<commit_before>package command

import (
	"github.com/chrislusf/seaweedfs/weed/filer2"
	"github.com/chrislusf/seaweedfs/weed/glog"
	"github.com/chrislusf/seaweedfs/weed/notification"
	"github.com/chrislusf/seaweedfs/weed/pb/filer_pb"
	"github.com/chrislusf/seaweedfs/weed/server"
	"github.com/spf13/viper"
)

func init() {
	cmdFilerExport.Run = runFilerExport // break init cycle
}

var cmdFilerExport = &Command{
	UsageLine: "filer.export -sourceStore=mysql -targetStroe=cassandra",
	Short:     "export meta data in filer store",
	Long: `Iterate the file tree and export all metadata out

	Both source and target store:
        * should be a store name already specified in filer.toml
        * do not need to be enabled state

	If target store is empty, only the directory tree will be listed.

	If target store is "notification", the list of entries will be sent to notification.
	This is usually used to bootstrap filer replication to a remote system.

  `,
}

var (
	// filerExportOutputFile  = cmdFilerExport.Flag.String("output", "", "the output file. If empty, only list out the directory tree")
	filerExportSourceStore = cmdFilerExport.Flag.String("sourceStore", "", "the source store name in filer.toml, default to currently enabled store")
	filerExportTargetStore = cmdFilerExport.Flag.String("targetStore", "", "the target store name in filer.toml, or \"notification\" to export all files to message queue")
	dir                    = cmdFilerExport.Flag.String("dir", "/", "only process files under this directory")
	dirListLimit           = cmdFilerExport.Flag.Int("dirListLimit", 100000, "limit directory list size")
	dryRun                 = cmdFilerExport.Flag.Bool("dryRun", false, "not actually moving data")
)

type statistics struct {
	directoryCount int
	fileCount      int
}

func runFilerExport(cmd *Command, args []string) bool {

	weed_server.LoadConfiguration("filer", true)
	config := viper.GetViper()

	var sourceStore, targetStore filer2.FilerStore

	for _, store := range filer2.Stores {
		if store.GetName() == *filerExportSourceStore || *filerExportSourceStore == "" && config.GetBool(store.GetName()+".enabled") {
			viperSub := config.Sub(store.GetName())
			if err := store.Initialize(viperSub); err != nil {
				glog.Fatalf("Failed to initialize source store for %s: %+v",
					store.GetName(), err)
			} else {
				sourceStore = store
			}
			break
		}
	}

	for _, store := range filer2.Stores {
		if store.GetName() == *filerExportTargetStore {
			viperSub := config.Sub(store.GetName())
			if err := store.Initialize(viperSub); err != nil {
				glog.Fatalf("Failed to initialize target store for %s: %+v",
					store.GetName(), err)
			} else {
				targetStore = store
			}
			break
		}
	}

	if sourceStore == nil {
		glog.Errorf("Failed to find source store %s", *filerExportSourceStore)
		println("existing data sources are:")
		for _, store := range filer2.Stores {
			println("    " + store.GetName())
		}
		return false
	}

	if targetStore == nil && *filerExportTargetStore != "" && *filerExportTargetStore != "notification" {
		glog.Errorf("Failed to find target store %s", *filerExportTargetStore)
		println("existing data sources are:")
		for _, store := range filer2.Stores {
			println("    " + store.GetName())
		}
		return false
	}

	stat := statistics{}

	var fn func(level int, entry *filer2.Entry) error

	if *filerExportTargetStore == "notification" {
		weed_server.LoadConfiguration("notification", false)
		v := viper.GetViper()
		notification.LoadConfiguration(v.Sub("notification"))

		fn = func(level int, entry *filer2.Entry) error {
			printout(level, entry)
			if *dryRun {
				return nil
			}
			return notification.Queue.SendMessage(
				string(entry.FullPath),
				&filer_pb.EventNotification{
					NewEntry: entry.ToProtoEntry(),
				},
			)
		}
	} else if targetStore == nil {
		fn = printout
	} else {
		fn = func(level int, entry *filer2.Entry) error {
			printout(level, entry)
			if *dryRun {
				return nil
			}
			return targetStore.InsertEntry(entry)
		}
	}

	doTraverse(&stat, sourceStore, filer2.FullPath(*dir), 0, fn)

	glog.Infof("processed %d directories, %d files", stat.directoryCount, stat.fileCount)

	return true
}

func doTraverse(stat *statistics, filerStore filer2.FilerStore, parentPath filer2.FullPath, level int, fn func(level int, entry *filer2.Entry) error) {

	limit := *dirListLimit
	lastEntryName := ""
	for {
		entries, err := filerStore.ListDirectoryEntries(parentPath, lastEntryName, false, limit)
		if err != nil {
			break
		}
		for _, entry := range entries {
			if fnErr := fn(level, entry); fnErr != nil {
				glog.Errorf("failed to process entry: %s", entry.FullPath)
			}
			if entry.IsDirectory() {
				stat.directoryCount++
				doTraverse(stat, filerStore, entry.FullPath, level+1, fn)
			} else {
				stat.fileCount++
			}
		}
		if len(entries) < limit {
			break
		}
	}
}

func printout(level int, entry *filer2.Entry) error {
	for i := 0; i < level; i++ {
		if i == level-1 {
			print("+-")
		} else {
			print("| ")
		}
	}
	print(entry.FullPath.Name())
	for _, chunk:=range entry.Chunks{
		print("[")
		print(chunk.FileId)
		print(",")
		print(chunk.Offset)
		print(",")
		print(chunk.Size)
		print(")")
	}
	println()
	return nil
}
<commit_msg>options to control filer.export verbosity<commit_after>package command

import (
	"github.com/chrislusf/seaweedfs/weed/filer2"
	"github.com/chrislusf/seaweedfs/weed/glog"
	"github.com/chrislusf/seaweedfs/weed/notification"
	"github.com/chrislusf/seaweedfs/weed/pb/filer_pb"
	"github.com/chrislusf/seaweedfs/weed/server"
	"github.com/spf13/viper"
)

func init() {
	cmdFilerExport.Run = runFilerExport // break init cycle
}

var cmdFilerExport = &Command{
	UsageLine: "filer.export -sourceStore=mysql -targetStroe=cassandra",
	Short:     "export meta data in filer store",
	Long: `Iterate the file tree and export all metadata out

	Both source and target store:
        * should be a store name already specified in filer.toml
        * do not need to be enabled state

	If target store is empty, only the directory tree will be listed.

	If target store is "notification", the list of entries will be sent to notification.
	This is usually used to bootstrap filer replication to a remote system.

  `,
}

var (
	// filerExportOutputFile  = cmdFilerExport.Flag.String("output", "", "the output file. If empty, only list out the directory tree")
	filerExportSourceStore = cmdFilerExport.Flag.String("sourceStore", "", "the source store name in filer.toml, default to currently enabled store")
	filerExportTargetStore = cmdFilerExport.Flag.String("targetStore", "", "the target store name in filer.toml, or \"notification\" to export all files to message queue")
	dir                    = cmdFilerExport.Flag.String("dir", "/", "only process files under this directory")
	dirListLimit           = cmdFilerExport.Flag.Int("dirListLimit", 100000, "limit directory list size")
	dryRun                 = cmdFilerExport.Flag.Bool("dryRun", false, "not actually moving data")
	verboseFilerExport     = cmdFilerExport.Flag.Bool("v", false, "verbose entry details")
)

type statistics struct {
	directoryCount int
	fileCount      int
}

func runFilerExport(cmd *Command, args []string) bool {

	weed_server.LoadConfiguration("filer", true)
	config := viper.GetViper()

	var sourceStore, targetStore filer2.FilerStore

	for _, store := range filer2.Stores {
		if store.GetName() == *filerExportSourceStore || *filerExportSourceStore == "" && config.GetBool(store.GetName()+".enabled") {
			viperSub := config.Sub(store.GetName())
			if err := store.Initialize(viperSub); err != nil {
				glog.Fatalf("Failed to initialize source store for %s: %+v",
					store.GetName(), err)
			} else {
				sourceStore = store
			}
			break
		}
	}

	for _, store := range filer2.Stores {
		if store.GetName() == *filerExportTargetStore {
			viperSub := config.Sub(store.GetName())
			if err := store.Initialize(viperSub); err != nil {
				glog.Fatalf("Failed to initialize target store for %s: %+v",
					store.GetName(), err)
			} else {
				targetStore = store
			}
			break
		}
	}

	if sourceStore == nil {
		glog.Errorf("Failed to find source store %s", *filerExportSourceStore)
		println("existing data sources are:")
		for _, store := range filer2.Stores {
			println("    " + store.GetName())
		}
		return false
	}

	if targetStore == nil && *filerExportTargetStore != "" && *filerExportTargetStore != "notification" {
		glog.Errorf("Failed to find target store %s", *filerExportTargetStore)
		println("existing data sources are:")
		for _, store := range filer2.Stores {
			println("    " + store.GetName())
		}
		return false
	}

	stat := statistics{}

	var fn func(level int, entry *filer2.Entry) error

	if *filerExportTargetStore == "notification" {
		weed_server.LoadConfiguration("notification", false)
		v := viper.GetViper()
		notification.LoadConfiguration(v.Sub("notification"))

		fn = func(level int, entry *filer2.Entry) error {
			printout(level, entry)
			if *dryRun {
				return nil
			}
			return notification.Queue.SendMessage(
				string(entry.FullPath),
				&filer_pb.EventNotification{
					NewEntry: entry.ToProtoEntry(),
				},
			)
		}
	} else if targetStore == nil {
		fn = printout
	} else {
		fn = func(level int, entry *filer2.Entry) error {
			printout(level, entry)
			if *dryRun {
				return nil
			}
			return targetStore.InsertEntry(entry)
		}
	}

	doTraverse(&stat, sourceStore, filer2.FullPath(*dir), 0, fn)

	glog.Infof("processed %d directories, %d files", stat.directoryCount, stat.fileCount)

	return true
}

func doTraverse(stat *statistics, filerStore filer2.FilerStore, parentPath filer2.FullPath, level int, fn func(level int, entry *filer2.Entry) error) {

	limit := *dirListLimit
	lastEntryName := ""
	for {
		entries, err := filerStore.ListDirectoryEntries(parentPath, lastEntryName, false, limit)
		if err != nil {
			break
		}
		for _, entry := range entries {
			if fnErr := fn(level, entry); fnErr != nil {
				glog.Errorf("failed to process entry: %s", entry.FullPath)
			}
			if entry.IsDirectory() {
				stat.directoryCount++
				doTraverse(stat, filerStore, entry.FullPath, level+1, fn)
			} else {
				stat.fileCount++
			}
		}
		if len(entries) < limit {
			break
		}
	}
}

func printout(level int, entry *filer2.Entry) error {
	for i := 0; i < level; i++ {
		if i == level-1 {
			print("+-")
		} else {
			print("| ")
		}
	}
	print(entry.FullPath.Name())
	if *verboseFilerExport{
		for _, chunk := range entry.Chunks {
			print("[")
			print(chunk.FileId)
			print(",")
			print(chunk.Offset)
			print(",")
			print(chunk.Size)
			print(")")
		}
	}
	println()
	return nil
}
<|endoftext|>


<commit_before>package transport

import (
	"crypto/tls"
	"time"

	"golang.org/x/net/context"
)

type Options struct {
	Addrs     []string
	Secure    bool
	TLSConfig *tls.Config

	// Other options for implementations of the interface
	// can be stored in a context
	Context context.Context
}

type DialOptions struct {
	Stream  bool
	Timeout time.Duration

	// TODO: add tls options when dialling
	// Currently set in global options

	// Other options for implementations of the interface
	// can be stored in a context
	Context context.Context
}

type ListenOptions struct {
	// TODO: add tls options when listening
	// Currently set in global options

	// Other options for implementations of the interface
	// can be stored in a context
	Context context.Context
}

// Addrs to use for transport
func Addrs(addrs ...string) Option {
	return func(o *Options) {
		o.Addrs = addrs
	}
}

// Use secure communication. If TLSConfig is not specified we
// use InsecureSkipVerify and generate a self signed cert
func Secure(b bool) Option {
	return func(o *Options) {
		o.Secure = b
	}
}

// TLSConfig to be used for the transport.
func TLSConfig(t *tls.Config) Option {
	return func(o *Options) {
		o.TLSConfig = t
	}
}

// Indicates whether this is a streaming connection
func WithStream() DialOption {
	return func(o *DialOptions) {
		o.Stream = true
	}
}

// Timeout used when dialling the remote side
func WithTimeout(d time.Duration) DialOption {
	return func(o *DialOptions) {
		o.Timeout = d
	}
}
<commit_msg>Add deadline option<commit_after>package transport

import (
	"crypto/tls"
	"time"

	"golang.org/x/net/context"
)

type Options struct {
	Addrs     []string
	Secure    bool
	TLSConfig *tls.Config
	// Deadline sets the time to wait to Send/Recv
	Deadline time.Duration
	// Other options for implementations of the interface
	// can be stored in a context
	Context context.Context
}

type DialOptions struct {
	Stream  bool
	Timeout time.Duration

	// TODO: add tls options when dialling
	// Currently set in global options

	// Other options for implementations of the interface
	// can be stored in a context
	Context context.Context
}

type ListenOptions struct {
	// TODO: add tls options when listening
	// Currently set in global options

	// Other options for implementations of the interface
	// can be stored in a context
	Context context.Context
}

// Addrs to use for transport
func Addrs(addrs ...string) Option {
	return func(o *Options) {
		o.Addrs = addrs
	}
}

// Deadline sets the time to wait for Send/Recv execution
func Deadline(t time.Duration) Option {
	return func(o *Options) {
		o.Deadline = t
	}
}

// Use secure communication. If TLSConfig is not specified we
// use InsecureSkipVerify and generate a self signed cert
func Secure(b bool) Option {
	return func(o *Options) {
		o.Secure = b
	}
}

// TLSConfig to be used for the transport.
func TLSConfig(t *tls.Config) Option {
	return func(o *Options) {
		o.TLSConfig = t
	}
}

// Indicates whether this is a streaming connection
func WithStream() DialOption {
	return func(o *DialOptions) {
		o.Stream = true
	}
}

// Timeout used when dialling the remote side
func WithTimeout(d time.Duration) DialOption {
	return func(o *DialOptions) {
		o.Timeout = d
	}
}
<|endoftext|>


<commit_before>// go build -ldflags "-X main.dest linux -X main.version local"  main.go

package main

import (
	"configurator"
	"fmt"
	"io"
	"log"
	"optionparser"
	"os"
	"os/exec"
	"os/signal"
	"path/filepath"
	"runtime"
	"strconv"
	"strings"
	"syscall"
	"time"
)

var (
	options               map[string]string
	defaults              map[string]string
	layoutoptions         map[string]string
	variables             map[string]string
	installdir            string
	libdir                string
	srcdir                string
	path_to_documentation string // Where the documentation (index.html) is
	inifile               string
	dest                  string // The platform which this script runs on.
	version               string
	homecfg               string
	systemcfg             string
	pwd                   string
	exe_suffix            string
	extra_dir             []string
	starttime             time.Time
	cfg                   *configurator.ConfigData
)

func init() {
	var err error
	log.SetFlags(0)
	starttime = time.Now()
	go signalCatcher()
	pwd, err = os.Getwd()
	if err != nil {
		log.Fatal(err)
	}
	variables = make(map[string]string)
	layoutoptions = make(map[string]string)
	options = make(map[string]string)
	defaults = map[string]string{
		"layout":  "layout.xml",
		"jobname": "publisher",
		"data":    "data.xml",
		"runs":    "1",
	}

	// The problem now is that we don't know where the executable file is
	// if it's in the PATH, it has no ../ prefix
	// if it is relative, make an absolute path from it.

	executable_name := filepath.Base(os.Args[0])
	var bindir string
	if executable_name == os.Args[0] {
		// most likely an absolute path
		bindir, err = exec.LookPath(executable_name)
		if err != nil {
			log.Fatal(err)
		}
		bindir = filepath.Dir(bindir)
	} else {
		// a relative path hopefully
		bindir, err = filepath.Abs(filepath.Dir(os.Args[0]))
		if err != nil {
			log.Fatal(err)
		}
	}
	installdir = filepath.Join(bindir, "..")

	if version == "" {
		version = "local"
	}

	extra_dir = append(extra_dir, pwd)
	// log.Print("Built for platform: ",dest)
	switch os := runtime.GOOS; os {
	case "darwin":
		defaults["opencommand"] = "open"
		exe_suffix = ""
	case "linux":
		defaults["opencommand"] = "xdg-open"
		exe_suffix = ""
	case "windows":
		defaults["opencommand"] = "cmd /C start"
		exe_suffix = ".exe"
	}

	switch dest {
	case "linux-usr":
		libdir = "/usr/share/speedata-publisher/lib"
		srcdir = "/usr/share/speedata-publisher/sw"
		os.Setenv("PUBLISHER_BASE_PATH", "/usr/share/speedata-publisher")
		os.Setenv("LUA_PATH", fmt.Sprintf("%s/lua/?.lua;%s/lua/common/?.lua;", srcdir, srcdir))
		path_to_documentation = "/usr/share/doc/speedata-publisher/index.html"
	case "directory":
		libdir = filepath.Join(installdir, "lib")
		srcdir = filepath.Join(installdir, "sw")
		path_to_documentation = filepath.Join(installdir, "share/doc/index.html")
		os.Setenv("PUBLISHER_BASE_PATH",installdir)
		os.Setenv("LUA_PATH", srcdir+"/lua/?.lua;"+installdir+"/lib/?.lua;"+srcdir+"/lua/common/?.lua;")
	default:
		// local git installation
		libdir = filepath.Join(installdir, "lib")
		srcdir = filepath.Join(installdir, "src")
		os.Setenv("PUBLISHER_BASE_PATH", srcdir)
		os.Setenv("LUA_PATH", srcdir+"/lua/?.lua;"+installdir+"/lib/?.lua;"+srcdir+"/lua/common/?.lua;")
		extra_dir = append(extra_dir, filepath.Join(installdir, "fonts"))
		extra_dir = append(extra_dir, filepath.Join(installdir, "img"))
		path_to_documentation = filepath.Join(installdir, "/build/handbuch_publisher/index.html")
	}
	inifile = filepath.Join(srcdir, "lua/sdini.lua")
	// cfg, err = configurator.ReadFiles("/Users/patrick/.publisher.cfg", path.Join(pwd, "publisher.cfg"))
	cfg, err = configurator.ReadFiles(filepath.Join(pwd, "publisher.cfg"), "/Users/patrick/.publisher.cfg")
	if err != nil {
		log.Fatal(err)
	}
}

func getOption(optionname string) string {
	if options[optionname] != "" {
		return options[optionname]
	}
	if cfg.String("DEFAULT", optionname) != "" {
		return cfg.String("DEFAULT", optionname)
	}
	if defaults[optionname] != "" {
		return defaults[optionname]
	}
	return ""
}

// Open the given file with the system's default program
func openFile(filename string) {
	opencommand := getOption("opencommand")
	cmdname := strings.SplitN(opencommand+" "+filename, " ", 2)
	cmd := exec.Command(cmdname[0], cmdname[1])
	err := cmd.Start()
	if err != nil {
		log.Fatal(err)
	}
	err = cmd.Wait()
	if err != nil {
		log.Fatal(err)
	}
}

// Put string a=b into the variabls map
func setVariable(str string) {
	a := strings.Split(str, "=")
	variables[a[0]] = a[1]
}

func showDuration() {
	log.Printf("Total run time: %v\n", time.Now().Sub(starttime))
}

func signalCatcher() {
	ch := make(chan os.Signal)
	signal.Notify(ch, syscall.SIGINT)
	sig := <-ch
	log.Printf("Signal received: %v", sig)
	showDuration()
	os.Exit(0)
}

// Run the given command line
func run(cmdline string) {
	// Todo: don't split quoted commands
	cmdline_array := strings.Split(cmdline, " ")
	cmd := exec.Command(cmdline_array[0])
	cmd.Args = cmdline_array
	stdout, err := cmd.StdoutPipe()
	if err != nil {
		log.Fatal(err)
	}
	stderr, err := cmd.StderrPipe()
	if err != nil {
		log.Fatal(err)
	}
	err = cmd.Start()
	if err != nil {
		log.Fatal(err)
	}
	go io.Copy(os.Stdout, stdout)
	go io.Copy(os.Stderr, stderr)
	err = cmd.Wait()
	if err != nil {
		showDuration()
		log.Print(err)
	}
}

func save_variables() {
	jobname := getOption("jobname")
	f, err := os.Create(jobname + ".vars")
	if err != nil {
		log.Fatal(err)
	}
	fmt.Fprintln(f, "return { ")
	for key, value := range variables {
		fmt.Fprintf(f, `["%s"] = "%s", `+"\n", key, value)
	}
	fmt.Fprintln(f, "} ")
	f.Close()
}

// add the command line argument (extra-dir) into the slice
func extradir(arg string) {
	extra_dir = append(extra_dir,arg)
}

// We don't know where the executable is and we don't know
// if we should run sdluatex (with libxml2 parser) or simple
// luatex
func getExecutablePath() string {
	// 1 check the installdir/bin for sdluatex(.exe)
	// 2 check PATH for sdluatex(.exe)
	// 3 assume simple installation and take luatex(.exe)
	// 4 check then installdir/bin for luatex(.exe)
	// 5 check PATH for luatex(.exe)
	// 6 panic!
	executable_name := "sdluatex" + exe_suffix
	var p string

	// 1 check the installdir/bin for sdluatex(.exe)
	p = fmt.Sprintf("%s/bin/%s", installdir, executable_name)
	fi, _ := os.Stat(p)
	if fi != nil {
		return p
	}

	// 2 check PATH for sdluatex(.exe)
	p, _ = exec.LookPath(executable_name)
	if p != "" {
		return p
	}

	// 3 assume simple installation and take luatex(.exe)
	executable_name = "luatex" + exe_suffix

	// 4 check then installdir/bin for luatex(.exe)
	p = fmt.Sprintf("%s/bin/%s", installdir, executable_name)
	fi, _ = os.Stat(p)
	if fi != nil {
		return p
	}
	// 5 check PATH for luatex(.exe)
	p, _ = exec.LookPath(executable_name)
	if p != "" {
		return p
	}

	// 6 panic!
	log.Fatal("Can't find sdluatex or luatex binary")
	return ""
}

// Print version information
func versioninfo() {
	log.Println("Version: ",version)
	os.Exit(0)
}

func runPublisher() {
	log.Print("run speedata publisher")

	save_variables()

	f, err := os.Create(getOption("jobname") + ".protocol")
	if err != nil {
		log.Fatal(err)
	}
	fmt.Fprintf(f, "Protocol file for speedata Publisher (%s)\n", version)
	fmt.Fprintln(f, "Time:", starttime.Format(time.ANSIC))
	f.Close()

	// layoutoptions are passed as a command line argument to the publisher
	var layoutoptions_ary []string
	if layoutoptions["grid"] != "" {
		layoutoptions_ary = append(layoutoptions_ary, `showgrid="`+layoutoptions["grid"]+`"`)
	}
	if layoutoptions["startpage"] != "" {
		layoutoptions_ary = append(layoutoptions_ary, `startpage="`+layoutoptions["startpage"]+`"`)
	}
	if layoutoptions["trace"] != "" {
		layoutoptions_ary = append(layoutoptions_ary, `trace="`+layoutoptions["trace"]+`"`)
	}
	layoutoptions_cmdline := strings.Join(layoutoptions_ary, ",")
	jobname := getOption("jobname")
	layoutname := getOption("layout")
	dataname := getOption("data")
	exec_name := getExecutablePath()
	dummy_data := getOption("dummy")
	if dummy_data == "true" {
		dataname = "-dummy"
	}

	runs, err := strconv.Atoi(getOption("runs"))
	if err != nil {
		log.Fatal(err)
	}
	for i := 1; i <= runs; i++ {
		cmdline := fmt.Sprintf("%s --interaction nonstopmode --jobname=%s --ini --lua=%s publisher.tex %s %s %s", exec_name, jobname, inifile, layoutname, dataname, layoutoptions_cmdline)
		run(cmdline)
	}
}

func main() {
	op := optionparser.NewOptionParser()
	op.On("--autoopen", "Open the PDF file (MacOS X and Linux only)", options)
	op.On("--data NAME", "Name of the XML data file. Defaults to 'data.xml'", options)
	op.On("--dummy","Don't read a data file, use '<data />' as input",options)
	op.On("--filter FILTER", "Run XPROC filter before publishing starts", options)
	op.On("--grid", "Display background grid. Disable with --no-grid", layoutoptions)
	op.On("--layout NAME", "Name of the layout file. Defaults to 'layout.xml'", options)
	op.On("--jobname NAME", "The name of the resulting PDF file, default is 'publisher.pdf'", options)
	op.On("--runs NUM", "Number of publishing runs ", options)
	op.On("--startpage NUM", "The first page number", layoutoptions)
	op.On("--trace","Show debug messages and some tracing PDF output",layoutoptions)
	op.On("-v", "--var VAR=VALUE", "Set a variable for the publishing run", setVariable)
	op.On("--version", "Show version information", versioninfo)
	op.On("-x", "--extra-dir DIR", "Additional directory for file search", extradir)
	op.On("--xml", "Output as (pseudo-)XML (for list-fonts)", options)

	op.Command("list-fonts", "List installed fonts (use together with --xml for copy/paste)")
	op.Command("doc", "Open documentation")
	op.Command("watch", "Start watchdog / hotfolder")
	op.Command("run", "Start publishing (default)")
	err := op.Parse()
	if err != nil {
		log.Fatal("Parse error: ", err)
	}

	var command string
	switch len(op.Extra) {
	case 0:
		// no command given, run is the default command
		command = "run"
	case 1:
		// great
		command = op.Extra[0]
	default:
		// more than one command given, what should I do?
		command = op.Extra[0]
	}

	os.Setenv("SD_EXTRA_DIRS", strings.Join(extra_dir, ":"))

	switch command {
	case "run":
		if filter := getOption("filter"); filter != "" {
			if filepath.Ext(filter) != ".xpl" {
				filter = filter + ".xpl"
			}
			log.Println("Run filter: ", filter)
			os.Setenv("CLASSPATH", libdir+"/calabash.jar:"+libdir+"/saxon9he.jar")
			cmdline := "java com.xmlcalabash.drivers.Main " + filter
			run(cmdline)
		}
		runPublisher()
		// open PDF if necessary
		if getOption("autoopen") == "true" {
			openFile(getOption("jobname") + ".pdf")
		}
	case "doc":
		openFile(path_to_documentation)
		os.Exit(0)
	case "list-fonts":
		var xml string
		if getOption("xml") == "true" {
			xml = "xml"
		}
		cmdline := fmt.Sprintf("%s/bin/sdluatex --luaonly %s/lua/sdscripts.lua %s list-fonts %s", installdir, srcdir, inifile, xml)
		run(cmdline)
	case "watch":
		log.Fatal("not implemented yet.")
	default:
		log.Fatal("unknown command:", command)
	}
	showDuration()
}
<commit_msg>sp: kill publishing process if SIGINT comes in<commit_after>// go build -ldflags "-X main.dest linux -X main.version local"  main.go

package main

import (
	"configurator"
	"fmt"
	"io"
	"log"
	"optionparser"
	"os"
	"os/exec"
	"os/signal"
	"path/filepath"
	"runtime"
	"strconv"
	"strings"
	"syscall"
	"time"
)

var (
	options               map[string]string
	defaults              map[string]string
	layoutoptions         map[string]string
	variables             map[string]string
	installdir            string
	libdir                string
	srcdir                string
	path_to_documentation string // Where the documentation (index.html) is
	inifile               string
	dest                  string // The platform which this script runs on.
	version               string
	homecfg               string
	systemcfg             string
	pwd                   string
	exe_suffix            string
	extra_dir             []string
	starttime             time.Time
	cfg                   *configurator.ConfigData
	running_processes     []*os.Process
)

func init() {
	var err error
	log.SetFlags(0)
	starttime = time.Now()
	go signalCatcher()
	pwd, err = os.Getwd()
	if err != nil {
		log.Fatal(err)
	}
	variables = make(map[string]string)
	layoutoptions = make(map[string]string)
	options = make(map[string]string)
	defaults = map[string]string{
		"layout":  "layout.xml",
		"jobname": "publisher",
		"data":    "data.xml",
		"runs":    "1",
	}

	// The problem now is that we don't know where the executable file is
	// if it's in the PATH, it has no ../ prefix
	// if it is relative, make an absolute path from it.

	executable_name := filepath.Base(os.Args[0])
	var bindir string
	if executable_name == os.Args[0] {
		// most likely an absolute path
		bindir, err = exec.LookPath(executable_name)
		if err != nil {
			log.Fatal(err)
		}
		bindir = filepath.Dir(bindir)
	} else {
		// a relative path hopefully
		bindir, err = filepath.Abs(filepath.Dir(os.Args[0]))
		if err != nil {
			log.Fatal(err)
		}
	}
	installdir = filepath.Join(bindir, "..")

	if version == "" {
		version = "local"
	}

	extra_dir = append(extra_dir, pwd)
	// log.Print("Built for platform: ",dest)
	switch os := runtime.GOOS; os {
	case "darwin":
		defaults["opencommand"] = "open"
		exe_suffix = ""
	case "linux":
		defaults["opencommand"] = "xdg-open"
		exe_suffix = ""
	case "windows":
		defaults["opencommand"] = "cmd /C start"
		exe_suffix = ".exe"
	}

	switch dest {
	case "linux-usr":
		libdir = "/usr/share/speedata-publisher/lib"
		srcdir = "/usr/share/speedata-publisher/sw"
		os.Setenv("PUBLISHER_BASE_PATH", "/usr/share/speedata-publisher")
		os.Setenv("LUA_PATH", fmt.Sprintf("%s/lua/?.lua;%s/lua/common/?.lua;", srcdir, srcdir))
		path_to_documentation = "/usr/share/doc/speedata-publisher/index.html"
	case "directory":
		libdir = filepath.Join(installdir, "lib")
		srcdir = filepath.Join(installdir, "sw")
		path_to_documentation = filepath.Join(installdir, "share/doc/index.html")
		os.Setenv("PUBLISHER_BASE_PATH", installdir)
		os.Setenv("LUA_PATH", srcdir+"/lua/?.lua;"+installdir+"/lib/?.lua;"+srcdir+"/lua/common/?.lua;")
	default:
		// local git installation
		libdir = filepath.Join(installdir, "lib")
		srcdir = filepath.Join(installdir, "src")
		os.Setenv("PUBLISHER_BASE_PATH", srcdir)
		os.Setenv("LUA_PATH", srcdir+"/lua/?.lua;"+installdir+"/lib/?.lua;"+srcdir+"/lua/common/?.lua;")
		extra_dir = append(extra_dir, filepath.Join(installdir, "fonts"))
		extra_dir = append(extra_dir, filepath.Join(installdir, "img"))
		path_to_documentation = filepath.Join(installdir, "/build/handbuch_publisher/index.html")
	}
	inifile = filepath.Join(srcdir, "lua/sdini.lua")
	// cfg, err = configurator.ReadFiles("/Users/patrick/.publisher.cfg", path.Join(pwd, "publisher.cfg"))
	cfg, err = configurator.ReadFiles(filepath.Join(pwd, "publisher.cfg"), "/Users/patrick/.publisher.cfg")
	if err != nil {
		log.Fatal(err)
	}
}

func getOption(optionname string) string {
	if options[optionname] != "" {
		return options[optionname]
	}
	if cfg.String("DEFAULT", optionname) != "" {
		return cfg.String("DEFAULT", optionname)
	}
	if defaults[optionname] != "" {
		return defaults[optionname]
	}
	return ""
}

// Open the given file with the system's default program
func openFile(filename string) {
	opencommand := getOption("opencommand")
	cmdname := strings.SplitN(opencommand+" "+filename, " ", 2)
	cmd := exec.Command(cmdname[0], cmdname[1])
	err := cmd.Start()
	if err != nil {
		log.Fatal(err)
	}
	err = cmd.Wait()
	if err != nil {
		log.Fatal(err)
	}
}

// Put string a=b into the variabls map
func setVariable(str string) {
	a := strings.Split(str, "=")
	variables[a[0]] = a[1]
}

func showDuration() {
	log.Printf("Total run time: %v\n", time.Now().Sub(starttime))
}

func signalCatcher() {
	ch := make(chan os.Signal)
	signal.Notify(ch, syscall.SIGINT)
	sig := <-ch
	log.Printf("Signal received: %v", sig)
	for _, proc := range running_processes {
		err := proc.Kill()
		if err != nil {
			log.Fatal(err)
		}
	}
	showDuration()
	os.Exit(0)
}

// Run the given command line
func run(cmdline string) {
	// Todo: don't split quoted commands
	cmdline_array := strings.Split(cmdline, " ")
	cmd := exec.Command(cmdline_array[0])
	cmd.Args = cmdline_array
	stdout, err := cmd.StdoutPipe()
	if err != nil {
		log.Fatal(err)
	}
	stderr, err := cmd.StderrPipe()
	if err != nil {
		log.Fatal(err)
	}
	err = cmd.Start()
	if err != nil {
		log.Fatal(err)
	}
	running_processes = append(running_processes, cmd.Process)
	go io.Copy(os.Stdout, stdout)
	go io.Copy(os.Stderr, stderr)
	err = cmd.Wait()
	if err != nil {
		showDuration()
		log.Print(err)
	}
}

func save_variables() {
	jobname := getOption("jobname")
	f, err := os.Create(jobname + ".vars")
	if err != nil {
		log.Fatal(err)
	}
	fmt.Fprintln(f, "return { ")
	for key, value := range variables {
		fmt.Fprintf(f, `["%s"] = "%s", `+"\n", key, value)
	}
	fmt.Fprintln(f, "} ")
	f.Close()
}

// add the command line argument (extra-dir) into the slice
func extradir(arg string) {
	extra_dir = append(extra_dir, arg)
}

// We don't know where the executable is and we don't know
// if we should run sdluatex (with libxml2 parser) or simple
// luatex
func getExecutablePath() string {
	// 1 check the installdir/bin for sdluatex(.exe)
	// 2 check PATH for sdluatex(.exe)
	// 3 assume simple installation and take luatex(.exe)
	// 4 check then installdir/bin for luatex(.exe)
	// 5 check PATH for luatex(.exe)
	// 6 panic!
	executable_name := "sdluatex" + exe_suffix
	var p string

	// 1 check the installdir/bin for sdluatex(.exe)
	p = fmt.Sprintf("%s/bin/%s", installdir, executable_name)
	fi, _ := os.Stat(p)
	if fi != nil {
		return p
	}

	// 2 check PATH for sdluatex(.exe)
	p, _ = exec.LookPath(executable_name)
	if p != "" {
		return p
	}

	// 3 assume simple installation and take luatex(.exe)
	executable_name = "luatex" + exe_suffix

	// 4 check then installdir/bin for luatex(.exe)
	p = fmt.Sprintf("%s/bin/%s", installdir, executable_name)
	fi, _ = os.Stat(p)
	if fi != nil {
		return p
	}
	// 5 check PATH for luatex(.exe)
	p, _ = exec.LookPath(executable_name)
	if p != "" {
		return p
	}

	// 6 panic!
	log.Fatal("Can't find sdluatex or luatex binary")
	return ""
}

// Print version information
func versioninfo() {
	log.Println("Version: ", version)
	os.Exit(0)
}

func runPublisher() {
	log.Print("run speedata publisher")

	save_variables()

	f, err := os.Create(getOption("jobname") + ".protocol")
	if err != nil {
		log.Fatal(err)
	}
	fmt.Fprintf(f, "Protocol file for speedata Publisher (%s)\n", version)
	fmt.Fprintln(f, "Time:", starttime.Format(time.ANSIC))
	f.Close()

	// layoutoptions are passed as a command line argument to the publisher
	var layoutoptions_ary []string
	if layoutoptions["grid"] != "" {
		layoutoptions_ary = append(layoutoptions_ary, `showgrid="`+layoutoptions["grid"]+`"`)
	}
	if layoutoptions["startpage"] != "" {
		layoutoptions_ary = append(layoutoptions_ary, `startpage="`+layoutoptions["startpage"]+`"`)
	}
	if layoutoptions["trace"] != "" {
		layoutoptions_ary = append(layoutoptions_ary, `trace="`+layoutoptions["trace"]+`"`)
	}
	layoutoptions_cmdline := strings.Join(layoutoptions_ary, ",")
	jobname := getOption("jobname")
	layoutname := getOption("layout")
	dataname := getOption("data")
	exec_name := getExecutablePath()
	dummy_data := getOption("dummy")
	if dummy_data == "true" {
		dataname = "-dummy"
	}

	runs, err := strconv.Atoi(getOption("runs"))
	if err != nil {
		log.Fatal(err)
	}
	for i := 1; i <= runs; i++ {
		cmdline := fmt.Sprintf("%s --interaction nonstopmode --jobname=%s --ini --lua=%s publisher.tex %s %s %s", exec_name, jobname, inifile, layoutname, dataname, layoutoptions_cmdline)
		run(cmdline)
	}
}

func main() {
	op := optionparser.NewOptionParser()
	op.On("--autoopen", "Open the PDF file (MacOS X and Linux only)", options)
	op.On("--data NAME", "Name of the XML data file. Defaults to 'data.xml'", options)
	op.On("--dummy", "Don't read a data file, use '<data />' as input", options)
	op.On("--filter FILTER", "Run XPROC filter before publishing starts", options)
	op.On("--grid", "Display background grid. Disable with --no-grid", layoutoptions)
	op.On("--layout NAME", "Name of the layout file. Defaults to 'layout.xml'", options)
	op.On("--jobname NAME", "The name of the resulting PDF file, default is 'publisher.pdf'", options)
	op.On("--runs NUM", "Number of publishing runs ", options)
	op.On("--startpage NUM", "The first page number", layoutoptions)
	op.On("--trace", "Show debug messages and some tracing PDF output", layoutoptions)
	op.On("-v", "--var VAR=VALUE", "Set a variable for the publishing run", setVariable)
	op.On("--version", "Show version information", versioninfo)
	op.On("-x", "--extra-dir DIR", "Additional directory for file search", extradir)
	op.On("--xml", "Output as (pseudo-)XML (for list-fonts)", options)

	op.Command("list-fonts", "List installed fonts (use together with --xml for copy/paste)")
	op.Command("doc", "Open documentation")
	op.Command("watch", "Start watchdog / hotfolder")
	op.Command("run", "Start publishing (default)")
	err := op.Parse()
	if err != nil {
		log.Fatal("Parse error: ", err)
	}

	var command string
	switch len(op.Extra) {
	case 0:
		// no command given, run is the default command
		command = "run"
	case 1:
		// great
		command = op.Extra[0]
	default:
		// more than one command given, what should I do?
		command = op.Extra[0]
	}

	os.Setenv("SD_EXTRA_DIRS", strings.Join(extra_dir, ":"))

	switch command {
	case "run":
		if filter := getOption("filter"); filter != "" {
			if filepath.Ext(filter) != ".xpl" {
				filter = filter + ".xpl"
			}
			log.Println("Run filter: ", filter)
			os.Setenv("CLASSPATH", libdir+"/calabash.jar:"+libdir+"/saxon9he.jar")
			cmdline := "java com.xmlcalabash.drivers.Main " + filter
			run(cmdline)
		}
		runPublisher()
		// open PDF if necessary
		if getOption("autoopen") == "true" {
			openFile(getOption("jobname") + ".pdf")
		}
	case "doc":
		openFile(path_to_documentation)
		os.Exit(0)
	case "list-fonts":
		var xml string
		if getOption("xml") == "true" {
			xml = "xml"
		}
		cmdline := fmt.Sprintf("%s/bin/sdluatex --luaonly %s/lua/sdscripts.lua %s list-fonts %s", installdir, srcdir, inifile, xml)
		run(cmdline)
	case "watch":
		log.Fatal("not implemented yet.")
	default:
		log.Fatal("unknown command:", command)
	}
	showDuration()
}
<|endoftext|>


<commit_before>// Copyright 2016 Lars Wiegman. All rights reserved. Use of this source code is
// governed by a BSD-style license that can be found in the LICENSE file.

package multipass

import "html/template"

const (
	loginPage = iota
	tokenInvalidPage
	tokenSentPage
	continueOrSignoutPage
)

type page struct {
	PJAX        bool
	Page        int
	NextURL     string
	LoginPath   string
	SignoutPath string
}

func loadTemplates() (*template.Template, error) {
	tmpl := template.New("")
	template.Must(tmpl.New("head").Parse(`<!DOCTYPE html>

	<meta charset="utf-8">
	<meta name="description" content="Multipass">
	<title>Multipass</title>

`))

	template.Must(tmpl.New("tail").Parse(`</body></html>`))

	template.Must(tmpl.New("loginform").Parse(`
<form action="{{ .LoginPath }}" method=POST class="login-form">
	<input type=hidden name=url value="{{ .NextURL }}" />
	<input type=text name=handle placeholder="Your handle ..." />
	<input type=submit value="Submit" class="btn btn-default">
</form>`))

	template.Must(tmpl.New("signoutform").Parse(`
<form action="{{ .SignoutPath }}" method=POST class="login-form">
	<input type=hidden name=url value="{{ .NextURL }}" />
	<input type=submit value="Signout" class="btn btn-danger">
</form>`))

	template.Must(tmpl.New("page").Parse(`
	{{ if ne .PJAX true }}{{ template "head"}}{{end}}

Multipass

		{{ if eq .Page 0 }}
			{{ template "loginform" . }}

This resource is protected. Submit your handle to gain access.

		{{ else if eq .Page 1 }}

Your access token has expired or is invalid. Submit your handle to request a one.

			{{ template "loginform" . }}

This resource is protected.

		{{ else if eq .Page 2 }}

A message with an access token was sent to your handle; Follow the login link in the message to gain access.

		{{ else if eq .Page 3 }}

Continue to access the private resource or signout.

			Continue
			{{ template "signoutform" . }}

This resource is protected.

		{{ else }}

Default page

		{{ end }}

{{ if ne .PJAX true }}{{ template "tail" }}{{ end }}`))

	template.Must(tmpl.New("style").Parse(`
* {
	-webkit-box-sizing: border-box;
	-moz-box-sizing: border-box;
	box-sizing: border-box;
}
article, aside, details, figcaption, figure, footer, header, hgroup,
main, nav, section, summary {
	display: block;
}
body {
	margin: 0;
}
.wrapper {
	display: -webkit-flex;
	-webkit-flex-direction: column;
}
.login-box {
	width: 480px;
	margin: 4rem auto;
	text-align: center;
	padding: 1rem 0;
	border: solid .2rem #0d8eba;
	border-radius: .8rem;
	padding: 1rem 2rem;
	background-color: #fff;
}
.login-box h1 {
	font-family: sans-serif;
	font-size: 3rem;
	color: #444;
}
.login-form {
}
.login-form input {
}
.login-form input[type=text] {
	font-size: 1.2rem;
	padding: 0 1rem;
	border-color: #ddd;
	color: #444;
	margin: .6rem 0;
	width: 100%;
	height: 2.8rem;
	border-radius: .4rem;
	border-style: solid;
	border-width: .2rem;
}
.login-form input[type=submit] {
}
.btn {
	font-family: sans-serif;
	font-size: 1.2rem;
	margin: 1rem 0;
	padding: .5rem 1rem;
	width: 100%;
	height: 2.8rem;
	border-radius: .4rem;
	border-style: solid;
	border-width: .16rem;
	text-transform: uppercase;
	color: #fff;
	height: 3rem;
	cursor: pointer;
	display: inline-block;
	line-height: 2rem;
}
a.btn {
	text-decoration: none;
	color: white;
}
.btn-default {
	background-color: #0d8eba;
	border-color: #0d8eba;
}
.btn-success {
	background-color: #0dba77;
	border-color: #0dba77;
}
.btn-danger {
	background-color: #ba2f0d;
	border-color: #ba2f0d;
}
.notice {
	font-style: italic;
	color: #666;
}
/* Narrow */
@media only screen and (max-width: 480px) {
	body {
		background-color: #0d8eba;
	}
	.login-box {
		width: 100%;
		border: none;
		border-radius: 0;
	}
}
/* Medium */
@media only screen and (min-width: 481px) and (max-width: 960px) {
}
/* Wide */
@media only screen and (min-width: 961px) {
}
`))
	return tmpl, nil
}
<commit_msg>Fix little typo<commit_after>// Copyright 2016 Lars Wiegman. All rights reserved. Use of this source code is
// governed by a BSD-style license that can be found in the LICENSE file.

package multipass

import "html/template"

const (
	loginPage = iota
	tokenInvalidPage
	tokenSentPage
	continueOrSignoutPage
)

type page struct {
	PJAX        bool
	Page        int
	NextURL     string
	LoginPath   string
	SignoutPath string
}

func loadTemplates() (*template.Template, error) {
	tmpl := template.New("")
	template.Must(tmpl.New("head").Parse(`<!DOCTYPE html>

	<meta charset="utf-8">
	<meta name="description" content="Multipass">
	<title>Multipass</title>

`))

	template.Must(tmpl.New("tail").Parse(`</body></html>`))

	template.Must(tmpl.New("loginform").Parse(`
<form action="{{ .LoginPath }}" method=POST class="login-form">
	<input type=hidden name=url value="{{ .NextURL }}" />
	<input type=text name=handle placeholder="Your handle ..." />
	<input type=submit value="Submit" class="btn btn-default">
</form>`))

	template.Must(tmpl.New("signoutform").Parse(`
<form action="{{ .SignoutPath }}" method=POST class="login-form">
	<input type=hidden name=url value="{{ .NextURL }}" />
	<input type=submit value="Signout" class="btn btn-danger">
</form>`))

	template.Must(tmpl.New("page").Parse(`
	{{ if ne .PJAX true }}{{ template "head"}}{{end}}

Multipass

		{{ if eq .Page 0 }}
			{{ template "loginform" . }}

This resource is protected. Submit your handle to gain access.

		{{ else if eq .Page 1 }}

Your access token has expired or is invalid. Submit your handle to request a new one.

			{{ template "loginform" . }}

This resource is protected.

		{{ else if eq .Page 2 }}

A message with an access token was sent to your handle; Follow the login link in the message to gain access.

		{{ else if eq .Page 3 }}

Continue to access the private resource or signout.

			Continue
			{{ template "signoutform" . }}

This resource is protected.

		{{ else }}

Default page

		{{ end }}

{{ if ne .PJAX true }}{{ template "tail" }}{{ end }}`))

	template.Must(tmpl.New("style").Parse(`
* {
	-webkit-box-sizing: border-box;
	-moz-box-sizing: border-box;
	box-sizing: border-box;
}
article, aside, details, figcaption, figure, footer, header, hgroup,
main, nav, section, summary {
	display: block;
}
body {
	margin: 0;
}
.wrapper {
	display: -webkit-flex;
	-webkit-flex-direction: column;
}
.login-box {
	width: 480px;
	margin: 4rem auto;
	text-align: center;
	padding: 1rem 0;
	border: solid .2rem #0d8eba;
	border-radius: .8rem;
	padding: 1rem 2rem;
	background-color: #fff;
}
.login-box h1 {
	font-family: sans-serif;
	font-size: 3rem;
	color: #444;
}
.login-form {
}
.login-form input {
}
.login-form input[type=text] {
	font-size: 1.2rem;
	padding: 0 1rem;
	border-color: #ddd;
	color: #444;
	margin: .6rem 0;
	width: 100%;
	height: 2.8rem;
	border-radius: .4rem;
	border-style: solid;
	border-width: .2rem;
}
.login-form input[type=submit] {
}
.btn {
	font-family: sans-serif;
	font-size: 1.2rem;
	margin: 1rem 0;
	padding: .5rem 1rem;
	width: 100%;
	height: 2.8rem;
	border-radius: .4rem;
	border-style: solid;
	border-width: .16rem;
	text-transform: uppercase;
	color: #fff;
	height: 3rem;
	cursor: pointer;
	display: inline-block;
	line-height: 2rem;
}
a.btn {
	text-decoration: none;
	color: white;
}
.btn-default {
	background-color: #0d8eba;
	border-color: #0d8eba;
}
.btn-success {
	background-color: #0dba77;
	border-color: #0dba77;
}
.btn-danger {
	background-color: #ba2f0d;
	border-color: #ba2f0d;
}
.notice {
	font-style: italic;
	color: #666;
}
/* Narrow */
@media only screen and (max-width: 480px) {
	body {
		background-color: #0d8eba;
	}
	.login-box {
		width: 100%;
		border: none;
		border-radius: 0;
	}
}
/* Medium */
@media only screen and (min-width: 481px) and (max-width: 960px) {
}
/* Wide */
@media only screen and (min-width: 961px) {
}
`))
	return tmpl, nil
}
<|endoftext|>


<commit_before>package main

import (
	"fmt"
	"os"
	//"path"
	//"runtime"

	"github.com/lazywei/go-opencv/opencv"
	//"../opencv" // can be used in forks, comment in real application
)

func main() {
	win := opencv.NewWindow("Go-OpenCV Webcam")
	defer win.Destroy()

	cap := opencv.NewCameraCapture(0)
	if cap == nil {
		panic("can not open camera")
	}
	defer cap.Release()

	win.CreateTrackbar("Thresh", 1, 100, func(pos int, param ...interface{}) {
		for {
			if cap.GrabFrame() {
				img := cap.RetrieveFrame(1)
				if img != nil {
					ProcessImage(img, win, pos)
				} else {
					fmt.Println("Image ins nil")
				}
			}

			if key := opencv.WaitKey(10); key == 27 {
				os.Exit(0)
			}
		}
	})
	opencv.WaitKey(0)
}

func ProcessImage(img *opencv.IplImage, win *opencv.Window, pos int) error {
	w := img.Width()
	h := img.Height()

	// Create the output image
	cedge := opencv.CreateImage(w, h, opencv.IPL_DEPTH_8U, 3)
	defer cedge.Release()

	// Convert to grayscale
	gray := opencv.CreateImage(w, h, opencv.IPL_DEPTH_8U, 1)
	edge := opencv.CreateImage(w, h, opencv.IPL_DEPTH_8U, 1)
	defer gray.Release()
	defer edge.Release()

	opencv.CvtColor(img, gray, opencv.CV_BGR2GRAY)

	opencv.Smooth(gray, edge, opencv.CV_BLUR, 3, 3, 0, 0)
	opencv.Not(gray, edge)

	// Run the edge detector on grayscale
	opencv.Canny(gray, edge, float64(pos), float64(pos*3), 3)

	opencv.Zero(cedge)
	// copy edge points
	opencv.Copy(img, cedge, edge)

	win.ShowImage(cedge)
	return nil
}
<commit_msg>Run processing in main thread instead of widget.<commit_after>package main

import (
	"fmt"
	"os"
	//"path"
	//"runtime"

	"github.com/lazywei/go-opencv/opencv"
	//"../opencv" // can be used in forks, comment in real application
)

func main() {
	var edge_threshold int

	win := opencv.NewWindow("Go-OpenCV Webcam")
	defer win.Destroy()

	cap := opencv.NewCameraCapture(0)
	if cap == nil {
		panic("can not open camera")
	}
	defer cap.Release()

	win.CreateTrackbar("Thresh", 1, 100, func(pos int, param ...interface{}) {
		edge_threshold = pos
	})

	fmt.Println("Press ESC to quit")
	for {
		if cap.GrabFrame() {
			img := cap.RetrieveFrame(1)
			if img != nil {
				ProcessImage(img, win, edge_threshold)
			} else {
				fmt.Println("Image ins nil")
			}
		}
		key := opencv.WaitKey(10)

		if key == 27 {
			os.Exit(0)
		}
	}
}

func ProcessImage(img *opencv.IplImage, win *opencv.Window, pos int) error {
	w := img.Width()
	h := img.Height()

	// Create the output image
	cedge := opencv.CreateImage(w, h, opencv.IPL_DEPTH_8U, 3)
	defer cedge.Release()

	// Convert to grayscale
	gray := opencv.CreateImage(w, h, opencv.IPL_DEPTH_8U, 1)
	edge := opencv.CreateImage(w, h, opencv.IPL_DEPTH_8U, 1)
	defer gray.Release()
	defer edge.Release()

	opencv.CvtColor(img, gray, opencv.CV_BGR2GRAY)

	opencv.Smooth(gray, edge, opencv.CV_BLUR, 3, 3, 0, 0)
	opencv.Not(gray, edge)

	// Run the edge detector on grayscale
	opencv.Canny(gray, edge, float64(pos), float64(pos*3), 3)

	opencv.Zero(cedge)
	// copy edge points
	opencv.Copy(img, cedge, edge)

	win.ShowImage(cedge)
	return nil
}
<|endoftext|>


<commit_before>/*
 * Copyright 2017 the original author or authors.
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 *      http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */

package v1

import (
	meta_v1 "k8s.io/apimachinery/pkg/apis/meta/v1"
)

// Represents the functions.extensions.sk8s.io CRD
type Function struct {
	meta_v1.TypeMeta   `json:",inline"`
	meta_v1.ObjectMeta `json:"metadata"`
	Spec               FunctionSpec   `json:"spec"`
	Status             FunctionStatus `json:"status,omitempty"`
}

// Spec (what the user wants) for a function
type FunctionSpec struct {

	// +optional
	Input string `json:"input,omitempty"`

	// +optional
	Ouput string `json:"output,omitempty"`

	// +optional
	Params []FunctionParam `json:"params,omitempty"`

	HandlerRef HandlerReference `json:"handlerRef"`

	// +optional
	Env []FunctionEnvVar `json:"env,omitempty"`
}

type FunctionParam struct {
	Name string `json:"name"`

	Value string `json:"value"` // TODO could be other than string?
}

type FunctionEnvVar struct {
	Name      string `json:"name"`
	ValueFrom string `json:"valueFrom"`
}

// An object reference to a Handler
// sub-set of https://kubernetes.io/docs/api-reference/v1.7/#objectreference-v1-core
// TODO: Add apiVersion, etc?
type HandlerReference struct {
	// Namespace of the referent.
	// More info: http://kubernetes.io/docs/user-guide/namespaces
	// +optional
	Namespace string `json:"namespace,omitempty" protobuf:"bytes,2,opt,name=namespace"`
	// Name of the referent.
	// More info: http://kubernetes.io/docs/user-guide/identifiers#names
	// +optional
	Name string `json:"name,omitempty" protobuf:"bytes,3,opt,name=name"`
}

// Status (computed) for a function
type FunctionStatus struct {
}

// Returned in list operations
type FunctionList struct {
	meta_v1.TypeMeta `json:",inline"`
	meta_v1.ListMeta `json:"metadata"`
	Items            []Function `json:"items"`
}
<commit_msg>refactoring Function<commit_after>/*
 * Copyright 2017 the original author or authors.
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 *      http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */

package v1

import (
	meta_v1 "k8s.io/apimachinery/pkg/apis/meta/v1"
)

// Represents the functions.extensions.sk8s.io CRD
type Function struct {
	meta_v1.TypeMeta   `json:",inline"`
	meta_v1.ObjectMeta `json:"metadata"`
	Spec               FunctionSpec   `json:"spec"`
	Status             FunctionStatus `json:"status,omitempty"`
}

// Spec (what the user wants) for a function
type FunctionSpec struct {

	Image string `json:"image"`

	// +optional
	Command []string `json:"command,omitempty"`

	// +optional
	Args []string `json:"args,omitempty"`

	Protocol string `json:"protocol"`

	// +optional
	Input string `json:"input,omitempty"`

	// +optional
	Output string `json:"output,omitempty"`

	// +optional
	Env []FunctionEnvVar `json:"env,omitempty"`
}

type FunctionEnvVar struct {
	Name      string `json:"name"`
	Value     string `json:"value"`
}

// Status (computed) for a function
type FunctionStatus struct {
}

// Returned in list operations
type FunctionList struct {
	meta_v1.TypeMeta `json:",inline"`
	meta_v1.ListMeta `json:"metadata"`
	Items            []Function `json:"items"`
}
<|endoftext|>


<commit_before>// Sync files and directories to and from swift
//
// Nick Craig-Wood <nick@craig-wood.com>
package main

import (
	"crypto/md5"
	"flag"
	"fmt"
	"github.com/ncw/swift"
	"io"
	"log"
	"os"
	"path/filepath"
	"runtime"
	"runtime/pprof"
	"strings"
	"sync"
)

// Globals
var (
	// Flags
	//fileSize      = flag.Int64("s", 1E9, "Size of the check files")
	cpuprofile = flag.String("cpuprofile", "", "Write cpu profile to file")
	//duration      = flag.Duration("duration", time.Hour*24, "Duration to run test")
	//statsInterval = flag.Duration("stats", time.Minute*1, "Interval to print stats")
	//logfile       = flag.String("logfile", "stressdisk.log", "File to write log to set to empty to ignore")

	snet    = flag.Bool("snet", false, "Use internal service network") // FIXME not implemented
	verbose = flag.Bool("verbose", false, "Print lots more stuff")
	quiet   = flag.Bool("quiet", false, "Print as little stuff as possible")
	// FIXME make these part of swift so we get a standard set of flags?
	authUrl   = flag.String("auth", os.Getenv("ST_AUTH"), "Auth URL for server. Defaults to environment var ST_AUTH.")
	userName  = flag.String("user", os.Getenv("ST_USER"), "User name. Defaults to environment var ST_USER.")
	apiKey    = flag.String("key", os.Getenv("ST_KEY"), "API key (password). Defaults to environment var ST_KEY.")
	checkers  = flag.Int("checkers", 8, "Number of checkers to run in parallel.")
	uploaders = flag.Int("uploaders", 4, "Number of uploaders to run in parallel.")
)

type FsObject struct {
	rel  string
	path string
	info os.FileInfo
}

type FsObjectsChan chan *FsObject

type FsObjects []FsObject

// Write debuging output for this FsObject
func (fs *FsObject) Debugf(text string, args ...interface{}) {
	out := fmt.Sprintf(text, args...)
	log.Printf("%s: %s", fs.rel, out)
}

// md5sum calculates the md5sum of a file returning a lowercase hex string
func (fs *FsObject) md5sum() (string, error) {
	in, err := os.Open(fs.path)
	if err != nil {
		fs.Debugf("Failed to open: %s", err)
		return "", err
	}
	defer in.Close() // FIXME ignoring error
	hash := md5.New()
	_, err = io.Copy(hash, in)
	if err != nil {
		fs.Debugf("Failed to read: %s", err)
		return "", err
	}
	return fmt.Sprintf("%x", hash.Sum(nil)), nil
}

// Checks to see if an object has changed or not by looking at its size, mtime and MD5SUM
//
// If the remote object size is different then it is considered to be
// changed.
//
// If the size is the same and the mtime is the same then it is
// considered to be unchanged.  This is the heuristic rsync uses when
// not using --checksum.
//
// If the size is the same and and mtime is different or unreadable
// and the MD5SUM is the same then the file is considered to be
// unchanged.  In this case the mtime on the object is updated.
//
// Otherwise the file is considered to be changed.
func (fs *FsObject) changed(c *swift.Connection, container string) bool {
	obj, h, err := c.Object(container, fs.rel)
	if err != nil {
		fs.Debugf("Failed to read info: %s", err)
		return true
	}
	if obj.Bytes != fs.info.Size() {
		fs.Debugf("Sizes differ")
		return true
	}

	// Size the same so check the mtime
	m := h.ObjectMetadata()
	t, err := m.GetModTime()
	if err != nil {
		fs.Debugf("Failed to read mtime: %s", err)
	} else if !t.Equal(fs.info.ModTime()) {
		fs.Debugf("Modification times differ")
	} else {
		fs.Debugf("Size and modification time the same - skipping")
		return false
	}

	// mtime is unreadable or different but size is the same so
	// check the MD5SUM
	localMd5, err := fs.md5sum()
	if err != nil {
		fs.Debugf("Failed to calculate md5: %s", err)
		return true
	}
	// fs.Debugf("Local  MD5 %s", localMd5)
	// fs.Debugf("Remote MD5 %s", obj.Hash)
	if localMd5 != strings.ToLower(obj.Hash) {
		fs.Debugf("Md5sums differ")
		return true
	}

	// Update the mtime of the remote object here
	m.SetModTime(fs.info.ModTime())
	err = c.ObjectUpdate(container, fs.rel, m.ObjectHeaders())
	if err != nil {
		fs.Debugf("Failed to update mtime: %s", err)
	}
	fs.Debugf("Updated mtime")

	fs.Debugf("Size and MD5SUM identical - skipping")
	return false
}

// Is this object storable
func (fs *FsObject) storable(c *swift.Connection, container string) bool {
	mode := fs.info.Mode()
	if mode&(os.ModeSymlink|os.ModeNamedPipe|os.ModeSocket|os.ModeDevice) != 0 {
		fs.Debugf("Can't transfer non file/directory")
		return false
	} else if mode&os.ModeDir != 0 {
		// Debug?
		fs.Debugf("FIXME Skipping directory")
		return false
	}
	return true
}

// Puts the FsObject into the container
func (fs *FsObject) put(c *swift.Connection, container string) {
	// FIXME content type
	in, err := os.Open(fs.path)
	if err != nil {
		fs.Debugf("Failed to open: %s", err)
		return
	}
	defer in.Close()
	m := swift.Metadata{}
	m.SetModTime(fs.info.ModTime())
	_, err = c.ObjectPut(container, fs.rel, in, true, "", "", m.ObjectHeaders())
	if err != nil {
		fs.Debugf("Failed to upload: %s", err)
		return
	}
	fs.Debugf("Uploaded")
}

// Walk the path returning a channel of FsObjects
//
// FIXME ignore symlinks?
// FIXME what about hardlinks / etc
func walk(root string) FsObjectsChan {
	out := make(FsObjectsChan, *checkers)
	go func() {
		err := filepath.Walk(root, func(path string, f os.FileInfo, err error) error {
			if err != nil {
				log.Printf("Failed to open directory: %s: %s", path, err)
			} else {
				info, err := os.Lstat(path)
				if err != nil {
					log.Printf("Failed to stat %s: %s", path, err)
					return nil
				}
				rel, err := filepath.Rel(root, path)
				if err != nil {
					log.Printf("Failed to get relative path %s: %s", path, err)
					return nil
				}
				if rel == "." {
					rel = ""
				}
				out <- &FsObject{rel: rel, path: path, info: info}
			}
			return nil
		})
		if err != nil {
			log.Printf("Failed to open directory: %s: %s", root, err)
		}
		close(out)
	}()
	return out
}

// syntaxError prints the syntax
func syntaxError() {
	fmt.Fprintf(os.Stderr, `Sync files and directores to and from swift

FIXME

Full options:
`)
	flag.PrintDefaults()
}

// Exit with the message
func fatal(message string, args ...interface{}) {
	syntaxError()
	fmt.Fprintf(os.Stderr, message, args...)
	os.Exit(1)
}

// checkArgs checks there are enough arguments and prints a message if not
func checkArgs(args []string, n int, message string) {
	if len(args) != n {
		syntaxError()
		fmt.Fprintf(os.Stderr, "%d arguments required: %s\n", n, message)
		os.Exit(1)
	}
}

// Read FsObjects on in and write them to out if they need uploading
//
// FIXME potentially doing lots of MD5SUMS at once
func checker(c *swift.Connection, container string, in, out FsObjectsChan, wg *sync.WaitGroup) {
	defer wg.Done()
	for fs := range in {
		// Check to see if can store this
		if !fs.storable(c, container) {
			continue
		}
		// Check to see if changed or not
		if !fs.changed(c, container) {
			fs.Debugf("Unchanged skipping")
			continue
		}
		out <- fs
	}
}

// Read FsObjects on in and upload them
func uploader(c *swift.Connection, container string, in FsObjectsChan, wg *sync.WaitGroup) {
	defer wg.Done()
	for fs := range in {
		fs.put(c, container)
	}
}

// Syncs a directory into a container
func upload(c *swift.Connection, root, container string) {
	to_be_checked := walk(root)
	to_be_uploaded := make(FsObjectsChan, *uploaders)

	var checkerWg sync.WaitGroup
	checkerWg.Add(*checkers)
	for i := 0; i < *checkers; i++ {
		go checker(c, container, to_be_checked, to_be_uploaded, &checkerWg)
	}

	var uploaderWg sync.WaitGroup
	uploaderWg.Add(*uploaders)
	for i := 0; i < *uploaders; i++ {
		go uploader(c, container, to_be_uploaded, &uploaderWg)
	}

	log.Printf("Waiting for checks to finish")
	checkerWg.Wait()
	close(to_be_uploaded)
	log.Printf("Waiting for uploads to finish")
	uploaderWg.Wait()
}

// Lists the containers
func listContainers(c *swift.Connection) {
	containers, err := c.ContainersAll(nil)
	if err != nil {
		log.Fatalf("Couldn't list containers: %s", err)
	}
	for _, container := range containers {
		fmt.Printf("%9d %12d %s\n", container.Count, container.Bytes, container.Name)
	}
}

// Lists files in a container
func list(c *swift.Connection, container string) {
	//objects, err := c.ObjectsAll(container, &swift.ObjectsOpts{Prefix: "", Delimiter: '/'})
	objects, err := c.ObjectsAll(container, nil)
	if err != nil {
		log.Fatalf("Couldn't read container %q: %s", container, err)
	}
	for _, object := range objects {
		if object.PseudoDirectory {
			fmt.Printf("%9s %19s %s\n", "Directory", "-", object.Name)
		} else {
			fmt.Printf("%9d %19s %s\n", object.Bytes, object.LastModified.Format("2006-01-02 15:04:05"), object.Name)
		}
	}
}

// Makes a container
func mkdir(c *swift.Connection, container string) {
	err := c.ContainerCreate(container, nil)
	if err != nil {
		log.Fatalf("Couldn't create container %q: %s", container, err)
	}
}

// Removes a container
func rmdir(c *swift.Connection, container string) {
	err := c.ContainerDelete(container)
	if err != nil {
		log.Fatalf("Couldn't delete container %q: %s", container, err)
	}
}

func main() {
	flag.Usage = syntaxError
	flag.Parse()
	args := flag.Args()
	runtime.GOMAXPROCS(runtime.NumCPU())

	// Setup profiling if desired
	if *cpuprofile != "" {
		f, err := os.Create(*cpuprofile)
		if err != nil {
			log.Fatal(err)
		}
		pprof.StartCPUProfile(f)
		defer pprof.StopCPUProfile()
	}

	fmt.Println(args)
	if len(args) < 1 {
		fatal("No command supplied\n")
	}

	if *userName == "" {
		log.Fatal("Need -user or environmental variable ST_USER")
	}
	if *apiKey == "" {
		log.Fatal("Need -key or environmental variable ST_KEY")
	}
	if *authUrl == "" {
		log.Fatal("Need -auth or environmental variable ST_AUTH")
	}
	c := &swift.Connection{
		UserName: *userName,
		ApiKey:   *apiKey,
		AuthUrl:  *authUrl,
	}
	err := c.Authenticate()
	if err != nil {
		log.Fatal("Failed to authenticate", err)
	}

	command := args[0]
	args = args[1:]

	switch command {
	case "up", "upload":
		checkArgs(args, 2, "Need directory to read from and container to write to")
		upload(c, args[0], args[1])
	case "list", "ls":
		if len(args) == 0 {
			listContainers(c)
		} else {
			checkArgs(args, 1, "Need container to list")
			list(c, args[0])
		}
	case "mkdir":
		checkArgs(args, 1, "Need container to make")
		mkdir(c, args[0])
	case "rmdir":
		checkArgs(args, 1, "Need container to delte")
		rmdir(c, args[0])
	default:
		log.Fatalf("Unknown command %q", command)
	}

}
<commit_msg>First attempt at download<commit_after>// Sync files and directories to and from swift
//
// Nick Craig-Wood <nick@craig-wood.com>
package main

import (
	"crypto/md5"
	"flag"
	"fmt"
	"github.com/ncw/swift"
	"io"
	"log"
	"os"
	"path"
	"path/filepath"
	"runtime"
	"runtime/pprof"
	"strings"
	"sync"
)

// Globals
var (
	// Flags
	//fileSize      = flag.Int64("s", 1E9, "Size of the check files")
	cpuprofile = flag.String("cpuprofile", "", "Write cpu profile to file")
	//duration      = flag.Duration("duration", time.Hour*24, "Duration to run test")
	//statsInterval = flag.Duration("stats", time.Minute*1, "Interval to print stats")
	//logfile       = flag.String("logfile", "stressdisk.log", "File to write log to set to empty to ignore")

	snet    = flag.Bool("snet", false, "Use internal service network") // FIXME not implemented
	verbose = flag.Bool("verbose", false, "Print lots more stuff")
	quiet   = flag.Bool("quiet", false, "Print as little stuff as possible")
	// FIXME make these part of swift so we get a standard set of flags?
	authUrl   = flag.String("auth", os.Getenv("ST_AUTH"), "Auth URL for server. Defaults to environment var ST_AUTH.")
	userName  = flag.String("user", os.Getenv("ST_USER"), "User name. Defaults to environment var ST_USER.")
	apiKey    = flag.String("key", os.Getenv("ST_KEY"), "API key (password). Defaults to environment var ST_KEY.")
	checkers  = flag.Int("checkers", 8, "Number of checkers to run in parallel.")
	uploaders = flag.Int("uploaders", 4, "Number of uploaders to run in parallel.")
)

type FsObject struct {
	rel  string
	path string
	info os.FileInfo
}

type FsObjectsChan chan *FsObject

type FsObjects []FsObject

// Write debuging output for this FsObject
func (fs *FsObject) Debugf(text string, args ...interface{}) {
	out := fmt.Sprintf(text, args...)
	log.Printf("%s: %s", fs.rel, out)
}

// md5sum calculates the md5sum of a file returning a lowercase hex string
func (fs *FsObject) md5sum() (string, error) {
	in, err := os.Open(fs.path)
	if err != nil {
		fs.Debugf("Failed to open: %s", err)
		return "", err
	}
	defer in.Close() // FIXME ignoring error
	hash := md5.New()
	_, err = io.Copy(hash, in)
	if err != nil {
		fs.Debugf("Failed to read: %s", err)
		return "", err
	}
	return fmt.Sprintf("%x", hash.Sum(nil)), nil
}

// Checks to see if an object has changed or not by looking at its size, mtime and MD5SUM
//
// If the remote object size is different then it is considered to be
// changed.
//
// If the size is the same and the mtime is the same then it is
// considered to be unchanged.  This is the heuristic rsync uses when
// not using --checksum.
//
// If the size is the same and and mtime is different or unreadable
// and the MD5SUM is the same then the file is considered to be
// unchanged.  In this case the mtime on the object is updated.
//
// Otherwise the file is considered to be changed.
func (fs *FsObject) changed(c *swift.Connection, container string) bool {
	obj, h, err := c.Object(container, fs.rel)
	if err != nil {
		fs.Debugf("Failed to read info: %s", err)
		return true
	}
	if obj.Bytes != fs.info.Size() {
		fs.Debugf("Sizes differ")
		return true
	}

	// Size the same so check the mtime
	m := h.ObjectMetadata()
	t, err := m.GetModTime()
	if err != nil {
		fs.Debugf("Failed to read mtime: %s", err)
	} else if !t.Equal(fs.info.ModTime()) {
		fs.Debugf("Modification times differ")
	} else {
		fs.Debugf("Size and modification time the same - skipping")
		return false
	}

	// mtime is unreadable or different but size is the same so
	// check the MD5SUM
	localMd5, err := fs.md5sum()
	if err != nil {
		fs.Debugf("Failed to calculate md5: %s", err)
		return true
	}
	// fs.Debugf("Local  MD5 %s", localMd5)
	// fs.Debugf("Remote MD5 %s", obj.Hash)
	if localMd5 != strings.ToLower(obj.Hash) {
		fs.Debugf("Md5sums differ")
		return true
	}

	// Update the mtime of the remote object here
	m.SetModTime(fs.info.ModTime())
	err = c.ObjectUpdate(container, fs.rel, m.ObjectHeaders())
	if err != nil {
		fs.Debugf("Failed to update mtime: %s", err)
	}
	fs.Debugf("Updated mtime")

	fs.Debugf("Size and MD5SUM identical - skipping")
	return false
}

// Is this object storable
func (fs *FsObject) storable(c *swift.Connection, container string) bool {
	mode := fs.info.Mode()
	if mode&(os.ModeSymlink|os.ModeNamedPipe|os.ModeSocket|os.ModeDevice) != 0 {
		fs.Debugf("Can't transfer non file/directory")
		return false
	} else if mode&os.ModeDir != 0 {
		// Debug?
		fs.Debugf("FIXME Skipping directory")
		return false
	}
	return true
}

// Puts the FsObject into the container
func (fs *FsObject) put(c *swift.Connection, container string) {
	// FIXME content type
	in, err := os.Open(fs.path)
	if err != nil {
		fs.Debugf("Failed to open: %s", err)
		return
	}
	defer in.Close()
	m := swift.Metadata{}
	m.SetModTime(fs.info.ModTime())
	_, err = c.ObjectPut(container, fs.rel, in, true, "", "", m.ObjectHeaders())
	if err != nil {
		fs.Debugf("Failed to upload: %s", err)
		return
	}
	fs.Debugf("Uploaded")
}

// Return an FsObject from a path
//
// May return nil if an error occurred
func NewFsObject(root, path string) *FsObject {
	info, err := os.Lstat(path)
	if err != nil {
		log.Printf("Failed to stat %s: %s", path, err)
		return nil
	}
	rel, err := filepath.Rel(root, path)
	if err != nil {
		log.Printf("Failed to get relative path %s: %s", path, err)
		return nil
	}
	if rel == "." {
		rel = ""
	}
	return &FsObject{rel: rel, path: path, info: info}
}

// Walk the path returning a channel of FsObjects
//
// FIXME ignore symlinks?
// FIXME what about hardlinks / etc
func walk(root string) FsObjectsChan {
	out := make(FsObjectsChan, *checkers)
	go func() {
		err := filepath.Walk(root, func(path string, f os.FileInfo, err error) error {
			if err != nil {
				log.Printf("Failed to open directory: %s: %s", path, err)
			} else {
				if fs := NewFsObject(root, path); fs != nil {
					out <- fs
				}
			}
			return nil
		})
		if err != nil {
			log.Printf("Failed to open directory: %s: %s", root, err)
		}
		close(out)
	}()
	return out
}

// syntaxError prints the syntax
func syntaxError() {
	fmt.Fprintf(os.Stderr, `Sync files and directores to and from swift

FIXME

Full options:
`)
	flag.PrintDefaults()
}

// Exit with the message
func fatal(message string, args ...interface{}) {
	syntaxError()
	fmt.Fprintf(os.Stderr, message, args...)
	os.Exit(1)
}

// checkArgs checks there are enough arguments and prints a message if not
func checkArgs(args []string, n int, message string) {
	if len(args) != n {
		syntaxError()
		fmt.Fprintf(os.Stderr, "%d arguments required: %s\n", n, message)
		os.Exit(1)
	}
}

// Read FsObjects on in and write them to out if they need uploading
//
// FIXME potentially doing lots of MD5SUMS at once
func checker(c *swift.Connection, container string, in, out FsObjectsChan, wg *sync.WaitGroup) {
	defer wg.Done()
	for fs := range in {
		// Check to see if can store this
		if !fs.storable(c, container) {
			continue
		}
		// Check to see if changed or not
		if !fs.changed(c, container) {
			fs.Debugf("Unchanged skipping")
			continue
		}
		out <- fs
	}
}

// Read FsObjects on in and upload them
func uploader(c *swift.Connection, container string, in FsObjectsChan, wg *sync.WaitGroup) {
	defer wg.Done()
	for fs := range in {
		fs.put(c, container)
	}
}

// Syncs a directory into a container
func upload(c *swift.Connection, root, container string) {
	to_be_checked := walk(root)
	to_be_uploaded := make(FsObjectsChan, *uploaders)

	var checkerWg sync.WaitGroup
	checkerWg.Add(*checkers)
	for i := 0; i < *checkers; i++ {
		go checker(c, container, to_be_checked, to_be_uploaded, &checkerWg)
	}

	var uploaderWg sync.WaitGroup
	uploaderWg.Add(*uploaders)
	for i := 0; i < *uploaders; i++ {
		go uploader(c, container, to_be_uploaded, &uploaderWg)
	}

	log.Printf("Waiting for checks to finish")
	checkerWg.Wait()
	close(to_be_uploaded)
	log.Printf("Waiting for uploads to finish")
	uploaderWg.Wait()
}

// Get an object to the filepath making directories if necessary
func get(c *swift.Connection, container, name, filepath string) {
	log.Printf("Download %s to %s", name, filepath)

	dir := path.Dir(filepath)
	err := os.MkdirAll(dir, 0770)
	if err != nil {
		log.Printf("Couldn't make directory %q: %s", dir, err)
		return
	}

	out, err := os.Create(filepath)
	if err != nil {
		log.Printf("Failed to open %q: %s", filepath, err)
		return
	}

	_, geterr := c.ObjectGet(container, name, out, true, nil)
	if geterr != nil {
		log.Printf("Failed to download %q: %s", name, geterr)
	}

	err = out.Close()
	if err != nil {
		log.Printf("Error closing %q: %s", filepath, err)
	}

	if geterr != nil {
		log.Printf("Removing failed download %q", filepath)
		err = os.Remove(filepath)
		if err != nil {
			log.Printf("Failed to remove %q: %s", filepath, err)
		}
	}
}

// Syncs a container into a directory
//
// FIXME don't want to update the modification times on the
// remote server if they are different - want to modify the local
// file!
//
// FIXME need optional stat in FsObject and to be able to make FsObjects from ObjectsAll
func download(c *swift.Connection, container, root string) {
	// FIXME this would be nice running into a channel!
	objects, err := c.ObjectsAll(container, nil)
	if err != nil {
		log.Fatalf("Couldn't read container %q: %s", container, err)
	}

	err = os.MkdirAll(root, 0770)
	if err != nil {
		log.Fatalf("Couldn't make directory %q: %s", root, err)
	}

	for i := range objects {
		object := &objects[i]
		filepath := path.Join(root, object.Name)
		fs := NewFsObject(root, filepath)
		if fs == nil {
			log.Printf("%s: Download: not found", object.Name)
		} else if !fs.storable(c, container) {
			fs.Debugf("Skip: not storable")
			continue
		} else if !fs.changed(c, container) {
			fs.Debugf("Skip: not changed")
			continue
		}
		get(c, container, object.Name, filepath)
	}
}

// Lists the containers
func listContainers(c *swift.Connection) {
	containers, err := c.ContainersAll(nil)
	if err != nil {
		log.Fatalf("Couldn't list containers: %s", err)
	}
	for _, container := range containers {
		fmt.Printf("%9d %12d %s\n", container.Count, container.Bytes, container.Name)
	}
}

// Lists files in a container
func list(c *swift.Connection, container string) {
	//objects, err := c.ObjectsAll(container, &swift.ObjectsOpts{Prefix: "", Delimiter: '/'})
	objects, err := c.ObjectsAll(container, nil)
	if err != nil {
		log.Fatalf("Couldn't read container %q: %s", container, err)
	}
	for _, object := range objects {
		if object.PseudoDirectory {
			fmt.Printf("%9s %19s %s\n", "Directory", "-", object.Name)
		} else {
			fmt.Printf("%9d %19s %s\n", object.Bytes, object.LastModified.Format("2006-01-02 15:04:05"), object.Name)
		}
	}
}

// Makes a container
func mkdir(c *swift.Connection, container string) {
	err := c.ContainerCreate(container, nil)
	if err != nil {
		log.Fatalf("Couldn't create container %q: %s", container, err)
	}
}

// Removes a container
func rmdir(c *swift.Connection, container string) {
	err := c.ContainerDelete(container)
	if err != nil {
		log.Fatalf("Couldn't delete container %q: %s", container, err)
	}
}

func main() {
	flag.Usage = syntaxError
	flag.Parse()
	args := flag.Args()
	runtime.GOMAXPROCS(runtime.NumCPU())

	// Setup profiling if desired
	if *cpuprofile != "" {
		f, err := os.Create(*cpuprofile)
		if err != nil {
			log.Fatal(err)
		}
		pprof.StartCPUProfile(f)
		defer pprof.StopCPUProfile()
	}

	fmt.Println(args)
	if len(args) < 1 {
		fatal("No command supplied\n")
	}

	if *userName == "" {
		log.Fatal("Need -user or environmental variable ST_USER")
	}
	if *apiKey == "" {
		log.Fatal("Need -key or environmental variable ST_KEY")
	}
	if *authUrl == "" {
		log.Fatal("Need -auth or environmental variable ST_AUTH")
	}
	c := &swift.Connection{
		UserName: *userName,
		ApiKey:   *apiKey,
		AuthUrl:  *authUrl,
	}
	err := c.Authenticate()
	if err != nil {
		log.Fatal("Failed to authenticate", err)
	}

	command := args[0]
	args = args[1:]

	switch command {
	case "up", "upload":
		checkArgs(args, 2, "Need directory to read from and container to write to")
		upload(c, args[0], args[1])
	case "down", "download":
		checkArgs(args, 2, "Need container to read from and directory to write to")
		download(c, args[0], args[1])
	case "list", "ls":
		if len(args) == 0 {
			listContainers(c)
		} else {
			checkArgs(args, 1, "Need container to list")
			list(c, args[0])
		}
	case "mkdir":
		checkArgs(args, 1, "Need container to make")
		mkdir(c, args[0])
	case "rmdir":
		checkArgs(args, 1, "Need container to delte")
		rmdir(c, args[0])
	default:
		log.Fatalf("Unknown command %q", command)
	}

}
<|endoftext|>


<commit_before>package sync

import (
	"math/rand"
	"radiusd/config"
	"radiusd/queue"
	"time"
)

func save() {
	entries := queue.Flush()
	if config.Verbose {
		config.Log.Printf("sync.flush %d metrics", len(entries))
	}
	for user, entry := range entries {
		if e := SessionAcct(user, time.Now().Format("2006-01-02 15:04"), entry.InOctet, entry.OutOctet, entry.InPacket, entry.OutPacket, config.Hostname); e != nil {
			config.Log.Printf("WARN: Losing statistic data err=" + e.Error())
		}
		if e := UpdateRemaining(user, entry.InOctet+entry.OutOctet); e != nil {
			config.Log.Printf("WARN: Losing statistic data err=" + e.Error())
		}
	}
}

func Loop() {
	rand.Seed(time.Now().Unix())
	rnd := time.Duration(rand.Int31n(20)) * time.Second
	sleep := time.Duration(time.Minute + rnd)
	if config.Verbose {
		config.Log.Printf("Sync every: %s", sleep.String())
	}

	for range time.Tick(sleep) {
		save()
	}
}

// Force writing stats now
func Force() {
	save()
}
<commit_msg>Bugfix. Wrong timezone<commit_after>package sync

import (
	"math/rand"
	"radiusd/config"
	"radiusd/queue"
	"time"
)

func save() {
	entries := queue.Flush()
	if config.Verbose {
		config.Log.Printf("sync.flush %d metrics", len(entries))
	}
	for user, entry := range entries {
		if e := SessionAcct(user, time.Now().UTC().Format("2006-01-02 15:04"), entry.InOctet, entry.OutOctet, entry.InPacket, entry.OutPacket, config.Hostname); e != nil {
			config.Log.Printf("WARN: Losing statistic data err=" + e.Error())
		}
		if e := UpdateRemaining(user, entry.InOctet+entry.OutOctet); e != nil {
			config.Log.Printf("WARN: Losing statistic data err=" + e.Error())
		}
	}
}

func Loop() {
	rand.Seed(time.Now().Unix())
	rnd := time.Duration(rand.Int31n(20)) * time.Second
	sleep := time.Duration(time.Minute + rnd)
	if config.Verbose {
		config.Log.Printf("Sync every: %s", sleep.String())
	}

	for range time.Tick(sleep) {
		save()
	}
}

// Force writing stats now
func Force() {
	save()
}
<|endoftext|>


<commit_before>// Copyright 2016 Mikio Hara. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

package tcpinfo

import (
	"strings"
	"time"
	"unsafe"

	"github.com/mikioh/tcpopt"
)

var options = [soMax]option{
	soInfo:   {ianaProtocolTCP, sysTCP_INFO, parseInfo},
	soCCInfo: {ianaProtocolTCP, sysTCP_CC_INFO, parseCCInfo},
	soCCAlgo: {ianaProtocolTCP, sysTCP_CONGESTION, parseCCAlgorithm},
}

// Marshal implements the Marshal method of tcpopt.Option interface.
func (i *Info) Marshal() ([]byte, error) { return (*[sizeofTCPInfo]byte)(unsafe.Pointer(i))[:], nil }

// A CAState represents a state of congestion avoidance.
type CAState int

var caStates = map[CAState]string{
	CAOpen:     "open",
	CADisorder: "disorder",
	CACWR:      "congestion window reduced",
	CARecovery: "recovery",
	CALoss:     "loss",
}

func (st CAState) String() string {
	s, ok := caStates[st]
	if !ok {
		return "<nil>"
	}
	return s
}

// A SysInfo represents platform-specific information.
type SysInfo struct {
	PathMTU         uint       `json:"path_mtu"`     // path maximum transmission unit
	AdvertisedMSS   MaxSegSize `json:"adv_mss"`      // advertised maximum segment size
	CAState         CAState    `json:"ca_state"`     // state of congestion avoidance
	KeepAliveProbes uint       `json:"ka_probes"`    // # of keep alive probes sent
	UnackSegs       uint       `json:"unack_segs"`   // # of unack'd segments in transmission queue
	SackSegs        uint       `json:"sack_segs"`    // # of sack'd segments in tranmission queue
	LostSegs        uint       `json:"lost_segs"`    // # of lost segments in transmission queue
	RetransSegs     uint       `json:"retrans_segs"` // # of retransmitting segments in transmission queue
	ForwardAckSegs  uint       `json:"fack_segs"`    // # of forward ack'd segments in transmission queue
}

var sysStates = [12]State{Unknown, Established, SynSent, SynReceived, FinWait1, FinWait2, TimeWait, Closed, CloseWait, LastAck, Listen, Closing}

func parseInfo(b []byte) (tcpopt.Option, error) {
	if len(b) < sizeofTCPInfo {
		return nil, errBufferTooShort
	}
	sti := (*sysTCPInfo)(unsafe.Pointer(&b[0]))
	i := &Info{State: sysStates[sti.State]}
	if sti.Options&sysTCPI_OPT_WSCALE != 0 {
		i.Options = append(i.Options, WindowScale(sti.Pad_cgo_0[0]>>4))
		i.PeerOptions = append(i.PeerOptions, WindowScale(sti.Pad_cgo_0[0]&0x0f))
	}
	if sti.Options&sysTCPI_OPT_TIMESTAMPS != 0 {
		i.Options = append(i.Options, Timestamps(true))
		i.PeerOptions = append(i.PeerOptions, Timestamps(true))
	}
	i.SenderMSS = MaxSegSize(sti.Snd_mss)
	i.ReceiverMSS = MaxSegSize(sti.Rcv_mss)
	i.RTT = time.Duration(sti.Rtt) * time.Microsecond
	i.RTTVar = time.Duration(sti.Rttvar) * time.Microsecond
	i.RTO = time.Duration(sti.Rto) * time.Microsecond
	i.ATO = time.Duration(sti.Ato) * time.Microsecond
	i.LastDataSent = time.Duration(sti.Last_data_sent) * time.Millisecond
	i.LastDataReceived = time.Duration(sti.Last_data_recv) * time.Millisecond
	i.LastAckReceived = time.Duration(sti.Last_ack_recv) * time.Millisecond
	i.FlowControl = &FlowControl{
		ReceiverWindow: uint(sti.Rcv_space),
	}
	i.CongestionControl = &CongestionControl{
		SenderSSThreshold:   uint(sti.Snd_ssthresh),
		ReceiverSSThreshold: uint(sti.Rcv_ssthresh),
		SenderWindow:        uint(sti.Snd_cwnd),
	}
	i.Sys = &SysInfo{
		PathMTU:         uint(sti.Pmtu),
		AdvertisedMSS:   MaxSegSize(sti.Advmss),
		CAState:         CAState(sti.Ca_state),
		KeepAliveProbes: uint(sti.Probes),
		UnackSegs:       uint(sti.Unacked),
		SackSegs:        uint(sti.Sacked),
		LostSegs:        uint(sti.Lost),
		RetransSegs:     uint(sti.Retrans),
		ForwardAckSegs:  uint(sti.Fackets),
	}
	return i, nil
}

// A VegasInfo represents Vegas congestion control information.
type VegasInfo struct {
	Enabled    bool          `json:"enabled"`
	RoundTrips uint          `json:"rnd_trips"` // # of round-trips
	RTT        time.Duration `json:"rtt"`       // round-trip time
	MinRTT     time.Duration `json:"min_rtt"`   // minimum round-trip time
}

// Algorithm implements the Algorithm method of CCAlgorithmInfo
// interface.
func (vi *VegasInfo) Algorithm() string { return "vegas" }

// A CEState represents a state of ECN congestion encountered (CE)
// codepoint.
type CEState int

// A DCTCPInfo represents Datacenter TCP congestion control
// information.
type DCTCPInfo struct {
	Enabled         bool    `json:"enabled"`
	CEState         CEState `json:"ce_state"`    // state of ECN CE codepoint
	Alpha           uint    `json:"alpha"`       // fraction of bytes sent
	ECNAckedBytes   uint    `json:"ecn_acked"`   // # of acked bytes with ECN
	TotalAckedBytes uint    `json:"total_acked"` // total # of acked bytes
}

// Algorithm implements the Algorithm method of CCAlgorithmInfo
// interface.
func (di *DCTCPInfo) Algorithm() string { return "dctcp" }

func parseCCAlgorithmInfo(name string, b []byte) (CCAlgorithmInfo, error) {
	if strings.HasPrefix(name, "dctcp") {
		if len(b) < sizeofTCPDCTCPInfo {
			return nil, errBufferTooShort
		}
		sdi := (*sysTCPDCTCPInfo)(unsafe.Pointer(&b[0]))
		di := &DCTCPInfo{Alpha: uint(sdi.Alpha)}
		if sdi.Enabled != 0 {
			di.Enabled = true
		}
		return di, nil
	}
	if len(b) < sizeofTCPVegasInfo {
		return nil, errBufferTooShort
	}
	svi := (*sysTCPVegasInfo)(unsafe.Pointer(&b[0]))
	vi := &VegasInfo{
		RoundTrips: uint(svi.Rttcnt),
		RTT:        time.Duration(svi.Rtt) * time.Microsecond,
		MinRTT:     time.Duration(svi.Minrtt) * time.Microsecond,
	}
	if svi.Enabled != 0 {
		vi.Enabled = true
	}
	return vi, nil
}
<commit_msg>tcpinfo: update linux-specific information<commit_after>// Copyright 2016 Mikio Hara. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

package tcpinfo

import (
	"strings"
	"time"
	"unsafe"

	"github.com/mikioh/tcpopt"
)

var options = [soMax]option{
	soInfo:   {ianaProtocolTCP, sysTCP_INFO, parseInfo},
	soCCInfo: {ianaProtocolTCP, sysTCP_CC_INFO, parseCCInfo},
	soCCAlgo: {ianaProtocolTCP, sysTCP_CONGESTION, parseCCAlgorithm},
}

// Marshal implements the Marshal method of tcpopt.Option interface.
func (i *Info) Marshal() ([]byte, error) { return (*[sizeofTCPInfo]byte)(unsafe.Pointer(i))[:], nil }

// A CAState represents a state of congestion avoidance.
type CAState int

var caStates = map[CAState]string{
	CAOpen:     "open",
	CADisorder: "disorder",
	CACWR:      "congestion window reduced",
	CARecovery: "recovery",
	CALoss:     "loss",
}

func (st CAState) String() string {
	s, ok := caStates[st]
	if !ok {
		return "<nil>"
	}
	return s
}

// A SysInfo represents platform-specific information.
type SysInfo struct {
	PathMTU         uint       `json:"path_mtu"`     // path maximum transmission unit
	AdvertisedMSS   MaxSegSize `json:"adv_mss"`      // advertised maximum segment size
	CAState         CAState    `json:"ca_state"`     // state of congestion avoidance
	KeepAliveProbes uint       `json:"ka_probes"`    // # of keep alive probes sent
	UnackSegs       uint       `json:"unack_segs"`   // # of unack'd segments in transmission queue
	SackSegs        uint       `json:"sack_segs"`    // # of sack'd segments in tranmission queue
	LostSegs        uint       `json:"lost_segs"`    // # of lost segments in transmission queue
	RetransSegs     uint       `json:"retrans_segs"` // # of retransmitting segments in transmission queue
	ForwardAckSegs  uint       `json:"fack_segs"`    // # of forward ack'd segments in transmission queue
}

var sysStates = [12]State{Unknown, Established, SynSent, SynReceived, FinWait1, FinWait2, TimeWait, Closed, CloseWait, LastAck, Listen, Closing}

func parseInfo(b []byte) (tcpopt.Option, error) {
	if len(b) < sizeofTCPInfo {
		return nil, errBufferTooShort
	}
	sti := (*sysTCPInfo)(unsafe.Pointer(&b[0]))
	i := &Info{State: sysStates[sti.State]}
	if sti.Options&sysTCPI_OPT_WSCALE != 0 {
		i.Options = append(i.Options, WindowScale(sti.Pad_cgo_0[0]>>4))
		i.PeerOptions = append(i.PeerOptions, WindowScale(sti.Pad_cgo_0[0]&0x0f))
	}
	if sti.Options&sysTCPI_OPT_SACK != 0 {
		i.Options = append(i.Options, SACKPermitted(true))
		i.PeerOptions = append(i.PeerOptions, SACKPermitted(true))
	}
	if sti.Options&sysTCPI_OPT_TIMESTAMPS != 0 {
		i.Options = append(i.Options, Timestamps(true))
		i.PeerOptions = append(i.PeerOptions, Timestamps(true))
	}
	i.SenderMSS = MaxSegSize(sti.Snd_mss)
	i.ReceiverMSS = MaxSegSize(sti.Rcv_mss)
	i.RTT = time.Duration(sti.Rtt) * time.Microsecond
	i.RTTVar = time.Duration(sti.Rttvar) * time.Microsecond
	i.RTO = time.Duration(sti.Rto) * time.Microsecond
	i.ATO = time.Duration(sti.Ato) * time.Microsecond
	i.LastDataSent = time.Duration(sti.Last_data_sent) * time.Millisecond
	i.LastDataReceived = time.Duration(sti.Last_data_recv) * time.Millisecond
	i.LastAckReceived = time.Duration(sti.Last_ack_recv) * time.Millisecond
	i.FlowControl = &FlowControl{
		ReceiverWindow: uint(sti.Rcv_space),
	}
	i.CongestionControl = &CongestionControl{
		SenderSSThreshold:   uint(sti.Snd_ssthresh),
		ReceiverSSThreshold: uint(sti.Rcv_ssthresh),
		SenderWindow:        uint(sti.Snd_cwnd),
	}
	i.Sys = &SysInfo{
		PathMTU:         uint(sti.Pmtu),
		AdvertisedMSS:   MaxSegSize(sti.Advmss),
		CAState:         CAState(sti.Ca_state),
		KeepAliveProbes: uint(sti.Probes),
		UnackSegs:       uint(sti.Unacked),
		SackSegs:        uint(sti.Sacked),
		LostSegs:        uint(sti.Lost),
		RetransSegs:     uint(sti.Retrans),
		ForwardAckSegs:  uint(sti.Fackets),
	}
	return i, nil
}

// A VegasInfo represents Vegas congestion control information.
type VegasInfo struct {
	Enabled    bool          `json:"enabled"`
	RoundTrips uint          `json:"rnd_trips"` // # of round-trips
	RTT        time.Duration `json:"rtt"`       // round-trip time
	MinRTT     time.Duration `json:"min_rtt"`   // minimum round-trip time
}

// Algorithm implements the Algorithm method of CCAlgorithmInfo
// interface.
func (vi *VegasInfo) Algorithm() string { return "vegas" }

// A CEState represents a state of ECN congestion encountered (CE)
// codepoint.
type CEState int

// A DCTCPInfo represents Datacenter TCP congestion control
// information.
type DCTCPInfo struct {
	Enabled         bool    `json:"enabled"`
	CEState         CEState `json:"ce_state"`    // state of ECN CE codepoint
	Alpha           uint    `json:"alpha"`       // fraction of bytes sent
	ECNAckedBytes   uint    `json:"ecn_acked"`   // # of acked bytes with ECN
	TotalAckedBytes uint    `json:"total_acked"` // total # of acked bytes
}

// Algorithm implements the Algorithm method of CCAlgorithmInfo
// interface.
func (di *DCTCPInfo) Algorithm() string { return "dctcp" }

func parseCCAlgorithmInfo(name string, b []byte) (CCAlgorithmInfo, error) {
	if strings.HasPrefix(name, "dctcp") {
		if len(b) < sizeofTCPDCTCPInfo {
			return nil, errBufferTooShort
		}
		sdi := (*sysTCPDCTCPInfo)(unsafe.Pointer(&b[0]))
		di := &DCTCPInfo{Alpha: uint(sdi.Alpha)}
		if sdi.Enabled != 0 {
			di.Enabled = true
		}
		return di, nil
	}
	if len(b) < sizeofTCPVegasInfo {
		return nil, errBufferTooShort
	}
	svi := (*sysTCPVegasInfo)(unsafe.Pointer(&b[0]))
	vi := &VegasInfo{
		RoundTrips: uint(svi.Rttcnt),
		RTT:        time.Duration(svi.Rtt) * time.Microsecond,
		MinRTT:     time.Duration(svi.Minrtt) * time.Microsecond,
	}
	if svi.Enabled != 0 {
		vi.Enabled = true
	}
	return vi, nil
}
<|endoftext|>


<commit_before>package urknall

import (
	"fmt"
	"github.com/dynport/dgtk/pubsub"
	"github.com/dynport/gocli"
	"io"
	"log"
	"strings"
	"time"
)

const (
	colorDryRun = 226
	colorCached = 33
	colorExec   = 34
)

var colorMapping = map[string]int{
	statusCached:       colorCached,
	statusExecFinished: colorExec,
}

// needs to be closed afterwards
func OpenStdoutLogger() (io.Closer, error) {
	logger := &stdoutLogger{}
	logger.Formatter = logger.DefaultFormatter
	e := logger.Start()
	if e != nil {
		return nil, e
	}
	return logger, nil
}

type stdoutLogger struct {
	Formatter    formatter
	maxLengths   map[int]int
	started      time.Time
	finished     chan interface{}
	pubSub       *pubsub.PubSub
	subscription *pubsub.Subscription
}

func (logger *stdoutLogger) Started() time.Time {
	if logger.started.IsZero() {
		logger.started = time.Now()
	}
	return logger.started
}

func (logger *stdoutLogger) formatCommandOuput(message *Message) string {
	prefix := fmt.Sprintf("[%s][%s][%s]", formatIp(message.HostIP()), formatRunlistName(message.RunlistName()), formatDuration(logger.sinceStarted()))
	line := message.line
	if message.IsStderr() {
		line = gocli.Red(line)
	}
	return prefix + " " + line
}

func formatIp(ip string) string {
	return fmt.Sprintf("%15s", ip)
}

type formatter func(urknallMessage *Message) string

func (logger *stdoutLogger) DefaultFormatter(message *Message) string {
	ignoreKeys := []string{MessageRunlistsPrecompile, MessageCleanupCacheEntries, MessageRunlistsProvision, MessageUrknallInternal}
	for _, k := range ignoreKeys {
		if strings.HasPrefix(message.Key(), k) {
			return ""
		}
	}
	if len(message.line) > 0 {
		return logger.formatCommandOuput(message)
	}
	ip := message.HostIP()
	runlistName := message.RunlistName()
	payload := ""
	if message.task != nil {
		payload = message.task.Command().Logging()
	}
	execStatus := message.execStatus
	if color := colorMapping[execStatus]; color > 0 {
		execStatus = gocli.Colorize(color, execStatus)
	}
	parts := []string{
		fmt.Sprintf("[%s][%s][%s][%-8s]%s",
			formatIp(ip),
			formatRunlistName(runlistName),
			formatDuration(logger.sinceStarted()),
			execStatus,
			payload,
		),
	}
	return strings.Join(parts, " ")
}

func formatRunlistName(name string) string {
	if len(name) > 8 {
		name = name[0:8]
	}
	return fmt.Sprintf("%-8s", name)
}

func formatDuration(dur time.Duration) string {
	durString := ""
	if dur >= 1*time.Millisecond {
		durString = fmt.Sprintf("%.03f", dur.Seconds())
	}
	return fmt.Sprintf("%7s", durString)
}

func (logger *stdoutLogger) sinceStarted() time.Duration {
	return time.Now().Sub(logger.Started())
}

func (logger *stdoutLogger) Start() error {
	logger.started = time.Now()
	if logger.Formatter == nil {
		return fmt.Errorf("Formatter must be set")
	}
	logger.pubSub = pubsub.New()
	RegisterPubSub(logger.pubSub)
	logger.subscription = logger.pubSub.Subscribe(func(m *Message) {
		if message := logger.Formatter(m); message != "" {
			log.Println(message)
		}
	})
	return nil
}

func (logger *stdoutLogger) Close() error {
	return logger.subscription.Close()
}

func init() {
	log.SetFlags(0)
}
<commit_msg>fixed width of exec status indicator<commit_after>package urknall

import (
	"fmt"
	"github.com/dynport/dgtk/pubsub"
	"github.com/dynport/gocli"
	"io"
	"log"
	"strings"
	"time"
)

const (
	colorDryRun = 226
	colorCached = 33
	colorExec   = 34
)

var colorMapping = map[string]int{
	statusCached:       colorCached,
	statusExecFinished: colorExec,
}

// needs to be closed afterwards
func OpenStdoutLogger() (io.Closer, error) {
	logger := &stdoutLogger{}
	logger.Formatter = logger.DefaultFormatter
	e := logger.Start()
	if e != nil {
		return nil, e
	}
	return logger, nil
}

type stdoutLogger struct {
	Formatter    formatter
	maxLengths   map[int]int
	started      time.Time
	finished     chan interface{}
	pubSub       *pubsub.PubSub
	subscription *pubsub.Subscription
}

func (logger *stdoutLogger) Started() time.Time {
	if logger.started.IsZero() {
		logger.started = time.Now()
	}
	return logger.started
}

func (logger *stdoutLogger) formatCommandOuput(message *Message) string {
	prefix := fmt.Sprintf("[%s][%s][%s]", formatIp(message.HostIP()), formatRunlistName(message.RunlistName()), formatDuration(logger.sinceStarted()))
	line := message.line
	if message.IsStderr() {
		line = gocli.Red(line)
	}
	return prefix + " " + line
}

func formatIp(ip string) string {
	return fmt.Sprintf("%15s", ip)
}

type formatter func(urknallMessage *Message) string

func (logger *stdoutLogger) DefaultFormatter(message *Message) string {
	ignoreKeys := []string{MessageRunlistsPrecompile, MessageCleanupCacheEntries, MessageRunlistsProvision, MessageUrknallInternal}
	for _, k := range ignoreKeys {
		if strings.HasPrefix(message.Key(), k) {
			return ""
		}
	}
	if len(message.line) > 0 {
		return logger.formatCommandOuput(message)
	}
	ip := message.HostIP()
	runlistName := message.RunlistName()
	payload := ""
	if message.task != nil {
		payload = message.task.Command().Logging()
	}
	execStatus := fmt.Sprintf("%-8s", message.execStatus)
	if color := colorMapping[message.execStatus]; color > 0 {
		execStatus = gocli.Colorize(color, execStatus)
	}
	parts := []string{
		fmt.Sprintf("[%s][%s][%s][%s]%s",
			formatIp(ip),
			formatRunlistName(runlistName),
			formatDuration(logger.sinceStarted()),
			execStatus,
			payload,
		),
	}
	return strings.Join(parts, " ")
}

func formatRunlistName(name string) string {
	if len(name) > 8 {
		name = name[0:8]
	}
	return fmt.Sprintf("%-8s", name)
}

func formatDuration(dur time.Duration) string {
	durString := ""
	if dur >= 1*time.Millisecond {
		durString = fmt.Sprintf("%.03f", dur.Seconds())
	}
	return fmt.Sprintf("%7s", durString)
}

func (logger *stdoutLogger) sinceStarted() time.Duration {
	return time.Now().Sub(logger.Started())
}

func (logger *stdoutLogger) Start() error {
	logger.started = time.Now()
	if logger.Formatter == nil {
		return fmt.Errorf("Formatter must be set")
	}
	logger.pubSub = pubsub.New()
	RegisterPubSub(logger.pubSub)
	logger.subscription = logger.pubSub.Subscribe(func(m *Message) {
		if message := logger.Formatter(m); message != "" {
			log.Println(message)
		}
	})
	return nil
}

func (logger *stdoutLogger) Close() error {
	return logger.subscription.Close()
}

func init() {
	log.SetFlags(0)
}
<|endoftext|>


<commit_before>// Package tt is a framework for writing table-driven tests
package tabby
<commit_msg>Update Tabby doc<commit_after>// Package tabby is a framework for writing table-driven tests
package tabby
<|endoftext|>