<commit_before>package main

import (
	"bytes"
	"flag"
	"github.com/ian-kent/go-log/appenders"
	"github.com/ian-kent/go-log/layout"
	"github.com/ian-kent/go-log/levels"
	"github.com/ian-kent/go-log/log"
	gotcha "github.com/ian-kent/gotcha/app"
	"github.com/ian-kent/gotcha/http"
	"net/url"
	"os"
	"strconv"
	"strings"
)

var maxlen = 262144000
var retain = 52428800
var applog bytes.Buffer

type Appender struct {
	a appenders.Appender
}

func (a *Appender) Write(level levels.LogLevel, message string, args ...interface{}) {
	a.a.Write(level, message, args...)
	applog.Write([]byte(a.Layout().Format(level, message, args...) + "\n"))
	if applog.Len() > maxlen {
		b := applog.Bytes()[retain:]
		applog = *new(bytes.Buffer)
		applog.Write(b)
	}
}
func (a *Appender) SetLayout(layout layout.Layout) {
	a.a.SetLayout(layout)
}
func (a *Appender) Layout() layout.Layout {
	return a.a.Layout()
}
func NewAppender() *Appender {
	return &Appender{
		a: appenders.Console(),
	}
}

func main() {
	log.Logger().SetAppender(NewAppender())

	global := "websysd.json"
	flag.StringVar(&global, "global", global, "global environment configuration")

	workspaces := make([]string, 0)
	flag.Var((*AppendSliceValue)(&workspaces), "workspace", "websysd workspace file (can be specified multiple times), defaults to './workspace.json'")

	// Create our Gotcha application
	var app = gotcha.Create(Asset)

	if len(workspaces) == 0 {
		workspaces = append(workspaces, "./workspace.json")
	}

	LoadConfig(global, workspaces)

	GlobalWorkspace = NewWorkspace(GlobalConfigWorkspace.Name, GlobalConfigWorkspace.Environment, make(map[string]map[string][]string), GlobalConfigWorkspace.InheritEnvironment)
	for fn, args := range GlobalConfigWorkspace.Functions {
		log.Info("=> Creating global function: %s", fn)
		GlobalWorkspace.Functions[fn] = &Function{
			Name:     fn,
			Args:     args.Args,
			Command:  args.Command,
			Executor: args.Executor,
		}
	}

	if GlobalWorkspace.InheritEnvironment {
		log.Info("=> Inheriting process environment into global workspace")
		for _, k := range os.Environ() {
			p := strings.SplitN(k, "=", 2)
			log.Info("  %s = %s", p[0], p[1])
			// TODO variable subst for current env vars
			if _, ok := GlobalWorkspace.Environment[p[0]]; !ok {
				GlobalWorkspace.Environment[p[0]] = p[1]
			}
		}
	}

	for _, ws := range ConfigWorkspaces {
		log.Info("=> Creating workspace: %s", ws.Name)

		var workspace *Workspace
		if wks, ok := Workspaces[ws.Name]; ok {
			log.Warn("Workspace %s already exists, merging tasks and environment")
			workspace = wks
		} else {
			workspace = NewWorkspace(ws.Name, ws.Environment, ws.Columns, ws.InheritEnvironment)
			Workspaces[ws.Name] = workspace
		}

		workspace.IsLocked = ws.IsLocked

		if workspace.InheritEnvironment && !GlobalWorkspace.InheritEnvironment {
			log.Info("=> Inheriting process environment into workspace")
			for _, k := range os.Environ() {
				p := strings.SplitN(k, "=", 2)
				log.Info("  %s = %s", p[0], p[1])
				// TODO variable subst for current env vars
				if _, ok := GlobalWorkspace.Environment[p[0]]; !ok {
					GlobalWorkspace.Environment[p[0]] = p[1]
				}
			}
		}

		for fn, args := range ws.Functions {
			log.Info("=> Creating workspace function: %s", fn)
			workspace.Functions[fn] = &Function{
				Name:     fn,
				Args:     args.Args,
				Command:  args.Command,
				Executor: args.Executor,
			}
		}

		for _, t := range ws.Tasks {
			log.Info("=> Creating task: %s", t.Name)

			if _, ok := workspace.Tasks[t.Name]; ok {
				log.Warn("Task %s already exists, overwriting")
			}

			env := make(map[string]string)
			for k, v := range GlobalWorkspace.Environment {
				env[k] = v
			}
			for k, v := range ws.Environment {
				env[k] = v
			}
			for k, v := range t.Environment {
				env[k] = v
			}

			task := NewTask(workspace, t.Name, t.Executor, t.Command, env, t.Service, t.Stdout, t.Stderr, t.Metadata, t.Pwd)
			workspace.Tasks[t.Name] = task
		}
	}

	// Get the router
	r := app.Router

	// Create some routes
	r.Get("/", list_workspaces)
	r.Get("/log", show_log)
	r.Get("/workspace/(?P<workspace>[^/]+)", list_tasks)

	// Serve static content (but really use a CDN)
	r.Get("/images/(?P<file>.*)", r.Static("assets/images/{{file}}"))
	r.Get("/css/(?P<file>.*)", r.Static("assets/css/{{file}}"))

	r.Post("/workspace/(?P<workspace>[^/]+)/task/(?P<task>[^/]+)/start", startTask)
	r.Post("/workspace/(?P<workspace>[^/]+)/task/(?P<task>[^/]+)/stop", stopTask)
	r.Post("/workspace/(?P<workspace>[^/]+)/task/(?P<task>[^/]+)/enable", enableServiceTask)
	r.Post("/workspace/(?P<workspace>[^/]+)/task/(?P<task>[^/]+)/disable", disableServiceTask)
	r.Get("/workspace/(?P<workspace>[^/]+)/task/(?P<task>[^/]+)", taskHistory)

	r.Get("/workspace/(?P<workspace>[^/]+)/task/(?P<task>[^/]+)/run/(?P<run>\\d+)", taskRun)
	r.Get("/workspace/(?P<workspace>[^/]+)/task/(?P<task>[^/]+)/run/(?P<run>\\d+)/stdout", taskRunStdout)
	r.Get("/workspace/(?P<workspace>[^/]+)/task/(?P<task>[^/]+)/run/(?P<run>\\d+)/stderr", taskRunStderr)

	// Start our application
	app.Start()

	defer func() {
		for _, ws := range Workspaces {
			for _, t := range ws.Tasks {
				if t.ActiveTask != nil && t.ActiveTask.Cmd != nil && t.ActiveTask.Cmd.Process != nil {
					t.ActiveTask.Cmd.Process.Kill()
				}
			}
		}
	}()

	<-make(chan int)
}

func redir(session *http.Session) {
	redir := "/"

	if k := session.Request.Referer(); len(k) > 0 {
		redir = k
	}

	session.Redirect(&url.URL{Path: redir})
}

func startTask(session *http.Session) {
	ws, _ := session.Stash["workspace"].(string)
	id, _ := session.Stash["task"].(string)

	Workspaces[ws].Tasks[id].Start()

	redir(session)
}

func stopTask(session *http.Session) {
	ws, _ := session.Stash["workspace"].(string)
	id, _ := session.Stash["task"].(string)

	Workspaces[ws].Tasks[id].Stop()

	redir(session)
}

func enableServiceTask(session *http.Session) {
	ws, _ := session.Stash["workspace"].(string)
	id, _ := session.Stash["task"].(string)

	Workspaces[ws].Tasks[id].Service = true

	if Workspaces[ws].Tasks[id].ActiveTask == nil {
		Workspaces[ws].Tasks[id].Start()
	}

	redir(session)
}

func disableServiceTask(session *http.Session) {
	ws, _ := session.Stash["workspace"].(string)
	id, _ := session.Stash["task"].(string)

	Workspaces[ws].Tasks[id].Service = false

	redir(session)
}

func taskHistory(session *http.Session) {
	ws, _ := session.Stash["workspace"].(string)
	id, _ := session.Stash["task"].(string)

	session.Stash["Title"] = "Task"
	session.Stash["Page"] = "History"
	session.Stash["Workspace"] = Workspaces[ws]
	session.Stash["Task"] = Workspaces[ws].Tasks[id]

	session.RenderWithLayout("task.html", "layout.html", "Content")
}

func taskRun(session *http.Session) {
	ws, _ := session.Stash["workspace"].(string)
	id, _ := session.Stash["task"].(string)
	run, _ := strconv.Atoi(session.Stash["run"].(string))

	session.Stash["Title"] = "Task run"
	session.Stash["Page"] = "TaskRun"
	session.Stash["Workspace"] = Workspaces[ws]
	session.Stash["Task"] = Workspaces[ws].Tasks[id]
	session.Stash["TaskRun"] = Workspaces[ws].Tasks[id].TaskRuns[run]

	session.RenderWithLayout("taskrun.html", "layout.html", "Content")
}

func taskRunStdout(session *http.Session) {
	ws, _ := session.Stash["workspace"].(string)
	id, _ := session.Stash["task"].(string)
	run, _ := strconv.Atoi(session.Stash["run"].(string))

	session.Stash["Title"] = "Task run stdout"
	session.Stash["Page"] = "TaskOutput"
	session.Stash["Type"] = "stdout"
	session.Stash["Workspace"] = Workspaces[ws]
	session.Stash["Task"] = Workspaces[ws].Tasks[id]
	session.Stash["TaskRun"] = Workspaces[ws].Tasks[id].TaskRuns[run]
	session.Stash["LogOutput"] = Workspaces[ws].Tasks[id].TaskRuns[run].StdoutBuf.String()

	session.RenderWithLayout("log.html", "layout.html", "Content")
}

func taskRunStderr(session *http.Session) {
	ws, _ := session.Stash["workspace"].(string)
	id, _ := session.Stash["task"].(string)
	run, _ := strconv.Atoi(session.Stash["run"].(string))

	session.Stash["Title"] = "Task run stderr"
	session.Stash["Page"] = "TaskOutput"
	session.Stash["Type"] = "stderr"
	session.Stash["Workspace"] = Workspaces[ws]
	session.Stash["Task"] = Workspaces[ws].Tasks[id]
	session.Stash["TaskRun"] = Workspaces[ws].Tasks[id].TaskRuns[run]
	session.Stash["LogOutput"] = Workspaces[ws].Tasks[id].TaskRuns[run].StderrBuf.String()

	session.RenderWithLayout("log.html", "layout.html", "Content")
}

func list_workspaces(session *http.Session) {
	// Stash a value and render a template
	session.Stash["Title"] = "websysd"
	session.Stash["Page"] = "Workspaces"
	session.Stash["Workspaces"] = Workspaces
	session.RenderWithLayout("workspaces.html", "layout.html", "Content")
}

func list_tasks(session *http.Session) {
	ws, _ := session.Stash["workspace"].(string)

	// Stash a value and render a template
	session.Stash["Title"] = "websysd"
	session.Stash["Page"] = "Tasks"
	session.Stash["Workspace"] = Workspaces[ws]
	session.Stash["Tasks"] = Workspaces[ws].Tasks
	session.RenderWithLayout("tasks.html", "layout.html", "Content")
}

func show_log(session *http.Session) {
	session.Stash["Title"] = "websysd log"
	session.Stash["Page"] = "AppLog"
	session.Stash["LogOutput"] = applog.String()

	session.RenderWithLayout("applog.html", "layout.html", "Content")
}
<commit_msg>Two fixes: - Cater for scenario where the environment KEY is empty - Set the workspace intended environment on the workspace instead of on `GlobalWorkspace`<commit_after>package main

import (
	"bytes"
	"flag"
	"github.com/ian-kent/go-log/appenders"
	"github.com/ian-kent/go-log/layout"
	"github.com/ian-kent/go-log/levels"
	"github.com/ian-kent/go-log/log"
	gotcha "github.com/ian-kent/gotcha/app"
	"github.com/ian-kent/gotcha/http"
	"net/url"
	"os"
	"strconv"
	"strings"
)

var maxlen = 262144000
var retain = 52428800
var applog bytes.Buffer

type Appender struct {
	a appenders.Appender
}

func (a *Appender) Write(level levels.LogLevel, message string, args ...interface{}) {
	a.a.Write(level, message, args...)
	applog.Write([]byte(a.Layout().Format(level, message, args...) + "\n"))
	if applog.Len() > maxlen {
		b := applog.Bytes()[retain:]
		applog = *new(bytes.Buffer)
		applog.Write(b)
	}
}
func (a *Appender) SetLayout(layout layout.Layout) {
	a.a.SetLayout(layout)
}
func (a *Appender) Layout() layout.Layout {
	return a.a.Layout()
}
func NewAppender() *Appender {
	return &Appender{
		a: appenders.Console(),
	}
}

func main() {
	log.Logger().SetAppender(NewAppender())

	global := "websysd.json"
	flag.StringVar(&global, "global", global, "global environment configuration")

	workspaces := make([]string, 0)
	flag.Var((*AppendSliceValue)(&workspaces), "workspace", "websysd workspace file (can be specified multiple times), defaults to './workspace.json'")

	// Create our Gotcha application
	var app = gotcha.Create(Asset)

	if len(workspaces) == 0 {
		workspaces = append(workspaces, "./workspace.json")
	}

	LoadConfig(global, workspaces)

	GlobalWorkspace = NewWorkspace(GlobalConfigWorkspace.Name, GlobalConfigWorkspace.Environment, make(map[string]map[string][]string), GlobalConfigWorkspace.InheritEnvironment)
	for fn, args := range GlobalConfigWorkspace.Functions {
		log.Info("=> Creating global function: %s", fn)
		GlobalWorkspace.Functions[fn] = &Function{
			Name:     fn,
			Args:     args.Args,
			Command:  args.Command,
			Executor: args.Executor,
		}
	}

	if GlobalWorkspace.InheritEnvironment {
		log.Info("=> Inheriting process environment into global workspace")
		for _, k := range os.Environ() {
			p := strings.SplitN(k, "=", 2)
			if strings.TrimSpace(p[0]) == "" {
				log.Warn("Skipping empty environment key")
				continue
			}
			log.Info("  %s = %s", p[0], p[1])
			// TODO variable subst for current env vars
			if _, ok := GlobalWorkspace.Environment[p[0]]; !ok {
				GlobalWorkspace.Environment[p[0]] = p[1]
			}
		}
	}

	for _, ws := range ConfigWorkspaces {
		log.Info("=> Creating workspace: %s", ws.Name)

		var workspace *Workspace
		if wks, ok := Workspaces[ws.Name]; ok {
			log.Warn("Workspace %s already exists, merging tasks and environment")
			workspace = wks
		} else {
			workspace = NewWorkspace(ws.Name, ws.Environment, ws.Columns, ws.InheritEnvironment)
			Workspaces[ws.Name] = workspace
		}

		workspace.IsLocked = ws.IsLocked

		if workspace.InheritEnvironment && !GlobalWorkspace.InheritEnvironment {
			log.Info("=> Inheriting process environment into workspace")
			for _, k := range os.Environ() {
				p := strings.SplitN(k, "=", 2)
				if strings.TrimSpace(p[0]) == "" {
					log.Warn("Skipping empty environment key")
					continue
				}
				log.Info("  %s = %s", p[0], p[1])
				// TODO variable subst for current env vars
				if _, ok := workspace.Environment[p[0]]; !ok {
					workspace.Environment[p[0]] = p[1]
				}
			}
		}

		for fn, args := range ws.Functions {
			log.Info("=> Creating workspace function: %s", fn)
			workspace.Functions[fn] = &Function{
				Name:     fn,
				Args:     args.Args,
				Command:  args.Command,
				Executor: args.Executor,
			}
		}

		for _, t := range ws.Tasks {
			log.Info("=> Creating task: %s", t.Name)

			if _, ok := workspace.Tasks[t.Name]; ok {
				log.Warn("Task %s already exists, overwriting")
			}

			env := make(map[string]string)
			for k, v := range GlobalWorkspace.Environment {
				env[k] = v
			}
			for k, v := range ws.Environment {
				env[k] = v
			}
			for k, v := range t.Environment {
				env[k] = v
			}

			task := NewTask(workspace, t.Name, t.Executor, t.Command, env, t.Service, t.Stdout, t.Stderr, t.Metadata, t.Pwd)
			workspace.Tasks[t.Name] = task
		}
	}

	// Get the router
	r := app.Router

	// Create some routes
	r.Get("/", list_workspaces)
	r.Get("/log", show_log)
	r.Get("/workspace/(?P<workspace>[^/]+)", list_tasks)

	// Serve static content (but really use a CDN)
	r.Get("/images/(?P<file>.*)", r.Static("assets/images/{{file}}"))
	r.Get("/css/(?P<file>.*)", r.Static("assets/css/{{file}}"))

	r.Post("/workspace/(?P<workspace>[^/]+)/task/(?P<task>[^/]+)/start", startTask)
	r.Post("/workspace/(?P<workspace>[^/]+)/task/(?P<task>[^/]+)/stop", stopTask)
	r.Post("/workspace/(?P<workspace>[^/]+)/task/(?P<task>[^/]+)/enable", enableServiceTask)
	r.Post("/workspace/(?P<workspace>[^/]+)/task/(?P<task>[^/]+)/disable", disableServiceTask)
	r.Get("/workspace/(?P<workspace>[^/]+)/task/(?P<task>[^/]+)", taskHistory)

	r.Get("/workspace/(?P<workspace>[^/]+)/task/(?P<task>[^/]+)/run/(?P<run>\\d+)", taskRun)
	r.Get("/workspace/(?P<workspace>[^/]+)/task/(?P<task>[^/]+)/run/(?P<run>\\d+)/stdout", taskRunStdout)
	r.Get("/workspace/(?P<workspace>[^/]+)/task/(?P<task>[^/]+)/run/(?P<run>\\d+)/stderr", taskRunStderr)

	// Start our application
	app.Start()

	defer func() {
		for _, ws := range Workspaces {
			for _, t := range ws.Tasks {
				if t.ActiveTask != nil && t.ActiveTask.Cmd != nil && t.ActiveTask.Cmd.Process != nil {
					t.ActiveTask.Cmd.Process.Kill()
				}
			}
		}
	}()

	<-make(chan int)
}

func redir(session *http.Session) {
	redir := "/"

	if k := session.Request.Referer(); len(k) > 0 {
		redir = k
	}

	session.Redirect(&url.URL{Path: redir})
}

func startTask(session *http.Session) {
	ws, _ := session.Stash["workspace"].(string)
	id, _ := session.Stash["task"].(string)

	Workspaces[ws].Tasks[id].Start()

	redir(session)
}

func stopTask(session *http.Session) {
	ws, _ := session.Stash["workspace"].(string)
	id, _ := session.Stash["task"].(string)

	Workspaces[ws].Tasks[id].Stop()

	redir(session)
}

func enableServiceTask(session *http.Session) {
	ws, _ := session.Stash["workspace"].(string)
	id, _ := session.Stash["task"].(string)

	Workspaces[ws].Tasks[id].Service = true

	if Workspaces[ws].Tasks[id].ActiveTask == nil {
		Workspaces[ws].Tasks[id].Start()
	}

	redir(session)
}

func disableServiceTask(session *http.Session) {
	ws, _ := session.Stash["workspace"].(string)
	id, _ := session.Stash["task"].(string)

	Workspaces[ws].Tasks[id].Service = false

	redir(session)
}

func taskHistory(session *http.Session) {
	ws, _ := session.Stash["workspace"].(string)
	id, _ := session.Stash["task"].(string)

	session.Stash["Title"] = "Task"
	session.Stash["Page"] = "History"
	session.Stash["Workspace"] = Workspaces[ws]
	session.Stash["Task"] = Workspaces[ws].Tasks[id]

	session.RenderWithLayout("task.html", "layout.html", "Content")
}

func taskRun(session *http.Session) {
	ws, _ := session.Stash["workspace"].(string)
	id, _ := session.Stash["task"].(string)
	run, _ := strconv.Atoi(session.Stash["run"].(string))

	session.Stash["Title"] = "Task run"
	session.Stash["Page"] = "TaskRun"
	session.Stash["Workspace"] = Workspaces[ws]
	session.Stash["Task"] = Workspaces[ws].Tasks[id]
	session.Stash["TaskRun"] = Workspaces[ws].Tasks[id].TaskRuns[run]

	session.RenderWithLayout("taskrun.html", "layout.html", "Content")
}

func taskRunStdout(session *http.Session) {
	ws, _ := session.Stash["workspace"].(string)
	id, _ := session.Stash["task"].(string)
	run, _ := strconv.Atoi(session.Stash["run"].(string))

	session.Stash["Title"] = "Task run stdout"
	session.Stash["Page"] = "TaskOutput"
	session.Stash["Type"] = "stdout"
	session.Stash["Workspace"] = Workspaces[ws]
	session.Stash["Task"] = Workspaces[ws].Tasks[id]
	session.Stash["TaskRun"] = Workspaces[ws].Tasks[id].TaskRuns[run]
	session.Stash["LogOutput"] = Workspaces[ws].Tasks[id].TaskRuns[run].StdoutBuf.String()

	session.RenderWithLayout("log.html", "layout.html", "Content")
}

func taskRunStderr(session *http.Session) {
	ws, _ := session.Stash["workspace"].(string)
	id, _ := session.Stash["task"].(string)
	run, _ := strconv.Atoi(session.Stash["run"].(string))

	session.Stash["Title"] = "Task run stderr"
	session.Stash["Page"] = "TaskOutput"
	session.Stash["Type"] = "stderr"
	session.Stash["Workspace"] = Workspaces[ws]
	session.Stash["Task"] = Workspaces[ws].Tasks[id]
	session.Stash["TaskRun"] = Workspaces[ws].Tasks[id].TaskRuns[run]
	session.Stash["LogOutput"] = Workspaces[ws].Tasks[id].TaskRuns[run].StderrBuf.String()

	session.RenderWithLayout("log.html", "layout.html", "Content")
}

func list_workspaces(session *http.Session) {
	// Stash a value and render a template
	session.Stash["Title"] = "websysd"
	session.Stash["Page"] = "Workspaces"
	session.Stash["Workspaces"] = Workspaces
	session.RenderWithLayout("workspaces.html", "layout.html", "Content")
}

func list_tasks(session *http.Session) {
	ws, _ := session.Stash["workspace"].(string)

	// Stash a value and render a template
	session.Stash["Title"] = "websysd"
	session.Stash["Page"] = "Tasks"
	session.Stash["Workspace"] = Workspaces[ws]
	session.Stash["Tasks"] = Workspaces[ws].Tasks
	session.RenderWithLayout("tasks.html", "layout.html", "Content")
}

func show_log(session *http.Session) {
	session.Stash["Title"] = "websysd log"
	session.Stash["Page"] = "AppLog"
	session.Stash["LogOutput"] = applog.String()

	session.RenderWithLayout("applog.html", "layout.html", "Content")
}
<|endoftext|>


<commit_before>/*
 * main.go - or-ctl-filter
 *
 * To the extent possible under law, Yawning Angel has waived all copyright and
 * related or neighboring rights to or-ctl-filter, using the creative commons
 * "cc0" public domain dedication. See LICENSE or
 * <http://creativecommons.org/publicdomain/zero/1.0/> for full details.
 */

// or-ctl-filter is a Tor Control Port filter in the spirit of
// "control-port-filter" by the Whonix developers.  It is more limited as the
// only use case considered is "I want to run Tor Browser on my desktop with a
// system tor service and have 'about:tor' and 'New Identity' work while
// disallowing scary control port commands".  But on a positive note, it's not
// a collection of bash and doesn't call netcat.
package main

import (
	"flag"
	"io/ioutil"
	"log"
	"net"
	"os"

	"github.com/yawning/bulb/utils"
)

const (
	defaultLogFile     = "or-ctl-filter.log"
	defaultControlAddr = "unix:///var/run/tor/control"

	defaultFilteredAddr = "tcp://127.0.0.1:9151"
	torControlAddr      = "127.0.0.1:9151" // Match ControlPort in torrc-defaults.
)

func main() {
	enableLogging := flag.Bool("enable-logging", false, "enable logging")
	logFile := flag.String("log-file", defaultLogFile, "log file")
	controlAddr := flag.String("control-address", defaultControlAddr, "tor control port address")
	filteredAddr := flag.String("filtered-control-address", defaultFilteredAddr, "filtered control port address")
	flag.Parse()

	// Deal with logging.
	if !*enableLogging {
		log.SetOutput(ioutil.Discard)
	} else if *logFile != "" {
		f, err := os.OpenFile(*logFile, os.O_CREATE|os.O_APPEND|os.O_WRONLY, 0600)
		if err != nil {
			log.Fatalf("Failed to create log file: %s\n", err)
		}
		log.SetOutput(f)
	}

	// Initialize the listener
	fNet, fAddr, err := utils.ParseControlPortString(*filteredAddr)
	if err != nil {
		log.Fatalf("Failed to resolved filtere port: %s\n", err)
	}
	ln, err := net.Listen(fNet, fAddr)
	if err != nil {
		log.Fatalf("Failed to listen on the filter port: %s\n", err)
	}
	defer ln.Close()

	// Listen for incoming connections, and dispatch workers.
	for {
		conn, err := ln.Accept()
		if err != nil {
			log.Printf("Failed to Accept(): %s\n", err)
			continue
		}
		// TODO: Allow specifying password.
		s := newSession(conn, *controlAddr, "")
		go s.FilterSession()
	}
}
<commit_msg>Remove unused constant, shorten a command line arg.<commit_after>/*
 * main.go - or-ctl-filter
 *
 * To the extent possible under law, Yawning Angel has waived all copyright and
 * related or neighboring rights to or-ctl-filter, using the creative commons
 * "cc0" public domain dedication. See LICENSE or
 * <http://creativecommons.org/publicdomain/zero/1.0/> for full details.
 */

// or-ctl-filter is a Tor Control Port filter in the spirit of
// "control-port-filter" by the Whonix developers.  It is more limited as the
// only use case considered is "I want to run Tor Browser on my desktop with a
// system tor service and have 'about:tor' and 'New Identity' work while
// disallowing scary control port commands".  But on a positive note, it's not
// a collection of bash and doesn't call netcat.
package main

import (
	"flag"
	"io/ioutil"
	"log"
	"net"
	"os"

	"github.com/yawning/bulb/utils"
)

const (
	defaultLogFile      = "or-ctl-filter.log"
	defaultControlAddr  = "unix:///var/run/tor/control"
	defaultFilteredAddr = "tcp://127.0.0.1:9151"
)

func main() {
	enableLogging := flag.Bool("enable-logging", false, "enable logging")
	logFile := flag.String("log-file", defaultLogFile, "log file")
	controlAddr := flag.String("control-address", defaultControlAddr, "tor control port address")
	filteredAddr := flag.String("filtered-address", defaultFilteredAddr, "filtered control port address")
	flag.Parse()

	// Deal with logging.
	if !*enableLogging {
		log.SetOutput(ioutil.Discard)
	} else if *logFile != "" {
		f, err := os.OpenFile(*logFile, os.O_CREATE|os.O_APPEND|os.O_WRONLY, 0600)
		if err != nil {
			log.Fatalf("Failed to create log file: %s\n", err)
		}
		log.SetOutput(f)
	}

	// Initialize the listener
	fNet, fAddr, err := utils.ParseControlPortString(*filteredAddr)
	if err != nil {
		log.Fatalf("Failed to resolved filtere port: %s\n", err)
	}
	ln, err := net.Listen(fNet, fAddr)
	if err != nil {
		log.Fatalf("Failed to listen on the filter port: %s\n", err)
	}
	defer ln.Close()

	// Listen for incoming connections, and dispatch workers.
	for {
		conn, err := ln.Accept()
		if err != nil {
			log.Printf("Failed to Accept(): %s\n", err)
			continue
		}
		// TODO: Allow specifying password.
		s := newSession(conn, *controlAddr, "")
		go s.FilterSession()
	}
}
<|endoftext|>


<commit_before>// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package main

import (
	"fmt"
	"log"
	"net/http"
	"os"

	"github.com/line/line-bot-sdk-go/linebot"
)

var bot *linebot.Client

func main() {
	var err error
	bot, err = linebot.New(os.Getenv("ChannelSecret"), os.Getenv("ChannelAccessToken"))
	log.Println("Bot:", bot, " err:", err)
	http.HandleFunc("/callback", callbackHandler)
	port := os.Getenv("PORT")
	addr := fmt.Sprintf(":%s", port)
	http.ListenAndServe(addr, nil)
}

func callbackHandler(w http.ResponseWriter, r *http.Request) {
	events, err := bot.ParseRequest(r)

	if err != nil {
		if err == linebot.ErrInvalidSignature {
			w.WriteHeader(400)
		} else {
			w.WriteHeader(500)
		}
		return
	}

	for _, event := range events {
		if event.Type == linebot.EventTypeMessage {
			switch message := event.Message.(type) {
			case *linebot.TextMessage:
				if _, err = bot.ReplyMessage(event.ReplyToken, linebot.NewTextMessage(message.ID+":"+message.Text+" OK!")).Do(); err != nil {
					log.Print(err)
				}
			}
		}
	}
}
<commit_msg>no message<commit_after>// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package main

import (
	"fmt"
	"log"
	"net/http"
	"os"

	"github.com/line/line-bot-sdk-go/linebot"
)

var bot *linebot.Client

func main() {
	var err error
	bot, err = linebot.New(os.Getenv("ChannelSecret"), os.Getenv("ChannelAccessToken"))
	log.Println("Bot:", bot, " err:", err)
	http.HandleFunc("/callback", callbackHandler)
	port := os.Getenv("PORT")
	addr := fmt.Sprintf(":%s", port)
	http.ListenAndServe(addr, nil)
}

func callbackHandler(w http.ResponseWriter, r *http.Request) {
	events, err := bot.ParseRequest(r)

	if err != nil {
		if err == linebot.ErrInvalidSignature {
			w.WriteHeader(400)
		} else {
			w.WriteHeader(500)
		}
		return
	}

	for _, event := range events {
		if event.Type == linebot.EventTypeMessage {
			switch message := event.Message.(type) {
			case *linebot.TextMessage:
				if _, err = bot.ReplyMessage(event.ReplyToken, linebot.NewTextMessage(message.Text+" OK!")).Do(); err != nil {
					log.Print(err)
				}
			}
		}
	}
}
<|endoftext|>


<commit_before>package main

import (
	"os"
	"strings"
	"time"

	"gopkg.in/mgo.v2/bson"

	log "github.com/Sirupsen/logrus"
	"github.com/bmuller/arrow/lib"
	"github.com/brandfolder/gin-gorelic"
	"github.com/carlescere/scheduler"
	"github.com/dwarvesf/working-on/db"
	"github.com/gin-gonic/contrib/ginrus"
	"github.com/gin-gonic/gin"
	"github.com/nlopes/slack"
	"github.com/olebedev/config"
)

type Item struct {
	ID        bson.ObjectId `json:"id" bson:"_id"`
	UserID    string        `json:"user_id" bson:"user_id"`
	Name      string        `json:"user_name" bson:"user_name"`
	Text      string        `json:"text" bson:"text"`
	CreatedAt time.Time     `json:"created_at" bson:"created_at"`
}

func main() {

	// Read configuration from file and env
	port := os.Getenv("PORT")
	digestTime := os.Getenv("DIGEST_TIME")
	gorelic.InitNewrelicAgent(os.Getenv("NEW_RELIC_LICENSE_KEY"), "working", false)

	// Setup schedule jobs
	digestJob := postDigest
	scheduler.Every().Day().At(digestTime).Run(digestJob)

	// Prepare router
	router := gin.New()
	router.Use(gorelic.Handler)
	router.Use(ginrus.Ginrus(log.StandardLogger(), time.RFC3339, true))

	// router.LoadHTMLGlob("templates/*.tmpl.html")
	router.Static("/static", "static")
	router.POST("/on", addItem)

	// Start server
	router.Run(":" + port)
}

// Message will be passed to server with '-' prefix via various way
//	+ Direct message with the bots
//	+ Use slash command `/working <message>`
//	+ Use cli `working-on <message>`
//	+ Might use Chrome plugin
//	+ ...
// Token is secondary param to indicate the user
func addItem(c *gin.Context) {

	// Parse token and message
	var item Item

	text := c.PostForm("text")
	text = strings.TrimSpace(text)

	if text == "" {
		log.Fatalln("Message is nil")
		return
	}

	userID := c.PostForm("user_id")
	userName := c.PostForm("user_name")

	item.ID = bson.NewObjectId()
	item.CreatedAt = time.Now()
	item.Name = userName
	item.UserID = userID
	item.Text = text

	ctx, err := db.NewContext()
	if err != nil {
		panic(err)
	}

	defer ctx.Close()

	// Add Item to database
	err = ctx.C("items").Insert(item)
	if err != nil {
		log.Fatalln(err)
	}

	// Repost to the target channel
	channel := os.Getenv("WORKING_CHANNEL")
	botToken := os.Getenv("BOT_TOKEN")

	if botToken == "" {
		log.Fatal("No token provided")
		os.Exit(1)
	}

	// <@U024BE7LH|bob>
	title := "*" + userName + "* is working on: " + text

	postWorkingItem(botToken, channel, title)

	// Parse configuration
	cfg, err := config.ParseJsonFile("setting.json")
	// var configurations []Configuration
	configurations, err := cfg.List("items")

	// Post item to project group
	for _, config := range configurations {
		for _, tag := range config.Tags {
			if strings.Contains(text, tag) {
				// Post to target channel
				postWorkingItem(config.Token, config.Channel, title)
			}
		}
	}
}

func postWorkingItem(token string, channel string, text string) {
	s := slack.New(token)
	params := slack.PostMessageParameters{}
	params.IconURL = "http://i.imgur.com/fLcxkel.png"
	params.Username = "oshin"
	s.PostMessage(channel, text, params)
}

type Configuration struct {
	Channel string   `json:"channel"`
	Tags    []string `json:"tags"`
	Token   string   `json:"token"`
}

// Post summary to Slack channel
func postDigest() {

	channel := os.Getenv("DIGEST_CHANNEL")
	botToken := os.Getenv("BOT_TOKEN")

	if botToken == "" {
		log.Fatal("No token provided")
		os.Exit(1)
	}

	s := slack.New(botToken)
	users, err := s.GetUsers()

	if err != nil {
		log.Fatal("Cannot get users")
		os.Exit(1)
	}

	ctx, err := db.NewContext()
	if err != nil {
		panic(err)
	}

	defer ctx.Close()

	log.Info("Preparing data")
	// If count > 0, it means there is data to show
	count := 0
	title := " >> Yesterday I did: "
	params := slack.PostMessageParameters{}
	fields := []slack.AttachmentField{}

	yesterday := arrow.Yesterday().UTC()
	toDate := time.Date(yesterday.Year(), yesterday.Month(), yesterday.Day(), 0, 0, 0, 0, time.UTC)

	// Prepare attachment of done items
	for _, user := range users {

		if user.IsBot || user.Deleted {
			continue
		}

		// log.Info("Process user: " + user.Name + " - " + user.Id)

		// Query done items from Database
		var values string
		var items []Item

		err = ctx.C("items").Find(bson.M{"$and": []bson.M{
			bson.M{"user_id": user.Id},
			bson.M{"created_at": bson.M{"$gt": toDate}},
		},
		}).All(&items)

		if err != nil {
			log.Fatal("Cannot query done items.")
			os.Exit(1)
		}

		for _, item := range items {
			values = "\t" + values + " + " + item.Text + "\n"
		}

		// <@U024BE7LH|bob>
		if len(items) > 0 {

			count = count + 1
			field := slack.AttachmentField{
				Title: "@" + user.Name,
				Value: values,
			}

			fields = append(fields, field)
		}
	}

	params.Attachments = []slack.Attachment{
		slack.Attachment{
			Color:  "#7CD197",
			Fields: fields,
		},
	}

	params.IconURL = "http://i.imgur.com/fLcxkel.png"
	params.Username = "oshin"

	if count > 0 {
		s.PostMessage(channel, title, params)
	}
}
<commit_msg>Post message to project channel<commit_after>package main

import (
	"os"
	"strings"
	"time"

	"gopkg.in/mgo.v2/bson"

	log "github.com/Sirupsen/logrus"
	"github.com/bmuller/arrow/lib"
	"github.com/brandfolder/gin-gorelic"
	"github.com/carlescere/scheduler"
	"github.com/dwarvesf/working-on/db"
	"github.com/gin-gonic/contrib/ginrus"
	"github.com/gin-gonic/gin"
	"github.com/nlopes/slack"
	"github.com/olebedev/config"
)

type Item struct {
	ID        bson.ObjectId `json:"id" bson:"_id"`
	UserID    string        `json:"user_id" bson:"user_id"`
	Name      string        `json:"user_name" bson:"user_name"`
	Text      string        `json:"text" bson:"text"`
	CreatedAt time.Time     `json:"created_at" bson:"created_at"`
}

func main() {

	// Read configuration from file and env
	port := os.Getenv("PORT")
	digestTime := os.Getenv("DIGEST_TIME")
	gorelic.InitNewrelicAgent(os.Getenv("NEW_RELIC_LICENSE_KEY"), "working", false)

	// Setup schedule jobs
	digestJob := postDigest
	scheduler.Every().Day().At(digestTime).Run(digestJob)

	// Prepare router
	router := gin.New()
	router.Use(gorelic.Handler)
	router.Use(ginrus.Ginrus(log.StandardLogger(), time.RFC3339, true))

	// router.LoadHTMLGlob("templates/*.tmpl.html")
	router.Static("/static", "static")
	router.POST("/on", addItem)

	// Start server
	router.Run(":" + port)
}

// Message will be passed to server with '-' prefix via various way
//	+ Direct message with the bots
//	+ Use slash command `/working <message>`
//	+ Use cli `working-on <message>`
//	+ Might use Chrome plugin
//	+ ...
// Token is secondary param to indicate the user
func addItem(c *gin.Context) {

	// Parse token and message
	var item Item

	text := c.PostForm("text")
	text = strings.TrimSpace(text)

	if text == "" {
		log.Fatalln("Message is nil")
		return
	}

	userID := c.PostForm("user_id")
	userName := c.PostForm("user_name")

	item.ID = bson.NewObjectId()
	item.CreatedAt = time.Now()
	item.Name = userName
	item.UserID = userID
	item.Text = text

	ctx, err := db.NewContext()
	if err != nil {
		panic(err)
	}

	defer ctx.Close()

	// Add Item to database
	err = ctx.C("items").Insert(item)
	if err != nil {
		log.Fatalln(err)
	}

	// Repost to the target channel
	channel := os.Getenv("WORKING_CHANNEL")
	botToken := os.Getenv("BOT_TOKEN")

	if botToken == "" {
		log.Fatal("No token provided")
		os.Exit(1)
	}

	// <@U024BE7LH|bob>
	title := "*" + userName + "* is working on: " + text

	postWorkingItem(botToken, channel, title)

	// Parse configuration
	cfg, err := config.ParseJsonFile("setting.json")
	configurations, err := cfg.List("items")

	// Post item to project group
	for _, item := range configurations {
		config, ok := item.(Configuration)
		if ok {
			for _, tag := range config.Tags {
				if strings.Contains(text, "#"+tag) {
					// Post to target channel
					postWorkingItem(config.Token, config.Channel, title)
				}
			}
		}
	}
}

func postWorkingItem(token string, channel string, text string) {
	s := slack.New(token)
	params := slack.PostMessageParameters{}
	params.IconURL = "http://i.imgur.com/fLcxkel.png"
	params.Username = "oshin"
	s.PostMessage(channel, text, params)
}

type Configuration struct {
	Channel string   `json:"channel"`
	Tags    []string `json:"tags"`
	Token   string   `json:"token"`
}

// Post summary to Slack channel
func postDigest() {

	channel := os.Getenv("DIGEST_CHANNEL")
	botToken := os.Getenv("BOT_TOKEN")

	if botToken == "" {
		log.Fatal("No token provided")
		os.Exit(1)
	}

	s := slack.New(botToken)
	users, err := s.GetUsers()

	if err != nil {
		log.Fatal("Cannot get users")
		os.Exit(1)
	}

	ctx, err := db.NewContext()
	if err != nil {
		panic(err)
	}

	defer ctx.Close()

	log.Info("Preparing data")
	// If count > 0, it means there is data to show
	count := 0
	title := " >> Yesterday I did: "
	params := slack.PostMessageParameters{}
	fields := []slack.AttachmentField{}

	yesterday := arrow.Yesterday().UTC()
	toDate := time.Date(yesterday.Year(), yesterday.Month(), yesterday.Day(), 0, 0, 0, 0, time.UTC)

	// Prepare attachment of done items
	for _, user := range users {

		if user.IsBot || user.Deleted {
			continue
		}

		// log.Info("Process user: " + user.Name + " - " + user.Id)

		// Query done items from Database
		var values string
		var items []Item

		err = ctx.C("items").Find(bson.M{"$and": []bson.M{
			bson.M{"user_id": user.Id},
			bson.M{"created_at": bson.M{"$gt": toDate}},
		},
		}).All(&items)

		if err != nil {
			log.Fatal("Cannot query done items.")
			os.Exit(1)
		}

		for _, item := range items {
			values = "\t" + values + " + " + item.Text + "\n"
		}

		// <@U024BE7LH|bob>
		if len(items) > 0 {

			count = count + 1
			field := slack.AttachmentField{
				Title: "@" + user.Name,
				Value: values,
			}

			fields = append(fields, field)
		}
	}

	params.Attachments = []slack.Attachment{
		slack.Attachment{
			Color:  "#7CD197",
			Fields: fields,
		},
	}

	params.IconURL = "http://i.imgur.com/fLcxkel.png"
	params.Username = "oshin"

	if count > 0 {
		s.PostMessage(channel, title, params)
	}
}
<|endoftext|>


<commit_before>package main

import (
	"flag"
	"fmt"
	"os"
)

func main() {
	flag.Usage = usage
	flag.Parse()
	args := flag.Args()
	if len(args) < 1 {
		usage()
	}
}

func usage() {

	usage := `ERROR: You need to pass a sub-command (e.g., knife SUB-COMMAND)

Usage: knife sub-command (options)
    -s, --server-url URL             Chef Server URL
        --chef-zero-host HOST        Host to start chef-zero on
        --chef-zero-port PORT        Port (or port range) to start chef-zero on.  Port ranges like 1000,1010 or 8889-9999 will try all given ports until one works.
    -k, --key KEY                    API Client Key
        --[no-]color                 Use colored output, defaults to false on Windows, true otherwise
    -c, --config CONFIG              The configuration file to use
        --defaults                   Accept default values for all questions
    -d, --disable-editing            Do not open EDITOR, just accept the data as is
    -e, --editor EDITOR              Set the editor to use for interactive commands
    -E, --environment ENVIRONMENT    Set the Chef environment (except for in searches, where this will be flagrantly ignored)
    -F, --format FORMAT              Which format to use for output
        --[no-]listen                Whether a local mode (-z) server binds to a port
    -z, --local-mode                 Point knife commands at local repository instead of server
    -u, --user USER                  API Client Username
        --print-after                Show the data after a destructive operation
    -V, --verbose                    More verbose output. Use twice for max verbosity
    -v, --version                    Show chef version
    -y, --yes                        Say yes to all prompts for confirmation
    -h, --help                       Show this message

Available subcommands: (for details, knife SUB-COMMAND --help)
`
	fmt.Println(usage)
	os.Exit(1)
}
<commit_msg>add chef client config<commit_after>package main

import (
	"flag"
	"fmt"
	"github.com/go-chef/chef"
	"io/ioutil"
	"os"
)

var (
	client *chef.Client
)

func init() {
	//get the chef config from env params
	clientName := os.Getenv("CLIENT_NAME")
	clientKey := os.Getenv("CLIENT_KEY")
	baseURL := os.Getenv("CHEF_SERVER_URL")

	// fmt.Printf("\n Client name: %s\n Client Key: %s\n BaseURL: %s\n", clientName, clientKey, baseURL)

	// read the client key
	key, err := ioutil.ReadFile(clientKey)
	if err != nil {
		fmt.Println("Couldn't read key", err)
		os.Exit(1)
	}

	// build a client
	client, err = chef.NewClient(&chef.Config{
		Name:    clientName,
		Key:     string(key),
		BaseURL: baseURL,
		// SkipSSL: true,
	})
	if err != nil {
		fmt.Println("error setting up client", err)
		os.Exit(1)
	}
}

func main() {
	flag.Usage = usage
	flag.Parse()
	args := flag.Args()
	if len(args) < 1 {
		usage()
		os.Exit(1)
	}

}

func usage() {

	usageMSG := `ERROR: You need to pass a sub-command (e.g., knife SUB-COMMAND)

Usage: knife sub-command (options)
    -s, --server-url URL             Chef Server URL
        --chef-zero-host HOST        Host to start chef-zero on
        --chef-zero-port PORT        Port (or port range) to start chef-zero on.  Port ranges like 1000,1010 or 8889-9999 will try all given ports until one works.
    -k, --key KEY                    API Client Key
        --[no-]color                 Use colored output, defaults to false on Windows, true otherwise
    -c, --config CONFIG              The configuration file to use
        --defaults                   Accept default values for all questions
    -d, --disable-editing            Do not open EDITOR, just accept the data as is
    -e, --editor EDITOR              Set the editor to use for interactive commands
    -E, --environment ENVIRONMENT    Set the Chef environment (except for in searches, where this will be flagrantly ignored)
    -F, --format FORMAT              Which format to use for output
        --[no-]listen                Whether a local mode (-z) server binds to a port
    -z, --local-mode                 Point knife commands at local repository instead of server
    -u, --user USER                  API Client Username
        --print-after                Show the data after a destructive operation
    -V, --verbose                    More verbose output. Use twice for max verbosity
    -v, --version                    Show chef version
    -y, --yes                        Say yes to all prompts for confirmation
    -h, --help                       Show this message

Available subcommands: (for details, knife SUB-COMMAND --help)
`
	fmt.Println(usageMSG)
}
<|endoftext|>


<commit_before>package main

import (
	"encoding/json"
	"flag"
	"github.com/Bpazy/wl520/welove"
	"github.com/bitly/go-simplejson"
	"github.com/ltt1987/alidayu"
	"io"
	"io/ioutil"
	"log"
	"os"
	"reflect"
	"strconv"
	"sync"
)

var w sync.WaitGroup

func main() {
	isServer := flag.Bool("s", false, "启动我们的家HTTP代理")
	path := flag.String("out", "wl520.json", "生成的配置文件路径")
	port := flag.String("port", ":8080", "我们的家Http代理端口号")
	allTask := flag.Bool("a", false, "完成所有我们的家互动任务")
	configPath := flag.String("c", "wl520.json", "配置文件位置")
	visitTimes := flag.Int("v", -1, "每日拜访次数")
	outputPath := flag.String("log", "welove.log", "日志路径")
	tree := flag.Bool("t", false, "完成爱情树任务")
	pet := flag.Bool("p", false, "完成宠物任务")
	buyItemId := flag.Int("buy", 0, "农场购买物品ID")
	coin := flag.Int("coin", -1, "农场被购买物品ID的价格上限(闭区间)")
	farmSign := flag.Bool("farm-sign", false, "农场签到")
	smsNotify := flag.Bool("sms-notify", false, "检查签到情况并发送短信")
	flag.Parse()

	welove.ServerRun(*path, *port, *isServer)    // 是否开启代理服务器
	love := initConfig(*outputPath, *configPath) // 读取配置文件

	log.Println("wl520 start.")
	goFunc(buyItem, love, *buyItemId, *coin, *buyItemId) // 购买指定物品
	goFunc(doAllTasks, love, *allTask)                   // 完成互动任务
	goFunc(doVisit, *visitTimes, love)                   // 拜访任务
	goFunc(doTreePost, love, *tree)                      // 爱情树任务
	goFunc(doPetTasks, love, *pet)                       // 宠物任务
	goFunc(doFarmSign, love, *farmSign)                  // 农场签到
	goFunc(doSmsNotify, love, *smsNotify)                // 检查签到情况并发送短信
	w.Wait()
	log.Println("wl520 end.")

}

func goFunc(f interface{}, args ...interface{}) {
	v := reflect.ValueOf(f)
	var a []reflect.Value
	for _, v := range args {
		a = append(a, reflect.ValueOf(v))
	}
	w.Add(1)
	go v.Call(a)
}

func doSmsNotify(love welove.Love, smsNotify bool) {
	defer w.Done()
	if !smsNotify {
		return
	} else {
		alidayu.AppKey = love.DayuAppKey
		alidayu.AppSecret = love.DayuAppSecret
	}
	loveTreeSms(love)
}

func loveTreeSms(love welove.Love) {
	bytes, err := welove.QueryTreeInfo(love.AccessToken, love.AppKey)
	if err != nil {
		log.Fatal(err)
	}
	js, _ := simplejson.NewJson(bytes)
	var loverLackWater, _ = js.Get("lover_lack_water").Int()
	var loverLackSunlight, _ = js.Get("lover_lack_sunlight").Int()
	var lackWater, _ = js.Get("lack_water").Int()
	var lackSunlight, _ = js.Get("lack_sunlight").Int()
	if loverLackSunlight == 1 || loverLackWater == 1 {
		success, resp := alidayu.SendSMS(love.LoverMobile, "ZY通知", love.DayuSunlightTemplateCode, `{"name":"女主人"}`)
		log.Println(success, resp)
	}
	if lackWater == 1 || lackSunlight == 1 {
		success, resp := alidayu.SendSMS(love.Mobile, "ZY通知", love.DayuSunlightTemplateCode, `{"name":"主人"}`)
		log.Println(success, resp)
	}
}

func doFarmSign(love welove.Love, do bool) {
	defer w.Done()
	if !do {
		return
	}
	res, err := welove.FarmSign(love)
	if err != nil {
		log.Fatal(err)
	}
	defer res.Body.Close()
	bytes, _ := ioutil.ReadAll(res.Body)
	log.Printf("农场签到: %s\n", string(bytes))
}

func buyItem(love welove.Love, itemId, coin, buyItemId int) {
	defer w.Done()
	if buyItemId == 0 {
		return
	}
	items := welove.QueryItems(love.AccessToken).Messages[0].AdItems
	for _, v := range items {
		if v.ItemID == itemId && v.Coin <= coin {
			status := welove.BuyItem(love.AccessToken, v.SellerFarmID, v.ID)
			log.Printf("农场购买result: %d, Raw: %+v\n", status.Result, status)
		}
	}
}

func doTreePost(love welove.Love, tree bool) {
	defer w.Done()
	if !tree {
		return
	}
	op := []int{1, 2}
	for _, v := range op {
		res, err := welove.TreePost(love.AccessToken, love.AppKey, v)
		if err != nil {
			log.Fatal(err)
		}
		bytes, _ := ioutil.ReadAll(res.Body)
		js, _ := simplejson.NewJson(bytes)
		result, _ := js.Get("result").Int()
		log.Printf("爱情树result: %d, Raw: %s\n", result, string(bytes))
	}
}

func initConfig(outputPath, configPath string) welove.Love {
	setLogFile(outputPath)
	return readConfig(configPath)
}

//读取配置文件
func readConfig(configPath string) welove.Love {
	bytes, err := ioutil.ReadFile(configPath)
	if err != nil {
		log.Fatal(err)
	}
	love := welove.Love{}
	json.Unmarshal(bytes, &love)
	return love
}

//配置日志
func setLogFile(outputPath string) {
	var file, err = os.OpenFile(outputPath, os.O_APPEND|os.O_CREATE|os.O_RDWR, os.ModeAppend)
	if err != nil {
		log.Fatal(err)
	}
	log.SetOutput(io.MultiWriter(os.Stdout, file))
}

func doVisit(visitTimes int, love welove.Love) {
	defer w.Done()
	if visitTimes == -1 {
		return
	}
	for i := 0; i < visitTimes; i++ {
		if id, ok := welove.RandomHouse(love.AccessToken); ok {
			res, err := welove.Visit(love.AccessToken, id)
			if err != nil {
				log.Fatal(err)
			}
			bytes, _ := ioutil.ReadAll(res.Body)
			js, _ := simplejson.NewJson(bytes)
			result, _ := js.Get("result").Int()
			log.Printf("拜访result: %d, Raw: %s\n", result, string(bytes))
		}
	}
}

func doAllTasks(love welove.Love, allTask bool) {
	defer w.Done()
	if !allTask {
		return
	}
	res, err := welove.GetLoveSpaceIdRaw(love.AccessToken, love.AppKey)
	if err != nil {
		log.Fatal(err)
	}
	bytes, _ := ioutil.ReadAll(res.Body)
	loveSpaceId := welove.GetLoveSpaceId(string(bytes))
	taskTypes := []int{1, 4, 5, 6, 7, 11}
	for _, v := range taskTypes {
		res, err := welove.HomePost(love.AccessToken, v, loveSpaceId)
		if err != nil {
			log.Printf("任务%d错误\n", v)
		}
		bytes, _ := ioutil.ReadAll(res.Body)
		m := make(map[string]interface{})
		json.Unmarshal(bytes, &m)
		log.Printf("任务%d, Raw: %s\n", v, string(bytes))
	}
}

func doPetTasks(love welove.Love, doPet bool) {
	defer w.Done()
	if !doPet {
		return
	}
	petStatus := welove.GetPetStatus(love.AccessToken)
	log.Printf("宠物状态Raw: %+v\n", petStatus)
	pet := petStatus.Messages[0].Pets[0]
	for _, v := range pet.PetTasks {
		if v.RemainTime != 0 {
			continue
		}
		taskResult := welove.DoPetTask(love.AccessToken, strconv.Itoa(pet.PetID), strconv.Itoa(v.TaskType))
		log.Printf("宠物任务%d, Raw: %+v\n", v.TaskType, taskResult)
	}
}
<commit_msg>extract commandline config<commit_after>package main

import (
	"encoding/json"
	"flag"
	"github.com/Bpazy/wl520/welove"
	"github.com/bitly/go-simplejson"
	"github.com/ltt1987/alidayu"
	"io"
	"io/ioutil"
	"log"
	"os"
	"reflect"
	"strconv"
	"sync"
)

var w sync.WaitGroup

var (
	isServer   *bool
	path       *string
	port       *string
	allTask    *bool
	configPath *string
	visitTimes *int
	outputPath *string
	tree       *bool
	pet        *bool
	buyItemId  *int
	coin       *int
	farmSign   *bool
	smsNotify  *bool
)

func init() {
	isServer = flag.Bool("s", false, "启动我们的家HTTP代理")
	path = flag.String("out", "wl520.json", "生成的配置文件路径")
	port = flag.String("port", ":8080", "我们的家Http代理端口号")
	allTask = flag.Bool("a", false, "完成所有我们的家互动任务")
	configPath = flag.String("c", "wl520.json", "配置文件位置")
	visitTimes = flag.Int("v", -1, "每日拜访次数")
	outputPath = flag.String("log", "welove.log", "日志路径")
	tree = flag.Bool("t", false, "完成爱情树任务")
	pet = flag.Bool("p", false, "完成宠物任务")
	buyItemId = flag.Int("buy", 0, "农场购买物品ID")
	coin = flag.Int("coin", -1, "农场被购买物品ID的价格上限(闭区间)")
	farmSign = flag.Bool("farm-sign", false, "农场签到")
	smsNotify = flag.Bool("sms-notify", false, "检查签到情况并发送短信")
	flag.Parse()
}

func main() {

	welove.ServerRun(*path, *port, *isServer)    // 是否开启代理服务器
	love := initConfig(*outputPath, *configPath) // 读取配置文件

	log.Println("wl520 start.")
	goFunc(buyItem, love, *buyItemId, *coin, *buyItemId) // 购买指定物品
	goFunc(doAllTasks, love, *allTask)                   // 完成互动任务
	goFunc(doVisit, *visitTimes, love)                   // 拜访任务
	goFunc(doTreePost, love, *tree)                      // 爱情树任务
	goFunc(doPetTasks, love, *pet)                       // 宠物任务
	goFunc(doFarmSign, love, *farmSign)                  // 农场签到
	goFunc(doSmsNotify, love, *smsNotify)                // 检查签到情况并发送短信
	w.Wait()
	log.Println("wl520 end.")

}

func goFunc(f interface{}, args ...interface{}) {
	v := reflect.ValueOf(f)
	var a []reflect.Value
	for _, v := range args {
		a = append(a, reflect.ValueOf(v))
	}
	w.Add(1)
	go v.Call(a)
}

func doSmsNotify(love welove.Love, smsNotify bool) {
	defer w.Done()
	if !smsNotify {
		return
	} else {
		alidayu.AppKey = love.DayuAppKey
		alidayu.AppSecret = love.DayuAppSecret
	}
	loveTreeSms(love)
}

func loveTreeSms(love welove.Love) {
	bytes, err := welove.QueryTreeInfo(love.AccessToken, love.AppKey)
	if err != nil {
		log.Fatal(err)
	}
	js, _ := simplejson.NewJson(bytes)
	var loverLackWater, _ = js.Get("lover_lack_water").Int()
	var loverLackSunlight, _ = js.Get("lover_lack_sunlight").Int()
	var lackWater, _ = js.Get("lack_water").Int()
	var lackSunlight, _ = js.Get("lack_sunlight").Int()
	if loverLackSunlight == 1 || loverLackWater == 1 {
		success, resp := alidayu.SendSMS(love.LoverMobile, "ZY通知", love.DayuSunlightTemplateCode, `{"name":"女主人"}`)
		log.Println(success, resp)
	}
	if lackWater == 1 || lackSunlight == 1 {
		success, resp := alidayu.SendSMS(love.Mobile, "ZY通知", love.DayuSunlightTemplateCode, `{"name":"主人"}`)
		log.Println(success, resp)
	}
}

func doFarmSign(love welove.Love, do bool) {
	defer w.Done()
	if !do {
		return
	}
	res, err := welove.FarmSign(love)
	if err != nil {
		log.Fatal(err)
	}
	defer res.Body.Close()
	bytes, _ := ioutil.ReadAll(res.Body)
	log.Printf("农场签到: %s\n", string(bytes))
}

func buyItem(love welove.Love, itemId, coin, buyItemId int) {
	defer w.Done()
	if buyItemId == 0 {
		return
	}
	items := welove.QueryItems(love.AccessToken).Messages[0].AdItems
	for _, v := range items {
		if v.ItemID == itemId && v.Coin <= coin {
			status := welove.BuyItem(love.AccessToken, v.SellerFarmID, v.ID)
			log.Printf("农场购买result: %d, Raw: %+v\n", status.Result, status)
		}
	}
}

func doTreePost(love welove.Love, tree bool) {
	defer w.Done()
	if !tree {
		return
	}
	op := []int{1, 2}
	for _, v := range op {
		res, err := welove.TreePost(love.AccessToken, love.AppKey, v)
		if err != nil {
			log.Fatal(err)
		}
		bytes, _ := ioutil.ReadAll(res.Body)
		js, _ := simplejson.NewJson(bytes)
		result, _ := js.Get("result").Int()
		log.Printf("爱情树result: %d, Raw: %s\n", result, string(bytes))
	}
}

func initConfig(outputPath, configPath string) welove.Love {
	setLogFile(outputPath)
	return readConfig(configPath)
}

//读取配置文件
func readConfig(configPath string) welove.Love {
	bytes, err := ioutil.ReadFile(configPath)
	if err != nil {
		log.Fatal(err)
	}
	love := welove.Love{}
	json.Unmarshal(bytes, &love)
	return love
}

//配置日志
func setLogFile(outputPath string) {
	var file, err = os.OpenFile(outputPath, os.O_APPEND|os.O_CREATE|os.O_RDWR, os.ModeAppend)
	if err != nil {
		log.Fatal(err)
	}
	log.SetOutput(io.MultiWriter(os.Stdout, file))
}

func doVisit(visitTimes int, love welove.Love) {
	defer w.Done()
	if visitTimes == -1 {
		return
	}
	for i := 0; i < visitTimes; i++ {
		if id, ok := welove.RandomHouse(love.AccessToken); ok {
			res, err := welove.Visit(love.AccessToken, id)
			if err != nil {
				log.Fatal(err)
			}
			bytes, _ := ioutil.ReadAll(res.Body)
			js, _ := simplejson.NewJson(bytes)
			result, _ := js.Get("result").Int()
			log.Printf("拜访result: %d, Raw: %s\n", result, string(bytes))
		}
	}
}

func doAllTasks(love welove.Love, allTask bool) {
	defer w.Done()
	if !allTask {
		return
	}
	res, err := welove.GetLoveSpaceIdRaw(love.AccessToken, love.AppKey)
	if err != nil {
		log.Fatal(err)
	}
	bytes, _ := ioutil.ReadAll(res.Body)
	loveSpaceId := welove.GetLoveSpaceId(string(bytes))
	taskTypes := []int{1, 4, 5, 6, 7, 11}
	for _, v := range taskTypes {
		res, err := welove.HomePost(love.AccessToken, v, loveSpaceId)
		if err != nil {
			log.Printf("任务%d错误\n", v)
		}
		bytes, _ := ioutil.ReadAll(res.Body)
		m := make(map[string]interface{})
		json.Unmarshal(bytes, &m)
		log.Printf("任务%d, Raw: %s\n", v, string(bytes))
	}
}

func doPetTasks(love welove.Love, doPet bool) {
	defer w.Done()
	if !doPet {
		return
	}
	petStatus := welove.GetPetStatus(love.AccessToken)
	log.Printf("宠物状态Raw: %+v\n", petStatus)
	pet := petStatus.Messages[0].Pets[0]
	for _, v := range pet.PetTasks {
		if v.RemainTime != 0 {
			continue
		}
		taskResult := welove.DoPetTask(love.AccessToken, strconv.Itoa(pet.PetID), strconv.Itoa(v.TaskType))
		log.Printf("宠物任务%d, Raw: %+v\n", v.TaskType, taskResult)
	}
}
<|endoftext|>


<commit_before>package main

import (
	"encoding/csv"
	"fmt"
	"log"
	"os"
	"strings"
)

const (
	SELLWIRE_TRANSACTION_COLUMN_STATUS=16
	SELLWIRE_TRANSACTION_COLUMN_TRANSACTION_ID=2
	SELLWIRE_TRANSACTION_COLUMN_TIMESTAMP=12
	SELLWIRE_TRANSACTION_COLUMN_CUSTOMER_NAME=3
	SELLWIRE_TRANSACTION_COLUMN_CUSTOMER_AMOUNT=7
	SELLWIRE_TRANSACTION_COLUMN_CUSTOMER_IS_EU=9
	SELLWIRE_TRANSACTION_COLUMN_CUSTOMER_COUNTRY_CODE=10
	SELLWIRE_TRANSACTION_COLUMN_CUSTOMER_TAX_NUMBER=11
)

type SellwireTransaction struct {
	TransactionId string
	Date string
	CustomerName string
	Amount string
	IsEU string
	CountryCode string
	TaxNumber string
}

func main() {
	file, err := os.Open("SellwireOrders_All_23_Feb_2016_30_Sep_2016.csv")
	if err != nil {
		log.Fatal(err)
	}
	r := csv.NewReader(file)

	records, err := r.ReadAll()
	if err != nil {
		log.Fatal(err)
	}

	var paypalTransactions []SellwireTransaction

	for _, record := range records {
		status := record[SELLWIRE_TRANSACTION_COLUMN_STATUS]
		if status != "complete" {
			// TODO do we need to handle refunded?
			continue
		}
		timestamp := record[SELLWIRE_TRANSACTION_COLUMN_TIMESTAMP]
		timestampParts := strings.Split(timestamp, " ")
		sellwireRecord := SellwireTransaction{
			TransactionId: record[SELLWIRE_TRANSACTION_COLUMN_TRANSACTION_ID],
			Date: timestampParts[0],
			CustomerName: record[SELLWIRE_TRANSACTION_COLUMN_CUSTOMER_NAME],
			Amount: record[SELLWIRE_TRANSACTION_COLUMN_CUSTOMER_AMOUNT],
			IsEU: record[SELLWIRE_TRANSACTION_COLUMN_CUSTOMER_IS_EU],
			CountryCode: record[SELLWIRE_TRANSACTION_COLUMN_CUSTOMER_COUNTRY_CODE],
			TaxNumber: record[SELLWIRE_TRANSACTION_COLUMN_CUSTOMER_TAX_NUMBER],
		}

		if !strings.HasPrefix(sellwireRecord.TransactionId, "ch_") {
			paypalTransactions = append(paypalTransactions, sellwireRecord)
			fmt.Print(sellwireRecord)
		}
	}
}<commit_msg>Read test file from input folder<commit_after>package main

import (
	"encoding/csv"
	"fmt"
	"log"
	"os"
	"strings"
)

const (
	SELLWIRE_TRANSACTION_COLUMN_STATUS=16
	SELLWIRE_TRANSACTION_COLUMN_TRANSACTION_ID=2
	SELLWIRE_TRANSACTION_COLUMN_TIMESTAMP=12
	SELLWIRE_TRANSACTION_COLUMN_CUSTOMER_NAME=3
	SELLWIRE_TRANSACTION_COLUMN_CUSTOMER_AMOUNT=7
	SELLWIRE_TRANSACTION_COLUMN_CUSTOMER_IS_EU=9
	SELLWIRE_TRANSACTION_COLUMN_CUSTOMER_COUNTRY_CODE=10
	SELLWIRE_TRANSACTION_COLUMN_CUSTOMER_TAX_NUMBER=11
)

type SellwireTransaction struct {
	TransactionId string
	Date string
	CustomerName string
	Amount string
	IsEU string
	CountryCode string
	TaxNumber string
}

func main() {
	file, err := os.Open("input/SellwireOrders_All_23_Feb_2016_30_Sep_2016.csv")
	if err != nil {
		log.Fatal(err)
	}
	r := csv.NewReader(file)

	records, err := r.ReadAll()
	if err != nil {
		log.Fatal(err)
	}

	var paypalTransactions []SellwireTransaction

	for _, record := range records {
		status := record[SELLWIRE_TRANSACTION_COLUMN_STATUS]
		if status != "complete" {
			// TODO do we need to handle refunded?
			continue
		}
		timestamp := record[SELLWIRE_TRANSACTION_COLUMN_TIMESTAMP]
		timestampParts := strings.Split(timestamp, " ")
		sellwireRecord := SellwireTransaction{
			TransactionId: record[SELLWIRE_TRANSACTION_COLUMN_TRANSACTION_ID],
			Date: timestampParts[0],
			CustomerName: record[SELLWIRE_TRANSACTION_COLUMN_CUSTOMER_NAME],
			Amount: record[SELLWIRE_TRANSACTION_COLUMN_CUSTOMER_AMOUNT],
			IsEU: record[SELLWIRE_TRANSACTION_COLUMN_CUSTOMER_IS_EU],
			CountryCode: record[SELLWIRE_TRANSACTION_COLUMN_CUSTOMER_COUNTRY_CODE],
			TaxNumber: record[SELLWIRE_TRANSACTION_COLUMN_CUSTOMER_TAX_NUMBER],
		}

		if !strings.HasPrefix(sellwireRecord.TransactionId, "ch_") {
			paypalTransactions = append(paypalTransactions, sellwireRecord)
			fmt.Print(sellwireRecord)
		}
	}
}<|endoftext|>


<commit_before>package s3

import (
	"fmt"
	"io"
	"os"

	"github.com/aws/aws-sdk-go/aws"
	"github.com/aws/aws-sdk-go/aws/session"
	"github.com/aws/aws-sdk-go/service/s3"
	"github.com/codegangsta/cli"
)

var (
	parallel     int
	dryRun       bool
	delete       bool
	public       bool
	quiet        bool
	ignoreErrors bool
	acl          string
)

var ValidACLs = map[string]bool{
	"private":                   true,
	"public-read":               true,
	"public-read-write":         true,
	"authenticated-read":        true,
	"bucket-owner-read":         true,
	"bucket-owner-full-control": true,
	"log-delivery-write":        true,
}

func validACL() bool {
	if acl != "" && !ValidACLs[acl] {
		fmt.Fprintln(os.Stderr, "acl should be one of: private, public-read, public-read-write, authenticated-read, bucket-owner-read, bucket-owner-full-control, log-delivery-write")
		return false
	}
	return true
}

func Main(conn S3er, args []string, output io.Writer) int {
	out = output
	exitCode := 0

	getConnection := func(c *cli.Context) S3er {
		if conn == nil {
			region := c.String("region")
			if region == "" {
				region = "us-east-1"
			}
			config := aws.Config{
				Region: aws.String(region),
			}
			conn = s3.New(session.New(), &config)
		}
		return conn
	}

	commonFlags := []cli.Flag{
		cli.IntFlag{
			Name:        "p",
			Value:       32,
			Usage:       "number of parallel operations to run",
			Destination: &parallel,
		},
		cli.BoolFlag{
			Name:        "n",
			Usage:       "dry-run, no actions taken",
			Destination: &dryRun,
		},
		cli.BoolFlag{
			Name:        "ignore-errors",
			Usage:       "",
			Destination: &ignoreErrors,
		},
		cli.BoolFlag{
			Name:        "q",
			Usage:       "",
			Destination: &quiet,
		},
		cli.StringFlag{
			Name:   "region",
			Usage:  "set region, otherwise environment variable AWS_REGION is checked, finally defaulting to us-east-1",
			EnvVar: "AWS_REGION",
		},
	}

	aclFlag := cli.StringFlag{
		Name:        "acl",
		Usage:       "set acl to one of: private, public-read, public-read-write, authenticated-read, bucket-owner-read, bucket-owner-full-control, log-delivery-write",
		Destination: &acl,
	}
	publicFlag := cli.BoolFlag{
		Name:        "public, P",
		Usage:       "",
		Destination: &public,
	}
	deleteFlag := cli.BoolFlag{
		Name:        "delete",
		Usage:       "delete extraneous files from destination",
		Destination: &delete,
	}

	app := cli.NewApp()
	app.Name = "s3"
	app.Usage = "S3 utility knife"
	app.Version = "0.0.1"
	app.Flags = commonFlags
	app.Commands = []cli.Command{
		{
			Name:      "cat",
			Usage:     "Cat key contents",
			ArgsUsage: "key ...",
			Flags:     commonFlags,
			Action: func(c *cli.Context) {
				if len(c.Args()) == 0 {
					cli.ShowCommandHelp(c, "cat")
					exitCode = 1
					return
				}
				conn := getConnection(c)
				catKeys(conn, c.Args())
			},
		},
		{
			Name:      "get",
			Usage:     "Download keys",
			ArgsUsage: "key ...",
			Action: func(c *cli.Context) {
				if len(c.Args()) == 0 {
					cli.ShowCommandHelp(c, "get")
					exitCode = 1
					return
				}
				conn := getConnection(c)
				getKeys(conn, c.Args())
			},
		},
		{
			Name:      "grep",
			Usage:     "Grep keys",
			ArgsUsage: "string key ...",
			Action: func(c *cli.Context) {
				if len(c.Args()) == 0 {
					cli.ShowCommandHelp(c, "grep")
					exitCode = 1
					return
				}
				conn := getConnection(c)
				grepKeys(conn, c.Args())
			},
		},
		{
			Name:      "ls",
			Usage:     "List buckets or keys",
			ArgsUsage: "[bucket]",
			Action: func(c *cli.Context) {
				if len(c.Args()) < 1 {
					conn := getConnection(c)
					listBuckets(conn)
				} else {
					conn := getConnection(c)
					listKeys(conn, c.Args())
				}
			},
		},
		{
			Name:      "mb",
			Usage:     "Create bucket",
			ArgsUsage: "bucket",
			Action: func(c *cli.Context) {
				if len(c.Args()) != 1 {
					cli.ShowCommandHelp(c, "mb")
					exitCode = 1
					return
				}
				conn := getConnection(c)
				putBuckets(conn, c.Args())
			},
		},
		{
			Name:      "put",
			Usage:     "Upload files",
			ArgsUsage: "source [source ...] dest",
			Flags:     []cli.Flag{aclFlag, publicFlag},
			Action: func(c *cli.Context) {
				if len(c.Args()) < 2 {
					cli.ShowCommandHelp(c, "put")
					exitCode = 1
					return
				}
				if public {
					acl = "public-read"
				}
				if !validACL() {
					exitCode = 1
					return
				}
				conn := getConnection(c)
				putKeys(conn, c.Args())
			},
		},
		{
			Name:      "rb",
			Usage:     "Remove bucket(s)",
			ArgsUsage: "bucket ...",
			Action: func(c *cli.Context) {
				if len(c.Args()) == 0 {
					cli.ShowCommandHelp(c, "rb")
					exitCode = 1
					return
				}
				conn := getConnection(c)
				rmBuckets(conn, c.Args())
			},
		},
		{
			Name:      "rm",
			Usage:     "Remove keys",
			ArgsUsage: "key ...",
			Action: func(c *cli.Context) {
				if len(c.Args()) == 0 {
					cli.ShowCommandHelp(c, "rm")
					exitCode = 1
					return
				}
				conn := getConnection(c)
				rmKeys(conn, c.Args())
			},
		},
		{
			Name:      "sync",
			Usage:     "Synchronise local to s3, s3 to s3 or s3 to local",
			ArgsUsage: "source dest",
			Flags:     []cli.Flag{aclFlag, publicFlag, deleteFlag},
			Action: func(c *cli.Context) {
				if len(c.Args()) != 2 {
					cli.ShowCommandHelp(c, "sync")
					exitCode = 1
					return
				}
				if public {
					acl = "public-read"
				}
				if !validACL() {
					exitCode = 1
					return
				}
				conn := getConnection(c)
				syncFiles(conn, c.Args())
			},
		},
	}
	app.Run(args)
	return exitCode
}
<commit_msg>Fix region<commit_after>package s3

import (
	"fmt"
	"io"
	"os"

	"github.com/aws/aws-sdk-go/aws"
	"github.com/aws/aws-sdk-go/aws/session"
	"github.com/aws/aws-sdk-go/service/s3"
	"github.com/codegangsta/cli"
)

var (
	parallel     int
	dryRun       bool
	delete       bool
	public       bool
	quiet        bool
	ignoreErrors bool
	acl          string
)

var ValidACLs = map[string]bool{
	"private":                   true,
	"public-read":               true,
	"public-read-write":         true,
	"authenticated-read":        true,
	"bucket-owner-read":         true,
	"bucket-owner-full-control": true,
	"log-delivery-write":        true,
}

func validACL() bool {
	if acl != "" && !ValidACLs[acl] {
		fmt.Fprintln(os.Stderr, "acl should be one of: private, public-read, public-read-write, authenticated-read, bucket-owner-read, bucket-owner-full-control, log-delivery-write")
		return false
	}
	return true
}

func Main(conn S3er, args []string, output io.Writer) int {
	out = output
	exitCode := 0

	getConnection := func(c *cli.Context) S3er {
		if conn == nil {
			region := c.Parent().String("region")
			config := aws.Config{
				Region: aws.String(region),
			}
			conn = s3.New(session.New(), &config)
		}
		return conn
	}

	commonFlags := []cli.Flag{
		cli.IntFlag{
			Name:        "p",
			Value:       32,
			Usage:       "number of parallel operations to run",
			Destination: &parallel,
		},
		cli.BoolFlag{
			Name:        "n",
			Usage:       "dry-run, no actions taken",
			Destination: &dryRun,
		},
		cli.BoolFlag{
			Name:        "ignore-errors",
			Usage:       "",
			Destination: &ignoreErrors,
		},
		cli.BoolFlag{
			Name:        "q",
			Usage:       "",
			Destination: &quiet,
		},
		cli.StringFlag{
			Name:   "region",
			Usage:  "set region, otherwise environment variable AWS_REGION is checked, finally defaulting to us-east-1",
			Value:  "us-east-1",
			EnvVar: "AWS_REGION",
		},
	}

	aclFlag := cli.StringFlag{
		Name:        "acl",
		Usage:       "set acl to one of: private, public-read, public-read-write, authenticated-read, bucket-owner-read, bucket-owner-full-control, log-delivery-write",
		Destination: &acl,
	}
	publicFlag := cli.BoolFlag{
		Name:        "public, P",
		Usage:       "",
		Destination: &public,
	}
	deleteFlag := cli.BoolFlag{
		Name:        "delete",
		Usage:       "delete extraneous files from destination",
		Destination: &delete,
	}

	app := cli.NewApp()
	app.Name = "s3"
	app.Usage = "S3 utility knife"
	app.Version = "0.0.1"
	app.Flags = commonFlags
	app.Commands = []cli.Command{
		{
			Name:      "cat",
			Usage:     "Cat key contents",
			ArgsUsage: "key ...",
			Flags:     commonFlags,
			Action: func(c *cli.Context) {
				if len(c.Args()) == 0 {
					cli.ShowCommandHelp(c, "cat")
					exitCode = 1
					return
				}
				conn := getConnection(c)
				catKeys(conn, c.Args())
			},
		},
		{
			Name:      "get",
			Usage:     "Download keys",
			ArgsUsage: "key ...",
			Action: func(c *cli.Context) {
				if len(c.Args()) == 0 {
					cli.ShowCommandHelp(c, "get")
					exitCode = 1
					return
				}
				conn := getConnection(c)
				getKeys(conn, c.Args())
			},
		},
		{
			Name:      "grep",
			Usage:     "Grep keys",
			ArgsUsage: "string key ...",
			Action: func(c *cli.Context) {
				if len(c.Args()) == 0 {
					cli.ShowCommandHelp(c, "grep")
					exitCode = 1
					return
				}
				conn := getConnection(c)
				grepKeys(conn, c.Args())
			},
		},
		{
			Name:      "ls",
			Usage:     "List buckets or keys",
			ArgsUsage: "[bucket]",
			Action: func(c *cli.Context) {
				if len(c.Args()) < 1 {
					conn := getConnection(c)
					listBuckets(conn)
				} else {
					conn := getConnection(c)
					listKeys(conn, c.Args())
				}
			},
		},
		{
			Name:      "mb",
			Usage:     "Create bucket",
			ArgsUsage: "bucket",
			Action: func(c *cli.Context) {
				if len(c.Args()) != 1 {
					cli.ShowCommandHelp(c, "mb")
					exitCode = 1
					return
				}
				conn := getConnection(c)
				putBuckets(conn, c.Args())
			},
		},
		{
			Name:      "put",
			Usage:     "Upload files",
			ArgsUsage: "source [source ...] dest",
			Flags:     []cli.Flag{aclFlag, publicFlag},
			Action: func(c *cli.Context) {
				if len(c.Args()) < 2 {
					cli.ShowCommandHelp(c, "put")
					exitCode = 1
					return
				}
				if public {
					acl = "public-read"
				}
				if !validACL() {
					exitCode = 1
					return
				}
				conn := getConnection(c)
				putKeys(conn, c.Args())
			},
		},
		{
			Name:      "rb",
			Usage:     "Remove bucket(s)",
			ArgsUsage: "bucket ...",
			Action: func(c *cli.Context) {
				if len(c.Args()) == 0 {
					cli.ShowCommandHelp(c, "rb")
					exitCode = 1
					return
				}
				conn := getConnection(c)
				rmBuckets(conn, c.Args())
			},
		},
		{
			Name:      "rm",
			Usage:     "Remove keys",
			ArgsUsage: "key ...",
			Action: func(c *cli.Context) {
				if len(c.Args()) == 0 {
					cli.ShowCommandHelp(c, "rm")
					exitCode = 1
					return
				}
				conn := getConnection(c)
				rmKeys(conn, c.Args())
			},
		},
		{
			Name:      "sync",
			Usage:     "Synchronise local to s3, s3 to s3 or s3 to local",
			ArgsUsage: "source dest",
			Flags:     []cli.Flag{aclFlag, publicFlag, deleteFlag},
			Action: func(c *cli.Context) {
				if len(c.Args()) != 2 {
					cli.ShowCommandHelp(c, "sync")
					exitCode = 1
					return
				}
				if public {
					acl = "public-read"
				}
				if !validACL() {
					exitCode = 1
					return
				}
				conn := getConnection(c)
				syncFiles(conn, c.Args())
			},
		},
	}
	app.Run(args)
	return exitCode
}
<|endoftext|>


<commit_before>package main

import (
	"encoding/json"
	"flag"
	"net/http"
	"os"
	"time"

	"github.com/getlantern/autoupdate-server/server"
	"github.com/getlantern/golog"
)

var (
	flagPrivateKey         = flag.String("k", "", "Path to private key.")
	flagLocalAddr          = flag.String("l", ":9999", "Local bind address.")
	flagPublicAddr         = flag.String("p", "http://127.0.0.1:9999/", "Public address.")
	flagGithubOrganization = flag.String("o", "getlantern", "Github organization.")
	flagGithubProject      = flag.String("n", "lantern", "Github project name.")
	flagHelp               = flag.Bool("h", false, "Shows help.")
)

var (
	log            = golog.LoggerFor("autoupdate-server")
	releaseManager *server.ReleaseManager
)

type updateHandler struct {
}

// updateAssets checks for new assets released on the github releases page.
func updateAssets() error {
	log.Debug("Updating assets...")
	if err := releaseManager.UpdateAssetsMap(); err != nil {
		return err
	}
	return nil
}

// backgroundUpdate periodically looks for releases.
func backgroundUpdate() {
	for {
		time.Sleep(githubRefreshTime)
		// Updating assets...
		if err := updateAssets(); err != nil {
			log.Debugf("updateAssets: %s", err)
		}
	}
}

func (u *updateHandler) closeWithStatus(w http.ResponseWriter, status int) {
	w.WriteHeader(status)
	if status == http.StatusNoContent {
		return
	}
	if _, err := w.Write([]byte(http.StatusText(status))); err != nil {
		log.Debugf("Unable to write status %d: %v", status, err)
	}
}

func (u *updateHandler) ServeHTTP(w http.ResponseWriter, r *http.Request) {
	var err error
	var res *server.Result

	if r.Method == "POST" {
		defer r.Body.Close()

		var params server.Params
		decoder := json.NewDecoder(r.Body)

		if err = decoder.Decode(&params); err != nil {
			u.closeWithStatus(w, http.StatusBadRequest)
			return
		}

		if res, err = releaseManager.CheckForUpdate(&params); err != nil {
			if err == server.ErrNoUpdateAvailable {
				u.closeWithStatus(w, http.StatusNoContent)
				return
			}
			log.Debugf("CheckForUpdate failed. OS/Version: %s/%s, error: %q", params.AppVersion, params.OS, err)
			u.closeWithStatus(w, http.StatusExpectationFailed)
			return
		}

		// log.Debugf("Got query from client %q/%q, resolved to upgrade to %q using %q strategy.", params.AppVersion, params.OS, res.Version, res.PatchType)

		if res.PatchURL != "" {
			res.PatchURL = *flagPublicAddr + res.PatchURL
		}

		var content []byte

		if content, err = json.Marshal(res); err != nil {
			u.closeWithStatus(w, http.StatusInternalServerError)
			return
		}

		w.WriteHeader(http.StatusOK)
		w.Header().Set("Content-Type", "application/json")
		if _, err := w.Write(content); err != nil {
			log.Debugf("Unable to write response: %v", err)
		}
		return
	}
	u.closeWithStatus(w, http.StatusNotFound)
	return
}

func main() {

	// Parsing flags
	flag.Parse()

	if *flagHelp || *flagPrivateKey == "" {
		flag.Usage()
		os.Exit(0)
	}

	server.SetPrivateKey(*flagPrivateKey)

	// Creating release manager.
	log.Debug("Starting release manager.")
	releaseManager = server.NewReleaseManager(*flagGithubOrganization, *flagGithubProject)
	// Getting assets...
	if err := updateAssets(); err != nil {
		// In this case we will not be able to continue.
		log.Fatal(err)
	}

	// Setting a goroutine for pulling updates periodically
	go backgroundUpdate()

	mux := http.NewServeMux()

	mux.Handle("/update", new(updateHandler))
	mux.Handle("/patches/", http.StripPrefix("/patches/", http.FileServer(http.Dir(localPatchesDirectory))))

	srv := http.Server{
		Addr:    *flagLocalAddr,
		Handler: mux,
	}

	log.Debugf("Starting up HTTP server at %s.", *flagLocalAddr)

	if err := srv.ListenAndServe(); err != nil {
		log.Fatalf("ListenAndServe: ", err)
	}

}
<commit_msg>Re-enable debug message.<commit_after>package main

import (
	"encoding/json"
	"flag"
	"net/http"
	"os"
	"time"

	"github.com/getlantern/autoupdate-server/server"
	"github.com/getlantern/golog"
)

var (
	flagPrivateKey         = flag.String("k", "", "Path to private key.")
	flagLocalAddr          = flag.String("l", ":9999", "Local bind address.")
	flagPublicAddr         = flag.String("p", "http://127.0.0.1:9999/", "Public address.")
	flagGithubOrganization = flag.String("o", "getlantern", "Github organization.")
	flagGithubProject      = flag.String("n", "lantern", "Github project name.")
	flagHelp               = flag.Bool("h", false, "Shows help.")
)

var (
	log            = golog.LoggerFor("autoupdate-server")
	releaseManager *server.ReleaseManager
)

type updateHandler struct {
}

// updateAssets checks for new assets released on the github releases page.
func updateAssets() error {
	log.Debug("Updating assets...")
	if err := releaseManager.UpdateAssetsMap(); err != nil {
		return err
	}
	return nil
}

// backgroundUpdate periodically looks for releases.
func backgroundUpdate() {
	for {
		time.Sleep(githubRefreshTime)
		// Updating assets...
		if err := updateAssets(); err != nil {
			log.Debugf("updateAssets: %s", err)
		}
	}
}

func (u *updateHandler) closeWithStatus(w http.ResponseWriter, status int) {
	w.WriteHeader(status)
	if status == http.StatusNoContent {
		return
	}
	if _, err := w.Write([]byte(http.StatusText(status))); err != nil {
		log.Debugf("Unable to write status %d: %v", status, err)
	}
}

func (u *updateHandler) ServeHTTP(w http.ResponseWriter, r *http.Request) {
	var err error
	var res *server.Result

	if r.Method == "POST" {
		defer r.Body.Close()

		var params server.Params
		decoder := json.NewDecoder(r.Body)

		if err = decoder.Decode(&params); err != nil {
			u.closeWithStatus(w, http.StatusBadRequest)
			return
		}

		if res, err = releaseManager.CheckForUpdate(&params); err != nil {
			if err == server.ErrNoUpdateAvailable {
				u.closeWithStatus(w, http.StatusNoContent)
				return
			}
			log.Debugf("CheckForUpdate failed. OS/Version: %s/%s, error: %q", params.AppVersion, params.OS, err)
			u.closeWithStatus(w, http.StatusExpectationFailed)
			return
		}

		log.Debugf("Got query from client %q/%q, resolved to upgrade to %q using %q strategy.", params.AppVersion, params.OS, res.Version, res.PatchType)

		if res.PatchURL != "" {
			res.PatchURL = *flagPublicAddr + res.PatchURL
		}

		var content []byte

		if content, err = json.Marshal(res); err != nil {
			u.closeWithStatus(w, http.StatusInternalServerError)
			return
		}

		w.WriteHeader(http.StatusOK)
		w.Header().Set("Content-Type", "application/json")
		if _, err := w.Write(content); err != nil {
			log.Debugf("Unable to write response: %v", err)
		}
		return
	}
	u.closeWithStatus(w, http.StatusNotFound)
	return
}

func main() {

	// Parsing flags
	flag.Parse()

	if *flagHelp || *flagPrivateKey == "" {
		flag.Usage()
		os.Exit(0)
	}

	server.SetPrivateKey(*flagPrivateKey)

	// Creating release manager.
	log.Debug("Starting release manager.")
	releaseManager = server.NewReleaseManager(*flagGithubOrganization, *flagGithubProject)
	// Getting assets...
	if err := updateAssets(); err != nil {
		// In this case we will not be able to continue.
		log.Fatal(err)
	}

	// Setting a goroutine for pulling updates periodically
	go backgroundUpdate()

	mux := http.NewServeMux()

	mux.Handle("/update", new(updateHandler))
	mux.Handle("/patches/", http.StripPrefix("/patches/", http.FileServer(http.Dir(localPatchesDirectory))))

	srv := http.Server{
		Addr:    *flagLocalAddr,
		Handler: mux,
	}

	log.Debugf("Starting up HTTP server at %s.", *flagLocalAddr)

	if err := srv.ListenAndServe(); err != nil {
		log.Fatalf("ListenAndServe: ", err)
	}

}
<|endoftext|>


<commit_before>package main

import (
	"bufio"
	"flag"
	"fmt"
	"log"
	"os"
	"runtime"
	"runtime/pprof"
	"sync"
	"sync/atomic"

	"github.com/caelifer/dups/balancer"
	"github.com/caelifer/dups/fstree"
	"github.com/caelifer/dups/mapreduce"
)

// Global stats for activity report
var stats struct {
	TotalDirs        uint64
	TotalFiles       uint64
	TotalCopies      uint64
	TotalWastedSpace uint64
}

// Internal constant to control number of Worker threads in balancer's worker pool
const workerPoolMultiplier = 2 // Use twice the available cores

// Flags
var (
	cpuprofile      = flag.String("cpuprofile", "", "write cpu profile to file")
	memprofile      = flag.String("memprofile", "", "write memory profile to file")
	maxWorkerNumber = flag.Int("jobs", runtime.NumCPU()*workerPoolMultiplier, "Number of parallel jobs")
)

// Global pool manager interfaced via WorkQueue
var WorkQueue = balancer.NewWorkQueue(*maxWorkerNumber)

func main() {
	// First parse flags
	flag.Parse()

	// Prep runtime to use the maxWorkerNumber real threads
	runtime.GOMAXPROCS(*maxWorkerNumber)

	if *cpuprofile != "" {
		f, err := os.Create(*cpuprofile)
		if err != nil {
			log.Fatal(err)
		}
		pprof.StartCPUProfile(f)
		defer pprof.StopCPUProfile()
	}

	if *memprofile != "" {
		f, err := os.Create(*memprofile)
		if err != nil {
			log.Fatal(err)
		}
		defer func() {
			pprof.WriteHeapProfile(f)
			f.Close()
		}()
	}

	// Process command line params
	paths := flag.Args()

	if len(paths) == 0 {
		// Default is current directory
		paths = []string{"."}
	}

	// Channel interfaces between Map() and Reduce() functions
	var keyValChan <-chan mapreduce.KeyValue
	var valChan <-chan mapreduce.Value

	// Start map-reduce and remove duplicate path nodes
	keyValChan = mapreduce.Map(makeNodeMapFnWithPaths(paths))
	valChan = mapreduce.Reduce(keyValChan, reduceByFileName)

	// Map by filesize
	keyValChan = mapreduce.Map(makeFileSizeMapFnFrom(valChan))
	valChan = mapreduce.Reduce(keyValChan, reduceByFileSize)

	// Map by fast SHA1 hash (first 1024 bytes)
	keyValChan = mapreduce.Map(makeFileHashMapFnFrom(valChan, true))
	valChan = mapreduce.Reduce(keyValChan, reduceByHash)

	// Map by SHA1 hash (full file hashing)
	keyValChan = mapreduce.Map(makeFileHashMapFnFrom(valChan, false))
	valChan = mapreduce.Reduce(keyValChan, reduceByHash)

	// Final reduce before reporting
	dups := make(chan Dup)
	go func(out chan<- Dup) {
		byHash := make(map[string][]*Node)

		for x := range valChan {
			n := x.Value().(*Node) // Type assert

			// Aggregate
			if v, ok := byHash[n.Hash]; ok {
				// Found node with the same file size
				byHash[n.Hash] = append(v, n)
			} else {
				byHash[n.Hash] = []*Node{n}
			}
		}

		// Reduce
		for _, nodes := range byHash {
			count := len(nodes)
			if count > 1 {
				// Update free size stats
				atomic.AddUint64(&stats.TotalWastedSpace, uint64(nodes[0].Size*int64(count-1)))

				for _, node := range nodes {
					// Update dups number stats
					atomic.AddUint64(&stats.TotalCopies, 1)
					out <- Dup{*node, count}
				}
			}
		}
		close(out)
	}(dups)

	// Buffer output
	w := bufio.NewWriter(os.Stdout)
	// Report
	for d := range dups {
		fmt.Fprintln(w, d)
	}
	// Flush any pending writes
	w.Flush() // Ignore error

	// Stats report
	log.Printf("Examined %d files in %d directories, found %d dups, total wasted space %.2fGB\n",
		stats.TotalFiles, stats.TotalDirs, stats.TotalCopies, float64(stats.TotalWastedSpace)/(1024*1024*1024))
}

// makeNodeMapFnWithPaths
func makeNodeMapFnWithPaths(paths []string) mapreduce.MapFn {
	return func(out chan<- mapreduce.KeyValue) {
		// Process all command line paths
		for _, path_ := range paths {
			// err := filepath.Walk(path_, func(path string, info os.FileInfo, err error) error {
			err := fstree.Walk(WorkQueue, path_, func(path string, info os.FileInfo, err error) error {
				// Handle passthrough error
				if err != nil {
					log.Println("WARN", err)
					return nil
				}

				// Only process simple files
				if info.IsDir() {
					// Increase seen directory counter
					atomic.AddUint64(&stats.TotalDirs, 1)
				}

				// Only process simple files
				if IsRegularFile(info) {
					size := info.Size()
					out <- mapreduce.NewKVType(
						mapreduce.KeyTypeFromString(path),
						&Node{Path: path, Size: size},
					)
					// Increase seen files counter
					atomic.AddUint64(&stats.TotalFiles, 1)
				}

				//	// Log symlinks
				//	if info.Mode()&os.ModeSymlink != 0 {
				//	fpath := path + string(os.PathSeparator) + info.Name()
				//		log.Println("DEBUG Found symlink", fpath)
				//	}

				return nil
			})

			if err != nil {
				log.Fatal(err)
			}
		}
	}
}

// reduceByFileName custom function remove nodes with duplicate paths
func reduceByFileName(out chan<- mapreduce.Value, in <-chan mapreduce.KeyValue) {
	byName := make(map[mapreduce.KeyType]*Node)

	for x := range in {
		path := x.Key()           // Get key
		node := x.Value().(*Node) // Assert type

		// Add values to the map for aggregation, skip nodes with the same path
		if _, ok := byName[path]; !ok {
			byName[path] = node
			out <- node // send first copy
		}
	}
}

// Very simple function to map nodes by size
func makeFileSizeMapFnFrom(in <-chan mapreduce.Value) mapreduce.MapFn {
	return func(out chan<- mapreduce.KeyValue) {
		for x := range in {
			node := x.Value().(*Node) // Assert type

			out <- mapreduce.NewKVType(
				mapreduce.KeyTypeFromInt64(node.Size),
				node,
			)
		}
	}
}

// reduceByFileSize custom function to filter files by size
func reduceByFileSize(out chan<- mapreduce.Value, in <-chan mapreduce.KeyValue) {
	bySize := make(map[mapreduce.KeyType][]*Node)

	for x := range in {
		size := x.Key()           // Get key
		node := x.Value().(*Node) // Assert type

		// Add values to the map for aggregation
		if v, ok := bySize[size]; ok {
			// Found node with the same file size
			if len(v) == 1 {
				// First time we found duplicate, send first node too
				out <- v[0]
			}
			bySize[size] = append(v, node)
			// Send duplicate downstream
			out <- node
		} else {
			// Store first copy
			bySize[size] = []*Node{node}
		}
	}
}

func makeFileHashMapFnFrom(in <-chan mapreduce.Value, fast bool) mapreduce.MapFn {
	return func(out chan<- mapreduce.KeyValue) {
		var wg sync.WaitGroup
		for x := range in {
			node := x.Value().(*Node) // Assert type

			// Add to wait group
			wg.Add(1)

			// Calculate hash using balancer
			go func(n *Node) {
				WorkQueue <- func() {
					defer wg.Done() // Signal done

					// Default hash value for files < 1024 if fast is true
					hash := "0h"

					// Little optimization to avoid calculating fast hash on small files
					if !fast || n.Size > blockSize {
						// Always calculate hash if fast == false or file size > blockSize
						hash = n.calculateHash(fast) // Fast hash calculation - SHA1 of first 1024 bytes
					}

					// Don't process files for which we failed to calculate SHA1 hash
					if hash == "" {
						// log.Printf("WARN Unable calculate SHA1 hash for %q\n", node.Path)
						return
					}

					out <- mapreduce.NewKVType(
						mapreduce.KeyTypeFromString(hash),
						n,
					)
				}
			}(node)
		}
		// Wait for all results be submitted
		wg.Wait()
	}
}

func reduceByHash(out chan<- mapreduce.Value, in <-chan mapreduce.KeyValue) {
	byHash := make(map[mapreduce.KeyType][]*Node)

	for x := range in {
		hash := x.Key()
		node := x.Value().(*Node) // Assert type

		// Add hash value to a node
		node.Hash = hash.String()

		if v, ok := byHash[hash]; ok {
			// Found node with the same SHA1 hash
			// Send out aggregeted results
			if len(v) == 1 {
				// First time we found duplicate, send first node too
				out <- v[0]
			}
			byHash[hash] = append(v, node)
			out <- node
		} else {
			byHash[hash] = []*Node{node}
		}
	}
}

func IsRegularFile(fi os.FileInfo) bool {
	return fi.Mode()&os.ModeType == 0
}
<commit_msg>- Use new node package<commit_after>package main

import (
	"bufio"
	"flag"
	"fmt"
	"log"
	"os"
	"runtime"
	"runtime/pprof"
	"sync"
	"sync/atomic"

	"github.com/caelifer/dups/balancer"
	"github.com/caelifer/dups/fstree"
	"github.com/caelifer/dups/mapreduce"
	"github.com/caelifer/dups/node"
)

// Global stats for activity report
var stats struct {
	TotalDirs        uint64
	TotalFiles       uint64
	TotalCopies      uint64
	TotalWastedSpace uint64
}

// Internal constant to control number of Worker threads in balancer's worker pool
const workerPoolMultiplier = 2 // Use twice the available cores

// Flags
var (
	cpuprofile      = flag.String("cpuprofile", "", "write cpu profile to file")
	memprofile      = flag.String("memprofile", "", "write memory profile to file")
	maxWorkerNumber = flag.Int("jobs", runtime.NumCPU()*workerPoolMultiplier, "Number of parallel jobs")
)

// Global pool manager interfaced via WorkQueue
var WorkQueue = balancer.NewWorkQueue(*maxWorkerNumber)

func main() {
	// First parse flags
	flag.Parse()

	// Prep runtime to use the maxWorkerNumber real threads
	runtime.GOMAXPROCS(*maxWorkerNumber)

	if *cpuprofile != "" {
		f, err := os.Create(*cpuprofile)
		if err != nil {
			log.Fatal(err)
		}
		pprof.StartCPUProfile(f)
		defer pprof.StopCPUProfile()
	}

	if *memprofile != "" {
		f, err := os.Create(*memprofile)
		if err != nil {
			log.Fatal(err)
		}
		defer func() {
			pprof.WriteHeapProfile(f)
			f.Close()
		}()
	}

	// Process command line params
	paths := flag.Args()

	if len(paths) == 0 {
		// Default is current directory
		paths = []string{"."}
	}

	// Channel interfaces between Map() and Reduce() functions
	var keyValChan <-chan mapreduce.KeyValue
	var valChan <-chan mapreduce.Value

	// Start map-reduce and remove duplicate path nodes
	keyValChan = mapreduce.Map(makeNodeMapFnWithPaths(paths))
	valChan = mapreduce.Reduce(keyValChan, reduceByFileName)

	// Map by filesize
	keyValChan = mapreduce.Map(makeFileSizeMapFnFrom(valChan))
	valChan = mapreduce.Reduce(keyValChan, reduceByFileSize)

	// Map by fast SHA1 hash (first 1024 bytes)
	keyValChan = mapreduce.Map(makeFileHashMapFnFrom(valChan, true))
	valChan = mapreduce.Reduce(keyValChan, reduceByHash)

	// Map by SHA1 hash (full file hashing)
	keyValChan = mapreduce.Map(makeFileHashMapFnFrom(valChan, false))
	valChan = mapreduce.Reduce(keyValChan, reduceByHash)

	// Final reduce before reporting
	dups := make(chan node.Dup)
	go func(out chan<- node.Dup) {
		byHash := make(map[string][]*node.Node)

		for x := range valChan {
			n := x.Value().(*node.Node) // Type assert

			// Aggregate
			if v, ok := byHash[n.Hash]; ok {
				// Found node with the same file size
				byHash[n.Hash] = append(v, n)
			} else {
				byHash[n.Hash] = []*node.Node{n}
			}
		}

		// Reduce
		for _, nodes := range byHash {
			count := len(nodes)
			if count > 1 {
				// Update free size stats
				atomic.AddUint64(&stats.TotalWastedSpace, uint64(nodes[0].Size*int64(count-1)))

				for _, n := range nodes {
					// Update dups number stats
					atomic.AddUint64(&stats.TotalCopies, 1)
					out <- node.Dup{*n, count}
				}
			}
		}
		close(out)
	}(dups)

	// Buffer output
	w := bufio.NewWriter(os.Stdout)
	// Report
	for d := range dups {
		fmt.Fprintln(w, d)
	}
	// Flush any pending writes
	w.Flush() // Ignore error

	// Stats report
	log.Printf("Examined %d files in %d directories, found %d dups, total wasted space %.2fGB\n",
		stats.TotalFiles, stats.TotalDirs, stats.TotalCopies, float64(stats.TotalWastedSpace)/(1024*1024*1024))
}

// makeNodeMapFnWithPaths
func makeNodeMapFnWithPaths(paths []string) mapreduce.MapFn {
	return func(out chan<- mapreduce.KeyValue) {
		// Process all command line paths
		for _, path_ := range paths {
			// err := filepath.Walk(path_, func(path string, info os.FileInfo, err error) error {
			err := fstree.Walk(WorkQueue, path_, func(path string, info os.FileInfo, err error) error {
				// Handle passthrough error
				if err != nil {
					log.Println("WARN", err)
					return nil
				}

				// Only process simple files
				if info.IsDir() {
					// Increase seen directory counter
					atomic.AddUint64(&stats.TotalDirs, 1)
				}

				// Only process simple files
				if IsRegularFile(info) {
					size := info.Size()
					out <- mapreduce.NewKVType(
						mapreduce.KeyTypeFromString(path),
						&node.Node{Path: path, Size: size},
					)
					// Increase seen files counter
					atomic.AddUint64(&stats.TotalFiles, 1)
				}

				//	// Log symlinks
				//	if info.Mode()&os.ModeSymlink != 0 {
				//	fpath := path + string(os.PathSeparator) + info.Name()
				//		log.Println("DEBUG Found symlink", fpath)
				//	}

				return nil
			})

			if err != nil {
				log.Fatal(err)
			}
		}
	}
}

// reduceByFileName custom function remove nodes with duplicate paths
func reduceByFileName(out chan<- mapreduce.Value, in <-chan mapreduce.KeyValue) {
	byName := make(map[mapreduce.KeyType]*node.Node)

	for x := range in {
		path := x.Key()                // Get key
		node := x.Value().(*node.Node) // Assert type

		// Add values to the map for aggregation, skip nodes with the same path
		if _, ok := byName[path]; !ok {
			byName[path] = node
			out <- node // send first copy
		}
	}
}

// Very simple function to map nodes by size
func makeFileSizeMapFnFrom(in <-chan mapreduce.Value) mapreduce.MapFn {
	return func(out chan<- mapreduce.KeyValue) {
		for x := range in {
			node := x.Value().(*node.Node) // Assert type

			out <- mapreduce.NewKVType(
				mapreduce.KeyTypeFromInt64(node.Size),
				node,
			)
		}
	}
}

// reduceByFileSize custom function to filter files by size
func reduceByFileSize(out chan<- mapreduce.Value, in <-chan mapreduce.KeyValue) {
	bySize := make(map[mapreduce.KeyType][]*node.Node)

	for x := range in {
		size := x.Key()             // Get key
		n := x.Value().(*node.Node) // Assert type

		// Add values to the map for aggregation
		if v, ok := bySize[size]; ok {
			// Found node with the same file size
			if len(v) == 1 {
				// First time we found duplicate, send first node too
				out <- v[0]
			}
			// Add new node to a list
			bySize[size] = append(v, n)
			// Send duplicate downstream
			out <- n
		} else {
			// Store first copy
			bySize[size] = []*node.Node{n}
		}
	}
}

func makeFileHashMapFnFrom(in <-chan mapreduce.Value, fast bool) mapreduce.MapFn {
	return func(out chan<- mapreduce.KeyValue) {
		var wg sync.WaitGroup
		for x := range in {
			node_ := x.Value().(*node.Node) // Assert type

			// Add to wait group
			wg.Add(1)

			// Calculate hash using balancer
			go func(n *node.Node) {
				WorkQueue <- func() {
					defer wg.Done() // Signal done

					// Calculate hash using fast calculation if required
					err := n.CalculateHash(fast) // Fast hash calculation - SHA1 of first 1024 bytes
					if err != nil {
						// Skip files for which we failed to calculate SHA1 hash
						// log.Printf("WARN Unable calculate SHA1 hash for %q\n", node.Path)
						return
					}

					out <- mapreduce.NewKVType(
						mapreduce.KeyTypeFromString(n.Hash),
						n,
					)
				}
			}(node_)
		}
		// Wait for all results be submitted
		wg.Wait()
	}
}

func reduceByHash(out chan<- mapreduce.Value, in <-chan mapreduce.KeyValue) {
	byHash := make(map[mapreduce.KeyType][]*node.Node)

	for x := range in {
		hash := x.Key()
		n := x.Value().(*node.Node) // Assert type

		// Add hash value to a node
		n.Hash = hash.String()

		if v, ok := byHash[hash]; ok {
			// Found node with the same SHA1 hash
			// Send out aggregeted results
			if len(v) == 1 {
				// First time we found duplicate, send first node too
				out <- v[0]
			}
			// Add new node to a list
			byHash[hash] = append(v, n)
			// Send new node
			out <- n
		} else {
			byHash[hash] = []*node.Node{n}
		}
	}
}

func IsRegularFile(fi os.FileInfo) bool {
	return fi.Mode()&os.ModeType == 0
}
<|endoftext|>


<commit_before>/*

For keeping a minimum running, perhaps when doing a routing table update, if destination hosts are all
 expired or about to expire we start more.

*/

package main

import (
	"fmt"
	"os"
	"strings"

	log "github.com/Sirupsen/logrus"
	"github.com/iron-io/functions/api/datastore"
	"github.com/iron-io/functions/api/models"
	"github.com/iron-io/functions/api/server"
	"github.com/iron-io/titan/common"
	"github.com/spf13/viper"
)

func main() {
	config := &models.Config{}

	InitConfig()
	common.SetLogLevel(viper.GetString("log_level"))

	err := config.Validate()
	if err != nil {
		log.WithError(err).Fatalln("Invalid config.")
	}

	ds, err := datastore.New(viper.GetString("db"))
	if err != nil {
		log.WithError(err).Fatalln("Invalid DB url.")
	}

	srv := server.New(ds, config)
	srv.Run()
}

func InitConfig() {
	cwd, _ := os.Getwd()
	viper.SetEnvKeyReplacer(strings.NewReplacer(".", "_"))
	viper.SetDefault("log_level", "info")
	viper.SetDefault("db", fmt.Sprintf("bolt://%s/bolt.db?bucket=funcs", cwd))
	viper.SetConfigName("config")
	viper.AddConfigPath(".")
	viper.AutomaticEnv() // picks up env vars automatically
	viper.ReadInConfig()
}
<commit_msg>fixing log level<commit_after>/*

For keeping a minimum running, perhaps when doing a routing table update, if destination hosts are all
 expired or about to expire we start more.

*/

package main

import (
	"fmt"
	"os"
	"strings"

	log "github.com/Sirupsen/logrus"
	"github.com/iron-io/functions/api/datastore"
	"github.com/iron-io/functions/api/models"
	"github.com/iron-io/functions/api/server"
	"github.com/spf13/viper"
)

func main() {
	config := &models.Config{}

	InitConfig()
	logLevel, err := log.ParseLevel(viper.GetString("log_level"))
	if err != nil {
		log.WithError(err).Fatalln("Invalid log level.")
	}
	log.SetLevel(logLevel)

	err = config.Validate()
	if err != nil {
		log.WithError(err).Fatalln("Invalid config.")
	}

	ds, err := datastore.New(viper.GetString("db"))
	if err != nil {
		log.WithError(err).Fatalln("Invalid DB url.")
	}

	srv := server.New(ds, config)
	srv.Run()
}

func InitConfig() {
	cwd, _ := os.Getwd()
	viper.SetEnvKeyReplacer(strings.NewReplacer(".", "_"))
	viper.SetDefault("log_level", "info")
	viper.SetDefault("db", fmt.Sprintf("bolt://%s/bolt.db?bucket=funcs", cwd))
	viper.SetConfigName("config")
	viper.AddConfigPath(".")
	viper.AutomaticEnv() // picks up env vars automatically
	viper.ReadInConfig()
}
<|endoftext|>


<commit_before>// This Source Code Form is subject to the terms of the Mozilla Public
// License, v. 2.0. If a copy of the MPL was not distributed with this
// file, You can obtain one at http://mozilla.org/MPL/2.0/

package main

import (
	"fmt"
	"github.com/gorilla/mux"
	"github.com/st3fan/moz-syncapi/syncapi"
	"log"
	"net/http"
)

const (
	DEFAULT_API_PREFIX         = "/"
	DEFAULT_API_LISTEN_ADDRESS = "0.0.0.0"
	DEFAULT_API_LISTEN_PORT    = 9091
)

func main() {

	config := syncapi.DefaultConfig()

	router := mux.NewRouter()
	_, err := syncapi.SetupRouter(router.PathPrefix(DEFAULT_API_PREFIX).Subrouter(), config)
	if err != nil {
		log.Fatal(err)
	}

	http.Handle("/", router)

	addr := fmt.Sprintf("%s:%d", DEFAULT_API_LISTEN_ADDRESS, DEFAULT_API_LISTEN_PORT)
	log.Printf("Starting syncapi server on http://%s%s", addr, DEFAULT_API_PREFIX)
	err = http.ListenAndServe(addr, nil)
	if err != nil {
		log.Fatal(err)
	}
}
<commit_msg>Added some flags to configure the web app<commit_after>// This Source Code Form is subject to the terms of the Mozilla Public
// License, v. 2.0. If a copy of the MPL was not distributed with this
// file, You can obtain one at http://mozilla.org/MPL/2.0/

package main

import (
	"flag"
	"fmt"
	"github.com/gorilla/mux"
	"github.com/st3fan/moz-syncapi/syncapi"
	"log"
	"net/http"
)

const (
	DEFAULT_API_ROOT    = "/"
	DEFAULT_API_ADDRESS = "0.0.0.0"
	DEFAULT_API_PORT    = 8080
)

func main() {

	root := flag.String("root", DEFAULT_API_ROOT, "web root context")
	address := flag.String("address", DEFAULT_API_ADDRESS, "address to bind to")
	port := flag.Int("port", DEFAULT_API_PORT, "port to listen on")

	flag.Parse()

	config := syncapi.DefaultConfig()

	router := mux.NewRouter()
	_, err := syncapi.SetupRouter(router.PathPrefix(*root).Subrouter(), config)
	if err != nil {
		log.Fatal(err)
	}

	http.Handle("/", router)

	addr := fmt.Sprintf("%s:%d", *address, *port)
	log.Printf("Starting syncapi server on http://%s%s", addr, *root)
	err = http.ListenAndServe(addr, nil)
	if err != nil {
		log.Fatal(err)
	}
}
<|endoftext|>