<commit_before>package main

import (
	"fmt"
	"github.com/gorilla/mux"
	. "github.com/mickael-kerjean/filestash/server/common"
	. "github.com/mickael-kerjean/filestash/server/ctrl"
	. "github.com/mickael-kerjean/filestash/server/middleware"
	_ "github.com/mickael-kerjean/filestash/server/plugin"
	"net/http"
    "net/http/pprof"
	"os"
	"runtime/debug"
	"strconv"
	"time"
)

func main() {
	app := App{}
	Init(&app)
}

func Init(a *App) {
	var middlewares []Middleware
	r := mux.NewRouter()

	// Profiling - handy to identify nasty leaks and/or bugs!
	if os.Getenv("DEBUG") == "true" {
		r.HandleFunc("/debug/pprof/", pprof.Index)
		r.HandleFunc("/debug/pprof/cmdline", pprof.Cmdline)
		r.HandleFunc("/debug/pprof/profile", pprof.Profile)
		r.HandleFunc("/debug/pprof/symbol", pprof.Symbol)
		r.HandleFunc("/debug/pprof/trace", pprof.Trace)
		r.Handle("/debug/pprof/goroutine", pprof.Handler("goroutine"))
		r.Handle("/debug/pprof/heap", pprof.Handler("heap"))
		r.Handle("/debug/pprof/threadcreate", pprof.Handler("threadcreate"))
		r.Handle("/debug/pprof/block", pprof.Handler("block"))
		r.HandleFunc("/debug/free", func(w http.ResponseWriter, r *http.Request) {
			debug.FreeOSMemory()
			w.Write([]byte("DONE"))
		})
	}

	// API for Session
	session := r.PathPrefix("/api/session").Subrouter()
	middlewares = []Middleware{ ApiHeaders, SecureHeaders, SecureAjax, SessionStart }
	session.HandleFunc("",                NewMiddlewareChain(SessionGet,          middlewares, *a)).Methods("GET")
	middlewares = []Middleware{ ApiHeaders, SecureHeaders, SecureAjax, BodyParser }
	session.HandleFunc("",                NewMiddlewareChain(SessionAuthenticate, middlewares, *a)).Methods("POST")
	middlewares = []Middleware{ ApiHeaders, SecureHeaders, SecureAjax, SessionTry }
	session.HandleFunc("",                NewMiddlewareChain(SessionLogout,       middlewares, *a)).Methods("DELETE")
	middlewares = []Middleware{ ApiHeaders, SecureHeaders, SecureAjax }
	session.HandleFunc("/auth/{service}", NewMiddlewareChain(SessionOAuthBackend, middlewares, *a)).Methods("GET")

	// API for admin
	middlewares = []Middleware{ ApiHeaders, SecureAjax }
	admin := r.PathPrefix("/admin/api").Subrouter()
	admin.HandleFunc("/session", NewMiddlewareChain(AdminSessionGet,            middlewares, *a)).Methods("GET")
	admin.HandleFunc("/session", NewMiddlewareChain(AdminSessionAuthenticate,   middlewares, *a)).Methods("POST")
	middlewares = []Middleware{ ApiHeaders, AdminOnly, SecureAjax }
	admin.HandleFunc("/plugin",  NewMiddlewareChain(FetchPluginsHandler,        middlewares, *a)).Methods("GET")
	admin.HandleFunc("/config",  NewMiddlewareChain(PrivateConfigHandler,       middlewares, *a)).Methods("GET")
	admin.HandleFunc("/config",  NewMiddlewareChain(PrivateConfigUpdateHandler, middlewares, *a)).Methods("POST")
	middlewares = []Middleware{ IndexHeaders }
	admin.HandleFunc("/log",                        NewMiddlewareChain(FetchLogHandler,          middlewares, *a)).Methods("GET")
	r.PathPrefix("/admin").Handler(http.HandlerFunc(NewMiddlewareChain(IndexHandler(FILE_INDEX), middlewares, *a))).Methods("GET")

	// API for File management
	files := r.PathPrefix("/api/files").Subrouter()
	middlewares = []Middleware{ ApiHeaders, SecureHeaders, SessionStart, LoggedInOnly }
	files.HandleFunc("/cat",    NewMiddlewareChain(FileCat,    middlewares, *a)).Methods("GET", "HEAD")
	middlewares = []Middleware{ ApiHeaders, SecureHeaders, SecureAjax, SessionStart, LoggedInOnly }
	files.HandleFunc("/cat",    NewMiddlewareChain(FileAccess, middlewares, *a)).Methods("OPTIONS")
	files.HandleFunc("/cat",    NewMiddlewareChain(FileSave,   middlewares, *a)).Methods("POST")
	files.HandleFunc("/ls",     NewMiddlewareChain(FileLs,     middlewares, *a)).Methods("GET")
	files.HandleFunc("/mv",     NewMiddlewareChain(FileMv,     middlewares, *a)).Methods("GET")
	files.HandleFunc("/rm",     NewMiddlewareChain(FileRm,     middlewares, *a)).Methods("GET")
	files.HandleFunc("/mkdir",  NewMiddlewareChain(FileMkdir,  middlewares, *a)).Methods("GET")
	files.HandleFunc("/touch",  NewMiddlewareChain(FileTouch,  middlewares, *a)).Methods("GET")
	middlewares = []Middleware{ ApiHeaders, SessionStart, LoggedInOnly }
	files.HandleFunc("/search",  NewMiddlewareChain(FileSearch,  middlewares, *a)).Methods("GET")

	// API for exporter
	middlewares = []Middleware{ ApiHeaders, SecureHeaders, RedirectSharedLoginIfNeeded, SessionStart, LoggedInOnly }
	r.PathPrefix("/api/export/{share}/{mtype0}/{mtype1}").Handler(NewMiddlewareChain(FileExport,  middlewares, *a))

	// API for Shared link
	share := r.PathPrefix("/api/share").Subrouter()
	middlewares = []Middleware{ ApiHeaders, SecureHeaders, SecureAjax, SessionStart, LoggedInOnly }
	share.HandleFunc("",               NewMiddlewareChain(ShareList,        middlewares, *a)).Methods("GET")
	middlewares = []Middleware{ ApiHeaders, SecureHeaders, SecureAjax, BodyParser }
	share.HandleFunc("/{share}/proof", NewMiddlewareChain(ShareVerifyProof, middlewares, *a)).Methods("POST")
	middlewares = []Middleware{ ApiHeaders, SecureHeaders, SecureAjax, CanManageShare }
	share.HandleFunc("/{share}",       NewMiddlewareChain(ShareDelete,      middlewares, *a)).Methods("DELETE")
	middlewares = []Middleware{ ApiHeaders, SecureHeaders, SecureAjax, BodyParser, CanManageShare }
	share.HandleFunc("/{share}",       NewMiddlewareChain(ShareUpsert,      middlewares, *a)).Methods("POST")

	// Webdav server / Shared Link
	middlewares = []Middleware{ IndexHeaders, SecureHeaders }
	r.HandleFunc("/s/{share}",         NewMiddlewareChain(IndexHandler(FILE_INDEX), middlewares, *a)).Methods("GET")
	middlewares = []Middleware{ WebdavBlacklist, SessionStart }
	r.PathPrefix("/s/{share}").Handler(NewMiddlewareChain(WebdavHandler,            middlewares, *a))

	// Application Resources
	middlewares = []Middleware{ ApiHeaders }
	r.HandleFunc("/api/config",  NewMiddlewareChain(PublicConfigHandler,                             middlewares, *a)).Methods("GET")
	r.HandleFunc("/api/backend", NewMiddlewareChain(AdminBackend,                                    middlewares, *a)).Methods("GET")
	middlewares = []Middleware{ StaticHeaders }
	r.PathPrefix("/assets").Handler(http.HandlerFunc(NewMiddlewareChain(StaticHandler(FILE_ASSETS),  middlewares, *a))).Methods("GET")
	r.HandleFunc("/favicon.ico", func(res http.ResponseWriter, req *http.Request) {
		http.Redirect(res, req, "/assets/logo/favicon.ico", http.StatusMovedPermanently)
	})
	middlewares = []Middleware{ IndexHeaders, SecureHeaders }
	r.HandleFunc("/about",                     NewMiddlewareChain(AboutHandler,                      middlewares, *a)).Methods("GET")
	for _, obj := range Hooks.Get.HttpEndpoint() {
		obj(r)
	}
	r.HandleFunc("/robots.txt", func(res http.ResponseWriter, req *http.Request) {
		res.Write([]byte(""))
	})
	r.PathPrefix("/").Handler(http.HandlerFunc(NewMiddlewareChain(IndexHandler(FILE_INDEX),          middlewares, *a))).Methods("GET")

	port := Config.Get("general.port").Int()
	srv := &http.Server{
		Addr:    ":" + strconv.Itoa(port),
		Handler: r,
	}
	Log.Stdout("Filestash %s: starting", APP_VERSION)
	go ensureAppHasBooted(fmt.Sprintf("http://localhost:%d/about", port), fmt.Sprintf("listening on :%d", port))
	if err := srv.ListenAndServe(); err != nil {
		Log.Stdout("error: %v", err)
		return
	}
}

func ensureAppHasBooted(address string, message string) {
	for {
		time.Sleep(100 * time.Millisecond)
		res, err := http.Get(address)
		if err != nil {
			continue
		}
		res.Body.Close()
		if res.StatusCode != http.StatusOK {
			continue
		}
		Log.Stdout(message)
		break
	}
}
<commit_msg>fix (boot): infinite loop when booting - #154<commit_after>package main

import (
	"fmt"
	"github.com/gorilla/mux"
	. "github.com/mickael-kerjean/filestash/server/common"
	. "github.com/mickael-kerjean/filestash/server/ctrl"
	. "github.com/mickael-kerjean/filestash/server/middleware"
	_ "github.com/mickael-kerjean/filestash/server/plugin"
	"net/http"
    "net/http/pprof"
	"os"
	"runtime/debug"
	"strconv"
	"time"
)

func main() {
	app := App{}
	Init(&app)
}

func Init(a *App) {
	var middlewares []Middleware
	r := mux.NewRouter()

	// Profiling - handy to identify nasty leaks and/or bugs!
	if os.Getenv("DEBUG") == "true" {
		r.HandleFunc("/debug/pprof/", pprof.Index)
		r.HandleFunc("/debug/pprof/cmdline", pprof.Cmdline)
		r.HandleFunc("/debug/pprof/profile", pprof.Profile)
		r.HandleFunc("/debug/pprof/symbol", pprof.Symbol)
		r.HandleFunc("/debug/pprof/trace", pprof.Trace)
		r.Handle("/debug/pprof/goroutine", pprof.Handler("goroutine"))
		r.Handle("/debug/pprof/heap", pprof.Handler("heap"))
		r.Handle("/debug/pprof/threadcreate", pprof.Handler("threadcreate"))
		r.Handle("/debug/pprof/block", pprof.Handler("block"))
		r.HandleFunc("/debug/free", func(w http.ResponseWriter, r *http.Request) {
			debug.FreeOSMemory()
			w.Write([]byte("DONE"))
		})
	}

	// API for Session
	session := r.PathPrefix("/api/session").Subrouter()
	middlewares = []Middleware{ ApiHeaders, SecureHeaders, SecureAjax, SessionStart }
	session.HandleFunc("",                NewMiddlewareChain(SessionGet,          middlewares, *a)).Methods("GET")
	middlewares = []Middleware{ ApiHeaders, SecureHeaders, SecureAjax, BodyParser }
	session.HandleFunc("",                NewMiddlewareChain(SessionAuthenticate, middlewares, *a)).Methods("POST")
	middlewares = []Middleware{ ApiHeaders, SecureHeaders, SecureAjax, SessionTry }
	session.HandleFunc("",                NewMiddlewareChain(SessionLogout,       middlewares, *a)).Methods("DELETE")
	middlewares = []Middleware{ ApiHeaders, SecureHeaders, SecureAjax }
	session.HandleFunc("/auth/{service}", NewMiddlewareChain(SessionOAuthBackend, middlewares, *a)).Methods("GET")

	// API for admin
	middlewares = []Middleware{ ApiHeaders, SecureAjax }
	admin := r.PathPrefix("/admin/api").Subrouter()
	admin.HandleFunc("/session", NewMiddlewareChain(AdminSessionGet,            middlewares, *a)).Methods("GET")
	admin.HandleFunc("/session", NewMiddlewareChain(AdminSessionAuthenticate,   middlewares, *a)).Methods("POST")
	middlewares = []Middleware{ ApiHeaders, AdminOnly, SecureAjax }
	admin.HandleFunc("/plugin",  NewMiddlewareChain(FetchPluginsHandler,        middlewares, *a)).Methods("GET")
	admin.HandleFunc("/config",  NewMiddlewareChain(PrivateConfigHandler,       middlewares, *a)).Methods("GET")
	admin.HandleFunc("/config",  NewMiddlewareChain(PrivateConfigUpdateHandler, middlewares, *a)).Methods("POST")
	middlewares = []Middleware{ IndexHeaders }
	admin.HandleFunc("/log",                        NewMiddlewareChain(FetchLogHandler,          middlewares, *a)).Methods("GET")
	r.PathPrefix("/admin").Handler(http.HandlerFunc(NewMiddlewareChain(IndexHandler(FILE_INDEX), middlewares, *a))).Methods("GET")

	// API for File management
	files := r.PathPrefix("/api/files").Subrouter()
	middlewares = []Middleware{ ApiHeaders, SecureHeaders, SessionStart, LoggedInOnly }
	files.HandleFunc("/cat",    NewMiddlewareChain(FileCat,    middlewares, *a)).Methods("GET", "HEAD")
	middlewares = []Middleware{ ApiHeaders, SecureHeaders, SecureAjax, SessionStart, LoggedInOnly }
	files.HandleFunc("/cat",    NewMiddlewareChain(FileAccess, middlewares, *a)).Methods("OPTIONS")
	files.HandleFunc("/cat",    NewMiddlewareChain(FileSave,   middlewares, *a)).Methods("POST")
	files.HandleFunc("/ls",     NewMiddlewareChain(FileLs,     middlewares, *a)).Methods("GET")
	files.HandleFunc("/mv",     NewMiddlewareChain(FileMv,     middlewares, *a)).Methods("GET")
	files.HandleFunc("/rm",     NewMiddlewareChain(FileRm,     middlewares, *a)).Methods("GET")
	files.HandleFunc("/mkdir",  NewMiddlewareChain(FileMkdir,  middlewares, *a)).Methods("GET")
	files.HandleFunc("/touch",  NewMiddlewareChain(FileTouch,  middlewares, *a)).Methods("GET")
	middlewares = []Middleware{ ApiHeaders, SessionStart, LoggedInOnly }
	files.HandleFunc("/search",  NewMiddlewareChain(FileSearch,  middlewares, *a)).Methods("GET")

	// API for exporter
	middlewares = []Middleware{ ApiHeaders, SecureHeaders, RedirectSharedLoginIfNeeded, SessionStart, LoggedInOnly }
	r.PathPrefix("/api/export/{share}/{mtype0}/{mtype1}").Handler(NewMiddlewareChain(FileExport,  middlewares, *a))

	// API for Shared link
	share := r.PathPrefix("/api/share").Subrouter()
	middlewares = []Middleware{ ApiHeaders, SecureHeaders, SecureAjax, SessionStart, LoggedInOnly }
	share.HandleFunc("",               NewMiddlewareChain(ShareList,        middlewares, *a)).Methods("GET")
	middlewares = []Middleware{ ApiHeaders, SecureHeaders, SecureAjax, BodyParser }
	share.HandleFunc("/{share}/proof", NewMiddlewareChain(ShareVerifyProof, middlewares, *a)).Methods("POST")
	middlewares = []Middleware{ ApiHeaders, SecureHeaders, SecureAjax, CanManageShare }
	share.HandleFunc("/{share}",       NewMiddlewareChain(ShareDelete,      middlewares, *a)).Methods("DELETE")
	middlewares = []Middleware{ ApiHeaders, SecureHeaders, SecureAjax, BodyParser, CanManageShare }
	share.HandleFunc("/{share}",       NewMiddlewareChain(ShareUpsert,      middlewares, *a)).Methods("POST")

	// Webdav server / Shared Link
	middlewares = []Middleware{ IndexHeaders, SecureHeaders }
	r.HandleFunc("/s/{share}",         NewMiddlewareChain(IndexHandler(FILE_INDEX), middlewares, *a)).Methods("GET")
	middlewares = []Middleware{ WebdavBlacklist, SessionStart }
	r.PathPrefix("/s/{share}").Handler(NewMiddlewareChain(WebdavHandler,            middlewares, *a))

	// Application Resources
	middlewares = []Middleware{ ApiHeaders }
	r.HandleFunc("/api/config",  NewMiddlewareChain(PublicConfigHandler,                             middlewares, *a)).Methods("GET")
	r.HandleFunc("/api/backend", NewMiddlewareChain(AdminBackend,                                    middlewares, *a)).Methods("GET")
	middlewares = []Middleware{ StaticHeaders }
	r.PathPrefix("/assets").Handler(http.HandlerFunc(NewMiddlewareChain(StaticHandler(FILE_ASSETS),  middlewares, *a))).Methods("GET")
	r.HandleFunc("/favicon.ico", func(res http.ResponseWriter, req *http.Request) {
		http.Redirect(res, req, "/assets/logo/favicon.ico", http.StatusMovedPermanently)
	})
	middlewares = []Middleware{ IndexHeaders }
	r.HandleFunc("/about",                     NewMiddlewareChain(AboutHandler,                      middlewares, *a)).Methods("GET")
	for _, obj := range Hooks.Get.HttpEndpoint() {
		obj(r)
	}
	r.HandleFunc("/robots.txt", func(res http.ResponseWriter, req *http.Request) {
		res.Write([]byte(""))
	})
	r.PathPrefix("/").Handler(http.HandlerFunc(NewMiddlewareChain(IndexHandler(FILE_INDEX),          middlewares, *a))).Methods("GET")

	port := Config.Get("general.port").Int()
	srv := &http.Server{
		Addr:    ":" + strconv.Itoa(port),
		Handler: r,
	}
	Log.Stdout("Filestash %s: starting", APP_VERSION)
	go ensureAppHasBooted(fmt.Sprintf("http://localhost:%d/about", port), fmt.Sprintf("listening on :%d", port))
	if err := srv.ListenAndServe(); err != nil {
		Log.Stdout("error: %v", err)
		return
	}
}

func ensureAppHasBooted(address string, message string) {
	i := 0
	for {
		if i > 10 {
			Log.Warning("Filestash hasn't boot ?!?")
			break
		}
		time.Sleep(250 * time.Millisecond)
		res, err := http.Get(address)
		if err != nil {
			i += 1
			continue
		}
		res.Body.Close()
		if res.StatusCode != http.StatusOK {
			i += 1
			continue
		}
		Log.Stdout(message)
		break
	}
}
<|endoftext|>


<commit_before>package main

import (
	"fmt"
	"os"
	"path/filepath"

	"github.com/BurntSushi/toml"
)

type config struct {
	GtdFile   string `toml:"gtdfile"`
	MemoDir   string `toml:"memodir"`
	OutputDir string `toml:"outputdir"`
	FilterCmd string `toml:"filtercmd"`
	Editor    string `toml:"editor"`
}

func (cfg *config) load() error {
	var dir string
	dir = filepath.Join(os.Getenv("HOME"), ".config", "gtd")

	if err := os.MkdirAll(dir, 0700); err != nil {
		return fmt.Errorf("cannot create directory: %v", err)
	}

	configfile := filepath.Join(dir, "config.toml")
	_, err := os.Stat(configfile)
	// if file exists in ~/.config/gtd/config.toml, decode file and return nil
	if err == nil {
		_, err := toml.DecodeFile(configfile, cfg)
		if err != nil {
			return fmt.Errorf("cannot decode toml file: %v", err)
		}
		return nil
	}

	cfg.Editor = "vi"
	gtdfile := filepath.Join(os.Getenv("HOME"), "gtd.json")
	cfg.GtdFile = gtdfile
	f, err := os.Create(configfile)
	// if file not exists in ~/.config/gtd/config.toml, write f with cfg
	return toml.NewEncoder(f).Encode(cfg)
}
<commit_msg>add runcmd<commit_after>package main

import (
	"fmt"
	"os"
	"os/exec"
	"path/filepath"
	"strings"

	"github.com/BurntSushi/toml"
)

type config struct {
	GtdFile   string `toml:"gtdfile"`
	MemoDir   string `toml:"memodir"`
	OutputDir string `toml:"outputdir"`
	FilterCmd string `toml:"filtercmd"`
	Editor    string `toml:"editor"`
}

func (cfg *config) load() error {
	var dir string
	dir = filepath.Join(os.Getenv("HOME"), ".config", "gtd")

	if err := os.MkdirAll(dir, 0700); err != nil {
		return fmt.Errorf("cannot create directory: %v", err)
	}

	configfile := filepath.Join(dir, "config.toml")
	_, err := os.Stat(configfile)
	// if file exists in ~/.config/gtd/config.toml, decode file and return nil
	if err == nil {
		_, err := toml.DecodeFile(configfile, cfg)
		if err != nil {
			return fmt.Errorf("cannot decode toml file: %v", err)
		}
		return nil
	}

	cfg.Editor = "vi"
	gtdfile := filepath.Join(os.Getenv("HOME"), "gtd.json")
	cfg.GtdFile = gtdfile
	f, err := os.Create(configfile)
	return toml.NewEncoder(f).Encode(cfg)
}

func (cfg *config) runcmd(command string, files ...string) error {
	var args []string
	for _, file := range files {
		args = append(args, fmt.Sprintf("%q", file))
	}
	cmdargs := strings.Join(args, " ")
	command += " " + cmdargs

	var cmd *exec.Cmd
	cmd = exec.Command("sh", "-c", command)
	cmd.Stderr = os.Stderr
	cmd.Stdout = os.Stdout
	cmd.Stdin = os.Stdin
	return cmd.Run()
}
<|endoftext|>


<commit_before>// Copyright 2012 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

package scanner_test

import (
	"fmt"
)

import (
	"code.google.com/p/gcfg/scanner"
	"code.google.com/p/gcfg/token"
)

func ExampleScanner_Scan() {
	// src is the input that we want to tokenize.
	src := []byte("cos(x) + 1i*sin(x) // Euler")

	// Initialize the scanner.
	var s scanner.Scanner
	fset := token.NewFileSet()                      // positions are relative to fset
	file := fset.AddFile("", fset.Base(), len(src)) // register input "file"
	s.Init(file, src, nil /* no error handler */, scanner.ScanComments)

	// Repeated calls to Scan yield the token sequence found in the input.
	for {
		pos, tok, lit := s.Scan()
		if tok == token.EOF {
			break
		}
		fmt.Printf("%s\t%s\t%q\n", fset.Position(pos), tok, lit)
	}

	// output:
	// 1:1	IDENT	"cos"
	// 1:4	(	""
	// 1:5	IDENT	"x"
	// 1:6	)	""
	// 1:8	+	""
	// 1:10	IMAG	"1i"
	// 1:12	*	""
	// 1:13	IDENT	"sin"
	// 1:16	(	""
	// 1:17	IDENT	"x"
	// 1:18	)	""
	// 1:20	;	"\n"
	// 1:20	COMMENT	"// Euler"
}
<commit_msg>scanner: adapt example_test.go<commit_after>// Copyright 2012 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

package scanner_test

import (
	"fmt"
)

import (
	"code.google.com/p/gcfg/scanner"
	"code.google.com/p/gcfg/token"
)

func ExampleScanner_Scan() {
	// src is the input that we want to tokenize.
	src := []byte(`[profile "A"]
color = blue ; Comment`)

	// Initialize the scanner.
	var s scanner.Scanner
	fset := token.NewFileSet()                      // positions are relative to fset
	file := fset.AddFile("", fset.Base(), len(src)) // register input "file"
	s.Init(file, src, nil /* no error handler */, scanner.ScanComments)

	// Repeated calls to Scan yield the token sequence found in the input.
	for {
		pos, tok, lit := s.Scan()
		if tok == token.EOF {
			break
		}
		fmt.Printf("%s\t%s\t%q\n", fset.Position(pos), tok, lit)
	}

	// output:
	// 1:1	[	""
	// 1:2	IDENT	"profile"
	// 1:10	STRING	"\"A\""
	// 1:13	]	""
	// 1:14	EOL	""
	// 2:1	IDENT	"color"
	// 2:7	=	""
	// 2:9	STRING	"blue"
	// 2:14	COMMENT	"; Comment"
}
<|endoftext|>


<commit_before>package poll

import (
	"context"
	"sync"
	"time"

	"github.com/rusenask/keel/provider"
	"github.com/rusenask/keel/types"

	log "github.com/Sirupsen/logrus"
)

// DefaultManager - default manager is responsible for scanning deployments and identifying
// deployments that have market
type DefaultManager struct {
	providers provider.Providers

	// repository watcher
	watcher Watcher

	mu *sync.Mutex

	// scanTick - scan interval in seconds, defaults to 60 seconds
	scanTick int

	// root context
	ctx context.Context
}

// NewPollManager - new default poller
func NewPollManager(providers provider.Providers, watcher Watcher) *DefaultManager {
	return &DefaultManager{
		providers: providers,
		watcher:   watcher,
		mu:        &sync.Mutex{},
		scanTick:  55,
	}
}

// Start - start scanning deployment for changes
func (s *DefaultManager) Start(ctx context.Context) error {
	// setting root context
	s.ctx = ctx

	log.Info("trigger.poll.manager: polling trigger configured")

	// initial scan
	err := s.scan(ctx)
	if err != nil {
		log.WithFields(log.Fields{
			"error": err,
		}).Error("trigger.poll.manager: scan failed")
	}

	for _ = range time.Tick(time.Duration(s.scanTick) * time.Second) {
		select {
		case <-ctx.Done():
			return nil
		default:
			log.Debug("performing scan")
			err := s.scan(ctx)
			if err != nil {
				log.WithFields(log.Fields{
					"error": err,
				}).Error("trigger.poll.manager: kubernetes scan failed")
			}
		}
	}

	return nil
}

func (s *DefaultManager) scan(ctx context.Context) error {
	trackedImages, err := s.providers.TrackedImages()
	if err != nil {
		return err
	}

	for _, trackedImage := range trackedImages {
		if trackedImage.Trigger != types.TriggerTypePoll {
			continue
		}

		err := s.watcher.Watch(trackedImage.Image.Remote(), trackedImage.PollSchedule, "", "")
		if err != nil {
			log.WithFields(log.Fields{
				"error":    err,
				"schedule": trackedImage.PollSchedule,
				"image":    trackedImage.Image.Remote(),
			}).Error("trigger.poll.manager: failed to start watching repository")
			return err
		}
	}
	return nil
}
<commit_msg>getting credentials for images<commit_after>package poll

import (
	"context"
	"sync"
	"time"

	"github.com/rusenask/keel/provider"
	"github.com/rusenask/keel/secrets"
	"github.com/rusenask/keel/types"

	log "github.com/Sirupsen/logrus"
)

// DefaultManager - default manager is responsible for scanning deployments and identifying
// deployments that have market
type DefaultManager struct {
	providers provider.Providers

	secretsGetter secrets.Getter

	// repository watcher
	watcher Watcher

	mu *sync.Mutex

	// scanTick - scan interval in seconds, defaults to 60 seconds
	scanTick int

	// root context
	ctx context.Context
}

// NewPollManager - new default poller
func NewPollManager(providers provider.Providers, watcher Watcher, secretsGetter secrets.Getter) *DefaultManager {
	return &DefaultManager{
		providers:     providers,
		secretsGetter: secretsGetter,
		watcher:       watcher,
		mu:            &sync.Mutex{},
		scanTick:      55,
	}
}

// Start - start scanning deployment for changes
func (s *DefaultManager) Start(ctx context.Context) error {
	// setting root context
	s.ctx = ctx

	log.Info("trigger.poll.manager: polling trigger configured")

	// initial scan
	err := s.scan(ctx)
	if err != nil {
		log.WithFields(log.Fields{
			"error": err,
		}).Error("trigger.poll.manager: scan failed")
	}

	for _ = range time.Tick(time.Duration(s.scanTick) * time.Second) {
		select {
		case <-ctx.Done():
			return nil
		default:
			log.Debug("performing scan")
			err := s.scan(ctx)
			if err != nil {
				log.WithFields(log.Fields{
					"error": err,
				}).Error("trigger.poll.manager: kubernetes scan failed")
			}
		}
	}

	return nil
}

func (s *DefaultManager) scan(ctx context.Context) error {
	trackedImages, err := s.providers.TrackedImages()
	if err != nil {
		return err
	}

	for _, trackedImage := range trackedImages {
		if trackedImage.Trigger != types.TriggerTypePoll {
			continue
		}

		// anonymous credentials
		creds := &types.Credentials{}

		if trackedImage.Namespace != "" && len(trackedImage.Secrets) > 0 {
			imageCreds, err := s.secretsGetter.Get(trackedImage)
			if err != nil {
				log.WithFields(log.Fields{
					"error":   err,
					"secrets": trackedImage.Secrets,
					"image":   trackedImage.Image.Remote(),
				}).Error("trigger.poll.manager: failed to get authentication credentials")
			} else {
				creds = imageCreds
			}
		}

		err := s.watcher.Watch(trackedImage.Image.Remote(), trackedImage.PollSchedule, creds.Username, creds.Password)
		if err != nil {
			log.WithFields(log.Fields{
				"error":    err,
				"schedule": trackedImage.PollSchedule,
				"image":    trackedImage.Image.Remote(),
			}).Error("trigger.poll.manager: failed to start watching repository")
			return err
		}
	}
	return nil
}
<|endoftext|>


<commit_before>/*
Package flagtag provides support for creating command line flags by tagging appropriate struct fields with the 'flag' tag.
*/
package flagtag

import (
	"errors"
	"flag"
	"reflect"
	"strconv"
	"strings"
	"time"
	"unsafe"
)

// MustConfigureAndParse is like ConfigureAndParse, the only difference is that
// it will panic in case of an error.
func MustConfigureAndParse(config interface{}) {
	if err := ConfigureAndParse(config); err != nil {
		panic(err)
	}
}

// MustConfigure is like Configure, the only difference is that it will panic
// in case of an error.
func MustConfigure(config interface{}) {
	if err := Configure(config); err != nil {
		panic(err)
	}
}

// ConfigureAndParse will first attempt to configure the flags according to the
// provided config type. If any error occurs, this error will be returned and
// the command line arguments will not be parsed. If no error occurs, the
// command line arguments will be parsed and the config type will contain the
// result.
// Using this function may remove the need to even import the flag package at
// all.
func ConfigureAndParse(config interface{}) error {
	if err := Configure(config); err != nil {
		return err
	}
	flag.Parse()
	return nil
}

// Configure will configure the flag parameters according to the tags of the
// provided data type. It is allowed to call this method multiple times with
// different data types. (As long as flag's Parse() method has not been called
// yet.)
// Fields without a 'flag' tag or with an empty 'flag' tag will be ignored.
//
// The 'flag' tag consists of 3 parts, similar to the *Var-functions of the
// flag package. Parts are separated by a comma. The parts are:
//  - 1st: flag name
//  - 2nd: default value
//  - 3rd: usage description
//
// Example:
//  `flag:"verbose,false,Enable verbose output."`.
//
// This will create a flag 'verbose', which defaults to 'false' and shows usage
// information "Enable verbose output.".
//
// If an error occurs, this error will be returned and the configuration of
// other struct fields will be aborted.
func Configure(config interface{}) error {
	val, err := getStructValue(config)
	if err != nil {
		return err
	}
	return configure(val)
}

// configure (recursively) configures flags as they are discovered in the provided type and value.
// In case of an error, the error is returned. Possible errors are:
// - Invalid default values, error of type ErrInvalidDefault.
// - nil pointer provided.
// - nil interface provided.
// - interface to nil value provided.
// - Tagged variable uses unsupported data type.
func configure(structValue reflect.Value) error {
	var structType = structValue.Type()
	for i := 0; i < structType.NumField(); i++ {
		field := structType.Field(i)
		fieldType := field.Type
		fieldValue := structValue.Field(i)
		t := field.Tag.Get("flag")
		if t == "" {
			// if field is not tagged then we do not need to flag the type itself
			if fieldType.Kind() == reflect.Struct {
				// kind is a struct => recurse into inner struct
				if err := configure(fieldValue); err != nil {
					return err
				}
			}
		} else {
			// field is tagged, continue investigating what kind of flag to create
			tag := parseTag(t, field.Tag.Get("flagopt"))
			if tag.Name == "" {
				// tag is invalid, since there is no name
				return errors.New("field '" + field.Name + "': invalid flag name: empty string")
			}
			switch fieldType.Kind() {
			case reflect.Ptr:
				// unwrap pointer
				if fieldValue.IsNil() {
					return errors.New("field '" + field.Name + "' (tag '" + tag.Name + "'): cannot use nil pointer")
				}
				fieldType = fieldType.Elem()
				fieldValue = fieldValue.Elem()
			case reflect.Interface:
				// check if interface is valid
				if fieldValue.IsNil() {
					return errors.New("field '" + field.Name + "' (tag '" + tag.Name + "'): cannot use nil interface")
				}
				var value = reflect.ValueOf(fieldValue.Interface())
				switch value.Type().Kind() {
				case reflect.Ptr, reflect.Interface:
					if value.IsNil() {
						return errors.New("field '" + field.Name + "' (tag '" + tag.Name + "'): cannot use nil interface value")
					}
				}
			}
			if !fieldValue.CanSet() {
				return errors.New("field '" + field.Name + "' (tag '" + tag.Name + "') is unexported or unaddressable: cannot use this field")
			}
			if !tag.Options.skipFlagValue && registerFlagByValueInterface(fieldValue, &tag) {
				// no error during registration => Var-flag registered => continue with next field
				continue
			}
			if err := registerFlagByPrimitive(field.Name, fieldValue, &tag); err != nil {
				return err
			}
		}
	}
	return nil
}

// registerFlagByValueInterface checks if the provided type can be treated as flag.Value.
// If so, a flag.Value flag is set and true is returned. If no flag is set, false is returned.
func registerFlagByValueInterface(fieldValue reflect.Value, tag *flagTag) bool {
	var value flag.Value
	switch fieldValue.Type().Kind() {
	case reflect.Interface:
		var ok bool
		value, ok = fieldValue.Interface().(flag.Value)
		if !ok {
			return false
		}
	default:
		var ok bool
		value, ok = fieldValue.Addr().Interface().(flag.Value)
		if !ok {
			return false
		}
	}
	flag.Var(value, tag.Name, tag.Description)
	if tag.DefaultValue != "" {
		// a default value is provided, first call value.Set() with the provided default value
		value.Set(tag.DefaultValue)
	}
	return true
}

// registerFlagByPrimitive registers a single field as one of the primitive flag types. Types are matched by
// kind, so types derived from one of the basic types are still eligible for a flag.
//
// If it is not possible to register a flag because of an unknown data type, an error will be returned.
// If the specified default value is invalid, an error of type ErrInvalidDefault will be returned.
func registerFlagByPrimitive(fieldName string, fieldValue reflect.Value, tag *flagTag) error {
	var fieldType = fieldValue.Type()
	// Check time.Duration first, since it will also match one of the basic kinds.
	if durationVar, ok := fieldValue.Addr().Interface().(*time.Duration); ok {
		// field is a time.Duration
		defaultVal, err := time.ParseDuration(tag.DefaultValue)
		if err != nil {
			return &ErrInvalidDefault{fieldName, tag.Name, err}
		}
		flag.DurationVar(durationVar, tag.Name, defaultVal, tag.Description)
		return nil
	}
	// Check basic kinds.
	// TODO convert to detected kind without using unsafe
	var fieldPtr = unsafe.Pointer(fieldValue.UnsafeAddr())
	switch fieldType.Kind() {
	case reflect.String:
		flag.StringVar((*string)(fieldPtr), tag.Name, tag.DefaultValue, tag.Description)
	case reflect.Bool:
		defaultVal, err := strconv.ParseBool(tag.DefaultValue)
		if err != nil {
			return &ErrInvalidDefault{fieldName, tag.Name, err}
		}
		flag.BoolVar((*bool)(fieldPtr), tag.Name, defaultVal, tag.Description)
	case reflect.Float64:
		defaultVal, err := strconv.ParseFloat(tag.DefaultValue, 64)
		if err != nil {
			return &ErrInvalidDefault{fieldName, tag.Name, err}
		}
		flag.Float64Var((*float64)(fieldPtr), tag.Name, defaultVal, tag.Description)
	case reflect.Int:
		defaultVal, err := strconv.ParseInt(tag.DefaultValue, 0, fieldType.Bits())
		if err != nil {
			return &ErrInvalidDefault{fieldName, tag.Name, err}
		}
		flag.IntVar((*int)(fieldPtr), tag.Name, int(defaultVal), tag.Description)
	case reflect.Int64:
		defaultVal, err := strconv.ParseInt(tag.DefaultValue, 0, 64)
		if err != nil {
			return &ErrInvalidDefault{fieldName, tag.Name, err}
		}
		flag.Int64Var((*int64)(fieldPtr), tag.Name, defaultVal, tag.Description)
	case reflect.Uint:
		defaultVal, err := strconv.ParseUint(tag.DefaultValue, 0, fieldType.Bits())
		if err != nil {
			return &ErrInvalidDefault{fieldName, tag.Name, err}
		}
		flag.UintVar((*uint)(fieldPtr), tag.Name, uint(defaultVal), tag.Description)
	case reflect.Uint64:
		defaultVal, err := strconv.ParseUint(tag.DefaultValue, 0, 64)
		if err != nil {
			return &ErrInvalidDefault{fieldName, tag.Name, err}
		}
		flag.Uint64Var((*uint64)(fieldPtr), tag.Name, defaultVal, tag.Description)
	default:
		return errors.New("unsupported data type (kind '" + strconv.FormatUint(uint64(fieldType.Kind()), 10) + "') for field '" + fieldName + "' (tag '" + tag.Name + "')")
	}
	return nil
}

// getStructValue checks that the provided config instance is actually a struct not a nil value.
func getStructValue(config interface{}) (reflect.Value, error) {
	var zero reflect.Value
	if config == nil {
		return zero, errors.New("config cannot be nil")
	}
	ptr := reflect.ValueOf(config)
	if ptr.IsNil() {
		return zero, errors.New("config cannot point to nil")
	}
	val := reflect.Indirect(ptr)
	if val.Kind() != reflect.Struct {
		return zero, errors.New("config instance is not a struct")
	}
	return val, nil
}

// parseTag parses a string of text and separates the various sections of the 'flag'-tag.
func parseTag(value string, optvalue string) flagTag {
	parts := strings.SplitN(value, ",", 3)
	for len(parts) < 3 {
		parts = append(parts, "")
	}
	var flag = flagTag{Name: parts[0], DefaultValue: parts[1], Description: parts[2]}
	if optvalue != "" {
		if strings.Contains(optvalue, "skipFlagValue") {
			flag.Options.skipFlagValue = true
		}
	}
	return flag
}

// flagTag contains the parsed tag values
type flagTag struct {
	Name         string
	DefaultValue string
	Description  string
	Options      struct {
		skipFlagValue bool
	}
}

type ErrInvalidDefault struct {
	field string
	tag   string
	err   error
}

func (e *ErrInvalidDefault) Error() string {
	return "invalid default value for field '" + e.field + "' (tag '" + e.tag + "'): " + e.err.Error()
}
<commit_msg>Added comments and export SkipFlagValue.<commit_after>/*
Package flagtag provides support for creating command line flags by tagging appropriate struct fields with the 'flag' tag.
*/
package flagtag

import (
	"errors"
	"flag"
	"reflect"
	"strconv"
	"strings"
	"time"
	"unsafe"
)

// MustConfigureAndParse is like ConfigureAndParse, the only difference is that
// it will panic in case of an error.
func MustConfigureAndParse(config interface{}) {
	if err := ConfigureAndParse(config); err != nil {
		panic(err)
	}
}

// MustConfigure is like Configure, the only difference is that it will panic
// in case of an error.
func MustConfigure(config interface{}) {
	if err := Configure(config); err != nil {
		panic(err)
	}
}

// ConfigureAndParse will first attempt to configure the flags according to the
// provided config type. If any error occurs, this error will be returned and
// the command line arguments will not be parsed. If no error occurs, the
// command line arguments will be parsed and the config type will contain the
// result.
// Using this function may remove the need to even import the flag package at
// all.
func ConfigureAndParse(config interface{}) error {
	if err := Configure(config); err != nil {
		return err
	}
	flag.Parse()
	return nil
}

// Configure will configure the flag parameters according to the tags of the
// provided data type. It is allowed to call this method multiple times with
// different data types. (As long as flag's Parse() method has not been called
// yet.)
// Fields without a 'flag' tag or with an empty 'flag' tag will be ignored.
//
// The 'flag' tag consists of 3 parts, similar to the *Var-functions of the
// flag package. Parts are separated by a comma. The parts are:
//  - 1st: flag name
//  - 2nd: default value
//  - 3rd: usage description
//
// Example:
//  `flag:"verbose,false,Enable verbose output."`.
//
// This will create a flag 'verbose', which defaults to 'false' and shows usage
// information "Enable verbose output.".
//
// If an error occurs, this error will be returned and the configuration of
// other struct fields will be aborted.
func Configure(config interface{}) error {
	val, err := getStructValue(config)
	if err != nil {
		return err
	}
	return configure(val)
}

// configure (recursively) configures flags as they are discovered in the provided type and value.
// In case of an error, the error is returned. Possible errors are:
// - Invalid default values, error of type ErrInvalidDefault.
// - nil pointer provided.
// - nil interface provided.
// - interface to nil value provided.
// - Tagged variable uses unsupported data type.
func configure(structValue reflect.Value) error {
	var structType = structValue.Type()
	for i := 0; i < structType.NumField(); i++ {
		field := structType.Field(i)
		fieldType := field.Type
		fieldValue := structValue.Field(i)
		t := field.Tag.Get("flag")
		if t == "" {
			// if field is not tagged then we do not need to flag the type itself
			if fieldType.Kind() == reflect.Struct {
				// kind is a struct => recurse into inner struct
				if err := configure(fieldValue); err != nil {
					return err
				}
			}
		} else {
			// field is tagged, continue investigating what kind of flag to create
			tag := parseTag(t, field.Tag.Get("flagopt"))
			if tag.Name == "" {
				// tag is invalid, since there is no name
				return errors.New("field '" + field.Name + "': invalid flag name: empty string")
			}
			switch fieldType.Kind() {
			case reflect.Ptr:
				// unwrap pointer
				if fieldValue.IsNil() {
					return errors.New("field '" + field.Name + "' (tag '" + tag.Name + "'): cannot use nil pointer")
				}
				fieldType = fieldType.Elem()
				fieldValue = fieldValue.Elem()
			case reflect.Interface:
				// check if interface is valid
				if fieldValue.IsNil() {
					return errors.New("field '" + field.Name + "' (tag '" + tag.Name + "'): cannot use nil interface")
				}
				var value = reflect.ValueOf(fieldValue.Interface())
				switch value.Type().Kind() {
				case reflect.Ptr, reflect.Interface:
					if value.IsNil() {
						return errors.New("field '" + field.Name + "' (tag '" + tag.Name + "'): cannot use nil interface value")
					}
				}
			}
			if !fieldValue.CanSet() {
				return errors.New("field '" + field.Name + "' (tag '" + tag.Name + "') is unexported or unaddressable: cannot use this field")
			}
			if !tag.Options.SkipFlagValue && registerFlagByValueInterface(fieldValue, &tag) {
				// no error during registration => Var-flag registered => continue with next field
				continue
			}
			if err := registerFlagByPrimitive(field.Name, fieldValue, &tag); err != nil {
				return err
			}
		}
	}
	return nil
}

// registerFlagByValueInterface checks if the provided type can be treated as flag.Value.
// If so, a flag.Value flag is set and true is returned. If no flag is set, false is returned.
func registerFlagByValueInterface(fieldValue reflect.Value, tag *flagTag) bool {
	var value flag.Value
	switch fieldValue.Type().Kind() {
	case reflect.Interface:
		var ok bool
		value, ok = fieldValue.Interface().(flag.Value)
		if !ok {
			return false
		}
	default:
		var ok bool
		value, ok = fieldValue.Addr().Interface().(flag.Value)
		if !ok {
			return false
		}
	}
	flag.Var(value, tag.Name, tag.Description)
	if tag.DefaultValue != "" {
		// a default value is provided, first call value.Set() with the provided default value
		value.Set(tag.DefaultValue)
	}
	return true
}

// registerFlagByPrimitive registers a single field as one of the primitive flag types. Types are matched by
// kind, so types derived from one of the basic types are still eligible for a flag.
//
// If it is not possible to register a flag because of an unknown data type, an error will be returned.
// If the specified default value is invalid, an error of type ErrInvalidDefault will be returned.
func registerFlagByPrimitive(fieldName string, fieldValue reflect.Value, tag *flagTag) error {
	var fieldType = fieldValue.Type()
	// Check time.Duration first, since it will also match one of the basic kinds.
	if durationVar, ok := fieldValue.Addr().Interface().(*time.Duration); ok {
		// field is a time.Duration
		defaultVal, err := time.ParseDuration(tag.DefaultValue)
		if err != nil {
			return &ErrInvalidDefault{fieldName, tag.Name, err}
		}
		flag.DurationVar(durationVar, tag.Name, defaultVal, tag.Description)
		return nil
	}
	// Check basic kinds.
	// TODO convert to detected kind without using unsafe
	var fieldPtr = unsafe.Pointer(fieldValue.UnsafeAddr())
	switch fieldType.Kind() {
	case reflect.String:
		flag.StringVar((*string)(fieldPtr), tag.Name, tag.DefaultValue, tag.Description)
	case reflect.Bool:
		defaultVal, err := strconv.ParseBool(tag.DefaultValue)
		if err != nil {
			return &ErrInvalidDefault{fieldName, tag.Name, err}
		}
		flag.BoolVar((*bool)(fieldPtr), tag.Name, defaultVal, tag.Description)
	case reflect.Float64:
		defaultVal, err := strconv.ParseFloat(tag.DefaultValue, 64)
		if err != nil {
			return &ErrInvalidDefault{fieldName, tag.Name, err}
		}
		flag.Float64Var((*float64)(fieldPtr), tag.Name, defaultVal, tag.Description)
	case reflect.Int:
		defaultVal, err := strconv.ParseInt(tag.DefaultValue, 0, fieldType.Bits())
		if err != nil {
			return &ErrInvalidDefault{fieldName, tag.Name, err}
		}
		flag.IntVar((*int)(fieldPtr), tag.Name, int(defaultVal), tag.Description)
	case reflect.Int64:
		defaultVal, err := strconv.ParseInt(tag.DefaultValue, 0, 64)
		if err != nil {
			return &ErrInvalidDefault{fieldName, tag.Name, err}
		}
		flag.Int64Var((*int64)(fieldPtr), tag.Name, defaultVal, tag.Description)
	case reflect.Uint:
		defaultVal, err := strconv.ParseUint(tag.DefaultValue, 0, fieldType.Bits())
		if err != nil {
			return &ErrInvalidDefault{fieldName, tag.Name, err}
		}
		flag.UintVar((*uint)(fieldPtr), tag.Name, uint(defaultVal), tag.Description)
	case reflect.Uint64:
		defaultVal, err := strconv.ParseUint(tag.DefaultValue, 0, 64)
		if err != nil {
			return &ErrInvalidDefault{fieldName, tag.Name, err}
		}
		flag.Uint64Var((*uint64)(fieldPtr), tag.Name, defaultVal, tag.Description)
	default:
		return errors.New("unsupported data type (kind '" + strconv.FormatUint(uint64(fieldType.Kind()), 10) + "') for field '" + fieldName + "' (tag '" + tag.Name + "')")
	}
	return nil
}

// getStructValue checks that the provided config instance is actually a struct not a nil value.
func getStructValue(config interface{}) (reflect.Value, error) {
	var zero reflect.Value
	if config == nil {
		return zero, errors.New("config cannot be nil")
	}
	ptr := reflect.ValueOf(config)
	if ptr.IsNil() {
		return zero, errors.New("config cannot point to nil")
	}
	val := reflect.Indirect(ptr)
	if val.Kind() != reflect.Struct {
		return zero, errors.New("config instance is not a struct")
	}
	return val, nil
}

// parseTag parses a string of text and separates the various sections of the 'flag'-tag.
func parseTag(value string, optvalue string) flagTag {
	parts := strings.SplitN(value, ",", 3)
	for len(parts) < 3 {
		parts = append(parts, "")
	}
	var flag = flagTag{Name: parts[0], DefaultValue: parts[1], Description: parts[2]}
	if optvalue != "" {
		if strings.Contains(optvalue, "skipFlagValue") {
			flag.Options.SkipFlagValue = true
		}
	}
	return flag
}

// flagTag contains the parsed tag values.
type flagTag struct {
	Name         string
	DefaultValue string
	Description  string
	Options      struct {
		SkipFlagValue bool
	}
}

// ErrInvalidDefault is an error type for the case of invalid defaults.
type ErrInvalidDefault struct {
	field string
	tag   string
	err   error
}

// Error returns the error explaining the bad default value.
func (e *ErrInvalidDefault) Error() string {
	return "invalid default value for field '" + e.field + "' (tag '" + e.tag + "'): " + e.err.Error()
}
<|endoftext|>


<commit_before>package streamtunnel

import (
	"bufio"
	"fmt"
	"io"
	"io/ioutil"
	"net"
	"net/http"
	"os"
	"time"

	"github.com/hashicorp/yamux"
	"github.com/koding/logging"
)

// Client is responsible for creating a control connection to a tunnel server,
// creating new tunnels and proxy them to tunnel server.
type Client struct {
	// underlying tcp connection which is used for multiplexing
	nc net.Conn

	// underlying yamux session
	session *yamux.Session

	// serverAddr is the address of the tunnel-server
	serverAddr string

	// localAddr is the address of a local server that will be tunneled to the
	// public. Currently only one server is supported.
	localAddr string

	// yamuxConfig is passed to new yamux.Session's
	yamuxConfig *yamux.Config

	log logging.Logger
}

// ClientConfig defines the configuration for the Client
type ClientConfig struct {
	// ServerAddr defines the TCP address of the tunnel server to be connected
	ServerAddr string

	// LocalAddr defines the TCP address of the local server. This is optional
	// if you want to specify a single TCP address. Otherwise the client will
	// always proxy to 127.0.0.1:incomingPort, where incomingPort is the
	// tunnelserver's public exposed Port
	LocalAddr string

	// Debug enables debug mode, enable only if you want to debug the server
	Debug bool

	// Log defines the logger. If nil a default logging.Logger is used.
	Log logging.Logger

	// YamuxConfig defines the config which passed to every new yamux.Session. If nil
	// yamux.DefaultConfig() is used.
	YamuxConfig *yamux.Config
}

// NewClient creates a new tunnel that is established between the serverAddr
// and localAddr. It exits if it can't create a new control connection to the
// server. If localAddr is empty client will always try to proxy to a local
// port.
func NewClient(cfg *ClientConfig) *Client {
	yamuxConfig := yamux.DefaultConfig()
	if cfg.YamuxConfig != nil {
		if err := yamux.VerifyConfig(cfg.YamuxConfig); err != nil {
			fmt.Fprintf(os.Stderr, err.Error())
			os.Exit(1)
		}

		yamuxConfig = cfg.YamuxConfig
	}

	log := newLogger("streamtunnel-server", cfg.Debug)
	if cfg.Log != nil {
		log = cfg.Log
	}

	client := &Client{
		serverAddr:  cfg.ServerAddr,
		localAddr:   cfg.LocalAddr,
		log:         log,
		yamuxConfig: yamuxConfig,
	}

	return client
}

func (c *Client) Start(identifier string) error {
	var err error
	c.nc, err = net.Dial("tcp", c.serverAddr)
	if err != nil {
		return err
	}

	remoteAddr := fmt.Sprintf("http://%s%s", c.nc.RemoteAddr(), ControlPath)
	req, err := http.NewRequest("CONNECT", remoteAddr, nil)
	if err != nil {
		return fmt.Errorf("CONNECT %s", err)
	}

	req.Header.Set(XKTunnelIdentifier, identifier)
	if err := req.Write(c.nc); err != nil {
		return err
	}

	resp, err := http.ReadResponse(bufio.NewReader(c.nc), req)
	if err != nil {
		return fmt.Errorf("read response %s", err)
	}
	defer resp.Body.Close()

	if resp.StatusCode != 200 && resp.Status != Connected {
		out, err := ioutil.ReadAll(resp.Body)
		if err != nil {
			return err
		}

		return fmt.Errorf("proxy server: %s. err: %s", resp.Status, string(out))
	}

	c.session, err = yamux.Client(c.nc, c.yamuxConfig)
	if err != nil {
		return err
	}

	stream, err := c.session.Open()
	if err != nil {
		return err
	}

	if _, err := stream.Write([]byte(ctHandshakeRequest)); err != nil {
		return err
	}

	buf := make([]byte, len(ctHandshakeResponse))
	if _, err := stream.Read(buf); err != nil {
		return err
	}

	if string(buf) != ctHandshakeResponse {
		return fmt.Errorf("handshake aborted. got: %s", string(buf))
	}

	ct := newControl(stream)
	c.log.Debug("client has started successfully.")

	return c.listenControl(ct)
}

func (c *Client) listenControl(ct *control) error {
	for {
		var msg ControlMsg
		err := ct.dec.Decode(&msg)
		if err != nil {
			return fmt.Errorf("decode err: '%s'", err)
		}

		c.log.Debug("controlMsg: %+v", msg)

		switch msg.Action {
		case RequestClientSession:
			go func() {
				if err := c.proxy(msg.LocalPort); err != nil {
					fmt.Fprintf(os.Stderr, "proxy err: '%s'\n", err)
				}
			}()
		}
	}
}

func (c *Client) proxy(port string) error {
	conn, err := c.session.Open()
	if err != nil {
		return err
	}

	if port == "0" {
		port = "80"
	}

	localAddr := "127.0.0.1:" + port
	if c.localAddr != "" {
		localAddr = c.localAddr
	}

	local, err := newLocalDial(localAddr)
	if err != nil {
		return err
	}

	go func() {
		<-join(local, conn)
		conn.Close()
	}()

	return nil
}

func newLocalDial(addr string) (net.Conn, error) {
	c, err := net.Dial("tcp", addr)
	if err != nil {
		return nil, err
	}

	c.SetDeadline(time.Time{})
	return c, nil
}

func join(local, remote io.ReadWriteCloser) chan error {
	errc := make(chan error, 2)

	copy := func(dst io.Writer, src io.Reader) {
		_, err := io.Copy(dst, src)
		errc <- err
	}

	go copy(local, remote)
	go copy(remote, local)

	return errc
}
<commit_msg>streamtunnel: also fix client side channel opening<commit_after>package streamtunnel

import (
	"bufio"
	"errors"
	"fmt"
	"io"
	"io/ioutil"
	"net"
	"net/http"
	"os"
	"time"

	"github.com/hashicorp/yamux"
	"github.com/koding/logging"
)

// Client is responsible for creating a control connection to a tunnel server,
// creating new tunnels and proxy them to tunnel server.
type Client struct {
	// underlying tcp connection which is used for multiplexing
	nc net.Conn

	// underlying yamux session
	session *yamux.Session

	// serverAddr is the address of the tunnel-server
	serverAddr string

	// localAddr is the address of a local server that will be tunneled to the
	// public. Currently only one server is supported.
	localAddr string

	// yamuxConfig is passed to new yamux.Session's
	yamuxConfig *yamux.Config

	log logging.Logger
}

// ClientConfig defines the configuration for the Client
type ClientConfig struct {
	// ServerAddr defines the TCP address of the tunnel server to be connected
	ServerAddr string

	// LocalAddr defines the TCP address of the local server. This is optional
	// if you want to specify a single TCP address. Otherwise the client will
	// always proxy to 127.0.0.1:incomingPort, where incomingPort is the
	// tunnelserver's public exposed Port
	LocalAddr string

	// Debug enables debug mode, enable only if you want to debug the server
	Debug bool

	// Log defines the logger. If nil a default logging.Logger is used.
	Log logging.Logger

	// YamuxConfig defines the config which passed to every new yamux.Session. If nil
	// yamux.DefaultConfig() is used.
	YamuxConfig *yamux.Config
}

// NewClient creates a new tunnel that is established between the serverAddr
// and localAddr. It exits if it can't create a new control connection to the
// server. If localAddr is empty client will always try to proxy to a local
// port.
func NewClient(cfg *ClientConfig) *Client {
	yamuxConfig := yamux.DefaultConfig()
	if cfg.YamuxConfig != nil {
		if err := yamux.VerifyConfig(cfg.YamuxConfig); err != nil {
			fmt.Fprintf(os.Stderr, err.Error())
			os.Exit(1)
		}

		yamuxConfig = cfg.YamuxConfig
	}

	log := newLogger("streamtunnel-server", cfg.Debug)
	if cfg.Log != nil {
		log = cfg.Log
	}

	client := &Client{
		serverAddr:  cfg.ServerAddr,
		localAddr:   cfg.LocalAddr,
		log:         log,
		yamuxConfig: yamuxConfig,
	}

	return client
}

func (c *Client) Start(identifier string) error {
	var err error
	c.nc, err = net.Dial("tcp", c.serverAddr)
	if err != nil {
		return err
	}

	remoteAddr := fmt.Sprintf("http://%s%s", c.nc.RemoteAddr(), ControlPath)
	req, err := http.NewRequest("CONNECT", remoteAddr, nil)
	if err != nil {
		return fmt.Errorf("CONNECT %s", err)
	}

	req.Header.Set(XKTunnelIdentifier, identifier)
	if err := req.Write(c.nc); err != nil {
		return err
	}

	resp, err := http.ReadResponse(bufio.NewReader(c.nc), req)
	if err != nil {
		return fmt.Errorf("read response %s", err)
	}
	defer resp.Body.Close()

	if resp.StatusCode != 200 && resp.Status != Connected {
		out, err := ioutil.ReadAll(resp.Body)
		if err != nil {
			return err
		}

		return fmt.Errorf("proxy server: %s. err: %s", resp.Status, string(out))
	}

	c.session, err = yamux.Client(c.nc, c.yamuxConfig)
	if err != nil {
		return err
	}

	var stream net.Conn

	// if we don't receive anything from the server, we'll timeout
	select {
	case <-async(func() {
		// this is blocking until client opens a session to us
		stream, err = c.session.Open()
		if err != nil {
			c.log.Error("session open err: %s", err)
		}
	}):
	case <-time.After(time.Second * 10):
		if stream != nil {
			stream.Close()
		}
		return errors.New("timeout opening session")
	}

	if _, err := stream.Write([]byte(ctHandshakeRequest)); err != nil {
		return err
	}

	buf := make([]byte, len(ctHandshakeResponse))
	if _, err := stream.Read(buf); err != nil {
		return err
	}

	if string(buf) != ctHandshakeResponse {
		return fmt.Errorf("handshake aborted. got: %s", string(buf))
	}

	ct := newControl(stream)
	c.log.Debug("client has started successfully.")

	return c.listenControl(ct)
}

func (c *Client) listenControl(ct *control) error {
	for {
		var msg ControlMsg
		err := ct.dec.Decode(&msg)
		if err != nil {
			return fmt.Errorf("decode err: '%s'", err)
		}

		c.log.Debug("controlMsg: %+v", msg)

		switch msg.Action {
		case RequestClientSession:
			go func() {
				if err := c.proxy(msg.LocalPort); err != nil {
					fmt.Fprintf(os.Stderr, "proxy err: '%s'\n", err)
				}
			}()
		}
	}
}

func (c *Client) proxy(port string) error {
	conn, err := c.session.Open()
	if err != nil {
		return err
	}

	if port == "0" {
		port = "80"
	}

	localAddr := "127.0.0.1:" + port
	if c.localAddr != "" {
		localAddr = c.localAddr
	}

	local, err := newLocalDial(localAddr)
	if err != nil {
		return err
	}

	go func() {
		<-join(local, conn)
		conn.Close()
	}()

	return nil
}

func newLocalDial(addr string) (net.Conn, error) {
	c, err := net.Dial("tcp", addr)
	if err != nil {
		return nil, err
	}

	c.SetDeadline(time.Time{})
	return c, nil
}

func join(local, remote io.ReadWriteCloser) chan error {
	errc := make(chan error, 2)

	copy := func(dst io.Writer, src io.Reader) {
		_, err := io.Copy(dst, src)
		errc <- err
	}

	go copy(local, remote)
	go copy(remote, local)

	return errc
}
<|endoftext|>


<commit_before>package torrent

import (
	"bitbucket.org/anacrolix/go.torrent/peer_protocol"
	"crypto"
	"crypto/rand"
	"errors"
	metainfo "github.com/nsf/libtorgo/torrent"
	"io"
	"launchpad.net/gommap"
	"log"
	"net"
	"os"
	"path/filepath"
)

const (
	PieceHash = crypto.SHA1
)

type InfoHash [20]byte

type pieceSum [20]byte

func copyHashSum(dst, src []byte) {
	if len(dst) != len(src) || copy(dst, src) != len(dst) {
		panic("hash sum sizes differ")
	}
}

func BytesInfoHash(b []byte) (ih InfoHash) {
	if len(b) != len(ih) {
		panic("bad infohash bytes")
	}
	return
}

type pieceState uint8

const (
	pieceStateUnknown = iota
	pieceStateComplete
	pieceStateIncomplete
)

type piece struct {
	State pieceState
	Hash  pieceSum
}

type connection struct {
	Socket net.Conn
	post   chan peer_protocol.Message

	Interested bool
	Choked     bool
	Requests   []peer_protocol.Request

	PeerId         [20]byte
	PeerInterested bool
	PeerChoked     bool
	PeerRequests   []peer_protocol.Request
}

type torrent struct {
	InfoHash InfoHash
	Pieces   []piece
	Data     MMapSpan
	MetaInfo *metainfo.MetaInfo
	Conns    []connection
	Peers    []Peer
}

type Peer struct {
	Id   [20]byte
	IP   net.IP
	Port int
}

func (t torrent) PieceSize(piece int) (size int64) {
	if piece == len(t.Pieces)-1 {
		size = t.Data.Size() % t.MetaInfo.PieceLength
	}
	if size == 0 {
		size = t.MetaInfo.PieceLength
	}
	return
}

func (t torrent) PieceReader(piece int) io.Reader {
	return io.NewSectionReader(t.Data, int64(piece)*t.MetaInfo.PieceLength, t.MetaInfo.PieceLength)
}

func (t torrent) HashPiece(piece int) (ps pieceSum) {
	hash := PieceHash.New()
	n, err := io.Copy(hash, t.PieceReader(piece))
	if err != nil {
		panic(err)
	}
	if n != t.PieceSize(piece) {
		panic("hashed wrong number of bytes")
	}
	copyHashSum(ps[:], hash.Sum(nil))
	return
}

type client struct {
	DataDir       string
	HalfOpenLimit int
	PeerId        [20]byte

	halfOpen int
	torrents map[InfoHash]*torrent

	noTorrents      chan struct{}
	addTorrent      chan *torrent
	torrentFinished chan InfoHash
	actorTask       chan func()
}

func NewClient(dataDir string) *client {
	c := &client{
		DataDir:       dataDir,
		HalfOpenLimit: 10,

		torrents: make(map[InfoHash]*torrent),

		noTorrents:      make(chan struct{}),
		addTorrent:      make(chan *torrent),
		torrentFinished: make(chan InfoHash),
		actorTask:       make(chan func()),
	}
	_, err := rand.Read(c.PeerId[:])
	if err != nil {
		panic("error generating peer id")
	}
	go c.run()
	return c
}

func mmapTorrentData(metaInfo *metainfo.MetaInfo, location string) (mms MMapSpan, err error) {
	defer func() {
		if err != nil {
			mms.Close()
			mms = nil
		}
	}()
	for _, miFile := range metaInfo.Files {
		fileName := filepath.Join(append([]string{location, metaInfo.Name}, miFile.Path...)...)
		err = os.MkdirAll(filepath.Dir(fileName), 0666)
		if err != nil {
			return
		}
		var file *os.File
		file, err = os.OpenFile(fileName, os.O_CREATE|os.O_RDWR, 0666)
		if err != nil {
			return
		}
		var fi os.FileInfo
		fi, err = file.Stat()
		if err != nil {
			return
		}
		if fi.Size() < miFile.Length {
			err = file.Truncate(miFile.Length)
			if err != nil {
				return
			}
		}
		var mMap gommap.MMap
		mMap, err = gommap.MapRegion(file.Fd(), 0, miFile.Length, gommap.PROT_READ|gommap.PROT_WRITE, gommap.MAP_SHARED)
		if err != nil {
			return
		}
		if int64(len(mMap)) != miFile.Length {
			panic("mmap has wrong length")
		}
		mms = append(mms, MMap{mMap})
	}
	return
}

func (me *client) torrent(ih InfoHash) *torrent {
	for _, t := range me.torrents {
		if t.InfoHash == ih {
			return t
		}
	}
	return nil
}

func (me *client) initiateConn(peer Peer, torrent *torrent) {
	if peer.Id == me.PeerId {
		return
	}
	me.halfOpen++
	go func() {
		conn, err := net.DialTCP("tcp", nil, &net.TCPAddr{
			IP:   peer.IP,
			Port: peer.Port,
		})
		me.withContext(func() {
			me.halfOpen--
			me.openNewConns()
		})
		if err != nil {
			log.Printf("error connecting to peer: %s", err)
			return
		}
		me.runConnection(conn, torrent, peer.Id)
	}()
}

func (me *client) runConnection(conn net.Conn, torrent *torrent, peerId [20]byte) {
	log.Fatalf("connected to %s", conn)
}

func (me *client) openNewConns() {
	for _, t := range me.torrents {
		for len(t.Peers) != 0 {
			if me.halfOpen >= me.HalfOpenLimit {
				return
			}
			p := t.Peers[0]
			t.Peers = t.Peers[1:]
			me.initiateConn(p, t)
		}
	}
}

func (me *client) AddPeers(infoHash InfoHash, peers []Peer) (err error) {
	me.withContext(func() {
		t := me.torrent(infoHash)
		if t == nil {
			err = errors.New("no such torrent")
			return
		}
		t.Peers = append(t.Peers, peers...)
		me.openNewConns()
	})
	return
}

func (me *client) AddTorrent(metaInfo *metainfo.MetaInfo) error {
	torrent := &torrent{
		InfoHash: BytesInfoHash(metaInfo.InfoHash),
	}
	for offset := 0; offset < len(metaInfo.Pieces); offset += PieceHash.Size() {
		hash := metaInfo.Pieces[offset : offset+PieceHash.Size()]
		if len(hash) != PieceHash.Size() {
			return errors.New("bad piece hash in metainfo")
		}
		piece := piece{}
		copyHashSum(piece.Hash[:], hash)
		torrent.Pieces = append(torrent.Pieces, piece)
	}
	var err error
	torrent.Data, err = mmapTorrentData(metaInfo, me.DataDir)
	if err != nil {
		return err
	}
	torrent.MetaInfo = metaInfo
	me.addTorrent <- torrent
	return nil
}

func (me *client) WaitAll() {
	<-me.noTorrents
}

func (me *client) Close() {
}

func (me *client) withContext(f func()) {
	me.actorTask <- f
}

func (me *client) pieceHashed(ih InfoHash, piece int, correct bool) {
	torrent := me.torrents[ih]
	torrent.Pieces[piece].State = func() pieceState {
		if correct {
			return pieceStateComplete
		} else {
			return pieceStateIncomplete
		}
	}()
	for _, piece := range torrent.Pieces {
		if piece.State == pieceStateUnknown {
			return
		}
	}
	go func() {
		me.torrentFinished <- ih
	}()
}

func (me *client) run() {
	for {
		noTorrents := me.noTorrents
		if len(me.torrents) != 0 {
			noTorrents = nil
		}
		select {
		case noTorrents <- struct{}{}:
		case torrent := <-me.addTorrent:
			if _, ok := me.torrents[torrent.InfoHash]; ok {
				break
			}
			me.torrents[torrent.InfoHash] = torrent
			for i := range torrent.Pieces {
				go func(piece int) {
					sum := torrent.HashPiece(piece)
					me.withContext(func() {
						me.pieceHashed(torrent.InfoHash, piece, sum == torrent.Pieces[piece].Hash)
					})
				}(i)
			}
		case infoHash := <-me.torrentFinished:
			delete(me.torrents, infoHash)
		case task := <-me.actorTask:
			task()
		}
	}
}
<commit_msg>Prevent false-positive race conditions<commit_after>package torrent

import (
	"bitbucket.org/anacrolix/go.torrent/peer_protocol"
	"crypto"
	"crypto/rand"
	"errors"
	metainfo "github.com/nsf/libtorgo/torrent"
	"io"
	"launchpad.net/gommap"
	"log"
	"net"
	"os"
	"path/filepath"
)

const (
	PieceHash = crypto.SHA1
)

type InfoHash [20]byte

type pieceSum [20]byte

func copyHashSum(dst, src []byte) {
	if len(dst) != len(src) || copy(dst, src) != len(dst) {
		panic("hash sum sizes differ")
	}
}

func BytesInfoHash(b []byte) (ih InfoHash) {
	if len(b) != len(ih) {
		panic("bad infohash bytes")
	}
	return
}

type pieceState uint8

const (
	pieceStateUnknown = iota
	pieceStateComplete
	pieceStateIncomplete
)

type piece struct {
	State pieceState
	Hash  pieceSum
}

type connection struct {
	Socket net.Conn
	post   chan peer_protocol.Message

	Interested bool
	Choked     bool
	Requests   []peer_protocol.Request

	PeerId         [20]byte
	PeerInterested bool
	PeerChoked     bool
	PeerRequests   []peer_protocol.Request
}

type torrent struct {
	InfoHash InfoHash
	Pieces   []piece
	Data     MMapSpan
	MetaInfo *metainfo.MetaInfo
	Conns    []connection
	Peers    []Peer
}

type Peer struct {
	Id   [20]byte
	IP   net.IP
	Port int
}

func (t torrent) PieceSize(piece int) (size int64) {
	if piece == len(t.Pieces)-1 {
		size = t.Data.Size() % t.MetaInfo.PieceLength
	}
	if size == 0 {
		size = t.MetaInfo.PieceLength
	}
	return
}

func (t *torrent) PieceReader(piece int) io.Reader {
	return io.NewSectionReader(t.Data, int64(piece)*t.MetaInfo.PieceLength, t.MetaInfo.PieceLength)
}

func (t *torrent) HashPiece(piece int) (ps pieceSum) {
	hash := PieceHash.New()
	n, err := io.Copy(hash, t.PieceReader(piece))
	if err != nil {
		panic(err)
	}
	if n != t.PieceSize(piece) {
		panic("hashed wrong number of bytes")
	}
	copyHashSum(ps[:], hash.Sum(nil))
	return
}

type client struct {
	DataDir       string
	HalfOpenLimit int
	PeerId        [20]byte

	halfOpen int
	torrents map[InfoHash]*torrent

	noTorrents      chan struct{}
	addTorrent      chan *torrent
	torrentFinished chan InfoHash
	actorTask       chan func()
}

func NewClient(dataDir string) *client {
	c := &client{
		DataDir:       dataDir,
		HalfOpenLimit: 10,

		torrents: make(map[InfoHash]*torrent),

		noTorrents:      make(chan struct{}),
		addTorrent:      make(chan *torrent),
		torrentFinished: make(chan InfoHash),
		actorTask:       make(chan func()),
	}
	_, err := rand.Read(c.PeerId[:])
	if err != nil {
		panic("error generating peer id")
	}
	go c.run()
	return c
}

func mmapTorrentData(metaInfo *metainfo.MetaInfo, location string) (mms MMapSpan, err error) {
	defer func() {
		if err != nil {
			mms.Close()
			mms = nil
		}
	}()
	for _, miFile := range metaInfo.Files {
		fileName := filepath.Join(append([]string{location, metaInfo.Name}, miFile.Path...)...)
		err = os.MkdirAll(filepath.Dir(fileName), 0666)
		if err != nil {
			return
		}
		var file *os.File
		file, err = os.OpenFile(fileName, os.O_CREATE|os.O_RDWR, 0666)
		if err != nil {
			return
		}
		var fi os.FileInfo
		fi, err = file.Stat()
		if err != nil {
			return
		}
		if fi.Size() < miFile.Length {
			err = file.Truncate(miFile.Length)
			if err != nil {
				return
			}
		}
		var mMap gommap.MMap
		mMap, err = gommap.MapRegion(file.Fd(), 0, miFile.Length, gommap.PROT_READ|gommap.PROT_WRITE, gommap.MAP_SHARED)
		if err != nil {
			return
		}
		if int64(len(mMap)) != miFile.Length {
			panic("mmap has wrong length")
		}
		mms = append(mms, MMap{mMap})
	}
	return
}

func (me *client) torrent(ih InfoHash) *torrent {
	for _, t := range me.torrents {
		if t.InfoHash == ih {
			return t
		}
	}
	return nil
}

func (me *client) initiateConn(peer Peer, torrent *torrent) {
	if peer.Id == me.PeerId {
		return
	}
	me.halfOpen++
	go func() {
		conn, err := net.DialTCP("tcp", nil, &net.TCPAddr{
			IP:   peer.IP,
			Port: peer.Port,
		})
		me.withContext(func() {
			me.halfOpen--
			me.openNewConns()
		})
		if err != nil {
			log.Printf("error connecting to peer: %s", err)
			return
		}
		me.runConnection(conn, torrent, peer.Id)
	}()
}

func (me *client) runConnection(conn net.Conn, torrent *torrent, peerId [20]byte) {
	log.Fatalf("connected to %s", conn)
}

func (me *client) openNewConns() {
	for _, t := range me.torrents {
		for len(t.Peers) != 0 {
			if me.halfOpen >= me.HalfOpenLimit {
				return
			}
			p := t.Peers[0]
			t.Peers = t.Peers[1:]
			me.initiateConn(p, t)
		}
	}
}

func (me *client) AddPeers(infoHash InfoHash, peers []Peer) (err error) {
	me.withContext(func() {
		t := me.torrent(infoHash)
		if t == nil {
			err = errors.New("no such torrent")
			return
		}
		t.Peers = append(t.Peers, peers...)
		me.openNewConns()
	})
	return
}

func (me *client) AddTorrent(metaInfo *metainfo.MetaInfo) error {
	torrent := &torrent{
		InfoHash: BytesInfoHash(metaInfo.InfoHash),
	}
	for offset := 0; offset < len(metaInfo.Pieces); offset += PieceHash.Size() {
		hash := metaInfo.Pieces[offset : offset+PieceHash.Size()]
		if len(hash) != PieceHash.Size() {
			return errors.New("bad piece hash in metainfo")
		}
		piece := piece{}
		copyHashSum(piece.Hash[:], hash)
		torrent.Pieces = append(torrent.Pieces, piece)
	}
	var err error
	torrent.Data, err = mmapTorrentData(metaInfo, me.DataDir)
	if err != nil {
		return err
	}
	torrent.MetaInfo = metaInfo
	me.addTorrent <- torrent
	return nil
}

func (me *client) WaitAll() {
	<-me.noTorrents
}

func (me *client) Close() {
}

func (me *client) withContext(f func()) {
	me.actorTask <- f
}

func (me *client) pieceHashed(ih InfoHash, piece int, correct bool) {
	torrent := me.torrents[ih]
	torrent.Pieces[piece].State = func() pieceState {
		if correct {
			return pieceStateComplete
		} else {
			return pieceStateIncomplete
		}
	}()
	for _, piece := range torrent.Pieces {
		if piece.State == pieceStateUnknown {
			return
		}
	}
	go func() {
		me.torrentFinished <- ih
	}()
}

func (me *client) run() {
	for {
		noTorrents := me.noTorrents
		if len(me.torrents) != 0 {
			noTorrents = nil
		}
		select {
		case noTorrents <- struct{}{}:
		case torrent := <-me.addTorrent:
			if _, ok := me.torrents[torrent.InfoHash]; ok {
				break
			}
			me.torrents[torrent.InfoHash] = torrent
			for i := range torrent.Pieces {
				go func(piece int) {
					sum := torrent.HashPiece(piece)
					me.withContext(func() {
						me.pieceHashed(torrent.InfoHash, piece, sum == torrent.Pieces[piece].Hash)
					})
				}(i)
			}
		case infoHash := <-me.torrentFinished:
			delete(me.torrents, infoHash)
		case task := <-me.actorTask:
			task()
		}
	}
}
<|endoftext|>


<commit_before>// Copyright 2014-2015 Apptimist, Inc. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

package main

import (
	"io/ioutil"
	"path/filepath"
	"strconv"
	"strings"

	"gopkg.in/yaml.v1"
)

var (
	SystemConfigFN = filepath.Join("etc", DefaultConfigFN)
	SystemReposDN  = filepath.Join("srv", AsnStr)
)

type Config struct {
	Name string
	// (e.g. "siren")
	Dir string
	// Local repository directory (e.g. /srv/asn/siren or siren.asn)
	Lat float64 `yaml:"lat,omitempty"`
	Lon float64 `yaml:"lon,omitempty"`
	// Latitude and Longitude of this server or administrator.
	Listen []*URL `yaml:"listen,omitempty"`
	// List of listening URLs.  All servers should listen to WebSockets
	// (e.g. ws://). Servers should also listen on a Unix socket file for
	// test and administration (e.g.unix:///). If available, also listen on
	// a TCP socket (e.g. tcp://) for mirror activity.
	// Assume admin mode if empty or not present.
	Server []struct {
		Name     string `yaml:"name,omitempty"`
		Url      *URL
		Lat, Lon float64
	} `yaml:"server,omitempty"`
	// In daemon mode this lists the other servers to replicate blobs.
	// In admin mode this lists the servers that may get/put blobs.
	Keys *ServiceKeys `yaml:"keys,omitempty"`
	// Usually generated with -new-keys then edited to remove the
	// unnecessary secrete keys.
}

// Bytes marshals the Config for output to a file.
func (c *Config) Bytes() []byte {
	buf, err := yaml.Marshal(c)
	if err != nil {
		return []byte(err.Error())
	}
	return buf
}

func (c *Config) Check(m Mode) (err error) {
	switch {
	case len(c.Name) == 0:
		err = &Error{"config", "no name"}
	case len(c.Dir) == 0:
		err = &Error{c.Name, "no repos"}
	case c.Keys == nil:
		err = &Error{c.Name, "no keys"}
	case c.Keys.Admin == nil:
		err = &Error{c.Name, "no admin keys"}
	case c.Keys.Admin.Pub == nil:
		err = &Error{c.Name, "no admin public keys"}
	case c.Keys.Admin.Pub.Encr == nil:
		err = &Error{c.Name, "no admin public encr key"}
	case c.Keys.Admin.Pub.Auth == nil:
		err = &Error{c.Name, "no admin public auth key"}
	case c.Keys.Server == nil:
		err = &Error{c.Name, "no server keys"}
	case c.Keys.Server.Pub == nil:
		err = &Error{c.Name, "no server public keys"}
	case c.Keys.Server.Pub.Encr == nil:
		err = &Error{c.Name, "no server public encr key"}
	case c.Keys.Server.Pub.Auth == nil:
		err = &Error{c.Name, "no server public auth key"}
	case m.Admin() && len(c.Server) == 0:
		err = &Error{c.Name, "no servers"}
	case m.Server() && len(c.Listen) == 0:
		err = &Error{c.Name, "no listeners"}
	}
	return
}

func (c *Config) Parse(fn string) (err error) {
	var b []byte
	var def struct{ name, dir string }
	var ok bool
	if b, ok = Builtin[fn]; ok {
		def.name = fn
		def.dir = fn + ReposExt
	} else if fn == DefaultConfigFN {
		b, err = ioutil.ReadFile(DefaultConfigFN)
		def.name = AsnStr
		if err != nil {
			b, err = ioutil.ReadFile(SystemConfigFN)
			if err != nil {
				return
			}
			def.dir = SystemReposDN
		} else {
			def.dir = DefaultReposDN
		}
	} else if b, err = ioutil.ReadFile(fn); err == nil {
		def.name = strings.TrimSuffix(fn, ConfigExt)
		def.dir = def.name + ReposExt
	} else {
		return
	}
	if err := yaml.Unmarshal(b, c); err != nil {
		return err
	}
	if c.Name == "" {
		c.Name = def.name
	}
	if c.Dir == "" {
		c.Dir = def.dir
	}
	return nil
}

// retrieve Server Index of named, url or numbered server.
func (c *Config) SI(s string) (i int, err error) {
	if s == "" {
		return
	}
	if i, err = strconv.Atoi(s); err == nil {
		if i < 0 || i >= len(c.Server) {
			err = &Error{s, strconv.ErrRange.Error()}
		}
		return
	}
	for x, se := range c.Server {
		if s == se.Name || s == se.Url.String() {
			i, err = x, nil
			return
		}
	}
	err = &Error{s, "no such server"}
	return
}

// String marshals the Config.
func (c *Config) String() string {
	buf, err := yaml.Marshal(c)
	if err != nil {
		return err.Error()
	}
	return string(buf)
}
<commit_msg>presume ".yaml" extension<commit_after>// Copyright 2014-2015 Apptimist, Inc. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

package main

import (
	"io/ioutil"
	"path/filepath"
	"strconv"
	"strings"

	"gopkg.in/yaml.v1"
)

var (
	SystemConfigFN = filepath.Join("etc", DefaultConfigFN)
	SystemReposDN  = filepath.Join("srv", AsnStr)
)

type Config struct {
	Name string
	// (e.g. "siren")
	Dir string
	// Local repository directory (e.g. /srv/asn/siren or siren.asn)
	Lat float64 `yaml:"lat,omitempty"`
	Lon float64 `yaml:"lon,omitempty"`
	// Latitude and Longitude of this server or administrator.
	Listen []*URL `yaml:"listen,omitempty"`
	// List of listening URLs.  All servers should listen to WebSockets
	// (e.g. ws://). Servers should also listen on a Unix socket file for
	// test and administration (e.g.unix:///). If available, also listen on
	// a TCP socket (e.g. tcp://) for mirror activity.
	// Assume admin mode if empty or not present.
	Server []struct {
		Name     string `yaml:"name,omitempty"`
		Url      *URL
		Lat, Lon float64
	} `yaml:"server,omitempty"`
	// In daemon mode this lists the other servers to replicate blobs.
	// In admin mode this lists the servers that may get/put blobs.
	Keys *ServiceKeys `yaml:"keys,omitempty"`
	// Usually generated with -new-keys then edited to remove the
	// unnecessary secrete keys.
}

// Bytes marshals the Config for output to a file.
func (c *Config) Bytes() []byte {
	buf, err := yaml.Marshal(c)
	if err != nil {
		return []byte(err.Error())
	}
	return buf
}

func (c *Config) Check(m Mode) (err error) {
	switch {
	case len(c.Name) == 0:
		err = &Error{"config", "no name"}
	case len(c.Dir) == 0:
		err = &Error{c.Name, "no repos"}
	case c.Keys == nil:
		err = &Error{c.Name, "no keys"}
	case c.Keys.Admin == nil:
		err = &Error{c.Name, "no admin keys"}
	case c.Keys.Admin.Pub == nil:
		err = &Error{c.Name, "no admin public keys"}
	case c.Keys.Admin.Pub.Encr == nil:
		err = &Error{c.Name, "no admin public encr key"}
	case c.Keys.Admin.Pub.Auth == nil:
		err = &Error{c.Name, "no admin public auth key"}
	case c.Keys.Server == nil:
		err = &Error{c.Name, "no server keys"}
	case c.Keys.Server.Pub == nil:
		err = &Error{c.Name, "no server public keys"}
	case c.Keys.Server.Pub.Encr == nil:
		err = &Error{c.Name, "no server public encr key"}
	case c.Keys.Server.Pub.Auth == nil:
		err = &Error{c.Name, "no server public auth key"}
	case m.Admin() && len(c.Server) == 0:
		err = &Error{c.Name, "no servers"}
	case m.Server() && len(c.Listen) == 0:
		err = &Error{c.Name, "no listeners"}
	}
	return
}

func (c *Config) Parse(fn string) (err error) {
	var b []byte
	var def struct{ name, dir string }
	var ok bool
	if b, ok = Builtin[fn]; ok {
		def.name = fn
		def.dir = fn + ReposExt
	} else if fn == DefaultConfigFN {
		b, err = ioutil.ReadFile(DefaultConfigFN)
		def.name = AsnStr
		if err != nil {
			b, err = ioutil.ReadFile(SystemConfigFN)
			if err != nil {
				return
			}
			def.dir = SystemReposDN
		} else {
			def.dir = DefaultReposDN
		}
	} else if b, err = ioutil.ReadFile(fn); err == nil {
		def.name = strings.TrimSuffix(fn, ConfigExt)
		def.dir = def.name + ReposExt
	} else if b, err = ioutil.ReadFile(fn + ConfigExt); err == nil {
		def.name = fn
		def.dir = def.name + ReposExt
	} else {
		return
	}
	if err := yaml.Unmarshal(b, c); err != nil {
		return err
	}
	if c.Name == "" {
		c.Name = def.name
	}
	if c.Dir == "" {
		c.Dir = def.dir
	}
	return nil
}

// retrieve Server Index of named, url or numbered server.
func (c *Config) SI(s string) (i int, err error) {
	if s == "" {
		return
	}
	if i, err = strconv.Atoi(s); err == nil {
		if i < 0 || i >= len(c.Server) {
			err = &Error{s, strconv.ErrRange.Error()}
		}
		return
	}
	for x, se := range c.Server {
		if s == se.Name || s == se.Url.String() {
			i, err = x, nil
			return
		}
	}
	err = &Error{s, "no such server"}
	return
}

// String marshals the Config.
func (c *Config) String() string {
	buf, err := yaml.Marshal(c)
	if err != nil {
		return err.Error()
	}
	return string(buf)
}
<|endoftext|>


<commit_before>package main

import (
	"bytes"
	"encoding/json"
	"fmt"
	"github.com/nutrun/lentil"
	"os"
	"strings"
)

type Client struct {
	q       *lentil.Beanstalkd
	verbose bool
}

func NewClient(verbose bool) (*Client, error) {
	this := new(Client)
	q, err := lentil.Dial(Config.QueueAddr)
	if err != nil {
		return nil, err
	}
	this.q = q
	this.verbose = verbose
	return this, nil
}

func (this *Client) put(msg *Message) error {
	message, e := json.Marshal(msg)
	if this.verbose {
		fmt.Fprintf(os.Stderr, "QUEUEING UP: %s\n", message)
	}
	if e != nil {
		return e
	}
	if msg.Tube != "default" {
		e = this.q.Use(msg.Tube)
		if e != nil {
			return e
		}
	}
	_, e = this.q.Put(msg.Priority, msg.Delay, 60*60, message) // An hour TTR?
	return e
}

func (this *Client) putMany(input []byte) error {
	jobs, e := MessagesFromJSON(input)
	if e != nil {
		return e
	}
	for _, job := range jobs {
		e = this.put(job)
		if e != nil {
			return e
		}
	}
	return nil
}

func (this *Client) stats() error {
	q := NewJobQueue(this.q, false, make([]string, 0))
	stats, err := json.Marshal(q)
	if err != nil {
		return err
	}
	buffer := bytes.NewBufferString("")
	err = json.Indent(buffer, stats, "", "\t")
	if err != nil {
		return err
	}
	fmt.Printf("%s\n", buffer.String())
	return nil
}

func (this *Client) drain(tubes string) error {
	drainedJobs := []byte("[\n")
	isFirstDrained := true
	for _, tube := range strings.Split(tubes, ",") {
		_, err := this.q.Watch(tube)
		if err != nil {
			return err
		}
		_, err = this.q.Ignore("default")
		if err != nil {
			return err
		}

		for job, err := this.q.ReserveWithTimeout(0); err == nil; job, err = this.q.ReserveWithTimeout(0) {
			this.q.Delete(job.Id)
			if !isFirstDrained {
				drainedJobs = append(drainedJobs, []byte(",\n")...)
			}
			drainedJobs = append(drainedJobs, job.Body...)
			isFirstDrained = false
		}
		if err != nil {
			return err
		}
	}
	drainedJobs = append(drainedJobs, []byte("\n]")...)
	fmt.Fprintf(os.Stderr, "%s", string(drainedJobs))
	return nil
}

func (this *Client) pause(tubes string, delay int) error {
	for _, tube := range strings.Split(tubes, ",") {
		e := this.q.PauseTube(tube, delay)
		if e != nil {
			return e
		}
		fmt.Fprintf(os.Stderr, "Paused %s for %d seconds", tubes, delay)
	}
	return nil
}
<commit_msg>fixed drain bug<commit_after>package main

import (
	"bytes"
	"encoding/json"
	"fmt"
	"github.com/nutrun/lentil"
	"os"
	"strings"
)

type Client struct {
	q       *lentil.Beanstalkd
	verbose bool
}

func NewClient(verbose bool) (*Client, error) {
	this := new(Client)
	q, err := lentil.Dial(Config.QueueAddr)
	if err != nil {
		return nil, err
	}
	this.q = q
	this.verbose = verbose
	return this, nil
}

func (this *Client) put(msg *Message) error {
	message, e := json.Marshal(msg)
	if this.verbose {
		fmt.Fprintf(os.Stderr, "QUEUEING UP: %s\n", message)
	}
	if e != nil {
		return e
	}
	if msg.Tube != "default" {
		e = this.q.Use(msg.Tube)
		if e != nil {
			return e
		}
	}
	_, e = this.q.Put(msg.Priority, msg.Delay, 60*60, message) // An hour TTR?
	return e
}

func (this *Client) putMany(input []byte) error {
	jobs, e := MessagesFromJSON(input)
	if e != nil {
		return e
	}
	for _, job := range jobs {
		e = this.put(job)
		if e != nil {
			return e
		}
	}
	return nil
}

func (this *Client) stats() error {
	q := NewJobQueue(this.q, false, make([]string, 0))
	stats, err := json.Marshal(q)
	if err != nil {
		return err
	}
	buffer := bytes.NewBufferString("")
	err = json.Indent(buffer, stats, "", "\t")
	if err != nil {
		return err
	}
	fmt.Printf("%s\n", buffer.String())
	return nil
}

func (this *Client) drain(tubes string) error {
	drainedJobs := make([]string, 0)
	for _, tube := range strings.Split(tubes, ",") {
		_, err := this.q.Watch(tube)
		if err != nil {
			return err
		}
		_, err = this.q.Ignore("default")
		if err != nil {
			return err
		}
		for {
			job, err := this.q.ReserveWithTimeout(0)
			if err != nil && strings.HasPrefix(err.Error(), "TIMED_OUT") {
				break
			}
			if err != nil {
				return err
			}
			err = this.q.Delete(job.Id)
			if err != nil {
				return err
			}
			drainedJobs = append(drainedJobs, string(job.Body))
		}
	}
	fmt.Fprintf(os.Stderr, "[%s]", strings.Join(drainedJobs, ","))
	return nil
}

func (this *Client) pause(tubes string, delay int) error {
	for _, tube := range strings.Split(tubes, ",") {
		e := this.q.PauseTube(tube, delay)
		if e != nil {
			return e
		}
		fmt.Fprintf(os.Stderr, "Paused %s for %d seconds", tubes, delay)
	}
	return nil
}
<|endoftext|>