<commit_before>// DCRSPY is a program to continuously monitor and log changes in various data
// on the Decred network.  It works by connecting to both dcrd and dcrwallet,
// and responding when a new block is detected via a notifier registered with
// dcrd.
//
// Two types of information are monitored:
//	1. Block chain data (from dcrd)
//  2. Stake information (from your wallet)
//
// Multiple destinations for the data are planned:
//	1. stdout.  JSON-formatted data is send to stdout. IMPLEMENTED.
//	2. File system.  JSON-formatted data is written to the file system.
//	   NOT YET IMPLEMENTED.
//  3. Database. Data is inserted into a MySQL database.  NOT YET IMPLEMENTED.
//
// See README.md and TODO for more information.
//
// by Jonathan Chappelow (chappjc)
//
// Borrowing logging and config file facilities, plus much boilerplate from
// main.go, from on dcrticketbuyer, by the Decred developers.

package main

import (
	"fmt"
	"io/ioutil"
	"os"
	_ "os/exec"
	"os/signal"
	"sync"
	"time"

	//"github.com/davecgh/go-spew/spew"
	"github.com/decred/dcrd/chaincfg/chainhash"
	"github.com/decred/dcrrpcclient"
)

const (
	// blockConnChanBuffer is the size of the block connected channel buffer.
	blockConnChanBuffer = 100
)

func main() {
	// Parse the configuration file.
	cfg, err := loadConfig()
	if err != nil {
		fmt.Printf("Failed to load dcrspy config: %s\n", err.Error())
		os.Exit(1)
	}
	defer backendLog.Flush()

	dcrrpcclient.UseLogger(clientLog)

	// Connect to dcrd RPC server using websockets. Set up the
	// notification handler to deliver blocks through a channel.
	var connectChan chan int32
	var stakeDiffChan chan int64
	if !cfg.NoCollectBlockData {
		connectChan = make(chan int32, blockConnChanBuffer)
		stakeDiffChan = make(chan int64, 2)
	}
	var connectChanStkInf chan int32
	if !cfg.NoCollectStakeInfo {
		connectChanStkInf = make(chan int32, blockConnChanBuffer)
	}

	ntfnHandlersDaemon := dcrrpcclient.NotificationHandlers{
		OnBlockConnected: func(hash *chainhash.Hash, height int32,
			time time.Time, vb uint16) {
			select {
			case connectChan <- height:
				// TODO: also call user-specified system command
				// what are required arguments? how to select?
				// cmdName, args := "pwd", ""
				// cmd := exec.Command(cmdName, args)
				// err := cmd.Start()
				// if err != nil {
				// 	log.Errorf("Failed to start system command %v. Error: %v",
				// 		cmdName, err)
				// }
				// cerr := make(chan error)
				// go func() { cerr <- cmd.Wait() }()
			default:
			}
			select {
			case connectChanStkInf <- height:
			default:
			}
		},
		// BUG: This appears to be currently broken, notifiying on every block.
		OnStakeDifficulty: func(hash *chainhash.Hash, height int64,
			stakeDiff int64) {
			select {
			case stakeDiffChan <- stakeDiff:
			default:
			}
		},
	}

sdifftest:
	for {
		select {
		case <-stakeDiffChan:
			fmt.Print("Something was already in the stake diff chan!")
		default:
			break sdifftest
		}
	}

	var dcrdCerts []byte
	if !cfg.DisableClientTLS {
		dcrdCerts, err = ioutil.ReadFile(cfg.DcrdCert)
		if err != nil {
			fmt.Printf("Failed to read dcrd cert file at %s: %s\n", cfg.DcrdCert,
				err.Error())
			os.Exit(1)
		}
	}

	log.Infof("Attempting to connect to dcrd RPC %s as user %s "+
		"using certificate located in %s",
		cfg.DcrdServ, cfg.DcrdUser, cfg.DcrdCert)

	connCfgDaemon := &dcrrpcclient.ConnConfig{
		Host:         cfg.DcrdServ,
		Endpoint:     "ws",
		User:         cfg.DcrdUser,
		Pass:         cfg.DcrdPass,
		Certificates: dcrdCerts,
		DisableTLS:   cfg.DisableClientTLS,
	}
	dcrdClient, err := dcrrpcclient.New(connCfgDaemon, &ntfnHandlersDaemon)
	if err != nil {
		fmt.Printf("Failed to start dcrd RPC client: %s\n", err.Error())
		os.Exit(1)
	}

	// Register for block connection notifications.
	if err := dcrdClient.NotifyBlocks(); err != nil {
		fmt.Printf("Failed to register daemon RPC client for  "+
			"block notifications: %s\n", err.Error())
		os.Exit(1)
	}

	// Register for stake difficulty change notifications.
	if err := dcrdClient.NotifyStakeDifficulty(); err != nil {
		fmt.Printf("Failed to register daemon RPC client for  "+
			"stake difficulty change notifications: %s\n", err.Error())
		os.Exit(1)
	}

	// Connect to the dcrwallet server RPC client.
	var dcrwCerts []byte
	if !cfg.DisableClientTLS {
		dcrwCerts, err = ioutil.ReadFile(cfg.DcrwCert)
		if err != nil {
			log.Errorf("Failed to read dcrwallet cert file at %s: %s\n",
				cfg.DcrwCert, err.Error())
			// but try anyway?
		}
	}

	log.Infof("Attempting to connect to dcrwallet RPC %s as user %s "+
		"using certificate located in %s",
		cfg.DcrwServ, cfg.DcrwUser, cfg.DcrwCert)

	connCfgWallet := &dcrrpcclient.ConnConfig{
		Host:         cfg.DcrwServ,
		Endpoint:     "ws",
		User:         cfg.DcrwUser,
		Pass:         cfg.DcrwPass,
		Certificates: dcrwCerts,
		DisableTLS:   cfg.DisableClientTLS,
	}

	dcrwClient, err := dcrrpcclient.New(connCfgWallet, nil)
	if err != nil {
		fmt.Printf("Failed to start dcrwallet RPC client: %s\n", err.Error())
		os.Exit(1)
	}

	// err = syncGlobalsStartup(dcrdClient, dcrwClient, cfg)
	// if err != nil {
	// 	log.Errorf("Failed to start sync globals on startup: %s\n", err.Error())
	// 	os.Exit(1)
	// }

	// Ctrl-C to shut down.
	// Nothing should be sent the quit channel.  It should only be closed.
	quit := make(chan struct{})
	// Only accept a single CTRL+C
	c := make(chan os.Signal, 1)
	signal.Notify(c, os.Interrupt)

	// Start waiting for the signal
	go func() {
		for range c {
			signal.Stop(c)
			// Close the channel so multiple goroutines can get the message
			log.Infof("CTRL+C hit.  Closing goroutines.")
			close(quit)
		}
	}()

	// WaitGroup for the chainMonitor and stakeMonitor
	var wg sync.WaitGroup

	// Block data collector
	var collector *blockDataCollector
	if !cfg.NoCollectBlockData {
		wg.Add(1)
		collector, err = newBlockDataCollector(cfg, dcrdClient)
		if err != nil {
			fmt.Printf("Failed to create block data collector: %s\n", err.Error())
			os.Exit(1)
		}

		// Blockchain monitor for the collector
		// if collector is nil, so is connectChan
		stdoutBlockDataSaver := &BlockDataToJSONStdOut{}
		wsChainMonitor := newChainMonitor(collector, connectChan,
			stdoutBlockDataSaver, quit, &wg)
		go wsChainMonitor.blockConnectedHandler()
	}

	// Stake info data (getstakeinfo) collector
	var stakeCollector *stakeInfoDataCollector
	if !cfg.NoCollectStakeInfo {
		wg.Add(1)
		stakeCollector, err = newStakeInfoDataCollector(cfg, dcrdClient, dcrwClient)
		if err != nil {
			fmt.Printf("Failed to create block data collector: %s\n", err.Error())
			os.Exit(1)
		}

		// Stake info monitor for the stakeCollector
		stdoutStakeInfoSaver := &StakeInfoDataToJSONStdOut{}
		wsStakeInfoMonitor := newStakeMonitor(stakeCollector, connectChanStkInf,
			stdoutStakeInfoSaver, quit, &wg)
		go wsStakeInfoMonitor.blockConnectedHandler()
	}

	// stakediff not implemented yet as the notifier appears broken
	go func() {
		for {
			select {
			case s, ok := <-stakeDiffChan:
				if !ok {
					log.Infof("Stake difficulty channel closed")
					return
				}
				log.Infof("Got stake difficulty change notification (%v). "+
					" Doing nothing for now.", s)
			case <-quit:
				log.Infof("Quitting getstakeinfo handler.")
				return
			}
		}
	}()

	log.Infof("RPC client(s) successfully connected. Now monitoring and " +
		"collecting data.")

	// When os.Interrupt arrives, the channel quit will be closed
	//<-quit
	wg.Wait()

	// Closing these channels should be unnecessary if quit was handled right
	if stakeDiffChan != nil {
		close(stakeDiffChan)
	}
	if connectChan != nil {
		close(connectChan)
	}
	if connectChanStkInf != nil {
		close(connectChanStkInf)
	}

	log.Infof("Closing connection to dcrd.")
	dcrdClient.Shutdown()
	log.Infof("Closing connection to dcrwallet.")
	dcrwClient.Shutdown()

	log.Infof("Bye!")
	time.Sleep(1 * time.Second)
	os.Exit(1)
}
<commit_msg>Allow --nostakeinfo to work.<commit_after>// DCRSPY is a program to continuously monitor and log changes in various data
// on the Decred network.  It works by connecting to both dcrd and dcrwallet,
// and responding when a new block is detected via a notifier registered with
// dcrd.
//
// Two types of information are monitored:
//	1. Block chain data (from dcrd)
//  2. Stake information (from your wallet)
//
// Multiple destinations for the data are planned:
//	1. stdout.  JSON-formatted data is send to stdout. IMPLEMENTED.
//	2. File system.  JSON-formatted data is written to the file system.
//	   NOT YET IMPLEMENTED.
//  3. Database. Data is inserted into a MySQL database.  NOT YET IMPLEMENTED.
//
// See README.md and TODO for more information.
//
// by Jonathan Chappelow (chappjc)
//
// Borrowing logging and config file facilities, plus much boilerplate from
// main.go, from on dcrticketbuyer, by the Decred developers.

package main

import (
	"fmt"
	"io/ioutil"
	"os"
	_ "os/exec"
	"os/signal"
	"sync"
	"time"

	//"github.com/davecgh/go-spew/spew"
	"github.com/decred/dcrd/chaincfg/chainhash"
	"github.com/decred/dcrrpcclient"
)

const (
	// blockConnChanBuffer is the size of the block connected channel buffer.
	blockConnChanBuffer = 100
)

func main() {
	// Parse the configuration file.
	cfg, err := loadConfig()
	if err != nil {
		fmt.Printf("Failed to load dcrspy config: %s\n", err.Error())
		os.Exit(1)
	}
	defer backendLog.Flush()

	dcrrpcclient.UseLogger(clientLog)

	// Connect to dcrd RPC server using websockets. Set up the
	// notification handler to deliver blocks through a channel.
	var connectChan chan int32
	var stakeDiffChan chan int64
	if !cfg.NoCollectBlockData {
		connectChan = make(chan int32, blockConnChanBuffer)
		stakeDiffChan = make(chan int64, 2)
	}
	var connectChanStkInf chan int32
	if !cfg.NoCollectStakeInfo {
		connectChanStkInf = make(chan int32, blockConnChanBuffer)
	}

	ntfnHandlersDaemon := dcrrpcclient.NotificationHandlers{
		OnBlockConnected: func(hash *chainhash.Hash, height int32,
			time time.Time, vb uint16) {
			select {
			case connectChan <- height:
				// TODO: also call user-specified system command
				// what are required arguments? how to select?
				// cmdName, args := "pwd", ""
				// cmd := exec.Command(cmdName, args)
				// err := cmd.Start()
				// if err != nil {
				// 	log.Errorf("Failed to start system command %v. Error: %v",
				// 		cmdName, err)
				// }
				// cerr := make(chan error)
				// go func() { cerr <- cmd.Wait() }()
			default:
			}
			select {
			case connectChanStkInf <- height:
			default:
			}
		},
		// BUG: This appears to be currently broken, notifiying on every block.
		OnStakeDifficulty: func(hash *chainhash.Hash, height int64,
			stakeDiff int64) {
			select {
			case stakeDiffChan <- stakeDiff:
			default:
			}
		},
	}

sdifftest:
	for {
		select {
		case <-stakeDiffChan:
			fmt.Print("Something was already in the stake diff chan!")
		default:
			break sdifftest
		}
	}

	var dcrdCerts []byte
	if !cfg.DisableClientTLS {
		dcrdCerts, err = ioutil.ReadFile(cfg.DcrdCert)
		if err != nil {
			fmt.Printf("Failed to read dcrd cert file at %s: %s\n", cfg.DcrdCert,
				err.Error())
			os.Exit(1)
		}
	}

	log.Infof("Attempting to connect to dcrd RPC %s as user %s "+
		"using certificate located in %s",
		cfg.DcrdServ, cfg.DcrdUser, cfg.DcrdCert)

	connCfgDaemon := &dcrrpcclient.ConnConfig{
		Host:         cfg.DcrdServ,
		Endpoint:     "ws",
		User:         cfg.DcrdUser,
		Pass:         cfg.DcrdPass,
		Certificates: dcrdCerts,
		DisableTLS:   cfg.DisableClientTLS,
	}
	dcrdClient, err := dcrrpcclient.New(connCfgDaemon, &ntfnHandlersDaemon)
	if err != nil {
		fmt.Printf("Failed to start dcrd RPC client: %s\n", err.Error())
		os.Exit(1)
	}

	// Register for block connection notifications.
	if err := dcrdClient.NotifyBlocks(); err != nil {
		fmt.Printf("Failed to register daemon RPC client for  "+
			"block notifications: %s\n", err.Error())
		os.Exit(1)
	}

	// Register for stake difficulty change notifications.
	if err := dcrdClient.NotifyStakeDifficulty(); err != nil {
		fmt.Printf("Failed to register daemon RPC client for  "+
			"stake difficulty change notifications: %s\n", err.Error())
		os.Exit(1)
	}

	var dcrwClient *dcrrpcclient.Client
	if !cfg.NoCollectStakeInfo {
		// Connect to the dcrwallet server RPC client.
		var dcrwCerts []byte
		if !cfg.DisableClientTLS {
			dcrwCerts, err = ioutil.ReadFile(cfg.DcrwCert)
			if err != nil {
				log.Errorf("Failed to read dcrwallet cert file at %s: %s\n",
					cfg.DcrwCert, err.Error())
				// but try anyway?
			}
		}

		log.Infof("Attempting to connect to dcrwallet RPC %s as user %s "+
			"using certificate located in %s",
			cfg.DcrwServ, cfg.DcrwUser, cfg.DcrwCert)

		connCfgWallet := &dcrrpcclient.ConnConfig{
			Host:         cfg.DcrwServ,
			Endpoint:     "ws",
			User:         cfg.DcrwUser,
			Pass:         cfg.DcrwPass,
			Certificates: dcrwCerts,
			DisableTLS:   cfg.DisableClientTLS,
		}

		dcrwClient, err = dcrrpcclient.New(connCfgWallet, nil)
		if err != nil {
			fmt.Printf("Failed to start dcrwallet RPC client: %s\nPerhaps you"+
			" wanted to start with --nostakeinfo?\n", err.Error())
			fmt.Printf("Verify that rpc.cert is for your wallet:\n\t%v",cfg.DcrwCert)
			os.Exit(1)
		}
	}

	// err = syncGlobalsStartup(dcrdClient, dcrwClient, cfg)
	// if err != nil {
	// 	log.Errorf("Failed to start sync globals on startup: %s\n", err.Error())
	// 	os.Exit(1)
	// }

	// Ctrl-C to shut down.
	// Nothing should be sent the quit channel.  It should only be closed.
	quit := make(chan struct{})
	// Only accept a single CTRL+C
	c := make(chan os.Signal, 1)
	signal.Notify(c, os.Interrupt)

	// Start waiting for the signal
	go func() {
		for range c {
			signal.Stop(c)
			// Close the channel so multiple goroutines can get the message
			log.Infof("CTRL+C hit.  Closing goroutines.")
			close(quit)
		}
	}()

	// WaitGroup for the chainMonitor and stakeMonitor
	var wg sync.WaitGroup

	// Block data collector
	var collector *blockDataCollector
	if !cfg.NoCollectBlockData {
		wg.Add(1)
		collector, err = newBlockDataCollector(cfg, dcrdClient)
		if err != nil {
			fmt.Printf("Failed to create block data collector: %s\n", err.Error())
			os.Exit(1)
		}

		// Blockchain monitor for the collector
		// if collector is nil, so is connectChan
		stdoutBlockDataSaver := &BlockDataToJSONStdOut{}
		wsChainMonitor := newChainMonitor(collector, connectChan,
			stdoutBlockDataSaver, quit, &wg)
		go wsChainMonitor.blockConnectedHandler()
	}

	// Stake info data (getstakeinfo) collector
	var stakeCollector *stakeInfoDataCollector
	if !cfg.NoCollectStakeInfo {
		wg.Add(1)
		stakeCollector, err = newStakeInfoDataCollector(cfg, dcrdClient, dcrwClient)
		if err != nil {
			fmt.Printf("Failed to create block data collector: %s\n", err.Error())
			os.Exit(1)
		}

		// Stake info monitor for the stakeCollector
		stdoutStakeInfoSaver := &StakeInfoDataToJSONStdOut{}
		wsStakeInfoMonitor := newStakeMonitor(stakeCollector, connectChanStkInf,
			stdoutStakeInfoSaver, quit, &wg)
		go wsStakeInfoMonitor.blockConnectedHandler()
	}

	// stakediff not implemented yet as the notifier appears broken
	go func() {
		for {
			select {
			case s, ok := <-stakeDiffChan:
				if !ok {
					log.Infof("Stake difficulty channel closed")
					return
				}
				log.Infof("Got stake difficulty change notification (%v). "+
					" Doing nothing for now.", s)
			case <-quit:
				log.Infof("Quitting getstakeinfo handler.")
				return
			}
		}
	}()

	log.Infof("RPC client(s) successfully connected. Now monitoring and " +
		"collecting data.")

	// When os.Interrupt arrives, the channel quit will be closed
	//<-quit
	wg.Wait()

	// Closing these channels should be unnecessary if quit was handled right
	if stakeDiffChan != nil {
		close(stakeDiffChan)
	}
	if connectChan != nil {
		close(connectChan)
	}
	if connectChanStkInf != nil {
		close(connectChanStkInf)
	}

	log.Infof("Closing connection to dcrd.")
	dcrdClient.Shutdown()

	if !cfg.NoCollectStakeInfo {
		log.Infof("Closing connection to dcrwallet.")
		dcrwClient.Shutdown()
	}

	log.Infof("Bye!")
	time.Sleep(1 * time.Second)
	os.Exit(1)
}
<|endoftext|>


<commit_before>package tabulate

import "math"

func Tabulate(screenWidth int, items []string) string {

	// find the largest column in the data
	var colWidth = 0

	for _, item := range items {
		l := len(item)
		if l > colWidth {
			colWidth = l + 2
		}
	}

	var base = float64(screenWidth) / float64(colWidth)
	var maxColumns = int(math.Floor(base))

	// build the results
	return makerow(items, colWidth, maxColumns)
}

func makerow(group []string, width int, maxColumns int) string {

	var output = ""
	var length = len(group)

	if length == 0 {
		return ""
	}

	var base = float64(length) / float64(maxColumns)
	var minRows = int(math.Ceil(base))

	for row := 0; row < minRows; row++ {
		for col := 0; col < maxColumns; col++ {

			var idx = row*maxColumns + col

			if idx >= length {
				break
			}

			var item = group[idx]
			output += item

			// add the padding to the end of the item
			if col < maxColumns-1 {
				itemLen := len(item)
				for s := 0; s < width-itemLen; s++ {
					output += " "
				}
			}
		}

		output += "\r\n"
	}

	return output
}
<commit_msg>slight cleanup<commit_after>package tabulate

import "math"

func Tabulate(screenWidth int, items []string) string {

	// find the largest column in the data
	var colWidth = 0

	for _, item := range items {
		l := len(item)
		if l > colWidth {
			colWidth = l + 2
		}
	}

	var maxBase = float64(screenWidth) / float64(colWidth)
	var maxColumns = int(math.Floor(maxBase))

	var output = ""
	var length = len(items)

	if length == 0 {
		return ""
	}

	var minBase = float64(length) / float64(maxColumns)
	var minRows = int(math.Ceil(minBase))

	for row := 0; row < minRows; row++ {
		for col := 0; col < maxColumns; col++ {

			var idx = row*maxColumns + col

			if idx >= length {
				break
			}

			var item = items[idx]
			output += item

			// add the padding to the end of the item
			if col < maxColumns-1 {
				itemLen := len(item)
				for s := 0; s < colWidth-itemLen; s++ {
					output += " "
				}
			}
		}

		output += "\r\n"
	}

	return output
}
<|endoftext|>


<commit_before>package main

import (
	"code.google.com/p/go-uuid/uuid"
	"encoding/json"
	"fmt"
	"github.com/gorilla/mux"
	gorillasession "github.com/gorilla/sessions"
	"github.com/ory-am/common/env"
	"github.com/ory-am/common/mgopath"
	"github.com/ory-am/event"
	gde "github.com/ory-am/gitdeploy/event"
	"github.com/ory-am/gitdeploy/job"
	gdLog "github.com/ory-am/gitdeploy/log"
	"github.com/ory-am/gitdeploy/sse"
	"github.com/ory-am/gitdeploy/storage"
	"github.com/ory-am/gitdeploy/storage/mongo"
	"github.com/ory-am/google-json-style-response/responder"
	"gopkg.in/mgo.v2"
	"log"
	"net/http"
	"os/exec"
	"regexp"
	"time"
	"runtime"
	"os"
    "strings"
)

const (
	sessionCurrentDeployment = "cdid"
	sessionName = "gdp"
)

var (
// API Version
	ApiVersion = "1.0"

// Generic configuration
	host = env.Getenv("HOST", "")
	port = env.Getenv("PORT", "7654")

	envAppTtl = env.Getenv("APP_TTL", "30m")
    envClusterConf = env.Getenv("FLYNN_CLUSTER_CONFIG", "")

// Configuration for CORS
	corsAllowOrigin = env.Getenv("CORS_ALLOW_ORIGIN", "http://localhost:9000")

	sessionStore = gorillasession.NewCookieStore([]byte(env.Getenv("SESSION_SECRET", "changme")))

// MongoDB
	envMongoPath = env.Getenv("MONGODB", "mongodb://localhost:27017/gitdeploy")
)

type deployRequest struct {
	Repository string `json:"repository"`
}

type appResponse struct {
	App string `json:"app"`
}

func main() {
	checkDependencies()
	eventManager := event.New()

	// mgo
	db, dbName, err := mgopath.Connect(envMongoPath)
	if err != nil {
		log.Fatal(err)
	}

	storage := mongo.New(db, dbName)
	eventManager.AttachListenerAggregate(storage)

	// SSE broker
	sseBroker := sse.New(storage)

	// Log listener
	eventManager.AttachListenerAggregate(new(gdLog.Listener))

	// Mux router
	r := mux.NewRouter()
	r.HandleFunc("/config", configHandler).Methods("GET")
	r.HandleFunc("/deployments", deployWrapperAction(sseBroker, eventManager, storage)).Methods("POST")
	r.HandleFunc("/deployments", setCORSHeaders).Methods("OPTIONS")
	r.HandleFunc("/deployments/{app:.+}/events", eventWrapperAction(sseBroker)).Methods("GET")
	r.HandleFunc("/apps/{app:.+}", getAppHandler(storage)).Methods("GET")
	r.PathPrefix("/").HandlerFunc(publicHandler("./app/dist"))
	http.Handle("/", r)

	go job.KillAppsOnHitList(storage)

	listen := fmt.Sprintf("%s:%s", host, port)
	log.Printf("Listening on %s", listen)
	log.Fatal(http.ListenAndServe(listen, nil))
}

func publicHandler(dir string) func(http.ResponseWriter, *http.Request) {
	return func(w http.ResponseWriter, r *http.Request) {
		path := dir + r.URL.Path
		if r.URL.Path == "/" {
			path = dir + "/index.html"
		}
		pattern := `!\.html|\.js|\.svg|\.css|\.png|\.jpg$`

		if f, err := os.Stat(path); err == nil {
			if !f.IsDir() {
				http.ServeFile(w, r, path)
				return
			} else {
                http.NotFound(w, r)
				return
			}
		}

		if matched, err := regexp.MatchString(pattern, path); err != nil {
			log.Printf("Could not exec regex: %s", err.Error())
		} else if !matched {
            http.ServeFile(w, r, dir + "/index.html")
            return
		} else {
			http.NotFound(w, r)
		}
	}
}

func configHandler(w http.ResponseWriter, r *http.Request) {
	setCORSHeaders(w, r)
	responseSuccess(w, struct {
		Now time.Time `json:"now"`
	}{
		time.Now(),
	})
}

func getAppHandler(store *mongo.MongoStorage) func(w http.ResponseWriter, r *http.Request) {
	return func(w http.ResponseWriter, r *http.Request) {
		if cleanUpSession(w, r) != nil {
			return
		}
		setCORSHeaders(w, r)
		vars := mux.Vars(r)
		id := vars["app"]
		app, err := store.GetApp(id)
		if err == mgo.ErrNotFound {
			responseError(w, http.StatusNotFound, fmt.Sprintf("App %s does not exist.", id))
			return
		} else if app.Killed {
			responseError(w, http.StatusNotFound, "App timed out and is no longer available.")
			return
		} else if err != nil {
			responseError(w, http.StatusInternalServerError, err.Error())
			return
		}
		logs, err := job.GetLogs(app.ID)
		if err != nil {
			responseError(w, http.StatusInternalServerError, err.Error())
			return
		}

		ps, err := job.GetPS(app.ID)
		if err != nil {
			responseError(w, http.StatusInternalServerError, err.Error())
			return
		}

		deployLogs, err := store.FindDeployLogsForApp(id)
		if err != nil {
			responseError(w, http.StatusInternalServerError, err.Error())
			return
		}

		responseSuccess(w, struct {
			*storage.App
			Logs       string                 `json:"logs"`
			DeployLogs []*storage.DeployEvent `json:"deployLogs"`
			PS         string                 `json:"ps"`
		}{app, logs, deployLogs, ps})
	}
}

func cleanUpSession(w http.ResponseWriter, r *http.Request) error {
	session, err := sessionStore.Get(r, sessionName)
	if err != nil {
		responseError(w, http.StatusBadRequest, "Please delete your cookies.")
		return err
	}
	delete(session.Values, sessionCurrentDeployment)
	if err := session.Save(r, w); err != nil {
		responseError(w, http.StatusInternalServerError, err.Error())
		return err
	}
	return nil
}

func eventWrapperAction(sseBroker *sse.Broker) func(w http.ResponseWriter, r *http.Request) {
	return func(w http.ResponseWriter, r *http.Request) {
		setCORSHeaders(w, r)
		sseBroker.EventHandler(w, r)
	}
}

func deployWrapperAction(sseBroker *sse.Broker, em *event.EventManager, store *mongo.MongoStorage) func(w http.ResponseWriter, r *http.Request) {
	return func(w http.ResponseWriter, r *http.Request) {
		setCORSHeaders(w, r)
		deployAction(w, r, sseBroker, em, store)
	}
}

func deployAction(w http.ResponseWriter, r *http.Request, sseBroker *sse.Broker, em *event.EventManager, store *mongo.MongoStorage) {
	session, err := sessionStore.Get(r, sessionName)
	if err != nil {
		responseError(w, http.StatusBadRequest, "Please delete your cookies.")
		return
	}

	// Check if the user is currently deploying an application and switch to that one.
	if v, ok := session.Values[sessionCurrentDeployment].(string); ok && len(v) > 0 {
		app, err := store.GetApp(v)
		if err != nil {
            cleanUpSession(w, r)
            log.Printf("Could not fetch app from cookie: %s", err.Error())
        } else if !sseBroker.IsChannelOpen(app.ID) {
            cleanUpSession(w, r)
            log.Printf("Channel %s does not exist any more", app.ID)
		} else {
			responseSuccess(w, app)
            return
		}
	}

	// Parse body
	dr := new(deployRequest)
	decoder := json.NewDecoder(r.Body)
	decoder.Decode(dr)

	// Validate URL
	regExpression := "(https|http):\\/\\/github\\.com\\/[a-zA-Z0-9\\-\\_]+/[a-zA-Z0-9\\-\\_]+\\.git"
	if match, _ := regexp.MatchString(regExpression, dr.Repository); !match {
		responseError(w, http.StatusBadRequest, "I only support GitHub.")
		return
	}

	app := uuid.NewRandom().String()
	if ttl, err := time.ParseDuration(envAppTtl); err != nil {
		log.Println(err.Error())
		responseError(w, http.StatusInternalServerError, err.Error())
		return
	} else {
		appEntity, err := store.AddApp(app, time.Now().Add(ttl), dr.Repository, getIP(r))
		if err != nil {
			responseError(w, http.StatusInternalServerError, err.Error())
			return
		}
		session.Values[sessionCurrentDeployment] = appEntity.ID
		if err := session.Save(r, w); err != nil {
			responseError(w, http.StatusInternalServerError, err.Error())
			return
		}
		go runJobs(w, r, em, dr, appEntity, sseBroker, session, store)
		responseSuccess(w, appEntity)
	}
}

func runJobs(w http.ResponseWriter, r *http.Request, em *event.EventManager, dr *deployRequest, app *storage.App, sseBroker *sse.Broker, session *gorillasession.Session, store *mongo.MongoStorage) {
	sseBroker.OpenChannel(app.ID)
	sseBroker.Start(app.ID)
	defer func() {
		// Give the client the chance to read the output...
		time.Sleep(15 * time.Second)
		sseBroker.CloseChannel(app.ID)
	}()

	em.Trigger("app.created", gde.New(app.ID, app.ID))

	destination, err := job.Clone(em, app.ID, dr.Repository)
	if err != nil {
		log.Printf("Error in job.clone %s: %s", app.ID, err.Error())
		return
	}

	if err = job.Parse(em, app.ID, destination); err != nil {
		log.Printf("Error in job.parse %s: %s", app.ID, err.Error())
		return
	}

	if err = job.Deploy(em, app.ID, destination); err != nil {
		log.Printf("Error in job.deploy %s: %s", app.ID, err.Error())
		return
	}

	cluster, err := job.GetCluster(em, app.ID)
	if err != nil {
		log.Printf("Error in job.getCluster %s: %s", app.ID, err.Error())
		return
	}

	app.URL = fmt.Sprintf("%s.%s", app.ID, cluster.Host)
	if err := store.UpdateApp(app); err != nil {
		responseError(w, http.StatusInternalServerError, err.Error())
		log.Printf("Error %s: %s", app.ID, err.Error())
		return
	}

    if err = job.Cleanup(em, app.ID, destination); err != nil {
        log.Printf("Error in job.cleanup %s: %s", app.ID, err.Error())
        return
    }

	log.Println("Deployment successful.")
	em.Trigger("app.deployed", gde.New(app.ID, app.URL))
}

// Set the different CORS headers required for CORS request
func setCORSHeaders(w http.ResponseWriter, r *http.Request) {
	w.Header().Add("Access-Control-Allow-Methods", "POST, GET, OPTIONS")
	w.Header().Set("Access-Control-Allow-Origin", corsAllowOrigin)
	w.Header().Set("Access-Control-Allow-Credentials", "true")
	w.Header().Set("Access-Control-Allow-Headers", "Origin, X-Requested-With, Content-Type, Accept")
}

func responseError(w http.ResponseWriter, code int, message string) {
	log.Printf("Error %d: %s", code, message)
	response := responder.New(ApiVersion)
	response.Write(w, response.Error(code, message))
}

func responseSuccess(w http.ResponseWriter, data interface{}) {
	response := responder.New(ApiVersion)
	response.Write(w, response.Success(data))
}

func checkDependencies() {
	go func() {
		_, err := exec.LookPath("git")
		if err != nil {
			log.Fatal("Git CLI is required but not installed or not in path.")
		}
	}()
	go checkIfFlynnExists()
}

func checkIfFlynnExists() {
	func() {
		_, err := exec.LookPath("flynn")
		if err != nil {
			if runtime.GOOS == "windows" {
				log.Fatal("Flynn CLI is required but not installed or not in path.")
			}
			log.Println("Could not find Flynn CLI, trying to install...")
			if o, err := exec.Command("sh", "bin/flynn-install.sh").CombinedOutput(); err != nil {
				log.Printf("Could not install Flynn CLI (%s): %s", err.Error(), o)
			} else if _, err := exec.LookPath("flynn"); err != nil {
				log.Fatal("Could not install Flynn CLI.")
			}
            log.Println("Flynn installed successfully!")
            log.Println("Adding flynn cluster...")
            log.Println(envClusterConf)
            args := append([]string{"cluster", "add"}, strings.Split(envClusterConf, " ")...)
            log.Printf("%s", args)
            if o, err := exec.Command("flynn", args...).CombinedOutput(); err != nil {
                log.Fatalf("Could not add cluster (status: %s) (output: %s) (args: %s)", err.Error(), o, args)
            } else {
                log.Printf("Adding cluster successful: %s", o)
            }
		}
	}()
}

func getIP(r *http.Request) string {
    ip := removePort(r.RemoteAddr)
    if len(r.Header.Get("X-FORWARDED-FOR")) > 0 {
        ip = r.Header.Get("X-FORWARDED-FOR")
    }
    return ip
}

func removePort(ip string) string {
    split := strings.Split(ip, ":")
    if len(split) < 2 {
        return ip
    }
    return strings.Join(split[:len(split)-1], ":")
}
<commit_msg>deploy: Fixed GitHub detection<commit_after>package main

import (
	"code.google.com/p/go-uuid/uuid"
	"encoding/json"
	"fmt"
	"github.com/gorilla/mux"
	gorillasession "github.com/gorilla/sessions"
	"github.com/ory-am/common/env"
	"github.com/ory-am/common/mgopath"
	"github.com/ory-am/event"
	gde "github.com/ory-am/gitdeploy/event"
	"github.com/ory-am/gitdeploy/job"
	gdLog "github.com/ory-am/gitdeploy/log"
	"github.com/ory-am/gitdeploy/sse"
	"github.com/ory-am/gitdeploy/storage"
	"github.com/ory-am/gitdeploy/storage/mongo"
	"github.com/ory-am/google-json-style-response/responder"
	"gopkg.in/mgo.v2"
	"log"
	"net/http"
	"os/exec"
	"regexp"
	"time"
	"runtime"
	"os"
    "strings"
)

const (
	sessionCurrentDeployment = "cdid"
	sessionName = "gdp"
)

var (
// API Version
	ApiVersion = "1.0"

// Generic configuration
	host = env.Getenv("HOST", "")
	port = env.Getenv("PORT", "7654")

	envAppTtl = env.Getenv("APP_TTL", "30m")
    envClusterConf = env.Getenv("FLYNN_CLUSTER_CONFIG", "")

// Configuration for CORS
	corsAllowOrigin = env.Getenv("CORS_ALLOW_ORIGIN", "http://localhost:9000")

	sessionStore = gorillasession.NewCookieStore([]byte(env.Getenv("SESSION_SECRET", "changme")))

// MongoDB
	envMongoPath = env.Getenv("MONGODB", "mongodb://localhost:27017/gitdeploy")
)

type deployRequest struct {
	Repository string `json:"repository"`
}

type appResponse struct {
	App string `json:"app"`
}

func main() {
	checkDependencies()
	eventManager := event.New()

	// mgo
	db, dbName, err := mgopath.Connect(envMongoPath)
	if err != nil {
		log.Fatal(err)
	}

	storage := mongo.New(db, dbName)
	eventManager.AttachListenerAggregate(storage)

	// SSE broker
	sseBroker := sse.New(storage)

	// Log listener
	eventManager.AttachListenerAggregate(new(gdLog.Listener))

	// Mux router
	r := mux.NewRouter()
	r.HandleFunc("/config", configHandler).Methods("GET")
	r.HandleFunc("/deployments", deployWrapperAction(sseBroker, eventManager, storage)).Methods("POST")
	r.HandleFunc("/deployments", setCORSHeaders).Methods("OPTIONS")
	r.HandleFunc("/deployments/{app:.+}/events", eventWrapperAction(sseBroker)).Methods("GET")
	r.HandleFunc("/apps/{app:.+}", getAppHandler(storage)).Methods("GET")
	r.PathPrefix("/").HandlerFunc(publicHandler("./app/dist"))
	http.Handle("/", r)

	go job.KillAppsOnHitList(storage)

	listen := fmt.Sprintf("%s:%s", host, port)
	log.Printf("Listening on %s", listen)
	log.Fatal(http.ListenAndServe(listen, nil))
}

func publicHandler(dir string) func(http.ResponseWriter, *http.Request) {
	return func(w http.ResponseWriter, r *http.Request) {
		path := dir + r.URL.Path
		if r.URL.Path == "/" {
			path = dir + "/index.html"
		}
		pattern := `!\.html|\.js|\.svg|\.css|\.png|\.jpg$`

		if f, err := os.Stat(path); err == nil {
			if !f.IsDir() {
				http.ServeFile(w, r, path)
				return
			} else {
                http.NotFound(w, r)
				return
			}
		}

		if matched, err := regexp.MatchString(pattern, path); err != nil {
			log.Printf("Could not exec regex: %s", err.Error())
		} else if !matched {
            http.ServeFile(w, r, dir + "/index.html")
            return
		} else {
			http.NotFound(w, r)
		}
	}
}

func configHandler(w http.ResponseWriter, r *http.Request) {
	setCORSHeaders(w, r)
	responseSuccess(w, struct {
		Now time.Time `json:"now"`
	}{
		time.Now(),
	})
}

func getAppHandler(store *mongo.MongoStorage) func(w http.ResponseWriter, r *http.Request) {
	return func(w http.ResponseWriter, r *http.Request) {
		if cleanUpSession(w, r) != nil {
			return
		}
		setCORSHeaders(w, r)
		vars := mux.Vars(r)
		id := vars["app"]
		app, err := store.GetApp(id)
		if err == mgo.ErrNotFound {
			responseError(w, http.StatusNotFound, fmt.Sprintf("App %s does not exist.", id))
			return
		} else if app.Killed {
			responseError(w, http.StatusNotFound, "App timed out and is no longer available.")
			return
		} else if err != nil {
			responseError(w, http.StatusInternalServerError, err.Error())
			return
		}
		logs, err := job.GetLogs(app.ID)
		if err != nil {
			responseError(w, http.StatusInternalServerError, err.Error())
			return
		}

		ps, err := job.GetPS(app.ID)
		if err != nil {
			responseError(w, http.StatusInternalServerError, err.Error())
			return
		}

		deployLogs, err := store.FindDeployLogsForApp(id)
		if err != nil {
			responseError(w, http.StatusInternalServerError, err.Error())
			return
		}

		responseSuccess(w, struct {
			*storage.App
			Logs       string                 `json:"logs"`
			DeployLogs []*storage.DeployEvent `json:"deployLogs"`
			PS         string                 `json:"ps"`
		}{app, logs, deployLogs, ps})
	}
}

func cleanUpSession(w http.ResponseWriter, r *http.Request) error {
	session, err := sessionStore.Get(r, sessionName)
	if err != nil {
		responseError(w, http.StatusBadRequest, "Please delete your cookies.")
		return err
	}
	delete(session.Values, sessionCurrentDeployment)
	if err := session.Save(r, w); err != nil {
		responseError(w, http.StatusInternalServerError, err.Error())
		return err
	}
	return nil
}

func eventWrapperAction(sseBroker *sse.Broker) func(w http.ResponseWriter, r *http.Request) {
	return func(w http.ResponseWriter, r *http.Request) {
		setCORSHeaders(w, r)
		sseBroker.EventHandler(w, r)
	}
}

func deployWrapperAction(sseBroker *sse.Broker, em *event.EventManager, store *mongo.MongoStorage) func(w http.ResponseWriter, r *http.Request) {
	return func(w http.ResponseWriter, r *http.Request) {
		setCORSHeaders(w, r)
		deployAction(w, r, sseBroker, em, store)
	}
}

func deployAction(w http.ResponseWriter, r *http.Request, sseBroker *sse.Broker, em *event.EventManager, store *mongo.MongoStorage) {
	session, err := sessionStore.Get(r, sessionName)
	if err != nil {
		responseError(w, http.StatusBadRequest, "Please delete your cookies.")
		return
	}

	// Check if the user is currently deploying an application and switch to that one.
	if v, ok := session.Values[sessionCurrentDeployment].(string); ok && len(v) > 0 {
		app, err := store.GetApp(v)
		if err != nil {
            cleanUpSession(w, r)
            log.Printf("Could not fetch app from cookie: %s", err.Error())
        } else if !sseBroker.IsChannelOpen(app.ID) {
            cleanUpSession(w, r)
            log.Printf("Channel %s does not exist any more", app.ID)
		} else {
			responseSuccess(w, app)
            return
		}
	}

	// Parse body
	dr := new(deployRequest)
	decoder := json.NewDecoder(r.Body)
	decoder.Decode(dr)

	// Validate URL
	regExpression := "(https|http):\\/\\/github\\.com\\/[a-zA-Z0-9\\-\\_\\.]+/[a-zA-Z0-9\\-\\_\\.]+\\.git"
	if match, _ := regexp.MatchString(regExpression, dr.Repository); !match {
		responseError(w, http.StatusBadRequest, "I only support GitHub.")
		return
	}

	app := uuid.NewRandom().String()
	if ttl, err := time.ParseDuration(envAppTtl); err != nil {
		log.Println(err.Error())
		responseError(w, http.StatusInternalServerError, err.Error())
		return
	} else {
		appEntity, err := store.AddApp(app, time.Now().Add(ttl), dr.Repository, getIP(r))
		if err != nil {
			responseError(w, http.StatusInternalServerError, err.Error())
			return
		}
		session.Values[sessionCurrentDeployment] = appEntity.ID
		if err := session.Save(r, w); err != nil {
			responseError(w, http.StatusInternalServerError, err.Error())
			return
		}
		go runJobs(w, r, em, dr, appEntity, sseBroker, session, store)
		responseSuccess(w, appEntity)
	}
}

func runJobs(w http.ResponseWriter, r *http.Request, em *event.EventManager, dr *deployRequest, app *storage.App, sseBroker *sse.Broker, session *gorillasession.Session, store *mongo.MongoStorage) {
	sseBroker.OpenChannel(app.ID)
	sseBroker.Start(app.ID)
	defer func() {
		// Give the client the chance to read the output...
		time.Sleep(15 * time.Second)
		sseBroker.CloseChannel(app.ID)
	}()

	em.Trigger("app.created", gde.New(app.ID, app.ID))

	destination, err := job.Clone(em, app.ID, dr.Repository)
	if err != nil {
		log.Printf("Error in job.clone %s: %s", app.ID, err.Error())
		return
	}

	if err = job.Parse(em, app.ID, destination); err != nil {
		log.Printf("Error in job.parse %s: %s", app.ID, err.Error())
		return
	}

	if err = job.Deploy(em, app.ID, destination); err != nil {
		log.Printf("Error in job.deploy %s: %s", app.ID, err.Error())
		return
	}

	cluster, err := job.GetCluster(em, app.ID)
	if err != nil {
		log.Printf("Error in job.getCluster %s: %s", app.ID, err.Error())
		return
	}

	app.URL = fmt.Sprintf("%s.%s", app.ID, cluster.Host)
	if err := store.UpdateApp(app); err != nil {
		responseError(w, http.StatusInternalServerError, err.Error())
		log.Printf("Error %s: %s", app.ID, err.Error())
		return
	}

    if err = job.Cleanup(em, app.ID, destination); err != nil {
        log.Printf("Error in job.cleanup %s: %s", app.ID, err.Error())
        return
    }

	log.Println("Deployment successful.")
	em.Trigger("app.deployed", gde.New(app.ID, app.URL))
}

// Set the different CORS headers required for CORS request
func setCORSHeaders(w http.ResponseWriter, r *http.Request) {
	w.Header().Add("Access-Control-Allow-Methods", "POST, GET, OPTIONS")
	w.Header().Set("Access-Control-Allow-Origin", corsAllowOrigin)
	w.Header().Set("Access-Control-Allow-Credentials", "true")
	w.Header().Set("Access-Control-Allow-Headers", "Origin, X-Requested-With, Content-Type, Accept")
}

func responseError(w http.ResponseWriter, code int, message string) {
	log.Printf("Error %d: %s", code, message)
	response := responder.New(ApiVersion)
	response.Write(w, response.Error(code, message))
}

func responseSuccess(w http.ResponseWriter, data interface{}) {
	response := responder.New(ApiVersion)
	response.Write(w, response.Success(data))
}

func checkDependencies() {
	go func() {
		_, err := exec.LookPath("git")
		if err != nil {
			log.Fatal("Git CLI is required but not installed or not in path.")
		}
	}()
	go checkIfFlynnExists()
}

func checkIfFlynnExists() {
	func() {
		_, err := exec.LookPath("flynn")
		if err != nil {
			if runtime.GOOS == "windows" {
				log.Fatal("Flynn CLI is required but not installed or not in path.")
			}
			log.Println("Could not find Flynn CLI, trying to install...")
			if o, err := exec.Command("sh", "bin/flynn-install.sh").CombinedOutput(); err != nil {
				log.Printf("Could not install Flynn CLI (%s): %s", err.Error(), o)
			} else if _, err := exec.LookPath("flynn"); err != nil {
				log.Fatal("Could not install Flynn CLI.")
			}
            log.Println("Flynn installed successfully!")
            log.Println("Adding flynn cluster...")
            log.Println(envClusterConf)
            args := append([]string{"cluster", "add"}, strings.Split(envClusterConf, " ")...)
            log.Printf("%s", args)
            if o, err := exec.Command("flynn", args...).CombinedOutput(); err != nil {
                log.Fatalf("Could not add cluster (status: %s) (output: %s) (args: %s)", err.Error(), o, args)
            } else {
                log.Printf("Adding cluster successful: %s", o)
            }
		}
	}()
}

func getIP(r *http.Request) string {
    ip := removePort(r.RemoteAddr)
    if len(r.Header.Get("X-FORWARDED-FOR")) > 0 {
        ip = r.Header.Get("X-FORWARDED-FOR")
    }
    return ip
}

func removePort(ip string) string {
    split := strings.Split(ip, ":")
    if len(split) < 2 {
        return ip
    }
    return strings.Join(split[:len(split)-1], ":")
}
<|endoftext|>


<commit_before>package main

import (
	"flag"
	"fmt"
	"log"
	"os"
	"os/exec"
	"path/filepath"
	"strings"
	"sync"

	"github.com/docker/go-plugins-helpers/volume"
)

const (
	sshfsId       = "_sshfs"
	socketAddress = "/run/docker/plugins/sshfs.sock"
)

var (
	defaultDir = filepath.Join(volume.DefaultDockerRootDirectory, sshfsId)
	root       = flag.String("root", defaultDir, "SshFS volumes root directory")
)

type volume_name struct {
	name        string
	connections int
}

type sshfsDriver struct {
	sync.Mutex

	root    string
	volumes map[string]*volume_name
}

func newSshfsDriver(root string) sshfsDriver {
	d := sshfsDriver{
		root:    root,
		volumes: map[string]*volume_name{},
	}

	return d
}

func (d sshfsDriver) Create(r volume.Request) volume.Response {
	return volume.Response{}
}

func (d sshfsDriver) Remove(r volume.Request) volume.Response {
	d.Lock()
	defer d.Unlock()
	m := d.mountpoint(r.Name)

	if s, ok := d.volumes[m]; ok {
		if s.connections <= 1 {
			delete(d.volumes, m)
		}
	}
	return volume.Response{}
}

func (d sshfsDriver) Path(r volume.Request) volume.Response {
	return volume.Response{Mountpoint: d.mountpoint(r.Name)}
}

func (d sshfsDriver) Mount(r volume.MountRequest) volume.Response {
	d.Lock()
	defer d.Unlock()
	m := d.mountpoint(r.Name)

	s, ok := d.volumes[m]
	if ok && s.connections > 0 {
		log.Printf("Volume %s already mounted on %s\n", r.Name, m)
		s.connections++
		return volume.Response{Mountpoint: m}
	}

	fi, err := os.Lstat(m)

	if os.IsNotExist(err) {
		if err := os.MkdirAll(m, 0755); err != nil {
			return volume.Response{Err: err.Error()}
		}
	} else if err != nil {
		return volume.Response{Err: err.Error()}
	}

	if fi != nil && !fi.IsDir() {
		return volume.Response{Err: fmt.Sprintf("%v already exist and it's not a directory", m)}
	}

	if err := d.mountVolume(r.Name, m); err != nil {
		return volume.Response{Err: err.Error()}
	}

	d.volumes[m] = &volume_name{name: r.Name, connections: 1}

	log.Printf("Mounting volume %s on %s\n", r.Name, m)
	return volume.Response{Mountpoint: m}
}

func (d sshfsDriver) Unmount(r volume.UnmountRequest) volume.Response {
	d.Lock()
	defer d.Unlock()
	m := d.mountpoint(r.Name)

	if s, ok := d.volumes[m]; ok {
		if s.connections == 1 {
			log.Printf("Unmounting volume %s from %s\n", r.Name, m)
			if err := d.unmountVolume(m); err != nil {
				return volume.Response{Err: err.Error()}
			}
		}
		s.connections--
	} else {
		return volume.Response{Err: fmt.Sprintf("Unable to find volume mounted on %s", m)}
	}

	return volume.Response{}
}

func (d sshfsDriver) Get(r volume.Request) volume.Response {
	d.Lock()
	defer d.Unlock()
	m := d.mountpoint(r.Name)
	if s, ok := d.volumes[m]; ok {
		return volume.Response{Volume: &volume.Volume{Name: s.name, Mountpoint: d.mountpoint(s.name)}}
	}

	return volume.Response{Err: fmt.Sprintf("Unable to find volume mounted on %s", m)}
}

func (d sshfsDriver) List(r volume.Request) volume.Response {
	d.Lock()
	defer d.Unlock()
	var vols []*volume.Volume
	for _, v := range d.volumes {
		vols = append(vols, &volume.Volume{Name: v.name, Mountpoint: d.mountpoint(v.name)})
	}
	return volume.Response{Volumes: vols}
}

func (d sshfsDriver) Capabilities(r volume.Request) volume.Response {
	return volume.Response{Capabilities: volume.Capability{Scope: "global"}}
}

func (d *sshfsDriver) mountpoint(name string) string {
	return filepath.Join(d.root, name)
}

func (d *sshfsDriver) mountVolume(name, destination string) error {
	parts := strings.Split(name, "#")
	if len(parts) == 1 {
		name = name + ":/"
	} else if len(parts) == 2 {
		name = parts[0] + ":" + parts[1]
	} else {
		return fmt.Errorf("invalid name, use [user@]host#[dir]")
	}
	cmd := fmt.Sprintf("sshfs %s  %s", name, destination)
	return exec.Command("sh", "-c", cmd).Run()
}

func (d *sshfsDriver) unmountVolume(target string) error {
	cmd := fmt.Sprintf("umount %s", target)
	return exec.Command("sh", "-c", cmd).Run()
}

func main() {
	flag.Parse()

	d := newSshfsDriver(*root)
	h := volume.NewHandler(d)
	fmt.Printf("listening on %s\n", socketAddress)
	fmt.Println(h.ServeUnix("root", socketAddress))
}
<commit_msg>fix lint<commit_after>package main

import (
	"flag"
	"fmt"
	"log"
	"os"
	"os/exec"
	"path/filepath"
	"strings"
	"sync"

	"github.com/docker/go-plugins-helpers/volume"
)

const (
	sshfsID       = "_sshfs"
	socketAddress = "/run/docker/plugins/sshfs.sock"
)

var (
	defaultDir = filepath.Join(volume.DefaultDockerRootDirectory, sshfsID)
	root       = flag.String("root", defaultDir, "SshFS volumes root directory")
)

type volumeName struct {
	name        string
	connections int
}

type sshfsDriver struct {
	sync.Mutex

	root    string
	volumes map[string]*volumeName
}

func newSshfsDriver(root string) sshfsDriver {
	d := sshfsDriver{
		root:    root,
		volumes: map[string]*volumeName{},
	}

	return d
}

func (d sshfsDriver) Create(r volume.Request) volume.Response {
	return volume.Response{}
}

func (d sshfsDriver) Remove(r volume.Request) volume.Response {
	d.Lock()
	defer d.Unlock()
	m := d.mountpoint(r.Name)

	if s, ok := d.volumes[m]; ok {
		if s.connections <= 1 {
			delete(d.volumes, m)
		}
	}
	return volume.Response{}
}

func (d sshfsDriver) Path(r volume.Request) volume.Response {
	return volume.Response{Mountpoint: d.mountpoint(r.Name)}
}

func (d sshfsDriver) Mount(r volume.MountRequest) volume.Response {
	d.Lock()
	defer d.Unlock()
	m := d.mountpoint(r.Name)

	s, ok := d.volumes[m]
	if ok && s.connections > 0 {
		log.Printf("Volume %s already mounted on %s\n", r.Name, m)
		s.connections++
		return volume.Response{Mountpoint: m}
	}

	fi, err := os.Lstat(m)

	if os.IsNotExist(err) {
		if err := os.MkdirAll(m, 0755); err != nil {
			return volume.Response{Err: err.Error()}
		}
	} else if err != nil {
		return volume.Response{Err: err.Error()}
	}

	if fi != nil && !fi.IsDir() {
		return volume.Response{Err: fmt.Sprintf("%v already exist and it's not a directory", m)}
	}

	if err := d.mountVolume(r.Name, m); err != nil {
		return volume.Response{Err: err.Error()}
	}

	d.volumes[m] = &volumeName{name: r.Name, connections: 1}

	log.Printf("Mounting volume %s on %s\n", r.Name, m)
	return volume.Response{Mountpoint: m}
}

func (d sshfsDriver) Unmount(r volume.UnmountRequest) volume.Response {
	d.Lock()
	defer d.Unlock()
	m := d.mountpoint(r.Name)

	if s, ok := d.volumes[m]; ok {
		if s.connections == 1 {
			log.Printf("Unmounting volume %s from %s\n", r.Name, m)
			if err := d.unmountVolume(m); err != nil {
				return volume.Response{Err: err.Error()}
			}
		}
		s.connections--
	} else {
		return volume.Response{Err: fmt.Sprintf("Unable to find volume mounted on %s", m)}
	}

	return volume.Response{}
}

func (d sshfsDriver) Get(r volume.Request) volume.Response {
	d.Lock()
	defer d.Unlock()
	m := d.mountpoint(r.Name)
	if s, ok := d.volumes[m]; ok {
		return volume.Response{Volume: &volume.Volume{Name: s.name, Mountpoint: d.mountpoint(s.name)}}
	}

	return volume.Response{Err: fmt.Sprintf("Unable to find volume mounted on %s", m)}
}

func (d sshfsDriver) List(r volume.Request) volume.Response {
	d.Lock()
	defer d.Unlock()
	var vols []*volume.Volume
	for _, v := range d.volumes {
		vols = append(vols, &volume.Volume{Name: v.name, Mountpoint: d.mountpoint(v.name)})
	}
	return volume.Response{Volumes: vols}
}

func (d sshfsDriver) Capabilities(r volume.Request) volume.Response {
	return volume.Response{Capabilities: volume.Capability{Scope: "global"}}
}

func (d *sshfsDriver) mountpoint(name string) string {
	return filepath.Join(d.root, name)
}

func (d *sshfsDriver) mountVolume(name, destination string) error {
	parts := strings.Split(name, "#")
	if len(parts) == 1 {
		name = name + ":/"
	} else if len(parts) == 2 {
		name = parts[0] + ":" + parts[1]
	} else {
		return fmt.Errorf("invalid name, use [user@]host#[dir]")
	}
	cmd := fmt.Sprintf("sshfs %s  %s", name, destination)
	return exec.Command("sh", "-c", cmd).Run()
}

func (d *sshfsDriver) unmountVolume(target string) error {
	cmd := fmt.Sprintf("umount %s", target)
	return exec.Command("sh", "-c", cmd).Run()
}

func main() {
	flag.Parse()

	d := newSshfsDriver(*root)
	h := volume.NewHandler(d)
	fmt.Printf("listening on %s\n", socketAddress)
	fmt.Println(h.ServeUnix("root", socketAddress))
}
<|endoftext|>


<commit_before>// transfer2go - Go implementation of loosely coupled, distributed agents for data transfer
//
// Copyright (c) 2017 - Valentin Kuznetsov <vkuznet@gmail.com>
//
package main

import (
	"encoding/json"
	"flag"
	"fmt"
	"io/ioutil"
	"log"
	"os"

	"github.com/vkuznet/transfer2go/client"
	"github.com/vkuznet/transfer2go/server"
	"github.com/vkuznet/transfer2go/utils"
)

func main() {

	// server options
	var agent string
	flag.StringVar(&agent, "agent", "", "Remote agent (registration) end-point")
	var configFile string
	flag.StringVar(&configFile, "config", "", "Agent configuration file")
	var verbose int
	flag.IntVar(&verbose, "verbose", 0, "Verbosity level")

	// client options
	var src string
	flag.StringVar(&src, "src", "", "Source end-point, either local file or AgentName:LFN")
	var dst string
	flag.StringVar(&dst, "dst", "", "Destination end-point, either AgentName or AgentName:LFN")
	var register string
	flag.StringVar(&register, "register", "", "File with meta-data of records in JSON data format to register at remote agent")

	var auth bool
	flag.Bool(&auth, "auth", true, "To disable the auth layer")

	flag.Parse()

	if auth {
		utils.CheckX509()
	}

	utils.VERBOSE = verbose
	if configFile != "" {
		data, err := ioutil.ReadFile(configFile)
		if err != nil {
			log.Println("Unable to read", configFile, err)
			os.Exit(1)
		}
		var config server.Config
		err = json.Unmarshal(data, &config)
		if err != nil {
			log.Println("Unable to parse", configFile, err)
			os.Exit(1)
		}
		if config.Catalog == "" {
			pwd, err := os.Getwd()
			if err != nil {
				log.Println("Unable to get current directory", err)
				os.Exit(1)
			}
			config.Catalog = pwd // use current directory as catalog
		}
		if config.Workers == 0 {
			config.Workers = 10 // default value
		}
		if config.QueueSize == 0 {
			config.QueueSize = 100 // default value
		}
		if config.Protocol == "" {
			config.Protocol = "http" // default value
		}
		if config.Port == 0 {
			config.Port = 8989
		}
		if agent != "" {
			config.Register = agent
		}
		if config.Register == "" {
			log.Println("WARNING this agent is not registered with remote ones, either provide register in your config or invoke register API call")
		}

		server.Server(config)
	} else {
		var err error
		if register != "" {
			err = client.Register(agent, register)
		} else if src == "" { // no transfer request
			client.Agent(agent)
		} else {
			err = client.Transfer(agent, src, dst)
		}
		if err != nil {
			log.Println(err)
			os.Exit(1)
		}
	}
}

// helper function to construct site name
func makeSiteName() string {
	host, err := os.Hostname()
	if err != nil {
		panic(fmt.Sprintf("Unable to get hostname, error=%v", err))
	}
	return fmt.Sprintf("T4_%s_%v", host, os.Getuid())
}
<commit_msg>Pass auth variable to server config<commit_after>// transfer2go - Go implementation of loosely coupled, distributed agents for data transfer
//
// Copyright (c) 2017 - Valentin Kuznetsov <vkuznet@gmail.com>
//
package main

import (
	"encoding/json"
	"flag"
	"fmt"
	"io/ioutil"
	"log"
	"os"

	"github.com/vkuznet/transfer2go/client"
	"github.com/vkuznet/transfer2go/server"
	"github.com/vkuznet/transfer2go/utils"
)

func main() {

	// server options
	var agent string
	flag.StringVar(&agent, "agent", "", "Remote agent (registration) end-point")
	var configFile string
	flag.StringVar(&configFile, "config", "", "Agent configuration file")
	var verbose int
	flag.IntVar(&verbose, "verbose", 0, "Verbosity level")

	// client options
	var src string
	flag.StringVar(&src, "src", "", "Source end-point, either local file or AgentName:LFN")
	var dst string
	flag.StringVar(&dst, "dst", "", "Destination end-point, either AgentName or AgentName:LFN")
	var register string
	flag.StringVar(&register, "register", "", "File with meta-data of records in JSON data format to register at remote agent")

	var auth bool
	flag.Bool(&auth, "auth", true, "To disable the auth layer")

	flag.Parse()

	if auth {
		utils.CheckX509()
	}

	utils.VERBOSE = verbose
	if configFile != "" {
		data, err := ioutil.ReadFile(configFile)
		if err != nil {
			log.Println("Unable to read", configFile, err)
			os.Exit(1)
		}
		var config server.Config
		err = json.Unmarshal(data, &config)
		if err != nil {
			log.Println("Unable to parse", configFile, err)
			os.Exit(1)
		}
		if config.Catalog == "" {
			pwd, err := os.Getwd()
			if err != nil {
				log.Println("Unable to get current directory", err)
				os.Exit(1)
			}
			config.Catalog = pwd // use current directory as catalog
		}
		if config.Workers == 0 {
			config.Workers = 10 // default value
		}
		if config.QueueSize == 0 {
			config.QueueSize = 100 // default value
		}
		if config.Protocol == "" {
			config.Protocol = "http" // default value
		}
		if config.Port == 0 {
			config.Port = 8989
		}
		if agent != "" {
			config.Register = agent
		}
		if config.Register == "" {
			log.Println("WARNING this agent is not registered with remote ones, either provide register in your config or invoke register API call")
		}

		config.Auth = auth

		server.Server(config)
	} else {
		var err error
		if register != "" {
			err = client.Register(agent, register)
		} else if src == "" { // no transfer request
			client.Agent(agent)
		} else {
			err = client.Transfer(agent, src, dst)
		}
		if err != nil {
			log.Println(err)
			os.Exit(1)
		}
	}
}

// helper function to construct site name
func makeSiteName() string {
	host, err := os.Hostname()
	if err != nil {
		panic(fmt.Sprintf("Unable to get hostname, error=%v", err))
	}
	return fmt.Sprintf("T4_%s_%v", host, os.Getuid())
}
<|endoftext|>


<commit_before>package main
import (
  "os"
  //"fmt"
  "log"
  "./config"
  "github.com/gin-gonic/gin"
)

// Request Parameters
type Item struct {
  Id int `json:"id"`
  Title string `json:"title"`
  Body string `json:"body"`
  Category int `json:"category"`
}
type User struct {
  Id int `json:"id"`
  SysId int `json:"system_id"`
  Device string `json:"device"`
}
type Calender struct {
  Id int `json:"id"`
  Date string `json:"date"`
  ItemId int `json:"item_id"`
  Users string `json:"users"`
  Device string `json:"device"`
}
type Category struct {
  Id int `json:"id"`
  Name string `json:"name"`
}

var con Config.Base = Config.Get()
func main() {
  router := gin.Default()
  var path string
  if con.Serv.ENV == "production" {
    path = "./production.log"
  } else {
    path = "./staging.log"
  }
  f, err := os.OpenFile(path, os.O_RDWR|os.O_CREATE|os.O_APPEND, 0666)
  if err != nil { panic(err) }
  gin.DefaultWriter = f
  v1 := router.Group("/v1")
  {
    v1.POST("/item", ItemFunc)
    v1.POST("/user", UserFunc)
    v1.POST("/calendar", CalenderFunc)
    v1.POST("/category", CategoryFunc)
  }
  router.Run()
}

func ItemFunc(c *gin.Context) { c.JSON(200, gin.H{"status":"200"}) }
func UserFunc(c *gin.Context) { c.JSON(200, gin.H{"status":"200"}) }
func CalenderFunc(c *gin.Context) { c.JSON(200, gin.H{"status":"200"}) }
func CategoryFunc(c *gin.Context) { c.JSON(200, gin.H{"status":"200"}) }
<commit_msg>modify BindJSON<commit_after>package main
import (
  "os"
  //"fmt"
  //"log"
  "./config"
  "github.com/gin-gonic/gin"
)

// Request Parameters
type Item struct {
  Id int `json:"id"`
  Title string `json:"title"`
  Body string `json:"body"`
  Category int `json:"category"`
}
type User struct {
  Id int `json:"id"`
  SysId int `json:"system_id"`
  Device string `json:"device"`
}
type Calender struct {
  Id int `json:"id"`
  Date string `json:"date"`
  ItemId int `json:"item_id"`
  Users string `json:"users"`
  Device string `json:"device"`
}
type Category struct {
  Id int `json:"id"`
  Name string `json:"name"`
}

var con Config.Base = Config.Get()
func main() {
  router := gin.Default()
  var path string
  if con.Serv.ENV == "production" {
    path = "./production.log"
  } else {
    path = "./staging.log"
  }
  f, err := os.OpenFile(path, os.O_RDWR|os.O_CREATE|os.O_APPEND, 0666)
  if err != nil { panic(err) }
  gin.DefaultWriter = f
  v1 := router.Group("/v1")
  {
    v1.POST("/item", ItemFunc)
    v1.POST("/user", UserFunc)
    v1.POST("/calendar", CalenderFunc)
    v1.POST("/category", CategoryFunc)
  }
  router.GET("/call/status/check.json", func(c *gin.Context){
    c.JSON(200,gin.H{
      "status":"200",
      "result":"OK",
    })
  })
  router.Run(con.Serv.PORT)
}

func ItemFunc(c *gin.Context) {
  var req Item
  c.BindJSON(&req)
  c.JSON(200, gin.H{"status":"200"})
}
func UserFunc(c *gin.Context) {
  var req User
  c.BindJSON(&req)
  c.JSON(200, gin.H{"status":"200"})
}
func CalenderFunc(c *gin.Context) {
  var req Calender
  c.BindJSON(&req)
  c.JSON(200, gin.H{"status":"200"})
}
func CategoryFunc(c *gin.Context) {
  var req Category
  c.BindJSON(&req)
  c.JSON(200, gin.H{"status":"200"})
}
<|endoftext|>


<commit_before>package main

import (
	"bufio"
	"encoding/json"
	"errors"
	"flag"
	"fmt"
	"sort"
	"github.com/tomsteele/blacksheepwall/bsw"
	"log"
	"net"
	"os"
	"runtime"
	"text/tabwriter"
)

var usage = `
  Usage: blacksheepwall [options] <ip address or CIDR>

  Options:
  -h, --help            Show Usage and exit.
  -version              Show version and exit.
  -debug                Enable debugging and show errors returned from tasks.
  -concurrency <int>    Max amount of concurrent tasks.    [default: 100]
  -cpus <int>           Max amount of cpus  for the go runtime.    [default: 1]
  -server <string>      DNS server address.    [default: "8.8.8.8"]
  -input <string>       Line separated file of networks (CIDR) or
                        IP Addresses.
  -ipv6	                Look for additional AAAA records where applicable.
  -domain <string>      Target domain to use for certain tasks.
  -dictionary <string>  Attempt to retrieve the CNAME and A record for
                        each subdomain in the line separated file.
  -yandex <string>      Provided a Yandex search XML API url. Use the Yandex
                        search 'rhost:' operator to find subdomains of a
                        provided domain..
  -bing	<string>        Provided a base64 encoded API key. Use the Bing search
                        API's 'ip:' operator to lookup hostnames for each host.
  -headers              Perform HTTP(s) requests to each host and look for
                        hostnames in a possible Location header.
  -reverse              Retrieve the PTR for each host.
  -tls                  Attempt to retrieve names from TLS certificates
                        (CommonName and Subject Alternative Name).
  -viewdns              Lookup each host using viewdns.info's Reverse IP
                        Lookup function.
  -fcrdns               Verify results by attempting to retrieve the A or AAAA record for
                        each result previously identified hostname.
  -clean                Print results as unique hostnames for each host.
  -csv                  Print results in csv format.
  -json                 Print results as JSON.

`

// Returns all ip addresses from each CIDR range in a list.
func linesToIpList(lines []string) ([]string, error) {
	ipList := make([]string, 0)
	for _, line := range lines {
		if net.ParseIP(line) != nil {
			ipList = append(ipList, line)
		} else if ip, network, err := net.ParseCIDR(line); err == nil {
			for ip := ip.Mask(network.Mask); network.Contains(ip); increaseIp(ip) {
				ipList = append(ipList, ip.String())
			}
		} else {
			return ipList, errors.New("\"" + line + "\" is not an IP Address or CIDR Network")
		}
	}
	return ipList, nil
}

// Increases each byte of a net.IP.
func increaseIp(ip net.IP) {
	for j := len(ip) - 1; j >= 0; j-- {
		ip[j]++
		if ip[j] > 0 {
			break
		}
	}
}

// Reads lines from a file and returns as a slice.
func readFileLines(path string) ([]string, error) {
	file, err := os.Open(path)
	if err != nil {
		return nil, err
	}
	defer file.Close()
	lines := make([]string, 0)
	scanner := bufio.NewScanner(file)
	for scanner.Scan() {
		lines = append(lines, scanner.Text())
	}
	return lines, scanner.Err()
}

// Sets the max CPUS for the runtime.
func setMaxProcs(cpus int) {
	if cpus < 1 {
		log.Fatal("-cpus must be atleast 1")
	}
	if cpus > runtime.NumCPU() {
		log.Fatal("-cpus count too high")
	}
	if cpus == 1 {
		return
	}
	runtime.GOMAXPROCS(cpus)
	return
}

type task func() (bsw.Results, error)
type empty struct{}

func main() {
	flVersion := flag.Bool("version", false, "Show version and exit.")
	flConcurrency := flag.Int("concurrency", 100, "Max amount of concurrent tasks.")
	flCpus := flag.Int("cpus", 1, "Max amount of cpus  for the go runtime.")
	flDebug := flag.Bool("debug", false, "Enable debugging and show errors returned from tasks.")
	flipv6 := flag.Bool("ipv6", false, "Look for AAAA records where applicable.")
	flServerAddr := flag.String("server", "8.8.8.8", "DNS server address.")
	flIpFile := flag.String("input", "", "Line separated file of networks (CIDR) or IP Addresses.")
	flReverse := flag.Bool("reverse", false, "Retrieve the PTR for each host.")
	flHeader := flag.Bool("headers", false, "Perform HTTP(s) requests to each host and look for hostnames in a possible Location header.")
	flTLS := flag.Bool("tls", false, "Attempt to retrieve names from TLS certificates (CommonName and Subject Alternative Name).")
	flViewDnsInfo := flag.Bool("viewdns", false, "Lookup each host using viewdns.info's Reverse IP Lookup function.")
	flBing := flag.String("bing", "", "Provided a base64 encoded API key. Use the Bing search API's 'ip:' operator to lookup hostnames for each host.")
	flYandex := flag.String("yandex", "", "Provided a Yandex search XML API url. Use the Yandex search 'rhost:' operator to find subdomains of a provided domain.")
	flDomain := flag.String("domain", "", "Target domain to use for certain tasks.")
	flDictFile := flag.String("dictionary", "", "Attempt to retrieve the CNAME and A record for each subdomain in the line separated file.")
	flFcrdns := flag.Bool("fcrdns", false, "Verify results by attempting to retrieve the A or AAAA record for each result previously identified hostname.")
	flClean := flag.Bool("clean", false, "Print results as unique hostnames for each host.")
	flCsv := flag.Bool("csv", false, "Print results in csv format.")
	flJson := flag.Bool("json", false, "Print results as JSON.")
	flag.Usage = func() { fmt.Print(usage) }
	flag.Parse()

	if *flVersion {
		fmt.Println("blacksheepwall version ", bsw.VERSION)
		os.Exit(0)
	}

	setMaxProcs(*flCpus)

	// Holds all ip addresses for testing
	ipAddrList := make([]string, 0)

	// Used to hold a ip or CIDR range passed as fl.Arg(0)
	var flNetwork string

	// Verify that some sort of work load was given in commands
	if *flIpFile == "" && *flDomain == "" && len(flag.Args()) < 1 {
		log.Fatal("You didn't provide any work for me to do")
	}
	if *flYandex != "" && *flDomain == "" {
		log.Fatal("Yandex API requires domain set with -domain")
	}
	if *flDictFile != "" && *flDomain == "" {
		log.Fatal("Dictionary lookup requires domain set with -domain")
	}
	if *flDomain != "" && *flYandex == "" && *flDictFile == "" {
		log.Fatal("-domain provided but no methods provided that use it")
	}

	// Get first argument that is not an option and turn it into a list of ips
	if len(flag.Args()) > 0 {
		flNetwork = flag.Arg(0)
		list, err := linesToIpList([]string{flNetwork})
		if err != nil {
			log.Fatal(err.Error())
		}
		ipAddrList = append(ipAddrList, list...)
	}

	// If file given as -input, read lines and turn each possible ip or network into
	// a list of ips. Appends list to ipAddrList. Will fail fatally if line in file
	// is not a valid ip or CIDR range.
	if *flIpFile != "" {
		lines, err := readFileLines(*flIpFile)
		if err != nil {
			log.Fatal("Error reading " + *flIpFile + " " + err.Error())
		}
		list, err := linesToIpList(lines)
		if err != nil {
			log.Fatal(err.Error())
		}
		ipAddrList = append(ipAddrList, list...)
	}

	// tracker: Chanel uses an empty struct to track when all goroutines in the pool
	//          have completed as well as a single call from the gatherer.
	//
	// tasks:   Chanel used in the goroutine pool to manage incoming work. A task is
	//          a function wrapper that returns a slice of results and a possible error.
	//
	// res:     When each task is called in the pool, it will send valid results to
	//          the res channel. A goroutine manages this channel and appends results
	//          to results slice.
	tracker := make(chan empty)
	tasks := make(chan task, *flConcurrency)
	res := make(chan bsw.Results, *flConcurrency)
	results := bsw.Results{}

	// Start up *flConcurrency amount of goroutines
	log.Printf("Spreading tasks across %d goroutines", *flConcurrency)
	for i := 0; i < *flConcurrency; i++ {
		go func() {
			var c = 0
			for def := range tasks {
				result, err := def()
				if m := c % 2; m == 0 {
					c = 3
					os.Stderr.WriteString("\rWorking \\")
				} else {
					c = 2
					os.Stderr.WriteString("\rWorking /")
				}
				if err != nil && *flDebug {
					log.Println(err.Error())
				}
				if err == nil {
					res <- result
				}
			}
			e := empty{}
			tracker <- e
		}()
	}

	// Ingest incoming results
	go func() {
		for result := range res {
			if len(result) < 1 {
				continue
			}
			if *flFcrdns {
				for _, r := range result {
					ip, err := bsw.LookupName(r.Hostname, *flServerAddr)
					if err == nil && len(ip) > 0 {
						results = append(results, bsw.Result{Source: "fcrdns", IP: ip, Hostname: r.Hostname})
					}
					ip, err = bsw.LookupName6(r.Hostname, *flServerAddr)
					if err == nil && len(ip) > 0 {
						results = append(results, bsw.Result{Source: "fcrdns", IP: ip, Hostname: r.Hostname})
					}
				}
			} else {
				results = append(results, result...)
			}
		}
		e := empty{}
		tracker <- e
	}()

	// Bing has two possible search paths. We need to find which one is valid.
	var bingPath string
	if *flBing != "" {
		p, err := bsw.FindBingSearchPath(*flBing)
		if err != nil {
			log.Fatal(err.Error())
		}
		bingPath = p
	}

	// ip based functionality should be added to the pool here
	for _, h := range ipAddrList {
		host := h
		if *flReverse {
			tasks <- func() (bsw.Results, error) {
				return bsw.Reverse(host, *flServerAddr)
			}
		}
		if *flTLS {
			tasks <- func() (bsw.Results, error) {
				return bsw.TLS(host)
			}
		}
		if *flViewDnsInfo {
			tasks <- func() (bsw.Results, error) {
				return bsw.ViewDnsInfo(host)
			}
		}
		if *flBing != "" && bingPath != "" {
			tasks <- func() (bsw.Results, error) {
				return bsw.BingAPI(host, *flBing, bingPath)
			}
		}
		if *flHeader {
			tasks <- func() (bsw.Results, error) {
				return bsw.Headers(host)
			}
		}

	}

	// Domain based functions will likely require separate blocks

	// Subdomain dictionary guessing
	if *flDictFile != "" && *flDomain != "" {
		nameList, err := readFileLines(*flDictFile)
		if err != nil {
			log.Fatal("Error reading " + *flDictFile + " " + err.Error())
		}
		// Get an ip for a possible wildcard domain and use it as a blacklist
		blacklist := bsw.GetWildCard(*flDomain, *flServerAddr)
		var blacklist6 string
		if *flipv6 {
			blacklist6 = bsw.GetWildCard6(*flDomain, *flServerAddr)
		}
		for _, n := range nameList {
			sub := n
			tasks <- func() (bsw.Results, error) {
				return bsw.Dictionary(*flDomain, sub, blacklist, *flServerAddr)
			}
			if *flipv6 {
				tasks <- func() (bsw.Results, error) {
					return bsw.Dictionary6(*flDomain, sub, blacklist6, *flServerAddr)
				}
			}
		}
	}

	if *flYandex != "" && *flDomain != "" {
		tasks <- func() (bsw.Results, error) {
			return bsw.YandexAPI(*flDomain, *flYandex, *flServerAddr)
		}
	}

	// Close the tasks channel after all jobs have completed and for each
	// goroutine in the pool receive an empty message from  tracker.
	close(tasks)
	for i := 0; i < *flConcurrency; i++ {
		<-tracker
	}
	close(res)
	// Receive and empty message from the result gatherer
	<-tracker

	os.Stderr.WriteString("\r")
	log.Println("All tasks completed\n")
	sort.Sort(results)

	// Output options
	if *flJson {
		j, _ := json.MarshalIndent(results, "", "    ")
		fmt.Println(string(j))
	} else if *flCsv {
		for _, r := range results {
			fmt.Printf("%s,%s,%s\n", r.Hostname, r.IP, r.Source)
		}
	} else if *flClean {
		cleanSet := make(map[string][]string)
		for _, r := range results {
			cleanSet[r.Hostname] = append(cleanSet[r.Hostname], r.IP)
		}
		for k, v := range cleanSet {
			fmt.Printf("%s:\n", k)
			for _, ip := range v {
				fmt.Printf("\t%s\n", ip)
			}
		}
	} else {
		w := tabwriter.NewWriter(os.Stdout, 0, 8, 4, ' ', 0)
		fmt.Fprintln(w, "IP\tHostname\tSource")
		for _, r := range results {
			fmt.Fprintf(w, "%s\t%s\t%s\n", r.IP, r.Hostname, r.Source)
		}
		w.Flush()
	}
}
<commit_msg>Cleaner flag definition<commit_after>package main

import (
	"bufio"
	"encoding/json"
	"errors"
	"flag"
	"fmt"
	"github.com/tomsteele/blacksheepwall/bsw"
	"log"
	"net"
	"os"
	"runtime"
	"sort"
	"text/tabwriter"
)

var usage = `
  Usage: blacksheepwall [options] <ip address or CIDR>

  Options:
  -h, --help            Show Usage and exit.
  -version              Show version and exit.
  -debug                Enable debugging and show errors returned from tasks.
  -concurrency <int>    Max amount of concurrent tasks.    [default: 100]
  -cpus <int>           Max amount of cpus  for the go runtime.    [default: 1]
  -server <string>      DNS server address.    [default: "8.8.8.8"]
  -input <string>       Line separated file of networks (CIDR) or
                        IP Addresses.
  -ipv6	                Look for additional AAAA records where applicable.
  -domain <string>      Target domain to use for certain tasks.
  -dictionary <string>  Attempt to retrieve the CNAME and A record for
                        each subdomain in the line separated file.
  -yandex <string>      Provided a Yandex search XML API url. Use the Yandex
                        search 'rhost:' operator to find subdomains of a
                        provided domain..
  -bing	<string>        Provided a base64 encoded API key. Use the Bing search
                        API's 'ip:' operator to lookup hostnames for each host.
  -headers              Perform HTTP(s) requests to each host and look for
                        hostnames in a possible Location header.
  -reverse              Retrieve the PTR for each host.
  -tls                  Attempt to retrieve names from TLS certificates
                        (CommonName and Subject Alternative Name).
  -viewdns              Lookup each host using viewdns.info's Reverse IP
                        Lookup function.
  -fcrdns               Verify results by attempting to retrieve the A or AAAA record for
                        each result previously identified hostname.
  -clean                Print results as unique hostnames for each host.
  -csv                  Print results in csv format.
  -json                 Print results as JSON.

`

// Returns all ip addresses from each CIDR range in a list.
func linesToIpList(lines []string) ([]string, error) {
	ipList := make([]string, 0)
	for _, line := range lines {
		if net.ParseIP(line) != nil {
			ipList = append(ipList, line)
		} else if ip, network, err := net.ParseCIDR(line); err == nil {
			for ip := ip.Mask(network.Mask); network.Contains(ip); increaseIp(ip) {
				ipList = append(ipList, ip.String())
			}
		} else {
			return ipList, errors.New("\"" + line + "\" is not an IP Address or CIDR Network")
		}
	}
	return ipList, nil
}

// Increases each byte of a net.IP.
func increaseIp(ip net.IP) {
	for j := len(ip) - 1; j >= 0; j-- {
		ip[j]++
		if ip[j] > 0 {
			break
		}
	}
}

// Reads lines from a file and returns as a slice.
func readFileLines(path string) ([]string, error) {
	file, err := os.Open(path)
	if err != nil {
		return nil, err
	}
	defer file.Close()
	lines := make([]string, 0)
	scanner := bufio.NewScanner(file)
	for scanner.Scan() {
		lines = append(lines, scanner.Text())
	}
	return lines, scanner.Err()
}

// Sets the max CPUS for the runtime.
func setMaxProcs(cpus int) {
	if cpus < 1 {
		log.Fatal("-cpus must be atleast 1")
	}
	if cpus > runtime.NumCPU() {
		log.Fatal("-cpus count too high")
	}
	if cpus == 1 {
		return
	}
	runtime.GOMAXPROCS(cpus)
	return
}

type task func() (bsw.Results, error)
type empty struct{}

func main() {
	var (
		flVersion     = flag.Bool("version", false, "Show version and exit.")
		flConcurrency = flag.Int("concurrency", 100, "Max amount of concurrent tasks.")
		flCpus        = flag.Int("cpus", 1, "Max amount of cpus  for the go runtime.")
		flDebug       = flag.Bool("debug", false, "Enable debugging and show errors returned from tasks.")
		flipv6        = flag.Bool("ipv6", false, "Look for AAAA records where applicable.")
		flServerAddr  = flag.String("server", "8.8.8.8", "DNS server address.")
		flIpFile      = flag.String("input", "", "Line separated file of networks (CIDR) or IP Addresses.")
		flReverse     = flag.Bool("reverse", false, "Retrieve the PTR for each host.")
		flHeader      = flag.Bool("headers", false, "Perform HTTP(s) requests to each host and look for hostnames in a possible Location header.")
		flTLS         = flag.Bool("tls", false, "Attempt to retrieve names from TLS certificates (CommonName and Subject Alternative Name).")
		flViewDnsInfo = flag.Bool("viewdns", false, "Lookup each host using viewdns.info's Reverse IP Lookup function.")
		flBing        = flag.String("bing", "", "Provided a base64 encoded API key. Use the Bing search API's 'ip:' operator to lookup hostnames for each host.")
		flYandex      = flag.String("yandex", "", "Provided a Yandex search XML API url. Use the Yandex search 'rhost:' operator to find subdomains of a provided domain.")
		flDomain      = flag.String("domain", "", "Target domain to use for certain tasks.")
		flDictFile    = flag.String("dictionary", "", "Attempt to retrieve the CNAME and A record for each subdomain in the line separated file.")
		flFcrdns      = flag.Bool("fcrdns", false, "Verify results by attempting to retrieve the A or AAAA record for each result previously identified hostname.")
		flClean       = flag.Bool("clean", false, "Print results as unique hostnames for each host.")
		flCsv         = flag.Bool("csv", false, "Print results in csv format.")
		flJson        = flag.Bool("json", false, "Print results as JSON.")
	)
	flag.Usage = func() { fmt.Print(usage) }
	flag.Parse()

	if *flVersion {
		fmt.Println("blacksheepwall version ", bsw.VERSION)
		os.Exit(0)
	}

	setMaxProcs(*flCpus)

	// Holds all ip addresses for testing
	ipAddrList := make([]string, 0)

	// Used to hold a ip or CIDR range passed as fl.Arg(0)
	var flNetwork string

	// Verify that some sort of work load was given in commands
	if *flIpFile == "" && *flDomain == "" && len(flag.Args()) < 1 {
		log.Fatal("You didn't provide any work for me to do")
	}
	if *flYandex != "" && *flDomain == "" {
		log.Fatal("Yandex API requires domain set with -domain")
	}
	if *flDictFile != "" && *flDomain == "" {
		log.Fatal("Dictionary lookup requires domain set with -domain")
	}
	if *flDomain != "" && *flYandex == "" && *flDictFile == "" {
		log.Fatal("-domain provided but no methods provided that use it")
	}

	// Get first argument that is not an option and turn it into a list of ips
	if len(flag.Args()) > 0 {
		flNetwork = flag.Arg(0)
		list, err := linesToIpList([]string{flNetwork})
		if err != nil {
			log.Fatal(err.Error())
		}
		ipAddrList = append(ipAddrList, list...)
	}

	// If file given as -input, read lines and turn each possible ip or network into
	// a list of ips. Appends list to ipAddrList. Will fail fatally if line in file
	// is not a valid ip or CIDR range.
	if *flIpFile != "" {
		lines, err := readFileLines(*flIpFile)
		if err != nil {
			log.Fatal("Error reading " + *flIpFile + " " + err.Error())
		}
		list, err := linesToIpList(lines)
		if err != nil {
			log.Fatal(err.Error())
		}
		ipAddrList = append(ipAddrList, list...)
	}

	// tracker: Chanel uses an empty struct to track when all goroutines in the pool
	//          have completed as well as a single call from the gatherer.
	//
	// tasks:   Chanel used in the goroutine pool to manage incoming work. A task is
	//          a function wrapper that returns a slice of results and a possible error.
	//
	// res:     When each task is called in the pool, it will send valid results to
	//          the res channel. A goroutine manages this channel and appends results
	//          to results slice.
	tracker := make(chan empty)
	tasks := make(chan task, *flConcurrency)
	res := make(chan bsw.Results, *flConcurrency)
	results := bsw.Results{}

	// Start up *flConcurrency amount of goroutines
	log.Printf("Spreading tasks across %d goroutines", *flConcurrency)
	for i := 0; i < *flConcurrency; i++ {
		go func() {
			var c = 0
			for def := range tasks {
				result, err := def()
				if m := c % 2; m == 0 {
					c = 3
					os.Stderr.WriteString("\rWorking \\")
				} else {
					c = 2
					os.Stderr.WriteString("\rWorking /")
				}
				if err != nil && *flDebug {
					log.Println(err.Error())
				}
				if err == nil {
					res <- result
				}
			}
			e := empty{}
			tracker <- e
		}()
	}

	// Ingest incoming results
	go func() {
		for result := range res {
			if len(result) < 1 {
				continue
			}
			if *flFcrdns {
				for _, r := range result {
					ip, err := bsw.LookupName(r.Hostname, *flServerAddr)
					if err == nil && len(ip) > 0 {
						results = append(results, bsw.Result{Source: "fcrdns", IP: ip, Hostname: r.Hostname})
					}
					ip, err = bsw.LookupName6(r.Hostname, *flServerAddr)
					if err == nil && len(ip) > 0 {
						results = append(results, bsw.Result{Source: "fcrdns", IP: ip, Hostname: r.Hostname})
					}
				}
			} else {
				results = append(results, result...)
			}
		}
		e := empty{}
		tracker <- e
	}()

	// Bing has two possible search paths. We need to find which one is valid.
	var bingPath string
	if *flBing != "" {
		p, err := bsw.FindBingSearchPath(*flBing)
		if err != nil {
			log.Fatal(err.Error())
		}
		bingPath = p
	}

	// ip based functionality should be added to the pool here
	for _, h := range ipAddrList {
		host := h
		if *flReverse {
			tasks <- func() (bsw.Results, error) {
				return bsw.Reverse(host, *flServerAddr)
			}
		}
		if *flTLS {
			tasks <- func() (bsw.Results, error) {
				return bsw.TLS(host)
			}
		}
		if *flViewDnsInfo {
			tasks <- func() (bsw.Results, error) {
				return bsw.ViewDnsInfo(host)
			}
		}
		if *flBing != "" && bingPath != "" {
			tasks <- func() (bsw.Results, error) {
				return bsw.BingAPI(host, *flBing, bingPath)
			}
		}
		if *flHeader {
			tasks <- func() (bsw.Results, error) {
				return bsw.Headers(host)
			}
		}

	}

	// Domain based functions will likely require separate blocks

	// Subdomain dictionary guessing
	if *flDictFile != "" && *flDomain != "" {
		nameList, err := readFileLines(*flDictFile)
		if err != nil {
			log.Fatal("Error reading " + *flDictFile + " " + err.Error())
		}
		// Get an ip for a possible wildcard domain and use it as a blacklist
		blacklist := bsw.GetWildCard(*flDomain, *flServerAddr)
		var blacklist6 string
		if *flipv6 {
			blacklist6 = bsw.GetWildCard6(*flDomain, *flServerAddr)
		}
		for _, n := range nameList {
			sub := n
			tasks <- func() (bsw.Results, error) {
				return bsw.Dictionary(*flDomain, sub, blacklist, *flServerAddr)
			}
			if *flipv6 {
				tasks <- func() (bsw.Results, error) {
					return bsw.Dictionary6(*flDomain, sub, blacklist6, *flServerAddr)
				}
			}
		}
	}

	if *flYandex != "" && *flDomain != "" {
		tasks <- func() (bsw.Results, error) {
			return bsw.YandexAPI(*flDomain, *flYandex, *flServerAddr)
		}
	}

	// Close the tasks channel after all jobs have completed and for each
	// goroutine in the pool receive an empty message from  tracker.
	close(tasks)
	for i := 0; i < *flConcurrency; i++ {
		<-tracker
	}
	close(res)
	// Receive and empty message from the result gatherer
	<-tracker

	os.Stderr.WriteString("\r")
	log.Println("All tasks completed\n")
	sort.Sort(results)

	// Output options
	if *flJson {
		j, _ := json.MarshalIndent(results, "", "    ")
		fmt.Println(string(j))
	} else if *flCsv {
		for _, r := range results {
			fmt.Printf("%s,%s,%s\n", r.Hostname, r.IP, r.Source)
		}
	} else if *flClean {
		cleanSet := make(map[string][]string)
		for _, r := range results {
			cleanSet[r.Hostname] = append(cleanSet[r.Hostname], r.IP)
		}
		for k, v := range cleanSet {
			fmt.Printf("%s:\n", k)
			for _, ip := range v {
				fmt.Printf("\t%s\n", ip)
			}
		}
	} else {
		w := tabwriter.NewWriter(os.Stdout, 0, 8, 4, ' ', 0)
		fmt.Fprintln(w, "IP\tHostname\tSource")
		for _, r := range results {
			fmt.Fprintf(w, "%s\t%s\t%s\n", r.IP, r.Hostname, r.Source)
		}
		w.Flush()
	}
}
<|endoftext|>


<commit_before>package main

import (
	"fmt"
	"log"
	"net/http"

	"github.com/hellodudu/comment/comt"
	"github.com/hellodudu/comment/task"
)

var td *task.TaskDispatcher
var ReqNum int = 0
var appMap map[int]comt.App

func taskHandler(w http.ResponseWriter, r *http.Request) {
	ReqNum = ReqNum + 1
	tk := &task.Task{Req: ReqNum}
	td.AddTask(tk)

	w.Write([]byte("It is done!"))
}

func createAppHandler(w http.ResponseWriter, r *http.Request) {
	// todo check app exist

	newApp := comt.App{AppID: 1, PubKey: "pub_key", PriKey: "pri_key"}
	appMap[newApp.AppID] = newApp

	fmt.Printf("create new app %v!\n", appMap[newApp.AppID])

	w.Write([]byte("It is done!"))
}

func main() {
	var err error
	if td, err = task.NewTaskDispatcher(); err != nil {
		panic("new task dispatcher failed!")
	}

	appMap = make(map[int]comt.App)

	http.HandleFunc("/create_app", createAppHandler)
	http.HandleFunc("/task", taskHandler)
	log.Fatal(http.ListenAndServe(":8080", nil))

}
<commit_msg>pack code<commit_after>package main

import (
	"fmt"
	"log"
	"net/http"

	"github.com/hellodudu/comment/comt"
	"github.com/hellodudu/comment/task"
)

var td *task.TaskDispatcher
var ReqNum int = 0
var appMap map[int]comt.App

func taskHandler(w http.ResponseWriter, r *http.Request) {
	ReqNum = ReqNum + 1
	tk := &task.Task{Req: ReqNum}
	td.AddTask(tk)

	w.Write([]byte("It is done!"))
}

func createAppHandler(w http.ResponseWriter, r *http.Request) {
	// todo check app exist
	var byReq []byte
	r.Body.Read(byReq)
	fmt.Printf("recv request: %s\n", byReq)

	newApp := comt.App{AppID: 1, PubKey: "pub_key", PriKey: "pri_key"}
	appMap[newApp.AppID] = newApp

	fmt.Printf("create new app %v!\n", appMap[newApp.AppID])

	w.Write([]byte("It is done!"))
}

func main() {
	var err error
	if td, err = task.NewTaskDispatcher(); err != nil {
		panic("new task dispatcher failed!")
	}

	appMap = make(map[int]comt.App)

	http.HandleFunc("/create_app", createAppHandler)
	http.HandleFunc("/task", taskHandler)
	log.Fatal(http.ListenAndServe(":8080", nil))

}
<|endoftext|>