<commit_before>package api

import (
	"encoding/json"
	"fmt"
	"net/http"
	"strings"

	"github.com/julienschmidt/httprouter"
)

// HttpGET is a utility function for making http get requests to sia with a whitelisted user-agent
func HttpGET(url string) (resp *http.Response, err error) {
	req, err := http.NewRequest("GET", url, nil)
	if err != nil {
		return nil, err
	}
	req.Header.Add("User-Agent", "Sia-Agent")
	return new(http.Client).Do(req)
}

// HttpPOST is a utility function for making post requests to sia with a whitelisted user-agent
func HttpPOST(url string, data string) (resp *http.Response, err error) {
	req, err := http.NewRequest("POST", url, strings.NewReader(data))
	if err != nil {
		return nil, err
	}
	req.Header.Add("User-Agent", "Sia-Agent")
	req.Header.Add("Content-Type", "application/x-www-form-urlencoded")
	return new(http.Client).Do(req)
}

// requireUserAgent is middleware that requires all requests to set a
// UserAgent that contains the specified string.
func requireUserAgent(h http.Handler, ua string) http.Handler {
	return http.HandlerFunc(func(w http.ResponseWriter, req *http.Request) {
		if !strings.Contains(req.UserAgent(), ua) {
			writeError(w, "Browser access disabled due to security vulnerability. Use Sia-UI or siac.", http.StatusBadRequest)
			return
		}
		h.ServeHTTP(w, req)
	})
}

// initAPI determines which functions handle each API call.
func (srv *Server) initAPI() {
	router := httprouter.New()
	router.NotFound = http.HandlerFunc(srv.unrecognizedCallHandler) // custom 404

	// Daemon API Calls - Unfinished
	router.GET("/daemon/constants", srv.daemonConstantsHandler)
	router.GET("/daemon/version", srv.daemonVersionHandler)
	router.GET("/daemon/stop", srv.daemonStopHandler)

	// Consensus API Calls
	if srv.cs != nil {
		router.GET("/consensus", srv.consensusHandler)
	}

	// Explorer API Calls
	if srv.explorer != nil {
		router.GET("/explorer", srv.explorerHandler)
		router.GET("/explorer/blocks/:height", srv.explorerBlocksHandler)
		router.GET("/explorer/hashes/:hash", srv.explorerHashHandler)
	}

	// Gateway API Calls - Unfinished
	if srv.gateway != nil {
		router.GET("/gateway", srv.gatewayHandler)
		router.POST("/gateway/add/:addr", srv.gatewayAddHandler)
		router.POST("/gateway/remove/:addr", srv.gatewayRemoveHandler)
	}

	// Host API Calls
	if srv.host != nil {
		router.GET("/host", srv.hostHandlerGET)
		router.POST("/host", srv.hostHandlerPOST)
		router.POST("/host/announce", srv.hostAnnounceHandler)
	}

	// Miner API Calls
	if srv.miner != nil {
		router.GET("/miner", srv.minerHandler)
		router.GET("/miner/header", srv.minerHeaderHandlerGET)
		router.POST("/miner/header", srv.minerHeaderHandlerPOST)
		router.GET("/miner/start", srv.minerStartHandler)
		router.GET("/miner/stop", srv.minerStopHandler)
		router.GET("/miner/headerforwork", srv.minerHeaderHandlerGET) // COMPATv0.4.8
		router.GET("/miner/submitheader", srv.minerHeaderHandlerPOST) // COMPATv0.4.8
	}

	// Renter API Calls - Unfinished
	if srv.renter != nil {
		router.GET("/renter/downloads", srv.renterDownloadsHandler)
		router.GET("/renter/files", srv.renterFilesHandler)

		router.POST("/renter/load", srv.renterLoadHandler)
		router.POST("/renter/loadascii", srv.renterLoadAsciiHandler)
		router.GET("/renter/share", srv.renterShareHandler)
		router.GET("/renter/shareascii", srv.renterShareAsciiHandler)

		router.POST("/renter/delete/*path", srv.renterDeleteHandler)
		router.GET("/renter/download/*path", srv.renterDownloadHandler)
		router.POST("/renter/rename/*path", srv.renterRenameHandler)
		router.POST("/renter/upload/*path", srv.renterUploadHandler)

		router.GET("/renter/hosts/active", srv.renterHostsActiveHandler)
		router.GET("/renter/hosts/all", srv.renterHostsAllHandler)
	}

	// TransactionPool API Calls - Unfinished
	if srv.tpool != nil {
		router.GET("/transactionpool/transactions", srv.transactionpoolTransactionsHandler)
	}

	// Wallet API Calls
	if srv.wallet != nil {
		router.GET("/wallet", srv.walletHandler)
		router.POST("/wallet/033x", srv.wallet033xHandler)
		router.GET("/wallet/address", srv.walletAddressHandler)
		router.GET("/wallet/addresses", srv.walletAddressesHandler)
		router.GET("/wallet/backup", srv.walletBackupHandler)
		router.POST("/wallet/init", srv.walletInitHandler)
		router.POST("/wallet/lock", srv.walletLockHandler)
		router.POST("/wallet/seed", srv.walletSeedHandler)
		router.GET("/wallet/seeds", srv.walletSeedsHandler)
		router.POST("/wallet/siacoins", srv.walletSiacoinsHandler)
		router.POST("/wallet/siafunds", srv.walletSiafundsHandler)
		router.POST("/wallet/siagkey", srv.walletSiagkeyHandler)
		router.GET("/wallet/transaction/:id", srv.walletTransactionHandler)
		router.GET("/wallet/transactions", srv.walletTransactionsHandler)
		router.GET("/wallet/transactions/:addr", srv.walletTransactionsAddrHandler)
		router.POST("/wallet/unlock", srv.walletUnlockHandler)
		router.POST("/wallet/encrypt", srv.walletInitHandler) // COMPATv0.4.0
	}

	// Apply UserAgent middleware and create HTTP server
	uaRouter := requireUserAgent(router, srv.requiredUserAgent)
	srv.apiServer = &http.Server{Handler: uaRouter}
}

// unrecognizedCallHandler handles calls to unknown pages (404).
func (srv *Server) unrecognizedCallHandler(w http.ResponseWriter, req *http.Request) {
	w.WriteHeader(http.StatusNotFound)
	fmt.Fprintf(w, "404 - Refer to API.md")
}

// writeError an error to the API caller.
func writeError(w http.ResponseWriter, msg string, err int) {
	http.Error(w, msg, err)
}

// writeJSON writes the object to the ResponseWriter. If the encoding fails, an
// error is written instead.
func writeJSON(w http.ResponseWriter, obj interface{}) {
	if json.NewEncoder(w).Encode(obj) != nil {
		http.Error(w, "Failed to encode response", http.StatusInternalServerError)
	}
}

// writeSuccess writes the success json object ({"Success":true}) to the
// ResponseWriter
func writeSuccess(w http.ResponseWriter) {
	writeJSON(w, struct{ Success bool }{true})
}
<commit_msg>declare API fully RESTful<commit_after>package api

import (
	"encoding/json"
	"net/http"
	"strings"

	"github.com/julienschmidt/httprouter"
)

// HttpGET is a utility function for making http get requests to sia with a whitelisted user-agent
func HttpGET(url string) (resp *http.Response, err error) {
	req, err := http.NewRequest("GET", url, nil)
	if err != nil {
		return nil, err
	}
	req.Header.Add("User-Agent", "Sia-Agent")
	return new(http.Client).Do(req)
}

// HttpPOST is a utility function for making post requests to sia with a whitelisted user-agent
func HttpPOST(url string, data string) (resp *http.Response, err error) {
	req, err := http.NewRequest("POST", url, strings.NewReader(data))
	if err != nil {
		return nil, err
	}
	req.Header.Add("User-Agent", "Sia-Agent")
	req.Header.Add("Content-Type", "application/x-www-form-urlencoded")
	return new(http.Client).Do(req)
}

// requireUserAgent is middleware that requires all requests to set a
// UserAgent that contains the specified string.
func requireUserAgent(h http.Handler, ua string) http.Handler {
	return http.HandlerFunc(func(w http.ResponseWriter, req *http.Request) {
		if !strings.Contains(req.UserAgent(), ua) {
			writeError(w, "Browser access disabled due to security vulnerability. Use Sia-UI or siac.", http.StatusBadRequest)
			return
		}
		h.ServeHTTP(w, req)
	})
}

// initAPI determines which functions handle each API call.
func (srv *Server) initAPI() {
	router := httprouter.New()
	router.NotFound = http.HandlerFunc(srv.unrecognizedCallHandler) // custom 404

	// Daemon API Calls
	router.GET("/daemon/constants", srv.daemonConstantsHandler)
	router.GET("/daemon/version", srv.daemonVersionHandler)
	router.GET("/daemon/stop", srv.daemonStopHandler)

	// Consensus API Calls
	if srv.cs != nil {
		router.GET("/consensus", srv.consensusHandler)
	}

	// Explorer API Calls
	if srv.explorer != nil {
		router.GET("/explorer", srv.explorerHandler)
		router.GET("/explorer/blocks/:height", srv.explorerBlocksHandler)
		router.GET("/explorer/hashes/:hash", srv.explorerHashHandler)
	}

	// Gateway API Calls
	if srv.gateway != nil {
		router.GET("/gateway", srv.gatewayHandler)
		router.POST("/gateway/add/:addr", srv.gatewayAddHandler)
		router.POST("/gateway/remove/:addr", srv.gatewayRemoveHandler)
	}

	// Host API Calls
	if srv.host != nil {
		router.GET("/host", srv.hostHandlerGET)
		router.POST("/host", srv.hostHandlerPOST)
		router.POST("/host/announce", srv.hostAnnounceHandler)
	}

	// Miner API Calls
	if srv.miner != nil {
		router.GET("/miner", srv.minerHandler)
		router.GET("/miner/header", srv.minerHeaderHandlerGET)
		router.POST("/miner/header", srv.minerHeaderHandlerPOST)
		router.GET("/miner/start", srv.minerStartHandler)
		router.GET("/miner/stop", srv.minerStopHandler)
		router.GET("/miner/headerforwork", srv.minerHeaderHandlerGET) // COMPATv0.4.8
		router.GET("/miner/submitheader", srv.minerHeaderHandlerPOST) // COMPATv0.4.8
	}

	// Renter API Calls
	if srv.renter != nil {
		router.GET("/renter/downloads", srv.renterDownloadsHandler)
		router.GET("/renter/files", srv.renterFilesHandler)

		router.POST("/renter/load", srv.renterLoadHandler)
		router.POST("/renter/loadascii", srv.renterLoadAsciiHandler)
		router.GET("/renter/share", srv.renterShareHandler)
		router.GET("/renter/shareascii", srv.renterShareAsciiHandler)

		router.POST("/renter/delete/*path", srv.renterDeleteHandler)
		router.GET("/renter/download/*path", srv.renterDownloadHandler)
		router.POST("/renter/rename/*path", srv.renterRenameHandler)
		router.POST("/renter/upload/*path", srv.renterUploadHandler)

		router.GET("/renter/hosts/active", srv.renterHostsActiveHandler)
		router.GET("/renter/hosts/all", srv.renterHostsAllHandler)
	}

	// TransactionPool API Calls
	if srv.tpool != nil {
		router.GET("/transactionpool/transactions", srv.transactionpoolTransactionsHandler)
	}

	// Wallet API Calls
	if srv.wallet != nil {
		router.GET("/wallet", srv.walletHandler)
		router.POST("/wallet/033x", srv.wallet033xHandler)
		router.GET("/wallet/address", srv.walletAddressHandler)
		router.GET("/wallet/addresses", srv.walletAddressesHandler)
		router.GET("/wallet/backup", srv.walletBackupHandler)
		router.POST("/wallet/init", srv.walletInitHandler)
		router.POST("/wallet/lock", srv.walletLockHandler)
		router.POST("/wallet/seed", srv.walletSeedHandler)
		router.GET("/wallet/seeds", srv.walletSeedsHandler)
		router.POST("/wallet/siacoins", srv.walletSiacoinsHandler)
		router.POST("/wallet/siafunds", srv.walletSiafundsHandler)
		router.POST("/wallet/siagkey", srv.walletSiagkeyHandler)
		router.GET("/wallet/transaction/:id", srv.walletTransactionHandler)
		router.GET("/wallet/transactions", srv.walletTransactionsHandler)
		router.GET("/wallet/transactions/:addr", srv.walletTransactionsAddrHandler)
		router.POST("/wallet/unlock", srv.walletUnlockHandler)
		router.POST("/wallet/encrypt", srv.walletInitHandler) // COMPATv0.4.0
	}

	// Apply UserAgent middleware and create HTTP server
	uaRouter := requireUserAgent(router, srv.requiredUserAgent)
	srv.apiServer = &http.Server{Handler: uaRouter}
}

// unrecognizedCallHandler handles calls to unknown pages (404).
func (srv *Server) unrecognizedCallHandler(w http.ResponseWriter, req *http.Request) {
	http.Error(w, "404 - Refer to API.md", http.StatusNotFound)
}

// writeError an error to the API caller.
func writeError(w http.ResponseWriter, msg string, err int) {
	http.Error(w, msg, err)
}

// writeJSON writes the object to the ResponseWriter. If the encoding fails, an
// error is written instead.
func writeJSON(w http.ResponseWriter, obj interface{}) {
	if json.NewEncoder(w).Encode(obj) != nil {
		http.Error(w, "Failed to encode response", http.StatusInternalServerError)
	}
}

// writeSuccess writes the success json object ({"Success":true}) to the
// ResponseWriter
func writeSuccess(w http.ResponseWriter) {
	writeJSON(w, struct{ Success bool }{true})
}
<|endoftext|>


<commit_before>package api

import (
	"crypto/md5"
	"crypto/sha1"
	"net/http"
	"os"

	"github.com/FederationOfFathers/dashboard/db"
	"github.com/gorilla/handlers"
	"github.com/gorilla/mux"
	"github.com/gorilla/securecookie"
	"go.uber.org/zap"
)

var ListenOn = ":8866"
var Router = mux.NewRouter()
var Logger *zap.Logger

var DB *db.DB

func Run() {
	jwtSecretBytes = []byte(JWTSecret)
	s := sha1.New()
	m := md5.New()
	s.Write(jwtSecretBytes)
	m.Write(jwtSecretBytes)
	cookie = securecookie.New(s.Sum(nil), m.Sum(nil))
	Logger.Fatal(
		"error starting API http server",
		zap.String("listenOn", ListenOn),
		zap.Error(http.ListenAndServe(ListenOn,
			handlers.ProxyHeaders(
				handlers.CompressHandler(
					handlers.CombinedLoggingHandler(os.Stdout,
						handlers.CORS(
							handlers.AllowCredentials(),
							handlers.AllowedHeaders([]string{
								"Content-Type",
								"Authorization",
								"X-Requested-With",
							}),
							handlers.AllowedMethods([]string{
								"GET",
								"HEAD",
								"PUT",
								"POST",
								"DELETE",
							}),
							handlers.AllowedOrigins([]string{
								"http://ui.fofgaming.com",
								"https://ui.fofgaming.com",
								"http://dev.fofgaming.com",
								"https://dev.fofgaming.com",
								"http://127.0.0.1",
								"http://localhost",
							}),
						)(Router),
					),
				),
			),
		)))
}
<commit_msg>allow dev ui port<commit_after>package api

import (
	"crypto/md5"
	"crypto/sha1"
	"net/http"
	"os"

	"github.com/FederationOfFathers/dashboard/db"
	"github.com/gorilla/handlers"
	"github.com/gorilla/mux"
	"github.com/gorilla/securecookie"
	"go.uber.org/zap"
)

var ListenOn = ":8866"
var Router = mux.NewRouter()
var Logger *zap.Logger

var DB *db.DB

func Run() {
	jwtSecretBytes = []byte(JWTSecret)
	s := sha1.New()
	m := md5.New()
	s.Write(jwtSecretBytes)
	m.Write(jwtSecretBytes)
	cookie = securecookie.New(s.Sum(nil), m.Sum(nil))
	Logger.Fatal(
		"error starting API http server",
		zap.String("listenOn", ListenOn),
		zap.Error(http.ListenAndServe(ListenOn,
			handlers.ProxyHeaders(
				handlers.CompressHandler(
					handlers.CombinedLoggingHandler(os.Stdout,
						handlers.CORS(
							handlers.AllowCredentials(),
							handlers.AllowedHeaders([]string{
								"Content-Type",
								"Authorization",
								"X-Requested-With",
							}),
							handlers.AllowedMethods([]string{
								"GET",
								"HEAD",
								"PUT",
								"POST",
								"DELETE",
							}),
							handlers.AllowedOrigins([]string{
								"http://ui.fofgaming.com",
								"https://ui.fofgaming.com",
								"http://dev.fofgaming.com",
								"https://dev.fofgaming.com",
								"http://127.0.0.1:3000",
								"http://localhost:3000",
								"http://127.0.0.1",
								"http://localhost",
							}),
						)(Router),
					),
				),
			),
		)))
}
<|endoftext|>


<commit_before>package kumoru

import (
	"bytes"
	"crypto/hmac"
	"crypto/md5"
	"crypto/sha256"
	"crypto/tls"
	"encoding/base64"
	"encoding/json"
	"fmt"
	"io/ioutil"
	"net/http"
	"net/http/httputil"
	"net/url"
	"os"
	"os/user"
	"reflect"
	"strings"
	"time"

	log "github.com/Sirupsen/logrus"
)

const (
	POST   = "POST"
	GET    = "GET"
	HEAD   = "HEAD"
	PUT    = "PUT"
	DELETE = "DELETE"
	PATCH  = "PATCH"
)

type Request *http.Request
type Response *http.Response

type KumoruClient struct {
	SliceData         []interface{}
	BasicAuth         struct{ UserName, Password string }
	BounceToRawString bool
	Client            *http.Client
	Data              map[string]interface{}
	Debug             bool
	EndPoint          *Endpoints
	Errors            []error
	FormData          url.Values
	Header            map[string]string
	Logger            *log.Logger
	Method            string
	QueryData         url.Values
	RawString         string
	Sign              bool
	TargetType        string
	Tokens            *Ktokens
	Transport         *http.Transport
	Url               string
}

// Used to create a new KumoruClient Object.
func New() *KumoruClient {

	config := os.Getenv("KUMORU_CONFIG")

	if config == "" {
		usr, err := user.Current()
		if err != nil {
			log.Fatal(err)
		}

		config = usr.HomeDir + "/.kumoru/config"
	}

	c, err := LoadCreds(config, "auth")
	if err != nil {
		log.Fatal(err)
	}

	e, err := LoadEndpoints(config, "endpoints")
	if err != nil {
		log.Fatal(err)
	}

	t, err := LoadTokens(config, "tokens")
	if err != nil {
		log.Fatal(err)
	}

	k := &KumoruClient{
		BasicAuth:         struct{ UserName, Password string }{c.UserName, c.Password},
		BounceToRawString: false,
		Client:            &http.Client{},
		Data:              make(map[string]interface{}),
		Debug:             false,
		SliceData:         []interface{}{},
		EndPoint:          &e,
		Errors:            nil,
		FormData:          url.Values{},
		Header:            make(map[string]string),
		Logger:            log.New(),
		QueryData:         url.Values{},
		RawString:         "",
		Sign:              false,
		TargetType:        "form",
		Tokens:            &t,
		Transport:         &http.Transport{},
		Url:               "",
	}

	return k
}

func (k *KumoruClient) SignRequest(enable bool) *KumoruClient {
	k.Sign = enable
	return k
}

func (k *KumoruClient) SetDebug(enable bool) *KumoruClient {
	k.Debug = enable
	return k
}

func (k *KumoruClient) SetLogger(logger *log.Logger) *KumoruClient {
	k.Logger = logger
	return k
}

// Clear KumoruClient data for a new request
func (k *KumoruClient) ClearKumoruClient() {
	k.BounceToRawString = false
	k.Data = make(map[string]interface{})
	k.Errors = nil
	k.FormData = url.Values{}
	k.Header = make(map[string]string)
	k.Method = ""
	k.QueryData = url.Values{}
	k.RawString = ""
	k.Sign = false
	k.TargetType = "form"
	k.Url = ""
	k.SliceData = []interface{}{}
}

func (k *KumoruClient) Get(targetUrl string) *KumoruClient {
	k.ClearKumoruClient()
	k.Method = GET
	k.Url = targetUrl
	k.Errors = nil

	return k
}

func (k *KumoruClient) Patch(targetUrl string) *KumoruClient {
	k.ClearKumoruClient()
	k.Method = PATCH
	k.Url = targetUrl
	k.Errors = nil

	return k
}

func (k *KumoruClient) Put(targetUrl string) *KumoruClient {
	k.ClearKumoruClient()
	k.Method = PUT
	k.Url = targetUrl
	k.Errors = nil

	return k
}

func (k *KumoruClient) Delete(targetUrl string) *KumoruClient {
	k.ClearKumoruClient()
	k.Method = DELETE
	k.Url = targetUrl
	k.Errors = nil

	return k
}

func (k *KumoruClient) Post(targetUrl string) *KumoruClient {
	k.ClearKumoruClient()
	k.Method = POST
	k.Url = targetUrl
	k.Errors = nil

	return k
}

func (k *KumoruClient) Head(targetUrl string) *KumoruClient {
	k.ClearKumoruClient()
	k.Method = POST
	k.Url = targetUrl
	k.Errors = nil

	return k
}

// SetHeader headers
// kumoru.New().
// POST("/application/B8658129-701E-432C-BD80-5D0F464EC932").
// SetHeader("Accept", "application/x-www-form-urlencoded")
func (k *KumoruClient) SetHeader(param string, value string) *KumoruClient {
	k.Header[param] = value
	return k
}

func (k *KumoruClient) Param(key string, value string) *KumoruClient {
	k.QueryData.Add(key, value)
	return k
}

// Probably can merge in the initialize section
func (k *KumoruClient) SetBasicAuth(username string, password string) *KumoruClient {
	k.BasicAuth = struct{ UserName, Password string }{username, password}
	return k
}

// Query fucntion forms a query-string in the url of GET method or body of POST method.
// Usage Example:
//
// kumoru.New().
// Get("/applications/").
// Query(`{ query: 'myapp' }`).
// Query(`{ limit: '5' }`).
// End()
//
// kumoru.New().
// Get("/applications/").
// Query("query=myapp&limit=5").
// End()
//
// kumoru.New().
// Get("/applications/").
// Query("query=myapp&limit=5").
// Query(`{ sort: 'asc' }`).
// End()

func (k *KumoruClient) Query(content interface{}) *KumoruClient {
	switch v := reflect.ValueOf(content); v.Kind() {

	case reflect.String:
		k.queryString(v.String())
	case reflect.Struct:
		k.queryStruct(v.Interface())
	default:
	}
	return k
}

func (k *KumoruClient) queryString(content string) *KumoruClient {
	var val map[string]string
	if err := json.Unmarshal([]byte(content), &val); err == nil {
		for key, v := range val {
			k.QueryData.Add(key, v)
		}
	} else {
		if queryVal, err := url.ParseQuery(content); err == nil {
			for key, _ := range queryVal {
				k.QueryData.Add(key, queryVal.Get(key))
			}
		} else {
			k.Errors = append(k.Errors, err)
		}
	}
	return k
}

func (k *KumoruClient) queryStruct(content interface{}) *KumoruClient {
	if marchalContent, err := json.Marshal(content); err != nil {
		k.Errors = append(k.Errors, err)
	} else {
		var val map[string]interface{}
		if err := json.Unmarshal(marchalContent, &val); err != nil {
			k.Errors = append(k.Errors, err)
		} else {
			for key, v := range val {
				key = strings.ToLower(key)
				k.QueryData.Add(key, v.(string))
			}
		}
	}
	return k
}

func (k *KumoruClient) TLSClientConfig(config *tls.Config) *KumoruClient {
	k.Transport.TLSClientConfig = config
	return k
}

// End() or EndBytes() must be called to execute the call otherwise it won't do a thing.
func (k *KumoruClient) End(callback ...func(response Response, body string, errs []error)) (Response, string, []error) {
	var bytesCallback []func(response Response, body []byte, errs []error)
	if len(callback) > 0 {
		bytesCallback = []func(response Response, body []byte, errs []error){
			func(response Response, body []byte, errs []error) {
				callback[0](response, string(body), errs)
			},
		}
	}
	resp, body, errs := k.EndBytes(bytesCallback...)
	bodyString := string(body)
	return resp, bodyString, errs
}

// EndBytes should be used when you want the body as bytes.
func (k *KumoruClient) EndBytes(callback ...func(response Response, body []byte, errs []error)) (Response, []byte, []error) {
	// check whether there is an error. if yes, return all errors
	if len(k.Errors) != 0 {
		return nil, nil, k.Errors
	}

	req, err := k.NewRequest()

	if err != nil {
		k.Errors = append(k.Errors, err)
		return nil, nil, k.Errors
	}

	for key, v := range k.Header {
		req.Header.Set(key, v)
	}

	// Add all querystring from Query func
	q := req.URL.Query()
	for key, v := range k.QueryData {
		for _, vv := range v {
			q.Add(key, vv)
		}
	}
	req.URL.RawQuery = q.Encode()

	// Sign Request
	if !k.Sign {
		if k.BasicAuth != struct{ UserName, Password string }{} {
			req.SetBasicAuth(k.BasicAuth.UserName, k.BasicAuth.Password)
		}
	} else {
		b1 := strings.NewReader(k.RawString)
		date := time.Now()
		signingString := k.Method + "\n"

		b, err := ioutil.ReadAll(b1)

		if err != nil {
			log.Fatal(err)
		}
		if len(b) != 0 {
			data := md5.Sum(b)
			newdata := fmt.Sprintf("%x", string(data[:16]))

			req.Header.Set("Content-MD5", newdata)

			signingString += newdata + "\n" + "application/x-www-form-urlencoded" + "\n"
		}
		u, _ := url.Parse(k.Url)
		signingString += "x-kumoru-date:" + date.String() + "\n" + u.Path
		req.Header.Set("X-Kumoru-Date", date.String())

		h := hmac.New(sha256.New, []byte(k.Tokens.Private))
		h.Write([]byte(signingString))
		digest := fmt.Sprintf("%x", h.Sum(nil))

		req.Header.Set("Authorization", base64.StdEncoding.EncodeToString([]byte(k.Tokens.Public+":"+digest)))
	}

	// Set Transport
	k.Client.Transport = k.Transport

	// Log details of this request
	if k.Debug {
		dump, err := httputil.DumpRequest(req, true)
		if err != nil {
			k.Logger.Println("Error:", err)
		} else {
			k.Logger.Printf("HTTP Request: %s", string(dump))
		}
	}

	// Send request
	resp, err := k.Client.Do(req)
	if err != nil {
		k.Errors = append(k.Errors, err)
		return nil, nil, k.Errors
	}
	defer resp.Body.Close()

	// Log details of this response
	if k.Debug {
		dump, err := httputil.DumpResponse(resp, true)
		if nil != err {
			k.Logger.Println("Error:", err)
		} else {
			k.Logger.Printf("HTTP Response: %s", string(dump))
		}
	}

	body, _ := ioutil.ReadAll(resp.Body)
	// Reset resp.Body so it can be use again
	resp.Body = ioutil.NopCloser(bytes.NewBuffer(body))
	// deep copy response to give it to both return and callback func
	respCallback := *resp
	if len(callback) != 0 {
		callback[0](&respCallback, body, k.Errors)
	}
	return resp, body, nil
}

func changeMapToURLValues(data map[string]interface{}) url.Values {
	var newUrlValues = url.Values{}
	for k, v := range data {
		switch val := v.(type) {
		case string:
			newUrlValues.Add(k, val)
		case []string:
			for _, element := range val {
				newUrlValues.Add(k, element)
			}
		// if a number, change to string
		// json.Number used to protect against a wrong (for GoRequest) default conversion
		// which always converts number to float64.
		// This type is caused by using Decoder.UseNumber()
		case json.Number:
			newUrlValues.Add(k, string(val))
		}
	}
	return newUrlValues
}
<commit_msg>Add required header key to signing string<commit_after>package kumoru

import (
	"bytes"
	"crypto/hmac"
	"crypto/md5"
	"crypto/sha256"
	"crypto/tls"
	"encoding/base64"
	"encoding/json"
	"fmt"
	"io/ioutil"
	"net/http"
	"net/http/httputil"
	"net/url"
	"os"
	"os/user"
	"reflect"
	"strings"
	"time"

	log "github.com/Sirupsen/logrus"
)

const (
	POST   = "POST"
	GET    = "GET"
	HEAD   = "HEAD"
	PUT    = "PUT"
	DELETE = "DELETE"
	PATCH  = "PATCH"
)

type Request *http.Request
type Response *http.Response

type KumoruClient struct {
	SliceData         []interface{}
	BasicAuth         struct{ UserName, Password string }
	BounceToRawString bool
	Client            *http.Client
	Data              map[string]interface{}
	Debug             bool
	EndPoint          *Endpoints
	Errors            []error
	FormData          url.Values
	Header            map[string]string
	Logger            *log.Logger
	Method            string
	QueryData         url.Values
	RawString         string
	Sign              bool
	TargetType        string
	Tokens            *Ktokens
	Transport         *http.Transport
	Url               string
}

// Used to create a new KumoruClient Object.
func New() *KumoruClient {

	config := os.Getenv("KUMORU_CONFIG")

	if config == "" {
		usr, err := user.Current()
		if err != nil {
			log.Fatal(err)
		}

		config = usr.HomeDir + "/.kumoru/config"
	}

	c, err := LoadCreds(config, "auth")
	if err != nil {
		log.Fatal(err)
	}

	e, err := LoadEndpoints(config, "endpoints")
	if err != nil {
		log.Fatal(err)
	}

	t, err := LoadTokens(config, "tokens")
	if err != nil {
		log.Fatal(err)
	}

	k := &KumoruClient{
		BasicAuth:         struct{ UserName, Password string }{c.UserName, c.Password},
		BounceToRawString: false,
		Client:            &http.Client{},
		Data:              make(map[string]interface{}),
		Debug:             false,
		SliceData:         []interface{}{},
		EndPoint:          &e,
		Errors:            nil,
		FormData:          url.Values{},
		Header:            make(map[string]string),
		Logger:            log.New(),
		QueryData:         url.Values{},
		RawString:         "",
		Sign:              false,
		TargetType:        "form",
		Tokens:            &t,
		Transport:         &http.Transport{},
		Url:               "",
	}

	return k
}

func (k *KumoruClient) SignRequest(enable bool) *KumoruClient {
	k.Sign = enable
	return k
}

func (k *KumoruClient) SetDebug(enable bool) *KumoruClient {
	k.Debug = enable
	return k
}

func (k *KumoruClient) SetLogger(logger *log.Logger) *KumoruClient {
	k.Logger = logger
	return k
}

// Clear KumoruClient data for a new request
func (k *KumoruClient) ClearKumoruClient() {
	k.BounceToRawString = false
	k.Data = make(map[string]interface{})
	k.Errors = nil
	k.FormData = url.Values{}
	k.Header = make(map[string]string)
	k.Method = ""
	k.QueryData = url.Values{}
	k.RawString = ""
	k.Sign = false
	k.TargetType = "form"
	k.Url = ""
	k.SliceData = []interface{}{}
}

func (k *KumoruClient) Get(targetUrl string) *KumoruClient {
	k.ClearKumoruClient()
	k.Method = GET
	k.Url = targetUrl
	k.Errors = nil

	return k
}

func (k *KumoruClient) Patch(targetUrl string) *KumoruClient {
	k.ClearKumoruClient()
	k.Method = PATCH
	k.Url = targetUrl
	k.Errors = nil

	return k
}

func (k *KumoruClient) Put(targetUrl string) *KumoruClient {
	k.ClearKumoruClient()
	k.Method = PUT
	k.Url = targetUrl
	k.Errors = nil

	return k
}

func (k *KumoruClient) Delete(targetUrl string) *KumoruClient {
	k.ClearKumoruClient()
	k.Method = DELETE
	k.Url = targetUrl
	k.Errors = nil

	return k
}

func (k *KumoruClient) Post(targetUrl string) *KumoruClient {
	k.ClearKumoruClient()
	k.Method = POST
	k.Url = targetUrl
	k.Errors = nil

	return k
}

func (k *KumoruClient) Head(targetUrl string) *KumoruClient {
	k.ClearKumoruClient()
	k.Method = POST
	k.Url = targetUrl
	k.Errors = nil

	return k
}

// SetHeader headers
// kumoru.New().
// POST("/application/B8658129-701E-432C-BD80-5D0F464EC932").
// SetHeader("Accept", "application/x-www-form-urlencoded")
func (k *KumoruClient) SetHeader(param string, value string) *KumoruClient {
	k.Header[param] = value
	return k
}

func (k *KumoruClient) Param(key string, value string) *KumoruClient {
	k.QueryData.Add(key, value)
	return k
}

// Probably can merge in the initialize section
func (k *KumoruClient) SetBasicAuth(username string, password string) *KumoruClient {
	k.BasicAuth = struct{ UserName, Password string }{username, password}
	return k
}

// Query fucntion forms a query-string in the url of GET method or body of POST method.
// Usage Example:
//
// kumoru.New().
// Get("/applications/").
// Query(`{ query: 'myapp' }`).
// Query(`{ limit: '5' }`).
// End()
//
// kumoru.New().
// Get("/applications/").
// Query("query=myapp&limit=5").
// End()
//
// kumoru.New().
// Get("/applications/").
// Query("query=myapp&limit=5").
// Query(`{ sort: 'asc' }`).
// End()

func (k *KumoruClient) Query(content interface{}) *KumoruClient {
	switch v := reflect.ValueOf(content); v.Kind() {

	case reflect.String:
		k.queryString(v.String())
	case reflect.Struct:
		k.queryStruct(v.Interface())
	default:
	}
	return k
}

func (k *KumoruClient) queryString(content string) *KumoruClient {
	var val map[string]string
	if err := json.Unmarshal([]byte(content), &val); err == nil {
		for key, v := range val {
			k.QueryData.Add(key, v)
		}
	} else {
		if queryVal, err := url.ParseQuery(content); err == nil {
			for key, _ := range queryVal {
				k.QueryData.Add(key, queryVal.Get(key))
			}
		} else {
			k.Errors = append(k.Errors, err)
		}
	}
	return k
}

func (k *KumoruClient) queryStruct(content interface{}) *KumoruClient {
	if marchalContent, err := json.Marshal(content); err != nil {
		k.Errors = append(k.Errors, err)
	} else {
		var val map[string]interface{}
		if err := json.Unmarshal(marchalContent, &val); err != nil {
			k.Errors = append(k.Errors, err)
		} else {
			for key, v := range val {
				key = strings.ToLower(key)
				k.QueryData.Add(key, v.(string))
			}
		}
	}
	return k
}

func (k *KumoruClient) TLSClientConfig(config *tls.Config) *KumoruClient {
	k.Transport.TLSClientConfig = config
	return k
}

// End() or EndBytes() must be called to execute the call otherwise it won't do a thing.
func (k *KumoruClient) End(callback ...func(response Response, body string, errs []error)) (Response, string, []error) {
	var bytesCallback []func(response Response, body []byte, errs []error)
	if len(callback) > 0 {
		bytesCallback = []func(response Response, body []byte, errs []error){
			func(response Response, body []byte, errs []error) {
				callback[0](response, string(body), errs)
			},
		}
	}
	resp, body, errs := k.EndBytes(bytesCallback...)
	bodyString := string(body)
	return resp, bodyString, errs
}

// EndBytes should be used when you want the body as bytes.
func (k *KumoruClient) EndBytes(callback ...func(response Response, body []byte, errs []error)) (Response, []byte, []error) {
	// check whether there is an error. if yes, return all errors
	if len(k.Errors) != 0 {
		return nil, nil, k.Errors
	}

	req, err := k.NewRequest()

	if err != nil {
		k.Errors = append(k.Errors, err)
		return nil, nil, k.Errors
	}

	for key, v := range k.Header {
		req.Header.Set(key, v)
	}

	// Add all querystring from Query func
	q := req.URL.Query()
	for key, v := range k.QueryData {
		for _, vv := range v {
			q.Add(key, vv)
		}
	}
	req.URL.RawQuery = q.Encode()

	// Sign Request
	if !k.Sign {
		if k.BasicAuth != struct{ UserName, Password string }{} {
			req.SetBasicAuth(k.BasicAuth.UserName, k.BasicAuth.Password)
		}
	} else {
		b1 := strings.NewReader(k.RawString)
		date := time.Now()
		signingString := k.Method + "\n"

		b, err := ioutil.ReadAll(b1)

		if err != nil {
			log.Fatal(err)
		}
		if len(b) != 0 {
			data := md5.Sum(b)
			newdata := fmt.Sprintf("content-md5:%x", string(data[:16]))

			req.Header.Set("Content-MD5", newdata)

			signingString += fmt.Sprintf("content-type:%v", req.Header.Get("Content-Type")+"\n")
		}
		u, _ := url.Parse(k.Url)
		signingString += "x-kumoru-date:" + date.String() + "\n" + u.Path
		req.Header.Set("X-Kumoru-Date", date.String())

		h := hmac.New(sha256.New, []byte(k.Tokens.Private))
		h.Write([]byte(signingString))
		digest := fmt.Sprintf("%x", h.Sum(nil))

		req.Header.Set("Authorization", base64.StdEncoding.EncodeToString([]byte(k.Tokens.Public+":"+digest)))
	}

	// Set Transport
	k.Client.Transport = k.Transport

	// Log details of this request
	if k.Debug {
		dump, err := httputil.DumpRequest(req, true)
		if err != nil {
			k.Logger.Println("Error:", err)
		} else {
			k.Logger.Printf("HTTP Request: %s", string(dump))
		}
	}

	// Send request
	resp, err := k.Client.Do(req)
	if err != nil {
		k.Errors = append(k.Errors, err)
		return nil, nil, k.Errors
	}
	defer resp.Body.Close()

	// Log details of this response
	if k.Debug {
		dump, err := httputil.DumpResponse(resp, true)
		if nil != err {
			k.Logger.Println("Error:", err)
		} else {
			k.Logger.Printf("HTTP Response: %s", string(dump))
		}
	}

	body, _ := ioutil.ReadAll(resp.Body)
	// Reset resp.Body so it can be use again
	resp.Body = ioutil.NopCloser(bytes.NewBuffer(body))
	// deep copy response to give it to both return and callback func
	respCallback := *resp
	if len(callback) != 0 {
		callback[0](&respCallback, body, k.Errors)
	}
	return resp, body, nil
}

func changeMapToURLValues(data map[string]interface{}) url.Values {
	var newUrlValues = url.Values{}
	for k, v := range data {
		switch val := v.(type) {
		case string:
			newUrlValues.Add(k, val)
		case []string:
			for _, element := range val {
				newUrlValues.Add(k, element)
			}
		// if a number, change to string
		// json.Number used to protect against a wrong (for GoRequest) default conversion
		// which always converts number to float64.
		// This type is caused by using Decoder.UseNumber()
		case json.Number:
			newUrlValues.Add(k, string(val))
		}
	}
	return newUrlValues
}
<|endoftext|>


<commit_before>// +build windows,!linux,!darwin

/*
Binary vnc_to_vm connects VNC to a remote VM via its socket file over SSH.
- connect to the local SSH agent in cygwin
- connect to the remote server with SSH
- relay the remote VNC socket file over SSH into a local listener
- start VNC viewer with it

Requirements:
- cygwin, with ssh and socat installed, ssh-agent running
- socat on the destination, TightVNC viewer locally

To avoid opening a command window:
  go build -ldflags "-H windowsgui" vnc_to_vm.go
*/

package main

import (
	"bufio"
	"flag"
	"fmt"
	"io"
	"io/ioutil"
	"net"
	"os"
	"os/exec"
	"path/filepath"
	"regexp"
	"strings"

	"golang.org/x/crypto/ssh"
	"golang.org/x/crypto/ssh/agent"
)

var (
	flagHost   = flag.String("host", "", "Host to SSH to (user@host[:port]).")
	flagSocket = flag.String("socket", "", "Path to remote VNC socket.")
	flagViewer = flag.String("viewer", `C:\Program Files\TightVNC\tvnviewer.exe`, "Path to local TightVNC viewer.")
	flagCygwin = flag.String("cygwin", `C:\cygwin64`, "Path to cygwin root (for SSH agent and socat).")
)

func main() {
	flag.Parse()
	if err := launch(); err != nil {
		fmt.Printf("Error: %v\n\n[press enter to exit]", err)
		bufio.NewScanner(os.Stdin).Scan()
	}
}

func launch() error {
	if *flagHost == "" || *flagSocket == "" {
		flag.PrintDefaults()
		return nil
	}
	userHost := regexp.MustCompile(`^([^@]*)@(.*)$`).FindStringSubmatch(*flagHost)
	if len(userHost) == 0 {
		return fmt.Errorf("invalid host")
	}
	remote, err := sshToVNC(userHost[1], userHost[2], *flagSocket)
	if err != nil {
		return err
	}
	return vncViewer(remote)
}

func sshToVNC(user, host, socket string) (ReadWriteCloseWaiter, error) {
	agentc, err := connectAgent()
	if err != nil {
		return nil, err
	}
	defer agentc.Wait()
	defer agentc.Close()
	config := &ssh.ClientConfig{
		User: user,
		Auth: []ssh.AuthMethod{
			ssh.PublicKeysCallback(agent.NewClient(agentc).Signers),
		},
	}
	client, err := ssh.Dial("tcp", host, config)
	if err != nil {
		return nil, err
	}
	fmt.Println("[+] Connected to remote SSH server")
	session, err := client.NewSession()
	if err != nil {
		return nil, err
	}
	stdin, err := session.StdinPipe()
	if err != nil {
		return nil, err
	}
	stdout, err := session.StdoutPipe()
	if err != nil {
		return nil, err
	}
	if strings.Contains(socket, `"`) || strings.Contains(socket, `'`) {
		return nil, fmt.Errorf("invalid socket path")
	}
	if err := session.Start(fmt.Sprintf(`socat 'UNIX:"%s"' -`, socket)); err != nil {
		return nil, err
	}
	fmt.Println("[+] Connected to remote VNC")
	return &RWCWaiter{ioutil.NopCloser(stdout), stdin, session}, nil
}

func connectAgent() (ReadWriteCloseWaiter, error) {
	matches, err := filepath.Glob(*flagCygwin + `\tmp\ssh-*\agent.*`)
	if err != nil {
		return nil, err
	}
	if len(matches) == 0 {
		return nil, fmt.Errorf("agent not found")
	}
	if len(matches) > 1 {
		return nil, fmt.Errorf("multiple agents found")
	}
	path := strings.Replace(strings.TrimPrefix(matches[0], *flagCygwin), `\`, `/`, -1)
	cmd := exec.Command(*flagCygwin+`\bin\socat`, fmt.Sprintf("UNIX:%s", path), "-")
	stdin, err := cmd.StdinPipe()
	if err != nil {
		return nil, err
	}
	stdout, err := cmd.StdoutPipe()
	if err != nil {
		return nil, err
	}
	if err := cmd.Start(); err != nil {
		return nil, err
	}
	fmt.Println("[+] Connected to SSH agent")
	return &RWCWaiter{stdout, stdin, cmd}, nil
}

func vncViewer(remote ReadWriteCloseWaiter) error {
	ln, err := net.Listen("tcp", "localhost:0")
	if err != nil {
		return err
	}
	port := ln.Addr().(*net.TCPAddr).Port
	cmd := exec.Command(*flagViewer, fmt.Sprintf("localhost::%d", port))
	if err := cmd.Start(); err != nil {
		return err
	}
	fmt.Println("[+] VNC viewer started")
	local, err := ln.Accept()
	if err != nil {
		return err
	}
	if err := ln.Close(); err != nil {
		return err
	}
	fmt.Println("[+] VNC viewer connected")
	done := make(chan struct{})
	cp := func(dst io.Writer, src io.Reader) {
		io.Copy(dst, src)
		done <- struct{}{}
	}
	go cp(remote, local)
	go cp(local, remote)
	go func() {
		cmd.Wait()
		done <- struct{}{}
	}()
	go func() {
		remote.Wait()
		done <- struct{}{}
	}()
	<-done
	return nil
}

type Waiter interface {
	Wait() error
}

type ReadWriteCloseWaiter interface {
	io.Reader
	io.Writer
	io.Closer
	Waiter
}

// RWCWaiter implements io.ReadWriteCloser and Waiter.
type RWCWaiter struct {
	io.ReadCloser
	io.WriteCloser
	Waiter
}

// Close closes both the reader and writer.
func (s *RWCWaiter) Close() error {
	if err := s.ReadCloser.Close(); err != nil {
		return err
	}
	return s.WriteCloser.Close()
}

// Wait waits for the waiter.
func (s *RWCWaiter) Wait() error {
	return s.Waiter.Wait()
}
<commit_msg>kvm/vnc_to_vm: reuse windows/cygwin and simplify<commit_after>// +build windows

/*
Binary vnc_to_vm connects VNC to a remote VM via its socket file over SSH.
- connect to the local SSH agent in cygwin
- connect to the remote server with SSH
- relay the remote VNC socket file over SSH into a local listener
- start VNC viewer with it

Requirements:
- cygwin, with ssh and socat installed, ssh-agent running
- socat on the destination, TightVNC viewer locally

To avoid opening a command window:
  go build -ldflags -H=windowsgui vnc_to_vm.go
*/

package main

import (
	"flag"
	"fmt"
	"io"
	"log"
	"net"
	"os/exec"
	"regexp"
	"strings"

	"github.com/StalkR/misc/windows/cygwin"
	"golang.org/x/crypto/ssh"
	"golang.org/x/crypto/ssh/agent"
)

var (
	flagHost   = flag.String("host", "", "Host to SSH to (user@host[:port]).")
	flagSocket = flag.String("socket", "", "Path to remote VNC socket.")
	flagViewer = flag.String("viewer", `C:\Program Files\TightVNC\tvnviewer.exe`, "Path to local TightVNC viewer.")
)

func main() {
	flag.Parse()
	if err := launch(); err != nil {
		log.Fatal(err)
	}
}

func launch() error {
	if *flagHost == "" || *flagSocket == "" {
		flag.PrintDefaults()
		return nil
	}
	userHost := regexp.MustCompile(`^([^@]*)@(.*)$`).FindStringSubmatch(*flagHost)
	if len(userHost) == 0 {
		return fmt.Errorf("invalid host")
	}
	user, host := userHost[1], userHost[2]
	if strings.Contains(*flagSocket, `"`) || strings.Contains(*flagSocket, `'`) {
		return fmt.Errorf("invalid socket path")
	}
	socket := *flagSocket

	vnc, err := vncViewer()
	if err != nil {
		return err
	}
	defer vnc.Close()
	return sshToVNC(user, host, socket, vnc)
}

func vncViewer() (io.ReadWriteCloser, error) {
	ln, err := net.Listen("tcp", "localhost:0")
	if err != nil {
		return nil, err
	}
	port := ln.Addr().(*net.TCPAddr).Port
	cmd := exec.Command(*flagViewer, fmt.Sprintf("localhost::%d", port))
	if err := cmd.Start(); err != nil {
		return nil, err
	}
	conn, err := ln.Accept()
	if err != nil {
		cmd.Wait()
		return nil, err
	}
	if err := ln.Close(); err != nil {
		cmd.Wait()
		return nil, err
	}
	log.Printf("VNC viewer connected (localhost:%v)", port)
	return &vncConn{conn, cmd}, nil
}

// vncConn implements io.ReadWriteCloser.
type vncConn struct {
	conn io.ReadWriteCloser
	cmd  *exec.Cmd
}

func (s *vncConn) Read(p []byte) (int, error)  { return s.conn.Read(p) }
func (s *vncConn) Write(p []byte) (int, error) { return s.conn.Write(p) }
func (s *vncConn) Close() error {
	var errors []error
	if err := s.conn.Close(); err != nil {
		errors = append(errors, err)
	}
	if err := s.cmd.Wait(); err != nil {
		errors = append(errors, err)
	}
	if len(errors) > 0 {
		if len(errors) == 1 {
			return errors[0]
		}
		return fmt.Errorf("close: %v errors, first: %v", len(errors), errors[0])
	}
	return nil
}

func sshToVNC(user, host, socket string, vnc io.ReadWriter) error {
	client, err := connect(user, host)
	if err != nil {
		return err
	}
	defer client.Close()
	log.Print("Connected to SSH server")
	session, err := client.NewSession()
	if err != nil {
		return err
	}
	defer session.Close()
	session.Stdin = vnc
	session.Stdout = vnc
	if err := session.Start(fmt.Sprintf(`socat 'UNIX:"%s"' -`, socket)); err != nil {
		return err
	}
	log.Print("Connected to VNC via SSH")
	return session.Wait()
}

func connect(user, host string) (*ssh.Client, error) {
	ag, err := cygwin.SSHAgent()
	if err != nil {
		return nil, err
	}
	defer ag.Close()
	config := &ssh.ClientConfig{
		User: user,
		Auth: []ssh.AuthMethod{
			ssh.PublicKeysCallback(agent.NewClient(ag).Signers),
		},
		HostKeyCallback: ssh.InsecureIgnoreHostKey(),
	}
	return ssh.Dial("tcp", host, config)
}
<|endoftext|>


<commit_before>package main

import "github.com/dmulholl/janus-go/janus"

import (
    "fmt"
    "os"
    "path/filepath"
)

const version = "2.1.0.dev"

var helptext = fmt.Sprintf(`
Usage: %s [FLAGS] [COMMAND]

  Ironclad is a command line password manager.

Flags:
  -h, --help        Print the application's help text.
  -v, --version     Print the application's version number.

Basic Commands:
  add               Add a new entry to a password database.
  delete            Delete one or more entries from a database.
  edit              Edit an existing database entry.
  gen               Generate a new random password.
  init              Initialize a new password database.
  list              List database entries.
  pass              Copy a password to the clipboard.
  url               Copy a url to the clipboard.
  user              Copy a username to the clipboard.

Additional Commands:
  cachepass         Change a database's cache password.
  config            Set or print a configuration option.
  decrypt           Decrypt a file.
  dump              Dump a database's internal JSON data store.
  encrypt           Encrypt a file.
  export            Export entries from a database.
  import            Import entries into a database.
  purge             Purge inactive (i.e. deleted) entries from a database.
  restore           Restore one or more previously deleted entries.
  masterpass        Change a database's master password.
  tags              List database tags.

Aliases:
  new               Alias for 'add'.
  show              Alias for 'list --verbose'.

Command Help:
  help <command>    Print the specified command's help text and exit.
`, filepath.Base(os.Args[0]))

func main() {

    // Instantiate an argument parser.
    parser := janus.NewParser()
    parser.Helptext = helptext
    parser.Version = version

    // Register commands.
    registerAddCmd(parser)
    registerCacheCmd(parser)
    registerConfigCmd(parser)
    registerDecryptCmd(parser)
    registerDeleteCmd(parser)
    registerDumpCmd(parser)
    registerEditCmd(parser)
    registerEncryptCmd(parser)
    registerExportCmd(parser)
    registerGenCmd(parser)
    registerImportCmd(parser)
    registerInitCmd(parser)
    registerListCmd(parser)
    registerPassCmd(parser)
    registerPurgeCmd(parser)
    registerRestoreCmd(parser)
    registerMasterpassCmd(parser)
    registerCachepassCmd(parser)
    registerTagsCmd(parser)
    registerUrlCmd(parser)
    registerUserCmd(parser)

    // Parse the command line arguments.
    parser.Parse()
    if !parser.HasCmd() {
        parser.ExitHelp()
    }
}
<commit_msg>Bump version to 2.1.0<commit_after>package main

import "github.com/dmulholl/janus-go/janus"

import (
    "fmt"
    "os"
    "path/filepath"
)

const version = "2.1.0"

var helptext = fmt.Sprintf(`
Usage: %s [FLAGS] [COMMAND]

  Ironclad is a command line password manager.

Flags:
  -h, --help        Print the application's help text.
  -v, --version     Print the application's version number.

Basic Commands:
  add               Add a new entry to a password database.
  delete            Delete one or more entries from a database.
  edit              Edit an existing database entry.
  gen               Generate a new random password.
  init              Initialize a new password database.
  list              List database entries.
  pass              Copy a password to the clipboard.
  url               Copy a url to the clipboard.
  user              Copy a username to the clipboard.

Additional Commands:
  cachepass         Change a database's cache password.
  config            Set or print a configuration option.
  decrypt           Decrypt a file.
  dump              Dump a database's internal JSON data store.
  encrypt           Encrypt a file.
  export            Export entries from a database.
  import            Import entries into a database.
  purge             Purge inactive (i.e. deleted) entries from a database.
  restore           Restore one or more previously deleted entries.
  masterpass        Change a database's master password.
  tags              List database tags.

Aliases:
  new               Alias for 'add'.
  show              Alias for 'list --verbose'.

Command Help:
  help <command>    Print the specified command's help text and exit.
`, filepath.Base(os.Args[0]))

func main() {

    // Instantiate an argument parser.
    parser := janus.NewParser()
    parser.Helptext = helptext
    parser.Version = version

    // Register commands.
    registerAddCmd(parser)
    registerCacheCmd(parser)
    registerConfigCmd(parser)
    registerDecryptCmd(parser)
    registerDeleteCmd(parser)
    registerDumpCmd(parser)
    registerEditCmd(parser)
    registerEncryptCmd(parser)
    registerExportCmd(parser)
    registerGenCmd(parser)
    registerImportCmd(parser)
    registerInitCmd(parser)
    registerListCmd(parser)
    registerPassCmd(parser)
    registerPurgeCmd(parser)
    registerRestoreCmd(parser)
    registerMasterpassCmd(parser)
    registerCachepassCmd(parser)
    registerTagsCmd(parser)
    registerUrlCmd(parser)
    registerUserCmd(parser)

    // Parse the command line arguments.
    parser.Parse()
    if !parser.HasCmd() {
        parser.ExitHelp()
    }
}
<|endoftext|>


<commit_before>package main

import (
    "fmt"
    "time"
    "github.com/nitram509/led"
    "image/color"
    "math/rand"
    "os"
    "sort"
    "strings"
    "regexp"
    "encoding/hex"
    "gopkg.in/alecthomas/kingpin.v1"
)

var (
    VERSION = "0.0.1"
    flagSetColor = kingpin.Flag("set-color", "Set color for device. The format must be \"#rrggbb\" or \"random\" or an HTML color name, like \"green\"").String()
    flagListColors = kingpin.Flag("list-colors", "List all available HTML color names").Bool()
    flagListDevices = kingpin.Flag("list-devices", "List all connected devices").Short('l').Bool()
)

func printListColorNames() {
    var colorNames []string
    for k := range colors {
        colorNames = append(colorNames, k)
    }
    sort.Strings(colorNames)
    for k := range colorNames {
        if (k > 0) {
            fmt.Print(",")
        }
        fmt.Print(colorNames[k])
    }
}

func printListDevices() {
    var i int = 0
    for devInfo := range led.Devices() {
        fmt.Printf("%d\t%s", i,  devInfo.GetType().String())
    }
}

func getFlagColor() color.Color {
    col := strings.ToLower(*flagSetColor)
    if (col == "random") {
        rand.Seed(time.Now().UnixNano())
        return color.RGBA{uint8(rand.Int()), uint8(rand.Int()), uint8(rand.Int()), 0xFF}
    }
    if (colors[col] != nil) {
        return colors[col]
    }
    validHexCode := regexp.MustCompile(`^#?([a-f0-9]{6})$`)
    if (validHexCode.MatchString(col)) {
        hexStr := validHexCode.FindStringSubmatch(col)
        bytes, err := hex.DecodeString(hexStr[1])
        if (err != nil) {
            fmt.Printf("invalid color code '%s'. use '#rrggbb' instead", hexStr[1])
            return nil
        }
        return color.RGBA{uint8(bytes[0]), uint8(bytes[1]), uint8(bytes[2]), 0xFF}
    }
    return nil
}

func main() {

    kingpin.Version(VERSION)
    kingpin.Parse()

    if (len(os.Args) <= 1) {
        kingpin.Usage()
        os.Exit(0)
    }

    if (flagListColors != nil && *flagListColors) {
        printListColorNames()
        os.Exit(0)
    }

    if (flagListDevices != nil && *flagListDevices) {
        printListDevices()
        os.Exit(0)
    }

    if (flagSetColor == nil) {
        os.Exit(0)
    }

    for devInfo := range led.Devices() {
        dev, err := devInfo.Open()
        if err != nil {
            fmt.Println(err)
            continue
        }
        defer func() {
            dev.Close()
        }()

        col := getFlagColor()
        if (col != nil) {
            dev.SetColor(col)
        }

    }

}
<commit_msg>fixed displaying index<commit_after>package main

import (
    "fmt"
    "time"
    "github.com/nitram509/led"
    "image/color"
    "math/rand"
    "os"
    "sort"
    "strings"
    "regexp"
    "encoding/hex"
    "gopkg.in/alecthomas/kingpin.v1"
)

var (
    VERSION = "0.0.1"
    flagSetColor = kingpin.Flag("set-color", "Set color for device. The format must be \"#rrggbb\", \"random\", \"off\" or an HTML color name, like \"green\"").String()
    flagListColors = kingpin.Flag("list-colors", "List all available HTML color names").Bool()
    flagListDevices = kingpin.Flag("list-devices", "List all connected devices").Short('l').Bool()
)

func printListColorNames() {
    var colorNames []string
    for k := range colors {
        colorNames = append(colorNames, k)
    }
    sort.Strings(colorNames)
    for k := range colorNames {
        if (k > 0) {
            fmt.Print(",")
        }
        fmt.Print(colorNames[k])
    }
}

func printListDevices() {
    var i int = 0
    for devInfo := range led.Devices() {
        fmt.Printf("%d\t%s\n", i,  devInfo.GetType().String())
        i++
    }
}

func getFlagColor() color.Color {
    col := strings.ToLower(*flagSetColor)
    if (col == "random") {
        rand.Seed(time.Now().UnixNano())
        return color.RGBA{uint8(rand.Int()), uint8(rand.Int()), uint8(rand.Int()), 0xFF}
    }
    if (col == "random") {
        return color.Black
    }
    if (colors[col] != nil) {
        return colors[col]
    }
    validHexCode := regexp.MustCompile(`^#?([a-f0-9]{6})$`)
    if (validHexCode.MatchString(col)) {
        hexStr := validHexCode.FindStringSubmatch(col)
        bytes, err := hex.DecodeString(hexStr[1])
        if (err != nil) {
            fmt.Printf("invalid color code '%s'. use '#rrggbb' instead", hexStr[1])
            return nil
        }
        return color.RGBA{uint8(bytes[0]), uint8(bytes[1]), uint8(bytes[2]), 0xFF}
    }
    return nil
}

func main() {

    kingpin.Version(VERSION)
    kingpin.Parse()

    if (len(os.Args) <= 1) {
        kingpin.Usage()
        os.Exit(0)
    }

    if (flagListColors != nil && *flagListColors) {
        printListColorNames()
        os.Exit(0)
    }

    if (flagListDevices != nil && *flagListDevices) {
        printListDevices()
        os.Exit(0)
    }

    if (flagSetColor == nil) {
        os.Exit(0)
    }

    for devInfo := range led.Devices() {
        dev, err := devInfo.Open()
        if err != nil {
            fmt.Println(err)
            continue
        }
        defer func() {
            dev.Close()
        }()

        col := getFlagColor()
        if (col != nil) {
            dev.SetColor(col)
        }

    }

}
<|endoftext|>


<commit_before>package lexer

// checker receives a pointer to a lexer and tries to
// lex a token. If it is successful, it will return it as
// well as true, otherwise it will return an empty token and false
type checker interface {
	Check(*Lexer) (Token, bool)
}

type delimiterChecker struct{}

func (d delimiterChecker) Check(l *Lexer) (Token, bool) {
	c := l.ch
	if l.ch == ',' {
		l.readChar()
		return newToken(COMMA, c, l.line), true
	} else if l.ch == ';' {
		l.readChar()
		return newToken(SEMICOLON, c, l.line), true
	} else if l.ch == '(' {
		l.readChar()
		return newToken(LPAREN, c, l.line), true
	} else if l.ch == ')' {
		l.readChar()
		return newToken(RPAREN, c, l.line), true
	} else if l.ch == '{' {
		l.readChar()
		return newToken(LBRACE, c, l.line), true
	} else if l.ch == '}' {
		l.readChar()
		return newToken(RBRACE, c, l.line), true
	}

	return Token{}, false
}

type eofChecker struct{}

func (c eofChecker) Check(l *Lexer) (Token, bool) {
	if l.ch == 0 {
		tok := Token{}
		tok.Type = EOF
		tok.Literal = ""
		l.readChar()
		return tok, true
	}

	return Token{}, false
}

type plusChecker struct{}

func (c plusChecker) Check(l *Lexer) (Token, bool) {
	if l.ch == '+' {
		l.readChar()
		if l.ch == '+' {
			tok := Token{}
			tok.Type = INC
			tok.Literal = "++"
			l.readChar()
			return tok, true
		}
		tok := newToken(PLUS, '+', l.line)
		return tok, true
	}

	return Token{}, false
}

type equalsChecker struct{}

func (c equalsChecker) Check(l *Lexer) (Token, bool) {
	tok := Token{}
	if l.ch == '=' {
		if l.input[l.position:l.position+3] == "===" {
			tok.Type = IDENTITY
			tok.Literal = "==="
			l.advance(3)

			return tok, true
		}
		if l.input[l.position:l.position+2] == "==" {
			tok.Type = EQUALS
			tok.Literal = "=="
			l.advance(2)

			return tok, true
		}
		if l.input[l.position:l.position+2] == "=>" {
			tok.Type = ARROW
			tok.Literal = "=>"
			l.advance(2)

			return tok, true
		}
		if l.input[l.position:l.position+2] == "=&" {
			tok.Type = REFERENCE
			tok.Literal = "=&"
			l.advance(2)
			return tok, true
		}
		tok = newToken(ASSIGN, l.ch, l.line)
		l.readChar()
		return tok, true
	}

	return tok, false
}

type numberChecker struct{}

func (c numberChecker) Check(l *Lexer) (Token, bool) {
	tok := Token{}
	if c.isInt(l.ch) {
		tok.Type = INT
		tok.Literal = c.readInteger(l)
		return tok, true
	}

	return tok, false
}

func (c numberChecker) isInt(b byte) bool {
	return b >= '0' && b <= '9'
}

func (c numberChecker) readInteger(l *Lexer) string {
	position := l.position
	for c.isInt(l.ch) {
		l.readChar()
	}

	return l.input[position:l.position]
}

type identifierChecker struct{}

func (i identifierChecker) Check(l *Lexer) (Token, bool) {
	tok := Token{}
	if i.isLetter(l.ch) {
		tok.Literal = i.readIdentifier(l)
		tok.Type = LookupIdent(tok.Literal)
		return tok, true
	}

	return tok, false
}

func (i identifierChecker) isLetter(ch byte) bool {
	return 'a' <= ch && ch <= 'z' || 'A' <= ch && ch <= 'Z' || ch == '_' || ch == '$'
}

func (i identifierChecker) readIdentifier(l *Lexer) string {
	position := l.position
	for i.isLetter(l.ch) || l.ch == '\\' {
		l.readChar()
	}

	return l.input[position:l.position]
}

type phptagChecker struct{}

func (p phptagChecker) Check(l *Lexer) (Token, bool) {
	tok := Token{}
	if l.input[l.position:l.position+5] == "<?php" {
		tok.Type = PHPTAG
		tok.Literal = "<?php"
		for range tok.Literal {
			l.readChar()
		}
		return tok, true
	}

	return tok, false
}

type compareChecker struct{}

func (c compareChecker) Check(l *Lexer) (Token, bool) {
	tok := Token{}
	if l.ch == '<' {
		if (l.input[l.position+1]) == '=' {
			tok.Type = LESSTHANOREQUAL
			tok.Literal = "<="
			l.advance(2)
			return tok, true
		}
		tok = newToken(LESSTHAN, l.ch, l.line)
		l.readChar()
		return tok, true
	}
	if l.ch == '>' {
		if (l.input[l.position+1]) == '=' {
			tok.Type = GREATERTHANOREQUAL
			tok.Literal = ">="
			l.advance(2)
			return tok, true
		}
		tok = newToken(GREATERTHAN, l.ch, l.line)
		l.readChar()
		return tok, true
	}

	return tok, false
}

type stringChecker struct {
	delimiter byte
	tokenType TokenType
}

func (s stringChecker) Check(l *Lexer) (Token, bool) {
	tok := Token{}
	if l.ch != s.delimiter {
		return tok, false
	}
	l.readChar()
	tok.Type = s.tokenType
	tok.Literal = s.readString(l)
	l.readChar()

	return tok, true
}

func (s stringChecker) readString(l *Lexer) string {
	pos := l.position
	var res []byte
	for {
		l.scan([]byte{s.delimiter, '\\'})
		if l.ch == '\\' {
			res = append(res, l.input[pos:l.position]...)
			l.advance(2)
			pos = l.position - 1
			continue
		}
		res = append(res, l.input[pos:l.position]...)
		break
	}
	return string(res)
}
<commit_msg>switch to switch statement in delimiter checker<commit_after>package lexer

// checker receives a pointer to a lexer and tries to
// lex a token. If it is successful, it will return it as
// well as true, otherwise it will return an empty token and false
type checker interface {
	Check(*Lexer) (Token, bool)
}

type delimiterChecker struct{}

func (d delimiterChecker) Check(l *Lexer) (Token, bool) {
	c := l.ch

	switch l.ch {
	case ',':
		l.readChar()
		return newToken(COMMA, c, l.line), true
	case ';':
		l.readChar()
		return newToken(SEMICOLON, c, l.line), true
	case '(':
		l.readChar()
		return newToken(LPAREN, c, l.line), true
	case ')':
		l.readChar()
		return newToken(RPAREN, c, l.line), true
	case '{':
		l.readChar()
		return newToken(LBRACE, c, l.line), true
	case '}':
		l.readChar()
		return newToken(RBRACE, c, l.line), true
	}

	return Token{}, false
}

type eofChecker struct{}

func (c eofChecker) Check(l *Lexer) (Token, bool) {
	if l.ch == 0 {
		tok := Token{}
		tok.Type = EOF
		tok.Literal = ""
		l.readChar()
		return tok, true
	}

	return Token{}, false
}

type plusChecker struct{}

func (c plusChecker) Check(l *Lexer) (Token, bool) {
	if l.ch == '+' {
		l.readChar()
		if l.ch == '+' {
			tok := Token{}
			tok.Type = INC
			tok.Literal = "++"
			l.readChar()
			return tok, true
		}
		tok := newToken(PLUS, '+', l.line)
		return tok, true
	}

	return Token{}, false
}

type equalsChecker struct{}

func (c equalsChecker) Check(l *Lexer) (Token, bool) {
	tok := Token{}
	if l.ch == '=' {
		if l.input[l.position:l.position+3] == "===" {
			tok.Type = IDENTITY
			tok.Literal = "==="
			l.advance(3)

			return tok, true
		}
		if l.input[l.position:l.position+2] == "==" {
			tok.Type = EQUALS
			tok.Literal = "=="
			l.advance(2)

			return tok, true
		}
		if l.input[l.position:l.position+2] == "=>" {
			tok.Type = ARROW
			tok.Literal = "=>"
			l.advance(2)

			return tok, true
		}
		if l.input[l.position:l.position+2] == "=&" {
			tok.Type = REFERENCE
			tok.Literal = "=&"
			l.advance(2)
			return tok, true
		}
		tok = newToken(ASSIGN, l.ch, l.line)
		l.readChar()
		return tok, true
	}

	return tok, false
}

type numberChecker struct{}

func (c numberChecker) Check(l *Lexer) (Token, bool) {
	tok := Token{}
	if c.isInt(l.ch) {
		tok.Type = INT
		tok.Literal = c.readInteger(l)
		return tok, true
	}

	return tok, false
}

func (c numberChecker) isInt(b byte) bool {
	return b >= '0' && b <= '9'
}

func (c numberChecker) readInteger(l *Lexer) string {
	position := l.position
	for c.isInt(l.ch) {
		l.readChar()
	}

	return l.input[position:l.position]
}

type identifierChecker struct{}

func (i identifierChecker) Check(l *Lexer) (Token, bool) {
	tok := Token{}
	if i.isLetter(l.ch) {
		tok.Literal = i.readIdentifier(l)
		tok.Type = LookupIdent(tok.Literal)
		return tok, true
	}

	return tok, false
}

func (i identifierChecker) isLetter(ch byte) bool {
	return 'a' <= ch && ch <= 'z' || 'A' <= ch && ch <= 'Z' || ch == '_' || ch == '$'
}

func (i identifierChecker) readIdentifier(l *Lexer) string {
	position := l.position
	for i.isLetter(l.ch) || l.ch == '\\' {
		l.readChar()
	}

	return l.input[position:l.position]
}

type phptagChecker struct{}

func (p phptagChecker) Check(l *Lexer) (Token, bool) {
	tok := Token{}
	if l.input[l.position:l.position+5] == "<?php" {
		tok.Type = PHPTAG
		tok.Literal = "<?php"
		for range tok.Literal {
			l.readChar()
		}
		return tok, true
	}

	return tok, false
}

type compareChecker struct{}

func (c compareChecker) Check(l *Lexer) (Token, bool) {
	tok := Token{}
	if l.ch == '<' {
		if (l.input[l.position+1]) == '=' {
			tok.Type = LESSTHANOREQUAL
			tok.Literal = "<="
			l.advance(2)
			return tok, true
		}
		tok = newToken(LESSTHAN, l.ch, l.line)
		l.readChar()
		return tok, true
	}
	if l.ch == '>' {
		if (l.input[l.position+1]) == '=' {
			tok.Type = GREATERTHANOREQUAL
			tok.Literal = ">="
			l.advance(2)
			return tok, true
		}
		tok = newToken(GREATERTHAN, l.ch, l.line)
		l.readChar()
		return tok, true
	}

	return tok, false
}

type stringChecker struct {
	delimiter byte
	tokenType TokenType
}

func (s stringChecker) Check(l *Lexer) (Token, bool) {
	tok := Token{}
	if l.ch != s.delimiter {
		return tok, false
	}
	l.readChar()
	tok.Type = s.tokenType
	tok.Literal = s.readString(l)
	l.readChar()

	return tok, true
}

func (s stringChecker) readString(l *Lexer) string {
	pos := l.position
	var res []byte
	for {
		l.scan([]byte{s.delimiter, '\\'})
		if l.ch == '\\' {
			res = append(res, l.input[pos:l.position]...)
			l.advance(2)
			pos = l.position - 1
			continue
		}
		res = append(res, l.input[pos:l.position]...)
		break
	}
	return string(res)
}
<|endoftext|>


<commit_before>package lfsapi

import (
	"context"
	"crypto/tls"
	"fmt"
	"io"
	"net"
	"net/http"
	"net/url"
	"os"
	"regexp"
	"strconv"
	"strings"
	"time"

	"github.com/git-lfs/git-lfs/config"
	"github.com/git-lfs/git-lfs/errors"
	"github.com/git-lfs/git-lfs/tools"
	"github.com/rubyist/tracerx"
)

const MediaType = "application/vnd.git-lfs+json; charset=utf-8"

var (
	UserAgent = "git-lfs"
	httpRE    = regexp.MustCompile(`\Ahttps?://`)
)

func (c *Client) NewRequest(method string, e Endpoint, suffix string, body interface{}) (*http.Request, error) {
	sshRes, err := c.SSH.Resolve(e, method)
	if err != nil {
		tracerx.Printf("ssh: %s failed, error: %s, message: %s",
			e.SshUserAndHost, err.Error(), sshRes.Message,
		)

		if len(sshRes.Message) > 0 {
			return nil, errors.Wrap(err, sshRes.Message)
		}
		return nil, err
	}

	prefix := e.Url
	if len(sshRes.Href) > 0 {
		prefix = sshRes.Href
	}

	if !httpRE.MatchString(prefix) {
		urlfragment := strings.SplitN(prefix, "?", 2)[0]
		return nil, fmt.Errorf("missing protocol: %q", urlfragment)
	}

	req, err := http.NewRequest(method, joinURL(prefix, suffix), nil)
	if err != nil {
		return req, err
	}

	for key, value := range sshRes.Header {
		req.Header.Set(key, value)
	}
	req.Header.Set("Accept", MediaType)

	if body != nil {
		if merr := MarshalToRequest(req, body); merr != nil {
			return req, merr
		}
		req.Header.Set("Content-Type", MediaType)
	}

	return req, err
}

const slash = "/"

func joinURL(prefix, suffix string) string {
	if strings.HasSuffix(prefix, slash) {
		return prefix + suffix
	}
	return prefix + slash + suffix
}

// Do sends an HTTP request to get an HTTP response. It wraps net/http, adding
// extra headers, redirection handling, and error reporting.
func (c *Client) Do(req *http.Request) (*http.Response, error) {
	req.Header = c.extraHeadersFor(req)

	return c.do(req)
}

// do performs an *http.Request respecting redirects, and handles the response
// as defined in c.handleResponse. Notably, it does not alter the headers for
// the request argument in any way.
func (c *Client) do(req *http.Request) (*http.Response, error) {
	req.Header.Set("User-Agent", UserAgent)

	res, err := c.doWithRedirects(c.httpClient(req.Host), req, nil)
	if err != nil {
		return res, err
	}

	return res, c.handleResponse(res)
}

// Close closes any resources that this client opened.
func (c *Client) Close() error {
	return c.httpLogger.Close()
}

func (c *Client) extraHeadersFor(req *http.Request) http.Header {
	extraHeaders := c.extraHeaders(req.URL)
	if len(extraHeaders) == 0 {
		return req.Header
	}

	copy := make(http.Header, len(req.Header))
	for k, vs := range req.Header {
		copy[k] = vs
	}

	for k, vs := range extraHeaders {
		for _, v := range vs {
			copy[k] = append(copy[k], v)
		}
	}
	return copy
}

func (c *Client) extraHeaders(u *url.URL) map[string][]string {
	hdrs := c.uc.GetAll("http", u.String(), "extraHeader")
	m := make(map[string][]string, len(hdrs))

	for _, hdr := range hdrs {
		parts := strings.SplitN(hdr, ":", 2)
		if len(parts) < 2 {
			continue
		}

		k, v := parts[0], strings.TrimSpace(parts[1])

		m[k] = append(m[k], v)
	}
	return m
}

func (c *Client) doWithRedirects(cli *http.Client, req *http.Request, via []*http.Request) (*http.Response, error) {
	tracedReq, err := c.traceRequest(req)
	if err != nil {
		return nil, err
	}

	var retries int
	if n, ok := Retries(req); ok {
		retries = n
	} else {
		retries = defaultRequestRetries
	}

	var res *http.Response

	requests := tools.MaxInt(0, retries) + 1
	for i := 0; i < requests; i++ {
		res, err = cli.Do(req)
		if err == nil {
			break
		}

		if seek, ok := req.Body.(io.Seeker); ok {
			seek.Seek(0, io.SeekStart)
		}

		c.traceResponse(req, tracedReq, nil)
	}

	if err != nil {
		c.traceResponse(req, tracedReq, nil)
		return nil, err
	}

	if res == nil {
		return nil, nil
	}

	c.traceResponse(req, tracedReq, res)

	if res.StatusCode != 301 &&
		res.StatusCode != 302 &&
		res.StatusCode != 303 &&
		res.StatusCode != 307 &&
		res.StatusCode != 308 {

		// Above are the list of 3xx status codes that we know
		// how to handle below. If the status code contained in
		// the HTTP response was none of them, return the (res,
		// err) tuple as-is, otherwise handle the redirect.
		return res, err
	}

	redirectTo := res.Header.Get("Location")
	locurl, err := url.Parse(redirectTo)
	if err == nil && !locurl.IsAbs() {
		locurl = req.URL.ResolveReference(locurl)
		redirectTo = locurl.String()
	}

	via = append(via, req)
	if len(via) >= 3 {
		return res, errors.New("too many redirects")
	}

	redirectedReq, err := newRequestForRetry(req, redirectTo)
	if err != nil {
		return res, err
	}

	return c.doWithRedirects(cli, redirectedReq, via)
}

func (c *Client) httpClient(host string) *http.Client {
	c.clientMu.Lock()
	defer c.clientMu.Unlock()

	if c.gitEnv == nil {
		c.gitEnv = make(testEnv)
	}

	if c.osEnv == nil {
		c.osEnv = make(testEnv)
	}

	if c.hostClients == nil {
		c.hostClients = make(map[string]*http.Client)
	}

	if client, ok := c.hostClients[host]; ok {
		return client
	}

	concurrentTransfers := c.ConcurrentTransfers
	if concurrentTransfers < 1 {
		concurrentTransfers = 8
	}

	dialtime := c.DialTimeout
	if dialtime < 1 {
		dialtime = 30
	}

	keepalivetime := c.KeepaliveTimeout
	if keepalivetime < 1 {
		keepalivetime = 1800
	}

	tlstime := c.TLSTimeout
	if tlstime < 1 {
		tlstime = 30
	}

	tr := &http.Transport{
		Proxy:               proxyFromClient(c),
		TLSHandshakeTimeout: time.Duration(tlstime) * time.Second,
		MaxIdleConnsPerHost: concurrentTransfers,
	}

	activityTimeout := 30
	if v, ok := c.uc.Get("lfs", fmt.Sprintf("https://%v", host), "activitytimeout"); ok {
		if i, err := strconv.Atoi(v); err == nil {
			activityTimeout = i
		} else {
			activityTimeout = 0
		}
	}

	dialer := &net.Dialer{
		Timeout:   time.Duration(dialtime) * time.Second,
		KeepAlive: time.Duration(keepalivetime) * time.Second,
		DualStack: true,
	}

	if activityTimeout > 0 {
		activityDuration := time.Duration(activityTimeout) * time.Second
		tr.DialContext = func(ctx context.Context, network, addr string) (net.Conn, error) {
			c, err := dialer.DialContext(ctx, network, addr)
			if c == nil {
				return c, err
			}
			if tc, ok := c.(*net.TCPConn); ok {
				tc.SetKeepAlive(true)
				tc.SetKeepAlivePeriod(dialer.KeepAlive)
			}
			return &deadlineConn{Timeout: activityDuration, Conn: c}, err
		}
	} else {
		tr.DialContext = dialer.DialContext
	}

	tr.TLSClientConfig = &tls.Config{}

	if isClientCertEnabledForHost(c, host) {
		tracerx.Printf("http: client cert for %s", host)
		tr.TLSClientConfig.Certificates = []tls.Certificate{getClientCertForHost(c, host)}
		tr.TLSClientConfig.BuildNameToCertificate()
	}

	if isCertVerificationDisabledForHost(c, host) {
		tr.TLSClientConfig.InsecureSkipVerify = true
	} else {
		tr.TLSClientConfig.RootCAs = getRootCAsForHost(c, host)
	}

	httpClient := &http.Client{
		Transport: tr,
		CheckRedirect: func(*http.Request, []*http.Request) error {
			return http.ErrUseLastResponse
		},
	}

	c.hostClients[host] = httpClient
	if c.VerboseOut == nil {
		c.VerboseOut = os.Stderr
	}

	return httpClient
}

func (c *Client) CurrentUser() (string, string) {
	userName, _ := c.gitEnv.Get("user.name")
	userEmail, _ := c.gitEnv.Get("user.email")
	return userName, userEmail
}

func newRequestForRetry(req *http.Request, location string) (*http.Request, error) {
	newReq, err := http.NewRequest(req.Method, location, nil)
	if err != nil {
		return nil, err
	}

	if req.URL.Scheme == "https" && newReq.URL.Scheme == "http" {
		return nil, errors.New("lfsapi/client: refusing insecure redirect, https->http")
	}

	sameHost := req.URL.Host == newReq.URL.Host
	for key := range req.Header {
		if key == "Authorization" {
			if !sameHost {
				continue
			}
		}
		newReq.Header.Set(key, req.Header.Get(key))
	}

	oldestURL := strings.SplitN(req.URL.String(), "?", 2)[0]
	newURL := strings.SplitN(newReq.URL.String(), "?", 2)[0]
	tracerx.Printf("api: redirect %s %s to %s", req.Method, oldestURL, newURL)

	// This body will have already been rewound from a call to
	// lfsapi.Client.traceRequest().
	newReq.Body = req.Body
	newReq.ContentLength = req.ContentLength

	// Copy the request's context.Context, if any.
	newReq = newReq.WithContext(req.Context())

	return newReq, nil
}

type deadlineConn struct {
	Timeout time.Duration
	net.Conn
}

func (c *deadlineConn) Read(b []byte) (int, error) {
	if err := c.Conn.SetDeadline(time.Now().Add(c.Timeout)); err != nil {
		return 0, err
	}
	return c.Conn.Read(b)
}

func (c *deadlineConn) Write(b []byte) (int, error) {
	if err := c.Conn.SetDeadline(time.Now().Add(c.Timeout)); err != nil {
		return 0, err
	}

	return c.Conn.Write(b)
}

func init() {
	UserAgent = config.VersionDesc
}
<commit_msg>Add hint that file:// URLs require a standalone transfer agent<commit_after>package lfsapi

import (
	"context"
	"crypto/tls"
	"fmt"
	"io"
	"net"
	"net/http"
	"net/url"
	"os"
	"regexp"
	"strconv"
	"strings"
	"time"

	"github.com/git-lfs/git-lfs/config"
	"github.com/git-lfs/git-lfs/errors"
	"github.com/git-lfs/git-lfs/tools"
	"github.com/rubyist/tracerx"
)

const MediaType = "application/vnd.git-lfs+json; charset=utf-8"

var (
	UserAgent = "git-lfs"
	httpRE    = regexp.MustCompile(`\Ahttps?://`)
)

var hintFileUrl = strings.TrimSpace(`
hint: The remote resolves to a file:// URL, which can only work with a
hint: standalone transfer agent.  See section "Using a Custom Transfer Type
hint: without the API server" in custom-transfers.md for details.
`)

func (c *Client) NewRequest(method string, e Endpoint, suffix string, body interface{}) (*http.Request, error) {
	if strings.HasPrefix(e.Url, "file://") {
		// Initial `\n` to avoid overprinting `Downloading LFS...`.
		fmt.Fprintf(os.Stderr, "\n%s\n", hintFileUrl)
	}

	sshRes, err := c.SSH.Resolve(e, method)
	if err != nil {
		tracerx.Printf("ssh: %s failed, error: %s, message: %s",
			e.SshUserAndHost, err.Error(), sshRes.Message,
		)

		if len(sshRes.Message) > 0 {
			return nil, errors.Wrap(err, sshRes.Message)
		}
		return nil, err
	}

	prefix := e.Url
	if len(sshRes.Href) > 0 {
		prefix = sshRes.Href
	}

	if !httpRE.MatchString(prefix) {
		urlfragment := strings.SplitN(prefix, "?", 2)[0]
		return nil, fmt.Errorf("missing protocol: %q", urlfragment)
	}

	req, err := http.NewRequest(method, joinURL(prefix, suffix), nil)
	if err != nil {
		return req, err
	}

	for key, value := range sshRes.Header {
		req.Header.Set(key, value)
	}
	req.Header.Set("Accept", MediaType)

	if body != nil {
		if merr := MarshalToRequest(req, body); merr != nil {
			return req, merr
		}
		req.Header.Set("Content-Type", MediaType)
	}

	return req, err
}

const slash = "/"

func joinURL(prefix, suffix string) string {
	if strings.HasSuffix(prefix, slash) {
		return prefix + suffix
	}
	return prefix + slash + suffix
}

// Do sends an HTTP request to get an HTTP response. It wraps net/http, adding
// extra headers, redirection handling, and error reporting.
func (c *Client) Do(req *http.Request) (*http.Response, error) {
	req.Header = c.extraHeadersFor(req)

	return c.do(req)
}

// do performs an *http.Request respecting redirects, and handles the response
// as defined in c.handleResponse. Notably, it does not alter the headers for
// the request argument in any way.
func (c *Client) do(req *http.Request) (*http.Response, error) {
	req.Header.Set("User-Agent", UserAgent)

	res, err := c.doWithRedirects(c.httpClient(req.Host), req, nil)
	if err != nil {
		return res, err
	}

	return res, c.handleResponse(res)
}

// Close closes any resources that this client opened.
func (c *Client) Close() error {
	return c.httpLogger.Close()
}

func (c *Client) extraHeadersFor(req *http.Request) http.Header {
	extraHeaders := c.extraHeaders(req.URL)
	if len(extraHeaders) == 0 {
		return req.Header
	}

	copy := make(http.Header, len(req.Header))
	for k, vs := range req.Header {
		copy[k] = vs
	}

	for k, vs := range extraHeaders {
		for _, v := range vs {
			copy[k] = append(copy[k], v)
		}
	}
	return copy
}

func (c *Client) extraHeaders(u *url.URL) map[string][]string {
	hdrs := c.uc.GetAll("http", u.String(), "extraHeader")
	m := make(map[string][]string, len(hdrs))

	for _, hdr := range hdrs {
		parts := strings.SplitN(hdr, ":", 2)
		if len(parts) < 2 {
			continue
		}

		k, v := parts[0], strings.TrimSpace(parts[1])

		m[k] = append(m[k], v)
	}
	return m
}

func (c *Client) doWithRedirects(cli *http.Client, req *http.Request, via []*http.Request) (*http.Response, error) {
	tracedReq, err := c.traceRequest(req)
	if err != nil {
		return nil, err
	}

	var retries int
	if n, ok := Retries(req); ok {
		retries = n
	} else {
		retries = defaultRequestRetries
	}

	var res *http.Response

	requests := tools.MaxInt(0, retries) + 1
	for i := 0; i < requests; i++ {
		res, err = cli.Do(req)
		if err == nil {
			break
		}

		if seek, ok := req.Body.(io.Seeker); ok {
			seek.Seek(0, io.SeekStart)
		}

		c.traceResponse(req, tracedReq, nil)
	}

	if err != nil {
		c.traceResponse(req, tracedReq, nil)
		return nil, err
	}

	if res == nil {
		return nil, nil
	}

	c.traceResponse(req, tracedReq, res)

	if res.StatusCode != 301 &&
		res.StatusCode != 302 &&
		res.StatusCode != 303 &&
		res.StatusCode != 307 &&
		res.StatusCode != 308 {

		// Above are the list of 3xx status codes that we know
		// how to handle below. If the status code contained in
		// the HTTP response was none of them, return the (res,
		// err) tuple as-is, otherwise handle the redirect.
		return res, err
	}

	redirectTo := res.Header.Get("Location")
	locurl, err := url.Parse(redirectTo)
	if err == nil && !locurl.IsAbs() {
		locurl = req.URL.ResolveReference(locurl)
		redirectTo = locurl.String()
	}

	via = append(via, req)
	if len(via) >= 3 {
		return res, errors.New("too many redirects")
	}

	redirectedReq, err := newRequestForRetry(req, redirectTo)
	if err != nil {
		return res, err
	}

	return c.doWithRedirects(cli, redirectedReq, via)
}

func (c *Client) httpClient(host string) *http.Client {
	c.clientMu.Lock()
	defer c.clientMu.Unlock()

	if c.gitEnv == nil {
		c.gitEnv = make(testEnv)
	}

	if c.osEnv == nil {
		c.osEnv = make(testEnv)
	}

	if c.hostClients == nil {
		c.hostClients = make(map[string]*http.Client)
	}

	if client, ok := c.hostClients[host]; ok {
		return client
	}

	concurrentTransfers := c.ConcurrentTransfers
	if concurrentTransfers < 1 {
		concurrentTransfers = 8
	}

	dialtime := c.DialTimeout
	if dialtime < 1 {
		dialtime = 30
	}

	keepalivetime := c.KeepaliveTimeout
	if keepalivetime < 1 {
		keepalivetime = 1800
	}

	tlstime := c.TLSTimeout
	if tlstime < 1 {
		tlstime = 30
	}

	tr := &http.Transport{
		Proxy:               proxyFromClient(c),
		TLSHandshakeTimeout: time.Duration(tlstime) * time.Second,
		MaxIdleConnsPerHost: concurrentTransfers,
	}

	activityTimeout := 30
	if v, ok := c.uc.Get("lfs", fmt.Sprintf("https://%v", host), "activitytimeout"); ok {
		if i, err := strconv.Atoi(v); err == nil {
			activityTimeout = i
		} else {
			activityTimeout = 0
		}
	}

	dialer := &net.Dialer{
		Timeout:   time.Duration(dialtime) * time.Second,
		KeepAlive: time.Duration(keepalivetime) * time.Second,
		DualStack: true,
	}

	if activityTimeout > 0 {
		activityDuration := time.Duration(activityTimeout) * time.Second
		tr.DialContext = func(ctx context.Context, network, addr string) (net.Conn, error) {
			c, err := dialer.DialContext(ctx, network, addr)
			if c == nil {
				return c, err
			}
			if tc, ok := c.(*net.TCPConn); ok {
				tc.SetKeepAlive(true)
				tc.SetKeepAlivePeriod(dialer.KeepAlive)
			}
			return &deadlineConn{Timeout: activityDuration, Conn: c}, err
		}
	} else {
		tr.DialContext = dialer.DialContext
	}

	tr.TLSClientConfig = &tls.Config{}

	if isClientCertEnabledForHost(c, host) {
		tracerx.Printf("http: client cert for %s", host)
		tr.TLSClientConfig.Certificates = []tls.Certificate{getClientCertForHost(c, host)}
		tr.TLSClientConfig.BuildNameToCertificate()
	}

	if isCertVerificationDisabledForHost(c, host) {
		tr.TLSClientConfig.InsecureSkipVerify = true
	} else {
		tr.TLSClientConfig.RootCAs = getRootCAsForHost(c, host)
	}

	httpClient := &http.Client{
		Transport: tr,
		CheckRedirect: func(*http.Request, []*http.Request) error {
			return http.ErrUseLastResponse
		},
	}

	c.hostClients[host] = httpClient
	if c.VerboseOut == nil {
		c.VerboseOut = os.Stderr
	}

	return httpClient
}

func (c *Client) CurrentUser() (string, string) {
	userName, _ := c.gitEnv.Get("user.name")
	userEmail, _ := c.gitEnv.Get("user.email")
	return userName, userEmail
}

func newRequestForRetry(req *http.Request, location string) (*http.Request, error) {
	newReq, err := http.NewRequest(req.Method, location, nil)
	if err != nil {
		return nil, err
	}

	if req.URL.Scheme == "https" && newReq.URL.Scheme == "http" {
		return nil, errors.New("lfsapi/client: refusing insecure redirect, https->http")
	}

	sameHost := req.URL.Host == newReq.URL.Host
	for key := range req.Header {
		if key == "Authorization" {
			if !sameHost {
				continue
			}
		}
		newReq.Header.Set(key, req.Header.Get(key))
	}

	oldestURL := strings.SplitN(req.URL.String(), "?", 2)[0]
	newURL := strings.SplitN(newReq.URL.String(), "?", 2)[0]
	tracerx.Printf("api: redirect %s %s to %s", req.Method, oldestURL, newURL)

	// This body will have already been rewound from a call to
	// lfsapi.Client.traceRequest().
	newReq.Body = req.Body
	newReq.ContentLength = req.ContentLength

	// Copy the request's context.Context, if any.
	newReq = newReq.WithContext(req.Context())

	return newReq, nil
}

type deadlineConn struct {
	Timeout time.Duration
	net.Conn
}

func (c *deadlineConn) Read(b []byte) (int, error) {
	if err := c.Conn.SetDeadline(time.Now().Add(c.Timeout)); err != nil {
		return 0, err
	}
	return c.Conn.Read(b)
}

func (c *deadlineConn) Write(b []byte) (int, error) {
	if err := c.Conn.SetDeadline(time.Now().Add(c.Timeout)); err != nil {
		return 0, err
	}

	return c.Conn.Write(b)
}

func init() {
	UserAgent = config.VersionDesc
}
<|endoftext|>


<commit_before>// +build windows

package bot

import (
	"fmt"
	"time"

	"golang.org/x/sys/windows/svc/eventlog"
)

var eventLog *eventlog.Log

// Log logs messages whenever the connector log level is
// less than the given level
func Log(l LogLevel, v ...interface{}) {

	logLock.Lock()
	currlevel := logLevel
	logLock.Unlock()

	if l >= currlevel || l == Audit {
		prefix := logLevelToStr(l) + ":"
		p := []interface{}{prefix}
		var msg string
		if len(v) == 1 {
			msg = fmt.Sprintln(prefix, v[0])
		} else {
			v = append(p, v...)
			msg = fmt.Sprintln(v...)
		}

		robot.logger.Print(msg)
		if eventLog != nil {
			switch l {
			case Info:
				eventLog.Info(1, msg)
			case Warn:
				eventLog.Warning(1, msg)
			case Error:
				eventLog.Error(1, msg)
			}
		}
		tsMsg := fmt.Sprintf("%s %s", time.Now().Format("Jan 2 15:04:05"), msg)
		logLock.Lock()
		logBuffer[logLine] = tsMsg
		logLine = (logLine + 1) % (buffLines - 1)
		logLock.Unlock()
	}
}
<commit_msg>Log Audit events as Info<commit_after>// +build windows

package bot

import (
	"fmt"
	"time"

	"golang.org/x/sys/windows/svc/eventlog"
)

var eventLog *eventlog.Log

// Log logs messages whenever the connector log level is
// less than the given level
func Log(l LogLevel, v ...interface{}) {

	logLock.Lock()
	currlevel := logLevel
	logLock.Unlock()

	if l >= currlevel || l == Audit {
		prefix := logLevelToStr(l) + ":"
		p := []interface{}{prefix}
		var msg string
		if len(v) == 1 {
			msg = fmt.Sprintln(prefix, v[0])
		} else {
			v = append(p, v...)
			msg = fmt.Sprintln(v...)
		}

		robot.logger.Print(msg)
		if eventLog != nil {
			switch l {
			case Info, Audit:
				eventLog.Info(1, msg)
			case Warn:
				eventLog.Warning(1, msg)
			case Error:
				eventLog.Error(1, msg)
			}
		}
		tsMsg := fmt.Sprintf("%s %s", time.Now().Format("Jan 2 15:04:05"), msg)
		logLock.Lock()
		logBuffer[logLine] = tsMsg
		logLine = (logLine + 1) % (buffLines - 1)
		logLock.Unlock()
	}
}
<|endoftext|>


<commit_before>package bql

import (
	"github.com/Sirupsen/logrus"
	"math/rand"
	"pfi/sensorbee/sensorbee/bql/execution"
	"pfi/sensorbee/sensorbee/bql/parser"
	"pfi/sensorbee/sensorbee/bql/udf"
	"pfi/sensorbee/sensorbee/core"
	"sync"
	"time"
)

type bqlBox struct {
	// stmt is the BQL statement executed by this box
	stmt *parser.SelectStmt
	// reg holds functions that can be used in this box
	reg udf.FunctionRegistry
	// plan is the execution plan for the SELECT statement in there
	execPlan execution.PhysicalPlan
	// mutex protects access to shared state
	mutex sync.Mutex
	// timeEmitterMutex protects access to those resources
	// accessed by the time-based emitter
	timeEmitterMutex sync.Mutex
	// emitterLimit holds a positive value if this box should
	// stop emitting items after a certain number of items
	emitterLimit int64
	// emitterSampling holds a positive value if this box should only
	// emit a certain subset of items (defined by emitterSamplingType)
	emitterSampling float64
	// emitterSamplingType holds a value different from
	// parser.UnspecifiedSamplingType if output sampling is active
	emitterSamplingType parser.EmitterSamplingType
	// genCount holds the number of items generated so far
	// (i.e. computed by the underlying execution plan). this is only
	// used if the count-based sampling is active.
	genCount int64
	// emitCount holds the number of items emitted so far
	emitCount int64
	// lastTuple points to the last tuple that was generated by
	// the underlying plan.
	lastTuple *core.Tuple
	// lastWriter points to the last writer that was passed to
	// `Process()`
	lastWriter core.Writer
	// stopped is an additional flag to signal the time-based emitter
	// that it should stop emitting items.
	stopped bool
	// removeMe is a function to remove this bqlBox from its
	// topology. A nil check must be done before calling.
	removeMe func()
}

func NewBQLBox(stmt *parser.SelectStmt, reg udf.FunctionRegistry) *bqlBox {
	return &bqlBox{stmt: stmt, reg: reg}
}

func (b *bqlBox) Init(ctx *core.Context) error {
	// create the execution plan
	analyzedPlan, err := execution.Analyze(*b.stmt, b.reg)
	if err != nil {
		return err
	}
	b.emitterLimit = analyzedPlan.EmitterLimit
	b.emitterSampling = analyzedPlan.EmitterSampling
	b.emitterSamplingType = analyzedPlan.EmitterSamplingType
	optimizedPlan, err := analyzedPlan.LogicalOptimize()
	if err != nil {
		return err
	}
	b.execPlan, err = optimizedPlan.MakePhysicalPlan(b.reg)
	if err != nil {
		return err
	}
	if b.emitterSamplingType == parser.TimeBasedSampling {
		go b.timeEmitter(ctx)
	}
	return nil
}

func (b *bqlBox) Process(ctx *core.Context, t *core.Tuple, s core.Writer) error {
	b.mutex.Lock()
	defer b.mutex.Unlock()

	// deal with statements that have an emitter limit. in particular,
	// if we are already over the limit, exit here
	b.timeEmitterMutex.Lock()
	emitterLimit := b.emitterLimit
	emitCount := b.emitCount
	b.timeEmitterMutex.Unlock()
	if emitterLimit >= 0 && emitCount >= emitterLimit {
		return nil
	}

	// feed tuple into plan
	resultData, err := b.execPlan.Process(t)
	if err != nil {
		return err
	}

	// emit result data as tuples
	for _, data := range resultData {
		tup := *t
		tup.Data = data
		if len(t.Trace) != 0 {
			tup.Trace = make([]core.TraceEvent, len(t.Trace))
			copy(tup.Trace, t.Trace)
		}

		// decide if we should emit a tuple for this item
		shouldWriteTuple := true
		if b.emitterSamplingType == parser.CountBasedSampling {
			shouldWriteTuple = b.genCount%int64(b.emitterSampling) == 0
			// with 1,000,000 items per second, the counter below will
			// overflow after running for 292,471 years. probably ok.
			b.genCount += 1
		} else if b.emitterSamplingType == parser.RandomizedSampling {
			// emitterSampling is in [0,1], not [0,100] any more
			shouldWriteTuple = rand.Float64() < b.emitterSampling
		} else if b.emitterSamplingType == parser.TimeBasedSampling {
			// we will never emit something from this function
			// when the time-based emitter is used
			b.timeEmitterMutex.Lock()
			b.lastTuple = &tup
			b.lastWriter = s
			b.timeEmitterMutex.Unlock()
			continue
		}

		// write the tuple to the connected box
		if shouldWriteTuple {
			if err := s.Write(ctx, &tup); err != nil {
				return err
			}
			b.timeEmitterMutex.Lock()
			b.emitCount += 1
			b.timeEmitterMutex.Unlock()
		}
		// stop emitting if we have hit the limit
		b.timeEmitterMutex.Lock()
		emitterLimit := b.emitterLimit
		emitCount := b.emitCount
		b.timeEmitterMutex.Unlock()
		if emitterLimit >= 0 && emitCount >= emitterLimit {
			break
		}
	}

	// remove this box if we are over the limit
	b.timeEmitterMutex.Lock()
	if b.emitterLimit >= 0 && b.emitCount >= b.emitterLimit {
		// avoid conflict with the timeEmitter (which will also perform
		// the same operation under some conditions)
		if b.removeMe != nil {
			b.removeMe()
			// don't call twice
			b.removeMe = nil
		}
	}
	b.timeEmitterMutex.Unlock()

	return nil
}

func (b *bqlBox) timeEmitter(ctx *core.Context) {
	// invariant: b.emitterSamplingType == TimeBasedSampling

	// generate a ticker that will tick every time we need to emit a tuple
	ticker := time.NewTicker(time.Duration(b.emitterSampling * float64(time.Second)))
	for _ = range ticker.C {
		// we need to lock here because we access the `stopped` flag, the
		// `lastTuple` and `lastWriter` pointer, as well as`emitCount`
		b.timeEmitterMutex.Lock()
		// b.stopped is set to true by either
		// - the Terminate function (in that case we may in no case
		//   write any further tuples to any writer)
		// - this function itself (if there is a LIMIT present that we hit)
		if b.stopped {
			b.timeEmitterMutex.Unlock()
			ticker.Stop()
			// eat the rest of the ticks
			continue
		}

		if b.lastTuple != nil && b.lastWriter != nil {
			if err := b.lastWriter.Write(ctx, b.lastTuple); err != nil {
				if ctx != nil {
					ctx.ErrLog(err).WithFields(logrus.Fields{
						"node_type": "box",
						"node_sink": b.lastWriter,
					}).Error("Cannot write tuple")
				}
			}
			// we do not want to emit the same tuple twice, so
			// we set it to null
			b.lastTuple = nil
			// increase the counter and check if we have reached the limit
			b.emitCount += 1
			if b.emitterLimit >= 0 && b.emitCount >= b.emitterLimit {
				b.stopped = true
				// if this function sets the b.stopped flag itself, it must
				// also remove the box from the topology. (if the b.stopped
				// flag is set by Terminate, we must not call removeMe,
				// therefore we cannot move this behind the loop.)
				if b.removeMe != nil {
					b.removeMe()
					// don't call twice
					b.removeMe = nil
				}
			}
		}
		// unlock the mutex while waiting
		b.timeEmitterMutex.Unlock()
	}
}

func (b *bqlBox) Terminate(ctx *core.Context) error {
	// signal to the time-based emitter that it should stop
	b.timeEmitterMutex.Lock()
	b.stopped = true
	b.timeEmitterMutex.Unlock()
	return nil
}
<commit_msg>Ignore panic in removeMe().<commit_after>package bql

import (
	"github.com/Sirupsen/logrus"
	"math/rand"
	"pfi/sensorbee/sensorbee/bql/execution"
	"pfi/sensorbee/sensorbee/bql/parser"
	"pfi/sensorbee/sensorbee/bql/udf"
	"pfi/sensorbee/sensorbee/core"
	"sync"
	"time"
)

type bqlBox struct {
	// stmt is the BQL statement executed by this box
	stmt *parser.SelectStmt
	// reg holds functions that can be used in this box
	reg udf.FunctionRegistry
	// plan is the execution plan for the SELECT statement in there
	execPlan execution.PhysicalPlan
	// mutex protects access to shared state
	mutex sync.Mutex
	// timeEmitterMutex protects access to those resources
	// accessed by the time-based emitter
	timeEmitterMutex sync.Mutex
	// emitterLimit holds a positive value if this box should
	// stop emitting items after a certain number of items
	emitterLimit int64
	// emitterSampling holds a positive value if this box should only
	// emit a certain subset of items (defined by emitterSamplingType)
	emitterSampling float64
	// emitterSamplingType holds a value different from
	// parser.UnspecifiedSamplingType if output sampling is active
	emitterSamplingType parser.EmitterSamplingType
	// genCount holds the number of items generated so far
	// (i.e. computed by the underlying execution plan). this is only
	// used if the count-based sampling is active.
	genCount int64
	// emitCount holds the number of items emitted so far
	emitCount int64
	// lastTuple points to the last tuple that was generated by
	// the underlying plan.
	lastTuple *core.Tuple
	// lastWriter points to the last writer that was passed to
	// `Process()`
	lastWriter core.Writer
	// stopped is an additional flag to signal the time-based emitter
	// that it should stop emitting items.
	stopped bool
	// removeMe is a function to remove this bqlBox from its
	// topology. A nil check must be done before calling.
	removeMe func()
}

func NewBQLBox(stmt *parser.SelectStmt, reg udf.FunctionRegistry) *bqlBox {
	return &bqlBox{stmt: stmt, reg: reg}
}

func (b *bqlBox) Init(ctx *core.Context) error {
	// create the execution plan
	analyzedPlan, err := execution.Analyze(*b.stmt, b.reg)
	if err != nil {
		return err
	}
	b.emitterLimit = analyzedPlan.EmitterLimit
	b.emitterSampling = analyzedPlan.EmitterSampling
	b.emitterSamplingType = analyzedPlan.EmitterSamplingType
	optimizedPlan, err := analyzedPlan.LogicalOptimize()
	if err != nil {
		return err
	}
	b.execPlan, err = optimizedPlan.MakePhysicalPlan(b.reg)
	if err != nil {
		return err
	}
	if b.emitterSamplingType == parser.TimeBasedSampling {
		go b.timeEmitter(ctx)
	}
	return nil
}

func (b *bqlBox) Process(ctx *core.Context, t *core.Tuple, s core.Writer) error {
	b.mutex.Lock()
	defer b.mutex.Unlock()

	// deal with statements that have an emitter limit. in particular,
	// if we are already over the limit, exit here
	b.timeEmitterMutex.Lock()
	emitterLimit := b.emitterLimit
	emitCount := b.emitCount
	b.timeEmitterMutex.Unlock()
	if emitterLimit >= 0 && emitCount >= emitterLimit {
		return nil
	}

	// feed tuple into plan
	resultData, err := b.execPlan.Process(t)
	if err != nil {
		return err
	}

	// emit result data as tuples
	for _, data := range resultData {
		tup := *t
		tup.Data = data
		if len(t.Trace) != 0 {
			tup.Trace = make([]core.TraceEvent, len(t.Trace))
			copy(tup.Trace, t.Trace)
		}

		// decide if we should emit a tuple for this item
		shouldWriteTuple := true
		if b.emitterSamplingType == parser.CountBasedSampling {
			shouldWriteTuple = b.genCount%int64(b.emitterSampling) == 0
			// with 1,000,000 items per second, the counter below will
			// overflow after running for 292,471 years. probably ok.
			b.genCount += 1
		} else if b.emitterSamplingType == parser.RandomizedSampling {
			// emitterSampling is in [0,1], not [0,100] any more
			shouldWriteTuple = rand.Float64() < b.emitterSampling
		} else if b.emitterSamplingType == parser.TimeBasedSampling {
			// we will never emit something from this function
			// when the time-based emitter is used
			b.timeEmitterMutex.Lock()
			b.lastTuple = &tup
			b.lastWriter = s
			b.timeEmitterMutex.Unlock()
			continue
		}

		// write the tuple to the connected box
		if shouldWriteTuple {
			if err := s.Write(ctx, &tup); err != nil {
				return err
			}
			b.timeEmitterMutex.Lock()
			b.emitCount += 1
			b.timeEmitterMutex.Unlock()
		}
		// stop emitting if we have hit the limit
		b.timeEmitterMutex.Lock()
		emitterLimit := b.emitterLimit
		emitCount := b.emitCount
		b.timeEmitterMutex.Unlock()
		if emitterLimit >= 0 && emitCount >= emitterLimit {
			break
		}
	}

	// remove this box if we are over the limit
	b.timeEmitterMutex.Lock()
	if b.emitterLimit >= 0 && b.emitCount >= b.emitterLimit {
		// avoid conflict with the timeEmitter (which will also perform
		// the same operation under some conditions)
		if b.removeMe != nil {
			b.callRemoveMeIgnoringPanic()
			// don't call twice
			b.removeMe = nil
		}
	}
	b.timeEmitterMutex.Unlock()

	return nil
}

func (b *bqlBox) timeEmitter(ctx *core.Context) {
	// invariant: b.emitterSamplingType == TimeBasedSampling

	// generate a ticker that will tick every time we need to emit a tuple
	ticker := time.NewTicker(time.Duration(b.emitterSampling * float64(time.Second)))
	for _ = range ticker.C {
		// we need to lock here because we access the `stopped` flag, the
		// `lastTuple` and `lastWriter` pointer, as well as`emitCount`
		b.timeEmitterMutex.Lock()
		// b.stopped is set to true by either
		// - the Terminate function (in that case we may in no case
		//   write any further tuples to any writer)
		// - this function itself (if there is a LIMIT present that we hit)
		if b.stopped {
			b.timeEmitterMutex.Unlock()
			ticker.Stop()
			// eat the rest of the ticks
			continue
		}

		if b.lastTuple != nil && b.lastWriter != nil {
			if err := b.lastWriter.Write(ctx, b.lastTuple); err != nil {
				if ctx != nil {
					ctx.ErrLog(err).WithFields(logrus.Fields{
						"node_type": "box",
						"node_sink": b.lastWriter,
					}).Error("Cannot write tuple")
				}
			}
			// we do not want to emit the same tuple twice, so
			// we set it to null
			b.lastTuple = nil
			// increase the counter and check if we have reached the limit
			b.emitCount += 1
			if b.emitterLimit >= 0 && b.emitCount >= b.emitterLimit {
				b.stopped = true
				// if this function sets the b.stopped flag itself, it must
				// also remove the box from the topology. (if the b.stopped
				// flag is set by Terminate, we must not call removeMe,
				// therefore we cannot move this behind the loop.)
				if b.removeMe != nil {
					b.callRemoveMeIgnoringPanic()
					// don't call twice
					b.removeMe = nil
				}
			}
		}
		// unlock the mutex while waiting
		b.timeEmitterMutex.Unlock()
	}
}

func (b *bqlBox) Terminate(ctx *core.Context) error {
	// signal to the time-based emitter that it should stop
	b.timeEmitterMutex.Lock()
	b.stopped = true
	b.timeEmitterMutex.Unlock()
	return nil
}

func (b *bqlBox) callRemoveMeIgnoringPanic() {
	defer func() {
		recover()
	}()
	b.removeMe()
}
<|endoftext|>