<commit_before>package main

import (
	"crypto/tls"
	"fmt"
	"io/ioutil"
	"net"
	"net/http"
	"path/filepath"
	"strings"
	"sync"
	"time"

	"github.com/litl/galaxy/log"
)

var (
	httpRouter *HostRouter
)

// This works along with the ServiceRegistry, and the individual Services to
// route http requests based on the Host header. The Resgistry hold the mapping
// of VHost names to individual services, and each service has it's own
// ReeverseProxy to fulfill the request.
// HostRouter contains the ReverseProxy http Listener, and has an http.Handler
// to service the requets.
type HostRouter struct {
	sync.Mutex
	// the http frontend
	server *http.Server

	// automatically redirect to https
	SSLOnly bool

	// HTTP/HTTPS
	Scheme string

	// track our listener so we can kill the server
	listener net.Listener
}

func NewHostRouter(httpServer *http.Server) *HostRouter {
	r := &HostRouter{
		Scheme: "http",
	}
	httpServer.Handler = r
	r.server = httpServer
	return r
}

func (r *HostRouter) ServeHTTP(w http.ResponseWriter, req *http.Request) {
	reqId := genId()
	req.Header.Set("X-Request-Id", reqId)

	if r.SSLOnly {
		if req.TLS != nil || req.Header.Get("X-Forwarded-Proto") != "https" {
			//TODO: verify RequestURI
			redirLoc := "https://" + req.Host + req.RequestURI
			http.Redirect(w, req, redirLoc, http.StatusMovedPermanently)
			return
		}
	}

	var err error
	host := req.Host

	if strings.Contains(host, ":") {
		host, _, err = net.SplitHostPort(req.Host)
		if err != nil {
			log.Warnf("%s", err)
		}
	}

	svc := Registry.GetVHostService(host)

	if svc != nil && svc.httpProxy != nil {
		// The vhost has a service registered, give it to the proxy
		svc.ServeHTTP(w, req)
		return
	}

	r.noHostHandler(w, req)
}

func (r *HostRouter) noHostHandler(w http.ResponseWriter, req *http.Request) {
	w.WriteHeader(http.StatusNotFound)
	fmt.Fprintln(w, "Not found")
}

// TODO: collect more stats?

// Start the HTTP Router frontend.
// Takes a channel to notify when the listener is started
// to safely synchronize tests.
func (r *HostRouter) Start(ready chan bool) {
	//FIXME: poor locking strategy
	r.Lock()
	var err error
	r.listener, err = newTimeoutListener("tcp", r.server.Addr, 300*time.Second)
	if err != nil {
		log.Errorf("%s", err)
		r.Unlock()
		return
	}

	listener := r.listener
	if r.Scheme == "https" {
		listener = tls.NewListener(listener, r.server.TLSConfig)
	}

	r.Unlock()

	log.Printf("%s server listening at %s", strings.ToUpper(r.Scheme), r.server.Addr)
	if ready != nil {
		close(ready)
	}

	// This will log a closed connection error every time we Stop
	// but that's mostly a testing issue.
	log.Errorf("%s", r.server.Serve(listener))
}

func (r *HostRouter) Stop() {
	r.listener.Close()
}

func startHTTPServer(wg *sync.WaitGroup) {
	defer wg.Done()

	//TODO: configure these timeouts somewhere
	httpServer := &http.Server{
		Addr:           httpAddr,
		ReadTimeout:    10 * time.Minute,
		WriteTimeout:   10 * time.Minute,
		MaxHeaderBytes: 1 << 20,
	}

	httpRouter = NewHostRouter(httpServer)
	httpRouter.SSLOnly = sslOnly

	httpRouter.Start(nil)
}

// find certs in and is the named directory, and match them up by their base
// name using '.pem' and '.key' as extensions.
func loadCerts(certDir string) (*tls.Config, error) {
	abs, err := filepath.Abs(certDir)
	if err != nil {
		return nil, err
	}

	dir, err := ioutil.ReadDir(abs)
	if err != nil {
		return nil, err
	}

	// [cert, key] pairs
	pairs := make(map[string][2]string)

	for _, f := range dir {
		name := f.Name()
		if strings.HasSuffix(name, ".pem") {
			p := pairs[name[:len(name)-4]]
			p[0] = filepath.Join(abs, name)
			pairs[name[:len(name)-4]] = p
		}
		if strings.HasSuffix(name, ".key") {
			p := pairs[name[:len(name)-4]]
			p[1] = filepath.Join(abs, name)
			pairs[name[:len(name)-4]] = p
		}
	}

	tlsCfg := &tls.Config{
		NextProtos: []string{"http/1.1"},
	}

	for key, pair := range pairs {
		if pair[0] == "" {
			log.Errorf("missing cert for key: %s", pair[1])
			continue
		}
		if pair[1] == "" {
			log.Errorf("missing key for cert: %s", pair[0])
			continue
		}

		cert, err := tls.LoadX509KeyPair(pair[0], pair[1])
		if err != nil {
			log.Error(err)
			continue
		}
		tlsCfg.Certificates = append(tlsCfg.Certificates, cert)
		log.Debugf("loaded X509KeyPair for %s", key)
	}

	if len(tlsCfg.Certificates) == 0 {
		return nil, fmt.Errorf("no tls certificates loaded")
	}

	tlsCfg.BuildNameToCertificate()

	return tlsCfg, nil
}

func startHTTPSServer(wg *sync.WaitGroup) {
	defer wg.Done()

	tlsCfg, err := loadCerts(certDir)
	if err != nil {
		log.Error(err)
		return
	}

	//TODO: configure these timeouts somewhere
	httpsServer := &http.Server{
		Addr:           httpsAddr,
		ReadTimeout:    10 * time.Minute,
		WriteTimeout:   10 * time.Minute,
		MaxHeaderBytes: 1 << 20,
		TLSConfig:      tlsCfg,
	}

	httpRouter = NewHostRouter(httpsServer)
	httpRouter.Scheme = "https"
	httpRouter.SSLOnly = sslOnly

	httpRouter.Start(nil)
}

type ErrorPage struct {
	// The Mutex protects access to the body slice, and headers
	// Everything else should be static once the ErrorPage is created.
	sync.Mutex

	Location    string
	StatusCodes []int

	// body contains the cached error page
	body []byte
	// important headers
	header http.Header
}

func (e *ErrorPage) Body() []byte {
	e.Lock()
	defer e.Unlock()
	return e.body
}

func (e *ErrorPage) SetBody(b []byte) {
	e.Lock()
	defer e.Unlock()
	e.body = b
}

func (e *ErrorPage) Header() http.Header {
	e.Lock()
	defer e.Unlock()
	return e.header
}

func (e *ErrorPage) SetHeader(h http.Header) {
	e.Lock()
	defer e.Unlock()
	e.header = h
}

// List of headers we want to cache for ErrorPages
var ErrorHeaders = []string{
	"Content-Type",
	"Content-Encoding",
	"Cache-Control",
	"Last-Modified",
	"Retry-After",
	"Set-Cookie",
}

// ErrorResponse provides a ReverProxy callback to process a response and
// insert custom error pages for a virtual host.
type ErrorResponse struct {
	sync.Mutex

	// map them by status for responses
	pages map[int]*ErrorPage

	// keep this handy to refresh the pages
	client *http.Client
}

func NewErrorResponse(pages map[string][]int) *ErrorResponse {
	errors := &ErrorResponse{
		pages: make(map[int]*ErrorPage),
	}

	// aggressively timeout connections
	errors.client = &http.Client{
		Transport: &http.Transport{
			Dial: (&net.Dialer{
				Timeout: 2 * time.Second,
			}).Dial,
			TLSHandshakeTimeout: 2 * time.Second,
		},
		Timeout: 5 * time.Second,
	}

	if pages != nil {
		errors.Update(pages)
	}
	return errors
}

// Get the ErrorPage, returning nil if the page was incomplete.
// We permanently cache error pages and headers once we've seen them.
func (e *ErrorResponse) Get(code int) *ErrorPage {
	e.Lock()
	page, ok := e.pages[code]
	e.Unlock()

	if !ok {
		// this is a code we don't handle
		return nil
	}

	body := page.Body()
	if body != nil {
		return page
	}

	// we haven't successfully fetched this error
	e.fetch(page)
	return page
}

func (e *ErrorResponse) fetch(page *ErrorPage) {
	log.Debugf("Fetching error page from %s", page.Location)
	resp, err := e.client.Get(page.Location)
	if err != nil {
		log.Warnf("Could not fetch %s: %s", page.Location, err.Error())
		return
	}
	defer resp.Body.Close()

	// If the StatusCode matches any of our registered codes, it's OK
	for _, code := range page.StatusCodes {
		if resp.StatusCode == code {
			resp.StatusCode = http.StatusOK
			break
		}
	}

	if resp.StatusCode != http.StatusOK {
		log.Warnf("Server returned %d when fetching %s", resp.StatusCode, page.Location)
		return
	}

	header := make(map[string][]string)
	for _, key := range ErrorHeaders {
		if hdr, ok := resp.Header[key]; ok {
			header[key] = hdr
		}
	}
	// set the headers along with the body below

	body, err := ioutil.ReadAll(resp.Body)
	if err != nil {
		log.Warnf("Error reading response from %s: %s", page.Location, err.Error())
		return
	}

	if len(body) > 0 {
		page.SetHeader(header)
		page.SetBody(body)
		return
	}
	log.Warnf("Empty response from %s", page.Location)
}

// This replaces all existing ErrorPages
func (e *ErrorResponse) Update(pages map[string][]int) {
	e.Lock()
	defer e.Unlock()

	e.pages = make(map[int]*ErrorPage)

	for loc, codes := range pages {
		page := &ErrorPage{
			StatusCodes: codes,
			Location:    loc,
		}

		for _, code := range codes {
			e.pages[code] = page
		}
		go e.fetch(page)
	}
}

func (e *ErrorResponse) CheckResponse(pr *ProxyRequest) bool {

	errPage := e.Get(pr.Response.StatusCode)
	if errPage != nil {
		// load the cached headers
		header := pr.ResponseWriter.Header()
		for key, val := range errPage.Header() {
			header[key] = val
		}

		pr.ResponseWriter.WriteHeader(pr.Response.StatusCode)
		pr.ResponseWriter.Write(errPage.Body())
		return false
	}

	return true
}

func logProxyRequest(pr *ProxyRequest) bool {
	// TODO: we may to be able to switch this off
	if pr == nil || pr.Request == nil {
		return true
	}

	var id, method, clientIP, url, backend, agent string
	var status int

	duration := pr.FinishTime.Sub(pr.StartTime)

	id = pr.Request.Header.Get("X-Request-Id")
	method = pr.Request.Method
	url = pr.Request.Host + pr.Request.RequestURI
	agent = pr.Request.UserAgent()
	status = pr.Response.StatusCode

	clientIP = pr.Request.Header.Get("X-Forwarded-For")
	if clientIP == "" {
		clientIP = pr.Request.RemoteAddr
	}

	if pr.Response != nil && pr.Response.Request != nil && pr.Response.Request.URL != nil {
		backend = pr.Response.Request.URL.Host
	}

	err := fmt.Sprintf("%v", pr.ProxyError)

	fmtStr := "id=%s method=%s client-ip=%s url=%s backend=%s status=%d duration=%s agent=%s, err=%s"

	log.Printf(fmtStr, id, method, clientIP, url, backend, status, duration, agent, err)
	return true
}
<commit_msg>shuttle: set X-Request-Id in reponse headers<commit_after>package main

import (
	"crypto/tls"
	"fmt"
	"io/ioutil"
	"net"
	"net/http"
	"path/filepath"
	"strings"
	"sync"
	"time"

	"github.com/litl/galaxy/log"
)

var (
	httpRouter *HostRouter
)

// This works along with the ServiceRegistry, and the individual Services to
// route http requests based on the Host header. The Resgistry hold the mapping
// of VHost names to individual services, and each service has it's own
// ReeverseProxy to fulfill the request.
// HostRouter contains the ReverseProxy http Listener, and has an http.Handler
// to service the requets.
type HostRouter struct {
	sync.Mutex
	// the http frontend
	server *http.Server

	// automatically redirect to https
	SSLOnly bool

	// HTTP/HTTPS
	Scheme string

	// track our listener so we can kill the server
	listener net.Listener
}

func NewHostRouter(httpServer *http.Server) *HostRouter {
	r := &HostRouter{
		Scheme: "http",
	}
	httpServer.Handler = r
	r.server = httpServer
	return r
}

func (r *HostRouter) ServeHTTP(w http.ResponseWriter, req *http.Request) {
	reqId := genId()
	req.Header.Set("X-Request-Id", reqId)
	w.Header().Add("X-Request-Id", reqId)

	if r.SSLOnly {
		if req.TLS != nil || req.Header.Get("X-Forwarded-Proto") != "https" {
			//TODO: verify RequestURI
			redirLoc := "https://" + req.Host + req.RequestURI
			http.Redirect(w, req, redirLoc, http.StatusMovedPermanently)
			return
		}
	}

	var err error
	host := req.Host

	if strings.Contains(host, ":") {
		host, _, err = net.SplitHostPort(req.Host)
		if err != nil {
			log.Warnf("%s", err)
		}
	}

	svc := Registry.GetVHostService(host)

	if svc != nil && svc.httpProxy != nil {
		// The vhost has a service registered, give it to the proxy
		svc.ServeHTTP(w, req)
		return
	}

	r.noHostHandler(w, req)
}

func (r *HostRouter) noHostHandler(w http.ResponseWriter, req *http.Request) {
	w.WriteHeader(http.StatusNotFound)
	fmt.Fprintln(w, "Not found")
}

// TODO: collect more stats?

// Start the HTTP Router frontend.
// Takes a channel to notify when the listener is started
// to safely synchronize tests.
func (r *HostRouter) Start(ready chan bool) {
	//FIXME: poor locking strategy
	r.Lock()
	var err error
	r.listener, err = newTimeoutListener("tcp", r.server.Addr, 300*time.Second)
	if err != nil {
		log.Errorf("%s", err)
		r.Unlock()
		return
	}

	listener := r.listener
	if r.Scheme == "https" {
		listener = tls.NewListener(listener, r.server.TLSConfig)
	}

	r.Unlock()

	log.Printf("%s server listening at %s", strings.ToUpper(r.Scheme), r.server.Addr)
	if ready != nil {
		close(ready)
	}

	// This will log a closed connection error every time we Stop
	// but that's mostly a testing issue.
	log.Errorf("%s", r.server.Serve(listener))
}

func (r *HostRouter) Stop() {
	r.listener.Close()
}

func startHTTPServer(wg *sync.WaitGroup) {
	defer wg.Done()

	//TODO: configure these timeouts somewhere
	httpServer := &http.Server{
		Addr:           httpAddr,
		ReadTimeout:    10 * time.Minute,
		WriteTimeout:   10 * time.Minute,
		MaxHeaderBytes: 1 << 20,
	}

	httpRouter = NewHostRouter(httpServer)
	httpRouter.SSLOnly = sslOnly

	httpRouter.Start(nil)
}

// find certs in and is the named directory, and match them up by their base
// name using '.pem' and '.key' as extensions.
func loadCerts(certDir string) (*tls.Config, error) {
	abs, err := filepath.Abs(certDir)
	if err != nil {
		return nil, err
	}

	dir, err := ioutil.ReadDir(abs)
	if err != nil {
		return nil, err
	}

	// [cert, key] pairs
	pairs := make(map[string][2]string)

	for _, f := range dir {
		name := f.Name()
		if strings.HasSuffix(name, ".pem") {
			p := pairs[name[:len(name)-4]]
			p[0] = filepath.Join(abs, name)
			pairs[name[:len(name)-4]] = p
		}
		if strings.HasSuffix(name, ".key") {
			p := pairs[name[:len(name)-4]]
			p[1] = filepath.Join(abs, name)
			pairs[name[:len(name)-4]] = p
		}
	}

	tlsCfg := &tls.Config{
		NextProtos: []string{"http/1.1"},
	}

	for key, pair := range pairs {
		if pair[0] == "" {
			log.Errorf("missing cert for key: %s", pair[1])
			continue
		}
		if pair[1] == "" {
			log.Errorf("missing key for cert: %s", pair[0])
			continue
		}

		cert, err := tls.LoadX509KeyPair(pair[0], pair[1])
		if err != nil {
			log.Error(err)
			continue
		}
		tlsCfg.Certificates = append(tlsCfg.Certificates, cert)
		log.Debugf("loaded X509KeyPair for %s", key)
	}

	if len(tlsCfg.Certificates) == 0 {
		return nil, fmt.Errorf("no tls certificates loaded")
	}

	tlsCfg.BuildNameToCertificate()

	return tlsCfg, nil
}

func startHTTPSServer(wg *sync.WaitGroup) {
	defer wg.Done()

	tlsCfg, err := loadCerts(certDir)
	if err != nil {
		log.Error(err)
		return
	}

	//TODO: configure these timeouts somewhere
	httpsServer := &http.Server{
		Addr:           httpsAddr,
		ReadTimeout:    10 * time.Minute,
		WriteTimeout:   10 * time.Minute,
		MaxHeaderBytes: 1 << 20,
		TLSConfig:      tlsCfg,
	}

	httpRouter = NewHostRouter(httpsServer)
	httpRouter.Scheme = "https"
	httpRouter.SSLOnly = sslOnly

	httpRouter.Start(nil)
}

type ErrorPage struct {
	// The Mutex protects access to the body slice, and headers
	// Everything else should be static once the ErrorPage is created.
	sync.Mutex

	Location    string
	StatusCodes []int

	// body contains the cached error page
	body []byte
	// important headers
	header http.Header
}

func (e *ErrorPage) Body() []byte {
	e.Lock()
	defer e.Unlock()
	return e.body
}

func (e *ErrorPage) SetBody(b []byte) {
	e.Lock()
	defer e.Unlock()
	e.body = b
}

func (e *ErrorPage) Header() http.Header {
	e.Lock()
	defer e.Unlock()
	return e.header
}

func (e *ErrorPage) SetHeader(h http.Header) {
	e.Lock()
	defer e.Unlock()
	e.header = h
}

// List of headers we want to cache for ErrorPages
var ErrorHeaders = []string{
	"Content-Type",
	"Content-Encoding",
	"Cache-Control",
	"Last-Modified",
	"Retry-After",
	"Set-Cookie",
}

// ErrorResponse provides a ReverProxy callback to process a response and
// insert custom error pages for a virtual host.
type ErrorResponse struct {
	sync.Mutex

	// map them by status for responses
	pages map[int]*ErrorPage

	// keep this handy to refresh the pages
	client *http.Client
}

func NewErrorResponse(pages map[string][]int) *ErrorResponse {
	errors := &ErrorResponse{
		pages: make(map[int]*ErrorPage),
	}

	// aggressively timeout connections
	errors.client = &http.Client{
		Transport: &http.Transport{
			Dial: (&net.Dialer{
				Timeout: 2 * time.Second,
			}).Dial,
			TLSHandshakeTimeout: 2 * time.Second,
		},
		Timeout: 5 * time.Second,
	}

	if pages != nil {
		errors.Update(pages)
	}
	return errors
}

// Get the ErrorPage, returning nil if the page was incomplete.
// We permanently cache error pages and headers once we've seen them.
func (e *ErrorResponse) Get(code int) *ErrorPage {
	e.Lock()
	page, ok := e.pages[code]
	e.Unlock()

	if !ok {
		// this is a code we don't handle
		return nil
	}

	body := page.Body()
	if body != nil {
		return page
	}

	// we haven't successfully fetched this error
	e.fetch(page)
	return page
}

func (e *ErrorResponse) fetch(page *ErrorPage) {
	log.Debugf("Fetching error page from %s", page.Location)
	resp, err := e.client.Get(page.Location)
	if err != nil {
		log.Warnf("Could not fetch %s: %s", page.Location, err.Error())
		return
	}
	defer resp.Body.Close()

	// If the StatusCode matches any of our registered codes, it's OK
	for _, code := range page.StatusCodes {
		if resp.StatusCode == code {
			resp.StatusCode = http.StatusOK
			break
		}
	}

	if resp.StatusCode != http.StatusOK {
		log.Warnf("Server returned %d when fetching %s", resp.StatusCode, page.Location)
		return
	}

	header := make(map[string][]string)
	for _, key := range ErrorHeaders {
		if hdr, ok := resp.Header[key]; ok {
			header[key] = hdr
		}
	}
	// set the headers along with the body below

	body, err := ioutil.ReadAll(resp.Body)
	if err != nil {
		log.Warnf("Error reading response from %s: %s", page.Location, err.Error())
		return
	}

	if len(body) > 0 {
		page.SetHeader(header)
		page.SetBody(body)
		return
	}
	log.Warnf("Empty response from %s", page.Location)
}

// This replaces all existing ErrorPages
func (e *ErrorResponse) Update(pages map[string][]int) {
	e.Lock()
	defer e.Unlock()

	e.pages = make(map[int]*ErrorPage)

	for loc, codes := range pages {
		page := &ErrorPage{
			StatusCodes: codes,
			Location:    loc,
		}

		for _, code := range codes {
			e.pages[code] = page
		}
		go e.fetch(page)
	}
}

func (e *ErrorResponse) CheckResponse(pr *ProxyRequest) bool {

	errPage := e.Get(pr.Response.StatusCode)
	if errPage != nil {
		// load the cached headers
		header := pr.ResponseWriter.Header()
		for key, val := range errPage.Header() {
			header[key] = val
		}

		pr.ResponseWriter.WriteHeader(pr.Response.StatusCode)
		pr.ResponseWriter.Write(errPage.Body())
		return false
	}

	return true
}

func logProxyRequest(pr *ProxyRequest) bool {
	// TODO: we may to be able to switch this off
	if pr == nil || pr.Request == nil {
		return true
	}

	var id, method, clientIP, url, backend, agent string
	var status int

	duration := pr.FinishTime.Sub(pr.StartTime)

	id = pr.Request.Header.Get("X-Request-Id")
	method = pr.Request.Method
	url = pr.Request.Host + pr.Request.RequestURI
	agent = pr.Request.UserAgent()
	status = pr.Response.StatusCode

	clientIP = pr.Request.Header.Get("X-Forwarded-For")
	if clientIP == "" {
		clientIP = pr.Request.RemoteAddr
	}

	if pr.Response != nil && pr.Response.Request != nil && pr.Response.Request.URL != nil {
		backend = pr.Response.Request.URL.Host
	}

	err := fmt.Sprintf("%v", pr.ProxyError)

	fmtStr := "id=%s method=%s client-ip=%s url=%s backend=%s status=%d duration=%s agent=%s, err=%s"

	log.Printf(fmtStr, id, method, clientIP, url, backend, status, duration, agent, err)
	return true
}
<|endoftext|>


<commit_before>package main

import (
	"fmt"
	"math/big"
	"os"
	"path/filepath"
	"text/tabwriter"

	"github.com/NebulousLabs/Sia/api"
	"github.com/NebulousLabs/Sia/modules"

	"github.com/spf13/cobra"
)

var (
	hostCmd = &cobra.Command{
		Use:   "host",
		Short: "Perform host actions",
		Long:  "View or modify host settings.",
		Run:   wrap(hostcmd),
	}

	hostConfigCmd = &cobra.Command{
		Use:   "config [setting] [value]",
		Short: "Modify host settings",
		Long: `Modify host settings.
Available settings:

Parameter                        Unit

acceptingcontracts               boolean
collateral                       currency/TB
collateralbudget                 currency
maxcollateral                    currency
maxdownloadbatchsize             int
maxduration                      int
maxrevisebatchsize               int
minimumcontractprice             currency
minimumdownloadbandwidthprice    currency/TB
minimumstorageprice              currency/TB/month
minimumuploadbandwidthprice      currency/TB
netaddress                       string
windowsize                       int

Currency units can be specified, e.g. 10SC; run 'siac help wallet' for details.

For a description of each parameter, see doc/API.md.

To configure the host to accept new contracts, set acceptingcontracts to true:
	siac host config acceptingcontracts true
`,
		Run: wrap(hostconfigcmd),
	}

	hostAnnounceCmd = &cobra.Command{
		Use:   "announce",
		Short: "Announce yourself as a host",
		Long: `Announce yourself as a host on the network.
You may also supply a specific address to be announced, e.g.:
	siac host announce my-host-domain.com:9001
Doing so will override the standard connectivity checks.`,
		Run: hostannouncecmd,
	}

	hostFolderCmd = &cobra.Command{
		Use:   "folder",
		Short: "Add, remove, or resize a storage folder",
		Long:  "Add, remove, or resize a storage folder.",
	}

	hostFolderAddCmd = &cobra.Command{
		Use:   "add [path] [size]",
		Short: "Add a storage folder to the host",
		Long:  "Add a storage folder to the host, specifying how much data it should store",
		Run:   wrap(hostfolderaddcmd),
	}

	hostFolderRemoveCmd = &cobra.Command{
		Use:   "remove [path]",
		Short: "Remove a storage folder from the host",
		Long: `Remove a storage folder from the host. Note that this does not delete any
data; it will instead be distributed across the remaining storage folders.`,

		Run: wrap(hostfolderremovecmd),
	}

	hostFolderResizeCmd = &cobra.Command{
		Use:   "resize [path] [size]",
		Short: "Resize a storage folder",
		Long: `Change how much data a storage folder should store. If the new size is less
than what the folder is currently storing, data will be distributed across the
other storage folders.`,
		Run: wrap(hostfolderresizecmd),
	}

	hostSectorCmd = &cobra.Command{
		Use:   "sector",
		Short: "Add or delete a sector (add not supported)",
		Long: `Add or delete a sector. Adding is not currently supported. Note that
deleting a sector may impact host revenue.`,
	}

	hostSectorDeleteCmd = &cobra.Command{
		Use:   "delete [root]",
		Short: "Delete a sector",
		Long: `Delete a sector, identified by its Merkle root. Note that deleting a
sector may impact host revenue.`,
		Run: wrap(hostsectordeletecmd),
	}
)

// hostcmd is the handler for the command `siac host`.
// Prints info about the host and its storage folders.
func hostcmd() {
	hg := new(api.HostGET)
	err := getAPI("/host", &hg)
	if err != nil {
		die("Could not fetch host settings:", err)
	}
	sg := new(api.StorageGET)
	err = getAPI("/storage", &sg)
	if err != nil {
		die("Could not fetch storage info:", err)
	}

	fm := hg.FinancialMetrics
	is := hg.InternalSettings
	nm := hg.NetworkMetrics

	// calculate total storage available and remaining
	var totalstorage, storageremaining uint64
	for _, folder := range sg.StorageFolderMetadata {
		totalstorage += folder.Capacity
		storageremaining += folder.CapacityRemaining
	}

	// convert accepting bool
	accept := yesNo(is.AcceptingContracts)
	// convert price to SC/TB/mo
	price, err := modules.StoragePriceToHuman(is.MinimumStoragePrice)
	if err != nil {
		price = ^uint64(0)
	}
	totalRevenue := fm.ContractCompensation.
		Add(fm.StorageRevenue).
		Add(fm.DownloadBandwidthRevenue).
		Add(fm.UploadBandwidthRevenue)
	totalPotentialRevenue := fm.PotentialContractCompensation.
		Add(fm.PotentialStorageRevenue).
		Add(fm.PotentialDownloadBandwidthRevenue).
		Add(fm.PotentialUploadBandwidthRevenue)
	fmt.Printf(`Host info:
	Storage:      %v (%v used)
	Price:        %v SC per TB per month
	Max Duration: %v Blocks

	Accepting Contracts: %v
	Anticipated Revenue: %v
	Revenue:             %v
	Lost Revenue:        %v
	Lost Collateral:     %v
`, filesizeUnits(int64(totalstorage)), filesizeUnits(int64(totalstorage-storageremaining)),
		price, is.MaxDuration, accept, currencyUnits(totalPotentialRevenue),
		currencyUnits(totalRevenue), currencyUnits(fm.LostRevenue),
		currencyUnits(fm.LostStorageCollateral))

	// display more info if verbose flag is set
	if hostVerbose {
		fmt.Printf(`
	Net Address: %v

RPC Stats:
	Error Calls:        %v
	Unrecognized Calls: %v
	Download Calls:     %v
	Renew Calls:        %v
	Revise Calls:       %v
	Settings Calls:     %v
	FormContract Calls: %v
`, nm.NetAddress, nm.ErrorCalls, nm.UnrecognizedCalls, nm.DownloadCalls,
			nm.RenewCalls, nm.ReviseCalls, nm.SettingsCalls, nm.FormContractCalls)
	}

	fmt.Println("\nStorage Folders:")

	// display storage folder info
	if len(sg.StorageFolderMetadata) == 0 {
		fmt.Println("No storage folders configured")
		return
	}
	w := tabwriter.NewWriter(os.Stdout, 0, 0, 4, ' ', 0)
	fmt.Fprintf(w, "\tUsed\tCapacity\t%% Used\tPath\n")
	for _, folder := range sg.StorageFolderMetadata {
		curSize := int64(folder.Capacity - folder.CapacityRemaining)
		pctUsed := 100 * (float64(curSize) / float64(folder.Capacity))
		fmt.Fprintf(w, "\t%s\t%s\t%.2f\t%s\n", filesizeUnits(curSize), filesizeUnits(int64(folder.Capacity)), pctUsed, folder.Path)
	}
	w.Flush()
}

// hostconfigcmd is the handler for the command `siac host config [setting] [value]`.
// Modifies host settings.
func hostconfigcmd(param, value string) {
	switch param {
	// currency (convert to hastings)
	case "collateralbudget", "maxcollateral", "minimumcontractprice":
		hastings, err := parseCurrency(value)
		if err != nil {
			die("Could not parse "+param+":", err)
		}
		value = hastings

	// currency/TB (convert to hastings/byte)
	case "collateral", "minimumdownloadbandwidthprice", "minimumuploadbandwidthprice":
		hastings, err := parseCurrency(value)
		if err != nil {
			die("Could not parse "+param+":", err)
		}
		i, _ := new(big.Int).SetString(hastings, 10)
		i.Div(i, big.NewInt(1e12)) // divide by 1e12 bytes/TB
		value = i.String()

	// currency/TB/month (convert to hastings/byte/block)
	case "minimumstorageprice":
		hastings, err := parseCurrency(value)
		if err != nil {
			die("Could not parse "+param+":", err)
		}
		i, _ := new(big.Int).SetString(hastings, 10)
		i.Div(i, big.NewInt(1e12)) // divide by 1e12 bytes/TB
		value = i.String()

	// other valid settings
	case "acceptingcontracts", "maxdownloadbatchsize", "maxduration",
		"maxrevisebatchsize", "netaddress", "windowsize":

	// invalid settings
	default:
		die("\"" + param + "\" is not a host setting")
	}
	err := post("/host", param+"="+value)
	if err != nil {
		die("Could not update host settings:", err)
	}
	fmt.Println("Host settings updated.")
}

// hostannouncecmd is the handler for the command `siac host announce`.
// Announces yourself as a host to the network. Optionally takes an address to
// announce as.
func hostannouncecmd(cmd *cobra.Command, args []string) {
	var err error
	switch len(args) {
	case 0:
		err = post("/host/announce", "")
	case 1:
		err = post("/host/announce", "netaddress="+args[0])
	default:
		cmd.Usage()
		os.Exit(exitCodeUsage)
	}
	if err != nil {
		die("Could not announce host:", err)
	}
	fmt.Println("Host announcement submitted to network.")
}

// hostfolderaddcmd adds a folder to the host.
func hostfolderaddcmd(path, size string) {
	size, err := parseFilesize(size)
	if err != nil {
		die("Could not parse size:", err)
	}
	err = post("/storage/folders/add"+filepath.ToSlash(abs(path)), "size="+size)
	if err != nil {
		die("Could not add folder:", err)
	}
	fmt.Println("Added folder", path)
}

// hostfolderremovecmd removes a folder from the host.
func hostfolderremovecmd(path string) {
	err := post("/storage/folders/remove"+filepath.ToSlash(abs(path)), "")
	if err != nil {
		die("Could not remove folder:", err)
	}
	fmt.Println("Removed folder", path)
}

// hostfolderresizecmd resizes a folder in the host.
func hostfolderresizecmd(path, newsize string) {
	newsize, err := parseFilesize(newsize)
	if err != nil {
		die("Could not parse size:", err)
	}
	err = post("/storage/folders/resize"+filepath.ToSlash(abs(path)), "newsize="+newsize)
	if err != nil {
		die("Could not resize folder:", err)
	}
	fmt.Printf("Resized folder %v to %v\n", path, newsize)
}

// hostsectordeletecmd deletes a sector from the host.
func hostsectordeletecmd(root string) {
	err := post("/storage/sectors/delete/"+root, "")
	if err != nil {
		die("Could not delete sector:", err)
	}
	fmt.Println("Deleted sector", root)
}
<commit_msg>describe net address<commit_after>package main

import (
	"fmt"
	"math/big"
	"os"
	"path/filepath"
	"text/tabwriter"

	"github.com/NebulousLabs/Sia/api"
	"github.com/NebulousLabs/Sia/modules"

	"github.com/spf13/cobra"
)

var (
	hostCmd = &cobra.Command{
		Use:   "host",
		Short: "Perform host actions",
		Long:  "View or modify host settings.",
		Run:   wrap(hostcmd),
	}

	hostConfigCmd = &cobra.Command{
		Use:   "config [setting] [value]",
		Short: "Modify host settings",
		Long: `Modify host settings.
Available settings:

Parameter                        Unit

acceptingcontracts               boolean
collateral                       currency/TB
collateralbudget                 currency
maxcollateral                    currency
maxdownloadbatchsize             int
maxduration                      int
maxrevisebatchsize               int
minimumcontractprice             currency
minimumdownloadbandwidthprice    currency/TB
minimumstorageprice              currency/TB/month
minimumuploadbandwidthprice      currency/TB
netaddress                       string
windowsize                       int

Currency units can be specified, e.g. 10SC; run 'siac help wallet' for details.

For a description of each parameter, see doc/API.md.

To configure the host to accept new contracts, set acceptingcontracts to true:
	siac host config acceptingcontracts true
`,
		Run: wrap(hostconfigcmd),
	}

	hostAnnounceCmd = &cobra.Command{
		Use:   "announce",
		Short: "Announce yourself as a host",
		Long: `Announce yourself as a host on the network.
You may also supply a specific address to be announced, e.g.:
	siac host announce my-host-domain.com:9001
Doing so will override the standard connectivity checks.`,
		Run: hostannouncecmd,
	}

	hostFolderCmd = &cobra.Command{
		Use:   "folder",
		Short: "Add, remove, or resize a storage folder",
		Long:  "Add, remove, or resize a storage folder.",
	}

	hostFolderAddCmd = &cobra.Command{
		Use:   "add [path] [size]",
		Short: "Add a storage folder to the host",
		Long:  "Add a storage folder to the host, specifying how much data it should store",
		Run:   wrap(hostfolderaddcmd),
	}

	hostFolderRemoveCmd = &cobra.Command{
		Use:   "remove [path]",
		Short: "Remove a storage folder from the host",
		Long: `Remove a storage folder from the host. Note that this does not delete any
data; it will instead be distributed across the remaining storage folders.`,

		Run: wrap(hostfolderremovecmd),
	}

	hostFolderResizeCmd = &cobra.Command{
		Use:   "resize [path] [size]",
		Short: "Resize a storage folder",
		Long: `Change how much data a storage folder should store. If the new size is less
than what the folder is currently storing, data will be distributed across the
other storage folders.`,
		Run: wrap(hostfolderresizecmd),
	}

	hostSectorCmd = &cobra.Command{
		Use:   "sector",
		Short: "Add or delete a sector (add not supported)",
		Long: `Add or delete a sector. Adding is not currently supported. Note that
deleting a sector may impact host revenue.`,
	}

	hostSectorDeleteCmd = &cobra.Command{
		Use:   "delete [root]",
		Short: "Delete a sector",
		Long: `Delete a sector, identified by its Merkle root. Note that deleting a
sector may impact host revenue.`,
		Run: wrap(hostsectordeletecmd),
	}
)

// hostcmd is the handler for the command `siac host`.
// Prints info about the host and its storage folders.
func hostcmd() {
	hg := new(api.HostGET)
	err := getAPI("/host", &hg)
	if err != nil {
		die("Could not fetch host settings:", err)
	}
	sg := new(api.StorageGET)
	err = getAPI("/storage", &sg)
	if err != nil {
		die("Could not fetch storage info:", err)
	}

	fm := hg.FinancialMetrics
	is := hg.InternalSettings
	nm := hg.NetworkMetrics

	// calculate total storage available and remaining
	var totalstorage, storageremaining uint64
	for _, folder := range sg.StorageFolderMetadata {
		totalstorage += folder.Capacity
		storageremaining += folder.CapacityRemaining
	}

	// convert accepting bool
	accept := yesNo(is.AcceptingContracts)
	// convert price to SC/TB/mo
	price, err := modules.StoragePriceToHuman(is.MinimumStoragePrice)
	if err != nil {
		price = ^uint64(0)
	}
	// calculate total revenue
	totalRevenue := fm.ContractCompensation.
		Add(fm.StorageRevenue).
		Add(fm.DownloadBandwidthRevenue).
		Add(fm.UploadBandwidthRevenue)
	totalPotentialRevenue := fm.PotentialContractCompensation.
		Add(fm.PotentialStorageRevenue).
		Add(fm.PotentialDownloadBandwidthRevenue).
		Add(fm.PotentialUploadBandwidthRevenue)
	fmt.Printf(`Host info:
	Storage:      %v (%v used)
	Price:        %v SC per TB per month
	Max Duration: %v Blocks

	Accepting Contracts: %v
	Anticipated Revenue: %v
	Revenue:             %v
	Lost Revenue:        %v
	Lost Collateral:     %v
`, filesizeUnits(int64(totalstorage)), filesizeUnits(int64(totalstorage-storageremaining)),
		price, is.MaxDuration, accept, currencyUnits(totalPotentialRevenue),
		currencyUnits(totalRevenue), currencyUnits(fm.LostRevenue),
		currencyUnits(fm.LostStorageCollateral))

	// display more info if verbose flag is set
	if hostVerbose {
		// describe net address
		netaddr := nm.NetAddress
		if is.NetAddress == "" {
			netaddr += " (automatically determined)"
		} else {
			netaddr += " (manually specified)"
		}
		fmt.Printf(`
	Net Address: %v

RPC Stats:
	Error Calls:        %v
	Unrecognized Calls: %v
	Download Calls:     %v
	Renew Calls:        %v
	Revise Calls:       %v
	Settings Calls:     %v
	FormContract Calls: %v
`, netaddr, nm.ErrorCalls, nm.UnrecognizedCalls, nm.DownloadCalls,
			nm.RenewCalls, nm.ReviseCalls, nm.SettingsCalls, nm.FormContractCalls)
	}

	fmt.Println("\nStorage Folders:")

	// display storage folder info
	if len(sg.StorageFolderMetadata) == 0 {
		fmt.Println("No storage folders configured")
		return
	}
	w := tabwriter.NewWriter(os.Stdout, 0, 0, 4, ' ', 0)
	fmt.Fprintf(w, "\tUsed\tCapacity\t%% Used\tPath\n")
	for _, folder := range sg.StorageFolderMetadata {
		curSize := int64(folder.Capacity - folder.CapacityRemaining)
		pctUsed := 100 * (float64(curSize) / float64(folder.Capacity))
		fmt.Fprintf(w, "\t%s\t%s\t%.2f\t%s\n", filesizeUnits(curSize), filesizeUnits(int64(folder.Capacity)), pctUsed, folder.Path)
	}
	w.Flush()
}

// hostconfigcmd is the handler for the command `siac host config [setting] [value]`.
// Modifies host settings.
func hostconfigcmd(param, value string) {
	switch param {
	// currency (convert to hastings)
	case "collateralbudget", "maxcollateral", "minimumcontractprice":
		hastings, err := parseCurrency(value)
		if err != nil {
			die("Could not parse "+param+":", err)
		}
		value = hastings

	// currency/TB (convert to hastings/byte)
	case "collateral", "minimumdownloadbandwidthprice", "minimumuploadbandwidthprice":
		hastings, err := parseCurrency(value)
		if err != nil {
			die("Could not parse "+param+":", err)
		}
		i, _ := new(big.Int).SetString(hastings, 10)
		i.Div(i, big.NewInt(1e12)) // divide by 1e12 bytes/TB
		value = i.String()

	// currency/TB/month (convert to hastings/byte/block)
	case "minimumstorageprice":
		hastings, err := parseCurrency(value)
		if err != nil {
			die("Could not parse "+param+":", err)
		}
		i, _ := new(big.Int).SetString(hastings, 10)
		i.Div(i, big.NewInt(1e12)) // divide by 1e12 bytes/TB
		value = i.String()

	// other valid settings
	case "acceptingcontracts", "maxdownloadbatchsize", "maxduration",
		"maxrevisebatchsize", "netaddress", "windowsize":

	// invalid settings
	default:
		die("\"" + param + "\" is not a host setting")
	}
	err := post("/host", param+"="+value)
	if err != nil {
		die("Could not update host settings:", err)
	}
	fmt.Println("Host settings updated.")
}

// hostannouncecmd is the handler for the command `siac host announce`.
// Announces yourself as a host to the network. Optionally takes an address to
// announce as.
func hostannouncecmd(cmd *cobra.Command, args []string) {
	var err error
	switch len(args) {
	case 0:
		err = post("/host/announce", "")
	case 1:
		err = post("/host/announce", "netaddress="+args[0])
	default:
		cmd.Usage()
		os.Exit(exitCodeUsage)
	}
	if err != nil {
		die("Could not announce host:", err)
	}
	fmt.Println("Host announcement submitted to network.")
}

// hostfolderaddcmd adds a folder to the host.
func hostfolderaddcmd(path, size string) {
	size, err := parseFilesize(size)
	if err != nil {
		die("Could not parse size:", err)
	}
	err = post("/storage/folders/add"+filepath.ToSlash(abs(path)), "size="+size)
	if err != nil {
		die("Could not add folder:", err)
	}
	fmt.Println("Added folder", path)
}

// hostfolderremovecmd removes a folder from the host.
func hostfolderremovecmd(path string) {
	err := post("/storage/folders/remove"+filepath.ToSlash(abs(path)), "")
	if err != nil {
		die("Could not remove folder:", err)
	}
	fmt.Println("Removed folder", path)
}

// hostfolderresizecmd resizes a folder in the host.
func hostfolderresizecmd(path, newsize string) {
	newsize, err := parseFilesize(newsize)
	if err != nil {
		die("Could not parse size:", err)
	}
	err = post("/storage/folders/resize"+filepath.ToSlash(abs(path)), "newsize="+newsize)
	if err != nil {
		die("Could not resize folder:", err)
	}
	fmt.Printf("Resized folder %v to %v\n", path, newsize)
}

// hostsectordeletecmd deletes a sector from the host.
func hostsectordeletecmd(root string) {
	err := post("/storage/sectors/delete/"+root, "")
	if err != nil {
		die("Could not delete sector:", err)
	}
	fmt.Println("Deleted sector", root)
}
<|endoftext|>


<commit_before>package main

import (
	"context"
	"crypto/tls"
	"flag"
	"net"
	"net/http"
	"os"
	"os/signal"
	"strings"
	"syscall"
	"time"

	"github.com/coreos/go-semver/semver"
	"github.com/fnproject/fn/fnlb/lb"
	"github.com/sirupsen/logrus"
)

const VERSION = "0.0.98"

func main() {
	// XXX (reed): normalize
	fnodes := flag.String("nodes", "", "comma separated list of functions nodes")
	minAPIVersion := flag.String("min-api-version", "0.0.65", "minimal node API to accept")

	var conf lb.Config
	flag.StringVar(&conf.DBurl, "db", "sqlite3://:memory:", "backend to store nodes, default to in memory")
	flag.StringVar(&conf.Listen, "listen", ":8081", "port to run on")
	flag.IntVar(&conf.HealthcheckInterval, "hc-interval", 3, "how often to check f(x) nodes, in seconds")
	flag.StringVar(&conf.HealthcheckEndpoint, "hc-path", "/version", "endpoint to determine node health")
	flag.IntVar(&conf.HealthcheckUnhealthy, "hc-unhealthy", 2, "threshold of failed checks to declare node unhealthy")
	flag.IntVar(&conf.HealthcheckTimeout, "hc-timeout", 5, "timeout of healthcheck endpoint, in seconds")
	flag.StringVar(&conf.ZipkinURL, "zipkin", "", "zipkin endpoint to send traces")
	flag.Parse()

	conf.MinAPIVersion = semver.New(*minAPIVersion)

	if len(*fnodes) > 0 {
		// starting w/o nodes is fine too
		conf.Nodes = strings.Split(*fnodes, ",")
	}

	conf.Transport = &http.Transport{
		Proxy: http.ProxyFromEnvironment,
		Dial: (&net.Dialer{
			Timeout:   10 * time.Second,
			KeepAlive: 120 * time.Second,
		}).Dial,
		MaxIdleConnsPerHost: 512,
		TLSHandshakeTimeout: 10 * time.Second,
		TLSClientConfig: &tls.Config{
			ClientSessionCache: tls.NewLRUClientSessionCache(4096),
		},
	}

	g, err := lb.NewAllGrouper(conf)
	if err != nil {
		logrus.WithError(err).Fatal("error setting up grouper")
	}

	r := lb.NewConsistentRouter(conf)
	k := func(r *http.Request) (string, error) {
		return r.URL.Path, nil
	}

	h := lb.NewProxy(k, g, r, conf)
	h = g.Wrap(h) // add/del/list endpoints
	h = r.Wrap(h) // stats / dash endpoint

	err = serve(conf.Listen, h)
	if err != nil {
		logrus.WithError(err).Fatal("server error")
	}
}

func serve(addr string, handler http.Handler) error {
	server := &http.Server{Addr: addr, Handler: handler}

	ch := make(chan os.Signal, 1)
	signal.Notify(ch, syscall.SIGQUIT, syscall.SIGINT)
	go func() {
		for sig := range ch {
			logrus.WithFields(logrus.Fields{"signal": sig}).Info("received signal")
			server.Shutdown(context.Background()) // safe shutdown
			return
		}
	}()
	return server.ListenAndServe()
}
<commit_msg>fnlb: 0.0.99 release [skip ci]<commit_after>package main

import (
	"context"
	"crypto/tls"
	"flag"
	"net"
	"net/http"
	"os"
	"os/signal"
	"strings"
	"syscall"
	"time"

	"github.com/coreos/go-semver/semver"
	"github.com/fnproject/fn/fnlb/lb"
	"github.com/sirupsen/logrus"
)

const VERSION = "0.0.99"

func main() {
	// XXX (reed): normalize
	fnodes := flag.String("nodes", "", "comma separated list of functions nodes")
	minAPIVersion := flag.String("min-api-version", "0.0.66", "minimal node API to accept")

	var conf lb.Config
	flag.StringVar(&conf.DBurl, "db", "sqlite3://:memory:", "backend to store nodes, default to in memory")
	flag.StringVar(&conf.Listen, "listen", ":8081", "port to run on")
	flag.IntVar(&conf.HealthcheckInterval, "hc-interval", 3, "how often to check f(x) nodes, in seconds")
	flag.StringVar(&conf.HealthcheckEndpoint, "hc-path", "/version", "endpoint to determine node health")
	flag.IntVar(&conf.HealthcheckUnhealthy, "hc-unhealthy", 2, "threshold of failed checks to declare node unhealthy")
	flag.IntVar(&conf.HealthcheckTimeout, "hc-timeout", 5, "timeout of healthcheck endpoint, in seconds")
	flag.StringVar(&conf.ZipkinURL, "zipkin", "", "zipkin endpoint to send traces")
	flag.Parse()

	conf.MinAPIVersion = semver.New(*minAPIVersion)

	if len(*fnodes) > 0 {
		// starting w/o nodes is fine too
		conf.Nodes = strings.Split(*fnodes, ",")
	}

	conf.Transport = &http.Transport{
		Proxy: http.ProxyFromEnvironment,
		Dial: (&net.Dialer{
			Timeout:   10 * time.Second,
			KeepAlive: 120 * time.Second,
		}).Dial,
		MaxIdleConnsPerHost: 512,
		TLSHandshakeTimeout: 10 * time.Second,
		TLSClientConfig: &tls.Config{
			ClientSessionCache: tls.NewLRUClientSessionCache(4096),
		},
	}

	g, err := lb.NewAllGrouper(conf)
	if err != nil {
		logrus.WithError(err).Fatal("error setting up grouper")
	}

	r := lb.NewConsistentRouter(conf)
	k := func(r *http.Request) (string, error) {
		return r.URL.Path, nil
	}

	h := lb.NewProxy(k, g, r, conf)
	h = g.Wrap(h) // add/del/list endpoints
	h = r.Wrap(h) // stats / dash endpoint

	err = serve(conf.Listen, h)
	if err != nil {
		logrus.WithError(err).Fatal("server error")
	}
}

func serve(addr string, handler http.Handler) error {
	server := &http.Server{Addr: addr, Handler: handler}

	ch := make(chan os.Signal, 1)
	signal.Notify(ch, syscall.SIGQUIT, syscall.SIGINT)
	go func() {
		for sig := range ch {
			logrus.WithFields(logrus.Fields{"signal": sig}).Info("received signal")
			server.Shutdown(context.Background()) // safe shutdown
			return
		}
	}()
	return server.ListenAndServe()
}
<|endoftext|>


<commit_before>package main

import (
	"context"
	"crypto/tls"
	"flag"
	"net"
	"net/http"
	"os"
	"os/signal"
	"strings"
	"syscall"
	"time"

	"github.com/coreos/go-semver/semver"
	"github.com/fnproject/fn/fnlb/lb"
	"github.com/sirupsen/logrus"
)

const VERSION = "0.0.63"

func main() {
	// XXX (reed): normalize
	fnodes := flag.String("nodes", "", "comma separated list of functions nodes")
	minAPIVersion := flag.String("min-api-version", "0.0.30", "minimal node API to accept")

	var conf lb.Config
	flag.StringVar(&conf.DBurl, "db", "sqlite3://:memory:", "backend to store nodes, default to in memory")
	flag.StringVar(&conf.Listen, "listen", ":8081", "port to run on")
	flag.IntVar(&conf.HealthcheckInterval, "hc-interval", 3, "how often to check f(x) nodes, in seconds")
	flag.StringVar(&conf.HealthcheckEndpoint, "hc-path", "/version", "endpoint to determine node health")
	flag.IntVar(&conf.HealthcheckUnhealthy, "hc-unhealthy", 2, "threshold of failed checks to declare node unhealthy")
	flag.IntVar(&conf.HealthcheckTimeout, "hc-timeout", 5, "timeout of healthcheck endpoint, in seconds")
	flag.StringVar(&conf.ZipkinURL, "zipkin", "", "zipkin endpoint to send traces")
	flag.Parse()

	conf.MinAPIVersion = semver.New(*minAPIVersion)

	if len(*fnodes) > 0 {
		// starting w/o nodes is fine too
		conf.Nodes = strings.Split(*fnodes, ",")
	}

	conf.Transport = &http.Transport{
		Proxy: http.ProxyFromEnvironment,
		Dial: (&net.Dialer{
			Timeout:   10 * time.Second,
			KeepAlive: 120 * time.Second,
		}).Dial,
		MaxIdleConnsPerHost: 512,
		TLSHandshakeTimeout: 10 * time.Second,
		TLSClientConfig: &tls.Config{
			ClientSessionCache: tls.NewLRUClientSessionCache(4096),
		},
	}

	g, err := lb.NewAllGrouper(conf)
	if err != nil {
		logrus.WithError(err).Fatal("error setting up grouper")
	}

	r := lb.NewConsistentRouter(conf)
	k := func(r *http.Request) (string, error) {
		return r.URL.Path, nil
	}

	h := lb.NewProxy(k, g, r, conf)
	h = g.Wrap(h) // add/del/list endpoints
	h = r.Wrap(h) // stats / dash endpoint

	err = serve(conf.Listen, h)
	if err != nil {
		logrus.WithError(err).Fatal("server error")
	}
}

func serve(addr string, handler http.Handler) error {
	server := &http.Server{Addr: addr, Handler: handler}

	ch := make(chan os.Signal, 1)
	signal.Notify(ch, syscall.SIGQUIT, syscall.SIGINT)
	go func() {
		for sig := range ch {
			logrus.WithFields(logrus.Fields{"signal": sig}).Info("received signal")
			server.Shutdown(context.Background()) // safe shutdown
			return
		}
	}()
	return server.ListenAndServe()
}
<commit_msg>fnlb: 0.0.64 release [skip ci]<commit_after>package main

import (
	"context"
	"crypto/tls"
	"flag"
	"net"
	"net/http"
	"os"
	"os/signal"
	"strings"
	"syscall"
	"time"

	"github.com/coreos/go-semver/semver"
	"github.com/fnproject/fn/fnlb/lb"
	"github.com/sirupsen/logrus"
)

const VERSION = "0.0.64"

func main() {
	// XXX (reed): normalize
	fnodes := flag.String("nodes", "", "comma separated list of functions nodes")
	minAPIVersion := flag.String("min-api-version", "0.0.31", "minimal node API to accept")

	var conf lb.Config
	flag.StringVar(&conf.DBurl, "db", "sqlite3://:memory:", "backend to store nodes, default to in memory")
	flag.StringVar(&conf.Listen, "listen", ":8081", "port to run on")
	flag.IntVar(&conf.HealthcheckInterval, "hc-interval", 3, "how often to check f(x) nodes, in seconds")
	flag.StringVar(&conf.HealthcheckEndpoint, "hc-path", "/version", "endpoint to determine node health")
	flag.IntVar(&conf.HealthcheckUnhealthy, "hc-unhealthy", 2, "threshold of failed checks to declare node unhealthy")
	flag.IntVar(&conf.HealthcheckTimeout, "hc-timeout", 5, "timeout of healthcheck endpoint, in seconds")
	flag.StringVar(&conf.ZipkinURL, "zipkin", "", "zipkin endpoint to send traces")
	flag.Parse()

	conf.MinAPIVersion = semver.New(*minAPIVersion)

	if len(*fnodes) > 0 {
		// starting w/o nodes is fine too
		conf.Nodes = strings.Split(*fnodes, ",")
	}

	conf.Transport = &http.Transport{
		Proxy: http.ProxyFromEnvironment,
		Dial: (&net.Dialer{
			Timeout:   10 * time.Second,
			KeepAlive: 120 * time.Second,
		}).Dial,
		MaxIdleConnsPerHost: 512,
		TLSHandshakeTimeout: 10 * time.Second,
		TLSClientConfig: &tls.Config{
			ClientSessionCache: tls.NewLRUClientSessionCache(4096),
		},
	}

	g, err := lb.NewAllGrouper(conf)
	if err != nil {
		logrus.WithError(err).Fatal("error setting up grouper")
	}

	r := lb.NewConsistentRouter(conf)
	k := func(r *http.Request) (string, error) {
		return r.URL.Path, nil
	}

	h := lb.NewProxy(k, g, r, conf)
	h = g.Wrap(h) // add/del/list endpoints
	h = r.Wrap(h) // stats / dash endpoint

	err = serve(conf.Listen, h)
	if err != nil {
		logrus.WithError(err).Fatal("server error")
	}
}

func serve(addr string, handler http.Handler) error {
	server := &http.Server{Addr: addr, Handler: handler}

	ch := make(chan os.Signal, 1)
	signal.Notify(ch, syscall.SIGQUIT, syscall.SIGINT)
	go func() {
		for sig := range ch {
			logrus.WithFields(logrus.Fields{"signal": sig}).Info("received signal")
			server.Shutdown(context.Background()) // safe shutdown
			return
		}
	}()
	return server.ListenAndServe()
}
<|endoftext|>


<commit_before>package bleve

import (
	"os"

	"github.com/blevesearch/bleve"
	"github.com/inconshreveable/log15"
	"github.com/leeola/errors"
	"github.com/leeola/kala"
	kq "github.com/leeola/kala/q"
)

type Config struct {
	// Path is the *directory* containing the bleve index.
	Path string `toml:"path"`

	Log log15.Logger
}

type Bleve struct {
	config Config
	log    log15.Logger
	bleve  bleve.Index
}

func New(c Config) (*Bleve, error) {
	if c.Path == "" {
		return nil, errors.New("missing required Config field: Path")
	}

	if c.Log == nil {
		c.Log = log15.New()
	}

	if err := os.MkdirAll(c.Path, 0755); err != nil {
		return nil, err
	}

	b, err := bleve.Open(c.Path)
	if err == bleve.ErrorIndexMetaMissing {
		mapping := bleve.NewIndexMapping()
		b, err = bleve.New(c.Path, mapping)
	}
	if err != nil {
		return nil, errors.Wrap(err, "failed to open or create bleve normal index")
	}

	return &Bleve{
		config: c,
		log:    c.Log,
		bleve:  b,
	}, nil
}

func (b *Bleve) Index(h, id string, fields []kala.Field) error {
	b.log.Warn("id indexing not yet implemented", "id", id)

	row := map[string]interface{}{}
	for _, f := range fields {
		// TODO(leeola): implement options

		row[f.Field] = f.Value
	}

	return b.bleve.Index(h, &row)
}

func (b *Bleve) Search(kq *kq.Query) ([]string, error) {
	search, err := ConvertQuery(kq)
	if err != nil {
		return nil, errors.Stack(err)
	}

	searchResults, err := b.bleve.Search(search)
	if err != nil {
		return nil, errors.Stack(err)
	}

	hashes := make([]string, len(searchResults.Hits))
	for i, documentMatch := range searchResults.Hits {
		hashes[i] = documentMatch.ID
	}

	return hashes, nil
}
<commit_msg>doc: added FTS note<commit_after>package bleve

import (
	"os"

	"github.com/blevesearch/bleve"
	"github.com/inconshreveable/log15"
	"github.com/leeola/errors"
	"github.com/leeola/kala"
	kq "github.com/leeola/kala/q"
)

type Config struct {
	// Path is the *directory* containing the bleve index.
	Path string `toml:"path"`

	Log log15.Logger
}

// Bleve implements the kala Index interface for Bleve Search.
//
// Supported Options:
//    FullTextSearch:
//      This option is default, so it's _technically_ ignored but it still works
//      for full text searches and should be defined in metadata regardless.
type Bleve struct {
	config Config
	log    log15.Logger
	bleve  bleve.Index
}

func New(c Config) (*Bleve, error) {
	if c.Path == "" {
		return nil, errors.New("missing required Config field: Path")
	}

	if c.Log == nil {
		c.Log = log15.New()
	}

	if err := os.MkdirAll(c.Path, 0755); err != nil {
		return nil, err
	}

	b, err := bleve.Open(c.Path)
	if err == bleve.ErrorIndexMetaMissing {
		mapping := bleve.NewIndexMapping()
		b, err = bleve.New(c.Path, mapping)
	}
	if err != nil {
		return nil, errors.Wrap(err, "failed to open or create bleve normal index")
	}

	return &Bleve{
		config: c,
		log:    c.Log,
		bleve:  b,
	}, nil
}

func (b *Bleve) Index(h, id string, fields []kala.Field) error {
	b.log.Warn("id indexing not yet implemented", "id", id)

	row := map[string]interface{}{}
	for _, f := range fields {
		// TODO(leeola): implement options

		row[f.Field] = f.Value
	}

	return b.bleve.Index(h, &row)
}

func (b *Bleve) Search(kq *kq.Query) ([]string, error) {
	search, err := ConvertQuery(kq)
	if err != nil {
		return nil, errors.Stack(err)
	}

	searchResults, err := b.bleve.Search(search)
	if err != nil {
		return nil, errors.Stack(err)
	}

	hashes := make([]string, len(searchResults.Hits))
	for i, documentMatch := range searchResults.Hits {
		hashes[i] = documentMatch.ID
	}

	return hashes, nil
}
<|endoftext|>


<commit_before>package pool

import (
	"io"
	"net"
	"os"
	"time"
)

func ExamplePool() {
	go func() {
		if l, err := net.Listen("tcp", "localhost:10000"); err == nil {
			p := NewPool(l.Accept)
			for {
				if c, err := p.Get(); err == nil {
					go func() {
						io.Copy(os.Stdout, c)
						c.Close()
					}()
					// Output:
					// hello world
					// hello world
					// hello world
					// hello world
					// hello world
					// hello world
				}
			}
		}
	}()

	p := NewPool(Limiter(func() (net.Conn, error) {
		return net.Dial("tcp", "localhost:10000")
	}, 3))

	for i := 0; i < 6; i++ {
		if c, err := p.Get(); err == nil {
			c.Write([]byte("hello world\n"))
			go func() {
				<-time.After(100 * time.Millisecond)
				c.Close()
			}()
		}
	}

	<-time.After(100 * time.Millisecond)
}
<commit_msg>adding an Example for Recycler<commit_after>package pool

import (
	"io"
	"net"
	"os"
	"time"
)

func ExampleRecycler() {
	go func() {
		if l, err := net.Listen("tcp", "localhost:10001"); err == nil {
			p := NewPool(l.Accept)
			if c, err := p.Get(); err == nil {
				io.Copy(os.Stdout, c)
				c.Close()
			}
			l.Close()
		}
	}()
	// Output:
	// Hello World
	// Hello World
	// Hello World

	r := NewRecycler(func() (net.Conn, error) {
		return net.Dial("tcp", "localhost:10001")
	}, 1)
	p := NewPool(r.Get)

	for i := 0; i < 3; i++ {
		if c, err := p.Get(); err == nil {
			io.WriteString(c, "Hello World\n")
			r.Put(c)
		}
	}

	<-time.After(100 * time.Millisecond)
}

func ExamplePool() {
	go func() {
		if l, err := net.Listen("tcp", "localhost:10000"); err == nil {
			p := NewPool(l.Accept)
			for {
				if c, err := p.Get(); err == nil {
					go func() {
						io.Copy(os.Stdout, c)
						c.Close()
					}()
					// Output:
					// hello world
					// hello world
					// hello world
					// hello world
					// hello world
					// hello world
				}
			}
		}
	}()

	p := NewPool(Limiter(func() (net.Conn, error) {
		return net.Dial("tcp", "localhost:10000")
	}, 3))

	for i := 0; i < 6; i++ {
		if c, err := p.Get(); err == nil {
			c.Write([]byte("hello world\n"))
			go func() {
				<-time.After(100 * time.Millisecond)
				c.Close()
			}()
		}
	}

	<-time.After(100 * time.Millisecond)
}
<|endoftext|>


<commit_before>package gocd

import (
	"context"
	"fmt"
	"io/ioutil"
	"net/http"
	"testing"
)

func TestPipelineTemplateService_ListPipelineTemplates(t *testing.T) {

	setup()
	defer teardown()

	mux.HandleFunc("/api/admin/templates", func(w http.ResponseWriter, r *http.Request) {
		testMethod(t, r, "GET")
		testAuth(t, r, mockAuthorization)
		j, _ := ioutil.ReadFile("test/resources/pipelinetemplates.0.json")
		fmt.Fprint(w, string(j))
	})

	templates, _, err := client.PipelineTemplates.ListPipelineTemplates(context.Background())

	if err != nil {
		t.Error(err)
	}

	testGotStringSlice(t, []TestStringSlice{
		{(*templates)[0].Name, "template0"},
		{(*templates)[0].Pipelines[0], "up42"},
		{string(len((*templates))), "1"},
	})
}

func TestPipelineTemplateService_GetPipelineTemplates(t *testing.T) {
	setup()
	defer teardown()

	mux.HandleFunc("/api/admin/templates/template1", func(w http.ResponseWriter, r *http.Request) {
		testMethod(t, r, "GET")
		testAuth(t, r, mockAuthorization)
		j, _ := ioutil.ReadFile("test/resources/pipelinetemplates.1.json")
		fmt.Fprint(w, string(j))
	})

	template, _, err := client.PipelineTemplates.GetPipelineTemplate(
		context.Background(),
		"template1",
	)

	if err != nil {
		t.Error(err)
	}

	testGotStringSlice(t, []TestStringSlice{
		{template.Name, "template1"},
		{string(len(template.Stages)), "1"},
	})
}
<commit_msg>Autocommit for 'gofmt -w'.<commit_after>package gocd

import (
	"context"
	"fmt"
	"io/ioutil"
	"net/http"
	"testing"
)

func TestPipelineTemplateService_ListPipelineTemplates(t *testing.T) {

	setup()
	defer teardown()

	mux.HandleFunc("/api/admin/templates", func(w http.ResponseWriter, r *http.Request) {
		testMethod(t, r, "GET")
		testAuth(t, r, mockAuthorization)
		j, _ := ioutil.ReadFile("test/resources/pipelinetemplates.0.json")
		fmt.Fprint(w, string(j))
	})

	templates, _, err := client.PipelineTemplates.ListPipelineTemplates(context.Background())

	if err != nil {
		t.Error(err)
	}

	testGotStringSlice(t, []TestStringSlice{
		{(*templates)[0].Name, "template0"},
		{(*templates)[0].Pipelines[0], "up42"},
		{string(len((*templates))), "1"},
	})
}

func TestPipelineTemplateService_GetPipelineTemplates(t *testing.T) {
	setup()
	defer teardown()

	mux.HandleFunc("/api/admin/templates/template1", func(w http.ResponseWriter, r *http.Request) {
		testMethod(t, r, "GET")
		testAuth(t, r, mockAuthorization)
		j, _ := ioutil.ReadFile("test/resources/pipelinetemplates.1.json")
		fmt.Fprint(w, string(j))
	})

	template, _, err := client.PipelineTemplates.GetPipelineTemplate(
		context.Background(),
		"template1",
	)

	if err != nil {
		t.Error(err)
	}

	testGotStringSlice(t, []TestStringSlice{
		{template.Name, "template1"},
		{string(len(template.Stages)), "1"},
	})
}
<|endoftext|>


<commit_before>package network

import (
	"io"
	"fmt"
	"errors"
	"github.com/yaricom/goNEAT/neat"
)

// A NODE is either a NEURON or a SENSOR.
//   - If it's a sensor, it can be loaded with a value for output
//   - If it's a neuron, it has a list of its incoming input signals ([]*Link is used)
// Use an activation count to avoid flushing
type NNode struct {
	// The ID of the node
	Id               int

	// If true the node is active
	IsActive         bool

	// The the type of the node (NEURON or SENSOR)
	NType            int
	// The type of node activation function (SIGMOID, ...)
	FType            int
	// The placement of the node in the network layers (INPUT, HIDDEN, OUTPUT)
	GenNodeLabel     int

	// The activation for current step
	ActiveOut        float64
	// The activation from PREVIOUS (time-delayed) time step, if there is one
	ActiveOutTd      float64
	// The node's activation value
	Activation       float64
	// The number of activations for current node
	ActivationsCount int32
	// The activation sum
	ActivationSum    float64

	// The list of all incoming connections
	Incoming         []*Link
	// The list of all outgoing connections
	Outgoing         []*Link
	// The trait linked to the node
	Trait            *neat.Trait

	// Used for Gene decoding
	Analogue         *NNode
	// Used for Genome duplication
	Duplicate        *NNode

	/* ************ LEARNING PARAMETERS *********** */
	// The following parameters are for use in neurons that learn through habituation,
	// sensitization, or Hebbian-type processes  */
	Params           []float64

	// Activation value of node at time t-1; Holds the previous step's activation for recurrency
	lastActivation   float64
	// Activation value of node at time t-2 Holds the activation before  the previous step's
	// This is necessary for a special recurrent case when the innode of a recurrent link is one time step ahead of the outnode.
	// The innode then needs to send from TWO time steps ago
	lastActivation2  float64
}

// Creates new node with specified type (NEURON or SENSOR) and ID
func NewNNode(ntype, nodeid int) *NNode {
	n := newNode()
	n.NType = ntype
	n.Id = nodeid
	return n
}

// Creates new node with specified type (NEURON or SENSOR), ID and in the specified
// layer (INPUT, HIDDEN, OUTPUT)
func NewNNodeInPlace(ntype, nodeid, placement int) *NNode {
	n := newNode()
	n.NType = ntype
	n.Id = nodeid
	n.GenNodeLabel = placement
	return n
}

// Construct a NNode off another NNode with given trait for genome purposes
func NewNNodeCopy(n *NNode, t *neat.Trait) *NNode {
	node := newNode()
	node.NType = n.NType
	node.Id = n.Id
	node.GenNodeLabel = n.GenNodeLabel
	node.Trait = t
	node.deriveTrait(t)
	return node
}

// Read a NNode from specified Reader and applies corresponding trait to it from a list of traits provided
func ReadNNode(r io.Reader, traits []*neat.Trait) *NNode {
	n := newNode()
	var trait_id int
	fmt.Fscanf(r, "%d %d %d %d ", &n.Id, &trait_id, &n.NType, &n.GenNodeLabel)
	if trait_id != 0 && traits != nil {
		// find corresponding node trait from list
		for _, t := range traits {
			if trait_id == t.Id {
				n.Trait = t
				n.deriveTrait(t)
				break
			}
		}
	} else {
		// just create empty params
		n.deriveTrait(nil)
	}
	return n
}

// The private default constructor
func newNode() *NNode {
	return &NNode{
		FType:SIGMOID,
		Incoming:make([]*Link, 0),
		Outgoing:make([]*Link, 0),
		GenNodeLabel:HIDDEN,
	}
}

// Copy trait parameters into this node's parameters
func (n *NNode) deriveTrait(t *neat.Trait) {
	n.Params = make([]float64, neat.Num_trait_params)
	if t != nil {
		for i, p := range t.Params {
			n.Params[i] = p
		}
	}
}

// Saves current node's activations for potential time delayed connections
func (n *NNode) saveActivations() {
	n.lastActivation2 = n.lastActivation
	n.lastActivation = n.Activation
}

// Returns activation for a current step
func (n *NNode) GetActiveOut() float64 {
	if n.ActivationsCount > 0 {
		return n.Activation
	} else {
		return 0.0
	}
}

// Returns activation from PREVIOUS time step
func (n *NNode) GetActiveOutTd() float64 {
	if n.ActivationsCount > 1 {
		return n.lastActivation
	} else {
		return 0.0
	}
}

// Returns true if this node is SENSOR
func (n *NNode) IsSensor() bool {
	return n.NType == SENSOR
}

// returns true if this node is NEURON
func (n *NNode) IsNeuron() bool {
	return n.NType == NEURON
}

// If the node is a SENSOR, returns TRUE and loads the value
func (n *NNode) SensorLoad(load float64) bool {
	if n.IsSensor() {
		// Keep a memory of activations for potential time delayed connections
		n.saveActivations()
		// Puts sensor into next time-step
		n.ActivationsCount++
		n.Activation = load
		return true
	} else {
		return false
	}
}

// Adds a NONRECURRENT Link to an incoming NNode in the incoming List
func (n *NNode) AddIncoming(in *NNode, weight float64) {
	newLink := NewLink(weight, in, n, false)
	n.Incoming = append(n.Incoming, newLink)
}

// Adds a Link to a new NNode in the incoming List
func (n *NNode) AddIncomingRecurrent(in *NNode, weight float64, recur bool) {
	newLink := NewLink(weight, in, n, recur)
	n.Incoming = append(n.Incoming, newLink)
}

// Recursively deactivate backwards through the network
func (n *NNode) Flushback() {
	n.ActivationsCount = 0
	n.Activation = 0
	n.lastActivation = 0
	n.lastActivation2 = 0
}

// Verify flushing for debuginh
func (n *NNode) FlushbackCheck() error {
	if n.ActivationsCount > 0 {
		return errors.New(fmt.Sprintf("ALERT: %s has activation count %d", n, n.ActivationsCount))
	}
	if n.Activation > 0 {
		return errors.New(fmt.Sprintf("ALERT: %s has activation %f", n, n.Activation))
	}
	if n.lastActivation > 0 {
		return errors.New(fmt.Sprintf("ALERT: %s has last_activation %f", n, n.lastActivation))
	}
	if n.lastActivation2 > 0 {
		return errors.New(fmt.Sprintf("ALERT: %s has last_activation2 %f", n, n.lastActivation2))
	}
	return nil
}

// Dump node to a writer
func (n *NNode) WriteNode(w io.Writer) {
	trait_id := 0
	if n.Trait != nil {
		trait_id = n.Trait.Id
	}
	fmt.Fprintf(w, "%d %d %d %d", n.Id, trait_id, n.NType, n.GenNodeLabel)
}

// Find the greatest depth starting from this neuron at depth d
func (n *NNode) Depth(d int32) (int32, error) {
	if d > 100 {
		return 10, errors.New("** DEPTH NOT DETERMINED FOR NETWORK WITH LOOP");
	}
	// Base Case
	if n.IsSensor() {
		return d, nil
	} else {
		// recursion
		max := d // The max depth
		for _, l := range n.Incoming {
			cur_depth, err := l.InNode.Depth(d + 1)
			if err != nil {
				return cur_depth, err
			}
			if cur_depth > max {
				max = cur_depth
			}
		}
		return max, nil
	}

}

func (n *NNode) String() string {
	if n.IsSensor() {
		return fmt.Sprintf("(S %3d, type: %d, step %d = %.3f %.3f)",
			n.Id, n.NType, n.ActivationsCount, n.Activation, n.Params)
	} else {
		return fmt.Sprintf("(N %3d, type: %d, step %d = %.3f %.3f)",
			n.Id, n.NType, n.ActivationsCount, n.Activation, n.Params)
	}
}

<commit_msg>Fixed comments<commit_after>package network

import (
	"io"
	"fmt"
	"errors"
	"github.com/yaricom/goNEAT/neat"
)

// A NODE is either a NEURON or a SENSOR.
//   - If it's a sensor, it can be loaded with a value for output
//   - If it's a neuron, it has a list of its incoming input signals ([]*Link is used)
// Use an activation count to avoid flushing
type NNode struct {
	// The ID of the node
	Id               int

	// If true the node is active
	IsActive         bool

	// The the type of the node (NEURON or SENSOR)
	NType            int
	// The type of node activation function (SIGMOID, ...)
	FType            int
	// The placement of the node in the network layers (HIDDEN, INPUT, OUTPUT, BIAS)
	GenNodeLabel     int

	// The activation for current step
	ActiveOut        float64
	// The activation from PREVIOUS (time-delayed) time step, if there is one
	ActiveOutTd      float64
	// The node's activation value
	Activation       float64
	// The number of activations for current node
	ActivationsCount int32
	// The activation sum
	ActivationSum    float64

	// The list of all incoming connections
	Incoming         []*Link
	// The list of all outgoing connections
	Outgoing         []*Link
	// The trait linked to the node
	Trait            *neat.Trait

	// Used for Gene decoding
	Analogue         *NNode
	// Used for Genome duplication
	Duplicate        *NNode

	/* ************ LEARNING PARAMETERS *********** */
	// The following parameters are for use in neurons that learn through habituation,
	// sensitization, or Hebbian-type processes  */
	Params           []float64

	// Activation value of node at time t-1; Holds the previous step's activation for recurrency
	lastActivation   float64
	// Activation value of node at time t-2 Holds the activation before  the previous step's
	// This is necessary for a special recurrent case when the innode of a recurrent link is one time step ahead of the outnode.
	// The innode then needs to send from TWO time steps ago
	lastActivation2  float64
}

// Creates new node with specified type (NEURON or SENSOR) and ID
func NewNNode(ntype, nodeid int) *NNode {
	n := newNode()
	n.NType = ntype
	n.Id = nodeid
	return n
}

// Creates new node with specified type (NEURON or SENSOR), ID and in the specified
// layer (INPUT, HIDDEN, OUTPUT)
func NewNNodeInPlace(ntype, nodeid, placement int) *NNode {
	n := newNode()
	n.NType = ntype
	n.Id = nodeid
	n.GenNodeLabel = placement
	return n
}

// Construct a NNode off another NNode with given trait for genome purposes
func NewNNodeCopy(n *NNode, t *neat.Trait) *NNode {
	node := newNode()
	node.NType = n.NType
	node.Id = n.Id
	node.GenNodeLabel = n.GenNodeLabel
	node.Trait = t
	node.deriveTrait(t)
	return node
}

// Read a NNode from specified Reader and applies corresponding trait to it from a list of traits provided
func ReadNNode(r io.Reader, traits []*neat.Trait) *NNode {
	n := newNode()
	var trait_id int
	fmt.Fscanf(r, "%d %d %d %d ", &n.Id, &trait_id, &n.NType, &n.GenNodeLabel)
	if trait_id != 0 && traits != nil {
		// find corresponding node trait from list
		for _, t := range traits {
			if trait_id == t.Id {
				n.Trait = t
				n.deriveTrait(t)
				break
			}
		}
	} else {
		// just create empty params
		n.deriveTrait(nil)
	}
	return n
}

// The private default constructor
func newNode() *NNode {
	return &NNode{
		FType:SIGMOID,
		Incoming:make([]*Link, 0),
		Outgoing:make([]*Link, 0),
		GenNodeLabel:HIDDEN,
	}
}

// Copy trait parameters into this node's parameters
func (n *NNode) deriveTrait(t *neat.Trait) {
	n.Params = make([]float64, neat.Num_trait_params)
	if t != nil {
		for i, p := range t.Params {
			n.Params[i] = p
		}
	}
}

// Saves current node's activations for potential time delayed connections
func (n *NNode) saveActivations() {
	n.lastActivation2 = n.lastActivation
	n.lastActivation = n.Activation
}

// Returns activation for a current step
func (n *NNode) GetActiveOut() float64 {
	if n.ActivationsCount > 0 {
		return n.Activation
	} else {
		return 0.0
	}
}

// Returns activation from PREVIOUS time step
func (n *NNode) GetActiveOutTd() float64 {
	if n.ActivationsCount > 1 {
		return n.lastActivation
	} else {
		return 0.0
	}
}

// Returns true if this node is SENSOR
func (n *NNode) IsSensor() bool {
	return n.NType == SENSOR
}

// returns true if this node is NEURON
func (n *NNode) IsNeuron() bool {
	return n.NType == NEURON
}

// If the node is a SENSOR, returns TRUE and loads the value
func (n *NNode) SensorLoad(load float64) bool {
	if n.IsSensor() {
		// Keep a memory of activations for potential time delayed connections
		n.saveActivations()
		// Puts sensor into next time-step
		n.ActivationsCount++
		n.Activation = load
		return true
	} else {
		return false
	}
}

// Adds a NONRECURRENT Link to an incoming NNode in the incoming List
func (n *NNode) AddIncoming(in *NNode, weight float64) {
	newLink := NewLink(weight, in, n, false)
	n.Incoming = append(n.Incoming, newLink)
}

// Adds a Link to a new NNode in the incoming List
func (n *NNode) AddIncomingRecurrent(in *NNode, weight float64, recur bool) {
	newLink := NewLink(weight, in, n, recur)
	n.Incoming = append(n.Incoming, newLink)
}

// Recursively deactivate backwards through the network
func (n *NNode) Flushback() {
	n.ActivationsCount = 0
	n.Activation = 0
	n.lastActivation = 0
	n.lastActivation2 = 0
}

// Verify flushing for debuginh
func (n *NNode) FlushbackCheck() error {
	if n.ActivationsCount > 0 {
		return errors.New(fmt.Sprintf("ALERT: %s has activation count %d", n, n.ActivationsCount))
	}
	if n.Activation > 0 {
		return errors.New(fmt.Sprintf("ALERT: %s has activation %f", n, n.Activation))
	}
	if n.lastActivation > 0 {
		return errors.New(fmt.Sprintf("ALERT: %s has last_activation %f", n, n.lastActivation))
	}
	if n.lastActivation2 > 0 {
		return errors.New(fmt.Sprintf("ALERT: %s has last_activation2 %f", n, n.lastActivation2))
	}
	return nil
}

// Dump node to a writer
func (n *NNode) WriteNode(w io.Writer) {
	trait_id := 0
	if n.Trait != nil {
		trait_id = n.Trait.Id
	}
	fmt.Fprintf(w, "%d %d %d %d", n.Id, trait_id, n.NType, n.GenNodeLabel)
}

// Find the greatest depth starting from this neuron at depth d
func (n *NNode) Depth(d int32) (int32, error) {
	if d > 100 {
		return 10, errors.New("** DEPTH NOT DETERMINED FOR NETWORK WITH LOOP");
	}
	// Base Case
	if n.IsSensor() {
		return d, nil
	} else {
		// recursion
		max := d // The max depth
		for _, l := range n.Incoming {
			cur_depth, err := l.InNode.Depth(d + 1)
			if err != nil {
				return cur_depth, err
			}
			if cur_depth > max {
				max = cur_depth
			}
		}
		return max, nil
	}

}

func (n *NNode) String() string {
	if n.IsSensor() {
		return fmt.Sprintf("(S %3d, type: %d, step %d = %.3f %.3f)",
			n.Id, n.NType, n.ActivationsCount, n.Activation, n.Params)
	} else {
		return fmt.Sprintf("(N %3d, type: %d, step %d = %.3f %.3f)",
			n.Id, n.NType, n.ActivationsCount, n.Activation, n.Params)
	}
}

<|endoftext|>


<commit_before>package replacer

import (
	"bytes"
	"strings"

	"github.com/json-iterator/go"
	"github.com/valyala/bytebufferpool"
	"github.com/valyala/fasthttp"
	"github.com/xtrafrancyz/vk-proxy/shared"
)

var json = jsoniter.ConfigFastest

type domainConfig struct {
	apiReplaces                []replace
	apiOfficialLongpollReplace replace
	apiVkmeLongpollReplace     replace
	apiLongpollReplace         replace

	siteHlsReplace replace

	headLocationReplace replace

	vkuiLangsHtml replace
	vkuiLangsJs   replace
	vkuiApiJs     replace
}

type Replacer struct {
	ProxyBaseDomain   string
	ProxyStaticDomain string

	config *domainConfig
}

type ReplaceContext struct {
	Method []byte
	Domain string
	Path   string

	FilterFeed bool
}

func (r *Replacer) getDomainConfig() *domainConfig {
	if r.config == nil {
		cfg := &domainConfig{}
		var replacementStart = []byte(`\/\/` + r.ProxyBaseDomain + `\/_\/`)
		var jsonUrlPrefix = []byte(`"https:`)
		cfg = &domainConfig{}
		cfg.apiReplaces = []replace{
			newStringReplace(`"https:\/\/vk.com\/video_hls.php`, `"https:\/\/`+r.ProxyBaseDomain+`\/@vk.com\/video_hls.php`),
			newRegexFuncReplace(`\\/\\/[-_a-zA-Z0-9]{1,15}\.(?:userapi\.com|vk-cdn\.net|vk\.(?:me|com)|vkuser(?:live|video|audio)\.(?:net|com))\\/`,
				func(src, dst []byte, start, end int) []byte {
					// check if url has valid prefix (like in regexp backreference)
					if start < 7 || !bytes.Equal(src[start-7:start], jsonUrlPrefix) {
						goto cancel
					}
					// do not proxy m.vk.com domain (bugged articles)
					if bytes.Equal(src[start+4:end-2], shared.DomainVkMobile) {
						goto cancel
					}
					dst = append(dst, replacementStart...)
					dst = append(dst, src[start+4:end]...)
					return dst
				cancel:
					return append(dst, src[start:end]...)
				}),
			newRegexReplace(`"https:\\/\\/vk\.com\\/((?:\\/)?images\\/|sticker(:?\\/|s_)|doc-?[0-9]+_)`, `"https:\/\/`+r.ProxyBaseDomain+`\/_\/vk.com\/$1`),
		}
		cfg.apiOfficialLongpollReplace = newStringReplace(`"server":"api.vk.com\/newuim`, `"server":"`+r.ProxyBaseDomain+`\/_\/api.vk.com\/newuim`)
		cfg.apiVkmeLongpollReplace = newStringReplace(`"server":"api.vk.me\/uim`, `"server":"`+r.ProxyBaseDomain+`\/_\/api.vk.me\/uim`)
		cfg.apiLongpollReplace = newStringReplace(`"server":"`, `"server":"`+r.ProxyBaseDomain+`\/_\/`)

		cfg.siteHlsReplace = newRegexReplace(`https:\/\/([-_a-zA-Z0-9]+\.(?:userapi\.com|vk-cdn\.net|vk\.me|vkuser(?:live|video)\.(?:net|com)))\/`, `https://`+r.ProxyBaseDomain+`/_/$1/`)

		cfg.headLocationReplace = newRegexReplace(`https?://([^/]+)(.*)`, `https://`+r.ProxyBaseDomain+`/@$1$2`)

		cfg.vkuiLangsHtml = newRegexReplace(` src="https://(?:vk.com|'[^']+')/js/vkui_lang`, ` src="https://`+r.ProxyBaseDomain+`/_/vk.com/js/vkui_lang`)
		cfg.vkuiLangsJs = newStringReplace(`langpackEntry:"https://vk.com"`, `langpackEntry:"https://`+r.ProxyBaseDomain+`/_/vk.com"`)
		cfg.vkuiApiJs = newStringReplace(`api.vk.com`, r.ProxyBaseDomain)
		r.config = cfg
	}
	return r.config
}

func (r *Replacer) DoReplaceRequest(req *fasthttp.Request, ctx ReplaceContext) {
	if bytes.Equal(ctx.Method, shared.MethodOptions) {
		// Заменяем заголовок Origin для CORS на заспросах со страниц VKUI
		// api.vk.com принимает только "https://static.vk.com" в заголовке Origin
		// Плюс к этому, если послать некорректный Referer, вк тоже пошлет нас куда подальше
		if origin := req.Header.Peek("Origin"); origin != nil {
			origins := string(origin)
			if strings.HasSuffix(origins, r.ProxyStaticDomain) {
				req.Header.Set("Origin", "https://static.vk.com")
				if referer := req.Header.Peek("Referer"); referer != nil {
					req.Header.Set("Referer", strings.Replace(string(referer), r.ProxyStaticDomain, "static.vk.com", 1))
				}
			}
		}
	}

	if ctx.Domain == "oauth.vk.com" {
		// Для авторизации страницы VKUI используют не уже готовый токен авторизации, а получают его при каждом
		// открытии страницы. В запросе авторизации передается текущий урл страницы VKUI, а так как она проксируется,
		// то она отличается от оригинальной. ВК проверяет этот урл страницы и отвергает авторизацию если он не
		// совпадает.
		// Зачем такие костыли - никто не знает, но нужно их обходить.
		// Если в запросе авторизации используется проксируемый статик домен - заменяем его на оригинальный.
		if strings.HasPrefix(ctx.Path, "/authorize?") {
			uri := fasthttp.AcquireURI()
			uri.Update(ctx.Path)
			args := uri.QueryArgs()
			sourceUrl := args.Peek("source_url")
			if sourceUrl != nil {
				sourceUrls := string(sourceUrl)
				modified := strings.Replace(sourceUrls, r.ProxyStaticDomain, "static.vk.com", 1)
				if modified != sourceUrls {
					args.Set("source_url", modified)
					req.SetRequestURIBytes(uri.RequestURI())
					req.SetHost("oauth.vk.com")
				}
			}
		}
	}
}

func (r *Replacer) DoReplaceResponse(res *fasthttp.Response, body *bytebufferpool.ByteBuffer, ctx ReplaceContext) *bytebufferpool.ByteBuffer {
	config := r.getDomainConfig()

	if bytes.Equal(ctx.Method, shared.MethodOptions) {
		// Если в ответ на запрос OPTIONS с заданным Origin придет какой-то кривой ответ, то запросы не будут проходить
		if corsOrigin := res.Header.Peek("Access-Control-Allow-Origin"); corsOrigin != nil {
			res.Header.Set("Access-Control-Allow-Origin", "https://"+r.ProxyStaticDomain)
		}
		return body
	}

	if ctx.Domain == "api.vk.com" {
		for _, replace := range config.apiReplaces {
			body = replace.apply(body)
		}

		// Replace longpoll server
		if ctx.Path == "/method/messages.getLongPollServer" {
			body = config.apiLongpollReplace.apply(body)
		} else

		// Replace longpoll server for official app
		if ctx.Path == "/method/execute" ||
			ctx.Path == "/method/execute.imGetLongPollHistoryExtended" ||
			ctx.Path == "/method/execute.imLpInit" {
			body = config.apiOfficialLongpollReplace.apply(body)
			body = config.apiVkmeLongpollReplace.apply(body)
		}

		if ctx.FilterFeed {
			if ctx.Path == "/method/execute.getNewsfeedSmart" ||
				ctx.Path == "/method/newsfeed.get" {
				var parsed map[string]interface{}
				if err := json.Unmarshal(body.B, &parsed); err == nil {
					if parsed["response"] != nil {
						response := parsed["response"].(map[string]interface{})
						modified := tryRemoveAds(response)
						modified = tryInsertPost(response) || modified
						if modified {
							body.B, err = json.Marshal(parsed)
						}
					}
				}
			}
		}

	} else if ctx.Domain == "vk.com" {
		if ctx.Path == "/video_hls.php" {
			body = config.siteHlsReplace.apply(body)
		}

	} else if ctx.Domain == "static.vk.com" {
		if location := res.Header.Peek("Location"); location != nil {
			// Абсолютный редирект на статик меняем на относительный
			locstr := string(location)
			if idx := strings.Index(locstr, "static.vk.com"); idx != -1 {
				relativePath := ctx.Path
				if idx0 := strings.LastIndexByte(relativePath, '/'); idx0 != -1 {
					relativePath = relativePath[:idx0+1]
				}
				relativeRedirectPath := locstr[idx+13 /*static.vk.com*/:]
				relativeRedirectPath = relativeRedirectPath[longestCommonPrefix(relativeRedirectPath, relativePath):]
				res.Header.Set("Location", relativeRedirectPath)
			} else {
				buf := bytebufferpool.Get()
				buf.Set(location)
				buf = config.headLocationReplace.apply(buf)
				res.Header.SetBytesV("Location", buf.Bytes())
			}
		}

		if strings.HasSuffix(ctx.Path, ".js") {
			body = config.vkuiApiJs.apply(body)
			if strings.HasPrefix(ctx.Path, "/community_manage") {
				body = config.vkuiLangsJs.apply(body)
			}
		} else {
			contentType := string(res.Header.ContentType())
			if strings.HasPrefix(contentType, "text/html") {
				body = config.vkuiLangsHtml.apply(body)
			}
		}
	}

	return body
}

func tryRemoveAds(response map[string]interface{}) bool {
	raw, ok := response["items"]
	if !ok {
		return false
	}
	items := raw.([]interface{})
	removed := 0
	for i := len(items) - 1; i >= 0; i-- {
		post := items[i].(map[string]interface{})
		if post["type"] == "ads" || (post["type"] == "post" && post["marked_as_ads"] != nil && post["marked_as_ads"].(float64) == 1) {
			items[i] = items[len(items)-1]
			items[len(items)-1] = nil
			items = items[:len(items)-1]
			removed++
		}
	}
	if removed > 0 {
		newItems := make([]interface{}, len(items))
		copy(newItems, items)
		response["items"] = newItems
		return true
	}
	return false
}

func AcquireBuffer() *bytebufferpool.ByteBuffer {
	return replaceBufferPool.Get()
}

func ReleaseBuffer(buffer *bytebufferpool.ByteBuffer) {
	replaceBufferPool.Put(buffer)
}

func longestCommonPrefix(a, b string) (i int) {
	for ; i < len(a) && i < len(b); i++ {
		if a[i] != b[i] {
			break
		}
	}
	return
}
<commit_msg>Исправлена утечка<commit_after>package replacer

import (
	"bytes"
	"strings"

	"github.com/json-iterator/go"
	"github.com/valyala/bytebufferpool"
	"github.com/valyala/fasthttp"
	"github.com/xtrafrancyz/vk-proxy/shared"
)

var json = jsoniter.ConfigFastest

type domainConfig struct {
	apiReplaces                []replace
	apiOfficialLongpollReplace replace
	apiVkmeLongpollReplace     replace
	apiLongpollReplace         replace

	siteHlsReplace replace

	headLocationReplace replace

	vkuiLangsHtml replace
	vkuiLangsJs   replace
	vkuiApiJs     replace
}

type Replacer struct {
	ProxyBaseDomain   string
	ProxyStaticDomain string

	config *domainConfig
}

type ReplaceContext struct {
	Method []byte
	Domain string
	Path   string

	FilterFeed bool
}

func (r *Replacer) getDomainConfig() *domainConfig {
	if r.config == nil {
		cfg := &domainConfig{}
		var replacementStart = []byte(`\/\/` + r.ProxyBaseDomain + `\/_\/`)
		var jsonUrlPrefix = []byte(`"https:`)
		cfg = &domainConfig{}
		cfg.apiReplaces = []replace{
			newStringReplace(`"https:\/\/vk.com\/video_hls.php`, `"https:\/\/`+r.ProxyBaseDomain+`\/@vk.com\/video_hls.php`),
			newRegexFuncReplace(`\\/\\/[-_a-zA-Z0-9]{1,15}\.(?:userapi\.com|vk-cdn\.net|vk\.(?:me|com)|vkuser(?:live|video|audio)\.(?:net|com))\\/`,
				func(src, dst []byte, start, end int) []byte {
					// check if url has valid prefix (like in regexp backreference)
					if start < 7 || !bytes.Equal(src[start-7:start], jsonUrlPrefix) {
						goto cancel
					}
					// do not proxy m.vk.com domain (bugged articles)
					if bytes.Equal(src[start+4:end-2], shared.DomainVkMobile) {
						goto cancel
					}
					dst = append(dst, replacementStart...)
					dst = append(dst, src[start+4:end]...)
					return dst
				cancel:
					return append(dst, src[start:end]...)
				}),
			newRegexReplace(`"https:\\/\\/vk\.com\\/((?:\\/)?images\\/|sticker(:?\\/|s_)|doc-?[0-9]+_)`, `"https:\/\/`+r.ProxyBaseDomain+`\/_\/vk.com\/$1`),
		}
		cfg.apiOfficialLongpollReplace = newStringReplace(`"server":"api.vk.com\/newuim`, `"server":"`+r.ProxyBaseDomain+`\/_\/api.vk.com\/newuim`)
		cfg.apiVkmeLongpollReplace = newStringReplace(`"server":"api.vk.me\/uim`, `"server":"`+r.ProxyBaseDomain+`\/_\/api.vk.me\/uim`)
		cfg.apiLongpollReplace = newStringReplace(`"server":"`, `"server":"`+r.ProxyBaseDomain+`\/_\/`)

		cfg.siteHlsReplace = newRegexReplace(`https:\/\/([-_a-zA-Z0-9]+\.(?:userapi\.com|vk-cdn\.net|vk\.me|vkuser(?:live|video)\.(?:net|com)))\/`, `https://`+r.ProxyBaseDomain+`/_/$1/`)

		cfg.headLocationReplace = newRegexReplace(`https?://([^/]+)(.*)`, `https://`+r.ProxyBaseDomain+`/@$1$2`)

		cfg.vkuiLangsHtml = newRegexReplace(` src="https://(?:vk.com|'[^']+')/js/vkui_lang`, ` src="https://`+r.ProxyBaseDomain+`/_/vk.com/js/vkui_lang`)
		cfg.vkuiLangsJs = newStringReplace(`langpackEntry:"https://vk.com"`, `langpackEntry:"https://`+r.ProxyBaseDomain+`/_/vk.com"`)
		cfg.vkuiApiJs = newStringReplace(`api.vk.com`, r.ProxyBaseDomain)
		r.config = cfg
	}
	return r.config
}

func (r *Replacer) DoReplaceRequest(req *fasthttp.Request, ctx ReplaceContext) {
	if bytes.Equal(ctx.Method, shared.MethodOptions) {
		// Заменяем заголовок Origin для CORS на заспросах со страниц VKUI
		// api.vk.com принимает только "https://static.vk.com" в заголовке Origin
		// Плюс к этому, если послать некорректный Referer, вк тоже пошлет нас куда подальше
		if origin := req.Header.Peek("Origin"); origin != nil {
			origins := string(origin)
			if strings.HasSuffix(origins, r.ProxyStaticDomain) {
				req.Header.Set("Origin", "https://static.vk.com")
				if referer := req.Header.Peek("Referer"); referer != nil {
					req.Header.Set("Referer", strings.Replace(string(referer), r.ProxyStaticDomain, "static.vk.com", 1))
				}
			}
		}
	}

	if ctx.Domain == "oauth.vk.com" {
		// Для авторизации страницы VKUI используют не уже готовый токен авторизации, а получают его при каждом
		// открытии страницы. В запросе авторизации передается текущий урл страницы VKUI, а так как она проксируется,
		// то она отличается от оригинальной. ВК проверяет этот урл страницы и отвергает авторизацию если он не
		// совпадает.
		// Зачем такие костыли - никто не знает, но нужно их обходить.
		// Если в запросе авторизации используется проксируемый статик домен - заменяем его на оригинальный.
		if strings.HasPrefix(ctx.Path, "/authorize?") {
			uri := fasthttp.AcquireURI()
			uri.Update(ctx.Path)
			args := uri.QueryArgs()
			sourceUrl := args.Peek("source_url")
			if sourceUrl != nil {
				sourceUrls := string(sourceUrl)
				modified := strings.Replace(sourceUrls, r.ProxyStaticDomain, "static.vk.com", 1)
				if modified != sourceUrls {
					args.Set("source_url", modified)
					req.SetRequestURIBytes(uri.RequestURI())
					req.SetHost("oauth.vk.com")
				}
			}
			fasthttp.ReleaseURI(uri)
		}
	}
}

func (r *Replacer) DoReplaceResponse(res *fasthttp.Response, body *bytebufferpool.ByteBuffer, ctx ReplaceContext) *bytebufferpool.ByteBuffer {
	config := r.getDomainConfig()

	if bytes.Equal(ctx.Method, shared.MethodOptions) {
		// Если в ответ на запрос OPTIONS с заданным Origin придет какой-то кривой ответ, то запросы не будут проходить
		if corsOrigin := res.Header.Peek("Access-Control-Allow-Origin"); corsOrigin != nil {
			res.Header.Set("Access-Control-Allow-Origin", "https://"+r.ProxyStaticDomain)
		}
		return body
	}

	if ctx.Domain == "api.vk.com" {
		for _, replace := range config.apiReplaces {
			body = replace.apply(body)
		}

		// Replace longpoll server
		if ctx.Path == "/method/messages.getLongPollServer" {
			body = config.apiLongpollReplace.apply(body)
		} else

		// Replace longpoll server for official app
		if ctx.Path == "/method/execute" ||
			ctx.Path == "/method/execute.imGetLongPollHistoryExtended" ||
			ctx.Path == "/method/execute.imLpInit" {
			body = config.apiOfficialLongpollReplace.apply(body)
			body = config.apiVkmeLongpollReplace.apply(body)
		}

		if ctx.FilterFeed {
			if ctx.Path == "/method/execute.getNewsfeedSmart" ||
				ctx.Path == "/method/newsfeed.get" {
				var parsed map[string]interface{}
				if err := json.Unmarshal(body.B, &parsed); err == nil {
					if parsed["response"] != nil {
						response := parsed["response"].(map[string]interface{})
						modified := tryRemoveAds(response)
						modified = tryInsertPost(response) || modified
						if modified {
							body.B, err = json.Marshal(parsed)
						}
					}
				}
			}
		}

	} else if ctx.Domain == "vk.com" {
		if ctx.Path == "/video_hls.php" {
			body = config.siteHlsReplace.apply(body)
		}

	} else if ctx.Domain == "static.vk.com" {
		if location := res.Header.Peek("Location"); location != nil {
			// Абсолютный редирект на статик меняем на относительный
			locstr := string(location)
			if idx := strings.Index(locstr, "static.vk.com"); idx != -1 {
				relativePath := ctx.Path
				if idx0 := strings.LastIndexByte(relativePath, '/'); idx0 != -1 {
					relativePath = relativePath[:idx0+1]
				}
				relativeRedirectPath := locstr[idx+13 /*static.vk.com*/:]
				relativeRedirectPath = relativeRedirectPath[longestCommonPrefix(relativeRedirectPath, relativePath):]
				res.Header.Set("Location", relativeRedirectPath)
			} else {
				buf := bytebufferpool.Get()
				buf.Set(location)
				buf = config.headLocationReplace.apply(buf)
				res.Header.SetBytesV("Location", buf.Bytes())
			}
		}

		if strings.HasSuffix(ctx.Path, ".js") {
			body = config.vkuiApiJs.apply(body)
			if strings.HasPrefix(ctx.Path, "/community_manage") {
				body = config.vkuiLangsJs.apply(body)
			}
		} else {
			contentType := string(res.Header.ContentType())
			if strings.HasPrefix(contentType, "text/html") {
				body = config.vkuiLangsHtml.apply(body)
			}
		}
	}

	return body
}

func tryRemoveAds(response map[string]interface{}) bool {
	raw, ok := response["items"]
	if !ok {
		return false
	}
	items := raw.([]interface{})
	removed := 0
	for i := len(items) - 1; i >= 0; i-- {
		post := items[i].(map[string]interface{})
		if post["type"] == "ads" || (post["type"] == "post" && post["marked_as_ads"] != nil && post["marked_as_ads"].(float64) == 1) {
			items[i] = items[len(items)-1]
			items[len(items)-1] = nil
			items = items[:len(items)-1]
			removed++
		}
	}
	if removed > 0 {
		newItems := make([]interface{}, len(items))
		copy(newItems, items)
		response["items"] = newItems
		return true
	}
	return false
}

func AcquireBuffer() *bytebufferpool.ByteBuffer {
	return replaceBufferPool.Get()
}

func ReleaseBuffer(buffer *bytebufferpool.ByteBuffer) {
	replaceBufferPool.Put(buffer)
}

func longestCommonPrefix(a, b string) (i int) {
	for ; i < len(a) && i < len(b); i++ {
		if a[i] != b[i] {
			break
		}
	}
	return
}
<|endoftext|>


<commit_before>// Copyright 2015 The Xorm Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

package xorm

import (
	"errors"
	"fmt"
	"net/url"
	"sort"
	"strings"

	"github.com/coscms/xorm/core"
)

type pqDriver struct {
	*core.Uri
}

type values map[string]string

func (vs values) Set(k, v string) {
	vs[k] = v
}

func (vs values) Get(k string) (v string) {
	return vs[k]
}

func errorf(s string, args ...interface{}) {
	panic(fmt.Errorf("pq: %s", fmt.Sprintf(s, args...)))
}

func parseURL(connstr string) (string, error) {
	u, err := url.Parse(connstr)
	if err != nil {
		return "", err
	}

	if u.Scheme != "postgresql" && u.Scheme != "postgres" {
		return "", fmt.Errorf("invalid connection protocol: %s", u.Scheme)
	}

	var kvs []string
	escaper := strings.NewReplacer(` `, `\ `, `'`, `\'`, `\`, `\\`)
	accrue := func(k, v string) {
		if v != "" {
			kvs = append(kvs, k+"="+escaper.Replace(v))
		}
	}

	if u.User != nil {
		v := u.User.Username()
		accrue("user", v)

		v, _ = u.User.Password()
		accrue("password", v)
	}

	i := strings.Index(u.Host, ":")
	if i < 0 {
		accrue("host", u.Host)
	} else {
		accrue("host", u.Host[:i])
		accrue("port", u.Host[i+1:])
	}

	if u.Path != "" {
		accrue("dbname", u.Path[1:])
	}

	q := u.Query()
	for k := range q {
		accrue(k, q.Get(k))
	}

	sort.Strings(kvs) // Makes testing easier (not a performance concern)
	return strings.Join(kvs, " "), nil
}

func parseOpts(name string, o values) {
	if len(name) == 0 {
		return
	}

	name = strings.TrimSpace(name)

	ps := strings.Split(name, " ")
	for _, p := range ps {
		kv := strings.Split(p, "=")
		if len(kv) < 2 {
			errorf("invalid option: %q", p)
		}
		o.Set(kv[0], kv[1])
	}
}

func (p *pqDriver) Parse(driverName, dataSourceName string) (*core.Uri, error) {
	db := &core.Uri{DbType: core.POSTGRES}
	o := make(values)
	var err error
	if strings.HasPrefix(dataSourceName, "postgresql://") || strings.HasPrefix(dataSourceName, "postgres://") {
		dataSourceName, err = parseURL(dataSourceName)
		if err != nil {
			return nil, err
		}
	}
	parseOpts(dataSourceName, o)

	db.DbName = o.Get("dbname")
	if db.DbName == "" {
		return nil, errors.New("dbname is empty")
	}
	db.Schema = o.Get("schema")
	if len(db.Schema) == 0 {
		db.Schema = "public"
	}
	p.Uri = db
	return db, nil
}

func (db *pqDriver) TableName(tableName string) string {
	if len(db.Schema) > 0 && db.Schema != `public` {
		return db.Schema + "." + tableName
	}
	return tableName
}
<commit_msg>update<commit_after>// Copyright 2015 The Xorm Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

package xorm

import (
	"errors"
	"fmt"
	"net/url"
	"sort"
	"strings"

	"github.com/coscms/xorm/core"
)

type pqDriver struct {
}

type values map[string]string

func (vs values) Set(k, v string) {
	vs[k] = v
}

func (vs values) Get(k string) (v string) {
	return vs[k]
}

func errorf(s string, args ...interface{}) {
	panic(fmt.Errorf("pq: %s", fmt.Sprintf(s, args...)))
}

func parseURL(connstr string) (string, error) {
	u, err := url.Parse(connstr)
	if err != nil {
		return "", err
	}

	if u.Scheme != "postgresql" && u.Scheme != "postgres" {
		return "", fmt.Errorf("invalid connection protocol: %s", u.Scheme)
	}

	var kvs []string
	escaper := strings.NewReplacer(` `, `\ `, `'`, `\'`, `\`, `\\`)
	accrue := func(k, v string) {
		if v != "" {
			kvs = append(kvs, k+"="+escaper.Replace(v))
		}
	}

	if u.User != nil {
		v := u.User.Username()
		accrue("user", v)

		v, _ = u.User.Password()
		accrue("password", v)
	}

	i := strings.Index(u.Host, ":")
	if i < 0 {
		accrue("host", u.Host)
	} else {
		accrue("host", u.Host[:i])
		accrue("port", u.Host[i+1:])
	}

	if u.Path != "" {
		accrue("dbname", u.Path[1:])
	}

	q := u.Query()
	for k := range q {
		accrue(k, q.Get(k))
	}

	sort.Strings(kvs) // Makes testing easier (not a performance concern)
	return strings.Join(kvs, " "), nil
}

func parseOpts(name string, o values) {
	if len(name) == 0 {
		return
	}

	name = strings.TrimSpace(name)

	ps := strings.Split(name, " ")
	for _, p := range ps {
		kv := strings.Split(p, "=")
		if len(kv) < 2 {
			errorf("invalid option: %q", p)
		}
		o.Set(kv[0], kv[1])
	}
}

func (p *pqDriver) Parse(driverName, dataSourceName string) (*core.Uri, error) {
	db := &core.Uri{DbType: core.POSTGRES}
	o := make(values)
	var err error
	if strings.HasPrefix(dataSourceName, "postgresql://") || strings.HasPrefix(dataSourceName, "postgres://") {
		dataSourceName, err = parseURL(dataSourceName)
		if err != nil {
			return nil, err
		}
	}
	parseOpts(dataSourceName, o)

	db.DbName = o.Get("dbname")
	if db.DbName == "" {
		return nil, errors.New("dbname is empty")
	}
	db.Schema = o.Get("schema")
	if len(db.Schema) == 0 {
		db.Schema = "public"
	}
	return db, nil
}
<|endoftext|>