<commit_before>// Copyright 2014 The Serviced Authors.
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package web

import (
	"crypto/tls"
	"fmt"
	"net"
	"sync"
	"time"

	"github.com/control-center/serviced/proxy"

	"github.com/control-center/serviced/coordinator/client"
	"github.com/control-center/serviced/dao"
	domainService "github.com/control-center/serviced/domain/service"
	"github.com/control-center/serviced/utils"
	"github.com/control-center/serviced/zzk"
	"github.com/control-center/serviced/zzk/registry"
	"github.com/control-center/serviced/zzk/service"
	"github.com/zenoss/glog"

	"net/http"
)

var (
	allportsLock sync.RWMutex
	allports     map[string]chan bool // map of port number to channel that destroys the server
	cpDao        dao.ControlPlane
)

func init() {
	allports = make(map[string]chan bool)
}

// Removes the port from our local cache and updates the service so the UI will flip to "disabled".
//  Only needs to be called if the port is being disabled unexpectedly due to an error
func disablePort(node service.ServicePublicEndpointNode) {
	//TODO: Add control plane methods to enable/disable public endpoints so we don't have to do a GetService and then UpdateService

	// remove the port from our local cache
	delete(allports, node.Name)

	// find the endpoint that matches this port number for this service (there will only be 1)
	var myService domainService.Service
	var myEndpoint domainService.ServiceEndpoint
	var unused int
	cpDao.GetService(node.ServiceID, &myService)
	for _, endpoint := range myService.Endpoints {
		for _, endpointPort := range endpoint.PortList {
			if endpointPort.PortAddr == node.Name {
				myEndpoint = endpoint
			}
		}
	}

	// disable port
	myService.EnablePort(myEndpoint.Name, node.Name, false)
	if err := cpDao.UpdateService(myService, &unused); err != nil {
		glog.Errorf("Error in disablePort(%s:%s): %v", node.ServiceID, node.Name, err)
	}
}

func (sc *ServiceConfig) ServePublicPorts(shutdown <-chan (interface{}), dao dao.ControlPlane) {
	cpDao = dao
	go sc.syncAllPublicPorts(shutdown)
}

// For HTTPS connections, we need to inject a header for downstream servers.
func (sc *ServiceConfig) createPortHttpServer(node service.ServicePublicEndpointNode) error {
	port := node.Name
	useTLS := true

	glog.V(1).Infof("About to listen on port (https) %s; UseTLS=%t", port, useTLS)

	// Setup a handler for the port https endpoint.  This differs from the
	// handler for cc/vhosts.
	httphandler := func(w http.ResponseWriter, r *http.Request) {
		glog.V(2).Infof("httphandler (port) handling request: %+v", r)

		pepKey := registry.GetPublicEndpointKey(node.Name, node.Type)
		pepEP, err := sc.getPublicEndpoint(string(pepKey))
		if err != nil {
			http.Error(w, err.Error(), http.StatusNotFound)
			return
		}

		rp := sc.getReverseProxy(pepEP.hostIP, sc.muxPort, pepEP.privateIP, pepEP.epPort, sc.muxTLS && (sc.muxPort > 0))
		glog.V(1).Infof("Time to set up %s public endpoint proxy for %v", pepKey, r.URL)

		// Set up the X-Forwarded-Proto header so that downstream servers know
		// the request originated as HTTPS.
		if _, found := r.Header["X-Forwarded-Proto"]; !found {
			r.Header.Set("X-Forwarded-Proto", "https")
		}

		rp.ServeHTTP(w, r)
		return
	}

	// Create a new port server with a default handler.
	portServer := http.NewServeMux()
	portServer.HandleFunc("/", httphandler)

	// Get the certificates and handle the error.
	certFile, keyFile, err := sc.getCertFiles()
	if err != nil {
		glog.Errorf("Error getting certificates for HTTPS port %s: %s", port, err)
		disablePort(node)
		return err
	}

	// Setup certificates and serve the requests.
	go func() {
		// This cipher suites and tls min version change may not be needed with golang 1.5
		// https://github.com/golang/go/issues/10094
		// https://github.com/golang/go/issues/9364
		config := &tls.Config{
			MinVersion:               utils.MinTLS(),
			PreferServerCipherSuites: true,
			CipherSuites:             utils.CipherSuites(),
		}
		server := &http.Server{Addr: port, TLSConfig: config, Handler: portServer}
		err := server.ListenAndServeTLS(certFile, keyFile)
		if err != nil {
			glog.Errorf("could not setup HTTPS (port) webserver: %s", err)
		}
	}()

	return nil
}

func (sc *ServiceConfig) createPublicPortServer(node service.ServicePublicEndpointNode, stopChan chan bool, shutdown <-chan (interface{})) error {
	port := node.Name
	useTLS := node.UseTLS
	proto := node.Protocol

	// Declare our listener..
	var listener net.Listener
	var err error

	glog.V(1).Infof("About to listen on port %s; UseTLS=%t", port, useTLS)

	if proto == "https" {
		// We have to set up an HttpListener to inject headers for downstram servers.
		return sc.createPortHttpServer(node)
	} else if useTLS {
		// Gather our certs files and handle the error.
		certFile, keyFile, err := sc.getCertFiles()
		if err != nil {
			glog.Errorf("Error getting certificates for TLS port %s: %s", port, err)
			disablePort(node)
			return err
		}

		// Create our certificate from the cert files (strings).
		glog.V(2).Infof("Loading certs from %s, %s", certFile, keyFile)
		cert, err := tls.LoadX509KeyPair(certFile, keyFile)
		if err != nil {
			glog.Errorf("Could not set up tls certificate for public endpoint on port %s for %s: %s", port, node.ServiceID, err)
			disablePort(node)
			return err
		}

		// The list of certs to use for our secure listener on this port.
		certs := []tls.Certificate { cert }

		// This cipher suites and tls min version change may not be needed with golang 1.5
		// https://github.com/golang/go/issues/10094
		// https://github.com/golang/go/issues/9364
		config := &tls.Config{
			MinVersion:               utils.MinTLS(),
			PreferServerCipherSuites: true,
			CipherSuites:             utils.CipherSuites(),
			Certificates:             certs,
		}

		glog.V(1).Infof("Listening with TLS")
		listener, err = tls.Listen("tcp", port, config)
	} else {
		glog.V(1).Infof("Listening without TLS")
		listener, err = net.Listen("tcp", port)
	}

	if err != nil {
		glog.Errorf("Could not setup TCP listener for port %s for public endpoint %s: %s", port, node.ServiceID, err)
		disablePort(node)
		return err
	}

	glog.Infof("Listening on port %s; UseTLS=%t", port, useTLS)

	go func() {
		for {
			// accept connection on public port
			localConn, err := listener.Accept()
			if err != nil {
				glog.V(1).Infof("Stopping accept on port %s", port)
				return
			}

			// lookup remote endpoint for this public port
			pepEPInfo, err := sc.getPublicEndpoint(fmt.Sprintf("%s-%d", node.Name, int(node.Type)))
			if err != nil {
				// This happens if an endpoint is accessed and the containers have died or not come up yet.
				glog.Errorf("Error retrieving public endpoint %s-%d: %s", node.Name, int(node.Type), err)
				// close the accepted connection and continue waiting for connections.
				if err := localConn.Close(); err != nil {
					glog.Errorf("Error closing client connection: %s", err)
				}
				continue
			}

			// setup remote connection
			var remoteAddr string
			_, isLocalContainer := sc.localAddrs[pepEPInfo.hostIP]
			if isLocalContainer {
				remoteAddr = fmt.Sprintf("%s:%d", pepEPInfo.privateIP, pepEPInfo.epPort)
			} else {
				remoteAddr = fmt.Sprintf("%s:%d", pepEPInfo.hostIP, sc.muxPort)
			}
			remoteConn, err := sc.getRemoteConnection(remoteAddr, isLocalContainer, sc.muxPort, pepEPInfo.privateIP, pepEPInfo.epPort, sc.muxTLS && (sc.muxPort > 0))
			if err != nil {
				glog.Errorf("Error getting remote connection for public endpoint %s-%d: %v", node.Name, int(node.Type), err)
				continue
			}

			glog.V(2).Infof("Established remote connection to %s", remoteConn.RemoteAddr())

			// Serve proxied requests/responses.  We pass our own port stop channel so that
			// all proxy loops end when our port is shutdown.
			go proxy.ProxyLoop(localConn, remoteConn, stopChan)
		}
	}()

	go func() {
		// Wait for shutdown, then kill all your connections
		select {
		case <-shutdown:
			// Received an application shutdown. Close the port channel to halt all proxy loops.
			glog.Infof("Shutting down port %s", port)
			close(stopChan)
		case <-stopChan:
		}

		listener.Close()
		glog.Infof("Closed port %s", port)
		return
	}()

	return nil
}

func (sc *ServiceConfig) syncAllPublicPorts(shutdown <-chan interface{}) error {
	rootConn, err := zzk.GetLocalConnection("/")
	if err != nil {
		glog.Errorf("syncAllPublicPorts - Error getting root zk connection: %v", err)
		return err
	}

	cancelChan := make(chan interface{})
	zkServicePEPService := service.ZKServicePublicEndpoints

	syncPorts := func(conn client.Connection, parentPath string, childIDs ...string) {
		allportsLock.Lock()
		defer allportsLock.Unlock()

		glog.V(1).Infof("syncPorts STARTING for parentPath:%s childIDs:%v", parentPath, childIDs)

		// start all servers that have been not started and enabled
		newPorts := make(map[string]chan bool)
		for _, pepID := range childIDs {

			// The pepID is the ZK child key. Get the node so we have all of the node data.
			glog.V(1).Infof("zkServicePEPService: %s, pepID: %s", zkServicePEPService, pepID)
			nodePath := fmt.Sprintf("%s/%s", zkServicePEPService, pepID)
			var node service.ServicePublicEndpointNode
			err := rootConn.Get(nodePath, &node)
			if err != nil {
				glog.Errorf("Unable to get the ZK Node from PepID")
				continue
			}

			if node.Type == registry.EPTypePort && node.Enabled {
				port := node.Name
				stopChan, running := allports[port]

				if !running {
					// recently enabled port - port should be opened
					stopChan = make(chan bool)
					if err := sc.createPublicPortServer(node, stopChan, shutdown); err != nil {
						continue
					}
				}

				newPorts[port] = stopChan
			}
		}

		// stop all servers that have been deleted or disabled
		for port, stopChan := range allports {
			_, found := newPorts[port]
			if !found {
				glog.V(2).Infof("Stopping port server for port %s", port)
				close(stopChan)
				glog.Infof("Port server shut down for port %s", port)
			}
		}

		allports = newPorts
		glog.V(2).Infof("Portserver allports: %+v", allports)
	}

	for {
		glog.V(1).Infof("Running registry.WatchChildren for zookeeper path: %s", zkServicePEPService)
		err := registry.WatchChildren(rootConn, zkServicePEPService, cancelChan, syncPorts, pepWatchError)
		if err != nil {
			glog.V(1).Infof("Will retry in 10 seconds to WatchChildren(%s) due to error: %v", zkServicePEPService, err)
			<-time.After(time.Second * 10)
			continue
		}
		select {
		case <-shutdown:
			close(cancelChan)
			return nil
		default:
		}
	}
}
<commit_msg>Add X-Forwarded-Proto to http type port endpoints<commit_after>// Copyright 2014 The Serviced Authors.
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package web

import (
	"crypto/tls"
	"fmt"
	"net"
	"sync"
	"time"

	"github.com/control-center/serviced/proxy"

	"github.com/control-center/serviced/coordinator/client"
	"github.com/control-center/serviced/dao"
	domainService "github.com/control-center/serviced/domain/service"
	"github.com/control-center/serviced/utils"
	"github.com/control-center/serviced/zzk"
	"github.com/control-center/serviced/zzk/registry"
	"github.com/control-center/serviced/zzk/service"
	"github.com/zenoss/glog"

	"net/http"
)

var (
	allportsLock sync.RWMutex
	allports     map[string]chan bool // map of port number to channel that destroys the server
	cpDao        dao.ControlPlane
)

func init() {
	allports = make(map[string]chan bool)
}

// Removes the port from our local cache and updates the service so the UI will flip to "disabled".
//  Only needs to be called if the port is being disabled unexpectedly due to an error
func disablePort(node service.ServicePublicEndpointNode) {
	//TODO: Add control plane methods to enable/disable public endpoints so we don't have to do a GetService and then UpdateService

	// remove the port from our local cache
	delete(allports, node.Name)

	// find the endpoint that matches this port number for this service (there will only be 1)
	var myService domainService.Service
	var myEndpoint domainService.ServiceEndpoint
	var unused int
	cpDao.GetService(node.ServiceID, &myService)
	for _, endpoint := range myService.Endpoints {
		for _, endpointPort := range endpoint.PortList {
			if endpointPort.PortAddr == node.Name {
				myEndpoint = endpoint
			}
		}
	}

	// disable port
	myService.EnablePort(myEndpoint.Name, node.Name, false)
	if err := cpDao.UpdateService(myService, &unused); err != nil {
		glog.Errorf("Error in disablePort(%s:%s): %v", node.ServiceID, node.Name, err)
	}
}

func (sc *ServiceConfig) ServePublicPorts(shutdown <-chan (interface{}), dao dao.ControlPlane) {
	cpDao = dao
	go sc.syncAllPublicPorts(shutdown)
}

// For HTTPS connections, we need to inject a header for downstream servers.
func (sc *ServiceConfig) createPortHttpServer(node service.ServicePublicEndpointNode) error {
	port := node.Name
	proto := node.Protocol
	useTLS := node.UseTLS

	glog.V(1).Infof("About to listen on port (%s) %s; UseTLS=%t", proto, port, useTLS)

	// Setup a handler for the port http(s) endpoint.  This differs from the
	// handler for cc/vhosts.
	httphandler := func(w http.ResponseWriter, r *http.Request) {
		glog.V(2).Infof("httphandler (port) handling request: %+v", r)

		pepKey := registry.GetPublicEndpointKey(node.Name, node.Type)
		pepEP, err := sc.getPublicEndpoint(string(pepKey))
		if err != nil {
			http.Error(w, err.Error(), http.StatusNotFound)
			return
		}

		rp := sc.getReverseProxy(pepEP.hostIP, sc.muxPort, pepEP.privateIP, pepEP.epPort, sc.muxTLS && (sc.muxPort > 0))
		glog.V(1).Infof("Time to set up %s public endpoint proxy for %v", pepKey, r.URL)

		// Set up the X-Forwarded-Proto header so that downstream servers know
		// the request originated as HTTPS.
		if _, found := r.Header["X-Forwarded-Proto"]; !found {
			r.Header.Set("X-Forwarded-Proto", proto)
		}

		rp.ServeHTTP(w, r)
		return
	}

	// Create a new port server with a default handler.
	portServer := http.NewServeMux()
	portServer.HandleFunc("/", httphandler)

	// HTTPS requires configuring the certificates for TLS.
	if useTLS {
		// Get the certificates and handle the error.
		glog.V(2).Infof("Getting TLS certificates for port: %s", port)
		certFile, keyFile, err := sc.getCertFiles()
		if err != nil {
			glog.Errorf("Error getting certificates for HTTPS port %s: %s", port, err)
			disablePort(node)
			return err
		}

		// Setup certificates and serve the requests.
		glog.V(2).Infof("Starting secure port endpoint server for port: %s", port)
		go func() {
			// This cipher suites and tls min version change may not be needed with golang 1.5
			// https://github.com/golang/go/issues/10094
			// https://github.com/golang/go/issues/9364
			config := &tls.Config{
				MinVersion:               utils.MinTLS(),
				PreferServerCipherSuites: true,
				CipherSuites:             utils.CipherSuites(),
			}
			server := &http.Server{Addr: port, TLSConfig: config, Handler: portServer}
			err := server.ListenAndServeTLS(certFile, keyFile)
			if err != nil {
				glog.Errorf("could not setup %s (port) webserver: %s", proto, err)
			}
		}()
	} else {
		// HTTP just needs a request server.
		glog.V(2).Infof("Starting port endpoint server for port: %s", port)
		go func() {
			server := &http.Server{Addr: port, Handler: portServer}
			err := server.ListenAndServe()
			if err != nil {
				glog.Errorf("could not setup %s (port) webserver: %s", proto, err)
			}
		}()
	}

	return nil
}

func (sc *ServiceConfig) createPublicPortServer(node service.ServicePublicEndpointNode, stopChan chan bool, shutdown <-chan (interface{})) error {
	port := node.Name
	useTLS := node.UseTLS
	proto := node.Protocol

	// Declare our listener..
	var listener net.Listener
	var err error

	glog.V(1).Infof("About to listen on port %s; UseTLS=%t", port, useTLS)

	if proto == "https" || proto == "http" {
		// We have to set up an HttpListener to inject headers for downstram servers.
		return sc.createPortHttpServer(node)
	} else if useTLS {
		// Gather our certs files and handle the error.
		certFile, keyFile, err := sc.getCertFiles()
		if err != nil {
			glog.Errorf("Error getting certificates for TLS port %s: %s", port, err)
			disablePort(node)
			return err
		}

		// Create our certificate from the cert files (strings).
		glog.V(2).Infof("Loading certs from %s, %s", certFile, keyFile)
		cert, err := tls.LoadX509KeyPair(certFile, keyFile)
		if err != nil {
			glog.Errorf("Could not set up tls certificate for public endpoint on port %s for %s: %s", port, node.ServiceID, err)
			disablePort(node)
			return err
		}

		// The list of certs to use for our secure listener on this port.
		certs := []tls.Certificate { cert }

		// This cipher suites and tls min version change may not be needed with golang 1.5
		// https://github.com/golang/go/issues/10094
		// https://github.com/golang/go/issues/9364
		config := &tls.Config{
			MinVersion:               utils.MinTLS(),
			PreferServerCipherSuites: true,
			CipherSuites:             utils.CipherSuites(),
			Certificates:             certs,
		}

		glog.V(1).Infof("Listening with TLS")
		listener, err = tls.Listen("tcp", port, config)
	} else {
		glog.V(1).Infof("Listening without TLS")
		listener, err = net.Listen("tcp", port)
	}

	if err != nil {
		glog.Errorf("Could not setup TCP listener for port %s for public endpoint %s: %s", port, node.ServiceID, err)
		disablePort(node)
		return err
	}

	glog.Infof("Listening on port %s; UseTLS=%t", port, useTLS)

	go func() {
		for {
			// accept connection on public port
			localConn, err := listener.Accept()
			if err != nil {
				glog.V(1).Infof("Stopping accept on port %s", port)
				return
			}

			// lookup remote endpoint for this public port
			pepEPInfo, err := sc.getPublicEndpoint(fmt.Sprintf("%s-%d", node.Name, int(node.Type)))
			if err != nil {
				// This happens if an endpoint is accessed and the containers have died or not come up yet.
				glog.Errorf("Error retrieving public endpoint %s-%d: %s", node.Name, int(node.Type), err)
				// close the accepted connection and continue waiting for connections.
				if err := localConn.Close(); err != nil {
					glog.Errorf("Error closing client connection: %s", err)
				}
				continue
			}

			// setup remote connection
			var remoteAddr string
			_, isLocalContainer := sc.localAddrs[pepEPInfo.hostIP]
			if isLocalContainer {
				remoteAddr = fmt.Sprintf("%s:%d", pepEPInfo.privateIP, pepEPInfo.epPort)
			} else {
				remoteAddr = fmt.Sprintf("%s:%d", pepEPInfo.hostIP, sc.muxPort)
			}
			remoteConn, err := sc.getRemoteConnection(remoteAddr, isLocalContainer, sc.muxPort, pepEPInfo.privateIP, pepEPInfo.epPort, sc.muxTLS && (sc.muxPort > 0))
			if err != nil {
				glog.Errorf("Error getting remote connection for public endpoint %s-%d: %v", node.Name, int(node.Type), err)
				continue
			}

			glog.V(2).Infof("Established remote connection to %s", remoteConn.RemoteAddr())

			// Serve proxied requests/responses.  We pass our own port stop channel so that
			// all proxy loops end when our port is shutdown.
			go proxy.ProxyLoop(localConn, remoteConn, stopChan)
		}
	}()

	go func() {
		// Wait for shutdown, then kill all your connections
		select {
		case <-shutdown:
			// Received an application shutdown. Close the port channel to halt all proxy loops.
			glog.Infof("Shutting down port %s", port)
			close(stopChan)
		case <-stopChan:
		}

		listener.Close()
		glog.Infof("Closed port %s", port)
		return
	}()

	return nil
}

func (sc *ServiceConfig) syncAllPublicPorts(shutdown <-chan interface{}) error {
	rootConn, err := zzk.GetLocalConnection("/")
	if err != nil {
		glog.Errorf("syncAllPublicPorts - Error getting root zk connection: %v", err)
		return err
	}

	cancelChan := make(chan interface{})
	zkServicePEPService := service.ZKServicePublicEndpoints

	syncPorts := func(conn client.Connection, parentPath string, childIDs ...string) {
		allportsLock.Lock()
		defer allportsLock.Unlock()

		glog.V(1).Infof("syncPorts STARTING for parentPath:%s childIDs:%v", parentPath, childIDs)

		// start all servers that have been not started and enabled
		newPorts := make(map[string]chan bool)
		for _, pepID := range childIDs {

			// The pepID is the ZK child key. Get the node so we have all of the node data.
			glog.V(1).Infof("zkServicePEPService: %s, pepID: %s", zkServicePEPService, pepID)
			nodePath := fmt.Sprintf("%s/%s", zkServicePEPService, pepID)
			var node service.ServicePublicEndpointNode
			err := rootConn.Get(nodePath, &node)
			if err != nil {
				glog.Errorf("Unable to get the ZK Node from PepID")
				continue
			}

			if node.Type == registry.EPTypePort && node.Enabled {
				port := node.Name
				stopChan, running := allports[port]

				if !running {
					// recently enabled port - port should be opened
					stopChan = make(chan bool)
					if err := sc.createPublicPortServer(node, stopChan, shutdown); err != nil {
						continue
					}
				}

				newPorts[port] = stopChan
			}
		}

		// stop all servers that have been deleted or disabled
		for port, stopChan := range allports {
			_, found := newPorts[port]
			if !found {
				glog.V(2).Infof("Stopping port server for port %s", port)
				close(stopChan)
				glog.Infof("Port server shut down for port %s", port)
			}
		}

		allports = newPorts
		glog.V(2).Infof("Portserver allports: %+v", allports)
	}

	for {
		glog.V(1).Infof("Running registry.WatchChildren for zookeeper path: %s", zkServicePEPService)
		err := registry.WatchChildren(rootConn, zkServicePEPService, cancelChan, syncPorts, pepWatchError)
		if err != nil {
			glog.V(1).Infof("Will retry in 10 seconds to WatchChildren(%s) due to error: %v", zkServicePEPService, err)
			<-time.After(time.Second * 10)
			continue
		}
		select {
		case <-shutdown:
			close(cancelChan)
			return nil
		default:
		}
	}
}
<|endoftext|>


<commit_before>package webapps

import (
	"log"
	"net/http"

	"github.com/ChristianSiegert/go-packages/i18n/languages"
	"github.com/ChristianSiegert/go-packages/sessions"
	"github.com/julienschmidt/httprouter"
)

type WebApp struct {
	// Default language to redirect to when requested language is not supported.
	defaultLanguageCode string

	languages    map[string]*languages.Language
	logger       *log.Logger
	router       *httprouter.Router
	serverHost   string
	serverPort   string
	sessionStore sessions.Store
}

func New(host, port string, logger *log.Logger, sessionStore sessions.Store) *WebApp {
	return &WebApp{
		languages:    make(map[string]*languages.Language, 1),
		logger:       logger,
		router:       httprouter.New(),
		serverHost:   host,
		serverPort:   port,
		sessionStore: sessionStore,
	}
}

func (w *WebApp) AddLanguage(language *languages.Language, isDefault bool) {
	w.languages[language.Code()] = language
	if isDefault {
		w.defaultLanguageCode = language.Code()
	}
}

func (w *WebApp) AddRoute(path string, handle httprouter.Handle, methods ...string) {
	for _, method := range methods {
		handle = w.handleLanguage(handle)
		handle = w.handleSession(handle)
		w.router.Handle(method, path, handle)
	}
}

func (w *WebApp) AddFileDir(urlPath, dirPath string) {
	w.router.ServeFiles(urlPath, http.Dir(dirPath))
}

func (w *WebApp) handleLanguage(handle httprouter.Handle) httprouter.Handle {
	return func(writer http.ResponseWriter, request *http.Request, params httprouter.Params) {
		languageCode := params.ByName("lang")

		// If language is not supported, redirect to default language
		language, ok := w.languages[languageCode]
		if !ok {
			if w.defaultLanguageCode == "" {
				panic("webapps: No default language set.")
			}
			http.Redirect(writer, request, "/"+w.defaultLanguageCode, http.StatusSeeOther)
			return
		}

		// Add language to context
		context := languages.NewContext(request.Context(), language)
		request = request.WithContext(context)

		// Execute given handle
		handle(writer, request, params)
	}
}

func (w *WebApp) handleSession(handle httprouter.Handle) httprouter.Handle {
	return func(writer http.ResponseWriter, request *http.Request, params httprouter.Params) {
		// Get session for this request
		session, err := w.sessionStore.Get(writer, request)
		if err != nil {
			log.Println("webapps: " + err.Error())
			http.Error(writer, "Interal Server Error", http.StatusInternalServerError)
			return
		}

		// Add session to context
		context := sessions.NewContext(request.Context(), session)
		request = request.WithContext(context)

		// Execute given handle
		handle(writer, request, params)
	}
}

// Start starts the HTTP server.
func (w *WebApp) Start() error {
	serverAddress := w.serverHost + ":" + w.serverPort
	return http.ListenAndServe(serverAddress, w.router)
}

// StartWithTls starts the HTTP server with TLS.
func (w *WebApp) StartWithTls(certificatePath, keyPath string) error {
	serverAddress := w.serverHost + w.serverPort
	return http.ListenAndServeTLS(serverAddress, certificatePath, keyPath, w.router)
}
<commit_msg>Added comments in package “webapps”.<commit_after>package webapps

import (
	"log"
	"net/http"

	"github.com/ChristianSiegert/go-packages/i18n/languages"
	"github.com/ChristianSiegert/go-packages/sessions"
	"github.com/julienschmidt/httprouter"
)

// WebApp represents a web application or web site.
type WebApp struct {
	// Default language to redirect to when requested language is not supported.
	defaultLanguageCode string

	languages    map[string]*languages.Language
	logger       *log.Logger
	router       *httprouter.Router
	serverHost   string
	serverPort   string
	sessionStore sessions.Store
}

// New returns a new WebApp.
func New(host, port string, logger *log.Logger, sessionStore sessions.Store) *WebApp {
	return &WebApp{
		languages:    make(map[string]*languages.Language, 1),
		logger:       logger,
		router:       httprouter.New(),
		serverHost:   host,
		serverPort:   port,
		sessionStore: sessionStore,
	}
}

// AddLanguage adds a language to w.
func (w *WebApp) AddLanguage(language *languages.Language, isDefault bool) {
	w.languages[language.Code()] = language
	if isDefault {
		w.defaultLanguageCode = language.Code()
	}
}

// AddRoute adds a route to w.
func (w *WebApp) AddRoute(path string, handle httprouter.Handle, methods ...string) {
	for _, method := range methods {
		handle = w.handleLanguage(handle)
		handle = w.handleSession(handle)
		w.router.Handle(method, path, handle)
	}
}

// AddFileDir makes files stored in dirPath accessible at urlPath. urlPath must
// end with “/*filepath”.
func (w *WebApp) AddFileDir(urlPath, dirPath string) {
	w.router.ServeFiles(urlPath, http.Dir(dirPath))
}

func (w *WebApp) handleLanguage(handle httprouter.Handle) httprouter.Handle {
	return func(writer http.ResponseWriter, request *http.Request, params httprouter.Params) {
		languageCode := params.ByName("lang")

		// If language is not supported, redirect to default language
		language, ok := w.languages[languageCode]
		if !ok {
			if w.defaultLanguageCode == "" {
				panic("webapps: No default language set.")
			}
			http.Redirect(writer, request, "/"+w.defaultLanguageCode, http.StatusSeeOther)
			return
		}

		// Add language to context
		context := languages.NewContext(request.Context(), language)
		request = request.WithContext(context)

		// Execute given handle
		handle(writer, request, params)
	}
}

func (w *WebApp) handleSession(handle httprouter.Handle) httprouter.Handle {
	return func(writer http.ResponseWriter, request *http.Request, params httprouter.Params) {
		// Get session for this request
		session, err := w.sessionStore.Get(writer, request)
		if err != nil {
			log.Println("webapps: " + err.Error())
			http.Error(writer, "Interal Server Error", http.StatusInternalServerError)
			return
		}

		// Add session to context
		context := sessions.NewContext(request.Context(), session)
		request = request.WithContext(context)

		// Execute given handle
		handle(writer, request, params)
	}
}

// Start starts the HTTP server.
func (w *WebApp) Start() error {
	serverAddress := w.serverHost + ":" + w.serverPort
	return http.ListenAndServe(serverAddress, w.router)
}

// StartWithTLS starts the HTTP server with TLS (Transport Layer Security).
func (w *WebApp) StartWithTLS(certificatePath, keyPath string) error {
	serverAddress := w.serverHost + w.serverPort
	return http.ListenAndServeTLS(serverAddress, certificatePath, keyPath, w.router)
}
<|endoftext|>


<commit_before>package main

// TODO: figure out how to handle varying numbers of pixels
// when we're getting pixels via our OPC server source

import (
	"fmt"
	"github.com/davecheney/profile"
	"github.com/droundy/goopt"
	"github.com/longears/pixelslinger/beaglebone"
	"github.com/longears/pixelslinger/config"
	"github.com/longears/pixelslinger/midi"
	"github.com/longears/pixelslinger/opc"
	"os"
	"runtime"
	"sort"
	"strings"
	"time"
)

const ONBOARD_LED_HEARTBEAT = 0
const ONBOARD_LED_MIDI = 1

const SPI_MAGIC_WORD = "spi"
const PRINT_MAGIC_WORD = "print"
const DEVNULL_MAGIC_WORD = "/dev/null"
const LOCALHOST = "localhost"
const SPI_FN = "/dev/spidev1.0"

func init() {
	runtime.GOMAXPROCS(2)
}

// these are pointers to the actual values from the command line parser
var LAYOUT_FN = goopt.String([]string{"-l", "--layout"}, "...", "layout file (required)")
var SOURCE = goopt.String([]string{"-s", "--source"}, "spatial-stripes", "pixel source (either a pattern name or "+LOCALHOST+"[:port])")
var DEST = goopt.String([]string{"-d", "--dest"}, "localhost", "destination (one of "+PRINT_MAGIC_WORD+", "+SPI_MAGIC_WORD+", "+DEVNULL_MAGIC_WORD+", or hostname[:port])")
var FPS = goopt.Int([]string{"-f", "--fps"}, 40, "max frames per second")
var SECONDS = goopt.Int([]string{"-n", "--seconds"}, 0, "quit after this many seconds")
var ONCE = goopt.Flag([]string{"-o", "--once"}, []string{}, "quit after one frame", "")

// Parse the command line flags.  If invalid, show help and quit.
// Add default ports if needed.
// Read the layout file.
// Return the number of pixels in the layout, the source and dest thread methods.
func parseFlags() (nPixels int, sourceThread, effectThread, destThread opc.ByteThread) {

	// get sorted pattern names
	patternNames := make([]string, len(opc.PATTERN_REGISTRY))
	ii := 0
	for k, _ := range opc.PATTERN_REGISTRY {
		patternNames[ii] = k
		ii++
	}
	sort.Strings(patternNames)

	goopt.Summary = "Available source patterns:\n"
	for _, patternName := range patternNames {
		goopt.Summary += "          " + patternName + "\n"
	}
	goopt.Parse(nil)

	// layout is required
	if *LAYOUT_FN == "..." {
		fmt.Println(goopt.Usage())
		fmt.Println("--------------------------------------------------------------------------------/")
		os.Exit(1)
	}

	// read locations
	locations := opc.ReadLocations(*LAYOUT_FN)
	nPixels = len(locations) / 3

	// choose source thread method
	if strings.Contains(*SOURCE, LOCALHOST) {
		// source is localhost, so we will start an OPC server.
		// add default port if needed
		if !strings.Contains(*SOURCE, ":") {
			*SOURCE += ":7890"
		}
		sourceThread = opc.MakeOpcServerThread(*SOURCE)
	} else if (*SOURCE)[0] == ':' {
		// source is ":4908"
		*SOURCE = "localhost" + *SOURCE
		sourceThread = opc.MakeOpcServerThread(*SOURCE)
	} else {
		// source is a pattern name
		sourceThreadMaker, ok := opc.PATTERN_REGISTRY[*SOURCE]
		if !ok {
			fmt.Printf("Error: unknown source or pattern \"%s\"\n", *SOURCE)
			fmt.Println("--------------------------------------------------------------------------------/")
			os.Exit(1)
		}
		sourceThread = sourceThreadMaker(locations)
	}

	// choose effect thread method
	effectThread = opc.MakeEffectFader(locations)

	// choose dest thread method
	switch *DEST {
	case DEVNULL_MAGIC_WORD:
		destThread = opc.MakeSendToDevNullThread()
	case PRINT_MAGIC_WORD:
		destThread = opc.MakeSendToScreenThread()
	case SPI_MAGIC_WORD:
		destThread = opc.MakeSendToLPD8806Thread(SPI_FN)
	default:
		// add default port if needed
		if !strings.Contains(*DEST, ":") {
			*DEST += ":7890"
		}
		destThread = opc.MakeSendToOpcThread(*DEST)
	}

	return // returns nPixels, sourceThread, destThread
}

// Launch the sourceThread and destThread methods and coordinate the transfer of bytes from one to the other.
// Run until timeToRun seconds have passed and return.  If timeToRun is 0, run forever.
// Turn on the CPU profiler if timeToRun seconds > 0.
// Limit the framerate to a max of fps unless fps is 0.
func mainLoop(nPixels int, sourceThread, effectThread, destThread opc.ByteThread, fps float64, timeToRun float64) {
	if timeToRun > 0 {
		fmt.Printf("[mainLoop] Running for %f seconds with profiling turned on, pixels and network\n", timeToRun)
		defer profile.Start(profile.CPUProfile).Stop()
	} else {
		fmt.Println("[mainLoop] Running forever")
	}

	// prepare the byte slices and channels that connect the source and dest threads
	fillingSlice := make([]byte, nPixels*3)
	sendingSlice := make([]byte, nPixels*3)

	bytesToFillChan := make(chan []byte, 0)
	toEffectChan := make(chan []byte, 0)
	bytesFilledChan := make(chan []byte, 0)
	bytesToSendChan := make(chan []byte, 0)
	bytesSentChan := make(chan []byte, 0)

	// set up midi
	midiMessageChan := midi.GetMidiMessageStream("/dev/midi1") // this launches the midi thread
	midiState := midi.MidiState{}
	// set initial values for controller knobs
	//  (because the midi hardware only sends us values when the knobs move)
	for knob, defaultVal := range config.DEFAULT_KNOB_VALUES {
		midiState.ControllerValues[knob] = defaultVal
	}

	// launch the threads
	go sourceThread(bytesToFillChan, toEffectChan, &midiState)
	go effectThread(toEffectChan, bytesFilledChan, &midiState)
	go destThread(bytesToSendChan, bytesSentChan, &midiState)

	// main loop
	frame_budget_ms := 1000.0 / fps
	startTime := float64(time.Now().UnixNano()) / 1.0e9
	lastPrintTime := startTime
	frameStartTime := startTime
	frameEndTime := startTime
	framesSinceLastPrint := 0
	firstIteration := true
	flipper := 0
	beaglebone.SetOnboardLED(0, 1)
	for {
		// if we have any frame budget left from last time around, sleep to control the framerate
		if fps > 0 {
			frameEndTime = float64(time.Now().UnixNano()) / 1.0e9
			timeRemaining := float64(frame_budget_ms)/1000 - (frameEndTime - frameStartTime)
			if timeRemaining > 0 {
				time.Sleep(time.Duration(timeRemaining*1000*1000) * time.Microsecond)
			}
		}

		// fps reporting and bookkeeping
		// print framerate occasionally
		frameStartTime = float64(time.Now().UnixNano()) / 1.0e9
		framesSinceLastPrint += 1
		if frameStartTime > lastPrintTime+1 {
			lastPrintTime = frameStartTime
			fmt.Printf("[mainLoop] %f ms/frame (%d fps)\n", 1000.0/float64(framesSinceLastPrint), framesSinceLastPrint)
			framesSinceLastPrint = 0
			// toggle LED
			beaglebone.SetOnboardLED(ONBOARD_LED_HEARTBEAT, flipper)
			flipper = 1 - flipper
		}

		// if profiling, quit after a while
		if timeToRun > 0 && frameStartTime > startTime+timeToRun {
			return
		}

		// get midi
		midiState.UpdateStateFromChannel(midiMessageChan)
		if len(midiState.RecentMidiMessages) > 0 {
			beaglebone.SetOnboardLED(ONBOARD_LED_MIDI, 1)
		} else {
			beaglebone.SetOnboardLED(ONBOARD_LED_MIDI, 0)
		}

		// start the threads filling and sending slices in parallel.
		// if this is the first time through the loop we have to skip
		//  the sending stage or we'll send out a whole bunch of zeros.
		bytesToFillChan <- fillingSlice
		if !firstIteration {
			bytesToSendChan <- sendingSlice
		}

		// if only sending one frame, let's just get it all over with now
		//  or we'd have to compute two frames worth of pixels because of
		//  the double buffering effect of the two parallel threads
		if *ONCE {
			// get filled bytes and send them
			bytesToSendChan <- <-bytesFilledChan
			// wait for sending to complete
			<-bytesSentChan
			fmt.Println("[mainLoop] just running once.  quitting now.")
			return
		}

		// wait until both filling and sending threads are done
		<-bytesFilledChan
		if !firstIteration {
			<-bytesSentChan
		}

		// swap the slices
		sendingSlice, fillingSlice = fillingSlice, sendingSlice

		firstIteration = false
	}
}

func main() {
	fmt.Println("--------------------------------------------------------------------------------\\")
	defer fmt.Println("--------------------------------------------------------------------------------/")

	nPixels, sourceThread, effectThread, destThread := parseFlags()
	mainLoop(nPixels, sourceThread, effectThread, destThread, float64(*FPS), float64(*SECONDS))
}
<commit_msg>update profile dependency<commit_after>package main

// TODO: figure out how to handle varying numbers of pixels
// when we're getting pixels via our OPC server source

import (
	"fmt"
	"github.com/droundy/goopt"
	"github.com/longears/pixelslinger/beaglebone"
	"github.com/longears/pixelslinger/config"
	"github.com/longears/pixelslinger/midi"
	"github.com/longears/pixelslinger/opc"
	"github.com/pkg/profile"
	"os"
	"runtime"
	"sort"
	"strings"
	"time"
)

const ONBOARD_LED_HEARTBEAT = 0
const ONBOARD_LED_MIDI = 1

const SPI_MAGIC_WORD = "spi"
const PRINT_MAGIC_WORD = "print"
const DEVNULL_MAGIC_WORD = "/dev/null"
const LOCALHOST = "localhost"
const SPI_FN = "/dev/spidev1.0"

func init() {
	runtime.GOMAXPROCS(2)
}

// these are pointers to the actual values from the command line parser
var LAYOUT_FN = goopt.String([]string{"-l", "--layout"}, "...", "layout file (required)")
var SOURCE = goopt.String([]string{"-s", "--source"}, "spatial-stripes", "pixel source (either a pattern name or "+LOCALHOST+"[:port])")
var DEST = goopt.String([]string{"-d", "--dest"}, "localhost", "destination (one of "+PRINT_MAGIC_WORD+", "+SPI_MAGIC_WORD+", "+DEVNULL_MAGIC_WORD+", or hostname[:port])")
var FPS = goopt.Int([]string{"-f", "--fps"}, 40, "max frames per second")
var SECONDS = goopt.Int([]string{"-n", "--seconds"}, 0, "quit after this many seconds")
var ONCE = goopt.Flag([]string{"-o", "--once"}, []string{}, "quit after one frame", "")

// Parse the command line flags.  If invalid, show help and quit.
// Add default ports if needed.
// Read the layout file.
// Return the number of pixels in the layout, the source and dest thread methods.
func parseFlags() (nPixels int, sourceThread, effectThread, destThread opc.ByteThread) {

	// get sorted pattern names
	patternNames := make([]string, len(opc.PATTERN_REGISTRY))
	ii := 0
	for k, _ := range opc.PATTERN_REGISTRY {
		patternNames[ii] = k
		ii++
	}
	sort.Strings(patternNames)

	goopt.Summary = "Available source patterns:\n"
	for _, patternName := range patternNames {
		goopt.Summary += "          " + patternName + "\n"
	}
	goopt.Parse(nil)

	// layout is required
	if *LAYOUT_FN == "..." {
		fmt.Println(goopt.Usage())
		fmt.Println("--------------------------------------------------------------------------------/")
		os.Exit(1)
	}

	// read locations
	locations := opc.ReadLocations(*LAYOUT_FN)
	nPixels = len(locations) / 3

	// choose source thread method
	if strings.Contains(*SOURCE, LOCALHOST) {
		// source is localhost, so we will start an OPC server.
		// add default port if needed
		if !strings.Contains(*SOURCE, ":") {
			*SOURCE += ":7890"
		}
		sourceThread = opc.MakeOpcServerThread(*SOURCE)
	} else if (*SOURCE)[0] == ':' {
		// source is ":4908"
		*SOURCE = "localhost" + *SOURCE
		sourceThread = opc.MakeOpcServerThread(*SOURCE)
	} else {
		// source is a pattern name
		sourceThreadMaker, ok := opc.PATTERN_REGISTRY[*SOURCE]
		if !ok {
			fmt.Printf("Error: unknown source or pattern \"%s\"\n", *SOURCE)
			fmt.Println("--------------------------------------------------------------------------------/")
			os.Exit(1)
		}
		sourceThread = sourceThreadMaker(locations)
	}

	// choose effect thread method
	effectThread = opc.MakeEffectFader(locations)

	// choose dest thread method
	switch *DEST {
	case DEVNULL_MAGIC_WORD:
		destThread = opc.MakeSendToDevNullThread()
	case PRINT_MAGIC_WORD:
		destThread = opc.MakeSendToScreenThread()
	case SPI_MAGIC_WORD:
		destThread = opc.MakeSendToLPD8806Thread(SPI_FN)
	default:
		// add default port if needed
		if !strings.Contains(*DEST, ":") {
			*DEST += ":7890"
		}
		destThread = opc.MakeSendToOpcThread(*DEST)
	}

	return // returns nPixels, sourceThread, destThread
}

// Launch the sourceThread and destThread methods and coordinate the transfer of bytes from one to the other.
// Run until timeToRun seconds have passed and return.  If timeToRun is 0, run forever.
// Turn on the CPU profiler if timeToRun seconds > 0.
// Limit the framerate to a max of fps unless fps is 0.
func mainLoop(nPixels int, sourceThread, effectThread, destThread opc.ByteThread, fps float64, timeToRun float64) {
	if timeToRun > 0 {
		fmt.Printf("[mainLoop] Running for %f seconds with profiling turned on, pixels and network\n", timeToRun)
		defer profile.Start(profile.CPUProfile).Stop()
	} else {
		fmt.Println("[mainLoop] Running forever")
	}

	// prepare the byte slices and channels that connect the source and dest threads
	fillingSlice := make([]byte, nPixels*3)
	sendingSlice := make([]byte, nPixels*3)

	bytesToFillChan := make(chan []byte, 0)
	toEffectChan := make(chan []byte, 0)
	bytesFilledChan := make(chan []byte, 0)
	bytesToSendChan := make(chan []byte, 0)
	bytesSentChan := make(chan []byte, 0)

	// set up midi
	midiMessageChan := midi.GetMidiMessageStream("/dev/midi1") // this launches the midi thread
	midiState := midi.MidiState{}
	// set initial values for controller knobs
	//  (because the midi hardware only sends us values when the knobs move)
	for knob, defaultVal := range config.DEFAULT_KNOB_VALUES {
		midiState.ControllerValues[knob] = defaultVal
	}

	// launch the threads
	go sourceThread(bytesToFillChan, toEffectChan, &midiState)
	go effectThread(toEffectChan, bytesFilledChan, &midiState)
	go destThread(bytesToSendChan, bytesSentChan, &midiState)

	// main loop
	frame_budget_ms := 1000.0 / fps
	startTime := float64(time.Now().UnixNano()) / 1.0e9
	lastPrintTime := startTime
	frameStartTime := startTime
	frameEndTime := startTime
	framesSinceLastPrint := 0
	firstIteration := true
	flipper := 0
	beaglebone.SetOnboardLED(0, 1)
	for {
		// if we have any frame budget left from last time around, sleep to control the framerate
		if fps > 0 {
			frameEndTime = float64(time.Now().UnixNano()) / 1.0e9
			timeRemaining := float64(frame_budget_ms)/1000 - (frameEndTime - frameStartTime)
			if timeRemaining > 0 {
				time.Sleep(time.Duration(timeRemaining*1000*1000) * time.Microsecond)
			}
		}

		// fps reporting and bookkeeping
		// print framerate occasionally
		frameStartTime = float64(time.Now().UnixNano()) / 1.0e9
		framesSinceLastPrint += 1
		if frameStartTime > lastPrintTime+1 {
			lastPrintTime = frameStartTime
			fmt.Printf("[mainLoop] %f ms/frame (%d fps)\n", 1000.0/float64(framesSinceLastPrint), framesSinceLastPrint)
			framesSinceLastPrint = 0
			// toggle LED
			beaglebone.SetOnboardLED(ONBOARD_LED_HEARTBEAT, flipper)
			flipper = 1 - flipper
		}

		// if profiling, quit after a while
		if timeToRun > 0 && frameStartTime > startTime+timeToRun {
			return
		}

		// get midi
		midiState.UpdateStateFromChannel(midiMessageChan)
		if len(midiState.RecentMidiMessages) > 0 {
			beaglebone.SetOnboardLED(ONBOARD_LED_MIDI, 1)
		} else {
			beaglebone.SetOnboardLED(ONBOARD_LED_MIDI, 0)
		}

		// start the threads filling and sending slices in parallel.
		// if this is the first time through the loop we have to skip
		//  the sending stage or we'll send out a whole bunch of zeros.
		bytesToFillChan <- fillingSlice
		if !firstIteration {
			bytesToSendChan <- sendingSlice
		}

		// if only sending one frame, let's just get it all over with now
		//  or we'd have to compute two frames worth of pixels because of
		//  the double buffering effect of the two parallel threads
		if *ONCE {
			// get filled bytes and send them
			bytesToSendChan <- <-bytesFilledChan
			// wait for sending to complete
			<-bytesSentChan
			fmt.Println("[mainLoop] just running once.  quitting now.")
			return
		}

		// wait until both filling and sending threads are done
		<-bytesFilledChan
		if !firstIteration {
			<-bytesSentChan
		}

		// swap the slices
		sendingSlice, fillingSlice = fillingSlice, sendingSlice

		firstIteration = false
	}
}

func main() {
	fmt.Println("--------------------------------------------------------------------------------\\")
	defer fmt.Println("--------------------------------------------------------------------------------/")

	nPixels, sourceThread, effectThread, destThread := parseFlags()
	mainLoop(nPixels, sourceThread, effectThread, destThread, float64(*FPS), float64(*SECONDS))
}
<|endoftext|>


<commit_before>// Copyright 2016 Google Inc.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package pixiecore

import (
	"bytes"
	"html/template"
	"io/ioutil"
	"mime"
	"net/http"
	"path/filepath"
	"strings"
	"time"

	"github.com/cockroachdb/cockroach/ui"
)

//go:generate go-bindata -o ui/autogen.go -ignore autogen.go -pkg ui -nometadata -nomemcopy -prefix ui/ ui/

const assetsPath = "/_/assets/"

func (s *Server) serveUI(mux *http.ServeMux) {
	mux.HandleFunc("/", s.handleUI)
	mux.HandleFunc(assetsPath, s.handleUI)
}

func (s *Server) handleUI(w http.ResponseWriter, r *http.Request) {
	if r.URL.Path == "/" {
		r.URL.Path = assetsPath + "index.html"
	}
	if !strings.HasPrefix(r.URL.Path, assetsPath) {
		http.NotFound(w, r)
		return
	}
	// Sticking a / in front of the path before cleaning it will strip
	// out any "../" attempts at path traversal. Then we remove the
	// leading /, and we end up with a path we can filepath.Join or
	// fetch from asset data.
	path := filepath.Clean("/" + r.URL.Path[len(assetsPath):])[1:]
	t, err := s.getTemplate(path)
	if err != nil {
		s.log("UI", "Failed to parse template for %q: %s", path, err)
		http.Error(w, "internal server error", http.StatusInternalServerError)
		return
	}
	var b bytes.Buffer
	s.eventsMu.Lock()
	defer s.eventsMu.Unlock()
	if err = t.Execute(&b, s.events); err != nil {
		s.log("UI", "Failed to expand template for %q: %s", path, err)
		http.Error(w, "internal server error", http.StatusInternalServerError)
		return
	}
	mimetype := mime.TypeByExtension(filepath.Ext(path))
	if mimetype != "" {
		w.Header().Set("Content-Type", mimetype)
	}

	b.WriteTo(w)
}

func (s *Server) getTemplate(name string) (*template.Template, error) {
	var (
		bs  []byte
		err error
	)
	if s.UIAssetsDir != "" {
		bs, err = ioutil.ReadFile(filepath.Join(s.UIAssetsDir, name))
	} else {
		bs, err = ui.Asset(name)
	}
	if err != nil {
		return nil, err
	}
	funcs := template.FuncMap{
		"dec": func(i int) int {
			return i - 1
		},
		"timestamp_millis": func(t time.Time) int64 {
			return t.UnixNano() / int64(time.Millisecond)
		},
	}
	return template.New(name).Funcs(funcs).Parse(string(bs))
}
<commit_msg>pixiecore: fix import mangled by goimports.<commit_after>// Copyright 2016 Google Inc.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package pixiecore

import (
	"bytes"
	"html/template"
	"io/ioutil"
	"mime"
	"net/http"
	"path/filepath"
	"strings"
	"time"

	"go.universe.tf/netboot/pixiecore/ui"
)

//go:generate go-bindata -o ui/autogen.go -ignore autogen.go -pkg ui -nometadata -nomemcopy -prefix ui/ ui/

const assetsPath = "/_/assets/"

func (s *Server) serveUI(mux *http.ServeMux) {
	mux.HandleFunc("/", s.handleUI)
	mux.HandleFunc(assetsPath, s.handleUI)
}

func (s *Server) handleUI(w http.ResponseWriter, r *http.Request) {
	if r.URL.Path == "/" {
		r.URL.Path = assetsPath + "index.html"
	}
	if !strings.HasPrefix(r.URL.Path, assetsPath) {
		http.NotFound(w, r)
		return
	}
	// Sticking a / in front of the path before cleaning it will strip
	// out any "../" attempts at path traversal. Then we remove the
	// leading /, and we end up with a path we can filepath.Join or
	// fetch from asset data.
	path := filepath.Clean("/" + r.URL.Path[len(assetsPath):])[1:]
	t, err := s.getTemplate(path)
	if err != nil {
		s.log("UI", "Failed to parse template for %q: %s", path, err)
		http.Error(w, "internal server error", http.StatusInternalServerError)
		return
	}
	var b bytes.Buffer
	s.eventsMu.Lock()
	defer s.eventsMu.Unlock()
	if err = t.Execute(&b, s.events); err != nil {
		s.log("UI", "Failed to expand template for %q: %s", path, err)
		http.Error(w, "internal server error", http.StatusInternalServerError)
		return
	}
	mimetype := mime.TypeByExtension(filepath.Ext(path))
	if mimetype != "" {
		w.Header().Set("Content-Type", mimetype)
	}

	b.WriteTo(w)
}

func (s *Server) getTemplate(name string) (*template.Template, error) {
	var (
		bs  []byte
		err error
	)
	if s.UIAssetsDir != "" {
		bs, err = ioutil.ReadFile(filepath.Join(s.UIAssetsDir, name))
	} else {
		bs, err = ui.Asset(name)
	}
	if err != nil {
		return nil, err
	}
	funcs := template.FuncMap{
		"dec": func(i int) int {
			return i - 1
		},
		"timestamp_millis": func(t time.Time) int64 {
			return t.UnixNano() / int64(time.Millisecond)
		},
	}
	return template.New(name).Funcs(funcs).Parse(string(bs))
}
<|endoftext|>


<commit_before>// Copyright Istio Authors. All Rights Reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package util

import (
	"bytes"
	"context"
	"fmt"
	"io/ioutil"
	"os"
	"os/exec"
	"strings"

	"github.com/google/go-github/v28/github"
	"golang.org/x/oauth2"

	"istio.io/pkg/log"
	"istio.io/release-builder/pkg/model"
)

// PushCommit will create a branch and push a commit with the specified commit text
func PushCommit(manifest model.Manifest, repo, branch, commitString string, dryrun bool, githubToken string) (changes bool, err error) {
	output := bytes.Buffer{}
	cmd := VerboseCommand("git", "status", "--porcelain")
	cmd.Dir = manifest.RepoDir(repo)
	cmd.Stdout = &output
	if err := cmd.Run(); err != nil {
		return false, err
	}
	if output.Len() == 0 {
		log.Infof("no changes found to commit")
		return false, nil
	}
	log.Infof("changes found:\n%s", &output)

	if !dryrun {
		cmd = VerboseCommand("git", "checkout", "-b", branch)
		cmd.Dir = manifest.RepoDir(repo)
		if err := cmd.Run(); err != nil {
			return true, err
		}

		cmd = VerboseCommand("git", "add", "-A")
		cmd.Dir = manifest.RepoDir(repo)
		if err := cmd.Run(); err != nil {
			return true, err
		}

		// Get user.email and user.name from the GitHub token
		ctx := context.Background()
		ts := oauth2.StaticTokenSource(
			&oauth2.Token{AccessToken: githubToken},
		)

		tc := oauth2.NewClient(ctx, ts)
		client := github.NewClient(tc)

		user, _, err := client.Users.Get(ctx, "")
		if err != nil {
			return true, err
		}

		cmd = VerboseCommand("git", "commit", "-m", commitString, "-c", "user.name="+*user.Name, "-c", "user.email="+*user.Email,
			"--author="+*user.Name+"<"+*user.Email+">")
		cmd.Dir = manifest.RepoDir(repo)
		if err := cmd.Run(); err != nil {
			return true, err
		}

		cmd = VerboseCommand("git", "push", "--set-upstream", "origin", branch)
		cmd.Dir = manifest.RepoDir(repo)
		if err := cmd.Run(); err != nil {
			return true, err
		}
	}
	return true, nil
}

// CreatePR will look for changes. If changes exist, it will create
// a branch and push a commit with the specified commit text
func CreatePR(manifest model.Manifest, repo, branch, commitString string, dryrun bool, githubToken string) error {
	changes, err := PushCommit(manifest, repo, branch, commitString, dryrun, githubToken)
	if err != nil {
		return err
	}

	if changes && !dryrun {
		var cmd *exec.Cmd
		if repo != "envoy" {
			cmd = VerboseCommand("gh", "pr", "create", "--repo", manifest.Dependencies.Get()[repo].Git,
				"--fill", "--head", branch, "--base", manifest.Dependencies.Get()[repo].Branch, "--label", "release-notes-none")
		} else {
			cmd = VerboseCommand("gh", "pr", "create", "--repo", manifest.Dependencies.Get()[repo].Git,
				"--fill", "--head", branch, "--base", manifest.Dependencies.Get()[repo].Branch)
		}
		cmd.Dir = manifest.RepoDir(repo)
		if err := cmd.Run(); err != nil {
			return err
		}
	}
	return nil
}

// GetGithubToken returns the GitHub token from the specified file. If the filename
// isn't specified, it will return the token set in the GITHUB_TOKEN environment variable.
func GetGithubToken(file string) (string, error) {
	if file != "" {
		b, err := ioutil.ReadFile(file)
		if err != nil {
			return "", fmt.Errorf("failed to read github token: %v", file)
		}
		return strings.TrimSpace(string(b)), nil
	}
	return os.Getenv("GITHUB_TOKEN"), nil
}
<commit_msg>Update parameter order in git command (#598)<commit_after>// Copyright Istio Authors. All Rights Reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package util

import (
	"bytes"
	"context"
	"fmt"
	"io/ioutil"
	"os"
	"os/exec"
	"strings"

	"github.com/google/go-github/v28/github"
	"golang.org/x/oauth2"

	"istio.io/pkg/log"
	"istio.io/release-builder/pkg/model"
)

// PushCommit will create a branch and push a commit with the specified commit text
func PushCommit(manifest model.Manifest, repo, branch, commitString string, dryrun bool, githubToken string) (changes bool, err error) {
	output := bytes.Buffer{}
	cmd := VerboseCommand("git", "status", "--porcelain")
	cmd.Dir = manifest.RepoDir(repo)
	cmd.Stdout = &output
	if err := cmd.Run(); err != nil {
		return false, err
	}
	if output.Len() == 0 {
		log.Infof("no changes found to commit")
		return false, nil
	}
	log.Infof("changes found:\n%s", &output)

	if !dryrun {
		cmd = VerboseCommand("git", "checkout", "-b", branch)
		cmd.Dir = manifest.RepoDir(repo)
		if err := cmd.Run(); err != nil {
			return true, err
		}

		cmd = VerboseCommand("git", "add", "-A")
		cmd.Dir = manifest.RepoDir(repo)
		if err := cmd.Run(); err != nil {
			return true, err
		}

		// Get user.email and user.name from the GitHub token
		ctx := context.Background()
		ts := oauth2.StaticTokenSource(
			&oauth2.Token{AccessToken: githubToken},
		)

		tc := oauth2.NewClient(ctx, ts)
		client := github.NewClient(tc)

		user, _, err := client.Users.Get(ctx, "")
		if err != nil {
			return true, err
		}

		// user.Email may be nil, so set to an empty string
		emptyString := ""
		if user.Email == nil {
			user.Email = &emptyString
		}

		cmd = VerboseCommand("git", "-c", "user.name="+*user.Name, "-c", "user.email="+*user.Email, "commit",
			"--message", commitString, "--author="+*user.Name+"<"+*user.Email+">")
		cmd.Dir = manifest.RepoDir(repo)
		if err := cmd.Run(); err != nil {
			return true, err
		}

		cmd = VerboseCommand("git", "push", "--set-upstream", "origin", branch)
		cmd.Dir = manifest.RepoDir(repo)
		if err := cmd.Run(); err != nil {
			return true, err
		}
	}
	return true, nil
}

// CreatePR will look for changes. If changes exist, it will create
// a branch and push a commit with the specified commit text
func CreatePR(manifest model.Manifest, repo, branch, commitString string, dryrun bool, githubToken string) error {
	changes, err := PushCommit(manifest, repo, branch, commitString, dryrun, githubToken)
	if err != nil {
		return err
	}

	if changes && !dryrun {
		var cmd *exec.Cmd
		// Don't label the envoy or non istio repos with `release-notes-none`
		if repo == "envoy" || !strings.Contains(manifest.Dependencies.Get()[repo].Git, "github.com/istio/") {
			cmd = VerboseCommand("gh", "pr", "create", "--repo", manifest.Dependencies.Get()[repo].Git,
				"--fill", "--head", branch, "--base", manifest.Dependencies.Get()[repo].Branch)
		} else {
			cmd = VerboseCommand("gh", "pr", "create", "--repo", manifest.Dependencies.Get()[repo].Git,
				"--fill", "--head", branch, "--base", manifest.Dependencies.Get()[repo].Branch, "--label", "release-notes-none")
		}
		cmd.Dir = manifest.RepoDir(repo)
		if err := cmd.Run(); err != nil {
			return err
		}
	}
	return nil
}

// GetGithubToken returns the GitHub token from the specified file. If the filename
// isn't specified, it will return the token set in the GITHUB_TOKEN environment variable.
func GetGithubToken(file string) (string, error) {
	if file != "" {
		b, err := ioutil.ReadFile(file)
		if err != nil {
			return "", fmt.Errorf("failed to read github token: %v", file)
		}
		return strings.TrimSpace(string(b)), nil
	}
	return os.Getenv("GITHUB_TOKEN"), nil
}
<|endoftext|>


<commit_before>package moves

import (
	"errors"
	"github.com/jkomoros/boardgame"
)

//ApplyUntil is a simple move that is legal to apply in succession until its
//ConditionMet returns nil. You need to override ConditionMet as well as
//provide an Apply method.
type ApplyUntil struct {
	Base
}

type conditionMetter interface {
	ConditionMet(state boardgame.State) error
}

//AllowMultipleInProgression returns true because the move is applied until
//ConditionMet returns nil.
func (a *ApplyUntil) AllowMultipleInProgression() bool {
	return true
}

//ConditionMet is called in ApplyUntil's Legal method. If the condition has
//been met, return nil. If it has not been met, return an error describing why
//it is not yet met. The default ConditionMet returns nil always; you almost
//certainly want to override it.
func (a *ApplyUntil) ConditionMet(state boardgame.State) error {
	return nil
}

func (a *ApplyUntil) ValidConfiguration(exampleState boardgame.MutableState) error {
	if _, ok := a.TopLevelStruct().(conditionMetter); !ok {
		return errors.New("Embedding Move doesn't have ConditionMet")
	}
	return nil
}

//Legal returns an error until ConditionMet returns nil.
func (a *ApplyUntil) Legal(state boardgame.State, proposer boardgame.PlayerIndex) error {
	if err := a.Base.Legal(state, proposer); err != nil {
		return err
	}

	conditionMet, ok := a.TopLevelStruct().(conditionMetter)

	if !ok {
		//This should be extremely rare since we ourselves have the right method.
		return errors.New("ApplyUntil top level struct unexpectedly did not have ConditionMet method")
	}

	if err := conditionMet.ConditionMet(state); err != nil {
		return errors.New("The condition was not yet met: " + err.Error())
	}

	return nil
}
<commit_msg>Fixed an error where ApplyUntil's Legal was backwards. Part of #516.<commit_after>package moves

import (
	"errors"
	"github.com/jkomoros/boardgame"
)

//ApplyUntil is a simple move that is legal to apply in succession until its
//ConditionMet returns nil. You need to override ConditionMet as well as
//provide an Apply method.
type ApplyUntil struct {
	Base
}

type conditionMetter interface {
	ConditionMet(state boardgame.State) error
}

//AllowMultipleInProgression returns true because the move is applied until
//ConditionMet returns nil.
func (a *ApplyUntil) AllowMultipleInProgression() bool {
	return true
}

//ConditionMet is called in ApplyUntil's Legal method. If the condition has
//been met, return nil. If it has not been met, return an error describing why
//it is not yet met. The default ConditionMet returns nil always; you almost
//certainly want to override it.
func (a *ApplyUntil) ConditionMet(state boardgame.State) error {
	return nil
}

func (a *ApplyUntil) ValidConfiguration(exampleState boardgame.MutableState) error {
	if _, ok := a.TopLevelStruct().(conditionMetter); !ok {
		return errors.New("Embedding Move doesn't have ConditionMet")
	}
	return nil
}

//Legal returns an error until ConditionMet returns nil.
func (a *ApplyUntil) Legal(state boardgame.State, proposer boardgame.PlayerIndex) error {
	if err := a.Base.Legal(state, proposer); err != nil {
		return err
	}

	conditionMet, ok := a.TopLevelStruct().(conditionMetter)

	if !ok {
		//This should be extremely rare since we ourselves have the right method.
		return errors.New("ApplyUntil top level struct unexpectedly did not have ConditionMet method")
	}

	if err := conditionMet.ConditionMet(state); err != nil {
		//The condition is not yet met, which means it's legal.
		return nil
	}

	return errors.New("The condition was met, so the move is no longer legal.")
}
<|endoftext|>


<commit_before>package diff

import (
	"fmt"
	"imposm3/cache"
	"imposm3/config"
	"imposm3/database"
	_ "imposm3/database/postgis"
	"imposm3/diff/parser"
	diffstate "imposm3/diff/state"
	"imposm3/element"
	"imposm3/expire"
	"imposm3/geom/geos"
	"imposm3/geom/limit"
	"imposm3/logging"
	"imposm3/mapping"
	"imposm3/stats"
	"imposm3/writer"
	"io"
)

var log = logging.NewLogger("diff")

func Update(oscFile string, geometryLimiter *limit.Limiter, force bool) {
	state, err := diffstate.ParseFromOsc(oscFile)
	if err != nil {
		log.Fatal(err)
	}
	lastState, err := diffstate.ParseLastState(config.BaseOptions.CacheDir)
	if err != nil {
		log.Fatal(err)
	}

	if lastState != nil && lastState.Sequence != 0 && state != nil && state.Sequence <= lastState.Sequence {
		if !force {
			log.Warn(state, " already imported")
			return
		}
	}

	defer log.StopStep(log.StartStep(fmt.Sprintf("Processing %s", oscFile)))

	elems, errc := parser.Parse(oscFile)

	osmCache := cache.NewOSMCache(config.BaseOptions.CacheDir)
	err = osmCache.Open()
	if err != nil {
		log.Fatal("osm cache: ", err)
	}

	diffCache := cache.NewDiffCache(config.BaseOptions.CacheDir)
	err = diffCache.Open()
	if err != nil {
		log.Fatal("diff cache: ", err)
	}

	tagmapping, err := mapping.NewMapping(config.BaseOptions.MappingFile)
	if err != nil {
		log.Fatal(err)
	}

	dbConf := database.Config{
		ConnectionParams: config.BaseOptions.Connection,
		Srid:             config.BaseOptions.Srid,
	}
	db, err := database.Open(dbConf, tagmapping)
	if err != nil {
		log.Fatal("database open: ", err)
	}

	err = db.Begin()
	if err != nil {
		log.Fatal(err)
	}

	delDb, ok := db.(database.Deleter)
	if !ok {
		log.Fatal("database not deletable")
	}

	genDb, ok := db.(database.Generalizer)
	if ok {
		genDb.EnableGeneralizeUpdates()
	}

	deleter := NewDeleter(
		delDb,
		osmCache,
		diffCache,
		tagmapping.PointMatcher(),
		tagmapping.LineStringMatcher(),
		tagmapping.PolygonMatcher(),
	)

	progress := stats.NewStatsReporter()

	expiredTiles := expire.NewTiles(14)

	relTagFilter := tagmapping.RelationTagFilter()
	wayTagFilter := tagmapping.WayTagFilter()
	nodeTagFilter := tagmapping.NodeTagFilter()

	relations := make(chan *element.Relation)
	ways := make(chan *element.Way)
	nodes := make(chan *element.Node)

	relWriter := writer.NewRelationWriter(osmCache, diffCache, relations,
		db, progress, config.BaseOptions.Srid)
	relWriter.SetLimiter(geometryLimiter)
	relWriter.SetExpireTiles(expiredTiles)
	relWriter.Start()

	wayWriter := writer.NewWayWriter(osmCache, diffCache, ways, db,
		progress, config.BaseOptions.Srid)
	wayWriter.SetLimiter(geometryLimiter)
	wayWriter.SetExpireTiles(expiredTiles)
	wayWriter.Start()

	nodeWriter := writer.NewNodeWriter(osmCache, nodes, db,
		progress, config.BaseOptions.Srid)
	nodeWriter.SetLimiter(geometryLimiter)
	nodeWriter.Start()

	nodeIds := make(map[int64]bool)
	wayIds := make(map[int64]bool)
	relIds := make(map[int64]bool)

	step := log.StartStep("Parsing changes, updating cache and removing elements")

	g := geos.NewGeos()
For:
	for {
		select {
		case elem := <-elems:
			if elem.Rel != nil {
				relTagFilter.Filter(&elem.Rel.Tags)
				progress.AddRelations(1)
			} else if elem.Way != nil {
				wayTagFilter.Filter(&elem.Way.Tags)
				progress.AddWays(1)
			} else if elem.Node != nil {
				nodeTagFilter.Filter(&elem.Node.Tags)
				if len(elem.Node.Tags) > 0 {
					progress.AddNodes(1)
				}
				progress.AddCoords(1)
			}
			if elem.Del {
				deleter.Delete(elem)
				if !elem.Add {
					if elem.Rel != nil {
						if err := osmCache.Relations.DeleteRelation(elem.Rel.Id); err != nil {
							log.Fatal(err)
						}
					} else if elem.Way != nil {
						if err := osmCache.Ways.DeleteWay(elem.Way.Id); err != nil {
							log.Fatal(err)
						}
						diffCache.Ways.Delete(elem.Way.Id)
					} else if elem.Node != nil {
						if err := osmCache.Nodes.DeleteNode(elem.Node.Id); err != nil {
							log.Fatal(err)
						}
						if err := osmCache.Coords.DeleteCoord(elem.Node.Id); err != nil {
							log.Fatal(err)
						}
					}
				}
			}
			if elem.Add {
				if elem.Rel != nil {
					// check if first member is cached to avoid caching
					// unneeded relations (typical outside of our coverage)
					if memberIsCached(elem.Rel.Members, osmCache.Ways) {
						osmCache.Relations.PutRelation(elem.Rel)
						relIds[elem.Rel.Id] = true
					}
				} else if elem.Way != nil {
					// check if first coord is cached to avoid caching
					// unneeded ways (typical outside of our coverage)
					if coordIsCached(elem.Way.Refs, osmCache.Coords) {
						osmCache.Ways.PutWay(elem.Way)
						wayIds[elem.Way.Id] = true
					}
				} else if elem.Node != nil {
					if geometryLimiter == nil || geometryLimiter.IntersectsBuffer(g, elem.Node.Long, elem.Node.Lat) {
						osmCache.Nodes.PutNode(elem.Node)
						osmCache.Coords.PutCoords([]element.Node{*elem.Node})
						nodeIds[elem.Node.Id] = true
					}
				}
			}
		case err := <-errc:
			if err != io.EOF {
				log.Fatal(err)
			}
			break For
		}
	}
	progress.Stop()
	log.StopStep(step)
	step = log.StartStep("Writing added/modified elements")

	progress = stats.NewStatsReporter()

	for nodeId, _ := range nodeIds {
		node, err := osmCache.Nodes.GetNode(nodeId)
		if err != nil {
			if err != cache.NotFound {
				log.Print(node, err)
			}
			// missing nodes can still be Coords
			// no `continue` here
		}
		if node != nil {
			// insert new node
			nodes <- node
		}
		dependers := diffCache.Coords.Get(nodeId)
		// mark depending ways for (re)insert
		for _, way := range dependers {
			wayIds[way] = true
		}
	}

	for wayId, _ := range wayIds {
		way, err := osmCache.Ways.GetWay(wayId)
		if err != nil {
			if err != cache.NotFound {
				log.Print(way, err)
			}
			continue
		}
		// insert new way
		ways <- way
		dependers := diffCache.Ways.Get(wayId)
		// mark depending relations for (re)insert
		for _, rel := range dependers {
			relIds[rel] = true
		}
	}

	for relId, _ := range relIds {
		rel, err := osmCache.Relations.GetRelation(relId)
		if err != nil {
			if err != cache.NotFound {
				log.Print(rel, err)
			}
			continue
		}
		// insert new relation
		relations <- rel
	}

	close(relations)
	close(ways)
	close(nodes)

	nodeWriter.Wait()
	relWriter.Wait()
	wayWriter.Wait()

	if genDb != nil {
		genDb.GeneralizeUpdates()
	}

	err = db.End()
	if err != nil {
		log.Fatal(err)
	}
	err = db.Close()
	if err != nil {
		log.Fatal(err)
	}

	osmCache.Close()
	diffCache.Close()
	log.StopStep(step)

	step = log.StartStep("Updating expired tiles db")
	expire.WriteTileExpireDb(
		expiredTiles.SortedTiles(),
		"/tmp/expire_tiles.db",
	)
	log.StopStep(step)
	progress.Stop()

	if state != nil {
		err = diffstate.WriteLastState(config.BaseOptions.CacheDir, state)
		if err != nil {
			log.Warn(err) // warn only
		}
	}
}

func memberIsCached(members []element.Member, wayCache *cache.WaysCache) bool {
	for _, m := range members {
		if m.Type == element.WAY {
			_, err := wayCache.GetWay(m.Id)
			if err != nil {
				return false
			}
			return true
		}
	}
	return false
}

func coordIsCached(refs []int64, coordCache *cache.DeltaCoordsCache) bool {
	if len(refs) <= 0 {
		return false
	}
	_, err := coordCache.GetCoord(refs[0])
	if err != nil {
		return false
	}
	return true
}
<commit_msg>log progress during writing of added/modified elements<commit_after>package diff

import (
	"fmt"
	"imposm3/cache"
	"imposm3/config"
	"imposm3/database"
	_ "imposm3/database/postgis"
	"imposm3/diff/parser"
	diffstate "imposm3/diff/state"
	"imposm3/element"
	"imposm3/expire"
	"imposm3/geom/geos"
	"imposm3/geom/limit"
	"imposm3/logging"
	"imposm3/mapping"
	"imposm3/stats"
	"imposm3/writer"
	"io"
)

var log = logging.NewLogger("diff")

func Update(oscFile string, geometryLimiter *limit.Limiter, force bool) {
	state, err := diffstate.ParseFromOsc(oscFile)
	if err != nil {
		log.Fatal(err)
	}
	lastState, err := diffstate.ParseLastState(config.BaseOptions.CacheDir)
	if err != nil {
		log.Fatal(err)
	}

	if lastState != nil && lastState.Sequence != 0 && state != nil && state.Sequence <= lastState.Sequence {
		if !force {
			log.Warn(state, " already imported")
			return
		}
	}

	defer log.StopStep(log.StartStep(fmt.Sprintf("Processing %s", oscFile)))

	elems, errc := parser.Parse(oscFile)

	osmCache := cache.NewOSMCache(config.BaseOptions.CacheDir)
	err = osmCache.Open()
	if err != nil {
		log.Fatal("osm cache: ", err)
	}

	diffCache := cache.NewDiffCache(config.BaseOptions.CacheDir)
	err = diffCache.Open()
	if err != nil {
		log.Fatal("diff cache: ", err)
	}

	tagmapping, err := mapping.NewMapping(config.BaseOptions.MappingFile)
	if err != nil {
		log.Fatal(err)
	}

	dbConf := database.Config{
		ConnectionParams: config.BaseOptions.Connection,
		Srid:             config.BaseOptions.Srid,
	}
	db, err := database.Open(dbConf, tagmapping)
	if err != nil {
		log.Fatal("database open: ", err)
	}

	err = db.Begin()
	if err != nil {
		log.Fatal(err)
	}

	delDb, ok := db.(database.Deleter)
	if !ok {
		log.Fatal("database not deletable")
	}

	genDb, ok := db.(database.Generalizer)
	if ok {
		genDb.EnableGeneralizeUpdates()
	}

	deleter := NewDeleter(
		delDb,
		osmCache,
		diffCache,
		tagmapping.PointMatcher(),
		tagmapping.LineStringMatcher(),
		tagmapping.PolygonMatcher(),
	)

	progress := stats.NewStatsReporter()

	expiredTiles := expire.NewTiles(14)

	relTagFilter := tagmapping.RelationTagFilter()
	wayTagFilter := tagmapping.WayTagFilter()
	nodeTagFilter := tagmapping.NodeTagFilter()

	relations := make(chan *element.Relation)
	ways := make(chan *element.Way)
	nodes := make(chan *element.Node)

	relWriter := writer.NewRelationWriter(osmCache, diffCache, relations,
		db, progress, config.BaseOptions.Srid)
	relWriter.SetLimiter(geometryLimiter)
	relWriter.SetExpireTiles(expiredTiles)
	relWriter.Start()

	wayWriter := writer.NewWayWriter(osmCache, diffCache, ways, db,
		progress, config.BaseOptions.Srid)
	wayWriter.SetLimiter(geometryLimiter)
	wayWriter.SetExpireTiles(expiredTiles)
	wayWriter.Start()

	nodeWriter := writer.NewNodeWriter(osmCache, nodes, db,
		progress, config.BaseOptions.Srid)
	nodeWriter.SetLimiter(geometryLimiter)
	nodeWriter.Start()

	nodeIds := make(map[int64]bool)
	wayIds := make(map[int64]bool)
	relIds := make(map[int64]bool)

	step := log.StartStep("Parsing changes, updating cache and removing elements")

	g := geos.NewGeos()
For:
	for {
		select {
		case elem := <-elems:
			if elem.Rel != nil {
				relTagFilter.Filter(&elem.Rel.Tags)
				progress.AddRelations(1)
			} else if elem.Way != nil {
				wayTagFilter.Filter(&elem.Way.Tags)
				progress.AddWays(1)
			} else if elem.Node != nil {
				nodeTagFilter.Filter(&elem.Node.Tags)
				if len(elem.Node.Tags) > 0 {
					progress.AddNodes(1)
				}
				progress.AddCoords(1)
			}
			if elem.Del {
				deleter.Delete(elem)
				if !elem.Add {
					if elem.Rel != nil {
						if err := osmCache.Relations.DeleteRelation(elem.Rel.Id); err != nil {
							log.Fatal(err)
						}
					} else if elem.Way != nil {
						if err := osmCache.Ways.DeleteWay(elem.Way.Id); err != nil {
							log.Fatal(err)
						}
						diffCache.Ways.Delete(elem.Way.Id)
					} else if elem.Node != nil {
						if err := osmCache.Nodes.DeleteNode(elem.Node.Id); err != nil {
							log.Fatal(err)
						}
						if err := osmCache.Coords.DeleteCoord(elem.Node.Id); err != nil {
							log.Fatal(err)
						}
					}
				}
			}
			if elem.Add {
				if elem.Rel != nil {
					// check if first member is cached to avoid caching
					// unneeded relations (typical outside of our coverage)
					if memberIsCached(elem.Rel.Members, osmCache.Ways) {
						osmCache.Relations.PutRelation(elem.Rel)
						relIds[elem.Rel.Id] = true
					}
				} else if elem.Way != nil {
					// check if first coord is cached to avoid caching
					// unneeded ways (typical outside of our coverage)
					if coordIsCached(elem.Way.Refs, osmCache.Coords) {
						osmCache.Ways.PutWay(elem.Way)
						wayIds[elem.Way.Id] = true
					}
				} else if elem.Node != nil {
					if geometryLimiter == nil || geometryLimiter.IntersectsBuffer(g, elem.Node.Long, elem.Node.Lat) {
						osmCache.Nodes.PutNode(elem.Node)
						osmCache.Coords.PutCoords([]element.Node{*elem.Node})
						nodeIds[elem.Node.Id] = true
					}
				}
			}
		case err := <-errc:
			if err != io.EOF {
				log.Fatal(err)
			}
			break For
		}
	}
	progress.Stop()
	log.StopStep(step)
	step = log.StartStep("Writing added/modified elements")

	progress = stats.NewStatsReporter()

	for nodeId, _ := range nodeIds {
		node, err := osmCache.Nodes.GetNode(nodeId)
		if err != nil {
			if err != cache.NotFound {
				log.Print(node, err)
			}
			// missing nodes can still be Coords
			// no `continue` here
		}
		if node != nil {
			// insert new node
			progress.AddNodes(1)
			nodes <- node
		}
		dependers := diffCache.Coords.Get(nodeId)
		// mark depending ways for (re)insert
		for _, way := range dependers {
			wayIds[way] = true
		}
	}

	for wayId, _ := range wayIds {
		way, err := osmCache.Ways.GetWay(wayId)
		if err != nil {
			if err != cache.NotFound {
				log.Print(way, err)
			}
			continue
		}
		// insert new way
		progress.AddWays(1)
		ways <- way
		dependers := diffCache.Ways.Get(wayId)
		// mark depending relations for (re)insert
		for _, rel := range dependers {
			relIds[rel] = true
		}
	}

	for relId, _ := range relIds {
		rel, err := osmCache.Relations.GetRelation(relId)
		if err != nil {
			if err != cache.NotFound {
				log.Print(rel, err)
			}
			continue
		}
		// insert new relation
		progress.AddRelations(1)
		relations <- rel
	}

	close(relations)
	close(ways)
	close(nodes)

	nodeWriter.Wait()
	relWriter.Wait()
	wayWriter.Wait()

	if genDb != nil {
		genDb.GeneralizeUpdates()
	}

	err = db.End()
	if err != nil {
		log.Fatal(err)
	}
	err = db.Close()
	if err != nil {
		log.Fatal(err)
	}

	osmCache.Close()
	diffCache.Close()
	log.StopStep(step)

	step = log.StartStep("Updating expired tiles db")
	expire.WriteTileExpireDb(
		expiredTiles.SortedTiles(),
		"/tmp/expire_tiles.db",
	)
	log.StopStep(step)
	progress.Stop()

	if state != nil {
		err = diffstate.WriteLastState(config.BaseOptions.CacheDir, state)
		if err != nil {
			log.Warn(err) // warn only
		}
	}
}

func memberIsCached(members []element.Member, wayCache *cache.WaysCache) bool {
	for _, m := range members {
		if m.Type == element.WAY {
			_, err := wayCache.GetWay(m.Id)
			if err != nil {
				return false
			}
			return true
		}
	}
	return false
}

func coordIsCached(refs []int64, coordCache *cache.DeltaCoordsCache) bool {
	if len(refs) <= 0 {
		return false
	}
	_, err := coordCache.GetCoord(refs[0])
	if err != nil {
		return false
	}
	return true
}
<|endoftext|>


<commit_before>package nlp

import (
	"fmt"

	"github.com/gonum/matrix"
	"github.com/gonum/matrix/mat64"
)

// TruncatedSVD implements the Singular Value Decomposition factorisation of matrices.
// This produces an approximation of the input matrix at a lower rank.  This is a core
// component of LSA (Latent Semantic Analsis)
type TruncatedSVD struct {
	transform *mat64.Dense

	// K is the number of dimensions to which the output, transformed, matrix should be
	// truncated to.  The matrix output by the FitTransform() and Transform() methods will
	// be n rows by min(m, n, K) columns, where n is the number of columns in the original,
	// input matrix and min(m, n, K) is the lowest value of m, n, K where m is the number of
	// rows in the original, input matrix.
	K int
}

// NewTruncatedSVD creates a new TruncatedSVD transformer with K (the truncated
// dimensionality) being set to the specified value k
func NewTruncatedSVD(k int) *TruncatedSVD {
	return &TruncatedSVD{K: k}
}

// Fit performs the SVD factorisation on the input training data matrix, mat and
// stores the output term matrix as a transform to apply to matrices in the Transform matrix.
func (t *TruncatedSVD) Fit(mat mat64.Matrix) Transformer {
	t.FitTransform(mat)
	return t
}

// Transform applies the transform decomposed from the training data matrix in Fit()
// to the input matrix.  The resulting output matrix will be the closest approximation
// to the input matrix at a reduced rank.
func (t *TruncatedSVD) Transform(mat mat64.Matrix) (*mat64.Dense, error) {
	var product mat64.Dense

	product.Product(t.transform.T(), mat)

	return &product, nil
}

// FitTransform is approximately equivalent to calling Fit() followed by Transform() on the // same matrix.  This is a useful shortcut where separate trianing data is not being
// used to fit the model i.e. the model is fitted on the fly to the test data.
func (t *TruncatedSVD) FitTransform(mat mat64.Matrix) (*mat64.Dense, error) {
	var svd mat64.SVD
	if ok := svd.Factorize(mat, matrix.SVDThin); !ok {
		return nil, fmt.Errorf("Failed SVD Factorisation of working matrix")
	}
	s, u, v := t.extractSVD(&svd)

	m, n := mat.Dims()
	min := minimum(t.K, m, n)

	// truncate matrix to k << min(m, n)
	uk, ok := u.Slice(0, m, 0, min).(*mat64.Dense)
	if !ok {
		return nil, fmt.Errorf("Failed to truncate U")
	}

	vk, ok := v.Slice(0, n, 0, min).(*mat64.Dense)
	if !ok {
		return nil, fmt.Errorf("Failed to truncate V")
	}

	// only build out eigenvalue matrix to k x k (truncate values) (or min(m, n) if lower)
	sigmak := mat64.NewDense(min, min, nil)
	for i := 0; i < min; i++ {
		sigmak.Set(i, i, s[i])
	}

	t.transform = uk

	var product mat64.Dense
	product.Product(sigmak, vk.T())

	return &product, nil
}

func minimum(k, m, n int) int {
	return min(k, min(m, n))
}

func min(m, n int) int {
	if m < n {
		return m
	}
	return n
}

func (t *TruncatedSVD) extractSVD(svd *mat64.SVD) (s []float64, u, v *mat64.Dense) {
	var um, vm mat64.Dense
	um.UFromSVD(svd)
	vm.VFromSVD(svd)
	s = svd.Values(nil)
	return s, &um, &vm
}
<commit_msg>optimised Sigma x V.T() matrix multiplication to simply multiply diagonal values from Sigma with V.T() elements<commit_after>package nlp

import (
	"fmt"

	"github.com/gonum/matrix"
	"github.com/gonum/matrix/mat64"
)

// TruncatedSVD implements the Singular Value Decomposition factorisation of matrices.
// This produces an approximation of the input matrix at a lower rank.  This is a core
// component of LSA (Latent Semantic Analsis)
type TruncatedSVD struct {
	transform *mat64.Dense

	// K is the number of dimensions to which the output, transformed, matrix should be
	// truncated to.  The matrix output by the FitTransform() and Transform() methods will
	// be n rows by min(m, n, K) columns, where n is the number of columns in the original,
	// input matrix and min(m, n, K) is the lowest value of m, n, K where m is the number of
	// rows in the original, input matrix.
	K int
}

// NewTruncatedSVD creates a new TruncatedSVD transformer with K (the truncated
// dimensionality) being set to the specified value k
func NewTruncatedSVD(k int) *TruncatedSVD {
	return &TruncatedSVD{K: k}
}

// Fit performs the SVD factorisation on the input training data matrix, mat and
// stores the output term matrix as a transform to apply to matrices in the Transform matrix.
func (t *TruncatedSVD) Fit(mat mat64.Matrix) Transformer {
	t.FitTransform(mat)
	return t
}

// Transform applies the transform decomposed from the training data matrix in Fit()
// to the input matrix.  The resulting output matrix will be the closest approximation
// to the input matrix at a reduced rank.
func (t *TruncatedSVD) Transform(mat mat64.Matrix) (*mat64.Dense, error) {
	var product mat64.Dense

	product.Product(t.transform.T(), mat)

	return &product, nil
}

// FitTransform is approximately equivalent to calling Fit() followed by Transform() on the // same matrix.  This is a useful shortcut where separate trianing data is not being
// used to fit the model i.e. the model is fitted on the fly to the test data.
func (t *TruncatedSVD) FitTransform(mat mat64.Matrix) (*mat64.Dense, error) {
	var svd mat64.SVD
	if ok := svd.Factorize(mat, matrix.SVDThin); !ok {
		return nil, fmt.Errorf("Failed SVD Factorisation of working matrix")
	}
	s, u, v := t.extractSVD(&svd)

	m, n := mat.Dims()
	min := minimum(t.K, m, n)

	// truncate matrix to k << min(m, n)
	uk, ok := u.Slice(0, m, 0, min).(*mat64.Dense)
	if !ok {
		return nil, fmt.Errorf("Failed to truncate U")
	}

	vk, ok := v.Slice(0, n, 0, min).(*mat64.Dense)
	if !ok {
		return nil, fmt.Errorf("Failed to truncate V")
	}

	t.transform = uk

	// multiply Sigma by transpose of V.  As sigma is a symmetrical (square) diagonal matrix it is
	// more efficient to simply multiply each element from the array of diagonal values with each
	// element from the matrix V rather than multiplying out the non-zero values from off the diagonal.
	var product mat64.Dense
	product.Apply(func(i, j int, v float64) float64 {
		return (v * s[i])
	}, vk.T())

	return &product, nil
}

func minimum(k, m, n int) int {
	return min(k, min(m, n))
}

func min(m, n int) int {
	if m < n {
		return m
	}
	return n
}

func (t *TruncatedSVD) extractSVD(svd *mat64.SVD) (s []float64, u, v *mat64.Dense) {
	var um, vm mat64.Dense
	um.UFromSVD(svd)
	vm.VFromSVD(svd)
	s = svd.Values(nil)
	return s, &um, &vm
}
<|endoftext|>


<commit_before>package graval

import (
	"bufio"
	"bytes"
	"crypto/rand"
	"crypto/sha256"
	"encoding/hex"
	"fmt"
	"io"
	"net"
	"path/filepath"
	"strings"
	"time"
)

type ftpConn struct {
	conn          net.Conn
	controlReader    *bufio.Reader
	controlWriter    *bufio.Writer
	dataConn         ftpDataSocket
	driver           FTPDriver
	logger           *ftpLogger
	serverName       string
	sessionId        string
	namePrefix       string
	reqUser          string
	user             string
	renameFrom       string
	minDataPort      int
	maxDataPort      int
	pasvAdvertisedIp string
}

// NewftpConn constructs a new object that will handle the FTP protocol over
// an active net.TCPConn. The TCP connection should already be open before
// it is handed to this functions. driver is an instance of FTPDriver that
// will handle all auth and persistence details.
func newftpConn(tcpConn net.Conn, driver FTPDriver, serverName string, minPort int, maxPort int, pasvAdvertisedIp string) *ftpConn {
	c := new(ftpConn)
	c.namePrefix = "/"
	c.conn = tcpConn
	c.controlReader = bufio.NewReader(tcpConn)
	c.controlWriter = bufio.NewWriter(tcpConn)
	c.driver = driver
	c.sessionId = newSessionId()
	c.logger = newFtpLogger(c.sessionId)
	c.serverName = serverName
	c.minDataPort = minPort
	c.maxDataPort = maxPort
	c.pasvAdvertisedIp = pasvAdvertisedIp
	return c
}

// returns a random 20 char string that can be used as a unique session ID
func newSessionId() string {
	hash := sha256.New()
	_, err := io.CopyN(hash, rand.Reader, 50)
	if err != nil {
		return "????????????????????"
	}
	md := hash.Sum(nil)
	mdStr := hex.EncodeToString(md)
	return mdStr[0:20]
}

// Serve starts an endless loop that reads FTP commands from the client and
// responds appropriately. terminated is a channel that will receive a true
// message when the connection closes. This loop will be running inside a
// goroutine, so use this channel to be notified when the connection can be
// cleaned up.
func (ftpConn *ftpConn) Serve() {
	defer func() {
		if r := recover(); r != nil {
			ftpConn.logger.Printf("Recovered in ftpConn Serve: %s", r)
		}

		ftpConn.Close()
	}()

	ftpConn.logger.Printf("Connection Established (local: %s, remote: %s)", ftpConn.localIP(), ftpConn.remoteIP())
	// send welcome
	ftpConn.writeMessage(220, ftpConn.serverName)
	// read commands
	for {
		line, err := ftpConn.controlReader.ReadString('\n')
		if err != nil {
			break
		}
		ftpConn.receiveLine(line)
	}
	ftpConn.logger.Print("Connection Terminated")
}

// Close will manually close this connection, even if the client isn't ready.
func (ftpConn *ftpConn) Close() {
	ftpConn.conn.Close()
	if ftpConn.dataConn != nil {
		ftpConn.dataConn.Close()
	}
}

// receiveLine accepts a single line FTP command and co-ordinates an
// appropriate response.
func (ftpConn *ftpConn) receiveLine(line string) {
	command, param := ftpConn.parseLine(line)
	ftpConn.logger.PrintCommand(command, param)
	cmdObj := commands[command]
	if cmdObj == nil {
		ftpConn.writeMessage(500, "Command not found")
		return
	}
	if cmdObj.RequireParam() && param == "" {
		ftpConn.writeMessage(553, "action aborted, required param missing")
	} else if cmdObj.RequireAuth() && ftpConn.user == "" {
		ftpConn.writeMessage(530, "not logged in")
	} else {
		cmdObj.Execute(ftpConn, param)
	}
}

func (ftpConn *ftpConn) parseLine(line string) (string, string) {
	params := strings.SplitN(strings.Trim(line, "\r\n"), " ", 2)
	if len(params) == 1 {
		return params[0], ""
	}
	return params[0], strings.TrimSpace(params[1])
}

// writeMessage will send a standard FTP response back to the client.
func (ftpConn *ftpConn) writeMessage(code int, message string) (wrote int, err error) {
	ftpConn.logger.PrintResponse(code, message)
	line := fmt.Sprintf("%d %s\r\n", code, message)
	wrote, err = ftpConn.controlWriter.WriteString(line)
	ftpConn.controlWriter.Flush()
	return
}

// writeLines will send a multiline FTP response back to the client.
func (ftpConn *ftpConn) writeLines(code int, lines ...string) (wrote int, err error) {
	message := strings.Join(lines, "\r\n") + "\r\n"
	ftpConn.logger.PrintResponse(code, message)
	wrote, err = ftpConn.controlWriter.WriteString(message)
	ftpConn.controlWriter.Flush()
	return
}

// buildPath takes a client supplied path or filename and generates a safe
// absolute path within their account sandbox.
//
//    buildpath("/")
//    => "/"
//    buildpath("one.txt")
//    => "/one.txt"
//    buildpath("/files/two.txt")
//    => "/files/two.txt"
//    buildpath("files/two.txt")
//    => "files/two.txt"
//    buildpath("/../../../../etc/passwd")
//    => "/etc/passwd"
//
// The driver implementation is responsible for deciding how to treat this path.
// Obviously they MUST NOT just read the path off disk. The probably want to
// prefix the path with something to scope the users access to a sandbox.
func (ftpConn *ftpConn) buildPath(filename string) (fullPath string) {
	if len(filename) > 0 && filename[0:1] == "/" {
		fullPath = filepath.Clean(filename)
	} else if len(filename) > 0 {
		fullPath = filepath.Clean(ftpConn.namePrefix + "/" + filename)
	} else {
		fullPath = filepath.Clean(ftpConn.namePrefix)
	}
	fullPath = strings.Replace(fullPath, "//", "/", -1)
	return
}

// the server IP that is being used for this connection. May be the same for all connections,
// or may vary if the server is listening on 0.0.0.0
func (ftpConn *ftpConn) localIP() string {
	lAddr := ftpConn.conn.LocalAddr().(*net.TCPAddr)
	return lAddr.IP.String()
}

// the client IP address
func (ftpConn *ftpConn) remoteIP() string {
	rAddr := ftpConn.conn.RemoteAddr().(*net.TCPAddr)
	return rAddr.IP.String()
}

// sendOutofbandData will copy data from reader to the client via the currently
// open data socket. Assumes the socket is open and ready to be used.
func (ftpConn *ftpConn) sendOutofbandReader(reader io.Reader) {
	defer ftpConn.dataConn.Close()

	_, err := io.Copy(ftpConn.dataConn, reader)

	if err != nil {
		ftpConn.logger.Printf("sendOutofbandReader copy error %s", err)
		ftpConn.writeMessage(550, "Action not taken")
		return
	}

	ftpConn.writeMessage(226, "Transfer complete.")

	// Chrome dies on localhost if we close connection to soon
	time.Sleep(10 * time.Millisecond)
}

// sendOutofbandData will send a string to the client via the currently open
// data socket. Assumes the socket is open and ready to be used.
func (ftpConn *ftpConn) sendOutofbandData(data string) {
	ftpConn.sendOutofbandReader(bytes.NewReader([]byte(data)))
}

func (ftpConn *ftpConn) newPassiveSocket() (socket *ftpPassiveSocket, err error) {
	if ftpConn.dataConn != nil {
		ftpConn.dataConn.Close()
		ftpConn.dataConn = nil
	}

	socket, err = newPassiveSocket(ftpConn.localIP(), ftpConn.minDataPort, ftpConn.maxDataPort, ftpConn.logger)

	if err == nil {
		ftpConn.dataConn = socket
	}

	return
}

func (ftpConn *ftpConn) newActiveSocket(host string, port int) (socket *ftpActiveSocket, err error) {
	if ftpConn.dataConn != nil {
		ftpConn.dataConn.Close()
		ftpConn.dataConn = nil
	}

	socket, err = newActiveSocket(host, port, ftpConn.logger)

	if err == nil {
		ftpConn.dataConn = socket
	}

	return
}
<commit_msg>go fmt<commit_after>package graval

import (
	"bufio"
	"bytes"
	"crypto/rand"
	"crypto/sha256"
	"encoding/hex"
	"fmt"
	"io"
	"net"
	"path/filepath"
	"strings"
	"time"
)

type ftpConn struct {
	conn             net.Conn
	controlReader    *bufio.Reader
	controlWriter    *bufio.Writer
	dataConn         ftpDataSocket
	driver           FTPDriver
	logger           *ftpLogger
	serverName       string
	sessionId        string
	namePrefix       string
	reqUser          string
	user             string
	renameFrom       string
	minDataPort      int
	maxDataPort      int
	pasvAdvertisedIp string
}

// NewftpConn constructs a new object that will handle the FTP protocol over
// an active net.TCPConn. The TCP connection should already be open before
// it is handed to this functions. driver is an instance of FTPDriver that
// will handle all auth and persistence details.
func newftpConn(tcpConn net.Conn, driver FTPDriver, serverName string, minPort int, maxPort int, pasvAdvertisedIp string) *ftpConn {
	c := new(ftpConn)
	c.namePrefix = "/"
	c.conn = tcpConn
	c.controlReader = bufio.NewReader(tcpConn)
	c.controlWriter = bufio.NewWriter(tcpConn)
	c.driver = driver
	c.sessionId = newSessionId()
	c.logger = newFtpLogger(c.sessionId)
	c.serverName = serverName
	c.minDataPort = minPort
	c.maxDataPort = maxPort
	c.pasvAdvertisedIp = pasvAdvertisedIp
	return c
}

// returns a random 20 char string that can be used as a unique session ID
func newSessionId() string {
	hash := sha256.New()
	_, err := io.CopyN(hash, rand.Reader, 50)
	if err != nil {
		return "????????????????????"
	}
	md := hash.Sum(nil)
	mdStr := hex.EncodeToString(md)
	return mdStr[0:20]
}

// Serve starts an endless loop that reads FTP commands from the client and
// responds appropriately. terminated is a channel that will receive a true
// message when the connection closes. This loop will be running inside a
// goroutine, so use this channel to be notified when the connection can be
// cleaned up.
func (ftpConn *ftpConn) Serve() {
	defer func() {
		if r := recover(); r != nil {
			ftpConn.logger.Printf("Recovered in ftpConn Serve: %s", r)
		}

		ftpConn.Close()
	}()

	ftpConn.logger.Printf("Connection Established (local: %s, remote: %s)", ftpConn.localIP(), ftpConn.remoteIP())
	// send welcome
	ftpConn.writeMessage(220, ftpConn.serverName)
	// read commands
	for {
		line, err := ftpConn.controlReader.ReadString('\n')
		if err != nil {
			break
		}
		ftpConn.receiveLine(line)
	}
	ftpConn.logger.Print("Connection Terminated")
}

// Close will manually close this connection, even if the client isn't ready.
func (ftpConn *ftpConn) Close() {
	ftpConn.conn.Close()
	if ftpConn.dataConn != nil {
		ftpConn.dataConn.Close()
	}
}

// receiveLine accepts a single line FTP command and co-ordinates an
// appropriate response.
func (ftpConn *ftpConn) receiveLine(line string) {
	command, param := ftpConn.parseLine(line)
	ftpConn.logger.PrintCommand(command, param)
	cmdObj := commands[command]
	if cmdObj == nil {
		ftpConn.writeMessage(500, "Command not found")
		return
	}
	if cmdObj.RequireParam() && param == "" {
		ftpConn.writeMessage(553, "action aborted, required param missing")
	} else if cmdObj.RequireAuth() && ftpConn.user == "" {
		ftpConn.writeMessage(530, "not logged in")
	} else {
		cmdObj.Execute(ftpConn, param)
	}
}

func (ftpConn *ftpConn) parseLine(line string) (string, string) {
	params := strings.SplitN(strings.Trim(line, "\r\n"), " ", 2)
	if len(params) == 1 {
		return params[0], ""
	}
	return params[0], strings.TrimSpace(params[1])
}

// writeMessage will send a standard FTP response back to the client.
func (ftpConn *ftpConn) writeMessage(code int, message string) (wrote int, err error) {
	ftpConn.logger.PrintResponse(code, message)
	line := fmt.Sprintf("%d %s\r\n", code, message)
	wrote, err = ftpConn.controlWriter.WriteString(line)
	ftpConn.controlWriter.Flush()
	return
}

// writeLines will send a multiline FTP response back to the client.
func (ftpConn *ftpConn) writeLines(code int, lines ...string) (wrote int, err error) {
	message := strings.Join(lines, "\r\n") + "\r\n"
	ftpConn.logger.PrintResponse(code, message)
	wrote, err = ftpConn.controlWriter.WriteString(message)
	ftpConn.controlWriter.Flush()
	return
}

// buildPath takes a client supplied path or filename and generates a safe
// absolute path within their account sandbox.
//
//    buildpath("/")
//    => "/"
//    buildpath("one.txt")
//    => "/one.txt"
//    buildpath("/files/two.txt")
//    => "/files/two.txt"
//    buildpath("files/two.txt")
//    => "files/two.txt"
//    buildpath("/../../../../etc/passwd")
//    => "/etc/passwd"
//
// The driver implementation is responsible for deciding how to treat this path.
// Obviously they MUST NOT just read the path off disk. The probably want to
// prefix the path with something to scope the users access to a sandbox.
func (ftpConn *ftpConn) buildPath(filename string) (fullPath string) {
	if len(filename) > 0 && filename[0:1] == "/" {
		fullPath = filepath.Clean(filename)
	} else if len(filename) > 0 {
		fullPath = filepath.Clean(ftpConn.namePrefix + "/" + filename)
	} else {
		fullPath = filepath.Clean(ftpConn.namePrefix)
	}
	fullPath = strings.Replace(fullPath, "//", "/", -1)
	return
}

// the server IP that is being used for this connection. May be the same for all connections,
// or may vary if the server is listening on 0.0.0.0
func (ftpConn *ftpConn) localIP() string {
	lAddr := ftpConn.conn.LocalAddr().(*net.TCPAddr)
	return lAddr.IP.String()
}

// the client IP address
func (ftpConn *ftpConn) remoteIP() string {
	rAddr := ftpConn.conn.RemoteAddr().(*net.TCPAddr)
	return rAddr.IP.String()
}

// sendOutofbandData will copy data from reader to the client via the currently
// open data socket. Assumes the socket is open and ready to be used.
func (ftpConn *ftpConn) sendOutofbandReader(reader io.Reader) {
	defer ftpConn.dataConn.Close()

	_, err := io.Copy(ftpConn.dataConn, reader)

	if err != nil {
		ftpConn.logger.Printf("sendOutofbandReader copy error %s", err)
		ftpConn.writeMessage(550, "Action not taken")
		return
	}

	ftpConn.writeMessage(226, "Transfer complete.")

	// Chrome dies on localhost if we close connection to soon
	time.Sleep(10 * time.Millisecond)
}

// sendOutofbandData will send a string to the client via the currently open
// data socket. Assumes the socket is open and ready to be used.
func (ftpConn *ftpConn) sendOutofbandData(data string) {
	ftpConn.sendOutofbandReader(bytes.NewReader([]byte(data)))
}

func (ftpConn *ftpConn) newPassiveSocket() (socket *ftpPassiveSocket, err error) {
	if ftpConn.dataConn != nil {
		ftpConn.dataConn.Close()
		ftpConn.dataConn = nil
	}

	socket, err = newPassiveSocket(ftpConn.localIP(), ftpConn.minDataPort, ftpConn.maxDataPort, ftpConn.logger)

	if err == nil {
		ftpConn.dataConn = socket
	}

	return
}

func (ftpConn *ftpConn) newActiveSocket(host string, port int) (socket *ftpActiveSocket, err error) {
	if ftpConn.dataConn != nil {
		ftpConn.dataConn.Close()
		ftpConn.dataConn = nil
	}

	socket, err = newActiveSocket(host, port, ftpConn.logger)

	if err == nil {
		ftpConn.dataConn = socket
	}

	return
}
<|endoftext|>