<commit_before>package main

import (
	"bufio"
	"errors"
	"io"
	"log"
	"net"
	"net/http"
	"os"
	"runtime"
	"sync"
	"syscall"
	"time"
)

type Server struct {
	http.Server
	Addr       string // host:port listening ("" if not listening yet)
	Err        error  // error encountered while running server (nil if still listening)
	config     Config
	listener   *net.TCPListener
	shutdown   bool // shutdown was requested by Close()
	done       bool // server is no longer listening
	*sync.Cond      // can wait for done to become true
	sourceMap  *SourceMap
}

// FlushyResponseWriter wraps http.ResponseWriter, calling Flush()
// after each Write() to minimize buffering on our (server) end of the
// connection.
type FlushyResponseWriter struct {
	http.ResponseWriter
}

// Write sends data to the client. It panics if the underlying
// ResponseWriter does not implement http.Flusher.
func (writer *FlushyResponseWriter) Write(data []byte) (int, error) {
	defer func() {
		flusher, ok := writer.ResponseWriter.(http.Flusher)
		if !ok {
			log.Fatal("ResponseWriter is not a flusher")
		}
		flusher.Flush()
	}()
	return writer.ResponseWriter.Write(data)
}

// Run starts an HTTP server with the given configuration.
func (srv *Server) Run(c *Config) (err error) {
	// c.Check() is just for tests -- main() has already done this.
	if err = c.Check(); err != nil {
		log.Fatal(err)
	}
	defer runtime.GOMAXPROCS(runtime.GOMAXPROCS(c.CPUMax))
	addr, err := net.ResolveTCPAddr("tcp", c.Addr)
	if err != nil {
		return
	}
	srv.listener, err = net.ListenTCP("tcp", addr)
	if err != nil {
		return
	}
	if c.UID != os.Getuid() {
		err = syscall.Setuid(c.UID)
		if err != nil {
			err = errors.New("Setuid: " + err.Error())
			return
		}
	}
	srv.Addr = srv.listener.Addr().String()
	srv.sourceMap = NewSourceMap()
	mux := http.NewServeMux()
	mux.HandleFunc("/", func(writer http.ResponseWriter, req *http.Request) {
		writer.Header().Set("Content-Type", config.ContentType)
		startTime := time.Now()
		sreader := srv.sourceMap.NewReader(c.Path, c)
		fwriter := &FlushyResponseWriter{writer}
		wroteBytes, err := io.Copy(fwriter,
			bufio.NewReaderSize(sreader, int(c.FrameBytes)))
		if e, ok := err.(*net.OpError); ok {
			if e, ok := e.Err.(syscall.Errno); ok {
				if e == syscall.ECONNRESET {
					// Not really an error: client disconnected.
					err = nil
				}
			}
		}
		if err != nil {
			log.Printf("client %s error: %s", req.RemoteAddr, err)
		}
		log.Println("client", req.RemoteAddr,
			"--", time.Since(startTime).String(), "elapsed",
			wroteBytes, "bytes",
			sreader.FramesRead, "frames +",
			sreader.FramesSkipped, "skipped")
		sreader.Close()
		if srv.sourceMap.Count() == 0 && !c.Reopen {
			// The only source path has ended and can't be reopened.
			srv.Close()
		}
	})
	srv.Handler = mux
	mutex := &sync.RWMutex{}
	srv.Cond = sync.NewCond(mutex.RLocker())
	go func() {
		err = srv.Serve(tcpKeepAliveListener{srv.listener})
		if !srv.shutdown {
			srv.Err = err
		}
		mutex.Lock()
		srv.done = true
		srv.Cond.Broadcast()
		mutex.Unlock()
	}()
	return nil
}

// Wait returns when the server stops listening. If the server stops
// because Close() was called, it returns nil. If it stops for some
// other reason, it returns the error that caused it to stop.
func (srv *Server) Wait() error {
	srv.Cond.L.Lock()
	defer srv.Cond.L.Unlock()
	for !srv.done {
		log.Print("waiting")
		srv.Cond.Wait()
	}
	return srv.Err
}

// Close shuts down the server. It returns an error if the server
// already stopped for some reason other than a prior call to Close().
func (srv *Server) Close() error {
	srv.shutdown = true
	srv.listener.Close()
	srv.sourceMap.Close()
	return srv.Wait()
}

// Copied from net/http because not exported.
//
type tcpKeepAliveListener struct {
	*net.TCPListener
}

func (ln tcpKeepAliveListener) Accept() (c net.Conn, err error) {
	tc, err := ln.AcceptTCP()
	if err != nil {
		return
	}
	tc.SetKeepAlive(true)
	tc.SetKeepAlivePeriod(3 * time.Minute)
	return tc, nil
}
<commit_msg>Accept zero value as no-op for testing convenience.<commit_after>package main

import (
	"bufio"
	"errors"
	"io"
	"log"
	"net"
	"net/http"
	"os"
	"runtime"
	"sync"
	"syscall"
	"time"
)

type Server struct {
	http.Server
	Addr       string // host:port listening ("" if not listening yet)
	Err        error  // error encountered while running server (nil if still listening)
	config     Config
	listener   *net.TCPListener
	shutdown   bool // shutdown was requested by Close()
	done       bool // server is no longer listening
	*sync.Cond      // can wait for done to become true
	sourceMap  *SourceMap
}

// FlushyResponseWriter wraps http.ResponseWriter, calling Flush()
// after each Write() to minimize buffering on our (server) end of the
// connection.
type FlushyResponseWriter struct {
	http.ResponseWriter
}

// Write sends data to the client. It panics if the underlying
// ResponseWriter does not implement http.Flusher.
func (writer *FlushyResponseWriter) Write(data []byte) (int, error) {
	defer func() {
		flusher, ok := writer.ResponseWriter.(http.Flusher)
		if !ok {
			log.Fatal("ResponseWriter is not a flusher")
		}
		flusher.Flush()
	}()
	return writer.ResponseWriter.Write(data)
}

// Run starts an HTTP server with the given configuration.
func (srv *Server) Run(c *Config) (err error) {
	// c.Check() is just for tests -- main() has already done this.
	if err = c.Check(); err != nil {
		log.Fatal(err)
	}
	defer runtime.GOMAXPROCS(runtime.GOMAXPROCS(c.CPUMax))
	addr, err := net.ResolveTCPAddr("tcp", c.Addr)
	if err != nil {
		return
	}
	srv.listener, err = net.ListenTCP("tcp", addr)
	if err != nil {
		return
	}
	if c.UID != os.Getuid() && c.UID != 0 {
		err = syscall.Setuid(c.UID)
		if err != nil {
			err = errors.New("Setuid: " + err.Error())
			return
		}
	}
	srv.Addr = srv.listener.Addr().String()
	srv.sourceMap = NewSourceMap()
	mux := http.NewServeMux()
	mux.HandleFunc("/", func(writer http.ResponseWriter, req *http.Request) {
		writer.Header().Set("Content-Type", config.ContentType)
		startTime := time.Now()
		sreader := srv.sourceMap.NewReader(c.Path, c)
		fwriter := &FlushyResponseWriter{writer}
		wroteBytes, err := io.Copy(fwriter,
			bufio.NewReaderSize(sreader, int(c.FrameBytes)))
		if e, ok := err.(*net.OpError); ok {
			if e, ok := e.Err.(syscall.Errno); ok {
				if e == syscall.ECONNRESET {
					// Not really an error: client disconnected.
					err = nil
				}
			}
		}
		if err != nil {
			log.Printf("client %s error: %s", req.RemoteAddr, err)
		}
		log.Println("client", req.RemoteAddr,
			"--", time.Since(startTime).String(), "elapsed",
			wroteBytes, "bytes",
			sreader.FramesRead, "frames +",
			sreader.FramesSkipped, "skipped")
		sreader.Close()
		if srv.sourceMap.Count() == 0 && !c.Reopen {
			// The only source path has ended and can't be reopened.
			srv.Close()
		}
	})
	srv.Handler = mux
	mutex := &sync.RWMutex{}
	srv.Cond = sync.NewCond(mutex.RLocker())
	go func() {
		err = srv.Serve(tcpKeepAliveListener{srv.listener})
		if !srv.shutdown {
			srv.Err = err
		}
		mutex.Lock()
		srv.done = true
		srv.Cond.Broadcast()
		mutex.Unlock()
	}()
	return nil
}

// Wait returns when the server stops listening. If the server stops
// because Close() was called, it returns nil. If it stops for some
// other reason, it returns the error that caused it to stop.
func (srv *Server) Wait() error {
	srv.Cond.L.Lock()
	defer srv.Cond.L.Unlock()
	for !srv.done {
		log.Print("waiting")
		srv.Cond.Wait()
	}
	return srv.Err
}

// Close shuts down the server. It returns an error if the server
// already stopped for some reason other than a prior call to Close().
func (srv *Server) Close() error {
	srv.shutdown = true
	srv.listener.Close()
	srv.sourceMap.Close()
	return srv.Wait()
}

// Copied from net/http because not exported.
//
type tcpKeepAliveListener struct {
	*net.TCPListener
}

func (ln tcpKeepAliveListener) Accept() (c net.Conn, err error) {
	tc, err := ln.AcceptTCP()
	if err != nil {
		return
	}
	tc.SetKeepAlive(true)
	tc.SetKeepAlivePeriod(3 * time.Minute)
	return tc, nil
}
<|endoftext|>


<commit_before>package main

import (
	"bones/config"
	"bones/repositories"
	"bones/web/filters"
	"bones/web/handlers"
	"github.com/gorilla/pat"
	"log"
	"net/http"
)

func main() {
	repositories.Connect(config.Database())
	defer repositories.Cleanup()

	repositories.EnableSessions()

	r := setupRouting()

	http.Handle("/", r)
	http.Handle("/assets/", http.StripPrefix("/assets/", http.FileServer(http.Dir("./assets"))))

	log.Println("Starting server on port 8080")

	log.Fatal(http.ListenAndServe(":8080", nil))
}

func setupRouting() *pat.Router {
	r := pat.New()

	handlers.SetRouter(r)

	r.Get("/users/{id:[0-9]+}/profile", filters.ApplyTo(handlers.LoadUserProfilePage, filters.Authenticate, filters.Params)).Name("userProfile")

	r.Get("/signup", handlers.LoadSignupPage)
	r.Post("/signup", handlers.CreateNewUser)

	r.Get("/login", handlers.LoadLoginPage)
	r.Post("/login", filters.ApplyTo(handlers.CreateNewSession, filters.Csrf))
	r.Get("/logout", handlers.Logout)

	r.Get("/", handlers.LoadHomePage)

	return r
}
<commit_msg>Support setting server port via "PORT" envvar<commit_after>package main

import (
	"bones/config"
	"bones/repositories"
	"bones/web/filters"
	"bones/web/handlers"
	"github.com/gorilla/pat"
	"log"
	"net/http"
	"os"
)

func main() {
	repositories.Connect(config.Database())
	defer repositories.Cleanup()

	repositories.EnableSessions()

	r := setupRouting()

	http.Handle("/", r)
	http.Handle("/assets/", http.StripPrefix("/assets/", http.FileServer(http.Dir("./assets"))))

	port := portFromEnvOrDefault()
	log.Println("Starting server on port", port)
	log.Fatal(http.ListenAndServe(":"+port, nil))
}

func setupRouting() *pat.Router {
	r := pat.New()

	handlers.SetRouter(r)

	r.Get("/users/{id:[0-9]+}/profile", filters.ApplyTo(handlers.LoadUserProfilePage, filters.Authenticate, filters.Params)).Name("userProfile")

	r.Get("/signup", handlers.LoadSignupPage)
	r.Post("/signup", handlers.CreateNewUser)

	r.Get("/login", handlers.LoadLoginPage)
	r.Post("/login", filters.ApplyTo(handlers.CreateNewSession, filters.Csrf))
	r.Get("/logout", handlers.Logout)

	r.Get("/", handlers.LoadHomePage)

	return r
}

func portFromEnvOrDefault() string {
	port := os.Getenv("PORT")

	if port != "" {
		return port
	}

	return "8080"
}
<|endoftext|>


<commit_before>// Copyright 2018 Jigsaw Operations LLC
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     https://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package main

import (
	"bytes"
	"errors"
	"flag"
	"fmt"
	"io"
	"log"
	"net"
	"net/http"
	"os"
	"os/signal"
	"strconv"
	"strings"
	"syscall"
	"time"

	"github.com/Jigsaw-Code/outline-ss-server/metrics"
	"github.com/prometheus/client_golang/prometheus/promhttp"
	"github.com/shadowsocks/go-shadowsocks2/core"
	ssnet "github.com/shadowsocks/go-shadowsocks2/net"
	"github.com/shadowsocks/go-shadowsocks2/shadowaead"
	"github.com/shadowsocks/go-shadowsocks2/socks"
)

var config struct {
	UDPTimeout time.Duration
}

func shadowConn(conn ssnet.DuplexConn, cipherList []shadowaead.Cipher) (ssnet.DuplexConn, int, error) {
	cipher, index, shadowReader, err := findCipher(conn, cipherList)
	if err != nil {
		return nil, -1, err
	}
	shadowWriter := shadowaead.NewShadowsocksWriter(conn, cipher)
	return ssnet.WrapDuplexConn(conn, shadowReader, shadowWriter), index, nil
}

func findCipher(clientReader io.Reader, cipherList []shadowaead.Cipher) (shadowaead.Cipher, int, io.Reader, error) {
	if len(cipherList) == 0 {
		return nil, -1, nil, errors.New("Empty cipher list")
	} else if len(cipherList) == 1 {
		return cipherList[0], 0, shadowaead.NewShadowsocksReader(clientReader, cipherList[0]), nil
	}
	// buffer saves the bytes read from shadowConn, in order to allow for replays.
	var buffer bytes.Buffer
	// Try each cipher until we find one that authenticates successfully.
	// This assumes that all ciphers are AEAD.
	// TODO: Reorder list to try previously successful ciphers first for the client IP.
	// TODO: Ban and log client IPs with too many failures too quick to protect against DoS.
	for i, cipher := range cipherList {
		log.Printf("Trying cipher %v", i)
		// tmpReader reuses the bytes read so far, falling back to shadowConn if it needs more
		// bytes. All bytes read from shadowConn are saved in buffer.
		tmpReader := io.MultiReader(bytes.NewReader(buffer.Bytes()), io.TeeReader(clientReader, &buffer))
		// Override the Reader of shadowConn so we can reset it for each cipher test.
		cipherReader := shadowaead.NewShadowsocksReader(tmpReader, cipher)
		// Read should read just enough data to authenticate the payload size.
		_, err := cipherReader.Read(make([]byte, 0))
		if err != nil {
			log.Printf("Failed cipher %v: %v", i, err)
			continue
		}
		log.Printf("Selected cipher %v", i)
		// We don't need to replay the bytes anymore, but we don't want to drop those
		// read so far.
		return cipher, i, shadowaead.NewShadowsocksReader(io.MultiReader(&buffer, clientReader), cipher), nil
	}
	return nil, -1, nil, fmt.Errorf("could not find valid cipher")
}

func getNetKey(addr net.Addr) (string, error) {
	host, _, err := net.SplitHostPort(addr.String())
	if err != nil {
		return "", err
	}
	ip := net.ParseIP(host)
	if ip == nil {
		return "", errors.New("Failed to parse ip")
	}
	ipNet := net.IPNet{IP: ip}
	if ip.To4() != nil {
		ipNet.Mask = net.CIDRMask(24, 32)
	} else {
		ipNet.Mask = net.CIDRMask(32, 128)
	}
	return ipNet.String(), nil
}

type connectionError struct {
	// TODO: create status enums and move to metrics.go
	status  string
	message string
	cause   error
}

// Listen on addr for incoming connections.
func tcpRemote(addr string, cipherList []shadowaead.Cipher, m metrics.TCPMetrics) {
	// TODO: Delete these and get from the already collected Prometheus metrics instead.
	accessKeyMetrics := metrics.NewMetricsMap()
	netMetrics := metrics.NewMetricsMap()
	l, err := net.Listen("tcp", addr)
	if err != nil {
		log.Printf("failed to listen on %s: %v", addr, err)
		return
	}

	log.Printf("listening TCP on %s", addr)
	for {
		var clientConn ssnet.DuplexConn
		clientConn, err := l.(*net.TCPListener).AcceptTCP()
		m.AddOpenTCPConnection()
		if err != nil {
			log.Printf("failed to accept: %v", err)
			return
		}

		go func() (connError *connectionError) {
			connStart := time.Now()
			clientConn.(*net.TCPConn).SetKeepAlive(true)
			netKey, err := getNetKey(clientConn.RemoteAddr())
			if err != nil {
				netKey = "INVALID"
			}
			accessKey := "INVALID"
			var proxyMetrics metrics.ProxyMetrics
			clientConn = metrics.MeasureConn(clientConn, &proxyMetrics.ProxyClient, &proxyMetrics.ClientProxy)
			defer func() {
				connEnd := time.Now()
				connDuration := connEnd.Sub(connStart)
				clientConn.Close()
				status := "OK"
				if connError != nil {
					log.Printf("ERROR %v: %v", connError.message, connError.cause)
					status = connError.status
				}
				log.Printf("Done with status %v, duration %v", status, connDuration)
				m.AddClosedTCPConnection(accessKey, status, proxyMetrics, connDuration)
				accessKeyMetrics.Add(accessKey, proxyMetrics)
				log.Printf("Key %v: %s", accessKey, metrics.SPrintMetrics(accessKeyMetrics.Get(accessKey)))
				netMetrics.Add(netKey, proxyMetrics)
				log.Printf("Net %v: %s", netKey, metrics.SPrintMetrics(netMetrics.Get(netKey)))
			}()

			clientConn, index, err := shadowConn(clientConn, cipherList)
			if err != nil {
				return &connectionError{"ERR_CIPHER", "Failed to find a valid cipher", err}
			}
			accessKey = strconv.Itoa(index)

			tgt, err := socks.ReadAddr(clientConn)
			if err != nil {
				return &connectionError{"ERR_READ_ADDRESS", "Failed to get target address", err}
			}

			c, err := net.Dial("tcp", tgt.String())
			if err != nil {
				return &connectionError{"ERR_CONNECT", "Failed to connect to target", err}
			}
			var tgtConn ssnet.DuplexConn = c.(*net.TCPConn)
			defer tgtConn.Close()
			tgtConn.(*net.TCPConn).SetKeepAlive(true)
			tgtConn = metrics.MeasureConn(tgtConn, &proxyMetrics.ProxyTarget, &proxyMetrics.TargetProxy)

			// TODO: Disable logging in production. This is sensitive.
			log.Printf("proxy %s <-> %s", clientConn.RemoteAddr(), tgt)
			_, _, err = ssnet.Relay(clientConn, tgtConn)
			if err != nil {
				return &connectionError{"ERR_RELAY", "Failed to relay traffic", err}
			}
			return nil
		}()
	}
}

type cipherList []shadowaead.Cipher

func main() {

	var flags struct {
		Server      string
		Ciphers     cipherList
		MetricsAddr string
	}

	flag.StringVar(&flags.Server, "s", "", "server listen address")
	flag.Var(&flags.Ciphers, "u", "available ciphers: "+strings.Join(core.ListCipher(), " "))
	flag.DurationVar(&config.UDPTimeout, "udptimeout", 5*time.Minute, "UDP tunnel timeout")
	flag.StringVar(&flags.MetricsAddr, "metrics", "", "address for the Prometheus metrics")
	flag.Parse()

	if flags.Server == "" || len(flags.Ciphers) == 0 {
		flag.Usage()
		return
	}

	if flags.MetricsAddr != "" {
		http.Handle("/metrics", promhttp.Handler())
		go func() {
			log.Fatal(http.ListenAndServe(flags.MetricsAddr, nil))
		}()
		log.Printf("Metrics on http://%v/metrics", flags.MetricsAddr)
	}

	go udpRemote(flags.Server, flags.Ciphers)
	go tcpRemote(flags.Server, flags.Ciphers, metrics.NewPrometheusTCPMetrics())

	sigCh := make(chan os.Signal, 1)
	signal.Notify(sigCh, syscall.SIGINT, syscall.SIGTERM)
	<-sigCh
}

func (sl *cipherList) Set(flagValue string) error {
	e := strings.SplitN(flagValue, ":", 2)
	if len(e) != 2 {
		return fmt.Errorf("Missing colon")
	}
	cipher, err := core.PickCipher(e[0], nil, e[1])
	if err != nil {
		return err
	}
	aead, ok := cipher.(shadowaead.Cipher)
	if !ok {
		log.Fatal("Only AEAD ciphers are supported")
	}
	*sl = append(*sl, aead)
	return nil
}

func (sl *cipherList) String() string {
	return fmt.Sprint(*sl)
}
<commit_msg>Simplify findCipher<commit_after>// Copyright 2018 Jigsaw Operations LLC
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     https://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package main

import (
	"bytes"
	"errors"
	"flag"
	"fmt"
	"io"
	"log"
	"net"
	"net/http"
	"os"
	"os/signal"
	"strconv"
	"strings"
	"syscall"
	"time"

	"github.com/Jigsaw-Code/outline-ss-server/metrics"
	"github.com/prometheus/client_golang/prometheus/promhttp"
	"github.com/shadowsocks/go-shadowsocks2/core"
	ssnet "github.com/shadowsocks/go-shadowsocks2/net"
	"github.com/shadowsocks/go-shadowsocks2/shadowaead"
	"github.com/shadowsocks/go-shadowsocks2/socks"
)

var config struct {
	UDPTimeout time.Duration
}

func findCipher(clientConn ssnet.DuplexConn, cipherList []shadowaead.Cipher) (int, ssnet.DuplexConn, error) {
	if len(cipherList) == 0 {
		return -1, nil, errors.New("Empty cipher list")
	} else if len(cipherList) == 1 {
		return 0, shadowaead.NewConn(clientConn, cipherList[0]), nil
	}
	// buffer saves the bytes read from shadowConn, in order to allow for replays.
	var buffer bytes.Buffer
	// Try each cipher until we find one that authenticates successfully.
	// This assumes that all ciphers are AEAD.
	// TODO: Reorder list to try previously successful ciphers first for the client IP.
	// TODO: Ban and log client IPs with too many failures too quick to protect against DoS.
	for i, cipher := range cipherList {
		log.Printf("Trying cipher %v", i)
		// tmpReader reuses the bytes read so far, falling back to shadowConn if it needs more
		// bytes. All bytes read from shadowConn are saved in buffer.
		tmpReader := io.MultiReader(bytes.NewReader(buffer.Bytes()), io.TeeReader(clientConn, &buffer))
		// Override the Reader of shadowConn so we can reset it for each cipher test.
		cipherReader := shadowaead.NewShadowsocksReader(tmpReader, cipher)
		// Read should read just enough data to authenticate the payload size.
		_, err := cipherReader.Read(make([]byte, 0))
		if err != nil {
			log.Printf("Failed cipher %v: %v", i, err)
			continue
		}
		log.Printf("Selected cipher %v", i)
		// We don't need to replay the bytes anymore, but we don't want to drop those
		// read so far.
		ssr := shadowaead.NewShadowsocksReader(io.MultiReader(&buffer, clientConn), cipher)
		ssw := shadowaead.NewShadowsocksWriter(clientConn, cipher)
		return i, ssnet.WrapDuplexConn(clientConn, ssr, ssw), nil
	}
	return -1, nil, fmt.Errorf("could not find valid cipher")
}

func getNetKey(addr net.Addr) (string, error) {
	host, _, err := net.SplitHostPort(addr.String())
	if err != nil {
		return "", err
	}
	ip := net.ParseIP(host)
	if ip == nil {
		return "", errors.New("Failed to parse ip")
	}
	ipNet := net.IPNet{IP: ip}
	if ip.To4() != nil {
		ipNet.Mask = net.CIDRMask(24, 32)
	} else {
		ipNet.Mask = net.CIDRMask(32, 128)
	}
	return ipNet.String(), nil
}

type connectionError struct {
	// TODO: create status enums and move to metrics.go
	status  string
	message string
	cause   error
}

// Listen on addr for incoming connections.
func tcpRemote(addr string, cipherList []shadowaead.Cipher, m metrics.TCPMetrics) {
	// TODO: Delete these and get from the already collected Prometheus metrics instead.
	accessKeyMetrics := metrics.NewMetricsMap()
	netMetrics := metrics.NewMetricsMap()
	l, err := net.Listen("tcp", addr)
	if err != nil {
		log.Printf("failed to listen on %s: %v", addr, err)
		return
	}

	log.Printf("listening TCP on %s", addr)
	for {
		var clientConn ssnet.DuplexConn
		clientConn, err := l.(*net.TCPListener).AcceptTCP()
		m.AddOpenTCPConnection()
		if err != nil {
			log.Printf("failed to accept: %v", err)
			return
		}

		go func() (connError *connectionError) {
			connStart := time.Now()
			clientConn.(*net.TCPConn).SetKeepAlive(true)
			netKey, err := getNetKey(clientConn.RemoteAddr())
			if err != nil {
				netKey = "INVALID"
			}
			accessKey := "INVALID"
			var proxyMetrics metrics.ProxyMetrics
			clientConn = metrics.MeasureConn(clientConn, &proxyMetrics.ProxyClient, &proxyMetrics.ClientProxy)
			defer func() {
				connEnd := time.Now()
				connDuration := connEnd.Sub(connStart)
				clientConn.Close()
				status := "OK"
				if connError != nil {
					log.Printf("ERROR %v: %v", connError.message, connError.cause)
					status = connError.status
				}
				log.Printf("Done with status %v, duration %v", status, connDuration)
				m.AddClosedTCPConnection(accessKey, status, proxyMetrics, connDuration)
				accessKeyMetrics.Add(accessKey, proxyMetrics)
				log.Printf("Key %v: %s", accessKey, metrics.SPrintMetrics(accessKeyMetrics.Get(accessKey)))
				netMetrics.Add(netKey, proxyMetrics)
				log.Printf("Net %v: %s", netKey, metrics.SPrintMetrics(netMetrics.Get(netKey)))
			}()

			index, clientConn, err := findCipher(clientConn, cipherList)
			if err != nil {
				return &connectionError{"ERR_CIPHER", "Failed to find a valid cipher", err}
			}
			accessKey = strconv.Itoa(index)

			tgt, err := socks.ReadAddr(clientConn)
			if err != nil {
				return &connectionError{"ERR_READ_ADDRESS", "Failed to get target address", err}
			}

			c, err := net.Dial("tcp", tgt.String())
			if err != nil {
				return &connectionError{"ERR_CONNECT", "Failed to connect to target", err}
			}
			var tgtConn ssnet.DuplexConn = c.(*net.TCPConn)
			defer tgtConn.Close()
			tgtConn.(*net.TCPConn).SetKeepAlive(true)
			tgtConn = metrics.MeasureConn(tgtConn, &proxyMetrics.ProxyTarget, &proxyMetrics.TargetProxy)

			// TODO: Disable logging in production. This is sensitive.
			log.Printf("proxy %s <-> %s", clientConn.RemoteAddr(), tgt)
			_, _, err = ssnet.Relay(clientConn, tgtConn)
			if err != nil {
				return &connectionError{"ERR_RELAY", "Failed to relay traffic", err}
			}
			return nil
		}()
	}
}

type cipherList []shadowaead.Cipher

func main() {

	var flags struct {
		Server      string
		Ciphers     cipherList
		MetricsAddr string
	}

	flag.StringVar(&flags.Server, "s", "", "server listen address")
	flag.Var(&flags.Ciphers, "u", "available ciphers: "+strings.Join(core.ListCipher(), " "))
	flag.DurationVar(&config.UDPTimeout, "udptimeout", 5*time.Minute, "UDP tunnel timeout")
	flag.StringVar(&flags.MetricsAddr, "metrics", "", "address for the Prometheus metrics")
	flag.Parse()

	if flags.Server == "" || len(flags.Ciphers) == 0 {
		flag.Usage()
		return
	}

	if flags.MetricsAddr != "" {
		http.Handle("/metrics", promhttp.Handler())
		go func() {
			log.Fatal(http.ListenAndServe(flags.MetricsAddr, nil))
		}()
		log.Printf("Metrics on http://%v/metrics", flags.MetricsAddr)
	}

	go udpRemote(flags.Server, flags.Ciphers)
	go tcpRemote(flags.Server, flags.Ciphers, metrics.NewPrometheusTCPMetrics())

	sigCh := make(chan os.Signal, 1)
	signal.Notify(sigCh, syscall.SIGINT, syscall.SIGTERM)
	<-sigCh
}

func (sl *cipherList) Set(flagValue string) error {
	e := strings.SplitN(flagValue, ":", 2)
	if len(e) != 2 {
		return fmt.Errorf("Missing colon")
	}
	cipher, err := core.PickCipher(e[0], nil, e[1])
	if err != nil {
		return err
	}
	aead, ok := cipher.(shadowaead.Cipher)
	if !ok {
		log.Fatal("Only AEAD ciphers are supported")
	}
	*sl = append(*sl, aead)
	return nil
}

func (sl *cipherList) String() string {
	return fmt.Sprint(*sl)
}
<|endoftext|>


<commit_before>package main

import (
	"crypto/md5"
	"encoding/hex"
	"fmt"
	"github.com/codegangsta/negroni"
	"github.com/fzzy/radix/extra/pool"
	"github.com/fzzy/radix/redis"
	"github.com/gorilla/mux"
	"github.com/namsral/flag"
	"github.com/unrolled/render"
	"net/http"
	"os"
	"runtime"
	// "io"
	// "time"
	// "log"
	"strconv"
	"strings"
	// "regexp"
)

func cleanWords(query string) []string {
	// r := regexp.MustCompile(regexp.QuoteMeta("[]()"))
	// query = r.ReplaceAllString(query, " ")
	query = strings.Replace(query, "\"", " ", -1)
	query = strings.Replace(query, "[", " ", -1)
	query = strings.Replace(query, "]", " ", -1)
	query = strings.Replace(query, "(", " ", -1)
	query = strings.Replace(query, ")", " ", -1)
	query = strings.Replace(query, "?", " ", -1)
	query = strings.Replace(query, "!", " ", -1)
	query = strings.Replace(query, ",", " ", -1)
	query = strings.Replace(query, "-", " ", -1)
	query = strings.Replace(query, ":", " ", -1)
	query = strings.Replace(query, ";", " ", -1)
	query = strings.Replace(query, ",", " ", -1)
	query = strings.Replace(query, "'", " ", -1)
	split := strings.Fields(strings.Trim(query, " "))
	terms := make([]string, len(split))
	for i, term := range split {
		terms[i] = strings.ToLower(strings.Trim(strings.Trim(term, " "), "."))
	}
	return terms
}

func encodeString(str string) string {
	h := md5.New()
	h.Write([]byte(str))
	return hex.EncodeToString(h.Sum(nil))[0:8]
}

func getPrefixes(title string) []string {
	var prefixes []string
	for _, word := range cleanWords(title) {
		for i := 1; i <= len(word); i++ {
			prefixes = append(prefixes, word[0:i])
		}
	}
	return prefixes
}

// func QueryScore(terms []string, title) float32 {
// 	return 1.0
// }

func errHndlr(err error) {
	if err != nil {
		fmt.Println("error:", err)
		panic(err)
		os.Exit(1)
	}
}

func IndexHandler(w http.ResponseWriter, req *http.Request) {
	// this assumes there's a `templates/index.tmpl` file
	renderer.HTML(w, http.StatusOK, "index", nil)
}

/* POSTING new stuff in */
func UpdateHandler(w http.ResponseWriter, req *http.Request) {
	values := make(map[string]string)

	required := []string{"domain", "url", "title"}
	errors := make(map[string]string)
	var value string
	for _, key := range required {
		value = strings.Trim(req.FormValue(key), " ")
		values[key] = value
		if value == "" {
			errors[key] = "Missing"
		}
	}

	optional := []string{"groups", "popularity"}
	for _, key := range optional {
		value = strings.Trim(req.FormValue(key), " ")
		if key == "groups" {
			// values[key] = strings.Split(value, ",")
			values[key] = value
		} else if key == "popularity" {
			if value == "" {
				values[key] = "0"
			} else {
				_, err := strconv.ParseFloat(value, 0)
				if err != nil {
					errors[key] = "Not a number"
				} else {
					values[key] = value
				}
			}
		} else {
			values[key] = value
		}
	}
	// fmt.Println("errors=", errors, len(errors))
	if len(errors) > 0 {
		error := make(map[string]interface{})
		error["error"] = errors
		renderer.JSON(w, http.StatusBadRequest, error)
		return
	}
	// popularity, _ := strconv.ParseInt(values["popularity"], 10, 0)
	popularity, _ := strconv.ParseFloat(values["popularity"], 0)

	encoded := encodeString(values["domain"])
	values["url_encoded"] = encodeString(values["url"])

	c, err := redis_pool.Get()
	errHndlr(err)
	defer redis_pool.Put(c)
	// c.Cmd("FLUSHALL")
	// fmt.Println("CAREFUL! Always flushing the database")
	piped_commands := 0
	for _, prefix := range getPrefixes(values["title"]) {
		c.Append("ZADD", encoded+prefix, popularity, values["url_encoded"])
		piped_commands += 1
	}
	c.Append("HSET", encoded+"$titles", values["url_encoded"], values["title"])
	piped_commands += 1
	c.Append("HSET", encoded+"$urls", values["url_encoded"], values["url"])
	piped_commands += 1
	for i := 1; i <= piped_commands; i++ {
		if err := c.GetReply().Err; err != nil {
			errHndlr(err)
		}
	}

	output := map[string]string{"message": "OK"}
	renderer.JSON(w, http.StatusCreated, output)
}

func FetchHandler(w http.ResponseWriter, req *http.Request) {

	n_str := req.FormValue("n")

	var n int64
	if n_str == "" {
		n = 10
	} else {
		n, err := strconv.ParseInt(n_str, 10, 0)
		if err != nil {
			// error := make(map[string]string{"error": "Not a number"})
			error := map[string]string{"error": "Not a number"}
			renderer.JSON(w, http.StatusBadRequest, error)
			return
		}
		if n <= 0 {
			error := map[string]string{"error": "Number too small"}
			renderer.JSON(w, http.StatusBadRequest, error)
			return
		}
		if n > 100 {
			error := map[string]string{"error": "Number too big"}
			renderer.JSON(w, http.StatusBadRequest, error)
			return
		}
	}

	domain := strings.Trim(req.FormValue("domain"), " ")
	if domain == "" {
		error := map[string]string{"error": "'domain' missing"}
		renderer.JSON(w, http.StatusBadRequest, error)
		return
	}

	encoded := encodeString(domain)
	// fmt.Println(domain, encoded)

	q := strings.Trim(req.FormValue("q"), " ")
	terms := cleanWords(q)

	c, err := redis_pool.Get()
	errHndlr(err)
	defer redis_pool.CarefullyPut(c, &err)

	encoded_terms := make([]string, len(terms))
	for i, term := range terms {
		encoded_terms[i] = encoded + term
	}
	// NOTE! Maybe we don't need the ZINTERSTORE if there's only 1 command
	c.Append("ZINTERSTORE", "$tmp", len(terms), encoded_terms, "AGGREGATE", "max")
	c.Append("ZREVRANGE", "$tmp", 0, n-1, "WITHSCORES")

	c.GetReply() // the ZINTERSTORE
	replies, err := c.GetReply().List()
	// fmt.Println("replies", replies, len(replies))
	errHndlr(err)

	encoded_urls := make([]string, n+1)
	scores := make([]string, n+1)
	evens := 0
	for i, element := range replies {
		if i%2 == 0 {
			encoded_urls[evens] = element
			evens = evens + 1
		} else {
			scores[evens-1] = element
		}
	}
	encoded_urls = encoded_urls[:evens]
	scores = scores[:evens]

	var titles []string
	var urls []string
	if len(encoded_urls) == 0 {
	} else {
		titles, err = c.Cmd("HMGET", encoded+"$titles", encoded_urls).List()
		errHndlr(err)
		urls, err = c.Cmd("HMGET", encoded+"$urls", encoded_urls).List()
		errHndlr(err)
	}
	rows := make([]interface{}, len(titles))
	for i, title := range titles {
		row := make([]string, 2)
		row[0] = urls[i]
		// fmt.Println("scores", scores[i]+ 1000)
		// row[1] = scores[i] * QueryScore(terms, title)
		row[1] = title
		rows[i] = row
	}
	rows = rows[:len(titles)]

	output := make(map[string]interface{})
	output["terms"] = terms
	output["results"] = rows
	// fmt.Println(output)
	w.Header().Set("Access-Control-Allow-Origin", "*")
	renderer.JSON(w, http.StatusOK, output)
}

var (
	renderer   *render.Render
	redis_pool *pool.Pool
	procs      int
	debug      = true
)

func main() {
	var port = 3001
	var redis_url = "127.0.0.1:6379"
	var redis_database = 0
	flag.IntVar(&port, "port", port, "Port to start the server on")
	flag.IntVar(&procs, "procs", 1, "Number of CPU processors (0 to use max)")
	flag.BoolVar(&debug, "debug", false, "Debug mode")
	flag.StringVar(
		&redis_url, "redis_url", redis_url,
		"Redis URL to tcp connect to")
	flag.IntVar(&redis_database, "redis_database", redis_database,
		"Redis database number to connect to")
	flag.Parse()

	// Figuring out how many processors to use.
	max_procs := runtime.NumCPU()
	if procs == 0 {
		procs = max_procs
	} else if procs < 0 {
		panic("PROCS < 0")
	} else if procs > max_procs {
		panic(fmt.Sprintf("PROCS > max (%v)", max_procs))
	}
	// fmt.Println("procs=", procs)
	runtime.GOMAXPROCS(procs)

	fmt.Println("DEBUG MODE:", debug)
	renderer = render.New(render.Options{
		IndentJSON:    debug,
		IsDevelopment: debug,
	})

	df := func(network, addr string) (*redis.Client, error) {
		client, err := redis.Dial(network, addr)
		// fmt.Println("DIaling")
		if err != nil {
			return nil, err
		}
		err = client.Cmd("SELECT", redis_database).Err
		if err != nil {
			return nil, err
		}
		// if err = client.Cmd("AUTH", "SUPERSECRET").Err; err != nil {
		// 	client.Close()
		// 	return nil, err
		// }
		return client, nil
	}

	var err error
	// fmt.Println("redis_url:", redis_url)
	// fmt.Println("redis_database:", redis_database)
	redis_pool, err = pool.NewCustomPool("tcp", redis_url, procs*10, df)
	errHndlr(err)

	mux := mux.NewRouter()
	mux.HandleFunc("/", IndexHandler).Methods("GET", "HEAD")
	mux.HandleFunc("/v1", FetchHandler).Methods("GET", "HEAD")
	mux.HandleFunc("/v1", UpdateHandler).Methods("POST", "PUT")

	n := negroni.Classic()
	n.UseHandler(mux)
	n.Run(fmt.Sprintf(":%d", port))
}
<commit_msg>more verbose<commit_after>package main

import (
	"crypto/md5"
	"encoding/hex"
	"fmt"
	"github.com/codegangsta/negroni"
	"github.com/fzzy/radix/extra/pool"
	"github.com/fzzy/radix/redis"
	"github.com/gorilla/mux"
	"github.com/namsral/flag"
	"github.com/unrolled/render"
	"net/http"
	"os"
	"runtime"
	// "io"
	// "time"
	// "log"
	"strconv"
	"strings"
	// "regexp"
)

func cleanWords(query string) []string {
	// r := regexp.MustCompile(regexp.QuoteMeta("[]()"))
	// query = r.ReplaceAllString(query, " ")
	query = strings.Replace(query, "\"", " ", -1)
	query = strings.Replace(query, "[", " ", -1)
	query = strings.Replace(query, "]", " ", -1)
	query = strings.Replace(query, "(", " ", -1)
	query = strings.Replace(query, ")", " ", -1)
	query = strings.Replace(query, "?", " ", -1)
	query = strings.Replace(query, "!", " ", -1)
	query = strings.Replace(query, ",", " ", -1)
	query = strings.Replace(query, "-", " ", -1)
	query = strings.Replace(query, ":", " ", -1)
	query = strings.Replace(query, ";", " ", -1)
	query = strings.Replace(query, ",", " ", -1)
	query = strings.Replace(query, "'", " ", -1)
	split := strings.Fields(strings.Trim(query, " "))
	terms := make([]string, len(split))
	for i, term := range split {
		terms[i] = strings.ToLower(strings.Trim(strings.Trim(term, " "), "."))
	}
	return terms
}

func encodeString(str string) string {
	h := md5.New()
	h.Write([]byte(str))
	return hex.EncodeToString(h.Sum(nil))[0:8]
}

func getPrefixes(title string) []string {
	var prefixes []string
	for _, word := range cleanWords(title) {
		for i := 1; i <= len(word); i++ {
			prefixes = append(prefixes, word[0:i])
		}
	}
	return prefixes
}

// func QueryScore(terms []string, title) float32 {
// 	return 1.0
// }

func errHndlr(err error) {
	if err != nil {
		fmt.Println("error:", err)
		panic(err)
		os.Exit(1)
	}
}

func IndexHandler(w http.ResponseWriter, req *http.Request) {
	// this assumes there's a `templates/index.tmpl` file
	renderer.HTML(w, http.StatusOK, "index", nil)
}

/* POSTING new stuff in */
func UpdateHandler(w http.ResponseWriter, req *http.Request) {
	values := make(map[string]string)

	required := []string{"domain", "url", "title"}
	errors := make(map[string]string)
	var value string
	for _, key := range required {
		value = strings.Trim(req.FormValue(key), " ")
		values[key] = value
		if value == "" {
			errors[key] = "Missing"
		}
	}

	optional := []string{"groups", "popularity"}
	for _, key := range optional {
		value = strings.Trim(req.FormValue(key), " ")
		if key == "groups" {
			// values[key] = strings.Split(value, ",")
			values[key] = value
		} else if key == "popularity" {
			if value == "" {
				values[key] = "0"
			} else {
				_, err := strconv.ParseFloat(value, 0)
				if err != nil {
					errors[key] = "Not a number"
				} else {
					values[key] = value
				}
			}
		} else {
			values[key] = value
		}
	}
	// fmt.Println("errors=", errors, len(errors))
	if len(errors) > 0 {
		error := make(map[string]interface{})
		error["error"] = errors
		renderer.JSON(w, http.StatusBadRequest, error)
		return
	}
	// popularity, _ := strconv.ParseInt(values["popularity"], 10, 0)
	popularity, _ := strconv.ParseFloat(values["popularity"], 0)

	encoded := encodeString(values["domain"])
	values["url_encoded"] = encodeString(values["url"])

	c, err := redis_pool.Get()
	errHndlr(err)
	defer redis_pool.Put(c)
	// c.Cmd("FLUSHALL")
	// fmt.Println("CAREFUL! Always flushing the database")
	piped_commands := 0
	for _, prefix := range getPrefixes(values["title"]) {
		c.Append("ZADD", encoded+prefix, popularity, values["url_encoded"])
		piped_commands += 1
	}
	c.Append("HSET", encoded+"$titles", values["url_encoded"], values["title"])
	piped_commands += 1
	c.Append("HSET", encoded+"$urls", values["url_encoded"], values["url"])
	piped_commands += 1
	for i := 1; i <= piped_commands; i++ {
		if err := c.GetReply().Err; err != nil {
			errHndlr(err)
		}
	}

	output := map[string]string{"message": "OK"}
	renderer.JSON(w, http.StatusCreated, output)
}

func FetchHandler(w http.ResponseWriter, req *http.Request) {

	n_str := req.FormValue("n")

	var n int64
	if n_str == "" {
		n = 10
	} else {
		n, err := strconv.ParseInt(n_str, 10, 0)
		if err != nil {
			// error := make(map[string]string{"error": "Not a number"})
			error := map[string]string{"error": "Not a number"}
			renderer.JSON(w, http.StatusBadRequest, error)
			return
		}
		if n <= 0 {
			error := map[string]string{"error": "Number too small"}
			renderer.JSON(w, http.StatusBadRequest, error)
			return
		}
		if n > 100 {
			error := map[string]string{"error": "Number too big"}
			renderer.JSON(w, http.StatusBadRequest, error)
			return
		}
	}

	domain := strings.Trim(req.FormValue("domain"), " ")
	if domain == "" {
		error := map[string]string{"error": "'domain' missing"}
		renderer.JSON(w, http.StatusBadRequest, error)
		return
	}

	encoded := encodeString(domain)
	// fmt.Println(domain, encoded)

	q := strings.Trim(req.FormValue("q"), " ")
	terms := cleanWords(q)

	c, err := redis_pool.Get()
	errHndlr(err)
	defer redis_pool.CarefullyPut(c, &err)

	encoded_terms := make([]string, len(terms))
	for i, term := range terms {
		encoded_terms[i] = encoded + term
	}
	// NOTE! Maybe we don't need the ZINTERSTORE if there's only 1 command
	c.Append("ZINTERSTORE", "$tmp", len(terms), encoded_terms, "AGGREGATE", "max")
	c.Append("ZREVRANGE", "$tmp", 0, n-1, "WITHSCORES")

	c.GetReply() // the ZINTERSTORE
	replies, err := c.GetReply().List()
	// fmt.Println("replies", replies, len(replies))
	errHndlr(err)

	encoded_urls := make([]string, n+1)
	scores := make([]string, n+1)
	evens := 0
	for i, element := range replies {
		if i%2 == 0 {
			encoded_urls[evens] = element
			evens = evens + 1
		} else {
			scores[evens-1] = element
		}
	}
	encoded_urls = encoded_urls[:evens]
	scores = scores[:evens]

	var titles []string
	var urls []string
	if len(encoded_urls) == 0 {
	} else {
		titles, err = c.Cmd("HMGET", encoded+"$titles", encoded_urls).List()
		errHndlr(err)
		urls, err = c.Cmd("HMGET", encoded+"$urls", encoded_urls).List()
		errHndlr(err)
	}
	rows := make([]interface{}, len(titles))
	for i, title := range titles {
		row := make([]string, 2)
		row[0] = urls[i]
		// fmt.Println("scores", scores[i]+ 1000)
		// row[1] = scores[i] * QueryScore(terms, title)
		row[1] = title
		rows[i] = row
	}
	rows = rows[:len(titles)]

	output := make(map[string]interface{})
	output["terms"] = terms
	output["results"] = rows
	// fmt.Println(output)
	w.Header().Set("Access-Control-Allow-Origin", "*")
	renderer.JSON(w, http.StatusOK, output)
}

var (
	renderer   *render.Render
	redis_pool *pool.Pool
	procs      int
	debug      = true
)

func main() {
	var port = 3001
	var redis_url = "127.0.0.1:6379"
	var redis_database = 0
	flag.IntVar(&port, "port", port, "Port to start the server on")
	flag.IntVar(&procs, "procs", 1, "Number of CPU processors (0 to use max)")
	flag.BoolVar(&debug, "debug", false, "Debug mode")
	flag.StringVar(
		&redis_url, "redis_url", redis_url,
		"Redis URL to tcp connect to")
	flag.IntVar(&redis_database, "redis_database", redis_database,
		"Redis database number to connect to")
	flag.Parse()

	// Figuring out how many processors to use.
	max_procs := runtime.NumCPU()
	if procs == 0 {
		procs = max_procs
	} else if procs < 0 {
		panic("PROCS < 0")
	} else if procs > max_procs {
		panic(fmt.Sprintf("PROCS > max (%v)", max_procs))
	}
	// fmt.Println("procs=", procs)
	fmt.Printf("Running on %d procs\n", procs)
	runtime.GOMAXPROCS(procs)

	fmt.Println("DEBUG MODE:", debug)
	renderer = render.New(render.Options{
		IndentJSON:    debug,
		IsDevelopment: debug,
	})

	df := func(network, addr string) (*redis.Client, error) {
		client, err := redis.Dial(network, addr)
		// fmt.Println("DIaling")
		if err != nil {
			return nil, err
		}
		err = client.Cmd("SELECT", redis_database).Err
		if err != nil {
			return nil, err
		}
		// if err = client.Cmd("AUTH", "SUPERSECRET").Err; err != nil {
		// 	client.Close()
		// 	return nil, err
		// }
		return client, nil
	}

	var err error
	// fmt.Println("redis_url:", redis_url)
	// fmt.Println("redis_database:", redis_database)
	// fmt.Println("pool size", procs*10)
	redis_pool, err = pool.NewCustomPool("tcp", redis_url, 100, df)
	errHndlr(err)

	mux := mux.NewRouter()
	mux.HandleFunc("/", IndexHandler).Methods("GET", "HEAD")
	mux.HandleFunc("/v1", FetchHandler).Methods("GET", "HEAD")
	mux.HandleFunc("/v1", UpdateHandler).Methods("POST", "PUT")

	n := negroni.Classic()
	n.UseHandler(mux)
	n.Run(fmt.Sprintf(":%d", port))
}
<|endoftext|>


<commit_before>package falcore

import (
	"bufio"
	"crypto/rand"
	"crypto/tls"
	"errors"
	"fmt"
	"io"
	"net"
	"net/http"
	"os"
	"strconv"
	"sync"
	"syscall"
	"time"
)

type Server struct {
	Addr             string
	Pipeline         *Pipeline
	listener         net.Listener
	listenerFile     *os.File
	stopAccepting    chan int
	handlerWaitGroup *sync.WaitGroup
	logPrefix        string
	AcceptReady      chan int
}

func NewServer(port int, pipeline *Pipeline) *Server {
	s := new(Server)
	s.Addr = fmt.Sprintf(":%v", port)
	s.Pipeline = pipeline
	s.stopAccepting = make(chan int)
	s.AcceptReady = make(chan int, 1)
	s.handlerWaitGroup = new(sync.WaitGroup)
	s.logPrefix = fmt.Sprintf("%d", syscall.Getpid())
	return s
}

func (srv *Server) FdListen(fd int) error {
	var err error
	srv.listenerFile = os.NewFile(uintptr(fd), "")
	if srv.listener, err = net.FileListener(srv.listenerFile); err != nil {
		return err
	}
	if _, ok := srv.listener.(*net.TCPListener); !ok {
		return errors.New("Broken listener isn't TCP")
	}
	return nil
}

func (srv *Server) socketListen() error {
	var la *net.TCPAddr
	var err error
	if la, err = net.ResolveTCPAddr("tcp", srv.Addr); err != nil {
		return err
	}

	var l *net.TCPListener
	if l, err = net.ListenTCP("tcp", la); err != nil {
		return err
	}
	srv.listener = l
	if srv.listenerFile, err = l.File(); err != nil {
		return err
	}
	if e := syscall.SetNonblock(int(srv.listenerFile.Fd()), true); e != nil {
		return e
	}
	return nil
}

func (srv *Server) ListenAndServe() error {
	if srv.Addr == "" {
		srv.Addr = ":http"
	}
	if srv.listener == nil {
		if err := srv.socketListen(); err != nil {
			return err
		}
	}
	return srv.serve()
}

func (srv *Server) SocketFd() int {
	return int(srv.listenerFile.Fd())
}

func (srv *Server) ListenAndServeTLS(certFile, keyFile string) error {
	if srv.Addr == "" {
		srv.Addr = ":https"
	}
	config := &tls.Config{
		Rand:       rand.Reader,
		Time:       time.Now,
		NextProtos: []string{"http/1.1"},
	}

	var err error
	config.Certificates = make([]tls.Certificate, 1)
	config.Certificates[0], err = tls.LoadX509KeyPair(certFile, keyFile)
	if err != nil {
		return err
	}

	if srv.listener == nil {
		if err := srv.socketListen(); err != nil {
			return err
		}
	}

	srv.listener = tls.NewListener(srv.listener, config)

	return srv.serve()
}

func (srv *Server) StopAccepting() {
	srv.stopAccepting <- 1
}

func (srv *Server) Port() int {
	if l := srv.listener; l != nil {
		a := l.Addr()
		if _, p, e := net.SplitHostPort(a.String()); e == nil && p != "" {
			server_port, _ := strconv.Atoi(p)
			return server_port
		}
	}
	return 0
}

func (srv *Server) serve() (e error) {
	var accept = true
	srv.AcceptReady <- 1
	for accept {
		var c net.Conn
		if l, ok := srv.listener.(*net.TCPListener); ok {
			l.SetDeadline(time.Now().Add(3e9))
		}
		c, e = srv.listener.Accept()
		if e != nil {
			if ope, ok := e.(*net.OpError); ok {
				if !(ope.Timeout() && ope.Temporary()) {
					Error("%s SERVER Accept Error: %v", srv.serverLogPrefix(), ope)
				}
			} else {
				Error("%s SERVER Accept Error: %v", srv.serverLogPrefix(), e)
			}
		} else {
			//Trace("Handling!")
			srv.handlerWaitGroup.Add(1)
			go srv.handler(c)
		}
		select {
		case <-srv.stopAccepting:
			accept = false
		default:
		}
	}
	Trace("Stopped accepting, waiting for handlers")
	// wait for handlers
	srv.handlerWaitGroup.Wait()
	return nil
}

func (srv *Server) handler(c net.Conn) {
	startTime := time.Now()
	defer srv.connectionFinished(c)
	buf := bufio.NewReaderSize(c, 8192)
	var err error
	var req *http.Request
	// no keepalive (for now)
	reqCount := 0
	keepAlive := true
	for err == nil && keepAlive {
		if req, err = http.ReadRequest(buf); err == nil {
			if req.Header.Get("Connection") != "Keep-Alive" {
				keepAlive = false
			}
			request := newRequest(req, c, startTime)
			reqCount++
			var res *http.Response

			pssInit := new(PipelineStageStat)
			pssInit.Name = "server.Init"
			pssInit.StartTime = startTime
			pssInit.EndTime = time.Now()
			request.appendPipelineStage(pssInit)
			// execute the pipeline
			if res = srv.Pipeline.execute(request); res == nil {
				res = SimpleResponse(req, 404, nil, "Not Found")
			}
			// cleanup
			request.startPipelineStage("server.ResponseWrite")
			req.Body.Close()
			wbuf := bufio.NewWriter(c)
			res.Write(wbuf)
			wbuf.Flush()
			if res.Body != nil {
				res.Body.Close()
			}
			request.finishPipelineStage()
			request.finishRequest()
			srv.requestFinished(request)
		} else {
			// EOF is socket closed
			if err != io.ErrUnexpectedEOF {
				Error("%s %v ERROR reading request: %v", srv.serverLogPrefix(), c.RemoteAddr(), err)
			}
		}
	}
	//Debug("%s Processed %v requests on connection %v", srv.serverLogPrefix(), reqCount, c.RemoteAddr())
}

func (srv *Server) serverLogPrefix() string {
	return srv.logPrefix
}

func (srv *Server) requestFinished(request *Request) {
	if srv.Pipeline.RequestDoneCallback != nil {
		// Don't block the connecion for this
		go srv.Pipeline.RequestDoneCallback.FilterRequest(request)
	}
}

func (srv *Server) connectionFinished(c net.Conn) {
	c.Close()
	srv.handlerWaitGroup.Done()
}
<commit_msg>disabled SetNonblock on windows since it's only required for hot restart and that's not supported on windows<commit_after>package falcore

import (
	"bufio"
	"crypto/rand"
	"crypto/tls"
	"errors"
	"fmt"
	"io"
	"net"
	"net/http"
	"os"
	"strconv"
	"sync"
	"syscall"
	"time"

	"reflect"
	"runtime"
)

type Server struct {
	Addr             string
	Pipeline         *Pipeline
	listener         net.Listener
	listenerFile     *os.File
	stopAccepting    chan int
	handlerWaitGroup *sync.WaitGroup
	logPrefix        string
	AcceptReady      chan int
}

func NewServer(port int, pipeline *Pipeline) *Server {
	s := new(Server)
	s.Addr = fmt.Sprintf(":%v", port)
	s.Pipeline = pipeline
	s.stopAccepting = make(chan int)
	s.AcceptReady = make(chan int, 1)
	s.handlerWaitGroup = new(sync.WaitGroup)
	s.logPrefix = fmt.Sprintf("%d", syscall.Getpid())
	return s
}

func (srv *Server) FdListen(fd int) error {
	var err error
	srv.listenerFile = os.NewFile(uintptr(fd), "")
	if srv.listener, err = net.FileListener(srv.listenerFile); err != nil {
		return err
	}
	if _, ok := srv.listener.(*net.TCPListener); !ok {
		return errors.New("Broken listener isn't TCP")
	}
	return nil
}

func (srv *Server) socketListen() error {
	var la *net.TCPAddr
	var err error
	if la, err = net.ResolveTCPAddr("tcp", srv.Addr); err != nil {
		return err
	}

	var l *net.TCPListener
	if l, err = net.ListenTCP("tcp", la); err != nil {
		return err
	}
	srv.listener = l
	// setup listener to be non-blocking if we're not on windows.
	// this is required for hot restart to work.
	if runtime.GOOS != "windows" {
		if srv.listenerFile, err = l.File(); err != nil {
			return err
		}
		if e := setupFDNonblock(int(srv.listenerFile.Fd())); e != nil {
			return e
		}
	}
	return nil
}

// Calling syscall.SetNonblock using reflection to avoid compile errors
// on windows.  This call is not used on windows as hot restart is not supported.
func setupFDNonblock(fd int) error {
	// if function exists
	if fun := reflect.ValueOf(syscall.SetNonblock); fun.Kind() == reflect.Func {
		// if first argument is an int
		if fun.Type().In(0).Kind() == reflect.Int {
			args := []reflect.Value{reflect.ValueOf(fd), reflect.ValueOf(true)}
			if res := fun.Call(args); len(res) == 1 && !res[0].IsNil() {
				err := res[0].Interface().(error)
				return err
			}
		}
	}

	return nil
}

func (srv *Server) ListenAndServe() error {
	if srv.Addr == "" {
		srv.Addr = ":http"
	}
	if srv.listener == nil {
		if err := srv.socketListen(); err != nil {
			return err
		}
	}
	return srv.serve()
}

func (srv *Server) SocketFd() int {
	return int(srv.listenerFile.Fd())
}

func (srv *Server) ListenAndServeTLS(certFile, keyFile string) error {
	if srv.Addr == "" {
		srv.Addr = ":https"
	}
	config := &tls.Config{
		Rand:       rand.Reader,
		Time:       time.Now,
		NextProtos: []string{"http/1.1"},
	}

	var err error
	config.Certificates = make([]tls.Certificate, 1)
	config.Certificates[0], err = tls.LoadX509KeyPair(certFile, keyFile)
	if err != nil {
		return err
	}

	if srv.listener == nil {
		if err := srv.socketListen(); err != nil {
			return err
		}
	}

	srv.listener = tls.NewListener(srv.listener, config)

	return srv.serve()
}

func (srv *Server) StopAccepting() {
	srv.stopAccepting <- 1
}

func (srv *Server) Port() int {
	if l := srv.listener; l != nil {
		a := l.Addr()
		if _, p, e := net.SplitHostPort(a.String()); e == nil && p != "" {
			server_port, _ := strconv.Atoi(p)
			return server_port
		}
	}
	return 0
}

func (srv *Server) serve() (e error) {
	var accept = true
	srv.AcceptReady <- 1
	for accept {
		var c net.Conn
		if l, ok := srv.listener.(*net.TCPListener); ok {
			l.SetDeadline(time.Now().Add(3e9))
		}
		c, e = srv.listener.Accept()
		if e != nil {
			if ope, ok := e.(*net.OpError); ok {
				if !(ope.Timeout() && ope.Temporary()) {
					Error("%s SERVER Accept Error: %v", srv.serverLogPrefix(), ope)
				}
			} else {
				Error("%s SERVER Accept Error: %v", srv.serverLogPrefix(), e)
			}
		} else {
			//Trace("Handling!")
			srv.handlerWaitGroup.Add(1)
			go srv.handler(c)
		}
		select {
		case <-srv.stopAccepting:
			accept = false
		default:
		}
	}
	Trace("Stopped accepting, waiting for handlers")
	// wait for handlers
	srv.handlerWaitGroup.Wait()
	return nil
}

func (srv *Server) handler(c net.Conn) {
	startTime := time.Now()
	defer srv.connectionFinished(c)
	buf := bufio.NewReaderSize(c, 8192)
	var err error
	var req *http.Request
	// no keepalive (for now)
	reqCount := 0
	keepAlive := true
	for err == nil && keepAlive {
		if req, err = http.ReadRequest(buf); err == nil {
			if req.Header.Get("Connection") != "Keep-Alive" {
				keepAlive = false
			}
			request := newRequest(req, c, startTime)
			reqCount++
			var res *http.Response

			pssInit := new(PipelineStageStat)
			pssInit.Name = "server.Init"
			pssInit.StartTime = startTime
			pssInit.EndTime = time.Now()
			request.appendPipelineStage(pssInit)
			// execute the pipeline
			if res = srv.Pipeline.execute(request); res == nil {
				res = SimpleResponse(req, 404, nil, "Not Found")
			}
			// cleanup
			request.startPipelineStage("server.ResponseWrite")
			req.Body.Close()
			wbuf := bufio.NewWriter(c)
			res.Write(wbuf)
			wbuf.Flush()
			if res.Body != nil {
				res.Body.Close()
			}
			request.finishPipelineStage()
			request.finishRequest()
			srv.requestFinished(request)
		} else {
			// EOF is socket closed
			if err != io.ErrUnexpectedEOF {
				Error("%s %v ERROR reading request: %v", srv.serverLogPrefix(), c.RemoteAddr(), err)
			}
		}
	}
	//Debug("%s Processed %v requests on connection %v", srv.serverLogPrefix(), reqCount, c.RemoteAddr())
}

func (srv *Server) serverLogPrefix() string {
	return srv.logPrefix
}

func (srv *Server) requestFinished(request *Request) {
	if srv.Pipeline.RequestDoneCallback != nil {
		// Don't block the connecion for this
		go srv.Pipeline.RequestDoneCallback.FilterRequest(request)
	}
}

func (srv *Server) connectionFinished(c net.Conn) {
	c.Close()
	srv.handlerWaitGroup.Done()
}
<|endoftext|>


<commit_before>package main

import (
	"encoding/json"
	"errors"
	"log"
	"net/http"
	"net/url"

	"github.com/rcrowley/go-tigertonic"
	"gopkg.in/mgo.v2"
	"gopkg.in/mgo.v2/bson"
)

//curl -H "Content-Type: application/json" -XPUT -d '{"P"}' -f -v http://localhost:3000/beacon/{id}

const (
	dbName = "artroomServer"
	//DBURI for the mongodb
	DBURI = "127.0.0.1"

	waltersAPIPrefix    = "http://api.thewalters.org/v1/objects?apikey=ShxvahaBFNIcfWR7E78xXdssKIlXtUAJk9rDDrrmlvbOlxQKtASCzV4op5aHv2Il&keyword="
	waltersImagePrefix  = "http://static.thewalters.org/images/"
	waltersImagePostfix = "?width=500"
)

var (
	s = Server{}
	//CollectionNames is the shiz
	CollectionNames = []string{"beacon", "art"}
	beaconIndex     = mgo.Index{
		Key:        []string{"MinorID"},
		Unique:     true,
		DropDups:   true,
		Background: true,
		Sparse:     true,
		Name:       "beaconIndex",
		//ExpireAf
	}

	artIndex = mgo.Index{
		Key:        []string{"Beacon", "Title"},
		Unique:     true,
		DropDups:   true,
		Background: true,
		Sparse:     true,
		Name:       "ArtIndex",
		//ExpireAf
	}
	indices = []mgo.Index{beaconIndex, artIndex}
	mux     *tigertonic.TrieServeMux
)

type (
	//Server is the name for the server deal wit it
	Server struct {
		Session *mgo.Session // The main session we'll we be cloning
		DBURI   string       // Where the DB is on the network
		dbName  string       // Name of the MongoDB
	}

	//Beacon is the struct that structures what the data for a beacon will look like
	Beacon struct {
		ProxID  string
		MajorID int
		MinorID int
	}
	//FormResponse is what we get back from the form
	FormResponse struct {
		Date        string
		Author      string
		Title       string
		ProxID      string
		MajorID     int
		MinorID     int
		Description string
	}

	//The structure the walter api gives us more or less
	WalterObj struct {
		ObjectID       int
		Collection     string
		Title          string
		Medium         string
		Description    string
		Images         string
		CuratorComment string
		Beacon         Beacon
		ImageURL       string
	}
	//Walters objects
	Walters struct {
		Items []WalterObj
	}
)

func main() {
	log.Println("Server is warming up...")
	initDB()
	initHandlers()
	defer s.Session.Close()
	server := tigertonic.NewServer("localhost:3000", mux)
	log.Fatal(server.ListenAndServe())
}

func initHandlers() {
	cors := tigertonic.NewCORSBuilder().AddAllowedOrigins("*").AddAllowedHeaders("Access-Control-Allow-Headers", "Origin", "X-Requested-With", "Content-Type", "Accept")
	mux = tigertonic.NewTrieServeMux()
	mux.Handle(
		"POST",
		"/beacon",
		cors.Build(tigertonic.Marshaled(handlePOSTBeacon)),
	)
	//Could use go-metrics to do hot piece of art
	mux.Handle(
		"GET",
		"/beacon/{minorID}",
		cors.Build(tigertonic.Marshaled(handleBeacon)),
	)
}

func handlePOSTBeacon(u *url.URL, h http.Header, formResponse *FormResponse) (status int, responseHeaders http.Header, _ interface{}, err error) {
	log.Printf("We have begun the ritual, %v", h)
	client := &http.Client{}
	req, _ := http.NewRequest("GET", waltersAPIPrefix+formResponse.Title, nil)
	req.Header.Add("Accept", "application/json")
	res, err := client.Do(req)
	if err != nil {
		return 500, h, nil, errors.New("Req didn't go through")
		//		log.Fatalf("Req didn't go through, %v", err)
	}
	defer res.Body.Close()
	var data json.RawMessage
	err = json.NewDecoder(res.Body).Decode(&data)
	if err != nil {
		return 500, h, nil, errors.New("Couldn't decode data...")
		//		log.Fatalf("Can't resolve the datum?%v", err)
	}
	w := Walters{}
	err = json.Unmarshal(data, &w)
	if err != nil {
		return 500, h, nil, errors.New("Couldn't marshall into individual waltOBJ'")
		//		log.Fatalf("Can't unmarshall into individ waltOBJ, %v", err)
	}
	waltersObject := w.Items[0]
	waltersObject.Beacon = Beacon{formResponse.ProxID, formResponse.MajorID, formResponse.MinorID}
	waltersObject.ImageURL = waltersImagePrefix + waltersObject.Images + waltersImagePostfix
	waltersObject.CuratorComment = formResponse.Description
	err = Insert("beacon", waltersObject.Beacon)
	if err != nil {
		//Return a 500
		return 500, h, nil, errors.New("Couldn't insert corresponding beacon.'")
	}

	err = Insert("art", waltersObject)
	if err != nil {
		//Return a 500
		return 500, h, nil, errors.New("Couldn't insert corresponding art piece.'")
	}

	return 200, h, nil, nil
}

func handleBeacon(u *url.URL, h http.Header, _ interface{}) (status int, responseHeaders http.Header, waltersObj *WalterObj, err error) {
	minorID := u.Query().Get("minorID")
	log.Println(minorID)
	beacon, err := SearchBeaconByID(minorID, 0, -1)
	if err != nil {
		//Beacon not found return 404
		return 404, responseHeaders, nil, errors.New("Beacon not found in db")
	}
	for _, i := range beacon {
		arts, error := SearchArtByBeacon(i, 0, -1)
		if error != nil {
			return 404, responseHeaders, nil, errors.New("Beacon not assigned to art piece")
		}
		return 200, responseHeaders, &arts[0], nil

	}
	//	responseHeaders.Add("Accept","application/json")
	return 404, responseHeaders, nil, errors.New("Beacon not found in db")
	//Send a get request to api with keyword param
}

func initDB() {
	s.DBURI = DBURI
	s.dbName = dbName
	s.getSession()
	s.Session.SetSafe(&mgo.Safe{})
	s.Session.SetMode(mgo.Monotonic, true)
	cNames, errors := EnsureIndex(CollectionNames, indices...)
	for k, err := range errors {
		if err != nil {
			log.Fatalf("Indices not taking for %v;%v\n", cNames[k], err)
		}
	}
}

//EnsureIndex ensures that when we store things, we get the expected results
func EnsureIndex(collectionNames []string, indices ...mgo.Index) (s []string, e []error) {
	for j, k := range indices {
		function := func(c *mgo.Collection) error {
			return c.EnsureIndex(k)
		}
		err := withCollection(collectionNames[j], function)
		if err != nil {
			s = append(s, collectionNames[j])
			e = append(e, err)
		}
	}
	return
}

func (s *Server) getSession() *mgo.Session {
	if s.Session == nil {
		var err error
		dialInfo := &mgo.DialInfo{
			Addrs:    []string{s.DBURI},
			Direct:   true,
			FailFast: false,
		}
		s.Session, err = mgo.DialWithInfo(dialInfo)
		if err != nil {
			log.Fatalf("Can't find MongoDB. Is it started? %v\n", err)
		}
	}
	// Returns a copy of the session so we don't waste le resources? Doesn't reuse socket however
	return s.Session.Copy()
}

func withCollection(collection string, fn func(*mgo.Collection) error) error {
	session := s.getSession()
	defer session.Close()
	c := session.DB(s.dbName).C(collection)
	return fn(c)
}

//Insert datum into a specific collection
func Insert(collectionName string, values ...interface{}) error {
	function := func(c *mgo.Collection) error {
		err := c.Insert(values...)
		if err != nil {
			log.Printf("Can't insert document, %v\n", err)
		}
		return err
	}
	return withCollection(collectionName, function)
}

//SearchBeacon searches for a beacon using a passed in struct
func SearchBeacon(q interface{}, skip int, limit int) (searchResults []Beacon, err error) {
	searchResults = []Beacon{}
	query := func(c *mgo.Collection) error {
		function := c.Find(q).Skip(skip).Limit(limit).All(&searchResults)
		if limit < 0 {
			function = c.Find(q).Skip(skip).All(&searchResults)
		}
		return function
	}
	search := func() error {
		return withCollection("beacon", query)
	}
	err = search()
	return
}

//Estimote id
//

//SearchBeaconByID is a
func SearchBeaconByID(beacon string, skip int, limit int) (searchResults []Beacon, err error) {
	if len(beacon) == 20 {
		return SearchBeacon(bson.M{"MinorID": beacon}, skip, limit)
	}
	return nil, errors.New("Not long enough to be a beacon")
}

//SearchArt is a
func SearchArt(q interface{}, skip int, limit int) (searchResults []WalterObj, err error) {
	searchResults = []WalterObj{}
	query := func(c *mgo.Collection) error {
		function := c.Find(q).Skip(skip).Limit(limit).All(&searchResults)
		if limit < 0 {
			function = c.Find(q).Skip(skip).All(&searchResults)
		}
		return function
	}
	search := func() error {
		return withCollection("art", query)
	}
	err = search()
	return
}

//SearchArtByBeacon is a specific version of searchArt
func SearchArtByBeacon(beacon Beacon, skip int, limit int) (searchResults []WalterObj, err error) {
	return SearchArt(bson.M{"Beacon": beacon}, skip, limit)
}
<commit_msg>POST and GET. I'm BAE of the night.<commit_after>package main

import (
	"encoding/json"
	"log"
	"net/http"
	"strconv"
	"strings"
	"time"

	"github.com/gorilla/mux"
	"gopkg.in/mgo.v2"
	"gopkg.in/mgo.v2/bson"
)

//curl -H "Content-Type: application/json" -XPUT -d '{"P"}' -f -v http://localhost:3000/beacon/{id}

const (
	dbName = "artroomServer"
	//DBURI for the mongodb
	DBURI = "127.0.0.1"

	waltersAPIPrefix    = "http://api.thewalters.org/v1/objects?apikey=ShxvahaBFNIcfWR7E78xXdssKIlXtUAJk9rDDrrmlvbOlxQKtASCzV4op5aHv2Il&title="
	waltersImagePrefix  = "http://static.thewalters.org/images/"
	waltersImagePostfix = "?width=500"
)

var (
	s = Server{}
	//CollectionNames is the shiz
	CollectionNames = []string{"beacon", "art"}
	beaconIndex     = mgo.Index{
		Key:        []string{"MinorID"},
		Unique:     true,
		DropDups:   true,
		Background: true,
		Sparse:     true,
		Name:       "beaconIndex",
		//ExpireAf
	}

	artIndex = mgo.Index{
		Key:        []string{"Beacon", "Title"},
		Unique:     true,
		DropDups:   true,
		Background: true,
		Sparse:     true,
		Name:       "ArtIndex",
		//ExpireAf
	}
	indices = []mgo.Index{beaconIndex, artIndex}
)

type (
	//Server is the name for the server deal wit it
	Server struct {
		Session *mgo.Session // The main session we'll we be cloning
		DBURI   string       // Where the DB is on the network
		dbName  string       // Name of the MongoDB
	}

	//Beacon is the struct that structures what the data for a beacon will look like
	Beacon struct {
		ProxID  string
		MajorID int
		MinorID int
	}
	//FormResponse is what we get back from the form
	FormResponse struct {
		Date        string
		Author      string
		Title       string
		ProxID      string
		MajorID     int
		MinorID     int
		Description string
	}

	//The structure the walter api gives us more or less
	WalterObj struct {
		ObjectID       int
		Collection     string
		Title          string
		Author         string
		Medium         string
		Description    string
		Images         string
		CuratorComment string
		Beacon         Beacon
		ImageURL       string
	}
	//Walters objects
	Walters struct {
		Items []WalterObj
	}
)

func main() {
	log.Println("Server is warming up...")
	initDB()
	defer s.Session.Close()
	http.Handle("/", initHandlers())
	log.Fatalln(http.ListenAndServe(":3000", nil))
}

func initHandlers() *mux.Router {
	r := mux.NewRouter()
	r.HandleFunc("/beacon", handlePOSTBeacon).Methods("POST", "OPTIONS")
	r.HandleFunc("/beacon/{minorID:[0-9]+}", handleBeacon).Methods("GET")
	//	cors := tigertonic.NewCORSBuilder().AddAllowedOrigins("*").AddAllowedHeaders("Access-Control-Allow-Headers", )
	//
	return r
}

func handlePOSTBeacon(w http.ResponseWriter, req *http.Request) {
	log.Println("I got here?")
	switch req.Method {
	case "OPTIONS":
		w.Header().Set("Access-Control-Allow-Headers", "Origin, X-Requested-With, Content-Type, Accept")
		w.Header().Set("Access-Control-Allow-Origin", "*")
		w.Header().Set("Content-Type", "application/json; charset=utf-8")
		fallthrough
	case "POST":
		//		w.Header().Set("Access-Control-Allow-Origin", "*")
		var formResponse FormResponse
		err := ReadJSON(req, &formResponse)
		log.Printf("We have begun the ritual, %v", formResponse)
		client := &http.Client{}
		request, _ := http.NewRequest("GET", waltersAPIPrefix+strings.Replace(formResponse.Title, " ", "%20", -1), nil)
		request.Header.Add("Accept", "application/json")
		res, err := client.Do(request)
		if err != nil {
			http.Error(w, "Req didn't go through", 500)
			return
			//		log.Fatalf("Req didn't go through, %v", err)
		}
		defer res.Body.Close()
		var data json.RawMessage
		err = json.NewDecoder(res.Body).Decode(&data)
		if err != nil {
			http.Error(w, "Couldn't decode data...", 500)
			return
			//		log.Fatalf("Can't resolve the datum?%v", err)
		}
		wal := Walters{}
		err = json.Unmarshal(data, &wal)
		if err != nil {
			http.Error(w, "Couldn't marshall into individual waltOBJ.", 500)
			return
			//		log.Fatalf("Can't unmarshall into individ waltOBJ, %v", err)
		}
		if len(wal.Items) < 1 {
			http.Error(w, "Couldn't marshall correctly", 500)
			return
		}
		waltersObject := wal.Items[0]
		waltersObject.Beacon = Beacon{formResponse.ProxID, formResponse.MajorID, formResponse.MinorID}
		waltersObject.ImageURL = waltersImagePrefix + string(waltersObject.Images[0]) + waltersImagePostfix
		waltersObject.CuratorComment = formResponse.Description
		err = Insert("beacon", waltersObject.Beacon)
		if err != nil {
			//Return a 500
			http.Error(w, "Couldn't insert corresponding beacon.", 500)
			return
		}

		err = Insert("art", waltersObject)
		if err != nil {
			//Return a 500
			http.Error(w, "Couldn't insert corresponding art piece.", 500)
			return
		}
		http.Error(w, http.StatusText(http.StatusCreated), http.StatusCreated)
		break
	}
}

func handleBeacon(w http.ResponseWriter, req *http.Request) {
	params := mux.Vars(req)
	minorID, err := strconv.Atoi(params["minorID"])
	if err != nil {
		http.Error(w, "Beacon can't be found with invalid id", 404)
	}
	log.Println(minorID)
	beacon, err := SearchBeaconByID(minorID, 0, -1)
	log.Printf("%v", beacon)
	if err != nil {
		//Beacon not found return 404
		http.Error(w, "Beacon not found in db", 404)
	}
	for _, i := range beacon {
		arts, error := SearchArtByBeacon(i, 0, -1)
		if error != nil {
			http.Error(w, "Beacon assigned to art piece", 404)
		}
		ServeJSON(w, arts)

	}
	//	responseHeaders.Add("Accept","application/json")
	http.Error(w, "Beacon not found", 404)
	//Send a get request to api with keyword param
}

func initDB() {
	s.DBURI = DBURI
	s.dbName = dbName
	s.getSession()
	s.Session.SetSafe(&mgo.Safe{})
	s.Session.SetMode(mgo.Monotonic, true)
	cNames, errors := EnsureIndex(CollectionNames, indices...)
	for k, err := range errors {
		if err != nil {
			log.Fatalf("Indices not taking for %v;%v\n", cNames[k], err)
		}
	}
}

//EnsureIndex ensures that when we store things, we get the expected results
func EnsureIndex(collectionNames []string, indices ...mgo.Index) (s []string, e []error) {
	for j, k := range indices {
		function := func(c *mgo.Collection) error {
			return c.EnsureIndex(k)
		}
		err := withCollection(collectionNames[j], function)
		if err != nil {
			s = append(s, collectionNames[j])
			e = append(e, err)
		}
	}
	return
}

func (s *Server) getSession() *mgo.Session {
	if s.Session == nil {
		var err error
		dialInfo := &mgo.DialInfo{
			Addrs:    []string{s.DBURI},
			Direct:   true,
			FailFast: false,
		}
		s.Session, err = mgo.DialWithInfo(dialInfo)
		if err != nil {
			log.Fatalf("Can't find MongoDB. Is it started? %v\n", err)
		}
	}
	// Returns a copy of the session so we don't waste le resources? Doesn't reuse socket however
	return s.Session.Copy()
}

func withCollection(collection string, fn func(*mgo.Collection) error) error {
	session := s.getSession()
	defer session.Close()
	c := session.DB(s.dbName).C(collection)
	return fn(c)
}

//Insert datum into a specific collection
func Insert(collectionName string, values ...interface{}) error {
	function := func(c *mgo.Collection) error {
		err := c.Insert(values...)
		if err != nil {
			log.Printf("Can't insert document, %v\n", err)
		}
		return err
	}
	return withCollection(collectionName, function)
}

//SearchBeacon searches for a beacon using a passed in struct
func SearchBeacon(q interface{}, skip int, limit int) (searchResults []Beacon, err error) {
	searchResults = []Beacon{}
	query := func(c *mgo.Collection) error {
		function := c.Find(q).Skip(skip).Limit(limit).All(&searchResults)
		if limit < 0 {
			function = c.Find(q).Skip(skip).All(&searchResults)
		}
		return function
	}
	search := func() error {
		return withCollection("beacon", query)
	}
	err = search()
	return
}

//Estimote id
//

//SearchBeaconByID is a
func SearchBeaconByID(minorID int, skip int, limit int) (searchResults []Beacon, err error) {
	//	if len(beacon) == 20 {
	return SearchBeacon(bson.M{"minorid": minorID}, skip, limit)
	//	}
	//	return nil, errors.New("Not long enough to be a beacon")
}

//SearchArt is a
func SearchArt(q interface{}, skip int, limit int) (searchResults []WalterObj, err error) {
	searchResults = []WalterObj{}
	query := func(c *mgo.Collection) error {
		function := c.Find(q).Skip(skip).Limit(limit).All(&searchResults)
		if limit < 0 {
			function = c.Find(q).Skip(skip).All(&searchResults)
		}
		return function
	}
	search := func() error {
		return withCollection("art", query)
	}
	err = search()
	return
}

//SearchArtByBeacon is a specific version of searchArt
func SearchArtByBeacon(beacon Beacon, skip int, limit int) (searchResults []WalterObj, err error) {
	return SearchArt(bson.M{"beacon": beacon}, skip, limit)
}

// ServeJSON replies to the request with a JSON
// representation of resource v.
func ServeJSON(w http.ResponseWriter, v interface{}) {
	// avoid json vulnerabilities, always wrap v in an object literal
	//	doc := map[string]interface{}{"d": v}
	if data, err := json.Marshal(v); err != nil {
		log.Printf("Error marshalling json: %v", err)
		http.Error(w, err.Error(), http.StatusInternalServerError)
	} else {
		w.Header().Set("Content-Length", strconv.Itoa(len(data)))
		w.Header().Set("Content-Type", "application/json; charset=utf-8")
		w.Write(data)
	}
}

// ReadJSON decodes JSON data into a provided struct which must be passed in as a pointer.
//If it's not a pointer you are basically putting your data into a bottomless gorge and willing it to
//show up right next to you. Just no.
func ReadJSON(req *http.Request, v interface{}) error {
	defer req.Body.Close()
	decoder := json.NewDecoder(req.Body)
	err := decoder.Decode(v)
	return err
}

//Use this method to debug things
func logRequest(handler func(http.ResponseWriter, *http.Request)) func(http.ResponseWriter, *http.Request) {
	return func(w http.ResponseWriter, r *http.Request) {
		var s = time.Now()
		handler(w, r)
		log.Printf("%s %s %6.3fms", r.Method, r.RequestURI, (time.Since(s).Seconds() * 1000))
	}
}
<|endoftext|>


<commit_before>package rpc

import (
	"errors"
	"github.com/mitchellh/packer/packer"
	"io"
	"log"
	"net"
	"net/rpc"
)

// An implementation of packer.Communicator where the communicator is actually
// executed over an RPC connection.
type communicator struct {
	client *rpc.Client
}

// CommunicatorServer wraps a packer.Communicator implementation and makes
// it exportable as part of a Golang RPC server.
type CommunicatorServer struct {
	c packer.Communicator
}

// RemoteCommandServer wraps a packer.RemoteCommand struct and makes it
// exportable as part of a Golang RPC server.
type RemoteCommandServer struct {
	rc *packer.RemoteCommand
}

type CommunicatorStartResponse struct {
	StdinAddress         string
	StdoutAddress        string
	StderrAddress        string
	RemoteCommandAddress string
}

type CommunicatorDownloadArgs struct {
	Path          string
	WriterAddress string
}

type CommunicatorUploadArgs struct {
	Path          string
	ReaderAddress string
}

func Communicator(client *rpc.Client) *communicator {
	return &communicator{client}
}

func (c *communicator) Start(cmd string) (rc *packer.RemoteCommand, err error) {
	var response CommunicatorStartResponse
	err = c.client.Call("Communicator.Start", &cmd, &response)
	if err != nil {
		return
	}

	// Connect to the three streams that will handle stdin, stdout,
	// and stderr and get net.Conns for them.
	stdinC, err := net.Dial("tcp", response.StdinAddress)
	if err != nil {
		return
	}

	stdoutC, err := net.Dial("tcp", response.StdoutAddress)
	if err != nil {
		return
	}

	stderrC, err := net.Dial("tcp", response.StderrAddress)
	if err != nil {
		return
	}

	// Connect to the RPC server for the remote command
	client, err := rpc.Dial("tcp", response.RemoteCommandAddress)
	if err != nil {
		return
	}

	// Build the response object using the streams we created
	rc = &packer.RemoteCommand{
		stdinC,
		stdoutC,
		stderrC,
		false,
		-1,
	}

	// In a goroutine, we wait for the process to exit, then we set
	// that it has exited.
	go func() {
		client.Call("RemoteCommand.Wait", new(interface{}), &rc.ExitStatus)
		rc.Exited = true
	}()

	return
}

func (c *communicator) Upload(path string, r io.Reader) (err error) {
	// We need to create a server that can proxy the reader data
	// over because we can't simply gob encode an io.Reader
	readerL := netListenerInRange(portRangeMin, portRangeMax)
	if readerL == nil {
		err = errors.New("couldn't allocate listener for upload reader")
		return
	}

	// Make sure at the end of this call, we close the listener
	defer readerL.Close()

	// Pipe the reader through to the connection
	go serveSingleCopy("uploadReader", readerL, nil, r)

	args := CommunicatorUploadArgs{
		path,
		readerL.Addr().String(),
	}

	err = c.client.Call("Communicator.Upload", &args, new(interface{}))
	return
}

func (c *communicator) Download(path string, w io.Writer) (err error) {
	// We need to create a server that can proxy that data downloaded
	// into the writer because we can't gob encode a writer directly.
	writerL := netListenerInRange(portRangeMin, portRangeMax)
	if writerL == nil {
		err = errors.New("couldn't allocate listener for download writer")
		return
	}

	// Make sure we close the listener once we're done because we'll be done
	defer writerL.Close()

	// Serve a single connection and a single copy
	go serveSingleCopy("downloadWriter", writerL, w, nil)

	args := CommunicatorDownloadArgs{
		path,
		writerL.Addr().String(),
	}

	err = c.client.Call("Communicator.Download", &args, new(interface{}))
	return
}

func (c *CommunicatorServer) Start(cmd *string, reply *CommunicatorStartResponse) (err error) {
	// Start executing the command.
	command, err := c.c.Start(*cmd)
	if err != nil {
		return
	}

	// If we didn't get a proper command... that's not right.
	if command == nil {
		return errors.New("communicator returned nil remote command")
	}

	// Next, we need to take the stdin/stdout and start a listener
	// for each because the client will connect to us via TCP and use
	// that connection as the io.Reader or io.Writer. These exist for
	// only a single connection that is persistent.
	stdinL := netListenerInRange(portRangeMin, portRangeMax)
	stdoutL := netListenerInRange(portRangeMin, portRangeMax)
	stderrL := netListenerInRange(portRangeMin, portRangeMax)
	go serveSingleCopy("stdin", stdinL, command.Stdin, nil)
	go serveSingleCopy("stdout", stdoutL, nil, command.Stdout)
	go serveSingleCopy("stderr", stderrL, nil, command.Stderr)

	// For the exit status, we use a simple RPC Server that serves
	// some of the RemoteComand methods.
	server := rpc.NewServer()
	server.RegisterName("RemoteCommand", &RemoteCommandServer{command})

	*reply = CommunicatorStartResponse{
		stdinL.Addr().String(),
		stdoutL.Addr().String(),
		stderrL.Addr().String(),
		serveSingleConn(server),
	}

	return
}

func (c *CommunicatorServer) Upload(args *CommunicatorUploadArgs, reply *interface{}) (err error) {
	readerC, err := net.Dial("tcp", args.ReaderAddress)
	if err != nil {
		return
	}

	defer readerC.Close()

	err = c.c.Upload(args.Path, readerC)
	return
}

func (c *CommunicatorServer) Download(args *CommunicatorDownloadArgs, reply *interface{}) (err error) {
	writerC, err := net.Dial("tcp", args.WriterAddress)
	if err != nil {
		return
	}

	defer writerC.Close()

	err = c.c.Download(args.Path, writerC)
	return
}

func (rc *RemoteCommandServer) Wait(args *interface{}, reply *int) error {
	rc.rc.Wait()
	*reply = rc.rc.ExitStatus
	return nil
}

func serveSingleCopy(name string, l net.Listener, dst io.Writer, src io.Reader) {
	defer l.Close()

	conn, err := l.Accept()
	if err != nil {
		return
	}

	// The connection is the destination/source that is nil
	if dst == nil {
		dst = conn
	} else {
		src = conn
	}

	written, err := io.Copy(dst, src)
	log.Printf("%d bytes written for '%s'", written, name)
	if err != nil {
		log.Printf("'%s' copy error: %s", name, err.Error())
	}
}
<commit_msg>packer/rpc: Better logging in communicator<commit_after>package rpc

import (
	"errors"
	"github.com/mitchellh/packer/packer"
	"io"
	"log"
	"net"
	"net/rpc"
)

// An implementation of packer.Communicator where the communicator is actually
// executed over an RPC connection.
type communicator struct {
	client *rpc.Client
}

// CommunicatorServer wraps a packer.Communicator implementation and makes
// it exportable as part of a Golang RPC server.
type CommunicatorServer struct {
	c packer.Communicator
}

// RemoteCommandServer wraps a packer.RemoteCommand struct and makes it
// exportable as part of a Golang RPC server.
type RemoteCommandServer struct {
	rc *packer.RemoteCommand
}

type CommunicatorStartResponse struct {
	StdinAddress         string
	StdoutAddress        string
	StderrAddress        string
	RemoteCommandAddress string
}

type CommunicatorDownloadArgs struct {
	Path          string
	WriterAddress string
}

type CommunicatorUploadArgs struct {
	Path          string
	ReaderAddress string
}

func Communicator(client *rpc.Client) *communicator {
	return &communicator{client}
}

func (c *communicator) Start(cmd string) (rc *packer.RemoteCommand, err error) {
	var response CommunicatorStartResponse
	err = c.client.Call("Communicator.Start", &cmd, &response)
	if err != nil {
		return
	}

	// Connect to the three streams that will handle stdin, stdout,
	// and stderr and get net.Conns for them.
	stdinC, err := net.Dial("tcp", response.StdinAddress)
	if err != nil {
		return
	}

	stdoutC, err := net.Dial("tcp", response.StdoutAddress)
	if err != nil {
		return
	}

	stderrC, err := net.Dial("tcp", response.StderrAddress)
	if err != nil {
		return
	}

	// Connect to the RPC server for the remote command
	client, err := rpc.Dial("tcp", response.RemoteCommandAddress)
	if err != nil {
		return
	}

	// Build the response object using the streams we created
	rc = &packer.RemoteCommand{
		stdinC,
		stdoutC,
		stderrC,
		false,
		-1,
	}

	// In a goroutine, we wait for the process to exit, then we set
	// that it has exited.
	go func() {
		client.Call("RemoteCommand.Wait", new(interface{}), &rc.ExitStatus)
		rc.Exited = true
	}()

	return
}

func (c *communicator) Upload(path string, r io.Reader) (err error) {
	// We need to create a server that can proxy the reader data
	// over because we can't simply gob encode an io.Reader
	readerL := netListenerInRange(portRangeMin, portRangeMax)
	if readerL == nil {
		err = errors.New("couldn't allocate listener for upload reader")
		return
	}

	// Make sure at the end of this call, we close the listener
	defer readerL.Close()

	// Pipe the reader through to the connection
	go serveSingleCopy("uploadReader", readerL, nil, r)

	args := CommunicatorUploadArgs{
		path,
		readerL.Addr().String(),
	}

	err = c.client.Call("Communicator.Upload", &args, new(interface{}))
	return
}

func (c *communicator) Download(path string, w io.Writer) (err error) {
	// We need to create a server that can proxy that data downloaded
	// into the writer because we can't gob encode a writer directly.
	writerL := netListenerInRange(portRangeMin, portRangeMax)
	if writerL == nil {
		err = errors.New("couldn't allocate listener for download writer")
		return
	}

	// Make sure we close the listener once we're done because we'll be done
	defer writerL.Close()

	// Serve a single connection and a single copy
	go serveSingleCopy("downloadWriter", writerL, w, nil)

	args := CommunicatorDownloadArgs{
		path,
		writerL.Addr().String(),
	}

	err = c.client.Call("Communicator.Download", &args, new(interface{}))
	return
}

func (c *CommunicatorServer) Start(cmd *string, reply *CommunicatorStartResponse) (err error) {
	// Start executing the command.
	command, err := c.c.Start(*cmd)
	if err != nil {
		return
	}

	// If we didn't get a proper command... that's not right.
	if command == nil {
		return errors.New("communicator returned nil remote command")
	}

	// Next, we need to take the stdin/stdout and start a listener
	// for each because the client will connect to us via TCP and use
	// that connection as the io.Reader or io.Writer. These exist for
	// only a single connection that is persistent.
	stdinL := netListenerInRange(portRangeMin, portRangeMax)
	stdoutL := netListenerInRange(portRangeMin, portRangeMax)
	stderrL := netListenerInRange(portRangeMin, portRangeMax)
	go serveSingleCopy("stdin", stdinL, command.Stdin, nil)
	go serveSingleCopy("stdout", stdoutL, nil, command.Stdout)
	go serveSingleCopy("stderr", stderrL, nil, command.Stderr)

	// For the exit status, we use a simple RPC Server that serves
	// some of the RemoteComand methods.
	server := rpc.NewServer()
	server.RegisterName("RemoteCommand", &RemoteCommandServer{command})

	*reply = CommunicatorStartResponse{
		stdinL.Addr().String(),
		stdoutL.Addr().String(),
		stderrL.Addr().String(),
		serveSingleConn(server),
	}

	return
}

func (c *CommunicatorServer) Upload(args *CommunicatorUploadArgs, reply *interface{}) (err error) {
	readerC, err := net.Dial("tcp", args.ReaderAddress)
	if err != nil {
		return
	}

	defer readerC.Close()

	err = c.c.Upload(args.Path, readerC)
	return
}

func (c *CommunicatorServer) Download(args *CommunicatorDownloadArgs, reply *interface{}) (err error) {
	writerC, err := net.Dial("tcp", args.WriterAddress)
	if err != nil {
		return
	}

	defer writerC.Close()

	err = c.c.Download(args.Path, writerC)
	return
}

func (rc *RemoteCommandServer) Wait(args *interface{}, reply *int) error {
	rc.rc.Wait()
	*reply = rc.rc.ExitStatus
	return nil
}

func serveSingleCopy(name string, l net.Listener, dst io.Writer, src io.Reader) {
	defer l.Close()

	conn, err := l.Accept()
	if err != nil {
		log.Printf("'%s' accept error: %s", name, err)
		return
	}

	// The connection is the destination/source that is nil
	if dst == nil {
		dst = conn
	} else {
		src = conn
	}

	written, err := io.Copy(dst, src)
	log.Printf("%d bytes written for '%s'", written, name)
	if err != nil {
		log.Printf("'%s' copy error: %s", name, err)
	}
}
<|endoftext|>