<commit_before>package air

import (
	"net"
	"sync"
	"time"

	"github.com/valyala/fasthttp"
)

type (
	// Server defines the interface for HTTP server.
	Server interface {
		// SetHandler sets the handler for the HTTP server.
		SetHandler(Handler)

		// SetLogger sets the logger for the HTTP server.
		SetLogger(Logger)

		// Start starts the HTTP server.
		Start() error
	}

	// FastServer implements `Server`.
	FastServer struct {
		*fasthttp.Server
		config  Config
		handler Handler
		logger  Logger
		pool    *pool
	}

	// Config defines fasthttp config.
	Config struct {
		Address      string        // TCP address to listen on.
		Listener     net.Listener  // Custom `net.Listener`. If set, server accepts connections on it.
		TLSCertFile  string        // TLS certificate file path.
		TLSKeyFile   string        // TLS key file path.
		ReadTimeout  time.Duration // Maximum duration before timing out read of the request.
		WriteTimeout time.Duration // Maximum duration before timing out write of the response.
	}

	// Handler defines an interface to server HTTP requests via `ServeHTTP(Request, Response)`
	// function.
	Handler interface {
		ServeHTTP(Request, Response)
	}

	// FastHandlerFunc is an adapter to allow the use of `func(Request, Response)` as
	// an HTTP handler.
	FastHandlerFunc func(Request, Response)

	pool struct {
		request        sync.Pool
		response       sync.Pool
		requestHeader  sync.Pool
		responseHeader sync.Pool
		uri            sync.Pool
	}
)

// ServeHTTP serves HTTP request.
func (h FastHandlerFunc) ServeHTTP(req Request, res Response) {
	h(req, res)
}

// NewServer returns `FastServer` with provided listen address.
func NewServer(addr string) *FastServer {
	c := Config{Address: addr}
	return WithConfig(c)
}

// WithTLS returns `FastServer` with provided TLS config.
func WithTLS(addr, certFile, keyFile string) *FastServer {
	c := Config{
		Address:     addr,
		TLSCertFile: certFile,
		TLSKeyFile:  keyFile,
	}
	return WithConfig(c)
}

// WithConfig returns `FastServer` with provided config.
func WithConfig(c Config) (s *FastServer) {
	s = &FastServer{
		Server: new(fasthttp.Server),
		config: c,
		pool: &pool{
			request: sync.Pool{
				New: func() interface{} {
					return &FastRequest{logger: s.logger}
				},
			},
			response: sync.Pool{
				New: func() interface{} {
					return &FastResponse{logger: s.logger}
				},
			},
			requestHeader: sync.Pool{
				New: func() interface{} {
					return &FastRequestHeader{}
				},
			},
			responseHeader: sync.Pool{
				New: func() interface{} {
					return &FastResponseHeader{}
				},
			},
			uri: sync.Pool{
				New: func() interface{} {
					return &FastURI{}
				},
			},
		},
		handler: FastHandlerFunc(func(req Request, res Response) {
			s.logger.Error("handler not set, use `SetHandler()` to set it.")
		}),
		logger: NewLogger("air"),
	}
	s.ReadTimeout = c.ReadTimeout
	s.WriteTimeout = c.WriteTimeout
	s.Handler = s.ServeHTTP
	return
}

// SetHandler implements `Server#SetHandler` function.
func (s *FastServer) SetHandler(h Handler) {
	s.handler = h
}

// SetLogger implements `Server#SetLogger` function.
func (s *FastServer) SetLogger(l Logger) {
	s.logger = l
}

// Start implements `Server#Start` function.
func (s *FastServer) Start() error {
	if s.config.Listener == nil {
		return s.startDefaultListener()
	}
	return s.startCustomListener()

}

func (s *FastServer) startDefaultListener() error {
	c := s.config
	if c.TLSCertFile != "" && c.TLSKeyFile != "" {
		return s.ListenAndServeTLS(c.Address, c.TLSCertFile, c.TLSKeyFile)
	}
	return s.ListenAndServe(c.Address)
}

func (s *FastServer) startCustomListener() error {
	c := s.config
	if c.TLSCertFile != "" && c.TLSKeyFile != "" {
		return s.ServeTLS(c.Listener, c.TLSCertFile, c.TLSKeyFile)
	}
	return s.Serve(c.Listener)
}

func (s *FastServer) ServeHTTP(c *fasthttp.RequestCtx) {
	// Request
	req := s.pool.request.Get().(*FastRequest)
	reqHdr := s.pool.requestHeader.Get().(*FastRequestHeader)
	reqURI := s.pool.uri.Get().(*FastURI)
	reqHdr.reset(&c.Request.Header)
	reqURI.reset(c.URI())
	req.reset(c, reqHdr, reqURI)

	// Response
	res := s.pool.response.Get().(*FastResponse)
	resHdr := s.pool.responseHeader.Get().(*FastResponseHeader)
	resHdr.reset(&c.Response.Header)
	res.reset(c, resHdr)

	s.handler.ServeHTTP(req, res)

	// Return to pool
	s.pool.request.Put(req)
	s.pool.requestHeader.Put(reqHdr)
	s.pool.uri.Put(reqURI)
	s.pool.response.Put(res)
	s.pool.responseHeader.Put(resHdr)
}

// FastWrapHandler wraps `fasthttp.RequestHandler` into `HandlerFunc`.
func FastWrapHandler(h fasthttp.RequestHandler) HandlerFunc {
	return func(c Context) error {
		req := c.Request().(*FastRequest)
		res := c.Response().(*FastResponse)
		ctx := req.RequestCtx
		h(ctx)
		res.status = ctx.Response.StatusCode()
		res.size = int64(ctx.Response.Header.ContentLength())
		return nil
	}
}

// FastWrapGas wraps `func(fasthttp.RequestHandler) fasthttp.RequestHandler`
// into `GasFunc`
func FastWrapGas(m func(fasthttp.RequestHandler) fasthttp.RequestHandler) GasFunc {
	return func(next HandlerFunc) HandlerFunc {
		return func(c Context) (err error) {
			req := c.Request().(*FastRequest)
			res := c.Response().(*FastResponse)
			ctx := req.RequestCtx
			m(func(ctx *fasthttp.RequestCtx) {
				next(c)
			})(ctx)
			res.status = ctx.Response.StatusCode()
			res.size = int64(ctx.Response.Header.ContentLength())
			return
		}
	}
}
<commit_msg>Format server.go<commit_after>package air

import (
	"net"
	"sync"
	"time"

	"github.com/valyala/fasthttp"
)

type (
	// Server defines the interface for HTTP server.
	Server interface {
		// SetHandler sets the handler for the HTTP server.
		SetHandler(Handler)

		// SetLogger sets the logger for the HTTP server.
		SetLogger(Logger)

		// Start starts the HTTP server.
		Start() error
	}

	// FastServer implements `Server`.
	FastServer struct {
		*fasthttp.Server
		config  Config
		handler Handler
		logger  Logger
		pool    *pool
	}

	// Config defines fasthttp config.
	Config struct {
		Address      string        // TCP address to listen on.
		Listener     net.Listener  // Custom `net.Listener`. If set, server accepts connections on it.
		TLSCertFile  string        // TLS certificate file path.
		TLSKeyFile   string        // TLS key file path.
		ReadTimeout  time.Duration // Maximum duration before timing out read of the request.
		WriteTimeout time.Duration // Maximum duration before timing out write of the response.
	}

	// Handler defines an interface to server HTTP requests via `ServeHTTP(Request, Response)`
	// function.
	Handler interface {
		ServeHTTP(Request, Response)
	}

	// FastHandlerFunc is an adapter to allow the use of `func(Request, Response)` as
	// an HTTP handler.
	FastHandlerFunc func(Request, Response)

	pool struct {
		request        sync.Pool
		response       sync.Pool
		requestHeader  sync.Pool
		responseHeader sync.Pool
		uri            sync.Pool
	}
)

// ServeHTTP serves HTTP request.
func (h FastHandlerFunc) ServeHTTP(req Request, res Response) {
	h(req, res)
}

// NewServer returns `FastServer` with provided listen address.
func NewServer(addr string) *FastServer {
	c := Config{Address: addr}
	return WithConfig(c)
}

// WithTLS returns `FastServer` with provided TLS config.
func WithTLS(addr, certFile, keyFile string) *FastServer {
	c := Config{
		Address:     addr,
		TLSCertFile: certFile,
		TLSKeyFile:  keyFile,
	}
	return WithConfig(c)
}

// WithConfig returns `FastServer` with provided config.
func WithConfig(c Config) *FastServer {
	s := &FastServer{
		Server: new(fasthttp.Server),
		config: c,
		pool: &pool{
			request: sync.Pool{
				New: func() interface{} {
					return &FastRequest{logger: s.logger}
				},
			},
			response: sync.Pool{
				New: func() interface{} {
					return &FastResponse{logger: s.logger}
				},
			},
			requestHeader: sync.Pool{
				New: func() interface{} {
					return &FastRequestHeader{}
				},
			},
			responseHeader: sync.Pool{
				New: func() interface{} {
					return &FastResponseHeader{}
				},
			},
			uri: sync.Pool{
				New: func() interface{} {
					return &FastURI{}
				},
			},
		},
		handler: FastHandlerFunc(func(req Request, res Response) {
			s.logger.Error("handler not set, use `SetHandler()` to set it.")
		}),
		logger: NewLogger("air"),
	}
	s.ReadTimeout = c.ReadTimeout
	s.WriteTimeout = c.WriteTimeout
	s.Handler = s.ServeHTTP
	return
}

// SetHandler implements `Server#SetHandler` function.
func (s *FastServer) SetHandler(h Handler) {
	s.handler = h
}

// SetLogger implements `Server#SetLogger` function.
func (s *FastServer) SetLogger(l Logger) {
	s.logger = l
}

// Start implements `Server#Start` function.
func (s *FastServer) Start() error {
	if s.config.Listener == nil {
		return s.startDefaultListener()
	}
	return s.startCustomListener()

}

func (s *FastServer) startDefaultListener() error {
	c := s.config
	if c.TLSCertFile != "" && c.TLSKeyFile != "" {
		return s.ListenAndServeTLS(c.Address, c.TLSCertFile, c.TLSKeyFile)
	}
	return s.ListenAndServe(c.Address)
}

func (s *FastServer) startCustomListener() error {
	c := s.config
	if c.TLSCertFile != "" && c.TLSKeyFile != "" {
		return s.ServeTLS(c.Listener, c.TLSCertFile, c.TLSKeyFile)
	}
	return s.Serve(c.Listener)
}

func (s *FastServer) ServeHTTP(c *fasthttp.RequestCtx) {
	// Request
	req := s.pool.request.Get().(*FastRequest)
	reqHdr := s.pool.requestHeader.Get().(*FastRequestHeader)
	reqURI := s.pool.uri.Get().(*FastURI)
	reqHdr.reset(&c.Request.Header)
	reqURI.reset(c.URI())
	req.reset(c, reqHdr, reqURI)

	// Response
	res := s.pool.response.Get().(*FastResponse)
	resHdr := s.pool.responseHeader.Get().(*FastResponseHeader)
	resHdr.reset(&c.Response.Header)
	res.reset(c, resHdr)

	s.handler.ServeHTTP(req, res)

	// Return to pool
	s.pool.request.Put(req)
	s.pool.requestHeader.Put(reqHdr)
	s.pool.uri.Put(reqURI)
	s.pool.response.Put(res)
	s.pool.responseHeader.Put(resHdr)
}

// FastWrapHandler wraps `fasthttp.RequestHandler` into `HandlerFunc`.
func FastWrapHandler(h fasthttp.RequestHandler) HandlerFunc {
	return func(c Context) error {
		req := c.Request().(*FastRequest)
		res := c.Response().(*FastResponse)
		ctx := req.RequestCtx
		h(ctx)
		res.status = ctx.Response.StatusCode()
		res.size = int64(ctx.Response.Header.ContentLength())
		return nil
	}
}

// FastWrapGas wraps `func(fasthttp.RequestHandler) fasthttp.RequestHandler`
// into `GasFunc`
func FastWrapGas(m func(fasthttp.RequestHandler) fasthttp.RequestHandler) GasFunc {
	return func(next HandlerFunc) HandlerFunc {
		return func(c Context) (err error) {
			req := c.Request().(*FastRequest)
			res := c.Response().(*FastResponse)
			ctx := req.RequestCtx
			m(func(ctx *fasthttp.RequestCtx) {
				next(c)
			})(ctx)
			res.status = ctx.Response.StatusCode()
			res.size = int64(ctx.Response.Header.ContentLength())
			return
		}
	}
}
<|endoftext|>


<commit_before>package main

import (
	"encoding/json"
	"fmt"
	"io"
	"io/ioutil"
	"log"
	"net/http"
	"strconv"
	"time"

	"github.com/influxdata/influxdb/client/v2"
)

// AkamaiPayload is a Golang representation of the Cloudmonitor JSON datastructure
type AkamaiPayload struct {
	// Content Provider ID
	CP string `json:"cp"`
	// Defines the format of the payload (?)
	Format  string        `json:"format"`
	Geo     GeoStruct     `json:"geo"`
	ID      string        `json:"id"`
	Message MessageStruct `json:"message"`
	NetPerf NetPerfStruct `json:"netPerf"`
	Network NetworkStruct `json:"network"`
	ReqHdr  ReqHdrStruct  `json:"reqHdr"`
	RespHdr RespHdrStruct `json:"respHdr"`
	Start   string        `json:"start"`
	Type    string        `json:"type"`
	Version string        `json:"version"`
}

// GeoStruct is used for storing the JSON subfields
type GeoStruct struct {
	City    string `json:"city"`
	Country string `json:"country"`
	Lat     string `json:"lat"`
	Long    string `json:"long"`
	Region  string `json:"region"`
}

// MessageStruct is used for storing the JSON subfields
type MessageStruct struct {
	UA        string `json:"UA"`
	Bytes     string `json:"bytes"`
	CliIP     string `json:"cliIP"`
	FwdHost   string `json:"fwdHost"`
	Proto     string `json:"proto"`
	ProtoVer  string `json:"protoVer"`
	ReqHost   string `json:"reqHost"`
	ReqMethod string `json:"reqMethod"`
	ReqPath   string `json:"reqPath"`
	ReqPort   string `json:"reqPort"`
	RespCT    string `json:"respCT"`
	RespLen   string `json:"respLen"`
	Status    string `json:"status"`
}

// NetPerfStruct is used for storing the JSON subfields
type NetPerfStruct struct {
	Asnum        string `json:"asnum"`
	CacheStatus  string `json:"cacheStatus"`
	DownloadTime string `json:"downloadTime"`
	EdgeIP       string `json:"edgeIP"`
	FirstByte    string `json:"firstByte"`
	LastByte     string `json:"lastByte"`
	LastMileRTT  string `json:"lastMileRTT"`
}

// NetworkStruct is used for storing the JSON subfields
type NetworkStruct struct {
	Asnum       string `json:"asnum"`
	EdgeIP      string `json:"edgeIP"`
	Network     string `json:"network"`
	NetworkType string `json:"networkType"`
}

// ReqHdrStruct is used for storing the JSON subfields
type ReqHdrStruct struct {
	Cookie string `json:"cookie"`
}

// RespHdrStruct is used for storing the JSON subfields
type RespHdrStruct struct {
	Server  string `json:"server"`
	ContEnc string `json:"contEnc"`
}

// CreateObjects creates a list of AkamaiPayloads from a raw byte slice
func CreateObjects(jsonFile []byte) (_ []AkamaiPayload, err error) {
	var arrayObject []AkamaiPayload
	err = json.Unmarshal(jsonFile, &arrayObject)
	return arrayObject, err
}

// Handle parses the incoming request data
func Handle(w http.ResponseWriter, r *http.Request) {
	io.WriteString(w, "OK! I got you Bro! -> ")
	body, err := ioutil.ReadAll(r.Body)
	if err != nil {
		log.Fatal(err)
	}
	obj, err := CreateObjects(body)
	if err != nil {
		log.Fatal(err)
	}
	c, err := client.NewHTTPClient(client.HTTPConfig{
		Addr: "http://influxDB:8086",
	})
	for key, o := range obj {
		//payload[0].Geo["city"]
		fmt.Println("Server is:", o.RespHdr.Server, "KEY:", key)

		if err != nil {
			log.Fatal(err)
		}

		// Create a new point batch
		bp, err := client.NewBatchPoints(client.BatchPointsConfig{
			Database: "metrics",
			//Precision: "s",
		})
		if err != nil {
			log.Fatal(err)
		}

		// Create a point and add to batch
		tags := map[string]string{
<<<<<<< HEAD
			"country": o.Geo.Country,
		}
		fields := map[string]interface{}{

			"lat":  o.Geo.Lat,
			"long": o.Geo.Long,
=======
			"city":    o.Geo.City,
			"country": o.Geo.Country,
		}
		lat, err := strconv.ParseFloat(o.Geo.Lat, 64)
		long, err := strconv.ParseFloat(o.Geo.Long, 64)
		if err != nil {
			log.Fatal(err)
		}
		fields := map[string]interface{}{
			"lat":  lat,
			"long": long,
>>>>>>> 18a2638cf846722aa417c36e554f33b0857f3ec3
		}

		pt, err := client.NewPoint("cp", tags, fields, time.Now())
		if err != nil {
			log.Fatal(err)
		}
		bp.AddPoint(pt)

		// Write the batch
		if err := c.Write(bp); err != nil {
			log.Fatal(err)
		}
	}
}

func main() {

	http.HandleFunc("/", Handle)
	http.ListenAndServe(":9143", nil)
}
<commit_msg>fix incoming changes<commit_after>package main

import (
	"encoding/json"
	"fmt"
	"io"
	"io/ioutil"
	"log"
	"net/http"
	"strconv"
	"time"

	"github.com/influxdata/influxdb/client/v2"
)

// AkamaiPayload is a Golang representation of the Cloudmonitor JSON datastructure
type AkamaiPayload struct {
	// Content Provider ID
	CP string `json:"cp"`
	// Defines the format of the payload (?)
	Format  string        `json:"format"`
	Geo     GeoStruct     `json:"geo"`
	ID      string        `json:"id"`
	Message MessageStruct `json:"message"`
	NetPerf NetPerfStruct `json:"netPerf"`
	Network NetworkStruct `json:"network"`
	ReqHdr  ReqHdrStruct  `json:"reqHdr"`
	RespHdr RespHdrStruct `json:"respHdr"`
	Start   string        `json:"start"`
	Type    string        `json:"type"`
	Version string        `json:"version"`
}

// GeoStruct is used for storing the JSON subfields
type GeoStruct struct {
	City    string `json:"city"`
	Country string `json:"country"`
	Lat     string `json:"lat"`
	Long    string `json:"long"`
	Region  string `json:"region"`
}

// MessageStruct is used for storing the JSON subfields
type MessageStruct struct {
	UA        string `json:"UA"`
	Bytes     string `json:"bytes"`
	CliIP     string `json:"cliIP"`
	FwdHost   string `json:"fwdHost"`
	Proto     string `json:"proto"`
	ProtoVer  string `json:"protoVer"`
	ReqHost   string `json:"reqHost"`
	ReqMethod string `json:"reqMethod"`
	ReqPath   string `json:"reqPath"`
	ReqPort   string `json:"reqPort"`
	RespCT    string `json:"respCT"`
	RespLen   string `json:"respLen"`
	Status    string `json:"status"`
}

// NetPerfStruct is used for storing the JSON subfields
type NetPerfStruct struct {
	Asnum        string `json:"asnum"`
	CacheStatus  string `json:"cacheStatus"`
	DownloadTime string `json:"downloadTime"`
	EdgeIP       string `json:"edgeIP"`
	FirstByte    string `json:"firstByte"`
	LastByte     string `json:"lastByte"`
	LastMileRTT  string `json:"lastMileRTT"`
}

// NetworkStruct is used for storing the JSON subfields
type NetworkStruct struct {
	Asnum       string `json:"asnum"`
	EdgeIP      string `json:"edgeIP"`
	Network     string `json:"network"`
	NetworkType string `json:"networkType"`
}

// ReqHdrStruct is used for storing the JSON subfields
type ReqHdrStruct struct {
	Cookie string `json:"cookie"`
}

// RespHdrStruct is used for storing the JSON subfields
type RespHdrStruct struct {
	Server  string `json:"server"`
	ContEnc string `json:"contEnc"`
}

// CreateObjects creates a list of AkamaiPayloads from a raw byte slice
func CreateObjects(jsonFile []byte) (_ []AkamaiPayload, err error) {
	var arrayObject []AkamaiPayload
	err = json.Unmarshal(jsonFile, &arrayObject)
	return arrayObject, err
}

// Handle parses the incoming request data
func Handle(w http.ResponseWriter, r *http.Request) {
	io.WriteString(w, "OK! I got you Bro! -> ")
	body, err := ioutil.ReadAll(r.Body)
	if err != nil {
		log.Fatal(err)
	}
	obj, err := CreateObjects(body)
	if err != nil {
		log.Fatal(err)
	}
	c, err := client.NewHTTPClient(client.HTTPConfig{
		Addr: "http://influxDB:8086",
	})
	for key, o := range obj {
		//payload[0].Geo["city"]
		fmt.Println("Server is:", o.RespHdr.Server, "KEY:", key)

		if err != nil {
			log.Fatal(err)
		}

		// Create a new point batch
		bp, err := client.NewBatchPoints(client.BatchPointsConfig{
			Database: "metrics",
			//Precision: "s",
		})
		if err != nil {
			log.Fatal(err)
		}

		// Create a point and add to batch
		tags := map[string]string{
			"city":    o.Geo.City,
			"country": o.Geo.Country,
		}
		lat, err := strconv.ParseFloat(o.Geo.Lat, 64)
		long, err := strconv.ParseFloat(o.Geo.Long, 64)
		if err != nil {
			log.Fatal(err)
		}
		fields := map[string]interface{}{
			"lat":  lat,
			"long": long,
		}

		pt, err := client.NewPoint("cp", tags, fields, time.Now())
		if err != nil {
			log.Fatal(err)
		}
		bp.AddPoint(pt)

		// Write the batch
		if err := c.Write(bp); err != nil {
			log.Fatal(err)
		}
	}
}

func main() {

	http.HandleFunc("/", Handle)
	http.ListenAndServe(":9143", nil)
}
<|endoftext|>


<commit_before>// Copyright 2013 Ryan Rogers. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

// Package server provides an easy to use HTTPS server.  It provides some
// benefits over using the standard library directly, such as the ability to
// gracefully shut down active connections, and to do low (zero?) downtime
// restarts.
package server

import (
	"crypto/tls"
	"syscall"
)

// A list of the possible cipher suite ids that are not already defined
// by the crypto/tls package. Taken from
// http://www.iana.org/assignments/tls-parameters/tls-parameters.xml
//
// Note that the reason they are not defined by the crypto/tls package is
// because they are only usable in TLS 1.2, which is not yet supported by Go.
const (
	TLS_RSA_WITH_AES_128_GCM_SHA256       uint16 = 0x009c
	TLS_RSA_WITH_AES_256_GCM_SHA384       uint16 = 0x009d
	TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256 uint16 = 0xc02f
	TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384 uint16 = 0xc030
)

// HTTPS creates a server that will serve HTTP requests using TLS.  If
// reuseListeners (obtained via Detach) is provided and matches an address in
// addrs, an attempt will be made to reuse the listener.  If that fails, it
// will fall back to creating a new listener.
func HTTPS(addrs []string, keyPairs map[string]string, reuseListeners DetachedListeners) error {
	// Load the server's certificates.
	tlsCertificates := make([]tls.Certificate, 0, len(keyPairs))
	for certFile, keyFile := range keyPairs {
		cert, err := tls.LoadX509KeyPair(certFile, keyFile)
		if err != nil {
			return err
		}
		tlsCertificates = append(tlsCertificates, cert)
	}

	var err error
	for _, addr := range addrs {
		// Configure TLS for each individual address to avoid all addresses
		// sharing the same SessionTicketKey.
		tlsConfig := &tls.Config{
			Certificates: tlsCertificates,
			NextProtos:   []string{"http/1.1"},
			CipherSuites: []uint16{
				TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,
				tls.TLS_ECDHE_RSA_WITH_RC4_128_SHA,
				tls.TLS_ECDHE_RSA_WITH_AES_128_CBC_SHA,
				TLS_RSA_WITH_AES_128_GCM_SHA256,
				tls.TLS_RSA_WITH_RC4_128_SHA,
				tls.TLS_RSA_WITH_AES_128_CBC_SHA,
			},
			PreferServerCipherSuites: true,  // Prefer our strong ciphers
			SessionTicketsDisabled:   false, // Support session resumption
		}
		tlsConfig.BuildNameToCertificate()

		var li *listener
		// Attempt to reuse one of the provided listeners.
		if reuse, exists := reuseListeners[addr]; exists {
			tlsConfig.SessionTicketKey = reuse.SessionTicketKey
			li, err = newListenerFromFd(reuse.Fd, addr, tlsConfig)
			if err != nil {
				// Failed to reuse the listener, so do some cleanup.
				tlsConfig.SessionTicketKey = [32]byte{}
				syscall.Close(int(reuse.Fd))
			}
		}

		// Create a new listener if needed.
		if li == nil || err != nil {
			li, err = newListener(addr, tlsConfig)
		}

		// If we still don't have a listener, there's nothing more we can do.
		if err != nil {
			// FIXME: We should probably continue iterating through the addresses.
			return err
		}
	}

	return err
}

// Serve begins serving connections.
func Serve() {
	managedListeners.serve()
}

// Shutdown gracefully shuts down the server, allowing any currently active
// connections to finish before doing so.
func Shutdown() {
	managedListeners.shutdown(true)
}

// ForceShutdown forcefully closes all currently active connections.  Little
// care is shown in making sure things are cleaned up, so this should generally
// only be used as a last resort.
func ForceShutdown() {
	managedListeners.shutdown(false)
}

// Detach returns information about all currently active listeners.  This
// information can be passed back to HTTPS in order to recreate the listeners.
func Detach() DetachedListeners {
	return managedListeners.detach()
}
<commit_msg>Update and explain the list of ciphers that we want to use.<commit_after>// Copyright 2013 Ryan Rogers. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

// Package server provides an easy to use HTTPS server.  It provides some
// benefits over using the standard library directly, such as the ability to
// gracefully shut down active connections, and to do low (zero?) downtime
// restarts.
package server

import (
	"crypto/tls"
	"syscall"
)

// A list of strong cipher suite IDs that are not defined by the crypto/tls
// package in the current stable version of Go. Values taken from
// http://www.iana.org/assignments/tls-parameters/tls-parameters.xml
//
// Note that the reason they are not defined by the crypto/tls package is
// because they are not (yet?) supported by Go. Defining them here allows us
// to immediately start using them, should Go support them in the future.
const (
	TLS_DHE_RSA_WITH_AES_128_CBC_SHA        uint16 = 0x0033
	TLS_DHE_RSA_WITH_AES_256_CBC_SHA        uint16 = 0x0039
	TLS_RSA_WITH_AES_128_GCM_SHA256         uint16 = 0x009c
	TLS_RSA_WITH_AES_256_GCM_SHA384         uint16 = 0x009d
	TLS_DHE_RSA_WITH_AES_128_GCM_SHA256     uint16 = 0x009e
	TLS_DHE_RSA_WITH_AES_256_GCM_SHA384     uint16 = 0x009f
	TLS_ECDHE_ECDSA_WITH_AES_128_CBC_SHA    uint16 = 0xc009
	TLS_ECDHE_ECDSA_WITH_AES_256_CBC_SHA    uint16 = 0xc00a
	TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256 uint16 = 0xc02b
	TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384 uint16 = 0xc02c
	TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256   uint16 = 0xc02f
	TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384   uint16 = 0xc030
)

// HTTPS creates a server that will serve HTTP requests using TLS.  If
// reuseListeners (obtained via Detach) is provided and matches an address in
// addrs, an attempt will be made to reuse the listener.  If that fails, it
// will fall back to creating a new listener.
func HTTPS(addrs []string, keyPairs map[string]string, reuseListeners DetachedListeners) error {
	// Load the server's certificates.
	tlsCertificates := make([]tls.Certificate, 0, len(keyPairs))
	for certFile, keyFile := range keyPairs {
		cert, err := tls.LoadX509KeyPair(certFile, keyFile)
		if err != nil {
			return err
		}
		tlsCertificates = append(tlsCertificates, cert)
	}

	var err error
	for _, addr := range addrs {
		// Configure TLS for each individual address to avoid all addresses
		// sharing the same SessionTicketKey.
		tlsConfig := &tls.Config{
			Certificates: tlsCertificates,
			NextProtos:   []string{"http/1.1"},
			// Reasoning behind the cipher suite ordering:
			//
			// - We want forward secrecy, so ECDHE/DHE come first. ECDHE comes
			//   before DHE since it's both stronger and faster.
			// - We prefer ECDSA over RSA since it's both stronger and faster.
			// - AES-GCM is currently our best choice of ciphers, since it is
			//   not vulnerable to any known attacks.
			// - Between CBC-mode ciphers and RC4, I'm not sure which is the
			//   lesser evil. CBC is vulnerable to BEAST (which is mostly
			//   mitigated by modern clients: https://community.qualys.com/blogs/securitylabs/2013/09/10/is-beast-still-a-threat)
			//   and Lucky13 (which is unlikely to be mitigated in Go: https://groups.google.com/d/msg/golang-nuts/HF5O5vAKRcQ/3cYWryRyZboJ),
			//   while RC4 has its own set of issues which lead to questionable
			//   security. For now, I'm opting to prefer RC4 just because that
			//   seems to be the consensus among the internet giants that
			//   employ people who are undoubtedly much smarter than me about
			//   this sort of thing.
			CipherSuites: []uint16{
				TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,
				TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,
				TLS_ECDHE_ECDSA_WITH_AES_256_CBC_SHA,
				TLS_ECDHE_ECDSA_WITH_AES_128_CBC_SHA,

				TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,
				TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,
				tls.TLS_ECDHE_RSA_WITH_RC4_128_SHA,
				tls.TLS_ECDHE_RSA_WITH_AES_256_CBC_SHA,
				tls.TLS_ECDHE_RSA_WITH_AES_128_CBC_SHA,

				TLS_DHE_RSA_WITH_AES_256_GCM_SHA384,
				TLS_DHE_RSA_WITH_AES_128_GCM_SHA256,
				TLS_DHE_RSA_WITH_AES_256_CBC_SHA,
				TLS_DHE_RSA_WITH_AES_128_CBC_SHA,

				TLS_RSA_WITH_AES_256_GCM_SHA384,
				TLS_RSA_WITH_AES_128_GCM_SHA256,
				tls.TLS_RSA_WITH_RC4_128_SHA,
				tls.TLS_RSA_WITH_AES_256_CBC_SHA,
				tls.TLS_RSA_WITH_AES_128_CBC_SHA,
			},
			PreferServerCipherSuites: true,  // Prefer our strong ciphers
			SessionTicketsDisabled:   false, // Support session resumption
		}
		tlsConfig.BuildNameToCertificate()

		var li *listener
		// Attempt to reuse one of the provided listeners.
		if reuse, exists := reuseListeners[addr]; exists {
			tlsConfig.SessionTicketKey = reuse.SessionTicketKey
			li, err = newListenerFromFd(reuse.Fd, addr, tlsConfig)
			if err != nil {
				// Failed to reuse the listener, so do some cleanup.
				tlsConfig.SessionTicketKey = [32]byte{}
				syscall.Close(int(reuse.Fd))
			}
		}

		// Create a new listener if needed.
		if li == nil || err != nil {
			li, err = newListener(addr, tlsConfig)
		}

		// If we still don't have a listener, there's nothing more we can do.
		if err != nil {
			// FIXME: We should probably continue iterating through the addresses.
			return err
		}
	}

	return err
}

// Serve begins serving connections.
func Serve() {
	managedListeners.serve()
}

// Shutdown gracefully shuts down the server, allowing any currently active
// connections to finish before doing so.
func Shutdown() {
	managedListeners.shutdown(true)
}

// ForceShutdown forcefully closes all currently active connections.  Little
// care is shown in making sure things are cleaned up, so this should generally
// only be used as a last resort.
func ForceShutdown() {
	managedListeners.shutdown(false)
}

// Detach returns information about all currently active listeners.  This
// information can be passed back to HTTPS in order to recreate the listeners.
func Detach() DetachedListeners {
	return managedListeners.detach()
}
<|endoftext|>


<commit_before>package operation

import (
	"bufio"
	"fmt"
	"io/ioutil"
	"log"
	"os/exec"

	json "github.com/bitly/go-simplejson"
	"github.com/iris-contrib/middleware/cors"
	"github.com/iris-contrib/middleware/logger"
	"github.com/iris-contrib/middleware/recovery"
	"github.com/kataras/go-template/html"
	"github.com/kataras/iris"
	"github.com/spf13/viper"
	"gopkg.in/mgo.v2"
	"gopkg.in/mgo.v2/bson"
)

var gSession *mgo.Session

func initDB(dbURL string) (*mgo.Session, error) {
	session, err := mgo.Dial(dbURL)
	if err != nil {
		return nil, fmt.Errorf("Cannot connect to mongo, %v", err)
	}
	return session, err
}

func getArgs(task *Task, frontTag, backTag string) []string {
	fabPath := viper.GetString("fabPath")
	if frontTag == "" {
		frontTag = task.Project.Front.Branch
	}
	if backTag == "" {
		backTag = task.Project.Backend.Branch
	}
	tmp, err := ioutil.TempDir("/tmp/", "operation")
	if err != nil {
		log.Panic(err)
	}
	deploy := fmt.Sprintf("deploy:tmp_path=%s,backend_url=%s,backend_branch=%s,front_url=%s,front_branch=%s,remote_path=%s,venv_path=%s,program=%s,workers=%s,worker_class=%s,bind=%s,user_group=%s,ext=%s,path=%s,include=%s,local_user=%s,local_password=%s,config_name=%s,nginx=%v",
		tmp, task.Project.Backend.Address, backTag, task.Project.Front.Address, frontTag, task.RemoteServer.Path, task.VenvPath, task.Gunicorn.Program, task.Gunicorn.Workers,
		task.Gunicorn.WorkerClass, task.Gunicorn.Bind, task.RemoteServer.Group, task.Supervisor.Extension, task.Supervisor.Path, task.Supervisor.Include, task.LocalServer.User, task.LocalServer.Password, task.ConfigName, task.Nginx)
	cmd := []string{
		"-f", fabPath, "-u", task.RemoteServer.User, "-p", task.RemoteServer.Password, "-H", task.RemoteServer.Host, deploy}
	return cmd
}

func updateTag(t *Task, frontTag, backTag string) {
	session := gSession.Clone()
	defer session.Close()
	var group Group
	coll := session.DB("operation").C("groups")
	err := coll.FindId(t.Group).One(&group)
	if err != nil {
		log.Fatal(err)
	}
	switch t.Project.Name {
	case "kelvin":
		if err := coll.UpdateId(group.ID, bson.M{"$set": bson.M{"front_tag": frontTag, "back_tag": backTag}}); err != nil {
			log.Fatal(err)
		}

	case "cms_plm":
		if err := coll.UpdateId(group.ID, bson.M{"$set": bson.M{"cms_tag": backTag}}); err != nil {
			log.Fatal(err)
		}
	}
}

// RunCommand ...
func RunCommand(c iris.WebsocketConnection, taskID, frontTag, backTag string) {
	session := gSession.Clone()
	defer session.Close()
	var task Task
	coll := session.DB("operation").C("tasks")
	err := coll.FindId(bson.ObjectIdHex(taskID)).One(&task)
	if err != nil {
		log.Fatal(err)
	}

	cmdArgs := getArgs(&task, frontTag, backTag)
	cmd := exec.Command("fab", cmdArgs...)
	stdout, err := cmd.StdoutPipe()
	if err != nil {
		log.Fatal(err)
	}
	cmd.Stderr = cmd.Stdout
	scanner := bufio.NewScanner(stdout)
	ch := make(chan int)
	go func() {
		for scanner.Scan() {
			c.EmitMessage(scanner.Bytes())
		}
		ch <- 0
	}()
	if err := cmd.Start(); err != nil {
		log.Panic(fmt.Errorf("Exec error, %v", err))
	}
	if err := cmd.Wait(); err != nil {
		log.Panic(fmt.Errorf("Finish error, %v", err))
	}
	log.Println("Deploy Over.")
	updateTag(&task, frontTag, backTag)
	<-ch
	c.EmitMessage([]byte("Deploy Over."))
}

// CreateApp ...
func CreateApp() *iris.Framework {
	var err error
	viper.AddConfigPath("/Users/sunyu/workspace/goprojects/src/github.com/syfun/operation")
	viper.AddConfigPath("D:/Workspace/gowork/src/github.com/syfun/operation")
	viper.AddConfigPath("/opt/operation")
	viper.SetConfigName("config")
	viper.SetConfigType("json")
	if err = viper.ReadInConfig(); err != nil {
		log.Panic(fmt.Errorf("Fatal error config file: %s \n", err))
	}
	gSession, err = initDB(viper.GetString("mongoURL"))
	if err != nil {
		log.Fatal(err)
	}

	app := iris.New()
	fmt.Println("##########")
	fmt.Println("templtePath", viper.GetString("templatePath"))
	fmt.Println("fabPath", viper.GetString("fabPath"))
	app.UseTemplate(html.New()).Directory(viper.GetString("templatePath"), ".html")
	app.Static("/static", viper.GetString("staticPath"), 1)

	app.Use(recovery.Handler)

	crs := cors.New(cors.Options{
		AllowedOrigins: []string{"*"},
		AllowedMethods: []string{"GET", "PUT", "POST", "DELETE"},
	})
	app.Use(crs)

	errorLogger := logger.New(logger.Config{Status: true, IP: true, Method: true, Path: true})
	app.Use(errorLogger)

	app.Get("/", func(c *iris.Context) {
		c.MustRender("index.html", nil)
	})
	app.Post("/api/v1/tasks", createTask)
	app.Get("/api/v1/tasks", queryTask)
	//app.Put("/api/v1/tasks/:taskID", updateTask)
	//app.Delete("/api/v1/tasks/:taskID", deleteTask)
	app.Get("/api/v1/groups", getGroups)

	app.Config.Websocket.Endpoint = "/ws"
	app.Websocket.OnConnection(func(c iris.WebsocketConnection) {
		fmt.Println("Connected.")
		c.OnMessage(func(message []byte) {
			fmt.Println(string(message))
			js, _ := json.NewJson(message)
			msgType, err := js.Get("type").String()
			if err != nil {
				log.Panic(err)
			}
			if msgType == "deploy" {
				taskID, err := js.Get("taskID").String()
				if err != nil {
					log.Println(err)
				}
				frontTag, err := js.Get("frontTag").String()
				if err != nil {
					log.Println(err)
				}
				backTag, err := js.Get("backTag").String()
				if err != nil {
					log.Println(err)
				}
				RunCommand(c, taskID, frontTag, backTag)
			}
		})
	})
	return app
}
<commit_msg>add  detail error info abount run command<commit_after>package operation

import (
	"bufio"
	"fmt"
	"io/ioutil"
	"log"
	"os/exec"

	json "github.com/bitly/go-simplejson"
	"github.com/iris-contrib/middleware/cors"
	"github.com/iris-contrib/middleware/logger"
	"github.com/iris-contrib/middleware/recovery"
	"github.com/kataras/go-template/html"
	"github.com/kataras/iris"
	"github.com/spf13/viper"
	"gopkg.in/mgo.v2"
	"gopkg.in/mgo.v2/bson"
)

var gSession *mgo.Session

func initDB(dbURL string) (*mgo.Session, error) {
	session, err := mgo.Dial(dbURL)
	if err != nil {
		return nil, fmt.Errorf("Cannot connect to mongo, %v", err)
	}
	return session, err
}

func getArgs(task *Task, frontTag, backTag string) []string {
	fabPath := viper.GetString("fabPath")
	if frontTag == "" {
		frontTag = task.Project.Front.Branch
	}
	if backTag == "" {
		backTag = task.Project.Backend.Branch
	}
	tmp, err := ioutil.TempDir("/tmp/", "operation")
	if err != nil {
		log.Panic(err)
	}
	deploy := fmt.Sprintf("deploy:tmp_path=%s,backend_url=%s,backend_branch=%s,front_url=%s,front_branch=%s,remote_path=%s,venv_path=%s,program=%s,workers=%s,worker_class=%s,bind=%s,user_group=%s,ext=%s,path=%s,include=%s,local_user=%s,local_password=%s,config_name=%s,nginx=%v",
		tmp, task.Project.Backend.Address, backTag, task.Project.Front.Address, frontTag, task.RemoteServer.Path, task.VenvPath, task.Gunicorn.Program, task.Gunicorn.Workers,
		task.Gunicorn.WorkerClass, task.Gunicorn.Bind, task.RemoteServer.Group, task.Supervisor.Extension, task.Supervisor.Path, task.Supervisor.Include, task.LocalServer.User, task.LocalServer.Password, task.ConfigName, task.Nginx)
	cmd := []string{
		"-f", fabPath, "-u", task.RemoteServer.User, "-p", task.RemoteServer.Password, "-H", task.RemoteServer.Host, deploy}
	return cmd
}

func updateTag(t *Task, frontTag, backTag string) {
	session := gSession.Clone()
	defer session.Close()
	var group Group
	coll := session.DB("operation").C("groups")
	err := coll.FindId(t.Group).One(&group)
	if err != nil {
		log.Fatal(err)
	}
	switch t.Project.Name {
	case "kelvin":
		if err := coll.UpdateId(group.ID, bson.M{"$set": bson.M{"front_tag": frontTag, "back_tag": backTag}}); err != nil {
			log.Fatal(err)
		}

	case "cms_plm":
		if err := coll.UpdateId(group.ID, bson.M{"$set": bson.M{"cms_tag": backTag}}); err != nil {
			log.Fatal(err)
		}
	}
}

// RunCommand ...
func RunCommand(c iris.WebsocketConnection, taskID, frontTag, backTag string) {
	session := gSession.Clone()
	defer session.Close()
	var task Task
	coll := session.DB("operation").C("tasks")
	err := coll.FindId(bson.ObjectIdHex(taskID)).One(&task)
	if err != nil {
		log.Fatal(err)
	}

	cmdArgs := getArgs(&task, frontTag, backTag)
	cmd := exec.Command("fab", cmdArgs...)
	stdout, err := cmd.StdoutPipe()
	if err != nil {
		log.Fatal(err)
	}
	cmd.Stderr = cmd.Stdout
	scanner := bufio.NewScanner(stdout)
	ch := make(chan int)
	go func() {
		for scanner.Scan() {
			if err := c.EmitMessage(scanner.Bytes()); err != nil {
				log.Panic("Emit error.\n", err)
			}
		}
		ch <- 0
	}()
	if err := cmd.Start(); err != nil {
		log.Panic("Exec error.\n", err)
	}
	if err := cmd.Wait(); err != nil {
		log.Panic("Finish error.\n", err)
	}
	updateTag(&task, frontTag, backTag)
	<-ch
}

// CreateApp ...
func CreateApp() *iris.Framework {
	var err error
	viper.AddConfigPath("/Users/sunyu/workspace/goprojects/src/github.com/syfun/operation")
	viper.AddConfigPath("D:/Workspace/gowork/src/github.com/syfun/operation")
	viper.AddConfigPath("/opt/operation")
	viper.SetConfigName("config")
	viper.SetConfigType("json")
	if err = viper.ReadInConfig(); err != nil {
		log.Panic(fmt.Errorf("Fatal error config file: %s \n", err))
	}
	gSession, err = initDB(viper.GetString("mongoURL"))
	if err != nil {
		log.Fatal(err)
	}

	app := iris.New()
	fmt.Println("##########")
	fmt.Println("templtePath", viper.GetString("templatePath"))
	fmt.Println("fabPath", viper.GetString("fabPath"))
	app.UseTemplate(html.New()).Directory(viper.GetString("templatePath"), ".html")
	app.Static("/static", viper.GetString("staticPath"), 1)

	app.Use(recovery.Handler)

	crs := cors.New(cors.Options{
		AllowedOrigins: []string{"*"},
		AllowedMethods: []string{"GET", "PUT", "POST", "DELETE"},
	})
	app.Use(crs)

	errorLogger := logger.New(logger.Config{Status: true, IP: true, Method: true, Path: true})
	app.Use(errorLogger)

	app.Get("/", func(c *iris.Context) {
		c.MustRender("index.html", nil)
	})
	app.Post("/api/v1/tasks", createTask)
	app.Get("/api/v1/tasks", queryTask)
	//app.Put("/api/v1/tasks/:taskID", updateTask)
	//app.Delete("/api/v1/tasks/:taskID", deleteTask)
	app.Get("/api/v1/groups", getGroups)

	app.Config.Websocket.Endpoint = "/ws"
	app.Websocket.OnConnection(func(c iris.WebsocketConnection) {
		fmt.Println("Connected.")
		c.OnMessage(func(message []byte) {
			fmt.Println(string(message))
			js, _ := json.NewJson(message)
			msgType, err := js.Get("type").String()
			if err != nil {
				log.Panic(err)
			}
			if msgType == "deploy" {
				taskID, err := js.Get("taskID").String()
				if err != nil {
					log.Println(err)
				}
				frontTag, err := js.Get("frontTag").String()
				if err != nil {
					log.Println(err)
				}
				backTag, err := js.Get("backTag").String()
				if err != nil {
					log.Println(err)
				}
				RunCommand(c, taskID, frontTag, backTag)
				log.Println("Deploy Over.")
				if err := c.EmitMessage([]byte("Deploy Over.")); err != nil {
					log.Panic("Over error.\n", err)
				}
			}
		})
		if err := c.Disconnect(); err != nil {
			log.Panic("Disconnect error.\n", err)
		}

	})
	return app
}
<|endoftext|>


<commit_before>package main

import (
	"encoding/json"
	"fmt"
	"math"
	"net/http"
	"strings"
	"time"

	log "github.com/Sirupsen/logrus"
	"github.com/didip/tollbooth"
	"github.com/didip/tollbooth/config"
)

// Server is the dispatch server
type Server struct {
	dispatch *Dispatch
}

// NewServer creates a new dispatch server
func NewServer(dispatch *Dispatch, limitMax int64, limitTTL time.Duration) *Server {
	s := new(Server)
	s.dispatch = dispatch

	// setup a rate limiter if needed
	if limitMax != math.MaxInt64 {
		log.Infof("setting webserver rate-limit to %d/%s", limitMax, limitTTL)
		limiter := tollbooth.NewLimiter(limitMax, limitTTL)
		limiter.IPLookups = []string{"X-Forwarded-For", "RemoteAddr", "X-Real-IP"}
		limiter.Methods = []string{"POST"}

		// setup endpoints
		http.Handle("/send", LimitFuncHandler(limiter, send))
	} else {

		http.HandleFunc("/send", send)
	}
	http.HandleFunc("/", defaultAction)

	return s
}

// Run the server
func (s Server) Run(address string) {
	log.Infof("starting webserver on %s", address)
	log.Fatal(http.ListenAndServe(address, WriteLogHandler(http.DefaultServeMux)))
}

type statusWriter struct {
	http.ResponseWriter
	statusCode int
	length     int
}

func (w *statusWriter) WriteHeader(status int) {
	w.statusCode = status
	w.ResponseWriter.WriteHeader(status)
}

func (w *statusWriter) Write(b []byte) (int, error) {
	if w.statusCode == 0 {
		w.statusCode = 200
	}
	w.length = len(b)
	return w.ResponseWriter.Write(b)
}

// WriteLogHandler returns a server log handler
func WriteLogHandler(handler http.Handler) http.Handler {
	return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
		writer := statusWriter{w, 0, 0}

		// calculate the latency
		t := time.Now()
		handler.ServeHTTP(&writer, r)
		latency := time.Since(t)

		clientIP := r.RemoteAddr
		proxyList := splitIPList(r.Header.Get("X-Forwarded-For"))
		if len(proxyList) > 0 {
			clientIP = proxyList[0]
		}
		statusCode := writer.statusCode
		path := r.URL.Path
		method := r.Method
		log.Printf("%s - %s %s %d %v", clientIP, method, path, statusCode, latency)
	})
}

// LimitHandler is a middleware that performs rate-limiting given http.Handler struct.
func LimitHandler(limiter *config.Limiter, next http.Handler) http.Handler {
	middle := func(w http.ResponseWriter, r *http.Request) {
		tollbooth.SetResponseHeaders(limiter, w)

		httpError := tollbooth.LimitByRequest(limiter, r)
		if httpError != nil {
			respondError(w, r, httpError.StatusCode, httpError.Message)
			return
		}

		// There's no rate-limit error, serve the next handler.
		next.ServeHTTP(w, r)
	}

	return http.HandlerFunc(middle)
}

// LimitFuncHandler is a middleware that performs rate-limiting given request handler function.
func LimitFuncHandler(limiter *config.Limiter, nextFunc func(http.ResponseWriter, *http.Request)) http.Handler {
	return LimitHandler(limiter, http.HandlerFunc(nextFunc))
}

func send(w http.ResponseWriter, r *http.Request) {
	recvTime := time.Now()
	if r.Method != "POST" {
		respondError(w, r, 404, "page not found")
		return
	}

	if r.Body == nil {
		respondError(w, r, 400, "request body missing")
		return
	}
	var msg DispatchRequest
	err := json.NewDecoder(r.Body).Decode(&msg)
	if err != nil {
		respondError(w, r, 400, "message format: %v", err)
		return
	}
	msg.Time = recvTime

	if len(msg.AuthToken) == 0 {
		respondError(w, r, 400, "field 'auth-token' missing")
		return
	}

	email, err := FormatEmail(msg.Email)
	if err != nil {
		respondError(w, r, 400, "email address is not valid")
		return
	}
	msg.Email = email

	err = dispatch.Send(msg)
	if err != nil {
		respondError(w, r, 400, "%v", err)
		return
	}

	respondSuccess(w, r)
}

func defaultAction(w http.ResponseWriter, r *http.Request) {
	respondError(w, r, 404, "page not found")
}

func respondError(w http.ResponseWriter, r *http.Request, code int, message string, a ...interface{}) {
	var msg string
	if r.Header.Get("Content-Type") == "application/json" {
		m := fmt.Sprintf(message, a...)
		msg = fmt.Sprintf("{\"status\": \"error\", \"message\": \"%s\"}", m)
		w.Header().Add("Content-Type", "application/json")
	} else { // default is text response
		m := fmt.Sprintf(message, a...)
		msg = fmt.Sprintf("%d %s", code, m)
		w.Header().Add("Content-Type", "text/plain")
	}

	w.WriteHeader(code)
	w.Write([]byte(msg))
}

func respondSuccess(w http.ResponseWriter, r *http.Request) {
	var msg string
	if r.Header.Get("Content-Type") == "application/json" {
		msg = "{\"status\": \"success\"}"
		w.Header().Add("Content-Type", "application/json")
	} else { // default is text response
		msg = "200 success"
		w.Header().Add("Content-Type", "text/plain")
	}

	w.WriteHeader(200)
	w.Write([]byte(msg))
}

func splitIPList(ipList string) []string {
	ips := strings.Split(ipList, ", ")
	list := make([]string, len(ips))
	for i, ip := range ips {
		list[i] = ip
	}
	return list
}
<commit_msg>remove port number from http logs<commit_after>package main

import (
	"encoding/json"
	"fmt"
	"math"
	"net"
	"net/http"
	"strings"
	"time"

	log "github.com/Sirupsen/logrus"
	"github.com/didip/tollbooth"
	"github.com/didip/tollbooth/config"
)

// Server is the dispatch server
type Server struct {
	dispatch *Dispatch
}

// NewServer creates a new dispatch server
func NewServer(dispatch *Dispatch, limitMax int64, limitTTL time.Duration) *Server {
	s := new(Server)
	s.dispatch = dispatch

	// setup a rate limiter if needed
	if limitMax != math.MaxInt64 {
		log.Infof("setting webserver rate-limit to %d/%s", limitMax, limitTTL)
		limiter := tollbooth.NewLimiter(limitMax, limitTTL)
		limiter.IPLookups = []string{"X-Forwarded-For", "RemoteAddr", "X-Real-IP"}
		limiter.Methods = []string{"POST"}

		// setup endpoints
		http.Handle("/send", LimitFuncHandler(limiter, send))
	} else {

		http.HandleFunc("/send", send)
	}
	http.HandleFunc("/", defaultAction)

	return s
}

// Run the server
func (s Server) Run(address string) {
	log.Infof("starting webserver on %s", address)
	log.Fatal(http.ListenAndServe(address, WriteLogHandler(http.DefaultServeMux)))
}

type statusWriter struct {
	http.ResponseWriter
	statusCode int
	length     int
}

func (w *statusWriter) WriteHeader(status int) {
	w.statusCode = status
	w.ResponseWriter.WriteHeader(status)
}

func (w *statusWriter) Write(b []byte) (int, error) {
	if w.statusCode == 0 {
		w.statusCode = 200
	}
	w.length = len(b)
	return w.ResponseWriter.Write(b)
}

// WriteLogHandler returns a server log handler
func WriteLogHandler(handler http.Handler) http.Handler {
	return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
		writer := statusWriter{w, 0, 0}

		// calculate the latency
		t := time.Now()
		handler.ServeHTTP(&writer, r)
		latency := time.Since(t)

		clientIP := getClientIP(r)
		statusCode := writer.statusCode
		path := r.URL.Path
		method := r.Method
		log.Printf("%s - %s %s %d %v", clientIP, method, path, statusCode, latency)
	})
}

// LimitHandler is a middleware that performs rate-limiting given http.Handler struct.
func LimitHandler(limiter *config.Limiter, next http.Handler) http.Handler {
	middle := func(w http.ResponseWriter, r *http.Request) {
		tollbooth.SetResponseHeaders(limiter, w)

		httpError := tollbooth.LimitByRequest(limiter, r)
		if httpError != nil {
			respondError(w, r, httpError.StatusCode, httpError.Message)
			return
		}

		// There's no rate-limit error, serve the next handler.
		next.ServeHTTP(w, r)
	}

	return http.HandlerFunc(middle)
}

// LimitFuncHandler is a middleware that performs rate-limiting given request handler function.
func LimitFuncHandler(limiter *config.Limiter, nextFunc func(http.ResponseWriter, *http.Request)) http.Handler {
	return LimitHandler(limiter, http.HandlerFunc(nextFunc))
}

func send(w http.ResponseWriter, r *http.Request) {
	recvTime := time.Now()
	if r.Method != "POST" {
		respondError(w, r, 404, "page not found")
		return
	}

	if r.Body == nil {
		respondError(w, r, 400, "request body missing")
		return
	}
	var msg DispatchRequest
	err := json.NewDecoder(r.Body).Decode(&msg)
	if err != nil {
		respondError(w, r, 400, "message format: %v", err)
		return
	}
	msg.Time = recvTime

	if len(msg.AuthToken) == 0 {
		respondError(w, r, 400, "field 'auth-token' missing")
		return
	}

	email, err := FormatEmail(msg.Email)
	if err != nil {
		respondError(w, r, 400, "email address is not valid")
		return
	}
	msg.Email = email

	err = dispatch.Send(msg)
	if err != nil {
		respondError(w, r, 400, "%v", err)
		return
	}

	respondSuccess(w, r)
}

func defaultAction(w http.ResponseWriter, r *http.Request) {
	respondError(w, r, 404, "page not found")
}

func respondError(w http.ResponseWriter, r *http.Request, code int, message string, a ...interface{}) {
	var msg string
	if r.Header.Get("Content-Type") == "application/json" {
		m := fmt.Sprintf(message, a...)
		msg = fmt.Sprintf("{\"status\": \"error\", \"message\": \"%s\"}", m)
		w.Header().Add("Content-Type", "application/json")
	} else { // default is text response
		m := fmt.Sprintf(message, a...)
		msg = fmt.Sprintf("%d %s", code, m)
		w.Header().Add("Content-Type", "text/plain")
	}

	w.WriteHeader(code)
	w.Write([]byte(msg))
}

func respondSuccess(w http.ResponseWriter, r *http.Request) {
	var msg string
	if r.Header.Get("Content-Type") == "application/json" {
		msg = "{\"status\": \"success\"}"
		w.Header().Add("Content-Type", "application/json")
	} else { // default is text response
		msg = "200 success"
		w.Header().Add("Content-Type", "text/plain")
	}

	w.WriteHeader(200)
	w.Write([]byte(msg))
}

func splitIPList(ipList string) []string {
	ips := strings.Split(ipList, ", ")
	var list []string
	for _, ip := range ips {
		ip = strings.TrimSpace(ip)
		if len(ip) > 0 {
			list = append(list, ip)
		}
	}
	return list
}

func getClientIP(r *http.Request) string {
	// first, figure out the correct IP to use
	clientHostPort := r.RemoteAddr
	proxyList := splitIPList(r.Header.Get("X-Forwarded-For"))
	if len(proxyList) > 0 {
		clientHostPort = proxyList[0]
	}

	// clean it up
	clientIP, _, err := net.SplitHostPort(clientHostPort)
	if err != nil {
		clientIP = clientHostPort
	}

	return clientIP
}
<|endoftext|>


<commit_before>package main

import (
	"fmt"
	"html"
	"log"
	"net/http"
	"strings"
)

func topHandler(w http.ResponseWriter, r *http.Request) {
	fmt.Fprintf(w, "Messtin")
}

func bookListHandler(w http.ResponseWriter, r *http.Request) {
	http.ServeFile(w, r, "books.json")
}

func bookHandler(w http.ResponseWriter, r *http.Request) {
	var path = html.EscapeString(r.URL.Path)
	var comps = strings.Split(path, "/")
	var book = comps[len(comps)-2]
	var id = comps[len(comps)-1]

	log.Println("request to book: " + book + ", " + id)
	if id == "manifest.json" {
		http.ServeFile(w, r, "contents/"+book+".json")
		return
	}
	http.ServeFile(w, r, "contents/"+book+"/"+id)
}

func main() {
	// route
	mux := http.NewServeMux()
	mux.Handle("/", http.HandlerFunc(topHandler))
	mux.Handle("/books.json", http.HandlerFunc(bookListHandler))
	mux.Handle("/book/", http.HandlerFunc(bookHandler))

	log.Fatal(http.ListenAndServe(":8080", mux))
}
<commit_msg>Concentrate on server dev with Node.js.<commit_after><|endoftext|>


<commit_before>package main

import (
	"code.google.com/p/go.net/websocket"
	"github.com/kr/pty"
	"html/template"
	"io"
	"io/ioutil"
	"net/http"
	"os"
	"os/exec"
	"os/user"
	"strconv"
	"strings"
	"syscall"
	"time"
)

var ide *template.Template
var api *template.Template
var hostname, _ = ioutil.ReadFile("/etc/hostname")
var users = make(map[string]string)
var config = map[string]string{"port": "80", "projectdir": "~/raspberrystem_projects/"}

func main() {
	settings, err := ioutil.ReadFile("/etc/ide/settings.conf")
	if err == nil {
		set := strings.Split(string(settings), "\n")
		for _, line := range set {
			if len(line) > 0 && line[0:1] != "#" {
				line = strings.Split(line, "#")[0]
				lsplit := strings.Split(line, " ")
				config[strings.ToLower(lsplit[0])] = strings.TrimRight(line[len(lsplit[0])+1:], " ")
			}
		}
	}
	currUser, _ := user.Current()
	config["projectdir"] = strings.Replace(config["projectdir"], "~", currUser.HomeDir, 1)
	os.Mkdir(config["projectdir"], 0775)
	if err != nil {
		panic(err)
	}
	http.HandleFunc("/", index) //All requests to / and 404s will route to Index
	http.HandleFunc("/ide.js", ideJs)
	http.HandleFunc("/cm.js", cmJs)
	http.HandleFunc("/cm.css", cmCss)
	http.HandleFunc("/python.js", pythonMode)
	http.HandleFunc("/shell.js", shellMode)
	http.HandleFunc("/api/listfiles", listFiles)
	http.HandleFunc("/api/listthemes", listThemes)
	http.HandleFunc("/api/readfile", readFile)
	http.HandleFunc("/api/savefile", saveFile)
	http.HandleFunc("/api/deletefile", deleteFile)
	http.HandleFunc("/api/hostname", hostnameOut)
	http.Handle("/api/socket", websocket.Handler(socketServer))
	http.Handle("/api/change", websocket.Handler(changeServer))
	http.Handle("/website/", http.StripPrefix("/website/", http.FileServer(http.Dir("/etc/ide/website"))))
	http.Handle("/images/", http.StripPrefix("/images/", http.FileServer(http.Dir("/etc/ide/assets/images"))))
	http.Handle("/themes/", http.StripPrefix("/themes/", http.FileServer(http.Dir("/etc/ide/assets/themes"))))
	err = http.ListenAndServe(":"+config["port"], nil)
	if err != nil {
		panic(err)
	}
}

//Called by requests to / and 404s
func index(w http.ResponseWriter, r *http.Request) {
	http.ServeFile(w, r, "/etc/ide/ide.html")
}

func ideJs(w http.ResponseWriter, r *http.Request) {
	http.ServeFile(w, r, "/etc/ide/assets/ide.js")
}

func cmJs(w http.ResponseWriter, r *http.Request) {
	http.ServeFile(w, r, "/etc/ide/assets/cmirror/codemirror.js")
}

func cmCss(w http.ResponseWriter, r *http.Request) {
	http.ServeFile(w, r, "/etc/ide/assets/cmirror/codemirror.css")
}

func pythonMode(w http.ResponseWriter, r *http.Request) {
	http.ServeFile(w, r, "/etc/ide/assets/cmirror/python.js")
}

func shellMode(w http.ResponseWriter, r *http.Request) {
	http.ServeFile(w, r, "/etc/ide/assets/cmirror/shell.js")
}

//Lists all files in the projects directory
func listFiles(w http.ResponseWriter, r *http.Request) {
	w.Header().Set("Content-Type", "text/plain")
	files, _ := ioutil.ReadDir(config["projectdir"])
	var list string
	for _, f := range files {
		if !f.IsDir() {
			list += "\n" + f.Name()
		}
	}
	if list != "" {
		io.WriteString(w, list[1:])
	}
}

//Lists the availible themes
func listThemes(w http.ResponseWriter, r *http.Request) {
	w.Header().Set("Content-Type", "text/plain")
	files, _ := ioutil.ReadDir("/etc/ide/assets/themes/")
	var list string
	for _, f := range files {
		if !f.IsDir() {
			list += "\n" + f.Name()
		}
	}
	if list != "" {
		io.WriteString(w, list[1:])
	}
}

//Outputs the contents of the requested file
func readFile(w http.ResponseWriter, r *http.Request) {
	w.Header().Set("Content-Type", "text/plain")
	opts := r.URL.Query()
	fname := strings.Trim(opts.Get("file"), " ./")
	contents, err := ioutil.ReadFile(config["projectdir"] + fname)
	if err != nil {
		io.WriteString(w, "error")
		return
	}
	io.WriteString(w, string(contents))
}

//Potential issue here because POST requests tend to have size limits
func saveFile(w http.ResponseWriter, r *http.Request) {
	r.ParseForm()
	name := strings.Trim(r.Form.Get("file"), " ./")
	content := r.Form.Get("content")
	ioutil.WriteFile(config["projectdir"]+name, []byte(content), 0744)
}

func deleteFile(w http.ResponseWriter, r *http.Request) {
	r.ParseForm()
	fname := strings.Trim(r.Form.Get("file"), " ./")
	os.Remove(config["projectdir"] + fname)
}

func hostnameOut(w http.ResponseWriter, r *http.Request) {
	w.Header().Set("Content-Type", "text/plain")
	w.Header().Set("Access-Control-Allow-Origin", "*")
	io.WriteString(w, string(hostname))
}

//Called as a goroutine to wait for the close command and kill the process.
func watchClose(s *websocket.Conn, p *os.Process) {
	data := make([]byte, 512)
	n, _ := s.Read(data)
	payload := string(data[:n]) //[:n] to cut out padding
	if payload == "close" {
		p.Signal(syscall.SIGINT)
	}
}

//Receives socket connections to /api/socket and creates a PTY to run the process
func socketServer(s *websocket.Conn) {
	data := make([]byte, 512)
	n, _ := s.Read(data)
	com := exec.Command(config["projectdir"] + string(data[:n])) //[:n] to cut out padding
	pt, err := pty.Start(com)
	if err != nil {
		s.Write([]byte("error: " + err.Error()))
		s.Close()
		return
	}
	go watchClose(s, com.Process)
	for {
		out := make([]byte, 1024)
		n, err := pt.Read(out)
		if err != nil {
			if err.Error() == "read /dev/ptmx: input/output error" || err.Error() == "EOF" {
				break
			}
			s.Write([]byte("error: " + err.Error()))
		} else if n > 0 {
			s.Write([]byte("output: " + string(out[:n])))
		}
		time.Sleep(time.Millisecond * 10) //For minimal CPU impact
	}
	s.Close()
}

var changeSockets = make(map[string][]*websocket.Conn)
var lastSocket *websocket.Conn

//Receives /api/change socket connections and manages concurrent editing
func changeServer(s *websocket.Conn) {
	var currFile string
	for {
		var data string
		err := websocket.Message.Receive(s, &data)
		if err != nil {
			break
		}
		//Change of file
		if data[:4] == "COF:" {
			if currFile != "" {
				for i, v := range changeSockets[currFile] {
					if v == s {
						changeSockets[currFile] = append(changeSockets[currFile][:i], changeSockets[currFile][i+1:]...)
						break
					}
				}
				users := strconv.Itoa(len(changeSockets[currFile]))
				for _, v := range changeSockets[currFile] {
					websocket.Message.Send(v, "USERS:"+users)
				}
			}
			currFile = data[4:]
			if len(changeSockets[currFile]) > 0 {
				lastSocket = s
				websocket.Message.Send(changeSockets[currFile][0], "FILE")
			}
			changeSockets[currFile] = append(changeSockets[currFile], s)
			users := strconv.Itoa(len(changeSockets[currFile]))
			for _, v := range changeSockets[currFile] {
				websocket.Message.Send(v, "USERS:"+users)
			}
		} else if data[:4] == "CIF:" { //Change in file
			for _, v := range changeSockets[currFile] {
				if v != s {
					websocket.Message.Send(v, data[4:])
				}
			}
		} else if data[:5] == "FILE:" {
			for _, v := range changeSockets[currFile] {
				if v != s {
					websocket.Message.Send(v, data)
				}
			}
			lastSocket = nil
		}
	}
	if currFile != "" {
		for i, v := range changeSockets[currFile] {
			if v == s {
				changeSockets[currFile] = append(changeSockets[currFile][:i], changeSockets[currFile][i+1:]...)
				break
			}
		}
		users := strconv.Itoa(len(changeSockets[currFile]))
		for _, v := range changeSockets[currFile] {
			websocket.Message.Send(v, "USERS:"+users)
		}
	}
	s.Close()
}
<commit_msg>Removed HTML templates<commit_after>package main

import (
	"code.google.com/p/go.net/websocket"
	"github.com/kr/pty"
	"io"
	"io/ioutil"
	"net/http"
	"os"
	"os/exec"
	"os/user"
	"strconv"
	"strings"
	"syscall"
	"time"
)

var hostname, _ = ioutil.ReadFile("/etc/hostname")
var users = make(map[string]string)
var config = map[string]string{"port": "80", "projectdir": "~/raspberrystem_projects/"}

func main() {
	settings, err := ioutil.ReadFile("/etc/ide/settings.conf")
	if err == nil {
		set := strings.Split(string(settings), "\n")
		for _, line := range set {
			if len(line) > 0 && line[0:1] != "#" {
				line = strings.Split(line, "#")[0]
				lsplit := strings.Split(line, " ")
				config[strings.ToLower(lsplit[0])] = strings.TrimRight(line[len(lsplit[0])+1:], " ")
			}
		}
	}
	currUser, _ := user.Current()
	config["projectdir"] = strings.Replace(config["projectdir"], "~", currUser.HomeDir, 1)
	os.Mkdir(config["projectdir"], 0775)
	if err != nil {
		panic(err)
	}
	http.HandleFunc("/", index) //All requests to / and 404s will route to Index
	http.HandleFunc("/ide.js", ideJs)
	http.HandleFunc("/cm.js", cmJs)
	http.HandleFunc("/cm.css", cmCss)
	http.HandleFunc("/python.js", pythonMode)
	http.HandleFunc("/shell.js", shellMode)
	http.HandleFunc("/api/listfiles", listFiles)
	http.HandleFunc("/api/listthemes", listThemes)
	http.HandleFunc("/api/readfile", readFile)
	http.HandleFunc("/api/savefile", saveFile)
	http.HandleFunc("/api/deletefile", deleteFile)
	http.HandleFunc("/api/hostname", hostnameOut)
	http.Handle("/api/socket", websocket.Handler(socketServer))
	http.Handle("/api/change", websocket.Handler(changeServer))
	http.Handle("/website/", http.StripPrefix("/website/", http.FileServer(http.Dir("/etc/ide/website"))))
	http.Handle("/images/", http.StripPrefix("/images/", http.FileServer(http.Dir("/etc/ide/assets/images"))))
	http.Handle("/themes/", http.StripPrefix("/themes/", http.FileServer(http.Dir("/etc/ide/assets/themes"))))
	err = http.ListenAndServe(":"+config["port"], nil)
	if err != nil {
		panic(err)
	}
}

//Called by requests to / and 404s
func index(w http.ResponseWriter, r *http.Request) {
	http.ServeFile(w, r, "/etc/ide/ide.html")
}

func ideJs(w http.ResponseWriter, r *http.Request) {
	http.ServeFile(w, r, "/etc/ide/assets/ide.js")
}

func cmJs(w http.ResponseWriter, r *http.Request) {
	http.ServeFile(w, r, "/etc/ide/assets/cmirror/codemirror.js")
}

func cmCss(w http.ResponseWriter, r *http.Request) {
	http.ServeFile(w, r, "/etc/ide/assets/cmirror/codemirror.css")
}

func pythonMode(w http.ResponseWriter, r *http.Request) {
	http.ServeFile(w, r, "/etc/ide/assets/cmirror/python.js")
}

func shellMode(w http.ResponseWriter, r *http.Request) {
	http.ServeFile(w, r, "/etc/ide/assets/cmirror/shell.js")
}

//Lists all files in the projects directory
func listFiles(w http.ResponseWriter, r *http.Request) {
	w.Header().Set("Content-Type", "text/plain")
	files, _ := ioutil.ReadDir(config["projectdir"])
	var list string
	for _, f := range files {
		if !f.IsDir() {
			list += "\n" + f.Name()
		}
	}
	if list != "" {
		io.WriteString(w, list[1:])
	}
}

//Lists the availible themes
func listThemes(w http.ResponseWriter, r *http.Request) {
	w.Header().Set("Content-Type", "text/plain")
	files, _ := ioutil.ReadDir("/etc/ide/assets/themes/")
	var list string
	for _, f := range files {
		if !f.IsDir() {
			list += "\n" + f.Name()
		}
	}
	if list != "" {
		io.WriteString(w, list[1:])
	}
}

//Outputs the contents of the requested file
func readFile(w http.ResponseWriter, r *http.Request) {
	w.Header().Set("Content-Type", "text/plain")
	opts := r.URL.Query()
	fname := strings.Trim(opts.Get("file"), " ./")
	contents, err := ioutil.ReadFile(config["projectdir"] + fname)
	if err != nil {
		io.WriteString(w, "error")
		return
	}
	io.WriteString(w, string(contents))
}

//Potential issue here because POST requests tend to have size limits
func saveFile(w http.ResponseWriter, r *http.Request) {
	r.ParseForm()
	name := strings.Trim(r.Form.Get("file"), " ./")
	content := r.Form.Get("content")
	ioutil.WriteFile(config["projectdir"]+name, []byte(content), 0744)
}

func deleteFile(w http.ResponseWriter, r *http.Request) {
	r.ParseForm()
	fname := strings.Trim(r.Form.Get("file"), " ./")
	os.Remove(config["projectdir"] + fname)
}

func hostnameOut(w http.ResponseWriter, r *http.Request) {
	w.Header().Set("Content-Type", "text/plain")
	w.Header().Set("Access-Control-Allow-Origin", "*")
	io.WriteString(w, string(hostname))
}

//Called as a goroutine to wait for the close command and kill the process.
func watchClose(s *websocket.Conn, p *os.Process) {
	data := make([]byte, 512)
	n, _ := s.Read(data)
	payload := string(data[:n]) //[:n] to cut out padding
	if payload == "close" {
		p.Signal(syscall.SIGINT)
	}
}

//Receives socket connections to /api/socket and creates a PTY to run the process
func socketServer(s *websocket.Conn) {
	data := make([]byte, 512)
	n, _ := s.Read(data)
	com := exec.Command(config["projectdir"] + string(data[:n])) //[:n] to cut out padding
	pt, err := pty.Start(com)
	if err != nil {
		s.Write([]byte("error: " + err.Error()))
		s.Close()
		return
	}
	go watchClose(s, com.Process)
	for {
		out := make([]byte, 1024)
		n, err := pt.Read(out)
		if err != nil {
			if err.Error() == "read /dev/ptmx: input/output error" || err.Error() == "EOF" {
				break
			}
			s.Write([]byte("error: " + err.Error()))
		} else if n > 0 {
			s.Write([]byte("output: " + string(out[:n])))
		}
		time.Sleep(time.Millisecond * 10) //For minimal CPU impact
	}
	s.Close()
}

var changeSockets = make(map[string][]*websocket.Conn)
var lastSocket *websocket.Conn

//Receives /api/change socket connections and manages concurrent editing
func changeServer(s *websocket.Conn) {
	var currFile string
	for {
		var data string
		err := websocket.Message.Receive(s, &data)
		if err != nil {
			break
		}
		//Change of file
		if data[:4] == "COF:" {
			if currFile != "" {
				for i, v := range changeSockets[currFile] {
					if v == s {
						changeSockets[currFile] = append(changeSockets[currFile][:i], changeSockets[currFile][i+1:]...)
						break
					}
				}
				users := strconv.Itoa(len(changeSockets[currFile]))
				for _, v := range changeSockets[currFile] {
					websocket.Message.Send(v, "USERS:"+users)
				}
			}
			currFile = data[4:]
			if len(changeSockets[currFile]) > 0 {
				lastSocket = s
				websocket.Message.Send(changeSockets[currFile][0], "FILE")
			}
			changeSockets[currFile] = append(changeSockets[currFile], s)
			users := strconv.Itoa(len(changeSockets[currFile]))
			for _, v := range changeSockets[currFile] {
				websocket.Message.Send(v, "USERS:"+users)
			}
		} else if data[:4] == "CIF:" { //Change in file
			for _, v := range changeSockets[currFile] {
				if v != s {
					websocket.Message.Send(v, data[4:])
				}
			}
		} else if data[:5] == "FILE:" {
			for _, v := range changeSockets[currFile] {
				if v != s {
					websocket.Message.Send(v, data)
				}
			}
			lastSocket = nil
		}
	}
	if currFile != "" {
		for i, v := range changeSockets[currFile] {
			if v == s {
				changeSockets[currFile] = append(changeSockets[currFile][:i], changeSockets[currFile][i+1:]...)
				break
			}
		}
		users := strconv.Itoa(len(changeSockets[currFile]))
		for _, v := range changeSockets[currFile] {
			websocket.Message.Send(v, "USERS:"+users)
		}
	}
	s.Close()
}
<|endoftext|>


<commit_before>package main

import (
	"log"
	"os"
	"net"
	"io/ioutil"
	"time"
	"regexp"
	"strings"
	"strconv"
	"bufio"
)

type Server struct {
	conf Config
	mon *Monitoring
	lg log.Logger
	ch chan string
	chS chan string
	chA chan string
}

func (s Server) combineMetricsWithSameName(metric string, metrics []Metric) []Metric {
	split := regexp.MustCompile("\\s").Split(metric, 3)

	value, err := strconv.ParseFloat(split[1], 64)
	if err != nil {s.lg.Println("Can not parse value of metric " + split[0] + ": " + split[1]) ; return metrics}
	timestamp, err := strconv.ParseInt(split[2], 10, 64)
	if err != nil {s.lg.Println("Can not parse timestamp of metric " + split[0] + ": " + split[2]) ; return metrics}

	/*
	Go through existing metrics and search for the same name of metric
	If there is no same metric - append it as a new
	 */
	for i,_ := range metrics {
		if metrics[i].name == split[0] {
			metrics[i].amount++
			metrics[i].value += value
			metrics[i].timestamp += timestamp
			return metrics
		}
	}
	metrics = append(metrics, Metric{split[0], 1, value, timestamp})
	return metrics
}
// Sum metrics with prefix
func (s Server) sumMetricsWithPrefix() {
	for ;; time.Sleep(time.Duration(s.conf.SumInterval)*time.Second) {
		var working_list[] Metric
		chanSize := len(s.chS)
		for i := 0; i < chanSize; i++ {
			working_list = s.combineMetricsWithSameName(strings.Replace(<-s.chS, s.conf.SumPrefix, "", -1), working_list)
		}
		for _,val := range working_list {
			s.ch <- val.name + " " +
				strconv.FormatFloat(val.value, 'f', 2, 32) + " " + strconv.FormatInt(val.timestamp/val.amount, 10)
		}
	}
}

// AVG metrics with prefix
func (s Server) avgMetricsWithPrefix() {
	for ;; time.Sleep(time.Duration(s.conf.AvgInterval)*time.Second) {
		var working_list[] Metric
		chanSize := len(s.chA)
		for i := 0; i < chanSize; i++ {
			working_list = s.combineMetricsWithSameName(strings.Replace(<-s.chA, s.conf.AvgPrefix, "", -1), working_list)
		}
		for _,val := range working_list {
			s.ch <- val.name + " " +
				strconv.FormatFloat(val.value/float64(val.amount), 'f', 2, 32) + " " + strconv.FormatInt(val.timestamp/val.amount, 10)
		}
	}
}

// Function checks and removed bad data and sorts it by SUM prefix
func (s Server)cleanAndUseIncomingData(metrics []string) {
	for _,metric := range metrics {
		if validateMetric(metric, s.conf.AllowedMetrics) {
			if strings.HasPrefix(metric, s.conf.SumPrefix) {
				if len(s.chS) < s.conf.SumsPerSecond*s.conf.SumInterval{
					s.chS <- metric
				} else {
					s.lg.Println("Too many metrics in the SUM queue (" + strconv.Itoa(len(s.chS)) + "). I have to drop icommings")
					s.mon.dropped++
				}
			} else if strings.HasPrefix(metric, s.conf.AvgPrefix) {
				if len(s.chA) < s.conf.AvgsPerSecond*s.conf.AvgInterval{
					s.chA <- metric
				} else {
					s.lg.Println("Too many metrics in the AVG queue (" + strconv.Itoa(len(s.chA)) + "). I have to drop icommings")
					s.mon.dropped++
				}
			} else {
				if len(s.ch) < s.conf.MaxMetrics*s.conf.ClientSendInterval {
					s.ch <- metric
				} else {
					s.lg.Println("Too many metrics in the main queue (" + strconv.Itoa(len(s.ch)) + "). I have to drop icommings")
					s.mon.dropped++
				}
			}
		} else {
			if metric != "" {
				s.mon.dropped++
				s.lg.Println("Removing bad metric \"" + metric + "\" from the list")
			}
		}
	}
	metrics = nil
}

// Reading metrics from network
func (s Server)handleRequest(conn net.Conn) {

	connbuf := bufio.NewReader(conn)
	for ;; {
		s.mon.got.net++
		metric, err := connbuf.ReadString('\n')
		if err!= nil {
			break
		}
		if s.mon.got.net < s.conf.MaxMetrics {
			s.cleanAndUseIncomingData([]string{strings.Replace(strings.Replace(metric, "\r", "", -1), "\n", "", -1)})
		} else {
			s.mon.dropped++
		}
	}
	conn.Close()
}

// Reading metrics from files in folder. This is a second way how to send metrics, except network
func (s Server)handleDirMetrics() {
	for ;; time.Sleep(time.Duration(s.conf.ClientSendInterval)*time.Second) {
		files, err := ioutil.ReadDir(s.conf.MetricDir)
		if err != nil {
			panic(err.Error())
		}
		for _, f := range files {
			results_list := readMetricsFromFile(s.conf.MetricDir+"/"+f.Name())
			s.mon.got.dir += len(results_list)
			s.cleanAndUseIncomingData(results_list)
		}

	}
}

func (s Server)runServer() {
	// Listen for incoming connections.
	l, err := net.Listen("tcp", s.conf.LocalBind)
	if err != nil {
		s.lg.Println("Failed to run server:", err.Error())
		os.Exit(1)
	} else {
		s.lg.Println("Server is running")
	}
	// Close the listener when the application closes.
	defer l.Close()

	// Run goroutine for reading metrics from metricDir
	go s.handleDirMetrics()
	// Run goroutine for sum metrics with prefix
	go s.sumMetricsWithPrefix()
	// Run goroutine for avg metrics with prefix
	go s.avgMetricsWithPrefix()

	for {
		// Listen for an incoming connection.
		conn, err := l.Accept()
		if err != nil {
			s.lg.Println("Error accepting: ", err.Error())
			os.Exit(1)
		}
		// Handle connections in a new goroutine.
		go s.handleRequest(conn)
	}
}<commit_msg>Last metric without \n at the end will be processed anyway<commit_after>package main

import (
	"log"
	"os"
	"net"
	"io/ioutil"
	"time"
	"regexp"
	"strings"
	"strconv"
	"bufio"
)

type Server struct {
	conf Config
	mon *Monitoring
	lg log.Logger
	ch chan string
	chS chan string
	chA chan string
}

func (s Server) combineMetricsWithSameName(metric string, metrics []Metric) []Metric {
	split := regexp.MustCompile("\\s").Split(metric, 3)

	value, err := strconv.ParseFloat(split[1], 64)
	if err != nil {s.lg.Println("Can not parse value of metric " + split[0] + ": " + split[1]) ; return metrics}
	timestamp, err := strconv.ParseInt(split[2], 10, 64)
	if err != nil {s.lg.Println("Can not parse timestamp of metric " + split[0] + ": " + split[2]) ; return metrics}

	/*
	Go through existing metrics and search for the same name of metric
	If there is no same metric - append it as a new
	 */
	for i,_ := range metrics {
		if metrics[i].name == split[0] {
			metrics[i].amount++
			metrics[i].value += value
			metrics[i].timestamp += timestamp
			return metrics
		}
	}
	metrics = append(metrics, Metric{split[0], 1, value, timestamp})
	return metrics
}
// Sum metrics with prefix
func (s Server) sumMetricsWithPrefix() {
	for ;; time.Sleep(time.Duration(s.conf.SumInterval)*time.Second) {
		var working_list[] Metric
		chanSize := len(s.chS)
		for i := 0; i < chanSize; i++ {
			working_list = s.combineMetricsWithSameName(strings.Replace(<-s.chS, s.conf.SumPrefix, "", -1), working_list)
		}
		for _,val := range working_list {
			s.ch <- val.name + " " +
				strconv.FormatFloat(val.value, 'f', 2, 32) + " " + strconv.FormatInt(val.timestamp/val.amount, 10)
		}
	}
}

// AVG metrics with prefix
func (s Server) avgMetricsWithPrefix() {
	for ;; time.Sleep(time.Duration(s.conf.AvgInterval)*time.Second) {
		var working_list[] Metric
		chanSize := len(s.chA)
		for i := 0; i < chanSize; i++ {
			working_list = s.combineMetricsWithSameName(strings.Replace(<-s.chA, s.conf.AvgPrefix, "", -1), working_list)
		}
		for _,val := range working_list {
			s.ch <- val.name + " " +
				strconv.FormatFloat(val.value/float64(val.amount), 'f', 2, 32) + " " + strconv.FormatInt(val.timestamp/val.amount, 10)
		}
	}
}

// Function checks and removed bad data and sorts it by SUM prefix
func (s Server)cleanAndUseIncomingData(metrics []string) {
	for _,metric := range metrics {
		if validateMetric(metric, s.conf.AllowedMetrics) {
			if strings.HasPrefix(metric, s.conf.SumPrefix) {
				if len(s.chS) < s.conf.SumsPerSecond*s.conf.SumInterval{
					s.chS <- metric
				} else {
					s.lg.Println("Too many metrics in the SUM queue (" + strconv.Itoa(len(s.chS)) + "). I have to drop icommings")
					s.mon.dropped++
				}
			} else if strings.HasPrefix(metric, s.conf.AvgPrefix) {
				if len(s.chA) < s.conf.AvgsPerSecond*s.conf.AvgInterval{
					s.chA <- metric
				} else {
					s.lg.Println("Too many metrics in the AVG queue (" + strconv.Itoa(len(s.chA)) + "). I have to drop icommings")
					s.mon.dropped++
				}
			} else {
				if len(s.ch) < s.conf.MaxMetrics*s.conf.ClientSendInterval {
					s.ch <- metric
				} else {
					s.lg.Println("Too many metrics in the main queue (" + strconv.Itoa(len(s.ch)) + "). I have to drop icommings")
					s.mon.dropped++
				}
			}
		} else {
			if metric != "" {
				s.mon.dropped++
				s.lg.Println("Removing bad metric \"" + metric + "\" from the list")
			}
		}
	}
	metrics = nil
}

// Reading metrics from network
func (s Server)handleRequest(conn net.Conn) {

	connbuf := bufio.NewReader(conn)
	for ;; {
		s.mon.got.net++
		metric, err := connbuf.ReadString('\n')
		// Even if error occurred we still put "metric" into analysis, cause it can be a valid metric, but without \n
		if s.mon.got.net < s.conf.MaxMetrics {
			s.cleanAndUseIncomingData([]string{strings.Replace(strings.Replace(metric, "\r", "", -1), "\n", "", -1)})
		} else {
			s.mon.dropped++
		}

		if err!= nil {
			break
		}
	}
	conn.Close()
}

// Reading metrics from files in folder. This is a second way how to send metrics, except network
func (s Server)handleDirMetrics() {
	for ;; time.Sleep(time.Duration(s.conf.ClientSendInterval)*time.Second) {
		files, err := ioutil.ReadDir(s.conf.MetricDir)
		if err != nil {
			panic(err.Error())
		}
		for _, f := range files {
			results_list := readMetricsFromFile(s.conf.MetricDir+"/"+f.Name())
			s.mon.got.dir += len(results_list)
			s.cleanAndUseIncomingData(results_list)
		}

	}
}

func (s Server)runServer() {
	// Listen for incoming connections.
	l, err := net.Listen("tcp", s.conf.LocalBind)
	if err != nil {
		s.lg.Println("Failed to run server:", err.Error())
		os.Exit(1)
	} else {
		s.lg.Println("Server is running")
	}
	// Close the listener when the application closes.
	defer l.Close()

	// Run goroutine for reading metrics from metricDir
	go s.handleDirMetrics()
	// Run goroutine for sum metrics with prefix
	go s.sumMetricsWithPrefix()
	// Run goroutine for avg metrics with prefix
	go s.avgMetricsWithPrefix()

	for {
		// Listen for an incoming connection.
		conn, err := l.Accept()
		if err != nil {
			s.lg.Println("Error accepting: ", err.Error())
			os.Exit(1)
		}
		// Handle connections in a new goroutine.
		go s.handleRequest(conn)
	}
}<|endoftext|>


<commit_before><commit_msg>Added more track functions<commit_after><|endoftext|>


<commit_before><commit_msg>Implements testing function to compare 2 slices of integrers<commit_after><|endoftext|>


<commit_before><commit_msg>Emit number of processes<commit_after><|endoftext|>


<commit_before>// Copyright 2018 the u-root Authors. All rights reserved
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

package cmdline

import (
	"strings"
	"testing"
)

func TestRemoveFilter(t *testing.T) {
	toRemove := []string{"remove-1", "remove_2"}

	cl := `keep=5 remove_1=wontbethere remove-2=nomore keep2`
	exp := `keep=5 keep2`

	res := removeFilter(cl, toRemove)
	if res != exp {
		t.Errorf("error, expected: `%v`, got: `%v`", exp, res)
	}
}

func TestUpdateFilter(t *testing.T) {
	exampleCmdLine := `BOOT_IMAGE=/vmlinuz-4.11.2 ro ` +
		`test-flag test2-flag=8 uroot.uinitflags="a=3 skipfork test2-flag" ` +
		`uroot.initflags="systemd test-flag=3  test2-flag runlevel=2" ` +
		`root=LABEL=/ biosdevname=0 net.ifnames=0 fsck.repair=yes ` +
		`ipv6.autoconf=0 erst_disable nox2apic crashkernel=128M ` +
		`systemd.unified_cgroup_hierarchy=1 cgroup_no_v1=all console=tty0 ` +
		`console=ttyS0,115200 security=selinux selinux=1 enforcing=0`

	// Do this once, we'll over-write soon
	once.Do(cmdLineOpener)
	cmdLineReader := strings.NewReader(exampleCmdLine)
	procCmdLine = parse(cmdLineReader)

	if procCmdLine.Err != nil {
		t.Errorf("procCmdLine threw an error: %v", procCmdLine.Err)
	}

	toRemove := []string{"console", "earlyconsole"}
	toReuse := []string{"console", "not-present"}
	toAppend := "append=me"

	cl := `keep=5 console=ttyS1 keep2 earlyconsole=ttyS1`
	exp := `keep=5 keep2 append=me console=ttyS0,115200`

	filter := NewUpdateFilter(toAppend, toRemove, toReuse)
	res := filter.Update(cl)
	if res != exp {
		t.Errorf("error, expected: `%v`, got: `%v`", exp, res)
	}
}
<commit_msg>pkg/cmdline: update filter tests as per review<commit_after>// Copyright 2018 the u-root Authors. All rights reserved
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

package cmdline

import (
	"strings"
	"testing"
)

func TestRemoveFilter(t *testing.T) {
	toRemove := []string{"remove-1", "remove_2"}

	cl := `keep=5 remove_1=wontbethere remove-2=nomore keep2`
	want := `keep=5 keep2`

	got := removeFilter(cl, toRemove)
	if got != want {
		t.Errorf("removeFilter(%v,%v) = %v, want %v", cl, toRemove, got, want)
	}
}

func TestUpdateFilter(t *testing.T) {
	exampleCmdLine := `BOOT_IMAGE=/vmlinuz-4.11.2 ro ` +
		`test-flag test2-flag=8 uroot.uinitflags="a=3 skipfork test2-flag" ` +
		`uroot.initflags="systemd test-flag=3  test2-flag runlevel=2" ` +
		`root=LABEL=/ biosdevname=0 net.ifnames=0 fsck.repair=yes ` +
		`ipv6.autoconf=0 erst_disable nox2apic crashkernel=128M ` +
		`systemd.unified_cgroup_hierarchy=1 cgroup_no_v1=all console=tty0 ` +
		`console=ttyS0,115200 security=selinux selinux=1 enforcing=0`

	// Do this once, we'll over-write soon
	once.Do(cmdLineOpener)
	cmdLineReader := strings.NewReader(exampleCmdLine)
	procCmdLine = parse(cmdLineReader)

	if procCmdLine.Err != nil {
		t.Errorf("procCmdLine threw an error: %v", procCmdLine.Err)
	}

	toRemove := []string{"console", "earlyconsole"}
	toReuse := []string{"console", "not-present"}
	toAppend := "append=me"

	cl := `keep=5 console=ttyS1 keep2 earlyconsole=ttyS1`
	want := `keep=5 keep2 append=me console=ttyS0,115200`

	filter := NewUpdateFilter(toAppend, toRemove, toReuse)
	got := filter.Update(cl)
	if got != want {
		t.Errorf("Update(%v) = %v, want %v", cl, got, want)
	}
}
<|endoftext|>


<commit_before>// Copyright 2020 Google LLC
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//      http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package config

import (
	"context"
	"strings"
	"time"

	"github.com/google/exposure-notifications-verification-server/pkg/cache"
	"github.com/google/exposure-notifications-verification-server/pkg/database"
	"github.com/google/exposure-notifications-verification-server/pkg/ratelimit"

	"github.com/google/exposure-notifications-server/pkg/observability"

	firebase "firebase.google.com/go"
	"github.com/sethvargo/go-envconfig"
)

var _ IssueAPIConfig = (*ServerConfig)(nil)

// ServerConfig represents the environment based config for the server.
type ServerConfig struct {
	Firebase      FirebaseConfig
	Database      database.Config
	Observability observability.Config
	Cache         cache.Config

	Port string `env:"PORT,default=8080"`

	// Login Config
	SessionDuration   time.Duration `env:"SESSION_DURATION,default=24h"`
	RevokeCheckPeriod time.Duration `env:"REVOKE_CHECK_DURATION,default=5m"`

	// CookieKeys is a slice of bytes. The first is 64 bytes, the second is 32.
	// They should be base64-encoded.
	CookieKeys Base64ByteSlice `env:"COOKIE_KEYS,required"`

	// CookieDomain is the domain for which cookie should be valid.
	CookieDomain string `env:"COOKIE_DOMAIN"`

	// CSRFAuthKey is the authentication key. It must be 32-bytes and can be
	// generated with tools/gen-secret. The value's should be base64 encoded.
	CSRFAuthKey envconfig.Base64Bytes `env:"CSRF_AUTH_KEY,required"`

	// Application Config
	ServerName          string        `env:"SERVER_NAME,default=Diagnosis Verification Server"`
	CollisionRetryCount uint          `env:"COLLISION_RETRY_COUNT,default=6"`
	AllowedSymptomAge   time.Duration `env:"ALLOWED_PAST_SYMPTOM_DAYS,default=336h"` // 336h is 14 days.
	EnforceRealmQuotas  bool          `env:"ENFORCE_REALM_QUOTAS, default=false"`

	AssetsPath string `env:"ASSETS_PATH,default=./cmd/server/assets"`

	// Certificate signing key settings, needed for public key / settings display.
	CertificateSigning CertificateSigningConfig

	// If Dev mode is true, cookies aren't required to be sent over secure channels.
	// This includes CSRF protection base cookie. You want this false in production (the default).
	DevMode bool `env:"DEV_MODE"`

	// Rate limiting configuration
	RateLimit ratelimit.Config
}

// NewServerConfig initializes and validates a ServerConfig struct.
func NewServerConfig(ctx context.Context) (*ServerConfig, error) {
	var config ServerConfig
	if err := ProcessWith(ctx, &config, envconfig.OsLookuper()); err != nil {
		return nil, err
	}
	// For the, when inserting into the javascript, gets escaped and becomes unusable.
	config.Firebase.DatabaseURL = strings.ReplaceAll(config.Firebase.DatabaseURL, "https://", "")
	return &config, nil
}

func (c *ServerConfig) Validate() error {
	fields := []struct {
		Var  time.Duration
		Name string
	}{
		{c.SessionDuration, "SESSION_DURATION"},
		{c.RevokeCheckPeriod, "REVOKE_CHECK_DURATION"},
		{c.AllowedSymptomAge, "ALLOWED_PAST_SYMPTOM_DAYS"},
	}

	for _, f := range fields {
		if err := checkPositiveDuration(f.Var, f.Name); err != nil {
			return err
		}
	}

	return nil
}

func (c *ServerConfig) GetCollisionRetryCount() uint {
	return c.CollisionRetryCount
}

func (c *ServerConfig) GetAllowedSymptomAge() time.Duration {
	return c.AllowedSymptomAge
}

func (c *ServerConfig) GetEnforceRealmQuotas() bool {
	return c.EnforceRealmQuotas
}

func (c *ServerConfig) GetRateLimitConfig() *ratelimit.Config {
	return &c.RateLimit
}

func (c *ServerConfig) ObservabilityExporterConfig() *observability.Config {
	return &c.Observability
}

// FirebaseConfig represents configuration specific to firebase auth.
type FirebaseConfig struct {
	APIKey          string `env:"FIREBASE_API_KEY,required"`
	AuthDomain      string `env:"FIREBASE_AUTH_DOMAIN,required"`
	DatabaseURL     string `env:"FIREBASE_DATABASE_URL,required"`
	ProjectID       string `env:"FIREBASE_PROJECT_ID,required"`
	StorageBucket   string `env:"FIREBASE_STORAGE_BUCKET,required"`
	MessageSenderID string `env:"FIREBASE_MESSAGE_SENDER_ID,required"`
	AppID           string `env:"FIREBASE_APP_ID,required"`
	MeasurementID   string `env:"FIREBASE_MEASUREMENT_ID,required"`

	TermsOfServiceURL string `env:"FIREBASE_TERMS_OF_SERVICE_URL,required"`
	PrivacyPolicyURL  string `env:"FIREBASE_PRIVACY_POLICY_URL,required"`
}

// FirebaseConfig returns the firebase SDK config based on the local env config.
func (c *ServerConfig) FirebaseConfig() *firebase.Config {
	return &firebase.Config{
		DatabaseURL:   c.Firebase.DatabaseURL,
		ProjectID:     c.Firebase.ProjectID,
		StorageBucket: c.Firebase.StorageBucket,
	}
}

// Base64ByteSlice is a slice of base64-encoded strings that we want to convert
// to bytes.
type Base64ByteSlice []envconfig.Base64Bytes

// AsBytes returns the value as a slice of bytes instead of its main type.
func (c Base64ByteSlice) AsBytes() [][]byte {
	s := make([][]byte, len(c))
	for i, v := range c {
		s[i] = []byte(v)
	}
	return s
}
<commit_msg>Make default session duration 20h (#578)<commit_after>// Copyright 2020 Google LLC
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//      http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package config

import (
	"context"
	"strings"
	"time"

	"github.com/google/exposure-notifications-verification-server/pkg/cache"
	"github.com/google/exposure-notifications-verification-server/pkg/database"
	"github.com/google/exposure-notifications-verification-server/pkg/ratelimit"

	"github.com/google/exposure-notifications-server/pkg/observability"

	firebase "firebase.google.com/go"
	"github.com/sethvargo/go-envconfig"
)

var _ IssueAPIConfig = (*ServerConfig)(nil)

// ServerConfig represents the environment based config for the server.
type ServerConfig struct {
	Firebase      FirebaseConfig
	Database      database.Config
	Observability observability.Config
	Cache         cache.Config

	Port string `env:"PORT,default=8080"`

	// Login Config
	SessionDuration   time.Duration `env:"SESSION_DURATION, default=20h"`
	RevokeCheckPeriod time.Duration `env:"REVOKE_CHECK_DURATION,default=5m"`

	// CookieKeys is a slice of bytes. The first is 64 bytes, the second is 32.
	// They should be base64-encoded.
	CookieKeys Base64ByteSlice `env:"COOKIE_KEYS,required"`

	// CookieDomain is the domain for which cookie should be valid.
	CookieDomain string `env:"COOKIE_DOMAIN"`

	// CSRFAuthKey is the authentication key. It must be 32-bytes and can be
	// generated with tools/gen-secret. The value's should be base64 encoded.
	CSRFAuthKey envconfig.Base64Bytes `env:"CSRF_AUTH_KEY,required"`

	// Application Config
	ServerName          string        `env:"SERVER_NAME,default=Diagnosis Verification Server"`
	CollisionRetryCount uint          `env:"COLLISION_RETRY_COUNT,default=6"`
	AllowedSymptomAge   time.Duration `env:"ALLOWED_PAST_SYMPTOM_DAYS,default=336h"` // 336h is 14 days.
	EnforceRealmQuotas  bool          `env:"ENFORCE_REALM_QUOTAS, default=false"`

	AssetsPath string `env:"ASSETS_PATH,default=./cmd/server/assets"`

	// Certificate signing key settings, needed for public key / settings display.
	CertificateSigning CertificateSigningConfig

	// If Dev mode is true, cookies aren't required to be sent over secure channels.
	// This includes CSRF protection base cookie. You want this false in production (the default).
	DevMode bool `env:"DEV_MODE"`

	// Rate limiting configuration
	RateLimit ratelimit.Config
}

// NewServerConfig initializes and validates a ServerConfig struct.
func NewServerConfig(ctx context.Context) (*ServerConfig, error) {
	var config ServerConfig
	if err := ProcessWith(ctx, &config, envconfig.OsLookuper()); err != nil {
		return nil, err
	}
	// For the, when inserting into the javascript, gets escaped and becomes unusable.
	config.Firebase.DatabaseURL = strings.ReplaceAll(config.Firebase.DatabaseURL, "https://", "")
	return &config, nil
}

func (c *ServerConfig) Validate() error {
	fields := []struct {
		Var  time.Duration
		Name string
	}{
		{c.SessionDuration, "SESSION_DURATION"},
		{c.RevokeCheckPeriod, "REVOKE_CHECK_DURATION"},
		{c.AllowedSymptomAge, "ALLOWED_PAST_SYMPTOM_DAYS"},
	}

	for _, f := range fields {
		if err := checkPositiveDuration(f.Var, f.Name); err != nil {
			return err
		}
	}

	return nil
}

func (c *ServerConfig) GetCollisionRetryCount() uint {
	return c.CollisionRetryCount
}

func (c *ServerConfig) GetAllowedSymptomAge() time.Duration {
	return c.AllowedSymptomAge
}

func (c *ServerConfig) GetEnforceRealmQuotas() bool {
	return c.EnforceRealmQuotas
}

func (c *ServerConfig) GetRateLimitConfig() *ratelimit.Config {
	return &c.RateLimit
}

func (c *ServerConfig) ObservabilityExporterConfig() *observability.Config {
	return &c.Observability
}

// FirebaseConfig represents configuration specific to firebase auth.
type FirebaseConfig struct {
	APIKey          string `env:"FIREBASE_API_KEY,required"`
	AuthDomain      string `env:"FIREBASE_AUTH_DOMAIN,required"`
	DatabaseURL     string `env:"FIREBASE_DATABASE_URL,required"`
	ProjectID       string `env:"FIREBASE_PROJECT_ID,required"`
	StorageBucket   string `env:"FIREBASE_STORAGE_BUCKET,required"`
	MessageSenderID string `env:"FIREBASE_MESSAGE_SENDER_ID,required"`
	AppID           string `env:"FIREBASE_APP_ID,required"`
	MeasurementID   string `env:"FIREBASE_MEASUREMENT_ID,required"`

	TermsOfServiceURL string `env:"FIREBASE_TERMS_OF_SERVICE_URL,required"`
	PrivacyPolicyURL  string `env:"FIREBASE_PRIVACY_POLICY_URL,required"`
}

// FirebaseConfig returns the firebase SDK config based on the local env config.
func (c *ServerConfig) FirebaseConfig() *firebase.Config {
	return &firebase.Config{
		DatabaseURL:   c.Firebase.DatabaseURL,
		ProjectID:     c.Firebase.ProjectID,
		StorageBucket: c.Firebase.StorageBucket,
	}
}

// Base64ByteSlice is a slice of base64-encoded strings that we want to convert
// to bytes.
type Base64ByteSlice []envconfig.Base64Bytes

// AsBytes returns the value as a slice of bytes instead of its main type.
func (c Base64ByteSlice) AsBytes() [][]byte {
	s := make([][]byte, len(c))
	for i, v := range c {
		s[i] = []byte(v)
	}
	return s
}
<|endoftext|>


<commit_before>// +build linux

package node

import (
	"fmt"
	"time"

	"github.com/golang/glog"

	kapierrors "k8s.io/apimachinery/pkg/api/errors"
	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
	utilwait "k8s.io/apimachinery/pkg/util/wait"
	"k8s.io/client-go/tools/cache"

	networkapi "github.com/openshift/origin/pkg/network/apis/network"
	"github.com/openshift/origin/pkg/network/common"
)

func (node *OsdnNode) SubnetStartNode() error {
	go utilwait.Forever(node.watchSubnets, 0)
	return nil
}

type hostSubnetMap map[string]*networkapi.HostSubnet

func (node *OsdnNode) getLocalSubnet() (string, error) {
	var subnet *networkapi.HostSubnet
	// If the HostSubnet doesn't already exist, it will be created by the SDN master in
	// response to the kubelet registering itself with the master (which should be
	// happening in another goroutine in parallel with this). Sometimes this takes
	// unexpectedly long though, so give it plenty of time before returning an error
	// (since that will cause the node process to exit).
	backoff := utilwait.Backoff{
		// ~2 mins total
		Duration: time.Second,
		Factor:   1.5,
		Steps:    11,
	}
	err := utilwait.ExponentialBackoff(backoff, func() (bool, error) {
		var err error
		subnet, err = node.networkClient.Network().HostSubnets().Get(node.hostName, metav1.GetOptions{})
		if err == nil {
			return true, nil
		} else if kapierrors.IsNotFound(err) {
			glog.Warningf("Could not find an allocated subnet for node: %s, Waiting...", node.hostName)
			return false, nil
		} else {
			return false, err
		}
	})
	if err != nil {
		return "", fmt.Errorf("failed to get subnet for this host: %s, error: %v", node.hostName, err)
	}

	if err = node.networkInfo.ValidateNodeIP(subnet.HostIP); err != nil {
		return "", fmt.Errorf("failed to validate own HostSubnet: %v", err)
	}

	return subnet.Subnet, nil
}

func (node *OsdnNode) updateVXLANMulticastRules(subnets hostSubnetMap) {
	remoteIPs := make([]string, 0, len(subnets))
	for _, subnet := range subnets {
		if subnet.HostIP != node.localIP {
			remoteIPs = append(remoteIPs, subnet.HostIP)
		}
	}
	if err := node.oc.UpdateVXLANMulticastFlows(remoteIPs); err != nil {
		glog.Errorf("Error updating OVS VXLAN multicast flows: %v", err)
	}
}

func (node *OsdnNode) watchSubnets() {
	subnets := make(hostSubnetMap)
	common.RunEventQueue(node.networkClient.Network().RESTClient(), common.HostSubnets, func(delta cache.Delta) error {
		hs := delta.Object.(*networkapi.HostSubnet)
		if hs.HostIP == node.localIP {
			return nil
		}

		glog.V(5).Infof("Watch %s event for HostSubnet %q", delta.Type, hs.ObjectMeta.Name)
		switch delta.Type {
		case cache.Sync, cache.Added, cache.Updated:
			oldSubnet, exists := subnets[string(hs.UID)]
			if exists {
				if oldSubnet.HostIP == hs.HostIP {
					break
				} else {
					// Delete old subnet rules
					node.DeleteHostSubnetRules(oldSubnet)
				}
			}
			if err := node.networkInfo.ValidateNodeIP(hs.HostIP); err != nil {
				glog.Warningf("Ignoring invalid subnet for node %s: %v", hs.HostIP, err)
				break
			}

			node.AddHostSubnetRules(hs)
			subnets[string(hs.UID)] = hs
		case cache.Deleted:
			delete(subnets, string(hs.UID))
			node.DeleteHostSubnetRules(hs)
		}
		// Update multicast rules after all other changes have been processed
		node.updateVXLANMulticastRules(subnets)
		return nil
	})
}
<commit_msg>Bug 1534720 - SDN node should fetch latest local HostSubnet for the node<commit_after>// +build linux

package node

import (
	"fmt"
	"time"

	"github.com/golang/glog"

	kapierrors "k8s.io/apimachinery/pkg/api/errors"
	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
	utilwait "k8s.io/apimachinery/pkg/util/wait"
	"k8s.io/client-go/tools/cache"

	networkapi "github.com/openshift/origin/pkg/network/apis/network"
	"github.com/openshift/origin/pkg/network/common"
)

func (node *OsdnNode) SubnetStartNode() error {
	go utilwait.Forever(node.watchSubnets, 0)
	return nil
}

type hostSubnetMap map[string]*networkapi.HostSubnet

func (node *OsdnNode) getLocalSubnet() (string, error) {
	var subnet *networkapi.HostSubnet
	// If the HostSubnet doesn't already exist, it will be created by the SDN master in
	// response to the kubelet registering itself with the master (which should be
	// happening in another goroutine in parallel with this). Sometimes this takes
	// unexpectedly long though, so give it plenty of time before returning an error
	// (since that will cause the node process to exit).
	backoff := utilwait.Backoff{
		// ~2 mins total
		Duration: time.Second,
		Factor:   1.5,
		Steps:    11,
	}
	err := utilwait.ExponentialBackoff(backoff, func() (bool, error) {
		var err error
		subnet, err = node.networkClient.Network().HostSubnets().Get(node.hostName, metav1.GetOptions{})
		if err == nil {
			if subnet.HostIP == node.localIP {
				return true, nil
			} else {
				glog.Warningf("HostIP %q for local subnet does not match with nodeIP %q, "+
					"Waiting for master to update subnet for node %q ...", subnet.HostIP, node.localIP, node.hostName)
				return false, nil
			}
		} else if kapierrors.IsNotFound(err) {
			glog.Warningf("Could not find an allocated subnet for node: %s, Waiting...", node.hostName)
			return false, nil
		} else {
			return false, err
		}
	})
	if err != nil {
		return "", fmt.Errorf("failed to get subnet for this host: %s, error: %v", node.hostName, err)
	}

	if err = node.networkInfo.ValidateNodeIP(subnet.HostIP); err != nil {
		return "", fmt.Errorf("failed to validate own HostSubnet: %v", err)
	}

	return subnet.Subnet, nil
}

func (node *OsdnNode) updateVXLANMulticastRules(subnets hostSubnetMap) {
	remoteIPs := make([]string, 0, len(subnets))
	for _, subnet := range subnets {
		if subnet.HostIP != node.localIP {
			remoteIPs = append(remoteIPs, subnet.HostIP)
		}
	}
	if err := node.oc.UpdateVXLANMulticastFlows(remoteIPs); err != nil {
		glog.Errorf("Error updating OVS VXLAN multicast flows: %v", err)
	}
}

func (node *OsdnNode) watchSubnets() {
	subnets := make(hostSubnetMap)
	common.RunEventQueue(node.networkClient.Network().RESTClient(), common.HostSubnets, func(delta cache.Delta) error {
		hs := delta.Object.(*networkapi.HostSubnet)
		if hs.HostIP == node.localIP {
			return nil
		}

		glog.V(5).Infof("Watch %s event for HostSubnet %q", delta.Type, hs.ObjectMeta.Name)
		switch delta.Type {
		case cache.Sync, cache.Added, cache.Updated:
			oldSubnet, exists := subnets[string(hs.UID)]
			if exists {
				if oldSubnet.HostIP == hs.HostIP {
					break
				} else {
					// Delete old subnet rules
					node.DeleteHostSubnetRules(oldSubnet)
				}
			}
			if err := node.networkInfo.ValidateNodeIP(hs.HostIP); err != nil {
				glog.Warningf("Ignoring invalid subnet for node %s: %v", hs.HostIP, err)
				break
			}

			node.AddHostSubnetRules(hs)
			subnets[string(hs.UID)] = hs
		case cache.Deleted:
			delete(subnets, string(hs.UID))
			node.DeleteHostSubnetRules(hs)
		}
		// Update multicast rules after all other changes have been processed
		node.updateVXLANMulticastRules(subnets)
		return nil
	})
}
<|endoftext|>