<commit_before>package main

import (
	"github.com/elazarl/goproxy"

	"flag"
	"fmt"
	"io"
	"log"
	"net"
	"net/http"
	"os"
	"os/signal"
	"regexp"
	"strings"
	"syscall"
	"time"
)

const (
	appendLog int = iota
	reopenLog int = iota
)

// Digest Auth. operation type
const (
	validateUser int = iota
	getNonce     int = iota
)

// Digest auth resp status
const (
	authOk     int = iota
	authFailed int = iota
	nonceOk    int = iota
)

const proxyForwardedForHeader = "X-Forwarded-For"

type Meta struct {
	action int
	req    *http.Request
	resp   *http.Response
	err    error
	time   time.Time
}

type BasicAuthRequest struct {
	data        *BasicAuthData
	respChannel chan *BasicAuthResponse
}

type BasicAuthResponse struct {
	status bool
}

type DigestAuthRequest struct {
	data        *DigestAuthData
	op          int
	respChannel chan *DigestAuthResponse
}

type DigestAuthResponse struct {
	data   string
	status int
}

func makeBasicAuthValidator(auth *BasicAuth) BasicAuthFunc {
	channel := make(chan *BasicAuthRequest)
	validator := func() {
		for e := range channel {
			status := auth.Validate(e.data)
			e.respChannel <- &BasicAuthResponse{status}
		}
	}

	go validator()

	return func(authData *BasicAuthData) *BasicAuthResponse {
		request := &BasicAuthRequest{authData, make(chan *BasicAuthResponse)}
		channel <- request
		return <-request.respChannel
	}
}

func makeDigestAuthValidator(auth *DigestAuth) DigestAuthFunc {
	channel := make(chan *DigestAuthRequest)
	processor := func() {
		for e := range channel {
			var response *DigestAuthResponse
			switch e.op {
			case validateUser:
				status := auth.Validate(e.data)
				if status {
					response = &DigestAuthResponse{status: authOk}
				} else {
					response = &DigestAuthResponse{status: authFailed}
				}
			case getNonce:
				nonce := auth.NewNonce()
				response = &DigestAuthResponse{status: nonceOk, data: nonce}
			default:
				panic("unexpected operation type")
			}
			e.respChannel <- response
		}
	}

	go processor()

	f := func(authData *DigestAuthData, op int) *DigestAuthResponse {
		request := &DigestAuthRequest{data: authData, op: op, respChannel: make(chan *DigestAuthResponse)}
		channel <- request
		return <-request.respChannel
	}

	return f
}

func fprintf(nr *int64, err *error, w io.Writer, pat string, a ...interface{}) {
	if *err != nil {
		return
	}
	var n int
	n, *err = fmt.Fprintf(w, pat, a...)
	*nr += int64(n)
}

func write(nr *int64, err *error, w io.Writer, b []byte) {
	if *err != nil {
		return
	}
	var n int
	n, *err = w.Write(b)
	*nr += int64(n)
}

func (m *Meta) WriteTo(w io.Writer) (nr int64, err error) {
	if m.resp != nil {
		if m.resp.Request != nil {
			fprintf(&nr, &err, w,
				"%v %v %v %v %v %v\n",
				m.time.Format(time.RFC3339),
				m.resp.Request.RemoteAddr,
				m.resp.Request.Method,
				m.resp.Request.URL,
				m.resp.StatusCode,
				m.resp.ContentLength)
		} else {
			fprintf(&nr, &err, w,
				"%v %v %v %v %v %v\n",
				m.time.Format(time.RFC3339),
				"-",
				"-",
				"-",
				m.resp.StatusCode,
				m.resp.ContentLength)
		}
	} else if m.req != nil {
		fprintf(&nr, &err, w,
			"%v %v %v %v %v %v\n",
			m.time.Format(time.RFC3339),
			m.req.RemoteAddr,
			m.req.Method,
			m.req.URL,
			"-",
			"-")
	}

	return
}

type HttpLogger struct {
	path        string
	logChannel  chan *Meta
	errorChanel chan error
}

func NewLogger(conf *Configuration) *HttpLogger {
	var fh *os.File

	if conf.AccessLog != "" {
		var err error
		fh, err = os.OpenFile(conf.AccessLog, os.O_APPEND|os.O_WRONLY|os.O_CREATE, 0600)
		if err != nil {
			log.Fatalf("Couldn't open log file: %v", err)
		}
	}

	logger := &HttpLogger{conf.AccessLog, make(chan *Meta), make(chan error)}

	go func() {
		for m := range logger.logChannel {
			if fh != nil {
				switch m.action {
				case appendLog:
					if _, err := m.WriteTo(fh); err != nil {
						log.Println("Can't write meta", err)
					}
				case reopenLog:
					fh.Close()
					var err error
					fh, err = os.OpenFile(conf.AccessLog, os.O_APPEND|os.O_WRONLY|os.O_CREATE, 0600)
					if err != nil {
						log.Fatalf("Couldn't reopenLog log file: %v", err)
					}
				}
			}
		}
		logger.errorChanel <- fh.Close()
	}()

	return logger
}

var emptyResp = &http.Response{}
var emptyReq = &http.Request{}

func (logger *HttpLogger) LogReq(req *http.Request, ctx *goproxy.ProxyCtx) {
	if req == nil {
		req = emptyReq
	}

	logger.LogMeta(&Meta{
		action: appendLog,
		req:    req,
		err:    ctx.Error,
		time:   time.Now()})
}

func (logger *HttpLogger) LogResp(resp *http.Response, ctx *goproxy.ProxyCtx) {
	if resp == nil {
		resp = emptyResp
	}

	logger.LogMeta(&Meta{
		action: appendLog,
		resp:   resp,
		err:    ctx.Error,
		time:   time.Now()})
}

func (logger *HttpLogger) LogMeta(m *Meta) {
	logger.logChannel <- m
}

func (logger *HttpLogger) Close() error {
	close(logger.logChannel)
	return <-logger.errorChanel
}

func (logger *HttpLogger) Reopen() {
	logger.LogMeta(&Meta{action: reopenLog})
}

func setAllowedNetworksHandler(conf *Configuration, proxy *goproxy.ProxyHttpServer) {
	if conf.AllowedNetworks != nil && len(conf.AllowedNetworks) > 0 {
		proxy.OnRequest(goproxy.Not(sourceIpMatches(conf.AllowedNetworks))).HandleConnect(goproxy.AlwaysReject)
		proxy.OnRequest(goproxy.Not(sourceIpMatches(conf.AllowedNetworks))).DoFunc(
			func(req *http.Request, ctx *goproxy.ProxyCtx) (*http.Request, *http.Response) {
				return req, goproxy.NewResponse(req, goproxy.ContentTypeHtml, http.StatusForbidden, "Access denied")
			})
	}

	if conf.DisallowedNetworks != nil && len(conf.DisallowedNetworks) > 0 {
		proxy.OnRequest(sourceIpMatches(conf.DisallowedNetworks)).HandleConnect(goproxy.AlwaysReject)
		proxy.OnRequest(sourceIpMatches(conf.DisallowedNetworks)).DoFunc(
			func(req *http.Request, ctx *goproxy.ProxyCtx) (*http.Request, *http.Response) {
				return req, goproxy.NewResponse(req, goproxy.ContentTypeHtml, http.StatusForbidden, "Access denied")
			})
	}
}

func sourceIpMatches(networks []string) goproxy.ReqConditionFunc {
	cidrs := make([](*net.IPNet), len(networks))
	for idx, network := range networks {
		_, cidrnet, _ := net.ParseCIDR(network)
		cidrs[idx] = cidrnet
	}

	return func(req *http.Request, ctx *goproxy.ProxyCtx) bool {
		ip, _, err := net.SplitHostPort(req.RemoteAddr)
		if err != nil {
			ctx.Warnf("coudn't parse remote address %v: %v", req.RemoteAddr, err)
			return false
		}
		addr := net.ParseIP(ip)
		for _, network := range cidrs {
			if network.Contains(addr) {
				return true
			}
		}
		return false
	}
}

func setAllowedConnectPortsHandler(conf *Configuration, proxy *goproxy.ProxyHttpServer) {
	if conf.AllowedConnectPorts != nil && len(conf.AllowedConnectPorts) > 0 {
		ports := make([]string, len(conf.AllowedConnectPorts))
		for i, v := range conf.AllowedConnectPorts {
			ports[i] = ":" + v
		}
		rx := "(" + strings.Join(ports, "|") + ")$"
		proxy.OnRequest(goproxy.Not(goproxy.ReqHostMatches(regexp.MustCompile(rx)))).HandleConnect(goproxy.AlwaysReject)
	}
}

func setForwardedForHeaderHandler(conf *Configuration, proxy *goproxy.ProxyHttpServer) {
	f := func(req *http.Request, ctx *goproxy.ProxyCtx) (*http.Request, *http.Response) {
		ip, _, err := net.SplitHostPort(req.RemoteAddr)
		if err != nil {
			ctx.Warnf("coudn't parse remote address %v: %v", req.RemoteAddr, err)
			return req, nil
		}

		switch conf.ForwardedFor {
		case "on":
			header := req.Header.Get(proxyForwardedForHeader)
			if header == "" {
				req.Header.Add(proxyForwardedForHeader, ip)
			} else {
				header = header + ", " + ip
				req.Header.Del(proxyForwardedForHeader)
				req.Header.Add(proxyForwardedForHeader, header)
			}
		case "delete":
			req.Header.Del(proxyForwardedForHeader)
		case "truncate":
			req.Header.Del(proxyForwardedForHeader)
			req.Header.Add(proxyForwardedForHeader, ip)
		}

		return req, nil
	}

	proxy.OnRequest().DoFunc(f)
}

func CreateProxy(conf *Configuration) *goproxy.ProxyHttpServer {
	proxy := goproxy.NewProxyHttpServer()
	setActivityLog(conf, proxy)

	return proxy
}

func setActivityLog(conf *Configuration, proxy *goproxy.ProxyHttpServer) {
	if conf.ActivityLog != "" {
		fh, err := os.OpenFile(conf.ActivityLog, os.O_APPEND|os.O_WRONLY|os.O_CREATE, 0600)
		if err != nil {
			log.Fatalf("couldn't open activity log file %v: %v", conf.ActivityLog, err)
		}
		proxy.Logger = log.New(fh, "", log.LstdFlags)
	}
}

func main() {
	config := flag.String("config", "microproxy.json", "proxy conf file")
	verbose := flag.Bool("v", false, "enable verbose debug mode")

	flag.Parse()

	conf := NewConfiguration(*config)

	proxy := CreateProxy(conf)
	proxy.Verbose = *verbose

	logger := NewLogger(conf)

	if conf.AuthFile != "" {
		if conf.AuthType == "basic" {
			auth, err := NewBasicAuthFromFile(conf.AuthFile)
			if err != nil {
				proxy.Logger.Fatalf("couldn't create basic auth structure: %v\n", err)
			}
			ProxyBasic(proxy, conf.AuthRealm, makeBasicAuthValidator(auth))
		} else {
			auth, err := NewDigestAuthFromFile(conf.AuthFile)
			if err != nil {
				proxy.Logger.Fatalf("couldn't create digest auth structure: %v\n", err)
			}
			ProxyDigest(proxy, conf.AuthRealm, makeDigestAuthValidator(auth))
		}
	}

	proxy.OnRequest().HandleConnectFunc(
		func(host string, ctx *goproxy.ProxyCtx) (*goproxy.ConnectAction, string) {
			if ctx.Req == nil {
				ctx.Req = emptyReq
			}

			logger.LogMeta(&Meta{
				action: appendLog,
				req:    ctx.Req,
				err:    ctx.Error,
				time:   time.Now()})

			return goproxy.OkConnect, host
		})

	setAllowedConnectPortsHandler(conf, proxy)
	setAllowedNetworksHandler(conf, proxy)
	setForwardedForHeaderHandler(conf, proxy)

	proxy.OnResponse().DoFunc(
		func(resp *http.Response, ctx *goproxy.ProxyCtx) *http.Response {
			logger.LogResp(resp, ctx)
			return resp
		})

	signalChannel := make(chan os.Signal)
	signal.Notify(signalChannel, os.Interrupt, syscall.SIGUSR1)

	go func() {
		for {
			sig := <-signalChannel
			switch sig {
			case os.Interrupt:
				proxy.Logger.Println("got interrupt signal, exiting")
				logger.Close()
				os.Exit(0)
			case syscall.SIGUSR1:
				proxy.Logger.Println("got USR1 signal, reopening logs")
				// Reopen access log
				logger.Reopen()
				// Reopen activity log
				setActivityLog(conf, proxy)
			}
		}
	}()

	proxy.Logger.Println("starting proxy")

	log.Fatal(http.ListenAndServe(conf.Listen, proxy))
}
<commit_msg>more descriptive messages<commit_after>package main

import (
	"github.com/elazarl/goproxy"

	"flag"
	"fmt"
	"io"
	"log"
	"net"
	"net/http"
	"os"
	"os/signal"
	"regexp"
	"strings"
	"syscall"
	"time"
)

const (
	appendLog int = iota
	reopenLog int = iota
)

// Digest auth. operation type
const (
	validateUser int = iota
	getNonce     int = iota
)

// Digest auth. resp status
const (
	authOk     int = iota
	authFailed int = iota
	nonceOk    int = iota
)

const proxyForwardedForHeader = "X-Forwarded-For"

type Meta struct {
	action int
	req    *http.Request
	resp   *http.Response
	err    error
	time   time.Time
}

type BasicAuthRequest struct {
	data        *BasicAuthData
	respChannel chan *BasicAuthResponse
}

type BasicAuthResponse struct {
	status bool
}

type DigestAuthRequest struct {
	data        *DigestAuthData
	op          int
	respChannel chan *DigestAuthResponse
}

type DigestAuthResponse struct {
	data   string
	status int
}

func makeBasicAuthValidator(auth *BasicAuth) BasicAuthFunc {
	channel := make(chan *BasicAuthRequest)
	validator := func() {
		for e := range channel {
			status := auth.Validate(e.data)
			e.respChannel <- &BasicAuthResponse{status}
		}
	}

	go validator()

	return func(authData *BasicAuthData) *BasicAuthResponse {
		request := &BasicAuthRequest{authData, make(chan *BasicAuthResponse)}
		channel <- request
		return <-request.respChannel
	}
}

func makeDigestAuthValidator(auth *DigestAuth) DigestAuthFunc {
	channel := make(chan *DigestAuthRequest)
	processor := func() {
		for e := range channel {
			var response *DigestAuthResponse
			switch e.op {
			case validateUser:
				status := auth.Validate(e.data)
				if status {
					response = &DigestAuthResponse{status: authOk}
				} else {
					response = &DigestAuthResponse{status: authFailed}
				}
			case getNonce:
				nonce := auth.NewNonce()
				response = &DigestAuthResponse{status: nonceOk, data: nonce}
			default:
				panic("unexpected operation type")
			}
			e.respChannel <- response
		}
	}

	go processor()

	f := func(authData *DigestAuthData, op int) *DigestAuthResponse {
		request := &DigestAuthRequest{data: authData, op: op, respChannel: make(chan *DigestAuthResponse)}
		channel <- request
		return <-request.respChannel
	}

	return f
}

func fprintf(nr *int64, err *error, w io.Writer, pat string, a ...interface{}) {
	if *err != nil {
		return
	}
	var n int
	n, *err = fmt.Fprintf(w, pat, a...)
	*nr += int64(n)
}

func write(nr *int64, err *error, w io.Writer, b []byte) {
	if *err != nil {
		return
	}
	var n int
	n, *err = w.Write(b)
	*nr += int64(n)
}

func (m *Meta) WriteTo(w io.Writer) (nr int64, err error) {
	if m.resp != nil {
		if m.resp.Request != nil {
			fprintf(&nr, &err, w,
				"%v %v %v %v %v %v\n",
				m.time.Format(time.RFC3339),
				m.resp.Request.RemoteAddr,
				m.resp.Request.Method,
				m.resp.Request.URL,
				m.resp.StatusCode,
				m.resp.ContentLength)
		} else {
			fprintf(&nr, &err, w,
				"%v %v %v %v %v %v\n",
				m.time.Format(time.RFC3339),
				"-",
				"-",
				"-",
				m.resp.StatusCode,
				m.resp.ContentLength)
		}
	} else if m.req != nil {
		fprintf(&nr, &err, w,
			"%v %v %v %v %v %v\n",
			m.time.Format(time.RFC3339),
			m.req.RemoteAddr,
			m.req.Method,
			m.req.URL,
			"-",
			"-")
	}

	return
}

type HttpLogger struct {
	path        string
	logChannel  chan *Meta
	errorChanel chan error
}

func NewLogger(conf *Configuration) *HttpLogger {
	var fh *os.File

	if conf.AccessLog != "" {
		var err error
		fh, err = os.OpenFile(conf.AccessLog, os.O_APPEND|os.O_WRONLY|os.O_CREATE, 0600)
		if err != nil {
			log.Fatalf("Couldn't open log file: %v", err)
		}
	}

	logger := &HttpLogger{conf.AccessLog, make(chan *Meta), make(chan error)}

	go func() {
		for m := range logger.logChannel {
			if fh != nil {
				switch m.action {
				case appendLog:
					if _, err := m.WriteTo(fh); err != nil {
						log.Println("Can't write meta", err)
					}
				case reopenLog:
					fh.Close()
					var err error
					fh, err = os.OpenFile(conf.AccessLog, os.O_APPEND|os.O_WRONLY|os.O_CREATE, 0600)
					if err != nil {
						log.Fatalf("Couldn't reopen log file: %v", err)
					}
				}
			}
		}
		logger.errorChanel <- fh.Close()
	}()

	return logger
}

var emptyResp = &http.Response{}
var emptyReq = &http.Request{}

func (logger *HttpLogger) LogReq(req *http.Request, ctx *goproxy.ProxyCtx) {
	if req == nil {
		req = emptyReq
	}

	logger.LogMeta(&Meta{
		action: appendLog,
		req:    req,
		err:    ctx.Error,
		time:   time.Now()})
}

func (logger *HttpLogger) LogResp(resp *http.Response, ctx *goproxy.ProxyCtx) {
	if resp == nil {
		resp = emptyResp
	}

	logger.LogMeta(&Meta{
		action: appendLog,
		resp:   resp,
		err:    ctx.Error,
		time:   time.Now()})
}

func (logger *HttpLogger) LogMeta(m *Meta) {
	logger.logChannel <- m
}

func (logger *HttpLogger) Close() error {
	close(logger.logChannel)
	return <-logger.errorChanel
}

func (logger *HttpLogger) Reopen() {
	logger.LogMeta(&Meta{action: reopenLog})
}

func setAllowedNetworksHandler(conf *Configuration, proxy *goproxy.ProxyHttpServer) {
	if conf.AllowedNetworks != nil && len(conf.AllowedNetworks) > 0 {
		proxy.OnRequest(goproxy.Not(sourceIpMatches(conf.AllowedNetworks))).HandleConnect(goproxy.AlwaysReject)
		proxy.OnRequest(goproxy.Not(sourceIpMatches(conf.AllowedNetworks))).DoFunc(
			func(req *http.Request, ctx *goproxy.ProxyCtx) (*http.Request, *http.Response) {
				return req, goproxy.NewResponse(req, goproxy.ContentTypeHtml, http.StatusForbidden, "Access denied")
			})
	}

	if conf.DisallowedNetworks != nil && len(conf.DisallowedNetworks) > 0 {
		proxy.OnRequest(sourceIpMatches(conf.DisallowedNetworks)).HandleConnect(goproxy.AlwaysReject)
		proxy.OnRequest(sourceIpMatches(conf.DisallowedNetworks)).DoFunc(
			func(req *http.Request, ctx *goproxy.ProxyCtx) (*http.Request, *http.Response) {
				return req, goproxy.NewResponse(req, goproxy.ContentTypeHtml, http.StatusForbidden, "Access denied")
			})
	}
}

func sourceIpMatches(networks []string) goproxy.ReqConditionFunc {
	cidrs := make([](*net.IPNet), len(networks))
	for idx, network := range networks {
		_, cidrnet, _ := net.ParseCIDR(network)
		cidrs[idx] = cidrnet
	}

	return func(req *http.Request, ctx *goproxy.ProxyCtx) bool {
		ip, _, err := net.SplitHostPort(req.RemoteAddr)
		if err != nil {
			ctx.Warnf("coudn't parse remote address %v: %v", req.RemoteAddr, err)
			return false
		}
		addr := net.ParseIP(ip)
		for _, network := range cidrs {
			if network.Contains(addr) {
				return true
			}
		}
		return false
	}
}

func setAllowedConnectPortsHandler(conf *Configuration, proxy *goproxy.ProxyHttpServer) {
	if conf.AllowedConnectPorts != nil && len(conf.AllowedConnectPorts) > 0 {
		ports := make([]string, len(conf.AllowedConnectPorts))
		for i, v := range conf.AllowedConnectPorts {
			ports[i] = ":" + v
		}
		rx := "(" + strings.Join(ports, "|") + ")$"
		proxy.OnRequest(goproxy.Not(goproxy.ReqHostMatches(regexp.MustCompile(rx)))).HandleConnect(goproxy.AlwaysReject)
	}
}

func setForwardedForHeaderHandler(conf *Configuration, proxy *goproxy.ProxyHttpServer) {
	f := func(req *http.Request, ctx *goproxy.ProxyCtx) (*http.Request, *http.Response) {
		ip, _, err := net.SplitHostPort(req.RemoteAddr)
		if err != nil {
			ctx.Warnf("coudn't parse remote address %v: %v", req.RemoteAddr, err)
			return req, nil
		}

		switch conf.ForwardedFor {
		case "on":
			header := req.Header.Get(proxyForwardedForHeader)
			if header == "" {
				req.Header.Add(proxyForwardedForHeader, ip)
			} else {
				header = header + ", " + ip
				req.Header.Del(proxyForwardedForHeader)
				req.Header.Add(proxyForwardedForHeader, header)
			}
		case "delete":
			req.Header.Del(proxyForwardedForHeader)
		case "truncate":
			req.Header.Del(proxyForwardedForHeader)
			req.Header.Add(proxyForwardedForHeader, ip)
		}

		return req, nil
	}

	proxy.OnRequest().DoFunc(f)
}

func CreateProxy(conf *Configuration) *goproxy.ProxyHttpServer {
	proxy := goproxy.NewProxyHttpServer()
	setActivityLog(conf, proxy)

	return proxy
}

func setActivityLog(conf *Configuration, proxy *goproxy.ProxyHttpServer) {
	if conf.ActivityLog != "" {
		fh, err := os.OpenFile(conf.ActivityLog, os.O_APPEND|os.O_WRONLY|os.O_CREATE, 0600)
		if err != nil {
			log.Fatalf("couldn't open activity log file %v: %v", conf.ActivityLog, err)
		}
		proxy.Logger = log.New(fh, "", log.LstdFlags)
	}
}

func main() {
	config := flag.String("config", "microproxy.json", "proxy configuration file")
	verbose := flag.Bool("v", false, "enable verbose debug mode")

	flag.Parse()

	conf := NewConfiguration(*config)

	proxy := CreateProxy(conf)
	proxy.Verbose = *verbose

	logger := NewLogger(conf)

	if conf.AuthFile != "" {
		if conf.AuthType == "basic" {
			auth, err := NewBasicAuthFromFile(conf.AuthFile)
			if err != nil {
				proxy.Logger.Fatalf("couldn't create basic auth structure: %v\n", err)
			}
			ProxyBasic(proxy, conf.AuthRealm, makeBasicAuthValidator(auth))
		} else {
			auth, err := NewDigestAuthFromFile(conf.AuthFile)
			if err != nil {
				proxy.Logger.Fatalf("couldn't create digest auth structure: %v\n", err)
			}
			ProxyDigest(proxy, conf.AuthRealm, makeDigestAuthValidator(auth))
		}
	}

	proxy.OnRequest().HandleConnectFunc(
		func(host string, ctx *goproxy.ProxyCtx) (*goproxy.ConnectAction, string) {
			if ctx.Req == nil {
				ctx.Req = emptyReq
			}

			logger.LogMeta(&Meta{
				action: appendLog,
				req:    ctx.Req,
				err:    ctx.Error,
				time:   time.Now()})

			return goproxy.OkConnect, host
		})

	setAllowedConnectPortsHandler(conf, proxy)
	setAllowedNetworksHandler(conf, proxy)
	setForwardedForHeaderHandler(conf, proxy)

	proxy.OnResponse().DoFunc(
		func(resp *http.Response, ctx *goproxy.ProxyCtx) *http.Response {
			logger.LogResp(resp, ctx)
			return resp
		})

	signalChannel := make(chan os.Signal)
	signal.Notify(signalChannel, os.Interrupt, syscall.SIGUSR1)

	go func() {
		for {
			sig := <-signalChannel
			switch sig {
			case os.Interrupt:
				proxy.Logger.Println("got interrupt signal, exiting")
				logger.Close()
				os.Exit(0)
			case syscall.SIGUSR1:
				proxy.Logger.Println("got USR1 signal, reopening logs")
				// Reopen access log
				logger.Reopen()
				// Reopen activity log
				setActivityLog(conf, proxy)
			}
		}
	}()

	proxy.Logger.Println("starting proxy")

	log.Fatal(http.ListenAndServe(conf.Listen, proxy))
}
<|endoftext|>


<commit_before>// Copyright 2014 Manu Martinez-Almeida.  All rights reserved.
// Use of this source code is governed by a MIT style
// license that can be found in the LICENSE file.

package mid

import (
	"fmt"
	"io"
	"os"
	"time"

	"github.com/go-ego/ego"
	"github.com/go-ego/ego/mid/util"

	"github.com/mattn/go-isatty"
)

var (
	green        = string([]byte{27, 91, 57, 55, 59, 52, 50, 109})
	white        = string([]byte{27, 91, 57, 48, 59, 52, 55, 109})
	yellow       = string([]byte{27, 91, 57, 55, 59, 52, 51, 109})
	red          = string([]byte{27, 91, 57, 55, 59, 52, 49, 109})
	blue         = string([]byte{27, 91, 57, 55, 59, 52, 52, 109})
	magenta      = string([]byte{27, 91, 57, 55, 59, 52, 53, 109})
	cyan         = string([]byte{27, 91, 57, 55, 59, 52, 54, 109})
	reset        = string([]byte{27, 91, 48, 109})
	disableColor = false
)

// DisableConsoleColor disables color output in the console
func DisableConsoleColor() {
	disableColor = true
}

// ErrorLogger returns a handlerfunc for any error type
func ErrorLogger() ego.HandlerFunc {
	return ErrorLoggerT(util.ErrorTypeAny)
}

// ErrorLoggerT returns a handlerfunc for a given error type
func ErrorLoggerT(typ util.ErrorType) ego.HandlerFunc {
	return func(c *ego.Context) {
		c.Next()
		errors := c.Errors.ByType(typ)
		if len(errors) > 0 {
			c.JSON(-1, errors)
		}
	}
}

// Logger instances a Logger middleware that will write the logs to mid.DefaultWriter
// By default mid.DefaultWriter = os.Stdout
func Logger() ego.HandlerFunc {
	return LoggerWithWriter(ego.DefaultWriter)
}

// LoggerWithWriter instance a Logger middleware with the specified writter buffer.
// Example: os.Stdout, a file opened in write mode, a socket...
func LoggerWithWriter(out io.Writer, notlogged ...string) ego.HandlerFunc {
	isTerm := true

	// if w, ok := out.(*os.File); !ok || !isatty.IsTerminal(w.Fd()) {
	if w, ok := out.(*os.File); !ok ||
		(os.Getenv("TERM") == "dumb" || (!isatty.IsTerminal(w.Fd()) && !isatty.IsCygwinTerminal(w.Fd()))) ||
		disableColor {
		isTerm = false
	}

	var skip map[string]struct{}

	if length := len(notlogged); length > 0 {
		skip = make(map[string]struct{}, length)

		for _, path := range notlogged {
			skip[path] = struct{}{}
		}
	}

	return func(c *ego.Context) {
		// Start timer
		start := time.Now()
		path := c.Request.URL.Path
		raw := c.Request.URL.RawQuery

		// Process request
		c.Next()

		// Log only when path is not being skipped
		if _, ok := skip[path]; !ok {
			// Stop timer
			end := time.Now()
			latency := end.Sub(start)

			clientIP := c.ClientIP()
			// method := Colorize(c.Request.Method, "note")
			method := c.Request.Method
			statusCode := c.Writer.Status()
			var statusColor, methodColor string
			if isTerm {
				statusColor = colorForStatus(statusCode)
				methodColor = colorForMethod(method)
			}
			comment := c.Errors.ByType(util.ErrorTypePrivate).String()

			if raw != "" {
				path = path + "?" + raw
			}
			// fmt.Fprintf(out, "[EGO] %v |%s %3d %s| %13v | %s |%s  %s %-7s %s\n%s",
			fmt.Fprintf(out, "[EGO] %v |%s %3d %s| %13v | %15s |%s  %s %-7s %s\n%s",
				end.Format("2006/01/02 - 15:04:05"),
				statusColor, statusCode, reset,
				latency,
				clientIP,
				methodColor, method, reset,
				path,
				comment,
			)
		}
	}
}

func colorForStatus(code int) string {
	switch {
	case code >= 200 && code < 300:
		return green
	case code >= 300 && code < 400:
		return white
	case code >= 400 && code < 500:
		return yellow
	default:
		return red
	}
}

func colorForMethod(method string) string {
	switch method {
	case "GET":
		return blue
	case "POST":
		return cyan
	case "PUT":
		return yellow
	case "DELETE":
		return red
	case "PATCH":
		return green
	case "HEAD":
		return magenta
	case "OPTIONS":
		return white
	default:
		return reset
	}
}
<commit_msg>log format<commit_after>// Copyright 2014 Manu Martinez-Almeida.  All rights reserved.
// Use of this source code is governed by a MIT style
// license that can be found in the LICENSE file.

package mid

import (
	"fmt"
	"io"
	"os"
	"time"

	"github.com/go-ego/ego"
	"github.com/go-ego/ego/mid/util"

	"github.com/mattn/go-isatty"
)

var (
	green        = string([]byte{27, 91, 57, 55, 59, 52, 50, 109})
	white        = string([]byte{27, 91, 57, 48, 59, 52, 55, 109})
	yellow       = string([]byte{27, 91, 57, 55, 59, 52, 51, 109})
	red          = string([]byte{27, 91, 57, 55, 59, 52, 49, 109})
	blue         = string([]byte{27, 91, 57, 55, 59, 52, 52, 109})
	magenta      = string([]byte{27, 91, 57, 55, 59, 52, 53, 109})
	cyan         = string([]byte{27, 91, 57, 55, 59, 52, 54, 109})
	reset        = string([]byte{27, 91, 48, 109})
	disableColor = false
)

// DisableConsoleColor disables color output in the console
func DisableConsoleColor() {
	disableColor = true
}

// ErrorLogger returns a handlerfunc for any error type
func ErrorLogger() ego.HandlerFunc {
	return ErrorLoggerT(util.ErrorTypeAny)
}

// ErrorLoggerT returns a handlerfunc for a given error type
func ErrorLoggerT(typ util.ErrorType) ego.HandlerFunc {
	return func(c *ego.Context) {
		c.Next()
		errors := c.Errors.ByType(typ)
		if len(errors) > 0 {
			c.JSON(-1, errors)
		}
	}
}

// Logger instances a Logger middleware that will write the logs to mid.DefaultWriter
// By default mid.DefaultWriter = os.Stdout
func Logger() ego.HandlerFunc {
	return LoggerWithWriter(ego.DefaultWriter)
}

// LoggerWithWriter instance a Logger middleware with the specified writter buffer.
// Example: os.Stdout, a file opened in write mode, a socket...
func LoggerWithWriter(out io.Writer, notlogged ...string) ego.HandlerFunc {
	isTerm := true

	// if w, ok := out.(*os.File); !ok || !isatty.IsTerminal(w.Fd()) {
	if w, ok := out.(*os.File); !ok ||
		(os.Getenv("TERM") == "dumb" || (!isatty.IsTerminal(w.Fd()) && !isatty.IsCygwinTerminal(w.Fd()))) ||
		disableColor {
		isTerm = false
	}

	var skip map[string]struct{}

	if length := len(notlogged); length > 0 {
		skip = make(map[string]struct{}, length)

		for _, path := range notlogged {
			skip[path] = struct{}{}
		}
	}

	return func(c *ego.Context) {
		// Start timer
		start := time.Now()
		path := c.Request.URL.Path
		raw := c.Request.URL.RawQuery

		// Process request
		c.Next()

		// Log only when path is not being skipped
		if _, ok := skip[path]; !ok {
			// Stop timer
			end := time.Now()
			latency := end.Sub(start)

			clientIP := c.ClientIP()
			// method := Colorize(c.Request.Method, "note")
			method := c.Request.Method
			statusCode := c.Writer.Status()
			var statusColor, methodColor string
			if isTerm {
				statusColor = colorForStatus(statusCode)
				methodColor = colorForMethod(method)
			}
			comment := c.Errors.ByType(util.ErrorTypePrivate).String()

			if raw != "" {
				path = path + "?" + raw
			}
			// fmt.Fprintf(out, "[EGO] %v |%s %3d %s| %13v | %15s |%s  %s %-7s %s\n%s",
			fmt.Fprintf(out, "[EGO] %v |%s %3d %s| %13v | %15s |%s %-7s %s %s\n%s",
				end.Format("2006/01/02 - 15:04:05"),
				statusColor, statusCode, reset,
				latency,
				clientIP,
				methodColor, method, reset,
				path,
				comment,
			)
		}
	}
}

func colorForStatus(code int) string {
	switch {
	case code >= 200 && code < 300:
		return green
	case code >= 300 && code < 400:
		return white
	case code >= 400 && code < 500:
		return yellow
	default:
		return red
	}
}

func colorForMethod(method string) string {
	switch method {
	case "GET":
		return blue
	case "POST":
		return cyan
	case "PUT":
		return yellow
	case "DELETE":
		return red
	case "PATCH":
		return green
	case "HEAD":
		return magenta
	case "OPTIONS":
		return white
	default:
		return reset
	}
}
<|endoftext|>


<commit_before>package bot

import (
	"os"
	"reflect"
	"sync"
	"testing"

	cfg "github.com/arachnist/gorepost/config"
	"github.com/arachnist/gorepost/irc"
)

var eventTests = []struct {
	in          irc.Message
	expectedOut []irc.Message
}{
	{ // "ping"
		in: irc.Message{
			Command:  "PING",
			Trailing: "foobar",
		},
		expectedOut: []irc.Message{
			{
				Command:  "PONG",
				Trailing: "foobar",
			},
		},
	},
	{ // "invitki"
		in: irc.Message{
			Command:  "INVITE",
			Trailing: "#test-channel",
		},
		expectedOut: []irc.Message{
			{
				Command: "JOIN",
				Params:  []string{"#test-channel"},
			},
		},
	},
	{ // "channel join"
		in: irc.Message{
			Command: "001",
			Context: map[string]string{
				"Network": "TestNetwork",
			},
		},
		expectedOut: []irc.Message{
			{
				Command: "JOIN",
				Params:  []string{"#testchan-1"},
			},
			{
				Command: "JOIN",
				Params:  []string{"#testchan-2"},
			},
		},
	},
	{ // "msgping",
		in: irc.Message{
			Command:  "PRIVMSG",
			Trailing: ":ping",
			Prefix: &irc.Prefix{
				Name: "idontexist",
			},
		},
		expectedOut: []irc.Message{
			{
				Command:  "PRIVMSG",
				Params:   []string{"idontexist"},
				Trailing: "pingity pong",
			},
		},
	},
	{ // non-matching
		in: irc.Message{
			Command:  "PRIVMSG",
			Trailing: "foo bar baz",
		},
		expectedOut: []irc.Message{},
	},
}

func TestPlugins(t *testing.T) {
	output := make(chan irc.Message, 1)
	quitCollector := make(chan struct{}, 1)
	var r []irc.Message
	var wg sync.WaitGroup

	for _, e := range eventTests {
		r = r[:0]

		wg.Add(len(e.expectedOut))
		go func(quit chan struct{}, input chan irc.Message) {
			for {
				select {
				case msg := <-input:
					wg.Done()
					r = append(r, msg)
				case <-quit:
				}
			}
		}(quitCollector, output)

		Dispatcher(output, e.in)

		wg.Wait()
		quitCollector <- struct{}{}

		if !reflect.DeepEqual(r, e.expectedOut) {
			t.Logf("expected: %+v\n", e.expectedOut)
			t.Logf("result: %+v\n", r)
			t.Fail()
		}
	}
}

func TestMain(m *testing.M) {
	cfg.SetFileListBuilder(configLookupHelper)
	os.Exit(m.Run())
}

func configLookupHelper(map[string]string) []string {
	return []string{".testconfig.json"}
}

func eventCollector(quit chan struct{}, input chan irc.Message) (r []irc.Message) {
	for {
		select {
		case msg := <-input:
			r = append(r, msg)
		case <-quit:
			return r
		}
	}
}
<commit_msg>That function is not needed anymore.<commit_after>package bot

import (
	"os"
	"reflect"
	"sync"
	"testing"

	cfg "github.com/arachnist/gorepost/config"
	"github.com/arachnist/gorepost/irc"
)

var eventTests = []struct {
	in          irc.Message
	expectedOut []irc.Message
}{
	{ // "ping"
		in: irc.Message{
			Command:  "PING",
			Trailing: "foobar",
		},
		expectedOut: []irc.Message{
			{
				Command:  "PONG",
				Trailing: "foobar",
			},
		},
	},
	{ // "invitki"
		in: irc.Message{
			Command:  "INVITE",
			Trailing: "#test-channel",
		},
		expectedOut: []irc.Message{
			{
				Command: "JOIN",
				Params:  []string{"#test-channel"},
			},
		},
	},
	{ // "channel join"
		in: irc.Message{
			Command: "001",
			Context: map[string]string{
				"Network": "TestNetwork",
			},
		},
		expectedOut: []irc.Message{
			{
				Command: "JOIN",
				Params:  []string{"#testchan-1"},
			},
			{
				Command: "JOIN",
				Params:  []string{"#testchan-2"},
			},
		},
	},
	{ // "msgping",
		in: irc.Message{
			Command:  "PRIVMSG",
			Trailing: ":ping",
			Prefix: &irc.Prefix{
				Name: "idontexist",
			},
		},
		expectedOut: []irc.Message{
			{
				Command:  "PRIVMSG",
				Params:   []string{"idontexist"},
				Trailing: "pingity pong",
			},
		},
	},
	{ // non-matching
		in: irc.Message{
			Command:  "PRIVMSG",
			Trailing: "foo bar baz",
		},
		expectedOut: []irc.Message{},
	},
}

func TestPlugins(t *testing.T) {
	output := make(chan irc.Message, 1)
	quitCollector := make(chan struct{}, 1)
	var r []irc.Message
	var wg sync.WaitGroup

	for _, e := range eventTests {
		r = r[:0]

		wg.Add(len(e.expectedOut))
		go func(quit chan struct{}, input chan irc.Message) {
			for {
				select {
				case msg := <-input:
					wg.Done()
					r = append(r, msg)
				case <-quit:
				}
			}
		}(quitCollector, output)

		Dispatcher(output, e.in)

		wg.Wait()
		quitCollector <- struct{}{}

		if !reflect.DeepEqual(r, e.expectedOut) {
			t.Logf("expected: %+v\n", e.expectedOut)
			t.Logf("result: %+v\n", r)
			t.Fail()
		}
	}
}

func TestMain(m *testing.M) {
	cfg.SetFileListBuilder(configLookupHelper)
	os.Exit(m.Run())
}

func configLookupHelper(map[string]string) []string {
	return []string{".testconfig.json"}
}
<|endoftext|>


<commit_before>package broadcaster_test

import (
	"fmt"
	"sync"
	"testing"
	"time"

	"github.com/stvnrhodes/broadcaster"
)

func ExampleCaster() {
	// Create a new broadcaster
	b := broadcaster.New()

	// Create a channel subscribed to the broadcaster
	done := make(chan struct{})
	ch := b.Subscribe(done)

	// Wait groups are added to make the example more deterministic.
	// In most cases, having no wait groups is fine.
	var wg, wg2 sync.WaitGroup
	wg.Add(1)
	wg2.Add(1)

	// Read all messages from the channel.
	go func() {
		for msg := range ch {
			fmt.Println("Hello", msg)
		}
		wg.Done()
	}()

	b.Cast("World")

	// Create another channel. This one won't unsubscribe, so the argument is nil.
	ch2 := b.Subscribe(nil)

	// Read all messages from the channel.
	go func() {
		for msg := range ch2 {
			fmt.Println("Goodbye", msg)
		}
		wg2.Done()
	}()

	// Cast to the channel
	b.Cast(123)
	close(done)
	wg.Wait()
	b.Cast("examples")
	b.Close()
	wg2.Wait()
	// Output:
	// Hello World
	// Hello 123
	// Goodbye 123
	// Goodbye examples
}

func runAndCheck(t *testing.T, ch <-chan interface{}, length int, wg *sync.WaitGroup) {
	i := 0
	for _ = range ch {
		i++
	}
	if i != length {
		t.Errorf("Wrong number of messages to subscriber: got %v, want %v", i, length)
	}
	wg.Done()
}

func TestSubscribeOne(t *testing.T) {
	b := broadcaster.New()
	var wg sync.WaitGroup

	wg.Add(1)
	ch := b.Subscribe(nil)
	go runAndCheck(t, ch, 3, &wg)

	b.Cast(struct{}{})
	b.Cast("test string")
	b.Cast(123)
	b.Close()
	wg.Wait()
}

func TestSubscribeMany(t *testing.T) {
	b := broadcaster.New()
	var wg sync.WaitGroup

	for i := 0; i < 10; i++ {
		wg.Add(1)
		ch := b.Subscribe(nil)
		go runAndCheck(t, ch, 3, &wg)
	}

	b.Cast(struct{}{})
	b.Cast("test string")
	b.Cast(123)
	b.Close()
	wg.Wait()
}

func TestUnsubscribeOne(t *testing.T) {
	b := broadcaster.New()
	var wg sync.WaitGroup

	done := make(chan struct{})
	wg.Add(1)
	ch := b.Subscribe(done)
	go runAndCheck(t, ch, 3, &wg)

	b.Cast(struct{}{})
	b.Cast("test string")
	b.Cast(123)
	close(done)
	wg.Wait()
	b.Close()
}

func TestUnsubscribeMany(t *testing.T) {
	b := broadcaster.New()
	var wg1, wg2, wg3 sync.WaitGroup

	done1 := make(chan struct{})
	for i := 0; i < 5; i++ {
		wg1.Add(1)
		ch := b.Subscribe(done1)
		go runAndCheck(t, ch, 3, &wg1)
	}
	done2 := make(chan struct{})
	for i := 0; i < 5; i++ {
		wg2.Add(1)
		ch := b.Subscribe(done2)
		go runAndCheck(t, ch, 4, &wg2)
	}
	for i := 0; i < 5; i++ {
		wg3.Add(1)
		ch := b.Subscribe(nil)
		go runAndCheck(t, ch, 5, &wg3)
	}

	b.Cast(struct{}{})
	b.Cast("test string")
	b.Cast(123)
	close(done1)
	wg1.Wait()
	b.Cast(5.0)
	close(done2)
	wg2.Wait()
	b.Cast([]int{3, 5, 7})
	b.Close()
	wg3.Wait()
}

func expectPanic(t *testing.T, f func()) {
	defer func() {
		if r := recover(); r == nil {
			t.Errorf("expected panic for %v", f)
		}
	}()
	f()
}

func TestAfterClose(t *testing.T) {
	b := broadcaster.New()
	var wg sync.WaitGroup

	wg.Add(1)
	ch := b.Subscribe(nil)
	go runAndCheck(t, ch, 0, &wg)

	b.Close()
	wg.Wait()

	done := make(chan struct{})
	closedCh := b.Subscribe(done)
	if v := <-closedCh; v != nil {
		t.Errorf("Unexpected non-nil value: %v", v)
	}
	close(done)
	if v := <-closedCh; v != nil {
		t.Errorf("Unexpected non-nil value: %v", v)
	}

	expectPanic(t, b.Close)
	expectPanic(t, func() { b.Cast(1) })
}

func BenchmarkSubscriberNum(b *testing.B) {
	caster := broadcaster.New()
	var wg sync.WaitGroup

	for i := 0; i < b.N; i++ {
		wg.Add(1)
		ch := caster.Subscribe(nil)
		go func() {
			for _ = range ch {
				// Empty the channel
			}
			wg.Done()
		}()
	}

	caster.Cast(struct{}{})
	caster.Cast("test string")
	caster.Cast(123)
	caster.Close()
	wg.Wait()
}

func Benchmark1Subscriber(b *testing.B)     { benchmarkNSubscribers(b, 1) }
func Benchmark10Subscribers(b *testing.B)   { benchmarkNSubscribers(b, 10) }
func Benchmark100Subscribers(b *testing.B)  { benchmarkNSubscribers(b, 100) }
func Benchmark1000Subscribers(b *testing.B) { benchmarkNSubscribers(b, 1000) }

func benchmarkNSubscribers(b *testing.B, n int) {
	caster := broadcaster.New(&broadcaster.Options{WaitTime: time.Hour})
	var wg sync.WaitGroup

	for i := 0; i < n; i++ {
		wg.Add(1)
		ch := caster.Subscribe(nil)
		go func() {
			for _ = range ch {
				// Empty the channel
			}
			wg.Done()
		}()
	}

	for i := 0; i < b.N; i++ {
		caster.Cast("test string")
	}
	caster.Close()
	wg.Wait()
}
<commit_msg>Print function type instead of value so that tests pass<commit_after>package broadcaster_test

import (
	"fmt"
	"sync"
	"testing"
	"time"

	"github.com/stvnrhodes/broadcaster"
)

func ExampleCaster() {
	// Create a new broadcaster
	b := broadcaster.New()

	// Create a channel subscribed to the broadcaster
	done := make(chan struct{})
	ch := b.Subscribe(done)

	// Wait groups are added to make the example more deterministic.
	// In most cases, having no wait groups is fine.
	var wg, wg2 sync.WaitGroup
	wg.Add(1)
	wg2.Add(1)

	// Read all messages from the channel.
	go func() {
		for msg := range ch {
			fmt.Println("Hello", msg)
		}
		wg.Done()
	}()

	b.Cast("World")

	// Create another channel. This one won't unsubscribe, so the argument is nil.
	ch2 := b.Subscribe(nil)

	// Read all messages from the channel.
	go func() {
		for msg := range ch2 {
			fmt.Println("Goodbye", msg)
		}
		wg2.Done()
	}()

	// Cast to the channel
	b.Cast(123)
	close(done)
	wg.Wait()
	b.Cast("examples")
	b.Close()
	wg2.Wait()
	// Output:
	// Hello World
	// Hello 123
	// Goodbye 123
	// Goodbye examples
}

func runAndCheck(t *testing.T, ch <-chan interface{}, length int, wg *sync.WaitGroup) {
	i := 0
	for _ = range ch {
		i++
	}
	if i != length {
		t.Errorf("Wrong number of messages to subscriber: got %v, want %v", i, length)
	}
	wg.Done()
}

func TestSubscribeOne(t *testing.T) {
	b := broadcaster.New()
	var wg sync.WaitGroup

	wg.Add(1)
	ch := b.Subscribe(nil)
	go runAndCheck(t, ch, 3, &wg)

	b.Cast(struct{}{})
	b.Cast("test string")
	b.Cast(123)
	b.Close()
	wg.Wait()
}

func TestSubscribeMany(t *testing.T) {
	b := broadcaster.New()
	var wg sync.WaitGroup

	for i := 0; i < 10; i++ {
		wg.Add(1)
		ch := b.Subscribe(nil)
		go runAndCheck(t, ch, 3, &wg)
	}

	b.Cast(struct{}{})
	b.Cast("test string")
	b.Cast(123)
	b.Close()
	wg.Wait()
}

func TestUnsubscribeOne(t *testing.T) {
	b := broadcaster.New()
	var wg sync.WaitGroup

	done := make(chan struct{})
	wg.Add(1)
	ch := b.Subscribe(done)
	go runAndCheck(t, ch, 3, &wg)

	b.Cast(struct{}{})
	b.Cast("test string")
	b.Cast(123)
	close(done)
	wg.Wait()
	b.Close()
}

func TestUnsubscribeMany(t *testing.T) {
	b := broadcaster.New()
	var wg1, wg2, wg3 sync.WaitGroup

	done1 := make(chan struct{})
	for i := 0; i < 5; i++ {
		wg1.Add(1)
		ch := b.Subscribe(done1)
		go runAndCheck(t, ch, 3, &wg1)
	}
	done2 := make(chan struct{})
	for i := 0; i < 5; i++ {
		wg2.Add(1)
		ch := b.Subscribe(done2)
		go runAndCheck(t, ch, 4, &wg2)
	}
	for i := 0; i < 5; i++ {
		wg3.Add(1)
		ch := b.Subscribe(nil)
		go runAndCheck(t, ch, 5, &wg3)
	}

	b.Cast(struct{}{})
	b.Cast("test string")
	b.Cast(123)
	close(done1)
	wg1.Wait()
	b.Cast(5.0)
	close(done2)
	wg2.Wait()
	b.Cast([]int{3, 5, 7})
	b.Close()
	wg3.Wait()
}

func expectPanic(t *testing.T, f func()) {
	defer func() {
		if r := recover(); r == nil {
			t.Errorf("expected panic for %T", f)
		}
	}()
	f()
}

func TestAfterClose(t *testing.T) {
	b := broadcaster.New()
	var wg sync.WaitGroup

	wg.Add(1)
	ch := b.Subscribe(nil)
	go runAndCheck(t, ch, 0, &wg)

	b.Close()
	wg.Wait()

	done := make(chan struct{})
	closedCh := b.Subscribe(done)
	if v := <-closedCh; v != nil {
		t.Errorf("Unexpected non-nil value: %v", v)
	}
	close(done)
	if v := <-closedCh; v != nil {
		t.Errorf("Unexpected non-nil value: %v", v)
	}

	expectPanic(t, b.Close)
	expectPanic(t, func() { b.Cast(1) })
}

func BenchmarkSubscriberNum(b *testing.B) {
	caster := broadcaster.New()
	var wg sync.WaitGroup

	for i := 0; i < b.N; i++ {
		wg.Add(1)
		ch := caster.Subscribe(nil)
		go func() {
			for _ = range ch {
				// Empty the channel
			}
			wg.Done()
		}()
	}

	caster.Cast(struct{}{})
	caster.Cast("test string")
	caster.Cast(123)
	caster.Close()
	wg.Wait()
}

func Benchmark1Subscriber(b *testing.B)     { benchmarkNSubscribers(b, 1) }
func Benchmark10Subscribers(b *testing.B)   { benchmarkNSubscribers(b, 10) }
func Benchmark100Subscribers(b *testing.B)  { benchmarkNSubscribers(b, 100) }
func Benchmark1000Subscribers(b *testing.B) { benchmarkNSubscribers(b, 1000) }

func benchmarkNSubscribers(b *testing.B, n int) {
	caster := broadcaster.New(&broadcaster.Options{WaitTime: time.Hour})
	var wg sync.WaitGroup

	for i := 0; i < n; i++ {
		wg.Add(1)
		ch := caster.Subscribe(nil)
		go func() {
			for _ = range ch {
				// Empty the channel
			}
			wg.Done()
		}()
	}

	for i := 0; i < b.N; i++ {
		caster.Cast("test string")
	}
	caster.Close()
	wg.Wait()
}
<|endoftext|>


<commit_before>package cmd

import (
	"fmt"
	"os"
	"os/exec"
	"reflect"

	"github.com/gobuffalo/buffalo/meta"
	"github.com/gobuffalo/envy"
	"github.com/pkg/errors"
	"github.com/spf13/cobra"
)

// infoCmd represents the info command
var infoCmd = &cobra.Command{
	Use:   "info",
	Short: "Prints off diagnostic information useful for debugging.",
	RunE: func(cmd *cobra.Command, args []string) error {
		bb := os.Stdout

		bb.WriteString(fmt.Sprintf("### Buffalo Version\n%s\n", Version))

		bb.WriteString("\n### App Information\n")
		app := meta.New(".")
		rv := reflect.ValueOf(app)
		rt := rv.Type()
		for i := 0; i < rt.NumField(); i++ {
			f := rt.Field(i)
			bb.WriteString(fmt.Sprintf("%s=%v\n", f.Name, rv.FieldByName(f.Name).Interface()))
		}

		err := checkGoInfo()
		if err != nil {
			return errors.WithStack(err)
		}

		checkExternalsTools()

		return nil
	},
}

func checkGoInfo() error {
	bb := os.Stdout

	bb.WriteString("\n### Go Version\n")
	c := exec.Command(envy.Get("GO_BIN", "go"), "version")
	c.Stdout = bb
	err := c.Run()
	if err != nil {
		return errors.WithStack(err)
	}

	bb.WriteString("\n### Go Env\n")
	c = exec.Command(envy.Get("GO_BIN", "go"), "env")
	c.Stdout = bb
	c.Stderr = bb
	c.Run()

	return nil
}

func checkExternalsTools() {
	bb := os.Stdout

	infoCommands := []struct {
		Name      string
		PathName  string
		Cmd       *exec.Cmd
		InfoLabel string
	}{
		{"Node", "node", exec.Command("node", "--version"), "\n### Node Version\n"},
		{"NPM", "npm", exec.Command("npm", "--version"), "\n### NPM Version\n"},
		{"Yarn", "yarn", exec.Command("yarn", "--version"), "\n### Yarn Version\n"},
		{"dep", "dep", exec.Command("dep", "version"), "\n### Dep Version\n"},
		{"dep", "dep", exec.Command("dep", "status"), "\n### Dep Status\n"},
		{"PostgreSQL", "pg_ctl", exec.Command("pg_ctl", "--version"), "\n### PostgreSQL Version\n"},
		{"MySQL", "mysql", exec.Command("mysql", "--version"), "\n### MySQL Version\n"},
		{"SQLite", "sqlite3", exec.Command("sqlite3", "--version"), "\n### SQLite Version\n"},
	}

	for _, cmd := range infoCommands {
		bb.WriteString(cmd.InfoLabel)
		execIfExists(cmd.Name, cmd.InfoLabel, cmd.Cmd)
	}

}

func execIfExists(name string, pathName string, c *exec.Cmd) {
	bb := os.Stdout

	if _, err := exec.LookPath("mysql"); err != nil {
		bb.WriteString(fmt.Sprintf("%s Not Found\n", name))
		return
	}

	c.Stdout = bb
	c.Stderr = bb
	c.Run()
}

func init() {
	decorate("info", RootCmd)
	RootCmd.AddCommand(infoCmd)
}
<commit_msg>fixing issues outlined by @markbates<commit_after>package cmd

import (
	"fmt"
	"os"
	"os/exec"
	"reflect"

	"github.com/gobuffalo/buffalo/meta"
	"github.com/gobuffalo/envy"
	"github.com/pkg/errors"
	"github.com/spf13/cobra"
)

// infoCmd represents the info command
var infoCmd = &cobra.Command{
	Use:   "info",
	Short: "Prints off diagnostic information useful for debugging.",
	RunE: func(cmd *cobra.Command, args []string) error {
		bb := os.Stdout

		bb.WriteString(fmt.Sprintf("### Buffalo Version\n%s\n", Version))

		bb.WriteString("\n### App Information\n")
		app := meta.New(".")
		rv := reflect.ValueOf(app)
		rt := rv.Type()

		for i := 0; i < rt.NumField(); i++ {
			f := rt.Field(i)
			bb.WriteString(fmt.Sprintf("%s=%v\n", f.Name, rv.FieldByName(f.Name).Interface()))
		}

		return runInfoCmds()
	},
}

// func checkGoInfo() error {
// 	bb := os.Stdout

// 	bb.WriteString("\n### Go Version\n")
// 	c := exec.Command(envy.Get("GO_BIN", "go"), "version")
// 	c.Stdout = bb
// 	err := c.Run()
// 	if err != nil {
// 		return errors.WithStack(err)
// 	}

// 	bb.WriteString("\n### Go Env\n")
// 	c = exec.Command(envy.Get("GO_BIN", "go"), "env")
// 	c.Stdout = bb
// 	c.Stderr = bb
// 	c.Run()

// 	return nil
// }

type infoCommand struct {
	Name      string
	PathName  string
	Cmd       *exec.Cmd
	InfoLabel string
}

func runInfoCmds() error {

	commands := []infoCommand{
		{"Go", envy.Get("GO_BIN", "go"), exec.Command(envy.Get("GO_BIN", "go"), "version"), "\n### Go Version\n"},
		{"Go", envy.Get("GO_BIN", "go"), exec.Command(envy.Get("GO_BIN", "go"), "env"), "\n### Go Env\n"},
		{"Node", "node", exec.Command("node", "--version"), "\n### Node Version\n"},
		{"NPM", "npm", exec.Command("npm", "--version"), "\n### NPM Version\n"},
		{"Yarn", "yarn", exec.Command("yarn", "--version"), "\n### Yarn Version\n"},
		{"PostgreSQL", "pg_ctl", exec.Command("pg_ctl", "--version"), "\n### PostgreSQL Version\n"},
		{"MySQL", "mysql", exec.Command("mysql", "--version"), "\n### MySQL Version\n"},
		{"SQLite", "sqlite3", exec.Command("sqlite3", "--version"), "\n### SQLite Version\n"},
		{"dep", "dep", exec.Command("dep", "status"), "\n### Dep Status\n"},
		{"dep", "dep", exec.Command("dep", "version"), "\n### Dep Version\n"},
	}

	for _, cmd := range commands {
		err := execIfExists(cmd)
		if err != nil {
			return errors.WithStack(err)
		}
	}

	return nil
}

func execIfExists(infoCmd infoCommand) error {
	bb := os.Stdout
	bb.WriteString(infoCmd.InfoLabel)

	if _, err := exec.LookPath(infoCmd.PathName); err != nil {
		bb.WriteString(fmt.Sprintf("%s Not Found\n", infoCmd.Name))
		return nil
	}

	infoCmd.Cmd.Stdout = bb
	infoCmd.Cmd.Stderr = bb

	err := infoCmd.Cmd.Run()
	return err
}

func init() {
	decorate("info", RootCmd)
	RootCmd.AddCommand(infoCmd)
}
<|endoftext|>


<commit_before>package main

import (
	"fmt"
	"html/template"
	"log"
	"math"
	"net/http"
	"regexp"
	"strconv"
	"time"

	"./database"
	"./stat"
	"./web"
)

var (
	reDayPathMatch  = regexp.MustCompile("/day/([0-9]+)[/\\-]([0-9]+)[/\\-]([0-9]+)")
	reFilePathMatch = regexp.MustCompile("/file/([^/]+)")
)

const (
	dateFormat     = "2006-01-02"
	dateFormatLong = "2006-01-02T15:04:05.000Z"
)

func SetupWebFace(wf *web.WebFace, dbPtr *database.StatTrackerDB) {
	sh := SummaryHandle{db: dbPtr}
	sh.Setup()
	wf.Router.Handle("/", sh)
	wf.Router.Handle("/day/", DayHandle{db: dbPtr})
	wf.Router.Handle("/file/", FileHandle{db: dbPtr})
}

////////////////////////////////////////////////////////////////////////////////
// Summary Handle
type svgBox struct {
	X         int
	Y         int
	W         int
	H         int
	Classname string
}

type gPoint struct {
	Stat  *stat.DailyUserStat
	X, Y  int
	Boxes []svgBox
}

type SummaryHandle struct {
	db           *database.StatTrackerDB
	DayList      map[int]map[time.Month]map[int]*stat.DailyUserStat
	LatestGraph  []gPoint
	GridLines    []int
	GridDayLines []int
	GridWidth    int
	GridHeight   int
}

func (sh *SummaryHandle) Setup() {
	sh.DayList = make(map[int]map[time.Month]map[int]*stat.DailyUserStat)

	// Sumary Setup
	d := sh.db.LoadNextDailyUserStat("")
	prevDate, dErr := time.Parse(dateFormat, d.ModDate)
	if dErr != nil {
		log.Fatalln("Cannot parse:", d)
	}

	for d != nil {
		// Skip Ahread Days
		newDate, nErr := time.Parse(dateFormat, d.ModDate)
		if nErr != nil {
			log.Fatalln("Cannot parse:", nErr, d)
		}

		for prevDate.Before(newDate) {
			sh.SetDayListDay(prevDate, nil)
			prevDate = prevDate.AddDate(0, 0, 1)
		}

		// Setup Day
		sh.SetDayListDay(prevDate, d)
		prevDate = prevDate.AddDate(0, 0, 1)

		// Onto Next Date
		d = sh.db.LoadNextDailyUserStat(d.ModDate)
	}

	newDate := time.Now()
	for prevDate.Before(newDate) {
		sh.SetDayListDay(prevDate, nil)
		prevDate = prevDate.AddDate(0, 0, 1)
	}

	// Graph Setup
	sh.LatestGraph = []gPoint{}
	firstDay := newDate.AddDate(0, 0, -100)

	sh.GridDayLines = []int{}
	sh.GridLines = []int{}
	sh.GridWidth = 800
	sh.GridHeight = 300
	XStep := sh.GridWidth / 100

	yH := sh.GridHeight
	yStep := 100
	for y := 0; yH > 0; y += yStep {
		yH = sh.GridHeight - int(math.Pow(float64(y), 0.7))
		sh.GridLines = append(sh.GridLines, yH)

		if y == 1000 {
			yStep = 1000
		}
	}

	newDate = firstDay
	dateList := make([]time.Time, 100)
	dayList := make([]*stat.DailyUserStat, 100)
	for i := 0; i < 100; i += 1 {
		dateList[i] = newDate
		dayList[i] = sh.GetDayListDay(newDate)
		newDate = newDate.AddDate(0, 0, 1)
	}

	dailyWordHistChart("./static/days.png", 700, 400, dayList, dateList)

	for i, day := range dateList {
		d := sh.GetDayListDay(day)
		if d != nil {
			h := int(math.Pow(float64(d.WordAdd), 0.7))

			p := gPoint{
				X:     i * XStep,
				Y:     sh.GridHeight - h,
				Stat:  d,
				Boxes: []svgBox{},
			}

			p.Boxes = append(p.Boxes, svgBox{X: i * XStep, Y: sh.GridHeight - h, W: XStep, H: h, Classname: "addBar"})
			p.Boxes = append(p.Boxes, svgBox{X: i * XStep, Y: sh.GridHeight - h, W: XStep, H: int(math.Pow(float64(0-d.WordSub), 0.7)), Classname: "subBar"})

			if p.Y < 0 {
				p.Y = 30
			}

			sh.LatestGraph = append(sh.LatestGraph, p)
		}

		if dateList[i].Weekday() == time.Monday {
			sh.GridDayLines = append(sh.GridDayLines, i*XStep)
		}
	}

	fmt.Println("Setup Summary Handle")
}

func (sh *SummaryHandle) GetDayListDay(dateKey time.Time) *stat.DailyUserStat {
	var ok bool
	var year map[time.Month]map[int]*stat.DailyUserStat
	var month map[int]*stat.DailyUserStat

	yKey := -dateKey.Year()
	year, ok = sh.DayList[yKey]
	if !ok {
		return nil
	}

	mKey := dateKey.Month()
	month, ok = year[mKey]
	if !ok {
		return nil
	}

	return month[dateKey.Day()]
}

func (sh *SummaryHandle) SetDayListDay(dateKey time.Time, data *stat.DailyUserStat) {
	var ok bool
	var year map[time.Month]map[int]*stat.DailyUserStat
	var month map[int]*stat.DailyUserStat

	yKey := -dateKey.Year()
	year, ok = sh.DayList[yKey]
	if !ok {
		sh.DayList[yKey] = make(map[time.Month]map[int]*stat.DailyUserStat)
		year = sh.DayList[yKey]
	}

	mKey := dateKey.Month()
	month, ok = year[mKey]
	if !ok {
		year[mKey] = make(map[int]*stat.DailyUserStat)
		month = year[mKey]

		firstDay := time.Date(dateKey.Year(), dateKey.Month(), 1, 0, 0, 0, 0, time.UTC).Weekday()
		dayOff := 1
		for firstDay != time.Monday {
			dayOff -= 1
			month[dayOff] = nil
			if firstDay == time.Sunday {
				firstDay = time.Saturday
			} else {
				firstDay -= 1
			}
		}
	}

	month[dateKey.Day()] = data
}

func (sh SummaryHandle) ServeHTTP(rw http.ResponseWriter, req *http.Request) {
	sumTemp, err := template.ParseFiles("./templates/summary.html")
	if err != nil {
		log.Fatalln("Error parsing:", err)
	}
	e := sumTemp.Execute(rw, sh)
	if e != nil {
		log.Println("Error in Temp", e)
	}
}

////////////////////////////////////////////////////////////////////////////////
// Day Handle
type DayHandle struct {
	db *database.StatTrackerDB
}

type DayData struct {
	FullDate  string
	Stat      *stat.DailyUserStat
	WordTotal int
	DocList   []*stat.DocStat
	RevList   []*stat.RevStat
}

func (dh DayHandle) ServeHTTP(rw http.ResponseWriter, req *http.Request) {

	matches := reDayPathMatch.FindAllStringSubmatch(req.URL.String(), -1)
	if len(matches) < 1 {
		http.Error(rw, "Invalid Path", 400)
		return
	}

	year, errYear := strconv.Atoi(matches[0][1])
	month, errMonth := strconv.Atoi(matches[0][2])
	day, errDate := strconv.Atoi(matches[0][3])

	if errYear != nil || errMonth != nil || errDate != nil {
		http.Error(rw, fmt.Sprintf("Error parsing Path: \n %s \n %s \n %s", errYear, errMonth, errDate), 400)
		return
	}

	date := time.Date(year, time.Month(month), day, 12, 0, 0, 0, time.UTC)

	shortDate := date.Format("2006-01-02")

	dayStat := dh.db.LoadDailyUserStats(shortDate)
	if dayStat == nil {
		fmt.Fprintf(rw, "No stats for %s", shortDate)
		return
	}

	sumTemp, err := template.ParseFiles("./templates/DailyUserStat.html")
	if err != nil {
		http.Error(rw, fmt.Sprintf("Error parsing: %s", err), 500)
		return
	}

	dList := []*stat.DocStat{}
	rList := []*stat.RevStat{}
	for k, v := range dayStat.FileRevs {
		file := dh.db.LoadFileStats(k)
		if file == nil {
			http.Error(rw, fmt.Sprintf("Error finding file: %s", k), 500)
			return
		}

		for _, vRev := range file.RevList {
			for _, vRevID := range v {
				if vRev.RevId == vRevID {
					dList = append(dList, file)
					rList = append(rList, &vRev)
				}
			}
		}

	}

	e := sumTemp.Execute(rw, DayData{
		FullDate:  date.Format("Monday, 2 Jan 2006"),
		Stat:      dayStat,
		WordTotal: dayStat.WordAdd + dayStat.WordSub,
		DocList:   dList,
		RevList:   rList,
	})
	if e != nil {
		log.Println("Error in Temp", e)
	}

}

////////////////////////////////////////////////////////////////////////////////
// File Handle
type FileHandle struct {
	db *database.StatTrackerDB
}

func (dh FileHandle) ServeHTTP(rw http.ResponseWriter, req *http.Request) {

	fileTemp, err := template.ParseFiles("./templates/fileStat.html")
	if err != nil {
		http.Error(rw, fmt.Sprintf("Error parsing: %s", err), 500)
		return
	}

	matches := reFilePathMatch.FindAllStringSubmatch(req.URL.String(), -1)
	if len(matches) < 1 {
		fmt.Println(matches, req.URL)
		http.Error(rw, "Invalid Path", 400)
		return
	}

	fileStat := dh.db.LoadFileStats(matches[0][1])
	if fileStat == nil {
		fmt.Fprintf(rw, "No stats for %s", matches[0][1])
		return
	}

	date, errDate := time.Parse(dateFormatLong, fileStat.LastMod)
	if errDate != nil {
		http.Error(rw, fmt.Sprintf("Error parsing: %s", fileStat.LastMod), 500)
		return
	}

	e := fileTemp.Execute(rw, struct {
		FullDate string
		ModDate  string
		Stat     *stat.DocStat
	}{
		date.Format("Monday, 2 Jan 2006"),
		date.Format(dateFormat),
		fileStat,
	})

	if e != nil {
		log.Println("Error in Temp", e)
	}

}
<commit_msg>pagetweak<commit_after>package main

import (
	"fmt"
	"html/template"
	"log"
	"math"
	"net/http"
	"regexp"
	"strconv"
	"time"

	"./database"
	"./stat"
	"./web"
)

var (
	reDayPathMatch  = regexp.MustCompile("/day/([0-9]+)[/\\-]([0-9]+)[/\\-]([0-9]+)")
	reFilePathMatch = regexp.MustCompile("/file/([^/]+)")
)

const (
	dateFormat     = "2006-01-02"
	dateFormatLong = "2006-01-02T15:04:05.000Z"
)

func SetupWebFace(wf *web.WebFace, dbPtr *database.StatTrackerDB) {
	sh := SummaryHandle{db: dbPtr}
	sh.Setup()
	wf.Router.Handle("/", sh)
	wf.Router.Handle("/day/", DayHandle{db: dbPtr})
	wf.Router.Handle("/file/", FileHandle{db: dbPtr})
}

////////////////////////////////////////////////////////////////////////////////
// Summary Handle
type svgBox struct {
	X         int
	Y         int
	W         int
	H         int
	Classname string
}

type gPoint struct {
	Stat  *stat.DailyUserStat
	X, Y  int
	Boxes []svgBox
}

type SummaryHandle struct {
	db           *database.StatTrackerDB
	DayList      map[int]map[time.Month]map[int]*stat.DailyUserStat
	LatestGraph  []gPoint
	GridLines    []int
	GridDayLines []int
	GridWidth    int
	GridHeight   int
}

func (sh *SummaryHandle) Setup() {
	sh.DayList = make(map[int]map[time.Month]map[int]*stat.DailyUserStat)

	// Sumary Setup
	d := sh.db.LoadNextDailyUserStat("")
	prevDate, dErr := time.Parse(dateFormat, d.ModDate)
	if dErr != nil {
		log.Fatalln("Cannot parse:", d)
	}

	for d != nil {
		// Skip Ahread Days
		newDate, nErr := time.Parse(dateFormat, d.ModDate)
		if nErr != nil {
			log.Fatalln("Cannot parse:", nErr, d)
		}

		for prevDate.Before(newDate) {
			sh.SetDayListDay(prevDate, nil)
			prevDate = prevDate.AddDate(0, 0, 1)
		}

		// Setup Day
		sh.SetDayListDay(prevDate, d)
		prevDate = prevDate.AddDate(0, 0, 1)

		// Onto Next Date
		d = sh.db.LoadNextDailyUserStat(d.ModDate)
	}

	newDate := time.Now()
	for prevDate.Before(newDate) {
		sh.SetDayListDay(prevDate, nil)
		prevDate = prevDate.AddDate(0, 0, 1)
	}

	// Graph Setup
	sh.LatestGraph = []gPoint{}
	firstDay := newDate.AddDate(0, 0, -100)

	sh.GridDayLines = []int{}
	sh.GridLines = []int{}
	sh.GridWidth = 800
	sh.GridHeight = 300
	XStep := sh.GridWidth / 100

	yH := sh.GridHeight
	yStep := 100
	for y := 0; yH > 0; y += yStep {
		yH = sh.GridHeight - int(math.Pow(float64(y), 0.7))
		sh.GridLines = append(sh.GridLines, yH)

		if y == 1000 {
			yStep = 1000
		}
	}

	newDate = firstDay
	dateList := make([]time.Time, 100)
	dayList := make([]*stat.DailyUserStat, 100)
	for i := 0; i < 100; i += 1 {
		dateList[i] = newDate
		dayList[i] = sh.GetDayListDay(newDate)
		newDate = newDate.AddDate(0, 0, 1)
	}

	dailyWordHistChart("./static/days.png", 700, 400, dayList, dateList)

	for i, day := range dateList {
		d := sh.GetDayListDay(day)
		if d != nil {
			h := int(math.Pow(float64(d.WordAdd), 0.7))

			p := gPoint{
				X:     i * XStep,
				Y:     sh.GridHeight - h,
				Stat:  d,
				Boxes: []svgBox{},
			}

			p.Boxes = append(p.Boxes, svgBox{X: i * XStep, Y: sh.GridHeight - h, W: XStep, H: h, Classname: "addBar"})
			p.Boxes = append(p.Boxes, svgBox{X: i * XStep, Y: sh.GridHeight - h, W: XStep, H: int(math.Pow(float64(0-d.WordSub), 0.7)), Classname: "subBar"})

			if p.Y < 0 {
				p.Y = 30
			}

			sh.LatestGraph = append(sh.LatestGraph, p)
		}

		if dateList[i].Weekday() == time.Monday {
			sh.GridDayLines = append(sh.GridDayLines, i*XStep)
		}
	}

	fmt.Println("Setup Summary Handle")
}

func (sh *SummaryHandle) GetDayListDay(dateKey time.Time) *stat.DailyUserStat {
	var ok bool
	var year map[time.Month]map[int]*stat.DailyUserStat
	var month map[int]*stat.DailyUserStat

	yKey := -dateKey.Year()
	year, ok = sh.DayList[yKey]
	if !ok {
		return nil
	}

	mKey := dateKey.Month()
	month, ok = year[mKey]
	if !ok {
		return nil
	}

	return month[dateKey.Day()]
}

func (sh *SummaryHandle) SetDayListDay(dateKey time.Time, data *stat.DailyUserStat) {
	var ok bool
	var year map[time.Month]map[int]*stat.DailyUserStat
	var month map[int]*stat.DailyUserStat

	yKey := -dateKey.Year()
	year, ok = sh.DayList[yKey]
	if !ok {
		sh.DayList[yKey] = make(map[time.Month]map[int]*stat.DailyUserStat)
		year = sh.DayList[yKey]
	}

	mKey := dateKey.Month()
	month, ok = year[mKey]
	if !ok {
		year[mKey] = make(map[int]*stat.DailyUserStat)
		month = year[mKey]

		firstDay := time.Date(dateKey.Year(), dateKey.Month(), 1, 0, 0, 0, 0, time.UTC).Weekday()
		dayOff := 1
		for firstDay != time.Monday {
			dayOff -= 1
			month[dayOff] = nil
			if firstDay == time.Sunday {
				firstDay = time.Saturday
			} else {
				firstDay -= 1
			}
		}
	}

	month[dateKey.Day()] = data
}

func (sh SummaryHandle) ServeHTTP(rw http.ResponseWriter, req *http.Request) {
	sumTemp, err := template.ParseFiles("./templates/summary.html")
	if err != nil {
		log.Fatalln("Error parsing:", err)
	}
	e := sumTemp.Execute(rw, sh)
	if e != nil {
		log.Println("Error in Temp", e)
	}
}

////////////////////////////////////////////////////////////////////////////////
// Day Handle
type DayHandle struct {
	db *database.StatTrackerDB
}

type DayData struct {
	FullDate  string
	Stat      *stat.DailyUserStat
	WordTotal int
	DocList   []*stat.DocStat
	RevList   []*stat.RevStat
}

func (dh DayHandle) ServeHTTP(rw http.ResponseWriter, req *http.Request) {

	matches := reDayPathMatch.FindAllStringSubmatch(req.URL.String(), -1)
	if len(matches) < 1 {
		http.Error(rw, "Invalid Path", 400)
		return
	}

	year, errYear := strconv.Atoi(matches[0][1])
	month, errMonth := strconv.Atoi(matches[0][2])
	day, errDate := strconv.Atoi(matches[0][3])

	if errYear != nil || errMonth != nil || errDate != nil {
		http.Error(rw, fmt.Sprintf("Error parsing Path: \n %s \n %s \n %s", errYear, errMonth, errDate), 400)
		return
	}

	date := time.Date(year, time.Month(month), day, 12, 0, 0, 0, time.UTC)

	shortDate := date.Format("2006-01-02")

	dayStat := dh.db.LoadDailyUserStats(shortDate)
	if dayStat == nil {
		fmt.Fprintf(rw, "No stats for %s", shortDate)
		return
	}

	sumTemp, err := template.ParseFiles("./templates/dailyStat.html")
	if err != nil {
		http.Error(rw, fmt.Sprintf("Error parsing: %s", err), 500)
		return
	}

	dList := []*stat.DocStat{}
	rList := []*stat.RevStat{}
	for k, v := range dayStat.FileRevs {
		file := dh.db.LoadFileStats(k)
		if file == nil {
			http.Error(rw, fmt.Sprintf("Error finding file: %s", k), 500)
			return
		}

		for _, vRev := range file.RevList {
			for _, vRevID := range v {
				if vRev.RevId == vRevID {
					dList = append(dList, file)
					rList = append(rList, &vRev)
				}
			}
		}

	}

	e := sumTemp.Execute(rw, DayData{
		FullDate:  date.Format("Monday, 2 Jan 2006"),
		Stat:      dayStat,
		WordTotal: dayStat.WordAdd + dayStat.WordSub,
		DocList:   dList,
		RevList:   rList,
	})
	if e != nil {
		log.Println("Error in Temp", e)
	}

}

////////////////////////////////////////////////////////////////////////////////
// File Handle
type FileHandle struct {
	db *database.StatTrackerDB
}

func (dh FileHandle) ServeHTTP(rw http.ResponseWriter, req *http.Request) {

	fileTemp, err := template.ParseFiles("./templates/fileStat.html")
	if err != nil {
		http.Error(rw, fmt.Sprintf("Error parsing: %s", err), 500)
		return
	}

	matches := reFilePathMatch.FindAllStringSubmatch(req.URL.String(), -1)
	if len(matches) < 1 {
		fmt.Println(matches, req.URL)
		http.Error(rw, "Invalid Path", 400)
		return
	}

	fileStat := dh.db.LoadFileStats(matches[0][1])
	if fileStat == nil {
		fmt.Fprintf(rw, "No stats for %s", matches[0][1])
		return
	}

	date, errDate := time.Parse(dateFormatLong, fileStat.LastMod)
	if errDate != nil {
		http.Error(rw, fmt.Sprintf("Error parsing: %s", fileStat.LastMod), 500)
		return
	}

	e := fileTemp.Execute(rw, struct {
		FullDate string
		ModDate  string
		Stat     *stat.DocStat
	}{
		date.Format("Monday, 2 Jan 2006"),
		date.Format(dateFormat),
		fileStat,
	})

	if e != nil {
		log.Println("Error in Temp", e)
	}

}
<|endoftext|>


<commit_before>// This Source Code Form is subject to the terms of the Mozilla Public
// License, v. 2.0. If a copy of the MPL was not distributed with this
// file, You can obtain one at http://mozilla.org/MPL/2.0/.
//
// Contributor: Zack Mullaly zmullaly@mozilla.com [:zack]

package api

import (
	"encoding/json"
	"fmt"
	"net/http"
	"strings"
	"testing"
	"time"
)

const migAPIPort uint16 = 12345

type heartbeatResponse struct {
	Error *string `json:"error"`
}

func postHeartbeat(port uint16, body string) (int, heartbeatResponse, error) {
	response, err := http.Post(
		fmt.Sprintf("http://127.0.0.1:%d/api/v1/heartbeat", port),
		"application/json",
		strings.NewReader(body))
	if err != nil {
		return 0, heartbeatResponse{nil}, err
	}

	respData := heartbeatResponse{}
	decoder := json.NewDecoder(response.Body)
	decodeErr := decoder.Decode(&respData)

	response.Body.Close()

	if decodeErr != nil {
		return 0, heartbeatResponse{nil}, decodeErr
	}

	return response.StatusCode, respData, nil
}

func TestPostHeartbeatWithValidRequest(t *testing.T) {
	statusCode, response, err := postHeartbeat(migAPIPort, fmt.Sprintf(`{
    "name": "testagent123",
    "mode": "checkin",
    "version": "20181012-0.abc123.dev",
    "pid": 1234,
    "queueLoc": "testagent",
    "startTime": "%s",
    "environment": {
      "init": "systemd",
      "ident": "Ubuntu 16.04",
      "os": "linux",
      "arch": "x86_64",
      "proxied": false,
      "proxy": "",
      "addresses": ["127.0.0.1"],
      "publicIP": "127.0.0.1",
      "modules": ["memory", "file"]
    },
    "tags": [
      {
        "name": "operator",
        "value": "tester"
      }
    ]
  }`, time.Now().Format(time.RFC3339)))

	errMsg := ""
	if response.Error != nil {
		errMsg = *response.Error
	}

	if err != nil {
		t.Fatalf("Failed to make request or decode response: %s", err.Error())
	}
	if statusCode != http.StatusOK {
		t.Errorf("Expected status code %d, got %d", http.StatusOK, statusCode)
	}
	if errMsg != "" {
		t.Errorf("Did not expect an error from API, got '%s'", errMsg)
	}
}
<commit_msg>Add postgres check for heartbeat to api integration test<commit_after>// This Source Code Form is subject to the terms of the Mozilla Public
// License, v. 2.0. If a copy of the MPL was not distributed with this
// file, You can obtain one at http://mozilla.org/MPL/2.0/.
//
// Contributor: Zack Mullaly zmullaly@mozilla.com [:zack]

package api

import (
	"database/sql"
	"encoding/json"
	"fmt"
	"gopkg.in/gcfg.v1"
	"net/http"
	"path/filepath"
	"strings"
	"testing"
	"time"

	_ "github.com/lib/pq"
)

const migAPIPort uint16 = 12345

type heartbeatResponse struct {
	Error *string `json:"error"`
}

func postHeartbeat(port uint16, body string) (int, heartbeatResponse, error) {
	response, err := http.Post(
		fmt.Sprintf("http://127.0.0.1:%d/api/v1/heartbeat", port),
		"application/json",
		strings.NewReader(body))
	if err != nil {
		return 0, heartbeatResponse{nil}, err
	}

	respData := heartbeatResponse{}
	decoder := json.NewDecoder(response.Body)
	decodeErr := decoder.Decode(&respData)

	response.Body.Close()

	if decodeErr != nil {
		return 0, heartbeatResponse{nil}, decodeErr
	}

	return response.StatusCode, respData, nil
}

type Config struct {
	Postgres struct {
		Host, User, Password, DBName, SSLMode string
		Port, MaxConn                         int
	}
}

func TestPostHeartbeatWithValidRequest(t *testing.T) {
	agentName := "testagent123"
	statusCode, response, err := postHeartbeat(migAPIPort, fmt.Sprintf(`{
    "name": "%s",
    "mode": "checkin",
    "version": "20181012-0.abc123.dev",
    "pid": 1234,
    "queueLoc": "testagent",
    "startTime": "%s",
    "environment": {
      "init": "systemd",
      "ident": "Ubuntu 16.04",
      "os": "linux",
      "arch": "x86_64",
      "proxied": false,
      "proxy": "",
      "addresses": ["127.0.0.1"],
      "publicIP": "127.0.0.1",
      "modules": ["memory", "file"]
    },
    "tags": [
      {
        "name": "operator",
        "value": "tester"
      }
    ]
  }`, agentName, time.Now().Format(time.RFC3339)))

	errMsg := ""
	if response.Error != nil {
		errMsg = *response.Error
	}

	if err != nil {
		t.Fatalf("Failed to make request or decode response: %s", err.Error())
	}
	if statusCode != http.StatusOK {
		t.Errorf("Expected status code %d, got %d", http.StatusOK, statusCode)
	}
	if errMsg != "" {
		t.Errorf("Did not expect an error from API, got '%s'", errMsg)
	}

	path, err := filepath.Abs("../../api.cfg")
	if err != nil {
		panic(err)
	}

	cfg := Config{}
	err = gcfg.FatalOnly(gcfg.ReadFileInto(&cfg, path))
	if err != nil {
		panic(err)
	}

	user := cfg.Postgres.User
	password := cfg.Postgres.Password
	host := cfg.Postgres.Host
	port := cfg.Postgres.Port
	dbname := cfg.Postgres.DBName
	sslmode := cfg.Postgres.SSLMode

	url := fmt.Sprintf("postgres://%s:%s@%s:%d/%s?sslmode=%s", user, password, host, port, dbname, sslmode)
	db_conn, err := sql.Open("postgres", url)
	if err != nil {
		t.Fatalf("Failed to connect to postgresl DB: %s", err.Error())
	}

	defer db_conn.Close()

	rows, err := db_conn.Query("SELECT name FROM agents WHERE mode='checkin'")
	if err != nil {
		t.Fatalf("Received error when querying for heartbeats: %v", err)
	}

	if rows != nil {
		defer rows.Close()
	}

	numRows := 0
	for rows.Next() {
		numRows++
		var name string
		err = rows.Scan(&name)
		if err != nil {
			err = fmt.Errorf("Error while parsing name: '%v'", err)
			return
		}
		if name != agentName {
			t.Fatalf("Expected heartbeat with agent name: %s got %s", agentName, name)
		}
	}
}
<|endoftext|>


<commit_before>package image

import (
	"archive/tar"
	"context"
	"crypto/sha256"
	"errors"
	"fmt"
	"io"

	"github.com/concourse/concourse/atc/runtime"

	"github.com/concourse/concourse/atc/resource"

	"github.com/hashicorp/go-multierror"

	"code.cloudfoundry.org/lager"
	"github.com/DataDog/zstd"
	"github.com/concourse/concourse/atc"
	"github.com/concourse/concourse/atc/db"

	"github.com/concourse/concourse/atc/worker"
)

const ImageMetadataFile = "metadata.json"

// ErrImageUnavailable is returned when a task's configured image resource
// has no versions.
var ErrImageUnavailable = errors.New("no versions of image available")

var ErrImageGetDidNotProduceVolume = errors.New("fetching the image did not produce a volume")

//go:generate counterfeiter . ImageResourceFetcherFactory

type ImageResourceFetcherFactory interface {
	NewImageResourceFetcher(
		worker.Worker,
		worker.ImageResource,
		atc.Version,
		int,
		atc.VersionedResourceTypes,
		worker.ImageFetchingDelegate,
	) ImageResourceFetcher
}

//go:generate counterfeiter . ImageResourceFetcher

type ImageResourceFetcher interface {
	Fetch(
		ctx context.Context,
		logger lager.Logger,
		container db.CreatingContainer,
		privileged bool,
	) (worker.Volume, io.ReadCloser, atc.Version, error)
}

type imageResourceFetcherFactory struct {
	dbResourceCacheFactory  db.ResourceCacheFactory
	dbResourceConfigFactory db.ResourceConfigFactory
	resourceFetcher         worker.Fetcher
}

func NewImageResourceFetcherFactory(
	dbResourceCacheFactory db.ResourceCacheFactory,
	dbResourceConfigFactory db.ResourceConfigFactory,
	resourceFetcher worker.Fetcher,
) ImageResourceFetcherFactory {
	return &imageResourceFetcherFactory{
		dbResourceCacheFactory:  dbResourceCacheFactory,
		dbResourceConfigFactory: dbResourceConfigFactory,
		resourceFetcher:         resourceFetcher,
	}
}

func (f *imageResourceFetcherFactory) NewImageResourceFetcher(
	worker worker.Worker,
	imageResource worker.ImageResource,
	version atc.Version,
	teamID int,
	customTypes atc.VersionedResourceTypes,
	imageFetchingDelegate worker.ImageFetchingDelegate,
) ImageResourceFetcher {
	return &imageResourceFetcher{
		worker:                  worker,
		resourceFetcher:         f.resourceFetcher,
		dbResourceCacheFactory:  f.dbResourceCacheFactory,
		dbResourceConfigFactory: f.dbResourceConfigFactory,

		imageResource:         imageResource,
		version:               version,
		teamID:                teamID,
		customTypes:           customTypes,
		imageFetchingDelegate: imageFetchingDelegate,
	}
}

type imageResourceFetcher struct {
	worker                  worker.Worker
	resourceFetcher         worker.Fetcher
	dbResourceCacheFactory  db.ResourceCacheFactory
	dbResourceConfigFactory db.ResourceConfigFactory

	imageResource         worker.ImageResource
	version               atc.Version
	teamID                int
	customTypes           atc.VersionedResourceTypes
	imageFetchingDelegate worker.ImageFetchingDelegate
}

func (i *imageResourceFetcher) Fetch(
	ctx context.Context,
	logger lager.Logger,
	container db.CreatingContainer,
	privileged bool,
) (worker.Volume, io.ReadCloser, atc.Version, error) {
	version := i.version
	if version == nil {
		var err error
		version, err = i.getLatestVersion(ctx, logger, container)
		if err != nil {
			logger.Error("failed-to-get-latest-image-version", err)
			return nil, nil, nil, err
		}
	}

	params := i.imageResource.Params

	resourceCache, err := i.dbResourceCacheFactory.FindOrCreateResourceCache(
		db.ForContainer(container.ID()),
		i.imageResource.Type,
		version,
		i.imageResource.Source,
		params,
		i.customTypes,
	)
	if err != nil {
		logger.Error("failed-to-create-resource-cache", err)
		return nil, nil, nil, err
	}

	err = i.imageFetchingDelegate.ImageVersionDetermined(resourceCache)
	if err != nil {
		return nil, nil, nil, err
	}

	containerMetadata := db.ContainerMetadata{
		Type: db.ContainerTypeGet,
	}

	containerSpec := worker.ContainerSpec{
		ImageSpec: worker.ImageSpec{
			ResourceType: i.imageResource.Type,
		},
		TeamID: i.teamID,
	}

	processSpec := runtime.ProcessSpec{
		Path:         "/opt/resource/in",
		Args:         []string{resource.ResourcesDir("get")},
		StdoutWriter: i.imageFetchingDelegate.Stdout(),
		StderrWriter: i.imageFetchingDelegate.Stderr(),
	}
	res := resource.NewResource(
		i.imageResource.Source,
		params,
		version,
	)

	sign, err := res.Signature()
	if err != nil {
		return nil, nil, nil, err
	}

	lockName := lockName(sign, i.worker.Name())
	imageFetcherSpec := worker.ImageFetcherSpec{
		ResourceTypes: i.customTypes,
		Delegate:      i.imageFetchingDelegate,
	}

	_, volume, err := i.resourceFetcher.Fetch(
		ctx,
		logger.Session("init-image"),
		containerMetadata,
		i.worker,
		containerSpec,
		processSpec,
		res,
		db.NewImageGetContainerOwner(container, i.teamID),
		imageFetcherSpec,
		resourceCache,
		lockName,
	)

	if err != nil {
		logger.Error("failed-to-fetch-image", err)
		return nil, nil, nil, err
	}

	if volume == nil {
		return nil, nil, nil, ErrImageGetDidNotProduceVolume
	}

	reader, err := volume.StreamOut(ctx, ImageMetadataFile)
	if err != nil {
		return nil, nil, nil, err
	}

	zstdReader := zstd.NewReader(reader)
	tarReader := tar.NewReader(zstdReader)

	_, err = tarReader.Next()
	if err != nil {
		return nil, nil, nil, fmt.Errorf("could not read file \"%s\" from tar", ImageMetadataFile)
	}

	releasingReader := &fileReadMultiCloser{
		reader: tarReader,
		closers: []io.Closer{
			reader,
			zstdReader,
		},
	}

	return volume, releasingReader, version, nil
}

func (i *imageResourceFetcher) ensureVersionOfType(
	ctx context.Context,
	logger lager.Logger,
	container db.CreatingContainer,
	resourceType atc.VersionedResourceType,
) error {
	containerSpec := worker.ContainerSpec{
		ImageSpec: worker.ImageSpec{
			ResourceType: resourceType.Name,
		},
		TeamID: i.teamID,
		BindMounts: []worker.BindMountSource{
			&worker.CertsVolumeMount{Logger: logger},
		},
	}

	owner := db.NewImageCheckContainerOwner(container, i.teamID)

	resourceTypeContainer, err := i.worker.FindOrCreateContainer(
		ctx,
		logger,
		worker.NoopImageFetchingDelegate{},
		owner,
		db.ContainerMetadata{
			Type: db.ContainerTypeCheck,
		},
		containerSpec,
		i.customTypes,
	)
	if err != nil {
		return err
	}

	processSpec := runtime.ProcessSpec{
		Path: "/opt/resource/check",
	}
	checkResourceType := resource.NewResource(resourceType.Source, resourceType.Params, resourceType.Version)
	versions, err := checkResourceType.Check(context.TODO(), processSpec, resourceTypeContainer)
	if err != nil {
		return err
	}

	if len(versions) == 0 {
		return ErrImageUnavailable
	}

	resourceType.Version = versions[0]

	i.customTypes = append(i.customTypes.Without(resourceType.Name), resourceType)

	return nil
}

func (i *imageResourceFetcher) getLatestVersion(
	ctx context.Context,
	logger lager.Logger,
	container db.CreatingContainer,
) (atc.Version, error) {

	resourceType, found := i.customTypes.Lookup(i.imageResource.Type)
	if found && resourceType.Version == nil {
		err := i.ensureVersionOfType(ctx, logger, container, resourceType)
		if err != nil {
			return nil, err
		}
	}

	resourceSpec := worker.ContainerSpec{
		ImageSpec: worker.ImageSpec{
			ResourceType: i.imageResource.Type,
		},
		TeamID: i.teamID,
		BindMounts: []worker.BindMountSource{
			&worker.CertsVolumeMount{Logger: logger},
		},
	}

	owner := db.NewImageCheckContainerOwner(container, i.teamID)

	imageContainer, err := i.worker.FindOrCreateContainer(
		ctx,
		logger,
		i.imageFetchingDelegate,
		owner,
		db.ContainerMetadata{
			Type: db.ContainerTypeCheck,
		},
		resourceSpec,
		i.customTypes,
	)
	if err != nil {
		return nil, err
	}

	processSpec := runtime.ProcessSpec{
		Path: "/opt/resource/check",
	}
	checkingResource := resource.NewResource(i.imageResource.Source, i.imageResource.Params, i.imageResource.Version)
	versions, err := checkingResource.Check(context.TODO(), processSpec, imageContainer)
	if err != nil {
		return nil, err
	}

	if len(versions) == 0 {
		return nil, ErrImageUnavailable
	}

	return versions[0], nil
}

type fileReadMultiCloser struct {
	reader  io.Reader
	closers []io.Closer
}

func (frc fileReadMultiCloser) Read(p []byte) (n int, err error) {
	return frc.reader.Read(p)
}

func (frc fileReadMultiCloser) Close() error {
	var closeErrors error

	for _, closer := range frc.closers {
		err := closer.Close()
		if err != nil {
			closeErrors = multierror.Append(closeErrors, err)
		}
	}

	return closeErrors
}

func lockName(resourceJson []byte, workerName string) string {
	jsonRes := append(resourceJson, []byte(workerName)...)
	return fmt.Sprintf("%x", sha256.Sum256(jsonRes))
}
<commit_msg>fix fetching of task image<commit_after>package image

import (
	"archive/tar"
	"context"
	"crypto/sha256"
	"errors"
	"fmt"
	"io"

	"github.com/concourse/concourse/atc/runtime"

	"github.com/concourse/concourse/atc/resource"

	"github.com/hashicorp/go-multierror"

	"code.cloudfoundry.org/lager"
	"github.com/DataDog/zstd"
	"github.com/concourse/concourse/atc"
	"github.com/concourse/concourse/atc/db"

	"github.com/concourse/concourse/atc/worker"
)

const ImageMetadataFile = "metadata.json"

// ErrImageUnavailable is returned when a task's configured image resource
// has no versions.
var ErrImageUnavailable = errors.New("no versions of image available")

var ErrImageGetDidNotProduceVolume = errors.New("fetching the image did not produce a volume")

//go:generate counterfeiter . ImageResourceFetcherFactory

type ImageResourceFetcherFactory interface {
	NewImageResourceFetcher(
		worker.Worker,
		worker.ImageResource,
		atc.Version,
		int,
		atc.VersionedResourceTypes,
		worker.ImageFetchingDelegate,
	) ImageResourceFetcher
}

//go:generate counterfeiter . ImageResourceFetcher

type ImageResourceFetcher interface {
	Fetch(
		ctx context.Context,
		logger lager.Logger,
		container db.CreatingContainer,
		privileged bool,
	) (worker.Volume, io.ReadCloser, atc.Version, error)
}

type imageResourceFetcherFactory struct {
	dbResourceCacheFactory  db.ResourceCacheFactory
	dbResourceConfigFactory db.ResourceConfigFactory
	resourceFetcher         worker.Fetcher
}

func NewImageResourceFetcherFactory(
	dbResourceCacheFactory db.ResourceCacheFactory,
	dbResourceConfigFactory db.ResourceConfigFactory,
	resourceFetcher worker.Fetcher,
) ImageResourceFetcherFactory {
	return &imageResourceFetcherFactory{
		dbResourceCacheFactory:  dbResourceCacheFactory,
		dbResourceConfigFactory: dbResourceConfigFactory,
		resourceFetcher:         resourceFetcher,
	}
}

func (f *imageResourceFetcherFactory) NewImageResourceFetcher(
	worker worker.Worker,
	imageResource worker.ImageResource,
	version atc.Version,
	teamID int,
	customTypes atc.VersionedResourceTypes,
	imageFetchingDelegate worker.ImageFetchingDelegate,
) ImageResourceFetcher {
	return &imageResourceFetcher{
		worker:                  worker,
		resourceFetcher:         f.resourceFetcher,
		dbResourceCacheFactory:  f.dbResourceCacheFactory,
		dbResourceConfigFactory: f.dbResourceConfigFactory,

		imageResource:         imageResource,
		version:               version,
		teamID:                teamID,
		customTypes:           customTypes,
		imageFetchingDelegate: imageFetchingDelegate,
	}
}

type imageResourceFetcher struct {
	worker                  worker.Worker
	resourceFetcher         worker.Fetcher
	dbResourceCacheFactory  db.ResourceCacheFactory
	dbResourceConfigFactory db.ResourceConfigFactory

	imageResource         worker.ImageResource
	version               atc.Version
	teamID                int
	customTypes           atc.VersionedResourceTypes
	imageFetchingDelegate worker.ImageFetchingDelegate
}

func (i *imageResourceFetcher) Fetch(
	ctx context.Context,
	logger lager.Logger,
	container db.CreatingContainer,
	privileged bool,
) (worker.Volume, io.ReadCloser, atc.Version, error) {
	version := i.version
	if version == nil {
		var err error
		version, err = i.getLatestVersion(ctx, logger, container)
		if err != nil {
			logger.Error("failed-to-get-latest-image-version", err)
			return nil, nil, nil, err
		}
	}

	params := i.imageResource.Params

	resourceCache, err := i.dbResourceCacheFactory.FindOrCreateResourceCache(
		db.ForContainer(container.ID()),
		i.imageResource.Type,
		version,
		i.imageResource.Source,
		params,
		i.customTypes,
	)
	if err != nil {
		logger.Error("failed-to-create-resource-cache", err)
		return nil, nil, nil, err
	}

	err = i.imageFetchingDelegate.ImageVersionDetermined(resourceCache)
	if err != nil {
		return nil, nil, nil, err
	}

	containerMetadata := db.ContainerMetadata{
		Type: db.ContainerTypeGet,
	}

	containerSpec := worker.ContainerSpec{
		ImageSpec: worker.ImageSpec{
			ResourceType: i.imageResource.Type,
		},
		TeamID: i.teamID,
	}

	processSpec := runtime.ProcessSpec{
		Path:         "/opt/resource/in",
		Args:         []string{resource.ResourcesDir("get")},
		Dir:          resource.ResourcesDir("get"),
		StdoutWriter: i.imageFetchingDelegate.Stdout(),
		StderrWriter: i.imageFetchingDelegate.Stderr(),
	}
	res := resource.NewResource(
		i.imageResource.Source,
		params,
		version,
	)

	sign, err := res.Signature()
	if err != nil {
		return nil, nil, nil, err
	}

	lockName := lockName(sign, i.worker.Name())
	imageFetcherSpec := worker.ImageFetcherSpec{
		ResourceTypes: i.customTypes,
		Delegate:      i.imageFetchingDelegate,
	}

	_, volume, err := i.resourceFetcher.Fetch(
		ctx,
		logger.Session("init-image"),
		containerMetadata,
		i.worker,
		containerSpec,
		processSpec,
		res,
		db.NewImageGetContainerOwner(container, i.teamID),
		imageFetcherSpec,
		resourceCache,
		lockName,
	)

	if err != nil {
		logger.Error("failed-to-fetch-image", err)
		return nil, nil, nil, err
	}

	if volume == nil {
		return nil, nil, nil, ErrImageGetDidNotProduceVolume
	}

	reader, err := volume.StreamOut(ctx, ImageMetadataFile)
	if err != nil {
		return nil, nil, nil, err
	}

	zstdReader := zstd.NewReader(reader)
	tarReader := tar.NewReader(zstdReader)

	_, err = tarReader.Next()
	if err != nil {
		return nil, nil, nil, fmt.Errorf("could not read file \"%s\" from tar", ImageMetadataFile)
	}

	releasingReader := &fileReadMultiCloser{
		reader: tarReader,
		closers: []io.Closer{
			reader,
			zstdReader,
		},
	}

	return volume, releasingReader, version, nil
}

func (i *imageResourceFetcher) ensureVersionOfType(
	ctx context.Context,
	logger lager.Logger,
	container db.CreatingContainer,
	resourceType atc.VersionedResourceType,
) error {
	containerSpec := worker.ContainerSpec{
		ImageSpec: worker.ImageSpec{
			ResourceType: resourceType.Name,
		},
		TeamID: i.teamID,
		BindMounts: []worker.BindMountSource{
			&worker.CertsVolumeMount{Logger: logger},
		},
	}

	owner := db.NewImageCheckContainerOwner(container, i.teamID)

	resourceTypeContainer, err := i.worker.FindOrCreateContainer(
		ctx,
		logger,
		worker.NoopImageFetchingDelegate{},
		owner,
		db.ContainerMetadata{
			Type: db.ContainerTypeCheck,
		},
		containerSpec,
		i.customTypes,
	)
	if err != nil {
		return err
	}

	processSpec := runtime.ProcessSpec{
		Path: "/opt/resource/check",
	}
	checkResourceType := resource.NewResource(resourceType.Source, resourceType.Params, resourceType.Version)
	versions, err := checkResourceType.Check(context.TODO(), processSpec, resourceTypeContainer)
	if err != nil {
		return err
	}

	if len(versions) == 0 {
		return ErrImageUnavailable
	}

	resourceType.Version = versions[0]

	i.customTypes = append(i.customTypes.Without(resourceType.Name), resourceType)

	return nil
}

func (i *imageResourceFetcher) getLatestVersion(
	ctx context.Context,
	logger lager.Logger,
	container db.CreatingContainer,
) (atc.Version, error) {

	resourceType, found := i.customTypes.Lookup(i.imageResource.Type)
	if found && resourceType.Version == nil {
		err := i.ensureVersionOfType(ctx, logger, container, resourceType)
		if err != nil {
			return nil, err
		}
	}

	resourceSpec := worker.ContainerSpec{
		ImageSpec: worker.ImageSpec{
			ResourceType: i.imageResource.Type,
		},
		TeamID: i.teamID,
		BindMounts: []worker.BindMountSource{
			&worker.CertsVolumeMount{Logger: logger},
		},
	}

	owner := db.NewImageCheckContainerOwner(container, i.teamID)

	imageContainer, err := i.worker.FindOrCreateContainer(
		ctx,
		logger,
		i.imageFetchingDelegate,
		owner,
		db.ContainerMetadata{
			Type: db.ContainerTypeCheck,
		},
		resourceSpec,
		i.customTypes,
	)
	if err != nil {
		return nil, err
	}

	processSpec := runtime.ProcessSpec{
		Path: "/opt/resource/check",
	}
	checkingResource := resource.NewResource(i.imageResource.Source, i.imageResource.Params, i.imageResource.Version)
	versions, err := checkingResource.Check(context.TODO(), processSpec, imageContainer)
	if err != nil {
		return nil, err
	}

	if len(versions) == 0 {
		return nil, ErrImageUnavailable
	}

	return versions[0], nil
}

type fileReadMultiCloser struct {
	reader  io.Reader
	closers []io.Closer
}

func (frc fileReadMultiCloser) Read(p []byte) (n int, err error) {
	return frc.reader.Read(p)
}

func (frc fileReadMultiCloser) Close() error {
	var closeErrors error

	for _, closer := range frc.closers {
		err := closer.Close()
		if err != nil {
			closeErrors = multierror.Append(closeErrors, err)
		}
	}

	return closeErrors
}

func lockName(resourceJson []byte, workerName string) string {
	jsonRes := append(resourceJson, []byte(workerName)...)
	return fmt.Sprintf("%x", sha256.Sum256(jsonRes))
}
<|endoftext|>


<commit_before>package controllers

import (
	"fmt"
	"github.com/gin-gonic/gin"
	"net/http"

	"github.com/eirka/eirka-libs/audit"
	"github.com/eirka/eirka-libs/config"
	e "github.com/eirka/eirka-libs/errors"
	"github.com/eirka/eirka-libs/redis"
	"github.com/eirka/eirka-libs/user"

	"github.com/eirka/eirka-post/models"
	u "github.com/eirka/eirka-post/utils"
)

// Input from new reply form
type replyForm struct {
	Comment string `form:"comment"`
	Thread  uint   `form:"thread" binding:"required"`
}

// ReplyController handles the creation of new threads
func ReplyController(c *gin.Context) {
	var err error
	var rf replyForm
	req := c.Request

	// get userdata from session middleware
	userdata := c.MustGet("userdata").(user.User)

	// check size of content
	if req.ContentLength > int64(config.Settings.Limits.ImageMaxSize) {
		c.JSON(http.StatusExpectationFailed, gin.H{"error_message": e.ErrImageSize.Error()})
		c.Error(e.ErrImageSize).SetMeta("ReplyController.ContentLength")
		return
	}

	// set max bytes reader
	req.Body = http.MaxBytesReader(c.Writer, req.Body, int64(config.Settings.Limits.ImageMaxSize))

	err = c.Bind(&rf)
	if err != nil {
		c.JSON(e.ErrorMessage(e.ErrInvalidParam))
		c.Error(err).SetMeta("ReplyController.Bind")
		return
	}

	// Set parameters to ReplyModel
	m := models.ReplyModel{
		Uid:     userdata.Id,
		Ip:      c.ClientIP(),
		Comment: rf.Comment,
		Thread:  rf.Thread,
		Image:   true,
	}

	image := u.ImageType{}

	// Check if theres a file
	image.File, image.Header, err = req.FormFile("file")
	if err == http.ErrMissingFile {
		m.Image = false
	}

	// Validate input parameters
	err = m.ValidateInput()
	if err != nil {
		c.JSON(http.StatusBadRequest, gin.H{"error_message": err.Error()})
		c.Error(err).SetMeta("ReplyController.ValidateInput")
		return
	}

	// Check thread status
	err = m.Status()
	if err == e.ErrThreadClosed {
		c.JSON(http.StatusBadRequest, gin.H{"error_message": err.Error()})
		c.Error(err).SetMeta("ReplyController.Status")
		return
	} else if err != nil {
		c.JSON(e.ErrorMessage(e.ErrInternalError))
		c.Error(err).SetMeta("ReplyController.Status")
		return
	}

	if m.Comment != "" {

		// Check comment in SFS and Akismet
		check := u.CheckComment{
			Ip:      m.Ip,
			Ua:      req.UserAgent(),
			Referer: req.Referer(),
			Comment: m.Comment,
		}

		err = check.Get()
		if err != nil {
			c.JSON(http.StatusBadRequest, gin.H{"error_message": err.Error()})
			c.Error(err).SetMeta("ReplyController.CheckComment")
			return
		}

	}

	if m.Image {

		// set the ib for duplicate checking
		image.Ib = m.Ib

		// Save the image to a file
		err = image.SaveImage()
		if err != nil {
			c.JSON(http.StatusBadRequest, gin.H{"error_message": err.Error()})
			c.Error(err).SetMeta("ReplyController.SaveImage")
			return
		}

		m.MD5 = image.MD5
		m.OrigWidth = image.OrigWidth
		m.OrigHeight = image.OrigHeight
		m.ThumbWidth = image.ThumbWidth
		m.ThumbHeight = image.ThumbHeight
		m.Filename = image.Filename
		m.Thumbnail = image.Thumbnail

	}

	// Post data
	err = m.Post()
	if err != nil {
		c.JSON(e.ErrorMessage(e.ErrInternalError))
		c.Error(err).SetMeta("ReplyController.Post")
		return
	}

	// Initialize cache handle
	cache := redis.RedisCache

	// Delete redis stuff
	index_key := fmt.Sprintf("%s:%d", "index", m.Ib)
	directory_key := fmt.Sprintf("%s:%d", "directory", m.Ib)
	thread_key := fmt.Sprintf("%s:%d:%d", "thread", m.Ib, m.Thread)
	image_key := fmt.Sprintf("%s:%d", "image", m.Ib)

	// lock the shared mutex
	err = cache.Lock()
	if err != nil {
		c.JSON(e.ErrorMessage(e.ErrInternalError))
		c.Error(err).SetMeta("ReplyController.cache.Lock")
		return
	}

	err = cache.Delete(index_key, directory_key, thread_key, image_key)
	if err != nil {
		c.JSON(e.ErrorMessage(e.ErrInternalError))
		c.Error(err).SetMeta("ReplyController.cache.Delete")
		return
	}

	// get board domain and redirect to it
	redirect, err := u.Link(m.Ib, req.Referer())
	if err != nil {
		c.JSON(e.ErrorMessage(e.ErrInternalError))
		c.Error(err).SetMeta("ReplyController.redirect")
		return
	}

	c.Redirect(303, redirect)

	audit := audit.Audit{
		User:   userdata.Id,
		Ib:     m.Ib,
		Ip:     m.Ip,
		Action: audit.AuditReply,
		Info:   fmt.Sprintf("%d", m.Thread),
	}

	// submit audit
	err = audit.Submit()
	if err != nil {
		c.Error(err).SetMeta("ReplyController.audit.Submit")
	}

	return

}
<commit_msg>shared mutex<commit_after>package controllers

import (
	"fmt"
	"github.com/gin-gonic/gin"
	"net/http"

	"github.com/eirka/eirka-libs/audit"
	"github.com/eirka/eirka-libs/config"
	e "github.com/eirka/eirka-libs/errors"
	"github.com/eirka/eirka-libs/redis"
	"github.com/eirka/eirka-libs/user"

	"github.com/eirka/eirka-post/models"
	u "github.com/eirka/eirka-post/utils"
)

// Input from new reply form
type replyForm struct {
	Comment string `form:"comment"`
	Thread  uint   `form:"thread" binding:"required"`
}

// ReplyController handles the creation of new threads
func ReplyController(c *gin.Context) {
	var err error
	var rf replyForm
	req := c.Request

	// get userdata from session middleware
	userdata := c.MustGet("userdata").(user.User)

	// check size of content
	if req.ContentLength > int64(config.Settings.Limits.ImageMaxSize) {
		c.JSON(http.StatusExpectationFailed, gin.H{"error_message": e.ErrImageSize.Error()})
		c.Error(e.ErrImageSize).SetMeta("ReplyController.ContentLength")
		return
	}

	// set max bytes reader
	req.Body = http.MaxBytesReader(c.Writer, req.Body, int64(config.Settings.Limits.ImageMaxSize))

	err = c.Bind(&rf)
	if err != nil {
		c.JSON(e.ErrorMessage(e.ErrInvalidParam))
		c.Error(err).SetMeta("ReplyController.Bind")
		return
	}

	// Set parameters to ReplyModel
	m := models.ReplyModel{
		Uid:     userdata.Id,
		Ip:      c.ClientIP(),
		Comment: rf.Comment,
		Thread:  rf.Thread,
		Image:   true,
	}

	image := u.ImageType{}

	// Check if theres a file
	image.File, image.Header, err = req.FormFile("file")
	if err == http.ErrMissingFile {
		m.Image = false
	}

	// Validate input parameters
	err = m.ValidateInput()
	if err != nil {
		c.JSON(http.StatusBadRequest, gin.H{"error_message": err.Error()})
		c.Error(err).SetMeta("ReplyController.ValidateInput")
		return
	}

	// Check thread status
	err = m.Status()
	if err == e.ErrThreadClosed {
		c.JSON(http.StatusBadRequest, gin.H{"error_message": err.Error()})
		c.Error(err).SetMeta("ReplyController.Status")
		return
	} else if err != nil {
		c.JSON(e.ErrorMessage(e.ErrInternalError))
		c.Error(err).SetMeta("ReplyController.Status")
		return
	}

	if m.Comment != "" {

		// Check comment in SFS and Akismet
		check := u.CheckComment{
			Ip:      m.Ip,
			Ua:      req.UserAgent(),
			Referer: req.Referer(),
			Comment: m.Comment,
		}

		err = check.Get()
		if err != nil {
			c.JSON(http.StatusBadRequest, gin.H{"error_message": err.Error()})
			c.Error(err).SetMeta("ReplyController.CheckComment")
			return
		}

	}

	if m.Image {

		// set the ib for duplicate checking
		image.Ib = m.Ib

		// Save the image to a file
		err = image.SaveImage()
		if err != nil {
			c.JSON(http.StatusBadRequest, gin.H{"error_message": err.Error()})
			c.Error(err).SetMeta("ReplyController.SaveImage")
			return
		}

		m.MD5 = image.MD5
		m.OrigWidth = image.OrigWidth
		m.OrigHeight = image.OrigHeight
		m.ThumbWidth = image.ThumbWidth
		m.ThumbHeight = image.ThumbHeight
		m.Filename = image.Filename
		m.Thumbnail = image.Thumbnail

	}

	// Post data
	err = m.Post()
	if err != nil {
		c.JSON(e.ErrorMessage(e.ErrInternalError))
		c.Error(err).SetMeta("ReplyController.Post")
		return
	}

	// Initialize cache handle
	cache := redis.RedisCache

	// Delete redis stuff
	index_key := fmt.Sprintf("%s:%d", "index", m.Ib)
	directory_key := fmt.Sprintf("%s:%d", "directory", m.Ib)
	thread_key := fmt.Sprintf("%s:%d:%d", "thread", m.Ib, m.Thread)
	image_key := fmt.Sprintf("%s:%d", "image", m.Ib)

	// lock the shared mutex
	err = cache.Lock(m.Ib)
	if err != nil {
		c.JSON(e.ErrorMessage(e.ErrInternalError))
		c.Error(err).SetMeta("ReplyController.cache.Lock")
		return
	}

	err = cache.Delete(index_key, directory_key, thread_key, image_key)
	if err != nil {
		c.JSON(e.ErrorMessage(e.ErrInternalError))
		c.Error(err).SetMeta("ReplyController.cache.Delete")
		return
	}

	// get board domain and redirect to it
	redirect, err := u.Link(m.Ib, req.Referer())
	if err != nil {
		c.JSON(e.ErrorMessage(e.ErrInternalError))
		c.Error(err).SetMeta("ReplyController.redirect")
		return
	}

	c.Redirect(303, redirect)

	audit := audit.Audit{
		User:   userdata.Id,
		Ib:     m.Ib,
		Ip:     m.Ip,
		Action: audit.AuditReply,
		Info:   fmt.Sprintf("%d", m.Thread),
	}

	// submit audit
	err = audit.Submit()
	if err != nil {
		c.Error(err).SetMeta("ReplyController.audit.Submit")
	}

	return

}
<|endoftext|>