<commit_before>// Copyright (C) 2014 Daniel Harrison

package hfile

import (
	"bytes"
	"encoding/binary"
	"fmt"
	"log"
	"sort"
)

type Scanner struct {
	reader  *Reader
	idx     int
	buf     *bytes.Reader
	lastKey *[]byte
}

func NewScanner(r *Reader) Scanner {
	return Scanner{r, 0, nil, nil}
}

func (s *Scanner) Reset() {
	s.idx = 0
	s.buf = nil
	s.lastKey = nil
}

func (s *Scanner) findBlock(key []byte) int {
	remaining := len(s.reader.index) - s.idx - 1

	if remaining <= 0 {
		return s.idx // s.cur is the last block, so it is only choice.
	}

	if s.reader.index[s.idx+1].IsAfter(key) {
		return s.idx
	}

	offset := sort.Search(remaining, func(i int) bool {
		return s.reader.index[s.idx+i+1].IsAfter(key)
	})

	return s.idx + offset
}

func (s *Scanner) CheckIfKeyOutOfOrder(key []byte) error {
	if s.lastKey != nil && bytes.Compare(*s.lastKey, key) > 0 {
		return fmt.Errorf("Keys our of order! %v > %v", *s.lastKey, key)
	}
	s.lastKey = &key
	return nil
}

func (s *Scanner) blockFor(key []byte) (*bytes.Reader, error, bool) {
	err := s.CheckIfKeyOutOfOrder(key)
	if err != nil {
		return nil, err, false
	}

	if s.reader.index[s.idx].IsAfter(key) {
		return nil, nil, false
	}

	idx := s.findBlock(key)

	if idx != s.idx || s.buf == nil { // need to load a new block
		data, err := s.reader.GetBlock(idx)
		if err != nil {
			return nil, err, false
		}
		s.idx = idx
		s.buf = data
	}

	return s.buf, nil, true
}

func (s *Scanner) GetFirst(key []byte) ([]byte, error, bool) {
	data, err, ok := s.blockFor(key)

	if !ok {
		return nil, err, ok
	}

	value, _, found := getValuesFromBuffer(data, key, true)
	return value, nil, found
}

func (s *Scanner) GetAll(key []byte) ([][]byte, error) {
	data, err, ok := s.blockFor(key)

	if !ok {
		log.Println("no block for key ", key)
		return nil, err
	}

	_, found, _ := getValuesFromBuffer(data, key, false)
	return found, err
}

func getValuesFromBuffer(buf *bytes.Reader, key []byte, first bool) ([]byte, [][]byte, bool) {
	var acc [][]byte

	for buf.Len() > 0 {
		var keyLen, valLen uint32
		binary.Read(buf, binary.BigEndian, &keyLen)
		binary.Read(buf, binary.BigEndian, &valLen)
		keyBytes := make([]byte, keyLen)
		valBytes := make([]byte, valLen)
		buf.Read(keyBytes)
		buf.Read(valBytes)
		if bytes.Compare(key, keyBytes) == 0 {
			if first {
				return valBytes, nil, true
			} else {
				acc = append(acc, valBytes)
			}
		}
	}

	return nil, acc, len(acc) > 0
}
<commit_msg>add lots of debug printing<commit_after>// Copyright (C) 2014 Daniel Harrison

package hfile

import (
	"bytes"
	"encoding/binary"
	"encoding/hex"
	"fmt"
	"log"
	"sort"
)

type Scanner struct {
	reader  *Reader
	idx     int
	buf     *bytes.Reader
	lastKey *[]byte
}

func NewScanner(r *Reader) Scanner {
	return Scanner{r, 0, nil, nil}
}

func (s *Scanner) Reset() {
	s.idx = 0
	s.buf = nil
	s.lastKey = nil
}

func (s *Scanner) findBlock(key []byte) int {
	remaining := len(s.reader.index) - s.idx - 1
	log.Printf("[Scanner.findBlock] cur %d, remaining %d\n", s.idx, remaining)

	if remaining <= 0 {
		log.Println("[Scanner.findBlock] last block")
		return s.idx // s.cur is the last block, so it is only choice.
	}

	if s.reader.index[s.idx+1].IsAfter(key) {
		log.Println("[Scanner.findBlock] next block is past key")
		return s.idx
	}

	offset := sort.Search(remaining, func(i int) bool {
		return s.reader.index[s.idx+i+1].IsAfter(key)
	})

	return s.idx + offset
}

func (s *Scanner) CheckIfKeyOutOfOrder(key []byte) error {
	if s.lastKey != nil && bytes.Compare(*s.lastKey, key) > 0 {
		return fmt.Errorf("Keys our of order! %v > %v", *s.lastKey, key)
	}
	s.lastKey = &key
	return nil
}

func (s *Scanner) blockFor(key []byte) (*bytes.Reader, error, bool) {
	err := s.CheckIfKeyOutOfOrder(key)
	if err != nil {
		return nil, err, false
	}

	if s.reader.index[s.idx].IsAfter(key) {
		log.Printf("[Scanner.blockFor] curBlock after key %s (cur: %d, start: %s)\n",
			hex.EncodeToString(key),
			s.idx,
			hex.EncodeToString(s.reader.index[s.idx].firstKeyBytes),
		)
		return nil, nil, false
	}

	idx := s.findBlock(key)
	log.Printf("[Scanner.blockFor] findBlock key: %s. Picked %d (starts: %s). Cur: %d (starts: %s)\n",
		hex.EncodeToString(key),
		idx,
		hex.EncodeToString(s.reader.index[idx].firstKeyBytes),
		s.idx,
		hex.EncodeToString(s.reader.index[s.idx].firstKeyBytes),
	)

	if idx != s.idx || s.buf == nil { // need to load a new block
		data, err := s.reader.GetBlock(idx)
		if err != nil {
			log.Printf("[Scanner.blockFor] read err %s (key: %s, idx: %d, start: %s)\n",
				err,
				hex.EncodeToString(key),
				idx,
				hex.EncodeToString(s.reader.index[idx].firstKeyBytes),
			)
			return nil, err, false
		}
		s.idx = idx
		s.buf = data
	}

	return s.buf, nil, true
}

func (s *Scanner) GetFirst(key []byte) ([]byte, error, bool) {
	data, err, ok := s.blockFor(key)

	if !ok {
		log.Printf("[Scanner.GetFirst] No Block for key: %s (err: %s, found: %s)\n", hex.EncodeToString(key), err, ok)
		return nil, err, ok
	}

	value, _, found := getValuesFromBuffer(data, key, true)
	return value, nil, found
}

func (s *Scanner) GetAll(key []byte) ([][]byte, error) {
	data, err, ok := s.blockFor(key)

	if !ok {
		log.Printf("[Scanner.GetAll] No Block for key: %s (err: %s, found: %s)\n", hex.EncodeToString(key), err, ok)
		return nil, err
	}

	_, found, _ := getValuesFromBuffer(data, key, false)
	return found, err
}

func getValuesFromBuffer(buf *bytes.Reader, key []byte, first bool) ([]byte, [][]byte, bool) {
	var acc [][]byte

	for buf.Len() > 0 {
		var keyLen, valLen uint32
		binary.Read(buf, binary.BigEndian, &keyLen)
		binary.Read(buf, binary.BigEndian, &valLen)
		keyBytes := make([]byte, keyLen)
		valBytes := make([]byte, valLen)
		buf.Read(keyBytes)
		buf.Read(valBytes)
		if bytes.Compare(key, keyBytes) == 0 {
			if first {
				return valBytes, nil, true
			} else {
				acc = append(acc, valBytes)
			}
		}
	}

	return nil, acc, len(acc) > 0
}
<|endoftext|>


<commit_before>package controllers

import (
	"github.com/astaxie/beego"
	"github.com/astaxie/beego/validation"
	"github.com/liulixiang1988/WIM-System/helper"
	"github.com/liulixiang1988/WIM-System/models/datalineinfo"
	"github.com/liulixiang1988/WIM-System/models/detail"
	"github.com/oal/beego-pongo2"
	"time"
)

type WeightController struct {
	beego.Controller
}

func (this *WeightController) Get() {
	last, err := datalineinfo.Last(1)
	var msg string
	if err != nil {
		msg = "error"
	}
	pongo2.Render(this.Ctx, "weight/home.html", pongo2.Context{
		"title":        "首页",
		"msg":          msg,
		"datalineinfo": last,
	})
}

func (this *WeightController) DailyStatistics() {
	pongo2.Render(this.Ctx, "weight/daily_statistics.html", pongo2.Context{
		"title": "日统计",
	})
}

//按班次查询
func (this *WeightController) WorkShift() {
	data := pongo2.Context{"title": "班次明细查询"}
	var msg []string = make([]string, 0)

	dayStr := this.GetString("day")
	if dayStr == "" {
		dayStr = time.Now().Format("2006-01-02")
	}
	data["day"] = dayStr

	workshift, err := this.GetInt("workshift")
	if err != nil {
		workshift = 1
	}
	data["workshift"] = workshift

	workarea, err := this.GetInt("workarea")
	if err != nil {
		workarea = 1
	}
	data["workarea"] = workarea

	valid := validation.Validation{}
	valid.Match(dayStr, helper.DatePatten, "day")
	valid.Required(workshift, "workshift")
	valid.Required(workarea, "workarea")
	if valid.HasErrors() {
		for _, err := range valid.Errors {
			msg = append(msg, err.String())
		}
		data["msg"] = msg
		pongo2.Render(this.Ctx, "weight/workshift.html", data)
		return
	}

	day, _ := time.Parse("2006-01-02", dayStr)

	results, err := datalineinfo.WorkShift(day, int8(workshift), workarea)
	if err != nil {
		msg = append(msg, "请选择日期与班次")
		msg = append(msg, err.Error())
		data["msg"] = msg
	}
	data["results"] = results

	pongo2.Render(this.Ctx, "weight/workshift.html", data)
}

func (this *WeightController) GetDetails() {
	data := pongo2.Context{"title": "批次明细"}
	var msg []string = make([]string, 0)
	batchNumber, err := this.GetInt("batch")
	if err != nil {
		batchNumber = 0
	}
	data["batch"] = batchNumber

	workarea, err := this.GetInt("workarea")
	if err != nil {
		workarea = 1
	}
	data["workarea"] = workarea

	results, err := detail.GetDetails(workarea, batchNumber)
	if err != nil {
		msg = append(msg, err.Error())
		data["msg"] = msg
	}
	data["results"] = results
	pongo2.Render(this.Ctx, "weight/details.html", data)
}
<commit_msg>添加时间范围统计<commit_after>package controllers

import (
	"github.com/astaxie/beego"
	"github.com/astaxie/beego/validation"
	"github.com/liulixiang1988/WIM-System/helper"
	"github.com/liulixiang1988/WIM-System/models/datalineinfo"
	"github.com/liulixiang1988/WIM-System/models/detail"
	"github.com/oal/beego-pongo2"
	"time"
)

type WeightController struct {
	beego.Controller
}

func (this *WeightController) Get() {
	last, err := datalineinfo.Last(1)
	var msg string
	if err != nil {
		msg = "error"
	}
	pongo2.Render(this.Ctx, "weight/home.html", pongo2.Context{
		"title":        "首页",
		"msg":          msg,
		"datalineinfo": last,
	})
}

func (this *WeightController) DailyStatistics() {
	pongo2.Render(this.Ctx, "weight/daily_statistics.html", pongo2.Context{
		"title": "日统计",
	})
}

//按班次查询
func (this *WeightController) WorkShift() {
	data := pongo2.Context{"title": "班次明细查询"}
	var msg []string = make([]string, 0)

	dayStr := this.GetString("day")
	if dayStr == "" {
		dayStr = time.Now().Format("2006-01-02")
	}
	data["day"] = dayStr

	workshift, err := this.GetInt("workshift")
	if err != nil {
		workshift = 1
	}
	data["workshift"] = workshift

	workarea, err := this.GetInt("workarea")
	if err != nil {
		workarea = 1
	}
	data["workarea"] = workarea

	valid := validation.Validation{}
	valid.Match(dayStr, helper.DatePatten, "day")
	valid.Required(workshift, "workshift")
	valid.Required(workarea, "workarea")
	if valid.HasErrors() {
		for _, err := range valid.Errors {
			msg = append(msg, err.String())
		}
		data["msg"] = msg
		pongo2.Render(this.Ctx, "weight/workshift.html", data)
		return
	}

	day, _ := time.Parse("2006-01-02", dayStr)

	results, err := datalineinfo.WorkShift(day, int8(workshift), workarea)
	if err != nil {
		msg = append(msg, "请选择日期与班次")
		msg = append(msg, err.Error())
		data["msg"] = msg
	}
	data["results"] = results

	pongo2.Render(this.Ctx, "weight/workshift.html", data)
}

func (this *WeightController) GetDetails() {
	data := pongo2.Context{"title": "批次明细"}
	var msg []string = make([]string, 0)
	batchNumber, err := this.GetInt("batch")
	if err != nil {
		batchNumber = 0
	}
	data["batch"] = batchNumber

	workarea, err := this.GetInt("workarea")
	if err != nil {
		workarea = 1
	}
	data["workarea"] = workarea

	results, err := detail.GetDetails(workarea, batchNumber)
	if err != nil {
		msg = append(msg, err.Error())
		data["msg"] = msg
	}
	data["results"] = results
	pongo2.Render(this.Ctx, "weight/details.html", data)
}

func (this *WeightController) GetStatics() {
	data := pongo2.Context{"title": "时间范围统计"}
	var msg []string = make([]string, 0)

}
<|endoftext|>


<commit_before>package sparta

import (
	"fmt"
	"net/http"
	"time"

	"github.com/Sirupsen/logrus"
)

// Port used for HTTP proxying communication
const defaultHTTPPort = 9999

// Execute creates an HTTP listener to dispatch execution. Typically
// called via Main() via command line arguments.
func Execute(lambdaAWSInfos []*LambdaAWSInfo, port int, parentProcessPID int, logger *logrus.Logger) error {
	if port <= 0 {
		port = defaultHTTPPort
	}

	server := &http.Server{
		Addr:         fmt.Sprintf(":%d", port),
		Handler:      NewLambdaHTTPHandler(lambdaAWSInfos, logger),
		ReadTimeout:  10 * time.Second,
		WriteTimeout: 10 * time.Second,
	}
	if 0 != parentProcessPID {
		platformKill(parentProcessPID)
	}
	logger.WithFields(logrus.Fields{
		"URL": fmt.Sprintf("http://localhost:%d", port),
	}).Info("Starting Sparta server")

	err := server.ListenAndServe()
	if err != nil {
		logger.WithFields(logrus.Fields{
			"Error": err.Error(),
		}).Error("Failed to launch server")
		return err
	}

	return nil
}
<commit_msg>Ensure preconditions met, additional logging for CloudWatch debugging<commit_after>package sparta

import (
	"fmt"
	"github.com/Sirupsen/logrus"
	"net/http"
	"os"
	"path"
	"time"
)

// Port used for HTTP proxying communication
const defaultHTTPPort = 9999

// Execute creates an HTTP listener to dispatch execution. Typically
// called via Main() via command line arguments.
func Execute(lambdaAWSInfos []*LambdaAWSInfo, port int, parentProcessPID int, logger *logrus.Logger) error {
	validationErr := validateSpartaPreconditions(lambdaAWSInfos, logger)
	if validationErr != nil {
		return validationErr
	}

	if port <= 0 {
		port = defaultHTTPPort
	}

	server := &http.Server{
		Addr:         fmt.Sprintf(":%d", port),
		Handler:      NewLambdaHTTPHandler(lambdaAWSInfos, logger),
		ReadTimeout:  10 * time.Second,
		WriteTimeout: 10 * time.Second,
	}
	logger.WithFields(logrus.Fields{
		"ParentPID": parentProcessPID,
	}).Info("Signaling parent process")

	if 0 != parentProcessPID {
		platformKill(parentProcessPID)
	}
	binaryName := path.Base(os.Args[0])
	logger.WithFields(logrus.Fields{
		"URL": fmt.Sprintf("http://localhost:%d", port),
	}).Info(fmt.Sprintf("Starting %s server", binaryName))

	err := server.ListenAndServe()
	if err != nil {
		logger.WithFields(logrus.Fields{
			"Error": err.Error(),
		}).Error("Failed to launch server")
		return err
	}

	return nil
}
<|endoftext|>


<commit_before>// The MIT License (MIT)
//
// Copyright (c) 2014, 2015 Jamie Alquiza
//
// Permission is hereby granted, free of charge, to any person obtaining a copy
// of this software and associated documentation files (the "Software"), to deal
// in the Software without restriction, including without limitation the rights
// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
// copies of the Software, and to permit persons to whom the Software is
// furnished to do so, subject to the following conditions:
//
// The above copyright notice and this permission notice shall be included in
// all copies or substantial portions of the Software.
//
// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
// THE SOFTWARE.
package main

import (
	"flag"
	"fmt"
	"log"
	"math/rand"
	"os"
	"os/signal"
	"strconv"
	"strings"
	"syscall"
	"time"

	kafka "github.com/Shopify/sarama"
	"github.com/jamiealquiza/tachymeter"
)

var (
	// Configs.
	brokers        []string
	topic          string
	msgSize        int
	msgRate        int64
	batchSize      int
	compressionOpt string
	compression    kafka.CompressionCodec
	clients        int
	producers      int
	noop           bool

	// Character selection for random messages.
	chars = []byte("abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ1234567890!@#$^&*(){}][:<>.")

	// Counters / misc.
	signals     = make(chan os.Signal)
	killClients = make(chan bool, 24)
	sentCntr    = make(chan int64, 1)
)

func init() {
	flag.StringVar(&topic, "topic", "sangrenel", "Topic to publish to")
	flag.IntVar(&msgSize, "size", 300, "Message size in bytes")
	flag.Int64Var(&msgRate, "rate", 100000000, "Apply a global message rate limit")
	flag.IntVar(&batchSize, "batch", 0, "Max messages per batch. Defaults to unlimited (0).")
	flag.StringVar(&compressionOpt, "compression", "none", "Message compression: none, gzip, snappy")
	flag.BoolVar(&noop, "noop", false, "Test message generation performance, do not transmit messages")
	flag.IntVar(&clients, "clients", 1, "Number of Kafka client workers")
	flag.IntVar(&producers, "producers", 5, "Number of producer instances per client")
	brokerString := flag.String("brokers", "localhost:9092", "Comma delimited list of Kafka brokers")
	flag.Parse()

	brokers = strings.Split(*brokerString, ",")

	switch compressionOpt {
	case "gzip":
		compression = kafka.CompressionGZIP
	case "snappy":
		compression = kafka.CompressionSnappy
	case "none":
		compression = kafka.CompressionNone
	default:
		fmt.Printf("Invalid compression option: %s\n", compressionOpt)
		os.Exit(1)
	}

	sentCntr <- 0
}

// clientProducer generates random messages and writes to Kafka.
// Workers track and limit message rates using incrSent() and fetchSent().
// Default 5 instances of clientProducer are created under each Kafka client.
func clientProducer(c kafka.Client, t *tachymeter.Tachymeter) {
	producer, err := kafka.NewSyncProducerFromClient(c)
	if err != nil {
		log.Println(err.Error())
	}
	defer producer.Close()

	source := rand.NewSource(time.Now().UnixNano())
	generator := rand.New(source)
	msgData := make([]byte, msgSize)

	// Use a local accumulator then periodically update global counter.
	// Global counter can become a bottleneck with too many threads.
	// tick := time.Tick(2 * time.Millisecond)
	var n int64
	var times [10]time.Duration

	for {
		// Message rate limit works by having all clientProducer loops incrementing
		// a global counter and tracking the aggregate per-second progress.
		// If the configured rate is met, the worker will sleep
		// for the remainder of the 1 second window.
		rateEnd := time.Now().Add(time.Second)
		countStart := fetchSent()
		var start time.Time
		for fetchSent()-countStart < msgRate {
			randMsg(msgData, *generator)
			msg := &kafka.ProducerMessage{Topic: topic, Value: kafka.ByteEncoder(msgData)}

			start = time.Now()
			_, _, err = producer.SendMessage(msg)
			if err != nil {
				log.Println(err)
			} else {
				// Increment global counter and
				// tachymeter every 10 messages.
				n++
				times[n-1] = time.Since(start)
				if n == 10 {
					incrSent(10)
					for _, ts := range times {
						t.AddTime(ts)
					}
					n = 0
				}
			}
		}
		// If the global per-second rate limit was met,
		// the inner loop breaks and the outer loop sleeps for the second remainder.
		time.Sleep(rateEnd.Sub(time.Now()) + time.Since(start))
	}
}

// clientDummyProducer is a dummy function that kafkaClient calls if noop is True.
// It is used in place of starting actual Kafka client connections to test message creation performance.
func clientDummyProducer(t *tachymeter.Tachymeter) {
	source := rand.NewSource(time.Now().UnixNano())
	generator := rand.New(source)
	msg := make([]byte, msgSize)

	var n int64
	var times [10]time.Duration

	for {
		start := time.Now()
		randMsg(msg, *generator)

		// Increment global counter and
		// tachymeter every 10 messages.
		n++
		times[n-1] = time.Since(start)
		if n == 10 {
			incrSent(10)
			for _, ts := range times {
				t.AddTime(ts)
			}
			n = 0
		}
	}
}

// kafkaClient initializes a connection to a Kafka cluster and
// initializes one or more clientProducer() (producer instances).
func kafkaClient(n int, t *tachymeter.Tachymeter) {
	switch noop {
	// If not noop, actually fire up Kafka connections and send messages.
	case false:
		cId := "client_" + strconv.Itoa(n)

		conf := kafka.NewConfig()
		if compression != kafka.CompressionNone {
			conf.Producer.Compression = compression
		}
		conf.Producer.Flush.MaxMessages = batchSize

		conf.Producer.MaxMessageBytes = msgSize + 50

		client, err := kafka.NewClient(brokers, conf)
		if err != nil {
			log.Println(err)
			os.Exit(1)
		} else {
			log.Printf("%s connected\n", cId)
		}
		for i := 0; i < producers; i++ {
			go clientProducer(client, t)
		}
	// If noop, we're not creating connections at all.
	// Just generate messages and burn CPU.
	default:
		for i := 0; i < producers; i++ {
			go clientDummyProducer(t)
		}
	}
	<-killClients
}

// Returns a random message generated from the chars byte slice.
// Message length of m bytes as defined by msgSize.
func randMsg(m []byte, generator rand.Rand) {
	for i := range m {
		m[i] = chars[generator.Intn(len(chars))]
	}
}

// Global counter functions.
func incrSent(n int64) {
	i := <-sentCntr
	sentCntr <- i + n
}
func fetchSent() int64 {
	i := <-sentCntr
	sentCntr <- i
	return i
}

// Calculates aggregate raw message output in human / network units.
func calcOutput(n int64) (float64, string) {
	m := (float64(n) / 5) * float64(msgSize)
	var o string
	switch {
	case m >= 131072:
		o = strconv.FormatFloat(m/131072, 'f', 0, 64) + "Mb/sec"
	case m < 131072:
		o = strconv.FormatFloat(m/1024, 'f', 0, 64) + "KB/sec"
	}
	return m, o
}

func main() {
	// Listens for signals.
	signal.Notify(signals, syscall.SIGINT, syscall.SIGTERM)

	if graphiteIp != "" {
		go graphiteWriter()
	}

	// Print Sangrenel startup info.
	fmt.Println("\n::: Sangrenel :::")
	fmt.Printf("\nStarting %d client workers, %d producers per worker\n", clients, producers)
	fmt.Printf("Message size %d bytes, %d message limit per batch\n", msgSize, batchSize)
	switch compressionOpt {
	case "none":
		fmt.Println("Compression: none")
	case "gzip":
		fmt.Println("Compression: GZIP")
	case "snappy":
		fmt.Println("Compression: Snappy")
	}

	t := tachymeter.New(&tachymeter.Config{Size: 1000, Safe: true})

	// Start client workers.
	for i := 0; i < clients; i++ {
		go kafkaClient(i+1, t)
	}

	// Start Sangrenel periodic info output.
	tick := time.Tick(5 * time.Second)

	var currCnt, lastCnt int64
	start := time.Now()
	for {
		select {
		case <-tick:
			// Set tachymeter wall time.
			t.SetWallTime(time.Since(start))

			// Set last and current to last read sent count.
			lastCnt = currCnt

			// Get actual current sent count, then delta from last count.
			// Delta is divided by update interval (5s) for per-second rate over a window.
			currCnt = fetchSent()
			deltaCnt := currCnt - lastCnt

			stats := t.Calc()

			outputBytes, outputString := calcOutput(deltaCnt)

			// Update the metrics map for the Graphite writer.
			metrics["rate"] = stats.Rate.Second
			metrics["output"] = outputBytes
			metrics["5p"] = (float64(stats.Time.Long5p.Nanoseconds()) / 1000) / 1000
			// Add ts for Graphite.
			now := time.Now()
			ts := float64(now.Unix())
			metrics["timestamp"] = ts

			if graphiteIp != "" {
				metricsOutgoing <- metrics
			}

			fmt.Println()
			log.Printf("Generating %s @ %.0f messages/sec | topic: %s | %.2fms top 5%% latency\n",
				outputString,
				metrics["rate"],
				topic,
				metrics["5p"])

			stats.Dump()

			// Check if the tacymeter size needs to be increased
			// to avoid sampling. Otherwise, just reset it.
			if int(deltaCnt) > len(t.Times) {
				newTachy := tachymeter.New(&tachymeter.Config{Size: int(2 * deltaCnt), Safe: true})
				*t = *newTachy
			} else {
				t.Reset()
			}

			// Reset interval time.
			start = time.Now()

		// Waits for signals. Currently just brutally kills Sangrenel.
		case <-signals:
			fmt.Println("Killing Connections")
			for i := 0; i < clients; i++ {
				killClients <- true
			}
			close(killClients)
			time.Sleep(2 * time.Second)
			os.Exit(0)
		}
	}
}
<commit_msg>defaults with a larger tachymeter size<commit_after>// The MIT License (MIT)
//
// Copyright (c) 2014, 2015 Jamie Alquiza
//
// Permission is hereby granted, free of charge, to any person obtaining a copy
// of this software and associated documentation files (the "Software"), to deal
// in the Software without restriction, including without limitation the rights
// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
// copies of the Software, and to permit persons to whom the Software is
// furnished to do so, subject to the following conditions:
//
// The above copyright notice and this permission notice shall be included in
// all copies or substantial portions of the Software.
//
// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
// THE SOFTWARE.
package main

import (
	"flag"
	"fmt"
	"log"
	"math/rand"
	"os"
	"os/signal"
	"strconv"
	"strings"
	"syscall"
	"time"

	kafka "github.com/Shopify/sarama"
	"github.com/jamiealquiza/tachymeter"
)

var (
	// Configs.
	brokers        []string
	topic          string
	msgSize        int
	msgRate        int64
	batchSize      int
	compressionOpt string
	compression    kafka.CompressionCodec
	clients        int
	producers      int
	noop           bool

	// Character selection for random messages.
	chars = []byte("abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ1234567890!@#$^&*(){}][:<>.")

	// Counters / misc.
	signals     = make(chan os.Signal)
	killClients = make(chan bool, 24)
	sentCntr    = make(chan int64, 1)
)

func init() {
	flag.StringVar(&topic, "topic", "sangrenel", "Topic to publish to")
	flag.IntVar(&msgSize, "size", 300, "Message size in bytes")
	flag.Int64Var(&msgRate, "rate", 100000000, "Apply a global message rate limit")
	flag.IntVar(&batchSize, "batch", 0, "Max messages per batch. Defaults to unlimited (0).")
	flag.StringVar(&compressionOpt, "compression", "none", "Message compression: none, gzip, snappy")
	flag.BoolVar(&noop, "noop", false, "Test message generation performance, do not transmit messages")
	flag.IntVar(&clients, "clients", 1, "Number of Kafka client workers")
	flag.IntVar(&producers, "producers", 5, "Number of producer instances per client")
	brokerString := flag.String("brokers", "localhost:9092", "Comma delimited list of Kafka brokers")
	flag.Parse()

	brokers = strings.Split(*brokerString, ",")

	switch compressionOpt {
	case "gzip":
		compression = kafka.CompressionGZIP
	case "snappy":
		compression = kafka.CompressionSnappy
	case "none":
		compression = kafka.CompressionNone
	default:
		fmt.Printf("Invalid compression option: %s\n", compressionOpt)
		os.Exit(1)
	}

	sentCntr <- 0
}

// clientProducer generates random messages and writes to Kafka.
// Workers track and limit message rates using incrSent() and fetchSent().
// Default 5 instances of clientProducer are created under each Kafka client.
func clientProducer(c kafka.Client, t *tachymeter.Tachymeter) {
	producer, err := kafka.NewSyncProducerFromClient(c)
	if err != nil {
		log.Println(err.Error())
	}
	defer producer.Close()

	source := rand.NewSource(time.Now().UnixNano())
	generator := rand.New(source)
	msgData := make([]byte, msgSize)

	// Use a local accumulator then periodically update global counter.
	// Global counter can become a bottleneck with too many threads.
	// tick := time.Tick(2 * time.Millisecond)
	var n int64
	var times [10]time.Duration

	for {
		// Message rate limit works by having all clientProducer loops incrementing
		// a global counter and tracking the aggregate per-second progress.
		// If the configured rate is met, the worker will sleep
		// for the remainder of the 1 second window.
		rateEnd := time.Now().Add(time.Second)
		countStart := fetchSent()
		var start time.Time
		for fetchSent()-countStart < msgRate {
			randMsg(msgData, *generator)
			msg := &kafka.ProducerMessage{Topic: topic, Value: kafka.ByteEncoder(msgData)}

			start = time.Now()
			_, _, err = producer.SendMessage(msg)
			if err != nil {
				log.Println(err)
			} else {
				// Increment global counter and
				// tachymeter every 10 messages.
				n++
				times[n-1] = time.Since(start)
				if n == 10 {
					incrSent(10)
					for _, ts := range times {
						t.AddTime(ts)
					}
					n = 0
				}
			}
		}
		// If the global per-second rate limit was met,
		// the inner loop breaks and the outer loop sleeps for the second remainder.
		time.Sleep(rateEnd.Sub(time.Now()) + time.Since(start))
	}
}

// clientDummyProducer is a dummy function that kafkaClient calls if noop is True.
// It is used in place of starting actual Kafka client connections to test message creation performance.
func clientDummyProducer(t *tachymeter.Tachymeter) {
	source := rand.NewSource(time.Now().UnixNano())
	generator := rand.New(source)
	msg := make([]byte, msgSize)

	var n int64
	var times [10]time.Duration

	for {
		start := time.Now()
		randMsg(msg, *generator)

		// Increment global counter and
		// tachymeter every 10 messages.
		n++
		times[n-1] = time.Since(start)
		if n == 10 {
			incrSent(10)
			for _, ts := range times {
				t.AddTime(ts)
			}
			n = 0
		}
	}
}

// kafkaClient initializes a connection to a Kafka cluster and
// initializes one or more clientProducer() (producer instances).
func kafkaClient(n int, t *tachymeter.Tachymeter) {
	switch noop {
	// If not noop, actually fire up Kafka connections and send messages.
	case false:
		cId := "client_" + strconv.Itoa(n)

		conf := kafka.NewConfig()
		if compression != kafka.CompressionNone {
			conf.Producer.Compression = compression
		}
		conf.Producer.Flush.MaxMessages = batchSize

		conf.Producer.MaxMessageBytes = msgSize + 50

		client, err := kafka.NewClient(brokers, conf)
		if err != nil {
			log.Println(err)
			os.Exit(1)
		} else {
			log.Printf("%s connected\n", cId)
		}
		for i := 0; i < producers; i++ {
			go clientProducer(client, t)
		}
	// If noop, we're not creating connections at all.
	// Just generate messages and burn CPU.
	default:
		for i := 0; i < producers; i++ {
			go clientDummyProducer(t)
		}
	}
	<-killClients
}

// Returns a random message generated from the chars byte slice.
// Message length of m bytes as defined by msgSize.
func randMsg(m []byte, generator rand.Rand) {
	for i := range m {
		m[i] = chars[generator.Intn(len(chars))]
	}
}

// Global counter functions.
func incrSent(n int64) {
	i := <-sentCntr
	sentCntr <- i + n
}
func fetchSent() int64 {
	i := <-sentCntr
	sentCntr <- i
	return i
}

// Calculates aggregate raw message output in human / network units.
func calcOutput(n int64) (float64, string) {
	m := (float64(n) / 5) * float64(msgSize)
	var o string
	switch {
	case m >= 131072:
		o = strconv.FormatFloat(m/131072, 'f', 0, 64) + "Mb/sec"
	case m < 131072:
		o = strconv.FormatFloat(m/1024, 'f', 0, 64) + "KB/sec"
	}
	return m, o
}

func main() {
	// Listens for signals.
	signal.Notify(signals, syscall.SIGINT, syscall.SIGTERM)

	if graphiteIp != "" {
		go graphiteWriter()
	}

	// Print Sangrenel startup info.
	fmt.Println("\n::: Sangrenel :::")
	fmt.Printf("\nStarting %d client workers, %d producers per worker\n", clients, producers)
	fmt.Printf("Message size %d bytes, %d message limit per batch\n", msgSize, batchSize)
	switch compressionOpt {
	case "none":
		fmt.Println("Compression: none")
	case "gzip":
		fmt.Println("Compression: GZIP")
	case "snappy":
		fmt.Println("Compression: Snappy")
	}

	t := tachymeter.New(&tachymeter.Config{Size: 300000, Safe: true})

	// Start client workers.
	for i := 0; i < clients; i++ {
		go kafkaClient(i+1, t)
	}

	// Start Sangrenel periodic info output.
	tick := time.Tick(5 * time.Second)

	var currCnt, lastCnt int64
	start := time.Now()
	for {
		select {
		case <-tick:
			// Set tachymeter wall time.
			t.SetWallTime(time.Since(start))

			// Set last and current to last read sent count.
			lastCnt = currCnt

			// Get actual current sent count, then delta from last count.
			// Delta is divided by update interval (5s) for per-second rate over a window.
			currCnt = fetchSent()
			deltaCnt := currCnt - lastCnt

			stats := t.Calc()

			outputBytes, outputString := calcOutput(deltaCnt)

			// Update the metrics map for the Graphite writer.
			metrics["rate"] = stats.Rate.Second
			metrics["output"] = outputBytes
			metrics["5p"] = (float64(stats.Time.Long5p.Nanoseconds()) / 1000) / 1000
			// Add ts for Graphite.
			now := time.Now()
			ts := float64(now.Unix())
			metrics["timestamp"] = ts

			if graphiteIp != "" {
				metricsOutgoing <- metrics
			}

			fmt.Println()
			log.Printf("Generating %s @ %.0f messages/sec | topic: %s | %.2fms top 5%% latency\n",
				outputString,
				metrics["rate"],
				topic,
				metrics["5p"])

			stats.Dump()

			// Check if the tacymeter size needs to be increased
			// to avoid sampling. Otherwise, just reset it.
			if int(deltaCnt) > len(t.Times) {
				newTachy := tachymeter.New(&tachymeter.Config{Size: int(2 * deltaCnt), Safe: true})
				// This is actually dangerous;
				// this could swap in a tachy with unlocked
				// mutexes while the current one has locks held.
				*t = *newTachy
			} else {
				t.Reset()
			}

			// Reset interval time.
			start = time.Now()

		// Waits for signals. Currently just brutally kills Sangrenel.
		case <-signals:
			fmt.Println("Killing Connections")
			for i := 0; i < clients; i++ {
				killClients <- true
			}
			close(killClients)
			time.Sleep(2 * time.Second)
			os.Exit(0)
		}
	}
}
<|endoftext|>


<commit_before>package main

import (
	"crypto/x509"
	"encoding/pem"
	"flag"
	"fmt"
	"io"
	"io/ioutil"
	"log"
	"math/rand"
	"net/http"
	"os"
	"os/exec"
	"path"
	"regexp"
)

var db = map[string]string{}
var csrLocation = "/var/csr/"
var crtLocation = "/var/crt/"
var confLocation = "/opt/pollendina/openssl-ca.cnf"

var port = flag.String("port", ":33004", "Default port for Pollendina CA.")

type Tuple struct{ CN, Token string }

var updates = make(chan Tuple)

var (
	Info    *log.Logger
	Warning *log.Logger
	Error   *log.Logger
)

func InitLogs(
	infoHandle io.Writer,
	warningHandle io.Writer,
	errorHandle io.Writer) {

	Info = log.New(infoHandle,
		"INFO: ",
		log.Ldate|log.Ltime|log.Lshortfile)

	Warning = log.New(warningHandle,
		"WARNING: ",
		log.Ldate|log.Ltime|log.Lshortfile)

	Error = log.New(errorHandle,
		"ERROR: ",
		log.Ldate|log.Ltime|log.Lshortfile)
}

func main() {

	flag.Parse()

	InitLogs(os.Stdout, os.Stdout, os.Stderr)

	rh := new(RegexHandler)

	authPathPattern, _ := regexp.Compile("/v1/authorize")
	signPathPattern, _ := regexp.Compile("/v1/sign/.*")

	rh.HandleFunc(authPathPattern, Authorize)
	rh.HandleFunc(signPathPattern, Sign)

	go MapWriter()

	// Placeholder for authentication / authorization middleware on authorize call.

	err := http.ListenAndServe(*port, rh)
	if err != nil {
		Error.Println(err)
	}
}

func MapWriter() {
	for {
		select {
		case t, ok := <-updates:
			if !ok {
				Error.Println("Publisher channel closed. Stopping.")
				return
			}
			Info.Println("Setting key %s to value %s", t.Token, t.CN)
			db[t.Token] = t.CN
		}
	}
}

func Authorize(w http.ResponseWriter, req *http.Request) {
	Info.Println("Received authorize call.")
	// Parse input
	cn := req.FormValue("cn")
	token := req.FormValue("token")
	// life := req.FormValue("lifeInSeconds")

	// TODO: sign certificate with provided expiration date
	fmt.Println("need to incorporate lifeInSeconds for signed cert expriation ts")

	// queue for write to map
	// ...
	t := Tuple{cn, token}
	updates <- t

	Info.Println("Service: %s", cn)
	Info.Println("Token: %s", token)

	req.Body.Close()
}

func Sign(w http.ResponseWriter, req *http.Request) {
	Info.Println("Received sign call.")

	// Pull the token out of the path
	_, token := path.Split(req.URL.Path)
	Info.Println("Received signing request for token %s", token)

	if len(token) == 0 {
		Warning.Println("No token provided.")
		w.WriteHeader(http.StatusBadRequest)
		return
	}

	// Get the registered CN for the provided token (or fail)
	authCn := db[token]

	if authCn == "" {
		Warning.Println("Unauthorized CN.")
		w.WriteHeader(http.StatusUnauthorized)
		return
	}

	// Upload the CSR and copy it to some known location
	body, err := ioutil.ReadAll(req.Body)
	if err != nil {
		Error.Println(err.Error())
		w.WriteHeader(http.StatusBadRequest)
		return
	}

	randoName := fmt.Sprintf("%d.csr", rand.Int63())
	csrFilename := csrLocation + randoName
	err = ioutil.WriteFile(csrFilename, body, 0777)
	if err != nil {
		Error.Println(err.Error())
		w.WriteHeader(http.StatusInternalServerError)
		return
	}
	Info.Println("File uploaded.")

	// Parse the CSR
	rawCSR, err := ioutil.ReadFile(csrFilename)
	if err != nil {
		Error.Println(err.Error())
		w.WriteHeader(http.StatusInternalServerError)
		return
	}
	decodedCSR, _ := pem.Decode(rawCSR)
	csr, err := x509.ParseCertificateRequest(decodedCSR.Bytes)
	if err != nil {
		Error.Println(err.Error())
		w.WriteHeader(http.StatusBadRequest)
		return
	}

	Info.Println("Received CSR for: %s", csr.Subject.CommonName)

	// check authorization for the provided commonname
	if csr.Subject.CommonName != authCn {
		Warning.Println("Unauthorized CN %s", csr.Subject.CommonName)
		w.WriteHeader(http.StatusUnauthorized)
		return
	}

	// Build the command for exec
	// openssl ca -config openssl-ca.cnf -policy signing_policy -extensions signing_req -out servercert.pem -infiles servercert.csr
	app := "openssl"
	command := "ca"
	c_flag := "-config"
	p_flag := "-policy"
	p_value := "signing_policy"
	e_flag := "-extensions"
	e_value := "signing_req"
	o_flag := "-out"
	outputFile := crtLocation + randoName + ".crt"
	i_flag := "-infiles"
	b_flag := "-batch"

	// Sign the CSR with OpenSSL
	cmd := exec.Command(app, command, b_flag, c_flag, confLocation, p_flag, p_value, e_flag, e_value, o_flag, outputFile, i_flag, csrFilename)
	args := fmt.Sprintf("%s %s %s %s %s %s %s %s %s %s %s %s %s", app, command, b_flag, c_flag, confLocation, p_flag, p_value, e_flag, e_value, o_flag, outputFile, i_flag, csrFilename)
	fmt.Println(args)
	stdOut, err := cmd.Output()
	if err != nil {
		Error.Println("OpenSSL stdout: %s", string(stdOut))
		Error.Println("OpenSSL stderr: %s", err.Error())
		w.WriteHeader(http.StatusInternalServerError)
		return
	}

	// Open the output file for reading and stream it back on the response
	outputData, err := ioutil.ReadFile(outputFile)
	w.Write(outputData)
}

type MuxRoute struct {
	pattern *regexp.Regexp
	handler http.Handler
}

type RegexHandler struct {
	rs []*MuxRoute
}

func (rh *RegexHandler) Handler(p *regexp.Regexp, h http.Handler) {
	rh.rs = append(rh.rs, &MuxRoute{p, h})
}

func (rh *RegexHandler) HandleFunc(p *regexp.Regexp, h func(http.ResponseWriter, *http.Request)) {
	rh.rs = append(rh.rs, &MuxRoute{p, http.HandlerFunc(h)})
}

func (rh *RegexHandler) ServeHTTP(w http.ResponseWriter, r *http.Request) {
	for _, route := range rh.rs {
		if route.pattern.MatchString(r.URL.Path) {
			route.handler.ServeHTTP(w, r)
			return
		}
	}
	Warning.Println("Route not found: %s", r.URL.Path)
	// no pattern matched; send 404 response
	http.NotFound(w, r)
}
<commit_msg>Fixing service logging.<commit_after>package main

import (
	"crypto/x509"
	"encoding/pem"
	"flag"
	"fmt"
	"io"
	"io/ioutil"
	"log"
	"math/rand"
	"net/http"
	"os"
	"os/exec"
	"path"
	"regexp"
)

var db = map[string]string{}
var csrLocation = "/var/csr/"
var crtLocation = "/var/crt/"
var confLocation = "/opt/pollendina/openssl-ca.cnf"

var port = flag.String("port", ":33004", "Default port for Pollendina CA.")

type Tuple struct{ CN, Token string }

var updates = make(chan Tuple)

var (
	Info    *log.Logger
	Warning *log.Logger
	Error   *log.Logger
)

func InitLogs(
	infoHandle io.Writer,
	warningHandle io.Writer,
	errorHandle io.Writer) {

	Info = log.New(infoHandle,
		"INFO: ",
		log.Ldate|log.Ltime|log.Lshortfile)

	Warning = log.New(warningHandle,
		"WARNING: ",
		log.Ldate|log.Ltime|log.Lshortfile)

	Error = log.New(errorHandle,
		"ERROR: ",
		log.Ldate|log.Ltime|log.Lshortfile)
}

func main() {

	flag.Parse()

	InitLogs(os.Stdout, os.Stdout, os.Stderr)

	rh := new(RegexHandler)

	authPathPattern, _ := regexp.Compile("/v1/authorize")
	signPathPattern, _ := regexp.Compile("/v1/sign/.*")

	rh.HandleFunc(authPathPattern, Authorize)
	rh.HandleFunc(signPathPattern, Sign)

	go MapWriter()

	// Placeholder for authentication / authorization middleware on authorize call.

	err := http.ListenAndServe(*port, rh)
	if err != nil {
		Error.Println(err)
	}
}

func MapWriter() {
	for {
		select {
		case t, ok := <-updates:
			if !ok {
				Error.Printf("Publisher channel closed. Stopping.")
				return
			}
			Info.Printf("Setting key %s to value %s", t.Token, t.CN)
			db[t.Token] = t.CN
		}
	}
}

func Authorize(w http.ResponseWriter, req *http.Request) {
	Info.Printf("Received authorize call.")
	// Parse input
	cn := req.FormValue("cn")
	token := req.FormValue("token")
	// life := req.FormValue("lifeInSeconds")

	// TODO: sign certificate with provided expiration date
	fmt.Printf("need to incorporate lifeInSeconds for signed cert expriation ts")

	// queue for write to map
	// ...
	t := Tuple{cn, token}
	updates <- t

	Info.Printf("Service: %s", cn)
	Info.Printf("Token: %s", token)

	req.Body.Close()
}

func Sign(w http.ResponseWriter, req *http.Request) {
	Info.Printf("Received sign call.")

	// Pull the token out of the path
	_, token := path.Split(req.URL.Path)
	Info.Printf("Received signing request for token %s", token)

	if len(token) == 0 {
		Warning.Printf("No token provided.")
		w.WriteHeader(http.StatusBadRequest)
		return
	}

	// Get the registered CN for the provided token (or fail)
	authCn := db[token]

	if authCn == "" {
		Warning.Printf("Unauthorized CN.")
		w.WriteHeader(http.StatusUnauthorized)
		return
	}

	// Upload the CSR and copy it to some known location
	body, err := ioutil.ReadAll(req.Body)
	if err != nil {
		Error.Printf(err.Error())
		w.WriteHeader(http.StatusBadRequest)
		return
	}

	randoName := fmt.Sprintf("%d.csr", rand.Int63())
	csrFilename := csrLocation + randoName
	err = ioutil.WriteFile(csrFilename, body, 0777)
	if err != nil {
		Error.Printf(err.Error())
		w.WriteHeader(http.StatusInternalServerError)
		return
	}
	Info.Printf("File uploaded.")

	// Parse the CSR
	rawCSR, err := ioutil.ReadFile(csrFilename)
	if err != nil {
		Error.Printf(err.Error())
		w.WriteHeader(http.StatusInternalServerError)
		return
	}
	decodedCSR, _ := pem.Decode(rawCSR)
	csr, err := x509.ParseCertificateRequest(decodedCSR.Bytes)
	if err != nil {
		Error.Printf(err.Error())
		w.WriteHeader(http.StatusBadRequest)
		return
	}

	Info.Printf("Received CSR for: %s", csr.Subject.CommonName)

	// check authorization for the provided commonname
	if csr.Subject.CommonName != authCn {
		Warning.Printf("Unauthorized CN %s", csr.Subject.CommonName)
		w.WriteHeader(http.StatusUnauthorized)
		return
	}

	// Build the command for exec
	// openssl ca -config openssl-ca.cnf -policy signing_policy -extensions signing_req -out servercert.pem -infiles servercert.csr
	app := "openssl"
	command := "ca"
	c_flag := "-config"
	p_flag := "-policy"
	p_value := "signing_policy"
	e_flag := "-extensions"
	e_value := "signing_req"
	o_flag := "-out"
	outputFile := crtLocation + randoName + ".crt"
	i_flag := "-infiles"
	b_flag := "-batch"

	// Sign the CSR with OpenSSL
	cmd := exec.Command(app, command, b_flag, c_flag, confLocation, p_flag, p_value, e_flag, e_value, o_flag, outputFile, i_flag, csrFilename)
	args := fmt.Sprintf("%s %s %s %s %s %s %s %s %s %s %s %s %s", app, command, b_flag, c_flag, confLocation, p_flag, p_value, e_flag, e_value, o_flag, outputFile, i_flag, csrFilename)
	fmt.Printf(args)
	stdOut, err := cmd.Output()
	if err != nil {
		Error.Printf("OpenSSL stdout: %s", string(stdOut))
		Error.Printf("OpenSSL stderr: %s", err.Error())
		w.WriteHeader(http.StatusInternalServerError)
		return
	}

	// Open the output file for reading and stream it back on the response
	outputData, err := ioutil.ReadFile(outputFile)
	w.Write(outputData)
}

type MuxRoute struct {
	pattern *regexp.Regexp
	handler http.Handler
}

type RegexHandler struct {
	rs []*MuxRoute
}

func (rh *RegexHandler) Handler(p *regexp.Regexp, h http.Handler) {
	rh.rs = append(rh.rs, &MuxRoute{p, h})
}

func (rh *RegexHandler) HandleFunc(p *regexp.Regexp, h func(http.ResponseWriter, *http.Request)) {
	rh.rs = append(rh.rs, &MuxRoute{p, http.HandlerFunc(h)})
}

func (rh *RegexHandler) ServeHTTP(w http.ResponseWriter, r *http.Request) {
	for _, route := range rh.rs {
		if route.pattern.MatchString(r.URL.Path) {
			route.handler.ServeHTTP(w, r)
			return
		}
	}
	Warning.Printf("Route not found: %s", r.URL.Path)
	// no pattern matched; send 404 response
	http.NotFound(w, r)
}
<|endoftext|>


<commit_before>package main

import (
	"crypto/x509"
	"flag"
        "fmt"
        "log"
	"io/ioutil"
        "math/rand"
	"net/http"
	"os/exec"
        "path"
        "regexp"
        "encoding/pem"
)
var db = map[string]string{}
var csrLocation = "/var/csr/"
var crtLocation = "/var/crt/"
var confLocation = "/opt/pollendina/openssl-ca.cnf"

type Tuple struct { CN, Token string }
var updates = make(chan Tuple)

func main() {

	flag.Parse()

        rh := new(RegexHandler)

        authPathPattern,_ := regexp.Compile("/v1/authorize")
        signPathPattern,_ := regexp.Compile("/v1/sign/.*")

	rh.HandleFunc(authPathPattern, Authorize)
	rh.HandleFunc(signPathPattern, Sign)

        go MapWriter()

	// Placeholder for authentication / authorization middleware on authorize call.

	err := http.ListenAndServe(":33004", rh)
	if err != nil {
		log.Fatal("ListenAndServe: ", err)
	}
}

func MapWriter() {
      for {
              select {
                      case t,ok := <-updates:
                          if (!ok) {
                               log.Printf("Publisher channel closed. Stopping.")
                               return;
                          }
                          log.Printf("Setting key %s to value %s", t.Token, t.CN)
                          db[t.Token] = t.CN
              }
      }
}

func Authorize(w http.ResponseWriter, req *http.Request) {
	log.Println("Received authorize call.")
	// Parse input
	cn := req.FormValue("cn")
	token := req.FormValue("token")

	// queue for write to map
	// ...
        t := Tuple{cn, token}
        updates <- t

        log.Println("Service: " + cn)
        log.Println("Token: " + token)

	req.Body.Close()
}

func Sign(w http.ResponseWriter, req *http.Request) {
	log.Println("Received sign call.")

        // Pull the token out of the path
        _, token := path.Split(req.URL.Path)
        log.Printf("Received signing request for token %s", token)

        if len(token) == 0 {
                log.Println("No token provided.")
                w.WriteHeader(http.StatusBadRequest)
                return
        }

        // Get the registered CN for the provided token (or fail)
        authCn := db[token]

        if (authCn == "") {
                log.Println("unauthorized")
                w.WriteHeader(http.StatusUnauthorized)
                return
        }

	// Upload the CSR and copy it to some known location
        body, err := ioutil.ReadAll(req.Body)
        if err != nil {
                log.Println(err)
                w.WriteHeader(http.StatusBadRequest)
                return
        }

        randoName := fmt.Sprintf("%d.csr", rand.Int63())
        csrFilename := csrLocation + randoName
        err = ioutil.WriteFile(csrFilename, body, 0777)
        if err != nil {
                log.Println(err)
                w.WriteHeader(http.StatusInternalServerError)
                return
        }
        log.Println("File uploaded.")

	// Parse the CSR
	rawCSR, _ := ioutil.ReadFile(csrFilename)
        decodedCSR, _ := pem.Decode(rawCSR)
	csr, err := x509.ParseCertificateRequest(decodedCSR.Bytes)
	if err != nil {
		log.Println(err)
		w.WriteHeader(http.StatusBadRequest)
		return
	}

	log.Println("Received CSR for: " + csr.Subject.CommonName)

	// check authorization for the provided commonname
        if (csr.Subject.CommonName != authCn) {
                log.Println("unauthorized")
                w.WriteHeader(http.StatusUnauthorized)
                return
        }

	// Build the command for exec
	// openssl ca -config openssl-ca.cnf -policy signing_policy -extensions signing_req -out servercert.pem -infiles servercert.csr
	app := "openssl"
	command := "ca"
	c_flag := "-config"
	p_flag := "-policy"
	p_value := "signing_policy"
	e_flag := "-extensions"
	e_value := "signing_req"
	o_flag := "-out"
	outputFile := crtLocation + randoName + ".crt"
	i_flag := "-infiles"
        b_flag := "-batch"

	// Sign the CSR with OpenSSL
	cmd := exec.Command(app, command, b_flag, c_flag, confLocation, p_flag, p_value, e_flag, e_value, o_flag, outputFile, i_flag, csrFilename)
        args := fmt.Sprintf("%s %s %s %s %s %s %s %s %s %s %s %s %s", app, command, b_flag, c_flag, confLocation, p_flag, p_value, e_flag, e_value, o_flag, outputFile, i_flag, csrFilename)
	fmt.Println(args)
        stdOut, err := cmd.Output()
	if err != nil {
                log.Println("STDOUT: " + string(stdOut))
		log.Println("STDERR: " + err.Error())
		w.WriteHeader(http.StatusInternalServerError)
		return
	}

	// Open the output file for reading and stream it back on the response
        outputData, err := ioutil.ReadFile(outputFile)
	w.Write(outputData)
}

type MuxRoute struct {
    pattern *regexp.Regexp
    handler http.Handler
}

type RegexHandler struct {
    rs []*MuxRoute
}

func (rh *RegexHandler) Handler(p *regexp.Regexp, h http.Handler) {
    rh.rs = append(rh.rs, &MuxRoute{p, h})
}

func (rh *RegexHandler) HandleFunc(p *regexp.Regexp, h func(http.ResponseWriter, *http.Request)) {
    rh.rs = append(rh.rs, &MuxRoute{p, http.HandlerFunc(h)})
}

func (rh *RegexHandler) ServeHTTP(w http.ResponseWriter, r *http.Request) {
    for _, route := range rh.rs {
        if route.pattern.MatchString(r.URL.Path) {
            route.handler.ServeHTTP(w, r)
            return
        }
    }
    log.Println("missed")
    // no pattern matched; send 404 response
    http.NotFound(w, r)
}

<commit_msg>Go fmt.<commit_after>package main

import (
	"crypto/x509"
	"encoding/pem"
	"flag"
	"fmt"
	"io/ioutil"
	"log"
	"math/rand"
	"net/http"
	"os/exec"
	"path"
	"regexp"
)

var db = map[string]string{}
var csrLocation = "/var/csr/"
var crtLocation = "/var/crt/"
var confLocation = "/opt/pollendina/openssl-ca.cnf"

type Tuple struct{ CN, Token string }

var updates = make(chan Tuple)

func main() {

	flag.Parse()

	rh := new(RegexHandler)

	authPathPattern, _ := regexp.Compile("/v1/authorize")
	signPathPattern, _ := regexp.Compile("/v1/sign/.*")

	rh.HandleFunc(authPathPattern, Authorize)
	rh.HandleFunc(signPathPattern, Sign)

	go MapWriter()

	// Placeholder for authentication / authorization middleware on authorize call.

	err := http.ListenAndServe(":33004", rh)
	if err != nil {
		log.Fatal("ListenAndServe: ", err)
	}
}

func MapWriter() {
	for {
		select {
		case t, ok := <-updates:
			if !ok {
				log.Printf("Publisher channel closed. Stopping.")
				return
			}
			log.Printf("Setting key %s to value %s", t.Token, t.CN)
			db[t.Token] = t.CN
		}
	}
}

func Authorize(w http.ResponseWriter, req *http.Request) {
	log.Println("Received authorize call.")
	// Parse input
	cn := req.FormValue("cn")
	token := req.FormValue("token")

	// queue for write to map
	// ...
	t := Tuple{cn, token}
	updates <- t

	log.Println("Service: " + cn)
	log.Println("Token: " + token)

	req.Body.Close()
}

func Sign(w http.ResponseWriter, req *http.Request) {
	log.Println("Received sign call.")

	// Pull the token out of the path
	_, token := path.Split(req.URL.Path)
	log.Printf("Received signing request for token %s", token)

	if len(token) == 0 {
		log.Println("No token provided.")
		w.WriteHeader(http.StatusBadRequest)
		return
	}

	// Get the registered CN for the provided token (or fail)
	authCn := db[token]

	if authCn == "" {
		log.Println("unauthorized")
		w.WriteHeader(http.StatusUnauthorized)
		return
	}

	// Upload the CSR and copy it to some known location
	body, err := ioutil.ReadAll(req.Body)
	if err != nil {
		log.Println(err)
		w.WriteHeader(http.StatusBadRequest)
		return
	}

	randoName := fmt.Sprintf("%d.csr", rand.Int63())
	csrFilename := csrLocation + randoName
	err = ioutil.WriteFile(csrFilename, body, 0777)
	if err != nil {
		log.Println(err)
		w.WriteHeader(http.StatusInternalServerError)
		return
	}
	log.Println("File uploaded.")

	// Parse the CSR
	rawCSR, _ := ioutil.ReadFile(csrFilename)
	decodedCSR, _ := pem.Decode(rawCSR)
	csr, err := x509.ParseCertificateRequest(decodedCSR.Bytes)
	if err != nil {
		log.Println(err)
		w.WriteHeader(http.StatusBadRequest)
		return
	}

	log.Println("Received CSR for: " + csr.Subject.CommonName)

	// check authorization for the provided commonname
	if csr.Subject.CommonName != authCn {
		log.Println("unauthorized")
		w.WriteHeader(http.StatusUnauthorized)
		return
	}

	// Build the command for exec
	// openssl ca -config openssl-ca.cnf -policy signing_policy -extensions signing_req -out servercert.pem -infiles servercert.csr
	app := "openssl"
	command := "ca"
	c_flag := "-config"
	p_flag := "-policy"
	p_value := "signing_policy"
	e_flag := "-extensions"
	e_value := "signing_req"
	o_flag := "-out"
	outputFile := crtLocation + randoName + ".crt"
	i_flag := "-infiles"
	b_flag := "-batch"

	// Sign the CSR with OpenSSL
	cmd := exec.Command(app, command, b_flag, c_flag, confLocation, p_flag, p_value, e_flag, e_value, o_flag, outputFile, i_flag, csrFilename)
	args := fmt.Sprintf("%s %s %s %s %s %s %s %s %s %s %s %s %s", app, command, b_flag, c_flag, confLocation, p_flag, p_value, e_flag, e_value, o_flag, outputFile, i_flag, csrFilename)
	fmt.Println(args)
	stdOut, err := cmd.Output()
	if err != nil {
		log.Println("STDOUT: " + string(stdOut))
		log.Println("STDERR: " + err.Error())
		w.WriteHeader(http.StatusInternalServerError)
		return
	}

	// Open the output file for reading and stream it back on the response
	outputData, err := ioutil.ReadFile(outputFile)
	w.Write(outputData)
}

type MuxRoute struct {
	pattern *regexp.Regexp
	handler http.Handler
}

type RegexHandler struct {
	rs []*MuxRoute
}

func (rh *RegexHandler) Handler(p *regexp.Regexp, h http.Handler) {
	rh.rs = append(rh.rs, &MuxRoute{p, h})
}

func (rh *RegexHandler) HandleFunc(p *regexp.Regexp, h func(http.ResponseWriter, *http.Request)) {
	rh.rs = append(rh.rs, &MuxRoute{p, http.HandlerFunc(h)})
}

func (rh *RegexHandler) ServeHTTP(w http.ResponseWriter, r *http.Request) {
	for _, route := range rh.rs {
		if route.pattern.MatchString(r.URL.Path) {
			route.handler.ServeHTTP(w, r)
			return
		}
	}
	log.Println("missed")
	// no pattern matched; send 404 response
	http.NotFound(w, r)
}
<|endoftext|>


<commit_before>package session

import (
	"testing"

	"github.com/alexedwards/scs/mem/engine"
)

func TestString(t *testing.T) {
	m := Manage(engine.New())
	h := m(testServeMux)

	_, body, cookie := testRequest(t, h, "/PutString", "")
	if body != "OK" {
		t.Fatalf("got %q: expected %q", body, "OK")
	}

	_, body, _ = testRequest(t, h, "/GetString", cookie)
	if body != "lorem ipsum" {
		t.Fatalf("got %q: expected %q", body, "lorem ipsum")
	}

	_, body, cookie = testRequest(t, h, "/PopString", cookie)
	if body != "lorem ipsum" {
		t.Fatalf("got %q: expected %q", body, "lorem ipsum")
	}

	_, body, _ = testRequest(t, h, "/GetString", cookie)
	if body != ErrKeyNotFound.Error() {
		t.Fatalf("got %q: expected %q", body, ErrKeyNotFound.Error())
	}
}

func TestBool(t *testing.T) {
	m := Manage(engine.New())
	h := m(testServeMux)

	_, body, cookie := testRequest(t, h, "/PutBool", "")
	if body != "OK" {
		t.Fatalf("got %q: expected %q", body, "OK")
	}

	_, body, _ = testRequest(t, h, "/GetBool", cookie)
	if body != "true" {
		t.Fatalf("got %q: expected %q", body, "true")
	}

	_, body, cookie = testRequest(t, h, "/PopBool", cookie)
	if body != "true" {
		t.Fatalf("got %q: expected %q", body, "true")
	}

	_, body, _ = testRequest(t, h, "/GetBool", cookie)
	if body != ErrKeyNotFound.Error() {
		t.Fatalf("got %q: expected %q", body, ErrKeyNotFound.Error())
	}
}

func TestInt(t *testing.T) {
	m := Manage(engine.New())
	h := m(testServeMux)

	_, body, cookie := testRequest(t, h, "/PutInt", "")
	if body != "OK" {
		t.Fatalf("got %q: expected %q", body, "OK")
	}

	_, body, _ = testRequest(t, h, "/GetInt", cookie)
	if body != "12345" {
		t.Fatalf("got %q: expected %q", body, "12345")
	}

	_, body, cookie = testRequest(t, h, "/PopInt", cookie)
	if body != "12345" {
		t.Fatalf("got %q: expected %q", body, "12345")
	}

	_, body, _ = testRequest(t, h, "/GetInt", cookie)
	if body != ErrKeyNotFound.Error() {
		t.Fatalf("got %q: expected %q", body, ErrKeyNotFound.Error())
	}
}

func TestRemove(t *testing.T) {
	m := Manage(engine.New())
	h := m(testServeMux)

	_, _, cookie := testRequest(t, h, "/PutString", "")
	_, _, cookie = testRequest(t, h, "/PutBool", cookie)

	_, body, cookie := testRequest(t, h, "/RemoveString", cookie)
	if body != "OK" {
		t.Fatalf("got %q: expected %q", body, "OK")
	}

	_, body, _ = testRequest(t, h, "/GetString", cookie)
	if body != ErrKeyNotFound.Error() {
		t.Fatalf("got %q: expected %q", body, ErrKeyNotFound.Error())
	}

	_, body, _ = testRequest(t, h, "/GetBool", cookie)
	if body != "true" {
		t.Fatalf("got %q: expected %q", body, "true")
	}
}

func TestClear(t *testing.T) {
	m := Manage(engine.New())
	h := m(testServeMux)

	_, _, cookie := testRequest(t, h, "/PutString", "")
	_, _, cookie = testRequest(t, h, "/PutBool", cookie)

	_, body, cookie := testRequest(t, h, "/Clear", cookie)
	if body != "OK" {
		t.Fatalf("got %q: expected %q", body, "OK")
	}

	_, body, _ = testRequest(t, h, "/GetString", cookie)
	if body != ErrKeyNotFound.Error() {
		t.Fatalf("got %q: expected %q", body, ErrKeyNotFound.Error())
	}

	_, body, _ = testRequest(t, h, "/GetBool", cookie)
	if body != ErrKeyNotFound.Error() {
		t.Fatalf("got %q: expected %q", body, ErrKeyNotFound.Error())
	}
}
<commit_msg>[tests] GetInt should work with values that haven't been encoded<commit_after>package session

import (
	"net/http"
	"testing"

	"github.com/alexedwards/scs/mem/engine"
)

func TestString(t *testing.T) {
	m := Manage(engine.New())
	h := m(testServeMux)

	_, body, cookie := testRequest(t, h, "/PutString", "")
	if body != "OK" {
		t.Fatalf("got %q: expected %q", body, "OK")
	}

	_, body, _ = testRequest(t, h, "/GetString", cookie)
	if body != "lorem ipsum" {
		t.Fatalf("got %q: expected %q", body, "lorem ipsum")
	}

	_, body, cookie = testRequest(t, h, "/PopString", cookie)
	if body != "lorem ipsum" {
		t.Fatalf("got %q: expected %q", body, "lorem ipsum")
	}

	_, body, _ = testRequest(t, h, "/GetString", cookie)
	if body != ErrKeyNotFound.Error() {
		t.Fatalf("got %q: expected %q", body, ErrKeyNotFound.Error())
	}
}

func TestBool(t *testing.T) {
	m := Manage(engine.New())
	h := m(testServeMux)

	_, body, cookie := testRequest(t, h, "/PutBool", "")
	if body != "OK" {
		t.Fatalf("got %q: expected %q", body, "OK")
	}

	_, body, _ = testRequest(t, h, "/GetBool", cookie)
	if body != "true" {
		t.Fatalf("got %q: expected %q", body, "true")
	}

	_, body, cookie = testRequest(t, h, "/PopBool", cookie)
	if body != "true" {
		t.Fatalf("got %q: expected %q", body, "true")
	}

	_, body, _ = testRequest(t, h, "/GetBool", cookie)
	if body != ErrKeyNotFound.Error() {
		t.Fatalf("got %q: expected %q", body, ErrKeyNotFound.Error())
	}
}

func TestInt(t *testing.T) {
	m := Manage(engine.New())
	h := m(testServeMux)

	_, body, cookie := testRequest(t, h, "/PutInt", "")
	if body != "OK" {
		t.Fatalf("got %q: expected %q", body, "OK")
	}

	_, body, _ = testRequest(t, h, "/GetInt", cookie)
	if body != "12345" {
		t.Fatalf("got %q: expected %q", body, "12345")
	}

	_, body, cookie = testRequest(t, h, "/PopInt", cookie)
	if body != "12345" {
		t.Fatalf("got %q: expected %q", body, "12345")
	}

	_, body, _ = testRequest(t, h, "/GetInt", cookie)
	if body != ErrKeyNotFound.Error() {
		t.Fatalf("got %q: expected %q", body, ErrKeyNotFound.Error())
	}

	r := requestWithSession(new(http.Request), &session{values: make(map[string]interface{})})

	_ = PutInt(r, "test_int", 12345)
	i, _ := GetInt(r, "test_int")
	if i != 12345 {
		t.Fatalf("got %d: expected %d", i, 12345)
	}
}

func TestRemove(t *testing.T) {
	m := Manage(engine.New())
	h := m(testServeMux)

	_, _, cookie := testRequest(t, h, "/PutString", "")
	_, _, cookie = testRequest(t, h, "/PutBool", cookie)

	_, body, cookie := testRequest(t, h, "/RemoveString", cookie)
	if body != "OK" {
		t.Fatalf("got %q: expected %q", body, "OK")
	}

	_, body, _ = testRequest(t, h, "/GetString", cookie)
	if body != ErrKeyNotFound.Error() {
		t.Fatalf("got %q: expected %q", body, ErrKeyNotFound.Error())
	}

	_, body, _ = testRequest(t, h, "/GetBool", cookie)
	if body != "true" {
		t.Fatalf("got %q: expected %q", body, "true")
	}
}

func TestClear(t *testing.T) {
	m := Manage(engine.New())
	h := m(testServeMux)

	_, _, cookie := testRequest(t, h, "/PutString", "")
	_, _, cookie = testRequest(t, h, "/PutBool", cookie)

	_, body, cookie := testRequest(t, h, "/Clear", cookie)
	if body != "OK" {
		t.Fatalf("got %q: expected %q", body, "OK")
	}

	_, body, _ = testRequest(t, h, "/GetString", cookie)
	if body != ErrKeyNotFound.Error() {
		t.Fatalf("got %q: expected %q", body, ErrKeyNotFound.Error())
	}

	_, body, _ = testRequest(t, h, "/GetBool", cookie)
	if body != ErrKeyNotFound.Error() {
		t.Fatalf("got %q: expected %q", body, ErrKeyNotFound.Error())
	}
}
<|endoftext|>


<commit_before>package main

import (
	"bufio"
	"errors"
	"fmt"
	"io"
	"log"
	"os"
	"regexp"
	"sort"
	"strconv"
	"strings"
	"time"

	"github.com/jlaffaye/ftp"
)

// Loggerer ...
type Loggerer struct {
	logger ILogger
	err    error
}

// SetLogger sets objects logger
func (l *Loggerer) SetLogger(logger ILogger) {
	l.logger = logger
}

// Log logs the input text
func (l *Loggerer) Log(loglevel int, text ...interface{}) {
	if l.logger != nil {
		l.logger.Log(loglevel, text...)
	}
}

// Error ...
func (l *Loggerer) Error(text ...interface{}) {
	if l.logger != nil {
		l.logger.Log(Error, "ERROR: "+fmt.Sprint(text...))
	}
	if l.err == nil {
		l.err = errors.New(fmt.Sprint(text...))
	}
}

// ResetError ...
func (l *Loggerer) ResetError() {
	l.err = nil
}

// GetError ...
func (l *Loggerer) GetError() error {
	return l.err
}

type ftpConn struct {
	Loggerer
	conn      *ftp.ServerConn
	connected bool
}

func newFtpConn() *ftpConn {
	return &ftpConn{}
}

func (f *ftpConn) dial(addr string) {
	f.ResetError()
	conn, err := ftp.Dial(addr)
	if err != nil {
		f.Error(err)
		return
	}
	f.conn = conn
	f.connected = true
	f.Log(Debug, "Connected to "+addr)
}

func (f *ftpConn) quit() {
	if !f.connected {
		return
	}
	f.conn.Quit()
	f.connected = false
	f.Log(Debug, "Connection closed correctly")
}

func (f *ftpConn) login(user string, pass string) {
	if f.GetError() != nil {
		return
	}
	err := f.conn.Login(user, pass)
	if err != nil {
		f.Error(err)
		return
	}
	f.Log(Debug, "Logged in as "+user)
}

func (f *ftpConn) cwd() string {
	if f.GetError() != nil {
		return ""
	}
	cwd, err := f.conn.CurrentDir()
	if err != nil {
		f.Error(err)
		return ""
	}
	return cwd
}

func (f *ftpConn) cd(path string) {
	if f.GetError() != nil {
		return
	}
	err := f.conn.ChangeDir(path)
	if err != nil {
		f.Error(err)
		return
	}
}

func (f *ftpConn) cdup() {
	if f.GetError() != nil {
		return
	}
	f.conn.ChangeDirToParent()
}

func (f *ftpConn) ls(path string) (entries []*ftp.Entry) {
	if f.GetError() != nil {
		return
	}
	entries, err := f.conn.List(path)
	if err != nil {
		f.Error(err)
		return
	}
	return entries
}

func (f *ftpConn) walk(fl map[string]fileEntry) {
	if f.GetError() != nil {
		return
	}
	entries := f.ls("")
	cwd := f.cwd()
	newLine := pad(cwd, len(lastLine))
	fmt.Print(newLine + "\r")
	lastLine = cwd
	for _, element := range entries {
		switch element.Type {
		case ftp.EntryTypeFile:
			if acceptFileName(element.Name) {
				key := cwd + "/" + element.Name
				entry, fileExists := fl[key]
				if fileExists {
					// Old file with new date
					if !entry.Time.Equal(element.Time) {
						f.Log(Notice, "~ "+truncPad(key, 40, 'l')+" datetime changed")
						fl[key] = newFileEntry(element)
					} else if entry.Size != element.Size {
						f.Log(Notice, "~ "+truncPad(key, 40, 'l')+" size changed")
						fl[key] = newFileEntry(element)
					} else {
						entry.Found = true
						fl[key] = entry
					}
				} else {
					// New file
					f.Log(Notice, "+ "+truncPad(key, 40, 'l')+" new file")
					fl[key] = newFileEntry(element)
				}
			}
		case ftp.EntryTypeFolder:
			f.cd(element.Name)
			f.walk(fl)
			f.cdup()
		}
	}
}

func pad(s string, n int) string {
	if n > len(s) {
		return s + strings.Repeat(" ", n-len(s))
	}
	return s
}

func truncPad(s string, n int, side byte) string {
	if len(s) > n {
		if n >= 3 {
			return "..." + s[len(s)-n+3:len(s)]
		}
		return s[len(s)-n : len(s)]
	}
	if side == 'r' {
		return strings.Repeat(" ", n-len(s)) + s
	}
	return s + strings.Repeat(" ", n-len(s))
}

func acceptFileName(fileName string) bool {
	if fileMask.MatchString(fileName) {
		return true
	}
	return false
}

func newFileEntry(entry *ftp.Entry) fileEntry {
	file := fileEntry{}
	file.Name = entry.Name
	file.Size = entry.Size
	file.Time = entry.Time
	file.Found = true
	return file
}

type fileEntry struct {
	Name  string
	Size  uint64
	Time  time.Time
	Found bool
}

func (fe *fileEntry) pack() string {
	return fe.Name + "?|" + fmt.Sprintf("%v", fe.Size) + "?|" + fe.Time.String()
}

type tFileList struct {
	Loggerer
	file map[string]fileEntry
}

func newFileList() *tFileList {
	return &tFileList{file: map[string]fileEntry{}}
}

func (fl *tFileList) pack() string {
	output := []string{}
	for key, value := range fl.file {
		output = append(output, "?{"+key+"?}"+value.pack()+"\n")
	}
	sort.Strings(output)
	return strings.Join(output, "")
}

func (fl *tFileList) clean() {
	for key, value := range fl.file {
		if !value.Found {
			delete(fl.file, key)
			fl.Log(Notice, "- "+truncPad(key, 40, 'l')+" deleted")
		} else {
			value.Found = false
		}
	}
}

func (fl tFileList) String() string {
	return fl.pack()
}

func (fl *tFileList) save(filepath string) {
	file, err := os.Create(filepath)
	if err != nil {
		log.Panicln(err)
	}
	defer file.Close()

	_, err = io.Copy(file, strings.NewReader(fl.pack()))
	if err != nil {
		log.Panicln(err)
	}
}

func (fl *tFileList) load(filepath string) {
	fl.Log(Debug, "Loading \""+filepath+"\"...")
	file, err := os.Open(filepath)
	if err != nil {
		fl.Log(Error, "\""+filepath+"\" not found")
		return
	}
	defer file.Close()

	scanner := bufio.NewScanner(file)
	for scanner.Scan() {
		key, entry := fl.parseLine(scanner.Text())
		if key != "" {
			fl.file[key] = entry
		}
	}

	err = scanner.Err()
	if err != nil {
		log.Panicln(err)
	}
}

func (fl *tFileList) parseLine(line string) (string, fileEntry) {
	// "?{/AMEDIATEKA/ANIMALS_2/SER_05620.mxf?}SER_05620.mxf?|13114515508?|2017-03-17 14:39:39 +0000 UTC"
	if !regExpLine.MatchString(line) {
		fl.Log(Error, "Wrong input in file list ("+line+")")
		return "", fileEntry{}
	}
	matches := regExpLine.FindStringSubmatch(line)
	key := matches[1]
	entry := fileEntry{}
	entry.Name = matches[2]
	entrySize, err := strconv.Atoi(matches[3])
	entry.Size = uint64(entrySize)
	entry.Time, err = time.Parse("2006-01-02 15:04:05 +0000 UTC", matches[4])
	if err != nil {
		fl.Log(Error, err)
		return "", fileEntry{}
	}
	return key, entry
}

// ILogger ...
type ILogger interface {
	Log(loglevel int, text ...interface{})
}

type logger struct {
	writers []tWriter
}

type tWriter struct {
	writer   io.Writer
	loglevel int
}

func newLogger() *logger {
	return &logger{}
}

func (l *logger) addLogger(loglevel int, writer io.Writer) {
	l.writers = append(l.writers, tWriter{writer, loglevel})
}

func (l *logger) Log(loglevel int, text ...interface{}) {
	for _, writer := range l.writers {
		if loglevel&writer.loglevel != 0 {
			_, err := writer.writer.Write([]byte(time.Now().Format("2006-01-02 15:04:05") + " " + logLeveltoStr(loglevel) + ": " + fmt.Sprint(text...) + "\n"))
			if err != nil {
				fmt.Println(err)
			}
		}
	}
	if loglevel&Panic != 0 {
		panic(fmt.Sprint(text...))
	}
}

// LogLevel flags
const (
	Quiet = 0
	Panic = 1 << iota
	Error
	Warning
	Notice
	Info
	Debug
)

func logLevelLeq(loglevel int) int {
	return loglevel - 1 | loglevel
}

func logLeveltoStr(loglevel int) string {
	s := []string{}
	if loglevel&Panic != 0 {
		s = append(s, "PNC")
	}
	if loglevel&Error != 0 {
		s = append(s, "ERR")
	}
	if loglevel&Warning != 0 {
		s = append(s, "WRN")
	}
	if loglevel&Notice != 0 {
		s = append(s, "NTC")
	}
	if loglevel&Info != 0 {
		s = append(s, "INF")
	}
	if loglevel&Debug != 0 {
		s = append(s, "DBG")
	}
	return strings.Join(s, "|")
}

func newFileWriter(path string) *os.File {
	file, err := os.OpenFile(path, os.O_APPEND|os.O_CREATE|os.O_WRONLY, 0755)
	if err != nil {
		log.Panicln(err)
	}
	return file
}

// Global variables are set in private file.
// Ftp server address with port.
// var addr = ""
// var user = ""
// var pass = ""

var regExpLine = regexp.MustCompile(`\?\{(.*)\?\}(.*)\?\|(\d+)\?\|(.*)$`)
var logFilePath = "shuher.log"
var fileListPath = "shuherFileList.txt"
var watcherRootPath = "/AMEDIATEKA"
var fileMask = regexp.MustCompile(`^.*\.mxf$`)
var lastLine string
var longSleepTime = 15 * time.Minute
var shortSleepTime = 1 * time.Minute

func main() {
	// Create objects.
	fileWriter := newFileWriter(logFilePath)
	defer fileWriter.Close()
	logger := newLogger()
	logger.addLogger(logLevelLeq(Debug), fileWriter)
	logger.addLogger(logLevelLeq(Info), os.Stdout)
	ftpConn := newFtpConn()
	ftpConn.SetLogger(logger)
	fileList := newFileList()
	fileList.SetLogger(logger)
	// Load file list.
	fileList.load(fileListPath)
	// Properly close the connection on exit.
	defer ftpConn.quit()

	for {
		// Initialize the connection to the specified ftp server address.
		ftpConn.dial(addr)
		// Authenticate the client with specified user and password.
		ftpConn.login(user, pass)
		// Change directory to watcherRootPath.
		ftpConn.cd(watcherRootPath)
		// Walk the directory tree.
		if ftpConn.GetError() == nil {
			logger.Log(Info, "Looking for new files...")
			ftpConn.walk(fileList.file)
			fmt.Print(pad("", len(lastLine)) + "\r")
		}
		// Terminate the FTP connection.
		ftpConn.quit()
		// Remove deleted files from the fileList.
		fileList.clean()
		// Save new fileList.
		if ftpConn.GetError() == nil {
			fileList.save(fileListPath)
			// Wait for sleepTime before checking again.
			time.Sleep(longSleepTime)
		} else {
			// Wait for sleepTime before checking again.
			time.Sleep(shortSleepTime)
		}
	}
}
<commit_msg>Add mail logger<commit_after>package main

import (
	"bufio"
	"errors"
	"fmt"
	"io"
	"log"
	"os"
	"regexp"
	"sort"
	"strconv"
	"strings"
	"time"

	"github.com/jlaffaye/ftp"
	"github.com/malashin/pochta"
)

// Loggerer ...
type Loggerer struct {
	logger ILogger
	err    error
}

// SetLogger sets objects logger
func (l *Loggerer) SetLogger(logger ILogger) {
	l.logger = logger
}

// Log logs the input text
func (l *Loggerer) Log(loglevel int, text ...interface{}) {
	if l.logger != nil {
		l.logger.Log(loglevel, text...)
	}
}

// Error ...
func (l *Loggerer) Error(text ...interface{}) {
	if l.logger != nil {
		l.logger.Log(Error, "ERROR: "+fmt.Sprint(text...))
	}
	if l.err == nil {
		l.err = errors.New(fmt.Sprint(text...))
	}
}

// ResetError ...
func (l *Loggerer) ResetError() {
	l.err = nil
}

// GetError ...
func (l *Loggerer) GetError() error {
	return l.err
}

type ftpConn struct {
	Loggerer
	conn      *ftp.ServerConn
	connected bool
}

func newFtpConn() *ftpConn {
	return &ftpConn{}
}

func (f *ftpConn) dial(addr string) {
	f.ResetError()
	conn, err := ftp.Dial(addr)
	if err != nil {
		f.Error(err)
		return
	}
	f.conn = conn
	f.connected = true
	f.Log(Debug, "Connected to "+addr)
}

func (f *ftpConn) quit() {
	if !f.connected {
		return
	}
	f.conn.Quit()
	f.connected = false
	f.Log(Debug, "Connection closed correctly")
}

func (f *ftpConn) login(user string, pass string) {
	if f.GetError() != nil {
		return
	}
	err := f.conn.Login(user, pass)
	if err != nil {
		f.Error(err)
		return
	}
	f.Log(Debug, "Logged in as "+user)
}

func (f *ftpConn) cwd() string {
	if f.GetError() != nil {
		return ""
	}
	cwd, err := f.conn.CurrentDir()
	if err != nil {
		f.Error(err)
		return ""
	}
	return cwd
}

func (f *ftpConn) cd(path string) {
	if f.GetError() != nil {
		return
	}
	err := f.conn.ChangeDir(path)
	if err != nil {
		f.Error(err)
		return
	}
}

func (f *ftpConn) cdup() {
	if f.GetError() != nil {
		return
	}
	f.conn.ChangeDirToParent()
}

func (f *ftpConn) ls(path string) (entries []*ftp.Entry) {
	if f.GetError() != nil {
		return
	}
	entries, err := f.conn.List(path)
	if err != nil {
		f.Error(err)
		return
	}
	return entries
}

func (f *ftpConn) walk(fl map[string]fileEntry) {
	if f.GetError() != nil {
		return
	}
	entries := f.ls("")
	cwd := f.cwd()
	newLine := pad(cwd, len(lastLine))
	fmt.Print(newLine + "\r")
	lastLine = cwd
	for _, element := range entries {
		switch element.Type {
		case ftp.EntryTypeFile:
			if acceptFileName(element.Name) {
				key := cwd + "/" + element.Name
				entry, fileExists := fl[key]
				if fileExists {
					// Old file with new date
					if !entry.Time.Equal(element.Time) {
						f.Log(Notice, "~ "+truncPad(key, 40, 'l')+" datetime changed")
						fl[key] = newFileEntry(element)
					} else if entry.Size != element.Size {
						f.Log(Notice, "~ "+truncPad(key, 40, 'l')+" size changed")
						fl[key] = newFileEntry(element)
					} else {
						entry.Found = true
						fl[key] = entry
					}
				} else {
					// New file
					f.Log(Notice, "+ "+truncPad(key, 40, 'l')+" new file")
					fl[key] = newFileEntry(element)
				}
			}
		case ftp.EntryTypeFolder:
			f.cd(element.Name)
			f.walk(fl)
			f.cdup()
		}
	}
}

func pad(s string, n int) string {
	if n > len(s) {
		return s + strings.Repeat(" ", n-len(s))
	}
	return s
}

func truncPad(s string, n int, side byte) string {
	if len(s) > n {
		if n >= 3 {
			return "..." + s[len(s)-n+3:len(s)]
		}
		return s[len(s)-n : len(s)]
	}
	if side == 'r' {
		return strings.Repeat(" ", n-len(s)) + s
	}
	return s + strings.Repeat(" ", n-len(s))
}

func acceptFileName(fileName string) bool {
	if fileMask.MatchString(fileName) {
		return true
	}
	return false
}

func newFileEntry(entry *ftp.Entry) fileEntry {
	file := fileEntry{}
	file.Name = entry.Name
	file.Size = entry.Size
	file.Time = entry.Time
	file.Found = true
	return file
}

type fileEntry struct {
	Name  string
	Size  uint64
	Time  time.Time
	Found bool
}

func (fe *fileEntry) pack() string {
	return fe.Name + "?|" + fmt.Sprintf("%v", fe.Size) + "?|" + fe.Time.String()
}

type tFileList struct {
	Loggerer
	file map[string]fileEntry
}

func newFileList() *tFileList {
	return &tFileList{file: map[string]fileEntry{}}
}

func (fl *tFileList) pack() string {
	output := []string{}
	for key, value := range fl.file {
		output = append(output, "?{"+key+"?}"+value.pack()+"\n")
	}
	sort.Strings(output)
	return strings.Join(output, "")
}

func (fl *tFileList) clean() {
	for key, value := range fl.file {
		if !value.Found {
			delete(fl.file, key)
			fl.Log(Notice, "- "+truncPad(key, 40, 'l')+" deleted")
		} else {
			value.Found = false
		}
	}
}

func (fl tFileList) String() string {
	return fl.pack()
}

func (fl *tFileList) save(filepath string) {
	file, err := os.Create(filepath)
	if err != nil {
		log.Panicln(err)
	}
	defer file.Close()

	_, err = io.Copy(file, strings.NewReader(fl.pack()))
	if err != nil {
		log.Panicln(err)
	}
}

func (fl *tFileList) load(filepath string) {
	fl.Log(Debug, "Loading \""+filepath+"\"...")
	file, err := os.Open(filepath)
	if err != nil {
		fl.Log(Error, "\""+filepath+"\" not found")
		return
	}
	defer file.Close()

	scanner := bufio.NewScanner(file)
	for scanner.Scan() {
		key, entry := fl.parseLine(scanner.Text())
		if key != "" {
			fl.file[key] = entry
		}
	}

	err = scanner.Err()
	if err != nil {
		log.Panicln(err)
	}
}

func (fl *tFileList) parseLine(line string) (string, fileEntry) {
	// "?{/AMEDIATEKA/ANIMALS_2/SER_05620.mxf?}SER_05620.mxf?|13114515508?|2017-03-17 14:39:39 +0000 UTC"
	if !regExpLine.MatchString(line) {
		fl.Log(Error, "Wrong input in file list ("+line+")")
		return "", fileEntry{}
	}
	matches := regExpLine.FindStringSubmatch(line)
	key := matches[1]
	entry := fileEntry{}
	entry.Name = matches[2]
	entrySize, err := strconv.Atoi(matches[3])
	entry.Size = uint64(entrySize)
	entry.Time, err = time.Parse("2006-01-02 15:04:05 +0000 UTC", matches[4])
	if err != nil {
		fl.Log(Error, err)
		return "", fileEntry{}
	}
	return key, entry
}

// ILogger ...
type ILogger interface {
	Log(loglevel int, text ...interface{})
}

type logger struct {
	writers []tWriter
}

type tWriter struct {
	writer   io.Writer
	loglevel int
}

func newLogger() *logger {
	return &logger{}
}

func (l *logger) addLogger(loglevel int, writer io.Writer) {
	l.writers = append(l.writers, tWriter{writer, loglevel})
}

func (l *logger) Log(loglevel int, text ...interface{}) {
	for _, writer := range l.writers {
		if loglevel&writer.loglevel != 0 {
			_, err := writer.writer.Write([]byte(time.Now().Format("2006-01-02 15:04:05") + " " + logLeveltoStr(loglevel) + ": " + fmt.Sprint(text...) + "\n"))
			if err != nil {
				fmt.Println(err)
			}
		}
	}
	if loglevel&Panic != 0 {
		panic(fmt.Sprint(text...))
	}
}

// LogLevel flags
const (
	Quiet = 0
	Panic = 1 << iota
	Error
	Warning
	Notice
	Info
	Debug
)

func logLevelLeq(loglevel int) int {
	return loglevel - 1 | loglevel
}

func logLeveltoStr(loglevel int) string {
	s := []string{}
	if loglevel&Panic != 0 {
		s = append(s, "PNC")
	}
	if loglevel&Error != 0 {
		s = append(s, "ERR")
	}
	if loglevel&Warning != 0 {
		s = append(s, "WRN")
	}
	if loglevel&Notice != 0 {
		s = append(s, "NTC")
	}
	if loglevel&Info != 0 {
		s = append(s, "INF")
	}
	if loglevel&Debug != 0 {
		s = append(s, "DBG")
	}
	return strings.Join(s, "|")
}

func newFileWriter(path string) *os.File {
	file, err := os.OpenFile(path, os.O_APPEND|os.O_CREATE|os.O_WRONLY, 0755)
	if err != nil {
		log.Panicln(err)
	}
	return file
}

type TMailWriter struct {
	msg []string
}

func NewMailWriter() *TMailWriter {
	return &TMailWriter{}
}

func (m *TMailWriter) Write(p []byte) (n int, err error) {
	m.msg = append(m.msg, string(p))
	return 0, nil
}

func (m *TMailWriter) Send() {
	if len(m.msg) != 0 {
		body := strings.Join(m.msg, "")
		err := pochta.SendMail(smtpserver, auth, from, to, subject, body)
		if err != nil {
			log.Panicln(err)
		}
		m.msg = []string{}
	}
}

// Global variables are set in private file.
// Ftp server address with port.
// var addr = ""
// var user = ""
// var pass = ""

// Mail config
// var smtpserver = "" // with port
// var auth = pochta.LoginAuth("", "")
// var from = mail.Address{Name: "", Address: ""}
// var to = mail.Address{Name: "", Address: ""}
// var subject = ""

var regExpLine = regexp.MustCompile(`\?\{(.*)\?\}(.*)\?\|(\d+)\?\|(.*)$`)
var logFilePath = "shuher.log"
var fileListPath = "shuherFileList.txt"
var watcherRootPath = "/AMEDIATEKA"
var fileMask = regexp.MustCompile(`^.*\.mxf$`)
var lastLine string
var longSleepTime = 15 * time.Minute
var shortSleepTime = 1 * time.Minute

func main() {
	// Create objects.
	fileWriter := newFileWriter(logFilePath)
	defer fileWriter.Close()
	mailWriter := NewMailWriter()
	logger := newLogger()
	logger.addLogger(logLevelLeq(Debug), fileWriter)
	logger.addLogger(logLevelLeq(Notice), mailWriter)
	logger.addLogger(logLevelLeq(Info), os.Stdout)
	ftpConn := newFtpConn()
	ftpConn.SetLogger(logger)
	fileList := newFileList()
	fileList.SetLogger(logger)
	// Load file list.
	fileList.load(fileListPath)
	// Properly close the connection on exit.
	defer ftpConn.quit()

	for {
		// Initialize the connection to the specified ftp server address.
		ftpConn.dial(addr)
		// Authenticate the client with specified user and password.
		ftpConn.login(user, pass)
		// Change directory to watcherRootPath.
		ftpConn.cd(watcherRootPath)
		// Walk the directory tree.
		if ftpConn.GetError() == nil {
			logger.Log(Info, "Looking for new files...")
			ftpConn.walk(fileList.file)
			fmt.Print(pad("", len(lastLine)) + "\r")
		}
		// Terminate the FTP connection.
		ftpConn.quit()
		// Remove deleted files from the fileList.
		fileList.clean()
		mailWriter.Send()
		if ftpConn.GetError() == nil {
			// Save new fileList.
			fileList.save(fileListPath)
			// Wait for sleepTime before checking again.
			time.Sleep(longSleepTime)
		} else {
			// Wait for sleepTime before checking again.
			time.Sleep(shortSleepTime)
		}
	}
}
<|endoftext|>


<commit_before>package ghlib

/*  types for github API */

type GhLimitResourcesCore struct {
    Limit int `json:"limit"`
    Remaining int `json:"remaining"`
    reset int `json:"reset"`
}

type GhLimitResourcesSearch struct {
    Limit int `json:"limit"`
    Remaining int `json:"remaining"`
    reset int `json:"reset"`
}

type GhLimitResources struct {
    Core GhLimitResourcesCore `json:"core"`
    Search GhLimitResourcesSearch `json:"search"`
}

type GhLimitRate struct {
    Limit int `json:"limit"`
    Remaining int `json:"remaining"`
    reset int `json:"reset"`
}

type GhLimit struct {
    Resources GhLimitResources `json:"resources"`
    Rate GhLimitRate `json:"rate"`
}

<commit_msg>Added GhUser type<commit_after>package ghlib

/*  types for github API limit check */

type GhLimitResourcesCore struct {
    Limit int `json:"limit"`
    Remaining int `json:"remaining"`
    reset int `json:"reset"`
}

type GhLimitResourcesSearch struct {
    Limit int `json:"limit"`
    Remaining int `json:"remaining"`
    reset int `json:"reset"`
}

type GhLimitResources struct {
    Core GhLimitResourcesCore `json:"core"`
    Search GhLimitResourcesSearch `json:"search"`
}

type GhLimitRate struct {
    Limit int `json:"limit"`
    Remaining int `json:"remaining"`
    reset int `json:"reset"`
}

type GhLimit struct {
    Resources GhLimitResources `json:"resources"`
    Rate GhLimitRate `json:"rate"`
}

/*  Github API types */

type GhUser struct {
    Login string `json:"login"`
    Id int `json:"id"`
    Avatar_url string `json:"avatar_url"`
    Gravatar_id string `json:"gravatar_id"`
    Url string `json:"url"`
    Html_url string `json:"html_url"`
    Followers_url string `json:"followers_url"`
    Following_url string `json:"following_url"`
    Gists_url string `json:"gists_url"`
    Starred_url string `json:"starred_url"`
    Subscriptions_url string `json:"subscriptions_url"`
    Organizations_url string `json:"organizations_url"`
    Repos_url string `json:"repos_url"`
    Events_url string `json:"events_url"`
    Received_events_url string `json:"received_events_url"`
    Usertype string `json:"type"`
    Site_admin bool `json:"site_admin"`
    Name string `json:"name"`
    Company string `json:"company"`
    Blog string `json:"blog"`
    Location string `json:"location"`
    Email string "email"
    Hireable bool `json:"hireable"`
    Bio string `json:"bio"`
    Public_repos int `json:"public_repos"`
    Public_gists int `json:"public_gists"`
    Followers int `json:"followers"`
    Following int `json:"following"`
    Created_at string `json:"created_at"`
    Updated_at string `json:"updated_at"`
}

<|endoftext|>


<commit_before>package main

import (
	"fmt"
	"math"
	"os"
	"text/tabwriter"

	"github.com/GaryBoone/GoStats/stats"
	"github.com/topher200/baseutil"
)

type criterionCalculationFunction func(teams []Team) Score
type criterion struct {
	name      string                       // human readable name
	calculate criterionCalculationFunction // how to calculate the raw score
	filter    PlayerFilter                 // cull down to players that match
	weight    int                          // how much weight to give this score
	// worstCase is calculated at runtime to be the absolute worst score we can
	// see this criterion getting, calculated using random sampling
	worstCase Score
}

var criteriaToScore = [...]criterion{
	criterion{"matching baggages", baggagesMatch, nil, 10000, 0},
	criterion{"number of players", playerCountDifference, nil, 15, 0},
	criterion{"number of males", playerCountDifference, IsMale, 12, 0},
	criterion{"number of females", playerCountDifference, IsFemale, 12, 0},
	criterion{"average rating players", ratingDifference, nil, 8, 0},
	criterion{"average rating males", ratingDifference, IsMale, 7, 0},
	criterion{"average rating females", ratingDifference, IsFemale, 7, 0},
	criterion{"std dev of team player ratings", ratingStdDev, nil, 6, 0},
	criterion{"std dev of team male ratings", ratingStdDev, IsMale, 5, 0},
	criterion{"std dev of team female ratings", ratingStdDev, IsFemale, 5, 0},
}

func playerCountDifference(teams []Team) Score {
	teamLengths := make([]int, numTeams)
	for i, team := range teams {
		teamLengths[i] = len(team.players)
	}
	return Score(baseutil.StandardDeviationInt(teamLengths))
}

func ratingDifference(teams []Team) Score {
	teamAverageRatings := make([]float64, numTeams)
	for i, team := range teams {
		teamAverageRatings[i] = float64(AverageRating(team))
	}
	return Score(stats.StatsSampleStandardDeviation(teamAverageRatings))
}

func ratingStdDev(teams []Team) Score {
	teamRatingsStdDev := make([]float64, numTeams)
	for i, team := range teams {
		if len(team.players) < 2 {
			teamRatingsStdDev[i] = 0
			continue
		}
		playerRatings := make([]float64, len(team.players))
		for j, player := range team.players {
			playerRatings[j] = float64(player.rating)
		}
		teamRatingsStdDev[i] = stats.StatsSampleStandardDeviation(playerRatings)
	}
	return Score(stats.StatsSampleStandardDeviation(teamRatingsStdDev))
}

func baggagesMatch(teams []Team) Score {
	score := Score(0)
	for _, team := range teams {
		for _, player := range team.players {
			if !player.HasBaggage() {
				continue
			}
			_, err := FindPlayer(team.players, player.baggage)
			if err != nil {
				// Player desired a baggage, but they're not on the team
				score += 1
			}
		}
	}
	return score
}

func AverageRating(team Team) Score {
	if len(team.players) == 0 {
		return Score(0)
	}
	sum := float32(0.0)
	for _, player := range team.players {
		sum += player.rating
	}
	return Score(sum / float32(len(team.players)))
}

// analyze criterion by filtering the input teams and running the criterion's
// function
func (c criterion) analyze(teams []Team) (
	rawScore Score, normalizedScore Score, weightedScore Score) {
	filteredTeams := make([]Team, len(teams))
	for i, _ := range teams {
		filteredTeams[i].players = Filter(teams[i].players, c.filter)
	}

	rawScore = c.calculate(filteredTeams)
	if c.worstCase != 0 {
		normalizedScore = rawScore / c.worstCase
	} else {
		normalizedScore = rawScore
	}
	weightedScore = normalizedScore * Score(c.weight)
	return rawScore, normalizedScore, weightedScore
}

func maxScore(a, b Score) Score {
	if a > b {
		return a
	} else {
		return b
	}
}

// PopulateWorstCases calculates the worst case of each criterion.
//
// The function has the side effect of filling in the worstCase param for each
// criterion in criteriaToScore.
func PopulateWorstCases(solutions []Solution) {
	for _, solution := range solutions {
		_, rawScores := ScoreSolution(solution.players)
		for i, criterion := range criteriaToScore {
			if math.IsNaN(float64(rawScores[i])) {
				continue
			}
			criteriaToScore[i].worstCase = maxScore(
				criterion.worstCase, rawScores[i])
		}
	}
}

// Score a solution based on all known criteria.
//
// Returns the total score for the solution, as well as the raw score found for
// each of the criteriaToScore.
func ScoreSolution(players []Player) (totalScore Score, rawScores []Score) {
	teams := splitIntoTeams(players)
	rawScores = make([]Score, len(criteriaToScore))
	for i, criterion := range criteriaToScore {
		rawScore, _, weightedScore := criterion.analyze(teams)
		rawScores[i] = rawScore
		totalScore += weightedScore
	}
	return totalScore, rawScores
}

func PrintSolutionScoring(solution Solution) {
	teams := splitIntoTeams(solution.players)
	totalScore := Score(0)
	writer := new(tabwriter.Writer)
	writer.Init(os.Stdout, 0, 0, 1, ' ', 0)
	for _, criterion := range criteriaToScore {
		rawScore, normalizedScore, weightedScore := criterion.analyze(teams)
		totalScore += weightedScore
		fmt.Fprintf(
			writer,
			"Balancing %s.\tScore: %.02f\t(= normalized score %.02f * weight %d)\t(raw score %0.2f, worst case %.02f)\tRunning total: %.02f\n",
			criterion.name, weightedScore, normalizedScore, criterion.weight,
			rawScore, criterion.worstCase, totalScore)
	}
	writer.Flush()
}
<commit_msg>balance the rating of the top players<commit_after>package main

import (
	"fmt"
	"math"
	"os"
	"sort"
	"text/tabwriter"

	"github.com/GaryBoone/GoStats/stats"
	"github.com/topher200/baseutil"
)

type criterionCalculationFunction func(teams []Team) Score
type criterion struct {
	name      string                       // human readable name
	calculate criterionCalculationFunction // how to calculate the raw score
	filter    PlayerFilter                 // cull down to players that match
	// numPlayers reduces the amount of players we analyze from each team.
	// Sometimes used to just grab the top players on the team, for example.
	// Ignored if 0.
	numPlayers int
	weight     int // how much weight to give this score
	// worstCase is calculated at runtime to be the absolute worst score we can
	// see this criterion getting, calculated using random sampling
	worstCase Score
}

var criteriaToScore = [...]criterion{
	criterion{"matching baggages", baggagesMatch, nil, 0, 10000, 0},
	criterion{"number of players", playerCountDifference, nil, 0, 15, 0},
	criterion{"number of males", playerCountDifference, IsMale, 0, 12, 0},
	criterion{"number of females", playerCountDifference, IsFemale, 0, 12, 0},
	criterion{"average rating players", ratingDifference, nil, 0, 8, 0},
	criterion{"average rating males", ratingDifference, IsMale, 0, 7, 0},
	criterion{"average rating females", ratingDifference, IsFemale, 0, 7, 0},
	criterion{"average rating top players", ratingDifference, nil, 3, 20, 0},
	criterion{"average rating top males", ratingDifference, IsMale, 3, 19, 0},
	criterion{"average rating top females", ratingDifference, IsFemale, 19, 7, 0},
	criterion{"std dev of team player ratings", ratingStdDev, nil, 0, 6, 0},
	criterion{"std dev of team male ratings", ratingStdDev, IsMale, 0, 5, 0},
	criterion{"std dev of team female ratings", ratingStdDev, IsFemale, 0, 5, 0},
}

func playerCountDifference(teams []Team) Score {
	teamLengths := make([]int, numTeams)
	for i, team := range teams {
		teamLengths[i] = len(team.players)
	}
	return Score(baseutil.StandardDeviationInt(teamLengths))
}

func ratingDifference(teams []Team) Score {
	teamAverageRatings := make([]float64, numTeams)
	for i, team := range teams {
		teamAverageRatings[i] = float64(AverageRating(team))
	}
	return Score(stats.StatsSampleStandardDeviation(teamAverageRatings))
}

func ratingStdDev(teams []Team) Score {
	teamRatingsStdDev := make([]float64, numTeams)
	for i, team := range teams {
		if len(team.players) < 2 {
			teamRatingsStdDev[i] = 0
			continue
		}
		playerRatings := make([]float64, len(team.players))
		for j, player := range team.players {
			playerRatings[j] = float64(player.rating)
		}
		teamRatingsStdDev[i] = stats.StatsSampleStandardDeviation(playerRatings)
	}
	return Score(stats.StatsSampleStandardDeviation(teamRatingsStdDev))
}

func baggagesMatch(teams []Team) Score {
	score := Score(0)
	for _, team := range teams {
		for _, player := range team.players {
			if !player.HasBaggage() {
				continue
			}
			_, err := FindPlayer(team.players, player.baggage)
			if err != nil {
				// Player desired a baggage, but they're not on the team
				score += 1
			}
		}
	}
	return score
}

func AverageRating(team Team) Score {
	if len(team.players) == 0 {
		return Score(0)
	}
	sum := float32(0.0)
	for _, player := range team.players {
		sum += player.rating
	}
	return Score(sum / float32(len(team.players)))
}

// analyze criterion by filtering the input teams and running the criterion's
// function
func (c criterion) analyze(teams []Team) (
	rawScore Score, normalizedScore Score, weightedScore Score) {
	filteredTeams := make([]Team, len(teams))
	for i, _ := range teams {
		players := Filter(teams[i].players, c.filter)
		// If the max num players to run this criterion on is set and we have at
		// least that many players, filter out all but the top ones
		if c.numPlayers > 0 && len(players) > c.numPlayers {
			sort.Sort(ByRating(players))
			players = players[:c.numPlayers]
		}
		filteredTeams[i].players = players
	}

	rawScore = c.calculate(filteredTeams)
	if c.worstCase != 0 {
		normalizedScore = rawScore / c.worstCase
	} else {
		normalizedScore = rawScore
	}
	weightedScore = normalizedScore * Score(c.weight)
	return rawScore, normalizedScore, weightedScore
}

func maxScore(a, b Score) Score {
	if a > b {
		return a
	} else {
		return b
	}
}

// PopulateWorstCases calculates the worst case of each criterion.
//
// The function has the side effect of filling in the worstCase param for each
// criterion in criteriaToScore.
func PopulateWorstCases(solutions []Solution) {
	for _, solution := range solutions {
		_, rawScores := ScoreSolution(solution.players)
		for i, criterion := range criteriaToScore {
			if math.IsNaN(float64(rawScores[i])) {
				continue
			}
			criteriaToScore[i].worstCase = maxScore(
				criterion.worstCase, rawScores[i])
		}
	}
}

// Score a solution based on all known criteria.
//
// Returns the total score for the solution, as well as the raw score found for
// each of the criteriaToScore.
func ScoreSolution(players []Player) (totalScore Score, rawScores []Score) {
	teams := splitIntoTeams(players)
	rawScores = make([]Score, len(criteriaToScore))
	for i, criterion := range criteriaToScore {
		rawScore, _, weightedScore := criterion.analyze(teams)
		rawScores[i] = rawScore
		totalScore += weightedScore
	}
	return totalScore, rawScores
}

func PrintSolutionScoring(solution Solution) {
	teams := splitIntoTeams(solution.players)
	totalScore := Score(0)
	writer := new(tabwriter.Writer)
	writer.Init(os.Stdout, 0, 0, 1, ' ', 0)
	for _, criterion := range criteriaToScore {
		rawScore, normalizedScore, weightedScore := criterion.analyze(teams)
		totalScore += weightedScore
		fmt.Fprintf(
			writer,
			"Balancing %s.\tScore: %.02f\t(= normalized score %.02f * weight %d)\t(raw score %0.2f, worst case %.02f)\tRunning total: %.02f\n",
			criterion.name, weightedScore, normalizedScore, criterion.weight,
			rawScore, criterion.worstCase, totalScore)
	}
	writer.Flush()
}
<|endoftext|>


<commit_before>package main

import (
	"fmt"
	"net/http"
)

func handler(w http.ResponseWriter, r *http.Request) {
	fmt.Fprintf(w, "Hiya, there poop!")
}

func main() {
	http.HandleFunc("/", handler)
	http.ListenAndServe(":8081", nil)
}

<commit_msg>still testing<commit_after>package main

import (
	"fmt"
	"net/http"
)

func handler(w http.ResponseWriter, r *http.Request) {
	fmt.Fprintf(w, "Hiyaaaaa, there poop!")
}

func main() {
	http.HandleFunc("/", handler)
	http.ListenAndServe(":8081", nil)
}

<|endoftext|>


<commit_before>package blockexplorer

import (
	"path/filepath"
	"testing"

	"github.com/NebulousLabs/Sia/build"
	"github.com/NebulousLabs/Sia/modules"
	"github.com/NebulousLabs/Sia/modules/consensus"
	"github.com/NebulousLabs/Sia/modules/gateway"
	"github.com/NebulousLabs/Sia/modules/miner"
	"github.com/NebulousLabs/Sia/modules/transactionpool"
	"github.com/NebulousLabs/Sia/modules/wallet"
)

// Explorer tester struct is the helper object for explorer
// testing. It holds the helper modules for its testing
type explorerTester struct {
	cs      *consensus.State
	gateway modules.Gateway
	miner   modules.Miner
	tpool   modules.TransactionPool
	wallet  modules.Wallet

	explorer *BlockExplorer

	csUpdateChan     <-chan struct{}
	beUpdateChan     <-chan struct{}
	tpoolUpdateChan  <-chan struct{}
	minerUpdateChan  <-chan struct{}
	walletUpdateChan <-chan struct{}

	t *testing.T
}

// csUpdateWait blocks until a consensus update has propagated to all
// modules.
func (et *explorerTester) csUpdateWait() {
	<-et.csUpdateChan
	<-et.beUpdateChan
	et.tpUpdateWait()
}

// tpUpdateWait blocks until a transaction pool update has propagated to all
// modules.
func (ht *explorerTester) tpUpdateWait() {
	<-ht.tpoolUpdateChan
	<-ht.minerUpdateChan
	<-ht.walletUpdateChan
}

func createExplorerTester(name string, t *testing.T) *explorerTester {
	testdir := build.TempDir(modules.HostDir, name)

	// Create the modules
	g, err := gateway.New(":0", filepath.Join(testdir, modules.GatewayDir))
	if err != nil {
		t.Fatal(err)
	}
	cs, err := consensus.New(g, filepath.Join(testdir, modules.ConsensusDir))
	if err != nil {
		t.Fatal(err)
	}
	tp, err := transactionpool.New(cs, g)
	if err != nil {
		t.Fatal(err)
	}
	w, err := wallet.New(cs, tp, filepath.Join(testdir, modules.WalletDir))
	if err != nil {
		t.Fatal(err)
	}
	m, err := miner.New(cs, tp, w)
	if err != nil {
		t.Fatal(err)
	}
	be, err := New(cs)
	if err != nil {
		t.Fatal(err)
	}

	et := &explorerTester{
		cs:      cs,
		gateway: g,
		miner:   m,
		tpool:   tp,
		wallet:  w,

		explorer: be,

		csUpdateChan:     cs.ConsensusSetNotify(),
		beUpdateChan:     be.BlockExplorerNotify(),
		tpoolUpdateChan:  tp.TransactionPoolNotify(),
		minerUpdateChan:  m.MinerNotify(),
		walletUpdateChan: w.WalletNotify(),

		t: t,
	}
	et.csUpdateWait()
	return et
}
<commit_msg>renamed ht to et in blockexplorer tests<commit_after>package blockexplorer

import (
	"path/filepath"
	"testing"

	"github.com/NebulousLabs/Sia/build"
	"github.com/NebulousLabs/Sia/modules"
	"github.com/NebulousLabs/Sia/modules/consensus"
	"github.com/NebulousLabs/Sia/modules/gateway"
	"github.com/NebulousLabs/Sia/modules/miner"
	"github.com/NebulousLabs/Sia/modules/transactionpool"
	"github.com/NebulousLabs/Sia/modules/wallet"
)

// Explorer tester struct is the helper object for explorer
// testing. It holds the helper modules for its testing
type explorerTester struct {
	cs      *consensus.State
	gateway modules.Gateway
	miner   modules.Miner
	tpool   modules.TransactionPool
	wallet  modules.Wallet

	explorer *BlockExplorer

	csUpdateChan     <-chan struct{}
	beUpdateChan     <-chan struct{}
	tpoolUpdateChan  <-chan struct{}
	minerUpdateChan  <-chan struct{}
	walletUpdateChan <-chan struct{}

	t *testing.T
}

// csUpdateWait blocks until a consensus update has propagated to all
// modules.
func (et *explorerTester) csUpdateWait() {
	<-et.csUpdateChan
	<-et.beUpdateChan
	et.tpUpdateWait()
}

// tpUpdateWait blocks until a transaction pool update has propagated to all
// modules.
func (et *explorerTester) tpUpdateWait() {
	<-et.tpoolUpdateChan
	<-et.minerUpdateChan
	<-et.walletUpdateChan
}

func createExplorerTester(name string, t *testing.T) *explorerTester {
	testdir := build.TempDir(modules.HostDir, name)

	// Create the modules
	g, err := gateway.New(":0", filepath.Join(testdir, modules.GatewayDir))
	if err != nil {
		t.Fatal(err)
	}
	cs, err := consensus.New(g, filepath.Join(testdir, modules.ConsensusDir))
	if err != nil {
		t.Fatal(err)
	}
	tp, err := transactionpool.New(cs, g)
	if err != nil {
		t.Fatal(err)
	}
	w, err := wallet.New(cs, tp, filepath.Join(testdir, modules.WalletDir))
	if err != nil {
		t.Fatal(err)
	}
	m, err := miner.New(cs, tp, w)
	if err != nil {
		t.Fatal(err)
	}
	be, err := New(cs)
	if err != nil {
		t.Fatal(err)
	}

	et := &explorerTester{
		cs:      cs,
		gateway: g,
		miner:   m,
		tpool:   tp,
		wallet:  w,

		explorer: be,

		csUpdateChan:     cs.ConsensusSetNotify(),
		beUpdateChan:     be.BlockExplorerNotify(),
		tpoolUpdateChan:  tp.TransactionPoolNotify(),
		minerUpdateChan:  m.MinerNotify(),
		walletUpdateChan: w.WalletNotify(),

		t: t,
	}
	et.csUpdateWait()
	return et
}
<|endoftext|>