<commit_before>// Naive implementation of the Mahalanobis distance using go.matrix
// (https://en.wikipedia.org/wiki/Mahalanobis_distance)
//
// This is me learning Go, it's probably broken, don't use it.
//
// Example:
//
//      package main
//
//      import (
//          "fmt"
//          "github.com/skelterjohn/go.matrix"
//          "github.com/ant0ine/go.mahalanobis"
//      )
//
//      func main() {
//
//          points, err := matrix.ParseMatlab("[1 4 3 4;4 2 3 4]")
//          if err != nil {
//          panic(err)
//          }
//          fmt.Println("4 points:\n", points)
//
//          target, err := matrix.ParseMatlab("[3;4]")
//          if err != nil {
//              panic(err)
//          }
//          fmt.Println("the target point:\n", target)
//
//          distance, err := mahalanobis.Distance(points, target)
//          if err != nil {
//              panic(err)
//          }
//          fmt.Println("Mahalanobis distance=", distance)
//      }
package mahalanobis

import (
	//    "fmt"
	"github.com/skelterjohn/go.matrix"
	"math"
)

// Given a set a points, return the mean vector.
func MeanVector(points *matrix.DenseMatrix) *matrix.DenseMatrix {
	mean := matrix.Zeros(points.Rows(), 1)
	for i := 0; i < points.Rows(); i++ {
		sum := 0.0
		for j := 0; j < points.Cols(); j++ {
			sum += points.Get(i, j)
		}
		mean.Set(i, 0, sum/float64(points.Cols()))
	}
	return mean
}

func sample_covariance_matrix(points, mean *matrix.DenseMatrix) *matrix.DenseMatrix {
	dim := points.Rows()
	cov := matrix.Zeros(dim, dim)
	for i := 0; i < dim; i++ {
		for j := 0; j < dim; j++ {
			if i > j {
				// symetric matrix
				continue
			}
			// TODO in go routines ?
			sum := 0.0
			for k := 0; k < points.Cols(); k++ {
				sum += (points.Get(i, k) - mean.Get(i, 0)) * (points.Get(j, k) - mean.Get(j, 0))
			}

			// this is the sample covariance, divide by (N - 1)
			covariance := sum / (float64(points.Cols() - 1))

			cov.Set(i, j, covariance)
			// symetric matrix
			cov.Set(j, i, covariance)

		}
	}
	return cov
}

// Return the covariance matrix for this set of points (sample covariance is used)
func CovarianceMatrix(points *matrix.DenseMatrix) *matrix.DenseMatrix {
	mean := MeanVector(points)
	return sample_covariance_matrix(points, mean)
}

// Return the square of the Mahalanobis distance
func DistanceSquare(points, target *matrix.DenseMatrix) (float64, error) {

	// TODO check the dimensions

	mean := MeanVector(points)
	//fmt.Println("mean:\n", mean)

	delta := target.Copy()
	delta.SubtractDense(mean)
	//fmt.Println("delta:\n", delta)

	cov := sample_covariance_matrix(points, mean)
	//fmt.Println("covariance:\n", cov)

	inv, err := cov.Inverse()
	if err != nil {
		return 0, err // XXX
	}
	//fmt.Println("inverse covariance:\n", inv)

	product1, err := inv.TimesDense(delta)
	if err != nil {
		return 0, err // XXX
	}
	delta_t := delta.Transpose()
	product2, err := delta_t.TimesDense(product1)
	if err != nil {
		return 0, err // XXX
	}

	return product2.Get(0, 0), nil
}

// Return the Mahalanobis distance
func Distance(points, target *matrix.DenseMatrix) (float64, error) {
	square, err := DistanceSquare(points, target)
	if err != nil {
		return 0, err // XXX
	}
	return math.Sqrt(square), nil

}
<commit_msg>-1 as a return value in addition to the error<commit_after>// Naive implementation of the Mahalanobis distance using go.matrix
// (https://en.wikipedia.org/wiki/Mahalanobis_distance)
//
// This is me learning Go, it's probably broken, don't use it.
//
// Example:
//
//      package main
//
//      import (
//          "fmt"
//          "github.com/skelterjohn/go.matrix"
//          "github.com/ant0ine/go.mahalanobis"
//      )
//
//      func main() {
//
//          points, err := matrix.ParseMatlab("[1 4 3 4;4 2 3 4]")
//          if err != nil {
//          panic(err)
//          }
//          fmt.Println("4 points:\n", points)
//
//          target, err := matrix.ParseMatlab("[3;4]")
//          if err != nil {
//              panic(err)
//          }
//          fmt.Println("the target point:\n", target)
//
//          distance, err := mahalanobis.Distance(points, target)
//          if err != nil {
//              panic(err)
//          }
//          fmt.Println("Mahalanobis distance=", distance)
//      }
package mahalanobis

import (
	//    "fmt"
	"github.com/skelterjohn/go.matrix"
	"math"
)

// Given a set a points, return the mean vector.
func MeanVector(points *matrix.DenseMatrix) *matrix.DenseMatrix {
	mean := matrix.Zeros(points.Rows(), 1)
	for i := 0; i < points.Rows(); i++ {
		sum := 0.0
		for j := 0; j < points.Cols(); j++ {
			sum += points.Get(i, j)
		}
		mean.Set(i, 0, sum/float64(points.Cols()))
	}
	return mean
}

func sample_covariance_matrix(points, mean *matrix.DenseMatrix) *matrix.DenseMatrix {
	dim := points.Rows()
	cov := matrix.Zeros(dim, dim)
	for i := 0; i < dim; i++ {
		for j := 0; j < dim; j++ {
			if i > j {
				// symetric matrix
				continue
			}
			// TODO in go routines ?
			sum := 0.0
			for k := 0; k < points.Cols(); k++ {
				sum += (points.Get(i, k) - mean.Get(i, 0)) * (points.Get(j, k) - mean.Get(j, 0))
			}

			// this is the sample covariance, divide by (N - 1)
			covariance := sum / (float64(points.Cols() - 1))

			cov.Set(i, j, covariance)
			// symetric matrix
			cov.Set(j, i, covariance)

		}
	}
	return cov
}

// Return the covariance matrix for this set of points (sample covariance is used)
func CovarianceMatrix(points *matrix.DenseMatrix) *matrix.DenseMatrix {
	mean := MeanVector(points)
	return sample_covariance_matrix(points, mean)
}

// Return the square of the Mahalanobis distance
func DistanceSquare(points, target *matrix.DenseMatrix) (float64, error) {

	// TODO check the dimensions

	mean := MeanVector(points)
	//fmt.Println("mean:\n", mean)

	delta := target.Copy()
	delta.SubtractDense(mean)
	//fmt.Println("delta:\n", delta)

	cov := sample_covariance_matrix(points, mean)
	//fmt.Println("covariance:\n", cov)

	inv, err := cov.Inverse()
	if err != nil {
		return -1, err
	}
	//fmt.Println("inverse covariance:\n", inv)

	product1, err := inv.TimesDense(delta)
	if err != nil {
		return -1, err
	}
	delta_t := delta.Transpose()
	product2, err := delta_t.TimesDense(product1)
	if err != nil {
		return -1, err
	}

	return product2.Get(0, 0), nil
}

// Return the Mahalanobis distance
func Distance(points, target *matrix.DenseMatrix) (float64, error) {
	square, err := DistanceSquare(points, target)
	if err != nil {
		return -1, err
	}
	return math.Sqrt(square), nil

}
<|endoftext|>


<commit_before>package nsqd

import (
	"bytes"
	"encoding/json"
	"net"
	"os"
	"strconv"
	"time"

	"github.com/nsqio/go-nsq"
	"github.com/nsqio/nsq/internal/version"
)

func connectCallback(n *NSQD, hostname string, syncTopicChan chan *lookupPeer) func(*lookupPeer) {
	return func(lp *lookupPeer) {
		ci := make(map[string]interface{})
		ci["version"] = version.Binary
		ci["tcp_port"] = n.RealTCPAddr().Port
		ci["http_port"] = n.RealHTTPAddr().Port
		ci["hostname"] = hostname
		ci["broadcast_address"] = n.getOpts().BroadcastAddress

		cmd, err := nsq.Identify(ci)
		if err != nil {
			lp.Close()
			return
		}
		resp, err := lp.Command(cmd)
		if err != nil {
			n.logf(LOG_ERROR, "LOOKUPD(%s): %s - %s", lp, cmd, err)
		} else if bytes.Equal(resp, []byte("E_INVALID")) {
			n.logf(LOG_INFO, "LOOKUPD(%s): lookupd returned %s", lp, resp)
		} else {
			err = json.Unmarshal(resp, &lp.Info)
			if err != nil {
				n.logf(LOG_ERROR, "LOOKUPD(%s): parsing response - %s", lp, resp)
			} else {
				n.logf(LOG_INFO, "LOOKUPD(%s): peer info %+v", lp, lp.Info)
				if lp.Info.BroadcastAddress == "" {
					n.logf(LOG_ERROR, "LOOKUPD(%s): no broadcast address", lp)
				}
			}
		}

		go func() {
			syncTopicChan <- lp
		}()
	}
}

func (n *NSQD) lookupLoop() {
	var lookupPeers []*lookupPeer
	var lookupAddrs []string
	syncTopicChan := make(chan *lookupPeer)
	connect := true

	hostname, err := os.Hostname()
	if err != nil {
		n.logf(LOG_FATAL, "failed to get hostname - %s", err)
		os.Exit(1)
	}

	// for announcements, lookupd determines the host automatically
	ticker := time.Tick(15 * time.Second)
	for {
		if connect {
			for _, host := range n.getOpts().NSQLookupdTCPAddresses {
				if in(host, lookupAddrs) {
					continue
				}
				n.logf(LOG_INFO, "LOOKUP(%s): adding peer", host)
				lookupPeer := newLookupPeer(host, n.getOpts().MaxBodySize, n.logf,
					connectCallback(n, hostname, syncTopicChan))
				lookupPeer.Command(nil) // start the connection
				lookupPeers = append(lookupPeers, lookupPeer)
				lookupAddrs = append(lookupAddrs, host)
			}
			n.lookupPeers.Store(lookupPeers)
			connect = false
		}

		select {
		case <-ticker:
			// send a heartbeat and read a response (read detects closed conns)
			for _, lookupPeer := range lookupPeers {
				n.logf(LOG_DEBUG, "LOOKUPD(%s): sending heartbeat", lookupPeer)
				cmd := nsq.Ping()
				_, err := lookupPeer.Command(cmd)
				if err != nil {
					n.logf(LOG_ERROR, "LOOKUPD(%s): %s - %s", lookupPeer, cmd, err)
				}
			}
		case val := <-n.notifyChan:
			var cmd *nsq.Command
			var branch string

			switch val.(type) {
			case *Channel:
				// notify all nsqlookupds that a new channel exists, or that it's removed
				branch = "channel"
				channel := val.(*Channel)
				if channel.Exiting() == true {
					cmd = nsq.UnRegister(channel.topicName, channel.name)
				} else {
					cmd = nsq.Register(channel.topicName, channel.name)
				}
			case *Topic:
				// notify all nsqlookupds that a new topic exists, or that it's removed
				branch = "topic"
				topic := val.(*Topic)
				if topic.Exiting() == true {
					cmd = nsq.UnRegister(topic.name, "")
				} else {
					cmd = nsq.Register(topic.name, "")
				}
			}

			for _, lookupPeer := range lookupPeers {
				n.logf(LOG_INFO, "LOOKUPD(%s): %s %s", lookupPeer, branch, cmd)
				_, err := lookupPeer.Command(cmd)
				if err != nil {
					n.logf(LOG_ERROR, "LOOKUPD(%s): %s - %s", lookupPeer, cmd, err)
				}
			}
		case lookupPeer := <-syncTopicChan:
			var commands []*nsq.Command
			// build all the commands first so we exit the lock(s) as fast as possible
			n.RLock()
			for _, topic := range n.topicMap {
				topic.RLock()
				if len(topic.channelMap) == 0 {
					commands = append(commands, nsq.Register(topic.name, ""))
				} else {
					for _, channel := range topic.channelMap {
						commands = append(commands, nsq.Register(channel.topicName, channel.name))
					}
				}
				topic.RUnlock()
			}
			n.RUnlock()

			for _, cmd := range commands {
				n.logf(LOG_INFO, "LOOKUPD(%s): %s", lookupPeer, cmd)
				_, err := lookupPeer.Command(cmd)
				if err != nil {
					n.logf(LOG_ERROR, "LOOKUPD(%s): %s - %s", lookupPeer, cmd, err)
					break
				}
			}
		case <-n.optsNotificationChan:
			var tmpPeers []*lookupPeer
			var tmpAddrs []string
			for _, lp := range lookupPeers {
				if in(lp.addr, n.getOpts().NSQLookupdTCPAddresses) {
					tmpPeers = append(tmpPeers, lp)
					tmpAddrs = append(tmpAddrs, lp.addr)
					continue
				}
				n.logf(LOG_INFO, "LOOKUP(%s): removing peer", lp)
				lp.Close()
			}
			lookupPeers = tmpPeers
			lookupAddrs = tmpAddrs
			connect = true
		case <-n.exitChan:
			goto exit
		}
	}

exit:
	n.logf(LOG_INFO, "LOOKUP: closing")
}

func in(s string, lst []string) bool {
	for _, v := range lst {
		if s == v {
			return true
		}
	}
	return false
}

func (n *NSQD) lookupdHTTPAddrs() []string {
	var lookupHTTPAddrs []string
	lookupPeers := n.lookupPeers.Load()
	if lookupPeers == nil {
		return nil
	}
	for _, lp := range lookupPeers.([]*lookupPeer) {
		if len(lp.Info.BroadcastAddress) <= 0 {
			continue
		}
		addr := net.JoinHostPort(lp.Info.BroadcastAddress, strconv.Itoa(lp.Info.HTTPPort))
		lookupHTTPAddrs = append(lookupHTTPAddrs, addr)
	}
	return lookupHTTPAddrs
}
<commit_msg>fix full topic sync accumulation<commit_after>package nsqd

import (
	"bytes"
	"encoding/json"
	"net"
	"os"
	"strconv"
	"time"

	"github.com/nsqio/go-nsq"
	"github.com/nsqio/nsq/internal/version"
)

func connectCallback(n *NSQD, hostname string, syncTopicChan chan *lookupPeer) func(*lookupPeer) {
	return func(lp *lookupPeer) {
		ci := make(map[string]interface{})
		ci["version"] = version.Binary
		ci["tcp_port"] = n.RealTCPAddr().Port
		ci["http_port"] = n.RealHTTPAddr().Port
		ci["hostname"] = hostname
		ci["broadcast_address"] = n.getOpts().BroadcastAddress

		cmd, err := nsq.Identify(ci)
		if err != nil {
			lp.Close()
			return
		}
		resp, err := lp.Command(cmd)
		if err != nil {
			n.logf(LOG_ERROR, "LOOKUPD(%s): %s - %s", lp, cmd, err)
			return
		} else if bytes.Equal(resp, []byte("E_INVALID")) {
			n.logf(LOG_INFO, "LOOKUPD(%s): lookupd returned %s", lp, resp)
		} else {
			err = json.Unmarshal(resp, &lp.Info)
			if err != nil {
				n.logf(LOG_ERROR, "LOOKUPD(%s): parsing response - %s", lp, resp)
			} else {
				n.logf(LOG_INFO, "LOOKUPD(%s): peer info %+v", lp, lp.Info)
				if lp.Info.BroadcastAddress == "" {
					n.logf(LOG_ERROR, "LOOKUPD(%s): no broadcast address", lp)
				}
			}
		}

		go func() {
			syncTopicChan <- lp
		}()
	}
}

func (n *NSQD) lookupLoop() {
	var lookupPeers []*lookupPeer
	var lookupAddrs []string
	syncTopicChan := make(chan *lookupPeer)
	connect := true

	hostname, err := os.Hostname()
	if err != nil {
		n.logf(LOG_FATAL, "failed to get hostname - %s", err)
		os.Exit(1)
	}

	// for announcements, lookupd determines the host automatically
	ticker := time.Tick(15 * time.Second)
	for {
		if connect {
			for _, host := range n.getOpts().NSQLookupdTCPAddresses {
				if in(host, lookupAddrs) {
					continue
				}
				n.logf(LOG_INFO, "LOOKUP(%s): adding peer", host)
				lookupPeer := newLookupPeer(host, n.getOpts().MaxBodySize, n.logf,
					connectCallback(n, hostname, syncTopicChan))
				lookupPeer.Command(nil) // start the connection
				lookupPeers = append(lookupPeers, lookupPeer)
				lookupAddrs = append(lookupAddrs, host)
			}
			n.lookupPeers.Store(lookupPeers)
			connect = false
		}

		select {
		case <-ticker:
			// send a heartbeat and read a response (read detects closed conns)
			for _, lookupPeer := range lookupPeers {
				n.logf(LOG_DEBUG, "LOOKUPD(%s): sending heartbeat", lookupPeer)
				cmd := nsq.Ping()
				_, err := lookupPeer.Command(cmd)
				if err != nil {
					n.logf(LOG_ERROR, "LOOKUPD(%s): %s - %s", lookupPeer, cmd, err)
				}
			}
		case val := <-n.notifyChan:
			var cmd *nsq.Command
			var branch string

			switch val.(type) {
			case *Channel:
				// notify all nsqlookupds that a new channel exists, or that it's removed
				branch = "channel"
				channel := val.(*Channel)
				if channel.Exiting() == true {
					cmd = nsq.UnRegister(channel.topicName, channel.name)
				} else {
					cmd = nsq.Register(channel.topicName, channel.name)
				}
			case *Topic:
				// notify all nsqlookupds that a new topic exists, or that it's removed
				branch = "topic"
				topic := val.(*Topic)
				if topic.Exiting() == true {
					cmd = nsq.UnRegister(topic.name, "")
				} else {
					cmd = nsq.Register(topic.name, "")
				}
			}

			for _, lookupPeer := range lookupPeers {
				n.logf(LOG_INFO, "LOOKUPD(%s): %s %s", lookupPeer, branch, cmd)
				_, err := lookupPeer.Command(cmd)
				if err != nil {
					n.logf(LOG_ERROR, "LOOKUPD(%s): %s - %s", lookupPeer, cmd, err)
				}
			}
		case lookupPeer := <-syncTopicChan:
			var commands []*nsq.Command
			// build all the commands first so we exit the lock(s) as fast as possible
			n.RLock()
			for _, topic := range n.topicMap {
				topic.RLock()
				if len(topic.channelMap) == 0 {
					commands = append(commands, nsq.Register(topic.name, ""))
				} else {
					for _, channel := range topic.channelMap {
						commands = append(commands, nsq.Register(channel.topicName, channel.name))
					}
				}
				topic.RUnlock()
			}
			n.RUnlock()

			for _, cmd := range commands {
				n.logf(LOG_INFO, "LOOKUPD(%s): %s", lookupPeer, cmd)
				_, err := lookupPeer.Command(cmd)
				if err != nil {
					n.logf(LOG_ERROR, "LOOKUPD(%s): %s - %s", lookupPeer, cmd, err)
					break
				}
			}
		case <-n.optsNotificationChan:
			var tmpPeers []*lookupPeer
			var tmpAddrs []string
			for _, lp := range lookupPeers {
				if in(lp.addr, n.getOpts().NSQLookupdTCPAddresses) {
					tmpPeers = append(tmpPeers, lp)
					tmpAddrs = append(tmpAddrs, lp.addr)
					continue
				}
				n.logf(LOG_INFO, "LOOKUP(%s): removing peer", lp)
				lp.Close()
			}
			lookupPeers = tmpPeers
			lookupAddrs = tmpAddrs
			connect = true
		case <-n.exitChan:
			goto exit
		}
	}

exit:
	n.logf(LOG_INFO, "LOOKUP: closing")
}

func in(s string, lst []string) bool {
	for _, v := range lst {
		if s == v {
			return true
		}
	}
	return false
}

func (n *NSQD) lookupdHTTPAddrs() []string {
	var lookupHTTPAddrs []string
	lookupPeers := n.lookupPeers.Load()
	if lookupPeers == nil {
		return nil
	}
	for _, lp := range lookupPeers.([]*lookupPeer) {
		if len(lp.Info.BroadcastAddress) <= 0 {
			continue
		}
		addr := net.JoinHostPort(lp.Info.BroadcastAddress, strconv.Itoa(lp.Info.HTTPPort))
		lookupHTTPAddrs = append(lookupHTTPAddrs, addr)
	}
	return lookupHTTPAddrs
}
<|endoftext|>


<commit_before>package main

var symtab = make(map[string]string)

func intern(s string) string {
	if v, ok := symtab[s]; ok {
		return v
	}
	symtab[s] = s
	return s
}

func internBytes(s []byte) string {
	if v, ok := symtab[string(s)]; ok {
		return v
	}
	v := string(s)
	symtab[v] = v
	return v
}
<commit_msg>Implement internBytes by intern<commit_after>package main

var symtab = make(map[string]string)

func intern(s string) string {
	if v, ok := symtab[s]; ok {
		return v
	}
	symtab[s] = s
	return s
}

func internBytes(s []byte) string {
	return intern(string(s))
}
<|endoftext|>


<commit_before>package storer

import (
	"bytes"
	"context"
	"crypto/sha256"
	"encoding/base64"
	"errors"
	"fmt"
	"io"
	"math/big"
	"time"

	"github.com/aws/aws-sdk-go-v2/service/s3"
	"github.com/aws/aws-sdk-go-v2/service/s3/types"
	"github.com/jmhodges/clock"
	"github.com/prometheus/client_golang/prometheus"
	"google.golang.org/protobuf/types/known/emptypb"

	"github.com/letsencrypt/boulder/crl"
	"github.com/letsencrypt/boulder/crl/crl_x509"
	cspb "github.com/letsencrypt/boulder/crl/storer/proto"
	"github.com/letsencrypt/boulder/issuance"
	blog "github.com/letsencrypt/boulder/log"
)

// s3Putter matches the subset of the s3.Client interface which we use, to allow
// simpler mocking in tests.
type s3Putter interface {
	PutObject(ctx context.Context, params *s3.PutObjectInput, optFns ...func(*s3.Options)) (*s3.PutObjectOutput, error)
}

type crlStorer struct {
	cspb.UnimplementedCRLStorerServer
	s3Client         s3Putter
	s3Bucket         string
	issuers          map[issuance.IssuerNameID]*issuance.Certificate
	uploadCount      *prometheus.CounterVec
	sizeHistogram    *prometheus.HistogramVec
	latencyHistogram *prometheus.HistogramVec
	log              blog.Logger
	clk              clock.Clock
}

func New(
	issuers []*issuance.Certificate,
	s3Client s3Putter,
	s3Bucket string,
	stats prometheus.Registerer,
	log blog.Logger,
	clk clock.Clock,
) (*crlStorer, error) {
	issuersByNameID := make(map[issuance.IssuerNameID]*issuance.Certificate, len(issuers))
	for _, issuer := range issuers {
		issuersByNameID[issuer.NameID()] = issuer
	}

	uploadCount := prometheus.NewCounterVec(prometheus.CounterOpts{
		Name: "crl_storer_uploads",
		Help: "A counter of the number of CRLs uploaded by crl-storer",
	}, []string{"issuer", "result"})
	stats.MustRegister(uploadCount)

	sizeHistogram := prometheus.NewHistogramVec(prometheus.HistogramOpts{
		Name:    "crl_storer_sizes",
		Help:    "A histogram of the sizes (in bytes) of CRLs uploaded by crl-storer",
		Buckets: []float64{0, 256, 1024, 4096, 16384, 65536},
	}, []string{"issuer"})
	stats.MustRegister(sizeHistogram)

	latencyHistogram := prometheus.NewHistogramVec(prometheus.HistogramOpts{
		Name:    "crl_storer_upload_times",
		Help:    "A histogram of the time (in seconds) it took crl-storer to upload CRLs",
		Buckets: []float64{0.01, 0.2, 0.5, 1, 2, 5, 10, 20, 50, 100, 200, 500, 1000, 2000, 5000},
	}, []string{"issuer"})
	stats.MustRegister(latencyHistogram)

	return &crlStorer{
		issuers:          issuersByNameID,
		s3Client:         s3Client,
		s3Bucket:         s3Bucket,
		uploadCount:      uploadCount,
		sizeHistogram:    sizeHistogram,
		latencyHistogram: latencyHistogram,
		log:              log,
		clk:              clk,
	}, nil
}

// TODO(#6261): Unify all error messages to identify the shard they're working
// on as a JSON object including issuer, crl number, and shard number.

func (cs *crlStorer) UploadCRL(stream cspb.CRLStorer_UploadCRLServer) error {
	var issuer *issuance.Certificate
	var shardIdx int64
	var crlNumber *big.Int
	crlBytes := make([]byte, 0)

	for {
		in, err := stream.Recv()
		if err != nil {
			if err == io.EOF {
				break
			}
			return err
		}

		switch payload := in.Payload.(type) {
		case *cspb.UploadCRLRequest_Metadata:
			if crlNumber != nil || issuer != nil {
				return errors.New("got more than one metadata message")
			}
			if payload.Metadata.IssuerNameID == 0 || payload.Metadata.Number == 0 {
				return errors.New("got incomplete metadata message")
			}

			shardIdx = payload.Metadata.ShardIdx
			crlNumber = crl.Number(time.Unix(0, payload.Metadata.Number))

			var ok bool
			issuer, ok = cs.issuers[issuance.IssuerNameID(payload.Metadata.IssuerNameID)]
			if !ok {
				return fmt.Errorf("got unrecognized IssuerID: %d", payload.Metadata.IssuerNameID)
			}

		case *cspb.UploadCRLRequest_CrlChunk:
			crlBytes = append(crlBytes, payload.CrlChunk...)
		}

	}

	if issuer == nil || crlNumber == nil {
		return errors.New("got no metadata message")
	}

	crlId, err := crl.Id(issuer.NameID(), crlNumber, int(shardIdx))
	if err != nil {
		return err
	}

	cs.sizeHistogram.WithLabelValues(issuer.Subject.CommonName).Observe(float64(len(crlBytes)))

	crl, err := crl_x509.ParseRevocationList(crlBytes)
	if err != nil {
		return fmt.Errorf("parsing CRL for %s: %w", crlId, err)
	}

	if crl.Number.Cmp(crlNumber) != 0 {
		return errors.New("got mismatched CRL Number")
	}

	err = crl.CheckSignatureFrom(issuer.Certificate)
	if err != nil {
		return fmt.Errorf("validating signature for %s: %w", crlId, err)
	}

	start := cs.clk.Now()

	filename := fmt.Sprintf("%d/%s/%d.crl", issuer.NameID(), crlNumber, shardIdx)
	checksum := sha256.Sum256(crlBytes)
	checksumb64 := base64.StdEncoding.EncodeToString(checksum[:])
	crlContentType := "application/pkix-crl"
	_, err = cs.s3Client.PutObject(stream.Context(), &s3.PutObjectInput{
		Bucket:            &cs.s3Bucket,
		Key:               &filename,
		Body:              bytes.NewReader(crlBytes),
		ChecksumAlgorithm: types.ChecksumAlgorithmSha256,
		ChecksumSHA256:    &checksumb64,
		ContentType:       &crlContentType,
		Metadata:          map[string]string{"crlNumber": crlNumber.String()},
	})
	if err != nil {
		cs.uploadCount.WithLabelValues(issuer.Subject.CommonName, "failed").Inc()
		cs.log.AuditErrf("CRL upload failed: id=[%s] err=[%s]", crlId, err)
	} else {
		cs.uploadCount.WithLabelValues(issuer.Subject.CommonName, "success").Inc()
		cs.log.AuditInfof(
			"CRL uploaded: issuerCN=[%s] id=[%s] thisUpdate=[%s] nextUpdate=[%s] numEntries=[%d]",
			issuer.Subject.CommonName, crlId, crl.ThisUpdate, crl.NextUpdate, len(crl.RevokedCertificates),
		)
	}

	latency := cs.clk.Now().Sub(start)
	cs.latencyHistogram.WithLabelValues(issuer.Subject.CommonName).Observe(latency.Seconds())

	if err != nil {
		return fmt.Errorf("uploading to S3: %w", err)
	}

	return stream.SendAndClose(&emptypb.Empty{})
}
<commit_msg>Remove crlNumber from CRL S3 paths (#6352)<commit_after>package storer

import (
	"bytes"
	"context"
	"crypto/sha256"
	"encoding/base64"
	"errors"
	"fmt"
	"io"
	"math/big"
	"time"

	"github.com/aws/aws-sdk-go-v2/service/s3"
	"github.com/aws/aws-sdk-go-v2/service/s3/types"
	"github.com/jmhodges/clock"
	"github.com/prometheus/client_golang/prometheus"
	"google.golang.org/protobuf/types/known/emptypb"

	"github.com/letsencrypt/boulder/crl"
	"github.com/letsencrypt/boulder/crl/crl_x509"
	cspb "github.com/letsencrypt/boulder/crl/storer/proto"
	"github.com/letsencrypt/boulder/issuance"
	blog "github.com/letsencrypt/boulder/log"
)

// s3Putter matches the subset of the s3.Client interface which we use, to allow
// simpler mocking in tests.
type s3Putter interface {
	PutObject(ctx context.Context, params *s3.PutObjectInput, optFns ...func(*s3.Options)) (*s3.PutObjectOutput, error)
}

type crlStorer struct {
	cspb.UnimplementedCRLStorerServer
	s3Client         s3Putter
	s3Bucket         string
	issuers          map[issuance.IssuerNameID]*issuance.Certificate
	uploadCount      *prometheus.CounterVec
	sizeHistogram    *prometheus.HistogramVec
	latencyHistogram *prometheus.HistogramVec
	log              blog.Logger
	clk              clock.Clock
}

func New(
	issuers []*issuance.Certificate,
	s3Client s3Putter,
	s3Bucket string,
	stats prometheus.Registerer,
	log blog.Logger,
	clk clock.Clock,
) (*crlStorer, error) {
	issuersByNameID := make(map[issuance.IssuerNameID]*issuance.Certificate, len(issuers))
	for _, issuer := range issuers {
		issuersByNameID[issuer.NameID()] = issuer
	}

	uploadCount := prometheus.NewCounterVec(prometheus.CounterOpts{
		Name: "crl_storer_uploads",
		Help: "A counter of the number of CRLs uploaded by crl-storer",
	}, []string{"issuer", "result"})
	stats.MustRegister(uploadCount)

	sizeHistogram := prometheus.NewHistogramVec(prometheus.HistogramOpts{
		Name:    "crl_storer_sizes",
		Help:    "A histogram of the sizes (in bytes) of CRLs uploaded by crl-storer",
		Buckets: []float64{0, 256, 1024, 4096, 16384, 65536},
	}, []string{"issuer"})
	stats.MustRegister(sizeHistogram)

	latencyHistogram := prometheus.NewHistogramVec(prometheus.HistogramOpts{
		Name:    "crl_storer_upload_times",
		Help:    "A histogram of the time (in seconds) it took crl-storer to upload CRLs",
		Buckets: []float64{0.01, 0.2, 0.5, 1, 2, 5, 10, 20, 50, 100, 200, 500, 1000, 2000, 5000},
	}, []string{"issuer"})
	stats.MustRegister(latencyHistogram)

	return &crlStorer{
		issuers:          issuersByNameID,
		s3Client:         s3Client,
		s3Bucket:         s3Bucket,
		uploadCount:      uploadCount,
		sizeHistogram:    sizeHistogram,
		latencyHistogram: latencyHistogram,
		log:              log,
		clk:              clk,
	}, nil
}

// TODO(#6261): Unify all error messages to identify the shard they're working
// on as a JSON object including issuer, crl number, and shard number.

func (cs *crlStorer) UploadCRL(stream cspb.CRLStorer_UploadCRLServer) error {
	var issuer *issuance.Certificate
	var shardIdx int64
	var crlNumber *big.Int
	crlBytes := make([]byte, 0)

	for {
		in, err := stream.Recv()
		if err != nil {
			if err == io.EOF {
				break
			}
			return err
		}

		switch payload := in.Payload.(type) {
		case *cspb.UploadCRLRequest_Metadata:
			if crlNumber != nil || issuer != nil {
				return errors.New("got more than one metadata message")
			}
			if payload.Metadata.IssuerNameID == 0 || payload.Metadata.Number == 0 {
				return errors.New("got incomplete metadata message")
			}

			shardIdx = payload.Metadata.ShardIdx
			crlNumber = crl.Number(time.Unix(0, payload.Metadata.Number))

			var ok bool
			issuer, ok = cs.issuers[issuance.IssuerNameID(payload.Metadata.IssuerNameID)]
			if !ok {
				return fmt.Errorf("got unrecognized IssuerID: %d", payload.Metadata.IssuerNameID)
			}

		case *cspb.UploadCRLRequest_CrlChunk:
			crlBytes = append(crlBytes, payload.CrlChunk...)
		}

	}

	if issuer == nil || crlNumber == nil {
		return errors.New("got no metadata message")
	}

	crlId, err := crl.Id(issuer.NameID(), crlNumber, int(shardIdx))
	if err != nil {
		return err
	}

	cs.sizeHistogram.WithLabelValues(issuer.Subject.CommonName).Observe(float64(len(crlBytes)))

	crl, err := crl_x509.ParseRevocationList(crlBytes)
	if err != nil {
		return fmt.Errorf("parsing CRL for %s: %w", crlId, err)
	}

	if crl.Number.Cmp(crlNumber) != 0 {
		return errors.New("got mismatched CRL Number")
	}

	err = crl.CheckSignatureFrom(issuer.Certificate)
	if err != nil {
		return fmt.Errorf("validating signature for %s: %w", crlId, err)
	}

	start := cs.clk.Now()

	filename := fmt.Sprintf("%d/%d.crl", issuer.NameID(), shardIdx)
	checksum := sha256.Sum256(crlBytes)
	checksumb64 := base64.StdEncoding.EncodeToString(checksum[:])
	crlContentType := "application/pkix-crl"
	_, err = cs.s3Client.PutObject(stream.Context(), &s3.PutObjectInput{
		Bucket:            &cs.s3Bucket,
		Key:               &filename,
		Body:              bytes.NewReader(crlBytes),
		ChecksumAlgorithm: types.ChecksumAlgorithmSha256,
		ChecksumSHA256:    &checksumb64,
		ContentType:       &crlContentType,
		Metadata:          map[string]string{"crlNumber": crlNumber.String()},
	})
	if err != nil {
		cs.uploadCount.WithLabelValues(issuer.Subject.CommonName, "failed").Inc()
		cs.log.AuditErrf("CRL upload failed: id=[%s] err=[%s]", crlId, err)
	} else {
		cs.uploadCount.WithLabelValues(issuer.Subject.CommonName, "success").Inc()
		cs.log.AuditInfof(
			"CRL uploaded: id=[%s] issuerCN=[%s] thisUpdate=[%s] nextUpdate=[%s] numEntries=[%d]",
			crlId, issuer.Subject.CommonName, crl.ThisUpdate, crl.NextUpdate, len(crl.RevokedCertificates),
		)
	}

	latency := cs.clk.Now().Sub(start)
	cs.latencyHistogram.WithLabelValues(issuer.Subject.CommonName).Observe(latency.Seconds())

	if err != nil {
		return fmt.Errorf("uploading to S3: %w", err)
	}

	return stream.SendAndClose(&emptypb.Empty{})
}
<|endoftext|>


<commit_before>package main

import (
	"fmt"
	"io"
	"io/ioutil"
	"net/http"
	"os"
	"os/exec"
	"regexp"
)

type Tagger struct {
	Movie    Movie
	FilePath string
	Format   string
}

func (t *Tagger) TempId() string {
	var file_id string
	pwd, _ := os.Getwd()
	files, _ := ioutil.ReadDir(pwd)
	for _, f := range files {
		r, _ := regexp.Compile("-temp-([0-9]+)")
		if r.MatchString(f.Name()) {
			file_id = r.FindString(f.Name())
		}
	}
	return file_id
}

func (t *Tagger) GetArtwork() {
	if t.Movie.Poster != "" {
		fmt.Printf("Downloading artwork...\n")
		file, err := os.Create("artwork.jpg")
		defer file.Close()

		check := http.Client{
			CheckRedirect: func(r *http.Request, via []*http.Request) error {
				r.URL.Opaque = r.URL.Path
				return nil
			},
		}

		resp, err := check.Get(t.Movie.Poster) // add a filter to check redirect
		if err != nil {
			fmt.Println(err)
			panic(err)
		}
		defer resp.Body.Close()
		io.Copy(file, resp.Body)

		if err != nil {
			panic(err)
		}
	}
}

func (t *Tagger) AtomicCommand() error {
	file_path := fmt.Sprintf("%s%s", t.FilePath, t.Format)
	file_args := []string{file_path}
	args := append(file_args, t.Movie.ParsleyFlags()...)

	if t.Movie.Poster != "" {
		artwork := []string{"--artwork", "REMOVE_ALL", "--artwork", "artwork.jpg"}
		args = append(args, artwork...)
	}

	_, err := exec.Command("AtomicParsley", args...).Output()
	return err
}

func (t *Tagger) CleanupCommand() error {
	old_file := fmt.Sprintf("%s%s%s", t.FilePath, t.TempId(), t.Format)
	new_file := fmt.Sprintf("%s%s", t.FilePath, t.Format)
	os.Remove("artwork.jpg")
	return os.Rename(old_file, new_file)
}

func (t *Tagger) SetTags() {
	t.GetArtwork()
	fmt.Printf("Setting tags...\n")
	t.AtomicCommand()
	fmt.Printf("Cleaning up tmp files...\n")
	t.CleanupCommand()
}
<commit_msg>Add functions for file names<commit_after>package main

import (
	"fmt"
	"io"
	"io/ioutil"
	"net/http"
	"os"
	"os/exec"
	"regexp"
)

type Tagger struct {
	Movie    Movie
	FilePath string
	Format   string
}

func (t *Tagger) FullFileName() string {
	return fmt.Sprintf("%s%s", t.FilePath, t.Format)
}

func (t *Tagger) TmpFileName() string {
	return fmt.Sprintf("%s%s%s", t.FilePath, t.TempId(), t.Format)
}

func (t *Tagger) TempId() string {
	var file_id string
	pwd, _ := os.Getwd()
	files, _ := ioutil.ReadDir(pwd)
	for _, f := range files {
		r, _ := regexp.Compile("-temp-([0-9]+)")
		if r.MatchString(f.Name()) {
			file_id = r.FindString(f.Name())
		}
	}
	return file_id
}

func (t *Tagger) GetArtwork() {
	if t.Movie.Poster != "" {
		fmt.Printf("Downloading artwork...\n")
		file, err := os.Create("artwork.jpg")
		defer file.Close()

		check := http.Client{
			CheckRedirect: func(r *http.Request, via []*http.Request) error {
				r.URL.Opaque = r.URL.Path
				return nil
			},
		}

		resp, err := check.Get(t.Movie.Poster) // add a filter to check redirect
		if err != nil {
			fmt.Println(err)
			panic(err)
		}
		defer resp.Body.Close()
		io.Copy(file, resp.Body)

		if err != nil {
			panic(err)
		}
	}
}

func (t *Tagger) AtomicCommand() error {
	file_args := []string{t.FullFileName()}
	args := append(file_args, t.Movie.ParsleyFlags()...)

	if t.Movie.Poster != "" {
		artwork := []string{"--artwork", "REMOVE_ALL", "--artwork", "artwork.jpg"}
		args = append(args, artwork...)
	}

	_, err := exec.Command("AtomicParsley", args...).Output()
	return err
}

func (t *Tagger) CleanupCommand() {
	os.Remove("artwork.jpg")
	os.Rename(t.TmpFileName(), t.FullFileName())
}

func (t *Tagger) SetTags() {
	t.GetArtwork()
	fmt.Printf("Setting tags...\n")
	t.AtomicCommand()
	fmt.Printf("Cleaning up tmp files...\n")
	t.CleanupCommand()
}
<|endoftext|>


<commit_before>package waveguide

import "html/template"

var head = template.HTML(`

                <title>Waveguide</title>

`)

var rootTmpl = template.Must(template.New("root").Parse(`

{{.Head}}

                                Location
                                Conditions
                                Wave Height
                                Last Updated

                                {{range .Spots}}

						{{.HTMLName}}
						{{if .LatestQuality}}
							{{ $q := .LatestQuality }}
							{{$q.Stars}}
							{{$q.WaveHeight}}
							{{$q.HowLong}} ago
						{{end}}

                                {{end}}

`))<commit_msg>Show a helpful message in the case of no data.<commit_after>package waveguide

import "html/template"

var head = template.HTML(`

                <title>Waveguide</title>

`)

var rootTmpl = template.Must(template.New("root").Parse(`

{{.Head}}

                	{{if .Spots}}

					Location
					Conditions
					Wave Height
					Last Updated

					{{range .Spots}}

							{{.HTMLName}}
							{{if .LatestQuality}}
								{{ $q := .LatestQuality }}
								{{$q.Stars}}
								{{$q.WaveHeight}}
								{{$q.HowLong}} ago
							{{end}}

					{{end}}

			{{else}}
				There's no data yet. You can get some by visiting /update_all.
			{{end}}

`))
<|endoftext|>


<commit_before>// Copyright 2015-2018 the u-root Authors. All rights reserved
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

package main

var templates = map[string][]string{
	"all": {
		"github.com/u-root/u-root/cmds/core/*",
		"github.com/u-root/u-root/cmds/boot/*",
		"github.com/u-root/u-root/cmds/nichrome/*",
	},
	"boot": {
		"github.com/u-root/u-root/cmds/boot/*",
	},
	// Core should be things you don't want to live without.
	"core": {
		"github.com/u-root/u-root/cmds/core/*",
	},
	// Minimal should be things you can't live without.
	"minimal": {
		"github.com/u-root/u-root/cmds/core/cat",
		"github.com/u-root/u-root/cmds/core/chmod",
		"github.com/u-root/u-root/cmds/core/cmp",
		"github.com/u-root/u-root/cmds/core/cp",
		"github.com/u-root/u-root/cmds/core/date",
		"github.com/u-root/u-root/cmds/core/dd",
		"github.com/u-root/u-root/cmds/core/df",
		"github.com/u-root/u-root/cmds/core/dhclient",
		"github.com/u-root/u-root/cmds/core/dmesg",
		"github.com/u-root/u-root/cmds/core/echo",
		"github.com/u-root/u-root/cmds/core/elvish",
		"github.com/u-root/u-root/cmds/core/find",
		"github.com/u-root/u-root/cmds/core/free",
		"github.com/u-root/u-root/cmds/core/gpgv",
		"github.com/u-root/u-root/cmds/core/grep",
		"github.com/u-root/u-root/cmds/core/gzip",
		"github.com/u-root/u-root/cmds/core/hostname",
		"github.com/u-root/u-root/cmds/core/id",
		"github.com/u-root/u-root/cmds/core/init",
		"github.com/u-root/u-root/cmds/core/insmod",
		"github.com/u-root/u-root/cmds/core/installcommand",
		"github.com/u-root/u-root/cmds/core/io",
		"github.com/u-root/u-root/cmds/core/ip",
		"github.com/u-root/u-root/cmds/core/kexec",
		"github.com/u-root/u-root/cmds/core/kill",
		"github.com/u-root/u-root/cmds/core/ln",
		"github.com/u-root/u-root/cmds/core/losetup",
		"github.com/u-root/u-root/cmds/core/ls",
		"github.com/u-root/u-root/cmds/core/lsmod",
		"github.com/u-root/u-root/cmds/core/mkdir",
		"github.com/u-root/u-root/cmds/core/mknod",
		"github.com/u-root/u-root/cmds/core/mount",
		"github.com/u-root/u-root/cmds/core/msr",
		"github.com/u-root/u-root/cmds/core/mv",
		"github.com/u-root/u-root/cmds/core/pci",
		"github.com/u-root/u-root/cmds/core/ping",
		"github.com/u-root/u-root/cmds/core/printenv",
		"github.com/u-root/u-root/cmds/core/ps",
		"github.com/u-root/u-root/cmds/core/pwd",
		"github.com/u-root/u-root/cmds/core/readlink",
		"github.com/u-root/u-root/cmds/core/rm",
		"github.com/u-root/u-root/cmds/core/rmmod",
		"github.com/u-root/u-root/cmds/core/seq",
		"github.com/u-root/u-root/cmds/core/shutdown",
		"github.com/u-root/u-root/cmds/core/sleep",
		"github.com/u-root/u-root/cmds/core/sync",
		"github.com/u-root/u-root/cmds/core/tail",
		"github.com/u-root/u-root/cmds/core/tee",
		"github.com/u-root/u-root/cmds/core/truncate",
		"github.com/u-root/u-root/cmds/core/umount",
		"github.com/u-root/u-root/cmds/core/uname",
		"github.com/u-root/u-root/cmds/core/unshare",
		"github.com/u-root/u-root/cmds/core/wc",
		"github.com/u-root/u-root/cmds/core/wget",
		"github.com/u-root/u-root/cmds/core/which",
	},
	// coreboot-app minimal environment
	"coreboot-app": {
		"github.com/u-root/u-root/cmds/core/cat",
		"github.com/u-root/u-root/cmds/core/cbmem",
		"github.com/u-root/u-root/cmds/core/chroot",
		"github.com/u-root/u-root/cmds/core/cp",
		"github.com/u-root/u-root/cmds/core/dd",
		"github.com/u-root/u-root/cmds/core/dhclient",
		"github.com/u-root/u-root/cmds/core/dmesg",
		"github.com/u-root/u-root/cmds/core/elvish",
		"github.com/u-root/u-root/cmds/core/find",
		"github.com/u-root/u-root/cmds/core/grep",
		"github.com/u-root/u-root/cmds/core/id",
		"github.com/u-root/u-root/cmds/core/init",
		"github.com/u-root/u-root/cmds/core/insmod",
		"github.com/u-root/u-root/cmds/core/ip",
		"github.com/u-root/u-root/cmds/core/kill",
		"github.com/u-root/u-root/cmds/core/ls",
		"github.com/u-root/u-root/cmds/core/mount",
		"github.com/u-root/u-root/cmds/core/pci",
		"github.com/u-root/u-root/cmds/core/ping",
		"github.com/u-root/u-root/cmds/core/ps",
		"github.com/u-root/u-root/cmds/core/pwd",
		"github.com/u-root/u-root/cmds/core/rm",
		"github.com/u-root/u-root/cmds/core/rmmod",
		"github.com/u-root/u-root/cmds/core/shutdown",
		"github.com/u-root/u-root/cmds/core/sshd",
		"github.com/u-root/u-root/cmds/core/switch_root",
		"github.com/u-root/u-root/cmds/core/tail",
		"github.com/u-root/u-root/cmds/core/tee",
		"github.com/u-root/u-root/cmds/core/uname",
		"github.com/u-root/u-root/cmds/core/wget",
	},
}
<commit_msg>Add systemboot template, which includes systemboot uinit<commit_after>// Copyright 2015-2018 the u-root Authors. All rights reserved
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

package main

var templates = map[string][]string{
	"all": {
		"github.com/u-root/u-root/cmds/core/*",
		"github.com/u-root/u-root/cmds/boot/*",
		"github.com/u-root/u-root/cmds/nichrome/*",
	},
	"boot": {
		"github.com/u-root/u-root/cmds/boot/boot",
		"github.com/u-root/u-root/cmds/boot/boot2",
		"github.com/u-root/u-root/cmds/boot/fbnetboot",
		"github.com/u-root/u-root/cmds/boot/pxeboot",
		"github.com/u-root/u-root/cmds/boot/localboot",
		"github.com/u-root/u-root/cmds/exp/esxiboot",
	},
	"systemboot": {
		"github.com/u-root/u-root/cmds/boot/boot",
		"github.com/u-root/u-root/cmds/boot/boot2",
		"github.com/u-root/u-root/cmds/boot/fbnetboot",
		"github.com/u-root/u-root/cmds/boot/pxeboot",
		"github.com/u-root/u-root/cmds/boot/localboot",
		"github.com/u-root/u-root/cmds/exp/esxiboot",
		"github.com/u-root/u-root/cmds/boot/uinit",
	},
	// Core should be things you don't want to live without.
	"core": {
		"github.com/u-root/u-root/cmds/core/*",
	},
	// Minimal should be things you can't live without.
	"minimal": {
		"github.com/u-root/u-root/cmds/core/cat",
		"github.com/u-root/u-root/cmds/core/chmod",
		"github.com/u-root/u-root/cmds/core/cmp",
		"github.com/u-root/u-root/cmds/core/cp",
		"github.com/u-root/u-root/cmds/core/date",
		"github.com/u-root/u-root/cmds/core/dd",
		"github.com/u-root/u-root/cmds/core/df",
		"github.com/u-root/u-root/cmds/core/dhclient",
		"github.com/u-root/u-root/cmds/core/dmesg",
		"github.com/u-root/u-root/cmds/core/echo",
		"github.com/u-root/u-root/cmds/core/elvish",
		"github.com/u-root/u-root/cmds/core/find",
		"github.com/u-root/u-root/cmds/core/free",
		"github.com/u-root/u-root/cmds/core/gpgv",
		"github.com/u-root/u-root/cmds/core/grep",
		"github.com/u-root/u-root/cmds/core/gzip",
		"github.com/u-root/u-root/cmds/core/hostname",
		"github.com/u-root/u-root/cmds/core/id",
		"github.com/u-root/u-root/cmds/core/init",
		"github.com/u-root/u-root/cmds/core/insmod",
		"github.com/u-root/u-root/cmds/core/installcommand",
		"github.com/u-root/u-root/cmds/core/io",
		"github.com/u-root/u-root/cmds/core/ip",
		"github.com/u-root/u-root/cmds/core/kexec",
		"github.com/u-root/u-root/cmds/core/kill",
		"github.com/u-root/u-root/cmds/core/ln",
		"github.com/u-root/u-root/cmds/core/losetup",
		"github.com/u-root/u-root/cmds/core/ls",
		"github.com/u-root/u-root/cmds/core/lsmod",
		"github.com/u-root/u-root/cmds/core/mkdir",
		"github.com/u-root/u-root/cmds/core/mknod",
		"github.com/u-root/u-root/cmds/core/mount",
		"github.com/u-root/u-root/cmds/core/msr",
		"github.com/u-root/u-root/cmds/core/mv",
		"github.com/u-root/u-root/cmds/core/pci",
		"github.com/u-root/u-root/cmds/core/ping",
		"github.com/u-root/u-root/cmds/core/printenv",
		"github.com/u-root/u-root/cmds/core/ps",
		"github.com/u-root/u-root/cmds/core/pwd",
		"github.com/u-root/u-root/cmds/core/readlink",
		"github.com/u-root/u-root/cmds/core/rm",
		"github.com/u-root/u-root/cmds/core/rmmod",
		"github.com/u-root/u-root/cmds/core/seq",
		"github.com/u-root/u-root/cmds/core/shutdown",
		"github.com/u-root/u-root/cmds/core/sleep",
		"github.com/u-root/u-root/cmds/core/sync",
		"github.com/u-root/u-root/cmds/core/tail",
		"github.com/u-root/u-root/cmds/core/tee",
		"github.com/u-root/u-root/cmds/core/truncate",
		"github.com/u-root/u-root/cmds/core/umount",
		"github.com/u-root/u-root/cmds/core/uname",
		"github.com/u-root/u-root/cmds/core/unshare",
		"github.com/u-root/u-root/cmds/core/wc",
		"github.com/u-root/u-root/cmds/core/wget",
		"github.com/u-root/u-root/cmds/core/which",
	},
	// coreboot-app minimal environment
	"coreboot-app": {
		"github.com/u-root/u-root/cmds/core/cat",
		"github.com/u-root/u-root/cmds/core/cbmem",
		"github.com/u-root/u-root/cmds/core/chroot",
		"github.com/u-root/u-root/cmds/core/cp",
		"github.com/u-root/u-root/cmds/core/dd",
		"github.com/u-root/u-root/cmds/core/dhclient",
		"github.com/u-root/u-root/cmds/core/dmesg",
		"github.com/u-root/u-root/cmds/core/elvish",
		"github.com/u-root/u-root/cmds/core/find",
		"github.com/u-root/u-root/cmds/core/grep",
		"github.com/u-root/u-root/cmds/core/id",
		"github.com/u-root/u-root/cmds/core/init",
		"github.com/u-root/u-root/cmds/core/insmod",
		"github.com/u-root/u-root/cmds/core/ip",
		"github.com/u-root/u-root/cmds/core/kill",
		"github.com/u-root/u-root/cmds/core/ls",
		"github.com/u-root/u-root/cmds/core/mount",
		"github.com/u-root/u-root/cmds/core/pci",
		"github.com/u-root/u-root/cmds/core/ping",
		"github.com/u-root/u-root/cmds/core/ps",
		"github.com/u-root/u-root/cmds/core/pwd",
		"github.com/u-root/u-root/cmds/core/rm",
		"github.com/u-root/u-root/cmds/core/rmmod",
		"github.com/u-root/u-root/cmds/core/shutdown",
		"github.com/u-root/u-root/cmds/core/sshd",
		"github.com/u-root/u-root/cmds/core/switch_root",
		"github.com/u-root/u-root/cmds/core/tail",
		"github.com/u-root/u-root/cmds/core/tee",
		"github.com/u-root/u-root/cmds/core/uname",
		"github.com/u-root/u-root/cmds/core/wget",
	},
}
<|endoftext|>


<commit_before>package main

// BaseTemplateContent is HTML for base.tmpl
const BaseTemplateContent = `

    <meta charset="utf-8">
    <meta http-equiv="X-UA-Compatible" content="IE=edge">
    <meta name="viewport" content="width=device-width, initial-scale=1">
    <title></title>

{{ template "content" . }}

`

// IndexTemplateContent is HTML for index page
const IndexTemplateContent = `{{ define "content" }}

Index

  {{ range .Posts }}

{{ .Title }}

  {{ end }}

{{ end }}
`

// PostTemplateContent is HTML for post page
const PostTemplateContent = `{{ define "content"}}

{{.Content}}

{{ end }}
`
<commit_msg>update default index template<commit_after>package main

// BaseTemplateContent is HTML for base.tmpl
const BaseTemplateContent = `

    <meta charset="utf-8">
    <meta http-equiv="X-UA-Compatible" content="IE=edge">
    <meta name="viewport" content="width=device-width, initial-scale=1">
    <title></title>

{{ template "content" . }}

`

// IndexTemplateContent is HTML for index page
const IndexTemplateContent = `{{ define "content" }}

Index

  {{ range .Posts }}

{{ .Title }}

  {{ end }}

{{ end }}
`

// PostTemplateContent is HTML for post page
const PostTemplateContent = `{{ define "content"}}

{{.Content}}

{{ end }}
`
<|endoftext|>


<commit_before>package jiraui

const (
	default_list_template = `{{ range .issues }}[{{ .key | printf "%-12s"}}](fg-red)  [{{ if .fields.assignee }}{{ .fields.assignee.name | printf "%-10s" }}{{else}}{{"Unassigned"| printf "%-10s" }}{{end}} ](fg-blue) [{{ .fields.status.name | printf "%-12s"}}](fg-blue) [{{ dateFormat "2006-01-02" .fields.created }}](fg-blue)/[{{ dateFormat "2006-01-02T15:04" .fields.updated }}](fg-green)  {{ .fields.summary | printf "%-75s"}}
{{ end }}`
	default_view_template = `
issue: [{{ .key }}](fg-red)
summary: [{{ .fields.summary }}](fg-blue)

self: {{ .self }}
browse: ENDPOINT/browse/{{ .key }}
priority: {{ .fields.priority.name }}
status: {{ .fields.status.name }}
votes: {{ .fields.votes.votes }}
created: {{ .fields.created }}
updated: {{ .fields.updated }}
assignee: {{ if .fields.assignee }}{{ .fields.assignee.name }}{{end}}
reporter: {{ if .fields.reporter }}{{ .fields.reporter.name }}{{end}}
issuetype: {{ .fields.issuetype.name }}
{{if eq .fields.issuetype.name "Epic" }}epic_links: [<click here to show>](fg-red){{end}}
{{if .fields.customfield_10001 }}epic: [{{ .fields.customfield_10001 }}](fg-red){{end}}
{{if .fields.parent }}parent: [{{ .fields.parent.key }}](fg-red) -- {{ .fields.parent.fields.summary }}{{end}}
subtasks:
{{ range .fields.subtasks }}  - [{{ .key }}](fg-red)[{{.fields.status.name}}] -- {{.fields.summary}}
{{end}}

[labels:](fg-green){{ range .fields.labels }} {{ . }}{{end}}
[components:](fg-green){{ range .fields.components }} {{ .name }}{{end}}
[watchers:](fg-green){{ range .fields.customfield_10304 }} {{ .name }}{{end}}
[blockers:](fg-green)
{{ range .fields.issuelinks }}{{if .outwardIssue}}  - [{{ .outwardIssue.key }}](fg-red)[{{.outwardIssue.fields.status.name}}] -- {{.outwardIssue.fields.summary}}
{{end}}{{end}}
[depends:](fg-green)
{{ range .fields.issuelinks }}{{if .inwardIssue}}  - [{{ .inwardIssue.key }}](fg-red)[{{.inwardIssue.fields.status.name}}] -- {{.inwardIssue.fields.summary}}
{{end}}{{end}}

[description:](fg-green)

  {{ or .fields.description "" | indent 2 }}

[comments:](fg-green)

{{ range .fields.comment.comments }}
  - [{{.author.name}} at {{.created}}](fg-blue)
    {{ or .body "" | indent 4}}

{{end}}
`
	default_help_template = `
[Quick reference for jira-ui](fg-white)

[Actions:](fg-blue)

    <enter>      - select query/ticket
    L            - Label view (query results page only)
    E            - Edit ticket
    S            - Select sort order (query results page only)
    v            - Vote for the selected ticket
    V            - Remove vote on the selected ticket

[Commands (a'la vim/tig):](fg-blue)

    :comment {single-line-comment} - add a short comment to ticket
    :label {labels}                - add labels to selected ticket
    :label add/remove {labels}     - add/remove labels to selected ticket
    :take                          - assign ticket to self
    :assign {user}                 - assign ticket to {user}
    :unassign                      - unassign ticket
    :watch                         - watch ticket
    :vote                          - vote for the selected ticket
    :unvote                        - remove vote for the selected ticket
    :<up>                          - select previous command
    :quit or :q                    - quit

[Navigation:](fg-blue)

    up/k         - previous line
    down/j       - next line
    C-f/<space>  - next page
    C-b          - previous page
    }            - next paragraph/section/fast-move
    {            - previous paragraph/section/fast-move
    n            - next search match
    g            - go to top of page
    G            - go to bottom of page
    q            - go back / quit
    C-c/Q        - quit

[Configuration:](fg-blue)

  It is very much recommended to read the go-jira documentation,
  particularly surrounding the .jira.d configuration directories.

  go-jira-ui uses this same mechanism, so can be used to load per-project
  defaults. It also leverages the templating engine, so you can customise
  the view of both the query output (use 'jira_ui_list' template), and the
  issue 'view' template.

  go-jira-ui reads its own [jira-ui-config.yml](fg-green) file in these
  jira.d directories, as not to pollute the go-jira config. You can add
  additional queries & sort orderings to the top-level Query page:

    $ cat ~/jira.d/jira-ui-config.yml:
    sorts:
      - name: "sort by vote count"
            jql:  "ORDER BY votes DESC"
    queries:
      - name: "alice assigned"
        jql:  "assignee = alice AND resolution = Unresolved"
      - name: "bob assigned"
        jql:  "assignee = bob AND resolution = Unresolved"
      - name: "unresolved must-do"
        jql:  "labels = 'must-do' AND resolution = Unresolved AND ( project = 'OPS' OR project = 'INFRA')"

  Learning JQL is highly recommended, the Atlassian Advanced Searching
  page is a good place to start.

`
)
<commit_msg>update help page for watch support<commit_after>package jiraui

const (
	default_list_template = `{{ range .issues }}[{{ .key | printf "%-12s"}}](fg-red)  [{{ if .fields.assignee }}{{ .fields.assignee.name | printf "%-10s" }}{{else}}{{"Unassigned"| printf "%-10s" }}{{end}} ](fg-blue) [{{ .fields.status.name | printf "%-12s"}}](fg-blue) [{{ dateFormat "2006-01-02" .fields.created }}](fg-blue)/[{{ dateFormat "2006-01-02T15:04" .fields.updated }}](fg-green)  {{ .fields.summary | printf "%-75s"}}
{{ end }}`
	default_view_template = `
issue: [{{ .key }}](fg-red)
summary: [{{ .fields.summary }}](fg-blue)

self: {{ .self }}
browse: ENDPOINT/browse/{{ .key }}
priority: {{ .fields.priority.name }}
status: {{ .fields.status.name }}
votes: {{ .fields.votes.votes }}
created: {{ .fields.created }}
updated: {{ .fields.updated }}
assignee: {{ if .fields.assignee }}{{ .fields.assignee.name }}{{end}}
reporter: {{ if .fields.reporter }}{{ .fields.reporter.name }}{{end}}
issuetype: {{ .fields.issuetype.name }}
{{if eq .fields.issuetype.name "Epic" }}epic_links: [<click here to show>](fg-red){{end}}
{{if .fields.customfield_10001 }}epic: [{{ .fields.customfield_10001 }}](fg-red){{end}}
{{if .fields.parent }}parent: [{{ .fields.parent.key }}](fg-red) -- {{ .fields.parent.fields.summary }}{{end}}
subtasks:
{{ range .fields.subtasks }}  - [{{ .key }}](fg-red)[{{.fields.status.name}}] -- {{.fields.summary}}
{{end}}

[labels:](fg-green){{ range .fields.labels }} {{ . }}{{end}}
[components:](fg-green){{ range .fields.components }} {{ .name }}{{end}}
[watchers:](fg-green){{ range .fields.customfield_10304 }} {{ .name }}{{end}}
[blockers:](fg-green)
{{ range .fields.issuelinks }}{{if .outwardIssue}}  - [{{ .outwardIssue.key }}](fg-red)[{{.outwardIssue.fields.status.name}}] -- {{.outwardIssue.fields.summary}}
{{end}}{{end}}
[depends:](fg-green)
{{ range .fields.issuelinks }}{{if .inwardIssue}}  - [{{ .inwardIssue.key }}](fg-red)[{{.inwardIssue.fields.status.name}}] -- {{.inwardIssue.fields.summary}}
{{end}}{{end}}

[description:](fg-green)

  {{ or .fields.description "" | indent 2 }}

[comments:](fg-green)

{{ range .fields.comment.comments }}
  - [{{.author.name}} at {{.created}}](fg-blue)
    {{ or .body "" | indent 4}}

{{end}}
`
	default_help_template = `
[Quick reference for jira-ui](fg-white)

[Actions:](fg-blue)

    <enter>      - select query/ticket
    L            - Label view (query results page only)
    E            - Edit ticket
    S            - Select sort order (query results page only)
    v            - Vote for the selected ticket
    V            - Remove vote on the selected ticket
    w            - Watch the selected ticket
    W            - Unwatch the selected ticket

[Commands (a'la vim/tig):](fg-blue)

    :comment {single-line-comment} - add a short comment to ticket
    :label {labels}                - add labels to selected ticket
    :label add/remove {labels}     - add/remove labels to selected ticket
    :take                          - assign ticket to self
    :assign {user}                 - assign ticket to {user}
    :unassign                      - unassign ticket
    :vote                          - vote for the selected ticket
    :unvote                        - remove vote for the selected ticket
    :watch [add/remove] [watcher]  - watch ticket (optionally as a different user)
    :<up>                          - select previous command
    :quit or :q                    - quit

[Navigation:](fg-blue)

    up/k         - previous line
    down/j       - next line
    C-f/<space>  - next page
    C-b          - previous page
    }            - next paragraph/section/fast-move
    {            - previous paragraph/section/fast-move
    n            - next search match
    g            - go to top of page
    G            - go to bottom of page
    q            - go back / quit
    C-c/Q        - quit

[Configuration:](fg-blue)

  It is very much recommended to read the go-jira documentation,
  particularly surrounding the .jira.d configuration directories.

  go-jira-ui uses this same mechanism, so can be used to load per-project
  defaults. It also leverages the templating engine, so you can customise
  the view of both the query output (use 'jira_ui_list' template), and the
  issue 'view' template.

  go-jira-ui reads its own [jira-ui-config.yml](fg-green) file in these
  jira.d directories, as not to pollute the go-jira config. You can add
  additional queries & sort orderings to the top-level Query page:

    $ cat ~/jira.d/jira-ui-config.yml:
    sorts:
      - name: "sort by vote count"
            jql:  "ORDER BY votes DESC"
    queries:
      - name: "alice assigned"
        jql:  "assignee = alice AND resolution = Unresolved"
      - name: "bob assigned"
        jql:  "assignee = bob AND resolution = Unresolved"
      - name: "unresolved must-do"
        jql:  "labels = 'must-do' AND resolution = Unresolved AND ( project = 'OPS' OR project = 'INFRA')"

  Learning JQL is highly recommended, the Atlassian Advanced Searching
  page is a good place to start.

`
)
<|endoftext|>


<commit_before>package main

import (
	"bonitosrv/elasticsearch"
	"encoding/json"
	"fmt"
	"io/ioutil"
	"time"
)

// Type grouping the methods of this API end point
type ByDimensionApi struct {
	es    *elasticsearch.Elasticsearch
	Index string
}

func NewByDimensionApi(index string) *ByDimensionApi {
	return &ByDimensionApi{
		es:    elasticsearch.NewElasticsearch(),
		Index: index,
	}
}

type ByDimensionRequest struct {
	Timerange
	Metrics          []string
	HistogramMetrics []string `json:"histogram_metrics"`
	Config           struct {
		Primary_dimension   string
		Secondary_dimension string
		Responsetime_field  string
		Status_field        string
		Timestamp_field     string
		Status_value_ok     string
		Count_field         string
		Percentiles         []float32
		Histogram_points    int
	}

	// Calculated values
	interval_seconds  float32
	histogram_seconds float32
}

// Fills in the request with an omitted configuration options that
// have defaults.
func (api *ByDimensionApi) setRequestDefaults(req *ByDimensionRequest) {
	c := &req.Config

	if len(c.Primary_dimension) == 0 {
		c.Primary_dimension = "service"
	}
	if len(c.Secondary_dimension) == 0 {
		c.Secondary_dimension = "host"
	}
	if len(c.Responsetime_field) == 0 {
		c.Responsetime_field = "responsetime"
	}
	if len(c.Status_field) == 0 {
		c.Status_field = "status"
	}
	if len(c.Status_value_ok) == 0 {
		c.Status_value_ok = "ok"
	}
	if len(c.Count_field) == 0 {
		c.Count_field = "count"
	}
	if len(c.Timestamp_field) == 0 {
		c.Timestamp_field = "timestamp"
	}
	if len(c.Percentiles) == 0 {
		c.Percentiles = []float32{50, 90, 99, 99.5}
	}
	if c.Histogram_points == 0 {
		c.Histogram_points = 10
	}

	if req.Timerange.IsZero() {
		req.Timerange.From = JsTime(time.Now().Add(-1 * time.Hour))
		req.Timerange.To = JsTime(time.Now())
	}
}

// ByDimensionResponse is the structure that gets marshaled to JSON
// and sent to the client in response to a bydimension query.
type ByDimensionResponse struct {
	Status  string             `json:"status"`
	Primary []PrimaryDimension `json:"primary"`
}

type PrimaryDimension struct {
	Name         string                      `json:"name"`
	Metrics      map[string]float32          `json:"metrics"`
	Hist_metrics map[string][]HistogramValue `json:"hist_metrics"`
}

type HistogramValue struct {
	Ts    JsTime  `json:"ts"`
	Value float32 `json:"value"`
}

// EsByDimensionReq is the structure that gets marshaled to JSON
// and is sent to Elasticsearch.
type EsByDimensionReq struct {
	Query struct {
		Filtered MapStr `json:"filtered"`
	} `json:"query"`
	Aggs struct {
		Primary struct {
			Terms struct {
				Field string `json:"field"`
				Size  int    `json:"size"`
			} `json:"terms"`
			Aggs MapStr `json:"aggs"`
		} `json:"primary"`
	} `json:"aggs"`
}

func (api *ByDimensionApi) buildTimeFilter(req *ByDimensionRequest) *MapStr {
	return &MapStr{
		"range": MapStr{
			req.Config.Timestamp_field: MapStr{
				"lte": elasticsearch.Time(req.Timerange.To),
				"gte": elasticsearch.Time(req.Timerange.From),
			},
		},
	}
}

func (api *ByDimensionApi) buildRequestAggs(req *ByDimensionRequest) (*MapStr, error) {
	aggs := MapStr{}
	for _, metric := range req.Metrics {
		switch metric {
		case "volume":
			aggs["volume"] = MapStr{
				"sum": MapStr{
					"field": req.Config.Count_field,
				},
			}
		case "rt_max", "rt_avg":
			aggs["rt_stats"] = MapStr{
				"stats": MapStr{
					"field": req.Config.Responsetime_field,
				},
			}
		case "rt_percentiles":
			aggs["rt_percentiles"] = MapStr{
				"percentiles": MapStr{
					"field":    req.Config.Responsetime_field,
					"percents": req.Config.Percentiles,
				},
			}
		case "secondary_count":
			aggs["secondary_card"] = MapStr{
				"cardinality": MapStr{
					"field": req.Config.Secondary_dimension,
				},
			}
		case "errors_rate":
			aggs["errors_count"] = MapStr{
				"filter": MapStr{
					"not": MapStr{
						"term": MapStr{
							req.Config.Status_field: req.Config.Status_value_ok,
						},
					},
				},
				"aggs": MapStr{
					"count": MapStr{
						"sum": MapStr{
							"field": "count",
						},
					},
				},
			}
			// make sure the volume is there
			aggs["volume"] = MapStr{
				"sum": MapStr{
					"field": req.Config.Count_field,
				},
			}
		default:
			return nil, fmt.Errorf("Unknown metric name '%s'", metric)
		}
	}

	return &aggs, nil
}

// Returns the Timerange interval as both float32 and string that can be passed
// to the Elasticseach date_histogram field. For example, 613.234s is a valid
// interval. The interval is computed in such a way so that there will be
// approximately the given number of points in the histogram.
func computeHistogramInterval(tr *Timerange, points int) (float32, string) {

	// the bucket interval in seconds (can be a float)
	total_interval := time.Time(tr.To).Sub(time.Time(tr.From))
	interval_secs := float32(int64(total_interval)/int64(points)/int64(1e6)) / 1000
	return interval_secs, fmt.Sprintf("%.3fs", interval_secs)
}

// Returns the real number of seconds in a bucket returned by Elasticsearch.
// This can be different from interval_secs for the first and last buckets,
// which can be smaller.
func computeRealSecondsInInterval(interval_secs float32, start_interval time.Time, tr *Timerange) float32 {
	// When dividing by the seconds, we have to be careful with
	// the first and the last interval which can be shorter.
	var from, to time.Time
	if start_interval.Before(time.Time(tr.From)) {
		from = time.Time(tr.From)
	} else {
		from = start_interval
	}

	end_interval := start_interval.Add(time.Duration(interval_secs*1e3) * time.Millisecond)
	if end_interval.After(time.Time(tr.To)) {
		to = time.Time(tr.To)
	} else {
		to = end_interval
	}

	return float32(int64(to.Sub(from))/1e6) / 1000.0
}

func (api *ByDimensionApi) buildRequestHistogramAggs(req *ByDimensionRequest) (*MapStr, error) {

	var interval string
	req.histogram_seconds, interval = computeHistogramInterval(&req.Timerange,
		req.Config.Histogram_points)

	aggs := MapStr{}
	for _, metric := range req.HistogramMetrics {
		switch metric {
		case "volume":
			aggs["volume_hist"] = MapStr{
				"date_histogram": MapStr{
					"field":    "timestamp",
					"interval": interval,
				},
				"aggs": MapStr{
					"volume": MapStr{
						"sum": MapStr{
							"field": req.Config.Count_field,
						},
					},
				},
			}
		default:
			return nil, fmt.Errorf("Unknown histogram metric name '%s'", metric)
		}

	}
	return &aggs, nil
}

func (api *ByDimensionApi) bucketToPrimary(req *ByDimensionRequest,
	bucket map[string]json.RawMessage) (*PrimaryDimension, error) {

	var primary PrimaryDimension

	err := json.Unmarshal(bucket["key"], &primary.Name)
	if err != nil {
		return nil, err
	}

	// transform metrics
	primary.Metrics = map[string]float32{}
	for _, metric := range req.Metrics {
		switch metric {
		case "volume":
			var volume struct {
				Value float32
			}

			err = json.Unmarshal(bucket["volume"], &volume)
			if err != nil {
				return nil, err
			}

			primary.Metrics["volume"] = volume.Value / req.interval_seconds
		case "rt_max":
		case "rt_avg":
			var stats struct {
				Max float32
				Avg float32
			}
			err = json.Unmarshal(bucket["rt_stats"], &stats)
			if err != nil {
				return nil, err
			}

			primary.Metrics["rt_max"] = stats.Max
			primary.Metrics["rt_avg"] = stats.Avg

		case "rt_percentiles":
			var percentiles struct {
				Values map[string]float32
			}
			err = json.Unmarshal(bucket["rt_percentiles"], &percentiles)
			for key, value := range percentiles.Values {
				primary.Metrics[fmt.Sprintf("rt_%sp", key)] = value
			}

		case "secondary_count":
			var secondary struct {
				Value float32
			}

			err = json.Unmarshal(bucket["secondary_card"], &secondary)
			if err != nil {
				return nil, err
			}

			primary.Metrics["secondary_count"] = secondary.Value
		case "errors_rate":
			var errors_count struct {
				Count struct {
					Value float32
				}
			}
			var volume1 struct {
				Value float32
			}

			err = json.Unmarshal(bucket["errors_count"], &errors_count)
			if err != nil {
				return nil, err
			}
			err = json.Unmarshal(bucket["volume"], &volume1)
			if err != nil {
				return nil, err
			}

			primary.Metrics["errors_rate"] = errors_count.Count.Value /
				volume1.Value
		}
	}

	// transform histogram metrics
	primary.Hist_metrics = map[string][]HistogramValue{}
	for _, metric := range req.HistogramMetrics {
		switch metric {
		case "volume":
			var volume_hist struct {
				Buckets []struct {
					Key_as_string elasticsearch.Time
					Volume        struct {
						Value float32
					}
				}
			}

			err = json.Unmarshal(bucket["volume_hist"], &volume_hist)
			if err != nil {
				return nil, err
			}

			values := []HistogramValue{}

			for _, bucket := range volume_hist.Buckets {
				bucket_secs := computeRealSecondsInInterval(req.histogram_seconds,
					time.Time(bucket.Key_as_string), &req.Timerange)

				fmt.Println("bucket_secs", bucket_secs, "histogram_seconds", req.histogram_seconds)
				fmt.Println(time.Time(bucket.Key_as_string))
				fmt.Println(req.Timerange)

				values = append(values, HistogramValue{
					Ts:    JsTime(bucket.Key_as_string),
					Value: bucket.Volume.Value / bucket_secs,
				})
			}

			primary.Hist_metrics["volume"] = values
		}
	}

	return &primary, nil
}

func (api *ByDimensionApi) Query(req *ByDimensionRequest) (*ByDimensionResponse, int, error) {

	var esreq EsByDimensionReq
	es := elasticsearch.NewElasticsearch()

	api.setRequestDefaults(req)

	req.interval_seconds = float32(time.Time(req.Timerange.To).Sub(time.Time(req.Timerange.From))) / 1e9

	primary := &esreq.Aggs.Primary
	primary.Terms.Field = req.Config.Primary_dimension

	// set timestamp filter
	esreq.Query.Filtered = MapStr{}
	esreq.Query.Filtered["filter"] = *api.buildTimeFilter(req)

	aggs, err := api.buildRequestAggs(req)
	if err != nil {
		return nil, 400, err
	}
	primary.Aggs = *aggs

	aggs, err = api.buildRequestHistogramAggs(req)
	if err != nil {
		return nil, 400, err
	}
	for k, v := range *aggs {
		primary.Aggs[k] = v
	}

	// up to here we assume there are client errors, from here on
	// it's on us.

	objreq, err := json.Marshal(&esreq)
	if err != nil {
		return nil, 500, err
	}

	//fmt.Println("Objreq=", string(objreq))

	resp, err := es.Search(api.Index, "?search_type=count",
		string(objreq))
	if err != nil {
		return nil, 500, err
	}
	defer resp.Body.Close()

	body, err := ioutil.ReadAll(resp.Body)
	if err != nil {
		return nil, 500, err
	}

	var answ struct {
		Aggregations struct {
			Primary struct {
				Buckets []map[string]json.RawMessage
			}
		}
	}

	err = json.Unmarshal(body, &answ)
	if err != nil {
		return nil, 500, err
	}

	var response ByDimensionResponse
	response.Primary = []PrimaryDimension{}

	for _, bucket := range answ.Aggregations.Primary.Buckets {

		primary, err := api.bucketToPrimary(req, bucket)
		if err != nil {
			return nil, 500, err
		}

		response.Primary = append(response.Primary, *primary)
	}

	// if we got so far, we're successful
	response.Status = "ok"

	return &response, 200, nil
}
<commit_msg>removed debugging statements<commit_after>package main

import (
	"bonitosrv/elasticsearch"
	"encoding/json"
	"fmt"
	"io/ioutil"
	"time"
)

// Type grouping the methods of this API end point
type ByDimensionApi struct {
	es    *elasticsearch.Elasticsearch
	Index string
}

func NewByDimensionApi(index string) *ByDimensionApi {
	return &ByDimensionApi{
		es:    elasticsearch.NewElasticsearch(),
		Index: index,
	}
}

type ByDimensionRequest struct {
	Timerange
	Metrics          []string
	HistogramMetrics []string `json:"histogram_metrics"`
	Config           struct {
		Primary_dimension   string
		Secondary_dimension string
		Responsetime_field  string
		Status_field        string
		Timestamp_field     string
		Status_value_ok     string
		Count_field         string
		Percentiles         []float32
		Histogram_points    int
	}

	// Calculated values
	interval_seconds  float32
	histogram_seconds float32
}

// Fills in the request with an omitted configuration options that
// have defaults.
func (api *ByDimensionApi) setRequestDefaults(req *ByDimensionRequest) {
	c := &req.Config

	if len(c.Primary_dimension) == 0 {
		c.Primary_dimension = "service"
	}
	if len(c.Secondary_dimension) == 0 {
		c.Secondary_dimension = "host"
	}
	if len(c.Responsetime_field) == 0 {
		c.Responsetime_field = "responsetime"
	}
	if len(c.Status_field) == 0 {
		c.Status_field = "status"
	}
	if len(c.Status_value_ok) == 0 {
		c.Status_value_ok = "ok"
	}
	if len(c.Count_field) == 0 {
		c.Count_field = "count"
	}
	if len(c.Timestamp_field) == 0 {
		c.Timestamp_field = "timestamp"
	}
	if len(c.Percentiles) == 0 {
		c.Percentiles = []float32{50, 90, 99, 99.5}
	}
	if c.Histogram_points == 0 {
		c.Histogram_points = 10
	}

	if req.Timerange.IsZero() {
		req.Timerange.From = JsTime(time.Now().Add(-1 * time.Hour))
		req.Timerange.To = JsTime(time.Now())
	}
}

// ByDimensionResponse is the structure that gets marshaled to JSON
// and sent to the client in response to a bydimension query.
type ByDimensionResponse struct {
	Status  string             `json:"status"`
	Primary []PrimaryDimension `json:"primary"`
}

type PrimaryDimension struct {
	Name         string                      `json:"name"`
	Metrics      map[string]float32          `json:"metrics"`
	Hist_metrics map[string][]HistogramValue `json:"hist_metrics"`
}

type HistogramValue struct {
	Ts    JsTime  `json:"ts"`
	Value float32 `json:"value"`
}

// EsByDimensionReq is the structure that gets marshaled to JSON
// and is sent to Elasticsearch.
type EsByDimensionReq struct {
	Query struct {
		Filtered MapStr `json:"filtered"`
	} `json:"query"`
	Aggs struct {
		Primary struct {
			Terms struct {
				Field string `json:"field"`
				Size  int    `json:"size"`
			} `json:"terms"`
			Aggs MapStr `json:"aggs"`
		} `json:"primary"`
	} `json:"aggs"`
}

func (api *ByDimensionApi) buildTimeFilter(req *ByDimensionRequest) *MapStr {
	return &MapStr{
		"range": MapStr{
			req.Config.Timestamp_field: MapStr{
				"lte": elasticsearch.Time(req.Timerange.To),
				"gte": elasticsearch.Time(req.Timerange.From),
			},
		},
	}
}

func (api *ByDimensionApi) buildRequestAggs(req *ByDimensionRequest) (*MapStr, error) {
	aggs := MapStr{}
	for _, metric := range req.Metrics {
		switch metric {
		case "volume":
			aggs["volume"] = MapStr{
				"sum": MapStr{
					"field": req.Config.Count_field,
				},
			}
		case "rt_max", "rt_avg":
			aggs["rt_stats"] = MapStr{
				"stats": MapStr{
					"field": req.Config.Responsetime_field,
				},
			}
		case "rt_percentiles":
			aggs["rt_percentiles"] = MapStr{
				"percentiles": MapStr{
					"field":    req.Config.Responsetime_field,
					"percents": req.Config.Percentiles,
				},
			}
		case "secondary_count":
			aggs["secondary_card"] = MapStr{
				"cardinality": MapStr{
					"field": req.Config.Secondary_dimension,
				},
			}
		case "errors_rate":
			aggs["errors_count"] = MapStr{
				"filter": MapStr{
					"not": MapStr{
						"term": MapStr{
							req.Config.Status_field: req.Config.Status_value_ok,
						},
					},
				},
				"aggs": MapStr{
					"count": MapStr{
						"sum": MapStr{
							"field": "count",
						},
					},
				},
			}
			// make sure the volume is there
			aggs["volume"] = MapStr{
				"sum": MapStr{
					"field": req.Config.Count_field,
				},
			}
		default:
			return nil, fmt.Errorf("Unknown metric name '%s'", metric)
		}
	}

	return &aggs, nil
}

// Returns the Timerange interval as both float32 and string that can be passed
// to the Elasticseach date_histogram field. For example, 613.234s is a valid
// interval. The interval is computed in such a way so that there will be
// approximately the given number of points in the histogram.
func computeHistogramInterval(tr *Timerange, points int) (float32, string) {

	// the bucket interval in seconds (can be a float)
	total_interval := time.Time(tr.To).Sub(time.Time(tr.From))
	interval_secs := float32(int64(total_interval)/int64(points)/int64(1e6)) / 1000
	return interval_secs, fmt.Sprintf("%.3fs", interval_secs)
}

// Returns the real number of seconds in a bucket returned by Elasticsearch.
// This can be different from interval_secs for the first and last buckets,
// which can be smaller.
func computeRealSecondsInInterval(interval_secs float32, start_interval time.Time, tr *Timerange) float32 {
	// When dividing by the seconds, we have to be careful with
	// the first and the last interval which can be shorter.
	var from, to time.Time
	if start_interval.Before(time.Time(tr.From)) {
		from = time.Time(tr.From)
	} else {
		from = start_interval
	}

	end_interval := start_interval.Add(time.Duration(interval_secs*1e3) * time.Millisecond)
	if end_interval.After(time.Time(tr.To)) {
		to = time.Time(tr.To)
	} else {
		to = end_interval
	}

	return float32(int64(to.Sub(from))/1e6) / 1000.0
}

func (api *ByDimensionApi) buildRequestHistogramAggs(req *ByDimensionRequest) (*MapStr, error) {

	var interval string
	req.histogram_seconds, interval = computeHistogramInterval(&req.Timerange,
		req.Config.Histogram_points)

	aggs := MapStr{}
	for _, metric := range req.HistogramMetrics {
		switch metric {
		case "volume":
			aggs["volume_hist"] = MapStr{
				"date_histogram": MapStr{
					"field":    "timestamp",
					"interval": interval,
				},
				"aggs": MapStr{
					"volume": MapStr{
						"sum": MapStr{
							"field": req.Config.Count_field,
						},
					},
				},
			}
		default:
			return nil, fmt.Errorf("Unknown histogram metric name '%s'", metric)
		}

	}
	return &aggs, nil
}

func (api *ByDimensionApi) bucketToPrimary(req *ByDimensionRequest,
	bucket map[string]json.RawMessage) (*PrimaryDimension, error) {

	var primary PrimaryDimension

	err := json.Unmarshal(bucket["key"], &primary.Name)
	if err != nil {
		return nil, err
	}

	// transform metrics
	primary.Metrics = map[string]float32{}
	for _, metric := range req.Metrics {
		switch metric {
		case "volume":
			var volume struct {
				Value float32
			}

			err = json.Unmarshal(bucket["volume"], &volume)
			if err != nil {
				return nil, err
			}

			primary.Metrics["volume"] = volume.Value / req.interval_seconds
		case "rt_max":
		case "rt_avg":
			var stats struct {
				Max float32
				Avg float32
			}
			err = json.Unmarshal(bucket["rt_stats"], &stats)
			if err != nil {
				return nil, err
			}

			primary.Metrics["rt_max"] = stats.Max
			primary.Metrics["rt_avg"] = stats.Avg

		case "rt_percentiles":
			var percentiles struct {
				Values map[string]float32
			}
			err = json.Unmarshal(bucket["rt_percentiles"], &percentiles)
			for key, value := range percentiles.Values {
				primary.Metrics[fmt.Sprintf("rt_%sp", key)] = value
			}

		case "secondary_count":
			var secondary struct {
				Value float32
			}

			err = json.Unmarshal(bucket["secondary_card"], &secondary)
			if err != nil {
				return nil, err
			}

			primary.Metrics["secondary_count"] = secondary.Value
		case "errors_rate":
			var errors_count struct {
				Count struct {
					Value float32
				}
			}
			var volume1 struct {
				Value float32
			}

			err = json.Unmarshal(bucket["errors_count"], &errors_count)
			if err != nil {
				return nil, err
			}
			err = json.Unmarshal(bucket["volume"], &volume1)
			if err != nil {
				return nil, err
			}

			primary.Metrics["errors_rate"] = errors_count.Count.Value /
				volume1.Value
		}
	}

	// transform histogram metrics
	primary.Hist_metrics = map[string][]HistogramValue{}
	for _, metric := range req.HistogramMetrics {
		switch metric {
		case "volume":
			var volume_hist struct {
				Buckets []struct {
					Key_as_string elasticsearch.Time
					Volume        struct {
						Value float32
					}
				}
			}

			err = json.Unmarshal(bucket["volume_hist"], &volume_hist)
			if err != nil {
				return nil, err
			}

			values := []HistogramValue{}

			for _, bucket := range volume_hist.Buckets {
				bucket_secs := computeRealSecondsInInterval(req.histogram_seconds,
					time.Time(bucket.Key_as_string), &req.Timerange)

				values = append(values, HistogramValue{
					Ts:    JsTime(bucket.Key_as_string),
					Value: bucket.Volume.Value / bucket_secs,
				})
			}

			primary.Hist_metrics["volume"] = values
		}
	}

	return &primary, nil
}

func (api *ByDimensionApi) Query(req *ByDimensionRequest) (*ByDimensionResponse, int, error) {

	var esreq EsByDimensionReq
	es := elasticsearch.NewElasticsearch()

	api.setRequestDefaults(req)

	req.interval_seconds = float32(time.Time(req.Timerange.To).Sub(time.Time(req.Timerange.From))) / 1e9

	primary := &esreq.Aggs.Primary
	primary.Terms.Field = req.Config.Primary_dimension

	// set timestamp filter
	esreq.Query.Filtered = MapStr{}
	esreq.Query.Filtered["filter"] = *api.buildTimeFilter(req)

	aggs, err := api.buildRequestAggs(req)
	if err != nil {
		return nil, 400, err
	}
	primary.Aggs = *aggs

	aggs, err = api.buildRequestHistogramAggs(req)
	if err != nil {
		return nil, 400, err
	}
	for k, v := range *aggs {
		primary.Aggs[k] = v
	}

	// up to here we assume there are client errors, from here on
	// it's on us.

	objreq, err := json.Marshal(&esreq)
	if err != nil {
		return nil, 500, err
	}

	//fmt.Println("Objreq=", string(objreq))

	resp, err := es.Search(api.Index, "?search_type=count",
		string(objreq))
	if err != nil {
		return nil, 500, err
	}
	defer resp.Body.Close()

	body, err := ioutil.ReadAll(resp.Body)
	if err != nil {
		return nil, 500, err
	}

	var answ struct {
		Aggregations struct {
			Primary struct {
				Buckets []map[string]json.RawMessage
			}
		}
	}

	err = json.Unmarshal(body, &answ)
	if err != nil {
		return nil, 500, err
	}

	var response ByDimensionResponse
	response.Primary = []PrimaryDimension{}

	for _, bucket := range answ.Aggregations.Primary.Buckets {

		primary, err := api.bucketToPrimary(req, bucket)
		if err != nil {
			return nil, 500, err
		}

		response.Primary = append(response.Primary, *primary)
	}

	// if we got so far, we're successful
	response.Status = "ok"

	return &response, 200, nil
}
<|endoftext|>


<commit_before>package main

import (
	"errors"
	"github.com/Comcast/webpa-common/health"
	"github.com/Comcast/webpa-common/logging"
	"github.com/stretchr/testify/assert"
	"github.com/stretchr/testify/mock"
	"github.com/stretchr/testify/require"
	"math"
	"net/http"
	"net/http/httptest"
	"os"
	"strings"
	"sync"
	"testing"
)

// Begin mock declarations

// mockHandler only needs to mock the `HandleRequest` method
type mockHandler struct {
	mock.Mock
}

func (m *mockHandler) HandleRequest(workerID int, inRequest CaduceusRequest) {
	m.Called(workerID, inRequest)
}

// mockHealthTracker needs to mock things from both the `HealthTracker`
// interface as well as the `health.Monitor` interface
type mockHealthTracker struct {
	mock.Mock
}

func (m *mockHealthTracker) SendEvent(healthFunc health.HealthFunc) {
	m.Called(healthFunc)
}

func (m *mockHealthTracker) ServeHTTP(response http.ResponseWriter, request *http.Request) {
	m.Called(response, request)
}

func (m *mockHealthTracker) IncrementBucket(inSize int) {
	m.Called(inSize)
}

// Begin test functions

func TestMain(m *testing.M) {
	os.Exit(m.Run())
}

func TestWorkerPool(t *testing.T) {
	assert := assert.New(t)

	workerPool := WorkerPoolFactory{
		NumWorkers: 1,
		QueueSize:  1,
	}.New()

	t.Run("TestWorkerPoolSend", func(t *testing.T) {
		testWG := new(sync.WaitGroup)
		testWG.Add(1)

		require.NotNil(t, workerPool)
		err := workerPool.Send(func(workerID int) {
			testWG.Done()
		})

		testWG.Wait()
		assert.Nil(err)
	})

	workerPool = WorkerPoolFactory{
		NumWorkers: 1,
		QueueSize:  0,
	}.New()

	/* TODO: This test is broken.
	t.Run("TestWorkerPoolFullQueue", func(t *testing.T) {
		require.NotNil(t, workerPool)
		err := workerPool.Send(func(workerID int) {
			assert.Fail("This should not execute because our worker queue is full.")
		})

		assert.NotNil(err)
	})
	*/
}

func TestCaduceusHealth(t *testing.T) {
	assert := assert.New(t)

	testData := []struct {
		inSize       int
		expectedStat health.Stat
	}{
		{inSize: -1, expectedStat: PayloadsOverZero},
		{inSize: 0, expectedStat: PayloadsOverZero},
		{inSize: 99, expectedStat: PayloadsOverZero},
		{inSize: 999, expectedStat: PayloadsOverHundred},
		{inSize: 9999, expectedStat: PayloadsOverThousand},
		{inSize: 10001, expectedStat: PayloadsOverTenThousand},
		{inSize: math.MaxInt32, expectedStat: PayloadsOverTenThousand},
	}

	t.Run("TestIncrementBucket", func(t *testing.T) {
		for _, data := range testData {
			fakeMonitor := new(mockHealthTracker)
			fakeMonitor.On("SendEvent", mock.AnythingOfType("health.HealthFunc")).Run(
				func(args mock.Arguments) {
					healthFunc := args.Get(0).(health.HealthFunc)
					stats := make(health.Stats)

					healthFunc(stats)
					assert.Equal(1, stats[data.expectedStat])
				}).Once()

			caduceusHealth := &CaduceusHealth{fakeMonitor}

			caduceusHealth.IncrementBucket(data.inSize)
			fakeMonitor.AssertExpectations(t)
		}
	})
}

func TestServeHandler(t *testing.T) {
	assert := assert.New(t)

	logger := logging.DefaultLogger()
	fakeHandler := new(mockHandler)
	fakeHandler.On("HandleRequest", mock.AnythingOfType("int"), mock.AnythingOfType("CaduceusRequest")).Return().Once()

	fakeHealth := new(mockHealthTracker)
	fakeHealth.On("IncrementBucket", mock.AnythingOfType("int")).Return().Once()

	requestSuccessful := func(func(workerID int)) error {
		fakeHandler.HandleRequest(0, CaduceusRequest{})
		return nil
	}

	serverWrapper := &ServerHandler{
		Logger:          logger,
		caduceusHandler: fakeHandler,
		caduceusHealth:  fakeHealth,
		doJob:           requestSuccessful,
	}

	req := httptest.NewRequest("POST", "localhost:8080", strings.NewReader("Test payload."))

	t.Run("TestServeHTTPHappyPath", func(t *testing.T) {
		req.Header.Set("Content-Type", "text/plain")

		w := httptest.NewRecorder()
		serverWrapper.ServeHTTP(w, req)
		resp := w.Result()

		assert.Equal(202, resp.StatusCode)
		fakeHandler.AssertExpectations(t)
		fakeHealth.AssertExpectations(t)
	})

	t.Run("TestServeHTTPTooManyHeaders", func(t *testing.T) {
		req.Header.Add("Content-Type", "too/many/headers")

		w := httptest.NewRecorder()
		serverWrapper.ServeHTTP(w, req)
		resp := w.Result()

		assert.Equal(400, resp.StatusCode)
		fakeHandler.AssertExpectations(t)
		fakeHealth.AssertExpectations(t)
	})

	t.Run("TestServeHTTPWrongHeader", func(t *testing.T) {
		req.Header.Del("Content-Type")

		w := httptest.NewRecorder()
		serverWrapper.ServeHTTP(w, req)
		resp := w.Result()

		assert.Equal(400, resp.StatusCode)
		fakeHandler.AssertExpectations(t)
		fakeHealth.AssertExpectations(t)
	})

	t.Run("TestServeHTTPFullQueue", func(t *testing.T) {
		req.Header.Set("Content-Type", "text/plain")

		w := httptest.NewRecorder()
		requestTimeout := func(func(workerID int)) error {
			return errors.New("Intentional error.")
		}
		serverWrapper.doJob = requestTimeout
		serverWrapper.ServeHTTP(w, req)
		resp := w.Result()

		assert.Equal(408, resp.StatusCode)
		fakeHandler.AssertExpectations(t)
		fakeHealth.AssertExpectations(t)
	})
}
<commit_msg>Fixing unit test for when worker pool queue is full.<commit_after>package main

import (
	"errors"
	"github.com/Comcast/webpa-common/health"
	"github.com/Comcast/webpa-common/logging"
	"github.com/stretchr/testify/assert"
	"github.com/stretchr/testify/mock"
	"github.com/stretchr/testify/require"
	"math"
	"net/http"
	"net/http/httptest"
	"os"
	"strings"
	"sync"
	"testing"
)

// Begin mock declarations

// mockHandler only needs to mock the `HandleRequest` method
type mockHandler struct {
	mock.Mock
}

func (m *mockHandler) HandleRequest(workerID int, inRequest CaduceusRequest) {
	m.Called(workerID, inRequest)
}

// mockHealthTracker needs to mock things from both the `HealthTracker`
// interface as well as the `health.Monitor` interface
type mockHealthTracker struct {
	mock.Mock
}

func (m *mockHealthTracker) SendEvent(healthFunc health.HealthFunc) {
	m.Called(healthFunc)
}

func (m *mockHealthTracker) ServeHTTP(response http.ResponseWriter, request *http.Request) {
	m.Called(response, request)
}

func (m *mockHealthTracker) IncrementBucket(inSize int) {
	m.Called(inSize)
}

// Begin test functions

func TestMain(m *testing.M) {
	os.Exit(m.Run())
}

func TestWorkerPool(t *testing.T) {
	assert := assert.New(t)

	workerPool := WorkerPoolFactory{
		NumWorkers: 1,
		QueueSize:  1,
	}.New()

	t.Run("TestWorkerPoolSend", func(t *testing.T) {
		testWG := new(sync.WaitGroup)
		testWG.Add(1)

		require.NotNil(t, workerPool)
		err := workerPool.Send(func(workerID int) {
			testWG.Done()
		})

		testWG.Wait()
		assert.Nil(err)
	})

	workerPool = WorkerPoolFactory{
		NumWorkers: 0,
		QueueSize:  0,
	}.New()

	t.Run("TestWorkerPoolFullQueue", func(t *testing.T) {
		require.NotNil(t, workerPool)
		err := workerPool.Send(func(workerID int) {
			assert.Fail("This should not execute because our worker queue is full and we have no workers.")
		})

		assert.NotNil(err)
	})
}

func TestCaduceusHealth(t *testing.T) {
	assert := assert.New(t)

	testData := []struct {
		inSize       int
		expectedStat health.Stat
	}{
		{inSize: -1, expectedStat: PayloadsOverZero},
		{inSize: 0, expectedStat: PayloadsOverZero},
		{inSize: 99, expectedStat: PayloadsOverZero},
		{inSize: 999, expectedStat: PayloadsOverHundred},
		{inSize: 9999, expectedStat: PayloadsOverThousand},
		{inSize: 10001, expectedStat: PayloadsOverTenThousand},
		{inSize: math.MaxInt32, expectedStat: PayloadsOverTenThousand},
	}

	t.Run("TestIncrementBucket", func(t *testing.T) {
		for _, data := range testData {
			fakeMonitor := new(mockHealthTracker)
			fakeMonitor.On("SendEvent", mock.AnythingOfType("health.HealthFunc")).Run(
				func(args mock.Arguments) {
					healthFunc := args.Get(0).(health.HealthFunc)
					stats := make(health.Stats)

					healthFunc(stats)
					assert.Equal(1, stats[data.expectedStat])
				}).Once()

			caduceusHealth := &CaduceusHealth{fakeMonitor}

			caduceusHealth.IncrementBucket(data.inSize)
			fakeMonitor.AssertExpectations(t)
		}
	})
}

func TestServeHandler(t *testing.T) {
	assert := assert.New(t)

	logger := logging.DefaultLogger()
	fakeHandler := new(mockHandler)
	fakeHandler.On("HandleRequest", mock.AnythingOfType("int"), mock.AnythingOfType("CaduceusRequest")).Return().Once()

	fakeHealth := new(mockHealthTracker)
	fakeHealth.On("IncrementBucket", mock.AnythingOfType("int")).Return().Once()

	requestSuccessful := func(func(workerID int)) error {
		fakeHandler.HandleRequest(0, CaduceusRequest{})
		return nil
	}

	serverWrapper := &ServerHandler{
		Logger:          logger,
		caduceusHandler: fakeHandler,
		caduceusHealth:  fakeHealth,
		doJob:           requestSuccessful,
	}

	req := httptest.NewRequest("POST", "localhost:8080", strings.NewReader("Test payload."))

	t.Run("TestServeHTTPHappyPath", func(t *testing.T) {
		req.Header.Set("Content-Type", "text/plain")

		w := httptest.NewRecorder()
		serverWrapper.ServeHTTP(w, req)
		resp := w.Result()

		assert.Equal(202, resp.StatusCode)
		fakeHandler.AssertExpectations(t)
		fakeHealth.AssertExpectations(t)
	})

	t.Run("TestServeHTTPTooManyHeaders", func(t *testing.T) {
		req.Header.Add("Content-Type", "too/many/headers")

		w := httptest.NewRecorder()
		serverWrapper.ServeHTTP(w, req)
		resp := w.Result()

		assert.Equal(400, resp.StatusCode)
		fakeHandler.AssertExpectations(t)
		fakeHealth.AssertExpectations(t)
	})

	t.Run("TestServeHTTPWrongHeader", func(t *testing.T) {
		req.Header.Del("Content-Type")

		w := httptest.NewRecorder()
		serverWrapper.ServeHTTP(w, req)
		resp := w.Result()

		assert.Equal(400, resp.StatusCode)
		fakeHandler.AssertExpectations(t)
		fakeHealth.AssertExpectations(t)
	})

	t.Run("TestServeHTTPFullQueue", func(t *testing.T) {
		req.Header.Set("Content-Type", "text/plain")

		w := httptest.NewRecorder()
		requestTimeout := func(func(workerID int)) error {
			return errors.New("Intentional error.")
		}
		serverWrapper.doJob = requestTimeout
		serverWrapper.ServeHTTP(w, req)
		resp := w.Result()

		assert.Equal(408, resp.StatusCode)
		fakeHandler.AssertExpectations(t)
		fakeHealth.AssertExpectations(t)
	})
}
<|endoftext|>


<commit_before>// Copyright 2014, Hǎiliàng Wáng. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

package html
<commit_msg>remove unused code.<commit_after><|endoftext|>


<commit_before>package main

import (
	"bytes"
	"encoding/json"
	"fmt"
	"log"
	"net/http"
	"runtime"
	"strconv"
	"strings"
	"sync"
	"time"

	"github.com/mingrammer/go-codelab/models"
)

// worker struct has necessary information to run sensor data producing job
type worker struct {
	ticker      *time.Ticker
	sensor      models.Sensor
	sensorError float64
	serverPort  int
}

// sensorWorker will be ran in goroutine to produce (random) sensor data and sends it to server
func sensorWorker(done <-chan struct{}, w worker) {
	for {
		select {
		case <-done:
			return
		case <-ticker.C:
			sensorData := sensor.GenerateSensorData(w.sensorError)
			url := getRequestServerURL(w.serverPort)

			fmt.Println(w.sensorData.SendingOutputString())

			sendJSONSensorData(w.url, w.sensorData)
		}
	}
}

func main() {
	numCPUs := runtime.NumCPU()
	runtime.GOMAXPROCS(numCPUs)

	var wg sync.WaitGroup
	const numWorkers = 3

	done := make(chan struct{})

	wg.Add(numWorkers)

	workerList := [numWorkers]worker{
		worker{
			ticker:      time.NewTicker(500 * time.Millisecond),
			sensor:      models.GyroSensor{},
			sensorError: 4.0,
			serverPort:  8001,
		},
		worker{
			ticker:      time.NewTicker(500 * time.Millisecond),
			sensor:      models.AccelSensor{},
			sensorError: 12.0,
			serverPort:  8002,
		},
		worker{
			ticker:      time.NewTicker(2 * time.Second),
			sensor:      models.TempSensor{},
			sensorError: 1.5,
			serverPort:  8003,
		},
	}

	for _, w := range workerList {
		go func(w worker) {
			sensorWorker(done, w)
			wg.Done()
		}(w)
	}

	wg.Wait()
}

// getRequestServerURL just concatenate the server url and port number to get complete server url
func getRequestServerURL(port int) string {
	urlComponents := []string{"http://127.0.0.1", strconv.Itoa(port)}

	return strings.Join(urlComponents, ":")
}

// sendJSONSensorData sends the sensor data as JSON type to server
func sendJSONSensorData(url string, sensorValues interface{}) {
	jsonBytes, err := json.Marshal(sensorValues)
	if err != nil {
		log.Fatal("Error occurs when mashaling the gyro sensor values")
	}

	buff := bytes.NewBuffer(jsonBytes)

	resp, err := http.Post(url, "application/json", buff)
	if err != nil || resp.StatusCode != 200 {
		log.Fatal("Error occurs when request the post data")
	}
}
<commit_msg>Fix bug about wrong access expression of worker (mistakes)<commit_after>package main

import (
	"bytes"
	"encoding/json"
	"fmt"
	"log"
	"net/http"
	"runtime"
	"strconv"
	"strings"
	"sync"
	"time"

	"github.com/mingrammer/go-codelab/models"
)

// worker struct has necessary information to run sensor data producing job
type worker struct {
	ticker      *time.Ticker
	sensor      models.Sensor
	sensorError float64
	serverPort  int
}

// sensorWorker will be ran in goroutine to produce (random) sensor data and sends it to server
func sensorWorker(done <-chan struct{}, w worker) {
	for {
		select {
		case <-done:
			return
		case <-w.ticker.C:
			sensorData := w.sensor.GenerateSensorData(w.sensorError)
			url := getRequestServerURL(w.serverPort)

			fmt.Println(sensorData.SendingOutputString())

			sendJSONSensorData(url, sensorData)
		}
	}
}

func main() {
	numCPUs := runtime.NumCPU()
	runtime.GOMAXPROCS(numCPUs)

	var wg sync.WaitGroup
	const numWorkers = 3

	done := make(chan struct{})

	wg.Add(numWorkers)

	workerList := [numWorkers]worker{
		worker{
			ticker:      time.NewTicker(500 * time.Millisecond),
			sensor:      models.GyroSensor{},
			sensorError: 4.0,
			serverPort:  8001,
		},
		worker{
			ticker:      time.NewTicker(500 * time.Millisecond),
			sensor:      models.AccelSensor{},
			sensorError: 12.0,
			serverPort:  8002,
		},
		worker{
			ticker:      time.NewTicker(2 * time.Second),
			sensor:      models.TempSensor{},
			sensorError: 1.5,
			serverPort:  8003,
		},
	}

	for _, w := range workerList {
		go func(w worker) {
			sensorWorker(done, w)
			wg.Done()
		}(w)
	}

	wg.Wait()
}

// getRequestServerURL just concatenate the server url and port number to get complete server url
func getRequestServerURL(port int) string {
	urlComponents := []string{"http://127.0.0.1", strconv.Itoa(port)}

	return strings.Join(urlComponents, ":")
}

// sendJSONSensorData sends the sensor data as JSON type to server
func sendJSONSensorData(url string, sensorValues interface{}) {
	jsonBytes, err := json.Marshal(sensorValues)
	if err != nil {
		log.Fatal("Error occurs when mashaling the gyro sensor values")
	}

	buff := bytes.NewBuffer(jsonBytes)

	resp, err := http.Post(url, "application/json", buff)
	if err != nil || resp.StatusCode != 200 {
		log.Fatal("Error occurs when request the post data")
	}
}
<|endoftext|>


<commit_before>package serf

import (
	"time"
)

type coalesceEvent struct {
	Type   EventType
	Member *Member
}

// coalescedEventCh returns an event channel where the events are coalesced
// over a period of time. This helps lower the number of events that are
// fired in the case where many nodes share similar events at one time.
// Examples where this is possible are if many new nodes are brought online
// at one time, events will be coalesced together into one event.
func coalescedEventCh(outCh chan<- Event, shutdownCh <-chan struct{},
	coalescePeriod time.Duration, quiescentPeriod time.Duration) chan<- Event {
	eventCh := make(chan Event)
	go coalescer(eventCh, outCh, shutdownCh, coalescePeriod, quiescentPeriod)
	return eventCh
}

// coalescer is the function that actually does the coalescence work.
// This function runs in a goroutine, processing events until the
// shutdown channel sends a message.
func coalescer(eventCh <-chan Event, newCh chan<- Event, shutdownCh <-chan struct{},
	coalescePeriod time.Duration, quiescentPeriod time.Duration) {
	var quiescent <-chan time.Time
	var quantum <-chan time.Time

	lastEvents := make(map[string]EventType)
	latestEvents := make(map[string]coalesceEvent)
	shutdown := false

	for {
		coalesce := false

		select {
		case rawEvent := <-eventCh:
			// Ignore any non-member related events
			eventType := rawEvent.EventType()
			if eventType != EventMemberJoin && eventType != EventMemberLeave && eventType != EventMemberFailed {
				newCh <- rawEvent
				continue
			}

			// Cast to a member event
			e := rawEvent.(MemberEvent)

			// Start a new quantum if we need to and update the quiescent
			// timer.
			if quantum == nil {
				quantum = time.After(coalescePeriod)
			}
			quiescent = time.After(quiescentPeriod)

			for _, m := range e.Members {
				latestEvents[m.Name] = coalesceEvent{
					Type:   e.Type,
					Member: &m,
				}
			}
		case <-quantum:
			coalesce = true
		case <-quiescent:
			coalesce = true
		case <-shutdownCh:
			// Make sure we coalesce one last time and then shut down
			coalesce = true
			shutdown = true
		}

		if !coalesce {
			continue
		}

		// Reset the timers
		quantum = nil
		quiescent = nil

		// Coalesce the various events we got into a single set of events.
		events := make(map[EventType]*MemberEvent)
		for name, cevent := range latestEvents {
			previous, ok := lastEvents[name]

			// If we sent the same event before, then ignore
			if ok && previous == cevent.Type {
				continue
			}

			// Update our last event
			lastEvents[name] = cevent.Type

			// Add it to our event
			newEvent, ok := events[cevent.Type]
			if !ok {
				newEvent = &MemberEvent{Type: cevent.Type}
				events[cevent.Type] = newEvent
			}
			newEvent.Members = append(newEvent.Members, *cevent.Member)
		}

		// Send out those events
		for _, event := range events {
			newCh <- *event
		}

		// If we were told to shutdown, then exit now
		if shutdown {
			break
		}
	}
}
<commit_msg>serf: buffer the coalesce channel<commit_after>package serf

import (
	"time"
)

type coalesceEvent struct {
	Type   EventType
	Member *Member
}

// coalescedEventCh returns an event channel where the events are coalesced
// over a period of time. This helps lower the number of events that are
// fired in the case where many nodes share similar events at one time.
// Examples where this is possible are if many new nodes are brought online
// at one time, events will be coalesced together into one event.
func coalescedEventCh(outCh chan<- Event, shutdownCh <-chan struct{},
	coalescePeriod time.Duration, quiescentPeriod time.Duration) chan<- Event {
	eventCh := make(chan Event, 1024)
	go coalescer(eventCh, outCh, shutdownCh, coalescePeriod, quiescentPeriod)
	return eventCh
}

// coalescer is the function that actually does the coalescence work.
// This function runs in a goroutine, processing events until the
// shutdown channel sends a message.
func coalescer(eventCh <-chan Event, newCh chan<- Event, shutdownCh <-chan struct{},
	coalescePeriod time.Duration, quiescentPeriod time.Duration) {
	var quiescent <-chan time.Time
	var quantum <-chan time.Time

	lastEvents := make(map[string]EventType)
	latestEvents := make(map[string]coalesceEvent)
	shutdown := false

	for {
		coalesce := false

		select {
		case rawEvent := <-eventCh:
			// Ignore any non-member related events
			eventType := rawEvent.EventType()
			if eventType != EventMemberJoin && eventType != EventMemberLeave && eventType != EventMemberFailed {
				newCh <- rawEvent
				continue
			}

			// Cast to a member event
			e := rawEvent.(MemberEvent)

			// Start a new quantum if we need to and update the quiescent
			// timer.
			if quantum == nil {
				quantum = time.After(coalescePeriod)
			}
			quiescent = time.After(quiescentPeriod)

			for _, m := range e.Members {
				latestEvents[m.Name] = coalesceEvent{
					Type:   e.Type,
					Member: &m,
				}
			}
		case <-quantum:
			coalesce = true
		case <-quiescent:
			coalesce = true
		case <-shutdownCh:
			// Make sure we coalesce one last time and then shut down
			coalesce = true
			shutdown = true
		}

		if !coalesce {
			continue
		}

		// Reset the timers
		quantum = nil
		quiescent = nil

		// Coalesce the various events we got into a single set of events.
		events := make(map[EventType]*MemberEvent)
		for name, cevent := range latestEvents {
			previous, ok := lastEvents[name]

			// If we sent the same event before, then ignore
			if ok && previous == cevent.Type {
				continue
			}

			// Update our last event
			lastEvents[name] = cevent.Type

			// Add it to our event
			newEvent, ok := events[cevent.Type]
			if !ok {
				newEvent = &MemberEvent{Type: cevent.Type}
				events[cevent.Type] = newEvent
			}
			newEvent.Members = append(newEvent.Members, *cevent.Member)
		}

		// Send out those events
		for _, event := range events {
			newCh <- *event
		}

		// If we were told to shutdown, then exit now
		if shutdown {
			break
		}
	}
}
<|endoftext|>