<commit_before>/*
Copyright 2019 The Kubernetes Authors.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package testutil

import (
	"fmt"
	"io"
	"math"
	"reflect"
	"sort"
	"strings"

	dto "github.com/prometheus/client_model/go"
	"github.com/prometheus/common/expfmt"
	"github.com/prometheus/common/model"

	"k8s.io/component-base/metrics"
)

var (
	// MetricNameLabel is label under which model.Sample stores metric name
	MetricNameLabel model.LabelName = model.MetricNameLabel
	// QuantileLabel is label under which model.Sample stores latency quantile value
	QuantileLabel model.LabelName = model.QuantileLabel
)

// Metrics is generic metrics for other specific metrics
type Metrics map[string]model.Samples

// Equal returns true if all metrics are the same as the arguments.
func (m *Metrics) Equal(o Metrics) bool {
	var leftKeySet []string
	var rightKeySet []string
	for k := range *m {
		leftKeySet = append(leftKeySet, k)
	}
	for k := range o {
		rightKeySet = append(rightKeySet, k)
	}
	if !reflect.DeepEqual(leftKeySet, rightKeySet) {
		return false
	}
	for _, k := range leftKeySet {
		if !(*m)[k].Equal(o[k]) {
			return false
		}
	}
	return true
}

// NewMetrics returns new metrics which are initialized.
func NewMetrics() Metrics {
	result := make(Metrics)
	return result
}

// ParseMetrics parses Metrics from data returned from prometheus endpoint
func ParseMetrics(data string, output *Metrics) error {
	dec := expfmt.NewDecoder(strings.NewReader(data), expfmt.FmtText)
	decoder := expfmt.SampleDecoder{
		Dec:  dec,
		Opts: &expfmt.DecodeOptions{},
	}

	for {
		var v model.Vector
		if err := decoder.Decode(&v); err != nil {
			if err == io.EOF {
				// Expected loop termination condition.
				return nil
			}
			continue
		}
		for _, metric := range v {
			name := string(metric.Metric[model.MetricNameLabel])
			(*output)[name] = append((*output)[name], metric)
		}
	}
}

// TextToMetricFamilies reads 'in' as the simple and flat text-based exchange
// format and creates MetricFamily proto messages. It returns the MetricFamily
// proto messages in a map where the metric names are the keys, along with any
// error encountered.
func TextToMetricFamilies(in io.Reader) (map[string]*dto.MetricFamily, error) {
	var textParser expfmt.TextParser
	return textParser.TextToMetricFamilies(in)
}

// ExtractMetricSamples parses the prometheus metric samples from the input string.
func ExtractMetricSamples(metricsBlob string) ([]*model.Sample, error) {
	dec := expfmt.NewDecoder(strings.NewReader(metricsBlob), expfmt.FmtText)
	decoder := expfmt.SampleDecoder{
		Dec:  dec,
		Opts: &expfmt.DecodeOptions{},
	}

	var samples []*model.Sample
	for {
		var v model.Vector
		if err := decoder.Decode(&v); err != nil {
			if err == io.EOF {
				// Expected loop termination condition.
				return samples, nil
			}
			return nil, err
		}
		samples = append(samples, v...)
	}
}

// PrintSample returns formatted representation of metric Sample
func PrintSample(sample *model.Sample) string {
	buf := make([]string, 0)
	// Id is a VERY special label. For 'normal' container it's useless, but it's necessary
	// for 'system' containers (e.g. /docker-daemon, /kubelet, etc.). We know if that's the
	// case by checking if there's a label "kubernetes_container_name" present. It's hacky
	// but it works...
	_, normalContainer := sample.Metric["kubernetes_container_name"]
	for k, v := range sample.Metric {
		if strings.HasPrefix(string(k), "__") {
			continue
		}

		if string(k) == "id" && normalContainer {
			continue
		}
		buf = append(buf, fmt.Sprintf("%v=%v", string(k), v))
	}
	return fmt.Sprintf("[%v] = %v", strings.Join(buf, ","), sample.Value)
}

// ComputeHistogramDelta computes the change in histogram metric for a selected label.
// Results are stored in after samples
func ComputeHistogramDelta(before, after model.Samples, label model.LabelName) {
	beforeSamplesMap := make(map[string]*model.Sample)
	for _, bSample := range before {
		beforeSamplesMap[makeKey(bSample.Metric[label], bSample.Metric["le"])] = bSample
	}
	for _, aSample := range after {
		if bSample, found := beforeSamplesMap[makeKey(aSample.Metric[label], aSample.Metric["le"])]; found {
			aSample.Value = aSample.Value - bSample.Value
		}
	}
}

func makeKey(a, b model.LabelValue) string {
	return string(a) + "___" + string(b)
}

// GetMetricValuesForLabel returns value of metric for a given dimension
func GetMetricValuesForLabel(ms Metrics, metricName, label string) map[string]int64 {
	samples, found := ms[metricName]
	result := make(map[string]int64, len(samples))
	if !found {
		return result
	}
	for _, sample := range samples {
		count := int64(sample.Value)
		dimensionName := string(sample.Metric[model.LabelName(label)])
		result[dimensionName] = count
	}
	return result
}

// ValidateMetrics verifies if every sample of metric has all expected labels
func ValidateMetrics(metrics Metrics, metricName string, expectedLabels ...string) error {
	samples, ok := metrics[metricName]
	if !ok {
		return fmt.Errorf("metric %q was not found in metrics", metricName)
	}
	for _, sample := range samples {
		for _, l := range expectedLabels {
			if _, ok := sample.Metric[model.LabelName(l)]; !ok {
				return fmt.Errorf("metric %q is missing label %q, sample: %q", metricName, l, sample.String())
			}
		}
	}
	return nil
}

// Histogram wraps prometheus histogram DTO (data transfer object)
type Histogram struct {
	*dto.Histogram
}

// GetHistogramFromGatherer collects a metric from a gatherer implementing k8s.io/component-base/metrics.Gatherer interface.
// Used only for testing purposes where we need to gather metrics directly from a running binary (without metrics endpoint).
func GetHistogramFromGatherer(gatherer metrics.Gatherer, metricName string) (Histogram, error) {
	var metricFamily *dto.MetricFamily
	m, err := gatherer.Gather()
	if err != nil {
		return Histogram{}, err
	}
	for _, mFamily := range m {
		if mFamily.GetName() == metricName {
			metricFamily = mFamily
			break
		}
	}

	if metricFamily == nil {
		return Histogram{}, fmt.Errorf("metric %q not found", metricName)
	}

	if metricFamily.GetMetric() == nil {
		return Histogram{}, fmt.Errorf("metric %q is empty", metricName)
	}

	if len(metricFamily.GetMetric()) == 0 {
		return Histogram{}, fmt.Errorf("metric %q is empty", metricName)
	}

	return Histogram{
		// Histograms are stored under the first index (based on observation).
		// Given there's only one histogram registered per each metric name, accessing
		// the first index is sufficient.
		metricFamily.GetMetric()[0].GetHistogram(),
	}, nil
}

func uint64Ptr(u uint64) *uint64 {
	return &u
}

// Bucket of a histogram
type bucket struct {
	upperBound float64
	count      float64
}

func bucketQuantile(q float64, buckets []bucket) float64 {
	if q < 0 {
		return math.Inf(-1)
	}
	if q > 1 {
		return math.Inf(+1)
	}

	if len(buckets) < 2 {
		return math.NaN()
	}

	rank := q * buckets[len(buckets)-1].count
	b := sort.Search(len(buckets)-1, func(i int) bool { return buckets[i].count >= rank })

	if b == 0 {
		return buckets[0].upperBound * (rank / buckets[0].count)
	}

	if b == len(buckets)-1 && math.IsInf(buckets[b].upperBound, 1) {
		return buckets[len(buckets)-2].upperBound
	}

	// linear approximation of b-th bucket
	brank := rank - buckets[b-1].count
	bSize := buckets[b].upperBound - buckets[b-1].upperBound
	bCount := buckets[b].count - buckets[b-1].count

	return buckets[b-1].upperBound + bSize*(brank/bCount)
}

// Quantile computes q-th quantile of a cumulative histogram.
// It's expected the histogram is valid (by calling Validate)
func (hist *Histogram) Quantile(q float64) float64 {
	var buckets []bucket

	for _, bckt := range hist.Bucket {
		buckets = append(buckets, bucket{
			count:      float64(bckt.GetCumulativeCount()),
			upperBound: bckt.GetUpperBound(),
		})
	}

	if len(buckets) == 0 || buckets[len(buckets)-1].upperBound != math.Inf(+1) {
		// The list of buckets in dto.Histogram doesn't include the final +Inf bucket, so we
		// add it here for the reset of the samples.
		buckets = append(buckets, bucket{
			count:      float64(hist.GetSampleCount()),
			upperBound: math.Inf(+1),
		})
	}

	return bucketQuantile(q, buckets)
}

// Average computes histogram's average value
func (hist *Histogram) Average() float64 {
	return hist.GetSampleSum() / float64(hist.GetSampleCount())
}

// Clear clears all fields of the wrapped histogram
func (hist *Histogram) Clear() {
	if hist.SampleCount != nil {
		*hist.SampleCount = 0
	}
	if hist.SampleSum != nil {
		*hist.SampleSum = 0
	}
	for _, b := range hist.Bucket {
		if b.CumulativeCount != nil {
			*b.CumulativeCount = 0
		}
	}
}

// Validate makes sure the wrapped histogram has all necessary fields set and with valid values.
func (hist *Histogram) Validate() error {
	if hist.SampleCount == nil || hist.GetSampleCount() == 0 {
		return fmt.Errorf("nil or empty histogram SampleCount")
	}

	if hist.SampleSum == nil || hist.GetSampleSum() == 0 {
		return fmt.Errorf("nil or empty histogram SampleSum")
	}

	for _, bckt := range hist.Bucket {
		if bckt == nil {
			return fmt.Errorf("empty histogram bucket")
		}
		if bckt.UpperBound == nil || bckt.GetUpperBound() < 0 {
			return fmt.Errorf("nil or negative histogram bucket UpperBound")
		}
	}

	return nil
}

// GetGaugeMetricValue extract metric value from GaugeMetric
func GetGaugeMetricValue(m metrics.GaugeMetric) (float64, error) {
	metricProto := &dto.Metric{}
	if err := m.Write(metricProto); err != nil {
		return 0, fmt.Errorf("error writing m: %v", err)
	}
	return metricProto.Gauge.GetValue(), nil
}

// GetCounterMetricValue extract metric value from CounterMetric
func GetCounterMetricValue(m metrics.CounterMetric) (float64, error) {
	metricProto := &dto.Metric{}
	if err := m.(metrics.Metric).Write(metricProto); err != nil {
		return 0, fmt.Errorf("error writing m: %v", err)
	}
	return metricProto.Counter.GetValue(), nil
}

// GetHistogramMetricValue extract sum of all samples from ObserverMetric
func GetHistogramMetricValue(m metrics.ObserverMetric) (float64, error) {
	metricProto := &dto.Metric{}
	if err := m.(metrics.Metric).Write(metricProto); err != nil {
		return 0, fmt.Errorf("error writing m: %v", err)
	}
	return metricProto.Histogram.GetSampleSum(), nil
}

// LabelsMatch returns true if metric has all expected labels otherwise false
func LabelsMatch(metric *dto.Metric, labelFilter map[string]string) bool {
	metricLabels := map[string]string{}

	for _, labelPair := range metric.Label {
		metricLabels[labelPair.GetName()] = labelPair.GetValue()
	}

	// length comparison then match key to values in the maps
	if len(labelFilter) > len(metricLabels) {
		return false
	}

	for labelName, labelValue := range labelFilter {
		if value, ok := metricLabels[labelName]; !ok || value != labelValue {
			return false
		}
	}

	return true
}
<commit_msg>clean up testutil/metrics content<commit_after>/*
Copyright 2019 The Kubernetes Authors.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package testutil

import (
	"fmt"
	"io"
	"math"
	"reflect"
	"sort"
	"strings"

	dto "github.com/prometheus/client_model/go"
	"github.com/prometheus/common/expfmt"
	"github.com/prometheus/common/model"

	"k8s.io/component-base/metrics"
)

var (
	// MetricNameLabel is label under which model.Sample stores metric name
	MetricNameLabel model.LabelName = model.MetricNameLabel
	// QuantileLabel is label under which model.Sample stores latency quantile value
	QuantileLabel model.LabelName = model.QuantileLabel
)

// Metrics is generic metrics for other specific metrics
type Metrics map[string]model.Samples

// Equal returns true if all metrics are the same as the arguments.
func (m *Metrics) Equal(o Metrics) bool {
	var leftKeySet []string
	var rightKeySet []string
	for k := range *m {
		leftKeySet = append(leftKeySet, k)
	}
	for k := range o {
		rightKeySet = append(rightKeySet, k)
	}
	if !reflect.DeepEqual(leftKeySet, rightKeySet) {
		return false
	}
	for _, k := range leftKeySet {
		if !(*m)[k].Equal(o[k]) {
			return false
		}
	}
	return true
}

// NewMetrics returns new metrics which are initialized.
func NewMetrics() Metrics {
	result := make(Metrics)
	return result
}

// ParseMetrics parses Metrics from data returned from prometheus endpoint
func ParseMetrics(data string, output *Metrics) error {
	dec := expfmt.NewDecoder(strings.NewReader(data), expfmt.FmtText)
	decoder := expfmt.SampleDecoder{
		Dec:  dec,
		Opts: &expfmt.DecodeOptions{},
	}

	for {
		var v model.Vector
		if err := decoder.Decode(&v); err != nil {
			if err == io.EOF {
				// Expected loop termination condition.
				return nil
			}
			continue
		}
		for _, metric := range v {
			name := string(metric.Metric[MetricNameLabel])
			(*output)[name] = append((*output)[name], metric)
		}
	}
}

// TextToMetricFamilies reads 'in' as the simple and flat text-based exchange
// format and creates MetricFamily proto messages. It returns the MetricFamily
// proto messages in a map where the metric names are the keys, along with any
// error encountered.
func TextToMetricFamilies(in io.Reader) (map[string]*dto.MetricFamily, error) {
	var textParser expfmt.TextParser
	return textParser.TextToMetricFamilies(in)
}

// PrintSample returns formatted representation of metric Sample
func PrintSample(sample *model.Sample) string {
	buf := make([]string, 0)
	// Id is a VERY special label. For 'normal' container it's useless, but it's necessary
	// for 'system' containers (e.g. /docker-daemon, /kubelet, etc.). We know if that's the
	// case by checking if there's a label "kubernetes_container_name" present. It's hacky
	// but it works...
	_, normalContainer := sample.Metric["kubernetes_container_name"]
	for k, v := range sample.Metric {
		if strings.HasPrefix(string(k), "__") {
			continue
		}

		if string(k) == "id" && normalContainer {
			continue
		}
		buf = append(buf, fmt.Sprintf("%v=%v", string(k), v))
	}
	return fmt.Sprintf("[%v] = %v", strings.Join(buf, ","), sample.Value)
}

// ComputeHistogramDelta computes the change in histogram metric for a selected label.
// Results are stored in after samples
func ComputeHistogramDelta(before, after model.Samples, label model.LabelName) {
	beforeSamplesMap := make(map[string]*model.Sample)
	for _, bSample := range before {
		beforeSamplesMap[makeKey(bSample.Metric[label], bSample.Metric["le"])] = bSample
	}
	for _, aSample := range after {
		if bSample, found := beforeSamplesMap[makeKey(aSample.Metric[label], aSample.Metric["le"])]; found {
			aSample.Value = aSample.Value - bSample.Value
		}
	}
}

func makeKey(a, b model.LabelValue) string {
	return string(a) + "___" + string(b)
}

// GetMetricValuesForLabel returns value of metric for a given dimension
func GetMetricValuesForLabel(ms Metrics, metricName, label string) map[string]int64 {
	samples, found := ms[metricName]
	result := make(map[string]int64, len(samples))
	if !found {
		return result
	}
	for _, sample := range samples {
		count := int64(sample.Value)
		dimensionName := string(sample.Metric[model.LabelName(label)])
		result[dimensionName] = count
	}
	return result
}

// ValidateMetrics verifies if every sample of metric has all expected labels
func ValidateMetrics(metrics Metrics, metricName string, expectedLabels ...string) error {
	samples, ok := metrics[metricName]
	if !ok {
		return fmt.Errorf("metric %q was not found in metrics", metricName)
	}
	for _, sample := range samples {
		for _, l := range expectedLabels {
			if _, ok := sample.Metric[model.LabelName(l)]; !ok {
				return fmt.Errorf("metric %q is missing label %q, sample: %q", metricName, l, sample.String())
			}
		}
	}
	return nil
}

// Histogram wraps prometheus histogram DTO (data transfer object)
type Histogram struct {
	*dto.Histogram
}

// GetHistogramFromGatherer collects a metric from a gatherer implementing k8s.io/component-base/metrics.Gatherer interface.
// Used only for testing purposes where we need to gather metrics directly from a running binary (without metrics endpoint).
func GetHistogramFromGatherer(gatherer metrics.Gatherer, metricName string) (Histogram, error) {
	var metricFamily *dto.MetricFamily
	m, err := gatherer.Gather()
	if err != nil {
		return Histogram{}, err
	}
	for _, mFamily := range m {
		if mFamily.GetName() == metricName {
			metricFamily = mFamily
			break
		}
	}

	if metricFamily == nil {
		return Histogram{}, fmt.Errorf("metric %q not found", metricName)
	}

	if metricFamily.GetMetric() == nil {
		return Histogram{}, fmt.Errorf("metric %q is empty", metricName)
	}

	if len(metricFamily.GetMetric()) == 0 {
		return Histogram{}, fmt.Errorf("metric %q is empty", metricName)
	}

	return Histogram{
		// Histograms are stored under the first index (based on observation).
		// Given there's only one histogram registered per each metric name, accessing
		// the first index is sufficient.
		metricFamily.GetMetric()[0].GetHistogram(),
	}, nil
}

func uint64Ptr(u uint64) *uint64 {
	return &u
}

// Bucket of a histogram
type bucket struct {
	upperBound float64
	count      float64
}

func bucketQuantile(q float64, buckets []bucket) float64 {
	if q < 0 {
		return math.Inf(-1)
	}
	if q > 1 {
		return math.Inf(+1)
	}

	if len(buckets) < 2 {
		return math.NaN()
	}

	rank := q * buckets[len(buckets)-1].count
	b := sort.Search(len(buckets)-1, func(i int) bool { return buckets[i].count >= rank })

	if b == 0 {
		return buckets[0].upperBound * (rank / buckets[0].count)
	}

	if b == len(buckets)-1 && math.IsInf(buckets[b].upperBound, 1) {
		return buckets[len(buckets)-2].upperBound
	}

	// linear approximation of b-th bucket
	brank := rank - buckets[b-1].count
	bSize := buckets[b].upperBound - buckets[b-1].upperBound
	bCount := buckets[b].count - buckets[b-1].count

	return buckets[b-1].upperBound + bSize*(brank/bCount)
}

// Quantile computes q-th quantile of a cumulative histogram.
// It's expected the histogram is valid (by calling Validate)
func (hist *Histogram) Quantile(q float64) float64 {
	var buckets []bucket

	for _, bckt := range hist.Bucket {
		buckets = append(buckets, bucket{
			count:      float64(bckt.GetCumulativeCount()),
			upperBound: bckt.GetUpperBound(),
		})
	}

	if len(buckets) == 0 || buckets[len(buckets)-1].upperBound != math.Inf(+1) {
		// The list of buckets in dto.Histogram doesn't include the final +Inf bucket, so we
		// add it here for the reset of the samples.
		buckets = append(buckets, bucket{
			count:      float64(hist.GetSampleCount()),
			upperBound: math.Inf(+1),
		})
	}

	return bucketQuantile(q, buckets)
}

// Average computes histogram's average value
func (hist *Histogram) Average() float64 {
	return hist.GetSampleSum() / float64(hist.GetSampleCount())
}

// Clear clears all fields of the wrapped histogram
func (hist *Histogram) Clear() {
	if hist.SampleCount != nil {
		*hist.SampleCount = 0
	}
	if hist.SampleSum != nil {
		*hist.SampleSum = 0
	}
	for _, b := range hist.Bucket {
		if b.CumulativeCount != nil {
			*b.CumulativeCount = 0
		}
	}
}

// Validate makes sure the wrapped histogram has all necessary fields set and with valid values.
func (hist *Histogram) Validate() error {
	if hist.SampleCount == nil || hist.GetSampleCount() == 0 {
		return fmt.Errorf("nil or empty histogram SampleCount")
	}

	if hist.SampleSum == nil || hist.GetSampleSum() == 0 {
		return fmt.Errorf("nil or empty histogram SampleSum")
	}

	for _, bckt := range hist.Bucket {
		if bckt == nil {
			return fmt.Errorf("empty histogram bucket")
		}
		if bckt.UpperBound == nil || bckt.GetUpperBound() < 0 {
			return fmt.Errorf("nil or negative histogram bucket UpperBound")
		}
	}

	return nil
}

// GetGaugeMetricValue extract metric value from GaugeMetric
func GetGaugeMetricValue(m metrics.GaugeMetric) (float64, error) {
	metricProto := &dto.Metric{}
	if err := m.Write(metricProto); err != nil {
		return 0, fmt.Errorf("error writing m: %v", err)
	}
	return metricProto.Gauge.GetValue(), nil
}

// GetCounterMetricValue extract metric value from CounterMetric
func GetCounterMetricValue(m metrics.CounterMetric) (float64, error) {
	metricProto := &dto.Metric{}
	if err := m.(metrics.Metric).Write(metricProto); err != nil {
		return 0, fmt.Errorf("error writing m: %v", err)
	}
	return metricProto.Counter.GetValue(), nil
}

// GetHistogramMetricValue extract sum of all samples from ObserverMetric
func GetHistogramMetricValue(m metrics.ObserverMetric) (float64, error) {
	metricProto := &dto.Metric{}
	if err := m.(metrics.Metric).Write(metricProto); err != nil {
		return 0, fmt.Errorf("error writing m: %v", err)
	}
	return metricProto.Histogram.GetSampleSum(), nil
}

// LabelsMatch returns true if metric has all expected labels otherwise false
func LabelsMatch(metric *dto.Metric, labelFilter map[string]string) bool {
	metricLabels := map[string]string{}

	for _, labelPair := range metric.Label {
		metricLabels[labelPair.GetName()] = labelPair.GetValue()
	}

	// length comparison then match key to values in the maps
	if len(labelFilter) > len(metricLabels) {
		return false
	}

	for labelName, labelValue := range labelFilter {
		if value, ok := metricLabels[labelName]; !ok || value != labelValue {
			return false
		}
	}

	return true
}
<|endoftext|>


<commit_before>package main

import (
	"encoding/json"
	"flag"
	"fmt"
	"io/ioutil"
	"net/http"
	"os"
	"strconv"
	"strings"

	log "github.com/sirupsen/logrus"
)

// CSPReport is the structure of the HTTP payload the system receives.
type CSPReport struct {
	Body CSPReportBody `json:"csp-report"`
}

// CSPReportBody contains the fields that are nested within the
// violation report.
type CSPReportBody struct {
	DocumentURI        string      `json:"document-uri"`
	Referrer           string      `json:"referrer"`
	BlockedURI         string      `json:"blocked-uri"`
	ViolatedDirective  string      `json:"violated-directive"`
	EffectiveDirective string      `json:"effective-directive"`
	OriginalPolicy     string      `json:"original-policy"`
	Disposition        string      `json:"disposition"`
	ScriptSample       string      `json:"script-sample"`
	StatusCode         interface{} `json:"status-code"`
}

var (
	// Rev is set at build time and holds the revision that the package
	// was created at.
	Rev = "dev"

	// Flag for health check url.
	healthCheckPath = "/_healthcheck"

	// Shared defaults for the logger output. This ensures that we are
	// using the same keys for the `FieldKey` values across both formatters.
	logFieldMapDefaults = log.FieldMap{
		log.FieldKeyTime:  "timestamp",
		log.FieldKeyLevel: "level",
		log.FieldKeyMsg:   "message",
	}

	// Default URI Filter list.
	ignoredBlockedURIs = []string{
		"resource://",
		"chromenull://",
		"chrome-extension://",
		"safari-extension://",
		"mxjscall://",
		"webviewprogressproxy://",
		"res://",
		"mx://",
		"safari-resource://",
		"chromeinvoke://",
		"chromeinvokeimmediate://",
		"mbinit://",
		"opera://",
		"ms-appx://",
		"ms-appx-web://",
		"localhost",
		"127.0.0.1",
		"none://",
		"about:blank",
		"android-webview",
		"ms-browser-extension",
		"wvjbscheme://__wvjb_queue_message__",
		"nativebaiduhd://adblock",
		"bdvideo://error",
	}
)

func init() {
	log.SetOutput(os.Stdout)
	log.SetLevel(log.InfoLevel)
}

func trimEmptyAndComments(s []string) []string {
	var r []string
	for _, str := range s {
		if str == "" {
			continue
		}

		// ignore comments
		if strings.HasPrefix(str, "#") {
			continue
		}

		r = append(r, str)
	}
	return r
}

func main() {
	version := flag.Bool("version", false, "Display the version")
	debugFlag := flag.Bool("debug", false, "Output additional logging for debugging")
	outputFormat := flag.String("output-format", "text", "Define how the violation reports are formatted for output.\nDefaults to 'text'. Valid options are 'text' or 'json'")
	blockedURIFile := flag.String("filter-file", "", "Blocked URI Filter file")
	listenPort := flag.Int("port", 8080, "Port to listen on")
	flag.StringVar(&healthCheckPath, "health-check-path", healthCheckPath, "Health checker path")

	flag.Parse()

	if *version {
		fmt.Printf("csp-collector (%s)\n", Rev)
		os.Exit(0)
	}

	if *debugFlag {
		log.SetLevel(log.DebugLevel)
	}

	if *outputFormat == "json" {
		log.SetFormatter(&log.JSONFormatter{
			FieldMap: logFieldMapDefaults,
		})
	} else {
		log.SetFormatter(&log.TextFormatter{
			FullTimestamp:          true,
			DisableLevelTruncation: true,
			QuoteEmptyFields:       true,
			DisableColors:          true,
			FieldMap:               logFieldMapDefaults,
		})
	}

	log.Debug("Starting up...")
	if *blockedURIFile != "" {
		log.Debugf("Using Filter list from file at: %s\n", *blockedURIFile)

		content, err := ioutil.ReadFile(*blockedURIFile)
		if err != nil {
			log.Fatalf("Error reading Blocked File list: %s", *blockedURIFile)
		}
		ignoredBlockedURIs = trimEmptyAndComments(strings.Split(string(content), "\n"))
	} else {
		log.Debug("Using Filter list from internal list")
	}

	log.Debugf("Blocked URI List: %s", ignoredBlockedURIs)
	log.Debugf("Listening on TCP Port: %s", strconv.Itoa(*listenPort))

	http.HandleFunc(healthCheckPath, func(w http.ResponseWriter, r *http.Request) {
		if r.Method != http.MethodGet {
			http.Error(w, "Method not allowed", http.StatusMethodNotAllowed)
			return
		}

		w.WriteHeader(http.StatusOK)
	})

	http.HandleFunc("/", handleViolationReport)
	log.Fatal(http.ListenAndServe(fmt.Sprintf(":%s", strconv.Itoa(*listenPort)), nil))
}

func handleViolationReport(w http.ResponseWriter, r *http.Request) {
	if r.Method != "POST" {
		w.WriteHeader(http.StatusMethodNotAllowed)
		log.WithFields(log.Fields{
			"http_method": r.Method,
		}).Debug("Received invalid HTTP method")
		return
	}

	decoder := json.NewDecoder(r.Body)
	var report CSPReport

	err := decoder.Decode(&report)
	if err != nil {
		w.WriteHeader(http.StatusUnprocessableEntity)
		log.Debugf("Unable to decode invalid JSON payload: %s", err)
		return
	}
	defer r.Body.Close()

	reportValidation := validateViolation(report)
	if reportValidation != nil {
		http.Error(w, reportValidation.Error(), http.StatusBadRequest)
		log.Debugf("Received invalid payload: %s", reportValidation.Error())
		return
	}

	metadatas, gotMetadata := r.URL.Query()["metadata"]
	var metadata string
	if gotMetadata {
		metadata = metadatas[0]
	}

	log.WithFields(log.Fields{
		"document_uri":        report.Body.DocumentURI,
		"referrer":            report.Body.Referrer,
		"blocked_uri":         report.Body.BlockedURI,
		"violated_directive":  report.Body.ViolatedDirective,
		"effective_directive": report.Body.EffectiveDirective,
		"original_policy":     report.Body.OriginalPolicy,
		"disposition":         report.Body.Disposition,
		"script_sample":       report.Body.ScriptSample,
		"status_code":         report.Body.StatusCode,
		"metadata":            metadata,
	}).Info()
}

func validateViolation(r CSPReport) error {
	for _, value := range ignoredBlockedURIs {
		if strings.HasPrefix(r.Body.BlockedURI, value) {
			err := fmt.Errorf("blocked URI ('%s') is an invalid resource", value)
			return err
		}
	}

	if !strings.HasPrefix(r.Body.DocumentURI, "http") {
		return fmt.Errorf("document URI ('%s') is invalid", r.Body.DocumentURI)
	}

	return nil
}
<commit_msg>Localize health check path<commit_after>package main

import (
	"encoding/json"
	"flag"
	"fmt"
	"io/ioutil"
	"net/http"
	"os"
	"strconv"
	"strings"

	log "github.com/sirupsen/logrus"
)

// CSPReport is the structure of the HTTP payload the system receives.
type CSPReport struct {
	Body CSPReportBody `json:"csp-report"`
}

// CSPReportBody contains the fields that are nested within the
// violation report.
type CSPReportBody struct {
	DocumentURI        string      `json:"document-uri"`
	Referrer           string      `json:"referrer"`
	BlockedURI         string      `json:"blocked-uri"`
	ViolatedDirective  string      `json:"violated-directive"`
	EffectiveDirective string      `json:"effective-directive"`
	OriginalPolicy     string      `json:"original-policy"`
	Disposition        string      `json:"disposition"`
	ScriptSample       string      `json:"script-sample"`
	StatusCode         interface{} `json:"status-code"`
}

const (
	// Default health check url.
	defaultHealthCheckPath = "/_healthcheck"
)

var (
	// Rev is set at build time and holds the revision that the package
	// was created at.
	Rev = "dev"

	// Shared defaults for the logger output. This ensures that we are
	// using the same keys for the `FieldKey` values across both formatters.
	logFieldMapDefaults = log.FieldMap{
		log.FieldKeyTime:  "timestamp",
		log.FieldKeyLevel: "level",
		log.FieldKeyMsg:   "message",
	}

	// Default URI Filter list.
	ignoredBlockedURIs = []string{
		"resource://",
		"chromenull://",
		"chrome-extension://",
		"safari-extension://",
		"mxjscall://",
		"webviewprogressproxy://",
		"res://",
		"mx://",
		"safari-resource://",
		"chromeinvoke://",
		"chromeinvokeimmediate://",
		"mbinit://",
		"opera://",
		"ms-appx://",
		"ms-appx-web://",
		"localhost",
		"127.0.0.1",
		"none://",
		"about:blank",
		"android-webview",
		"ms-browser-extension",
		"wvjbscheme://__wvjb_queue_message__",
		"nativebaiduhd://adblock",
		"bdvideo://error",
	}
)

func init() {
	log.SetOutput(os.Stdout)
	log.SetLevel(log.InfoLevel)
}

func trimEmptyAndComments(s []string) []string {
	var r []string
	for _, str := range s {
		if str == "" {
			continue
		}

		// ignore comments
		if strings.HasPrefix(str, "#") {
			continue
		}

		r = append(r, str)
	}
	return r
}

func main() {
	version := flag.Bool("version", false, "Display the version")
	debugFlag := flag.Bool("debug", false, "Output additional logging for debugging")
	outputFormat := flag.String("output-format", "text", "Define how the violation reports are formatted for output.\nDefaults to 'text'. Valid options are 'text' or 'json'")
	blockedURIFile := flag.String("filter-file", "", "Blocked URI Filter file")
	listenPort := flag.Int("port", 8080, "Port to listen on")
	healthCheckPath := flag.String("health-check-path", defaultHealthCheckPath, "Health checker path")

	flag.Parse()

	if *version {
		fmt.Printf("csp-collector (%s)\n", Rev)
		os.Exit(0)
	}

	if *debugFlag {
		log.SetLevel(log.DebugLevel)
	}

	if *outputFormat == "json" {
		log.SetFormatter(&log.JSONFormatter{
			FieldMap: logFieldMapDefaults,
		})
	} else {
		log.SetFormatter(&log.TextFormatter{
			FullTimestamp:          true,
			DisableLevelTruncation: true,
			QuoteEmptyFields:       true,
			DisableColors:          true,
			FieldMap:               logFieldMapDefaults,
		})
	}

	log.Debug("Starting up...")
	if *blockedURIFile != "" {
		log.Debugf("Using Filter list from file at: %s\n", *blockedURIFile)

		content, err := ioutil.ReadFile(*blockedURIFile)
		if err != nil {
			log.Fatalf("Error reading Blocked File list: %s", *blockedURIFile)
		}
		ignoredBlockedURIs = trimEmptyAndComments(strings.Split(string(content), "\n"))
	} else {
		log.Debug("Using Filter list from internal list")
	}

	log.Debugf("Blocked URI List: %s", ignoredBlockedURIs)
	log.Debugf("Listening on TCP Port: %s", strconv.Itoa(*listenPort))

	http.HandleFunc(*healthCheckPath, func(w http.ResponseWriter, r *http.Request) {
		if r.Method != http.MethodGet {
			http.Error(w, "Method not allowed", http.StatusMethodNotAllowed)
			return
		}

		w.WriteHeader(http.StatusOK)
	})

	http.HandleFunc("/", handleViolationReport)
	log.Fatal(http.ListenAndServe(fmt.Sprintf(":%s", strconv.Itoa(*listenPort)), nil))
}

func handleViolationReport(w http.ResponseWriter, r *http.Request) {
	if r.Method != "POST" {
		w.WriteHeader(http.StatusMethodNotAllowed)
		log.WithFields(log.Fields{
			"http_method": r.Method,
		}).Debug("Received invalid HTTP method")
		return
	}

	decoder := json.NewDecoder(r.Body)
	var report CSPReport

	err := decoder.Decode(&report)
	if err != nil {
		w.WriteHeader(http.StatusUnprocessableEntity)
		log.Debugf("Unable to decode invalid JSON payload: %s", err)
		return
	}
	defer r.Body.Close()

	reportValidation := validateViolation(report)
	if reportValidation != nil {
		http.Error(w, reportValidation.Error(), http.StatusBadRequest)
		log.Debugf("Received invalid payload: %s", reportValidation.Error())
		return
	}

	metadatas, gotMetadata := r.URL.Query()["metadata"]
	var metadata string
	if gotMetadata {
		metadata = metadatas[0]
	}

	log.WithFields(log.Fields{
		"document_uri":        report.Body.DocumentURI,
		"referrer":            report.Body.Referrer,
		"blocked_uri":         report.Body.BlockedURI,
		"violated_directive":  report.Body.ViolatedDirective,
		"effective_directive": report.Body.EffectiveDirective,
		"original_policy":     report.Body.OriginalPolicy,
		"disposition":         report.Body.Disposition,
		"script_sample":       report.Body.ScriptSample,
		"status_code":         report.Body.StatusCode,
		"metadata":            metadata,
	}).Info()
}

func validateViolation(r CSPReport) error {
	for _, value := range ignoredBlockedURIs {
		if strings.HasPrefix(r.Body.BlockedURI, value) {
			err := fmt.Errorf("blocked URI ('%s') is an invalid resource", value)
			return err
		}
	}

	if !strings.HasPrefix(r.Body.DocumentURI, "http") {
		return fmt.Errorf("document URI ('%s') is invalid", r.Body.DocumentURI)
	}

	return nil
}
<|endoftext|>


<commit_before>// 语音处理
// 利用百度RESTFul API 进行语音及文字的相互转换
package voice

import (
	"errors"

	"io/ioutil"

	sdk "github.com/chenqinghe/baidu-ai-go-sdk/internal"
	"github.com/imroc/req"
	"math/rand"
	"net"
	"strconv"
	"strings"
)

const (
	TTS_URL string = "http://tsn.baidu.com/text2audio"
	ASR_URL string = "http://vop.baidu.com/server_api"
)
const (
	B int = 1 << (10 * iota)
	KB
	MB
)

var (
	ErrNoTTSConfig = errors.New("No TTSConfig.please set TTSConfig correctlly first or call method UseDefaultTTSConfig")
	ErrTextTooLong = errors.New("The input string is too long")
)

//VoiceClient 代表一个语音服务应用
type VoiceClient struct {
	*sdk.Client
	TTSConfig *TTSConfig
}

//语音合成参数
type TTSConfig struct {
	SPD int //语速，取值0-9，默认为5中语速
	PIT int //音调，取值0-9，默认为5中语调
	VOL int //音量，取值0-15，默认为5中音量
	PER int //发音人选择, 0为普通女声，1为普通男声，3为情感合成-度逍遥，4为情感合成-度丫丫，默认为普通女声
}

var defaultTTSConfig = &TTSConfig{
	SPD: 5,
	PIT: 5,
	VOL: 5,
	PER: 0,
}

//语音识别响应信息
type ASRResponse struct {
	CorpusNo string   `json:"corpus_no"`
	ERRMSG   string   `json:"err_msg"`
	ERRNO    int      `json:"err_no"`
	Result   []string `json:"result"`
	SN       string   `json:"sn"`
}

//语音识别参数
type ASRParams struct {
	Format  string `json:"format"`  //语音的格式，pcm 或者 wav 或者 amr。不区分大小写
	Rate    int    `json:"rate"`    //采样率，支持 8000 或者 16000
	Channel int    `json:"channel"` //声道数，仅支持单声道，请填写固定值 1
	Cuid    string `json:"cuid"`    //用户唯一标识，用来区分用户，计算UV值。建议填写能区分用户的机器 MAC 地址或 IMEI 码，长度为60字符以内
	Token   string `json:"token"`   //开放平台获取到的开发者access_token
	Lan     string `json:"lan"`     //语种选择，默认中文（zh）。 中文=zh、粤语=ct、英文=en，不区分大小写
	Speech  string `json:"speech"`  //真实的语音数据 ，需要进行base64 编码。与len参数连一起使用
	Len     int    `json:"len"`     //原始语音长度，单位字节
}

//TextToSpeech 语音合成，将文字转换为语音
func (vc *VoiceClient) TextToSpeech(txt string) ([]byte, error) {

	if len(txt) >= 1024 {
		return []byte{}, ErrTextTooLong
	}
	if err := vc.Auth(); err != nil {
		return []byte{}, err
	}
	if vc.TTSConfig == nil {
		return []byte{}, ErrNoTTSConfig
	}
	itfcs, err := net.Interfaces()
	if err != nil {
		return []byte{}, err
	}
	mac := itfcs[0].HardwareAddr.String()
	if mac == "" {
		mac = randomStr(10)
	}
	params := req.Param{
		"tex":  txt,                            //必填	合成的文本，使用UTF-8编码，请注意文本长度必须小于1024字节
		"lan":  "zh",                           //必填	语言选择,目前只有中英文混合模式，填写固定值zh
		"tok":  vc.AccessToken,                 //必填	开放平台获取到的开发者access_token（见上面的“鉴权认证机制”段落）
		"ctp":  "1",                            //必填	客户端类型选择，web端填写固定值1
		"cuid": mac,                            //必填	用户唯一标识，用来区分用户，计算UV值。建议填写能区分用户的机器 MAC 地址或 IMEI 码，长度为60字符以内
		"spd":  strconv.Itoa(vc.TTSConfig.SPD), //选填	语速，取值0-9，默认为5中语速
		"pit":  strconv.Itoa(vc.TTSConfig.PIT), //选填	音调，取值0-9，默认为5中语调
		"vol":  strconv.Itoa(vc.TTSConfig.VOL), //选填	音量，取值0-15，默认为5中音量
		"per":  strconv.Itoa(vc.TTSConfig.PER), //选填 发音人选择, 0为普通女声，1为普通男声，3为情感合成-度逍遥，4为情感合成-度丫丫，默认为普通女声
	}
	resp, err := req.Post(TTS_URL, params)
	if err != nil {
		return []byte{}, nil
	}
	respHeader := resp.Response().Header
	contentType, ok := respHeader["Content-Type"]
	if !ok {
		return []byte{}, errors.New("No Content-Type Set.")
	}
	if contentType[0] == "audio/mp3" {
		respBody, err := ioutil.ReadAll(resp.Response().Body)
		if err != nil {
			return []byte{}, nil
		}
		return respBody, nil
	} else {
		respStr, err := resp.ToString()
		if err != nil {
			return []byte{}, err
		}
		return []byte{}, errors.New("调用服务失败：" + respStr)
	}
}

//SpeechToText 语音识别，将语音翻译成文字
func (vc *VoiceClient) SpeechToText(ap ASRParams) ([]string, error) {
	if ap.Len > 8*10*MB {
		return []string{}, errors.New("文件大小不能超过10M")
	}
	if err := vc.Auth(); err != nil {
		return []string{}, err
	}
	ap.Token = vc.AccessToken
	resp, err := req.Post(ASR_URL, req.Header{
		"Content-Type": "application/json",
	}, req.BodyJSON(ap))
	if err != nil {
		return []string{}, err
	}
	var rs ASRResponse
	if err := resp.ToJSON(&rs); err != nil {
		return []string{}, err
	}
	if !strings.Contains(rs.ERRMSG, "success") || rs.ERRNO != 0 {
		return []string{}, errors.New("调用服务失败：" + rs.ERRMSG)
	}
	return rs.Result, nil
}

func NewVoiceClient(ApiKey, secretKey string) *VoiceClient {
	return &VoiceClient{
		Client:    sdk.NewClient(ApiKey, secretKey),
		TTSConfig: defaultTTSConfig,
	}
}

func randomStr(length int) string {
	var baseStr = "qwertyuiopasdfghjklzxcvbnmQWERTYUIOPLKJHGFDSAZXCVBNM0123456789"
	var bt []byte = make([]byte, length)
	for i := 0; i < length; i++ {
		k := rand.Intn(62)
		bt[i] = baseStr[k]
	}
	return string(bt)
}
<commit_msg>fix typo<commit_after>// 语音处理
// 利用百度RESTFul API 进行语音及文字的相互转换
package voice

import (
	"errors"

	"io/ioutil"

	sdk "github.com/chenqinghe/baidu-ai-go-sdk/internal"
	"github.com/imroc/req"
	"math/rand"
	"net"
	"strconv"
	"strings"
)

const (
	TTS_URL string = "http://tsn.baidu.com/text2audio"
	ASR_URL string = "http://vop.baidu.com/server_api"
)
const (
	B int = 1 << (10 * iota)
	KB
	MB
)

var (
	ErrNoTTSConfig = errors.New("No TTSConfig.please set TTSConfig correctlly first or call method UseDefaultTTSConfig")
	ErrTextTooLong = errors.New("The input string is too long")
)

//VoiceClient 代表一个语音服务应用
type VoiceClient struct {
	*sdk.Client
	TTSConfig *TTSConfig
}

//语音合成参数
type TTSConfig struct {
	SPD int //语速，取值0-9，默认为5中语速
	PIT int //音调，取值0-9，默认为5中语调
	VOL int //音量，取值0-15，默认为5中音量
	PER int //发音人选择, 0为普通女声，1为普通男声，3为情感合成-度逍遥，4为情感合成-度丫丫，默认为普通女声
}

var defaultTTSConfig = &TTSConfig{
	SPD: 5,
	PIT: 5,
	VOL: 5,
	PER: 0,
}

//语音识别响应信息
type ASRResponse struct {
	CorpusNo string   `json:"corpus_no"`
	ERRMSG   string   `json:"err_msg"`
	ERRNO    int      `json:"err_no"`
	Result   []string `json:"result"`
	SN       string   `json:"sn"`
}

//语音识别参数
type ASRParams struct {
	Format  string `json:"format"`  //语音的格式，pcm 或者 wav 或者 amr。不区分大小写
	Rate    int    `json:"rate"`    //采样率，支持 8000 或者 16000
	Channel int    `json:"channel"` //声道数，仅支持单声道，请填写固定值 1
	Cuid    string `json:"cuid"`    //用户唯一标识，用来区分用户，计算UV值。建议填写能区分用户的机器 MAC 地址或 IMEI 码，长度为60字符以内
	Token   string `json:"token"`   //开放平台获取到的开发者access_token
	Lan     string `json:"lan"`     //语种选择，默认中文（zh）。 中文=zh、粤语=ct、英文=en，不区分大小写
	Speech  string `json:"speech"`  //真实的语音数据 ，需要进行base64 编码。与len参数连一起使用
	Len     int    `json:"len"`     //原始语音长度，单位字节
}

//TextToSpeech 语音合成，将文字转换为语音
func (vc *VoiceClient) TextToSpeech(txt string) ([]byte, error) {

	if len(txt) >= 1024 {
		return []byte{}, ErrTextTooLong
	}
	if err := vc.Auth(); err != nil {
		return []byte{}, err
	}
	if vc.TTSConfig == nil {
		return []byte{}, ErrNoTTSConfig
	}
	itfcs, err := net.Interfaces()
	if err != nil {
		return []byte{}, err
	}
	mac := itfcs[0].HardwareAddr.String()
	if mac == "" {
		mac = randomStr(10)
	}
	params := req.Param{
		"tex":  txt,                            //必填	合成的文本，使用UTF-8编码，请注意文本长度必须小于1024字节
		"lan":  "zh",                           //必填	语言选择,目前只有中英文混合模式，填写固定值zh
		"tok":  vc.AccessToken,                 //必填	开放平台获取到的开发者access_token（见上面的“鉴权认证机制”段落）
		"ctp":  "1",                            //必填	客户端类型选择，web端填写固定值1
		"cuid": mac,                            //必填	用户唯一标识，用来区分用户，计算UV值。建议填写能区分用户的机器 MAC 地址或 IMEI 码，长度为60字符以内
		"spd":  strconv.Itoa(vc.TTSConfig.SPD), //选填	语速，取值0-9，默认为5中语速
		"pit":  strconv.Itoa(vc.TTSConfig.PIT), //选填	音调，取值0-9，默认为5中语调
		"vol":  strconv.Itoa(vc.TTSConfig.VOL), //选填	音量，取值0-15，默认为5中音量
		"per":  strconv.Itoa(vc.TTSConfig.PER), //选填 发音人选择, 0为普通女声，1为普通男声，3为情感合成-度逍遥，4为情感合成-度丫丫，默认为普通女声
	}
	resp, err := req.Post(TTS_URL, params)
	if err != nil {
		return []byte{}, err
	}
	respHeader := resp.Response().Header
	contentType, ok := respHeader["Content-Type"]
	if !ok {
		return []byte{}, errors.New("No Content-Type Set.")
	}
	if contentType[0] == "audio/mp3" {
		respBody, err := ioutil.ReadAll(resp.Response().Body)
		if err != nil {
			return []byte{}, err
		}
		return respBody, nil
	} else {
		respStr, err := resp.ToString()
		if err != nil {
			return []byte{}, err
		}
		return []byte{}, errors.New("调用服务失败：" + respStr)
	}
}

//SpeechToText 语音识别，将语音翻译成文字
func (vc *VoiceClient) SpeechToText(ap ASRParams) ([]string, error) {
	if ap.Len > 8*10*MB {
		return []string{}, errors.New("文件大小不能超过10M")
	}
	if err := vc.Auth(); err != nil {
		return []string{}, err
	}
	ap.Token = vc.AccessToken
	resp, err := req.Post(ASR_URL, req.Header{
		"Content-Type": "application/json",
	}, req.BodyJSON(ap))
	if err != nil {
		return []string{}, err
	}
	var rs ASRResponse
	if err := resp.ToJSON(&rs); err != nil {
		return []string{}, err
	}
	if !strings.Contains(rs.ERRMSG, "success") || rs.ERRNO != 0 {
		return []string{}, errors.New("调用服务失败：" + rs.ERRMSG)
	}
	return rs.Result, nil
}

func NewVoiceClient(ApiKey, secretKey string) *VoiceClient {
	return &VoiceClient{
		Client:    sdk.NewClient(ApiKey, secretKey),
		TTSConfig: defaultTTSConfig,
	}
}

func randomStr(length int) string {
	var baseStr = "qwertyuiopasdfghjklzxcvbnmQWERTYUIOPLKJHGFDSAZXCVBNM0123456789"
	var bt []byte = make([]byte, length)
	for i := 0; i < length; i++ {
		k := rand.Intn(62)
		bt[i] = baseStr[k]
	}
	return string(bt)
}
<|endoftext|>


<commit_before>package main

import (
	"context"
	"fmt"
	"html/template"
	"log"
	"net/http"
	"os"
	"os/exec"
	"runtime"
	"strconv"
	"time"

	"github.com/gorilla/mux"
)

const indexTemplStr = `<!doctype html>

<title>Plot Viewer</title>

	{{range $index, $p := .Plots}}

			<object data="/images/{{$index}}" type="image/svg+xml" style="pointer-events: none;"></object>

{{$p}}

	{{end}}

			<button id="open" onclick="openInNewTab()">Open in Tab</button>
			<button id="cp-png" onclick="openPNG()">Open PNG</button>
			<button id="exit" onclick="quit()">Quit</button>

		<object id="im-box" data="/images/0" type="image/svg+xml" style="pointer-events: none;">
		</object>

`

var indexTempl = template.Must(template.New("index").Parse(indexTemplStr))

type plotViewer struct {
	pdfPaths []string
	svgCache [][]byte
	pngCache [][]byte
}

func (v plotViewer) rootHandler(w http.ResponseWriter, r *http.Request) {
	err := indexTempl.Execute(w, struct {
		Plots []string
	}{
		v.pdfPaths,
	})
	if err != nil {
		log.Println("GET /: %v", err)
	}
}

func (v plotViewer) imageHandler(w http.ResponseWriter, r *http.Request) {
	vars := mux.Vars(r)
	id, err := strconv.Atoi(vars["id"])
	if err != nil {
		http.Error(w, err.Error(), http.StatusBadRequest)
		return
	}
	if id < 0 || id >= len(v.pdfPaths) {
		http.Error(w, "image id out of bounds", http.StatusBadRequest)
		return
	}

	if v.svgCache[id] == nil {
		out, err := exec.Command("pdf2svg", v.pdfPaths[id], "/dev/stdout").Output()
		if err != nil {
			http.Error(w, fmt.Sprintf("unable to convert to svg: %v", err), http.StatusInternalServerError)
			return
		}

		v.svgCache[id] = out
	}

	w.Header().Set("content-type", "image/svg+xml")
	w.Write(v.svgCache[id])
}

func (v plotViewer) pngHandler(w http.ResponseWriter, r *http.Request) {
	vars := mux.Vars(r)
	id, err := strconv.Atoi(vars["id"])
	if err != nil {
		http.Error(w, err.Error(), http.StatusBadRequest)
		return
	}
	if id < 0 || id >= len(v.pdfPaths) {
		http.Error(w, "image id out of bounds", http.StatusBadRequest)
		return
	}

	if v.pngCache[id] == nil {
		out, err := exec.Command("convert", "-density", "300", v.pdfPaths[id], "png:-").Output()
		if err != nil {
			http.Error(w, fmt.Sprintf("unable to convert to png: %v", err), http.StatusInternalServerError)
			return
		}

		v.pngCache[id] = out
	}

	w.Header().Set("content-type", "image/png")
	w.Write(v.pngCache[id])
}

type quitHandler struct {
	s *http.Server
}

func (h quitHandler) ServeHTTP(w http.ResponseWriter, r *http.Request) {
	ctx, _ := context.WithTimeout(r.Context(), 5*time.Second)
	h.s.Shutdown(ctx)
}

func main() {
	log.SetFlags(0)
	log.SetPrefix("vplots: ")
	if len(os.Args) < 2 {
		fmt.Fprintln(os.Stderr, "usage: vplots plot.pdf...")
		os.Exit(2)
	}

	v := plotViewer{
		pdfPaths: os.Args[1:],
		svgCache: make([][]byte, len(os.Args)-1),
		pngCache: make([][]byte, len(os.Args)-1),
	}

	r := mux.NewRouter()
	r.HandleFunc("/", v.rootHandler)
	r.HandleFunc("/images/{id:[0-9]+}", v.imageHandler)
	r.HandleFunc("/pngs/{id:[0-9]+}", v.pngHandler)

	s := &http.Server{
		Addr:    "127.0.0.1:6544",
		Handler: r,
	}
	r.Handle("/quit", quitHandler{s})
	go openURL(s.Addr)
	s.ListenAndServe()
}

func openURL(url string) {
	url = "http://" + url

	for tries := 5; tries > 5; tries-- {
		if _, err := http.Get(url); err == nil {
			break
		}
		time.Sleep(10 * time.Millisecond)
	}

	var cmd *exec.Cmd
	switch runtime.GOOS {
	case "darwin":
		cmd = exec.Command("open", url)
	case "linux", "freebsd", "solaris":
		cmd = exec.Command("xdg-open", url)
	default:
		log.Printf("no known open command, connect to %s", url)
		return
	}
	if err := cmd.Start(); err != nil {
		log.Printf("open command failed, connect to %s: %v", url, err)
	}
}
<commit_msg>vplots moved to github.com/uluyol/vplots<commit_after><|endoftext|>


<commit_before>package pinocchio

import (
  "fmt"
  "math/big"
  "github.com/46bit/pinocchio/ec"
)

type DualECDRBG struct {
  C *ec.PrimeCurve
  Q *ec.Point

  Z *big.Int
  z_bytes []byte
  S *big.Int
  StateIndex uint64
  StateBit uint32
}

func NewDualECDRBG(c *ec.PrimeCurve, qx, qy, seed *big.Int) *DualECDRBG {
  g := DualECDRBG{c, &ec.Point{qx, qy, true}, big.NewInt(0), nil, big.NewInt(0), 0, 0}
  g.seed(seed)
  return &g
}

// Seed is twice security_strength bits long (at least 256)
func (g *DualECDRBG) seed(seed *big.Int) {
  g.S = seed
  g.generate_number()
}

func (g *DualECDRBG) generate_number() {
  g.StateIndex++

  s := g.C.ScalarMultiply(g.S, g.C.G)
  z := g.C.ScalarMultiply(s.X, g.Q)

  fmt.Printf("s is\nx = %X\ny = %X\n\n", s.X, s.Y)
  fmt.Printf("z is\nx = %X\ny = %X\n\n", z.X, z.Y)

  if !g.C.Satisfied(s) {
    fmt.Printf("s = g.C.G * %X not on curve\n", g.S)
  }
  if !g.C.Satisfied(z) {
    fmt.Printf("z = g.Q * %X not on curve\n", s.X)
  }

  if !s.Finite {
    fmt.Println("s infinite")
  }
  if !z.Finite {
    fmt.Println("z infinite")
  }

  g.S = s.X
  g.Z = z.X

  g.z_bytes = g.Z.Bytes()
  g.StateBit = 16
  fmt.Printf("%x\n", g.z_bytes)
}

func (g *DualECDRBG) Bit() uint32 {
  z_byte := g.z_bytes[g.StateBit / 8]
  z_bit_shift := 7 - (g.StateBit % 8)
  bit := 1 & (z_byte >> z_bit_shift)

  g.StateBit++
  if int(g.StateBit) >= g.Z.BitLen() { // @TODO: think if the bitlen-16 is off by 1
    g.generate_number()
  }
  return uint32(bit)
}

func (g *DualECDRBG) Urand32() uint32 {
  v := uint32(0)
  for i := 0; i < 32; i++ {
    v = (v<<1) + g.Bit()
  }
  return v
}
<commit_msg>Added Byte output method which now matches OpenSSL output.<commit_after>package pinocchio

import (
  "errors"
  "math/big"
  "github.com/46bit/pinocchio/ec"
)

type DualECDRBG struct {
  C *ec.PrimeCurve
  Q *ec.Point

  S *big.Int
  Sp *ec.Point
  Z *big.Int
  Zp *ec.Point

  ZBytes []byte
  StateIndex uint64
  StateBit uint32
}

func NewDualECDRBG(c *ec.PrimeCurve, qx, qy, seed *big.Int) *DualECDRBG {
  g := DualECDRBG{c, &ec.Point{qx, qy, true}, big.NewInt(0), nil, big.NewInt(0), nil, nil, 0, 0}
  g.seed(seed)
  return &g
}

// Seed is twice security_strength bits long (at least 256)
func (g *DualECDRBG) seed(seed *big.Int) {
  g.S = seed
  // To match OpenSSL, we perform two rounds before giving any output.
  g.generate_number()
  g.generate_number()
}

func (g *DualECDRBG) generate_number() {
  g.StateIndex++

  g.Sp = g.C.ScalarMultiply(g.S, g.C.G)
  g.S = g.Sp.X

  g.Zp = g.C.ScalarMultiply(g.S, g.Q)
  g.Z = g.Zp.X

  // We use .Bytes() so as to extract Z in big-endian format. This matches
  // OpenSSL behaviour. We drop the first 16 bits by initialising g.StateBit
  // as 16.
  g.ZBytes = g.Z.Bytes()
  g.StateBit = 16
}

func (g *DualECDRBG) Selfcheck() error {
  if !g.C.Satisfied(g.Sp) {
    return errors.New("S is not on curve.")
  }
  if !g.C.Satisfied(g.Zp) {
    return errors.New("Z is not on curve.")
  }

  if !g.Sp.Finite {
    return errors.New("S is infinite.")
  }
  if !g.Zp.Finite {
    return errors.New("Z is infinite.")
  }

  return nil
}

func (g *DualECDRBG) Bit() uint32 {
  if int(g.StateBit) >= len(g.ZBytes) * 8 {
    g.generate_number()
  }

  z_byte := g.ZBytes[g.StateBit / 8]
  z_bit_shift := 7 - (g.StateBit % 8)
  z_bit := 1 & (z_byte >> z_bit_shift)

  g.StateBit++

  return uint32(z_bit)
}

func (g *DualECDRBG) Urand32() uint32 {
  v := uint32(0)
  for i := 0; i < 32; i++ {
    v = (v<<1) + g.Bit()
  }
  return v
}

func (g *DualECDRBG) Byte() byte {
  if g.StateBit % 8 != 0 {
    g.StateBit = ((g.StateBit + 8) >> 3) << 3
  }

  if int(g.StateBit) >= len(g.ZBytes) * 8 {
    g.generate_number()
  }

  z_byte := g.ZBytes[g.StateBit / 8]

  g.StateBit += 8

  return z_byte
}
<|endoftext|>


<commit_before>// Copyright 2016 Factom Foundation
// Use of this source code is governed by the MIT
// license that can be found in the LICENSE file.

package wallet

import (
	"bytes"
	"crypto/sha256"

	"github.com/FactomProject/btcutil/base58"
)

const (
	SeedLength = 64
	Version    = "0.2.2.5"
	ApiVersion = "2.0"
)

// seed address prefix
var seedPrefix = []byte{0x13, 0xdd}

// SeedString returnes the string representation of a raw Wallet Seed or Next
// Wallet Seed.
func SeedString(seed []byte) string {
	if len(seed) != SeedLength {
		return ""
	}

	buf := new(bytes.Buffer)

	// 2 byte Seed Address Prefix
	buf.Write(seedPrefix)

	// 64 byte Seed
	buf.Write(seed)

	// 4 byte Checksum
	check := shad(buf.Bytes())[:4]
	buf.Write(check)

	return base58.Encode(buf.Bytes())
}

// shad Double Sha256 Hash; sha256(sha256(data))
func shad(data []byte) []byte {
	h1 := sha256.Sum256(data)
	h2 := sha256.Sum256(h1[:])
	return h2[:]
}

// newCounter is used to generate the ID field for the JSON2Request
func newCounter() func() int {
	count := 0
	return func() int {
		count += 1
		return count
	}
}

var APICounter = newCounter()
<commit_msg>increase version<commit_after>// Copyright 2016 Factom Foundation
// Use of this source code is governed by the MIT
// license that can be found in the LICENSE file.

package wallet

import (
	"bytes"
	"crypto/sha256"

	"github.com/FactomProject/btcutil/base58"
)

const (
	SeedLength = 64
	Version    = "0.2.2.6"
	ApiVersion = "2.0"
)

// seed address prefix
var seedPrefix = []byte{0x13, 0xdd}

// SeedString returnes the string representation of a raw Wallet Seed or Next
// Wallet Seed.
func SeedString(seed []byte) string {
	if len(seed) != SeedLength {
		return ""
	}

	buf := new(bytes.Buffer)

	// 2 byte Seed Address Prefix
	buf.Write(seedPrefix)

	// 64 byte Seed
	buf.Write(seed)

	// 4 byte Checksum
	check := shad(buf.Bytes())[:4]
	buf.Write(check)

	return base58.Encode(buf.Bytes())
}

// shad Double Sha256 Hash; sha256(sha256(data))
func shad(data []byte) []byte {
	h1 := sha256.Sum256(data)
	h2 := sha256.Sum256(h1[:])
	return h2[:]
}

// newCounter is used to generate the ID field for the JSON2Request
func newCounter() func() int {
	count := 0
	return func() int {
		count += 1
		return count
	}
}

var APICounter = newCounter()
<|endoftext|>


<commit_before>// Package watch detects raw changes to files.
package watch

import (
	"context"
)

// Watcher watches a file at the given path for changes.
type Watcher interface {
	// Read blocks and returns the next update for a file. An error is returned
	// when the file cannot be read. Subsequent calls block until the underlying
	// contents or error changes. When multiple updates have occurred for a file,
	// Read coalesces and returns the latest update.
	//
	// If ctx is cancelled, Read returns an error.
	Read(ctx context.Context) (Update, error)

	// Close causes Watcher to stop watching a file and release its resources.
	Close()
}

// Update represents contents for a file. Path can represent fan-out on
// individual files when watching a directory path.
type Update struct {
	Path     string
	Contents []byte
}
<commit_msg>Allow addition and removal of Watcher paths.<commit_after>// Package watch detects raw changes to files.
package watch

import (
	"context"
)

// Watcher watches files at the given paths for changes.
type Watcher interface {
	// Read blocks and returns the next update for a file. An error is returned
	// when the file cannot be read. Subsequent calls block until the underlying
	// contents or error changes. When multiple updates have occurred for a file,
	// Read coalesces and returns the latest update.
	//
	// If ctx is cancelled, Read returns an error.
	Read(ctx context.Context) (Update, error)

	// Add causes Watcher to monitor an additional file. The format is
	// filesystem-specific. If Close has been called, this has no effect.
	Add(path string) error

	// Remove causes Watcher to stop monitoring a file. The path must match one
	// already monitored in the same format. The format is filesystem-specific.
	Remove(path string) error

	// Close causes Watcher to stop watching all files and release its resources.
	Close()
}

// Update represents contents for a file. Path can represent fan-out on
// individual files when watching a directory path.
type Update struct {
	Path     string
	Contents []byte
}
<|endoftext|>


<commit_before>package edit

// XXX
// Taken from github.com/nsf/godit, not a genuine wcwidth implementation.
// wcwidth of glibc is broken, meaning we cannot just use cgo to wrap the
// native wcwidth. Possible reference implementation:
// http://www.cl.cam.ac.uk/~mgk25/ucs/wcwidth.c
//
// Original comment in godit source:
// somewhat close to what wcwidth does, except rune_width doesn't return 0 or
// -1, it's always 1 or 2
func wcwidth(r rune) int {
	if r >= 0x1100 &&
		(r <= 0x115f || r == 0x2329 || r == 0x232a ||
			(r >= 0x2e80 && r <= 0xa4cf && r != 0x303f) ||
			(r >= 0xac00 && r <= 0xd7a3) ||
			(r >= 0xf900 && r <= 0xfaff) ||
			(r >= 0xfe30 && r <= 0xfe6f) ||
			(r >= 0xff00 && r <= 0xff60) ||
			(r >= 0xffe0 && r <= 0xffe6) ||
			(r >= 0x20000 && r <= 0x2fffd) ||
			(r >= 0x30000 && r <= 0x3fffd)) {
		return 2
	}
	return 1
}

func wcwidths(s string) (w int) {
	for _, r := range s {
		w += wcwidth(r)
	}
	return
}

func trimWcwidth(s string, wmax int) string {
	w := 0
	for i, r := range s {
		w += wcwidth(r)
		if w > wmax {
			return s[:i]
		}
	}
	return s
}
<commit_msg>Use a real wcwidth implementation<commit_after>package edit

import (
	"sort"
)

// Taken from http://www.cl.cam.ac.uk/~mgk25/ucs/wcwidth.c (public domain)
var combining = [][2]rune{
	{0x0300, 0x036F}, {0x0483, 0x0486}, {0x0488, 0x0489},
	{0x0591, 0x05BD}, {0x05BF, 0x05BF}, {0x05C1, 0x05C2},
	{0x05C4, 0x05C5}, {0x05C7, 0x05C7}, {0x0600, 0x0603},
	{0x0610, 0x0615}, {0x064B, 0x065E}, {0x0670, 0x0670},
	{0x06D6, 0x06E4}, {0x06E7, 0x06E8}, {0x06EA, 0x06ED},
	{0x070F, 0x070F}, {0x0711, 0x0711}, {0x0730, 0x074A},
	{0x07A6, 0x07B0}, {0x07EB, 0x07F3}, {0x0901, 0x0902},
	{0x093C, 0x093C}, {0x0941, 0x0948}, {0x094D, 0x094D},
	{0x0951, 0x0954}, {0x0962, 0x0963}, {0x0981, 0x0981},
	{0x09BC, 0x09BC}, {0x09C1, 0x09C4}, {0x09CD, 0x09CD},
	{0x09E2, 0x09E3}, {0x0A01, 0x0A02}, {0x0A3C, 0x0A3C},
	{0x0A41, 0x0A42}, {0x0A47, 0x0A48}, {0x0A4B, 0x0A4D},
	{0x0A70, 0x0A71}, {0x0A81, 0x0A82}, {0x0ABC, 0x0ABC},
	{0x0AC1, 0x0AC5}, {0x0AC7, 0x0AC8}, {0x0ACD, 0x0ACD},
	{0x0AE2, 0x0AE3}, {0x0B01, 0x0B01}, {0x0B3C, 0x0B3C},
	{0x0B3F, 0x0B3F}, {0x0B41, 0x0B43}, {0x0B4D, 0x0B4D},
	{0x0B56, 0x0B56}, {0x0B82, 0x0B82}, {0x0BC0, 0x0BC0},
	{0x0BCD, 0x0BCD}, {0x0C3E, 0x0C40}, {0x0C46, 0x0C48},
	{0x0C4A, 0x0C4D}, {0x0C55, 0x0C56}, {0x0CBC, 0x0CBC},
	{0x0CBF, 0x0CBF}, {0x0CC6, 0x0CC6}, {0x0CCC, 0x0CCD},
	{0x0CE2, 0x0CE3}, {0x0D41, 0x0D43}, {0x0D4D, 0x0D4D},
	{0x0DCA, 0x0DCA}, {0x0DD2, 0x0DD4}, {0x0DD6, 0x0DD6},
	{0x0E31, 0x0E31}, {0x0E34, 0x0E3A}, {0x0E47, 0x0E4E},
	{0x0EB1, 0x0EB1}, {0x0EB4, 0x0EB9}, {0x0EBB, 0x0EBC},
	{0x0EC8, 0x0ECD}, {0x0F18, 0x0F19}, {0x0F35, 0x0F35},
	{0x0F37, 0x0F37}, {0x0F39, 0x0F39}, {0x0F71, 0x0F7E},
	{0x0F80, 0x0F84}, {0x0F86, 0x0F87}, {0x0F90, 0x0F97},
	{0x0F99, 0x0FBC}, {0x0FC6, 0x0FC6}, {0x102D, 0x1030},
	{0x1032, 0x1032}, {0x1036, 0x1037}, {0x1039, 0x1039},
	{0x1058, 0x1059}, {0x1160, 0x11FF}, {0x135F, 0x135F},
	{0x1712, 0x1714}, {0x1732, 0x1734}, {0x1752, 0x1753},
	{0x1772, 0x1773}, {0x17B4, 0x17B5}, {0x17B7, 0x17BD},
	{0x17C6, 0x17C6}, {0x17C9, 0x17D3}, {0x17DD, 0x17DD},
	{0x180B, 0x180D}, {0x18A9, 0x18A9}, {0x1920, 0x1922},
	{0x1927, 0x1928}, {0x1932, 0x1932}, {0x1939, 0x193B},
	{0x1A17, 0x1A18}, {0x1B00, 0x1B03}, {0x1B34, 0x1B34},
	{0x1B36, 0x1B3A}, {0x1B3C, 0x1B3C}, {0x1B42, 0x1B42},
	{0x1B6B, 0x1B73}, {0x1DC0, 0x1DCA}, {0x1DFE, 0x1DFF},
	{0x200B, 0x200F}, {0x202A, 0x202E}, {0x2060, 0x2063},
	{0x206A, 0x206F}, {0x20D0, 0x20EF}, {0x302A, 0x302F},
	{0x3099, 0x309A}, {0xA806, 0xA806}, {0xA80B, 0xA80B},
	{0xA825, 0xA826}, {0xFB1E, 0xFB1E}, {0xFE00, 0xFE0F},
	{0xFE20, 0xFE23}, {0xFEFF, 0xFEFF}, {0xFFF9, 0xFFFB},
	{0x10A01, 0x10A03}, {0x10A05, 0x10A06}, {0x10A0C, 0x10A0F},
	{0x10A38, 0x10A3A}, {0x10A3F, 0x10A3F}, {0x1D167, 0x1D169},
	{0x1D173, 0x1D182}, {0x1D185, 0x1D18B}, {0x1D1AA, 0x1D1AD},
	{0x1D242, 0x1D244}, {0xE0001, 0xE0001}, {0xE0020, 0xE007F},
	{0xE0100, 0xE01EF},
}

func isCombining(r rune) bool {
	n := len(combining)
	i := sort.Search(n, func(i int) bool { return r <= combining[i][1] })
	return i < n && r >= combining[i][0]
}

func wcwidth(r rune) int {
	switch {
	case r == 0:
		return 0
	case r < 32 || (0x7f <= r && r < 0xa0): // Control character
		return -1
	case isCombining(r):
		return 0
	}

	if r >= 0x1100 &&
		(r <= 0x115f || /* Hangul Jamo init. consonants */
			r == 0x2329 || r == 0x232a ||
			(r >= 0x2e80 && r <= 0xa4cf &&
				r != 0x303f) || /* CJK ... Yi */
			(r >= 0xac00 && r <= 0xd7a3) || /* Hangul Syllables */
			(r >= 0xf900 && r <= 0xfaff) || /* CJK Compatibility Ideographs */
			(r >= 0xfe10 && r <= 0xfe19) || /* Vertical forms */
			(r >= 0xfe30 && r <= 0xfe6f) || /* CJK Compatibility Forms */
			(r >= 0xff00 && r <= 0xff60) || /* Fullwidth Forms */
			(r >= 0xffe0 && r <= 0xffe6) ||
			(r >= 0x20000 && r <= 0x2fffd) ||
			(r >= 0x30000 && r <= 0x3fffd)) {
		return 2
	}
	return 1
}

func wcwidths(s string) (w int) {
	for _, r := range s {
		w += wcwidth(r)
	}
	return
}

func trimWcwidth(s string, wmax int) string {
	w := 0
	for i, r := range s {
		w += wcwidth(r)
		if w > wmax {
			return s[:i]
		}
	}
	return s
}
<|endoftext|>


<commit_before>// Copyright (c) 2015 Mute Communications Ltd.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

package ctrlengine

import (
	"bufio"
	"bytes"
	"fmt"
	"io"
	"os"
	"os/exec"
	"path/filepath"
	"strconv"
	"syscall"

	"github.com/agl/ed25519"
	"github.com/codegangsta/cli"
	"github.com/mutecomm/mute/cipher"
	"github.com/mutecomm/mute/encode/base64"
	"github.com/mutecomm/mute/log"
	"github.com/mutecomm/mute/msgdb"
	"github.com/mutecomm/mute/util/bzero"
)

func createKeyDB(c *cli.Context, passphrase []byte) error {
	cmd := exec.Command("mutecrypt",
		"--passphrase-fd", strconv.FormatUint(uint64(syscall.Stdin), 10),
		"--homedir", c.GlobalString("homedir"),
		"--loglevel", c.GlobalString("loglevel"),
		"--logdir", c.GlobalString("logdir"),
		"db", "create",
		"--iterations", strconv.Itoa(c.Int("iterations")))
	stdin, err := cmd.StdinPipe()
	if err != nil {
		return err
	}
	var errbuf bytes.Buffer
	cmd.Stderr = &errbuf
	if err := cmd.Start(); err != nil {
		return err
	}
	plen := len(passphrase)
	buf := make([]byte, plen+1+plen+1)
	defer bzero.Bytes(buf)
	copy(buf, passphrase)
	copy(buf[plen:], []byte("\n"))
	copy(buf[plen+1:], passphrase)
	copy(buf[plen+1+plen:], []byte("\n"))
	if _, err := stdin.Write(buf); err != nil {
		return err
	}
	stdin.Close()
	if err := cmd.Wait(); err != nil {
		return fmt.Errorf("%s: %s", err, errbuf.String())
	}
	return nil
}

// create a new MsgDB and KeyDB.
func (ce *CtrlEngine) dbCreate(
	w, statusfp io.Writer,
	homedir string,
	c *cli.Context,
) error {
	msgdbname := filepath.Join(c.GlobalString("homedir"), "msgs")
	// read passphrase
	passphraseFD := c.GlobalInt("passphrase-fd")
	fmt.Fprintf(statusfp, "read passphrase from fd %d\n", passphraseFD)
	log.Infof("read passphrase from fd %d", passphraseFD)
	fp := os.NewFile(uintptr(passphraseFD), "passphrase-fd")
	scanner := bufio.NewScanner(fp)
	var passphrase []byte
	defer bzero.Bytes(passphrase)
	if scanner.Scan() {
		passphrase = scanner.Bytes()
	} else if err := scanner.Err(); err != nil {
		return log.Error(err)
	}
	log.Info("done")
	// read passphrase again
	fmt.Fprintf(statusfp, "read passphrase from fd %d again\n", passphraseFD)
	log.Infof("read passphrase from fd %d again", passphraseFD)
	var passphrase2 []byte
	defer bzero.Bytes(passphrase2)
	if scanner.Scan() {
		passphrase2 = scanner.Bytes()
	} else if err := scanner.Err(); err != nil {
		return log.Error(err)
	}
	log.Info("done")
	// compare passphrases
	if !bytes.Equal(passphrase, passphrase2) {
		return log.Error(ErrPassphrasesDiffer)
	}
	// create msgDB
	log.Infof("create msgDB '%s'", msgdbname)
	if err := msgdb.Create(msgdbname, passphrase, c.Int("iterations")); err != nil {
		return err
	}
	// open msgDB
	msgDB, err := msgdb.Open(msgdbname, passphrase)
	if err != nil {
		return err
	}
	defer msgDB.Close()
	// configure to make sure mutecrypt has config file
	err = ce.upkeepFetchconf(msgDB, homedir, false, nil, statusfp)
	if err != nil {
		return err
	}
	// create keyDB
	log.Info("create keyDB")
	if err := createKeyDB(c, passphrase); err != nil {
		return err
	}
	// status
	fmt.Fprintf(statusfp, "database files created\n")
	log.Info("database files created")
	// determine private walletKey
	walletKey := c.String("walletkey")
	if walletKey == "" {
		// generate wallet key
		_, privateKey, err := ed25519.GenerateKey(cipher.RandReader)
		if err != nil {
			return err
		}
		walletKey = base64.Encode(privateKey[:])
	}
	// store wallet key
	if err := msgDB.AddValue(msgdb.WalletKey, walletKey); err != nil {
		return err
	}
	// print wallet key
	if err := printWalletKey(w, walletKey); err != nil {
		return err
	}
	return nil
}

func rekeyKeyDB(c *cli.Context, oldPassphrase, newPassphrase []byte) error {
	cmd := exec.Command("mutecrypt",
		"--passphrase-fd", "0",
		"--homedir", c.GlobalString("homedir"),
		"--loglevel", c.GlobalString("loglevel"),
		"--logdir", c.GlobalString("logdir"),
		"db", "rekey",
		"--iterations", strconv.Itoa(c.Int("iterations")))
	stdin, err := cmd.StdinPipe()
	if err != nil {
		return err
	}
	var errbuf bytes.Buffer
	cmd.Stderr = &errbuf
	if err := cmd.Start(); err != nil {
		return err
	}
	olen := len(oldPassphrase)
	nlen := len(newPassphrase)
	buf := make([]byte, olen+1+nlen+1+nlen+1)
	defer bzero.Bytes(buf)
	copy(buf, oldPassphrase)
	copy(buf[olen:], []byte("\n"))
	copy(buf[olen+1:], newPassphrase)
	copy(buf[olen+1+nlen:], []byte("\n"))
	copy(buf[olen+1+nlen+1:], newPassphrase)
	copy(buf[olen+1+nlen+1+nlen:], []byte("\n"))
	if _, err := stdin.Write(buf); err != nil {
		return err
	}
	stdin.Close()
	if err := cmd.Wait(); err != nil {
		return fmt.Errorf("%s: %s", err, errbuf.String())
	}
	return nil
}

// rekey MsgDB and KeyDB.
func (ce *CtrlEngine) dbRekey(statusfp io.Writer, c *cli.Context) error {
	msgdbname := filepath.Join(c.GlobalString("homedir"), "msgs")
	// read old passphrase
	passphraseFD := c.GlobalInt("passphrase-fd")
	fmt.Fprintf(statusfp, "read old passphrase from fd %d\n", passphraseFD)
	log.Infof("read old passphrase from fd %d", passphraseFD)
	fp := os.NewFile(uintptr(passphraseFD), "passphrase-fd")
	scanner := bufio.NewScanner(fp)
	var oldPassphrase []byte
	defer bzero.Bytes(oldPassphrase)
	if scanner.Scan() {
		oldPassphrase = scanner.Bytes()
	} else if err := scanner.Err(); err != nil {
		return log.Error(err)
	}
	log.Info("done")
	// read new passphrase
	fmt.Fprintf(statusfp, "read new passphrase from fd %d\n", passphraseFD)
	log.Infof("read new passphrase from fd %d", passphraseFD)
	var newPassphrase []byte
	defer bzero.Bytes(newPassphrase)
	if scanner.Scan() {
		newPassphrase = scanner.Bytes()
	} else if err := scanner.Err(); err != nil {
		return log.Error(err)
	}
	log.Info("done")
	// read new passphrase again
	fmt.Fprintf(statusfp, "read new passphrase from fd %d again\n", passphraseFD)
	log.Infof("read new passphrase from fd %d again", passphraseFD)
	var newPassphrase2 []byte
	defer bzero.Bytes(newPassphrase2)
	if scanner.Scan() {
		newPassphrase2 = scanner.Bytes()
	} else if err := scanner.Err(); err != nil {
		return log.Error(err)
	}
	log.Info("done")
	// compare new passphrases
	if !bytes.Equal(newPassphrase, newPassphrase2) {
		return log.Error(ErrPassphrasesDiffer)
	}
	// rekey msgDB
	log.Infof("rekey msgDB '%s'", msgdbname)
	if err := msgdb.Rekey(msgdbname, oldPassphrase, newPassphrase, c.Int("iterations")); err != nil {
		return err
	}
	// rekey keyDB
	log.Info("rekey keyDB")
	if err := rekeyKeyDB(c, oldPassphrase, newPassphrase); err != nil {
		return err
	}
	return nil
}

func mutecryptDBStatus(c *cli.Context, w io.Writer, passphrase []byte) error {
	args := []string{
		"--homedir", c.GlobalString("homedir"),
		"--loglevel", c.GlobalString("loglevel"),
		"--logdir", c.GlobalString("logdir"),
		"db", "status",
	}
	cmd := exec.Command("mutecrypt", args...)
	cmd.Stdout = w
	var errbuf bytes.Buffer
	cmd.Stderr = &errbuf
	ppR, ppW, err := os.Pipe()
	if err != nil {
		return err
	}
	defer ppR.Close()
	ppW.Write(passphrase)
	ppW.Close()
	cmd.ExtraFiles = append(cmd.ExtraFiles, ppR)
	if err := cmd.Start(); err != nil {
		return err
	}
	if err := cmd.Wait(); err != nil {
		return fmt.Errorf("%s: %s", err, errbuf.String())
	}
	return nil
}

func (ce *CtrlEngine) dbStatus(c *cli.Context, w io.Writer) error {
	autoVacuum, freelistCount, err := ce.msgDB.Status()
	if err != nil {
		return err
	}
	fmt.Fprintf(w, "msgdb:\n")
	fmt.Fprintf(w, "auto_vacuum=%s\n", autoVacuum)
	fmt.Fprintf(w, "freelist_count=%d\n", freelistCount)
	if err := mutecryptDBStatus(c, w, ce.passphrase); err != nil {
		return log.Error(err)
	}
	return nil
}

func mutecryptDBVacuum(
	c *cli.Context,
	passphrase []byte,
	autoVacuumMode string,
) error {
	args := []string{
		"--homedir", c.GlobalString("homedir"),
		"--loglevel", c.GlobalString("loglevel"),
		"--logdir", c.GlobalString("logdir"),
		"db", "vacuum",
	}
	if autoVacuumMode != "" {
		args = append(args, "--auto-vacuum", autoVacuumMode)
	}
	cmd := exec.Command("mutecrypt", args...)
	var errbuf bytes.Buffer
	cmd.Stderr = &errbuf
	ppR, ppW, err := os.Pipe()
	if err != nil {
		return err
	}
	defer ppR.Close()
	ppW.Write(passphrase)
	ppW.Close()
	cmd.ExtraFiles = append(cmd.ExtraFiles, ppR)
	if err := cmd.Start(); err != nil {
		return err
	}
	if err := cmd.Wait(); err != nil {
		return fmt.Errorf("%s: %s", err, errbuf.String())
	}
	return nil
}

func (ce *CtrlEngine) dbVacuum(c *cli.Context, autoVacuumMode string) error {
	if err := ce.msgDB.Vacuum(autoVacuumMode); err != nil {
		return err
	}
	if err := mutecryptDBVacuum(c, ce.passphrase, autoVacuumMode); err != nil {
		return log.Error(err)
	}
	return nil
}

func mutecryptDBIncremental(
	c *cli.Context,
	passphrase []byte,
	pages int64,
) error {
	args := []string{
		"--homedir", c.GlobalString("homedir"),
		"--loglevel", c.GlobalString("loglevel"),
		"--logdir", c.GlobalString("logdir"),
		"db", "incremental",
	}
	if pages != 0 {
		args = append(args, "--pages", strconv.FormatInt(pages, 10))
	}
	cmd := exec.Command("mutecrypt", args...)
	var errbuf bytes.Buffer
	cmd.Stderr = &errbuf
	ppR, ppW, err := os.Pipe()
	if err != nil {
		return err
	}
	defer ppR.Close()
	ppW.Write(passphrase)
	ppW.Close()
	cmd.ExtraFiles = append(cmd.ExtraFiles, ppR)
	if err := cmd.Start(); err != nil {
		return err
	}
	if err := cmd.Wait(); err != nil {
		return fmt.Errorf("%s: %s", err, errbuf.String())
	}
	return nil
}

func (ce *CtrlEngine) dbIncremental(c *cli.Context, pagesToRemove int64) error {
	if err := ce.msgDB.Incremental(pagesToRemove); err != nil {
		return err
	}
	err := mutecryptDBIncremental(c, ce.passphrase, pagesToRemove)
	if err != nil {
		return log.Error(err)
	}
	return nil
}

func mutecryptDBVersion(c *cli.Context, w io.Writer, passphrase []byte) error {
	args := []string{
		"--homedir", c.GlobalString("homedir"),
		"--loglevel", c.GlobalString("loglevel"),
		"--logdir", c.GlobalString("logdir"),
		"db", "version",
	}
	cmd := exec.Command("mutecrypt", args...)
	cmd.Stdout = w
	var errbuf bytes.Buffer
	cmd.Stderr = &errbuf
	ppR, ppW, err := os.Pipe()
	if err != nil {
		return err
	}
	defer ppR.Close()
	ppW.Write(passphrase)
	ppW.Close()
	cmd.ExtraFiles = append(cmd.ExtraFiles, ppR)
	if err := cmd.Start(); err != nil {
		return err
	}
	if err := cmd.Wait(); err != nil {
		return fmt.Errorf("%s: %s", err, errbuf.String())
	}
	return nil
}

func (ce *CtrlEngine) dbVersion(c *cli.Context, w io.Writer) error {
	version, err := ce.msgDB.Version()
	if err != nil {
		return err
	}
	fmt.Fprintf(w, "msgdb:\n")
	fmt.Fprintf(w, "version=%s\n", version)
	if err := mutecryptDBVersion(c, w, ce.passphrase); err != nil {
		return log.Error(err)
	}
	return nil
}
<commit_msg>ctrlengine: loop through mutecrypt --logconsole<commit_after>// Copyright (c) 2015 Mute Communications Ltd.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

package ctrlengine

import (
	"bufio"
	"bytes"
	"fmt"
	"io"
	"os"
	"os/exec"
	"path/filepath"
	"strconv"
	"syscall"

	"github.com/agl/ed25519"
	"github.com/codegangsta/cli"
	"github.com/mutecomm/mute/cipher"
	"github.com/mutecomm/mute/encode/base64"
	"github.com/mutecomm/mute/log"
	"github.com/mutecomm/mute/msgdb"
	"github.com/mutecomm/mute/util/bzero"
)

func createKeyDB(c *cli.Context, w io.Writer, passphrase []byte) error {
	args := []string{
		"--output-fd", strconv.Itoa(c.GlobalInt("output-fd")),
		"--passphrase-fd", strconv.FormatUint(uint64(syscall.Stdin), 10),
		"--homedir", c.GlobalString("homedir"),
		"--loglevel", c.GlobalString("loglevel"),
		"--logdir", c.GlobalString("logdir"),
	}
	if c.GlobalBool("logconsole") {
		args = append(args, "--logconsole")
	}
	args = append(args,
		"db", "create",
		"--iterations", strconv.Itoa(c.Int("iterations")),
	)
	cmd := exec.Command("mutecrypt", args...)
	stdin, err := cmd.StdinPipe()
	if err != nil {
		return err
	}
	var errbuf bytes.Buffer
	cmd.Stdout = w
	cmd.Stderr = &errbuf
	if err := cmd.Start(); err != nil {
		return err
	}
	plen := len(passphrase)
	buf := make([]byte, plen+1+plen+1)
	defer bzero.Bytes(buf)
	copy(buf, passphrase)
	copy(buf[plen:], []byte("\n"))
	copy(buf[plen+1:], passphrase)
	copy(buf[plen+1+plen:], []byte("\n"))
	if _, err := stdin.Write(buf); err != nil {
		return err
	}
	stdin.Close()
	if err := cmd.Wait(); err != nil {
		return fmt.Errorf("%s: %s", err, errbuf.String())
	}
	return nil
}

// create a new MsgDB and KeyDB.
func (ce *CtrlEngine) dbCreate(
	w, statusfp io.Writer,
	homedir string,
	c *cli.Context,
) error {
	msgdbname := filepath.Join(c.GlobalString("homedir"), "msgs")
	// read passphrase
	passphraseFD := c.GlobalInt("passphrase-fd")
	fmt.Fprintf(statusfp, "read passphrase from fd %d\n", passphraseFD)
	log.Infof("read passphrase from fd %d", passphraseFD)
	fp := os.NewFile(uintptr(passphraseFD), "passphrase-fd")
	scanner := bufio.NewScanner(fp)
	var passphrase []byte
	defer bzero.Bytes(passphrase)
	if scanner.Scan() {
		passphrase = scanner.Bytes()
	} else if err := scanner.Err(); err != nil {
		return log.Error(err)
	}
	log.Info("done")
	// read passphrase again
	fmt.Fprintf(statusfp, "read passphrase from fd %d again\n", passphraseFD)
	log.Infof("read passphrase from fd %d again", passphraseFD)
	var passphrase2 []byte
	defer bzero.Bytes(passphrase2)
	if scanner.Scan() {
		passphrase2 = scanner.Bytes()
	} else if err := scanner.Err(); err != nil {
		return log.Error(err)
	}
	log.Info("done")
	// compare passphrases
	if !bytes.Equal(passphrase, passphrase2) {
		return log.Error(ErrPassphrasesDiffer)
	}
	// create msgDB
	log.Infof("create msgDB '%s'", msgdbname)
	if err := msgdb.Create(msgdbname, passphrase, c.Int("iterations")); err != nil {
		return err
	}
	// open msgDB
	msgDB, err := msgdb.Open(msgdbname, passphrase)
	if err != nil {
		return err
	}
	defer msgDB.Close()
	// configure to make sure mutecrypt has config file
	err = ce.upkeepFetchconf(msgDB, homedir, false, nil, statusfp)
	if err != nil {
		return err
	}
	// create keyDB
	log.Info("create keyDB")
	if err := createKeyDB(c, w, passphrase); err != nil {
		return err
	}
	// status
	fmt.Fprintf(statusfp, "database files created\n")
	log.Info("database files created")
	// determine private walletKey
	walletKey := c.String("walletkey")
	if walletKey == "" {
		// generate wallet key
		_, privateKey, err := ed25519.GenerateKey(cipher.RandReader)
		if err != nil {
			return err
		}
		walletKey = base64.Encode(privateKey[:])
	}
	// store wallet key
	if err := msgDB.AddValue(msgdb.WalletKey, walletKey); err != nil {
		return err
	}
	// print wallet key
	if err := printWalletKey(w, walletKey); err != nil {
		return err
	}
	return nil
}

func rekeyKeyDB(c *cli.Context, oldPassphrase, newPassphrase []byte) error {
	cmd := exec.Command("mutecrypt",
		"--passphrase-fd", "0",
		"--homedir", c.GlobalString("homedir"),
		"--loglevel", c.GlobalString("loglevel"),
		"--logdir", c.GlobalString("logdir"),
		"db", "rekey",
		"--iterations", strconv.Itoa(c.Int("iterations")))
	stdin, err := cmd.StdinPipe()
	if err != nil {
		return err
	}
	var errbuf bytes.Buffer
	cmd.Stderr = &errbuf
	if err := cmd.Start(); err != nil {
		return err
	}
	olen := len(oldPassphrase)
	nlen := len(newPassphrase)
	buf := make([]byte, olen+1+nlen+1+nlen+1)
	defer bzero.Bytes(buf)
	copy(buf, oldPassphrase)
	copy(buf[olen:], []byte("\n"))
	copy(buf[olen+1:], newPassphrase)
	copy(buf[olen+1+nlen:], []byte("\n"))
	copy(buf[olen+1+nlen+1:], newPassphrase)
	copy(buf[olen+1+nlen+1+nlen:], []byte("\n"))
	if _, err := stdin.Write(buf); err != nil {
		return err
	}
	stdin.Close()
	if err := cmd.Wait(); err != nil {
		return fmt.Errorf("%s: %s", err, errbuf.String())
	}
	return nil
}

// rekey MsgDB and KeyDB.
func (ce *CtrlEngine) dbRekey(statusfp io.Writer, c *cli.Context) error {
	msgdbname := filepath.Join(c.GlobalString("homedir"), "msgs")
	// read old passphrase
	passphraseFD := c.GlobalInt("passphrase-fd")
	fmt.Fprintf(statusfp, "read old passphrase from fd %d\n", passphraseFD)
	log.Infof("read old passphrase from fd %d", passphraseFD)
	fp := os.NewFile(uintptr(passphraseFD), "passphrase-fd")
	scanner := bufio.NewScanner(fp)
	var oldPassphrase []byte
	defer bzero.Bytes(oldPassphrase)
	if scanner.Scan() {
		oldPassphrase = scanner.Bytes()
	} else if err := scanner.Err(); err != nil {
		return log.Error(err)
	}
	log.Info("done")
	// read new passphrase
	fmt.Fprintf(statusfp, "read new passphrase from fd %d\n", passphraseFD)
	log.Infof("read new passphrase from fd %d", passphraseFD)
	var newPassphrase []byte
	defer bzero.Bytes(newPassphrase)
	if scanner.Scan() {
		newPassphrase = scanner.Bytes()
	} else if err := scanner.Err(); err != nil {
		return log.Error(err)
	}
	log.Info("done")
	// read new passphrase again
	fmt.Fprintf(statusfp, "read new passphrase from fd %d again\n", passphraseFD)
	log.Infof("read new passphrase from fd %d again", passphraseFD)
	var newPassphrase2 []byte
	defer bzero.Bytes(newPassphrase2)
	if scanner.Scan() {
		newPassphrase2 = scanner.Bytes()
	} else if err := scanner.Err(); err != nil {
		return log.Error(err)
	}
	log.Info("done")
	// compare new passphrases
	if !bytes.Equal(newPassphrase, newPassphrase2) {
		return log.Error(ErrPassphrasesDiffer)
	}
	// rekey msgDB
	log.Infof("rekey msgDB '%s'", msgdbname)
	if err := msgdb.Rekey(msgdbname, oldPassphrase, newPassphrase, c.Int("iterations")); err != nil {
		return err
	}
	// rekey keyDB
	log.Info("rekey keyDB")
	if err := rekeyKeyDB(c, oldPassphrase, newPassphrase); err != nil {
		return err
	}
	return nil
}

func mutecryptDBStatus(c *cli.Context, w io.Writer, passphrase []byte) error {
	args := []string{
		"--homedir", c.GlobalString("homedir"),
		"--loglevel", c.GlobalString("loglevel"),
		"--logdir", c.GlobalString("logdir"),
		"db", "status",
	}
	cmd := exec.Command("mutecrypt", args...)
	cmd.Stdout = w
	var errbuf bytes.Buffer
	cmd.Stderr = &errbuf
	ppR, ppW, err := os.Pipe()
	if err != nil {
		return err
	}
	defer ppR.Close()
	ppW.Write(passphrase)
	ppW.Close()
	cmd.ExtraFiles = append(cmd.ExtraFiles, ppR)
	if err := cmd.Start(); err != nil {
		return err
	}
	if err := cmd.Wait(); err != nil {
		return fmt.Errorf("%s: %s", err, errbuf.String())
	}
	return nil
}

func (ce *CtrlEngine) dbStatus(c *cli.Context, w io.Writer) error {
	autoVacuum, freelistCount, err := ce.msgDB.Status()
	if err != nil {
		return err
	}
	fmt.Fprintf(w, "msgdb:\n")
	fmt.Fprintf(w, "auto_vacuum=%s\n", autoVacuum)
	fmt.Fprintf(w, "freelist_count=%d\n", freelistCount)
	if err := mutecryptDBStatus(c, w, ce.passphrase); err != nil {
		return log.Error(err)
	}
	return nil
}

func mutecryptDBVacuum(
	c *cli.Context,
	passphrase []byte,
	autoVacuumMode string,
) error {
	args := []string{
		"--homedir", c.GlobalString("homedir"),
		"--loglevel", c.GlobalString("loglevel"),
		"--logdir", c.GlobalString("logdir"),
		"db", "vacuum",
	}
	if autoVacuumMode != "" {
		args = append(args, "--auto-vacuum", autoVacuumMode)
	}
	cmd := exec.Command("mutecrypt", args...)
	var errbuf bytes.Buffer
	cmd.Stderr = &errbuf
	ppR, ppW, err := os.Pipe()
	if err != nil {
		return err
	}
	defer ppR.Close()
	ppW.Write(passphrase)
	ppW.Close()
	cmd.ExtraFiles = append(cmd.ExtraFiles, ppR)
	if err := cmd.Start(); err != nil {
		return err
	}
	if err := cmd.Wait(); err != nil {
		return fmt.Errorf("%s: %s", err, errbuf.String())
	}
	return nil
}

func (ce *CtrlEngine) dbVacuum(c *cli.Context, autoVacuumMode string) error {
	if err := ce.msgDB.Vacuum(autoVacuumMode); err != nil {
		return err
	}
	if err := mutecryptDBVacuum(c, ce.passphrase, autoVacuumMode); err != nil {
		return log.Error(err)
	}
	return nil
}

func mutecryptDBIncremental(
	c *cli.Context,
	passphrase []byte,
	pages int64,
) error {
	args := []string{
		"--homedir", c.GlobalString("homedir"),
		"--loglevel", c.GlobalString("loglevel"),
		"--logdir", c.GlobalString("logdir"),
		"db", "incremental",
	}
	if pages != 0 {
		args = append(args, "--pages", strconv.FormatInt(pages, 10))
	}
	cmd := exec.Command("mutecrypt", args...)
	var errbuf bytes.Buffer
	cmd.Stderr = &errbuf
	ppR, ppW, err := os.Pipe()
	if err != nil {
		return err
	}
	defer ppR.Close()
	ppW.Write(passphrase)
	ppW.Close()
	cmd.ExtraFiles = append(cmd.ExtraFiles, ppR)
	if err := cmd.Start(); err != nil {
		return err
	}
	if err := cmd.Wait(); err != nil {
		return fmt.Errorf("%s: %s", err, errbuf.String())
	}
	return nil
}

func (ce *CtrlEngine) dbIncremental(c *cli.Context, pagesToRemove int64) error {
	if err := ce.msgDB.Incremental(pagesToRemove); err != nil {
		return err
	}
	err := mutecryptDBIncremental(c, ce.passphrase, pagesToRemove)
	if err != nil {
		return log.Error(err)
	}
	return nil
}

func mutecryptDBVersion(c *cli.Context, w io.Writer, passphrase []byte) error {
	args := []string{
		"--homedir", c.GlobalString("homedir"),
		"--loglevel", c.GlobalString("loglevel"),
		"--logdir", c.GlobalString("logdir"),
		"db", "version",
	}
	cmd := exec.Command("mutecrypt", args...)
	cmd.Stdout = w
	var errbuf bytes.Buffer
	cmd.Stderr = &errbuf
	ppR, ppW, err := os.Pipe()
	if err != nil {
		return err
	}
	defer ppR.Close()
	ppW.Write(passphrase)
	ppW.Close()
	cmd.ExtraFiles = append(cmd.ExtraFiles, ppR)
	if err := cmd.Start(); err != nil {
		return err
	}
	if err := cmd.Wait(); err != nil {
		return fmt.Errorf("%s: %s", err, errbuf.String())
	}
	return nil
}

func (ce *CtrlEngine) dbVersion(c *cli.Context, w io.Writer) error {
	version, err := ce.msgDB.Version()
	if err != nil {
		return err
	}
	fmt.Fprintf(w, "msgdb:\n")
	fmt.Fprintf(w, "version=%s\n", version)
	if err := mutecryptDBVersion(c, w, ce.passphrase); err != nil {
		return log.Error(err)
	}
	return nil
}
<|endoftext|>


<commit_before>package email

import (
	"crypto/tls"
	"git.yichui.net/zj/mall/api/libs/uuid"
	"io"
	"net"
	"net/mail"
	"net/smtp"
	"runtime"
	"sync"
	"time"
)

type Worker struct {
	baseSender
	Tag       string
	Client    *smtp.Client
	config    *Config
	sender    baseSender
	address   *mail.Address
	isValid   bool
	isRunning bool
	tryCnt    int
	mutex     *sync.Mutex
}

func NewWorker(tag string, c *Config, s baseSender) *Worker {
	return &Worker{Tag: tag, config: c, sender: s, address: &mail.Address{c.User, c.User}, isValid: true, isRunning: false, tryCnt: 0, mutex: &sync.Mutex{}}
}

func (s *Worker) initClient() error {
	var (
		auth = smtp.PlainAuth(uuid.NewUUID(), s.config.User, s.config.Password, s.config.Host)
		host = s.config.Host + ":" + s.config.Port
		tc   = &tls.Config{
			InsecureSkipVerify: true,
			ServerName:         s.config.Host,
		}
		cnn net.Conn
		err error
	)
	if s.config.SSL == true {
		if cnn, err = tls.Dial("tcp", host, tc); err != nil {
			return err
		}
		if s.Client, err = smtp.NewClient(cnn, s.config.Host); err != nil {
			return err
		}
	} else {
		if s.Client, err = smtp.Dial(host); err != nil {
			return err
		}
	}
	if err = s.Client.Auth(auth); err != nil {
		return err
	}
	s.isValid = (err == nil)
	return err
}

func (s *Worker) quitClient() error {
	return s.Client.Quit()
}

func (s *Worker) IsIdle() bool {
	return !s.isRunning
}

func (s *Worker) Send(e *Email) error {
	if e == nil {
		return ERR_EMAIL_INVALID
	}
	e.AddTag(s.Tag)
	if !s.IsIdle() {
		return ERR_WORKER_RUNNING
	}
	s.mutex.Lock()
	defer s.mutex.Unlock()
	if err := s.initClient(); err != nil {
		return err
	}
	s.isRunning = true
	go s.handleEmail(e)
	return nil
}

func (s *Worker) handleEmail(e *Email) {
	runtime.Gosched()
	s.mutex.Lock()
	defer func() {
		s.isRunning = false
		s.mutex.Unlock()
		s.fetch()
	}()
	s.reset()
	for _, t := range e.To {
		s.sendOne(t, e)
	}
	s.quitClient()
}

func (s *Worker) fetch() {
	s.sender.fetch()
}

func (s *Worker) reset() {
	s.tryCnt = 0
}

func (s *Worker) sendOne(t string, e *Email) {
	var (
		err error
		msg []byte
		wc  io.WriteCloser
	)
	msg = []byte("To: " + t + "\r\n" +
		"From: " + s.address.String() + "\r\n" +
		"Subject: " + e.Subject + "\r\n" +
		e.Type + "\r\n\r\n" +
		e.Content)
	err = s.Client.Mail(s.address.Address)
	if err != nil {
		goto final
	}
	err = s.Client.Rcpt(t)
	if err != nil {
		goto final
	}
	wc, err = s.Client.Data()
	if err != nil {
		goto final
	}
	_, err = wc.Write(msg)
	if err != nil {
		goto final
	}
	err = wc.Close()
	if err != nil {
		goto final
	}
final:
	if err != nil {
		s.tryCnt++
		if s.tryCnt <= e.MaxTryCnt {
			time.Sleep(s.config.Sleep)
			s.sendOne(t, e)
		} else {
			s.pushOneFailure(t, *e, err)
		}
	} else {
		s.pushSuccess(e)
	}
}

func (s *Worker) pushSuccess(e *Email) {
	s.sender.pushSuccess(e)
}

func (s *Worker) pushFailure(e *Email, err error) {
	s.sender.pushFailure(e, err)
}

func (s *Worker) pushOneFailure(to string, e Email, err error) {
	e.To = []string{to}
	s.pushFailure(&e, err)
}
<commit_msg>fix: uuid<commit_after>package email

import (
	"crypto/tls"
	"github.com/WindomZ/go-macro/uuid"
	"io"
	"net"
	"net/mail"
	"net/smtp"
	"runtime"
	"sync"
	"time"
)

type Worker struct {
	baseSender
	Tag       string
	Client    *smtp.Client
	config    *Config
	sender    baseSender
	address   *mail.Address
	isValid   bool
	isRunning bool
	tryCnt    int
	mutex     *sync.Mutex
}

func NewWorker(tag string, c *Config, s baseSender) *Worker {
	return &Worker{Tag: tag, config: c, sender: s, address: &mail.Address{c.User, c.User}, isValid: true, isRunning: false, tryCnt: 0, mutex: &sync.Mutex{}}
}

func (s *Worker) initClient() error {
	var (
		auth = smtp.PlainAuth(uuid.NewUUID(), s.config.User, s.config.Password, s.config.Host)
		host = s.config.Host + ":" + s.config.Port
		tc   = &tls.Config{
			InsecureSkipVerify: true,
			ServerName:         s.config.Host,
		}
		cnn net.Conn
		err error
	)
	if s.config.SSL == true {
		if cnn, err = tls.Dial("tcp", host, tc); err != nil {
			return err
		}
		if s.Client, err = smtp.NewClient(cnn, s.config.Host); err != nil {
			return err
		}
	} else {
		if s.Client, err = smtp.Dial(host); err != nil {
			return err
		}
	}
	if err = s.Client.Auth(auth); err != nil {
		return err
	}
	s.isValid = (err == nil)
	return err
}

func (s *Worker) quitClient() error {
	return s.Client.Quit()
}

func (s *Worker) IsIdle() bool {
	return !s.isRunning
}

func (s *Worker) Send(e *Email) error {
	if e == nil {
		return ERR_EMAIL_INVALID
	}
	e.AddTag(s.Tag)
	if !s.IsIdle() {
		return ERR_WORKER_RUNNING
	}
	s.mutex.Lock()
	defer s.mutex.Unlock()
	if err := s.initClient(); err != nil {
		return err
	}
	s.isRunning = true
	go s.handleEmail(e)
	return nil
}

func (s *Worker) handleEmail(e *Email) {
	runtime.Gosched()
	s.mutex.Lock()
	defer func() {
		s.isRunning = false
		s.mutex.Unlock()
		s.fetch()
	}()
	s.reset()
	for _, t := range e.To {
		s.sendOne(t, e)
	}
	s.quitClient()
}

func (s *Worker) fetch() {
	s.sender.fetch()
}

func (s *Worker) reset() {
	s.tryCnt = 0
}

func (s *Worker) sendOne(t string, e *Email) {
	var (
		err error
		msg []byte
		wc  io.WriteCloser
	)
	msg = []byte("To: " + t + "\r\n" +
		"From: " + s.address.String() + "\r\n" +
		"Subject: " + e.Subject + "\r\n" +
		e.Type + "\r\n\r\n" +
		e.Content)
	err = s.Client.Mail(s.address.Address)
	if err != nil {
		goto final
	}
	err = s.Client.Rcpt(t)
	if err != nil {
		goto final
	}
	wc, err = s.Client.Data()
	if err != nil {
		goto final
	}
	_, err = wc.Write(msg)
	if err != nil {
		goto final
	}
	err = wc.Close()
	if err != nil {
		goto final
	}
final:
	if err != nil {
		s.tryCnt++
		if s.tryCnt <= e.MaxTryCnt {
			time.Sleep(s.config.Sleep)
			s.sendOne(t, e)
		} else {
			s.pushOneFailure(t, *e, err)
		}
	} else {
		s.pushSuccess(e)
	}
}

func (s *Worker) pushSuccess(e *Email) {
	s.sender.pushSuccess(e)
}

func (s *Worker) pushFailure(e *Email, err error) {
	s.sender.pushFailure(e, err)
}

func (s *Worker) pushOneFailure(to string, e Email, err error) {
	e.To = []string{to}
	s.pushFailure(&e, err)
}
<|endoftext|>


<commit_before>package marathon

import (
	"encoding/json"
	"github.com/QubitProducts/bamboo/configuration"
	"io/ioutil"
	"net/http"
	"sort"
	"strings"
)

// Describes an app process running
type Task struct {
	Host  string
	Port  int
	Ports []int
}

// A health check on the application
type HealthCheck struct {
	// One of TCP, HTTP or COMMAND
	Protocol string
	// The path (if Protocol is HTTP)
	Path string
	// The position of the port targeted in the ports array
	PortIndex int
}

// An app may have multiple processes
type App struct {
	Id              string
	EscapedId       string
	HealthCheckPath string
	HealthChecks    []HealthCheck
	Tasks           []Task
	ServicePort     int
	ServicePorts    []int
	Env             map[string]string
	Labels          map[string]string
}

type AppList []App

func (slice AppList) Len() int {
	return len(slice)
}

func (slice AppList) Less(i, j int) bool {
	return slice[i].Id < slice[j].Id
}

func (slice AppList) Swap(i, j int) {
	slice[i], slice[j] = slice[j], slice[i]
}

type marathonTaskList []marathonTask

type marathonTasks struct {
	Tasks marathonTaskList `json:"tasks"`
}

type marathonTask struct {
	AppId        string
	Id           string
	Host         string
	Ports        []int
	ServicePorts []int
	StartedAt    string
	StagedAt     string
	Version      string
}

func (slice marathonTaskList) Len() int {
	return len(slice)
}

func (slice marathonTaskList) Less(i, j int) bool {
	return slice[i].StagedAt < slice[j].StagedAt
}

func (slice marathonTaskList) Swap(i, j int) {
	slice[i], slice[j] = slice[j], slice[i]
}

type marathonApps struct {
	Apps []marathonApp `json:"apps"`
}

type marathonApp struct {
	Id           string            `json:"id"`
	HealthChecks []marathonHealthCheck     `json:"healthChecks"`
	Ports        []int             `json:"ports"`
	Env          map[string]string `json:"env"`
	Labels       map[string]string `json:"labels"`
}

type marathonHealthCheck struct {
	Path     string `json:"path"`
	Protocol string `json:"protocol"`
	PortIndex int `json:"portIndex"`
}

func fetchMarathonApps(endpoint string) (map[string]marathonApp, error) {
	response, err := http.Get(endpoint + "/v2/apps")

	if err != nil {
		return nil, err
	}

	defer response.Body.Close()
	var appResponse marathonApps

	contents, err := ioutil.ReadAll(response.Body)
	if err != nil {
		return nil, err
	}

	err = json.Unmarshal(contents, &appResponse)
	if err != nil {
		return nil, err
	}

	dataById := map[string]marathonApp{}

	for _, appConfig := range appResponse.Apps {
		dataById[appConfig.Id] = appConfig
	}

	return dataById, nil
}

func fetchTasks(endpoint string) (map[string][]marathonTask, error) {
	client := &http.Client{}
	req, err := http.NewRequest("GET", endpoint+"/v2/tasks", nil)
	req.Header.Add("Accept", "application/json")
	response, err := client.Do(req)

	var tasks marathonTasks

	if err != nil {
		return nil, err
	}

	contents, err := ioutil.ReadAll(response.Body)
	defer response.Body.Close()
	if err != nil {
		return nil, err
	}

	err = json.Unmarshal(contents, &tasks)
	if err != nil {
		return nil, err
	}

	taskList := tasks.Tasks
	sort.Sort(taskList)

	tasksById := map[string][]marathonTask{}
	for _, task := range taskList {
		if tasksById[task.AppId] == nil {
			tasksById[task.AppId] = []marathonTask{}
		}
		tasksById[task.AppId] = append(tasksById[task.AppId], task)
	}

	return tasksById, nil
}

func createApps(tasksById map[string][]marathonTask, marathonApps map[string]marathonApp) AppList {

	apps := AppList{}

	for appId, tasks := range tasksById {
		marathonApp := marathonApps[appId]

		simpleTasks := []Task{}

		for _, task := range tasks {
			if len(task.Ports) > 0 {
				simpleTasks = append(simpleTasks, Task{Host: task.Host, Port: task.Ports[0], Ports: task.Ports})
			}
		}

		// Try to handle old app id format without slashes
		appPath := appId
		if !strings.HasPrefix(appId, "/") {
			appPath = "/" + appId
		}

		app := App{
			// Since Marathon 0.7, apps are namespaced with path
			Id: appPath,
			// Used for template
			EscapedId:       strings.Replace(appId, "/", "::", -1),
			Tasks:           simpleTasks,
			HealthCheckPath: parseHealthCheckPath(marathonApp.HealthChecks),
			Env:             marathonApps[appId].Env,
			Labels:          marathonApps[appId].Labels,
		}

		app.HealthChecks = make([]HealthCheck, 0, len(marathonApp.HealthChecks))
		for _, marathonCheck := range marathonApp.HealthChecks {
			check := HealthCheck{
				Protocol: marathonCheck.Protocol,
				Path: marathonCheck.Path,
				PortIndex: marathonCheck.PortIndex,
			}
			app.HealthChecks = append(app.HealthChecks, check)
		}

		if len(marathonApps[appId].Ports) > 0 {
			app.ServicePort = marathonApps[appId].Ports[0]
			app.ServicePorts = marathonApps[appId].Ports
		}

		apps = append(apps, app)
	}
	return apps
}

func parseHealthCheckPath(checks []marathonHealthCheck) string {
	for _, check := range checks {
		if check.Protocol != "HTTP" {
			continue
		}
		return check.Path
	}
	return ""
}

/*
	Apps returns a struct that describes Marathon current app and their
	sub tasks information.

	Parameters:
		endpoint: Marathon HTTP endpoint, e.g. http://localhost:8080
*/
func FetchApps(maraconf configuration.Marathon) (AppList, error) {

	var applist AppList
	var err error

	// try all configured endpoints until one succeeds
	for _, url := range maraconf.Endpoints() {
		applist, err = _fetchApps(url)
		if err == nil {
			return applist, err
		}
	}
	// return last error
	return nil, err
}

func _fetchApps(url string) (AppList, error) {
	tasks, err := fetchTasks(url)
	if err != nil {
		return nil, err
	}

	marathonApps, err := fetchMarathonApps(url)
	if err != nil {
		return nil, err
	}

	apps := createApps(tasks, marathonApps)
	sort.Sort(apps)
	return apps, nil
}
<commit_msg>go fmt ./...<commit_after>package marathon

import (
	"encoding/json"
	"github.com/QubitProducts/bamboo/configuration"
	"io/ioutil"
	"net/http"
	"sort"
	"strings"
)

// Describes an app process running
type Task struct {
	Host  string
	Port  int
	Ports []int
}

// A health check on the application
type HealthCheck struct {
	// One of TCP, HTTP or COMMAND
	Protocol string
	// The path (if Protocol is HTTP)
	Path string
	// The position of the port targeted in the ports array
	PortIndex int
}

// An app may have multiple processes
type App struct {
	Id              string
	EscapedId       string
	HealthCheckPath string
	HealthChecks    []HealthCheck
	Tasks           []Task
	ServicePort     int
	ServicePorts    []int
	Env             map[string]string
	Labels          map[string]string
}

type AppList []App

func (slice AppList) Len() int {
	return len(slice)
}

func (slice AppList) Less(i, j int) bool {
	return slice[i].Id < slice[j].Id
}

func (slice AppList) Swap(i, j int) {
	slice[i], slice[j] = slice[j], slice[i]
}

type marathonTaskList []marathonTask

type marathonTasks struct {
	Tasks marathonTaskList `json:"tasks"`
}

type marathonTask struct {
	AppId        string
	Id           string
	Host         string
	Ports        []int
	ServicePorts []int
	StartedAt    string
	StagedAt     string
	Version      string
}

func (slice marathonTaskList) Len() int {
	return len(slice)
}

func (slice marathonTaskList) Less(i, j int) bool {
	return slice[i].StagedAt < slice[j].StagedAt
}

func (slice marathonTaskList) Swap(i, j int) {
	slice[i], slice[j] = slice[j], slice[i]
}

type marathonApps struct {
	Apps []marathonApp `json:"apps"`
}

type marathonApp struct {
	Id           string                `json:"id"`
	HealthChecks []marathonHealthCheck `json:"healthChecks"`
	Ports        []int                 `json:"ports"`
	Env          map[string]string     `json:"env"`
	Labels       map[string]string     `json:"labels"`
}

type marathonHealthCheck struct {
	Path      string `json:"path"`
	Protocol  string `json:"protocol"`
	PortIndex int    `json:"portIndex"`
}

func fetchMarathonApps(endpoint string) (map[string]marathonApp, error) {
	response, err := http.Get(endpoint + "/v2/apps")

	if err != nil {
		return nil, err
	}

	defer response.Body.Close()
	var appResponse marathonApps

	contents, err := ioutil.ReadAll(response.Body)
	if err != nil {
		return nil, err
	}

	err = json.Unmarshal(contents, &appResponse)
	if err != nil {
		return nil, err
	}

	dataById := map[string]marathonApp{}

	for _, appConfig := range appResponse.Apps {
		dataById[appConfig.Id] = appConfig
	}

	return dataById, nil
}

func fetchTasks(endpoint string) (map[string][]marathonTask, error) {
	client := &http.Client{}
	req, err := http.NewRequest("GET", endpoint+"/v2/tasks", nil)
	req.Header.Add("Accept", "application/json")
	response, err := client.Do(req)

	var tasks marathonTasks

	if err != nil {
		return nil, err
	}

	contents, err := ioutil.ReadAll(response.Body)
	defer response.Body.Close()
	if err != nil {
		return nil, err
	}

	err = json.Unmarshal(contents, &tasks)
	if err != nil {
		return nil, err
	}

	taskList := tasks.Tasks
	sort.Sort(taskList)

	tasksById := map[string][]marathonTask{}
	for _, task := range taskList {
		if tasksById[task.AppId] == nil {
			tasksById[task.AppId] = []marathonTask{}
		}
		tasksById[task.AppId] = append(tasksById[task.AppId], task)
	}

	return tasksById, nil
}

func createApps(tasksById map[string][]marathonTask, marathonApps map[string]marathonApp) AppList {

	apps := AppList{}

	for appId, tasks := range tasksById {
		marathonApp := marathonApps[appId]

		simpleTasks := []Task{}

		for _, task := range tasks {
			if len(task.Ports) > 0 {
				simpleTasks = append(simpleTasks, Task{Host: task.Host, Port: task.Ports[0], Ports: task.Ports})
			}
		}

		// Try to handle old app id format without slashes
		appPath := appId
		if !strings.HasPrefix(appId, "/") {
			appPath = "/" + appId
		}

		app := App{
			// Since Marathon 0.7, apps are namespaced with path
			Id: appPath,
			// Used for template
			EscapedId:       strings.Replace(appId, "/", "::", -1),
			Tasks:           simpleTasks,
			HealthCheckPath: parseHealthCheckPath(marathonApp.HealthChecks),
			Env:             marathonApps[appId].Env,
			Labels:          marathonApps[appId].Labels,
		}

		app.HealthChecks = make([]HealthCheck, 0, len(marathonApp.HealthChecks))
		for _, marathonCheck := range marathonApp.HealthChecks {
			check := HealthCheck{
				Protocol:  marathonCheck.Protocol,
				Path:      marathonCheck.Path,
				PortIndex: marathonCheck.PortIndex,
			}
			app.HealthChecks = append(app.HealthChecks, check)
		}

		if len(marathonApps[appId].Ports) > 0 {
			app.ServicePort = marathonApps[appId].Ports[0]
			app.ServicePorts = marathonApps[appId].Ports
		}

		apps = append(apps, app)
	}
	return apps
}

func parseHealthCheckPath(checks []marathonHealthCheck) string {
	for _, check := range checks {
		if check.Protocol != "HTTP" {
			continue
		}
		return check.Path
	}
	return ""
}

/*
	Apps returns a struct that describes Marathon current app and their
	sub tasks information.

	Parameters:
		endpoint: Marathon HTTP endpoint, e.g. http://localhost:8080
*/
func FetchApps(maraconf configuration.Marathon) (AppList, error) {

	var applist AppList
	var err error

	// try all configured endpoints until one succeeds
	for _, url := range maraconf.Endpoints() {
		applist, err = _fetchApps(url)
		if err == nil {
			return applist, err
		}
	}
	// return last error
	return nil, err
}

func _fetchApps(url string) (AppList, error) {
	tasks, err := fetchTasks(url)
	if err != nil {
		return nil, err
	}

	marathonApps, err := fetchMarathonApps(url)
	if err != nil {
		return nil, err
	}

	apps := createApps(tasks, marathonApps)
	sort.Sort(apps)
	return apps, nil
}
<|endoftext|>