<commit_before>// Copyright 2014 Canonical Ltd.
// Licensed under the AGPLv3, see LICENCE file for details.

package state

import (
	"encoding/json"
	"fmt"
	"time"

	"github.com/juju/loggo"

	"github.com/juju/errors"
	"github.com/juju/names"
	"github.com/juju/utils"
	"gopkg.in/juju/charm.v3"
	"gopkg.in/mgo.v2"
	"gopkg.in/mgo.v2/bson"
	"gopkg.in/mgo.v2/txn"
)

var metricsLogger = loggo.GetLogger("juju.state.metrics")

// MetricBatch represents a batch of metrics reported from a unit.
// These will be received from the unit in batches.
// The main contents of the metric (key, value) is defined
// by the charm author and sent from the unit via a call to
// add-metric
type MetricBatch struct {
	st  *State
	doc metricBatchDoc
}

type metricBatchDoc struct {
	UUID     string    `bson:"_id"`
	EnvUUID  string    `bson:"envuuid"`
	Unit     string    `bson:"unit"`
	CharmUrl string    `bson:"charmurl"`
	Sent     bool      `bson:"sent"`
	Created  time.Time `bson:"created"`
	Metrics  []Metric  `bson:"metrics"`
}

// Metric represents a single Metric.
type Metric struct {
	Key         string    `bson:"key"`
	Value       string    `bson:"value"`
	Time        time.Time `bson:"time"`
	Credentials []byte    `bson:"credentials"`
}

// AddMetric adds a new batch of metrics to the database.
// A UUID for the metric will be generated and the new MetricBatch will be returned
func (st *State) addMetrics(unitTag names.UnitTag, charmUrl *charm.URL, created time.Time, metrics []Metric) (*MetricBatch, error) {
	if len(metrics) == 0 {
		return nil, errors.New("cannot add a batch of 0 metrics")
	}
	uuid, err := utils.NewUUID()
	if err != nil {
		return nil, err
	}

	metric := &MetricBatch{
		st: st,
		doc: metricBatchDoc{
			UUID:     uuid.String(),
			EnvUUID:  st.EnvironTag().String(),
			Unit:     unitTag.Id(),
			CharmUrl: charmUrl.String(),
			Sent:     false,
			Created:  created,
			Metrics:  metrics,
		}}
	buildTxn := func(attempt int) ([]txn.Op, error) {
		if attempt > 0 {
			notDead, err := isNotDead(st.db, unitsC, unitTag.Id())
			if err != nil || !notDead {
				return nil, errors.NotFoundf(unitTag.Id())
			}
		}
		ops := []txn.Op{{
			C:      unitsC,
			Id:     unitTag.Id(),
			Assert: notDeadDoc,
		}, {
			C:      metricsC,
			Id:     metric.UUID(),
			Assert: txn.DocMissing,
			Insert: &metric.doc,
		}}
		return ops, nil
	}
	err = st.run(buildTxn)
	if err != nil {
		return nil, errors.Trace(err)
	}

	return metric, nil
}

// MetricBatches returns all metric batches currently stored in state.
// TODO (tasdomas): this method is currently only used in the uniter worker test -
//                  it needs to be modified to restrict the scope of the values it
//                  returns if it is to be used outside of tests.
func (st *State) MetricBatches() ([]MetricBatch, error) {
	c, closer := st.getCollection(metricsC)
	defer closer()
	docs := []metricBatchDoc{}
	err := c.Find(nil).All(&docs)
	if err != nil {
		return nil, errors.Trace(err)
	}
	results := make([]MetricBatch, len(docs))
	for i, doc := range docs {
		results[i] = MetricBatch{st: st, doc: doc}
	}
	return results, nil
}

// MetricBatch returns the metric batch with the given id.
func (st *State) MetricBatch(id string) (*MetricBatch, error) {
	c, closer := st.getCollection(metricsC)
	defer closer()
	doc := metricBatchDoc{}
	err := c.Find(bson.M{"_id": id}).One(&doc)
	if err == mgo.ErrNotFound {
		return nil, errors.NotFoundf("metric %v", id)
	}
	if err != nil {
		return nil, err
	}
	return &MetricBatch{st: st, doc: doc}, nil
}

// CleanupOldMetrics looks for metrics that are 24 hours old (or older)
// and have been sent. Any metrics it finds are deleted.
func (st *State) CleanupOldMetrics() error {
	age := time.Now().Add(-(time.Hour * 24))
	c, closer := st.getCollection(metricsC)
	defer closer()
	// Nothing else in the system will interact with sent metrics, and nothing needs
	// to watch them either; so in this instance it's safe to do an end run around the
	// mgo/txn package. See State.cleanupRelationSettings for a similar situation.
	err := c.Remove(bson.M{
		"sent":    true,
		"created": bson.M{"$lte": age},
	})
	if err == mgo.ErrNotFound {
		metricsLogger.Infof("no metrics found to cleanup")
		return nil
	}
	return err
}

// MetricSender defines the interface used to send metrics
// to a collection service.
type MetricSender interface {
	Send([]*MetricBatch) error
}

// SendMetrics will send any unsent metrics
// over the MetricSender interface in batches
// no larger than batchSize.
func (st *State) SendMetrics(sender MetricSender, batchSize int) error {
	for {
		var docs []metricBatchDoc
		c, closer := st.getCollection(metricsC)
		defer closer()
		err := c.Find(bson.M{
			"sent": false,
		}).Limit(batchSize).All(&docs)
		if err != nil {
			return errors.Trace(err)
		}

		if len(docs) == 0 {
			break
		}
		batch := make([]*MetricBatch, len(docs))
		for i, doc := range docs {
			batch[i] = &MetricBatch{st: st, doc: doc}

		}

		err = st.sendBatch(sender, batch)
		if err != nil {
			return errors.Trace(err)
		}
	}
	unsent, err := st.countofUnsentMetrics()
	if err != nil {
		return errors.Trace(err)
	}
	sent, err := st.countofSentMetrics()
	if err != nil {
		return errors.Trace(err)
	}
	metricsLogger.Infof("metrics collection summary: sent:%d unsent:%d", sent, unsent)

	return nil
}

// countofUnsentMetrics returns the number of metrics that
// haven't been sent to the collection service.
func (st *State) countofUnsentMetrics() (int, error) {
	c, closer := st.getCollection(metricsC)
	defer closer()
	return c.Find(bson.M{
		"sent": false,
	}).Count()
}

// countofSentMetrics returns the number of metrics that
// have been sent to the collection service and have not
// been removed by the cleanup worker.
func (st *State) countofSentMetrics() (int, error) {
	c, closer := st.getCollection(metricsC)
	defer closer()
	return c.Find(bson.M{
		"sent": true,
	}).Count()
}

// sendBatch send metrics over the MetricSender interface and
// sets the metric's sent attribute.
func (st *State) sendBatch(sender MetricSender, metrics []*MetricBatch) error {
	err := sender.Send(metrics)
	if err != nil {
		return errors.Trace(err)
	}
	err = st.setMetricBatchesSent(metrics)
	if err != nil {
		metricsLogger.Warningf("failed to setsent on metrics", err)
	}
	return nil
}

// MarshalJSON defines how the MetricBatch type should be
// converted to json.
func (m *MetricBatch) MarshalJSON() ([]byte, error) {
	return json.Marshal(m.doc)
}

// Refresh refreshes the contents of the MetricBatch from the underlying
// state. It an error that satisfies errors.IsNotFound if the unit has
// been removed.
func (m *MetricBatch) Refresh() error {
	metrics, closer := m.st.getCollection(metricsC)
	defer closer()

	err := metrics.FindId(m.doc.UUID).One(&m.doc)
	if err == mgo.ErrNotFound {
		return errors.NotFoundf("metric %q", m)
	}
	if err != nil {
		return fmt.Errorf("cannot refresh metric %q: %v", m, err)
	}
	return nil
}

// UUID returns to uuid of the metric.
func (m *MetricBatch) UUID() string {
	return m.doc.UUID
}

// EnvUUID returns the environment UUID this metric applies to.
func (m *MetricBatch) EnvUUID() string {
	return m.doc.EnvUUID
}

// Unit returns the name of the unit this metric was generated in.
func (m *MetricBatch) Unit() string {
	return m.doc.Unit
}

// CharmURL returns the charm url for the charm this metric was generated in.
func (m *MetricBatch) CharmURL() string {
	return m.doc.CharmUrl
}

// Sent returns a flag to tell us if this metric has been sent to the metric
// collection service
func (m *MetricBatch) Sent() bool {
	return m.doc.Sent
}

// Metrics returns the metrics in this batch.
func (m *MetricBatch) Metrics() []Metric {
	result := make([]Metric, len(m.doc.Metrics))
	copy(result, m.doc.Metrics)
	return result
}

// SetSent sets the sent flag to true
func (m *MetricBatch) SetSent() error {
	ops := setSentOps([]*MetricBatch{m})
	if err := m.st.runTransaction(ops); err != nil {
		return errors.Annotatef(err, "cannot set metric sent for metric %q", m.UUID())
	}

	m.doc.Sent = true
	return nil
}

func setSentOps(metrics []*MetricBatch) []txn.Op {
	ops := make([]txn.Op, len(metrics))
	for i, m := range metrics {
		ops[i] = txn.Op{
			C:      metricsC,
			Id:     m.UUID(),
			Assert: txn.DocExists,
			Update: bson.M{"$set": bson.M{"sent": true}},
		}
	}
	return ops
}

func (st *State) setMetricBatchesSent(metrics []*MetricBatch) error {
	ops := setSentOps(metrics)
	if err := st.runTransaction(ops); err != nil {
		return errors.Annotatef(err, "cannot set metric sent in bulk call")
	}
	for _, m := range metrics {
		m.doc.Sent = true
	}
	return nil
}
<commit_msg>fixes from review<commit_after>// Copyright 2014 Canonical Ltd.
// Licensed under the AGPLv3, see LICENCE file for details.

package state

import (
	"encoding/json"
	"time"

	"github.com/juju/loggo"

	"github.com/juju/errors"
	"github.com/juju/names"
	"github.com/juju/utils"
	"gopkg.in/juju/charm.v3"
	"gopkg.in/mgo.v2"
	"gopkg.in/mgo.v2/bson"
	"gopkg.in/mgo.v2/txn"
)

var metricsLogger = loggo.GetLogger("juju.state.metrics")

// MetricBatch represents a batch of metrics reported from a unit.
// These will be received from the unit in batches.
// The main contents of the metric (key, value) is defined
// by the charm author and sent from the unit via a call to
// add-metric
type MetricBatch struct {
	st  *State
	doc metricBatchDoc
}

type metricBatchDoc struct {
	UUID     string    `bson:"_id"`
	EnvUUID  string    `bson:"envuuid"`
	Unit     string    `bson:"unit"`
	CharmUrl string    `bson:"charmurl"`
	Sent     bool      `bson:"sent"`
	Created  time.Time `bson:"created"`
	Metrics  []Metric  `bson:"metrics"`
}

// Metric represents a single Metric.
type Metric struct {
	Key         string    `bson:"key"`
	Value       string    `bson:"value"`
	Time        time.Time `bson:"time"`
	Credentials []byte    `bson:"credentials"`
}

// AddMetric adds a new batch of metrics to the database.
// A UUID for the metric will be generated and the new MetricBatch will be returned
func (st *State) addMetrics(unitTag names.UnitTag, charmUrl *charm.URL, created time.Time, metrics []Metric) (*MetricBatch, error) {
	if len(metrics) == 0 {
		return nil, errors.New("cannot add a batch of 0 metrics")
	}
	uuid, err := utils.NewUUID()
	if err != nil {
		return nil, err
	}

	metric := &MetricBatch{
		st: st,
		doc: metricBatchDoc{
			UUID:     uuid.String(),
			EnvUUID:  st.EnvironTag().String(),
			Unit:     unitTag.Id(),
			CharmUrl: charmUrl.String(),
			Sent:     false,
			Created:  created,
			Metrics:  metrics,
		}}
	buildTxn := func(attempt int) ([]txn.Op, error) {
		if attempt > 0 {
			notDead, err := isNotDead(st.db, unitsC, unitTag.Id())
			if err != nil || !notDead {
				return nil, errors.NotFoundf(unitTag.Id())
			}
		}
		ops := []txn.Op{{
			C:      unitsC,
			Id:     unitTag.Id(),
			Assert: notDeadDoc,
		}, {
			C:      metricsC,
			Id:     metric.UUID(),
			Assert: txn.DocMissing,
			Insert: &metric.doc,
		}}
		return ops, nil
	}
	err = st.run(buildTxn)
	if err != nil {
		return nil, errors.Trace(err)
	}

	return metric, nil
}

// MetricBatches returns all metric batches currently stored in state.
// TODO (tasdomas): this method is currently only used in the uniter worker test -
//                  it needs to be modified to restrict the scope of the values it
//                  returns if it is to be used outside of tests.
func (st *State) MetricBatches() ([]MetricBatch, error) {
	c, closer := st.getCollection(metricsC)
	defer closer()
	docs := []metricBatchDoc{}
	err := c.Find(nil).All(&docs)
	if err != nil {
		return nil, errors.Trace(err)
	}
	results := make([]MetricBatch, len(docs))
	for i, doc := range docs {
		results[i] = MetricBatch{st: st, doc: doc}
	}
	return results, nil
}

// MetricBatch returns the metric batch with the given id.
func (st *State) MetricBatch(id string) (*MetricBatch, error) {
	c, closer := st.getCollection(metricsC)
	defer closer()
	doc := metricBatchDoc{}
	err := c.Find(bson.M{"_id": id}).One(&doc)
	if err == mgo.ErrNotFound {
		return nil, errors.NotFoundf("metric %v", id)
	}
	if err != nil {
		return nil, err
	}
	return &MetricBatch{st: st, doc: doc}, nil
}

// CleanupOldMetrics looks for metrics that are 24 hours old (or older)
// and have been sent. Any metrics it finds are deleted.
func (st *State) CleanupOldMetrics() error {
	age := time.Now().Add(-(time.Hour * 24))
	c, closer := st.getCollection(metricsC)
	defer closer()
	// Nothing else in the system will interact with sent metrics, and nothing needs
	// to watch them either; so in this instance it's safe to do an end run around the
	// mgo/txn package. See State.cleanupRelationSettings for a similar situation.
	err := c.Remove(bson.M{
		"sent":    true,
		"created": bson.M{"$lte": age},
	})
	if err == mgo.ErrNotFound {
		metricsLogger.Infof("no metrics found to cleanup")
		return nil
	}
	return err
}

// MetricSender defines the interface used to send metrics
// to a collection service.
type MetricSender interface {
	Send([]*MetricBatch) error
}

// SendMetrics will send any unsent metrics
// over the MetricSender interface in batches
// no larger than batchSize.
func (st *State) SendMetrics(sender MetricSender, batchSize int) error {
	for {
		var docs []metricBatchDoc
		c, closer := st.getCollection(metricsC)
		defer closer()
		err := c.Find(bson.M{
			"sent": false,
		}).Limit(batchSize).All(&docs)
		if err != nil {
			return errors.Trace(err)
		}

		if len(docs) == 0 {
			break
		}
		batch := make([]*MetricBatch, len(docs))
		for i, doc := range docs {
			batch[i] = &MetricBatch{st: st, doc: doc}

		}

		err = st.sendBatch(sender, batch)
		if err != nil {
			return errors.Trace(err)
		}
	}
	unsent, err := st.countofUnsentMetrics()
	if err != nil {
		return errors.Trace(err)
	}
	sent, err := st.countofSentMetrics()
	if err != nil {
		return errors.Trace(err)
	}
	metricsLogger.Infof("metrics collection summary: sent:%d unsent:%d", sent, unsent)

	return nil
}

// countofUnsentMetrics returns the number of metrics that
// haven't been sent to the collection service.
func (st *State) countofUnsentMetrics() (int, error) {
	c, closer := st.getCollection(metricsC)
	defer closer()
	return c.Find(bson.M{
		"sent": false,
	}).Count()
}

// countofSentMetrics returns the number of metrics that
// have been sent to the collection service and have not
// been removed by the cleanup worker.
func (st *State) countofSentMetrics() (int, error) {
	c, closer := st.getCollection(metricsC)
	defer closer()
	return c.Find(bson.M{
		"sent": true,
	}).Count()
}

// sendBatch send metrics over the MetricSender interface and
// sets the metric's sent attribute.
func (st *State) sendBatch(sender MetricSender, metrics []*MetricBatch) error {
	err := sender.Send(metrics)
	if err != nil {
		return errors.Trace(err)
	}
	err = st.setMetricBatchesSent(metrics)
	if err != nil {
		metricsLogger.Warningf("failed to setsent on metrics", err)
	}
	return nil
}

// MarshalJSON defines how the MetricBatch type should be
// converted to json.
func (m *MetricBatch) MarshalJSON() ([]byte, error) {
	return json.Marshal(m.doc)
}

// Refresh refreshes the contents of the MetricBatch from the underlying state.
func (m *MetricBatch) Refresh() error {
	metrics, closer := m.st.getCollection(metricsC)
	defer closer()

	err := metrics.FindId(m.doc.UUID).One(&m.doc)
	if err == mgo.ErrNotFound {
		return errors.NotFoundf("metric %q", m)
	}
	if err != nil {
		return errors.Annotatef(err, "cannot refresh metric %q", m)
	}
	return nil
}

// UUID returns to uuid of the metric.
func (m *MetricBatch) UUID() string {
	return m.doc.UUID
}

// EnvUUID returns the environment UUID this metric applies to.
func (m *MetricBatch) EnvUUID() string {
	return m.doc.EnvUUID
}

// Unit returns the name of the unit this metric was generated in.
func (m *MetricBatch) Unit() string {
	return m.doc.Unit
}

// CharmURL returns the charm url for the charm this metric was generated in.
func (m *MetricBatch) CharmURL() string {
	return m.doc.CharmUrl
}

// Sent returns a flag to tell us if this metric has been sent to the metric
// collection service
func (m *MetricBatch) Sent() bool {
	return m.doc.Sent
}

// Metrics returns the metrics in this batch.
func (m *MetricBatch) Metrics() []Metric {
	result := make([]Metric, len(m.doc.Metrics))
	copy(result, m.doc.Metrics)
	return result
}

// SetSent sets the sent flag to true
func (m *MetricBatch) SetSent() error {
	ops := setSentOps([]*MetricBatch{m})
	if err := m.st.runTransaction(ops); err != nil {
		return errors.Annotatef(err, "cannot set metric sent for metric %q", m.UUID())
	}

	m.doc.Sent = true
	return nil
}

func setSentOps(metrics []*MetricBatch) []txn.Op {
	ops := make([]txn.Op, len(metrics))
	for i, m := range metrics {
		ops[i] = txn.Op{
			C:      metricsC,
			Id:     m.UUID(),
			Assert: txn.DocExists,
			Update: bson.M{"$set": bson.M{"sent": true}},
		}
	}
	return ops
}

func (st *State) setMetricBatchesSent(metrics []*MetricBatch) error {
	ops := setSentOps(metrics)
	if err := st.runTransaction(ops); err != nil {
		return errors.Annotatef(err, "cannot set metric sent in bulk call")
	}
	for _, m := range metrics {
		m.doc.Sent = true
	}
	return nil
}
<|endoftext|>


<commit_before>package gbrotli

import (
	"io"

	"github.com/dsnet/compress/brotli"
	"github.com/itchio/wharf/pwr"
)

type brotliDecompressor struct{}

func (bc *brotliDecompressor) Apply(reader io.Reader) (io.Reader, error) {
	br := brotli.NewReader(reader, nil)
	return br, nil
}

func init() {
	pwr.RegisterDecompressor(pwr.CompressionAlgorithm_BROTLI, &brotliDecompressor{})
}
<commit_msg>Adjust to new dsnet/compress API, pt. 2<commit_after>package gbrotli

import (
	"io"

	"github.com/dsnet/compress/brotli"
	"github.com/go-errors/errors"
	"github.com/itchio/wharf/pwr"
)

type brotliDecompressor struct{}

func (bc *brotliDecompressor) Apply(reader io.Reader) (io.Reader, error) {
	br, err := brotli.NewReader(reader, nil)
	if err != nil {
		return nil, errors.Wrap(err, 1)
	}
	return br, nil
}

func init() {
	pwr.RegisterDecompressor(pwr.CompressionAlgorithm_BROTLI, &brotliDecompressor{})
}
<|endoftext|>


<commit_before>package main

import "fmt"
import "net/http"
import "net/http/httputil"
import "net/url"
import "time"
import "log"
import "encoding/base64"
import "flag"
import "os"
import "github.com/elazarl/goproxy"

func Log(handler http.Handler) http.Handler {
    return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
        log.Printf("%s %s %s", r.RemoteAddr, r.Method, r.URL)
        handler.ServeHTTP(w, r)
    })
}

var record = flag.Bool("record", false, "run a http proxy that will record requests")
var playback = flag.String("playback", "", "playback a recoreded session")
var target = flag.String("target", "", "the target host to record")

func getTimeMilis() int64 {
    return time.Now().UnixNano()/1e6
}

func runProxy() {
    proxy := goproxy.NewProxyHttpServer()
    proxy.Verbose = true
    client := http.Client{}
    fo, err := os.Create(".requests")
    if err != nil { panic(err) }
    // close fo on exit and check for its returned error
    defer func() {
        if err := fo.Close(); err != nil {
            panic(err)
        }
    }()

    startTimeMilis := getTimeMilis()

    var c chan string = make(chan string)
    go func() {
        for {
            line := <- c
            fo.WriteString(line)
            fo.Sync()
        }
    }()

    proxy.OnRequest().DoFunc(func(r *http.Request,ctx *goproxy.ProxyCtx)(*http.Request,*http.Response) {
        fmt.Println(r.Host)
        request_url, err := url.Parse(r.RequestURI)
        if err != nil {
            panic(err)
        }

        log.Print("Rewriting request on ", r.Host, " to ", *target)

        r.RequestURI = ""
        r.URL        = request_url
        r.URL.Scheme = "https"
        r.Host       = *target
        r.URL.Host   = *target
        r.URL.User   = nil

        currentTimeMilis := getTimeMilis()
        var requestBytes []byte
        requestBytes, err = httputil.DumpRequest(r, true)

        c <- fmt.Sprint(currentTimeMilis-startTimeMilis)
        c <- "\nLOLPONIES\n"
        c <- base64.StdEncoding.EncodeToString(requestBytes)
        c <- "\nLOLPONIES\n"

        resp, err := client.Do(r)
        if err != nil {
            panic(err)
        }

        return nil, resp
    })
    http.Handle("/static/", http.StripPrefix("/static/", http.FileServer(http.Dir("static/"))))
    fmt.Println("Serving on http://localhost:8080")
    log.Fatal(http.ListenAndServe(":8080", proxy))
}

func runPlayback() {
}

func main() {
    flag.Parse()
    if (*playback == "" && *record == false) {
        fmt.Println("One of --playback <filename> or --record is required")
        flag.Usage()
        os.Exit(1)
    } else if (*record && *target == "") {
        fmt.Println("--target <url> is required with --record")
        flag.Usage()
        os.Exit(1)
    } else if (*record && *target != "") {
        runProxy()
    }
}
<commit_msg>buffering<commit_after>package main

import "fmt"
import "net/http"
import "net/http/httputil"
import "net/url"
import "time"
import "log"
import "encoding/base64"
import "flag"
import "os"
import "github.com/elazarl/goproxy"

func Log(handler http.Handler) http.Handler {
    return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
        log.Printf("%s %s %s", r.RemoteAddr, r.Method, r.URL)
        handler.ServeHTTP(w, r)
    })
}

var record = flag.Bool("record", false, "run a http proxy that will record requests")
var playback = flag.String("playback", "", "playback a recoreded session")
var target = flag.String("target", "", "the target host to record")

func getTimeMilis() int64 {
    return time.Now().UnixNano()/1e6
}

func runProxy() {
    proxy := goproxy.NewProxyHttpServer()
    proxy.Verbose = true
    client := http.Client{}
    fo, err := os.Create(".requests")
    if err != nil { panic(err) }
    // close fo on exit and check for its returned error
    defer func() {
        if err := fo.Close(); err != nil {
            panic(err)
        }
    }()

    startTimeMilis := getTimeMilis()

    var c chan string = make(chan string, 100)
    go func() {
        for {
            line := <- c
            fo.WriteString(line)
            fo.Sync()
        }
    }()

    proxy.OnRequest().DoFunc(func(r *http.Request,ctx *goproxy.ProxyCtx)(*http.Request,*http.Response) {
        fmt.Println(r.Host)
        request_url, err := url.Parse(r.RequestURI)
        if err != nil {
            panic(err)
        }

        log.Print("Rewriting request on ", r.Host, " to ", *target)

        r.RequestURI = ""
        r.URL        = request_url
        r.URL.Scheme = "https"
        r.Host       = *target
        r.URL.Host   = *target
        r.URL.User   = nil

        currentTimeMilis := getTimeMilis()
        var requestBytes []byte
        requestBytes, err = httputil.DumpRequest(r, true)

        c <- fmt.Sprint(currentTimeMilis-startTimeMilis)
        c <- "\nLOLPONIES\n"
        c <- base64.StdEncoding.EncodeToString(requestBytes)
        c <- "\nLOLPONIES\n"

        resp, err := client.Do(r)
        if err != nil {
            panic(err)
        }

        return nil, resp
    })
    http.Handle("/static/", http.StripPrefix("/static/", http.FileServer(http.Dir("static/"))))
    fmt.Println("Serving on http://localhost:8080")
    log.Fatal(http.ListenAndServe(":8080", proxy))
}

func runPlayback() {
}

func main() {
    flag.Parse()
    if (*playback == "" && *record == false) {
        fmt.Println("One of --playback <filename> or --record is required")
        flag.Usage()
        os.Exit(1)
    } else if (*record && *target == "") {
        fmt.Println("--target <url> is required with --record")
        flag.Usage()
        os.Exit(1)
    } else if (*record && *target != "") {
        runProxy()
    }
}
<|endoftext|>


<commit_before>// Copyright 2015 Matthew Holt and The Caddy Authors
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package caddyhttp

import (
	"crypto/tls"
	"net/http"

	"go.uber.org/zap/zapcore"
)

// LoggableHTTPRequest makes an HTTP request loggable with zap.Object().
type LoggableHTTPRequest struct{ *http.Request }

// MarshalLogObject satisfies the zapcore.ObjectMarshaler interface.
func (r LoggableHTTPRequest) MarshalLogObject(enc zapcore.ObjectEncoder) error {
	enc.AddString("remote_addr", r.RemoteAddr)
	enc.AddString("proto", r.Proto)
	enc.AddString("method", r.Method)
	enc.AddString("host", r.Host)
	enc.AddString("uri", r.RequestURI)
	enc.AddObject("headers", LoggableHTTPHeader(r.Header))
	if r.TLS != nil {
		enc.AddObject("tls", LoggableTLSConnState(*r.TLS))
	}
	return nil
}

// LoggableHTTPHeader makes an HTTP header loggable with zap.Object().
type LoggableHTTPHeader http.Header

// MarshalLogObject satisfies the zapcore.ObjectMarshaler interface.
func (h LoggableHTTPHeader) MarshalLogObject(enc zapcore.ObjectEncoder) error {
	if h == nil {
		return nil
	}
	for key, val := range h {
		enc.AddArray(key, LoggableStringArray(val))
	}
	return nil
}

// LoggableStringArray makes a slice of strings marshalable for logging.
type LoggableStringArray []string

// MarshalLogArray satisfies the zapcore.ArrayMarshaler interface.
func (sa LoggableStringArray) MarshalLogArray(enc zapcore.ArrayEncoder) error {
	if sa == nil {
		return nil
	}
	for _, s := range sa {
		enc.AppendString(s)
	}
	return nil
}

// LoggableTLSConnState makes a TLS connection state loggable with zap.Object().
type LoggableTLSConnState tls.ConnectionState

// MarshalLogObject satisfies the zapcore.ObjectMarshaler interface.
func (t LoggableTLSConnState) MarshalLogObject(enc zapcore.ObjectEncoder) error {
	enc.AddBool("resumed", t.DidResume)
	enc.AddUint16("version", t.Version)
	enc.AddUint16("ciphersuite", t.CipherSuite)
	enc.AddString("proto", t.NegotiatedProtocol)
	enc.AddBool("proto_mutual", t.NegotiatedProtocolIsMutual)
	enc.AddString("server_name", t.ServerName)
	return nil
}

// Interface guards
var (
	_ zapcore.ObjectMarshaler = (*LoggableHTTPRequest)(nil)
	_ zapcore.ObjectMarshaler = (*LoggableHTTPHeader)(nil)
	_ zapcore.ArrayMarshaler  = (*LoggableStringArray)(nil)
	_ zapcore.ObjectMarshaler = (*LoggableTLSConnState)(nil)
)
<commit_msg>caddyhttp: Add TLS client cert info to logs (#3640)<commit_after>// Copyright 2015 Matthew Holt and The Caddy Authors
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package caddyhttp

import (
	"crypto/tls"
	"net/http"

	"go.uber.org/zap/zapcore"
)

// LoggableHTTPRequest makes an HTTP request loggable with zap.Object().
type LoggableHTTPRequest struct{ *http.Request }

// MarshalLogObject satisfies the zapcore.ObjectMarshaler interface.
func (r LoggableHTTPRequest) MarshalLogObject(enc zapcore.ObjectEncoder) error {
	enc.AddString("remote_addr", r.RemoteAddr)
	enc.AddString("proto", r.Proto)
	enc.AddString("method", r.Method)
	enc.AddString("host", r.Host)
	enc.AddString("uri", r.RequestURI)
	enc.AddObject("headers", LoggableHTTPHeader(r.Header))
	if r.TLS != nil {
		enc.AddObject("tls", LoggableTLSConnState(*r.TLS))
	}
	return nil
}

// LoggableHTTPHeader makes an HTTP header loggable with zap.Object().
type LoggableHTTPHeader http.Header

// MarshalLogObject satisfies the zapcore.ObjectMarshaler interface.
func (h LoggableHTTPHeader) MarshalLogObject(enc zapcore.ObjectEncoder) error {
	if h == nil {
		return nil
	}
	for key, val := range h {
		enc.AddArray(key, LoggableStringArray(val))
	}
	return nil
}

// LoggableStringArray makes a slice of strings marshalable for logging.
type LoggableStringArray []string

// MarshalLogArray satisfies the zapcore.ArrayMarshaler interface.
func (sa LoggableStringArray) MarshalLogArray(enc zapcore.ArrayEncoder) error {
	if sa == nil {
		return nil
	}
	for _, s := range sa {
		enc.AppendString(s)
	}
	return nil
}

// LoggableTLSConnState makes a TLS connection state loggable with zap.Object().
type LoggableTLSConnState tls.ConnectionState

// MarshalLogObject satisfies the zapcore.ObjectMarshaler interface.
func (t LoggableTLSConnState) MarshalLogObject(enc zapcore.ObjectEncoder) error {
	enc.AddBool("resumed", t.DidResume)
	enc.AddUint16("version", t.Version)
	enc.AddUint16("cipher_suite", t.CipherSuite)
	enc.AddString("proto", t.NegotiatedProtocol)
	enc.AddBool("proto_mutual", t.NegotiatedProtocolIsMutual)
	enc.AddString("server_name", t.ServerName)
	if len(t.PeerCertificates) > 0 {
		enc.AddString("client_common_name", t.PeerCertificates[0].Subject.CommonName)
		enc.AddString("client_serial", t.PeerCertificates[0].SerialNumber.String())
	}
	return nil
}

// Interface guards
var (
	_ zapcore.ObjectMarshaler = (*LoggableHTTPRequest)(nil)
	_ zapcore.ObjectMarshaler = (*LoggableHTTPHeader)(nil)
	_ zapcore.ArrayMarshaler  = (*LoggableStringArray)(nil)
	_ zapcore.ObjectMarshaler = (*LoggableTLSConnState)(nil)
)
<|endoftext|>


<commit_before>package pipe

import (
	"fmt"
	_ "testing"
	"time"
)

func Example() {
	Print(
		Find(FILES, "."),
		Grep(`pipe.*\.go$`),
		NumberLines(),
	)
	// Output:
	//     1 pipe.go
	//     2 pipe_test.go
}

func ExampleEmpty() {
	Print()
	// Output:
}

func ExampleSingle() {
	Print(Echo("foo"))
	// Output:
	// foo
}

func ExampleMultiple() {
	Print(Echo("foo"), Echo("bar"))
	// Output:
	// foo
	// bar
}

func ExampleSequence_empty() {
	Print(Sequence())
	// Output:
}

func ExampleSequence_multi() {
	Print(Sequence(Echo("1 of 2"), Echo("2 of 2")))
	// Output:
	// 1 of 2
	// 2 of 2
}

func ExampleSequence_single() {
	Print(Sequence(Echo("1 of 1")))
	// Output:
	// 1 of 1
}

func ExampleForEach() {
	for s := range ForEach(Numbers(1, 10), First(3)) {
		fmt.Print(s)
	}
	// Output:
	// 123
}

func ExamplePrint() {
	Print(Echo("a"), Echo("b"), Echo("c"))
	// Output:
	// a
	// b
	// c
}

func ExampleEcho() {
	Print(Echo("hello", "world"))
	// Output:
	// hello
	// world
}

func ExampleNumbers() {
	Print(Numbers(2, 5))
	// Output:
	// 2
	// 3
	// 4
	// 5
}

func ExampleMap() {
	Print(
		Echo("hello", "there", "how", "are", "you?"),
		Map(func(s string) string {
			return fmt.Sprintf("%d %s", len(s), s)
		}),
	)
	// Output:
	// 5 hello
	// 5 there
	// 3 how
	// 3 are
	// 4 you?
}

func ExampleIf() {
	Print(Numbers(1, 12), If(func(s string) bool { return len(s) > 1 }))
	// Output:
	// 10
	// 11
	// 12
}

func ExampleGrep() {
	Print(Numbers(1, 12), Grep(".."))
	// Output:
	// 10
	// 11
	// 12
}

func ExampleGrepNot() {
	Print(Numbers(1, 12), GrepNot("^.$"))
	// Output:
	// 10
	// 11
	// 12
}

func ExampleUniq() {
	Print(Echo("a", "b", "b", "c", "b"), Uniq())
	// Output:
	// a
	// b
	// c
	// b
}

func ExampleUniqWithCount() {
	Print(
		Echo("a", "b", "b", "c"),
		UniqWithCount(),
	)
	// Output:
	// 1 a
	// 2 b
	// 1 c
}

func ExampleParallelMap() {
	Print(
		Echo("hello", "there", "how", "are", "you?"),
		ParallelMap(4, func(s string) string {
			// Sleep some amount to ensure that ParalellMap
			// implementation handles out of order results.
			time.Sleep(100 * time.Duration(len(s)) * time.Millisecond)
			return fmt.Sprintf("%d %s", len(s), s)
		}),
	)
	// Output:
	// 5 hello
	// 5 there
	// 3 how
	// 3 are
	// 4 you?
}

func ExampleSubstitute() {
	Print(Numbers(1, 5), Substitute("(3)", "$1$1"))
	// Output:
	// 1
	// 2
	// 33
	// 4
	// 5
}

func ExampleNumeric() {
	Print(
		Echo(
			"a 100",
			"b 20",
			"c notanumber", // Will sort last since column 2 is not a number
			"d",            // Will sort earliest since column 2 is missing
		),
		Sort(Numeric(2)),
	)
	// Output:
	// d
	// b 20
	// a 100
	// c notanumber
}

func ExampleTextual() {
	Print(
		Echo(
			"10 bananas",
			"20 apples",
			"30", // Will sort first since column 2 is missing
		),
		Sort(Textual(2)),
	)
	// Output:
	// 30
	// 20 apples
	// 10 bananas
}

func ExampleDescending() {
	Print(
		Echo(
			"100",
			"20",
			"50",
		),
		Sort(Descending(Numeric(1))),
	)
	// Output:
	// 100
	// 50
	// 20
}

func ExampleSort() {
	Print(Echo("banana", "apple", "cheese", "apple"), Sort())
	// Output:
	// apple
	// apple
	// banana
	// cheese
}
func ExampleSort_twoTextColumns() {
	Print(
		Echo(
			"2 green bananas",
			"3 red apples",
			"4 yellow bananas",
			"5 brown pears",
			"6 green apples",
		),
		Sort(Textual(2), Textual(3)),
	)
	// Output:
	// 5 brown pears
	// 6 green apples
	// 2 green bananas
	// 3 red apples
	// 4 yellow bananas
}

func ExampleSort_twoNumericColumns() {
	Print(
		Echo(
			"1970 12",
			"1970 6",
			"1950 6",
			"1980 9",
		),
		Sort(Numeric(1), Numeric(2)))
	// Output:
	// 1950 6
	// 1970 6
	// 1970 12
	// 1980 9
}

func ExampleSort_mixedColumns() {
	Print(
		Echo(
			"1970 march",
			"1970 feb",
			"1950 june",
			"1980 sep",
		),
		Sort(Numeric(1), Textual(2)))
	// Output:
	// 1950 june
	// 1970 feb
	// 1970 march
	// 1980 sep
}

func ExampleReverse() {
	Print(Echo("a", "b"), Reverse())
	// Output:
	// b
	// a
}

func ExampleSample() {
	Print(
		Numbers(100, 200),
		Sample(4),
		Sort(),
	)
	// Output:
	// 122
	// 150
	// 154
	// 158
}

func ExampleFirst() {
	Print(Numbers(1, 10), First(3))
	// Output:
	// 1
	// 2
	// 3
}

func ExampleLast() {
	Print(Numbers(1, 10), Last(2))
	// Output:
	// 9
	// 10
}

func ExampleDropFirst() {
	Print(Numbers(1, 10), DropFirst(8))
	// Output:
	// 9
	// 10
}

func ExampleDropLast() {
	Print(Numbers(1, 10), DropLast(3))
	// Output:
	// 1
	// 2
	// 3
	// 4
	// 5
	// 6
	// 7
}

func ExampleNumberLines() {
	Print(Echo("a", "b"), NumberLines())
	// Output:
	//     1 a
	//     2 b
}

func ExampleCut() {
	Print(Echo("hello", "world."), Cut(2, 4))
	// Output:
	// llo
	// rld
}

func ExmapleSelect() {
	Print(Echo("hello world"), Select(2, 0, 1))
	// Output:
	// world hello world hello
}

func ExampleFind() {
	Print(Find(FILES, "."), Grep("pipe"))
	// Output:
	// pipe.go
	// pipe_test.go
}

func ExampleFind_dirs() {
	Print(Find(DIRS, "."), GrepNot("git"))
	// Output:
	// .
}

func ExampleCat() {
	Print(Cat("pipe_test.go"), Grep("^func ExampleCat"))
	// Output:
	// func ExampleCat() {
}

func ExampleSystem() {
	Print(
		System("find", ".", "-type", "f", "-print"),
		Grep(`^\./pipe.*\.go$`),
		Sort(),
	)

	// Output:
	// ./pipe.go
	// ./pipe_test.go
}
<commit_msg>reduce sleep in ParallelMap test<commit_after>package pipe

import (
	"fmt"
	_ "testing"
	"time"
)

func Example() {
	Print(
		Find(FILES, "."),
		Grep(`pipe.*\.go$`),
		NumberLines(),
	)
	// Output:
	//     1 pipe.go
	//     2 pipe_test.go
}

func ExampleEmpty() {
	Print()
	// Output:
}

func ExampleSingle() {
	Print(Echo("foo"))
	// Output:
	// foo
}

func ExampleMultiple() {
	Print(Echo("foo"), Echo("bar"))
	// Output:
	// foo
	// bar
}

func ExampleSequence_empty() {
	Print(Sequence())
	// Output:
}

func ExampleSequence_multi() {
	Print(Sequence(Echo("1 of 2"), Echo("2 of 2")))
	// Output:
	// 1 of 2
	// 2 of 2
}

func ExampleSequence_single() {
	Print(Sequence(Echo("1 of 1")))
	// Output:
	// 1 of 1
}

func ExampleForEach() {
	for s := range ForEach(Numbers(1, 10), First(3)) {
		fmt.Print(s)
	}
	// Output:
	// 123
}

func ExamplePrint() {
	Print(Echo("a"), Echo("b"), Echo("c"))
	// Output:
	// a
	// b
	// c
}

func ExampleEcho() {
	Print(Echo("hello", "world"))
	// Output:
	// hello
	// world
}

func ExampleNumbers() {
	Print(Numbers(2, 5))
	// Output:
	// 2
	// 3
	// 4
	// 5
}

func ExampleMap() {
	Print(
		Echo("hello", "there", "how", "are", "you?"),
		Map(func(s string) string {
			return fmt.Sprintf("%d %s", len(s), s)
		}),
	)
	// Output:
	// 5 hello
	// 5 there
	// 3 how
	// 3 are
	// 4 you?
}

func ExampleIf() {
	Print(Numbers(1, 12), If(func(s string) bool { return len(s) > 1 }))
	// Output:
	// 10
	// 11
	// 12
}

func ExampleGrep() {
	Print(Numbers(1, 12), Grep(".."))
	// Output:
	// 10
	// 11
	// 12
}

func ExampleGrepNot() {
	Print(Numbers(1, 12), GrepNot("^.$"))
	// Output:
	// 10
	// 11
	// 12
}

func ExampleUniq() {
	Print(Echo("a", "b", "b", "c", "b"), Uniq())
	// Output:
	// a
	// b
	// c
	// b
}

func ExampleUniqWithCount() {
	Print(
		Echo("a", "b", "b", "c"),
		UniqWithCount(),
	)
	// Output:
	// 1 a
	// 2 b
	// 1 c
}

func ExampleParallelMap() {
	Print(
		Echo("hello", "there", "how", "are", "you?"),
		ParallelMap(4, func(s string) string {
			// Sleep some amount to ensure that ParalellMap
			// implementation handles out of order results.
			time.Sleep(10 * time.Duration(len(s)) * time.Millisecond)
			return fmt.Sprintf("%d %s", len(s), s)
		}),
	)
	// Output:
	// 5 hello
	// 5 there
	// 3 how
	// 3 are
	// 4 you?
}

func ExampleSubstitute() {
	Print(Numbers(1, 5), Substitute("(3)", "$1$1"))
	// Output:
	// 1
	// 2
	// 33
	// 4
	// 5
}

func ExampleNumeric() {
	Print(
		Echo(
			"a 100",
			"b 20",
			"c notanumber", // Will sort last since column 2 is not a number
			"d",            // Will sort earliest since column 2 is missing
		),
		Sort(Numeric(2)),
	)
	// Output:
	// d
	// b 20
	// a 100
	// c notanumber
}

func ExampleTextual() {
	Print(
		Echo(
			"10 bananas",
			"20 apples",
			"30", // Will sort first since column 2 is missing
		),
		Sort(Textual(2)),
	)
	// Output:
	// 30
	// 20 apples
	// 10 bananas
}

func ExampleDescending() {
	Print(
		Echo(
			"100",
			"20",
			"50",
		),
		Sort(Descending(Numeric(1))),
	)
	// Output:
	// 100
	// 50
	// 20
}

func ExampleSort() {
	Print(Echo("banana", "apple", "cheese", "apple"), Sort())
	// Output:
	// apple
	// apple
	// banana
	// cheese
}
func ExampleSort_twoTextColumns() {
	Print(
		Echo(
			"2 green bananas",
			"3 red apples",
			"4 yellow bananas",
			"5 brown pears",
			"6 green apples",
		),
		Sort(Textual(2), Textual(3)),
	)
	// Output:
	// 5 brown pears
	// 6 green apples
	// 2 green bananas
	// 3 red apples
	// 4 yellow bananas
}

func ExampleSort_twoNumericColumns() {
	Print(
		Echo(
			"1970 12",
			"1970 6",
			"1950 6",
			"1980 9",
		),
		Sort(Numeric(1), Numeric(2)))
	// Output:
	// 1950 6
	// 1970 6
	// 1970 12
	// 1980 9
}

func ExampleSort_mixedColumns() {
	Print(
		Echo(
			"1970 march",
			"1970 feb",
			"1950 june",
			"1980 sep",
		),
		Sort(Numeric(1), Textual(2)))
	// Output:
	// 1950 june
	// 1970 feb
	// 1970 march
	// 1980 sep
}

func ExampleReverse() {
	Print(Echo("a", "b"), Reverse())
	// Output:
	// b
	// a
}

func ExampleSample() {
	Print(
		Numbers(100, 200),
		Sample(4),
		Sort(),
	)
	// Output:
	// 122
	// 150
	// 154
	// 158
}

func ExampleFirst() {
	Print(Numbers(1, 10), First(3))
	// Output:
	// 1
	// 2
	// 3
}

func ExampleLast() {
	Print(Numbers(1, 10), Last(2))
	// Output:
	// 9
	// 10
}

func ExampleDropFirst() {
	Print(Numbers(1, 10), DropFirst(8))
	// Output:
	// 9
	// 10
}

func ExampleDropLast() {
	Print(Numbers(1, 10), DropLast(3))
	// Output:
	// 1
	// 2
	// 3
	// 4
	// 5
	// 6
	// 7
}

func ExampleNumberLines() {
	Print(Echo("a", "b"), NumberLines())
	// Output:
	//     1 a
	//     2 b
}

func ExampleCut() {
	Print(Echo("hello", "world."), Cut(2, 4))
	// Output:
	// llo
	// rld
}

func ExmapleSelect() {
	Print(Echo("hello world"), Select(2, 0, 1))
	// Output:
	// world hello world hello
}

func ExampleFind() {
	Print(Find(FILES, "."), Grep("pipe"))
	// Output:
	// pipe.go
	// pipe_test.go
}

func ExampleFind_dirs() {
	Print(Find(DIRS, "."), GrepNot("git"))
	// Output:
	// .
}

func ExampleCat() {
	Print(Cat("pipe_test.go"), Grep("^func ExampleCat"))
	// Output:
	// func ExampleCat() {
}

func ExampleSystem() {
	Print(
		System("find", ".", "-type", "f", "-print"),
		Grep(`^\./pipe.*\.go$`),
		Sort(),
	)

	// Output:
	// ./pipe.go
	// ./pipe_test.go
}
<|endoftext|>


<commit_before>// Package inspect parses a file and prints its output.
package main

import (
	"bramp.net/dsector/ufwb"
	"flag"
	"fmt"
	"os"
)

var (
	// TODO Change this to be arg1 and arg2, instead of flags
	grammar = flag.String("grammar", "", "grammar filename")
	target = flag.String("target", "", "target filename")
)

func openGrammar() *ufwb.Ufwb {
	file, err := os.Open(*grammar)
	if err != nil {
		fmt.Fprintf(os.Stderr, "failed to open grammar %q: %s\n", grammar, err)
		os.Exit(1)
	}
	defer file.Close()

	g, errs := ufwb.ParseXmlGrammar(file)
	if len(errs) > 0 {
		fmt.Fprintf(os.Stderr, "failed to parse grammar %q:\n", grammar)
		for i, err := range errs {
			fmt.Fprintf(os.Stderr, "[%d] %s\n", i, err.Error())
		}
		os.Exit(1)
	}

	return g
}

func decode(u *ufwb.Ufwb, f ufwb.File) {
	decoder := ufwb.NewDecoder(u, f)
	value, err := decoder.Decode()
	if err != nil {
		fmt.Fprintf(os.Stderr, "Failed to parse target: %s\n", err.Error())
		os.Exit(1)
	}

	str, err := value.Format(f)
	if err != nil {
		fmt.Fprintf(os.Stderr, "Failed to format parsed output: %s\n", err.Error())
		os.Exit(1)
	}

	fmt.Print(str)
}

func parseFlags() {
	flag.Parse()

	if grammar == nil || *grammar == "" {
		fmt.Fprintf(os.Stderr, "Please specify a grammar file\n")
		os.Exit(1)
	}

	if target == nil || *target == "" {
		fmt.Fprintf(os.Stderr, "Please specify a target file\n")
		os.Exit(1)
	}

}

func main() {

	parseFlags()

	g := openGrammar()

	file, err := ufwb.OpenOSFile(*target)
	if err != nil {
		fmt.Fprintf(os.Stderr, "Failed to open target %q: %s\n", target, err.Error())
		return
	}
	defer file.Close()

	decode(g, file)
}
<commit_msg>Fixed up the inspect tool to accept two arguments, and to use the API after the refactor.<commit_after>// Package inspect parses a file and prints its output.
package main

import (
	"bramp.net/dsector/input"
	"bramp.net/dsector/ufwb"
	"flag"
	"fmt"
	"os"
)

func openGrammar(grammar string) *ufwb.Ufwb {
	file, err := os.Open(grammar)
	if err != nil {
		fmt.Fprintf(os.Stderr, "failed to open grammar %q: %s\n", grammar, err)
		os.Exit(1)
	}
	defer file.Close()

	g, errs := ufwb.ParseXmlGrammar(file)
	if len(errs) > 0 {
		fmt.Fprintf(os.Stderr, "failed to parse grammar %q:\n", grammar)
		for i, err := range errs {
			fmt.Fprintf(os.Stderr, "[%d] %s\n", i, err.Error())
		}
		os.Exit(1)
	}

	return g
}

func decode(u *ufwb.Ufwb, f input.Input) {
	decoder := ufwb.NewDecoder(u, f)
	value, err := decoder.Decode()
	if err != nil {
		fmt.Fprintf(os.Stderr, "Failed to parse target: %s\n", err.Error())
		os.Exit(1)
	}

	str, err := value.Format(f)
	if err != nil {
		fmt.Fprintf(os.Stderr, "Failed to format parsed output: %s\n", err.Error())
		os.Exit(1)
	}

	fmt.Print(str)
}

func main() {

	flag.Parse()
	flag.Usage = func() {
		fmt.Println("inspect [grammar] [target]")
	}

	args := flag.Args()
	if len(args) < 2 {
		flag.Usage()
		os.Exit(1)
	}

	grammar := args[0]
	target := args[1]

	g := openGrammar(grammar)

	file, err := input.OpenOSFile(target)
	if err != nil {
		fmt.Fprintf(os.Stderr, "Failed to open target %q: %s\n", target, err.Error())
		return
	}
	defer file.Close()

	decode(g, file)
}
<|endoftext|>


<commit_before>// Copyright 2015 CNI authors
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package ns

import (
	"fmt"
	"os"
	"runtime"
	"syscall"
)

var setNsMap = map[string]uintptr{
	"386":   346,
	"amd64": 308,
	"arm":   374,
}

// SetNS sets the network namespace on a target file.
func SetNS(f *os.File, flags uintptr) error {
	if runtime.GOOS != "linux" {
		return fmt.Errorf("unsupported OS: %s", runtime.GOOS)
	}

	trap, ok := setNsMap[runtime.GOARCH]
	if !ok {
		return fmt.Errorf("unsupported arch: %s", runtime.GOARCH)
	}

	_, _, err := syscall.RawSyscall(trap, f.Fd(), flags, 0)
	if err != 0 {
		return err
	}

	return nil
}

// WithNetNSPath executes the passed closure under the given network
// namespace, restoring the original namespace afterwards.
// Changing namespaces must be done on a goroutine that has been
// locked to an OS thread. If lockThread arg is true, this function
// locks the goroutine prior to change namespace and unlocks before
// returning
func WithNetNSPath(nspath string, lockThread bool, f func(*os.File) error) error {
	ns, err := os.Open(nspath)
	if err != nil {
		return fmt.Errorf("Failed to open %v: %v", nspath, err)
	}
	defer ns.Close()
	return WithNetNS(ns, lockThread, f)
}

// WithNetNS executes the passed closure under the given network
// namespace, restoring the original namespace afterwards.
// Changing namespaces must be done on a goroutine that has been
// locked to an OS thread. If lockThread arg is true, this function
// locks the goroutine prior to change namespace and unlocks before
// returning.  If the closure returns an error, WithNetNS attempts to
// restore the original namespace before returning.
func WithNetNS(ns *os.File, lockThread bool, f func(*os.File) error) error {
	if lockThread {
		runtime.LockOSThread()
		defer runtime.UnlockOSThread()
	}
	// save a handle to current (host) network namespace
	thisNS, err := os.Open("/proc/self/ns/net")
	if err != nil {
		return fmt.Errorf("Failed to open /proc/self/ns/net: %v", err)
	}
	defer thisNS.Close()

	if err = SetNS(ns, syscall.CLONE_NEWNET); err != nil {
		return fmt.Errorf("Error switching to ns %v: %v", ns.Name(), err)
	}
	defer SetNS(thisNS, syscall.CLONE_NEWNET) // switch back

	if err = f(thisNS); err != nil {
		return err
	}

	return nil
}
<commit_msg>pkg/ns: use correct syscall number on arm<commit_after>// Copyright 2015 CNI authors
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package ns

import (
	"fmt"
	"os"
	"runtime"
	"syscall"
)

var setNsMap = map[string]uintptr{
	"386":   346,
	"amd64": 308,
	"arm":   375,
}

// SetNS sets the network namespace on a target file.
func SetNS(f *os.File, flags uintptr) error {
	if runtime.GOOS != "linux" {
		return fmt.Errorf("unsupported OS: %s", runtime.GOOS)
	}

	trap, ok := setNsMap[runtime.GOARCH]
	if !ok {
		return fmt.Errorf("unsupported arch: %s", runtime.GOARCH)
	}

	_, _, err := syscall.RawSyscall(trap, f.Fd(), flags, 0)
	if err != 0 {
		return err
	}

	return nil
}

// WithNetNSPath executes the passed closure under the given network
// namespace, restoring the original namespace afterwards.
// Changing namespaces must be done on a goroutine that has been
// locked to an OS thread. If lockThread arg is true, this function
// locks the goroutine prior to change namespace and unlocks before
// returning
func WithNetNSPath(nspath string, lockThread bool, f func(*os.File) error) error {
	ns, err := os.Open(nspath)
	if err != nil {
		return fmt.Errorf("Failed to open %v: %v", nspath, err)
	}
	defer ns.Close()
	return WithNetNS(ns, lockThread, f)
}

// WithNetNS executes the passed closure under the given network
// namespace, restoring the original namespace afterwards.
// Changing namespaces must be done on a goroutine that has been
// locked to an OS thread. If lockThread arg is true, this function
// locks the goroutine prior to change namespace and unlocks before
// returning.  If the closure returns an error, WithNetNS attempts to
// restore the original namespace before returning.
func WithNetNS(ns *os.File, lockThread bool, f func(*os.File) error) error {
	if lockThread {
		runtime.LockOSThread()
		defer runtime.UnlockOSThread()
	}
	// save a handle to current (host) network namespace
	thisNS, err := os.Open("/proc/self/ns/net")
	if err != nil {
		return fmt.Errorf("Failed to open /proc/self/ns/net: %v", err)
	}
	defer thisNS.Close()

	if err = SetNS(ns, syscall.CLONE_NEWNET); err != nil {
		return fmt.Errorf("Error switching to ns %v: %v", ns.Name(), err)
	}
	defer SetNS(thisNS, syscall.CLONE_NEWNET) // switch back

	if err = f(thisNS); err != nil {
		return err
	}

	return nil
}
<|endoftext|>


<commit_before>// Reddit audiences crawler
// Rémy Mathieu © 2016

package app

import (
	"fmt"
	"log"
	"math/rand"
	"net/http"
	"strconv"
	"strings"
	"time"

	"github.com/PuerkitoBio/goquery"
)

const (
	REDDIT_SUBREDDIT_URL           = "https://reddit.com/r/"
	MIN_SECONDS_BETWEEN_EACH_CRAWL = 1
)

var subredditsToCrawl chan string

func StartCrawlingJob(a *App) {
	subredditsToCrawl = make(chan string)

	// starts the worker
	go Worker(a)

	// starts the main loop
	// regularly feeding the worker.
	if a.Config.Crawl {
		log.Println("info: starts crawling job.")
		ticker := time.NewTicker(time.Second * 30)
		for range ticker.C {
			log.Println("info: crawling job is running.")
			Feeder(a)
		}
		ticker.Stop()
	}
}

// Worker is the routine dealing with the HTTP
// call + reading hte DOM.
func Worker(a *App) {
	for subreddit := range subredditsToCrawl {
		if audience, subscribers, articles, err := readDOMData(subreddit); err == nil {
			// store the audience+subscribers and update the last crawl time
			// ----------------------
			if err := a.DB().InsertAudienceValue(subreddit, audience, subscribers); err != nil {
				log.Println("err:", err.Error())
			} else {
				log.Printf("info: subreddit %s has %d active users (%d subscribers)\n", subreddit, audience, subscribers)
			}

			// store each articles if not already present at this rank
			// ----------------------
			if err := storeArticles(a, articles); err != nil {
				log.Printf("err: %s", err.Error())
			}
		} else if err != nil {
			log.Println("err:", err.Error())
		}

		r := time.Duration(((rand.Int() % 2) + MIN_SECONDS_BETWEEN_EACH_CRAWL)) * time.Second

		time.Sleep(r) // wait a bit before the next crawl
	}
}

// Feeder retrieves the audience of subreddits for which
// the last crawl time is more than some minutes.
func Feeder(a *App) {
	// crawl each subreddit each 5 minutes
	five := time.Minute * 5
	t := time.Now().Add(-five)
	subreddits, err := a.DB().FindSubredditsToCrawl(t)

	if err != nil {
		log.Printf("err: can't retrieve subreddits to crawl: %s\n", err.Error())
	}

	for _, subreddit := range subreddits {
		log.Println("info: crawling", subreddit)
		subredditsToCrawl <- subreddit
	}
}

// readDOMData gets the subreddit page on reddit
// and gets the current audience, the subscribers and
// the article infos from the DOM.
// NOTE(remy): we stop as soon as we have a DOM error because
// it has great chances that the full DOM is corrupted/not retrieved.
func readDOMData(subreddit string) (int64, int64, []Article, error) {
	var audience int64
	var subscribers int64
	var err error

	doc, err := getSubredditPage(REDDIT_SUBREDDIT_URL + subreddit)
	if err != nil {
		return 0, 0, nil, fmt.Errorf("while crawling %s: %s", subreddit, err.Error())
	}

	// audience
	// ----------------------

	s := doc.Find("p.users-online span.number").First()

	value := s.Text()
	if len(value) == 0 {
		return 0, 0, nil, fmt.Errorf("can't retrieve subreddit %s audience: no text value in the dom node.", subreddit)
	}

	if audience, err = cleanInt(value); err != nil {
		return 0, 0, nil, err
	}

	// subscribers
	// ----------------------

	s = doc.Find("span.subscribers span.number").First()

	value = s.Text()
	if len(value) == 0 {
		return 0, 0, nil, fmt.Errorf("can't retrieve subreddit %s subscribers: no text value in the dom node.", subreddit)
	}

	if subscribers, err = cleanInt(value); err != nil {
		return 0, 0, nil, err
	}

	// articles
	// ----------------------

	now := time.Now()
	articles := make([]Article, 0)
	s = doc.Find(".link").Each(func(i int, selec *goquery.Selection) {
		l := selec.Find("p.title a.title")
		title := l.First()
		link, _ := l.Attr("href")
		strPos := selec.ChildrenFiltered(".rank").First()
		articleId, _ := selec.Attr("data-fullname")
		author, _ := selec.Attr("data-author")

		// remove the t[1-3]_ from the article id
		for i := 0; i < 4; i++ {
			articleId = strings.TrimPrefix(articleId, fmt.Sprintf("t%d_", i))
		}

		rank, err := strconv.Atoi(strPos.Text())
		if err != nil {
			rank = 0 // it's probably a promoted or stickied article
		}

		promoted := false
		if selec.HasClass("promoted") {
			promoted = true
		}

		sticky := false
		if selec.HasClass("stickied") {
			sticky = true
		}

		articles = append(articles, Article{
			Subreddit:    subreddit,
			ArticleId:    articleId,
			ArticleTitle: title.Text(),
			ArticleLink:  link,
			Author:       author,
			Rank:         rank,
			CrawlTime:    now,
			Promoted:     promoted,
			Sticky:       sticky,
		})
	})

	return audience, subscribers, articles, err
}

// storeArticles checks for each article if the
// info isn't already present in database, if not,
// it stores it. If changed, it also stores it.
func storeArticles(a *App, articles []Article) error {
	if len(articles) == 0 {
		return nil
	}

	for _, article := range articles {
		id, rank, err := a.DB().FindArticleLastState(article.Subreddit, article.ArticleId)
		if err != nil {
			return fmt.Errorf("while retrieving article last state: %s", err.Error())
		}

		// already stored at this rank
		if article.ArticleId == id && rank == article.Rank {
			continue
		}

		// not already store, do it now
		if _, err := a.DB().InsertArticle(article); err != nil {
			return err
		}
	}

	return nil
}

func getSubredditPage(url string) (*goquery.Document, error) {
	r, err := NewRequest(url)
	if err != nil {
		return nil, err
	}

	client := http.Client{}
	resp, err := client.Do(r)

	if err != nil {
		resp.Body.Close()
		return nil, err
	}

	if resp.StatusCode != 200 {
		resp.Body.Close()
		return nil, fmt.Errorf(resp.Status)
	}

	return goquery.NewDocumentFromResponse(resp)
}

func cleanInt(str string) (int64, error) {
	// sometimes it starts with ~
	if strings.HasPrefix(str, "~") {
		str = str[1:]
	}
	// , for thousands etc.
	str = strings.Replace(str, ",", "", -1)
	// finally trim
	str = strings.Trim(str, " ")

	return strconv.ParseInt(str, 10, 64)
}
<commit_msg>app: avoid a 301.<commit_after>// Reddit audiences crawler
// Rémy Mathieu © 2016

package app

import (
	"fmt"
	"log"
	"math/rand"
	"net/http"
	"strconv"
	"strings"
	"time"

	"github.com/PuerkitoBio/goquery"
)

const (
	REDDIT_SUBREDDIT_URL           = "https://www.reddit.com/r/"
	MIN_SECONDS_BETWEEN_EACH_CRAWL = 1
)

var subredditsToCrawl chan string

func StartCrawlingJob(a *App) {
	subredditsToCrawl = make(chan string)

	// starts the worker
	go Worker(a)

	// starts the main loop
	// regularly feeding the worker.
	if a.Config.Crawl {
		log.Println("info: starts crawling job.")
		ticker := time.NewTicker(time.Second * 30)
		for range ticker.C {
			log.Println("info: crawling job is running.")
			Feeder(a)
		}
		ticker.Stop()
	}
}

// Worker is the routine dealing with the HTTP
// call + reading hte DOM.
func Worker(a *App) {
	for subreddit := range subredditsToCrawl {
		if audience, subscribers, articles, err := readDOMData(subreddit); err == nil {
			// store the audience+subscribers and update the last crawl time
			// ----------------------
			if err := a.DB().InsertAudienceValue(subreddit, audience, subscribers); err != nil {
				log.Println("err:", err.Error())
			} else {
				log.Printf("info: subreddit %s has %d active users (%d subscribers)\n", subreddit, audience, subscribers)
			}

			// store each articles if not already present at this rank
			// ----------------------
			if err := storeArticles(a, articles); err != nil {
				log.Printf("err: %s", err.Error())
			}
		} else if err != nil {
			log.Println("err:", err.Error())
		}

		r := time.Duration(((rand.Int() % 2) + MIN_SECONDS_BETWEEN_EACH_CRAWL)) * time.Second

		time.Sleep(r) // wait a bit before the next crawl
	}
}

// Feeder retrieves the audience of subreddits for which
// the last crawl time is more than some minutes.
func Feeder(a *App) {
	// crawl each subreddit each 5 minutes
	five := time.Minute * 5
	t := time.Now().Add(-five)
	subreddits, err := a.DB().FindSubredditsToCrawl(t)

	if err != nil {
		log.Printf("err: can't retrieve subreddits to crawl: %s\n", err.Error())
	}

	for _, subreddit := range subreddits {
		log.Println("info: crawling", subreddit)
		subredditsToCrawl <- subreddit
	}
}

// readDOMData gets the subreddit page on reddit
// and gets the current audience, the subscribers and
// the article infos from the DOM.
// NOTE(remy): we stop as soon as we have a DOM error because
// it has great chances that the full DOM is corrupted/not retrieved.
func readDOMData(subreddit string) (int64, int64, []Article, error) {
	var audience int64
	var subscribers int64
	var err error

	doc, err := getSubredditPage(REDDIT_SUBREDDIT_URL + subreddit)
	if err != nil {
		return 0, 0, nil, fmt.Errorf("while crawling %s: %s", subreddit, err.Error())
	}

	// audience
	// ----------------------

	s := doc.Find("p.users-online span.number").First()

	value := s.Text()
	if len(value) == 0 {
		return 0, 0, nil, fmt.Errorf("can't retrieve subreddit %s audience: no text value in the dom node.", subreddit)
	}

	if audience, err = cleanInt(value); err != nil {
		return 0, 0, nil, err
	}

	// subscribers
	// ----------------------

	s = doc.Find("span.subscribers span.number").First()

	value = s.Text()
	if len(value) == 0 {
		return 0, 0, nil, fmt.Errorf("can't retrieve subreddit %s subscribers: no text value in the dom node.", subreddit)
	}

	if subscribers, err = cleanInt(value); err != nil {
		return 0, 0, nil, err
	}

	// articles
	// ----------------------

	now := time.Now()
	articles := make([]Article, 0)
	s = doc.Find(".link").Each(func(i int, selec *goquery.Selection) {
		l := selec.Find("p.title a.title")
		title := l.First()
		link, _ := l.Attr("href")
		strPos := selec.ChildrenFiltered(".rank").First()
		articleId, _ := selec.Attr("data-fullname")
		author, _ := selec.Attr("data-author")

		// remove the t[1-3]_ from the article id
		for i := 0; i < 4; i++ {
			articleId = strings.TrimPrefix(articleId, fmt.Sprintf("t%d_", i))
		}

		rank, err := strconv.Atoi(strPos.Text())
		if err != nil {
			rank = 0 // it's probably a promoted or stickied article
		}

		promoted := false
		if selec.HasClass("promoted") {
			promoted = true
		}

		sticky := false
		if selec.HasClass("stickied") {
			sticky = true
		}

		articles = append(articles, Article{
			Subreddit:    subreddit,
			ArticleId:    articleId,
			ArticleTitle: title.Text(),
			ArticleLink:  link,
			Author:       author,
			Rank:         rank,
			CrawlTime:    now,
			Promoted:     promoted,
			Sticky:       sticky,
		})
	})

	return audience, subscribers, articles, err
}

// storeArticles checks for each article if the
// info isn't already present in database, if not,
// it stores it. If changed, it also stores it.
func storeArticles(a *App, articles []Article) error {
	if len(articles) == 0 {
		return nil
	}

	for _, article := range articles {
		id, rank, err := a.DB().FindArticleLastState(article.Subreddit, article.ArticleId)
		if err != nil {
			return fmt.Errorf("while retrieving article last state: %s", err.Error())
		}

		// already stored at this rank
		if article.ArticleId == id && rank == article.Rank {
			continue
		}

		// not already store, do it now
		if _, err := a.DB().InsertArticle(article); err != nil {
			return err
		}
	}

	return nil
}

func getSubredditPage(url string) (*goquery.Document, error) {
	r, err := NewRequest(url)
	if err != nil {
		return nil, err
	}

	client := http.Client{}
	resp, err := client.Do(r)

	if err != nil {
		resp.Body.Close()
		return nil, err
	}

	if resp.StatusCode != 200 {
		resp.Body.Close()
		return nil, fmt.Errorf(resp.Status)
	}

	return goquery.NewDocumentFromResponse(resp)
}

func cleanInt(str string) (int64, error) {
	// sometimes it starts with ~
	if strings.HasPrefix(str, "~") {
		str = str[1:]
	}
	// , for thousands etc.
	str = strings.Replace(str, ",", "", -1)
	// finally trim
	str = strings.Trim(str, " ")

	return strconv.ParseInt(str, 10, 64)
}
<|endoftext|>


<commit_before>package statsd

import (
	"fmt"
	"log"
	"math/rand"
	"net"
	"strings"
	"sync"
	"time"
)

const metricTypeCount = "c"
const metricTypeGauge = "g"
const metricTypeTiming = "t"
const metricTypeSet = "s"

// The Client type
type Client struct {
	host          string
	port          int
	conn          net.Conn          // UDP connection to StatsD server
	rand          *rand.Rand        // rand generator to skip messages by sample rate
	keyBuffer     map[string]string // array of messages to send
	keyBufferLock sync.RWMutex      // mutex to lock buffer of keys
	autoflush     bool              // send metrics on every call
}

// NewClient creates new StatsD client
func NewClient(host string, port int) *Client {
	client := Client{
		host:      host,
		port:      port,
		rand:      rand.New(rand.NewSource(time.Now().Unix())),
		keyBuffer: make(map[string]string),
		autoflush: false,
	}
	return &client
}

// Open UDP connection to statsd server
func (client *Client) Open() {
	connectionString := fmt.Sprintf("%s:%d", client.host, client.port)
	conn, err := net.Dial("udp", connectionString)
	if err != nil {
		log.Println(err)
	}
	client.conn = conn
}

// Close UDP connection to statsd server
func (client *Client) Close() {
	client.conn.Close()
}

// SetAutoflush enables/disables buffered mode
// In buffered mode requires manual call of Flush()
// In autoflush mode message sends to server on every call
func (client *Client) SetAutoflush(autoflush bool) {
	client.autoflush = autoflush
}

// Timing track in milliseconds with sampling
func (client *Client) Timing(key string, time int64, sampleRate float32) {
	metricValue := fmt.Sprintf("%d|%s", time, metricTypeTiming)
	if sampleRate < 1 {
		if client.isSendAcceptedBySampleRate(sampleRate) {
			metricValue = fmt.Sprintf("%s|@%f", metricValue, sampleRate)
		} else {
			return
		}
	}

	client.addToBuffer(key, metricValue)
}

// Count tack
func (client *Client) Count(key string, delta int, sampleRate float32) {
	metricValue := fmt.Sprintf("%d|%s", delta, metricTypeCount)
	if sampleRate < 1 {
		if client.isSendAcceptedBySampleRate(sampleRate) {
			metricValue = fmt.Sprintf("%s|@%f", metricValue, sampleRate)
		} else {
			return
		}
	}

	client.addToBuffer(key, metricValue)
}

// Gauge track
func (client *Client) Gauge(key string, value int) {
	metricValue := fmt.Sprintf("%d|%s", value, metricTypeGauge)
	client.addToBuffer(key, metricValue)
}

// Set tracking
func (client *Client) Set(key string, value int) {
	metricValue := fmt.Sprintf("%d|%s", value, metricTypeSet)
	client.addToBuffer(key, metricValue)
}

// add to buffer and flush if auto flush enabled
func (client *Client) addToBuffer(key string, metricValue string) {
	// flush
	if client.autoflush {
		// send metric now
		client.send(fmt.Sprintf("%s:%s", key, metricValue))
	} else {
		// add metric to buffer for next manual flush
		client.keyBufferLock.Lock()
		client.keyBuffer[key] = metricValue
		client.keyBufferLock.Unlock()

	}
}

// Check if acceptable by sample rate
func (client *Client) isSendAcceptedBySampleRate(sampleRate float32) bool {
	if sampleRate >= 1 {
		return true
	}
	randomNumber := client.rand.Float32()
	return randomNumber <= sampleRate
}

// Flush buffer to statsd daemon by UDP
func (client *Client) Flush() {
	// check if buffer has metrics
	if len(client.keyBuffer) == 0 {
		return
	}

	// prepare metric packet
	var metricPacketArray []string

	// lock
	client.keyBufferLock.Lock()

	// build packet
	for key, metricValue := range client.keyBuffer {
		metricPacketArray = append(metricPacketArray, fmt.Sprintf("%s:%s", key, metricValue))
	}
	metricPacket := strings.Join(metricPacketArray, "\n")

	// clear key buffer
	client.keyBuffer = make(map[string]string)

	// lock
	client.keyBufferLock.Unlock()

	// send packet
	client.send(metricPacket)
}

// Send StatsD packet
func (client *Client) send(metricPacket string) {
	// send metric packet
	_, err := fmt.Fprintf(client.conn, metricPacket)
	if err != nil {
		log.Println(err)
	}
}
<commit_msg>async send to statsd<commit_after>package statsd

import (
	"fmt"
	"log"
	"math/rand"
	"net"
	"strings"
	"sync"
	"time"
)

const metricTypeCount = "c"
const metricTypeGauge = "g"
const metricTypeTiming = "t"
const metricTypeSet = "s"

// The Client type
type Client struct {
	host          string
	port          int
	conn          net.Conn          // UDP connection to StatsD server
	rand          *rand.Rand        // rand generator to skip messages by sample rate
	keyBuffer     map[string]string // array of messages to send
	keyBufferLock sync.RWMutex      // mutex to lock buffer of keys
	autoflush     bool              // send metrics on every call
}

// NewClient creates new StatsD client
func NewClient(host string, port int) *Client {
	client := Client{
		host:      host,
		port:      port,
		rand:      rand.New(rand.NewSource(time.Now().Unix())),
		keyBuffer: make(map[string]string),
		autoflush: false,
	}
	return &client
}

// Open UDP connection to statsd server
func (client *Client) Open() {
	connectionString := fmt.Sprintf("%s:%d", client.host, client.port)
	conn, err := net.Dial("udp", connectionString)
	if err != nil {
		log.Println(err)
	}
	client.conn = conn
}

// Close UDP connection to statsd server
func (client *Client) Close() {
	client.conn.Close()
}

// SetAutoflush enables/disables buffered mode
// In buffered mode requires manual call of Flush()
// In autoflush mode message sends to server on every call
func (client *Client) SetAutoflush(autoflush bool) {
	client.autoflush = autoflush
}

// Timing track in milliseconds with sampling
func (client *Client) Timing(key string, time int64, sampleRate float32) {
	metricValue := fmt.Sprintf("%d|%s", time, metricTypeTiming)
	if sampleRate < 1 {
		if client.isSendAcceptedBySampleRate(sampleRate) {
			metricValue = fmt.Sprintf("%s|@%f", metricValue, sampleRate)
		} else {
			return
		}
	}

	client.addToBuffer(key, metricValue)
}

// Count tack
func (client *Client) Count(key string, delta int, sampleRate float32) {
	metricValue := fmt.Sprintf("%d|%s", delta, metricTypeCount)
	if sampleRate < 1 {
		if client.isSendAcceptedBySampleRate(sampleRate) {
			metricValue = fmt.Sprintf("%s|@%f", metricValue, sampleRate)
		} else {
			return
		}
	}

	client.addToBuffer(key, metricValue)
}

// Gauge track
func (client *Client) Gauge(key string, value int) {
	metricValue := fmt.Sprintf("%d|%s", value, metricTypeGauge)
	client.addToBuffer(key, metricValue)
}

// Set tracking
func (client *Client) Set(key string, value int) {
	metricValue := fmt.Sprintf("%d|%s", value, metricTypeSet)
	client.addToBuffer(key, metricValue)
}

// add to buffer and flush if auto flush enabled
func (client *Client) addToBuffer(key string, metricValue string) {
	// flush
	if client.autoflush {
		// send metric now
		go client.send(fmt.Sprintf("%s:%s", key, metricValue))
	} else {
		// add metric to buffer for next manual flush
		client.keyBufferLock.Lock()
		client.keyBuffer[key] = metricValue
		client.keyBufferLock.Unlock()

	}
}

// Check if acceptable by sample rate
func (client *Client) isSendAcceptedBySampleRate(sampleRate float32) bool {
	if sampleRate >= 1 {
		return true
	}
	randomNumber := client.rand.Float32()
	return randomNumber <= sampleRate
}

// Flush buffer to statsd daemon by UDP
func (client *Client) Flush() {
	// check if buffer has metrics
	if len(client.keyBuffer) == 0 {
		return
	}

	// prepare metric packet
	var metricPacketArray []string

	// lock
	client.keyBufferLock.Lock()

	// build packet
	for key, metricValue := range client.keyBuffer {
		metricPacketArray = append(metricPacketArray, fmt.Sprintf("%s:%s", key, metricValue))
	}
	metricPacket := strings.Join(metricPacketArray, "\n")

	// clear key buffer
	client.keyBuffer = make(map[string]string)

	// lock
	client.keyBufferLock.Unlock()

	// send packet
	go client.send(metricPacket)
}

// Send StatsD packet
func (client *Client) send(metricPacket string) {
	// send metric packet
	_, err := fmt.Fprintf(client.conn, metricPacket)
	if err != nil {
		log.Println(err)
	}
}
<|endoftext|>


<commit_before>// Copyright 2013 tsuru authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

package app

import (
	"fmt"
	"github.com/globocom/tsuru/log"
	"github.com/globocom/tsuru/provision"
	"github.com/globocom/tsuru/queue"
	"io/ioutil"
)

func ensureAppIsStarted(msg *queue.Message) (App, error) {
	a := App{Name: msg.Args[0]}
	err := a.Get()
	if err != nil {
		return a, fmt.Errorf("Error handling %q: app %q does not exist.", msg.Action, a.Name)
	}
	units := getUnits(&a, msg.Args[1:])
	if a.State != "started" || !units.Started() {
		format := "Error handling %q for the app %q:"
		switch a.State {
		case "error":
			format += " the app is in %q state."
			queue.Delete(msg)
		case "down":
			format += " the app is %s."
			queue.Delete(msg)
		default:
			format += ` The status of the app and all units should be "started" (the app is %q).`
		}
		return a, fmt.Errorf(format, msg.Action, a.Name, a.State)
	}
	return a, nil
}

func handle(msg *queue.Message) {
	switch msg.Action {
	case RegenerateApprc:
		if len(msg.Args) < 1 {
			log.Printf("Error handling %q: this action requires at least 1 argument.", msg.Action)
			return
		}
		app, err := ensureAppIsStarted(msg)
		if err != nil {
			log.Print(err)
			return
		}
		queue.Delete(msg)
		app.SerializeEnvVars()
	case StartApp:
		if len(msg.Args) < 1 {
			log.Printf("Error handling %q: this action requires at least 1 argument.", msg.Action)
		}
		app, err := ensureAppIsStarted(msg)
		if err != nil {
			log.Print(err)
			return
		}
		err = app.Restart(ioutil.Discard)
		if err != nil {
			log.Printf("Error handling %q. App failed to start:\n%s.", msg.Action, err)
			return
		}
		queue.Delete(msg)
	default:
		log.Printf("Error handling %q: invalid action.", msg.Action)
		msg.Release()
	}
}

type unitList []Unit

func (l unitList) Started() bool {
	for _, unit := range l {
		if unit.State != string(provision.StatusStarted) {
			return false
		}
	}
	return true
}

func getUnits(a *App, names []string) unitList {
	var units []Unit
	if len(names) > 0 {
		units = make([]Unit, len(names))
		i := 0
		for _, unitName := range names {
			for _, appUnit := range a.Units {
				if appUnit.Name == unitName {
					units[i] = appUnit
					i++
					break
				}
			}
		}
	}
	return unitList(units)
}

var handler = &queue.Handler{F: handle}

func enqueue(msgs ...queue.Message) {
	for _, msg := range msgs {
		copy := msg
		queue.Put(&copy)
	}
	handler.Start()
}
<commit_msg>queue: use msg.Release where we used PutBack<commit_after>// Copyright 2013 tsuru authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

package app

import (
	"fmt"
	"github.com/globocom/tsuru/log"
	"github.com/globocom/tsuru/provision"
	"github.com/globocom/tsuru/queue"
	"io/ioutil"
)

func ensureAppIsStarted(msg *queue.Message) (App, error) {
	a := App{Name: msg.Args[0]}
	err := a.Get()
	if err != nil {
		return a, fmt.Errorf("Error handling %q: app %q does not exist.", msg.Action, a.Name)
	}
	units := getUnits(&a, msg.Args[1:])
	if a.State != "started" || !units.Started() {
		format := "Error handling %q for the app %q:"
		switch a.State {
		case "error":
			format += " the app is in %q state."
			queue.Delete(msg)
		case "down":
			format += " the app is %s."
			queue.Delete(msg)
		default:
			format += ` The status of the app and all units should be "started" (the app is %q).`
			msg.Release()
		}
		return a, fmt.Errorf(format, msg.Action, a.Name, a.State)
	}
	return a, nil
}

func handle(msg *queue.Message) {
	switch msg.Action {
	case RegenerateApprc:
		if len(msg.Args) < 1 {
			log.Printf("Error handling %q: this action requires at least 1 argument.", msg.Action)
			return
		}
		app, err := ensureAppIsStarted(msg)
		if err != nil {
			log.Print(err)
			return
		}
		queue.Delete(msg)
		app.SerializeEnvVars()
	case StartApp:
		if len(msg.Args) < 1 {
			log.Printf("Error handling %q: this action requires at least 1 argument.", msg.Action)
		}
		app, err := ensureAppIsStarted(msg)
		if err != nil {
			log.Print(err)
			return
		}
		err = app.Restart(ioutil.Discard)
		if err != nil {
			log.Printf("Error handling %q. App failed to start:\n%s.", msg.Action, err)
			return
		}
		queue.Delete(msg)
	default:
		log.Printf("Error handling %q: invalid action.", msg.Action)
		msg.Release()
	}
}

type unitList []Unit

func (l unitList) Started() bool {
	for _, unit := range l {
		if unit.State != string(provision.StatusStarted) {
			return false
		}
	}
	return true
}

func getUnits(a *App, names []string) unitList {
	var units []Unit
	if len(names) > 0 {
		units = make([]Unit, len(names))
		i := 0
		for _, unitName := range names {
			for _, appUnit := range a.Units {
				if appUnit.Name == unitName {
					units[i] = appUnit
					i++
					break
				}
			}
		}
	}
	return unitList(units)
}

var handler = &queue.Handler{F: handle}

func enqueue(msgs ...queue.Message) {
	for _, msg := range msgs {
		copy := msg
		queue.Put(&copy)
	}
	handler.Start()
}
<|endoftext|>


<commit_before>// This module configures a redis connect for the application
package revelRedis

import (
	"github.com/gosexy/redis"
	"github.com/robfig/revel"
)

var (
	Redis *redis.Client
)

func Init() {
	// Read configuration.
	var found bool
	var host string
	var port int
	if host, found = revel.Config.String("redis.host"); !found {
		revel.ERROR.Fatal("No redis.host found.")
	}
	if port, found = revel.Config.Int("redis.port"); !found {
		port = 6379
	}

	Redis = redis.New()

	// Open a connection.
	var err error
	err = Redis.Connect(host, uint(port))
	if err != nil {
		revel.ERROR.Fatal(err)
	}
}

type RedisController struct {
	*revel.Controller
	Redis *redis.Client
}

func (c *RedisController) Begin() revel.Result {
	c.Redis = Redis
	return nil
}

func init() {
	revel.OnAppStart(Init)
	revel.InterceptMethod((*RedisController).Begin, revel.BEFORE)
}
<commit_msg>Handle REDIS_URL env value<commit_after>// This module configures a redis connect for the application
package revelRedis

import (
	"github.com/gosexy/redis"
	"github.com/robfig/revel"
	"os"
	"regexp"
	"strings"
	"strconv"
)

var (
	Redis *redis.Client
)

func Init() {
	// Read configuration.
	var found bool
	var host string
	var port int

	// First look in the environment for REDIS_URL
	url := os.Getenv("REDIS_URL")

	// Check it matches a redis url format
	if match, _ := regexp.MatchString("^redis://.*:[0-9]+$", url); match {

		// Remove the scheme
		url = strings.Replace(url, "redis://", "", 1)

		// Split to get the port off the end
		parts := strings.Split(url, ":")
		port64, _ := strconv.ParseInt(parts[len(parts)-1], 0, 0)
		if port64 > 0{
			port = int(port64)
		}

		// Remove the port part and join to get the hostname
		parts = parts[:len(parts)-1]
		host = strings.Join(parts, ":")
	}

	// Then look into the configuration for redis.host and redis.port
	if len(host) == 0 {
		if host, found = revel.Config.String("redis.host"); !found {
			revel.ERROR.Fatal("No redis.host found.")
		}
	}
	if port == 0 {
		if port, found = revel.Config.Int("redis.port"); !found {
			port = 6379
		}
	}

	Redis = redis.New()

	// Open a connection.
	var err error
	err = Redis.Connect(host, uint(port))
	if err != nil {
		revel.ERROR.Fatal(err)
	}
}

type RedisController struct {
	*revel.Controller
	Redis *redis.Client
}

func (c *RedisController) Begin() revel.Result {
	c.Redis = Redis
	return nil
}

func init() {
	revel.OnAppStart(Init)
	revel.InterceptMethod((*RedisController).Begin, revel.BEFORE)
}
<|endoftext|>


<commit_before>package app

import merkle "github.com/tendermint/go-merkle"

// State represents the app states, separating the commited state (for queries)
// from the working state (for CheckTx and AppendTx)
type State struct {
	committed merkle.Tree
	deliverTx merkle.Tree
	checkTx   merkle.Tree
}

func NewState(tree merkle.Tree, persistent bool) State {
	return State{
		committed: tree,
		deliverTx: tree.Copy(),
		checkTx:   tree.Copy(),
	}
}

func (s State) Committed() merkle.Tree {
	return s.committed
}

func (s State) Append() merkle.Tree {
	return s.deliverTx
}

func (s State) Check() merkle.Tree {
	return s.checkTx
}

// Commit stores the current Append() state as committed
// starts new Append/Check state, and
// returns the hash for the commit
func (s *State) Commit() []byte {
	// TODO: use save, broken for now, cuz it requires a backing db
	hash := s.deliverTx.Hash()
	s.committed = s.deliverTx
	s.deliverTx = s.committed.Copy()
	s.checkTx = s.committed.Copy()
	return hash
}
<commit_msg>persistent variable passed<commit_after>package app

import merkle "github.com/tendermint/go-merkle"

// State represents the app states, separating the commited state (for queries)
// from the working state (for CheckTx and AppendTx)
type State struct {
	committed  merkle.Tree
	deliverTx  merkle.Tree
	checkTx    merkle.Tree
	persistent bool
}

func NewState(tree merkle.Tree, persistent bool) State {
	return State{
		committed:  tree,
		deliverTx:  tree.Copy(),
		checkTx:    tree.Copy(),
		persistent: persistent,
	}
}

func (s State) Committed() merkle.Tree {
	return s.committed
}

func (s State) Append() merkle.Tree {
	return s.deliverTx
}

func (s State) Check() merkle.Tree {
	return s.checkTx
}

// Commit stores the current Append() state as committed
// starts new Append/Check state, and
// returns the hash for the commit
func (s *State) Commit() []byte {
	var hash []byte
	if s.persistent {
		hash = s.deliverTx.Save()
	} else {
		hash = s.deliverTx.Hash()
	}

	s.committed = s.deliverTx
	s.deliverTx = s.committed.Copy()
	s.checkTx = s.committed.Copy()
	return hash
}
<|endoftext|>


<commit_before>package storage

import (
	"fmt"
	"github.com/MatthewHartstonge/storage/client"
	"github.com/MatthewHartstonge/storage/request"
	"github.com/ory/fosite"
	"github.com/pkg/errors"
	"gopkg.in/mgo.v2"
	"strconv"
	"strings"
	"time"
)

// Config provides a way to define the specific pieces that make up a mongo connection
type Config struct {
	// Default connection settings
	Hostname     string
	Hostnames    []string
	Port         uint16 // 0 to 65,535
	DatabaseName string

	// Credential Access
	Username string
	Password string

	// Replica Set
	Replset string

	// Timeout specified in seconds.
	Timeout uint
}

// DefaultConfig returns a configuration for a locally hosted, unauthenticated mongo
func DefaultConfig() *Config {
	return &Config{
		Hostname:     "127.0.0.1",
		Port:         27017,
		DatabaseName: "OAuth2",
	}
}

// ConnectionURI generates a formatted Mongo Connection URL
func ConnectionURI(cfg *Config) string {
	connectionString := "mongodb://"
	credentials := ""

	if cfg.Username != "" && cfg.Password != "" {
		credentials = fmt.Sprintf("%s:%s@", cfg.Username, cfg.Password)
	}

	hosts := ""
	if cfg.Hostnames != nil && cfg.Hostname == "" {
		hosts = strings.Join(cfg.Hostnames, fmt.Sprintf(":%s,", strconv.Itoa(int(cfg.Port))))
		cfg.Hostname = hosts
	}

	connectionString = fmt.Sprintf("%s%s%s:%s/%s",
		connectionString,
		credentials,
		cfg.Hostname,
		strconv.Itoa(int(cfg.Port)),
		cfg.DatabaseName,
	)

	if cfg.Replset != "" {
		connectionString += "?replicaSet=" + cfg.Replset
	}

	return connectionString
}

// MongoStore provides a fosite Datastore.
type MongoStore struct {
	Clients *client.MongoManager

	// OAuth Stores
	AuthorizeCodes *request.MongoManager
	IDSessions     *request.MongoManager
	Implicit       *request.MongoManager

	// TODO: Work out if tokens should be stored using fosite.Requester or look at jwk.Manager from ory/hydra
	// TODO: Work out what is actually stored with a token requester
	AccessTokens  *request.MongoManager
	RefreshTokens *request.MongoManager

	// TODO: Create User Manager
	//Users *user.MongoManager

	// TODO: Create different cache storage backends?
	// - *cache.MemoryManager
	// - *cache.MongoManager
	// - *cache.RedisManager
	//AccessTokenRequestIDs *cache.MemoryManager
	//RefreshTokenRequestIDs *cache.MemoryManager
}

// Close ensures that each endpoint has it's connection closed properly.
func (m *MongoStore) Close() {
	m.Clients.DB.Session.Close()
}

// ConnectToMongo returns a connection to mongo.
func ConnectToMongo(cfg *Config) (*mgo.Database, error) {
	uri := ConnectionURI(cfg)
	if cfg.Timeout == 0 {
		cfg.Timeout = 10
	}
	session, err := mgo.DialWithTimeout(uri, time.Second*time.Duration(cfg.Timeout))
	if err != nil {
		return nil, errors.WithStack(err)
	}
	return session.DB(cfg.DatabaseName), nil
}

// NewDefaultMongoStore returns a MongoStore configured with the default mongo configuration and default hasher.
func NewDefaultMongoStore() (*MongoStore, error) {
	cfg := DefaultConfig()
	sess, err := ConnectToMongo(cfg)
	if err != nil {
		return nil, errors.WithStack(err)
	}
	h := &fosite.BCrypt{WorkFactor: 10}
	store := &MongoStore{
		Clients: &client.MongoManager{
			DB:     sess,
			Hasher: h,
		},
	}
	return store, err
}

// NewMongoStore allows for custom mongo configuration and custom hashers.
func NewMongoStore(c *Config, h fosite.Hasher) (*MongoStore, error) {
	sess, err := ConnectToMongo(c)
	if err != nil {
		return nil, errors.WithStack(err)
	}
	if h == nil {
		h = &fosite.BCrypt{WorkFactor: 10}
	}
	store := &MongoStore{
		Clients: &client.MongoManager{
			DB:     sess,
			Hasher: h,
		},
	}
	return store, nil
}
<commit_msg>:arrow_up: request: Implement request.MongoManager into the NewDatastore functions<commit_after>package storage

import (
	"fmt"
	"github.com/MatthewHartstonge/storage/client"
	"github.com/MatthewHartstonge/storage/request"
	"github.com/ory/fosite"
	"github.com/pkg/errors"
	"gopkg.in/mgo.v2"
	"strconv"
	"strings"
	"time"
)

// Config provides a way to define the specific pieces that make up a mongo connection
type Config struct {
	// Default connection settings
	Hostname     string
	Hostnames    []string
	Port         uint16 // 0 to 65,535
	DatabaseName string

	// Credential Access
	Username string
	Password string

	// Replica Set
	Replset string

	// Timeout specified in seconds.
	Timeout uint
}

// DefaultConfig returns a configuration for a locally hosted, unauthenticated mongo
func DefaultConfig() *Config {
	return &Config{
		Hostname:     "127.0.0.1",
		Port:         27017,
		DatabaseName: "OAuth2",
	}
}

// ConnectionURI generates a formatted Mongo Connection URL
func ConnectionURI(cfg *Config) string {
	connectionString := "mongodb://"
	credentials := ""

	if cfg.Username != "" && cfg.Password != "" {
		credentials = fmt.Sprintf("%s:%s@", cfg.Username, cfg.Password)
	}

	hosts := ""
	if cfg.Hostnames != nil && cfg.Hostname == "" {
		hosts = strings.Join(cfg.Hostnames, fmt.Sprintf(":%s,", strconv.Itoa(int(cfg.Port))))
		cfg.Hostname = hosts
	}

	connectionString = fmt.Sprintf("%s%s%s:%s/%s",
		connectionString,
		credentials,
		cfg.Hostname,
		strconv.Itoa(int(cfg.Port)),
		cfg.DatabaseName,
	)

	if cfg.Replset != "" {
		connectionString += "?replicaSet=" + cfg.Replset
	}

	return connectionString
}

// MongoStore provides a fosite Datastore.
type MongoStore struct {
	Clients *client.MongoManager

	// OAuth Stores
	AuthorizeCodes *request.MongoManager
	IDSessions     *request.MongoManager
	Implicit       *request.MongoManager
	AccessTokens   *request.MongoManager
	RefreshTokens  *request.MongoManager

	// TODO: Create User Manager
	//Users *user.MongoManager

	// TODO: Create different cache storage backends?
	// - *cache.MemoryManager
	// - *cache.MongoManager
	// - *cache.RedisManager
	//AccessTokenRequestIDs *cache.MemoryManager
	//RefreshTokenRequestIDs *cache.MemoryManager
}

// Close ensures that each endpoint has it's connection closed properly.
func (m *MongoStore) Close() {
	m.Clients.DB.Session.Close()
}

// ConnectToMongo returns a connection to mongo.
func ConnectToMongo(cfg *Config) (*mgo.Database, error) {
	uri := ConnectionURI(cfg)
	if cfg.Timeout == 0 {
		cfg.Timeout = 10
	}
	session, err := mgo.DialWithTimeout(uri, time.Second*time.Duration(cfg.Timeout))
	if err != nil {
		return nil, errors.WithStack(err)
	}
	return session.DB(cfg.DatabaseName), nil
}

// NewDefaultMongoStore returns a MongoStore configured with the default mongo configuration and default hasher.
func NewDefaultMongoStore() (*MongoStore, error) {
	cfg := DefaultConfig()
	sess, err := ConnectToMongo(cfg)
	if err != nil {
		return nil, errors.WithStack(err)
	}
	h := &fosite.BCrypt{WorkFactor: 10}
	c := &client.MongoManager{
		DB:     sess,
		Hasher: h,
	}
	r := &request.MongoManager{
		MongoManager: *c,
		DB:           sess,
	}
	return &MongoStore{
		Clients:        c,
		AuthorizeCodes: r,
		IDSessions:     r,
		Implicit:       r,
		AccessTokens:   r,
		RefreshTokens:  r,
	}, nil
}

// NewMongoStore allows for custom mongo configuration and custom hashers.
func NewMongoStore(cfg *Config, h fosite.Hasher) (*MongoStore, error) {
	sess, err := ConnectToMongo(cfg)
	if err != nil {
		return nil, errors.WithStack(err)
	}
	if h == nil {
		h = &fosite.BCrypt{WorkFactor: 10}
	}
	c := &client.MongoManager{
		DB:     sess,
		Hasher: h,
	}
	r := &request.MongoManager{
		MongoManager: *c,
		DB:           sess,
	}
	return &MongoStore{
		Clients:        c,
		AuthorizeCodes: r,
		IDSessions:     r,
		Implicit:       r,
		AccessTokens:   r,
		RefreshTokens:  r,
	}, nil
}
<|endoftext|>