<commit_before>// Copyright 2012 The Plotinum Authors. All rights reserved.
// Use of this source code is governed by an MIT-style license
// that can be found in the LICENSE file.

// The vgpdf implemens the vg.Canvas interface
// using gopdf (bitbucket.org/zombiezen/gopdf/pdf).
package vgpdf

import (
	"bitbucket.org/zombiezen/gopdf/pdf"
	"bufio"
	"code.google.com/p/plotinum/vg"
	"fmt"
	"image/color"
	"io"
	"math"
)

// Canvas implements the vg.Canvas interface,
// drawing to a PDF.
type Canvas struct {
	doc         *pdf.Document
	w, h        vg.Length
	page        *pdf.Canvas
	lineVisible bool
}

// New creates a new PDF Canvas.
func New(w, h vg.Length) *Canvas {
	c := &Canvas{
		doc: pdf.New(),
		w:   w,
		h:   h,
	}
	c.page = c.doc.NewPage(unit(w), unit(h))
	vg.Initialize(c)
	return c
}

func (c *Canvas) Size() (w, h vg.Length) {
	return c.w, c.h
}

func (c *Canvas) SetLineWidth(w vg.Length) {
	c.page.SetLineWidth(unit(w))
	c.lineVisible = w != 0
}

func (c *Canvas) SetLineDash(dashes []vg.Length, offs vg.Length) {
	ds := make([]pdf.Unit, len(dashes))
	for i, d := range dashes {
		ds[i] = unit(d)
	}
	c.page.SetLineDash(unit(offs), ds)
}

func (c *Canvas) SetColor(clr color.Color) {
	c.page.SetStrokeColor(pdfColor(clr))
	c.page.SetColor(pdfColor(clr))
}

func (c *Canvas) Rotate(r float64) {
	c.page.Rotate(float32(r))
}

func (c *Canvas) Translate(x vg.Length, y vg.Length) {
	c.page.Translate(unit(x), unit(y))
}

func (c *Canvas) Scale(x float64, y float64) {
	c.page.Scale(float32(x), float32(y))
}

func (c *Canvas) Push() {
	c.page.Push()
}

func (c *Canvas) Pop() {
	c.page.Pop()
}

func (c *Canvas) Stroke(p vg.Path) {
	if c.lineVisible {
		c.page.Stroke(pdfPath(c, p))
	}
}

func (c *Canvas) Fill(p vg.Path) {
	c.page.Fill(pdfPath(c, p))
}

func (c *Canvas) FillString(fnt vg.Font, x, y vg.Length, str string) {
	t := new(pdf.Text)
	t.SetFont(fnt.Name(), unit(fnt.Size))
	t.NextLineOffset(unit(x), unit(y))
	t.Text(str)
	c.page.DrawText(t)
}

func (*Canvas) DPI() float64 {
	return float64(pdf.Inch)
}

// pdfPath returns a pdf.Path from a vg.Path.
func pdfPath(c *Canvas, path vg.Path) *pdf.Path {
	p := new(pdf.Path)
	for _, comp := range path {
		switch comp.Type {
		case vg.MoveComp:
			p.Move(pdfPoint(comp.X, comp.Y))
		case vg.LineComp:
			p.Line(pdfPoint(comp.X, comp.Y))
		case vg.ArcComp:
			arc(p, comp)
		case vg.CloseComp:
			p.Close()
		default:
			panic(fmt.Sprintf("Unknown path component type: %d\n", comp.Type))
		}
	}
	return p
}

// Approximate a circular arc using multiple
// cubic Bézier curves, one for each π/2 segment.
//
// This is from:
// 	http://hansmuller-flex.blogspot.com/2011/04/approximating-circular-arc-with-cubic.html
func arc(p *pdf.Path, comp vg.PathComp) {
	x0 := comp.X + comp.Radius*vg.Length(math.Cos(comp.Start))
	y0 := comp.Y + comp.Radius*vg.Length(math.Sin(comp.Start))
	p.Line(pdfPoint(x0, y0))

	a1 := comp.Start
	end := a1 + comp.Angle
	sign := 1.0
	if end < a1 {
		sign = -1.0
	}
	left := math.Abs(comp.Angle)
	for left > 0 {
		a2 := a1 + sign*math.Min(math.Pi/2, left)
		partialArc(p, comp.X, comp.Y, comp.Radius, a1, a2)
		left -= math.Abs(a2 - a1)
		a1 = a2
	}
}

// Approximate a circular arc of fewer than π/2
// radians with cubic Bézier curve.
func partialArc(p *pdf.Path, x, y, r vg.Length, a1, a2 float64) {
	a := (a2 - a1) / 2
	x4 := r * vg.Length(math.Cos(a))
	y4 := r * vg.Length(math.Sin(a))
	x1 := x4
	y1 := -y4

	const k = 0.5522847498 // some magic constant
	f := k * vg.Length(math.Tan(a))
	x2 := x1 + f*y4
	y2 := y1 + f*x4
	x3 := x2
	y3 := -y2

	// Rotate and translate points into position.
	ar := a + a1
	sinar := vg.Length(math.Sin(ar))
	cosar := vg.Length(math.Cos(ar))
	x2r := x2*cosar - y2*sinar + x
	y2r := x2*sinar + y2*cosar + y
	x3r := x3*cosar - y3*sinar + x
	y3r := x3*sinar + y3*cosar + y
	x4 = r*vg.Length(math.Cos(a2)) + x
	y4 = r*vg.Length(math.Sin(a2)) + y
	p.Curve(pdfPoint(x2r, y2r), pdfPoint(x3r, y3r), pdfPoint(x4, y4))
}

func pdfPoint(x, y vg.Length) pdf.Point {
	return pdf.Point{X: unit(x), Y: unit(y)}
}

func pdfColor(clr color.Color) (float32, float32, float32) {
	if clr == nil {
		clr = color.Black
	}
	r, g, b, _ := clr.RGBA()
	return float32(r) / math.MaxUint16,
		float32(g) / math.MaxUint16,
		float32(b) / math.MaxUint16
}

// unit returns a pdf.Unit, converted from a vg.Length.
func unit(l vg.Length) pdf.Unit {
	return pdf.Unit(l.Points()) * pdf.Pt
}

// WriterCounter implements the io.Writer interface, and counts
// the total number of bytes written.
type writerCounter struct {
	io.Writer
	n int64
}

func (w *writerCounter) Write(p []byte) (int, error) {
	n, err := w.Writer.Write(p)
	w.n += int64(n)
	return n, err
}

// WriteTo writes the Canvas to an io.Writer.
// After calling Write, the canvas is closed
// and may no longer be used for drawing.
func (c *Canvas) WriteTo(w io.Writer) (int64, error) {
	c.page.Close()
	wc := writerCounter{Writer: w}
	b := bufio.NewWriter(&wc)
	if err := c.doc.Encode(b); err != nil {
		return wc.n, err
	}
	err := b.Flush()
	return wc.n, err
}
<commit_msg>PDF line width defaults to visible.<commit_after>// Copyright 2012 The Plotinum Authors. All rights reserved.
// Use of this source code is governed by an MIT-style license
// that can be found in the LICENSE file.

// The vgpdf implemens the vg.Canvas interface
// using gopdf (bitbucket.org/zombiezen/gopdf/pdf).
package vgpdf

import (
	"bitbucket.org/zombiezen/gopdf/pdf"
	"bufio"
	"code.google.com/p/plotinum/vg"
	"fmt"
	"image/color"
	"io"
	"math"
)

// Canvas implements the vg.Canvas interface,
// drawing to a PDF.
type Canvas struct {
	doc         *pdf.Document
	w, h        vg.Length
	page        *pdf.Canvas
	lineVisible bool
}

// New creates a new PDF Canvas.
func New(w, h vg.Length) *Canvas {
	c := &Canvas{
		doc:         pdf.New(),
		w:           w,
		h:           h,
		lineVisible: true,
	}
	c.page = c.doc.NewPage(unit(w), unit(h))
	vg.Initialize(c)
	return c
}

func (c *Canvas) Size() (w, h vg.Length) {
	return c.w, c.h
}

func (c *Canvas) SetLineWidth(w vg.Length) {
	c.page.SetLineWidth(unit(w))
	c.lineVisible = w != 0
}

func (c *Canvas) SetLineDash(dashes []vg.Length, offs vg.Length) {
	ds := make([]pdf.Unit, len(dashes))
	for i, d := range dashes {
		ds[i] = unit(d)
	}
	c.page.SetLineDash(unit(offs), ds)
}

func (c *Canvas) SetColor(clr color.Color) {
	c.page.SetStrokeColor(pdfColor(clr))
	c.page.SetColor(pdfColor(clr))
}

func (c *Canvas) Rotate(r float64) {
	c.page.Rotate(float32(r))
}

func (c *Canvas) Translate(x vg.Length, y vg.Length) {
	c.page.Translate(unit(x), unit(y))
}

func (c *Canvas) Scale(x float64, y float64) {
	c.page.Scale(float32(x), float32(y))
}

func (c *Canvas) Push() {
	c.page.Push()
}

func (c *Canvas) Pop() {
	c.page.Pop()
}

func (c *Canvas) Stroke(p vg.Path) {
	if c.lineVisible {
		c.page.Stroke(pdfPath(c, p))
	}
}

func (c *Canvas) Fill(p vg.Path) {
	c.page.Fill(pdfPath(c, p))
}

func (c *Canvas) FillString(fnt vg.Font, x, y vg.Length, str string) {
	t := new(pdf.Text)
	t.SetFont(fnt.Name(), unit(fnt.Size))
	t.NextLineOffset(unit(x), unit(y))
	t.Text(str)
	c.page.DrawText(t)
}

func (*Canvas) DPI() float64 {
	return float64(pdf.Inch)
}

// pdfPath returns a pdf.Path from a vg.Path.
func pdfPath(c *Canvas, path vg.Path) *pdf.Path {
	p := new(pdf.Path)
	for _, comp := range path {
		switch comp.Type {
		case vg.MoveComp:
			p.Move(pdfPoint(comp.X, comp.Y))
		case vg.LineComp:
			p.Line(pdfPoint(comp.X, comp.Y))
		case vg.ArcComp:
			arc(p, comp)
		case vg.CloseComp:
			p.Close()
		default:
			panic(fmt.Sprintf("Unknown path component type: %d\n", comp.Type))
		}
	}
	return p
}

// Approximate a circular arc using multiple
// cubic Bézier curves, one for each π/2 segment.
//
// This is from:
// 	http://hansmuller-flex.blogspot.com/2011/04/approximating-circular-arc-with-cubic.html
func arc(p *pdf.Path, comp vg.PathComp) {
	x0 := comp.X + comp.Radius*vg.Length(math.Cos(comp.Start))
	y0 := comp.Y + comp.Radius*vg.Length(math.Sin(comp.Start))
	p.Line(pdfPoint(x0, y0))

	a1 := comp.Start
	end := a1 + comp.Angle
	sign := 1.0
	if end < a1 {
		sign = -1.0
	}
	left := math.Abs(comp.Angle)
	for left > 0 {
		a2 := a1 + sign*math.Min(math.Pi/2, left)
		partialArc(p, comp.X, comp.Y, comp.Radius, a1, a2)
		left -= math.Abs(a2 - a1)
		a1 = a2
	}
}

// Approximate a circular arc of fewer than π/2
// radians with cubic Bézier curve.
func partialArc(p *pdf.Path, x, y, r vg.Length, a1, a2 float64) {
	a := (a2 - a1) / 2
	x4 := r * vg.Length(math.Cos(a))
	y4 := r * vg.Length(math.Sin(a))
	x1 := x4
	y1 := -y4

	const k = 0.5522847498 // some magic constant
	f := k * vg.Length(math.Tan(a))
	x2 := x1 + f*y4
	y2 := y1 + f*x4
	x3 := x2
	y3 := -y2

	// Rotate and translate points into position.
	ar := a + a1
	sinar := vg.Length(math.Sin(ar))
	cosar := vg.Length(math.Cos(ar))
	x2r := x2*cosar - y2*sinar + x
	y2r := x2*sinar + y2*cosar + y
	x3r := x3*cosar - y3*sinar + x
	y3r := x3*sinar + y3*cosar + y
	x4 = r*vg.Length(math.Cos(a2)) + x
	y4 = r*vg.Length(math.Sin(a2)) + y
	p.Curve(pdfPoint(x2r, y2r), pdfPoint(x3r, y3r), pdfPoint(x4, y4))
}

func pdfPoint(x, y vg.Length) pdf.Point {
	return pdf.Point{X: unit(x), Y: unit(y)}
}

func pdfColor(clr color.Color) (float32, float32, float32) {
	if clr == nil {
		clr = color.Black
	}
	r, g, b, _ := clr.RGBA()
	return float32(r) / math.MaxUint16,
		float32(g) / math.MaxUint16,
		float32(b) / math.MaxUint16
}

// unit returns a pdf.Unit, converted from a vg.Length.
func unit(l vg.Length) pdf.Unit {
	return pdf.Unit(l.Points()) * pdf.Pt
}

// WriterCounter implements the io.Writer interface, and counts
// the total number of bytes written.
type writerCounter struct {
	io.Writer
	n int64
}

func (w *writerCounter) Write(p []byte) (int, error) {
	n, err := w.Writer.Write(p)
	w.n += int64(n)
	return n, err
}

// WriteTo writes the Canvas to an io.Writer.
// After calling Write, the canvas is closed
// and may no longer be used for drawing.
func (c *Canvas) WriteTo(w io.Writer) (int64, error) {
	c.page.Close()
	wc := writerCounter{Writer: w}
	b := bufio.NewWriter(&wc)
	if err := c.doc.Encode(b); err != nil {
		return wc.n, err
	}
	err := b.Flush()
	return wc.n, err
}
<|endoftext|>


<commit_before>// Package httpcontrol allows for HTTP transport level control around
// timeouts and retries.
package httpcontrol

import (
	"container/heap"
	"crypto/tls"
	"flag"
	"io"
	"log"
	"net"
	"net/http"
	"net/url"
	"strings"
	"sync"
	"time"

	"github.com/daaku/go.pqueue"
)

// Stats for a RoundTrip.
type Stats struct {
	// The RoundTrip request.
	Request *http.Request

	// May not always be available.
	Response *http.Response

	// Will be set if the RoundTrip resulted in an error. Note that these are
	// RoundTrip errors and we do not care about the HTTP Status.
	Error error

	// Each duration is independent and the sum of all of them is the total
	// request duration. One or more durations may be zero.
	Duration struct {
		Header, Body time.Duration
	}

	Retry struct {
		// Will be incremented for each retry. The initial request will have this set
		// to 0, and the first retry to 1 and so on.
		Count uint

		// Will be set if and only if an error was encountered and a retry is
		// pending.
		Pending bool
	}
}

// Transport is an implementation of RoundTripper that supports http, https,
// and http proxies (for either http or https with CONNECT). Transport can also
// cache connections for future re-use. It also provides various timeouts,
// retry logic and the ability to track request statistics.
type Transport struct {

	// Proxy specifies a function to return a proxy for a given
	// *http.Request. If the function returns a non-nil error, the
	// request is aborted with the provided error.
	// If Proxy is nil or returns a nil *url.URL, no proxy is used.
	Proxy func(*http.Request) (*url.URL, error)

	// TLSClientConfig specifies the TLS configuration to use with
	// tls.Client. If nil, the default configuration is used.
	TLSClientConfig *tls.Config

	// DisableKeepAlives, if true, prevents re-use of TCP connections
	// between different HTTP requests.
	DisableKeepAlives bool

	// DisableCompression, if true, prevents the Transport from
	// requesting compression with an "Accept-Encoding: gzip"
	// request header when the Request contains no existing
	// Accept-Encoding value. If the Transport requests gzip on
	// its own and gets a gzipped response, it's transparently
	// decoded in the Response.Body. However, if the user
	// explicitly requested gzip it is not automatically
	// uncompressed.
	DisableCompression bool

	// MaxIdleConnsPerHost, if non-zero, controls the maximum idle
	// (keep-alive) to keep per-host.  If zero,
	// http.DefaultMaxIdleConnsPerHost is used.
	MaxIdleConnsPerHost int

	// Timeout is the maximum amount of time a dial will wait for
	// a connect to complete.
	//
	// The default is no timeout.
	//
	// With or without a timeout, the operating system may impose
	// its own earlier timeout. For instance, TCP timeouts are
	// often around 3 minutes.
	DialTimeout time.Duration

	// ResponseHeaderTimeout, if non-zero, specifies the amount of
	// time to wait for a server's response headers after fully
	// writing the request (including its body, if any). This
	// time does not include the time to read the response body.
	ResponseHeaderTimeout time.Duration

	// RequestTimeout, if non-zero, specifies the amount of time for the entire
	// request. This includes dialing (if necessary), the response header as well
	// as the entire body.
	RequestTimeout time.Duration

	// MaxTries, if non-zero, specifies the number of times we will retry on
	// failure. Retries are only attempted for temporary network errors or known
	// safe failures.
	MaxTries uint

	// Stats allows for capturing the result of a request and is useful for
	// monitoring purposes.
	Stats func(*Stats)

	// Debug, if true, will trigger logging of interesting events to aid in
	// debugging the request flow.
	Debug bool

	transport    *http.Transport
	closeMonitor chan bool
	pqMutex      sync.Mutex
	pq           pqueue.PriorityQueue
}

var knownFailureSuffixes = []string{
	"connection refused",
	"connection reset by peer.",
	"connection timed out.",
	"no such host.",
	"remote error: handshake failure",
	"unexpected EOF.",
}

func shouldRetryError(err error) bool {
	if neterr, ok := err.(net.Error); ok {
		if neterr.Temporary() {
			return true
		}
	}

	s := err.Error()
	for _, suffix := range knownFailureSuffixes {
		if strings.HasSuffix(s, suffix) {
			return true
		}
	}
	return false
}

// Start the Transport.
func (t *Transport) Start() error {
	if t.Debug {
		log.Println("httpcontrol: Start")
	}
	dialer := &net.Dialer{Timeout: t.DialTimeout}
	t.transport = &http.Transport{
		Dial:                dialer.Dial,
		Proxy:               t.Proxy,
		TLSClientConfig:     t.TLSClientConfig,
		DisableKeepAlives:   t.DisableKeepAlives,
		DisableCompression:  t.DisableCompression,
		MaxIdleConnsPerHost: t.MaxIdleConnsPerHost,
	}
	t.closeMonitor = make(chan bool)
	t.pq = pqueue.New(16)
	go t.monitor()
	return nil
}

// Close the Transport.
func (t *Transport) Close() error {
	t.transport.CloseIdleConnections()
	t.closeMonitor <- true
	<-t.closeMonitor
	if t.Debug {
		log.Println("httpcontrol: Close")
	}
	return nil
}

func (t *Transport) monitor() {
	ticker := time.NewTicker(25 * time.Millisecond)
	for {
		select {
		case <-t.closeMonitor:
			ticker.Stop()
			close(t.closeMonitor)
			return
		case n := <-ticker.C:
			now := n.UnixNano()
			for {
				t.pqMutex.Lock()
				item, _ := t.pq.PeekAndShift(now)
				t.pqMutex.Unlock()

				if item == nil {
					break
				}

				req := item.Value.(*http.Request)
				if t.Debug {
					log.Printf("httpcontrol: Request Timeout: %s", req.URL)
				}
				t.CancelRequest(req)
			}
		}
	}
}

func (t *Transport) CancelRequest(req *http.Request) {
	t.transport.CancelRequest(req)
}

func (t *Transport) tries(req *http.Request, try uint) (*http.Response, error) {
	startTime := time.Now()
	deadline := startTime.Add(t.RequestTimeout).UnixNano()
	item := &pqueue.Item{Value: req, Priority: deadline}
	t.pqMutex.Lock()
	heap.Push(&t.pq, item)
	t.pqMutex.Unlock()
	res, err := t.transport.RoundTrip(req)
	headerTime := time.Now()
	if err != nil {
		t.pqMutex.Lock()
		if item.Index != -1 {
			heap.Remove(&t.pq, item.Index)
		}
		t.pqMutex.Unlock()

		var stats *Stats
		if t.Stats != nil {
			stats = &Stats{
				Request:  req,
				Response: res,
				Error:    err,
			}
			stats.Duration.Header = headerTime.Sub(startTime)
			stats.Retry.Count = try
		}

		if try < t.MaxTries && req.Method == "GET" && shouldRetryError(err) {
			if t.Stats != nil {
				stats.Retry.Pending = true
				t.Stats(stats)
			}
			return t.tries(req, try+1)
		}

		if t.Stats != nil {
			t.Stats(stats)
		}
		return nil, err
	}

	res.Body = &bodyCloser{
		ReadCloser: res.Body,
		res:        res,
		item:       item,
		transport:  t,
		startTime:  startTime,
		headerTime: headerTime,
	}
	return res, nil
}

func (t *Transport) RoundTrip(req *http.Request) (*http.Response, error) {
	if t.Debug {
		log.Printf("httpcontrol: Request: %s", req.URL)
	}
	return t.tries(req, 0)
}

type bodyCloser struct {
	io.ReadCloser
	res        *http.Response
	item       *pqueue.Item
	transport  *Transport
	startTime  time.Time
	headerTime time.Time
}

func (b *bodyCloser) Close() error {
	err := b.ReadCloser.Close()
	closeTime := time.Now()
	b.transport.pqMutex.Lock()
	if b.item.Index != -1 {
		heap.Remove(&b.transport.pq, b.item.Index)
	}
	b.transport.pqMutex.Unlock()
	if b.transport.Stats != nil {
		stats := &Stats{
			Request:  b.res.Request,
			Response: b.res,
		}
		stats.Duration.Header = b.headerTime.Sub(b.startTime)
		stats.Duration.Body = closeTime.Sub(b.startTime) - stats.Duration.Header
		b.transport.Stats(stats)
	}
	return err
}

// A Flag configured Transport instance.
func TransportFlag(name string) *Transport {
	t := &Transport{TLSClientConfig: &tls.Config{}}
	flag.BoolVar(
		&t.TLSClientConfig.InsecureSkipVerify,
		name+".insecure-tls",
		false,
		name+" skip tls certificate verification",
	)
	flag.BoolVar(
		&t.DisableKeepAlives,
		name+".disable-keepalive",
		false,
		name+" disable keep-alives",
	)
	flag.BoolVar(
		&t.DisableCompression,
		name+".disable-compression",
		false,
		name+" disable compression",
	)
	flag.IntVar(
		&t.MaxIdleConnsPerHost,
		name+".max-idle-conns-per-host",
		http.DefaultMaxIdleConnsPerHost,
		name+" max idle connections per host",
	)
	flag.DurationVar(
		&t.DialTimeout,
		name+".dial-timeout",
		2*time.Second,
		name+" dial timeout",
	)
	flag.DurationVar(
		&t.ResponseHeaderTimeout,
		name+".response-header-timeout",
		3*time.Second,
		name+" response header timeout",
	)
	flag.DurationVar(
		&t.RequestTimeout,
		name+".request-timeout",
		30*time.Second,
		name+" request timeout",
	)
	flag.UintVar(
		&t.MaxTries,
		name+".max-tries",
		0,
		name+" max retries for known safe failures",
	)
	flag.BoolVar(
		&t.Debug,
		name+".debug",
		false,
		name+" debug logging",
	)
	return t
}
<commit_msg>String method on Stats for debugging purposes<commit_after>// Package httpcontrol allows for HTTP transport level control around
// timeouts and retries.
package httpcontrol

import (
	"bytes"
	"container/heap"
	"crypto/tls"
	"flag"
	"fmt"
	"io"
	"log"
	"net"
	"net/http"
	"net/url"
	"strings"
	"sync"
	"time"

	"github.com/daaku/go.pqueue"
)

// Stats for a RoundTrip.
type Stats struct {
	// The RoundTrip request.
	Request *http.Request

	// May not always be available.
	Response *http.Response

	// Will be set if the RoundTrip resulted in an error. Note that these are
	// RoundTrip errors and we do not care about the HTTP Status.
	Error error

	// Each duration is independent and the sum of all of them is the total
	// request duration. One or more durations may be zero.
	Duration struct {
		Header, Body time.Duration
	}

	Retry struct {
		// Will be incremented for each retry. The initial request will have this set
		// to 0, and the first retry to 1 and so on.
		Count uint

		// Will be set if and only if an error was encountered and a retry is
		// pending.
		Pending bool
	}
}

func (s *Stats) String() string {
	var buf bytes.Buffer
	fmt.Fprintf(&buf, "%s %s", s.Request.Method, s.Request.URL)

	if s.Response != nil {
		fmt.Fprintf(&buf, " got response with status %s", s.Response.Status)
	}

	return buf.String()
}

// Transport is an implementation of RoundTripper that supports http, https,
// and http proxies (for either http or https with CONNECT). Transport can also
// cache connections for future re-use. It also provides various timeouts,
// retry logic and the ability to track request statistics.
type Transport struct {

	// Proxy specifies a function to return a proxy for a given
	// *http.Request. If the function returns a non-nil error, the
	// request is aborted with the provided error.
	// If Proxy is nil or returns a nil *url.URL, no proxy is used.
	Proxy func(*http.Request) (*url.URL, error)

	// TLSClientConfig specifies the TLS configuration to use with
	// tls.Client. If nil, the default configuration is used.
	TLSClientConfig *tls.Config

	// DisableKeepAlives, if true, prevents re-use of TCP connections
	// between different HTTP requests.
	DisableKeepAlives bool

	// DisableCompression, if true, prevents the Transport from
	// requesting compression with an "Accept-Encoding: gzip"
	// request header when the Request contains no existing
	// Accept-Encoding value. If the Transport requests gzip on
	// its own and gets a gzipped response, it's transparently
	// decoded in the Response.Body. However, if the user
	// explicitly requested gzip it is not automatically
	// uncompressed.
	DisableCompression bool

	// MaxIdleConnsPerHost, if non-zero, controls the maximum idle
	// (keep-alive) to keep per-host.  If zero,
	// http.DefaultMaxIdleConnsPerHost is used.
	MaxIdleConnsPerHost int

	// Timeout is the maximum amount of time a dial will wait for
	// a connect to complete.
	//
	// The default is no timeout.
	//
	// With or without a timeout, the operating system may impose
	// its own earlier timeout. For instance, TCP timeouts are
	// often around 3 minutes.
	DialTimeout time.Duration

	// ResponseHeaderTimeout, if non-zero, specifies the amount of
	// time to wait for a server's response headers after fully
	// writing the request (including its body, if any). This
	// time does not include the time to read the response body.
	ResponseHeaderTimeout time.Duration

	// RequestTimeout, if non-zero, specifies the amount of time for the entire
	// request. This includes dialing (if necessary), the response header as well
	// as the entire body.
	RequestTimeout time.Duration

	// MaxTries, if non-zero, specifies the number of times we will retry on
	// failure. Retries are only attempted for temporary network errors or known
	// safe failures.
	MaxTries uint

	// Stats allows for capturing the result of a request and is useful for
	// monitoring purposes.
	Stats func(*Stats)

	// Debug, if true, will trigger logging of interesting events to aid in
	// debugging the request flow.
	Debug bool

	transport    *http.Transport
	closeMonitor chan bool
	pqMutex      sync.Mutex
	pq           pqueue.PriorityQueue
}

var knownFailureSuffixes = []string{
	"connection refused",
	"connection reset by peer.",
	"connection timed out.",
	"no such host.",
	"remote error: handshake failure",
	"unexpected EOF.",
}

func shouldRetryError(err error) bool {
	if neterr, ok := err.(net.Error); ok {
		if neterr.Temporary() {
			return true
		}
	}

	s := err.Error()
	for _, suffix := range knownFailureSuffixes {
		if strings.HasSuffix(s, suffix) {
			return true
		}
	}
	return false
}

// Start the Transport.
func (t *Transport) Start() error {
	if t.Debug {
		log.Println("httpcontrol: Start")
	}
	dialer := &net.Dialer{Timeout: t.DialTimeout}
	t.transport = &http.Transport{
		Dial:                dialer.Dial,
		Proxy:               t.Proxy,
		TLSClientConfig:     t.TLSClientConfig,
		DisableKeepAlives:   t.DisableKeepAlives,
		DisableCompression:  t.DisableCompression,
		MaxIdleConnsPerHost: t.MaxIdleConnsPerHost,
	}
	t.closeMonitor = make(chan bool)
	t.pq = pqueue.New(16)
	go t.monitor()
	return nil
}

// Close the Transport.
func (t *Transport) Close() error {
	t.transport.CloseIdleConnections()
	t.closeMonitor <- true
	<-t.closeMonitor
	if t.Debug {
		log.Println("httpcontrol: Close")
	}
	return nil
}

func (t *Transport) monitor() {
	ticker := time.NewTicker(25 * time.Millisecond)
	for {
		select {
		case <-t.closeMonitor:
			ticker.Stop()
			close(t.closeMonitor)
			return
		case n := <-ticker.C:
			now := n.UnixNano()
			for {
				t.pqMutex.Lock()
				item, _ := t.pq.PeekAndShift(now)
				t.pqMutex.Unlock()

				if item == nil {
					break
				}

				req := item.Value.(*http.Request)
				if t.Debug {
					log.Printf("httpcontrol: Request Timeout: %s", req.URL)
				}
				t.CancelRequest(req)
			}
		}
	}
}

func (t *Transport) CancelRequest(req *http.Request) {
	t.transport.CancelRequest(req)
}

func (t *Transport) tries(req *http.Request, try uint) (*http.Response, error) {
	startTime := time.Now()
	deadline := startTime.Add(t.RequestTimeout).UnixNano()
	item := &pqueue.Item{Value: req, Priority: deadline}
	t.pqMutex.Lock()
	heap.Push(&t.pq, item)
	t.pqMutex.Unlock()
	res, err := t.transport.RoundTrip(req)
	headerTime := time.Now()
	if err != nil {
		t.pqMutex.Lock()
		if item.Index != -1 {
			heap.Remove(&t.pq, item.Index)
		}
		t.pqMutex.Unlock()

		var stats *Stats
		if t.Stats != nil {
			stats = &Stats{
				Request:  req,
				Response: res,
				Error:    err,
			}
			stats.Duration.Header = headerTime.Sub(startTime)
			stats.Retry.Count = try
		}

		if try < t.MaxTries && req.Method == "GET" && shouldRetryError(err) {
			if t.Stats != nil {
				stats.Retry.Pending = true
				t.Stats(stats)
			}
			return t.tries(req, try+1)
		}

		if t.Stats != nil {
			t.Stats(stats)
		}
		return nil, err
	}

	res.Body = &bodyCloser{
		ReadCloser: res.Body,
		res:        res,
		item:       item,
		transport:  t,
		startTime:  startTime,
		headerTime: headerTime,
	}
	return res, nil
}

func (t *Transport) RoundTrip(req *http.Request) (*http.Response, error) {
	if t.Debug {
		log.Printf("httpcontrol: Request: %s", req.URL)
	}
	return t.tries(req, 0)
}

type bodyCloser struct {
	io.ReadCloser
	res        *http.Response
	item       *pqueue.Item
	transport  *Transport
	startTime  time.Time
	headerTime time.Time
}

func (b *bodyCloser) Close() error {
	err := b.ReadCloser.Close()
	closeTime := time.Now()
	b.transport.pqMutex.Lock()
	if b.item.Index != -1 {
		heap.Remove(&b.transport.pq, b.item.Index)
	}
	b.transport.pqMutex.Unlock()
	if b.transport.Stats != nil {
		stats := &Stats{
			Request:  b.res.Request,
			Response: b.res,
		}
		stats.Duration.Header = b.headerTime.Sub(b.startTime)
		stats.Duration.Body = closeTime.Sub(b.startTime) - stats.Duration.Header
		b.transport.Stats(stats)
	}
	return err
}

// A Flag configured Transport instance.
func TransportFlag(name string) *Transport {
	t := &Transport{TLSClientConfig: &tls.Config{}}
	flag.BoolVar(
		&t.TLSClientConfig.InsecureSkipVerify,
		name+".insecure-tls",
		false,
		name+" skip tls certificate verification",
	)
	flag.BoolVar(
		&t.DisableKeepAlives,
		name+".disable-keepalive",
		false,
		name+" disable keep-alives",
	)
	flag.BoolVar(
		&t.DisableCompression,
		name+".disable-compression",
		false,
		name+" disable compression",
	)
	flag.IntVar(
		&t.MaxIdleConnsPerHost,
		name+".max-idle-conns-per-host",
		http.DefaultMaxIdleConnsPerHost,
		name+" max idle connections per host",
	)
	flag.DurationVar(
		&t.DialTimeout,
		name+".dial-timeout",
		2*time.Second,
		name+" dial timeout",
	)
	flag.DurationVar(
		&t.ResponseHeaderTimeout,
		name+".response-header-timeout",
		3*time.Second,
		name+" response header timeout",
	)
	flag.DurationVar(
		&t.RequestTimeout,
		name+".request-timeout",
		30*time.Second,
		name+" request timeout",
	)
	flag.UintVar(
		&t.MaxTries,
		name+".max-tries",
		0,
		name+" max retries for known safe failures",
	)
	flag.BoolVar(
		&t.Debug,
		name+".debug",
		false,
		name+" debug logging",
	)
	return t
}
<|endoftext|>


<commit_before>package cli

import (
	"context"
	"errors"
	"flag"
	"fmt"
	"strings"
)

// Context is a type that is passed through to
// each Handler action in a cli application. Context
// can be used to retrieve context-specific args and
// parsed command-line options.
type Context struct {
	context.Context
	App           *App
	Command       *Command
	shellComplete bool
	flagSet       *flag.FlagSet
	parentContext *Context
}

// NewContext creates a new context. For use in when invoking an App or Command action.
func NewContext(app *App, set *flag.FlagSet, parentCtx *Context) *Context {
	c := &Context{App: app, flagSet: set, parentContext: parentCtx}
	if parentCtx != nil {
		c.Context = parentCtx.Context
		c.shellComplete = parentCtx.shellComplete
		if parentCtx.flagSet == nil {
			parentCtx.flagSet = &flag.FlagSet{}
		}
	}

	c.Command = &Command{}

	if c.Context == nil {
		c.Context = context.Background()
	}

	return c
}

// NumFlags returns the number of flags set
func (c *Context) NumFlags() int {
			return c.flagSet.NFlag()
}

// Set sets a context flag to a value.
func (c *Context) Set(name, value string) error {
	return c.flagSet.Set(name, value)
}

// IsSet determines if the flag was actually set
func (c *Context) IsSet(name string) bool {
	if fs := lookupFlagSet(name, c); fs != nil {
		if fs := lookupFlagSet(name, c); fs != nil {
			isSet := false
			fs.Visit(func(f *flag.Flag) {
				if f.Name == name {
					isSet = true
				}
			})
			if isSet {
				return true
			}
		}

		f := lookupFlag(name, c)
		if f == nil {
			return false
		}

		return f.IsSet()
	}

	return false
}

// LocalFlagNames returns a slice of flag names used in this context.
func (c *Context) LocalFlagNames() []string {
	var names []string
	c.flagSet.Visit(makeFlagNameVisitor(&names))
	return names
}

// FlagNames returns a slice of flag names used by the this context and all of
// its parent contexts.
func (c *Context) FlagNames() []string {
	var names []string
	for _, ctx := range c.Lineage() {
		ctx.flagSet.Visit(makeFlagNameVisitor(&names))
	}
	return names
}

// Lineage returns *this* context and all of its ancestor contexts in order from
// child to parent
func (c *Context) Lineage() []*Context {
	var lineage []*Context

	for cur := c; cur != nil; cur = cur.parentContext {
		lineage = append(lineage, cur)
	}

	return lineage
}

// Value returns the value of the flag corresponding to `name`
func (c *Context) Value(name string) interface{} {
	return c.flagSet.Lookup(name).Value.(flag.Getter).Get()
}

// Args returns the command line arguments associated with the context.
func (c *Context) Args() Args {
	ret := args(c.flagSet.Args())
	return &ret
}

// NArg returns the number of the command line arguments.
func (c *Context) NArg() int {
	return c.Args().Len()
}

func lookupFlag(name string, ctx *Context) Flag {
	for _, c := range ctx.Lineage() {
		if c.Command == nil {
			continue
		}

		for _, f := range c.Command.Flags {
			for _, n := range f.Names() {
				if n == name {
					return f
				}
			}
		}
	}

	if ctx.App != nil {
		for _, f := range ctx.App.Flags {
			for _, n := range f.Names() {
				if n == name {
					return f
				}
			}
		}
	}

	return nil
}

func lookupFlagSet(name string, ctx *Context) *flag.FlagSet {
	for _, c := range ctx.Lineage() {
		if f := c.flagSet.Lookup(name); f != nil {
			return c.flagSet
		}
	}

	return nil
}

func copyFlag(name string, ff *flag.Flag, set *flag.FlagSet) {
	switch ff.Value.(type) {
	case Serializer:
		_ = set.Set(name, ff.Value.(Serializer).Serialize())
	default:
		_ = set.Set(name, ff.Value.String())
	}
}

func normalizeFlags(flags []Flag, set *flag.FlagSet) error {
	visited := make(map[string]bool)
	set.Visit(func(f *flag.Flag) {
		visited[f.Name] = true
	})
	for _, f := range flags {
		parts := f.Names()
		if len(parts) == 1 {
			continue
		}
		var ff *flag.Flag
		for _, name := range parts {
			name = strings.Trim(name, " ")
			if visited[name] {
				if ff != nil {
					return errors.New("Cannot use two forms of the same flag: " + name + " " + ff.Name)
				}
				ff = set.Lookup(name)
			}
		}
		if ff == nil {
			continue
		}
		for _, name := range parts {
			name = strings.Trim(name, " ")
			if !visited[name] {
				copyFlag(name, ff, set)
			}
		}
	}
	return nil
}

func makeFlagNameVisitor(names *[]string) func(*flag.Flag) {
	return func(f *flag.Flag) {
		nameParts := strings.Split(f.Name, ",")
		name := strings.TrimSpace(nameParts[0])

		for _, part := range nameParts {
			part = strings.TrimSpace(part)
			if len(part) > len(name) {
				name = part
			}
		}

		if name != "" {
			*names = append(*names, name)
		}
	}
}

type requiredFlagsErr interface {
	error
	getMissingFlags() []string
}

type errRequiredFlags struct {
	missingFlags []string
}

func (e *errRequiredFlags) Error() string {
	numberOfMissingFlags := len(e.missingFlags)
	if numberOfMissingFlags == 1 {
		return fmt.Sprintf("Required flag %q not set", e.missingFlags[0])
	}
	joinedMissingFlags := strings.Join(e.missingFlags, ", ")
	return fmt.Sprintf("Required flags %q not set", joinedMissingFlags)
}

func (e *errRequiredFlags) getMissingFlags() []string {
	return e.missingFlags
}

func checkRequiredFlags(flags []Flag, context *Context) requiredFlagsErr {
	var missingFlags []string
	for _, f := range flags {
		if rf, ok := f.(RequiredFlag); ok && rf.IsRequired() {
			var flagPresent bool
			var flagName string

			for _, key := range f.Names() {
				if len(key) > 1 {
					flagName = key
				}

				if context.IsSet(strings.TrimSpace(key)) {
					flagPresent = true
				}
			}

			if !flagPresent && flagName != "" {
				missingFlags = append(missingFlags, flagName)
			}
		}
	}

	if len(missingFlags) != 0 {
		return &errRequiredFlags{missingFlags: missingFlags}
	}

	return nil
}
<commit_msg>revert back to correct formatting<commit_after>package cli

import (
	"context"
	"errors"
	"flag"
	"fmt"
	"strings"
)

// Context is a type that is passed through to
// each Handler action in a cli application. Context
// can be used to retrieve context-specific args and
// parsed command-line options.
type Context struct {
	context.Context
	App           *App
	Command       *Command
	shellComplete bool
	flagSet       *flag.FlagSet
	parentContext *Context
}

// NewContext creates a new context. For use in when invoking an App or Command action.
func NewContext(app *App, set *flag.FlagSet, parentCtx *Context) *Context {
	c := &Context{App: app, flagSet: set, parentContext: parentCtx}
	if parentCtx != nil {
		c.Context = parentCtx.Context
		c.shellComplete = parentCtx.shellComplete
		if parentCtx.flagSet == nil {
			parentCtx.flagSet = &flag.FlagSet{}
		}
	}

	c.Command = &Command{}

	if c.Context == nil {
		c.Context = context.Background()
	}

	return c
}

// NumFlags returns the number of flags set
func (c *Context) NumFlags() int {
	return c.flagSet.NFlag()
}

// Set sets a context flag to a value.
func (c *Context) Set(name, value string) error {
	return c.flagSet.Set(name, value)
}

// IsSet determines if the flag was actually set
func (c *Context) IsSet(name string) bool {
	if fs := lookupFlagSet(name, c); fs != nil {
		if fs := lookupFlagSet(name, c); fs != nil {
			isSet := false
			fs.Visit(func(f *flag.Flag) {
				if f.Name == name {
					isSet = true
				}
			})
			if isSet {
				return true
			}
		}

		f := lookupFlag(name, c)
		if f == nil {
			return false
		}

		return f.IsSet()
	}

	return false
}

// LocalFlagNames returns a slice of flag names used in this context.
func (c *Context) LocalFlagNames() []string {
	var names []string
	c.flagSet.Visit(makeFlagNameVisitor(&names))
	return names
}

// FlagNames returns a slice of flag names used by the this context and all of
// its parent contexts.
func (c *Context) FlagNames() []string {
	var names []string
	for _, ctx := range c.Lineage() {
		ctx.flagSet.Visit(makeFlagNameVisitor(&names))
	}
	return names
}

// Lineage returns *this* context and all of its ancestor contexts in order from
// child to parent
func (c *Context) Lineage() []*Context {
	var lineage []*Context

	for cur := c; cur != nil; cur = cur.parentContext {
		lineage = append(lineage, cur)
	}

	return lineage
}

// Value returns the value of the flag corresponding to `name`
func (c *Context) Value(name string) interface{} {
	return c.flagSet.Lookup(name).Value.(flag.Getter).Get()
}

// Args returns the command line arguments associated with the context.
func (c *Context) Args() Args {
	ret := args(c.flagSet.Args())
	return &ret
}

// NArg returns the number of the command line arguments.
func (c *Context) NArg() int {
	return c.Args().Len()
}

func lookupFlag(name string, ctx *Context) Flag {
	for _, c := range ctx.Lineage() {
		if c.Command == nil {
			continue
		}

		for _, f := range c.Command.Flags {
			for _, n := range f.Names() {
				if n == name {
					return f
				}
			}
		}
	}

	if ctx.App != nil {
		for _, f := range ctx.App.Flags {
			for _, n := range f.Names() {
				if n == name {
					return f
				}
			}
		}
	}

	return nil
}

func lookupFlagSet(name string, ctx *Context) *flag.FlagSet {
	for _, c := range ctx.Lineage() {
		if f := c.flagSet.Lookup(name); f != nil {
			return c.flagSet
		}
	}

	return nil
}

func copyFlag(name string, ff *flag.Flag, set *flag.FlagSet) {
	switch ff.Value.(type) {
	case Serializer:
		_ = set.Set(name, ff.Value.(Serializer).Serialize())
	default:
		_ = set.Set(name, ff.Value.String())
	}
}

func normalizeFlags(flags []Flag, set *flag.FlagSet) error {
	visited := make(map[string]bool)
	set.Visit(func(f *flag.Flag) {
		visited[f.Name] = true
	})
	for _, f := range flags {
		parts := f.Names()
		if len(parts) == 1 {
			continue
		}
		var ff *flag.Flag
		for _, name := range parts {
			name = strings.Trim(name, " ")
			if visited[name] {
				if ff != nil {
					return errors.New("Cannot use two forms of the same flag: " + name + " " + ff.Name)
				}
				ff = set.Lookup(name)
			}
		}
		if ff == nil {
			continue
		}
		for _, name := range parts {
			name = strings.Trim(name, " ")
			if !visited[name] {
				copyFlag(name, ff, set)
			}
		}
	}
	return nil
}

func makeFlagNameVisitor(names *[]string) func(*flag.Flag) {
	return func(f *flag.Flag) {
		nameParts := strings.Split(f.Name, ",")
		name := strings.TrimSpace(nameParts[0])

		for _, part := range nameParts {
			part = strings.TrimSpace(part)
			if len(part) > len(name) {
				name = part
			}
		}

		if name != "" {
			*names = append(*names, name)
		}
	}
}

type requiredFlagsErr interface {
	error
	getMissingFlags() []string
}

type errRequiredFlags struct {
	missingFlags []string
}

func (e *errRequiredFlags) Error() string {
	numberOfMissingFlags := len(e.missingFlags)
	if numberOfMissingFlags == 1 {
		return fmt.Sprintf("Required flag %q not set", e.missingFlags[0])
	}
	joinedMissingFlags := strings.Join(e.missingFlags, ", ")
	return fmt.Sprintf("Required flags %q not set", joinedMissingFlags)
}

func (e *errRequiredFlags) getMissingFlags() []string {
	return e.missingFlags
}

func checkRequiredFlags(flags []Flag, context *Context) requiredFlagsErr {
	var missingFlags []string
	for _, f := range flags {
		if rf, ok := f.(RequiredFlag); ok && rf.IsRequired() {
			var flagPresent bool
			var flagName string

			for _, key := range f.Names() {
				if len(key) > 1 {
					flagName = key
				}

				if context.IsSet(strings.TrimSpace(key)) {
					flagPresent = true
				}
			}

			if !flagPresent && flagName != "" {
				missingFlags = append(missingFlags, flagName)
			}
		}
	}

	if len(missingFlags) != 0 {
		return &errRequiredFlags{missingFlags: missingFlags}
	}

	return nil
}
<|endoftext|>


<commit_before>package main

import (
	"fmt"
	"net/http"
)

type Context struct {
	*Injector
	handlers      []Handler
	routerHandler Handler
	index         int
}

func (c *Context) Next() {
	c.index += 1
	c.Run()
}

func (c *Context) Run() {
	// Run all Duck handles
	for c.index < len(c.handlers) {

		if ret, err := c.Invoke(c.handlers[c.index]); err == nil {
			if len(ret) != 0 {
				rv := c.Get(c.GetType((*http.ResponseWriter)(nil)))
				res := rv.Interface().(http.ResponseWriter)
				fmt.Fprintf(res, ret[0].Interface().(string))
			}
		} else {
			fmt.Println(err)
		}
		c.index++
	}
	// Run all route handles
	if c.index == len(c.handlers) {
		c.Invoke(c.routerHandler)
	}
}

type RouterContext struct {
	*Injector
	handlers []Handler // all route handler
	index    int
}

func (rc *RouterContext) Run() {
	for i := 0; i < len(rc.handlers); i++ {
		if ret, err := rc.Invoke(rc.handlers[i]); err == nil {
			if len(ret) != 0 {
				rv := rc.Get(rc.GetType((*http.ResponseWriter)(nil)))
				res := rv.Interface().(http.ResponseWriter)
				fmt.Fprintf(res, ret[0].Interface().(string))
			}
		} else {
			fmt.Println(err)
		}
	}
}
<commit_msg>Add return handler<commit_after>package main

import (
	"fmt"
	"net/http"
	"reflect"
)

type Context struct {
	*Injector
	handlers      []Handler
	routerHandler Handler
	index         int
}

func (c *Context) Next() {
	c.index += 1
	c.Run()
}

func (c *Context) Run() {
	// Run all Duck handles
	for c.index < len(c.handlers) {
		if val, err := c.Invoke(c.handlers[c.index]); err != nil {
			ReturnHandler(val, c)
		}
		c.index++
	}
	// Run all route handles
	if c.index == len(c.handlers) {
		c.Invoke(c.routerHandler)
	}
}

type RouterContext struct {
	*Injector
	handlers []Handler // all route handler
	index    int
}

type Contexter interface {
	GetType(interface{}) reflect.Type
	Get(reflect.Type) reflect.Value
}

func (rc *RouterContext) Run() {
	for i := 0; i < len(rc.handlers); i++ {
		if val, err := rc.Invoke(rc.handlers[i]); err == nil {
			ReturnHandler(val, rc)
		} else {
			fmt.Println(err)
		}
	}
}

func ReturnHandler(val []reflect.Value, c Contexter) {
	rv := c.Get(c.GetType((*http.ResponseWriter)(nil)))
	res := rv.Interface().(http.ResponseWriter)
	if len(val) == 1 {
		fmt.Fprintf(res, val[0].Interface().(string))
	} else if len(val) == 2 {
		// status code
		res.WriteHeader(int(val[0].Int()))
		fmt.Fprintf(res, val[1].Interface().(string))
	}
}
<|endoftext|>


<commit_before>package cssminify

func Files() []Cssfile {
	return findFiles()
}

func findFiles() []Cssfile {
	return make([]Cssfile, 1)
}

type Cssfile struct {
	pathname string
	content string
}
<commit_msg>Walking files and getting content is done.<commit_after>package cssminify

import (
	"path/filepath"
	"os"
	"regexp"
	"io/ioutil"
)

func Files() []Cssfile {
	return findFiles()
}

func findFiles() []Cssfile {
	files := make([]Cssfile, 0)
	filepath.Walk(".",
		func (root string, info os.FileInfo, err error) error {

		matched, _ := regexp.MatchString(".css$", root)
		if matched {
			file := Cssfile{pathname: root, content: readFile(root)}
			files = append(files, file)
		}
		return err
	})
	return files
}

func readFile(root string) string {
	content, err := ioutil.ReadFile(root)
	if err != nil {
		panic(err)
	}
	return string(content)
}

type Cssfile struct {
	pathname string
	content string
}
<|endoftext|>


<commit_before>package cli

import (
	"errors"
	"flag"
	"fmt"
	"os"
	"reflect"
	"strings"
	"syscall"
)

// Context is a type that is passed through to
// each Handler action in a cli application. Context
// can be used to retrieve context-specific Args and
// parsed command-line options.
type Context struct {
	App           *App
	Command       Command
	shellComplete bool
	flagSet       *flag.FlagSet
	setFlags      map[string]bool
	parentContext *Context
}

// NewContext creates a new context. For use in when invoking an App or Command action.
func NewContext(app *App, set *flag.FlagSet, parentCtx *Context) *Context {
	c := &Context{App: app, flagSet: set, parentContext: parentCtx}

	if parentCtx != nil {
		c.shellComplete = parentCtx.shellComplete
	}

	return c
}

// NumFlags returns the number of flags set
func (c *Context) NumFlags() int {
	return c.flagSet.NFlag()
}

// Set sets a context flag to a value.
func (c *Context) Set(name, value string) error {
	c.setFlags = nil
	return c.flagSet.Set(name, value)
}

// GlobalSet sets a context flag to a value on the global flagset
func (c *Context) GlobalSet(name, value string) error {
	globalContext(c).setFlags = nil
	return globalContext(c).flagSet.Set(name, value)
}

// IsSet determines if the flag was actually set
func (c *Context) IsSet(name string) bool {
	if c.setFlags == nil {
		c.setFlags = make(map[string]bool)

		c.flagSet.Visit(func(f *flag.Flag) {
			c.setFlags[f.Name] = true
		})

		c.flagSet.VisitAll(func(f *flag.Flag) {
			if _, ok := c.setFlags[f.Name]; ok {
				return
			}
			c.setFlags[f.Name] = false
		})

		// XXX hack to support IsSet for flags with EnvVar
		//
		// There isn't an easy way to do this with the current implementation since
		// whether a flag was set via an environment variable is very difficult to
		// determine here. Instead, we intend to introduce a backwards incompatible
		// change in version 2 to add `IsSet` to the Flag interface to push the
		// responsibility closer to where the information required to determine
		// whether a flag is set by non-standard means such as environment
		// variables is available.
		//
		// See https://github.com/urfave/cli/issues/294 for additional discussion
		flags := c.Command.Flags
		if c.Command.Name == "" { // cannot == Command{} since it contains slice types
			if c.App != nil {
				flags = c.App.Flags
			}
		}
		for _, f := range flags {
			eachName(f.GetName(), func(name string) {
				if isSet, ok := c.setFlags[name]; isSet || !ok {
					return
				}

				val := reflect.ValueOf(f)
				if val.Kind() == reflect.Ptr {
					val = val.Elem()
				}

				filePathValue := val.FieldByName("FilePath")
				if filePathValue.IsValid() {
					eachName(filePathValue.String(), func(filePath string) {
						if _, err := os.Stat(filePath); err == nil {
							c.setFlags[name] = true
							return
						}
					})
				}

				envVarValue := val.FieldByName("EnvVar")
				if envVarValue.IsValid() {
					eachName(envVarValue.String(), func(envVar string) {
						envVar = strings.TrimSpace(envVar)
						if _, ok := syscall.Getenv(envVar); ok {
							c.setFlags[name] = true
							return
						}
					})
				}
			})
		}
	}

	return c.setFlags[name]
}

// GlobalIsSet determines if the global flag was actually set
func (c *Context) GlobalIsSet(name string) bool {
	ctx := c
	if ctx.parentContext != nil {
		ctx = ctx.parentContext
	}

	for ; ctx != nil; ctx = ctx.parentContext {
		if ctx.IsSet(name) {
			return true
		}
	}
	return false
}

// FlagNames returns a slice of flag names used in this context.
func (c *Context) FlagNames() (names []string) {
	for _, flag := range c.Command.Flags {
		name := strings.Split(flag.GetName(), ",")[0]
		if name == "help" {
			continue
		}
		names = append(names, name)
	}
	return
}

// GlobalFlagNames returns a slice of global flag names used by the app.
func (c *Context) GlobalFlagNames() (names []string) {
	for _, flag := range c.App.Flags {
		name := strings.Split(flag.GetName(), ",")[0]
		if name == "help" || name == "version" {
			continue
		}
		names = append(names, name)
	}
	return
}

// Parent returns the parent context, if any
func (c *Context) Parent() *Context {
	return c.parentContext
}

// value returns the value of the flag coressponding to `name`
func (c *Context) value(name string) interface{} {
	return c.flagSet.Lookup(name).Value.(flag.Getter).Get()
}

// Args contains apps console arguments
type Args []string

// Args returns the command line arguments associated with the context.
func (c *Context) Args() Args {
	args := Args(c.flagSet.Args())
	return args
}

// NArg returns the number of the command line arguments.
func (c *Context) NArg() int {
	return len(c.Args())
}

// Get returns the nth argument, or else a blank string
func (a Args) Get(n int) string {
	if len(a) > n {
		return a[n]
	}
	return ""
}

// First returns the first argument, or else a blank string
func (a Args) First() string {
	return a.Get(0)
}

// Tail returns the rest of the arguments (not the first one)
// or else an empty string slice
func (a Args) Tail() []string {
	if len(a) >= 2 {
		return []string(a)[1:]
	}
	return []string{}
}

// Present checks if there are any arguments present
func (a Args) Present() bool {
	return len(a) != 0
}

// Swap swaps arguments at the given indexes
func (a Args) Swap(from, to int) error {
	if from >= len(a) || to >= len(a) {
		return errors.New("index out of range")
	}
	a[from], a[to] = a[to], a[from]
	return nil
}

func globalContext(ctx *Context) *Context {
	if ctx == nil {
		return nil
	}

	for {
		if ctx.parentContext == nil {
			return ctx
		}
		ctx = ctx.parentContext
	}
}

func lookupGlobalFlagSet(name string, ctx *Context) *flag.FlagSet {
	if ctx.parentContext != nil {
		ctx = ctx.parentContext
	}
	for ; ctx != nil; ctx = ctx.parentContext {
		if f := ctx.flagSet.Lookup(name); f != nil {
			return ctx.flagSet
		}
	}
	return nil
}

func copyFlag(name string, ff *flag.Flag, set *flag.FlagSet) {
	switch ff.Value.(type) {
	case *StringSlice:
	default:
		set.Set(name, ff.Value.String())
	}
}

func normalizeFlags(flags []Flag, set *flag.FlagSet) error {
	visited := make(map[string]bool)
	set.Visit(func(f *flag.Flag) {
		visited[f.Name] = true
	})
	for _, f := range flags {
		parts := strings.Split(f.GetName(), ",")
		if len(parts) == 1 {
			continue
		}
		var ff *flag.Flag
		for _, name := range parts {
			name = strings.Trim(name, " ")
			if visited[name] {
				if ff != nil {
					return errors.New("Cannot use two forms of the same flag: " + name + " " + ff.Name)
				}
				ff = set.Lookup(name)
			}
		}
		if ff == nil {
			continue
		}
		for _, name := range parts {
			name = strings.Trim(name, " ")
			if !visited[name] {
				copyFlag(name, ff, set)
			}
		}
	}
	return nil
}

func checkRequiredFlags(flags []Flag, set *flag.FlagSet) error {
	visited := make(map[string]bool)
	set.Visit(func(f *flag.Flag) {
		visited[f.Name] = true
	})

	var missingFlags []string
	for _, f := range flags {
		if rf, ok := f.(RequiredFlag); ok && rf.IsRequired() {
			key := strings.Split(f.GetName(), ",")[0]
			if !visited[key] {
				missingFlags = append(missingFlags, f.GetName())
			}
		}
	}

	numberOfMissingFlags := len(missingFlags)
	if numberOfMissingFlags == 1 {
		return fmt.Errorf("Required flag %q not set", missingFlags[0])
	}
	if numberOfMissingFlags >= 2 {
		var missingFlagsOutput string
		for idx, f := range missingFlags {
			// if not the last item, append with a ", "
			if idx != numberOfMissingFlags-1 {
				missingFlagsOutput = fmt.Sprintf("%s%s, ", missingFlagsOutput, f)
			} else {
				missingFlagsOutput = fmt.Sprintf("%s%s", missingFlagsOutput, f)
			}
		}
		return fmt.Errorf("Required flags %q not set", missingFlagsOutput)
	}

	return nil
}
<commit_msg>use strings.Join<commit_after>package cli

import (
	"errors"
	"flag"
	"fmt"
	"os"
	"reflect"
	"strings"
	"syscall"
)

// Context is a type that is passed through to
// each Handler action in a cli application. Context
// can be used to retrieve context-specific Args and
// parsed command-line options.
type Context struct {
	App           *App
	Command       Command
	shellComplete bool
	flagSet       *flag.FlagSet
	setFlags      map[string]bool
	parentContext *Context
}

// NewContext creates a new context. For use in when invoking an App or Command action.
func NewContext(app *App, set *flag.FlagSet, parentCtx *Context) *Context {
	c := &Context{App: app, flagSet: set, parentContext: parentCtx}

	if parentCtx != nil {
		c.shellComplete = parentCtx.shellComplete
	}

	return c
}

// NumFlags returns the number of flags set
func (c *Context) NumFlags() int {
	return c.flagSet.NFlag()
}

// Set sets a context flag to a value.
func (c *Context) Set(name, value string) error {
	c.setFlags = nil
	return c.flagSet.Set(name, value)
}

// GlobalSet sets a context flag to a value on the global flagset
func (c *Context) GlobalSet(name, value string) error {
	globalContext(c).setFlags = nil
	return globalContext(c).flagSet.Set(name, value)
}

// IsSet determines if the flag was actually set
func (c *Context) IsSet(name string) bool {
	if c.setFlags == nil {
		c.setFlags = make(map[string]bool)

		c.flagSet.Visit(func(f *flag.Flag) {
			c.setFlags[f.Name] = true
		})

		c.flagSet.VisitAll(func(f *flag.Flag) {
			if _, ok := c.setFlags[f.Name]; ok {
				return
			}
			c.setFlags[f.Name] = false
		})

		// XXX hack to support IsSet for flags with EnvVar
		//
		// There isn't an easy way to do this with the current implementation since
		// whether a flag was set via an environment variable is very difficult to
		// determine here. Instead, we intend to introduce a backwards incompatible
		// change in version 2 to add `IsSet` to the Flag interface to push the
		// responsibility closer to where the information required to determine
		// whether a flag is set by non-standard means such as environment
		// variables is available.
		//
		// See https://github.com/urfave/cli/issues/294 for additional discussion
		flags := c.Command.Flags
		if c.Command.Name == "" { // cannot == Command{} since it contains slice types
			if c.App != nil {
				flags = c.App.Flags
			}
		}
		for _, f := range flags {
			eachName(f.GetName(), func(name string) {
				if isSet, ok := c.setFlags[name]; isSet || !ok {
					return
				}

				val := reflect.ValueOf(f)
				if val.Kind() == reflect.Ptr {
					val = val.Elem()
				}

				filePathValue := val.FieldByName("FilePath")
				if filePathValue.IsValid() {
					eachName(filePathValue.String(), func(filePath string) {
						if _, err := os.Stat(filePath); err == nil {
							c.setFlags[name] = true
							return
						}
					})
				}

				envVarValue := val.FieldByName("EnvVar")
				if envVarValue.IsValid() {
					eachName(envVarValue.String(), func(envVar string) {
						envVar = strings.TrimSpace(envVar)
						if _, ok := syscall.Getenv(envVar); ok {
							c.setFlags[name] = true
							return
						}
					})
				}
			})
		}
	}

	return c.setFlags[name]
}

// GlobalIsSet determines if the global flag was actually set
func (c *Context) GlobalIsSet(name string) bool {
	ctx := c
	if ctx.parentContext != nil {
		ctx = ctx.parentContext
	}

	for ; ctx != nil; ctx = ctx.parentContext {
		if ctx.IsSet(name) {
			return true
		}
	}
	return false
}

// FlagNames returns a slice of flag names used in this context.
func (c *Context) FlagNames() (names []string) {
	for _, flag := range c.Command.Flags {
		name := strings.Split(flag.GetName(), ",")[0]
		if name == "help" {
			continue
		}
		names = append(names, name)
	}
	return
}

// GlobalFlagNames returns a slice of global flag names used by the app.
func (c *Context) GlobalFlagNames() (names []string) {
	for _, flag := range c.App.Flags {
		name := strings.Split(flag.GetName(), ",")[0]
		if name == "help" || name == "version" {
			continue
		}
		names = append(names, name)
	}
	return
}

// Parent returns the parent context, if any
func (c *Context) Parent() *Context {
	return c.parentContext
}

// value returns the value of the flag coressponding to `name`
func (c *Context) value(name string) interface{} {
	return c.flagSet.Lookup(name).Value.(flag.Getter).Get()
}

// Args contains apps console arguments
type Args []string

// Args returns the command line arguments associated with the context.
func (c *Context) Args() Args {
	args := Args(c.flagSet.Args())
	return args
}

// NArg returns the number of the command line arguments.
func (c *Context) NArg() int {
	return len(c.Args())
}

// Get returns the nth argument, or else a blank string
func (a Args) Get(n int) string {
	if len(a) > n {
		return a[n]
	}
	return ""
}

// First returns the first argument, or else a blank string
func (a Args) First() string {
	return a.Get(0)
}

// Tail returns the rest of the arguments (not the first one)
// or else an empty string slice
func (a Args) Tail() []string {
	if len(a) >= 2 {
		return []string(a)[1:]
	}
	return []string{}
}

// Present checks if there are any arguments present
func (a Args) Present() bool {
	return len(a) != 0
}

// Swap swaps arguments at the given indexes
func (a Args) Swap(from, to int) error {
	if from >= len(a) || to >= len(a) {
		return errors.New("index out of range")
	}
	a[from], a[to] = a[to], a[from]
	return nil
}

func globalContext(ctx *Context) *Context {
	if ctx == nil {
		return nil
	}

	for {
		if ctx.parentContext == nil {
			return ctx
		}
		ctx = ctx.parentContext
	}
}

func lookupGlobalFlagSet(name string, ctx *Context) *flag.FlagSet {
	if ctx.parentContext != nil {
		ctx = ctx.parentContext
	}
	for ; ctx != nil; ctx = ctx.parentContext {
		if f := ctx.flagSet.Lookup(name); f != nil {
			return ctx.flagSet
		}
	}
	return nil
}

func copyFlag(name string, ff *flag.Flag, set *flag.FlagSet) {
	switch ff.Value.(type) {
	case *StringSlice:
	default:
		set.Set(name, ff.Value.String())
	}
}

func normalizeFlags(flags []Flag, set *flag.FlagSet) error {
	visited := make(map[string]bool)
	set.Visit(func(f *flag.Flag) {
		visited[f.Name] = true
	})
	for _, f := range flags {
		parts := strings.Split(f.GetName(), ",")
		if len(parts) == 1 {
			continue
		}
		var ff *flag.Flag
		for _, name := range parts {
			name = strings.Trim(name, " ")
			if visited[name] {
				if ff != nil {
					return errors.New("Cannot use two forms of the same flag: " + name + " " + ff.Name)
				}
				ff = set.Lookup(name)
			}
		}
		if ff == nil {
			continue
		}
		for _, name := range parts {
			name = strings.Trim(name, " ")
			if !visited[name] {
				copyFlag(name, ff, set)
			}
		}
	}
	return nil
}

func checkRequiredFlags(flags []Flag, set *flag.FlagSet) error {
	visited := make(map[string]bool)
	set.Visit(func(f *flag.Flag) {
		visited[f.Name] = true
	})

	var missingFlags []string
	for _, f := range flags {
		if rf, ok := f.(RequiredFlag); ok && rf.IsRequired() {
			key := strings.Split(f.GetName(), ",")[0]
			if !visited[key] {
				missingFlags = append(missingFlags, f.GetName())
			}
		}
	}

	numberOfMissingFlags := len(missingFlags)
	if numberOfMissingFlags == 1 {
		return fmt.Errorf("Required flag %q not set", missingFlags[0])
	}
	if numberOfMissingFlags >= 2 {
		joinedMissingFlags := strings.Join(missingFlags, ", ")
		return fmt.Errorf("Required flags %q not set", joinedMissingFlags)
	}

	return nil
}
<|endoftext|>


<commit_before>package chuper

import (
	"net/url"

	"github.com/PuerkitoBio/fetchbot"
)

type Context struct {
	*fetchbot.Context
	C Cache
}

func (c *Context) Cache() Cache {
	return c.C
}

func (c *Context) Queue() *fetchbot.Queue {
	return c.Q
}

func (c *Context) URL() *url.URL {
	return c.Cmd.URL()
}

func (c *Context) SourceURL() *url.URL {
	switch cmd := c.Cmd.(type) {
	case Cmd:
		return cmd.SourceURL()
	default:
		return nil
	}
}
<commit_msg>Add SourceUrl to Context when it's a command with BasicAuth.<commit_after>package chuper

import (
	"net/url"

	"github.com/PuerkitoBio/fetchbot"
)

type Context struct {
	*fetchbot.Context
	C Cache
}

func (c *Context) Cache() Cache {
	return c.C
}

func (c *Context) Queue() *fetchbot.Queue {
	return c.Q
}

func (c *Context) URL() *url.URL {
	return c.Cmd.URL()
}

func (c *Context) SourceURL() *url.URL {
	switch cmd := c.Cmd.(type) {
	case Cmd:
		return cmd.SourceURL()
	case CmdBasicAuth:
		return cmd.SourceURL()
	default:
		return nil
	}
}
<|endoftext|>


<commit_before>//
// Variable frequency CPU usage sampling on Linux via /proc
// Maybe this will turn into something like prstat on Solaris
//

// TOOD - check for rollover
// TODO - random large values sometimes, perhaps when we see a pid for the first time
// TODO - pull in /proc/stat
// TODO - pull in /proc/schedstat
// TODO - check max field length for assumed present fields

package main

import (
	"flag"
	"fmt"
	"log"
	"os"
	"os/signal"
	"runtime/pprof"
	"sort"
	"time"
)

const histMin = 0
const histMax = 100000000
const histSigFigs = 2

func main() {
	var interval = flag.Int("i", 1000, "interval (ms) between measurements")
	var samples = flag.Int("s", 60, "sample counts to aggregate for output")
	var topN = flag.Int("n", 10, "show top N processes")
	var cpuprofile = flag.String("cpuprofile", "", "write cpu profile to file")
	var jiffy = flag.Int("jiffy", 100, "length of a jiffy")

	flag.Parse()

	if *cpuprofile != "" {
		f, err := os.Create(*cpuprofile)
		if err != nil {
			log.Fatal(err)
		}
		if err = pprof.StartCPUProfile(f); err != nil {
			log.Fatal(err)
		}
	}

	sigChan := make(chan os.Signal, 1)
	signal.Notify(sigChan, os.Interrupt)

	go func() {
		_ = <-sigChan
		pprof.StopCPUProfile()
		fmt.Fprintln(os.Stderr, "SIGINT, probably")
		os.Exit(0)
	}()

	procCur := make(procStatsMap)
	procPrev := make(procStatsMap)
	procSum := make(procStatsMap)
	procHist := make(procStatsHistMap)

	schedCur := make(schedStatsMap)
	schedPrev := make(schedStatsMap)
	schedSum := make(schedStatsMap)

	var sysCur *systemStats
	var sysPrev *systemStats
	var sysSum *systemStats
	var sysHist *systemStatsHist

	// run all scans one time to establish a baseline
	pids := getPidList()
	procPrev = statReader(pids)
	sysPrev = statReaderGlobal()
	schedPrev = schedReaderPids(pids)
	sysSum = &systemStats{}
	sysHist = &systemStatsHist{}

	for {
		for count := 0; count < *samples; count++ {
			time.Sleep(time.Duration(*interval) * time.Millisecond)

			pids = getPidList()

			procCur = statReader(pids)
			statRecord(procCur, procPrev, procSum, procHist)
			procPrev = procCur

			sysCur = statReaderGlobal()
			statsRecordGlobal(sysCur, sysPrev, sysSum, sysHist)
			sysPrev = sysCur
		}

		schedCur = schedReaderPids(pids)
		schedRecord(schedCur, schedPrev, schedSum)
		schedPrev = schedCur

		dumpStats(procSum, procHist, sysSum, sysHist, schedSum, topN, jiffy, interval, samples)
		procHist = make(procStatsHistMap)
		procSum = make(procStatsMap)
		sysHist = &systemStatsHist{}
		sysSum = &systemStats{}
	}
}

type bar struct {
	baz int
}

// Wrapper to sort histograms by max but remember which pid they are
type sortHist struct {
	pid  int
	hist *procStatsHist
}

// ByMax sorts stats by max usage
type ByMax []*sortHist

func (m ByMax) Len() int {
	return len(m)
}
func (m ByMax) Swap(i, j int) {
	m[i], m[j] = m[j], m[i]
}
func (m ByMax) Less(i, j int) bool {
	maxI := maxList([]float64{
		m[i].hist.ustime.Mean(),
		m[i].hist.custime.Mean(),
		m[i].hist.delayacctBlkioTicks.Mean(),
	})
	maxJ := maxList([]float64{
		m[j].hist.ustime.Mean(),
		m[j].hist.custime.Mean(),
		m[j].hist.delayacctBlkioTicks.Mean(),
	})
	return maxI > maxJ
}
func maxList(list []float64) float64 {
	ret := list[0]
	for i := 1; i < len(list); i++ {
		if list[i] > ret {
			ret = list[i]
		}
	}
	return ret
}

func formatMem(num uint64) string {
	letter := string("K")

	num = num * 4
	if num >= 10000 {
		num = (num + 512) / 1024
		letter = "M"
		if num >= 10000 {
			num = (num + 512) / 1024
			letter = "G"
		}
	}
	return fmt.Sprintf("%d%s", num, letter)
}

func dumpStats(sumStats procStatsMap, histStats procStatsHistMap, sysSum *systemStats, sysHist *systemStatsHist,
	sumSched schedStatsMap, topN, jiffy, interval, samples *int) {

	scale := func(val float64) float64 {
		return val / float64(*jiffy) / float64(*interval) * 1000 * 100
	}
	scaleSum := func(val float64, count int64) float64 {
		valSec := val / float64(*jiffy)
		sampleSec := float64(*interval) * float64(count) / 1000.0
		ret := (valSec / sampleSec) * 100
		return ret
	}
	scaleSched := func(val float64) float64 {
		return val / float64(*jiffy) / float64((*interval)*(*samples)) * 100
	}

	var list []*sortHist

	for pid, hist := range histStats {
		list = append(list, &sortHist{pid, hist})
	}
	sort.Sort(ByMax(list))
	if len(list) > *topN {
		list = list[:*topN]
	}
	fmt.Printf("usr:    %4s/%4s/%4s   sys:%4s/%4s/%4s  nice:%4s/%4s/%4s    idle:%4s/%4s/%4s\n",
		trim(scale(float64(sysHist.usr.Min())), 4),
		trim(scale(float64(sysHist.usr.Max())), 4),
		trim(scale(sysHist.usr.Mean()), 4),

		trim(scale(float64(sysHist.sys.Min())), 4),
		trim(scale(float64(sysHist.sys.Max())), 4),
		trim(scale(sysHist.sys.Mean()), 4),

		trim(scale(float64(sysHist.nice.Min())), 4),
		trim(scale(float64(sysHist.nice.Max())), 4),
		trim(scale(sysHist.nice.Mean()), 4),

		trim(scale(float64(sysHist.idle.Min())), 4),
		trim(scale(float64(sysHist.idle.Max())), 4),
		trim(scale(sysHist.idle.Mean()), 4),
	)
	fmt.Printf("iowait: %4s/%4s/%4s  ctxt:%4s/%4s/%4s  prun:%4s/%4s/%4s  pblock:%4s/%4s/%4s  pstart: %4d\n",
		trim(scale(float64(sysHist.iowait.Min())), 4),
		trim(scale(float64(sysHist.iowait.Max())), 4),
		trim(scale(sysHist.iowait.Mean()), 4),

		trim(scale(float64(sysHist.ctxt.Min())), 4),
		trim(scale(float64(sysHist.ctxt.Max())), 4),
		trim(scale(sysHist.ctxt.Mean()), 4),

		trim(float64(sysHist.procsRunning.Min()), 4),
		trim(float64(sysHist.procsRunning.Max()), 4),
		trim(sysHist.procsRunning.Mean(), 4),

		trim(float64(sysHist.procsBlocked.Min()), 4),
		trim(float64(sysHist.procsBlocked.Max()), 4),
		trim(sysHist.procsBlocked.Mean(), 4),

		sysSum.procsTotal,
	)

	fmt.Print("                comm     pid     min     max     usr     sys   ctime    slat     ctx     icx     rss  iowait thrd  sam\n")
	for _, thisStat := range list {
		hist := thisStat.hist
		pid := thisStat.pid
		var schedWait, nrSwitches, nrInvoluntarySwitches string
		sched, ok := sumSched[pid]
		if ok == true {
			schedWait = trim(scaleSched(sched.waitSum), 7)
			nrSwitches = fmt.Sprintf("%d", sched.nrSwitches)
			nrInvoluntarySwitches = fmt.Sprintf("%d", sched.nrInvoluntarySwitches)
		} else {
			schedWait = "-"
			nrSwitches = "-"
			nrInvoluntarySwitches = "-"
		}
		sampleCount := hist.utime.TotalCount()
		fmt.Printf("%20s %7d %7s %7s %7s %7s %7s %7s %7s %7s %7s %7s %4d %4d\n",
			sumStats[pid].comm,
			pid,
			trim(scale(float64(hist.ustime.Min())), 7),
			trim(scale(float64(hist.ustime.Max())), 7),
			trim(scaleSum(float64(sumStats[pid].utime), sampleCount), 7),
			trim(scaleSum(float64(sumStats[pid].stime), sampleCount), 7),
			trim(scaleSum(float64(sumStats[pid].cutime+sumStats[pid].cstime), sampleCount), 7),
			schedWait,
			nrSwitches,
			nrInvoluntarySwitches,
			formatMem(sumStats[pid].rss),
			trim(scaleSum(float64(sumStats[pid].delayacctBlkioTicks), sampleCount), 7),
			sumStats[pid].numThreads,
			sampleCount,
		)
	}
	fmt.Println()
}
<commit_msg>Try to calibrate sleep intervals based on work done.<commit_after>//
// Variable frequency CPU usage sampling on Linux via /proc
// Maybe this will turn into something like prstat on Solaris
//

// TOOD - check for rollover
// TODO - random large values sometimes, perhaps when we see a pid for the first time
// TODO - pull in /proc/stat
// TODO - pull in /proc/schedstat
// TODO - check max field length for assumed present fields

package main

import (
	"flag"
	"fmt"
	"log"
	"os"
	"os/signal"
	"runtime/pprof"
	"sort"
	"time"
)

const histMin = 0
const histMax = 100000000
const histSigFigs = 2

func main() {
	var interval = flag.Int("i", 1000, "interval (ms) between measurements")
	var samples = flag.Int("s", 60, "sample counts to aggregate for output")
	var topN = flag.Int("n", 10, "show top N processes")
	var cpuprofile = flag.String("cpuprofile", "", "write cpu profile to file")
	var jiffy = flag.Int("jiffy", 100, "length of a jiffy")

	flag.Parse()

	if *cpuprofile != "" {
		f, err := os.Create(*cpuprofile)
		if err != nil {
			log.Fatal(err)
		}
		if err = pprof.StartCPUProfile(f); err != nil {
			log.Fatal(err)
		}
	}

	sigChan := make(chan os.Signal, 1)
	signal.Notify(sigChan, os.Interrupt)

	go func() {
		_ = <-sigChan
		pprof.StopCPUProfile()
		fmt.Fprintln(os.Stderr, "SIGINT, probably")
		os.Exit(0)
	}()

	procCur := make(procStatsMap)
	procPrev := make(procStatsMap)
	procSum := make(procStatsMap)
	procHist := make(procStatsHistMap)

	schedCur := make(schedStatsMap)
	schedPrev := make(schedStatsMap)
	schedSum := make(schedStatsMap)

	var sysCur *systemStats
	var sysPrev *systemStats
	var sysSum *systemStats
	var sysHist *systemStatsHist

	var t1, t2 time.Time

	// run all scans one time to establish a baseline
	pids := getPidList()

	schedPrev = schedReaderPids(pids)
	t1 = time.Now()
	procPrev = statReader(pids)
	sysPrev = statReaderGlobal()
	sysSum = &systemStats{}
	sysHist = &systemStatsHist{}
	t2 = time.Now()

	targetSleep := time.Duration(*interval) * time.Millisecond
	adjustedSleep := targetSleep - t2.Sub(t1)

	for {
		for count := 0; count < *samples; count++ {
			time.Sleep(adjustedSleep)

			t1 = time.Now()
			pids = getPidList()

			procCur = statReader(pids)
			statRecord(procCur, procPrev, procSum, procHist)
			procPrev = procCur

			sysCur = statReaderGlobal()
			statsRecordGlobal(sysCur, sysPrev, sysSum, sysHist)
			sysPrev = sysCur

			t2 = time.Now()
			adjustedSleep = targetSleep - t2.Sub(t1)
		}

		topHist := sortList(procHist, *topN)
		topPids := make(pidlist, *topN)
		for i := 0; i < *topN; i++ {
			topPids[i] = topHist[i].pid
		}

		schedCur = schedReaderPids(topPids)
		schedRecord(schedCur, schedPrev, schedSum)
		schedPrev = schedCur

		dumpStats(procSum, procHist, sysSum, sysHist, schedSum, topN, jiffy, interval, samples)
		procHist = make(procStatsHistMap)
		procSum = make(procStatsMap)
		sysHist = &systemStatsHist{}
		sysSum = &systemStats{}
		t2 = time.Now()
		adjustedSleep = targetSleep - t2.Sub(t1)
	}
}

type bar struct {
	baz int
}

// Wrapper to sort histograms by max but remember which pid they are
type sortHist struct {
	pid  int
	hist *procStatsHist
}

// ByMax sorts stats by max usage
type ByMax []*sortHist

func (m ByMax) Len() int {
	return len(m)
}
func (m ByMax) Swap(i, j int) {
	m[i], m[j] = m[j], m[i]
}
func (m ByMax) Less(i, j int) bool {
	maxI := maxList([]float64{
		m[i].hist.ustime.Mean(),
		m[i].hist.custime.Mean(),
		m[i].hist.delayacctBlkioTicks.Mean(),
	})
	maxJ := maxList([]float64{
		m[j].hist.ustime.Mean(),
		m[j].hist.custime.Mean(),
		m[j].hist.delayacctBlkioTicks.Mean(),
	})
	return maxI > maxJ
}
func maxList(list []float64) float64 {
	ret := list[0]
	for i := 1; i < len(list); i++ {
		if list[i] > ret {
			ret = list[i]
		}
	}
	return ret
}

func sortList(histStats procStatsHistMap, limit int) []*sortHist {
	var list []*sortHist

	for pid, hist := range histStats {
		list = append(list, &sortHist{pid, hist})
	}
	sort.Sort(ByMax(list))
	if len(list) > limit {
		list = list[:limit]
	}

	return list
}

func formatMem(num uint64) string {
	letter := string("K")

	num = num * 4
	if num >= 10000 {
		num = (num + 512) / 1024
		letter = "M"
		if num >= 10000 {
			num = (num + 512) / 1024
			letter = "G"
		}
	}
	return fmt.Sprintf("%d%s", num, letter)
}

func dumpStats(sumStats procStatsMap, histStats procStatsHistMap, sysSum *systemStats, sysHist *systemStatsHist,
	sumSched schedStatsMap, topN, jiffy, interval, samples *int) {

	scale := func(val float64) float64 {
		return val / float64(*jiffy) / float64(*interval) * 1000 * 100
	}
	scaleSum := func(val float64, count int64) float64 {
		valSec := val / float64(*jiffy)
		sampleSec := float64(*interval) * float64(count) / 1000.0
		ret := (valSec / sampleSec) * 100
		return ret
	}
	scaleSched := func(val float64) float64 {
		return val / float64(*jiffy) / float64((*interval)*(*samples)) * 100
	}

	fmt.Printf("usr:    %4s/%4s/%4s   sys:%4s/%4s/%4s  nice:%4s/%4s/%4s    idle:%4s/%4s/%4s\n",
		trim(scale(float64(sysHist.usr.Min())), 4),
		trim(scale(float64(sysHist.usr.Max())), 4),
		trim(scale(sysHist.usr.Mean()), 4),

		trim(scale(float64(sysHist.sys.Min())), 4),
		trim(scale(float64(sysHist.sys.Max())), 4),
		trim(scale(sysHist.sys.Mean()), 4),

		trim(scale(float64(sysHist.nice.Min())), 4),
		trim(scale(float64(sysHist.nice.Max())), 4),
		trim(scale(sysHist.nice.Mean()), 4),

		trim(scale(float64(sysHist.idle.Min())), 4),
		trim(scale(float64(sysHist.idle.Max())), 4),
		trim(scale(sysHist.idle.Mean()), 4),
	)
	fmt.Printf("iowait: %4s/%4s/%4s  ctxt:%4s/%4s/%4s  prun:%4s/%4s/%4s  pblock:%4s/%4s/%4s  pstart: %4d\n",
		trim(scale(float64(sysHist.iowait.Min())), 4),
		trim(scale(float64(sysHist.iowait.Max())), 4),
		trim(scale(sysHist.iowait.Mean()), 4),

		trim(scale(float64(sysHist.ctxt.Min())), 4),
		trim(scale(float64(sysHist.ctxt.Max())), 4),
		trim(scale(sysHist.ctxt.Mean()), 4),

		trim(float64(sysHist.procsRunning.Min()), 4),
		trim(float64(sysHist.procsRunning.Max()), 4),
		trim(sysHist.procsRunning.Mean(), 4),

		trim(float64(sysHist.procsBlocked.Min()), 4),
		trim(float64(sysHist.procsBlocked.Max()), 4),
		trim(sysHist.procsBlocked.Mean(), 4),

		sysSum.procsTotal,
	)

	list := sortList(histStats, *topN)

	fmt.Print("                comm     pid     min     max     usr     sys   ctime    slat     ctx     icx     rss  iowait thrd  sam\n")
	for _, thisStat := range list {
		hist := thisStat.hist
		pid := thisStat.pid
		var schedWait, nrSwitches, nrInvoluntarySwitches string
		sched, ok := sumSched[pid]
		if ok == true {
			schedWait = trim(scaleSched(sched.waitSum), 7)
			nrSwitches = fmt.Sprintf("%d", sched.nrSwitches)
			nrInvoluntarySwitches = fmt.Sprintf("%d", sched.nrInvoluntarySwitches)
		} else {
			schedWait = "-"
			nrSwitches = "-"
			nrInvoluntarySwitches = "-"
		}
		sampleCount := hist.utime.TotalCount()
		fmt.Printf("%20s %7d %7s %7s %7s %7s %7s %7s %7s %7s %7s %7s %4d %4d\n",
			sumStats[pid].comm,
			pid,
			trim(scale(float64(hist.ustime.Min())), 7),
			trim(scale(float64(hist.ustime.Max())), 7),
			trim(scaleSum(float64(sumStats[pid].utime), sampleCount), 7),
			trim(scaleSum(float64(sumStats[pid].stime), sampleCount), 7),
			trim(scaleSum(float64(sumStats[pid].cutime+sumStats[pid].cstime), sampleCount), 7),
			schedWait,
			nrSwitches,
			nrInvoluntarySwitches,
			formatMem(sumStats[pid].rss),
			trim(scaleSum(float64(sumStats[pid].delayacctBlkioTicks), sampleCount), 7),
			sumStats[pid].numThreads,
			sampleCount,
		)
	}
	fmt.Println()
}
<|endoftext|>


<commit_before>package routes

import (
	"log"
	"net/http"
	"text/template"
	"os"
	"time"

	"golang.org/x/oauth2"
	"github.com/google/go-github/github"
	"github.com/julienschmidt/httprouter"

	"github.com/gophergala2016/hugoku/ci"
	"github.com/gophergala2016/hugoku/store"
	"github.com/gophergala2016/hugoku/util/session"
	"github.com/gophergala2016/hugoku/util/cmd"
	"github.com/gophergala2016/hugoku/util/repo"
)

func PostProject(w http.ResponseWriter, r *http.Request, _ httprouter.Params) {
	var buildStatus = "ok"
	user, err := session.GetUser(r)
	if err != nil {
		log.Fatal(err)
	}
	projectName := r.PostFormValue("name")
	ts := oauth2.StaticTokenSource(
		&oauth2.Token{AccessToken: user.Token.AccessToken},
	)
	tc := oauth2.NewClient(oauth2.NoContext, ts)
	client := github.NewClient(tc)

	log.Printf("Creating %s...", projectName)

	startTime := time.Now()
	//path, err := ci.Deploy(username.(string), projectName)
	_, err = ci.Deploy(user.Username, projectName)

	buildDuration := time.Since(startTime)
	if err != nil {
		log.Fatalf("Error while trying to create project: %s", err)
		buildStatus = "fail"
	}

	buildInfo := store.BuildInfo{BuildTime: time.Now(), BuildDuration: buildDuration, BuildStatus: buildStatus}
	project := store.Project{Name: projectName, BuildsInfo: []store.BuildInfo{buildInfo}, LastBuildInfo: buildInfo}
	user.Projects = append(user.Projects, project)

	err = store.SaveUser(user)

	if !repo.Exists(client, user.Username, projectName) {
		repo := &github.Repository{
			Name:    github.String(projectName),
			Private: github.Bool(false),
		}
		_, _, err = client.Repositories.Create("", repo)
		if err != nil {
			log.Fatalf("Error while trying to create repo: %s", err)
		}

		// Push the repo
		wd, _ := os.Getwd()
		err := os.Chdir(wd + "/repos/" + user.Username + "/" + projectName + "/")
		if err != nil {
			log.Fatal(err)
		}
		cmd.Run("git", []string{"init", "--quiet"})
		cmd.Run("git", []string{"add", "."})
		cmd.Run("git", []string{"commit", "-m", "'initial source code'"})
		cmd.Run("git", []string{"remote", "add", "origin", "git@github.com:" + user.Username + "/" + projectName + ".git"})
		cmd.Run("git", []string{"push", "--quiet", "-u", "origin", "master"})
	}

	http.Redirect(w, r, "/", http.StatusTemporaryRedirect)
}

// getProject is the Hugoku project page handdler and shows the project and the build history.
func GetProject(w http.ResponseWriter, r *http.Request, ps httprouter.Params) {
	var id = ps.ByName("id")
	user, err := session.GetUser(r)
	if err != nil {
		log.Fatal("Error getting the user from the session: ", err)
	}
	// TODO: sanitize id
	log.Printf("getProject %s!\n", id)

	if id == "" {
		log.Println("Missing project id!")
		http.Redirect(w, r, "/", http.StatusTemporaryRedirect)
		return
	}
	project, err := user.GetProject(id)
	if err != nil {
		log.Println("Error getting project: ", id, err)
	}
	t, err := template.ParseFiles("templates/project.html",
		"templates/partials/footer.html",
		"templates/partials/header.html")
	if err != nil {
		log.Fatal("Error parsing the project page template:", err)
	}
	err = t.Execute(w, project)
	if err != nil {
		log.Fatal("Error executing the project page template:", err)
	}
}
<commit_msg>cwd back to the main folder<commit_after>package routes

import (
	"log"
	"net/http"
	"os"
	"text/template"
	"time"

	"github.com/google/go-github/github"
	"github.com/julienschmidt/httprouter"
	"golang.org/x/oauth2"

	"github.com/gophergala2016/hugoku/ci"
	"github.com/gophergala2016/hugoku/store"
	"github.com/gophergala2016/hugoku/util/cmd"
	"github.com/gophergala2016/hugoku/util/repo"
	"github.com/gophergala2016/hugoku/util/session"
)

func PostProject(w http.ResponseWriter, r *http.Request, _ httprouter.Params) {
	var buildStatus = "ok"
	user, err := session.GetUser(r)
	if err != nil {
		log.Fatal(err)
	}
	projectName := r.PostFormValue("name")
	ts := oauth2.StaticTokenSource(
		&oauth2.Token{AccessToken: user.Token.AccessToken},
	)
	tc := oauth2.NewClient(oauth2.NoContext, ts)
	client := github.NewClient(tc)

	log.Printf("Creating %s...", projectName)

	startTime := time.Now()
	//path, err := ci.Deploy(username.(string), projectName)
	_, err = ci.Deploy(user.Username, projectName)

	buildDuration := time.Since(startTime)
	if err != nil {
		log.Fatalf("Error while trying to create project: %s", err)
		buildStatus = "fail"
	}

	buildInfo := store.BuildInfo{BuildTime: time.Now(), BuildDuration: buildDuration, BuildStatus: buildStatus}
	project := store.Project{Name: projectName, BuildsInfo: []store.BuildInfo{buildInfo}, LastBuildInfo: buildInfo}
	user.Projects = append(user.Projects, project)

	err = store.SaveUser(user)

	if !repo.Exists(client, user.Username, projectName) {
		repo := &github.Repository{
			Name:    github.String(projectName),
			Private: github.Bool(false),
		}
		_, _, err = client.Repositories.Create("", repo)
		if err != nil {
			log.Fatalf("Error while trying to create repo: %s", err)
		}

		// Push the repo
		wd, _ := os.Getwd()
		err := os.Chdir(wd + "/repos/" + user.Username + "/" + projectName + "/")
		if err != nil {
			log.Fatal(err)
		}
		cmd.Run("git", []string{"init", "--quiet"})
		cmd.Run("git", []string{"add", "."})
		cmd.Run("git", []string{"commit", "-m", "'initial source code'"})
		cmd.Run("git", []string{"remote", "add", "origin", "git@github.com:" + user.Username + "/" + projectName + ".git"})
		cmd.Run("git", []string{"push", "--quiet", "-u", "origin", "master"})
		err = os.Chdir(wd)
		if err != nil {
			log.Fatal(err)
		}
	}

	http.Redirect(w, r, "/", http.StatusTemporaryRedirect)
}

// getProject is the Hugoku project page handdler and shows the project and the build history.
func GetProject(w http.ResponseWriter, r *http.Request, ps httprouter.Params) {
	var id = ps.ByName("id")
	user, err := session.GetUser(r)
	if err != nil {
		log.Fatal("Error getting the user from the session: ", err)
	}
	// TODO: sanitize id
	log.Printf("getProject %s!\n", id)

	if id == "" {
		log.Println("Missing project id!")
		http.Redirect(w, r, "/", http.StatusTemporaryRedirect)
		return
	}
	project, err := user.GetProject(id)
	if err != nil {
		log.Println("Error getting project: ", id, err)
	}
	t, err := template.ParseFiles("templates/project.html",
		"templates/partials/footer.html",
		"templates/partials/header.html")
	if err != nil {
		log.Fatal("Error parsing the project page template:", err)
	}
	err = t.Execute(w, project)
	if err != nil {
		log.Fatal("Error executing the project page template:", err)
	}
}
<|endoftext|>


<commit_before>package ruler

import (
	"sync"
	"time"

	"github.com/jonboulle/clockwork"
	"github.com/prometheus/client_golang/prometheus"
	"github.com/prometheus/common/log"
	"github.com/prometheus/prometheus/rules"
	"github.com/weaveworks/scope/common/instrument"
	"golang.org/x/net/context"

	"github.com/weaveworks/cortex/util"
)

const (
	// Backoff for loading initial configuration set.
	minBackoff = 100 * time.Millisecond
	maxBackoff = 2 * time.Second
)

var (
	queueLength = prometheus.NewGauge(prometheus.GaugeOpts{
		Namespace: "cortex",
		Name:      "rules_queue_length",
		Help:      "The length of the rules queue.",
	})
	totalConfigs = prometheus.NewGauge(prometheus.GaugeOpts{
		Namespace: "cortex",
		Name:      "configs",
		Help:      "How many configs the scheduler knows about.",
	})
	configsRequestDuration = prometheus.NewHistogramVec(prometheus.HistogramOpts{
		Namespace: "cortex",
		Name:      "configs_request_duration_seconds",
		Help:      "Time spent requesting configs.",
		Buckets:   prometheus.DefBuckets,
	}, []string{"operation", "status_code"})
)

func init() {
	prometheus.MustRegister(queueLength)
	prometheus.MustRegister(configsRequestDuration)
	prometheus.MustRegister(totalConfigs)
}

type workItem struct {
	userID    string
	rules     []rules.Rule
	scheduled time.Time
}

// Key implements ScheduledItem
func (w workItem) Key() string {
	return w.userID
}

// Scheduled implements ScheduledItem
func (w workItem) Scheduled() time.Time {
	return w.scheduled
}

// Defer returns a copy of this work item, rescheduled to a later time.
func (w workItem) Defer(interval time.Duration) workItem {
	return workItem{w.userID, w.rules, w.scheduled.Add(interval)}
}

type scheduler struct {
	configsAPI         configsAPI // XXX: Maybe make this an interface ConfigSource or similar.
	evaluationInterval time.Duration
	q                  *util.SchedulingQueue

	// All the configurations that we have. Only used for instrumentation.
	cfgs map[string]cortexConfig

	pollInterval time.Duration

	latestConfig configID
	latestMutex  sync.RWMutex

	done       chan struct{}
	terminated chan struct{}
}

// newScheduler makes a new scheduler.
func newScheduler(configsAPI configsAPI, evaluationInterval, pollInterval time.Duration) scheduler {
	return scheduler{
		configsAPI:         configsAPI,
		evaluationInterval: evaluationInterval,
		pollInterval:       pollInterval,
		q:                  util.NewSchedulingQueue(clockwork.NewRealClock()),
		cfgs:               map[string]cortexConfig{},
	}
}

// Run polls the source of configurations for changes.
func (s *scheduler) Run() {
	log.Debugf("Scheduler started")
	defer close(s.terminated)
	// Load initial set of all configurations before polling for new ones.
	s.addNewConfigs(time.Now(), s.loadAllConfigs())
	ticker := time.NewTicker(s.pollInterval)
	for {
		select {
		case now := <-ticker.C:
			err := s.updateConfigs(now)
			if err != nil {
				log.Warnf("Scheduler: error updating configs: %v", err)
			}
		case <-s.done:
			ticker.Stop()
		}
	}
}

func (s *scheduler) Stop() {
	close(s.done)
	s.q.Close()
	<-s.terminated
	log.Debugf("Scheduler stopped")
}

// Load the full set of configurations from the server, retrying with backoff
// until we can get them.
func (s *scheduler) loadAllConfigs() map[string]cortexConfigView {
	backoff := minBackoff
	for {
		cfgs, err := s.poll()
		if err == nil {
			log.Debugf("Scheduler: found %d configurations in initial load", len(cfgs))
			return cfgs
		}
		log.Warnf("Scheduler: error fetching all configurations, backing off: %v", err)
		time.Sleep(backoff)
		backoff *= 2
		if backoff > maxBackoff {
			backoff = maxBackoff
		}
	}
}

func (s *scheduler) updateConfigs(now time.Time) error {
	cfgs, err := s.poll()
	if err != nil {
		return err
	}
	s.addNewConfigs(now, cfgs)
	return nil
}

func (s *scheduler) poll() (map[string]cortexConfigView, error) {
	s.latestMutex.RLock()
	configID := s.latestConfig
	s.latestMutex.RUnlock()
	var cfgs map[string]cortexConfigView
	err := instrument.TimeRequestHistogram(context.Background(), "Configs.GetOrgConfigs", configsRequestDuration, func(_ context.Context) error {
		var err error
		cfgs, err = s.configsAPI.getOrgConfigs(configID)
		return err
	})
	if err != nil {
		log.Warnf("Scheduler: configs server poll failed: %v", err)
		return nil, err
	}
	s.latestMutex.Lock()
	s.latestConfig = getLatestConfigID(cfgs)
	s.latestMutex.Unlock()
	return cfgs, nil
}

func (s *scheduler) addNewConfigs(now time.Time, cfgs map[string]cortexConfigView) {
	// TODO: instrument how many configs we have, both valid & invalid.
	log.Debugf("Adding %d configurations", len(cfgs))
	for userID, config := range cfgs {
		rules, err := config.Config.GetRules()
		if err != nil {
			// XXX: This means that if a user has a working configuration and
			// they submit a broken one, we'll keep processing the last known
			// working configuration, and they'll never know.
			// TODO: Provide a way of deleting / cancelling recording rules.
			log.Warnf("Scheduler: invalid Cortex configuration for %v", userID)
			continue
		}

		s.addWorkItem(workItem{userID, rules, now})
		s.cfgs[userID] = config.Config
	}
	totalConfigs.Set(float64(len(s.cfgs)))
}

func (s *scheduler) addWorkItem(i workItem) {
	s.q.Enqueue(i)
	queueLength.Set(float64(s.q.Length()))
	log.Debugf("Scheduler: work item added: %v", i)
}

// Get the next scheduled work item, blocking if none.
//
// Call `workItemDone` on the returned item to indicate that it is ready to be
// rescheduled.
func (s *scheduler) nextWorkItem() *workItem {
	log.Debugf("Scheduler: work item requested. Pending...")
	// TODO: We are blocking here on the second Dequeue event. Write more
	// tests for the scheduling queue.
	op := s.q.Dequeue()
	if op == nil {
		log.Infof("Queue closed. No more work items.")
		return nil
	}
	queueLength.Set(float64(s.q.Length()))
	item := op.(workItem)
	log.Debugf("Scheduler: work item granted: %v", item)
	return &item
}

// workItemDone marks the given item as being ready to be rescheduled.
func (s *scheduler) workItemDone(i workItem) {
	next := i.Defer(s.evaluationInterval)
	log.Debugf("Scheduler: work item %v rescheduled for %v", i, next.scheduled.Format("2006-01-02 15:04:05"))
	s.addWorkItem(next)
}
<commit_msg>More instrumentation improvements<commit_after>package ruler

import (
	"sync"
	"time"

	"github.com/jonboulle/clockwork"
	"github.com/prometheus/client_golang/prometheus"
	"github.com/prometheus/common/log"
	"github.com/prometheus/prometheus/rules"
	"github.com/weaveworks/scope/common/instrument"
	"golang.org/x/net/context"

	"github.com/weaveworks/cortex/util"
)

const (
	// Backoff for loading initial configuration set.
	minBackoff = 100 * time.Millisecond
	maxBackoff = 2 * time.Second
)

var (
	queueLength = prometheus.NewGauge(prometheus.GaugeOpts{
		Namespace: "cortex",
		Name:      "rules_queue_length",
		Help:      "The length of the rules queue.",
	})
	totalConfigs = prometheus.NewGauge(prometheus.GaugeOpts{
		Namespace: "cortex",
		Name:      "configs",
		Help:      "How many configs the scheduler knows about.",
	})
	configsRequestDuration = prometheus.NewHistogramVec(prometheus.HistogramOpts{
		Namespace: "cortex",
		Name:      "configs_request_duration_seconds",
		Help:      "Time spent requesting configs.",
		Buckets:   prometheus.DefBuckets,
	}, []string{"operation", "status_code"})
)

func init() {
	prometheus.MustRegister(queueLength)
	prometheus.MustRegister(configsRequestDuration)
	prometheus.MustRegister(totalConfigs)
}

type workItem struct {
	userID    string
	rules     []rules.Rule
	scheduled time.Time
}

// Key implements ScheduledItem
func (w workItem) Key() string {
	return w.userID
}

// Scheduled implements ScheduledItem
func (w workItem) Scheduled() time.Time {
	return w.scheduled
}

// Defer returns a copy of this work item, rescheduled to a later time.
func (w workItem) Defer(interval time.Duration) workItem {
	return workItem{w.userID, w.rules, w.scheduled.Add(interval)}
}

type scheduler struct {
	configsAPI         configsAPI // XXX: Maybe make this an interface ConfigSource or similar.
	evaluationInterval time.Duration
	q                  *util.SchedulingQueue

	// All the configurations that we have. Only used for instrumentation.
	cfgs map[string]cortexConfig

	pollInterval time.Duration

	latestConfig configID
	latestMutex  sync.RWMutex

	done       chan struct{}
	terminated chan struct{}
}

// newScheduler makes a new scheduler.
func newScheduler(configsAPI configsAPI, evaluationInterval, pollInterval time.Duration) scheduler {
	return scheduler{
		configsAPI:         configsAPI,
		evaluationInterval: evaluationInterval,
		pollInterval:       pollInterval,
		q:                  util.NewSchedulingQueue(clockwork.NewRealClock()),
		cfgs:               map[string]cortexConfig{},
	}
}

// Run polls the source of configurations for changes.
func (s *scheduler) Run() {
	log.Debugf("Scheduler started")
	defer close(s.terminated)
	// Load initial set of all configurations before polling for new ones.
	s.addNewConfigs(time.Now(), s.loadAllConfigs())
	ticker := time.NewTicker(s.pollInterval)
	for {
		select {
		case now := <-ticker.C:
			err := s.updateConfigs(now)
			if err != nil {
				log.Warnf("Scheduler: error updating configs: %v", err)
			}
		case <-s.done:
			ticker.Stop()
		}
	}
}

func (s *scheduler) Stop() {
	close(s.done)
	s.q.Close()
	<-s.terminated
	log.Debugf("Scheduler stopped")
}

// Load the full set of configurations from the server, retrying with backoff
// until we can get them.
func (s *scheduler) loadAllConfigs() map[string]cortexConfigView {
	backoff := minBackoff
	for {
		cfgs, err := s.poll()
		if err == nil {
			log.Debugf("Scheduler: found %d configurations in initial load", len(cfgs))
			return cfgs
		}
		log.Warnf("Scheduler: error fetching all configurations, backing off: %v", err)
		time.Sleep(backoff)
		backoff *= 2
		if backoff > maxBackoff {
			backoff = maxBackoff
		}
	}
}

func (s *scheduler) updateConfigs(now time.Time) error {
	cfgs, err := s.poll()
	if err != nil {
		return err
	}
	s.addNewConfigs(now, cfgs)
	return nil
}

func (s *scheduler) poll() (map[string]cortexConfigView, error) {
	s.latestMutex.RLock()
	configID := s.latestConfig
	s.latestMutex.RUnlock()
	var cfgs map[string]cortexConfigView
	err := instrument.TimeRequestHistogram(context.Background(), "Configs.GetOrgConfigs", configsRequestDuration, func(_ context.Context) error {
		var err error
		cfgs, err = s.configsAPI.getOrgConfigs(configID)
		return err
	})
	if err != nil {
		log.Warnf("Scheduler: configs server poll failed: %v", err)
		return nil, err
	}
	s.latestMutex.Lock()
	s.latestConfig = getLatestConfigID(cfgs)
	s.latestMutex.Unlock()
	return cfgs, nil
}

func (s *scheduler) addNewConfigs(now time.Time, cfgs map[string]cortexConfigView) {
	// TODO: instrument how many configs we have, both valid & invalid.
	log.Debugf("Adding %d configurations", len(cfgs))
	for userID, config := range cfgs {
		rules, err := config.Config.GetRules()
		if err != nil {
			// XXX: This means that if a user has a working configuration and
			// they submit a broken one, we'll keep processing the last known
			// working configuration, and they'll never know.
			// TODO: Provide a way of deleting / cancelling recording rules.
			log.Warnf("Scheduler: invalid Cortex configuration for %v: %v", userID, err)
			continue
		}

		s.addWorkItem(workItem{userID, rules, now})
		s.cfgs[userID] = config.Config
	}
	totalConfigs.Set(float64(len(s.cfgs)))
}

func (s *scheduler) addWorkItem(i workItem) {
	s.q.Enqueue(i)
	queueLength.Set(float64(s.q.Length()))
	log.Debugf("Scheduler: work item added: %v", i)
}

// Get the next scheduled work item, blocking if none.
//
// Call `workItemDone` on the returned item to indicate that it is ready to be
// rescheduled.
func (s *scheduler) nextWorkItem() *workItem {
	log.Debugf("Scheduler: work item requested. Pending...")
	// TODO: We are blocking here on the second Dequeue event. Write more
	// tests for the scheduling queue.
	op := s.q.Dequeue()
	if op == nil {
		log.Infof("Queue closed. No more work items.")
		return nil
	}
	queueLength.Set(float64(s.q.Length()))
	item := op.(workItem)
	log.Debugf("Scheduler: work item granted: %v", item)
	return &item
}

// workItemDone marks the given item as being ready to be rescheduled.
func (s *scheduler) workItemDone(i workItem) {
	next := i.Defer(s.evaluationInterval)
	log.Debugf("Scheduler: work item %v rescheduled for %v", i, next.scheduled.Format("2006-01-02 15:04:05"))
	s.addWorkItem(next)
}
<|endoftext|>