<commit_before>package backoff

import (
	"crypto/rand"
	"math"
	"math/big"
	"time"
)

// BackoffStrategy can be implemented to provide different backoff algorithms.
type BackoffStrategy interface {
	// GetBackoffDuration calculates the next time.Duration that the current thread will sleep for when backing off.
	// It receives the current backoff count, the initial backoff duration and the last back off duration.
	GetBackoffDuration(int, time.Duration, time.Duration) time.Duration
}

// Backoff tracks the generic state of the configured back off strategy.
type Backoff struct {
	// LastDuration contains the duration that was previously waited, or 0 if no backoff has occurred yet.
	LastDuration time.Duration
	// NextDuration contains the duration that will be waited on the next call to Backoff().
	NextDuration time.Duration
	start        time.Duration
	limit        time.Duration
	count        int
	strategy     BackoffStrategy
}

// NewBackoff creates a new Backoff using the specified BackoffStrategy, start duration and limit.
func NewBackoff(strategy BackoffStrategy, start time.Duration, limit time.Duration) *Backoff {
	backoff := Backoff{strategy: strategy, start: start, limit: limit}
	backoff.Reset()
	return &backoff
}

// Reset sets the Backoff to its initial conditions ready to start over.
func (b *Backoff) Reset() {
	b.count = 0
	b.LastDuration = 0
	b.NextDuration = b.getNextDuration()
}

// Backoff causes the current thread/routine to sleep for NextDuration.
func (b *Backoff) Backoff() {
	time.Sleep(b.NextDuration)
	b.count++
	b.LastDuration = b.NextDuration
	b.NextDuration = b.getNextDuration()
}

func (b *Backoff) getNextDuration() time.Duration {
	backoff := b.strategy.GetBackoffDuration(b.count, b.start, b.LastDuration)
	if b.limit > 0 && backoff > b.limit {
		backoff = b.limit
	}
	return backoff
}

type exponential struct{}

func (exponential) GetBackoffDuration(backoffCount int, start time.Duration, lastBackoff time.Duration) time.Duration {
	period := int64(math.Pow(2, float64(backoffCount)))
	return time.Duration(period) * start
}

// NewExponential creates a new backoff using the exponential backoff algorithm.
func NewExponential(start time.Duration, limit time.Duration) *Backoff {
	return NewBackoff(exponential{}, start, limit)
}

type exponentialFullJitter struct{}

func (exponentialFullJitter) GetBackoffDuration(backoffCount int, start time.Duration, lastBackoff time.Duration) time.Duration {
	backoff := exponential{}.GetBackoffDuration(backoffCount, start, lastBackoff)
	if backoff <= 0 {
		return backoff
	}
	jitter, _ := rand.Int(rand.Reader, big.NewInt(int64(backoff)))
	return time.Duration(jitter.Int64())
}

// NewExponentialFullJitter creates a new backoff using the exponential with full jitter backoff algorithm.
func NewExponentialFullJitter(start time.Duration, limit time.Duration) *Backoff {
	return NewBackoff(exponentialFullJitter{}, start, limit)
}

type linear struct{}

func (linear) GetBackoffDuration(backoffCount int, start time.Duration, lastBackoff time.Duration) time.Duration {
	return time.Duration(backoffCount) * start
}

// NewLinear creates a new backoff using the linear backoff algorithm.
func NewLinear(start time.Duration, limit time.Duration) *Backoff {
	return NewBackoff(linear{}, start, limit)
}
<commit_msg>Apply limit before calculating jitter<commit_after>package backoff

import (
	"crypto/rand"
	"math"
	"math/big"
	"time"
)

// BackoffStrategy can be implemented to provide different backoff algorithms.
type BackoffStrategy interface {
	// GetBackoffDuration calculates the next time.Duration that the current thread will sleep for when backing off.
	// It receives the current backoff count, the initial backoff duration and the last back off duration.
	GetBackoffDuration(int, time.Duration, time.Duration) time.Duration
}

// Backoff tracks the generic state of the configured back off strategy.
type Backoff struct {
	// LastDuration contains the duration that was previously waited, or 0 if no backoff has occurred yet.
	LastDuration time.Duration
	// NextDuration contains the duration that will be waited on the next call to Backoff().
	NextDuration time.Duration
	start        time.Duration
	limit        time.Duration
	count        int
	strategy     BackoffStrategy
}

// NewBackoff creates a new Backoff using the specified BackoffStrategy, start duration and limit.
func NewBackoff(strategy BackoffStrategy, start time.Duration, limit time.Duration) *Backoff {
	backoff := Backoff{strategy: strategy, start: start, limit: limit}
	backoff.Reset()
	return &backoff
}

// Reset sets the Backoff to its initial conditions ready to start over.
func (b *Backoff) Reset() {
	b.count = 0
	b.LastDuration = 0
	b.NextDuration = b.getNextDuration()
}

// Backoff causes the current thread/routine to sleep for NextDuration.
func (b *Backoff) Backoff() {
	time.Sleep(b.NextDuration)
	b.count++
	b.LastDuration = b.NextDuration
	b.NextDuration = b.getNextDuration()
}

func (b *Backoff) getNextDuration() time.Duration {
	backoff := b.strategy.GetBackoffDuration(b.count, b.start, b.LastDuration)
	if b.limit > 0 && backoff > b.limit {
		backoff = b.limit
	}
	return backoff
}

type exponential struct{}

func (exponential) GetBackoffDuration(backoffCount int, start time.Duration, lastBackoff time.Duration) time.Duration {
	period := int64(math.Pow(2, float64(backoffCount)))
	return time.Duration(period) * start
}

// NewExponential creates a new backoff using the exponential backoff algorithm.
func NewExponential(start time.Duration, limit time.Duration) *Backoff {
	return NewBackoff(exponential{}, start, limit)
}

type exponentialFullJitter struct {
	limit time.Duration
}

func (b exponentialFullJitter) GetBackoffDuration(backoffCount int, start time.Duration, lastBackoff time.Duration) time.Duration {
	backoff := exponential{}.GetBackoffDuration(backoffCount, start, lastBackoff)
	if backoff <= 0 {
		return backoff
	}
	// apply limit here to ensure the jitter falls btween 0-min(limit,backoff)
	if b.limit > 0 && backoff > b.limit {
		backoff = b.limit
	}
	jitter, _ := rand.Int(rand.Reader, big.NewInt(int64(backoff)))
	return time.Duration(jitter.Int64())
}

// NewExponentialFullJitter creates a new backoff using the exponential with full jitter backoff algorithm.
func NewExponentialFullJitter(start time.Duration, limit time.Duration) *Backoff {
	return NewBackoff(exponentialFullJitter{limit: limit}, start, limit)
}

type linear struct{}

func (linear) GetBackoffDuration(backoffCount int, start time.Duration, lastBackoff time.Duration) time.Duration {
	return time.Duration(backoffCount) * start
}

// NewLinear creates a new backoff using the linear backoff algorithm.
func NewLinear(start time.Duration, limit time.Duration) *Backoff {
	return NewBackoff(linear{}, start, limit)
}
<|endoftext|>


<commit_before>package resource

import (
	"errors"
	"net/http"

	"github.com/jhh/puka-api/lib"
	"github.com/jhh/puka-api/lib/storage"
	"github.com/manyminds/api2go"
)

const typeErrMsg = "Invalid instance given"

// The BookmarkResource struct implements api2go routes
type BookmarkResource struct {
	BookmarkStorage storage.BookmarkStorage
}

// FindAll satisfies api2go.FindAll interface
func (s BookmarkResource) FindAll(r api2go.Request) (api2go.Responder, error) {
	// check for authentication error set by middleware
	if err, ok := r.Context.Get("error"); ok {
		return &Response{}, err.(error)
	}
	bookmarks, err := s.BookmarkStorage.GetAll(storage.NewQuery(r))
	if err != nil {
		return &Response{}, api2go.NewHTTPError(err, err.Error(), http.StatusInternalServerError)
	}
	return &Response{Res: bookmarks, Code: http.StatusOK}, nil
}

// PaginatedFindAll satisfies the api2go.PaginatedFindAll interface
func (s BookmarkResource) PaginatedFindAll(r api2go.Request) (uint, api2go.Responder, error) {
	// check for authentication error set by middleware
	if err, ok := r.Context.Get("error"); ok {
		return 0, &Response{}, err.(error)
	}

	p, err := NewPaginator(r)
	if err != nil {
		return 0, &Response{}, api2go.NewHTTPError(err, err.Error(), http.StatusBadRequest)
	}
	q := storage.NewQuery(r)
	count, err := s.BookmarkStorage.Count(q)
	if err != nil {
		return 0, &Response{}, api2go.NewHTTPError(err, err.Error(), http.StatusInternalServerError)
	}
	bookmarks, err := s.BookmarkStorage.GetPage(q, p.Skip, p.Limit)
	if err != nil {
		return 0, &Response{}, err
	}

	// total count, response, error
	return uint(count), &Response{Res: bookmarks, Code: http.StatusOK}, nil
}

// FindOne satisfies api2go.CRUD interface
func (s BookmarkResource) FindOne(id string, r api2go.Request) (api2go.Responder, error) {
	// check for authentication error set by middleware
	if err, ok := r.Context.Get("error"); ok {
		return &Response{}, err.(error)
	}

	bookmark, err := s.BookmarkStorage.GetOne(id)
	if err != nil {
		return &Response{}, api2go.NewHTTPError(err, err.Error(), http.StatusNotFound)
	}
	return &Response{Res: bookmark, Code: http.StatusOK}, nil
}

// Create satisfies api2go.CRUD interface
func (s BookmarkResource) Create(obj interface{}, r api2go.Request) (api2go.Responder, error) {
	// check for authentication error set by middleware
	if err, ok := r.Context.Get("error"); ok {
		return &Response{}, err.(error)
	}

	bookmark, ok := obj.(lib.Bookmark)
	if !ok {
		return &Response{}, api2go.NewHTTPError(errors.New(typeErrMsg), typeErrMsg, http.StatusBadRequest)
	}

	err := s.BookmarkStorage.Insert(&bookmark)
	if err != nil {
		return &Response{}, api2go.NewHTTPError(err, err.Error(), http.StatusInternalServerError)
	}

	return &Response{Res: bookmark, Code: http.StatusCreated}, nil
}

// Delete satisfies api2go.CRUD interface
func (s BookmarkResource) Delete(id string, r api2go.Request) (api2go.Responder, error) {
	// check for authentication error set by middleware
	if err, ok := r.Context.Get("error"); ok {
		return &Response{}, err.(error)
	}

	if err := s.BookmarkStorage.Delete(id); err != nil {
		return &Response{}, api2go.NewHTTPError(err, err.Error(), http.StatusNotFound)
	}
	return &Response{Code: http.StatusNoContent}, nil
}

// Update satisfies api2go.CRUD interface
func (s BookmarkResource) Update(obj interface{}, r api2go.Request) (api2go.Responder, error) {
	// check for authentication error set by middleware
	if err, ok := r.Context.Get("error"); ok {
		return &Response{}, err.(error)
	}

	bookmark, ok := obj.(lib.Bookmark)
	if !ok {
		return &Response{}, api2go.NewHTTPError(errors.New(typeErrMsg), typeErrMsg, http.StatusBadRequest)
	}

	if err := s.BookmarkStorage.Update(&bookmark); err != nil {
		return &Response{}, api2go.NewHTTPError(err, err.Error(), http.StatusInternalServerError)
	}

	return &Response{Res: bookmark, Code: http.StatusOK}, nil
}
<commit_msg>Fix doc comment.<commit_after>package resource

import (
	"errors"
	"net/http"

	"github.com/jhh/puka-api/lib"
	"github.com/jhh/puka-api/lib/storage"
	"github.com/manyminds/api2go"
)

const typeErrMsg = "Invalid instance given"

// The BookmarkResource struct implements api2go routes.
type BookmarkResource struct {
	BookmarkStorage storage.BookmarkStorage
}

// FindAll satisfies api2go.FindAll interface
func (s BookmarkResource) FindAll(r api2go.Request) (api2go.Responder, error) {
	// check for authentication error set by middleware
	if err, ok := r.Context.Get("error"); ok {
		return &Response{}, err.(error)
	}
	bookmarks, err := s.BookmarkStorage.GetAll(storage.NewQuery(r))
	if err != nil {
		return &Response{}, api2go.NewHTTPError(err, err.Error(), http.StatusInternalServerError)
	}
	return &Response{Res: bookmarks, Code: http.StatusOK}, nil
}

// PaginatedFindAll satisfies the api2go.PaginatedFindAll interface
func (s BookmarkResource) PaginatedFindAll(r api2go.Request) (uint, api2go.Responder, error) {
	// check for authentication error set by middleware
	if err, ok := r.Context.Get("error"); ok {
		return 0, &Response{}, err.(error)
	}

	p, err := NewPaginator(r)
	if err != nil {
		return 0, &Response{}, api2go.NewHTTPError(err, err.Error(), http.StatusBadRequest)
	}
	q := storage.NewQuery(r)
	count, err := s.BookmarkStorage.Count(q)
	if err != nil {
		return 0, &Response{}, api2go.NewHTTPError(err, err.Error(), http.StatusInternalServerError)
	}
	bookmarks, err := s.BookmarkStorage.GetPage(q, p.Skip, p.Limit)
	if err != nil {
		return 0, &Response{}, err
	}

	// total count, response, error
	return uint(count), &Response{Res: bookmarks, Code: http.StatusOK}, nil
}

// FindOne satisfies api2go.CRUD interface
func (s BookmarkResource) FindOne(id string, r api2go.Request) (api2go.Responder, error) {
	// check for authentication error set by middleware
	if err, ok := r.Context.Get("error"); ok {
		return &Response{}, err.(error)
	}

	bookmark, err := s.BookmarkStorage.GetOne(id)
	if err != nil {
		return &Response{}, api2go.NewHTTPError(err, err.Error(), http.StatusNotFound)
	}
	return &Response{Res: bookmark, Code: http.StatusOK}, nil
}

// Create satisfies api2go.CRUD interface
func (s BookmarkResource) Create(obj interface{}, r api2go.Request) (api2go.Responder, error) {
	// check for authentication error set by middleware
	if err, ok := r.Context.Get("error"); ok {
		return &Response{}, err.(error)
	}

	bookmark, ok := obj.(lib.Bookmark)
	if !ok {
		return &Response{}, api2go.NewHTTPError(errors.New(typeErrMsg), typeErrMsg, http.StatusBadRequest)
	}

	err := s.BookmarkStorage.Insert(&bookmark)
	if err != nil {
		return &Response{}, api2go.NewHTTPError(err, err.Error(), http.StatusInternalServerError)
	}

	return &Response{Res: bookmark, Code: http.StatusCreated}, nil
}

// Delete satisfies api2go.CRUD interface
func (s BookmarkResource) Delete(id string, r api2go.Request) (api2go.Responder, error) {
	// check for authentication error set by middleware
	if err, ok := r.Context.Get("error"); ok {
		return &Response{}, err.(error)
	}

	if err := s.BookmarkStorage.Delete(id); err != nil {
		return &Response{}, api2go.NewHTTPError(err, err.Error(), http.StatusNotFound)
	}
	return &Response{Code: http.StatusNoContent}, nil
}

// Update satisfies api2go.CRUD interface
func (s BookmarkResource) Update(obj interface{}, r api2go.Request) (api2go.Responder, error) {
	// check for authentication error set by middleware
	if err, ok := r.Context.Get("error"); ok {
		return &Response{}, err.(error)
	}

	bookmark, ok := obj.(lib.Bookmark)
	if !ok {
		return &Response{}, api2go.NewHTTPError(errors.New(typeErrMsg), typeErrMsg, http.StatusBadRequest)
	}

	if err := s.BookmarkStorage.Update(&bookmark); err != nil {
		return &Response{}, api2go.NewHTTPError(err, err.Error(), http.StatusInternalServerError)
	}

	return &Response{Res: bookmark, Code: http.StatusOK}, nil
}
<|endoftext|>


<commit_before>package core

import (
	"github.com/v2ray/v2ray-core/common/log"
	v2net "github.com/v2ray/v2ray-core/common/net"
	"github.com/v2ray/v2ray-core/common/retry"
	"github.com/v2ray/v2ray-core/config"
)

var (
	inboundFactories  = make(map[string]InboundConnectionHandlerFactory)
	outboundFactories = make(map[string]OutboundConnectionHandlerFactory)
)

func RegisterInboundConnectionHandlerFactory(name string, factory InboundConnectionHandlerFactory) error {
	// TODO check name
	inboundFactories[name] = factory
	return nil
}

func RegisterOutboundConnectionHandlerFactory(name string, factory OutboundConnectionHandlerFactory) error {
	// TODO check name
	outboundFactories[name] = factory
	return nil
}

// Point is an single server in V2Ray system.
type Point struct {
	port uint16
	ich  InboundConnectionHandler
	och  OutboundConnectionHandler
}

// NewPoint returns a new Point server based on given configuration.
// The server is not started at this point.
func NewPoint(pConfig config.PointConfig) (*Point, error) {
	var vpoint = new(Point)
	vpoint.port = pConfig.Port()

	ichFactory, ok := inboundFactories[pConfig.InboundConfig().Protocol()]
	if !ok {
		log.Error("Unknown inbound connection handler factory %s", pConfig.InboundConfig().Protocol())
		return nil, config.BadConfiguration
	}
	ichConfig := pConfig.InboundConfig().Settings(config.TypeInbound)
	ich, err := ichFactory.Create(vpoint, ichConfig)
	if err != nil {
		log.Error("Failed to create inbound connection handler: %v", err)
		return nil, err
	}
	vpoint.ich = ich

	ochFactory, ok := outboundFactories[pConfig.OutboundConfig().Protocol()]
	if !ok {
		log.Error("Unknown outbound connection handler factory %s", pConfig.OutboundConfig().Protocol())
		return nil, config.BadConfiguration
	}
	ochConfig := pConfig.OutboundConfig().Settings(config.TypeOutbound)
	och, err := ochFactory.Create(vpoint, ochConfig)
	if err != nil {
		log.Error("Failed to create outbound connection handler: %v", err)
		return nil, err
	}
	vpoint.och = och

	return vpoint, nil
}

type InboundConnectionHandlerFactory interface {
	Create(vp *Point, config interface{}) (InboundConnectionHandler, error)
}

type InboundConnectionHandler interface {
	Listen(port uint16) error
}

type OutboundConnectionHandlerFactory interface {
	Create(VP *Point, config interface{}) (OutboundConnectionHandler, error)
}

type OutboundConnectionHandler interface {
	Dispatch(firstPacket v2net.Packet, ray OutboundRay) error
}

// Start starts the Point server, and return any error during the process.
// In the case of any errors, the state of the server is unpredicatable.
func (vp *Point) Start() error {
	if vp.port <= 0 {
		log.Error("Invalid port %d", vp.port)
		return config.BadConfiguration
	}

	return retry.Timed(100 /* times */, 100 /* ms */).On(func() error {
		return vp.ich.Listen(vp.port)
	})
}

func (p *Point) DispatchToOutbound(packet v2net.Packet) InboundRay {
	ray := NewRay()
	go p.och.Dispatch(packet, ray)
	return ray
}
<commit_msg>Always show running message.<commit_after>package core

import (
	"github.com/v2ray/v2ray-core/common/log"
	v2net "github.com/v2ray/v2ray-core/common/net"
	"github.com/v2ray/v2ray-core/common/retry"
	"github.com/v2ray/v2ray-core/config"
)

var (
	inboundFactories  = make(map[string]InboundConnectionHandlerFactory)
	outboundFactories = make(map[string]OutboundConnectionHandlerFactory)
)

func RegisterInboundConnectionHandlerFactory(name string, factory InboundConnectionHandlerFactory) error {
	// TODO check name
	inboundFactories[name] = factory
	return nil
}

func RegisterOutboundConnectionHandlerFactory(name string, factory OutboundConnectionHandlerFactory) error {
	// TODO check name
	outboundFactories[name] = factory
	return nil
}

// Point is an single server in V2Ray system.
type Point struct {
	port uint16
	ich  InboundConnectionHandler
	och  OutboundConnectionHandler
}

// NewPoint returns a new Point server based on given configuration.
// The server is not started at this point.
func NewPoint(pConfig config.PointConfig) (*Point, error) {
	var vpoint = new(Point)
	vpoint.port = pConfig.Port()

	ichFactory, ok := inboundFactories[pConfig.InboundConfig().Protocol()]
	if !ok {
		log.Error("Unknown inbound connection handler factory %s", pConfig.InboundConfig().Protocol())
		return nil, config.BadConfiguration
	}
	ichConfig := pConfig.InboundConfig().Settings(config.TypeInbound)
	ich, err := ichFactory.Create(vpoint, ichConfig)
	if err != nil {
		log.Error("Failed to create inbound connection handler: %v", err)
		return nil, err
	}
	vpoint.ich = ich

	ochFactory, ok := outboundFactories[pConfig.OutboundConfig().Protocol()]
	if !ok {
		log.Error("Unknown outbound connection handler factory %s", pConfig.OutboundConfig().Protocol())
		return nil, config.BadConfiguration
	}
	ochConfig := pConfig.OutboundConfig().Settings(config.TypeOutbound)
	och, err := ochFactory.Create(vpoint, ochConfig)
	if err != nil {
		log.Error("Failed to create outbound connection handler: %v", err)
		return nil, err
	}
	vpoint.och = och

	return vpoint, nil
}

type InboundConnectionHandlerFactory interface {
	Create(vp *Point, config interface{}) (InboundConnectionHandler, error)
}

type InboundConnectionHandler interface {
	Listen(port uint16) error
}

type OutboundConnectionHandlerFactory interface {
	Create(VP *Point, config interface{}) (OutboundConnectionHandler, error)
}

type OutboundConnectionHandler interface {
	Dispatch(firstPacket v2net.Packet, ray OutboundRay) error
}

// Start starts the Point server, and return any error during the process.
// In the case of any errors, the state of the server is unpredicatable.
func (vp *Point) Start() error {
	if vp.port <= 0 {
		log.Error("Invalid port %d", vp.port)
		return config.BadConfiguration
	}

	return retry.Timed(100 /* times */, 100 /* ms */).On(func() error {
		err := vp.ich.Listen(vp.port)
    if err == nil {
      log.Warning("Point server started on port %d", vp.port)
      return nil
    }
    return err
	})
}

func (p *Point) DispatchToOutbound(packet v2net.Packet) InboundRay {
	ray := NewRay()
	go p.och.Dispatch(packet, ray)
	return ray
}
<|endoftext|>


<commit_before>package main

import (
	"bytes"
	"io/ioutil"
	"log"
	"regexp"
	"strconv"
	"strings"
	"text/template"
	"time"

	"github.com/russross/blackfriday"
)

type Post struct {
	Author    string
	Date      time.Time
	Title     string
	Content   string
	Preview   string
	Template  string
	Permalink string
	Comments  bool
}

// Implement the sort.Interface for []Post by Date
type ByDate []Post

func (a ByDate) Len() int           { return len(a) }
func (a ByDate) Swap(i, j int)      { a[i], a[j] = a[j], a[i] }
func (a ByDate) Less(i, j int) bool { return a[i].Date.After(a[j].Date) }

func loadPost(filename string) Post {
	var post Post
	const dateFormat = "01-02-2006 15:04"

	// Read the post file
	content, err := ioutil.ReadFile(filename)
	if err != nil {
		log.Fatal(err)
	}

	// Extract metadata and content
	x := strings.Split(string(content), "\n\n")
	metadata := x[0]

	// Convert the markdown to HTML
	post.Content = string(blackfriday.MarkdownCommon([]byte(strings.Trim(strings.Join(x[1:], "\n\n"), "\n"))))

	// Get the excerpt
	words := strings.Split(post.Content, " ")
	if len(words) <= site.Config.PreviewLength {
		post.Preview = post.Content
	} else {
		post.Preview = strings.Join(words[:site.Config.PreviewLength], " ") + "..."
	}

	// Process metadata
	rauthor := regexp.MustCompile(`Author: (.*)`)
	rdate := regexp.MustCompile(`Date: (.*)`)
	rtitle := regexp.MustCompile(`Title: (.*)`)
	rtemplate := regexp.MustCompile(`Template: (.*)`)
	rcomments := regexp.MustCompile(`Comments: (.*)`)

	if author := rauthor.FindStringSubmatch(metadata); author != nil {
		post.Author = author[1]
	} else {
		log.Fatal("Author not defined for post ", filename)
	}

	if date := rdate.FindStringSubmatch(metadata); date != nil {
		post.Date, _ = time.Parse(dateFormat, date[1])

		// Generate the url
		year := strconv.Itoa(post.Date.Year())
		month := strconv.Itoa(int(post.Date.Month()))
		day := strconv.Itoa(post.Date.Day())
		fname := strings.Split(strings.Split(filename, ".")[0], "/")[1]
		post.Permalink = "/" + year + "/" + month + "/" + day + "/" + fname + "/"
	} else {
		log.Fatal("Date not defined for post ", filename)
	}

	if title := rtitle.FindStringSubmatch(metadata); title != nil {
		post.Title = title[1]
	} else {
		log.Fatal("Title not defined for post ", filename)
	}

	if template := rtemplate.FindStringSubmatch(metadata); template != nil {
		post.Template = template[1]
	} else {
		log.Fatal("Template not defined for post ", filename)
	}

	if comments := rcomments.FindStringSubmatch(metadata); comments != nil {
		if comments[1] == "enabled" {
			post.Comments = true
		}
	}

	return post
}

func (p Post) convertPost() string {
	// Check if enabled comments
	htmlComments := &bytes.Buffer{}
	if p.Comments {
		data := struct {
			DisqusShortname string
			Permalink       string
		}{
			site.Config.DisqusShortname,
			p.Permalink,
		}
		template_file := site.Config.Templates + "/comments.html"
		layout, err := ioutil.ReadFile(template_file)
		if err != nil {
			log.Fatal(err)
		}
		commentsLayout := template.Must(template.New(p.Template).Parse(string(layout)))
		if err := commentsLayout.Execute(htmlComments, data); err != nil {
			log.Fatal(err)
		}
	}

	// Read the post template
	template_file := site.Config.Templates + "/" + p.Template + ".html"
	layout, err := ioutil.ReadFile(template_file)
	if err != nil {
		log.Fatal(err)
	}

	// Run the template
	htmlPost := &bytes.Buffer{}
	data := struct {
		Config   Config
		Post     Post
		Comments string
	}{
		site.Config,
		p,
		htmlComments.String(),
	}
	postLayout := template.Must(template.New(p.Template).Parse(string(layout)))
	if err := postLayout.Execute(htmlPost, data); err != nil {
		log.Fatal(err)
	}

	return htmlPost.String()
}
<commit_msg>Fixed issue #3: Preview strips images and code.<commit_after>package main

import (
	"bytes"
	"io/ioutil"
	"log"
	"regexp"
	"strconv"
	"strings"
	"text/template"
	"time"

	"github.com/russross/blackfriday"
)

type Post struct {
	Author    string
	Date      time.Time
	Title     string
	Content   string
	Preview   string
	Template  string
	Permalink string
	Comments  bool
}

// Implement the sort.Interface for []Post by Date
type ByDate []Post

func (a ByDate) Len() int           { return len(a) }
func (a ByDate) Swap(i, j int)      { a[i], a[j] = a[j], a[i] }
func (a ByDate) Less(i, j int) bool { return a[i].Date.After(a[j].Date) }

func loadPost(filename string) Post {
	var post Post
	const dateFormat = "01-02-2006 15:04"

	// Read the post file
	content, err := ioutil.ReadFile(filename)
	if err != nil {
		log.Fatal(err)
	}

	// Extract metadata and content
	x := strings.Split(string(content), "\n\n")
	metadata := x[0]

	// Convert the markdown to HTML
	post.Content = string(blackfriday.MarkdownCommon([]byte(strings.Trim(strings.Join(x[1:], "\n\n"), "\n"))))

	// Get the preview
	rtags := regexp.MustCompile(`<(pre|code|img).*>(.|\s)*?(</(pre|code|img)>)+`)
	stripped := rtags.ReplaceAllString(post.Content, "[...]")

	words := strings.Split(stripped, " ")
	if len(words) <= site.Config.PreviewLength {
		post.Preview = stripped
	} else {
		post.Preview = strings.Join(words[:site.Config.PreviewLength], " ") + "..."
	}

	// Process metadata
	rauthor := regexp.MustCompile(`Author: (.*)`)
	rdate := regexp.MustCompile(`Date: (.*)`)
	rtitle := regexp.MustCompile(`Title: (.*)`)
	rtemplate := regexp.MustCompile(`Template: (.*)`)
	rcomments := regexp.MustCompile(`Comments: (.*)`)

	if author := rauthor.FindStringSubmatch(metadata); author != nil {
		post.Author = author[1]
	} else {
		log.Fatal("Author not defined for post ", filename)
	}

	if date := rdate.FindStringSubmatch(metadata); date != nil {
		post.Date, _ = time.Parse(dateFormat, date[1])

		// Generate the url
		year := strconv.Itoa(post.Date.Year())
		month := strconv.Itoa(int(post.Date.Month()))
		day := strconv.Itoa(post.Date.Day())
		fname := strings.Split(strings.Split(filename, ".")[0], "/")[1]
		post.Permalink = "/" + year + "/" + month + "/" + day + "/" + fname + "/"
	} else {
		log.Fatal("Date not defined for post ", filename)
	}

	if title := rtitle.FindStringSubmatch(metadata); title != nil {
		post.Title = title[1]
	} else {
		log.Fatal("Title not defined for post ", filename)
	}

	if template := rtemplate.FindStringSubmatch(metadata); template != nil {
		post.Template = template[1]
	} else {
		log.Fatal("Template not defined for post ", filename)
	}

	if comments := rcomments.FindStringSubmatch(metadata); comments != nil {
		if comments[1] == "enabled" {
			post.Comments = true
		}
	}

	return post
}

func (p Post) convertPost() string {
	// Check if enabled comments
	htmlComments := &bytes.Buffer{}
	if p.Comments {
		data := struct {
			DisqusShortname string
			Permalink       string
		}{
			site.Config.DisqusShortname,
			p.Permalink,
		}
		template_file := site.Config.Templates + "/comments.html"
		layout, err := ioutil.ReadFile(template_file)
		if err != nil {
			log.Fatal(err)
		}
		commentsLayout := template.Must(template.New(p.Template).Parse(string(layout)))
		if err := commentsLayout.Execute(htmlComments, data); err != nil {
			log.Fatal(err)
		}
	}

	// Read the post template
	template_file := site.Config.Templates + "/" + p.Template + ".html"
	layout, err := ioutil.ReadFile(template_file)
	if err != nil {
		log.Fatal(err)
	}

	// Run the template
	htmlPost := &bytes.Buffer{}
	data := struct {
		Config   Config
		Post     Post
		Comments string
	}{
		site.Config,
		p,
		htmlComments.String(),
	}
	postLayout := template.Must(template.New(p.Template).Parse(string(layout)))
	if err := postLayout.Execute(htmlPost, data); err != nil {
		log.Fatal(err)
	}

	return htmlPost.String()
}
<|endoftext|>


<commit_before>package main

import (
	"fmt"
	"os"
	"strings"
	"sync"

	"github.com/urfave/cli"
)

var cmdPrint = cli.Command{
	Name:  "print",
	Usage: "Prints the prompt with the specified syntax",
	// TODO: improve description
	Description: "Example: bronze print dir:blue:black",
	Action: func(ctx *cli.Context) error {
		if len(ctx.Args()) == 0 {
			cli.ShowCommandHelpAndExit(ctx, "print", 1)
		}

		cmdPrintAction(ctx.Args())
		return nil
	},
}

type segment struct {
	background string
	foreground string
	value      string
}

func cmdPrintAction(args []string) {
	var segments []*segment
	waitgroup := new(sync.WaitGroup)
	waitgroup.Add(len(args))

	for _, arg := range args {
		// validate argument
		fields := strings.Split(arg, ":")
		if len(fields) != 3 {
			fmt.Fprintf(os.Stderr, "bronze: Invalid argument: %q. Exactly three fields expected.\n", arg)
			os.Exit(1)
		}

		segment := &segment{
			background: fields[1],
			foreground: fields[2],
		}
		segments = append(segments, segment)

		go func() {
			handleModule(fields[0], segment)
			waitgroup.Done()
		}()
	}

	// wait for all the async segments
	waitgroup.Wait()

	// print the prompt
	first := true
	lastSegment := &segment{
		background: "black",
		foreground: "white",
	}
	for _, segment := range segments {
		if segment.value == "" {
			continue
		}

		// if this isn't the first segment, before printing the next segment, separate them
		if !first {
			if segment.background == lastSegment.background {
				printSegment(segment.background, lastSegment.foreground, "\ue0b1")
			} else {
				// use the last background as the current foreground
				printSegment(segment.background, lastSegment.background, separator)
			}
		}
		first = false

		printSegment(segment.background, segment.foreground, " "+segment.value+" ")
		lastSegment = segment
	}
	// print final separator
	printSegment("none", lastSegment.background, separator)
	resetColors()
}

func printSegment(background, foreground, value string) {
	fmt.Print(escapeBackground(background) + escapeForeground(foreground) + value)
}
<commit_msg>fixed using thin separator constant<commit_after>package main

import (
	"fmt"
	"os"
	"strings"
	"sync"

	"github.com/urfave/cli"
)

var cmdPrint = cli.Command{
	Name:  "print",
	Usage: "Prints the prompt with the specified syntax",
	// TODO: improve description
	Description: "Example: bronze print dir:blue:black",
	Action: func(ctx *cli.Context) error {
		if len(ctx.Args()) == 0 {
			cli.ShowCommandHelpAndExit(ctx, "print", 1)
		}

		cmdPrintAction(ctx.Args())
		return nil
	},
}

type segment struct {
	background string
	foreground string
	value      string
}

func cmdPrintAction(args []string) {
	var segments []*segment
	waitgroup := new(sync.WaitGroup)
	waitgroup.Add(len(args))

	for _, arg := range args {
		// validate argument
		fields := strings.Split(arg, ":")
		if len(fields) != 3 {
			fmt.Fprintf(os.Stderr, "bronze: Invalid argument: %q. Exactly three fields expected.\n", arg)
			os.Exit(1)
		}

		segment := &segment{
			background: fields[1],
			foreground: fields[2],
		}
		segments = append(segments, segment)

		go func() {
			handleModule(fields[0], segment)
			waitgroup.Done()
		}()
	}

	// wait for all the async segments
	waitgroup.Wait()

	// print the prompt
	first := true
	lastSegment := &segment{
		background: "black",
		foreground: "white",
	}
	for _, segment := range segments {
		if segment.value == "" {
			continue
		}

		// if this isn't the first segment, before printing the next segment, separate them
		if !first {
			if segment.background == lastSegment.background {
				printSegment(segment.background, lastSegment.foreground, thinSeparator)
			} else {
				// use the last background as the current foreground
				printSegment(segment.background, lastSegment.background, separator)
			}
		}
		first = false

		printSegment(segment.background, segment.foreground, " "+segment.value+" ")
		lastSegment = segment
	}
	// print final separator
	printSegment("none", lastSegment.background, separator)
	resetColors()
}

func printSegment(background, foreground, value string) {
	fmt.Print(escapeBackground(background) + escapeForeground(foreground) + value)
}
<|endoftext|>


<commit_before>package main

import "bufio"
import "container/list"
import "io"
import "net"
import "strings"

func handleHTTPConnection(downstream net.Conn) {
    reader := bufio.NewReader(downstream)
    hostname := ""
    readLines := list.New()
    for hostname == "" {
        bytes, _, error := reader.ReadLine()
        if error != nil {
            println("Error reading")
            return
        }
        line := string(bytes)
        readLines.PushBack(line)
        if strings.HasPrefix(line, "Host: ") {
            hostname = strings.TrimPrefix(line, "Host: ")
            break
        }
    }
    if hostname == "" {
        println("No host!")
        return
    }

    upstream, error := net.Dial("tcp", hostname + ":80")
    if error != nil {
        println("Couldn't connect to upstream", error)
        return
    }

    for element := readLines.Front(); element != nil; element = element.Next() {
        line := element.Value.(string)
        upstream.Write([]byte(line))
        upstream.Write([]byte("\n"))
    }

    go io.Copy(upstream, reader)
    go io.Copy(downstream, upstream)
}

func doHTTPProxy(done chan int) {
    listener, error := net.Listen("tcp", "0.0.0.0:80")
    if error != nil {
        println("Couldn't start listening", error)
    }
    println("Started, listening...")
    for {
        connection, error := listener.Accept()
        if error != nil {
            println("Accept error", error)
            return
        }

        go handleHTTPConnection(connection)
    }
    done <- 1
}

func main() {
    httpDone := make(chan int)
    go doHTTPProxy(httpDone)

    <- httpDone
}
<commit_msg>Parameterised handler function for proxy so we don't need a dupe for SSL<commit_after>package main

import "bufio"
import "container/list"
import "io"
import "net"
import "strings"

func handleHTTPConnection(downstream net.Conn) {
    reader := bufio.NewReader(downstream)
    hostname := ""
    readLines := list.New()
    for hostname == "" {
        bytes, _, error := reader.ReadLine()
        if error != nil {
            println("Error reading")
            return
        }
        line := string(bytes)
        readLines.PushBack(line)
        if strings.HasPrefix(line, "Host: ") {
            hostname = strings.TrimPrefix(line, "Host: ")
            break
        }
    }
    if hostname == "" {
        println("No host!")
        return
    }

    upstream, error := net.Dial("tcp", hostname + ":80")
    if error != nil {
        println("Couldn't connect to upstream", error)
        return
    }

    for element := readLines.Front(); element != nil; element = element.Next() {
        line := element.Value.(string)
        upstream.Write([]byte(line))
        upstream.Write([]byte("\n"))
    }

    go io.Copy(upstream, reader)
    go io.Copy(downstream, upstream)
}

func doHTTPProxy(done chan int, handle func(net.Conn)) {
    listener, error := net.Listen("tcp", "0.0.0.0:80")
    if error != nil {
        println("Couldn't start listening", error)
    }
    println("Started, listening...")
    for {
        connection, error := listener.Accept()
        if error != nil {
            println("Accept error", error)
            return
        }

        go handle(connection)
    }
    done <- 1
}

func main() {
    httpDone := make(chan int)
    go doHTTPProxy(httpDone, handleHTTPConnection)

    <- httpDone
}
<|endoftext|>


<commit_before>package main

import (
	"bytes"
	"crypto/tls"
	"log"
	"net/url"
	"runtime/debug"
	"strings"
	"sync"
	"time"

	"github.com/valyala/bytebufferpool"
	"github.com/valyala/fasthttp"
	"github.com/xtrafrancyz/vk-proxy/bytefmt"
	"github.com/xtrafrancyz/vk-proxy/replacer"
)

const (
	readBufferSize = 8192

	byteBufferPoolSetupRounds = 42000   // == bytebufferpool.calibrateCallsThreshold
	byteBufferPoolSetupSize   = 2097152 // 2**21
)

func init() {
	// Настройка размера буфера для ответа
	r := fasthttp.Response{}
	for i := 0; i <= byteBufferPoolSetupRounds; i++ {
		r.SetBodyString("")
		if b := r.Body(); cap(b) != byteBufferPoolSetupSize {
			r.SwapBody(make([]byte, byteBufferPoolSetupSize))
		} else {
			r.SwapBody(b[:cap(b)])
		}
		r.ResetBody()
	}

	// Настройка размера реплейсера
	for i := 0; i <= byteBufferPoolSetupRounds; i++ {
		b := replacer.AcquireBuffer()
		if cap(b.B) != byteBufferPoolSetupSize {
			b.B = make([]byte, byteBufferPoolSetupSize)
		} else {
			b.B = b.B[:cap(b.B)]
		}
		replacer.ReleaseBuffer(b)
	}
}

var (
	gzip            = []byte("gzip")
	vkProxyName     = []byte("vk-proxy")
	serverHeader    = []byte("Server")
	setCookie       = []byte("Set-Cookie")
	acceptEncoding  = []byte("Accept-Encoding")
	contentEncoding = []byte("Content-Encoding")
)

type ProxyConfig struct {
	ReduceMemoryUsage bool
	BaseDomain        string
	BaseStaticDomain  string
	LogVerbosity      int
	GzipUpstream      bool
	FilterFeed        bool
}

type Proxy struct {
	client   *fasthttp.Client
	server   *fasthttp.Server
	replacer *replacer.Replacer
	tracker  *tracker
	config   ProxyConfig
}

func NewProxy(config ProxyConfig) *Proxy {
	p := &Proxy{
		client: &fasthttp.Client{
			Name:           "vk-proxy",
			ReadBufferSize: readBufferSize,
			TLSConfig:      &tls.Config{InsecureSkipVerify: true},
			ReadTimeout:    30 * time.Second,
			WriteTimeout:   10 * time.Second,
			RetryIf: func(request *fasthttp.Request) bool {
				return false
			},
		},
		replacer: &replacer.Replacer{
			ProxyBaseDomain:   config.BaseDomain,
			ProxyStaticDomain: config.BaseStaticDomain,
		},
		tracker: &tracker{
			uniqueUsers: make(map[string]bool),
		},
		config: config,
	}
	p.server = &fasthttp.Server{
		Handler:           p.handleProxy,
		ReduceMemoryUsage: config.ReduceMemoryUsage,
		ReadBufferSize:    readBufferSize,
		Name:              "vk-proxy",
	}
	if p.config.LogVerbosity > 0 {
		p.tracker.start()
	}
	return p
}

func (p *Proxy) ListenTCP(host string) error {
	log.Printf("Starting server on http://%s", host)
	return p.server.ListenAndServe(host)
}

func (p *Proxy) ListenUnix(path string) error {
	log.Printf("Starting server on http://unix:%s", path)
	return p.server.ListenAndServeUNIX(path, 0777)
}

func (p *Proxy) handleProxy(ctx *fasthttp.RequestCtx) {
	defer func() {
		if r := recover(); r != nil {
			log.Printf("panic when proxying the request: %s%s", r, debug.Stack())
			ctx.Error("500 Internal Server Error", 500)
		}
	}()
	start := time.Now()

	replaceContext := &replacer.ReplaceContext{
		Method:     ctx.Method(),
		OriginHost: string(ctx.Request.Host()),
		FilterFeed: p.config.FilterFeed,
	}

	if !p.prepareProxyRequest(ctx, replaceContext) {
		ctx.Error("400 Bad Request", 400)
		return
	}

	if replaceContext.Host == "api.vk.com" &&
		(replaceContext.Path == "/away" || replaceContext.Path == "/away.php") {
		p.handleAway(ctx)
		return
	}

	err := p.client.Do(&ctx.Request, &ctx.Response)
	if err == nil {
		err = p.processProxyResponse(ctx, replaceContext)
	}

	elapsed := time.Since(start).Round(100 * time.Microsecond)

	if err != nil {
		log.Printf("%s %s %s%s error: %s", elapsed, ctx.Request.Header.Method(), ctx.Host(), ctx.Path(), err)
		if strings.Contains(err.Error(), "timed out") || strings.Contains(err.Error(), "timeout") {
			ctx.Error("408 Request Timeout", 408)
		} else {
			ctx.Error("500 Internal Server Error", 500)
		}
		return
	}

	if p.config.LogVerbosity > 0 {
		ip := ctx.Request.Header.Peek("CF-Connecting-IP") // Cloudflare
		if ip == nil {
			ip = ctx.Request.Header.Peek("X-Real-IP") // nginx
		}
		if ip == nil {
			ip = []byte(ctx.RemoteIP().String()) // real
		}
		p.tracker.trackRequest(string(ip), len(ctx.Response.Body()))
	}

	if p.config.LogVerbosity == 2 {
		log.Printf("%s %s %s%s %s", elapsed, ctx.Request.Header.Method(), ctx.Host(), ctx.Path(),
			bytefmt.ByteSize(uint64(len(ctx.Response.Body()))))
	} else if p.config.LogVerbosity == 3 {
		log.Printf("%s %s %s%s %s\n%s", elapsed, ctx.Request.Header.Method(), ctx.Host(), ctx.Path(),
			bytefmt.ByteSize(uint64(len(ctx.Response.Body()))), ctx.Response.Body())
	}
}

func (p *Proxy) handleAway(ctx *fasthttp.RequestCtx) {
	to := string(ctx.QueryArgs().Peek("to"))
	if to == "" {
		ctx.Error("Bad Request: 'to' argument is not set", 400)
		return
	}
	to, err := url.QueryUnescape(to)
	if err != nil {
		ctx.Error("Bad Request: could not unescape url", 400)
		return
	}
	ctx.Redirect(to, fasthttp.StatusMovedPermanently)
}

func (p *Proxy) prepareProxyRequest(ctx *fasthttp.RequestCtx, replaceContext *replacer.ReplaceContext) bool {
	// Routing
	req := &ctx.Request
	uri := string(req.RequestURI())
	host := ""
	if strings.HasPrefix(uri, "/@") {
		slashIndex := strings.IndexByte(uri[2:], '/')
		if slashIndex == -1 {
			return false
		}
		endpoint := uri[2 : slashIndex+2]
		if endpoint != "vk.com" &&
			!strings.HasSuffix(endpoint, ".vk.com") &&
			!strings.HasSuffix(endpoint, ".vkuseraudio.net") &&
			!strings.HasSuffix(endpoint, ".vkuseraudio.com") {
			return false
		}
		host = endpoint
		uri = uri[2+slashIndex:]
		req.SetRequestURI(uri)
	} else if altHost := req.Header.Peek("Proxy-Host"); altHost != nil {
		host = string(altHost)
		switch host {
		case "static.vk.com":
		case "oauth.vk.com":
		default:
			return false
		}
		req.Header.Del("Proxy-Host")
	} else {
		host = "api.vk.com"
	}
	req.SetHost(host)

	// Replace some request data
	replaceContext.Host = host
	replaceContext.Path = string(ctx.Path())
	p.replacer.DoReplaceRequest(req, replaceContext)

	// After req.URI() call it is impossible to modify URI
	req.URI().SetScheme("https")
	if p.config.GzipUpstream {
		req.Header.SetBytesKV(acceptEncoding, gzip)
	} else {
		req.Header.DelBytes(acceptEncoding)
	}
	return true
}

func (p *Proxy) processProxyResponse(ctx *fasthttp.RequestCtx, replaceContext *replacer.ReplaceContext) error {
	res := &ctx.Response
	res.Header.DelBytes(setCookie)
	res.Header.SetBytesKV(serverHeader, vkProxyName)

	var buf *bytebufferpool.ByteBuffer
	// Gunzip body if needed
	if bytes.Contains(res.Header.PeekBytes(contentEncoding), gzip) {
		res.Header.DelBytes(contentEncoding)
		buf = replacer.AcquireBuffer()
		_, err := fasthttp.WriteGunzip(buf, res.Body())
		if err != nil {
			replacer.ReleaseBuffer(buf)
			return err
		}
		replacer.ReleaseBuffer(&bytebufferpool.ByteBuffer{
			B: res.SwapBody(nil),
		})
	} else {
		buf = &bytebufferpool.ByteBuffer{
			B: res.SwapBody(nil),
		}
	}

	buf = p.replacer.DoReplaceResponse(res, buf, replaceContext)

	// avoid copying and save old buffer
	buf.B = res.SwapBody(buf.B)
	if cap(buf.B) > 10 {
		replacer.ReleaseBuffer(buf)
	}
	return nil
}

type tracker struct {
	lock        sync.Mutex
	requests    uint32
	bytes       uint64
	uniqueUsers map[string]bool
}

func (t *tracker) start() {
	go func() {
		for range time.Tick(60 * time.Second) {
			t.lock.Lock()
			log.Printf("Requests: %d, Traffic: %s, Online: %d", t.requests, bytefmt.ByteSize(t.bytes), len(t.uniqueUsers))
			t.requests = 0
			t.bytes = 0
			t.uniqueUsers = make(map[string]bool)
			t.lock.Unlock()
		}
	}()
}

func (t *tracker) trackRequest(ip string, size int) {
	t.lock.Lock()

	t.uniqueUsers[ip] = true
	t.requests++
	t.bytes += uint64(size)

	t.lock.Unlock()
}
<commit_msg>В лог пишется текущее количество запросов в обработке<commit_after>package main

import (
	"bytes"
	"crypto/tls"
	"log"
	"net/url"
	"runtime/debug"
	"strings"
	"sync"
	"time"

	"github.com/valyala/bytebufferpool"
	"github.com/valyala/fasthttp"
	"github.com/xtrafrancyz/vk-proxy/bytefmt"
	"github.com/xtrafrancyz/vk-proxy/replacer"
)

const (
	readBufferSize = 8192

	byteBufferPoolSetupRounds = 42000   // == bytebufferpool.calibrateCallsThreshold
	byteBufferPoolSetupSize   = 2097152 // 2**21
)

func init() {
	// Настройка размера буфера для ответа
	r := fasthttp.Response{}
	for i := 0; i <= byteBufferPoolSetupRounds; i++ {
		r.SetBodyString("")
		if b := r.Body(); cap(b) != byteBufferPoolSetupSize {
			r.SwapBody(make([]byte, byteBufferPoolSetupSize))
		} else {
			r.SwapBody(b[:cap(b)])
		}
		r.ResetBody()
	}

	// Настройка размера реплейсера
	for i := 0; i <= byteBufferPoolSetupRounds; i++ {
		b := replacer.AcquireBuffer()
		if cap(b.B) != byteBufferPoolSetupSize {
			b.B = make([]byte, byteBufferPoolSetupSize)
		} else {
			b.B = b.B[:cap(b.B)]
		}
		replacer.ReleaseBuffer(b)
	}
}

var (
	gzip            = []byte("gzip")
	vkProxyName     = []byte("vk-proxy")
	serverHeader    = []byte("Server")
	setCookie       = []byte("Set-Cookie")
	acceptEncoding  = []byte("Accept-Encoding")
	contentEncoding = []byte("Content-Encoding")
)

type ProxyConfig struct {
	ReduceMemoryUsage bool
	BaseDomain        string
	BaseStaticDomain  string
	LogVerbosity      int
	GzipUpstream      bool
	FilterFeed        bool
}

type Proxy struct {
	client   *fasthttp.Client
	server   *fasthttp.Server
	replacer *replacer.Replacer
	tracker  *tracker
	config   ProxyConfig
}

func NewProxy(config ProxyConfig) *Proxy {
	p := &Proxy{
		client: &fasthttp.Client{
			Name:           "vk-proxy",
			ReadBufferSize: readBufferSize,
			TLSConfig:      &tls.Config{InsecureSkipVerify: true},
			ReadTimeout:    30 * time.Second,
			WriteTimeout:   10 * time.Second,
			RetryIf: func(request *fasthttp.Request) bool {
				return false
			},
		},
		replacer: &replacer.Replacer{
			ProxyBaseDomain:   config.BaseDomain,
			ProxyStaticDomain: config.BaseStaticDomain,
		},
		tracker: &tracker{
			uniqueUsers: make(map[string]bool),
		},
		config: config,
	}
	p.server = &fasthttp.Server{
		Handler:           p.handleProxy,
		ReduceMemoryUsage: config.ReduceMemoryUsage,
		ReadBufferSize:    readBufferSize,
		Name:              "vk-proxy",
	}
	p.tracker.server = p.server
	if p.config.LogVerbosity > 0 {
		p.tracker.start()
	}
	return p
}

func (p *Proxy) ListenTCP(host string) error {
	log.Printf("Starting server on http://%s", host)
	return p.server.ListenAndServe(host)
}

func (p *Proxy) ListenUnix(path string) error {
	log.Printf("Starting server on http://unix:%s", path)
	return p.server.ListenAndServeUNIX(path, 0777)
}

func (p *Proxy) handleProxy(ctx *fasthttp.RequestCtx) {
	defer func() {
		if r := recover(); r != nil {
			log.Printf("panic when proxying the request: %s%s", r, debug.Stack())
			ctx.Error("500 Internal Server Error", 500)
		}
	}()
	start := time.Now()

	replaceContext := &replacer.ReplaceContext{
		Method:     ctx.Method(),
		OriginHost: string(ctx.Request.Host()),
		FilterFeed: p.config.FilterFeed,
	}

	if !p.prepareProxyRequest(ctx, replaceContext) {
		ctx.Error("400 Bad Request", 400)
		return
	}

	if replaceContext.Host == "api.vk.com" &&
		(replaceContext.Path == "/away" || replaceContext.Path == "/away.php") {
		p.handleAway(ctx)
		return
	}

	err := p.client.Do(&ctx.Request, &ctx.Response)
	if err == nil {
		err = p.processProxyResponse(ctx, replaceContext)
	}

	elapsed := time.Since(start).Round(100 * time.Microsecond)

	if err != nil {
		log.Printf("%s %s %s%s error: %s", elapsed, ctx.Request.Header.Method(), ctx.Host(), ctx.Path(), err)
		if strings.Contains(err.Error(), "timed out") || strings.Contains(err.Error(), "timeout") {
			ctx.Error("408 Request Timeout", 408)
		} else {
			ctx.Error("500 Internal Server Error", 500)
		}
		return
	}

	if p.config.LogVerbosity > 0 {
		ip := ctx.Request.Header.Peek("CF-Connecting-IP") // Cloudflare
		if ip == nil {
			ip = ctx.Request.Header.Peek("X-Real-IP") // nginx
		}
		if ip == nil {
			ip = []byte(ctx.RemoteIP().String()) // real
		}
		p.tracker.trackRequest(string(ip), len(ctx.Response.Body()))
	}

	if p.config.LogVerbosity == 2 {
		log.Printf("%s %s %s%s %s", elapsed, ctx.Request.Header.Method(), ctx.Host(), ctx.Path(),
			bytefmt.ByteSize(uint64(len(ctx.Response.Body()))))
	} else if p.config.LogVerbosity == 3 {
		log.Printf("%s %s %s%s %s\n%s", elapsed, ctx.Request.Header.Method(), ctx.Host(), ctx.Path(),
			bytefmt.ByteSize(uint64(len(ctx.Response.Body()))), ctx.Response.Body())
	}
}

func (p *Proxy) handleAway(ctx *fasthttp.RequestCtx) {
	to := string(ctx.QueryArgs().Peek("to"))
	if to == "" {
		ctx.Error("Bad Request: 'to' argument is not set", 400)
		return
	}
	to, err := url.QueryUnescape(to)
	if err != nil {
		ctx.Error("Bad Request: could not unescape url", 400)
		return
	}
	ctx.Redirect(to, fasthttp.StatusMovedPermanently)
}

func (p *Proxy) prepareProxyRequest(ctx *fasthttp.RequestCtx, replaceContext *replacer.ReplaceContext) bool {
	// Routing
	req := &ctx.Request
	uri := string(req.RequestURI())
	host := ""
	if strings.HasPrefix(uri, "/@") {
		slashIndex := strings.IndexByte(uri[2:], '/')
		if slashIndex == -1 {
			return false
		}
		endpoint := uri[2 : slashIndex+2]
		if endpoint != "vk.com" &&
			!strings.HasSuffix(endpoint, ".vk.com") &&
			!strings.HasSuffix(endpoint, ".vkuseraudio.net") &&
			!strings.HasSuffix(endpoint, ".vkuseraudio.com") {
			return false
		}
		host = endpoint
		uri = uri[2+slashIndex:]
		req.SetRequestURI(uri)
	} else if altHost := req.Header.Peek("Proxy-Host"); altHost != nil {
		host = string(altHost)
		switch host {
		case "static.vk.com":
		case "oauth.vk.com":
		default:
			return false
		}
		req.Header.Del("Proxy-Host")
	} else {
		host = "api.vk.com"
	}
	req.SetHost(host)

	// Replace some request data
	replaceContext.Host = host
	replaceContext.Path = string(ctx.Path())
	p.replacer.DoReplaceRequest(req, replaceContext)

	// After req.URI() call it is impossible to modify URI
	req.URI().SetScheme("https")
	if p.config.GzipUpstream {
		req.Header.SetBytesKV(acceptEncoding, gzip)
	} else {
		req.Header.DelBytes(acceptEncoding)
	}
	return true
}

func (p *Proxy) processProxyResponse(ctx *fasthttp.RequestCtx, replaceContext *replacer.ReplaceContext) error {
	res := &ctx.Response
	res.Header.DelBytes(setCookie)
	res.Header.SetBytesKV(serverHeader, vkProxyName)

	var buf *bytebufferpool.ByteBuffer
	// Gunzip body if needed
	if bytes.Contains(res.Header.PeekBytes(contentEncoding), gzip) {
		res.Header.DelBytes(contentEncoding)
		buf = replacer.AcquireBuffer()
		_, err := fasthttp.WriteGunzip(buf, res.Body())
		if err != nil {
			replacer.ReleaseBuffer(buf)
			return err
		}
		replacer.ReleaseBuffer(&bytebufferpool.ByteBuffer{
			B: res.SwapBody(nil),
		})
	} else {
		buf = &bytebufferpool.ByteBuffer{
			B: res.SwapBody(nil),
		}
	}

	buf = p.replacer.DoReplaceResponse(res, buf, replaceContext)

	// avoid copying and save old buffer
	buf.B = res.SwapBody(buf.B)
	if cap(buf.B) > 10 {
		replacer.ReleaseBuffer(buf)
	}
	return nil
}

type tracker struct {
	lock        sync.Mutex
	requests    uint32
	bytes       uint64
	uniqueUsers map[string]bool
	server      *fasthttp.Server
}

func (t *tracker) start() {
	go func() {
		for range time.Tick(60 * time.Second) {
			t.lock.Lock()
			log.Printf("Requests: %d, Traffic: %s, Online: %d, Concurrency: %d",
				t.requests, bytefmt.ByteSize(t.bytes), len(t.uniqueUsers),
				t.server.GetCurrentConcurrency(),
			)
			t.requests = 0
			t.bytes = 0
			t.uniqueUsers = make(map[string]bool)
			t.lock.Unlock()
		}
	}()
}

func (t *tracker) trackRequest(ip string, size int) {
	t.lock.Lock()

	t.uniqueUsers[ip] = true
	t.requests++
	t.bytes += uint64(size)

	t.lock.Unlock()
}
<|endoftext|>


<commit_before>package main

import (
	"code.google.com/p/go.net/html/charset"
	"compress/gzip"
	"crypto/tls"
	"errors"
	"flag"
	"fmt"
	"io"
	"io/ioutil"
	"log"
	"net"
	"net/http"
	"strings"
	"time"
)

var disableGzip = flag.Bool("disable-gzip", false, "Don't compress HTTP responses with gzip.")

type proxyHandler struct {
	// TLS is whether this is an HTTPS connection.
	TLS bool

	// connectPort is the server port that was specified in a CONNECT request.
	connectPort string

	// user is a user that has already been authenticated.
	user string

	// rt is the RoundTripper that will be used to fulfill the requests.
	// If it is nil, a default Transport will be used.
	rt http.RoundTripper
}

// lanAddress returns whether addr is in one of the LAN address ranges.
func lanAddress(addr string) bool {
	ip := net.ParseIP(addr)
	if ip4 := ip.To4(); ip4 != nil {
		switch ip4[0] {
		case 10, 127:
			return true
		case 172:
			return ip4[1]&0xf0 == 16
		case 192:
			return ip4[1] == 168
		}
		return false
	}

	if ip[0]&0xfe == 0xfc {
		return true
	}
	if ip[0] == 0xfe && (ip[1]&0xfc) == 0x80 {
		return true
	}

	return false
}

func (h proxyHandler) ServeHTTP(w http.ResponseWriter, r *http.Request) {
	activeConnections.Add(1)
	defer activeConnections.Done()

	if len(r.URL.String()) > 10000 {
		http.Error(w, "URL too long", http.StatusRequestURITooLong)
		return
	}

	client := r.RemoteAddr
	host, _, err := net.SplitHostPort(client)
	if err == nil {
		client = host
	}
	user := client

	if h.user != "" {
		user = h.user
	} else if !*authNever && !h.TLS && (*authAlways || !lanAddress(client)) {
		u := authenticate(w, r)
		if u == "" {
			return
		}
		user = u
	}

	if r.Host == localServer {
		http.DefaultServeMux.ServeHTTP(w, r)
		return
	}

	if r.Method == "CONNECT" {
		if !tlsReady {
			sc := scorecard{
				tally: URLRules.MatchingRules(r.URL),
			}
			sc.calculate(user)
			logAccess(r, nil, sc, "", 0, false, user)
			if sc.action == BLOCK {
				showBlockPage(w, r, &sc, user)
				return
			}
		}

		conn, err := newHijackedConn(w)
		if err != nil {
			fmt.Fprintln(conn, "HTTP/1.1 500 Internal Server Error")
			fmt.Fprintln(conn)
			fmt.Fprintln(conn, err)
			conn.Close()
			return
		}
		fmt.Fprint(conn, "HTTP/1.1 200 Connection Established\r\n\r\n")
		if tlsReady {
			SSLBump(conn, r.URL.Host, user)
		} else {
			connectDirect(conn, r.URL.Host, nil)
		}
		return
	}

	if r.Header.Get("Upgrade") == "websocket" {
		h.makeWebsocketConnection(w, r)
		return
	}

	r.Header.Add("Via", r.Proto+" Redwood")
	r.Header.Add("X-Forwarded-For", client)

	gzipOK := !*disableGzip && strings.Contains(r.Header.Get("Accept-Encoding"), "gzip")
	r.Header.Del("Accept-Encoding")

	// Reconstruct the URL if it is incomplete (i.e. on a transparent proxy).
	if r.URL.Host == "" {
		r.URL.Host = r.Host
	}
	if r.URL.Scheme == "" {
		if h.TLS {
			r.URL.Scheme = "https"
		} else {
			r.URL.Scheme = "http"
		}
	}

	sc := scorecard{
		tally: URLRules.MatchingRules(r.URL),
	}
	sc.calculate(user)
	if sc.action == BLOCK {
		showBlockPage(w, r, &sc, user)
		logAccess(r, nil, sc, "", 0, false, user)
		return
	}

	changeQuery(r.URL)

	var resp *http.Response
	if h.rt == nil {
		resp, err = transport.RoundTrip(r)
	} else {
		resp, err = h.rt.RoundTrip(r)
	}

	if err != nil {
		http.Error(w, err.Error(), http.StatusServiceUnavailable)
		log.Printf("error fetching %s: %s", r.URL, err)
		logAccess(r, nil, sc, "", 0, false, user)
		return
	}
	defer resp.Body.Close()

	contentType, action := checkContentType(resp)

	switch action {
	case BLOCK:
		sc.action = BLOCK
		sc.blocked = []string{"blocked-mime"}
		showBlockPage(w, r, &sc, user)
		logAccess(r, resp, sc, contentType, 0, false, user)
		return

	case ALLOW:
		sc.action = IGNORE
		copyResponseHeader(w, resp)
		io.Copy(w, resp.Body)
		logAccess(r, resp, sc, contentType, 0, false, user)
		return
	}

	lr := &io.LimitedReader{
		R: resp.Body,
		N: 1e7,
	}
	content, err := ioutil.ReadAll(lr)
	if err != nil {
		log.Printf("error while reading response body (URL: %s): %s", r.URL, err)
	}
	if lr.N == 0 {
		log.Println("response body too long to filter:", r.URL)
		copyResponseHeader(w, resp)
		w.Write(content)
		n, err := io.Copy(w, resp.Body)
		if err != nil {
			log.Printf("error while copying response (URL: %s): %s", r.URL, err)
		}
		sc.action = IGNORE
		logAccess(r, resp, sc, contentType, int(n)+len(content), false, user)
		return
	}

	modified := false
	_, cs, _ := charset.DetermineEncoding(content, resp.Header.Get("Content-Type"))
	if strings.Contains(contentType, "html") {
		modified = pruneContent(r.URL, &content, cs)
		if modified {
			resp.Header.Set("Content-Type", "text/html; charset=utf-8")
			cs = "utf-8"
		}
	}

	scanContent(content, contentType, cs, sc.tally)
	sc.calculate(user)

	if sc.action == BLOCK {
		showBlockPage(w, r, &sc, user)
		logAccess(r, resp, sc, contentType, len(content), modified, user)
		return
	}

	if resp.Header.Get("Content-Type") == "" {
		// If the server didn't specify a content type, don't let the http
		// package add one by doing MIME sniffing.
		resp.Header.Set("Content-Type", "")
	}

	if gzipOK && len(content) > 1000 {
		resp.Header.Set("Content-Encoding", "gzip")
		resp.Header.Del("Content-Length")
		copyResponseHeader(w, resp)
		gzw := gzip.NewWriter(w)
		gzw.Write(content)
		gzw.Close()
	} else {
		copyResponseHeader(w, resp)
		w.Write(content)
	}

	logAccess(r, resp, sc, contentType, len(content), modified, user)
}

// copyResponseHeader writes resp's header and status code to w.
func copyResponseHeader(w http.ResponseWriter, resp *http.Response) {
	newHeader := w.Header()
	for key, values := range resp.Header {
		for _, v := range values {
			newHeader.Add(key, v)
		}
	}

	w.WriteHeader(resp.StatusCode)
}

// A hijackedConn is a connection that has been hijacked (to fulfill a CONNECT
// request).
type hijackedConn struct {
	net.Conn
	io.Reader
}

func (hc *hijackedConn) Read(b []byte) (int, error) {
	return hc.Reader.Read(b)
}

func newHijackedConn(w http.ResponseWriter) (*hijackedConn, error) {
	hj, ok := w.(http.Hijacker)
	if !ok {
		return nil, errors.New("connection doesn't support hijacking")
	}
	conn, bufrw, err := hj.Hijack()
	if err != nil {
		return nil, err
	}
	err = bufrw.Flush()
	if err != nil {
		return nil, err
	}
	return &hijackedConn{
		Conn:   conn,
		Reader: bufrw.Reader,
	}, nil
}

var transport = http.Transport{
	TLSClientConfig: unverifiedClientConfig,
	Proxy:           http.ProxyFromEnvironment,
}

// This is to deal with the problem of stale keepalive connections, which cause
// transport.RoundTrip to return io.EOF.
func init() {
	go func() {
		for _ = range time.Tick(10 * time.Second) {
			transport.CloseIdleConnections()
		}
	}()

	transport.RegisterProtocol("ftp", FTPTransport{})
}

func (h proxyHandler) makeWebsocketConnection(w http.ResponseWriter, r *http.Request) {
	addr := r.Host
	if _, _, err := net.SplitHostPort(addr); err != nil {
		// There is no port specified; we need to add it.
		port := h.connectPort
		if port == "" {
			port = "80"
		}
		addr = net.JoinHostPort(addr, port)
	}
	var err error
	var serverConn net.Conn
	if h.TLS {
		serverConn, err = tls.Dial("tcp", addr, unverifiedClientConfig)
	} else {
		serverConn, err = net.Dial("tcp", addr)
	}
	if err != nil {
		http.Error(w, err.Error(), http.StatusInternalServerError)
		return
	}

	err = r.Write(serverConn)
	if err != nil {
		http.Error(w, err.Error(), http.StatusInternalServerError)
		return
	}

	hj, ok := w.(http.Hijacker)
	if !ok {
		http.Error(w, "Couldn't create a websocket connection", http.StatusInternalServerError)
		return
	}
	conn, bufrw, err := hj.Hijack()
	if err != nil {
		http.Error(w, err.Error(), http.StatusInternalServerError)
		return
	}

	go func() {
		io.Copy(conn, serverConn)
		conn.Close()
	}()
	io.Copy(serverConn, bufrw)
	serverConn.Close()
	return
}
<commit_msg>Use the same URL escaping talking to server that client used.<commit_after>package main

import (
	"code.google.com/p/go.net/html/charset"
	"compress/gzip"
	"crypto/tls"
	"errors"
	"flag"
	"fmt"
	"io"
	"io/ioutil"
	"log"
	"net"
	"net/http"
	"strings"
	"time"
)

var disableGzip = flag.Bool("disable-gzip", false, "Don't compress HTTP responses with gzip.")

type proxyHandler struct {
	// TLS is whether this is an HTTPS connection.
	TLS bool

	// connectPort is the server port that was specified in a CONNECT request.
	connectPort string

	// user is a user that has already been authenticated.
	user string

	// rt is the RoundTripper that will be used to fulfill the requests.
	// If it is nil, a default Transport will be used.
	rt http.RoundTripper
}

// lanAddress returns whether addr is in one of the LAN address ranges.
func lanAddress(addr string) bool {
	ip := net.ParseIP(addr)
	if ip4 := ip.To4(); ip4 != nil {
		switch ip4[0] {
		case 10, 127:
			return true
		case 172:
			return ip4[1]&0xf0 == 16
		case 192:
			return ip4[1] == 168
		}
		return false
	}

	if ip[0]&0xfe == 0xfc {
		return true
	}
	if ip[0] == 0xfe && (ip[1]&0xfc) == 0x80 {
		return true
	}

	return false
}

func (h proxyHandler) ServeHTTP(w http.ResponseWriter, r *http.Request) {
	activeConnections.Add(1)
	defer activeConnections.Done()

	if len(r.URL.String()) > 10000 {
		http.Error(w, "URL too long", http.StatusRequestURITooLong)
		return
	}

	client := r.RemoteAddr
	host, _, err := net.SplitHostPort(client)
	if err == nil {
		client = host
	}
	user := client

	if h.user != "" {
		user = h.user
	} else if !*authNever && !h.TLS && (*authAlways || !lanAddress(client)) {
		u := authenticate(w, r)
		if u == "" {
			return
		}
		user = u
	}

	if r.Host == localServer {
		http.DefaultServeMux.ServeHTTP(w, r)
		return
	}

	if r.Method == "CONNECT" {
		if !tlsReady {
			sc := scorecard{
				tally: URLRules.MatchingRules(r.URL),
			}
			sc.calculate(user)
			logAccess(r, nil, sc, "", 0, false, user)
			if sc.action == BLOCK {
				showBlockPage(w, r, &sc, user)
				return
			}
		}

		conn, err := newHijackedConn(w)
		if err != nil {
			fmt.Fprintln(conn, "HTTP/1.1 500 Internal Server Error")
			fmt.Fprintln(conn)
			fmt.Fprintln(conn, err)
			conn.Close()
			return
		}
		fmt.Fprint(conn, "HTTP/1.1 200 Connection Established\r\n\r\n")
		if tlsReady {
			SSLBump(conn, r.URL.Host, user)
		} else {
			connectDirect(conn, r.URL.Host, nil)
		}
		return
	}

	if r.Header.Get("Upgrade") == "websocket" {
		h.makeWebsocketConnection(w, r)
		return
	}

	r.Header.Add("Via", r.Proto+" Redwood")
	r.Header.Add("X-Forwarded-For", client)

	gzipOK := !*disableGzip && strings.Contains(r.Header.Get("Accept-Encoding"), "gzip")
	r.Header.Del("Accept-Encoding")

	// Reconstruct the URL if it is incomplete (i.e. on a transparent proxy).
	if r.URL.Host == "" {
		r.URL.Host = r.Host
	}
	if r.URL.Scheme == "" {
		if h.TLS {
			r.URL.Scheme = "https"
		} else {
			r.URL.Scheme = "http"
		}
	}

	sc := scorecard{
		tally: URLRules.MatchingRules(r.URL),
	}
	sc.calculate(user)
	if sc.action == BLOCK {
		showBlockPage(w, r, &sc, user)
		logAccess(r, nil, sc, "", 0, false, user)
		return
	}

	urlChanged := changeQuery(r.URL)

	if !urlChanged {
		// Rebuild the URL in a way that will preserve which characters are escaped
		// and which aren't, for compatibility with broken servers.
		rawURL := r.RequestURI
		if strings.HasPrefix(rawURL, r.URL.Scheme) {
			rawURL = rawURL[len(r.URL.Scheme):]
			rawURL = strings.TrimPrefix(rawURL, "://")
			slash := strings.Index(rawURL, "/")
			if slash == -1 {
				rawURL = "/"
			} else {
				rawURL = rawURL[slash:]
			}
		}
		q := strings.Index(rawURL, "?")
		if q != -1 {
			rawURL = rawURL[:q]
		}
		r.URL.Opaque = rawURL
	}

	var resp *http.Response
	if h.rt == nil {
		if r.URL.Opaque != "" && transport.Proxy != nil {
			if p, _ := transport.Proxy(r); p != nil {
				// If the request is going through a proxy, the host needs to be
				// included in the opaque element.
				r.URL.Opaque = "//" + r.URL.Host + r.URL.Opaque
			}
		}
		resp, err = transport.RoundTrip(r)
	} else {
		resp, err = h.rt.RoundTrip(r)
	}

	r.URL.Opaque = ""

	if err != nil {
		http.Error(w, err.Error(), http.StatusServiceUnavailable)
		log.Printf("error fetching %s: %s", r.URL, err)
		logAccess(r, nil, sc, "", 0, false, user)
		return
	}
	defer resp.Body.Close()

	contentType, action := checkContentType(resp)

	switch action {
	case BLOCK:
		sc.action = BLOCK
		sc.blocked = []string{"blocked-mime"}
		showBlockPage(w, r, &sc, user)
		logAccess(r, resp, sc, contentType, 0, false, user)
		return

	case ALLOW:
		sc.action = IGNORE
		copyResponseHeader(w, resp)
		io.Copy(w, resp.Body)
		logAccess(r, resp, sc, contentType, 0, false, user)
		return
	}

	lr := &io.LimitedReader{
		R: resp.Body,
		N: 1e7,
	}
	content, err := ioutil.ReadAll(lr)
	if err != nil {
		log.Printf("error while reading response body (URL: %s): %s", r.URL, err)
	}
	if lr.N == 0 {
		log.Println("response body too long to filter:", r.URL)
		copyResponseHeader(w, resp)
		w.Write(content)
		n, err := io.Copy(w, resp.Body)
		if err != nil {
			log.Printf("error while copying response (URL: %s): %s", r.URL, err)
		}
		sc.action = IGNORE
		logAccess(r, resp, sc, contentType, int(n)+len(content), false, user)
		return
	}

	modified := false
	_, cs, _ := charset.DetermineEncoding(content, resp.Header.Get("Content-Type"))
	if strings.Contains(contentType, "html") {
		modified = pruneContent(r.URL, &content, cs)
		if modified {
			resp.Header.Set("Content-Type", "text/html; charset=utf-8")
			cs = "utf-8"
		}
	}

	scanContent(content, contentType, cs, sc.tally)
	sc.calculate(user)

	if sc.action == BLOCK {
		showBlockPage(w, r, &sc, user)
		logAccess(r, resp, sc, contentType, len(content), modified, user)
		return
	}

	if resp.Header.Get("Content-Type") == "" {
		// If the server didn't specify a content type, don't let the http
		// package add one by doing MIME sniffing.
		resp.Header.Set("Content-Type", "")
	}

	if gzipOK && len(content) > 1000 {
		resp.Header.Set("Content-Encoding", "gzip")
		resp.Header.Del("Content-Length")
		copyResponseHeader(w, resp)
		gzw := gzip.NewWriter(w)
		gzw.Write(content)
		gzw.Close()
	} else {
		copyResponseHeader(w, resp)
		w.Write(content)
	}

	logAccess(r, resp, sc, contentType, len(content), modified, user)
}

// copyResponseHeader writes resp's header and status code to w.
func copyResponseHeader(w http.ResponseWriter, resp *http.Response) {
	newHeader := w.Header()
	for key, values := range resp.Header {
		for _, v := range values {
			newHeader.Add(key, v)
		}
	}

	w.WriteHeader(resp.StatusCode)
}

// A hijackedConn is a connection that has been hijacked (to fulfill a CONNECT
// request).
type hijackedConn struct {
	net.Conn
	io.Reader
}

func (hc *hijackedConn) Read(b []byte) (int, error) {
	return hc.Reader.Read(b)
}

func newHijackedConn(w http.ResponseWriter) (*hijackedConn, error) {
	hj, ok := w.(http.Hijacker)
	if !ok {
		return nil, errors.New("connection doesn't support hijacking")
	}
	conn, bufrw, err := hj.Hijack()
	if err != nil {
		return nil, err
	}
	err = bufrw.Flush()
	if err != nil {
		return nil, err
	}
	return &hijackedConn{
		Conn:   conn,
		Reader: bufrw.Reader,
	}, nil
}

var transport = http.Transport{
	TLSClientConfig: unverifiedClientConfig,
	Proxy:           http.ProxyFromEnvironment,
}

// This is to deal with the problem of stale keepalive connections, which cause
// transport.RoundTrip to return io.EOF.
func init() {
	go func() {
		for _ = range time.Tick(10 * time.Second) {
			transport.CloseIdleConnections()
		}
	}()

	transport.RegisterProtocol("ftp", FTPTransport{})
}

func (h proxyHandler) makeWebsocketConnection(w http.ResponseWriter, r *http.Request) {
	addr := r.Host
	if _, _, err := net.SplitHostPort(addr); err != nil {
		// There is no port specified; we need to add it.
		port := h.connectPort
		if port == "" {
			port = "80"
		}
		addr = net.JoinHostPort(addr, port)
	}
	var err error
	var serverConn net.Conn
	if h.TLS {
		serverConn, err = tls.Dial("tcp", addr, unverifiedClientConfig)
	} else {
		serverConn, err = net.Dial("tcp", addr)
	}
	if err != nil {
		http.Error(w, err.Error(), http.StatusInternalServerError)
		return
	}

	err = r.Write(serverConn)
	if err != nil {
		http.Error(w, err.Error(), http.StatusInternalServerError)
		return
	}

	hj, ok := w.(http.Hijacker)
	if !ok {
		http.Error(w, "Couldn't create a websocket connection", http.StatusInternalServerError)
		return
	}
	conn, bufrw, err := hj.Hijack()
	if err != nil {
		http.Error(w, err.Error(), http.StatusInternalServerError)
		return
	}

	go func() {
		io.Copy(conn, serverConn)
		conn.Close()
	}()
	io.Copy(serverConn, bufrw)
	serverConn.Close()
	return
}
<|endoftext|>


<commit_before>package goproxy

import (
	"bufio"
	"io"
	"log"
	"net"
	"net/http"
	"os"
	"regexp"
	"sync/atomic"
)

// The basic proxy type. Implements http.Handler.
type ProxyHttpServer struct {
	// session variable must be aligned in i386
	// see http://golang.org/src/pkg/sync/atomic/doc.go#L41
	sess int64
	// KeepDestinationHeaders indicates the proxy should retain any headers present in the http.Response before proxying
	KeepDestinationHeaders bool
	// setting Verbose to true will log information on each request sent to the proxy
	Verbose         bool
	Logger          Logger
	NonproxyHandler http.Handler
	reqHandlers     []ReqHandler
	respHandlers    []RespHandler
	httpsHandlers   []HttpsHandler
	Tr              *http.Transport
	// ConnectDial will be used to create TCP connections for CONNECT requests
	// if nil Tr.Dial will be used
	ConnectDial func(network string, addr string) (net.Conn, error)
	CertStore   CertStorage
	KeepHeader  bool
}

var hasPort = regexp.MustCompile(`:\d+$`)

func copyHeaders(dst, src http.Header, keepDestHeaders bool) {
	if !keepDestHeaders {
		for k := range dst {
			dst.Del(k)
		}
	}
	for k, vs := range src {
		for _, v := range vs {
			dst.Add(k, v)
		}
	}
}

func isEof(r *bufio.Reader) bool {
	_, err := r.Peek(1)
	if err == io.EOF {
		return true
	}
	return false
}

func (proxy *ProxyHttpServer) filterRequest(r *http.Request, ctx *ProxyCtx) (req *http.Request, resp *http.Response) {
	req = r
	for _, h := range proxy.reqHandlers {
		req, resp = h.Handle(r, ctx)
		// non-nil resp means the handler decided to skip sending the request
		// and return canned response instead.
		if resp != nil {
			break
		}
	}
	return
}
func (proxy *ProxyHttpServer) filterResponse(respOrig *http.Response, ctx *ProxyCtx) (resp *http.Response) {
	resp = respOrig
	for _, h := range proxy.respHandlers {
		ctx.Resp = resp
		resp = h.Handle(resp, ctx)
	}
	return
}

func removeProxyHeaders(ctx *ProxyCtx, r *http.Request) {
	r.RequestURI = "" // this must be reset when serving a request with the client
	ctx.Logf("Sending request %v %v", r.Method, r.URL.String())
	// If no Accept-Encoding header exists, Transport will add the headers it can accept
	// and would wrap the response body with the relevant reader.
	r.Header.Del("Accept-Encoding")
	// curl can add that, see
	// https://jdebp.eu./FGA/web-proxy-connection-header.html
	r.Header.Del("Proxy-Connection")
	r.Header.Del("Proxy-Authenticate")
	r.Header.Del("Proxy-Authorization")
	// Connection, Authenticate and Authorization are single hop Header:
	// http://www.w3.org/Protocols/rfc2616/rfc2616.txt
	// 14.10 Connection
	//   The Connection general-header field allows the sender to specify
	//   options that are desired for that particular connection and MUST NOT
	//   be communicated by proxies over further connections.

	// ASK: I'm not sure about this... as far as I see when server reads http request
	// /usr/local/go/src/net/http/request.go:1085 it sets Close only in two cases
	// when there is Connection: close header set or when req.isH2Upgrade (have
	// no idea what is it).
	// We need to set r.Close to false otherwise
	// /usr/local/go/src/net/http/transfer.go:275 will add Connection:close header
	// back. This is the reason of why test fails.
	if r.Header.Get("Connection") == "close" && r.Close {
		r.Close = false
	}
	r.Header.Del("Connection")
}

// Standard net/http function. Shouldn't be used directly, http.Serve will use it.
func (proxy *ProxyHttpServer) ServeHTTP(w http.ResponseWriter, r *http.Request) {
	//r.Header["X-Forwarded-For"] = w.RemoteAddr()
	if r.Method == "CONNECT" {
		proxy.handleHttps(w, r)
	} else {
		ctx := &ProxyCtx{Req: r, Session: atomic.AddInt64(&proxy.sess, 1), Proxy: proxy}

		var err error
		ctx.Logf("Got request %v %v %v %v", r.URL.Path, r.Host, r.Method, r.URL.String())
		if !r.URL.IsAbs() {
			proxy.NonproxyHandler.ServeHTTP(w, r)
			return
		}
		r, resp := proxy.filterRequest(r, ctx)

		if resp == nil {
			if isWebSocketRequest(r) {
				ctx.Logf("Request looks like websocket upgrade.")
				proxy.serveWebsocket(ctx, w, r)
			}

			if !proxy.KeepHeader {
				removeProxyHeaders(ctx, r)
			}
			resp, err = ctx.RoundTrip(r)
			if err != nil {
				ctx.Error = err
				resp = proxy.filterResponse(nil, ctx)

			}
			if resp != nil {
				ctx.Logf("Received response %v", resp.Status)
			}
		}

		var origBody io.ReadCloser

		if resp != nil {
			origBody = resp.Body
			defer origBody.Close()
		}

		resp = proxy.filterResponse(resp, ctx)

		if resp == nil {
			var errorString string
			if ctx.Error != nil {
				errorString = "error read response " + r.URL.Host + " : " + ctx.Error.Error()
				ctx.Logf(errorString)
				http.Error(w, ctx.Error.Error(), 500)
			} else {
				errorString = "error read response " + r.URL.Host
				ctx.Logf(errorString)
				http.Error(w, errorString, 500)
			}
			return
		}
		ctx.Logf("Copying response to client %v [%d]", resp.Status, resp.StatusCode)
		// http.ResponseWriter will take care of filling the correct response length
		// Setting it now, might impose wrong value, contradicting the actual new
		// body the user returned.
		// We keep the original body to remove the header only if things changed.
		// This will prevent problems with HEAD requests where there's no body, yet,
		// the Content-Length header should be set.
		if origBody != resp.Body {
			resp.Header.Del("Content-Length")
		}
		copyHeaders(w.Header(), resp.Header, proxy.KeepDestinationHeaders)
		w.WriteHeader(resp.StatusCode)
		nr, err := io.Copy(w, resp.Body)
		if err := resp.Body.Close(); err != nil {
			ctx.Warnf("Can't close response body %v", err)
		}
		ctx.Logf("Copied %v bytes to client error=%v", nr, err)
	}
}

// NewProxyHttpServer creates and returns a proxy server, logging to stderr by default
func NewProxyHttpServer() *ProxyHttpServer {
	proxy := ProxyHttpServer{
		Logger:        log.New(os.Stderr, "", log.LstdFlags),
		reqHandlers:   []ReqHandler{},
		respHandlers:  []RespHandler{},
		httpsHandlers: []HttpsHandler{},
		NonproxyHandler: http.HandlerFunc(func(w http.ResponseWriter, req *http.Request) {
			http.Error(w, "This is a proxy server. Does not respond to non-proxy requests.", 500)
		}),
		Tr: &http.Transport{TLSClientConfig: tlsClientSkipVerify, Proxy: http.ProxyFromEnvironment},
	}

	proxy.ConnectDial = dialerFromEnv(&proxy)

	return &proxy
}
<commit_msg>update comment to change<commit_after>package goproxy

import (
	"bufio"
	"io"
	"log"
	"net"
	"net/http"
	"os"
	"regexp"
	"sync/atomic"
)

// The basic proxy type. Implements http.Handler.
type ProxyHttpServer struct {
	// session variable must be aligned in i386
	// see http://golang.org/src/pkg/sync/atomic/doc.go#L41
	sess int64
	// KeepDestinationHeaders indicates the proxy should retain any headers present in the http.Response before proxying
	KeepDestinationHeaders bool
	// setting Verbose to true will log information on each request sent to the proxy
	Verbose         bool
	Logger          Logger
	NonproxyHandler http.Handler
	reqHandlers     []ReqHandler
	respHandlers    []RespHandler
	httpsHandlers   []HttpsHandler
	Tr              *http.Transport
	// ConnectDial will be used to create TCP connections for CONNECT requests
	// if nil Tr.Dial will be used
	ConnectDial func(network string, addr string) (net.Conn, error)
	CertStore   CertStorage
	KeepHeader  bool
}

var hasPort = regexp.MustCompile(`:\d+$`)

func copyHeaders(dst, src http.Header, keepDestHeaders bool) {
	if !keepDestHeaders {
		for k := range dst {
			dst.Del(k)
		}
	}
	for k, vs := range src {
		for _, v := range vs {
			dst.Add(k, v)
		}
	}
}

func isEof(r *bufio.Reader) bool {
	_, err := r.Peek(1)
	if err == io.EOF {
		return true
	}
	return false
}

func (proxy *ProxyHttpServer) filterRequest(r *http.Request, ctx *ProxyCtx) (req *http.Request, resp *http.Response) {
	req = r
	for _, h := range proxy.reqHandlers {
		req, resp = h.Handle(r, ctx)
		// non-nil resp means the handler decided to skip sending the request
		// and return canned response instead.
		if resp != nil {
			break
		}
	}
	return
}
func (proxy *ProxyHttpServer) filterResponse(respOrig *http.Response, ctx *ProxyCtx) (resp *http.Response) {
	resp = respOrig
	for _, h := range proxy.respHandlers {
		ctx.Resp = resp
		resp = h.Handle(resp, ctx)
	}
	return
}

func removeProxyHeaders(ctx *ProxyCtx, r *http.Request) {
	r.RequestURI = "" // this must be reset when serving a request with the client
	ctx.Logf("Sending request %v %v", r.Method, r.URL.String())
	// If no Accept-Encoding header exists, Transport will add the headers it can accept
	// and would wrap the response body with the relevant reader.
	r.Header.Del("Accept-Encoding")
	// curl can add that, see
	// https://jdebp.eu./FGA/web-proxy-connection-header.html
	r.Header.Del("Proxy-Connection")
	r.Header.Del("Proxy-Authenticate")
	r.Header.Del("Proxy-Authorization")
	// Connection, Authenticate and Authorization are single hop Header:
	// http://www.w3.org/Protocols/rfc2616/rfc2616.txt
	// 14.10 Connection
	//   The Connection general-header field allows the sender to specify
	//   options that are desired for that particular connection and MUST NOT
	//   be communicated by proxies over further connections.

	// When server reads http request it sets req.Close to true if
	// "Connection" header contains "close".
	// https://github.com/golang/go/blob/master/src/net/http/request.go#L1080
	// Later, transfer.go adds "Connection: close" back when req.Close is true
	// https://github.com/golang/go/blob/master/src/net/http/transfer.go#L275
	// That's why tests that checks "Connection: close" removal fail
	if r.Header.Get("Connection") == "close" {
		r.Close = false
	}
	r.Header.Del("Connection")
}

// Standard net/http function. Shouldn't be used directly, http.Serve will use it.
func (proxy *ProxyHttpServer) ServeHTTP(w http.ResponseWriter, r *http.Request) {
	//r.Header["X-Forwarded-For"] = w.RemoteAddr()
	if r.Method == "CONNECT" {
		proxy.handleHttps(w, r)
	} else {
		ctx := &ProxyCtx{Req: r, Session: atomic.AddInt64(&proxy.sess, 1), Proxy: proxy}

		var err error
		ctx.Logf("Got request %v %v %v %v", r.URL.Path, r.Host, r.Method, r.URL.String())
		if !r.URL.IsAbs() {
			proxy.NonproxyHandler.ServeHTTP(w, r)
			return
		}
		r, resp := proxy.filterRequest(r, ctx)

		if resp == nil {
			if isWebSocketRequest(r) {
				ctx.Logf("Request looks like websocket upgrade.")
				proxy.serveWebsocket(ctx, w, r)
			}

			if !proxy.KeepHeader {
				removeProxyHeaders(ctx, r)
			}
			resp, err = ctx.RoundTrip(r)
			if err != nil {
				ctx.Error = err
				resp = proxy.filterResponse(nil, ctx)

			}
			if resp != nil {
				ctx.Logf("Received response %v", resp.Status)
			}
		}

		var origBody io.ReadCloser

		if resp != nil {
			origBody = resp.Body
			defer origBody.Close()
		}

		resp = proxy.filterResponse(resp, ctx)

		if resp == nil {
			var errorString string
			if ctx.Error != nil {
				errorString = "error read response " + r.URL.Host + " : " + ctx.Error.Error()
				ctx.Logf(errorString)
				http.Error(w, ctx.Error.Error(), 500)
			} else {
				errorString = "error read response " + r.URL.Host
				ctx.Logf(errorString)
				http.Error(w, errorString, 500)
			}
			return
		}
		ctx.Logf("Copying response to client %v [%d]", resp.Status, resp.StatusCode)
		// http.ResponseWriter will take care of filling the correct response length
		// Setting it now, might impose wrong value, contradicting the actual new
		// body the user returned.
		// We keep the original body to remove the header only if things changed.
		// This will prevent problems with HEAD requests where there's no body, yet,
		// the Content-Length header should be set.
		if origBody != resp.Body {
			resp.Header.Del("Content-Length")
		}
		copyHeaders(w.Header(), resp.Header, proxy.KeepDestinationHeaders)
		w.WriteHeader(resp.StatusCode)
		nr, err := io.Copy(w, resp.Body)
		if err := resp.Body.Close(); err != nil {
			ctx.Warnf("Can't close response body %v", err)
		}
		ctx.Logf("Copied %v bytes to client error=%v", nr, err)
	}
}

// NewProxyHttpServer creates and returns a proxy server, logging to stderr by default
func NewProxyHttpServer() *ProxyHttpServer {
	proxy := ProxyHttpServer{
		Logger:        log.New(os.Stderr, "", log.LstdFlags),
		reqHandlers:   []ReqHandler{},
		respHandlers:  []RespHandler{},
		httpsHandlers: []HttpsHandler{},
		NonproxyHandler: http.HandlerFunc(func(w http.ResponseWriter, req *http.Request) {
			http.Error(w, "This is a proxy server. Does not respond to non-proxy requests.", 500)
		}),
		Tr: &http.Transport{TLSClientConfig: tlsClientSkipVerify, Proxy: http.ProxyFromEnvironment},
	}

	proxy.ConnectDial = dialerFromEnv(&proxy)

	return &proxy
}
<|endoftext|>


<commit_before>package bbs

import (
	"time"

	"github.com/cloudfoundry-incubator/consuladapter"
	"github.com/cloudfoundry-incubator/runtime-schema/bbs/lock_bbs"
	"github.com/cloudfoundry-incubator/runtime-schema/bbs/lrp_bbs"
	"github.com/cloudfoundry-incubator/runtime-schema/bbs/services_bbs"
	"github.com/cloudfoundry-incubator/runtime-schema/bbs/task_bbs"
	"github.com/cloudfoundry-incubator/runtime-schema/cb"
	"github.com/cloudfoundry-incubator/runtime-schema/models"
	"github.com/cloudfoundry/storeadapter"
	"github.com/pivotal-golang/clock"
	"github.com/pivotal-golang/lager"
	"github.com/tedsuo/ifrit"
)

//Bulletin Board System/Store

//go:generate counterfeiter -o fake_bbs/fake_receptor_bbs.go . ReceptorBBS
type ReceptorBBS interface {
	//desired lrp
	DesireLRP(lager.Logger, models.DesiredLRP) error
	UpdateDesiredLRP(logger lager.Logger, processGuid string, update models.DesiredLRPUpdate) error
	RemoveDesiredLRPByProcessGuid(logger lager.Logger, processGuid string) error

	// cells
	Cells() ([]models.CellPresence, error)
}

//go:generate counterfeiter -o fake_bbs/fake_rep_bbs.go . RepBBS
type RepBBS interface {
	//services
	NewCellPresence(cellPresence models.CellPresence, retryInterval time.Duration) ifrit.Runner
}

//go:generate counterfeiter -o fake_bbs/fake_converger_bbs.go . ConvergerBBS
type ConvergerBBS interface {
	//lock
	NewConvergeLock(convergerID string, retryInterval time.Duration) ifrit.Runner

	//lrp
	ConvergeLRPs(logger lager.Logger, cellsLoader *services_bbs.CellsLoader)

	//task
	ConvergeTasks(logger lager.Logger, timeToClaim, convergenceInterval, timeToResolve time.Duration, cellsLoader *services_bbs.CellsLoader)

	//cell loader
	NewCellsLoader() *services_bbs.CellsLoader

	//cells
	CellEvents() <-chan services_bbs.CellEvent
}

const ConvergerBBSWorkPoolSize = 50

//go:generate counterfeiter -o fake_bbs/fake_nsync_bbs.go . NsyncBBS
type NsyncBBS interface {
	//lock
	NewNsyncBulkerLock(bulkerID string, retryInterval time.Duration) ifrit.Runner
}

//go:generate counterfeiter -o fake_bbs/fake_auctioneer_bbs.go . AuctioneerBBS
type AuctioneerBBS interface {
	//services
	Cells() ([]models.CellPresence, error)

	//lock
	NewAuctioneerLock(auctioneerPresence models.AuctioneerPresence, retryInterval time.Duration) (ifrit.Runner, error)
}

//go:generate counterfeiter -o fake_bbs/fake_metrics_bbs.go . MetricsBBS
type MetricsBBS interface {
	//lock
	NewRuntimeMetricsLock(runtimeMetricsID string, retryInterval time.Duration) ifrit.Runner
}

//go:generate counterfeiter -o fake_bbs/fake_route_emitter_bbs.go . RouteEmitterBBS
type RouteEmitterBBS interface {
	//lock
	NewRouteEmitterLock(emitterID string, retryInterval time.Duration) ifrit.Runner
}

//go:generate counterfeiter -o fake_bbs/fake_tps_bbs.go . TpsBBS
type TpsBBS interface {
	//lock
	NewTpsWatcherLock(watcherID string, retryInterval time.Duration) ifrit.Runner
}

type VeritasBBS interface {
	//lrp
	DesireLRP(lager.Logger, models.DesiredLRP) error
	RemoveDesiredLRPByProcessGuid(logger lager.Logger, guid string) error

	//services
	Cells() ([]models.CellPresence, error)
	AuctioneerAddress() (string, error)
}

func NewReceptorBBS(store storeadapter.StoreAdapter, consul *consuladapter.Session, receptorTaskHandlerURL string, clock clock.Clock, logger lager.Logger) ReceptorBBS {
	return NewBBS(store, consul, receptorTaskHandlerURL, clock, logger)
}

func NewRepBBS(store storeadapter.StoreAdapter, consul *consuladapter.Session, receptorTaskHandlerURL string, clock clock.Clock, logger lager.Logger) RepBBS {
	return NewBBS(store, consul, receptorTaskHandlerURL, clock, logger)
}

func NewConvergerBBS(store storeadapter.StoreAdapter, consul *consuladapter.Session, receptorTaskHandlerURL string, clock clock.Clock, logger lager.Logger) ConvergerBBS {
	return NewBBS(store, consul, receptorTaskHandlerURL, clock, logger)
}

func NewNsyncBBS(consul *consuladapter.Session, clock clock.Clock, logger lager.Logger) NsyncBBS {
	return lock_bbs.New(consul, clock, logger.Session("lock-bbs"))
}

func NewAuctioneerBBS(store storeadapter.StoreAdapter, consul *consuladapter.Session, receptorTaskHandlerURL string, clock clock.Clock, logger lager.Logger) AuctioneerBBS {
	return NewBBS(store, consul, receptorTaskHandlerURL, clock, logger)
}

func NewMetricsBBS(consul *consuladapter.Session, clock clock.Clock, logger lager.Logger) MetricsBBS {
	return lock_bbs.New(consul, clock, logger.Session("metrics-bbs"))
}

func NewRouteEmitterBBS(consul *consuladapter.Session, clock clock.Clock, logger lager.Logger) RouteEmitterBBS {
	return lock_bbs.New(consul, clock, logger.Session("lock-bbs"))
}

func NewTpsBBS(consul *consuladapter.Session, clock clock.Clock, logger lager.Logger) TpsBBS {
	return lock_bbs.New(consul, clock, logger.Session("lock-bbs"))
}

func NewVeritasBBS(store storeadapter.StoreAdapter, consul *consuladapter.Session, clock clock.Clock, logger lager.Logger) VeritasBBS {
	return NewBBS(store, consul, "", clock, logger)
}

func NewBBS(store storeadapter.StoreAdapter, consul *consuladapter.Session, receptorTaskHandlerURL string, clock clock.Clock, logger lager.Logger) *BBS {
	services := services_bbs.New(consul, clock, logger.Session("services-bbs"))
	auctioneerClient := cb.NewAuctioneerClient()
	cellClient := cb.NewCellClient()

	return &BBS{
		LockBBS:     lock_bbs.New(consul, clock, logger.Session("lock-bbs")),
		LRPBBS:      lrp_bbs.New(store, clock, cellClient, auctioneerClient, services),
		ServicesBBS: services,
		TaskBBS:     task_bbs.New(store, consul, clock, cb.NewTaskClient(), auctioneerClient, cellClient, services, receptorTaskHandlerURL),
	}
}

type BBS struct {
	*lock_bbs.LockBBS
	*lrp_bbs.LRPBBS
	*services_bbs.ServicesBBS
	*task_bbs.TaskBBS
}
<commit_msg>remove ConvergeTasks from bbs interface<commit_after>package bbs

import (
	"time"

	"github.com/cloudfoundry-incubator/consuladapter"
	"github.com/cloudfoundry-incubator/runtime-schema/bbs/lock_bbs"
	"github.com/cloudfoundry-incubator/runtime-schema/bbs/lrp_bbs"
	"github.com/cloudfoundry-incubator/runtime-schema/bbs/services_bbs"
	"github.com/cloudfoundry-incubator/runtime-schema/bbs/task_bbs"
	"github.com/cloudfoundry-incubator/runtime-schema/cb"
	"github.com/cloudfoundry-incubator/runtime-schema/models"
	"github.com/cloudfoundry/storeadapter"
	"github.com/pivotal-golang/clock"
	"github.com/pivotal-golang/lager"
	"github.com/tedsuo/ifrit"
)

//Bulletin Board System/Store

//go:generate counterfeiter -o fake_bbs/fake_receptor_bbs.go . ReceptorBBS
type ReceptorBBS interface {
	//desired lrp
	DesireLRP(lager.Logger, models.DesiredLRP) error
	UpdateDesiredLRP(logger lager.Logger, processGuid string, update models.DesiredLRPUpdate) error
	RemoveDesiredLRPByProcessGuid(logger lager.Logger, processGuid string) error

	// cells
	Cells() ([]models.CellPresence, error)
}

//go:generate counterfeiter -o fake_bbs/fake_rep_bbs.go . RepBBS
type RepBBS interface {
	//services
	NewCellPresence(cellPresence models.CellPresence, retryInterval time.Duration) ifrit.Runner
}

//go:generate counterfeiter -o fake_bbs/fake_converger_bbs.go . ConvergerBBS
type ConvergerBBS interface {
	//lock
	NewConvergeLock(convergerID string, retryInterval time.Duration) ifrit.Runner

	//lrp
	ConvergeLRPs(logger lager.Logger, cellsLoader *services_bbs.CellsLoader)

	//cell loader
	NewCellsLoader() *services_bbs.CellsLoader

	//cells
	CellEvents() <-chan services_bbs.CellEvent
}

const ConvergerBBSWorkPoolSize = 50

//go:generate counterfeiter -o fake_bbs/fake_nsync_bbs.go . NsyncBBS
type NsyncBBS interface {
	//lock
	NewNsyncBulkerLock(bulkerID string, retryInterval time.Duration) ifrit.Runner
}

//go:generate counterfeiter -o fake_bbs/fake_auctioneer_bbs.go . AuctioneerBBS
type AuctioneerBBS interface {
	//services
	Cells() ([]models.CellPresence, error)

	//lock
	NewAuctioneerLock(auctioneerPresence models.AuctioneerPresence, retryInterval time.Duration) (ifrit.Runner, error)
}

//go:generate counterfeiter -o fake_bbs/fake_metrics_bbs.go . MetricsBBS
type MetricsBBS interface {
	//lock
	NewRuntimeMetricsLock(runtimeMetricsID string, retryInterval time.Duration) ifrit.Runner
}

//go:generate counterfeiter -o fake_bbs/fake_route_emitter_bbs.go . RouteEmitterBBS
type RouteEmitterBBS interface {
	//lock
	NewRouteEmitterLock(emitterID string, retryInterval time.Duration) ifrit.Runner
}

//go:generate counterfeiter -o fake_bbs/fake_tps_bbs.go . TpsBBS
type TpsBBS interface {
	//lock
	NewTpsWatcherLock(watcherID string, retryInterval time.Duration) ifrit.Runner
}

type VeritasBBS interface {
	//lrp
	DesireLRP(lager.Logger, models.DesiredLRP) error
	RemoveDesiredLRPByProcessGuid(logger lager.Logger, guid string) error

	//services
	Cells() ([]models.CellPresence, error)
	AuctioneerAddress() (string, error)
}

func NewReceptorBBS(store storeadapter.StoreAdapter, consul *consuladapter.Session, receptorTaskHandlerURL string, clock clock.Clock, logger lager.Logger) ReceptorBBS {
	return NewBBS(store, consul, receptorTaskHandlerURL, clock, logger)
}

func NewRepBBS(store storeadapter.StoreAdapter, consul *consuladapter.Session, receptorTaskHandlerURL string, clock clock.Clock, logger lager.Logger) RepBBS {
	return NewBBS(store, consul, receptorTaskHandlerURL, clock, logger)
}

func NewConvergerBBS(store storeadapter.StoreAdapter, consul *consuladapter.Session, receptorTaskHandlerURL string, clock clock.Clock, logger lager.Logger) ConvergerBBS {
	return NewBBS(store, consul, receptorTaskHandlerURL, clock, logger)
}

func NewNsyncBBS(consul *consuladapter.Session, clock clock.Clock, logger lager.Logger) NsyncBBS {
	return lock_bbs.New(consul, clock, logger.Session("lock-bbs"))
}

func NewAuctioneerBBS(store storeadapter.StoreAdapter, consul *consuladapter.Session, receptorTaskHandlerURL string, clock clock.Clock, logger lager.Logger) AuctioneerBBS {
	return NewBBS(store, consul, receptorTaskHandlerURL, clock, logger)
}

func NewMetricsBBS(consul *consuladapter.Session, clock clock.Clock, logger lager.Logger) MetricsBBS {
	return lock_bbs.New(consul, clock, logger.Session("metrics-bbs"))
}

func NewRouteEmitterBBS(consul *consuladapter.Session, clock clock.Clock, logger lager.Logger) RouteEmitterBBS {
	return lock_bbs.New(consul, clock, logger.Session("lock-bbs"))
}

func NewTpsBBS(consul *consuladapter.Session, clock clock.Clock, logger lager.Logger) TpsBBS {
	return lock_bbs.New(consul, clock, logger.Session("lock-bbs"))
}

func NewVeritasBBS(store storeadapter.StoreAdapter, consul *consuladapter.Session, clock clock.Clock, logger lager.Logger) VeritasBBS {
	return NewBBS(store, consul, "", clock, logger)
}

func NewBBS(store storeadapter.StoreAdapter, consul *consuladapter.Session, receptorTaskHandlerURL string, clock clock.Clock, logger lager.Logger) *BBS {
	services := services_bbs.New(consul, clock, logger.Session("services-bbs"))
	auctioneerClient := cb.NewAuctioneerClient()
	cellClient := cb.NewCellClient()

	return &BBS{
		LockBBS:     lock_bbs.New(consul, clock, logger.Session("lock-bbs")),
		LRPBBS:      lrp_bbs.New(store, clock, cellClient, auctioneerClient, services),
		ServicesBBS: services,
		TaskBBS:     task_bbs.New(store, consul, clock, cb.NewTaskClient(), auctioneerClient, cellClient, services, receptorTaskHandlerURL),
	}
}

type BBS struct {
	*lock_bbs.LockBBS
	*lrp_bbs.LRPBBS
	*services_bbs.ServicesBBS
	*task_bbs.TaskBBS
}
<|endoftext|>


<commit_before>package rapid

import (
	"fmt"
	"html/template"
	"log"
	"net/http"
	"strconv"
	"sync"
)

var wg sync.WaitGroup
var srv *http.Server

// Connection - struct for handling http request and write
type Connection struct {
	R      *http.Request
	W      http.ResponseWriter
	Params map[string]string
}

// Send - Return plain text string back to http request
func (c *Connection) Send(message string) {
	fmt.Fprintf(c.W, message)
}

// View - Render HTML view without templating
func (c *Connection) View(path string) {
	c.Render(path, nil)
}

// Render - Render HTML view with templating
// Templating uses standard library templating
func (c *Connection) Render(path string, object interface{}) {
	t, _ := template.ParseFiles(path)
	c.W.Header().Set("Content-Type", "text/html; charset=utf-8")
	t.Execute(c.W, object)
}

// Redirect - Redirect a request to another rest end point
func (c *Connection) Redirect(path string) {
	http.Redirect(c.W, c.R, path, 301)
}

// StaticFolder - Specify application public/static folder
func StaticFolder(path string, dir string) {
	http.Handle("/"+path+"/", http.StripPrefix("/"+path+"/", http.FileServer(http.Dir(dir))))
}

// Listen - Start webserver on specified port
// Returns the instance of the http server currently runing.
// You can use this instance to shutdown the server if need be.
func Listen(port int) {
	ListenAndWait(port, true)
}

// ListenAndWait - Gives user option of waiting for server or not
func ListenAndWait(port int, wait bool) {
	portString := strconv.Itoa(port)
	srv = &http.Server{Addr: ":" + portString}

	go func() {
		if err := srv.ListenAndServe(); err != nil {
			// cannot panic, because this probably is an intentional close
			log.Printf("Httpserver: ListenAndServe() error: %s", err)
		}
	}()

	wg.Add(1)
	if wait {
		wg.Wait()
	}
}

func ShutdownServer() {
	wg.Done()
	if err := srv.Shutdown(nil); err != nil {
		panic(err) // failure/timeout shutting down the server gracefully
	}
}
<commit_msg>Added comment for the shut down server method.<commit_after>package rapid

import (
	"fmt"
	"html/template"
	"log"
	"net/http"
	"strconv"
	"sync"
)

var wg sync.WaitGroup
var srv *http.Server

// Connection - struct for handling http request and write
type Connection struct {
	R      *http.Request
	W      http.ResponseWriter
	Params map[string]string
}

// Send - Return plain text string back to http request
func (c *Connection) Send(message string) {
	fmt.Fprintf(c.W, message)
}

// View - Render HTML view without templating
func (c *Connection) View(path string) {
	c.Render(path, nil)
}

// Render - Render HTML view with templating
// Templating uses standard library templating
func (c *Connection) Render(path string, object interface{}) {
	t, _ := template.ParseFiles(path)
	c.W.Header().Set("Content-Type", "text/html; charset=utf-8")
	t.Execute(c.W, object)
}

// Redirect - Redirect a request to another rest end point
func (c *Connection) Redirect(path string) {
	http.Redirect(c.W, c.R, path, 301)
}

// StaticFolder - Specify application public/static folder
func StaticFolder(path string, dir string) {
	http.Handle("/"+path+"/", http.StripPrefix("/"+path+"/", http.FileServer(http.Dir(dir))))
}

// Listen - Start webserver on specified port
// Returns the instance of the http server currently runing.
// You can use this instance to shutdown the server if need be.
func Listen(port int) {
	ListenAndWait(port, true)
}

// ListenAndWait - Gives user option of waiting for server or not
func ListenAndWait(port int, wait bool) {
	portString := strconv.Itoa(port)
	srv = &http.Server{Addr: ":" + portString}

	go func() {
		if err := srv.ListenAndServe(); err != nil {
			// cannot panic, because this probably is an intentional close
			log.Printf("Httpserver: ListenAndServe() error: %s", err)
		}
	}()

	wg.Add(1)
	if wait {
		wg.Wait()
	}
}

// ShutdownServer - Gracefully shut down the server and unblock
// the server thread.
func ShutdownServer() {
	wg.Done()
	if err := srv.Shutdown(nil); err != nil {
		panic(err) // failure/timeout shutting down the server gracefully
	}
}
<|endoftext|>