<commit_before>package graph

import (
	"os"
	"path"
	"reflect"
	"sort"
	"syscall"
	"unsafe"
	"ellipsoid"
)

type Node interface {
	Edges() []Edge
	LatLng() (float64, float64)
}

type Edge interface {
	Length() float64
	StartPoint() Node // e.g. via binary search on the node array
	EndPoint() Node
	ReverseEdge() (Edge, bool)
	Steps() []Step
	// Label() string
}

// "partial edge" is returned by the k-d tree
type Way struct {
	Length float64
	Node   Node // StartPoint or EndPoint
	Steps  []Step
}

type Graph interface {
	NodeCount() int
	EdgeCount() int
	Node(uint) Node
	Edge(uint) Edge
	Positions() Positions
}

// Implementation sketch (wrapper around graph):
// The graph is loaded before and is given to this.
// Init: Both position files (vertexes and inner steps in an edge) are alread loaded
//   for the graph (Nodes.LatLng(), Edge.Steps() - Step.Lat/Lng).
//   Thus despite storing a pointer the graph, nothing to do here.
// For every method where an index is given, there is a branch
//   if index < Graph.NodeCount()
//      work with Graph.Node(index)
//   else
//      work with the underlying Step array (index - Graph.NodeCount())
//      not possible efficiently with the current interface
//
// Positions is used for both creating the k-d tree in the preprocessing phase
// and for doing the nearest neighbor lookup during runtime.
type Positions interface {
	Len() int
	Lat(int) float64
	Lng(int) float64
	Step(int) Step
	Ways(int, bool) []Way // index, forward (i.e. looking at the edge in forward or in backward order)
}

type Step struct {
	Lat float64
	Lng float64
}

// Implementation

type graphFile struct {
	// ellipsoid for distance calculations
	geo ellipsoid.Ellipsoid
	// vertices maps a vertex index to the index of its first out edge
	vertices []uint32
	// edges maps edge indices to the index of the vertex endpoint
	edges []uint32
	// rev_edges maps an edge index to its reverse
	revEdges []uint32
	// map edge indices to distances
	distances []float64
	// map node indices to positions (lat: 2 * i, lng: 2 * i + 1)
	positions []float64
	// map an edge index to the inex of the first intermediate step
	steps []uint32
	// positions as interleaved lat/lng pairs as above
	stepPositions []float64
}

func mapFile(base, name string) ([]byte, error) {
	file, err := os.Open(path.Join(base, name))
	if err != nil {
		return nil, err
	}
	info, err := file.Stat()
	if err != nil {
		return nil, err
	}
	// Thanks to Windows compatibility file.Fd is not declared int...
	fdfu := file.Fd()
	fd := *(*int)(unsafe.Pointer(&fdfu))
	// This is bad. Slices have int size and capacity fields, which
	// means that we might truncate here. We can work around this issue
	// by using unsafe.Pointer internally and only convert to slices for
	// individual edge/step lists... But for now our files are small
	// and this works:
	size := int(info.Size())
	return syscall.Mmap(fd, 0, size, syscall.PROT_READ, syscall.MAP_PRIVATE)
}

func mapFileUint32(base, name string) ([]uint32, error) {
	m, err := mapFile(base, name)
	if err != nil {
		return nil, err
	}

	dh := (*reflect.SliceHeader)(unsafe.Pointer(&m))
	dh.Len /= 4
	dh.Cap /= 4
	return *(*[]uint32)(unsafe.Pointer(&m)), nil
}

func mapFileFloat64(base, name string) ([]float64, error) {
	m, err := mapFile(base, name)
	if err != nil {
		return nil, err
	}

	dh := (*reflect.SliceHeader)(unsafe.Pointer(&m))
	dh.Len /= 8
	dh.Cap /= 8
	return *(*[]float64)(unsafe.Pointer(&m)), nil
}

func Open(base string) (Graph, error) {
	graph := graphFile{}

	var err error
	graph.vertices, err = mapFileUint32(base, "vertices.ftf")
	if err != nil {
		return nil, err
	}

	graph.edges, err = mapFileUint32(base, "edges.ftf")
	if err != nil {
		return nil, err
	}

	graph.revEdges, err = mapFileUint32(base, "rev_edges.ftf")
	if err != nil {
		return nil, err
	}

	graph.distances, err = mapFileFloat64(base, "distances.ftf")
	if err != nil {
		return nil, err
	}

	graph.positions, err = mapFileFloat64(base, "positions.ftf")
	if err != nil {
		return nil, err
	}

	graph.steps, err = mapFileUint32(base, "steps.ftf")
	if err != nil {
		return nil, err
	}

	graph.stepPositions, err = mapFileFloat64(base, "step_positions.ftf")
	if err != nil {
		return nil, err
	}

	return &graph, nil
}

// Interface Implementation

type nodeReference struct {
	g     *graphFile
	index uint
}

type edgeReference struct {
	g     *graphFile
	index uint
}

// Graph

func (g *graphFile) NodeCount() int {
	return len(g.vertices) - 1
}

func (g *graphFile) EdgeCount() int {
	return len(g.edges)
}

func (g *graphFile) Node(i uint) Node {
	if i >= uint(g.NodeCount()) {
		panic("Node access out of bounds.")
	}
	return nodeReference{g, i}
}

func (g *graphFile) Edge(i uint) Edge {
	if i >= uint(g.EdgeCount()) {
		panic("Edge access out of bounds.")
	}
	return edgeReference{g, i}
}

func (g *graphFile) Positions() Positions {
	return g
}

// Node

func (ref nodeReference) Edges() []Edge {
	start := ref.g.vertices[ref.index]
	stop := ref.g.vertices[ref.index+1]
	degree := stop - start
	edges := make([]Edge, degree)
	for i, _ := range edges {
		edges[i] = edgeReference{ref.g, uint(ref.g.edges[int(start)+i])}
	}
	return edges
}

func (ref nodeReference) LatLng() (float64, float64) {
	lat := ref.g.positions[2*ref.index]
	lng := ref.g.positions[2*ref.index+1]
	return lat, lng
}

// Edge

func (ref edgeReference) Length() float64 {
	return ref.g.distances[ref.index]
}

func (ref edgeReference) StartPoint() Node {
	i := sort.Search(len(ref.g.vertices),
		func(i int) bool { return uint(ref.g.vertices[i]) > ref.index })
	return nodeReference{ref.g, uint(i - 1)}
}

func (ref edgeReference) EndPoint() Node {
	index := ref.g.edges[ref.index]
	return nodeReference{ref.g, uint(index)}
}

func (ref edgeReference) ReverseEdge() (Edge, bool) {
	index := ref.g.revEdges[ref.index]
	if uint(index) == ref.index {
		return ref, false
	}
	return edgeReference{ref.g, uint(index)}, true
}

func (ref edgeReference) Steps() []Step {
	start := ref.g.steps[ref.index]
	stop := ref.g.steps[ref.index+1]
	size := stop - start
	steps := make([]Step, size)
	for i, _ := range steps {
		lat := ref.g.stepPositions[2*(int(start)+i)]
		lng := ref.g.stepPositions[2*(int(start)+i)+1]
		steps[i] = Step{lat, lng}
	}
	return steps
}

// Positions

func (g *graphFile) Len() int {
	return g.NodeCount() + len(g.stepPositions) / 2
}

func (g *graphFile) Lat(i int) float64 {
	if i < g.NodeCount() {
		lat, _ := g.Node(uint(i)).LatLng()
		return lat
	}
	i -= g.NodeCount()
	return g.stepPositions[2*i]
}

func (g *graphFile) Lng(i int) float64 {
	if i < g.NodeCount() {
		_, lng := g.Node(uint(i)).LatLng()
		return lng
	}
	i -= g.NodeCount()
	return g.stepPositions[2*i+1]
}

func (g *graphFile) Step(i int) Step {
	if i < g.NodeCount() {
		lat, lng := g.Node(uint(i)).LatLng()
		return Step{lat, lng}
	}
	i -= g.NodeCount()
	return Step{g.stepPositions[2*i], g.stepPositions[2*i+1]}
}

func wayLength(steps []Step, geo ellipsoid.Ellipsoid) float64 {
	if len(steps) == 0 {
		return 0.0
	}
	total := 0.0
	prev  := steps[0]
	for _, step := range steps {
		distance, _ := geo.To(prev.Lat, prev.Lng, step.Lat, step.Lng)
		total += distance
		prev = step
	}
	return total
}

func reverse(steps []Step) {
	for i, j := 0, len(steps)-1; i < j; i, j = i+1, j-1 {
		steps[i], steps[j] = steps[j], steps[i]
	}
}

func (g *graphFile) Ways(i int, forward bool) []Way {
    if i < g.NodeCount() {
        w := make([]Way, 1)
		w[0] = Way{Length: 0, Node: g.Node(uint(i)), Steps: nil}
        return w
    }
    i -= g.NodeCount()
    // find the (edge, offset) pair for step i
	edgeIndex := sort.Search(len(g.steps),
		func(j int) bool { return uint(g.steps[j]) > uint(i) }) - 1
	offset := uint32(i) - g.steps[edgeIndex]
	edge   := g.Edge(uint(edgeIndex))
	// now we can allocate the way corresponding to (edge,offset),
	// but there are three cases to consider:
	// - if the way is bidirectional we have to compute both directions,
	//   if forward == true the from the offset two both endpoints,
	//   and the reverse otherwise
	// - if the way is unidirectional then we have to compute the way
	//   from the StartPoint to offset if forward == false
	// - otherwise we have to compute the way from offset to the EndPoint
	// Strictly speaking only the second case needs an additional binary
	// search in the form of edge.StartPoint, but let's keep this simple
	// for now.
	steps := edge.Steps()
	b1 := steps[:offset + 1]
	b2 := steps[offset:]
	l1 := wayLength(b1, g.geo)
	l2 := wayLength(b2, g.geo)
	t1 := edge.StartPoint()
	t2 := edge.EndPoint()

	if !forward {
		reverse(b1)
		reverse(b2)
	}

	var w []Way
	if _, ok := edge.ReverseEdge(); ok {
		w = make([]Way, 2) // bidirectional
		w[0] = Way{Length: l1, Node: t1, Steps: b1}
		w[1] = Way{Length: l2, Node: t2, Steps: b2}
	} else {
		w = make([]Way, 1) // one way
		if forward {
			w[0] = Way{Length: l2, Node: t2, Steps: b2}
		} else {
			w[0] = Way{Length: l1, Node: t1, Steps: b1}
		}
	}
	return w
}
<commit_msg>... guess.<commit_after>package graph

import (
	"os"
	"path"
	"reflect"
	"sort"
	"syscall"
	"unsafe"
	"ellipsoid"
)

type Node interface {
	Edges() []Edge
	LatLng() (float64, float64)
}

type Edge interface {
	Length() float64
	StartPoint() Node // e.g. via binary search on the node array
	EndPoint() Node
	ReverseEdge() (Edge, bool)
	Steps() []Step
	// Label() string
}

// "partial edge" is returned by the k-d tree
type Way struct {
	Length float64
	Node   Node // StartPoint or EndPoint
	Steps  []Step
}

type Graph interface {
	NodeCount() int
	EdgeCount() int
	Node(uint) Node
	Edge(uint) Edge
	Positions() Positions
}

// Implementation sketch (wrapper around graph):
// The graph is loaded before and is given to this.
// Init: Both position files (vertexes and inner steps in an edge) are alread loaded
//   for the graph (Nodes.LatLng(), Edge.Steps() - Step.Lat/Lng).
//   Thus despite storing a pointer the graph, nothing to do here.
// For every method where an index is given, there is a branch
//   if index < Graph.NodeCount()
//      work with Graph.Node(index)
//   else
//      work with the underlying Step array (index - Graph.NodeCount())
//      not possible efficiently with the current interface
//
// Positions is used for both creating the k-d tree in the preprocessing phase
// and for doing the nearest neighbor lookup during runtime.
type Positions interface {
	Len() int
	Lat(int) float64
	Lng(int) float64
	Step(int) Step
	Ways(int, bool) []Way // index, forward (i.e. looking at the edge in forward or in backward order)
}

type Step struct {
	Lat float64
	Lng float64
}

// Implementation

type graphFile struct {
	// ellipsoid for distance calculations
	geo ellipsoid.Ellipsoid
	// vertices maps a vertex index to the index of its first out edge
	vertices []uint32
	// edges maps edge indices to the index of the vertex endpoint
	edges []uint32
	// rev_edges maps an edge index to its reverse
	revEdges []uint32
	// map edge indices to distances
	distances []float64
	// map node indices to positions (lat: 2 * i, lng: 2 * i + 1)
	positions []float64
	// map an edge index to the inex of the first intermediate step
	steps []uint32
	// positions as interleaved lat/lng pairs as above
	stepPositions []float64
}

func mapFile(base, name string) ([]byte, error) {
	file, err := os.Open(path.Join(base, name))
	if err != nil {
		return nil, err
	}
	info, err := file.Stat()
	if err != nil {
		return nil, err
	}
	// Thanks to Windows compatibility file.Fd is not declared int...
	fdfu := file.Fd()
	fd := *(*int)(unsafe.Pointer(&fdfu))
	// This is bad. Slices have int size and capacity fields, which
	// means that we might truncate here. We can work around this issue
	// by using unsafe.Pointer internally and only convert to slices for
	// individual edge/step lists... But for now our files are small
	// and this works:
	size := int(info.Size())
	return syscall.Mmap(fd, 0, size, syscall.PROT_READ, syscall.MAP_PRIVATE)
}

func mapFileUint32(base, name string) ([]uint32, error) {
	m, err := mapFile(base, name)
	if err != nil {
		return nil, err
	}

	dh := (*reflect.SliceHeader)(unsafe.Pointer(&m))
	dh.Len /= 4
	dh.Cap /= 4
	return *(*[]uint32)(unsafe.Pointer(&m)), nil
}

func mapFileFloat64(base, name string) ([]float64, error) {
	m, err := mapFile(base, name)
	if err != nil {
		return nil, err
	}

	dh := (*reflect.SliceHeader)(unsafe.Pointer(&m))
	dh.Len /= 8
	dh.Cap /= 8
	return *(*[]float64)(unsafe.Pointer(&m)), nil
}

func Open(base string) (Graph, error) {
	graph := graphFile{}

	graph.geo = ellipsoid.Init("WGS84", ellipsoid.Degrees, ellipsoid.Meter,
		ellipsoid.Longitude_is_symmetric, ellipsoid.Bearing_is_symmetric)

	var err error
	graph.vertices, err = mapFileUint32(base, "vertices.ftf")
	if err != nil {
		return nil, err
	}

	graph.edges, err = mapFileUint32(base, "edges.ftf")
	if err != nil {
		return nil, err
	}

	graph.revEdges, err = mapFileUint32(base, "rev_edges.ftf")
	if err != nil {
		return nil, err
	}

	graph.distances, err = mapFileFloat64(base, "distances.ftf")
	if err != nil {
		return nil, err
	}

	graph.positions, err = mapFileFloat64(base, "positions.ftf")
	if err != nil {
		return nil, err
	}

	graph.steps, err = mapFileUint32(base, "steps.ftf")
	if err != nil {
		return nil, err
	}

	graph.stepPositions, err = mapFileFloat64(base, "step_positions.ftf")
	if err != nil {
		return nil, err
	}

	return &graph, nil
}

// Interface Implementation

type nodeReference struct {
	g     *graphFile
	index uint
}

type edgeReference struct {
	g     *graphFile
	index uint
}

// Graph

func (g *graphFile) NodeCount() int {
	return len(g.vertices) - 1
}

func (g *graphFile) EdgeCount() int {
	return len(g.edges)
}

func (g *graphFile) Node(i uint) Node {
	if i >= uint(g.NodeCount()) {
		panic("Node access out of bounds.")
	}
	return nodeReference{g, i}
}

func (g *graphFile) Edge(i uint) Edge {
	if i >= uint(g.EdgeCount()) {
		panic("Edge access out of bounds.")
	}
	return edgeReference{g, i}
}

func (g *graphFile) Positions() Positions {
	return g
}

// Node

func (ref nodeReference) Edges() []Edge {
	start := ref.g.vertices[ref.index]
	stop := ref.g.vertices[ref.index+1]
	degree := stop - start
	edges := make([]Edge, degree)
	for i, _ := range edges {
		edges[i] = edgeReference{ref.g, uint(ref.g.edges[int(start)+i])}
	}
	return edges
}

func (ref nodeReference) LatLng() (float64, float64) {
	lat := ref.g.positions[2*ref.index]
	lng := ref.g.positions[2*ref.index+1]
	return lat, lng
}

// Edge

func (ref edgeReference) Length() float64 {
	return ref.g.distances[ref.index]
}

func (ref edgeReference) StartPoint() Node {
	i := sort.Search(len(ref.g.vertices),
		func(i int) bool { return uint(ref.g.vertices[i]) > ref.index })
	return nodeReference{ref.g, uint(i - 1)}
}

func (ref edgeReference) EndPoint() Node {
	index := ref.g.edges[ref.index]
	return nodeReference{ref.g, uint(index)}
}

func (ref edgeReference) ReverseEdge() (Edge, bool) {
	index := ref.g.revEdges[ref.index]
	if uint(index) == ref.index {
		return ref, false
	}
	return edgeReference{ref.g, uint(index)}, true
}

func (ref edgeReference) Steps() []Step {
	start := ref.g.steps[ref.index]
	stop := ref.g.steps[ref.index+1]
	size := stop - start
	steps := make([]Step, size)
	for i, _ := range steps {
		lat := ref.g.stepPositions[2*(int(start)+i)]
		lng := ref.g.stepPositions[2*(int(start)+i)+1]
		steps[i] = Step{lat, lng}
	}
	return steps
}

// Positions

func (g *graphFile) Len() int {
	return g.NodeCount() + len(g.stepPositions) / 2
}

func (g *graphFile) Lat(i int) float64 {
	if i < g.NodeCount() {
		lat, _ := g.Node(uint(i)).LatLng()
		return lat
	}
	i -= g.NodeCount()
	return g.stepPositions[2*i]
}

func (g *graphFile) Lng(i int) float64 {
	if i < g.NodeCount() {
		_, lng := g.Node(uint(i)).LatLng()
		return lng
	}
	i -= g.NodeCount()
	return g.stepPositions[2*i+1]
}

func (g *graphFile) Step(i int) Step {
	if i < g.NodeCount() {
		lat, lng := g.Node(uint(i)).LatLng()
		return Step{lat, lng}
	}
	i -= g.NodeCount()
	return Step{g.stepPositions[2*i], g.stepPositions[2*i+1]}
}

func wayLength(steps []Step, geo ellipsoid.Ellipsoid) float64 {
	if len(steps) == 0 {
		return 0.0
	}
	total := 0.0
	prev  := steps[0]
	for _, step := range steps {
		distance, _ := geo.To(prev.Lat, prev.Lng, step.Lat, step.Lng)
		total += distance
		prev = step
	}
	return total
}

func reverse(steps []Step) {
	for i, j := 0, len(steps)-1; i < j; i, j = i+1, j-1 {
		steps[i], steps[j] = steps[j], steps[i]
	}
}

func (g *graphFile) Ways(i int, forward bool) []Way {
    if i < g.NodeCount() {
        w := make([]Way, 1)
		w[0] = Way{Length: 0, Node: g.Node(uint(i)), Steps: nil}
        return w
    }
    i -= g.NodeCount()
    // find the (edge, offset) pair for step i
	edgeIndex := sort.Search(len(g.steps),
		func(j int) bool { return uint(g.steps[j]) > uint(i) }) - 1
	offset := uint32(i) - g.steps[edgeIndex]
	edge   := g.Edge(uint(edgeIndex))
	// now we can allocate the way corresponding to (edge,offset),
	// but there are three cases to consider:
	// - if the way is bidirectional we have to compute both directions,
	//   if forward == true the from the offset two both endpoints,
	//   and the reverse otherwise
	// - if the way is unidirectional then we have to compute the way
	//   from the StartPoint to offset if forward == false
	// - otherwise we have to compute the way from offset to the EndPoint
	// Strictly speaking only the second case needs an additional binary
	// search in the form of edge.StartPoint, but let's keep this simple
	// for now.
	steps := edge.Steps()
	b1 := steps[:offset + 1]
	b2 := steps[offset:]
	l1 := wayLength(b1, g.geo)
	l2 := wayLength(b2, g.geo)
	t1 := edge.StartPoint()
	t2 := edge.EndPoint()

	if !forward {
		reverse(b1)
		reverse(b2)
	}

	var w []Way
	if _, ok := edge.ReverseEdge(); ok {
		w = make([]Way, 2) // bidirectional
		w[0] = Way{Length: l1, Node: t1, Steps: b1}
		w[1] = Way{Length: l2, Node: t2, Steps: b2}
	} else {
		w = make([]Way, 1) // one way
		if forward {
			w[0] = Way{Length: l2, Node: t2, Steps: b2}
		} else {
			w[0] = Way{Length: l1, Node: t1, Steps: b1}
		}
	}
	return w
}
<|endoftext|>


<commit_before>// Copyright 2014 Brave New Software

//    Licensed under the Apache License, Version 2.0 (the "License");
//    you may not use this file except in compliance with the License.
//    You may obtain a copy of the License at

//        http://www.apache.org/licenses/LICENSE-2.0

//    Unless required by applicable law or agreed to in writing, software
//    distributed under the License is distributed on an "AS IS" BASIS,
//    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
//    See the License for the specific language governing permissions and
//    limitations under the License.
//

package statshub

import (
	"encoding/json"
	"fmt"
	"log"
	"net/http"
	"path"
	"strings"
	"time"
)

const (
	cacheExpiration = 1 * time.Minute
)

var (
	countriesCached     bool
	countryStats        []byte
	countryStatsRequest = make(chan chan []byte, 1000)
)

// ClientQueryResponse is a Response to a StatsQuery
type ClientQueryResponse struct {
	Response
	Dims map[string]map[string]*Stats `json:"dims"`
}

// Response is a response to a stats request (update or query)
type Response struct {
	Succeeded bool   `json:"succeeded"`
	Error     string `json:"error"`
}

func init() {
	http.HandleFunc("/stats/", statsHandler)
	go cacheCountries()
}

// statsHandler handles requests to /stats
func statsHandler(w http.ResponseWriter, r *http.Request) {
	var id string
	var err error
	if id, err = path.Base(r.URL.Path); err != nil {
		fail(w, 400, err)
	}

	if "POST" == r.Method {
		if id == "" {
			id = "unknown"
		}

		w.Header().Set("Content-Type", "application/json")

		statusCode, resp, err := postStats(r, id)
		if err != nil {
			fail(w, statusCode, err)
		} else {
			write(w, 200, resp)
		}
	} else if "GET" == r.Method {
		w.Header().Set("Content-Type", "application/json")

		// check cache
		// TODO: should probably cache more than just the 1 url
		path := r.URL.Path
		requestedCountryStats := path == "/stats/country" || path == "/stats/country/"
		if requestedCountryStats {
			countryResponse := make(chan []byte)
			countryStatsRequest <- countryResponse
			cached := <-countryResponse
			if cached == nil {
				fail(w, 500, fmt.Errorf("No countries cached"))
			} else {
				w.WriteHeader(200)
				w.Write(cached)
				return
			}
		} else {
			log.Printf("Not using cache for path: %s", path)
		}
		statusCode, resp, err := getStats(id)
		if err != nil {
			fail(w, statusCode, err)
		} else {
			w.WriteHeader(statusCode)
			bytes, err := json.Marshal(resp)
			if err == nil {
				w.Write(bytes)
			} else {
				log.Printf("Unable to respond to client: %s", err)
			}
		}
	} else {
		log.Printf("Query: %s", r.URL.Query())
		w.WriteHeader(405)
	}
}

func cacheCountries() {
	queryCountries()
	for {
		nextInterval := time.Now().Truncate(cacheExpiration).Add(cacheExpiration)
		waitTime := nextInterval.Sub(time.Now())
		select {
		case req := <-countryStatsRequest:
			if countriesCached {
				req <- countryStats
			} else {
				req <- nil
			}
		case <-time.After(waitTime):
			queryCountries()
		}
	}
}

func queryCountries() {
	log.Printf("Querying countries for cache")
	_, resp, err := getStats("country")
	if err != nil {
		log.Printf("Unable to cache countries: %s", err)
		countriesCached = false
	} else {
		bytes, err := json.Marshal(resp)
		if err == nil {
			countryStats = bytes
			countriesCached = true
		} else {
			countriesCached = false
			log.Printf("Unable to cache countries: %s", err)
		}
	}
}

// postStats handles a POST request to /stats
func postStats(r *http.Request, id string) (statusCode int, resp interface{}, err error) {
	decoder := json.NewDecoder(r.Body)
	stats := &StatsUpdate{}
	err = decoder.Decode(stats)
	if err != nil {
		return 400, nil, fmt.Errorf("Unable to decode request: %s", err)
	}

	if err = stats.write(id); err != nil {
		formattedError := fmt.Errorf("Unable to post stats: %s", err)
		log.Println(formattedError)
		return 500, nil, formattedError
	}

	return 200, &Response{Succeeded: true}, nil
}

// getStats handles a GET request to /stats
func getStats(dim string) (statusCode int, resp interface{}, err error) {
	clientResp := &ClientQueryResponse{
		Response: Response{Succeeded: true},
	}

	dimNames := dimNamesFor(dim)

	if clientResp.Dims, err = QueryDims(dimNames); err != nil {
		return 500, nil, fmt.Errorf("Unable to query stats: %s", err)
	}

	return 200, clientResp, nil
}

func dimNamesFor(dim string) []string {
	if dim != "" {
		return []string{dim}
	} else {
		return nil
	}
}

func fail(w http.ResponseWriter, statusCode int, err error) {
	response := Response{Succeeded: false, Error: fmt.Sprintf("%s", err)}
	write(w, statusCode, response)
}

func write(w http.ResponseWriter, statusCode int, data interface{}) {
	w.WriteHeader(statusCode)
	bytes, err := json.Marshal(data)
	if err == nil {
		w.Write(bytes)
	} else {
		log.Printf("Unable to respond to client: %s", err)
	}
}
<commit_msg>Better path parsing<commit_after>// Copyright 2014 Brave New Software

//    Licensed under the Apache License, Version 2.0 (the "License");
//    you may not use this file except in compliance with the License.
//    You may obtain a copy of the License at

//        http://www.apache.org/licenses/LICENSE-2.0

//    Unless required by applicable law or agreed to in writing, software
//    distributed under the License is distributed on an "AS IS" BASIS,
//    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
//    See the License for the specific language governing permissions and
//    limitations under the License.
//

package statshub

import (
	"encoding/json"
	"fmt"
	"log"
	"net/http"
	"path"
	"time"
)

const (
	cacheExpiration = 1 * time.Minute
)

var (
	countriesCached     bool
	countryStats        []byte
	countryStatsRequest = make(chan chan []byte, 1000)
)

// ClientQueryResponse is a Response to a StatsQuery
type ClientQueryResponse struct {
	Response
	Dims map[string]map[string]*Stats `json:"dims"`
}

// Response is a response to a stats request (update or query)
type Response struct {
	Succeeded bool   `json:"succeeded"`
	Error     string `json:"error"`
}

func init() {
	http.HandleFunc("/stats/", statsHandler)
	go cacheCountries()
}

// statsHandler handles requests to /stats
func statsHandler(w http.ResponseWriter, r *http.Request) {
	id := path.Base(r.URL.Path)

	if "POST" == r.Method {
		if id == "" {
			id = "unknown"
		}

		w.Header().Set("Content-Type", "application/json")

		statusCode, resp, err := postStats(r, id)
		if err != nil {
			fail(w, statusCode, err)
		} else {
			write(w, 200, resp)
		}
	} else if "GET" == r.Method {
		w.Header().Set("Content-Type", "application/json")

		// check cache
		// TODO: should probably cache more than just the 1 url
		path := r.URL.Path
		requestedCountryStats := path == "/stats/country" || path == "/stats/country/"
		if requestedCountryStats {
			countryResponse := make(chan []byte)
			countryStatsRequest <- countryResponse
			cached := <-countryResponse
			if cached == nil {
				fail(w, 500, fmt.Errorf("No countries cached"))
			} else {
				w.WriteHeader(200)
				w.Write(cached)
				return
			}
		} else {
			log.Printf("Not using cache for path: %s", path)
		}
		statusCode, resp, err := getStats(id)
		if err != nil {
			fail(w, statusCode, err)
		} else {
			w.WriteHeader(statusCode)
			bytes, err := json.Marshal(resp)
			if err == nil {
				w.Write(bytes)
			} else {
				log.Printf("Unable to respond to client: %s", err)
			}
		}
	} else {
		log.Printf("Query: %s", r.URL.Query())
		w.WriteHeader(405)
	}
}

func cacheCountries() {
	queryCountries()
	for {
		nextInterval := time.Now().Truncate(cacheExpiration).Add(cacheExpiration)
		waitTime := nextInterval.Sub(time.Now())
		select {
		case req := <-countryStatsRequest:
			if countriesCached {
				req <- countryStats
			} else {
				req <- nil
			}
		case <-time.After(waitTime):
			queryCountries()
		}
	}
}

func queryCountries() {
	log.Printf("Querying countries for cache")
	_, resp, err := getStats("country")
	if err != nil {
		log.Printf("Unable to cache countries: %s", err)
		countriesCached = false
	} else {
		bytes, err := json.Marshal(resp)
		if err == nil {
			countryStats = bytes
			countriesCached = true
		} else {
			countriesCached = false
			log.Printf("Unable to cache countries: %s", err)
		}
	}
}

// postStats handles a POST request to /stats
func postStats(r *http.Request, id string) (statusCode int, resp interface{}, err error) {
	decoder := json.NewDecoder(r.Body)
	stats := &StatsUpdate{}
	err = decoder.Decode(stats)
	if err != nil {
		return 400, nil, fmt.Errorf("Unable to decode request: %s", err)
	}

	if err = stats.write(id); err != nil {
		formattedError := fmt.Errorf("Unable to post stats: %s", err)
		log.Println(formattedError)
		return 500, nil, formattedError
	}

	return 200, &Response{Succeeded: true}, nil
}

// getStats handles a GET request to /stats
func getStats(dim string) (statusCode int, resp interface{}, err error) {
	clientResp := &ClientQueryResponse{
		Response: Response{Succeeded: true},
	}

	dimNames := dimNamesFor(dim)

	if clientResp.Dims, err = QueryDims(dimNames); err != nil {
		return 500, nil, fmt.Errorf("Unable to query stats: %s", err)
	}

	return 200, clientResp, nil
}

func dimNamesFor(dim string) []string {
	if dim != "" {
		return []string{dim}
	} else {
		return nil
	}
}

func fail(w http.ResponseWriter, statusCode int, err error) {
	response := Response{Succeeded: false, Error: fmt.Sprintf("%s", err)}
	write(w, statusCode, response)
}

func write(w http.ResponseWriter, statusCode int, data interface{}) {
	w.WriteHeader(statusCode)
	bytes, err := json.Marshal(data)
	if err == nil {
		w.Write(bytes)
	} else {
		log.Printf("Unable to respond to client: %s", err)
	}
}
<|endoftext|>


<commit_before>package store_test

import (
	"bytes"
	"fmt"
	"io/ioutil"
	. "launchpad.net/gocheck"
	"launchpad.net/juju-core/charm"
	"launchpad.net/juju-core/store"
	"launchpad.net/juju-core/testing"
	"os"
	"os/exec"
	"path/filepath"
	"strings"
	"time"
)

func (s *StoreSuite) dummyBranch(c *C, suffix string) bzrDir {
	tmpDir := c.MkDir()
	if suffix != "" {
		tmpDir = filepath.Join(tmpDir, suffix)
		err := os.MkdirAll(tmpDir, 0755)
		c.Assert(err, IsNil)
	}
	branch := bzrDir(tmpDir)
	branch.init()

	copyCharmDir(branch.path(), testing.Charms.Dir("dummy"))
	branch.add()
	branch.commit("Imported charm.")
	return branch
}

var urls = []*charm.URL{
	charm.MustParseURL("cs:~joe/oneiric/dummy"),
	charm.MustParseURL("cs:oneiric/dummy"),
}

type fakePlugin struct {
	oldEnv string
}

func (p *fakePlugin) install(dir string, content string) {
	p.oldEnv = os.Getenv("BZR_PLUGINS_AT")
	err := ioutil.WriteFile(filepath.Join(dir, "__init__.py"), []byte(content), 0644)
	if err != nil {
		panic(err)
	}
	os.Setenv("BZR_PLUGINS_AT", "fakePlugin@" + dir)
}

func (p *fakePlugin) uninstall() {
	os.Setenv("BZR_PLUGINS_AT", p.oldEnv)
}

func (s *StoreSuite) TestPublish(c *C) {
	branch := s.dummyBranch(c, "")

	// Ensure that the streams are parsed separately by inserting
	// garbage on stderr. The wanted information is still there.
	plugin := fakePlugin{}
	plugin.install(c.MkDir(), `import sys; sys.stderr.write("STDERR STUFF FROM TEST\n")`)
	defer plugin.uninstall()

	err := store.PublishBazaarBranch(s.store, urls, branch.path(), "wrong-rev")
	c.Assert(err, IsNil)

	for _, url := range urls {
		info, rc, err := s.store.OpenCharm(url)
		c.Assert(err, IsNil)
		defer rc.Close()
		c.Assert(info.Revision(), Equals, 0)
		c.Assert(info.Meta().Name, Equals, "dummy")

		data, err := ioutil.ReadAll(rc)
		c.Assert(err, IsNil)

		bundle, err := charm.ReadBundleBytes(data)
		c.Assert(err, IsNil)
		c.Assert(bundle.Revision(), Equals, 0)
		c.Assert(bundle.Meta().Name, Equals, "dummy")
	}

	// Attempt to publish the same content again while providing the wrong
	// tip revision. It must pick the real revision from the branch and
	// note this was previously published.
	err = store.PublishBazaarBranch(s.store, urls, branch.path(), "wrong-rev")
	c.Assert(err, Equals, store.ErrRedundantUpdate)

	// Bump the content revision and lie again about the known tip revision.
	// This time, though, pretend it's the same as the real branch revision
	// previously published. It must error and not publish the new revision
	// because it will use the revision provided as a parameter to check if
	// publishing was attempted before. This is the mechanism that enables
	// stopping fast without having to download every single branch. Real
	// revision is picked in the next scan.
	digest1 := branch.digest()
	branch.change()
	err = store.PublishBazaarBranch(s.store, urls, branch.path(), digest1)
	c.Assert(err, Equals, store.ErrRedundantUpdate)

	// Now allow it to publish the new content by providing an unseen revision.
	err = store.PublishBazaarBranch(s.store, urls, branch.path(), "wrong-rev")
	c.Assert(err, IsNil)
	digest2 := branch.digest()

	info, err := s.store.CharmInfo(urls[0])
	c.Assert(err, IsNil)
	c.Assert(info.Revision(), Equals, 1)
	c.Assert(info.Meta().Name, Equals, "dummy")

	// There are two events published, for each of the successful attempts.
	// The failures are ignored given that they are artifacts of the
	// publishing mechanism rather than actual problems.
	_, err = s.store.CharmEvent(urls[0], "wrong-rev")
	c.Assert(err, Equals, store.ErrNotFound)
	for i, digest := range []string{digest1, digest2} {
		event, err := s.store.CharmEvent(urls[0], digest)
		c.Assert(err, IsNil)
		c.Assert(event.Kind, Equals, store.EventPublished)
		c.Assert(event.Revision, Equals, i)
		c.Assert(event.Errors, IsNil)
		c.Assert(event.Warnings, IsNil)
	}
}

func (s *StoreSuite) TestPublishErrorFromBzr(c *C) {
	branch := s.dummyBranch(c, "")

	// In TestPublish we ensure that the streams are parsed
	// separately by inserting garbage on stderr. Now make
	// sure that stderr isn't simply trashed, as we want to
	// know about what a real error tells us.
	plugin := fakePlugin{}
	plugin.install(c.MkDir(), `import sys; sys.stderr.write("STDERR STUFF FROM TEST\n"); sys.exit(1)`)
	defer plugin.uninstall()

	err := store.PublishBazaarBranch(s.store, urls, branch.path(), "wrong-rev")
	c.Assert(err, ErrorMatches, "(?s).*STDERR STUFF.*")
}

func (s *StoreSuite) TestPublishErrorInCharm(c *C) {
	branch := s.dummyBranch(c, "")

	// Corrupt the charm.
	branch.remove("metadata.yaml")
	branch.commit("Removed metadata.yaml.")

	// Attempt to publish the erroneous content.
	err := store.PublishBazaarBranch(s.store, urls, branch.path(), "wrong-rev")
	c.Assert(err, ErrorMatches, ".*/metadata.yaml: no such file or directory")

	// The event should be logged as well, since this was an error in the charm
	// that won't go away and must be communicated to the author.
	event, err := s.store.CharmEvent(urls[0], branch.digest())
	c.Assert(err, IsNil)
	c.Assert(event.Kind, Equals, store.EventPublishError)
	c.Assert(event.Revision, Equals, 0)
	c.Assert(event.Errors, NotNil)
	c.Assert(event.Errors[0], Matches, ".*/metadata.yaml: no such file or directory")
	c.Assert(event.Warnings, IsNil)
}

type bzrDir string

func (dir bzrDir) path(args ...string) string {
	return filepath.Join(append([]string{string(dir)}, args...)...)
}

func (dir bzrDir) run(args ...string) []byte {
	cmd := exec.Command("bzr", args...)
	cmd.Dir = string(dir)
	output, err := cmd.Output()
	if err != nil {
		panic(fmt.Sprintf("command failed: bzr %s\n%s", strings.Join(args, " "), output))
	}
	return output
}

func (dir bzrDir) init() {
	dir.run("init")
}

func (dir bzrDir) add(paths ...string) {
	dir.run(append([]string{"add"}, paths...)...)
}

func (dir bzrDir) remove(paths ...string) {
	dir.run(append([]string{"rm"}, paths...)...)
}

func (dir bzrDir) commit(msg string) {
	dir.run("commit", "-m", msg)
}

func (dir bzrDir) write(path string, data string) {
	err := ioutil.WriteFile(dir.path(path), []byte(data), 0644)
	if err != nil {
		panic(err)
	}
}

func (dir bzrDir) change() {
	t := time.Now().String()
	dir.write("timestamp", t)
	dir.add("timestamp")
	dir.commit("Revision bumped at " + t)
}

func (dir bzrDir) digest() string {
	output := dir.run("revision-info")
	f := bytes.Fields(output)
	if len(f) != 2 {
		panic("revision-info returned bad output: " + string(output))
	}
	return string(f[1])
}

func copyCharmDir(dst string, dir *charm.Dir) {
	var b bytes.Buffer
	err := dir.BundleTo(&b)
	if err != nil {
		panic(err)
	}
	bundle, err := charm.ReadBundleBytes(b.Bytes())
	if err != nil {
		panic(err)
	}
	err = bundle.ExpandTo(dst)
	if err != nil {
		panic(err)
	}
}
<commit_msg>go fmt<commit_after>package store_test

import (
	"bytes"
	"fmt"
	"io/ioutil"
	. "launchpad.net/gocheck"
	"launchpad.net/juju-core/charm"
	"launchpad.net/juju-core/store"
	"launchpad.net/juju-core/testing"
	"os"
	"os/exec"
	"path/filepath"
	"strings"
	"time"
)

func (s *StoreSuite) dummyBranch(c *C, suffix string) bzrDir {
	tmpDir := c.MkDir()
	if suffix != "" {
		tmpDir = filepath.Join(tmpDir, suffix)
		err := os.MkdirAll(tmpDir, 0755)
		c.Assert(err, IsNil)
	}
	branch := bzrDir(tmpDir)
	branch.init()

	copyCharmDir(branch.path(), testing.Charms.Dir("dummy"))
	branch.add()
	branch.commit("Imported charm.")
	return branch
}

var urls = []*charm.URL{
	charm.MustParseURL("cs:~joe/oneiric/dummy"),
	charm.MustParseURL("cs:oneiric/dummy"),
}

type fakePlugin struct {
	oldEnv string
}

func (p *fakePlugin) install(dir string, content string) {
	p.oldEnv = os.Getenv("BZR_PLUGINS_AT")
	err := ioutil.WriteFile(filepath.Join(dir, "__init__.py"), []byte(content), 0644)
	if err != nil {
		panic(err)
	}
	os.Setenv("BZR_PLUGINS_AT", "fakePlugin@"+dir)
}

func (p *fakePlugin) uninstall() {
	os.Setenv("BZR_PLUGINS_AT", p.oldEnv)
}

func (s *StoreSuite) TestPublish(c *C) {
	branch := s.dummyBranch(c, "")

	// Ensure that the streams are parsed separately by inserting
	// garbage on stderr. The wanted information is still there.
	plugin := fakePlugin{}
	plugin.install(c.MkDir(), `import sys; sys.stderr.write("STDERR STUFF FROM TEST\n")`)
	defer plugin.uninstall()

	err := store.PublishBazaarBranch(s.store, urls, branch.path(), "wrong-rev")
	c.Assert(err, IsNil)

	for _, url := range urls {
		info, rc, err := s.store.OpenCharm(url)
		c.Assert(err, IsNil)
		defer rc.Close()
		c.Assert(info.Revision(), Equals, 0)
		c.Assert(info.Meta().Name, Equals, "dummy")

		data, err := ioutil.ReadAll(rc)
		c.Assert(err, IsNil)

		bundle, err := charm.ReadBundleBytes(data)
		c.Assert(err, IsNil)
		c.Assert(bundle.Revision(), Equals, 0)
		c.Assert(bundle.Meta().Name, Equals, "dummy")
	}

	// Attempt to publish the same content again while providing the wrong
	// tip revision. It must pick the real revision from the branch and
	// note this was previously published.
	err = store.PublishBazaarBranch(s.store, urls, branch.path(), "wrong-rev")
	c.Assert(err, Equals, store.ErrRedundantUpdate)

	// Bump the content revision and lie again about the known tip revision.
	// This time, though, pretend it's the same as the real branch revision
	// previously published. It must error and not publish the new revision
	// because it will use the revision provided as a parameter to check if
	// publishing was attempted before. This is the mechanism that enables
	// stopping fast without having to download every single branch. Real
	// revision is picked in the next scan.
	digest1 := branch.digest()
	branch.change()
	err = store.PublishBazaarBranch(s.store, urls, branch.path(), digest1)
	c.Assert(err, Equals, store.ErrRedundantUpdate)

	// Now allow it to publish the new content by providing an unseen revision.
	err = store.PublishBazaarBranch(s.store, urls, branch.path(), "wrong-rev")
	c.Assert(err, IsNil)
	digest2 := branch.digest()

	info, err := s.store.CharmInfo(urls[0])
	c.Assert(err, IsNil)
	c.Assert(info.Revision(), Equals, 1)
	c.Assert(info.Meta().Name, Equals, "dummy")

	// There are two events published, for each of the successful attempts.
	// The failures are ignored given that they are artifacts of the
	// publishing mechanism rather than actual problems.
	_, err = s.store.CharmEvent(urls[0], "wrong-rev")
	c.Assert(err, Equals, store.ErrNotFound)
	for i, digest := range []string{digest1, digest2} {
		event, err := s.store.CharmEvent(urls[0], digest)
		c.Assert(err, IsNil)
		c.Assert(event.Kind, Equals, store.EventPublished)
		c.Assert(event.Revision, Equals, i)
		c.Assert(event.Errors, IsNil)
		c.Assert(event.Warnings, IsNil)
	}
}

func (s *StoreSuite) TestPublishErrorFromBzr(c *C) {
	branch := s.dummyBranch(c, "")

	// In TestPublish we ensure that the streams are parsed
	// separately by inserting garbage on stderr. Now make
	// sure that stderr isn't simply trashed, as we want to
	// know about what a real error tells us.
	plugin := fakePlugin{}
	plugin.install(c.MkDir(), `import sys; sys.stderr.write("STDERR STUFF FROM TEST\n"); sys.exit(1)`)
	defer plugin.uninstall()

	err := store.PublishBazaarBranch(s.store, urls, branch.path(), "wrong-rev")
	c.Assert(err, ErrorMatches, "(?s).*STDERR STUFF.*")
}

func (s *StoreSuite) TestPublishErrorInCharm(c *C) {
	branch := s.dummyBranch(c, "")

	// Corrupt the charm.
	branch.remove("metadata.yaml")
	branch.commit("Removed metadata.yaml.")

	// Attempt to publish the erroneous content.
	err := store.PublishBazaarBranch(s.store, urls, branch.path(), "wrong-rev")
	c.Assert(err, ErrorMatches, ".*/metadata.yaml: no such file or directory")

	// The event should be logged as well, since this was an error in the charm
	// that won't go away and must be communicated to the author.
	event, err := s.store.CharmEvent(urls[0], branch.digest())
	c.Assert(err, IsNil)
	c.Assert(event.Kind, Equals, store.EventPublishError)
	c.Assert(event.Revision, Equals, 0)
	c.Assert(event.Errors, NotNil)
	c.Assert(event.Errors[0], Matches, ".*/metadata.yaml: no such file or directory")
	c.Assert(event.Warnings, IsNil)
}

type bzrDir string

func (dir bzrDir) path(args ...string) string {
	return filepath.Join(append([]string{string(dir)}, args...)...)
}

func (dir bzrDir) run(args ...string) []byte {
	cmd := exec.Command("bzr", args...)
	cmd.Dir = string(dir)
	output, err := cmd.Output()
	if err != nil {
		panic(fmt.Sprintf("command failed: bzr %s\n%s", strings.Join(args, " "), output))
	}
	return output
}

func (dir bzrDir) init() {
	dir.run("init")
}

func (dir bzrDir) add(paths ...string) {
	dir.run(append([]string{"add"}, paths...)...)
}

func (dir bzrDir) remove(paths ...string) {
	dir.run(append([]string{"rm"}, paths...)...)
}

func (dir bzrDir) commit(msg string) {
	dir.run("commit", "-m", msg)
}

func (dir bzrDir) write(path string, data string) {
	err := ioutil.WriteFile(dir.path(path), []byte(data), 0644)
	if err != nil {
		panic(err)
	}
}

func (dir bzrDir) change() {
	t := time.Now().String()
	dir.write("timestamp", t)
	dir.add("timestamp")
	dir.commit("Revision bumped at " + t)
}

func (dir bzrDir) digest() string {
	output := dir.run("revision-info")
	f := bytes.Fields(output)
	if len(f) != 2 {
		panic("revision-info returned bad output: " + string(output))
	}
	return string(f[1])
}

func copyCharmDir(dst string, dir *charm.Dir) {
	var b bytes.Buffer
	err := dir.BundleTo(&b)
	if err != nil {
		panic(err)
	}
	bundle, err := charm.ReadBundleBytes(b.Bytes())
	if err != nil {
		panic(err)
	}
	err = bundle.ExpandTo(dst)
	if err != nil {
		panic(err)
	}
}
<|endoftext|>


<commit_before>// Copyright 2012 Aaron Jacobs. All Rights Reserved.
// Author: aaronjjacobs@gmail.com (Aaron Jacobs)
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package blob

import (
	"fmt"
)

// Return a blob store that wraps the supplied one, confirming that the blob
// contents and scores it returns are correct, guarding against silent data
// corruption.
func NewCheckingStore(wrapped Store) Store {
	return &checkingStore{wrapped}
}

type checkingStore struct {
	wrapped Store
}

func (s *checkingStore) Store(blob []byte) (score Score, err error) {
	// Call the wrapped store.
	if score, err = s.wrapped.Store(blob); err != nil {
		return
	}

	// Check its result.
	expected := ComputeScore(blob)
	if score != expected {
		err = fmt.Errorf(
			"Incorrect score returned for blob; %s vs %s.",
			score.Hex(),
			expected.Hex())

		return
	}

	return
}

func (s *checkingStore) Load(score Score) (blob []byte, err error) {
	// Call the wrapped store.
	if blob, err = s.wrapped.Load(score); err != nil {
		return
	}

	// Check its result.
	actual := ComputeScore(blob)
	if actual != score {
		return nil, fmt.Errorf(
			"Incorrect data returned for blob; requested score is %s actual is %s.",
			score.Hex(),
			actual.Hex())
	}

	return
}

func (s *checkingStore) Flush() (err error) {
	err = s.wrapped.Flush()
	return
}

func (s *checkingStore) Contains(score Score) (b bool) {
	b = s.wrapped.Contains(score)
	return
}
<commit_msg>checkingStore<commit_after>// Copyright 2012 Aaron Jacobs. All Rights Reserved.
// Author: aaronjjacobs@gmail.com (Aaron Jacobs)
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package blob

import (
	"fmt"

	"golang.org/x/net/context"
)

// Return a blob store that wraps the supplied one, confirming that the blob
// contents and scores it returns are correct, guarding against silent data
// corruption.
func NewCheckingStore(wrapped Store) Store {
	return &checkingStore{wrapped}
}

type checkingStore struct {
	wrapped Store
}

func (s *checkingStore) Store(
	ctx context.Context,
	blob []byte) (score Score, err error) {
	// Call the wrapped store.
	if score, err = s.wrapped.Store(ctx, blob); err != nil {
		return
	}

	// Check its result.
	expected := ComputeScore(blob)
	if score != expected {
		err = fmt.Errorf(
			"Incorrect score returned for blob; %s vs %s.",
			score.Hex(),
			expected.Hex())

		return
	}

	return
}

func (s *checkingStore) Load(
	ctx context.Context,
	score Score) (blob []byte, err error) {
	// Call the wrapped store.
	if blob, err = s.wrapped.Load(ctx, score); err != nil {
		return
	}

	// Check its result.
	actual := ComputeScore(blob)
	if actual != score {
		return nil, fmt.Errorf(
			"Incorrect data returned for blob; requested score is %s actual is %s.",
			score.Hex(),
			actual.Hex())
	}

	return
}

func (s *checkingStore) Flush(ctx context.Context) (err error) {
	err = s.wrapped.Flush(ctx)
	return
}

func (s *checkingStore) Contains(ctx context.Context, score Score) (b bool) {
	b = s.wrapped.Contains(ctx, score)
	return
}
<|endoftext|>


<commit_before>package main

import (
	"encoding/json"
	"errors"
	"fmt"
	"io"
	"io/ioutil"
	"net/http"
	"strings"
)

type elasticEngine struct {
	client  *http.Client
	baseURL string
}

func NewElasticEngine(elasticURL string) Engine {
	transport := &http.Transport{MaxIdleConnsPerHost: 30}
	e := &elasticEngine{
		client:  &http.Client{Transport: transport},
		baseURL: elasticURL,
	}
	for strings.HasSuffix(e.baseURL, "/") {
		e.baseURL = e.baseURL[0 : len(e.baseURL)-1]
	}

	return e
}

func (ee *elasticEngine) Drop(collection string) {
	req, err := http.NewRequest("DELETE", fmt.Sprintf("%s/store/%s/", ee.baseURL, collection), nil)
	if err != nil {
		panic(err)
	}
	resp, err := ee.client.Do(req)
	if err != nil {
		panic(err)
	}
	if resp.StatusCode != 200 && resp.StatusCode != 404 {
		panic("drop fail")
	}
}

func (ee *elasticEngine) Write(collection, id string, cont Document) error {
	if id == "" {
		return errors.New("missing id")
	}

	r, w := io.Pipe()
	go func() {
		je := json.NewEncoder(w)
		err := je.Encode(cont)
		if err != nil {
			panic(err)
		}
		w.Close()
	}()

	req, err := http.NewRequest("PUT", fmt.Sprintf("%s/store/%s/%s", ee.baseURL, collection, id), r)
	if err != nil {
		panic(err)
	}
	resp, err := ee.client.Do(req)
	if err != nil {
		panic(err)
	}
	io.Copy(ioutil.Discard, resp.Body)
	resp.Body.Close()

	return nil
}

func (ee *elasticEngine) Count(collection string) int {
	res, err := ee.client.Get(fmt.Sprintf("%s/store/%s/_count", ee.baseURL, collection))
	if err != nil {
		panic(err)
	}
	defer res.Body.Close()
	if res.StatusCode == 200 {
		dec := json.NewDecoder(res.Body)
		var result esCountResult
		err := dec.Decode(&result)
		if err != nil {
			panic(err)
		}
		return result.Count
	}
	panic(res.StatusCode)
}

type esCountResult struct {
	Count int `json:"count"`
}

func (ee *elasticEngine) Load(collection, id string) (bool, Document, error) {
	res, err := ee.client.Get(fmt.Sprintf("%s/store/%s/%s", ee.baseURL, collection, id))
	if err != nil {
		panic(err)
	}
	defer res.Body.Close()
	switch {
	case res.StatusCode == 200:
		dec := json.NewDecoder(res.Body)
		var result esGetResult
		err := dec.Decode(&result)
		if err != nil {
			panic(err)
		}
		return true, result.Source, nil
	case res.StatusCode == 404:
		return false, Document{}, nil
	default:
		panic(res.StatusCode)
	}
}

type esGetResult struct {
	Source Document `json:"_source"`
}

func (ee elasticEngine) All(collection string, closechan chan struct{}) (chan Document, error) {
	q := fmt.Sprintf("{\"query\":{\"match_all\": {}}, \"fields\":[], \"size\": %d,  \"from\": 0}", ee.Count(collection)+1000)
	return ee.query(collection, q, closechan)
}

func (ee elasticEngine) query(collection, q string, closechan chan struct{}) (chan Document, error) {
	cont := make(chan Document)

	res, err := ee.client.Post(fmt.Sprintf("%s/store/%s/_search", ee.baseURL, collection), "application/json", strings.NewReader(q))
	if err != nil {
		panic(err)
	}

	switch {
	case res.StatusCode == 200:
	case res.StatusCode == 400:
		res.Body.Close()
		return nil, ErrInvalidQuery
	default:
		panic(res.StatusCode)
	}

	go func() {
		defer close(cont)

		defer res.Body.Close()
		dec := json.NewDecoder(res.Body)
		var result esRecentResult
		err := dec.Decode(&result)
		if err != nil {
			panic(err)
		}
		for _, h := range result.Hits.Hits {
			id := h.Id
			found, c, err := ee.Load(collection, id)
			if err != nil {
				panic(err)
			}
			if !found {
				panic("fail")
			}
			select {
			case cont <- c:
			case <-closechan:
				break
			}
		}
	}()

	return cont, nil
}

func (ee elasticEngine) Close() {
}

type esRecentResult struct {
	Hits esRecentHit `json:"hits"`
}
type esRecentHit struct {
	Hits []esRecentId `json:"hits"`
}
type esRecentId struct {
	Id string `json:"_id"`
}

type esSearchResult struct {
	Hits hitResult `json:"hits"`
}
type hitResult struct {
	Hits []esSearchId `json:"hits"`
}
type esSearchId struct {
	Id string `json:"_id"`
}
<commit_msg>cleanup<commit_after>package main

import (
	"encoding/json"
	"errors"
	"fmt"
	"io"
	"io/ioutil"
	"net/http"
	"strings"
)

type elasticEngine struct {
	client  *http.Client
	baseURL string
}

func NewElasticEngine(elasticURL string) Engine {
	transport := &http.Transport{MaxIdleConnsPerHost: 30}
	e := &elasticEngine{
		client:  &http.Client{Transport: transport},
		baseURL: elasticURL,
	}
	for strings.HasSuffix(e.baseURL, "/") {
		e.baseURL = e.baseURL[0 : len(e.baseURL)-1]
	}

	return e
}

func (ee *elasticEngine) Drop(collection string) {
	req, err := http.NewRequest("DELETE", fmt.Sprintf("%s/store/%s/", ee.baseURL, collection), nil)
	if err != nil {
		panic(err)
	}
	resp, err := ee.client.Do(req)
	if err != nil {
		panic(err)
	}
	if resp.StatusCode != 200 && resp.StatusCode != 404 {
		panic("drop fail")
	}
}

func (ee *elasticEngine) Write(collection, id string, cont Document) error {
	if id == "" {
		return errors.New("missing id")
	}

	r, w := io.Pipe()
	go func() {
		je := json.NewEncoder(w)
		err := je.Encode(cont)
		if err != nil {
			panic(err)
		}
		w.Close()
	}()

	req, err := http.NewRequest("PUT", fmt.Sprintf("%s/store/%s/%s", ee.baseURL, collection, id), r)
	if err != nil {
		panic(err)
	}
	resp, err := ee.client.Do(req)
	if err != nil {
		panic(err)
	}
	io.Copy(ioutil.Discard, resp.Body)
	resp.Body.Close()

	return nil
}

func (ee *elasticEngine) Count(collection string) int {
	res, err := ee.client.Get(fmt.Sprintf("%s/store/%s/_count", ee.baseURL, collection))
	if err != nil {
		panic(err)
	}
	defer res.Body.Close()
	if res.StatusCode == 200 {
		dec := json.NewDecoder(res.Body)
		var result esCountResult
		err := dec.Decode(&result)
		if err != nil {
			panic(err)
		}
		return result.Count
	}
	panic(res.StatusCode)
}

type esCountResult struct {
	Count int `json:"count"`
}

func (ee *elasticEngine) Load(collection, id string) (bool, Document, error) {
	res, err := ee.client.Get(fmt.Sprintf("%s/store/%s/%s", ee.baseURL, collection, id))
	if err != nil {
		panic(err)
	}
	defer res.Body.Close()
	switch {
	case res.StatusCode == 200:
		dec := json.NewDecoder(res.Body)
		var result esGetResult
		err := dec.Decode(&result)
		if err != nil {
			panic(err)
		}
		return true, result.Source, nil
	case res.StatusCode == 404:
		return false, Document{}, nil
	default:
		panic(res.StatusCode)
	}
}

type esGetResult struct {
	Source Document `json:"_source"`
}

func (ee elasticEngine) All(collection string, closechan chan struct{}) (chan Document, error) {
	q := fmt.Sprintf("{\"query\":{\"match_all\": {}}, \"fields\":[], \"size\": %d,  \"from\": 0}", ee.Count(collection)+1000)
	return ee.query(collection, q, closechan)
}

func (ee elasticEngine) query(collection, q string, closechan chan struct{}) (chan Document, error) {
	cont := make(chan Document)

	res, err := ee.client.Post(fmt.Sprintf("%s/store/%s/_search", ee.baseURL, collection), "application/json", strings.NewReader(q))
	if err != nil {
		panic(err)
	}

	switch {
	case res.StatusCode == 200:
	case res.StatusCode == 400:
		res.Body.Close()
		return nil, ErrInvalidQuery
	default:
		panic(res.StatusCode)
	}

	go func() {
		defer close(cont)

		defer res.Body.Close()
		dec := json.NewDecoder(res.Body)
		var result esSearchResult
		err := dec.Decode(&result)
		if err != nil {
			panic(err)
		}
		for _, h := range result.Hits.Hits {
			id := h.Id
			found, c, err := ee.Load(collection, id)
			if err != nil {
				panic(err)
			}
			if !found {
				panic("fail")
			}
			select {
			case cont <- c:
			case <-closechan:
				break
			}
		}
	}()

	return cont, nil
}

func (ee elasticEngine) Close() {
}

type esSearchResult struct {
	Hits hitResult `json:"hits"`
}
type hitResult struct {
	Hits []esSearchId `json:"hits"`
}
type esSearchId struct {
	Id string `json:"_id"`
}
<|endoftext|>


<commit_before>package pstoremem

import (
	"context"
	"sort"
	"sync"
	"time"

	logging "github.com/ipfs/go-log"
	peer "github.com/libp2p/go-libp2p-peer"
	ma "github.com/multiformats/go-multiaddr"

	pstore "github.com/libp2p/go-libp2p-peerstore"
	addr "github.com/libp2p/go-libp2p-peerstore/addr"
)

var log = logging.Logger("peerstore")

type expiringAddr struct {
	Addr    ma.Multiaddr
	TTL     time.Duration
	Expires time.Time
}

func (e *expiringAddr) ExpiredBy(t time.Time) bool {
	return t.After(e.Expires)
}

var _ pstore.AddrBook = (*memoryAddrBook)(nil)

// memoryAddrBook manages addresses.
type memoryAddrBook struct {
	addrmu sync.Mutex
	addrs  map[peer.ID]map[string]expiringAddr

	subManager *AddrSubManager
}

func NewAddrBook() pstore.AddrBook {
	return &memoryAddrBook{
		addrs:      make(map[peer.ID]map[string]expiringAddr),
		subManager: NewAddrSubManager(),
	}
}

func (mab *memoryAddrBook) PeersWithAddrs() peer.IDSlice {
	mab.addrmu.Lock()
	defer mab.addrmu.Unlock()
	if mab.addrs == nil {
		return nil
	}

	pids := make(peer.IDSlice, 0, len(mab.addrs))
	for pid := range mab.addrs {
		pids = append(pids, pid)
	}
	return pids
}

// AddAddr calls AddAddrs(p, []ma.Multiaddr{addr}, ttl)
func (mab *memoryAddrBook) AddAddr(p peer.ID, addr ma.Multiaddr, ttl time.Duration) {
	mab.AddAddrs(p, []ma.Multiaddr{addr}, ttl)
}

// AddAddrs gives memoryAddrBook addresses to use, with a given ttl
// (time-to-live), after which the address is no longer valid.
// If the manager has a longer TTL, the operation is a no-op for that address
func (mab *memoryAddrBook) AddAddrs(p peer.ID, addrs []ma.Multiaddr, ttl time.Duration) {
	mab.addrmu.Lock()
	defer mab.addrmu.Unlock()

	// if ttl is zero, exit. nothing to do.
	if ttl <= 0 {
		return
	}

	amap := mab.addrs[p]
	if amap == nil {
		amap = make(map[string]expiringAddr, len(addrs))
		mab.addrs[p] = amap
	}
	exp := time.Now().Add(ttl)
	for _, addr := range addrs {
		if addr == nil {
			log.Warningf("was passed nil multiaddr for %s", p)
			continue
		}
		addrstr := string(addr.Bytes())
		a, found := amap[addrstr]
		if !found || exp.After(a.Expires) {
			amap[addrstr] = expiringAddr{Addr: addr, Expires: exp, TTL: ttl}

			mab.subManager.BroadcastAddr(p, addr)
		}
	}
}

// SetAddr calls mgr.SetAddrs(p, addr, ttl)
func (mab *memoryAddrBook) SetAddr(p peer.ID, addr ma.Multiaddr, ttl time.Duration) {
	mab.SetAddrs(p, []ma.Multiaddr{addr}, ttl)
}

// SetAddrs sets the ttl on addresses. This clears any TTL there previously.
// This is used when we receive the best estimate of the validity of an address.
func (mab *memoryAddrBook) SetAddrs(p peer.ID, addrs []ma.Multiaddr, ttl time.Duration) {
	mab.addrmu.Lock()
	defer mab.addrmu.Unlock()

	amap := mab.addrs[p]
	if amap == nil {
		amap = make(map[string]expiringAddr, len(addrs))
		mab.addrs[p] = amap
	}

	exp := time.Now().Add(ttl)
	for _, addr := range addrs {
		if addr == nil {
			log.Warningf("was passed nil multiaddr for %s", p)
			continue
		}
		// re-set all of them for new ttl.
		addrstr := string(addr.Bytes())

		if ttl > 0 {
			amap[addrstr] = expiringAddr{Addr: addr, Expires: exp, TTL: ttl}

			mab.subManager.BroadcastAddr(p, addr)
		} else {
			delete(amap, addrstr)
		}
	}
}

// UpdateAddrs updates the addresses associated with the given peer that have
// the given oldTTL to have the given newTTL.
func (mab *memoryAddrBook) UpdateAddrs(p peer.ID, oldTTL time.Duration, newTTL time.Duration) {
	mab.addrmu.Lock()
	defer mab.addrmu.Unlock()

	if mab.addrs == nil {
		return
	}

	amap, found := mab.addrs[p]
	if !found {
		return
	}

	exp := time.Now().Add(newTTL)
	for k, addr := range amap {
		if oldTTL == addr.TTL {
			addr.TTL = newTTL
			addr.Expires = exp
			amap[k] = addr
		}
	}
}

// Addresses returns all known (and valid) addresses for a given
func (mab *memoryAddrBook) Addrs(p peer.ID) []ma.Multiaddr {
	mab.addrmu.Lock()
	defer mab.addrmu.Unlock()

	// not initialized? nothing to give.
	if mab.addrs == nil {
		return nil
	}

	amap, found := mab.addrs[p]
	if !found {
		return nil
	}

	now := time.Now()
	good := make([]ma.Multiaddr, 0, len(amap))
	for k, m := range amap {
		if !m.ExpiredBy(now) {
			good = append(good, m.Addr)
		} else {
			delete(amap, k)
		}
	}

	return good
}

// ClearAddrs removes all previously stored addresses
func (mab *memoryAddrBook) ClearAddrs(p peer.ID) {
	mab.addrmu.Lock()
	defer mab.addrmu.Unlock()

	delete(mab.addrs, p)
}

// AddrStream returns a channel on which all new addresses discovered for a
// given peer ID will be published.
func (mab *memoryAddrBook) AddrStream(ctx context.Context, p peer.ID) <-chan ma.Multiaddr {
	mab.addrmu.Lock()
	defer mab.addrmu.Unlock()

	baseaddrslice := mab.addrs[p]
	initial := make([]ma.Multiaddr, 0, len(baseaddrslice))
	for _, a := range baseaddrslice {
		initial = append(initial, a.Addr)
	}

	return mab.subManager.AddrStream(ctx, p, initial)
}

type addrSub struct {
	pubch  chan ma.Multiaddr
	lk     sync.Mutex
	buffer []ma.Multiaddr
	ctx    context.Context
}

func (s *addrSub) pubAddr(a ma.Multiaddr) {
	select {
	case s.pubch <- a:
	case <-s.ctx.Done():
	}
}

// An abstracted, pub-sub manager for address streams. Extracted from
// memoryAddrBook in order to support additional implementations.
type AddrSubManager struct {
	mu   sync.RWMutex
	subs map[peer.ID][]*addrSub
}

// NewAddrSubManager initializes an AddrSubManager.
func NewAddrSubManager() *AddrSubManager {
	return &AddrSubManager{
		subs: make(map[peer.ID][]*addrSub),
	}
}

// Used internally by the address stream coroutine to remove a subscription
// from the manager.
func (mgr *AddrSubManager) removeSub(p peer.ID, s *addrSub) {
	mgr.mu.Lock()
	defer mgr.mu.Unlock()

	subs := mgr.subs[p]
	if len(subs) == 1 {
		if subs[0] != s {
			return
		}
		delete(mgr.subs, p)
		return
	}

	for i, v := range subs {
		if v == s {
			subs[i] = subs[len(subs)-1]
			subs[len(subs)-1] = nil
			mgr.subs[p] = subs[:len(subs)-1]
			return
		}
	}
}

// BroadcastAddr broadcasts a new address to all subscribed streams.
func (mgr *AddrSubManager) BroadcastAddr(p peer.ID, addr ma.Multiaddr) {
	mgr.mu.RLock()
	defer mgr.mu.RUnlock()

	if subs, ok := mgr.subs[p]; ok {
		for _, sub := range subs {
			sub.pubAddr(addr)
		}
	}
}

// AddrStream creates a new subscription for a given peer ID, pre-populating the
// channel with any addresses we might already have on file.
func (mgr *AddrSubManager) AddrStream(ctx context.Context, p peer.ID, initial []ma.Multiaddr) <-chan ma.Multiaddr {
	sub := &addrSub{pubch: make(chan ma.Multiaddr), ctx: ctx}
	out := make(chan ma.Multiaddr)

	mgr.mu.Lock()
	if _, ok := mgr.subs[p]; ok {
		mgr.subs[p] = append(mgr.subs[p], sub)
	} else {
		mgr.subs[p] = []*addrSub{sub}
	}
	mgr.mu.Unlock()

	sort.Sort(addr.AddrList(initial))

	go func(buffer []ma.Multiaddr) {
		defer close(out)

		sent := make(map[string]bool, len(buffer))
		var outch chan ma.Multiaddr

		for _, a := range buffer {
			sent[string(a.Bytes())] = true
		}

		var next ma.Multiaddr
		if len(buffer) > 0 {
			next = buffer[0]
			buffer = buffer[1:]
			outch = out
		}

		for {
			select {
			case outch <- next:
				if len(buffer) > 0 {
					next = buffer[0]
					buffer = buffer[1:]
				} else {
					outch = nil
					next = nil
				}
			case naddr := <-sub.pubch:
				if sent[string(naddr.Bytes())] {
					continue
				}

				sent[string(naddr.Bytes())] = true
				if next == nil {
					next = naddr
					outch = out
				} else {
					buffer = append(buffer, naddr)
				}
			case <-ctx.Done():
				mgr.removeSub(p, sub)
				return
			}
		}

	}(initial)

	return out
}
<commit_msg>add garbage collection for in-memory peerstore<commit_after>package pstoremem

import (
	"context"
	"sort"
	"sync"
	"time"

	logging "github.com/ipfs/go-log"
	peer "github.com/libp2p/go-libp2p-peer"
	ma "github.com/multiformats/go-multiaddr"

	pstore "github.com/libp2p/go-libp2p-peerstore"
	addr "github.com/libp2p/go-libp2p-peerstore/addr"
)

var log = logging.Logger("peerstore")

type expiringAddr struct {
	Addr    ma.Multiaddr
	TTL     time.Duration
	Expires time.Time
}

func (e *expiringAddr) ExpiredBy(t time.Time) bool {
	return t.After(e.Expires)
}

var _ pstore.AddrBook = (*memoryAddrBook)(nil)

// memoryAddrBook manages addresses.
type memoryAddrBook struct {
	addrmu sync.Mutex
	addrs  map[peer.ID]map[string]expiringAddr

	nextGC time.Time

	subManager *AddrSubManager
}

func NewAddrBook() pstore.AddrBook {
	return &memoryAddrBook{
		addrs:      make(map[peer.ID]map[string]expiringAddr),
		subManager: NewAddrSubManager(),
	}
}

func (mab *memoryAddrBook) gc() {
	now := time.Now()
	if !now.After(mab.nextGC) {
		return
	}
	for p, amap := range mab.addrs {
		for k, addr := range amap {
			if addr.ExpiredBy(now) {
				delete(amap, k)
			}
		}
		if len(amap) == 0 {
			delete(mab.addrs, p)
		}
	}
	mab.nextGC = time.Now().Add(pstore.AddressTTL)
}

func (mab *memoryAddrBook) PeersWithAddrs() peer.IDSlice {
	mab.addrmu.Lock()
	defer mab.addrmu.Unlock()

	pids := make(peer.IDSlice, 0, len(mab.addrs))
	for pid := range mab.addrs {
		pids = append(pids, pid)
	}
	return pids
}

// AddAddr calls AddAddrs(p, []ma.Multiaddr{addr}, ttl)
func (mab *memoryAddrBook) AddAddr(p peer.ID, addr ma.Multiaddr, ttl time.Duration) {
	mab.AddAddrs(p, []ma.Multiaddr{addr}, ttl)
}

// AddAddrs gives memoryAddrBook addresses to use, with a given ttl
// (time-to-live), after which the address is no longer valid.
// If the manager has a longer TTL, the operation is a no-op for that address
func (mab *memoryAddrBook) AddAddrs(p peer.ID, addrs []ma.Multiaddr, ttl time.Duration) {
	mab.addrmu.Lock()
	defer mab.addrmu.Unlock()

	// if ttl is zero, exit. nothing to do.
	if ttl <= 0 {
		return
	}

	amap := mab.addrs[p]
	if amap == nil {
		amap = make(map[string]expiringAddr, len(addrs))
		mab.addrs[p] = amap
	}
	exp := time.Now().Add(ttl)
	for _, addr := range addrs {
		if addr == nil {
			log.Warningf("was passed nil multiaddr for %s", p)
			continue
		}
		addrstr := string(addr.Bytes())
		a, found := amap[addrstr]
		if !found || exp.After(a.Expires) {
			amap[addrstr] = expiringAddr{Addr: addr, Expires: exp, TTL: ttl}

			mab.subManager.BroadcastAddr(p, addr)
		}
	}
	mab.gc()
}

// SetAddr calls mgr.SetAddrs(p, addr, ttl)
func (mab *memoryAddrBook) SetAddr(p peer.ID, addr ma.Multiaddr, ttl time.Duration) {
	mab.SetAddrs(p, []ma.Multiaddr{addr}, ttl)
}

// SetAddrs sets the ttl on addresses. This clears any TTL there previously.
// This is used when we receive the best estimate of the validity of an address.
func (mab *memoryAddrBook) SetAddrs(p peer.ID, addrs []ma.Multiaddr, ttl time.Duration) {
	mab.addrmu.Lock()
	defer mab.addrmu.Unlock()

	amap := mab.addrs[p]
	if amap == nil {
		amap = make(map[string]expiringAddr, len(addrs))
		mab.addrs[p] = amap
	}

	exp := time.Now().Add(ttl)
	for _, addr := range addrs {
		if addr == nil {
			log.Warningf("was passed nil multiaddr for %s", p)
			continue
		}
		// re-set all of them for new ttl.
		addrstr := string(addr.Bytes())

		if ttl > 0 {
			amap[addrstr] = expiringAddr{Addr: addr, Expires: exp, TTL: ttl}

			mab.subManager.BroadcastAddr(p, addr)
		} else {
			delete(amap, addrstr)
		}
	}
	mab.gc()
}

// UpdateAddrs updates the addresses associated with the given peer that have
// the given oldTTL to have the given newTTL.
func (mab *memoryAddrBook) UpdateAddrs(p peer.ID, oldTTL time.Duration, newTTL time.Duration) {
	mab.addrmu.Lock()
	defer mab.addrmu.Unlock()

	amap, found := mab.addrs[p]
	if !found {
		return
	}

	exp := time.Now().Add(newTTL)
	for k, addr := range amap {
		if oldTTL == addr.TTL {
			addr.TTL = newTTL
			addr.Expires = exp
			amap[k] = addr
		}
	}
	mab.gc()
}

// Addresses returns all known (and valid) addresses for a given
func (mab *memoryAddrBook) Addrs(p peer.ID) []ma.Multiaddr {
	mab.addrmu.Lock()
	defer mab.addrmu.Unlock()

	amap, found := mab.addrs[p]
	if !found {
		return nil
	}

	now := time.Now()
	good := make([]ma.Multiaddr, 0, len(amap))
	for k, m := range amap {
		if !m.ExpiredBy(now) {
			good = append(good, m.Addr)
		} else {
			delete(amap, k)
		}
	}

	return good
}

// ClearAddrs removes all previously stored addresses
func (mab *memoryAddrBook) ClearAddrs(p peer.ID) {
	mab.addrmu.Lock()
	defer mab.addrmu.Unlock()

	delete(mab.addrs, p)
}

// AddrStream returns a channel on which all new addresses discovered for a
// given peer ID will be published.
func (mab *memoryAddrBook) AddrStream(ctx context.Context, p peer.ID) <-chan ma.Multiaddr {
	mab.addrmu.Lock()
	defer mab.addrmu.Unlock()

	baseaddrslice := mab.addrs[p]
	initial := make([]ma.Multiaddr, 0, len(baseaddrslice))
	for _, a := range baseaddrslice {
		initial = append(initial, a.Addr)
	}

	return mab.subManager.AddrStream(ctx, p, initial)
}

type addrSub struct {
	pubch  chan ma.Multiaddr
	lk     sync.Mutex
	buffer []ma.Multiaddr
	ctx    context.Context
}

func (s *addrSub) pubAddr(a ma.Multiaddr) {
	select {
	case s.pubch <- a:
	case <-s.ctx.Done():
	}
}

// An abstracted, pub-sub manager for address streams. Extracted from
// memoryAddrBook in order to support additional implementations.
type AddrSubManager struct {
	mu   sync.RWMutex
	subs map[peer.ID][]*addrSub
}

// NewAddrSubManager initializes an AddrSubManager.
func NewAddrSubManager() *AddrSubManager {
	return &AddrSubManager{
		subs: make(map[peer.ID][]*addrSub),
	}
}

// Used internally by the address stream coroutine to remove a subscription
// from the manager.
func (mgr *AddrSubManager) removeSub(p peer.ID, s *addrSub) {
	mgr.mu.Lock()
	defer mgr.mu.Unlock()

	subs := mgr.subs[p]
	if len(subs) == 1 {
		if subs[0] != s {
			return
		}
		delete(mgr.subs, p)
		return
	}

	for i, v := range subs {
		if v == s {
			subs[i] = subs[len(subs)-1]
			subs[len(subs)-1] = nil
			mgr.subs[p] = subs[:len(subs)-1]
			return
		}
	}
}

// BroadcastAddr broadcasts a new address to all subscribed streams.
func (mgr *AddrSubManager) BroadcastAddr(p peer.ID, addr ma.Multiaddr) {
	mgr.mu.RLock()
	defer mgr.mu.RUnlock()

	if subs, ok := mgr.subs[p]; ok {
		for _, sub := range subs {
			sub.pubAddr(addr)
		}
	}
}

// AddrStream creates a new subscription for a given peer ID, pre-populating the
// channel with any addresses we might already have on file.
func (mgr *AddrSubManager) AddrStream(ctx context.Context, p peer.ID, initial []ma.Multiaddr) <-chan ma.Multiaddr {
	sub := &addrSub{pubch: make(chan ma.Multiaddr), ctx: ctx}
	out := make(chan ma.Multiaddr)

	mgr.mu.Lock()
	if _, ok := mgr.subs[p]; ok {
		mgr.subs[p] = append(mgr.subs[p], sub)
	} else {
		mgr.subs[p] = []*addrSub{sub}
	}
	mgr.mu.Unlock()

	sort.Sort(addr.AddrList(initial))

	go func(buffer []ma.Multiaddr) {
		defer close(out)

		sent := make(map[string]bool, len(buffer))
		var outch chan ma.Multiaddr

		for _, a := range buffer {
			sent[string(a.Bytes())] = true
		}

		var next ma.Multiaddr
		if len(buffer) > 0 {
			next = buffer[0]
			buffer = buffer[1:]
			outch = out
		}

		for {
			select {
			case outch <- next:
				if len(buffer) > 0 {
					next = buffer[0]
					buffer = buffer[1:]
				} else {
					outch = nil
					next = nil
				}
			case naddr := <-sub.pubch:
				if sent[string(naddr.Bytes())] {
					continue
				}

				sent[string(naddr.Bytes())] = true
				if next == nil {
					next = naddr
					outch = out
				} else {
					buffer = append(buffer, naddr)
				}
			case <-ctx.Done():
				mgr.removeSub(p, sub)
				return
			}
		}

	}(initial)

	return out
}
<|endoftext|>


<commit_before>package ssms

import (
	"context"
	"fmt"
	"net"

	peer "github.com/libp2p/go-libp2p-peer"
	ss "github.com/libp2p/go-stream-security"
	mss "github.com/multiformats/go-multistream"
)

// SSMuxer is a multistream stream security transport multiplexer.
//
// SSMuxer is safe to use without initialization. However, it's not safe to move
// after use.
type SSMuxer struct {
	mux             mss.MultistreamMuxer
	tpts            map[string]ss.Transport
	OrderPreference []string
}

// AddTransport adds a stream security transport to this multistream muxer.
//
// This method is *not* thread-safe. It should be called only when initializing
// the SSMuxer.
func (sm *SSMuxer) AddTransport(path string, transport ss.Transport) {
	if sm.tpts == nil {
		sm.tpts = make(map[string]ss.Transport, 1)
	}

	sm.mux.AddHandler(path, nil)
	sm.tpts[path] = transport
	sm.OrderPreference = append(sm.OrderPreference, path)
}

// SecureInbound secures an inbound connection using this multistream
// multiplexed stream security transport.
func (sm *SSMuxer) SecureInbound(ctx context.Context, insecure net.Conn) (ss.Conn, error) {
	tpt, err := sm.selectProto(ctx, insecure, true)
	if err != nil {
		return nil, err
	}
	return tpt.SecureInbound(ctx, insecure)
}

// SecureOutbound secures an outbound connection using this multistream
// multiplexed stream security transport.
func (sm *SSMuxer) SecureOutbound(ctx context.Context, insecure net.Conn, p peer.ID) (ss.Conn, error) {
	tpt, err := sm.selectProto(ctx, insecure, false)
	if err != nil {
		return nil, err
	}
	return tpt.SecureOutbound(ctx, insecure, p)
}

func (sm *SSMuxer) selectProto(ctx context.Context, insecure net.Conn, server bool) (ss.Transport, error) {
	var proto string
	var err error
	done := make(chan struct{})
	go func() {
		defer close(done)
		if server {
			proto, _, err = sm.mux.Negotiate(insecure)
		} else {
			proto, err = mss.SelectOneOf(sm.OrderPreference, insecure)
		}
	}()

	select {
	case <-done:
		if err != nil {
			return nil, err
		}
		if tpt, ok := sm.tpts[proto]; ok {
			return tpt, nil
		}
		return nil, fmt.Errorf("selected unknown security transport")
	case <-ctx.Done():
		// We *must* do this. We have outstanding work on the connection
		// and it's no longer safe to use.
		insecure.Close()
		return nil, ctx.Err()
	}
}
<commit_msg>type assert transport<commit_after>package ssms

import (
	"context"
	"fmt"
	"net"

	peer "github.com/libp2p/go-libp2p-peer"
	ss "github.com/libp2p/go-stream-security"
	mss "github.com/multiformats/go-multistream"
)

// SSMuxer is a multistream stream security transport multiplexer.
//
// SSMuxer is safe to use without initialization. However, it's not safe to move
// after use.
type SSMuxer struct {
	mux             mss.MultistreamMuxer
	tpts            map[string]ss.Transport
	OrderPreference []string
}

var _ ss.Transport = (*SSMuxer)(nil)

// AddTransport adds a stream security transport to this multistream muxer.
//
// This method is *not* thread-safe. It should be called only when initializing
// the SSMuxer.
func (sm *SSMuxer) AddTransport(path string, transport ss.Transport) {
	if sm.tpts == nil {
		sm.tpts = make(map[string]ss.Transport, 1)
	}

	sm.mux.AddHandler(path, nil)
	sm.tpts[path] = transport
	sm.OrderPreference = append(sm.OrderPreference, path)
}

// SecureInbound secures an inbound connection using this multistream
// multiplexed stream security transport.
func (sm *SSMuxer) SecureInbound(ctx context.Context, insecure net.Conn) (ss.Conn, error) {
	tpt, err := sm.selectProto(ctx, insecure, true)
	if err != nil {
		return nil, err
	}
	return tpt.SecureInbound(ctx, insecure)
}

// SecureOutbound secures an outbound connection using this multistream
// multiplexed stream security transport.
func (sm *SSMuxer) SecureOutbound(ctx context.Context, insecure net.Conn, p peer.ID) (ss.Conn, error) {
	tpt, err := sm.selectProto(ctx, insecure, false)
	if err != nil {
		return nil, err
	}
	return tpt.SecureOutbound(ctx, insecure, p)
}

func (sm *SSMuxer) selectProto(ctx context.Context, insecure net.Conn, server bool) (ss.Transport, error) {
	var proto string
	var err error
	done := make(chan struct{})
	go func() {
		defer close(done)
		if server {
			proto, _, err = sm.mux.Negotiate(insecure)
		} else {
			proto, err = mss.SelectOneOf(sm.OrderPreference, insecure)
		}
	}()

	select {
	case <-done:
		if err != nil {
			return nil, err
		}
		if tpt, ok := sm.tpts[proto]; ok {
			return tpt, nil
		}
		return nil, fmt.Errorf("selected unknown security transport")
	case <-ctx.Done():
		// We *must* do this. We have outstanding work on the connection
		// and it's no longer safe to use.
		insecure.Close()
		return nil, ctx.Err()
	}
}
<|endoftext|>


<commit_before>package libp2pquic

import (
	"net"

	ma "github.com/multiformats/go-multiaddr"

	. "github.com/onsi/ginkgo"
	. "github.com/onsi/gomega"
)

var _ = Describe("QUIC Multiaddr", func() {
	It("converts a net.Addr to a QUIC Multiaddr", func() {
		addr := &net.UDPAddr{IP: net.IPv4(192, 168, 0, 42), Port: 1337}
		maddr, err := toQuicMultiaddr(addr)
		Expect(err).ToNot(HaveOccurred())
		Expect(maddr.String()).To(Equal("/ip4/192.168.0.42/udp/1337/quic"))
	})

	It("converts a QUIC Multiaddr to a net.Addr", func() {
		maddr, err := ma.NewMultiaddr("/ip4/192.168.0.42/udp/1337/quic")
		Expect(err).ToNot(HaveOccurred())
		addr, err := fromQuicMultiaddr(maddr)
		Expect(err).ToNot(HaveOccurred())
		Expect(addr).To(BeAssignableToTypeOf(&net.UDPAddr{}))
		udpAddr := addr.(*net.UDPAddr)
		Expect(udpAddr.IP).To(Equal(net.IPv4(192, 168, 0, 42)))
		Expect(udpAddr.Port).To(Equal(1337))
	})
})
<commit_msg>migrate the multiaddr tests away from Ginkgo<commit_after>package libp2pquic

import (
	"net"
	"testing"

	ma "github.com/multiformats/go-multiaddr"
	"github.com/stretchr/testify/require"
)

func TestConvertToQuicMultiaddr(t *testing.T) {
	addr := &net.UDPAddr{IP: net.IPv4(192, 168, 0, 42), Port: 1337}
	maddr, err := toQuicMultiaddr(addr)
	require.NoError(t, err)
	require.Equal(t, maddr.String(), "/ip4/192.168.0.42/udp/1337/quic")
}

func TestConvertFromQuicMultiaddr(t *testing.T) {
	maddr, err := ma.NewMultiaddr("/ip4/192.168.0.42/udp/1337/quic")
	require.NoError(t, err)
	addr, err := fromQuicMultiaddr(maddr)
	require.NoError(t, err)
	udpAddr, ok := addr.(*net.UDPAddr)
	require.True(t, ok)
	require.Equal(t, udpAddr.IP, net.IPv4(192, 168, 0, 42))
	require.Equal(t, udpAddr.Port, 1337)
}
<|endoftext|>


<commit_before>package glubcms

import (
	"encoding/json"
	"log"
	"net/url"
	"os"
	"path/filepath"
	"strings"
	"time"
)

type Entry struct {
	active bool
	author string
	date   time.Time
	html   []byte
	link   url.URL
	title  string
}

func (e Entry) Title() string {
	return e.title
}
func (e Entry) Link() string {
	return e.link.String()
}
func (e Entry) IsActive() bool {
	return e.active
}
func (a Entry) Author() string {
	return a.author
}
func (e Entry) Date() time.Time {
	return e.date
}
func (e Entry) HTML() []byte {
	return e.html
}
func (e Entry) IsArticle() bool {
	return e.HTML() != nil
}

type PTreeList []*PTree
type PTree struct {
	Entry
	Children PTreeList
}

func (ptl PTreeList) Filter(filterfunc func(*PTree) bool) PTreeList {
	var ret PTreeList
	for _, e := range ptl {
		if filterfunc(e) {
			ret = append(ret, e)
		}
	}
	return ret
}

func (pt *PTree) Articles() PTreeList {
	return pt.Children.Filter((*PTree).IsArticle)
}

//func (pt *PTree) MenuEntries() []*PTree {
//	return pt.Children.Filter((*PTree).IsMenu)
//}
func (pt *PTree) Active() *PTree {
	ret := pt.Children.Filter((*PTree).IsActive)
	switch len(ret) {
	case 1:
		return ret[0]
	case 0:
		return nil
	default:
		log.Fatalf("Multiple active children. %+v", ret)
		return nil
	}
}
func (pt *PTree) IsLeaf() bool {
	return pt.Children == nil
}

func splitfirstdir(s string) (string, string) {
	if len(s) == 0 {
		return "", ""
	}
	if s[0] == os.PathSeparator {
		return splitfirstdir(s[1:])
	}
	split := strings.SplitN(s, string(os.PathSeparator), 2)
	switch len(split) {
	case 2:
		return split[0], split[1]
	case 1:
		return split[0], ""
	default:
		return "", ""
	}
}

func PTreeFromDir(prefix string, path string) *PTree {
	var ret *PTree
	var dirname string
	path = filepath.Clean(path)
	for {
		dirname, path = splitfirstdir(path)
		if dirname == "" {
			break
		}

	}
	return ret
}

func pTreeFromDir(prefix string, path string) *PTree {
	var ret PTree

	metafile, err := os.Open(filepath.Join(path, "meta.json"))
	if err != nil {
		log.Fatal(err)
	}
	err = json.NewDecoder(metafile).Decode(&ret)
	metafile.Close()
	if err != nil {
		log.Fatal(err)
	}
	ret.link = url.URL{}

	paths, err := filepath.Glob(filepath.Join(path, "*", "meta.json"))
	if err != nil {
		log.Fatal(err)
	}
	for _, path := range paths {
		path = filepath.Dir(path)
		newChild := pTreeFromDir(prefix, path)
		if newChild != nil {
			ret.Children = append(ret.Children, newChild)
		}
	}

	if ret.Children != nil {
	}

	return &ret
}

//type Page struct {
//	Menu     [][]MenuEntry
//	Articles []Article
//}
//
//func (p *Page) RenderHTML(t *template.Template) []byte {
//	b := bytes.Buffer{}
//	fmt.Println(t.Execute(&b, p))
//	return b.Bytes()
//}
//
//func (p *Page) Path() []MenuEntry {
//	var ret []MenuEntry
//level:
//	for _, menuLevel := range p.Menu {
//		for _, m := range menuLevel {
//			if m.Active() {
//				ret = append(ret, m)
//				continue level
//			}
//		}
//	}
//
//	return ret
//}
//
//func (p *Page) Title(sep ...string) string {
//	for _, a := range p.Articles {
//		if a.Active() {
//			return a.Title()
//		}
//	}
//
//	var menutitles []string
//	for _, m := range p.Path() {
//		menutitles = append(menutitles, m.Title())
//	}
//
//	if len(sep) == 0 {
//		sep = " - "
//	}
//	return strings.Join(menutitles, sep)
//}
<commit_msg>POC implementation, filesystem backend<commit_after>package glubcms

import (
	"bytes"
	"encoding/json"
	"fmt"
	"html/template"
	"io/ioutil"
	"log"
	"net/url"
	"os"
	"path/filepath"
	"sort"
	"strings"
	"time"

	bf "github.com/russross/blackfriday"
)

type Entry interface {
	Active() bool
	Author() string
	Date() time.Time
	HTML() template.HTML
	IsArticle() bool
	Link() string
	Title() string
}

type Entries []Entry

func (e Entries) Len() int {
	return len(e)
}
func (e Entries) Less(i, j int) bool {
	return e[i].Date().After(e[j].Date())
}
func (e Entries) Swap(i, j int) {
	e[i], e[j] = e[j], e[i]
}
func (e Entries) Split() (Menu, Articles Entries) {
	for _, v := range e {
		if v.IsArticle() {
			Articles = append(Articles, v)
		} else {
			Menu = append(Menu, v)
		}
	}

	return
}

type mtime time.Time

func (t *mtime) UnmarshalJSON(b []byte) error {
	layout := "2006-01-02 15:04"
	tmp, err := time.Parse(layout, strings.Trim(string(b), "\""))
	*t = mtime(tmp)
	return err
}

type Meta struct {
	Author string
	Date   mtime
	Title  string
}

type entry struct {
	meta      Meta
	active    bool
	html      []byte
	isarticle bool
	link      url.URL
}

func (e entry) Active() bool {
	return e.active
}
func (a entry) Author() string {
	return a.meta.Author
}
func (e entry) Date() time.Time {
	return time.Time(e.meta.Date)
}
func (e entry) HTML() template.HTML {
	return template.HTML(e.html)
}
func (e entry) IsArticle() bool {
	return e.isarticle
}
func (e entry) Link() string {
	return e.link.String()
}
func (e entry) Title() string {
	return e.meta.Title
}

func entryFromMeta(path string) (*entry, error) {
	ret := entry{}
	f, err := os.Open(filepath.Join(path))
	if err != nil {
		return nil, err
	}
	defer f.Close()

	err = json.NewDecoder(f).Decode(&ret.meta)
	return &ret, err
}

func entryFromDir(prefix, path, activepath string) Entry {
	ret, err := entryFromMeta(filepath.Join(prefix, path, "meta.json"))
	if err != nil {
		log.Println(err)
		return nil
	}

	ret.link = url.URL{Path: path}
	if strings.HasPrefix(activepath, path) {
		ret.active = true
	}

	md, err := os.Open(filepath.Join(prefix, path, "article.md"))
	if err != nil {
		if !os.IsNotExist(err) {
			log.Println(err)
		}
		return ret
	}
	defer md.Close()

	// only attempt to convert the markdown if it's the active path
	if activepath == path {
		b, err := ioutil.ReadAll(md)
		if err != nil {
			log.Println(err)
			return ret
		}
		ret.html = bf.MarkdownBasic(b)
	}

	ret.isarticle = true

	return ret
}

func entriesFromDir(prefix, path, activepath string) Entries {
	var ret Entries

	dir, err := os.Open(filepath.Join(prefix, path))
	if err != nil {
		log.Println(err)
		return nil
	}

	dirlist, err := dir.Readdir(0)
	if err != nil {
		log.Println(err)
	}
	dir.Close()

	for _, fi := range dirlist {
		if !fi.IsDir() {
			continue
		}
		entry := entryFromDir(prefix, filepath.Join(path, fi.Name()), activepath)
		if entry != nil {
			ret = append(ret, entry)
		}
	}

	sort.Sort(ret)
	return ret
}

type Page struct {
	Menu     []Entries
	Articles Entries
	Content  Entry
}

func PageFromDir(prefix, path string) Page {
	var p Page
	path = filepath.Clean(path)
	activepath := path

	// look for an article in current path
	if c := entryFromDir(prefix, path, activepath); c.IsArticle() {
		p.Content = c
	}

	for {
		menu, articles := entriesFromDir(prefix, path, activepath).Split()
		if len(menu) > 0 {
			sort.Sort(menu)
			p.Menu = append(p.Menu, menu)
		}
		if p.Articles == nil && len(articles) > 0 {
			sort.Sort(articles)
			p.Articles = articles
			if p.Content == nil {
				// Parse again with activepath set, to get the markdown
				cpath := p.Articles[0].Link()
				p.Content = entryFromDir(prefix, cpath, cpath)
				p.Articles[0] = p.Content
			}
		}
		if path == "." || path == "/" {
			break
		}
		path = filepath.Dir(path)
	}

	// building from deepest path to /, need to reverse
	for l, r := 0, len(p.Menu)-1; l < r; l, r = l+1, r-1 {
		p.Menu[l], p.Menu[r] = p.Menu[r], p.Menu[l]
	}

	return p
}

// For debugging
func (p Page) Outline() string {
	buf := bytes.Buffer{}

	fmt.Fprintln(&buf, "Menu:")
	for level, ms := range p.Menu {
		for _, m := range ms {
			fmt.Fprintf(&buf, "%s%q", strings.Repeat("\t", level), m.Title())
			if m.Active() {
				fmt.Fprintf(&buf, " (active)")
			}
			fmt.Fprintln(&buf)
		}
	}

	fmt.Fprintln(&buf, "Articles:")
	for _, a := range p.Articles {
		fmt.Fprintf(&buf, "%q", a.Title())
		if a.Active() {
			fmt.Fprintf(&buf, " (active)")
		}
		fmt.Fprintln(&buf)
	}
	fmt.Fprintln(&buf, "Content:")
	if p.Content != nil {
		fmt.Fprint(&buf, p.Content.Title())
	}

	return buf.String()
}

func init() {
	log.SetFlags(log.Flags() | log.Lshortfile)
}
<|endoftext|>


<commit_before>package page

import (
	"os"
	"io"
	"time"
	"bytes"
	"text/template"
	"log"
)

type Page struct {
	pageName string
	templateName string

	template *template.Template
	Head        []string
	Scripts     []string
	Stylesheets []string
	Title       string
	Description string
	Keywords    []string
	Body        string
	Now         time.Time
}

func NewPage(templateName string) (p *Page) {
	p = new(Page)
	p.Now = time.Now()
	p.templateName = templateName
	return
}

func (p *Page) Execute(w io.Writer) {
	var err error
	p.template, err = template.ParseFiles("template/" + p.templateName)
	if err == nil {
		p.template.Execute(w, p)
	}else{
		log.Fatal(err)
	}
}

func (p *Page) FromFile(slug string) (*Page) {
	body, _ := template.ParseFiles("page/" + slug)
	var b bytes.Buffer
	body.Execute(&b, p)
	p.Body = b.String()
	return p
}

func Exists(slug string) bool {
	fh, err := os.Open("page/" + slug)
	defer fh.Close()
	if err == nil {
		return true
	}
	return false
}

func TemplateExists(slug string) bool {
	fh, err := os.Open("template/" + slug)
	defer fh.Close()
	if err == nil {
		return true
	}
	return false
}

<commit_msg>Refactored, go fmt<commit_after>package page

import (
	"bytes"
	"io"
	_ "log"
	"path/filepath"
	"text/template"
	"time"
)

type Page struct {
	body     *template.Template
	template *template.Template

	Head        []string
	Scripts     []string
	Stylesheets []string
	Title       string
	Description string
	Keywords    []string
	Body        string
	Now         time.Time
}

func NewPage() (p *Page) {
	p = new(Page)
	p.Now = time.Now()
	return
}

func (p *Page) LoadTemplate(templateFile string) (err error) {
	p.template, err = template.ParseFiles(filepath.Join("template", templateFile))
	return
}

func (p *Page) LoadBody(bodyFile string) (err error) {
	p.body, err = template.ParseFiles(filepath.Join("page", bodyFile))
	return
}

func (p *Page) Execute(w io.Writer) (err error) {

	if p.template != nil && p.body == nil {
		err = p.template.Execute(w, p)
		return
	}

	if p.template == nil && p.body != nil {
		err = p.body.Execute(w, p)
		return
	}

	if p.template != nil && p.body != nil {
		var b bytes.Buffer
		err = p.body.Execute(&b, p)
		if err != nil {
			return
		}
		p.Body = b.String()
		err = p.template.Execute(w, p)
		return
	}

	return
}

func (p *Page) SetTitle(title string) (s string) {
	p.Title = title
	return
}

func (p *Page) AddJS(path string) (s string) {
	p.Scripts = append(p.Scripts, path)
	return
}

func (p *Page) UseTemplate(path string) (s string) {
	p.LoadTemplate(path)
	return
}

/*

func Exists(slug string) bool {
	fh, err := os.Open("page/" + slug)
	defer fh.Close()
	if err == nil {
		return true
	}
	return false
}

func TemplateExists(slug string) bool {
	fh, err := os.Open("template/" + slug)
	defer fh.Close()
	if err == nil {
		return true
	}
	return false
}

*/
<|endoftext|>