<commit_before>package api

import (
	"encoding/base64"
	"encoding/json"
	"errors"
	"fmt"
	"net/http"
	"path"

	log "github.com/coreos/fleet/third_party/github.com/golang/glog"

	"github.com/coreos/fleet/job"
	"github.com/coreos/fleet/registry"
	"github.com/coreos/fleet/schema"
	"github.com/coreos/fleet/unit"
)

func wireUpUnitsResource(mux *http.ServeMux, prefix string, reg registry.Registry) {
	base := path.Join(prefix, "units")
	ur := unitsResource{reg, base}
	mux.Handle(base, &ur)
	mux.Handle(base+"/", &ur)
}

type unitsResource struct {
	reg      registry.Registry
	basePath string
}

func (ur *unitsResource) ServeHTTP(rw http.ResponseWriter, req *http.Request) {
	if isCollectionPath(ur.basePath, req.URL.Path) {
		switch req.Method {
		case "GET":
			ur.list(rw, req)
		default:
			sendError(rw, http.StatusMethodNotAllowed, errors.New("only GET supported against this resource"))
		}
	} else if item, ok := isItemPath(ur.basePath, req.URL.Path); ok {
		switch req.Method {
		case "GET":
			ur.get(rw, req, item)
		case "DELETE":
			ur.destroy(rw, req, item)
		case "PUT":
			ur.set(rw, req, item)
		default:
			sendError(rw, http.StatusMethodNotAllowed, errors.New("only GET, PUT and DELETE supported against this resource"))
		}
	} else {
		rw.WriteHeader(http.StatusNotFound)
	}
}

func (ur *unitsResource) set(rw http.ResponseWriter, req *http.Request, item string) {
	if validateContentType(req) != nil {
		sendError(rw, http.StatusNotAcceptable, errors.New("application/json is only supported Content-Type"))
		return
	}

	var dus schema.DesiredUnitState
	dec := json.NewDecoder(req.Body)
	err := dec.Decode(&dus)
	if err != nil {
		sendError(rw, http.StatusBadRequest, fmt.Errorf("unable to decode body: %v", err))
		return
	}

	j, err := ur.reg.Job(item)
	if err != nil {
		log.Errorf("Failed fetching Job(%s) from Registry: %v", item, err)
		rw.WriteHeader(http.StatusInternalServerError)
		return
	}

	var u *unit.Unit
	if len(dus.FileContents) > 0 {
		u, err = decodeUnitContents(dus.FileContents)
		if err != nil {
			sendError(rw, http.StatusBadRequest, fmt.Errorf("invalid fileContents: %v", err))
			return
		}
	}

	// TODO(bcwaldon): Assert value of DesiredState is launched, loaded or inactive
	ds := job.JobState(dus.DesiredState)

	if j != nil {
		ur.update(rw, j, ds, u)
	} else if u != nil {
		ur.create(rw, item, ds, u)
	} else {
		sendError(rw, http.StatusConflict, errors.New("unit does not exist and no fileContents provided"))
	}
}

func (ur *unitsResource) create(rw http.ResponseWriter, item string, ds job.JobState, u *unit.Unit) {
	j := job.NewJob(item, *u)

	if err := ur.reg.CreateJob(j); err != nil {
		log.Errorf("Failed creating Job(%s) in Registry: %v", j.Name, err)
		rw.WriteHeader(http.StatusInternalServerError)
		return
	}

	if err := ur.reg.SetJobTargetState(j.Name, ds); err != nil {
		log.Errorf("Failed setting target state of Job(%s): %v", j.Name, err)
		rw.WriteHeader(http.StatusInternalServerError)
		return
	}

	rw.WriteHeader(http.StatusNoContent)
}

func (ur *unitsResource) update(rw http.ResponseWriter, j *job.Job, ds job.JobState, cmp *unit.Unit) {
	// Assert that the Job's Unit matches the Unit in the request, if provided
	if cmp != nil && cmp.Hash() != j.Unit.Hash() {
		sendError(rw, http.StatusConflict, errors.New("hash of provided fileContents does not match that of existing unit"))
		return
	}

	err := ur.reg.SetJobTargetState(j.Name, ds)
	if err != nil {
		log.Errorf("Failed setting target state of Job(%s): %v", j.Name, err)
		rw.WriteHeader(http.StatusInternalServerError)
		return
	}

	rw.WriteHeader(http.StatusNoContent)
}

func (ur *unitsResource) destroy(rw http.ResponseWriter, req *http.Request, item string) {
	if validateContentType(req) != nil {
		sendError(rw, http.StatusNotAcceptable, errors.New("application/json is only supported Content-Type"))
		return
	}

	var du schema.DeletableUnit
	dec := json.NewDecoder(req.Body)
	err := dec.Decode(&du)
	if err != nil {
		sendError(rw, http.StatusBadRequest, fmt.Errorf("unable to decode body: %v", err))
		return
	}

	var u *unit.Unit
	if len(du.FileContents) > 0 {
		u, err = decodeUnitContents(du.FileContents)
		if err != nil {
			sendError(rw, http.StatusBadRequest, fmt.Errorf("invalid fileContents: %v", err))
			return
		}
	}

	j, err := ur.reg.Job(item)
	if err != nil {
		log.Errorf("Failed fetching Job(%s): %v", item, err)
		rw.WriteHeader(http.StatusInternalServerError)
		return
	}

	if j == nil {
		sendError(rw, http.StatusNotFound, errors.New("unit does not exist"))
		return
	}

	if u != nil && u.Hash() != j.Unit.Hash() {
		sendError(rw, http.StatusConflict, errors.New("hash of provided fileContents does not match that of existing unit"))
		return
	}

	err = ur.reg.DestroyJob(item)
	if err != nil {
		log.Errorf("Failed destroying Job(%s): %v", item, err)
		rw.WriteHeader(http.StatusInternalServerError)
		return
	}

	rw.WriteHeader(http.StatusNoContent)
}

func (ur *unitsResource) get(rw http.ResponseWriter, req *http.Request, item string) {
	j, err := ur.reg.Job(item)
	if err != nil {
		log.Errorf("Failed fetching Job(%s) from Registry: %v", item, err)
		rw.WriteHeader(http.StatusInternalServerError)
		return
	}

	if j == nil {
		sendError(rw, http.StatusNotFound, errors.New("unit does not exist"))
		return
	}

	u, err := mapJobToSchema(ur.reg, j)
	if err != nil {
		log.Errorf("Failed mapping Job(%s) to schema: %v", item, err)
		rw.WriteHeader(http.StatusInternalServerError)
		return
	}

	sendResponse(rw, http.StatusOK, *u)
}

func (ur *unitsResource) list(rw http.ResponseWriter, req *http.Request) {
	token, err := findNextPageToken(req.URL)
	if err != nil {
		sendError(rw, http.StatusBadRequest, err)
		return
	}

	if token == nil {
		def := DefaultPageToken()
		token = &def
	}

	page, err := getUnitPage(ur.reg, *token)
	if err != nil {
		log.Errorf("Failed fetching page of Units: %v", err)
		rw.WriteHeader(http.StatusInternalServerError)
		return
	}

	sendResponse(rw, http.StatusOK, page)
}

func getUnitPage(reg registry.Registry, tok PageToken) (*schema.UnitPage, error) {
	all, err := reg.Jobs()
	if err != nil {
		return nil, err
	}

	page, err := extractUnitPage(reg, all, tok)
	if err != nil {
		return nil, err
	}

	return page, nil
}

func extractUnitPage(reg registry.Registry, all []job.Job, tok PageToken) (*schema.UnitPage, error) {
	total := len(all)

	startIndex := int((tok.Page - 1) * tok.Limit)
	stopIndex := int(tok.Page * tok.Limit)

	var items []job.Job
	var next *PageToken

	if startIndex < total {
		if stopIndex > total {
			stopIndex = total
		} else {
			n := tok.Next()
			next = &n
		}

		items = all[startIndex:stopIndex]
	}

	return newUnitPage(reg, items, next)
}

func newUnitPage(reg registry.Registry, items []job.Job, tok *PageToken) (*schema.UnitPage, error) {
	sup := schema.UnitPage{
		Units: make([]*schema.Unit, 0, len(items)),
	}

	if tok != nil {
		sup.NextPageToken = tok.Encode()
	}

	for _, j := range items {
		u, err := mapJobToSchema(reg, &j)
		if err != nil {
			return nil, err
		}
		sup.Units = append(sup.Units, u)
	}
	return &sup, nil
}

func mapJobToSchema(reg registry.Registry, j *job.Job) (*schema.Unit, error) {
	su := schema.Unit{
		Name:         j.Name,
		FileHash:     j.UnitHash.String(),
		FileContents: encodeUnitContents(&j.Unit),
	}

	if j.State != nil {
		su.CurrentState = string(*(j.State))
	}

	if j.UnitState != nil {
		su.Systemd = &schema.SystemdState{
			LoadState:   j.UnitState.LoadState,
			ActiveState: j.UnitState.ActiveState,
			SubState:    j.UnitState.SubState,
		}
		if j.UnitState.MachineState != nil {
			su.Systemd.MachineID = j.UnitState.MachineState.ID
		}
	}

	tgtMachID, err := reg.JobTarget(j.Name)
	if err != nil {
		return nil, err
	}

	su.TargetMachineID = tgtMachID

	tgtState, err := reg.JobTargetState(j.Name)
	if err != nil {
		return nil, err
	}

	if tgtState != nil {
		su.DesiredState = string(*tgtState)
	}

	return &su, nil
}

func encodeUnitContents(u *unit.Unit) string {
	return base64.StdEncoding.EncodeToString([]byte(u.Raw))
}

func decodeUnitContents(c string) (*unit.Unit, error) {
	dec, err := base64.StdEncoding.DecodeString(c)
	if err != nil {
		return nil, err
	}

	return unit.NewUnit(string(dec))
}
<commit_msg>feat(API): Return error entities with unit 500s<commit_after>package api

import (
	"encoding/base64"
	"encoding/json"
	"errors"
	"fmt"
	"net/http"
	"path"

	log "github.com/coreos/fleet/third_party/github.com/golang/glog"

	"github.com/coreos/fleet/job"
	"github.com/coreos/fleet/registry"
	"github.com/coreos/fleet/schema"
	"github.com/coreos/fleet/unit"
)

func wireUpUnitsResource(mux *http.ServeMux, prefix string, reg registry.Registry) {
	base := path.Join(prefix, "units")
	ur := unitsResource{reg, base}
	mux.Handle(base, &ur)
	mux.Handle(base+"/", &ur)
}

type unitsResource struct {
	reg      registry.Registry
	basePath string
}

func (ur *unitsResource) ServeHTTP(rw http.ResponseWriter, req *http.Request) {
	if isCollectionPath(ur.basePath, req.URL.Path) {
		switch req.Method {
		case "GET":
			ur.list(rw, req)
		default:
			sendError(rw, http.StatusMethodNotAllowed, errors.New("only GET supported against this resource"))
		}
	} else if item, ok := isItemPath(ur.basePath, req.URL.Path); ok {
		switch req.Method {
		case "GET":
			ur.get(rw, req, item)
		case "DELETE":
			ur.destroy(rw, req, item)
		case "PUT":
			ur.set(rw, req, item)
		default:
			sendError(rw, http.StatusMethodNotAllowed, errors.New("only GET, PUT and DELETE supported against this resource"))
		}
	} else {
		rw.WriteHeader(http.StatusNotFound)
	}
}

func (ur *unitsResource) set(rw http.ResponseWriter, req *http.Request, item string) {
	if validateContentType(req) != nil {
		sendError(rw, http.StatusNotAcceptable, errors.New("application/json is only supported Content-Type"))
		return
	}

	var dus schema.DesiredUnitState
	dec := json.NewDecoder(req.Body)
	err := dec.Decode(&dus)
	if err != nil {
		sendError(rw, http.StatusBadRequest, fmt.Errorf("unable to decode body: %v", err))
		return
	}

	j, err := ur.reg.Job(item)
	if err != nil {
		log.Errorf("Failed fetching Job(%s) from Registry: %v", item, err)
		sendError(rw, http.StatusInternalServerError, nil)
		return
	}

	var u *unit.Unit
	if len(dus.FileContents) > 0 {
		u, err = decodeUnitContents(dus.FileContents)
		if err != nil {
			sendError(rw, http.StatusBadRequest, fmt.Errorf("invalid fileContents: %v", err))
			return
		}
	}

	// TODO(bcwaldon): Assert value of DesiredState is launched, loaded or inactive
	ds := job.JobState(dus.DesiredState)

	if j != nil {
		ur.update(rw, j, ds, u)
	} else if u != nil {
		ur.create(rw, item, ds, u)
	} else {
		sendError(rw, http.StatusConflict, errors.New("unit does not exist and no fileContents provided"))
	}
}

func (ur *unitsResource) create(rw http.ResponseWriter, item string, ds job.JobState, u *unit.Unit) {
	j := job.NewJob(item, *u)

	if err := ur.reg.CreateJob(j); err != nil {
		log.Errorf("Failed creating Job(%s) in Registry: %v", j.Name, err)
		sendError(rw, http.StatusInternalServerError, nil)
		return
	}

	if err := ur.reg.SetJobTargetState(j.Name, ds); err != nil {
		log.Errorf("Failed setting target state of Job(%s): %v", j.Name, err)
		sendError(rw, http.StatusInternalServerError, nil)
		return
	}

	rw.WriteHeader(http.StatusNoContent)
}

func (ur *unitsResource) update(rw http.ResponseWriter, j *job.Job, ds job.JobState, cmp *unit.Unit) {
	// Assert that the Job's Unit matches the Unit in the request, if provided
	if cmp != nil && cmp.Hash() != j.Unit.Hash() {
		sendError(rw, http.StatusConflict, errors.New("hash of provided fileContents does not match that of existing unit"))
		return
	}

	err := ur.reg.SetJobTargetState(j.Name, ds)
	if err != nil {
		log.Errorf("Failed setting target state of Job(%s): %v", j.Name, err)
		sendError(rw, http.StatusInternalServerError, nil)
		return
	}

	rw.WriteHeader(http.StatusNoContent)
}

func (ur *unitsResource) destroy(rw http.ResponseWriter, req *http.Request, item string) {
	if validateContentType(req) != nil {
		sendError(rw, http.StatusNotAcceptable, errors.New("application/json is only supported Content-Type"))
		return
	}

	var du schema.DeletableUnit
	dec := json.NewDecoder(req.Body)
	err := dec.Decode(&du)
	if err != nil {
		sendError(rw, http.StatusBadRequest, fmt.Errorf("unable to decode body: %v", err))
		return
	}

	var u *unit.Unit
	if len(du.FileContents) > 0 {
		u, err = decodeUnitContents(du.FileContents)
		if err != nil {
			sendError(rw, http.StatusBadRequest, fmt.Errorf("invalid fileContents: %v", err))
			return
		}
	}

	j, err := ur.reg.Job(item)
	if err != nil {
		log.Errorf("Failed fetching Job(%s): %v", item, err)
		sendError(rw, http.StatusInternalServerError, nil)
		return
	}

	if j == nil {
		sendError(rw, http.StatusNotFound, errors.New("unit does not exist"))
		return
	}

	if u != nil && u.Hash() != j.Unit.Hash() {
		sendError(rw, http.StatusConflict, errors.New("hash of provided fileContents does not match that of existing unit"))
		return
	}

	err = ur.reg.DestroyJob(item)
	if err != nil {
		log.Errorf("Failed destroying Job(%s): %v", item, err)
		sendError(rw, http.StatusInternalServerError, nil)
		return
	}

	rw.WriteHeader(http.StatusNoContent)
}

func (ur *unitsResource) get(rw http.ResponseWriter, req *http.Request, item string) {
	j, err := ur.reg.Job(item)
	if err != nil {
		log.Errorf("Failed fetching Job(%s) from Registry: %v", item, err)
		sendError(rw, http.StatusInternalServerError, nil)
		return
	}

	if j == nil {
		sendError(rw, http.StatusNotFound, errors.New("unit does not exist"))
		return
	}

	u, err := mapJobToSchema(ur.reg, j)
	if err != nil {
		log.Errorf("Failed mapping Job(%s) to schema: %v", item, err)
		sendError(rw, http.StatusInternalServerError, nil)
		return
	}

	sendResponse(rw, http.StatusOK, *u)
}

func (ur *unitsResource) list(rw http.ResponseWriter, req *http.Request) {
	token, err := findNextPageToken(req.URL)
	if err != nil {
		sendError(rw, http.StatusBadRequest, err)
		return
	}

	if token == nil {
		def := DefaultPageToken()
		token = &def
	}

	page, err := getUnitPage(ur.reg, *token)
	if err != nil {
		log.Errorf("Failed fetching page of Units: %v", err)
		sendError(rw, http.StatusInternalServerError, nil)
		return
	}

	sendResponse(rw, http.StatusOK, page)
}

func getUnitPage(reg registry.Registry, tok PageToken) (*schema.UnitPage, error) {
	all, err := reg.Jobs()
	if err != nil {
		return nil, err
	}

	page, err := extractUnitPage(reg, all, tok)
	if err != nil {
		return nil, err
	}

	return page, nil
}

func extractUnitPage(reg registry.Registry, all []job.Job, tok PageToken) (*schema.UnitPage, error) {
	total := len(all)

	startIndex := int((tok.Page - 1) * tok.Limit)
	stopIndex := int(tok.Page * tok.Limit)

	var items []job.Job
	var next *PageToken

	if startIndex < total {
		if stopIndex > total {
			stopIndex = total
		} else {
			n := tok.Next()
			next = &n
		}

		items = all[startIndex:stopIndex]
	}

	return newUnitPage(reg, items, next)
}

func newUnitPage(reg registry.Registry, items []job.Job, tok *PageToken) (*schema.UnitPage, error) {
	sup := schema.UnitPage{
		Units: make([]*schema.Unit, 0, len(items)),
	}

	if tok != nil {
		sup.NextPageToken = tok.Encode()
	}

	for _, j := range items {
		u, err := mapJobToSchema(reg, &j)
		if err != nil {
			return nil, err
		}
		sup.Units = append(sup.Units, u)
	}
	return &sup, nil
}

func mapJobToSchema(reg registry.Registry, j *job.Job) (*schema.Unit, error) {
	su := schema.Unit{
		Name:         j.Name,
		FileHash:     j.UnitHash.String(),
		FileContents: encodeUnitContents(&j.Unit),
	}

	if j.State != nil {
		su.CurrentState = string(*(j.State))
	}

	if j.UnitState != nil {
		su.Systemd = &schema.SystemdState{
			LoadState:   j.UnitState.LoadState,
			ActiveState: j.UnitState.ActiveState,
			SubState:    j.UnitState.SubState,
		}
		if j.UnitState.MachineState != nil {
			su.Systemd.MachineID = j.UnitState.MachineState.ID
		}
	}

	tgtMachID, err := reg.JobTarget(j.Name)
	if err != nil {
		return nil, err
	}

	su.TargetMachineID = tgtMachID

	tgtState, err := reg.JobTargetState(j.Name)
	if err != nil {
		return nil, err
	}

	if tgtState != nil {
		su.DesiredState = string(*tgtState)
	}

	return &su, nil
}

func encodeUnitContents(u *unit.Unit) string {
	return base64.StdEncoding.EncodeToString([]byte(u.Raw))
}

func decodeUnitContents(c string) (*unit.Unit, error) {
	dec, err := base64.StdEncoding.DecodeString(c)
	if err != nil {
		return nil, err
	}

	return unit.NewUnit(string(dec))
}
<|endoftext|>


<commit_before>package gophercloud

import(
 "fmt"
 "github.com/mitchellh/mapstructure"
)

//The default generic openstack api
var OpenstackApi = map[string]interface{}{
        "Name":      "nova",
	"UrlChoice": PublicURL,
}

// Api for use with rackspace
var RackspaceApi = map[string]interface{}{
	"Name":      "cloudServersOpenStack",
	"VersionId": "2",
	"UrlChoice": PublicURL,
}

//Populates an ApiCriteria struct with the api values
//from one of the api maps
func PopulateApi(variant string) (ApiCriteria, error){
	var Api ApiCriteria
	var variantMap map[string]interface{}

	switch variant {
	case "":
		variantMap = OpenstackApi

	case "openstack":
		variantMap = OpenstackApi

	case "rackspace":
		variantMap = RackspaceApi

	default:
		var err = fmt.Errorf(
			"PopulateApi: Unknown variant %# v; legal values: \"openstack\", \"rackspace\"", variant)
		return Api, err
	}

	err := mapstructure.Decode(variantMap,&Api)
		if err != nil{
			return Api,err
		}
	return Api, err
}
<commit_msg>Query for type "compute" instead of name "nova"<commit_after>package gophercloud

import(
 "fmt"
 "github.com/mitchellh/mapstructure"
)

//The default generic openstack api
var OpenstackApi = map[string]interface{}{
	"Type": "compute",
	"UrlChoice": PublicURL,
}

// Api for use with rackspace
var RackspaceApi = map[string]interface{}{
	"Name":      "cloudServersOpenStack",
	"VersionId": "2",
	"UrlChoice": PublicURL,
}

//Populates an ApiCriteria struct with the api values
//from one of the api maps
func PopulateApi(variant string) (ApiCriteria, error){
	var Api ApiCriteria
	var variantMap map[string]interface{}

	switch variant {
	case "":
		variantMap = OpenstackApi

	case "openstack":
		variantMap = OpenstackApi

	case "rackspace":
		variantMap = RackspaceApi

	default:
		var err = fmt.Errorf(
			"PopulateApi: Unknown variant %# v; legal values: \"openstack\", \"rackspace\"", variant)
		return Api, err
	}

	err := mapstructure.Decode(variantMap,&Api)
		if err != nil{
			return Api,err
		}
	return Api, err
}
<|endoftext|>


<commit_before>package sync

import (
	"sync"
	"testing"
)

// TestTryMutexBasicMutex verifies that Lock and Unlock work the same as a
// normal mutex would.
func TestTryMutexBasicMutex(t *testing.T) {
	// Check that two calls to lock will execute in the correct order.
	var tm TryMutex
	var data int
	tm.Lock()
	go func() {
		data = 15
		tm.Unlock()
	}()
	tm.Lock()
	if data != 15 {
		t.Error("Locking did not safely protect the data")
	}
	tm.Unlock()
}

// TestTryMutexConcurrentLocking checks that doing lots of concurrent locks is
// handled as expected.
func TestTryMutexConcurrentLocking(t *testing.T) {
	// Try executing multiple additions concurrently.
	var tm TryMutex
	var data int
	var wg sync.WaitGroup
	for i := 0; i < 250; i++ {
		wg.Add(1)
		go func() {
			tm.Lock()
			data++
			tm.Unlock()
			wg.Done()
		}()
	}
	wg.Wait()
	if data != 250 {
		t.Error("Locking did not safely protect the data")
	}
}

// TestTryMutexBasicTryLock checks that a TryLock will succeed if nobody is
// holding a lock, and will fail if the lock is being held.
func TestTryMutexBasicTryLock(t *testing.T) {
	// Lock and then TryLock.
	var tm TryMutex
	tm.Lock()
	if tm.TryLock() {
		t.Error("TryLock should have failed")
	}
	tm.Unlock()

	tm.Lock()
	tm.Unlock()

	// TryLock and then TryLock.
	if !tm.TryLock() {
		t.Error("Could not get a blank lock")
	}
	if tm.TryLock() {
		t.Error("should not have been able to get the lock")
	}
	tm.Unlock()
}

// TestTryMutexConcurrentTries attempts to grab locks from many threads, giving
// the race detector a chance to detect any issues.
func TestTryMutexConncurrentTries(t *testing.T) {
	// Try executing multiple additions concurrently.
	var tm TryMutex
	var data int
	var wg sync.WaitGroup
	for i := 0; i < 250; i++ {
		wg.Add(1)
		go func() {
			for !tm.TryLock() {
			}

			data++
			tm.Unlock()
			wg.Done()
		}()
	}
	wg.Wait()
	if data != 250 {
		t.Error("Locking did not safely protect the data")
	}
}
<commit_msg>add tests for trylocktimed<commit_after>package sync

import (
	"sync"
	"testing"
	"time"
)

// TestTryMutexBasicMutex verifies that Lock and Unlock work the same as a
// normal mutex would.
func TestTryMutexBasicMutex(t *testing.T) {
	// Check that two calls to lock will execute in the correct order.
	var tm TryMutex
	var data int
	tm.Lock()
	go func() {
		data = 15
		tm.Unlock()
	}()
	tm.Lock()
	if data != 15 {
		t.Error("Locking did not safely protect the data")
	}
	tm.Unlock()
}

// TestTryMutexConcurrentLocking checks that doing lots of concurrent locks is
// handled as expected.
func TestTryMutexConcurrentLocking(t *testing.T) {
	// Try executing multiple additions concurrently.
	var tm TryMutex
	var data int
	var wg sync.WaitGroup
	for i := 0; i < 250; i++ {
		wg.Add(1)
		go func() {
			tm.Lock()
			data++
			tm.Unlock()
			wg.Done()
		}()
	}
	wg.Wait()
	if data != 250 {
		t.Error("Locking did not safely protect the data")
	}
}

// TestTryMutexBasicTryLock checks that a TryLock will succeed if nobody is
// holding a lock, and will fail if the lock is being held.
func TestTryMutexBasicTryLock(t *testing.T) {
	// Lock and then TryLock.
	var tm TryMutex
	tm.Lock()
	if tm.TryLock() {
		t.Error("TryLock should have failed")
	}
	tm.Unlock()

	tm.Lock()
	tm.Unlock()

	// TryLock and then TryLock.
	if !tm.TryLock() {
		t.Error("Could not get a blank lock")
	}
	if tm.TryLock() {
		t.Error("should not have been able to get the lock")
	}
	tm.Unlock()
}

// TestTryMutexConcurrentTries attempts to grab locks from many threads, giving
// the race detector a chance to detect any issues.
func TestTryMutexConncurrentTries(t *testing.T) {
	// Try executing multiple additions concurrently.
	var tm TryMutex
	var data int
	var wg sync.WaitGroup
	for i := 0; i < 250; i++ {
		wg.Add(1)
		go func() {
			for !tm.TryLock() {
			}

			data++
			tm.Unlock()
			wg.Done()
		}()
	}
	wg.Wait()
	if data != 250 {
		t.Error("Locking did not safely protect the data")
	}
}

// TestTryMutexTimed checks that a timed lock will correctly time out if it
// cannot grab a lock.
func TestTryMutexTimed(t *testing.T) {
	var tm TryMutex
	tm.Lock()

	startTime := time.Now()
	if tm.TryLockTimed(time.Millisecond * 500) {
		t.Error("was able to grab a locked lock")
	}

	if time.Now().Sub(startTime) < time.Millisecond*500 {
		t.Error("lock did not wait the correct amount of time before timing out")
	}
	if time.Now().Sub(startTime) > time.Millisecond*750 {
		t.Error("lock waited too long before timing out")
	}

	tm.Unlock()
	if !tm.TryLockTimed(time.Millisecond * 1) {
		t.Error("Unable to get an unlocked lock")
	}
	tm.Unlock()
}

// TestTryMutexTimedConcurrent checks that a timed lock will correctly time out
// if it cannot grab a lock.
func TestTryMutexTimedConcurrent(t *testing.T) {
	var tm TryMutex

	// Engage a lock and launch a gothread to wait for a lock, fail, and then
	// call unlock.
	tm.Lock()
	go func() {
		startTime := time.Now()
		if tm.TryLockTimed(time.Millisecond * 500) {
			t.Error("was able to grab a locked lock")
		}

		if time.Now().Sub(startTime) < time.Millisecond*500 {
			t.Error("lock did not wait the correct amount of time before timing out")
		}
		if time.Now().Sub(startTime) > time.Millisecond*750 {
			t.Error("lock waited too long before timing out")
		}

		tm.Unlock()
	}()

	// Try to get a lock, but don't wait long enough.
	if tm.TryLockTimed(time.Millisecond * 250) {
		// Lock shoud time out because the gothread responsible for releasing
		// the lock will be idle for 500 milliseconds.
		t.Error("Lock should have timed out")
	}
	if !tm.TryLockTimed(time.Millisecond * 750) {
		// Lock should be successful - the above thread should finish in under
		// 750 milliseconds.
		t.Error("Lock should have been successful")
	}
	tm.Unlock()
}
<|endoftext|>


<commit_before>// Copyright 2013 tsuru authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

package app

import (
	"fmt"
	"github.com/globocom/tsuru/db"
	"github.com/globocom/tsuru/log"
	"github.com/globocom/tsuru/provision"
	"github.com/globocom/tsuru/queue"
	"github.com/globocom/tsuru/service"
	"io/ioutil"
	"labix.org/v2/mgo/bson"
)

const (
	// queue actions
	regenerateApprc         = "regenerate-apprc"
	startApp                = "start-app"
	RegenerateApprcAndStart = "regenerate-apprc-start-app"
	bindService             = "bind-service"

	// name of the queue for internal messages.
	queueName = "tsuru-app"

	// Name of the queue for external messages.
	QueueName = "tsuru-app-public"
)

func ensureAppIsStarted(msg *queue.Message) (App, error) {
	a := App{Name: msg.Args[0]}
	err := a.Get()
	if err != nil {
		return a, fmt.Errorf("Error handling %q: app %q does not exist.", msg.Action, a.Name)
	}
	units := getUnits(&a, msg.Args[1:])
	if a.State != "started" || !units.Started() {
		format := "Error handling %q for the app %q:"
		switch a.State {
		case "error":
			format += " the app is in %q state."
			msg.Delete()
		case "down":
			format += " the app is %s."
			msg.Delete()
		default:
			format += ` The status of the app and all units should be "started" (the app is %q).`
			msg.Release(5e9)
		}
		return a, fmt.Errorf(format, msg.Action, a.Name, a.State)
	}
	return a, nil
}

func bindUnit(msg *queue.Message) error {
	a := App{Name: msg.Args[0]}
	err := a.Get()
	if err != nil {
		return fmt.Errorf("Error handling %q: app %q does not exist.", msg.Action, a.Name)
	}
	unit := getUnits(&a, msg.Args[1:])[0]
	var instances []service.ServiceInstance
	err = db.Session.ServiceInstances().Find(bson.M{"apps": bson.M{"$in": []string{msg.Args[0]}}}).All(&instances)
	if err != nil {
		return err
	}
	for _, instance := range instances {
		_, err = instance.BindUnit(&unit)
		if err != nil {
			log.Printf("Error binding the unit %s with the service instance %s.", unit.Name, instance.Name)
		}
	}
	return nil
}

func handle(msg *queue.Message) {
	switch msg.Action {
	case RegenerateApprcAndStart:
		fallthrough
	case regenerateApprc:
		if len(msg.Args) < 1 {
			log.Printf("Error handling %q: this action requires at least 1 argument.", msg.Action)
			return
		}
		app, err := ensureAppIsStarted(msg)
		if err != nil {
			log.Print(err)
			return
		}
		msg.Delete()
		app.SerializeEnvVars()
		fallthrough
	case startApp:
		if msg.Action == regenerateApprc {
			break
		}
		if len(msg.Args) < 1 {
			log.Printf("Error handling %q: this action requires at least 1 argument.", msg.Action)
		}
		app, err := ensureAppIsStarted(msg)
		if err != nil {
			log.Print(err)
			return
		}
		err = app.Restart(ioutil.Discard)
		if err != nil {
			log.Printf("Error handling %q. App failed to start:\n%s.", msg.Action, err)
			return
		}
		msg.Delete()
	case bindService:
		err := bindUnit(msg)
		if err != nil {
			log.Print(err)
			return
		}
		msg.Delete()
	default:
		log.Printf("Error handling %q: invalid action.", msg.Action)
		msg.Release(0)
	}
}

type unitList []Unit

func (l unitList) Started() bool {
	for _, unit := range l {
		if unit.State != string(provision.StatusStarted) {
			return false
		}
	}
	return true
}

func getUnits(a *App, names []string) unitList {
	var units []Unit
	if len(names) > 0 {
		units = make([]Unit, len(names))
		i := 0
		for _, unitName := range names {
			for _, appUnit := range a.Units {
				if appUnit.Name == unitName {
					units[i] = appUnit
					i++
					break
				}
			}
		}
	}
	return unitList(units)
}

var handler = &queue.Handler{F: handle, Queues: []string{queueName, QueueName}}

func enqueue(msgs ...queue.Message) {
	for _, msg := range msgs {
		copy := msg
		copy.Put(queueName, 0)
	}
	handler.Start()
}
<commit_msg>Using app.Avaliable in queue.<commit_after>// Copyright 2013 tsuru authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

package app

import (
	"fmt"
	"github.com/globocom/tsuru/db"
	"github.com/globocom/tsuru/log"
	"github.com/globocom/tsuru/provision"
	"github.com/globocom/tsuru/queue"
	"github.com/globocom/tsuru/service"
	"io/ioutil"
	"labix.org/v2/mgo/bson"
)

const (
	// queue actions
	regenerateApprc         = "regenerate-apprc"
	startApp                = "start-app"
	RegenerateApprcAndStart = "regenerate-apprc-start-app"
	bindService             = "bind-service"

	// name of the queue for internal messages.
	queueName = "tsuru-app"

	// Name of the queue for external messages.
	QueueName = "tsuru-app-public"
)

func ensureAppIsStarted(msg *queue.Message) (App, error) {
	a := App{Name: msg.Args[0]}
	err := a.Get()
	if err != nil {
		return a, fmt.Errorf("Error handling %q: app %q does not exist.", msg.Action, a.Name)
	}
	units := getUnits(&a, msg.Args[1:])
	if !a.Avaliable() || !units.Started() {
		format := "Error handling %q for the app %q:"
		switch a.State {
		case "error":
			format += " the app is in %q state."
			msg.Delete()
		case "down":
			format += " the app is %s."
			msg.Delete()
		default:
			format += ` The status of the app and all units should be "started" (the app is %q).`
			msg.Release(5e9)
		}
		return a, fmt.Errorf(format, msg.Action, a.Name, a.State)
	}
	return a, nil
}

func bindUnit(msg *queue.Message) error {
	a := App{Name: msg.Args[0]}
	err := a.Get()
	if err != nil {
		return fmt.Errorf("Error handling %q: app %q does not exist.", msg.Action, a.Name)
	}
	unit := getUnits(&a, msg.Args[1:])[0]
	var instances []service.ServiceInstance
	err = db.Session.ServiceInstances().Find(bson.M{"apps": bson.M{"$in": []string{msg.Args[0]}}}).All(&instances)
	if err != nil {
		return err
	}
	for _, instance := range instances {
		_, err = instance.BindUnit(&unit)
		if err != nil {
			log.Printf("Error binding the unit %s with the service instance %s.", unit.Name, instance.Name)
		}
	}
	return nil
}

func handle(msg *queue.Message) {
	switch msg.Action {
	case RegenerateApprcAndStart:
		fallthrough
	case regenerateApprc:
		if len(msg.Args) < 1 {
			log.Printf("Error handling %q: this action requires at least 1 argument.", msg.Action)
			return
		}
		app, err := ensureAppIsStarted(msg)
		if err != nil {
			log.Print(err)
			return
		}
		msg.Delete()
		app.SerializeEnvVars()
		fallthrough
	case startApp:
		if msg.Action == regenerateApprc {
			break
		}
		if len(msg.Args) < 1 {
			log.Printf("Error handling %q: this action requires at least 1 argument.", msg.Action)
		}
		app, err := ensureAppIsStarted(msg)
		if err != nil {
			log.Print(err)
			return
		}
		err = app.Restart(ioutil.Discard)
		if err != nil {
			log.Printf("Error handling %q. App failed to start:\n%s.", msg.Action, err)
			return
		}
		msg.Delete()
	case bindService:
		err := bindUnit(msg)
		if err != nil {
			log.Print(err)
			return
		}
		msg.Delete()
	default:
		log.Printf("Error handling %q: invalid action.", msg.Action)
		msg.Release(0)
	}
}

type unitList []Unit

func (l unitList) Started() bool {
	for _, unit := range l {
		if unit.State != string(provision.StatusStarted) {
			return false
		}
	}
	return true
}

func getUnits(a *App, names []string) unitList {
	var units []Unit
	if len(names) > 0 {
		units = make([]Unit, len(names))
		i := 0
		for _, unitName := range names {
			for _, appUnit := range a.Units {
				if appUnit.Name == unitName {
					units[i] = appUnit
					i++
					break
				}
			}
		}
	}
	return unitList(units)
}

var handler = &queue.Handler{F: handle, Queues: []string{queueName, QueueName}}

func enqueue(msgs ...queue.Message) {
	for _, msg := range msgs {
		copy := msg
		copy.Put(queueName, 0)
	}
	handler.Start()
}
<|endoftext|>


<commit_before>package main

import (
	"context"
	"fmt"
	"time"

	"google.golang.org/appengine"
	"google.golang.org/appengine/datastore"
	"google.golang.org/appengine/delay"
	"google.golang.org/appengine/log"
	"google.golang.org/appengine/taskqueue"

	"github.com/google/go-github/github"
)

const (
	VintageDateFormat = "January 2, 2006"
	VintageChunkSize  = 1000
)

type RepoVintage struct {
	UserId  int       `datastore:",noindex"`
	RepoId  int       `datastore:",noindex"`
	Vintage time.Time `datastore:",noindex"`
}

func getVintageKey(c context.Context, userId int, repoId int) *datastore.Key {
	return datastore.NewKey(c, "RepoVintage", fmt.Sprintf("%d-%d", userId, repoId), 0, nil)
}

var computeVintageFunc *delay.Function

func computeVintage(c context.Context, userId int, userLogin string, repoId int, repoOwnerLogin string, repoName string) error {
	account, err := getAccount(c, userId)
	if err != nil {
		log.Errorf(c, "Could not load account %d: %s. Presumed deleted, aborting computing vintage for %s/%s", userId, err.Error(), repoOwnerLogin, repoName)
		return nil
	}

	oauthTransport := githubOAuthTransport(c)
	oauthTransport.Token = &account.OAuthToken
	githubClient := github.NewClient(oauthTransport.Client())

	repo, response, err := githubClient.Repositories.Get(repoOwnerLogin, repoName)
	if response.StatusCode == 403 || response.StatusCode == 404 {
		log.Warningf(c, "Got a %d when trying to look up %s/%s (%d)", response.StatusCode, repoOwnerLogin, repoName, repoId)
		_, err = datastore.Put(c, getVintageKey(c, userId, repoId), &RepoVintage{
			UserId:  userId,
			RepoId:  repoId,
			Vintage: time.Unix(0, 0),
		})
		return err
	} else if err != nil {
		log.Errorf(c, "Could not load repo %s/%s (%d): %s", repoOwnerLogin, repoName, repoId, err.Error())
		return err
	}

	// Cheap check to see if there are commits before the creation time.
	vintage := repo.CreatedAt.UTC()
	beforeCreationTime := repo.CreatedAt.UTC().AddDate(0, 0, -1)
	commits, response, err := githubClient.Repositories.ListCommits(
		repoOwnerLogin,
		repoName,
		&github.CommitsListOptions{
			ListOptions: github.ListOptions{PerPage: 1},
			Author:      userLogin,
			Until:       beforeCreationTime,
		})
	if response != nil && response.StatusCode == 409 {
		// GitHub returns with a 409 when a repository is empty.
		commits = make([]github.RepositoryCommit, 0)
	} else if err != nil {
		log.Errorf(c, "Could not load commits for repo %s (%d): %s", *repo.FullName, repoId, err.Error())
		return err
	}

	// If there are, then we use the contributor stats API to figure out when
	// the user's first commit in the repository was.
	if len(commits) > 0 {
		stats, response, err := githubClient.Repositories.ListContributorsStats(repoOwnerLogin, repoName)
		if response.StatusCode == 202 {
			log.Infof(c, "Stats were not available for %s, will try again later", *repo.FullName)
			task, err := computeVintageFunc.Task(userId, userLogin, repoId, repoOwnerLogin, repoName)
			if err != nil {
				log.Errorf(c, "Could create delayed task for %s: %s", *repo.FullName, err.Error())
				return err
			}
			task.Delay = time.Second * 10
			taskqueue.Add(c, task, "")
			return nil
		}
		if err != nil {
			log.Errorf(c, "Could not load stats for repo %s: %s", *repo.FullName, err.Error())
			return err
		}
		for _, stat := range stats {
			if *stat.Author.ID == userId {
				for i := range stat.Weeks {
					weekTimestamp := stat.Weeks[i].Week.UTC()
					if weekTimestamp.Before(vintage) {
						vintage = weekTimestamp
					}
				}
				break
			}
		}
	}

	_, err = datastore.Put(c, getVintageKey(c, userId, repoId), &RepoVintage{
		UserId:  userId,
		RepoId:  repoId,
		Vintage: vintage,
	})
	if err != nil {
		log.Errorf(c, "Could save vintage for repo %s: %s", *repo.FullName, err.Error())
		return err
	}

	return nil
}

func init() {
	computeVintageFunc = delay.Func("computeVintage", computeVintage)
}

func fillVintages(c context.Context, user *github.User, repos []*Repo) error {
	if len(repos) > VintageChunkSize {
		for chunkStart := 0; chunkStart < len(repos); chunkStart += VintageChunkSize {
			chunkEnd := chunkStart + VintageChunkSize
			if chunkEnd > len(repos) {
				chunkEnd = len(repos)
			}
			err := fillVintages(c, user, repos[chunkStart:chunkEnd])
			if err != nil {
				return err
			}
		}
		return nil
	}
	keys := make([]*datastore.Key, len(repos))
	for i := range repos {
		keys[i] = getVintageKey(c, *user.ID, *repos[i].ID)
	}
	vintages := make([]*RepoVintage, len(repos))
	for i := range vintages {
		vintages[i] = new(RepoVintage)
	}
	err := datastore.GetMulti(c, keys, vintages)
	if err != nil {
		if errs, ok := err.(appengine.MultiError); ok {
			for i, err := range errs {
				if err == datastore.ErrNoSuchEntity {
					vintages[i] = nil
				} else if err != nil {
					log.Errorf(c, "%d/%s vintage fetch error: %s", i, *repos[i].FullName, err.Error())
					return err
				}
			}
		} else {
			return err
		}
	}
	for i := range vintages {
		repo := repos[i]
		vintage := vintages[i]
		if vintage != nil {
			if vintage.Vintage.Unix() != 0 {
				repo.Vintage = vintage.Vintage
			}
			continue
		}
		computeVintageFunc.Call(c, *user.ID, *user.Login, *repo.ID, *repo.Owner.Login, *repo.Name)
	}
	return nil
}

type Repos struct {
	AllRepos       []*Repo
	UserRepos      []*Repo
	OtherUserRepos []*UserRepos
	OldestVintage  time.Time
}

func (repos *Repos) Redact() {
	for _, repo := range repos.UserRepos {
		*repo.HTMLURL = "https://redacted"
		*repo.FullName = "redacted/redacted"
	}
	for _, otherUserRepos := range repos.OtherUserRepos {
		*otherUserRepos.User.Login = "redacted"
		*otherUserRepos.User.AvatarURL = "https://redacted"
		for _, repo := range otherUserRepos.Repos {
			*repo.HTMLURL = "https://redacted"
			*repo.FullName = "redacted/redacted"
		}
	}
}

type Repo struct {
	*github.Repository
	Vintage         time.Time
	IncludeInDigest bool
}

func newRepo(githubRepo *github.Repository, account *Account) *Repo {
	return &Repo{
		Repository:      githubRepo,
		Vintage:         githubRepo.CreatedAt.UTC(),
		IncludeInDigest: !account.IsRepoIdExcluded(*githubRepo.ID),
	}
}

func (repo *Repo) TypeAsOcticonName() string {
	if *repo.Fork {
		return "repo-forked"
	}
	if *repo.Private {
		return "lock"
	}
	return "repo"
}

func (repo *Repo) TypeAsClassName() string {
	if *repo.Fork {
		return "fork"
	}
	if *repo.Private {
		return "private"
	}
	return ""
}

func (repo *Repo) DisplayVintage() string {
	return repo.Vintage.Format(VintageDateFormat)
}

type UserRepos struct {
	User  *github.User
	Repos []*Repo
}

func getRepos(c context.Context, githubClient *github.Client, account *Account, user *github.User) (*Repos, error) {
	clientUserRepos := make([]github.Repository, 0)
	page := 1
	for {
		pageClientUserRepos, response, err := githubClient.Repositories.List(
			// The username parameter must be left blank so that we can get all
			// of the repositories the user has access to, not just ones that
			// they own.
			"",
			&github.RepositoryListOptions{
				ListOptions: github.ListOptions{
					Page:    page,
					PerPage: 100,
				},
			})
		if err != nil {
			return nil, err
		}
		clientUserRepos = append(clientUserRepos, pageClientUserRepos...)
		if response.NextPage == 0 {
			break
		}
		page = response.NextPage
	}

	repos := &Repos{}
	repos.UserRepos = make([]*Repo, 0, len(clientUserRepos))
	repos.OtherUserRepos = make([]*UserRepos, 0)
	for i := range clientUserRepos {
		ownerID := *clientUserRepos[i].Owner.ID
		if ownerID == *user.ID {
			repos.UserRepos = append(repos.UserRepos, newRepo(&clientUserRepos[i], account))
		} else {
			var userRepos *UserRepos
			for j := range repos.OtherUserRepos {
				if *repos.OtherUserRepos[j].User.ID == ownerID {
					userRepos = repos.OtherUserRepos[j]
					break
				}
			}
			if userRepos == nil {
				userRepos = &UserRepos{
					User:  clientUserRepos[i].Owner,
					Repos: make([]*Repo, 0),
				}
				repos.OtherUserRepos = append(repos.OtherUserRepos, userRepos)
			}
			userRepos.Repos = append(userRepos.Repos, newRepo(&clientUserRepos[i], account))
		}
	}

	repos.AllRepos = make([]*Repo, 0, len(clientUserRepos))
	repos.AllRepos = append(repos.AllRepos, repos.UserRepos...)
	for _, userRepos := range repos.OtherUserRepos {
		repos.AllRepos = append(repos.AllRepos, userRepos.Repos...)
	}

	err := fillVintages(c, user, repos.AllRepos)
	if err != nil {
		return nil, err
	}

	repos.OldestVintage = time.Now().UTC()
	for _, repo := range repos.AllRepos {
		repoVintage := repo.Vintage
		if repoVintage.Before(repos.OldestVintage) {
			repos.OldestVintage = repoVintage
		}
	}

	return repos, nil
}
<commit_msg>Avoid infinite task queue retries when trying to compute repo vintages<commit_after>package main

import (
	"context"
	"fmt"
	"time"

	"google.golang.org/appengine"
	"google.golang.org/appengine/datastore"
	"google.golang.org/appengine/delay"
	"google.golang.org/appengine/log"
	"google.golang.org/appengine/taskqueue"

	"github.com/google/go-github/github"
)

const (
	VintageDateFormat = "January 2, 2006"
	VintageChunkSize  = 1000
)

type RepoVintage struct {
	UserId  int       `datastore:",noindex"`
	RepoId  int       `datastore:",noindex"`
	Vintage time.Time `datastore:",noindex"`
}

func getVintageKey(c context.Context, userId int, repoId int) *datastore.Key {
	return datastore.NewKey(c, "RepoVintage", fmt.Sprintf("%d-%d", userId, repoId), 0, nil)
}

var computeVintageFunc *delay.Function

func computeVintage(c context.Context, userId int, userLogin string, repoId int, repoOwnerLogin string, repoName string) error {
	account, err := getAccount(c, userId)
	if err != nil {
		log.Errorf(c, "Could not load account %d: %s. Presumed deleted, aborting computing vintage for %s/%s", userId, err.Error(), repoOwnerLogin, repoName)
		return nil
	}

	oauthTransport := githubOAuthTransport(c)
	oauthTransport.Token = &account.OAuthToken
	githubClient := github.NewClient(oauthTransport.Client())

	repo, response, err := githubClient.Repositories.Get(repoOwnerLogin, repoName)
	if response.StatusCode == 403 || response.StatusCode == 404 {
		log.Warningf(c, "Got a %d when trying to look up %s/%s (%d)", response.StatusCode, repoOwnerLogin, repoName, repoId)
		_, err = datastore.Put(c, getVintageKey(c, userId, repoId), &RepoVintage{
			UserId:  userId,
			RepoId:  repoId,
			Vintage: time.Unix(0, 0),
		})
		return err
	} else if err != nil {
		log.Errorf(c, "Could not load repo %s/%s (%d): %s", repoOwnerLogin, repoName, repoId, err.Error())
		return err
	}

	// Cheap check to see if there are commits before the creation time.
	vintage := repo.CreatedAt.UTC()
	beforeCreationTime := repo.CreatedAt.UTC().AddDate(0, 0, -1)
	commits, response, err := githubClient.Repositories.ListCommits(
		repoOwnerLogin,
		repoName,
		&github.CommitsListOptions{
			ListOptions: github.ListOptions{PerPage: 1},
			Author:      userLogin,
			Until:       beforeCreationTime,
		})
	if response != nil && response.StatusCode == 409 {
		// GitHub returns with a 409 when a repository is empty.
		commits = make([]github.RepositoryCommit, 0)
	} else if response != nil && response.StatusCode >= 500 {
		// Avoid retries if GitHub can't load commits (this happens for repos
		// like AOSiP-Devices/kernel_xiaomi_laurel_sprout, presumably because
		// they have too many commits).
		log.Warningf(c, "Could not load commits for repo %s (%d), not retrying: %s", *repo.FullName, repoId, err.Error())
		commits = make([]github.RepositoryCommit, 0)
	} else if err != nil {
		log.Errorf(c, "Could not load commits for repo %s (%d), will retry: %s", *repo.FullName, repoId, err.Error())
		return err
	}

	// If there are, then we use the contributor stats API to figure out when
	// the user's first commit in the repository was.
	if len(commits) > 0 {
		stats, response, err := githubClient.Repositories.ListContributorsStats(repoOwnerLogin, repoName)
		if response.StatusCode == 202 {
			log.Infof(c, "Stats were not available for %s, will try again later", *repo.FullName)
			task, err := computeVintageFunc.Task(userId, userLogin, repoId, repoOwnerLogin, repoName)
			if err != nil {
				log.Errorf(c, "Could create delayed task for %s: %s", *repo.FullName, err.Error())
				return err
			}
			task.Delay = time.Second * 10
			taskqueue.Add(c, task, "")
			return nil
		}
		if err != nil {
			log.Errorf(c, "Could not load stats for repo %s: %s", *repo.FullName, err.Error())
			return err
		}
		for _, stat := range stats {
			if *stat.Author.ID == userId {
				for i := range stat.Weeks {
					weekTimestamp := stat.Weeks[i].Week.UTC()
					if weekTimestamp.Before(vintage) {
						vintage = weekTimestamp
					}
				}
				break
			}
		}
	}

	_, err = datastore.Put(c, getVintageKey(c, userId, repoId), &RepoVintage{
		UserId:  userId,
		RepoId:  repoId,
		Vintage: vintage,
	})
	if err != nil {
		log.Errorf(c, "Could save vintage for repo %s: %s", *repo.FullName, err.Error())
		return err
	}

	return nil
}

func init() {
	computeVintageFunc = delay.Func("computeVintage", computeVintage)
}

func fillVintages(c context.Context, user *github.User, repos []*Repo) error {
	if len(repos) > VintageChunkSize {
		for chunkStart := 0; chunkStart < len(repos); chunkStart += VintageChunkSize {
			chunkEnd := chunkStart + VintageChunkSize
			if chunkEnd > len(repos) {
				chunkEnd = len(repos)
			}
			err := fillVintages(c, user, repos[chunkStart:chunkEnd])
			if err != nil {
				return err
			}
		}
		return nil
	}
	keys := make([]*datastore.Key, len(repos))
	for i := range repos {
		keys[i] = getVintageKey(c, *user.ID, *repos[i].ID)
	}
	vintages := make([]*RepoVintage, len(repos))
	for i := range vintages {
		vintages[i] = new(RepoVintage)
	}
	err := datastore.GetMulti(c, keys, vintages)
	if err != nil {
		if errs, ok := err.(appengine.MultiError); ok {
			for i, err := range errs {
				if err == datastore.ErrNoSuchEntity {
					vintages[i] = nil
				} else if err != nil {
					log.Errorf(c, "%d/%s vintage fetch error: %s", i, *repos[i].FullName, err.Error())
					return err
				}
			}
		} else {
			return err
		}
	}
	for i := range vintages {
		repo := repos[i]
		vintage := vintages[i]
		if vintage != nil {
			if vintage.Vintage.Unix() != 0 {
				repo.Vintage = vintage.Vintage
			}
			continue
		}
		computeVintageFunc.Call(c, *user.ID, *user.Login, *repo.ID, *repo.Owner.Login, *repo.Name)
	}
	return nil
}

type Repos struct {
	AllRepos       []*Repo
	UserRepos      []*Repo
	OtherUserRepos []*UserRepos
	OldestVintage  time.Time
}

func (repos *Repos) Redact() {
	for _, repo := range repos.UserRepos {
		*repo.HTMLURL = "https://redacted"
		*repo.FullName = "redacted/redacted"
	}
	for _, otherUserRepos := range repos.OtherUserRepos {
		*otherUserRepos.User.Login = "redacted"
		*otherUserRepos.User.AvatarURL = "https://redacted"
		for _, repo := range otherUserRepos.Repos {
			*repo.HTMLURL = "https://redacted"
			*repo.FullName = "redacted/redacted"
		}
	}
}

type Repo struct {
	*github.Repository
	Vintage         time.Time
	IncludeInDigest bool
}

func newRepo(githubRepo *github.Repository, account *Account) *Repo {
	return &Repo{
		Repository:      githubRepo,
		Vintage:         githubRepo.CreatedAt.UTC(),
		IncludeInDigest: !account.IsRepoIdExcluded(*githubRepo.ID),
	}
}

func (repo *Repo) TypeAsOcticonName() string {
	if *repo.Fork {
		return "repo-forked"
	}
	if *repo.Private {
		return "lock"
	}
	return "repo"
}

func (repo *Repo) TypeAsClassName() string {
	if *repo.Fork {
		return "fork"
	}
	if *repo.Private {
		return "private"
	}
	return ""
}

func (repo *Repo) DisplayVintage() string {
	return repo.Vintage.Format(VintageDateFormat)
}

type UserRepos struct {
	User  *github.User
	Repos []*Repo
}

func getRepos(c context.Context, githubClient *github.Client, account *Account, user *github.User) (*Repos, error) {
	clientUserRepos := make([]github.Repository, 0)
	page := 1
	for {
		pageClientUserRepos, response, err := githubClient.Repositories.List(
			// The username parameter must be left blank so that we can get all
			// of the repositories the user has access to, not just ones that
			// they own.
			"",
			&github.RepositoryListOptions{
				ListOptions: github.ListOptions{
					Page:    page,
					PerPage: 100,
				},
			})
		if err != nil {
			return nil, err
		}
		clientUserRepos = append(clientUserRepos, pageClientUserRepos...)
		if response.NextPage == 0 {
			break
		}
		page = response.NextPage
	}

	repos := &Repos{}
	repos.UserRepos = make([]*Repo, 0, len(clientUserRepos))
	repos.OtherUserRepos = make([]*UserRepos, 0)
	for i := range clientUserRepos {
		ownerID := *clientUserRepos[i].Owner.ID
		if ownerID == *user.ID {
			repos.UserRepos = append(repos.UserRepos, newRepo(&clientUserRepos[i], account))
		} else {
			var userRepos *UserRepos
			for j := range repos.OtherUserRepos {
				if *repos.OtherUserRepos[j].User.ID == ownerID {
					userRepos = repos.OtherUserRepos[j]
					break
				}
			}
			if userRepos == nil {
				userRepos = &UserRepos{
					User:  clientUserRepos[i].Owner,
					Repos: make([]*Repo, 0),
				}
				repos.OtherUserRepos = append(repos.OtherUserRepos, userRepos)
			}
			userRepos.Repos = append(userRepos.Repos, newRepo(&clientUserRepos[i], account))
		}
	}

	repos.AllRepos = make([]*Repo, 0, len(clientUserRepos))
	repos.AllRepos = append(repos.AllRepos, repos.UserRepos...)
	for _, userRepos := range repos.OtherUserRepos {
		repos.AllRepos = append(repos.AllRepos, userRepos.Repos...)
	}

	err := fillVintages(c, user, repos.AllRepos)
	if err != nil {
		return nil, err
	}

	repos.OldestVintage = time.Now().UTC()
	for _, repo := range repos.AllRepos {
		repoVintage := repo.Vintage
		if repoVintage.Before(repos.OldestVintage) {
			repos.OldestVintage = repoVintage
		}
	}

	return repos, nil
}
<|endoftext|>


<commit_before>package auth

import (
	"crypto/sha1"
	"encoding/json"
	"fmt"
	. "github.com/olsdavis/goelan/player"
	"io/ioutil"
	"net/http"
	"strings"
	"github.com/olsdavis/goelan/util"
)

const (
	BASE_URL = "https://sessionserver.mojang.com/session/minecraft/hasJoined?username=%v&serverId=%v"
)

// Authenticates the current player.
func Auth(username string, sharedSecret, publicKey []byte) (*PlayerProfile, error) {
	digest := authDigest(sharedSecret, publicKey)
	resp, err := http.Get(fmt.Sprintf(BASE_URL, username, digest))
	if err != nil {
		return nil, err
	}
	defer resp.Body.Close()
	content, err := ioutil.ReadAll(resp.Body)
	if err != nil {
		return nil, err
	}
	var profile PlayerProfile
	err = json.Unmarshal(content, &profile)
	if err != nil {
		return nil, err
	}
	realUniqueId, err := util.StringToUUID(util.ToHypenUUID(profile.UUID))
	if err != nil {
		panic(err)
	}
	profile.RealUUID = realUniqueId
	return &profile, nil
}

// THE FOLLOWING CODE HAS BEEN TAKEN FROM: https://gist.github.com/toqueteos/5372776

// AuthDigest computes a special SHA-1 digest required for Minecraft web
// authentication on Premium servers (online-mode=true).
// Source: http://wiki.vg/Protocol_Encryption#Server
//
// Also many, many thanks to SirCmpwn and his wonderful gist (C#):
// https://gist.github.com/SirCmpwn/404223052379e82f91e6
func authDigest(sharedSecret, publicKey []byte) string {
	h := sha1.New()
	h.Write(sharedSecret)
	h.Write(publicKey)
	hash := h.Sum(nil)

	// Check for negative hashes
	negative := (hash[0] & 0x80) == 0x80
	if negative {
		hash = twosComplement(hash)
	}

	// Trim away zeroes
	res := strings.TrimLeft(fmt.Sprintf("%x", hash), "0")
	if negative {
		res = "-" + res
	}

	return res
}

// little endian
func twosComplement(p []byte) []byte {
	carry := true
	for i := len(p) - 1; i >= 0; i-- {
		p[i] = byte(^p[i])
		if carry {
			carry = p[i] == 0xff
			p[i]++
		}
	}
	return p
}
<commit_msg>:hammer: Quick fix, Go conventions.<commit_after>package auth

import (
	"crypto/sha1"
	"encoding/json"
	"fmt"
	. "github.com/olsdavis/goelan/player"
	"io/ioutil"
	"net/http"
	"strings"
	"github.com/olsdavis/goelan/util"
)

const (
	baseUrl = "https://sessionserver.mojang.com/session/minecraft/hasJoined?username=%v&serverId=%v"
)

// Authenticates the current player.
func Auth(username string, sharedSecret, publicKey []byte) (*PlayerProfile, error) {
	digest := authDigest(sharedSecret, publicKey)
	resp, err := http.Get(fmt.Sprintf(baseUrl, username, digest))
	if err != nil {
		return nil, err
	}
	defer resp.Body.Close()
	content, err := ioutil.ReadAll(resp.Body)
	if err != nil {
		return nil, err
	}
	var profile PlayerProfile
	err = json.Unmarshal(content, &profile)
	if err != nil {
		return nil, err
	}
	realUniqueId, err := util.StringToUUID(util.ToHypenUUID(profile.UUID))
	if err != nil {
		panic(err)
	}
	profile.RealUUID = realUniqueId
	return &profile, nil
}

// THE FOLLOWING CODE HAS BEEN TAKEN FROM: https://gist.github.com/toqueteos/5372776

// AuthDigest computes a special SHA-1 digest required for Minecraft web
// authentication on Premium servers (online-mode=true).
// Source: http://wiki.vg/Protocol_Encryption#Server
//
// Also many, many thanks to SirCmpwn and his wonderful gist (C#):
// https://gist.github.com/SirCmpwn/404223052379e82f91e6
func authDigest(sharedSecret, publicKey []byte) string {
	h := sha1.New()
	h.Write(sharedSecret)
	h.Write(publicKey)
	hash := h.Sum(nil)

	// Check for negative hashes
	negative := (hash[0] & 0x80) == 0x80
	if negative {
		hash = twosComplement(hash)
	}

	// Trim away zeroes
	res := strings.TrimLeft(fmt.Sprintf("%x", hash), "0")
	if negative {
		res = "-" + res
	}

	return res
}

// little endian
func twosComplement(p []byte) []byte {
	carry := true
	for i := len(p) - 1; i >= 0; i-- {
		p[i] = byte(^p[i])
		if carry {
			carry = p[i] == 0xff
			p[i]++
		}
	}
	return p
}
<|endoftext|>


<commit_before>package auth

import (
	"os"

	"github.com/dgrijalva/jwt-go"
	"github.com/generationtux/brizo/database"
	"github.com/jinzhu/gorm"
)

// User represents a Brizo user
type User struct {
	database.Model
	Username       string `gorm:"not null;unique_index"`
	Name           string
	Email          string
	GithubUsername string
	GithubToken    string
}

// CreateUser creates a Brizo specific user without forcing any validation on
// the input
func CreateUser(db *gorm.DB, user *User) (bool, error) {
	result := db.Create(&user)

	return result.RowsAffected == 1, result.Error
}

// UpdateUser updates an existing Brizo user based on his/her username
func UpdateUser(db *gorm.DB, user *User) (bool, error) {
	result := db.Model(user).Where("username = ?", user.Username).UpdateColumns(user)

	return result.RowsAffected == 1, result.Error
}

// create JWT Token
func CreateToken(user *User) (string, error) {
	token := jwt.NewWithClaims(tokenSigningMethod(os.Getenv("JWT_ALGO")), jwt.MapClaims{
		"username": user.Username,
		"name":     user.Name,
		"email":    user.Email,
	})

	tokenString, err := token.SignedString(os.Getenv("JWT_SECRET"))

	return tokenString, err
}

func tokenSigningMethod(method string) string {
	switch method {
	case "HS256":
		return jwt.SigningMethodHS256
	case "HS384":
		return jwt.SigningMethodHS384
	case "HS512":
		return jwt.SigningMethodHS512
	default:
		return nil
	}
}
<commit_msg>fixed tokenSigningMethod return type<commit_after>package auth

import (
	"os"

	"github.com/dgrijalva/jwt-go"
	"github.com/generationtux/brizo/database"
	"github.com/jinzhu/gorm"
)

// User represents a Brizo user
type User struct {
	database.Model
	Username       string `gorm:"not null;unique_index"`
	Name           string
	Email          string
	GithubUsername string
	GithubToken    string
}

// CreateUser creates a Brizo specific user without forcing any validation on
// the input
func CreateUser(db *gorm.DB, user *User) (bool, error) {
	result := db.Create(&user)

	return result.RowsAffected == 1, result.Error
}

// UpdateUser updates an existing Brizo user based on his/her username
func UpdateUser(db *gorm.DB, user *User) (bool, error) {
	result := db.Model(user).Where("username = ?", user.Username).UpdateColumns(user)

	return result.RowsAffected == 1, result.Error
}

// create JWT Token
func CreateToken(user *User) (string, error) {
	token := jwt.NewWithClaims(tokenSigningMethod(os.Getenv("JWT_ALGO")), jwt.MapClaims{
		"username": user.Username,
		"name":     user.Name,
		"email":    user.Email,
	})

	tokenString, err := token.SignedString(os.Getenv("JWT_SECRET"))

	return tokenString, err
}

func tokenSigningMethod(method string) jwt.SigningMethod {
	switch method {
	case "HS256":
		return jwt.SigningMethodHS256
	case "HS384":
		return jwt.SigningMethodHS384
	case "HS512":
		return jwt.SigningMethodHS512
	default:
		return nil
	}
}
<|endoftext|>


<commit_before>// Copyright 2014 Google Inc. All rights reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

// Package base provides common utilities for other stenographer libraries.
package base

import (
	"container/heap"
	"flag"
	"fmt"
	"io"
	"log"
	"sort"
	"sync"
	"syscall"

	"code.google.com/p/gopacket"
	"code.google.com/p/gopacket/layers"
	"code.google.com/p/gopacket/pcapgo"
	"golang.org/x/net/context"
)

var VerboseLogging = flag.Int("v", -1, "log many verbose logs")

// V provides verbose logging which can be turned on/off with the -v flag.
func V(level int, fmt string, args ...interface{}) {
	if *VerboseLogging >= level {
		log.Printf(fmt, args...)
	}
}

// Packet is a single packet with its metadata.
type Packet struct {
	Data                 []byte // The actual bytes that make up the packet
	gopacket.CaptureInfo        // Metadata about when/how the packet was captured
}

// PacketChan provides an async method for passing multiple ordered packets
// between goroutines.
type PacketChan struct {
	mu   sync.Mutex
	c    chan *Packet
	err  error
	done chan struct{}
}

// Receive provides the channel from which to read packets.  It always
// returns the same channel.
func (p *PacketChan) Receive() <-chan *Packet { return p.c }

// Send sends a single packet on the channel to the receiver.
func (p *PacketChan) Send(pkt *Packet) { p.c <- pkt }

// Close closes the sending channel and sets the PacketChan's error based
// in its input.
func (p *PacketChan) Close(err error) {
	p.mu.Lock()
	p.err = err
	p.mu.Unlock()
	close(p.c)
	close(p.done)
}

// Done returns a channel that is closed when this packet channel is complete.
func (p *PacketChan) Done() <-chan struct{} {
	return p.done
}

// NewPacketChan returns a new PacketChan channel for passing packets around.
func NewPacketChan(buffer int) *PacketChan {
	return &PacketChan{
		c:    make(chan *Packet, buffer),
		done: make(chan struct{}),
	}
}

// Discard discards all remaining packets on the receiving end.  If you stop
// using the channel before reading all packets, you must call this function.
// It's a good idea to defer this regardless.
func (p *PacketChan) Discard() {
	go func() {
		discarded := 0
		for _ = range p.c {
			discarded++
		}
		if discarded > 0 {
			V(2, "discarded %v", discarded)
		}
	}()
}

// Err gets the current error for the channel, if any exists.  This may be
// called during Next(), but if an error occurs it may only be set after Next()
// returns false the first time.
func (p *PacketChan) Err() error {
	p.mu.Lock()
	defer p.mu.Unlock()
	return p.err
}

// indexedPacket is used internally by MergePacketChans.
type indexedPacket struct {
	*Packet
	i int
}

// packetHeap is used internally by MergePacketChans.
type packetHeap []indexedPacket

func (p packetHeap) Len() int            { return len(p) }
func (p packetHeap) Swap(i, j int)       { p[i], p[j] = p[j], p[i] }
func (p packetHeap) Less(i, j int) bool  { return p[i].Timestamp.Before(p[j].Timestamp) }
func (p *packetHeap) Push(x interface{}) { *p = append(*p, x.(indexedPacket)) }
func (p *packetHeap) Pop() (x interface{}) {
	index := len(*p) - 1
	*p, x = (*p)[:index], (*p)[index]
	return
}

// ConcatPacketChans concatenates packet chans in order.
func ConcatPacketChans(ctx context.Context, in <-chan *PacketChan) *PacketChan {
	out := NewPacketChan(100)
	go func() {
		for c := range in {
			c := c
			defer c.Discard()
		L:
			for c.Err() == nil {
				select {
				case pkt := <-c.Receive():
					if pkt == nil {
						break L
					}
					out.Send(pkt)
				case <-ctx.Done():
					out.Close(ctx.Err())
					return
				}
			}
			if err := c.Err(); err != nil {
				out.Close(err)
				return
			}
		}
		out.Close(nil)
	}()
	return out
}

// MergePacketChans merges an incoming set of packet chans, each sorted by
// time, returning a new single packet chan that's also sorted by time.
func MergePacketChans(ctx context.Context, in []*PacketChan) *PacketChan {
	out := NewPacketChan(100)
	go func() {
		count := 0
		defer func() {
			V(1, "merged %d streams for %d total packets", len(in), count)
		}()
		var h packetHeap
		for i := range in {
			defer in[i].Discard()
		}
		for i, c := range in {
			select {
			case pkt := <-c.Receive():
				if pkt != nil {
					heap.Push(&h, indexedPacket{Packet: pkt, i: i})
				}
				if err := c.Err(); err != nil {
					out.Close(err)
					return
				}
			case <-ctx.Done():
				out.Close(ctx.Err())
				return
			}
		}
		for h.Len() > 0 && !ContextDone(ctx) {
			p := heap.Pop(&h).(indexedPacket)
			count++
			if pkt := <-in[p.i].Receive(); pkt != nil {
				heap.Push(&h, indexedPacket{Packet: pkt, i: p.i})
			}
			out.c <- p.Packet
			if err := in[p.i].Err(); err != nil {
				out.Close(err)
				return
			}
		}
		out.Close(ctx.Err())
	}()
	return out
}

// Positions detail the offsets of packets within a blockfile.
type Positions []int64

var (
	AllPositions = Positions{-1}
	NoPositions  = Positions{}
)

func (p Positions) IsAllPositions() bool {
	return len(p) == 1 && p[0] == -1
}

func (a Positions) Less(i, j int) bool {
	return a[i] < a[j]
}
func (a Positions) Swap(i, j int) {
	a[i], a[j] = a[j], a[i]
}
func (a Positions) Len() int {
	return len(a)
}
func (a Positions) Sort() {
	sort.Sort(a)
}

// Union returns the union of a and b.  a and b must be sorted in advance.
// Returned slice will be sorted.
func (a Positions) Union(b Positions) (out Positions) {
	switch {
	case a.IsAllPositions():
		return a
	case b.IsAllPositions():
		return b
	case len(a) == 0:
		return b
	case len(b) == 0:
		return a
	}
	out = make(Positions, 0, len(a)+len(b)/2)
	ib := 0
	for _, pos := range a {
		for ib < len(b) && b[ib] < pos {
			out = append(out, b[ib])
			ib++
		}
		if ib < len(b) && b[ib] == pos {
			ib++
		}
		out = append(out, pos)
	}
	out = append(out, b[ib:]...)
	return out
}

// Intersect returns the intersection of a and b.  a and b must be sorted in
// advance.  Returned slice will be sorted.
func (a Positions) Intersect(b Positions) (out Positions) {
	switch {
	case a.IsAllPositions():
		return b
	case b.IsAllPositions():
		return a
	case len(a) == 0:
		return a
	case len(b) == 0:
		return b
	}
	out = make(Positions, 0, len(a)/2)
	ib := 0
	for _, pos := range a {
		for ib < len(b) && b[ib] < pos {
			ib++
		}
		if ib < len(b) && b[ib] == pos {
			out = append(out, pos)
			ib++
		}
	}
	return out
}

func PathDiskFreePercentage(path string) (int, error) {
	var stat syscall.Statfs_t
	if err := syscall.Statfs(path, &stat); err != nil {
		return 0, err
	}
	return int(100 * stat.Bfree / stat.Blocks), nil
}

// snapLen is the max packet size we'll return in pcap files to users.
const snapLen = 65536

// PacketsToFile writes all packets from 'in' to 'out', writing out all packets
// in a valid PCAP file format.
func PacketsToFile(in *PacketChan, out io.Writer) error {
	w := pcapgo.NewWriter(out)
	w.WriteFileHeader(snapLen, layers.LinkTypeEthernet)
	count := 0
	defer in.Discard()
	for p := range in.Receive() {
		if len(p.Data) > snapLen {
			p.Data = p.Data[:snapLen]
		}
		if err := w.WritePacket(p.CaptureInfo, p.Data); err != nil {
			// This can happen if our pipe is broken, and we don't want to blow stack
			// traces all over our users when that happens, so Error/Exit instead of
			// Fatal.
			return fmt.Errorf("error writing packet: %v", err)
		}
		count++
	}
	return in.Err()
}

// ContextDone returns true if a context is complete.
func ContextDone(ctx context.Context) bool {
	// There's two ways we could do this:  by checking ctx.Done or by
	// seeing if ctx.Err != nil.  The latter, though, uses a single
	// exclusive mutex, so when the context is shared by a ton of
	// goroutines, it can actually block things quite a bit.  Checking
	// ctx.Done is much more scalable across multiple goroutines.
	select {
	case <-ctx.Done():
		return true
	default:
		return false
	}
}
<commit_msg>Use bavail instead of bfree when calculating free disk space.<commit_after>// Copyright 2014 Google Inc. All rights reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

// Package base provides common utilities for other stenographer libraries.
package base

import (
	"container/heap"
	"flag"
	"fmt"
	"io"
	"log"
	"sort"
	"sync"
	"syscall"

	"code.google.com/p/gopacket"
	"code.google.com/p/gopacket/layers"
	"code.google.com/p/gopacket/pcapgo"
	"golang.org/x/net/context"
)

var VerboseLogging = flag.Int("v", -1, "log many verbose logs")

// V provides verbose logging which can be turned on/off with the -v flag.
func V(level int, fmt string, args ...interface{}) {
	if *VerboseLogging >= level {
		log.Printf(fmt, args...)
	}
}

// Packet is a single packet with its metadata.
type Packet struct {
	Data                 []byte // The actual bytes that make up the packet
	gopacket.CaptureInfo        // Metadata about when/how the packet was captured
}

// PacketChan provides an async method for passing multiple ordered packets
// between goroutines.
type PacketChan struct {
	mu   sync.Mutex
	c    chan *Packet
	err  error
	done chan struct{}
}

// Receive provides the channel from which to read packets.  It always
// returns the same channel.
func (p *PacketChan) Receive() <-chan *Packet { return p.c }

// Send sends a single packet on the channel to the receiver.
func (p *PacketChan) Send(pkt *Packet) { p.c <- pkt }

// Close closes the sending channel and sets the PacketChan's error based
// in its input.
func (p *PacketChan) Close(err error) {
	p.mu.Lock()
	p.err = err
	p.mu.Unlock()
	close(p.c)
	close(p.done)
}

// Done returns a channel that is closed when this packet channel is complete.
func (p *PacketChan) Done() <-chan struct{} {
	return p.done
}

// NewPacketChan returns a new PacketChan channel for passing packets around.
func NewPacketChan(buffer int) *PacketChan {
	return &PacketChan{
		c:    make(chan *Packet, buffer),
		done: make(chan struct{}),
	}
}

// Discard discards all remaining packets on the receiving end.  If you stop
// using the channel before reading all packets, you must call this function.
// It's a good idea to defer this regardless.
func (p *PacketChan) Discard() {
	go func() {
		discarded := 0
		for _ = range p.c {
			discarded++
		}
		if discarded > 0 {
			V(2, "discarded %v", discarded)
		}
	}()
}

// Err gets the current error for the channel, if any exists.  This may be
// called during Next(), but if an error occurs it may only be set after Next()
// returns false the first time.
func (p *PacketChan) Err() error {
	p.mu.Lock()
	defer p.mu.Unlock()
	return p.err
}

// indexedPacket is used internally by MergePacketChans.
type indexedPacket struct {
	*Packet
	i int
}

// packetHeap is used internally by MergePacketChans.
type packetHeap []indexedPacket

func (p packetHeap) Len() int            { return len(p) }
func (p packetHeap) Swap(i, j int)       { p[i], p[j] = p[j], p[i] }
func (p packetHeap) Less(i, j int) bool  { return p[i].Timestamp.Before(p[j].Timestamp) }
func (p *packetHeap) Push(x interface{}) { *p = append(*p, x.(indexedPacket)) }
func (p *packetHeap) Pop() (x interface{}) {
	index := len(*p) - 1
	*p, x = (*p)[:index], (*p)[index]
	return
}

// ConcatPacketChans concatenates packet chans in order.
func ConcatPacketChans(ctx context.Context, in <-chan *PacketChan) *PacketChan {
	out := NewPacketChan(100)
	go func() {
		for c := range in {
			c := c
			defer c.Discard()
		L:
			for c.Err() == nil {
				select {
				case pkt := <-c.Receive():
					if pkt == nil {
						break L
					}
					out.Send(pkt)
				case <-ctx.Done():
					out.Close(ctx.Err())
					return
				}
			}
			if err := c.Err(); err != nil {
				out.Close(err)
				return
			}
		}
		out.Close(nil)
	}()
	return out
}

// MergePacketChans merges an incoming set of packet chans, each sorted by
// time, returning a new single packet chan that's also sorted by time.
func MergePacketChans(ctx context.Context, in []*PacketChan) *PacketChan {
	out := NewPacketChan(100)
	go func() {
		count := 0
		defer func() {
			V(1, "merged %d streams for %d total packets", len(in), count)
		}()
		var h packetHeap
		for i := range in {
			defer in[i].Discard()
		}
		for i, c := range in {
			select {
			case pkt := <-c.Receive():
				if pkt != nil {
					heap.Push(&h, indexedPacket{Packet: pkt, i: i})
				}
				if err := c.Err(); err != nil {
					out.Close(err)
					return
				}
			case <-ctx.Done():
				out.Close(ctx.Err())
				return
			}
		}
		for h.Len() > 0 && !ContextDone(ctx) {
			p := heap.Pop(&h).(indexedPacket)
			count++
			if pkt := <-in[p.i].Receive(); pkt != nil {
				heap.Push(&h, indexedPacket{Packet: pkt, i: p.i})
			}
			out.c <- p.Packet
			if err := in[p.i].Err(); err != nil {
				out.Close(err)
				return
			}
		}
		out.Close(ctx.Err())
	}()
	return out
}

// Positions detail the offsets of packets within a blockfile.
type Positions []int64

var (
	AllPositions = Positions{-1}
	NoPositions  = Positions{}
)

func (p Positions) IsAllPositions() bool {
	return len(p) == 1 && p[0] == -1
}

func (a Positions) Less(i, j int) bool {
	return a[i] < a[j]
}
func (a Positions) Swap(i, j int) {
	a[i], a[j] = a[j], a[i]
}
func (a Positions) Len() int {
	return len(a)
}
func (a Positions) Sort() {
	sort.Sort(a)
}

// Union returns the union of a and b.  a and b must be sorted in advance.
// Returned slice will be sorted.
func (a Positions) Union(b Positions) (out Positions) {
	switch {
	case a.IsAllPositions():
		return a
	case b.IsAllPositions():
		return b
	case len(a) == 0:
		return b
	case len(b) == 0:
		return a
	}
	out = make(Positions, 0, len(a)+len(b)/2)
	ib := 0
	for _, pos := range a {
		for ib < len(b) && b[ib] < pos {
			out = append(out, b[ib])
			ib++
		}
		if ib < len(b) && b[ib] == pos {
			ib++
		}
		out = append(out, pos)
	}
	out = append(out, b[ib:]...)
	return out
}

// Intersect returns the intersection of a and b.  a and b must be sorted in
// advance.  Returned slice will be sorted.
func (a Positions) Intersect(b Positions) (out Positions) {
	switch {
	case a.IsAllPositions():
		return b
	case b.IsAllPositions():
		return a
	case len(a) == 0:
		return a
	case len(b) == 0:
		return b
	}
	out = make(Positions, 0, len(a)/2)
	ib := 0
	for _, pos := range a {
		for ib < len(b) && b[ib] < pos {
			ib++
		}
		if ib < len(b) && b[ib] == pos {
			out = append(out, pos)
			ib++
		}
	}
	return out
}

func PathDiskFreePercentage(path string) (int, error) {
	var stat syscall.Statfs_t
	if err := syscall.Statfs(path, &stat); err != nil {
		return 0, err
	}
	return int(100 * stat.Bavail / stat.Blocks), nil
}

// snapLen is the max packet size we'll return in pcap files to users.
const snapLen = 65536

// PacketsToFile writes all packets from 'in' to 'out', writing out all packets
// in a valid PCAP file format.
func PacketsToFile(in *PacketChan, out io.Writer) error {
	w := pcapgo.NewWriter(out)
	w.WriteFileHeader(snapLen, layers.LinkTypeEthernet)
	count := 0
	defer in.Discard()
	for p := range in.Receive() {
		if len(p.Data) > snapLen {
			p.Data = p.Data[:snapLen]
		}
		if err := w.WritePacket(p.CaptureInfo, p.Data); err != nil {
			// This can happen if our pipe is broken, and we don't want to blow stack
			// traces all over our users when that happens, so Error/Exit instead of
			// Fatal.
			return fmt.Errorf("error writing packet: %v", err)
		}
		count++
	}
	return in.Err()
}

// ContextDone returns true if a context is complete.
func ContextDone(ctx context.Context) bool {
	// There's two ways we could do this:  by checking ctx.Done or by
	// seeing if ctx.Err != nil.  The latter, though, uses a single
	// exclusive mutex, so when the context is shared by a ton of
	// goroutines, it can actually block things quite a bit.  Checking
	// ctx.Done is much more scalable across multiple goroutines.
	select {
	case <-ctx.Done():
		return true
	default:
		return false
	}
}
<|endoftext|>


<commit_before>// Copyright 2013 The Go Authors.  All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

// Package blog implements a web server for articles written in present format.
package blog

import (
	"bytes"
	"encoding/json"
	"encoding/xml"
	"fmt"
	"html/template"
	"log"
	"net/http"
	"os"
	"path/filepath"
	"regexp"
	"sort"
	"strings"
	"time"

	"code.google.com/p/go.tools/blog/atom"
	"code.google.com/p/go.tools/present"

	_ "code.google.com/p/go.tools/playground"
)

var validJSONPFunc = regexp.MustCompile(`(?i)^[a-z_][a-z0-9_.]*$`)

// Config specifies Server configuration values.
type Config struct {
	ContentPath  string // Relative or absolute location of article files and related content.
	TemplatePath string // Relative or absolute location of template files.

	BaseURL  string // Absolute base URL (for permalinks; no trailing slash).
	BasePath string // Base URL path relative to server root (no trailing slash).
	GodocURL string // The base URL of godoc (for menu bar; no trailing slash).
	Hostname string // Server host name, used for rendering ATOM feeds.

	HomeArticles int // Articles to display on the home page.
	FeedArticles int // Articles to include in Atom and JSON feeds.

	PlayEnabled bool
}

// Doc represents an article adorned with presentation data.
type Doc struct {
	*present.Doc
	Permalink string        // Canonical URL for this document.
	Path      string        // Path relative to server root (including base).
	HTML      template.HTML // rendered article

	Related      []*Doc
	Newer, Older *Doc
}

// Server implements an http.Handler that serves blog articles.
type Server struct {
	cfg      Config
	docs     []*Doc
	tags     []string
	docPaths map[string]*Doc // key is path without BasePath.
	docTags  map[string][]*Doc
	template struct {
		home, index, article, doc *template.Template
	}
	atomFeed []byte // pre-rendered Atom feed
	jsonFeed []byte // pre-rendered JSON feed
	content  http.Handler
}

// NewServer constructs a new Server using the specified config.
func NewServer(cfg Config) (*Server, error) {
	present.PlayEnabled = cfg.PlayEnabled

	root := filepath.Join(cfg.TemplatePath, "root.tmpl")
	parse := func(name string) (*template.Template, error) {
		t := template.New("").Funcs(funcMap)
		return t.ParseFiles(root, filepath.Join(cfg.TemplatePath, name))
	}

	s := &Server{cfg: cfg}

	// Parse templates.
	var err error
	s.template.home, err = parse("home.tmpl")
	if err != nil {
		return nil, err
	}
	s.template.index, err = parse("index.tmpl")
	if err != nil {
		return nil, err
	}
	s.template.article, err = parse("article.tmpl")
	if err != nil {
		return nil, err
	}
	p := present.Template().Funcs(funcMap)
	s.template.doc, err = p.ParseFiles(filepath.Join(cfg.TemplatePath, "doc.tmpl"))
	if err != nil {
		return nil, err
	}

	// Load content.
	err = s.loadDocs(filepath.Clean(cfg.ContentPath))
	if err != nil {
		return nil, err
	}

	err = s.renderAtomFeed()
	if err != nil {
		return nil, err
	}

	err = s.renderJSONFeed()
	if err != nil {
		return nil, err
	}

	// Set up content file server.
	s.content = http.FileServer(http.Dir(cfg.ContentPath))

	return s, nil
}

var funcMap = template.FuncMap{
	"sectioned": sectioned,
	"authors":   authors,
}

// sectioned returns true if the provided Doc contains more than one section.
// This is used to control whether to display the table of contents and headings.
func sectioned(d *present.Doc) bool {
	return len(d.Sections) > 1
}

// authors returns a comma-separated list of author names.
func authors(authors []present.Author) string {
	var b bytes.Buffer
	last := len(authors) - 1
	for i, a := range authors {
		if i > 0 {
			if i == last {
				b.WriteString(" and ")
			} else {
				b.WriteString(", ")
			}
		}
		b.WriteString(authorName(a))
	}
	return b.String()
}

// authorName returns the first line of the Author text: the author's name.
func authorName(a present.Author) string {
	el := a.TextElem()
	if len(el) == 0 {
		return ""
	}
	text, ok := el[0].(present.Text)
	if !ok || len(text.Lines) == 0 {
		return ""
	}
	return text.Lines[0]
}

// loadDocs reads all content from the provided file system root, renders all
// the articles it finds, adds them to the Server's docs field, computes the
// denormalized docPaths, docTags, and tags fields, and populates the various
// helper fields (Next, Previous, Related) for each Doc.
func (s *Server) loadDocs(root string) error {
	// Read content into docs field.
	const ext = ".article"
	fn := func(p string, info os.FileInfo, err error) error {
		if filepath.Ext(p) != ext {
			return nil
		}
		f, err := os.Open(p)
		if err != nil {
			return err
		}
		defer f.Close()
		d, err := present.Parse(f, p, 0)
		if err != nil {
			return err
		}
		html := new(bytes.Buffer)
		err = d.Render(html, s.template.doc)
		if err != nil {
			return err
		}
		p = p[len(root) : len(p)-len(ext)] // trim root and extension
		s.docs = append(s.docs, &Doc{
			Doc:       d,
			Path:      s.cfg.BasePath + p,
			Permalink: s.cfg.BaseURL + p,
			HTML:      template.HTML(html.String()),
		})
		return nil
	}
	err := filepath.Walk(root, fn)
	if err != nil {
		return err
	}
	sort.Sort(docsByTime(s.docs))

	// Pull out doc paths and tags and put in reverse-associating maps.
	s.docPaths = make(map[string]*Doc)
	s.docTags = make(map[string][]*Doc)
	for _, d := range s.docs {
		s.docPaths[strings.TrimPrefix(d.Path, s.cfg.BasePath)] = d
		for _, t := range d.Tags {
			s.docTags[t] = append(s.docTags[t], d)
		}
	}

	// Pull out unique sorted list of tags.
	for t := range s.docTags {
		s.tags = append(s.tags, t)
	}
	sort.Strings(s.tags)

	// Set up presentation-related fields, Newer, Older, and Related.
	for _, doc := range s.docs {
		// Newer, Older: docs adjacent to doc
		for i := range s.docs {
			if s.docs[i] != doc {
				continue
			}
			if i > 0 {
				doc.Newer = s.docs[i-1]
			}
			if i+1 < len(s.docs) {
				doc.Older = s.docs[i+1]
			}
			break
		}

		// Related: all docs that share tags with doc.
		related := make(map[*Doc]bool)
		for _, t := range doc.Tags {
			for _, d := range s.docTags[t] {
				if d != doc {
					related[d] = true
				}
			}
		}
		for d := range related {
			doc.Related = append(doc.Related, d)
		}
		sort.Sort(docsByTime(doc.Related))
	}

	return nil
}

// renderAtomFeed generates an XML Atom feed and stores it in the Server's
// atomFeed field.
func (s *Server) renderAtomFeed() error {
	var updated time.Time
	if len(s.docs) > 1 {
		updated = s.docs[0].Time
	}
	feed := atom.Feed{
		Title:   "The Go Programming Language Blog",
		ID:      "tag:" + s.cfg.Hostname + ",2013:" + s.cfg.Hostname,
		Updated: atom.Time(updated),
		Link: []atom.Link{{
			Rel:  "self",
			Href: s.cfg.BaseURL + "/feed.atom",
		}},
	}
	for i, doc := range s.docs {
		if i >= s.cfg.FeedArticles {
			break
		}
		e := &atom.Entry{
			Title: doc.Title,
			ID:    feed.ID + doc.Path,
			Link: []atom.Link{{
				Rel:  "alternate",
				Href: doc.Permalink,
			}},
			Published: atom.Time(doc.Time),
			Updated:   atom.Time(doc.Time),
			Summary: &atom.Text{
				Type: "html",
				Body: summary(doc),
			},
			Content: &atom.Text{
				Type: "html",
				Body: string(doc.HTML),
			},
			Author: &atom.Person{
				Name: authors(doc.Authors),
			},
		}
		feed.Entry = append(feed.Entry, e)
	}
	data, err := xml.Marshal(&feed)
	if err != nil {
		return err
	}
	s.atomFeed = data
	return nil
}

type jsonItem struct {
	Title   string
	Link    string
	Time    time.Time
	Summary string
	Content string
	Author  string
}

// renderJSONFeed generates a JSON feed and stores it in the Server's jsonFeed
// field.
func (s *Server) renderJSONFeed() error {
	var feed []jsonItem
	for i, doc := range s.docs {
		if i >= s.cfg.FeedArticles {
			break
		}
		item := jsonItem{
			Title:   doc.Title,
			Link:    doc.Permalink,
			Time:    doc.Time,
			Summary: summary(doc),
			Content: string(doc.HTML),
			Author:  authors(doc.Authors),
		}
		feed = append(feed, item)
	}
	data, err := json.Marshal(feed)
	if err != nil {
		return err
	}
	s.jsonFeed = data
	return nil
}

// summary returns the first paragraph of text from the provided Doc.
func summary(d *Doc) string {
	if len(d.Sections) == 0 {
		return ""
	}
	for _, elem := range d.Sections[0].Elem {
		text, ok := elem.(present.Text)
		if !ok || text.Pre {
			// skip everything but non-text elements
			continue
		}
		var buf bytes.Buffer
		for _, s := range text.Lines {
			buf.WriteString(string(present.Style(s)))
			buf.WriteByte('\n')
		}
		return buf.String()
	}
	return ""
}

// rootData encapsulates data destined for the root template.
type rootData struct {
	Doc      *Doc
	BasePath string
	GodocURL string
	Data     interface{}
}

// ServeHTTP serves the front, index, and article pages
// as well as the ATOM and JSON feeds.
func (s *Server) ServeHTTP(w http.ResponseWriter, r *http.Request) {
	var (
		d = rootData{BasePath: s.cfg.BasePath, GodocURL: s.cfg.GodocURL}
		t *template.Template
	)
	switch p := strings.TrimPrefix(r.URL.Path, s.cfg.BasePath); p {
	case "/":
		d.Data = s.docs
		if len(s.docs) > s.cfg.HomeArticles {
			d.Data = s.docs[:s.cfg.HomeArticles]
		}
		t = s.template.home
	case "/index":
		d.Data = s.docs
		t = s.template.index
	case "/feed.atom", "/feeds/posts/default":
		w.Header().Set("Content-type", "application/atom+xml; charset=utf-8")
		w.Write(s.atomFeed)
		return
	case "/.json":
		if p := r.FormValue("jsonp"); validJSONPFunc.MatchString(p) {
			w.Header().Set("Content-type", "application/javascript; charset=utf-8")
			fmt.Fprintf(w, "%v(%s)", p, s.jsonFeed)
			return
		}
		w.Header().Set("Content-type", "application/json; charset=utf-8")
		w.Write(s.jsonFeed)
		return
	default:
		doc, ok := s.docPaths[p]
		if !ok {
			// Not a doc; try to just serve static content.
			s.content.ServeHTTP(w, r)
			return
		}
		d.Doc = doc
		t = s.template.article
	}
	err := t.ExecuteTemplate(w, "root", d)
	if err != nil {
		log.Println(err)
	}
}

// docsByTime implements sort.Interface, sorting Docs by their Time field.
type docsByTime []*Doc

func (s docsByTime) Len() int           { return len(s) }
func (s docsByTime) Swap(i, j int)      { s[i], s[j] = s[j], s[i] }
func (s docsByTime) Less(i, j int) bool { return s[i].Time.After(s[j].Time) }
<commit_msg>go.tools/blog: remove playground package import<commit_after>// Copyright 2013 The Go Authors.  All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

// Package blog implements a web server for articles written in present format.
package blog

import (
	"bytes"
	"encoding/json"
	"encoding/xml"
	"fmt"
	"html/template"
	"log"
	"net/http"
	"os"
	"path/filepath"
	"regexp"
	"sort"
	"strings"
	"time"

	"code.google.com/p/go.tools/blog/atom"
	"code.google.com/p/go.tools/present"
)

var validJSONPFunc = regexp.MustCompile(`(?i)^[a-z_][a-z0-9_.]*$`)

// Config specifies Server configuration values.
type Config struct {
	ContentPath  string // Relative or absolute location of article files and related content.
	TemplatePath string // Relative or absolute location of template files.

	BaseURL  string // Absolute base URL (for permalinks; no trailing slash).
	BasePath string // Base URL path relative to server root (no trailing slash).
	GodocURL string // The base URL of godoc (for menu bar; no trailing slash).
	Hostname string // Server host name, used for rendering ATOM feeds.

	HomeArticles int // Articles to display on the home page.
	FeedArticles int // Articles to include in Atom and JSON feeds.

	PlayEnabled bool
}

// Doc represents an article adorned with presentation data.
type Doc struct {
	*present.Doc
	Permalink string        // Canonical URL for this document.
	Path      string        // Path relative to server root (including base).
	HTML      template.HTML // rendered article

	Related      []*Doc
	Newer, Older *Doc
}

// Server implements an http.Handler that serves blog articles.
type Server struct {
	cfg      Config
	docs     []*Doc
	tags     []string
	docPaths map[string]*Doc // key is path without BasePath.
	docTags  map[string][]*Doc
	template struct {
		home, index, article, doc *template.Template
	}
	atomFeed []byte // pre-rendered Atom feed
	jsonFeed []byte // pre-rendered JSON feed
	content  http.Handler
}

// NewServer constructs a new Server using the specified config.
func NewServer(cfg Config) (*Server, error) {
	present.PlayEnabled = cfg.PlayEnabled

	root := filepath.Join(cfg.TemplatePath, "root.tmpl")
	parse := func(name string) (*template.Template, error) {
		t := template.New("").Funcs(funcMap)
		return t.ParseFiles(root, filepath.Join(cfg.TemplatePath, name))
	}

	s := &Server{cfg: cfg}

	// Parse templates.
	var err error
	s.template.home, err = parse("home.tmpl")
	if err != nil {
		return nil, err
	}
	s.template.index, err = parse("index.tmpl")
	if err != nil {
		return nil, err
	}
	s.template.article, err = parse("article.tmpl")
	if err != nil {
		return nil, err
	}
	p := present.Template().Funcs(funcMap)
	s.template.doc, err = p.ParseFiles(filepath.Join(cfg.TemplatePath, "doc.tmpl"))
	if err != nil {
		return nil, err
	}

	// Load content.
	err = s.loadDocs(filepath.Clean(cfg.ContentPath))
	if err != nil {
		return nil, err
	}

	err = s.renderAtomFeed()
	if err != nil {
		return nil, err
	}

	err = s.renderJSONFeed()
	if err != nil {
		return nil, err
	}

	// Set up content file server.
	s.content = http.FileServer(http.Dir(cfg.ContentPath))

	return s, nil
}

var funcMap = template.FuncMap{
	"sectioned": sectioned,
	"authors":   authors,
}

// sectioned returns true if the provided Doc contains more than one section.
// This is used to control whether to display the table of contents and headings.
func sectioned(d *present.Doc) bool {
	return len(d.Sections) > 1
}

// authors returns a comma-separated list of author names.
func authors(authors []present.Author) string {
	var b bytes.Buffer
	last := len(authors) - 1
	for i, a := range authors {
		if i > 0 {
			if i == last {
				b.WriteString(" and ")
			} else {
				b.WriteString(", ")
			}
		}
		b.WriteString(authorName(a))
	}
	return b.String()
}

// authorName returns the first line of the Author text: the author's name.
func authorName(a present.Author) string {
	el := a.TextElem()
	if len(el) == 0 {
		return ""
	}
	text, ok := el[0].(present.Text)
	if !ok || len(text.Lines) == 0 {
		return ""
	}
	return text.Lines[0]
}

// loadDocs reads all content from the provided file system root, renders all
// the articles it finds, adds them to the Server's docs field, computes the
// denormalized docPaths, docTags, and tags fields, and populates the various
// helper fields (Next, Previous, Related) for each Doc.
func (s *Server) loadDocs(root string) error {
	// Read content into docs field.
	const ext = ".article"
	fn := func(p string, info os.FileInfo, err error) error {
		if filepath.Ext(p) != ext {
			return nil
		}
		f, err := os.Open(p)
		if err != nil {
			return err
		}
		defer f.Close()
		d, err := present.Parse(f, p, 0)
		if err != nil {
			return err
		}
		html := new(bytes.Buffer)
		err = d.Render(html, s.template.doc)
		if err != nil {
			return err
		}
		p = p[len(root) : len(p)-len(ext)] // trim root and extension
		s.docs = append(s.docs, &Doc{
			Doc:       d,
			Path:      s.cfg.BasePath + p,
			Permalink: s.cfg.BaseURL + p,
			HTML:      template.HTML(html.String()),
		})
		return nil
	}
	err := filepath.Walk(root, fn)
	if err != nil {
		return err
	}
	sort.Sort(docsByTime(s.docs))

	// Pull out doc paths and tags and put in reverse-associating maps.
	s.docPaths = make(map[string]*Doc)
	s.docTags = make(map[string][]*Doc)
	for _, d := range s.docs {
		s.docPaths[strings.TrimPrefix(d.Path, s.cfg.BasePath)] = d
		for _, t := range d.Tags {
			s.docTags[t] = append(s.docTags[t], d)
		}
	}

	// Pull out unique sorted list of tags.
	for t := range s.docTags {
		s.tags = append(s.tags, t)
	}
	sort.Strings(s.tags)

	// Set up presentation-related fields, Newer, Older, and Related.
	for _, doc := range s.docs {
		// Newer, Older: docs adjacent to doc
		for i := range s.docs {
			if s.docs[i] != doc {
				continue
			}
			if i > 0 {
				doc.Newer = s.docs[i-1]
			}
			if i+1 < len(s.docs) {
				doc.Older = s.docs[i+1]
			}
			break
		}

		// Related: all docs that share tags with doc.
		related := make(map[*Doc]bool)
		for _, t := range doc.Tags {
			for _, d := range s.docTags[t] {
				if d != doc {
					related[d] = true
				}
			}
		}
		for d := range related {
			doc.Related = append(doc.Related, d)
		}
		sort.Sort(docsByTime(doc.Related))
	}

	return nil
}

// renderAtomFeed generates an XML Atom feed and stores it in the Server's
// atomFeed field.
func (s *Server) renderAtomFeed() error {
	var updated time.Time
	if len(s.docs) > 1 {
		updated = s.docs[0].Time
	}
	feed := atom.Feed{
		Title:   "The Go Programming Language Blog",
		ID:      "tag:" + s.cfg.Hostname + ",2013:" + s.cfg.Hostname,
		Updated: atom.Time(updated),
		Link: []atom.Link{{
			Rel:  "self",
			Href: s.cfg.BaseURL + "/feed.atom",
		}},
	}
	for i, doc := range s.docs {
		if i >= s.cfg.FeedArticles {
			break
		}
		e := &atom.Entry{
			Title: doc.Title,
			ID:    feed.ID + doc.Path,
			Link: []atom.Link{{
				Rel:  "alternate",
				Href: doc.Permalink,
			}},
			Published: atom.Time(doc.Time),
			Updated:   atom.Time(doc.Time),
			Summary: &atom.Text{
				Type: "html",
				Body: summary(doc),
			},
			Content: &atom.Text{
				Type: "html",
				Body: string(doc.HTML),
			},
			Author: &atom.Person{
				Name: authors(doc.Authors),
			},
		}
		feed.Entry = append(feed.Entry, e)
	}
	data, err := xml.Marshal(&feed)
	if err != nil {
		return err
	}
	s.atomFeed = data
	return nil
}

type jsonItem struct {
	Title   string
	Link    string
	Time    time.Time
	Summary string
	Content string
	Author  string
}

// renderJSONFeed generates a JSON feed and stores it in the Server's jsonFeed
// field.
func (s *Server) renderJSONFeed() error {
	var feed []jsonItem
	for i, doc := range s.docs {
		if i >= s.cfg.FeedArticles {
			break
		}
		item := jsonItem{
			Title:   doc.Title,
			Link:    doc.Permalink,
			Time:    doc.Time,
			Summary: summary(doc),
			Content: string(doc.HTML),
			Author:  authors(doc.Authors),
		}
		feed = append(feed, item)
	}
	data, err := json.Marshal(feed)
	if err != nil {
		return err
	}
	s.jsonFeed = data
	return nil
}

// summary returns the first paragraph of text from the provided Doc.
func summary(d *Doc) string {
	if len(d.Sections) == 0 {
		return ""
	}
	for _, elem := range d.Sections[0].Elem {
		text, ok := elem.(present.Text)
		if !ok || text.Pre {
			// skip everything but non-text elements
			continue
		}
		var buf bytes.Buffer
		for _, s := range text.Lines {
			buf.WriteString(string(present.Style(s)))
			buf.WriteByte('\n')
		}
		return buf.String()
	}
	return ""
}

// rootData encapsulates data destined for the root template.
type rootData struct {
	Doc      *Doc
	BasePath string
	GodocURL string
	Data     interface{}
}

// ServeHTTP serves the front, index, and article pages
// as well as the ATOM and JSON feeds.
func (s *Server) ServeHTTP(w http.ResponseWriter, r *http.Request) {
	var (
		d = rootData{BasePath: s.cfg.BasePath, GodocURL: s.cfg.GodocURL}
		t *template.Template
	)
	switch p := strings.TrimPrefix(r.URL.Path, s.cfg.BasePath); p {
	case "/":
		d.Data = s.docs
		if len(s.docs) > s.cfg.HomeArticles {
			d.Data = s.docs[:s.cfg.HomeArticles]
		}
		t = s.template.home
	case "/index":
		d.Data = s.docs
		t = s.template.index
	case "/feed.atom", "/feeds/posts/default":
		w.Header().Set("Content-type", "application/atom+xml; charset=utf-8")
		w.Write(s.atomFeed)
		return
	case "/.json":
		if p := r.FormValue("jsonp"); validJSONPFunc.MatchString(p) {
			w.Header().Set("Content-type", "application/javascript; charset=utf-8")
			fmt.Fprintf(w, "%v(%s)", p, s.jsonFeed)
			return
		}
		w.Header().Set("Content-type", "application/json; charset=utf-8")
		w.Write(s.jsonFeed)
		return
	default:
		doc, ok := s.docPaths[p]
		if !ok {
			// Not a doc; try to just serve static content.
			s.content.ServeHTTP(w, r)
			return
		}
		d.Doc = doc
		t = s.template.article
	}
	err := t.ExecuteTemplate(w, "root", d)
	if err != nil {
		log.Println(err)
	}
}

// docsByTime implements sort.Interface, sorting Docs by their Time field.
type docsByTime []*Doc

func (s docsByTime) Len() int           { return len(s) }
func (s docsByTime) Swap(i, j int)      { s[i], s[j] = s[j], s[i] }
func (s docsByTime) Less(i, j int) bool { return s[i].Time.After(s[j].Time) }
<|endoftext|>


<commit_before>package blx

const (
	PayloadTypeTransferBlock string = "transfer-block"
	PayloadTypeRequestBlock         = "request-block"
)

const (
	StatusOK = iota
	StatusNotFound
)

type Meta struct {
	Values map[string][]byte `json:"values,omitempty"`
}

type Block struct {
	Key  string `json:"key"`
	Meta Meta   `json:"meta"`
	Data []byte `json:"data"`
}

type payloadTransferBlock struct {
	Status int    `json:"status"`
	Nonce  string `json:"nonce"`
	Block  *Block `json:"block,omitempty"`
}

type payloadTransferRequestBlock struct {
	Nonce    string `json:"nonce"`
	Key      string `json:"key,omitempty"`
	response chan interface{}
}
<commit_msg>Add block comments<commit_after>package blx

const (
	PayloadTypeTransferBlock string = "transfer-block"
	PayloadTypeRequestBlock         = "request-block"
)

const (
	StatusOK = iota
	StatusNotFound
)

// Meta is a struct that holds metadata for the specific block
type Meta struct {
	Values map[string][]byte `json:"values,omitempty"`
}

// Block is the base struct for data transmission and storage
type Block struct {
	Key  string `json:"key"`
	Meta Meta   `json:"meta"`
	Data []byte `json:"data"`
}

type payloadTransferBlock struct {
	Status int    `json:"status"`
	Nonce  string `json:"nonce"`
	Block  *Block `json:"block,omitempty"`
}

type payloadTransferRequestBlock struct {
	Nonce    string `json:"nonce"`
	Key      string `json:"key,omitempty"`
	response chan interface{}
}
<|endoftext|>