<commit_before>/*
** Copyright [2012-2014] [Megam Systems]
**
** Licensed under the Apache License, Version 2.0 (the "License");
** you may not use this file except in compliance with the License.
** You may obtain a copy of the License at
**
** http://www.apache.org/licenses/LICENSE-2.0
**
** Unless required by applicable law or agreed to in writing, software
** distributed under the License is distributed on an "AS IS" BASIS,
** WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
** See the License for the specific language governing permissions and
** limitations under the License.
 */
package amqp

import (
	"fmt"
	log "github.com/Sirupsen/logrus"
	"github.com/streadway/amqp"
	"sync"
)

type rabbitmqQ struct {
	name    string
	prefix  string
	factory *rabbitmqQFactory
	psc     *amqp.Channel
}

func (r *rabbitmqQ) exchname() string {
	return r.prefix + "_" + r.name + "_exchange"
}

func (r *rabbitmqQ) qname() string {
	return r.prefix + "_" + r.name + "_queue"
}

func (r *rabbitmqQ) tag() string {
	return r.prefix + "_" + r.name + "_tag"
}

func (r *rabbitmqQ) key() string {
	return r.prefix + "_" + r.name + "_key"
}

func (r *rabbitmqQ) Pub(msg []byte) error {
	chnl, err := r.factory.dial(r.exchname()) // return amqp.Channel
	if err != nil {
		return err
	}

	if err = chnl.Publish(
		r.exchname(), // publish to an exchange
		r.key(),      // routing to 0 or more queues
		false,        // mandatory
		false,        // immediate
		amqp.Publishing{
			Headers:         amqp.Table{},
			ContentType:     "text/plain",
			ContentEncoding: "",
			Body:            msg,
			DeliveryMode:    amqp.Transient, // 1=non-persistent, 2=persistent
			Priority:        0,              // 0-9
			// a bunch of application/implementation-specific fields
		},
	); err != nil {
		return fmt.Errorf("Failed to publish message in exchange: %s", err)
	}
	log.Debugf("[amqp] pub >%s< SUCCESS.",r.exchname())
	return err
}

func (r *rabbitmqQ) UnSub() error {
	if r.psc == nil {
		return nil
	}
	err := r.psc.Cancel(r.tag(), false)
	if err != nil {
		return err
	}
	return nil
}

func (r *rabbitmqQ) Sub() (chan []byte, error) {
	chnl, err := r.factory.getChonn(r.key(), r.exchname(), r.qname())
	if err != nil {
		return nil, err
	}

	r.psc = chnl

	msgChan := make(chan []byte)

	deliveries, err := chnl.Consume(
		r.qname(), // name
		r.tag(),   // consumerTag,
		true,      // noAck
		false,     // exclusive
		false,     // noLocal
		false,     // noWait
		nil,       // arguments
	)
	if err != nil {
		return nil, err
	}
	log.Debugf("[amqp] sub >%s< (%s) SUCCESS.", r.qname(), r.tag())

	//This is asynchronous, the callee will have to wait.
	go func() {
		//defer close(msgChan)
		for d := range deliveries {
			msgChan <- d.Body
		}

	}()
	return msgChan, nil
}

type rabbitmqQFactory struct {
	BindAddress string
	sync.Mutex
}

func (f *rabbitmqQFactory) Get(name string) (PubSubQ, error) {
	return &rabbitmqQ{name: name, prefix: "megam", factory: f}, nil
}

func (f *rabbitmqQFactory) Dial() (*amqp.Connection, error) {
	conn, err := amqp.Dial(f.BindAddress)

	if err != nil {
		return nil, err
	}
	return conn, nil
}

func (f *rabbitmqQFactory) dial(exchname string) (*amqp.Channel, error) {
	conn, err := f.Dial()

	if err != nil {
		return nil, err
	}

	log.Debugf("[amqp] dial (%s) SUCCESS", f.BindAddress)

	chnl, err := conn.Channel()

	if err != nil {
		return nil, err
	}

	//defer chnl.Close()

	if err = chnl.ExchangeDeclare(
		exchname, // name of the exchange
		"fanout", // exchange Type
		true,     // durable
		false,    // delete when complete
		false,    // internal
		false,    // noWait
		nil,      // arguments
	); err != nil {
		return nil, err
	}

	return chnl, err
}

func (factory *rabbitmqQFactory) getChonn(key string, exchname string, qname string) (*amqp.Channel, error) {
	chnl, err := factory.dial(exchname)
	if err != nil {
		return nil, err
	}

	qu, err := chnl.QueueDeclare(
		qname, // name of the queue
		true,  // durable
		false, // delete when usused
		false, // exclusive
		false, // noWait
		nil,   // arguments
	)
	if err != nil {
		return nil, err
	}

	log.Debugf("[amqp] queue (%s) SUCCESS", qname)

	if err = chnl.QueueBind(
		qu.Name, // name of the queue
		key,
		exchname,
		false, // noWait
		nil,   // arguments
	); err != nil {
		return nil, err
	}

	return chnl, nil
}
<commit_msg>minor text log change in rabbitmq during pub and sub<commit_after>/*
** Copyright [2012-2014] [Megam Systems]
**
** Licensed under the Apache License, Version 2.0 (the "License");
** you may not use this file except in compliance with the License.
** You may obtain a copy of the License at
**
** http://www.apache.org/licenses/LICENSE-2.0
**
** Unless required by applicable law or agreed to in writing, software
** distributed under the License is distributed on an "AS IS" BASIS,
** WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
** See the License for the specific language governing permissions and
** limitations under the License.
 */
package amqp

import (
	"fmt"
	log "github.com/Sirupsen/logrus"
	"github.com/streadway/amqp"
	"sync"
)

type rabbitmqQ struct {
	name    string
	prefix  string
	factory *rabbitmqQFactory
	psc     *amqp.Channel
}

func (r *rabbitmqQ) exchname() string {
	return r.prefix + "_" + r.name + "_exchange"
}

func (r *rabbitmqQ) qname() string {
	return r.prefix + "_" + r.name + "_queue"
}

func (r *rabbitmqQ) tag() string {
	return r.prefix + "_" + r.name + "_tag"
}

func (r *rabbitmqQ) key() string {
	return r.prefix + "_" + r.name + "_key"
}

func (r *rabbitmqQ) Pub(msg []byte) error {
	chnl, err := r.factory.dial(r.exchname()) // return amqp.Channel
	if err != nil {
		return err
	}

	if err = chnl.Publish(
		r.exchname(), // publish to an exchange
		r.key(),      // routing to 0 or more queues
		false,        // mandatory
		false,        // immediate
		amqp.Publishing{
			Headers:         amqp.Table{},
			ContentType:     "text/plain",
			ContentEncoding: "",
			Body:            msg,
			DeliveryMode:    amqp.Transient, // 1=non-persistent, 2=persistent
			Priority:        0,              // 0-9
			// a bunch of application/implementation-specific fields
		},
	); err != nil {
		return fmt.Errorf("Failed to publish message in exchange: %s", err)
	}
	log.Debugf("[amqp] pub (%s) SUCCESS.",r.exchname())
	return err
}

func (r *rabbitmqQ) UnSub() error {
	if r.psc == nil {
		return nil
	}
	err := r.psc.Cancel(r.tag(), false)
	if err != nil {
		return err
	}
	return nil
}

func (r *rabbitmqQ) Sub() (chan []byte, error) {
	chnl, err := r.factory.getChonn(r.key(), r.exchname(), r.qname())
	if err != nil {
		return nil, err
	}

	r.psc = chnl

	msgChan := make(chan []byte)

	deliveries, err := chnl.Consume(
		r.qname(), // name
		r.tag(),   // consumerTag,
		true,      // noAck
		false,     // exclusive
		false,     // noLocal
		false,     // noWait
		nil,       // arguments
	)
	if err != nil {
		return nil, err
	}
	log.Debugf("[amqp] sub (%s, %s) SUCCESS.", r.qname(), r.tag())

	//This is asynchronous, the callee will have to wait.
	go func() {
		//defer close(msgChan)
		for d := range deliveries {
			msgChan <- d.Body
		}

	}()
	return msgChan, nil
}

type rabbitmqQFactory struct {
	BindAddress string
	sync.Mutex
}

func (f *rabbitmqQFactory) Get(name string) (PubSubQ, error) {
	return &rabbitmqQ{name: name, prefix: "megam", factory: f}, nil
}

func (f *rabbitmqQFactory) Dial() (*amqp.Connection, error) {
	conn, err := amqp.Dial(f.BindAddress)

	if err != nil {
		return nil, err
	}
	return conn, nil
}

func (f *rabbitmqQFactory) dial(exchname string) (*amqp.Channel, error) {
	conn, err := f.Dial()

	if err != nil {
		return nil, err
	}

	log.Debugf("[amqp] dial (%s) SUCCESS", f.BindAddress)

	chnl, err := conn.Channel()

	if err != nil {
		return nil, err
	}

	//defer chnl.Close()

	if err = chnl.ExchangeDeclare(
		exchname, // name of the exchange
		"fanout", // exchange Type
		true,     // durable
		false,    // delete when complete
		false,    // internal
		false,    // noWait
		nil,      // arguments
	); err != nil {
		return nil, err
	}

	return chnl, err
}

func (factory *rabbitmqQFactory) getChonn(key string, exchname string, qname string) (*amqp.Channel, error) {
	chnl, err := factory.dial(exchname)
	if err != nil {
		return nil, err
	}

	qu, err := chnl.QueueDeclare(
		qname, // name of the queue
		true,  // durable
		false, // delete when usused
		false, // exclusive
		false, // noWait
		nil,   // arguments
	)
	if err != nil {
		return nil, err
	}

	log.Debugf("[amqp] queue (%s) SUCCESS", qname)

	if err = chnl.QueueBind(
		qu.Name, // name of the queue
		key,
		exchname,
		false, // noWait
		nil,   // arguments
	); err != nil {
		return nil, err
	}

	return chnl, nil
}
<|endoftext|>


<commit_before>// Copyright 2021 The LUCI Authors.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//      http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

// Package tree implements fetching tree status from Tree Status App.
package tree

import (
	"context"
	"encoding/json"
	"fmt"
	"io/ioutil"
	"net/http"
	"strings"
	"time"

	"go.chromium.org/luci/common/errors"
	"go.chromium.org/luci/common/logging"
	"go.chromium.org/luci/common/retry/transient"
	"go.chromium.org/luci/server/auth"
)

// Client defines the interface that interacts with Tree status App.
type Client interface {
	// FetchLatest fetches the latest tree status.
	FetchLatest(ctx context.Context, endpoint string) (Status, error)
}

// Status models the status returned by tree status app.
//
// Note that only fields that are needed in CV are included.
// Source of Truth: https://source.chromium.org/chromium/infra/infra/+/52a8cfcb436b0012e668630a2f261237046a033a:appengine/chromium_status/appengine_module/chromium_status/status.py;l=209-252
type Status struct {
	// State describes the Tree state.
	State State
	// Since is the timestamp when the tree obtains the current state.
	Since time.Time
}

// State enumerates possible values for tree state.
//
// Source of Truth: https://source.chromium.org/chromium/infra/infra/+/52a8cfcb436b0012e668630a2f261237046a033a:appengine/chromium_status/appengine_module/chromium_status/status.py;l=233-248
type State int8

const (
	StateUnknown State = iota
	Open
	Closed
	Throttled
	InMaintenance
)

func convertToTreeState(s string) State {
	switch s {
	case "open":
		return Open
	case "close":
		return Closed
	case "throttled":
		return Throttled
	case "maintenance":
		return InMaintenance
	default:
		return StateUnknown
	}
}

var clientCtxKey = "go.chromium.org/luci/cv/internal/tree.Client"

// InstallProd puts a production `Client` implementation.
func InstallProd(ctx context.Context) (context.Context, error) {
	t, err := auth.GetRPCTransport(ctx, auth.NoAuth, nil)
	if err != nil {
		return nil, err
	}
	return Install(ctx, httpClientImpl{&http.Client{Transport: t}}), nil
}

// Install puts the given `Client` implementation into the context.
func Install(ctx context.Context, c Client) context.Context {
	return context.WithValue(ctx, &clientCtxKey, c)
}

// mustClient returns the `Client` implementation stored in the context.
//
// Panics if not found.
func mustClient(ctx context.Context) Client {
	c := ctx.Value(&clientCtxKey)
	if c == nil {
		panic("Tree Status Client not found in the context")
	}
	return c.(Client)
}

// FetchLatest fetches the latest tree status.
//
// This is a shortcut of `tree.mustClient(ctx).FetchLatest(ctx, endpoint)`.
func FetchLatest(ctx context.Context, endpoint string) (Status, error) {
	return mustClient(ctx).FetchLatest(ctx, endpoint)
}

type httpClientImpl struct {
	*http.Client
}

// FetchLatest fetches the latest tree status.
func (c httpClientImpl) FetchLatest(ctx context.Context, endpoint string) (Status, error) {
	url := fmt.Sprintf("%s/current?format=json", strings.TrimSuffix(endpoint, "/"))

	req, err := http.NewRequestWithContext(ctx, "GET", url, nil)
	if err != nil {
		return Status{}, errors.Annotate(err, "failed to create new request").Err()
	}
	resp, err := c.Do(req)
	if err != nil {
		return Status{}, errors.Annotate(err, "failed to get latest tree status from %s", url).Tag(transient.Tag).Err()
	}
	defer resp.Body.Close()
	bs, err := ioutil.ReadAll(resp.Body)
	if err != nil {
		return Status{}, errors.Annotate(err, "failed to read response body from %s", url).Tag(transient.Tag).Err()
	}
	if resp.StatusCode >= 400 {
		logging.Errorf(ctx, "received error response when calling %s; response body: %q", url, string(bs))
		return Status{}, errors.Reason("received error when caling %s", url).Err()
	}
	var raw struct {
		State string `json:"general_state"`
		Date  string
	}
	if err := json.Unmarshal(bs, &raw); err != nil {
		return Status{}, errors.Annotate(err, "failed to unmarshal JSON %q", string(bs)).Err()
	}
	const dateFormat = "2006-01-02 15:04:05.999999"
	t, err := time.Parse(dateFormat, raw.Date)
	if err != nil {
		return Status{}, errors.Annotate(err, "failed to parse date %s", raw.Date).Err()
	}
	return Status{
		State: convertToTreeState(raw.State),
		Since: t,
	}, nil
}
<commit_msg>cv: Stop passing nil RPCOption when creating transport for tree client<commit_after>// Copyright 2021 The LUCI Authors.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//      http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

// Package tree implements fetching tree status from Tree Status App.
package tree

import (
	"context"
	"encoding/json"
	"fmt"
	"io/ioutil"
	"net/http"
	"strings"
	"time"

	"go.chromium.org/luci/common/errors"
	"go.chromium.org/luci/common/logging"
	"go.chromium.org/luci/common/retry/transient"
	"go.chromium.org/luci/server/auth"
)

// Client defines the interface that interacts with Tree status App.
type Client interface {
	// FetchLatest fetches the latest tree status.
	FetchLatest(ctx context.Context, endpoint string) (Status, error)
}

// Status models the status returned by tree status app.
//
// Note that only fields that are needed in CV are included.
// Source of Truth: https://source.chromium.org/chromium/infra/infra/+/52a8cfcb436b0012e668630a2f261237046a033a:appengine/chromium_status/appengine_module/chromium_status/status.py;l=209-252
type Status struct {
	// State describes the Tree state.
	State State
	// Since is the timestamp when the tree obtains the current state.
	Since time.Time
}

// State enumerates possible values for tree state.
//
// Source of Truth: https://source.chromium.org/chromium/infra/infra/+/52a8cfcb436b0012e668630a2f261237046a033a:appengine/chromium_status/appengine_module/chromium_status/status.py;l=233-248
type State int8

const (
	StateUnknown State = iota
	Open
	Closed
	Throttled
	InMaintenance
)

func convertToTreeState(s string) State {
	switch s {
	case "open":
		return Open
	case "close":
		return Closed
	case "throttled":
		return Throttled
	case "maintenance":
		return InMaintenance
	default:
		return StateUnknown
	}
}

var clientCtxKey = "go.chromium.org/luci/cv/internal/tree.Client"

// InstallProd puts a production `Client` implementation.
func InstallProd(ctx context.Context) (context.Context, error) {
	t, err := auth.GetRPCTransport(ctx, auth.NoAuth)
	if err != nil {
		return nil, err
	}
	return Install(ctx, httpClientImpl{&http.Client{Transport: t}}), nil
}

// Install puts the given `Client` implementation into the context.
func Install(ctx context.Context, c Client) context.Context {
	return context.WithValue(ctx, &clientCtxKey, c)
}

// mustClient returns the `Client` implementation stored in the context.
//
// Panics if not found.
func mustClient(ctx context.Context) Client {
	c := ctx.Value(&clientCtxKey)
	if c == nil {
		panic("Tree Status Client not found in the context")
	}
	return c.(Client)
}

// FetchLatest fetches the latest tree status.
//
// This is a shortcut of `tree.mustClient(ctx).FetchLatest(ctx, endpoint)`.
func FetchLatest(ctx context.Context, endpoint string) (Status, error) {
	return mustClient(ctx).FetchLatest(ctx, endpoint)
}

type httpClientImpl struct {
	*http.Client
}

// FetchLatest fetches the latest tree status.
func (c httpClientImpl) FetchLatest(ctx context.Context, endpoint string) (Status, error) {
	url := fmt.Sprintf("%s/current?format=json", strings.TrimSuffix(endpoint, "/"))

	req, err := http.NewRequestWithContext(ctx, "GET", url, nil)
	if err != nil {
		return Status{}, errors.Annotate(err, "failed to create new request").Err()
	}
	resp, err := c.Do(req)
	if err != nil {
		return Status{}, errors.Annotate(err, "failed to get latest tree status from %s", url).Tag(transient.Tag).Err()
	}
	defer resp.Body.Close()
	bs, err := ioutil.ReadAll(resp.Body)
	if err != nil {
		return Status{}, errors.Annotate(err, "failed to read response body from %s", url).Tag(transient.Tag).Err()
	}
	if resp.StatusCode >= 400 {
		logging.Errorf(ctx, "received error response when calling %s; response body: %q", url, string(bs))
		return Status{}, errors.Reason("received error when caling %s", url).Err()
	}
	var raw struct {
		State string `json:"general_state"`
		Date  string
	}
	if err := json.Unmarshal(bs, &raw); err != nil {
		return Status{}, errors.Annotate(err, "failed to unmarshal JSON %q", string(bs)).Err()
	}
	const dateFormat = "2006-01-02 15:04:05.999999"
	t, err := time.Parse(dateFormat, raw.Date)
	if err != nil {
		return Status{}, errors.Annotate(err, "failed to parse date %s", raw.Date).Err()
	}
	return Status{
		State: convertToTreeState(raw.State),
		Since: t,
	}, nil
}
<|endoftext|>


<commit_before>package shorty

import (
	"crypto/rand"
	"fmt"
	"github.com/golang/glog"
	"github.com/gorilla/mux"
	omni_http "github.com/qorio/omni/http"
	"io"
	"net/http"
	"net/url"
	"strings"
	"time"
)

var secureCookie *omni_http.SecureCookie

var (
	contextQueryParam = "__xrlc"
	appUrlSchemeParam = "__xrlp"
	shortCodeParam    = "__xrls"
	noAppInstallParam = "__xrl_noapp"

	uuidCookieKey       = "uuid"
	lastViewedCookieKey = "last"

	regexFmt string = "[_A-Za-z0-9\\.\\-]{%d,}"
)

type ShortyEndPointSettings struct {
	Redirect404     string
	GeoIpDbFilePath string
}

type ShortyEndPoint struct {
	settings      ShortyEndPointSettings
	router        *mux.Router
	requestParser *omni_http.RequestParser
	service       Shorty
}

func init() {
	var err error
	secureCookie, err = omni_http.NewSecureCookie([]byte(""), nil)
	if err != nil {
		glog.Warningln("Cannot initialize secure cookie!")
		panic(err)
	}
}

func NewApiEndPoint(settings ShortyEndPointSettings, service Shorty) (api *ShortyEndPoint, err error) {
	if requestParser, err := omni_http.NewRequestParser(settings.GeoIpDbFilePath); err == nil {
		api = &ShortyEndPoint{
			settings:      settings,
			router:        mux.NewRouter(),
			requestParser: requestParser,
			service:       service,
		}
		regex := fmt.Sprintf(regexFmt, service.UrlLength())
		api.router.HandleFunc("/{id:"+regex+"}", api.RedirectHandler).Methods("GET").Name("redirect")

		// Campaign based
		api.router.HandleFunc("/api/v1/campaign", api.ApiAddCampaignHandler).
			Methods("POST").Name("add_campaign")
		api.router.HandleFunc("/api/v1/campaign/{campaignId}", api.ApiGetCampaignHandler).
			Methods("GET").Name("get_campaign")
		api.router.HandleFunc("/api/v1/campaign/{campaignId}", api.ApiUpdateCampaignHandler).
			Methods("POST").Name("update_campaign")
		api.router.HandleFunc("/api/v1/campaign/{campaignId}/url", api.ApiAddCampaignUrlHandler).
			Methods("POST").Name("add_campaign_url")

		// Stand-alone
		api.router.HandleFunc("/api/v1/url", api.ApiAddUrlHandler).
			Methods("POST").Name("add")

		// First attempt if the app starts up organically -- this gives the server an opportunity
		// to match by fingerprinting.  If fingerprinting cannot match a user then the response will tell
		// the SDK to use safari for first launch install reporting.
		api.router.HandleFunc("/api/v1/tryfp/{scheme}/{app_uuid}",
			api.ApiTryMatchInstallOnOrganicAppLaunch).Methods("POST").Name("app_install_try_fingerprint")

		api.router.HandleFunc("/api/v1/events/install/{scheme}/{app_uuid}",
			api.ApiReportInstallOnReferredAppLaunch).Methods("POST").Name("app_install_referred_launch")

		api.router.HandleFunc("/api/v1/events/openurl/{scheme}/{app_uuid}",
			api.ApiReportAppOpenUrl).Methods("POST").Name("app_ping")

		return api, nil
	} else {
		return nil, err
	}
}

func NewRedirector(settings ShortyEndPointSettings, service Shorty) (api *ShortyEndPoint, err error) {
	if requestParser, err := omni_http.NewRequestParser(settings.GeoIpDbFilePath); err == nil {
		api = &ShortyEndPoint{
			settings:      settings,
			router:        mux.NewRouter(),
			requestParser: requestParser,
			service:       service,
		}

		regex := fmt.Sprintf(regexFmt, service.UrlLength())
		api.router.HandleFunc("/{id:"+regex+"}", api.RedirectHandler).Methods("GET").Name("redirect")

		// Install reporting when no context is known (organic install, first launch)
		api.router.HandleFunc("/i/{scheme}/{app_uuid}",
			api.ReportInstallOnOrganicAppLaunch).Methods("GET").Name("app_install_on_direct_launch")

		api.router.HandleFunc("/m/{scheme}/{uuid}/{shortCode:"+regex+"}/",
			api.CheckAppInstallInterstitialHandler).Methods("GET").Name("app_install_interstitial")

		// Dynamically generated so that any html @fetchUrl loaded by above can reference it as src="../deeplink.js"
		api.router.HandleFunc("/m/{scheme}/{uuid}/{shortCode:"+regex+"}/deeplink.js",
			api.CheckAppInstallInterstitialJSHandler).Methods("GET").Name("app_install_interstitial_js")

		return api, nil
	} else {
		return nil, err
	}
}

func (this *ShortyEndPoint) ServeHTTP(resp http.ResponseWriter, request *http.Request) {
	this.router.ServeHTTP(resp, request)
}

func (this *ShortyEndPoint) RedirectHandler(resp http.ResponseWriter, req *http.Request) {
	req.ParseForm()
	vars := mux.Vars(req)
	shortUrl, err := this.service.FindUrl(vars["id"])

	if err != nil {
		renderError(resp, req, err.Error(), http.StatusInternalServerError)
		return
	} else if shortUrl == nil {
		if this.settings.Redirect404 != "" {
			originalUrl, err := this.router.Get("redirect").URL("id", vars["id"])
			if err != nil {
				renderError(resp, req, err.Error(), http.StatusInternalServerError)
				return
			}
			url404 := strings.Replace(this.settings.Redirect404,
				"$origURL", url.QueryEscape(fmt.Sprintf("http://%s%s", req.Host, originalUrl.String())), 1)
			http.Redirect(resp, req, url404, http.StatusTemporaryRedirect)
			return
		}
		renderError(resp, req, "No URL was found with that shorty code", http.StatusNotFound)
		return
	}

	var destination string = shortUrl.Destination
	var renderInline bool = false
	var appOpen *AppOpen
	var found bool

	omni_http.SetNoCachingHeaders(resp)
	cookies := omni_http.NewCookieHandler(secureCookie, resp, req)

	visits, cookied, last, userId := processCookies(cookies, shortUrl.Id)

	var matchedRule *RoutingRule

	userAgent := omni_http.ParseUserAgent(req)
	origin, _ := this.requestParser.Parse(req)

	matchedRule, notFound := shortUrl.MatchRule(this.service, userAgent, origin, cookies)
	if notFound != nil || matchedRule == nil {
		http.Redirect(resp, req, shortUrl.Destination, http.StatusMovedPermanently)
		goto event
	}

	glog.Infoln("REDIRECT: matched rule:", matchedRule.Id)

	// check if there's been an appOpen
	appOpen, found, _ = this.service.FindAppOpen(UrlScheme(matchedRule.AppUrlScheme), UUID(userId))
	if !found || float64(time.Now().Unix()-appOpen.Timestamp) >= matchedRule.AppOpenTTLDays*24.*60.*60. {

		glog.V(10).Infoln("REDIRECT: no app-open in days:", matchedRule.AppOpenTTLDays)

		switch {

		case matchedRule.SendToInterstitial.isTrue():

			// Special handling of platforms where apps using webviews don't shared
			// cookies -- aka sandboxed uuids -- (e.g. iOS):
			// If the option to harvest the uuid cookie in this context is set to true
			// then do a redirect to a special harvest url where the uuid in the cookie
			// can be collected.  The harvest url will directly render content using the
			// rule's content properties e.g. rule.InlineContent or rule.FetchFromUrl.
			// This will let the calling browser think it's got the final content.
			// The user is then prompted to follow that harvest url in another browser
			// (eg. mobile Safari).  When the user uses another browser to go to that
			// url (which contains the uuid in this context), the handler of that url
			// will have a uuid-uuid pair.
			// For example, if on iOS, the Facebook app accesses the shortlink and arrives
			// here, we will have the uuid generated (userId).  We then redirect to a harvest
			// url //harvest/<shortCode>/<userId> and render a static html page instructing
			// the user to open the harvest url in Safari.  When the user opens the harvest
			// link in Safari, a different userId is generated (because the original userId
			// in the FB app webview is not shared).  Because the harvest url has the userId
			// in the FB app context, an association between the uuid-safari and uuid-fbapp
			// can be created.

			// If we need to harvest the cookied uuid - then
			// just redirect to the special landing page url where the cookied uuid (userId)
			// can be harvested.
			renderInline = false
			fetchUrl := url.QueryEscape(matchedRule.ContentSourceUrl)
			appUrlScheme := url.QueryEscape(matchedRule.AppUrlScheme)
			destination = fmt.Sprintf("/m/%s/%s/%s/?f=%s", appUrlScheme, userId, shortUrl.Id, fetchUrl)

		case matchedRule.ContentSourceUrl != "":

			renderInline = true
			destination = omni_http.FetchFromUrl(userAgent.Header, matchedRule.ContentSourceUrl)

		default:
			renderInline = false
			destination = matchedRule.AppStoreUrl
		}

	} else {

		glog.V(10).Infoln("REDIRECT- found app-open. redirecting", matchedRule.Destination)

		renderInline = false
		destination = matchedRule.Destination
	}

	if renderInline {
		resp.Write([]byte(destination))
	} else {
		destination = injectContext(destination, matchedRule, shortUrl, userId)
		http.Redirect(resp, req, destination, http.StatusMovedPermanently)
	}

event:
	// Record stats asynchronously
	timestamp := timestamp()

	go func() {
		origin, geoParseErr := this.requestParser.Parse(req)
		origin.Cookied = cookied
		origin.Visits = visits
		origin.LastVisit = last
		origin.Destination = destination
		origin.ShortCode = shortUrl.Id
		if geoParseErr != nil {
			glog.Warningln("can-not-determine-location", geoParseErr)
		}
		glog.Infoln(
			"uuid:", userId, "url:", shortUrl.Id, "send-to:", destination,
			"ip:", origin.Ip, "mobile:", origin.UserAgent.Mobile,
			"platform:", origin.UserAgent.Platform, "os:", origin.UserAgent.OS, "make:", origin.UserAgent.Make,
			"browser:", origin.UserAgent.Browser, "version:", origin.UserAgent.BrowserVersion,
			"location:", *origin.Location,
			"useragent:", origin.UserAgent.Header,
			"cookied", cookied)

		// Save a fingerprint
		deeplink := ""
		matchedRuleId := ""
		if matchedRule != nil {
			deeplink = matchedRule.Destination
			matchedRuleId = matchedRule.Id
		} else {
			glog.Warningln(">>>>>>>>>>>>  MATCHED RULE IS NIL", origin, req)
		}

		fingerprint := omni_http.FingerPrint(origin)
		this.service.SaveFingerprintedVisit(&FingerprintedVisit{
			Fingerprint:   fingerprint,
			Context:       UUID(userId),
			ShortCode:     shortUrl.Id,
			Visit:         destination,
			Deeplink:      deeplink,
			Timestamp:     timestamp,
			Referrer:      origin.Referrer,
			MatchedRuleId: matchedRuleId,
			UserAgent:     origin.UserAgent.Header,
		})

		this.service.PublishDecode(&DecodeEvent{
			RequestOrigin: origin,
			Destination:   destination,
			Context:       UUID(userId),
			Origin:        shortUrl.Origin,
			AppKey:        shortUrl.AppKey,
			CampaignKey:   shortUrl.CampaignKey,
			MatchedRuleId: matchedRuleId,
		})
		shortUrl.Record(origin, visits > 1)
	}()

	return
}

func injectContext(dest string, matchedRule *RoutingRule, shortUrl *ShortUrl, userId string) string {
	destination := dest
	if matchedRule != nil {
		parsedUrl, pErr := url.Parse(destination)
		// If the schemes match, then it's a deeplink.  Add additional params if the destination is
		// a deeplink or a http url that is mapped in Android's intent filter (where an app can also handle
		// url with http as scheme.
		if pErr == nil && (parsedUrl.Scheme == matchedRule.AppUrlScheme || matchedRule.IsAndroidIntentFilter.isTrue()) {
			destination = addQueryParam(destination, contextQueryParam, userId)
			destination = addQueryParam(destination, appUrlSchemeParam, matchedRule.AppUrlScheme)
			destination = addQueryParam(destination, shortCodeParam, shortUrl.Id)
		}
	}
	return destination
}

func addQueryParam(url, key, value string) string {
	if strings.ContainsRune(url, '?') {
		return url + "&" + key + "=" + value
	} else {
		return url + "?" + key + "=" + value
	}
}

func renderJsonError(resp http.ResponseWriter, req *http.Request, message string, code int) (err error) {
	resp.WriteHeader(code)
	resp.Write([]byte(fmt.Sprintf("{\"error\":\"%s\"}", message)))
	return
}

func renderError(resp http.ResponseWriter, req *http.Request, message string, code int) (err error) {
	resp.WriteHeader(code)
	resp.Write([]byte(fmt.Sprintf("<html><body>Error: %s </body></html>", message)))
	return
}

// newUUID generates a random UUID according to RFC 4122
func newUUID() (string, error) {
	uuid := make([]byte, 16)
	n, err := io.ReadFull(rand.Reader, uuid)
	if n != len(uuid) || err != nil {
		return "", err
	}
	// variant bits; see section 4.1.1
	uuid[8] = uuid[8]&^0xc0 | 0x80
	// version 4 (pseudo-random); see section 4.1.3
	uuid[6] = uuid[6]&^0xf0 | 0x40
	return fmt.Sprintf("%x-%x-%x-%x-%x", uuid[0:4], uuid[4:6], uuid[6:8], uuid[8:10], uuid[10:]), nil
}

// cookied = if the user uuid is cookied
func processCookies(cookies omni_http.Cookies, shortCode string) (visits int, cookied bool, last, uuid string) {
	last, _ = cookies.GetPlainString(lastViewedCookieKey)

	sc := shortCode
	if sc == "" {
		sc = last
	}
	cookies.Get(sc, &visits)

	var cookieError error

	uuid, _ = cookies.GetPlainString(uuidCookieKey)
	if uuid == "" {
		if uuid, _ = newUUID(); uuid != "" {
			cookieError = cookies.SetPlainString(uuidCookieKey, uuid)
			cookied = cookieError == nil
		}
	}

	visits++
	cookieError = cookies.SetPlainString(lastViewedCookieKey, sc)
	cookieError = cookies.Set(sc, visits)

	return
}
<commit_msg>Logging on visits; fingerprints<commit_after>package shorty

import (
	"crypto/rand"
	"fmt"
	"github.com/golang/glog"
	"github.com/gorilla/mux"
	omni_http "github.com/qorio/omni/http"
	"io"
	"net/http"
	"net/url"
	"strings"
	"time"
)

var secureCookie *omni_http.SecureCookie

var (
	contextQueryParam = "__xrlc"
	appUrlSchemeParam = "__xrlp"
	shortCodeParam    = "__xrls"
	noAppInstallParam = "__xrl_noapp"

	uuidCookieKey       = "uuid"
	lastViewedCookieKey = "last"

	regexFmt string = "[_A-Za-z0-9\\.\\-]{%d,}"
)

type ShortyEndPointSettings struct {
	Redirect404     string
	GeoIpDbFilePath string
}

type ShortyEndPoint struct {
	settings      ShortyEndPointSettings
	router        *mux.Router
	requestParser *omni_http.RequestParser
	service       Shorty
}

func init() {
	var err error
	secureCookie, err = omni_http.NewSecureCookie([]byte(""), nil)
	if err != nil {
		glog.Warningln("Cannot initialize secure cookie!")
		panic(err)
	}
}

func NewApiEndPoint(settings ShortyEndPointSettings, service Shorty) (api *ShortyEndPoint, err error) {
	if requestParser, err := omni_http.NewRequestParser(settings.GeoIpDbFilePath); err == nil {
		api = &ShortyEndPoint{
			settings:      settings,
			router:        mux.NewRouter(),
			requestParser: requestParser,
			service:       service,
		}
		regex := fmt.Sprintf(regexFmt, service.UrlLength())
		api.router.HandleFunc("/{id:"+regex+"}", api.RedirectHandler).Methods("GET").Name("redirect")

		// Campaign based
		api.router.HandleFunc("/api/v1/campaign", api.ApiAddCampaignHandler).
			Methods("POST").Name("add_campaign")
		api.router.HandleFunc("/api/v1/campaign/{campaignId}", api.ApiGetCampaignHandler).
			Methods("GET").Name("get_campaign")
		api.router.HandleFunc("/api/v1/campaign/{campaignId}", api.ApiUpdateCampaignHandler).
			Methods("POST").Name("update_campaign")
		api.router.HandleFunc("/api/v1/campaign/{campaignId}/url", api.ApiAddCampaignUrlHandler).
			Methods("POST").Name("add_campaign_url")

		// Stand-alone
		api.router.HandleFunc("/api/v1/url", api.ApiAddUrlHandler).
			Methods("POST").Name("add")

		// First attempt if the app starts up organically -- this gives the server an opportunity
		// to match by fingerprinting.  If fingerprinting cannot match a user then the response will tell
		// the SDK to use safari for first launch install reporting.
		api.router.HandleFunc("/api/v1/tryfp/{scheme}/{app_uuid}",
			api.ApiTryMatchInstallOnOrganicAppLaunch).Methods("POST").Name("app_install_try_fingerprint")

		api.router.HandleFunc("/api/v1/events/install/{scheme}/{app_uuid}",
			api.ApiReportInstallOnReferredAppLaunch).Methods("POST").Name("app_install_referred_launch")

		api.router.HandleFunc("/api/v1/events/openurl/{scheme}/{app_uuid}",
			api.ApiReportAppOpenUrl).Methods("POST").Name("app_ping")

		return api, nil
	} else {
		return nil, err
	}
}

func NewRedirector(settings ShortyEndPointSettings, service Shorty) (api *ShortyEndPoint, err error) {
	if requestParser, err := omni_http.NewRequestParser(settings.GeoIpDbFilePath); err == nil {
		api = &ShortyEndPoint{
			settings:      settings,
			router:        mux.NewRouter(),
			requestParser: requestParser,
			service:       service,
		}

		regex := fmt.Sprintf(regexFmt, service.UrlLength())
		api.router.HandleFunc("/{id:"+regex+"}", api.RedirectHandler).Methods("GET").Name("redirect")

		// Install reporting when no context is known (organic install, first launch)
		api.router.HandleFunc("/i/{scheme}/{app_uuid}",
			api.ReportInstallOnOrganicAppLaunch).Methods("GET").Name("app_install_on_direct_launch")

		api.router.HandleFunc("/m/{scheme}/{uuid}/{shortCode:"+regex+"}/",
			api.CheckAppInstallInterstitialHandler).Methods("GET").Name("app_install_interstitial")

		// Dynamically generated so that any html @fetchUrl loaded by above can reference it as src="../deeplink.js"
		api.router.HandleFunc("/m/{scheme}/{uuid}/{shortCode:"+regex+"}/deeplink.js",
			api.CheckAppInstallInterstitialJSHandler).Methods("GET").Name("app_install_interstitial_js")

		return api, nil
	} else {
		return nil, err
	}
}

func (this *ShortyEndPoint) ServeHTTP(resp http.ResponseWriter, request *http.Request) {
	this.router.ServeHTTP(resp, request)
}

func (this *ShortyEndPoint) RedirectHandler(resp http.ResponseWriter, req *http.Request) {
	req.ParseForm()
	vars := mux.Vars(req)
	shortUrl, err := this.service.FindUrl(vars["id"])

	if err != nil {
		renderError(resp, req, err.Error(), http.StatusInternalServerError)
		return
	} else if shortUrl == nil {
		if this.settings.Redirect404 != "" {
			originalUrl, err := this.router.Get("redirect").URL("id", vars["id"])
			if err != nil {
				renderError(resp, req, err.Error(), http.StatusInternalServerError)
				return
			}
			url404 := strings.Replace(this.settings.Redirect404,
				"$origURL", url.QueryEscape(fmt.Sprintf("http://%s%s", req.Host, originalUrl.String())), 1)
			http.Redirect(resp, req, url404, http.StatusTemporaryRedirect)
			return
		}
		renderError(resp, req, "No URL was found with that shorty code", http.StatusNotFound)
		return
	}

	var destination string = shortUrl.Destination
	var renderInline bool = false
	var appOpen *AppOpen
	var found bool

	omni_http.SetNoCachingHeaders(resp)
	cookies := omni_http.NewCookieHandler(secureCookie, resp, req)

	visits, cookied, last, userId := processCookies(cookies, shortUrl.Id)

	var matchedRule *RoutingRule
	matchedRuleId, deeplink := "no-match", "?"

	userAgent := omni_http.ParseUserAgent(req)
	origin, _ := this.requestParser.Parse(req)

	matchedRule, notFound := shortUrl.MatchRule(this.service, userAgent, origin, cookies)
	if notFound != nil || matchedRule == nil {
		http.Redirect(resp, req, shortUrl.Destination, http.StatusMovedPermanently)
		goto event
	}

	glog.Infoln("REDIRECT: matched rule:", matchedRule.Id)
	matchedRuleId = matchedRule.Id
	deeplink = matchedRule.Destination

	// check if there's been an appOpen
	appOpen, found, _ = this.service.FindAppOpen(UrlScheme(matchedRule.AppUrlScheme), UUID(userId))
	if !found || float64(time.Now().Unix()-appOpen.Timestamp) >= matchedRule.AppOpenTTLDays*24.*60.*60. {

		glog.V(10).Infoln("REDIRECT: no app-open in days:", matchedRule.AppOpenTTLDays)

		switch {

		case matchedRule.SendToInterstitial.isTrue():

			// Special handling of platforms where apps using webviews don't shared
			// cookies -- aka sandboxed uuids -- (e.g. iOS):
			// If the option to harvest the uuid cookie in this context is set to true
			// then do a redirect to a special harvest url where the uuid in the cookie
			// can be collected.  The harvest url will directly render content using the
			// rule's content properties e.g. rule.InlineContent or rule.FetchFromUrl.
			// This will let the calling browser think it's got the final content.
			// The user is then prompted to follow that harvest url in another browser
			// (eg. mobile Safari).  When the user uses another browser to go to that
			// url (which contains the uuid in this context), the handler of that url
			// will have a uuid-uuid pair.
			// For example, if on iOS, the Facebook app accesses the shortlink and arrives
			// here, we will have the uuid generated (userId).  We then redirect to a harvest
			// url //harvest/<shortCode>/<userId> and render a static html page instructing
			// the user to open the harvest url in Safari.  When the user opens the harvest
			// link in Safari, a different userId is generated (because the original userId
			// in the FB app webview is not shared).  Because the harvest url has the userId
			// in the FB app context, an association between the uuid-safari and uuid-fbapp
			// can be created.

			// If we need to harvest the cookied uuid - then
			// just redirect to the special landing page url where the cookied uuid (userId)
			// can be harvested.
			renderInline = false
			fetchUrl := url.QueryEscape(matchedRule.ContentSourceUrl)
			appUrlScheme := url.QueryEscape(matchedRule.AppUrlScheme)
			destination = fmt.Sprintf("/m/%s/%s/%s/?f=%s", appUrlScheme, userId, shortUrl.Id, fetchUrl)

		case matchedRule.ContentSourceUrl != "":

			renderInline = true
			destination = omni_http.FetchFromUrl(userAgent.Header, matchedRule.ContentSourceUrl)

		default:
			renderInline = false
			destination = matchedRule.AppStoreUrl
		}

	} else {

		glog.V(10).Infoln("REDIRECT- found app-open. redirecting", matchedRule.Destination)

		renderInline = false
		destination = matchedRule.Destination
	}

	if renderInline {
		resp.Write([]byte(destination))
	} else {
		destination = injectContext(destination, matchedRule, shortUrl, userId)
		http.Redirect(resp, req, destination, http.StatusMovedPermanently)
	}

event:
	// Record stats asynchronously
	timestamp := timestamp()

	go func() {
		origin, geoParseErr := this.requestParser.Parse(req)
		origin.Cookied = cookied
		origin.Visits = visits
		origin.LastVisit = last
		origin.Destination = destination
		origin.ShortCode = shortUrl.Id
		if geoParseErr != nil {
			glog.Warningln("can-not-determine-location", geoParseErr)
		}
		glog.Infoln(
			"uuid:", userId, "url:", shortUrl.Id, "send-to:", destination,
			"ip:", origin.Ip, "mobile:", origin.UserAgent.Mobile,
			"platform:", origin.UserAgent.Platform, "os:", origin.UserAgent.OS, "make:", origin.UserAgent.Make,
			"browser:", origin.UserAgent.Browser, "version:", origin.UserAgent.BrowserVersion,
			"location:", *origin.Location,
			"useragent:", origin.UserAgent.Header,
			"cookied", cookied)

		// Save a fingerprint
		fingerprint := omni_http.FingerPrint(origin)
		fv := &FingerprintedVisit{
			Fingerprint:   fingerprint,
			Context:       UUID(userId),
			ShortCode:     shortUrl.Id,
			Visit:         destination,
			Deeplink:      deeplink,
			Timestamp:     timestamp,
			Referrer:      origin.Referrer,
			MatchedRuleId: matchedRuleId,
			UserAgent:     origin.UserAgent.Header,
		}
		this.service.SaveFingerprintedVisit(fv)
		glog.V(20).Infoln("saved visit", "fingerprint=", fingerprint, "visit=", fv)

		this.service.PublishDecode(&DecodeEvent{
			RequestOrigin: origin,
			Destination:   destination,
			Context:       UUID(userId),
			Origin:        shortUrl.Origin,
			AppKey:        shortUrl.AppKey,
			CampaignKey:   shortUrl.CampaignKey,
			MatchedRuleId: matchedRuleId,
		})
		shortUrl.Record(origin, visits > 1)
	}()

	return
}

func injectContext(dest string, matchedRule *RoutingRule, shortUrl *ShortUrl, userId string) string {
	destination := dest
	if matchedRule != nil {
		parsedUrl, pErr := url.Parse(destination)
		// If the schemes match, then it's a deeplink.  Add additional params if the destination is
		// a deeplink or a http url that is mapped in Android's intent filter (where an app can also handle
		// url with http as scheme.
		if pErr == nil && (parsedUrl.Scheme == matchedRule.AppUrlScheme || matchedRule.IsAndroidIntentFilter.isTrue()) {
			destination = addQueryParam(destination, contextQueryParam, userId)
			destination = addQueryParam(destination, appUrlSchemeParam, matchedRule.AppUrlScheme)
			destination = addQueryParam(destination, shortCodeParam, shortUrl.Id)
		}
	}
	return destination
}

func addQueryParam(url, key, value string) string {
	if strings.ContainsRune(url, '?') {
		return url + "&" + key + "=" + value
	} else {
		return url + "?" + key + "=" + value
	}
}

func renderJsonError(resp http.ResponseWriter, req *http.Request, message string, code int) (err error) {
	resp.WriteHeader(code)
	resp.Write([]byte(fmt.Sprintf("{\"error\":\"%s\"}", message)))
	return
}

func renderError(resp http.ResponseWriter, req *http.Request, message string, code int) (err error) {
	resp.WriteHeader(code)
	resp.Write([]byte(fmt.Sprintf("<html><body>Error: %s </body></html>", message)))
	return
}

// newUUID generates a random UUID according to RFC 4122
func newUUID() (string, error) {
	uuid := make([]byte, 16)
	n, err := io.ReadFull(rand.Reader, uuid)
	if n != len(uuid) || err != nil {
		return "", err
	}
	// variant bits; see section 4.1.1
	uuid[8] = uuid[8]&^0xc0 | 0x80
	// version 4 (pseudo-random); see section 4.1.3
	uuid[6] = uuid[6]&^0xf0 | 0x40
	return fmt.Sprintf("%x-%x-%x-%x-%x", uuid[0:4], uuid[4:6], uuid[6:8], uuid[8:10], uuid[10:]), nil
}

// cookied = if the user uuid is cookied
func processCookies(cookies omni_http.Cookies, shortCode string) (visits int, cookied bool, last, uuid string) {
	last, _ = cookies.GetPlainString(lastViewedCookieKey)

	sc := shortCode
	if sc == "" {
		sc = last
	}
	cookies.Get(sc, &visits)

	var cookieError error

	uuid, _ = cookies.GetPlainString(uuidCookieKey)
	if uuid == "" {
		if uuid, _ = newUUID(); uuid != "" {
			cookieError = cookies.SetPlainString(uuidCookieKey, uuid)
			cookied = cookieError == nil
		}
	}

	visits++
	cookieError = cookies.SetPlainString(lastViewedCookieKey, sc)
	cookieError = cookies.Set(sc, visits)

	return
}
<|endoftext|>


<commit_before>package should

import (
	"bytes"
	"fmt"
	"net/http"
	"testing"

	"github.com/kindrid/gotest/rest"
	"github.com/y0ssar1an/q"
)

// RESTHarness provides an engine that can construct requests, run them, and
// prepare the results for testing.
type RESTHarness struct {
	API       rest.Describer
	Requester RequestMaker
	Parser    StructureParser
}

// RESTExchange holds one HTTP request, expected response, and actual response
type RESTExchange struct {
	Request  *http.Request   // The request
	Expected *BodiedResponse // The response we have got
	Actual   *BodiedResponse // The response we actually got
	Err      error           // any error running the request
}

// RequesterMaker is a function that the tested code will use to simulate or actually perform a request.
type RequestMaker func(*http.Request) (*http.Response, error)

// BodiedResponse holds a response with the Body already read and parsed
type BodiedResponse struct {
	Response *http.Response    // incorporate response
	Raw      string            // the raw body
	Parsed   StructureExplorer // parsed body
}

func ReadResponseBody(rsp *http.Response, parser StructureParser) (result *BodiedResponse, err error) {
	result = &BodiedResponse{Response: rsp}
	if rsp.Body != nil {
		buf := new(bytes.Buffer)
		buf.ReadFrom(rsp.Body)
		result.Raw = buf.String()
	}
	if result.Raw != "" {
		result.Parsed, err = parser(result.Raw)
	}
	return
}

// RunRequest executes an HTTP request and returns the expected and actual response in a
// *RESTExchange. For the format of params, see rest.Describer's documentation, currently:
//
// Params is a list of strings, [name1, value1, name2, value2, ...]. Keys have one
// of these prefixes:
//
// 	  ":" - indicates an html header as a string
//    "&" - indicates a URL param as a string
//    "=" - treated as a raw string in path and body templating, ADD QUOTES if you want quotes.
func (har *RESTHarness) RunRequest(requestID string, params []string, body string) (result *RESTExchange) {
	var expected, actual *http.Response
	// Grab information from the Describer (API specification)
	result = &RESTExchange{}
	// q.Q("Before get Request", requestID, params, body)
	result.Request, expected, result.Err = har.API.GetRequest(requestID, params, body)
	if result.Err != nil {
		return
	}

	if expected != nil {
		result.Expected, result.Err = ReadResponseBody(expected, har.Parser)
		if result.Err != nil {
			return
		}
	}

	// // Run the request
	if har.Requester == nil {
		result.Err = fmt.Errorf("a RESTHarness needs a request function to run a request")
	}
	actual, result.Err = har.Requester(result.Request)
	if result.Err != nil {
		return
	}
	result.Actual, result.Err = ReadResponseBody(actual, har.Parser)
	if result.Err != nil {
		return
	}

	return
}

// TestRequest works like RunRequest but makes some basic assertions about the return.
func (har *RESTHarness) TestRequest(t *testing.T, requestID string, params []string, body string) (result *RESTExchange) {
	result = har.RunRequest(requestID, params, body)
	q.Q(result)

	fail := ""
	if result.Err != nil {
		fail = result.Err.Error()
	}
	if fail == "" && (result.Expected == nil || result.Expected.Response == nil) {
		fail = "No expected response supplied for this path"
	}
	if fail == "" && (result.Actual == nil || result.Actual.Response == nil) {
		fail = "No actual response supplied for this path"
	}
	if fail == "" {
		fail = MatchHTTPStatusCode(result.Actual.Response, result.Expected.Response.StatusCode)
	}

	// q.Q(result.Expected.Response)

	if fail != "" {
		t.Error(fail)
	}

	// could check JSON:API content type
	return
}
<commit_msg>Remove debugging code<commit_after>package should

import (
	"bytes"
	"fmt"
	"net/http"
	"testing"

	"github.com/kindrid/gotest/rest"
)

// RESTHarness provides an engine that can construct requests, run them, and
// prepare the results for testing.
type RESTHarness struct {
	API       rest.Describer
	Requester RequestMaker
	Parser    StructureParser
}

// RESTExchange holds one HTTP request, expected response, and actual response
type RESTExchange struct {
	Request  *http.Request   // The request
	Expected *BodiedResponse // The response we have got
	Actual   *BodiedResponse // The response we actually got
	Err      error           // any error running the request
}

// RequesterMaker is a function that the tested code will use to simulate or actually perform a request.
type RequestMaker func(*http.Request) (*http.Response, error)

// BodiedResponse holds a response with the Body already read and parsed
type BodiedResponse struct {
	Response *http.Response    // incorporate response
	Raw      string            // the raw body
	Parsed   StructureExplorer // parsed body
}

func ReadResponseBody(rsp *http.Response, parser StructureParser) (result *BodiedResponse, err error) {
	result = &BodiedResponse{Response: rsp}
	if rsp.Body != nil {
		buf := new(bytes.Buffer)
		buf.ReadFrom(rsp.Body)
		result.Raw = buf.String()
	}
	if result.Raw != "" {
		result.Parsed, err = parser(result.Raw)
	}
	return
}

// RunRequest executes an HTTP request and returns the expected and actual response in a
// *RESTExchange. For the format of params, see rest.Describer's documentation, currently:
//
// Params is a list of strings, [name1, value1, name2, value2, ...]. Keys have one
// of these prefixes:
//
// 	  ":" - indicates an html header as a string
//    "&" - indicates a URL param as a string
//    "=" - treated as a raw string in path and body templating, ADD QUOTES if you want quotes.
func (har *RESTHarness) RunRequest(requestID string, params []string, body string) (result *RESTExchange) {
	var expected, actual *http.Response
	// Grab information from the Describer (API specification)
	result = &RESTExchange{}
	result.Request, expected, result.Err = har.API.GetRequest(requestID, params, body)
	if result.Err != nil {
		return
	}

	if expected != nil {
		result.Expected, result.Err = ReadResponseBody(expected, har.Parser)
		if result.Err != nil {
			return
		}
	}

	// // Run the request
	if har.Requester == nil {
		result.Err = fmt.Errorf("a RESTHarness needs a request function to run a request")
	}
	actual, result.Err = har.Requester(result.Request)
	if result.Err != nil {
		return
	}
	result.Actual, result.Err = ReadResponseBody(actual, har.Parser)
	if result.Err != nil {
		return
	}

	return
}

// TestRequest works like RunRequest but makes some basic assertions about the return.
func (har *RESTHarness) TestRequest(t *testing.T, requestID string, params []string, body string) (result *RESTExchange) {
	result = har.RunRequest(requestID, params, body)

	fail := ""
	if result.Err != nil {
		fail = result.Err.Error()
	}
	if fail == "" && (result.Expected == nil || result.Expected.Response == nil) {
		fail = "No expected response supplied for this path"
	}
	if fail == "" && (result.Actual == nil || result.Actual.Response == nil) {
		fail = "No actual response supplied for this path"
	}
	if fail == "" {
		fail = MatchHTTPStatusCode(result.Actual.Response, result.Expected.Response.StatusCode)
	}

	if fail != "" {
		t.Error(fail)
	}

	// could check JSON:API content type
	return
}
<|endoftext|>


<commit_before>package main

import (
	"errors"
	"fmt"
	"os"
	"path/filepath"
	"strconv"
	"strings"
	"time"

	"github.com/NebulousLabs/Sia/api"
	"github.com/NebulousLabs/Sia/crypto"
	"github.com/NebulousLabs/Sia/modules"
	"github.com/NebulousLabs/Sia/modules/consensus"
	"github.com/NebulousLabs/Sia/modules/explorer"
	"github.com/NebulousLabs/Sia/modules/gateway"
	"github.com/NebulousLabs/Sia/modules/host"
	"github.com/NebulousLabs/Sia/modules/miner"
	"github.com/NebulousLabs/Sia/modules/renter"
	"github.com/NebulousLabs/Sia/modules/transactionpool"
	"github.com/NebulousLabs/Sia/modules/wallet"
	"github.com/NebulousLabs/Sia/profile"

	"github.com/spf13/cobra"
)

// processNetAddr adds a ':' to a bare integer, so that it is a proper port
// number.
func processNetAddr(addr string) string {
	_, err := strconv.Atoi(addr)
	if err == nil {
		return ":" + addr
	}
	return addr
}

// processModules makes the modules string lowercase to make checking if a
// module in the string easier, and returns an error if the string contains an
// invalid module character.
func processModules(modules string) (string, error) {
	modules = strings.ToLower(modules)
	validModules := "cghmrtwe"
	invalidModules := modules
	for _, m := range validModules {
		invalidModules = strings.Replace(invalidModules, string(m), "", 1)
	}
	if len(invalidModules) > 0 {
		return "", errors.New("Unable to parse --modules flag, unrecognized modules: " + invalidModules)
	}
	return modules, nil
}

// processConfig checks the configuration values and performs cleanup on
// incorrect-but-allowed values.
func processConfig(config Config) (Config, error) {
	var err error
	config.Siad.APIaddr = processNetAddr(config.Siad.APIaddr)
	config.Siad.RPCaddr = processNetAddr(config.Siad.RPCaddr)
	config.Siad.HostAddr = processNetAddr(config.Siad.HostAddr)
	config.Siad.Modules, err = processModules(config.Siad.Modules)
	if err != nil {
		return Config{}, err
	}
	return config, nil
}

// startDaemonCmd uses the config parameters to start siad.
func startDaemon(config Config) (err error) {
	// Print a startup message.
	fmt.Println("Loading...")
	loadStart := time.Now()

	// Create all of the modules.
	i := 0
	var g modules.Gateway
	if strings.Contains(config.Siad.Modules, "g") {
		i++
		fmt.Printf("(%d/%d) Loading gateway...\n", i, len(config.Siad.Modules))
		g, err = gateway.New(config.Siad.RPCaddr, filepath.Join(config.Siad.SiaDir, modules.GatewayDir))
		if err != nil {
			return err
		}
	}
	var cs modules.ConsensusSet
	if strings.Contains(config.Siad.Modules, "c") {
		i++
		fmt.Printf("(%d/%d) Loading consensus...\n", i, len(config.Siad.Modules))
		cs, err = consensus.New(g, filepath.Join(config.Siad.SiaDir, modules.ConsensusDir))
		if err != nil {
			return err
		}
	}
	var e modules.Explorer
	if strings.Contains(config.Siad.Modules, "e") {
		i++
		fmt.Printf("(%d/%d) Loading explorer...\n", i, len(config.Siad.Modules))
		e, err = explorer.New(cs, filepath.Join(config.Siad.SiaDir, modules.ExplorerDir))
		if err != nil {
			return err
		}
	}
	var tpool modules.TransactionPool
	if strings.Contains(config.Siad.Modules, "t") {
		i++
		fmt.Printf("(%d/%d) Loading transaction pool...\n", i, len(config.Siad.Modules))
		tpool, err = transactionpool.New(cs, g)
		if err != nil {
			return err
		}
	}
	var w modules.Wallet
	if strings.Contains(config.Siad.Modules, "w") {
		i++
		fmt.Printf("(%d/%d) Loading wallet...\n", i, len(config.Siad.Modules))
		w, err = wallet.New(cs, tpool, filepath.Join(config.Siad.SiaDir, modules.WalletDir))
		if err != nil {
			return err
		}
	}
	var m modules.Miner
	if strings.Contains(config.Siad.Modules, "m") {
		i++
		fmt.Printf("(%d/%d) Loading miner...\n", i, len(config.Siad.Modules))
		m, err = miner.New(cs, tpool, w, filepath.Join(config.Siad.SiaDir, modules.MinerDir))
		if err != nil {
			return err
		}
	}
	var h modules.Host
	if strings.Contains(config.Siad.Modules, "h") {
		i++
		fmt.Printf("(%d/%d) Loading host...\n", i, len(config.Siad.Modules))
		h, err = host.New(cs, tpool, w, config.Siad.HostAddr, filepath.Join(config.Siad.SiaDir, modules.HostDir))
		if err != nil {
			return err
		}
	}
	var r modules.Renter
	if strings.Contains(config.Siad.Modules, "r") {
		i++
		fmt.Printf("(%d/%d) Loading renter...\n", i, len(config.Siad.Modules))
		r, err = renter.New(cs, w, tpool, filepath.Join(config.Siad.SiaDir, modules.RenterDir))
		if err != nil {
			return err
		}
	}
	srv, err := api.NewServer(
		config.Siad.APIaddr,
		config.Siad.RequiredUserAgent,
		cs,
		e,
		g,
		h,
		m,
		r,
		tpool,
		w,
	)
	if err != nil {
		return err
	}

	// Bootstrap to the network.
	if !config.Siad.NoBootstrap && g != nil {
		// connect to 3 random bootstrap nodes
		perm, err := crypto.Perm(len(modules.BootstrapPeers))
		if err != nil {
			return err
		}
		for _, i := range perm[:3] {
			go g.Connect(modules.BootstrapPeers[i])
		}
	}

	// Print a 'startup complete' message.
	startupTime := time.Since(loadStart)
	fmt.Println("Finished loading in", startupTime.Seconds(), "seconds")

	// Start serving api requests.
	err = srv.Serve()
	if err != nil {
		return err
	}
	return nil
}

// startDaemonCmd is a passthrough function for startDaemon.
func startDaemonCmd(cmd *cobra.Command, _ []string) {
	// Create the profiling directory if profiling is enabled.
	if globalConfig.Siad.Profile {
		go profile.StartContinuousProfile(globalConfig.Siad.ProfileDir)
	}

	// Process the config variables after they are parsed by cobra.
	config, err := processConfig(globalConfig)
	if err != nil {
		fmt.Fprintln(os.Stderr, err)
		cmd.Usage()
		os.Exit(exitCodeUsage)
	}

	// Start siad. startDaemon will only return when it is shutting down.
	err = startDaemon(config)
	if err != nil {
		die(err)
	}
}
<commit_msg>Clarify invalid modules error<commit_after>package main

import (
	"errors"
	"fmt"
	"os"
	"path/filepath"
	"strconv"
	"strings"
	"time"

	"github.com/NebulousLabs/Sia/api"
	"github.com/NebulousLabs/Sia/crypto"
	"github.com/NebulousLabs/Sia/modules"
	"github.com/NebulousLabs/Sia/modules/consensus"
	"github.com/NebulousLabs/Sia/modules/explorer"
	"github.com/NebulousLabs/Sia/modules/gateway"
	"github.com/NebulousLabs/Sia/modules/host"
	"github.com/NebulousLabs/Sia/modules/miner"
	"github.com/NebulousLabs/Sia/modules/renter"
	"github.com/NebulousLabs/Sia/modules/transactionpool"
	"github.com/NebulousLabs/Sia/modules/wallet"
	"github.com/NebulousLabs/Sia/profile"

	"github.com/spf13/cobra"
)

// processNetAddr adds a ':' to a bare integer, so that it is a proper port
// number.
func processNetAddr(addr string) string {
	_, err := strconv.Atoi(addr)
	if err == nil {
		return ":" + addr
	}
	return addr
}

// processModules makes the modules string lowercase to make checking if a
// module in the string easier, and returns an error if the string contains an
// invalid module character.
func processModules(modules string) (string, error) {
	modules = strings.ToLower(modules)
	validModules := "cghmrtwe"
	invalidModules := modules
	for _, m := range validModules {
		invalidModules = strings.Replace(invalidModules, string(m), "", 1)
	}
	if len(invalidModules) > 0 {
		return "", errors.New("Unable to parse --modules flag, unrecognized or duplicate modules: " + invalidModules)
	}
	return modules, nil
}

// processConfig checks the configuration values and performs cleanup on
// incorrect-but-allowed values.
func processConfig(config Config) (Config, error) {
	var err error
	config.Siad.APIaddr = processNetAddr(config.Siad.APIaddr)
	config.Siad.RPCaddr = processNetAddr(config.Siad.RPCaddr)
	config.Siad.HostAddr = processNetAddr(config.Siad.HostAddr)
	config.Siad.Modules, err = processModules(config.Siad.Modules)
	if err != nil {
		return Config{}, err
	}
	return config, nil
}

// startDaemonCmd uses the config parameters to start siad.
func startDaemon(config Config) (err error) {
	// Print a startup message.
	fmt.Println("Loading...")
	loadStart := time.Now()

	// Create all of the modules.
	i := 0
	var g modules.Gateway
	if strings.Contains(config.Siad.Modules, "g") {
		i++
		fmt.Printf("(%d/%d) Loading gateway...\n", i, len(config.Siad.Modules))
		g, err = gateway.New(config.Siad.RPCaddr, filepath.Join(config.Siad.SiaDir, modules.GatewayDir))
		if err != nil {
			return err
		}
	}
	var cs modules.ConsensusSet
	if strings.Contains(config.Siad.Modules, "c") {
		i++
		fmt.Printf("(%d/%d) Loading consensus...\n", i, len(config.Siad.Modules))
		cs, err = consensus.New(g, filepath.Join(config.Siad.SiaDir, modules.ConsensusDir))
		if err != nil {
			return err
		}
	}
	var e modules.Explorer
	if strings.Contains(config.Siad.Modules, "e") {
		i++
		fmt.Printf("(%d/%d) Loading explorer...\n", i, len(config.Siad.Modules))
		e, err = explorer.New(cs, filepath.Join(config.Siad.SiaDir, modules.ExplorerDir))
		if err != nil {
			return err
		}
	}
	var tpool modules.TransactionPool
	if strings.Contains(config.Siad.Modules, "t") {
		i++
		fmt.Printf("(%d/%d) Loading transaction pool...\n", i, len(config.Siad.Modules))
		tpool, err = transactionpool.New(cs, g)
		if err != nil {
			return err
		}
	}
	var w modules.Wallet
	if strings.Contains(config.Siad.Modules, "w") {
		i++
		fmt.Printf("(%d/%d) Loading wallet...\n", i, len(config.Siad.Modules))
		w, err = wallet.New(cs, tpool, filepath.Join(config.Siad.SiaDir, modules.WalletDir))
		if err != nil {
			return err
		}
	}
	var m modules.Miner
	if strings.Contains(config.Siad.Modules, "m") {
		i++
		fmt.Printf("(%d/%d) Loading miner...\n", i, len(config.Siad.Modules))
		m, err = miner.New(cs, tpool, w, filepath.Join(config.Siad.SiaDir, modules.MinerDir))
		if err != nil {
			return err
		}
	}
	var h modules.Host
	if strings.Contains(config.Siad.Modules, "h") {
		i++
		fmt.Printf("(%d/%d) Loading host...\n", i, len(config.Siad.Modules))
		h, err = host.New(cs, tpool, w, config.Siad.HostAddr, filepath.Join(config.Siad.SiaDir, modules.HostDir))
		if err != nil {
			return err
		}
	}
	var r modules.Renter
	if strings.Contains(config.Siad.Modules, "r") {
		i++
		fmt.Printf("(%d/%d) Loading renter...\n", i, len(config.Siad.Modules))
		r, err = renter.New(cs, w, tpool, filepath.Join(config.Siad.SiaDir, modules.RenterDir))
		if err != nil {
			return err
		}
	}
	srv, err := api.NewServer(
		config.Siad.APIaddr,
		config.Siad.RequiredUserAgent,
		cs,
		e,
		g,
		h,
		m,
		r,
		tpool,
		w,
	)
	if err != nil {
		return err
	}

	// Bootstrap to the network.
	if !config.Siad.NoBootstrap && g != nil {
		// connect to 3 random bootstrap nodes
		perm, err := crypto.Perm(len(modules.BootstrapPeers))
		if err != nil {
			return err
		}
		for _, i := range perm[:3] {
			go g.Connect(modules.BootstrapPeers[i])
		}
	}

	// Print a 'startup complete' message.
	startupTime := time.Since(loadStart)
	fmt.Println("Finished loading in", startupTime.Seconds(), "seconds")

	// Start serving api requests.
	err = srv.Serve()
	if err != nil {
		return err
	}
	return nil
}

// startDaemonCmd is a passthrough function for startDaemon.
func startDaemonCmd(cmd *cobra.Command, _ []string) {
	// Create the profiling directory if profiling is enabled.
	if globalConfig.Siad.Profile {
		go profile.StartContinuousProfile(globalConfig.Siad.ProfileDir)
	}

	// Process the config variables after they are parsed by cobra.
	config, err := processConfig(globalConfig)
	if err != nil {
		fmt.Fprintln(os.Stderr, err)
		cmd.Usage()
		os.Exit(exitCodeUsage)
	}

	// Start siad. startDaemon will only return when it is shutting down.
	err = startDaemon(config)
	if err != nil {
		die(err)
	}
}
<|endoftext|>


<commit_before>package auth

import (
	"bytes"
	"encoding/json"
	"errors"
	"github.com/guregu/kami"
	"github.com/the-information/ori/account"
	"github.com/the-information/ori/config"
	"github.com/the-information/ori/rest"
	"golang.org/x/net/context"
	"google.golang.org/appengine/log"
	"io/ioutil"
	"net/http"
)

var authKey = "__auth_ctx"

var ErrNotInAuthContext = errors.New("That context object was not run through auth.Middleware!")

// Middleware sets up the request context so account information can be
// retrieved with auth.GetAccount(ctx). It panics if config.Get(ctx) fails.
func Middleware(ctx context.Context, w http.ResponseWriter, r *http.Request) context.Context {

	var conf config.Global
	if err := config.Get(ctx, &conf); err != nil {
		panic("Could not get AuthSecret: " + err.Error())
	}
	claimSet, err := Decode([]byte(r.Header.Get("Authorization")), []byte(conf.AuthSecret))
	if err != nil {
		return context.WithValue(ctx, authKey, err)
	} else if claimSet == SuperClaimSet {
		return context.WithValue(ctx, authKey, &account.Super)
	} else if claimSet == NobodyClaimSet {
		return context.WithValue(ctx, authKey, &account.Nobody)
	}

	var acct account.Account
	if err := account.Get(ctx, claimSet.Sub, &acct); err != nil {
		rest.WriteJSON(w, &rest.Response{
			Code: http.StatusUnauthorized,
			Body: &rest.Message{"Could not retrieve account with key " + claimSet.Sub + ": " + err.Error()},
		})
		return nil
	} else {
		return context.WithValue(ctx, authKey, &acct)
	}

}

var forbiddenMessage = "You do not have permission to access this resource."

type key int

// An AuthCheck is a handler-like function that determines whether a given HTTP request ought to be
// allowed to continue executing based on the request's authentication status. They can be
// used with Check to check credentials on a route without having to do so in the
// core HTTP handler for the route, which is very convenient.
type AuthCheck func(ctx context.Context, w http.ResponseWriter, r *http.Request) bool

// Super is an AuthCheck that grants access to the superuser (that is, the user
// identified by `Authorization: <secret>`).
func Super(ctx context.Context, w http.ResponseWriter, r *http.Request) bool {

	var acct account.Account
	GetAccount(ctx, &acct)
	return acct.Super()

}

// HasRole returns an AuthCheck that grants access if `account.HasRole(role)` is true.
func HasRole(role string) AuthCheck {

	return func(ctx context.Context, w http.ResponseWriter, r *http.Request) bool {

		var acct account.Account
		GetAccount(ctx, &acct)
		return acct.HasRole(role)

	}

}

// AccountMatchesParam returns an AuthCheck that grants access if paramName is the same
// as the account's ID; so, for instance, on a route to /accounts/:accountId, with
// a request to /accounts/asdf, the AuthCheck will return true if the account's ID is asdf.
func AccountMatchesParam(paramName string) AuthCheck {

	return func(ctx context.Context, w http.ResponseWriter, r *http.Request) bool {

		var acct account.Account
		err := GetAccount(ctx, &acct)
		return err == nil && acct.Key(ctx).Encode() == kami.Param(ctx, paramName)

	}

}

// AccountOwnsObject returns an AuthCheck that reads the request body and grants
// access if the request body is an object that has the property `property` and its
// value is the same as the account's encoded datastore key. So, for instance,
// on a request with a body shaped like `{"owner": "asdf"}`, AccountOwnsObject("owner")
// will return true if the authorized account's encoded datastore key is `asdf`.
func AccountOwnsObject(property string) AuthCheck {

	return func(ctx context.Context, w http.ResponseWriter, r *http.Request) bool {

		if r.Body == nil {
			return false
		}

		var acct account.Account
		err := GetAccount(ctx, &acct)
		if err != nil {
			return false
		} else if acct.Super() {
			return true
		}

		// buffer the http body so we can look at it
		buf := bytes.Buffer{}
		buf.ReadFrom(r.Body)

		r.Body = ioutil.NopCloser(&buf)
		data := buf.Bytes()

		props := make(map[string]interface{}, 8)

		if err := json.Unmarshal(data, props); err != nil {
			return false
		}

		val, ok := props[property]
		if !ok {
			return false
		}

		return val.(string) == acct.Key(ctx).Encode()
	}

}

// Checker is an object returned by auth.Check.
type Checker struct {
	checks []AuthCheck
}

// Check produces a Checker for a set of AuthChecks. If all of the AuthChecks
// fail, Checker.Then will return a 403 to the user and prevent the underlying
// handler from being called.
func Check(checks ...AuthCheck) *Checker {
	return &Checker{checks: checks}
}

// Then produces a kami.Handler that wraps another handler with authentication magic.
// If every AuthCheck associated with the Checker fails, Then will reject the HTTP request
// with a 403. If even one of them passes, Then will call h.
// This produces a very pleasant syntax for authenticating routes:
// 	func listAccounts(ctx context.Context, w http.ResponseWriter, r *http.Request) {
//		w.Write([]byte("[]"))
//	}
//
// 	kami.Get("/accounts", auth.Check(auth.Super, auth.HasRole("admin")).Then(listAccounts))
func (c *Checker) Then(h kami.HandlerFunc) kami.HandlerFunc {

	return func(ctx context.Context, w http.ResponseWriter, r *http.Request) {

		var acct account.Account
		GetAccount(ctx, &acct)

		// run the checks
		passed := false
		for _, check := range c.checks {
			if check(ctx, w, r) {
				passed = true
			}
		}

		// did one of them pass?
		if passed {
			log.Infof(ctx, "%s: access granted", acct.Email)
			// mark this as a context which passed authentication
			ctx = context.WithValue(ctx, "__auth_check_ctx", "passed")
			h(ctx, w, r)
		} else {

			log.Warningf(ctx, "%s: access denied", acct.Email)
			rest.WriteJSON(w, &rest.Response{
				Code: http.StatusForbidden,
				Body: &rest.Message{forbiddenMessage},
			})

		}

	}

}

// GetAccount retrieves the authorized account for ctx and copies it into account.
// It returns the error if one was encountered.
// Note that for special user types (account.Nobody and account.Super) both
// the returned key and error will be nil.
func GetAccount(ctx context.Context, acct *account.Account) error {

	switch t := ctx.Value(authKey).(type) {
	case error:
		return t
	case *account.Account:
		*acct = *t
		return nil
	default:
		return ErrNotInAuthContext
	}

}
<commit_msg>add auth.CheckAll<commit_after>package auth

import (
	"bytes"
	"encoding/json"
	"errors"
	"github.com/guregu/kami"
	"github.com/the-information/ori/account"
	"github.com/the-information/ori/config"
	"github.com/the-information/ori/rest"
	"golang.org/x/net/context"
	"google.golang.org/appengine/log"
	"io/ioutil"
	"net/http"
)

var (
	authKey = "__auth_ctx"
)

var ErrNotInAuthContext = errors.New("That context object was not run through auth.Middleware!")

// Middleware sets up the request context so account information can be
// retrieved with auth.GetAccount(ctx). It panics if config.Get(ctx) fails.
func Middleware(ctx context.Context, w http.ResponseWriter, r *http.Request) context.Context {

	var conf config.Global
	if err := config.Get(ctx, &conf); err != nil {
		panic("Could not get AuthSecret: " + err.Error())
	}
	claimSet, err := Decode([]byte(r.Header.Get("Authorization")), []byte(conf.AuthSecret))
	if err != nil {
		return context.WithValue(ctx, authKey, err)
	} else if claimSet == SuperClaimSet {
		return context.WithValue(ctx, authKey, &account.Super)
	} else if claimSet == NobodyClaimSet {
		return context.WithValue(ctx, authKey, &account.Nobody)
	}

	var acct account.Account
	if err := account.Get(ctx, claimSet.Sub, &acct); err != nil {
		rest.WriteJSON(w, &rest.Response{
			Code: http.StatusUnauthorized,
			Body: &rest.Message{"Could not retrieve account with key " + claimSet.Sub + ": " + err.Error()},
		})
		return nil
	} else {
		return context.WithValue(ctx, authKey, &acct)
	}

}

var forbiddenMessage = "You do not have permission to access this resource."

type key int

// An AuthCheck is a handler-like function that determines whether a given HTTP request ought to be
// allowed to continue executing based on the request's authentication status. They can be
// used with Check to check credentials on a route without having to do so in the
// core HTTP handler for the route, which is very convenient.
type AuthCheck func(ctx context.Context, w http.ResponseWriter, r *http.Request) bool

// Super is an AuthCheck that grants access to the superuser (that is, the user
// identified by `Authorization: <secret>`).
func Super(ctx context.Context, w http.ResponseWriter, r *http.Request) bool {

	var acct account.Account
	GetAccount(ctx, &acct)
	return acct.Super()

}

// HasRole returns an AuthCheck that grants access if `account.HasRole(role)` is true.
func HasRole(role string) AuthCheck {

	return func(ctx context.Context, w http.ResponseWriter, r *http.Request) bool {

		var acct account.Account
		GetAccount(ctx, &acct)
		return acct.HasRole(role)

	}

}

// AccountMatchesParam returns an AuthCheck that grants access if paramName is the same
// as the account's ID; so, for instance, on a route to /accounts/:accountId, with
// a request to /accounts/asdf, the AuthCheck will return true if the account's ID is asdf.
func AccountMatchesParam(paramName string) AuthCheck {

	return func(ctx context.Context, w http.ResponseWriter, r *http.Request) bool {

		var acct account.Account
		err := GetAccount(ctx, &acct)
		return err == nil && acct.Key(ctx).Encode() == kami.Param(ctx, paramName)

	}

}

// AccountOwnsObject returns an AuthCheck that reads the request body and grants
// access if the request body is an object that has the property `property` and its
// value is the same as the account's encoded datastore key. So, for instance,
// on a request with a body shaped like `{"owner": "asdf"}`, AccountOwnsObject("owner")
// will return true if the authorized account's encoded datastore key is `asdf`.
func AccountOwnsObject(property string) AuthCheck {

	return func(ctx context.Context, w http.ResponseWriter, r *http.Request) bool {

		if r.Body == nil {
			return false
		}

		var acct account.Account
		err := GetAccount(ctx, &acct)
		if err != nil {
			return false
		} else if acct.Super() {
			return true
		}

		// buffer the http body so we can look at it
		buf := bytes.Buffer{}
		buf.ReadFrom(r.Body)

		r.Body = ioutil.NopCloser(&buf)
		data := buf.Bytes()

		props := make(map[string]interface{}, 8)

		if err := json.Unmarshal(data, props); err != nil {
			return false
		}

		val, ok := props[property]
		if !ok {
			return false
		}

		return val.(string) == acct.Key(ctx).Encode()
	}

}

// Checker is an object returned by auth.Check.
type Checker struct {
	checks []AuthCheck
	all    bool
}

// Check produces a Checker for a set of AuthChecks. If none of the AuthChecks
// pass, Checker.Then will return a 403 to the user and prevent the underlying
// handler from being called.
func Check(checks ...AuthCheck) *Checker {
	return &Checker{checks: checks}
}

// CheckAll produces a Checker for a set of AuthChecks. If any of the AuthChecks
// fail, Checker.Then will return a 403 to the user and prevent the underlying
// handler from being called.
func CheckAll(checks ...AuthCheck) *Checker {
	return &Checker{checks: checks, all: true}
}

// Then produces a kami.Handler that wraps another handler with authentication magic.
// If every AuthCheck associated with the Checker fails, Then will reject the HTTP request
// with a 403. If only one of them passes, Then will call h.
// This produces a very pleasant syntax for authenticating routes:
// 	func listAccounts(ctx context.Context, w http.ResponseWriter, r *http.Request) {
//		w.Write([]byte("[]"))
//	}
//
// 	kami.Get("/accounts", auth.Check(auth.Super, auth.HasRole("admin")).Then(listAccounts))
func (c *Checker) Then(h kami.HandlerFunc) kami.HandlerFunc {

	return func(ctx context.Context, w http.ResponseWriter, r *http.Request) {

		var acct account.Account
		GetAccount(ctx, &acct)

		// run the checks
		passed := false
		for _, check := range c.checks {
			if check(ctx, w, r) {
				passed = true
			} else if c.all {
				passed = false
				break
			}
		}

		// did one of them pass?
		if passed {
			log.Infof(ctx, "%s: access granted", acct.Email)
			// mark this as a context which passed authentication
			ctx = context.WithValue(ctx, "__auth_check_ctx", "passed")
			h(ctx, w, r)
		} else {

			log.Warningf(ctx, "%s: access denied", acct.Email)
			rest.WriteJSON(w, &rest.Response{
				Code: http.StatusForbidden,
				Body: &rest.Message{forbiddenMessage},
			})

		}

	}

}

// GetAccount retrieves the authorized account for ctx and copies it into account.
// It returns the error if one was encountered.
// Note that for special user types (account.Nobody and account.Super) both
// the returned key and error will be nil.
func GetAccount(ctx context.Context, acct *account.Account) error {

	switch t := ctx.Value(authKey).(type) {
	case error:
		return t
	case *account.Account:
		*acct = *t
		return nil
	default:
		return ErrNotInAuthContext
	}

}
<|endoftext|>


<commit_before>package simra

import (
	"io"

	mp3 "github.com/hajimehoshi/go-mp3"
	"github.com/hajimehoshi/oto"
	"golang.org/x/mobile/asset"
)

type audio struct {
	isClosed chan bool
}

// Audioer is an interface for treating audio
type Audioer interface {
	Play(resource asset.File, loop bool, doneCallback func(err error)) error
	Stop() error
}

// NewAudio returns new audio instance that implements Audioer interface
func NewAudio() Audioer {
	return &audio{
		isClosed: make(chan bool, 1),
	}
}

func (a *audio) Play(resource asset.File, loop bool, doneCallback func(err error)) error {
	dec, err := mp3.NewDecoder(resource)
	if err != nil {
		return err
	}

	player, err := oto.NewPlayer(dec.SampleRate(), 2, 2, 8192)
	if err != nil {
		return err
	}

	doneChan := make(chan error)
	go func() {
		doneChan <- a.doPlay(player, dec, loop)
	}()

	go func() {
		defer func() {
			err := dec.Close()
			if err != nil {
				LogError(err.Error())
			}
			err = player.Close()
			if err != nil {
				LogError(err.Error())
			}
		}()

		<-doneChan
		doneCallback(err)
	}()

	return nil
}

func (a *audio) doPlay(player *oto.Player, r io.ReadSeeker, loop bool) error {
	var offset, written int64
	var err error
	readByte := (int64)(4096 * 30)
loop:
	for {
	playback:
		for {
			r.Seek(offset, io.SeekStart)
			select {
			case <-a.isClosed:
				readByte = 0
				r.Seek(0, io.SeekEnd)
			default:
				// for non-blocking
			}
			written, err = io.CopyN(player, r, readByte)
			if err != nil || written == 0 {
				// error or EOF
				break playback
			}
			offset += written
		}
		if !loop || err != io.EOF {
			break loop
		}
		offset = 0
	}
	return err
}

func (a *audio) Stop() error {
	a.isClosed <- true
	return nil
}
<commit_msg>optimize audio playback. remove unnecessary seeking<commit_after>package simra

import (
	"io"

	mp3 "github.com/hajimehoshi/go-mp3"
	"github.com/hajimehoshi/oto"
	"golang.org/x/mobile/asset"
)

type audio struct {
	isClosed chan bool
}

// Audioer is an interface for treating audio
type Audioer interface {
	Play(resource asset.File, loop bool, doneCallback func(err error)) error
	Stop() error
}

// NewAudio returns new audio instance that implements Audioer interface
func NewAudio() Audioer {
	return &audio{
		isClosed: make(chan bool, 1),
	}
}

func (a *audio) Play(resource asset.File, loop bool, doneCallback func(err error)) error {
	dec, err := mp3.NewDecoder(resource)
	if err != nil {
		return err
	}

	player, err := oto.NewPlayer(dec.SampleRate(), 2, 2, 8192)
	if err != nil {
		return err
	}

	doneChan := make(chan error)
	go func() {
		doneChan <- a.doPlay(player, dec, loop)
	}()

	go func() {
		defer func() {
			err := dec.Close()
			if err != nil {
				LogError(err.Error())
			}
			err = player.Close()
			if err != nil {
				LogError(err.Error())
			}
		}()

		<-doneChan
		doneCallback(err)
	}()

	return nil
}

func (a *audio) doPlay(player *oto.Player, r io.ReadSeeker, loop bool) error {
	var written int64
	var err error
	readByte := (int64)(8192)
loop:
	for {
		r.Seek(0, io.SeekStart)
	playback:
		for {
			select {
			case <-a.isClosed:
				readByte = 0
				r.Seek(0, io.SeekEnd)
			default:
				// for non-blocking
			}
			written, err = io.CopyN(player, r, readByte)
			if err != nil || written == 0 {
				// error or EOF
				break playback
			}
		}
		if !loop || err != io.EOF {
			break loop
		}
	}
	return err
}

func (a *audio) Stop() error {
	a.isClosed <- true
	return nil
}
<|endoftext|>


<commit_before>// since.go

package main

import (
	"flag"
	"fmt"
	"github.com/MichaelTJones/walk"
	"log"
	"os"
	"runtime"
	"sort"
	"sync"
	"time"
)

/* REVISION HISTORY
   21 Oct 2018 -- First started playing w/ MichaelTJones' code.  I added a help flag
   21 Oct 2022 -- In the code again, after running golangci-lint.  Changed how help flag contents is output.  If an absolute time is given, use that, else use duration.
*/

var LastAlteredDate = "Oct 21, 2022"

//var duration = flag.String("d", "", "find files modified within DURATION")
var duration = flag.Duration("dur", 5*time.Minute, "find files modified within this duration")
var format = flag.String("f", "2006-01-02 03:04:05", "time format")
var instant = flag.String("t", "", "find files modified since TIME")
var quiet = flag.Bool("q", false, "do not print filenames")
var verbose = flag.Bool("v", false, "print summary statistics")
var days = flag.Int("d", 0, "days duration")
var weeks = flag.Int("w", 0, "weeks duration")

//var help = flag.Bool("h", false, "print help message")

func main() {

	execName, _ := os.Executable()
	ExecFI, _ := os.Stat(execName)
	ExecTimeStamp := ExecFI.ModTime().Format("Mon Jan-2-2006_15:04:05 MST")

	flag.Usage = func() {
		fmt.Fprintf(flag.CommandLine.Output(), " %s last modified %s, compiled with %s, last linked %s.\n", os.Args[0], LastAlteredDate, runtime.Version(), ExecTimeStamp)
		fmt.Fprintf(flag.CommandLine.Output(), " Usage: since <options> <start-dir-list> \n")
		fmt.Fprintf(flag.CommandLine.Output(), " Valid time units for duration are ns, us, ms, s, m, h. \n")
		fmt.Fprintf(flag.CommandLine.Output(), " since -d 5m -- show all files changed within last 5 minutes starting at current directory \n")
		fmt.Fprintf(flag.CommandLine.Output(), " since -d 5m $HOME or %%userprofile -- show all files changed within last 5 minutes starting at home directory \n")
		flag.PrintDefaults()
	}
	flag.Parse()

	fmt.Printf(" since written in Go.  LastAltered %s, compiled with %s, last linked %s.\n", LastAlteredDate, runtime.Version(), ExecTimeStamp)

	now := time.Now()
	when := now
	switch {
	case *instant != "":
		t, err := time.Parse(*format, *instant)
		if err != nil {
			fmt.Printf("error parsing time %q, %s\n", *instant, err)
			os.Exit(1)
		}
		when = t
	default:
		//d, err := time.ParseDuration(*duration)
		//if err != nil {
		//	fmt.Printf("error parsing duration %q, %s\n", *duration, err)
		//	os.Exit(2)
		//}
		*duration = *duration + time.Duration(*weeks)*7*24*time.Hour + time.Duration(*days)*24*time.Hour
		when = now.Add(-*duration) // subtract duration from now.
	}

	if *verbose {
		fmt.Printf(" weeks = %d, days = %d, duration = %s\n", *weeks, *days, *duration)
		fmt.Printf(" when = %s\n", when)
	}

	// goroutine to collect names of recently-modified files
	var result []string
	done := make(chan bool)
	results := make(chan string, 1024)
	go func() {
		for r := range results {
			result = append(result, r)
		}
		sort.Strings(result) // simulate ordered traversal
		done <- true
	}()

	// parallel walker and walk to find recently-modified files
	var lock sync.Mutex
	var tFiles, tBytes int // total files and bytes
	var rFiles, rBytes int // recent files and bytes
	sizeVisitor := func(path string, info os.FileInfo, err error) error {
		if err == nil {
			lock.Lock()
			tFiles += 1
			tBytes += int(info.Size())
			lock.Unlock()

			if info.ModTime().After(when) {
				lock.Lock()
				rFiles += 1
				rBytes += int(info.Size())
				lock.Unlock()

				if !*quiet {
					// fmt.Printf("%s %s\n", info.ModTime(), path) // simple
					results <- path // allows sorting into "normal" order
				}
			}
		}
		return nil
	}
	if len(flag.Args()) < 1 {
		dir, err := os.Getwd()
		if err != nil {
			log.Fatalln(" error from Getwd is", err)
		}
		err = walk.Walk(dir, sizeVisitor)
		if err != nil {
			log.Fatalln(" error from walk.Walk is", err)
		}
	} else {
		for _, root := range flag.Args() {
			err := walk.Walk(root, sizeVisitor)
			if err != nil {
				log.Fatalln(" error from walk.Walk is", err)
			}
		}
	}

	// wait for traversal results and print
	close(results) // no more results
	<-done         // wait for final results and sorting
	𝛥t := float64(time.Since(now)) / 1e9

	for _, r := range result {
		fmt.Printf("%s\n", r)
	}

	// print optional verbose summary report
	if *verbose {
		log.Printf("     total: %8d files (%7.2f%%), %13d bytes (%7.2f%%)\n",
			tFiles, 100.0, tBytes, 100.0)

		rfp := 100 * float64(rFiles) / float64(tFiles)
		rbp := 100 * float64(rBytes) / float64(tBytes)
		log.Printf("    recent: %8d files (%7.2f%%), %13d bytes (%7.2f%%) in %.4f seconds\n",
			rFiles, rfp, rBytes, rbp, 𝛥t)
	}
}
<commit_msg>10/26/2022 07:30:59 since/since.go<commit_after>// since.go

package main

import (
	"flag"
	"fmt"
	"github.com/MichaelTJones/walk"
	"log"
	"os"
	"runtime"
	"sort"
	"strings"
	"sync"
	"time"
)

/* REVISION HISTORY
   21 Oct 2018 -- First started playing w/ MichaelTJones' code.  I added a help flag
   21 Oct 2022 -- In the code again, after running golangci-lint.  Changed how help flag contents is output.  If an absolute time is given, use that, else use duration.
   26 Oct 2022 -- Added '~' processing
*/

var LastAlteredDate = "Oct 26, 2022"

//var duration = flag.String("d", "", "find files modified within DURATION")
var duration = flag.Duration("dur", 5*time.Minute, "find files modified within this duration")
var format = flag.String("f", "2006-01-02 03:04:05", "time format")
var instant = flag.String("t", "", "find files modified since TIME")
var quiet = flag.Bool("q", false, "do not print filenames")
var verbose = flag.Bool("v", false, "print summary statistics")
var days = flag.Int("d", 0, "days duration")
var weeks = flag.Int("w", 0, "weeks duration")

//var help = flag.Bool("h", false, "print help message")

func main() {

	execName, _ := os.Executable()
	ExecFI, _ := os.Stat(execName)
	ExecTimeStamp := ExecFI.ModTime().Format("Mon Jan-2-2006_15:04:05 MST")

	flag.Usage = func() {
		fmt.Fprintf(flag.CommandLine.Output(), " %s last modified %s, compiled with %s, last linked %s.\n", os.Args[0], LastAlteredDate, runtime.Version(), ExecTimeStamp)
		fmt.Fprintf(flag.CommandLine.Output(), " Usage: since <options> <start-dir-list> \n")
		fmt.Fprintf(flag.CommandLine.Output(), " Valid time units for duration are ns, us, ms, s, m, h. \n")
		fmt.Fprintf(flag.CommandLine.Output(), " since -d 5m -- show all files changed within last 5 minutes starting at current directory \n")
		fmt.Fprintf(flag.CommandLine.Output(), " since -d 5m $HOME or %%userprofile -- show all files changed within last 5 minutes starting at home directory \n")
		flag.PrintDefaults()
	}
	flag.Parse()

	fmt.Printf(" since written in Go.  LastAltered %s, compiled with %s, last linked %s.\n", LastAlteredDate, runtime.Version(), ExecTimeStamp)

	now := time.Now()
	when := now
	switch {
	case *instant != "":
		t, err := time.Parse(*format, *instant)
		if err != nil {
			fmt.Printf("error parsing time %q, %s\n", *instant, err)
			os.Exit(1)
		}
		when = t
	default:
		//d, err := time.ParseDuration(*duration)
		//if err != nil {
		//	fmt.Printf("error parsing duration %q, %s\n", *duration, err)
		//	os.Exit(2)
		//}
		*duration = *duration + time.Duration(*weeks)*7*24*time.Hour + time.Duration(*days)*24*time.Hour
		when = now.Add(-*duration) // subtract duration from now.
	}

	home, err := os.UserHomeDir()
	if err != nil {
		fmt.Fprintf(os.Stderr, " Error from UserHomeDir is %v.\n", err)
	}

	if *verbose {
		fmt.Printf(" weeks = %d, days = %d, duration = %s\n", *weeks, *days, *duration)
		fmt.Printf(" when = %s\n", when)
	}

	// goroutine to collect names of recently-modified files
	var result []string
	done := make(chan bool)
	results := make(chan string, 1024)
	go func() {
		for r := range results {
			result = append(result, r)
		}
		sort.Strings(result) // simulate ordered traversal
		done <- true
	}()

	// parallel walker and walk to find recently-modified files
	var lock sync.Mutex
	var tFiles, tBytes int // total files and bytes
	var rFiles, rBytes int // recent files and bytes
	sizeVisitor := func(path string, info os.FileInfo, err error) error {
		if err == nil {
			lock.Lock()
			tFiles += 1
			tBytes += int(info.Size())
			lock.Unlock()

			if info.ModTime().After(when) {
				lock.Lock()
				rFiles += 1
				rBytes += int(info.Size())
				lock.Unlock()

				if !*quiet {
					// fmt.Printf("%s %s\n", info.ModTime(), path) // simple
					results <- path // allows sorting into "normal" order
				}
			}
		}
		return nil
	}
	if len(flag.Args()) < 1 {
		dir, err := os.Getwd()
		if err != nil {
			log.Fatalln(" error from Getwd is", err)
		}
		err = walk.Walk(dir, sizeVisitor)
		if err != nil {
			log.Fatalln(" error from walk.Walk is", err)
		}
	} else {
		for _, root := range flag.Args() {
			dir := strings.Replace(root, "~", home, 1) // I decided to not test for windows or presence of ~.  This is pretty fast as it is.
			err := walk.Walk(dir, sizeVisitor)
			if err != nil {
				log.Fatalln(" error from walk.Walk is", err)
			}
		}
	}

	// wait for traversal results and print
	close(results) // no more results
	<-done         // wait for final results and sorting
	𝛥t := float64(time.Since(now)) / 1e9

	for _, r := range result {
		fmt.Printf("%s\n", r)
	}

	// print optional verbose summary report
	if *verbose {
		log.Printf("     total: %8d files (%7.2f%%), %13d bytes (%7.2f%%)\n",
			tFiles, 100.0, tBytes, 100.0)

		rfp := 100 * float64(rFiles) / float64(tFiles)
		rbp := 100 * float64(rBytes) / float64(tBytes)
		log.Printf("    recent: %8d files (%7.2f%%), %13d bytes (%7.2f%%) in %.4f seconds\n",
			rFiles, rfp, rBytes, rbp, 𝛥t)
	}
}
<|endoftext|>