<commit_before>// Copyright 2016 The LUCI Authors.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//      http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package main

import (
	"context"
	"time"

	"github.com/golang/protobuf/ptypes"
	"go.chromium.org/luci/common/clock"
	"go.chromium.org/luci/common/data/rand/mathrand"
	"go.chromium.org/luci/common/errors"
	"go.chromium.org/luci/common/gcloud/gs"
	"go.chromium.org/luci/common/logging"
	log "go.chromium.org/luci/common/logging"
	"go.chromium.org/luci/common/tsmon/distribution"
	"go.chromium.org/luci/common/tsmon/field"
	"go.chromium.org/luci/common/tsmon/metric"
	montypes "go.chromium.org/luci/common/tsmon/types"
	"go.chromium.org/luci/logdog/api/config/svcconfig"
	logdog "go.chromium.org/luci/logdog/api/endpoints/coordinator/services/v1"
	"go.chromium.org/luci/logdog/appengine/coordinator"
	"go.chromium.org/luci/logdog/common/types"
	"go.chromium.org/luci/logdog/server/archivist"
	"go.chromium.org/luci/logdog/server/bundleServicesClient"
	"go.chromium.org/luci/logdog/server/service"

	"go.chromium.org/luci/hardcoded/chromeinfra"
)

var (
	errInvalidConfig = errors.New("invalid configuration")

	errNoWorkToDo = errors.New("no work to do")

	// batchSize is the number of jobs to lease from taskqueue per cycle.
	// TaskQueue has a limit of 10qps for leasing tasks, so the batch size must be set to:
	// batchSize * 10 > (max expected stream creation QPS)
	// In 2019, max stream QPS is approximately 400 QPS
	// TODO(hinoka): Make these configurable in settings.
	batchSize = int64(50)

	// leaseTime is the amount of time to to lease the batch of tasks for.
	// We need:
	// (time to process avg stream) * batchSize < leaseTime
	// As of 2019, 90th percentile process time per stream is ~4s.
	leaseTime = 15 * time.Minute

	// maxSleepTime is the max amount of time to sleep in-between errors, in seconds.
	maxSleepTime = 32

	// tsTaskProcessingTime measures the amount of time spent processing a single
	// task.
	//
	// The "consumed" field is true if the underlying task was consumed and
	// false if it was not.
	tsTaskProcessingTime = metric.NewCumulativeDistribution("logdog/archivist/task_processing_time_ms_ng",
		"The amount of time (in milliseconds) that a single task takes to process in the new pipeline.",
		&montypes.MetricMetadata{Units: montypes.Milliseconds},
		distribution.DefaultBucketer,
		field.Bool("consumed"))

	tsLoopCycleTime = metric.NewCumulativeDistribution("logdog/archivist/loop_cycle_time_ms",
		"The amount of time a single batch of leases takes to process.",
		&montypes.MetricMetadata{Units: montypes.Milliseconds},
		distribution.DefaultBucketer)

	tsLeaseCount = metric.NewCounter("logdog/archivist/tasks_leased",
		"Number of tasks leased.",
		nil)

	tsNackCount = metric.NewCounter("logdog/archivist/tasks_not_acked",
		"Number of tasks leased but failed.",
		nil)

	tsAckCount = metric.NewCounter("logdog/archivist/tasks_acked",
		"Number of tasks successfully completed and acked.",
		nil)
)

// application is the Archivist application state.
type application struct {
	service.Service
}

// runForever runs the archivist loop forever.
func runForever(c context.Context, ar archivist.Archivist) error {
	sleepTime := 1
	client := ar.Service
	// Forever loop.
	for {
		// Only exit out if the context is cancelled (I.E. Ctrl + c).
		if c.Err() != nil {
			return c.Err()
		}
		if err := func() error {
			cycleStartTime := clock.Now(c)
			leaseTimeProto := ptypes.DurationProto(leaseTime)
			nc, _ := context.WithTimeout(c, leaseTime)

			tasks, err := client.LeaseArchiveTasks(c, &logdog.LeaseRequest{
				MaxTasks:  batchSize,
				LeaseTime: leaseTimeProto,
			})
			if err != nil {
				return err
			}
			logging.Fields{
				"cycleStart": true,
				"numLeased":  len(tasks.Tasks),
				"noWork":     len(tasks.Tasks) == 0,
			}.Infof(c, "Start of cycled, leased %d tasks", len(tasks.Tasks))
			tsLeaseCount.Add(c, int64(len(tasks.Tasks)))
			if len(tasks.Tasks) == 0 {
				return errNoWorkToDo
			}
			merr := errors.NewLazyMultiError(len(tasks.Tasks))
			ackTasks := make([]*logdog.ArchiveTask, 0, len(tasks.Tasks))
			for i, task := range tasks.Tasks {
				deleteTask := false
				startTime := clock.Now(c)
				if err := ar.ArchiveTask(nc, task); err != nil {
					merr.Assign(i, err)
				} else { // err == nil
					ackTasks = append(ackTasks, task)
					deleteTask = true
				}
				duration := clock.Now(c).Sub(startTime)
				tsTaskProcessingTime.Add(c, float64(duration.Nanoseconds())/1000000, deleteTask)
			}
			tsNackCount.Add(c, int64(len(tasks.Tasks)-len(ackTasks)))
			// Ack the successful tasks.  We log the error here since there's nothing we can do.
			if len(ackTasks) > 0 {
				tsAckCount.Add(c, int64(len(ackTasks)))
				c := log.SetFields(c, log.Fields{"Tasks": ackTasks})
				if _, err := client.DeleteArchiveTasks(c, &logdog.DeleteRequest{Tasks: ackTasks}); err != nil {
					log.WithError(err).Errorf(c, "error while acking tasks (%s)", ackTasks)
				}
			}

			duration := clock.Now(c).Sub(cycleStartTime)
			logging.Fields{
				"cycleEnd":     true,
				"numLeased":    len(tasks.Tasks),
				"numCompleted": len(ackTasks),
				"timeSpentSec": duration.Seconds(),
			}.Infof(c, "Done archiving %d items (%d successful) took %.2fs", len(tasks.Tasks), len(ackTasks), duration.Seconds())
			tsLoopCycleTime.Add(c, float64(duration.Nanoseconds()/1000000))
			return merr.Get()
		}(); err != nil {
			// Back off on errors.
			sleepTime *= 2
			if sleepTime > maxSleepTime {
				sleepTime = maxSleepTime
			}
			switch err {
			case errNoWorkToDo:
				logging.Infof(c, "no work to do, sleeping for %d seconds", sleepTime)
			default:
				logging.WithError(err).Errorf(c, "got error in loop, sleeping for %d seconds", sleepTime)
			}
			clock.Sleep(c, time.Duration(sleepTime)*time.Second)
		} else {
			sleepTime = 1
		}
	}
}

// settingsUpdater updates the settings from datastore every once in a while.
func settingsUpdater(c context.Context) {
	for {
		set := coordinator.GetSettings(c)
		logging.Debugf(c, "updating settings: %v", set)
		if batchSize != set.ArchivistBatchSize {
			batchSize = set.ArchivistBatchSize
		}
		if leaseTime != set.ArchivistLeaseTime {
			leaseTime = set.ArchivistLeaseTime
		}
		clock.Sleep(c, 5*time.Minute)
	}
}

// run is the main execution function.
func (a *application) runArchivist(c context.Context) error {
	cfg := a.ServiceConfig()

	coordCfg, acfg := cfg.GetCoordinator(), cfg.GetArchivist()
	switch {
	case coordCfg == nil:
		fallthrough

	case acfg == nil:
		return errors.New("missing required config: archivist")
	case acfg.GsStagingBucket == "":
		return errors.New("missing required config: archivist.gs_staging_bucket")
	}

	// Initialize our Storage.
	//
	// NOTE: We're requesting read/write access even though we only need read-only
	// access because GKE doesn't understand the read-only scope:
	// https://www.googleapis.com/auth/bigtable.readonly
	st, err := a.IntermediateStorage(c, true)
	if err != nil {
		log.WithError(err).Errorf(c, "Failed to get storage instance.")
		return err
	}
	defer st.Close()

	// Defines our Google Storage client project scoped factory.
	gsClientFactory := func(ctx context.Context, proj types.ProjectName) (gs.Client, error) {
		gsClient, err := a.GSClient(ctx, proj)
		if err != nil {
			log.WithError(err).Errorf(c, "Failed to get Google Storage client.")
			return nil, err
		}
		return gsClient, nil
	}

	// Initialize a Coordinator client that bundles requests together.
	coordClient := &bundleServicesClient.Client{
		ServicesClient:       a.Coordinator(),
		DelayThreshold:       time.Second,
		BundleCountThreshold: 100,
	}
	defer coordClient.Flush()

	ar := archivist.Archivist{
		Service:         coordClient,
		SettingsLoader:  a.GetSettingsLoader(acfg),
		Storage:         st,
		GSClientFactory: gsClientFactory,
	}

	// Application shutdown will now operate by stopping the Iterator.
	c, cancelFunc := context.WithCancel(c)
	defer cancelFunc()

	// Application shutdown will now operate by cancelling the Archivist's
	// shutdown Context.
	a.SetShutdownFunc(cancelFunc)

	// Update our settings periodicall.
	go settingsUpdater(c)

	return runForever(c, ar)
}

// GetSettingsLoader is an archivist.SettingsLoader implementation that merges
// global and project-specific settings.
//
// The resulting settings object will be verified by the Archivist.
func (a *application) GetSettingsLoader(acfg *svcconfig.Archivist) archivist.SettingsLoader {
	serviceID := a.ServiceID()

	return func(c context.Context, proj types.ProjectName) (*archivist.Settings, error) {
		// Fold in our project-specific configuration, if valid.
		pcfg, err := a.ProjectConfig(c, proj)
		if err != nil {
			log.Fields{
				log.ErrorKey: err,
				"project":    proj,
			}.Errorf(c, "Failed to fetch project configuration.")
			return nil, err
		}

		indexParam := func(get func(ic *svcconfig.ArchiveIndexConfig) int32) int {
			if ic := pcfg.ArchiveIndexConfig; ic != nil {
				if v := get(ic); v > 0 {
					return int(v)
				}
			}

			if ic := acfg.ArchiveIndexConfig; ic != nil {
				if v := get(ic); v > 0 {
					return int(v)
				}
			}

			return 0
		}

		// Load our base settings.
		//
		// Archival bases are:
		// Staging: gs://<services:gs_staging_bucket>/<project-id>/...
		// Archive: gs://<project:archive_gs_bucket>/<project-id>/...
		st := archivist.Settings{
			GSBase:        gs.MakePath(pcfg.ArchiveGsBucket, "").Concat(serviceID),
			GSStagingBase: gs.MakePath(acfg.GsStagingBucket, "").Concat(serviceID),

			IndexStreamRange: indexParam(func(ic *svcconfig.ArchiveIndexConfig) int32 { return ic.StreamRange }),
			IndexPrefixRange: indexParam(func(ic *svcconfig.ArchiveIndexConfig) int32 { return ic.PrefixRange }),
			IndexByteRange:   indexParam(func(ic *svcconfig.ArchiveIndexConfig) int32 { return ic.ByteRange }),
			AlwaysRender:     (acfg.RenderAllStreams || pcfg.RenderAllStreams),
		}

		// Fold project settings into loaded ones.
		return &st, nil
	}
}

// Entry point.
func main() {
	mathrand.SeedRandomly()
	a := application{
		Service: service.Service{
			Name:               "archivist",
			DefaultAuthOptions: chromeinfra.DefaultAuthOptions(),
		},
	}
	a.Run(context.Background(), a.runArchivist)
}
<commit_msg>[logdog] Fix likely archivist memory leak.<commit_after>// Copyright 2016 The LUCI Authors.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//      http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package main

import (
	"context"
	"time"

	"github.com/golang/protobuf/ptypes"
	"go.chromium.org/luci/common/clock"
	"go.chromium.org/luci/common/data/rand/mathrand"
	"go.chromium.org/luci/common/errors"
	"go.chromium.org/luci/common/gcloud/gs"
	"go.chromium.org/luci/common/logging"
	log "go.chromium.org/luci/common/logging"
	"go.chromium.org/luci/common/tsmon/distribution"
	"go.chromium.org/luci/common/tsmon/field"
	"go.chromium.org/luci/common/tsmon/metric"
	montypes "go.chromium.org/luci/common/tsmon/types"
	"go.chromium.org/luci/logdog/api/config/svcconfig"
	logdog "go.chromium.org/luci/logdog/api/endpoints/coordinator/services/v1"
	"go.chromium.org/luci/logdog/appengine/coordinator"
	"go.chromium.org/luci/logdog/common/types"
	"go.chromium.org/luci/logdog/server/archivist"
	"go.chromium.org/luci/logdog/server/bundleServicesClient"
	"go.chromium.org/luci/logdog/server/service"

	"go.chromium.org/luci/hardcoded/chromeinfra"
)

var (
	errInvalidConfig = errors.New("invalid configuration")

	errNoWorkToDo = errors.New("no work to do")

	// batchSize is the number of jobs to lease from taskqueue per cycle.
	// TaskQueue has a limit of 10qps for leasing tasks, so the batch size must be set to:
	// batchSize * 10 > (max expected stream creation QPS)
	// In 2019, max stream QPS is approximately 400 QPS
	// TODO(hinoka): Make these configurable in settings.
	batchSize = int64(50)

	// leaseTime is the amount of time to to lease the batch of tasks for.
	// We need:
	// (time to process avg stream) * batchSize < leaseTime
	// As of 2019, 90th percentile process time per stream is ~4s.
	leaseTime = 15 * time.Minute

	// maxSleepTime is the max amount of time to sleep in-between errors, in seconds.
	maxSleepTime = 32

	// tsTaskProcessingTime measures the amount of time spent processing a single
	// task.
	//
	// The "consumed" field is true if the underlying task was consumed and
	// false if it was not.
	tsTaskProcessingTime = metric.NewCumulativeDistribution("logdog/archivist/task_processing_time_ms_ng",
		"The amount of time (in milliseconds) that a single task takes to process in the new pipeline.",
		&montypes.MetricMetadata{Units: montypes.Milliseconds},
		distribution.DefaultBucketer,
		field.Bool("consumed"))

	tsLoopCycleTime = metric.NewCumulativeDistribution("logdog/archivist/loop_cycle_time_ms",
		"The amount of time a single batch of leases takes to process.",
		&montypes.MetricMetadata{Units: montypes.Milliseconds},
		distribution.DefaultBucketer)

	tsLeaseCount = metric.NewCounter("logdog/archivist/tasks_leased",
		"Number of tasks leased.",
		nil)

	tsNackCount = metric.NewCounter("logdog/archivist/tasks_not_acked",
		"Number of tasks leased but failed.",
		nil)

	tsAckCount = metric.NewCounter("logdog/archivist/tasks_acked",
		"Number of tasks successfully completed and acked.",
		nil)
)

// application is the Archivist application state.
type application struct {
	service.Service
}

// runForever runs the archivist loop forever.
func runForever(c context.Context, ar archivist.Archivist) error {
	sleepTime := 1
	client := ar.Service
	// Forever loop.
	for {
		// Only exit out if the context is cancelled (I.E. Ctrl + c).
		if c.Err() != nil {
			return c.Err()
		}
		if err := func() error {
			cycleStartTime := clock.Now(c)
			leaseTimeProto := ptypes.DurationProto(leaseTime)
			nc, cancel := context.WithTimeout(c, leaseTime)
			defer cancel()

			tasks, err := client.LeaseArchiveTasks(c, &logdog.LeaseRequest{
				MaxTasks:  batchSize,
				LeaseTime: leaseTimeProto,
			})
			if err != nil {
				return err
			}
			logging.Fields{
				"cycleStart": true,
				"numLeased":  len(tasks.Tasks),
				"noWork":     len(tasks.Tasks) == 0,
			}.Infof(c, "Start of cycled, leased %d tasks", len(tasks.Tasks))
			tsLeaseCount.Add(c, int64(len(tasks.Tasks)))
			if len(tasks.Tasks) == 0 {
				return errNoWorkToDo
			}
			merr := errors.NewLazyMultiError(len(tasks.Tasks))
			ackTasks := make([]*logdog.ArchiveTask, 0, len(tasks.Tasks))
			for i, task := range tasks.Tasks {
				deleteTask := false
				startTime := clock.Now(c)
				if err := ar.ArchiveTask(nc, task); err != nil {
					merr.Assign(i, err)
				} else { // err == nil
					ackTasks = append(ackTasks, task)
					deleteTask = true
				}
				duration := clock.Now(c).Sub(startTime)
				tsTaskProcessingTime.Add(c, float64(duration.Nanoseconds())/1000000, deleteTask)
			}
			tsNackCount.Add(c, int64(len(tasks.Tasks)-len(ackTasks)))
			// Ack the successful tasks.  We log the error here since there's nothing we can do.
			if len(ackTasks) > 0 {
				tsAckCount.Add(c, int64(len(ackTasks)))
				c := log.SetFields(c, log.Fields{"Tasks": ackTasks})
				if _, err := client.DeleteArchiveTasks(c, &logdog.DeleteRequest{Tasks: ackTasks}); err != nil {
					log.WithError(err).Errorf(c, "error while acking tasks (%s)", ackTasks)
				}
			}

			duration := clock.Now(c).Sub(cycleStartTime)
			logging.Fields{
				"cycleEnd":     true,
				"numLeased":    len(tasks.Tasks),
				"numCompleted": len(ackTasks),
				"timeSpentSec": duration.Seconds(),
			}.Infof(c, "Done archiving %d items (%d successful) took %.2fs", len(tasks.Tasks), len(ackTasks), duration.Seconds())
			tsLoopCycleTime.Add(c, float64(duration.Nanoseconds()/1000000))
			return merr.Get()
		}(); err != nil {
			// Back off on errors.
			sleepTime *= 2
			if sleepTime > maxSleepTime {
				sleepTime = maxSleepTime
			}
			switch err {
			case errNoWorkToDo:
				logging.Infof(c, "no work to do, sleeping for %d seconds", sleepTime)
			default:
				logging.WithError(err).Errorf(c, "got error in loop, sleeping for %d seconds", sleepTime)
			}
			clock.Sleep(c, time.Duration(sleepTime)*time.Second)
		} else {
			sleepTime = 1
		}
	}
}

// settingsUpdater updates the settings from datastore every once in a while.
func settingsUpdater(c context.Context) {
	for {
		set := coordinator.GetSettings(c)
		logging.Debugf(c, "updating settings: %v", set)
		if batchSize != set.ArchivistBatchSize {
			batchSize = set.ArchivistBatchSize
		}
		if leaseTime != set.ArchivistLeaseTime {
			leaseTime = set.ArchivistLeaseTime
		}
		clock.Sleep(c, 5*time.Minute)
	}
}

// run is the main execution function.
func (a *application) runArchivist(c context.Context) error {
	cfg := a.ServiceConfig()

	coordCfg, acfg := cfg.GetCoordinator(), cfg.GetArchivist()
	switch {
	case coordCfg == nil:
		fallthrough

	case acfg == nil:
		return errors.New("missing required config: archivist")
	case acfg.GsStagingBucket == "":
		return errors.New("missing required config: archivist.gs_staging_bucket")
	}

	// Initialize our Storage.
	//
	// NOTE: We're requesting read/write access even though we only need read-only
	// access because GKE doesn't understand the read-only scope:
	// https://www.googleapis.com/auth/bigtable.readonly
	st, err := a.IntermediateStorage(c, true)
	if err != nil {
		log.WithError(err).Errorf(c, "Failed to get storage instance.")
		return err
	}
	defer st.Close()

	// Defines our Google Storage client project scoped factory.
	gsClientFactory := func(ctx context.Context, proj types.ProjectName) (gs.Client, error) {
		gsClient, err := a.GSClient(ctx, proj)
		if err != nil {
			log.WithError(err).Errorf(c, "Failed to get Google Storage client.")
			return nil, err
		}
		return gsClient, nil
	}

	// Initialize a Coordinator client that bundles requests together.
	coordClient := &bundleServicesClient.Client{
		ServicesClient:       a.Coordinator(),
		DelayThreshold:       time.Second,
		BundleCountThreshold: 100,
	}
	defer coordClient.Flush()

	ar := archivist.Archivist{
		Service:         coordClient,
		SettingsLoader:  a.GetSettingsLoader(acfg),
		Storage:         st,
		GSClientFactory: gsClientFactory,
	}

	// Application shutdown will now operate by stopping the Iterator.
	c, cancelFunc := context.WithCancel(c)
	defer cancelFunc()

	// Application shutdown will now operate by cancelling the Archivist's
	// shutdown Context.
	a.SetShutdownFunc(cancelFunc)

	// Update our settings periodicall.
	go settingsUpdater(c)

	return runForever(c, ar)
}

// GetSettingsLoader is an archivist.SettingsLoader implementation that merges
// global and project-specific settings.
//
// The resulting settings object will be verified by the Archivist.
func (a *application) GetSettingsLoader(acfg *svcconfig.Archivist) archivist.SettingsLoader {
	serviceID := a.ServiceID()

	return func(c context.Context, proj types.ProjectName) (*archivist.Settings, error) {
		// Fold in our project-specific configuration, if valid.
		pcfg, err := a.ProjectConfig(c, proj)
		if err != nil {
			log.Fields{
				log.ErrorKey: err,
				"project":    proj,
			}.Errorf(c, "Failed to fetch project configuration.")
			return nil, err
		}

		indexParam := func(get func(ic *svcconfig.ArchiveIndexConfig) int32) int {
			if ic := pcfg.ArchiveIndexConfig; ic != nil {
				if v := get(ic); v > 0 {
					return int(v)
				}
			}

			if ic := acfg.ArchiveIndexConfig; ic != nil {
				if v := get(ic); v > 0 {
					return int(v)
				}
			}

			return 0
		}

		// Load our base settings.
		//
		// Archival bases are:
		// Staging: gs://<services:gs_staging_bucket>/<project-id>/...
		// Archive: gs://<project:archive_gs_bucket>/<project-id>/...
		st := archivist.Settings{
			GSBase:        gs.MakePath(pcfg.ArchiveGsBucket, "").Concat(serviceID),
			GSStagingBase: gs.MakePath(acfg.GsStagingBucket, "").Concat(serviceID),

			IndexStreamRange: indexParam(func(ic *svcconfig.ArchiveIndexConfig) int32 { return ic.StreamRange }),
			IndexPrefixRange: indexParam(func(ic *svcconfig.ArchiveIndexConfig) int32 { return ic.PrefixRange }),
			IndexByteRange:   indexParam(func(ic *svcconfig.ArchiveIndexConfig) int32 { return ic.ByteRange }),
			AlwaysRender:     (acfg.RenderAllStreams || pcfg.RenderAllStreams),
		}

		// Fold project settings into loaded ones.
		return &st, nil
	}
}

// Entry point.
func main() {
	mathrand.SeedRandomly()
	a := application{
		Service: service.Service{
			Name:               "archivist",
			DefaultAuthOptions: chromeinfra.DefaultAuthOptions(),
		},
	}
	a.Run(context.Background(), a.runArchivist)
}
<|endoftext|>


<commit_before>package handlers

import (
	"net/http"
	"net/http/httptest"
	"net/url"
	"strings"
	"testing"
	"time"

	"github.com/golang/protobuf/jsonpb"
	"github.com/golang/protobuf/ptypes"

	"pixur.org/pixur/schema"
	"pixur.org/pixur/status"
	"pixur.org/pixur/tasks"
)

func TestGetRefreshTokenFailsOnNonPost(t *testing.T) {
	s := httptest.NewServer(&GetRefreshTokenHandler{})
	defer s.Close()

	res, err := (&testClient{}).Get(s.URL)
	if err != nil {
		t.Fatal(err)
	}
	defer res.Body.Close()

	if have, want := res.StatusCode, http.StatusMethodNotAllowed; have != want {
		t.Error("have", have, "want", want)
	}
}

func TestGetRefreshTokenFailsOnMissingXsrf(t *testing.T) {
	s := httptest.NewServer(&GetRefreshTokenHandler{})
	defer s.Close()

	res, err := (&testClient{
		DisableXSRF: true,
	}).PostForm(s.URL, url.Values{})
	if err != nil {
		t.Fatal(err)
	}
	defer res.Body.Close()

	if have, want := res.StatusCode, http.StatusUnauthorized; have != want {
		t.Error("have", have, "want", want)
	}
	if have, want := bodyToText(res.Body), "xsrf cookie"; !strings.Contains(have, want) {
		t.Error("have", have, "want", want)
	}
}

func TestGetRefreshTokenSucceedsOnIdentSecret(t *testing.T) {
	var taskCap *tasks.AuthUserTask
	successRunner := func(task tasks.Task) status.S {
		taskCap = task.(*tasks.AuthUserTask)
		taskCap.NewTokenID = 3
		taskCap.User = &schema.User{
			UserId: 2,
		}
		return nil
	}
	s := httptest.NewServer(&GetRefreshTokenHandler{
		Runner: tasks.TestTaskRunner(successRunner),
		Now:    time.Now,
	})
	defer s.Close()

	res, err := (&testClient{}).PostForm(s.URL, url.Values{
		"ident":  []string{"a"},
		"secret": []string{"b"},
	})
	if err != nil {
		t.Fatal(err)
	}
	defer res.Body.Close()

	if have, want := res.StatusCode, http.StatusOK; have != want {
		t.Error("have", have, "want", want)
	}

	resp := new(GetRefreshTokenResponse)
	if err := jsonpb.Unmarshal(res.Body, resp); err != nil {
		t.Error(err)
	}
	if resp.RefreshToken != "" || resp.AuthToken != "" || resp.PixToken != "" {
		t.Error("tokens should have been removed", resp)
	}
	if resp.RefreshPayload.Subject != "2" || resp.RefreshPayload.TokenId != 3 {
		t.Error("wrong token ids", resp)
	}
	if taskCap == nil {
		t.Fatal("task didn't run")
	}
	if taskCap.Email != "a" || taskCap.Secret != "b" {
		t.Error("wrong task input", taskCap.Email, taskCap.Secret)
	}

	// Cookie verification
	cs := make(map[string]*http.Cookie)
	for _, c := range res.Cookies() {
		cs[c.Name] = c
	}
	if len(cs) != 3 {
		t.Error("expected 3 cookies", cs)
	}
	refreshCookie, ok := cs[refreshPwtCookieName]
	if !ok || refreshCookie.Value == "" {
		t.Error("missing", refreshPwtCookieName, "have", refreshCookie, ok)
	}
	if have, want := refreshCookie.Path, "/api/getRefreshToken"; have != want {
		t.Error("have", have, "want", want)
	}
	if !refreshCookie.Secure || !refreshCookie.HttpOnly {
		t.Error("expected secure and httponly", refreshCookie)
	}
	if !within(time.Unix(resp.RefreshPayload.NotAfter.Seconds, 0), refreshCookie.Expires, time.Minute) {
		t.Error("wrong expires", resp.RefreshPayload.NotAfter, " != ", refreshCookie.Expires)
	}

	authCookie, ok := cs[authPwtCookieName]
	if !ok || authCookie.Value == "" {
		t.Error("missing", authPwtCookieName, "have", authCookie, ok)
	}
	if have, want := authCookie.Path, "/api/"; have != want {
		t.Error("have", have, "want", want)
	}
	if !authCookie.Secure || !authCookie.HttpOnly {
		t.Error("expected secure and httponly", authCookie)
	}
	if !within(time.Unix(resp.AuthPayload.NotAfter.Seconds, 0), authCookie.Expires, time.Minute) {
		t.Error("wrong expires", resp.AuthPayload.NotAfter, " != ", authCookie.Expires)
	}

	pixCookie, ok := cs[pixPwtCookieName]
	if !ok || pixCookie.Value == "" {
		t.Error("missing", pixPwtCookieName, "have", pixCookie, ok)
	}
	if have, want := pixCookie.Path, "/pix/"; have != want {
		t.Error("have", have, "want", want)
	}
	if !pixCookie.Secure || !pixCookie.HttpOnly {
		t.Error("expected secure and httponly", pixCookie)
	}
	if !within(time.Unix(resp.PixPayload.NotAfter.Seconds, 0), pixCookie.Expires, time.Minute) {
		t.Error("wrong expires", resp.PixPayload.NotAfter, " != ", pixCookie.Expires)
	}
}

func TestGetRefreshTokenSucceedsOnRefreshToken(t *testing.T) {
	var taskCap *tasks.AuthUserTask
	successRunner := func(task tasks.Task) status.S {
		taskCap = task.(*tasks.AuthUserTask)
		taskCap.NewTokenID = 3
		taskCap.User = &schema.User{
			UserId: 2,
		}
		return nil
	}
	s := httptest.NewServer(&GetRefreshTokenHandler{
		Runner: tasks.TestTaskRunner(successRunner),
		Now:    time.Now,
	})
	defer s.Close()

	req, err := http.NewRequest("POST", s.URL, nil)
	if err != nil {
		t.Fatal(err)
	}
	cookie, payload := testRefreshToken()
	req.AddCookie(cookie)
	res, err := (&testClient{}).Do(req)
	if err != nil {
		t.Fatal(err)
	}
	defer res.Body.Close()

	if have, want := res.StatusCode, http.StatusOK; have != want {
		t.Error("have", have, "want", want)
	}

	resp := new(GetRefreshTokenResponse)
	if err := jsonpb.Unmarshal(res.Body, resp); err != nil {
		t.Error(err)
	}
	if resp.RefreshToken != "" || resp.AuthToken != "" || resp.PixToken != "" {
		t.Error("tokens should have been removed", resp)
	}
	if resp.RefreshPayload.Subject != "2" || resp.RefreshPayload.TokenId != 3 {
		t.Error("wrong token ids", resp)
	}
	if taskCap == nil {
		t.Fatal("task didn't run")
	}
	if taskCap.TokenID != payload.TokenId || taskCap.UserID != 9 /* payload.Subject */ {
		t.Error("wrong task input", taskCap.Email, taskCap.Secret)
	}

	// Cookie verification
	cs := make(map[string]*http.Cookie)
	for _, c := range res.Cookies() {
		cs[c.Name] = c
	}
	if len(cs) != 3 {
		t.Error("expected 3 cookies", cs)
	}
	refreshCookie, ok := cs[refreshPwtCookieName]
	if !ok || refreshCookie.Value == "" {
		t.Error("missing", refreshPwtCookieName, "have", refreshCookie, ok)
	}
	if have, want := refreshCookie.Path, "/api/getRefreshToken"; have != want {
		t.Error("have", have, "want", want)
	}
	if !refreshCookie.Secure || !refreshCookie.HttpOnly {
		t.Error("expected secure and httponly", refreshCookie)
	}
	if !within(time.Unix(resp.RefreshPayload.NotAfter.Seconds, 0), refreshCookie.Expires, time.Minute) {
		t.Error("wrong expires", resp.RefreshPayload.NotAfter, " != ", refreshCookie.Expires)
	}

	authCookie, ok := cs[authPwtCookieName]
	if !ok || authCookie.Value == "" {
		t.Error("missing", authPwtCookieName, "have", authCookie, ok)
	}
	if have, want := authCookie.Path, "/api/"; have != want {
		t.Error("have", have, "want", want)
	}
	if !authCookie.Secure || !authCookie.HttpOnly {
		t.Error("expected secure and httponly", authCookie)
	}
	if !within(time.Unix(resp.AuthPayload.NotAfter.Seconds, 0), authCookie.Expires, time.Minute) {
		t.Error("wrong expires", resp.AuthPayload.NotAfter, " != ", authCookie.Expires)
	}

	pixCookie, ok := cs[pixPwtCookieName]
	if !ok || pixCookie.Value == "" {
		t.Error("missing", pixPwtCookieName, "have", pixCookie, ok)
	}
	if have, want := pixCookie.Path, "/pix/"; have != want {
		t.Error("have", have, "want", want)
	}
	if !pixCookie.Secure || !pixCookie.HttpOnly {
		t.Error("expected secure and httponly", pixCookie)
	}
	if !within(time.Unix(resp.PixPayload.NotAfter.Seconds, 0), pixCookie.Expires, time.Minute) {
		t.Error("wrong expires", resp.PixPayload.NotAfter, " != ", pixCookie.Expires)
	}
}

func within(t1, t2 time.Time, diff time.Duration) bool {
	d := t1.Sub(t2)
	if d < 0 {
		d = -d
	}
	return d <= diff
}

func testRefreshToken() (*http.Cookie, *PwtPayload) {
	notafter, _ := ptypes.TimestampProto(time.Now().Add(refreshPwtDuration))
	notbefore, _ := ptypes.TimestampProto(time.Now().Add(-1 * time.Minute))
	payload := &PwtPayload{
		Subject:   "9",
		NotAfter:  notafter,
		NotBefore: notbefore,
		Type:      PwtPayload_REFRESH,
		TokenId:   10,
	}
	refreshToken, err := defaultPwtCoder.encode(payload)
	if err != nil {
		panic(err)
	}
	return &http.Cookie{
		Name:  refreshPwtCookieName,
		Value: string(refreshToken),
	}, payload
}
<commit_msg>handlers: add refresh token tests<commit_after>package handlers

import (
	"context"
	"net/http"
	"net/http/httptest"
	"net/url"
	"strings"
	"testing"
	"time"

	"github.com/golang/protobuf/jsonpb"
	"github.com/golang/protobuf/proto"
	"github.com/golang/protobuf/ptypes"
	tspb "github.com/golang/protobuf/ptypes/timestamp"

	"pixur.org/pixur/schema"
	"pixur.org/pixur/status"
	"pixur.org/pixur/tasks"
)

func TestGetRefreshTokenFailsOnNonPost(t *testing.T) {
	s := httptest.NewServer(&GetRefreshTokenHandler{})
	defer s.Close()

	res, err := (&testClient{}).Get(s.URL)
	if err != nil {
		t.Fatal(err)
	}
	defer res.Body.Close()

	if have, want := res.StatusCode, http.StatusMethodNotAllowed; have != want {
		t.Error("have", have, "want", want)
	}
}

func TestGetRefreshTokenFailsOnMissingXsrf(t *testing.T) {
	s := httptest.NewServer(&GetRefreshTokenHandler{})
	defer s.Close()

	res, err := (&testClient{
		DisableXSRF: true,
	}).PostForm(s.URL, url.Values{})
	if err != nil {
		t.Fatal(err)
	}
	defer res.Body.Close()

	if have, want := res.StatusCode, http.StatusUnauthorized; have != want {
		t.Error("have", have, "want", want)
	}
	if have, want := bodyToText(res.Body), "xsrf cookie"; !strings.Contains(have, want) {
		t.Error("have", have, "want", want)
	}
}

func TestGetRefreshTokenSucceedsOnIdentSecret(t *testing.T) {
	var taskCap *tasks.AuthUserTask
	successRunner := func(task tasks.Task) status.S {
		taskCap = task.(*tasks.AuthUserTask)
		taskCap.NewTokenID = 3
		taskCap.User = &schema.User{
			UserId: 2,
		}
		return nil
	}
	s := httptest.NewServer(&GetRefreshTokenHandler{
		Runner: tasks.TestTaskRunner(successRunner),
		Now:    time.Now,
	})
	defer s.Close()

	res, err := (&testClient{}).PostForm(s.URL, url.Values{
		"ident":  []string{"a"},
		"secret": []string{"b"},
	})
	if err != nil {
		t.Fatal(err)
	}
	defer res.Body.Close()

	if have, want := res.StatusCode, http.StatusOK; have != want {
		t.Error("have", have, "want", want)
	}

	resp := new(GetRefreshTokenResponse)
	if err := jsonpb.Unmarshal(res.Body, resp); err != nil {
		t.Error(err)
	}
	if resp.RefreshToken != "" || resp.AuthToken != "" || resp.PixToken != "" {
		t.Error("tokens should have been removed", resp)
	}
	if resp.RefreshPayload.Subject != "2" || resp.RefreshPayload.TokenId != 3 {
		t.Error("wrong token ids", resp)
	}
	if taskCap == nil {
		t.Fatal("task didn't run")
	}
	if taskCap.Email != "a" || taskCap.Secret != "b" {
		t.Error("wrong task input", taskCap.Email, taskCap.Secret)
	}

	// Cookie verification
	cs := make(map[string]*http.Cookie)
	for _, c := range res.Cookies() {
		cs[c.Name] = c
	}
	if len(cs) != 3 {
		t.Error("expected 3 cookies", cs)
	}
	refreshCookie, ok := cs[refreshPwtCookieName]
	if !ok || refreshCookie.Value == "" {
		t.Error("missing", refreshPwtCookieName, "have", refreshCookie, ok)
	}
	if have, want := refreshCookie.Path, "/api/getRefreshToken"; have != want {
		t.Error("have", have, "want", want)
	}
	if !refreshCookie.Secure || !refreshCookie.HttpOnly {
		t.Error("expected secure and httponly", refreshCookie)
	}
	if !within(time.Unix(resp.RefreshPayload.NotAfter.Seconds, 0), refreshCookie.Expires, time.Minute) {
		t.Error("wrong expires", resp.RefreshPayload.NotAfter, " != ", refreshCookie.Expires)
	}

	authCookie, ok := cs[authPwtCookieName]
	if !ok || authCookie.Value == "" {
		t.Error("missing", authPwtCookieName, "have", authCookie, ok)
	}
	if have, want := authCookie.Path, "/api/"; have != want {
		t.Error("have", have, "want", want)
	}
	if !authCookie.Secure || !authCookie.HttpOnly {
		t.Error("expected secure and httponly", authCookie)
	}
	if !within(time.Unix(resp.AuthPayload.NotAfter.Seconds, 0), authCookie.Expires, time.Minute) {
		t.Error("wrong expires", resp.AuthPayload.NotAfter, " != ", authCookie.Expires)
	}

	pixCookie, ok := cs[pixPwtCookieName]
	if !ok || pixCookie.Value == "" {
		t.Error("missing", pixPwtCookieName, "have", pixCookie, ok)
	}
	if have, want := pixCookie.Path, "/pix/"; have != want {
		t.Error("have", have, "want", want)
	}
	if !pixCookie.Secure || !pixCookie.HttpOnly {
		t.Error("expected secure and httponly", pixCookie)
	}
	if !within(time.Unix(resp.PixPayload.NotAfter.Seconds, 0), pixCookie.Expires, time.Minute) {
		t.Error("wrong expires", resp.PixPayload.NotAfter, " != ", pixCookie.Expires)
	}
}

func TestGetRefreshTokenSucceedsOnRefreshToken(t *testing.T) {
	var taskCap *tasks.AuthUserTask
	successRunner := func(task tasks.Task) status.S {
		taskCap = task.(*tasks.AuthUserTask)
		taskCap.NewTokenID = 3
		taskCap.User = &schema.User{
			UserId: 2,
		}
		return nil
	}
	s := httptest.NewServer(&GetRefreshTokenHandler{
		Runner: tasks.TestTaskRunner(successRunner),
		Now:    time.Now,
	})
	defer s.Close()

	req, err := http.NewRequest("POST", s.URL, nil)
	if err != nil {
		t.Fatal(err)
	}
	cookie, payload := testRefreshToken()
	req.AddCookie(cookie)
	res, err := (&testClient{}).Do(req)
	if err != nil {
		t.Fatal(err)
	}
	defer res.Body.Close()

	if have, want := res.StatusCode, http.StatusOK; have != want {
		t.Error("have", have, "want", want)
	}

	resp := new(GetRefreshTokenResponse)
	if err := jsonpb.Unmarshal(res.Body, resp); err != nil {
		t.Error(err)
	}
	if resp.RefreshToken != "" || resp.AuthToken != "" || resp.PixToken != "" {
		t.Error("tokens should have been removed", resp)
	}
	if resp.RefreshPayload.Subject != "2" || resp.RefreshPayload.TokenId != 3 {
		t.Error("wrong token ids", resp)
	}
	if taskCap == nil {
		t.Fatal("task didn't run")
	}
	if taskCap.TokenID != payload.TokenId || taskCap.UserID != 9 /* payload.Subject */ {
		t.Error("wrong task input", taskCap.Email, taskCap.Secret)
	}

	// Cookie verification
	cs := make(map[string]*http.Cookie)
	for _, c := range res.Cookies() {
		cs[c.Name] = c
	}
	if len(cs) != 3 {
		t.Error("expected 3 cookies", cs)
	}
	refreshCookie, ok := cs[refreshPwtCookieName]
	if !ok || refreshCookie.Value == "" {
		t.Error("missing", refreshPwtCookieName, "have", refreshCookie, ok)
	}
	if have, want := refreshCookie.Path, "/api/getRefreshToken"; have != want {
		t.Error("have", have, "want", want)
	}
	if !refreshCookie.Secure || !refreshCookie.HttpOnly {
		t.Error("expected secure and httponly", refreshCookie)
	}
	if !within(time.Unix(resp.RefreshPayload.NotAfter.Seconds, 0), refreshCookie.Expires, time.Minute) {
		t.Error("wrong expires", resp.RefreshPayload.NotAfter, " != ", refreshCookie.Expires)
	}

	authCookie, ok := cs[authPwtCookieName]
	if !ok || authCookie.Value == "" {
		t.Error("missing", authPwtCookieName, "have", authCookie, ok)
	}
	if have, want := authCookie.Path, "/api/"; have != want {
		t.Error("have", have, "want", want)
	}
	if !authCookie.Secure || !authCookie.HttpOnly {
		t.Error("expected secure and httponly", authCookie)
	}
	if !within(time.Unix(resp.AuthPayload.NotAfter.Seconds, 0), authCookie.Expires, time.Minute) {
		t.Error("wrong expires", resp.AuthPayload.NotAfter, " != ", authCookie.Expires)
	}

	pixCookie, ok := cs[pixPwtCookieName]
	if !ok || pixCookie.Value == "" {
		t.Error("missing", pixPwtCookieName, "have", pixCookie, ok)
	}
	if have, want := pixCookie.Path, "/pix/"; have != want {
		t.Error("have", have, "want", want)
	}
	if !pixCookie.Secure || !pixCookie.HttpOnly {
		t.Error("expected secure and httponly", pixCookie)
	}
	if !within(time.Unix(resp.PixPayload.NotAfter.Seconds, 0), pixCookie.Expires, time.Minute) {
		t.Error("wrong expires", resp.PixPayload.NotAfter, " != ", pixCookie.Expires)
	}
}

func TestGetRefreshTokenFailsOnInvalidToken(t *testing.T) {
	h := &GetRefreshTokenHandler{}
	_, sts := h.GetRefreshToken(context.Background(), &GetRefreshTokenRequest{
		RefreshToken: "invalid",
	})

	if have, want := sts.Code(), status.Code_UNAUTHENTICATED; have != want {
		t.Error("have", have, "want", want)
	}

	if have, want := sts.Message(), "can't decode token"; !strings.Contains(have, want) {
		t.Error("have", have, "want", want)
	}
}

func TestGetRefreshTokenFailsOnNonRefreshToken(t *testing.T) {
	h := &GetRefreshTokenHandler{}
	notafter, _ := ptypes.TimestampProto(time.Now().Add(refreshPwtDuration))
	notbefore, _ := ptypes.TimestampProto(time.Now().Add(-1 * time.Minute))
	payload := &PwtPayload{
		Subject:   "9",
		NotAfter:  notafter,
		NotBefore: notbefore,
		Type:      PwtPayload_AUTH,
	}
	refreshToken, err := defaultPwtCoder.encode(payload)
	if err != nil {
		panic(err)
	}

	_, sts := h.GetRefreshToken(context.Background(), &GetRefreshTokenRequest{
		RefreshToken: string(refreshToken),
	})

	if have, want := sts.Code(), status.Code_UNAUTHENTICATED; have != want {
		t.Error("have", have, "want", want)
	}

	if have, want := sts.Message(), "can't decode non refresh token"; !strings.Contains(have, want) {
		t.Error("have", have, "want", want)
	}
}

func TestGetRefreshTokenFailsOnBadSubject(t *testing.T) {
	h := &GetRefreshTokenHandler{}
	notafter, _ := ptypes.TimestampProto(time.Now().Add(refreshPwtDuration))
	notbefore, _ := ptypes.TimestampProto(time.Now().Add(-1 * time.Minute))
	payload := &PwtPayload{
		Subject:   "invalid",
		NotAfter:  notafter,
		NotBefore: notbefore,
		Type:      PwtPayload_REFRESH,
	}
	refreshToken, err := defaultPwtCoder.encode(payload)
	if err != nil {
		panic(err)
	}

	_, sts := h.GetRefreshToken(context.Background(), &GetRefreshTokenRequest{
		RefreshToken: string(refreshToken),
	})

	if have, want := sts.Code(), status.Code_UNAUTHENTICATED; have != want {
		t.Error("have", have, "want", want)
	}

	if have, want := sts.Message(), "can't decode subject"; !strings.Contains(have, want) {
		t.Error("have", have, "want", want)
	}
}

func TestGetRefreshTokenFailsOnTaskError(t *testing.T) {
	failureRunner := func(task tasks.Task) status.S {
		return status.InternalError(nil, "bad")
	}
	h := &GetRefreshTokenHandler{
		Runner: tasks.TestTaskRunner(failureRunner),
	}

	_, sts := h.GetRefreshToken(context.Background(), &GetRefreshTokenRequest{})

	if have, want := sts.Code(), status.Code_INTERNAL_ERROR; have != want {
		t.Error("have", have, "want", want)
	}

	if have, want := sts.Message(), "bad"; !strings.Contains(have, want) {
		t.Error("have", have, "want", want)
	}
}

func TestGetRefreshToken(t *testing.T) {
	var taskCap *tasks.AuthUserTask
	successRunner := func(task tasks.Task) status.S {
		taskCap = task.(*tasks.AuthUserTask)
		taskCap.User = &schema.User{
			UserId: 2,
		}
		taskCap.NewTokenID = 4
		return nil
	}
	notafter, _ := ptypes.TimestampProto(time.Now().Add(refreshPwtDuration))
	notbefore, _ := ptypes.TimestampProto(time.Now().Add(-1 * time.Minute))
	payload := &PwtPayload{
		Subject:   "2",
		NotAfter:  notafter,
		NotBefore: notbefore,
		Type:      PwtPayload_REFRESH,
		TokenId:   3,
	}
	refreshToken, err := defaultPwtCoder.encode(payload)
	if err != nil {
		panic(err)
	}

	h := &GetRefreshTokenHandler{
		Runner: tasks.TestTaskRunner(successRunner),
		Now:    time.Now,
	}

	resp, sts := h.GetRefreshToken(context.Background(), &GetRefreshTokenRequest{
		Ident:        "ident",
		Secret:       "secret",
		RefreshToken: string(refreshToken),
	})
	if sts != nil {
		t.Fatal(err)
	}

	if have, want := taskCap.Email, "ident"; have != want {
		t.Error("have", have, "want", want)
	}
	if have, want := taskCap.Secret, "secret"; have != want {
		t.Error("have", have, "want", want)
	}
	if have, want := taskCap.UserID, int64(2); have != want {
		t.Error("have", have, "want", want)
	}
	if have, want := taskCap.TokenID, int64(3); have != want {
		t.Error("have", have, "want", want)
	}

	if len(resp.RefreshToken) == 0 || len(resp.AuthToken) == 0 || len(resp.PixToken) == 0 {
		t.Error("expected non-empty token", resp.RefreshToken, resp.AuthToken, resp.PixToken)
	}

	if !withinProto(resp.RefreshPayload.NotBefore, time.Now(), time.Minute*2) {
		t.Error("wrong before", resp.RefreshPayload.NotBefore)
	}
	if !withinProto(resp.RefreshPayload.NotAfter, time.Now().Add(refreshPwtDuration), time.Minute) {
		t.Error("wrong after", resp.RefreshPayload.NotAfter)
	}
	resp.RefreshPayload.NotBefore = nil
	resp.RefreshPayload.NotAfter = nil
	expectedRefresh := &PwtPayload{
		Subject: "2",
		TokenId: 4,
		Type:    PwtPayload_REFRESH,
	}
	if !proto.Equal(resp.RefreshPayload, expectedRefresh) {
		t.Error("have", resp.RefreshPayload, "want", expectedRefresh)
	}

	if !withinProto(resp.AuthPayload.NotBefore, time.Now(), time.Minute*2) {
		t.Error("wrong before", resp.AuthPayload.NotBefore)
	}
	if !withinProto(resp.AuthPayload.NotAfter, time.Now().Add(authPwtDuration), time.Minute) {
		t.Error("wrong after", resp.AuthPayload.NotAfter)
	}
	resp.AuthPayload.NotBefore = nil
	resp.AuthPayload.NotAfter = nil
	expectedAuth := &PwtPayload{
		Subject:       "2",
		TokenParentId: 4,
		Type:          PwtPayload_AUTH,
	}
	if !proto.Equal(resp.AuthPayload, expectedAuth) {
		t.Error("have", resp.AuthPayload, "want", expectedAuth)
	}

	if !withinProto(resp.PixPayload.NotBefore, time.Now(), time.Minute*2) {
		t.Error("wrong before", resp.PixPayload.NotBefore)
	}
	if !withinProto(resp.PixPayload.NotAfter, time.Now().Add(refreshPwtDuration), time.Minute) {
		t.Error("wrong after", resp.PixPayload.NotAfter)
	}
	if !withinProto(resp.PixPayload.SoftNotAfter, time.Now().Add(authPwtDuration), time.Minute) {
		t.Error("wrong soft after", resp.PixPayload.SoftNotAfter)
	}
	resp.PixPayload.NotBefore = nil
	resp.PixPayload.NotAfter = nil
	resp.PixPayload.SoftNotAfter = nil
	expectedPix := &PwtPayload{
		Subject:       "2",
		TokenParentId: 4,
		Type:          PwtPayload_PIX,
	}
	if !proto.Equal(resp.PixPayload, expectedPix) {
		t.Error("have", resp.PixPayload, "want", expectedPix)
	}
}

func within(t1, t2 time.Time, diff time.Duration) bool {
	d := t1.Sub(t2)
	if d < 0 {
		d = -d
	}
	return d <= diff
}

func withinProto(t1pb *tspb.Timestamp, t2 time.Time, diff time.Duration) bool {
	t1, err := ptypes.Timestamp(t1pb)
	if err != nil {
		panic(err)
	}
	d := t1.Sub(t2)
	if d < 0 {
		d = -d
	}
	return d <= diff
}

func testRefreshToken() (*http.Cookie, *PwtPayload) {
	notafter, _ := ptypes.TimestampProto(time.Now().Add(refreshPwtDuration))
	notbefore, _ := ptypes.TimestampProto(time.Now().Add(-1 * time.Minute))
	payload := &PwtPayload{
		Subject:   "9",
		NotAfter:  notafter,
		NotBefore: notbefore,
		Type:      PwtPayload_REFRESH,
		TokenId:   10,
	}
	refreshToken, err := defaultPwtCoder.encode(payload)
	if err != nil {
		panic(err)
	}
	return &http.Cookie{
		Name:  refreshPwtCookieName,
		Value: string(refreshToken),
	}, payload
}
<|endoftext|>


<commit_before>package parser

import (
	"fmt"
	"html/template"
	"net/http"

	"ratings/responses"

	"github.com/labstack/echo"
	"github.com/leebenson/conform"
	"github.com/microcosm-cc/bluemonday"
)

type (
	app struct {
		Key     string `json:"key" validate:"required,len=32,alphanum,excludesall= " conform:"lower"`
		Version string `json:"version" validate:"required,alphanum,gte=1,lte=15,excludesall= " conform:"lower"`
	}

	user struct {
		Name   string `json:"name" validate:"required,alphanum,gte=3,slte=70" conform:"name"`
		Email  string `json:"email" validate:"required,email,gte=3,lte=100,excludesall= " conform:"email"`
		MiBAID string `json:"mibaId" validate:"required,alphanum,gte=1,excludesall= " conform:"lower"`
	}

	platform struct {
		Key     string `json:"key" validate:"required,len=32,alphanum,excludesall= " conform:"lower"`
		Version string `json:"version" validate:"required,alphanum,gte=1,lte=15,excludesall= " conform:"lower"`
	}

	device struct {
		Name   string `json:"name" validate:"required,alphanum,gte=1,lte=30" conform:"trim"`
		Brand  string `json:"brand" validate:"required,alphanum,gte=1,lte=30" conform:"trim"`
		Screen screen `json:"screen" validate:"required"`
	}

	screen struct {
		Width  int `json:"width" validate:"required,numeric,gt=0"`
		Height int `json:"height" validate:"required,numeric,gt=0"`
		PPI    int `json:"ppi" validate:"required,numeric,gt=0"`
	}

	browser struct {
		Name    string `json:"name" validate:"required,alphanum,gte=1,lte=15" conform:"trim"`
		Version string `json:"version" validate:"required,alphanum,gte=1,lte=15,excludesall= " conform:"lower"`
	}

	// Request holds the mapped fields from the request's JSON body
	Request struct {
		Rating      uint8    `json:"rating" validate:"required,numeric,min=-127,max=127"`
		Description string   `json:"description" validate:"alphanum,gte=3,lte=30,omitempty" conform:"trim"`
		Comment     string   `json:"comment" validate:"gte=3,lte=1000,omitempty" conform:"trim,ucfirst"`
		Range       string   `json:"range" validate:"required,len=32,alphanum,excludesall= " conform:"lower"`
		App         app      `json:"app" validate:"required"`
		Platform    platform `json:"platform" validate:"required"`
		User        *user    `json:"user" validate:"omitempty"`
		Device      *device  `json:"device" validate:"omitempty"`
		Browser     *browser `json:"browser" validate:"omitempty"`
	}
)

// Parse parses, scrubs and escapes a request's JSON body and maps it to a struct
func Parse(context echo.Context) (*Request, error) {
	request := new(Request)

	if err := bind(request, context); err != nil {
		return request, err
	}

	if err := validate(request, context); err != nil {
		return request, err
	}

	conform.Strings(request)
	sanitizeComment(request)

	return request, nil
}

func bind(request *Request, context echo.Context) error {
	if err := context.Bind(request); err != nil {
		errorMessage := fmt.Sprintf("Error parsing request: %s", err.Error())

		return responses.ErrorResponse(http.StatusBadRequest, errorMessage, context)
	}

	return nil
}

func validate(request *Request, context echo.Context) error {
	if err := context.Validate(request); err != nil {
		errorMessage := fmt.Sprintf("Error validating request: %s", err.Error())

		return responses.ErrorResponse(http.StatusUnprocessableEntity, errorMessage, context)
	}

	return nil
}

func sanitizeComment(request *Request) {
	sanitizer := bluemonday.StrictPolicy()

	request.Comment = sanitizer.Sanitize(request.Comment)
}
<commit_msg>Replaced sanitizer<commit_after>package parser

import (
	"fmt"
	"net/http"

	"ratings/responses"

	"github.com/labstack/echo"
	"github.com/leebenson/conform"
	"github.com/microcosm-cc/bluemonday"
)

type (
	app struct {
		Key     string `json:"key" validate:"required,len=32,alphanum,excludesall= " conform:"lower"`
		Version string `json:"version" validate:"required,alphanum,gte=1,lte=15,excludesall= " conform:"lower"`
	}

	user struct {
		Name   string `json:"name" validate:"required,alphanum,gte=3,slte=70" conform:"name"`
		Email  string `json:"email" validate:"required,email,gte=3,lte=100,excludesall= " conform:"email"`
		MiBAID string `json:"mibaId" validate:"required,alphanum,gte=1,excludesall= " conform:"lower"`
	}

	platform struct {
		Key     string `json:"key" validate:"required,len=32,alphanum,excludesall= " conform:"lower"`
		Version string `json:"version" validate:"required,alphanum,gte=1,lte=15,excludesall= " conform:"lower"`
	}

	device struct {
		Name   string `json:"name" validate:"required,alphanum,gte=1,lte=30" conform:"trim"`
		Brand  string `json:"brand" validate:"required,alphanum,gte=1,lte=30" conform:"trim"`
		Screen screen `json:"screen" validate:"required"`
	}

	screen struct {
		Width  int `json:"width" validate:"required,numeric,gt=0"`
		Height int `json:"height" validate:"required,numeric,gt=0"`
		PPI    int `json:"ppi" validate:"required,numeric,gt=0"`
	}

	browser struct {
		Name    string `json:"name" validate:"required,alphanum,gte=1,lte=15" conform:"trim"`
		Version string `json:"version" validate:"required,alphanum,gte=1,lte=15,excludesall= " conform:"lower"`
	}

	// Request holds the mapped fields from the request's JSON body
	Request struct {
		Rating      uint8    `json:"rating" validate:"required,numeric,min=-127,max=127"`
		Description string   `json:"description" validate:"alphanum,gte=3,lte=30,omitempty" conform:"trim"`
		Comment     string   `json:"comment" validate:"gte=3,lte=1000,omitempty" conform:"trim,ucfirst"`
		Range       string   `json:"range" validate:"required,len=32,alphanum,excludesall= " conform:"lower"`
		App         app      `json:"app" validate:"required"`
		Platform    platform `json:"platform" validate:"required"`
		User        *user    `json:"user" validate:"omitempty"`
		Device      *device  `json:"device" validate:"omitempty"`
		Browser     *browser `json:"browser" validate:"omitempty"`
	}
)

// Parse parses, scrubs and escapes a request's JSON body and maps it to a struct
func Parse(context echo.Context) (*Request, error) {
	request := new(Request)

	if err := bind(request, context); err != nil {
		return request, err
	}

	if err := validate(request, context); err != nil {
		return request, err
	}

	conform.Strings(request)
	escape(request)

	return request, nil
}

func bind(request *Request, context echo.Context) error {
	if err := context.Bind(request); err != nil {
		errorMessage := fmt.Sprintf("Error parsing request: %s", err.Error())

		return responses.ErrorResponse(http.StatusBadRequest, errorMessage, context)
	}

	return nil
}

func validate(request *Request, context echo.Context) error {
	if err := context.Validate(request); err != nil {
		errorMessage := fmt.Sprintf("Error validating request: %s", err.Error())

		return responses.ErrorResponse(http.StatusUnprocessableEntity, errorMessage, context)
	}

	return nil
}

func escape(request *Request) {
	sanitizer := bluemonday.StrictPolicy()

	request.Comment = sanitizer.Sanitize(request.Comment)
	request.Description = sanitizer.Sanitize(request.Description)
}
<|endoftext|>


<commit_before>// Copyright 2015 go-swagger maintainers
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//    http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package validate

import (
	"log"
	"reflect"
	"regexp"

	"github.com/go-openapi/errors"
	"github.com/go-openapi/spec"
	"github.com/go-openapi/strfmt"
)

type objectValidator struct {
	Path                 string
	In                   string
	MaxProperties        *int64
	MinProperties        *int64
	Required             []string
	Properties           map[string]spec.Schema
	AdditionalProperties *spec.SchemaOrBool
	PatternProperties    map[string]spec.Schema
	Root                 interface{}
	KnownFormats         strfmt.Registry
}

func (o *objectValidator) SetPath(path string) {
	o.Path = path
}

func (o *objectValidator) Applies(source interface{}, kind reflect.Kind) bool {
	// TODO: this should also work for structs
	// there is a problem in the type validator where it will be unhappy about null values
	// so that requires more testing
	r := reflect.TypeOf(source) == specSchemaType && (kind == reflect.Map || kind == reflect.Struct)
	if Debug {
		log.Printf("object validator for %q applies %t for %T (kind: %v)\n", o.Path, r, source, kind)
	}
	return r
}

func (o *objectValidator) Validate(data interface{}) *Result {
	val := data.(map[string]interface{})
	numKeys := int64(len(val))

	if o.MinProperties != nil && numKeys < *o.MinProperties {
		return sErr(errors.TooFewProperties(o.Path, o.In, *o.MinProperties))
	}
	if o.MaxProperties != nil && numKeys > *o.MaxProperties {
		return sErr(errors.TooManyProperties(o.Path, o.In, *o.MaxProperties))
	}

	res := new(Result)

	if o.AdditionalProperties != nil && !o.AdditionalProperties.Allows {
		for k := range val {
			_, regularProperty := o.Properties[k]
			matched := false

			for pk := range o.PatternProperties {
				if matches, _ := regexp.MatchString(pk, k); matches {
					matched = true
					break
				}
			}
			if !regularProperty && k != "$schema" && k != "id" && !matched {
				res.AddErrors(errors.PropertyNotAllowed(o.Path, o.In, k))
			}
		}
	} else {
		for key, value := range val {
			_, regularProperty := o.Properties[key]
			matched, succeededOnce, _ := o.validatePatternProperty(key, value, res)
			if !(regularProperty || matched || succeededOnce) {
				if o.AdditionalProperties != nil && o.AdditionalProperties.Schema != nil {
					res.Merge(NewSchemaValidator(o.AdditionalProperties.Schema, o.Root, o.Path+"."+key, o.KnownFormats).Validate(value))
				} else if regularProperty && !(matched || succeededOnce) {
					res.AddErrors(errors.FailedAllPatternProperties(o.Path, o.In, key))
				}
			}
		}
	}

	createdFromDefaults := map[string]bool{}

	for pName, pSchema := range o.Properties {
		rName := pName
		if o.Path != "" {
			rName = o.Path + "." + pName
		}

		if v, ok := val[pName]; ok {
			r := NewSchemaValidator(&pSchema, o.Root, rName, o.KnownFormats).Validate(v)
			res.Merge(r)
		} else if pSchema.Default != nil {
			createdFromDefaults[pName] = true
			pName := pName // shaddow
			def := pSchema.Default
			res.Defaulters = append(res.Defaulters, DefaulterFunc(func() {
				val[pName] = def
			}))
		}
	}

	if len(o.Required) > 0 {
		for _, k := range o.Required {
			if _, ok := val[k]; !ok && !createdFromDefaults[k] {
				res.AddErrors(errors.Required(o.Path+"."+k, o.In))
				continue
			}
		}
	}

	for key, value := range val {
		_, regularProperty := o.Properties[key]
		matched, succeededOnce, patterns := o.validatePatternProperty(key, value, res)
		if !regularProperty && (matched || succeededOnce) {
			for _, pName := range patterns {
				if v, ok := o.PatternProperties[pName]; ok {
					res.Merge(NewSchemaValidator(&v, o.Root, o.Path+"."+key, o.KnownFormats).Validate(value))
				}
			}
		}
	}
	return res
}

func (o *objectValidator) validatePatternProperty(key string, value interface{}, result *Result) (bool, bool, []string) {
	matched := false
	succeededOnce := false
	var patterns []string

	for k, schema := range o.PatternProperties {
		if match, _ := regexp.MatchString(k, key); match {
			patterns = append(patterns, k)
			matched = true
			validator := NewSchemaValidator(&schema, o.Root, o.Path+"."+key, o.KnownFormats)

			res := validator.Validate(value)
			result.Merge(res)
		}
	}

	if succeededOnce {
		result.Inc()
	}

	return matched, succeededOnce, patterns
}
<commit_msg>check that any object with items key should have a type array<commit_after>// Copyright 2015 go-swagger maintainers
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//    http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package validate

import (
	"log"
	"reflect"
	"regexp"

	"github.com/go-openapi/errors"
	"github.com/go-openapi/spec"
	"github.com/go-openapi/strfmt"
)

type objectValidator struct {
	Path                 string
	In                   string
	MaxProperties        *int64
	MinProperties        *int64
	Required             []string
	Properties           map[string]spec.Schema
	AdditionalProperties *spec.SchemaOrBool
	PatternProperties    map[string]spec.Schema
	Root                 interface{}
	KnownFormats         strfmt.Registry
}

func (o *objectValidator) SetPath(path string) {
	o.Path = path
}

func (o *objectValidator) Applies(source interface{}, kind reflect.Kind) bool {
	// TODO: this should also work for structs
	// there is a problem in the type validator where it will be unhappy about null values
	// so that requires more testing
	r := reflect.TypeOf(source) == specSchemaType && (kind == reflect.Map || kind == reflect.Struct)
	if Debug {
		log.Printf("object validator for %q applies %t for %T (kind: %v)\n", o.Path, r, source, kind)
	}
	return r
}

func (o *objectValidator) Validate(data interface{}) *Result {
	val := data.(map[string]interface{})
	numKeys := int64(len(val))

	if o.MinProperties != nil && numKeys < *o.MinProperties {
		return sErr(errors.TooFewProperties(o.Path, o.In, *o.MinProperties))
	}
	if o.MaxProperties != nil && numKeys > *o.MaxProperties {
		return sErr(errors.TooManyProperties(o.Path, o.In, *o.MaxProperties))
	}

	res := new(Result)

	// This is to check that the items keyword is only valid for array types
	if _, itemsKeyFound := val["items"]; itemsKeyFound {
		if t, typeFound := val["type"]; typeFound {
			if tpe, ok := t.(string); !ok || tpe != "array" {
				res.AddErrors(errors.InvalidType(o.Path, o.In, "array", nil))
			}
		}
	}

	if o.AdditionalProperties != nil && !o.AdditionalProperties.Allows {
		for k := range val {
			_, regularProperty := o.Properties[k]
			matched := false

			for pk := range o.PatternProperties {
				if matches, _ := regexp.MatchString(pk, k); matches {
					matched = true
					break
				}
			}
			if !regularProperty && k != "$schema" && k != "id" && !matched {
				res.AddErrors(errors.PropertyNotAllowed(o.Path, o.In, k))
			}
		}
	} else {
		for key, value := range val {
			_, regularProperty := o.Properties[key]
			matched, succeededOnce, _ := o.validatePatternProperty(key, value, res)
			if !(regularProperty || matched || succeededOnce) {
				if o.AdditionalProperties != nil && o.AdditionalProperties.Schema != nil {
					res.Merge(NewSchemaValidator(o.AdditionalProperties.Schema, o.Root, o.Path+"."+key, o.KnownFormats).Validate(value))
				} else if regularProperty && !(matched || succeededOnce) {
					res.AddErrors(errors.FailedAllPatternProperties(o.Path, o.In, key))
				}
			}
		}
	}

	createdFromDefaults := map[string]bool{}

	for pName, pSchema := range o.Properties {
		rName := pName
		if o.Path != "" {
			rName = o.Path + "." + pName
		}

		if v, ok := val[pName]; ok {
			r := NewSchemaValidator(&pSchema, o.Root, rName, o.KnownFormats).Validate(v)
			res.Merge(r)
		} else if pSchema.Default != nil {
			createdFromDefaults[pName] = true
			pName := pName // shaddow
			def := pSchema.Default
			res.Defaulters = append(res.Defaulters, DefaulterFunc(func() {
				val[pName] = def
			}))
		}
	}

	if len(o.Required) > 0 {
		for _, k := range o.Required {
			if _, ok := val[k]; !ok && !createdFromDefaults[k] {
				res.AddErrors(errors.Required(o.Path+"."+k, o.In))
				continue
			}
		}
	}

	for key, value := range val {
		_, regularProperty := o.Properties[key]
		matched, succeededOnce, patterns := o.validatePatternProperty(key, value, res)
		if !regularProperty && (matched || succeededOnce) {
			for _, pName := range patterns {
				if v, ok := o.PatternProperties[pName]; ok {
					res.Merge(NewSchemaValidator(&v, o.Root, o.Path+"."+key, o.KnownFormats).Validate(value))
				}
			}
		}
	}
	return res
}

func (o *objectValidator) validatePatternProperty(key string, value interface{}, result *Result) (bool, bool, []string) {
	matched := false
	succeededOnce := false
	var patterns []string

	for k, schema := range o.PatternProperties {
		if match, _ := regexp.MatchString(k, key); match {
			patterns = append(patterns, k)
			matched = true
			validator := NewSchemaValidator(&schema, o.Root, o.Path+"."+key, o.KnownFormats)

			res := validator.Validate(value)
			result.Merge(res)
		}
	}

	if succeededOnce {
		result.Inc()
	}

	return matched, succeededOnce, patterns
}
<|endoftext|>


<commit_before>package ops

import (
	"bsearch/index"
)

type Attr struct {
	docs    []index.IbDoc
	currptr int
}

func NewAttr(in *index.Index, key string) *Attr {
	a := in.Attrs[key]
	if a == nil {
		return nil
	}
	return &Attr{in.Attrs[key], 0}
}

func (ba *Attr) CurrentDoc() *index.IbDoc {
	if ba.currptr == -1 {
		return nil
	}
	return &ba.docs[ba.currptr]
}

func (ba *Attr) NextDoc(search *index.IbDoc) *index.IbDoc {
	from := ba.currptr
	to := len(ba.docs) - 1

	if search == nil {
		r := &ba.docs[ba.currptr]
		ba.currptr++
		return r
	}

	for true {
		middle := from + (to-from)/2
		d := &ba.docs[middle]
		c := search.Cmp(d)
		if c == 0 {
			ba.currptr = middle
			return d
		}
		if to == from {
			if c > 0 {
				ba.currptr = to
				return d
			}
			break
		}
		if c < 0 {
			from = middle + 1
		} else {
			if from == middle {
				ba.currptr = middle
				return d
			}
			to = middle
		}
	}
	ba.currptr = -1
	return nil
}
<commit_msg>No need to export Attr, no one will poke at its innards.<commit_after>package ops

import (
	"bsearch/index"
)

type attr struct {
	docs    []index.IbDoc
	currptr int
}

func NewAttr(in *index.Index, key string) QueryOp {
	a := in.Attrs[key]
	if a == nil {
		return nil
	}
	return &attr{in.Attrs[key], 0}
}

func (ba *attr) CurrentDoc() *index.IbDoc {
	if ba.currptr == -1 {
		return nil
	}
	return &ba.docs[ba.currptr]
}

func (ba *attr) NextDoc(search *index.IbDoc) *index.IbDoc {
	from := ba.currptr
	to := len(ba.docs) - 1

	if search == nil {
		r := &ba.docs[ba.currptr]
		ba.currptr++
		return r
	}

	for true {
		middle := from + (to-from)/2
		d := &ba.docs[middle]
		c := search.Cmp(d)
		if c == 0 {
			ba.currptr = middle
			return d
		}
		if to == from {
			if c > 0 {
				ba.currptr = to
				return d
			}
			break
		}
		if c < 0 {
			from = middle + 1
		} else {
			if from == middle {
				ba.currptr = middle
				return d
			}
			to = middle
		}
	}
	ba.currptr = -1
	return nil
}
<|endoftext|>


<commit_before>package protocol

import (
	"fmt"
	"io"
	"sync"

	context "github.com/jbenet/go-ipfs/Godeps/_workspace/src/code.google.com/p/go.net/context"

	inet "github.com/jbenet/go-ipfs/p2p/net"
	eventlog "github.com/jbenet/go-ipfs/thirdparty/eventlog"
	lgbl "github.com/jbenet/go-ipfs/util/eventlog/loggables"
)

var log = eventlog.Logger("net/mux")

type streamHandlerMap map[ID]inet.StreamHandler

// Mux provides simple stream multixplexing.
// It helps you precisely when:
//  * You have many streams
//  * You have function handlers
//
// It contains the handlers for each protocol accepted.
// It dispatches handlers for streams opened by remote peers.
type Mux struct {
	// defaultHandler handles unknown protocols. Callers modify at your own risk.
	defaultHandler inet.StreamHandler

	lock     sync.RWMutex
	handlers streamHandlerMap
}

func NewMux() *Mux {
	return &Mux{
		handlers: streamHandlerMap{},
	}
}

// Protocols returns the list of protocols this muxer has handlers for
func (m *Mux) Protocols() []ID {
	m.lock.RLock()
	l := make([]ID, 0, len(m.handlers))
	for p := range m.handlers {
		l = append(l, p)
	}
	m.lock.RUnlock()
	return l
}

// readHeader reads the stream and returns the next Handler function
// according to the muxer encoding.
func (m *Mux) readHeader(s io.Reader) (ID, inet.StreamHandler, error) {
	// log.Error("ReadProtocolHeader")
	p, err := ReadHeader(s)
	if err != nil {
		return "", nil, err
	}

	// log.Debug("readHeader got:", p)
	m.lock.RLock()
	h, found := m.handlers[p]
	m.lock.RUnlock()

	switch {
	case !found && m.defaultHandler != nil:
		return p, m.defaultHandler, nil
	case !found && m.defaultHandler == nil:
		return p, nil, fmt.Errorf("%s no handler with name: %s (%d)", m, p, len(p))
	default:
		return p, h, nil
	}
}

// String returns the muxer's printing representation
func (m *Mux) String() string {
	m.lock.RLock()
	defer m.lock.RUnlock()
	return fmt.Sprintf("<Muxer %p %d>", m, len(m.handlers))
}

func (m *Mux) SetDefaultHandler(h inet.StreamHandler) {
	m.lock.Lock()
	m.defaultHandler = h
	m.lock.Unlock()
}

// SetHandler sets the protocol handler on the Network's Muxer.
// This operation is threadsafe.
func (m *Mux) SetHandler(p ID, h inet.StreamHandler) {
	log.Debugf("%s setting handler for protocol: %s (%d)", m, p, len(p))
	m.lock.Lock()
	m.handlers[p] = h
	m.lock.Unlock()
}

// Handle reads the next name off the Stream, and calls a handler function
// This is done in its own goroutine, to avoid blocking the caller.
func (m *Mux) Handle(s inet.Stream) {
	go m.HandleSync(s)
}

// HandleSync reads the next name off the Stream, and calls a handler function
// This is done synchronously. The handler function will return before
// HandleSync returns.
func (m *Mux) HandleSync(s inet.Stream) {
	ctx := context.Background()

	name, handler, err := m.readHeader(s)
	if err != nil {
		err = fmt.Errorf("protocol mux error: %s", err)
		log.Error(err)
		log.Event(ctx, "muxError", lgbl.Error(err))
		s.Close()
		return
	}

	log.Infof("muxer handle protocol: %s", name)
	log.Event(ctx, "muxHandle", eventlog.Metadata{"protocol": name})
	handler(s)
}

// ReadLengthPrefix reads the name from Reader with a length-byte-prefix.
func ReadLengthPrefix(r io.Reader) (string, error) {
	// c-string identifier
	// the first byte is our length
	l := make([]byte, 1)
	if _, err := io.ReadFull(r, l); err != nil {
		return "", err
	}
	length := int(l[0])

	// the next are our identifier
	name := make([]byte, length)
	if _, err := io.ReadFull(r, name); err != nil {
		return "", err
	}

	return string(name), nil
}
<commit_msg>eventlog: rm noisy low-signal statement<commit_after>package protocol

import (
	"fmt"
	"io"
	"sync"

	context "github.com/jbenet/go-ipfs/Godeps/_workspace/src/code.google.com/p/go.net/context"

	inet "github.com/jbenet/go-ipfs/p2p/net"
	eventlog "github.com/jbenet/go-ipfs/thirdparty/eventlog"
	lgbl "github.com/jbenet/go-ipfs/util/eventlog/loggables"
)

var log = eventlog.Logger("net/mux")

type streamHandlerMap map[ID]inet.StreamHandler

// Mux provides simple stream multixplexing.
// It helps you precisely when:
//  * You have many streams
//  * You have function handlers
//
// It contains the handlers for each protocol accepted.
// It dispatches handlers for streams opened by remote peers.
type Mux struct {
	// defaultHandler handles unknown protocols. Callers modify at your own risk.
	defaultHandler inet.StreamHandler

	lock     sync.RWMutex
	handlers streamHandlerMap
}

func NewMux() *Mux {
	return &Mux{
		handlers: streamHandlerMap{},
	}
}

// Protocols returns the list of protocols this muxer has handlers for
func (m *Mux) Protocols() []ID {
	m.lock.RLock()
	l := make([]ID, 0, len(m.handlers))
	for p := range m.handlers {
		l = append(l, p)
	}
	m.lock.RUnlock()
	return l
}

// readHeader reads the stream and returns the next Handler function
// according to the muxer encoding.
func (m *Mux) readHeader(s io.Reader) (ID, inet.StreamHandler, error) {
	// log.Error("ReadProtocolHeader")
	p, err := ReadHeader(s)
	if err != nil {
		return "", nil, err
	}

	// log.Debug("readHeader got:", p)
	m.lock.RLock()
	h, found := m.handlers[p]
	m.lock.RUnlock()

	switch {
	case !found && m.defaultHandler != nil:
		return p, m.defaultHandler, nil
	case !found && m.defaultHandler == nil:
		return p, nil, fmt.Errorf("%s no handler with name: %s (%d)", m, p, len(p))
	default:
		return p, h, nil
	}
}

// String returns the muxer's printing representation
func (m *Mux) String() string {
	m.lock.RLock()
	defer m.lock.RUnlock()
	return fmt.Sprintf("<Muxer %p %d>", m, len(m.handlers))
}

func (m *Mux) SetDefaultHandler(h inet.StreamHandler) {
	m.lock.Lock()
	m.defaultHandler = h
	m.lock.Unlock()
}

// SetHandler sets the protocol handler on the Network's Muxer.
// This operation is threadsafe.
func (m *Mux) SetHandler(p ID, h inet.StreamHandler) {
	log.Debugf("%s setting handler for protocol: %s (%d)", m, p, len(p))
	m.lock.Lock()
	m.handlers[p] = h
	m.lock.Unlock()
}

// Handle reads the next name off the Stream, and calls a handler function
// This is done in its own goroutine, to avoid blocking the caller.
func (m *Mux) Handle(s inet.Stream) {
	go m.HandleSync(s)
}

// HandleSync reads the next name off the Stream, and calls a handler function
// This is done synchronously. The handler function will return before
// HandleSync returns.
func (m *Mux) HandleSync(s inet.Stream) {
	ctx := context.Background()

	name, handler, err := m.readHeader(s)
	if err != nil {
		err = fmt.Errorf("protocol mux error: %s", err)
		log.Error(err)
		log.Event(ctx, "muxError", lgbl.Error(err))
		s.Close()
		return
	}

	log.Infof("muxer handle protocol: %s", name)
	handler(s)
}

// ReadLengthPrefix reads the name from Reader with a length-byte-prefix.
func ReadLengthPrefix(r io.Reader) (string, error) {
	// c-string identifier
	// the first byte is our length
	l := make([]byte, 1)
	if _, err := io.ReadFull(r, l); err != nil {
		return "", err
	}
	length := int(l[0])

	// the next are our identifier
	name := make([]byte, length)
	if _, err := io.ReadFull(r, name); err != nil {
		return "", err
	}

	return string(name), nil
}
<|endoftext|>


<commit_before>package aws

import (
	"fmt"
	"log"
	"strings"
	"time"

	"github.com/aws/aws-sdk-go/aws"
	"github.com/aws/aws-sdk-go/aws/awserr"
	"github.com/aws/aws-sdk-go/service/sns"
	"github.com/hashicorp/errwrap"
	"github.com/hashicorp/terraform/helper/resource"
	"github.com/hashicorp/terraform/helper/schema"
)

// Mutable attributes
var SNSAttributeMap = map[string]string{
	"arn":             "TopicArn",
	"display_name":    "DisplayName",
	"policy":          "Policy",
	"delivery_policy": "DeliveryPolicy",
}

func resourceAwsSnsTopic() *schema.Resource {
	return &schema.Resource{
		Create: resourceAwsSnsTopicCreate,
		Read:   resourceAwsSnsTopicRead,
		Update: resourceAwsSnsTopicUpdate,
		Delete: resourceAwsSnsTopicDelete,
		Importer: &schema.ResourceImporter{
			State: schema.ImportStatePassthrough,
		},

		Schema: map[string]*schema.Schema{
			"name": &schema.Schema{
				Type:     schema.TypeString,
				Required: true,
				ForceNew: true,
			},
			"display_name": &schema.Schema{
				Type:     schema.TypeString,
				Optional: true,
				ForceNew: false,
			},
			"policy": &schema.Schema{
				Type:             schema.TypeString,
				Optional:         true,
				Computed:         true,
				ValidateFunc:     validateJsonString,
				DiffSuppressFunc: suppressEquivalentAwsPolicyDiffs,
				StateFunc: func(v interface{}) string {
					json, _ := normalizeJsonString(v)
					return json
				},
			},
			"delivery_policy": &schema.Schema{
				Type:             schema.TypeString,
				Optional:         true,
				ForceNew:         false,
				ValidateFunc:     validateJsonString,
				DiffSuppressFunc: suppressEquivalentJsonDiffs,
				StateFunc: func(v interface{}) string {
					json, _ := normalizeJsonString(v)
					return json
				},
			},
			"arn": &schema.Schema{
				Type:     schema.TypeString,
				Computed: true,
			},
		},
	}
}

func resourceAwsSnsTopicCreate(d *schema.ResourceData, meta interface{}) error {
	snsconn := meta.(*AWSClient).snsconn

	name := d.Get("name").(string)

	log.Printf("[DEBUG] SNS create topic: %s", name)

	req := &sns.CreateTopicInput{
		Name: aws.String(name),
	}

	output, err := snsconn.CreateTopic(req)
	if err != nil {
		return fmt.Errorf("Error creating SNS topic: %s", err)
	}

	d.SetId(*output.TopicArn)

	// Write the ARN to the 'arn' field for export
	d.Set("arn", *output.TopicArn)

	return resourceAwsSnsTopicUpdate(d, meta)
}

func resourceAwsSnsTopicUpdate(d *schema.ResourceData, meta interface{}) error {
	r := *resourceAwsSnsTopic()

	for k, _ := range r.Schema {
		if attrKey, ok := SNSAttributeMap[k]; ok {
			if d.HasChange(k) {
				log.Printf("[DEBUG] Updating %s", attrKey)
				_, n := d.GetChange(k)
				// Ignore an empty policy
				if !(k == "policy" && n == "") {
					// Make API call to update attributes
					req := sns.SetTopicAttributesInput{
						TopicArn:       aws.String(d.Id()),
						AttributeName:  aws.String(attrKey),
						AttributeValue: aws.String(n.(string)),
					}

					// Retry the update in the event of an eventually consistent style of
					// error, where say an IAM resource is successfully created but not
					// actually available. See https://github.com/hashicorp/terraform/issues/3660
					log.Printf("[DEBUG] Updating SNS Topic (%s) attributes request: %s", d.Id(), req)
					stateConf := &resource.StateChangeConf{
						Pending:    []string{"retrying"},
						Target:     []string{"success"},
						Refresh:    resourceAwsSNSUpdateRefreshFunc(meta, req),
						Timeout:    1 * time.Minute,
						MinTimeout: 3 * time.Second,
					}
					_, err := stateConf.WaitForState()
					if err != nil {
						return err
					}
				}
			}
		}
	}

	return resourceAwsSnsTopicRead(d, meta)
}

func resourceAwsSNSUpdateRefreshFunc(
	meta interface{}, params sns.SetTopicAttributesInput) resource.StateRefreshFunc {
	return func() (interface{}, string, error) {
		snsconn := meta.(*AWSClient).snsconn
		if _, err := snsconn.SetTopicAttributes(&params); err != nil {
			log.Printf("[WARN] Erroring updating topic attributes: %s", err)
			if awsErr, ok := err.(awserr.Error); ok {
				// if the error contains the PrincipalNotFound message, we can retry
				if strings.Contains(awsErr.Message(), "PrincipalNotFound") {
					log.Printf("[DEBUG] Retrying AWS SNS Topic Update: %s", params)
					return nil, "retrying", err
				}
			}
			return nil, "failed", err
		}
		return 42, "success", nil
	}
}

func resourceAwsSnsTopicRead(d *schema.ResourceData, meta interface{}) error {
	snsconn := meta.(*AWSClient).snsconn

	attributeOutput, err := snsconn.GetTopicAttributes(&sns.GetTopicAttributesInput{
		TopicArn: aws.String(d.Id()),
	})
	if err != nil {
		if awsErr, ok := err.(awserr.Error); ok && awsErr.Code() == "NotFound" {
			log.Printf("[WARN] SNS Topic (%s) not found, error code (404)", d.Id())
			d.SetId("")
			return nil
		}

		return err
	}

	if attributeOutput.Attributes != nil && len(attributeOutput.Attributes) > 0 {
		attrmap := attributeOutput.Attributes
		resource := *resourceAwsSnsTopic()
		// iKey = internal struct key, oKey = AWS Attribute Map key
		for iKey, oKey := range SNSAttributeMap {
			log.Printf("[DEBUG] Reading %s => %s", iKey, oKey)

			if attrmap[oKey] != nil {
				// Some of the fetched attributes are stateful properties such as
				// the number of subscriptions, the owner, etc. skip those
				if resource.Schema[iKey] != nil {
					var value string
					if iKey == "policy" {
						value, err = normalizeJsonString(*attrmap[oKey])
						if err != nil {
							return errwrap.Wrapf("policy contains an invalid JSON: {{err}}", err)
						}
					} else {
						value = *attrmap[oKey]
					}
					log.Printf("[DEBUG] Reading %s => %s -> %s", iKey, oKey, value)
					d.Set(iKey, value)
				}
			}
		}
	}

	// If we have no name set (import) then determine it from the ARN.
	// This is a bit of a heuristic for now since AWS provides no other
	// way to get it.
	if _, ok := d.GetOk("name"); !ok {
		arn := d.Get("arn").(string)
		idx := strings.LastIndex(arn, ":")
		if idx > -1 {
			d.Set("name", arn[idx+1:])
		}
	}

	return nil
}

func resourceAwsSnsTopicDelete(d *schema.ResourceData, meta interface{}) error {
	snsconn := meta.(*AWSClient).snsconn

	log.Printf("[DEBUG] SNS Delete Topic: %s", d.Id())
	_, err := snsconn.DeleteTopic(&sns.DeleteTopicInput{
		TopicArn: aws.String(d.Id()),
	})
	if err != nil {
		return err
	}
	return nil
}
<commit_msg>Using resource.Retry<commit_after>package aws

import (
	"fmt"
	"log"
	"strings"

	"github.com/aws/aws-sdk-go/aws"
	"github.com/aws/aws-sdk-go/aws/awserr"
	"github.com/aws/aws-sdk-go/service/sns"
	"github.com/hashicorp/errwrap"
	"github.com/hashicorp/terraform/helper/resource"
	"github.com/hashicorp/terraform/helper/schema"
)

// Mutable attributes
var SNSAttributeMap = map[string]string{
	"arn":             "TopicArn",
	"display_name":    "DisplayName",
	"policy":          "Policy",
	"delivery_policy": "DeliveryPolicy",
}

func resourceAwsSnsTopic() *schema.Resource {
	return &schema.Resource{
		Create: resourceAwsSnsTopicCreate,
		Read:   resourceAwsSnsTopicRead,
		Update: resourceAwsSnsTopicUpdate,
		Delete: resourceAwsSnsTopicDelete,
		Importer: &schema.ResourceImporter{
			State: schema.ImportStatePassthrough,
		},

		Schema: map[string]*schema.Schema{
			"name": &schema.Schema{
				Type:     schema.TypeString,
				Required: true,
				ForceNew: true,
			},
			"display_name": &schema.Schema{
				Type:     schema.TypeString,
				Optional: true,
				ForceNew: false,
			},
			"policy": &schema.Schema{
				Type:             schema.TypeString,
				Optional:         true,
				Computed:         true,
				ValidateFunc:     validateJsonString,
				DiffSuppressFunc: suppressEquivalentAwsPolicyDiffs,
				StateFunc: func(v interface{}) string {
					json, _ := normalizeJsonString(v)
					return json
				},
			},
			"delivery_policy": &schema.Schema{
				Type:             schema.TypeString,
				Optional:         true,
				ForceNew:         false,
				ValidateFunc:     validateJsonString,
				DiffSuppressFunc: suppressEquivalentJsonDiffs,
				StateFunc: func(v interface{}) string {
					json, _ := normalizeJsonString(v)
					return json
				},
			},
			"arn": &schema.Schema{
				Type:     schema.TypeString,
				Computed: true,
			},
		},
	}
}

func resourceAwsSnsTopicCreate(d *schema.ResourceData, meta interface{}) error {
	snsconn := meta.(*AWSClient).snsconn

	name := d.Get("name").(string)

	log.Printf("[DEBUG] SNS create topic: %s", name)

	req := &sns.CreateTopicInput{
		Name: aws.String(name),
	}

	output, err := snsconn.CreateTopic(req)
	if err != nil {
		return fmt.Errorf("Error creating SNS topic: %s", err)
	}

	d.SetId(*output.TopicArn)

	// Write the ARN to the 'arn' field for export
	d.Set("arn", *output.TopicArn)

	return resourceAwsSnsTopicUpdate(d, meta)
}

func resourceAwsSnsTopicUpdate(d *schema.ResourceData, meta interface{}) error {
	r := *resourceAwsSnsTopic()

	for k, _ := range r.Schema {
		if attrKey, ok := SNSAttributeMap[k]; ok {
			if d.HasChange(k) {
				log.Printf("[DEBUG] Updating %s", attrKey)
				_, n := d.GetChange(k)
				// Ignore an empty policy
				if !(k == "policy" && n == "") {
					// Make API call to update attributes
					req := sns.SetTopicAttributesInput{
						TopicArn:       aws.String(d.Id()),
						AttributeName:  aws.String(attrKey),
						AttributeValue: aws.String(n.(string)),
					}
					conn := meta.(*AWSClient).snsconn
					// Retry the update in the event of an eventually consistent style of
					// error, where say an IAM resource is successfully created but not
					// actually available. See https://github.com/hashicorp/terraform/issues/3660
					_, err := retryOnAwsCode("InvalidParameter", func() (interface{}, error) {
						return conn.SetTopicAttributes(&req)
					})
					return err
				}
			}
		}
	}

	return resourceAwsSnsTopicRead(d, meta)
}

func resourceAwsSNSUpdateRefreshFunc(
	meta interface{}, params sns.SetTopicAttributesInput) resource.StateRefreshFunc {
	return func() (interface{}, string, error) {
		snsconn := meta.(*AWSClient).snsconn
		if _, err := snsconn.SetTopicAttributes(&params); err != nil {
			log.Printf("[WARN] Erroring updating topic attributes: %s", err)
			if awsErr, ok := err.(awserr.Error); ok {
				// if the error contains the PrincipalNotFound message, we can retry
				if strings.Contains(awsErr.Message(), "PrincipalNotFound") {
					log.Printf("[DEBUG] Retrying AWS SNS Topic Update: %s", params)
					return nil, "retrying", nil
				}
			}
			return nil, "failed", err
		}
		return 42, "success", nil
	}
}

func resourceAwsSnsTopicRead(d *schema.ResourceData, meta interface{}) error {
	snsconn := meta.(*AWSClient).snsconn

	attributeOutput, err := snsconn.GetTopicAttributes(&sns.GetTopicAttributesInput{
		TopicArn: aws.String(d.Id()),
	})
	if err != nil {
		if awsErr, ok := err.(awserr.Error); ok && awsErr.Code() == "NotFound" {
			log.Printf("[WARN] SNS Topic (%s) not found, error code (404)", d.Id())
			d.SetId("")
			return nil
		}

		return err
	}

	if attributeOutput.Attributes != nil && len(attributeOutput.Attributes) > 0 {
		attrmap := attributeOutput.Attributes
		resource := *resourceAwsSnsTopic()
		// iKey = internal struct key, oKey = AWS Attribute Map key
		for iKey, oKey := range SNSAttributeMap {
			log.Printf("[DEBUG] Reading %s => %s", iKey, oKey)

			if attrmap[oKey] != nil {
				// Some of the fetched attributes are stateful properties such as
				// the number of subscriptions, the owner, etc. skip those
				if resource.Schema[iKey] != nil {
					var value string
					if iKey == "policy" {
						value, err = normalizeJsonString(*attrmap[oKey])
						if err != nil {
							return errwrap.Wrapf("policy contains an invalid JSON: {{err}}", err)
						}
					} else {
						value = *attrmap[oKey]
					}
					log.Printf("[DEBUG] Reading %s => %s -> %s", iKey, oKey, value)
					d.Set(iKey, value)
				}
			}
		}
	}

	// If we have no name set (import) then determine it from the ARN.
	// This is a bit of a heuristic for now since AWS provides no other
	// way to get it.
	if _, ok := d.GetOk("name"); !ok {
		arn := d.Get("arn").(string)
		idx := strings.LastIndex(arn, ":")
		if idx > -1 {
			d.Set("name", arn[idx+1:])
		}
	}

	return nil
}

func resourceAwsSnsTopicDelete(d *schema.ResourceData, meta interface{}) error {
	snsconn := meta.(*AWSClient).snsconn

	log.Printf("[DEBUG] SNS Delete Topic: %s", d.Id())
	_, err := snsconn.DeleteTopic(&sns.DeleteTopicInput{
		TopicArn: aws.String(d.Id()),
	})
	if err != nil {
		return err
	}
	return nil
}
<|endoftext|>