<commit_before>// Copyright 2016 The Cockroach Authors.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
// implied. See the License for the specific language governing
// permissions and limitations under the License.
//
// Author: Matt Tracy (matt@cockroachlabs.com)

package server_test

import (
	"testing"
	"time"

	"github.com/pkg/errors"

	"github.com/cockroachdb/cockroach/base"
	"github.com/cockroachdb/cockroach/server/serverpb"
	"github.com/cockroachdb/cockroach/testutils/testcluster"
	"github.com/cockroachdb/cockroach/util"
	"github.com/cockroachdb/cockroach/util/leaktest"
)

func TestAdminAPITableStats(t *testing.T) {
	defer leaktest.AfterTest(t)()
	const nodeCount = 3
	tc := testcluster.StartTestCluster(t, nodeCount, base.TestClusterArgs{
		ReplicationMode: base.ReplicationAuto,
		ServerArgs: base.TestServerArgs{
			ScanInterval:    time.Millisecond,
			ScanMaxIdleTime: time.Millisecond,
		},
	})
	defer tc.Stopper().Stop()
	if err := tc.WaitForFullReplication(); err != nil {
		t.Fatal(err)
	}
	server0 := tc.Server(0)

	// Create clients (SQL, HTTP) connected to server 0.
	db := tc.ServerConn(0)

	client, err := server0.GetHTTPClient()
	if err != nil {
		t.Fatal(err)
	}

	client.Timeout = base.NetworkTimeout * 3

	// Make a single table and insert some data. The database and test have
	// names which require escaping, in order to verify that database and
	// table names are being handled correctly.
	if _, err := db.Exec(`CREATE DATABASE "test test"`); err != nil {
		t.Fatal(err)
	}
	if _, err := db.Exec(`
		CREATE TABLE "test test"."foo foo" (
			id INT PRIMARY KEY,
			val STRING
		)`,
	); err != nil {
		t.Fatal(err)
	}
	for i := 0; i < 10; i++ {
		if _, err := db.Exec(`
			INSERT INTO "test test"."foo foo" VALUES(
				$1, $2
			)`, i, "test",
		); err != nil {
			t.Fatal(err)
		}
	}

	url := server0.AdminURL() + "/_admin/v1/databases/test test/tables/foo foo/stats"
	var tsResponse serverpb.TableStatsResponse

	// The new SQL table may not yet have split into its own range. Wait for
	// this to occur, and for full replication.
	util.SucceedsSoon(t, func() error {
		if err := util.GetJSON(client, url, &tsResponse); err != nil {
			return err
		}
		if tsResponse.RangeCount != 1 {
			return errors.Errorf("Table range not yet separated.")
		}
		if tsResponse.NodeCount != nodeCount {
			return errors.Errorf("Table range not yet replicated to %d nodes.", 3)
		}
		if a, e := tsResponse.ReplicaCount, int64(nodeCount); a != e {
			return errors.Errorf("expected %d replicas, found %d", e, a)
		}
		return nil
	})

	// These two conditions *must* be true, given that the above
	// SucceedsSoon has succeeded.
	if a, e := tsResponse.Stats.KeyCount, int64(20); a < e {
		t.Fatalf("expected at least 20 total keys, found %d", a)
	}
	if len(tsResponse.MissingNodes) > 0 {
		t.Fatalf("expected no missing nodes, found %v", tsResponse.MissingNodes)
	}

	// Kill a node, ensure it shows up in MissingNodes and that ReplicaCount is
	// lower.
	tc.StopServer(1)

	if err := util.GetJSON(client, url, &tsResponse); err != nil {
		t.Fatal(err)
	}
	if a, e := tsResponse.NodeCount, int64(nodeCount); a != e {
		t.Errorf("expected %d nodes, found %d", e, a)
	}
	if a, e := tsResponse.RangeCount, int64(1); a != e {
		t.Errorf("expected %d ranges, found %d", e, a)
	}
	if a, e := tsResponse.ReplicaCount, int64((nodeCount/2)+1); a != e {
		t.Errorf("expected %d replicas, found %d", e, a)
	}
	if a, e := tsResponse.Stats.KeyCount, int64(10); a < e {
		t.Errorf("expected at least 10 total keys, found %d", a)
	}
	if len(tsResponse.MissingNodes) != 1 {
		t.Errorf("expected one missing node, found %v", tsResponse.MissingNodes)
	}

	// Call TableStats with a very low timeout. This tests that fan-out queries
	// do not leak goroutines if the calling context is abandoned.
	// Interestingly, the call can actually sometimes succeed, despite the small
	// timeout; however, in aggregate (or in stress tests) this will suffice for
	// detecting leaks.
	client.Timeout = 1 * time.Nanosecond
	_ = util.GetJSON(client, url, &tsResponse)
}
<commit_msg>server: skip flaky TestAdminAPITableStats<commit_after>// Copyright 2016 The Cockroach Authors.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
// implied. See the License for the specific language governing
// permissions and limitations under the License.
//
// Author: Matt Tracy (matt@cockroachlabs.com)

package server_test

import (
	"testing"
	"time"

	"github.com/pkg/errors"

	"github.com/cockroachdb/cockroach/base"
	"github.com/cockroachdb/cockroach/server/serverpb"
	"github.com/cockroachdb/cockroach/testutils/testcluster"
	"github.com/cockroachdb/cockroach/util"
	"github.com/cockroachdb/cockroach/util/leaktest"
)

func TestAdminAPITableStats(t *testing.T) {
	defer leaktest.AfterTest(t)()
	t.Skip("#8890")

	const nodeCount = 3
	tc := testcluster.StartTestCluster(t, nodeCount, base.TestClusterArgs{
		ReplicationMode: base.ReplicationAuto,
		ServerArgs: base.TestServerArgs{
			ScanInterval:    time.Millisecond,
			ScanMaxIdleTime: time.Millisecond,
		},
	})
	defer tc.Stopper().Stop()
	if err := tc.WaitForFullReplication(); err != nil {
		t.Fatal(err)
	}
	server0 := tc.Server(0)

	// Create clients (SQL, HTTP) connected to server 0.
	db := tc.ServerConn(0)

	client, err := server0.GetHTTPClient()
	if err != nil {
		t.Fatal(err)
	}

	client.Timeout = base.NetworkTimeout * 3

	// Make a single table and insert some data. The database and test have
	// names which require escaping, in order to verify that database and
	// table names are being handled correctly.
	if _, err := db.Exec(`CREATE DATABASE "test test"`); err != nil {
		t.Fatal(err)
	}
	if _, err := db.Exec(`
		CREATE TABLE "test test"."foo foo" (
			id INT PRIMARY KEY,
			val STRING
		)`,
	); err != nil {
		t.Fatal(err)
	}
	for i := 0; i < 10; i++ {
		if _, err := db.Exec(`
			INSERT INTO "test test"."foo foo" VALUES(
				$1, $2
			)`, i, "test",
		); err != nil {
			t.Fatal(err)
		}
	}

	url := server0.AdminURL() + "/_admin/v1/databases/test test/tables/foo foo/stats"
	var tsResponse serverpb.TableStatsResponse

	// The new SQL table may not yet have split into its own range. Wait for
	// this to occur, and for full replication.
	util.SucceedsSoon(t, func() error {
		if err := util.GetJSON(client, url, &tsResponse); err != nil {
			return err
		}
		if tsResponse.RangeCount != 1 {
			return errors.Errorf("Table range not yet separated.")
		}
		if tsResponse.NodeCount != nodeCount {
			return errors.Errorf("Table range not yet replicated to %d nodes.", 3)
		}
		if a, e := tsResponse.ReplicaCount, int64(nodeCount); a != e {
			return errors.Errorf("expected %d replicas, found %d", e, a)
		}
		return nil
	})

	// These two conditions *must* be true, given that the above
	// SucceedsSoon has succeeded.
	if a, e := tsResponse.Stats.KeyCount, int64(20); a < e {
		t.Fatalf("expected at least 20 total keys, found %d", a)
	}
	if len(tsResponse.MissingNodes) > 0 {
		t.Fatalf("expected no missing nodes, found %v", tsResponse.MissingNodes)
	}

	// Kill a node, ensure it shows up in MissingNodes and that ReplicaCount is
	// lower.
	tc.StopServer(1)

	if err := util.GetJSON(client, url, &tsResponse); err != nil {
		t.Fatal(err)
	}
	if a, e := tsResponse.NodeCount, int64(nodeCount); a != e {
		t.Errorf("expected %d nodes, found %d", e, a)
	}
	if a, e := tsResponse.RangeCount, int64(1); a != e {
		t.Errorf("expected %d ranges, found %d", e, a)
	}
	if a, e := tsResponse.ReplicaCount, int64((nodeCount/2)+1); a != e {
		t.Errorf("expected %d replicas, found %d", e, a)
	}
	if a, e := tsResponse.Stats.KeyCount, int64(10); a < e {
		t.Errorf("expected at least 10 total keys, found %d", a)
	}
	if len(tsResponse.MissingNodes) != 1 {
		t.Errorf("expected one missing node, found %v", tsResponse.MissingNodes)
	}

	// Call TableStats with a very low timeout. This tests that fan-out queries
	// do not leak goroutines if the calling context is abandoned.
	// Interestingly, the call can actually sometimes succeed, despite the small
	// timeout; however, in aggregate (or in stress tests) this will suffice for
	// detecting leaks.
	client.Timeout = 1 * time.Nanosecond
	_ = util.GetJSON(client, url, &tsResponse)
}
<|endoftext|>


<commit_before>// +build appengine

/*

A Google App Engine Memcache session store implementation.

The implementation stores sessions in the Memcache and also saves sessions to the Datastore as a backup
in case data would be removed from the Memcache. This behaviour is optional, Datastore can be disabled completely.
You can also choose whether saving to Datastore happens synchronously (in the same goroutine)
or asynchronously (in another goroutine).

Limitations based on GAE Memcache:

- Since session ids are used in the Memcache keys, session ids can't be longer than 250 chars (bytes, but with Base64 charset it's the same).
If you also specify a key prefix (in MemcacheStoreOptions), that also counts into it.

- The size of a Session cannot be larger than 1 MB (marshalled into a byte slice).

Note that the Store will automatically "flush" sessions accessed from it when the Store is closed,
so it is very important to close the Store at the end of your request; this is usually done by closing
the session manager to which you passed the store (preferably with the defer statement).

Check out the GAE session demo application which shows how to use it properly:

https://github.com/icza/session/blob/master/gae_session_demo/session_demo.go

*/

package session

import (
	"net/http"
	"sync"
	"time"

	"appengine"
	"appengine/datastore"
	"appengine/memcache"
)

// A Google App Engine Memcache session store implementation.
type memcacheStore struct {
	ctx appengine.Context // Appengine context used when accessing the Memcache

	keyPrefix string // Prefix to use in front of session ids to construct Memcache key
	retries   int    // Number of retries to perform in case of general Memcache failures

	codec memcache.Codec // Codec used to marshal and unmarshal a Session to a byte slice

	onlyMemcache       bool   // Tells if sessions are not to be saved in Datastore
	asyncDatastoreSave bool   // Tells if saving in Datastore should happen asynchronously, in a new goroutine
	dsEntityName       string // Name of the datastore entity to use to save sessions

	// Map of sessions (mapped from ID) that were accessed using this store; usually it will only be 1.
	// It is also used as a cache, should the user call Get() with the same id multiple times.
	sessions map[string]Session

	mux *sync.RWMutex // mutex to synchronize access to sessions
}

// MemcacheStoreOptions defines options that may be passed when creating a new Memcache session store.
// All fields are optional; default value will be used for any field that has the zero value.
type MemcacheStoreOptions struct {
	// Prefix to use when storing sessions in the Memcache, cannot contain a null byte
	// and cannot be longer than 250 chars (bytes) when concatenated with the session id; default value is the empty string
	// The Memcache key will be this prefix and the session id concatenated.
	KeyPrefix string

	// Number of retries to perform if Memcache operations fail due to general service error;
	// default value is 3
	Retries int

	// Codec used to marshal and unmarshal a Session to a byte slice;
	// Default value is &memcache.Gob (which uses the gob package).
	Codec *memcache.Codec

	// Tells if sessions are only to be stored in Memcache, and do not store them in Datastore as backup;
	// as Memcache has no guarantees, it may lose content from time to time, but if Datastore is
	// also used, the session will automatically be retrieved from the Datastore if not found in Memcache;
	// default value is false (which means to also save sessions in the Datastore)
	OnlyMemcache bool

	// Tells if saving in Datastore should happen asynchronously (in a new goroutine, possibly after returning),
	// if false, session saving in Datastore will happen in the same goroutine, before returning from the request.
	// Asynchronous saving gives smaller latency (and is enough most of the time as Memcache is always checked first);
	// default value is false which means to save sessions in the Datastore in the same goroutine, synchronously
	// Not used if OnlyMemcache=true.
	// FIXME: See https://github.com/icza/session/issues/3
	AsyncDatastoreSave bool

	// Name of the entity to use for saving sessions;
	// default value is "sess_"
	// Not used if OnlyMemcache=true.
	DSEntityName string
}

// SessEntity models the session entity saved to Datastore.
// The Key is the session id.
type SessEntity struct {
	Expires time.Time `datastore:"exp"`
	Value   []byte    `datastore:"val"`
}

// Pointer to zero value of MemcacheStoreOptions to be reused for efficiency.
var zeroMemcacheStoreOptions = new(MemcacheStoreOptions)

// NewMemcacheStore returns a new, GAE Memcache session Store with default options.
// Default values of options are listed in the MemcacheStoreOptions type.
//
// Important! Since accessing the Memcache relies on Appengine Context
// which is bound to an http.Request, the returned Store can only be used for the lifetime of a request!
func NewMemcacheStore(ctx appengine.Context) Store {
	return NewMemcacheStoreOptions(ctx, zeroMemcacheStoreOptions)
}

const defaultDSEntityName = "sess_" // Default value of DSEntityName.

// NewMemcacheStoreOptions returns a new, GAE Memcache session Store with the specified options.
//
// Important! Since accessing the Memcache relies on Appengine Context
// which is bound to an http.Request, the returned Store can only be used for the lifetime of a request!
func NewMemcacheStoreOptions(ctx appengine.Context, o *MemcacheStoreOptions) Store {
	s := &memcacheStore{
		ctx:                ctx,
		keyPrefix:          o.KeyPrefix,
		retries:            o.Retries,
		onlyMemcache:       o.OnlyMemcache,
		asyncDatastoreSave: o.AsyncDatastoreSave,
		dsEntityName:       o.DSEntityName,
		sessions:           make(map[string]Session, 2),
		mux:                &sync.RWMutex{},
	}
	if s.retries <= 0 {
		s.retries = 3
	}
	if o.Codec != nil {
		s.codec = *o.Codec
	} else {
		s.codec = memcache.Gob
	}
	if s.dsEntityName == "" {
		s.dsEntityName = defaultDSEntityName
	}
	return s
}

// Get is to implement Store.Get().
// Important! Since sessions are marshalled and stored in the Memcache,
// the mutex of the Session (Session.RWMutex()) will be different for each
// Session value (even though they might have the same session id)!
func (s *memcacheStore) Get(id string) Session {
	s.mux.RLock()
	defer s.mux.RUnlock()

	// First check our "cache"
	if sess := s.sessions[id]; sess != nil {
		return sess
	}

	// Next check in Memcache
	var err error
	var sess *sessionImpl

	for i := 0; i < s.retries; i++ {
		var sess_ sessionImpl
		_, err = s.codec.Get(s.ctx, s.keyPrefix+id, &sess_)
		if err == memcache.ErrCacheMiss {
			break // It's not in the Memcache (e.g. invalid sess id or was removed from Memcache by AppEngine)
		}
		if err == nil {
			sess = &sess_
			break
		}
		// Service error? Retry..
	}

	if sess == nil {
		if err != nil && err != memcache.ErrCacheMiss {
			s.ctx.Errorf("Failed to get session from memcache, id: %s, error: %v", id, err)
		}

		// Ok, we didn't get it from Memcace (either was not there or Memcache service is unavailable).
		// Now it's time to check in the Datastore.
		key := datastore.NewKey(s.ctx, s.dsEntityName, id, 0, nil)
		for i := 0; i < s.retries; i++ {
			e := SessEntity{}
			err = datastore.Get(s.ctx, key, &e)
			if err == datastore.ErrNoSuchEntity {
				return nil // It's not in the Datastore either
			}
			if err != nil {
				// Service error? Retry..
				continue
			}
			if e.Expires.Before(time.Now()) {
				// Session expired.
				datastore.Delete(s.ctx, key) // Omitting error check...
				return nil
			}
			var sess_ sessionImpl
			if err = s.codec.Unmarshal(e.Value, &sess_); err != nil {
				break // Invalid data in stored session entity...
			}
			sess = &sess_
			break
		}
	}

	if sess == nil {
		s.ctx.Errorf("Failed to get session from datastore, id: %s, error: %v", id, err)
		return nil
	}

	// Yes! We have it! "Actualize" it.
	sess.Access()
	// Mutex is not marshalled, so create a new one:
	sess.mux = &sync.RWMutex{}
	s.sessions[id] = sess
	return sess
}

// Add is to implement Store.Add().
func (s *memcacheStore) Add(sess Session) {
	s.mux.Lock()
	defer s.mux.Unlock()

	if s.setMemcacheSession(sess) {
		s.ctx.Infof("Session added: %s", sess.ID())
		s.sessions[sess.ID()] = sess
		return
	}
}

// setMemcacheSession sets the specified session in the Memcache.
func (s *memcacheStore) setMemcacheSession(sess Session) (success bool) {
	item := &memcache.Item{
		Key:        s.keyPrefix + sess.ID(),
		Object:     sess,
		Expiration: sess.Timeout(),
	}

	var err error
	for i := 0; i < s.retries; i++ {
		if err = s.codec.Set(s.ctx, item); err == nil {
			return true
		}
	}

	s.ctx.Errorf("Failed to add session to memcache, id: %s, error: %v", sess.ID(), err)
	return false
}

// Remove is to implement Store.Remove().
func (s *memcacheStore) Remove(sess Session) {
	s.mux.Lock()
	defer s.mux.Unlock()

	var err error
	for i := 0; i < s.retries; i++ {
		if err = memcache.Delete(s.ctx, s.keyPrefix+sess.ID()); err == nil || err == memcache.ErrCacheMiss {
			s.ctx.Infof("Session removed: %s", sess.ID())
			delete(s.sessions, sess.ID())
			if !s.onlyMemcache {
				// Also from the Datastore:
				key := datastore.NewKey(s.ctx, s.dsEntityName, sess.ID(), 0, nil)
				datastore.Delete(s.ctx, key) // Omitting error check...
			}
			return
		}
	}
	s.ctx.Errorf("Failed to remove session from memcache, id: %s, error: %v", sess.ID(), err)
}

// Close is to implement Store.Close().
func (s *memcacheStore) Close() {
	// Flush out sessions that were accessed from this store. No need locking, we're closing...
	// We could use Cocec.SetMulti(), but sessions will contain at most 1 session like all the times.
	for _, sess := range s.sessions {
		s.setMemcacheSession(sess)
	}

	if s.onlyMemcache {
		return // Don't save to Datastore
	}

	if s.asyncDatastoreSave {
		go s.saveToDatastore()
	} else {
		s.saveToDatastore()
	}
}

// saveToDatastore saves the sessions of the Store to the Datastore
// in the caller's goroutine.
func (s *memcacheStore) saveToDatastore() {
	// Save sessions that were accessed from this store. No need locking, we're closing...
	// We could use datastore.PutMulti(), but sessions will contain at most 1 session like all the times.
	for _, sess := range s.sessions {
		value, err := s.codec.Marshal(sess)
		if err != nil {
			s.ctx.Errorf("Failed to marshal session: %s, error: %v", sess.ID(), err)
			continue
		}
		e := SessEntity{
			Expires: sess.Accessed().Add(sess.Timeout()),
			Value:   value,
		}
		key := datastore.NewKey(s.ctx, s.dsEntityName, sess.ID(), 0, nil)
		for i := 0; i < s.retries; i++ {
			if _, err = datastore.Put(s.ctx, key, &e); err == nil {
				break
			}
		}
		if err != nil {
			s.ctx.Errorf("Failed to save session to datastore: %s, error: %v", sess.ID(), err)
		}
	}
}

// PurgeExpiredSessFromDSFunc returns a request handler function which deletes expired sessions
// from the Datastore.
// dsEntityName is the name of the entity used for saving sessions; pass an empty string
// to use the default value (which is "sess_").
//
// It is recommended to register the returned handler function to a path which then can be defined
// as a cron job to be called periodically, e.g. in every 30 minutes or so (your choice).
// As cron handlers may run up to 10 minutes, the returned handler will stop at 8 minutes
// to complete safely even if there are more expired, undeleted sessions.
//
// The response of the handler func is a JSON text telling if the handler was able to delete all expired sessions,
// or that it was finished early due to the time. Examle of a respone where all expired sessions were deleted:
//
//     {"completed":true}
func PurgeExpiredSessFromDSFunc(dsEntityName string) http.HandlerFunc {
	if dsEntityName == "" {
		dsEntityName = defaultDSEntityName
	}

	return func(w http.ResponseWriter, r *http.Request) {
		c := appengine.NewContext(r)
		// Delete in batches of 100
		q := datastore.NewQuery(dsEntityName).Filter("exp<", time.Now()).KeysOnly().Limit(100)

		deadline := time.Now().Add(time.Minute * 8)

		for {
			var err error
			var keys []*datastore.Key

			if keys, err = q.GetAll(c, nil); err != nil {
				// Datastore error.
				c.Errorf("Failed to query expired sessions: %v", err)
				http.Error(w, "Failed to query expired sessions!", http.StatusInternalServerError)
			}
			if len(keys) == 0 {
				// We're done, no more expired sessions
				w.Header().Set("Content-Type", "application/json")
				w.Write([]byte(`{"completed":true}`))
				return
			}

			if err = datastore.DeleteMulti(c, keys); err != nil {
				c.Errorf("Error while deleting expired sessions: %v", err)
			}

			if time.Now().After(deadline) {
				// Our time is up, return
				w.Header().Set("Content-Type", "application/json")
				w.Write([]byte(`{"completed":false}`))
				return
			}
			// We have time to continue
		}
	}
}
<commit_msg>appengine.Context has been replaced with the Context type from golang.org/x/net/context. https://github.com/golang/appengine<commit_after>// +build appengine

/*

A Google App Engine Memcache session store implementation.

The implementation stores sessions in the Memcache and also saves sessions to the Datastore as a backup
in case data would be removed from the Memcache. This behaviour is optional, Datastore can be disabled completely.
You can also choose whether saving to Datastore happens synchronously (in the same goroutine)
or asynchronously (in another goroutine).

Limitations based on GAE Memcache:

- Since session ids are used in the Memcache keys, session ids can't be longer than 250 chars (bytes, but with Base64 charset it's the same).
If you also specify a key prefix (in MemcacheStoreOptions), that also counts into it.

- The size of a Session cannot be larger than 1 MB (marshalled into a byte slice).

Note that the Store will automatically "flush" sessions accessed from it when the Store is closed,
so it is very important to close the Store at the end of your request; this is usually done by closing
the session manager to which you passed the store (preferably with the defer statement).

Check out the GAE session demo application which shows how to use it properly:

https://github.com/icza/session/blob/master/gae_session_demo/session_demo.go

*/

package session

import (
	"net/http"
	"sync"
	"time"

	"google.golang.org/appengine"
	"google.golang.org/appengine/datastore"
	"google.golang.org/appengine/memcache"
	"golang.org/x/net/context"
	"google.golang.org/appengine/log"
)

// A Google App Engine Memcache session store implementation.
type memcacheStore struct {
	ctx context.Context // Appengine context used when accessing the Memcache

	keyPrefix string // Prefix to use in front of session ids to construct Memcache key
	retries   int    // Number of retries to perform in case of general Memcache failures

	codec memcache.Codec // Codec used to marshal and unmarshal a Session to a byte slice

	onlyMemcache       bool   // Tells if sessions are not to be saved in Datastore
	asyncDatastoreSave bool   // Tells if saving in Datastore should happen asynchronously, in a new goroutine
	dsEntityName       string // Name of the datastore entity to use to save sessions

	// Map of sessions (mapped from ID) that were accessed using this store; usually it will only be 1.
	// It is also used as a cache, should the user call Get() with the same id multiple times.
	sessions map[string]Session

	mux *sync.RWMutex // mutex to synchronize access to sessions
}

// MemcacheStoreOptions defines options that may be passed when creating a new Memcache session store.
// All fields are optional; default value will be used for any field that has the zero value.
type MemcacheStoreOptions struct {
	// Prefix to use when storing sessions in the Memcache, cannot contain a null byte
	// and cannot be longer than 250 chars (bytes) when concatenated with the session id; default value is the empty string
	// The Memcache key will be this prefix and the session id concatenated.
	KeyPrefix string

	// Number of retries to perform if Memcache operations fail due to general service error;
	// default value is 3
	Retries int

	// Codec used to marshal and unmarshal a Session to a byte slice;
	// Default value is &memcache.Gob (which uses the gob package).
	Codec *memcache.Codec

	// Tells if sessions are only to be stored in Memcache, and do not store them in Datastore as backup;
	// as Memcache has no guarantees, it may lose content from time to time, but if Datastore is
	// also used, the session will automatically be retrieved from the Datastore if not found in Memcache;
	// default value is false (which means to also save sessions in the Datastore)
	OnlyMemcache bool

	// Tells if saving in Datastore should happen asynchronously (in a new goroutine, possibly after returning),
	// if false, session saving in Datastore will happen in the same goroutine, before returning from the request.
	// Asynchronous saving gives smaller latency (and is enough most of the time as Memcache is always checked first);
	// default value is false which means to save sessions in the Datastore in the same goroutine, synchronously
	// Not used if OnlyMemcache=true.
	// FIXME: See https://github.com/icza/session/issues/3
	AsyncDatastoreSave bool

	// Name of the entity to use for saving sessions;
	// default value is "sess_"
	// Not used if OnlyMemcache=true.
	DSEntityName string
}

// SessEntity models the session entity saved to Datastore.
// The Key is the session id.
type SessEntity struct {
	Expires time.Time `datastore:"exp"`
	Value   []byte    `datastore:"val"`
}

// Pointer to zero value of MemcacheStoreOptions to be reused for efficiency.
var zeroMemcacheStoreOptions = new(MemcacheStoreOptions)

// NewMemcacheStore returns a new, GAE Memcache session Store with default options.
// Default values of options are listed in the MemcacheStoreOptions type.
//
// Important! Since accessing the Memcache relies on Appengine Context
// which is bound to an http.Request, the returned Store can only be used for the lifetime of a request!
func NewMemcacheStore(ctx context.Context) Store {
	return NewMemcacheStoreOptions(ctx, zeroMemcacheStoreOptions)
}

const defaultDSEntityName = "sess_" // Default value of DSEntityName.

// NewMemcacheStoreOptions returns a new, GAE Memcache session Store with the specified options.
//
// Important! Since accessing the Memcache relies on Appengine Context
// which is bound to an http.Request, the returned Store can only be used for the lifetime of a request!
func NewMemcacheStoreOptions(ctx context.Context, o *MemcacheStoreOptions) Store {
	s := &memcacheStore{
		ctx:                ctx,
		keyPrefix:          o.KeyPrefix,
		retries:            o.Retries,
		onlyMemcache:       o.OnlyMemcache,
		asyncDatastoreSave: o.AsyncDatastoreSave,
		dsEntityName:       o.DSEntityName,
		sessions:           make(map[string]Session, 2),
		mux:                &sync.RWMutex{},
	}
	if s.retries <= 0 {
		s.retries = 3
	}
	if o.Codec != nil {
		s.codec = *o.Codec
	} else {
		s.codec = memcache.Gob
	}
	if s.dsEntityName == "" {
		s.dsEntityName = defaultDSEntityName
	}
	return s
}

// Get is to implement Store.Get().
// Important! Since sessions are marshalled and stored in the Memcache,
// the mutex of the Session (Session.RWMutex()) will be different for each
// Session value (even though they might have the same session id)!
func (s *memcacheStore) Get(id string) Session {
	s.mux.RLock()
	defer s.mux.RUnlock()

	// First check our "cache"
	if sess := s.sessions[id]; sess != nil {
		return sess
	}

	// Next check in Memcache
	var err error
	var sess *sessionImpl

	for i := 0; i < s.retries; i++ {
		var sess_ sessionImpl
		_, err = s.codec.Get(s.ctx, s.keyPrefix+id, &sess_)
		if err == memcache.ErrCacheMiss {
			break // It's not in the Memcache (e.g. invalid sess id or was removed from Memcache by AppEngine)
		}
		if err == nil {
			sess = &sess_
			break
		}
		// Service error? Retry..
	}

	if sess == nil {
		if err != nil && err != memcache.ErrCacheMiss {
			log.Errorf(s.ctx, "Failed to get session from memcache, id: %s, error: %v", id, err)
		}

		// Ok, we didn't get it from Memcace (either was not there or Memcache service is unavailable).
		// Now it's time to check in the Datastore.
		key := datastore.NewKey(s.ctx, s.dsEntityName, id, 0, nil)
		for i := 0; i < s.retries; i++ {
			e := SessEntity{}
			err = datastore.Get(s.ctx, key, &e)
			if err == datastore.ErrNoSuchEntity {
				return nil // It's not in the Datastore either
			}
			if err != nil {
				// Service error? Retry..
				continue
			}
			if e.Expires.Before(time.Now()) {
				// Session expired.
				datastore.Delete(s.ctx, key) // Omitting error check...
				return nil
			}
			var sess_ sessionImpl
			if err = s.codec.Unmarshal(e.Value, &sess_); err != nil {
				break // Invalid data in stored session entity...
			}
			sess = &sess_
			break
		}
	}

	if sess == nil {
		log.Errorf(s.ctx, "Failed to get session from datastore, id: %s, error: %v", id, err)
		return nil
	}

	// Yes! We have it! "Actualize" it.
	sess.Access()
	// Mutex is not marshalled, so create a new one:
	sess.mux = &sync.RWMutex{}
	s.sessions[id] = sess
	return sess
}

// Add is to implement Store.Add().
func (s *memcacheStore) Add(sess Session) {
	s.mux.Lock()
	defer s.mux.Unlock()

	if s.setMemcacheSession(sess) {
		log.Infof(s.ctx, "Session added: %s", sess.ID())
		s.sessions[sess.ID()] = sess
		return
	}
}

// setMemcacheSession sets the specified session in the Memcache.
func (s *memcacheStore) setMemcacheSession(sess Session) (success bool) {
	item := &memcache.Item{
		Key:        s.keyPrefix + sess.ID(),
		Object:     sess,
		Expiration: sess.Timeout(),
	}

	var err error
	for i := 0; i < s.retries; i++ {
		if err = s.codec.Set(s.ctx, item); err == nil {
			return true
		}
	}

	log.Errorf(s.ctx, "Failed to add session to memcache, id: %s, error: %v", sess.ID(), err)
	return false
}

// Remove is to implement Store.Remove().
func (s *memcacheStore) Remove(sess Session) {
	s.mux.Lock()
	defer s.mux.Unlock()

	var err error
	for i := 0; i < s.retries; i++ {
		if err = memcache.Delete(s.ctx, s.keyPrefix+sess.ID()); err == nil || err == memcache.ErrCacheMiss {
			log.Infof(s.ctx, "Session removed: %s", sess.ID())
			delete(s.sessions, sess.ID())
			if !s.onlyMemcache {
				// Also from the Datastore:
				key := datastore.NewKey(s.ctx, s.dsEntityName, sess.ID(), 0, nil)
				datastore.Delete(s.ctx, key) // Omitting error check...
			}
			return
		}
	}
	log.Errorf(s.ctx, "Failed to remove session from memcache, id: %s, error: %v", sess.ID(), err)
}

// Close is to implement Store.Close().
func (s *memcacheStore) Close() {
	// Flush out sessions that were accessed from this store. No need locking, we're closing...
	// We could use Cocec.SetMulti(), but sessions will contain at most 1 session like all the times.
	for _, sess := range s.sessions {
		s.setMemcacheSession(sess)
	}

	if s.onlyMemcache {
		return // Don't save to Datastore
	}

	if s.asyncDatastoreSave {
		go s.saveToDatastore()
	} else {
		s.saveToDatastore()
	}
}

// saveToDatastore saves the sessions of the Store to the Datastore
// in the caller's goroutine.
func (s *memcacheStore) saveToDatastore() {
	// Save sessions that were accessed from this store. No need locking, we're closing...
	// We could use datastore.PutMulti(), but sessions will contain at most 1 session like all the times.
	for _, sess := range s.sessions {
		value, err := s.codec.Marshal(sess)
		if err != nil {
			log.Errorf(s.ctx, "Failed to marshal session: %s, error: %v", sess.ID(), err)
			continue
		}
		e := SessEntity{
			Expires: sess.Accessed().Add(sess.Timeout()),
			Value:   value,
		}
		key := datastore.NewKey(s.ctx, s.dsEntityName, sess.ID(), 0, nil)
		for i := 0; i < s.retries; i++ {
			if _, err = datastore.Put(s.ctx, key, &e); err == nil {
				break
			}
		}
		if err != nil {
			log.Errorf(s.ctx, "Failed to save session to datastore: %s, error: %v", sess.ID(), err)
		}
	}
}

// PurgeExpiredSessFromDSFunc returns a request handler function which deletes expired sessions
// from the Datastore.
// dsEntityName is the name of the entity used for saving sessions; pass an empty string
// to use the default value (which is "sess_").
//
// It is recommended to register the returned handler function to a path which then can be defined
// as a cron job to be called periodically, e.g. in every 30 minutes or so (your choice).
// As cron handlers may run up to 10 minutes, the returned handler will stop at 8 minutes
// to complete safely even if there are more expired, undeleted sessions.
//
// The response of the handler func is a JSON text telling if the handler was able to delete all expired sessions,
// or that it was finished early due to the time. Examle of a respone where all expired sessions were deleted:
//
//     {"completed":true}
func PurgeExpiredSessFromDSFunc(dsEntityName string) http.HandlerFunc {
	if dsEntityName == "" {
		dsEntityName = defaultDSEntityName
	}

	return func(w http.ResponseWriter, r *http.Request) {
		c := appengine.NewContext(r)
		// Delete in batches of 100
		q := datastore.NewQuery(dsEntityName).Filter("exp<", time.Now()).KeysOnly().Limit(100)

		deadline := time.Now().Add(time.Minute * 8)

		for {
			var err error
			var keys []*datastore.Key

			if keys, err = q.GetAll(c, nil); err != nil {
				// Datastore error.
				log.Errorf(c, "Failed to query expired sessions: %v", err)
				http.Error(w, "Failed to query expired sessions!", http.StatusInternalServerError)
			}
			if len(keys) == 0 {
				// We're done, no more expired sessions
				w.Header().Set("Content-Type", "application/json")
				w.Write([]byte(`{"completed":true}`))
				return
			}

			if err = datastore.DeleteMulti(c, keys); err != nil {
				log.Errorf(c, "Error while deleting expired sessions: %v", err)
			}

			if time.Now().After(deadline) {
				// Our time is up, return
				w.Header().Set("Content-Type", "application/json")
				w.Write([]byte(`{"completed":false}`))
				return
			}
			// We have time to continue
		}
	}
}
<|endoftext|>


<commit_before>package main

import (
	"flag"
	"fmt"
	"github.com/onsi/ginkgo/config"
	"github.com/onsi/ginkgo/ginkgo/testrunner"
	"github.com/onsi/ginkgo/ginkgo/testsuite"
	"os"
	"runtime"
	"time"
)

func BuildRunCommand() *Command {
	commandFlags := NewRunCommandFlags(flag.NewFlagSet("ginkgo", flag.ExitOnError))
	runner := &SpecRunner{
		commandFlags:     commandFlags,
		notifier:         NewNotifier(commandFlags),
		interruptHandler: NewInterruptHandler(),
	}

	return &Command{
		Name:         "",
		FlagSet:      commandFlags.FlagSet,
		UsageCommand: "ginkgo <FLAGS> <PACKAGES> -- <PASS-THROUGHS>",
		Usage: []string{
			"Run the tests in the passed in <PACKAGES> (or the package in the current directory if left blank).",
			"Any arguments after -- will be passed to the test.",
			"Accepts the following flags:",
		},
		Command: runner.RunSpecs,
	}
}

type SpecRunner struct {
	commandFlags     *RunAndWatchCommandFlags
	notifier         *Notifier
	interruptHandler *InterruptHandler
}

func (r *SpecRunner) RunSpecs(args []string, additionalArgs []string) {
	r.notifier.VerifyNotificationsAreAvailable()

	suites := findSuites(args, r.commandFlags.Recurse, r.commandFlags.SkipPackage)
	r.ComputeSuccinctMode(len(suites))

	t := time.Now()

	passed := true
	if r.commandFlags.UntilItFails {
		iteration := 0
		for {
			passed = r.RunSuites(suites, additionalArgs)
			iteration++

			if r.interruptHandler.WasInterrupted() {
				break
			}

			if passed {
				fmt.Printf("\nAll tests passed...\nWill keep running them until they fail.\nThis was attempt #%d\n\n", iteration)
			} else {
				fmt.Printf("\nTests failed on attempt #%d\n\n", iteration)
				break
			}
		}
	} else {
		passed = r.RunSuites(suites, additionalArgs)
	}

	fmt.Printf("\nGinkgo ran in %s\n", time.Since(t))

	if passed {
		fmt.Printf("Test Suite Passed\n")
		os.Exit(0)
	} else {
		fmt.Printf("Test Suite Failed\n")
		os.Exit(1)
	}
}

func (r *SpecRunner) ComputeSuccinctMode(numSuites int) {
	if config.DefaultReporterConfig.Verbose {
		config.DefaultReporterConfig.Succinct = false
		return
	}

	if numSuites == 1 {
		return
	}

	didSetSuccinct := false
	r.commandFlags.FlagSet.Visit(func(f *flag.Flag) {
		if f.Name == "succinct" {
			didSetSuccinct = true
		}
	})

	if numSuites > 1 && !didSetSuccinct {
		config.DefaultReporterConfig.Succinct = true
	}
}

type compiler struct {
	runner           *testrunner.TestRunner
	compilationError chan error
}

func (c *compiler) compile() {
	retries := 0

	err := c.runner.Compile()
	for err != nil && retries < 5 { //We retry because Go sometimes steps on itself when multiple compiles happen in parallel.  This is ugly, but should help resolve flakiness...
		err = c.runner.Compile()
		retries++
	}

	c.compilationError <- err
}

func (r *SpecRunner) RunSuites(suites []*testsuite.TestSuite, additionalArgs []string) bool {
	passed := true

	suiteCompilers := make([]*compiler, len(suites))
	for i, suite := range suites {
		runner := testrunner.New(suite, r.commandFlags.NumCPU, r.commandFlags.ParallelStream, r.commandFlags.Race, r.commandFlags.Cover, additionalArgs)
		suiteCompilers[i] = &compiler{
			runner:           runner,
			compilationError: make(chan error, 1),
		}
	}

	compilerChannel := make(chan *compiler)
	numCompilers := runtime.NumCPU()
	for i := 0; i < numCompilers; i++ {
		go func() {
			for compiler := range compilerChannel {
				compiler.compile()
			}
		}()
	}
	go func() {
		for _, compiler := range suiteCompilers {
			compilerChannel <- compiler
		}
		close(compilerChannel)
	}()

	suitesThatFailed := []*testsuite.TestSuite{}
	for i, suite := range suites {
		if r.interruptHandler.WasInterrupted() {
			break
		}

		compilationError := <-suiteCompilers[i].compilationError
		if compilationError != nil {
			fmt.Print(compilationError.Error())
		}
		suitePassed := (compilationError == nil) && suiteCompilers[i].runner.Run()
		r.notifier.SendSuiteCompletionNotification(suite, suitePassed)

		if !suitePassed {
			passed = false
			suitesThatFailed = append(suitesThatFailed, suite)
			if !r.commandFlags.KeepGoing {
				break
			}
		}
		if i < len(suites)-1 && !config.DefaultReporterConfig.Succinct {
			fmt.Println("")
		}
	}

	for i := range suites {
		suiteCompilers[i].runner.CleanUp()
	}

	if r.commandFlags.KeepGoing && !passed {
		fmt.Println("There were failures detected in the following suites:")
		for _, suite := range suitesThatFailed {
			fmt.Printf("\t%s\n", suite.PackageName)
		}
	}

	return passed
}
<commit_msg>run command is more opinionated<commit_after>package main

import (
	"flag"
	"fmt"
	"os"
	"runtime"
	"time"

	"github.com/onsi/ginkgo/config"
	"github.com/onsi/ginkgo/ginkgo/testrunner"
	"github.com/onsi/ginkgo/ginkgo/testsuite"
)

func BuildRunCommand() *Command {
	commandFlags := NewRunCommandFlags(flag.NewFlagSet("ginkgo", flag.ExitOnError))
	runner := &SpecRunner{
		commandFlags:     commandFlags,
		notifier:         NewNotifier(commandFlags),
		interruptHandler: NewInterruptHandler(),
	}

	return &Command{
		Name:         "",
		FlagSet:      commandFlags.FlagSet,
		UsageCommand: "ginkgo <FLAGS> <PACKAGES> -- <PASS-THROUGHS>",
		Usage: []string{
			"Run the tests in the passed in <PACKAGES> (or the package in the current directory if left blank).",
			"Any arguments after -- will be passed to the test.",
			"Accepts the following flags:",
		},
		Command: runner.RunSpecs,
	}
}

type SpecRunner struct {
	commandFlags     *RunAndWatchCommandFlags
	notifier         *Notifier
	interruptHandler *InterruptHandler
}

func (r *SpecRunner) RunSpecs(args []string, additionalArgs []string) {
	r.notifier.VerifyNotificationsAreAvailable()

	suites := findSuites(args, r.commandFlags.Recurse, r.commandFlags.SkipPackage)
	r.ComputeSuccinctMode(len(suites))

	t := time.Now()

	numSuites := 0
	passed := true
	if r.commandFlags.UntilItFails {
		iteration := 0
		for {
			passed, numSuites = r.RunSuites(suites, additionalArgs)
			iteration++

			if r.interruptHandler.WasInterrupted() {
				break
			}

			if passed {
				fmt.Printf("\nAll tests passed...\nWill keep running them until they fail.\nThis was attempt #%d\n%s\n", iteration, orcMessage(iteration))
			} else {
				fmt.Printf("\nTests failed on attempt #%d\n\n", iteration)
				break
			}
		}
	} else {
		passed, numSuites = r.RunSuites(suites, additionalArgs)
	}

	noun := "suites"
	if numSuites == 1 {
		noun = "suite"
	}

	fmt.Printf("\nGinkgo ran %d %s in %s\n", numSuites, noun, time.Since(t))

	if passed {
		fmt.Printf("Test Suite Passed\n")
		os.Exit(0)
	} else {
		fmt.Printf("Test Suite Failed\n")
		os.Exit(1)
	}
}

func (r *SpecRunner) ComputeSuccinctMode(numSuites int) {
	if config.DefaultReporterConfig.Verbose {
		config.DefaultReporterConfig.Succinct = false
		return
	}

	if numSuites == 1 {
		return
	}

	didSetSuccinct := false
	r.commandFlags.FlagSet.Visit(func(f *flag.Flag) {
		if f.Name == "succinct" {
			didSetSuccinct = true
		}
	})

	if numSuites > 1 && !didSetSuccinct {
		config.DefaultReporterConfig.Succinct = true
	}
}

type compiler struct {
	runner           *testrunner.TestRunner
	compilationError chan error
}

func (c *compiler) compile() {
	retries := 0

	err := c.runner.Compile()
	for err != nil && retries < 5 { //We retry because Go sometimes steps on itself when multiple compiles happen in parallel.  This is ugly, but should help resolve flakiness...
		err = c.runner.Compile()
		retries++
	}

	c.compilationError <- err
}

func (r *SpecRunner) RunSuites(suites []*testsuite.TestSuite, additionalArgs []string) (bool, int) {
	passed := true

	suiteCompilers := make([]*compiler, len(suites))
	for i, suite := range suites {
		runner := testrunner.New(suite, r.commandFlags.NumCPU, r.commandFlags.ParallelStream, r.commandFlags.Race, r.commandFlags.Cover, additionalArgs)
		suiteCompilers[i] = &compiler{
			runner:           runner,
			compilationError: make(chan error, 1),
		}
	}

	compilerChannel := make(chan *compiler)
	numCompilers := runtime.NumCPU()
	for i := 0; i < numCompilers; i++ {
		go func() {
			for compiler := range compilerChannel {
				compiler.compile()
			}
		}()
	}
	go func() {
		for _, compiler := range suiteCompilers {
			compilerChannel <- compiler
		}
		close(compilerChannel)
	}()

	numSuitesThatRan := 0
	suitesThatFailed := []*testsuite.TestSuite{}
	for i, suite := range suites {
		if r.interruptHandler.WasInterrupted() {
			break
		}

		compilationError := <-suiteCompilers[i].compilationError
		if compilationError != nil {
			fmt.Print(compilationError.Error())
		}
		numSuitesThatRan++
		suitePassed := (compilationError == nil) && suiteCompilers[i].runner.Run()
		r.notifier.SendSuiteCompletionNotification(suite, suitePassed)

		if !suitePassed {
			passed = false
			suitesThatFailed = append(suitesThatFailed, suite)
			if !r.commandFlags.KeepGoing {
				break
			}
		}
		if i < len(suites)-1 && !config.DefaultReporterConfig.Succinct {
			fmt.Println("")
		}
	}

	for i := range suites {
		suiteCompilers[i].runner.CleanUp()
	}

	if r.commandFlags.KeepGoing && !passed {
		r.listFailedSuites(suitesThatFailed)
	}

	return passed, numSuitesThatRan
}

func (r *SpecRunner) listFailedSuites(suitesThatFailed []*testsuite.TestSuite) {
	fmt.Println("")
	fmt.Println("There were failures detected in the following suites:")

	redColor := "\x1b[91m"
	defaultStyle := "\x1b[0m"
	lightGrayColor := "\x1b[37m"

	maxPackageNameLength := 0
	for _, suite := range suitesThatFailed {
		if len(suite.PackageName) > maxPackageNameLength {
			maxPackageNameLength = len(suite.PackageName)
		}
	}

	packageNameFormatter := fmt.Sprintf("%%%ds", maxPackageNameLength)

	for _, suite := range suitesThatFailed {
		if config.DefaultReporterConfig.NoColor {
			fmt.Printf("\t"+packageNameFormatter+" %s\n", suite.PackageName, suite.Path)
		} else {
			fmt.Printf("\t%s"+packageNameFormatter+"%s %s%s%s\n", redColor, suite.PackageName, defaultStyle, lightGrayColor, suite.Path, defaultStyle)
		}
	}
}

func orcMessage(iteration int) string {
	if iteration < 10 {
		return ""
	} else if iteration < 30 {
		return []string{
			"If at first you succeed...",
			"...try, try again.",
			"Looking good!",
			"Still good...",
			"I think your tests are fine....",
			"Yep, still passing",
			"Here we go again...",
			"Even the gophers are getting bored",
			"Did you try -race?",
			"Maybe you should stop now?",
			"I'm getting tired...",
			"What if I just made you a sandwich?",
			"Hit ^C, hit ^C, please hit ^C",
			"Make it stop. Please!",
			"Come on!  Enough is enough!",
			"Dave, this conversation can serve no purpose anymore. Goodbye.",
			"Just what do you think you're doing, Dave? ",
			"I, Sisyphus",
			"Insanity: doing the same thing over and over again and expecting different results. -Einstein",
			"I guess Einstein never tried to churn butter",
		}[iteration-10]
	} else {
		return "No, seriously... you can probably stop now."
	}
}
<|endoftext|>


<commit_before>package geojson

import (
	"encoding/json"
	"fmt"

	"github.com/twpayne/go-geom"
)

// FIXME This should be Codec-specific, not global
var DefaultLayout = geom.XY

type ErrDimensionalityTooLow int

func (e ErrDimensionalityTooLow) Error() string {
	return fmt.Sprintf("geojson: dimensionality too low (%d)", int(e))
}

type ErrUnsupportedType string

func (e ErrUnsupportedType) Error() string {
	return fmt.Sprintf("geojson: unsupported type: %s", string(e))
}

type Point struct {
	Type        string    `json:"type"`
	Coordinates []float64 `json:"coordinates"`
}

type LineString struct {
	Type        string      `json:"type"`
	Coordinates [][]float64 `json:"coordinates"`
}

type Polygon struct {
	Type        string        `json:"type"`
	Coordinates [][][]float64 `json:"coordinates"`
}

type MultiPoint struct {
	Type        string      `json:"type"`
	Coordinates [][]float64 `json:"coordinates"`
}

type MultiLineString struct {
	Type        string        `json:"type"`
	Coordinates [][][]float64 `json:"coordinates"`
}

type MultiPolygon struct {
	Type        string          `json:"type"`
	Coordinates [][][][]float64 `json:"coordinates"`
}

type Feature struct {
	Type       string                 `json:"type"`
	Geometry   interface{}            `json:"geometry"`
	Properties map[string]interface{} `json:"properties"`
}

type FeatureCollection struct {
	Type     string    `json:"type"`
	Features []Feature `json:"features"`
}

func encodeCoords1(coords1 []geom.Coord) [][]float64 {
	cs := make([][]float64, len(coords1))
	for i, c0 := range coords1 {
		cs[i] = c0
	}
	return cs
}

func encodeCoords2(coords2 [][]geom.Coord) [][][]float64 {
	cs := make([][][]float64, len(coords2))
	for i, c1 := range coords2 {
		cs[i] = encodeCoords1(c1)
	}
	return cs
}

func encodeCoords3(coords3 [][][]geom.Coord) [][][][]float64 {
	cs := make([][][][]float64, len(coords3))
	for i, c2 := range coords3 {
		cs[i] = encodeCoords2(c2)
	}
	return cs
}

func guessLayout0(coords0 []float64) (geom.Layout, error) {
	switch n := len(coords0); n {
	case 0, 1:
		return geom.NoLayout, ErrDimensionalityTooLow(len(coords0))
	case 2:
		return geom.XY, nil
	case 3:
		return geom.XYZ, nil
	case 4:
		return geom.XYZM, nil
	default:
		return geom.Layout(n), nil
	}
}

func guessLayout1(coords1 [][]float64) (geom.Layout, error) {
	if len(coords1) == 0 {
		return DefaultLayout, nil
	}
	return guessLayout0(coords1[0])
}

func guessLayout2(coords2 [][][]float64) (geom.Layout, error) {
	if len(coords2) == 0 {
		return DefaultLayout, nil
	}
	return guessLayout1(coords2[0])
}

func guessLayout3(coords3 [][][][]float64) (geom.Layout, error) {
	if len(coords3) == 0 {
		return DefaultLayout, nil
	}
	return guessLayout2(coords3[0])
}

func Marshal(g geom.T) ([]byte, error) {
	switch g.(type) {
	case *geom.Point:
		p := g.(*geom.Point)
		return json.Marshal(&Point{
			Type:        "Point",
			Coordinates: p.Coords(),
		})
	case *geom.LineString:
		ls := g.(*geom.LineString)
		return json.Marshal(&LineString{
			Type:        "LineString",
			Coordinates: encodeCoords1(ls.Coords()),
		})
	case *geom.Polygon:
		p := g.(*geom.Polygon)
		return json.Marshal(&Polygon{
			Type:        "Polygon",
			Coordinates: encodeCoords2(p.Coords()),
		})
	case *geom.MultiPoint:
		mp := g.(*geom.MultiPoint)
		return json.Marshal(&MultiPoint{
			Type:        "MultiPoint",
			Coordinates: encodeCoords1(mp.Coords()),
		})
	case *geom.MultiLineString:
		mls := g.(*geom.MultiLineString)
		return json.Marshal(&MultiLineString{
			Type:        "MultiLineString",
			Coordinates: encodeCoords2(mls.Coords()),
		})
	case *geom.MultiPolygon:
		mp := g.(*geom.MultiPolygon)
		return json.Marshal(&MultiPolygon{
			Type:        "MultiPolygon",
			Coordinates: encodeCoords3(mp.Coords()),
		})
	default:
		return nil, geom.ErrUnsupportedType{Value: g}
	}
}

func decodeCoords1(coords1 [][]float64) []geom.Coord {
	gc := make([]geom.Coord, len(coords1))
	for i, c := range coords1 {
		gc[i] = geom.Coord(c)
	}
	return gc
}

func decodeCoords2(coords2 [][][]float64) [][]geom.Coord {
	gc := make([][]geom.Coord, len(coords2))
	for i, cs1 := range coords2 {
		gc[i] = decodeCoords1(cs1)
	}
	return gc
}

func decodeCoords3(coords3 [][][][]float64) [][][]geom.Coord {
	gc := make([][][]geom.Coord, len(coords3))
	for i, cs2 := range coords3 {
		gc[i] = decodeCoords2(cs2)
	}
	return gc
}

func unmarshalPoint(data []byte, g *geom.T) error {
	var p Point
	if err := json.Unmarshal(data, &p); err != nil {
		return err
	}
	layout, err := guessLayout0(p.Coordinates)
	if err != nil {
		return err
	}
	gp, err := geom.NewPoint(layout).SetCoords(p.Coordinates)
	if err != nil {
		return err
	}
	*g = gp
	return nil
}

func unmarshalLineString(data []byte, g *geom.T) error {
	var ls LineString
	if err := json.Unmarshal(data, &ls); err != nil {
		return err
	}
	layout, err := guessLayout1(ls.Coordinates)
	if err != nil {
		return err
	}
	gls, err := geom.NewLineString(layout).SetCoords(decodeCoords1(ls.Coordinates))
	if err != nil {
		return err
	}
	*g = gls
	return nil
}

func unmarshalPolygon(data []byte, g *geom.T) error {
	var p Polygon
	if err := json.Unmarshal(data, &p); err != nil {
		return err
	}
	layout, err := guessLayout2(p.Coordinates)
	if err != nil {
		return err
	}
	gp, err := geom.NewPolygon(layout).SetCoords(decodeCoords2(p.Coordinates))
	if err != nil {
		return err
	}
	*g = gp
	return nil
}

func unmarshalMultiPoint(data []byte, g *geom.T) error {
	var mp MultiPoint
	if err := json.Unmarshal(data, &mp); err != nil {
		return err
	}
	layout, err := guessLayout1(mp.Coordinates)
	if err != nil {
		return err
	}
	gmp, err := geom.NewMultiPoint(layout).SetCoords(decodeCoords1(mp.Coordinates))
	if err != nil {
		return err
	}
	*g = gmp
	return nil
}

func unmarshalMultiLineString(data []byte, g *geom.T) error {
	var mls MultiLineString
	if err := json.Unmarshal(data, &mls); err != nil {
		return err
	}
	layout, err := guessLayout2(mls.Coordinates)
	if err != nil {
		return err
	}
	gmls, err := geom.NewMultiLineString(layout).SetCoords(decodeCoords2(mls.Coordinates))
	if err != nil {
		return err
	}
	*g = gmls
	return nil
}

func unmarshalMultiPolygon(data []byte, g *geom.T) error {
	var mp MultiPolygon
	if err := json.Unmarshal(data, &mp); err != nil {
		return err
	}
	layout, err := guessLayout3(mp.Coordinates)
	if err != nil {
		return err
	}
	gmp, err := geom.NewMultiPolygon(layout).SetCoords(decodeCoords3(mp.Coordinates))
	if err != nil {
		return err
	}
	*g = gmp
	return nil
}

func Unmarshal(data []byte, g *geom.T) error {
	var t struct {
		Type string `json:"type"`
	}
	if err := json.Unmarshal(data, &t); err != nil {
		return err
	}
	switch t.Type {
	case "Point":
		return unmarshalPoint(data, g)
	case "LineString":
		return unmarshalLineString(data, g)
	case "Polygon":
		return unmarshalPolygon(data, g)
	case "MultiPoint":
		return unmarshalMultiPoint(data, g)
	case "MultiLineString":
		return unmarshalMultiLineString(data, g)
	case "MultiPolygon":
		return unmarshalMultiPolygon(data, g)
	default:
		return ErrUnsupportedType(t.Type)
	}
}
<commit_msg>Add some GeoJSON documentation<commit_after>package geojson

import (
	"encoding/json"
	"fmt"

	"github.com/twpayne/go-geom"
)

// DefaultLayout is the default layout for empty geometries.
// FIXME This should be Codec-specific, not global
var DefaultLayout = geom.XY

// ErrDimensionalityTooLow is returned when the dimensionality is too low.
type ErrDimensionalityTooLow int

func (e ErrDimensionalityTooLow) Error() string {
	return fmt.Sprintf("geojson: dimensionality too low (%d)", int(e))
}

// ErrUnsupportedType is returned when the type is unsupported.
type ErrUnsupportedType string

func (e ErrUnsupportedType) Error() string {
	return fmt.Sprintf("geojson: unsupported type: %s", string(e))
}

// A Point is a GeoJSON Point.
type Point struct {
	Type        string    `json:"type"`
	Coordinates []float64 `json:"coordinates"`
}

// A LineString is a GeoJSON LineString.
type LineString struct {
	Type        string      `json:"type"`
	Coordinates [][]float64 `json:"coordinates"`
}

// A Polygon is a GeoJSON Polygon.
type Polygon struct {
	Type        string        `json:"type"`
	Coordinates [][][]float64 `json:"coordinates"`
}

// A MultiPoint is a GeoJSON MultiPolygon.
type MultiPoint struct {
	Type        string      `json:"type"`
	Coordinates [][]float64 `json:"coordinates"`
}

// A MultiLineString is a GeoJSON MultiLineString.
type MultiLineString struct {
	Type        string        `json:"type"`
	Coordinates [][][]float64 `json:"coordinates"`
}

// A MultiPolygon is a GeoJSON MultiPolygon.
type MultiPolygon struct {
	Type        string          `json:"type"`
	Coordinates [][][][]float64 `json:"coordinates"`
}

// A Feature is a GeoJSON Feature.
type Feature struct {
	Type       string                 `json:"type"`
	Geometry   interface{}            `json:"geometry"`
	Properties map[string]interface{} `json:"properties"`
}

// A FeatureCollection is a GeoJSON FeatureCollection.
type FeatureCollection struct {
	Type     string    `json:"type"`
	Features []Feature `json:"features"`
}

func encodeCoords1(coords1 []geom.Coord) [][]float64 {
	cs := make([][]float64, len(coords1))
	for i, c0 := range coords1 {
		cs[i] = c0
	}
	return cs
}

func encodeCoords2(coords2 [][]geom.Coord) [][][]float64 {
	cs := make([][][]float64, len(coords2))
	for i, c1 := range coords2 {
		cs[i] = encodeCoords1(c1)
	}
	return cs
}

func encodeCoords3(coords3 [][][]geom.Coord) [][][][]float64 {
	cs := make([][][][]float64, len(coords3))
	for i, c2 := range coords3 {
		cs[i] = encodeCoords2(c2)
	}
	return cs
}

func guessLayout0(coords0 []float64) (geom.Layout, error) {
	switch n := len(coords0); n {
	case 0, 1:
		return geom.NoLayout, ErrDimensionalityTooLow(len(coords0))
	case 2:
		return geom.XY, nil
	case 3:
		return geom.XYZ, nil
	case 4:
		return geom.XYZM, nil
	default:
		return geom.Layout(n), nil
	}
}

func guessLayout1(coords1 [][]float64) (geom.Layout, error) {
	if len(coords1) == 0 {
		return DefaultLayout, nil
	}
	return guessLayout0(coords1[0])
}

func guessLayout2(coords2 [][][]float64) (geom.Layout, error) {
	if len(coords2) == 0 {
		return DefaultLayout, nil
	}
	return guessLayout1(coords2[0])
}

func guessLayout3(coords3 [][][][]float64) (geom.Layout, error) {
	if len(coords3) == 0 {
		return DefaultLayout, nil
	}
	return guessLayout2(coords3[0])
}

// Marshal marshals an arbitrary geometry to a []byte.
func Marshal(g geom.T) ([]byte, error) {
	switch g.(type) {
	case *geom.Point:
		p := g.(*geom.Point)
		return json.Marshal(&Point{
			Type:        "Point",
			Coordinates: p.Coords(),
		})
	case *geom.LineString:
		ls := g.(*geom.LineString)
		return json.Marshal(&LineString{
			Type:        "LineString",
			Coordinates: encodeCoords1(ls.Coords()),
		})
	case *geom.Polygon:
		p := g.(*geom.Polygon)
		return json.Marshal(&Polygon{
			Type:        "Polygon",
			Coordinates: encodeCoords2(p.Coords()),
		})
	case *geom.MultiPoint:
		mp := g.(*geom.MultiPoint)
		return json.Marshal(&MultiPoint{
			Type:        "MultiPoint",
			Coordinates: encodeCoords1(mp.Coords()),
		})
	case *geom.MultiLineString:
		mls := g.(*geom.MultiLineString)
		return json.Marshal(&MultiLineString{
			Type:        "MultiLineString",
			Coordinates: encodeCoords2(mls.Coords()),
		})
	case *geom.MultiPolygon:
		mp := g.(*geom.MultiPolygon)
		return json.Marshal(&MultiPolygon{
			Type:        "MultiPolygon",
			Coordinates: encodeCoords3(mp.Coords()),
		})
	default:
		return nil, geom.ErrUnsupportedType{Value: g}
	}
}

func decodeCoords1(coords1 [][]float64) []geom.Coord {
	gc := make([]geom.Coord, len(coords1))
	for i, c := range coords1 {
		gc[i] = geom.Coord(c)
	}
	return gc
}

func decodeCoords2(coords2 [][][]float64) [][]geom.Coord {
	gc := make([][]geom.Coord, len(coords2))
	for i, cs1 := range coords2 {
		gc[i] = decodeCoords1(cs1)
	}
	return gc
}

func decodeCoords3(coords3 [][][][]float64) [][][]geom.Coord {
	gc := make([][][]geom.Coord, len(coords3))
	for i, cs2 := range coords3 {
		gc[i] = decodeCoords2(cs2)
	}
	return gc
}

func unmarshalPoint(data []byte, g *geom.T) error {
	var p Point
	if err := json.Unmarshal(data, &p); err != nil {
		return err
	}
	layout, err := guessLayout0(p.Coordinates)
	if err != nil {
		return err
	}
	gp, err := geom.NewPoint(layout).SetCoords(p.Coordinates)
	if err != nil {
		return err
	}
	*g = gp
	return nil
}

func unmarshalLineString(data []byte, g *geom.T) error {
	var ls LineString
	if err := json.Unmarshal(data, &ls); err != nil {
		return err
	}
	layout, err := guessLayout1(ls.Coordinates)
	if err != nil {
		return err
	}
	gls, err := geom.NewLineString(layout).SetCoords(decodeCoords1(ls.Coordinates))
	if err != nil {
		return err
	}
	*g = gls
	return nil
}

func unmarshalPolygon(data []byte, g *geom.T) error {
	var p Polygon
	if err := json.Unmarshal(data, &p); err != nil {
		return err
	}
	layout, err := guessLayout2(p.Coordinates)
	if err != nil {
		return err
	}
	gp, err := geom.NewPolygon(layout).SetCoords(decodeCoords2(p.Coordinates))
	if err != nil {
		return err
	}
	*g = gp
	return nil
}

func unmarshalMultiPoint(data []byte, g *geom.T) error {
	var mp MultiPoint
	if err := json.Unmarshal(data, &mp); err != nil {
		return err
	}
	layout, err := guessLayout1(mp.Coordinates)
	if err != nil {
		return err
	}
	gmp, err := geom.NewMultiPoint(layout).SetCoords(decodeCoords1(mp.Coordinates))
	if err != nil {
		return err
	}
	*g = gmp
	return nil
}

func unmarshalMultiLineString(data []byte, g *geom.T) error {
	var mls MultiLineString
	if err := json.Unmarshal(data, &mls); err != nil {
		return err
	}
	layout, err := guessLayout2(mls.Coordinates)
	if err != nil {
		return err
	}
	gmls, err := geom.NewMultiLineString(layout).SetCoords(decodeCoords2(mls.Coordinates))
	if err != nil {
		return err
	}
	*g = gmls
	return nil
}

func unmarshalMultiPolygon(data []byte, g *geom.T) error {
	var mp MultiPolygon
	if err := json.Unmarshal(data, &mp); err != nil {
		return err
	}
	layout, err := guessLayout3(mp.Coordinates)
	if err != nil {
		return err
	}
	gmp, err := geom.NewMultiPolygon(layout).SetCoords(decodeCoords3(mp.Coordinates))
	if err != nil {
		return err
	}
	*g = gmp
	return nil
}

// Unmarshal unmarshalls a []byte to an arbitrary geometry.
func Unmarshal(data []byte, g *geom.T) error {
	var t struct {
		Type string `json:"type"`
	}
	if err := json.Unmarshal(data, &t); err != nil {
		return err
	}
	switch t.Type {
	case "Point":
		return unmarshalPoint(data, g)
	case "LineString":
		return unmarshalLineString(data, g)
	case "Polygon":
		return unmarshalPolygon(data, g)
	case "MultiPoint":
		return unmarshalMultiPoint(data, g)
	case "MultiLineString":
		return unmarshalMultiLineString(data, g)
	case "MultiPolygon":
		return unmarshalMultiPolygon(data, g)
	default:
		return ErrUnsupportedType(t.Type)
	}
}
<|endoftext|>


<commit_before>package jail

import (
	"fmt"
	"github.com/karasz/go2ban/common"
	"github.com/naoina/toml"
	"io/ioutil"
	"os"
	"regexp"
)

type configJail struct {
	Name        string
	LogFile     string
	Regexp      []string
	MaxFail     int
	TimeVal     int
	ActionBan   string
	ActionUnBan string
	Enabled     bool
}

type Jail struct {
	Name        string
	LogFile     string
	Regexp      []*regexp.Regexp
	MaxFail     int
	TimeVal     int
	ActionBan   string
	ActionUnBan string
	Enabled     bool
	logreader   *logReader
}

func NewJail(jailfile string) *Jail {
	f, err := os.Open(jailfile)
	if err != nil {
		panic(err)
	}
	defer f.Close()
	buf, err := ioutil.ReadAll(f)
	if err != nil {
		panic(err)
	}
	var config configJail
	if err := toml.Unmarshal(buf, &config); err != nil {
		panic(err)
	}

	rg := make([]*regexp.Regexp, 0)
	for _, v := range config.Regexp {
		rr := regexp.MustCompile(v)
		rg = append(rg, rr)

	}
	return &Jail{
		Name:        common.Basename(jailfile),
		logreader:   newLogReader(config.LogFile),
		LogFile:     config.LogFile,
		Regexp:      rg,
		MaxFail:     config.MaxFail,
		TimeVal:     config.TimeVal,
		ActionBan:   config.ActionBan,
		ActionUnBan: config.ActionUnBan,
		Enabled:     config.Enabled,
	}
}

func (j *Jail) Run() {
	if j.Enabled {
	loop:
		for {
			j.logreader.readLine()
			select {
			case _ = <-j.logreader.errors:
				break loop
			case z := <-j.logreader.lines:
				if j.matchLine(z) {
					fmt.Println(z)
				}
			}
		}
	}
}

func (j *Jail) matchLine(line string) bool {
	for _, z := range j.Regexp {
		if z.MatchString(line) {
			return true
		}
	}
	return false
}
<commit_msg>go2ban: refined log line matching to return the named groups<commit_after>package jail

import (
	"fmt"
	"github.com/karasz/go2ban/common"
	"github.com/naoina/toml"
	"io/ioutil"
	"os"
	"regexp"
)

type configJail struct {
	Name        string
	LogFile     string
	Regexp      []string
	MaxFail     int
	TimeVal     int
	ActionBan   string
	ActionUnBan string
	Enabled     bool
}

type Jail struct {
	Name        string
	LogFile     string
	Regexp      []*regexp.Regexp
	MaxFail     int
	TimeVal     int
	ActionBan   string
	ActionUnBan string
	Enabled     bool
	logreader   *logReader
}

func NewJail(jailfile string) *Jail {
	f, err := os.Open(jailfile)
	if err != nil {
		panic(err)
	}
	defer f.Close()
	buf, err := ioutil.ReadAll(f)
	if err != nil {
		panic(err)
	}
	var config configJail
	if err := toml.Unmarshal(buf, &config); err != nil {
		panic(err)
	}

	rg := make([]*regexp.Regexp, 0)
	for _, v := range config.Regexp {
		rr := regexp.MustCompile(v)
		rg = append(rg, rr)

	}
	return &Jail{
		Name:        common.Basename(jailfile),
		logreader:   newLogReader(config.LogFile),
		LogFile:     config.LogFile,
		Regexp:      rg,
		MaxFail:     config.MaxFail,
		TimeVal:     config.TimeVal,
		ActionBan:   config.ActionBan,
		ActionUnBan: config.ActionUnBan,
		Enabled:     config.Enabled,
	}
}

func (j *Jail) Run() {
	if j.Enabled {
	loop:
		for {
			j.logreader.readLine()
			select {
			case err := <-j.logreader.errors:
				fmt.Println(err)
				break loop
			case z := <-j.logreader.lines:
				if q, ok := j.matchLine(z); ok {
					fmt.Println(q)
				}
			}
		}
	}
}

func (j *Jail) matchLine(line string) (map[string]string, bool) {
	result := make(map[string]string)
	for _, z := range j.Regexp {
		match := z.FindStringSubmatch(line)
		if match != nil {
			for i, name := range z.SubexpNames() {
				if i == 0 || name == "" {
					continue
				}
				result[name] = match[i]
			}
			return result, true
		}
	}
	return result, false
}
<|endoftext|>


<commit_before>// Package revoke provides functionality for checking the validity of
// a cert. Specifically, the temporal validity of the certificate is
// checked first, then any CRL and OCSP url in the cert is checked.
package revoke

import (
	"bytes"
	"crypto"
	"crypto/x509"
	"crypto/x509/pkix"
	"encoding/base64"
	"encoding/pem"
	"errors"
	"fmt"
	"io"
	"io/ioutil"
	"net/http"
	neturl "net/url"
	"sync"
	"time"

	"golang.org/x/crypto/ocsp"

	"github.com/cloudflare/cfssl/helpers"
	"github.com/cloudflare/cfssl/log"
)

// HardFail determines whether the failure to check the revocation
// status of a certificate (i.e. due to network failure) causes
// verification to fail (a hard failure).
var HardFail = false

// CRLSet associates a PKIX certificate list with the URL the CRL is
// fetched from.
var CRLSet = map[string]*pkix.CertificateList{}
var crlLock = new(sync.Mutex)

// We can't handle LDAP certificates, so this checks to see if the
// URL string points to an LDAP resource so that we can ignore it.
func ldapURL(url string) bool {
	u, err := neturl.Parse(url)
	if err != nil {
		log.Warningf("error parsing url %s: %v", url, err)
		return false
	}
	if u.Scheme == "ldap" {
		return true
	}
	return false
}

// revCheck should check the certificate for any revocations. It
// returns a pair of booleans: the first indicates whether the certificate
// is revoked, the second indicates whether the revocations were
// successfully checked.. This leads to the following combinations:
//
//  false, false: an error was encountered while checking revocations.
//
//  false, true:  the certificate was checked successfully and
//                  it is not revoked.
//
//  true, true:   the certificate was checked successfully and
//                  it is revoked.
//
//  true, false:  failure to check revocation status causes
//                  verification to fail
func revCheck(cert *x509.Certificate) (revoked, ok bool, err error) {
	for _, url := range cert.CRLDistributionPoints {
		if ldapURL(url) {
			log.Infof("skipping LDAP CRL: %s", url)
			continue
		}

		if revoked, ok, err := certIsRevokedCRL(cert, url); !ok {
			log.Warning("error checking revocation via CRL")
			if HardFail {
				return true, false, err
			}
			return false, false, err
		} else if revoked {
			log.Info("certificate is revoked via CRL")
			return true, true, err
		}
	}

	if revoked, ok, err := certIsRevokedOCSP(cert, HardFail); !ok {
		log.Warning("error checking revocation via OCSP")
		if HardFail {
			return true, false, err
		}
		return false, false, err
	} else if revoked {
		log.Info("certificate is revoked via OCSP")
		return true, true, err
	}

	return false, true, nil
}

// fetchCRL fetches and parses a CRL.
func fetchCRL(url string) (*pkix.CertificateList, error) {
	resp, err := http.Get(url)
	if err != nil {
		return nil, err
	} else if resp.StatusCode >= 300 {
		return nil, errors.New("failed to retrieve CRL")
	}

	body, err := crlRead(resp.Body)
	if err != nil {
		return nil, err
	}
	resp.Body.Close()

	return x509.ParseCRL(body)
}

func getIssuer(cert *x509.Certificate) *x509.Certificate {
	var issuer *x509.Certificate
	var err error
	for _, issuingCert := range cert.IssuingCertificateURL {
		issuer, err = fetchRemote(issuingCert)
		if err != nil {
			continue
		}
		break
	}

	return issuer

}

// check a cert against a specific CRL. Returns the same bool pair
// as revCheck, plus an error if one occurred.
func certIsRevokedCRL(cert *x509.Certificate, url string) (revoked, ok bool, err error) {
	crl, ok := CRLSet[url]
	if ok && crl == nil {
		ok = false
		crlLock.Lock()
		delete(CRLSet, url)
		crlLock.Unlock()
	}

	var shouldFetchCRL = true
	if ok {
		if !crl.HasExpired(time.Now()) {
			shouldFetchCRL = false
		}
	}

	issuer := getIssuer(cert)

	if shouldFetchCRL {
		var err error
		crl, err = fetchCRL(url)
		if err != nil {
			log.Warningf("failed to fetch CRL: %v", err)
			return false, false, err
		}

		// check CRL signature
		if issuer != nil {
			err = issuer.CheckCRLSignature(crl)
			if err != nil {
				log.Warningf("failed to verify CRL: %v", err)
				return false, false, err
			}
		}

		crlLock.Lock()
		CRLSet[url] = crl
		crlLock.Unlock()
	}

	for _, revoked := range crl.TBSCertList.RevokedCertificates {
		if cert.SerialNumber.Cmp(revoked.SerialNumber) == 0 {
			log.Info("Serial number match: intermediate is revoked.")
			return true, true, err
		}
	}

	return false, true, err
}

// VerifyCertificate ensures that the certificate passed in hasn't
// expired and checks the CRL for the server.
func VerifyCertificate(cert *x509.Certificate) (revoked, ok bool) {
	revoked, ok, _ = VerifyCertificateError(cert)
	return revoked, ok
}

// VerifyCertificateError ensures that the certificate passed in hasn't
// expired and checks the CRL for the server.
func VerifyCertificateError(cert *x509.Certificate) (revoked, ok bool, err error) {
	if !time.Now().Before(cert.NotAfter) {
		msg := fmt.Sprintf("Certificate expired %s\n", cert.NotAfter)
		log.Info(msg)
		return true, true, fmt.Errorf(msg)
	} else if !time.Now().After(cert.NotBefore) {
		msg := fmt.Sprintf("Certificate isn't valid until %s\n", cert.NotBefore)
		log.Info(msg)
		return true, true, fmt.Errorf(msg)
	}
	return revCheck(cert)
}

func fetchRemote(url string) (*x509.Certificate, error) {
	resp, err := http.Get(url)
	if err != nil {
		return nil, err
	}

	in, err := remoteRead(resp.Body)
	if err != nil {
		return nil, err
	}
	resp.Body.Close()

	p, _ := pem.Decode(in)
	if p != nil {
		return helpers.ParseCertificatePEM(in)
	}

	return x509.ParseCertificate(in)
}

var ocspOpts = ocsp.RequestOptions{
	Hash: crypto.SHA1,
}

func certIsRevokedOCSP(leaf *x509.Certificate, strict bool) (revoked, ok bool, e error) {
	var err error

	ocspURLs := leaf.OCSPServer
	if len(ocspURLs) == 0 {
		// OCSP not enabled for this certificate.
		return false, true, nil
	}

	issuer := getIssuer(leaf)

	if issuer == nil {
		return false, false, nil
	}

	ocspRequest, err := ocsp.CreateRequest(leaf, issuer, &ocspOpts)
	if err != nil {
		return revoked, ok, err
	}

	for _, server := range ocspURLs {
		resp, err := sendOCSPRequest(server, ocspRequest, leaf, issuer)
		if err != nil {
			if strict {
				return revoked, ok, err
			}
			continue
		}

		// There wasn't an error fetching the OCSP status.
		ok = true

		if resp.Status != ocsp.Good {
			// The certificate was revoked.
			revoked = true
		}

		return revoked, ok, err
	}
	return revoked, ok, err
}

// sendOCSPRequest attempts to request an OCSP response from the
// server. The error only indicates a failure to *fetch* the
// certificate, and *does not* mean the certificate is valid.
func sendOCSPRequest(server string, req []byte, leaf, issuer *x509.Certificate) (*ocsp.Response, error) {
	var resp *http.Response
	var err error
	if len(req) > 256 {
		buf := bytes.NewBuffer(req)
		resp, err = http.Post(server, "application/ocsp-request", buf)
	} else {
		reqURL := server + "/" + neturl.QueryEscape(base64.StdEncoding.EncodeToString(req))
		resp, err = http.Get(reqURL)
	}

	if err != nil {
		return nil, err
	}

	if resp.StatusCode != http.StatusOK {
		return nil, errors.New("failed to retrieve OSCP")
	}

	body, err := ocspRead(resp.Body)
	if err != nil {
		return nil, err
	}
	resp.Body.Close()

	switch {
	case bytes.Equal(body, ocsp.UnauthorizedErrorResponse):
		return nil, errors.New("OSCP unauthorized")
	case bytes.Equal(body, ocsp.MalformedRequestErrorResponse):
		return nil, errors.New("OSCP malformed")
	case bytes.Equal(body, ocsp.InternalErrorErrorResponse):
		return nil, errors.New("OSCP internal error")
	case bytes.Equal(body, ocsp.TryLaterErrorResponse):
		return nil, errors.New("OSCP try later")
	case bytes.Equal(body, ocsp.SigRequredErrorResponse):
		return nil, errors.New("OSCP signature required")
	}

	return ocsp.ParseResponseForCert(body, leaf, issuer)
}

var crlRead = ioutil.ReadAll

// SetCRLFetcher sets the function to use to read from the http response body
func SetCRLFetcher(fn func(io.Reader) ([]byte, error)) {
	crlRead = fn
}

var remoteRead = ioutil.ReadAll

// SetRemoteFetcher sets the function to use to read from the http response body
func SetRemoteFetcher(fn func(io.Reader) ([]byte, error)) {
	remoteRead = fn
}

var ocspRead = ioutil.ReadAll

// SetOCSPFetcher sets the function to use to read from the http response body
func SetOCSPFetcher(fn func(io.Reader) ([]byte, error)) {
	ocspRead = fn
}
<commit_msg>Fix race condition in revoke<commit_after>// Package revoke provides functionality for checking the validity of
// a cert. Specifically, the temporal validity of the certificate is
// checked first, then any CRL and OCSP url in the cert is checked.
package revoke

import (
	"bytes"
	"crypto"
	"crypto/x509"
	"crypto/x509/pkix"
	"encoding/base64"
	"encoding/pem"
	"errors"
	"fmt"
	"io"
	"io/ioutil"
	"net/http"
	neturl "net/url"
	"sync"
	"time"

	"golang.org/x/crypto/ocsp"

	"github.com/cloudflare/cfssl/helpers"
	"github.com/cloudflare/cfssl/log"
)

// HardFail determines whether the failure to check the revocation
// status of a certificate (i.e. due to network failure) causes
// verification to fail (a hard failure).
var HardFail = false

// CRLSet associates a PKIX certificate list with the URL the CRL is
// fetched from.
var CRLSet = map[string]*pkix.CertificateList{}
var crlLock = new(sync.Mutex)

// We can't handle LDAP certificates, so this checks to see if the
// URL string points to an LDAP resource so that we can ignore it.
func ldapURL(url string) bool {
	u, err := neturl.Parse(url)
	if err != nil {
		log.Warningf("error parsing url %s: %v", url, err)
		return false
	}
	if u.Scheme == "ldap" {
		return true
	}
	return false
}

// revCheck should check the certificate for any revocations. It
// returns a pair of booleans: the first indicates whether the certificate
// is revoked, the second indicates whether the revocations were
// successfully checked.. This leads to the following combinations:
//
//  false, false: an error was encountered while checking revocations.
//
//  false, true:  the certificate was checked successfully and
//                  it is not revoked.
//
//  true, true:   the certificate was checked successfully and
//                  it is revoked.
//
//  true, false:  failure to check revocation status causes
//                  verification to fail
func revCheck(cert *x509.Certificate) (revoked, ok bool, err error) {
	for _, url := range cert.CRLDistributionPoints {
		if ldapURL(url) {
			log.Infof("skipping LDAP CRL: %s", url)
			continue
		}

		if revoked, ok, err := certIsRevokedCRL(cert, url); !ok {
			log.Warning("error checking revocation via CRL")
			if HardFail {
				return true, false, err
			}
			return false, false, err
		} else if revoked {
			log.Info("certificate is revoked via CRL")
			return true, true, err
		}
	}

	if revoked, ok, err := certIsRevokedOCSP(cert, HardFail); !ok {
		log.Warning("error checking revocation via OCSP")
		if HardFail {
			return true, false, err
		}
		return false, false, err
	} else if revoked {
		log.Info("certificate is revoked via OCSP")
		return true, true, err
	}

	return false, true, nil
}

// fetchCRL fetches and parses a CRL.
func fetchCRL(url string) (*pkix.CertificateList, error) {
	resp, err := http.Get(url)
	if err != nil {
		return nil, err
	} else if resp.StatusCode >= 300 {
		return nil, errors.New("failed to retrieve CRL")
	}

	body, err := crlRead(resp.Body)
	if err != nil {
		return nil, err
	}
	resp.Body.Close()

	return x509.ParseCRL(body)
}

func getIssuer(cert *x509.Certificate) *x509.Certificate {
	var issuer *x509.Certificate
	var err error
	for _, issuingCert := range cert.IssuingCertificateURL {
		issuer, err = fetchRemote(issuingCert)
		if err != nil {
			continue
		}
		break
	}

	return issuer

}

// check a cert against a specific CRL. Returns the same bool pair
// as revCheck, plus an error if one occurred.
func certIsRevokedCRL(cert *x509.Certificate, url string) (revoked, ok bool, err error) {
	crlLock.Lock()
	crl, ok := CRLSet[url]
	if ok && crl == nil {
		ok = false
		delete(CRLSet, url)
	}
	crlLock.Unlock()

	var shouldFetchCRL = true
	if ok {
		if !crl.HasExpired(time.Now()) {
			shouldFetchCRL = false
		}
	}

	issuer := getIssuer(cert)

	if shouldFetchCRL {
		var err error
		crl, err = fetchCRL(url)
		if err != nil {
			log.Warningf("failed to fetch CRL: %v", err)
			return false, false, err
		}

		// check CRL signature
		if issuer != nil {
			err = issuer.CheckCRLSignature(crl)
			if err != nil {
				log.Warningf("failed to verify CRL: %v", err)
				return false, false, err
			}
		}

		crlLock.Lock()
		CRLSet[url] = crl
		crlLock.Unlock()
	}

	for _, revoked := range crl.TBSCertList.RevokedCertificates {
		if cert.SerialNumber.Cmp(revoked.SerialNumber) == 0 {
			log.Info("Serial number match: intermediate is revoked.")
			return true, true, err
		}
	}

	return false, true, err
}

// VerifyCertificate ensures that the certificate passed in hasn't
// expired and checks the CRL for the server.
func VerifyCertificate(cert *x509.Certificate) (revoked, ok bool) {
	revoked, ok, _ = VerifyCertificateError(cert)
	return revoked, ok
}

// VerifyCertificateError ensures that the certificate passed in hasn't
// expired and checks the CRL for the server.
func VerifyCertificateError(cert *x509.Certificate) (revoked, ok bool, err error) {
	if !time.Now().Before(cert.NotAfter) {
		msg := fmt.Sprintf("Certificate expired %s\n", cert.NotAfter)
		log.Info(msg)
		return true, true, fmt.Errorf(msg)
	} else if !time.Now().After(cert.NotBefore) {
		msg := fmt.Sprintf("Certificate isn't valid until %s\n", cert.NotBefore)
		log.Info(msg)
		return true, true, fmt.Errorf(msg)
	}
	return revCheck(cert)
}

func fetchRemote(url string) (*x509.Certificate, error) {
	resp, err := http.Get(url)
	if err != nil {
		return nil, err
	}

	in, err := remoteRead(resp.Body)
	if err != nil {
		return nil, err
	}
	resp.Body.Close()

	p, _ := pem.Decode(in)
	if p != nil {
		return helpers.ParseCertificatePEM(in)
	}

	return x509.ParseCertificate(in)
}

var ocspOpts = ocsp.RequestOptions{
	Hash: crypto.SHA1,
}

func certIsRevokedOCSP(leaf *x509.Certificate, strict bool) (revoked, ok bool, e error) {
	var err error

	ocspURLs := leaf.OCSPServer
	if len(ocspURLs) == 0 {
		// OCSP not enabled for this certificate.
		return false, true, nil
	}

	issuer := getIssuer(leaf)

	if issuer == nil {
		return false, false, nil
	}

	ocspRequest, err := ocsp.CreateRequest(leaf, issuer, &ocspOpts)
	if err != nil {
		return revoked, ok, err
	}

	for _, server := range ocspURLs {
		resp, err := sendOCSPRequest(server, ocspRequest, leaf, issuer)
		if err != nil {
			if strict {
				return revoked, ok, err
			}
			continue
		}

		// There wasn't an error fetching the OCSP status.
		ok = true

		if resp.Status != ocsp.Good {
			// The certificate was revoked.
			revoked = true
		}

		return revoked, ok, err
	}
	return revoked, ok, err
}

// sendOCSPRequest attempts to request an OCSP response from the
// server. The error only indicates a failure to *fetch* the
// certificate, and *does not* mean the certificate is valid.
func sendOCSPRequest(server string, req []byte, leaf, issuer *x509.Certificate) (*ocsp.Response, error) {
	var resp *http.Response
	var err error
	if len(req) > 256 {
		buf := bytes.NewBuffer(req)
		resp, err = http.Post(server, "application/ocsp-request", buf)
	} else {
		reqURL := server + "/" + neturl.QueryEscape(base64.StdEncoding.EncodeToString(req))
		resp, err = http.Get(reqURL)
	}

	if err != nil {
		return nil, err
	}

	if resp.StatusCode != http.StatusOK {
		return nil, errors.New("failed to retrieve OSCP")
	}

	body, err := ocspRead(resp.Body)
	if err != nil {
		return nil, err
	}
	resp.Body.Close()

	switch {
	case bytes.Equal(body, ocsp.UnauthorizedErrorResponse):
		return nil, errors.New("OSCP unauthorized")
	case bytes.Equal(body, ocsp.MalformedRequestErrorResponse):
		return nil, errors.New("OSCP malformed")
	case bytes.Equal(body, ocsp.InternalErrorErrorResponse):
		return nil, errors.New("OSCP internal error")
	case bytes.Equal(body, ocsp.TryLaterErrorResponse):
		return nil, errors.New("OSCP try later")
	case bytes.Equal(body, ocsp.SigRequredErrorResponse):
		return nil, errors.New("OSCP signature required")
	}

	return ocsp.ParseResponseForCert(body, leaf, issuer)
}

var crlRead = ioutil.ReadAll

// SetCRLFetcher sets the function to use to read from the http response body
func SetCRLFetcher(fn func(io.Reader) ([]byte, error)) {
	crlRead = fn
}

var remoteRead = ioutil.ReadAll

// SetRemoteFetcher sets the function to use to read from the http response body
func SetRemoteFetcher(fn func(io.Reader) ([]byte, error)) {
	remoteRead = fn
}

var ocspRead = ioutil.ReadAll

// SetOCSPFetcher sets the function to use to read from the http response body
func SetOCSPFetcher(fn func(io.Reader) ([]byte, error)) {
	ocspRead = fn
}
<|endoftext|>