<commit_before>/*
Package nutz is a helper for interactions with bolt databases.
*/
package nutz

import (
	"fmt"
	"os"

	"github.com/boltdb/bolt"
)

// Storage contains metadata, which enable accessing the underlying bolt databases.
type Storage struct {
	// Data holds the value returned from the database.
	Data []byte

	// DataList holds key value pairs, retrievend by the GetAll method. These are the
	// values found in the bucket.
	DataList map[string][]byte

	// Error stores the last encountered error.
	Error error

	// DBName filename which will be used as the database.
	DBName string
	mode   os.FileMode
	db     *bolt.DB
	opts   *bolt.Options
}

// StorageFunc is the interface for a function which is used by Execute method
type StorageFunc func(s Storage, bucket, key string, value []byte, nested ...string) Storage

// NewStorage initializes a new storage object
func NewStorage(dbname string, mode os.FileMode, opts *bolt.Options) Storage {
	return Storage{
		DBName: dbname,
		mode:   mode,
		opts:   opts,
	}
}

// Create stores a given key value pairs . It takes an optional coma
// separated strings to act as nested buckets.
func (s Storage) Create(bucket, key string, value []byte, nested ...string) Storage {
	return s.execute(bucket, key, value, nested, create)
}

func create(s Storage, bucket, key string, value []byte, nested ...string) Storage {
	if len(nested) == 0 {
		s.Error = s.db.Update(func(tx *bolt.Tx) error {
			b, err := tx.CreateBucketIfNotExists([]byte(bucket))
			if err != nil {
				return err
			}
			return b.Put([]byte(key), value)
		})
	} else {
		s.Error = s.db.Update(func(tx *bolt.Tx) error {
			b, err := tx.CreateBucketIfNotExists([]byte(bucket))
			if err != nil {
				return err
			}
			prev, perr := createNestedBuckets(nested, b)
			if perr != nil {
				return perr
			}
			err = prev.Put([]byte(key), value)
			if err != nil {
				return err
			}

			rst := prev.Get([]byte(key))
			if rst != nil {
				s.Data = make([]byte, len((rst)))
				copy(s.Data, rst)
			}
			return nil
		})
	}

	if s.Error != nil {
		s.Data = nil // Make sure no previous data is returned
	}
	return s
}

// Get retrives a record from the database. The order of the optional nested buckets matter.
// if a key does not exist it returns an error
func (s Storage) Get(bucket, key string, nested ...string) Storage {
	return s.execute(bucket, key, nil, nested, getData)
}

func getData(s Storage, bucket, key string, value []byte, buckets ...string) Storage {
	if len(buckets) == 0 {
		s.Error = s.db.View(func(tx *bolt.Tx) error {
			b := tx.Bucket([]byte(bucket))
			if b == nil {
				return bolt.ErrBucketNotFound
			}
			res := b.Get([]byte(key))
			if res != nil {
				s.Data = make([]byte, len(res))
				copy(s.Data, res)
			}
			if res == nil {
				return notFound("key", key)
			}
			return nil
		})
	} else {
		s.Error = s.db.View(func(tx *bolt.Tx) error {
			b := tx.Bucket([]byte(bucket))
			if b == nil {
				return notFound("bucket", bucket)
			}
			prev, perr := getNestedBucket(buckets, b)
			if perr != nil {
				return perr
			}

			rst := prev.Get([]byte(key))
			if rst == nil {
				return bolt.ErrBucketNotFound
			}
			s.Data = make([]byte, len(rst))
			copy(s.Data, rst)
			return nil
		})
	}

	if s.Error != nil {
		s.Data = nil
	}
	return s
}

// Update replaces the old data stored in a given key with a new one. If the Key is not found
// it returns an error. Update does not attempt to create missing buckets, instead it returns an error
//
// This shares the same api as the Create method,
// except it returns an error if the record does not exist,
// and in case of buckets it returns a error if they dont exist.
//
// For a nested bucket list, and missing bucket in the list,
// or missarrangement result in an error.
func (s Storage) Update(bucket, key string, value []byte, nested ...string) Storage {
	return s.execute(bucket, key, value, nested, update)
}

func update(s Storage, bucket, key string, value []byte, nested ...string) Storage {
	if len(nested) == 0 {
		s.Error = s.db.Update(func(tx *bolt.Tx) error {
			b := tx.Bucket([]byte(bucket))
			if b == nil {
				return notFound("bucket", bucket)
			}
			gkey := b.Get([]byte(key))
			if gkey == nil {
				return notFound("key", key)
			}
			return b.Put([]byte(key), value)
		})
		s.Data = value
	} else {
		s.Error = s.db.Update(func(tx *bolt.Tx) error {
			b := tx.Bucket([]byte(bucket))
			if b == nil {
				return notFound("nucket", bucket)
			}
			prev, perr := getNestedBucket(nested, b)
			if perr != nil {
				return perr
			}
			return prev.Put([]byte(key), value)
		})
		s.Data = value
	}
	if s.Error != nil {
		s.Data = nil
	}
	return s
}

// GetAll retrieves all key value pairs in a bucket, it stores the map of key value pairs
// inside the DataList attribute.
func (s Storage) GetAll(bucket string, nested ...string) Storage {
	return s.execute(bucket, "", nil, nested, getAll)
}

func getAll(s Storage, bucket, key string, value []byte, nested ...string) Storage {
	s.DataList = make(map[string][]byte)
	if len(nested) == 0 {
		s.Error = s.db.View(func(tx *bolt.Tx) error {
			b := tx.Bucket([]byte(bucket))
			if b == nil {
				return bolt.ErrBucketNotFound
			}
			return b.ForEach(func(k, v []byte) error {
				dv := make([]byte, len(v))
				copy(dv, v)
				s.DataList[string(k)] = dv
				return nil
			})
		})
	} else {
		s.Error = s.db.View(func(tx *bolt.Tx) error {
			b := tx.Bucket([]byte(bucket))
			if b == nil {
				return notFound("bucket", bucket)
			}
			prev, perr := getNestedBucket(nested, b)
			if perr != nil {
				return perr
			}
			return prev.ForEach(func(k, v []byte) error {
				dv := make([]byte, len(v))
				copy(dv, v)
				s.DataList[string(k)] = dv
				return nil
			})

		})
	}

	if s.Error != nil {
		s.Data = nil // Make sure no previous data is reurned
	}
	return s
}

// Delete removes a record from the database
func (s Storage) Delete(bucket, key string, nested ...string) Storage {
	return s.execute(bucket, key, nil, nested, remove)
}

func remove(s Storage, bucket, key string, value []byte, nested ...string) Storage {
	if len(nested) == 0 {
		s.Error = s.db.Update(func(tx *bolt.Tx) error {
			b := tx.Bucket([]byte(bucket))
			if b == nil {
				return notFound("bucket", bucket)
			}
			return b.Delete([]byte(key))
		})
		s.Data = []byte(key)
	} else {
		s.Error = s.db.Update(func(tx *bolt.Tx) error {
			var prev *bolt.Bucket
			b := tx.Bucket([]byte(bucket))
			if b == nil {
				return notFound("bucket", bucket)
			}
			prev, perr := getNestedBucket(nested, b)
			if perr != nil {
				return perr
			}

			return prev.Delete([]byte(key))
		})
		s.Data = []byte(key)
	}

	if s.Error != nil {
		s.Data = nil // Make sure no previous data is reurned
	}
	return s
}

// DeleteDatabase removes the given database file as specified in the NewStorage function
// from the filesystem
func (s Storage) DeleteDatabase() error {
	return os.Remove(s.DBName)
}

// Execute opens a database, and pass all arguments together with a Storage object
// that has an open bolt database to a given StorageFunc. This is an ideal way to tap into
// the *bolt.DB object and do the whole stuffs you would like do do, without even bothering
// if the database exist of if the connection was closed.
//
// It makes sure the database is closed after the function exits. Note that, the nested
// buckets should be provided as a slice of strings.
func (s *Storage) Execute(bucket, key string, value []byte, nested []string, fn StorageFunc) Storage {
	return s.execute(bucket, key, value, nested, fn)
}

func (s Storage) execute(bucket, key string, value []byte, nested []string, fn StorageFunc) Storage {
	s.db, s.Error = bolt.Open(s.DBName, s.mode, s.opts)
	if s.Error != nil {
		return s
	}
	defer s.db.Close()
	return fn(s, bucket, key, value, nested...)
}

func notFound(kind, msg string) error {
	return fmt.Errorf("nutz: %s  %s  not found", kind, msg)
}

func getNestedBucket(n []string, b *bolt.Bucket) (*bolt.Bucket, error) {
	var prev *bolt.Bucket
	var uerr error
	prev = b
	for i := 0; i < len(n); i++ {
		curr := prev.Bucket([]byte(n[i]))
		if curr == nil {
			uerr = notFound("bucket", n[i])
			break
		}
		prev = curr
	}
	return prev, uerr
}

func createNestedBuckets(n []string, b *bolt.Bucket) (*bolt.Bucket, error) {
	var (
		prev, curr *bolt.Bucket
		err        error
	)
	prev = b
	for i := 0; i < len(n); i++ {
		if i == len(n)-1 {
			curr, err = prev.CreateBucket([]byte(n[i]))
		} else {
			curr, err = prev.CreateBucketIfNotExists([]byte(n[i]))
		}
		if err != nil {
			break
		}
		prev = curr
	}
	return prev, err
}
<commit_msg>Fixed nested bucket multiple keys<commit_after>/*
Package nutz is a helper for interactions with bolt databases.
*/
package nutz

import (
	"fmt"
	"os"

	"github.com/boltdb/bolt"
)

// Storage contains metadata, which enable accessing the underlying bolt databases.
type Storage struct {
	// Data holds the value returned from the database.
	Data []byte

	// DataList holds key value pairs, retrievend by the GetAll method. These are the
	// values found in the bucket.
	DataList map[string][]byte

	// Error stores the last encountered error.
	Error error

	// DBName filename which will be used as the database.
	DBName string
	mode   os.FileMode
	db     *bolt.DB
	opts   *bolt.Options
}

// StorageFunc is the interface for a function which is used by Execute method
type StorageFunc func(s Storage, bucket, key string, value []byte, nested ...string) Storage

// NewStorage initializes a new storage object
func NewStorage(dbname string, mode os.FileMode, opts *bolt.Options) Storage {
	return Storage{
		DBName: dbname,
		mode:   mode,
		opts:   opts,
	}
}

// Create stores a given key value pairs . It takes an optional coma
// separated strings to act as nested buckets.
func (s Storage) Create(bucket, key string, value []byte, nested ...string) Storage {
	return s.execute(bucket, key, value, nested, create)
}

func create(s Storage, bucket, key string, value []byte, nested ...string) Storage {
	if len(nested) == 0 {
		s.Error = s.db.Update(func(tx *bolt.Tx) error {
			b, err := tx.CreateBucketIfNotExists([]byte(bucket))
			if err != nil {
				return err
			}
			return b.Put([]byte(key), value)
		})
	} else {
		s.Error = s.db.Update(func(tx *bolt.Tx) error {
			b, err := tx.CreateBucketIfNotExists([]byte(bucket))
			if err != nil {
				return err
			}
			prev, perr := createNestedBuckets(nested, b)
			if perr != nil {
				return perr
			}
			err = prev.Put([]byte(key), value)
			if err != nil {
				return err
			}

			rst := prev.Get([]byte(key))
			if rst != nil {
				s.Data = make([]byte, len((rst)))
				copy(s.Data, rst)
			}
			return nil
		})
	}

	if s.Error != nil {
		s.Data = nil // Make sure no previous data is returned
	}
	return s
}

// Get retrives a record from the database. The order of the optional nested buckets matter.
// if a key does not exist it returns an error
func (s Storage) Get(bucket, key string, nested ...string) Storage {
	return s.execute(bucket, key, nil, nested, getData)
}

func getData(s Storage, bucket, key string, value []byte, buckets ...string) Storage {
	if len(buckets) == 0 {
		s.Error = s.db.View(func(tx *bolt.Tx) error {
			b := tx.Bucket([]byte(bucket))
			if b == nil {
				return bolt.ErrBucketNotFound
			}
			res := b.Get([]byte(key))
			if res != nil {
				s.Data = make([]byte, len(res))
				copy(s.Data, res)
			}
			if res == nil {
				return notFound("key", key)
			}
			return nil
		})
	} else {
		s.Error = s.db.View(func(tx *bolt.Tx) error {
			b := tx.Bucket([]byte(bucket))
			if b == nil {
				return notFound("bucket", bucket)
			}
			prev, perr := getNestedBucket(buckets, b)
			if perr != nil {
				return perr
			}

			rst := prev.Get([]byte(key))
			if rst == nil {
				return bolt.ErrBucketNotFound
			}
			s.Data = make([]byte, len(rst))
			copy(s.Data, rst)
			return nil
		})
	}

	if s.Error != nil {
		s.Data = nil
	}
	return s
}

// Update replaces the old data stored in a given key with a new one. If the Key is not found
// it returns an error. Update does not attempt to create missing buckets, instead it returns an error
//
// This shares the same api as the Create method,
// except it returns an error if the record does not exist,
// and in case of buckets it returns a error if they dont exist.
//
// For a nested bucket list, and missing bucket in the list,
// or missarrangement result in an error.
func (s Storage) Update(bucket, key string, value []byte, nested ...string) Storage {
	return s.execute(bucket, key, value, nested, update)
}

func update(s Storage, bucket, key string, value []byte, nested ...string) Storage {
	if len(nested) == 0 {
		s.Error = s.db.Update(func(tx *bolt.Tx) error {
			b := tx.Bucket([]byte(bucket))
			if b == nil {
				return notFound("bucket", bucket)
			}
			gkey := b.Get([]byte(key))
			if gkey == nil {
				return notFound("key", key)
			}
			return b.Put([]byte(key), value)
		})
		s.Data = value
	} else {
		s.Error = s.db.Update(func(tx *bolt.Tx) error {
			b := tx.Bucket([]byte(bucket))
			if b == nil {
				return notFound("nucket", bucket)
			}
			prev, perr := getNestedBucket(nested, b)
			if perr != nil {
				return perr
			}
			return prev.Put([]byte(key), value)
		})
		s.Data = value
	}
	if s.Error != nil {
		s.Data = nil
	}
	return s
}

// GetAll retrieves all key value pairs in a bucket, it stores the map of key value pairs
// inside the DataList attribute.
func (s Storage) GetAll(bucket string, nested ...string) Storage {
	return s.execute(bucket, "", nil, nested, getAll)
}

func getAll(s Storage, bucket, key string, value []byte, nested ...string) Storage {
	s.DataList = make(map[string][]byte)
	if len(nested) == 0 {
		s.Error = s.db.View(func(tx *bolt.Tx) error {
			b := tx.Bucket([]byte(bucket))
			if b == nil {
				return bolt.ErrBucketNotFound
			}
			return b.ForEach(func(k, v []byte) error {
				dv := make([]byte, len(v))
				copy(dv, v)
				s.DataList[string(k)] = dv
				return nil
			})
		})
	} else {
		s.Error = s.db.View(func(tx *bolt.Tx) error {
			b := tx.Bucket([]byte(bucket))
			if b == nil {
				return notFound("bucket", bucket)
			}
			prev, perr := getNestedBucket(nested, b)
			if perr != nil {
				return perr
			}
			return prev.ForEach(func(k, v []byte) error {
				dv := make([]byte, len(v))
				copy(dv, v)
				s.DataList[string(k)] = dv
				return nil
			})

		})
	}

	if s.Error != nil {
		s.Data = nil // Make sure no previous data is reurned
	}
	return s
}

// Delete removes a record from the database
func (s Storage) Delete(bucket, key string, nested ...string) Storage {
	return s.execute(bucket, key, nil, nested, remove)
}

func remove(s Storage, bucket, key string, value []byte, nested ...string) Storage {
	if len(nested) == 0 {
		s.Error = s.db.Update(func(tx *bolt.Tx) error {
			b := tx.Bucket([]byte(bucket))
			if b == nil {
				return notFound("bucket", bucket)
			}
			return b.Delete([]byte(key))
		})
		s.Data = []byte(key)
	} else {
		s.Error = s.db.Update(func(tx *bolt.Tx) error {
			var prev *bolt.Bucket
			b := tx.Bucket([]byte(bucket))
			if b == nil {
				return notFound("bucket", bucket)
			}
			prev, perr := getNestedBucket(nested, b)
			if perr != nil {
				return perr
			}

			return prev.Delete([]byte(key))
		})
		s.Data = []byte(key)
	}

	if s.Error != nil {
		s.Data = nil // Make sure no previous data is reurned
	}
	return s
}

// DeleteDatabase removes the given database file as specified in the NewStorage function
// from the filesystem
func (s Storage) DeleteDatabase() error {
	return os.Remove(s.DBName)
}

// Execute opens a database, and pass all arguments together with a Storage object
// that has an open bolt database to a given StorageFunc. This is an ideal way to tap into
// the *bolt.DB object and do the whole stuffs you would like do do, without even bothering
// if the database exist of if the connection was closed.
//
// It makes sure the database is closed after the function exits. Note that, the nested
// buckets should be provided as a slice of strings.
func (s *Storage) Execute(bucket, key string, value []byte, nested []string, fn StorageFunc) Storage {
	return s.execute(bucket, key, value, nested, fn)
}

func (s Storage) execute(bucket, key string, value []byte, nested []string, fn StorageFunc) Storage {
	s.db, s.Error = bolt.Open(s.DBName, s.mode, s.opts)
	if s.Error != nil {
		return s
	}
	defer s.db.Close()
	return fn(s, bucket, key, value, nested...)
}

func notFound(kind, msg string) error {
	return fmt.Errorf("nutz: %s  %s  not found", kind, msg)
}

func getNestedBucket(n []string, b *bolt.Bucket) (*bolt.Bucket, error) {
	var prev *bolt.Bucket
	var uerr error
	prev = b
	for i := 0; i < len(n); i++ {
		curr := prev.Bucket([]byte(n[i]))
		if curr == nil {
			uerr = notFound("bucket", n[i])
			break
		}
		prev = curr
	}
	return prev, uerr
}

func createNestedBuckets(n []string, b *bolt.Bucket) (*bolt.Bucket, error) {
	var (
		prev, curr *bolt.Bucket
		err        error
	)
	prev = b
	for i := 0; i < len(n); i++ {
		if i == len(n)-1 {
			curr, err = prev.CreateBucketIfNotExists([]byte(n[i]))
		} else {
			curr, err = prev.CreateBucketIfNotExists([]byte(n[i]))
		}
		if err != nil {
			break
		}
		prev = curr
	}
	return prev, err
}
<|endoftext|>


<commit_before>// Package testutil provides initalization and utility routines for unit tests.
//
// All tests should import it, even if only for its initialization:
//   import _ "veyron/lib/testutil"
//
package testutil

import (
	"flag"
	"math/rand"
	"os"
	"runtime"
	"strconv"
	// Need to import all of the packages that could possibly
	// define flags that we care about. In practice, this is the
	// flags defined by the testing package, the logging library
	// and any flags defined by the blackbox package below.
	_ "testing"
	"time"

	// Import blackbox to ensure that it gets to define its flags.
	_ "veyron/lib/testutil/blackbox"

	"veyron2/vlog"
)

const (
	SeedEnv = "VEYRON_RNG_SEED"
)

var (
	Rand *rand.Rand
)

func init() {
	if os.Getenv("GOMAXPROCS") == "" {
		// Set the number of logical processors to the number of CPUs,
		// if GOMAXPROCS is not set in the environment.
		runtime.GOMAXPROCS(runtime.NumCPU())
	}
	// At this point all of the flags that we're going to use for
	// tests must be defined.
	flag.Parse()
	vlog.ConfigureLibraryLoggerFromFlags()
	// Initialize pseudo-random number generator.
	seed := time.Now().UnixNano()
	seedString := os.Getenv(SeedEnv)
	if seedString != "" {
		var err error
		base, bitSize := 0, 64
		seed, err = strconv.ParseInt(seedString, base, bitSize)
		if err != nil {
			vlog.Fatalf("ParseInt(%v, %v, %v) failed: %v", seedString, base, bitSize, err)
		}
	}
	vlog.Infof("Seeding pseudo-random number generator with %v", seed)
	Rand = rand.New(rand.NewSource(seed))
}
<commit_msg>veyron/lib/testutil: Make the source of randomness thread-safe.<commit_after>// Package testutil provides initalization and utility routines for unit tests.
//
// All tests should import it, even if only for its initialization:
//   import _ "veyron/lib/testutil"
//
package testutil

import (
	"flag"
	"math/rand"
	"os"
	"runtime"
	"strconv"
	"sync"
	// Need to import all of the packages that could possibly
	// define flags that we care about. In practice, this is the
	// flags defined by the testing package, the logging library
	// and any flags defined by the blackbox package below.
	_ "testing"
	"time"

	// Import blackbox to ensure that it gets to define its flags.
	_ "veyron/lib/testutil/blackbox"

	"veyron2/vlog"
)

const (
	SeedEnv = "VEYRON_RNG_SEED"
)

// Random is a concurrent-access friendly source of randomness.
type Random struct {
	mu   sync.Mutex
	rand *rand.Rand
}

// Int returns a non-negative pseudo-random int.
func (r *Random) Int() int {
	r.mu.Lock()
	defer r.mu.Unlock()
	return r.rand.Int()
}

// Intn returns a non-negative pseudo-random int in the range [0, n).
func (r *Random) Intn(n int) int {
	r.mu.Lock()
	defer r.mu.Unlock()
	return r.rand.Intn(n)
}

// Int63 returns a non-negative 63-bit pseudo-random integer as an int64.
func (r *Random) Int63() int64 {
	r.mu.Lock()
	defer r.mu.Unlock()
	return r.rand.Int63()
}

var (
	Rand *Random
)

func init() {
	if os.Getenv("GOMAXPROCS") == "" {
		// Set the number of logical processors to the number of CPUs,
		// if GOMAXPROCS is not set in the environment.
		runtime.GOMAXPROCS(runtime.NumCPU())
	}
	// At this point all of the flags that we're going to use for
	// tests must be defined.
	flag.Parse()
	vlog.ConfigureLibraryLoggerFromFlags()
	// Initialize pseudo-random number generator.
	seed := time.Now().UnixNano()
	seedString := os.Getenv(SeedEnv)
	if seedString != "" {
		var err error
		base, bitSize := 0, 64
		seed, err = strconv.ParseInt(seedString, base, bitSize)
		if err != nil {
			vlog.Fatalf("ParseInt(%v, %v, %v) failed: %v", seedString, base, bitSize, err)
		}
	}
	vlog.Infof("Seeding pseudo-random number generator with %v", seed)
	Rand = &Random{rand: rand.New(rand.NewSource(seed))}
}
<|endoftext|>


<commit_before>package scheduler

import (
	"encoding/json"
	"fmt"
	"sync"
	"time"

	"github.com/docker/distribution/context"
	"github.com/docker/distribution/registry/storage/driver"
)

// onTTLExpiryFunc is called when a repository's TTL expires
type expiryFunc func(string) error

const (
	entryTypeBlob = iota
	entryTypeManifest
)

// schedulerEntry represents an entry in the scheduler
// fields are exported for serialization
type schedulerEntry struct {
	Key       string    `json:"Key"`
	Expiry    time.Time `json:"ExpiryData"`
	EntryType int       `json:"EntryType"`

	timer *time.Timer
}

// New returns a new instance of the scheduler
func New(ctx context.Context, driver driver.StorageDriver, path string) *TTLExpirationScheduler {
	return &TTLExpirationScheduler{
		entries:         make(map[string]*schedulerEntry),
		driver:          driver,
		pathToStateFile: path,
		ctx:             ctx,
		stopped:         true,
	}
}

// TTLExpirationScheduler is a scheduler used to perform actions
// when TTLs expire
type TTLExpirationScheduler struct {
	sync.Mutex

	entries map[string]*schedulerEntry

	driver          driver.StorageDriver
	ctx             context.Context
	pathToStateFile string

	stopped bool

	onBlobExpire     expiryFunc
	onManifestExpire expiryFunc
}

// OnBlobExpire is called when a scheduled blob's TTL expires
func (ttles *TTLExpirationScheduler) OnBlobExpire(f expiryFunc) {
	ttles.Lock()
	defer ttles.Unlock()

	ttles.onBlobExpire = f
}

// OnManifestExpire is called when a scheduled manifest's TTL expires
func (ttles *TTLExpirationScheduler) OnManifestExpire(f expiryFunc) {
	ttles.Lock()
	defer ttles.Unlock()

	ttles.onManifestExpire = f
}

// AddBlob schedules a blob cleanup after ttl expires
func (ttles *TTLExpirationScheduler) AddBlob(dgst string, ttl time.Duration) error {
	ttles.Lock()
	defer ttles.Unlock()

	if ttles.stopped {
		return fmt.Errorf("scheduler not started")
	}
	ttles.add(dgst, ttl, entryTypeBlob)
	return nil
}

// AddManifest schedules a manifest cleanup after ttl expires
func (ttles *TTLExpirationScheduler) AddManifest(repoName string, ttl time.Duration) error {
	ttles.Lock()
	defer ttles.Unlock()

	if ttles.stopped {
		return fmt.Errorf("scheduler not started")
	}

	ttles.add(repoName, ttl, entryTypeManifest)
	return nil
}

// Start starts the scheduler
func (ttles *TTLExpirationScheduler) Start() error {
	ttles.Lock()
	defer ttles.Unlock()

	err := ttles.readState()
	if err != nil {
		return err
	}

	if !ttles.stopped {
		return fmt.Errorf("Scheduler already started")
	}

	context.GetLogger(ttles.ctx).Infof("Starting cached object TTL expiration scheduler...")
	ttles.stopped = false

	// Start timer for each deserialized entry
	for _, entry := range ttles.entries {
		entry.timer = ttles.startTimer(entry, entry.Expiry.Sub(time.Now()))
	}

	return nil
}

func (ttles *TTLExpirationScheduler) add(key string, ttl time.Duration, eType int) {
	entry := &schedulerEntry{
		Key:       key,
		Expiry:    time.Now().Add(ttl),
		EntryType: eType,
	}
	context.GetLogger(ttles.ctx).Infof("Adding new scheduler entry for %s with ttl=%s", entry.Key, entry.Expiry.Sub(time.Now()))
	if oldEntry, present := ttles.entries[key]; present && oldEntry.timer != nil {
		oldEntry.timer.Stop()
	}
	ttles.entries[key] = entry
	entry.timer = ttles.startTimer(entry, ttl)

	if err := ttles.writeState(); err != nil {
		context.GetLogger(ttles.ctx).Errorf("Error writing scheduler state: %s", err)
	}
}

func (ttles *TTLExpirationScheduler) startTimer(entry *schedulerEntry, ttl time.Duration) *time.Timer {
	return time.AfterFunc(ttl, func() {
		ttles.Lock()
		defer ttles.Unlock()

		var f expiryFunc

		switch entry.EntryType {
		case entryTypeBlob:
			f = ttles.onBlobExpire
		case entryTypeManifest:
			f = ttles.onManifestExpire
		default:
			f = func(repoName string) error {
				return fmt.Errorf("Unexpected scheduler entry type")
			}
		}

		if err := f(entry.Key); err != nil {
			context.GetLogger(ttles.ctx).Errorf("Scheduler error returned from OnExpire(%s): %s", entry.Key, err)
		}

		delete(ttles.entries, entry.Key)
		if err := ttles.writeState(); err != nil {
			context.GetLogger(ttles.ctx).Errorf("Error writing scheduler state: %s", err)
		}
	})
}

// Stop stops the scheduler.
func (ttles *TTLExpirationScheduler) Stop() {
	ttles.Lock()
	defer ttles.Unlock()

	if err := ttles.writeState(); err != nil {
		context.GetLogger(ttles.ctx).Errorf("Error writing scheduler state: %s", err)
	}

	for _, entry := range ttles.entries {
		entry.timer.Stop()
	}
	ttles.stopped = true
}

func (ttles *TTLExpirationScheduler) writeState() error {
	jsonBytes, err := json.Marshal(ttles.entries)
	if err != nil {
		return err
	}

	err = ttles.driver.PutContent(ttles.ctx, ttles.pathToStateFile, jsonBytes)
	if err != nil {
		return err
	}
	return nil
}

func (ttles *TTLExpirationScheduler) readState() error {
	if _, err := ttles.driver.Stat(ttles.ctx, ttles.pathToStateFile); err != nil {
		switch err := err.(type) {
		case driver.PathNotFoundError:
			return nil
		default:
			return err
		}
	}

	bytes, err := ttles.driver.GetContent(ttles.ctx, ttles.pathToStateFile)
	if err != nil {
		return err
	}

	err = json.Unmarshal(bytes, &ttles.entries)
	if err != nil {
		return err
	}

	return nil
}
<commit_msg>Buffer writing the scheduler entry state to disk by periodically checking for changes to the entries index and saving it to the filesystem.<commit_after>package scheduler

import (
	"encoding/json"
	"fmt"
	"sync"
	"time"

	"github.com/docker/distribution/context"
	"github.com/docker/distribution/registry/storage/driver"
)

// onTTLExpiryFunc is called when a repository's TTL expires
type expiryFunc func(string) error

const (
	entryTypeBlob = iota
	entryTypeManifest
	indexSaveFrequency = 5 * time.Second
)

// schedulerEntry represents an entry in the scheduler
// fields are exported for serialization
type schedulerEntry struct {
	Key       string    `json:"Key"`
	Expiry    time.Time `json:"ExpiryData"`
	EntryType int       `json:"EntryType"`

	timer *time.Timer
}

// New returns a new instance of the scheduler
func New(ctx context.Context, driver driver.StorageDriver, path string) *TTLExpirationScheduler {
	return &TTLExpirationScheduler{
		entries:         make(map[string]*schedulerEntry),
		driver:          driver,
		pathToStateFile: path,
		ctx:             ctx,
		stopped:         true,
		doneChan:        make(chan struct{}),
		saveTimer:       time.NewTicker(indexSaveFrequency),
	}
}

// TTLExpirationScheduler is a scheduler used to perform actions
// when TTLs expire
type TTLExpirationScheduler struct {
	sync.Mutex

	entries map[string]*schedulerEntry

	driver          driver.StorageDriver
	ctx             context.Context
	pathToStateFile string

	stopped bool

	onBlobExpire     expiryFunc
	onManifestExpire expiryFunc

	indexDirty bool
	saveTimer  *time.Ticker
	doneChan   chan struct{}
}

// OnBlobExpire is called when a scheduled blob's TTL expires
func (ttles *TTLExpirationScheduler) OnBlobExpire(f expiryFunc) {
	ttles.Lock()
	defer ttles.Unlock()

	ttles.onBlobExpire = f
}

// OnManifestExpire is called when a scheduled manifest's TTL expires
func (ttles *TTLExpirationScheduler) OnManifestExpire(f expiryFunc) {
	ttles.Lock()
	defer ttles.Unlock()

	ttles.onManifestExpire = f
}

// AddBlob schedules a blob cleanup after ttl expires
func (ttles *TTLExpirationScheduler) AddBlob(dgst string, ttl time.Duration) error {
	ttles.Lock()
	defer ttles.Unlock()

	if ttles.stopped {
		return fmt.Errorf("scheduler not started")
	}
	ttles.add(dgst, ttl, entryTypeBlob)
	return nil
}

// AddManifest schedules a manifest cleanup after ttl expires
func (ttles *TTLExpirationScheduler) AddManifest(repoName string, ttl time.Duration) error {
	ttles.Lock()
	defer ttles.Unlock()

	if ttles.stopped {
		return fmt.Errorf("scheduler not started")
	}

	ttles.add(repoName, ttl, entryTypeManifest)
	return nil
}

// Start starts the scheduler
func (ttles *TTLExpirationScheduler) Start() error {
	ttles.Lock()
	defer ttles.Unlock()

	err := ttles.readState()
	if err != nil {
		return err
	}

	if !ttles.stopped {
		return fmt.Errorf("Scheduler already started")
	}

	context.GetLogger(ttles.ctx).Infof("Starting cached object TTL expiration scheduler...")
	ttles.stopped = false

	// Start timer for each deserialized entry
	for _, entry := range ttles.entries {
		entry.timer = ttles.startTimer(entry, entry.Expiry.Sub(time.Now()))
	}

	// Start a ticker to periodically save the entries index

	go func() {
		for {
			select {
			case <-ttles.saveTimer.C:
				if !ttles.indexDirty {
					continue
				}

				ttles.Lock()
				err := ttles.writeState()
				if err != nil {
					context.GetLogger(ttles.ctx).Errorf("Error writing scheduler state: %s", err)
				} else {
					ttles.indexDirty = false
				}
				ttles.Unlock()

			case <-ttles.doneChan:
				return
			}
		}
	}()

	return nil
}

func (ttles *TTLExpirationScheduler) add(key string, ttl time.Duration, eType int) {
	entry := &schedulerEntry{
		Key:       key,
		Expiry:    time.Now().Add(ttl),
		EntryType: eType,
	}
	context.GetLogger(ttles.ctx).Infof("Adding new scheduler entry for %s with ttl=%s", entry.Key, entry.Expiry.Sub(time.Now()))
	if oldEntry, present := ttles.entries[key]; present && oldEntry.timer != nil {
		oldEntry.timer.Stop()
	}
	ttles.entries[key] = entry
	entry.timer = ttles.startTimer(entry, ttl)
	ttles.indexDirty = true
}

func (ttles *TTLExpirationScheduler) startTimer(entry *schedulerEntry, ttl time.Duration) *time.Timer {
	return time.AfterFunc(ttl, func() {
		ttles.Lock()
		defer ttles.Unlock()

		var f expiryFunc

		switch entry.EntryType {
		case entryTypeBlob:
			f = ttles.onBlobExpire
		case entryTypeManifest:
			f = ttles.onManifestExpire
		default:
			f = func(repoName string) error {
				return fmt.Errorf("Unexpected scheduler entry type")
			}
		}

		if err := f(entry.Key); err != nil {
			context.GetLogger(ttles.ctx).Errorf("Scheduler error returned from OnExpire(%s): %s", entry.Key, err)
		}

		delete(ttles.entries, entry.Key)
		ttles.indexDirty = true
	})
}

// Stop stops the scheduler.
func (ttles *TTLExpirationScheduler) Stop() {
	ttles.Lock()
	defer ttles.Unlock()

	if err := ttles.writeState(); err != nil {
		context.GetLogger(ttles.ctx).Errorf("Error writing scheduler state: %s", err)
	}

	for _, entry := range ttles.entries {
		entry.timer.Stop()
	}

	close(ttles.doneChan)
	ttles.saveTimer.Stop()
	ttles.stopped = true
}

func (ttles *TTLExpirationScheduler) writeState() error {
	jsonBytes, err := json.Marshal(ttles.entries)
	if err != nil {
		return err
	}

	err = ttles.driver.PutContent(ttles.ctx, ttles.pathToStateFile, jsonBytes)
	if err != nil {
		return err
	}

	return nil
}

func (ttles *TTLExpirationScheduler) readState() error {
	if _, err := ttles.driver.Stat(ttles.ctx, ttles.pathToStateFile); err != nil {
		switch err := err.(type) {
		case driver.PathNotFoundError:
			return nil
		default:
			return err
		}
	}

	bytes, err := ttles.driver.GetContent(ttles.ctx, ttles.pathToStateFile)
	if err != nil {
		return err
	}

	err = json.Unmarshal(bytes, &ttles.entries)
	if err != nil {
		return err
	}

	return nil
}
<|endoftext|>


<commit_before>// This Source Code Form is subject to the terms of the Mozilla Public
// License, v. 2.0. If a copy of the MPL was not distributed with this
// file, You can obtain one at http://mozilla.org/MPL/2.0/.

package runtime

// TODO: dynamic, setf dynamic, set-dynamic, dynamic-let
<commit_msg>Added DynamicVariable operations<commit_after>// This Source Code Form is subject to the terms of the Mozilla Public
// License, v. 2.0. If a copy of the MPL was not distributed with this
// file, You can obtain one at http://mozilla.org/MPL/2.0/.

package runtime

import (
	"github.com/ta2gch/iris/runtime/environment"
	"github.com/ta2gch/iris/runtime/ilos"
	"github.com/ta2gch/iris/runtime/ilos/class"
	"github.com/ta2gch/iris/runtime/ilos/instance"
)

// TODO: setf dynamic

// Dynamic denotes a reference to the identifier denoting a
// dynamic variable. This special form is not allowed in the scope of a
// definition of var which is not done by defdynamic or dynamic-let.
//
// During activation, the current dynamic binding of the variable var is
// returned that was established most recently and is still in effect. An
// error shall be signaled if such a binding does not exist
// (error-id. unbound-variable).
func Dynamic(local, global *environment.Environment, var1 ilos.Instance) (ilos.Instance, ilos.Instance) {
	if err := ensure(class.Symbol, var1); err != nil {
		return nil, err
	}
	if v, ok := local.DynamicVariable.Get(var1); ok {
		return v, nil
	}
	if v, ok := global.DynamicVariable.Get(var1); ok {
		return v, nil
	}
	return nil, instance.New(class.UndefinedVariable, map[string]ilos.Instance{
		"NAME":      var1,
		"NAMESPACE": instance.New(class.Symbol, "Variable"),
	})
}

// SetDynamic denotes an assignment to a dynamic variable. This
// form can appear anywhere that (dynamic var) can appear.
//
// form is evaluated and the result of the evaluation is used to change
// the dynamic binding of var.
//
// An error shall be signaled if var has no dynamic value
// (error-id.  unbound-variable). setf of dynamic can be used only for
// modifying bindings, and not for establishing them.
func SetDynamic(local, global *environment.Environment, form, var1 ilos.Instance) (ilos.Instance, ilos.Instance) {
	if err := ensure(class.Symbol, var1); err != nil {
		return nil, err
	}
	form, err := Eval(local, global, form)
	if err != nil {
		return nil, form
	}
	if local.DynamicVariable.Set(var1, form) {
		return form, nil
	}
	if global.DynamicVariable.Set(var1, form) {
		return form, nil
	}
	return nil, instance.New(class.UndefinedFunction, map[string]ilos.Instance{
		"NAME":      var1,
		"NAMESPACE": instance.New(class.Symbol, "FUNCTION"),
	})
}

// DyamicLet is used to establish dynamic variable bindings.
// The first subform (the dynamic-let variable list) is a list of pairs (var
// form). The scope of an identifier var defined by dynamic-let is the current
// toplevel scope. The extent of the bindings of each var is the extent of the
// body of the dynamic-let. The dynamic-let special form establishes dynamic
// variables for all vars.
//
// References to a dynamic variable named by var must be made through the
// dynamic special form.
//
// All the initializing forms are evaluated sequentially from left to right, and
// then the values are associated with the corresponding vars. Using these
// additional dynamic bindings and the already existing bindings of visible
// identifiers, the forms body-form* are evaluated in sequential order. The
// returned value of dynamic-let is that of the last body-form of the body (or
// nil if there is none). The bindings are undone when control leaves the
// prepared dynamic-let special form.
func DyamicLet(local, global *environment.Environment, varForm ilos.Instance, bodyForm ...ilos.Instance) (ilos.Instance, ilos.Instance) {
	vfs := map[ilos.Instance]ilos.Instance{}
	if err := ensure(class.List, varForm); err != nil {
		return nil, err
	}
	for _, cadr := range varForm.(instance.List).Slice() {
		if err := ensure(class.List, cadr); err != nil {
			return nil, err
		}
		s := cadr.(instance.List).Slice()
		if len(s) != 2 {
			return nil, instance.New(class.ProgramError)
		}
		f, err := Eval(local, global, s[1])
		if err != nil {
			return nil, err
		}
		vfs[s[0]] = f
	}
	for v, f := range vfs {
		if !local.DynamicVariable.Define(v, f) {
			return nil, instance.New(class.ProgramError)
		}
	}
	return Progn(local, global, bodyForm...)
}
<|endoftext|>


<commit_before>package trust_test

import (
	"crypto/rand"
	"crypto/rsa"
	"github.com/stretchr/testify/assert"
	"net/http"
	"net/http/httptest"
	"testing"
	"time"

	"github.com/ory/hydra/oauth2/trust"

	"github.com/go-openapi/strfmt"
	"github.com/google/uuid"
	"github.com/stretchr/testify/suite"
	"gopkg.in/square/go-jose.v2"

	"github.com/ory/hydra/driver"
	"github.com/ory/hydra/jwk"

	"github.com/ory/hydra/driver/config"
	"github.com/ory/hydra/internal"
	hydra "github.com/ory/hydra/internal/httpclient/client"
	"github.com/ory/hydra/internal/httpclient/client/admin"
	"github.com/ory/hydra/internal/httpclient/models"
	"github.com/ory/hydra/x"
	"github.com/ory/x/urlx"
)

// Define the suite, and absorb the built-in basic suite
// functionality from testify - including a T() method which
// returns the current testing context.
type HandlerTestSuite struct {
	suite.Suite
	registry    driver.Registry
	server      *httptest.Server
	hydraClient *hydra.OryHydra
	publicKey   *rsa.PublicKey
}

// Setup will run before the tests in the suite are run.
func (s *HandlerTestSuite) SetupSuite() {
	conf := internal.NewConfigurationWithDefaults()
	conf.MustSet(config.KeySubjectTypesSupported, []string{"public"})
	conf.MustSet(config.KeyDefaultClientScope, []string{"foo", "bar"})
	s.registry = internal.NewRegistryMemory(s.T(), conf)

	router := x.NewRouterAdmin()
	handler := trust.NewHandler(s.registry)
	handler.SetRoutes(router)
	jwkHandler := jwk.NewHandler(s.registry, conf)
	jwkHandler.SetRoutes(router, x.NewRouterPublic(), func(h http.Handler) http.Handler {
		return h
	})
	s.server = httptest.NewServer(router)

	s.hydraClient = hydra.NewHTTPClientWithConfig(nil, &hydra.TransportConfig{Schemes: []string{"http"}, Host: urlx.ParseOrPanic(s.server.URL).Host})
	s.publicKey = s.generatePublicKey()
}

// Setup before each test.
func (s *HandlerTestSuite) SetupTest() {
}

// Will run after all the tests in the suite have been run.
func (s *HandlerTestSuite) TearDownSuite() {
}

// Will run after each test in the suite.
func (s *HandlerTestSuite) TearDownTest() {
	internal.CleanAndMigrate(s.registry)(s.T())
}

// In order for 'go test' to run this suite, we need to create
// a normal test function and pass our suite to suite.Run.
func TestHandlerTestSuite(t *testing.T) {
	suite.Run(t, new(HandlerTestSuite))
}

func (s *HandlerTestSuite) TestGrantCanBeCreatedAndFetched() {
	createRequestParams := s.newCreateJwtBearerGrantParams(
		"ory",
		"hackerman@example.com",
		false,
		[]string{"openid", "offline", "profile"},
		time.Now().Add(time.Hour),
	)
	model := createRequestParams.Body

	createResult, err := s.hydraClient.Admin.TrustJwtGrantIssuer(createRequestParams)

	s.Require().NoError(err, "no errors expected on grant creation")
	s.NotEmpty(createResult.Payload.ID, " grant id expected to be non-empty")
	s.Equal(*model.Issuer, createResult.Payload.Issuer, "issuer must match")
	s.Equal(model.Subject, createResult.Payload.Subject, "subject must match")
	s.Equal(model.Scope, createResult.Payload.Scope, "scopes must match")
	s.Equal(*model.Issuer, createResult.Payload.PublicKey.Set, "public key set must match grant issuer")
	s.Equal(*model.Jwk.Kid, createResult.Payload.PublicKey.Kid, "public key id must match")
	s.Equal(model.ExpiresAt.String(), createResult.Payload.ExpiresAt.String(), "expiration date must match")

	getRequestParams := admin.NewGetTrustedJwtGrantIssuerParams()
	getRequestParams.ID = createResult.Payload.ID
	getResult, err := s.hydraClient.Admin.GetTrustedJwtGrantIssuer(getRequestParams)

	s.Require().NoError(err, "no errors expected on grant fetching")
	s.Equal(getRequestParams.ID, getResult.Payload.ID, " grant id must match")
	s.Equal(*model.Issuer, getResult.Payload.Issuer, "issuer must match")
	s.Equal(model.Subject, getResult.Payload.Subject, "subject must match")
	s.Equal(model.Scope, getResult.Payload.Scope, "scopes must match")
	s.Equal(*model.Issuer, getResult.Payload.PublicKey.Set, "public key set must match grant issuer")
	s.Equal(*model.Jwk.Kid, getResult.Payload.PublicKey.Kid, "public key id must match")
	s.Equal(model.ExpiresAt.String(), getResult.Payload.ExpiresAt.String(), "expiration date must match")
}

func (s *HandlerTestSuite) TestGrantCanNotBeCreatedWithSameIssuerSubjectKey() {
	createRequestParams := s.newCreateJwtBearerGrantParams(
		"ory",
		"hackerman@example.com",
		false,
		[]string{"openid", "offline", "profile"},
		time.Now().Add(time.Hour),
	)

	_, err := s.hydraClient.Admin.TrustJwtGrantIssuer(createRequestParams)
	s.Require().NoError(err, "no errors expected on grant creation")

	_, err = s.hydraClient.Admin.TrustJwtGrantIssuer(createRequestParams)
	s.Require().Error(err, "expected error, because grant with same issuer+subject+kid exists")

	kid := uuid.New().String()
	createRequestParams.Body.Jwk.Kid = &kid
	_, err = s.hydraClient.Admin.TrustJwtGrantIssuer(createRequestParams)
	s.NoError(err, "no errors expected on grant creation, because kid is now different")
}

func (s *HandlerTestSuite) TestGrantCanNotBeCreatedWithSubjectAndAnySubject() {
	createRequestParams := s.newCreateJwtBearerGrantParams(
		"ory",
		"hackerman@example.com",
		true,
		[]string{"openid", "offline", "profile"},
		time.Now().Add(time.Hour),
	)

	_, err := s.hydraClient.Admin.TrustJwtGrantIssuer(createRequestParams)
	s.Require().Error(err, "expected error, because a grant with a subject and allow_any_subject cannot be created")
}

func (s *HandlerTestSuite) TestGrantCanNotBeCreatedWithMissingFields() {
	createRequestParams := s.newCreateJwtBearerGrantParams(
		"",
		"hackerman@example.com",
		false,
		[]string{"openid", "offline", "profile"},
		time.Now().Add(time.Hour),
	)

	_, err := s.hydraClient.Admin.TrustJwtGrantIssuer(createRequestParams)
	s.Require().Error(err, "expected error, because grant missing issuer")

	createRequestParams = s.newCreateJwtBearerGrantParams(
		"ory",
		"",
		false,
		[]string{"openid", "offline", "profile"},
		time.Now().Add(time.Hour),
	)

	_, err = s.hydraClient.Admin.TrustJwtGrantIssuer(createRequestParams)
	s.Require().Error(err, "expected error, because grant missing subject")

	createRequestParams = s.newCreateJwtBearerGrantParams(
		"ory",
		"hackerman@example.com",
		false,
		[]string{"openid", "offline", "profile"},
		time.Time{},
	)

	_, err = s.hydraClient.Admin.TrustJwtGrantIssuer(createRequestParams)
	s.Error(err, "expected error, because grant missing expiration date")
}

func (s *HandlerTestSuite) TestGrantPublicCanBeFetched() {
	createRequestParams := s.newCreateJwtBearerGrantParams(
		"ory",
		"hackerman@example.com",
		false,
		[]string{"openid", "offline", "profile"},
		time.Now().Add(time.Hour),
	)
	model := createRequestParams.Body

	_, err := s.hydraClient.Admin.TrustJwtGrantIssuer(createRequestParams)
	s.Require().NoError(err, "no error expected on grant creation")

	getJWKRequestParams := admin.NewGetJSONWebKeyParams()
	getJWKRequestParams.Kid = *model.Jwk.Kid
	getJWKRequestParams.Set = *model.Issuer

	getResult, err := s.hydraClient.Admin.GetJSONWebKey(getJWKRequestParams)

	s.Require().NoError(err, "no error expected on fetching public key")
	s.Equal(*model.Jwk.Kid, *getResult.Payload.Keys[0].Kid)
}

func (s *HandlerTestSuite) TestGrantWithAnySubjectCanBeCreated() {
	createRequestParams := s.newCreateJwtBearerGrantParams(
		"ory",
		"",
		true,
		[]string{"openid", "offline", "profile"},
		time.Now().Add(time.Hour),
	)

	grant, err := s.hydraClient.Admin.TrustJwtGrantIssuer(createRequestParams)
	s.Require().NoError(err, "no error expected on grant creation")

	assert.Empty(s.T(), grant.Payload.Subject)
	assert.Truef(s.T(), grant.Payload.AllowAnySubject, "grant with any subject must be true")
}

func (s *HandlerTestSuite) TestGrantListCanBeFetched() {
	createRequestParams := s.newCreateJwtBearerGrantParams(
		"ory",
		"hackerman@example.com",
		false,
		[]string{"openid", "offline", "profile"},
		time.Now().Add(time.Hour),
	)
	createRequestParams2 := s.newCreateJwtBearerGrantParams(
		"ory2",
		"safetyman@example.com",
		false,
		[]string{"profile"},
		time.Now().Add(time.Hour),
	)

	_, err := s.hydraClient.Admin.TrustJwtGrantIssuer(createRequestParams)
	s.Require().NoError(err, "no errors expected on grant creation")

	_, err = s.hydraClient.Admin.TrustJwtGrantIssuer(createRequestParams2)
	s.Require().NoError(err, "no errors expected on grant creation")

	getRequestParams := admin.NewListTrustedJwtGrantIssuersParams()
	getResult, err := s.hydraClient.Admin.ListTrustedJwtGrantIssuers(getRequestParams)

	s.Require().NoError(err, "no errors expected on grant list fetching")
	s.Len(getResult.Payload, 2, "expected to get list of 2 grants")

	getRequestParams.Issuer = createRequestParams2.Body.Issuer
	getResult, err = s.hydraClient.Admin.ListTrustedJwtGrantIssuers(getRequestParams)

	s.Require().NoError(err, "no errors expected on grant list fetching")
	s.Len(getResult.Payload, 1, "expected to get list of 1 grant, when filtering by issuer")
	s.Equal(*createRequestParams2.Body.Issuer, getResult.Payload[0].Issuer, "issuer must match")
}

func (s *HandlerTestSuite) TestGrantCanBeDeleted() {
	createRequestParams := s.newCreateJwtBearerGrantParams(
		"ory",
		"hackerman@example.com",
		false,
		[]string{"openid", "offline", "profile"},
		time.Now().Add(time.Hour),
	)

	createResult, err := s.hydraClient.Admin.TrustJwtGrantIssuer(createRequestParams)
	s.Require().NoError(err, "no errors expected on grant creation")

	deleteRequestParams := admin.NewDeleteTrustedJwtGrantIssuerParams()
	deleteRequestParams.ID = createResult.Payload.ID
	_, err = s.hydraClient.Admin.DeleteTrustedJwtGrantIssuer(deleteRequestParams)

	s.Require().NoError(err, "no errors expected on grant deletion")

	_, err = s.hydraClient.Admin.DeleteTrustedJwtGrantIssuer(deleteRequestParams)
	s.Error(err, "expected error, because grant has been already deleted")
}

func (s *HandlerTestSuite) generateJWK(publicKey *rsa.PublicKey) *models.JSONWebKey {
	jwk := jose.JSONWebKey{
		Key:       publicKey,
		KeyID:     uuid.New().String(),
		Algorithm: string(jose.RS256),
		Use:       "sig",
	}
	b, err := jwk.MarshalJSON()
	s.Require().NoError(err)

	mJWK := &models.JSONWebKey{}
	err = mJWK.UnmarshalBinary(b)
	s.Require().NoError(err)

	return mJWK
}

func (s *HandlerTestSuite) newCreateJwtBearerGrantParams(
	issuer, subject string, allowAnySubject bool, scope []string, expiresAt time.Time,
) *admin.TrustJwtGrantIssuerParams {
	createRequestParams := admin.NewTrustJwtGrantIssuerParams()
	exp := strfmt.DateTime(expiresAt.UTC().Round(time.Second))
	model := &models.TrustJwtGrantIssuerBody{
		ExpiresAt:       &exp,
		Issuer:          &issuer,
		Jwk:             s.generateJWK(s.publicKey),
		Scope:           scope,
		Subject:         subject,
		AllowAnySubject: allowAnySubject,
	}
	createRequestParams.SetBody(model)

	return createRequestParams
}

func (s *HandlerTestSuite) generatePublicKey() *rsa.PublicKey {
	privateKey, err := rsa.GenerateKey(rand.Reader, 2048)
	s.Require().NoError(err)
	return &privateKey.PublicKey
}
<commit_msg>autogen(openapi): regenerate swagger spec and internal client<commit_after>package trust_test

import (
	"crypto/rand"
	"crypto/rsa"
	"net/http"
	"net/http/httptest"
	"testing"
	"time"

	"github.com/stretchr/testify/assert"

	"github.com/ory/hydra/oauth2/trust"

	"github.com/go-openapi/strfmt"
	"github.com/google/uuid"
	"github.com/stretchr/testify/suite"
	"gopkg.in/square/go-jose.v2"

	"github.com/ory/hydra/driver"
	"github.com/ory/hydra/jwk"

	"github.com/ory/hydra/driver/config"
	"github.com/ory/hydra/internal"
	hydra "github.com/ory/hydra/internal/httpclient/client"
	"github.com/ory/hydra/internal/httpclient/client/admin"
	"github.com/ory/hydra/internal/httpclient/models"
	"github.com/ory/hydra/x"
	"github.com/ory/x/urlx"
)

// Define the suite, and absorb the built-in basic suite
// functionality from testify - including a T() method which
// returns the current testing context.
type HandlerTestSuite struct {
	suite.Suite
	registry    driver.Registry
	server      *httptest.Server
	hydraClient *hydra.OryHydra
	publicKey   *rsa.PublicKey
}

// Setup will run before the tests in the suite are run.
func (s *HandlerTestSuite) SetupSuite() {
	conf := internal.NewConfigurationWithDefaults()
	conf.MustSet(config.KeySubjectTypesSupported, []string{"public"})
	conf.MustSet(config.KeyDefaultClientScope, []string{"foo", "bar"})
	s.registry = internal.NewRegistryMemory(s.T(), conf)

	router := x.NewRouterAdmin()
	handler := trust.NewHandler(s.registry)
	handler.SetRoutes(router)
	jwkHandler := jwk.NewHandler(s.registry, conf)
	jwkHandler.SetRoutes(router, x.NewRouterPublic(), func(h http.Handler) http.Handler {
		return h
	})
	s.server = httptest.NewServer(router)

	s.hydraClient = hydra.NewHTTPClientWithConfig(nil, &hydra.TransportConfig{Schemes: []string{"http"}, Host: urlx.ParseOrPanic(s.server.URL).Host})
	s.publicKey = s.generatePublicKey()
}

// Setup before each test.
func (s *HandlerTestSuite) SetupTest() {
}

// Will run after all the tests in the suite have been run.
func (s *HandlerTestSuite) TearDownSuite() {
}

// Will run after each test in the suite.
func (s *HandlerTestSuite) TearDownTest() {
	internal.CleanAndMigrate(s.registry)(s.T())
}

// In order for 'go test' to run this suite, we need to create
// a normal test function and pass our suite to suite.Run.
func TestHandlerTestSuite(t *testing.T) {
	suite.Run(t, new(HandlerTestSuite))
}

func (s *HandlerTestSuite) TestGrantCanBeCreatedAndFetched() {
	createRequestParams := s.newCreateJwtBearerGrantParams(
		"ory",
		"hackerman@example.com",
		false,
		[]string{"openid", "offline", "profile"},
		time.Now().Add(time.Hour),
	)
	model := createRequestParams.Body

	createResult, err := s.hydraClient.Admin.TrustJwtGrantIssuer(createRequestParams)

	s.Require().NoError(err, "no errors expected on grant creation")
	s.NotEmpty(createResult.Payload.ID, " grant id expected to be non-empty")
	s.Equal(*model.Issuer, createResult.Payload.Issuer, "issuer must match")
	s.Equal(model.Subject, createResult.Payload.Subject, "subject must match")
	s.Equal(model.Scope, createResult.Payload.Scope, "scopes must match")
	s.Equal(*model.Issuer, createResult.Payload.PublicKey.Set, "public key set must match grant issuer")
	s.Equal(*model.Jwk.Kid, createResult.Payload.PublicKey.Kid, "public key id must match")
	s.Equal(model.ExpiresAt.String(), createResult.Payload.ExpiresAt.String(), "expiration date must match")

	getRequestParams := admin.NewGetTrustedJwtGrantIssuerParams()
	getRequestParams.ID = createResult.Payload.ID
	getResult, err := s.hydraClient.Admin.GetTrustedJwtGrantIssuer(getRequestParams)

	s.Require().NoError(err, "no errors expected on grant fetching")
	s.Equal(getRequestParams.ID, getResult.Payload.ID, " grant id must match")
	s.Equal(*model.Issuer, getResult.Payload.Issuer, "issuer must match")
	s.Equal(model.Subject, getResult.Payload.Subject, "subject must match")
	s.Equal(model.Scope, getResult.Payload.Scope, "scopes must match")
	s.Equal(*model.Issuer, getResult.Payload.PublicKey.Set, "public key set must match grant issuer")
	s.Equal(*model.Jwk.Kid, getResult.Payload.PublicKey.Kid, "public key id must match")
	s.Equal(model.ExpiresAt.String(), getResult.Payload.ExpiresAt.String(), "expiration date must match")
}

func (s *HandlerTestSuite) TestGrantCanNotBeCreatedWithSameIssuerSubjectKey() {
	createRequestParams := s.newCreateJwtBearerGrantParams(
		"ory",
		"hackerman@example.com",
		false,
		[]string{"openid", "offline", "profile"},
		time.Now().Add(time.Hour),
	)

	_, err := s.hydraClient.Admin.TrustJwtGrantIssuer(createRequestParams)
	s.Require().NoError(err, "no errors expected on grant creation")

	_, err = s.hydraClient.Admin.TrustJwtGrantIssuer(createRequestParams)
	s.Require().Error(err, "expected error, because grant with same issuer+subject+kid exists")

	kid := uuid.New().String()
	createRequestParams.Body.Jwk.Kid = &kid
	_, err = s.hydraClient.Admin.TrustJwtGrantIssuer(createRequestParams)
	s.NoError(err, "no errors expected on grant creation, because kid is now different")
}

func (s *HandlerTestSuite) TestGrantCanNotBeCreatedWithSubjectAndAnySubject() {
	createRequestParams := s.newCreateJwtBearerGrantParams(
		"ory",
		"hackerman@example.com",
		true,
		[]string{"openid", "offline", "profile"},
		time.Now().Add(time.Hour),
	)

	_, err := s.hydraClient.Admin.TrustJwtGrantIssuer(createRequestParams)
	s.Require().Error(err, "expected error, because a grant with a subject and allow_any_subject cannot be created")
}

func (s *HandlerTestSuite) TestGrantCanNotBeCreatedWithMissingFields() {
	createRequestParams := s.newCreateJwtBearerGrantParams(
		"",
		"hackerman@example.com",
		false,
		[]string{"openid", "offline", "profile"},
		time.Now().Add(time.Hour),
	)

	_, err := s.hydraClient.Admin.TrustJwtGrantIssuer(createRequestParams)
	s.Require().Error(err, "expected error, because grant missing issuer")

	createRequestParams = s.newCreateJwtBearerGrantParams(
		"ory",
		"",
		false,
		[]string{"openid", "offline", "profile"},
		time.Now().Add(time.Hour),
	)

	_, err = s.hydraClient.Admin.TrustJwtGrantIssuer(createRequestParams)
	s.Require().Error(err, "expected error, because grant missing subject")

	createRequestParams = s.newCreateJwtBearerGrantParams(
		"ory",
		"hackerman@example.com",
		false,
		[]string{"openid", "offline", "profile"},
		time.Time{},
	)

	_, err = s.hydraClient.Admin.TrustJwtGrantIssuer(createRequestParams)
	s.Error(err, "expected error, because grant missing expiration date")
}

func (s *HandlerTestSuite) TestGrantPublicCanBeFetched() {
	createRequestParams := s.newCreateJwtBearerGrantParams(
		"ory",
		"hackerman@example.com",
		false,
		[]string{"openid", "offline", "profile"},
		time.Now().Add(time.Hour),
	)
	model := createRequestParams.Body

	_, err := s.hydraClient.Admin.TrustJwtGrantIssuer(createRequestParams)
	s.Require().NoError(err, "no error expected on grant creation")

	getJWKRequestParams := admin.NewGetJSONWebKeyParams()
	getJWKRequestParams.Kid = *model.Jwk.Kid
	getJWKRequestParams.Set = *model.Issuer

	getResult, err := s.hydraClient.Admin.GetJSONWebKey(getJWKRequestParams)

	s.Require().NoError(err, "no error expected on fetching public key")
	s.Equal(*model.Jwk.Kid, *getResult.Payload.Keys[0].Kid)
}

func (s *HandlerTestSuite) TestGrantWithAnySubjectCanBeCreated() {
	createRequestParams := s.newCreateJwtBearerGrantParams(
		"ory",
		"",
		true,
		[]string{"openid", "offline", "profile"},
		time.Now().Add(time.Hour),
	)

	grant, err := s.hydraClient.Admin.TrustJwtGrantIssuer(createRequestParams)
	s.Require().NoError(err, "no error expected on grant creation")

	assert.Empty(s.T(), grant.Payload.Subject)
	assert.Truef(s.T(), grant.Payload.AllowAnySubject, "grant with any subject must be true")
}

func (s *HandlerTestSuite) TestGrantListCanBeFetched() {
	createRequestParams := s.newCreateJwtBearerGrantParams(
		"ory",
		"hackerman@example.com",
		false,
		[]string{"openid", "offline", "profile"},
		time.Now().Add(time.Hour),
	)
	createRequestParams2 := s.newCreateJwtBearerGrantParams(
		"ory2",
		"safetyman@example.com",
		false,
		[]string{"profile"},
		time.Now().Add(time.Hour),
	)

	_, err := s.hydraClient.Admin.TrustJwtGrantIssuer(createRequestParams)
	s.Require().NoError(err, "no errors expected on grant creation")

	_, err = s.hydraClient.Admin.TrustJwtGrantIssuer(createRequestParams2)
	s.Require().NoError(err, "no errors expected on grant creation")

	getRequestParams := admin.NewListTrustedJwtGrantIssuersParams()
	getResult, err := s.hydraClient.Admin.ListTrustedJwtGrantIssuers(getRequestParams)

	s.Require().NoError(err, "no errors expected on grant list fetching")
	s.Len(getResult.Payload, 2, "expected to get list of 2 grants")

	getRequestParams.Issuer = createRequestParams2.Body.Issuer
	getResult, err = s.hydraClient.Admin.ListTrustedJwtGrantIssuers(getRequestParams)

	s.Require().NoError(err, "no errors expected on grant list fetching")
	s.Len(getResult.Payload, 1, "expected to get list of 1 grant, when filtering by issuer")
	s.Equal(*createRequestParams2.Body.Issuer, getResult.Payload[0].Issuer, "issuer must match")
}

func (s *HandlerTestSuite) TestGrantCanBeDeleted() {
	createRequestParams := s.newCreateJwtBearerGrantParams(
		"ory",
		"hackerman@example.com",
		false,
		[]string{"openid", "offline", "profile"},
		time.Now().Add(time.Hour),
	)

	createResult, err := s.hydraClient.Admin.TrustJwtGrantIssuer(createRequestParams)
	s.Require().NoError(err, "no errors expected on grant creation")

	deleteRequestParams := admin.NewDeleteTrustedJwtGrantIssuerParams()
	deleteRequestParams.ID = createResult.Payload.ID
	_, err = s.hydraClient.Admin.DeleteTrustedJwtGrantIssuer(deleteRequestParams)

	s.Require().NoError(err, "no errors expected on grant deletion")

	_, err = s.hydraClient.Admin.DeleteTrustedJwtGrantIssuer(deleteRequestParams)
	s.Error(err, "expected error, because grant has been already deleted")
}

func (s *HandlerTestSuite) generateJWK(publicKey *rsa.PublicKey) *models.JSONWebKey {
	jwk := jose.JSONWebKey{
		Key:       publicKey,
		KeyID:     uuid.New().String(),
		Algorithm: string(jose.RS256),
		Use:       "sig",
	}
	b, err := jwk.MarshalJSON()
	s.Require().NoError(err)

	mJWK := &models.JSONWebKey{}
	err = mJWK.UnmarshalBinary(b)
	s.Require().NoError(err)

	return mJWK
}

func (s *HandlerTestSuite) newCreateJwtBearerGrantParams(
	issuer, subject string, allowAnySubject bool, scope []string, expiresAt time.Time,
) *admin.TrustJwtGrantIssuerParams {
	createRequestParams := admin.NewTrustJwtGrantIssuerParams()
	exp := strfmt.DateTime(expiresAt.UTC().Round(time.Second))
	model := &models.TrustJwtGrantIssuerBody{
		ExpiresAt:       &exp,
		Issuer:          &issuer,
		Jwk:             s.generateJWK(s.publicKey),
		Scope:           scope,
		Subject:         subject,
		AllowAnySubject: allowAnySubject,
	}
	createRequestParams.SetBody(model)

	return createRequestParams
}

func (s *HandlerTestSuite) generatePublicKey() *rsa.PublicKey {
	privateKey, err := rsa.GenerateKey(rand.Reader, 2048)
	s.Require().NoError(err)
	return &privateKey.PublicKey
}
<|endoftext|>


<commit_before>package test

import (
	"fmt"
	"reflect"
	"testing"

	driver "github.com/arangodb/go-driver"
)

func TestDatabaseTransaction(t *testing.T) {
	c := createClientFromEnv(t, true)
	db := ensureDatabase(nil, c, "transaction_test", nil, t)

	testCases := []struct {
		name         string
		action       string
		options      *driver.TransactionOptions
		expectResult interface{}
		expectError  error
	}{
		{"ReturnValue", "function () { return 'worked!'; }", nil, "worked!", nil},
		{"ReturnError", "function () { error error; }", nil, nil, fmt.Errorf("missing/invalid action definition for transaction - Uncaught SyntaxError: Unexpected identifier - SyntaxError: Unexpected identifier\n    at new Function (<anonymous>)")},
	}

	for _, testCase := range testCases {
		t.Run(testCase.name, func(t *testing.T) {
			result, err := db.Transaction(nil, testCase.action, testCase.options)
			if !reflect.DeepEqual(testCase.expectResult, result) {
				t.Errorf("expected result %v, got %v", testCase.expectResult, result)
			}
			if testCase.expectError != nil {
				if testCase.expectError.Error() != err.Error() {
					t.Errorf("expected error %v, got %v", testCase.expectError.Error(), err.Error())
				}
			}
		})
	}
}
<commit_msg>made test skip below version 3.2<commit_after>package test

import (
	"fmt"
	"reflect"
	"testing"

	driver "github.com/arangodb/go-driver"
)

func TestDatabaseTransaction(t *testing.T) {
	c := createClientFromEnv(t, true)
	skipBelowVersion(c, "3.2", t)
	db := ensureDatabase(nil, c, "transaction_test", nil, t)

	testCases := []struct {
		name         string
		action       string
		options      *driver.TransactionOptions
		expectResult interface{}
		expectError  error
	}{
		{"ReturnValue", "function () { return 'worked!'; }", nil, "worked!", nil},
		{"ReturnError", "function () { error error; }", nil, nil, fmt.Errorf("missing/invalid action definition for transaction - Uncaught SyntaxError: Unexpected identifier - SyntaxError: Unexpected identifier\n    at new Function (<anonymous>)")},
	}

	for _, testCase := range testCases {
		t.Run(testCase.name, func(t *testing.T) {
			result, err := db.Transaction(nil, testCase.action, testCase.options)
			if !reflect.DeepEqual(testCase.expectResult, result) {
				t.Errorf("expected result %v, got %v", testCase.expectResult, result)
			}
			if testCase.expectError != nil {
				if testCase.expectError.Error() != err.Error() {
					t.Errorf("expected error %v, got %v", testCase.expectError.Error(), err.Error())
				}
			}
		})
	}
}
<|endoftext|>


<commit_before>// +build e2e

/*
Copyright 2020 The Knative Authors

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
package autotls

import (
	"context"
	"crypto/tls"
	"crypto/x509"
	"net/http"
	"testing"
	"time"

	corev1 "k8s.io/api/core/v1"
	apierrors "k8s.io/apimachinery/pkg/api/errors"
	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
	"k8s.io/apimachinery/pkg/util/wait"

	"knative.dev/serving/pkg/apis/networking"
	routenames "knative.dev/serving/pkg/reconciler/route/resources/names"
	"knative.dev/serving/test"
	testingress "knative.dev/serving/test/conformance/ingress"
	"knative.dev/serving/test/e2e"
	v1test "knative.dev/serving/test/v1"
)

const (
	systemNamespace = "knative-serving"
)

// To run this test locally with cert-manager, you need to
// 1. Install cert-manager from `third_party/` directory.
// 2. Run the command below to do the configuration:
// kubectl apply -f test/config/autotls/certmanager/selfsigned/
func TestPerKsvcCert_localCA(t *testing.T) {
	clients := e2e.Setup(t)
	disableNamespaceCert(t, clients)

	names := test.ResourceNames{
		Service: test.ObjectNameForTest(t),
		Image:   "runtime",
	}
	test.CleanupOnInterrupt(func() { test.TearDown(clients, names) })
	defer test.TearDown(clients, names)

	objects, err := v1test.CreateServiceReady(t, clients, &names)
	if err != nil {
		t.Fatalf("Failed to create initial Service: %v: %v", names.Service, err)
	}

	cancel := turnOnAutoTLS(t, clients)
	defer cancel()

	// wait for certificate to be ready
	waitForCertificateReady(t, clients, routenames.Certificate(objects.Route))

	// curl HTTPS
	secretName := routenames.Certificate(objects.Route)
	rootCAs := createRootCAs(t, clients, objects.Route.Namespace, secretName)

	// The TLS info is added to the ingress after the service is created, that's
	// why we need to wait again
	err = v1test.WaitForServiceState(clients.ServingClient, names.Service, v1test.IsServiceReady, "ServiceIsReady")
	if err != nil {
		t.Fatalf("Service %s did not become ready: %v", names.Service, err)
	}

	httpsClient := createHTTPSClient(t, clients, objects, rootCAs)
	testingress.RuntimeRequest(t, httpsClient, "https://"+objects.Service.Status.URL.Host)
}

func createRootCAs(t *testing.T, clients *test.Clients, ns, secretName string) *x509.CertPool {
	secret, err := clients.KubeClient.Kube.CoreV1().Secrets(ns).Get(
		secretName, metav1.GetOptions{})
	if err != nil {
		t.Fatalf("Failed to get Secret %s: %v", secretName, err)
	}

	rootCAs, _ := x509.SystemCertPool()
	if rootCAs == nil {
		rootCAs = x509.NewCertPool()
	}
	if !rootCAs.AppendCertsFromPEM(secret.Data[corev1.TLSCertKey]) {
		t.Fatal("Failed to add the certificate to the root CA")
	}
	return rootCAs
}

func createHTTPSClient(t *testing.T, clients *test.Clients, objects *v1test.ResourceObjects, rootCAs *x509.CertPool) *http.Client {
	ing, err := clients.NetworkingClient.Ingresses.Get(routenames.Ingress(objects.Route), metav1.GetOptions{})
	if err != nil {
		t.Fatalf("Failed to get Ingress %s: %v", routenames.Ingress(objects.Route), err)
	}
	dialer := testingress.CreateDialContext(t, ing, clients)
	tlsConfig := &tls.Config{
		RootCAs: rootCAs,
	}
	return &http.Client{
		Transport: &http.Transport{
			DialContext:     dialer,
			TLSClientConfig: tlsConfig,
		}}
}

func disableNamespaceCert(t *testing.T, clients *test.Clients) {
	namespaces, err := clients.KubeClient.Kube.CoreV1().Namespaces().List(metav1.ListOptions{})
	if err != nil {
		t.Fatalf("Failed to list namespaces: %v", err)
	}
	for _, ns := range namespaces.Items {
		if ns.Labels == nil {
			ns.Labels = map[string]string{}
		}
		ns.Labels[networking.DisableWildcardCertLabelKey] = "true"
		if _, err := clients.KubeClient.Kube.CoreV1().Namespaces().Update(&ns); err != nil {
			t.Errorf("Fail to disable namespace cert: %v", err)
		}
	}
}

func turnOnAutoTLS(t *testing.T, clients *test.Clients) context.CancelFunc {
	configNetworkCM, err := clients.KubeClient.Kube.CoreV1().ConfigMaps(systemNamespace).Get("config-network", metav1.GetOptions{})
	if err != nil {
		t.Fatalf("Failed to get config-network ConfigMap: %v", err)
	}
	configNetworkCM.Data["autoTLS"] = "Enabled"
	test.CleanupOnInterrupt(func() {
		turnOffAutoTLS(t, clients)
	})
	if _, err := clients.KubeClient.Kube.CoreV1().ConfigMaps(systemNamespace).Update(configNetworkCM); err != nil {
		t.Fatalf("Failed to update config-network ConfigMap: %v", err)
	}
	return func() {
		turnOffAutoTLS(t, clients)
	}
}

func turnOffAutoTLS(t *testing.T, clients *test.Clients) {
	configNetworkCM, err := clients.KubeClient.Kube.CoreV1().ConfigMaps(systemNamespace).Get("config-network", metav1.GetOptions{})
	if err != nil {
		t.Errorf("Failed to get config-network ConfigMap: %v", err)
		return
	}
	delete(configNetworkCM.Data, "autoTLS")
	if _, err := clients.KubeClient.Kube.CoreV1().ConfigMaps(configNetworkCM.Namespace).Update(configNetworkCM); err != nil {
		t.Errorf("Failed to turn off Auto TLS: %v", err)
	}
}

func waitForCertificateReady(t *testing.T, clients *test.Clients, certName string) {
	if err := wait.Poll(10*time.Second, 300*time.Second, func() (bool, error) {
		cert, err := clients.NetworkingClient.Certificates.Get(certName, metav1.GetOptions{})
		if err != nil {
			if apierrors.IsNotFound(err) {
				t.Logf("Certificate %s has not been created: %v", certName, err)
				return false, nil
			}
			return false, err
		}
		return cert.Status.IsReady(), nil
	}); err != nil {
		t.Fatalf("Certificate %s is not ready: %v", certName, err)
	}
}
<commit_msg>E2E test for namespace cert for Auto TLS (#6631)<commit_after>// +build e2e

/*
Copyright 2020 The Knative Authors

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
package autotls

import (
	"context"
	"crypto/tls"
	"crypto/x509"
	"net/http"
	"strings"
	"testing"
	"time"

	corev1 "k8s.io/api/core/v1"
	apierrors "k8s.io/apimachinery/pkg/api/errors"
	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
	"k8s.io/apimachinery/pkg/util/sets"
	"k8s.io/apimachinery/pkg/util/wait"

	"knative.dev/pkg/system"
	"knative.dev/serving/pkg/apis/networking"
	routenames "knative.dev/serving/pkg/reconciler/route/resources/names"
	"knative.dev/serving/test"
	testingress "knative.dev/serving/test/conformance/ingress"
	"knative.dev/serving/test/e2e"
	v1test "knative.dev/serving/test/v1"
)

// To run this test locally with cert-manager, you need to
// 1. Install cert-manager from `third_party/` directory.
// 2. Run the command below to do the configuration:
// kubectl apply -f test/config/autotls/certmanager/selfsigned/
func TestPerKsvcCert_localCA(t *testing.T) {
	clients := e2e.Setup(t)
	disableNamespaceCertWithWhiteList(t, clients, sets.String{})

	names := test.ResourceNames{
		Service: test.ObjectNameForTest(t),
		Image:   "runtime",
	}
	test.CleanupOnInterrupt(func() { test.TearDown(clients, names) })
	defer test.TearDown(clients, names)

	objects, err := v1test.CreateServiceReady(t, clients, &names)
	if err != nil {
		t.Fatalf("Failed to create initial Service: %v: %v", names.Service, err)
	}

	cancel := turnOnAutoTLS(t, clients)
	test.CleanupOnInterrupt(cancel)
	defer cancel()

	// wait for certificate to be ready
	waitForCertificateReady(t, clients, routenames.Certificate(objects.Route))

	// curl HTTPS
	secretName := routenames.Certificate(objects.Route)
	rootCAs := createRootCAs(t, clients, objects.Route.Namespace, secretName)

	// The TLS info is added to the ingress after the service is created, that's
	// why we need to wait again
	err = v1test.WaitForServiceState(clients.ServingClient, names.Service, v1test.IsServiceReady, "ServiceIsReady")
	if err != nil {
		t.Fatalf("Service %s did not become ready: %v", names.Service, err)
	}

	httpsClient := createHTTPSClient(t, clients, objects, rootCAs)
	testingress.RuntimeRequest(t, httpsClient, "https://"+objects.Service.Status.URL.Host)
}

// To run this test locally with cert-manager, you need to
// 1. Install cert-manager from `third_party/` directory.
// 2. Run the command below to do the configuration:
// kubectl apply -f test/config/autotls/certmanager/selfsigned/
func TestPerNamespaceCert_localCA(t *testing.T) {
	clients := e2e.Setup(t)
	disableNamespaceCertWithWhiteList(t, clients, sets.NewString(test.ServingNamespace))
	defer disableNamespaceCertWithWhiteList(t, clients, sets.String{})

	cancel := turnOnAutoTLS(t, clients)
	test.CleanupOnInterrupt(cancel)
	defer cancel()

	// wait for certificate to be ready
	certName, err := waitForNamespaceCertReady(clients)
	if err != nil {
		t.Fatalf("Namespace Cert failed to become ready: %v", err)
	}

	names := test.ResourceNames{
		Service: test.ObjectNameForTest(t),
		Image:   "runtime",
	}
	test.CleanupOnInterrupt(func() { test.TearDown(clients, names) })
	defer test.TearDown(clients, names)

	objects, err := v1test.CreateServiceReady(t, clients, &names)
	if err != nil {
		t.Fatalf("Failed to create initial Service: %s: %v", names.Service, err)
	}

	// curl HTTPS
	rootCAs := createRootCAs(t, clients, test.ServingNamespace, certName)
	httpsClient := createHTTPSClient(t, clients, objects, rootCAs)
	testingress.RuntimeRequest(t, httpsClient, "https://"+objects.Service.Status.URL.Host)
}

func createRootCAs(t *testing.T, clients *test.Clients, ns, secretName string) *x509.CertPool {
	secret, err := clients.KubeClient.Kube.CoreV1().Secrets(ns).Get(
		secretName, metav1.GetOptions{})
	if err != nil {
		t.Fatalf("Failed to get Secret %s: %v", secretName, err)
	}

	rootCAs, err := x509.SystemCertPool()
	if rootCAs == nil || err != nil {
		if err != nil {
			t.Logf("Failed to load cert poll from system: %v. Will create a new cert pool.", err)
		}
		rootCAs = x509.NewCertPool()
	}
	if !rootCAs.AppendCertsFromPEM(secret.Data[corev1.TLSCertKey]) {
		t.Fatal("Failed to add the certificate to the root CA")
	}
	return rootCAs
}

func createHTTPSClient(t *testing.T, clients *test.Clients, objects *v1test.ResourceObjects, rootCAs *x509.CertPool) *http.Client {
	ing, err := clients.NetworkingClient.Ingresses.Get(routenames.Ingress(objects.Route), metav1.GetOptions{})
	if err != nil {
		t.Fatalf("Failed to get Ingress %s: %v", routenames.Ingress(objects.Route), err)
	}
	dialer := testingress.CreateDialContext(t, ing, clients)
	tlsConfig := &tls.Config{
		RootCAs: rootCAs,
	}
	return &http.Client{
		Transport: &http.Transport{
			DialContext:     dialer,
			TLSClientConfig: tlsConfig,
		}}
}

func disableNamespaceCertWithWhiteList(t *testing.T, clients *test.Clients, whiteLists sets.String) {
	namespaces, err := clients.KubeClient.Kube.CoreV1().Namespaces().List(metav1.ListOptions{})
	if err != nil {
		t.Fatalf("Failed to list namespaces: %v", err)
	}
	for _, ns := range namespaces.Items {
		if ns.Labels == nil {
			ns.Labels = map[string]string{}
		}
		if whiteLists.Has(ns.Name) {
			delete(ns.Labels, networking.DisableWildcardCertLabelKey)
		} else {
			ns.Labels[networking.DisableWildcardCertLabelKey] = "true"
		}
		if _, err := clients.KubeClient.Kube.CoreV1().Namespaces().Update(&ns); err != nil {
			t.Errorf("Fail to disable namespace cert: %v", err)
		}
	}
}

func turnOnAutoTLS(t *testing.T, clients *test.Clients) context.CancelFunc {
	configNetworkCM, err := clients.KubeClient.Kube.CoreV1().ConfigMaps(system.Namespace()).Get("config-network", metav1.GetOptions{})
	if err != nil {
		t.Fatalf("Failed to get config-network ConfigMap: %v", err)
	}
	configNetworkCM.Data["autoTLS"] = "Enabled"
	test.CleanupOnInterrupt(func() {
		turnOffAutoTLS(t, clients)
	})
	if _, err := clients.KubeClient.Kube.CoreV1().ConfigMaps(system.Namespace()).Update(configNetworkCM); err != nil {
		t.Fatalf("Failed to update config-network ConfigMap: %v", err)
	}
	return func() {
		turnOffAutoTLS(t, clients)
	}
}

func turnOffAutoTLS(t *testing.T, clients *test.Clients) {
	configNetworkCM, err := clients.KubeClient.Kube.CoreV1().ConfigMaps(system.Namespace()).Get("config-network", metav1.GetOptions{})
	if err != nil {
		t.Errorf("Failed to get config-network ConfigMap: %v", err)
		return
	}
	delete(configNetworkCM.Data, "autoTLS")
	if _, err := clients.KubeClient.Kube.CoreV1().ConfigMaps(configNetworkCM.Namespace).Update(configNetworkCM); err != nil {
		t.Errorf("Failed to turn off Auto TLS: %v", err)
	}
}

func waitForCertificateReady(t *testing.T, clients *test.Clients, certName string) {
	if err := wait.Poll(10*time.Second, 300*time.Second, func() (bool, error) {
		cert, err := clients.NetworkingClient.Certificates.Get(certName, metav1.GetOptions{})
		if err != nil {
			if apierrors.IsNotFound(err) {
				t.Logf("Certificate %s has not been created: %v", certName, err)
				return false, nil
			}
			return false, err
		}
		return cert.Status.IsReady(), nil
	}); err != nil {
		t.Fatalf("Certificate %s is not ready: %v", certName, err)
	}
}

func waitForNamespaceCertReady(clients *test.Clients) (string, error) {
	var certName string
	err := wait.Poll(10*time.Second, 300*time.Second, func() (bool, error) {
		certs, err := clients.NetworkingClient.Certificates.List(metav1.ListOptions{})
		if err != nil {
			return false, err
		}
		for _, cert := range certs.Items {
			if strings.Contains(cert.Name, test.ServingNamespace) {
				certName = cert.Name
				return cert.Status.IsReady(), nil
			}
		}
		// Namespace certificate has not been created.
		return false, nil
	})
	return certName, err
}
<|endoftext|>


<commit_before>// Copyright 2012 Aaron Jacobs. All Rights Reserved.
// Author: aaronjjacobs@gmail.com (Aaron Jacobs)
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
//
// An integration test that uses a real S3 account. Run as follows:
//
//     go run integration_test/*.go \
//         -key_id <key ID> \
//         -bucket <bucket> \
//         -region s3-ap-northeast-1.amazonaws.com
//
// Before doing this, create an empty bucket (or delete the contents of an
// existing bucket) using the S3 management console.

package main

import (
	"fmt"
	"github.com/jacobsa/aws/s3"
	. "github.com/jacobsa/oglematchers"
	. "github.com/jacobsa/ogletest"
	"sync"
)

////////////////////////////////////////////////////////////////////////
// Helpers
////////////////////////////////////////////////////////////////////////

// Run the supplied function for every integer in [0, n) with some degree of
// parallelism, returning an error if any invocation returns an error.
func runForRange(n int, f func (int) error) (err error) {
		// Set up channels. The work channel should be buffered so that we don't
		// have to block writing to it before checking for errors below. The error
		// channel must be buffered so that no worker goroutine gets stuck writing
		// a result to it and never returns. The stop channel must not be buffered
		// so that we can be sure that no more work will be processed when we
		// return below.
		work := make(chan int, n)
		errs := make(chan error, n)
		stop := make(chan bool)

		// Launch worker functions that attempt to do work, returning if a read
		// from the stop channel succeeds.
		processWork := func() {
			for {
				select {
				case i := <-work:
					errs<- f(i)
				case <-stop:
					return
				}
			}
		}

		const numWorkers = 17
		for i := 0; i < numWorkers; i++ {
			go processWork()
		}

		// Feed the workers work.
		for i := 0; i < n; i++ {
			work<- i
		}

		// Read results, stopping immediately if there is an error.
		for i := 0; i < n; i++ {
			err = <-errs
			if err != nil {
				break
			}
		}

		// Stop all of the workers, and wait for them to stop. This ensures that
		// no piece of work is in progress when we return.
		for i := 0; i < numWorkers; i++ {
			stop<- true
		}

		return
	}

type BucketTest struct {
	bucket s3.Bucket

	mutex sync.Mutex
	keysToDelete []string
}

func init() { RegisterTestSuite(&BucketTest{}) }

// Ensure that the given key is deleted before the test finishes.
func (t *BucketTest) ensureDeleted(key string) {
	t.mutex.Lock()
	defer t.mutex.Unlock()

	t.keysToDelete = append(t.keysToDelete, key)
}

func (t *BucketTest) SetUp(i *TestInfo) {
	var err error

	// Open a bucket.
	t.bucket, err = s3.OpenBucket(*g_bucketName, s3.Region(*g_region), g_accessKey)
	AssertEq(nil, err)
}

func (t *BucketTest) TearDown() {
	t.mutex.Lock()
	defer t.mutex.Unlock()

	err := runForRange(len(t.keysToDelete), func (i int) error {
		key := t.keysToDelete[i]
		if err := t.bucket.DeleteObject(key); err != nil {
			return fmt.Errorf("Couldn't delete key %s: %v", key, err)
		}

		return nil
	})

	AssertEq(nil, err)
}

////////////////////////////////////////////////////////////////////////
// Tests
////////////////////////////////////////////////////////////////////////

func (t *BucketTest) WrongAccessKeySecret() {
	// Open a bucket with the wrong key.
	wrongKey := g_accessKey
	wrongKey.Secret += "taco"

	bucket, err := s3.OpenBucket(*g_bucketName, s3.Region(*g_region), wrongKey)
	AssertEq(nil, err)

	// Attempt to do something.
	_, err = bucket.ListKeys("")
	ExpectThat(err, Error(HasSubstr("signature")))
}

func (t *BucketTest) InvalidUtf8Keys() {
	ExpectEq("TODO", "")
}

func (t *BucketTest) LongKeys() {
	ExpectEq("TODO", "")
}

func (t *BucketTest) NullBytesInKeys() {
	ExpectEq("TODO", "")
}

func (t *BucketTest) NonGraphicalCharacterInKeys() {
	ExpectEq("TODO", "")
}

func (t *BucketTest) EmptyKeys() {
	ExpectEq("TODO", "")
}

func (t *BucketTest) GetNonExistentObject() {
	_, err := t.bucket.GetObject("some_key")

	ExpectThat(err, Error(HasSubstr("404")))
	ExpectThat(err, Error(HasSubstr("some_key")))
	ExpectThat(err, Error(HasSubstr("exist")))
}

func (t *BucketTest) StoreThenGetEmptyObject() {
	key := "some_key"
	t.ensureDeleted(key)

	data := []byte{}

	// Store
	err := t.bucket.StoreObject(key, data)
	AssertEq(nil, err)

	// Get
	returnedData, err := t.bucket.GetObject(key)
	AssertEq(nil, err)
	ExpectThat(returnedData, DeepEquals(data))
}

func (t *BucketTest) StoreThenGetNonEmptyObject() {
	key := "some_key"
	t.ensureDeleted(key)

	data := []byte{0x17, 0x19, 0x00, 0x02, 0x03}

	// Store
	err := t.bucket.StoreObject(key, data)
	AssertEq(nil, err)

	// Get
	returnedData, err := t.bucket.GetObject(key)
	AssertEq(nil, err)
	ExpectThat(returnedData, DeepEquals(data))
}

func (t *BucketTest) ListEmptyBucket() {
	var keys []string
	var err error

	// From start.
	keys, err = t.bucket.ListKeys("")
	AssertEq(nil, err)
	ExpectThat(keys, ElementsAre())

	// From middle.
	keys, err = t.bucket.ListKeys("foo")
	AssertEq(nil, err)
	ExpectThat(keys, ElementsAre())
}

func (t *BucketTest) ListWithInvalidUtf8Minimum() {
	ExpectFalse(true, "TODO")
}

func (t *BucketTest) ListWithLongMinimum() {
	ExpectFalse(true, "TODO")
}

func (t *BucketTest) ListWithNullByteInMinimum() {
	ExpectFalse(true, "TODO")
}

func (t *BucketTest) ListFewKeys() {
	var keys []string
	var err error

	// Create several keys. S3 returns keys in an XML 1.0 document, and according
	// to Section 2.2 of the spec the smallest legal character is #x9, so a
	// string's successor in that space of strings is computed by appending \x09.
	//
	// S3 will actually allow you to create a smaller key, e.g. "bar\x01", but
	// Go's xml package will then refuse to parse its LIST responses.
	toCreate := []string{
		"foo",
		"bar",
		"bar\x09",
		"bar\x09\x09",
		"baz",
	}

	err = runForRange(len(toCreate), func(i int) error {
		key := toCreate[i]
		t.ensureDeleted(key)
		return t.bucket.StoreObject(key, []byte{})
	})

	AssertEq(nil, err)

	// From start.
	keys, err = t.bucket.ListKeys("")
	AssertEq(nil, err)
	ExpectThat(
		keys,
		ElementsAre(
		"bar",
		"bar\x09",
		"bar\x09\x09",
		"baz",
		"foo",
	))

	// Just before bar\x09.
	keys, err = t.bucket.ListKeys("bar")
	AssertEq(nil, err)
	ExpectThat(
		keys,
		ElementsAre(
		"bar\x09",
		"bar\x09\x09",
		"baz",
		"foo",
	))

	// At bar\x09.
	keys, err = t.bucket.ListKeys("bar\x09")
	AssertEq(nil, err)
	ExpectThat(
		keys,
		ElementsAre(
		"bar\x09\x09",
		"baz",
		"foo",
	))

	// Just after bar\x09.
	keys, err = t.bucket.ListKeys("bar\x09\x09")
	AssertEq(nil, err)
	ExpectThat(
		keys,
		ElementsAre(
		"baz",
		"foo",
	))

	// At last key.
	keys, err = t.bucket.ListKeys("foo")
	AssertEq(nil, err)
	ExpectThat(keys, ElementsAre())

	// Just after last key.
	keys, err = t.bucket.ListKeys("foo\x09")
	AssertEq(nil, err)
	ExpectThat(keys, ElementsAre())

	// Well after last key.
	keys, err = t.bucket.ListKeys("qux")
	AssertEq(nil, err)
	ExpectThat(keys, ElementsAre())
}

func (t *BucketTest) ListManyKeys() {
	var err error

	// Decide on many keys.
	const numKeys = 3072
	allKeys := make([]string, numKeys)

	for i, _ := range allKeys {
		allKeys[i] = fmt.Sprintf("%08x", i)
	}

	// Create them.
	err = runForRange(numKeys, func(i int) error {
		key := allKeys[i]
		t.ensureDeleted(key)
		return t.bucket.StoreObject(key, []byte{})
	})

	AssertEq(nil, err)

	// List them progressively.
	lb := ""
	keysListed := []string{}

	for {
		keys, err := t.bucket.ListKeys(lb)

		AssertEq(nil, err)
		AssertLt(len(keys), numKeys)

		if len(keys) == 0 {
			break
		}

		keysListed = append(keysListed, keys...)
		lb = keys[len(keys) - 1]
	}

	// We should have gotten them all back.
	ExpectThat(keysListed, DeepEquals(allKeys))
}

func (t *BucketTest) KeysWithSpecialCharacters() {
	ExpectFalse(true, "TODO")
}

func (t *BucketTest) DeleteNonExistentObject() {
	ExpectFalse(true, "TODO")
}

func (t *BucketTest) DeleteThenListAndGetObject() {
	ExpectFalse(true, "TODO")
}
<commit_msg>Adjusted numbers.<commit_after>// Copyright 2012 Aaron Jacobs. All Rights Reserved.
// Author: aaronjjacobs@gmail.com (Aaron Jacobs)
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
//
// An integration test that uses a real S3 account. Run as follows:
//
//     go run integration_test/*.go \
//         -key_id <key ID> \
//         -bucket <bucket> \
//         -region s3-ap-northeast-1.amazonaws.com
//
// Before doing this, create an empty bucket (or delete the contents of an
// existing bucket) using the S3 management console.

package main

import (
	"fmt"
	"github.com/jacobsa/aws/s3"
	. "github.com/jacobsa/oglematchers"
	. "github.com/jacobsa/ogletest"
	"sync"
)

////////////////////////////////////////////////////////////////////////
// Helpers
////////////////////////////////////////////////////////////////////////

// Run the supplied function for every integer in [0, n) with some degree of
// parallelism, returning an error if any invocation returns an error.
func runForRange(n int, f func (int) error) (err error) {
		// Set up channels. The work channel should be buffered so that we don't
		// have to block writing to it before checking for errors below. The error
		// channel must be buffered so that no worker goroutine gets stuck writing
		// a result to it and never returns. The stop channel must not be buffered
		// so that we can be sure that no more work will be processed when we
		// return below.
		work := make(chan int, n)
		errs := make(chan error, n)
		stop := make(chan bool)

		// Launch worker functions that attempt to do work, returning if a read
		// from the stop channel succeeds.
		processWork := func() {
			for {
				select {
				case i := <-work:
					errs<- f(i)
				case <-stop:
					return
				}
			}
		}

		const numWorkers = 128
		for i := 0; i < numWorkers; i++ {
			go processWork()
		}

		// Feed the workers work.
		for i := 0; i < n; i++ {
			work<- i
		}

		// Read results, stopping immediately if there is an error.
		for i := 0; i < n; i++ {
			err = <-errs
			if err != nil {
				break
			}
		}

		// Stop all of the workers, and wait for them to stop. This ensures that
		// no piece of work is in progress when we return.
		for i := 0; i < numWorkers; i++ {
			stop<- true
		}

		return
	}

type BucketTest struct {
	bucket s3.Bucket

	mutex sync.Mutex
	keysToDelete []string
}

func init() { RegisterTestSuite(&BucketTest{}) }

// Ensure that the given key is deleted before the test finishes.
func (t *BucketTest) ensureDeleted(key string) {
	t.mutex.Lock()
	defer t.mutex.Unlock()

	t.keysToDelete = append(t.keysToDelete, key)
}

func (t *BucketTest) SetUp(i *TestInfo) {
	var err error

	// Open a bucket.
	t.bucket, err = s3.OpenBucket(*g_bucketName, s3.Region(*g_region), g_accessKey)
	AssertEq(nil, err)
}

func (t *BucketTest) TearDown() {
	t.mutex.Lock()
	defer t.mutex.Unlock()

	err := runForRange(len(t.keysToDelete), func (i int) error {
		key := t.keysToDelete[i]
		if err := t.bucket.DeleteObject(key); err != nil {
			return fmt.Errorf("Couldn't delete key %s: %v", key, err)
		}

		return nil
	})

	AssertEq(nil, err)
}

////////////////////////////////////////////////////////////////////////
// Tests
////////////////////////////////////////////////////////////////////////

func (t *BucketTest) WrongAccessKeySecret() {
	// Open a bucket with the wrong key.
	wrongKey := g_accessKey
	wrongKey.Secret += "taco"

	bucket, err := s3.OpenBucket(*g_bucketName, s3.Region(*g_region), wrongKey)
	AssertEq(nil, err)

	// Attempt to do something.
	_, err = bucket.ListKeys("")
	ExpectThat(err, Error(HasSubstr("signature")))
}

func (t *BucketTest) InvalidUtf8Keys() {
	ExpectEq("TODO", "")
}

func (t *BucketTest) LongKeys() {
	ExpectEq("TODO", "")
}

func (t *BucketTest) NullBytesInKeys() {
	ExpectEq("TODO", "")
}

func (t *BucketTest) NonGraphicalCharacterInKeys() {
	ExpectEq("TODO", "")
}

func (t *BucketTest) EmptyKeys() {
	ExpectEq("TODO", "")
}

func (t *BucketTest) GetNonExistentObject() {
	_, err := t.bucket.GetObject("some_key")

	ExpectThat(err, Error(HasSubstr("404")))
	ExpectThat(err, Error(HasSubstr("some_key")))
	ExpectThat(err, Error(HasSubstr("exist")))
}

func (t *BucketTest) StoreThenGetEmptyObject() {
	key := "some_key"
	t.ensureDeleted(key)

	data := []byte{}

	// Store
	err := t.bucket.StoreObject(key, data)
	AssertEq(nil, err)

	// Get
	returnedData, err := t.bucket.GetObject(key)
	AssertEq(nil, err)
	ExpectThat(returnedData, DeepEquals(data))
}

func (t *BucketTest) StoreThenGetNonEmptyObject() {
	key := "some_key"
	t.ensureDeleted(key)

	data := []byte{0x17, 0x19, 0x00, 0x02, 0x03}

	// Store
	err := t.bucket.StoreObject(key, data)
	AssertEq(nil, err)

	// Get
	returnedData, err := t.bucket.GetObject(key)
	AssertEq(nil, err)
	ExpectThat(returnedData, DeepEquals(data))
}

func (t *BucketTest) ListEmptyBucket() {
	var keys []string
	var err error

	// From start.
	keys, err = t.bucket.ListKeys("")
	AssertEq(nil, err)
	ExpectThat(keys, ElementsAre())

	// From middle.
	keys, err = t.bucket.ListKeys("foo")
	AssertEq(nil, err)
	ExpectThat(keys, ElementsAre())
}

func (t *BucketTest) ListWithInvalidUtf8Minimum() {
	ExpectFalse(true, "TODO")
}

func (t *BucketTest) ListWithLongMinimum() {
	ExpectFalse(true, "TODO")
}

func (t *BucketTest) ListWithNullByteInMinimum() {
	ExpectFalse(true, "TODO")
}

func (t *BucketTest) ListFewKeys() {
	var keys []string
	var err error

	// Create several keys. S3 returns keys in an XML 1.0 document, and according
	// to Section 2.2 of the spec the smallest legal character is #x9, so a
	// string's successor in that space of strings is computed by appending \x09.
	//
	// S3 will actually allow you to create a smaller key, e.g. "bar\x01", but
	// Go's xml package will then refuse to parse its LIST responses.
	toCreate := []string{
		"foo",
		"bar",
		"bar\x09",
		"bar\x09\x09",
		"baz",
	}

	err = runForRange(len(toCreate), func(i int) error {
		key := toCreate[i]
		t.ensureDeleted(key)
		return t.bucket.StoreObject(key, []byte{})
	})

	AssertEq(nil, err)

	// From start.
	keys, err = t.bucket.ListKeys("")
	AssertEq(nil, err)
	ExpectThat(
		keys,
		ElementsAre(
		"bar",
		"bar\x09",
		"bar\x09\x09",
		"baz",
		"foo",
	))

	// Just before bar\x09.
	keys, err = t.bucket.ListKeys("bar")
	AssertEq(nil, err)
	ExpectThat(
		keys,
		ElementsAre(
		"bar\x09",
		"bar\x09\x09",
		"baz",
		"foo",
	))

	// At bar\x09.
	keys, err = t.bucket.ListKeys("bar\x09")
	AssertEq(nil, err)
	ExpectThat(
		keys,
		ElementsAre(
		"bar\x09\x09",
		"baz",
		"foo",
	))

	// Just after bar\x09.
	keys, err = t.bucket.ListKeys("bar\x09\x09")
	AssertEq(nil, err)
	ExpectThat(
		keys,
		ElementsAre(
		"baz",
		"foo",
	))

	// At last key.
	keys, err = t.bucket.ListKeys("foo")
	AssertEq(nil, err)
	ExpectThat(keys, ElementsAre())

	// Just after last key.
	keys, err = t.bucket.ListKeys("foo\x09")
	AssertEq(nil, err)
	ExpectThat(keys, ElementsAre())

	// Well after last key.
	keys, err = t.bucket.ListKeys("qux")
	AssertEq(nil, err)
	ExpectThat(keys, ElementsAre())
}

func (t *BucketTest) ListManyKeys() {
	var err error

	// Decide on many keys.
	const numKeys = 1001
	allKeys := make([]string, numKeys)

	for i, _ := range allKeys {
		allKeys[i] = fmt.Sprintf("%08x", i)
	}

	// Create them.
	err = runForRange(numKeys, func(i int) error {
		key := allKeys[i]
		t.ensureDeleted(key)
		return t.bucket.StoreObject(key, []byte{})
	})

	AssertEq(nil, err)

	// List them progressively.
	lb := ""
	keysListed := []string{}

	for {
		keys, err := t.bucket.ListKeys(lb)

		AssertEq(nil, err)
		AssertLt(len(keys), numKeys)

		if len(keys) == 0 {
			break
		}

		keysListed = append(keysListed, keys...)
		lb = keys[len(keys) - 1]
	}

	// We should have gotten them all back.
	ExpectThat(keysListed, DeepEquals(allKeys))
}

func (t *BucketTest) KeysWithSpecialCharacters() {
	ExpectFalse(true, "TODO")
}

func (t *BucketTest) DeleteNonExistentObject() {
	ExpectFalse(true, "TODO")
}

func (t *BucketTest) DeleteThenListAndGetObject() {
	ExpectFalse(true, "TODO")
}
<|endoftext|>


<commit_before>/*
Copyright 2018 The Kubernetes Authors.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package storage

import (
	"github.com/onsi/ginkgo/v2"
	v1 "k8s.io/api/core/v1"
	clientset "k8s.io/client-go/kubernetes"
	v1helper "k8s.io/kubernetes/pkg/apis/core/v1/helper"
	"k8s.io/kubernetes/test/e2e/framework"
	e2enode "k8s.io/kubernetes/test/e2e/framework/node"
	e2eskipper "k8s.io/kubernetes/test/e2e/framework/skipper"
	"k8s.io/kubernetes/test/e2e/storage/utils"
	admissionapi "k8s.io/pod-security-admission/api"
)

var _ = utils.SIGDescribe("Volume limits", func() {
	var (
		c clientset.Interface
	)
	f := framework.NewDefaultFramework("volume-limits-on-node")
	f.NamespacePodSecurityEnforceLevel = admissionapi.LevelPrivileged
	ginkgo.BeforeEach(func() {
		e2eskipper.SkipUnlessProviderIs("aws", "gce", "gke")
		c = f.ClientSet
		framework.ExpectNoError(e2enode.WaitForAllNodesSchedulable(c, framework.TestContext.NodeSchedulableTimeout))
	})

	ginkgo.It("should verify that all nodes have volume limits", func() {
		nodeList, err := e2enode.GetReadySchedulableNodes(f.ClientSet)
		framework.ExpectNoError(err)
		for _, node := range nodeList.Items {
			volumeLimits := getVolumeLimit(&node)
			if len(volumeLimits) == 0 {
				framework.Failf("Expected volume limits to be set")
			}
		}
	})
})

func getVolumeLimit(node *v1.Node) map[v1.ResourceName]int64 {
	volumeLimits := map[v1.ResourceName]int64{}
	nodeAllocatables := node.Status.Allocatable
	for k, v := range nodeAllocatables {
		if v1helper.IsAttachableVolumeResourceName(k) {
			volumeLimits[k] = v.Value()
		}
	}
	return volumeLimits
}
<commit_msg>remove in-tree volume limits test now that CSIMigration is GA<commit_after><|endoftext|>