<commit_before>// Copyright 2014 The rkt Authors
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

// Package keystore implements the ACI keystore.
package keystore

import (
	"bytes"
	"errors"
	"fmt"
	"io"
	"io/ioutil"
	"os"
	"path"
	"path/filepath"
	"strings"

	"github.com/coreos/rkt/Godeps/_workspace/src/github.com/appc/spec/schema/types"
	"github.com/coreos/rkt/Godeps/_workspace/src/golang.org/x/crypto/openpgp"
)

// A Config structure is used to configure a Keystore.
type Config struct {
	LocalRootPath    string
	LocalPrefixPath  string
	SystemRootPath   string
	SystemPrefixPath string
}

// A Keystore represents a repository of trusted public keys which can be
// used to verify PGP signatures.
type Keystore struct {
	*Config
}

// New returns a new Keystore based on config.
func New(config *Config) *Keystore {
	return &Keystore{config}
}

func NewConfig(systemPath, localPath string) *Config {
	return &Config{
		LocalRootPath:    filepath.Join(localPath, "trustedkeys", "root.d"),
		LocalPrefixPath:  filepath.Join(localPath, "trustedkeys", "prefix.d"),
		SystemRootPath:   filepath.Join(systemPath, "trustedkeys", "root.d"),
		SystemPrefixPath: filepath.Join(systemPath, "trustedkeys", "prefix.d"),
	}
}

// CheckSignature takes a signed file and a detached signature and returns the signer
// if the signature is signed by a trusted signer.
// If the signer is unknown or not trusted, opengpg.ErrUnknownIssuer is returned.
func (ks *Keystore) CheckSignature(prefix string, signed, signature io.ReadSeeker) (*openpgp.Entity, error) {
	return checkSignature(ks, prefix, signed, signature)
}

func checkSignature(ks *Keystore, prefix string, signed, signature io.ReadSeeker) (*openpgp.Entity, error) {
	acidentifier, err := types.NewACIdentifier(prefix)
	if err != nil {
		return nil, err
	}
	keyring, err := ks.loadKeyring(acidentifier.String())
	if err != nil {
		return nil, fmt.Errorf("keystore: error loading keyring %v", err)
	}
	entities, err := openpgp.CheckArmoredDetachedSignature(keyring, signed, signature)
	if err == io.EOF {
		// When the signature is binary instead of armored, the error is io.EOF.
		// Let's try with binary signatures as well
		if _, err := signed.Seek(0, 0); err != nil {
			return nil, fmt.Errorf("error seeking ACI file: %v", err)
		}
		if _, err := signature.Seek(0, 0); err != nil {
			return nil, fmt.Errorf("error seeking signature file: %v", err)
		}
		entities, err = openpgp.CheckDetachedSignature(keyring, signed, signature)
	}
	if err == io.EOF {
		// otherwise, the client failure is just "EOF", which is not helpful
		return nil, fmt.Errorf("keystore: no valid signatures found in signature file")
	}
	return entities, err
}

// DeleteTrustedKeyPrefix deletes the prefix trusted key identified by fingerprint.
func (ks *Keystore) DeleteTrustedKeyPrefix(prefix, fingerprint string) error {
	acidentifier, err := types.NewACIdentifier(prefix)
	if err != nil {
		return err
	}
	return os.Remove(path.Join(ks.LocalPrefixPath, acidentifier.String(), fingerprint))
}

// MaskTrustedKeySystemPrefix masks the system prefix trusted key identified by fingerprint.
func (ks *Keystore) MaskTrustedKeySystemPrefix(prefix, fingerprint string) (string, error) {
	acidentifier, err := types.NewACIdentifier(prefix)
	if err != nil {
		return "", err
	}
	dst := path.Join(ks.LocalPrefixPath, acidentifier.String(), fingerprint)
	return dst, ioutil.WriteFile(dst, []byte(""), 0644)
}

// DeleteTrustedKeyRoot deletes the root trusted key identified by fingerprint.
func (ks *Keystore) DeleteTrustedKeyRoot(fingerprint string) error {
	return os.Remove(path.Join(ks.LocalRootPath, fingerprint))
}

// MaskTrustedKeySystemRoot masks the system root trusted key identified by fingerprint.
func (ks *Keystore) MaskTrustedKeySystemRoot(fingerprint string) (string, error) {
	dst := path.Join(ks.LocalRootPath, fingerprint)
	return dst, ioutil.WriteFile(dst, []byte(""), 0644)
}

// StoreTrustedKeyPrefix stores the contents of public key r as a prefix trusted key.
func (ks *Keystore) StoreTrustedKeyPrefix(prefix string, r io.Reader) (string, error) {
	acidentifier, err := types.NewACIdentifier(prefix)
	if err != nil {
		return "", err
	}
	return storeTrustedKey(path.Join(ks.LocalPrefixPath, acidentifier.String()), r)
}

// StoreTrustedKeyRoot stores the contents of public key r as a root trusted key.
func (ks *Keystore) StoreTrustedKeyRoot(r io.Reader) (string, error) {
	return storeTrustedKey(ks.LocalRootPath, r)
}

func storeTrustedKey(dir string, r io.Reader) (string, error) {
	pubkeyBytes, err := ioutil.ReadAll(r)
	if err != nil {
		return "", err
	}
	if err := os.MkdirAll(dir, 0755); err != nil {
		return "", err
	}
	entityList, err := openpgp.ReadArmoredKeyRing(bytes.NewReader(pubkeyBytes))
	if err != nil {
		return "", err
	}
	pubKey := entityList[0].PrimaryKey
	trustedKeyPath := path.Join(dir, fingerprintToFilename(pubKey.Fingerprint))
	if err := ioutil.WriteFile(trustedKeyPath, pubkeyBytes, 0644); err != nil {
		return "", err
	}
	return trustedKeyPath, nil
}

func entityFromFile(path string) (*openpgp.Entity, error) {
	trustedKey, err := os.Open(path)
	if err != nil {
		return nil, err
	}
	defer trustedKey.Close()
	entityList, err := openpgp.ReadArmoredKeyRing(trustedKey)
	if err != nil {
		return nil, err
	}
	if len(entityList) < 1 {
		return nil, errors.New("missing opengpg entity")
	}
	fingerprint := fingerprintToFilename(entityList[0].PrimaryKey.Fingerprint)
	keyFile := filepath.Base(trustedKey.Name())
	if fingerprint != keyFile {
		return nil, fmt.Errorf("fingerprint mismatch: %q:%q", keyFile, fingerprint)
	}
	return entityList[0], nil
}

func (ks *Keystore) loadKeyring(prefix string) (openpgp.KeyRing, error) {
	acidentifier, err := types.NewACIdentifier(prefix)
	if err != nil {
		return nil, err
	}
	var keyring openpgp.EntityList
	trustedKeys := make(map[string]*openpgp.Entity)

	prefixRoot := strings.Split(acidentifier.String(), "/")[0]
	paths := []struct {
		root     string
		fullPath string
	}{
		{ks.SystemRootPath, ks.SystemRootPath},
		{ks.LocalRootPath, ks.LocalRootPath},
		{path.Join(ks.SystemPrefixPath, prefixRoot), path.Join(ks.SystemPrefixPath, acidentifier.String())},
		{path.Join(ks.LocalPrefixPath, prefixRoot), path.Join(ks.LocalPrefixPath, acidentifier.String())},
	}
	for _, p := range paths {
		err := filepath.Walk(p.root, func(path string, info os.FileInfo, err error) error {
			if err != nil && !os.IsNotExist(err) {
				return err
			}
			if info == nil {
				return nil
			}
			if info.IsDir() {
				switch {
				case strings.HasPrefix(p.fullPath, path):
					return nil
				default:
					return filepath.SkipDir
				}
			}
			// Remove trust for default keys.
			if info.Size() == 0 {
				delete(trustedKeys, info.Name())
				return nil
			}
			entity, err := entityFromFile(path)
			if err != nil {
				return err
			}
			trustedKeys[fingerprintToFilename(entity.PrimaryKey.Fingerprint)] = entity
			return nil
		})
		if err != nil {
			return nil, err
		}
	}

	for _, v := range trustedKeys {
		keyring = append(keyring, v)
	}
	return keyring, nil
}

func fingerprintToFilename(fp [20]byte) string {
	return fmt.Sprintf("%x", fp)
}

// NewTestKeystore creates a new KeyStore backed by a temp directory.
// NewTestKeystore returns a KeyStore, the path to the temp directory, and
// an error if any.
func NewTestKeystore() (*Keystore, string, error) {
	// The system and local config directories defined here are only for
	// the tests so they can be arbitrary. For simplicity, we use the same
	// values as common.DefaultSystemConfigDir and
	// common.DefaultLocalConfigDir.
	const TestSystemConfigDir = "/usr/lib/rkt"
	const TestLocalConfigDir = "/etc/rkt"

	dir, err := ioutil.TempDir("", "keystore-test")
	if err != nil {
		return nil, "", err
	}
	systemDir := filepath.Join(dir, TestSystemConfigDir)
	localDir := filepath.Join(dir, TestLocalConfigDir)
	c := NewConfig(systemDir, localDir)
	for _, path := range []string{c.LocalRootPath, c.SystemRootPath, c.LocalPrefixPath, c.SystemPrefixPath} {
		if err := os.MkdirAll(path, 0755); err != nil {
			return nil, "", err
		}
	}
	return New(c), dir, nil
}
<commit_msg>keys: revert changes in pkg/keystore/keystore.go<commit_after>// Copyright 2014 The rkt Authors
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

// Package keystore implements the ACI keystore.
package keystore

import (
	"bytes"
	"errors"
	"fmt"
	"io"
	"io/ioutil"
	"os"
	"path"
	"path/filepath"
	"strings"

	"github.com/coreos/rkt/Godeps/_workspace/src/github.com/appc/spec/schema/types"
	"github.com/coreos/rkt/Godeps/_workspace/src/golang.org/x/crypto/openpgp"
	"github.com/coreos/rkt/common"
)

// A Config structure is used to configure a Keystore.
type Config struct {
	LocalRootPath    string
	LocalPrefixPath  string
	SystemRootPath   string
	SystemPrefixPath string
}

// A Keystore represents a repository of trusted public keys which can be
// used to verify PGP signatures.
type Keystore struct {
	*Config
}

// New returns a new Keystore based on config.
func New(config *Config) *Keystore {
	if config == nil {
		config = defaultConfig
	}
	return &Keystore{config}
}

func NewConfig(systemPath, localPath string) *Config {
	return &Config{
		LocalRootPath:    filepath.Join(localPath, "trustedkeys", "root.d"),
		LocalPrefixPath:  filepath.Join(localPath, "trustedkeys", "prefix.d"),
		SystemRootPath:   filepath.Join(systemPath, "trustedkeys", "root.d"),
		SystemPrefixPath: filepath.Join(systemPath, "trustedkeys", "prefix.d"),
	}
}

var defaultConfig = NewConfig(common.DefaultSystemConfigDir, common.DefaultLocalConfigDir)

// CheckSignature is a convenience method for creating a Keystore with a default
// configuration and invoking CheckSignature.
func CheckSignature(prefix string, signed, signature io.ReadSeeker) (*openpgp.Entity, error) {
	ks := New(defaultConfig)
	return checkSignature(ks, prefix, signed, signature)
}

// CheckSignature takes a signed file and a detached signature and returns the signer
// if the signature is signed by a trusted signer.
// If the signer is unknown or not trusted, opengpg.ErrUnknownIssuer is returned.
func (ks *Keystore) CheckSignature(prefix string, signed, signature io.ReadSeeker) (*openpgp.Entity, error) {
	return checkSignature(ks, prefix, signed, signature)
}

func checkSignature(ks *Keystore, prefix string, signed, signature io.ReadSeeker) (*openpgp.Entity, error) {
	acidentifier, err := types.NewACIdentifier(prefix)
	if err != nil {
		return nil, err
	}
	keyring, err := ks.loadKeyring(acidentifier.String())
	if err != nil {
		return nil, fmt.Errorf("keystore: error loading keyring %v", err)
	}
	entities, err := openpgp.CheckArmoredDetachedSignature(keyring, signed, signature)
	if err == io.EOF {
		// When the signature is binary instead of armored, the error is io.EOF.
		// Let's try with binary signatures as well
		if _, err := signed.Seek(0, 0); err != nil {
			return nil, fmt.Errorf("error seeking ACI file: %v", err)
		}
		if _, err := signature.Seek(0, 0); err != nil {
			return nil, fmt.Errorf("error seeking signature file: %v", err)
		}
		entities, err = openpgp.CheckDetachedSignature(keyring, signed, signature)
	}
	if err == io.EOF {
		// otherwise, the client failure is just "EOF", which is not helpful
		return nil, fmt.Errorf("keystore: no valid signatures found in signature file")
	}
	return entities, err
}

// DeleteTrustedKeyPrefix deletes the prefix trusted key identified by fingerprint.
func (ks *Keystore) DeleteTrustedKeyPrefix(prefix, fingerprint string) error {
	acidentifier, err := types.NewACIdentifier(prefix)
	if err != nil {
		return err
	}
	return os.Remove(path.Join(ks.LocalPrefixPath, acidentifier.String(), fingerprint))
}

// MaskTrustedKeySystemPrefix masks the system prefix trusted key identified by fingerprint.
func (ks *Keystore) MaskTrustedKeySystemPrefix(prefix, fingerprint string) (string, error) {
	acidentifier, err := types.NewACIdentifier(prefix)
	if err != nil {
		return "", err
	}
	dst := path.Join(ks.LocalPrefixPath, acidentifier.String(), fingerprint)
	return dst, ioutil.WriteFile(dst, []byte(""), 0644)
}

// DeleteTrustedKeyRoot deletes the root trusted key identified by fingerprint.
func (ks *Keystore) DeleteTrustedKeyRoot(fingerprint string) error {
	return os.Remove(path.Join(ks.LocalRootPath, fingerprint))
}

// MaskTrustedKeySystemRoot masks the system root trusted key identified by fingerprint.
func (ks *Keystore) MaskTrustedKeySystemRoot(fingerprint string) (string, error) {
	dst := path.Join(ks.LocalRootPath, fingerprint)
	return dst, ioutil.WriteFile(dst, []byte(""), 0644)
}

// StoreTrustedKeyPrefix stores the contents of public key r as a prefix trusted key.
func (ks *Keystore) StoreTrustedKeyPrefix(prefix string, r io.Reader) (string, error) {
	acidentifier, err := types.NewACIdentifier(prefix)
	if err != nil {
		return "", err
	}
	return storeTrustedKey(path.Join(ks.LocalPrefixPath, acidentifier.String()), r)
}

// StoreTrustedKeyRoot stores the contents of public key r as a root trusted key.
func (ks *Keystore) StoreTrustedKeyRoot(r io.Reader) (string, error) {
	return storeTrustedKey(ks.LocalRootPath, r)
}

func storeTrustedKey(dir string, r io.Reader) (string, error) {
	pubkeyBytes, err := ioutil.ReadAll(r)
	if err != nil {
		return "", err
	}
	if err := os.MkdirAll(dir, 0755); err != nil {
		return "", err
	}
	entityList, err := openpgp.ReadArmoredKeyRing(bytes.NewReader(pubkeyBytes))
	if err != nil {
		return "", err
	}
	pubKey := entityList[0].PrimaryKey
	trustedKeyPath := path.Join(dir, fingerprintToFilename(pubKey.Fingerprint))
	if err := ioutil.WriteFile(trustedKeyPath, pubkeyBytes, 0644); err != nil {
		return "", err
	}
	return trustedKeyPath, nil
}

func entityFromFile(path string) (*openpgp.Entity, error) {
	trustedKey, err := os.Open(path)
	if err != nil {
		return nil, err
	}
	defer trustedKey.Close()
	entityList, err := openpgp.ReadArmoredKeyRing(trustedKey)
	if err != nil {
		return nil, err
	}
	if len(entityList) < 1 {
		return nil, errors.New("missing opengpg entity")
	}
	fingerprint := fingerprintToFilename(entityList[0].PrimaryKey.Fingerprint)
	keyFile := filepath.Base(trustedKey.Name())
	if fingerprint != keyFile {
		return nil, fmt.Errorf("fingerprint mismatch: %q:%q", keyFile, fingerprint)
	}
	return entityList[0], nil
}

func (ks *Keystore) loadKeyring(prefix string) (openpgp.KeyRing, error) {
	acidentifier, err := types.NewACIdentifier(prefix)
	if err != nil {
		return nil, err
	}
	var keyring openpgp.EntityList
	trustedKeys := make(map[string]*openpgp.Entity)

	prefixRoot := strings.Split(acidentifier.String(), "/")[0]
	paths := []struct {
		root     string
		fullPath string
	}{
		{ks.SystemRootPath, ks.SystemRootPath},
		{ks.LocalRootPath, ks.LocalRootPath},
		{path.Join(ks.SystemPrefixPath, prefixRoot), path.Join(ks.SystemPrefixPath, acidentifier.String())},
		{path.Join(ks.LocalPrefixPath, prefixRoot), path.Join(ks.LocalPrefixPath, acidentifier.String())},
	}
	for _, p := range paths {
		err := filepath.Walk(p.root, func(path string, info os.FileInfo, err error) error {
			if err != nil && !os.IsNotExist(err) {
				return err
			}
			if info == nil {
				return nil
			}
			if info.IsDir() {
				switch {
				case strings.HasPrefix(p.fullPath, path):
					return nil
				default:
					return filepath.SkipDir
				}
			}
			// Remove trust for default keys.
			if info.Size() == 0 {
				delete(trustedKeys, info.Name())
				return nil
			}
			entity, err := entityFromFile(path)
			if err != nil {
				return err
			}
			trustedKeys[fingerprintToFilename(entity.PrimaryKey.Fingerprint)] = entity
			return nil
		})
		if err != nil {
			return nil, err
		}
	}

	for _, v := range trustedKeys {
		keyring = append(keyring, v)
	}
	return keyring, nil
}

func fingerprintToFilename(fp [20]byte) string {
	return fmt.Sprintf("%x", fp)
}

// NewTestKeystore creates a new KeyStore backed by a temp directory.
// NewTestKeystore returns a KeyStore, the path to the temp directory, and
// an error if any.
func NewTestKeystore() (*Keystore, string, error) {
	dir, err := ioutil.TempDir("", "keystore-test")
	if err != nil {
		return nil, "", err
	}
	systemDir := filepath.Join(dir, common.DefaultSystemConfigDir)
	localDir := filepath.Join(dir, common.DefaultLocalConfigDir)
	c := NewConfig(systemDir, localDir)
	for _, path := range []string{c.LocalRootPath, c.SystemRootPath, c.LocalPrefixPath, c.SystemPrefixPath} {
		if err := os.MkdirAll(path, 0755); err != nil {
			return nil, "", err
		}
	}
	return New(c), dir, nil
}
<|endoftext|>


<commit_before>// Copyright 2016-2019 Authors of Cilium
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

// +build !privileged_tests

package kvstore

import (
	"context"
	"fmt"
	"time"

	"github.com/cilium/cilium/pkg/checker"

	. "gopkg.in/check.v1"
)

// BaseTests is the struct that needs to be embedded into all test suite
// structs of backend implementations. It contains all test functions that are
// agnostic to the backend implementation.
type BaseTests struct{}

func (s *BaseTests) TestLock(c *C) {
	prefix := "locktest/"

	DeletePrefix(prefix)
	defer DeletePrefix(prefix)

	for i := 0; i < 10; i++ {
		lock, err := LockPath(context.Background(), fmt.Sprintf("%sfoo/%d", prefix, i))
		c.Assert(err, IsNil)
		c.Assert(lock, Not(IsNil))
		lock.Unlock()
	}
}

func testKey(prefix string, i int) string {
	return fmt.Sprintf("%s%s/%010d", prefix, "foo", i)
}

func testValue(i int) []byte {
	return []byte(fmt.Sprintf("blah %d blah %d", i, i))
}

func (s *BaseTests) TestGetSet(c *C) {
	prefix := "unit-test/"
	maxID := 256

	DeletePrefix(prefix)
	defer DeletePrefix(prefix)

	key, val, err := GetPrefix(context.Background(), prefix)
	c.Assert(err, IsNil)
	c.Assert(val, IsNil)
	c.Assert(key, Equals, "")

	for i := 0; i < maxID; i++ {
		val, err = Get(testKey(prefix, i))
		c.Assert(err, IsNil)
		c.Assert(val, IsNil)

		key, val, err = GetPrefix(context.Background(), testKey(prefix, i))
		c.Assert(err, IsNil)
		c.Assert(val, IsNil)
		c.Assert(key, Equals, "")

		c.Assert(Set(testKey(prefix, i), testValue(i)), IsNil)

		val, err = Get(testKey(prefix, i))
		c.Assert(err, IsNil)
		c.Assert(val, checker.DeepEquals, testValue(i))

		val, err = Get(testKey(prefix, i))
		c.Assert(err, IsNil)
		c.Assert(val, checker.DeepEquals, testValue(i))
	}

	for i := 0; i < maxID; i++ {
		c.Assert(Delete(testKey(prefix, i)), IsNil)

		val, err = Get(testKey(prefix, i))
		c.Assert(err, IsNil)
		c.Assert(val, IsNil)

		key, val, err = GetPrefix(context.Background(), testKey(prefix, i))
		c.Assert(err, IsNil)
		c.Assert(val, IsNil)
		c.Assert(key, Equals, "")
	}

	key, val, err = GetPrefix(context.Background(), prefix)
	c.Assert(err, IsNil)
	c.Assert(val, IsNil)
	c.Assert(key, Equals, "")
}

func (s *BaseTests) BenchmarkGet(c *C) {
	prefix := "unit-test/"
	DeletePrefix(prefix)
	defer DeletePrefix(prefix)

	key := testKey(prefix, 1)
	c.Assert(Set(key, testValue(100)), IsNil)

	c.ResetTimer()
	for i := 0; i < c.N; i++ {
		Get(key)
	}
}

func (s *BaseTests) BenchmarkSet(c *C) {
	prefix := "unit-test/"
	DeletePrefix(prefix)
	defer DeletePrefix(prefix)

	key, val := testKey(prefix, 1), testValue(100)
	c.ResetTimer()
	for i := 0; i < c.N; i++ {
		Set(key, val)
	}
}

func (s *BaseTests) TestUpdate(c *C) {
	prefix := "unit-test/"

	DeletePrefix(prefix)
	defer DeletePrefix(prefix)

	key, val, err := GetPrefix(context.Background(), prefix)
	c.Assert(err, IsNil)
	c.Assert(val, IsNil)
	c.Assert(key, Equals, "")

	// create
	c.Assert(Update(context.Background(), testKey(prefix, 0), testValue(0), true), IsNil)

	val, err = Get(testKey(prefix, 0))
	c.Assert(err, IsNil)
	c.Assert(val, checker.DeepEquals, testValue(0))

	// update
	c.Assert(Update(context.Background(), testKey(prefix, 0), testValue(0), true), IsNil)

	val, err = Get(testKey(prefix, 0))
	c.Assert(err, IsNil)
	c.Assert(val, checker.DeepEquals, testValue(0))
}

func (s *BaseTests) TestCreateOnly(c *C) {
	prefix := "unit-test/"

	DeletePrefix(prefix)
	defer DeletePrefix(prefix)

	key, val, err := GetPrefix(context.Background(), prefix)
	c.Assert(err, IsNil)
	c.Assert(val, IsNil)
	c.Assert(key, Equals, "")

	c.Assert(CreateOnly(context.Background(), testKey(prefix, 0), testValue(0), false), IsNil)

	val, err = Get(testKey(prefix, 0))
	c.Assert(err, IsNil)
	c.Assert(val, checker.DeepEquals, testValue(0))

	c.Assert(CreateOnly(context.Background(), testKey(prefix, 0), testValue(1), false), Not(IsNil))

	val, err = Get(testKey(prefix, 0))
	c.Assert(err, IsNil)
	c.Assert(val, checker.DeepEquals, testValue(0))

	// key 1 does not exist so CreateIfExists should fail
	c.Assert(CreateIfExists(testKey(prefix, 1), testKey(prefix, 2), testValue(2), false), Not(IsNil))

	val, err = Get(testKey(prefix, 2))
	c.Assert(err, IsNil)
	c.Assert(val, IsNil)

	// key 0 exists so CreateIfExists should succeed
	c.Assert(CreateIfExists(testKey(prefix, 0), testKey(prefix, 2), testValue(2), false), IsNil)

	val, err = Get(testKey(prefix, 2))
	c.Assert(err, IsNil)
	c.Assert(val, checker.DeepEquals, testValue(2))
}

func drainEvents(w *Watcher) {
	for len(w.Events) > 0 {
		<-w.Events
	}
}

func expectEvent(c *C, w *Watcher, typ EventType, key string, val []byte) {
	select {
	case event := <-w.Events:
		c.Assert(event.Typ, Equals, typ)

		if event.Typ != EventTypeListDone {
			c.Assert(event.Key, checker.DeepEquals, key)

			// etcd does not provide the value of deleted keys
			if selectedModule == "consul" {
				c.Assert(event.Value, checker.DeepEquals, val)
			}
		}
	case <-time.After(10 * time.Second):
		c.Fatal("timeout while waiting for kvstore watcher event")
	}
}

func (s *BaseTests) TestListAndWatch(c *C) {
	key1, key2 := "foo2/key1", "foo2/key2"
	val1, val2 := []byte("val1"), []byte("val2")

	DeletePrefix("foo2/")
	defer DeletePrefix("foo2/")

	err := CreateOnly(context.Background(), key1, val1, false)
	c.Assert(err, IsNil)

	w := ListAndWatch("testWatcher2", "foo2/", 100)
	c.Assert(c, Not(IsNil))

	expectEvent(c, w, EventTypeCreate, key1, val1)
	expectEvent(c, w, EventTypeListDone, "", []byte{})

	err = CreateOnly(context.Background(), key2, val2, false)
	c.Assert(err, IsNil)
	expectEvent(c, w, EventTypeCreate, key2, val2)

	err = Delete(key1)
	c.Assert(err, IsNil)
	expectEvent(c, w, EventTypeDelete, key1, val1)

	err = CreateOnly(context.Background(), key1, val1, false)
	c.Assert(err, IsNil)
	expectEvent(c, w, EventTypeCreate, key1, val1)

	err = Delete(key1)
	c.Assert(err, IsNil)
	expectEvent(c, w, EventTypeDelete, key1, val1)

	err = Delete(key2)
	c.Assert(err, IsNil)
	expectEvent(c, w, EventTypeDelete, key2, val2)

	w.Stop()
}
<commit_msg>kvstore: Add test for GetPrefix()<commit_after>// Copyright 2016-2019 Authors of Cilium
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

// +build !privileged_tests

package kvstore

import (
	"context"
	"fmt"
	"time"

	"github.com/cilium/cilium/pkg/checker"

	. "gopkg.in/check.v1"
)

// BaseTests is the struct that needs to be embedded into all test suite
// structs of backend implementations. It contains all test functions that are
// agnostic to the backend implementation.
type BaseTests struct{}

func (s *BaseTests) TestLock(c *C) {
	prefix := "locktest/"

	DeletePrefix(prefix)
	defer DeletePrefix(prefix)

	for i := 0; i < 10; i++ {
		lock, err := LockPath(context.Background(), fmt.Sprintf("%sfoo/%d", prefix, i))
		c.Assert(err, IsNil)
		c.Assert(lock, Not(IsNil))
		lock.Unlock()
	}
}

func testKey(prefix string, i int) string {
	return fmt.Sprintf("%s%s/%010d", prefix, "foo", i)
}

func testValue(i int) []byte {
	return []byte(fmt.Sprintf("blah %d blah %d", i, i))
}

func (s *BaseTests) TestGetSet(c *C) {
	prefix := "unit-test/"
	maxID := 256

	DeletePrefix(prefix)
	defer DeletePrefix(prefix)

	key, val, err := GetPrefix(context.Background(), prefix)
	c.Assert(err, IsNil)
	c.Assert(val, IsNil)
	c.Assert(key, Equals, "")

	for i := 0; i < maxID; i++ {
		val, err = Get(testKey(prefix, i))
		c.Assert(err, IsNil)
		c.Assert(val, IsNil)

		key, val, err = GetPrefix(context.Background(), testKey(prefix, i))
		c.Assert(err, IsNil)
		c.Assert(val, IsNil)
		c.Assert(key, Equals, "")

		c.Assert(Set(testKey(prefix, i), testValue(i)), IsNil)

		val, err = Get(testKey(prefix, i))
		c.Assert(err, IsNil)
		c.Assert(val, checker.DeepEquals, testValue(i))

		val, err = Get(testKey(prefix, i))
		c.Assert(err, IsNil)
		c.Assert(val, checker.DeepEquals, testValue(i))
	}

	for i := 0; i < maxID; i++ {
		c.Assert(Delete(testKey(prefix, i)), IsNil)

		val, err = Get(testKey(prefix, i))
		c.Assert(err, IsNil)
		c.Assert(val, IsNil)

		key, val, err = GetPrefix(context.Background(), testKey(prefix, i))
		c.Assert(err, IsNil)
		c.Assert(val, IsNil)
		c.Assert(key, Equals, "")
	}

	key, val, err = GetPrefix(context.Background(), prefix)
	c.Assert(err, IsNil)
	c.Assert(val, IsNil)
	c.Assert(key, Equals, "")
}

func (s *BaseTests) TestGetPrefix(c *C) {
	prefix := "unit-test/"

	DeletePrefix(prefix)
	defer DeletePrefix(prefix)

	key, val, err := GetPrefix(context.Background(), prefix)
	c.Assert(err, IsNil)
	c.Assert(val, IsNil)
	c.Assert(key, Equals, "")

	// create
	labelsLong := "foo;/;bar;"
	labelsShort := "foo;/"
	testKey := fmt.Sprintf("%s%s/%010d", prefix, labelsLong, 0)
	c.Assert(Update(context.Background(), testKey, testValue(0), true), IsNil)

	val, err = Get(testKey)
	c.Assert(err, IsNil)
	c.Assert(val, checker.DeepEquals, testValue(0))

	prefixes := []string{
		prefix,
		fmt.Sprintf("%s%s", prefix, labelsLong),
		fmt.Sprintf("%s%s", prefix, labelsShort),
	}
	for _, p := range prefixes {
		key, val, err = GetPrefix(context.Background(), p)
		c.Assert(err, IsNil)
		c.Assert(val, checker.DeepEquals, testValue(0))
		c.Assert(key, Equals, testKey)
	}
}

func (s *BaseTests) BenchmarkGet(c *C) {
	prefix := "unit-test/"
	DeletePrefix(prefix)
	defer DeletePrefix(prefix)

	key := testKey(prefix, 1)
	c.Assert(Set(key, testValue(100)), IsNil)

	c.ResetTimer()
	for i := 0; i < c.N; i++ {
		Get(key)
	}
}

func (s *BaseTests) BenchmarkSet(c *C) {
	prefix := "unit-test/"
	DeletePrefix(prefix)
	defer DeletePrefix(prefix)

	key, val := testKey(prefix, 1), testValue(100)
	c.ResetTimer()
	for i := 0; i < c.N; i++ {
		Set(key, val)
	}
}

func (s *BaseTests) TestUpdate(c *C) {
	prefix := "unit-test/"

	DeletePrefix(prefix)
	defer DeletePrefix(prefix)

	key, val, err := GetPrefix(context.Background(), prefix)
	c.Assert(err, IsNil)
	c.Assert(val, IsNil)
	c.Assert(key, Equals, "")

	// create
	c.Assert(Update(context.Background(), testKey(prefix, 0), testValue(0), true), IsNil)

	val, err = Get(testKey(prefix, 0))
	c.Assert(err, IsNil)
	c.Assert(val, checker.DeepEquals, testValue(0))

	// update
	c.Assert(Update(context.Background(), testKey(prefix, 0), testValue(0), true), IsNil)

	val, err = Get(testKey(prefix, 0))
	c.Assert(err, IsNil)
	c.Assert(val, checker.DeepEquals, testValue(0))
}

func (s *BaseTests) TestCreateOnly(c *C) {
	prefix := "unit-test/"

	DeletePrefix(prefix)
	defer DeletePrefix(prefix)

	key, val, err := GetPrefix(context.Background(), prefix)
	c.Assert(err, IsNil)
	c.Assert(val, IsNil)
	c.Assert(key, Equals, "")

	c.Assert(CreateOnly(context.Background(), testKey(prefix, 0), testValue(0), false), IsNil)

	val, err = Get(testKey(prefix, 0))
	c.Assert(err, IsNil)
	c.Assert(val, checker.DeepEquals, testValue(0))

	c.Assert(CreateOnly(context.Background(), testKey(prefix, 0), testValue(1), false), Not(IsNil))

	val, err = Get(testKey(prefix, 0))
	c.Assert(err, IsNil)
	c.Assert(val, checker.DeepEquals, testValue(0))

	// key 1 does not exist so CreateIfExists should fail
	c.Assert(CreateIfExists(testKey(prefix, 1), testKey(prefix, 2), testValue(2), false), Not(IsNil))

	val, err = Get(testKey(prefix, 2))
	c.Assert(err, IsNil)
	c.Assert(val, IsNil)

	// key 0 exists so CreateIfExists should succeed
	c.Assert(CreateIfExists(testKey(prefix, 0), testKey(prefix, 2), testValue(2), false), IsNil)

	val, err = Get(testKey(prefix, 2))
	c.Assert(err, IsNil)
	c.Assert(val, checker.DeepEquals, testValue(2))
}

func drainEvents(w *Watcher) {
	for len(w.Events) > 0 {
		<-w.Events
	}
}

func expectEvent(c *C, w *Watcher, typ EventType, key string, val []byte) {
	select {
	case event := <-w.Events:
		c.Assert(event.Typ, Equals, typ)

		if event.Typ != EventTypeListDone {
			c.Assert(event.Key, checker.DeepEquals, key)

			// etcd does not provide the value of deleted keys
			if selectedModule == "consul" {
				c.Assert(event.Value, checker.DeepEquals, val)
			}
		}
	case <-time.After(10 * time.Second):
		c.Fatal("timeout while waiting for kvstore watcher event")
	}
}

func (s *BaseTests) TestListAndWatch(c *C) {
	key1, key2 := "foo2/key1", "foo2/key2"
	val1, val2 := []byte("val1"), []byte("val2")

	DeletePrefix("foo2/")
	defer DeletePrefix("foo2/")

	err := CreateOnly(context.Background(), key1, val1, false)
	c.Assert(err, IsNil)

	w := ListAndWatch("testWatcher2", "foo2/", 100)
	c.Assert(c, Not(IsNil))

	expectEvent(c, w, EventTypeCreate, key1, val1)
	expectEvent(c, w, EventTypeListDone, "", []byte{})

	err = CreateOnly(context.Background(), key2, val2, false)
	c.Assert(err, IsNil)
	expectEvent(c, w, EventTypeCreate, key2, val2)

	err = Delete(key1)
	c.Assert(err, IsNil)
	expectEvent(c, w, EventTypeDelete, key1, val1)

	err = CreateOnly(context.Background(), key1, val1, false)
	c.Assert(err, IsNil)
	expectEvent(c, w, EventTypeCreate, key1, val1)

	err = Delete(key1)
	c.Assert(err, IsNil)
	expectEvent(c, w, EventTypeDelete, key1, val1)

	err = Delete(key2)
	c.Assert(err, IsNil)
	expectEvent(c, w, EventTypeDelete, key2, val2)

	w.Stop()
}
<|endoftext|>


<commit_before>package models

import (
	"errors"
	"time"

	"github.com/grafana/grafana/pkg/components/securejsondata"
	"github.com/grafana/grafana/pkg/components/simplejson"
)

const (
	DS_GRAPHITE      = "graphite"
	DS_INFLUXDB      = "influxdb"
	DS_INFLUXDB_08   = "influxdb_08"
	DS_ES            = "elasticsearch"
	DS_OPENTSDB      = "opentsdb"
	DS_CLOUDWATCH    = "cloudwatch"
	DS_KAIROSDB      = "kairosdb"
	DS_PROMETHEUS    = "prometheus"
	DS_POSTGRES      = "postgres"
	DS_MYSQL         = "mysql"
	DS_MSSQL         = "mssql"
	DS_ACCESS_DIRECT = "direct"
	DS_ACCESS_PROXY  = "proxy"
)

var (
	ErrDataSourceNotFound           = errors.New("Data source not found")
	ErrDataSourceNameExists         = errors.New("Data source with same name already exists")
	ErrDataSourceUpdatingOldVersion = errors.New("Trying to update old version of datasource")
	ErrDatasourceIsReadOnly         = errors.New("Data source is readonly. Can only be updated from configuration.")
)

type DsAccess string

type DataSource struct {
	Id      int64
	OrgId   int64
	Version int

	Name              string
	Type              string
	Access            DsAccess
	Url               string
	Password          string
	User              string
	Database          string
	BasicAuth         bool
	BasicAuthUser     string
	BasicAuthPassword string
	WithCredentials   bool
	IsDefault         bool
	JsonData          *simplejson.Json
	SecureJsonData    securejsondata.SecureJsonData
	ReadOnly          bool

	Created time.Time
	Updated time.Time
}

var knownDatasourcePlugins = map[string]bool{
	DS_ES:                       true,
	DS_GRAPHITE:                 true,
	DS_INFLUXDB:                 true,
	DS_INFLUXDB_08:              true,
	DS_KAIROSDB:                 true,
	DS_CLOUDWATCH:               true,
	DS_PROMETHEUS:               true,
	DS_OPENTSDB:                 true,
	DS_POSTGRES:                 true,
	DS_MYSQL:                    true,
	DS_MSSQL:                    true,
	"opennms":                   true,
	"abhisant-druid-datasource": true,
	"dalmatinerdb-datasource":   true,
	"gnocci":                    true,
	"zabbix":                    true,
	"alexanderzobnin-zabbix-datasource":   true,
	"newrelic-app":                        true,
	"grafana-datadog-datasource":          true,
	"grafana-simple-json":                 true,
	"grafana-splunk-datasource":           true,
	"udoprog-heroic-datasource":           true,
	"grafana-openfalcon-datasource":       true,
	"opennms-datasource":                  true,
	"rackerlabs-blueflood-datasource":     true,
	"crate-datasource":                    true,
	"ayoungprogrammer-finance-datasource": true,
	"monasca-datasource":                  true,
	"vertamedia-clickhouse-datasource":    true,
}

func IsKnownDataSourcePlugin(dsType string) bool {
	_, exists := knownDatasourcePlugins[dsType]
	return exists
}

// ----------------------
// COMMANDS

// Also acts as api DTO
type AddDataSourceCommand struct {
	Name              string            `json:"name" binding:"Required"`
	Type              string            `json:"type" binding:"Required"`
	Access            DsAccess          `json:"access" binding:"Required"`
	Url               string            `json:"url"`
	Password          string            `json:"password"`
	Database          string            `json:"database"`
	User              string            `json:"user"`
	BasicAuth         bool              `json:"basicAuth"`
	BasicAuthUser     string            `json:"basicAuthUser"`
	BasicAuthPassword string            `json:"basicAuthPassword"`
	WithCredentials   bool              `json:"withCredentials"`
	IsDefault         bool              `json:"isDefault"`
	JsonData          *simplejson.Json  `json:"jsonData"`
	SecureJsonData    map[string]string `json:"secureJsonData"`
	ReadOnly          bool              `json:"readOnly"`

	OrgId int64 `json:"-"`

	Result *DataSource
}

// Also acts as api DTO
type UpdateDataSourceCommand struct {
	Name              string            `json:"name" binding:"Required"`
	Type              string            `json:"type" binding:"Required"`
	Access            DsAccess          `json:"access" binding:"Required"`
	Url               string            `json:"url"`
	Password          string            `json:"password"`
	User              string            `json:"user"`
	Database          string            `json:"database"`
	BasicAuth         bool              `json:"basicAuth"`
	BasicAuthUser     string            `json:"basicAuthUser"`
	BasicAuthPassword string            `json:"basicAuthPassword"`
	WithCredentials   bool              `json:"withCredentials"`
	IsDefault         bool              `json:"isDefault"`
	JsonData          *simplejson.Json  `json:"jsonData"`
	SecureJsonData    map[string]string `json:"secureJsonData"`
	Version           int               `json:"version"`
	ReadOnly          bool              `json:"readOnly"`

	OrgId int64 `json:"-"`
	Id    int64 `json:"-"`

	Result *DataSource
}

type DeleteDataSourceByIdCommand struct {
	Id    int64
	OrgId int64

	DeletedDatasourcesCount int64
}

type DeleteDataSourceByNameCommand struct {
	Name  string
	OrgId int64

	DeletedDatasourcesCount int64
}

// ---------------------
// QUERIES

type GetDataSourcesQuery struct {
	OrgId  int64
	Result []*DataSource
}

type GetAllDataSourcesQuery struct {
	Result []*DataSource
}

type GetDataSourceByIdQuery struct {
	Id     int64
	OrgId  int64
	Result *DataSource
}

type GetDataSourceByNameQuery struct {
	Name   string
	OrgId  int64
	Result *DataSource
}

// ---------------------
// EVENTS
type DataSourceCreatedEvent struct {
}
<commit_msg>go fmt fixes<commit_after>package models

import (
	"errors"
	"time"

	"github.com/grafana/grafana/pkg/components/securejsondata"
	"github.com/grafana/grafana/pkg/components/simplejson"
)

const (
	DS_GRAPHITE      = "graphite"
	DS_INFLUXDB      = "influxdb"
	DS_INFLUXDB_08   = "influxdb_08"
	DS_ES            = "elasticsearch"
	DS_OPENTSDB      = "opentsdb"
	DS_CLOUDWATCH    = "cloudwatch"
	DS_KAIROSDB      = "kairosdb"
	DS_PROMETHEUS    = "prometheus"
	DS_POSTGRES      = "postgres"
	DS_MYSQL         = "mysql"
	DS_MSSQL         = "mssql"
	DS_ACCESS_DIRECT = "direct"
	DS_ACCESS_PROXY  = "proxy"
)

var (
	ErrDataSourceNotFound           = errors.New("Data source not found")
	ErrDataSourceNameExists         = errors.New("Data source with same name already exists")
	ErrDataSourceUpdatingOldVersion = errors.New("Trying to update old version of datasource")
	ErrDatasourceIsReadOnly         = errors.New("Data source is readonly. Can only be updated from configuration.")
)

type DsAccess string

type DataSource struct {
	Id      int64
	OrgId   int64
	Version int

	Name              string
	Type              string
	Access            DsAccess
	Url               string
	Password          string
	User              string
	Database          string
	BasicAuth         bool
	BasicAuthUser     string
	BasicAuthPassword string
	WithCredentials   bool
	IsDefault         bool
	JsonData          *simplejson.Json
	SecureJsonData    securejsondata.SecureJsonData
	ReadOnly          bool

	Created time.Time
	Updated time.Time
}

var knownDatasourcePlugins = map[string]bool{
	DS_ES:                                 true,
	DS_GRAPHITE:                           true,
	DS_INFLUXDB:                           true,
	DS_INFLUXDB_08:                        true,
	DS_KAIROSDB:                           true,
	DS_CLOUDWATCH:                         true,
	DS_PROMETHEUS:                         true,
	DS_OPENTSDB:                           true,
	DS_POSTGRES:                           true,
	DS_MYSQL:                              true,
	DS_MSSQL:                              true,
	"opennms":                             true,
	"abhisant-druid-datasource":           true,
	"dalmatinerdb-datasource":             true,
	"gnocci":                              true,
	"zabbix":                              true,
	"alexanderzobnin-zabbix-datasource":   true,
	"newrelic-app":                        true,
	"grafana-datadog-datasource":          true,
	"grafana-simple-json":                 true,
	"grafana-splunk-datasource":           true,
	"udoprog-heroic-datasource":           true,
	"grafana-openfalcon-datasource":       true,
	"opennms-datasource":                  true,
	"rackerlabs-blueflood-datasource":     true,
	"crate-datasource":                    true,
	"ayoungprogrammer-finance-datasource": true,
	"monasca-datasource":                  true,
	"vertamedia-clickhouse-datasource":    true,
}

func IsKnownDataSourcePlugin(dsType string) bool {
	_, exists := knownDatasourcePlugins[dsType]
	return exists
}

// ----------------------
// COMMANDS

// Also acts as api DTO
type AddDataSourceCommand struct {
	Name              string            `json:"name" binding:"Required"`
	Type              string            `json:"type" binding:"Required"`
	Access            DsAccess          `json:"access" binding:"Required"`
	Url               string            `json:"url"`
	Password          string            `json:"password"`
	Database          string            `json:"database"`
	User              string            `json:"user"`
	BasicAuth         bool              `json:"basicAuth"`
	BasicAuthUser     string            `json:"basicAuthUser"`
	BasicAuthPassword string            `json:"basicAuthPassword"`
	WithCredentials   bool              `json:"withCredentials"`
	IsDefault         bool              `json:"isDefault"`
	JsonData          *simplejson.Json  `json:"jsonData"`
	SecureJsonData    map[string]string `json:"secureJsonData"`
	ReadOnly          bool              `json:"readOnly"`

	OrgId int64 `json:"-"`

	Result *DataSource
}

// Also acts as api DTO
type UpdateDataSourceCommand struct {
	Name              string            `json:"name" binding:"Required"`
	Type              string            `json:"type" binding:"Required"`
	Access            DsAccess          `json:"access" binding:"Required"`
	Url               string            `json:"url"`
	Password          string            `json:"password"`
	User              string            `json:"user"`
	Database          string            `json:"database"`
	BasicAuth         bool              `json:"basicAuth"`
	BasicAuthUser     string            `json:"basicAuthUser"`
	BasicAuthPassword string            `json:"basicAuthPassword"`
	WithCredentials   bool              `json:"withCredentials"`
	IsDefault         bool              `json:"isDefault"`
	JsonData          *simplejson.Json  `json:"jsonData"`
	SecureJsonData    map[string]string `json:"secureJsonData"`
	Version           int               `json:"version"`
	ReadOnly          bool              `json:"readOnly"`

	OrgId int64 `json:"-"`
	Id    int64 `json:"-"`

	Result *DataSource
}

type DeleteDataSourceByIdCommand struct {
	Id    int64
	OrgId int64

	DeletedDatasourcesCount int64
}

type DeleteDataSourceByNameCommand struct {
	Name  string
	OrgId int64

	DeletedDatasourcesCount int64
}

// ---------------------
// QUERIES

type GetDataSourcesQuery struct {
	OrgId  int64
	Result []*DataSource
}

type GetAllDataSourcesQuery struct {
	Result []*DataSource
}

type GetDataSourceByIdQuery struct {
	Id     int64
	OrgId  int64
	Result *DataSource
}

type GetDataSourceByNameQuery struct {
	Name   string
	OrgId  int64
	Result *DataSource
}

// ---------------------
// EVENTS
type DataSourceCreatedEvent struct {
}
<|endoftext|>


<commit_before>package main

import (
//	"io"
//	"io/ioutil"
//	"os"
	"flag"

	"github.com/dropbox/rust-brotli/c/go/brotli"
)

var useWriter = flag.Bool("use_writer", false, "should use writer for brotli")
var parallelism = flag.Int("workers", 16, "number of workers to run with")
var threads = flag.Int("threads", 2, "number of threads to compress the file")
var iters = flag.Int("iter", 1024, "number of times to run the compression/decompression cycle")
var size = flag.Int("size", 4096 * 1024, "Size of the test data")
var quality = flag.Int("quality", 5, "brotli quality level")

func main() {
	flag.Parse()
	options := brotli.CompressionOptions{
		NumThreads: *threads,
		Quality:    5,
		Catable:    true,
		Appendable: true,
		Magic:      true,
	}
	_ = useWriter
    _ = options
}
<commit_msg>first iter of the stress test with some threading options<commit_after>package main

import (
	"syscall"
	//	"io/ioutil"
	"bytes"
	//	"os"
	"compress/zlib"
	"crypto/md5"
	"io"
	"flag"
	"sync"

	"github.com/dropbox/rust-brotli/c/go/brotli"
)

var useWriter = flag.Bool("use_writer", false, "should use writer for brotli")
var parallelism = flag.Int("workers", 16, "number of workers to run with")
var threads = flag.Int("threads", 2, "number of threads to compress the file")
var iters = flag.Int("iter", 1024 * 1024, "number of times to run the compression/decompression cycle")
var sizeHint = flag.Uint64("size", 4096 * 1024, "Size of the test data")
var quality = flag.Float64("quality", 5, "brotli quality level")
var useZlib = flag.Int("zlib", 0, "1 for zlib 2 for both zlib and brotli")
var nongraceful = flag.Bool("hardexit", false, "use syscall.ExitGroup to end the program")
var anongoroutines = flag.Bool("anon", false, "use a separate anonymous goroutine for each invocation")
func main() {
	flag.Parse()
	options := brotli.CompressionOptions{
		NumThreads: *threads,
		Quality:    float32(*quality),
		Catable:    true,
		Appendable: true,
		Magic:      true,
		SizeHint:   uint32(*sizeHint),
	}
	file := testData(int(options.SizeHint))
	if *anongoroutines {
		var wg sync.WaitGroup
		wg.Add(*parallelism)
		for par :=0; par < *parallelism; par += 1 {
			go func() {
				for iter := 0; iter < *iters; iter += 1 {
					_ = stresst(options, file, *useZlib % 2 == 0)
					if *useZlib == 2 {
						_ = stresst(options, file, false)
					}
				}
				if *nongraceful {
					syscall.Exit(0)
				}
				wg.Done()
			}()
		}
		wg.Wait()
	} else {
		for iter := 0; iter < *iters; iter += 1 {
			var wg sync.WaitGroup
			wg.Add(*parallelism)
			for par :=0; par < *parallelism; par += 1 {
				go func() {
					_ = stresst(options, file, *useZlib % 2 == 0)
					if *useZlib == 2 {
						_ = stresst(options, file, false)
					}
					wg.Done()
				}()
			}
			wg.Wait()
		}
		if *nongraceful {
			syscall.Exit(0)
		}
	}
}

func stresst(
	options brotli.CompressionOptions,
	file []byte,
	useBrotli bool,
) (
	md5out [md5.Size]byte,
) {
	var output bytes.Buffer
	input := bytes.NewBuffer(file)
	var compressionWriter io.WriteCloser
	var err error
	if useBrotli {
		compressionWriter = brotli.NewMultiCompressionWriter(
			&output, options)
	} else {
		compressionWriter = zlib.NewWriter(&output)
	}
	if err != nil {
		panic(err)
	}
	_, err = io.Copy(compressionWriter, input)
	if err != nil {
		panic(err)
	}
	err = compressionWriter.Close()
	if err != nil {
		panic(err)
	}
	md5out = md5.Sum(output.Bytes())
	compressed := bytes.NewBuffer(output.Bytes())
	var compressionReader io.ReadCloser
	if useBrotli {
		compressionReader = brotli.NewDecompressionReader(compressed)
	} else {
		compressionReader, err = zlib.NewReader(compressed)
	}
	if err != nil {
		panic(err)
	}
	var rt bytes.Buffer
	_, err = io.Copy(&rt, compressionReader)
	if err != nil {
		panic(err)
	}
	err = compressionReader.Close()
	if err != nil {
		panic(err)
	}
	if !bytes.Equal(rt.Bytes(), file) {
		panic ("Files differ " + string(rt.Bytes()))
	}
	return
}
<|endoftext|>


<commit_before>package main

import (
    zmq "github.com/alecthomas/gozmq"
    "log"
    "github.com/howeyc/fsnotify"
    "os"
)

func main() {
    // Create and bind socket
    context, _ := zmq.NewContext()
    socket, _ := context.NewSocket(zmq.PUB)
    defer context.Close()
    defer socket.Close()
    socket.Bind("tcp://*:5556")
    socket.Bind("ipc://weather.ipc")

    // Open file and seek to end
    FILE := "/var/log/system.log"
    log_file, err := os.Open(FILE)

    if err != nil {
        println(err)
    }

    stat, _ := os.Stat(FILE)
    size := stat.Size()

    log_file.Seek(0, 2)

    if err != nil {
        log.Fatal(err)
    }

    watcher, err := fsnotify.NewWatcher()

    done := make(chan bool)

    println("Watching file for changes...")
    // Process events
    go func() {
        for {
            ev := <-watcher.Event
            if (ev.IsModify()) {

                // Create a buffer for reading the new data
                stat, _ = os.Stat(FILE)
                new_size := stat.Size()
                bytes := make([]byte, new_size - size)

                if (new_size - size > 0) {
                    _, err := log_file.Read(bytes)

                    if err != nil {
                        log.Fatal(err)
                    }

                    err = socket.Send(bytes, 0)

                    if err != nil {
                       log.Fatal(err)
                    }
                }
                size = new_size
            }
        }
    }()

    err = watcher.Watch(FILE)
    if err != nil {
        log.Fatal(err)
    }

    <-done

    watcher.Close()
}
<commit_msg>what the hell is this<commit_after>package main

import (
    zmq "github.com/alecthomas/gozmq"
    "log"
    "github.com/howeyc/fsnotify"
    "os"
)

func main() {
    // Create and bind socket
    context, _ := zmq.NewContext()
    socket, _ := context.NewSocket(zmq.PUB)
    defer context.Close()
    defer socket.Close()
    socket.Bind("tcp://*:5556")

    // Open file and seek to end
    FILE := "/var/log/system.log"
    log_file, err := os.Open(FILE)

    if err != nil {
        println(err)
    }

    stat, _ := os.Stat(FILE)
    size := stat.Size()

    log_file.Seek(0, 2)

    if err != nil {
        log.Fatal(err)
    }

    watcher, err := fsnotify.NewWatcher()

    done := make(chan bool)

    println("Watching file for changes...")
    // Process events
    go func() {
        for {
            ev := <-watcher.Event
            if (ev.IsModify()) {

                // Create a buffer for reading the new data
                stat, _ = os.Stat(FILE)
                new_size := stat.Size()
                bytes := make([]byte, new_size - size)

                if (new_size - size > 0) {
                    _, err := log_file.Read(bytes)

                    if err != nil {
                        log.Fatal(err)
                    }

                    err = socket.Send(bytes, 0)

                    if err != nil {
                       log.Fatal(err)
                    }
                }
                size = new_size
            }
        }
    }()

    err = watcher.Watch(FILE)
    if err != nil {
        log.Fatal(err)
    }

    <-done

    watcher.Close()
}
<|endoftext|>


<commit_before>// Copyright 2016 CodisLabs. All Rights Reserved.
// Licensed under the MIT (MIT-LICENSE.txt) license.

package topom

import (
	"time"

	"github.com/CodisLabs/codis/pkg/models"
	"github.com/CodisLabs/codis/pkg/proxy"
	"github.com/CodisLabs/codis/pkg/utils/log"
	"github.com/CodisLabs/codis/pkg/utils/rpc"
	"github.com/CodisLabs/codis/pkg/utils/sync2"
)

type RedisStats struct {
	Stats map[string]string `json:"stats,omitempty"`
	Error *rpc.RemoteError  `json:"error,omitempty"`

	UnixTime int64 `json:"unixtime"`
	Timeout  bool  `json:"timeout,omitempty"`
}

func (s *Topom) newRedisStats(addr string, timeout time.Duration) *RedisStats {
	var ch = make(chan struct{})
	stats := &RedisStats{}

	go func() {
		defer close(ch)
		x, err := s.redisp.Info(addr)
		if err != nil {
			stats.Error = rpc.NewRemoteError(err)
		} else {
			stats.Stats = x
		}
	}()

	select {
	case <-ch:
		return stats
	case <-time.After(timeout):
		return &RedisStats{Timeout: true}
	}
}

func (s *Topom) RefreshRedisStats(timeout time.Duration) (*sync2.Future, error) {
	s.mu.Lock()
	defer s.mu.Unlock()
	ctx, err := s.newContext()
	if err != nil {
		return nil, err
	}
	var fut sync2.Future
	for _, g := range ctx.group {
		for _, x := range g.Servers {
			fut.Add()
			go func(addr string) {
				stats := s.newRedisStats(addr, timeout)
				stats.UnixTime = time.Now().Unix()
				fut.Done(addr, stats)
			}(x.Addr)
		}
	}
	go func() {
		stats := make(map[string]*RedisStats)
		for k, v := range fut.Wait() {
			stats[k] = v.(*RedisStats)
		}
		s.mu.Lock()
		defer s.mu.Unlock()
		s.stats.servers = stats
	}()
	return &fut, nil
}

type ProxyStats struct {
	Stats *proxy.Stats     `json:"stats,omitempty"`
	Error *rpc.RemoteError `json:"error,omitempty"`

	UnixTime int64 `json:"unixtime"`
	Timeout  bool  `json:"timeout,omitempty"`
}

func (s *Topom) newProxyStats(p *models.Proxy, timeout time.Duration) *ProxyStats {
	var ch = make(chan struct{})
	stats := &ProxyStats{}

	go func() {
		defer close(ch)
		x, err := s.newProxyClient(p).Stats()
		if err != nil {
			stats.Error = rpc.NewRemoteError(err)
		} else {
			stats.Stats = x
		}
	}()

	select {
	case <-ch:
		return stats
	case <-time.After(timeout):
		return &ProxyStats{Timeout: true}
	}
}

func (s *Topom) RefreshProxyStats(timeout time.Duration) (*sync2.Future, error) {
	s.mu.Lock()
	defer s.mu.Unlock()
	ctx, err := s.newContext()
	if err != nil {
		return nil, err
	}
	var fut sync2.Future
	for _, p := range ctx.proxy {
		fut.Add()
		go func(p *models.Proxy) {
			stats := s.newProxyStats(p, timeout)
			stats.UnixTime = time.Now().Unix()
			fut.Done(p.Token, stats)

			switch x := stats.Stats; {
			case x == nil:
			case x.Closed || x.Online:
			default:
				if err := s.ReinitProxy(p.Token); err != nil {
					log.WarnErrorf(err, "auto reinit proxy-[%s] failed", p.Token)
				}
			}
		}(p)
	}
	go func() {
		stats := make(map[string]*ProxyStats)
		for k, v := range fut.Wait() {
			stats[k] = v.(*ProxyStats)
		}
		s.mu.Lock()
		defer s.mu.Unlock()
		s.stats.proxies = stats
	}()
	return &fut, nil
}
<commit_msg>topom: fix, call OnlineProxy instead of ReinitProxy in topom_stats.go<commit_after>// Copyright 2016 CodisLabs. All Rights Reserved.
// Licensed under the MIT (MIT-LICENSE.txt) license.

package topom

import (
	"time"

	"github.com/CodisLabs/codis/pkg/models"
	"github.com/CodisLabs/codis/pkg/proxy"
	"github.com/CodisLabs/codis/pkg/utils/log"
	"github.com/CodisLabs/codis/pkg/utils/rpc"
	"github.com/CodisLabs/codis/pkg/utils/sync2"
)

type RedisStats struct {
	Stats map[string]string `json:"stats,omitempty"`
	Error *rpc.RemoteError  `json:"error,omitempty"`

	UnixTime int64 `json:"unixtime"`
	Timeout  bool  `json:"timeout,omitempty"`
}

func (s *Topom) newRedisStats(addr string, timeout time.Duration) *RedisStats {
	var ch = make(chan struct{})
	stats := &RedisStats{}

	go func() {
		defer close(ch)
		x, err := s.redisp.Info(addr)
		if err != nil {
			stats.Error = rpc.NewRemoteError(err)
		} else {
			stats.Stats = x
		}
	}()

	select {
	case <-ch:
		return stats
	case <-time.After(timeout):
		return &RedisStats{Timeout: true}
	}
}

func (s *Topom) RefreshRedisStats(timeout time.Duration) (*sync2.Future, error) {
	s.mu.Lock()
	defer s.mu.Unlock()
	ctx, err := s.newContext()
	if err != nil {
		return nil, err
	}
	var fut sync2.Future
	for _, g := range ctx.group {
		for _, x := range g.Servers {
			fut.Add()
			go func(addr string) {
				stats := s.newRedisStats(addr, timeout)
				stats.UnixTime = time.Now().Unix()
				fut.Done(addr, stats)
			}(x.Addr)
		}
	}
	go func() {
		stats := make(map[string]*RedisStats)
		for k, v := range fut.Wait() {
			stats[k] = v.(*RedisStats)
		}
		s.mu.Lock()
		defer s.mu.Unlock()
		s.stats.servers = stats
	}()
	return &fut, nil
}

type ProxyStats struct {
	Stats *proxy.Stats     `json:"stats,omitempty"`
	Error *rpc.RemoteError `json:"error,omitempty"`

	UnixTime int64 `json:"unixtime"`
	Timeout  bool  `json:"timeout,omitempty"`
}

func (s *Topom) newProxyStats(p *models.Proxy, timeout time.Duration) *ProxyStats {
	var ch = make(chan struct{})
	stats := &ProxyStats{}

	go func() {
		defer close(ch)
		x, err := s.newProxyClient(p).Stats()
		if err != nil {
			stats.Error = rpc.NewRemoteError(err)
		} else {
			stats.Stats = x
		}
	}()

	select {
	case <-ch:
		return stats
	case <-time.After(timeout):
		return &ProxyStats{Timeout: true}
	}
}

func (s *Topom) RefreshProxyStats(timeout time.Duration) (*sync2.Future, error) {
	s.mu.Lock()
	defer s.mu.Unlock()
	ctx, err := s.newContext()
	if err != nil {
		return nil, err
	}
	var fut sync2.Future
	for _, p := range ctx.proxy {
		fut.Add()
		go func(p *models.Proxy) {
			stats := s.newProxyStats(p, timeout)
			stats.UnixTime = time.Now().Unix()
			fut.Done(p.Token, stats)

			switch x := stats.Stats; {
			case x == nil:
			case x.Closed || x.Online:
			default:
				if err := s.OnlineProxy(p.AdminAddr); err != nil {
					log.WarnErrorf(err, "auto online proxy-[%s] failed", p.Token)
				}
			}
		}(p)
	}
	go func() {
		stats := make(map[string]*ProxyStats)
		for k, v := range fut.Wait() {
			stats[k] = v.(*ProxyStats)
		}
		s.mu.Lock()
		defer s.mu.Unlock()
		s.stats.proxies = stats
	}()
	return &fut, nil
}
<|endoftext|>


<commit_before>//go:build linux && cgo && !agent

package sys

import (
	"io/ioutil"
	"os/user"
	"path/filepath"
	"strconv"
	"strings"
	"sync"
	"time"

	"github.com/mdlayher/vsock"

	"github.com/lxc/lxd/lxd/cgroup"
	"github.com/lxc/lxd/lxd/db"
	"github.com/lxc/lxd/lxd/storage/filesystem"
	"github.com/lxc/lxd/lxd/util"
	"github.com/lxc/lxd/shared"
	"github.com/lxc/lxd/shared/idmap"
	"github.com/lxc/lxd/shared/logger"
	"github.com/lxc/lxd/shared/osarch"
	"github.com/lxc/lxd/shared/version"
)

// InotifyTargetInfo records the inotify information associated with a given
// inotify target
type InotifyTargetInfo struct {
	Mask uint32
	Wd   int
	Path string
}

// InotifyInfo records the inotify information associated with a given
// inotify instance
type InotifyInfo struct {
	Fd int
	sync.RWMutex
	Targets map[string]*InotifyTargetInfo
}

// OS is a high-level facade for accessing all operating-system
// level functionality that LXD uses.
type OS struct {
	// Directories
	CacheDir string // Cache directory (e.g. /var/cache/lxd/).
	LogDir   string // Log directory (e.g. /var/log/lxd).
	VarDir   string // Data directory (e.g. /var/lib/lxd/).

	// Daemon environment
	Architectures   []int           // Cache of detected system architectures
	BackingFS       string          // Backing filesystem of $LXD_DIR/containers
	ExecPath        string          // Absolute path to the LXD executable
	IdmapSet        *idmap.IdmapSet // Information about user/group ID mapping
	InotifyWatch    InotifyInfo
	LxcPath         string // Path to the $LXD_DIR/containers directory
	MockMode        bool   // If true some APIs will be mocked (for testing)
	Nodev           bool
	RunningInUserNS bool

	// Privilege dropping
	UnprivUser  string
	UnprivUID   uint32
	UnprivGroup string
	UnprivGID   uint32

	// Apparmor features
	AppArmorAdmin     bool
	AppArmorAvailable bool
	AppArmorConfined  bool
	AppArmorStacked   bool
	AppArmorStacking  bool

	// Cgroup features
	CGInfo cgroup.Info

	// Kernel features
	CloseRange              bool
	CoreScheduling          bool
	IdmappedMounts          bool
	NetnsGetifaddrs         bool
	PidFdSetns              bool
	SeccompListener         bool
	SeccompListenerContinue bool
	Shiftfs                 bool
	UeventInjection         bool
	VFS3Fscaps              bool

	ContainerCoreScheduling bool
	NativeTerminals         bool
	PidFds                  bool
	SeccompListenerAddfd    bool

	// LXC features
	LXCFeatures map[string]bool

	// VM features
	VsockID uint32

	// OS info
	ReleaseInfo   map[string]string
	KernelVersion version.DottedVersion
	Uname         *shared.Utsname
	BootTime      time.Time
}

// DefaultOS returns a fresh uninitialized OS instance with default values.
func DefaultOS() *OS {
	newOS := &OS{
		VarDir:   shared.VarPath(),
		CacheDir: shared.CachePath(),
		LogDir:   shared.LogPath(),
	}
	newOS.InotifyWatch.Fd = -1
	newOS.InotifyWatch.Targets = make(map[string]*InotifyTargetInfo)
	newOS.ReleaseInfo = make(map[string]string)
	return newOS
}

// Init our internal data structures.
func (s *OS) Init() ([]db.Warning, error) {
	var dbWarnings []db.Warning

	err := s.initDirs()
	if err != nil {
		return nil, err
	}

	s.Architectures, err = util.GetArchitectures()
	if err != nil {
		return nil, err
	}

	s.LxcPath = filepath.Join(s.VarDir, "containers")

	s.BackingFS, err = filesystem.Detect(s.LxcPath)
	if err != nil {
		logger.Error("Error detecting backing fs", logger.Ctx{"err": err})
	}

	// Detect if it is possible to run daemons as an unprivileged user and group.
	for _, userName := range []string{"lxd", "nobody"} {
		u, err := user.Lookup(userName)
		if err != nil {
			continue
		}

		uid, err := strconv.ParseUint(u.Uid, 10, 32)
		if err != nil {
			return nil, err
		}

		s.UnprivUser = userName
		s.UnprivUID = uint32(uid)
		break
	}

	for _, groupName := range []string{"lxd", "nogroup"} {
		g, err := user.LookupGroup(groupName)
		if err != nil {
			continue
		}

		gid, err := strconv.ParseUint(g.Gid, 10, 32)
		if err != nil {
			return nil, err
		}

		s.UnprivGroup = groupName
		s.UnprivGID = uint32(gid)
		break
	}

	s.IdmapSet = idmap.GetIdmapSet()
	s.ExecPath = util.GetExecPath()
	s.RunningInUserNS = shared.RunningInUserNS()

	dbWarnings = s.initAppArmor()
	cgroup.Init()
	s.CGInfo = cgroup.GetInfo()

	// Fill in the VsockID.
	_ = util.LoadModule("vhost_vsock")

	vsockID, err := vsock.ContextID()
	if err != nil || vsockID > 2147483647 {
		// Fallback to the default ID for a host system if we're getting
		// an error or are getting a clearly invalid value.
		vsockID = 2
	}

	s.VsockID = vsockID

	// Fill in the OS release info.
	osInfo, err := osarch.GetLSBRelease()
	if err != nil {
		return nil, err
	}

	s.ReleaseInfo = osInfo

	uname, err := shared.Uname()
	if err != nil {
		return nil, err
	}
	s.Uname = uname

	kernelVersion, err := version.Parse(strings.Split(uname.Release, "-")[0])
	if err == nil {
		s.KernelVersion = *kernelVersion
	}

	// Fill in the boot time.
	out, err := ioutil.ReadFile("/proc/stat")
	if err != nil {
		return nil, err
	}

	btime := int64(0)
	for _, line := range strings.Split(string(out), "\n") {
		if !strings.HasPrefix(line, "btime ") {
			continue
		}

		fields := strings.Fields(line)
		btime, err = strconv.ParseInt(fields[1], 10, 64)
		if err != nil {
			return nil, err
		}

		break
	}

	if btime > 0 {
		s.BootTime = time.Unix(btime, 0)
	}

	return dbWarnings, nil
}

// InitStorage initialises the storage layer after it has been mounted.
func (s *OS) InitStorage() error {
	return s.initStorageDirs()
}
<commit_msg>lxd/sys: Ends all comments with a full-stop.<commit_after>//go:build linux && cgo && !agent

package sys

import (
	"io/ioutil"
	"os/user"
	"path/filepath"
	"strconv"
	"strings"
	"sync"
	"time"

	"github.com/mdlayher/vsock"

	"github.com/lxc/lxd/lxd/cgroup"
	"github.com/lxc/lxd/lxd/db"
	"github.com/lxc/lxd/lxd/storage/filesystem"
	"github.com/lxc/lxd/lxd/util"
	"github.com/lxc/lxd/shared"
	"github.com/lxc/lxd/shared/idmap"
	"github.com/lxc/lxd/shared/logger"
	"github.com/lxc/lxd/shared/osarch"
	"github.com/lxc/lxd/shared/version"
)

// InotifyTargetInfo records the inotify information associated with a given
// inotify target.
type InotifyTargetInfo struct {
	Mask uint32
	Wd   int
	Path string
}

// InotifyInfo records the inotify information associated with a given
// inotify instance.
type InotifyInfo struct {
	Fd int
	sync.RWMutex
	Targets map[string]*InotifyTargetInfo
}

// OS is a high-level facade for accessing all operating-system
// level functionality that LXD uses.
type OS struct {
	// Directories
	CacheDir string // Cache directory (e.g. /var/cache/lxd/).
	LogDir   string // Log directory (e.g. /var/log/lxd).
	VarDir   string // Data directory (e.g. /var/lib/lxd/).

	// Daemon environment
	Architectures   []int           // Cache of detected system architectures
	BackingFS       string          // Backing filesystem of $LXD_DIR/containers
	ExecPath        string          // Absolute path to the LXD executable
	IdmapSet        *idmap.IdmapSet // Information about user/group ID mapping
	InotifyWatch    InotifyInfo
	LxcPath         string // Path to the $LXD_DIR/containers directory
	MockMode        bool   // If true some APIs will be mocked (for testing)
	Nodev           bool
	RunningInUserNS bool

	// Privilege dropping
	UnprivUser  string
	UnprivUID   uint32
	UnprivGroup string
	UnprivGID   uint32

	// Apparmor features
	AppArmorAdmin     bool
	AppArmorAvailable bool
	AppArmorConfined  bool
	AppArmorStacked   bool
	AppArmorStacking  bool

	// Cgroup features
	CGInfo cgroup.Info

	// Kernel features
	CloseRange              bool
	CoreScheduling          bool
	IdmappedMounts          bool
	NetnsGetifaddrs         bool
	PidFdSetns              bool
	SeccompListener         bool
	SeccompListenerContinue bool
	Shiftfs                 bool
	UeventInjection         bool
	VFS3Fscaps              bool

	ContainerCoreScheduling bool
	NativeTerminals         bool
	PidFds                  bool
	SeccompListenerAddfd    bool

	// LXC features
	LXCFeatures map[string]bool

	// VM features
	VsockID uint32

	// OS info
	ReleaseInfo   map[string]string
	KernelVersion version.DottedVersion
	Uname         *shared.Utsname
	BootTime      time.Time
}

// DefaultOS returns a fresh uninitialized OS instance with default values.
func DefaultOS() *OS {
	newOS := &OS{
		VarDir:   shared.VarPath(),
		CacheDir: shared.CachePath(),
		LogDir:   shared.LogPath(),
	}
	newOS.InotifyWatch.Fd = -1
	newOS.InotifyWatch.Targets = make(map[string]*InotifyTargetInfo)
	newOS.ReleaseInfo = make(map[string]string)
	return newOS
}

// Init our internal data structures.
func (s *OS) Init() ([]db.Warning, error) {
	var dbWarnings []db.Warning

	err := s.initDirs()
	if err != nil {
		return nil, err
	}

	s.Architectures, err = util.GetArchitectures()
	if err != nil {
		return nil, err
	}

	s.LxcPath = filepath.Join(s.VarDir, "containers")

	s.BackingFS, err = filesystem.Detect(s.LxcPath)
	if err != nil {
		logger.Error("Error detecting backing fs", logger.Ctx{"err": err})
	}

	// Detect if it is possible to run daemons as an unprivileged user and group.
	for _, userName := range []string{"lxd", "nobody"} {
		u, err := user.Lookup(userName)
		if err != nil {
			continue
		}

		uid, err := strconv.ParseUint(u.Uid, 10, 32)
		if err != nil {
			return nil, err
		}

		s.UnprivUser = userName
		s.UnprivUID = uint32(uid)
		break
	}

	for _, groupName := range []string{"lxd", "nogroup"} {
		g, err := user.LookupGroup(groupName)
		if err != nil {
			continue
		}

		gid, err := strconv.ParseUint(g.Gid, 10, 32)
		if err != nil {
			return nil, err
		}

		s.UnprivGroup = groupName
		s.UnprivGID = uint32(gid)
		break
	}

	s.IdmapSet = idmap.GetIdmapSet()
	s.ExecPath = util.GetExecPath()
	s.RunningInUserNS = shared.RunningInUserNS()

	dbWarnings = s.initAppArmor()
	cgroup.Init()
	s.CGInfo = cgroup.GetInfo()

	// Fill in the VsockID.
	_ = util.LoadModule("vhost_vsock")

	vsockID, err := vsock.ContextID()
	if err != nil || vsockID > 2147483647 {
		// Fallback to the default ID for a host system if we're getting
		// an error or are getting a clearly invalid value.
		vsockID = 2
	}

	s.VsockID = vsockID

	// Fill in the OS release info.
	osInfo, err := osarch.GetLSBRelease()
	if err != nil {
		return nil, err
	}

	s.ReleaseInfo = osInfo

	uname, err := shared.Uname()
	if err != nil {
		return nil, err
	}
	s.Uname = uname

	kernelVersion, err := version.Parse(strings.Split(uname.Release, "-")[0])
	if err == nil {
		s.KernelVersion = *kernelVersion
	}

	// Fill in the boot time.
	out, err := ioutil.ReadFile("/proc/stat")
	if err != nil {
		return nil, err
	}

	btime := int64(0)
	for _, line := range strings.Split(string(out), "\n") {
		if !strings.HasPrefix(line, "btime ") {
			continue
		}

		fields := strings.Fields(line)
		btime, err = strconv.ParseInt(fields[1], 10, 64)
		if err != nil {
			return nil, err
		}

		break
	}

	if btime > 0 {
		s.BootTime = time.Unix(btime, 0)
	}

	return dbWarnings, nil
}

// InitStorage initialises the storage layer after it has been mounted.
func (s *OS) InitStorage() error {
	return s.initStorageDirs()
}
<|endoftext|>


<commit_before>package lynx

import (
	"database/sql/driver"
	"encoding/json"
	"fmt"
	"strconv"
)

// EncryptedFloat represents a float that is encryptable as a string.
type EncryptedFloat struct {
	StringValue *string
}

// NewEncryptedFloat returns a newly initialized EncryptedFloat.
func NewEncryptedFloat(f float64) EncryptedFloat {
	ef := EncryptedFloat{}
	ef.SetFloat(f)
	return ef
}

// SetFloat updates the underlying float value.
func (ef *EncryptedFloat) SetFloat(f float64) {
	s := fmt.Sprintf("%g", f)
	ef.StringValue = &s
}

// Float64 returns a float64 representation of the EncryptedFloat.
func (ef *EncryptedFloat) Float64() (float64, error) {
	if ef.StringValue == nil {
		return 0, fmt.Errorf("EncryptedFloat.Float64: attempting to return nil float64")
	}
	return strconv.ParseFloat(*ef.StringValue, 64)
}

// UnmarshalJSON will unmarshall a raw json representation of a number into an EncryptedFloat.
func (ef *EncryptedFloat) UnmarshalJSON(raw []byte) error {
	f := 0.0
	// Ensure that the value is a valid json number.
	err := json.Unmarshal(raw, &f)
	if err != nil {
		return err
	}
	// Use the unmarshaled float value.
	s := fmt.Sprintf("%g", f)
	ef.StringValue = &s
	return nil
}

// MarshalJSON marshals the current value into a json number.
func (ef EncryptedFloat) MarshalJSON() ([]byte, error) {
	if ef.StringValue == nil {
		return []byte("null"), nil
	}
	// Convert to float if possible.
	f, err := strconv.ParseFloat(*ef.Str, 64)
	if err != nil {
		return nil, fmt.Errorf("Unable to convert encrypted float to float: %s", err)
	}
	// Then json marshal the float value.
	return json.Marshal(f)
}

// EncryptableString returns the underlying string pointer.
func (ef EncryptedFloat) EncryptableString() *string {
	return ef.StringValue
}

// String implements the stringer interface.
func (ef EncryptedFloat) String() string {
	if ef.StringValue == nil {
		return ""
	}
	return *ef.StringValue
}

// Scan implements sql.Scanner for scanning database values.
func (ef *EncryptedFloat) Scan(value interface{}) error {
	switch val := value.(type) {
	case float64:
		s := fmt.Sprintf("%f", val)
		ef.StringValue = &s
	case int64:
		s := fmt.Sprintf("%d", val)
		ef.StringValue = &s
	case []byte:
		s := string(val)
		ef.Str = &s
	case string:
		ef.StringValue = &val
	case nil:
		ef.StringValue = nil
	default:
		return fmt.Errorf("EncryptedFloat.Scan: invalid scan type %T", value)
	}
	return nil
}

// Value implements value.Valuer to provide database values.
func (ef EncryptedFloat) Value() (driver.Value, error) {
	if ef.StringValue == nil {
		return nil, nil
	}
	return *ef.StringValue, nil
}
<commit_msg>Now using StringValue instead of Str for exported lynx.EncryptedFloat<commit_after>package lynx

import (
	"database/sql/driver"
	"encoding/json"
	"fmt"
	"strconv"
)

// EncryptedFloat represents a float that is encryptable as a string.
type EncryptedFloat struct {
	StringValue *string
}

// NewEncryptedFloat returns a newly initialized EncryptedFloat.
func NewEncryptedFloat(f float64) EncryptedFloat {
	ef := EncryptedFloat{}
	ef.SetFloat(f)
	return ef
}

// SetFloat updates the underlying float value.
func (ef *EncryptedFloat) SetFloat(f float64) {
	s := fmt.Sprintf("%g", f)
	ef.StringValue = &s
}

// Float64 returns a float64 representation of the EncryptedFloat.
func (ef *EncryptedFloat) Float64() (float64, error) {
	if ef.StringValue == nil {
		return 0, fmt.Errorf("EncryptedFloat.Float64: attempting to return nil float64")
	}
	return strconv.ParseFloat(*ef.StringValue, 64)
}

// UnmarshalJSON will unmarshall a raw json representation of a number into an EncryptedFloat.
func (ef *EncryptedFloat) UnmarshalJSON(raw []byte) error {
	f := 0.0
	// Ensure that the value is a valid json number.
	err := json.Unmarshal(raw, &f)
	if err != nil {
		return err
	}
	// Use the unmarshaled float value.
	s := fmt.Sprintf("%g", f)
	ef.StringValue = &s
	return nil
}

// MarshalJSON marshals the current value into a json number.
func (ef EncryptedFloat) MarshalJSON() ([]byte, error) {
	if ef.StringValue == nil {
		return []byte("null"), nil
	}
	// Convert to float if possible.
	f, err := strconv.ParseFloat(*ef.StringValue, 64)
	if err != nil {
		return nil, fmt.Errorf("Unable to convert encrypted float to float: %s", err)
	}
	// Then json marshal the float value.
	return json.Marshal(f)
}

// EncryptableString returns the underlying string pointer.
func (ef EncryptedFloat) EncryptableString() *string {
	return ef.StringValue
}

// String implements the stringer interface.
func (ef EncryptedFloat) String() string {
	if ef.StringValue == nil {
		return ""
	}
	return *ef.StringValue
}

// Scan implements sql.Scanner for scanning database values.
func (ef *EncryptedFloat) Scan(value interface{}) error {
	switch val := value.(type) {
	case float64:
		s := fmt.Sprintf("%f", val)
		ef.StringValue = &s
	case int64:
		s := fmt.Sprintf("%d", val)
		ef.StringValue = &s
	case []byte:
		s := string(val)
		ef.StringValue = &s
	case string:
		ef.StringValue = &val
	case nil:
		ef.StringValue = nil
	default:
		return fmt.Errorf("EncryptedFloat.Scan: invalid scan type %T", value)
	}
	return nil
}

// Value implements value.Valuer to provide database values.
func (ef EncryptedFloat) Value() (driver.Value, error) {
	if ef.StringValue == nil {
		return nil, nil
	}
	return *ef.StringValue, nil
}
<|endoftext|>


<commit_before>package lzb

import "errors"

type buffer struct {
	data       []byte
	bottom     int64 // bottom == max(top - len(data), 0)
	top        int64
	writeLimit int64
}

const maxWriteLimit = 1<<63 - 1

var (
	errOffset = errors.New("offset outside buffer range")
	errAgain  = errors.New("buffer overflow; repeat")
	errNegLen = errors.New("length is negative")
	errLimit  = errors.New("write limit reached")
)

func initBuffer(b *buffer, capacity int) {
	*b = buffer{data: make([]byte, capacity), writeLimit: maxWriteLimit}
}

func newBuffer(capacity int) *buffer {
	b := new(buffer)
	initBuffer(b, capacity)
	return b
}

func (b *buffer) capacity() int {
	return len(b.data)
}

func (b *buffer) length() int {
	return int(b.top - b.bottom)
}

func (b *buffer) setTop(off int64) {
	if off < 0 {
		panic("b.Top overflow?")
	}
	if off > b.writeLimit {
		panic("off exceeds writeLimit")
	}
	b.top = off
	b.bottom = off - int64(len(b.data))
	if b.bottom < 0 {
		b.bottom = 0
	}
}

func (b *buffer) index(off int64) int {
	if off < 0 {
		panic("negative offset?")
	}
	return int(off % int64(len(b.data)))
}

func (b *buffer) Write(p []byte) (n int, err error) {
	if len(p) == 0 {
		return
	}
	var m int
	off := b.top
	if off+int64(len(p)) > b.writeLimit {
		m = int(b.writeLimit - off)
		p = p[:m]
		err = errLimit
	}
	m = len(p) - len(b.data)
	if m > 0 {
		off += int64(m)
		p = p[m:]
	}
	for len(p) > 0 {
		m = copy(b.data[b.index(off):], p)
		off += int64(m)
		p = p[m:]
	}
	n = int(off - b.top)
	b.setTop(off)
	return n, err
}

func (b *buffer) WriteByte(c byte) error {
	if b.top >= b.writeLimit {
		return errLimit
	}
	b.data[b.index(b.top)] = c
	b.setTop(b.top + 1)
	return nil
}

func (b *buffer) writeRep(off int64, n int) (written int, err error) {
	if n < 0 {
		return 0, errNegLen
	}
	if !(b.bottom <= off && off < b.top) {
		return 0, errOffset
	}

	start, end := off, off+int64(n)
loop:
	for off < end {
		var next int64
		if b.top < end {
			next = b.top
		} else {
			next = end
		}
		e := b.index(next)
		for off < next {
			s := b.index(off)
			var q []byte
			if s < e {
				q = b.data[s:e]
			} else {
				q = b.data[s:]
			}
			var k int
			k, err = b.Write(q)
			off += int64(k)
			if err != nil {
				break loop
			}
		}
	}
	return int(off - start), err
}

// equalBytes count the equal bytes at off1 and off2 until max is reached.
func (b *buffer) equalBytes(off1, off2 int64, max int) int {
	if off1 < b.bottom || off2 < b.bottom || max <= 0 {
		return 0
	}
	n := b.top - off1
	if n < int64(max) {
		if n <= 0 {
			return 0
		}
		max = int(n)
	}
	n = b.top - off2
	if n < int64(max) {
		if n <= 0 {
			return 0
		}
		max = int(n)
	}
	for k := 0; k < max; k++ {
		i, j := b.index(off1+int64(k)), b.index(off2+int64(k))
		if b.data[i] != b.data[j] {
			return k
		}
	}
	return max
}
<commit_msg>lzb: added documentation comments to buffer.go<commit_after>package lzb

import "errors"

// buffer provides a circular buffer. The type support the io.Writer
// interface and other functions required to implement a dictionary.
//
// The top offset tracks the position of the buffer in the byte stream
// covered. The bottom offset marks the bottom of the buffer. The
// writeLimit marks the limit for additional writes.
type buffer struct {
	data       []byte
	bottom     int64 // bottom == max(top - len(data), 0)
	top        int64
	writeLimit int64
}

// maxWriteLimit provides the maximum value. Setting the writeLimit to
// this value disables the writeLimit for all practical purposes.
const maxWriteLimit = 1<<63 - 1

// Errors returned by buffer methods.
var (
	errOffset = errors.New("offset outside buffer range")
	errAgain  = errors.New("buffer overflow; repeat")
	errNegLen = errors.New("length is negative")
	errLimit  = errors.New("write limit reached")
)

// initBuffer initializes a buffer variable.
func initBuffer(b *buffer, capacity int) {
	*b = buffer{data: make([]byte, capacity), writeLimit: maxWriteLimit}
}

// newBuffer creates a new buffer.
func newBuffer(capacity int) *buffer {
	b := new(buffer)
	initBuffer(b, capacity)
	return b
}

// capacity returns the maximum capacity of the buffer.
func (b *buffer) capacity() int {
	return len(b.data)
}

// length returns the actual length of the buffer.
func (b *buffer) length() int {
	return int(b.top - b.bottom)
}

// setTop sets the top and bottom offset. Any modification of the top
// offset must use this method.
func (b *buffer) setTop(off int64) {
	if off < 0 {
		panic("b.Top overflow?")
	}
	if off > b.writeLimit {
		panic("off exceeds writeLimit")
	}
	b.top = off
	b.bottom = off - int64(len(b.data))
	if b.bottom < 0 {
		b.bottom = 0
	}
}

// index converts a byte stream offset into an index of the data field.
func (b *buffer) index(off int64) int {
	if off < 0 {
		panic("negative offset?")
	}
	return int(off % int64(len(b.data)))
}

// Write writes a byte slice into the buffer. It satisfies the io.Write
// interface.
func (b *buffer) Write(p []byte) (n int, err error) {
	if len(p) == 0 {
		return
	}
	var m int
	off := b.top
	if off+int64(len(p)) > b.writeLimit {
		m = int(b.writeLimit - off)
		p = p[:m]
		err = errLimit
	}
	m = len(p) - len(b.data)
	if m > 0 {
		off += int64(m)
		p = p[m:]
	}
	for len(p) > 0 {
		m = copy(b.data[b.index(off):], p)
		off += int64(m)
		p = p[m:]
	}
	n = int(off - b.top)
	b.setTop(off)
	return n, err
}

// WriteByte writes a single byte into the buffer. The method satisfies
// the io.ByteWriter interface.
func (b *buffer) WriteByte(c byte) error {
	if b.top >= b.writeLimit {
		return errLimit
	}
	b.data[b.index(b.top)] = c
	b.setTop(b.top + 1)
	return nil
}

// writeRep writes a repetition into the buffer. Obviously the method is
// used to handle matches during decoding the LZMA stream.
func (b *buffer) writeRep(off int64, n int) (written int, err error) {
	if n < 0 {
		return 0, errNegLen
	}
	if !(b.bottom <= off && off < b.top) {
		return 0, errOffset
	}

	start, end := off, off+int64(n)
loop:
	for off < end {
		var next int64
		if b.top < end {
			next = b.top
		} else {
			next = end
		}
		e := b.index(next)
		for off < next {
			s := b.index(off)
			var q []byte
			if s < e {
				q = b.data[s:e]
			} else {
				q = b.data[s:]
			}
			var k int
			k, err = b.Write(q)
			off += int64(k)
			if err != nil {
				break loop
			}
		}
	}
	return int(off - start), err
}

// equalBytes count the equal bytes at off1 and off2 until max is reached.
func (b *buffer) equalBytes(off1, off2 int64, max int) int {
	if off1 < b.bottom || off2 < b.bottom || max <= 0 {
		return 0
	}
	n := b.top - off1
	if n < int64(max) {
		if n <= 0 {
			return 0
		}
		max = int(n)
	}
	n = b.top - off2
	if n < int64(max) {
		if n <= 0 {
			return 0
		}
		max = int(n)
	}
	for k := 0; k < max; k++ {
		i, j := b.index(off1+int64(k)), b.index(off2+int64(k))
		if b.data[i] != b.data[j] {
			return k
		}
	}
	return max
}
<|endoftext|>


<commit_before>package godict

import (
	"math/rand"
	"runtime"
	"testing"
	"time"
)

const chars = "abcdefghijklmnopqrstuvwxyz" +
	"ABCDEFGHIJKLMNOPQRSTUVWXYZ" +
	"~!@#$%^&*()-_+={}[]\\|<,>.?/\"';:` "

func randomString(length int) string {

	res := make([]byte, length)

	for i := 0; i < length; i++ {
		res[i] = chars[rand.Intn(len(chars))]
	}

	return string(res)
}

func isEmpty(e entry, t *testing.T) {
	if e.data != nil {
		t.Error("Element is not nil")
	}
}

//Test_CreateNew tests parameters of new dictionary
func Test_CreateNew(t *testing.T) {
	d := New()

	if d.active != 0 {
		t.Error("There is not 0 active elements count in new dict")
	}

	if d.mask != 7 {
		t.Error("Mask of new dict is not equal 7")
	}

	if len(d.dict) != 8 {
		t.Error("Size of new dict is not equal 8")
	}

	if d.sparedict != nil {
		t.Error("Spare dict is not nil")
	}

	if cap(d.dict) != 8 {
		t.Error("Cap of new dict is not equal 8")
	}
	for _, e := range d.dict {
		isEmpty(e, t)
	}
}

//Test_Set tests setting elements in dict
func Test_Set(t *testing.T) {
	d := New()

	d.Set("a", "1")

	if d.active != 1 {
		t.Errorf("Wrong number of active slots: %v, must be 1", d.active)
	}

	var elts []entry

	for _, e := range d.dict {
		if e.data != nil && e.key == "a" {
			elts = append(elts, e)
		}
	}

	if len(elts) == 0 {
		t.Error("Element not found in dict")
	}

	if len(elts) > 1 {
		t.Error("Element found but in multiple slots")
	}

	elt := elts[0]

	if elt.value != "1" {
		t.Error("Element found, but value is not equal 1")
	}

	hsh := GenHash("a")

	if elt.hash != hsh {
		t.Error("Element found but with wrong hash")
	}

	if elt.deleted {
		t.Error("Element found but already marked as deleted")
	}
}

func BenchmarkSet(b *testing.B) {
	testTable := make([]string, 10000)
	for i := 0; i < len(testTable); i++ {
		testTable[i] = randomString(3)
	}
	b.ResetTimer()
	for i := 0; i < b.N; i++ {
		d := New()
		for _, elt := range testTable {
			d.Set(elt, "1")
		}
	}
}

//Test_Set tests setting elements in dict and resize on 2/3 fill
func Test_SetAndResize(t *testing.T) {
	runtime.GOMAXPROCS(2)

	d := New()

	test_keys1 := map[string]string{
		"a": "1",
		"b": "2",
		"c": "3",
		"d": "4",
		"e": "5",
		"f": "6",
	}

	for key, value := range test_keys1 {
		err := d.Set(key, value)
		if err != nil {
			t.Errorf("Error %v while inserting key %v", err, key)
		}
	}

	if d.active != uint32(len(test_keys1)) {
		t.Errorf("Wrong number of active slots: %v, must be %v",
			d.active, len(test_keys1))
	}

	for key, value := range test_keys1 {
		res, err := d.Get(key)
		if err != nil {
			t.Errorf("Error %v on retrieving key %v", err, key)
		}
		if res.value != value {
			t.Errorf("Wrong value %v for key %v", res, key)
		}
	}

	// waiting for rehash, cause of deadlock
	time.Sleep(time.Second / 10)

	if d.mask != 15 {
		t.Errorf("Wrong mask %v after resize, must be 15", d.mask)
	}

	if len(d.dict) != 16 {
		t.Errorf("Wrong len of resized dict: %v, must be 16", len(d.dict))
	}

	if cap(d.dict) != 16 {
		t.Errorf("Wrong cap of resized dict: %v, must be 16", cap(d.dict))
	}

	test_keys2 := map[string]string{
		"g": "7",
		"h": "8",
		"i": "9",
		"j": "10",
		"k": "11",
	}

	for key, value := range test_keys2 {
		err := d.Set(key, value)
		if err != nil {
			t.Errorf("Error %v while inserting key %v", err, key)
		}
	}

	if d.active != uint32(len(test_keys1)+len(test_keys2)) {
		t.Errorf("Wrong number of active slots: %v, must be %v",
			d.active, len(test_keys1)+len(test_keys2))
	}

	for key, value := range test_keys1 {
		res, err := d.Get(key)
		if err != nil {
			t.Errorf("Error %v on retrieving key %v", err, key)
		}
		if res.value != value {
			t.Errorf("Wrong value %v for key %v", res, key)
		}
	}

	for key, value := range test_keys2 {
		res, err := d.Get(key)
		if err != nil {
			t.Errorf("Error %v on retrieving key %v", err, key)
		}
		if res.value != value {
			t.Errorf("Wrong value %v for key %v", res, key)
		}
	}

	time.Sleep(time.Second / 10)

	if d.mask != 31 {
		t.Errorf("Wrong mask %v after resize, must be 31", d.mask)
	}

	if len(d.dict) != 32 {
		t.Errorf("Wrong len of resized dict: %v, must be 32", len(d.dict))
	}

	if cap(d.dict) != 32 {
		t.Errorf("Wrong cap of resized dict: %v, must be 32", cap(d.dict))
	}
}

//Test_Get tests getting element from dict
func Test_Get(t *testing.T) {
	d := New()

	d.Set("a", "1")

	res, err := d.Get("a")

	if err != nil {
		t.Errorf("Get failed with error %v", err)
	}

	if res.value != "1" {
		t.Errorf("Get wrong value %v", res)
	}
}

//Test_GetUnexisting tests getting element from dict without setting it first
func Test_GetUnexisting(t *testing.T) {
	d := New()

	slot, err := d.Get("a")

	if err == nil {
		t.Logf("Get did not fail. slot = %v. What value he returned?", slot)
		v := slot.Value()
		t.Errorf("Get did not fail and returned %v", v)
	}

}

//Test_Delete tests deleting element from dict
func Test_Delete(t *testing.T) {
	d := New()

	d.Set("a", "1")

	err := d.Delete("a")

	if err != nil {
		t.Errorf("Delete failed with error %v", err)
	}

	slot, err := d.Get("a")

	if err == nil {
		t.Logf("Get did not fail after delete. slot = %v. What value he returned?", slot)
		v := slot.Value()
		t.Errorf("Get did not fail and returned %v", v)
	}

	d.Set("a", "1")

	slot, err = d.Get("a")

	if err != nil {
		t.Errorf("Get failed with error %v", err)
	}

	if slot.value != "1" {
		t.Errorf("Get wrong value %v", slot.value)
	}

	if slot.deleted {
		t.Errorf("Key was set again, but still deleted, slot %v", slot)
	}
}

//Test_DeleteUnexisting tests delete element from dict without setting it first
func Test_DeleteUnexisting(t *testing.T) {
	d := New()

	err := d.Delete("a")

	if err == nil {
		t.Errorf("Delete did not fail. dict: %v", d)
	}

}
<commit_msg>Tests on expire command<commit_after>package godict

import (
	"math/rand"
	"runtime"
	"testing"
	"time"
)

const chars = "abcdefghijklmnopqrstuvwxyz" +
	"ABCDEFGHIJKLMNOPQRSTUVWXYZ" +
	"~!@#$%^&*()-_+={}[]\\|<,>.?/\"';:` "

func randomString(length int) string {

	res := make([]byte, length)

	for i := 0; i < length; i++ {
		res[i] = chars[rand.Intn(len(chars))]
	}

	return string(res)
}

func isEmpty(e entry, t *testing.T) {
	if e.data != nil {
		t.Error("Element is not nil")
	}
}

//TestCreateNew tests parameters of new dictionary
func TestCreateNew(t *testing.T) {
	d := New()

	if d.active != 0 {
		t.Error("There is not 0 active elements count in new dict")
	}

	if d.mask != 7 {
		t.Error("Mask of new dict is not equal 7")
	}

	if len(d.dict) != 8 {
		t.Error("Size of new dict is not equal 8")
	}

	if d.sparedict != nil {
		t.Error("Spare dict is not nil")
	}

	if cap(d.dict) != 8 {
		t.Error("Cap of new dict is not equal 8")
	}
	for _, e := range d.dict {
		isEmpty(e, t)
	}
}

//TestSet tests setting elements in dict
func TestSet(t *testing.T) {
	d := New()

	d.Set("a", "1")

	if d.active != 1 {
		t.Errorf("Wrong number of active slots: %v, must be 1", d.active)
	}

	var elts []entry

	for _, e := range d.dict {
		if e.data != nil && e.key == "a" {
			elts = append(elts, e)
		}
	}

	if len(elts) == 0 {
		t.Error("Element not found in dict")
	}

	if len(elts) > 1 {
		t.Error("Element found but in multiple slots")
	}

	elt := elts[0]

	if elt.value != "1" {
		t.Error("Element found, but value is not equal 1")
	}

	hsh := GenHash("a")

	if elt.hash != hsh {
		t.Error("Element found but with wrong hash")
	}

	if elt.deleted {
		t.Error("Element found but already marked as deleted")
	}
}

func BenchmarkSet(b *testing.B) {
	testTable := make([]string, 10000)
	for i := 0; i < len(testTable); i++ {
		testTable[i] = randomString(3)
	}
	b.ResetTimer()
	for i := 0; i < b.N; i++ {
		d := New()
		for _, elt := range testTable {
			d.Set(elt, "1")
		}
	}
}

//TestSet tests setting elements in dict and resize on 2/3 fill
func TestSetAndResize(t *testing.T) {
	runtime.GOMAXPROCS(2)

	d := New()

	test_keys1 := map[string]string{
		"a": "1",
		"b": "2",
		"c": "3",
		"d": "4",
		"e": "5",
		"f": "6",
	}

	for key, value := range test_keys1 {
		err := d.Set(key, value)
		if err != nil {
			t.Errorf("Error %v while inserting key %v", err, key)
		}
	}

	if d.active != uint32(len(test_keys1)) {
		t.Errorf("Wrong number of active slots: %v, must be %v",
			d.active, len(test_keys1))
	}

	for key, value := range test_keys1 {
		res, err := d.Get(key)
		if err != nil {
			t.Errorf("Error %v on retrieving key %v", err, key)
		}
		if res.value != value {
			t.Errorf("Wrong value %v for key %v", res, key)
		}
	}

	// waiting for rehash, cause of deadlock
	time.Sleep(time.Second / 10)

	if d.mask != 15 {
		t.Errorf("Wrong mask %v after resize, must be 15", d.mask)
	}

	if len(d.dict) != 16 {
		t.Errorf("Wrong len of resized dict: %v, must be 16", len(d.dict))
	}

	if cap(d.dict) != 16 {
		t.Errorf("Wrong cap of resized dict: %v, must be 16", cap(d.dict))
	}

	test_keys2 := map[string]string{
		"g": "7",
		"h": "8",
		"i": "9",
		"j": "10",
		"k": "11",
	}

	for key, value := range test_keys2 {
		err := d.Set(key, value)
		if err != nil {
			t.Errorf("Error %v while inserting key %v", err, key)
		}
	}

	if d.active != uint32(len(test_keys1)+len(test_keys2)) {
		t.Errorf("Wrong number of active slots: %v, must be %v",
			d.active, len(test_keys1)+len(test_keys2))
	}

	for key, value := range test_keys1 {
		res, err := d.Get(key)
		if err != nil {
			t.Errorf("Error %v on retrieving key %v", err, key)
		}
		if res.value != value {
			t.Errorf("Wrong value %v for key %v", res, key)
		}
	}

	for key, value := range test_keys2 {
		res, err := d.Get(key)
		if err != nil {
			t.Errorf("Error %v on retrieving key %v", err, key)
		}
		if res.value != value {
			t.Errorf("Wrong value %v for key %v", res, key)
		}
	}

	time.Sleep(time.Second / 10)

	if d.mask != 31 {
		t.Errorf("Wrong mask %v after resize, must be 31", d.mask)
	}

	if len(d.dict) != 32 {
		t.Errorf("Wrong len of resized dict: %v, must be 32", len(d.dict))
	}

	if cap(d.dict) != 32 {
		t.Errorf("Wrong cap of resized dict: %v, must be 32", cap(d.dict))
	}
}

//TestGet tests getting element from dict
func TestGet(t *testing.T) {
	d := New()

	d.Set("a", "1")

	res, err := d.Get("a")

	if err != nil {
		t.Errorf("Get failed with error %v", err)
	}

	if res.value != "1" {
		t.Errorf("Get wrong value %v", res)
	}
}

//TestGetUnexisting tests getting element from dict without setting it first
func TestGetUnexisting(t *testing.T) {
	d := New()

	slot, err := d.Get("a")

	if err == nil {
		t.Logf("Get did not fail. slot = %v. What value he returned?", slot)
		v := slot.Value()
		t.Errorf("Get did not fail and returned %v", v)
	}

}

//TestDelete tests deleting element from dict
func TestDelete(t *testing.T) {
	d := New()

	d.Set("a", "1")

	err := d.Delete("a")

	if err != nil {
		t.Errorf("Delete failed with error %v", err)
	}

	slot, err := d.Get("a")

	if err == nil {
		t.Logf("Get did not fail after delete. slot = %v. What value he returned?", slot)
		v := slot.Value()
		t.Errorf("Get did not fail and returned %v", v)
	}

	d.Set("a", "1")

	slot, err = d.Get("a")

	if err != nil {
		t.Errorf("Get failed with error %v", err)
	}

	if slot.value != "1" {
		t.Errorf("Get wrong value %v", slot.value)
	}

	if slot.deleted {
		t.Errorf("Key was set again, but still deleted, slot %v", slot)
	}
}

//TestDeleteUnexisting tests delete element from dict without setting it first
func TestDeleteUnexisting(t *testing.T) {
	d := New()

	err := d.Delete("a")

	if err == nil {
		t.Errorf("Delete did not fail. dict: %v", d)
	}
}

func TestExpire(t *testing.T) {
	d := New()
	d.Set("a", "1")
	d.Expire("a", 1)
	timeout := time.After(time.Second)
	slot, err := d.Get("a")
	if err != nil {
		t.Errorf("Error before expire: %v", err)
	}
	if slot.Value() != "1" {
		t.Errorf("Wrong slot value: %v", slot)
	}
	<-timeout
	slot, err = d.Get("a")
	if err == nil {
		t.Errorf("Object wasn`t expired. dict: %v, slot: %v", d, slot)
	}
}

func TestExpireUnexisting(t *testing.T) {
	d := New()

	err := d.Expire("a", 1)

	if err == nil {
		t.Errorf("Expire did not fail. dict: %v", d)
	}
}
<|endoftext|>


<commit_before>package main

import "github.com/mndrix/tap-go"

func main() {
	t := tap.New()
	t.Header(1)
	t.Diagnostic("expecting all to be well")
	t.Diagnosticf("here's some perfectly magical output: %d %s 0x%X.", 6, "abracadabra", 28)
	t.Diagnostic("some\nmultiline\ntext\n")
	t.Diagnosticf("%d lines\n%s multiline\ntext", 3, "more")
	t.Pass("all good")
}
<commit_msg>Explicitly test diagnostic output<commit_after>package main

import (
	"bytes"
	"io"
	"os"

	tap "github.com/mndrix/tap-go"
)

func main() {
	// collect output for comparison later
	buf := new(bytes.Buffer)
	t := tap.New()
	t.Writer = io.MultiWriter(os.Stdout, buf)

	t.Header(1)
	t.Diagnostic("expecting all to be well")
	t.Diagnosticf("here's some perfectly magical output: %d %s 0x%X.", 6, "abracadabra", 28)
	t.Diagnostic("some\nmultiline\ntext\n")
	t.Diagnosticf("%d lines\n%s multiline\ntext", 3, "more")

	got := buf.String()
	t.Ok(got == expected, "diagnostics gave expected output")
}

const expected = `TAP version 13
1..1
# expecting all to be well
# here's some perfectly magical output: 6 abracadabra 0x1C.
# some
# multiline
# text
# 3 lines
# more multiline
# text
`
<|endoftext|>


<commit_before>package certgen

import (
	"crypto/x509/pkix"
	"encoding/asn1"
	"encoding/hex"
	"errors"
	"net"
	"testing"
)

func TestComputePublicKeyKeyID(t *testing.T) {
	userPub, _, _ := setupX509Generator(t)
	_, err := ComputePublicKeyKeyID(userPub)
	if err != nil {
		t.Fatal(err)
	}
}

func TestGenDelegationExtension(t *testing.T) {

	netblock := net.IPNet{
		IP:   net.ParseIP("10.11.12.0"),
		Mask: net.CIDRMask(24, 32),
	}
	netblock2 := net.IPNet{
		IP:   net.ParseIP("13.14.128.0"),
		Mask: net.CIDRMask(20, 32),
	}
	netblockList := []net.IPNet{netblock, netblock2}
	var extension *pkix.Extension
	var err error
	extension, err = genDelegationExtension(netblockList)
	if err != nil {
		t.Fatal(err)
	}
	extensionDer, err := asn1.Marshal(*extension)
	if err != nil {
		t.Fatal(err)
	}

	t.Logf("encodedExt=\n%s", hex.Dump(extensionDer))
	/*
	 */
	var addressFamilyList []IpAdressFamily
	_, err = asn1.Unmarshal(extension.Value, &addressFamilyList)
	if err != nil {
		t.Fatal(err)
	}
	t.Logf("%+v", addressFamilyList)
	var roundTripBlockList []net.IPNet
	for _, encodedNetblock := range addressFamilyList[0].Addresses {
		decoded, err := decodeIPV4AddressChoice(encodedNetblock)
		if err != nil {
			t.Fatal(err)
		}
		roundTripBlockList = append(roundTripBlockList, decoded)
	}
	t.Logf("%+v", roundTripBlockList)
	if len(roundTripBlockList) != len(netblockList) {
		t.Fatal(errors.New("bad rountrip lenght"))
	}

}

func TestGenIPRestrictedX509Cert(t *testing.T) {
	userPub, caCert, caPriv := setupX509Generator(t)
	netblock := net.IPNet{
		IP:   net.ParseIP("127.0.0.0"),
		Mask: net.CIDRMask(8, 32),
	}
	netblock2 := net.IPNet{
		IP:   net.ParseIP("10.0.0.0"),
		Mask: net.CIDRMask(8, 32),
	}
	netblockList := []net.IPNet{netblock, netblock2}
	derCert, err := GenIPRestrictedX509Cert("username", userPub, caCert, caPriv, netblockList, testDuration, nil, nil)
	if err != nil {
		t.Fatal(err)
	}
	cert, _, err := derBytesCertToCertAndPem(derCert)
	if err != nil {
		t.Fatal(err)
	}
	//t.Logf("%+v", cert)
	var ok bool
	ok, err = VerifyIPRestrictedX509CertIP(cert, "10.0.0.1:234")
	if err != nil {
		t.Fatal(err)
	}
	if !ok {
		t.Fatal("should have passed")
	}
	ok, err = VerifyIPRestrictedX509CertIP(cert, "1.1.1.1:234")
	if err != nil {
		t.Fatal(err)
	}
	if ok {
		t.Fatal("should have failed bad ip range")
	}
	ok, err = VerifyIPRestrictedX509CertIP(caCert, "1.1.1.1:234")
	if err != nil {
		t.Fatal(err)
	}
	if ok {
		t.Fatal("should have failed extension not found")
	}
}
<commit_msg>missed nit<commit_after>package certgen

import (
	"crypto/x509/pkix"
	"encoding/asn1"
	"encoding/hex"
	"errors"
	"net"
	"testing"
)

func TestComputePublicKeyKeyID(t *testing.T) {
	userPub, _, _ := setupX509Generator(t)
	_, err := ComputePublicKeyKeyID(userPub)
	if err != nil {
		t.Fatal(err)
	}
}

func TestGenDelegationExtension(t *testing.T) {

	netblock := net.IPNet{
		IP:   net.ParseIP("10.11.12.0"),
		Mask: net.CIDRMask(24, 32),
	}
	netblock2 := net.IPNet{
		IP:   net.ParseIP("13.14.128.0"),
		Mask: net.CIDRMask(20, 32),
	}
	netblockList := []net.IPNet{netblock, netblock2}
	var extension *pkix.Extension
	var err error
	extension, err = genDelegationExtension(netblockList)
	if err != nil {
		t.Fatal(err)
	}
	extensionDer, err := asn1.Marshal(*extension)
	if err != nil {
		t.Fatal(err)
	}

	t.Logf("encodedExt=\n%s", hex.Dump(extensionDer))
	var addressFamilyList []IpAdressFamily
	_, err = asn1.Unmarshal(extension.Value, &addressFamilyList)
	if err != nil {
		t.Fatal(err)
	}
	t.Logf("%+v", addressFamilyList)
	var roundTripBlockList []net.IPNet
	for _, encodedNetblock := range addressFamilyList[0].Addresses {
		decoded, err := decodeIPV4AddressChoice(encodedNetblock)
		if err != nil {
			t.Fatal(err)
		}
		roundTripBlockList = append(roundTripBlockList, decoded)
	}
	t.Logf("%+v", roundTripBlockList)
	if len(roundTripBlockList) != len(netblockList) {
		t.Fatal(errors.New("bad rountrip lenght"))
	}

}

func TestGenIPRestrictedX509Cert(t *testing.T) {
	userPub, caCert, caPriv := setupX509Generator(t)
	netblock := net.IPNet{
		IP:   net.ParseIP("127.0.0.0"),
		Mask: net.CIDRMask(8, 32),
	}
	netblock2 := net.IPNet{
		IP:   net.ParseIP("10.0.0.0"),
		Mask: net.CIDRMask(8, 32),
	}
	netblockList := []net.IPNet{netblock, netblock2}
	derCert, err := GenIPRestrictedX509Cert("username", userPub, caCert, caPriv, netblockList, testDuration, nil, nil)
	if err != nil {
		t.Fatal(err)
	}
	cert, _, err := derBytesCertToCertAndPem(derCert)
	if err != nil {
		t.Fatal(err)
	}
	//t.Logf("%+v", cert)
	var ok bool
	ok, err = VerifyIPRestrictedX509CertIP(cert, "10.0.0.1:234")
	if err != nil {
		t.Fatal(err)
	}
	if !ok {
		t.Fatal("should have passed")
	}
	ok, err = VerifyIPRestrictedX509CertIP(cert, "1.1.1.1:234")
	if err != nil {
		t.Fatal(err)
	}
	if ok {
		t.Fatal("should have failed bad ip range")
	}
	ok, err = VerifyIPRestrictedX509CertIP(caCert, "1.1.1.1:234")
	if err != nil {
		t.Fatal(err)
	}
	if ok {
		t.Fatal("should have failed extension not found")
	}
}
<|endoftext|>