<commit_before>// Copyright 2016 The Upspin Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

// Package gcp implements upspin.Store using Google Cloud Platform as its storage.
package gcp

import (
	"bytes"
	"fmt"
	"io/ioutil"
	"net/url"
	"os"
	"strings"
	"sync"

	"upspin.io/bind"
	"upspin.io/cloud/storage"
	"upspin.io/errors"
	"upspin.io/key/sha256key"
	"upspin.io/log"
	"upspin.io/store/gcp/cache"
	"upspin.io/upspin"
)

// Configuration options for this package.
const (
	// ConfigProjectID specifies which GCP project to use for talking to GCP.
	//
	ConfigProjectID = "gcpProjectId"

	// ConfigBucketName specifies which GCS bucket to store data in.
	// If not specified, "g-upspin-store" is used.
	ConfigBucketName = "gcpBucketName"

	// ConfigTemporaryDir specifies which temporary directory to write files to before they're
	// uploaded to the destination bucket. If not present, one will be created in the
	// system's default location.
	ConfigTemporaryDir = "gcpTemporaryDir"
)

var (
	errNotConfigured = errors.E(errors.Invalid, errors.Str("GCP Store service not configured"))
)

// Server implements upspin.Store.
type server struct {
	context  upspin.Context
	endpoint upspin.Endpoint
}

var _ upspin.Store = (*server)(nil)

var (
	mu          sync.RWMutex // Protects fields below.
	refCount    uint64       // How many clones of us exist.
	cloudClient storage.Storage
	fileCache   *cache.FileCache
)

// New returns a new, unconfigured Store bound to the user in the context.
func New(context *upspin.Context) upspin.Store {
	return &server{
		context: *context, // Make a copy to prevent user making further changes.
	}
}

// Put implements upspin.Store.
func (s *server) Put(data []byte) (upspin.Reference, error) {
	const Put = "Put"
	reader := bytes.NewReader(data)
	// TODO: check that userName has permission to write to this store server.
	if !s.isConfigured() {
		return "", errors.E(Put, errNotConfigured)
	}
	mu.RLock()
	sha := sha256key.NewShaReader(reader)
	initialRef := fileCache.RandomRef()
	err := fileCache.Put(initialRef, sha)
	if err != nil {
		mu.RUnlock()
		return "", errors.E(Put, err)
	}
	// Figure out the appropriate reference for this blob.
	ref := sha.EncodedSum()

	// Rename it in the cache
	fileCache.Rename(ref, initialRef)

	// Now go store it in the cloud.
	go func() {
		if _, err := cloudClient.PutLocalFile(fileCache.GetFileLocation(ref), ref); err == nil {
			// Remove the locally-cached entry so we never
			// keep files locally, as we're a tiny server
			// compared with our much better-provisioned
			// storage backend.  This is safe to do
			// because FileCache is thread safe.
			fileCache.Purge(ref)
		}
		mu.RUnlock()
	}()
	return upspin.Reference(ref), nil
}

// Get implements upspin.Store.
func (s *server) Get(ref upspin.Reference) ([]byte, []upspin.Location, error) {
	file, loc, err := s.innerGet(s.context.UserName, ref)
	if err != nil {
		return nil, nil, err
	}
	if file != nil {
		defer file.Close()
		bytes, err := ioutil.ReadAll(file)
		if err != nil {
			err = errors.E("Get", err)
		}
		return bytes, nil, err
	}
	return nil, []upspin.Location{loc}, nil
}

// innerGet gets a local file descriptor or a new location for the reference. It returns only one of the two return
// values or an error. file is non-nil when the ref is found locally; the file is open for read and the
// caller should close it. If location is non-zero ref is in the backend at that location.
func (s *server) innerGet(userName upspin.UserName, ref upspin.Reference) (file *os.File, location upspin.Location, err error) {
	const Get = "Get"
	if !s.isConfigured() {
		return nil, upspin.Location{}, errors.E(Get, errNotConfigured)
	}
	mu.RLock()
	defer mu.RUnlock()
	file, err = fileCache.OpenRefForRead(string(ref))
	if err == nil {
		// Ref is in the local cache. Send the file and be done.
		log.Debug.Printf("ref %s is in local cache. Returning it as file: %s", ref, file.Name())
		return
	}

	// File is not local, try to get it from our storage.
	var link string
	link, err = cloudClient.Get(string(ref))
	if err != nil {
		err = errors.E(Get, err)
		return
	}
	// GCP should return an http link
	if !strings.HasPrefix(link, "http") {
		err = errors.E(Get, errors.Errorf("invalid link returned from GCP: %s", link))
		log.Error.Println(err)
		return
	}

	url, err := url.Parse(link)
	if err != nil {
		err = errors.E(Get, errors.Errorf("can't parse url: %s: %s", link, err))
		log.Error.Print(err)
		return
	}
	location.Reference = upspin.Reference(link)
	// Go fetch using the provided link. NetAddr is important so we can both ping the server and also cache the
	// HTTPS transport client efficiently.
	location.Endpoint.Transport = upspin.HTTPS
	location.Endpoint.NetAddr = upspin.NetAddr(fmt.Sprintf("%s://%s", url.Scheme, url.Host))
	log.Debug.Printf("Ref %s returned as link: %s", ref, link)
	return
}

// Delete implements upspin.Store.
func (s *server) Delete(ref upspin.Reference) error {
	const Delete = "Delete"
	if !s.isConfigured() {
		return errors.E(Delete, errNotConfigured)
	}
	mu.RLock()
	defer mu.RUnlock()
	// TODO: verify ownership and proper ACLs to delete blob
	err := cloudClient.Delete(string(ref))
	if err != nil {
		return errors.E(Delete, errors.Errorf("%s: %s", ref, err))
	}
	return nil
}

// Dial implements upspin.Service.
func (s *server) Dial(context *upspin.Context, e upspin.Endpoint) (upspin.Service, error) {
	if e.Transport != upspin.GCP {
		return nil, errors.E("Dial", errors.Invalid, errors.Str("unrecognized transport"))
	}

	mu.Lock()
	defer mu.Unlock()
	refCount++

	this := *s              // Clone ourselves.
	this.context = *context // Make a copy of the context, to prevent changes.
	this.endpoint = e
	return &this, nil
}

// Configure configures the connection to the backing store (namely, GCP) once the service
// has been dialed. The details of the configuration are explained at the package comments.
func (s *server) Configure(options ...string) error {
	const Configure = "Configure"
	// These are defaults that only make sense for those running upspin.io.
	bucketName := "g-upspin-store"
	projectID := "upspin"
	tempDir := ""
	for _, option := range options {
		opts := strings.Split(option, "=")
		if len(opts) != 2 {
			return errors.E(Configure, errors.Invalid, errors.Errorf("invalid option format: %q", option))
		}
		switch opts[0] {
		case ConfigBucketName:
			bucketName = opts[1]
		case ConfigProjectID:
			projectID = opts[1]
		case ConfigTemporaryDir:
			tempDir = opts[1]
		default:
			return errors.E(Configure, errors.Invalid, errors.Errorf("invalid configuration option: %q", opts[0]))
		}
	}

	mu.Lock()
	defer mu.Unlock()

	cloudClient = storage.New(projectID, bucketName, storage.PublicRead)
	fileCache = cache.NewFileCache(tempDir)
	if fileCache == nil {
		return errors.E(Configure, errors.Str("filecache failed to create temp directory"))
	}
	log.Debug.Printf("Configured GCP store: %v", options)
	return nil
}

// isConfigured returns whether this server is configured properly.
func (s *server) isConfigured() bool {
	mu.RLock()
	defer mu.RUnlock()
	return cloudClient != nil && fileCache != nil
}

// Ping implements upspin.Service.
func (s *server) Ping() bool {
	return true
}

// Close implements upspin.Service.
func (s *server) Close() {
	mu.Lock()
	defer mu.Unlock()

	if refCount == 0 {
		log.Error.Printf("Closing non-dialed gcp store")
		return
	}
	refCount--

	if refCount == 0 {
		cloudClient = nil
		if fileCache != nil {
			fileCache.Delete()
		}
		fileCache = nil
		// Do other cleanups here.
	}
}

// Authenticate implements upspin.Service.
func (s *server) Authenticate(*upspin.Context) error {
	// Authentication is not dealt here. It happens at other layers.
	return nil
}

// Endpoint implements upspin.Service.
func (s *server) Endpoint() upspin.Endpoint {
	return s.endpoint
}

func init() {
	bind.RegisterStore(upspin.GCP, New(&upspin.Context{}))
}
<commit_msg>store/gcp: fix configuration lock<commit_after>// Copyright 2016 The Upspin Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

// Package gcp implements upspin.Store using Google Cloud Platform as its storage.
package gcp

import (
	"bytes"
	"fmt"
	"io/ioutil"
	"net/url"
	"os"
	"strings"
	"sync"

	"upspin.io/bind"
	"upspin.io/cloud/storage"
	"upspin.io/errors"
	"upspin.io/key/sha256key"
	"upspin.io/log"
	"upspin.io/store/gcp/cache"
	"upspin.io/upspin"
)

// Configuration options for this package.
const (
	// ConfigProjectID specifies which GCP project to use for talking to GCP.
	//
	ConfigProjectID = "gcpProjectId"

	// ConfigBucketName specifies which GCS bucket to store data in.
	// If not specified, "g-upspin-store" is used.
	ConfigBucketName = "gcpBucketName"

	// ConfigTemporaryDir specifies which temporary directory to write files to before they're
	// uploaded to the destination bucket. If not present, one will be created in the
	// system's default location.
	ConfigTemporaryDir = "gcpTemporaryDir"
)

var (
	errNotConfigured = errors.E(errors.Invalid, errors.Str("GCP Store service not configured"))
)

// Server implements upspin.Store.
type server struct {
	context  upspin.Context
	endpoint upspin.Endpoint
}

var _ upspin.Store = (*server)(nil)

var (
	mu          sync.RWMutex // Protects fields below.
	refCount    uint64       // How many clones of us exist.
	cloudClient storage.Storage
	fileCache   *cache.FileCache
)

// New returns a new, unconfigured Store bound to the user in the context.
func New(context *upspin.Context) upspin.Store {
	return &server{
		context: *context, // Make a copy to prevent user making further changes.
	}
}

// Put implements upspin.Store.
func (s *server) Put(data []byte) (upspin.Reference, error) {
	const Put = "Put"
	reader := bytes.NewReader(data)
	// TODO: check that userName has permission to write to this store server.
	mu.RLock()
	if !s.isConfigured() {
		return "", errors.E(Put, errNotConfigured)
	}
	sha := sha256key.NewShaReader(reader)
	initialRef := fileCache.RandomRef()
	err := fileCache.Put(initialRef, sha)
	if err != nil {
		mu.RUnlock()
		return "", errors.E(Put, err)
	}
	// Figure out the appropriate reference for this blob.
	ref := sha.EncodedSum()

	// Rename it in the cache
	fileCache.Rename(ref, initialRef)

	// Now go store it in the cloud.
	go func() {
		if _, err := cloudClient.PutLocalFile(fileCache.GetFileLocation(ref), ref); err == nil {
			// Remove the locally-cached entry so we never
			// keep files locally, as we're a tiny server
			// compared with our much better-provisioned
			// storage backend.  This is safe to do
			// because FileCache is thread safe.
			fileCache.Purge(ref)
		}
		mu.RUnlock()
	}()
	return upspin.Reference(ref), nil
}

// Get implements upspin.Store.
func (s *server) Get(ref upspin.Reference) ([]byte, []upspin.Location, error) {
	file, loc, err := s.innerGet(s.context.UserName, ref)
	if err != nil {
		return nil, nil, err
	}
	if file != nil {
		defer file.Close()
		bytes, err := ioutil.ReadAll(file)
		if err != nil {
			err = errors.E("Get", err)
		}
		return bytes, nil, err
	}
	return nil, []upspin.Location{loc}, nil
}

// innerGet gets a local file descriptor or a new location for the reference. It returns only one of the two return
// values or an error. file is non-nil when the ref is found locally; the file is open for read and the
// caller should close it. If location is non-zero ref is in the backend at that location.
func (s *server) innerGet(userName upspin.UserName, ref upspin.Reference) (file *os.File, location upspin.Location, err error) {
	const Get = "Get"
	mu.RLock()
	defer mu.RUnlock()
	if !s.isConfigured() {
		return nil, upspin.Location{}, errors.E(Get, errNotConfigured)
	}
	file, err = fileCache.OpenRefForRead(string(ref))
	if err == nil {
		// Ref is in the local cache. Send the file and be done.
		log.Debug.Printf("ref %s is in local cache. Returning it as file: %s", ref, file.Name())
		return
	}

	// File is not local, try to get it from our storage.
	var link string
	link, err = cloudClient.Get(string(ref))
	if err != nil {
		err = errors.E(Get, err)
		return
	}
	// GCP should return an http link
	if !strings.HasPrefix(link, "http") {
		err = errors.E(Get, errors.Errorf("invalid link returned from GCP: %s", link))
		log.Error.Println(err)
		return
	}

	url, err := url.Parse(link)
	if err != nil {
		err = errors.E(Get, errors.Errorf("can't parse url: %s: %s", link, err))
		log.Error.Print(err)
		return
	}
	location.Reference = upspin.Reference(link)
	// Go fetch using the provided link. NetAddr is important so we can both ping the server and also cache the
	// HTTPS transport client efficiently.
	location.Endpoint.Transport = upspin.HTTPS
	location.Endpoint.NetAddr = upspin.NetAddr(fmt.Sprintf("%s://%s", url.Scheme, url.Host))
	log.Debug.Printf("Ref %s returned as link: %s", ref, link)
	return
}

// Delete implements upspin.Store.
func (s *server) Delete(ref upspin.Reference) error {
	const Delete = "Delete"
	mu.RLock()
	defer mu.RUnlock()
	if !s.isConfigured() {
		return errors.E(Delete, errNotConfigured)
	}
	// TODO: verify ownership and proper ACLs to delete blob
	err := cloudClient.Delete(string(ref))
	if err != nil {
		return errors.E(Delete, errors.Errorf("%s: %s", ref, err))
	}
	return nil
}

// Dial implements upspin.Service.
func (s *server) Dial(context *upspin.Context, e upspin.Endpoint) (upspin.Service, error) {
	if e.Transport != upspin.GCP {
		return nil, errors.E("Dial", errors.Invalid, errors.Str("unrecognized transport"))
	}

	mu.Lock()
	defer mu.Unlock()
	refCount++

	this := *s              // Clone ourselves.
	this.context = *context // Make a copy of the context, to prevent changes.
	this.endpoint = e
	return &this, nil
}

// Configure configures the connection to the backing store (namely, GCP) once the service
// has been dialed. The details of the configuration are explained at the package comments.
func (s *server) Configure(options ...string) error {
	const Configure = "Configure"
	// These are defaults that only make sense for those running upspin.io.
	bucketName := "g-upspin-store"
	projectID := "upspin"
	tempDir := ""
	for _, option := range options {
		opts := strings.Split(option, "=")
		if len(opts) != 2 {
			return errors.E(Configure, errors.Invalid, errors.Errorf("invalid option format: %q", option))
		}
		switch opts[0] {
		case ConfigBucketName:
			bucketName = opts[1]
		case ConfigProjectID:
			projectID = opts[1]
		case ConfigTemporaryDir:
			tempDir = opts[1]
		default:
			return errors.E(Configure, errors.Invalid, errors.Errorf("invalid configuration option: %q", opts[0]))
		}
	}

	mu.Lock()
	defer mu.Unlock()

	cloudClient = storage.New(projectID, bucketName, storage.PublicRead)
	fileCache = cache.NewFileCache(tempDir)
	if fileCache == nil {
		return errors.E(Configure, errors.Str("filecache failed to create temp directory"))
	}
	log.Debug.Printf("Configured GCP store: %v", options)
	return nil
}

// isConfigured returns whether this server is configured properly.
// It must be called with mu read locked.
func (s *server) isConfigured() bool {
	return cloudClient != nil && fileCache != nil
}

// Ping implements upspin.Service.
func (s *server) Ping() bool {
	return true
}

// Close implements upspin.Service.
func (s *server) Close() {
	mu.Lock()
	defer mu.Unlock()

	if refCount == 0 {
		log.Error.Printf("Closing non-dialed gcp store")
		return
	}
	refCount--

	if refCount == 0 {
		cloudClient = nil
		if fileCache != nil {
			fileCache.Delete()
		}
		fileCache = nil
		// Do other cleanups here.
	}
}

// Authenticate implements upspin.Service.
func (s *server) Authenticate(*upspin.Context) error {
	// Authentication is not dealt here. It happens at other layers.
	return nil
}

// Endpoint implements upspin.Service.
func (s *server) Endpoint() upspin.Endpoint {
	return s.endpoint
}

func init() {
	bind.RegisterStore(upspin.GCP, New(&upspin.Context{}))
}
<|endoftext|>


<commit_before>package mdb

import (
	mdb "github.com/influxdb/gomdb"
	"github.com/siddontang/ledisdb/store/driver"
	"os"
)

type Config struct {
	Path    string `json:"path"`
	MapSize int    `json:"map_size"`
}

type MDB struct {
	env  *mdb.Env
	db   mdb.DBI
	path string
}

func Open(c *Config) (MDB, error) {
	path := c.Path
	if c.MapSize == 0 {
		c.MapSize = 1024 * 1024 * 1024
	}

	env, err := mdb.NewEnv()
	if err != nil {
		return MDB{}, err
	}

	// TODO: max dbs should be configurable
	if err := env.SetMaxDBs(1); err != nil {
		return MDB{}, err
	}
	if err := env.SetMapSize(uint64(c.MapSize)); err != nil {
		return MDB{}, err
	}

	if _, err := os.Stat(path); err != nil {
		err = os.MkdirAll(path, 0755)
		if err != nil {
			return MDB{}, err
		}
	}

	err = env.Open(path, mdb.NOSYNC|mdb.NOMETASYNC|mdb.WRITEMAP|mdb.MAPASYNC|mdb.CREATE, 0755)
	if err != nil {
		return MDB{}, err
	}

	tx, err := env.BeginTxn(nil, 0)
	if err != nil {
		return MDB{}, err
	}

	dbi, err := tx.DBIOpen(nil, mdb.CREATE)
	if err != nil {
		return MDB{}, err
	}

	if err := tx.Commit(); err != nil {
		return MDB{}, err
	}

	db := MDB{
		env:  env,
		db:   dbi,
		path: path,
	}

	return db, nil
}

func Repair(c *Config) error {
	println("llmd not supports repair")
	return nil
}

func (db MDB) Put(key, value []byte) error {
	itr := db.iterator(false)
	defer itr.Close()

	itr.err = itr.c.Put(key, value, 0)
	itr.setState()
	return itr.Error()
}

func (db MDB) BatchPut(writes []Write) error {
	itr := db.iterator(false)

	for _, w := range writes {
		if w.Value == nil {
			itr.key, itr.value, itr.err = itr.c.Get(w.Key, mdb.SET)
			if itr.err == nil {
				itr.err = itr.c.Del(0)
			}
		} else {
			itr.err = itr.c.Put(w.Key, w.Value, 0)
		}

		if itr.err != nil && itr.err != mdb.NotFound {
			break
		}
	}
	itr.setState()

	return itr.Close()
}

func (db MDB) Get(key []byte) ([]byte, error) {
	tx, err := db.env.BeginTxn(nil, mdb.RDONLY)
	if err != nil {
		return nil, err
	}
	defer tx.Commit()

	v, err := tx.Get(db.db, key)
	if err == mdb.NotFound {
		return nil, nil
	}
	return v, err
}

func (db MDB) Delete(key []byte) error {
	itr := db.iterator(false)
	defer itr.Close()

	itr.key, itr.value, itr.err = itr.c.Get(key, mdb.SET)
	if itr.err == nil {
		itr.err = itr.c.Del(0)
	}
	itr.setState()
	return itr.Error()
}

type MDBIterator struct {
	key   []byte
	value []byte
	c     *mdb.Cursor
	tx    *mdb.Txn
	valid bool
	err   error

	closeAutoCommit bool
}

func (itr *MDBIterator) Key() []byte {
	return itr.key
}

func (itr *MDBIterator) Value() []byte {
	return itr.value
}

func (itr *MDBIterator) Valid() bool {
	return itr.valid
}

func (itr *MDBIterator) Error() error {
	return itr.err
}

func (itr *MDBIterator) getCurrent() {
	itr.key, itr.value, itr.err = itr.c.Get(nil, mdb.GET_CURRENT)
	itr.setState()
}

func (itr *MDBIterator) Seek(key []byte) {
	itr.key, itr.value, itr.err = itr.c.Get(key, mdb.SET_RANGE)
	itr.setState()
}
func (itr *MDBIterator) Next() {
	itr.key, itr.value, itr.err = itr.c.Get(nil, mdb.NEXT)
	itr.setState()
}

func (itr *MDBIterator) Prev() {
	itr.key, itr.value, itr.err = itr.c.Get(nil, mdb.PREV)
	itr.setState()
}

func (itr *MDBIterator) First() {
	itr.key, itr.value, itr.err = itr.c.Get(nil, mdb.FIRST)
	itr.setState()
}

func (itr *MDBIterator) Last() {
	itr.key, itr.value, itr.err = itr.c.Get(nil, mdb.LAST)
	itr.setState()
}

func (itr *MDBIterator) setState() {
	if itr.err != nil {
		if itr.err == mdb.NotFound {
			itr.err = nil
		}
		itr.valid = false
	}
}

func (itr *MDBIterator) Close() error {
	if err := itr.c.Close(); err != nil {
		itr.tx.Abort()
		return err
	}

	if !itr.closeAutoCommit {
		return itr.err
	}

	if itr.err != nil {
		itr.tx.Abort()
		return itr.err
	}
	return itr.tx.Commit()
}

func (_ MDB) Name() string {
	return "lmdb"
}

func (db MDB) Path() string {
	return db.path
}

func (db MDB) Compact() {
}

func (db MDB) iterator(rdonly bool) *MDBIterator {
	flags := uint(0)
	if rdonly {
		flags = mdb.RDONLY
	}
	tx, err := db.env.BeginTxn(nil, flags)
	if err != nil {
		return &MDBIterator{nil, nil, nil, nil, false, err, true}
	}

	c, err := tx.CursorOpen(db.db)
	if err != nil {
		tx.Abort()
		return &MDBIterator{nil, nil, nil, nil, false, err, true}
	}

	return &MDBIterator{nil, nil, c, tx, true, nil, true}
}

func (db MDB) Close() error {
	db.env.DBIClose(db.db)
	if err := db.env.Close(); err != nil {
		panic(err)
	}
	return nil
}

func (db MDB) NewIterator() driver.IIterator {
	return db.iterator(true)
}

func (db MDB) NewWriteBatch() driver.IWriteBatch {
	return &WriteBatch{&db, []Write{}}
}

func (db MDB) Begin() (driver.Tx, error) {
	return newTx(db)
}
<commit_msg>lmdv update<commit_after>package mdb

import (
	mdb "github.com/influxdb/gomdb"
	"github.com/siddontang/ledisdb/store/driver"
	"os"
)

type Config struct {
	Path    string `json:"path"`
	MapSize int    `json:"map_size"`
}

type MDB struct {
	env  *mdb.Env
	db   mdb.DBI
	path string
}

func Open(c *Config) (MDB, error) {
	path := c.Path
	if c.MapSize == 0 {
		c.MapSize = 1024 * 1024 * 1024
	}

	env, err := mdb.NewEnv()
	if err != nil {
		return MDB{}, err
	}

	// TODO: max dbs should be configurable
	if err := env.SetMaxDBs(1); err != nil {
		return MDB{}, err
	}

	if err := env.SetMapSize(uint64(c.MapSize)); err != nil {
		return MDB{}, err
	}

	if _, err := os.Stat(path); err != nil {
		err = os.MkdirAll(path, 0755)
		if err != nil {
			return MDB{}, err
		}
	}

	err = env.Open(path, mdb.NOSYNC|mdb.NOMETASYNC|mdb.WRITEMAP|mdb.MAPASYNC|mdb.CREATE, 0755)
	if err != nil {
		return MDB{}, err
	}

	tx, err := env.BeginTxn(nil, 0)
	if err != nil {
		return MDB{}, err
	}

	dbi, err := tx.DBIOpen(nil, mdb.CREATE)
	if err != nil {
		return MDB{}, err
	}

	if err := tx.Commit(); err != nil {
		return MDB{}, err
	}

	db := MDB{
		env:  env,
		db:   dbi,
		path: path,
	}

	return db, nil
}

func Repair(c *Config) error {
	println("llmd not supports repair")
	return nil
}

func (db MDB) Put(key, value []byte) error {
	itr := db.iterator(false)

	itr.err = itr.c.Put(key, value, 0)
	itr.setState()
	return itr.Close()
}

func (db MDB) BatchPut(writes []Write) error {
	itr := db.iterator(false)

	for _, w := range writes {
		if w.Value == nil {
			itr.key, itr.value, itr.err = itr.c.Get(w.Key, mdb.SET)
			if itr.err == nil {
				itr.err = itr.c.Del(0)
			}
		} else {
			itr.err = itr.c.Put(w.Key, w.Value, 0)
		}

		if itr.err != nil && itr.err != mdb.NotFound {
			break
		}
	}
	itr.setState()

	return itr.Close()
}

func (db MDB) Get(key []byte) ([]byte, error) {
	tx, err := db.env.BeginTxn(nil, mdb.RDONLY)
	if err != nil {
		return nil, err
	}
	defer tx.Commit()

	v, err := tx.Get(db.db, key)
	if err == mdb.NotFound {
		return nil, nil
	}
	return v, err
}

func (db MDB) Delete(key []byte) error {
	itr := db.iterator(false)
	defer itr.Close()

	itr.key, itr.value, itr.err = itr.c.Get(key, mdb.SET)
	if itr.err == nil {
		itr.err = itr.c.Del(0)
	}
	itr.setState()
	return itr.Error()
}

type MDBIterator struct {
	key   []byte
	value []byte
	c     *mdb.Cursor
	tx    *mdb.Txn
	valid bool
	err   error

	closeAutoCommit bool
}

func (itr *MDBIterator) Key() []byte {
	return itr.key
}

func (itr *MDBIterator) Value() []byte {
	return itr.value
}

func (itr *MDBIterator) Valid() bool {
	return itr.valid
}

func (itr *MDBIterator) Error() error {
	return itr.err
}

func (itr *MDBIterator) getCurrent() {
	itr.key, itr.value, itr.err = itr.c.Get(nil, mdb.GET_CURRENT)
	itr.setState()
}

func (itr *MDBIterator) Seek(key []byte) {
	itr.key, itr.value, itr.err = itr.c.Get(key, mdb.SET_RANGE)
	itr.setState()
}
func (itr *MDBIterator) Next() {
	itr.key, itr.value, itr.err = itr.c.Get(nil, mdb.NEXT)
	itr.setState()
}

func (itr *MDBIterator) Prev() {
	itr.key, itr.value, itr.err = itr.c.Get(nil, mdb.PREV)
	itr.setState()
}

func (itr *MDBIterator) First() {
	itr.key, itr.value, itr.err = itr.c.Get(nil, mdb.FIRST)
	itr.setState()
}

func (itr *MDBIterator) Last() {
	itr.key, itr.value, itr.err = itr.c.Get(nil, mdb.LAST)
	itr.setState()
}

func (itr *MDBIterator) setState() {
	if itr.err != nil {
		if itr.err == mdb.NotFound {
			itr.err = nil
		}
		itr.valid = false
	}
}

func (itr *MDBIterator) Close() error {
	if err := itr.c.Close(); err != nil {
		itr.tx.Abort()
		return err
	}

	if !itr.closeAutoCommit {
		return itr.err
	}

	if itr.err != nil {
		itr.tx.Abort()
		return itr.err
	}
	return itr.tx.Commit()
}

func (_ MDB) Name() string {
	return "lmdb"
}

func (db MDB) Path() string {
	return db.path
}

func (db MDB) Compact() {
}

func (db MDB) iterator(rdonly bool) *MDBIterator {
	flags := uint(0)
	if rdonly {
		flags = mdb.RDONLY
	}
	tx, err := db.env.BeginTxn(nil, flags)
	if err != nil {
		return &MDBIterator{nil, nil, nil, nil, false, err, true}
	}

	c, err := tx.CursorOpen(db.db)
	if err != nil {
		tx.Abort()
		return &MDBIterator{nil, nil, nil, nil, false, err, true}
	}

	return &MDBIterator{nil, nil, c, tx, true, nil, true}
}

func (db MDB) Close() error {
	db.env.DBIClose(db.db)
	if err := db.env.Close(); err != nil {
		panic(err)
	}
	return nil
}

func (db MDB) NewIterator() driver.IIterator {
	return db.iterator(true)
}

func (db MDB) NewWriteBatch() driver.IWriteBatch {
	return &WriteBatch{&db, []Write{}}
}

func (db MDB) Begin() (driver.Tx, error) {
	return newTx(db)
}
<|endoftext|>


<commit_before>package strftime

import (
	"fmt"
	"testing"
	"time"
)

func ExampleFormat() {
	t := time.Unix(1340244776, 0)
	utc, _ := time.LoadLocation("UTC")
	t = t.In(utc)
	fmt.Println(Format("%Y-%m-%d %H:%M:%S", t))
	// Output:
	// 2012-06-21 02:12:56
}

func TestNoLeadingPercentSign(t *testing.T) {
	tm := time.Unix(1340244776, 0)
	utc, _ := time.LoadLocation("UTC")
	tm = tm.In(utc)
	result := Format("aaabbb0123456789%Y", tm)
	if result != "aaabbb01234567892012" {
		t.Logf("%s != %s", result, "aaabbb01234567892012")
		t.Fail()
	}
}

func TestUnsupported(t *testing.T) {
	tm := time.Unix(1340244776, 0)
	utc, _ := time.LoadLocation("UTC")
	tm = tm.In(utc)
	result := Format("%0%1%%%2", tm)
	if result != "%0%1%%2" {
		t.Logf("%s != %s", result, "%0%1%%2")
		t.Fail()
	}
}

func TestRubyStftime(t *testing.T) {
	tm := time.Unix(1340244776, 0)
	utc, _ := time.LoadLocation("UTC")
	tm = tm.In(utc)
	result := Format("%H:%M:%S%L", tm)
	if result != "02:12:56.000" {
		t.Logf("%s != %s", result, "02:12:56.000")
		t.Fail()
	}
}
<commit_msg>fix typo in Ruby strftime test function name<commit_after>package strftime

import (
	"fmt"
	"testing"
	"time"
)

func ExampleFormat() {
	t := time.Unix(1340244776, 0)
	utc, _ := time.LoadLocation("UTC")
	t = t.In(utc)
	fmt.Println(Format("%Y-%m-%d %H:%M:%S", t))
	// Output:
	// 2012-06-21 02:12:56
}

func TestNoLeadingPercentSign(t *testing.T) {
	tm := time.Unix(1340244776, 0)
	utc, _ := time.LoadLocation("UTC")
	tm = tm.In(utc)
	result := Format("aaabbb0123456789%Y", tm)
	if result != "aaabbb01234567892012" {
		t.Logf("%s != %s", result, "aaabbb01234567892012")
		t.Fail()
	}
}

func TestUnsupported(t *testing.T) {
	tm := time.Unix(1340244776, 0)
	utc, _ := time.LoadLocation("UTC")
	tm = tm.In(utc)
	result := Format("%0%1%%%2", tm)
	if result != "%0%1%%2" {
		t.Logf("%s != %s", result, "%0%1%%2")
		t.Fail()
	}
}

func TestRubyStrftime(t *testing.T) {
	tm := time.Unix(1340244776, 0)
	utc, _ := time.LoadLocation("UTC")
	tm = tm.In(utc)
	result := Format("%H:%M:%S%L", tm)
	if result != "02:12:56.000" {
		t.Logf("%s != %s", result, "02:12:56.000")
		t.Fail()
	}
}
<|endoftext|>


<commit_before>package app

import (
	"fmt"
	"github.com/heysquirrel/tribe/git"
	tlog "github.com/heysquirrel/tribe/log"
	"github.com/jroimartin/gocui"
	"github.com/olekukonko/tablewriter"
	"log"
	"strconv"
	"time"
)

const (
	changesView         = "changes"
	contributorsView    = "contributors"
	associatedFilesView = "associatedfiles"
	logsView            = "logs"
	feedView            = "feed"
	relevantWorkView    = "relevantwork"
	legendView          = "legend"
	debugView           = "debug"
)

type View struct {
	title      string
	text       string
	x1         float64
	y1         float64
	x2         float64
	y2         float64
	highlight  bool
	hidden     bool
	selBgColor gocui.Attribute
	selFgColor gocui.Attribute
}

var views = map[string]View{
	changesView: {
		title:      "Last Changed",
		text:       "",
		x1:         0.0,
		y1:         0.0,
		x2:         0.3,
		y2:         0.2,
		highlight:  true,
		selBgColor: gocui.ColorCyan,
		selFgColor: gocui.ColorBlack,
	},
	contributorsView: {
		title: "Recent Contributors",
		text:  "",
		x1:    0.3,
		y1:    0.0,
		x2:    1.0,
		y2:    0.2,
	},
	associatedFilesView: {
		title: "Associated Files",
		text:  "",
		x1:    0.0,
		y1:    0.2,
		x2:    0.3,
		y2:    0.4,
	},
	logsView: {
		title: "Logs",
		text:  "",
		x1:    0.3,
		y1:    0.2,
		x2:    1.0,
		y2:    0.45,
	},
	feedView: {
		title: "Feed",
		text:  "",
		x1:    0.0,
		y1:    0.4,
		x2:    0.3,
		y2:    0.99,
	},
	relevantWorkView: {
		title: "Relevant Work Items",
		text:  "",
		x1:    0.3,
		y1:    0.45,
		x2:    1.0,
		y2:    0.9,
	},
	legendView: {
		title: "Legend",
		text:  "",
		x1:    0.3,
		y1:    0.9,
		x2:    1.0,
		y2:    0.99,
	},
	debugView: {
		title:  "Debug",
		text:   "",
		x1:     0.25,
		y1:     0.25,
		x2:     0.7,
		y2:     0.7,
		hidden: true,
	},
}

var defaultViews = []string{
	changesView,
	contributorsView,
	associatedFilesView,
	logsView,
	feedView,
	relevantWorkView,
	legendView,
	debugView,
}

func (a *App) Layout(g *gocui.Gui) error {
	maxX, maxY := g.Size()

	for _, name := range defaultViews {
		view := views[name]

		x1 := int(view.x1 * float64(maxX))
		y1 := int(view.y1 * float64(maxY))
		x2 := int(view.x2*float64(maxX)) - 1
		y2 := int(view.y2*float64(maxY)) - 1

		v, err := g.SetView(name, x1, y1, x2, y2)
		if err != gocui.ErrUnknownView {
			return err
		}

		v.Title = view.title
		if view.highlight {
			v.Highlight = view.highlight
			v.SelBgColor = view.selBgColor
			v.SelFgColor = view.selFgColor
		}
		fmt.Fprint(v, view.text)

		if view.hidden {
			g.SetViewOnBottom(name)
		}
	}

	_, err := g.SetCurrentView(changesView)
	if err != nil {
		return err
	}

	return a.setKeyBindings()
}

func (a *App) currentFileSelection() string {
	v, err := a.Gui.View(changesView)
	if err != nil {
		log.Panicln(err)
	}

	_, cy := v.Cursor()
	file, err := v.Line(cy)
	if err != nil {
		file = ""
	}

	return file
}

func (a *App) UpdateChanges(files []string) {
	a.updateView(changesView, func(v *gocui.View) {
		for _, file := range files {
			fmt.Fprintln(v, file)
		}
		a.currentFileChanged()
	})
}

func (a *App) UpdateDebug(entries []*tlog.LogEntry) {
	a.updateView(debugView, func(v *gocui.View) {
		table := tablewriter.NewWriter(v)
		table.SetHeader([]string{"Created At", "Message"})
		table.SetBorder(false)

		for _, entry := range entries {
			createdAt := entry.CreatedAt.UTC().Format(time.UnixDate)
			table.Append([]string{createdAt, entry.Message})
		}

		table.Render()
	})
}

func (a *App) UpdateContributors(contributors []*git.Contributor) {
	a.updateView(contributorsView, func(v *gocui.View) {
		table := tablewriter.NewWriter(v)
		table.SetHeader([]string{"Name", "Commits", "Last Commit"})
		table.SetBorder(false)

		for _, contributor := range contributors {
			table.Append([]string{contributor.Name, strconv.Itoa(contributor.Count), contributor.RelativeDate})
		}

		table.Render()
	})
}

func (a *App) updateView(view string, fn func(*gocui.View)) {
	a.Gui.Update(func(g *gocui.Gui) error {
		v, err := g.View(view)
		if err != nil {
			return err
		}
		v.Clear()

		fn(v)

		return nil
	})
}
<commit_msg>Make the debug window a little wider<commit_after>package app

import (
	"fmt"
	"github.com/heysquirrel/tribe/git"
	tlog "github.com/heysquirrel/tribe/log"
	"github.com/jroimartin/gocui"
	"github.com/olekukonko/tablewriter"
	"log"
	"strconv"
	"time"
)

const (
	changesView         = "changes"
	contributorsView    = "contributors"
	associatedFilesView = "associatedfiles"
	logsView            = "logs"
	feedView            = "feed"
	relevantWorkView    = "relevantwork"
	legendView          = "legend"
	debugView           = "debug"
)

type View struct {
	title      string
	text       string
	x1         float64
	y1         float64
	x2         float64
	y2         float64
	highlight  bool
	hidden     bool
	selBgColor gocui.Attribute
	selFgColor gocui.Attribute
}

var views = map[string]View{
	changesView: {
		title:      "Last Changed",
		text:       "",
		x1:         0.0,
		y1:         0.0,
		x2:         0.3,
		y2:         0.2,
		highlight:  true,
		selBgColor: gocui.ColorCyan,
		selFgColor: gocui.ColorBlack,
	},
	contributorsView: {
		title: "Recent Contributors",
		text:  "",
		x1:    0.3,
		y1:    0.0,
		x2:    1.0,
		y2:    0.2,
	},
	associatedFilesView: {
		title: "Associated Files",
		text:  "",
		x1:    0.0,
		y1:    0.2,
		x2:    0.3,
		y2:    0.4,
	},
	logsView: {
		title: "Logs",
		text:  "",
		x1:    0.3,
		y1:    0.2,
		x2:    1.0,
		y2:    0.45,
	},
	feedView: {
		title: "Feed",
		text:  "",
		x1:    0.0,
		y1:    0.4,
		x2:    0.3,
		y2:    0.99,
	},
	relevantWorkView: {
		title: "Relevant Work Items",
		text:  "",
		x1:    0.3,
		y1:    0.45,
		x2:    1.0,
		y2:    0.9,
	},
	legendView: {
		title: "Legend",
		text:  "",
		x1:    0.3,
		y1:    0.9,
		x2:    1.0,
		y2:    0.99,
	},
	debugView: {
		title:  "Debug",
		text:   "",
		x1:     0.25,
		y1:     0.25,
		x2:     0.8,
		y2:     0.8,
		hidden: true,
	},
}

var defaultViews = []string{
	changesView,
	contributorsView,
	associatedFilesView,
	logsView,
	feedView,
	relevantWorkView,
	legendView,
	debugView,
}

func (a *App) Layout(g *gocui.Gui) error {
	maxX, maxY := g.Size()

	for _, name := range defaultViews {
		view := views[name]

		x1 := int(view.x1 * float64(maxX))
		y1 := int(view.y1 * float64(maxY))
		x2 := int(view.x2*float64(maxX)) - 1
		y2 := int(view.y2*float64(maxY)) - 1

		v, err := g.SetView(name, x1, y1, x2, y2)
		if err != gocui.ErrUnknownView {
			return err
		}

		v.Title = view.title
		if view.highlight {
			v.Highlight = view.highlight
			v.SelBgColor = view.selBgColor
			v.SelFgColor = view.selFgColor
		}
		fmt.Fprint(v, view.text)

		if view.hidden {
			g.SetViewOnBottom(name)
		}
	}

	_, err := g.SetCurrentView(changesView)
	if err != nil {
		return err
	}

	return a.setKeyBindings()
}

func (a *App) currentFileSelection() string {
	v, err := a.Gui.View(changesView)
	if err != nil {
		log.Panicln(err)
	}

	_, cy := v.Cursor()
	file, err := v.Line(cy)
	if err != nil {
		file = ""
	}

	return file
}

func (a *App) UpdateChanges(files []string) {
	a.updateView(changesView, func(v *gocui.View) {
		for _, file := range files {
			fmt.Fprintln(v, file)
		}
		a.currentFileChanged()
	})
}

func (a *App) UpdateDebug(entries []*tlog.LogEntry) {
	a.updateView(debugView, func(v *gocui.View) {
		table := tablewriter.NewWriter(v)
		table.SetHeader([]string{"Created At", "Message"})
		table.SetBorder(false)

		for _, entry := range entries {
			createdAt := entry.CreatedAt.UTC().Format(time.UnixDate)
			table.Append([]string{createdAt, entry.Message})
		}

		table.Render()
	})
}

func (a *App) UpdateContributors(contributors []*git.Contributor) {
	a.updateView(contributorsView, func(v *gocui.View) {
		table := tablewriter.NewWriter(v)
		table.SetHeader([]string{"Name", "Commits", "Last Commit"})
		table.SetBorder(false)

		for _, contributor := range contributors {
			table.Append([]string{contributor.Name, strconv.Itoa(contributor.Count), contributor.RelativeDate})
		}

		table.Render()
	})
}

func (a *App) updateView(view string, fn func(*gocui.View)) {
	a.Gui.Update(func(g *gocui.Gui) error {
		v, err := g.View(view)
		if err != nil {
			return err
		}
		v.Clear()

		fn(v)

		return nil
	})
}
<|endoftext|>


<commit_before>// Copyright 2019 Google LLC
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     https://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package main

import (
	"bytes"
	"context"
	"strings"
	"testing"

	"cloud.google.com/go/bigquery"
	"cloud.google.com/go/datastore"
	"cloud.google.com/go/storage"
	"github.com/GoogleCloudPlatform/golang-samples/internal/testutil"
	dlppb "google.golang.org/genproto/googleapis/privacy/dlp/v2"
)

const (
	inspectTopicName        = "dlp-inspect-test-topic"
	inspectSubscriptionName = "dlp-inspect-test-sub"
)

func TestInspectString(t *testing.T) {
	testutil.SystemTest(t)
	tests := []struct {
		s    string
		want bool
	}{
		{
			s:    "My SSN is 111222333",
			want: true,
		},
		{
			s: "Does not match",
		},
	}
	for _, test := range tests {
		buf := new(bytes.Buffer)
		inspectString(buf, client, projectID, dlppb.Likelihood_POSSIBLE, 0, true, []string{"US_SOCIAL_SECURITY_NUMBER"}, nil, nil, test.s)
		if got := buf.String(); test.want != strings.Contains(got, "US_SOCIAL_SECURITY_NUMBER") {
			if test.want {
				t.Errorf("inspectString(%s) = %q, want 'US_SOCIAL_SECURITY_NUMBER' substring", test.s, got)
			} else {
				t.Errorf("inspectString(%s) = %q, want to not contain 'US_SOCIAL_SECURITY_NUMBER'", test.s, got)
			}
		}
		buf.Reset()
		inspectString(buf, client, projectID, dlppb.Likelihood_POSSIBLE, 0, true, nil, []string{"SSN"}, []string{"\\d{9}"}, test.s)
		if got := buf.String(); test.want != strings.Contains(got, "CUSTOM_DICTIONARY_0") && strings.Contains(got, "CUSTOM_REGEX_0") {
			if test.want {
				t.Errorf("inspectString(%s) = %q, want 'CUSTOM_DICTIONARY_0' and 'CUSTOM_REGEX_0' substring", test.s, got)
			} else {
				t.Errorf("inspectString(%s) = %q, want to not contain 'CUSTOM_DICTIONARY_0' and 'CUSTOM_REGEX_0'", test.s, got)
			}
		}
	}
}

func TestInspectFile(t *testing.T) {
	testutil.SystemTest(t)
	tests := []struct {
		s    string
		want bool
	}{
		{
			s:    "My SSN is 111222333",
			want: true,
		},
		{
			s: "Does not match",
		},
	}
	for _, test := range tests {
		buf := new(bytes.Buffer)
		inspectFile(buf, client, projectID, dlppb.Likelihood_POSSIBLE, 0, true, []string{"US_SOCIAL_SECURITY_NUMBER"}, nil, nil, dlppb.ByteContentItem_TEXT_UTF8, strings.NewReader(test.s))
		if got := buf.String(); test.want != strings.Contains(got, "US_SOCIAL_SECURITY_NUMBER") {
			if test.want {
				t.Errorf("inspectString(%s) = %q, want 'US_SOCIAL_SECURITY_NUMBER' substring", test.s, got)
			} else {
				t.Errorf("inspectString(%s) = %q, want to not contain 'US_SOCIAL_SECURITY_NUMBER'", test.s, got)
			}
		}
		buf.Reset()
		inspectFile(buf, client, projectID, dlppb.Likelihood_POSSIBLE, 0, true, nil, []string{"SSN"}, []string{"\\d{9}"}, dlppb.ByteContentItem_TEXT_UTF8, strings.NewReader(test.s))
		if got := buf.String(); test.want != strings.Contains(got, "CUSTOM_DICTIONARY_0") && strings.Contains(got, "CUSTOM_REGEX_0") {
			if test.want {
				t.Errorf("inspectString(%s) = %q, want 'CUSTOM_DICTIONARY_0' and 'CUSTOM_REGEX_0' substring", test.s, got)
			} else {
				t.Errorf("inspectString(%s) = %q, want to not contain 'CUSTOM_DICTIONARY_0' and 'CUSTOM_REGEX_0'", test.s, got)
			}
		}
	}
}

const (
	ssnFileName             = "fake_ssn.txt"
	nothingEventfulFileName = "nothing_eventful.txt"
	bucketName              = "golang-samples-dlp-test"
)

func writeTestGCSFiles(t *testing.T, projectID string) {
	ctx := context.Background()
	client, err := storage.NewClient(ctx)
	if err != nil {
		t.Fatalf("Failed to create client: %v", err)
	}
	bucket := client.Bucket(bucketName)
	_, err = bucket.Attrs(ctx)
	if err != nil {
		switch err {
		case storage.ErrObjectNotExist:
			if err := bucket.Create(ctx, projectID, nil); err != nil {
				t.Fatalf("Failed to create bucket: %v", err)
			}
		default:
			t.Fatalf("error getting bucket attrs: %v", err)
		}
	}
	if err := writeObject(ctx, bucket, ssnFileName, "My SSN is 111222333"); err != nil {
		t.Fatalf("error writing object: %v", err)
	}
	if err := writeObject(ctx, bucket, nothingEventfulFileName, "Nothing eventful"); err != nil {
		t.Fatalf("error writing object: %v", err)
	}
}

func writeObject(ctx context.Context, bucket *storage.BucketHandle, fileName, content string) error {
	obj := bucket.Object(fileName)
	_, err := obj.Attrs(ctx)
	if err != nil {
		switch err {
		case storage.ErrObjectNotExist:
			w := obj.NewWriter(ctx)
			w.Write([]byte(content))
			if err := w.Close(); err != nil {
				return err
			}
		default:
			return err
		}
	}
	return nil
}

func TestInspectGCS(t *testing.T) {
	testutil.SystemTest(t)
	writeTestGCSFiles(t, projectID)
	tests := []struct {
		fileName string
		want     string
	}{
		{
			fileName: ssnFileName,
			want:     "US_SOCIAL_SECURITY_NUMBER",
		},
		{
			fileName: nothingEventfulFileName,
			want:     "No results",
		},
	}
	for _, test := range tests {
		buf := new(bytes.Buffer)
		inspectGCSFile(buf, client, projectID, dlppb.Likelihood_POSSIBLE, 0, true, []string{"US_SOCIAL_SECURITY_NUMBER"}, []string{}, []string{}, inspectTopicName, inspectSubscriptionName, bucketName, test.fileName)
		if got := buf.String(); !strings.Contains(got, test.want) {
			t.Errorf("inspectString(%s) = %q, want %q substring", test.fileName, got, test.want)
		}
	}
}

type SSNTask struct {
	Description string
}

type BoringTask struct {
	Description string
}

func writeTestDatastoreFiles(t *testing.T, projectID string) {
	ctx := context.Background()
	client, err := datastore.NewClient(ctx, projectID)
	if err != nil {
		t.Fatalf("Failed to create client: %v", err)
	}
	kind := "SSNTask"
	name := "ssntask1"
	ssnKey := datastore.NameKey(kind, name, nil)
	task := SSNTask{
		Description: "My SSN is 111222333",
	}
	if _, err := client.Put(ctx, ssnKey, &task); err != nil {
		t.Fatalf("Failed to save task: %v", err)
	}

	kind = "BoringTask"
	name = "boringtask1"
	boringKey := datastore.NameKey(kind, name, nil)
	boringTask := BoringTask{
		Description: "Nothing meaningful",
	}
	if _, err := client.Put(ctx, boringKey, &boringTask); err != nil {
		t.Fatalf("Failed to save task: %v", err)
	}
}

func TestInspectDatastore(t *testing.T) {
	testutil.SystemTest(t)
	writeTestDatastoreFiles(t, projectID)
	tests := []struct {
		kind string
		want string
	}{
		{
			kind: "SSNTask",
			want: "US_SOCIAL_SECURITY_NUMBER",
		},
		{
			kind: "BoringTask",
			want: "No results",
		},
	}
	for _, test := range tests {
		buf := new(bytes.Buffer)
		inspectDatastore(buf, client, projectID, dlppb.Likelihood_POSSIBLE, 0, true, []string{"US_SOCIAL_SECURITY_NUMBER"}, []string{}, []string{}, inspectTopicName, inspectSubscriptionName, projectID, "", test.kind)
		if got := buf.String(); !strings.Contains(got, test.want) {
			t.Errorf("inspectDatastore(%s) = %q, want %q substring", test.kind, got, test.want)
		}
	}
}

type Item struct {
	Description string
}

const (
	harmlessTable = "harmless"
	harmfulTable  = "harmful"
	bqDatasetID   = "golang_samples_dlp"
)

func createBigqueryTestFiles(projectID, datasetID string) error {
	ctx := context.Background()
	client, err := bigquery.NewClient(ctx, projectID)
	if err != nil {
		return err
	}
	defer client.Close()
	d := client.Dataset(datasetID)
	if _, err := d.Metadata(ctx); err != nil {
		if err := d.Create(ctx, &bigquery.DatasetMetadata{}); err != nil {
			return err
		}
	}
	schema, err := bigquery.InferSchema(Item{})
	if err != nil {
		return err
	}
	if err := uploadBigQuery(ctx, d, schema, harmlessTable, "Nothing meaningful"); err != nil {
		return err
	}
	return uploadBigQuery(ctx, d, schema, harmfulTable, "My SSN is 111222333")
}

func uploadBigQuery(ctx context.Context, d *bigquery.Dataset, schema bigquery.Schema, table, content string) error {
	t := d.Table(table)
	if _, err := t.Metadata(ctx); err == nil {
		return nil
	}
	if err := t.Create(ctx, &bigquery.TableMetadata{Schema: schema}); err != nil {
		return err
	}
	source := bigquery.NewReaderSource(strings.NewReader(content))
	l := t.LoaderFrom(source)
	job, err := l.Run(ctx)
	if err != nil {
		return err
	}
	status, err := job.Wait(ctx)
	if err != nil {
		return err
	}
	return status.Err()
}

func TestInspectBigquery(t *testing.T) {
	testutil.SystemTest(t)
	if err := createBigqueryTestFiles(projectID, bqDatasetID); err != nil {
		t.Fatalf("error creating test BigQuery files: %v", err)
	}
	tests := []struct {
		table string
		want  string
	}{
		{
			table: harmfulTable,
			want:  "US_SOCIAL_SECURITY_NUMBER",
		},
		{
			table: harmlessTable,
			want:  "No results",
		},
	}
	for _, test := range tests {
		buf := new(bytes.Buffer)
		inspectBigquery(buf, client, projectID, dlppb.Likelihood_POSSIBLE, 0, true, []string{"US_SOCIAL_SECURITY_NUMBER"}, []string{}, []string{}, inspectTopicName, inspectSubscriptionName, projectID, bqDatasetID, test.table)
		if got := buf.String(); !strings.Contains(got, test.want) {
			t.Errorf("inspectBigquery(%s) = %q, want %q substring", test.table, got, test.want)
		}
	}
}
<commit_msg>dlp: make BQ & Datastore tests E2E (#770)<commit_after>// Copyright 2019 Google LLC
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     https://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package main

import (
	"bytes"
	"context"
	"strings"
	"testing"

	"cloud.google.com/go/bigquery"
	"cloud.google.com/go/datastore"
	"cloud.google.com/go/storage"
	"github.com/GoogleCloudPlatform/golang-samples/internal/testutil"
	dlppb "google.golang.org/genproto/googleapis/privacy/dlp/v2"
)

const (
	inspectTopicName        = "dlp-inspect-test-topic"
	inspectSubscriptionName = "dlp-inspect-test-sub"
)

func TestInspectString(t *testing.T) {
	testutil.SystemTest(t)
	tests := []struct {
		s    string
		want bool
	}{
		{
			s:    "My SSN is 111222333",
			want: true,
		},
		{
			s: "Does not match",
		},
	}
	for _, test := range tests {
		buf := new(bytes.Buffer)
		inspectString(buf, client, projectID, dlppb.Likelihood_POSSIBLE, 0, true, []string{"US_SOCIAL_SECURITY_NUMBER"}, nil, nil, test.s)
		if got := buf.String(); test.want != strings.Contains(got, "US_SOCIAL_SECURITY_NUMBER") {
			if test.want {
				t.Errorf("inspectString(%s) = %q, want 'US_SOCIAL_SECURITY_NUMBER' substring", test.s, got)
			} else {
				t.Errorf("inspectString(%s) = %q, want to not contain 'US_SOCIAL_SECURITY_NUMBER'", test.s, got)
			}
		}
		buf.Reset()
		inspectString(buf, client, projectID, dlppb.Likelihood_POSSIBLE, 0, true, nil, []string{"SSN"}, []string{"\\d{9}"}, test.s)
		if got := buf.String(); test.want != strings.Contains(got, "CUSTOM_DICTIONARY_0") && strings.Contains(got, "CUSTOM_REGEX_0") {
			if test.want {
				t.Errorf("inspectString(%s) = %q, want 'CUSTOM_DICTIONARY_0' and 'CUSTOM_REGEX_0' substring", test.s, got)
			} else {
				t.Errorf("inspectString(%s) = %q, want to not contain 'CUSTOM_DICTIONARY_0' and 'CUSTOM_REGEX_0'", test.s, got)
			}
		}
	}
}

func TestInspectFile(t *testing.T) {
	testutil.SystemTest(t)
	tests := []struct {
		s    string
		want bool
	}{
		{
			s:    "My SSN is 111222333",
			want: true,
		},
		{
			s: "Does not match",
		},
	}
	for _, test := range tests {
		buf := new(bytes.Buffer)
		inspectFile(buf, client, projectID, dlppb.Likelihood_POSSIBLE, 0, true, []string{"US_SOCIAL_SECURITY_NUMBER"}, nil, nil, dlppb.ByteContentItem_TEXT_UTF8, strings.NewReader(test.s))
		if got := buf.String(); test.want != strings.Contains(got, "US_SOCIAL_SECURITY_NUMBER") {
			if test.want {
				t.Errorf("inspectString(%s) = %q, want 'US_SOCIAL_SECURITY_NUMBER' substring", test.s, got)
			} else {
				t.Errorf("inspectString(%s) = %q, want to not contain 'US_SOCIAL_SECURITY_NUMBER'", test.s, got)
			}
		}
		buf.Reset()
		inspectFile(buf, client, projectID, dlppb.Likelihood_POSSIBLE, 0, true, nil, []string{"SSN"}, []string{"\\d{9}"}, dlppb.ByteContentItem_TEXT_UTF8, strings.NewReader(test.s))
		if got := buf.String(); test.want != strings.Contains(got, "CUSTOM_DICTIONARY_0") && strings.Contains(got, "CUSTOM_REGEX_0") {
			if test.want {
				t.Errorf("inspectString(%s) = %q, want 'CUSTOM_DICTIONARY_0' and 'CUSTOM_REGEX_0' substring", test.s, got)
			} else {
				t.Errorf("inspectString(%s) = %q, want to not contain 'CUSTOM_DICTIONARY_0' and 'CUSTOM_REGEX_0'", test.s, got)
			}
		}
	}
}

const (
	ssnFileName             = "fake_ssn.txt"
	nothingEventfulFileName = "nothing_eventful.txt"
	bucketName              = "golang-samples-dlp-test"
)

func writeTestGCSFiles(t *testing.T, projectID string) {
	ctx := context.Background()
	client, err := storage.NewClient(ctx)
	if err != nil {
		t.Fatalf("Failed to create client: %v", err)
	}
	bucket := client.Bucket(bucketName)
	_, err = bucket.Attrs(ctx)
	if err != nil {
		switch err {
		case storage.ErrObjectNotExist:
			if err := bucket.Create(ctx, projectID, nil); err != nil {
				t.Fatalf("Failed to create bucket: %v", err)
			}
		default:
			t.Fatalf("error getting bucket attrs: %v", err)
		}
	}
	if err := writeObject(ctx, bucket, ssnFileName, "My SSN is 111222333"); err != nil {
		t.Fatalf("error writing object: %v", err)
	}
	if err := writeObject(ctx, bucket, nothingEventfulFileName, "Nothing eventful"); err != nil {
		t.Fatalf("error writing object: %v", err)
	}
}

func writeObject(ctx context.Context, bucket *storage.BucketHandle, fileName, content string) error {
	obj := bucket.Object(fileName)
	_, err := obj.Attrs(ctx)
	if err != nil {
		switch err {
		case storage.ErrObjectNotExist:
			w := obj.NewWriter(ctx)
			w.Write([]byte(content))
			if err := w.Close(); err != nil {
				return err
			}
		default:
			return err
		}
	}
	return nil
}

func TestInspectGCS(t *testing.T) {
	testutil.SystemTest(t)
	writeTestGCSFiles(t, projectID)
	tests := []struct {
		fileName string
		want     string
	}{
		{
			fileName: ssnFileName,
			want:     "US_SOCIAL_SECURITY_NUMBER",
		},
		{
			fileName: nothingEventfulFileName,
			want:     "No results",
		},
	}
	for _, test := range tests {
		buf := new(bytes.Buffer)
		inspectGCSFile(buf, client, projectID, dlppb.Likelihood_POSSIBLE, 0, true, []string{"US_SOCIAL_SECURITY_NUMBER"}, []string{}, []string{}, inspectTopicName, inspectSubscriptionName, bucketName, test.fileName)
		if got := buf.String(); !strings.Contains(got, test.want) {
			t.Errorf("inspectString(%s) = %q, want %q substring", test.fileName, got, test.want)
		}
	}
}

type SSNTask struct {
	Description string
}

type BoringTask struct {
	Description string
}

func writeTestDatastoreFiles(t *testing.T, projectID string) {
	ctx := context.Background()
	client, err := datastore.NewClient(ctx, projectID)
	if err != nil {
		t.Fatalf("Failed to create client: %v", err)
	}
	kind := "SSNTask"
	name := "ssntask1"
	ssnKey := datastore.NameKey(kind, name, nil)
	task := SSNTask{
		Description: "My SSN is 111222333",
	}
	if _, err := client.Put(ctx, ssnKey, &task); err != nil {
		t.Fatalf("Failed to save task: %v", err)
	}

	kind = "BoringTask"
	name = "boringtask1"
	boringKey := datastore.NameKey(kind, name, nil)
	boringTask := BoringTask{
		Description: "Nothing meaningful",
	}
	if _, err := client.Put(ctx, boringKey, &boringTask); err != nil {
		t.Fatalf("Failed to save task: %v", err)
	}
}

func TestInspectDatastore(t *testing.T) {
	testutil.EndToEndTest(t)
	writeTestDatastoreFiles(t, projectID)
	tests := []struct {
		kind string
		want string
	}{
		{
			kind: "SSNTask",
			want: "US_SOCIAL_SECURITY_NUMBER",
		},
		{
			kind: "BoringTask",
			want: "No results",
		},
	}
	for _, test := range tests {
		buf := new(bytes.Buffer)
		inspectDatastore(buf, client, projectID, dlppb.Likelihood_POSSIBLE, 0, true, []string{"US_SOCIAL_SECURITY_NUMBER"}, []string{}, []string{}, inspectTopicName, inspectSubscriptionName, projectID, "", test.kind)
		if got := buf.String(); !strings.Contains(got, test.want) {
			t.Errorf("inspectDatastore(%s) = %q, want %q substring", test.kind, got, test.want)
		}
	}
}

type Item struct {
	Description string
}

const (
	harmlessTable = "harmless"
	harmfulTable  = "harmful"
	bqDatasetID   = "golang_samples_dlp"
)

func createBigqueryTestFiles(projectID, datasetID string) error {
	ctx := context.Background()
	client, err := bigquery.NewClient(ctx, projectID)
	if err != nil {
		return err
	}
	defer client.Close()
	d := client.Dataset(datasetID)
	if _, err := d.Metadata(ctx); err != nil {
		if err := d.Create(ctx, &bigquery.DatasetMetadata{}); err != nil {
			return err
		}
	}
	schema, err := bigquery.InferSchema(Item{})
	if err != nil {
		return err
	}
	if err := uploadBigQuery(ctx, d, schema, harmlessTable, "Nothing meaningful"); err != nil {
		return err
	}
	return uploadBigQuery(ctx, d, schema, harmfulTable, "My SSN is 111222333")
}

func uploadBigQuery(ctx context.Context, d *bigquery.Dataset, schema bigquery.Schema, table, content string) error {
	t := d.Table(table)
	if _, err := t.Metadata(ctx); err == nil {
		return nil
	}
	if err := t.Create(ctx, &bigquery.TableMetadata{Schema: schema}); err != nil {
		return err
	}
	source := bigquery.NewReaderSource(strings.NewReader(content))
	l := t.LoaderFrom(source)
	job, err := l.Run(ctx)
	if err != nil {
		return err
	}
	status, err := job.Wait(ctx)
	if err != nil {
		return err
	}
	return status.Err()
}

func TestInspectBigquery(t *testing.T) {
	testutil.EndToEndTest(t)
	if err := createBigqueryTestFiles(projectID, bqDatasetID); err != nil {
		t.Fatalf("error creating test BigQuery files: %v", err)
	}
	tests := []struct {
		table string
		want  string
	}{
		{
			table: harmfulTable,
			want:  "US_SOCIAL_SECURITY_NUMBER",
		},
		{
			table: harmlessTable,
			want:  "No results",
		},
	}
	for _, test := range tests {
		buf := new(bytes.Buffer)
		inspectBigquery(buf, client, projectID, dlppb.Likelihood_POSSIBLE, 0, true, []string{"US_SOCIAL_SECURITY_NUMBER"}, []string{}, []string{}, inspectTopicName, inspectSubscriptionName, projectID, bqDatasetID, test.table)
		if got := buf.String(); !strings.Contains(got, test.want) {
			t.Errorf("inspectBigquery(%s) = %q, want %q substring", test.table, got, test.want)
		}
	}
}
<|endoftext|>


<commit_before>/*
Copyright 2015 The Kubernetes Authors.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package controller

import (
	"fmt"
	"reflect"
	"strings"
	"time"

	"github.com/golang/glog"

	"k8s.io/kubernetes/pkg/api"
	"k8s.io/kubernetes/pkg/apis/extensions"
	"k8s.io/kubernetes/pkg/client/cache"

	"k8s.io/ingress/core/pkg/ingress"
	"k8s.io/ingress/core/pkg/ingress/annotations/parser"
	ssl "k8s.io/ingress/core/pkg/net/ssl"
)

// syncSecret keeps in sync Secrets used by Ingress rules with the files on
// disk to allow being used in controllers.
func (ic *GenericController) syncSecret(k interface{}) error {
	if ic.secretQueue.IsShuttingDown() {
		return nil
	}
	if !ic.controllersInSync() {
		time.Sleep(podStoreSyncedPollPeriod)
		return fmt.Errorf("deferring sync till endpoints controller has synced")
	}

	var key string
	var cert *ingress.SSLCert
	var err error

	key = k.(string)

	secObj, exists, err := ic.secrLister.Store.GetByKey(key)
	if err != nil {
		return fmt.Errorf("error getting secret %v: %v", key, err)
	}
	if !exists {
		return fmt.Errorf("secret %v was not found", key)
	}
	sec := secObj.(*api.Secret)
	if !ic.secrReferenced(sec.Name, sec.Namespace) {
		glog.V(3).Infof("secret %v/%v is not used in Ingress rules. skipping ", sec.Namespace, sec.Name)
		return nil
	}

	cert, err = ic.getPemCertificate(key)
	if err != nil {
		return err
	}

	// create certificates and add or update the item in the store
	cur, exists := ic.sslCertTracker.Get(key)
	if exists {
		s := cur.(*ingress.SSLCert)
		if reflect.DeepEqual(s, cert) {
			// no need to update
			return nil
		}
		glog.Infof("updating secret %v/%v in the local store", sec.Namespace, sec.Name)
		ic.sslCertTracker.Update(key, cert)
		return nil
	}
	glog.Infof("adding secret %v/%v to the local store", sec.Namespace, sec.Name)
	ic.sslCertTracker.Add(key, cert)
	return nil
}

// getPemCertificate receives a secret, and creates a ingress.SSLCert as return.
// It parses the secret and verifies if it's a keypair, or a 'ca.crt' secret only.
func (ic *GenericController) getPemCertificate(secretName string) (*ingress.SSLCert, error) {
	secretInterface, exists, err := ic.secrLister.Store.GetByKey(secretName)
	if err != nil {
		return nil, fmt.Errorf("error retrieving secret %v: %v", secretName, err)
	}
	if !exists {
		return nil, fmt.Errorf("secret named %v does not exist", secretName)
	}

	secret := secretInterface.(*api.Secret)
	cert, okcert := secret.Data[api.TLSCertKey]
	key, okkey := secret.Data[api.TLSPrivateKeyKey]

	ca := secret.Data["ca.crt"]

	nsSecName := strings.Replace(secretName, "/", "-", -1)

	var s *ingress.SSLCert
	if okcert && okkey {
		glog.Infof("found certificate and private key, configuring %v as a TLS Secret", secretName)
		s, err = ssl.AddOrUpdateCertAndKey(nsSecName, cert, key, ca)
	} else if ca != nil {
		glog.Infof("found only ca.crt, configuring %v as an Certificate Authentication secret", secretName)
		s, err = ssl.AddCertAuth(nsSecName, ca)
	} else {
		return nil, fmt.Errorf("ko keypair or CA cert could be found in %v", secretName)
	}

	if err != nil {
		return nil, err
	}

	s.Name = secret.Name
	s.Namespace = secret.Namespace
	return s, nil
}

// secrReferenced checks if a secret is referenced or not by one or more Ingress rules
func (ic *GenericController) secrReferenced(name, namespace string) bool {
	for _, ingIf := range ic.ingLister.Store.List() {
		ing := ingIf.(*extensions.Ingress)

		if ic.annotations.ContainsCertificateAuth(ing) {
			str, _ := parser.GetStringAnnotation("ingress.kubernetes.io/auth-tls-secret", ing)
			if str == fmt.Sprintf("%v/%v", namespace, name) {
				return true
			}
		}

		if ing.Namespace != namespace {
			continue
		}
		for _, tls := range ing.Spec.TLS {
			if tls.SecretName == name {
				return true
			}
		}
	}
	return false
}

// sslCertTracker holds a store of referenced Secrets in Ingress rules
type sslCertTracker struct {
	cache.ThreadSafeStore
}

func newSSLCertTracker() *sslCertTracker {
	return &sslCertTracker{
		cache.NewThreadSafeStore(cache.Indexers{}, cache.Indices{}),
	}
}
<commit_msg>Fix a typo in an error message<commit_after>/*
Copyright 2015 The Kubernetes Authors.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package controller

import (
	"fmt"
	"reflect"
	"strings"
	"time"

	"github.com/golang/glog"

	"k8s.io/kubernetes/pkg/api"
	"k8s.io/kubernetes/pkg/apis/extensions"
	"k8s.io/kubernetes/pkg/client/cache"

	"k8s.io/ingress/core/pkg/ingress"
	"k8s.io/ingress/core/pkg/ingress/annotations/parser"
	ssl "k8s.io/ingress/core/pkg/net/ssl"
)

// syncSecret keeps in sync Secrets used by Ingress rules with the files on
// disk to allow being used in controllers.
func (ic *GenericController) syncSecret(k interface{}) error {
	if ic.secretQueue.IsShuttingDown() {
		return nil
	}
	if !ic.controllersInSync() {
		time.Sleep(podStoreSyncedPollPeriod)
		return fmt.Errorf("deferring sync till endpoints controller has synced")
	}

	var key string
	var cert *ingress.SSLCert
	var err error

	key = k.(string)

	secObj, exists, err := ic.secrLister.Store.GetByKey(key)
	if err != nil {
		return fmt.Errorf("error getting secret %v: %v", key, err)
	}
	if !exists {
		return fmt.Errorf("secret %v was not found", key)
	}
	sec := secObj.(*api.Secret)
	if !ic.secrReferenced(sec.Name, sec.Namespace) {
		glog.V(3).Infof("secret %v/%v is not used in Ingress rules. skipping ", sec.Namespace, sec.Name)
		return nil
	}

	cert, err = ic.getPemCertificate(key)
	if err != nil {
		return err
	}

	// create certificates and add or update the item in the store
	cur, exists := ic.sslCertTracker.Get(key)
	if exists {
		s := cur.(*ingress.SSLCert)
		if reflect.DeepEqual(s, cert) {
			// no need to update
			return nil
		}
		glog.Infof("updating secret %v/%v in the local store", sec.Namespace, sec.Name)
		ic.sslCertTracker.Update(key, cert)
		return nil
	}
	glog.Infof("adding secret %v/%v to the local store", sec.Namespace, sec.Name)
	ic.sslCertTracker.Add(key, cert)
	return nil
}

// getPemCertificate receives a secret, and creates a ingress.SSLCert as return.
// It parses the secret and verifies if it's a keypair, or a 'ca.crt' secret only.
func (ic *GenericController) getPemCertificate(secretName string) (*ingress.SSLCert, error) {
	secretInterface, exists, err := ic.secrLister.Store.GetByKey(secretName)
	if err != nil {
		return nil, fmt.Errorf("error retrieving secret %v: %v", secretName, err)
	}
	if !exists {
		return nil, fmt.Errorf("secret named %v does not exist", secretName)
	}

	secret := secretInterface.(*api.Secret)
	cert, okcert := secret.Data[api.TLSCertKey]
	key, okkey := secret.Data[api.TLSPrivateKeyKey]

	ca := secret.Data["ca.crt"]

	nsSecName := strings.Replace(secretName, "/", "-", -1)

	var s *ingress.SSLCert
	if okcert && okkey {
		glog.Infof("found certificate and private key, configuring %v as a TLS Secret", secretName)
		s, err = ssl.AddOrUpdateCertAndKey(nsSecName, cert, key, ca)
	} else if ca != nil {
		glog.Infof("found only ca.crt, configuring %v as an Certificate Authentication secret", secretName)
		s, err = ssl.AddCertAuth(nsSecName, ca)
	} else {
		return nil, fmt.Errorf("no keypair or CA cert could be found in %v", secretName)
	}

	if err != nil {
		return nil, err
	}

	s.Name = secret.Name
	s.Namespace = secret.Namespace
	return s, nil
}

// secrReferenced checks if a secret is referenced or not by one or more Ingress rules
func (ic *GenericController) secrReferenced(name, namespace string) bool {
	for _, ingIf := range ic.ingLister.Store.List() {
		ing := ingIf.(*extensions.Ingress)

		if ic.annotations.ContainsCertificateAuth(ing) {
			str, _ := parser.GetStringAnnotation("ingress.kubernetes.io/auth-tls-secret", ing)
			if str == fmt.Sprintf("%v/%v", namespace, name) {
				return true
			}
		}

		if ing.Namespace != namespace {
			continue
		}
		for _, tls := range ing.Spec.TLS {
			if tls.SecretName == name {
				return true
			}
		}
	}
	return false
}

// sslCertTracker holds a store of referenced Secrets in Ingress rules
type sslCertTracker struct {
	cache.ThreadSafeStore
}

func newSSLCertTracker() *sslCertTracker {
	return &sslCertTracker{
		cache.NewThreadSafeStore(cache.Indexers{}, cache.Indices{}),
	}
}
<|endoftext|>


<commit_before>package site

import (
	"bytes"
	"fmt"
	"io"
	"os"
	"path/filepath"
	"runtime"
	"strings"

	"github.com/osteele/gojekyll/pages"
	"github.com/osteele/gojekyll/plugins"
	"github.com/osteele/gojekyll/utils"
)

// WriteFiles writes output files.
// It attends to options.dry_run.
func (s *Site) WriteFiles() (count int, err error) {
	errs := make(chan error, runtime.NumCPU())
	for _, p := range s.OutputDocs() {
		count++
		go func(d pages.Document) {
			errs <- s.WriteDoc(d)
		}(p)
	}
	var errList []error
	for i := 0; i < count; i++ {
		if e := <-errs; e != nil {
			errList = append(errList, e)
		}
	}
	return count, combineErrors(errList)
}

// WriteDoc writes a document to the destination directory.
// It attends to options.dry_run.
func (s *Site) WriteDoc(d pages.Document) error {
	from := d.SourcePath()
	to := filepath.Join(s.DestDir(), d.Permalink())
	if !d.Static() && filepath.Ext(to) == "" {
		to = filepath.Join(to, "index.html")
	}
	if s.config.Verbose {
		fmt.Println("create", to, "from", d.SourcePath())
	}
	if s.config.DryRun {
		// FIXME render the page, just don't write it
		return nil
	}
	// nolint: gas
	if err := os.MkdirAll(filepath.Dir(to), 0755); err != nil {
		return err
	}
	switch {
	case d.Static():
		return utils.CopyFileContents(to, from, 0644)
	default:
		return utils.VisitCreatedFile(to, func(w io.Writer) error {
			return s.WriteDocument(w, d)
		})
	}
}

// WriteDocument writes the document to a writer.
func (s *Site) WriteDocument(w io.Writer, d pages.Document) error {
	switch p := d.(type) {
	case pages.Page:
		return s.WritePage(w, p)
	default:
		return d.Write(w)
	}
}

// WritePage writes the page to a writer.
func (s *Site) WritePage(w io.Writer, p pages.Page) error {
	if err := s.prepareRendering(); err != nil {
		return err
	}
	buf := new(bytes.Buffer)
	if err := p.Write(buf); err != nil {
		return err
	}
	b := buf.Bytes()
	err := s.runHooks(func(p plugins.Plugin) error {
		b = p.PostRender(b)
		return nil
	})
	if err != nil {
		return err
	}
	_, err = w.Write(b)
	return err
}

func combineErrors(errs []error) error {
	switch len(errs) {
	case 0:
		return nil
	case 1:
		return errs[0]
	default:
		messages := make([]string, len(errs))
		for i, e := range errs {
			messages[i] = e.Error()
		}
		return fmt.Errorf(strings.Join(messages, "\n"))
	}
}

func (s *Site) prepareRendering() error {
	if !s.preparedToRender {
		if err := s.initializeRenderingPipeline(); err != nil {
			return err
		}
		if err := s.setPageContent(); err != nil {
			return err
		}
		s.preparedToRender = true
	}
	return nil
}
<commit_msg>Fix running out of file descriptors<commit_after>package site

import (
	"bytes"
	"fmt"
	"io"
	"os"
	"path/filepath"
	"strings"

	"github.com/osteele/gojekyll/pages"
	"github.com/osteele/gojekyll/plugins"
	"github.com/osteele/gojekyll/utils"
)

// WriteFiles writes output files.
// It attends to options.dry_run.
func (s *Site) WriteFiles() (count int, err error) {
	errs := make(chan error)
	// without this, large sites run out of file descriptors
	sem := make(chan bool, 20)
	for i, n := 0, cap(sem); i < n; i++ {
		sem <- true
	}
	for _, d := range s.OutputDocs() {
		count++
		go func(d pages.Document) {
			<-sem
			errs <- s.WriteDoc(d)
			sem <- true
		}(d)
	}
	var errList []error
	for i := 0; i < count; i++ {
		if e := <-errs; e != nil {
			errList = append(errList, e)
		}
	}
	return count, combineErrors(errList)
}

// WriteDoc writes a document to the destination directory.
// It attends to options.dry_run.
func (s *Site) WriteDoc(d pages.Document) error {
	from := d.SourcePath()
	to := filepath.Join(s.DestDir(), d.Permalink())
	if !d.Static() && filepath.Ext(to) == "" {
		to = filepath.Join(to, "index.html")
	}
	if s.config.Verbose {
		fmt.Println("create", to, "from", d.SourcePath())
	}
	if s.config.DryRun {
		// FIXME render the page, just don't write it
		return nil
	}
	// nolint: gas
	if err := os.MkdirAll(filepath.Dir(to), 0755); err != nil {
		return err
	}
	switch {
	case d.Static():
		return utils.CopyFileContents(to, from, 0644)
	default:
		return utils.VisitCreatedFile(to, func(w io.Writer) error {
			return s.WriteDocument(w, d)
		})
	}
}

// WriteDocument writes the document to a writer.
func (s *Site) WriteDocument(w io.Writer, d pages.Document) error {
	switch p := d.(type) {
	case pages.Page:
		return s.WritePage(w, p)
	default:
		return d.Write(w)
	}
}

// WritePage writes the page to a writer.
func (s *Site) WritePage(w io.Writer, p pages.Page) error {
	if err := s.prepareRendering(); err != nil {
		return err
	}
	buf := new(bytes.Buffer)
	if err := p.Write(buf); err != nil {
		return err
	}
	b := buf.Bytes()
	err := s.runHooks(func(p plugins.Plugin) error {
		b = p.PostRender(b)
		return nil
	})
	if err != nil {
		return err
	}
	_, err = w.Write(b)
	return err
}

func combineErrors(errs []error) error {
	switch len(errs) {
	case 0:
		return nil
	case 1:
		return errs[0]
	default:
		messages := make([]string, len(errs))
		for i, e := range errs {
			messages[i] = e.Error()
		}
		return fmt.Errorf(strings.Join(messages, "\n"))
	}
}

func (s *Site) prepareRendering() error {
	if !s.preparedToRender {
		if err := s.initializeRenderingPipeline(); err != nil {
			return err
		}
		if err := s.setPageContent(); err != nil {
			return err
		}
		s.preparedToRender = true
	}
	return nil
}
<|endoftext|>


<commit_before>package sema

import (
	"github.com/rhysd/gocaml/common"
	. "github.com/rhysd/gocaml/types"
	"github.com/rhysd/locerr"
)

// Check cyclic dependency. When unifying t and u where t is type variable and
// u is a type which contains t, it results in infinite-length type.
// It should be reported as semantic error.
func occur(v *Var, rhs Type) bool {
	switch t := rhs.(type) {
	case *Tuple:
		for _, e := range t.Elems {
			if occur(v, e) {
				return true
			}
		}
	case *Array:
		return occur(v, t.Elem)
	case *Option:
		return occur(v, t.Elem)
	case *Fun:
		if occur(v, t.Ret) {
			return true
		}
		for _, p := range t.Params {
			if occur(v, p) {
				return true
			}
		}
	case *Var:
		if t.Ref != nil {
			return occur(v, t.Ref)
		}
		if t.IsGeneric() {
			panic("FATAL: Generic type variable must not appear in occur check")
		}
		if v == t {
			return true
		}
		if t.Level > v.Level {
			// Adjust levels
			t.Level = v.Level
		}
	}
	return false
}

func unifyTuple(left, right *Tuple) *locerr.Error {
	length := len(left.Elems)
	if length != len(right.Elems) {
		return locerr.Errorf("Number of elements of tuple does not match: %d vs %d (between '%s' and '%s')", length, len(right.Elems), left.String(), right.String())
	}

	for i := 0; i < length; i++ {
		l := left.Elems[i]
		r := right.Elems[i]
		if err := Unify(l, r); err != nil {
			return locerr.Notef(err, "On unifying tuples' %s elements of '%s' and '%s'", common.Ordinal(i+1), left.String(), right.String())
		}
	}

	return nil
}

func unifyFun(left, right *Fun) *locerr.Error {
	if err := Unify(left.Ret, right.Ret); err != nil {
		return locerr.Notef(err, "On unifying functions' return types of '%s' and '%s'", left.String(), right.String())
	}

	if len(left.Params) != len(right.Params) {
		return locerr.Errorf("Number of parameters of function does not match: %d vs %d (between '%s' and '%s')", len(left.Params), len(right.Params), left.String(), right.String())
	}

	for i, l := range left.Params {
		r := right.Params[i]
		if err := Unify(l, r); err != nil {
			return locerr.Notef(err, "On unifying %s parameter of function '%s' and '%s'", common.Ordinal(i+1), left.String(), right.String())
		}
	}

	return nil
}

func assignVar(v *Var, t Type) *locerr.Error {
	// When rv.Ref == nil
	if occur(v, t) {
		return locerr.Errorf("Cannot resolve free type variable. Cyclic dependency found for free type variable '%s' while unification with '%s'", v.String(), t.String())
	}

	if v.IsGeneric() {
		panic("FATAL: Generic variable must not appear in unification")
	}

	v.Ref = t
	return nil
}

func Unify(left, right Type) *locerr.Error {
	switch l := left.(type) {
	case *Unit, *Bool, *Int, *Float, *String:
		// Types for Unit, Bool, Int, Float and String are singleton instance.
		// So comparing directly is OK.
		if l == right {
			return nil
		}
	case *Tuple:
		if r, ok := right.(*Tuple); ok {
			return unifyTuple(l, r)
		}
	case *Array:
		if r, ok := right.(*Array); ok {
			return Unify(l.Elem, r.Elem)
		}
	case *Option:
		if r, ok := right.(*Option); ok {
			return Unify(l.Elem, r.Elem)
		}
	case *Fun:
		if r, ok := right.(*Fun); ok {
			return unifyFun(l, r)
		}
	}

	lv, lok := left.(*Var)
	rv, rok := right.(*Var)

	// Order of below 'if' statements is important! (#15)

	if (lok && rok) && (lv == rv) {
		return nil
	}
	if lok && lv.Ref != nil {
		return Unify(lv.Ref, right)
	}
	if rok && rv.Ref != nil {
		return Unify(left, rv.Ref)
	}
	if lok {
		// When lv.Ref == nil
		return assignVar(lv, right)
	}
	if rok {
		// When rv.Ref == nil
		return assignVar(rv, left)
	}

	return locerr.Errorf("Cannot unify types. Type mismatch between '%s' and '%s'", left.String(), right.String())
}
<commit_msg>do not check generic type variable on unification<commit_after>package sema

import (
	"github.com/rhysd/gocaml/common"
	. "github.com/rhysd/gocaml/types"
	"github.com/rhysd/locerr"
)

// Check cyclic dependency. When unifying t and u where t is type variable and
// u is a type which contains t, it results in infinite-length type.
// It should be reported as semantic error.
func occur(v *Var, rhs Type) bool {
	switch t := rhs.(type) {
	case *Tuple:
		for _, e := range t.Elems {
			if occur(v, e) {
				return true
			}
		}
	case *Array:
		return occur(v, t.Elem)
	case *Option:
		return occur(v, t.Elem)
	case *Fun:
		if occur(v, t.Ret) {
			return true
		}
		for _, p := range t.Params {
			if occur(v, p) {
				return true
			}
		}
	case *Var:
		if t.Ref != nil {
			return occur(v, t.Ref)
		}
		if t.IsGeneric() {
			panic("FATAL: Generic type variable must not appear in occur check")
		}
		if v == t {
			return true
		}
		if t.Level > v.Level {
			// Adjust levels
			t.Level = v.Level
		}
	}
	return false
}

func unifyTuple(left, right *Tuple) *locerr.Error {
	length := len(left.Elems)
	if length != len(right.Elems) {
		return locerr.Errorf("Number of elements of tuple does not match: %d vs %d (between '%s' and '%s')", length, len(right.Elems), left.String(), right.String())
	}

	for i := 0; i < length; i++ {
		l := left.Elems[i]
		r := right.Elems[i]
		if err := Unify(l, r); err != nil {
			return locerr.Notef(err, "On unifying tuples' %s elements of '%s' and '%s'", common.Ordinal(i+1), left.String(), right.String())
		}
	}

	return nil
}

func unifyFun(left, right *Fun) *locerr.Error {
	if err := Unify(left.Ret, right.Ret); err != nil {
		return locerr.Notef(err, "On unifying functions' return types of '%s' and '%s'", left.String(), right.String())
	}

	if len(left.Params) != len(right.Params) {
		return locerr.Errorf("Number of parameters of function does not match: %d vs %d (between '%s' and '%s')", len(left.Params), len(right.Params), left.String(), right.String())
	}

	for i, l := range left.Params {
		r := right.Params[i]
		if err := Unify(l, r); err != nil {
			return locerr.Notef(err, "On unifying %s parameter of function '%s' and '%s'", common.Ordinal(i+1), left.String(), right.String())
		}
	}

	return nil
}

func assignVar(v *Var, t Type) *locerr.Error {
	// When rv.Ref == nil
	if occur(v, t) {
		return locerr.Errorf("Cannot resolve free type variable. Cyclic dependency found for free type variable '%s' while unification with '%s'", v.String(), t.String())
	}

	// Note:
	// 'v' may be generic type variable because of external symbols.
	// e.g.
	//   let _ = x in x + x
	// The `x` is an external symbol and typed as ?. And it is bound to `_` in `let` expression.
	// The `_` is typed as 'a so the type of `x` will be 'a.
	// In `x + x`, type of `x` is unified although its type is generic.

	v.Ref = t
	return nil
}

func Unify(left, right Type) *locerr.Error {
	switch l := left.(type) {
	case *Unit, *Bool, *Int, *Float, *String:
		// Types for Unit, Bool, Int, Float and String are singleton instance.
		// So comparing directly is OK.
		if l == right {
			return nil
		}
	case *Tuple:
		if r, ok := right.(*Tuple); ok {
			return unifyTuple(l, r)
		}
	case *Array:
		if r, ok := right.(*Array); ok {
			return Unify(l.Elem, r.Elem)
		}
	case *Option:
		if r, ok := right.(*Option); ok {
			return Unify(l.Elem, r.Elem)
		}
	case *Fun:
		if r, ok := right.(*Fun); ok {
			return unifyFun(l, r)
		}
	}

	lv, lok := left.(*Var)
	rv, rok := right.(*Var)

	// Order of below 'if' statements is important! (#15)

	if (lok && rok) && (lv == rv) {
		return nil
	}
	if lok && lv.Ref != nil {
		return Unify(lv.Ref, right)
	}
	if rok && rv.Ref != nil {
		return Unify(left, rv.Ref)
	}
	if lok {
		// When lv.Ref == nil
		return assignVar(lv, right)
	}
	if rok {
		// When rv.Ref == nil
		return assignVar(rv, left)
	}

	return locerr.Errorf("Cannot unify types. Type mismatch between '%s' and '%s'", left.String(), right.String())
}
<|endoftext|>