<commit_before>// This file is an example of an S3-pulling plugin.  This is a real-world
// plugin that can actually be used in a production environment (compared to
// the more general but dangerous "external-images" plugin).  This requires you
// to put your AWS access key information into the environment per AWS's
// standard credential management: AWS_ACCESS_KEY_ID and AWS_SECRET_ACCESS_KEY.
// You may also put access keys in $HOME/.aws/credentials (or
// docker/s3credentials if you're using the docker-compose example override
// setup).  See docker/s3credentials.example for an example credentials file.
//
// When a resource is requested, if its IIIF id begins with "s3://", we treat
// the rest of the id as an s3 bucket and id to be pulled from S3 object
// storage.  As credentials are configured on the server end, attack vectors
// seen in the external images plugin are effectively nullified.
//
// We assume the asset is already a format RAIS can serve (preferably JP2), and
// we cache it locally with the same extension it has in S3.  The IDToPath
// return is the cached path so that RAIS can use the cached file immediately
// after download.  The JP2 cache is configurable via `S3Cache` in the RAIS
// toml file or by setting `RAIS_S3CACHE` in the environment, and defaults to
// `/var/cache/rais-s3`.
//
// Expiration of cached files must be managed externally (to avoid
// over-complicating this plugin).  A simple approach could be a cron job that
// wipes out all cached data if it hasn't been accessed in the past 24 hours:
//
//     find /var/cache/rais-s3 -type f -atime +1 -exec rm {} \;
//
// Depending how fast the cache grows, how much disk space you have available,
// and how much variety you have in S3, you may want to monitor the cache
// closely and tweak this cron job example as needed, or come up with something
// more sophisticated.

package main

import (
	"errors"
	"rais/src/iiif"
	"rais/src/plugins"
	"time"

	"github.com/spf13/viper"
	"github.com/uoregon-libraries/gopkg/fileutil"
	"github.com/uoregon-libraries/gopkg/logger"
)

var l = logger.Named("rais/s3-plugin", logger.Debug)

var s3cache, s3zone, s3endpoint string
var cacheLifetime time.Duration

// Disabled lets the plugin manager know not to add this plugin's functions to
// the global list unless sanity checks in Initialize() pass
var Disabled = true

// Initialize sets up package variables for the s3 pulls and verifies sanity of
// some of the configuration
func Initialize() {
	viper.SetDefault("S3Cache", "/var/local/rais-s3")
	s3cache = viper.GetString("S3Cache")
	s3zone = viper.GetString("S3Zone")
	s3endpoint = viper.GetString("S3Endpoint")

	if s3zone == "" {
		l.Infof("S3 plugin will not be enabled: S3Zone must be set in rais.toml or RAIS_S3ZONE must be set in the environment")
		return
	}

	// This is an undocumented feature: it's a bit experimental, and really not
	// something that should be relied upon until it gets some testing.
	viper.SetDefault("S3CacheLifetime", "0")
	var lifetimeString = viper.GetString("S3CacheLifetime")
	var err error
	cacheLifetime, err = time.ParseDuration(lifetimeString)
	if err != nil {
		l.Fatalf("S3 plugin failure: malformed S3CacheLifetime (%q): %s", lifetimeString, err)
	}

	l.Debugf("Setting S3 cache location to %q", s3cache)
	l.Debugf("Setting S3 zone to %q", s3zone)
	if cacheLifetime > time.Duration(0) {
		l.Debugf("Setting S3 cache expiration to %s", cacheLifetime)
		go purgeLoop()
	}
	Disabled = false

	if fileutil.IsDir(s3cache) {
		return
	}
	if !fileutil.MustNotExist(s3cache) {
		l.Fatalf("S3 plugin failure: %q must not exist or else must be a directory", s3cache)
	}
}

// SetLogger is called by the RAIS server's plugin manager to let plugins use
// the central logger
func SetLogger(raisLogger *logger.Logger) {
	l = raisLogger
}

// IDToPath implements the auto-download logic when a IIIF ID
// starts with "s3://"
func IDToPath(id iiif.ID) (path string, err error) {
	var a, _ = lookupAsset(id)
	if a.key == "" {
		return "", plugins.ErrSkipped
	}

	// See if this file is currently being downloaded; if so we need to wait
	var timeout = time.Now().Add(time.Second * 10)
	for a.tryFLock() == false {
		time.Sleep(time.Millisecond * 250)
		if time.Now().After(timeout) {
			return "", errors.New("timed out waiting for locked asset (probably very slow download)")
		}
	}

	// Let the asset know it's being read
	a.read()

	// Attempt to download the asset content
	err = a.download()
	a.fUnlock()

	return a.path, err
}

// PurgeCaches deletes all cached files this plugin is tracking.  Deletion
// happens in the background so the API isn't sitting for potentially many
// minutes prior to responding to the caller.
//
// TODO: this plugin should index files on the filesystem to see if there are
// any it should be tracking (this happens if RAIS is ever shut down while
// tracking files).  We don't want to delay startup, though.  Options:
//     - On shutdown, write out the assets map - then on startup we can just
//       read it in again and reset purge times
//     - On startup fire up a background thread that just instantiates assets
//       via lookupAsset(basename(file-".ext")).  If the filename is always the
//       IIIF ID, this should work, and doesn't need to block since it'll only
//       lock on the lookupAsset call.
func PurgeCaches() {
	// lock all assets while indexing them so we can index everything RAIS
	// *currently* knows about without things getting weird if new stuff is being
	// indexed during the process
	assetMutex.Lock()
	var ids []iiif.ID
	for _, a := range assets {
		ids = append(ids, a.id)
	}
	assetMutex.Unlock()
	go purgeCaches(ids)
}

// purgeCaches synchronously purges a list of assets from the filesystem cache,
// pausing briefly between each purge so this can run in the background without
// hammering the disk.
func purgeCaches(ids []iiif.ID) {
	for _, id := range ids {
		ExpireCachedImage(id)
		time.Sleep(time.Millisecond * 250)
	}
	l.Infof("s3-images plugin: mass-purged %d assets", len(ids))
}

// ExpireCachedImage gets rid of any cached image for the given id, should it
// exist.  We don't really care if it doesn't exist, though, as that can mean
// it's already been purged, or RAIS was restarted and the whole cache removed,
// etc.
func ExpireCachedImage(id iiif.ID) {
	var a, ok = lookupAsset(id)
	var infoMsgFmt = "s3-images plugin: purging %q: %s"
	if ok {
		doPurge(a)
		l.Infof(infoMsgFmt, id, "success")
	} else {
		assetMutex.Lock()
		delete(assets, a.id)
		assetMutex.Unlock()
		l.Infof(infoMsgFmt, id, "no local asset cached")
	}
}
<commit_msg>plugins/s3-images: report "cache not purged" quieter<commit_after>// This file is an example of an S3-pulling plugin.  This is a real-world
// plugin that can actually be used in a production environment (compared to
// the more general but dangerous "external-images" plugin).  This requires you
// to put your AWS access key information into the environment per AWS's
// standard credential management: AWS_ACCESS_KEY_ID and AWS_SECRET_ACCESS_KEY.
// You may also put access keys in $HOME/.aws/credentials (or
// docker/s3credentials if you're using the docker-compose example override
// setup).  See docker/s3credentials.example for an example credentials file.
//
// When a resource is requested, if its IIIF id begins with "s3://", we treat
// the rest of the id as an s3 bucket and id to be pulled from S3 object
// storage.  As credentials are configured on the server end, attack vectors
// seen in the external images plugin are effectively nullified.
//
// We assume the asset is already a format RAIS can serve (preferably JP2), and
// we cache it locally with the same extension it has in S3.  The IDToPath
// return is the cached path so that RAIS can use the cached file immediately
// after download.  The JP2 cache is configurable via `S3Cache` in the RAIS
// toml file or by setting `RAIS_S3CACHE` in the environment, and defaults to
// `/var/cache/rais-s3`.
//
// Expiration of cached files must be managed externally (to avoid
// over-complicating this plugin).  A simple approach could be a cron job that
// wipes out all cached data if it hasn't been accessed in the past 24 hours:
//
//     find /var/cache/rais-s3 -type f -atime +1 -exec rm {} \;
//
// Depending how fast the cache grows, how much disk space you have available,
// and how much variety you have in S3, you may want to monitor the cache
// closely and tweak this cron job example as needed, or come up with something
// more sophisticated.

package main

import (
	"errors"
	"rais/src/iiif"
	"rais/src/plugins"
	"time"

	"github.com/spf13/viper"
	"github.com/uoregon-libraries/gopkg/fileutil"
	"github.com/uoregon-libraries/gopkg/logger"
)

var l = logger.Named("rais/s3-plugin", logger.Debug)

var s3cache, s3zone, s3endpoint string
var cacheLifetime time.Duration

// Disabled lets the plugin manager know not to add this plugin's functions to
// the global list unless sanity checks in Initialize() pass
var Disabled = true

// Initialize sets up package variables for the s3 pulls and verifies sanity of
// some of the configuration
func Initialize() {
	viper.SetDefault("S3Cache", "/var/local/rais-s3")
	s3cache = viper.GetString("S3Cache")
	s3zone = viper.GetString("S3Zone")
	s3endpoint = viper.GetString("S3Endpoint")

	if s3zone == "" {
		l.Infof("S3 plugin will not be enabled: S3Zone must be set in rais.toml or RAIS_S3ZONE must be set in the environment")
		return
	}

	// This is an undocumented feature: it's a bit experimental, and really not
	// something that should be relied upon until it gets some testing.
	viper.SetDefault("S3CacheLifetime", "0")
	var lifetimeString = viper.GetString("S3CacheLifetime")
	var err error
	cacheLifetime, err = time.ParseDuration(lifetimeString)
	if err != nil {
		l.Fatalf("S3 plugin failure: malformed S3CacheLifetime (%q): %s", lifetimeString, err)
	}

	l.Debugf("Setting S3 cache location to %q", s3cache)
	l.Debugf("Setting S3 zone to %q", s3zone)
	if cacheLifetime > time.Duration(0) {
		l.Debugf("Setting S3 cache expiration to %s", cacheLifetime)
		go purgeLoop()
	}
	Disabled = false

	if fileutil.IsDir(s3cache) {
		return
	}
	if !fileutil.MustNotExist(s3cache) {
		l.Fatalf("S3 plugin failure: %q must not exist or else must be a directory", s3cache)
	}
}

// SetLogger is called by the RAIS server's plugin manager to let plugins use
// the central logger
func SetLogger(raisLogger *logger.Logger) {
	l = raisLogger
}

// IDToPath implements the auto-download logic when a IIIF ID
// starts with "s3://"
func IDToPath(id iiif.ID) (path string, err error) {
	var a, _ = lookupAsset(id)
	if a.key == "" {
		return "", plugins.ErrSkipped
	}

	// See if this file is currently being downloaded; if so we need to wait
	var timeout = time.Now().Add(time.Second * 10)
	for a.tryFLock() == false {
		time.Sleep(time.Millisecond * 250)
		if time.Now().After(timeout) {
			return "", errors.New("timed out waiting for locked asset (probably very slow download)")
		}
	}

	// Let the asset know it's being read
	a.read()

	// Attempt to download the asset content
	err = a.download()
	a.fUnlock()

	return a.path, err
}

// PurgeCaches deletes all cached files this plugin is tracking.  Deletion
// happens in the background so the API isn't sitting for potentially many
// minutes prior to responding to the caller.
//
// TODO: this plugin should index files on the filesystem to see if there are
// any it should be tracking (this happens if RAIS is ever shut down while
// tracking files).  We don't want to delay startup, though.  Options:
//     - On shutdown, write out the assets map - then on startup we can just
//       read it in again and reset purge times
//     - On startup fire up a background thread that just instantiates assets
//       via lookupAsset(basename(file-".ext")).  If the filename is always the
//       IIIF ID, this should work, and doesn't need to block since it'll only
//       lock on the lookupAsset call.
func PurgeCaches() {
	// lock all assets while indexing them so we can index everything RAIS
	// *currently* knows about without things getting weird if new stuff is being
	// indexed during the process
	assetMutex.Lock()
	var ids []iiif.ID
	for _, a := range assets {
		ids = append(ids, a.id)
	}
	assetMutex.Unlock()
	go purgeCaches(ids)
}

// purgeCaches synchronously purges a list of assets from the filesystem cache,
// pausing briefly between each purge so this can run in the background without
// hammering the disk.
func purgeCaches(ids []iiif.ID) {
	for _, id := range ids {
		ExpireCachedImage(id)
		time.Sleep(time.Millisecond * 250)
	}
	l.Infof("s3-images plugin: mass-purged %d assets", len(ids))
}

// ExpireCachedImage gets rid of any cached image for the given id, should it
// exist.  We don't really care if it doesn't exist, though, as that can mean
// it's already been purged, or RAIS was restarted and the whole cache removed,
// etc.
func ExpireCachedImage(id iiif.ID) {
	var a, ok = lookupAsset(id)
	var infoMsgFmt = "s3-images plugin: purging %q: %s"
	if ok {
		doPurge(a)
		l.Infof(infoMsgFmt, id, "success")
	} else {
		assetMutex.Lock()
		delete(assets, a.id)
		assetMutex.Unlock()
		l.Debugf(infoMsgFmt, id, "no local asset cached")
	}
}
<|endoftext|>


<commit_before>package isitrandom

import "testing"

type alternatingRNG struct{}

func (rng alternatingRNG) Read(p []byte) (n int, err error) {
	for i := 0; i < len(p); i++ {
		p[i] = 0xaa // 10101010
	}
	return len(p), nil
}

type constantRNG struct{}

func (rng constantRNG) Read(p []byte) (n int, err error) {
	for i := 0; i < len(p); i++ {
		p[i] = 0xff // 11111111
	}
	return len(p), nil
}

func TestFrequencyTest(t *testing.T) {
	var p float64
	p = FrequencyTest(alternatingRNG{})
	// TODO fix this
	if p != 0.0 {
		t.Errorf("Expected %f, got %f", 0.5, p)
	}
	// TODO fix this
	p = FrequencyTest(constantRNG{})
	if p != 0.9999 {
		t.Errorf("Expected %f, got %f", 0.5, p)
	}
}
<commit_msg>Added slightly alternating sequence<commit_after>package isitrandom

import "testing"

type alternatingRNG struct{}

func (rng alternatingRNG) Read(p []byte) (n int, err error) {
	for i := 0; i < len(p); i++ {
		p[i] = 0xaa // 10101010
	}
	return len(p), nil
}

type slightlyAlternatingRNG struct{}

func (rng slightlyAlternatingRNG) Read(p []byte) (n int, err error) {
	for i := 0; i < len(p); i++ {
		p[i] = 0xbb // 10111011
	}
	return len(p), nil
}

type constantRNG struct{}

func (rng constantRNG) Read(p []byte) (n int, err error) {
	for i := 0; i < len(p); i++ {
		p[i] = 0xff // 11111111
	}
	return len(p), nil
}

func TestFrequencyTest(t *testing.T) {
	var p float64

	p = FrequencyTest(alternatingRNG{})
	if p != 0.0 {
		t.Errorf("Expected %f, got %f", 0.0, p)
	}

	p = FrequencyTest(slightlyAlternatingRNG{})
	if p != 0.5 {
		t.Errorf("slightlyAlternatingRNG, Expected %f, got %f", 0.9999, p)
	}

	p = FrequencyTest(constantRNG{})
	if p != 0.9999 {
		t.Errorf("Expected %f, got %f", 0.9999, p)
	}
}
<|endoftext|>


<commit_before>// Copyright (c) 2017 Cisco and/or its affiliates.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at:
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package kvdbsync

import (
	"errors"

	"github.com/golang/protobuf/proto"
	"github.com/ligato/cn-infra/datasync"
	"github.com/ligato/cn-infra/datasync/resync"
	"github.com/ligato/cn-infra/datasync/syncbase"
	"github.com/ligato/cn-infra/db/keyval"
	"github.com/ligato/cn-infra/infra"
	"github.com/ligato/cn-infra/logging"
	"github.com/ligato/cn-infra/servicelabel"
)

var (
	// ErrNotReady is an error returned when KVDBSync plugin is being used before the KVPlugin is ready.
	ErrNotReady = errors.New("transport adapter is not ready yet (probably called before AfterInit)")
)

// Plugin dbsync implements synchronization between local memory and db.
// Other plugins can be notified when DB changes occur or resync is needed.
// This plugin reads/pulls the data from db when resync is needed.
type Plugin struct {
	Deps

	adapter  *watcher
	registry *syncbase.Registry
}

// Deps groups dependencies injected into the plugin so that they are
// logically separated from other plugin fields.
type Deps struct {
	infra.PluginName
	Log          logging.PluginLogger
	ServiceLabel servicelabel.ReaderAPI
	KvPlugin     keyval.KvProtoPlugin // inject
	ResyncOrch   resync.Subscriber    // inject
}

// Init only initializes plugin.registry.
func (p *Plugin) Init() error {
	p.registry = syncbase.NewRegistry()

	return nil
}

// AfterInit uses provided connection to build new transport watcher.
//
// Plugin.registry subscriptions (registered by Watch method) are used for resync.
// Resync is called only if ResyncOrch was injected (i.e. is not nil).
// The order of plugins in flavor is not important to resync
// since Watch() is called in Plugin.Init() and Resync.Register()
// is called in Plugin.AfterInit().
//
// If provided connection is not ready (not connected), AfterInit starts new goroutine in order to
// 'wait' for the connection. After that, the new transport watcher is built as usual.
func (p *Plugin) AfterInit() error {
	if !p.isKvEnabled() {
		p.Log.Debugf("KVPlugin is nil or disabled, skipping AfterInit")
		return nil
	}

	// set function to be executed on KVPlugin connection
	p.KvPlugin.OnConnect(p.initKvPlugin)

	return nil
}

func (p *Plugin) isKvEnabled() bool {
	return p.KvPlugin != nil && !p.KvPlugin.Disabled()
}

func (p *Plugin) initKvPlugin() error {
	if !p.isKvEnabled() {
		p.Log.Debugf("KVPlugin is nil or disabled, skipping initKvPlugin")
		return nil
	}

	p.adapter = &watcher{
		db:   p.KvPlugin.NewBroker(p.ServiceLabel.GetAgentPrefix()),
		dbW:  p.KvPlugin.NewWatcher(p.ServiceLabel.GetAgentPrefix()),
		base: p.registry,
	}

	if p.ResyncOrch != nil {
		for name, sub := range p.registry.Subscriptions() {
			reg := p.ResyncOrch.Register(name)
			_, err := watchAndResyncBrokerKeys(reg, sub.ChangeChan, sub.ResyncChan, sub.CloseChan,
				p.adapter, sub.KeyPrefixes...)
			if err != nil {
				return err
			}
		}
	} else {
		p.Log.Debugf("ResyncOrch is nil, skipping registration")
	}

	return nil
}

// Watch adds entry to the plugin.registry. By doing this, other plugins will receive notifications
// about data changes and data resynchronization.
//
// This method is supposed to be called in Plugin.Init().
// Calling this method later than kvdbsync.Plugin.AfterInit() will have no effect
// (no notifications will be received).
func (p *Plugin) Watch(resyncName string, changeChan chan datasync.ChangeEvent,
	resyncChan chan datasync.ResyncEvent, keyPrefixes ...string) (datasync.WatchRegistration, error) {

	return p.registry.Watch(resyncName, changeChan, resyncChan, keyPrefixes...)
}

// Put propagates this call to a particular kvdb.Plugin unless the kvdb.Plugin is Disabled().
//
// This method is supposed to be called in Plugin.AfterInit() or later (even from different go routine).
func (p *Plugin) Put(key string, data proto.Message, opts ...datasync.PutOption) error {
	if !p.isKvEnabled() {
		return nil
	}

	if p.adapter != nil {
		return p.adapter.db.Put(key, data, opts...)
	}

	return ErrNotReady
}

// Delete propagates this call to a particular kvdb.Plugin unless the kvdb.Plugin is Disabled().
//
// This method is supposed to be called in Plugin.AfterInit() or later (even from different go routine).
func (p *Plugin) Delete(key string, opts ...datasync.DelOption) (existed bool, err error) {
	if !p.isKvEnabled() {
		return false, nil
	}

	if p.adapter != nil {
		return p.adapter.db.Delete(key, opts...)
	}

	return false, ErrNotReady
}

// Close resources.
func (plugin *Plugin) Close() error {
	return nil
}
<commit_msg>Satisfy linter<commit_after>// Copyright (c) 2017 Cisco and/or its affiliates.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at:
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package kvdbsync

import (
	"errors"

	"github.com/golang/protobuf/proto"
	"github.com/ligato/cn-infra/datasync"
	"github.com/ligato/cn-infra/datasync/resync"
	"github.com/ligato/cn-infra/datasync/syncbase"
	"github.com/ligato/cn-infra/db/keyval"
	"github.com/ligato/cn-infra/infra"
	"github.com/ligato/cn-infra/logging"
	"github.com/ligato/cn-infra/servicelabel"
)

var (
	// ErrNotReady is an error returned when KVDBSync plugin is being used before the KVPlugin is ready.
	ErrNotReady = errors.New("transport adapter is not ready yet (probably called before AfterInit)")
)

// Plugin dbsync implements synchronization between local memory and db.
// Other plugins can be notified when DB changes occur or resync is needed.
// This plugin reads/pulls the data from db when resync is needed.
type Plugin struct {
	Deps

	adapter  *watcher
	registry *syncbase.Registry
}

// Deps groups dependencies injected into the plugin so that they are
// logically separated from other plugin fields.
type Deps struct {
	infra.PluginName
	Log          logging.PluginLogger
	ServiceLabel servicelabel.ReaderAPI
	KvPlugin     keyval.KvProtoPlugin // inject
	ResyncOrch   resync.Subscriber    // inject
}

// Init only initializes plugin.registry.
func (p *Plugin) Init() error {
	p.registry = syncbase.NewRegistry()

	return nil
}

// AfterInit uses provided connection to build new transport watcher.
//
// Plugin.registry subscriptions (registered by Watch method) are used for resync.
// Resync is called only if ResyncOrch was injected (i.e. is not nil).
// The order of plugins in flavor is not important to resync
// since Watch() is called in Plugin.Init() and Resync.Register()
// is called in Plugin.AfterInit().
//
// If provided connection is not ready (not connected), AfterInit starts new goroutine in order to
// 'wait' for the connection. After that, the new transport watcher is built as usual.
func (p *Plugin) AfterInit() error {
	if !p.isKvEnabled() {
		p.Log.Debugf("KVPlugin is nil or disabled, skipping AfterInit")
		return nil
	}

	// set function to be executed on KVPlugin connection
	p.KvPlugin.OnConnect(p.initKvPlugin)

	return nil
}

func (p *Plugin) isKvEnabled() bool {
	return p.KvPlugin != nil && !p.KvPlugin.Disabled()
}

func (p *Plugin) initKvPlugin() error {
	if !p.isKvEnabled() {
		p.Log.Debugf("KVPlugin is nil or disabled, skipping initKvPlugin")
		return nil
	}

	p.adapter = &watcher{
		db:   p.KvPlugin.NewBroker(p.ServiceLabel.GetAgentPrefix()),
		dbW:  p.KvPlugin.NewWatcher(p.ServiceLabel.GetAgentPrefix()),
		base: p.registry,
	}

	if p.ResyncOrch != nil {
		for name, sub := range p.registry.Subscriptions() {
			reg := p.ResyncOrch.Register(name)
			_, err := watchAndResyncBrokerKeys(reg, sub.ChangeChan, sub.ResyncChan, sub.CloseChan,
				p.adapter, sub.KeyPrefixes...)
			if err != nil {
				return err
			}
		}
	} else {
		p.Log.Debugf("ResyncOrch is nil, skipping registration")
	}

	return nil
}

// Watch adds entry to the plugin.registry. By doing this, other plugins will receive notifications
// about data changes and data resynchronization.
//
// This method is supposed to be called in Plugin.Init().
// Calling this method later than kvdbsync.Plugin.AfterInit() will have no effect
// (no notifications will be received).
func (p *Plugin) Watch(resyncName string, changeChan chan datasync.ChangeEvent,
	resyncChan chan datasync.ResyncEvent, keyPrefixes ...string) (datasync.WatchRegistration, error) {

	return p.registry.Watch(resyncName, changeChan, resyncChan, keyPrefixes...)
}

// Put propagates this call to a particular kvdb.Plugin unless the kvdb.Plugin is Disabled().
//
// This method is supposed to be called in Plugin.AfterInit() or later (even from different go routine).
func (p *Plugin) Put(key string, data proto.Message, opts ...datasync.PutOption) error {
	if !p.isKvEnabled() {
		return nil
	}

	if p.adapter != nil {
		return p.adapter.db.Put(key, data, opts...)
	}

	return ErrNotReady
}

// Delete propagates this call to a particular kvdb.Plugin unless the kvdb.Plugin is Disabled().
//
// This method is supposed to be called in Plugin.AfterInit() or later (even from different go routine).
func (p *Plugin) Delete(key string, opts ...datasync.DelOption) (existed bool, err error) {
	if !p.isKvEnabled() {
		return false, nil
	}

	if p.adapter != nil {
		return p.adapter.db.Delete(key, opts...)
	}

	return false, ErrNotReady
}

// Close resources.
func (p *Plugin) Close() error {
	return nil
}
<|endoftext|>


<commit_before>/*
Copyright 2014 Google Inc. All rights reserved.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package gce_pd

import (
	"fmt"
	"os"
	"path"
	"strconv"

	"github.com/GoogleCloudPlatform/kubernetes/pkg/api"
	"github.com/GoogleCloudPlatform/kubernetes/pkg/kubelet/volume"
	"github.com/GoogleCloudPlatform/kubernetes/pkg/types"
	"github.com/GoogleCloudPlatform/kubernetes/pkg/util/mount"
	"github.com/golang/glog"
)

// This is the primary entrypoint for volume plugins.
func ProbeVolumePlugins() []volume.Plugin {
	return []volume.Plugin{&gcePersistentDiskPlugin{nil, false}, &gcePersistentDiskPlugin{nil, true}}
}

type gcePersistentDiskPlugin struct {
	host       volume.Host
	legacyMode bool // if set, plugin answers to the legacy name
}

var _ volume.Plugin = &gcePersistentDiskPlugin{}

const (
	gcePersistentDiskPluginName       = "kubernetes.io/gce-pd"
	gcePersistentDiskPluginLegacyName = "gce-pd"
)

func (plugin *gcePersistentDiskPlugin) Init(host volume.Host) {
	plugin.host = host
}

func (plugin *gcePersistentDiskPlugin) Name() string {
	if plugin.legacyMode {
		return gcePersistentDiskPluginLegacyName
	}
	return gcePersistentDiskPluginName
}

func (plugin *gcePersistentDiskPlugin) CanSupport(spec *api.Volume) bool {
	if plugin.legacyMode {
		// Legacy mode instances can be cleaned up but not created anew.
		return false
	}

	if spec.Source.GCEPersistentDisk != nil {
		return true
	}
	return false
}

func (plugin *gcePersistentDiskPlugin) NewBuilder(spec *api.Volume, podUID types.UID) (volume.Builder, error) {
	// Inject real implementations here, test through the internal function.
	return plugin.newBuilderInternal(spec, podUID, &GCEDiskUtil{}, mount.New())
}

func (plugin *gcePersistentDiskPlugin) newBuilderInternal(spec *api.Volume, podUID types.UID, manager pdManager, mounter mount.Interface) (volume.Builder, error) {
	if plugin.legacyMode {
		// Legacy mode instances can be cleaned up but not created anew.
		return nil, fmt.Errorf("legacy mode: can not create new instances")
	}

	pdName := spec.Source.GCEPersistentDisk.PDName
	fsType := spec.Source.GCEPersistentDisk.FSType
	partition := ""
	if spec.Source.GCEPersistentDisk.Partition != 0 {
		partition = strconv.Itoa(spec.Source.GCEPersistentDisk.Partition)
	}
	readOnly := spec.Source.GCEPersistentDisk.ReadOnly

	return &gcePersistentDisk{
		podUID:     podUID,
		volName:    spec.Name,
		pdName:     pdName,
		fsType:     fsType,
		partition:  partition,
		readOnly:   readOnly,
		manager:    manager,
		mounter:    mounter,
		plugin:     plugin,
		legacyMode: false,
	}, nil
}

func (plugin *gcePersistentDiskPlugin) NewCleaner(volName string, podUID types.UID) (volume.Cleaner, error) {
	// Inject real implementations here, test through the internal function.
	return plugin.newCleanerInternal(volName, podUID, &GCEDiskUtil{}, mount.New())
}

func (plugin *gcePersistentDiskPlugin) newCleanerInternal(volName string, podUID types.UID, manager pdManager, mounter mount.Interface) (volume.Cleaner, error) {
	legacy := false
	if plugin.legacyMode {
		legacy = true
	}
	return &gcePersistentDisk{
		podUID:     podUID,
		volName:    volName,
		manager:    manager,
		mounter:    mounter,
		plugin:     plugin,
		legacyMode: legacy,
	}, nil
}

// Abstract interface to PD operations.
type pdManager interface {
	// Attaches the disk to the kubelet's host machine.
	AttachDisk(pd *gcePersistentDisk) error
	// Detaches the disk from the kubelet's host machine.
	DetachDisk(pd *gcePersistentDisk, devicePath string) error
}

// gcePersistentDisk volumes are disk resources provided by Google Compute Engine
// that are attached to the kubelet's host machine and exposed to the pod.
type gcePersistentDisk struct {
	volName string
	podUID  types.UID
	// Unique identifier of the PD, used to find the disk resource in the provider.
	pdName string
	// Filesystem type, optional.
	fsType string
	// Specifies the partition to mount
	partition string
	// Specifies whether the disk will be attached as read-only.
	readOnly bool
	// Utility interface that provides API calls to the provider to attach/detach disks.
	manager pdManager
	// Mounter interface that provides system calls to mount the disks.
	mounter    mount.Interface
	plugin     *gcePersistentDiskPlugin
	legacyMode bool
}

func detachDiskLogError(pd *gcePersistentDisk) {
	err := pd.manager.DetachDisk(pd, "/dev/disk/by-id/google-"+pd.pdName)
	if err != nil {
		glog.Warningf("Failed to detach disk: %v (%v)", pd, err)
	}
}

// SetUp attaches the disk and bind mounts to the volume path.
func (pd *gcePersistentDisk) SetUp() error {
	if pd.legacyMode {
		return fmt.Errorf("legacy mode: can not create new instances")
	}

	// TODO: handle failed mounts here.
	mountpoint, err := isMountPoint(pd.GetPath())
	glog.V(4).Infof("PersistentDisk set up: %s %v %v", pd.GetPath(), mountpoint, err)
	if err != nil && !os.IsNotExist(err) {
		return err
	}
	if mountpoint {
		return nil
	}

	if err := pd.manager.AttachDisk(pd); err != nil {
		return err
	}

	flags := uintptr(0)
	if pd.readOnly {
		flags = mount.FlagReadOnly
	}

	volPath := pd.GetPath()
	if err := os.MkdirAll(volPath, 0750); err != nil {
		// TODO: we should really eject the attach/detach out into its own control loop.
		detachDiskLogError(pd)
		return err
	}

	// Perform a bind mount to the full path to allow duplicate mounts of the same PD.
	globalPDPath := makeGlobalPDName(pd.plugin.host, pd.pdName, pd.readOnly)
	err = pd.mounter.Mount(globalPDPath, pd.GetPath(), "", mount.FlagBind|flags, "")
	if err != nil {
		os.RemoveAll(pd.GetPath())
		// TODO: we should really eject the attach/detach out into its own control loop.
		detachDiskLogError(pd)
		return err
	}

	return nil
}

func makeGlobalPDName(host volume.Host, devName string, readOnly bool) string {
	return path.Join(host.GetPluginDir(gcePersistentDiskPluginName), "mounts", devName)
}

func (pd *gcePersistentDisk) GetPath() string {
	name := gcePersistentDiskPluginName
	if pd.legacyMode {
		name = gcePersistentDiskPluginLegacyName
	}
	return pd.plugin.host.GetPodVolumeDir(pd.podUID, volume.EscapePluginName(name), pd.volName)
}

// Unmounts the bind mount, and detaches the disk only if the PD
// resource was the last reference to that disk on the kubelet.
func (pd *gcePersistentDisk) TearDown() error {
	mountpoint, err := isMountPoint(pd.GetPath())
	if err != nil {
		return err
	}
	if !mountpoint {
		return os.RemoveAll(pd.GetPath())
	}

	devicePath, refCount, err := getMountRefCount(pd.mounter, pd.GetPath())
	if err != nil {
		return err
	}
	if err := pd.mounter.Unmount(pd.GetPath(), 0); err != nil {
		return err
	}
	refCount--
	if err := os.RemoveAll(pd.GetPath()); err != nil {
		return err
	}
	// If refCount is 1, then all bind mounts have been removed, and the
	// remaining reference is the global mount. It is safe to detach.
	if refCount == 1 {
		if err := pd.manager.DetachDisk(pd, devicePath); err != nil {
			return err
		}
	}
	return nil
}
<commit_msg>Reverse the order of unmounting and removing the pd disk. This should mean that we retry detaching if the original detach fails for some reason.<commit_after>/*
Copyright 2014 Google Inc. All rights reserved.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package gce_pd

import (
	"fmt"
	"os"
	"path"
	"strconv"

	"github.com/GoogleCloudPlatform/kubernetes/pkg/api"
	"github.com/GoogleCloudPlatform/kubernetes/pkg/kubelet/volume"
	"github.com/GoogleCloudPlatform/kubernetes/pkg/types"
	"github.com/GoogleCloudPlatform/kubernetes/pkg/util/mount"
	"github.com/golang/glog"
)

// This is the primary entrypoint for volume plugins.
func ProbeVolumePlugins() []volume.Plugin {
	return []volume.Plugin{&gcePersistentDiskPlugin{nil, false}, &gcePersistentDiskPlugin{nil, true}}
}

type gcePersistentDiskPlugin struct {
	host       volume.Host
	legacyMode bool // if set, plugin answers to the legacy name
}

var _ volume.Plugin = &gcePersistentDiskPlugin{}

const (
	gcePersistentDiskPluginName       = "kubernetes.io/gce-pd"
	gcePersistentDiskPluginLegacyName = "gce-pd"
)

func (plugin *gcePersistentDiskPlugin) Init(host volume.Host) {
	plugin.host = host
}

func (plugin *gcePersistentDiskPlugin) Name() string {
	if plugin.legacyMode {
		return gcePersistentDiskPluginLegacyName
	}
	return gcePersistentDiskPluginName
}

func (plugin *gcePersistentDiskPlugin) CanSupport(spec *api.Volume) bool {
	if plugin.legacyMode {
		// Legacy mode instances can be cleaned up but not created anew.
		return false
	}

	if spec.Source.GCEPersistentDisk != nil {
		return true
	}
	return false
}

func (plugin *gcePersistentDiskPlugin) NewBuilder(spec *api.Volume, podUID types.UID) (volume.Builder, error) {
	// Inject real implementations here, test through the internal function.
	return plugin.newBuilderInternal(spec, podUID, &GCEDiskUtil{}, mount.New())
}

func (plugin *gcePersistentDiskPlugin) newBuilderInternal(spec *api.Volume, podUID types.UID, manager pdManager, mounter mount.Interface) (volume.Builder, error) {
	if plugin.legacyMode {
		// Legacy mode instances can be cleaned up but not created anew.
		return nil, fmt.Errorf("legacy mode: can not create new instances")
	}

	pdName := spec.Source.GCEPersistentDisk.PDName
	fsType := spec.Source.GCEPersistentDisk.FSType
	partition := ""
	if spec.Source.GCEPersistentDisk.Partition != 0 {
		partition = strconv.Itoa(spec.Source.GCEPersistentDisk.Partition)
	}
	readOnly := spec.Source.GCEPersistentDisk.ReadOnly

	return &gcePersistentDisk{
		podUID:     podUID,
		volName:    spec.Name,
		pdName:     pdName,
		fsType:     fsType,
		partition:  partition,
		readOnly:   readOnly,
		manager:    manager,
		mounter:    mounter,
		plugin:     plugin,
		legacyMode: false,
	}, nil
}

func (plugin *gcePersistentDiskPlugin) NewCleaner(volName string, podUID types.UID) (volume.Cleaner, error) {
	// Inject real implementations here, test through the internal function.
	return plugin.newCleanerInternal(volName, podUID, &GCEDiskUtil{}, mount.New())
}

func (plugin *gcePersistentDiskPlugin) newCleanerInternal(volName string, podUID types.UID, manager pdManager, mounter mount.Interface) (volume.Cleaner, error) {
	legacy := false
	if plugin.legacyMode {
		legacy = true
	}
	return &gcePersistentDisk{
		podUID:     podUID,
		volName:    volName,
		manager:    manager,
		mounter:    mounter,
		plugin:     plugin,
		legacyMode: legacy,
	}, nil
}

// Abstract interface to PD operations.
type pdManager interface {
	// Attaches the disk to the kubelet's host machine.
	AttachDisk(pd *gcePersistentDisk) error
	// Detaches the disk from the kubelet's host machine.
	DetachDisk(pd *gcePersistentDisk, devicePath string) error
}

// gcePersistentDisk volumes are disk resources provided by Google Compute Engine
// that are attached to the kubelet's host machine and exposed to the pod.
type gcePersistentDisk struct {
	volName string
	podUID  types.UID
	// Unique identifier of the PD, used to find the disk resource in the provider.
	pdName string
	// Filesystem type, optional.
	fsType string
	// Specifies the partition to mount
	partition string
	// Specifies whether the disk will be attached as read-only.
	readOnly bool
	// Utility interface that provides API calls to the provider to attach/detach disks.
	manager pdManager
	// Mounter interface that provides system calls to mount the disks.
	mounter    mount.Interface
	plugin     *gcePersistentDiskPlugin
	legacyMode bool
}

func detachDiskLogError(pd *gcePersistentDisk) {
	err := pd.manager.DetachDisk(pd, "/dev/disk/by-id/google-"+pd.pdName)
	if err != nil {
		glog.Warningf("Failed to detach disk: %v (%v)", pd, err)
	}
}

// SetUp attaches the disk and bind mounts to the volume path.
func (pd *gcePersistentDisk) SetUp() error {
	if pd.legacyMode {
		return fmt.Errorf("legacy mode: can not create new instances")
	}

	// TODO: handle failed mounts here.
	mountpoint, err := isMountPoint(pd.GetPath())
	glog.V(4).Infof("PersistentDisk set up: %s %v %v", pd.GetPath(), mountpoint, err)
	if err != nil && !os.IsNotExist(err) {
		return err
	}
	if mountpoint {
		return nil
	}

	if err := pd.manager.AttachDisk(pd); err != nil {
		return err
	}

	flags := uintptr(0)
	if pd.readOnly {
		flags = mount.FlagReadOnly
	}

	volPath := pd.GetPath()
	if err := os.MkdirAll(volPath, 0750); err != nil {
		// TODO: we should really eject the attach/detach out into its own control loop.
		detachDiskLogError(pd)
		return err
	}

	// Perform a bind mount to the full path to allow duplicate mounts of the same PD.
	globalPDPath := makeGlobalPDName(pd.plugin.host, pd.pdName, pd.readOnly)
	err = pd.mounter.Mount(globalPDPath, pd.GetPath(), "", mount.FlagBind|flags, "")
	if err != nil {
		os.RemoveAll(pd.GetPath())
		// TODO: we should really eject the attach/detach out into its own control loop.
		detachDiskLogError(pd)
		return err
	}

	return nil
}

func makeGlobalPDName(host volume.Host, devName string, readOnly bool) string {
	return path.Join(host.GetPluginDir(gcePersistentDiskPluginName), "mounts", devName)
}

func (pd *gcePersistentDisk) GetPath() string {
	name := gcePersistentDiskPluginName
	if pd.legacyMode {
		name = gcePersistentDiskPluginLegacyName
	}
	return pd.plugin.host.GetPodVolumeDir(pd.podUID, volume.EscapePluginName(name), pd.volName)
}

// Unmounts the bind mount, and detaches the disk only if the PD
// resource was the last reference to that disk on the kubelet.
func (pd *gcePersistentDisk) TearDown() error {
	mountpoint, err := isMountPoint(pd.GetPath())
	if err != nil {
		return err
	}
	if !mountpoint {
		return os.RemoveAll(pd.GetPath())
	}

	devicePath, refCount, err := getMountRefCount(pd.mounter, pd.GetPath())
	if err != nil {
		return err
	}
	if err := pd.mounter.Unmount(pd.GetPath(), 0); err != nil {
		return err
	}
	refCount--
	// If refCount is 1, then all bind mounts have been removed, and the
	// remaining reference is the global mount. It is safe to detach.
	if refCount == 1 {
		if err := pd.manager.DetachDisk(pd, devicePath); err != nil {
			return err
		}
	}
	if err := os.RemoveAll(pd.GetPath()); err != nil {
		return err
	}
	return nil
}
<|endoftext|>


<commit_before>// Package sexp provides a high level intermediate representation
// that contains both Go and Emacs Lisp traits.
package sexp

import (
	"go/types"
	"lisp/function"
)

type Form interface {
	form()
}

// Atoms.
type (
	// Bool = true or false literal.
	Bool struct{ Val bool }
	// Int = rune constant or integer literal.
	Int struct{ Val int64 }
	// Float = floating point literal (of any supported format).
	Float struct{ Val float64 }
	// String = raw/normal string literal.
	String struct{ Val string }
	// Symbol = lisp.Symbol literal.
	Symbol struct{ Val string }
)

// Composite literals.
type (
	// ArrayLit = [N]T{...}.
	ArrayLit struct{ Vals []Form }
	// QuotedArray = ArrayLit where each element is constant.
	QuotedArray struct{ Vals []Form }
)

// Call expression is normal (direct) function invocation.
type Call struct {
	Fn   *function.Type
	Args []Form
}

// CallStmt is a Call which discards returned results.
type CallStmt struct {
	*Call
}

// Var - reference to lexical variable.
type Var struct{ Name string }

/* Special forms */

// Panic causes runtime panic and carries data along.
type Panic struct {
	ErrorData Form
}

// Bind associates name with expression (initializer).
// Binding has lexical scoping.
type Bind struct {
	Name string
	Init Form
}

// Rebind changes symbol value.
type Rebind struct {
	Name string
	Expr Form
}

// TypeAssert coerces expression to specified type; panics on failure.
type TypeAssert struct {
	Expr Form
	Type types.Type
}

// LispTypeAssert is a special case of type assert, it
// operates on unboxed Elisp values.
type LispTypeAssert struct {
	Expr Form
	Type types.Type
}

// FormList packs multiple forms together (like "progn").
type FormList struct {
	Forms []Form
}

// Block is a list of statements.
// Unlike FormList, it creates a new lexical scope.
type Block struct {
	Forms []Form
	Scope *types.Scope
}

// If statement evaluates test expression and,
// depending on the result, one of the branches gets
// executed. Else branch is optional.
type If struct {
	Cond Form
	Then *Block
	Else Form
}

// Return statement exits the function and returns
// one or more values to the caller.
type Return struct {
	Results []Form
}

type While struct {
	Cond Form
	Body *Block
}

/* Builtin ops */

// type MakeMap struct {
// 	SizeHint Form
// }

type (
	BitOr  struct{ Args [2]Form }
	BitAnd struct{ Args [2]Form }
	BitXor struct{ Args [2]Form }

	NumAdd struct {
		Args [2]Form
		Type *types.Basic
	}
	NumSub struct {
		Args [2]Form
		Type *types.Basic
	}
	NumMul struct {
		Args [2]Form
		Type *types.Basic
	}
	NumQuo struct {
		Args [2]Form
		Type *types.Basic
	}
	NumEq struct {
		Args [2]Form
		Type *types.Basic
	}
	NumNotEq struct {
		Args [2]Form
		Type *types.Basic
	}
	NumLt struct {
		Args [2]Form
		Type *types.Basic
	}
	NumLte struct {
		Args [2]Form
		Type *types.Basic
	}
	NumGt struct {
		Args [2]Form
		Type *types.Basic
	}
	NumGte struct {
		Args [2]Form
		Type *types.Basic
	}

	Concat      struct{ Args [2]Form }
	StringEq    struct{ Args [2]Form }
	StringNotEq struct{ Args [2]Form }
	StringLt    struct{ Args [2]Form }
	StringLte   struct{ Args [2]Form }
	StringGt    struct{ Args [2]Form }
	StringGte   struct{ Args [2]Form }
)
<commit_msg>removing commented-out code<commit_after>// Package sexp provides a high level intermediate representation
// that contains both Go and Emacs Lisp traits.
package sexp

import (
	"go/types"
	"lisp/function"
)

type Form interface {
	form()
}

// Atoms.
type (
	// Bool = true or false literal.
	Bool struct{ Val bool }
	// Int = rune constant or integer literal.
	Int struct{ Val int64 }
	// Float = floating point literal (of any supported format).
	Float struct{ Val float64 }
	// String = raw/normal string literal.
	String struct{ Val string }
	// Symbol = lisp.Symbol literal.
	Symbol struct{ Val string }
)

// Composite literals.
type (
	// ArrayLit = [N]T{...}.
	ArrayLit struct{ Vals []Form }
	// QuotedArray = ArrayLit where each element is constant.
	QuotedArray struct{ Vals []Form }
)

// Call expression is normal (direct) function invocation.
type Call struct {
	Fn   *function.Type
	Args []Form
}

// CallStmt is a Call which discards returned results.
type CallStmt struct {
	*Call
}

// Var - reference to lexical variable.
type Var struct{ Name string }

/* Special forms */

// Panic causes runtime panic and carries data along.
type Panic struct {
	ErrorData Form
}

// Bind associates name with expression (initializer).
// Binding has lexical scoping.
type Bind struct {
	Name string
	Init Form
}

// Rebind changes symbol value.
type Rebind struct {
	Name string
	Expr Form
}

// TypeAssert coerces expression to specified type; panics on failure.
type TypeAssert struct {
	Expr Form
	Type types.Type
}

// LispTypeAssert is a special case of type assert, it
// operates on unboxed Elisp values.
type LispTypeAssert struct {
	Expr Form
	Type types.Type
}

// FormList packs multiple forms together (like "progn").
type FormList struct {
	Forms []Form
}

// Block is a list of statements.
// Unlike FormList, it creates a new lexical scope.
type Block struct {
	Forms []Form
	Scope *types.Scope
}

// If statement evaluates test expression and,
// depending on the result, one of the branches gets
// executed. Else branch is optional.
type If struct {
	Cond Form
	Then *Block
	Else Form
}

// Return statement exits the function and returns
// one or more values to the caller.
type Return struct {
	Results []Form
}

type While struct {
	Cond Form
	Body *Block
}

/* Builtin ops */

type (
	BitOr  struct{ Args [2]Form }
	BitAnd struct{ Args [2]Form }
	BitXor struct{ Args [2]Form }

	NumAdd struct {
		Args [2]Form
		Type *types.Basic
	}
	NumSub struct {
		Args [2]Form
		Type *types.Basic
	}
	NumMul struct {
		Args [2]Form
		Type *types.Basic
	}
	NumQuo struct {
		Args [2]Form
		Type *types.Basic
	}
	NumEq struct {
		Args [2]Form
		Type *types.Basic
	}
	NumNotEq struct {
		Args [2]Form
		Type *types.Basic
	}
	NumLt struct {
		Args [2]Form
		Type *types.Basic
	}
	NumLte struct {
		Args [2]Form
		Type *types.Basic
	}
	NumGt struct {
		Args [2]Form
		Type *types.Basic
	}
	NumGte struct {
		Args [2]Form
		Type *types.Basic
	}

	Concat      struct{ Args [2]Form }
	StringEq    struct{ Args [2]Form }
	StringNotEq struct{ Args [2]Form }
	StringLt    struct{ Args [2]Form }
	StringLte   struct{ Args [2]Form }
	StringGt    struct{ Args [2]Form }
	StringGte   struct{ Args [2]Form }
)
<|endoftext|>


<commit_before>/*
Copyright 2017 The Nuclio Authors.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package http

import (
	"github.com/nuclio/nuclio/pkg/errors"
	"github.com/nuclio/nuclio/pkg/functionconfig"
	"github.com/nuclio/nuclio/pkg/processor/runtime"
	"github.com/nuclio/nuclio/pkg/processor/trigger"

	"github.com/mitchellh/mapstructure"
)

type Configuration struct {
	trigger.Configuration
	ReadBufferSize int
}

func NewConfiguration(ID string,
	triggerConfiguration *functionconfig.Trigger,
	runtimeConfiguration *runtime.Configuration) (*Configuration, error) {
	newConfiguration := Configuration{}

	// create base
	newConfiguration.Configuration = *trigger.NewConfiguration(ID, triggerConfiguration, runtimeConfiguration)

	// parse attributes
	if err := mapstructure.Decode(newConfiguration.Configuration.Attributes, &newConfiguration); err != nil {
		return nil, errors.Wrap(err, "Failed to decode attributes")
	}

	if newConfiguration.URL == "" {
		newConfiguration.URL = ":8080"
	}

	if newConfiguration.ReadBufferSize == 0 {
		newConfiguration.ReadBufferSize = 4 * 1024
	}

	return &newConfiguration, nil
}
<commit_msg>Increased fasthttp server read buffer size from 4kb to 16kb (#1431)<commit_after>/*
Copyright 2017 The Nuclio Authors.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package http

import (
	"github.com/nuclio/nuclio/pkg/errors"
	"github.com/nuclio/nuclio/pkg/functionconfig"
	"github.com/nuclio/nuclio/pkg/processor/runtime"
	"github.com/nuclio/nuclio/pkg/processor/trigger"

	"github.com/mitchellh/mapstructure"
)

type Configuration struct {
	trigger.Configuration
	ReadBufferSize int
}

func NewConfiguration(ID string,
	triggerConfiguration *functionconfig.Trigger,
	runtimeConfiguration *runtime.Configuration) (*Configuration, error) {
	newConfiguration := Configuration{}

	// create base
	newConfiguration.Configuration = *trigger.NewConfiguration(ID, triggerConfiguration, runtimeConfiguration)

	// parse attributes
	if err := mapstructure.Decode(newConfiguration.Configuration.Attributes, &newConfiguration); err != nil {
		return nil, errors.Wrap(err, "Failed to decode attributes")
	}

	if newConfiguration.URL == "" {
		newConfiguration.URL = ":8080"
	}

	if newConfiguration.ReadBufferSize == 0 {
		newConfiguration.ReadBufferSize = 16 * 1024
	}

	return &newConfiguration, nil
}
<|endoftext|>


<commit_before>/*
Copyright 2016 The Kubernetes Authors.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package apiservice

import (
	"context"
	"fmt"

	"k8s.io/apimachinery/pkg/fields"
	"k8s.io/apimachinery/pkg/labels"
	"k8s.io/apimachinery/pkg/runtime"
	"k8s.io/apimachinery/pkg/util/validation/field"
	"k8s.io/apiserver/pkg/registry/generic"
	"k8s.io/apiserver/pkg/storage"
	"k8s.io/apiserver/pkg/storage/names"

	"k8s.io/kube-aggregator/pkg/apis/apiregistration"
	"k8s.io/kube-aggregator/pkg/apis/apiregistration/validation"
)

type apiServerStrategy struct {
	runtime.ObjectTyper
	names.NameGenerator
}

func NewStrategy(typer runtime.ObjectTyper) apiServerStrategy {
	return apiServerStrategy{typer, names.SimpleNameGenerator}
}

func (apiServerStrategy) NamespaceScoped() bool {
	return false
}

func (apiServerStrategy) PrepareForCreate(ctx context.Context, obj runtime.Object) {
	apiservice := obj.(*apiregistration.APIService)
	apiservice.Status = apiregistration.APIServiceStatus{}

	// mark local API services as immediately available on create
	if apiservice.Spec.Service == nil {
		apiregistration.SetAPIServiceCondition(apiservice, apiregistration.NewLocalAvailableAPIServiceCondition())
	}
}

func (apiServerStrategy) PrepareForUpdate(ctx context.Context, obj, old runtime.Object) {
	newAPIService := obj.(*apiregistration.APIService)
	oldAPIService := old.(*apiregistration.APIService)
	newAPIService.Status = oldAPIService.Status
}

func (apiServerStrategy) Validate(ctx context.Context, obj runtime.Object) field.ErrorList {
	return validation.ValidateAPIService(obj.(*apiregistration.APIService))
}

func (apiServerStrategy) AllowCreateOnUpdate() bool {
	return false
}

func (apiServerStrategy) AllowUnconditionalUpdate() bool {
	return false
}

func (apiServerStrategy) Canonicalize(obj runtime.Object) {
}

func (apiServerStrategy) ValidateUpdate(ctx context.Context, obj, old runtime.Object) field.ErrorList {
	return validation.ValidateAPIServiceUpdate(obj.(*apiregistration.APIService), old.(*apiregistration.APIService))
}

type apiServerStatusStrategy struct {
	runtime.ObjectTyper
	names.NameGenerator
}

func NewStatusStrategy(typer runtime.ObjectTyper) apiServerStatusStrategy {
	return apiServerStatusStrategy{typer, names.SimpleNameGenerator}
}

func (apiServerStatusStrategy) NamespaceScoped() bool {
	return false
}

func (apiServerStatusStrategy) PrepareForUpdate(ctx context.Context, obj, old runtime.Object) {
	newAPIService := obj.(*apiregistration.APIService)
	oldAPIService := old.(*apiregistration.APIService)
	newAPIService.Spec = oldAPIService.Spec
	newAPIService.Labels = oldAPIService.Labels
	newAPIService.Annotations = oldAPIService.Annotations
	newAPIService.Finalizers = oldAPIService.Finalizers
	newAPIService.OwnerReferences = oldAPIService.OwnerReferences
}

func (apiServerStatusStrategy) AllowCreateOnUpdate() bool {
	return false
}

func (apiServerStatusStrategy) AllowUnconditionalUpdate() bool {
	return false
}

func (apiServerStatusStrategy) Canonicalize(obj runtime.Object) {
}

func (apiServerStatusStrategy) ValidateUpdate(ctx context.Context, obj, old runtime.Object) field.ErrorList {
	return validation.ValidateAPIServiceStatusUpdate(obj.(*apiregistration.APIService), old.(*apiregistration.APIService))
}

func GetAttrs(obj runtime.Object) (labels.Set, fields.Set, bool, error) {
	apiserver, ok := obj.(*apiregistration.APIService)
	if !ok {
		return nil, nil, false, fmt.Errorf("given object is not a APIService.")
	}
	return labels.Set(apiserver.ObjectMeta.Labels), APIServiceToSelectableFields(apiserver), apiserver.Initializers != nil, nil
}

// MatchAPIService is the filter used by the generic etcd backend to watch events
// from etcd to clients of the apiserver only interested in specific labels/fields.
func MatchAPIService(label labels.Selector, field fields.Selector) storage.SelectionPredicate {
	return storage.SelectionPredicate{
		Label:    label,
		Field:    field,
		GetAttrs: GetAttrs,
	}
}

// APIServiceToSelectableFields returns a field set that represents the object.
func APIServiceToSelectableFields(obj *apiregistration.APIService) fields.Set {
	return generic.ObjectMetaFieldsSet(&obj.ObjectMeta, true)
}
<commit_msg>Deprecate and remove use of alpha metadata.initializers field, remove IncludeUninitialized options<commit_after>/*
Copyright 2016 The Kubernetes Authors.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package apiservice

import (
	"context"
	"fmt"

	"k8s.io/apimachinery/pkg/fields"
	"k8s.io/apimachinery/pkg/labels"
	"k8s.io/apimachinery/pkg/runtime"
	"k8s.io/apimachinery/pkg/util/validation/field"
	"k8s.io/apiserver/pkg/registry/generic"
	"k8s.io/apiserver/pkg/storage"
	"k8s.io/apiserver/pkg/storage/names"

	"k8s.io/kube-aggregator/pkg/apis/apiregistration"
	"k8s.io/kube-aggregator/pkg/apis/apiregistration/validation"
)

type apiServerStrategy struct {
	runtime.ObjectTyper
	names.NameGenerator
}

func NewStrategy(typer runtime.ObjectTyper) apiServerStrategy {
	return apiServerStrategy{typer, names.SimpleNameGenerator}
}

func (apiServerStrategy) NamespaceScoped() bool {
	return false
}

func (apiServerStrategy) PrepareForCreate(ctx context.Context, obj runtime.Object) {
	apiservice := obj.(*apiregistration.APIService)
	apiservice.Status = apiregistration.APIServiceStatus{}

	// mark local API services as immediately available on create
	if apiservice.Spec.Service == nil {
		apiregistration.SetAPIServiceCondition(apiservice, apiregistration.NewLocalAvailableAPIServiceCondition())
	}
}

func (apiServerStrategy) PrepareForUpdate(ctx context.Context, obj, old runtime.Object) {
	newAPIService := obj.(*apiregistration.APIService)
	oldAPIService := old.(*apiregistration.APIService)
	newAPIService.Status = oldAPIService.Status
}

func (apiServerStrategy) Validate(ctx context.Context, obj runtime.Object) field.ErrorList {
	return validation.ValidateAPIService(obj.(*apiregistration.APIService))
}

func (apiServerStrategy) AllowCreateOnUpdate() bool {
	return false
}

func (apiServerStrategy) AllowUnconditionalUpdate() bool {
	return false
}

func (apiServerStrategy) Canonicalize(obj runtime.Object) {
}

func (apiServerStrategy) ValidateUpdate(ctx context.Context, obj, old runtime.Object) field.ErrorList {
	return validation.ValidateAPIServiceUpdate(obj.(*apiregistration.APIService), old.(*apiregistration.APIService))
}

type apiServerStatusStrategy struct {
	runtime.ObjectTyper
	names.NameGenerator
}

func NewStatusStrategy(typer runtime.ObjectTyper) apiServerStatusStrategy {
	return apiServerStatusStrategy{typer, names.SimpleNameGenerator}
}

func (apiServerStatusStrategy) NamespaceScoped() bool {
	return false
}

func (apiServerStatusStrategy) PrepareForUpdate(ctx context.Context, obj, old runtime.Object) {
	newAPIService := obj.(*apiregistration.APIService)
	oldAPIService := old.(*apiregistration.APIService)
	newAPIService.Spec = oldAPIService.Spec
	newAPIService.Labels = oldAPIService.Labels
	newAPIService.Annotations = oldAPIService.Annotations
	newAPIService.Finalizers = oldAPIService.Finalizers
	newAPIService.OwnerReferences = oldAPIService.OwnerReferences
}

func (apiServerStatusStrategy) AllowCreateOnUpdate() bool {
	return false
}

func (apiServerStatusStrategy) AllowUnconditionalUpdate() bool {
	return false
}

func (apiServerStatusStrategy) Canonicalize(obj runtime.Object) {
}

func (apiServerStatusStrategy) ValidateUpdate(ctx context.Context, obj, old runtime.Object) field.ErrorList {
	return validation.ValidateAPIServiceStatusUpdate(obj.(*apiregistration.APIService), old.(*apiregistration.APIService))
}

func GetAttrs(obj runtime.Object) (labels.Set, fields.Set, error) {
	apiserver, ok := obj.(*apiregistration.APIService)
	if !ok {
		return nil, nil, fmt.Errorf("given object is not a APIService.")
	}
	return labels.Set(apiserver.ObjectMeta.Labels), APIServiceToSelectableFields(apiserver), nil
}

// MatchAPIService is the filter used by the generic etcd backend to watch events
// from etcd to clients of the apiserver only interested in specific labels/fields.
func MatchAPIService(label labels.Selector, field fields.Selector) storage.SelectionPredicate {
	return storage.SelectionPredicate{
		Label:    label,
		Field:    field,
		GetAttrs: GetAttrs,
	}
}

// APIServiceToSelectableFields returns a field set that represents the object.
func APIServiceToSelectableFields(obj *apiregistration.APIService) fields.Set {
	return generic.ObjectMetaFieldsSet(&obj.ObjectMeta, true)
}
<|endoftext|>


<commit_before>package sqlstore

import (
	"testing"

	"github.com/grafana/grafana/pkg/components/simplejson"
	m "github.com/grafana/grafana/pkg/models"
	. "github.com/smartystreets/goconvey/convey"
)

func TestAlertingDataAccess(t *testing.T) {
	Convey("Testing Alerting data access", t, func() {
		InitTestDB(t)

		testDash := insertTestDashboard("dashboard with alerts", 1, "alert")

		items := []*m.Alert{
			{
				PanelId:     1,
				DashboardId: testDash.Id,
				OrgId:       testDash.OrgId,
				Name:        "Alerting title",
				Message:     "Alerting message",
				Settings:    simplejson.New(),
				Frequency:   1,
			},
		}

		cmd := m.SaveAlertsCommand{
			Alerts:      items,
			DashboardId: testDash.Id,
			OrgId:       1,
			UserId:      1,
		}

		err := SaveAlerts(&cmd)

		Convey("Can create one alert", func() {
			So(err, ShouldBeNil)
		})

		Convey("Can read properties", func() {
			alertQuery := m.GetAlertsQuery{DashboardId: testDash.Id, PanelId: 1, OrgId: 1}
			err2 := HandleAlertsQuery(&alertQuery)

			alert := alertQuery.Result[0]
			So(err2, ShouldBeNil)
			So(alert.Name, ShouldEqual, "Alerting title")
			So(alert.Message, ShouldEqual, "Alerting message")
			So(alert.State, ShouldEqual, "no_data")
			So(alert.Frequency, ShouldEqual, 1)
		})

		Convey("Alerts with same dashboard id and panel id should update", func() {
			modifiedItems := items
			modifiedItems[0].Name = "Name"

			modifiedCmd := m.SaveAlertsCommand{
				DashboardId: testDash.Id,
				OrgId:       1,
				UserId:      1,
				Alerts:      modifiedItems,
			}

			err := SaveAlerts(&modifiedCmd)

			Convey("Can save alerts with same dashboard and panel id", func() {
				So(err, ShouldBeNil)
			})

			Convey("Alerts should be updated", func() {
				query := m.GetAlertsQuery{DashboardId: testDash.Id, OrgId: 1}
				err2 := HandleAlertsQuery(&query)

				So(err2, ShouldBeNil)
				So(len(query.Result), ShouldEqual, 1)
				So(query.Result[0].Name, ShouldEqual, "Name")

				Convey("Alert state should not be updated", func() {
					So(query.Result[0].State, ShouldEqual, "no_data")
				})
			})

			Convey("Updates without changes should be ignored", func() {
				err3 := SaveAlerts(&modifiedCmd)
				So(err3, ShouldBeNil)
			})
		})

		Convey("Multiple alerts per dashboard", func() {
			multipleItems := []*m.Alert{
				{
					DashboardId: testDash.Id,
					PanelId:     1,
					Name:        "1",
					OrgId:       1,
					Settings:    simplejson.New(),
				},
				{
					DashboardId: testDash.Id,
					PanelId:     2,
					Name:        "2",
					OrgId:       1,
					Settings:    simplejson.New(),
				},
				{
					DashboardId: testDash.Id,
					PanelId:     3,
					Name:        "3",
					OrgId:       1,
					Settings:    simplejson.New(),
				},
			}

			cmd.Alerts = multipleItems
			err = SaveAlerts(&cmd)

			Convey("Should save 3 dashboards", func() {
				So(err, ShouldBeNil)

				queryForDashboard := m.GetAlertsQuery{DashboardId: testDash.Id, OrgId: 1}
				err2 := HandleAlertsQuery(&queryForDashboard)

				So(err2, ShouldBeNil)
				So(len(queryForDashboard.Result), ShouldEqual, 3)
			})

			Convey("should updated two dashboards and delete one", func() {
				missingOneAlert := multipleItems[:2]

				cmd.Alerts = missingOneAlert
				err = SaveAlerts(&cmd)

				Convey("should delete the missing alert", func() {
					query := m.GetAlertsQuery{DashboardId: testDash.Id, OrgId: 1}
					err2 := HandleAlertsQuery(&query)
					So(err2, ShouldBeNil)
					So(len(query.Result), ShouldEqual, 2)
				})
			})
		})

		Convey("When dashboard is removed", func() {
			items := []*m.Alert{
				{
					PanelId:     1,
					DashboardId: testDash.Id,
					Name:        "Alerting title",
					Message:     "Alerting message",
				},
			}

			cmd := m.SaveAlertsCommand{
				Alerts:      items,
				DashboardId: testDash.Id,
				OrgId:       1,
				UserId:      1,
			}

			SaveAlerts(&cmd)

			err = DeleteDashboard(&m.DeleteDashboardCommand{
				OrgId: 1,
				Slug:  testDash.Slug,
			})

			So(err, ShouldBeNil)

			Convey("Alerts should be removed", func() {
				query := m.GetAlertsQuery{DashboardId: testDash.Id, OrgId: 1}
				err2 := HandleAlertsQuery(&query)

				So(testDash.Id, ShouldEqual, 1)
				So(err2, ShouldBeNil)
				So(len(query.Result), ShouldEqual, 0)
			})
		})
	})
}
<commit_msg>Fixed tests<commit_after>package sqlstore

import (
	"testing"

	"github.com/grafana/grafana/pkg/components/simplejson"
	m "github.com/grafana/grafana/pkg/models"
	. "github.com/smartystreets/goconvey/convey"
)

func TestAlertingDataAccess(t *testing.T) {
	Convey("Testing Alerting data access", t, func() {
		InitTestDB(t)

		testDash := insertTestDashboard("dashboard with alerts", 1, "alert")

		items := []*m.Alert{
			{
				PanelId:     1,
				DashboardId: testDash.Id,
				OrgId:       testDash.OrgId,
				Name:        "Alerting title",
				Message:     "Alerting message",
				Settings:    simplejson.New(),
				Frequency:   1,
			},
		}

		cmd := m.SaveAlertsCommand{
			Alerts:      items,
			DashboardId: testDash.Id,
			OrgId:       1,
			UserId:      1,
		}

		err := SaveAlerts(&cmd)

		Convey("Can create one alert", func() {
			So(err, ShouldBeNil)
		})

		Convey("Can read properties", func() {
			alertQuery := m.GetAlertsQuery{DashboardId: testDash.Id, PanelId: 1, OrgId: 1}
			err2 := HandleAlertsQuery(&alertQuery)

			alert := alertQuery.Result[0]
			So(err2, ShouldBeNil)
			So(alert.Name, ShouldEqual, "Alerting title")
			So(alert.Message, ShouldEqual, "Alerting message")
			So(alert.State, ShouldEqual, "initialized")
			So(alert.Frequency, ShouldEqual, 1)
		})

		Convey("Alerts with same dashboard id and panel id should update", func() {
			modifiedItems := items
			modifiedItems[0].Name = "Name"

			modifiedCmd := m.SaveAlertsCommand{
				DashboardId: testDash.Id,
				OrgId:       1,
				UserId:      1,
				Alerts:      modifiedItems,
			}

			err := SaveAlerts(&modifiedCmd)

			Convey("Can save alerts with same dashboard and panel id", func() {
				So(err, ShouldBeNil)
			})

			Convey("Alerts should be updated", func() {
				query := m.GetAlertsQuery{DashboardId: testDash.Id, OrgId: 1}
				err2 := HandleAlertsQuery(&query)

				So(err2, ShouldBeNil)
				So(len(query.Result), ShouldEqual, 1)
				So(query.Result[0].Name, ShouldEqual, "Name")

				Convey("Alert state should not be updated", func() {
					So(query.Result[0].State, ShouldEqual, "initialized")
				})
			})

			Convey("Updates without changes should be ignored", func() {
				err3 := SaveAlerts(&modifiedCmd)
				So(err3, ShouldBeNil)
			})
		})

		Convey("Multiple alerts per dashboard", func() {
			multipleItems := []*m.Alert{
				{
					DashboardId: testDash.Id,
					PanelId:     1,
					Name:        "1",
					OrgId:       1,
					Settings:    simplejson.New(),
				},
				{
					DashboardId: testDash.Id,
					PanelId:     2,
					Name:        "2",
					OrgId:       1,
					Settings:    simplejson.New(),
				},
				{
					DashboardId: testDash.Id,
					PanelId:     3,
					Name:        "3",
					OrgId:       1,
					Settings:    simplejson.New(),
				},
			}

			cmd.Alerts = multipleItems
			err = SaveAlerts(&cmd)

			Convey("Should save 3 dashboards", func() {
				So(err, ShouldBeNil)

				queryForDashboard := m.GetAlertsQuery{DashboardId: testDash.Id, OrgId: 1}
				err2 := HandleAlertsQuery(&queryForDashboard)

				So(err2, ShouldBeNil)
				So(len(queryForDashboard.Result), ShouldEqual, 3)
			})

			Convey("should updated two dashboards and delete one", func() {
				missingOneAlert := multipleItems[:2]

				cmd.Alerts = missingOneAlert
				err = SaveAlerts(&cmd)

				Convey("should delete the missing alert", func() {
					query := m.GetAlertsQuery{DashboardId: testDash.Id, OrgId: 1}
					err2 := HandleAlertsQuery(&query)
					So(err2, ShouldBeNil)
					So(len(query.Result), ShouldEqual, 2)
				})
			})
		})

		Convey("When dashboard is removed", func() {
			items := []*m.Alert{
				{
					PanelId:     1,
					DashboardId: testDash.Id,
					Name:        "Alerting title",
					Message:     "Alerting message",
				},
			}

			cmd := m.SaveAlertsCommand{
				Alerts:      items,
				DashboardId: testDash.Id,
				OrgId:       1,
				UserId:      1,
			}

			SaveAlerts(&cmd)

			err = DeleteDashboard(&m.DeleteDashboardCommand{
				OrgId: 1,
				Slug:  testDash.Slug,
			})

			So(err, ShouldBeNil)

			Convey("Alerts should be removed", func() {
				query := m.GetAlertsQuery{DashboardId: testDash.Id, OrgId: 1}
				err2 := HandleAlertsQuery(&query)

				So(testDash.Id, ShouldEqual, 1)
				So(err2, ShouldBeNil)
				So(len(query.Result), ShouldEqual, 0)
			})
		})
	})
}
<|endoftext|>


<commit_before>package acr

import (
	"bufio"
	"context"
	cr "github.com/Azure/azure-sdk-for-go/services/containerregistry/mgmt/2018-09-01/containerregistry"
	"github.com/Azure/go-autorest/autorest/azure/auth"
	"github.com/GoogleContainerTools/skaffold/pkg/skaffold/build"
	"github.com/GoogleContainerTools/skaffold/pkg/skaffold/build/tag"
	"github.com/GoogleContainerTools/skaffold/pkg/skaffold/docker"
	"github.com/GoogleContainerTools/skaffold/pkg/skaffold/schema/latest"
	"github.com/GoogleContainerTools/skaffold/pkg/skaffold/util"
	"github.com/pkg/errors"
	"io"
	"net/http"
	"regexp"
	"time"
)

const BUILD_STATUS_HEADER = "x-ms-meta-Complete"

func (b *Builder) Build(ctx context.Context, out io.Writer, tagger tag.Tagger, artifacts []*latest.Artifact) ([]build.Artifact, error) {
	return build.InParallel(ctx, out, tagger, artifacts, b.buildArtifact)
}

func (b *Builder) buildArtifact(ctx context.Context, out io.Writer, tagger tag.Tagger, artifact *latest.Artifact) (string, error) {
	client := cr.NewRegistriesClient(b.Credentials.SubscriptionId)
	authorizer, err := auth.NewClientCredentialsConfig(b.Credentials.ClientId, b.Credentials.ClientSecret, b.Credentials.TenantId).Authorizer()
	if err != nil {
		return "", errors.Wrap(err, "authorizing client")
	}
	client.Authorizer = authorizer

	result, err := client.GetBuildSourceUploadURL(ctx, b.ResourceGroup, b.ContainerRegistry)
	if err != nil {
		return "", errors.Wrap(err, "build source upload url")
	}
	blob := NewBlobStorage(*result.UploadURL)

	err = docker.CreateDockerTarGzContext(blob.Writer(), artifact.Workspace, artifact.DockerArtifact)
	if err != nil {
		return "", errors.Wrap(err, "create context tar.gz")
	}

	err = blob.UploadFileToBlob()
	if err != nil {
		return "", errors.Wrap(err, "upload file to blob")
	}

	imageTag, err := tagger.GenerateFullyQualifiedImageName(artifact.Workspace, &tag.Options{
		Digest:    util.RandomID(),
		ImageName: artifact.ImageName,
	})
	if err != nil {
		return "", errors.Wrap(err, "create fully qualified image name")
	}

	imageTag, err = getImageTagWithoutFQDN(imageTag)
	if err != nil {
		return "", errors.Wrap(err, "get azure image tag")
	}

	buildRequest := cr.DockerBuildRequest{
		ImageNames:     &[]string{imageTag},
		IsPushEnabled:  &[]bool{true}[0], //who invented bool pointers
		SourceLocation: result.RelativePath,
		Platform: &cr.PlatformProperties{
			Variant:      cr.V8,
			Os:           cr.Linux,
			Architecture: cr.Amd64,
		},
		DockerFilePath: &artifact.DockerArtifact.DockerfilePath,
		Type:           cr.TypeDockerBuildRequest,
	}
	future, err := client.ScheduleRun(ctx, b.ResourceGroup, b.ContainerRegistry, buildRequest)
	if err != nil {
		return "", errors.Wrap(err, "schedule build request")
	}

	run, err := future.Result(client)
	if err != nil {
		return "", errors.Wrap(err, "get run id")
	}
	runId := *run.RunID

	runsClient := cr.NewRunsClient(b.Credentials.SubscriptionId)
	runsClient.Authorizer = client.Authorizer
	logUrl, err := runsClient.GetLogSasURL(ctx, b.ResourceGroup, b.ContainerRegistry, runId)
	if err != nil {
		return "", errors.Wrap(err, "get log url")
	}

	err = pollBuildStatus(*logUrl.LogLink, out)
	if err != nil {
		return "", errors.Wrap(err, "polling build status")
	}

	return imageTag, nil
}

func pollBuildStatus(logUrl string, out io.Writer) error {
	offset := int32(0)
	for {
		resp, err := http.Get(logUrl)
		if err != nil {
			return err
		}

		if resp.StatusCode == http.StatusNotFound {
			//if blob is not available yet, try again
			time.Sleep(2 * time.Second)
			continue
		}

		scanner := bufio.NewScanner(resp.Body)
		line := int32(0)
		for scanner.Scan() {
			if line > offset {
				out.Write(scanner.Bytes())
				offset++
			}
			line++
		}
		resp.Body.Close()

		if offset > 0 {
			switch resp.Header.Get(BUILD_STATUS_HEADER) {
			case "": //run succeeded when there is no status header
				return nil
			case "internalerror":
			case "failed":
				return errors.New("run failed")
			case "timedout":
				return errors.New("run timed out")
			case "canceled":
				return errors.New("run was canceled")
			}
		}

		time.Sleep(2 * time.Second)
	}
}

// ACR needs the image tag in the following format
// <registryName>/<repository>:<tag>
func getImageTagWithoutFQDN(imageTag string) (string, error) {
	r, err := regexp.Compile("(.*)\\..*\\..*(/.*)")
	if err != nil {
		return "", errors.Wrap(err, "create regexp")
	}

	matches := r.FindStringSubmatch(imageTag)
	if len(matches) < 3 {
		return "", errors.New("invalid image tag")
	}

	return matches[1] + matches[2], nil
}
<commit_msg>Fix output<commit_after>package acr

import (
	"bufio"
	"context"
	cr "github.com/Azure/azure-sdk-for-go/services/containerregistry/mgmt/2018-09-01/containerregistry"
	"github.com/Azure/go-autorest/autorest/azure/auth"
	"github.com/GoogleContainerTools/skaffold/pkg/skaffold/build"
	"github.com/GoogleContainerTools/skaffold/pkg/skaffold/build/tag"
	"github.com/GoogleContainerTools/skaffold/pkg/skaffold/docker"
	"github.com/GoogleContainerTools/skaffold/pkg/skaffold/schema/latest"
	"github.com/GoogleContainerTools/skaffold/pkg/skaffold/util"
	"github.com/pkg/errors"
	"io"
	"net/http"
	"regexp"
	"time"
)

const BUILD_STATUS_HEADER = "x-ms-meta-Complete"

func (b *Builder) Build(ctx context.Context, out io.Writer, tagger tag.Tagger, artifacts []*latest.Artifact) ([]build.Artifact, error) {
	return build.InParallel(ctx, out, tagger, artifacts, b.buildArtifact)
}

func (b *Builder) buildArtifact(ctx context.Context, out io.Writer, tagger tag.Tagger, artifact *latest.Artifact) (string, error) {
	client := cr.NewRegistriesClient(b.Credentials.SubscriptionId)
	authorizer, err := auth.NewClientCredentialsConfig(b.Credentials.ClientId, b.Credentials.ClientSecret, b.Credentials.TenantId).Authorizer()
	if err != nil {
		return "", errors.Wrap(err, "authorizing client")
	}
	client.Authorizer = authorizer

	result, err := client.GetBuildSourceUploadURL(ctx, b.ResourceGroup, b.ContainerRegistry)
	if err != nil {
		return "", errors.Wrap(err, "build source upload url")
	}
	blob := NewBlobStorage(*result.UploadURL)

	err = docker.CreateDockerTarGzContext(blob.Writer(), artifact.Workspace, artifact.DockerArtifact)
	if err != nil {
		return "", errors.Wrap(err, "create context tar.gz")
	}

	err = blob.UploadFileToBlob()
	if err != nil {
		return "", errors.Wrap(err, "upload file to blob")
	}

	imageTag, err := tagger.GenerateFullyQualifiedImageName(artifact.Workspace, &tag.Options{
		Digest:    util.RandomID(),
		ImageName: artifact.ImageName,
	})
	if err != nil {
		return "", errors.Wrap(err, "create fully qualified image name")
	}

	imageTag, err = getImageTagWithoutFQDN(imageTag)
	if err != nil {
		return "", errors.Wrap(err, "get azure image tag")
	}

	buildRequest := cr.DockerBuildRequest{
		ImageNames:     &[]string{imageTag},
		IsPushEnabled:  &[]bool{true}[0], //who invented bool pointers
		SourceLocation: result.RelativePath,
		Platform: &cr.PlatformProperties{
			Variant:      cr.V8,
			Os:           cr.Linux,
			Architecture: cr.Amd64,
		},
		DockerFilePath: &artifact.DockerArtifact.DockerfilePath,
		Type:           cr.TypeDockerBuildRequest,
	}
	future, err := client.ScheduleRun(ctx, b.ResourceGroup, b.ContainerRegistry, buildRequest)
	if err != nil {
		return "", errors.Wrap(err, "schedule build request")
	}

	run, err := future.Result(client)
	if err != nil {
		return "", errors.Wrap(err, "get run id")
	}
	runId := *run.RunID

	runsClient := cr.NewRunsClient(b.Credentials.SubscriptionId)
	runsClient.Authorizer = client.Authorizer
	logUrl, err := runsClient.GetLogSasURL(ctx, b.ResourceGroup, b.ContainerRegistry, runId)
	if err != nil {
		return "", errors.Wrap(err, "get log url")
	}

	err = pollBuildStatus(*logUrl.LogLink, out)
	if err != nil {
		return "", errors.Wrap(err, "polling build status")
	}

	return imageTag, nil
}

func pollBuildStatus(logUrl string, out io.Writer) error {
	offset := int32(0)
	for {
		resp, err := http.Get(logUrl)
		if err != nil {
			return err
		}

		if resp.StatusCode == http.StatusNotFound {
			//if blob is not available yet, try again
			time.Sleep(2 * time.Second)
			continue
		}

		scanner := bufio.NewScanner(resp.Body)
		line := int32(0)
		for scanner.Scan() {
			if line >= offset {
				out.Write(scanner.Bytes())
				out.Write([]byte("\n"))
				offset++
			}
			line++
		}
		resp.Body.Close()

		if offset > 0 {
			switch resp.Header.Get(BUILD_STATUS_HEADER) {
			case "":
				continue
			case "internalerror":
			case "failed":
				return errors.New("run failed")
			case "timedout":
				return errors.New("run timed out")
			case "canceled":
				return errors.New("run was canceled")
			default:
				return nil
			}
		}

		time.Sleep(2 * time.Second)
	}
}

// ACR needs the image tag in the following format
// <registryName>/<repository>:<tag>
func getImageTagWithoutFQDN(imageTag string) (string, error) {
	r, err := regexp.Compile("(.*)\\..*\\..*(/.*)")
	if err != nil {
		return "", errors.Wrap(err, "create regexp")
	}

	matches := r.FindStringSubmatch(imageTag)
	if len(matches) < 3 {
		return "", errors.New("invalid image tag")
	}

	return matches[1] + matches[2], nil
}
<|endoftext|>


<commit_before>package testcover

import (
	"bytes"
	"crypto/sha1"
	"encoding/binary"
	"encoding/hex"
)

func Fuzz(data []byte) int {
	if len(data) < 20 {
		return 0
	}
	x := binary.BigEndian.Uint32(data[12:])
	if x == 0x45839281 {
		bingo()
	}
	if data[10] == 0xfd && data[15] == 0x9a && data[17] == 0x71 {
		bingo()
	}
	switch binary.LittleEndian.Uint32(data[10:]) {
	default:
		bingo()
	case 0x12345678:
		bingo()
	case 0x98765432:
		bingo()
	}
	switch {
	case binary.LittleEndian.Uint32(data[8:]) == 0x12345678:
		bingo()
	default:
		bingo()
	case 0x98765432 == binary.BigEndian.Uint32(data[7:]):
		bingo()
	}

	switch string(data[5:9]) {
	case "ABCD":
		bingo()
	case "QWER":
		bingo()
	case "ZXCV":
		bingo()
	}

	n := binary.BigEndian.Uint32(data[0:4])
	if int(n) <= len(data)-4 {
		s := string(data[4 : 4+n])
		if s == "eat this" {
			bingo()
		}
	}

	if f := binary.BigEndian.Uint32(data[9:]) > 0xfffffffd; f {
		bingo()
	}

	type Hdr struct {
		Magic [8]byte
		N     uint32
	}
	var hdr Hdr
	binary.Read(bytes.NewReader(data), binary.LittleEndian, &hdr)
	if hdr.Magic == [8]byte{'m', 'a', 'g', 'i', 'c', 'h', 'd', 'r'} {
		bingo()
	}

	type Name string
	name := Name(data[4:9])
	if name == "12345" {
		bingo()
	}

	if len(data) > 40 {
		hash1 := sha1.Sum(data[0:20])
		var hash2 [20]byte
		binary.Read(bytes.NewReader(data[20:40]), binary.LittleEndian, &hash2)
		if hash1 == hash2 {
			bingo()
		}
	}

	for i := 0; i < 6; i++ {
		if data[i] != "CDATA["[i] {
			goto fail
		}
	}
	bingo()
fail:

	if varx, _ := binary.Uvarint(data[3:]); varx == 0xbadbeefc0ffee {
		bingo()
	}

	if data, err := hex.DecodeString(string(data[:6])); err == nil && string(data) == "foo" {
		bingo()
	}

        if data[0] != 'R' || data[1] != 'I' || data[2] != 'F' || data[3] != 'F' {
		bingo()
        }

	if x := binary.LittleEndian.Uint32(data[8:]); (x>>1)&(1<<24-1) == 11118709 {
		bingo()
	}

	return 0
}

func bingo() {
	if theFalse {
		bingo()
	}
}

var theFalse = false
<commit_msg>support upper/lower-case replacements in sonar also some improvements to versifier<commit_after>package testcover

import (
	"bytes"
	"crypto/sha1"
	"encoding/binary"
	"encoding/hex"
	"strings"
	"unicode"
)

func Fuzz(data []byte) int {
	if len(data) < 20 {
		return 0
	}
	x := binary.BigEndian.Uint32(data[12:])
	if x == 0x45839281 {
		bingo()
	}
	if data[10] == 0xfd && data[15] == 0x9a && data[17] == 0x71 {
		bingo()
	}
	switch binary.LittleEndian.Uint32(data[10:]) {
	default:
		bingo()
	case 0x12345678:
		bingo()
	case 0x98765432:
		bingo()
	}
	switch {
	case binary.LittleEndian.Uint32(data[8:]) == 0x12345678:
		bingo()
	default:
		bingo()
	case 0x98765432 == binary.BigEndian.Uint32(data[7:]):
		bingo()
	}

	switch string(data[5:9]) {
	case "ABCD":
		bingo()
	case "QWER":
		bingo()
	case "ZXCV":
		bingo()
	}

	n := binary.BigEndian.Uint32(data[0:4])
	if int(n) <= len(data)-4 {
		s := string(data[4 : 4+n])
		if s == "eat this" {
			bingo()
		}
	}

	if f := binary.BigEndian.Uint32(data[9:]) > 0xfffffffd; f {
		bingo()
	}

	type Hdr struct {
		Magic [8]byte
		N     uint32
	}
	var hdr Hdr
	binary.Read(bytes.NewReader(data), binary.LittleEndian, &hdr)
	if hdr.Magic == [8]byte{'m', 'a', 'g', 'i', 'c', 'h', 'd', 'r'} {
		bingo()
	}

	type Name string
	name := Name(data[4:9])
	if name == "12345" {
		bingo()
	}

	if len(data) > 40 {
		hash1 := sha1.Sum(data[0:20])
		var hash2 [20]byte
		binary.Read(bytes.NewReader(data[20:40]), binary.LittleEndian, &hash2)
		if hash1 == hash2 {
			bingo()
		}
	}

	for i := 0; i < 6; i++ {
		if data[i] != "CDATA["[i] {
			goto fail
		}
	}
	bingo()
fail:

	if varx, _ := binary.Uvarint(data[3:]); varx == 0xbadbeefc0ffee {
		bingo()
	}

	if data, err := hex.DecodeString(string(data[:6])); err == nil && string(data) == "foo" {
		bingo()
	}

	if data[0] != 'R' || data[1] != 'I' || data[2] != 'F' || data[3] != 'F' {
		bingo()
	}
	if unicode.ToLower(rune(data[0])) != 'a' &&
		unicode.ToLower(rune(data[1])) != 'b' &&
		unicode.ToLower(rune(data[2])) != 'c' &&
		unicode.ToLower(rune(data[3])) != 'd' &&
		unicode.ToLower(rune(data[4])) != 'e' &&
		unicode.ToLower(rune(data[5])) != 'f' &&
		unicode.ToLower(rune(data[6])) != 'g' &&
		unicode.ToLower(rune(data[7])) != 'h' {
		bingo()
	}

	if x := binary.LittleEndian.Uint32(data[8:]); (x>>1)&(1<<24-1) == 11118709 {
		bingo()
	}

	if strings.HasPrefix(string(data[2:]), "some prefix") {
		bingo()
	}
	if strings.HasSuffix(string(data[3:]), "some suffix") {
		bingo()
	}
	if strings.Index(string(data[4:]), "some index") != -1 {
		bingo()
	}
	if strings.IndexByte(string(data[4:]), 'X') != -1 {
		bingo()
	}
	if strings.Contains(string(data[1:]), "contained") {
		bingo()
	}
	if strings.ToLower(string(data[2:])) == "lower substr" {
		bingo()
	}
	if strings.ToUpper(string(data[2:])) == "UPPER SUBSTR" {
		bingo()
	}
	if strings.HasPrefix(strings.ToUpper(string(data[2:])), "UPPER PREFIX") {
		bingo()
	}

	if bytes.HasPrefix(data[3:], []byte("some prefix")) {
		bingo()
	}
	if bytes.HasSuffix(data[:len(data)-1], []byte("some suffix")) {
		bingo()
	}
	if bytes.Index(data[2:], []byte("bytes index")) != -1 {
		bingo()
	}
	if bytes.IndexByte(data[4:], 'Y') != -1 {
		bingo()
	}
	if bytes.Contains(data[1:], []byte("magic word")) {
		bingo()
	}
	if bytes.HasSuffix(bytes.ToUpper(data[:len(data)-1]), []byte("UPPER BYTE")) {
		bingo()
	}
	if bytes.Equal(bytes.ToLower(data[:11]), []byte("lower equal")) {
		bingo()
	}
	if bytes.Index(bytes.ToUpper(data[2:]), []byte("lower index")) != -1 {
		bingo()
	}

	return 0
}

func bingo() {
	if theFalse {
		bingo()
	}
}

var theFalse = false
<|endoftext|>


<commit_before>package tester

import (
	"encoding/json"
	"io/ioutil"
	"log"
	"os"
	"strings"

	"github.com/hashicorp/consul/api"
	"github.com/mitchellh/cli"
)

func UpgradeCommandFactory() (cli.Command, error) {
	return &Upgrade{}, nil
}

type Upgrade struct {
}

func (c *Upgrade) Help() string {
	helpText := `
Usage consul-live upgrade base version1 ... versionN

  Starts Consul using the base executable then shuts it down and upgrades in
  place using the supplied version executables. The base version is populated
  with some test data and that data is verified after each upgrade.
`
	return strings.TrimSpace(helpText)
}

func (c *Upgrade) Synopsis() string {
	return c.Help()
}

func (c *Upgrade) Run(args []string) int {
	if len(args) < 2 {
		log.Println("At least two versions must be given")
		return 1
	}

	if err := c.upgrade(args); err != nil {
		log.Println(err)
		return 1
	}
	return 0
}

type ServerConfig struct {
	Server           bool   `json:"server,omitempty"`
	Bootstrap        bool   `json:"bootstrap,omitempty"`
	DataDir          string `json:"data_dir,omitempty"`
	Datacenter       string `json:"datacenter,omitempty"`
	ACLMasterToken   string `json:"acl_master_token,omitempty"`
	ACLDatacenter    string `json:"acl_datacenter,omitempty"`
	ACLDefaultPolicy string `json:"acl_default_policy,omitempty"`
	LogLevel         string `json:"log_level,omitempty"`
}

func (c *Upgrade) upgrade(versions []string) error {
	base := versions[0]
	versions = versions[1:]

	var dir string
	var err error
	dir, err = ioutil.TempDir("", "consul")
	if err != nil {
		return err
	}
	defer os.RemoveAll(dir)

	config, err := ioutil.TempFile(dir, "config")
	if err != nil {
		return err
	}

	content, err := json.Marshal(ServerConfig{
		Server:           true,
		Bootstrap:        true,
		DataDir:          dir,
		Datacenter:       "dc1",
		ACLMasterToken:   "root",
		ACLDatacenter:    "dc1",
		ACLDefaultPolicy: "allow",
	})
	if err != nil {
		return err
	}
	if _, err := config.Write(content); err != nil {
		return err
	}
	if err := config.Close(); err != nil {
		return err
	}

	// Start the first version of Consul, which is our base.
	log.Printf("Starting base Consul from '%s'...\n", base)
	args := []string{
		"agent",
		"-config-file",
		config.Name(),
	}
	consul, err := NewConsul(base, args)
	if err != nil {
		return err
	}
	if err := consul.Start(); err != nil {
		return err
	}
	defer func() {
		if err := consul.Shutdown(); err != nil {
			log.Println(err)
		}
	}()

	// Wait for it to start up and elect itself.
	if err := consul.WaitForLeader(); err != nil {
		return err
	}

	// Populate it with some realistic data, enough to kick out a snapshot.
	log.Println("Populating with initial state store data...")
	client, err := api.NewClient(api.DefaultConfig())
	if err != nil {
		return err
	}
	fuzz, err := NewFuzz(client)
	if err != nil {
		return err
	}
	for {
		if err := fuzz.Populate(); err != nil {
			return err
		}

		entries, err := ioutil.ReadDir(dir + "/raft/snapshots/")
		if err != nil {
			return err
		}
		if len(entries) > 0 {
			break
		}
	}

	// Push some data in post-snapshot to make sure there's some stuff
	// in the Raft log as well.
	if err := fuzz.Populate(); err != nil {
		return err
	}
	if err := fuzz.Verify(); err != nil {
		return err
	}

	// Now shutdown the base version and try upgrading through the given
	// versions.
	if err := consul.Shutdown(); err != nil {
		return err
	}
	for _, version := range versions {
		// Start the upgraded version with the same data-dir.
		log.Printf("Upgrading to Consul from '%s'...\n", version)
		upgrade, err := NewConsul(version, args)
		if err != nil {
			return err
		}
		if err := upgrade.Start(); err != nil {
			return err
		}
		defer func() {
			if err := upgrade.Shutdown(); err != nil {
				log.Println(err)
			}
		}()

		// Wait for it to start up and elect itself.
		if err := upgrade.WaitForLeader(); err != nil {
			return err
		}

		// Make sure the data is still present.
		if err := fuzz.Verify(); err != nil {
			return err
		}

		// Add some new data for this version of Consul.
		if err := fuzz.Populate(); err != nil {
			return err
		}
		if err := fuzz.Verify(); err != nil {
			return err
		}

		// Shut it down in anticipation of the next upgrade.
		if err := upgrade.Shutdown(); err != nil {
			return err
		}
	}

	log.Println("Upgrade series complete")
	return nil
}
<commit_msg>Adds a bind parameter.<commit_after>package tester

import (
	"encoding/json"
	"io/ioutil"
	"log"
	"os"
	"strings"

	"github.com/hashicorp/consul/api"
	"github.com/mitchellh/cli"
)

func UpgradeCommandFactory() (cli.Command, error) {
	return &Upgrade{}, nil
}

type Upgrade struct {
}

func (c *Upgrade) Help() string {
	helpText := `
Usage consul-live upgrade base version1 ... versionN

  Starts Consul using the base executable then shuts it down and upgrades in
  place using the supplied version executables. The base version is populated
  with some test data and that data is verified after each upgrade.
`
	return strings.TrimSpace(helpText)
}

func (c *Upgrade) Synopsis() string {
	return c.Help()
}

func (c *Upgrade) Run(args []string) int {
	if len(args) < 2 {
		log.Println("At least two versions must be given")
		return 1
	}

	if err := c.upgrade(args); err != nil {
		log.Println(err)
		return 1
	}
	return 0
}

type ServerConfig struct {
	Server           bool   `json:"server,omitempty"`
	Bootstrap        bool   `json:"bootstrap,omitempty"`
	Bind             string `json:"bind_addr,omitempty"`
	DataDir          string `json:"data_dir,omitempty"`
	Datacenter       string `json:"datacenter,omitempty"`
	ACLMasterToken   string `json:"acl_master_token,omitempty"`
	ACLDatacenter    string `json:"acl_datacenter,omitempty"`
	ACLDefaultPolicy string `json:"acl_default_policy,omitempty"`
	LogLevel         string `json:"log_level,omitempty"`
}

func (c *Upgrade) upgrade(versions []string) error {
	base := versions[0]
	versions = versions[1:]

	var dir string
	var err error
	dir, err = ioutil.TempDir("", "consul")
	if err != nil {
		return err
	}
	defer os.RemoveAll(dir)

	config, err := ioutil.TempFile(dir, "config")
	if err != nil {
		return err
	}

	content, err := json.Marshal(ServerConfig{
		Server:           true,
		Bootstrap:        true,
		Bind:             "127.0.0.1",
		DataDir:          dir,
		Datacenter:       "dc1",
		ACLMasterToken:   "root",
		ACLDatacenter:    "dc1",
		ACLDefaultPolicy: "allow",
	})
	if err != nil {
		return err
	}
	if _, err := config.Write(content); err != nil {
		return err
	}
	if err := config.Close(); err != nil {
		return err
	}

	// Start the first version of Consul, which is our base.
	log.Printf("Starting base Consul from '%s'...\n", base)
	args := []string{
		"agent",
		"-config-file",
		config.Name(),
	}
	consul, err := NewConsul(base, args)
	if err != nil {
		return err
	}
	if err := consul.Start(); err != nil {
		return err
	}
	defer func() {
		if err := consul.Shutdown(); err != nil {
			log.Println(err)
		}
	}()

	// Wait for it to start up and elect itself.
	if err := consul.WaitForLeader(); err != nil {
		return err
	}

	// Populate it with some realistic data, enough to kick out a snapshot.
	log.Println("Populating with initial state store data...")
	client, err := api.NewClient(api.DefaultConfig())
	if err != nil {
		return err
	}
	fuzz, err := NewFuzz(client)
	if err != nil {
		return err
	}
	for {
		if err := fuzz.Populate(); err != nil {
			return err
		}

		entries, err := ioutil.ReadDir(dir + "/raft/snapshots/")
		if err != nil {
			return err
		}
		if len(entries) > 0 {
			break
		}
	}

	// Push some data in post-snapshot to make sure there's some stuff
	// in the Raft log as well.
	if err := fuzz.Populate(); err != nil {
		return err
	}
	if err := fuzz.Verify(); err != nil {
		return err
	}

	// Now shutdown the base version and try upgrading through the given
	// versions.
	if err := consul.Shutdown(); err != nil {
		return err
	}
	for _, version := range versions {
		// Start the upgraded version with the same data-dir.
		log.Printf("Upgrading to Consul from '%s'...\n", version)
		upgrade, err := NewConsul(version, args)
		if err != nil {
			return err
		}
		if err := upgrade.Start(); err != nil {
			return err
		}
		defer func() {
			if err := upgrade.Shutdown(); err != nil {
				log.Println(err)
			}
		}()

		// Wait for it to start up and elect itself.
		if err := upgrade.WaitForLeader(); err != nil {
			return err
		}

		// Make sure the data is still present.
		if err := fuzz.Verify(); err != nil {
			return err
		}

		// Add some new data for this version of Consul.
		if err := fuzz.Populate(); err != nil {
			return err
		}
		if err := fuzz.Verify(); err != nil {
			return err
		}

		// Shut it down in anticipation of the next upgrade.
		if err := upgrade.Shutdown(); err != nil {
			return err
		}
	}

	log.Println("Upgrade series complete")
	return nil
}
<|endoftext|>


<commit_before>package models

import (
  "path"
  "io/ioutil"
  "os"
  "encoding/json"
  "ghighlighter/utils"
)

type GhReadings struct {
  Items []GhReading
}

func (m *GhReadings) FindByTitle(title string) *GhReading {
  for _, reading := range m.Items {
    if reading.Title == title {
      return &reading
    }
  }
  return nil
}

func readingsFilePath() string {
  return path.Join(utils.DataDir(), "readings.json")
}

func (m *GhReadings) Read() {
  data, error := ioutil.ReadFile(readingsFilePath())
  if error != nil && !os.IsExist(error) {
    m.Write()
    m.Read()
  }

  var tmpItems []GhReading
  json.Unmarshal(data, &tmpItems)

  for _, reading := range tmpItems {
    m.Items = append(m.Items, reading)
  }
}

func (m *GhReadings) Write() {
  if m.Items == nil {
    m.Items = make([]GhReading, 0)
  }

  data, error := json.Marshal(m.Items)
  if error != nil { return }

  error = ioutil.WriteFile(readingsFilePath(), data, 0755)
}

func Readings() *GhReadings {
  readings := &GhReadings{}
  readings.Read()
  return readings
}

<commit_msg>Add Readings.FindByReadmillId<commit_after>package models

import (
  "path"
  "io/ioutil"
  "os"
  "encoding/json"
  "ghighlighter/utils"
)

type GhReadings struct {
  Items []GhReading
}

func (m *GhReadings) FindByTitle(title string) *GhReading {
  for _, reading := range m.Items {
    if reading.Title == title {
      return &reading
    }
  }
  return nil
}

func (m *GhReadings) FindByReadmillId(readmillId int) *GhReading {
  for _, reading := range m.Items {
    if reading.ReadmillId == readmillId {
      return &reading
    }
  }
  return nil
}

func readingsFilePath() string {
  return path.Join(utils.DataDir(), "readings.json")
}

func (m *GhReadings) Read() {
  data, error := ioutil.ReadFile(readingsFilePath())
  if error != nil && !os.IsExist(error) {
    m.Write()
    m.Read()
  }

  var tmpItems []GhReading
  json.Unmarshal(data, &tmpItems)

  for _, reading := range tmpItems {
    m.Items = append(m.Items, reading)
  }
}

func (m *GhReadings) Write() {
  if m.Items == nil {
    m.Items = make([]GhReading, 0)
  }

  data, error := json.Marshal(m.Items)
  if error != nil { return }

  error = ioutil.WriteFile(readingsFilePath(), data, 0755)
}

func Readings() *GhReadings {
  readings := &GhReadings{}
  readings.Read()
  return readings
}

<|endoftext|>