<commit_before>package jiracmd

import (
	"bytes"
	"fmt"
	"io/ioutil"
	"os"
	"path"

	"github.com/coryb/figtree"
	"github.com/coryb/oreo"
	"gopkg.in/Netflix-Skunkworks/go-jira.v1/jiracli"
	kingpin "gopkg.in/alecthomas/kingpin.v2"
)

func CmdUnexportTemplatesRegistry() *jiracli.CommandRegistryEntry {
	opts := ExportTemplatesOptions{}

	return &jiracli.CommandRegistryEntry{
		"Remove unmodified exported templates",
		func(fig *figtree.FigTree, cmd *kingpin.CmdClause) error {
			jiracli.LoadConfigs(cmd, fig, &opts)
			return CmdExportTemplatesUsage(cmd, &opts)
		},
		func(o *oreo.Client, globals *jiracli.GlobalOptions) error {
			if opts.Dir != "" {
				opts.Dir = fmt.Sprintf("%s/.jira.d/templates", jiracli.Homedir())
			}
			return CmdUnexportTemplates(globals, &opts)
		},
	}
}

// CmdUnexportTemplates will remove unmodified templates from export directory
func CmdUnexportTemplates(globals *jiracli.GlobalOptions, opts *ExportTemplatesOptions) error {
	for name, template := range jiracli.AllTemplates {
		if opts.Template != "" && opts.Template != name {
			continue
		}
		templateFile := path.Join(opts.Dir, name)
		if _, err := os.Stat(templateFile); err != nil {
			log.Warning("Skipping %s, not found", templateFile)
			continue
		}
		// open, read, compare
		contents, err := ioutil.ReadFile(templateFile)
		if err != nil {
			return err
		}
		if bytes.Compare([]byte(template), contents) == 0 {
			if !globals.Quiet.Value {
				log.Notice("Removing %s, template identical to default", templateFile)
			}
			os.Remove(templateFile)
		} else {
			if !globals.Quiet.Value {
				log.Notice("Skipping %s, found customizations to template", templateFile)
			}
		}
	}
	return nil
}
<commit_msg>Update unexportTemplates.go<commit_after>package jiracmd

import (
	"bytes"
	"fmt"
	"io/ioutil"
	"os"
	"path"

	"github.com/coryb/figtree"
	"github.com/coryb/oreo"
	"gopkg.in/Netflix-Skunkworks/go-jira.v1/jiracli"
	kingpin "gopkg.in/alecthomas/kingpin.v2"
)

func CmdUnexportTemplatesRegistry() *jiracli.CommandRegistryEntry {
	opts := ExportTemplatesOptions{}

	return &jiracli.CommandRegistryEntry{
		"Remove unmodified exported templates",
		func(fig *figtree.FigTree, cmd *kingpin.CmdClause) error {
			jiracli.LoadConfigs(cmd, fig, &opts)
			return CmdExportTemplatesUsage(cmd, &opts)
		},
		func(o *oreo.Client, globals *jiracli.GlobalOptions) error {
			if opts.Dir == "" {
				opts.Dir = fmt.Sprintf("%s/.jira.d/templates", jiracli.Homedir())
			}
			return CmdUnexportTemplates(globals, &opts)
		},
	}
}

// CmdUnexportTemplates will remove unmodified templates from export directory
func CmdUnexportTemplates(globals *jiracli.GlobalOptions, opts *ExportTemplatesOptions) error {
	for name, template := range jiracli.AllTemplates {
		if opts.Template != "" && opts.Template != name {
			continue
		}
		templateFile := path.Join(opts.Dir, name)
		if _, err := os.Stat(templateFile); err != nil {
			log.Warning("Skipping %s, not found", templateFile)
			continue
		}
		// open, read, compare
		contents, err := ioutil.ReadFile(templateFile)
		if err != nil {
			return err
		}
		if bytes.Compare([]byte(template), contents) == 0 {
			if !globals.Quiet.Value {
				log.Notice("Removing %s, template identical to default", templateFile)
			}
			os.Remove(templateFile)
		} else {
			if !globals.Quiet.Value {
				log.Notice("Skipping %s, found customizations to template", templateFile)
			}
		}
	}
	return nil
}
<|endoftext|>


<commit_before>/* This Source Code Form is subject to the terms of the Mozilla Public
 * License, v. 2.0. If a copy of the MPL was not distributed with this
 * file, You can obtain one at http://mozilla.org/MPL/2.0/. */

package simplepush

import (
	"fmt"
	"math/rand"
	"path"
	"strings"
	"sync"
	"time"

	"github.com/coreos/go-etcd/etcd"
)

const (
	minTTL = 2 * time.Second
)

var ErrMinTTL = fmt.Errorf("Default TTL too short; want at least %s", minTTL)

type EtcdLocatorConf struct {
	// Dir is the etcd key prefix for storing contacts. Defaults to
	// "push_hosts".
	Dir string `toml:"dir"`

	// BucketSize is the maximum number of requests that the router should send
	// before checking replies. Defaults to 10.
	BucketSize int `toml:"bucket_size"`

	// Servers is a list of etcd servers.
	Servers []string

	// DefaultTTL is the maximum amount of time that registered contacts will be
	// considered valid. Defaults to "24h".
	DefaultTTL string

	// RefreshInterval is the maximum amount of time that a cached contact list
	// will be considered valid. Defaults to "5m".
	RefreshInterval string `toml:"refresh_interval"`
}

// EtcdLocator stores routing endpoints in etcd and periodically polls for new
// contacts.
type EtcdLocator struct {
	sync.Mutex
	logger          *SimpleLogger
	metrics         *Metrics
	refreshInterval time.Duration
	defaultTTL      time.Duration
	bucketSize      int
	serverList      []string
	dir             string
	authority       string
	key             string
	client          *etcd.Client
	lastRefresh     time.Time
	isClosing       bool
	closeSignal     chan bool
	closeWait       sync.WaitGroup
	closeLock       sync.Mutex
	lastErr         error
}

func NewEtcdLocator() *EtcdLocator {
	return &EtcdLocator{
		closeSignal: make(chan bool),
	}
}

func (*EtcdLocator) ConfigStruct() interface{} {
	return &EtcdLocatorConf{
		Dir:             "push_hosts",
		BucketSize:      10,
		Servers:         []string{"http://localhost:4001"},
		DefaultTTL:      "24h",
		RefreshInterval: "5m",
	}
}

func (l *EtcdLocator) Init(app *Application, config interface{}) (err error) {
	conf := config.(*EtcdLocatorConf)
	l.logger = app.Logger()
	l.metrics = app.Metrics()

	l.refreshInterval, err = time.ParseDuration(conf.RefreshInterval)
	if err != nil {
		l.logger.Error("etcd", "Could not parse refreshInterval",
			LogFields{"error": err.Error(),
				"refreshInterval": conf.RefreshInterval})
		return
	}
	// default time for the server to be "live"
	l.defaultTTL, err = time.ParseDuration(conf.DefaultTTL)
	if err != nil {
		l.logger.Critical("etcd",
			"Could not parse etcd default TTL",
			LogFields{"value": conf.DefaultTTL, "error": err.Error()})
		return
	}
	if l.defaultTTL < minTTL {
		l.logger.Critical("etcd",
			"default TTL too short",
			LogFields{"value": conf.DefaultTTL})
		return ErrMinTTL
	}

	l.bucketSize = conf.BucketSize
	l.serverList = conf.Servers
	l.dir = path.Clean(conf.Dir)

	// The authority of the current server is used as the etcd key.
	router := app.Router()
	if hostname := router.hostname; hostname != "" {
		if router.scheme == "https" && router.port != 443 || router.scheme == "http" && router.port != 80 {
			l.authority = fmt.Sprintf("%s:%d", hostname, router.port)
		} else {
			l.authority = hostname
		}
		l.key = path.Join(l.dir, l.authority)
	}

	l.logger.Debug("etcd", "connecting to etcd servers",
		LogFields{"list": strings.Join(l.serverList, ";")})
	l.client = etcd.NewClient(l.serverList)

	// create the push hosts directory (if not already there)
	_, err = l.client.CreateDir(l.dir, 0)
	if err != nil {
		clientErr, ok := err.(*etcd.EtcdError)
		if !ok || clientErr.ErrorCode != 105 {
			l.logger.Error("etcd", "etcd createDir error", LogFields{
				"error": err.Error()})
			return
		}
	}
	if _, err = l.getServers(); err != nil {
		l.logger.Critical("etcd", "Could not initialize server list",
			LogFields{"error": err.Error()})
		return
	}
	if err = l.Register(); err != nil {
		l.logger.Critical("etcd", "Could not register with etcd",
			LogFields{"error": err.Error()})
		return
	}

	l.closeWait.Add(1)
	go l.refresh()
	return
}

// Close stops the locator and closes the etcd client connection. Implements
// Locator.Close().
func (l *EtcdLocator) Close() (err error) {
	defer l.closeLock.Unlock()
	l.closeLock.Lock()
	if l.isClosing {
		return l.lastErr
	}
	close(l.closeSignal)
	l.closeWait.Wait()
	if l.key != "" {
		_, err = l.client.Delete(l.key, false)
	}
	l.isClosing = true
	l.lastErr = err
	return
}

// Contacts returns a shuffled list of all nodes in the Simple Push cluster.
// Implements Locator.Contacts().
func (l *EtcdLocator) Contacts(string) ([]string, error) {
	contacts, err := l.getServers()
	if err != nil {
		l.logger.Error("etcd", "Could not get server list",
			LogFields{"error": err.Error()})
		return nil, err
	}
	for length := len(contacts); length > 0; {
		index := rand.Intn(length)
		length--
		contacts[index], contacts[length] = contacts[length], contacts[index]
	}
	return contacts, nil
}

// MaxParallel returns the maximum number of requests that the router should
// send in parallel. Implements Locator.MaxParallel().
func (l *EtcdLocator) MaxParallel() int {
	return l.bucketSize
}

// Register registers the server to the etcd cluster.
func (l *EtcdLocator) Register() error {
	if l.logger.ShouldLog(DEBUG) {
		l.logger.Debug("etcd", "Registering host", LogFields{"host": l.authority})
	}
	if _, err := l.client.Set(l.key, l.authority, uint64(l.defaultTTL/time.Second)); err != nil {
		l.logger.Error("etcd", "Failed to register",
			LogFields{"error": err.Error(),
				"key":  l.key,
				"host": l.authority})
		return err
	}
	return nil
}

// getServers gets the contact list from etcd.
func (l *EtcdLocator) getServers() ([]string, error) {
	l.Lock()
	defer l.Unlock()
	if time.Now().Sub(l.lastRefresh) < l.refreshInterval {
		return l.serverList, nil
	}
	nodeList, err := l.client.Get(l.dir, false, false)
	if err != nil {
		return nil, err
	}
	reply := make([]string, 0, len(nodeList.Node.Nodes))
	for _, node := range nodeList.Node.Nodes {
		if node.Value == l.authority || node.Value == "" {
			continue
		}
		reply = append(reply, node.Value)
	}
	l.serverList = reply
	l.lastRefresh = time.Now()
	return reply, nil
}

// refresh periodically re-registers the host with etcd.
func (l *EtcdLocator) refresh() {
	defer l.closeWait.Done()
	// auto refresh slightly more often than the TTL
	timeout := 0.75 * l.defaultTTL.Seconds()
	ticker := time.NewTicker(time.Duration(timeout) * time.Second)
	for ok := true; ok; {
		select {
		case ok = <-l.closeSignal:
		case <-ticker.C:
			l.Register()
		}
	}
	ticker.Stop()
}

func init() {
	AvailableLocators["etcd"] = func() HasConfigStruct { return NewEtcdLocator() }
}
<commit_msg>`EtcdLocator`: Periodically refresh contact lists.<commit_after>/* This Source Code Form is subject to the terms of the Mozilla Public
 * License, v. 2.0. If a copy of the MPL was not distributed with this
 * file, You can obtain one at http://mozilla.org/MPL/2.0/. */

package simplepush

import (
	"fmt"
	"io"
	"math/rand"
	"path"
	"strings"
	"sync"
	"time"

	"github.com/coreos/go-etcd/etcd"
)

const (
	minTTL = 2 * time.Second
)

var ErrMinTTL = fmt.Errorf("Default TTL too short; want at least %s", minTTL)

type EtcdLocatorConf struct {
	// Dir is the etcd key prefix for storing contacts. Defaults to
	// "push_hosts".
	Dir string `toml:"dir"`

	// BucketSize is the maximum number of requests that the router should send
	// before checking replies. Defaults to 10.
	BucketSize int `toml:"bucket_size"`

	// Servers is a list of etcd servers.
	Servers []string

	// DefaultTTL is the maximum amount of time that registered contacts will be
	// considered valid. Defaults to "24h".
	DefaultTTL string

	// RefreshInterval is the maximum amount of time that a cached contact list
	// will be considered valid. Defaults to "5m".
	RefreshInterval string `toml:"refresh_interval"`
}

// etcdFetch is an etcd contact list request.
type etcdFetch struct {
	replies chan []string
	errors  chan error
}

// EtcdLocator stores routing endpoints in etcd and polls for new contacts.
type EtcdLocator struct {
	sync.Mutex
	logger          *SimpleLogger
	metrics         *Metrics
	refreshInterval time.Duration
	defaultTTL      time.Duration
	bucketSize      int
	serverList      []string
	dir             string
	authority       string
	key             string
	client          *etcd.Client
	fetches         chan etcdFetch
	isClosing       bool
	closeSignal     chan bool
	closeWait       sync.WaitGroup
	closeLock       sync.Mutex
	lastErr         error
}

func NewEtcdLocator() *EtcdLocator {
	return &EtcdLocator{
		fetches:     make(chan etcdFetch),
		closeSignal: make(chan bool),
	}
}

func (*EtcdLocator) ConfigStruct() interface{} {
	return &EtcdLocatorConf{
		Dir:             "push_hosts",
		BucketSize:      10,
		Servers:         []string{"http://localhost:4001"},
		DefaultTTL:      "24h",
		RefreshInterval: "5m",
	}
}

func (l *EtcdLocator) Init(app *Application, config interface{}) (err error) {
	conf := config.(*EtcdLocatorConf)
	l.logger = app.Logger()
	l.metrics = app.Metrics()

	if l.refreshInterval, err = time.ParseDuration(conf.RefreshInterval); err != nil {
		l.logger.Error("etcd", "Could not parse refreshInterval",
			LogFields{"error": err.Error(),
				"refreshInterval": conf.RefreshInterval})
		return err
	}
	// default time for the server to be "live"
	if l.defaultTTL, err = time.ParseDuration(conf.DefaultTTL); err != nil {
		l.logger.Critical("etcd",
			"Could not parse etcd default TTL",
			LogFields{"value": conf.DefaultTTL, "error": err.Error()})
		return err
	}
	if l.defaultTTL < minTTL {
		l.logger.Critical("etcd",
			"default TTL too short",
			LogFields{"value": conf.DefaultTTL})
		return ErrMinTTL
	}

	l.bucketSize = conf.BucketSize
	l.serverList = conf.Servers
	l.dir = path.Clean(conf.Dir)

	// The authority of the current server is used as the etcd key.
	router := app.Router()
	if hostname := router.hostname; hostname != "" {
		if router.scheme == "https" && router.port != 443 || router.scheme == "http" && router.port != 80 {
			l.authority = fmt.Sprintf("%s:%d", hostname, router.port)
		} else {
			l.authority = hostname
		}
		l.key = path.Join(l.dir, l.authority)
	}

	l.logger.Debug("etcd", "connecting to etcd servers",
		LogFields{"list": strings.Join(l.serverList, ";")})
	l.client = etcd.NewClient(l.serverList)

	// create the push hosts directory (if not already there)
	if _, err = l.client.CreateDir(l.dir, 0); err != nil {
		clientErr, ok := err.(*etcd.EtcdError)
		if !ok || clientErr.ErrorCode != 105 {
			l.logger.Error("etcd", "etcd createDir error", LogFields{
				"error": err.Error()})
			return err
		}
	}
	if err = l.Register(); err != nil {
		l.logger.Critical("etcd", "Could not register with etcd",
			LogFields{"error": err.Error()})
		return err
	}

	l.closeWait.Add(2)
	go l.registerLoop()
	go l.fetchLoop()
	return nil
}

// Close stops the locator and closes the etcd client connection. Implements
// Locator.Close().
func (l *EtcdLocator) Close() (err error) {
	defer l.closeLock.Unlock()
	l.closeLock.Lock()
	if l.isClosing {
		return l.lastErr
	}
	close(l.closeSignal)
	l.closeWait.Wait()
	if l.key != "" {
		_, err = l.client.Delete(l.key, false)
	}
	l.isClosing = true
	l.lastErr = err
	return
}

// Contacts returns a shuffled list of all nodes in the Simple Push cluster.
// Implements Locator.Contacts().
func (l *EtcdLocator) Contacts(string) (contacts []string, err error) {
	replies, errors := make(chan []string, 1), make(chan error, 1)
	l.fetches <- etcdFetch{replies, errors}
	select {
	case <-l.closeSignal:
		return nil, io.EOF

	case contacts = <-replies:
	case err = <-errors:
	}
	if err != nil {
		l.logger.Error("etcd", "Could not get server list",
			LogFields{"error": err.Error()})
		return
	}
	for length := len(contacts); length > 0; {
		index := rand.Intn(length)
		length--
		contacts[index], contacts[length] = contacts[length], contacts[index]
	}
	return
}

// MaxParallel returns the maximum number of requests that the router should
// send in parallel. Implements Locator.MaxParallel().
func (l *EtcdLocator) MaxParallel() int {
	return l.bucketSize
}

// Register registers the server to the etcd cluster.
func (l *EtcdLocator) Register() (err error) {
	if l.logger.ShouldLog(DEBUG) {
		l.logger.Debug("etcd", "Registering host", LogFields{"host": l.authority})
	}
	if _, err = l.client.Set(l.key, l.authority, uint64(l.defaultTTL/time.Second)); err != nil {
		l.logger.Error("etcd", "Failed to register",
			LogFields{"error": err.Error(),
				"key":  l.key,
				"host": l.authority})
		return
	}
	return
}

// getServers gets the current contact list from etcd.
func (l *EtcdLocator) getServers() (servers []string, err error) {
	nodeList, err := l.client.Get(l.dir, false, false)
	if err != nil {
		return nil, err
	}
	reply := make([]string, 0, len(nodeList.Node.Nodes))
	for _, node := range nodeList.Node.Nodes {
		if node.Value == l.authority || node.Value == "" {
			continue
		}
		reply = append(reply, node.Value)
	}
	return reply, nil
}

// refreshLoop periodically re-registers the current node with etcd.
func (l *EtcdLocator) registerLoop() {
	defer l.closeWait.Done()
	// auto refresh slightly more often than the TTL
	timeout := 0.75 * l.defaultTTL.Seconds()
	ticker := time.NewTicker(time.Duration(timeout) * time.Second)
	for ok := true; ok; {
		select {
		case ok = <-l.closeSignal:
		case <-ticker.C:
			l.Register()
		}
	}
	ticker.Stop()
}

// fetchLoop polls etcd for new nodes and responds to requests for contacts.
func (l *EtcdLocator) fetchLoop() {
	defer l.closeWait.Done()
	var (
		lastReply   []string
		lastRefresh time.Time
	)
	for ok := true; ok; {
		select {
		case ok = <-l.closeSignal:
		case <-time.After(l.refreshInterval):
			if reply, err := l.getServers(); err == nil {
				lastReply = reply
				lastRefresh = time.Now()
			}

		case fetch := <-l.fetches:
			if !lastRefresh.IsZero() && time.Now().Sub(lastRefresh) < l.refreshInterval {
				fetch.replies <- lastReply
				break
			}
			reply, err := l.getServers()
			if err != nil {
				fetch.errors <- err
				break
			}
			lastReply = reply
			lastRefresh = time.Now()
			fetch.replies <- reply
		}
	}
}

func init() {
	AvailableLocators["etcd"] = func() HasConfigStruct { return NewEtcdLocator() }
}
<|endoftext|>


<commit_before>// Copyright 2015 The Chromium Authors. All rights reserved.
// Use of this source code is governed by a BSD-style license that can be
// found in the LICENSE file.

package helper

import (
	"bytes"
	"io"
	"testing"
	"time"

	"github.com/luci/gae"
	"github.com/luci/luci-go/common/cmpbin"
	. "github.com/smartystreets/goconvey/convey"
)

func init() {
	WriteDSPropertyMapDeterministic = true
}

type dspmapTC struct {
	name  string
	props gae.DSPropertyMap
}

func TestDSPropertyMapSerialization(t *testing.T) {
	t.Parallel()

	tests := []dspmapTC{
		{
			"basic",
			gae.DSPropertyMap{
				"R": {mp(false), mp(2.1), mpNI(3)},
				"S": {mp("hello"), mp("world")},
			},
		},
		{
			"keys",
			gae.DSPropertyMap{
				"DS": {mp(mkKey("appy", "ns", "Foo", 7)), mp(mkKey("other", "", "Yot", "wheeep"))},
				"BS": {mp(gae.BSKey("sup")), mp(gae.BSKey("nerds"))},
			},
		},
		{
			"geo",
			gae.DSPropertyMap{
				"G": {mp(gae.DSGeoPoint{Lat: 1, Lng: 2})},
			},
		},
		{
			"data",
			gae.DSPropertyMap{
				"S":          {mp("sup"), mp("fool"), mp("nerd")},
				"D.Foo.Nerd": {mp([]byte("sup")), mp([]byte("fool"))},
				"B":          {mp(gae.DSByteString("sup")), mp(gae.DSByteString("fool"))},
			},
		},
		{
			"time",
			gae.DSPropertyMap{
				"T": {
					mp(time.Now().UTC()),
					mp(time.Now().Add(time.Second).UTC())},
			},
		},
		{
			"empty vals",
			gae.DSPropertyMap{
				"T": {mp(true), mp(true)},
				"F": {mp(false), mp(false)},
				"N": {mp(nil), mp(nil)},
				"E": {},
			},
		},
	}

	Convey("DSPropertyMap serialization", t, func() {
		Convey("round trip", func() {
			for _, tc := range tests {
				tc := tc
				Convey(tc.name, func() {
					buf := &bytes.Buffer{}
					WriteDSPropertyMap(buf, tc.props, WithContext)
					dec, err := ReadDSPropertyMap(buf, WithContext, "", "")
					So(err, ShouldBeNil)
					So(dec, ShouldResemble, tc.props)
				})
			}
		})
	})
}

func TestSerializationReadMisc(t *testing.T) {
	t.Parallel()

	Convey("Misc Serialization tests", t, func() {
		Convey("ReadDSKey", func() {
			Convey("good cases", func() {
				Convey("w/ ctx decodes normally w/ ctx", func() {
					k := mkKey("aid", "ns", "knd", "yo", "other", 10)
					buf := &bytes.Buffer{}
					WriteDSKey(buf, WithContext, k)
					dk, err := ReadDSKey(buf, WithContext, "", "")
					So(err, ShouldBeNil)
					So(dk, ShouldEqualKey, k)
				})
				Convey("w/ ctx decodes normally w/o ctx", func() {
					k := mkKey("aid", "ns", "knd", "yo", "other", 10)
					buf := &bytes.Buffer{}
					WriteDSKey(buf, WithContext, k)
					dk, err := ReadDSKey(buf, WithoutContext, "spam", "nerd")
					So(err, ShouldBeNil)
					So(dk, ShouldEqualKey, mkKey("spam", "nerd", "knd", "yo", "other", 10))
				})
				Convey("w/o ctx decodes normally w/ ctx", func() {
					k := mkKey("aid", "ns", "knd", "yo", "other", 10)
					buf := &bytes.Buffer{}
					WriteDSKey(buf, WithoutContext, k)
					dk, err := ReadDSKey(buf, WithContext, "spam", "nerd")
					So(err, ShouldBeNil)
					So(dk, ShouldEqualKey, mkKey("", "", "knd", "yo", "other", 10))
				})
				Convey("w/o ctx decodes normally w/o ctx", func() {
					k := mkKey("aid", "ns", "knd", "yo", "other", 10)
					buf := &bytes.Buffer{}
					WriteDSKey(buf, WithoutContext, k)
					dk, err := ReadDSKey(buf, WithoutContext, "spam", "nerd")
					So(err, ShouldBeNil)
					So(dk, ShouldEqualKey, mkKey("spam", "nerd", "knd", "yo", "other", 10))
				})
				Convey("IntIDs always sort before StringIDs", func() {
					// -1 writes as almost all 1's in the first byte under cmpbin, even
					// though it's technically not a valid key.
					k := mkKey("aid", "ns", "knd", -1)
					buf := &bytes.Buffer{}
					WriteDSKey(buf, WithoutContext, k)

					k = mkKey("aid", "ns", "knd", "hat")
					buf2 := &bytes.Buffer{}
					WriteDSKey(buf2, WithoutContext, k)

					So(bytes.Compare(buf.Bytes(), buf2.Bytes()), ShouldBeLessThan, 0)
				})
			})

			Convey("err cases", func() {
				Convey("nil", func() {
					buf := &bytes.Buffer{}
					_, err := ReadDSKey(buf, WithContext, "", "")
					So(err, ShouldEqual, io.EOF)
				})
				Convey("str", func() {
					buf := &bytes.Buffer{}
					buf.WriteString("sup")
					_, err := ReadDSKey(buf, WithContext, "", "")
					So(err, ShouldErrLike, "expected actualCtx")
				})
				Convey("truncated 1", func() {
					buf := &bytes.Buffer{}
					buf.WriteByte(1) // actualCtx == 1
					_, err := ReadDSKey(buf, WithContext, "", "")
					So(err, ShouldEqual, io.EOF)
				})
				Convey("truncated 2", func() {
					buf := &bytes.Buffer{}
					buf.WriteByte(1) // actualCtx == 1
					cmpbin.WriteString(buf, "aid")
					_, err := ReadDSKey(buf, WithContext, "", "")
					So(err, ShouldEqual, io.EOF)
				})
				Convey("truncated 3", func() {
					buf := &bytes.Buffer{}
					buf.WriteByte(1) // actualCtx == 1
					cmpbin.WriteString(buf, "aid")
					cmpbin.WriteString(buf, "ns")
					_, err := ReadDSKey(buf, WithContext, "", "")
					So(err, ShouldEqual, io.EOF)
				})
				Convey("huge key", func() {
					buf := &bytes.Buffer{}
					buf.WriteByte(1) // actualCtx == 1
					cmpbin.WriteString(buf, "aid")
					cmpbin.WriteString(buf, "ns")
					cmpbin.WriteUint(buf, 1000)
					_, err := ReadDSKey(buf, WithContext, "", "")
					So(err, ShouldErrLike, "huge key")
				})
				Convey("insufficient tokens", func() {
					buf := &bytes.Buffer{}
					buf.WriteByte(1) // actualCtx == 1
					cmpbin.WriteString(buf, "aid")
					cmpbin.WriteString(buf, "ns")
					cmpbin.WriteUint(buf, 2)
					_, err := ReadDSKey(buf, WithContext, "", "")
					So(err, ShouldEqual, io.EOF)
				})
				Convey("partial token 1", func() {
					buf := &bytes.Buffer{}
					buf.WriteByte(1) // actualCtx == 1
					cmpbin.WriteString(buf, "aid")
					cmpbin.WriteString(buf, "ns")
					cmpbin.WriteUint(buf, 2)
					cmpbin.WriteString(buf, "hi")
					_, err := ReadDSKey(buf, WithContext, "", "")
					So(err, ShouldEqual, io.EOF)
				})
				Convey("partial token 2", func() {
					buf := &bytes.Buffer{}
					buf.WriteByte(1) // actualCtx == 1
					cmpbin.WriteString(buf, "aid")
					cmpbin.WriteString(buf, "ns")
					cmpbin.WriteUint(buf, 2)
					cmpbin.WriteString(buf, "hi")
					buf.WriteByte(byte(gae.DSPTString))
					_, err := ReadDSKey(buf, WithContext, "", "")
					So(err, ShouldEqual, io.EOF)
				})
				Convey("bad token (invalid type)", func() {
					buf := &bytes.Buffer{}
					buf.WriteByte(1) // actualCtx == 1
					cmpbin.WriteString(buf, "aid")
					cmpbin.WriteString(buf, "ns")
					cmpbin.WriteUint(buf, 2)
					cmpbin.WriteString(buf, "hi")
					buf.WriteByte(byte(gae.DSPTBlobKey))
					_, err := ReadDSKey(buf, WithContext, "", "")
					So(err, ShouldErrLike, "invalid type DSPTBlobKey")
				})
				Convey("bad token (invalid IntID)", func() {
					buf := &bytes.Buffer{}
					buf.WriteByte(1) // actualCtx == 1
					cmpbin.WriteString(buf, "aid")
					cmpbin.WriteString(buf, "ns")
					cmpbin.WriteUint(buf, 2)
					cmpbin.WriteString(buf, "hi")
					buf.WriteByte(byte(gae.DSPTInt))
					cmpbin.WriteInt(buf, -2)
					_, err := ReadDSKey(buf, WithContext, "", "")
					So(err, ShouldErrLike, "zero/negative")
				})
			})
		})

		Convey("ReadDSGeoPoint", func() {
			Convey("trunc 1", func() {
				buf := &bytes.Buffer{}
				_, err := ReadDSGeoPoint(buf)
				So(err, ShouldEqual, io.EOF)
			})
			Convey("trunc 2", func() {
				buf := &bytes.Buffer{}
				cmpbin.WriteFloat64(buf, 100)
				_, err := ReadDSGeoPoint(buf)
				So(err, ShouldEqual, io.EOF)
			})
			Convey("invalid", func() {
				buf := &bytes.Buffer{}
				cmpbin.WriteFloat64(buf, 100)
				cmpbin.WriteFloat64(buf, 1000)
				_, err := ReadDSGeoPoint(buf)
				So(err, ShouldErrLike, "invalid DSGeoPoint")
			})
		})

		Convey("WriteTime", func() {
			Convey("in non-UTC!", func() {
				buf := &bytes.Buffer{}
				So(func() {
					WriteTime(buf, time.Now())
				}, ShouldPanic)
			})
		})

		Convey("ReadTime", func() {
			Convey("trunc 1", func() {
				buf := &bytes.Buffer{}
				_, err := ReadTime(buf)
				So(err, ShouldEqual, io.EOF)
			})
		})

		Convey("ReadDSProperty", func() {
			Convey("trunc 1", func() {
				buf := &bytes.Buffer{}
				_, err := ReadDSProperty(buf, WithContext, "", "")
				So(err, ShouldEqual, io.EOF)
			})
			Convey("trunc (DSPTBytes)", func() {
				buf := &bytes.Buffer{}
				buf.WriteByte(byte(gae.DSPTBytes))
				_, err := ReadDSProperty(buf, WithContext, "", "")
				So(err, ShouldEqual, io.EOF)
			})
			Convey("trunc (DSPTBlobKey)", func() {
				buf := &bytes.Buffer{}
				buf.WriteByte(byte(gae.DSPTBlobKey))
				_, err := ReadDSProperty(buf, WithContext, "", "")
				So(err, ShouldEqual, io.EOF)
			})
			Convey("invalid type", func() {
				buf := &bytes.Buffer{}
				buf.WriteByte(byte(gae.DSPTUnknown + 1))
				_, err := ReadDSProperty(buf, WithContext, "", "")
				So(err, ShouldErrLike, "unknown type!")
			})
		})

		Convey("ReadDSPropertyMap", func() {
			Convey("trunc 1", func() {
				buf := &bytes.Buffer{}
				_, err := ReadDSPropertyMap(buf, WithContext, "", "")
				So(err, ShouldEqual, io.EOF)
			})
			Convey("too many rows", func() {
				buf := &bytes.Buffer{}
				cmpbin.WriteUint(buf, 1000000)
				_, err := ReadDSPropertyMap(buf, WithContext, "", "")
				So(err, ShouldErrLike, "huge number of rows")
			})
			Convey("trunc 2", func() {
				buf := &bytes.Buffer{}
				cmpbin.WriteUint(buf, 10)
				_, err := ReadDSPropertyMap(buf, WithContext, "", "")
				So(err, ShouldEqual, io.EOF)
			})
			Convey("trunc 3", func() {
				buf := &bytes.Buffer{}
				cmpbin.WriteUint(buf, 10)
				cmpbin.WriteString(buf, "ohai")
				_, err := ReadDSPropertyMap(buf, WithContext, "", "")
				So(err, ShouldEqual, io.EOF)
			})
			Convey("too many values", func() {
				buf := &bytes.Buffer{}
				cmpbin.WriteUint(buf, 10)
				cmpbin.WriteString(buf, "ohai")
				cmpbin.WriteUint(buf, 100000)
				_, err := ReadDSPropertyMap(buf, WithContext, "", "")
				So(err, ShouldErrLike, "huge number of properties")
			})
			Convey("trunc 4", func() {
				buf := &bytes.Buffer{}
				cmpbin.WriteUint(buf, 10)
				cmpbin.WriteString(buf, "ohai")
				cmpbin.WriteUint(buf, 10)
				_, err := ReadDSPropertyMap(buf, WithContext, "", "")
				So(err, ShouldEqual, io.EOF)
			})
		})
	})
}
<commit_msg>Force timezone to be non-UTC when testing non-UTC<commit_after>// Copyright 2015 The Chromium Authors. All rights reserved.
// Use of this source code is governed by a BSD-style license that can be
// found in the LICENSE file.

package helper

import (
	"bytes"
	"io"
	"testing"
	"time"

	"github.com/luci/gae"
	"github.com/luci/luci-go/common/cmpbin"
	. "github.com/smartystreets/goconvey/convey"
)

func init() {
	WriteDSPropertyMapDeterministic = true
}

type dspmapTC struct {
	name  string
	props gae.DSPropertyMap
}

func TestDSPropertyMapSerialization(t *testing.T) {
	t.Parallel()

	tests := []dspmapTC{
		{
			"basic",
			gae.DSPropertyMap{
				"R": {mp(false), mp(2.1), mpNI(3)},
				"S": {mp("hello"), mp("world")},
			},
		},
		{
			"keys",
			gae.DSPropertyMap{
				"DS": {mp(mkKey("appy", "ns", "Foo", 7)), mp(mkKey("other", "", "Yot", "wheeep"))},
				"BS": {mp(gae.BSKey("sup")), mp(gae.BSKey("nerds"))},
			},
		},
		{
			"geo",
			gae.DSPropertyMap{
				"G": {mp(gae.DSGeoPoint{Lat: 1, Lng: 2})},
			},
		},
		{
			"data",
			gae.DSPropertyMap{
				"S":          {mp("sup"), mp("fool"), mp("nerd")},
				"D.Foo.Nerd": {mp([]byte("sup")), mp([]byte("fool"))},
				"B":          {mp(gae.DSByteString("sup")), mp(gae.DSByteString("fool"))},
			},
		},
		{
			"time",
			gae.DSPropertyMap{
				"T": {
					mp(time.Now().UTC()),
					mp(time.Now().Add(time.Second).UTC())},
			},
		},
		{
			"empty vals",
			gae.DSPropertyMap{
				"T": {mp(true), mp(true)},
				"F": {mp(false), mp(false)},
				"N": {mp(nil), mp(nil)},
				"E": {},
			},
		},
	}

	Convey("DSPropertyMap serialization", t, func() {
		Convey("round trip", func() {
			for _, tc := range tests {
				tc := tc
				Convey(tc.name, func() {
					buf := &bytes.Buffer{}
					WriteDSPropertyMap(buf, tc.props, WithContext)
					dec, err := ReadDSPropertyMap(buf, WithContext, "", "")
					So(err, ShouldBeNil)
					So(dec, ShouldResemble, tc.props)
				})
			}
		})
	})
}

func TestSerializationReadMisc(t *testing.T) {
	t.Parallel()

	Convey("Misc Serialization tests", t, func() {
		Convey("ReadDSKey", func() {
			Convey("good cases", func() {
				Convey("w/ ctx decodes normally w/ ctx", func() {
					k := mkKey("aid", "ns", "knd", "yo", "other", 10)
					buf := &bytes.Buffer{}
					WriteDSKey(buf, WithContext, k)
					dk, err := ReadDSKey(buf, WithContext, "", "")
					So(err, ShouldBeNil)
					So(dk, ShouldEqualKey, k)
				})
				Convey("w/ ctx decodes normally w/o ctx", func() {
					k := mkKey("aid", "ns", "knd", "yo", "other", 10)
					buf := &bytes.Buffer{}
					WriteDSKey(buf, WithContext, k)
					dk, err := ReadDSKey(buf, WithoutContext, "spam", "nerd")
					So(err, ShouldBeNil)
					So(dk, ShouldEqualKey, mkKey("spam", "nerd", "knd", "yo", "other", 10))
				})
				Convey("w/o ctx decodes normally w/ ctx", func() {
					k := mkKey("aid", "ns", "knd", "yo", "other", 10)
					buf := &bytes.Buffer{}
					WriteDSKey(buf, WithoutContext, k)
					dk, err := ReadDSKey(buf, WithContext, "spam", "nerd")
					So(err, ShouldBeNil)
					So(dk, ShouldEqualKey, mkKey("", "", "knd", "yo", "other", 10))
				})
				Convey("w/o ctx decodes normally w/o ctx", func() {
					k := mkKey("aid", "ns", "knd", "yo", "other", 10)
					buf := &bytes.Buffer{}
					WriteDSKey(buf, WithoutContext, k)
					dk, err := ReadDSKey(buf, WithoutContext, "spam", "nerd")
					So(err, ShouldBeNil)
					So(dk, ShouldEqualKey, mkKey("spam", "nerd", "knd", "yo", "other", 10))
				})
				Convey("IntIDs always sort before StringIDs", func() {
					// -1 writes as almost all 1's in the first byte under cmpbin, even
					// though it's technically not a valid key.
					k := mkKey("aid", "ns", "knd", -1)
					buf := &bytes.Buffer{}
					WriteDSKey(buf, WithoutContext, k)

					k = mkKey("aid", "ns", "knd", "hat")
					buf2 := &bytes.Buffer{}
					WriteDSKey(buf2, WithoutContext, k)

					So(bytes.Compare(buf.Bytes(), buf2.Bytes()), ShouldBeLessThan, 0)
				})
			})

			Convey("err cases", func() {
				Convey("nil", func() {
					buf := &bytes.Buffer{}
					_, err := ReadDSKey(buf, WithContext, "", "")
					So(err, ShouldEqual, io.EOF)
				})
				Convey("str", func() {
					buf := &bytes.Buffer{}
					buf.WriteString("sup")
					_, err := ReadDSKey(buf, WithContext, "", "")
					So(err, ShouldErrLike, "expected actualCtx")
				})
				Convey("truncated 1", func() {
					buf := &bytes.Buffer{}
					buf.WriteByte(1) // actualCtx == 1
					_, err := ReadDSKey(buf, WithContext, "", "")
					So(err, ShouldEqual, io.EOF)
				})
				Convey("truncated 2", func() {
					buf := &bytes.Buffer{}
					buf.WriteByte(1) // actualCtx == 1
					cmpbin.WriteString(buf, "aid")
					_, err := ReadDSKey(buf, WithContext, "", "")
					So(err, ShouldEqual, io.EOF)
				})
				Convey("truncated 3", func() {
					buf := &bytes.Buffer{}
					buf.WriteByte(1) // actualCtx == 1
					cmpbin.WriteString(buf, "aid")
					cmpbin.WriteString(buf, "ns")
					_, err := ReadDSKey(buf, WithContext, "", "")
					So(err, ShouldEqual, io.EOF)
				})
				Convey("huge key", func() {
					buf := &bytes.Buffer{}
					buf.WriteByte(1) // actualCtx == 1
					cmpbin.WriteString(buf, "aid")
					cmpbin.WriteString(buf, "ns")
					cmpbin.WriteUint(buf, 1000)
					_, err := ReadDSKey(buf, WithContext, "", "")
					So(err, ShouldErrLike, "huge key")
				})
				Convey("insufficient tokens", func() {
					buf := &bytes.Buffer{}
					buf.WriteByte(1) // actualCtx == 1
					cmpbin.WriteString(buf, "aid")
					cmpbin.WriteString(buf, "ns")
					cmpbin.WriteUint(buf, 2)
					_, err := ReadDSKey(buf, WithContext, "", "")
					So(err, ShouldEqual, io.EOF)
				})
				Convey("partial token 1", func() {
					buf := &bytes.Buffer{}
					buf.WriteByte(1) // actualCtx == 1
					cmpbin.WriteString(buf, "aid")
					cmpbin.WriteString(buf, "ns")
					cmpbin.WriteUint(buf, 2)
					cmpbin.WriteString(buf, "hi")
					_, err := ReadDSKey(buf, WithContext, "", "")
					So(err, ShouldEqual, io.EOF)
				})
				Convey("partial token 2", func() {
					buf := &bytes.Buffer{}
					buf.WriteByte(1) // actualCtx == 1
					cmpbin.WriteString(buf, "aid")
					cmpbin.WriteString(buf, "ns")
					cmpbin.WriteUint(buf, 2)
					cmpbin.WriteString(buf, "hi")
					buf.WriteByte(byte(gae.DSPTString))
					_, err := ReadDSKey(buf, WithContext, "", "")
					So(err, ShouldEqual, io.EOF)
				})
				Convey("bad token (invalid type)", func() {
					buf := &bytes.Buffer{}
					buf.WriteByte(1) // actualCtx == 1
					cmpbin.WriteString(buf, "aid")
					cmpbin.WriteString(buf, "ns")
					cmpbin.WriteUint(buf, 2)
					cmpbin.WriteString(buf, "hi")
					buf.WriteByte(byte(gae.DSPTBlobKey))
					_, err := ReadDSKey(buf, WithContext, "", "")
					So(err, ShouldErrLike, "invalid type DSPTBlobKey")
				})
				Convey("bad token (invalid IntID)", func() {
					buf := &bytes.Buffer{}
					buf.WriteByte(1) // actualCtx == 1
					cmpbin.WriteString(buf, "aid")
					cmpbin.WriteString(buf, "ns")
					cmpbin.WriteUint(buf, 2)
					cmpbin.WriteString(buf, "hi")
					buf.WriteByte(byte(gae.DSPTInt))
					cmpbin.WriteInt(buf, -2)
					_, err := ReadDSKey(buf, WithContext, "", "")
					So(err, ShouldErrLike, "zero/negative")
				})
			})
		})

		Convey("ReadDSGeoPoint", func() {
			Convey("trunc 1", func() {
				buf := &bytes.Buffer{}
				_, err := ReadDSGeoPoint(buf)
				So(err, ShouldEqual, io.EOF)
			})
			Convey("trunc 2", func() {
				buf := &bytes.Buffer{}
				cmpbin.WriteFloat64(buf, 100)
				_, err := ReadDSGeoPoint(buf)
				So(err, ShouldEqual, io.EOF)
			})
			Convey("invalid", func() {
				buf := &bytes.Buffer{}
				cmpbin.WriteFloat64(buf, 100)
				cmpbin.WriteFloat64(buf, 1000)
				_, err := ReadDSGeoPoint(buf)
				So(err, ShouldErrLike, "invalid DSGeoPoint")
			})
		})

		Convey("WriteTime", func() {
			Convey("in non-UTC!", func() {
				buf := &bytes.Buffer{}
				pst, err := time.LoadLocation("America/Los_Angeles")
				So(err, ShouldBeNil)
				So(func() {
					WriteTime(buf, time.Now().In(pst))
				}, ShouldPanic)
			})
		})

		Convey("ReadTime", func() {
			Convey("trunc 1", func() {
				buf := &bytes.Buffer{}
				_, err := ReadTime(buf)
				So(err, ShouldEqual, io.EOF)
			})
		})

		Convey("ReadDSProperty", func() {
			Convey("trunc 1", func() {
				buf := &bytes.Buffer{}
				_, err := ReadDSProperty(buf, WithContext, "", "")
				So(err, ShouldEqual, io.EOF)
			})
			Convey("trunc (DSPTBytes)", func() {
				buf := &bytes.Buffer{}
				buf.WriteByte(byte(gae.DSPTBytes))
				_, err := ReadDSProperty(buf, WithContext, "", "")
				So(err, ShouldEqual, io.EOF)
			})
			Convey("trunc (DSPTBlobKey)", func() {
				buf := &bytes.Buffer{}
				buf.WriteByte(byte(gae.DSPTBlobKey))
				_, err := ReadDSProperty(buf, WithContext, "", "")
				So(err, ShouldEqual, io.EOF)
			})
			Convey("invalid type", func() {
				buf := &bytes.Buffer{}
				buf.WriteByte(byte(gae.DSPTUnknown + 1))
				_, err := ReadDSProperty(buf, WithContext, "", "")
				So(err, ShouldErrLike, "unknown type!")
			})
		})

		Convey("ReadDSPropertyMap", func() {
			Convey("trunc 1", func() {
				buf := &bytes.Buffer{}
				_, err := ReadDSPropertyMap(buf, WithContext, "", "")
				So(err, ShouldEqual, io.EOF)
			})
			Convey("too many rows", func() {
				buf := &bytes.Buffer{}
				cmpbin.WriteUint(buf, 1000000)
				_, err := ReadDSPropertyMap(buf, WithContext, "", "")
				So(err, ShouldErrLike, "huge number of rows")
			})
			Convey("trunc 2", func() {
				buf := &bytes.Buffer{}
				cmpbin.WriteUint(buf, 10)
				_, err := ReadDSPropertyMap(buf, WithContext, "", "")
				So(err, ShouldEqual, io.EOF)
			})
			Convey("trunc 3", func() {
				buf := &bytes.Buffer{}
				cmpbin.WriteUint(buf, 10)
				cmpbin.WriteString(buf, "ohai")
				_, err := ReadDSPropertyMap(buf, WithContext, "", "")
				So(err, ShouldEqual, io.EOF)
			})
			Convey("too many values", func() {
				buf := &bytes.Buffer{}
				cmpbin.WriteUint(buf, 10)
				cmpbin.WriteString(buf, "ohai")
				cmpbin.WriteUint(buf, 100000)
				_, err := ReadDSPropertyMap(buf, WithContext, "", "")
				So(err, ShouldErrLike, "huge number of properties")
			})
			Convey("trunc 4", func() {
				buf := &bytes.Buffer{}
				cmpbin.WriteUint(buf, 10)
				cmpbin.WriteString(buf, "ohai")
				cmpbin.WriteUint(buf, 10)
				_, err := ReadDSPropertyMap(buf, WithContext, "", "")
				So(err, ShouldEqual, io.EOF)
			})
		})
	})
}
<|endoftext|>


<commit_before>// Copyright 2015 The LUCI Authors.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//      http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package memory

import (
	"context"
	"errors"
	"fmt"

	ds "go.chromium.org/luci/gae/service/datastore"
	"go.chromium.org/luci/gae/service/info"
)

//////////////////////////////////// public ////////////////////////////////////

// useRDS adds a gae.Datastore implementation to context, accessible
// by gae.GetDS(c)
func useRDS(c context.Context) context.Context {
	return ds.SetRawFactory(c, func(ic context.Context) ds.RawInterface {
		kc := ds.GetKeyContext(ic)
		memCtx, isTxn := cur(ic)
		dsd := memCtx.Get(memContextDSIdx)
		if isTxn {
			return &txnDsImpl{ic, dsd.(*txnDataStoreData), kc}
		}
		return &dsImpl{ic, dsd.(*dataStoreData), kc}
	})
}

// NewDatastore creates a new standalone memory implementation of the datastore,
// suitable for embedding for doing in-memory data organization.
//
// It's configured by default with the following settings:
//   * AutoIndex(true)
//   * Consistent(true)
//   * DisableSpecialEntities(true)
//
// These settings can of course be changed by using the Testable interface.
func NewDatastore(c context.Context, inf info.RawInterface) ds.RawInterface {
	kc := ds.GetKeyContext(c)

	memctx := newMemContext(kc.AppID)

	dsCtx := info.Set(context.Background(), inf)
	rds := &dsImpl{dsCtx, memctx.Get(memContextDSIdx).(*dataStoreData), kc}

	ret := ds.Raw(ds.SetRaw(dsCtx, rds))
	t := ret.GetTestable()
	t.AutoIndex(true)
	t.Consistent(true)
	t.DisableSpecialEntities(true)

	return ret
}

//////////////////////////////////// dsImpl ////////////////////////////////////

// dsImpl exists solely to bind the current c to the datastore data.
type dsImpl struct {
	context.Context

	data *dataStoreData
	kc   ds.KeyContext
}

var _ ds.RawInterface = (*dsImpl)(nil)

func (d *dsImpl) AllocateIDs(keys []*ds.Key, cb ds.NewKeyCB) error {
	return d.data.allocateIDs(keys, cb)
}

func (d *dsImpl) PutMulti(keys []*ds.Key, vals []ds.PropertyMap, cb ds.NewKeyCB) error {
	d.data.putMulti(keys, vals, cb, false)
	return nil
}

func (d *dsImpl) GetMulti(keys []*ds.Key, _meta ds.MultiMetaGetter, cb ds.GetMultiCB) error {
	return d.data.getMulti(keys, cb)
}

func (d *dsImpl) DeleteMulti(keys []*ds.Key, cb ds.DeleteMultiCB) error {
	d.data.delMulti(keys, cb, false)
	return nil
}

func (d *dsImpl) DecodeCursor(s string) (ds.Cursor, error) {
	return newCursor(s)
}

func (d *dsImpl) Run(fq *ds.FinalizedQuery, cb ds.RawRunCB) error {
	cb = d.data.stripSpecialPropsRunCB(cb)
	idx, head := d.data.getQuerySnaps(!fq.EventuallyConsistent())
	err := executeQuery(fq, d.kc, false, idx, head, cb)
	if d.data.maybeAutoIndex(err) {
		idx, head = d.data.getQuerySnaps(!fq.EventuallyConsistent())
		err = executeQuery(fq, d.kc, false, idx, head, cb)
	}
	return err
}

func (d *dsImpl) Count(fq *ds.FinalizedQuery) (ret int64, err error) {
	idx, head := d.data.getQuerySnaps(!fq.EventuallyConsistent())
	ret, err = countQuery(fq, d.kc, false, idx, head)
	if d.data.maybeAutoIndex(err) {
		idx, head := d.data.getQuerySnaps(!fq.EventuallyConsistent())
		ret, err = countQuery(fq, d.kc, false, idx, head)
	}
	return
}

func (d *dsImpl) WithoutTransaction() context.Context {
	// Already not in a Transaction.
	return d
}

func (*dsImpl) CurrentTransaction() ds.Transaction { return nil }

func (d *dsImpl) AddIndexes(idxs ...*ds.IndexDefinition) {
	if len(idxs) == 0 {
		return
	}

	for _, i := range idxs {
		if !i.Compound() {
			panic(fmt.Errorf("Attempted to add non-compound index: %s", i))
		}
	}

	d.data.addIndexes(idxs)
}

func (d *dsImpl) Constraints() ds.Constraints { return d.data.getConstraints() }

func (d *dsImpl) TakeIndexSnapshot() ds.TestingSnapshot {
	return d.data.takeSnapshot()
}

func (d *dsImpl) SetIndexSnapshot(snap ds.TestingSnapshot) {
	d.data.setSnapshot(snap.(memStore))
}

func (d *dsImpl) CatchupIndexes() {
	d.data.catchupIndexes()
}

func (d *dsImpl) SetTransactionRetryCount(count int) {
	d.data.setTxnRetry(count)
}

func (d *dsImpl) Consistent(always bool) {
	d.data.setConsistent(always)
}

func (d *dsImpl) AutoIndex(enable bool) {
	d.data.setAutoIndex(enable)
}

func (d *dsImpl) DisableSpecialEntities(disabled bool) {
	d.data.setDisableSpecialEntities(disabled)
}

func (d *dsImpl) ShowSpecialProperties(show bool) {
	d.data.setShowSpecialProperties(show)
}

func (d *dsImpl) SetConstraints(c *ds.Constraints) error {
	if c == nil {
		c = &ds.Constraints{}
	}
	d.data.setConstraints(*c)
	return nil
}

func (d *dsImpl) GetTestable() ds.Testable { return d }

////////////////////////////////// txnDsImpl ///////////////////////////////////

type txnDsImpl struct {
	context.Context

	data *txnDataStoreData
	kc   ds.KeyContext
}

var _ ds.RawInterface = (*txnDsImpl)(nil)

func (d *txnDsImpl) AllocateIDs(keys []*ds.Key, cb ds.NewKeyCB) error {
	return d.data.parent.allocateIDs(keys, cb)
}

func (d *txnDsImpl) PutMulti(keys []*ds.Key, vals []ds.PropertyMap, cb ds.NewKeyCB) error {
	return d.data.run(func() error {
		d.data.putMulti(keys, vals, cb)
		return nil
	})
}

func (d *txnDsImpl) GetMulti(keys []*ds.Key, _meta ds.MultiMetaGetter, cb ds.GetMultiCB) error {
	return d.data.run(func() error {
		return d.data.getMulti(keys, cb)
	})
}

func (d *txnDsImpl) DeleteMulti(keys []*ds.Key, cb ds.DeleteMultiCB) error {
	return d.data.run(func() error {
		return d.data.delMulti(keys, cb)
	})
}

func (d *txnDsImpl) DecodeCursor(s string) (ds.Cursor, error) { return newCursor(s) }

func (d *txnDsImpl) Run(q *ds.FinalizedQuery, cb ds.RawRunCB) error {
	// note that autoIndex has no effect inside transactions. This is because
	// the transaction guarantees a consistent view of head at the time that the
	// transaction opens. At best, we could add the index on head, but then return
	// the error anyway, but adding the index then re-snapping at head would
	// potentially reveal other entities not in the original transaction snapshot.
	//
	// It's possible that if you have full-consistency and also auto index enabled
	// that this would make sense... but at that point you should probably just
	// add the index up front.
	cb = d.data.parent.stripSpecialPropsRunCB(cb)
	return executeQuery(q, d.kc, true, d.data.snap, d.data.snap, cb)
}

func (d *txnDsImpl) Count(fq *ds.FinalizedQuery) (ret int64, err error) {
	return countQuery(fq, d.kc, true, d.data.snap, d.data.snap)
}

func (*txnDsImpl) RunInTransaction(func(c context.Context) error, *ds.TransactionOptions) error {
	return errors.New("datastore: nested transactions are not supported")
}

func (d *txnDsImpl) WithoutTransaction() context.Context {
	return context.WithValue(d, &currentTxnKey, nil)
}

func (d *txnDsImpl) CurrentTransaction() ds.Transaction {
	return d.data.txn
}

func (d *txnDsImpl) Constraints() ds.Constraints { return d.data.parent.getConstraints() }

func (d *txnDsImpl) GetTestable() ds.Testable { return nil }
<commit_msg>[datastore] enable safe get in tests.<commit_after>// Copyright 2015 The LUCI Authors.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//      http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package memory

import (
	"context"
	"errors"
	"fmt"

	ds "go.chromium.org/luci/gae/service/datastore"
	_ "go.chromium.org/luci/gae/service/datastore/crbug1242998safeget"
	"go.chromium.org/luci/gae/service/info"
)

//////////////////////////////////// public ////////////////////////////////////

// useRDS adds a gae.Datastore implementation to context, accessible
// by gae.GetDS(c)
func useRDS(c context.Context) context.Context {
	return ds.SetRawFactory(c, func(ic context.Context) ds.RawInterface {
		kc := ds.GetKeyContext(ic)
		memCtx, isTxn := cur(ic)
		dsd := memCtx.Get(memContextDSIdx)
		if isTxn {
			return &txnDsImpl{ic, dsd.(*txnDataStoreData), kc}
		}
		return &dsImpl{ic, dsd.(*dataStoreData), kc}
	})
}

// NewDatastore creates a new standalone memory implementation of the datastore,
// suitable for embedding for doing in-memory data organization.
//
// It's configured by default with the following settings:
//   * AutoIndex(true)
//   * Consistent(true)
//   * DisableSpecialEntities(true)
//
// These settings can of course be changed by using the Testable interface.
func NewDatastore(c context.Context, inf info.RawInterface) ds.RawInterface {
	kc := ds.GetKeyContext(c)

	memctx := newMemContext(kc.AppID)

	dsCtx := info.Set(context.Background(), inf)
	rds := &dsImpl{dsCtx, memctx.Get(memContextDSIdx).(*dataStoreData), kc}

	ret := ds.Raw(ds.SetRaw(dsCtx, rds))
	t := ret.GetTestable()
	t.AutoIndex(true)
	t.Consistent(true)
	t.DisableSpecialEntities(true)

	return ret
}

//////////////////////////////////// dsImpl ////////////////////////////////////

// dsImpl exists solely to bind the current c to the datastore data.
type dsImpl struct {
	context.Context

	data *dataStoreData
	kc   ds.KeyContext
}

var _ ds.RawInterface = (*dsImpl)(nil)

func (d *dsImpl) AllocateIDs(keys []*ds.Key, cb ds.NewKeyCB) error {
	return d.data.allocateIDs(keys, cb)
}

func (d *dsImpl) PutMulti(keys []*ds.Key, vals []ds.PropertyMap, cb ds.NewKeyCB) error {
	d.data.putMulti(keys, vals, cb, false)
	return nil
}

func (d *dsImpl) GetMulti(keys []*ds.Key, _meta ds.MultiMetaGetter, cb ds.GetMultiCB) error {
	return d.data.getMulti(keys, cb)
}

func (d *dsImpl) DeleteMulti(keys []*ds.Key, cb ds.DeleteMultiCB) error {
	d.data.delMulti(keys, cb, false)
	return nil
}

func (d *dsImpl) DecodeCursor(s string) (ds.Cursor, error) {
	return newCursor(s)
}

func (d *dsImpl) Run(fq *ds.FinalizedQuery, cb ds.RawRunCB) error {
	cb = d.data.stripSpecialPropsRunCB(cb)
	idx, head := d.data.getQuerySnaps(!fq.EventuallyConsistent())
	err := executeQuery(fq, d.kc, false, idx, head, cb)
	if d.data.maybeAutoIndex(err) {
		idx, head = d.data.getQuerySnaps(!fq.EventuallyConsistent())
		err = executeQuery(fq, d.kc, false, idx, head, cb)
	}
	return err
}

func (d *dsImpl) Count(fq *ds.FinalizedQuery) (ret int64, err error) {
	idx, head := d.data.getQuerySnaps(!fq.EventuallyConsistent())
	ret, err = countQuery(fq, d.kc, false, idx, head)
	if d.data.maybeAutoIndex(err) {
		idx, head := d.data.getQuerySnaps(!fq.EventuallyConsistent())
		ret, err = countQuery(fq, d.kc, false, idx, head)
	}
	return
}

func (d *dsImpl) WithoutTransaction() context.Context {
	// Already not in a Transaction.
	return d
}

func (*dsImpl) CurrentTransaction() ds.Transaction { return nil }

func (d *dsImpl) AddIndexes(idxs ...*ds.IndexDefinition) {
	if len(idxs) == 0 {
		return
	}

	for _, i := range idxs {
		if !i.Compound() {
			panic(fmt.Errorf("Attempted to add non-compound index: %s", i))
		}
	}

	d.data.addIndexes(idxs)
}

func (d *dsImpl) Constraints() ds.Constraints { return d.data.getConstraints() }

func (d *dsImpl) TakeIndexSnapshot() ds.TestingSnapshot {
	return d.data.takeSnapshot()
}

func (d *dsImpl) SetIndexSnapshot(snap ds.TestingSnapshot) {
	d.data.setSnapshot(snap.(memStore))
}

func (d *dsImpl) CatchupIndexes() {
	d.data.catchupIndexes()
}

func (d *dsImpl) SetTransactionRetryCount(count int) {
	d.data.setTxnRetry(count)
}

func (d *dsImpl) Consistent(always bool) {
	d.data.setConsistent(always)
}

func (d *dsImpl) AutoIndex(enable bool) {
	d.data.setAutoIndex(enable)
}

func (d *dsImpl) DisableSpecialEntities(disabled bool) {
	d.data.setDisableSpecialEntities(disabled)
}

func (d *dsImpl) ShowSpecialProperties(show bool) {
	d.data.setShowSpecialProperties(show)
}

func (d *dsImpl) SetConstraints(c *ds.Constraints) error {
	if c == nil {
		c = &ds.Constraints{}
	}
	d.data.setConstraints(*c)
	return nil
}

func (d *dsImpl) GetTestable() ds.Testable { return d }

////////////////////////////////// txnDsImpl ///////////////////////////////////

type txnDsImpl struct {
	context.Context

	data *txnDataStoreData
	kc   ds.KeyContext
}

var _ ds.RawInterface = (*txnDsImpl)(nil)

func (d *txnDsImpl) AllocateIDs(keys []*ds.Key, cb ds.NewKeyCB) error {
	return d.data.parent.allocateIDs(keys, cb)
}

func (d *txnDsImpl) PutMulti(keys []*ds.Key, vals []ds.PropertyMap, cb ds.NewKeyCB) error {
	return d.data.run(func() error {
		d.data.putMulti(keys, vals, cb)
		return nil
	})
}

func (d *txnDsImpl) GetMulti(keys []*ds.Key, _meta ds.MultiMetaGetter, cb ds.GetMultiCB) error {
	return d.data.run(func() error {
		return d.data.getMulti(keys, cb)
	})
}

func (d *txnDsImpl) DeleteMulti(keys []*ds.Key, cb ds.DeleteMultiCB) error {
	return d.data.run(func() error {
		return d.data.delMulti(keys, cb)
	})
}

func (d *txnDsImpl) DecodeCursor(s string) (ds.Cursor, error) { return newCursor(s) }

func (d *txnDsImpl) Run(q *ds.FinalizedQuery, cb ds.RawRunCB) error {
	// note that autoIndex has no effect inside transactions. This is because
	// the transaction guarantees a consistent view of head at the time that the
	// transaction opens. At best, we could add the index on head, but then return
	// the error anyway, but adding the index then re-snapping at head would
	// potentially reveal other entities not in the original transaction snapshot.
	//
	// It's possible that if you have full-consistency and also auto index enabled
	// that this would make sense... but at that point you should probably just
	// add the index up front.
	cb = d.data.parent.stripSpecialPropsRunCB(cb)
	return executeQuery(q, d.kc, true, d.data.snap, d.data.snap, cb)
}

func (d *txnDsImpl) Count(fq *ds.FinalizedQuery) (ret int64, err error) {
	return countQuery(fq, d.kc, true, d.data.snap, d.data.snap)
}

func (*txnDsImpl) RunInTransaction(func(c context.Context) error, *ds.TransactionOptions) error {
	return errors.New("datastore: nested transactions are not supported")
}

func (d *txnDsImpl) WithoutTransaction() context.Context {
	return context.WithValue(d, &currentTxnKey, nil)
}

func (d *txnDsImpl) CurrentTransaction() ds.Transaction {
	return d.data.txn
}

func (d *txnDsImpl) Constraints() ds.Constraints { return d.data.parent.getConstraints() }

func (d *txnDsImpl) GetTestable() ds.Testable { return nil }
<|endoftext|>


<commit_before>// Copyright 2013, Cong Ding. All rights reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//   http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
//
// author: Cong Ding <dinggnu@gmail.com>

// Package logging implements log library for other applications. It provides
// functions Debug, Info, Warning, Error, Critical, and formatting version
// Logf.
//
// Example:
//
//	logger := logging.SimpleLogger("main")
//	logger.SetLevel(logging.WARNING)
//	logger.Error("test for error")
//	logger.Warning("test for warning", "second parameter")
//	logger.Debug("test for debug")
//
package logging

import (
	"github.com/ccding/go-config-reader/config"
	"io"
	"os"
	"strconv"
	"sync"
	"sync/atomic"
	"time"
)

// Pre-defined formats
const (
	DefaultFileName   = "logging.log"                   // default logging filename
	DefaultConfigFile = "logging.conf"                  // default logging configuration file
	DefaultTimeFormat = "2006-01-02 15:04:05.999999999" // defaulttime format
	bufSize           = 1000                            // buffer size for writer
	queueSize         = 10000                           // chan queue size in async logging
	reqSize           = 10000                           // chan queue size in async logging
)

// Logger is the logging struct.
type Logger struct {

	// Be careful of the alignment issue of the variable seqid because it
	// uses the sync/atomic.AddUint64() operation. If the alignment is
	// wrong, it will cause a panic. To solve the alignment issue in an
	// easy way, we put seqid to the beginning of the structure.
	// seqid is only visiable internally.
	seqid uint64 // last used sequence number in record

	// These variables can be configured by users.
	name         string    // logger name
	level        Level     // record level higher than this will be printed
	recordFormat string    // format of the record
	recordArgs   []string  // arguments to be used in the recordFormat
	out          io.Writer // writer
	sync         bool      // use sync or async way to record logs
	timeFormat   string    // format for time

	// These variables are visible to users.
	startTime time.Time // start time of the logger

	// Internally used variables, which don't have get and set functions.
	wlock   sync.Mutex   // writer lock
	queue   chan string  // queue used in async logging
	request chan request // queue used in non-runtime logging
	flush   chan bool    // flush signal for the watcher to write
	finish  chan bool    // finish flush signal for the flush function to return
	quit    chan bool    // quit signal for the watcher to quit
	fd      *os.File     // file handler, used to close the file on destroy
	runtime bool         // with runtime operation or not
}

// SimpleLogger creates a new logger with simple configuration.
func SimpleLogger(name string) (*Logger, error) {
	return createLogger(name, WARNING, BasicFormat, DefaultTimeFormat, os.Stdout, false)
}

// BasicLogger creates a new logger with basic configuration.
func BasicLogger(name string) (*Logger, error) {
	return FileLogger(name, WARNING, BasicFormat, DefaultTimeFormat, DefaultFileName, false)
}

// RichLogger creates a new logger with simple configuration.
func RichLogger(name string) (*Logger, error) {
	return FileLogger(name, NOTSET, RichFormat, DefaultTimeFormat, DefaultFileName, false)
}

// FileLogger creates a new logger with file output.
func FileLogger(name string, level Level, format string, timeFormat string, file string, sync bool) (*Logger, error) {
	out, err := os.OpenFile(file, os.O_WRONLY|os.O_APPEND|os.O_CREATE, os.ModeAppend|0666)
	if err != nil {
		return nil, err
	}
	logger, err := createLogger(name, level, format, timeFormat, out, sync)
	if err == nil {
		logger.fd = out
		return logger, nil
	} else {
		return nil, err
	}
}

// WriterLogger creates a new logger with a writer
func WriterLogger(name string, level Level, format string, timeFormat string, out io.Writer, sync bool) (*Logger, error) {
	return createLogger(name, level, format, timeFormat, out, sync)
}

// WriterLogger creates a new logger from a configuration file
func ConfigLogger(filename string) (*Logger, error) {
	conf := config.NewConfig(filename)
	err := conf.Read()
	if err != nil {
		return nil, err
	}
	name := conf.Get("", "name")
	slevel := conf.Get("", "level")
	if slevel == "" {
		slevel = "0"
	}
	l, err := strconv.Atoi(slevel)
	if err != nil {
		return nil, err
	}
	level := Level(l)
	format := conf.Get("", "format")
	if format == "" {
		format = BasicFormat
	}
	timeFormat := conf.Get("", "timeFormat")
	if timeFormat == "" {
		timeFormat = DefaultTimeFormat
	}
	ssync := conf.Get("", "sync")
	if ssync == "" {
		ssync = "0"
	}
	file := conf.Get("", "file")
	if file == "" {
		file = DefaultFileName
	}
	sync := true
	if ssync == "0" {
		sync = false
	} else if ssync == "1" {
		sync = true
	} else {
		return nil, err
	}
	return FileLogger(name, level, format, timeFormat, file, sync)
}

// createLogger create a new logger
func createLogger(name string, level Level, format string, timeFormat string, out io.Writer, sync bool) (*Logger, error) {
	logger := new(Logger)

	err := logger.parseFormat(format)
	if err != nil {
		return nil, err
	}

	// asign values to logger
	logger.name = name
	logger.level = level
	logger.out = out
	logger.seqid = 0
	logger.sync = sync
	logger.queue = make(chan string, queueSize)
	logger.request = make(chan request, reqSize)
	logger.flush = make(chan bool)
	logger.finish = make(chan bool)
	logger.quit = make(chan bool)
	logger.startTime = time.Now()
	logger.fd = nil
	logger.timeFormat = timeFormat

	// start watcher to write logs if it is async or no runtime field
	if !logger.sync {
		go logger.watcher()
	}

	return logger, nil
}

// Destroy sends quit signal to watcher and releases all the resources.
func (logger *Logger) Destroy() {
	if !logger.sync {
		// quit watcher
		logger.quit <- true
		// wait for watcher quit
		<-logger.quit
	}
	// clean up
	if logger.fd != nil {
		logger.fd.Close()
	}
}

// Flush the writer
func (logger *Logger) Flush() {
	if !logger.sync {
		// send flush signal
		logger.flush <- true
		// wait for the flush finish
		<-logger.finish
	}
}

// Getter functions

func (logger *Logger) Name() string {
	return logger.name
}

func (logger *Logger) StartTime() int64 {
	return logger.startTime.UnixNano()
}

func (logger *Logger) TimeFormat() string {
	return logger.timeFormat
}

func (logger *Logger) Level() Level {
	return Level(atomic.LoadInt32((*int32)(&logger.level)))
}

func (logger *Logger) RecordFormat() string {
	return logger.recordFormat
}

func (logger *Logger) RecordArgs() []string {
	return logger.recordArgs
}

func (logger *Logger) Writer() io.Writer {
	return logger.out
}

func (logger *Logger) Sync() bool {
	return logger.sync
}

// Setter functions

func (logger *Logger) SetLevel(level Level) {
	atomic.StoreInt32((*int32)(&logger.level), int32(level))
}

func (logger *Logger) SetWriter(out ...io.Writer) {
	logger.out = io.MultiWriter(out...)
}
<commit_msg>change default filereader mask<commit_after>// Copyright 2013, Cong Ding. All rights reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//   http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
//
// author: Cong Ding <dinggnu@gmail.com>

// Package logging implements log library for other applications. It provides
// functions Debug, Info, Warning, Error, Critical, and formatting version
// Logf.
//
// Example:
//
//	logger := logging.SimpleLogger("main")
//	logger.SetLevel(logging.WARNING)
//	logger.Error("test for error")
//	logger.Warning("test for warning", "second parameter")
//	logger.Debug("test for debug")
//
package logging

import (
	"github.com/ccding/go-config-reader/config"
	"io"
	"os"
	"strconv"
	"sync"
	"sync/atomic"
	"time"
)

// Pre-defined formats
const (
	DefaultFileName   = "logging.log"                   // default logging filename
	DefaultConfigFile = "logging.conf"                  // default logging configuration file
	DefaultTimeFormat = "2006-01-02 15:04:05.999999999" // defaulttime format
	bufSize           = 1000                            // buffer size for writer
	queueSize         = 10000                           // chan queue size in async logging
	reqSize           = 10000                           // chan queue size in async logging
)

// Logger is the logging struct.
type Logger struct {

	// Be careful of the alignment issue of the variable seqid because it
	// uses the sync/atomic.AddUint64() operation. If the alignment is
	// wrong, it will cause a panic. To solve the alignment issue in an
	// easy way, we put seqid to the beginning of the structure.
	// seqid is only visiable internally.
	seqid uint64 // last used sequence number in record

	// These variables can be configured by users.
	name         string    // logger name
	level        Level     // record level higher than this will be printed
	recordFormat string    // format of the record
	recordArgs   []string  // arguments to be used in the recordFormat
	out          io.Writer // writer
	sync         bool      // use sync or async way to record logs
	timeFormat   string    // format for time

	// These variables are visible to users.
	startTime time.Time // start time of the logger

	// Internally used variables, which don't have get and set functions.
	wlock   sync.Mutex   // writer lock
	queue   chan string  // queue used in async logging
	request chan request // queue used in non-runtime logging
	flush   chan bool    // flush signal for the watcher to write
	finish  chan bool    // finish flush signal for the flush function to return
	quit    chan bool    // quit signal for the watcher to quit
	fd      *os.File     // file handler, used to close the file on destroy
	runtime bool         // with runtime operation or not
}

// SimpleLogger creates a new logger with simple configuration.
func SimpleLogger(name string) (*Logger, error) {
	return createLogger(name, WARNING, BasicFormat, DefaultTimeFormat, os.Stdout, false)
}

// BasicLogger creates a new logger with basic configuration.
func BasicLogger(name string) (*Logger, error) {
	return FileLogger(name, WARNING, BasicFormat, DefaultTimeFormat, DefaultFileName, false)
}

// RichLogger creates a new logger with simple configuration.
func RichLogger(name string) (*Logger, error) {
	return FileLogger(name, NOTSET, RichFormat, DefaultTimeFormat, DefaultFileName, false)
}

// FileLogger creates a new logger with file output.
func FileLogger(name string, level Level, format string, timeFormat string, file string, sync bool) (*Logger, error) {
	out, err := os.OpenFile(file, os.O_WRONLY|os.O_APPEND|os.O_CREATE, os.ModeAppend|0644)
	if err != nil {
		return nil, err
	}
	logger, err := createLogger(name, level, format, timeFormat, out, sync)
	if err == nil {
		logger.fd = out
		return logger, nil
	} else {
		return nil, err
	}
}

// WriterLogger creates a new logger with a writer
func WriterLogger(name string, level Level, format string, timeFormat string, out io.Writer, sync bool) (*Logger, error) {
	return createLogger(name, level, format, timeFormat, out, sync)
}

// WriterLogger creates a new logger from a configuration file
func ConfigLogger(filename string) (*Logger, error) {
	conf := config.NewConfig(filename)
	err := conf.Read()
	if err != nil {
		return nil, err
	}
	name := conf.Get("", "name")
	slevel := conf.Get("", "level")
	if slevel == "" {
		slevel = "0"
	}
	l, err := strconv.Atoi(slevel)
	if err != nil {
		return nil, err
	}
	level := Level(l)
	format := conf.Get("", "format")
	if format == "" {
		format = BasicFormat
	}
	timeFormat := conf.Get("", "timeFormat")
	if timeFormat == "" {
		timeFormat = DefaultTimeFormat
	}
	ssync := conf.Get("", "sync")
	if ssync == "" {
		ssync = "0"
	}
	file := conf.Get("", "file")
	if file == "" {
		file = DefaultFileName
	}
	sync := true
	if ssync == "0" {
		sync = false
	} else if ssync == "1" {
		sync = true
	} else {
		return nil, err
	}
	return FileLogger(name, level, format, timeFormat, file, sync)
}

// createLogger create a new logger
func createLogger(name string, level Level, format string, timeFormat string, out io.Writer, sync bool) (*Logger, error) {
	logger := new(Logger)

	err := logger.parseFormat(format)
	if err != nil {
		return nil, err
	}

	// asign values to logger
	logger.name = name
	logger.level = level
	logger.out = out
	logger.seqid = 0
	logger.sync = sync
	logger.queue = make(chan string, queueSize)
	logger.request = make(chan request, reqSize)
	logger.flush = make(chan bool)
	logger.finish = make(chan bool)
	logger.quit = make(chan bool)
	logger.startTime = time.Now()
	logger.fd = nil
	logger.timeFormat = timeFormat

	// start watcher to write logs if it is async or no runtime field
	if !logger.sync {
		go logger.watcher()
	}

	return logger, nil
}

// Destroy sends quit signal to watcher and releases all the resources.
func (logger *Logger) Destroy() {
	if !logger.sync {
		// quit watcher
		logger.quit <- true
		// wait for watcher quit
		<-logger.quit
	}
	// clean up
	if logger.fd != nil {
		logger.fd.Close()
	}
}

// Flush the writer
func (logger *Logger) Flush() {
	if !logger.sync {
		// send flush signal
		logger.flush <- true
		// wait for the flush finish
		<-logger.finish
	}
}

// Getter functions

func (logger *Logger) Name() string {
	return logger.name
}

func (logger *Logger) StartTime() int64 {
	return logger.startTime.UnixNano()
}

func (logger *Logger) TimeFormat() string {
	return logger.timeFormat
}

func (logger *Logger) Level() Level {
	return Level(atomic.LoadInt32((*int32)(&logger.level)))
}

func (logger *Logger) RecordFormat() string {
	return logger.recordFormat
}

func (logger *Logger) RecordArgs() []string {
	return logger.recordArgs
}

func (logger *Logger) Writer() io.Writer {
	return logger.out
}

func (logger *Logger) Sync() bool {
	return logger.sync
}

// Setter functions

func (logger *Logger) SetLevel(level Level) {
	atomic.StoreInt32((*int32)(&logger.level), int32(level))
}

func (logger *Logger) SetWriter(out ...io.Writer) {
	logger.out = io.MultiWriter(out...)
}
<|endoftext|>


<commit_before>package main

import (
  "fmt"
  "io/ioutil"
  utils "github.com/hyperledger/fabric/protos/utils"
)

func main(){
  fmt.Println("Use codes to verify fabric")
  blockFile := "/root/go/src/github.com/hyperledger/fabric/examples/e2e_cli/channel-artifacts/genesis.block"
  data, err := ioutil.ReadFile(blockFile)
	if err != nil {
		fmt.Errorf("Could not read block %s", blockFile)
  }
  block, err := utils.UnmarshalBlock(data)
	if err != nil {
		fmt.Errorf("error unmarshaling to block: %s", err)
  }
  fmt.Println(utils.GetChainIDFromBlock(block))
  fmt.Println(" get what we want")
}
<commit_msg>Update main.go<commit_after>package main

import (
  "fmt"
  "io/ioutil"
  utils "github.com/hyperledger/fabric/protos/utils"
)

func main(){
  fmt.Println("Use codes to verify fabric")
  blockFile := "/root/go/src/github.com/hyperledger/fabric/examples/e2e_cli/channel-artifacts/genesis.block"
  data, err := ioutil.ReadFile(blockFile)
  if err != nil {
		fmt.Errorf("Could not read block %s", blockFile)
  }

  fmt.Println(utils.GetChainIDFromBlockBytes(block))
  fmt.Println(" get what we want")
}
<|endoftext|>


<commit_before>package dockermachine

import (
	"bufio"
	"bytes"
	"net/http"
	"os/exec"
	"path/filepath"
	"runtime"
	"strconv"
	"strings"

	dockerclient "github.com/docker/engine-api/client"
	"github.com/docker/go-connections/tlsconfig"
	docker "github.com/fsouza/go-dockerclient"
	"github.com/openshift/origin/pkg/bootstrap/docker/errors"
	"github.com/openshift/origin/pkg/bootstrap/docker/localcmd"
	"k8s.io/kubernetes/pkg/util/net"
)

const (
	defaultMachineMemory     = 2048
	defaultMachineProcessors = 2
)

// Builder can be used to create a new Docker machine on the local system
type Builder struct {
	name       string
	memory     int
	processors int
}

// NewBuilder creates a Docker machine Builder object used to create a Docker machine
func NewBuilder() *Builder {
	return &Builder{}
}

// Name sets the name of the Docker machine to build
func (b *Builder) Name(name string) *Builder {
	b.name = name
	return b
}

// Memory sets the amount of memory (in MB) to give a Docker machine when creating it
func (b *Builder) Memory(mem int) *Builder {
	b.memory = mem
	return b
}

// Processors sets the number of processors to give a Docker machine when creating it
func (b *Builder) Processors(proc int) *Builder {
	b.processors = proc
	return b
}

// Create creates a new Docker machine
func (b *Builder) Create() error {
	if Exists(b.name) {
		return ErrDockerMachineExists
	}
	if IsAvailable() {
		return ErrDockerMachineNotAvailable
	}
	mem := b.memory
	if mem == 0 {
		mem = determineMachineMemory()
	}
	proc := b.processors
	if proc == 0 {
		proc = determineMachineProcessors()
	}
	return localcmd.New(dockerMachineBinary()).Args(
		"create",
		"--driver", "virtualbox",
		"--virtualbox-cpu-count", strconv.Itoa(proc),
		"--virtualbox-memory", strconv.Itoa(mem),
		"--engine-insecure-registry", "172.30.0.0/16",
		b.name).Run()
}

// IsRunning returns true if a Docker machine is running
func IsRunning(name string) bool {
	err := localcmd.New(dockerMachineBinary()).Args("ip", name).Run()
	return err == nil
}

// IP returns the IP address of the Docker machine
func IP(name string) (string, error) {
	output, _, err := localcmd.New(dockerMachineBinary()).Args("ip", name).Output()
	if err != nil {
		return "", ErrDockerMachineExec("ip", err)
	}
	return strings.TrimSpace(output), nil
}

// Exists returns true if a Docker machine exists
func Exists(name string) bool {
	err := localcmd.New(dockerMachineBinary()).Args("inspect", name).Run()
	return err == nil
}

// Start starts up an existing Docker machine
func Start(name string) error {
	err := localcmd.New(dockerMachineBinary()).Args("start", name).Run()
	if err != nil {
		return ErrDockerMachineExec("start", err)
	}
	return nil
}

// Client returns a Docker client for the given Docker machine
func Client(name string) (*docker.Client, *dockerclient.Client, error) {
	output, _, err := localcmd.New(dockerMachineBinary()).Args("env", name).Output()
	if err != nil {
		return nil, nil, ErrDockerMachineExec("env", err)
	}
	scanner := bufio.NewScanner(bytes.NewBufferString(output))
	var (
		dockerHost, certPath string
		tlsVerify            bool
	)
	prefix := "export "
	if runtime.GOOS == "windows" {
		prefix = "SET "
	}
	for scanner.Scan() {
		line := scanner.Text()
		if strings.HasPrefix(line, prefix) {
			line = strings.TrimPrefix(line, prefix)
			parts := strings.SplitN(line, "=", 2)
			if len(parts) != 2 {
				continue
			}
			switch strings.ToUpper(parts[0]) {
			case "DOCKER_HOST":
				dockerHost = strings.Trim(parts[1], "\"")
			case "DOCKER_CERT_PATH":
				certPath = strings.Trim(parts[1], "\"")
			case "DOCKER_TLS_VERIFY":
				tlsVerify = len(parts[1]) > 0
			}
		}
	}
	var client *docker.Client
	if len(certPath) > 0 {
		cert := filepath.Join(certPath, "cert.pem")
		key := filepath.Join(certPath, "key.pem")
		ca := filepath.Join(certPath, "ca.pem")
		client, err = docker.NewVersionedTLSClient(dockerHost, cert, key, ca, "")
	} else {
		client, err = docker.NewVersionedClient(dockerHost, "")
	}
	if err != nil {
		return nil, nil, errors.NewError("could not get Docker client for machine %s", name).WithCause(err)
	}
	client.SkipServerVersionCheck = true

	var httpClient *http.Client
	if len(certPath) > 0 {
		tlscOptions := tlsconfig.Options{
			CAFile:             filepath.Join(certPath, "ca.pem"),
			CertFile:           filepath.Join(certPath, "cert.pem"),
			KeyFile:            filepath.Join(certPath, "key.pem"),
			InsecureSkipVerify: !tlsVerify,
		}
		tlsc, tlsErr := tlsconfig.Client(tlscOptions)
		if tlsErr != nil {
			return nil, nil, errors.NewError("could not create TLS config client for machine %s", name).WithCause(tlsErr)
		}
		httpClient = &http.Client{
			Transport: net.SetTransportDefaults(&http.Transport{
				TLSClientConfig: tlsc,
			}),
		}
	}

	engineAPIClient, err := dockerclient.NewClient(dockerHost, "", httpClient, nil)
	if err != nil {
		return nil, nil, errors.NewError("cannot create Docker engine API client").WithCause(err)
	}

	return client, engineAPIClient, nil
}

// IsAvailable returns true if the docker-machine executable can be found in the PATH
func IsAvailable() bool {
	_, err := exec.LookPath(dockerMachineBinary())
	return err != nil
}

// determineMachineMemory determines a reasonable default for machine memory
// TODO: implement linux & windows
func determineMachineMemory() int {
	if runtime.GOOS == "darwin" {
		output, _, err := localcmd.New("sysctl").Args("-n", "hw.memsize").Output()
		if err == nil {
			mem, perr := strconv.ParseInt(strings.TrimSpace(output), 10, 64)
			if perr == nil {
				return int(mem / (1024 * 1024 * 2)) // half of available megs
			}
		}
	}
	return defaultMachineMemory
}

// determineMachineProcs determines a reasonable default for machine processors
// TODO: implement linux & windows
func determineMachineProcessors() int {
	if runtime.GOOS == "darwin" {
		output, _, err := localcmd.New("sysctl").Args("-n", "hw.ncpus").Output()
		if err == nil {
			ncpus, aerr := strconv.Atoi(strings.TrimSpace(output))
			if aerr == nil {
				return ncpus // use all cpus
			}
		}
	}
	return defaultMachineProcessors
}

func dockerMachineBinary() string {
	if runtime.GOOS == "windows" {
		return "docker-machine.exe"
	}
	return "docker-machine"
}
<commit_msg>Updated sysctl usage for cpu<commit_after>package dockermachine

import (
	"bufio"
	"bytes"
	"net/http"
	"os/exec"
	"path/filepath"
	"runtime"
	"strconv"
	"strings"

	dockerclient "github.com/docker/engine-api/client"
	"github.com/docker/go-connections/tlsconfig"
	docker "github.com/fsouza/go-dockerclient"
	"github.com/openshift/origin/pkg/bootstrap/docker/errors"
	"github.com/openshift/origin/pkg/bootstrap/docker/localcmd"
	"k8s.io/kubernetes/pkg/util/net"
)

const (
	defaultMachineMemory     = 2048
	defaultMachineProcessors = 2
)

// Builder can be used to create a new Docker machine on the local system
type Builder struct {
	name       string
	memory     int
	processors int
}

// NewBuilder creates a Docker machine Builder object used to create a Docker machine
func NewBuilder() *Builder {
	return &Builder{}
}

// Name sets the name of the Docker machine to build
func (b *Builder) Name(name string) *Builder {
	b.name = name
	return b
}

// Memory sets the amount of memory (in MB) to give a Docker machine when creating it
func (b *Builder) Memory(mem int) *Builder {
	b.memory = mem
	return b
}

// Processors sets the number of processors to give a Docker machine when creating it
func (b *Builder) Processors(proc int) *Builder {
	b.processors = proc
	return b
}

// Create creates a new Docker machine
func (b *Builder) Create() error {
	if Exists(b.name) {
		return ErrDockerMachineExists
	}
	if IsAvailable() {
		return ErrDockerMachineNotAvailable
	}
	mem := b.memory
	if mem == 0 {
		mem = determineMachineMemory()
	}
	proc := b.processors
	if proc == 0 {
		proc = determineMachineProcessors()
	}
	return localcmd.New(dockerMachineBinary()).Args(
		"create",
		"--driver", "virtualbox",
		"--virtualbox-cpu-count", strconv.Itoa(proc),
		"--virtualbox-memory", strconv.Itoa(mem),
		"--engine-insecure-registry", "172.30.0.0/16",
		b.name).Run()
}

// IsRunning returns true if a Docker machine is running
func IsRunning(name string) bool {
	err := localcmd.New(dockerMachineBinary()).Args("ip", name).Run()
	return err == nil
}

// IP returns the IP address of the Docker machine
func IP(name string) (string, error) {
	output, _, err := localcmd.New(dockerMachineBinary()).Args("ip", name).Output()
	if err != nil {
		return "", ErrDockerMachineExec("ip", err)
	}
	return strings.TrimSpace(output), nil
}

// Exists returns true if a Docker machine exists
func Exists(name string) bool {
	err := localcmd.New(dockerMachineBinary()).Args("inspect", name).Run()
	return err == nil
}

// Start starts up an existing Docker machine
func Start(name string) error {
	err := localcmd.New(dockerMachineBinary()).Args("start", name).Run()
	if err != nil {
		return ErrDockerMachineExec("start", err)
	}
	return nil
}

// Client returns a Docker client for the given Docker machine
func Client(name string) (*docker.Client, *dockerclient.Client, error) {
	output, _, err := localcmd.New(dockerMachineBinary()).Args("env", name).Output()
	if err != nil {
		return nil, nil, ErrDockerMachineExec("env", err)
	}
	scanner := bufio.NewScanner(bytes.NewBufferString(output))
	var (
		dockerHost, certPath string
		tlsVerify            bool
	)
	prefix := "export "
	if runtime.GOOS == "windows" {
		prefix = "SET "
	}
	for scanner.Scan() {
		line := scanner.Text()
		if strings.HasPrefix(line, prefix) {
			line = strings.TrimPrefix(line, prefix)
			parts := strings.SplitN(line, "=", 2)
			if len(parts) != 2 {
				continue
			}
			switch strings.ToUpper(parts[0]) {
			case "DOCKER_HOST":
				dockerHost = strings.Trim(parts[1], "\"")
			case "DOCKER_CERT_PATH":
				certPath = strings.Trim(parts[1], "\"")
			case "DOCKER_TLS_VERIFY":
				tlsVerify = len(parts[1]) > 0
			}
		}
	}
	var client *docker.Client
	if len(certPath) > 0 {
		cert := filepath.Join(certPath, "cert.pem")
		key := filepath.Join(certPath, "key.pem")
		ca := filepath.Join(certPath, "ca.pem")
		client, err = docker.NewVersionedTLSClient(dockerHost, cert, key, ca, "")
	} else {
		client, err = docker.NewVersionedClient(dockerHost, "")
	}
	if err != nil {
		return nil, nil, errors.NewError("could not get Docker client for machine %s", name).WithCause(err)
	}
	client.SkipServerVersionCheck = true

	var httpClient *http.Client
	if len(certPath) > 0 {
		tlscOptions := tlsconfig.Options{
			CAFile:             filepath.Join(certPath, "ca.pem"),
			CertFile:           filepath.Join(certPath, "cert.pem"),
			KeyFile:            filepath.Join(certPath, "key.pem"),
			InsecureSkipVerify: !tlsVerify,
		}
		tlsc, tlsErr := tlsconfig.Client(tlscOptions)
		if tlsErr != nil {
			return nil, nil, errors.NewError("could not create TLS config client for machine %s", name).WithCause(tlsErr)
		}
		httpClient = &http.Client{
			Transport: net.SetTransportDefaults(&http.Transport{
				TLSClientConfig: tlsc,
			}),
		}
	}

	engineAPIClient, err := dockerclient.NewClient(dockerHost, "", httpClient, nil)
	if err != nil {
		return nil, nil, errors.NewError("cannot create Docker engine API client").WithCause(err)
	}

	return client, engineAPIClient, nil
}

// IsAvailable returns true if the docker-machine executable can be found in the PATH
func IsAvailable() bool {
	_, err := exec.LookPath(dockerMachineBinary())
	return err != nil
}

// determineMachineMemory determines a reasonable default for machine memory
// TODO: implement linux & windows
func determineMachineMemory() int {
	if runtime.GOOS == "darwin" {
		output, _, err := localcmd.New("sysctl").Args("-n", "hw.memsize").Output()
		if err == nil {
			mem, perr := strconv.ParseInt(strings.TrimSpace(output), 10, 64)
			if perr == nil {
				return int(mem / (1024 * 1024 * 2)) // half of available megs
			}
		}
	}
	return defaultMachineMemory
}

// determineMachineProcs determines a reasonable default for machine processors
// TODO: implement linux & windows
func determineMachineProcessors() int {
	if runtime.GOOS == "darwin" {
		output, _, err := localcmd.New("sysctl").Args("-n", "hw.logicalcpu").Output()
		if err == nil {
			cpus, aerr := strconv.Atoi(strings.TrimSpace(output))
			if aerr == nil {
				return cpus // use all cpus
			}
		}
	}
	return defaultMachineProcessors
}

func dockerMachineBinary() string {
	if runtime.GOOS == "windows" {
		return "docker-machine.exe"
	}
	return "docker-machine"
}
<|endoftext|>


<commit_before>package cassandra

import (
	"github.com/golang/glog"
	apiv1 "k8s.io/api/core/v1"
	"k8s.io/client-go/tools/record"

	v1alpha1 "github.com/jetstack/navigator/pkg/apis/navigator/v1alpha1"
	"github.com/jetstack/navigator/pkg/controllers"
	"github.com/jetstack/navigator/pkg/controllers/cassandra/actions"
	"github.com/jetstack/navigator/pkg/controllers/cassandra/nodepool"
	"github.com/jetstack/navigator/pkg/controllers/cassandra/pilot"
	"github.com/jetstack/navigator/pkg/controllers/cassandra/role"
	"github.com/jetstack/navigator/pkg/controllers/cassandra/rolebinding"
	"github.com/jetstack/navigator/pkg/controllers/cassandra/seedlabeller"
	"github.com/jetstack/navigator/pkg/controllers/cassandra/service"
	"github.com/jetstack/navigator/pkg/controllers/cassandra/serviceaccount"
)

const (
	ErrorSync   = "ErrSync"
	SuccessSync = "SuccessSync"

	MessageErrorSyncServiceAccount = "Error syncing service account: %s"
	MessageErrorSyncRole           = "Error syncing role: %s"
	MessageErrorSyncRoleBinding    = "Error syncing role binding: %s"
	MessageErrorSyncConfigMap      = "Error syncing config map: %s"
	MessageErrorSyncService        = "Error syncing service: %s"
	MessageErrorSyncNodePools      = "Error syncing node pools: %s"
	MessageErrorSyncPilots         = "Error syncing pilots: %s"
	MessageErrorSyncSeedLabels     = "Error syncing seed labels: %s"
	MessageErrorSync               = "Error syncing: %s"
	MessageSuccessSync             = "Successfully synced CassandraCluster"
)

type ControlInterface interface {
	Sync(*v1alpha1.CassandraCluster) error
}

var _ ControlInterface = &defaultCassandraClusterControl{}

type defaultCassandraClusterControl struct {
	seedProviderServiceControl service.Interface
	nodesServiceControl        service.Interface
	nodepoolControl            nodepool.Interface
	pilotControl               pilot.Interface
	serviceAccountControl      serviceaccount.Interface
	roleControl                role.Interface
	roleBindingControl         rolebinding.Interface
	seedLabellerControl        seedlabeller.Interface
	recorder                   record.EventRecorder
	state                      *controllers.State
}

func NewControl(
	seedProviderServiceControl service.Interface,
	nodesServiceControl service.Interface,
	nodepoolControl nodepool.Interface,
	pilotControl pilot.Interface,
	serviceAccountControl serviceaccount.Interface,
	roleControl role.Interface,
	roleBindingControl rolebinding.Interface,
	seedlabellerControl seedlabeller.Interface,
	recorder record.EventRecorder,
	state *controllers.State,
) ControlInterface {
	return &defaultCassandraClusterControl{
		seedProviderServiceControl: seedProviderServiceControl,
		nodesServiceControl:        nodesServiceControl,
		nodepoolControl:            nodepoolControl,
		pilotControl:               pilotControl,
		serviceAccountControl:      serviceAccountControl,
		roleControl:                roleControl,
		roleBindingControl:         roleBindingControl,
		seedLabellerControl:        seedlabellerControl,
		recorder:                   recorder,
		state:                      state,
	}
}

// checkPausedConditions checks if the given cluster is paused or not and adds an appropriate condition.
func (e *defaultCassandraClusterControl) checkPausedConditions(c *v1alpha1.CassandraCluster) error {
	cond := c.Status.GetStatusCondition(v1alpha1.ClusterConditionProgressing)
	pausedCondExists := cond != nil && cond.Reason == v1alpha1.PausedClusterReason

	needsUpdate := false
	if c.Spec.Paused && !pausedCondExists {
		c.Status.UpdateStatusCondition(
			v1alpha1.ClusterConditionProgressing,
			v1alpha1.ConditionFalse,
			v1alpha1.PausedClusterReason,
			"Cluster is paused",
		)
		needsUpdate = true
	} else if !c.Spec.Paused && pausedCondExists {
		c.Status.UpdateStatusCondition(
			v1alpha1.ClusterConditionProgressing,
			v1alpha1.ConditionTrue,
			v1alpha1.ResumedClusterReason,
			"Cluster is resumed",
		)
		needsUpdate = true
	}

	if !needsUpdate {
		return nil
	}

	var err error
	c, err = e.state.NavigatorClientset.NavigatorV1alpha1().CassandraClusters(c.Namespace).UpdateStatus(c)
	return err
}

func (e *defaultCassandraClusterControl) Sync(c *v1alpha1.CassandraCluster) error {
	c = c.DeepCopy()
	var err error

	e.checkPausedConditions(c)

	if c.Spec.Paused == true {
		glog.V(4).Infof("defaultCassandraClusterControl.Sync skipped, since cluster is paused")
		return nil
	}

	glog.V(4).Infof("defaultCassandraClusterControl.Sync")
	err = e.seedProviderServiceControl.Sync(c)
	if err != nil {
		e.recorder.Eventf(
			c,
			apiv1.EventTypeWarning,
			ErrorSync,
			MessageErrorSyncService,
			err,
		)
		return err
	}
	err = e.nodesServiceControl.Sync(c)
	if err != nil {
		e.recorder.Eventf(
			c,
			apiv1.EventTypeWarning,
			ErrorSync,
			MessageErrorSyncService,
			err,
		)
		return err
	}
	err = e.nodepoolControl.Sync(c)
	if err != nil {
		e.recorder.Eventf(
			c,
			apiv1.EventTypeWarning,
			ErrorSync,
			MessageErrorSyncNodePools,
			err,
		)
		return err
	}
	err = e.pilotControl.Sync(c)
	if err != nil {
		e.recorder.Eventf(
			c,
			apiv1.EventTypeWarning,
			ErrorSync,
			MessageErrorSyncPilots,
			err,
		)
		return err
	}
	err = e.serviceAccountControl.Sync(c)
	if err != nil {
		e.recorder.Eventf(
			c,
			apiv1.EventTypeWarning,
			ErrorSync,
			MessageErrorSyncServiceAccount,
			err,
		)
		return err
	}
	err = e.roleControl.Sync(c)
	if err != nil {
		e.recorder.Eventf(
			c,
			apiv1.EventTypeWarning,
			ErrorSync,
			MessageErrorSyncRole,
			err,
		)
		return err
	}
	err = e.roleBindingControl.Sync(c)
	if err != nil {
		e.recorder.Eventf(
			c,
			apiv1.EventTypeWarning,
			ErrorSync,
			MessageErrorSyncRoleBinding,
			err,
		)
		return err
	}
	err = e.seedLabellerControl.Sync(c)
	if err != nil {
		e.recorder.Eventf(
			c,
			apiv1.EventTypeWarning,
			ErrorSync,
			MessageErrorSyncSeedLabels,
			err,
		)
		return err
	}

	a := NextAction(c)
	if a != nil {
		err = a.Execute(e.state)
		if err != nil {
			e.recorder.Eventf(
				c,
				apiv1.EventTypeWarning,
				ErrorSync,
				MessageErrorSync,
				err,
			)
			return err
		}
	}

	e.recorder.Event(
		c,
		apiv1.EventTypeNormal,
		SuccessSync,
		MessageSuccessSync,
	)
	return nil
}

func NextAction(c *v1alpha1.CassandraCluster) controllers.Action {
	for _, np := range c.Spec.NodePools {
		_, found := c.Status.NodePools[np.Name]
		if !found {
			return &actions.CreateNodePool{
				Cluster:  c,
				NodePool: &np,
			}
		}
	}
	for _, np := range c.Spec.NodePools {
		nps := c.Status.NodePools[np.Name]
		if *np.Replicas > nps.ReadyReplicas {
			return &actions.ScaleOut{
				Cluster:  c,
				NodePool: &np,
			}
		}
	}
	return nil
}
<commit_msg>Remove unused variable<commit_after>package cassandra

import (
	"github.com/golang/glog"
	apiv1 "k8s.io/api/core/v1"
	"k8s.io/client-go/tools/record"

	v1alpha1 "github.com/jetstack/navigator/pkg/apis/navigator/v1alpha1"
	"github.com/jetstack/navigator/pkg/controllers"
	"github.com/jetstack/navigator/pkg/controllers/cassandra/actions"
	"github.com/jetstack/navigator/pkg/controllers/cassandra/nodepool"
	"github.com/jetstack/navigator/pkg/controllers/cassandra/pilot"
	"github.com/jetstack/navigator/pkg/controllers/cassandra/role"
	"github.com/jetstack/navigator/pkg/controllers/cassandra/rolebinding"
	"github.com/jetstack/navigator/pkg/controllers/cassandra/seedlabeller"
	"github.com/jetstack/navigator/pkg/controllers/cassandra/service"
	"github.com/jetstack/navigator/pkg/controllers/cassandra/serviceaccount"
)

const (
	ErrorSync   = "ErrSync"
	SuccessSync = "SuccessSync"

	MessageErrorSyncServiceAccount = "Error syncing service account: %s"
	MessageErrorSyncRole           = "Error syncing role: %s"
	MessageErrorSyncRoleBinding    = "Error syncing role binding: %s"
	MessageErrorSyncConfigMap      = "Error syncing config map: %s"
	MessageErrorSyncService        = "Error syncing service: %s"
	MessageErrorSyncNodePools      = "Error syncing node pools: %s"
	MessageErrorSyncPilots         = "Error syncing pilots: %s"
	MessageErrorSyncSeedLabels     = "Error syncing seed labels: %s"
	MessageErrorSync               = "Error syncing: %s"
	MessageSuccessSync             = "Successfully synced CassandraCluster"
)

type ControlInterface interface {
	Sync(*v1alpha1.CassandraCluster) error
}

var _ ControlInterface = &defaultCassandraClusterControl{}

type defaultCassandraClusterControl struct {
	seedProviderServiceControl service.Interface
	nodesServiceControl        service.Interface
	nodepoolControl            nodepool.Interface
	pilotControl               pilot.Interface
	serviceAccountControl      serviceaccount.Interface
	roleControl                role.Interface
	roleBindingControl         rolebinding.Interface
	seedLabellerControl        seedlabeller.Interface
	recorder                   record.EventRecorder
	state                      *controllers.State
}

func NewControl(
	seedProviderServiceControl service.Interface,
	nodesServiceControl service.Interface,
	nodepoolControl nodepool.Interface,
	pilotControl pilot.Interface,
	serviceAccountControl serviceaccount.Interface,
	roleControl role.Interface,
	roleBindingControl rolebinding.Interface,
	seedlabellerControl seedlabeller.Interface,
	recorder record.EventRecorder,
	state *controllers.State,
) ControlInterface {
	return &defaultCassandraClusterControl{
		seedProviderServiceControl: seedProviderServiceControl,
		nodesServiceControl:        nodesServiceControl,
		nodepoolControl:            nodepoolControl,
		pilotControl:               pilotControl,
		serviceAccountControl:      serviceAccountControl,
		roleControl:                roleControl,
		roleBindingControl:         roleBindingControl,
		seedLabellerControl:        seedlabellerControl,
		recorder:                   recorder,
		state:                      state,
	}
}

// checkPausedConditions checks if the given cluster is paused or not and adds an appropriate condition.
func (e *defaultCassandraClusterControl) checkPausedConditions(c *v1alpha1.CassandraCluster) error {
	cond := c.Status.GetStatusCondition(v1alpha1.ClusterConditionProgressing)
	pausedCondExists := cond != nil && cond.Reason == v1alpha1.PausedClusterReason

	needsUpdate := false
	if c.Spec.Paused && !pausedCondExists {
		c.Status.UpdateStatusCondition(
			v1alpha1.ClusterConditionProgressing,
			v1alpha1.ConditionFalse,
			v1alpha1.PausedClusterReason,
			"Cluster is paused",
		)
		needsUpdate = true
	} else if !c.Spec.Paused && pausedCondExists {
		c.Status.UpdateStatusCondition(
			v1alpha1.ClusterConditionProgressing,
			v1alpha1.ConditionTrue,
			v1alpha1.ResumedClusterReason,
			"Cluster is resumed",
		)
		needsUpdate = true
	}

	if !needsUpdate {
		return nil
	}

	var err error
	_, err = e.state.NavigatorClientset.NavigatorV1alpha1().CassandraClusters(c.Namespace).UpdateStatus(c)
	return err
}

func (e *defaultCassandraClusterControl) Sync(c *v1alpha1.CassandraCluster) error {
	c = c.DeepCopy()
	var err error

	e.checkPausedConditions(c)

	if c.Spec.Paused == true {
		glog.V(4).Infof("defaultCassandraClusterControl.Sync skipped, since cluster is paused")
		return nil
	}

	glog.V(4).Infof("defaultCassandraClusterControl.Sync")
	err = e.seedProviderServiceControl.Sync(c)
	if err != nil {
		e.recorder.Eventf(
			c,
			apiv1.EventTypeWarning,
			ErrorSync,
			MessageErrorSyncService,
			err,
		)
		return err
	}
	err = e.nodesServiceControl.Sync(c)
	if err != nil {
		e.recorder.Eventf(
			c,
			apiv1.EventTypeWarning,
			ErrorSync,
			MessageErrorSyncService,
			err,
		)
		return err
	}
	err = e.nodepoolControl.Sync(c)
	if err != nil {
		e.recorder.Eventf(
			c,
			apiv1.EventTypeWarning,
			ErrorSync,
			MessageErrorSyncNodePools,
			err,
		)
		return err
	}
	err = e.pilotControl.Sync(c)
	if err != nil {
		e.recorder.Eventf(
			c,
			apiv1.EventTypeWarning,
			ErrorSync,
			MessageErrorSyncPilots,
			err,
		)
		return err
	}
	err = e.serviceAccountControl.Sync(c)
	if err != nil {
		e.recorder.Eventf(
			c,
			apiv1.EventTypeWarning,
			ErrorSync,
			MessageErrorSyncServiceAccount,
			err,
		)
		return err
	}
	err = e.roleControl.Sync(c)
	if err != nil {
		e.recorder.Eventf(
			c,
			apiv1.EventTypeWarning,
			ErrorSync,
			MessageErrorSyncRole,
			err,
		)
		return err
	}
	err = e.roleBindingControl.Sync(c)
	if err != nil {
		e.recorder.Eventf(
			c,
			apiv1.EventTypeWarning,
			ErrorSync,
			MessageErrorSyncRoleBinding,
			err,
		)
		return err
	}
	err = e.seedLabellerControl.Sync(c)
	if err != nil {
		e.recorder.Eventf(
			c,
			apiv1.EventTypeWarning,
			ErrorSync,
			MessageErrorSyncSeedLabels,
			err,
		)
		return err
	}

	a := NextAction(c)
	if a != nil {
		err = a.Execute(e.state)
		if err != nil {
			e.recorder.Eventf(
				c,
				apiv1.EventTypeWarning,
				ErrorSync,
				MessageErrorSync,
				err,
			)
			return err
		}
	}

	e.recorder.Event(
		c,
		apiv1.EventTypeNormal,
		SuccessSync,
		MessageSuccessSync,
	)
	return nil
}

func NextAction(c *v1alpha1.CassandraCluster) controllers.Action {
	for _, np := range c.Spec.NodePools {
		_, found := c.Status.NodePools[np.Name]
		if !found {
			return &actions.CreateNodePool{
				Cluster:  c,
				NodePool: &np,
			}
		}
	}
	for _, np := range c.Spec.NodePools {
		nps := c.Status.NodePools[np.Name]
		if *np.Replicas > nps.ReadyReplicas {
			return &actions.ScaleOut{
				Cluster:  c,
				NodePool: &np,
			}
		}
	}
	return nil
}
<|endoftext|>