<commit_before>package s3vfs

import (
	"bytes"
	"fmt"
	"io"
	"io/ioutil"
	"net/url"
	"os"
	pathpkg "path"
	"path/filepath"
	"time"

	"code.google.com/p/go.tools/godoc/vfs"

	"github.com/sourcegraph/rwvfs"
	"github.com/sqs/s3"
	"github.com/sqs/s3/s3util"
)

var DefaultS3Config = s3util.Config{
	Keys: &s3.Keys{
		AccessKey: os.Getenv("AWS_ACCESS_KEY_ID"),
		SecretKey: os.Getenv("AWS_SECRET_KEY"),
	},
	Service: s3.DefaultService,
}

// S3 returns an implementation of FileSystem using the specified S3 bucket and
// config. If config is nil, DefaultS3Config is used.
//
// The bucket URL is the full URL to the bucket on Amazon S3, including the
// bucket name and AWS region (e.g.,
// https://s3-us-west-2.amazonaws.com/mybucket).
func S3(bucket *url.URL, config *s3util.Config) rwvfs.FileSystem {
	if config == nil {
		config = &DefaultS3Config
	}
	return &S3FS{bucket, config}
}

type S3FS struct {
	bucket *url.URL
	config *s3util.Config
}

func (fs *S3FS) String() string {
	return fmt.Sprintf("S3 filesystem at %s", fs.bucket)
}

func (fs *S3FS) url(path string) string {
	path = pathpkg.Join(fs.bucket.Path, path)
	return fs.bucket.ResolveReference(&url.URL{Path: path}).String()
}

func (fs *S3FS) Open(name string) (vfs.ReadSeekCloser, error) {
	rdr, err := s3util.Open(fs.url(name), fs.config)
	if err != nil {
		return nil, err
	}

	b, err := ioutil.ReadAll(rdr)
	if err != nil {
		return nil, err
	}

	return nopCloser{bytes.NewReader(b)}, nil
}

func (fs *S3FS) ReadDir(path string) ([]os.FileInfo, error) {
	dir, err := s3util.NewFile(fs.url(path), fs.config)
	if err != nil {
		return nil, err
	}

	fis, err := dir.Readdir(0)
	if err != nil {
		return nil, err
	}
	for i, fi := range fis {
		fis[i] = &fileInfo{
			name:    pathpkg.Base(fi.Name()),
			size:    fi.Size(),
			mode:    fi.Mode(),
			modTime: fi.ModTime(),
			sys:     fi.Sys(),
		}
	}
	return fis, nil
}

func (fs *S3FS) Lstat(name string) (os.FileInfo, error) {
	name = filepath.Clean(name)

	if name == "." {
		return &fileInfo{
			name:    ".",
			size:    0,
			mode:    os.ModeDir,
			modTime: time.Time{},
		}, nil
	}

	fis, err := fs.ReadDir(pathpkg.Dir(name))
	if err != nil {
		return nil, err
	}
	for _, fi := range fis {
		if fi.Name() == pathpkg.Base(name) {
			return fi, nil
		}
	}
	return nil, os.ErrNotExist
}

func (fs *S3FS) Stat(name string) (os.FileInfo, error) {
	return fs.Lstat(name)
}

// Create opens the file at path for writing, creating the file if it doesn't
// exist and truncating it otherwise.
func (fs *S3FS) Create(path string) (io.WriteCloser, error) {
	return s3util.Create(fs.url(path), nil, fs.config)
}

func (fs *S3FS) Mkdir(name string) error {
	// S3 doesn't have directories.
	return nil
}

func (fs *S3FS) Remove(name string) error {
	rdr, err := s3util.Delete(fs.url(name), fs.config)
	if rdr != nil {
		rdr.Close()
	}
	return err
}

type nopCloser struct {
	io.ReadSeeker
}

func (nc nopCloser) Close() error { return nil }

type fileInfo struct {
	name    string
	size    int64
	mode    os.FileMode
	modTime time.Time
	sys     interface{}
}

func (f *fileInfo) Name() string       { return f.name }
func (f *fileInfo) Size() int64        { return f.size }
func (f *fileInfo) Mode() os.FileMode  { return f.mode }
func (f *fileInfo) ModTime() time.Time { return f.modTime }
func (f *fileInfo) IsDir() bool        { return f.mode&os.ModeDir != 0 }
func (f *fileInfo) Sys() interface{}   { return f.sys }
<commit_msg>use new golang.org/x/... subrepo naming (see https://docs.google.com/document/d/1VdB_ecg6wTBC-iRzAT-eCwvzj0rlRw7iL5KRdzGtlu4/edit)<commit_after>package s3vfs

import (
	"bytes"
	"fmt"
	"io"
	"io/ioutil"
	"net/url"
	"os"
	pathpkg "path"
	"path/filepath"
	"time"

	"golang.org/x/tools/godoc/vfs"

	"github.com/sourcegraph/rwvfs"
	"github.com/sqs/s3"
	"github.com/sqs/s3/s3util"
)

var DefaultS3Config = s3util.Config{
	Keys: &s3.Keys{
		AccessKey: os.Getenv("AWS_ACCESS_KEY_ID"),
		SecretKey: os.Getenv("AWS_SECRET_KEY"),
	},
	Service: s3.DefaultService,
}

// S3 returns an implementation of FileSystem using the specified S3 bucket and
// config. If config is nil, DefaultS3Config is used.
//
// The bucket URL is the full URL to the bucket on Amazon S3, including the
// bucket name and AWS region (e.g.,
// https://s3-us-west-2.amazonaws.com/mybucket).
func S3(bucket *url.URL, config *s3util.Config) rwvfs.FileSystem {
	if config == nil {
		config = &DefaultS3Config
	}
	return &S3FS{bucket, config}
}

type S3FS struct {
	bucket *url.URL
	config *s3util.Config
}

func (fs *S3FS) String() string {
	return fmt.Sprintf("S3 filesystem at %s", fs.bucket)
}

func (fs *S3FS) url(path string) string {
	path = pathpkg.Join(fs.bucket.Path, path)
	return fs.bucket.ResolveReference(&url.URL{Path: path}).String()
}

func (fs *S3FS) Open(name string) (vfs.ReadSeekCloser, error) {
	rdr, err := s3util.Open(fs.url(name), fs.config)
	if err != nil {
		return nil, err
	}

	b, err := ioutil.ReadAll(rdr)
	if err != nil {
		return nil, err
	}

	return nopCloser{bytes.NewReader(b)}, nil
}

func (fs *S3FS) ReadDir(path string) ([]os.FileInfo, error) {
	dir, err := s3util.NewFile(fs.url(path), fs.config)
	if err != nil {
		return nil, err
	}

	fis, err := dir.Readdir(0)
	if err != nil {
		return nil, err
	}
	for i, fi := range fis {
		fis[i] = &fileInfo{
			name:    pathpkg.Base(fi.Name()),
			size:    fi.Size(),
			mode:    fi.Mode(),
			modTime: fi.ModTime(),
			sys:     fi.Sys(),
		}
	}
	return fis, nil
}

func (fs *S3FS) Lstat(name string) (os.FileInfo, error) {
	name = filepath.Clean(name)

	if name == "." {
		return &fileInfo{
			name:    ".",
			size:    0,
			mode:    os.ModeDir,
			modTime: time.Time{},
		}, nil
	}

	fis, err := fs.ReadDir(pathpkg.Dir(name))
	if err != nil {
		return nil, err
	}
	for _, fi := range fis {
		if fi.Name() == pathpkg.Base(name) {
			return fi, nil
		}
	}
	return nil, os.ErrNotExist
}

func (fs *S3FS) Stat(name string) (os.FileInfo, error) {
	return fs.Lstat(name)
}

// Create opens the file at path for writing, creating the file if it doesn't
// exist and truncating it otherwise.
func (fs *S3FS) Create(path string) (io.WriteCloser, error) {
	return s3util.Create(fs.url(path), nil, fs.config)
}

func (fs *S3FS) Mkdir(name string) error {
	// S3 doesn't have directories.
	return nil
}

func (fs *S3FS) Remove(name string) error {
	rdr, err := s3util.Delete(fs.url(name), fs.config)
	if rdr != nil {
		rdr.Close()
	}
	return err
}

type nopCloser struct {
	io.ReadSeeker
}

func (nc nopCloser) Close() error { return nil }

type fileInfo struct {
	name    string
	size    int64
	mode    os.FileMode
	modTime time.Time
	sys     interface{}
}

func (f *fileInfo) Name() string       { return f.name }
func (f *fileInfo) Size() int64        { return f.size }
func (f *fileInfo) Mode() os.FileMode  { return f.mode }
func (f *fileInfo) ModTime() time.Time { return f.modTime }
func (f *fileInfo) IsDir() bool        { return f.mode&os.ModeDir != 0 }
func (f *fileInfo) Sys() interface{}   { return f.sys }
<|endoftext|>


<commit_before>// Copyright 2018 Authors of Cilium
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package clustermesh

import (
	"context"
	"fmt"
	"path"
	"time"

	"github.com/cilium/cilium/api/v1/models"
	"github.com/cilium/cilium/pkg/allocator"
	"github.com/cilium/cilium/pkg/controller"
	"github.com/cilium/cilium/pkg/ipcache"
	"github.com/cilium/cilium/pkg/kvstore"
	"github.com/cilium/cilium/pkg/kvstore/store"
	"github.com/cilium/cilium/pkg/lock"
	nodeStore "github.com/cilium/cilium/pkg/node/store"
	serviceStore "github.com/cilium/cilium/pkg/service/store"

	"github.com/sirupsen/logrus"
)

// remoteCluster represents another cluster other than the cluster the agent is
// running in
type remoteCluster struct {
	// name is the name of the cluster
	name string

	// configPath is the path to the etcd configuration to be used to
	// connect to the etcd cluster of the remote cluster
	configPath string

	// changed receives an event when the remote cluster configuration has
	// changed and is closed when the configuration file was removed
	changed chan bool

	// mesh is the cluster mesh this remote cluster belongs to
	mesh *ClusterMesh

	controllers *controller.Manager

	// remoteConnectionControllerName is the name of the backing controller
	// that maintains the remote connection
	remoteConnectionControllerName string

	// mutex protects the following variables
	// - backend
	// - store
	// - remoteNodes
	// - ipCacheWatcher
	// - remoteIdentityCache
	mutex lock.RWMutex

	// store is the shared store representing all nodes in the remote cluster
	remoteNodes *store.SharedStore

	// remoteServices is the shared store representing services in remote
	// clusters
	remoteServices *store.SharedStore

	// ipCacheWatcher is the watcher that notifies about IP<->identity
	// changes in the remote cluster
	ipCacheWatcher *ipcache.IPIdentityWatcher

	// remoteIdentityCache is a locally cached copy of the identity
	// allocations in the remote cluster
	remoteIdentityCache *allocator.RemoteCache

	// backend is the kvstore backend being used
	backend kvstore.BackendOperations

	swg *lock.StoppableWaitGroup
}

var (
	// skipKvstoreConnection skips the etcd connection, used for testing
	skipKvstoreConnection bool
)

func (rc *remoteCluster) getLogger() *logrus.Entry {
	var (
		status string
		err    error
	)

	if rc.backend != nil {
		status, err = rc.backend.Status()
	}

	return log.WithFields(logrus.Fields{
		fieldClusterName:   rc.name,
		fieldConfig:        rc.configPath,
		fieldKVStoreStatus: status,
		fieldKVStoreErr:    err,
	})
}

// releaseOldConnection releases the etcd connection to a remote cluster. Must
// be called with rc.mutex held for writing.
func (rc *remoteCluster) releaseOldConnection() {
	if rc.ipCacheWatcher != nil {
		rc.ipCacheWatcher.Close()
		rc.ipCacheWatcher = nil
	}

	if rc.remoteNodes != nil {
		rc.remoteNodes.Close(context.TODO())
		rc.remoteNodes = nil
	}
	if rc.remoteIdentityCache != nil {
		rc.remoteIdentityCache.Close()
		rc.remoteIdentityCache = nil
	}
	if rc.remoteServices != nil {
		rc.remoteServices.Close(context.TODO())
		rc.remoteServices = nil
	}
	if rc.backend != nil {
		rc.backend.Close()
		rc.backend = nil
	}
}

func (rc *remoteCluster) restartRemoteConnection(allocator RemoteIdentityWatcher) {
	rc.controllers.UpdateController(rc.remoteConnectionControllerName,
		controller.ControllerParams{
			DoFunc: func(ctx context.Context) error {
				rc.mutex.Lock()
				if rc.backend != nil {
					rc.releaseOldConnection()
				}
				rc.mutex.Unlock()

				backend, errChan := kvstore.NewClient(context.TODO(), kvstore.EtcdBackendName,
					map[string]string{
						kvstore.EtcdOptionConfig: rc.configPath,
					},
					&kvstore.ExtraOptions{NoLockQuorumCheck: true})

				// Block until either an error is returned or
				// the channel is closed due to success of the
				// connection
				rc.getLogger().Debugf("Waiting for connection to be established")
				err, isErr := <-errChan
				if isErr {
					if backend != nil {
						backend.Close()
					}
					rc.getLogger().WithError(err).Warning("Unable to establish etcd connection to remote cluster")
					return err
				}

				rc.getLogger().Info("Connection to remote cluster established")

				remoteNodes, err := store.JoinSharedStore(store.Configuration{
					Prefix:                  path.Join(nodeStore.NodeStorePrefix, rc.name),
					KeyCreator:              rc.mesh.conf.NodeKeyCreator,
					SynchronizationInterval: time.Minute,
					Backend:                 backend,
					Observer:                rc.mesh.conf.NodeObserver(),
				})
				if err != nil {
					backend.Close()
					return err
				}

				remoteServices, err := store.JoinSharedStore(store.Configuration{
					Prefix: path.Join(serviceStore.ServiceStorePrefix, rc.name),
					KeyCreator: func() store.Key {
						svc := serviceStore.ClusterService{}
						return &svc
					},
					SynchronizationInterval: time.Minute,
					Backend:                 backend,
					Observer: &remoteServiceObserver{
						remoteCluster: rc,
						swg:           rc.swg,
					},
				})
				if err != nil {
					remoteNodes.Close(context.TODO())
					backend.Close()
					return err
				}
				rc.swg.Stop()

				remoteIdentityCache, err := allocator.WatchRemoteIdentities(backend)
				if err != nil {
					remoteServices.Close(context.TODO())
					remoteNodes.Close(context.TODO())
					backend.Close()
					return err
				}

				ipCacheWatcher := ipcache.NewIPIdentityWatcher(backend)
				go ipCacheWatcher.Watch(ctx)

				rc.mutex.Lock()
				rc.remoteNodes = remoteNodes
				rc.remoteServices = remoteServices
				rc.backend = backend
				rc.ipCacheWatcher = ipCacheWatcher
				rc.remoteIdentityCache = remoteIdentityCache
				rc.mutex.Unlock()

				rc.getLogger().Info("Established connection to remote etcd")

				return nil
			},
			StopFunc: func(ctx context.Context) error {
				rc.mutex.Lock()
				rc.releaseOldConnection()
				rc.mutex.Unlock()

				rc.getLogger().Info("All resources of remote cluster cleaned up")

				return nil
			},
		},
	)
}

func (rc *remoteCluster) onInsert(allocator RemoteIdentityWatcher) {
	rc.getLogger().Info("New remote cluster configuration")

	if skipKvstoreConnection {
		return
	}

	rc.remoteConnectionControllerName = fmt.Sprintf("remote-etcd-%s", rc.name)
	rc.restartRemoteConnection(allocator)

	go func() {
		for {
			val := <-rc.changed
			if val {
				rc.getLogger().Info("etcd configuration has changed, re-creating connection")
				rc.restartRemoteConnection(allocator)
			} else {
				rc.getLogger().Info("Closing connection to remote etcd")
				return
			}
		}
	}()

	go func() {
		for {
			select {
			// terminate routine when remote cluster is removed
			case _, ok := <-rc.changed:
				if !ok {
					return
				}
			default:
			}

			// wait for backend to appear
			rc.mutex.RLock()
			if rc.backend == nil {
				rc.mutex.RUnlock()
				time.Sleep(10 * time.Millisecond)
				continue
			}
			statusCheckErrors := rc.backend.StatusCheckErrors()
			rc.mutex.RUnlock()

			err, ok := <-statusCheckErrors
			if ok && err != nil {
				rc.getLogger().WithError(err).Warning("Error observed on etcd connection, reconnecting etcd")
				rc.restartRemoteConnection(allocator)
			}
		}
	}()

}

func (rc *remoteCluster) onRemove() {
	rc.controllers.RemoveAllAndWait()
	close(rc.changed)

	rc.getLogger().Info("Remote cluster disconnected")
}

func (rc *remoteCluster) isReady() bool {
	rc.mutex.RLock()
	defer rc.mutex.RUnlock()

	return rc.isReadyLocked()
}

func (rc *remoteCluster) isReadyLocked() bool {
	return rc.backend != nil && rc.remoteNodes != nil && rc.ipCacheWatcher != nil
}

func (rc *remoteCluster) status() *models.RemoteCluster {
	rc.mutex.RLock()
	defer rc.mutex.RUnlock()

	// This can happen when the controller in restartRemoteConnection is waiting
	// for the first connection to succeed.
	var backendStatus = "Waiting for initial connection to be established"
	if rc.backend != nil {
		var backendError error
		backendStatus, backendError = rc.backend.Status()
		if backendError != nil {
			backendStatus = backendError.Error()
		}
	}

	return &models.RemoteCluster{
		Name:              rc.name,
		Ready:             rc.isReadyLocked(),
		NumNodes:          int64(rc.remoteNodes.NumEntries()),
		NumSharedServices: int64(rc.remoteServices.NumEntries()),
		NumIdentities:     int64(rc.remoteIdentityCache.NumEntries()),
		Status:            backendStatus,
	}
}
<commit_msg>clustermesh: Release old connection asynchronously<commit_after>// Copyright 2018 Authors of Cilium
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package clustermesh

import (
	"context"
	"fmt"
	"path"
	"time"

	"github.com/cilium/cilium/api/v1/models"
	"github.com/cilium/cilium/pkg/allocator"
	"github.com/cilium/cilium/pkg/controller"
	"github.com/cilium/cilium/pkg/ipcache"
	"github.com/cilium/cilium/pkg/kvstore"
	"github.com/cilium/cilium/pkg/kvstore/store"
	"github.com/cilium/cilium/pkg/lock"
	nodeStore "github.com/cilium/cilium/pkg/node/store"
	serviceStore "github.com/cilium/cilium/pkg/service/store"

	"github.com/sirupsen/logrus"
)

// remoteCluster represents another cluster other than the cluster the agent is
// running in
type remoteCluster struct {
	// name is the name of the cluster
	name string

	// configPath is the path to the etcd configuration to be used to
	// connect to the etcd cluster of the remote cluster
	configPath string

	// changed receives an event when the remote cluster configuration has
	// changed and is closed when the configuration file was removed
	changed chan bool

	// mesh is the cluster mesh this remote cluster belongs to
	mesh *ClusterMesh

	controllers *controller.Manager

	// remoteConnectionControllerName is the name of the backing controller
	// that maintains the remote connection
	remoteConnectionControllerName string

	// mutex protects the following variables
	// - backend
	// - store
	// - remoteNodes
	// - ipCacheWatcher
	// - remoteIdentityCache
	mutex lock.RWMutex

	// store is the shared store representing all nodes in the remote cluster
	remoteNodes *store.SharedStore

	// remoteServices is the shared store representing services in remote
	// clusters
	remoteServices *store.SharedStore

	// ipCacheWatcher is the watcher that notifies about IP<->identity
	// changes in the remote cluster
	ipCacheWatcher *ipcache.IPIdentityWatcher

	// remoteIdentityCache is a locally cached copy of the identity
	// allocations in the remote cluster
	remoteIdentityCache *allocator.RemoteCache

	// backend is the kvstore backend being used
	backend kvstore.BackendOperations

	swg *lock.StoppableWaitGroup
}

var (
	// skipKvstoreConnection skips the etcd connection, used for testing
	skipKvstoreConnection bool
)

func (rc *remoteCluster) getLogger() *logrus.Entry {
	var (
		status string
		err    error
	)

	if rc.backend != nil {
		status, err = rc.backend.Status()
	}

	return log.WithFields(logrus.Fields{
		fieldClusterName:   rc.name,
		fieldConfig:        rc.configPath,
		fieldKVStoreStatus: status,
		fieldKVStoreErr:    err,
	})
}

// releaseOldConnection releases the etcd connection to a remote cluster
func (rc *remoteCluster) releaseOldConnection() {
	rc.mutex.Lock()
	ipCacheWatcher := rc.ipCacheWatcher
	rc.ipCacheWatcher = nil

	remoteNodes := rc.remoteNodes
	rc.remoteNodes = nil

	remoteIdentityCache := rc.remoteIdentityCache
	rc.remoteIdentityCache = nil

	remoteServices := rc.remoteServices
	rc.remoteServices = nil

	backend := rc.backend
	rc.backend = nil
	rc.mutex.Unlock()

	// Release resources asynchroneously in the background. Many of these
	// operations may time out if the connection was closed due to an error
	// condition.
	go func() {
		if ipCacheWatcher != nil {
			ipCacheWatcher.Close()
		}
		if remoteNodes != nil {
			remoteNodes.Close(context.TODO())
		}
		if remoteIdentityCache != nil {
			remoteIdentityCache.Close()
		}
		if remoteServices != nil {
			remoteServices.Close(context.TODO())
		}
		if backend != nil {
			backend.Close()
		}
	}()
}

func (rc *remoteCluster) restartRemoteConnection(allocator RemoteIdentityWatcher) {
	rc.controllers.UpdateController(rc.remoteConnectionControllerName,
		controller.ControllerParams{
			DoFunc: func(ctx context.Context) error {
				rc.releaseOldConnection()

				backend, errChan := kvstore.NewClient(context.TODO(), kvstore.EtcdBackendName,
					map[string]string{
						kvstore.EtcdOptionConfig: rc.configPath,
					},
					&kvstore.ExtraOptions{NoLockQuorumCheck: true})

				// Block until either an error is returned or
				// the channel is closed due to success of the
				// connection
				rc.getLogger().Debugf("Waiting for connection to be established")
				err, isErr := <-errChan
				if isErr {
					if backend != nil {
						backend.Close()
					}
					rc.getLogger().WithError(err).Warning("Unable to establish etcd connection to remote cluster")
					return err
				}

				rc.getLogger().Info("Connection to remote cluster established")

				remoteNodes, err := store.JoinSharedStore(store.Configuration{
					Prefix:                  path.Join(nodeStore.NodeStorePrefix, rc.name),
					KeyCreator:              rc.mesh.conf.NodeKeyCreator,
					SynchronizationInterval: time.Minute,
					Backend:                 backend,
					Observer:                rc.mesh.conf.NodeObserver(),
				})
				if err != nil {
					backend.Close()
					return err
				}

				remoteServices, err := store.JoinSharedStore(store.Configuration{
					Prefix: path.Join(serviceStore.ServiceStorePrefix, rc.name),
					KeyCreator: func() store.Key {
						svc := serviceStore.ClusterService{}
						return &svc
					},
					SynchronizationInterval: time.Minute,
					Backend:                 backend,
					Observer: &remoteServiceObserver{
						remoteCluster: rc,
						swg:           rc.swg,
					},
				})
				if err != nil {
					remoteNodes.Close(context.TODO())
					backend.Close()
					return err
				}
				rc.swg.Stop()

				remoteIdentityCache, err := allocator.WatchRemoteIdentities(backend)
				if err != nil {
					remoteServices.Close(context.TODO())
					remoteNodes.Close(context.TODO())
					backend.Close()
					return err
				}

				ipCacheWatcher := ipcache.NewIPIdentityWatcher(backend)
				go ipCacheWatcher.Watch(ctx)

				rc.mutex.Lock()
				rc.remoteNodes = remoteNodes
				rc.remoteServices = remoteServices
				rc.backend = backend
				rc.ipCacheWatcher = ipCacheWatcher
				rc.remoteIdentityCache = remoteIdentityCache
				rc.mutex.Unlock()

				rc.getLogger().Info("Established connection to remote etcd")

				return nil
			},
			StopFunc: func(ctx context.Context) error {
				rc.releaseOldConnection()
				rc.getLogger().Info("All resources of remote cluster cleaned up")
				return nil
			},
		},
	)
}

func (rc *remoteCluster) onInsert(allocator RemoteIdentityWatcher) {
	rc.getLogger().Info("New remote cluster configuration")

	if skipKvstoreConnection {
		return
	}

	rc.remoteConnectionControllerName = fmt.Sprintf("remote-etcd-%s", rc.name)
	rc.restartRemoteConnection(allocator)

	go func() {
		for {
			val := <-rc.changed
			if val {
				rc.getLogger().Info("etcd configuration has changed, re-creating connection")
				rc.restartRemoteConnection(allocator)
			} else {
				rc.getLogger().Info("Closing connection to remote etcd")
				return
			}
		}
	}()

	go func() {
		for {
			select {
			// terminate routine when remote cluster is removed
			case _, ok := <-rc.changed:
				if !ok {
					return
				}
			default:
			}

			// wait for backend to appear
			rc.mutex.RLock()
			if rc.backend == nil {
				rc.mutex.RUnlock()
				time.Sleep(10 * time.Millisecond)
				continue
			}
			statusCheckErrors := rc.backend.StatusCheckErrors()
			rc.mutex.RUnlock()

			err, ok := <-statusCheckErrors
			if ok && err != nil {
				rc.getLogger().WithError(err).Warning("Error observed on etcd connection, reconnecting etcd")
				rc.restartRemoteConnection(allocator)
			}
		}
	}()

}

func (rc *remoteCluster) onRemove() {
	rc.controllers.RemoveAllAndWait()
	close(rc.changed)

	rc.getLogger().Info("Remote cluster disconnected")
}

func (rc *remoteCluster) isReady() bool {
	rc.mutex.RLock()
	defer rc.mutex.RUnlock()

	return rc.isReadyLocked()
}

func (rc *remoteCluster) isReadyLocked() bool {
	return rc.backend != nil && rc.remoteNodes != nil && rc.ipCacheWatcher != nil
}

func (rc *remoteCluster) status() *models.RemoteCluster {
	rc.mutex.RLock()
	defer rc.mutex.RUnlock()

	// This can happen when the controller in restartRemoteConnection is waiting
	// for the first connection to succeed.
	var backendStatus = "Waiting for initial connection to be established"
	if rc.backend != nil {
		var backendError error
		backendStatus, backendError = rc.backend.Status()
		if backendError != nil {
			backendStatus = backendError.Error()
		}
	}

	return &models.RemoteCluster{
		Name:              rc.name,
		Ready:             rc.isReadyLocked(),
		NumNodes:          int64(rc.remoteNodes.NumEntries()),
		NumSharedServices: int64(rc.remoteServices.NumEntries()),
		NumIdentities:     int64(rc.remoteIdentityCache.NumEntries()),
		Status:            backendStatus,
	}
}
<|endoftext|>


<commit_before>/*
Copyright 2017 The Nuclio Authors.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package functionconfig

import (
	"strings"
	"testing"

	"github.com/nuclio/logger"
	"github.com/nuclio/zap"
	"github.com/stretchr/testify/suite"
	"k8s.io/api/core/v1"
)

type ReaderTestSuite struct {
	suite.Suite
	logger logger.Logger
	reader *Reader
}

func (suite *ReaderTestSuite) SetupTest() {
	suite.logger, _ = nucliozap.NewNuclioZapTest("test")
	suite.reader, _ = NewReader(suite.logger)
}

func (suite *ReaderTestSuite) TestPartitions() {
	configData := `
metadata:
  name: python handler
spec:
  runtime: python
  handler: reverser:handler
  triggers:
    http:
      maxWorkers: 4
      kind: http
    franz:
      kind: "kafka"
      url: "127.0.0.1:9092"
      total_tasks: 2
      max_task_allocation: 3
      partitions:
      - id: "0"
        checkpoint: "7"
      - id: "1"
      attributes:
        topic: trial
`

	config := Config{}
	reader, err := NewReader(suite.logger)
	suite.Require().NoError(err, "Can't create reader")
	err = reader.Read(strings.NewReader(configData), "processor", &config)
	suite.Require().NoError(err, "Can't reader configuration")

	trigger := config.Spec.Triggers["franz"]
	suite.Require().Equal(2, trigger.TotalTasks, "Bad total_tasks")
	suite.Require().Equal(3, trigger.MaxTaskAllocation, "Bad max_task_allocations")

	suite.Require().Equal(2, len(trigger.Partitions), "Wrong number of partitions")
	for _, partition := range trigger.Partitions {
		switch partition.ID {
		case "0":
			suite.Require().Equal("7", *partition.Checkpoint, "Bad checkpoint")
		case "1":
			suite.Require().Nil(partition.Checkpoint)
		default:
			suite.Require().Failf("Unknown partition ID - %s", partition.ID)
		}
	}
}

func (suite *ReaderTestSuite) TestCodeEntryConfigCaseInsensitivity() {
	configData := `
metadata:
  name: code_entry_name
  namespace: code_entry_namespace
spec:
  runtime: python3.6
  handler: code_entry_handler
  targetCpu: 13  # instead of targetCPU to test case insensitivity
`

	config := Config{
		Meta: Meta{
			Name:      "my_name",
			Namespace: "my_namespace",
		},
		Spec: Spec{
			Runtime: "python2.7",
			Handler: "my_handler",
		},
	}
	reader, err := NewReader(suite.logger)
	suite.Require().NoError(err, "Can't create reader")
	err = reader.Read(strings.NewReader(configData), "processor", &config)
	suite.Require().NoError(err, "Can't reader configuration")

	suite.Require().Equal(13, config.Spec.TargetCPU, "Bad target cpu")
}

func (suite *ReaderTestSuite) TestCodeEntryConfigDontOverrideConfigValues() {
	configData := `
metadata:
  name: code_entry_name
  namespace: code_entry_namespace
  labels:
    label_key: label_val
spec:
  runtime: python3.6
  handler: code_entry_handler
  targetCpu: 13
  build:
    commands:
    - pip install code_entry_package
  env:
    - name: env_var
      value: code_entry_env_val
    - name: code_entry_env_var
      value: code_entry_env_val_2
`

	config := Config{
		Meta: Meta{
			Name:      "my_name",
			Namespace: "my_namespace",
			Labels:    map[string]string{}, // empty map
		},
		Spec: Spec{
			Runtime:   "python2.7",
			Handler:   "my_handler",
			Env:       []v1.EnvVar{{Name: "env_var", Value: "my_env_val"}},
			TargetCPU: 51,
		},
	}
	reader, err := NewReader(suite.logger)
	suite.Require().NoError(err, "Can't create reader")
	err = reader.Read(strings.NewReader(configData), "processor", &config)
	suite.Require().NoError(err, "Can't reader configuration")

	suite.Require().Equal("my_name", config.Meta.Name, "Bad name")
	suite.Require().Equal("my_namespace", config.Meta.Namespace, "Bad namespace")

	expectedEnvVariables := []v1.EnvVar{
		{Name: "env_var", Value: "my_env_val"},
		{Name: "code_entry_env_var", Value: "code_entry_env_val_2"},
	}
	suite.Require().Equal(expectedEnvVariables, config.Spec.Env, "Bad env vars")

	suite.Require().Equal("my_handler", config.Spec.Handler, "Bad handler")
	suite.Require().Equal("python2.7", config.Spec.Runtime, "Bad runtime")
	suite.Require().Equal([]string{"pip install code_entry_package"}, config.Spec.Build.Commands, "Bad commands")
	suite.Require().Equal(map[string]string{"label_key": "label_val"}, config.Meta.Labels, "Bad labels")
	suite.Require().Equal(51, config.Spec.TargetCPU, "Bad target cpu")
}

func (suite *ReaderTestSuite) TestToDeployOptions() {
	suite.T().Skip("TODO")
	//	flatConfigurationContents := `
	//
	//name: function-name
	//namespace: function-namespace
	//runtime: golang
	//handler: some.module:handler
	//triggers:
	//
	//  http:
	//    maxWorkers: 4
	//    kind: http
	//
	//  rmq:
	//    kind: rabbit-mq
	//    url: amqp://guest:guest@34.224.60.166:5672
	//    attributes:
	//      exchangeName: functions
	//      queueName: functions
	//
	//dataBindings:
	//  db0:
	//    class: v3io
	//    secret: something
	//    url: http://192.168.51.240:8081/1024
	//
	//build:
	//  commands:
	//  - command1
	//  - command2
	//  - command3
	//  baseImage: someBaseImage
	//`

	//createFunctionOptions := platform.NewDeployOptions(nil)
	//
	//err := suite.reader.Read(bytes.NewBufferString(flatConfigurationContents), "yaml")
	//suite.Require().NoError(err)
	//
	//err = suite.reader.ToDeployOptions(createFunctionOptions)
	//suite.Require().NoError(err)
	//

	// compare.CompareNoOrder(&createFunctionOptions, &createFunctionOptions)
	// TODO
}

func TestRegistryTestSuite(t *testing.T) {
	suite.Run(t, new(ReaderTestSuite))
}
<commit_msg>Added external code entry type merger test for http trigger serviceType (#1973)<commit_after>/*
Copyright 2017 The Nuclio Authors.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package functionconfig

import (
	"strings"
	"testing"

	"github.com/ghodss/yaml"
	"github.com/nuclio/logger"
	"github.com/nuclio/zap"
	"github.com/stretchr/testify/suite"
	"k8s.io/api/core/v1"
)

type ReaderTestSuite struct {
	suite.Suite
	logger logger.Logger
	reader *Reader
}

func (suite *ReaderTestSuite) SetupTest() {
	suite.logger, _ = nucliozap.NewNuclioZapTest("test")
	suite.reader, _ = NewReader(suite.logger)
}

func (suite *ReaderTestSuite) TestPartitions() {
	configData := `
metadata:
  name: python handler
spec:
  runtime: python
  handler: reverser:handler
  triggers:
    http:
      maxWorkers: 4
      kind: http
    franz:
      kind: "kafka"
      url: "127.0.0.1:9092"
      total_tasks: 2
      max_task_allocation: 3
      partitions:
      - id: "0"
        checkpoint: "7"
      - id: "1"
      attributes:
        topic: trial
`

	config := Config{}
	reader, err := NewReader(suite.logger)
	suite.Require().NoError(err, "Can't create reader")
	err = reader.Read(strings.NewReader(configData), "processor", &config)
	suite.Require().NoError(err, "Can't reader configuration")

	trigger := config.Spec.Triggers["franz"]
	suite.Require().Equal(2, trigger.TotalTasks, "Bad total_tasks")
	suite.Require().Equal(3, trigger.MaxTaskAllocation, "Bad max_task_allocations")

	suite.Require().Equal(2, len(trigger.Partitions), "Wrong number of partitions")
	for _, partition := range trigger.Partitions {
		switch partition.ID {
		case "0":
			suite.Require().Equal("7", *partition.Checkpoint, "Bad checkpoint")
		case "1":
			suite.Require().Nil(partition.Checkpoint)
		default:
			suite.Require().Failf("Unknown partition ID - %s", partition.ID)
		}
	}
}

func (suite *ReaderTestSuite) TestCodeEntryConfigCaseInsensitivity() {
	configData := `
metadata:
  name: code_entry_name
  namespace: code_entry_namespace
spec:
  runtime: python3.6
  handler: code_entry_handler
  targetCpu: 13  # instead of targetCPU to test case insensitivity
`

	config := Config{
		Meta: Meta{
			Name:      "my_name",
			Namespace: "my_namespace",
		},
		Spec: Spec{
			Runtime: "python2.7",
			Handler: "my_handler",
		},
	}
	reader, err := NewReader(suite.logger)
	suite.Require().NoError(err, "Can't create reader")
	err = reader.Read(strings.NewReader(configData), "processor", &config)
	suite.Require().NoError(err, "Can't reader configuration")

	suite.Require().Equal(13, config.Spec.TargetCPU, "Bad target cpu")
}

func (suite *ReaderTestSuite) TestCodeEntryConfigDontOverrideConfigValues() {
	configData := `
metadata:
  name: code_entry_name
  namespace: code_entry_namespace
  labels:
    label_key: label_val
spec:
  runtime: python3.6
  handler: code_entry_handler
  targetCpu: 13
  build:
    commands:
    - pip install code_entry_package
  env:
    - name: env_var
      value: code_entry_env_val
    - name: code_entry_env_var
      value: code_entry_env_val_2
`

	config := Config{
		Meta: Meta{
			Name:      "my_name",
			Namespace: "my_namespace",
			Labels:    map[string]string{}, // empty map
		},
		Spec: Spec{
			Runtime:   "python2.7",
			Handler:   "my_handler",
			Env:       []v1.EnvVar{{Name: "env_var", Value: "my_env_val"}},
			TargetCPU: 51,
		},
	}
	reader, err := NewReader(suite.logger)
	suite.Require().NoError(err, "Can't create reader")
	err = reader.Read(strings.NewReader(configData), "processor", &config)
	suite.Require().NoError(err, "Can't reader configuration")

	suite.Require().Equal("my_name", config.Meta.Name, "Bad name")
	suite.Require().Equal("my_namespace", config.Meta.Namespace, "Bad namespace")

	expectedEnvVariables := []v1.EnvVar{
		{Name: "env_var", Value: "my_env_val"},
		{Name: "code_entry_env_var", Value: "code_entry_env_val_2"},
	}
	suite.Require().Equal(expectedEnvVariables, config.Spec.Env, "Bad env vars")

	suite.Require().Equal("my_handler", config.Spec.Handler, "Bad handler")
	suite.Require().Equal("python2.7", config.Spec.Runtime, "Bad runtime")
	suite.Require().Equal([]string{"pip install code_entry_package"}, config.Spec.Build.Commands, "Bad commands")
	suite.Require().Equal(map[string]string{"label_key": "label_val"}, config.Meta.Labels, "Bad labels")
	suite.Require().Equal(51, config.Spec.TargetCPU, "Bad target cpu")
}

func (suite *ReaderTestSuite) TestToDeployOptions() {
	suite.T().Skip("TODO")
	//	flatConfigurationContents := `
	//
	//name: function-name
	//namespace: function-namespace
	//runtime: golang
	//handler: some.module:handler
	//triggers:
	//
	//  http:
	//    maxWorkers: 4
	//    kind: http
	//
	//  rmq:
	//    kind: rabbit-mq
	//    url: amqp://guest:guest@34.224.60.166:5672
	//    attributes:
	//      exchangeName: functions
	//      queueName: functions
	//
	//dataBindings:
	//  db0:
	//    class: v3io
	//    secret: something
	//    url: http://192.168.51.240:8081/1024
	//
	//build:
	//  commands:
	//  - command1
	//  - command2
	//  - command3
	//  baseImage: someBaseImage
	//`

	//createFunctionOptions := platform.NewDeployOptions(nil)
	//
	//err := suite.reader.Read(bytes.NewBufferString(flatConfigurationContents), "yaml")
	//suite.Require().NoError(err)
	//
	//err = suite.reader.ToDeployOptions(createFunctionOptions)
	//suite.Require().NoError(err)
	//

	// compare.CompareNoOrder(&createFunctionOptions, &createFunctionOptions)
	// TODO
}

func (suite *ReaderTestSuite) TestCodeEntryConfigTriggerMerge() {
	type TestTrigger struct {
		Name        string
		ServiceType string
	}

	testCases := []struct {
		name                  string
		configTrigger         TestTrigger
		codeEntryTrigger      TestTrigger
		expectedConfigTrigger TestTrigger
		expectValidityError   bool
	}{
		{
			name: "bothNamesDefault",
			configTrigger: TestTrigger{
				Name:        "default-http",
				ServiceType: "ClusterIP",
			},
			codeEntryTrigger: TestTrigger{
				Name:        "default-http",
				ServiceType: "NodePort",
			},
			expectedConfigTrigger: TestTrigger{
				Name:        "default-http",
				ServiceType: "ClusterIP",
			},
			expectValidityError: false,
		},
		{
			name: "codeEntryDefaultNameConfigCustomName",
			configTrigger: TestTrigger{
				Name:        "my-trigger",
				ServiceType: "ClusterIP",
			},
			codeEntryTrigger: TestTrigger{
				Name:        "default-http",
				ServiceType: "NodePort",
			},
			expectedConfigTrigger: TestTrigger{
				Name:        "my-trigger",
				ServiceType: "ClusterIP",
			},
			expectValidityError: false,
		},
		{
			name: "codeEntryCustomNameConfigDefaultName",
			configTrigger: TestTrigger{
				Name:        "default-http",
				ServiceType: "ClusterIP",
			},
			codeEntryTrigger: TestTrigger{
				Name:        "my-trigger",
				ServiceType: "NodePort",
			},
			expectedConfigTrigger: TestTrigger{
				Name:        "my-trigger",
				ServiceType: "NodePort",
			},
			expectValidityError: false,
		},
		{
			name: "bothSameCustomNames",
			configTrigger: TestTrigger{
				Name:        "my-trigger",
				ServiceType: "ClusterIP",
			},
			codeEntryTrigger: TestTrigger{
				Name:        "my-trigger",
				ServiceType: "NodePort",
			},
			expectedConfigTrigger: TestTrigger{
				Name:        "my-trigger",
				ServiceType: "ClusterIP",
			},
			expectValidityError: false,
		},
		{
			name: "differentCustomNames",
			configTrigger: TestTrigger{
				Name:        "my-trigger",
				ServiceType: "ClusterIP",
			},
			codeEntryTrigger: TestTrigger{
				Name:        "not-my-trigger",
				ServiceType: "NodePort",
			},
			expectValidityError: true,
		},
	}

	for _, testCase := range testCases {
		suite.Run(testCase.name, func() {
			config := Config{
				Meta: Meta{
					Name:      "my_name",
					Namespace: "my_namespace",
				},
				Spec: Spec{
					Runtime: "python3.7",
					Handler: "my_handler",
				},
			}
			config.Spec.Triggers = map[string]Trigger{
				testCase.codeEntryTrigger.Name: {
					Name:       testCase.codeEntryTrigger.Name,
					Kind:       "http",
					Attributes: map[string]interface{}{"serviceType": testCase.codeEntryTrigger.ServiceType},
				},
			}
			configData, err := yaml.Marshal(config)
			suite.Require().NoError(err, "Can't marshal config")

			config.Spec.Triggers = map[string]Trigger{
				testCase.configTrigger.Name: {
					Name:       testCase.configTrigger.Name,
					Kind:       "http",
					Attributes: map[string]interface{}{"serviceType": testCase.configTrigger.ServiceType},
				},
			}

			reader, err := NewReader(suite.logger)
			suite.Require().NoError(err)

			err = reader.Read(strings.NewReader(string(configData)), "processor", &config)
			suite.Require().NoError(err)

			err = reader.validateConfigurationFileFunctionConfig(&config)
			if testCase.expectValidityError {
				suite.Require().Error(err)
			} else {
				suite.Require().NoError(err)
				suite.Assert().Equal(1, len(config.Spec.Triggers))
				suite.Assert().Equal(testCase.expectedConfigTrigger.Name,
					config.Spec.Triggers[testCase.expectedConfigTrigger.Name].Name)
				suite.Assert().Equal(testCase.expectedConfigTrigger.ServiceType,
					config.Spec.Triggers[testCase.expectedConfigTrigger.Name].Attributes["serviceType"])
			}
		})
	}
}

func TestRegistryTestSuite(t *testing.T) {
	suite.Run(t, new(ReaderTestSuite))
}
<|endoftext|>


<commit_before>package cli

import (
	"bytes"
	"io/ioutil"
	"testing"

	"github.com/spf13/pflag"

	"k8s.io/apimachinery/pkg/util/sets"
	kcmd "k8s.io/kubernetes/pkg/kubectl/cmd"

	"github.com/openshift/origin/pkg/cmd/util/clientcmd"
)

// MissingCommands is the list of commands we're already missing.
// NEVER ADD TO THIS LIST
// TODO kill this list
var MissingCommands = sets.NewString(
	"namespace", "rolling-update",
	"cluster-info", "api-versions",
	"stop",

	// are on admin commands
	"cordon",
	"drain",
	"uncordon",
	"taint",
	"top",
	"certificate",

	// TODO commands to assess
	"apiversions",
	"clusterinfo",
	"resize",
	"rollingupdate",
	"run-container",
	"update",
)

// WhitelistedCommands is the list of commands we're never going to have,
// defend each one with a comment
var WhitelistedCommands = sets.NewString()

func TestKubectlCompatibility(t *testing.T) {
	f := clientcmd.New(pflag.NewFlagSet("name", pflag.ContinueOnError))

	oc := NewCommandCLI("oc", "oc", &bytes.Buffer{}, ioutil.Discard, ioutil.Discard)
	kubectl := kcmd.NewKubectlCommand(f, nil, ioutil.Discard, ioutil.Discard)

kubectlLoop:
	for _, kubecmd := range kubectl.Commands() {
		for _, occmd := range oc.Commands() {
			if kubecmd.Name() == occmd.Name() {
				if MissingCommands.Has(kubecmd.Name()) {
					t.Errorf("%s was supposed to be missing", kubecmd.Name())
					continue
				}
				if WhitelistedCommands.Has(kubecmd.Name()) {
					t.Errorf("%s was supposed to be whitelisted", kubecmd.Name())
					continue
				}
				continue kubectlLoop
			}
		}
		if MissingCommands.Has(kubecmd.Name()) || WhitelistedCommands.Has(kubecmd.Name()) {
			continue
		}

		t.Errorf("missing %q,", kubecmd.Name())
	}
}

// this only checks one level deep for nested commands, but it does ensure that we've gotten several
// --validate flags.  Based on that we can reasonably assume we got them in the kube commands since they
// all share the same registration.
func TestValidateDisabled(t *testing.T) {
	f := clientcmd.New(pflag.NewFlagSet("name", pflag.ContinueOnError))

	oc := NewCommandCLI("oc", "oc", &bytes.Buffer{}, ioutil.Discard, ioutil.Discard)
	kubectl := kcmd.NewKubectlCommand(f, nil, ioutil.Discard, ioutil.Discard)

	for _, kubecmd := range kubectl.Commands() {
		for _, occmd := range oc.Commands() {
			if kubecmd.Name() == occmd.Name() {
				ocValidateFlag := occmd.Flags().Lookup("validate")
				if ocValidateFlag == nil {
					continue
				}

				if ocValidateFlag.Value.String() != "false" {
					t.Errorf("%s --validate is not defaulting to false", occmd.Name())
				}
			}
		}
	}

}
<commit_msg>boring: exclude new alpha kubectl command<commit_after>package cli

import (
	"bytes"
	"io/ioutil"
	"testing"

	"github.com/spf13/pflag"

	"k8s.io/apimachinery/pkg/util/sets"
	kcmd "k8s.io/kubernetes/pkg/kubectl/cmd"

	"github.com/openshift/origin/pkg/cmd/util/clientcmd"
)

// MissingCommands is the list of commands we're already missing.
// NEVER ADD TO THIS LIST
// TODO kill this list
var MissingCommands = sets.NewString(
	"namespace", "rolling-update",
	"cluster-info", "api-versions",
	"stop",

	// are on admin commands
	"cordon",
	"drain",
	"uncordon",
	"taint",
	"top",
	"certificate",

	// TODO commands to assess
	"apiversions",
	"clusterinfo",
	"resize",
	"rollingupdate",
	"run-container",
	"update",
	"alpha",
)

// WhitelistedCommands is the list of commands we're never going to have,
// defend each one with a comment
var WhitelistedCommands = sets.NewString()

func TestKubectlCompatibility(t *testing.T) {
	f := clientcmd.New(pflag.NewFlagSet("name", pflag.ContinueOnError))

	oc := NewCommandCLI("oc", "oc", &bytes.Buffer{}, ioutil.Discard, ioutil.Discard)
	kubectl := kcmd.NewKubectlCommand(f, nil, ioutil.Discard, ioutil.Discard)

kubectlLoop:
	for _, kubecmd := range kubectl.Commands() {
		for _, occmd := range oc.Commands() {
			if kubecmd.Name() == occmd.Name() {
				if MissingCommands.Has(kubecmd.Name()) {
					t.Errorf("%s was supposed to be missing", kubecmd.Name())
					continue
				}
				if WhitelistedCommands.Has(kubecmd.Name()) {
					t.Errorf("%s was supposed to be whitelisted", kubecmd.Name())
					continue
				}
				continue kubectlLoop
			}
		}
		if MissingCommands.Has(kubecmd.Name()) || WhitelistedCommands.Has(kubecmd.Name()) {
			continue
		}

		t.Errorf("missing %q,", kubecmd.Name())
	}
}

// this only checks one level deep for nested commands, but it does ensure that we've gotten several
// --validate flags.  Based on that we can reasonably assume we got them in the kube commands since they
// all share the same registration.
func TestValidateDisabled(t *testing.T) {
	f := clientcmd.New(pflag.NewFlagSet("name", pflag.ContinueOnError))

	oc := NewCommandCLI("oc", "oc", &bytes.Buffer{}, ioutil.Discard, ioutil.Discard)
	kubectl := kcmd.NewKubectlCommand(f, nil, ioutil.Discard, ioutil.Discard)

	for _, kubecmd := range kubectl.Commands() {
		for _, occmd := range oc.Commands() {
			if kubecmd.Name() == occmd.Name() {
				ocValidateFlag := occmd.Flags().Lookup("validate")
				if ocValidateFlag == nil {
					continue
				}

				if ocValidateFlag.Value.String() != "false" {
					t.Errorf("%s --validate is not defaulting to false", occmd.Name())
				}
			}
		}
	}

}
<|endoftext|>


<commit_before>/*
Copyright 2016 The Kubernetes Authors.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

// Package statusupdater implements interfaces that enable updating the status
// of API objects.
package statusupdater

import (
	"encoding/json"
	"fmt"

	"github.com/golang/glog"

	"k8s.io/kubernetes/pkg/api"
	kcache "k8s.io/kubernetes/pkg/client/cache"
	"k8s.io/kubernetes/pkg/client/clientset_generated/internalclientset"
	"k8s.io/kubernetes/pkg/controller/volume/attachdetach/cache"
	"k8s.io/kubernetes/pkg/conversion"
	"k8s.io/kubernetes/pkg/util/strategicpatch"
)

// NodeStatusUpdater defines a set of operations for updating the
// VolumesAttached field in the Node Status.
type NodeStatusUpdater interface {
	// Gets a list of node statuses that should be updated from the actual state
	// of the world and updates them.
	UpdateNodeStatuses() error
}

// NewNodeStatusUpdater returns a new instance of NodeStatusUpdater.
func NewNodeStatusUpdater(
	kubeClient internalclientset.Interface,
	nodeInformer kcache.SharedInformer,
	actualStateOfWorld cache.ActualStateOfWorld) NodeStatusUpdater {
	return &nodeStatusUpdater{
		actualStateOfWorld: actualStateOfWorld,
		nodeInformer:       nodeInformer,
		kubeClient:         kubeClient,
	}
}

type nodeStatusUpdater struct {
	kubeClient         internalclientset.Interface
	nodeInformer       kcache.SharedInformer
	actualStateOfWorld cache.ActualStateOfWorld
}

func (nsu *nodeStatusUpdater) UpdateNodeStatuses() error {
	// TODO: investigate right behavior if nodeName is empty
	// kubernetes/kubernetes/issues/37777
	nodesToUpdate := nsu.actualStateOfWorld.GetVolumesToReportAttached()
	for nodeName, attachedVolumes := range nodesToUpdate {
		nodeObj, exists, err := nsu.nodeInformer.GetStore().GetByKey(string(nodeName))
		if nodeObj == nil || !exists || err != nil {
			// If node does not exist, its status cannot be updated, log error and
			// reset flag statusUpdateNeeded back to true to indicate this node status
			// needs to be udpated again
			glog.V(2).Infof(
				"Could not update node status. Failed to find node %q in NodeInformer cache. %v",
				nodeName,
				err)
			nsu.actualStateOfWorld.SetNodeStatusUpdateNeeded(nodeName)
			continue
		}

		clonedNode, err := conversion.NewCloner().DeepCopy(nodeObj)
		if err != nil {
			return fmt.Errorf("error cloning node %q: %v",
				nodeName,
				err)
		}

		node, ok := clonedNode.(*api.Node)
		if !ok || node == nil {
			return fmt.Errorf(
				"failed to cast %q object %#v to Node",
				nodeName,
				clonedNode)
		}

		oldData, err := json.Marshal(node)
		if err != nil {
			return fmt.Errorf(
				"failed to Marshal oldData for node %q. %v",
				nodeName,
				err)
		}

		node.Status.VolumesAttached = attachedVolumes

		newData, err := json.Marshal(node)
		if err != nil {
			return fmt.Errorf(
				"failed to Marshal newData for node %q. %v",
				nodeName,
				err)
		}

		patchBytes, err :=
			strategicpatch.CreateStrategicMergePatch(oldData, newData, node)
		if err != nil {
			return fmt.Errorf(
				"failed to CreateStrategicMergePatch for node %q. %v",
				nodeName,
				err)
		}

		_, err = nsu.kubeClient.Core().Nodes().PatchStatus(string(nodeName), patchBytes)
		if err != nil {
			// If update node status fails, reset flag statusUpdateNeeded back to true
			// to indicate this node status needs to be udpated again
			nsu.actualStateOfWorld.SetNodeStatusUpdateNeeded(nodeName)
			return fmt.Errorf(
				"failed to kubeClient.Core().Nodes().Patch for node %q. %v",
				nodeName,
				err)
		}
		glog.V(2).Infof(
			"Updating status for node %q succeeded. patchBytes: %q VolumesAttached: %v",
			nodeName,
			string(patchBytes),
			node.Status.VolumesAttached)

	}
	return nil
}
<commit_msg>fix typo<commit_after>/*
Copyright 2016 The Kubernetes Authors.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

// Package statusupdater implements interfaces that enable updating the status
// of API objects.
package statusupdater

import (
	"encoding/json"
	"fmt"

	"github.com/golang/glog"

	"k8s.io/kubernetes/pkg/api"
	kcache "k8s.io/kubernetes/pkg/client/cache"
	"k8s.io/kubernetes/pkg/client/clientset_generated/internalclientset"
	"k8s.io/kubernetes/pkg/controller/volume/attachdetach/cache"
	"k8s.io/kubernetes/pkg/conversion"
	"k8s.io/kubernetes/pkg/util/strategicpatch"
)

// NodeStatusUpdater defines a set of operations for updating the
// VolumesAttached field in the Node Status.
type NodeStatusUpdater interface {
	// Gets a list of node statuses that should be updated from the actual state
	// of the world and updates them.
	UpdateNodeStatuses() error
}

// NewNodeStatusUpdater returns a new instance of NodeStatusUpdater.
func NewNodeStatusUpdater(
	kubeClient internalclientset.Interface,
	nodeInformer kcache.SharedInformer,
	actualStateOfWorld cache.ActualStateOfWorld) NodeStatusUpdater {
	return &nodeStatusUpdater{
		actualStateOfWorld: actualStateOfWorld,
		nodeInformer:       nodeInformer,
		kubeClient:         kubeClient,
	}
}

type nodeStatusUpdater struct {
	kubeClient         internalclientset.Interface
	nodeInformer       kcache.SharedInformer
	actualStateOfWorld cache.ActualStateOfWorld
}

func (nsu *nodeStatusUpdater) UpdateNodeStatuses() error {
	// TODO: investigate right behavior if nodeName is empty
	// kubernetes/kubernetes/issues/37777
	nodesToUpdate := nsu.actualStateOfWorld.GetVolumesToReportAttached()
	for nodeName, attachedVolumes := range nodesToUpdate {
		nodeObj, exists, err := nsu.nodeInformer.GetStore().GetByKey(string(nodeName))
		if nodeObj == nil || !exists || err != nil {
			// If node does not exist, its status cannot be updated, log error and
			// reset flag statusUpdateNeeded back to true to indicate this node status
			// needs to be updated again
			glog.V(2).Infof(
				"Could not update node status. Failed to find node %q in NodeInformer cache. %v",
				nodeName,
				err)
			nsu.actualStateOfWorld.SetNodeStatusUpdateNeeded(nodeName)
			continue
		}

		clonedNode, err := conversion.NewCloner().DeepCopy(nodeObj)
		if err != nil {
			return fmt.Errorf("error cloning node %q: %v",
				nodeName,
				err)
		}

		node, ok := clonedNode.(*api.Node)
		if !ok || node == nil {
			return fmt.Errorf(
				"failed to cast %q object %#v to Node",
				nodeName,
				clonedNode)
		}

		oldData, err := json.Marshal(node)
		if err != nil {
			return fmt.Errorf(
				"failed to Marshal oldData for node %q. %v",
				nodeName,
				err)
		}

		node.Status.VolumesAttached = attachedVolumes

		newData, err := json.Marshal(node)
		if err != nil {
			return fmt.Errorf(
				"failed to Marshal newData for node %q. %v",
				nodeName,
				err)
		}

		patchBytes, err :=
			strategicpatch.CreateStrategicMergePatch(oldData, newData, node)
		if err != nil {
			return fmt.Errorf(
				"failed to CreateStrategicMergePatch for node %q. %v",
				nodeName,
				err)
		}

		_, err = nsu.kubeClient.Core().Nodes().PatchStatus(string(nodeName), patchBytes)
		if err != nil {
			// If update node status fails, reset flag statusUpdateNeeded back to true
			// to indicate this node status needs to be updated again
			nsu.actualStateOfWorld.SetNodeStatusUpdateNeeded(nodeName)
			return fmt.Errorf(
				"failed to kubeClient.Core().Nodes().Patch for node %q. %v",
				nodeName,
				err)
		}
		glog.V(2).Infof(
			"Updating status for node %q succeeded. patchBytes: %q VolumesAttached: %v",
			nodeName,
			string(patchBytes),
			node.Status.VolumesAttached)

	}
	return nil
}
<|endoftext|>


<commit_before>// Copyright 2016 Marc-Antoine Ruel. All rights reserved.
// Use of this source code is governed under the Apache License, Version 2.0
// that can be found in the LICENSE file.

// Packages the static files in a .go file.
//go:generate go run ../package/main.go -out static_files_gen.go ../../../web

// dlibox drives the dlibox LED strip on a Raspberry Pi. It runs a web server
// for remote control.
package main

import (
	"encoding/json"
	"flag"
	"fmt"
	"io/ioutil"
	"log"
	"os"
	"os/signal"
	"runtime"
	"runtime/pprof"
	"syscall"

	"github.com/kardianos/osext"
	"github.com/maruel/dlibox/go/anim1d"
	"github.com/maruel/dlibox/go/bw2d"
	"github.com/maruel/dlibox/go/pio/buses"
	"github.com/maruel/dlibox/go/pio/buses/bcm283x"
	"github.com/maruel/dlibox/go/pio/buses/i2c"
	"github.com/maruel/dlibox/go/pio/buses/ir"
	"github.com/maruel/dlibox/go/pio/buses/spi"
	"github.com/maruel/dlibox/go/pio/devices"
	"github.com/maruel/dlibox/go/pio/devices/apa102"
	"github.com/maruel/dlibox/go/pio/devices/ssd1306"
	"github.com/maruel/dlibox/go/pio/fakes/screen"
	"github.com/maruel/dlibox/go/psf"
	"github.com/maruel/interrupt"
)

func initDisplay() (devices.Display, error) {
	i2cBus, err := i2c.Make(1)
	if err != nil {
		return nil, err
	}
	display, err := ssd1306.MakeI2C(i2cBus, 128, 64, false)
	if err != nil {
		return nil, err
	}
	f12, err := psf.Load("Terminus12x6")
	if err != nil {
		return nil, err
	}
	f20, err := psf.Load("Terminus20x10")
	if err != nil {
		return nil, err
	}
	// TODO(maruel): Leverage bme280 while at it but don't fail if not
	// connected.
	img, err := bw2d.Make(display.W, display.H)
	if err != nil {
		return nil, err
	}
	f20.Draw(img, 0, 0, bw2d.On, nil, "dlibox!")
	f12.Draw(img, 0, display.H-f12.H-1, bw2d.On, nil, "is awesome")
	if _, err = display.Write(img.Buf); err != nil {
		return nil, err
	}
	return display, nil
}

func initIR(painter *anim1d.Painter, config *IR) error {
	bus, err := ir.Make()
	if err != nil {
		return err
	}
	go func() {
		c := bus.Channel()
		for {
			select {
			case msg, ok := <-c:
				if !ok {
					break
				}
				if !msg.Repeat {
					// TODO(maruel): Locking.
					if pat := config.Mapping[msg.Key]; len(pat) != 0 {
						painter.SetPattern(string(pat))
					}
				}
			}
		}
	}()
	return nil
}

func initPIR(painter *anim1d.Painter, config *PIR) error {
	// TODO(maruel): Cleaner.
	pin := bcm283x.GetPin(config.Pin)
	if pin.String() == "INVALID" {
		return nil
	}
	if err := pin.In(buses.Down, buses.Rising); err != nil {
		return err
	}
	go func() {
		for {
			pin.ReadEdge()
			// TODO(maruel): Locking.
			painter.SetPattern(string(config.Pattern))
		}
	}()
	return nil
}

func mainImpl() error {
	thisFile, err := osext.Executable()
	if err != nil {
		return err
	}

	cpuprofile := flag.String("cpuprofile", "", "dump CPU profile in file")
	port := flag.Int("port", 8010, "http port to listen on")
	verbose := flag.Bool("verbose", false, "enable log output")
	fake := flag.Bool("fake", false, "use a terminal mock, useful to test without the hardware")
	flag.Parse()
	if flag.NArg() != 0 {
		return fmt.Errorf("unexpected argument: %s", flag.Args())
	}

	if !*verbose {
		log.SetOutput(ioutil.Discard)
	}

	interrupt.HandleCtrlC()
	defer interrupt.Set()
	chanSignal := make(chan os.Signal)
	go func() {
		<-chanSignal
		interrupt.Set()
	}()
	signal.Notify(chanSignal, syscall.SIGTERM)

	var properties []string
	if *cpuprofile != "" {
		// Run with cpuprofile, then use 'go tool pprof' to analyze it. See
		// http://blog.golang.org/profiling-go-programs for more details.
		f, err := os.Create(*cpuprofile)
		if err != nil {
			return err
		}
		pprof.StartCPUProfile(f)
		defer pprof.StopCPUProfile()
		properties = append(properties, "profiled=1")
	}

	// Config.
	config := ConfigMgr{}
	config.ResetDefault()
	if err := config.Load(); err != nil {
		return err
	}
	defer config.Close()

	b, err := json.MarshalIndent(config, "", "  ")
	if err != nil {
		return err
	}
	log.Printf("Config:\n%s", string(b))

	fps := 60
	if bcm283x.MaxSpeed < 900000 || runtime.NumCPU() < 4 {
		// Use 30Hz on slower devices because it is too slow.
		fps = 30
	}

	// Output (terminal with ANSI codes or APA102).
	var leds devices.Display
	if *fake {
		// Hardcode to 100 characters when using a terminal output.
		// TODO(maruel): Query the terminal and use its width.
		leds = screen.Make(100)
		defer os.Stdout.Write([]byte("\033[0m\n"))
		// Use lower refresh rate too.
		fps = 30
		properties = append(properties, "fake=1")
	} else {
		spiBus, err := spi.Make(0, 0, config.Settings.APA102.SPIspeed)
		if err != nil {
			return err
		}
		defer spiBus.Close()
		if leds, err = apa102.Make(spiBus, config.Settings.APA102.NumberLights, 255, 6500); err != nil {
			return err
		}
		properties = append(properties, fmt.Sprintf("APA102=%d", config.Settings.APA102.NumberLights))
	}

	// Try to initialize the display.
	if _, err = initDisplay(); err != nil {
		log.Printf("Display not connected")
	}

	// Painter.
	p := anim1d.MakePainter(leds, fps)
	if err := config.Init(p); err != nil {
		return err
	}
	startWebServer(*port, p, &config.Config)

	if err = initIR(p, &config.Settings.IR); err != nil {
		log.Printf("IR not connected")
	}

	if err = initPIR(p, &config.Settings.PIR); err != nil {
		log.Printf("PIR not connected")
	}

	/*
		service, err := initmDNS(*port, properties)
		if err != nil {
			return err
		}
		defer service.Close()
	*/

	return watchFile(thisFile)
}

func main() {
	if err := mainImpl(); err != nil {
		fmt.Fprintf(os.Stderr, "\ndlibox: %s.\n", err)
		os.Exit(1)
	}
}
<commit_msg>Fail less often; survive lack of SPI<commit_after>// Copyright 2016 Marc-Antoine Ruel. All rights reserved.
// Use of this source code is governed under the Apache License, Version 2.0
// that can be found in the LICENSE file.

// Packages the static files in a .go file.
//go:generate go run ../package/main.go -out static_files_gen.go ../../../web

// dlibox drives the dlibox LED strip on a Raspberry Pi. It runs a web server
// for remote control.
package main

import (
	"encoding/json"
	"flag"
	"fmt"
	"image"
	"io/ioutil"
	"log"
	"os"
	"os/signal"
	"runtime"
	"runtime/pprof"
	"syscall"

	"github.com/kardianos/osext"
	"github.com/maruel/dlibox/go/anim1d"
	"github.com/maruel/dlibox/go/bw2d"
	"github.com/maruel/dlibox/go/pio/buses"
	"github.com/maruel/dlibox/go/pio/buses/bcm283x"
	"github.com/maruel/dlibox/go/pio/buses/i2c"
	"github.com/maruel/dlibox/go/pio/buses/ir"
	"github.com/maruel/dlibox/go/pio/buses/spi"
	"github.com/maruel/dlibox/go/pio/devices"
	"github.com/maruel/dlibox/go/pio/devices/apa102"
	"github.com/maruel/dlibox/go/pio/devices/ssd1306"
	"github.com/maruel/dlibox/go/pio/fakes"
	"github.com/maruel/dlibox/go/pio/fakes/screen"
	"github.com/maruel/dlibox/go/psf"
	"github.com/maruel/interrupt"
)

func initDisplay() (devices.Display, error) {
	i2cBus, err := i2c.Make(1)
	if err != nil {
		return nil, err
	}
	display, err := ssd1306.MakeI2C(i2cBus, 128, 64, false)
	if err != nil {
		return nil, err
	}
	f12, err := psf.Load("Terminus12x6")
	if err != nil {
		return nil, err
	}
	f20, err := psf.Load("Terminus20x10")
	if err != nil {
		return nil, err
	}
	// TODO(maruel): Leverage bme280 while at it but don't fail if not
	// connected.
	img, err := bw2d.Make(display.W, display.H)
	if err != nil {
		return nil, err
	}
	f20.Draw(img, 0, 0, bw2d.On, nil, "dlibox!")
	f12.Draw(img, 0, display.H-f12.H-1, bw2d.On, nil, "is awesome")
	if _, err = display.Write(img.Buf); err != nil {
		return nil, err
	}
	return display, nil
}

func initIR(painter *anim1d.Painter, config *IR) error {
	bus, err := ir.Make()
	if err != nil {
		return err
	}
	go func() {
		c := bus.Channel()
		for {
			select {
			case msg, ok := <-c:
				if !ok {
					break
				}
				if !msg.Repeat {
					// TODO(maruel): Locking.
					if pat := config.Mapping[msg.Key]; len(pat) != 0 {
						painter.SetPattern(string(pat))
					}
				}
			}
		}
	}()
	return nil
}

func initPIR(painter *anim1d.Painter, config *PIR) error {
	// TODO(maruel): Cleaner.
	pin := bcm283x.GetPin(config.Pin)
	if pin.String() == "INVALID" {
		return nil
	}
	if err := pin.In(buses.Down, buses.Rising); err != nil {
		return err
	}
	go func() {
		for {
			pin.ReadEdge()
			// TODO(maruel): Locking.
			painter.SetPattern(string(config.Pattern))
		}
	}()
	return nil
}

func mainImpl() error {
	thisFile, err := osext.Executable()
	if err != nil {
		return err
	}

	cpuprofile := flag.String("cpuprofile", "", "dump CPU profile in file")
	port := flag.Int("port", 8010, "http port to listen on")
	verbose := flag.Bool("verbose", false, "enable log output")
	fake := flag.Bool("fake", false, "use a terminal mock, useful to test without the hardware")
	flag.Parse()
	if flag.NArg() != 0 {
		return fmt.Errorf("unexpected argument: %s", flag.Args())
	}

	if !*verbose {
		log.SetOutput(ioutil.Discard)
	}

	interrupt.HandleCtrlC()
	defer interrupt.Set()
	chanSignal := make(chan os.Signal)
	go func() {
		<-chanSignal
		interrupt.Set()
	}()
	signal.Notify(chanSignal, syscall.SIGTERM)

	var properties []string
	if *cpuprofile != "" {
		// Run with cpuprofile, then use 'go tool pprof' to analyze it. See
		// http://blog.golang.org/profiling-go-programs for more details.
		f, err := os.Create(*cpuprofile)
		if err != nil {
			return err
		}
		pprof.StartCPUProfile(f)
		defer pprof.StopCPUProfile()
		properties = append(properties, "profiled=1")
	}

	// Config.
	config := ConfigMgr{}
	config.ResetDefault()
	if err := config.Load(); err != nil {
		log.Printf("Loading config failed: %v", err)
	}
	defer config.Close()

	b, err := json.MarshalIndent(config, "", "  ")
	if err != nil {
		return err
	}
	log.Printf("Config:\n%s", string(b))

	fps := 60
	if bcm283x.MaxSpeed < 900000 || runtime.NumCPU() < 4 {
		// Use 30Hz on slower devices because it is too slow.
		fps = 30
	}

	// Output (terminal with ANSI codes or APA102).
	var leds devices.Display
	if *fake {
		// Hardcode to 100 characters when using a terminal output.
		// TODO(maruel): Query the terminal and use its width.
		leds = screen.Make(100)
		defer os.Stdout.Write([]byte("\033[0m\n"))
		// Use lower refresh rate too.
		fps = 30
		properties = append(properties, "fake=1")
	} else {
		spiBus, err := spi.Make(0, 0, config.Settings.APA102.SPIspeed)
		if err != nil {
			log.Printf("SPI failed: %v", err)
			leds = &fakes.Display{image.NewNRGBA(image.Rect(0, 0, config.Settings.APA102.NumberLights, 1))}
		} else {
			defer spiBus.Close()
			if leds, err = apa102.Make(spiBus, config.Settings.APA102.NumberLights, 255, 6500); err != nil {
				return err
			}
			properties = append(properties, fmt.Sprintf("APA102=%d", config.Settings.APA102.NumberLights))
		}
	}

	// Try to initialize the display.
	if _, err = initDisplay(); err != nil {
		log.Printf("Display not connected")
	}

	// Painter.
	p := anim1d.MakePainter(leds, fps)
	if err := config.Init(p); err != nil {
		return err
	}
	startWebServer(*port, p, &config.Config)

	if err = initIR(p, &config.Settings.IR); err != nil {
		log.Printf("IR not connected: %v", err)
	}

	if err = initPIR(p, &config.Settings.PIR); err != nil {
		log.Printf("PIR not connected: %v", err)
	}

	/*
		service, err := initmDNS(*port, properties)
		if err != nil {
			return err
		}
		defer service.Close()
	*/

	return watchFile(thisFile)
}

func main() {
	if err := mainImpl(); err != nil {
		fmt.Fprintf(os.Stderr, "\ndlibox: %s.\n", err)
		os.Exit(1)
	}
}
<|endoftext|>


<commit_before>package libkb

import (
	"os"
	"runtime"
	"time"

	keybase1 "github.com/keybase/client/protocol/go"
)

const (
	DevelServerURI      = "http://localhost:3000"
	ProductionServerURI = "https://keybase.io"
)

var ServerURI = DevelServerURI

const (
	ConfigFile  = "config.json"
	SessionFile = "session.json"
	DBFile      = "keybase.leveldb"
	SocketFile  = "keybased.sock"
	PIDFile     = "keybased.pid"

	SecretKeyringTemplate = "secretkeys.%u.mpack"

	APIVersion       = "1.0"
	APIURIPathPrefix = "/_/api/" + APIVersion
	DaemonPort       = 40933
	GoClientID       = "keybase.io go client"
	IdentifyAs       = GoClientID + " v" + Version + " " + runtime.GOOS
)

var UserAgent = "Keybase-Go-CLI/" + Version + " (" + runtime.Version() + " on " + runtime.GOOS + ")"

const (
	PermFile          os.FileMode = 0600
	PermDir           os.FileMode = 0700
	UmaskablePermFile os.FileMode = 0666
)

const (
	UserCacheSize        = 0x1000
	PGPFingerprintHexLen = 40

	ProofCacheSize      = 0x1000
	ProofCacheLongDur   = 6 * time.Hour
	ProofCacheMediumDur = 30 * time.Minute
	ProofCacheShortDur  = 1 * time.Minute

	SigShortIDBytes = 27
)

var MerkleProdKIDs = []string{
	"010159baae6c7d43c66adf8fb7bb2b8b4cbe408c062cfc369e693ccb18f85631dbcd0a",
}
var MerkleTestKIDs = []string{
	"0101be58b6c82db64f6ccabb05088db443c69f87d5d48857d709ed6f73948dabe67d0a",
}

const (
	KeybaseKIDV1       = 1 // Uses SHA-256
	KeybaseSignatureV1 = 1

	SigExpireIn       = 24 * 60 * 60 * 365 * 10 // 10 years
	NaclEdDSAExpireIn = 24 * 60 * 60 * 365 * 3  // 3 years
	NaclDHExpireIn    = 24 * 60 * 60 * 365 * 3  // 3 years
	AuthExpireIn      = 24 * 60 * 60 * 365      // 1 year
	KeyExpireIn       = 24 * 60 * 60 * 365 * 8  // 8 years
	SubkeyExpireIn    = 24 * 60 * 60 * 365 * 4  // 4 years
)

// Status codes.  This list should match keybase/lib/constants.iced.
const (
	SCOk                     = 0
	SCLoginRequired          = 201
	SCBadSession             = 202
	SCBadLoginPassword       = 204
	SCNotFound               = 205
	SCGeneric                = 218
	SCAlreadyLoggedIn        = 235
	SCCanceled               = 237
	SCBadSignupUsernameTaken = 701
	SCKeyNotFound            = 901
	SCKeyInUse               = 907
	SCKeyBadGen              = 913
	SCKeyNoSecret            = 914
	SCKeyNoActive            = 915
	SCBadTrackSession        = 1301
	SCStreamExists           = 1501
	SCStreamNotFound         = 1502
	SCStreamWrongKind        = 1503
	SCStreamEOF              = 1504
	SCAPINetworkError        = 1601
	SCTimeout                = 1602
	SCProofError             = 1701
	SCIdentificationExpired  = 1702
)

const (
	IDSuffixKID = 0x0a
)

const (
	MerkleTreeNode = 1
	MerkleTreeLeaf = 2
)

const (
	SibkeyType = "sibkey"
	SubkeyType = "subkey"
	EldestType = "eldest"
)

const (
	SigTypeNone           = 0
	SigTypeSelfSig        = 1
	SigTypeRemoteProof    = 2
	SigTypeTrack          = 3
	SigTypeUntrack        = 4
	SigTypeRevoke         = 5
	SigTypeCryptocurrency = 6
	SigTypeAnnouncement   = 7
)

type KeyType int

const (
	KeyTypeNone                  KeyType = 0
	KeyTypeOpenPGPPublic                 = 1
	KeyTypeP3skbPrivate                  = 2
	KeyTypeKbNaclEddsa                   = 3
	KeyTypeKbNaclDH                      = 4
	KeyTypeKbNaclEddsaServerHalf         = 5
	KeyTypeKbNaclDHServerHalf            = 6
)

const (
	DeviceStatusNone    = 0
	DeviceStatusActive  = 1
	DeviceStatusDefunct = 2
)

// these strings need to match the keys in
// keybase/lib_public/public_constants.iced ->
// public_constants.device.type
const (
	DeviceTypeDesktop = "desktop"
	DeviceTypeMobile  = "mobile"
	DeviceTypeWeb     = "web"
	DeviceTypePaper   = "backup"
)

const DownloadURL = "https://keybase.io/download"

var PGPVersion = "Keybase Go " + Version + " (" + runtime.GOOS + ")"

var PGPArmorHeaders = map[string]string{
	"Version": PGPVersion,
	"Comment": DownloadURL,
}

var RemoteServiceTypes = map[string]keybase1.ProofType{
	"keybase":    keybase1.ProofType_KEYBASE,
	"twitter":    keybase1.ProofType_TWITTER,
	"github":     keybase1.ProofType_GITHUB,
	"reddit":     keybase1.ProofType_REDDIT,
	"coinbase":   keybase1.ProofType_COINBASE,
	"hackernews": keybase1.ProofType_HACKERNEWS,
	"https":      keybase1.ProofType_GENERIC_WEB_SITE,
	"http":       keybase1.ProofType_GENERIC_WEB_SITE,
	"dns":        keybase1.ProofType_DNS,
	"rooter":     keybase1.ProofType_ROOTER,
}

var RemoteServiceOrder = []keybase1.ProofType{
	keybase1.ProofType_KEYBASE,
	keybase1.ProofType_TWITTER,
	keybase1.ProofType_GITHUB,
	keybase1.ProofType_REDDIT,
	keybase1.ProofType_COINBASE,
	keybase1.ProofType_HACKERNEWS,
	keybase1.ProofType_GENERIC_WEB_SITE,
	keybase1.ProofType_ROOTER,
}

const CanonicalHost = "keybase.io"

const (
	HTTPDefaultTimeout = 10 * time.Second
)

// Packet tags for OpenPGP and also Keybase packets
const (
	KeybasePacketV1 = 1
	TagP3skb        = 513
	TagSignature    = 514
	TagEncryption   = 515
)

const (
	KIDPGPBase    AlgoType = 0x00
	KIDPGPRsa              = 0x1
	KIDPGPElgamal          = 0x10
	KIDPGPDsa              = 0x11
	KIDPGPEcdh             = 0x12
	KIDPGPEcdsa            = 0x13
	KIDNaclEddsa           = 0x20
	KIDNaclDH              = 0x21
)

// OpenPGP hash IDs, taken from http://tools.ietf.org/html/rfc4880#section-9.4
const (
	HashPGPMd5       = 1
	HashPGPSha1      = 2
	HashPGPRipemd160 = 3
	HashPGPSha256    = 8
	HashPGPSha384    = 9
	HashPGPSha512    = 10
	HashPGPSha224    = 11
)

const (
	SigKbEddsa = KIDNaclEddsa
)

const (
	ServerUpdateLag = time.Minute
)

// key_revocation_types
const (
	RevSimpleDelete = 0
	RevFull         = 1
	RevDated        = 2
)

type KeyStatus int

const (
	KeyUncancelled KeyStatus = iota
	KeyRevoked
	KeyDeleted
	KeySuperseded
)

type KeyRole int

const (
	DLGNone KeyRole = iota
	DLGSibkey
	DLGSubkey
)

const (
	KexScryptCost       = 32768
	KexScryptR          = 8
	KexScryptP          = 1
	KexScryptKeylen     = 32
	KexSessionIDEntropy = 65 // kex doc specifies 65 bits of entropy
)

const (
	PaperKeyScryptCost    = 32768
	PaperKeyScryptR       = 8
	PaperKeyScryptP       = 1
	PaperKeyScryptKeylen  = 128
	PaperKeyPhraseEntropy = 144
	PaperKeyVersion       = 0
)

const UserSummaryLimit = 500 // max number of user summaries in one request
<commit_msg>Move all expirations (except auth) to 10 years. Also cleanup the constant so its clearer<commit_after>package libkb

import (
	"os"
	"runtime"
	"time"

	keybase1 "github.com/keybase/client/protocol/go"
)

const (
	DevelServerURI      = "http://localhost:3000"
	ProductionServerURI = "https://keybase.io"
)

var ServerURI = DevelServerURI

const (
	ConfigFile  = "config.json"
	SessionFile = "session.json"
	DBFile      = "keybase.leveldb"
	SocketFile  = "keybased.sock"
	PIDFile     = "keybased.pid"

	SecretKeyringTemplate = "secretkeys.%u.mpack"

	APIVersion       = "1.0"
	APIURIPathPrefix = "/_/api/" + APIVersion
	DaemonPort       = 40933
	GoClientID       = "keybase.io go client"
	IdentifyAs       = GoClientID + " v" + Version + " " + runtime.GOOS
)

var UserAgent = "Keybase-Go-CLI/" + Version + " (" + runtime.Version() + " on " + runtime.GOOS + ")"

const (
	PermFile          os.FileMode = 0600
	PermDir           os.FileMode = 0700
	UmaskablePermFile os.FileMode = 0666
)

const (
	UserCacheSize        = 0x1000
	PGPFingerprintHexLen = 40

	ProofCacheSize      = 0x1000
	ProofCacheLongDur   = 6 * time.Hour
	ProofCacheMediumDur = 30 * time.Minute
	ProofCacheShortDur  = 1 * time.Minute

	SigShortIDBytes = 27
)

var MerkleProdKIDs = []string{
	"010159baae6c7d43c66adf8fb7bb2b8b4cbe408c062cfc369e693ccb18f85631dbcd0a",
}
var MerkleTestKIDs = []string{
	"0101be58b6c82db64f6ccabb05088db443c69f87d5d48857d709ed6f73948dabe67d0a",
}

const (
	KeybaseKIDV1       = 1 // Uses SHA-256
	KeybaseSignatureV1 = 1
	OneYearInMinutes   = 24 * 60 * 60 * 365

	SigExpireIn       = OneYearInMinutes * 10 // 10 years
	NaclEdDSAExpireIn = OneYearInMinutes * 10 // 10 years
	NaclDHExpireIn    = OneYearInMinutes * 10 // 10 years
	KeyExpireIn       = OneYearInMinutes * 10 // 10 years
	SubkeyExpireIn    = OneYearInMinutes * 10 // 10 years
	AuthExpireIn      = OneYearInMinutes      // 1 year
)

// Status codes.  This list should match keybase/lib/constants.iced.
const (
	SCOk                     = 0
	SCLoginRequired          = 201
	SCBadSession             = 202
	SCBadLoginPassword       = 204
	SCNotFound               = 205
	SCGeneric                = 218
	SCAlreadyLoggedIn        = 235
	SCCanceled               = 237
	SCBadSignupUsernameTaken = 701
	SCKeyNotFound            = 901
	SCKeyInUse               = 907
	SCKeyBadGen              = 913
	SCKeyNoSecret            = 914
	SCKeyNoActive            = 915
	SCBadTrackSession        = 1301
	SCStreamExists           = 1501
	SCStreamNotFound         = 1502
	SCStreamWrongKind        = 1503
	SCStreamEOF              = 1504
	SCAPINetworkError        = 1601
	SCTimeout                = 1602
	SCProofError             = 1701
	SCIdentificationExpired  = 1702
)

const (
	IDSuffixKID = 0x0a
)

const (
	MerkleTreeNode = 1
	MerkleTreeLeaf = 2
)

const (
	SibkeyType = "sibkey"
	SubkeyType = "subkey"
	EldestType = "eldest"
)

const (
	SigTypeNone           = 0
	SigTypeSelfSig        = 1
	SigTypeRemoteProof    = 2
	SigTypeTrack          = 3
	SigTypeUntrack        = 4
	SigTypeRevoke         = 5
	SigTypeCryptocurrency = 6
	SigTypeAnnouncement   = 7
)

type KeyType int

const (
	KeyTypeNone                  KeyType = 0
	KeyTypeOpenPGPPublic                 = 1
	KeyTypeP3skbPrivate                  = 2
	KeyTypeKbNaclEddsa                   = 3
	KeyTypeKbNaclDH                      = 4
	KeyTypeKbNaclEddsaServerHalf         = 5
	KeyTypeKbNaclDHServerHalf            = 6
)

const (
	DeviceStatusNone    = 0
	DeviceStatusActive  = 1
	DeviceStatusDefunct = 2
)

// these strings need to match the keys in
// keybase/lib_public/public_constants.iced ->
// public_constants.device.type
const (
	DeviceTypeDesktop = "desktop"
	DeviceTypeMobile  = "mobile"
	DeviceTypeWeb     = "web"
	DeviceTypePaper   = "backup"
)

const DownloadURL = "https://keybase.io/download"

var PGPVersion = "Keybase Go " + Version + " (" + runtime.GOOS + ")"

var PGPArmorHeaders = map[string]string{
	"Version": PGPVersion,
	"Comment": DownloadURL,
}

var RemoteServiceTypes = map[string]keybase1.ProofType{
	"keybase":    keybase1.ProofType_KEYBASE,
	"twitter":    keybase1.ProofType_TWITTER,
	"github":     keybase1.ProofType_GITHUB,
	"reddit":     keybase1.ProofType_REDDIT,
	"coinbase":   keybase1.ProofType_COINBASE,
	"hackernews": keybase1.ProofType_HACKERNEWS,
	"https":      keybase1.ProofType_GENERIC_WEB_SITE,
	"http":       keybase1.ProofType_GENERIC_WEB_SITE,
	"dns":        keybase1.ProofType_DNS,
	"rooter":     keybase1.ProofType_ROOTER,
}

var RemoteServiceOrder = []keybase1.ProofType{
	keybase1.ProofType_KEYBASE,
	keybase1.ProofType_TWITTER,
	keybase1.ProofType_GITHUB,
	keybase1.ProofType_REDDIT,
	keybase1.ProofType_COINBASE,
	keybase1.ProofType_HACKERNEWS,
	keybase1.ProofType_GENERIC_WEB_SITE,
	keybase1.ProofType_ROOTER,
}

const CanonicalHost = "keybase.io"

const (
	HTTPDefaultTimeout = 10 * time.Second
)

// Packet tags for OpenPGP and also Keybase packets
const (
	KeybasePacketV1 = 1
	TagP3skb        = 513
	TagSignature    = 514
	TagEncryption   = 515
)

const (
	KIDPGPBase    AlgoType = 0x00
	KIDPGPRsa              = 0x1
	KIDPGPElgamal          = 0x10
	KIDPGPDsa              = 0x11
	KIDPGPEcdh             = 0x12
	KIDPGPEcdsa            = 0x13
	KIDNaclEddsa           = 0x20
	KIDNaclDH              = 0x21
)

// OpenPGP hash IDs, taken from http://tools.ietf.org/html/rfc4880#section-9.4
const (
	HashPGPMd5       = 1
	HashPGPSha1      = 2
	HashPGPRipemd160 = 3
	HashPGPSha256    = 8
	HashPGPSha384    = 9
	HashPGPSha512    = 10
	HashPGPSha224    = 11
)

const (
	SigKbEddsa = KIDNaclEddsa
)

const (
	ServerUpdateLag = time.Minute
)

// key_revocation_types
const (
	RevSimpleDelete = 0
	RevFull         = 1
	RevDated        = 2
)

type KeyStatus int

const (
	KeyUncancelled KeyStatus = iota
	KeyRevoked
	KeyDeleted
	KeySuperseded
)

type KeyRole int

const (
	DLGNone KeyRole = iota
	DLGSibkey
	DLGSubkey
)

const (
	KexScryptCost       = 32768
	KexScryptR          = 8
	KexScryptP          = 1
	KexScryptKeylen     = 32
	KexSessionIDEntropy = 65 // kex doc specifies 65 bits of entropy
)

const (
	PaperKeyScryptCost    = 32768
	PaperKeyScryptR       = 8
	PaperKeyScryptP       = 1
	PaperKeyScryptKeylen  = 128
	PaperKeyPhraseEntropy = 144
	PaperKeyVersion       = 0
)

const UserSummaryLimit = 500 // max number of user summaries in one request
<|endoftext|>


<commit_before>// Copyright 2015 Keybase, Inc. All rights reserved. Use of
// this source code is governed by the included BSD license.

package libkb

import (
	"fmt"
	"strings"
	"testing"

	"github.com/stretchr/testify/require"
)

type cmpTest struct {
	a, b string
	eq   bool
}

var nameCmpTest = []cmpTest{
	{a: "UpperCase", b: "uppercase", eq: true},
	{a: " Space prefix", b: "Space prefix", eq: true},
	{a: "Space suffix ", b: "Space suffix", eq: true},
	{a: "Space Inside", b: "SpaceInside", eq: true},
	{a: "work iPad", b: "work ipad", eq: true},
	{a: "my_ipad", b: "MY IPAD", eq: true},
	{a: "device a", b: "device b", eq: false},
	{a: "mike's computer", b: "mikes computer", eq: true},
	{a: "my+-'_device", b: "my device", eq: true},
}

func TestNameCmp(t *testing.T) {
	for _, test := range nameCmpTest {
		eq := NameCmp(test.a, test.b)
		if eq != test.eq {
			t.Errorf("name compare %q == %q => %v, expected %v", test.a, test.b, eq, test.eq)
		}
	}
}

func TestCombineErrors(t *testing.T) {
	err := CombineErrors(fmt.Errorf("error1"), nil, fmt.Errorf("error3"))
	expected := "There were multiple errors: error1; error3"
	if err.Error() != expected {
		t.Errorf("Wrong output for combine errors: %#v != %#v", err.Error(), expected)
	}
}

func TestWhitespaceNormalize(t *testing.T) {

	data := []struct {
		in, out string
	}{
		{" ab   cd    ef   gh ", "ab cd ef gh"},
		{"a\nb  c\nd", "a b c d"},
		{" a ", "a"},
		{"\na\nb ", "a b"},
		{
			" Verifying myself: I am pomf on Keybase.io. 8a6cewzit2o7zuLKGbDqQADhzfOlGerGuBpq\n/ https://keybase.io/pomf/sigs/8a6cewzit2o7zuLKGbDqQADhzfOlGerGuBpq ",
			"Verifying myself: I am pomf on Keybase.io. 8a6cewzit2o7zuLKGbDqQADhzfOlGerGuBpq / https://keybase.io/pomf/sigs/8a6cewzit2o7zuLKGbDqQADhzfOlGerGuBpq",
		},
	}

	for i, p := range data {
		out := WhitespaceNormalize(p.in)
		if out != p.out {
			t.Errorf("Failed on test %d: %s != %s", i, out, p.out)
		}
	}

}

func TestMakeByte24(t *testing.T) {
	var x1 [24]byte
	var x2 [31]byte
	var x3 [33]byte

	x1[3] = 5

	y := MakeByte24(x1[:])
	require.Equal(t, x1, y)

	require.Panics(t, func() {
		MakeByte24(x2[:])
	})

	require.Panics(t, func() {
		MakeByte24(x3[:])
	})
}

func TestMakeByte32(t *testing.T) {
	var x1 [32]byte
	var x2 [31]byte
	var x3 [33]byte

	x1[3] = 5

	y := MakeByte32(x1[:])
	require.Equal(t, x1, y)

	require.Panics(t, func() {
		MakeByte32(x2[:])
	})

	require.Panics(t, func() {
		MakeByte32(x3[:])
	})
}

func TestAppDataDir(t *testing.T) {
	dir, err := AppDataDir()
	if err != nil {
		// Non-Windows case.
		require.True(t, strings.HasPrefix(err.Error(), "unsupported"))
		return
	}

	// Windows case. AppDataDir should exist, at least on our test
	// machines.
	require.NoError(t, err)
	exists, err := FileExists(dir)
	require.NoError(t, err)
	require.True(t, exists)
}

func TestLocalDataDir(t *testing.T) {
	dir, err := LocalDataDir()
	if err != nil {
		// Non-Windows case.
		require.True(t, strings.HasPrefix(err.Error(), "unsupported"))
		return
	}

	// Windows case. LocalDataDir should exist, at least on our
	// test machines.
	require.NoError(t, err)
	exists, err := FileExists(dir)
	require.NoError(t, err)
	require.True(t, exists)
}
<commit_msg>test ForceWallClock (#11787)<commit_after>// Copyright 2015 Keybase, Inc. All rights reserved. Use of
// this source code is governed by the included BSD license.

package libkb

import (
	"fmt"
	"regexp"
	"strings"
	"testing"
	"time"

	"github.com/stretchr/testify/require"
)

type cmpTest struct {
	a, b string
	eq   bool
}

var nameCmpTest = []cmpTest{
	{a: "UpperCase", b: "uppercase", eq: true},
	{a: " Space prefix", b: "Space prefix", eq: true},
	{a: "Space suffix ", b: "Space suffix", eq: true},
	{a: "Space Inside", b: "SpaceInside", eq: true},
	{a: "work iPad", b: "work ipad", eq: true},
	{a: "my_ipad", b: "MY IPAD", eq: true},
	{a: "device a", b: "device b", eq: false},
	{a: "mike's computer", b: "mikes computer", eq: true},
	{a: "my+-'_device", b: "my device", eq: true},
}

func TestNameCmp(t *testing.T) {
	for _, test := range nameCmpTest {
		eq := NameCmp(test.a, test.b)
		if eq != test.eq {
			t.Errorf("name compare %q == %q => %v, expected %v", test.a, test.b, eq, test.eq)
		}
	}
}

func TestCombineErrors(t *testing.T) {
	err := CombineErrors(fmt.Errorf("error1"), nil, fmt.Errorf("error3"))
	expected := "There were multiple errors: error1; error3"
	if err.Error() != expected {
		t.Errorf("Wrong output for combine errors: %#v != %#v", err.Error(), expected)
	}
}

func TestWhitespaceNormalize(t *testing.T) {

	data := []struct {
		in, out string
	}{
		{" ab   cd    ef   gh ", "ab cd ef gh"},
		{"a\nb  c\nd", "a b c d"},
		{" a ", "a"},
		{"\na\nb ", "a b"},
		{
			" Verifying myself: I am pomf on Keybase.io. 8a6cewzit2o7zuLKGbDqQADhzfOlGerGuBpq\n/ https://keybase.io/pomf/sigs/8a6cewzit2o7zuLKGbDqQADhzfOlGerGuBpq ",
			"Verifying myself: I am pomf on Keybase.io. 8a6cewzit2o7zuLKGbDqQADhzfOlGerGuBpq / https://keybase.io/pomf/sigs/8a6cewzit2o7zuLKGbDqQADhzfOlGerGuBpq",
		},
	}

	for i, p := range data {
		out := WhitespaceNormalize(p.in)
		if out != p.out {
			t.Errorf("Failed on test %d: %s != %s", i, out, p.out)
		}
	}

}

func TestMakeByte24(t *testing.T) {
	var x1 [24]byte
	var x2 [31]byte
	var x3 [33]byte

	x1[3] = 5

	y := MakeByte24(x1[:])
	require.Equal(t, x1, y)

	require.Panics(t, func() {
		MakeByte24(x2[:])
	})

	require.Panics(t, func() {
		MakeByte24(x3[:])
	})
}

func TestMakeByte32(t *testing.T) {
	var x1 [32]byte
	var x2 [31]byte
	var x3 [33]byte

	x1[3] = 5

	y := MakeByte32(x1[:])
	require.Equal(t, x1, y)

	require.Panics(t, func() {
		MakeByte32(x2[:])
	})

	require.Panics(t, func() {
		MakeByte32(x3[:])
	})
}

func TestAppDataDir(t *testing.T) {
	dir, err := AppDataDir()
	if err != nil {
		// Non-Windows case.
		require.True(t, strings.HasPrefix(err.Error(), "unsupported"))
		return
	}

	// Windows case. AppDataDir should exist, at least on our test
	// machines.
	require.NoError(t, err)
	exists, err := FileExists(dir)
	require.NoError(t, err)
	require.True(t, exists)
}

func TestLocalDataDir(t *testing.T) {
	dir, err := LocalDataDir()
	if err != nil {
		// Non-Windows case.
		require.True(t, strings.HasPrefix(err.Error(), "unsupported"))
		return
	}

	// Windows case. LocalDataDir should exist, at least on our
	// test machines.
	require.NoError(t, err)
	exists, err := FileExists(dir)
	require.NoError(t, err)
	require.True(t, exists)
}

func hasMonotonicClock(t time.Time) bool {
	re := regexp.MustCompile(" m=[-+]([.0-9]+)$")
	return re.FindString(t.String()) != ""
}

func TestForceWallClock(t *testing.T) {
	n := time.Now()
	require.True(t, hasMonotonicClock(n))
	require.False(t, hasMonotonicClock(ForceWallClock(n)))
}
<|endoftext|>


<commit_before>package twitter

import (
	"errors"
	"os"
	"strconv"

	"github.com/dghubble/go-twitter/twitter"
	"github.com/dghubble/oauth1"
	"github.com/g-hyoga/kyuko/go/model"
)

var (
	T_CONSUMER_KEY        = os.Getenv("T_CONSUMER_KEY")
	T_CONSUMER_SECRET     = os.Getenv("T_CONSUMER_SECRET")
	T_ACCESS_TOKEN        = os.Getenv("T_ACCESS_TOKEN")
	T_ACCESS_TOKEN_SECRET = os.Getenv("T_ACCESS_TOKEN_SECRET")

	I_CONSUMER_KEY        = os.Getenv("I_CONSUMER_KEY")
	I_CONSUMER_SECRET     = os.Getenv("I_CONSUMER_SECRET")
	I_ACCESS_TOKEN        = os.Getenv("I_ACCESS_TOKEN")
	I_ACCESS_TOKEN_SECRET = os.Getenv("I_ACCESS_TOKEN_SECRET")
)

var tClient twitter.Client

func init() {
	//京田辺
	config := oauth1.NewConfig(T_CONSUMER_KEY, T_CONSUMER_SECRET)
	token := oauth1.NewToken(T_ACCESS_TOKEN, T_ACCESS_TOKEN_SECRET)
	httpClient := config.Client(oauth1.NoContext, token)
	tClient = *twitter.NewClient(httpClient)
}

// create line of template
// period:className(Instructor)
func CreateLine(kyuko model.KyukoData) (string, error) {
	if kyuko.ClassName == "" || kyuko.Instructor == "" || kyuko.Period == 0 {
		return "", errors.New("休講情報がないです")
	}

	period := strconv.Itoa(kyuko.Period)

	line := period + "限:" + kyuko.ClassName + "(" + kyuko.Instructor + ")\n"
	return line, nil
}

// tweet argment
func Update(text string) error {
	_, _, err := tClient.Statuses.Update(text, nil)
	if err != nil {
		return err
	}

	return nil
}
<commit_msg>tweetのテンプレート作成の途中(>weekdayを漢字に直す)<commit_after>package twitter

import (
	"errors"
	"os"
	"strconv"

	"github.com/dghubble/go-twitter/twitter"
	"github.com/dghubble/oauth1"
	"github.com/g-hyoga/kyuko/go/model"
)

var (
	T_CONSUMER_KEY        = os.Getenv("T_CONSUMER_KEY")
	T_CONSUMER_SECRET     = os.Getenv("T_CONSUMER_SECRET")
	T_ACCESS_TOKEN        = os.Getenv("T_ACCESS_TOKEN")
	T_ACCESS_TOKEN_SECRET = os.Getenv("T_ACCESS_TOKEN_SECRET")

	I_CONSUMER_KEY        = os.Getenv("I_CONSUMER_KEY")
	I_CONSUMER_SECRET     = os.Getenv("I_CONSUMER_SECRET")
	I_ACCESS_TOKEN        = os.Getenv("I_ACCESS_TOKEN")
	I_ACCESS_TOKEN_SECRET = os.Getenv("I_ACCESS_TOKEN_SECRET")
)

var tClient twitter.Client

func init() {
	//京田辺
	config := oauth1.NewConfig(T_CONSUMER_KEY, T_CONSUMER_SECRET)
	token := oauth1.NewToken(T_ACCESS_TOKEN, T_ACCESS_TOKEN_SECRET)
	httpClient := config.Client(oauth1.NoContext, token)
	tClient = *twitter.NewClient(httpClient)
}

// create line of template
// period:className(Instructor)
func CreateLine(kyuko model.KyukoData) (string, error) {
	if kyuko.ClassName == "" || kyuko.Instructor == "" || kyuko.Period == 0 {
		return "", errors.New("休講情報がないです")
	}

	period := strconv.Itoa(kyuko.Period)

	line := period + "限:" + kyuko.ClassName + "(" + kyuko.Instructor + ")\n"
	return line, nil
}

// create tweet template
// exsample
//
// hoge曜日の休講情報
// period:className(Instructor)
// period:className(Instructor)
// ...
//
// in 140 characters
func CreateContent(kyuko []model.KyukoDara) ([]string, error) {
	var tws []string

	var lines []string
	for _, v := range kyuko {
		line, err := CreateLine(v)
		if err != nil {
			return tws, err
		}
		lines = append(lines, line)
	}

	weekday := kyuko[0].Weekday
	stringWeekday := time.Weekday(weekday)
	tw := "曜日の休講情報\n"
	for i, v := range lines {

	}
}

// tweet argment
func Update(text string) error {
	_, _, err := tClient.Statuses.Update(text, nil)
	if err != nil {
		return err
	}

	return nil
}
<|endoftext|>


<commit_before>// Copyright 2013, Google Inc. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

package zktopo

import (
	"fmt"
	"path"
	"sort"
	"strconv"
	"strings"
	"time"

	log "github.com/golang/glog"
	"github.com/youtube/vitess/go/vt/topo"
	"github.com/youtube/vitess/go/zk"
	"launchpad.net/gozk/zookeeper"
)

/*
This file contains the code to support the local agent process for zktopo.Server
*/

func (zkts *Server) ValidateTabletActions(tabletAlias topo.TabletAlias) error {
	actionPath := TabletActionPathForAlias(tabletAlias)

	// Ensure that the action node is there. There is no conflict creating
	// this node.
	_, err := zkts.zconn.Create(actionPath, "", 0, zookeeper.WorldACL(zookeeper.PERM_ALL))
	if err != nil && !zookeeper.IsError(err, zookeeper.ZNODEEXISTS) {
		return err
	}
	return nil
}

func (zkts *Server) CreateTabletPidNode(tabletAlias topo.TabletAlias, contents string, done chan struct{}) error {
	zkTabletPath := TabletPathForAlias(tabletAlias)
	path := path.Join(zkTabletPath, "pid")
	return zk.CreatePidNode(zkts.zconn, path, contents, done)
}

func (zkts *Server) ValidateTabletPidNode(tabletAlias topo.TabletAlias) error {
	zkTabletPath := TabletPathForAlias(tabletAlias)
	path := path.Join(zkTabletPath, "pid")
	_, _, err := zkts.zconn.Get(path)
	return err
}

func (zkts *Server) GetSubprocessFlags() []string {
	return zk.GetZkSubprocessFlags()
}

func (zkts *Server) handleActionQueue(tabletAlias topo.TabletAlias, dispatchAction func(actionPath, data string) error) (<-chan zookeeper.Event, error) {
	zkActionPath := TabletActionPathForAlias(tabletAlias)

	// This read may seem a bit pedantic, but it makes it easier
	// for the system to trend towards consistency if an action
	// fails or somehow the action queue gets mangled by an errant
	// process.
	children, _, watch, err := zkts.zconn.ChildrenW(zkActionPath)
	if err != nil {
		return watch, err
	}
	if len(children) > 0 {
		sort.Strings(children)
		for _, child := range children {
			actionPath := zkActionPath + "/" + child
			if _, err := strconv.ParseUint(child, 10, 64); err != nil {
				// This is handy if you want to restart a stuck queue.
				// FIXME(msolomon) could listen on the queue node for a change
				// generated by a "touch", but listening on two things is a bit
				// more complex.
				log.Warningf("remove invalid event from action queue: %v", child)
				zkts.zconn.Delete(actionPath, -1)
			}

			data, _, err := zkts.zconn.Get(actionPath)
			if err != nil {
				log.Errorf("cannot read action %v from zk: %v", actionPath, err)
				break
			}

			if err := dispatchAction(actionPath, data); err != nil {
				break
			}
		}
	}
	return watch, nil
}

func (zkts *Server) ActionEventLoop(tabletAlias topo.TabletAlias, dispatchAction func(actionPath, data string) error, done chan struct{}) {
	for {
		// Process any pending actions when we startup, before we start listening
		// for events.
		watch, err := zkts.handleActionQueue(tabletAlias, dispatchAction)
		if err != nil {
			log.Warningf("action queue failed: %v", err)
			time.Sleep(5 * time.Second)
			continue
		}

		// FIXME(msolomon) Add a skewing timer here to guarantee we wakeup
		// periodically even if events are missed?
		select {
		case event := <-watch:
			if !event.Ok() {
				// NOTE(msolomon) The zk meta conn will reconnect automatically, or
				// error out. At this point, there isn't much to do.
				log.Warningf("zookeeper not OK: %v", event)
				time.Sleep(5 * time.Second)
			}
			// Otherwise, just handle the queue above.
		case <-done:
			return
		}
	}
}

// actionPathToTabletAlias parses an actionPath back
// zkActionPath is /zk/<cell>/vt/tablets/<uid>/action/<number>
func actionPathToTabletAlias(actionPath string) (topo.TabletAlias, error) {
	pathParts := strings.Split(actionPath, "/")
	if len(pathParts) != 8 || pathParts[0] != "" || pathParts[1] != "zk" || pathParts[3] != "vt" || pathParts[4] != "tablets" || pathParts[6] != "action" {
		return topo.TabletAlias{}, fmt.Errorf("invalid action path: %v", actionPath)
	}
	return topo.ParseTabletAliasString(pathParts[2] + "-" + pathParts[5])
}

func (zkts *Server) ReadTabletActionPath(actionPath string) (topo.TabletAlias, string, int64, error) {
	tabletAlias, err := actionPathToTabletAlias(actionPath)
	if err != nil {
		return topo.TabletAlias{}, "", 0, err
	}

	data, stat, err := zkts.zconn.Get(actionPath)
	if err != nil {
		return topo.TabletAlias{}, "", 0, err
	}

	return tabletAlias, data, int64(stat.Version()), nil
}

func (zkts *Server) UpdateTabletAction(actionPath, data string, version int64) error {
	_, err := zkts.zconn.Set(actionPath, data, int(version))
	if err != nil {
		if zookeeper.IsError(err, zookeeper.ZBADVERSION) {
			err = topo.ErrBadVersion
		}
		return err
	}
	return nil
}

// StoreTabletActionResponse stores the data both in action and actionlog
func (zkts *Server) StoreTabletActionResponse(actionPath, data string) error {
	_, err := zkts.zconn.Set(actionPath, data, -1)
	if err != nil {
		return err
	}

	actionLogPath := strings.Replace(actionPath, "/action/", "/actionlog/", 1)
	_, err = zk.CreateRecursive(zkts.zconn, actionLogPath, data, 0, zookeeper.WorldACL(zookeeper.PERM_ALL))
	return err
}

func (zkts *Server) UnblockTabletAction(actionPath string) error {
	return zkts.zconn.Delete(actionPath, -1)
}
<commit_msg>Small cleanup in this file.<commit_after>// Copyright 2013, Google Inc. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

package zktopo

import (
	"fmt"
	"path"
	"sort"
	"strconv"
	"strings"
	"time"

	log "github.com/golang/glog"
	"github.com/youtube/vitess/go/vt/topo"
	"github.com/youtube/vitess/go/zk"
	"launchpad.net/gozk/zookeeper"
)

/*
This file contains the code to support the local agent process for zktopo.Server
*/

func (zkts *Server) ValidateTabletActions(tabletAlias topo.TabletAlias) error {
	actionPath := TabletActionPathForAlias(tabletAlias)

	// Ensure that the action node is there. There is no conflict creating
	// this node.
	_, err := zkts.zconn.Create(actionPath, "", 0, zookeeper.WorldACL(zookeeper.PERM_ALL))
	if err != nil && !zookeeper.IsError(err, zookeeper.ZNODEEXISTS) {
		return err
	}
	return nil
}

func (zkts *Server) CreateTabletPidNode(tabletAlias topo.TabletAlias, contents string, done chan struct{}) error {
	zkTabletPath := TabletPathForAlias(tabletAlias)
	path := path.Join(zkTabletPath, "pid")
	return zk.CreatePidNode(zkts.zconn, path, contents, done)
}

func (zkts *Server) ValidateTabletPidNode(tabletAlias topo.TabletAlias) error {
	zkTabletPath := TabletPathForAlias(tabletAlias)
	path := path.Join(zkTabletPath, "pid")
	_, _, err := zkts.zconn.Get(path)
	return err
}

func (zkts *Server) GetSubprocessFlags() []string {
	return zk.GetZkSubprocessFlags()
}

// handleActionQueue will set the watch on the action queue,
// or return an error if it can't.
// It will also process all pending actions, until it can't read one
// or one fails. No error is returned for action failures.
func (zkts *Server) handleActionQueue(tabletAlias topo.TabletAlias, dispatchAction func(actionPath, data string) error) (<-chan zookeeper.Event, error) {
	zkActionPath := TabletActionPathForAlias(tabletAlias)

	// This read may seem a bit pedantic, but it makes it easier
	// for the system to trend towards consistency if an action
	// fails or somehow the action queue gets mangled by an errant
	// process.
	children, _, watch, err := zkts.zconn.ChildrenW(zkActionPath)
	if err != nil {
		return watch, err
	}
	if len(children) > 0 {
		sort.Strings(children)
		for _, child := range children {
			actionPath := zkActionPath + "/" + child
			if _, err := strconv.ParseUint(child, 10, 64); err != nil {
				// This is handy if you want to restart a stuck queue.
				// FIXME(msolomon) could listen on the queue node for a change
				// generated by a "touch", but listening on two things is a bit
				// more complex.
				log.Warningf("remove invalid event from action queue: %v", child)
				zkts.zconn.Delete(actionPath, -1)
				continue
			}

			data, _, err := zkts.zconn.Get(actionPath)
			if err != nil {
				log.Errorf("cannot read action %v from zk: %v", actionPath, err)
				break
			}

			if err := dispatchAction(actionPath, data); err != nil {
				break
			}
		}
	}
	return watch, nil
}

func (zkts *Server) ActionEventLoop(tabletAlias topo.TabletAlias, dispatchAction func(actionPath, data string) error, done chan struct{}) {
	for {
		// Process any pending actions when we startup, before
		// we start listening for events.
		watch, err := zkts.handleActionQueue(tabletAlias, dispatchAction)
		if err != nil {
			log.Warningf("failed to set the watch on action queue, will try again in 5 seconds: %v", err)
			time.Sleep(5 * time.Second)
			continue
		}

		select {
		case event := <-watch:
			if !event.Ok() {
				// NOTE(msolomon) The zk meta conn will
				// reconnect automatically, or error out.
				// At this point, there isn't much to do.
				log.Warningf("zookeeper not OK: %v, will try again in 5 seconds", event)
				time.Sleep(5 * time.Second)
			}
			// Otherwise, just handle the queue above.
		case <-done:
			return
		}
	}
}

// actionPathToTabletAlias parses an actionPath back
// zkActionPath is /zk/<cell>/vt/tablets/<uid>/action/<number>
func actionPathToTabletAlias(actionPath string) (topo.TabletAlias, error) {
	pathParts := strings.Split(actionPath, "/")
	if len(pathParts) != 8 || pathParts[0] != "" || pathParts[1] != "zk" || pathParts[3] != "vt" || pathParts[4] != "tablets" || pathParts[6] != "action" {
		return topo.TabletAlias{}, fmt.Errorf("invalid action path: %v", actionPath)
	}
	return topo.ParseTabletAliasString(pathParts[2] + "-" + pathParts[5])
}

func (zkts *Server) ReadTabletActionPath(actionPath string) (topo.TabletAlias, string, int64, error) {
	tabletAlias, err := actionPathToTabletAlias(actionPath)
	if err != nil {
		return topo.TabletAlias{}, "", 0, err
	}

	data, stat, err := zkts.zconn.Get(actionPath)
	if err != nil {
		return topo.TabletAlias{}, "", 0, err
	}

	return tabletAlias, data, int64(stat.Version()), nil
}

func (zkts *Server) UpdateTabletAction(actionPath, data string, version int64) error {
	_, err := zkts.zconn.Set(actionPath, data, int(version))
	if err != nil {
		if zookeeper.IsError(err, zookeeper.ZBADVERSION) {
			err = topo.ErrBadVersion
		}
		return err
	}
	return nil
}

// StoreTabletActionResponse stores the data both in action and actionlog
func (zkts *Server) StoreTabletActionResponse(actionPath, data string) error {
	_, err := zkts.zconn.Set(actionPath, data, -1)
	if err != nil {
		return err
	}

	actionLogPath := strings.Replace(actionPath, "/action/", "/actionlog/", 1)
	_, err = zk.CreateRecursive(zkts.zconn, actionLogPath, data, 0, zookeeper.WorldACL(zookeeper.PERM_ALL))
	return err
}

func (zkts *Server) UnblockTabletAction(actionPath string) error {
	return zkts.zconn.Delete(actionPath, -1)
}
<|endoftext|>