<commit_before>package main

import (
	//"encoding/json"
	"fmt"
	//"io/ioutil"
	//"log"
	//"os"
	//"path/filepath"
	//"time"

	"github.com/mitchellh/packer/common"
	"github.com/mitchellh/packer/helper/config"
	"github.com/mitchellh/packer/packer"
	"github.com/mitchellh/packer/packer/plugin"
	"github.com/mitchellh/packer/template/interpolate"
)

type Config struct {
	common.PackerConfig `mapstructure:",squash"`

	OutputPath string `mapstructure:"output"`
	StripPath  bool   `mapstructure:"strip_path"`
	ctx        interpolate.Context
}

type PostProcessor struct {
	config Config
}

type ManifestFile struct {
	Builds      []Artifact `json:"builds"`
	LastRunUUID string     `json:"last_run_uuid"`
}

func (p *PostProcessor) Configure(raws ...interface{}) error {
	err := config.Decode(&p.config, &config.DecodeOpts{
		Interpolate:        true,
		InterpolateContext: &p.config.ctx,
		InterpolateFilter: &interpolate.RenderFilter{
			Exclude: []string{},
		},
	}, raws...)
	if err != nil {
		return err
	}

	if p.config.OutputPath == "" {
		p.config.OutputPath = "packer-manifest.json"
	}

	if err = interpolate.Validate(p.config.OutputPath, &p.config.ctx); err != nil {
		return fmt.Errorf("Error parsing target template: %s", err)
	}

	return nil
}

func (p *PostProcessor) PostProcess(ui packer.Ui, source packer.Artifact) (packer.Artifact, bool, error) {
	//artifact := &Artifact{}
	//
	//var err error
	//var fi os.FileInfo
	//
	//// Create the current artifact.
	//for _, name := range source.Files() {
	//	af := ArtifactFile{}
	//	if fi, err = os.Stat(name); err == nil {
	//		af.Size = fi.Size()
	//	}
	//	if p.config.StripPath {
	//		af.Name = filepath.Base(name)
	//	} else {
	//		af.Name = name
	//	}
	//	artifact.ArtifactFiles = append(artifact.ArtifactFiles, af)
	//}
	//artifact.ArtifactId = source.Id()
	//artifact.BuilderType = p.config.PackerBuilderType
	//artifact.BuildName = p.config.PackerBuildName
	//artifact.BuildTime = time.Now().Unix()
	//// Since each post-processor runs in a different process we need a way to
	//// coordinate between various post-processors in a single packer run. We do
	//// this by setting a UUID per run and tracking this in the manifest file.
	//// When we detect that the UUID in the file is the same, we know that we are
	//// part of the same run and we simply add our data to the list. If the UUID
	//// is different we will check the -force flag and decide whether to truncate
	//// the file before we proceed.
	//artifact.PackerRunUUID = os.Getenv("PACKER_RUN_UUID")
	//
	//// Create a lock file with exclusive access. If this fails we will retry
	//// after a delay.
	//lockFilename := p.config.OutputPath + ".lock"
	//for i := 0; i < 3; i++ {
	//	// The file should not be locked for very long so we'll keep this short.
	//	time.Sleep((time.Duration(i) * 200 * time.Millisecond))
	//	_, err = os.OpenFile(lockFilename, os.O_RDWR|os.O_CREATE|os.O_EXCL, 0600)
	//	if err == nil {
	//		break
	//	}
	//	log.Printf("Error locking manifest file for reading and writing. Will sleep and retry. %s", err)
	//}
	//defer os.Remove(lockFilename)
	//
	//// Read the current manifest file from disk
	//contents := []byte{}
	//if contents, err = ioutil.ReadFile(p.config.OutputPath); err != nil && !os.IsNotExist(err) {
	//	return source, true, fmt.Errorf("Unable to open %s for reading: %s", p.config.OutputPath, err)
	//}
	//
	//// Parse the manifest file JSON, if we have one
	//manifestFile := &ManifestFile{}
	//if len(contents) > 0 {
	//	if err = json.Unmarshal(contents, manifestFile); err != nil {
	//		return source, true, fmt.Errorf("Unable to parse content from %s: %s", p.config.OutputPath, err)
	//	}
	//}
	//
	//// If -force is set and we are not on same run, truncate the file. Otherwise
	//// we will continue to add new builds to the existing manifest file.
	//if p.config.PackerForce && os.Getenv("PACKER_RUN_UUID") != manifestFile.LastRunUUID {
	//	manifestFile = &ManifestFile{}
	//}
	//
	//// Add the current artifact to the manifest file
	//manifestFile.Builds = append(manifestFile.Builds, *artifact)
	//manifestFile.LastRunUUID = os.Getenv("PACKER_RUN_UUID")
	//
	//// Write JSON to disk
	//if out, err := json.MarshalIndent(manifestFile, "", "  "); err == nil {
	//	if err = ioutil.WriteFile(p.config.OutputPath, out, 0664); err != nil {
	//		return source, true, fmt.Errorf("Unable to write %s: %s", p.config.OutputPath, err)
	//	}
	//} else {
	//	return source, true, fmt.Errorf("Unable to marshal JSON %s", err)
	//}

	fmt.Printf("ArtifactId : %v\n", source.Id())

	return source, true, nil
}

func main() {
	plugin.ServePostProcessor(new(PostProcessor))
}
<commit_msg>Backup, doesn't work; added main() function for post-processor, cannot install packer/packer/plugin<commit_after>package main

import (
	"fmt"

	"github.com/mitchellh/packer/common"
	"github.com/mitchellh/packer/helper/config"
	"github.com/mitchellh/packer/packer"
	// The library I cannot properly install
	"github.com/mitchellh/packer/packer/plugin"
	"github.com/mitchellh/packer/template/interpolate"
)

type Config struct {
	common.PackerConfig `mapstructure:",squash"`

	OutputPath string `mapstructure:"output"`
	StripPath  bool   `mapstructure:"strip_path"`
	ctx        interpolate.Context
}

type PostProcessor struct {
	config Config
}

type ManifestFile struct {
	Builds      []Artifact `json:"builds"`
	LastRunUUID string     `json:"last_run_uuid"`
}

func (p *PostProcessor) Configure(raws ...interface{}) error {
	err := config.Decode(&p.config, &config.DecodeOpts{
		Interpolate:        true,
		InterpolateContext: &p.config.ctx,
		InterpolateFilter: &interpolate.RenderFilter{
			Exclude: []string{},
		},
	}, raws...)
	if err != nil {
		return err
	}

	if p.config.OutputPath == "" {
		p.config.OutputPath = "packer-manifest.json"
	}

	if err = interpolate.Validate(p.config.OutputPath, &p.config.ctx); err != nil {
		return fmt.Errorf("Error parsing target template: %s", err)
	}

	return nil
}

func (p *PostProcessor) PostProcess(ui packer.Ui, source packer.Artifact) (packer.Artifact, bool, error) {
	fmt.Printf("ArtifactId : %v\n", source.Id())
	return source, true, nil
}

func main() {
	plugin.ServePostProcessor(new(PostProcessor))
}
<|endoftext|>


<commit_before>package targets

import (
	"github.com/openshift/installer/pkg/asset"
	"github.com/openshift/installer/pkg/asset/cluster"
	"github.com/openshift/installer/pkg/asset/ignition/bootstrap"
	"github.com/openshift/installer/pkg/asset/ignition/machine"
	"github.com/openshift/installer/pkg/asset/installconfig"
	"github.com/openshift/installer/pkg/asset/kubeconfig"
	"github.com/openshift/installer/pkg/asset/manifests"
	"github.com/openshift/installer/pkg/asset/templates/content/bootkube"
	"github.com/openshift/installer/pkg/asset/templates/content/openshift"
	"github.com/openshift/installer/pkg/asset/tls"
)

var (
	// InstallConfig are the install-config targeted assets.
	InstallConfig = []asset.WritableAsset{
		&installconfig.InstallConfig{},
	}

	// Manifests are the manifests targeted assets.
	Manifests = []asset.WritableAsset{
		&manifests.Manifests{},
		&manifests.Openshift{},
	}

	// ManifestTemplates are the manifest-templates targeted assets.
	ManifestTemplates = []asset.WritableAsset{
		&bootkube.KubeCloudConfig{},
		&bootkube.MachineConfigServerTLSSecret{},
		&bootkube.OpenshiftServiceCertSignerSecret{},
		&bootkube.Pull{},
		&bootkube.CVOOverrides{},
		&bootkube.HostEtcdServiceEndpointsKubeSystem{},
		&bootkube.KubeSystemConfigmapEtcdServingCA{},
		&bootkube.KubeSystemConfigmapRootCA{},
		&bootkube.KubeSystemSecretEtcdClient{},
		&bootkube.OpenshiftMachineConfigOperator{},
		&bootkube.OpenshiftClusterAPINamespace{},
		&bootkube.OpenshiftServiceCertSignerNamespace{},
		&bootkube.EtcdServiceKubeSystem{},
		&bootkube.HostEtcdServiceKubeSystem{},
		&openshift.BindingDiscovery{},
		&openshift.CloudCredsSecret{},
		&openshift.KubeadminPasswordSecret{},
		&openshift.RoleCloudCredsSecretReader{},
	}

	// IgnitionConfigs are the ignition-configs targeted assets.
	IgnitionConfigs = []asset.WritableAsset{
		&kubeconfig.Admin{},
		&machine.Master{},
		&machine.Worker{},
		&bootstrap.Bootstrap{},
		&cluster.Metadata{},
	}

	// Cluster are the cluster targeted assets.
	Cluster = []asset.WritableAsset{
		&cluster.TerraformVariables{},
		&kubeconfig.Admin{},
		&tls.JournalCertKey{},
		&cluster.Cluster{},
		&cluster.Metadata{},
	}
)
<commit_msg>pkg/asset/targets: Render the metadata asset before cluster<commit_after>package targets

import (
	"github.com/openshift/installer/pkg/asset"
	"github.com/openshift/installer/pkg/asset/cluster"
	"github.com/openshift/installer/pkg/asset/ignition/bootstrap"
	"github.com/openshift/installer/pkg/asset/ignition/machine"
	"github.com/openshift/installer/pkg/asset/installconfig"
	"github.com/openshift/installer/pkg/asset/kubeconfig"
	"github.com/openshift/installer/pkg/asset/manifests"
	"github.com/openshift/installer/pkg/asset/templates/content/bootkube"
	"github.com/openshift/installer/pkg/asset/templates/content/openshift"
	"github.com/openshift/installer/pkg/asset/tls"
)

var (
	// InstallConfig are the install-config targeted assets.
	InstallConfig = []asset.WritableAsset{
		&installconfig.InstallConfig{},
	}

	// Manifests are the manifests targeted assets.
	Manifests = []asset.WritableAsset{
		&manifests.Manifests{},
		&manifests.Openshift{},
	}

	// ManifestTemplates are the manifest-templates targeted assets.
	ManifestTemplates = []asset.WritableAsset{
		&bootkube.KubeCloudConfig{},
		&bootkube.MachineConfigServerTLSSecret{},
		&bootkube.OpenshiftServiceCertSignerSecret{},
		&bootkube.Pull{},
		&bootkube.CVOOverrides{},
		&bootkube.HostEtcdServiceEndpointsKubeSystem{},
		&bootkube.KubeSystemConfigmapEtcdServingCA{},
		&bootkube.KubeSystemConfigmapRootCA{},
		&bootkube.KubeSystemSecretEtcdClient{},
		&bootkube.OpenshiftMachineConfigOperator{},
		&bootkube.OpenshiftClusterAPINamespace{},
		&bootkube.OpenshiftServiceCertSignerNamespace{},
		&bootkube.EtcdServiceKubeSystem{},
		&bootkube.HostEtcdServiceKubeSystem{},
		&openshift.BindingDiscovery{},
		&openshift.CloudCredsSecret{},
		&openshift.KubeadminPasswordSecret{},
		&openshift.RoleCloudCredsSecretReader{},
	}

	// IgnitionConfigs are the ignition-configs targeted assets.
	IgnitionConfigs = []asset.WritableAsset{
		&kubeconfig.Admin{},
		&machine.Master{},
		&machine.Worker{},
		&bootstrap.Bootstrap{},
		&cluster.Metadata{},
	}

	// Cluster are the cluster targeted assets.
	Cluster = []asset.WritableAsset{
		&cluster.TerraformVariables{},
		&kubeconfig.Admin{},
		&tls.JournalCertKey{},
		&cluster.Metadata{},
		&cluster.Cluster{},
	}
)
<|endoftext|>


<commit_before>package controller<commit_msg>Added basic db connection logic<commit_after>package controller

import (
	"github.com/jinzhu/gorm"
	_ "github.com/jinzhu/gorm/dialects/sqlite" // sqlite driver
)

func connectDB() {
	db, err := gorm.Open("../../../../admin/database", "database.sqlite")
	if err != nil {
		panic("Failed to connect to database")
	}
	defer db.Close()
}
<|endoftext|>


<commit_before>/*
Copyright 2020 The cert-manager Authors.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package controller

import (
	"context"
	"fmt"
	"strings"
	"sync"
	"time"

	"k8s.io/apimachinery/pkg/util/wait"
	genericregistry "k8s.io/apiserver/pkg/registry/generic/registry"
	"k8s.io/client-go/tools/cache"
	"k8s.io/client-go/util/workqueue"

	logf "github.com/jetstack/cert-manager/pkg/logs"
	"github.com/jetstack/cert-manager/pkg/metrics"
)

type runFunc func(context.Context)

type runDurationFunc struct {
	fn       runFunc
	duration time.Duration
}

type queueingController interface {
	Register(*Context) (workqueue.RateLimitingInterface, []cache.InformerSynced, error)
	ProcessItem(ctx context.Context, key string) error
}

func NewController(
	ctx context.Context,
	name string,
	metrics *metrics.Metrics,
	syncFunc func(ctx context.Context, key string) error,
	mustSync []cache.InformerSynced,
	runDurationFuncs []runDurationFunc,
	queue workqueue.RateLimitingInterface,
) Interface {
	return &controller{
		ctx:              ctx,
		name:             name,
		metrics:          metrics,
		syncHandler:      syncFunc,
		mustSync:         mustSync,
		runDurationFuncs: runDurationFuncs,
		queue:            queue,
	}
}

type controller struct {
	// ctx is the root golang context for the controller
	ctx context.Context

	// name is the name for this controller
	name string

	// the function that should be called when an item is popped
	// off the workqueue
	syncHandler func(ctx context.Context, key string) error

	// mustSync is a slice of informers that must have synced before
	// this controller can start
	mustSync []cache.InformerSynced

	// a set of functions that will be called just after controller initialisation, once.
	runFirstFuncs []runFunc

	// a set of functions that should be called every duration.
	runDurationFuncs []runDurationFunc

	// queue is a reference to the queue used to enqueue resources
	// to be processed
	queue workqueue.RateLimitingInterface

	// metrics is used to expose Prometheus, shared by all controllers
	metrics *metrics.Metrics
}

// Run starts the controller loop
func (c *controller) Run(workers int, stopCh <-chan struct{}) error {
	ctx, cancel := context.WithCancel(c.ctx)
	defer cancel()
	log := logf.FromContext(ctx)

	log.V(logf.DebugLevel).Info("starting control loop")
	// wait for all the informer caches we depend on are synced
	if !cache.WaitForCacheSync(stopCh, c.mustSync...) {
		return fmt.Errorf("error waiting for informer caches to sync")
	}

	var wg sync.WaitGroup
	for i := 0; i < workers; i++ {
		wg.Add(1)
		// TODO (@munnerz): make time.Second duration configurable
		go wait.Until(func() {
			defer wg.Done()
			c.worker(ctx)
		}, time.Second, stopCh)
	}

	for _, f := range c.runFirstFuncs {
		f(ctx)
	}

	for _, f := range c.runDurationFuncs {
		go wait.Until(func() { f.fn(ctx) }, f.duration, stopCh)
	}

	<-stopCh
	log.V(logf.InfoLevel).Info("shutting down queue as workqueue signaled shutdown")
	c.queue.ShutDown()
	log.V(logf.DebugLevel).Info("waiting for workers to exit...")
	wg.Wait()
	log.V(logf.DebugLevel).Info("workers exited")
	return nil
}

func (b *controller) worker(ctx context.Context) {
	log := logf.FromContext(b.ctx)

	log.V(logf.DebugLevel).Info("starting worker")
	for {
		obj, shutdown := b.queue.Get()
		if shutdown {
			break
		}

		var key string
		// use an inlined function so we can use defer
		func() {
			defer b.queue.Done(obj)
			var ok bool
			if key, ok = obj.(string); !ok {
				return
			}
			log := log.WithValues("key", key)
			log.V(logf.DebugLevel).Info("syncing item")

			// Increase sync count for this controller
			b.metrics.IncrementSyncCallCount(b.name)

			err := b.syncHandler(ctx, key)
			if err != nil {
				if strings.Contains(err.Error(), genericregistry.OptimisticLockErrorMsg) {
					log.V(logf.DebugLevel).Error(err, "re-queuing item due to optimistic locking")
				} else {
					log.Error(err, "re-queuing item due to error processing")
				}

				b.queue.AddRateLimited(obj)
				return
			}
			log.V(logf.DebugLevel).Info("finished processing work item")
			b.queue.Forget(obj)
		}()
	}
	log.V(logf.DebugLevel).Info("exiting worker loop")
}
<commit_msg>Change optimistic logging to be Info, rather than debug<commit_after>/*
Copyright 2020 The cert-manager Authors.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package controller

import (
	"context"
	"fmt"
	"strings"
	"sync"
	"time"

	"k8s.io/apimachinery/pkg/util/wait"
	genericregistry "k8s.io/apiserver/pkg/registry/generic/registry"
	"k8s.io/client-go/tools/cache"
	"k8s.io/client-go/util/workqueue"

	logf "github.com/jetstack/cert-manager/pkg/logs"
	"github.com/jetstack/cert-manager/pkg/metrics"
)

type runFunc func(context.Context)

type runDurationFunc struct {
	fn       runFunc
	duration time.Duration
}

type queueingController interface {
	Register(*Context) (workqueue.RateLimitingInterface, []cache.InformerSynced, error)
	ProcessItem(ctx context.Context, key string) error
}

func NewController(
	ctx context.Context,
	name string,
	metrics *metrics.Metrics,
	syncFunc func(ctx context.Context, key string) error,
	mustSync []cache.InformerSynced,
	runDurationFuncs []runDurationFunc,
	queue workqueue.RateLimitingInterface,
) Interface {
	return &controller{
		ctx:              ctx,
		name:             name,
		metrics:          metrics,
		syncHandler:      syncFunc,
		mustSync:         mustSync,
		runDurationFuncs: runDurationFuncs,
		queue:            queue,
	}
}

type controller struct {
	// ctx is the root golang context for the controller
	ctx context.Context

	// name is the name for this controller
	name string

	// the function that should be called when an item is popped
	// off the workqueue
	syncHandler func(ctx context.Context, key string) error

	// mustSync is a slice of informers that must have synced before
	// this controller can start
	mustSync []cache.InformerSynced

	// a set of functions that will be called just after controller initialisation, once.
	runFirstFuncs []runFunc

	// a set of functions that should be called every duration.
	runDurationFuncs []runDurationFunc

	// queue is a reference to the queue used to enqueue resources
	// to be processed
	queue workqueue.RateLimitingInterface

	// metrics is used to expose Prometheus, shared by all controllers
	metrics *metrics.Metrics
}

// Run starts the controller loop
func (c *controller) Run(workers int, stopCh <-chan struct{}) error {
	ctx, cancel := context.WithCancel(c.ctx)
	defer cancel()
	log := logf.FromContext(ctx)

	log.V(logf.DebugLevel).Info("starting control loop")
	// wait for all the informer caches we depend on are synced
	if !cache.WaitForCacheSync(stopCh, c.mustSync...) {
		return fmt.Errorf("error waiting for informer caches to sync")
	}

	var wg sync.WaitGroup
	for i := 0; i < workers; i++ {
		wg.Add(1)
		// TODO (@munnerz): make time.Second duration configurable
		go wait.Until(func() {
			defer wg.Done()
			c.worker(ctx)
		}, time.Second, stopCh)
	}

	for _, f := range c.runFirstFuncs {
		f(ctx)
	}

	for _, f := range c.runDurationFuncs {
		go wait.Until(func() { f.fn(ctx) }, f.duration, stopCh)
	}

	<-stopCh
	log.V(logf.InfoLevel).Info("shutting down queue as workqueue signaled shutdown")
	c.queue.ShutDown()
	log.V(logf.DebugLevel).Info("waiting for workers to exit...")
	wg.Wait()
	log.V(logf.DebugLevel).Info("workers exited")
	return nil
}

func (b *controller) worker(ctx context.Context) {
	log := logf.FromContext(b.ctx)

	log.V(logf.DebugLevel).Info("starting worker")
	for {
		obj, shutdown := b.queue.Get()
		if shutdown {
			break
		}

		var key string
		// use an inlined function so we can use defer
		func() {
			defer b.queue.Done(obj)
			var ok bool
			if key, ok = obj.(string); !ok {
				return
			}
			log := log.WithValues("key", key)
			log.V(logf.DebugLevel).Info("syncing item")

			// Increase sync count for this controller
			b.metrics.IncrementSyncCallCount(b.name)

			err := b.syncHandler(ctx, key)
			if err != nil {
				if strings.Contains(err.Error(), genericregistry.OptimisticLockErrorMsg) {
					log.Info("re-queuing item due to optimistic locking on resource", "error", err.Error())
				} else {
					log.Error(err, "re-queuing item due to error processing")
				}

				b.queue.AddRateLimited(obj)
				return
			}
			log.V(logf.DebugLevel).Info("finished processing work item")
			b.queue.Forget(obj)
		}()
	}
	log.V(logf.DebugLevel).Info("exiting worker loop")
}
<|endoftext|>


<commit_before>/*
 * Minimalist Object Storage, (C) 2015 Minio, Inc.
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 *     http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */

package memory

import (
	"bufio"
	"bytes"
	"io"
	"sort"
	"strings"
	"sync"
	"time"

	"crypto/md5"
	"encoding/hex"

	"github.com/minio-io/minio/pkg/drivers"
)

// memoryDriver - local variables
type memoryDriver struct {
	bucketdata map[string]storedBucket
	objectdata map[string]storedObject
	lock       *sync.RWMutex
}

type storedBucket struct {
	metadata drivers.BucketMetadata
	//	owner    string // TODO
	//	id       string // TODO
}

type storedObject struct {
	metadata drivers.ObjectMetadata
	data     []byte
}

// Start memory object server
func Start() (chan<- string, <-chan error, drivers.Driver) {
	ctrlChannel := make(chan string)
	errorChannel := make(chan error)

	memory := new(memoryDriver)
	memory.bucketdata = make(map[string]storedBucket)
	memory.objectdata = make(map[string]storedObject)
	memory.lock = new(sync.RWMutex)

	go start(ctrlChannel, errorChannel)
	return ctrlChannel, errorChannel, memory
}

func start(ctrlChannel <-chan string, errorChannel chan<- error) {
	close(errorChannel)
}

// GetObject - GET object from memory buffer
func (memory memoryDriver) GetObject(w io.Writer, bucket string, object string) (int64, error) {
	// get object
	key := object
	if val, ok := memory.objectdata[key]; ok {
		objectBuffer := bytes.NewBuffer(val.data)
		written, err := io.Copy(w, objectBuffer)
		return written, err
	}
	return 0, drivers.ObjectNotFound{Bucket: bucket, Object: object}
}

// GetPartialObject - GET object from memory buffer range
func (memory memoryDriver) GetPartialObject(w io.Writer, bucket, object string, start, length int64) (int64, error) {
	var sourceBuffer bytes.Buffer
	if _, err := memory.GetObject(&sourceBuffer, bucket, object); err != nil {
		return 0, err
	}
	var nilBuffer bytes.Buffer
	if _, err := io.CopyN(&nilBuffer, &sourceBuffer, start); err != nil {
		return 0, err
	}
	return io.CopyN(w, &sourceBuffer, length)
}

// CreateBucketPolicy - Not implemented
func (memory memoryDriver) CreateBucketPolicy(bucket string, policy drivers.BucketPolicy) error {
	return drivers.APINotImplemented{API: "PutBucketPolicy"}
}

// GetBucketPolicy - Not implemented
func (memory memoryDriver) GetBucketPolicy(bucket string) (drivers.BucketPolicy, error) {
	return drivers.BucketPolicy{}, drivers.APINotImplemented{API: "GetBucketPolicy"}
}

// CreateObject - PUT object to memory buffer
func (memory memoryDriver) CreateObject(bucket, key, contentType, md5sum string, data io.Reader) error {
	memory.lock.Lock()
	defer memory.lock.Unlock()

	if _, ok := memory.bucketdata[bucket]; ok == false {
		return drivers.BucketNotFound{Bucket: bucket}
	}

	if _, ok := memory.objectdata[key]; ok == true {
		return drivers.ObjectExists{Bucket: bucket, Object: key}
	}

	if contentType == "" {
		contentType = "application/octet-stream"
	}

	contentType = strings.TrimSpace(contentType)

	var bytesBuffer bytes.Buffer
	var newObject = storedObject{}
	if _, ok := io.Copy(&bytesBuffer, data); ok == nil {
		size := bytesBuffer.Len()
		md5SumBytes := md5.Sum(bytesBuffer.Bytes())
		md5Sum := hex.EncodeToString(md5SumBytes[:])
		newObject.metadata = drivers.ObjectMetadata{
			Bucket: bucket,
			Key:    key,

			ContentType: contentType,
			Created:     time.Now(),
			Md5:         md5Sum,
			Size:        int64(size),
		}
		newObject.data = bytesBuffer.Bytes()
	}
	memory.objectdata[key] = newObject
	return nil
}

// CreateBucket - create bucket in memory
func (memory memoryDriver) CreateBucket(bucketName string) error {
	memory.lock.Lock()
	defer memory.lock.Unlock()
	if !drivers.IsValidBucket(bucketName) {
		return drivers.BucketNameInvalid{Bucket: bucketName}
	}

	if _, ok := memory.bucketdata[bucketName]; ok == true {
		return drivers.BucketExists{Bucket: bucketName}
	}

	var newBucket = storedBucket{}
	newBucket.metadata = drivers.BucketMetadata{}
	newBucket.metadata.Name = bucketName
	newBucket.metadata.Created = time.Now()
	memory.bucketdata[bucketName] = newBucket

	return nil
}

func delimiter(object, delimiter string) string {
	readBuffer := bytes.NewBufferString(object)
	reader := bufio.NewReader(readBuffer)
	stringReader := strings.NewReader(delimiter)
	delimited, _ := stringReader.ReadByte()
	delimitedStr, _ := reader.ReadString(delimited)
	return delimitedStr
}

func appendUniq(slice []string, i string) []string {
	for _, ele := range slice {
		if ele == i {
			return slice
		}
	}
	return append(slice, i)
}

func (memory memoryDriver) filterDelimiterPrefix(keys []string, key, delimitedName string, resources drivers.BucketResourcesMetadata) (drivers.BucketResourcesMetadata, []string) {
	switch true {
	case key == resources.Prefix:
		keys = appendUniq(keys, key)
		// DelimitedName - requires resources.Prefix as it was trimmed off earlier in the flow
	case key == resources.Prefix+delimitedName:
		keys = appendUniq(keys, key)
	case delimitedName != "":
		if delimitedName == resources.Delimiter {
			resources.CommonPrefixes = appendUniq(resources.CommonPrefixes, resources.Prefix+delimitedName)
		} else {
			resources.CommonPrefixes = appendUniq(resources.CommonPrefixes, delimitedName)
		}
	}
	return resources, keys
}

// ListObjects - list objects from memory
func (memory memoryDriver) ListObjects(bucket string, resources drivers.BucketResourcesMetadata) ([]drivers.ObjectMetadata, drivers.BucketResourcesMetadata, error) {
	if _, ok := memory.bucketdata[bucket]; ok == false {
		return []drivers.ObjectMetadata{}, drivers.BucketResourcesMetadata{IsTruncated: false}, drivers.BucketNotFound{Bucket: bucket}
	}
	var results []drivers.ObjectMetadata
	var keys []string
	for key := range memory.objectdata {
		switch true {
		// Prefix absent, delimit object key based on delimiter
		case resources.IsDelimiterSet():
			delimitedName := delimiter(key, resources.Delimiter)
			switch true {
			case delimitedName == "" || delimitedName == key:
				keys = appendUniq(keys, key)
			case delimitedName != "":
				resources.CommonPrefixes = appendUniq(resources.CommonPrefixes, delimitedName)
			}
		// Prefix present, delimit object key with prefix key based on delimiter
		case resources.IsDelimiterPrefixSet():
			if strings.HasPrefix(key, resources.Prefix) {
				trimmedName := strings.TrimPrefix(key, resources.Prefix)
				delimitedName := delimiter(trimmedName, resources.Delimiter)
				resources, keys = memory.filterDelimiterPrefix(keys, key, delimitedName, resources)
			}
		// Prefix present, nothing to delimit
		case resources.IsPrefixSet():
			keys = appendUniq(keys, key)
		// Prefix and delimiter absent
		case resources.IsDefault():
			keys = appendUniq(keys, key)
		}
	}
	sort.Strings(keys)
	for _, key := range keys {
		if len(results) == resources.Maxkeys {
			return results, drivers.BucketResourcesMetadata{IsTruncated: true}, nil
		}
		object := memory.objectdata[key]
		if bucket == object.metadata.Bucket {
			results = append(results, object.metadata)
		}
	}
	return results, resources, nil
}

// ByBucketName is a type for sorting bucket metadata by bucket name
type ByBucketName []drivers.BucketMetadata

// Len of bucket name
func (b ByBucketName) Len() int { return len(b) }

// Swap bucket i, j
func (b ByBucketName) Swap(i, j int) { b[i], b[j] = b[j], b[i] }

// Less
func (b ByBucketName) Less(i, j int) bool { return b[i].Name < b[j].Name }

// ListBuckets - List buckets from memory
func (memory memoryDriver) ListBuckets() ([]drivers.BucketMetadata, error) {
	var results []drivers.BucketMetadata
	for _, bucket := range memory.bucketdata {
		results = append(results, bucket.metadata)
	}
	sort.Sort(ByBucketName(results))
	return results, nil
}

// GetObjectMetadata - get object metadata from memory
func (memory memoryDriver) GetObjectMetadata(bucket, key, prefix string) (drivers.ObjectMetadata, error) {
	if object, ok := memory.objectdata[key]; ok == true {
		return object.metadata, nil
	}
	return drivers.ObjectMetadata{}, drivers.ObjectNotFound{Bucket: bucket, Object: key}
}
<commit_msg>Minor cleanup of memory get partial object<commit_after>/*
 * Minimalist Object Storage, (C) 2015 Minio, Inc.
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 *     http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */

package memory

import (
	"bufio"
	"bytes"
	"io"
	"sort"
	"strings"
	"sync"
	"time"

	"crypto/md5"
	"encoding/hex"

	"github.com/minio-io/minio/pkg/drivers"
	"io/ioutil"
)

// memoryDriver - local variables
type memoryDriver struct {
	bucketdata map[string]storedBucket
	objectdata map[string]storedObject
	lock       *sync.RWMutex
}

type storedBucket struct {
	metadata drivers.BucketMetadata
	//	owner    string // TODO
	//	id       string // TODO
}

type storedObject struct {
	metadata drivers.ObjectMetadata
	data     []byte
}

// Start memory object server
func Start() (chan<- string, <-chan error, drivers.Driver) {
	ctrlChannel := make(chan string)
	errorChannel := make(chan error)

	memory := new(memoryDriver)
	memory.bucketdata = make(map[string]storedBucket)
	memory.objectdata = make(map[string]storedObject)
	memory.lock = new(sync.RWMutex)

	go start(ctrlChannel, errorChannel)
	return ctrlChannel, errorChannel, memory
}

func start(ctrlChannel <-chan string, errorChannel chan<- error) {
	close(errorChannel)
}

// GetObject - GET object from memory buffer
func (memory memoryDriver) GetObject(w io.Writer, bucket string, object string) (int64, error) {
	// get object
	key := object
	if val, ok := memory.objectdata[key]; ok {
		objectBuffer := bytes.NewBuffer(val.data)
		written, err := io.Copy(w, objectBuffer)
		return written, err
	}
	return 0, drivers.ObjectNotFound{Bucket: bucket, Object: object}
}

// GetPartialObject - GET object from memory buffer range
func (memory memoryDriver) GetPartialObject(w io.Writer, bucket, object string, start, length int64) (int64, error) {
	var sourceBuffer bytes.Buffer
	if _, err := memory.GetObject(&sourceBuffer, bucket, object); err != nil {
		return 0, err
	}
	if _, err := io.CopyN(ioutil.Discard, &sourceBuffer, start); err != nil {
		return 0, err
	}
	return io.CopyN(w, &sourceBuffer, length)
}

// CreateBucketPolicy - Not implemented
func (memory memoryDriver) CreateBucketPolicy(bucket string, policy drivers.BucketPolicy) error {
	return drivers.APINotImplemented{API: "PutBucketPolicy"}
}

// GetBucketPolicy - Not implemented
func (memory memoryDriver) GetBucketPolicy(bucket string) (drivers.BucketPolicy, error) {
	return drivers.BucketPolicy{}, drivers.APINotImplemented{API: "GetBucketPolicy"}
}

// CreateObject - PUT object to memory buffer
func (memory memoryDriver) CreateObject(bucket, key, contentType, md5sum string, data io.Reader) error {
	memory.lock.Lock()
	defer memory.lock.Unlock()

	if _, ok := memory.bucketdata[bucket]; ok == false {
		return drivers.BucketNotFound{Bucket: bucket}
	}

	if _, ok := memory.objectdata[key]; ok == true {
		return drivers.ObjectExists{Bucket: bucket, Object: key}
	}

	if contentType == "" {
		contentType = "application/octet-stream"
	}

	contentType = strings.TrimSpace(contentType)

	var bytesBuffer bytes.Buffer
	var newObject = storedObject{}
	if _, ok := io.Copy(&bytesBuffer, data); ok == nil {
		size := bytesBuffer.Len()
		md5SumBytes := md5.Sum(bytesBuffer.Bytes())
		md5Sum := hex.EncodeToString(md5SumBytes[:])
		newObject.metadata = drivers.ObjectMetadata{
			Bucket: bucket,
			Key:    key,

			ContentType: contentType,
			Created:     time.Now(),
			Md5:         md5Sum,
			Size:        int64(size),
		}
		newObject.data = bytesBuffer.Bytes()
	}
	memory.objectdata[key] = newObject
	return nil
}

// CreateBucket - create bucket in memory
func (memory memoryDriver) CreateBucket(bucketName string) error {
	memory.lock.Lock()
	defer memory.lock.Unlock()
	if !drivers.IsValidBucket(bucketName) {
		return drivers.BucketNameInvalid{Bucket: bucketName}
	}

	if _, ok := memory.bucketdata[bucketName]; ok == true {
		return drivers.BucketExists{Bucket: bucketName}
	}

	var newBucket = storedBucket{}
	newBucket.metadata = drivers.BucketMetadata{}
	newBucket.metadata.Name = bucketName
	newBucket.metadata.Created = time.Now()
	memory.bucketdata[bucketName] = newBucket

	return nil
}

func delimiter(object, delimiter string) string {
	readBuffer := bytes.NewBufferString(object)
	reader := bufio.NewReader(readBuffer)
	stringReader := strings.NewReader(delimiter)
	delimited, _ := stringReader.ReadByte()
	delimitedStr, _ := reader.ReadString(delimited)
	return delimitedStr
}

func appendUniq(slice []string, i string) []string {
	for _, ele := range slice {
		if ele == i {
			return slice
		}
	}
	return append(slice, i)
}

func (memory memoryDriver) filterDelimiterPrefix(keys []string, key, delimitedName string, resources drivers.BucketResourcesMetadata) (drivers.BucketResourcesMetadata, []string) {
	switch true {
	case key == resources.Prefix:
		keys = appendUniq(keys, key)
		// DelimitedName - requires resources.Prefix as it was trimmed off earlier in the flow
	case key == resources.Prefix+delimitedName:
		keys = appendUniq(keys, key)
	case delimitedName != "":
		if delimitedName == resources.Delimiter {
			resources.CommonPrefixes = appendUniq(resources.CommonPrefixes, resources.Prefix+delimitedName)
		} else {
			resources.CommonPrefixes = appendUniq(resources.CommonPrefixes, delimitedName)
		}
	}
	return resources, keys
}

// ListObjects - list objects from memory
func (memory memoryDriver) ListObjects(bucket string, resources drivers.BucketResourcesMetadata) ([]drivers.ObjectMetadata, drivers.BucketResourcesMetadata, error) {
	if _, ok := memory.bucketdata[bucket]; ok == false {
		return []drivers.ObjectMetadata{}, drivers.BucketResourcesMetadata{IsTruncated: false}, drivers.BucketNotFound{Bucket: bucket}
	}
	var results []drivers.ObjectMetadata
	var keys []string
	for key := range memory.objectdata {
		switch true {
		// Prefix absent, delimit object key based on delimiter
		case resources.IsDelimiterSet():
			delimitedName := delimiter(key, resources.Delimiter)
			switch true {
			case delimitedName == "" || delimitedName == key:
				keys = appendUniq(keys, key)
			case delimitedName != "":
				resources.CommonPrefixes = appendUniq(resources.CommonPrefixes, delimitedName)
			}
		// Prefix present, delimit object key with prefix key based on delimiter
		case resources.IsDelimiterPrefixSet():
			if strings.HasPrefix(key, resources.Prefix) {
				trimmedName := strings.TrimPrefix(key, resources.Prefix)
				delimitedName := delimiter(trimmedName, resources.Delimiter)
				resources, keys = memory.filterDelimiterPrefix(keys, key, delimitedName, resources)
			}
		// Prefix present, nothing to delimit
		case resources.IsPrefixSet():
			keys = appendUniq(keys, key)
		// Prefix and delimiter absent
		case resources.IsDefault():
			keys = appendUniq(keys, key)
		}
	}
	sort.Strings(keys)
	for _, key := range keys {
		if len(results) == resources.Maxkeys {
			return results, drivers.BucketResourcesMetadata{IsTruncated: true}, nil
		}
		object := memory.objectdata[key]
		if bucket == object.metadata.Bucket {
			results = append(results, object.metadata)
		}
	}
	return results, resources, nil
}

// ByBucketName is a type for sorting bucket metadata by bucket name
type ByBucketName []drivers.BucketMetadata

// Len of bucket name
func (b ByBucketName) Len() int { return len(b) }

// Swap bucket i, j
func (b ByBucketName) Swap(i, j int) { b[i], b[j] = b[j], b[i] }

// Less
func (b ByBucketName) Less(i, j int) bool { return b[i].Name < b[j].Name }

// ListBuckets - List buckets from memory
func (memory memoryDriver) ListBuckets() ([]drivers.BucketMetadata, error) {
	var results []drivers.BucketMetadata
	for _, bucket := range memory.bucketdata {
		results = append(results, bucket.metadata)
	}
	sort.Sort(ByBucketName(results))
	return results, nil
}

// GetObjectMetadata - get object metadata from memory
func (memory memoryDriver) GetObjectMetadata(bucket, key, prefix string) (drivers.ObjectMetadata, error) {
	if object, ok := memory.objectdata[key]; ok == true {
		return object.metadata, nil
	}
	return drivers.ObjectMetadata{}, drivers.ObjectNotFound{Bucket: bucket, Object: key}
}
<|endoftext|>


<commit_before>package donut

import (
	"bytes"
	"io"
	"strconv"
	"time"

	"crypto/md5"
	"encoding/hex"
	"errors"
	"github.com/minio-io/minio/pkg/encoding/erasure"
	"github.com/minio-io/minio/pkg/utils/split"
	"strings"
)

func erasureReader(readers []io.ReadCloser, donutMetadata map[string]string, writer *io.PipeWriter) {
	// TODO handle errors
	totalChunks, _ := strconv.Atoi(donutMetadata["chunkCount"])
	totalLeft, _ := strconv.Atoi(donutMetadata["totalLength"])
	blockSize, _ := strconv.Atoi(donutMetadata["blockSize"])
	k, _ := strconv.Atoi(donutMetadata["erasureK"])
	m, _ := strconv.Atoi(donutMetadata["erasureM"])
	expectedMd5sum, _ := hex.DecodeString(donutMetadata["md5"])
	summer := md5.New()
	// TODO select technique properly
	params, _ := erasure.ParseEncoderParams(uint8(k), uint8(m), erasure.Cauchy)
	encoder := erasure.NewEncoder(params)
	for _, reader := range readers {
		defer reader.Close()
	}
	for i := 0; i < totalChunks; i++ {
		curBlockSize := totalLeft
		if blockSize < totalLeft {
			curBlockSize = blockSize
		}
		curChunkSize := erasure.GetEncodedChunkLen(curBlockSize, uint8(k))

		encodedBytes := make([][]byte, 16)
		for i, reader := range readers {
			var bytesBuffer bytes.Buffer
			// TODO watch for errors
			io.CopyN(&bytesBuffer, reader, int64(curChunkSize))
			encodedBytes[i] = bytesBuffer.Bytes()
		}
		decodedData, err := encoder.Decode(encodedBytes, curBlockSize)
		if err != nil {
			writer.CloseWithError(err)
			return
		}
		summer.Write(decodedData)
		io.Copy(writer, bytes.NewBuffer(decodedData))
		totalLeft = totalLeft - blockSize
	}
	actualMd5sum := summer.Sum(nil)
	if bytes.Compare(expectedMd5sum, actualMd5sum) != 0 {
		writer.CloseWithError(errors.New("decoded md5sum did not match"))
		return
	}
	writer.Close()
}

// erasure writer

type erasureWriter struct {
	writers       []Writer
	metadata      map[string]string
	donutMetadata map[string]string // not exposed
	erasureWriter *io.PipeWriter
	isClosed      <-chan bool
}

func newErasureWriter(writers []Writer) ObjectWriter {
	r, w := io.Pipe()
	isClosed := make(chan bool)
	writer := erasureWriter{
		writers:       writers,
		metadata:      make(map[string]string),
		erasureWriter: w,
		isClosed:      isClosed,
	}
	go erasureGoroutine(r, writer, isClosed)
	return writer
}

func erasureGoroutine(r *io.PipeReader, eWriter erasureWriter, isClosed chan<- bool) {
	chunks := split.Stream(r, 10*1024*1024)
	params, _ := erasure.ParseEncoderParams(8, 8, erasure.Cauchy)
	encoder := erasure.NewEncoder(params)
	chunkCount := 0
	totalLength := 0
	summer := md5.New()
	for chunk := range chunks {
		if chunk.Err == nil {
			totalLength = totalLength + len(chunk.Data)
			encodedBlocks, _ := encoder.Encode(chunk.Data)
			summer.Write(chunk.Data)
			for blockIndex, block := range encodedBlocks {
				io.Copy(eWriter.writers[blockIndex], bytes.NewBuffer(block))
			}
		}
		chunkCount = chunkCount + 1
	}
	dataMd5sum := summer.Sum(nil)
	metadata := make(map[string]string)
	metadata["blockSize"] = strconv.Itoa(10 * 1024 * 1024)
	metadata["chunkCount"] = strconv.Itoa(chunkCount)
	metadata["created"] = time.Now().Format(time.RFC3339Nano)
	metadata["erasureK"] = "8"
	metadata["erasureM"] = "8"
	metadata["erasureTechnique"] = "Cauchy"
	metadata["md5"] = hex.EncodeToString(dataMd5sum)
	metadata["totalLength"] = strconv.Itoa(totalLength)
	for _, nodeWriter := range eWriter.writers {
		if nodeWriter != nil {
			nodeWriter.SetMetadata(eWriter.metadata)
			nodeWriter.SetDonutMetadata(metadata)
			nodeWriter.Close()
		}
	}
	isClosed <- true
}

func (d erasureWriter) Write(data []byte) (int, error) {
	io.Copy(d.erasureWriter, bytes.NewBuffer(data))
	return len(data), nil
}

func (d erasureWriter) Close() error {
	d.erasureWriter.Close()
	<-d.isClosed
	return nil
}

func (d erasureWriter) CloseWithError(err error) error {
	for _, writer := range d.writers {
		if writer != nil {
			writer.CloseWithError(err)
		}
	}
	return nil
}

func (d erasureWriter) SetMetadata(metadata map[string]string) error {
	for k, _ := range metadata {
		if strings.HasPrefix(k, "sys.") {
			return errors.New("Invalid key '" + k + "', cannot start with sys.'")
		}
	}
	for k := range d.metadata {
		delete(d.metadata, k)
	}
	for k, v := range metadata {
		d.metadata[k] = v
	}
	return nil
}

func (d erasureWriter) GetMetadata() (map[string]string, error) {
	metadata := make(map[string]string)
	for k, v := range d.metadata {
		metadata[k] = v
	}
	return metadata, nil
}
<commit_msg>Refactoring method variable names for erasureWriter struct<commit_after>package donut

import (
	"bytes"
	"io"
	"strconv"
	"time"

	"crypto/md5"
	"encoding/hex"
	"errors"
	"github.com/minio-io/minio/pkg/encoding/erasure"
	"github.com/minio-io/minio/pkg/utils/split"
	"strings"
)

func erasureReader(readers []io.ReadCloser, donutMetadata map[string]string, writer *io.PipeWriter) {
	// TODO handle errors
	totalChunks, _ := strconv.Atoi(donutMetadata["chunkCount"])
	totalLeft, _ := strconv.Atoi(donutMetadata["totalLength"])
	blockSize, _ := strconv.Atoi(donutMetadata["blockSize"])
	k, _ := strconv.Atoi(donutMetadata["erasureK"])
	m, _ := strconv.Atoi(donutMetadata["erasureM"])
	expectedMd5sum, _ := hex.DecodeString(donutMetadata["md5"])
	summer := md5.New()
	// TODO select technique properly
	params, _ := erasure.ParseEncoderParams(uint8(k), uint8(m), erasure.Cauchy)
	encoder := erasure.NewEncoder(params)
	for _, reader := range readers {
		defer reader.Close()
	}
	for i := 0; i < totalChunks; i++ {
		curBlockSize := totalLeft
		if blockSize < totalLeft {
			curBlockSize = blockSize
		}
		curChunkSize := erasure.GetEncodedChunkLen(curBlockSize, uint8(k))

		encodedBytes := make([][]byte, 16)
		for i, reader := range readers {
			var bytesBuffer bytes.Buffer
			// TODO watch for errors
			io.CopyN(&bytesBuffer, reader, int64(curChunkSize))
			encodedBytes[i] = bytesBuffer.Bytes()
		}
		decodedData, err := encoder.Decode(encodedBytes, curBlockSize)
		if err != nil {
			writer.CloseWithError(err)
			return
		}
		summer.Write(decodedData)
		io.Copy(writer, bytes.NewBuffer(decodedData))
		totalLeft = totalLeft - blockSize
	}
	actualMd5sum := summer.Sum(nil)
	if bytes.Compare(expectedMd5sum, actualMd5sum) != 0 {
		writer.CloseWithError(errors.New("decoded md5sum did not match"))
		return
	}
	writer.Close()
}

// erasure writer

type erasureWriter struct {
	writers       []Writer
	metadata      map[string]string
	donutMetadata map[string]string // not exposed
	erasureWriter *io.PipeWriter
	isClosed      <-chan bool
}

func newErasureWriter(writers []Writer) ObjectWriter {
	r, w := io.Pipe()
	isClosed := make(chan bool)
	writer := erasureWriter{
		writers:       writers,
		metadata:      make(map[string]string),
		erasureWriter: w,
		isClosed:      isClosed,
	}
	go erasureGoroutine(r, writer, isClosed)
	return writer
}

func erasureGoroutine(r *io.PipeReader, eWriter erasureWriter, isClosed chan<- bool) {
	chunks := split.Stream(r, 10*1024*1024)
	params, _ := erasure.ParseEncoderParams(8, 8, erasure.Cauchy)
	encoder := erasure.NewEncoder(params)
	chunkCount := 0
	totalLength := 0
	summer := md5.New()
	for chunk := range chunks {
		if chunk.Err == nil {
			totalLength = totalLength + len(chunk.Data)
			encodedBlocks, _ := encoder.Encode(chunk.Data)
			summer.Write(chunk.Data)
			for blockIndex, block := range encodedBlocks {
				io.Copy(eWriter.writers[blockIndex], bytes.NewBuffer(block))
			}
		}
		chunkCount = chunkCount + 1
	}
	dataMd5sum := summer.Sum(nil)
	metadata := make(map[string]string)
	metadata["blockSize"] = strconv.Itoa(10 * 1024 * 1024)
	metadata["chunkCount"] = strconv.Itoa(chunkCount)
	metadata["created"] = time.Now().Format(time.RFC3339Nano)
	metadata["erasureK"] = "8"
	metadata["erasureM"] = "8"
	metadata["erasureTechnique"] = "Cauchy"
	metadata["md5"] = hex.EncodeToString(dataMd5sum)
	metadata["totalLength"] = strconv.Itoa(totalLength)
	for _, nodeWriter := range eWriter.writers {
		if nodeWriter != nil {
			nodeWriter.SetMetadata(eWriter.metadata)
			nodeWriter.SetDonutMetadata(metadata)
			nodeWriter.Close()
		}
	}
	isClosed <- true
}

func (eWriter erasureWriter) Write(data []byte) (int, error) {
	io.Copy(eWriter.erasureWriter, bytes.NewBuffer(data))
	return len(data), nil
}

func (eWriter erasureWriter) Close() error {
	eWriter.erasureWriter.Close()
	<-eWriter.isClosed
	return nil
}

func (eWriter erasureWriter) CloseWithError(err error) error {
	for _, writer := range eWriter.writers {
		if writer != nil {
			writer.CloseWithError(err)
		}
	}
	return nil
}

func (eWriter erasureWriter) SetMetadata(metadata map[string]string) error {
	for k, _ := range metadata {
		if strings.HasPrefix(k, "sys.") {
			return errors.New("Invalid key '" + k + "', cannot start with sys.'")
		}
	}
	for k := range eWriter.metadata {
		delete(eWriter.metadata, k)
	}
	for k, v := range metadata {
		eWriter.metadata[k] = v
	}
	return nil
}

func (eWriter erasureWriter) GetMetadata() (map[string]string, error) {
	metadata := make(map[string]string)
	for k, v := range eWriter.metadata {
		metadata[k] = v
	}
	return metadata, nil
}
<|endoftext|>


<commit_before>// This package is the entrypoint for the Firmware Transparency monitor.
// The monitor follows the growth of the Firmware Transparency log server,
// inspects new firmware metadata as it appears, and prints out a short
// summary.
//
// TODO(al): Extend monitor to verify claims.
//
// Start the monitor using:
// go run ./cmd/ft_monitor/main.go --logtostderr -v=2 --ftlog=http://localhost:8000/
package main

import (
	"flag"
	"net/url"
	"time"

	"github.com/golang/glog"
	"github.com/google/trillian-examples/binary_transparency/firmware/api"
	"github.com/google/trillian-examples/binary_transparency/firmware/internal/client"
)

var (
	ftLog        = flag.String("ftlog", "http://localhost:8000", "Base URL of FT Log server")
	pollInterval = flag.Duration("poll_interval", 5*time.Second, "Duration to wait between polling for new entries")
)

func main() {
	flag.Parse()

	if len(*ftLog) == 0 {
		glog.Exit("ftlog is required")
	}

	ftURL, err := url.Parse(*ftLog)
	if err != nil {
		glog.Exitf("Failed to parse FT log URL: %q", err)
	}

	glog.Infof("Monitoring FT log %q...", *ftLog)
	ticker := time.NewTicker(*pollInterval)

	c := client.Client{LogURL: ftURL}
	// TODO(al): check signature on checkpoint when they're added.
	var latestCP api.LogCheckpoint

	for {
		<-ticker.C
		cp, err := c.GetCheckpoint()
		if err != nil {
			glog.Warningf("Failed to update LogCheckpoint: %q", err)
			continue
		}
		// TODO(al): check signature on checkpoint when they're added.

		if cp.TreeSize <= latestCP.TreeSize {
			continue
		}
		glog.V(1).Infof("Got newer checkpoint %s", cp)

		// Fetch all the manifests from now till new checkpoint
		for idx := latestCP.TreeSize; idx < cp.TreeSize; idx++ {

			manifest, err := c.GetManifestEntryAndProof(api.GetFirmwareManifestRequest{idx, cp.TreeSize})
			if err != nil {
				glog.Warningf("Failed to fetch the Manifest: %q", err)
				continue
			}
			glog.V(1).Infof("Received New Manifest with Information=")
			glog.V(1).Infof("Manifest Value = %s", manifest.Value)
			glog.V(1).Infof("LeafIndex = %d", manifest.LeafIndex)
			glog.V(1).Infof("Proof = %x", manifest.Proof)
			// TODO verify the inclusion proof
		}

		// Perform consistency check only for non-zero initial tree size
		if latestCP.TreeSize != 0 {
			consistency, err := c.GetConsistencyProof(api.GetConsistencyRequest{latestCP.TreeSize, cp.TreeSize})
			if err != nil {
				glog.Warningf("Failed to fetch the Consistency: %q", err)
				continue
			}
			glog.V(1).Infof("Printing the latest Consistency Proof Information")
			glog.V(1).Infof("Consistency Proof = %x", consistency.Proof)
		}
		// TODO verify the fetched consistency proof, before updating the checkpoint
		latestCP = *cp
	}
}
<commit_msg>Add Inclusion to fetched manifest<commit_after>// This package is the entrypoint for the Firmware Transparency monitor.
// The monitor follows the growth of the Firmware Transparency log server,
// inspects new firmware metadata as it appears, and prints out a short
// summary.
//
// TODO(al): Extend monitor to verify claims.
//
// Start the monitor using:
// go run ./cmd/ft_monitor/main.go --logtostderr -v=2 --ftlog=http://localhost:8000/
package main

import (
	"flag"
	"net/url"
	"time"

	"github.com/golang/glog"
	"github.com/google/trillian-examples/binary_transparency/firmware/api"
	"github.com/google/trillian-examples/binary_transparency/firmware/internal/client"
	"github.com/google/trillian-examples/binary_transparency/firmware/internal/verify"
)

var (
	ftLog        = flag.String("ftlog", "http://localhost:8000", "Base URL of FT Log server")
	pollInterval = flag.Duration("poll_interval", 5*time.Second, "Duration to wait between polling for new entries")
)

func main() {
	flag.Parse()

	if len(*ftLog) == 0 {
		glog.Exit("ftlog is required")
	}

	ftURL, err := url.Parse(*ftLog)
	if err != nil {
		glog.Exitf("Failed to parse FT log URL: %q", err)
	}

	glog.Infof("Monitoring FT log %q...", *ftLog)
	ticker := time.NewTicker(*pollInterval)

	c := client.Client{LogURL: ftURL}
	var latestCP api.LogCheckpoint

	lv := verify.NewLogVerifier()

	for {
		<-ticker.C
		cp, err := c.GetCheckpoint()
		if err != nil {
			glog.Warningf("Failed to update LogCheckpoint: %q", err)
			continue
		}
		// TODO(al): check signature on checkpoint when they're added.

		if cp.TreeSize <= latestCP.TreeSize {
			continue
		}
		glog.V(1).Infof("Got newer checkpoint %s", cp)

		// Fetch all the manifests from now till new checkpoint
		for idx := latestCP.TreeSize; idx < cp.TreeSize; idx++ {

			manifest, err := c.GetManifestEntryAndProof(api.GetFirmwareManifestRequest{idx, cp.TreeSize})
			if err != nil {
				glog.Warningf("Failed to fetch the Manifest: %q", err)
				continue
			}
			glog.V(1).Infof("Received New Manifest with Information=")
			glog.V(1).Infof("Manifest Value = %s", manifest.Value)
			glog.V(1).Infof("LeafIndex = %d", manifest.LeafIndex)
			glog.V(1).Infof("Proof = %x", manifest.Proof)

			lh := client.HashLeaf(manifest.Value)
			if err := lv.VerifyInclusionProof(int64(manifest.LeafIndex), int64(cp.TreeSize), manifest.Proof, cp.RootHash, lh); err != nil {
				// Report Failed Inclusion Proof
				glog.Warning("Invalid inclusion proof received for LeafIndex %d", manifest.LeafIndex)
				continue
			}

			glog.V(1).Infof("Inclusion proof for leafhash 0x%x verified", lh)
		}

		// Perform consistency check only for non-zero initial tree size
		if latestCP.TreeSize != 0 {
			consistency, err := c.GetConsistencyProof(api.GetConsistencyRequest{latestCP.TreeSize, cp.TreeSize})
			if err != nil {
				glog.Warningf("Failed to fetch the Consistency: %q", err)
				continue
			}
			glog.V(1).Infof("Printing the latest Consistency Proof Information")
			glog.V(1).Infof("Consistency Proof = %x", consistency.Proof)
		}
		// TODO verify the fetched consistency proof, before updating the checkpoint
		latestCP = *cp
	}
}
<|endoftext|>


<commit_before>// This work is subject to the CC0 1.0 Universal (CC0 1.0) Public Domain Dedication
// license. Its contents can be found at:
// http://creativecommons.org/publicdomain/zero/1.0/

package glu

//#include "GL/glu.h"
import "C"
import "github.com/banthar/gl"
import "unsafe"

// Opaque object used for book keeping on the go side.
type Tesselator struct {
	tess *C.GLUtesselator

	polyData interface{}

	// vertData keeps references to the vertices specifed by TessVertex
	// so that the garbage collector does not invalidate them.
	vertData []*vertexDataWrapper

	beginData      TessBeginHandler
	vertexData     TessVertexHandler
	endData        TessEndHandler
	errorData      TessErrorHandler
	edgeFlagData   TessEdgeFlagHandler
	combineData    TessCombineHandler
}

// Wrapper around an interface. Does go not support (*interface{})(ptr)?
type vertexDataWrapper struct {
	data interface{}
}

// Create a new tesselator.
func NewTess() (tess *Tesselator) {
	tess = new(Tesselator)
	tess.tess = C.gluNewTess()

	if tess.tess == nil {
		panic("Out of memory.")
	}

	return
}

// Clean up resources held by the tesselator. Go's garbage collector cannot
// do this automatically.
func (tess *Tesselator) Delete() {
	C.gluDeleteTess(tess.tess)
	tess.tess = nil
}

// Begin the drawing of the polygon, with the data parameter that will
// be provided to callbacks.
func (tess *Tesselator) BeginPolygon(data interface{}) {
	tess.polyData = data
	C.gluTessBeginPolygon(tess.tess, unsafe.Pointer(tess))
}

// End the drawing of the polygon.
func (tess *Tesselator) EndPolygon() {
	C.gluTessEndPolygon(tess.tess)
}

// Begin a contour within the polygon.
func (tess *Tesselator) BeginContour() {
	C.gluTessBeginContour(tess.tess)
}

// End a contour within the polygon.
func (tess *Tesselator) EndContour() {
	C.gluTessEndContour(tess.tess)
}

// Add a vertex to the polygon, with the data parameter that will be
// provided to callbacks.
func (tess *Tesselator) Vertex(location [3]float64, data interface{}) {
	_data := &vertexDataWrapper{data}
	tess.vertData = append(tess.vertData, _data)
	C.gluTessVertex(tess.tess, (*C.GLdouble)(&location[0]), unsafe.Pointer(_data))
}

// Set the normal of the plane onto which points are projected onto before tesselation.
func (tess *Tesselator) Normal(valueX, valueY, valueZ float64) {
	cx := C.GLdouble(valueX)
	cy := C.GLdouble(valueY)
	cz := C.GLdouble(valueZ)
	C.gluTessNormal(tess.tess, cx, cy, cz)
}

// Set a property of the tesselator.
func (tess *Tesselator) Property(which gl.GLenum, data float64) {
	C.gluTessProperty(tess.tess, C.GLenum(which), C.GLdouble(data))
}

<commit_msg>Copy vertex location to safe memory before passing to c routine.<commit_after>// This work is subject to the CC0 1.0 Universal (CC0 1.0) Public Domain Dedication
// license. Its contents can be found at:
// http://creativecommons.org/publicdomain/zero/1.0/

package glu

//#include "GL/glu.h"
import "C"
import "github.com/banthar/gl"
import "unsafe"

// Opaque object used for book keeping on the go side.
type Tesselator struct {
	tess *C.GLUtesselator

	polyData interface{}

	// vertData keeps references to the vertices specifed by TessVertex
	// so that the garbage collector does not invalidate them.
	vertData []*vertexDataWrapper

	// vertLocs stores a copy of the vertices' locations as specified
	// to TessVertex. Again, so the garbage collector doesn't get them.
	vertLocs [][3]float64

	beginData      TessBeginHandler
	vertexData     TessVertexHandler
	endData        TessEndHandler
	errorData      TessErrorHandler
	edgeFlagData   TessEdgeFlagHandler
	combineData    TessCombineHandler
}

// Wrapper around an interface. Does go not support (*interface{})(ptr)?
type vertexDataWrapper struct {
	data interface{}
}

// Create a new tesselator.
func NewTess() (tess *Tesselator) {
	tess = new(Tesselator)
	tess.tess = C.gluNewTess()

	if tess.tess == nil {
		panic("Out of memory.")
	}

	return
}

// Clean up resources held by the tesselator. Go's garbage collector cannot
// do this automatically.
func (tess *Tesselator) Delete() {
	C.gluDeleteTess(tess.tess)
	tess.tess = nil
}

// Begin the drawing of the polygon, with the data parameter that will
// be provided to callbacks.
func (tess *Tesselator) BeginPolygon(data interface{}) {
	tess.polyData = data
	C.gluTessBeginPolygon(tess.tess, unsafe.Pointer(tess))
}

// End the drawing of the polygon.
func (tess *Tesselator) EndPolygon() {
	C.gluTessEndPolygon(tess.tess)
}

// Begin a contour within the polygon.
func (tess *Tesselator) BeginContour() {
	C.gluTessBeginContour(tess.tess)
}

// End a contour within the polygon.
func (tess *Tesselator) EndContour() {
	C.gluTessEndContour(tess.tess)
}

// Add a vertex to the polygon, with the data parameter that will be
// provided to callbacks.
func (tess *Tesselator) Vertex(location [3]float64, data interface{}) {
	// Wrap and safeguard data pointer.
	_data := &vertexDataWrapper{data}
	tess.vertData = append(tess.vertData, _data)

	// Copy location to a safe memory location.
	tess.vertLocs = append(tess.vertLocs, location)
	_location := unsafe.Pointer(&tess.vertLocs[len(tess.vertLocs)-1])

	C.gluTessVertex(tess.tess, (*C.GLdouble)(_location), unsafe.Pointer(_data))
}

// Set the normal of the plane onto which points are projected onto before tesselation.
func (tess *Tesselator) Normal(valueX, valueY, valueZ float64) {
	cx := C.GLdouble(valueX)
	cy := C.GLdouble(valueY)
	cz := C.GLdouble(valueZ)
	C.gluTessNormal(tess.tess, cx, cy, cz)
}

// Set a property of the tesselator.
func (tess *Tesselator) Property(which gl.GLenum, data float64) {
	C.gluTessProperty(tess.tess, C.GLenum(which), C.GLdouble(data))
}

<|endoftext|>


<commit_before><commit_msg>Ajustado endereço do servidor<commit_after><|endoftext|>


<commit_before><commit_msg>fix shadowserver markdowntable<commit_after><|endoftext|>


<commit_before>package util

import (
	"fmt"

	"github.com/satori/go.uuid"
	"gopkg.in/resty.v0"
)

func (u SomaUtil) TryGetCustomPropertyByUUIDOrName(s string, r string) string {
	id, err := uuid.FromString(s)
	if err != nil {
		// aborts on failure
		return u.GetPropertyIdByName("custom", s, r)
	}
	return id.String()
}

func (u SomaUtil) TryGetServicePropertyByUUIDOrName(s string, t string) string {
	id, err := uuid.FromString(s)
	if err != nil {
		// aborts on failure
		return u.GetPropertyIdByName("service", s, t)
	}
	return id.String()
}

func (u SomaUtil) TryGetSystemPropertyByUUIDOrName(s string) string {
	id, err := uuid.FromString(s)
	if err != nil {
		// aborts on failure
		return u.GetPropertyIdByName("system", s, "none")
	}
	return id.String()
}

func (u SomaUtil) TryGetTemplatePropertyByUUIDOrName(s string) string {
	id, err := uuid.FromString(s)
	if err != nil {
		// aborts on failure
		return u.GetPropertyIdByName("template", s, "none")
	}
	return id.String()
}

func (u SomaUtil) GetPropertyIdByName(pType string, prop string, ctx string) string {
	var (
		req         proto.Request
		ctxIdString string
		path        string
	)
	req = proto.Request{
		Filter: &proto.Filter{
			Property: &proto.PropertyFilter{
				Name: prop,
			},
		},
	}

	switch pType {
	case "custom":
		// context ctx is repository
		ctxIdString = u.TryGetRepositoryByUUIDOrName(ctx)
		path = fmt.Sprintf("/property/custom/%s/", ctxIdString)
	case "system":
		path = "/property/system/"
	case "template":
		path = "/property/service/global/"
	case "service":
		// context ctx is team
		ctxIdString = u.TryGetTeamByUUIDOrName(ctx)
		path = fmt.Sprintf("/property/service/team/%s/", ctxIdString)
	default:
		u.Abort("Unsupported property type in util.GetPropertyIdByName()")
	}

	resp := u.GetRequestWithBody(req, path)
	propResult := u.DecodeProtoResultPropertyFromResponse(resp)

	switch prop {
	case (*propResult.Properties)[0].Custom.Name:
		return (*propResult.Properties)[0].Custom.Id
	case (*propResult.Properties)[0].System.Name:
		return (*propResult.Properties)[0].System.Name
	case (*propResult.Properties)[0].Service.Name:
		return (*propResult.Properties)[0].Service.Name
	default:
		u.Abort("Received result set for incorrect property")
	}

	// required to silence the compiler, since ending in a switch is not
	// analyzed to always return:
	// http://code.google.com/p/go/issues/detail?id=65
	panic("unreachable")
}

func (u SomaUtil) CheckStringIsSystemProperty(s string) {
	resp := u.GetRequest("/property/system/")
	res := u.DecodeProtoResultPropertyFromResponse(resp)

	for _, prop := range *res.Properties {
		if prop.System.Name == s {
			return
		}
	}
	u.Abort("Invalid system property requested")
}

func (u SomaUtil) DecodeProtoResultPropertyFromResponse(resp *resty.Response) *proto.Result {
	return u.DecodeResultFromResponse(resp)
}

// vim: ts=4 sw=4 sts=4 noet fenc=utf-8 ffs=unix
<commit_msg>Improve abort message<commit_after>package util

import (
	"fmt"

	"github.com/satori/go.uuid"
	"gopkg.in/resty.v0"
)

func (u SomaUtil) TryGetCustomPropertyByUUIDOrName(s string, r string) string {
	id, err := uuid.FromString(s)
	if err != nil {
		// aborts on failure
		return u.GetPropertyIdByName("custom", s, r)
	}
	return id.String()
}

func (u SomaUtil) TryGetServicePropertyByUUIDOrName(s string, t string) string {
	id, err := uuid.FromString(s)
	if err != nil {
		// aborts on failure
		return u.GetPropertyIdByName("service", s, t)
	}
	return id.String()
}

func (u SomaUtil) TryGetSystemPropertyByUUIDOrName(s string) string {
	id, err := uuid.FromString(s)
	if err != nil {
		// aborts on failure
		return u.GetPropertyIdByName("system", s, "none")
	}
	return id.String()
}

func (u SomaUtil) TryGetTemplatePropertyByUUIDOrName(s string) string {
	id, err := uuid.FromString(s)
	if err != nil {
		// aborts on failure
		return u.GetPropertyIdByName("template", s, "none")
	}
	return id.String()
}

func (u SomaUtil) GetPropertyIdByName(pType string, prop string, ctx string) string {
	var (
		req         proto.Request
		ctxIdString string
		path        string
	)
	req = proto.Request{
		Filter: &proto.Filter{
			Property: &proto.PropertyFilter{
				Name: prop,
			},
		},
	}

	switch pType {
	case "custom":
		// context ctx is repository
		ctxIdString = u.TryGetRepositoryByUUIDOrName(ctx)
		path = fmt.Sprintf("/property/custom/%s/", ctxIdString)
	case "system":
		path = "/property/system/"
	case "template":
		path = "/property/service/global/"
	case "service":
		// context ctx is team
		ctxIdString = u.TryGetTeamByUUIDOrName(ctx)
		path = fmt.Sprintf("/property/service/team/%s/", ctxIdString)
	default:
		u.Abort("Unsupported property type in util.GetPropertyIdByName()")
	}

	resp := u.GetRequestWithBody(req, path)
	propResult := u.DecodeProtoResultPropertyFromResponse(resp)

	switch prop {
	case (*propResult.Properties)[0].Custom.Name:
		return (*propResult.Properties)[0].Custom.Id
	case (*propResult.Properties)[0].System.Name:
		return (*propResult.Properties)[0].System.Name
	case (*propResult.Properties)[0].Service.Name:
		return (*propResult.Properties)[0].Service.Name
	default:
		u.Abort("Received result set for incorrect property")
	}

	// required to silence the compiler, since ending in a switch is not
	// analyzed to always return:
	// http://code.google.com/p/go/issues/detail?id=65
	panic("unreachable")
}

func (u SomaUtil) CheckStringIsSystemProperty(s string) {
	resp := u.GetRequest("/property/system/")
	res := u.DecodeProtoResultPropertyFromResponse(resp)

	for _, prop := range *res.Properties {
		if prop.System.Name == s {
			return
		}
	}
	u.Abort(fmt.Sprintf("Invalid system property requested: %s", s))
}

func (u SomaUtil) DecodeProtoResultPropertyFromResponse(resp *resty.Response) *proto.Result {
	return u.DecodeResultFromResponse(resp)
}

// vim: ts=4 sw=4 sts=4 noet fenc=utf-8 ffs=unix
<|endoftext|>


<commit_before>// Copyright 2021 The Ebiten Authors
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

//go:build generate
// +build generate

//go:generate go run gen.go

package oboe
<commit_msg>audio/internal/oboe: Remove 'generate' build tag<commit_after>// Copyright 2021 The Ebiten Authors
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

//go:generate go run gen.go

package oboe
<|endoftext|>


<commit_before>package sasl

import (
	"testing"
	"time"

	"github.com/gogo/protobuf/proto"
	"github.com/mesos/mesos-go/auth/callback"
	"github.com/mesos/mesos-go/auth/sasl/mech/crammd5"
	mesos "github.com/mesos/mesos-go/mesosproto"
	"github.com/mesos/mesos-go/messenger"
	"github.com/mesos/mesos-go/upid"
	"github.com/stretchr/testify/assert"
	"github.com/stretchr/testify/mock"
	"golang.org/x/net/context"
)

type MockTransport struct {
	*messenger.MockedMessenger
}

func (m *MockTransport) Send(ctx context.Context, upid *upid.UPID, msg proto.Message) error {
	return m.Called(mock.Anything, upid, msg).Error(0)
}

func TestAuthticatee_validLogin(t *testing.T) {
	assert := assert.New(t)
	ctx := context.TODO()
	client := upid.UPID{
		ID:   "someFramework",
		Host: "b.net",
		Port: "789",
	}
	server := upid.UPID{
		ID:   "serv",
		Host: "a.com",
		Port: "123",
	}
	tpid := upid.UPID{
		ID:   "sasl_transport",
		Host: "g.org",
		Port: "456",
	}
	handler := callback.HandlerFunc(func(cb ...callback.Interface) error {
		for _, c := range cb {
			switch c := c.(type) {
			case *callback.Name:
				c.Set("foo")
			case *callback.Password:
				c.Set([]byte("bar"))
			case *callback.Interprocess:
				c.Set(server, client)
			default:
				return &callback.Unsupported{Callback: c}
			}
		}
		return nil
	})
	var transport *MockTransport
	factory := transportFactoryFunc(func() messenger.Messenger {
		transport = &MockTransport{messenger.NewMockedMessenger()}
		transport.On("Install").Return(nil)
		transport.On("UPID").Return(&tpid)
		transport.On("Start").Return(nil)
		transport.On("Stop").Return(nil)
		transport.On("Send", mock.Anything, &server, &mesos.AuthenticateMessage{
			Pid: proto.String(client.String()),
		}).Return(nil).Run(func(_ mock.Arguments) {
			transport.Recv(&server, &mesos.AuthenticationMechanismsMessage{
				Mechanisms: []string{crammd5.Name},
			})
		}).Once()

		transport.On("Send", mock.Anything, &server, &mesos.AuthenticationStartMessage{
			Mechanism: proto.String(crammd5.Name),
			Data:      []byte(""), // may be nil, depends on init step
		}).Return(nil).Run(func(_ mock.Arguments) {
			transport.Recv(&server, &mesos.AuthenticationStepMessage{
				Data: []byte(`lsd;lfkgjs;dlfkgjs;dfklg`),
			})
		}).Once()

		transport.On("Send", mock.Anything, &server, &mesos.AuthenticationStepMessage{
			Data: []byte(`foo cc7fd96cd80123ea844a7dba29a594ed`),
		}).Return(nil).Run(func(_ mock.Arguments) {
			transport.Recv(&server, &mesos.AuthenticationCompletedMessage{})
		}).Once()

		return transport
	})
	login, err := makeAuthenticatee(handler, factory)
	assert.Nil(err)

	err = login.Authenticate(ctx, handler)
	assert.Nil(err)
	assert.NotNil(transport)
	time.Sleep(1 * time.Second) // wait for the authenticator to shut down
	transport.AssertExpectations(t)
}
<commit_msg>fixed broken unit test<commit_after>package sasl

import (
	"testing"
	"time"

	"github.com/gogo/protobuf/proto"
	"github.com/mesos/mesos-go/auth/callback"
	"github.com/mesos/mesos-go/auth/sasl/mech/crammd5"
	mesos "github.com/mesos/mesos-go/mesosproto"
	"github.com/mesos/mesos-go/messenger"
	"github.com/mesos/mesos-go/upid"
	"github.com/stretchr/testify/assert"
	"github.com/stretchr/testify/mock"
	"golang.org/x/net/context"
)

type MockTransport struct {
	*messenger.MockedMessenger
}

func (m *MockTransport) Send(ctx context.Context, upid *upid.UPID, msg proto.Message) error {
	return m.Called(mock.Anything, upid, msg).Error(0)
}

func TestAuthticatee_validLogin(t *testing.T) {
	assert := assert.New(t)
	ctx := context.TODO()
	client := upid.UPID{
		ID:   "someFramework",
		Host: "b.net",
		Port: "789",
	}
	server := upid.UPID{
		ID:   "serv",
		Host: "a.com",
		Port: "123",
	}
	tpid := upid.UPID{
		ID:   "sasl_transport",
		Host: "g.org",
		Port: "456",
	}
	handler := callback.HandlerFunc(func(cb ...callback.Interface) error {
		for _, c := range cb {
			switch c := c.(type) {
			case *callback.Name:
				c.Set("foo")
			case *callback.Password:
				c.Set([]byte("bar"))
			case *callback.Interprocess:
				c.Set(server, client)
			default:
				return &callback.Unsupported{Callback: c}
			}
		}
		return nil
	})
	var transport *MockTransport
	factory := transportFactoryFunc(func() messenger.Messenger {
		transport = &MockTransport{messenger.NewMockedMessenger()}
		transport.On("Install").Return(nil)
		transport.On("UPID").Return(&tpid)
		transport.On("Start").Return(nil)
		transport.On("Stop").Return(nil)
		transport.On("Send", mock.Anything, &server, &mesos.AuthenticateMessage{
			Pid: proto.String(client.String()),
		}).Return(nil).Run(func(_ mock.Arguments) {
			transport.Recv(&server, &mesos.AuthenticationMechanismsMessage{
				Mechanisms: []string{crammd5.Name},
			})
		}).Once()

		transport.On("Send", mock.Anything, &server, &mesos.AuthenticationStartMessage{
			Mechanism: proto.String(crammd5.Name),
		}).Return(nil).Run(func(_ mock.Arguments) {
			transport.Recv(&server, &mesos.AuthenticationStepMessage{
				Data: []byte(`lsd;lfkgjs;dlfkgjs;dfklg`),
			})
		}).Once()

		transport.On("Send", mock.Anything, &server, &mesos.AuthenticationStepMessage{
			Data: []byte(`foo cc7fd96cd80123ea844a7dba29a594ed`),
		}).Return(nil).Run(func(_ mock.Arguments) {
			transport.Recv(&server, &mesos.AuthenticationCompletedMessage{})
		}).Once()

		return transport
	})
	login, err := makeAuthenticatee(handler, factory)
	assert.Nil(err)

	err = login.Authenticate(ctx, handler)
	assert.Nil(err)
	assert.NotNil(transport)
	time.Sleep(1 * time.Second) // wait for the authenticator to shut down
	transport.AssertExpectations(t)
}
<|endoftext|>


<commit_before>package processor

import (
	"fmt"
	"os"

	"github.com/nanobox-io/nanobox/models"
	"github.com/nanobox-io/nanobox/util"
	"github.com/nanobox-io/nanobox/util/data"
	"github.com/nanobox-io/nanobox/util/production_api"
)

type link struct {
	config ProcessConfig
}

func init() {
	Register("link", linkFunc)
}

func linkFunc(conf ProcessConfig) (Processor, error) {
	return link{conf}, nil
}

func (self link) Results() ProcessConfig {
	return self.config
}

func (self link) Process() error {
	if self.config.Meta["name"] == "" {
		fmt.Println("you need to provide a app name to link to")
		os.Exit(1)
	}

	// get app id
	app, err := production_api.App(self.config.Meta["name"])
	if err != nil {
		return err
	}

	// store the auth token
	link := models.AppLinks{}
	data.Get(util.AppName(), "links", &link)
	link[self.config.Meta["alias"]] = app.ID
	return data.Put(util.AppName(), "links", link)
}
<commit_msg>move link data from app to app_meta<commit_after>package processor

import (
	"fmt"
	"os"

	"github.com/nanobox-io/nanobox/models"
	"github.com/nanobox-io/nanobox/util"
	"github.com/nanobox-io/nanobox/util/data"
	"github.com/nanobox-io/nanobox/util/production_api"
)

type link struct {
	config ProcessConfig
}

func init() {
	Register("link", linkFunc)
}

func linkFunc(conf ProcessConfig) (Processor, error) {
	return link{conf}, nil
}

func (self link) Results() ProcessConfig {
	return self.config
}

func (self link) Process() error {
	if self.config.Meta["name"] == "" {
		fmt.Println("you need to provide a app name to link to")
		os.Exit(1)
	}

	// get app id
	app, err := production_api.App(self.config.Meta["name"])
	if err != nil {
		return err
	}

	// store the auth token
	link := models.AppLinks{}
	data.Get(util.AppName()"_meta", "links", &link)
	link[self.config.Meta["alias"]] = app.ID
	return data.Put(util.AppName()"_meta", "links", link)
}
<|endoftext|>


<commit_before>package main

import (
	"fmt"
	"os"

	"github.com/spf13/pflag"

	"k8s.io/apiserver/pkg/util/flag"
	"k8s.io/apiserver/pkg/util/logs"

	"k8s.io/kube-deploy/cluster-api/machinecontroller/controller"
)

func main() {
	c := controller.NewConfiguration()
	c.AddFlags(pflag.CommandLine)

	flag.InitFlags()
	logs.InitLogs()
	defer logs.FlushLogs()

	mc := controller.NewMachineController(c)
	if err := mc.Run(); err != nil {
		fmt.Fprintf(os.Stderr, "%v\n", err)
		os.Exit(1)
	}
}
<commit_msg>supress uneeded warning<commit_after>package main

import (
	"fmt"
	"os"
	goflag "flag"

	"github.com/spf13/pflag"

	"k8s.io/apiserver/pkg/util/flag"
	"k8s.io/apiserver/pkg/util/logs"

	"k8s.io/kube-deploy/cluster-api/machinecontroller/controller"
)

func main() {
	c := controller.NewConfiguration()
	c.AddFlags(pflag.CommandLine)

	flag.InitFlags()
	pflag.Parse()
	// Suppress warning
	// https://github.com/kubernetes/kubernetes/issues/17162
	goflag.CommandLine.Parse([]string{})
	logs.InitLogs()
	defer logs.FlushLogs()

	mc := controller.NewMachineController(c)
	if err := mc.Run(); err != nil {
		fmt.Fprintf(os.Stderr, "%v\n", err)
		os.Exit(1)
	}
}
<|endoftext|>


<commit_before>package kateway

import (
	"sync"
	"time"

	"github.com/funkygao/gafka/cmd/kguard/monitor"
	"github.com/funkygao/gafka/zk"
	log "github.com/funkygao/log4go"
	"github.com/rcrowley/go-metrics"
)

func init() {
	monitor.RegisterWatcher("kateway.sublag", func() monitor.Watcher {
		return &WatchSubLag{
			Tick: time.Minute,
		}
	})
}

// SubLag monitors aliveness of kateway cluster.
type WatchSubLag struct {
	Zkzone *zk.ZkZone
	Stop   <-chan struct{}
	Tick   time.Duration
	Wg     *sync.WaitGroup

	zkcluster *zk.ZkCluster

	suspects map[string]struct{}
}

func (this *WatchSubLag) Init(ctx monitor.Context) {
	this.Zkzone = ctx.ZkZone()
	this.Stop = ctx.StopChan()
	this.Wg = ctx.Inflight()
	this.suspects = make(map[string]struct{})
}

func (this *WatchSubLag) Run() {
	defer this.Wg.Done()

	this.zkcluster = this.Zkzone.NewCluster("bigtopic") // TODO

	ticker := time.NewTicker(this.Tick)
	defer ticker.Stop()

	subLagGroups := metrics.NewRegisteredGauge("sub.lags", nil)
	for {
		select {
		case <-this.Stop:
			log.Info("kateway.sublag stopped")
			return

		case <-ticker.C:
			subLagGroups.Update(int64(this.report()))

		}
	}
}

func (this *WatchSubLag) isSuspect(group string, topic string) bool {
	if _, present := this.suspects[group+"|"+topic]; present {
		return true
	}

	return false
}

func (this *WatchSubLag) suspect(group, topic string) {
	this.suspects[group+"|"+topic] = struct{}{}
}

func (this *WatchSubLag) unsuspect(group string, topic string) {
	delete(this.suspects, group+"|"+topic)
}

func (this *WatchSubLag) report() (lags int) {
	for group, consumers := range this.zkcluster.ConsumersByGroup("") {
		for _, c := range consumers {
			if !c.Online {
				continue
			}

			if c.ConsumerZnode == nil {
				log.Warn("group[%s] topic[%s/%s] unrecognized consumer", group, c.Topic, c.PartitionId)

				continue
			}

			if time.Since(c.ConsumerZnode.Uptime()) < time.Minute*2 {
				log.Info("group[%s] just started, topic[%s/%s]", group, c.Topic, c.PartitionId)

				this.unsuspect(group, c.Topic)
				continue
			}

			// offset commit every 1m, sublag runs every 1m, so the gap might be 2m
			// TODO lag too much, even if it's still alive, emit alarm
			elapsed := time.Since(c.Mtime.Time())
			if c.Lag == 0 || elapsed < time.Minute*3 {
				this.unsuspect(group, c.Topic)
				continue
			}

			// it might be lagging, but need confirm with last round
			if !this.isSuspect(group, c.Topic) {
				// suspect it, next round if it is still lagging, put on trial
				log.Warn("group[%s] suspected topic[%s/%s] %d - %d = %d, offset commit elapsed: %s",
					group, c.Topic, c.PartitionId, c.ProducerOffset, c.ConsumerOffset, c.Lag, elapsed.String())

				this.suspect(group, c.Topic)
				continue
			}

			// bingo! it IS lagging
			log.Warn("group[%s] confirmed topic[%s/%s] %d - %d = %d, offset commit elapsed: %s",
				group, c.Topic, c.PartitionId, c.ProducerOffset, c.ConsumerOffset, c.Lag, elapsed.String())

			lags++
		}
	}

	return
}
<commit_msg>kguard hunt for sub groups that consumes more than 1 topics<commit_after>package kateway

import (
	"sort"
	"strings"
	"sync"
	"time"

	"github.com/funkygao/gafka/cmd/kguard/monitor"
	"github.com/funkygao/gafka/zk"
	log "github.com/funkygao/log4go"
	"github.com/rcrowley/go-metrics"
)

func init() {
	monitor.RegisterWatcher("kateway.sublag", func() monitor.Watcher {
		return &WatchSubLag{
			Tick: time.Minute,
		}
	})
}

// SubLag monitors aliveness of kateway cluster.
type WatchSubLag struct {
	Zkzone *zk.ZkZone
	Stop   <-chan struct{}
	Tick   time.Duration
	Wg     *sync.WaitGroup

	zkcluster *zk.ZkCluster

	suspects map[string]struct{}
}

func (this *WatchSubLag) Init(ctx monitor.Context) {
	this.Zkzone = ctx.ZkZone()
	this.Stop = ctx.StopChan()
	this.Wg = ctx.Inflight()
	this.suspects = make(map[string]struct{})
}

func (this *WatchSubLag) Run() {
	defer this.Wg.Done()

	this.zkcluster = this.Zkzone.NewCluster("bigtopic") // TODO

	ticker := time.NewTicker(this.Tick)
	defer ticker.Stop()

	subLagGroups := metrics.NewRegisteredGauge("sub.lags", nil)
	subConflictGroup := metrics.NewRegisteredGauge("sub.group.conflict", nil)
	for {
		select {
		case <-this.Stop:
			log.Info("kateway.sublag stopped")
			return

		case <-ticker.C:
			lags, conflictGroups := this.report()
			subLagGroups.Update(int64(lags))
			subConflictGroup.Update(int64(conflictGroups))

		}
	}
}

func (this *WatchSubLag) isSuspect(group string, topic string) bool {
	if _, present := this.suspects[group+"|"+topic]; present {
		return true
	}

	return false
}

func (this *WatchSubLag) suspect(group, topic string) {
	this.suspects[group+"|"+topic] = struct{}{}
}

func (this *WatchSubLag) unsuspect(group string, topic string) {
	delete(this.suspects, group+"|"+topic)
}

func (this *WatchSubLag) report() (lags, conflictGroups int) {
	groupTopicsMap := make(map[string]map[string]struct{}) // group:sub topics
	for group, consumers := range this.zkcluster.ConsumersByGroup("") {
		for _, c := range consumers {
			if !c.Online {
				continue
			}

			if c.ConsumerZnode == nil {
				log.Warn("group[%s] topic[%s/%s] unrecognized consumer", group, c.Topic, c.PartitionId)

				continue
			}

			// record each group is consuming what topics
			for topic, _ := range c.ConsumerZnode.Subscription {
				if _, present := groupTopicsMap[group]; !present {
					groupTopicsMap[group] = make(map[string]struct{}, 5)
				}
				groupTopicsMap[group][topic] = struct{}{}
			}

			if time.Since(c.ConsumerZnode.Uptime()) < time.Minute*2 {
				log.Info("group[%s] just started, topic[%s/%s]", group, c.Topic, c.PartitionId)

				this.unsuspect(group, c.Topic)
				continue
			}

			// offset commit every 1m, sublag runs every 1m, so the gap might be 2m
			// TODO lag too much, even if it's still alive, emit alarm
			elapsed := time.Since(c.Mtime.Time())
			if c.Lag == 0 || elapsed < time.Minute*3 {
				this.unsuspect(group, c.Topic)
				continue
			}

			// it might be lagging, but need confirm with last round
			if !this.isSuspect(group, c.Topic) {
				// suspect it, next round if it is still lagging, put on trial
				log.Warn("group[%s] suspected topic[%s/%s] %d - %d = %d, offset commit elapsed: %s",
					group, c.Topic, c.PartitionId, c.ProducerOffset, c.ConsumerOffset, c.Lag, elapsed.String())

				this.suspect(group, c.Topic)
				continue
			}

			// bingo! it IS lagging
			log.Warn("group[%s] confirmed topic[%s/%s] %d - %d = %d, offset commit elapsed: %s",
				group, c.Topic, c.PartitionId, c.ProducerOffset, c.ConsumerOffset, c.Lag, elapsed.String())

			lags++
		}
	}

	// Sub disallow the same group to sub multiple topics
	for group, topics := range groupTopicsMap {
		if len(topics) <= 1 {
			continue
		}

		// conflict found!
		conflictGroups++

		// the same consumer group is consuming more than 1 topics
		topicsLabel := make([]string, 0, len(topics))
		for t, _ := range topics {
			topicsLabel = append(topicsLabel, t)
		}
		sort.Strings(topicsLabel)

		log.Warn("group[%s] consuming more than 1 topics: %s", group, strings.Join(topicsLabel, ","))
	}

	return
}
<|endoftext|>


<commit_before>
package cgocuishark

/*
#cgo pkg-config: glib-2.0
#cgo CFLAGS: -I./lib
#cgo LDFLAGS: -L./lib
#cgo LDFLAGS: -L/usr/local/lib
#cgo LDFLAGS: -lcuishark
#cgo LDFLAGS: -lwireshark -lgmodule-2.0
#cgo LDFLAGS: -pthread -lgthread-2.0 -pthread -lglib-2.0
#cgo LDFLAGS: -lwiretap -lwsutil -lz -lm -lpcap -lstdc++
#include <cuishark.h>
*/
import "C"
import "bytes"
import "unsafe"
import "encoding/binary"

type Packet struct {
  raw *C.packet_t
}

type Node struct {
  raw *C.node_t
}

func (pkt *Packet) Bytes() []uint8 {
  array := C.cuishark_msg_data_ptr(pkt.raw)
  length := C.cuishark_msg_data_len(pkt.raw)

	slice := make([]uint8, int(length))
	b := C.GoBytes(unsafe.Pointer(array), C.int(length))
	err := binary.Read(bytes.NewReader(b), binary.LittleEndian, slice)
	if err != nil {
		panic(err)
	}

  return slice
}

func (pkt *Packet) Node() (*Node) {
  pkt_raw := pkt.raw
  node_raw := C.cuishark_msg_node(pkt_raw)
  return &Node{node_raw}
}

func (node *Node) ChildsNum() int {
  cint := C.node_childs_num(node.raw)
  return int(cint)
}

func (node *Node) Child(idx int) (*Node) {
  c_idx := C.ulong(idx)
  c_child_node := C.node_child(node.raw, c_idx)
  return &Node{c_child_node}
}

func (node *Node) IsOpen() bool {
  b := C.node_isopen(node.raw)
  return bool(b)
}

func (node *Node) IsOpenSwitch() {
  C.node_isopen_switch(node.raw)
}

func (node *Node) Search(idx int) *Node {
  i := (C.int)(idx)
  c := C.get_node_from_root(node.raw, i)
  return &Node{c}
}

func (node *Node) Line() string {
  s := C.GoString(C.node_line(node.raw))
  return s
}

func Init(args []string) {
  argc := C.int(len(args))
  argv := make([]*C.char, argc)
  for i, arg := range args {
    argv[i] = C.CString(arg)
  }

  C.cuishark_init(argc, (**C.char)(&argv[0]))
}

func Capture() {
  C.cuishark_capture()
}

func Fini() {
  C.cuishark_fini()
}

func ApplyDfilter(filter string) {
  cstr := C.CString(filter)
  C.cuishark_apply_dfilter(cstr)
}

func LoopRunning() bool {
  b := C.cuishark_loop_running()
  bb := bool(b)
  return bb
}

func MsgQueuePop() *Packet {
  pkt := C.cuishark_msgqueue_pop()
  gopkt := Packet{pkt}
  return &gopkt
}

func MsgQueueEmpty() bool {
  cb := C.cuishark_msg_queue_empty()
  return bool(cb)
}

func GetInterfaceName() string {
  cstr := C.get_interface_name()
  return C.GoString(cstr)
}

func PacketsDump() {
  C.cuishark_packets_dump()
}

func StatusDump() {
  C.cuishark_status_dump()
}

<commit_msg>2018.07.02-17:21<commit_after>
package cgocuishark

/*
#cgo pkg-config: glib-2.0
#cgo CFLAGS: -I./lib
#cgo LDFLAGS: -L./lib
#cgo LDFLAGS: -L/usr/local/lib
#cgo LDFLAGS: -lcuishark
#cgo LDFLAGS: -pthread -lgthread-2.0 -lglib-2.0 -lgmodule-2.0
#cgo LDFLAGS: -lwiretap -lwsutil -lwireshark
#cgo LDFLAGS: -lz -lm -lpcap -lstdc++
#include <cuishark.h>
*/
import "C"
import "bytes"
import "unsafe"
import "encoding/binary"

type Packet struct {
  raw *C.packet_t
}

type Node struct {
  raw *C.node_t
}

func (pkt *Packet) Bytes() []uint8 {
  array := C.cuishark_msg_data_ptr(pkt.raw)
  length := C.cuishark_msg_data_len(pkt.raw)

	slice := make([]uint8, int(length))
	b := C.GoBytes(unsafe.Pointer(array), C.int(length))
	err := binary.Read(bytes.NewReader(b), binary.LittleEndian, slice)
	if err != nil {
		panic(err)
	}

  return slice
}

func (pkt *Packet) Node() (*Node) {
  pkt_raw := pkt.raw
  node_raw := C.cuishark_msg_node(pkt_raw)
  return &Node{node_raw}
}

func (node *Node) ChildsNum() int {
  cint := C.node_childs_num(node.raw)
  return int(cint)
}

func (node *Node) Child(idx int) (*Node) {
  c_idx := C.ulong(idx)
  c_child_node := C.node_child(node.raw, c_idx)
  return &Node{c_child_node}
}

func (node *Node) IsOpen() bool {
  b := C.node_isopen(node.raw)
  return bool(b)
}

func (node *Node) IsOpenSwitch() {
  C.node_isopen_switch(node.raw)
}

func (node *Node) Search(idx int) *Node {
  i := (C.int)(idx)
  c := C.get_node_from_root(node.raw, i)
  return &Node{c}
}

func (node *Node) Line() string {
  s := C.GoString(C.node_line(node.raw))
  return s
}

func Init(args []string) {
  argc := C.int(len(args))
  argv := make([]*C.char, argc)
  for i, arg := range args {
    argv[i] = C.CString(arg)
  }

  C.cuishark_init(argc, (**C.char)(&argv[0]))
}

func Capture() {
  C.cuishark_capture()
}

func Fini() {
  C.cuishark_fini()
}

func ApplyDfilter(filter string) {
  cstr := C.CString(filter)
  C.cuishark_apply_dfilter(cstr)
}

func LoopRunning() bool {
  b := C.cuishark_loop_running()
  bb := bool(b)
  return bb
}

func MsgQueuePop() *Packet {
  pkt := C.cuishark_msgqueue_pop()
  gopkt := Packet{pkt}
  return &gopkt
}

func MsgQueueEmpty() bool {
  cb := C.cuishark_msg_queue_empty()
  return bool(cb)
}

func GetInterfaceName() string {
  cstr := C.get_interface_name()
  return C.GoString(cstr)
}

func PacketsDump() {
  C.cuishark_packets_dump()
}

func StatusDump() {
  C.cuishark_status_dump()
}

<|endoftext|>


<commit_before>// Copyright 2017 tsuru authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

package mongodb

import (
	"github.com/tsuru/tsuru/db"
	dbStorage "github.com/tsuru/tsuru/db/storage"
	"github.com/tsuru/tsuru/types/auth"
	mgo "gopkg.in/mgo.v2"
	"gopkg.in/mgo.v2/bson"
)

type TeamService struct{}

type team struct {
	Name         string `bson:"_id"`
	CreatingUser string
}

func teamsCollection(conn *db.Storage) *dbStorage.Collection {
	return conn.Collection("teams")
}

func (r *TeamService) Insert(t auth.Team) error {
	conn, err := db.Conn()
	if err != nil {
		return err
	}
	defer conn.Close()
	err = teamsCollection(conn).Insert(team(t))
	if mgo.IsDup(err) {
		return auth.ErrTeamAlreadyExists
	}
	return err
}

func (r *TeamService) FindAll() ([]auth.Team, error) {
	conn, err := db.Conn()
	if err != nil {
		return nil, err
	}
	defer conn.Close()
	var teams []team
	err = teamsCollection(conn).Find(nil).All(&teams)
	if err != nil {
		return nil, err
	}
	authTeams := make([]auth.Team, len(teams))
	for i, t := range teams {
		authTeams[i] = auth.Team(t)
	}
	return authTeams, nil
}

func (r *TeamService) FindByName(name string) (*auth.Team, error) {
	var t team
	conn, err := db.Conn()
	if err != nil {
		return nil, err
	}
	defer conn.Close()
	err = teamsCollection(conn).FindId(name).One(&t)
	if err != nil {
		if err == mgo.ErrNotFound {
			err = auth.ErrTeamNotFound
		}
		return nil, err
	}
	team := auth.Team(t)
	return &team, nil
}

func (r *TeamService) FindByNames(names []string) ([]auth.Team, error) {
	conn, err := db.Conn()
	if err != nil {
		return nil, err
	}
	defer conn.Close()
	var teams []team
	err = teamsCollection(conn).Find(bson.M{"_id": bson.M{"$in": names}}).All(&teams)
	if err != nil {
		return nil, err
	}
	authTeams := make([]auth.Team, len(teams))
	for i, t := range teams {
		authTeams[i] = auth.Team(t)
	}
	return authTeams, nil
}

func (r *TeamService) Delete(t auth.Team) error {
	conn, err := db.Conn()
	if err != nil {
		return err
	}
	defer conn.Close()
	err = teamsCollection(conn).RemoveId(t.Name)
	if err == mgo.ErrNotFound {
		return auth.ErrTeamNotFound
	}
	return err
}
<commit_msg>storage/mongodb: remove code duplication<commit_after>// Copyright 2017 tsuru authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

package mongodb

import (
	"github.com/tsuru/tsuru/db"
	dbStorage "github.com/tsuru/tsuru/db/storage"
	"github.com/tsuru/tsuru/types/auth"
	mgo "gopkg.in/mgo.v2"
	"gopkg.in/mgo.v2/bson"
)

type TeamService struct{}

type team struct {
	Name         string `bson:"_id"`
	CreatingUser string
}

func teamsCollection(conn *db.Storage) *dbStorage.Collection {
	return conn.Collection("teams")
}

func (s *TeamService) Insert(t auth.Team) error {
	conn, err := db.Conn()
	if err != nil {
		return err
	}
	defer conn.Close()
	err = teamsCollection(conn).Insert(team(t))
	if mgo.IsDup(err) {
		return auth.ErrTeamAlreadyExists
	}
	return err
}

func (s *TeamService) FindAll() ([]auth.Team, error) {
	return s.findByQuery(nil)
}

func (s *TeamService) FindByName(name string) (*auth.Team, error) {
	var t team
	conn, err := db.Conn()
	if err != nil {
		return nil, err
	}
	defer conn.Close()
	err = teamsCollection(conn).FindId(name).One(&t)
	if err != nil {
		if err == mgo.ErrNotFound {
			err = auth.ErrTeamNotFound
		}
		return nil, err
	}
	team := auth.Team(t)
	return &team, nil
}

func (s *TeamService) FindByNames(names []string) ([]auth.Team, error) {
	query := bson.M{"_id": bson.M{"$in": names}}
	return s.findByQuery(query)
}

func (s *TeamService) findByQuery(query bson.M) ([]auth.Team, error) {
	conn, err := db.Conn()
	if err != nil {
		return nil, err
	}
	defer conn.Close()
	var teams []team
	err = teamsCollection(conn).Find(query).All(&teams)
	if err != nil {
		return nil, err
	}
	authTeams := make([]auth.Team, len(teams))
	for i, t := range teams {
		authTeams[i] = auth.Team(t)
	}
	return authTeams, nil
}

func (s *TeamService) Delete(t auth.Team) error {
	conn, err := db.Conn()
	if err != nil {
		return err
	}
	defer conn.Close()
	err = teamsCollection(conn).RemoveId(t.Name)
	if err == mgo.ErrNotFound {
		return auth.ErrTeamNotFound
	}
	return err
}
<|endoftext|>


<commit_before>package fsutil

import (
	"errors"
	"io"
	"io/ioutil"
	"os"
	"path"
	"testing"
	"time"
)

var errorTimeout = errors.New("timeout")

func TestWatchFileDir(t *testing.T) {
	dirname, err := ioutil.TempDir("", "WatchFileTests")
	if err != nil {
		t.Fatal(err)
	}
	defer os.RemoveAll(dirname)
	testWatchFile(t, dirname)
}

func TestWatchFileCwd(t *testing.T) {
	cwd, err := os.Getwd()
	if err != nil {
		t.Fatal(err)
	}
	defer os.Chdir(cwd)
	dirname, err := ioutil.TempDir("", "WatchFileTests")
	if err != nil {
		t.Fatal(err)
	}
	defer os.RemoveAll(dirname)
	if err := os.Chdir(dirname); err != nil {
		t.Fatal(err)
	}
	testWatchFile(t, "")
}

func testWatchFile(t *testing.T, dirname string) {
	pathNotExist := path.Join(dirname, "never-exists")
	ch := WatchFile(pathNotExist, nil)
	rc, err := watchTimeout(ch, time.Millisecond)
	if err != errorTimeout {
		rc.Close()
		t.Fatal("Expected timeout error for non-existant file")
	}
	pathExists := path.Join(dirname, "exists")
	file, err := os.Create(pathExists)
	if err != nil {
		t.Fatal(err)
	}
	file.Close()
	ch = WatchFile(pathExists, nil)
	rc, err = watchTimeout(ch, time.Millisecond)
	if err != nil {
		t.Fatal(err)
	} else {
		rc.Close()
	}
	pathExistsLater := path.Join(dirname, "exists-later")
	go func() {
		time.Sleep(time.Millisecond * 50)
		file, err := os.Create(pathExistsLater)
		if err != nil {
			t.Fatal(err)
		}
		file.Close()
	}()
	ch = WatchFile(pathExistsLater, nil)
	rc, err = watchTimeout(ch, time.Millisecond*10)
	if err != errorTimeout {
		t.Fatal("Expected timeout error for non-existant file")
	}
	rc, err = watchTimeout(ch, time.Millisecond*90)
	if err != nil {
		t.Fatal(err)
	} else {
		rc.Close()
	}
	pathWillBeRenamed := path.Join(dirname, "will-be-renamed")
	file, err = os.Create(pathWillBeRenamed)
	if err != nil {
		t.Fatal(err)
	}
	file.Close()
	rc, err = watchTimeout(ch, time.Millisecond*10)
	if err != errorTimeout {
		rc.Close()
		t.Fatal("Expected timeout error for unchanged file")
	}
	if err := os.Rename(pathWillBeRenamed, pathExistsLater); err != nil {
		t.Fatal(err)
	}
	rc, err = watchTimeout(ch, time.Millisecond*10)
	if err != nil {
		t.Fatal(err)
	} else {
		rc.Close()
	}
}

func watchTimeout(channel <-chan io.ReadCloser, timeout time.Duration) (
	io.ReadCloser, error) {
	select {
	case readCloser := <-channel:
		return readCloser, nil
	case <-time.After(timeout):
		return nil, errorTimeout
	}
	panic("impossible")
}
<commit_msg>Fix testing bug for lib/fsutil.WatchFile(): do not pass nil logger.<commit_after>package fsutil

import (
	"errors"
	"github.com/Symantec/Dominator/lib/log/testlogger"
	"io"
	"io/ioutil"
	"os"
	"path"
	"testing"
	"time"
)

var errorTimeout = errors.New("timeout")

func TestWatchFileDir(t *testing.T) {
	dirname, err := ioutil.TempDir("", "WatchFileTests")
	if err != nil {
		t.Fatal(err)
	}
	defer os.RemoveAll(dirname)
	testWatchFile(t, dirname)
}

func TestWatchFileCwd(t *testing.T) {
	cwd, err := os.Getwd()
	if err != nil {
		t.Fatal(err)
	}
	defer os.Chdir(cwd)
	dirname, err := ioutil.TempDir("", "WatchFileTests")
	if err != nil {
		t.Fatal(err)
	}
	defer os.RemoveAll(dirname)
	if err := os.Chdir(dirname); err != nil {
		t.Fatal(err)
	}
	testWatchFile(t, "")
}

func testWatchFile(t *testing.T, dirname string) {
	logger := testlogger.New(t)
	pathNotExist := path.Join(dirname, "never-exists")
	ch := WatchFile(pathNotExist, logger)
	rc, err := watchTimeout(ch, time.Millisecond)
	if err != errorTimeout {
		rc.Close()
		t.Fatal("Expected timeout error for non-existant file")
	}
	pathExists := path.Join(dirname, "exists")
	file, err := os.Create(pathExists)
	if err != nil {
		t.Fatal(err)
	}
	file.Close()
	ch = WatchFile(pathExists, logger)
	rc, err = watchTimeout(ch, time.Millisecond)
	if err != nil {
		t.Fatal(err)
	} else {
		rc.Close()
	}
	pathExistsLater := path.Join(dirname, "exists-later")
	go func() {
		time.Sleep(time.Millisecond * 50)
		file, err := os.Create(pathExistsLater)
		if err != nil {
			t.Fatal(err)
		}
		file.Close()
	}()
	ch = WatchFile(pathExistsLater, logger)
	rc, err = watchTimeout(ch, time.Millisecond*10)
	if err != errorTimeout {
		t.Fatal("Expected timeout error for non-existant file")
	}
	rc, err = watchTimeout(ch, time.Millisecond*90)
	if err != nil {
		t.Fatal(err)
	} else {
		rc.Close()
	}
	pathWillBeRenamed := path.Join(dirname, "will-be-renamed")
	file, err = os.Create(pathWillBeRenamed)
	if err != nil {
		t.Fatal(err)
	}
	file.Close()
	rc, err = watchTimeout(ch, time.Millisecond*10)
	if err != errorTimeout {
		rc.Close()
		t.Fatal("Expected timeout error for unchanged file")
	}
	if err := os.Rename(pathWillBeRenamed, pathExistsLater); err != nil {
		t.Fatal(err)
	}
	rc, err = watchTimeout(ch, time.Millisecond*10)
	if err != nil {
		t.Fatal(err)
	} else {
		rc.Close()
	}
}

func watchTimeout(channel <-chan io.ReadCloser, timeout time.Duration) (
	io.ReadCloser, error) {
	select {
	case readCloser := <-channel:
		return readCloser, nil
	case <-time.After(timeout):
		return nil, errorTimeout
	}
	panic("impossible")
}
<|endoftext|>