<commit_before>package transporter

import (
	"log"
	"time"

	"github.com/compose/transporter/pkg/adaptor"
	"github.com/compose/transporter/pkg/events"
)

// VERSION the library
const (
	VERSION = "0.0.2"
)

// A Pipeline is a the end to end description of a transporter data flow.
// including the source, sink, and all the transformers along the way
type Pipeline struct {
	source        *Node
	emitter       events.Emitter
	metricsTicker *time.Ticker

	// Err is the fatal error that was sent from the adaptor
	// that caused us to stop this process.  If this is nil, then
	// the transporter is running
	Err error
}

// NewDefaultPipeline returns a new Transporter Pipeline with the given node tree, and
// uses the events.HttpPostEmitter to deliver metrics.
// eg.
//   source :=
//   	transporter.NewNode("source", "mongo", adaptor.Config{"uri": "mongodb://localhost/", "namespace": "boom.foo", "debug": false, "tail": true}).
// 	  	Add(transporter.NewNode("out", "file", adaptor.Config{"uri": "stdout://"}))
//   pipeline, err := transporter.NewDefaultPipeline(source, events.Api{URI: "http://localhost/endpoint"}, 1*time.Second)
//   if err != nil {
// 	  fmt.Println(err)
// 	  os.Exit(1)
//   }
// pipeline.Run()
func NewDefaultPipeline(source *Node, uri, key, pid string, interval time.Duration) (*Pipeline, error) {
	emitter := events.NewHTTPPostEmitter(uri, key, pid)
	return NewPipeline(source, emitter, interval)
}

// NewPipeline creates a new Transporter Pipeline using the given tree of nodes, and Event Emitter
// eg.
//   source :=
//   	transporter.NewNode("source", "mongo", adaptor.Config{"uri": "mongodb://localhost/", "namespace": "boom.foo", "debug": false, "tail": true}).
// 	  	Add(transporter.NewNode("out", "file", adaptor.Config{"uri": "stdout://"}))
//   pipeline, err := transporter.NewPipeline(source, events.NewNoopEmitter(), 1*time.Second)
//   if err != nil {
// 	  fmt.Println(err)
// 	  os.Exit(1)
//   }
// pipeline.Run()
func NewPipeline(source *Node, emitter events.Emitter, interval time.Duration) (*Pipeline, error) {
	pipeline := &Pipeline{
		source:        source,
		emitter:       emitter,
		metricsTicker: time.NewTicker(interval),
	}

	// init the pipeline
	err := pipeline.source.Init(interval)
	if err != nil {
		return pipeline, err
	}

	// init the emitter with the right chan
	pipeline.emitter.Init(source.pipe.Event)

	// start the emitters
	go pipeline.startErrorListener(source.pipe.Err)
	go pipeline.startMetricsGatherer()
	pipeline.emitter.Start()

	return pipeline, nil
}

func (pipeline *Pipeline) String() string {
	out := pipeline.source.String()
	return out
}

// Stop sends a stop signal to the emitter and all the nodes, whether they are running or not.
// the node's database adaptors are expected to clean up after themselves, and stop will block until
// all nodes have stopped successfully
func (pipeline *Pipeline) Stop() {
	pipeline.source.Stop()
	pipeline.emitter.Stop()
	pipeline.metricsTicker.Stop()
}

// Run the pipeline
func (pipeline *Pipeline) Run() error {
	endpoints := pipeline.source.Endpoints()
	// send a boot event
	pipeline.source.pipe.Event <- events.NewBootEvent(time.Now().Unix(), VERSION, endpoints)

	// start the source
	err := pipeline.source.Start()
	if err != nil && pipeline.Err == nil {
		pipeline.Err = err // only set it if it hasn't been set already.
	}

	// pipeline has stopped, emit one last round of metrics and send the exit event
	pipeline.emitMetrics()
	pipeline.source.pipe.Event <- events.NewExitEvent(time.Now().Unix(), VERSION, endpoints)

	// the source has exited, stop all the other nodes
	pipeline.Stop()

	return pipeline.Err
}

// start error listener consumes all the events on the pipe's Err channel, and stops the pipeline
// when it receives one
func (pipeline *Pipeline) startErrorListener(cherr chan error) {
	for err := range cherr {
		if aerr, ok := err.(adaptor.Error); ok {
			pipeline.source.pipe.Event <- events.NewErrorEvent(time.Now().Unix(), aerr.Path, aerr.Record, aerr.Error())
			if aerr.Lvl == adaptor.ERROR || aerr.Lvl == adaptor.CRITICAL {
				log.Println(aerr)
			}
		} else {
			if pipeline.Err == nil {
				pipeline.Err = err
			}
			pipeline.Stop()
		}
	}
}

func (pipeline *Pipeline) startMetricsGatherer() {
	for _ = range pipeline.metricsTicker.C {
		pipeline.emitMetrics()
	}
}

// emit the metrics
func (pipeline *Pipeline) emitMetrics() {

	frontier := make([]*Node, 1)
	frontier[0] = pipeline.source

	for {
		// pop the first item
		node := frontier[0]
		frontier = frontier[1:]

		// do something with the node
		pipeline.source.pipe.Event <- events.NewMetricsEvent(time.Now().Unix(), node.Path(), node.pipe.MessageCount)

		// add this nodes children to the frontier
		for _, child := range node.Children {
			frontier = append(frontier, child)
		}

		// if we're empty
		if len(frontier) == 0 {
			break
		}
	}
}
<commit_msg>0.0.3 release<commit_after>package transporter

import (
	"log"
	"time"

	"github.com/compose/transporter/pkg/adaptor"
	"github.com/compose/transporter/pkg/events"
)

// VERSION the library
const (
	VERSION = "0.0.3"
)

// A Pipeline is a the end to end description of a transporter data flow.
// including the source, sink, and all the transformers along the way
type Pipeline struct {
	source        *Node
	emitter       events.Emitter
	metricsTicker *time.Ticker

	// Err is the fatal error that was sent from the adaptor
	// that caused us to stop this process.  If this is nil, then
	// the transporter is running
	Err error
}

// NewDefaultPipeline returns a new Transporter Pipeline with the given node tree, and
// uses the events.HttpPostEmitter to deliver metrics.
// eg.
//   source :=
//   	transporter.NewNode("source", "mongo", adaptor.Config{"uri": "mongodb://localhost/", "namespace": "boom.foo", "debug": false, "tail": true}).
// 	  	Add(transporter.NewNode("out", "file", adaptor.Config{"uri": "stdout://"}))
//   pipeline, err := transporter.NewDefaultPipeline(source, events.Api{URI: "http://localhost/endpoint"}, 1*time.Second)
//   if err != nil {
// 	  fmt.Println(err)
// 	  os.Exit(1)
//   }
// pipeline.Run()
func NewDefaultPipeline(source *Node, uri, key, pid string, interval time.Duration) (*Pipeline, error) {
	emitter := events.NewHTTPPostEmitter(uri, key, pid)
	return NewPipeline(source, emitter, interval)
}

// NewPipeline creates a new Transporter Pipeline using the given tree of nodes, and Event Emitter
// eg.
//   source :=
//   	transporter.NewNode("source", "mongo", adaptor.Config{"uri": "mongodb://localhost/", "namespace": "boom.foo", "debug": false, "tail": true}).
// 	  	Add(transporter.NewNode("out", "file", adaptor.Config{"uri": "stdout://"}))
//   pipeline, err := transporter.NewPipeline(source, events.NewNoopEmitter(), 1*time.Second)
//   if err != nil {
// 	  fmt.Println(err)
// 	  os.Exit(1)
//   }
// pipeline.Run()
func NewPipeline(source *Node, emitter events.Emitter, interval time.Duration) (*Pipeline, error) {
	pipeline := &Pipeline{
		source:        source,
		emitter:       emitter,
		metricsTicker: time.NewTicker(interval),
	}

	// init the pipeline
	err := pipeline.source.Init(interval)
	if err != nil {
		return pipeline, err
	}

	// init the emitter with the right chan
	pipeline.emitter.Init(source.pipe.Event)

	// start the emitters
	go pipeline.startErrorListener(source.pipe.Err)
	go pipeline.startMetricsGatherer()
	pipeline.emitter.Start()

	return pipeline, nil
}

func (pipeline *Pipeline) String() string {
	out := pipeline.source.String()
	return out
}

// Stop sends a stop signal to the emitter and all the nodes, whether they are running or not.
// the node's database adaptors are expected to clean up after themselves, and stop will block until
// all nodes have stopped successfully
func (pipeline *Pipeline) Stop() {
	pipeline.source.Stop()
	pipeline.emitter.Stop()
	pipeline.metricsTicker.Stop()
}

// Run the pipeline
func (pipeline *Pipeline) Run() error {
	endpoints := pipeline.source.Endpoints()
	// send a boot event
	pipeline.source.pipe.Event <- events.NewBootEvent(time.Now().Unix(), VERSION, endpoints)

	// start the source
	err := pipeline.source.Start()
	if err != nil && pipeline.Err == nil {
		pipeline.Err = err // only set it if it hasn't been set already.
	}

	// pipeline has stopped, emit one last round of metrics and send the exit event
	pipeline.emitMetrics()
	pipeline.source.pipe.Event <- events.NewExitEvent(time.Now().Unix(), VERSION, endpoints)

	// the source has exited, stop all the other nodes
	pipeline.Stop()

	return pipeline.Err
}

// start error listener consumes all the events on the pipe's Err channel, and stops the pipeline
// when it receives one
func (pipeline *Pipeline) startErrorListener(cherr chan error) {
	for err := range cherr {
		if aerr, ok := err.(adaptor.Error); ok {
			pipeline.source.pipe.Event <- events.NewErrorEvent(time.Now().Unix(), aerr.Path, aerr.Record, aerr.Error())
			if aerr.Lvl == adaptor.ERROR || aerr.Lvl == adaptor.CRITICAL {
				log.Println(aerr)
			}
		} else {
			if pipeline.Err == nil {
				pipeline.Err = err
			}
			pipeline.Stop()
		}
	}
}

func (pipeline *Pipeline) startMetricsGatherer() {
	for _ = range pipeline.metricsTicker.C {
		pipeline.emitMetrics()
	}
}

// emit the metrics
func (pipeline *Pipeline) emitMetrics() {

	frontier := make([]*Node, 1)
	frontier[0] = pipeline.source

	for {
		// pop the first item
		node := frontier[0]
		frontier = frontier[1:]

		// do something with the node
		pipeline.source.pipe.Event <- events.NewMetricsEvent(time.Now().Unix(), node.Path(), node.pipe.MessageCount)

		// add this nodes children to the frontier
		for _, child := range node.Children {
			frontier = append(frontier, child)
		}

		// if we're empty
		if len(frontier) == 0 {
			break
		}
	}
}
<|endoftext|>


<commit_before>/*
Copyright 2019 The Vitess Authors.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package dbconfigs

// This file contains logic for a pluggable credentials system.
// The default implementation is file based.
// The flags are global, but only programs that need to access the database
// link with this library, so we should be safe.

import (
	"encoding/json"
	"errors"
	"flag"
	"os"
	"os/signal"
	"strings"
	"sync"
	"syscall"
	"time"

	vaultapi "github.com/aquarapid/vaultlib"
	"github.com/spf13/pflag"

	"vitess.io/vitess/go/mysql"
	"vitess.io/vitess/go/vt/log"
	"vitess.io/vitess/go/vt/servenv"
)

var (
	dbCredentialsServer   = "file"
	dbCredentialsFile     string
	vaultAddr             string
	vaultTimeout          = 10 * time.Second
	vaultCACert           string
	vaultPath             string
	vaultCacheTTL         = 30 * time.Minute
	vaultTokenFile        string
	vaultRoleID           string
	vaultRoleSecretIDFile string
	vaultRoleMountPoint   = "approle"

	// ErrUnknownUser is returned by credential server when the
	// user doesn't exist
	ErrUnknownUser = errors.New("unknown user")

	cmdsWithDBCredentials = []string{
		"mysqlctl",
		"mysqlctld",
		"vtbackup",
		"vtcombo",
		"vtgr",
		"vttablet",
	}
)

// CredentialsServer is the interface for a credential server
type CredentialsServer interface {
	// GetUserAndPassword returns the user / password to use for a given
	// user. May return ErrUnknownUser. The user might be altered
	// to support versioned users.
	// Note this call needs to be thread safe, as we may call this from
	// multiple go routines.
	GetUserAndPassword(user string) (string, string, error)
}

// AllCredentialsServers contains all the known CredentialsServer
// implementations.  Note we will only access this after flags have
// been parsed.
var AllCredentialsServers = make(map[string]CredentialsServer)

func init() {
	AllCredentialsServers["file"] = &FileCredentialsServer{}
	AllCredentialsServers["vault"] = &VaultCredentialsServer{}

	sigChan := make(chan os.Signal, 1)
	signal.Notify(sigChan, syscall.SIGHUP)
	go func() {
		for range sigChan {
			if fcs, ok := AllCredentialsServers["file"].(*FileCredentialsServer); ok {
				fcs.mu.Lock()
				fcs.dbCredentials = nil
				fcs.mu.Unlock()
			}
			if vcs, ok := AllCredentialsServers["vault"].(*VaultCredentialsServer); ok {
				vcs.mu.Lock()
				vcs.dbCredsCache = nil
				vcs.mu.Unlock()
			}
		}
	}()

	for _, cmd := range cmdsWithDBCredentials {
		servenv.OnParseFor(cmd, func(fs *pflag.FlagSet) {
			// generic flags
			fs.StringVar(&dbCredentialsServer, "db-credentials-server", dbCredentialsServer, "db credentials server type ('file' - file implementation; 'vault' - HashiCorp Vault implementation)")

			// 'file' implementation flags
			fs.StringVar(&dbCredentialsFile, "db-credentials-file", dbCredentialsFile, "db credentials file; send SIGHUP to reload this file")

			// 'vault' implementation flags
			flag.StringVar(&vaultAddr, "db-credentials-vault-addr", vaultAddr, "URL to Vault server")
			flag.DurationVar(&vaultTimeout, "db-credentials-vault-timeout", vaultTimeout, "Timeout for vault API operations")
			flag.StringVar(&vaultCACert, "db-credentials-vault-tls-ca", vaultCACert, "Path to CA PEM for validating Vault server certificate")
			flag.StringVar(&vaultPath, "db-credentials-vault-path", vaultPath, "Vault path to credentials JSON blob, e.g.: secret/data/prod/dbcreds")
			flag.DurationVar(&vaultCacheTTL, "db-credentials-vault-ttl", vaultCacheTTL, "How long to cache DB credentials from the Vault server")
			flag.StringVar(&vaultTokenFile, "db-credentials-vault-tokenfile", vaultTokenFile, "Path to file containing Vault auth token; token can also be passed using VAULT_TOKEN environment variable")
			flag.StringVar(&vaultRoleID, "db-credentials-vault-roleid", vaultRoleID, "Vault AppRole id; can also be passed using VAULT_ROLEID environment variable")
			flag.StringVar(&vaultRoleSecretIDFile, "db-credentials-vault-role-secretidfile", vaultRoleSecretIDFile, "Path to file containing Vault AppRole secret_id; can also be passed using VAULT_SECRETID environment variable")
			flag.StringVar(&vaultRoleMountPoint, "db-credentials-vault-role-mountpoint", vaultRoleMountPoint, "Vault AppRole mountpoint; can also be passed using VAULT_MOUNTPOINT environment variable")
		})
	}
}

// GetCredentialsServer returns the current CredentialsServer. Only valid
// after flag.Init was called.
func GetCredentialsServer() CredentialsServer {
	cs, ok := AllCredentialsServers[dbCredentialsServer]
	if !ok {
		log.Exitf("Invalid credential server: %v", dbCredentialsServer)
	}
	return cs
}

// FileCredentialsServer is a simple implementation of CredentialsServer using
// a json file. Protected by mu.
type FileCredentialsServer struct {
	mu            sync.Mutex
	dbCredentials map[string][]string
}

// VaultCredentialsServer implements CredentialsServer using
// a Vault backend from HashiCorp.
type VaultCredentialsServer struct {
	mu                     sync.Mutex
	dbCredsCache           map[string][]string
	vaultCacheExpireTicker *time.Ticker
	vaultClient            *vaultapi.Client
	// We use a separate valid flag to allow invalidating the cache
	// without destroying it, in case Vault is temp down.
	cacheValid bool
}

// GetUserAndPassword is part of the CredentialsServer interface
func (fcs *FileCredentialsServer) GetUserAndPassword(user string) (string, string, error) {
	fcs.mu.Lock()
	defer fcs.mu.Unlock()

	if dbCredentialsFile == "" {
		return "", "", ErrUnknownUser
	}

	// read the json file only once
	if fcs.dbCredentials == nil {
		fcs.dbCredentials = make(map[string][]string)

		data, err := os.ReadFile(dbCredentialsFile)
		if err != nil {
			log.Warningf("Failed to read dbCredentials file: %v", dbCredentialsFile)
			return "", "", err
		}

		if err = json.Unmarshal(data, &fcs.dbCredentials); err != nil {
			log.Warningf("Failed to parse dbCredentials file: %v", dbCredentialsFile)
			return "", "", err
		}
	}

	passwd, ok := fcs.dbCredentials[user]
	if !ok {
		return "", "", ErrUnknownUser
	}
	return user, passwd[0], nil
}

// GetUserAndPassword for Vault implementation
func (vcs *VaultCredentialsServer) GetUserAndPassword(user string) (string, string, error) {
	vcs.mu.Lock()
	defer vcs.mu.Unlock()

	if vcs.vaultCacheExpireTicker == nil {
		vcs.vaultCacheExpireTicker = time.NewTicker(vaultCacheTTL)
		go func() {
			for range vcs.vaultCacheExpireTicker.C {
				if vcs, ok := AllCredentialsServers["vault"].(*VaultCredentialsServer); ok {
					vcs.cacheValid = false
				}
			}
		}()
	}

	if vcs.cacheValid && vcs.dbCredsCache != nil {
		if vcs.dbCredsCache[user] == nil {
			log.Errorf("Vault cache is valid, but user %s unknown in cache, will retry", user)
			return "", "", ErrUnknownUser
		}
		return user, vcs.dbCredsCache[user][0], nil
	}

	if vaultAddr == "" {
		return "", "", errors.New("No Vault server specified")
	}

	token, err := readFromFile(vaultTokenFile)
	if err != nil {
		return "", "", errors.New("No Vault token in provided filename")
	}
	secretID, err := readFromFile(vaultRoleSecretIDFile)
	if err != nil {
		return "", "", errors.New("No Vault secret_id in provided filename")
	}

	// From here on, errors might be transient, so we use ErrUnknownUser
	// for everything, so we get retries
	if vcs.vaultClient == nil {
		config := vaultapi.NewConfig()

		// All these can be overriden by environment
		//   so we need to check if they have been set by NewConfig
		if config.Address == "" {
			config.Address = vaultAddr
		}
		if config.Timeout == (0 * time.Second) {
			config.Timeout = vaultTimeout
		}
		if config.CACert == "" {
			config.CACert = vaultCACert
		}
		if config.Token == "" {
			config.Token = token
		}
		if config.AppRoleCredentials.RoleID == "" {
			config.AppRoleCredentials.RoleID = vaultRoleID
		}
		if config.AppRoleCredentials.SecretID == "" {
			config.AppRoleCredentials.SecretID = secretID
		}
		if config.AppRoleCredentials.MountPoint == "" {
			config.AppRoleCredentials.MountPoint = vaultRoleMountPoint
		}

		if config.CACert != "" {
			// If we provide a CA, ensure we actually use it
			config.InsecureSSL = false
		}

		var err error
		vcs.vaultClient, err = vaultapi.NewClient(config)
		if err != nil || vcs.vaultClient == nil {
			log.Errorf("Error in vault client initialization, will retry: %v", err)
			vcs.vaultClient = nil
			return "", "", ErrUnknownUser
		}
	}

	secret, err := vcs.vaultClient.GetSecret(vaultPath)
	if err != nil {
		log.Errorf("Error in Vault server params: %v", err)
		return "", "", ErrUnknownUser
	}

	if secret.JSONSecret == nil {
		log.Errorf("Empty DB credentials retrieved from Vault server")
		return "", "", ErrUnknownUser
	}

	dbCreds := make(map[string][]string)
	if err = json.Unmarshal(secret.JSONSecret, &dbCreds); err != nil {
		log.Errorf("Error unmarshaling DB credentials from Vault server")
		return "", "", ErrUnknownUser
	}
	if dbCreds[user] == nil {
		log.Warningf("Vault lookup for user not found: %v\n", user)
		return "", "", ErrUnknownUser
	}
	log.Infof("Vault client status: %s", vcs.vaultClient.GetStatus())

	vcs.dbCredsCache = dbCreds
	vcs.cacheValid = true
	return user, dbCreds[user][0], nil
}

func readFromFile(filePath string) (string, error) {
	if filePath == "" {
		return "", nil
	}
	fileBytes, err := os.ReadFile(filePath)
	if err != nil {
		return "", err
	}
	return strings.TrimSpace(string(fileBytes)), nil
}

// WithCredentials returns a copy of the provided ConnParams that we can use
// to connect, after going through the CredentialsServer.
func withCredentials(cp *mysql.ConnParams) (*mysql.ConnParams, error) {
	result := *cp
	user, passwd, err := GetCredentialsServer().GetUserAndPassword(cp.Uname)
	switch err {
	case nil:
		result.Uname = user
		result.Pass = passwd
	case ErrUnknownUser:
		// we just use what we have, and will fail later anyway
		// except if the actual password is empty, in which case
		// things will just "work"
		err = nil
	}
	return &result, err
}
<commit_msg>Fix missing flag usage (#11582)<commit_after>/*
Copyright 2019 The Vitess Authors.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package dbconfigs

// This file contains logic for a pluggable credentials system.
// The default implementation is file based.
// The flags are global, but only programs that need to access the database
// link with this library, so we should be safe.

import (
	"encoding/json"
	"errors"
	"os"
	"os/signal"
	"strings"
	"sync"
	"syscall"
	"time"

	vaultapi "github.com/aquarapid/vaultlib"
	"github.com/spf13/pflag"

	"vitess.io/vitess/go/mysql"
	"vitess.io/vitess/go/vt/log"
	"vitess.io/vitess/go/vt/servenv"
)

var (
	dbCredentialsServer   = "file"
	dbCredentialsFile     string
	vaultAddr             string
	vaultTimeout          = 10 * time.Second
	vaultCACert           string
	vaultPath             string
	vaultCacheTTL         = 30 * time.Minute
	vaultTokenFile        string
	vaultRoleID           string
	vaultRoleSecretIDFile string
	vaultRoleMountPoint   = "approle"

	// ErrUnknownUser is returned by credential server when the
	// user doesn't exist
	ErrUnknownUser = errors.New("unknown user")

	cmdsWithDBCredentials = []string{
		"mysqlctl",
		"mysqlctld",
		"vtbackup",
		"vtcombo",
		"vtgr",
		"vttablet",
	}
)

// CredentialsServer is the interface for a credential server
type CredentialsServer interface {
	// GetUserAndPassword returns the user / password to use for a given
	// user. May return ErrUnknownUser. The user might be altered
	// to support versioned users.
	// Note this call needs to be thread safe, as we may call this from
	// multiple go routines.
	GetUserAndPassword(user string) (string, string, error)
}

// AllCredentialsServers contains all the known CredentialsServer
// implementations.  Note we will only access this after flags have
// been parsed.
var AllCredentialsServers = make(map[string]CredentialsServer)

func init() {
	AllCredentialsServers["file"] = &FileCredentialsServer{}
	AllCredentialsServers["vault"] = &VaultCredentialsServer{}

	sigChan := make(chan os.Signal, 1)
	signal.Notify(sigChan, syscall.SIGHUP)
	go func() {
		for range sigChan {
			if fcs, ok := AllCredentialsServers["file"].(*FileCredentialsServer); ok {
				fcs.mu.Lock()
				fcs.dbCredentials = nil
				fcs.mu.Unlock()
			}
			if vcs, ok := AllCredentialsServers["vault"].(*VaultCredentialsServer); ok {
				vcs.mu.Lock()
				vcs.dbCredsCache = nil
				vcs.mu.Unlock()
			}
		}
	}()

	for _, cmd := range cmdsWithDBCredentials {
		servenv.OnParseFor(cmd, func(fs *pflag.FlagSet) {
			// generic flags
			fs.StringVar(&dbCredentialsServer, "db-credentials-server", dbCredentialsServer, "db credentials server type ('file' - file implementation; 'vault' - HashiCorp Vault implementation)")

			// 'file' implementation flags
			fs.StringVar(&dbCredentialsFile, "db-credentials-file", dbCredentialsFile, "db credentials file; send SIGHUP to reload this file")

			// 'vault' implementation flags
			fs.StringVar(&vaultAddr, "db-credentials-vault-addr", vaultAddr, "URL to Vault server")
			fs.DurationVar(&vaultTimeout, "db-credentials-vault-timeout", vaultTimeout, "Timeout for vault API operations")
			fs.StringVar(&vaultCACert, "db-credentials-vault-tls-ca", vaultCACert, "Path to CA PEM for validating Vault server certificate")
			fs.StringVar(&vaultPath, "db-credentials-vault-path", vaultPath, "Vault path to credentials JSON blob, e.g.: secret/data/prod/dbcreds")
			fs.DurationVar(&vaultCacheTTL, "db-credentials-vault-ttl", vaultCacheTTL, "How long to cache DB credentials from the Vault server")
			fs.StringVar(&vaultTokenFile, "db-credentials-vault-tokenfile", vaultTokenFile, "Path to file containing Vault auth token; token can also be passed using VAULT_TOKEN environment variable")
			fs.StringVar(&vaultRoleID, "db-credentials-vault-roleid", vaultRoleID, "Vault AppRole id; can also be passed using VAULT_ROLEID environment variable")
			fs.StringVar(&vaultRoleSecretIDFile, "db-credentials-vault-role-secretidfile", vaultRoleSecretIDFile, "Path to file containing Vault AppRole secret_id; can also be passed using VAULT_SECRETID environment variable")
			fs.StringVar(&vaultRoleMountPoint, "db-credentials-vault-role-mountpoint", vaultRoleMountPoint, "Vault AppRole mountpoint; can also be passed using VAULT_MOUNTPOINT environment variable")
		})
	}
}

// GetCredentialsServer returns the current CredentialsServer. Only valid
// after flag.Init was called.
func GetCredentialsServer() CredentialsServer {
	cs, ok := AllCredentialsServers[dbCredentialsServer]
	if !ok {
		log.Exitf("Invalid credential server: %v", dbCredentialsServer)
	}
	return cs
}

// FileCredentialsServer is a simple implementation of CredentialsServer using
// a json file. Protected by mu.
type FileCredentialsServer struct {
	mu            sync.Mutex
	dbCredentials map[string][]string
}

// VaultCredentialsServer implements CredentialsServer using
// a Vault backend from HashiCorp.
type VaultCredentialsServer struct {
	mu                     sync.Mutex
	dbCredsCache           map[string][]string
	vaultCacheExpireTicker *time.Ticker
	vaultClient            *vaultapi.Client
	// We use a separate valid flag to allow invalidating the cache
	// without destroying it, in case Vault is temp down.
	cacheValid bool
}

// GetUserAndPassword is part of the CredentialsServer interface
func (fcs *FileCredentialsServer) GetUserAndPassword(user string) (string, string, error) {
	fcs.mu.Lock()
	defer fcs.mu.Unlock()

	if dbCredentialsFile == "" {
		return "", "", ErrUnknownUser
	}

	// read the json file only once
	if fcs.dbCredentials == nil {
		fcs.dbCredentials = make(map[string][]string)

		data, err := os.ReadFile(dbCredentialsFile)
		if err != nil {
			log.Warningf("Failed to read dbCredentials file: %v", dbCredentialsFile)
			return "", "", err
		}

		if err = json.Unmarshal(data, &fcs.dbCredentials); err != nil {
			log.Warningf("Failed to parse dbCredentials file: %v", dbCredentialsFile)
			return "", "", err
		}
	}

	passwd, ok := fcs.dbCredentials[user]
	if !ok {
		return "", "", ErrUnknownUser
	}
	return user, passwd[0], nil
}

// GetUserAndPassword for Vault implementation
func (vcs *VaultCredentialsServer) GetUserAndPassword(user string) (string, string, error) {
	vcs.mu.Lock()
	defer vcs.mu.Unlock()

	if vcs.vaultCacheExpireTicker == nil {
		vcs.vaultCacheExpireTicker = time.NewTicker(vaultCacheTTL)
		go func() {
			for range vcs.vaultCacheExpireTicker.C {
				if vcs, ok := AllCredentialsServers["vault"].(*VaultCredentialsServer); ok {
					vcs.cacheValid = false
				}
			}
		}()
	}

	if vcs.cacheValid && vcs.dbCredsCache != nil {
		if vcs.dbCredsCache[user] == nil {
			log.Errorf("Vault cache is valid, but user %s unknown in cache, will retry", user)
			return "", "", ErrUnknownUser
		}
		return user, vcs.dbCredsCache[user][0], nil
	}

	if vaultAddr == "" {
		return "", "", errors.New("No Vault server specified")
	}

	token, err := readFromFile(vaultTokenFile)
	if err != nil {
		return "", "", errors.New("No Vault token in provided filename")
	}
	secretID, err := readFromFile(vaultRoleSecretIDFile)
	if err != nil {
		return "", "", errors.New("No Vault secret_id in provided filename")
	}

	// From here on, errors might be transient, so we use ErrUnknownUser
	// for everything, so we get retries
	if vcs.vaultClient == nil {
		config := vaultapi.NewConfig()

		// All these can be overriden by environment
		//   so we need to check if they have been set by NewConfig
		if config.Address == "" {
			config.Address = vaultAddr
		}
		if config.Timeout == (0 * time.Second) {
			config.Timeout = vaultTimeout
		}
		if config.CACert == "" {
			config.CACert = vaultCACert
		}
		if config.Token == "" {
			config.Token = token
		}
		if config.AppRoleCredentials.RoleID == "" {
			config.AppRoleCredentials.RoleID = vaultRoleID
		}
		if config.AppRoleCredentials.SecretID == "" {
			config.AppRoleCredentials.SecretID = secretID
		}
		if config.AppRoleCredentials.MountPoint == "" {
			config.AppRoleCredentials.MountPoint = vaultRoleMountPoint
		}

		if config.CACert != "" {
			// If we provide a CA, ensure we actually use it
			config.InsecureSSL = false
		}

		var err error
		vcs.vaultClient, err = vaultapi.NewClient(config)
		if err != nil || vcs.vaultClient == nil {
			log.Errorf("Error in vault client initialization, will retry: %v", err)
			vcs.vaultClient = nil
			return "", "", ErrUnknownUser
		}
	}

	secret, err := vcs.vaultClient.GetSecret(vaultPath)
	if err != nil {
		log.Errorf("Error in Vault server params: %v", err)
		return "", "", ErrUnknownUser
	}

	if secret.JSONSecret == nil {
		log.Errorf("Empty DB credentials retrieved from Vault server")
		return "", "", ErrUnknownUser
	}

	dbCreds := make(map[string][]string)
	if err = json.Unmarshal(secret.JSONSecret, &dbCreds); err != nil {
		log.Errorf("Error unmarshaling DB credentials from Vault server")
		return "", "", ErrUnknownUser
	}
	if dbCreds[user] == nil {
		log.Warningf("Vault lookup for user not found: %v\n", user)
		return "", "", ErrUnknownUser
	}
	log.Infof("Vault client status: %s", vcs.vaultClient.GetStatus())

	vcs.dbCredsCache = dbCreds
	vcs.cacheValid = true
	return user, dbCreds[user][0], nil
}

func readFromFile(filePath string) (string, error) {
	if filePath == "" {
		return "", nil
	}
	fileBytes, err := os.ReadFile(filePath)
	if err != nil {
		return "", err
	}
	return strings.TrimSpace(string(fileBytes)), nil
}

// WithCredentials returns a copy of the provided ConnParams that we can use
// to connect, after going through the CredentialsServer.
func withCredentials(cp *mysql.ConnParams) (*mysql.ConnParams, error) {
	result := *cp
	user, passwd, err := GetCredentialsServer().GetUserAndPassword(cp.Uname)
	switch err {
	case nil:
		result.Uname = user
		result.Pass = passwd
	case ErrUnknownUser:
		// we just use what we have, and will fail later anyway
		// except if the actual password is empty, in which case
		// things will just "work"
		err = nil
	}
	return &result, err
}
<|endoftext|>


<commit_before><commit_msg>Fixes for tests<commit_after><|endoftext|>


<commit_before>/*
Copyright 2014 The Kubernetes Authors All rights reserved.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package gce_pd

import (
	"fmt"
	"os"
	"path"
	"strconv"

	"github.com/golang/glog"
	"k8s.io/kubernetes/pkg/api"
	"k8s.io/kubernetes/pkg/api/resource"
	"k8s.io/kubernetes/pkg/types"
	"k8s.io/kubernetes/pkg/util/mount"
	"k8s.io/kubernetes/pkg/util/strings"
	"k8s.io/kubernetes/pkg/volume"
)

// This is the primary entrypoint for volume plugins.
func ProbeVolumePlugins() []volume.VolumePlugin {
	return []volume.VolumePlugin{&gcePersistentDiskPlugin{nil}}
}

type gcePersistentDiskPlugin struct {
	host volume.VolumeHost
}

var _ volume.VolumePlugin = &gcePersistentDiskPlugin{}
var _ volume.PersistentVolumePlugin = &gcePersistentDiskPlugin{}
var _ volume.DeletableVolumePlugin = &gcePersistentDiskPlugin{}
var _ volume.ProvisionableVolumePlugin = &gcePersistentDiskPlugin{}

const (
	gcePersistentDiskPluginName = "kubernetes.io/gce-pd"
)

func getPath(uid types.UID, volName string, host volume.VolumeHost) string {
	return host.GetPodVolumeDir(uid, strings.EscapeQualifiedNameForDisk(gcePersistentDiskPluginName), volName)
}

func (plugin *gcePersistentDiskPlugin) Init(host volume.VolumeHost) error {
	plugin.host = host
	return nil
}

func (plugin *gcePersistentDiskPlugin) GetPluginName() string {
	return gcePersistentDiskPluginName
}

func (plugin *gcePersistentDiskPlugin) GetVolumeName(spec *volume.Spec) (string, error) {
	volumeSource, _, err := getVolumeSource(spec)
	if err != nil {
		return "", err
	}

	return volumeSource.PDName, nil
}

func (plugin *gcePersistentDiskPlugin) CanSupport(spec *volume.Spec) bool {
	return (spec.PersistentVolume != nil && spec.PersistentVolume.Spec.GCEPersistentDisk != nil) ||
		(spec.Volume != nil && spec.Volume.GCEPersistentDisk != nil)
}

func (plugin *gcePersistentDiskPlugin) RequiresRemount() bool {
	return false
}

func (plugin *gcePersistentDiskPlugin) GetAccessModes() []api.PersistentVolumeAccessMode {
	return []api.PersistentVolumeAccessMode{
		api.ReadWriteOnce,
		api.ReadOnlyMany,
	}
}

func (plugin *gcePersistentDiskPlugin) NewMounter(spec *volume.Spec, pod *api.Pod, _ volume.VolumeOptions) (volume.Mounter, error) {
	// Inject real implementations here, test through the internal function.
	return plugin.newMounterInternal(spec, pod.UID, &GCEDiskUtil{}, plugin.host.GetMounter())
}

func getVolumeSource(
	spec *volume.Spec) (*api.GCEPersistentDiskVolumeSource, bool, error) {
	if spec.Volume != nil && spec.Volume.GCEPersistentDisk != nil {
		glog.V(4).Infof(
			"volume source %v spec %v, readonly flag retrieved from source: %v",
			spec.Volume.GCEPersistentDisk.PDName,
			spec.Name(),
			spec.Volume.GCEPersistentDisk.ReadOnly)
		return spec.Volume.GCEPersistentDisk, spec.Volume.GCEPersistentDisk.ReadOnly, nil
	} else if spec.PersistentVolume != nil &&
		spec.PersistentVolume.Spec.GCEPersistentDisk != nil {
		glog.V(4).Infof(
			"volume source %v spec %v, readonly flag retrieved from spec: %v",
			spec.PersistentVolume.Spec.GCEPersistentDisk.PDName,
			spec.Name(),
			spec.ReadOnly)
		return spec.PersistentVolume.Spec.GCEPersistentDisk, spec.ReadOnly, nil
	}

	return nil, false, fmt.Errorf("Spec does not reference a GCE volume type")
}

func (plugin *gcePersistentDiskPlugin) newMounterInternal(spec *volume.Spec, podUID types.UID, manager pdManager, mounter mount.Interface) (volume.Mounter, error) {
	// GCEPDs used directly in a pod have a ReadOnly flag set by the pod author.
	// GCEPDs used as a PersistentVolume gets the ReadOnly flag indirectly through the persistent-claim volume used to mount the PV
	volumeSource, readOnly, err := getVolumeSource(spec)
	if err != nil {
		return nil, err
	}

	pdName := volumeSource.PDName
	partition := ""
	if volumeSource.Partition != 0 {
		partition = strconv.Itoa(int(volumeSource.Partition))
	}

	return &gcePersistentDiskMounter{
		gcePersistentDisk: &gcePersistentDisk{
			podUID:          podUID,
			volName:         spec.Name(),
			pdName:          pdName,
			partition:       partition,
			mounter:         mounter,
			manager:         manager,
			plugin:          plugin,
			MetricsProvider: volume.NewMetricsStatFS(getPath(podUID, spec.Name(), plugin.host)),
		},
		readOnly: readOnly}, nil
}

func (plugin *gcePersistentDiskPlugin) NewUnmounter(volName string, podUID types.UID) (volume.Unmounter, error) {
	// Inject real implementations here, test through the internal function.
	return plugin.newUnmounterInternal(volName, podUID, &GCEDiskUtil{}, plugin.host.GetMounter())
}

func (plugin *gcePersistentDiskPlugin) newUnmounterInternal(volName string, podUID types.UID, manager pdManager, mounter mount.Interface) (volume.Unmounter, error) {
	return &gcePersistentDiskUnmounter{&gcePersistentDisk{
		podUID:          podUID,
		volName:         volName,
		manager:         manager,
		mounter:         mounter,
		plugin:          plugin,
		MetricsProvider: volume.NewMetricsStatFS(getPath(podUID, volName, plugin.host)),
	}}, nil
}

func (plugin *gcePersistentDiskPlugin) NewDeleter(spec *volume.Spec) (volume.Deleter, error) {
	return plugin.newDeleterInternal(spec, &GCEDiskUtil{})
}

func (plugin *gcePersistentDiskPlugin) newDeleterInternal(spec *volume.Spec, manager pdManager) (volume.Deleter, error) {
	if spec.PersistentVolume != nil && spec.PersistentVolume.Spec.GCEPersistentDisk == nil {
		return nil, fmt.Errorf("spec.PersistentVolumeSource.GCEPersistentDisk is nil")
	}
	return &gcePersistentDiskDeleter{
		gcePersistentDisk: &gcePersistentDisk{
			volName: spec.Name(),
			pdName:  spec.PersistentVolume.Spec.GCEPersistentDisk.PDName,
			manager: manager,
			plugin:  plugin,
		}}, nil
}

func (plugin *gcePersistentDiskPlugin) NewProvisioner(options volume.VolumeOptions) (volume.Provisioner, error) {
	if len(options.AccessModes) == 0 {
		options.AccessModes = plugin.GetAccessModes()
	}
	return plugin.newProvisionerInternal(options, &GCEDiskUtil{})
}

func (plugin *gcePersistentDiskPlugin) newProvisionerInternal(options volume.VolumeOptions, manager pdManager) (volume.Provisioner, error) {
	return &gcePersistentDiskProvisioner{
		gcePersistentDisk: &gcePersistentDisk{
			manager: manager,
			plugin:  plugin,
		},
		options: options,
	}, nil
}

// Abstract interface to PD operations.
type pdManager interface {
	// Creates a volume
	CreateVolume(provisioner *gcePersistentDiskProvisioner) (volumeID string, volumeSizeGB int, labels map[string]string, err error)
	// Deletes a volume
	DeleteVolume(deleter *gcePersistentDiskDeleter) error
}

// gcePersistentDisk volumes are disk resources provided by Google Compute Engine
// that are attached to the kubelet's host machine and exposed to the pod.
type gcePersistentDisk struct {
	volName string
	podUID  types.UID
	// Unique identifier of the PD, used to find the disk resource in the provider.
	pdName string
	// Specifies the partition to mount
	partition string
	// Utility interface to provision and delete disks
	manager pdManager
	// Mounter interface that provides system calls to mount the global path to the pod local path.
	mounter mount.Interface
	plugin  *gcePersistentDiskPlugin
	volume.MetricsProvider
}

type gcePersistentDiskMounter struct {
	*gcePersistentDisk
	// Specifies whether the disk will be mounted as read-only.
	readOnly bool
}

var _ volume.Mounter = &gcePersistentDiskMounter{}

func (b *gcePersistentDiskMounter) GetAttributes() volume.Attributes {
	return volume.Attributes{
		ReadOnly:        b.readOnly,
		Managed:         !b.readOnly,
		SupportsSELinux: true,
	}
}

// SetUp bind mounts the disk global mount to the volume path.
func (b *gcePersistentDiskMounter) SetUp(fsGroup *int64) error {
	return b.SetUpAt(b.GetPath(), fsGroup)
}

// SetUp bind mounts the disk global mount to the give volume path.
func (b *gcePersistentDiskMounter) SetUpAt(dir string, fsGroup *int64) error {
	// TODO: handle failed mounts here.
	notMnt, err := b.mounter.IsLikelyNotMountPoint(dir)
	glog.V(4).Infof("PersistentDisk set up: %s %v %v, pd name %v readOnly %v", dir, !notMnt, err, b.pdName, b.readOnly)
	if err != nil && !os.IsNotExist(err) {
		return err
	}
	if !notMnt {
		return nil
	}

	if err := os.MkdirAll(dir, 0750); err != nil {
		return err
	}

	// Perform a bind mount to the full path to allow duplicate mounts of the same PD.
	options := []string{"bind"}
	if b.readOnly {
		options = append(options, "ro")
	}

	globalPDPath := makeGlobalPDName(b.plugin.host, b.pdName)
	err = b.mounter.Mount(globalPDPath, dir, "", options)
	if err != nil {
		notMnt, mntErr := b.mounter.IsLikelyNotMountPoint(dir)
		if mntErr != nil {
			glog.Errorf("IsLikelyNotMountPoint check failed: %v", mntErr)
			return err
		}
		if !notMnt {
			if mntErr = b.mounter.Unmount(dir); mntErr != nil {
				glog.Errorf("Failed to unmount: %v", mntErr)
				return err
			}
			notMnt, mntErr := b.mounter.IsLikelyNotMountPoint(dir)
			if mntErr != nil {
				glog.Errorf("IsLikelyNotMountPoint check failed: %v", mntErr)
				return err
			}
			if !notMnt {
				// This is very odd, we don't expect it.  We'll try again next sync loop.
				glog.Errorf("%s is still mounted, despite call to unmount().  Will try again next sync loop.", dir)
				return err
			}
		}
		os.Remove(dir)
		return err
	}

	if !b.readOnly {
		volume.SetVolumeOwnership(b, fsGroup)
	}

	return nil
}

func makeGlobalPDName(host volume.VolumeHost, devName string) string {
	return path.Join(host.GetPluginDir(gcePersistentDiskPluginName), "mounts", devName)
}

func (b *gcePersistentDiskMounter) GetPath() string {
	return getPath(b.podUID, b.volName, b.plugin.host)
}

type gcePersistentDiskUnmounter struct {
	*gcePersistentDisk
}

var _ volume.Unmounter = &gcePersistentDiskUnmounter{}

func (c *gcePersistentDiskUnmounter) GetPath() string {
	return getPath(c.podUID, c.volName, c.plugin.host)
}

// Unmounts the bind mount, and detaches the disk only if the PD
// resource was the last reference to that disk on the kubelet.
func (c *gcePersistentDiskUnmounter) TearDown() error {
	return c.TearDownAt(c.GetPath())
}

// TearDownAt unmounts the bind mount
func (c *gcePersistentDiskUnmounter) TearDownAt(dir string) error {
	notMnt, err := c.mounter.IsLikelyNotMountPoint(dir)
	if err != nil {
		return err
	}
	if notMnt {
		return os.Remove(dir)
	}
	if err := c.mounter.Unmount(dir); err != nil {
		return err
	}
	notMnt, mntErr := c.mounter.IsLikelyNotMountPoint(dir)
	if mntErr != nil {
		glog.Errorf("IsLikelyNotMountPoint check failed: %v", mntErr)
		return err
	}
	if notMnt {
		return os.Remove(dir)
	}
	return fmt.Errorf("Failed to unmount volume dir")
}

type gcePersistentDiskDeleter struct {
	*gcePersistentDisk
}

var _ volume.Deleter = &gcePersistentDiskDeleter{}

func (d *gcePersistentDiskDeleter) GetPath() string {
	return getPath(d.podUID, d.volName, d.plugin.host)
}

func (d *gcePersistentDiskDeleter) Delete() error {
	return d.manager.DeleteVolume(d)
}

type gcePersistentDiskProvisioner struct {
	*gcePersistentDisk
	options volume.VolumeOptions
}

var _ volume.Provisioner = &gcePersistentDiskProvisioner{}

func (c *gcePersistentDiskProvisioner) Provision() (*api.PersistentVolume, error) {
	volumeID, sizeGB, labels, err := c.manager.CreateVolume(c)
	if err != nil {
		return nil, err
	}

	pv := &api.PersistentVolume{
		ObjectMeta: api.ObjectMeta{
			Name:   c.options.PVName,
			Labels: map[string]string{},
			Annotations: map[string]string{
				"kubernetes.io/createdby": "gce-pd-dynamic-provisioner",
			},
		},
		Spec: api.PersistentVolumeSpec{
			PersistentVolumeReclaimPolicy: c.options.PersistentVolumeReclaimPolicy,
			AccessModes:                   c.options.AccessModes,
			Capacity: api.ResourceList{
				api.ResourceName(api.ResourceStorage): resource.MustParse(fmt.Sprintf("%dGi", sizeGB)),
			},
			PersistentVolumeSource: api.PersistentVolumeSource{
				GCEPersistentDisk: &api.GCEPersistentDiskVolumeSource{
					PDName:    volumeID,
					Partition: 0,
					ReadOnly:  false,
				},
			},
		},
	}

	if len(labels) != 0 {
		if pv.Labels == nil {
			pv.Labels = make(map[string]string)
		}
		for k, v := range labels {
			pv.Labels[k] = v
		}
	}

	return pv, nil
}
<commit_msg>Remove spam log messages from gce pd<commit_after>/*
Copyright 2014 The Kubernetes Authors All rights reserved.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package gce_pd

import (
	"fmt"
	"os"
	"path"
	"strconv"

	"github.com/golang/glog"
	"k8s.io/kubernetes/pkg/api"
	"k8s.io/kubernetes/pkg/api/resource"
	"k8s.io/kubernetes/pkg/types"
	"k8s.io/kubernetes/pkg/util/mount"
	"k8s.io/kubernetes/pkg/util/strings"
	"k8s.io/kubernetes/pkg/volume"
)

// This is the primary entrypoint for volume plugins.
func ProbeVolumePlugins() []volume.VolumePlugin {
	return []volume.VolumePlugin{&gcePersistentDiskPlugin{nil}}
}

type gcePersistentDiskPlugin struct {
	host volume.VolumeHost
}

var _ volume.VolumePlugin = &gcePersistentDiskPlugin{}
var _ volume.PersistentVolumePlugin = &gcePersistentDiskPlugin{}
var _ volume.DeletableVolumePlugin = &gcePersistentDiskPlugin{}
var _ volume.ProvisionableVolumePlugin = &gcePersistentDiskPlugin{}

const (
	gcePersistentDiskPluginName = "kubernetes.io/gce-pd"
)

func getPath(uid types.UID, volName string, host volume.VolumeHost) string {
	return host.GetPodVolumeDir(uid, strings.EscapeQualifiedNameForDisk(gcePersistentDiskPluginName), volName)
}

func (plugin *gcePersistentDiskPlugin) Init(host volume.VolumeHost) error {
	plugin.host = host
	return nil
}

func (plugin *gcePersistentDiskPlugin) GetPluginName() string {
	return gcePersistentDiskPluginName
}

func (plugin *gcePersistentDiskPlugin) GetVolumeName(spec *volume.Spec) (string, error) {
	volumeSource, _, err := getVolumeSource(spec)
	if err != nil {
		return "", err
	}

	return volumeSource.PDName, nil
}

func (plugin *gcePersistentDiskPlugin) CanSupport(spec *volume.Spec) bool {
	return (spec.PersistentVolume != nil && spec.PersistentVolume.Spec.GCEPersistentDisk != nil) ||
		(spec.Volume != nil && spec.Volume.GCEPersistentDisk != nil)
}

func (plugin *gcePersistentDiskPlugin) RequiresRemount() bool {
	return false
}

func (plugin *gcePersistentDiskPlugin) GetAccessModes() []api.PersistentVolumeAccessMode {
	return []api.PersistentVolumeAccessMode{
		api.ReadWriteOnce,
		api.ReadOnlyMany,
	}
}

func (plugin *gcePersistentDiskPlugin) NewMounter(spec *volume.Spec, pod *api.Pod, _ volume.VolumeOptions) (volume.Mounter, error) {
	// Inject real implementations here, test through the internal function.
	return plugin.newMounterInternal(spec, pod.UID, &GCEDiskUtil{}, plugin.host.GetMounter())
}

func getVolumeSource(
	spec *volume.Spec) (*api.GCEPersistentDiskVolumeSource, bool, error) {
	if spec.Volume != nil && spec.Volume.GCEPersistentDisk != nil {
		return spec.Volume.GCEPersistentDisk, spec.Volume.GCEPersistentDisk.ReadOnly, nil
	} else if spec.PersistentVolume != nil &&
		spec.PersistentVolume.Spec.GCEPersistentDisk != nil {
		return spec.PersistentVolume.Spec.GCEPersistentDisk, spec.ReadOnly, nil
	}

	return nil, false, fmt.Errorf("Spec does not reference a GCE volume type")
}

func (plugin *gcePersistentDiskPlugin) newMounterInternal(spec *volume.Spec, podUID types.UID, manager pdManager, mounter mount.Interface) (volume.Mounter, error) {
	// GCEPDs used directly in a pod have a ReadOnly flag set by the pod author.
	// GCEPDs used as a PersistentVolume gets the ReadOnly flag indirectly through the persistent-claim volume used to mount the PV
	volumeSource, readOnly, err := getVolumeSource(spec)
	if err != nil {
		return nil, err
	}

	pdName := volumeSource.PDName
	partition := ""
	if volumeSource.Partition != 0 {
		partition = strconv.Itoa(int(volumeSource.Partition))
	}

	return &gcePersistentDiskMounter{
		gcePersistentDisk: &gcePersistentDisk{
			podUID:          podUID,
			volName:         spec.Name(),
			pdName:          pdName,
			partition:       partition,
			mounter:         mounter,
			manager:         manager,
			plugin:          plugin,
			MetricsProvider: volume.NewMetricsStatFS(getPath(podUID, spec.Name(), plugin.host)),
		},
		readOnly: readOnly}, nil
}

func (plugin *gcePersistentDiskPlugin) NewUnmounter(volName string, podUID types.UID) (volume.Unmounter, error) {
	// Inject real implementations here, test through the internal function.
	return plugin.newUnmounterInternal(volName, podUID, &GCEDiskUtil{}, plugin.host.GetMounter())
}

func (plugin *gcePersistentDiskPlugin) newUnmounterInternal(volName string, podUID types.UID, manager pdManager, mounter mount.Interface) (volume.Unmounter, error) {
	return &gcePersistentDiskUnmounter{&gcePersistentDisk{
		podUID:          podUID,
		volName:         volName,
		manager:         manager,
		mounter:         mounter,
		plugin:          plugin,
		MetricsProvider: volume.NewMetricsStatFS(getPath(podUID, volName, plugin.host)),
	}}, nil
}

func (plugin *gcePersistentDiskPlugin) NewDeleter(spec *volume.Spec) (volume.Deleter, error) {
	return plugin.newDeleterInternal(spec, &GCEDiskUtil{})
}

func (plugin *gcePersistentDiskPlugin) newDeleterInternal(spec *volume.Spec, manager pdManager) (volume.Deleter, error) {
	if spec.PersistentVolume != nil && spec.PersistentVolume.Spec.GCEPersistentDisk == nil {
		return nil, fmt.Errorf("spec.PersistentVolumeSource.GCEPersistentDisk is nil")
	}
	return &gcePersistentDiskDeleter{
		gcePersistentDisk: &gcePersistentDisk{
			volName: spec.Name(),
			pdName:  spec.PersistentVolume.Spec.GCEPersistentDisk.PDName,
			manager: manager,
			plugin:  plugin,
		}}, nil
}

func (plugin *gcePersistentDiskPlugin) NewProvisioner(options volume.VolumeOptions) (volume.Provisioner, error) {
	if len(options.AccessModes) == 0 {
		options.AccessModes = plugin.GetAccessModes()
	}
	return plugin.newProvisionerInternal(options, &GCEDiskUtil{})
}

func (plugin *gcePersistentDiskPlugin) newProvisionerInternal(options volume.VolumeOptions, manager pdManager) (volume.Provisioner, error) {
	return &gcePersistentDiskProvisioner{
		gcePersistentDisk: &gcePersistentDisk{
			manager: manager,
			plugin:  plugin,
		},
		options: options,
	}, nil
}

// Abstract interface to PD operations.
type pdManager interface {
	// Creates a volume
	CreateVolume(provisioner *gcePersistentDiskProvisioner) (volumeID string, volumeSizeGB int, labels map[string]string, err error)
	// Deletes a volume
	DeleteVolume(deleter *gcePersistentDiskDeleter) error
}

// gcePersistentDisk volumes are disk resources provided by Google Compute Engine
// that are attached to the kubelet's host machine and exposed to the pod.
type gcePersistentDisk struct {
	volName string
	podUID  types.UID
	// Unique identifier of the PD, used to find the disk resource in the provider.
	pdName string
	// Specifies the partition to mount
	partition string
	// Utility interface to provision and delete disks
	manager pdManager
	// Mounter interface that provides system calls to mount the global path to the pod local path.
	mounter mount.Interface
	plugin  *gcePersistentDiskPlugin
	volume.MetricsProvider
}

type gcePersistentDiskMounter struct {
	*gcePersistentDisk
	// Specifies whether the disk will be mounted as read-only.
	readOnly bool
}

var _ volume.Mounter = &gcePersistentDiskMounter{}

func (b *gcePersistentDiskMounter) GetAttributes() volume.Attributes {
	return volume.Attributes{
		ReadOnly:        b.readOnly,
		Managed:         !b.readOnly,
		SupportsSELinux: true,
	}
}

// SetUp bind mounts the disk global mount to the volume path.
func (b *gcePersistentDiskMounter) SetUp(fsGroup *int64) error {
	return b.SetUpAt(b.GetPath(), fsGroup)
}

// SetUp bind mounts the disk global mount to the give volume path.
func (b *gcePersistentDiskMounter) SetUpAt(dir string, fsGroup *int64) error {
	// TODO: handle failed mounts here.
	notMnt, err := b.mounter.IsLikelyNotMountPoint(dir)
	glog.V(4).Infof("PersistentDisk set up: %s %v %v, pd name %v readOnly %v", dir, !notMnt, err, b.pdName, b.readOnly)
	if err != nil && !os.IsNotExist(err) {
		return err
	}
	if !notMnt {
		return nil
	}

	if err := os.MkdirAll(dir, 0750); err != nil {
		return err
	}

	// Perform a bind mount to the full path to allow duplicate mounts of the same PD.
	options := []string{"bind"}
	if b.readOnly {
		options = append(options, "ro")
	}

	globalPDPath := makeGlobalPDName(b.plugin.host, b.pdName)
	err = b.mounter.Mount(globalPDPath, dir, "", options)
	if err != nil {
		notMnt, mntErr := b.mounter.IsLikelyNotMountPoint(dir)
		if mntErr != nil {
			glog.Errorf("IsLikelyNotMountPoint check failed: %v", mntErr)
			return err
		}
		if !notMnt {
			if mntErr = b.mounter.Unmount(dir); mntErr != nil {
				glog.Errorf("Failed to unmount: %v", mntErr)
				return err
			}
			notMnt, mntErr := b.mounter.IsLikelyNotMountPoint(dir)
			if mntErr != nil {
				glog.Errorf("IsLikelyNotMountPoint check failed: %v", mntErr)
				return err
			}
			if !notMnt {
				// This is very odd, we don't expect it.  We'll try again next sync loop.
				glog.Errorf("%s is still mounted, despite call to unmount().  Will try again next sync loop.", dir)
				return err
			}
		}
		os.Remove(dir)
		return err
	}

	if !b.readOnly {
		volume.SetVolumeOwnership(b, fsGroup)
	}

	return nil
}

func makeGlobalPDName(host volume.VolumeHost, devName string) string {
	return path.Join(host.GetPluginDir(gcePersistentDiskPluginName), "mounts", devName)
}

func (b *gcePersistentDiskMounter) GetPath() string {
	return getPath(b.podUID, b.volName, b.plugin.host)
}

type gcePersistentDiskUnmounter struct {
	*gcePersistentDisk
}

var _ volume.Unmounter = &gcePersistentDiskUnmounter{}

func (c *gcePersistentDiskUnmounter) GetPath() string {
	return getPath(c.podUID, c.volName, c.plugin.host)
}

// Unmounts the bind mount, and detaches the disk only if the PD
// resource was the last reference to that disk on the kubelet.
func (c *gcePersistentDiskUnmounter) TearDown() error {
	return c.TearDownAt(c.GetPath())
}

// TearDownAt unmounts the bind mount
func (c *gcePersistentDiskUnmounter) TearDownAt(dir string) error {
	notMnt, err := c.mounter.IsLikelyNotMountPoint(dir)
	if err != nil {
		return err
	}
	if notMnt {
		return os.Remove(dir)
	}
	if err := c.mounter.Unmount(dir); err != nil {
		return err
	}
	notMnt, mntErr := c.mounter.IsLikelyNotMountPoint(dir)
	if mntErr != nil {
		glog.Errorf("IsLikelyNotMountPoint check failed: %v", mntErr)
		return err
	}
	if notMnt {
		return os.Remove(dir)
	}
	return fmt.Errorf("Failed to unmount volume dir")
}

type gcePersistentDiskDeleter struct {
	*gcePersistentDisk
}

var _ volume.Deleter = &gcePersistentDiskDeleter{}

func (d *gcePersistentDiskDeleter) GetPath() string {
	return getPath(d.podUID, d.volName, d.plugin.host)
}

func (d *gcePersistentDiskDeleter) Delete() error {
	return d.manager.DeleteVolume(d)
}

type gcePersistentDiskProvisioner struct {
	*gcePersistentDisk
	options volume.VolumeOptions
}

var _ volume.Provisioner = &gcePersistentDiskProvisioner{}

func (c *gcePersistentDiskProvisioner) Provision() (*api.PersistentVolume, error) {
	volumeID, sizeGB, labels, err := c.manager.CreateVolume(c)
	if err != nil {
		return nil, err
	}

	pv := &api.PersistentVolume{
		ObjectMeta: api.ObjectMeta{
			Name:   c.options.PVName,
			Labels: map[string]string{},
			Annotations: map[string]string{
				"kubernetes.io/createdby": "gce-pd-dynamic-provisioner",
			},
		},
		Spec: api.PersistentVolumeSpec{
			PersistentVolumeReclaimPolicy: c.options.PersistentVolumeReclaimPolicy,
			AccessModes:                   c.options.AccessModes,
			Capacity: api.ResourceList{
				api.ResourceName(api.ResourceStorage): resource.MustParse(fmt.Sprintf("%dGi", sizeGB)),
			},
			PersistentVolumeSource: api.PersistentVolumeSource{
				GCEPersistentDisk: &api.GCEPersistentDiskVolumeSource{
					PDName:    volumeID,
					Partition: 0,
					ReadOnly:  false,
				},
			},
		},
	}

	if len(labels) != 0 {
		if pv.Labels == nil {
			pv.Labels = make(map[string]string)
		}
		for k, v := range labels {
			pv.Labels[k] = v
		}
	}

	return pv, nil
}
<|endoftext|>


<commit_before>/*
Copyright 2017 Google Inc.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

// Package topoproto contains utility functions to deal with the proto3
// structures defined in proto/topodata.
package topoproto

import (
	"fmt"
	"net"
	"sort"
	"strconv"
	"strings"

	"github.com/golang/protobuf/proto"

	topodatapb "github.com/youtube/vitess/go/vt/proto/topodata"
)

// This file contains the topodata.Tablet utility functions.

const (
	// Default name for databases is the prefix plus keyspace
	vtDbPrefix = "vt_"
)

// cache the conversion from tablet type enum to lower case string.
var tabletTypeLowerName map[int32]string

func init() {
	tabletTypeLowerName = make(map[int32]string, len(topodatapb.TabletType_name))
	for k, v := range topodatapb.TabletType_name {
		tabletTypeLowerName[k] = strings.ToLower(v)
	}
}

// TabletAliasIsZero returns true iff cell and uid are empty
func TabletAliasIsZero(ta *topodatapb.TabletAlias) bool {
	return ta == nil || (ta.Cell == "" && ta.Uid == 0)
}

// TabletAliasEqual returns true if two TabletAlias match
func TabletAliasEqual(left, right *topodatapb.TabletAlias) bool {
	return proto.Equal(left, right)
}

// TabletAliasString formats a TabletAlias
func TabletAliasString(ta *topodatapb.TabletAlias) string {
	if ta == nil {
		return "<nil>"
	}
	return fmt.Sprintf("%v-%010d", ta.Cell, ta.Uid)
}

// TabletAliasUIDStr returns a string version of the uid
func TabletAliasUIDStr(ta *topodatapb.TabletAlias) string {
	return fmt.Sprintf("%010d", ta.Uid)
}

// ParseTabletAlias returns a TabletAlias for the input string,
// of the form <cell>-<uid>
func ParseTabletAlias(aliasStr string) (*topodatapb.TabletAlias, error) {
	nameParts := strings.Split(aliasStr, "-")
	if len(nameParts) != 2 {
		return nil, fmt.Errorf("invalid tablet alias: %v", aliasStr)
	}
	uid, err := ParseUID(nameParts[1])
	if err != nil {
		return nil, fmt.Errorf("invalid tablet uid %v: %v", aliasStr, err)
	}
	return &topodatapb.TabletAlias{
		Cell: nameParts[0],
		Uid:  uid,
	}, nil
}

// ParseUID parses just the uid (a number)
func ParseUID(value string) (uint32, error) {
	uid, err := strconv.ParseUint(value, 10, 32)
	if err != nil {
		return 0, fmt.Errorf("bad tablet uid %v", err)
	}
	return uint32(uid), nil
}

// TabletAliasList is used mainly for sorting
type TabletAliasList []*topodatapb.TabletAlias

// Len is part of sort.Interface
func (tal TabletAliasList) Len() int {
	return len(tal)
}

// Less is part of sort.Interface
func (tal TabletAliasList) Less(i, j int) bool {
	if tal[i].Cell < tal[j].Cell {
		return true
	} else if tal[i].Cell > tal[j].Cell {
		return false
	}
	return tal[i].Uid < tal[j].Uid
}

// Swap is part of sort.Interface
func (tal TabletAliasList) Swap(i, j int) {
	tal[i], tal[j] = tal[j], tal[i]
}

// AllTabletTypes lists all the possible tablet types
var AllTabletTypes = []topodatapb.TabletType{
	topodatapb.TabletType_MASTER,
	topodatapb.TabletType_REPLICA,
	topodatapb.TabletType_RDONLY,
	topodatapb.TabletType_BATCH,
	topodatapb.TabletType_SPARE,
	topodatapb.TabletType_EXPERIMENTAL,
	topodatapb.TabletType_BACKUP,
	topodatapb.TabletType_RESTORE,
	topodatapb.TabletType_DRAINED,
}

// SlaveTabletTypes contains all the tablet type that can have replication
// enabled.
var SlaveTabletTypes = []topodatapb.TabletType{
	topodatapb.TabletType_REPLICA,
	topodatapb.TabletType_RDONLY,
	topodatapb.TabletType_BATCH,
	topodatapb.TabletType_SPARE,
	topodatapb.TabletType_EXPERIMENTAL,
	topodatapb.TabletType_BACKUP,
	topodatapb.TabletType_RESTORE,
	topodatapb.TabletType_DRAINED,
}

// ParseTabletType parses the tablet type into the enum.
func ParseTabletType(param string) (topodatapb.TabletType, error) {
	value, ok := topodatapb.TabletType_value[strings.ToUpper(param)]
	if !ok {
		return topodatapb.TabletType_UNKNOWN, fmt.Errorf("unknown TabletType %v", param)
	}
	return topodatapb.TabletType(value), nil
}

// ParseTabletTypes parses a comma separated list of tablet types and returns a slice with the respective enums.
func ParseTabletTypes(param string) ([]topodatapb.TabletType, error) {
	var tabletTypes []topodatapb.TabletType
	for _, typeStr := range strings.Split(param, ",") {
		t, err := ParseTabletType(typeStr)
		if err != nil {
			return nil, err
		}
		tabletTypes = append(tabletTypes, t)
	}
	return tabletTypes, nil
}

// TabletTypeLString returns a lower case version of the tablet type,
// or "unknown" if not known.
func TabletTypeLString(tabletType topodatapb.TabletType) string {
	value, ok := tabletTypeLowerName[int32(tabletType)]
	if !ok {
		return "unknown"
	}
	return value
}

// IsTypeInList returns true if the given type is in the list.
// Use it with AllTabletType and SlaveTabletType for instance.
func IsTypeInList(tabletType topodatapb.TabletType, types []topodatapb.TabletType) bool {
	for _, t := range types {
		if tabletType == t {
			return true
		}
	}
	return false
}

// MakeStringTypeList returns a list of strings that match the input list.
func MakeStringTypeList(types []topodatapb.TabletType) []string {
	strs := make([]string, len(types))
	for i, t := range types {
		strs[i] = strings.ToLower(t.String())
	}
	sort.Strings(strs)
	return strs
}

// SetMysqlPort sets the mysql port for tablet. This function
// also handles legacy by setting the port in PortMap.
// TODO(sougou); deprecate this function after 3.0.
func SetMysqlPort(tablet *topodatapb.Tablet, port int32) {
	if tablet.MysqlHostname == "" || tablet.MysqlHostname == tablet.Hostname {
		tablet.PortMap["mysql"] = port
	}
	// If it's the legacy form, preserve old behavior to prevent
	// confusion between new and old code.
	if tablet.MysqlHostname != "" {
		tablet.MysqlPort = port
	}
}

// MysqlAddr returns the host:port of the mysql server.
func MysqlAddr(tablet *topodatapb.Tablet) string {
	return fmt.Sprintf("%v:%v", MysqlHostname(tablet), MysqlPort(tablet))
}

// MysqlHostname returns the mysql host name. This function
// also handles legacy behavior: it uses the tablet's hostname
// if MysqlHostname is not specified.
// TODO(sougou); deprecate this function after 3.0.
func MysqlHostname(tablet *topodatapb.Tablet) string {
	if tablet.MysqlHostname == "" {
		return tablet.Hostname
	}
	return tablet.MysqlHostname
}

// MysqlPort returns the mysql port. This function
// also handles legacy behavior: it uses the tablet's port map
// if MysqlHostname is not specified.
// TODO(sougou); deprecate this function after 3.0.
func MysqlPort(tablet *topodatapb.Tablet) int32 {
	if tablet.MysqlHostname == "" {
		return tablet.PortMap["mysql"]
	}
	return tablet.MysqlPort
}

// MySQLIP returns the MySQL server's IP by resolvign the host name.
func MySQLIP(tablet *topodatapb.Tablet) (string, error) {
	ipAddrs, err := net.LookupHost(MysqlHostname(tablet))
	if err != nil {
		return "", err
	}
	return ipAddrs[0], nil
}

// TabletDbName is usually implied by keyspace. Having the shard
// information in the database name complicates mysql replication.
func TabletDbName(tablet *topodatapb.Tablet) string {
	if tablet.DbNameOverride != "" {
		return tablet.DbNameOverride
	}
	if tablet.Keyspace == "" {
		return ""
	}
	return vtDbPrefix + tablet.Keyspace
}

// TabletIsAssigned returns if this tablet is assigned to a keyspace and shard.
// A "scrap" node will show up as assigned even though its data cannot be used
// for serving.
func TabletIsAssigned(tablet *topodatapb.Tablet) bool {
	return tablet != nil && tablet.Keyspace != "" && tablet.Shard != ""
}
<commit_msg>ParseTabletAlias: Help the user by explaining the expected format.<commit_after>/*
Copyright 2017 Google Inc.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

// Package topoproto contains utility functions to deal with the proto3
// structures defined in proto/topodata.
package topoproto

import (
	"fmt"
	"net"
	"sort"
	"strconv"
	"strings"

	"github.com/golang/protobuf/proto"

	topodatapb "github.com/youtube/vitess/go/vt/proto/topodata"
)

// This file contains the topodata.Tablet utility functions.

const (
	// Default name for databases is the prefix plus keyspace
	vtDbPrefix = "vt_"
)

// cache the conversion from tablet type enum to lower case string.
var tabletTypeLowerName map[int32]string

func init() {
	tabletTypeLowerName = make(map[int32]string, len(topodatapb.TabletType_name))
	for k, v := range topodatapb.TabletType_name {
		tabletTypeLowerName[k] = strings.ToLower(v)
	}
}

// TabletAliasIsZero returns true iff cell and uid are empty
func TabletAliasIsZero(ta *topodatapb.TabletAlias) bool {
	return ta == nil || (ta.Cell == "" && ta.Uid == 0)
}

// TabletAliasEqual returns true if two TabletAlias match
func TabletAliasEqual(left, right *topodatapb.TabletAlias) bool {
	return proto.Equal(left, right)
}

// TabletAliasString formats a TabletAlias
func TabletAliasString(ta *topodatapb.TabletAlias) string {
	if ta == nil {
		return "<nil>"
	}
	return fmt.Sprintf("%v-%010d", ta.Cell, ta.Uid)
}

// TabletAliasUIDStr returns a string version of the uid
func TabletAliasUIDStr(ta *topodatapb.TabletAlias) string {
	return fmt.Sprintf("%010d", ta.Uid)
}

// ParseTabletAlias returns a TabletAlias for the input string,
// of the form <cell>-<uid>
func ParseTabletAlias(aliasStr string) (*topodatapb.TabletAlias, error) {
	nameParts := strings.Split(aliasStr, "-")
	if len(nameParts) != 2 {
		return nil, fmt.Errorf("invalid tablet alias: %q, expecting format: %q", aliasStr, "<cell>-<uid>")
	}
	uid, err := ParseUID(nameParts[1])
	if err != nil {
		return nil, fmt.Errorf("invalid tablet uid %v: %v", aliasStr, err)
	}
	return &topodatapb.TabletAlias{
		Cell: nameParts[0],
		Uid:  uid,
	}, nil
}

// ParseUID parses just the uid (a number)
func ParseUID(value string) (uint32, error) {
	uid, err := strconv.ParseUint(value, 10, 32)
	if err != nil {
		return 0, fmt.Errorf("bad tablet uid %v", err)
	}
	return uint32(uid), nil
}

// TabletAliasList is used mainly for sorting
type TabletAliasList []*topodatapb.TabletAlias

// Len is part of sort.Interface
func (tal TabletAliasList) Len() int {
	return len(tal)
}

// Less is part of sort.Interface
func (tal TabletAliasList) Less(i, j int) bool {
	if tal[i].Cell < tal[j].Cell {
		return true
	} else if tal[i].Cell > tal[j].Cell {
		return false
	}
	return tal[i].Uid < tal[j].Uid
}

// Swap is part of sort.Interface
func (tal TabletAliasList) Swap(i, j int) {
	tal[i], tal[j] = tal[j], tal[i]
}

// AllTabletTypes lists all the possible tablet types
var AllTabletTypes = []topodatapb.TabletType{
	topodatapb.TabletType_MASTER,
	topodatapb.TabletType_REPLICA,
	topodatapb.TabletType_RDONLY,
	topodatapb.TabletType_BATCH,
	topodatapb.TabletType_SPARE,
	topodatapb.TabletType_EXPERIMENTAL,
	topodatapb.TabletType_BACKUP,
	topodatapb.TabletType_RESTORE,
	topodatapb.TabletType_DRAINED,
}

// SlaveTabletTypes contains all the tablet type that can have replication
// enabled.
var SlaveTabletTypes = []topodatapb.TabletType{
	topodatapb.TabletType_REPLICA,
	topodatapb.TabletType_RDONLY,
	topodatapb.TabletType_BATCH,
	topodatapb.TabletType_SPARE,
	topodatapb.TabletType_EXPERIMENTAL,
	topodatapb.TabletType_BACKUP,
	topodatapb.TabletType_RESTORE,
	topodatapb.TabletType_DRAINED,
}

// ParseTabletType parses the tablet type into the enum.
func ParseTabletType(param string) (topodatapb.TabletType, error) {
	value, ok := topodatapb.TabletType_value[strings.ToUpper(param)]
	if !ok {
		return topodatapb.TabletType_UNKNOWN, fmt.Errorf("unknown TabletType %v", param)
	}
	return topodatapb.TabletType(value), nil
}

// ParseTabletTypes parses a comma separated list of tablet types and returns a slice with the respective enums.
func ParseTabletTypes(param string) ([]topodatapb.TabletType, error) {
	var tabletTypes []topodatapb.TabletType
	for _, typeStr := range strings.Split(param, ",") {
		t, err := ParseTabletType(typeStr)
		if err != nil {
			return nil, err
		}
		tabletTypes = append(tabletTypes, t)
	}
	return tabletTypes, nil
}

// TabletTypeLString returns a lower case version of the tablet type,
// or "unknown" if not known.
func TabletTypeLString(tabletType topodatapb.TabletType) string {
	value, ok := tabletTypeLowerName[int32(tabletType)]
	if !ok {
		return "unknown"
	}
	return value
}

// IsTypeInList returns true if the given type is in the list.
// Use it with AllTabletType and SlaveTabletType for instance.
func IsTypeInList(tabletType topodatapb.TabletType, types []topodatapb.TabletType) bool {
	for _, t := range types {
		if tabletType == t {
			return true
		}
	}
	return false
}

// MakeStringTypeList returns a list of strings that match the input list.
func MakeStringTypeList(types []topodatapb.TabletType) []string {
	strs := make([]string, len(types))
	for i, t := range types {
		strs[i] = strings.ToLower(t.String())
	}
	sort.Strings(strs)
	return strs
}

// SetMysqlPort sets the mysql port for tablet. This function
// also handles legacy by setting the port in PortMap.
// TODO(sougou); deprecate this function after 3.0.
func SetMysqlPort(tablet *topodatapb.Tablet, port int32) {
	if tablet.MysqlHostname == "" || tablet.MysqlHostname == tablet.Hostname {
		tablet.PortMap["mysql"] = port
	}
	// If it's the legacy form, preserve old behavior to prevent
	// confusion between new and old code.
	if tablet.MysqlHostname != "" {
		tablet.MysqlPort = port
	}
}

// MysqlAddr returns the host:port of the mysql server.
func MysqlAddr(tablet *topodatapb.Tablet) string {
	return fmt.Sprintf("%v:%v", MysqlHostname(tablet), MysqlPort(tablet))
}

// MysqlHostname returns the mysql host name. This function
// also handles legacy behavior: it uses the tablet's hostname
// if MysqlHostname is not specified.
// TODO(sougou); deprecate this function after 3.0.
func MysqlHostname(tablet *topodatapb.Tablet) string {
	if tablet.MysqlHostname == "" {
		return tablet.Hostname
	}
	return tablet.MysqlHostname
}

// MysqlPort returns the mysql port. This function
// also handles legacy behavior: it uses the tablet's port map
// if MysqlHostname is not specified.
// TODO(sougou); deprecate this function after 3.0.
func MysqlPort(tablet *topodatapb.Tablet) int32 {
	if tablet.MysqlHostname == "" {
		return tablet.PortMap["mysql"]
	}
	return tablet.MysqlPort
}

// MySQLIP returns the MySQL server's IP by resolvign the host name.
func MySQLIP(tablet *topodatapb.Tablet) (string, error) {
	ipAddrs, err := net.LookupHost(MysqlHostname(tablet))
	if err != nil {
		return "", err
	}
	return ipAddrs[0], nil
}

// TabletDbName is usually implied by keyspace. Having the shard
// information in the database name complicates mysql replication.
func TabletDbName(tablet *topodatapb.Tablet) string {
	if tablet.DbNameOverride != "" {
		return tablet.DbNameOverride
	}
	if tablet.Keyspace == "" {
		return ""
	}
	return vtDbPrefix + tablet.Keyspace
}

// TabletIsAssigned returns if this tablet is assigned to a keyspace and shard.
// A "scrap" node will show up as assigned even though its data cannot be used
// for serving.
func TabletIsAssigned(tablet *topodatapb.Tablet) bool {
	return tablet != nil && tablet.Keyspace != "" && tablet.Shard != ""
}
<|endoftext|>


<commit_before>package zeusmaster

import (
	"errors"
	"math/rand"
	"net"
	"os"
	"path/filepath"
	"strconv"
	"time"

	slog "github.com/burke/zeus/go/shinylog"
	"github.com/burke/zeus/go/unixsocket"
)

const zeusSockName string = ".zeus.sock"

func StartClientHandler(tree *ProcessTree, done chan bool) chan bool {
	quit := make(chan bool)
	go func() {
		path, _ := filepath.Abs(zeusSockName)
		addr, err := net.ResolveUnixAddr("unix", path)
		if err != nil {
			Error("Can't open socket.")
		}
		listener, err := net.ListenUnix("unix", addr)
		if err != nil {
			ErrorCantCreateListener()
		}

		connections := make(chan *unixsocket.Usock)
		go func() {
			for {
				if conn, err := listener.AcceptUnix(); err != nil {
					errorUnableToAcceptSocketConnection()
					time.Sleep(500 * time.Millisecond)
				} else {
					connections <- unixsocket.NewUsock(conn)
				}
			}
		}()

		for {
			select {
			case <-quit:
				listener.Close()
				done <- true
				return
			case conn := <-connections:
				go handleClientConnection(tree, conn)
			}
		}
	}()
	return quit
}

// see docs/client_master_handshake.md
func handleClientConnection(tree *ProcessTree, usock *unixsocket.Usock) {
	defer usock.Close()
	// we have established first contact to the client.

	command, clientPid, arguments, err := receiveCommandArgumentsAndPid(usock, nil)
	commandNode, slaveNode, err := findCommandAndSlaveNodes(tree, command, err)
	command = commandNode.Name // resolve aliases

	clientFile, err := receiveTTY(usock, err)
	defer clientFile.Close()

	if err == nil && slaveNode.Error != "" {
		// we can skip steps 3-5 as they deal with the command process we're not spawning.
		// Write a fake pid (step 6)
		usock.WriteMessage("0")
		// Write the error message to the terminal
		clientFile.Write([]byte(slaveNode.Error))
		// Skip step 7, and write an exit code to the client (step 8)
		usock.WriteMessage("1")
		return
	}

	commandUsock, err := bootNewCommand(slaveNode, command, err)
	defer commandUsock.Close()

	err = sendClientPidAndArgumentsToCommand(commandUsock, clientPid, arguments, err)

	err = sendTTYToCommand(commandUsock, clientFile, err)

	cmdPid, err := receivePidFromCommand(commandUsock, err)

	err = sendCommandPidToClient(usock, cmdPid, err)

	exitStatus, err := receiveExitStatus(commandUsock, err)

	err = sendExitStatus(usock, exitStatus, err)

	if err != nil {
		slog.Error(err)
	}
	// Done! Hooray!
}

func receiveCommandArgumentsAndPid(usock *unixsocket.Usock, err error) (string, int, string, error) {
	if err != nil {
		return "", -1, "", err
	}

	msg, err := usock.ReadMessage()
	if err != nil {
		return "", -1, "", err
	}
	command, clientPid, arguments, err := ParseClientCommandRequestMessage(msg)
	if err != nil {
		return "", -1, "", err
	}

	return command, clientPid, arguments, err
}

func findCommandAndSlaveNodes(tree *ProcessTree, command string, err error) (*CommandNode, *SlaveNode, error) {
	if err != nil {
		return nil, nil, err
	}

	commandNode := tree.FindCommand(command)
	if commandNode == nil {
		return nil, nil, errors.New("ERROR: Node not found!: " + command)
	}
	command = commandNode.Name
	slaveNode := commandNode.Parent

	return commandNode, slaveNode, nil
}

func receiveTTY(usock *unixsocket.Usock, err error) (*os.File, error) {
	if err != nil {
		return nil, err
	}

	clientFd, err := usock.ReadFD()
	if err != nil {
		return nil, errors.New("Expected FD, none received!")
	}
	fileName := strconv.Itoa(rand.Int())
	clientFile := unixsocket.FdToFile(clientFd, fileName)

	return clientFile, nil
}

func sendClientPidAndArgumentsToCommand(commandUsock *unixsocket.Usock, clientPid int, arguments string, err error) error {
	if err != nil {
		return err
	}

	msg := CreatePidAndArgumentsMessage(clientPid, arguments)
	_, err = commandUsock.WriteMessage(msg)
	return err
}

func receiveExitStatus(commandUsock *unixsocket.Usock, err error) (string, error) {
	if err != nil {
		return "", err
	}

	return commandUsock.ReadMessage()
}

func sendExitStatus(usock *unixsocket.Usock, exitStatus string, err error) error {
	if err != nil {
		return err
	}

	_, err = usock.WriteMessage(exitStatus)
	return err
}

func receivePidFromCommand(commandUsock *unixsocket.Usock, err error) (int, error) {
	if err != nil {
		return -1, err
	}

	msg, err := commandUsock.ReadMessage()
	if err != nil {
		return -1, err
	}
	intPid, _, _ := ParsePidMessage(msg)

	return intPid, err
}

func sendCommandPidToClient(usock *unixsocket.Usock, pid int, err error) error {
	if err != nil {
		return err
	}

	strPid := strconv.Itoa(pid)
	_, err = usock.WriteMessage(strPid)

	return err
}

type CommandRequest struct {
	Name    string
	Retchan chan *os.File
}

func bootNewCommand(slaveNode *SlaveNode, command string, err error) (*unixsocket.Usock, error) {
	if err != nil {
		return nil, err
	}

	request := &CommandRequest{command, make(chan *os.File)}
	slaveNode.RequestCommandBoot(request)
	commandFile := <-request.Retchan // TODO: don't really want to wait indefinitely.
	// defer commandFile.Close() // TODO: can't do this here anymore.

	return unixsocket.NewUsockFromFile(commandFile)
}

func sendTTYToCommand(commandUsock *unixsocket.Usock, clientFile *os.File, err error) error {
	if err != nil {
		return err
	}

	return commandUsock.WriteFD(int(clientFile.Fd()))
}
<commit_msg>Handle unexpected disconnect of client process [fix #185]<commit_after>package zeusmaster

import (
	"errors"
	"math/rand"
	"net"
	"os"
	"path/filepath"
	"strconv"
	"time"

	slog "github.com/burke/zeus/go/shinylog"
	"github.com/burke/zeus/go/unixsocket"
)

const zeusSockName string = ".zeus.sock"

func StartClientHandler(tree *ProcessTree, done chan bool) chan bool {
	quit := make(chan bool)
	go func() {
		path, _ := filepath.Abs(zeusSockName)
		addr, err := net.ResolveUnixAddr("unix", path)
		if err != nil {
			Error("Can't open socket.")
		}
		listener, err := net.ListenUnix("unix", addr)
		if err != nil {
			ErrorCantCreateListener()
		}

		connections := make(chan *unixsocket.Usock)
		go func() {
			for {
				if conn, err := listener.AcceptUnix(); err != nil {
					errorUnableToAcceptSocketConnection()
					time.Sleep(500 * time.Millisecond)
				} else {
					connections <- unixsocket.NewUsock(conn)
				}
			}
		}()

		for {
			select {
			case <-quit:
				listener.Close()
				done <- true
				return
			case conn := <-connections:
				go handleClientConnection(tree, conn)
			}
		}
	}()
	return quit
}

// see docs/client_master_handshake.md
func handleClientConnection(tree *ProcessTree, usock *unixsocket.Usock) {
	defer usock.Close()
	// we have established first contact to the client.

	command, clientPid, arguments, err := receiveCommandArgumentsAndPid(usock, nil)
	commandNode, slaveNode, err := findCommandAndSlaveNodes(tree, command, err)
	if err != nil {
		// connection was established, no data was sent. Ignore.
		return
	}
	command = commandNode.Name // resolve aliases

	clientFile, err := receiveTTY(usock, err)
	defer clientFile.Close()

	if err == nil && slaveNode.Error != "" {
		// we can skip steps 3-5 as they deal with the command process we're not spawning.
		// Write a fake pid (step 6)
		usock.WriteMessage("0")
		// Write the error message to the terminal
		clientFile.Write([]byte(slaveNode.Error))
		// Skip step 7, and write an exit code to the client (step 8)
		usock.WriteMessage("1")
		return
	}

	commandUsock, err := bootNewCommand(slaveNode, command, err)
	defer commandUsock.Close()

	err = sendClientPidAndArgumentsToCommand(commandUsock, clientPid, arguments, err)

	err = sendTTYToCommand(commandUsock, clientFile, err)

	cmdPid, err := receivePidFromCommand(commandUsock, err)

	err = sendCommandPidToClient(usock, cmdPid, err)

	exitStatus, err := receiveExitStatus(commandUsock, err)

	err = sendExitStatus(usock, exitStatus, err)

	if err != nil {
		slog.Error(err)
	}
	// Done! Hooray!
}

func receiveCommandArgumentsAndPid(usock *unixsocket.Usock, err error) (string, int, string, error) {
	if err != nil {
		return "", -1, "", err
	}

	msg, err := usock.ReadMessage()
	if err != nil {
		return "", -1, "", err
	}
	command, clientPid, arguments, err := ParseClientCommandRequestMessage(msg)
	if err != nil {
		return "", -1, "", err
	}

	return command, clientPid, arguments, err
}

func findCommandAndSlaveNodes(tree *ProcessTree, command string, err error) (*CommandNode, *SlaveNode, error) {
	if err != nil {
		return nil, nil, err
	}

	commandNode := tree.FindCommand(command)
	if commandNode == nil {
		return nil, nil, errors.New("ERROR: Node not found!: " + command)
	}
	command = commandNode.Name
	slaveNode := commandNode.Parent

	return commandNode, slaveNode, nil
}

func receiveTTY(usock *unixsocket.Usock, err error) (*os.File, error) {
	if err != nil {
		return nil, err
	}

	clientFd, err := usock.ReadFD()
	if err != nil {
		return nil, errors.New("Expected FD, none received!")
	}
	fileName := strconv.Itoa(rand.Int())
	clientFile := unixsocket.FdToFile(clientFd, fileName)

	return clientFile, nil
}

func sendClientPidAndArgumentsToCommand(commandUsock *unixsocket.Usock, clientPid int, arguments string, err error) error {
	if err != nil {
		return err
	}

	msg := CreatePidAndArgumentsMessage(clientPid, arguments)
	_, err = commandUsock.WriteMessage(msg)
	return err
}

func receiveExitStatus(commandUsock *unixsocket.Usock, err error) (string, error) {
	if err != nil {
		return "", err
	}

	return commandUsock.ReadMessage()
}

func sendExitStatus(usock *unixsocket.Usock, exitStatus string, err error) error {
	if err != nil {
		return err
	}

	_, err = usock.WriteMessage(exitStatus)
	return err
}

func receivePidFromCommand(commandUsock *unixsocket.Usock, err error) (int, error) {
	if err != nil {
		return -1, err
	}

	msg, err := commandUsock.ReadMessage()
	if err != nil {
		return -1, err
	}
	intPid, _, _ := ParsePidMessage(msg)

	return intPid, err
}

func sendCommandPidToClient(usock *unixsocket.Usock, pid int, err error) error {
	if err != nil {
		return err
	}

	strPid := strconv.Itoa(pid)
	_, err = usock.WriteMessage(strPid)

	return err
}

type CommandRequest struct {
	Name    string
	Retchan chan *os.File
}

func bootNewCommand(slaveNode *SlaveNode, command string, err error) (*unixsocket.Usock, error) {
	if err != nil {
		return nil, err
	}

	request := &CommandRequest{command, make(chan *os.File)}
	slaveNode.RequestCommandBoot(request)
	commandFile := <-request.Retchan // TODO: don't really want to wait indefinitely.
	// defer commandFile.Close() // TODO: can't do this here anymore.

	return unixsocket.NewUsockFromFile(commandFile)
}

func sendTTYToCommand(commandUsock *unixsocket.Usock, clientFile *os.File, err error) error {
	if err != nil {
		return err
	}

	return commandUsock.WriteFD(int(clientFile.Fd()))
}
<|endoftext|>


<commit_before>package client

import (
	"os"
	"testing"
)

func TestBasicCall(t *testing.T) {
	if v := os.Getenv("IN_TRAVIS"); v == "yes" {
		return
	}

	response := map[string]interface{}{}
	if err := NewClient(&Options{
		Token: os.Getenv("TOKEN"),
	}).Call("groups", "list", map[string]interface{}{
		"memberId": "random",
	}, &response); err != nil {
		t.Fatal(err)
	}
	if len(response) > 0 {
		t.Fatal(len(response))
	}
}
<commit_msg>Update client_test.go<commit_after>package client

import (
	"os"
	"testing"
)

func TestBasicCall(t *testing.T) {
	if v := os.Getenv("IN_TRAVIS_CI"); v == "yes" {
		return
	}

	response := map[string]interface{}{}
	if err := NewClient(&Options{
		Token: os.Getenv("TOKEN"),
	}).Call("groups", "list", map[string]interface{}{
		"memberId": "random",
	}, &response); err != nil {
		t.Fatal(err)
	}
	if len(response) > 0 {
		t.Fatal(len(response))
	}
}
<|endoftext|>


<commit_before>package rest

import (
	"encoding/json"
	"fmt"
	"socialapi/models"

	"labix.org/v2/mgo/bson"
)

func CreateChannelParticipants(channelId, accountId int64, c int) ([]*models.ChannelParticipant, error) {
	var participants []*models.ChannelParticipant
	for i := 0; i < c; i++ {
		participant, err := CreateChannelParticipant(channelId, accountId)
		if err != nil {
			return nil, err
		}

		participants = append(participants, participant)
	}

	return participants, nil
}

func CreateChannelParticipant(channelId, requesterId int64) (*models.ChannelParticipant, error) {
	account := models.NewAccount()
	account.OldId = bson.NewObjectId().Hex()
	account, _ = CreateAccount(account)
	return AddChannelParticipant(channelId, requesterId, account.Id)
}

func ListChannelParticipants(channelId, accountId int64) ([]*models.ChannelParticipant, error) {

	url := fmt.Sprintf("/channel/%d/participants?accountId=%d", channelId, accountId)
	res, err := sendRequest("GET", url, nil)
	if err != nil {
		return nil, err
	}
	var participants []*models.ChannelParticipant
	err = json.Unmarshal(res, &participants)
	if err != nil {
		return nil, err
	}

	return participants, nil
}

func AddChannelParticipant(channelId, requesterId int64, accountIds ...int64) (*models.ChannelParticipant, error) {
	url := fmt.Sprintf("/channel/%d/participants/add?accountId=%d", channelId, requesterId)
	return channelParticipantOp(
		url,
		channelId,
		requesterId,
		accountIds...,
	)
}

func DeleteChannelParticipant(channelId int64, requesterId, accountId ...int64) (*models.ChannelParticipant, error) {
	url := fmt.Sprintf("/channel/%d/participants/remove?accountId=%d", channelId, requesterId)
	return channelParticipantOp(
		url,
		channelId,
		requesterId,
		accountIds...,
	)
}

func BlockChannelParticipant(channelId int64, requesterId, accountId ...int64) (*models.ChannelParticipant, error) {
	url := fmt.Sprintf("/channel/%d/participants/block?accountId=%d", channelId, requesterId)
	return channelParticipantOp(
		url,
		channelId,
		requesterId,
		accountIds...,
	)
}

func UnblockChannelParticipant(channelId, requesterId int64, accountIds ...int64) (*models.ChannelParticipant, error) {
	url := fmt.Sprintf("/channel/%d/participants/unblock?accountId=%d", channelId, requesterId)
	return channelParticipantOp(
		url,
		channelId,
		requesterId,
		accountIds...,
	)
}

func channelParticipantOp(url string, channelId, requesterId int64, accountIds ...int64) (*models.ChannelParticipant, error) {

	res := make([]*models.ChannelParticipant, 0)
	for _, accountId := range accountIds {
		c := models.NewChannelParticipant()
		c.AccountId = accountId
		res = append(res, c)
	}

	cps, err := sendModel("POST", url, &res)
	if err != nil {
		return nil, err
	}

	a := *(cps.(*[]*models.ChannelParticipant))

	return a[0], nil
}
<commit_msg>Socialapi: fix function signatures<commit_after>package rest

import (
	"encoding/json"
	"fmt"
	"socialapi/models"

	"labix.org/v2/mgo/bson"
)

func CreateChannelParticipants(channelId, accountId int64, c int) ([]*models.ChannelParticipant, error) {
	var participants []*models.ChannelParticipant
	for i := 0; i < c; i++ {
		participant, err := CreateChannelParticipant(channelId, accountId)
		if err != nil {
			return nil, err
		}

		participants = append(participants, participant)
	}

	return participants, nil
}

func CreateChannelParticipant(channelId, requesterId int64) (*models.ChannelParticipant, error) {
	account := models.NewAccount()
	account.OldId = bson.NewObjectId().Hex()
	account, _ = CreateAccount(account)
	return AddChannelParticipant(channelId, requesterId, account.Id)
}

func ListChannelParticipants(channelId, accountId int64) ([]*models.ChannelParticipant, error) {

	url := fmt.Sprintf("/channel/%d/participants?accountId=%d", channelId, accountId)
	res, err := sendRequest("GET", url, nil)
	if err != nil {
		return nil, err
	}
	var participants []*models.ChannelParticipant
	err = json.Unmarshal(res, &participants)
	if err != nil {
		return nil, err
	}

	return participants, nil
}

func AddChannelParticipant(channelId, requesterId int64, accountIds ...int64) (*models.ChannelParticipant, error) {
	url := fmt.Sprintf("/channel/%d/participants/add?accountId=%d", channelId, requesterId)
	return channelParticipantOp(
		url,
		channelId,
		requesterId,
		accountIds...,
	)
}

func DeleteChannelParticipant(channelId int64, requesterId int64, accountIds ...int64) (*models.ChannelParticipant, error) {
	url := fmt.Sprintf("/channel/%d/participants/remove?accountId=%d", channelId, requesterId)
	return channelParticipantOp(
		url,
		channelId,
		requesterId,
		accountIds...,
	)
}

func BlockChannelParticipant(channelId int64, requesterId int64, accountIds ...int64) (*models.ChannelParticipant, error) {
	url := fmt.Sprintf("/channel/%d/participants/block?accountId=%d", channelId, requesterId)
	return channelParticipantOp(
		url,
		channelId,
		requesterId,
		accountIds...,
	)
}

func UnblockChannelParticipant(channelId, requesterId int64, accountIds ...int64) (*models.ChannelParticipant, error) {
	url := fmt.Sprintf("/channel/%d/participants/unblock?accountId=%d", channelId, requesterId)
	return channelParticipantOp(
		url,
		channelId,
		requesterId,
		accountIds...,
	)
}

func channelParticipantOp(url string, channelId, requesterId int64, accountIds ...int64) (*models.ChannelParticipant, error) {

	res := make([]*models.ChannelParticipant, 0)
	for _, accountId := range accountIds {
		c := models.NewChannelParticipant()
		c.AccountId = accountId
		res = append(res, c)
	}

	cps, err := sendModel("POST", url, &res)
	if err != nil {
		return nil, err
	}

	a := *(cps.(*[]*models.ChannelParticipant))

	return a[0], nil
}
<|endoftext|>


<commit_before>package gosnmpquerier

import (
	"testing"
	"time"

	"github.com/soniah/gosnmp"
	"github.com/stretchr/testify/assert"
)

type FakeSnmpClient struct{}

func (snmpClient *FakeSnmpClient) get(destination, community string, oids []string, timeout time.Duration, retries int) ([]gosnmp.SnmpPDU, error) {
	return makeSnmpPDu()
}

func (snmpClient *FakeSnmpClient) walk(destination, community, oid string, timeout time.Duration, retries int) ([]gosnmp.SnmpPDU, error) {
	return makeSnmpPDu()
}

func (snmpClient *FakeSnmpClient) getnext(destination, community string, oids []string, timeout time.Duration, retries int) ([]gosnmp.SnmpPDU, error) {
	return makeSnmpPDu()
}

func makeSnmpPDu() ([]gosnmp.SnmpPDU, error) {
	return []gosnmp.SnmpPDU{gosnmp.SnmpPDU{Name: "foo", Type: 1, Value: 1}}, nil
}

func newSyncQuerier() *syncQuerier {
	querier := NewSyncQuerier(1, 3, 3*time.Second)
	querier.asyncQuerier.snmpClient = &FakeSnmpClient{}
	return querier
}

func expectedSnmpResult() []gosnmp.SnmpPDU {
	return []gosnmp.SnmpPDU{gosnmp.SnmpPDU{Name: "foo", Type: 0x1, Value: 1}}
}
func TestGetReturnsSnmpGetResult(t *testing.T) {
	querier := newSyncQuerier()
	result, _ := querier.Get("192.168.5.15", "alea2", []string{"1.3.6.1.2.1.1.1.0"}, 1*time.Second, 1)
	assert.Equal(t, result, expectedSnmpResult())
}

func TestGetNextReturnsSnmpGetNextResult(t *testing.T) {
	querier := newSyncQuerier()
	result, _ := querier.GetNext("192.168.5.15", "alea2", []string{"1.3.6.1.2.1.1.1.0"}, 1*time.Second, 1)
	assert.Equal(t, result, expectedSnmpResult())
}

func TestWalkReturnsSnmpWalkResult(t *testing.T) {
	querier := newSyncQuerier()
	result, _ := querier.Walk("192.168.5.15", "alea2", "1.3.6.1.2.1.1", 1*time.Second, 1)
	assert.Equal(t, result, expectedSnmpResult())
}
<commit_msg>Rename<commit_after>package gosnmpquerier

import (
	"testing"
	"time"

	"github.com/soniah/gosnmp"
	"github.com/stretchr/testify/assert"
)

type FakeSnmpClient struct{}

func (snmpClient *FakeSnmpClient) get(destination, community string, oids []string, timeout time.Duration, retries int) ([]gosnmp.SnmpPDU, error) {
	return makeSnmpPDU()
}

func (snmpClient *FakeSnmpClient) walk(destination, community, oid string, timeout time.Duration, retries int) ([]gosnmp.SnmpPDU, error) {
	return makeSnmpPDU()
}

func (snmpClient *FakeSnmpClient) getnext(destination, community string, oids []string, timeout time.Duration, retries int) ([]gosnmp.SnmpPDU, error) {
	return makeSnmpPDU()
}

func makeSnmpPDU() ([]gosnmp.SnmpPDU, error) {
	return []gosnmp.SnmpPDU{gosnmp.SnmpPDU{Name: "foo", Type: 1, Value: 1}}, nil
}

func newSyncQuerier() *syncQuerier {
	querier := NewSyncQuerier(1, 3, 3*time.Second)
	querier.asyncQuerier.snmpClient = &FakeSnmpClient{}
	return querier
}

func expectedSnmpResult() []gosnmp.SnmpPDU {
	return []gosnmp.SnmpPDU{gosnmp.SnmpPDU{Name: "foo", Type: 0x1, Value: 1}}
}

func TestGetReturnsSnmpGetResult(t *testing.T) {
	querier := newSyncQuerier()
	result, _ := querier.Get("192.168.5.15", "alea2", []string{"1.3.6.1.2.1.1.1.0"}, 1*time.Second, 1)
	assert.Equal(t, result, expectedSnmpResult())
}

func TestGetNextReturnsSnmpGetNextResult(t *testing.T) {
	querier := newSyncQuerier()
	result, _ := querier.GetNext("192.168.5.15", "alea2", []string{"1.3.6.1.2.1.1.1.0"}, 1*time.Second, 1)
	assert.Equal(t, result, expectedSnmpResult())
}

func TestWalkReturnsSnmpWalkResult(t *testing.T) {
	querier := newSyncQuerier()
	result, _ := querier.Walk("192.168.5.15", "alea2", "1.3.6.1.2.1.1", 1*time.Second, 1)
	assert.Equal(t, result, expectedSnmpResult())
}
<|endoftext|>


<commit_before>package loader

import (
	"errors"
	"fmt"
	"os"
	"path/filepath"
	"plugin"

	iplugin "github.com/ipfs/go-ipfs/plugin"
)

func init() {
	loadPluginsFunc = linuxLoadFunc
}

func linuxLoadFunc(pluginDir string) ([]iplugin.Plugin, error) {
	var plugins []iplugin.Plugin

	err := filepath.Walk(pluginDir, func(fi string, info os.FileInfo, err error) error {
		if err != nil {
			return err
		}
		if info.IsDir() {
			if fi != pluginDir {
				log.Warningf("found directory inside plugins directory: %s", fi)
			}
			return nil
		}

		if info.Mode().Perm()&0111 == 0 {
			// file is not executable let's not load it
			// this is to prevent loading plugins from for example non-executable
			// mounts, some /tmp mounts are marked as such for security
			log.Errorf("non-executable file in plugins directory: %s", fi)
			return nil
		}

		if newPlugins, err := loadPlugin(fi); err == nil {
			plugins = append(plugins, newPlugins...)
		} else {
			return fmt.Errorf("loading plugin %s: %s", fi, err)
		}
		return nil
	})

	return plugins, err
}

func loadPlugin(fi string) ([]iplugin.Plugin, error) {
	pl, err := plugin.Open(fi)
	if err != nil {
		return nil, err
	}
	pls, err := pl.Lookup("Plugins")
	if err != nil {
		return nil, err
	}
	log.Errorf("plugins: %T", pls)

	typePls, ok := pls.(*[]iplugin.Plugin)
	if !ok {
		return nil, errors.New("filed 'Plugins' didn't contain correct type")
	}

	return *typePls, nil
}
<commit_msg>plugin: remove debug error log<commit_after>package loader

import (
	"errors"
	"fmt"
	"os"
	"path/filepath"
	"plugin"

	iplugin "github.com/ipfs/go-ipfs/plugin"
)

func init() {
	loadPluginsFunc = linuxLoadFunc
}

func linuxLoadFunc(pluginDir string) ([]iplugin.Plugin, error) {
	var plugins []iplugin.Plugin

	err := filepath.Walk(pluginDir, func(fi string, info os.FileInfo, err error) error {
		if err != nil {
			return err
		}
		if info.IsDir() {
			if fi != pluginDir {
				log.Warningf("found directory inside plugins directory: %s", fi)
			}
			return nil
		}

		if info.Mode().Perm()&0111 == 0 {
			// file is not executable let's not load it
			// this is to prevent loading plugins from for example non-executable
			// mounts, some /tmp mounts are marked as such for security
			log.Errorf("non-executable file in plugins directory: %s", fi)
			return nil
		}

		if newPlugins, err := loadPlugin(fi); err == nil {
			plugins = append(plugins, newPlugins...)
		} else {
			return fmt.Errorf("loading plugin %s: %s", fi, err)
		}
		return nil
	})

	return plugins, err
}

func loadPlugin(fi string) ([]iplugin.Plugin, error) {
	pl, err := plugin.Open(fi)
	if err != nil {
		return nil, err
	}
	pls, err := pl.Lookup("Plugins")
	if err != nil {
		return nil, err
	}

	typePls, ok := pls.(*[]iplugin.Plugin)
	if !ok {
		return nil, errors.New("filed 'Plugins' didn't contain correct type")
	}

	return *typePls, nil
}
<|endoftext|>