<commit_before>// Package cache is an interface to multiple storage backends for Shade.  It
// centralizes the implementation of reading and writing to multiple
// drive.Clients.
package cache

import (
	"errors"
	"flag"
	"fmt"
	"log"

	"github.com/asjoyner/shade"
	"github.com/asjoyner/shade/drive"
)

var (
	cacheDebug = flag.Bool("cacheDebug", false, "Print cache debugging traces")
)

func init() {
	drive.RegisterProvider("cache", NewClient)
}

type refreshReq struct {
	sha256sum []byte
	content   []byte
	f         *shade.File
}

// NewClient returns a Drive client which centralizes reading and writing to
// multiple Providers.
func NewClient(c drive.Config) (drive.Client, error) {
	if len(c.Children) == 0 {
		return nil, errors.New("no clients provided")
	}
	d := &Drive{config: c}
	for _, conf := range c.Children {
		child, err := drive.NewClient(conf)
		if err != nil {
			return nil, fmt.Errorf("%s: %s", conf.Provider, err)
		}
		if child.GetConfig().Write {
			d.log(fmt.Sprintf("child %s is writable.", conf.Provider))
			d.config.Write = true
		} else {
			d.log(fmt.Sprintf("child %s is NOT writable.", conf.Provider))
		}
		d.clients = append(d.clients, child)
	}
	d.log(fmt.Sprintf("my final status is: %v", d.config.Write))
	d.files = make(chan refreshReq, 100)
	go func(d *Drive) {
		for r := range d.files {
			d.refreshFile(r.sha256sum, r.content)
		}
	}(d)
	d.chunks = make(chan refreshReq, 100)
	go func(d *Drive) {
		for r := range d.chunks {
			d.refreshChunk(r.sha256sum, r.content, r.f)
		}
	}(d)
	return d, nil
}

// Drive implements the drive.Client interface by reading and writing to the
// slice of drive.Client interfaces it was provided.  It can return a config
// which describes only its name.
//
// If any of its clients are not Local(), it reports itself as not Local() by
// returning false.  If any of its clients are Persistent(), it requires writes
// to at least one of those backends to succeed, and reports itself as
// Persistent().
type Drive struct {
	config  drive.Config
	clients []drive.Client
	chunks  chan refreshReq
	files   chan refreshReq
	debug   bool
}

// ListFiles retrieves all of the File objects known to all of the provided
// clients.  The return is a list of sha256sums of the file object.  The keys
// may be passed to GetChunk() to retrieve the corresponding shade.File.
func (s *Drive) ListFiles() ([][]byte, error) {
	c := make(chan [][]byte, len(s.clients))
	for _, client := range s.clients {
		go func(client drive.Client) {
			f, err := client.ListFiles()
			if err != nil {
				s.log(fmt.Sprintf("Error reading from %q: %s", client.GetConfig().Provider, err))
			}
			c <- f
		}(client)
	}

	var resp [][]byte
	for i := 0; i < len(s.clients); i++ {
		resp = append(resp, <-c...)
	}

	return resp, nil
}

// GetFile retrieves a file with a given SHA-256 sum.  It will be returned
// from the first client in the slice of structs that returns the chunk.
func (s *Drive) GetFile(sha256sum []byte) ([]byte, error) {
	for _, client := range s.clients {
		file, err := client.GetFile(sha256sum)
		if err != nil {
			s.log(fmt.Sprintf("File %x not found in %q: %s", sha256sum, client.GetConfig().Provider, err))
			continue
		}
		s.files <- refreshReq{sha256sum: sha256sum, content: file, f: nil}
		return file, nil
	}
	return nil, errors.New("file not found")
}

// PutFile writes the metadata describing a new file.  It will be written to
// all shade backends configured to Write.  If any backends are Persistent, it
// returns an error if all Persistent backends fail to write.
// f should be marshalled JSON, and may be encrypted.
func (s *Drive) PutFile(sha256sum, f []byte) error {
	if s.config.Write == false {
		return errors.New("no clients configured to write")
	}

	persisted := make(chan struct{}, len(s.clients))
	done := make(chan struct{}, len(s.clients))
	for _, client := range s.clients {
		go func(client drive.Client) {
			if err := client.PutFile(sha256sum, f); err != nil {
				s.log(fmt.Sprintf("%s.PutFile(%x) failed: %s", client.GetConfig().Provider, sha256sum, err))
				done <- struct{}{}
				return
			}
			if !s.Persistent() || client.Persistent() {
				persisted <- struct{}{}
				return
			}
			done <- struct{}{}
		}(client)
	}
	for range s.clients {
		select {
		case <-persisted:
			return nil
		case <-done:
		}
	}
	return fmt.Errorf("persistent storage configured, but all writes failed: %x", sha256sum)
}

// GetChunk retrieves a chunk with a given SHA-256 sum.  It will be returned
// from the first client in the slice of structs that returns the chunk.
func (s *Drive) GetChunk(sha256sum []byte, f *shade.File) ([]byte, error) {
	// TODO(asjoyner): consider adding the ability to cancel GetChunk, then
	// paralellize this with a slight delay between launching each request.
	for _, client := range s.clients {
		chunk, err := client.GetChunk(sha256sum, f)
		if err != nil {
			s.log(fmt.Sprintf("Chunk %x not found in %q: %s", sha256sum, client.GetConfig().Provider, err))
			continue
		}
		s.chunks <- refreshReq{sha256sum: sha256sum, content: chunk, f: f}
		return chunk, nil
	}
	return nil, errors.New("chunk not found")
}

// PutChunk writes a chunk associated with a SHA-256 sum.  It will attempt to write to
// all shade backends configured to Write.  If any backends are Persistent, it
// returns an error if all Persistent backends fail to write.
func (s *Drive) PutChunk(sha256sum []byte, chunk []byte, f *shade.File) error {
	if s.config.Write == false {
		return errors.New("no clients configured to write")
	}

	persisted := make(chan struct{}, len(s.clients))
	done := make(chan struct{}, len(s.clients))
	for _, client := range s.clients {
		go func(client drive.Client) {
			if err := client.PutChunk(sha256sum, chunk, f); err != nil {
				s.log(fmt.Sprintf("%s.PutChunk(%x) failed: %s", client.GetConfig().Provider, sha256sum, err))
				done <- struct{}{}
				return
			}
			if !s.Persistent() || client.Persistent() {
				persisted <- struct{}{}
				return
			}
			done <- struct{}{}
		}(client)
	}
	for range s.clients {
		select {
		case <-persisted:
			return nil
		case <-done:
		}
	}
	return fmt.Errorf("persistent storage configured, but all writes failed: %x", sha256sum)
}

// GetConfig returns the config used to initialize this client.
func (s *Drive) GetConfig() drive.Config {
	return s.config
}

// Local returns true only if all configured storage backends are local to this
// machine.
func (s *Drive) Local() bool {
	for _, c := range s.clients {
		if !c.Local() {
			return false
		}
	}
	return true
}

// Persistent returns true if at least one configured storage backend is
// Persistent().
func (s *Drive) Persistent() bool {
	for _, c := range s.clients {
		if c.Persistent() {
			return true
		}
	}
	return false
}

// Debug enables debug statements to STDERR for non-critical failures to read or
// write from clients.
func (s *Drive) Debug() {
	flag.Set("cacheDebug", "true")
}

func (s *Drive) log(output string) {
	if *cacheDebug {
		log.Printf("drive.Cache: %s\n", output)
	}
}

func (s *Drive) refreshWorker() {
	select {
	case r := <-s.files:
		s.refreshFile(r.sha256sum, r.content)
	case r := <-s.chunks:
		s.refreshChunk(r.sha256sum, r.content, r.f)
	}
}

// refreshFile calls PutFile on each client which is Local()
// This populates eg. memory and disk clients with files that are
// fetched from remote clients.  Errors are logged, but not returned.
func (s *Drive) refreshFile(sha256sum, file []byte) {
	for _, client := range s.clients {
		if client.Local() {
			client.PutFile(sha256sum, file)
		}
	}
}

// refreshChunk calls PutChunk on each client which is Local()
// This populates eg. memory and disk clients with chunks that are
// fetched from remote clients.  Errors are logged, but not returned.
func (s *Drive) refreshChunk(sha256sum, chunk []byte, f *shade.File) {
	for _, client := range s.clients {
		if client.Local() {
			client.PutChunk(sha256sum, chunk, f)
		}
	}
}
<commit_msg>Convert cache from custom debug logging to glog<commit_after>// Package cache is an interface to multiple storage backends for Shade.  It
// centralizes the implementation of reading and writing to multiple
// drive.Clients.
package cache

import (
	"errors"
	"fmt"

	"github.com/golang/glog"

	"github.com/asjoyner/shade"
	"github.com/asjoyner/shade/drive"
)

func init() {
	drive.RegisterProvider("cache", NewClient)
}

type refreshReq struct {
	sha256sum []byte
	content   []byte
	f         *shade.File
}

// NewClient returns a Drive client which centralizes reading and writing to
// multiple Providers.
func NewClient(c drive.Config) (drive.Client, error) {
	if len(c.Children) == 0 {
		return nil, errors.New("no clients provided")
	}
	d := &Drive{config: c}
	for _, conf := range c.Children {
		child, err := drive.NewClient(conf)
		if err != nil {
			return nil, fmt.Errorf("%s: %s", conf.Provider, err)
		}
		if child.GetConfig().Write {
			glog.V(2).Infof("child %s is writable.", conf.Provider)
			d.config.Write = true
		} else {
			glog.V(2).Infof("child %s is NOT writable.", conf.Provider)
		}
		d.clients = append(d.clients, child)
	}
	glog.V(2).Infof("my final status is: %v", d.config.Write)
	d.files = make(chan refreshReq, 100)
	go func(d *Drive) {
		for r := range d.files {
			d.refreshFile(r.sha256sum, r.content)
		}
	}(d)
	d.chunks = make(chan refreshReq, 100)
	go func(d *Drive) {
		for r := range d.chunks {
			d.refreshChunk(r.sha256sum, r.content, r.f)
		}
	}(d)
	return d, nil
}

// Drive implements the drive.Client interface by reading and writing to the
// slice of drive.Client interfaces it was provided.  It can return a config
// which describes only its name.
//
// If any of its clients are not Local(), it reports itself as not Local() by
// returning false.  If any of its clients are Persistent(), it requires writes
// to at least one of those backends to succeed, and reports itself as
// Persistent().
type Drive struct {
	config  drive.Config
	clients []drive.Client
	chunks  chan refreshReq
	files   chan refreshReq
	debug   bool
}

// ListFiles retrieves all of the File objects known to all of the provided
// clients.  The return is a list of sha256sums of the file object.  The keys
// may be passed to GetChunk() to retrieve the corresponding shade.File.
func (s *Drive) ListFiles() ([][]byte, error) {
	c := make(chan [][]byte, len(s.clients))
	for _, client := range s.clients {
		// TODO: spawn goroutines for this in advance, one per client?
		// careful to keep it threadsafe
		go func(client drive.Client) {
			f, err := client.ListFiles()
			if err != nil {
				glog.Warningf("error reading from %q: %s", client.GetConfig().Provider, err)
			}
			c <- f
		}(client)
	}

	var resp [][]byte
	for i := 0; i < len(s.clients); i++ {
		resp = append(resp, <-c...)
	}

	return resp, nil
}

// GetFile retrieves a file with a given SHA-256 sum.  It will be returned
// from the first client in the slice of structs that returns the chunk.
func (s *Drive) GetFile(sha256sum []byte) ([]byte, error) {
	for _, client := range s.clients {
		file, err := client.GetFile(sha256sum)
		if err != nil {
			glog.V(2).Infof("File %x not found in %q: %s", sha256sum, client.GetConfig().Provider, err)
			continue
		}
		s.files <- refreshReq{sha256sum: sha256sum, content: file, f: nil}
		return file, nil
	}
	return nil, errors.New("file not found")
}

// PutFile writes the metadata describing a new file.  It will be written to
// all shade backends configured to Write.  If any backends are Persistent, it
// returns an error if all Persistent backends fail to write.
// f should be marshalled JSON, and may be encrypted.
func (s *Drive) PutFile(sha256sum, f []byte) error {
	if s.config.Write == false {
		return errors.New("no clients configured to write")
	}

	persisted := make(chan struct{}, len(s.clients))
	done := make(chan struct{}, len(s.clients))
	for _, client := range s.clients {
		go func(client drive.Client) {
			if err := client.PutFile(sha256sum, f); err != nil {
				glog.Warningf("%s.PutFile(%x) failed: %s", client.GetConfig().Provider, sha256sum, err)
				done <- struct{}{}
				return
			}
			if !s.Persistent() || client.Persistent() {
				persisted <- struct{}{}
				return
			}
			done <- struct{}{}
		}(client)
	}
	for range s.clients {
		select {
		case <-persisted:
			return nil
		case <-done:
		}
	}
	return fmt.Errorf("persistent storage configured, but all writes failed: %x", sha256sum)
}

// GetChunk retrieves a chunk with a given SHA-256 sum.  It will be returned
// from the first client in the slice of structs that returns the chunk.
func (s *Drive) GetChunk(sha256sum []byte, f *shade.File) ([]byte, error) {
	// TODO(asjoyner): consider adding the ability to cancel GetChunk, then
	// paralellize this with a slight delay between launching each request.
	for _, client := range s.clients {
		chunk, err := client.GetChunk(sha256sum, f)
		if err != nil {
			glog.V(2).Infof("Chunk %x not found in %q: %s", sha256sum, client.GetConfig().Provider, err)
			continue
		}
		s.chunks <- refreshReq{sha256sum: sha256sum, content: chunk, f: f}
		return chunk, nil
	}
	return nil, errors.New("chunk not found")
}

// PutChunk writes a chunk associated with a SHA-256 sum.  It will attempt to write to
// all shade backends configured to Write.  If any backends are Persistent, it
// returns an error if all Persistent backends fail to write.
func (s *Drive) PutChunk(sha256sum []byte, chunk []byte, f *shade.File) error {
	if s.config.Write == false {
		return errors.New("no clients configured to write")
	}

	persisted := make(chan struct{}, len(s.clients))
	done := make(chan struct{}, len(s.clients))
	for _, client := range s.clients {
		go func(client drive.Client) {
			if err := client.PutChunk(sha256sum, chunk, f); err != nil {
				glog.Warningf("%s.PutChunk(%x) failed: %s", client.GetConfig().Provider, sha256sum, err)
				done <- struct{}{}
				return
			}
			if !s.Persistent() || client.Persistent() {
				persisted <- struct{}{}
				return
			}
			done <- struct{}{}
		}(client)
	}
	for range s.clients {
		select {
		case <-persisted:
			return nil
		case <-done:
		}
	}
	return fmt.Errorf("persistent storage configured, but all writes failed: %x", sha256sum)
}

// GetConfig returns the config used to initialize this client.
func (s *Drive) GetConfig() drive.Config {
	return s.config
}

// Local returns true only if all configured storage backends are local to this
// machine.
func (s *Drive) Local() bool {
	for _, c := range s.clients {
		if !c.Local() {
			return false
		}
	}
	return true
}

// Persistent returns true if at least one configured storage backend is
// Persistent().
func (s *Drive) Persistent() bool {
	for _, c := range s.clients {
		if c.Persistent() {
			return true
		}
	}
	return false
}

func (s *Drive) refreshWorker() {
	select {
	case r := <-s.files:
		s.refreshFile(r.sha256sum, r.content)
	case r := <-s.chunks:
		s.refreshChunk(r.sha256sum, r.content, r.f)
	}
}

// refreshFile calls PutFile on each client which is Local()
// This populates eg. memory and disk clients with files that are
// fetched from remote clients.  Errors are logged, but not returned.
func (s *Drive) refreshFile(sha256sum, file []byte) {
	for _, client := range s.clients {
		if client.Local() {
			client.PutFile(sha256sum, file)
		}
	}
}

// refreshChunk calls PutChunk on each client which is Local()
// This populates eg. memory and disk clients with chunks that are
// fetched from remote clients.  Errors are logged, but not returned.
func (s *Drive) refreshChunk(sha256sum, chunk []byte, f *shade.File) {
	for _, client := range s.clients {
		if client.Local() {
			client.PutChunk(sha256sum, chunk, f)
		}
	}
}
<|endoftext|>


<commit_before>package aws

import (
	mach "github.com/jeffjen/machine/lib/machine"

	"github.com/urfave/cli"

	"encoding/json"
	"fmt"
	"reflect"
	"strconv"
	"strings"
)

var (
	ErrFieldFromTag = fmt.Errorf("Unable to find field from tag")
)

func syncFromAWS() cli.Command {
	return cli.Command{
		Name:  "sync",
		Usage: "Sync AWS settings and assets",
		Flags: []cli.Flag{
			cli.StringFlag{Name: "name", Value: "default", Usage: "Name of the profile"},
			cli.StringFlag{Name: "vpc-id", Value: "default", Usage: "AWS VPC identifier"},
		},
		Action: func(c *cli.Context) error {
			defer profile.Dump()
			defer mach.InstList.Dump()

			p := &Profile{Name: c.String("name"), Region: c.GlobalString("region")}
			if account_id, err := vpcInit(c, &p.VPC); err != nil {
				return cli.NewExitError(err.Error(), 1)
			} else {
				p.AccntId = account_id
			}
			if err := amiInit(c, p); err != nil {
				return cli.NewExitError(err.Error(), 1)
			}
			if err := keypairInit(c, p); err != nil {
				return cli.NewExitError(err.Error(), 1)
			}
			if err := ec2Init(); err != nil {
				return cli.NewExitError(err.Error(), 1)
			}
			if _, ok := profile[p.Region]; !ok {
				profile[p.Region] = make(RegionProfile)
			}
			profile[p.Region][p.Name] = p
			return nil
		},
	}
}

func getFieldFromTag(t reflect.Type, s string) (f string, e error) {
	for idx := 0; idx < t.NumField(); idx++ {
		field := t.Field(idx)
		if tag := field.Tag.Get("json"); tag == s {
			return field.Name, nil
		}
	}
	return "", ErrFieldFromTag
}

func getFromAWSConfig() cli.Command {
	return cli.Command{
		Name:  "get",
		Usage: "Get config value from local store",
		Flags: []cli.Flag{
			cli.StringFlag{Name: "name", Value: "default", Usage: "Name of the profile"},
		},
		Action: func(c *cli.Context) error {
			var (
				name   = c.String("name")
				region = c.GlobalString("region")

				qpath = c.Args().First()
			)

			// Retrieve user provide query path
			if qpath == "" {
				return nil // nothing to do here, abort
			}

			if _, ok := profile[region]; !ok {
				return cli.NewExitError("Selected region had no stored profile", 1)
			}

			if _, ok := profile[region][name]; !ok {
				return cli.NewExitError("Selected name had no profile", 1)
			}

			// TODO: report config value by dotted notation argument
			// e.g. .vpc.subnet.0.cidr
			var v interface{} = profile[region][name]

			for _, s := range strings.Split(qpath, ".") {
				val := reflect.ValueOf(v)

				// NOTE: dereference the value ahead of time
				if val.Kind() == reflect.Ptr {
					val = val.Elem()
				}

				switch val.Kind() {
				default:
					return cli.NewExitError(fmt.Sprintln("Path past leaf node -", qpath), 1)

				case reflect.Struct:
					if chk := val.FieldByName(s); chk.IsValid() {
						v = chk.Interface()
					} else if f, err := getFieldFromTag(val.Type(), s); err == nil {
						v = val.FieldByName(f).Interface()
					} else {
						return cli.NewExitError(fmt.Sprintln("invalid token -", s), 1)
					}

				case reflect.Slice:
					idx, err := strconv.Atoi(s)
					if err != nil {
						return cli.NewExitError(fmt.Sprintln("invalid token -", s), 1)
					}
					if idx < 0 || idx >= val.Len() {
						return cli.NewExitError(fmt.Sprintln("invalid token -", s), 1)
					}
					v = val.Index(idx).Interface()
				}
			}

			if output, err := json.MarshalIndent(v, "", "    "); err != nil {
				return cli.NewExitError(fmt.Sprintln("Corrupt profile -", name), 1)
			} else {
				fmt.Println(string(output))
			}

			return nil
		},
	}
}
<commit_msg>Properly format get config value<commit_after>package aws

import (
	mach "github.com/jeffjen/machine/lib/machine"

	"github.com/urfave/cli"

	"encoding/json"
	"fmt"
	"reflect"
	"strconv"
	"strings"
)

var (
	ErrFieldFromTag = fmt.Errorf("Unable to find field from tag")
)

func syncFromAWS() cli.Command {
	return cli.Command{
		Name:  "sync",
		Usage: "Sync AWS settings and assets",
		Flags: []cli.Flag{
			cli.StringFlag{Name: "name", Value: "default", Usage: "Name of the profile"},
			cli.StringFlag{Name: "vpc-id", Value: "default", Usage: "AWS VPC identifier"},
		},
		Action: func(c *cli.Context) error {
			defer profile.Dump()
			defer mach.InstList.Dump()

			p := &Profile{Name: c.String("name"), Region: c.GlobalString("region")}
			if account_id, err := vpcInit(c, &p.VPC); err != nil {
				return cli.NewExitError(err.Error(), 1)
			} else {
				p.AccntId = account_id
			}
			if err := amiInit(c, p); err != nil {
				return cli.NewExitError(err.Error(), 1)
			}
			if err := keypairInit(c, p); err != nil {
				return cli.NewExitError(err.Error(), 1)
			}
			if err := ec2Init(); err != nil {
				return cli.NewExitError(err.Error(), 1)
			}
			if _, ok := profile[p.Region]; !ok {
				profile[p.Region] = make(RegionProfile)
			}
			profile[p.Region][p.Name] = p
			return nil
		},
	}
}

func getFieldFromTag(t reflect.Type, s string) (f string, e error) {
	for idx := 0; idx < t.NumField(); idx++ {
		field := t.Field(idx)
		if tag := field.Tag.Get("json"); tag == s {
			return field.Name, nil
		}
	}
	return "", ErrFieldFromTag
}

func getFromAWSConfig() cli.Command {
	return cli.Command{
		Name:  "get",
		Usage: "Get config value from local store",
		Flags: []cli.Flag{
			cli.StringFlag{Name: "name", Value: "default", Usage: "Name of the profile"},
		},
		Action: func(c *cli.Context) error {
			var (
				name   = c.String("name")
				region = c.GlobalString("region")

				qpath = c.Args().First()
			)

			// Retrieve user provide query path
			if qpath == "" {
				return nil // nothing to do here, abort
			}

			if _, ok := profile[region]; !ok {
				return cli.NewExitError("Selected region had no stored profile", 1)
			}

			if _, ok := profile[region][name]; !ok {
				return cli.NewExitError("Selected name had no profile", 1)
			}

			// TODO: report config value by dotted notation argument
			// e.g. .vpc.subnet.0.cidr
			var v interface{} = profile[region][name]

			for _, s := range strings.Split(qpath, ".") {
				val := reflect.ValueOf(v)

				// NOTE: dereference the value ahead of time
				if val.Kind() == reflect.Ptr {
					val = val.Elem()
				}

				switch val.Kind() {
				default:
					return cli.NewExitError(fmt.Sprintln("Path past leaf node -", qpath), 1)

				case reflect.Struct:
					if chk := val.FieldByName(s); chk.IsValid() {
						v = chk.Interface()
					} else if f, err := getFieldFromTag(val.Type(), s); err == nil {
						v = val.FieldByName(f).Interface()
					} else {
						return cli.NewExitError(fmt.Sprintln("invalid token -", s), 1)
					}

				case reflect.Slice:
					idx, err := strconv.Atoi(s)
					if err != nil {
						return cli.NewExitError(fmt.Sprintln("invalid token -", s), 1)
					}
					if idx < 0 || idx >= val.Len() {
						return cli.NewExitError(fmt.Sprintln("invalid token -", s), 1)
					}
					v = val.Index(idx).Interface()
				}
			}

			output, err := json.MarshalIndent(v, "", "    ")
			if err != nil {
				return cli.NewExitError(fmt.Sprintln("Corrupt profile -", name), 1)
			}

			var text = string(output)
			if unquoted, err := strconv.Unquote(text); err == nil {
				fmt.Println(unquoted)
			} else {
				fmt.Println(text)
			}

			return nil
		},
	}
}
<|endoftext|>


<commit_before>package driver

import (
	"context"
	"fmt"
	"syscall"

	log "github.com/Sirupsen/logrus"
	"github.com/containerd/containerd"
	"github.com/containerd/containerd/namespaces"
)

const defaultContainerdPath = "/run/containerd/containerd.sock"

// ContainerdDriver is an implementation of the driver interface for using Containerd.
// This uses the provided client library which abstracts using the gRPC APIs directly.
// IMPORTANT: This implementation does not protect instance metadata for thread safely.
// At this time there is no understood use case for multi-threaded use of this implementation.
type ContainerdDriver struct {
	ctrdAddress string
	client      *containerd.Client
	context     context.Context
}

// ContainerdContainer is an implementation of the container metadata needed for containerd
type ContainerdContainer struct {
	name      string
	imageName string
	state     string
	process   string
	trace     bool
}

// NewContainerdDriver creates an instance of the containerd driver, providing a path to the ctr client
func NewContainerdDriver(path string) (Driver, error) {
	if path == "" {
		path = defaultContainerdPath
	}
	client, err := containerd.New(path)
	if err != nil {
		return &ContainerdDriver{}, err
	}
	bbCtx := namespaces.WithNamespace(context.Background(), "bb")
	driver := &ContainerdDriver{
		ctrdAddress: path,
		client:      client,
		context:     bbCtx,
	}
	return driver, nil
}

// newContainerdContainer creates the metadata object of a containerd-specific container with
// bundle, name, and any required additional information
func newContainerdContainer(name, image string, trace bool) Container {
	return &ContainerdContainer{
		name:      name,
		imageName: image,
		trace:     trace,
	}
}

// Name returns the name of the container
func (c *ContainerdContainer) Name() string {
	return c.name
}

// Trace returns whether the container should be started with tracing enabled
func (c *ContainerdContainer) Trace() bool {
	return c.trace
}

// Image returns the bundle path that runc will use
func (c *ContainerdContainer) Image() string {
	return c.imageName
}

// Process returns the process name in cases where this container instance is
// wrapping a potentially running container
func (c *ContainerdContainer) Process() string {
	return c.process
}

// State returns the queried state of the container (if available)
func (c *ContainerdContainer) State() string {
	return c.state
}

// Detached always returns true for containerd as IO streams are always detached
func (c *ContainerdContainer) Detached() bool {
	return true
}

// Type returns a driver.Type to indentify the driver implementation
func (r *ContainerdDriver) Type() Type {
	return Containerd
}

// Info returns
func (r *ContainerdDriver) Info() (string, error) {
	version, err := r.client.Version(r.context)
	if err != nil {
		return "", err
	}
	info := "containerd gRPC client driver (daemon: " + version.Version + "[Revision: " + version.Revision + "] )"
	return info, nil
}

// Create will create a container instance matching the specific needs
// of a driver
func (r *ContainerdDriver) Create(name, image string, detached bool, trace bool) (Container, error) {
	if _, err := r.client.GetImage(r.context, image); err != nil {
		// if the image isn't already in our namespaced context, then pull it
		// using the reference and default resolver (most likely DockerHub)
		if _, err := r.client.Pull(r.context, image, containerd.WithPullUnpack); err != nil {
			// error pulling the image
			return nil, err
		}
	}
	return newContainerdContainer(name, image, trace), nil
}

// Clean will clean the environment; removing any remaining containers in the runc metadata
func (r *ContainerdDriver) Clean() error {
	var tries int
	list, err := r.client.Containers(r.context)
	if err != nil {
		return fmt.Errorf("Error getting containerd list output: %v", err)
	}
	// try up to 3 times to handle any active containers in the list
	log.Infof("Attempting to cleanup containerd containers/metadata; %d listed", len(list))
	for len(list) > 0 && tries < 3 {
		log.Infof("containerd cleanup: Pass #%d", tries+1)
		// kill/stop and remove containers
		for _, ctr := range list {
			if err := ctr.Delete(r.context); err != nil {
				log.Errorf("Error deleting container %v: %v", ctr, err)
			}
		}
		tries++
		list, err = r.client.Containers(r.context)
		if err != nil {
			return fmt.Errorf("Error getting containerd list output: %v", err)
		}
	}
	log.Infof("containerd cleanup complete.")
	return nil
}

// Run will execute a container using the containerd driver.
func (r *ContainerdDriver) Run(ctr Container) (string, int, error) {
	image, err := r.client.GetImage(r.context, ctr.Image())
	if err != nil {
		return "", 0, err
	}
	spec, err := containerd.GenerateSpec(containerd.WithImageConfig(r.context, image))
	if err != nil {
		return "", 0, err
	}
	container, err := r.client.NewContainer(r.context, ctr.Name(),
		containerd.WithSpec(spec),
		containerd.WithImage(image),
		containerd.WithNewRootFS(ctr.Name(), image))
	if err != nil {
		return "", 0, err
	}
	task, err := container.NewTask(r.context, containerd.Stdio)
	if err != nil {
		return "", 0, err
	}
	if err := task.Start(r.context); err != nil {
		task.Delete(r.context)
		return "", 0, err
	}
	return "", 0, nil
}

// Stop will stop/kill a container
func (r *ContainerdDriver) Stop(ctr Container) (string, int, error) {
	container, err := r.client.LoadContainer(r.context, ctr.Name())
	if err != nil {
		return "", 0, err
	}
	task, err := container.Task(r.context, nil)
	if err != nil {
		return "", 0, err
	}
	err = task.Kill(r.context, syscall.SIGKILL)
	if err != nil {
		return "", 0, err
	}
	_, err = task.Delete(r.context)
	if err != nil {
		return "", 0, err
	}
	return "", 0, nil
}

// Remove will remove a container; in the containerd case we simply call kill
// which will remove any container metadata if it was running
func (r *ContainerdDriver) Remove(ctr Container) (string, int, error) {
	container, err := r.client.LoadContainer(r.context, ctr.Name())
	if err != nil {
		return "", 0, err
	}
	err = container.Delete(r.context)
	if err != nil {
		return "", 0, err
	}
	return "", 0, nil
}

// Pause will pause a container
func (r *ContainerdDriver) Pause(ctr Container) (string, int, error) {
	container, err := r.client.LoadContainer(r.context, ctr.Name())
	if err != nil {
		return "", 0, err
	}
	task, err := container.Task(r.context, nil)
	if err != nil {
		return "", 0, err
	}
	err = task.Pause(r.context)
	if err != nil {
		return "", 0, err
	}
	return "", 0, nil
}

// Unpause will unpause/resume a container
func (r *ContainerdDriver) Unpause(ctr Container) (string, int, error) {
	container, err := r.client.LoadContainer(r.context, ctr.Name())
	if err != nil {
		return "", 0, err
	}
	task, err := container.Task(r.context, nil)
	if err != nil {
		return "", 0, err
	}
	err = task.Resume(r.context)
	if err != nil {
		return "", 0, err
	}
	return "", 0, nil
}
<commit_msg>Add timings to new gRPC containerd driver calls<commit_after>package driver

import (
	"context"
	"fmt"
	"syscall"
	"time"

	log "github.com/Sirupsen/logrus"
	"github.com/containerd/containerd"
	"github.com/containerd/containerd/namespaces"
)

const defaultContainerdPath = "/run/containerd/containerd.sock"

// ContainerdDriver is an implementation of the driver interface for using Containerd.
// This uses the provided client library which abstracts using the gRPC APIs directly.
// IMPORTANT: This implementation does not protect instance metadata for thread safely.
// At this time there is no understood use case for multi-threaded use of this implementation.
type ContainerdDriver struct {
	ctrdAddress string
	client      *containerd.Client
	context     context.Context
}

// ContainerdContainer is an implementation of the container metadata needed for containerd
type ContainerdContainer struct {
	name      string
	imageName string
	state     string
	process   string
	trace     bool
}

// NewContainerdDriver creates an instance of the containerd driver, providing a path to the ctr client
func NewContainerdDriver(path string) (Driver, error) {
	if path == "" {
		path = defaultContainerdPath
	}
	client, err := containerd.New(path)
	if err != nil {
		return &ContainerdDriver{}, err
	}
	bbCtx := namespaces.WithNamespace(context.Background(), "bb")
	driver := &ContainerdDriver{
		ctrdAddress: path,
		client:      client,
		context:     bbCtx,
	}
	return driver, nil
}

// newContainerdContainer creates the metadata object of a containerd-specific container with
// bundle, name, and any required additional information
func newContainerdContainer(name, image string, trace bool) Container {
	return &ContainerdContainer{
		name:      name,
		imageName: image,
		trace:     trace,
	}
}

// Name returns the name of the container
func (c *ContainerdContainer) Name() string {
	return c.name
}

// Trace returns whether the container should be started with tracing enabled
func (c *ContainerdContainer) Trace() bool {
	return c.trace
}

// Image returns the bundle path that runc will use
func (c *ContainerdContainer) Image() string {
	return c.imageName
}

// Process returns the process name in cases where this container instance is
// wrapping a potentially running container
func (c *ContainerdContainer) Process() string {
	return c.process
}

// State returns the queried state of the container (if available)
func (c *ContainerdContainer) State() string {
	return c.state
}

// Detached always returns true for containerd as IO streams are always detached
func (c *ContainerdContainer) Detached() bool {
	return true
}

// Type returns a driver.Type to indentify the driver implementation
func (r *ContainerdDriver) Type() Type {
	return Containerd
}

// Info returns
func (r *ContainerdDriver) Info() (string, error) {
	version, err := r.client.Version(r.context)
	if err != nil {
		return "", err
	}
	info := "containerd gRPC client driver (daemon: " + version.Version + "[Revision: " + version.Revision + "] )"
	return info, nil
}

// Create will create a container instance matching the specific needs
// of a driver
func (r *ContainerdDriver) Create(name, image string, detached bool, trace bool) (Container, error) {
	if _, err := r.client.GetImage(r.context, image); err != nil {
		// if the image isn't already in our namespaced context, then pull it
		// using the reference and default resolver (most likely DockerHub)
		if _, err := r.client.Pull(r.context, image, containerd.WithPullUnpack); err != nil {
			// error pulling the image
			return nil, err
		}
	}
	return newContainerdContainer(name, image, trace), nil
}

// Clean will clean the environment; removing any remaining containers in the runc metadata
func (r *ContainerdDriver) Clean() error {
	var tries int
	list, err := r.client.Containers(r.context)
	if err != nil {
		return fmt.Errorf("Error getting containerd list output: %v", err)
	}
	// try up to 3 times to handle any active containers in the list
	log.Infof("Attempting to cleanup containerd containers/metadata; %d listed", len(list))
	for len(list) > 0 && tries < 3 {
		log.Infof("containerd cleanup: Pass #%d", tries+1)
		// kill/stop and remove containers
		for _, ctr := range list {
			if err := ctr.Delete(r.context); err != nil {
				log.Errorf("Error deleting container %v: %v", ctr, err)
			}
		}
		tries++
		list, err = r.client.Containers(r.context)
		if err != nil {
			return fmt.Errorf("Error getting containerd list output: %v", err)
		}
	}
	log.Infof("containerd cleanup complete.")
	return nil
}

// Run will execute a container using the containerd driver.
func (r *ContainerdDriver) Run(ctr Container) (string, int, error) {
	start := time.Now()
	image, err := r.client.GetImage(r.context, ctr.Image())
	if err != nil {
		return "", 0, err
	}
	spec, err := containerd.GenerateSpec(containerd.WithImageConfig(r.context, image))
	if err != nil {
		return "", 0, err
	}
	container, err := r.client.NewContainer(r.context, ctr.Name(),
		containerd.WithSpec(spec),
		containerd.WithImage(image),
		containerd.WithNewRootFS(ctr.Name(), image))
	if err != nil {
		return "", 0, err
	}
	task, err := container.NewTask(r.context, containerd.Stdio)
	if err != nil {
		return "", 0, err
	}
	if err := task.Start(r.context); err != nil {
		task.Delete(r.context)
		return "", 0, err
	}
	elapsed := time.Since(start)
	msElapsed := int(elapsed.Nanoseconds() / 1000000)
	return "", msElapsed, nil
}

// Stop will stop/kill a container
func (r *ContainerdDriver) Stop(ctr Container) (string, int, error) {
	start := time.Now()
	container, err := r.client.LoadContainer(r.context, ctr.Name())
	if err != nil {
		return "", 0, err
	}
	task, err := container.Task(r.context, nil)
	if err != nil {
		return "", 0, err
	}
	err = task.Kill(r.context, syscall.SIGKILL)
	if err != nil {
		return "", 0, err
	}
	_, err = task.Delete(r.context)
	if err != nil {
		return "", 0, err
	}
	elapsed := time.Since(start)
	msElapsed := int(elapsed.Nanoseconds() / 1000000)
	return "", msElapsed, nil
}

// Remove will remove a container; in the containerd case we simply call kill
// which will remove any container metadata if it was running
func (r *ContainerdDriver) Remove(ctr Container) (string, int, error) {
	start := time.Now()
	container, err := r.client.LoadContainer(r.context, ctr.Name())
	if err != nil {
		return "", 0, err
	}
	err = container.Delete(r.context)
	if err != nil {
		return "", 0, err
	}
	elapsed := time.Since(start)
	msElapsed := int(elapsed.Nanoseconds() / 1000000)
	return "", msElapsed, nil
}

// Pause will pause a container
func (r *ContainerdDriver) Pause(ctr Container) (string, int, error) {
	start := time.Now()
	container, err := r.client.LoadContainer(r.context, ctr.Name())
	if err != nil {
		return "", 0, err
	}
	task, err := container.Task(r.context, nil)
	if err != nil {
		return "", 0, err
	}
	err = task.Pause(r.context)
	if err != nil {
		return "", 0, err
	}
	elapsed := time.Since(start)
	msElapsed := int(elapsed.Nanoseconds() / 1000000)
	return "", msElapsed, nil
}

// Unpause will unpause/resume a container
func (r *ContainerdDriver) Unpause(ctr Container) (string, int, error) {
	start := time.Now()
	container, err := r.client.LoadContainer(r.context, ctr.Name())
	if err != nil {
		return "", 0, err
	}
	task, err := container.Task(r.context, nil)
	if err != nil {
		return "", 0, err
	}
	err = task.Resume(r.context)
	if err != nil {
		return "", 0, err
	}
	elapsed := time.Since(start)
	msElapsed := int(elapsed.Nanoseconds() / 1000000)
	return "", msElapsed, nil
}
<|endoftext|>


<commit_before>// Package drivers talks to various database backends and retrieves table,
// column, type, and foreign key information
package drivers

import (
	"sort"
	"sync"

	"github.com/friendsofgo/errors"
	"github.com/volatiletech/sqlboiler/v4/importers"
	"github.com/volatiletech/strmangle"
)

// These constants are used in the config map passed into the driver
const (
	ConfigBlacklist      = "blacklist"
	ConfigWhitelist      = "whitelist"
	ConfigSchema         = "schema"
	ConfigAddEnumTypes   = "add-enum-types"
	ConfigEnumNullPrefix = "enum-null-prefix"

	ConfigUser    = "user"
	ConfigPass    = "pass"
	ConfigHost    = "host"
	ConfigPort    = "port"
	ConfigDBName  = "dbname"
	ConfigSSLMode = "sslmode"
)

// Interface abstracts either a side-effect imported driver or a binary
// that is called in order to produce the data required for generation.
type Interface interface {
	// Assemble the database information into a nice struct
	Assemble(config Config) (*DBInfo, error)
	// Templates to add/replace for generation
	Templates() (map[string]string, error)
	// Imports to merge for generation
	Imports() (importers.Collection, error)
}

// DBInfo is the database's table data and dialect.
type DBInfo struct {
	Schema  string  `json:"schema"`
	Tables  []Table `json:"tables"`
	Dialect Dialect `json:"dialect"`
}

// Dialect describes the databases requirements in terms of which features
// it speaks and what kind of quoting mechanisms it uses.
//
// WARNING: When updating this struct there is a copy of it inside
// the boil_queries template that is used for users to create queries
// without having to figure out what their dialect is.
type Dialect struct {
	LQ rune `json:"lq"`
	RQ rune `json:"rq"`

	UseIndexPlaceholders bool `json:"use_index_placeholders"`
	UseLastInsertID      bool `json:"use_last_insert_id"`
	UseSchema            bool `json:"use_schema"`
	UseDefaultKeyword    bool `json:"use_default_keyword"`

	// The following is mostly for T-SQL/MSSQL, what a show
	UseTopClause            bool `json:"use_top_clause"`
	UseOutputClause         bool `json:"use_output_clause"`
	UseCaseWhenExistsClause bool `json:"use_case_when_exists_clause"`

	// No longer used, left for backwards compatibility
	// should be removed in v5
	UseAutoColumns bool `json:"use_auto_columns"`
}

// Constructor breaks down the functionality required to implement a driver
// such that the drivers.Tables method can be used to reduce duplication in driver
// implementations.
type Constructor interface {
	TableNames(schema string, whitelist, blacklist []string) ([]string, error)
	Columns(schema, tableName string, whitelist, blacklist []string) ([]Column, error)
	PrimaryKeyInfo(schema, tableName string) (*PrimaryKey, error)
	ForeignKeyInfo(schema, tableName string) ([]ForeignKey, error)

	// TranslateColumnType takes a Database column type and returns a go column type.
	TranslateColumnType(Column) Column
}

// Constructor breaks down the functionality required to implement a driver
// such that the drivers.Views method can be used to reduce duplication in driver
// implementations.
type ViewConstructor interface {
	ViewNames(schema string, whitelist, blacklist []string) ([]string, error)
	ViewCapabilities(schema, viewName string) (ViewCapabilities, error)
	ViewColumns(schema, tableName string, whitelist, blacklist []string) ([]Column, error)

	// TranslateColumnType takes a Database column type and returns a go column type.
	TranslateColumnType(Column) Column
}

type TableColumnTypeTranslator interface {
	// TranslateTableColumnType takes a Database column type and table name and returns a go column type.
	TranslateTableColumnType(c Column, tableName string) Column
}

// table returns columns info for a given table
func table(c Constructor, schema string, name string, whitelist, blacklist []string) (Table, error) {
	var err error
	t := &Table{
		Name: name,
	}

	if t.Columns, err = c.Columns(schema, name, whitelist, blacklist); err != nil {
		return Table{}, errors.Wrapf(err, "unable to fetch table column info (%s)", name)
	}

	tr, ok := c.(TableColumnTypeTranslator)
	if ok {
		for i, col := range t.Columns {
			t.Columns[i] = tr.TranslateTableColumnType(col, name)
		}
	} else {
		for i, col := range t.Columns {
			t.Columns[i] = c.TranslateColumnType(col)
		}
	}

	if t.PKey, err = c.PrimaryKeyInfo(schema, name); err != nil {
		return Table{}, errors.Wrapf(err, "unable to fetch table pkey info (%s)", name)
	}

	if t.FKeys, err = c.ForeignKeyInfo(schema, name); err != nil {
		return Table{}, errors.Wrapf(err, "unable to fetch table fkey info (%s)", name)
	}

	filterPrimaryKey(t, whitelist, blacklist)
	filterForeignKeys(t, whitelist, blacklist)

	setIsJoinTable(t)

	return *t, nil
}

// Tables returns the metadata for all tables, minus the tables
// specified in the blacklist.
func Tables(c Constructor, schema string, whitelist, blacklist []string) ([]Table, error) {
	var err error

	names, err := c.TableNames(schema, whitelist, blacklist)
	if err != nil {
		return nil, errors.Wrap(err, "unable to get table names")
	}

	sort.Strings(names)

	tables := make([]Table, len(names))

	// gather tables info in parallel but restrict amount of concurrent requests by 10
	const concurrency = 10
	concurrentTickets := make(chan struct{}, concurrency)
	for i := 0; i < concurrency; i++ {
		concurrentTickets <- struct{}{}
	}

	wg := sync.WaitGroup{}
	errs := make(chan error, len(names))
	for i, name := range names {
		<-concurrentTickets
		wg.Add(1)
		go func(i int, name string) {
			defer wg.Done()
			defer func() { concurrentTickets <- struct{}{} }()
			t, err := table(c, schema, name, whitelist, blacklist)
			if err != nil {
				errs <- err
				return
			}
			tables[i] = t
		}(i, name)
	}

	wg.Wait()

	// return first error occurred if any
	if len(errs) != 0 {
		return nil, <-errs
	}

	// Relationships have a dependency on foreign key nullability.
	for i := range tables {
		tbl := &tables[i]
		setForeignKeyConstraints(tbl, tables)
	}
	for i := range tables {
		tbl := &tables[i]
		setRelationships(tbl, tables)
	}

	if vc, ok := c.(ViewConstructor); ok {
		viewTables, err := views(vc, schema, whitelist, blacklist)
		if err != nil {
			return nil, errors.Wrap(err, "unable to load views")
		}

		tables = append(tables, viewTables...)
	}

	return tables, nil
}

// views returns the metadata for all views, minus the views
// specified in the blacklist.
func views(c ViewConstructor, schema string, whitelist, blacklist []string) ([]Table, error) {
	var err error

	names, err := c.ViewNames(schema, whitelist, blacklist)
	if err != nil {
		return nil, errors.Wrap(err, "unable to get view names")
	}

	sort.Strings(names)

	var views []Table
	for _, name := range names {
		t := Table{
			IsView: true,
			Name:   name,
		}

		if t.ViewCapabilities, err = c.ViewCapabilities(schema, name); err != nil {
			return nil, errors.Wrapf(err, "unable to fetch view capabilities info (%s)", name)
		}

		if t.Columns, err = c.ViewColumns(schema, name, whitelist, blacklist); err != nil {
			return nil, errors.Wrapf(err, "unable to fetch view column info (%s)", name)
		}

		tr, ok := c.(TableColumnTypeTranslator)
		if ok {
			for i, col := range t.Columns {
				t.Columns[i] = tr.TranslateTableColumnType(col, name)
			}
		} else {
			for i, col := range t.Columns {
				t.Columns[i] = c.TranslateColumnType(col)
			}
		}

		views = append(views, t)
	}

	return views, nil
}

func knownColumn(table string, column string, whitelist, blacklist []string) bool {
	return (len(whitelist) == 0 ||
		strmangle.SetInclude(table, whitelist) ||
		strmangle.SetInclude(table+"."+column, whitelist) ||
		strmangle.SetInclude("*."+column, whitelist)) &&
		(len(blacklist) == 0 || (!strmangle.SetInclude(table, blacklist) &&
			!strmangle.SetInclude(table+"."+column, blacklist) &&
			!strmangle.SetInclude("*."+column, blacklist)))
}

// filterPrimaryKey filter columns from the primary key that are not in whitelist or in blacklist
func filterPrimaryKey(t *Table, whitelist, blacklist []string) {
	if t.PKey == nil {
		return
	}

	pkeyColumns := make([]string, 0, len(t.PKey.Columns))
	for _, c := range t.PKey.Columns {
		if knownColumn(t.Name, c, whitelist, blacklist) {
			pkeyColumns = append(pkeyColumns, c)
		}
	}
	t.PKey.Columns = pkeyColumns
}

// filterForeignKeys filter FK whose ForeignTable is not in whitelist or in blacklist
func filterForeignKeys(t *Table, whitelist, blacklist []string) {
	var fkeys []ForeignKey

	for _, fkey := range t.FKeys {
		if knownColumn(fkey.ForeignTable, fkey.ForeignColumn, whitelist, blacklist) &&
			knownColumn(fkey.Table, fkey.Column, whitelist, blacklist) {
			fkeys = append(fkeys, fkey)
		}
	}
	t.FKeys = fkeys
}

// setIsJoinTable if there are:
// A composite primary key involving two columns
// Both primary key columns are also foreign keys
func setIsJoinTable(t *Table) {
	if t.PKey == nil || len(t.PKey.Columns) != 2 || len(t.FKeys) < 2 || len(t.Columns) > 2 {
		return
	}

	for _, c := range t.PKey.Columns {
		found := false
		for _, f := range t.FKeys {
			if c == f.Column {
				found = true
				break
			}
		}
		if !found {
			return
		}
	}

	t.IsJoinTable = true
}

func setForeignKeyConstraints(t *Table, tables []Table) {
	for i, fkey := range t.FKeys {
		localColumn := t.GetColumn(fkey.Column)
		foreignTable := GetTable(tables, fkey.ForeignTable)
		foreignColumn := foreignTable.GetColumn(fkey.ForeignColumn)

		t.FKeys[i].Nullable = localColumn.Nullable
		t.FKeys[i].Unique = localColumn.Unique
		t.FKeys[i].ForeignColumnNullable = foreignColumn.Nullable
		t.FKeys[i].ForeignColumnUnique = foreignColumn.Unique
	}
}

func setRelationships(t *Table, tables []Table) {
	t.ToOneRelationships = toOneRelationships(*t, tables)
	t.ToManyRelationships = toManyRelationships(*t, tables)
}
<commit_msg>getting views info in parallel<commit_after>// Package drivers talks to various database backends and retrieves table,
// column, type, and foreign key information
package drivers

import (
	"sort"
	"sync"

	"github.com/friendsofgo/errors"
	"github.com/volatiletech/sqlboiler/v4/importers"
	"github.com/volatiletech/strmangle"
)

// These constants are used in the config map passed into the driver
const (
	ConfigBlacklist      = "blacklist"
	ConfigWhitelist      = "whitelist"
	ConfigSchema         = "schema"
	ConfigAddEnumTypes   = "add-enum-types"
	ConfigEnumNullPrefix = "enum-null-prefix"

	ConfigUser    = "user"
	ConfigPass    = "pass"
	ConfigHost    = "host"
	ConfigPort    = "port"
	ConfigDBName  = "dbname"
	ConfigSSLMode = "sslmode"

	// number of threads while getting tables and views info
	// TODO: allow override from config and cmdline
	concurrency = 10
)

// Interface abstracts either a side-effect imported driver or a binary
// that is called in order to produce the data required for generation.
type Interface interface {
	// Assemble the database information into a nice struct
	Assemble(config Config) (*DBInfo, error)
	// Templates to add/replace for generation
	Templates() (map[string]string, error)
	// Imports to merge for generation
	Imports() (importers.Collection, error)
}

// DBInfo is the database's table data and dialect.
type DBInfo struct {
	Schema  string  `json:"schema"`
	Tables  []Table `json:"tables"`
	Dialect Dialect `json:"dialect"`
}

// Dialect describes the databases requirements in terms of which features
// it speaks and what kind of quoting mechanisms it uses.
//
// WARNING: When updating this struct there is a copy of it inside
// the boil_queries template that is used for users to create queries
// without having to figure out what their dialect is.
type Dialect struct {
	LQ rune `json:"lq"`
	RQ rune `json:"rq"`

	UseIndexPlaceholders bool `json:"use_index_placeholders"`
	UseLastInsertID      bool `json:"use_last_insert_id"`
	UseSchema            bool `json:"use_schema"`
	UseDefaultKeyword    bool `json:"use_default_keyword"`

	// The following is mostly for T-SQL/MSSQL, what a show
	UseTopClause            bool `json:"use_top_clause"`
	UseOutputClause         bool `json:"use_output_clause"`
	UseCaseWhenExistsClause bool `json:"use_case_when_exists_clause"`

	// No longer used, left for backwards compatibility
	// should be removed in v5
	UseAutoColumns bool `json:"use_auto_columns"`
}

// Constructor breaks down the functionality required to implement a driver
// such that the drivers.Tables method can be used to reduce duplication in driver
// implementations.
type Constructor interface {
	TableNames(schema string, whitelist, blacklist []string) ([]string, error)
	Columns(schema, tableName string, whitelist, blacklist []string) ([]Column, error)
	PrimaryKeyInfo(schema, tableName string) (*PrimaryKey, error)
	ForeignKeyInfo(schema, tableName string) ([]ForeignKey, error)

	// TranslateColumnType takes a Database column type and returns a go column type.
	TranslateColumnType(Column) Column
}

// Constructor breaks down the functionality required to implement a driver
// such that the drivers.Views method can be used to reduce duplication in driver
// implementations.
type ViewConstructor interface {
	ViewNames(schema string, whitelist, blacklist []string) ([]string, error)
	ViewCapabilities(schema, viewName string) (ViewCapabilities, error)
	ViewColumns(schema, tableName string, whitelist, blacklist []string) ([]Column, error)

	// TranslateColumnType takes a Database column type and returns a go column type.
	TranslateColumnType(Column) Column
}

type TableColumnTypeTranslator interface {
	// TranslateTableColumnType takes a Database column type and table name and returns a go column type.
	TranslateTableColumnType(c Column, tableName string) Column
}

// Tables returns the metadata for all tables, minus the tables
// specified in the blacklist.
func Tables(c Constructor, schema string, whitelist, blacklist []string) ([]Table, error) {
	var err error
	var ret []Table

	ret, err = tables(c, schema, whitelist, blacklist)
	if err != nil {
		return nil, errors.Wrap(err, "unable to load tables")
	}

	if vc, ok := c.(ViewConstructor); ok {
		v, err := views(vc, schema, whitelist, blacklist)
		if err != nil {
			return nil, errors.Wrap(err, "unable to load views")
		}
		ret = append(ret, v...)
	}

	return ret, nil
}

func tables(c Constructor, schema string, whitelist, blacklist []string) ([]Table, error) {
	var err error

	names, err := c.TableNames(schema, whitelist, blacklist)
	if err != nil {
		return nil, errors.Wrap(err, "unable to get table names")
	}

	sort.Strings(names)

	ret := make([]Table, len(names))

	limiter := newConcurrencyLimiter(concurrency)
	wg := sync.WaitGroup{}
	errs := make(chan error, len(names))
	for i, name := range names {
		limiter.get()
		wg.Add(1)
		go func(i int, name string) {
			defer wg.Done()
			defer limiter.put()
			t, err := table(c, schema, name, whitelist, blacklist)
			if err != nil {
				errs <- err
				return
			}
			ret[i] = t
		}(i, name)
	}

	wg.Wait()

	// return first error occurred if any
	if len(errs) > 0 {
		return nil, <-errs
	}

	// Relationships have a dependency on foreign key nullability.
	for i := range ret {
		tbl := &ret[i]
		setForeignKeyConstraints(tbl, ret)
	}
	for i := range ret {
		tbl := &ret[i]
		setRelationships(tbl, ret)
	}

	return ret, nil
}

// table returns columns info for a given table
func table(c Constructor, schema string, name string, whitelist, blacklist []string) (Table, error) {
	var err error
	t := &Table{
		Name: name,
	}

	if t.Columns, err = c.Columns(schema, name, whitelist, blacklist); err != nil {
		return Table{}, errors.Wrapf(err, "unable to fetch table column info (%s)", name)
	}

	tr, ok := c.(TableColumnTypeTranslator)
	if ok {
		for i, col := range t.Columns {
			t.Columns[i] = tr.TranslateTableColumnType(col, name)
		}
	} else {
		for i, col := range t.Columns {
			t.Columns[i] = c.TranslateColumnType(col)
		}
	}

	if t.PKey, err = c.PrimaryKeyInfo(schema, name); err != nil {
		return Table{}, errors.Wrapf(err, "unable to fetch table pkey info (%s)", name)
	}

	if t.FKeys, err = c.ForeignKeyInfo(schema, name); err != nil {
		return Table{}, errors.Wrapf(err, "unable to fetch table fkey info (%s)", name)
	}

	filterPrimaryKey(t, whitelist, blacklist)
	filterForeignKeys(t, whitelist, blacklist)

	setIsJoinTable(t)

	return *t, nil
}

// views returns the metadata for all views, minus the views
// specified in the blacklist.
func views(c ViewConstructor, schema string, whitelist, blacklist []string) ([]Table, error) {
	var err error

	names, err := c.ViewNames(schema, whitelist, blacklist)
	if err != nil {
		return nil, errors.Wrap(err, "unable to get view names")
	}

	sort.Strings(names)

	ret := make([]Table, len(names))

	limiter := newConcurrencyLimiter(concurrency)
	wg := sync.WaitGroup{}
	errs := make(chan error, len(names))
	for i, name := range names {
		limiter.get()
		wg.Add(1)
		go func(i int, name string) {
			defer wg.Done()
			defer limiter.put()
			t, err := view(c, schema, name, whitelist, blacklist)
			if err != nil {
				errs <- err
				return
			}
			ret[i] = t
		}(i, name)
	}

	wg.Wait()

	// return first error occurred if any
	if len(errs) > 0 {
		return nil, <-errs
	}

	return ret, nil
}

// view returns columns info for a given view
func view(c ViewConstructor, schema string, name string, whitelist, blacklist []string) (Table, error) {
	var err error
	t := Table{
		IsView: true,
		Name:   name,
	}

	if t.ViewCapabilities, err = c.ViewCapabilities(schema, name); err != nil {
		return Table{}, errors.Wrapf(err, "unable to fetch view capabilities info (%s)", name)
	}

	if t.Columns, err = c.ViewColumns(schema, name, whitelist, blacklist); err != nil {
		return Table{}, errors.Wrapf(err, "unable to fetch view column info (%s)", name)
	}

	tr, ok := c.(TableColumnTypeTranslator)
	if ok {
		for i, col := range t.Columns {
			t.Columns[i] = tr.TranslateTableColumnType(col, name)
		}
	} else {
		for i, col := range t.Columns {
			t.Columns[i] = c.TranslateColumnType(col)
		}
	}

	return t, nil
}

func knownColumn(table string, column string, whitelist, blacklist []string) bool {
	return (len(whitelist) == 0 ||
		strmangle.SetInclude(table, whitelist) ||
		strmangle.SetInclude(table+"."+column, whitelist) ||
		strmangle.SetInclude("*."+column, whitelist)) &&
		(len(blacklist) == 0 || (!strmangle.SetInclude(table, blacklist) &&
			!strmangle.SetInclude(table+"."+column, blacklist) &&
			!strmangle.SetInclude("*."+column, blacklist)))
}

// filterPrimaryKey filter columns from the primary key that are not in whitelist or in blacklist
func filterPrimaryKey(t *Table, whitelist, blacklist []string) {
	if t.PKey == nil {
		return
	}

	pkeyColumns := make([]string, 0, len(t.PKey.Columns))
	for _, c := range t.PKey.Columns {
		if knownColumn(t.Name, c, whitelist, blacklist) {
			pkeyColumns = append(pkeyColumns, c)
		}
	}
	t.PKey.Columns = pkeyColumns
}

// filterForeignKeys filter FK whose ForeignTable is not in whitelist or in blacklist
func filterForeignKeys(t *Table, whitelist, blacklist []string) {
	var fkeys []ForeignKey

	for _, fkey := range t.FKeys {
		if knownColumn(fkey.ForeignTable, fkey.ForeignColumn, whitelist, blacklist) &&
			knownColumn(fkey.Table, fkey.Column, whitelist, blacklist) {
			fkeys = append(fkeys, fkey)
		}
	}
	t.FKeys = fkeys
}

// setIsJoinTable if there are:
// A composite primary key involving two columns
// Both primary key columns are also foreign keys
func setIsJoinTable(t *Table) {
	if t.PKey == nil || len(t.PKey.Columns) != 2 || len(t.FKeys) < 2 || len(t.Columns) > 2 {
		return
	}

	for _, c := range t.PKey.Columns {
		found := false
		for _, f := range t.FKeys {
			if c == f.Column {
				found = true
				break
			}
		}
		if !found {
			return
		}
	}

	t.IsJoinTable = true
}

func setForeignKeyConstraints(t *Table, tables []Table) {
	for i, fkey := range t.FKeys {
		localColumn := t.GetColumn(fkey.Column)
		foreignTable := GetTable(tables, fkey.ForeignTable)
		foreignColumn := foreignTable.GetColumn(fkey.ForeignColumn)

		t.FKeys[i].Nullable = localColumn.Nullable
		t.FKeys[i].Unique = localColumn.Unique
		t.FKeys[i].ForeignColumnNullable = foreignColumn.Nullable
		t.FKeys[i].ForeignColumnUnique = foreignColumn.Unique
	}
}

func setRelationships(t *Table, tables []Table) {
	t.ToOneRelationships = toOneRelationships(*t, tables)
	t.ToManyRelationships = toManyRelationships(*t, tables)
}

// concurrencyCounter is a helper structure that can limit amount of concurrently processed requests
type concurrencyLimiter chan struct{}

func newConcurrencyLimiter(capacity int) concurrencyLimiter {
	ret := make(concurrencyLimiter, capacity)
	for i := 0; i < capacity; i++ {
		ret <- struct{}{}
	}

	return ret
}

func (c concurrencyLimiter) get() {
	<-c
}

func (c concurrencyLimiter) put() {
	c <- struct{}{}
}
<|endoftext|>


<commit_before>// Package dyncrc16 implements the Dynastream CRC-16 checksum.
package dyncrc16

// The size of a Dynastream CRC-16 checksum in bytes.
const Size = 2

var crcTable = [...]uint16{
	0x0000, 0xCC01, 0xD801, 0x1400,
	0xF001, 0x3C00, 0x2800, 0xE401,
	0xA001, 0x6C00, 0x7800, 0xB401,
	0x5000, 0x9C01, 0x8801, 0x4400,
}

// crc16 represents the partial evaluation of a checksum.
type crc16 uint16

// New returns a new hash.Hash16 computing the Dynastream CRC-16 checksum.
func New() Hash16 {
	c := new(crc16)
	return c
}

func (c *crc16) Sum16() uint16 {
	return uint16(*c)
}

func (c *crc16) Reset() {
	*c = 0
}

func (c *crc16) Size() int {
	return Size
}

func (c *crc16) BlockSize() int {
	return 1
}

func (c *crc16) Sum(in []byte) []byte {
	s := c.Sum16()
	return append(in, byte(s>>8), byte(s))
}

func (c *crc16) Write(data []byte) (int, error) {
	*c = update(*c, data)
	return len(data), nil
}

// Checksum returns the Dynastream CRC-16 checksum of data.
func Checksum(data []byte) uint16 {
	var c crc16
	c = update(c, data)
	return c.Sum16()
}

// Add data to the running checksum c.
func update(c crc16, data []byte) crc16 {
	for _, d := range data {
		c = updateByte(c, d)
	}
	return c
}

// Add data to the running checksum c.
func updateByte(c crc16, data byte) crc16 {
	d := uint16(c)

	// compute checksum of lower four bits of byte
	tmp := crcTable[d&0x0F]
	d = (d >> 4) & 0x0FFF
	d = d ^ tmp ^ crcTable[data&0x0F]

	// now compute checksum of upper four bits of byte
	tmp = crcTable[d&0x0F]
	d = (d >> 4) & 0x0FFF
	d = d ^ tmp ^ crcTable[(data>>4)&0x0F]

	return crc16(d)
}
<commit_msg>dyncrc16: unexport size constant<commit_after>// Package dyncrc16 implements the Dynastream CRC-16 checksum.
package dyncrc16

const size = 2

var crcTable = [...]uint16{
	0x0000, 0xCC01, 0xD801, 0x1400,
	0xF001, 0x3C00, 0x2800, 0xE401,
	0xA001, 0x6C00, 0x7800, 0xB401,
	0x5000, 0x9C01, 0x8801, 0x4400,
}

// crc16 represents the partial evaluation of a checksum.
type crc16 uint16

// New returns a new hash.Hash16 computing the Dynastream CRC-16 checksum.
func New() Hash16 {
	c := new(crc16)
	return c
}

func (c *crc16) Sum16() uint16 {
	return uint16(*c)
}

func (c *crc16) Reset() {
	*c = 0
}

func (c *crc16) Size() int {
	return size
}

func (c *crc16) BlockSize() int {
	return 1
}

func (c *crc16) Sum(in []byte) []byte {
	s := c.Sum16()
	return append(in, byte(s>>8), byte(s))
}

func (c *crc16) Write(data []byte) (int, error) {
	*c = update(*c, data)
	return len(data), nil
}

// Checksum returns the Dynastream CRC-16 checksum of data.
func Checksum(data []byte) uint16 {
	var c crc16
	c = update(c, data)
	return c.Sum16()
}

// Add data to the running checksum c.
func update(c crc16, data []byte) crc16 {
	for _, d := range data {
		c = updateByte(c, d)
	}
	return c
}

// Add data to the running checksum c.
func updateByte(c crc16, data byte) crc16 {
	d := uint16(c)

	// compute checksum of lower four bits of byte
	tmp := crcTable[d&0x0F]
	d = (d >> 4) & 0x0FFF
	d = d ^ tmp ^ crcTable[data&0x0F]

	// now compute checksum of upper four bits of byte
	tmp = crcTable[d&0x0F]
	d = (d >> 4) & 0x0FFF
	d = d ^ tmp ^ crcTable[(data>>4)&0x0F]

	return crc16(d)
}
<|endoftext|>


<commit_before>// Copyright 2016 Documize Inc. <legal@documize.com>. All rights reserved.
//
// This software (Documize Community Edition) is licensed under
// GNU AGPL v3 http://www.gnu.org/licenses/agpl-3.0.en.html
//
// You can operate outside the AGPL restrictions by purchasing
// Documize Enterprise Edition and obtaining a commercial license
// by contacting <sales@documize.com>.
//
// https://documize.com

// This package provides Documize as a single executable.
package main

import (
	"crypto/rand"
	"fmt"
	"strings"
	"time"

	"github.com/documize/community/core/api"
	"github.com/documize/community/core/api/endpoint"
	"github.com/documize/community/core/api/request"
	"github.com/documize/community/core/database"
	"github.com/documize/community/core/env"
	"github.com/documize/community/core/section"
	"github.com/documize/community/core/web"
	"github.com/documize/community/edition/logging"
	_ "github.com/documize/community/embed" // the compressed front-end code and static data
	_ "github.com/go-sql-driver/mysql"      // the mysql driver is required behind the scenes
	"github.com/jmoiron/sqlx"
)

func init() {
	// runtime stores server/application level information
	runtime := env.Runtime{}

	// wire up logging implementation
	runtime.Log = logging.NewLogger()

	// define product edition details
	runtime.Product = env.ProdInfo{}
	runtime.Product.Major = "1"
	runtime.Product.Minor = "50"
	runtime.Product.Patch = "0"
	runtime.Product.Version = fmt.Sprintf("%s.%s.%s", runtime.Product.Major, runtime.Product.Minor, runtime.Product.Patch)
	runtime.Product.Edition = "Community"
	runtime.Product.Title = fmt.Sprintf("%s Edition", runtime.Product.Edition)
	runtime.Product.License = env.License{}
	runtime.Product.License.Seats = 1
	runtime.Product.License.Valid = true
	runtime.Product.License.Trial = false
	runtime.Product.License.Edition = "Community"

	runtime.Flags = env.ParseFlags()
	flagPrep(&runtime)

	// temp code repair
	api.Runtime = runtime
	request.Db = runtime.Db
}

func main() {

	// process the db value first
	// env.Parse("db")

	section.Register()

	ready := make(chan struct{}, 1) // channel is used for testing
	endpoint.Serve(ready)
}

func flagPrep(r *env.Runtime) bool {
	// Prepare DB
	db, err := sqlx.Open("mysql", stdConn(r.Flags.DBConn))
	if err != nil {
		r.Log.Error("unable to setup database", err)
	}

	r.Db = db
	r.Db.SetMaxIdleConns(30)
	r.Db.SetMaxOpenConns(100)
	r.Db.SetConnMaxLifetime(time.Second * 14400)

	err = r.Db.Ping()
	if err != nil {
		r.Log.Error("unable to connect to database, connection string should be of the form: '"+
			"username:password@tcp(host:3306)/database'", err)
		return false
	}

	// go into setup mode if required
	if r.Flags.SiteMode != web.SiteModeOffline {
		if database.Check(*r) {
			if err := database.Migrate(true /* the config table exists */); err != nil {
				r.Log.Error("unable to run database migration", err)
				return false
			}
		} else {
			r.Log.Info("going into setup mode to prepare new database")
		}
	}

	// Prepare SALT
	if r.Flags.Salt == "" {
		b := make([]byte, 17)

		_, err := rand.Read(b)
		if err != nil {
			r.Log.Error("problem using crypto/rand", err)
			return false
		}

		for k, v := range b {
			if (v >= 'a' && v <= 'z') || (v >= 'A' && v <= 'Z') || (v >= '0' && v <= '0') {
				b[k] = v
			} else {
				s := fmt.Sprintf("%x", v)
				b[k] = s[0]
			}
		}

		r.Flags.Salt = string(b)
		r.Log.Info("please set DOCUMIZESALT or use -salt with this value: " + r.Flags.Salt)
	}

	// Prepare HTTP ports
	if r.Flags.SSLCertFile == "" && r.Flags.SSLKeyFile == "" {
		if r.Flags.HTTPPort == "" {
			r.Flags.HTTPPort = "80"
		}
	} else {
		if r.Flags.HTTPPort == "" {
			r.Flags.HTTPPort = "443"
		}
	}

	return true
}

var stdParams = map[string]string{
	"charset":          "utf8",
	"parseTime":        "True",
	"maxAllowedPacket": "4194304", // 4194304 // 16777216 = 16MB
}

func stdConn(cs string) string {
	queryBits := strings.Split(cs, "?")
	ret := queryBits[0] + "?"
	retFirst := true
	if len(queryBits) == 2 {
		paramBits := strings.Split(queryBits[1], "&")
		for _, pb := range paramBits {
			found := false
			if assignBits := strings.Split(pb, "="); len(assignBits) == 2 {
				_, found = stdParams[strings.TrimSpace(assignBits[0])]
			}
			if !found { // if we can't work out what it is, put it through
				if retFirst {
					retFirst = false
				} else {
					ret += "&"
				}
				ret += pb
			}
		}
	}
	for k, v := range stdParams {
		if retFirst {
			retFirst = false
		} else {
			ret += "&"
		}
		ret += k + "=" + v
	}
	return ret
}
<commit_msg>check parsed flags<commit_after>// Copyright 2016 Documize Inc. <legal@documize.com>. All rights reserved.
//
// This software (Documize Community Edition) is licensed under
// GNU AGPL v3 http://www.gnu.org/licenses/agpl-3.0.en.html
//
// You can operate outside the AGPL restrictions by purchasing
// Documize Enterprise Edition and obtaining a commercial license
// by contacting <sales@documize.com>.
//
// https://documize.com

// This package provides Documize as a single executable.
package main

import (
	"crypto/rand"
	"fmt"
	"strings"
	"time"

	"github.com/documize/community/core/api"
	"github.com/documize/community/core/api/endpoint"
	"github.com/documize/community/core/api/request"
	"github.com/documize/community/core/database"
	"github.com/documize/community/core/env"
	"github.com/documize/community/core/section"
	"github.com/documize/community/core/web"
	"github.com/documize/community/edition/logging"
	_ "github.com/documize/community/embed" // the compressed front-end code and static data
	_ "github.com/go-sql-driver/mysql"      // the mysql driver is required behind the scenes
	"github.com/jmoiron/sqlx"
)

func init() {
	// runtime stores server/application level information
	runtime := env.Runtime{}

	// wire up logging implementation
	runtime.Log = logging.NewLogger()

	// define product edition details
	runtime.Product = env.ProdInfo{}
	runtime.Product.Major = "1"
	runtime.Product.Minor = "50"
	runtime.Product.Patch = "0"
	runtime.Product.Version = fmt.Sprintf("%s.%s.%s", runtime.Product.Major, runtime.Product.Minor, runtime.Product.Patch)
	runtime.Product.Edition = "Community"
	runtime.Product.Title = fmt.Sprintf("%s Edition", runtime.Product.Edition)
	runtime.Product.License = env.License{}
	runtime.Product.License.Seats = 1
	runtime.Product.License.Valid = true
	runtime.Product.License.Trial = false
	runtime.Product.License.Edition = "Community"

	// parse settings from command line and environment
	runtime.Flags = env.ParseFlags()
	flagsOK := flagPrep(&runtime)

	if flagsOK {
		// runtime.Log = runtime.Log.SetDB(runtime.Db)
	}

	// temp code repair
	api.Runtime = runtime
	request.Db = runtime.Db
}

func main() {

	// process the db value first
	// env.Parse("db")

	section.Register()

	ready := make(chan struct{}, 1) // channel is used for testing
	endpoint.Serve(ready)
}

func flagPrep(r *env.Runtime) bool {
	// Prepare DB
	db, err := sqlx.Open("mysql", stdConn(r.Flags.DBConn))
	if err != nil {
		r.Log.Error("unable to setup database", err)
	}

	r.Db = db
	r.Db.SetMaxIdleConns(30)
	r.Db.SetMaxOpenConns(100)
	r.Db.SetConnMaxLifetime(time.Second * 14400)

	err = r.Db.Ping()
	if err != nil {
		r.Log.Error("unable to connect to database, connection string should be of the form: '"+
			"username:password@tcp(host:3306)/database'", err)
		return false
	}

	// go into setup mode if required
	if r.Flags.SiteMode != web.SiteModeOffline {
		if database.Check(*r) {
			if err := database.Migrate(true /* the config table exists */); err != nil {
				r.Log.Error("unable to run database migration", err)
				return false
			}
		} else {
			r.Log.Info("going into setup mode to prepare new database")
		}
	}

	// Prepare SALT
	if r.Flags.Salt == "" {
		b := make([]byte, 17)

		_, err := rand.Read(b)
		if err != nil {
			r.Log.Error("problem using crypto/rand", err)
			return false
		}

		for k, v := range b {
			if (v >= 'a' && v <= 'z') || (v >= 'A' && v <= 'Z') || (v >= '0' && v <= '0') {
				b[k] = v
			} else {
				s := fmt.Sprintf("%x", v)
				b[k] = s[0]
			}
		}

		r.Flags.Salt = string(b)
		r.Log.Info("please set DOCUMIZESALT or use -salt with this value: " + r.Flags.Salt)
	}

	// Prepare HTTP ports
	if r.Flags.SSLCertFile == "" && r.Flags.SSLKeyFile == "" {
		if r.Flags.HTTPPort == "" {
			r.Flags.HTTPPort = "80"
		}
	} else {
		if r.Flags.HTTPPort == "" {
			r.Flags.HTTPPort = "443"
		}
	}

	return true
}

var stdParams = map[string]string{
	"charset":          "utf8",
	"parseTime":        "True",
	"maxAllowedPacket": "4194304", // 4194304 // 16777216 = 16MB
}

func stdConn(cs string) string {
	queryBits := strings.Split(cs, "?")
	ret := queryBits[0] + "?"
	retFirst := true
	if len(queryBits) == 2 {
		paramBits := strings.Split(queryBits[1], "&")
		for _, pb := range paramBits {
			found := false
			if assignBits := strings.Split(pb, "="); len(assignBits) == 2 {
				_, found = stdParams[strings.TrimSpace(assignBits[0])]
			}
			if !found { // if we can't work out what it is, put it through
				if retFirst {
					retFirst = false
				} else {
					ret += "&"
				}
				ret += pb
			}
		}
	}
	for k, v := range stdParams {
		if retFirst {
			retFirst = false
		} else {
			ret += "&"
		}
		ret += k + "=" + v
	}
	return ret
}
<|endoftext|>


<commit_before>// Copyright 2014 Arne Roomann-Kurrik
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package twodee

import (
	"time"
)

type Layer interface {
	Render()
	Update(elapsed time.Duration)
	Delete()
	HandleEvent(evt Event) bool
}

type Layers struct {
	layers []Layer
}

func NewLayers() *Layers {
	return &Layers{}
}

func (l *Layers) Push(layer Layer) {
	l.layers = append(l.layers, layer)
}

func (l *Layers) Pop() (layer Layer) {
	var (
		index = len(l.layers) - 1
	)
	layer = l.layers[index]
	l.layers = l.layers[:index]
	return
}

func (l *Layers) Render() {
	for _, layer := range l.layers {
		layer.Render()
	}
}

func (l *Layers) Update(elapsed time.Duration) {
	for _, layer := range l.layers {
		layer.Update(elapsed)
	}
}

func (l *Layers) Delete() {
	for _, layer := range l.layers {
		layer.Delete()
	}
}

func (l *Layers) HandleEvent(evt Event) bool {
	for _, layer := range l.layers {
		if layer.HandleEvent(evt) == false {
			return false
		}
	}
	return true
}
<commit_msg>Process events in reverse layer order.<commit_after>// Copyright 2014 Arne Roomann-Kurrik
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package twodee

import (
	"time"
)

type Layer interface {
	Render()
	Update(elapsed time.Duration)
	Delete()
	HandleEvent(evt Event) bool
}

type Layers struct {
	layers []Layer
}

func NewLayers() *Layers {
	return &Layers{}
}

func (l *Layers) Push(layer Layer) {
	l.layers = append(l.layers, layer)
}

func (l *Layers) Pop() (layer Layer) {
	var (
		index = len(l.layers) - 1
	)
	layer = l.layers[index]
	l.layers = l.layers[:index]
	return
}

func (l *Layers) Render() {
	for _, layer := range l.layers {
		layer.Render()
	}
}

func (l *Layers) Update(elapsed time.Duration) {
	for _, layer := range l.layers {
		layer.Update(elapsed)
	}
}

func (l *Layers) Delete() {
	for _, layer := range l.layers {
		layer.Delete()
	}
}

func (l *Layers) HandleEvent(evt Event) bool {
	for i := len(l.layers) - 1; i >= 0; i-- {
		if l.layers[i].HandleEvent(evt) == false {
			return false
		}
	}
	return true
}
<|endoftext|>


<commit_before>package gitosis

import (
	"errors"
	"fmt"
	"github.com/timeredbull/tsuru/log"
	"os"
	"path"
	"syscall"
)

// AddKeys adds a user's public key to the keydir
func AddKey(group, member, key string) error {
	c, err := getConfig()
	if err != nil {
		return err
	}
	if !c.HasSection("group " + group) {
		return errors.New("Group not found")
	}
	p, err := getKeydirPath()
	if err != nil {
		return err
	}
	err = os.MkdirAll(p, 0755)
	if err != nil {
		return err
	}
	dir, err := os.Open(p)
	if err != nil {
		return err
	}
	filenames, err := dir.Readdirnames(0)
	if err != nil {
		return err
	}
	pattern := member + "_key%d"
	counter := 1
	actualMember := fmt.Sprintf(pattern, counter)
	filename := actualMember + ".pub"
	for _, f := range filenames {
		if f == filename {
			counter++
			actualMember = fmt.Sprintf(pattern, counter)
			filename = actualMember + ".pub"
		}
	}
	keyfilename := path.Join(p, filename)
	keyfile, err := os.OpenFile(keyfilename, syscall.O_WRONLY|syscall.O_CREAT, 0644)
	if err != nil {
		return err
	}
	defer keyfile.Close()
	n, err := keyfile.WriteString(key)
	if err != nil || n != len(key) {
		return err
	}
	err = addMember(group, actualMember)
	if err != nil {
		err = os.Remove(keyfilename)
		if err != nil {
			log.Panicf("Fatal error: the key file %s was left in the keydir", keyfilename)
			return err
		}
		return errors.New("Failed to add member to the group, the key file was not saved")
	}
	return nil
}
<commit_msg>gitosis: extracting key file name calculation<commit_after>package gitosis

import (
	"errors"
	"fmt"
	"github.com/timeredbull/tsuru/log"
	"os"
	"path"
	"syscall"
)

// AddKeys adds a user's public key to the keydir
func AddKey(group, member, key string) error {
	c, err := getConfig()
	if err != nil {
		return err
	}
	if !c.HasSection("group " + group) {
		return errors.New("Group not found")
	}
	p, err := getKeydirPath()
	if err != nil {
		return err
	}
	err = os.MkdirAll(p, 0755)
	if err != nil {
		return err
	}
	filename, actualMember, err := nextAvailableKey(p, member)
	if err != nil {
		return err
	}
	keyfilename := path.Join(p, filename)
	keyfile, err := os.OpenFile(keyfilename, syscall.O_WRONLY|syscall.O_CREAT, 0644)
	if err != nil {
		return err
	}
	defer keyfile.Close()
	n, err := keyfile.WriteString(key)
	if err != nil || n != len(key) {
		return err
	}
	err = addMember(group, actualMember)
	if err != nil {
		err = os.Remove(keyfilename)
		if err != nil {
			log.Panicf("Fatal error: the key file %s was left in the keydir", keyfilename)
			return err
		}
		return errors.New("Failed to add member to the group, the key file was not saved")
	}
	return nil
}

func nextAvailableKey(keydirname, member string) (string, string, error) {
	keydir, err := os.Open(keydirname)
	if err != nil {
		return "", "", err
	}
	defer keydir.Close()
	filenames, err := keydir.Readdirnames(0)
	if err != nil {
		return "", "", err
	}
	pattern := member + "_key%d"
	counter := 1
	actualMember := fmt.Sprintf(pattern, counter)
	filename := actualMember + ".pub"
	for _, f := range filenames {
		if f == filename {
			counter++
			actualMember = fmt.Sprintf(pattern, counter)
			filename = actualMember + ".pub"
		}
	}
	return filename, actualMember, nil
}
<|endoftext|>


<commit_before>/*
Package sdk is the gRPC implementation of the SDK gRPC server
Copyright 2018 Portworx

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

	http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
package sdk

import (
	"context"
	"fmt"

	"github.com/libopenstorage/openstorage/api"
	"github.com/libopenstorage/openstorage/volume"
	"google.golang.org/grpc/codes"
	"google.golang.org/grpc/status"
)

// CredentialServer is an implementation of the gRPC OpenStorageCredential interface
type CredentialServer struct {
	driver volume.VolumeDriver
}

// Create method creates credentials
func (s *CredentialServer) Create(
	ctx context.Context,
	req *api.SdkCredentialCreateRequest,
) (*api.SdkCredentialCreateResponse, error) {

	if len(req.GetName()) == 0 {
		return nil, status.Error(codes.InvalidArgument, "Must supply a name")
	} else if aws := req.GetAwsCredential(); aws != nil {
		return s.awsCreate(ctx, req, aws)
	} else if azure := req.GetAzureCredential(); azure != nil {
		return s.azureCreate(ctx, req, azure)
	} else if google := req.GetGoogleCredential(); google != nil {
		return s.googleCreate(ctx, req, google)
	}
	return nil, status.Error(codes.InvalidArgument, "Unknown credential type")

}

func (s *CredentialServer) awsCreate(
	ctx context.Context,
	req *api.SdkCredentialCreateRequest,
	aws *api.SdkAwsCredentialRequest,
) (*api.SdkCredentialCreateResponse, error) {

	if len(aws.GetAccessKey()) == 0 {
		return nil, status.Error(codes.InvalidArgument, "Must supply Access Key")
	}

	if len(aws.GetSecretKey()) == 0 {
		return nil, status.Error(codes.InvalidArgument, "Must supply Secret Key")
	}

	if len(aws.GetRegion()) == 0 {
		return nil, status.Error(codes.InvalidArgument, "Must supply Region Key")
	}

	if len(aws.GetEndpoint()) == 0 {
		return nil, status.Error(codes.InvalidArgument, "Must supply Endpoint Key")
	}

	params := make(map[string]string)

	params[api.OptCredType] = "s3"
	params[api.OptCredName] = req.GetName()
	params[api.OptCredEncrKey] = req.GetEncryptionKey()
	params[api.OptCredBucket] = req.GetBucket()
	params[api.OptCredRegion] = aws.GetRegion()
	params[api.OptCredEndpoint] = aws.GetEndpoint()
	params[api.OptCredAccessKey] = aws.GetAccessKey()
	params[api.OptCredSecretKey] = aws.GetSecretKey()
	params[api.OptCredDisableSSL] = fmt.Sprintf("%v", aws.GetDisableSsl())

	uuid, err := s.driver.CredsCreate(params)

	if err != nil {
		return nil, status.Errorf(
			codes.Internal,
			"failed to create aws credentials: %v",
			err.Error())
	}

	err = validateAndDeleteIfInvalid(s, uuid)

	if err != nil {
		return nil, err
	}
	return &api.SdkCredentialCreateResponse{CredentialId: uuid}, nil
}

func (s *CredentialServer) azureCreate(
	ctx context.Context,
	req *api.SdkCredentialCreateRequest,
	azure *api.SdkAzureCredentialRequest,
) (*api.SdkCredentialCreateResponse, error) {

	if len(azure.GetAccountKey()) == 0 {
		return nil, status.Error(codes.InvalidArgument, "Must supply Account Key")
	}

	if len(azure.GetAccountName()) == 0 {
		return nil, status.Error(codes.InvalidArgument, "Must supply Account name")
	}

	params := make(map[string]string)

	params[api.OptCredType] = "azure"
	params[api.OptCredName] = req.GetName()
	params[api.OptCredEncrKey] = req.GetEncryptionKey()
	params[api.OptCredBucket] = req.GetBucket()
	params[api.OptCredAzureAccountKey] = azure.GetAccountKey()
	params[api.OptCredAzureAccountName] = azure.GetAccountName()

	uuid, err := s.driver.CredsCreate(params)

	if err != nil {
		return nil, status.Errorf(
			codes.Internal,
			"failed to create Azure credentials: %v",
			err.Error())
	}

	err = validateAndDeleteIfInvalid(s, uuid)

	if err != nil {
		return nil, err
	}
	return &api.SdkCredentialCreateResponse{CredentialId: uuid}, nil
}

func (s *CredentialServer) googleCreate(
	ctx context.Context,
	req *api.SdkCredentialCreateRequest,
	google *api.SdkGoogleCredentialRequest,
) (*api.SdkCredentialCreateResponse, error) {

	if len(google.GetJsonKey()) == 0 {
		return nil, status.Error(codes.InvalidArgument, "Must supply JSON Key")
	}

	if len(google.GetProjectId()) == 0 {
		return nil, status.Error(codes.InvalidArgument, "Must supply Project ID")
	}

	params := make(map[string]string)

	params[api.OptCredType] = "google"
	params[api.OptCredName] = req.GetName()
	params[api.OptCredEncrKey] = req.GetEncryptionKey()
	params[api.OptCredBucket] = req.GetBucket()
	params[api.OptCredGoogleProjectID] = google.GetProjectId()
	params[api.OptCredGoogleJsonKey] = google.GetJsonKey()

	uuid, err := s.driver.CredsCreate(params)

	if err != nil {
		return nil, status.Errorf(
			codes.Internal,
			"failed to create Google credentials: %v",
			err.Error())
	}

	err = validateAndDeleteIfInvalid(s, uuid)

	if err != nil {
		return nil, err
	}

	return &api.SdkCredentialCreateResponse{CredentialId: uuid}, nil
}

// Validate validates a specified Credential.
func (s *CredentialServer) Validate(
	ctx context.Context,
	req *api.SdkCredentialValidateRequest,
) (*api.SdkCredentialValidateResponse, error) {

	if len(req.GetCredentialId()) == 0 {
		return nil, status.Error(codes.InvalidArgument, "Must provide credentials uuid")
	}

	validateReq := &api.SdkCredentialValidateRequest{CredentialId: req.GetCredentialId()}

	err := s.driver.CredsValidate(validateReq.GetCredentialId())

	if err != nil {
		return nil, status.Errorf(
			codes.Internal,
			"failed to validate credentials: %v",
			err.Error())
	}
	return &api.SdkCredentialValidateResponse{}, nil

}

// Delete deletes a specified credential
func (s *CredentialServer) Delete(
	ctx context.Context,
	req *api.SdkCredentialDeleteRequest,
) (*api.SdkCredentialDeleteResponse, error) {

	if len(req.GetCredentialId()) == 0 {
		return nil, status.Error(codes.InvalidArgument, "Must provide credentials uuid")
	}

	err := s.driver.CredsDelete(req.GetCredentialId())
	if err != nil {
		return nil, status.Errorf(
			codes.Internal,
			"failed to  delete credentials: %v",
			err.Error())
	}

	return &api.SdkCredentialDeleteResponse{}, nil
}

// Enumerate returns a list credentials ids
func (s *CredentialServer) Enumerate(
	ctx context.Context,
	req *api.SdkCredentialEnumerateRequest,
) (*api.SdkCredentialEnumerateResponse, error) {

	credList, err := s.driver.CredsEnumerate()
	if err != nil {
		return nil, status.Errorf(
			codes.Internal,
			"Unable to enumerate credentials AWS: %v",
			err.Error())
	}

	ids := make([]string, len(credList))
	i := 0
	for id := range credList {
		ids[i] = id
		i++
	}

	return &api.SdkCredentialEnumerateResponse{
		CredentialIds: ids,
	}, nil

}

// Inspect returns information about credential id
func (s *CredentialServer) Inspect(
	ctx context.Context,
	req *api.SdkCredentialInspectRequest,
) (*api.SdkCredentialInspectResponse, error) {

	if len(req.GetCredentialId()) == 0 {
		return nil, status.Error(codes.InvalidArgument, "Must provide a credential id")
	}

	credList, err := s.driver.CredsEnumerate()
	if err != nil {
		return nil, status.Errorf(
			codes.Internal,
			"Unable to enumerate credentials: %v",
			err.Error())
	}

	val, ok := credList[req.GetCredentialId()]
	if !ok {
		return nil, status.Errorf(codes.NotFound, "Credential id %s not found", req.GetCredentialId())
	}
	info, ok := val.(map[string]interface{})
	if !ok {
		return nil, status.Error(codes.Internal, "Unable to get credential id information")
	}

	credName, ok := info[api.OptCredName].(string)
	if !ok {
		// The code to support names may not be available
		credName = ""
	}
	bucket, ok := info[api.OptCredBucket].(string)
	if !ok {
		return nil, status.Error(codes.Internal, "Unable to get bucket name")
	}

	resp := &api.SdkCredentialInspectResponse{
		CredentialId: req.GetCredentialId(),
		Name:         credName,
		Bucket:       bucket,
	}

	switch info[api.OptCredType] {
	case "s3":
		accessKey, ok := info[api.OptCredAccessKey].(string)
		if !ok {
			return nil, status.Error(codes.Internal, "Unable to parse accessKey")
		}
		endpoint, ok := info[api.OptCredEndpoint].(string)
		if !ok {
			return nil, status.Error(codes.Internal, "Unable to parse endpoint")
		}
		region, ok := info[api.OptCredRegion].(string)
		if !ok {
			return nil, status.Error(codes.Internal, "Unable to parse region")
		}
		disableSsl, ok := info[api.OptCredDisableSSL].(string)
		if !ok {
			return nil, status.Error(codes.Internal, "Unable to parse disabling ssl was requested")
		}

		resp.CredentialType = &api.SdkCredentialInspectResponse_AwsCredential{
			AwsCredential: &api.SdkAwsCredentialResponse{
				AccessKey:  accessKey,
				Endpoint:   endpoint,
				Region:     region,
				DisableSsl: disableSsl == "true",
			},
		}
	case "azure":
		accountName, ok := info[api.OptCredAzureAccountName].(string)
		if !ok {
			return nil, status.Error(codes.Internal, "Unable to parse account name")
		}

		resp.CredentialType = &api.SdkCredentialInspectResponse_AzureCredential{
			AzureCredential: &api.SdkAzureCredentialResponse{
				AccountName: accountName,
			},
		}
	case "google":
		projectId, ok := info[api.OptCredGoogleProjectID].(string)
		if !ok {
			return nil, status.Error(codes.Internal, "Unable to parse project id")
		}
		resp.CredentialType = &api.SdkCredentialInspectResponse_GoogleCredential{
			GoogleCredential: &api.SdkGoogleCredentialResponse{
				ProjectId: projectId,
			},
		}
	default:
		return nil, status.Errorf(
			codes.Internal,
			"Received unknown credential type of %s",
			info[api.OptCredType])
	}

	return resp, nil
}

func validateAndDeleteIfInvalid(s *CredentialServer, uuid string) error {
	// Validate if the credentials provided were correct or not
	req := &api.SdkCredentialValidateRequest{CredentialId: uuid}

	validateErr := s.driver.CredsValidate(req.GetCredentialId())

	if validateErr != nil {
		deleteCred := &api.SdkCredentialDeleteRequest{CredentialId: uuid}
		err := s.driver.CredsDelete(deleteCred.GetCredentialId())

		if err != nil {
			return status.Errorf(
				codes.Internal,
				"failed to delete invalid Google credentials: %v",
				err.Error())
		}

		return status.Errorf(
			codes.PermissionDenied,
			"credentials could not be validated: %v",
			validateErr.Error())
	}

	return nil
}
<commit_msg>sdk:Allow bucket not to be set in credentials<commit_after>/*
Package sdk is the gRPC implementation of the SDK gRPC server
Copyright 2018 Portworx

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

	http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
package sdk

import (
	"context"
	"fmt"

	"github.com/libopenstorage/openstorage/api"
	"github.com/libopenstorage/openstorage/volume"
	"google.golang.org/grpc/codes"
	"google.golang.org/grpc/status"
)

// CredentialServer is an implementation of the gRPC OpenStorageCredential interface
type CredentialServer struct {
	driver volume.VolumeDriver
}

// Create method creates credentials
func (s *CredentialServer) Create(
	ctx context.Context,
	req *api.SdkCredentialCreateRequest,
) (*api.SdkCredentialCreateResponse, error) {

	if len(req.GetName()) == 0 {
		return nil, status.Error(codes.InvalidArgument, "Must supply a name")
	} else if aws := req.GetAwsCredential(); aws != nil {
		return s.awsCreate(ctx, req, aws)
	} else if azure := req.GetAzureCredential(); azure != nil {
		return s.azureCreate(ctx, req, azure)
	} else if google := req.GetGoogleCredential(); google != nil {
		return s.googleCreate(ctx, req, google)
	}
	return nil, status.Error(codes.InvalidArgument, "Unknown credential type")

}

func (s *CredentialServer) awsCreate(
	ctx context.Context,
	req *api.SdkCredentialCreateRequest,
	aws *api.SdkAwsCredentialRequest,
) (*api.SdkCredentialCreateResponse, error) {

	if len(aws.GetAccessKey()) == 0 {
		return nil, status.Error(codes.InvalidArgument, "Must supply Access Key")
	}

	if len(aws.GetSecretKey()) == 0 {
		return nil, status.Error(codes.InvalidArgument, "Must supply Secret Key")
	}

	if len(aws.GetRegion()) == 0 {
		return nil, status.Error(codes.InvalidArgument, "Must supply Region Key")
	}

	if len(aws.GetEndpoint()) == 0 {
		return nil, status.Error(codes.InvalidArgument, "Must supply Endpoint Key")
	}

	params := make(map[string]string)

	params[api.OptCredType] = "s3"
	params[api.OptCredName] = req.GetName()
	params[api.OptCredEncrKey] = req.GetEncryptionKey()
	params[api.OptCredBucket] = req.GetBucket()
	params[api.OptCredRegion] = aws.GetRegion()
	params[api.OptCredEndpoint] = aws.GetEndpoint()
	params[api.OptCredAccessKey] = aws.GetAccessKey()
	params[api.OptCredSecretKey] = aws.GetSecretKey()
	params[api.OptCredDisableSSL] = fmt.Sprintf("%v", aws.GetDisableSsl())

	uuid, err := s.driver.CredsCreate(params)

	if err != nil {
		return nil, status.Errorf(
			codes.Internal,
			"failed to create aws credentials: %v",
			err.Error())
	}

	err = validateAndDeleteIfInvalid(s, uuid)

	if err != nil {
		return nil, err
	}
	return &api.SdkCredentialCreateResponse{CredentialId: uuid}, nil
}

func (s *CredentialServer) azureCreate(
	ctx context.Context,
	req *api.SdkCredentialCreateRequest,
	azure *api.SdkAzureCredentialRequest,
) (*api.SdkCredentialCreateResponse, error) {

	if len(azure.GetAccountKey()) == 0 {
		return nil, status.Error(codes.InvalidArgument, "Must supply Account Key")
	}

	if len(azure.GetAccountName()) == 0 {
		return nil, status.Error(codes.InvalidArgument, "Must supply Account name")
	}

	params := make(map[string]string)

	params[api.OptCredType] = "azure"
	params[api.OptCredName] = req.GetName()
	params[api.OptCredEncrKey] = req.GetEncryptionKey()
	params[api.OptCredBucket] = req.GetBucket()
	params[api.OptCredAzureAccountKey] = azure.GetAccountKey()
	params[api.OptCredAzureAccountName] = azure.GetAccountName()

	uuid, err := s.driver.CredsCreate(params)

	if err != nil {
		return nil, status.Errorf(
			codes.Internal,
			"failed to create Azure credentials: %v",
			err.Error())
	}

	err = validateAndDeleteIfInvalid(s, uuid)

	if err != nil {
		return nil, err
	}
	return &api.SdkCredentialCreateResponse{CredentialId: uuid}, nil
}

func (s *CredentialServer) googleCreate(
	ctx context.Context,
	req *api.SdkCredentialCreateRequest,
	google *api.SdkGoogleCredentialRequest,
) (*api.SdkCredentialCreateResponse, error) {

	if len(google.GetJsonKey()) == 0 {
		return nil, status.Error(codes.InvalidArgument, "Must supply JSON Key")
	}

	if len(google.GetProjectId()) == 0 {
		return nil, status.Error(codes.InvalidArgument, "Must supply Project ID")
	}

	params := make(map[string]string)

	params[api.OptCredType] = "google"
	params[api.OptCredName] = req.GetName()
	params[api.OptCredEncrKey] = req.GetEncryptionKey()
	params[api.OptCredBucket] = req.GetBucket()
	params[api.OptCredGoogleProjectID] = google.GetProjectId()
	params[api.OptCredGoogleJsonKey] = google.GetJsonKey()

	uuid, err := s.driver.CredsCreate(params)

	if err != nil {
		return nil, status.Errorf(
			codes.Internal,
			"failed to create Google credentials: %v",
			err.Error())
	}

	err = validateAndDeleteIfInvalid(s, uuid)

	if err != nil {
		return nil, err
	}

	return &api.SdkCredentialCreateResponse{CredentialId: uuid}, nil
}

// Validate validates a specified Credential.
func (s *CredentialServer) Validate(
	ctx context.Context,
	req *api.SdkCredentialValidateRequest,
) (*api.SdkCredentialValidateResponse, error) {

	if len(req.GetCredentialId()) == 0 {
		return nil, status.Error(codes.InvalidArgument, "Must provide credentials uuid")
	}

	validateReq := &api.SdkCredentialValidateRequest{CredentialId: req.GetCredentialId()}

	err := s.driver.CredsValidate(validateReq.GetCredentialId())

	if err != nil {
		return nil, status.Errorf(
			codes.Internal,
			"failed to validate credentials: %v",
			err.Error())
	}
	return &api.SdkCredentialValidateResponse{}, nil

}

// Delete deletes a specified credential
func (s *CredentialServer) Delete(
	ctx context.Context,
	req *api.SdkCredentialDeleteRequest,
) (*api.SdkCredentialDeleteResponse, error) {

	if len(req.GetCredentialId()) == 0 {
		return nil, status.Error(codes.InvalidArgument, "Must provide credentials uuid")
	}

	err := s.driver.CredsDelete(req.GetCredentialId())
	if err != nil {
		return nil, status.Errorf(
			codes.Internal,
			"failed to  delete credentials: %v",
			err.Error())
	}

	return &api.SdkCredentialDeleteResponse{}, nil
}

// Enumerate returns a list credentials ids
func (s *CredentialServer) Enumerate(
	ctx context.Context,
	req *api.SdkCredentialEnumerateRequest,
) (*api.SdkCredentialEnumerateResponse, error) {

	credList, err := s.driver.CredsEnumerate()
	if err != nil {
		return nil, status.Errorf(
			codes.Internal,
			"Unable to enumerate credentials AWS: %v",
			err.Error())
	}

	ids := make([]string, len(credList))
	i := 0
	for id := range credList {
		ids[i] = id
		i++
	}

	return &api.SdkCredentialEnumerateResponse{
		CredentialIds: ids,
	}, nil

}

// Inspect returns information about credential id
func (s *CredentialServer) Inspect(
	ctx context.Context,
	req *api.SdkCredentialInspectRequest,
) (*api.SdkCredentialInspectResponse, error) {

	if len(req.GetCredentialId()) == 0 {
		return nil, status.Error(codes.InvalidArgument, "Must provide a credential id")
	}

	credList, err := s.driver.CredsEnumerate()
	if err != nil {
		return nil, status.Errorf(
			codes.Internal,
			"Unable to enumerate credentials: %v",
			err.Error())
	}

	val, ok := credList[req.GetCredentialId()]
	if !ok {
		return nil, status.Errorf(codes.NotFound, "Credential id %s not found", req.GetCredentialId())
	}
	info, ok := val.(map[string]interface{})
	if !ok {
		return nil, status.Error(codes.Internal, "Unable to get credential id information")
	}

	credName, ok := info[api.OptCredName].(string)
	if !ok {
		// The code to support names may not be available
		credName = ""
	}
	bucket, ok := info[api.OptCredBucket].(string)
	if !ok {
		// The code to support bucket may not be available
		bucket = ""
	}

	resp := &api.SdkCredentialInspectResponse{
		CredentialId: req.GetCredentialId(),
		Name:         credName,
		Bucket:       bucket,
	}

	switch info[api.OptCredType] {
	case "s3":
		accessKey, ok := info[api.OptCredAccessKey].(string)
		if !ok {
			return nil, status.Error(codes.Internal, "Unable to parse accessKey")
		}
		endpoint, ok := info[api.OptCredEndpoint].(string)
		if !ok {
			return nil, status.Error(codes.Internal, "Unable to parse endpoint")
		}
		region, ok := info[api.OptCredRegion].(string)
		if !ok {
			return nil, status.Error(codes.Internal, "Unable to parse region")
		}
		disableSsl, ok := info[api.OptCredDisableSSL].(string)
		if !ok {
			return nil, status.Error(codes.Internal, "Unable to parse disabling ssl was requested")
		}

		resp.CredentialType = &api.SdkCredentialInspectResponse_AwsCredential{
			AwsCredential: &api.SdkAwsCredentialResponse{
				AccessKey:  accessKey,
				Endpoint:   endpoint,
				Region:     region,
				DisableSsl: disableSsl == "true",
			},
		}
	case "azure":
		accountName, ok := info[api.OptCredAzureAccountName].(string)
		if !ok {
			return nil, status.Error(codes.Internal, "Unable to parse account name")
		}

		resp.CredentialType = &api.SdkCredentialInspectResponse_AzureCredential{
			AzureCredential: &api.SdkAzureCredentialResponse{
				AccountName: accountName,
			},
		}
	case "google":
		projectId, ok := info[api.OptCredGoogleProjectID].(string)
		if !ok {
			return nil, status.Error(codes.Internal, "Unable to parse project id")
		}
		resp.CredentialType = &api.SdkCredentialInspectResponse_GoogleCredential{
			GoogleCredential: &api.SdkGoogleCredentialResponse{
				ProjectId: projectId,
			},
		}
	default:
		return nil, status.Errorf(
			codes.Internal,
			"Received unknown credential type of %s",
			info[api.OptCredType])
	}

	return resp, nil
}

func validateAndDeleteIfInvalid(s *CredentialServer, uuid string) error {
	// Validate if the credentials provided were correct or not
	req := &api.SdkCredentialValidateRequest{CredentialId: uuid}

	validateErr := s.driver.CredsValidate(req.GetCredentialId())

	if validateErr != nil {
		deleteCred := &api.SdkCredentialDeleteRequest{CredentialId: uuid}
		err := s.driver.CredsDelete(deleteCred.GetCredentialId())

		if err != nil {
			return status.Errorf(
				codes.Internal,
				"failed to delete invalid Google credentials: %v",
				err.Error())
		}

		return status.Errorf(
			codes.PermissionDenied,
			"credentials could not be validated: %v",
			validateErr.Error())
	}

	return nil
}
<|endoftext|>