<commit_before>package protocol

import (
	"errors"
	"reflect"
	"regexp"

	"golang-refactoring.org/go-doctor/doctor"
)

type Setdir struct {
	Mode string `json:"mode" chk:"local|web"`
}

func (s *Setdir) Run(state *State, input map[string]interface{}) (Reply, error) {

	if valid, err := s.Validate(state, input); valid {
		// assuming everything is good?
		mode := input["mode"]
		state.Mode = mode.(string)

		// local mode? get directory and local filesystem
		if mode == "local" {
			state.Dir = input["directory"].(string)
			state.Filesystem = doctor.NewLocalFileSystem()
		}

		// web mode? get that virtual filesystem
		if mode == "web" {
			state.Filesystem = doctor.NewVirtualFileSystem()
		}

		state.State = 2
		return Reply{map[string]interface{}{"reply": "OK"}}, nil
	} else {
		return Reply{map[string]interface{}{"reply": "Error", "message": err.Error()}}, err
	}

}

func (s *Setdir) Validate(state *State, input map[string]interface{}) (bool, error) {
	if state.State < 1 {
		return false, errors.New("State must be non-zero for \"setdir\" command")
	}

	// mode key?
	if mode, found := input["mode"]; !found {
		err := errors.New("\"mode\" key is required")
		return false, err
	} else {
		// validate the mode value
		field, _ := reflect.TypeOf(s).Elem().FieldByName("Mode")
		modeValidator := regexp.MustCompile(field.Tag.Get("chk"))
		if valid := modeValidator.MatchString(mode.(string)); !valid {
			return false, errors.New("\"mode\" key must be \"web|local\"")
		}
		// check for directory key if mode == local
		if mode == "local" {
			if _, found := input["directory"]; !found {
				return false, errors.New("\"directory\" key required if \"mode\" is local")
			}
			// validate directory
			fs := doctor.NewLocalFileSystem()
			_, err := fs.ReadDir(input["directory"].(string))
			if err != nil {
				return false, err
			}
		}
	}
	return true, nil
}
<commit_msg>Changed setdir since VirtualFileSystem removed<commit_after>package protocol

import (
	"errors"
	"reflect"
	"regexp"

	"golang-refactoring.org/go-doctor/doctor"
)

type Setdir struct {
	Mode string `json:"mode" chk:"local|web"`
}

func (s *Setdir) Run(state *State, input map[string]interface{}) (Reply, error) {

	if valid, err := s.Validate(state, input); valid {
		// assuming everything is good?
		mode := input["mode"]
		state.Mode = mode.(string)

		// local mode? get directory and local filesystem
		if mode == "local" {
			state.Dir = input["directory"].(string)
			state.Filesystem = doctor.NewLocalFileSystem()
		}

		// web mode? get that virtual filesystem
		if mode == "web" {
			//state.Filesystem = doctor.NewVirtualFileSystem()
			return Reply{map[string]interface{}{"reply": "Error", "message": "Web mode not supported"}}, err
		}

		state.State = 2
		return Reply{map[string]interface{}{"reply": "OK"}}, nil
	} else {
		return Reply{map[string]interface{}{"reply": "Error", "message": err.Error()}}, err
	}

}

func (s *Setdir) Validate(state *State, input map[string]interface{}) (bool, error) {
	if state.State < 1 {
		return false, errors.New("State must be non-zero for \"setdir\" command")
	}

	// mode key?
	if mode, found := input["mode"]; !found {
		err := errors.New("\"mode\" key is required")
		return false, err
	} else {
		// validate the mode value
		field, _ := reflect.TypeOf(s).Elem().FieldByName("Mode")
		modeValidator := regexp.MustCompile(field.Tag.Get("chk"))
		if valid := modeValidator.MatchString(mode.(string)); !valid {
			return false, errors.New("\"mode\" key must be \"web|local\"")
		}
		// check for directory key if mode == local
		if mode == "local" {
			if _, found := input["directory"]; !found {
				return false, errors.New("\"directory\" key required if \"mode\" is local")
			}
			// validate directory
			fs := doctor.NewLocalFileSystem()
			_, err := fs.ReadDir(input["directory"].(string))
			if err != nil {
				return false, err
			}
		}
	}
	return true, nil
}
<|endoftext|>


<commit_before>// Copyright 2015 Google Inc. All Rights Reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//      http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

// +build !windows,!openbsd

// Package fuse provides a connection source wherein the user does not need to
// specify which instance they are connecting to before they start the
// executable. Instead, simply attempting to access a file in the provided
// directory will transparently create a proxied connection to an instance
// which has that name.
//
// Specifically, given that NewConnSrc was called with the mounting directory
// as /cloudsql:
//
// 1) Execute `mysql -S /cloudsql/speckle:instance`
// 2) The 'mysql' executable looks up the file "speckle:instance" inside "/cloudsql"
// 3) This lookup is intercepted by the code in this package. A local unix socket
//    located in a temporary directory is opened for listening and the lookup for
//    "speckle:instance" returns to mysql saying that it is a symbolic link
//    pointing to this new local socket.
// 4) mysql dials the local unix socket, creating a new connection to the
//    specified instance.
package fuse

import (
	"bytes"
	"errors"
	"fmt"
	"io"
	"net"
	"os"
	"path/filepath"
	"strings"
	"sync"
	"time"

	"bazil.org/fuse"
	"bazil.org/fuse/fs"
	"github.com/GoogleCloudPlatform/cloudsql-proxy/logging"
	"github.com/GoogleCloudPlatform/cloudsql-proxy/proxy/proxy"
	"golang.org/x/net/context"
)

// Supported returns true if the current system supports FUSE.
// TODO: for OSX, check to see if OSX FUSE is installed.
func Supported() bool {
	return true
}

// NewConnSrc returns a source of new connections based on Lookups in the
// provided mount directory. If there isn't a directory located at tmpdir one
// is created. The second return parameter can be used to shutdown and release
// any resources. As a result of this shutdown, or during any other fatal
// error, the returned chan will be closed.
//
// The connset parameter is optional.
func NewConnSrc(mountdir, tmpdir string, client *proxy.Client, connset *proxy.ConnSet) (<-chan proxy.Conn, io.Closer, error) {
	if err := os.MkdirAll(tmpdir, 0777); err != nil {
		return nil, nil, err
	}

	if err := fuse.Unmount(mountdir); err != nil {
		// The error is too verbose to be useful to print out
	}
	logging.Verbosef("Mounting %v...", mountdir)
	c, err := fuse.Mount(mountdir, fuse.AllowOther())
	if err != nil {
		return nil, nil, fmt.Errorf("cannot mount %q: %v", mountdir, err)
	}
	logging.Infof("Mounted %v", mountdir)

	if connset == nil {
		// Make a dummy one.
		connset = proxy.NewConnSet()
	}
	conns := make(chan proxy.Conn, 1)
	root := &fsRoot{
		tmpDir:  tmpdir,
		linkDir: mountdir,
		dst:     conns,
		links:   make(map[string]symlink),
		closers: []io.Closer{c},
		connset: connset,
		client:  client,
	}

	server := fs.New(c, &fs.Config{
		Debug: func(msg interface{}) {
			if false {
				logging.Verbosef("%s", msg)
			}
		},
	})

	go func() {
		if err := server.Serve(root); err != nil {
			logging.Errorf("serve %q exited due to error: %v", mountdir, err)
		}
		// The server exited but we don't know whether this is because of a
		// graceful reason (via root.Close) or via an external force unmounting.
		// Closing the root will ensure the 'dst' chan is closed correctly to
		// signify that no new connections are possible.
		if err := root.Close(); err != nil {
			logging.Errorf("root.Close() error: %v", err)
		}
		logging.Infof("FUSE exited")
	}()

	return conns, root, nil
}

// symlink implements a symbolic link, returning the underlying string when
// Readlink is called.
type symlink string

var _ interface {
	fs.Node
	fs.NodeReadlinker
} = symlink("")

func (s symlink) Readlink(context.Context, *fuse.ReadlinkRequest) (string, error) {
	return string(s), nil
}

// Attr helps implement fs.Node.
func (symlink) Attr(ctx context.Context, a *fuse.Attr) error {
	*a = fuse.Attr{
		Mode: 0777 | os.ModeSymlink,
	}
	return nil
}

type fsRoot struct {
	tmpDir, linkDir string

	client  *proxy.Client
	connset *proxy.ConnSet

	// sockLock protects fields in this struct related to sockets; specifically
	// 'links' and 'closers'.
	sockLock sync.Mutex
	links    map[string]symlink
	// closers holds a slice of things to close when fsRoot.Close is called.
	closers []io.Closer

	sync.RWMutex
	dst chan<- proxy.Conn
}

// Ensure that fsRoot implements the following interfaces.
var _ interface {
	fs.FS
	fs.Node
	fs.NodeRequestLookuper
	fs.HandleReadDirAller
} = &fsRoot{}

func (r *fsRoot) newConn(instance string, c net.Conn) {
	r.RLock()
	// dst will be nil if Close has been called already.
	if ch := r.dst; ch != nil {
		ch <- proxy.Conn{instance, c}
	} else {
		logging.Errorf("Ignored new conn request to %q: system has been closed", instance)
	}
	r.RUnlock()
}

func (r *fsRoot) Forget() {
	logging.Verbosef("Forget called on %q", r.linkDir)
}

func (r *fsRoot) Destroy() {
	logging.Verbosef("Destroy called on %q", r.linkDir)
}

func (r *fsRoot) Close() error {
	r.Lock()
	if r.dst != nil {
		// Since newConn only sends on dst while holding a reader lock, holding the
		// writer lock is sufficient to ensure there are no pending sends on the
		// channel when it is closed.
		close(r.dst)
		// Setting it to nil prevents further sends.
		r.dst = nil
	}
	r.Unlock()

	logging.Infof("unmount %q", r.linkDir)
	if err := fuse.Unmount(r.linkDir); err != nil {
		return err
	}

	var errs bytes.Buffer
	r.sockLock.Lock()
	for _, c := range r.closers {
		if err := c.Close(); err != nil {
			fmt.Fprintln(&errs, err)
		}
	}
	r.sockLock.Unlock()

	if errs.Len() == 0 {
		return nil
	}
	logging.Errorf("Close %q: %v", r.linkDir, errs.String())
	return errors.New(errs.String())
}

// Root returns the fsRoot itself as the root directory.
func (r *fsRoot) Root() (fs.Node, error) {
	return r, nil
}

// Attr helps implement fs.Node
func (r *fsRoot) Attr(ctx context.Context, a *fuse.Attr) error {
	*a = fuse.Attr{
		Mode: 0555 | os.ModeDir,
	}
	return nil
}

// Lookup helps implement fs.NodeRequestLookuper. If the requested file isn't
// the README, it returns a node which is a symbolic link to a socket which
// provides connectivity to a remote instance.  The instance which is connected
// to is determined by req.Name.
func (r *fsRoot) Lookup(_ context.Context, req *fuse.LookupRequest, resp *fuse.LookupResponse) (fs.Node, error) {
	if req.Name == "README" {
		return readme{}, nil
	}
	instance := req.Name
	r.sockLock.Lock()
	defer r.sockLock.Unlock()

	if ret, ok := r.links[instance]; ok {
		return ret, nil
	}

	path := filepath.Join(r.tmpDir, instance)
	os.RemoveAll(path) // Best effort; the following will fail if this does.
	linkpath := path

	// MySQL expects a Unix domain socket at the symlinked path whereas Postgres expects
	// a socket named ".s.PGSQL.5432" in the directory given by the database path.
	// Look up instance database version to determine the correct socket path.
	// Client is nil in unit tests.
	if r.client != nil {
		version, err := r.client.InstanceVersion(instance)
		if err != nil {
			return nil, fuse.ENOENT
		}
		if strings.HasPrefix(strings.ToLower(version), "postgres") {
			if err := os.MkdirAll(path, 0755); err != nil {
				return nil, fuse.EIO
			}
			path = filepath.Join(linkpath, ".s.PGSQL.5432")
		}
	}

	sock, err := net.Listen("unix", path)
	if err != nil {
		logging.Errorf("couldn't listen at %q: %v", path, err)
		return nil, fuse.EEXIST
	}
	if err := os.Chmod(path, 0777|os.ModeSocket); err != nil {
		logging.Errorf("couldn't update permissions for socket file %q: %v; other users may be unable to connect", path, err)
	}

	go r.listenerLifecycle(sock, instance, path)

	ret := symlink(linkpath)
	r.links[instance] = ret
	// TODO(chowski): memory leak when listeners exit on their own via removeListener.
	r.closers = append(r.closers, sock)

	return ret, nil
}

// removeListener marks that a Listener for an instance has exited and is no
// longer serving new connections.
func (r *fsRoot) removeListener(instance, path string) {
	r.sockLock.Lock()
	v, ok := r.links[instance]
	if ok && string(v) == path {
		delete(r.links, instance)
	} else {
		logging.Errorf("Removing a listener for %q at %q which was already replaced", instance, path)
	}
	r.sockLock.Unlock()
}

// listenerLifecycle calls l.Accept in a loop, and for each new connection
// r.newConn is called. After the Listener returns an error it is removed.
func (r *fsRoot) listenerLifecycle(l net.Listener, instance, path string) {
	for {
		start := time.Now()
		c, err := l.Accept()
		if err != nil {
			logging.Errorf("error in Accept for %q: %v", instance, err)
			if nerr, ok := err.(net.Error); ok && nerr.Temporary() {
				d := 10*time.Millisecond - time.Since(start)
				if d > 0 {
					time.Sleep(d)
				}
				continue
			}
			break
		}
		r.newConn(instance, c)
	}
	r.removeListener(instance, path)
	l.Close()
	if err := os.Remove(path); err != nil {
		logging.Errorf("couldn't remove %q: %v", path, err)
	}
}

// ReadDirAll returns a list of files contained in the root directory.
// It contains a README file which explains how to use the directory.
// In addition, there will be a file for each instance to which the
// proxy is actively connected.
func (r *fsRoot) ReadDirAll(context.Context) ([]fuse.Dirent, error) {
	ret := []fuse.Dirent{
		{Name: "README", Type: fuse.DT_File},
	}

	for _, v := range r.connset.IDs() {
		ret = append(ret, fuse.Dirent{Name: v, Type: fuse.DT_Socket})
	}

	return ret, nil
}

// readme implements the REAME file found in the mounted folder. It is a
// static read-only text file.
type readme struct{}

var _ interface {
	fs.Node
	fs.HandleReadAller
} = readme{}

const readmeText = `
When programs attempt to open files in this directory, a remote connection to
the Cloud SQL instance of the same name will be established.

That is, running :

	mysql -u root -S "/path/to/this/directory/project:instance-2"

will open a new connection to project:instance-2, given you have the correct
permissions.

Listing the contents of this directory will show all instances with active
connections.
`

// Attr helps implement fs.Node.
func (readme) Attr(ctx context.Context, a *fuse.Attr) error {
	*a = fuse.Attr{
		Mode: 0444,
		Size: uint64(len(readmeText)),
	}
	return nil
}

// ReadAll helps implement fs.HandleReadAller.
func (readme) ReadAll(context.Context) ([]byte, error) {
	return []byte(readmeText), nil
}
<commit_msg>fix: only allow fuse mode to unmount if an error occurs first (#537)<commit_after>// Copyright 2015 Google Inc. All Rights Reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//      http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

// +build !windows,!openbsd

// Package fuse provides a connection source wherein the user does not need to
// specify which instance they are connecting to before they start the
// executable. Instead, simply attempting to access a file in the provided
// directory will transparently create a proxied connection to an instance
// which has that name.
//
// Specifically, given that NewConnSrc was called with the mounting directory
// as /cloudsql:
//
// 1) Execute `mysql -S /cloudsql/speckle:instance`
// 2) The 'mysql' executable looks up the file "speckle:instance" inside "/cloudsql"
// 3) This lookup is intercepted by the code in this package. A local unix socket
//    located in a temporary directory is opened for listening and the lookup for
//    "speckle:instance" returns to mysql saying that it is a symbolic link
//    pointing to this new local socket.
// 4) mysql dials the local unix socket, creating a new connection to the
//    specified instance.
package fuse

import (
	"bytes"
	"errors"
	"fmt"
	"io"
	"net"
	"os"
	"path/filepath"
	"strings"
	"sync"
	"time"

	"bazil.org/fuse"
	"bazil.org/fuse/fs"
	"github.com/GoogleCloudPlatform/cloudsql-proxy/logging"
	"github.com/GoogleCloudPlatform/cloudsql-proxy/proxy/proxy"
	"golang.org/x/net/context"
)

// Supported returns true if the current system supports FUSE.
// TODO: for OSX, check to see if OSX FUSE is installed.
func Supported() bool {
	return true
}

// NewConnSrc returns a source of new connections based on Lookups in the
// provided mount directory. If there isn't a directory located at tmpdir one
// is created. The second return parameter can be used to shutdown and release
// any resources. As a result of this shutdown, or during any other fatal
// error, the returned chan will be closed.
//
// The connset parameter is optional.
func NewConnSrc(mountdir, tmpdir string, client *proxy.Client, connset *proxy.ConnSet) (<-chan proxy.Conn, io.Closer, error) {
	if err := os.MkdirAll(tmpdir, 0777); err != nil {
		return nil, nil, err
	}

	logging.Verbosef("Mounting %v...", mountdir)
	c, err := fuse.Mount(mountdir, fuse.AllowOther())
	if err != nil {
		// a common cause of failed mounts is that a previous instance did not shutdown cleanly, leaving an abandoned mount
		logging.Errorf("WARNING: Mount failed - attempting to unmount dir to resolve...", mountdir)
		if err = fuse.Unmount(mountdir); err != nil {
			logging.Errorf("Unmount failed: %v", err)
		}
		if c, err = fuse.Mount(mountdir, fuse.AllowOther()); err != nil {
			return nil, nil, fmt.Errorf("cannot mount %q: %v", mountdir, err)
		}
	}
	logging.Infof("Mounted %v", mountdir)

	if connset == nil {
		// Make a dummy one.
		connset = proxy.NewConnSet()
	}
	conns := make(chan proxy.Conn, 1)
	root := &fsRoot{
		tmpDir:  tmpdir,
		linkDir: mountdir,
		dst:     conns,
		links:   make(map[string]symlink),
		closers: []io.Closer{c},
		connset: connset,
		client:  client,
	}

	server := fs.New(c, &fs.Config{
		Debug: func(msg interface{}) {
			if false {
				logging.Verbosef("%s", msg)
			}
		},
	})

	go func() {
		if err := server.Serve(root); err != nil {
			logging.Errorf("serve %q exited due to error: %v", mountdir, err)
		}
		// The server exited but we don't know whether this is because of a
		// graceful reason (via root.Close) or via an external force unmounting.
		// Closing the root will ensure the 'dst' chan is closed correctly to
		// signify that no new connections are possible.
		if err := root.Close(); err != nil {
			logging.Errorf("root.Close() error: %v", err)
		}
		logging.Infof("FUSE exited")
	}()

	return conns, root, nil
}

// symlink implements a symbolic link, returning the underlying string when
// Readlink is called.
type symlink string

var _ interface {
	fs.Node
	fs.NodeReadlinker
} = symlink("")

func (s symlink) Readlink(context.Context, *fuse.ReadlinkRequest) (string, error) {
	return string(s), nil
}

// Attr helps implement fs.Node.
func (symlink) Attr(ctx context.Context, a *fuse.Attr) error {
	*a = fuse.Attr{
		Mode: 0777 | os.ModeSymlink,
	}
	return nil
}

type fsRoot struct {
	tmpDir, linkDir string

	client  *proxy.Client
	connset *proxy.ConnSet

	// sockLock protects fields in this struct related to sockets; specifically
	// 'links' and 'closers'.
	sockLock sync.Mutex
	links    map[string]symlink
	// closers holds a slice of things to close when fsRoot.Close is called.
	closers []io.Closer

	sync.RWMutex
	dst chan<- proxy.Conn
}

// Ensure that fsRoot implements the following interfaces.
var _ interface {
	fs.FS
	fs.Node
	fs.NodeRequestLookuper
	fs.HandleReadDirAller
} = &fsRoot{}

func (r *fsRoot) newConn(instance string, c net.Conn) {
	r.RLock()
	// dst will be nil if Close has been called already.
	if ch := r.dst; ch != nil {
		ch <- proxy.Conn{instance, c}
	} else {
		logging.Errorf("Ignored new conn request to %q: system has been closed", instance)
	}
	r.RUnlock()
}

func (r *fsRoot) Forget() {
	logging.Verbosef("Forget called on %q", r.linkDir)
}

func (r *fsRoot) Destroy() {
	logging.Verbosef("Destroy called on %q", r.linkDir)
}

func (r *fsRoot) Close() error {
	r.Lock()
	if r.dst != nil {
		// Since newConn only sends on dst while holding a reader lock, holding the
		// writer lock is sufficient to ensure there are no pending sends on the
		// channel when it is closed.
		close(r.dst)
		// Setting it to nil prevents further sends.
		r.dst = nil
	}
	r.Unlock()

	logging.Infof("unmount %q", r.linkDir)
	if err := fuse.Unmount(r.linkDir); err != nil {
		return err
	}

	var errs bytes.Buffer
	r.sockLock.Lock()
	for _, c := range r.closers {
		if err := c.Close(); err != nil {
			fmt.Fprintln(&errs, err)
		}
	}
	r.sockLock.Unlock()

	if errs.Len() == 0 {
		return nil
	}
	logging.Errorf("Close %q: %v", r.linkDir, errs.String())
	return errors.New(errs.String())
}

// Root returns the fsRoot itself as the root directory.
func (r *fsRoot) Root() (fs.Node, error) {
	return r, nil
}

// Attr helps implement fs.Node
func (r *fsRoot) Attr(ctx context.Context, a *fuse.Attr) error {
	*a = fuse.Attr{
		Mode: 0555 | os.ModeDir,
	}
	return nil
}

// Lookup helps implement fs.NodeRequestLookuper. If the requested file isn't
// the README, it returns a node which is a symbolic link to a socket which
// provides connectivity to a remote instance.  The instance which is connected
// to is determined by req.Name.
func (r *fsRoot) Lookup(_ context.Context, req *fuse.LookupRequest, resp *fuse.LookupResponse) (fs.Node, error) {
	if req.Name == "README" {
		return readme{}, nil
	}
	instance := req.Name
	r.sockLock.Lock()
	defer r.sockLock.Unlock()

	if ret, ok := r.links[instance]; ok {
		return ret, nil
	}

	path := filepath.Join(r.tmpDir, instance)
	os.RemoveAll(path) // Best effort; the following will fail if this does.
	linkpath := path

	// MySQL expects a Unix domain socket at the symlinked path whereas Postgres expects
	// a socket named ".s.PGSQL.5432" in the directory given by the database path.
	// Look up instance database version to determine the correct socket path.
	// Client is nil in unit tests.
	if r.client != nil {
		version, err := r.client.InstanceVersion(instance)
		if err != nil {
			return nil, fuse.ENOENT
		}
		if strings.HasPrefix(strings.ToLower(version), "postgres") {
			if err := os.MkdirAll(path, 0755); err != nil {
				return nil, fuse.EIO
			}
			path = filepath.Join(linkpath, ".s.PGSQL.5432")
		}
	}

	sock, err := net.Listen("unix", path)
	if err != nil {
		logging.Errorf("couldn't listen at %q: %v", path, err)
		return nil, fuse.EEXIST
	}
	if err := os.Chmod(path, 0777|os.ModeSocket); err != nil {
		logging.Errorf("couldn't update permissions for socket file %q: %v; other users may be unable to connect", path, err)
	}

	go r.listenerLifecycle(sock, instance, path)

	ret := symlink(linkpath)
	r.links[instance] = ret
	// TODO(chowski): memory leak when listeners exit on their own via removeListener.
	r.closers = append(r.closers, sock)

	return ret, nil
}

// removeListener marks that a Listener for an instance has exited and is no
// longer serving new connections.
func (r *fsRoot) removeListener(instance, path string) {
	r.sockLock.Lock()
	v, ok := r.links[instance]
	if ok && string(v) == path {
		delete(r.links, instance)
	} else {
		logging.Errorf("Removing a listener for %q at %q which was already replaced", instance, path)
	}
	r.sockLock.Unlock()
}

// listenerLifecycle calls l.Accept in a loop, and for each new connection
// r.newConn is called. After the Listener returns an error it is removed.
func (r *fsRoot) listenerLifecycle(l net.Listener, instance, path string) {
	for {
		start := time.Now()
		c, err := l.Accept()
		if err != nil {
			logging.Errorf("error in Accept for %q: %v", instance, err)
			if nerr, ok := err.(net.Error); ok && nerr.Temporary() {
				d := 10*time.Millisecond - time.Since(start)
				if d > 0 {
					time.Sleep(d)
				}
				continue
			}
			break
		}
		r.newConn(instance, c)
	}
	r.removeListener(instance, path)
	l.Close()
	if err := os.Remove(path); err != nil {
		logging.Errorf("couldn't remove %q: %v", path, err)
	}
}

// ReadDirAll returns a list of files contained in the root directory.
// It contains a README file which explains how to use the directory.
// In addition, there will be a file for each instance to which the
// proxy is actively connected.
func (r *fsRoot) ReadDirAll(context.Context) ([]fuse.Dirent, error) {
	ret := []fuse.Dirent{
		{Name: "README", Type: fuse.DT_File},
	}

	for _, v := range r.connset.IDs() {
		ret = append(ret, fuse.Dirent{Name: v, Type: fuse.DT_Socket})
	}

	return ret, nil
}

// readme implements the REAME file found in the mounted folder. It is a
// static read-only text file.
type readme struct{}

var _ interface {
	fs.Node
	fs.HandleReadAller
} = readme{}

const readmeText = `
When programs attempt to open files in this directory, a remote connection to
the Cloud SQL instance of the same name will be established.

That is, running :

	mysql -u root -S "/path/to/this/directory/project:instance-2"

will open a new connection to project:instance-2, given you have the correct
permissions.

Listing the contents of this directory will show all instances with active
connections.
`

// Attr helps implement fs.Node.
func (readme) Attr(ctx context.Context, a *fuse.Attr) error {
	*a = fuse.Attr{
		Mode: 0444,
		Size: uint64(len(readmeText)),
	}
	return nil
}

// ReadAll helps implement fs.HandleReadAller.
func (readme) ReadAll(context.Context) ([]byte, error) {
	return []byte(readmeText), nil
}
<|endoftext|>


<commit_before>package main

import (
	"math/rand"
	"os"
	"os/signal"
	"runtime"
	"syscall"
	"time"

	"github.com/btcsuite/btcd/wire"
	"github.com/op/go-logging"

	"github.com/CIRCL/pbtc/logger"
	"github.com/CIRCL/pbtc/manager"
	"github.com/CIRCL/pbtc/recorder"
	"github.com/CIRCL/pbtc/repository"
)

func main() {
	// catch signals
	sigc := make(chan os.Signal, 1)
	signal.Notify(sigc, os.Interrupt)

	// use all cpu cores
	runtime.GOMAXPROCS(runtime.NumCPU())

	// seed the random generator
	rand.Seed(time.Now().UnixNano())

	// logging
	log, err := logger.New(
		logger.EnableConsole(),
		logger.SetConsoleLevel(logging.INFO),
		logger.EnableFile(),
		logger.SetFileLevel(logging.DEBUG),
	)
	if err != nil {
		os.Exit(1)
	}

	log.Inf("[PBTC] Starting modules")

	// repository
	repo, err := repository.New(
		repository.SetLogger(log),
		repository.SetSeeds("seed.bitcoin.sipa.be"),
		repository.SetDefaultPort(8333),
		repository.DisableRestore(),
	)
	if err != nil {
		log.Critical("Unable to create repository (%v)", err)
		os.Exit(1)
	}

	// recorder
	rec, err := recorder.New(
		recorder.SetLogger(log),
		recorder.SetTypes(wire.CmdTx, wire.CmdVersion, wire.CmdInv,
			wire.CmdAddr),
		recorder.SetFileSize(0),
		recorder.SetFileAge(time.Minute*5),
		recorder.EnableReset(),
	)
	if err != nil {
		log.Critical("Unable to initialize recorder (%v)", err)
		os.Exit(1)
	}

	// manager
	mgr, err := manager.New(
		manager.SetLogger(log),
		manager.SetRepository(repo),
		manager.SetRecorder(rec),
		manager.SetNetwork(wire.MainNet),
		manager.SetVersion(wire.RejectVersion),
		manager.SetConnectionRate(time.Second/25),
		manager.SetInformationRate(time.Second*2),
		manager.SetPeerLimit(1000),
	)
	if err != nil {
		log.Critical("Unable to create manager (%v)", err)
		os.Exit(1)
	}

	log.Info("[PBTC] All modules initialization complete")

	// wait for signals in blocking loop
SigLoop:
	for sig := range sigc {
		log.Notice("Signal caught (%v)", sig.String())

		switch sig {
		case syscall.SIGINT:
			break SigLoop
		}
	}

	log.Info("[PBTC] Stopping modules")

	mgr.Stop()
	repo.Stop()

	log.Info("[PBTC] All modules shutdown complete")

	os.Exit(0)
}
<commit_msg>Fixed typo<commit_after>package main

import (
	"math/rand"
	"os"
	"os/signal"
	"runtime"
	"syscall"
	"time"

	"github.com/btcsuite/btcd/wire"
	"github.com/op/go-logging"

	"github.com/CIRCL/pbtc/logger"
	"github.com/CIRCL/pbtc/manager"
	"github.com/CIRCL/pbtc/recorder"
	"github.com/CIRCL/pbtc/repository"
)

func main() {
	// catch signals
	sigc := make(chan os.Signal, 1)
	signal.Notify(sigc, os.Interrupt)

	// use all cpu cores
	runtime.GOMAXPROCS(runtime.NumCPU())

	// seed the random generator
	rand.Seed(time.Now().UnixNano())

	// logging
	log, err := logger.New(
		logger.EnableConsole(),
		logger.SetConsoleLevel(logging.INFO),
		logger.EnableFile(),
		logger.SetFileLevel(logging.DEBUG),
	)
	if err != nil {
		os.Exit(1)
	}

	log.Info("[PBTC] Starting modules")

	// repository
	repo, err := repository.New(
		repository.SetLogger(log),
		repository.SetSeeds("seed.bitcoin.sipa.be"),
		repository.SetDefaultPort(8333),
		repository.DisableRestore(),
	)
	if err != nil {
		log.Critical("Unable to create repository (%v)", err)
		os.Exit(1)
	}

	// recorder
	rec, err := recorder.New(
		recorder.SetLogger(log),
		recorder.SetTypes(wire.CmdTx, wire.CmdVersion, wire.CmdInv,
			wire.CmdAddr),
		recorder.SetFileSize(0),
		recorder.SetFileAge(time.Minute*5),
		recorder.EnableReset(),
	)
	if err != nil {
		log.Critical("Unable to initialize recorder (%v)", err)
		os.Exit(1)
	}

	// manager
	mgr, err := manager.New(
		manager.SetLogger(log),
		manager.SetRepository(repo),
		manager.SetRecorder(rec),
		manager.SetNetwork(wire.MainNet),
		manager.SetVersion(wire.RejectVersion),
		manager.SetConnectionRate(time.Second/25),
		manager.SetInformationRate(time.Second*2),
		manager.SetPeerLimit(1000),
	)
	if err != nil {
		log.Critical("Unable to create manager (%v)", err)
		os.Exit(1)
	}

	log.Info("[PBTC] All modules initialization complete")

	// wait for signals in blocking loop
SigLoop:
	for sig := range sigc {
		log.Notice("Signal caught (%v)", sig.String())

		switch sig {
		case syscall.SIGINT:
			break SigLoop
		}
	}

	log.Info("[PBTC] Stopping modules")

	mgr.Stop()
	repo.Stop()

	log.Info("[PBTC] All modules shutdown complete")

	os.Exit(0)
}
<|endoftext|>


<commit_before>package main_test

import (
	"fmt"
	"time"

	"github.com/tecnickcom/statsd"

	"github.com/cloudfoundry-incubator/cf-test-helpers/helpers"

	. "github.com/onsi/ginkgo"
	. "github.com/onsi/gomega"

	"github.com/onsi/ginkgo"
	"github.com/onsi/gomega"
	"os"
	"strconv"
	"testing"
)

var (
	context        helpers.SuiteContext
	config         helpers.Config
	defaultTimeout time.Duration = 30 * time.Second
	cfPushTimeout  time.Duration = 5 * time.Minute
	statsdClient   *statsd.Client
	metricsPrefix  string
	loopCount      int
)

func TestBitsServicePerformanceTests(t *testing.T) {
	config = helpers.LoadConfig()

	if config.DefaultTimeout > 0 {
		defaultTimeout = config.DefaultTimeout * time.Second
	}

	if config.CfPushTimeout > 0 {
		cfPushTimeout = config.CfPushTimeout * time.Second
	}

	metricsPrefix = os.Getenv("PERFORMANCE_TEST_METRICS_PREFIX")

	if metricsPrefix != "" {
		metricsPrefix += "."
	}
	fmt.Printf("Using metricsPrefix=%v", metricsPrefix)

	loopCount = loopCountFromEnv()
	fmt.Printf("Using loopCount=%v", loopCount)

	context = helpers.NewContext(config)
	environment := helpers.NewEnvironment(context)

	BeforeSuite(func() {
		environment.Setup()

		var e error
		statsdClient, e = statsd.New() // Connect to the UDP port 8125 by default.
		Expect(e).NotTo(HaveOccurred())
	})

	AfterSuite(func() {
		statsdClient.Close()
		environment.Teardown()
	})

	gomega.RegisterFailHandler(ginkgo.Fail)
	ginkgo.RunSpecs(t, "BitsServicePerformanceTests Suite")
}

func loopCountFromEnv() int {
	loopCountStr := os.Getenv("LOOP_COUNT")

	if loopCountStr == "" {
		return 1
	} else {
		loopCount, err := strconv.Atoi(loopCountStr)

		if err != nil {
			panic(err)
		}

		return loopCount
	}
}
<commit_msg>Replace metrics prefix from . to _<commit_after>package main_test

import (
	"fmt"
	"time"

	"github.com/tecnickcom/statsd"

	"github.com/cloudfoundry-incubator/cf-test-helpers/helpers"

	. "github.com/onsi/ginkgo"
	. "github.com/onsi/gomega"

	"github.com/onsi/ginkgo"
	"github.com/onsi/gomega"
	"os"
	"strconv"
	"testing"
)

var (
	context        helpers.SuiteContext
	config         helpers.Config
	defaultTimeout time.Duration = 30 * time.Second
	cfPushTimeout  time.Duration = 5 * time.Minute
	statsdClient   *statsd.Client
	metricsPrefix  string
	loopCount      int
)

func TestBitsServicePerformanceTests(t *testing.T) {
	config = helpers.LoadConfig()

	if config.DefaultTimeout > 0 {
		defaultTimeout = config.DefaultTimeout * time.Second
	}

	if config.CfPushTimeout > 0 {
		cfPushTimeout = config.CfPushTimeout * time.Second
	}

	metricsPrefix = os.Getenv("PERFORMANCE_TEST_METRICS_PREFIX")

	if metricsPrefix != "" {
		metricsPrefix += "_"
	}
	fmt.Printf("Using metricsPrefix=%v", metricsPrefix)

	loopCount = loopCountFromEnv()
	fmt.Printf("Using loopCount=%v", loopCount)

	context = helpers.NewContext(config)
	environment := helpers.NewEnvironment(context)

	BeforeSuite(func() {
		environment.Setup()

		var e error
		statsdClient, e = statsd.New() // Connect to the UDP port 8125 by default.
		Expect(e).NotTo(HaveOccurred())
	})

	AfterSuite(func() {
		statsdClient.Close()
		environment.Teardown()
	})

	gomega.RegisterFailHandler(ginkgo.Fail)
	ginkgo.RunSpecs(t, "BitsServicePerformanceTests Suite")
}

func loopCountFromEnv() int {
	loopCountStr := os.Getenv("LOOP_COUNT")

	if loopCountStr == "" {
		return 1
	} else {
		loopCount, err := strconv.Atoi(loopCountStr)

		if err != nil {
			panic(err)
		}

		return loopCount
	}
}
<|endoftext|>


<commit_before>package prime

import (
	"reflect"
	"sort"
	"testing"
)

func TestPrimeFactors(t *testing.T) {
	for _, test := range tests {
		actual := Factors(test.input)
		sort.Sort(ascending(actual))
		if !reflect.DeepEqual(actual, test.expected) {
			t.Fatalf("FAIL %s\nFactors(%d) = %#v;\nexpected %#v",
				test.description, test.input,
				actual, test.expected)
		}
		t.Logf("PASS %s", test.description)
	}
}

func BenchmarkPrimeFactors(b *testing.B) {
	for i := 0; i < b.N; i++ {
		for _, test := range tests {
			Factors(test.input)
		}
	}
}

type ascending []int64

func (l ascending) Len() int             { return len(l) }
func (l ascending) Swap(ii, jj int)      { l[ii], l[jj] = l[jj], l[ii] }
func (l ascending) Less(ii, jj int) bool { return l[ii] < l[jj] }
<commit_msg>prime-factors: sort test cases.\nGuard against unordered test cases by sorting them along with results.<commit_after>package prime

import (
	"reflect"
	"sort"
	"testing"
)

func TestPrimeFactors(t *testing.T) {
	for _, test := range tests {
		actual := Factors(test.input)
		sort.Slice(actual, ascending(actual))
		sort.Slice(test.expected, ascending(test.expected))
		if !reflect.DeepEqual(actual, test.expected) {
			t.Fatalf("FAIL %s\nFactors(%d) = %#v;\nexpected %#v",
				test.description, test.input,
				actual, test.expected)
		}
		t.Logf("PASS %s", test.description)
	}
}

func BenchmarkPrimeFactors(b *testing.B) {
	for i := 0; i < b.N; i++ {
		for _, test := range tests {
			Factors(test.input)
		}
	}
}

func ascending(list []int64) func(int, int) bool {
	return func(ii, jj int) bool {
		return list[ii] < list[jj]
	}
}
<|endoftext|>


<commit_before>// Copyright 2020 Google LLC
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// https://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package server

import (
	"crypto/sha256"
	"encoding/hex"
	"encoding/json"
	"fmt"
	"log"
	"strconv"
	"strings"

	"github.com/kubeflow/pipelines/backend/src/cache/client"
	"github.com/kubeflow/pipelines/backend/src/cache/model"
	"github.com/kubeflow/pipelines/backend/src/cache/storage"
	"k8s.io/api/admission/v1beta1"
	corev1 "k8s.io/api/core/v1"
	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
)

const (
	KFPCacheEnabledLabelKey   string = "pipelines.kubeflow.org/cache_enabled"
	KFPCacheEnabledLabelValue string = "true"
	ArgoWorkflowNodeName      string = "workflows.argoproj.io/node-name"
	ArgoWorkflowTemplate      string = "workflows.argoproj.io/template"
	ExecutionKey              string = "pipelines.kubeflow.org/execution_cache_key"
	CacheIDLabelKey           string = "pipelines.kubeflow.org/cache_id"
	ArgoWorkflowOutputs       string = "workflows.argoproj.io/outputs"
	MetadataWrittenKey        string = "pipelines.kubeflow.org/metadata_written"
	AnnotationPath            string = "/metadata/annotations"
	LabelPath                 string = "/metadata/labels"
	SpecContainersPath        string = "/spec/containers"
	SpecInitContainersPath    string = "/spec/initContainers"
	TFXPodSuffix              string = "tfx/orchestration/kubeflow/container_entrypoint.py"
	ArchiveLocationKey        string = "archiveLocation"
)

var (
	podResource = metav1.GroupVersionResource{Version: "v1", Resource: "pods"}
)

type ClientManagerInterface interface {
	CacheStore() storage.ExecutionCacheStoreInterface
	KubernetesCoreClient() client.KubernetesCoreInterface
}

// MutatePodIfCached will check whether the execution has already been run before from MLMD and apply the output into pod.metadata.output
func MutatePodIfCached(req *v1beta1.AdmissionRequest, clientMgr ClientManagerInterface) ([]patchOperation, error) {
	// This handler should only get called on Pod objects as per the MutatingWebhookConfiguration in the YAML file.
	// However, if (for whatever reason) this gets invoked on an object of a different kind, issue a log message but
	// let the object request pass through otherwise.
	if req.Resource != podResource {
		log.Printf("Expect resource to be %q, but found %q", podResource, req.Resource)
		return nil, nil
	}

	// Parse the Pod object.
	raw := req.Object.Raw
	pod := corev1.Pod{}
	if _, _, err := universalDeserializer.Decode(raw, nil, &pod); err != nil {
		return nil, fmt.Errorf("could not deserialize pod object: %v", err)
	}

	// Pod filtering to only cache KFP argo pods except TFX pods
	// TODO: Switch to objectSelector once Kubernetes 1.15 hits the GKE stable channel. See
	// https://github.com/kubernetes/kubernetes/pull/78505
	// https://cloud.google.com/kubernetes-engine/docs/release-notes-stable
	if !isKFPCacheEnabled(&pod) {
		log.Printf("This pod %s does not enable cache.", pod.ObjectMeta.Name)
		return nil, nil
	}

	if isTFXPod(&pod) {
		log.Printf("This pod %s is created by tfx pipelines.", pod.ObjectMeta.Name)
		return nil, nil
	}

	var patches []patchOperation
	annotations := pod.ObjectMeta.Annotations
	labels := pod.ObjectMeta.Labels
	template, exists := annotations[ArgoWorkflowTemplate]
	var executionHashKey string
	if !exists {
		return patches, nil
	}

	// Generate the executionHashKey based on pod.metadata.annotations.workflows.argoproj.io/template
	executionHashKey, err := generateCacheKeyFromTemplate(template)
	log.Println(executionHashKey)
	if err != nil {
		log.Printf("Unable to generate cache key for pod %s : %s", pod.ObjectMeta.Name, err.Error())
		return patches, nil
	}

	annotations[ExecutionKey] = executionHashKey
	labels[CacheIDLabelKey] = ""
	var maxCacheStalenessInSeconds int64 = -1
	maxCacheStaleness, exists := annotations[MaxCacheStalenessKey]
	if exists {
		maxCacheStalenessInSeconds = getMaxCacheStaleness(maxCacheStaleness)
	}

	var cachedExecution *model.ExecutionCache
	cachedExecution, err = clientMgr.CacheStore().GetExecutionCache(executionHashKey, maxCacheStalenessInSeconds)
	if err != nil {
		log.Println(err.Error())
	}
	// Found cached execution, add cached output and cache_id and replace container images.
	if cachedExecution != nil {
		log.Println("Cached output: " + cachedExecution.ExecutionOutput)

		annotations[ArgoWorkflowOutputs] = getValueFromSerializedMap(cachedExecution.ExecutionOutput, ArgoWorkflowOutputs)
		labels[CacheIDLabelKey] = strconv.FormatInt(cachedExecution.ID, 10)
		labels[MetadataExecutionIDKey] = getValueFromSerializedMap(cachedExecution.ExecutionOutput, MetadataExecutionIDKey)
		labels[MetadataWrittenKey] = "true"

		dummyContainer := corev1.Container{
			Name:    "main",
			Image:   "alpine",
			Command: []string{`echo`, `"This step output is taken from cache."`},
		}
		dummyContainers := []corev1.Container{
			dummyContainer,
		}
		patches = append(patches, patchOperation{
			Op:    OperationTypeReplace,
			Path:  SpecContainersPath,
			Value: dummyContainers,
		})
		if pod.Spec.InitContainers != nil || len(pod.Spec.InitContainers) != 0 {
			patches = append(patches, patchOperation{
				Op:   OperationTypeRemove,
				Path: SpecInitContainersPath,
			})
		}
	}

	// Add executionKey to pod.metadata.annotations
	patches = append(patches, patchOperation{
		Op:    OperationTypeAdd,
		Path:  AnnotationPath,
		Value: annotations,
	})

	// Add cache_id label key
	patches = append(patches, patchOperation{
		Op:    OperationTypeAdd,
		Path:  LabelPath,
		Value: labels,
	})

	return patches, nil
}

func generateCacheKeyFromTemplate(template string) (string, error) {
	var templateMap map[string]interface{}
	b := []byte(template)
	err := json.Unmarshal(b, &templateMap)
	if err != nil {
		return "", err
	}

	// template[archiveLocation] needs to be removed when calculating cache key.
	// Because archiveLocation.key is different in every single run.
	_, exists := templateMap[ArchiveLocationKey]
	if exists {
		log.Println("ArchiveLocation exists in template.")
		delete(templateMap, ArchiveLocationKey)
	}
	b, err = json.Marshal(templateMap)
	if err != nil {
		return "", err
	}
	hash := sha256.New()
	hash.Write(b)
	md := hash.Sum(nil)
	executionHashKey := hex.EncodeToString(md)

	return executionHashKey, nil
}

func getValueFromSerializedMap(serializedMap string, key string) string {
	var outputMap map[string]interface{}
	b := []byte(serializedMap)
	err := json.Unmarshal(b, &outputMap)
	if err != nil {
		return ""
	}

	value, exist := outputMap[key].(string)
	if !exist || value == "" {
		return ""
	}
	return value
}

func isKFPCacheEnabled(pod *corev1.Pod) bool {
	cacheEnabled, exists := pod.ObjectMeta.Labels[KFPCacheEnabledLabelKey]
	if !exists {
		log.Printf("This pod %s is not created by KFP.", pod.ObjectMeta.Name)
		return false
	}
	return cacheEnabled == KFPCacheEnabledLabelValue
}

func isTFXPod(pod *corev1.Pod) bool {
	containers := pod.Spec.Containers
	if containers == nil || len(containers) == 0 {
		log.Printf("This pod container does not exist.")
		return true
	}
	var mainContainers []corev1.Container
	for _, c := range containers {
		if c.Name != "" && c.Name == "main" {
			mainContainers = append(mainContainers, c)
		}
	}
	if len(mainContainers) != 1 {
		return false
	}
	mainContainer := mainContainers[0]
	return len(mainContainer.Command) != 0 && strings.HasSuffix(mainContainer.Command[len(mainContainer.Command)-1], TFXPodSuffix)
}
<commit_msg>[Caching] Add a cached label for cached pods (#3623)<commit_after>// Copyright 2020 Google LLC
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// https://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package server

import (
	"crypto/sha256"
	"encoding/hex"
	"encoding/json"
	"fmt"
	"log"
	"strconv"
	"strings"

	"github.com/kubeflow/pipelines/backend/src/cache/client"
	"github.com/kubeflow/pipelines/backend/src/cache/model"
	"github.com/kubeflow/pipelines/backend/src/cache/storage"
	"k8s.io/api/admission/v1beta1"
	corev1 "k8s.io/api/core/v1"
	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
)

const (
	KFPCacheEnabledLabelKey   string = "pipelines.kubeflow.org/cache_enabled"
	KFPCacheEnabledLabelValue string = "true"
	KFPCachedLabelKey         string = "pipelines.kubeflow.org/reused_from_cache"
	KFPCachedLabelValue       string = "true"
	ArgoWorkflowNodeName      string = "workflows.argoproj.io/node-name"
	ArgoWorkflowTemplate      string = "workflows.argoproj.io/template"
	ExecutionKey              string = "pipelines.kubeflow.org/execution_cache_key"
	CacheIDLabelKey           string = "pipelines.kubeflow.org/cache_id"
	ArgoWorkflowOutputs       string = "workflows.argoproj.io/outputs"
	MetadataWrittenKey        string = "pipelines.kubeflow.org/metadata_written"
	AnnotationPath            string = "/metadata/annotations"
	LabelPath                 string = "/metadata/labels"
	SpecContainersPath        string = "/spec/containers"
	SpecInitContainersPath    string = "/spec/initContainers"
	TFXPodSuffix              string = "tfx/orchestration/kubeflow/container_entrypoint.py"
	ArchiveLocationKey        string = "archiveLocation"
)

var (
	podResource = metav1.GroupVersionResource{Version: "v1", Resource: "pods"}
)

type ClientManagerInterface interface {
	CacheStore() storage.ExecutionCacheStoreInterface
	KubernetesCoreClient() client.KubernetesCoreInterface
}

// MutatePodIfCached will check whether the execution has already been run before from MLMD and apply the output into pod.metadata.output
func MutatePodIfCached(req *v1beta1.AdmissionRequest, clientMgr ClientManagerInterface) ([]patchOperation, error) {
	// This handler should only get called on Pod objects as per the MutatingWebhookConfiguration in the YAML file.
	// However, if (for whatever reason) this gets invoked on an object of a different kind, issue a log message but
	// let the object request pass through otherwise.
	if req.Resource != podResource {
		log.Printf("Expect resource to be %q, but found %q", podResource, req.Resource)
		return nil, nil
	}

	// Parse the Pod object.
	raw := req.Object.Raw
	pod := corev1.Pod{}
	if _, _, err := universalDeserializer.Decode(raw, nil, &pod); err != nil {
		return nil, fmt.Errorf("could not deserialize pod object: %v", err)
	}

	// Pod filtering to only cache KFP argo pods except TFX pods
	// TODO: Switch to objectSelector once Kubernetes 1.15 hits the GKE stable channel. See
	// https://github.com/kubernetes/kubernetes/pull/78505
	// https://cloud.google.com/kubernetes-engine/docs/release-notes-stable
	if !isKFPCacheEnabled(&pod) {
		log.Printf("This pod %s does not enable cache.", pod.ObjectMeta.Name)
		return nil, nil
	}

	if isTFXPod(&pod) {
		log.Printf("This pod %s is created by tfx pipelines.", pod.ObjectMeta.Name)
		return nil, nil
	}

	var patches []patchOperation
	annotations := pod.ObjectMeta.Annotations
	labels := pod.ObjectMeta.Labels
	template, exists := annotations[ArgoWorkflowTemplate]
	var executionHashKey string
	if !exists {
		return patches, nil
	}

	// Generate the executionHashKey based on pod.metadata.annotations.workflows.argoproj.io/template
	executionHashKey, err := generateCacheKeyFromTemplate(template)
	log.Println(executionHashKey)
	if err != nil {
		log.Printf("Unable to generate cache key for pod %s : %s", pod.ObjectMeta.Name, err.Error())
		return patches, nil
	}

	annotations[ExecutionKey] = executionHashKey
	labels[CacheIDLabelKey] = ""
	var maxCacheStalenessInSeconds int64 = -1
	maxCacheStaleness, exists := annotations[MaxCacheStalenessKey]
	if exists {
		maxCacheStalenessInSeconds = getMaxCacheStaleness(maxCacheStaleness)
	}

	var cachedExecution *model.ExecutionCache
	cachedExecution, err = clientMgr.CacheStore().GetExecutionCache(executionHashKey, maxCacheStalenessInSeconds)
	if err != nil {
		log.Println(err.Error())
	}
	// Found cached execution, add cached output and cache_id and replace container images.
	if cachedExecution != nil {
		log.Println("Cached output: " + cachedExecution.ExecutionOutput)

		annotations[ArgoWorkflowOutputs] = getValueFromSerializedMap(cachedExecution.ExecutionOutput, ArgoWorkflowOutputs)
		labels[CacheIDLabelKey] = strconv.FormatInt(cachedExecution.ID, 10)
		labels[KFPCachedLabelKey] = KFPCachedLabelValue // This label indicates the pod is taken from cache.

		// These labels cache results for metadata-writer.
		labels[MetadataExecutionIDKey] = getValueFromSerializedMap(cachedExecution.ExecutionOutput, MetadataExecutionIDKey)
		labels[MetadataWrittenKey] = "true"

		dummyContainer := corev1.Container{
			Name:    "main",
			Image:   "alpine",
			Command: []string{`echo`, `"This step output is taken from cache."`},
		}
		dummyContainers := []corev1.Container{
			dummyContainer,
		}
		patches = append(patches, patchOperation{
			Op:    OperationTypeReplace,
			Path:  SpecContainersPath,
			Value: dummyContainers,
		})
		if pod.Spec.InitContainers != nil || len(pod.Spec.InitContainers) != 0 {
			patches = append(patches, patchOperation{
				Op:   OperationTypeRemove,
				Path: SpecInitContainersPath,
			})
		}
	}

	// Add executionKey to pod.metadata.annotations
	patches = append(patches, patchOperation{
		Op:    OperationTypeAdd,
		Path:  AnnotationPath,
		Value: annotations,
	})

	// Add cache_id label key
	patches = append(patches, patchOperation{
		Op:    OperationTypeAdd,
		Path:  LabelPath,
		Value: labels,
	})

	return patches, nil
}

func generateCacheKeyFromTemplate(template string) (string, error) {
	var templateMap map[string]interface{}
	b := []byte(template)
	err := json.Unmarshal(b, &templateMap)
	if err != nil {
		return "", err
	}

	// template[archiveLocation] needs to be removed when calculating cache key.
	// Because archiveLocation.key is different in every single run.
	_, exists := templateMap[ArchiveLocationKey]
	if exists {
		log.Println("ArchiveLocation exists in template.")
		delete(templateMap, ArchiveLocationKey)
	}
	b, err = json.Marshal(templateMap)
	if err != nil {
		return "", err
	}
	hash := sha256.New()
	hash.Write(b)
	md := hash.Sum(nil)
	executionHashKey := hex.EncodeToString(md)

	return executionHashKey, nil
}

func getValueFromSerializedMap(serializedMap string, key string) string {
	var outputMap map[string]interface{}
	b := []byte(serializedMap)
	err := json.Unmarshal(b, &outputMap)
	if err != nil {
		return ""
	}

	value, exist := outputMap[key].(string)
	if !exist || value == "" {
		return ""
	}
	return value
}

func isKFPCacheEnabled(pod *corev1.Pod) bool {
	cacheEnabled, exists := pod.ObjectMeta.Labels[KFPCacheEnabledLabelKey]
	if !exists {
		log.Printf("This pod %s is not created by KFP.", pod.ObjectMeta.Name)
		return false
	}
	return cacheEnabled == KFPCacheEnabledLabelValue
}

func isTFXPod(pod *corev1.Pod) bool {
	containers := pod.Spec.Containers
	if containers == nil || len(containers) == 0 {
		log.Printf("This pod container does not exist.")
		return true
	}
	var mainContainers []corev1.Container
	for _, c := range containers {
		if c.Name != "" && c.Name == "main" {
			mainContainers = append(mainContainers, c)
		}
	}
	if len(mainContainers) != 1 {
		return false
	}
	mainContainer := mainContainers[0]
	return len(mainContainer.Command) != 0 && strings.HasSuffix(mainContainer.Command[len(mainContainer.Command)-1], TFXPodSuffix)
}
<|endoftext|>


<commit_before>package signprocessor

import (
	"encoding/json"
	"fmt"
	"io/ioutil"
	"log"
	"strings"
	"time"

	"github.com/google/go-github/github"
	"github.com/mrjones/oauth"
	"github.com/zabawaba99/fireauth"
	"github.com/zabawaba99/firego"

	"golang.org/x/oauth2"
)

func EventLoop() {
	data := make(fireauth.Data)
	options := fireauth.Option{
		Admin: true,
	}
	token, _ := fireauth.New(Secrets.FireBaseSecret).CreateToken(data, &options)
	f := firego.New(Config.FireBaseDB, nil)
	f.Auth(token)

	t := oauth.NewConsumer(Secrets.TwitterKey, Secrets.TwitterSecret, oauth.ServiceProvider{
		RequestTokenUrl:   "https://api.twitter.com/oauth/request_token",
		AuthorizeTokenUrl: "https://api.twitter.com/oauth/authorize",
		AccessTokenUrl:    "https://api.twitter.com/oauth/access_token",
	})

	ts := oauth2.StaticTokenSource(
		&oauth2.Token{AccessToken: Secrets.GithubToken},
	)
	tc := oauth2.NewClient(oauth2.NoContext, ts)
	gh := github.NewClient(tc)

	notifications := make(chan firego.Event)
	if err := f.Watch(notifications); err != nil {
		log.Fatalf("Error setting up watch: %v", err)
	}

	defer f.StopWatching()
	for event := range notifications {
		if event.Path == "/" && event.Data != nil {
			if users, ok := event.Data.(map[string]interface{})["users"]; ok {
				for uid, d := range users.(map[string]interface{}) {
					details := d.(map[string]interface{})
					if process(uid, details, t, gh) {
						f.Child("users").Child(uid).Remove()
					}
				}
			}
		} else if event.Type == "put" && strings.HasPrefix(event.Path, "/users/") {
			uid := strings.TrimPrefix(event.Path, "/users/")
			if details, ok := event.Data.(map[string]interface{}); ok {
				if process(uid, details, t, gh) {
					f.Child("users").Child(uid).Remove()
				}
			}
		}
	}
	fmt.Printf("Notifications have stopped\n")
}

func process(uid string, details map[string]interface{}, t *oauth.Consumer, gh *github.Client) bool {
	linkProfile := details["linkProfile"].(bool)
	link := details["link"].(string)
	personalPage := details["personalPage"].(string)
	name := details["name"].(string)
	title := details["title"].(string)
	affiliation := details["affiliation"].(string)

	secret := details["twitterSecret"].(string)
	token := details["twitterToken"].(string)
	tclient, err := t.MakeHttpClient(&oauth.AccessToken{Token: token, Secret: secret})
	resp, err := tclient.Get("https://api.twitter.com/1.1/account/verify_credentials.json?skip_status=true&include_entities=false")
	if err != nil {
		log.Printf("Twitter API error: %v processing %s", err, uid)
		return false
	}
	defer resp.Body.Close()

	bits, _ := ioutil.ReadAll(resp.Body)

	var user map[string]interface{}
	json.Unmarshal(bits, &user)
	egg := user["default_profile_image"].(bool)
	created, _ := time.Parse(time.RubyDate, user["created_at"].(string))
	description := user["description"].(string)
	followers := int(user["followers_count"].(float64))
	following := int(user["friends_count"].(float64))
	displayName := user["name"].(string)
	handle := user["screen_name"].(string)
	tweets := int(user["statuses_count"].(float64))
	var url string
	switch user["url"].(type) {
	case string:
		url = user["url"].(string)
	}

	if name == "" {
		log.Printf("No signatory name specified for %s (%s)", uid, handle)
		return false
	}

	if score := Score(handle, displayName, url, created, followers, following, tweets, egg, description, personalPage); score < 0 {
		log.Printf("Not creating pull for %s (%s) due to score %d", uid, handle, score)
		return false
	}

	body := fmt.Sprintf("Twitter user: https://twitter.com/%s\nDisplay name: %s\nURL: %s\nCreated: %v, Followers: %d, Following: %d, Tweets: %d, Egg: %v\nTagline: %s\nPersonal page: %s", handle, displayName, url, created, followers, following, tweets, egg, description, personalPage)

	var linkMd, affiliationMd, titleMd string
	if linkProfile {
		linkMd = fmt.Sprintf("  link: https://twitter.com/%s\n", handle)
	} else if link != "" {
		linkMd = fmt.Sprintf("  link: %s\n", link)
	}
	if affiliation != "" {
		affiliationMd = fmt.Sprintf("  affiliation: \"%s\"\n", affiliation)
	}
	if title != "" {
		titleMd = fmt.Sprintf("  occupation_title: \"%s\"\n", title)
	}
	contents := fmt.Sprintf("---\n  name: \"%s\"\n%s%s%s---", name, linkMd, affiliationMd, titleMd)

	// Ensure we are forking from a clean state.
	g := gh.Git
	ref, _, err := g.GetRef("neveragaindottech", "neveragaindottech.github.io", "heads/master")
	if err != nil {
		log.Printf("Error: %v processing %s", err, uid)
		return false
	}
	refStr := fmt.Sprintf("heads/%s", uid)
	ref.Ref = &refStr
	if _, _, err = g.UpdateRef(Config.GithubUser, "neveragaindottech.github.io", ref, true); err != nil {
		if _, _, err = g.CreateRef(Config.GithubUser, "neveragaindottech.github.io", ref); err != nil {
			log.Printf("Error: %v processing %s", err, uid)
			return false
		}
	}
	baseC, _, err := g.GetCommit(Config.GithubUser, "neveragaindottech.github.io", *ref.Object.SHA)
	if err != nil {
		log.Printf("Error: %v processing %s", err, uid)
		return false
	}
	path := fmt.Sprintf("_signatures/%s.md", uid)
	mode := "100644"
	kind := "blob"
	newT, _, err := g.CreateTree(Config.GithubUser, "neveragaindottech.github.io", *baseC.Tree.SHA, []github.TreeEntry{
		{Path: &path, Mode: &mode, Type: &kind, Content: &contents},
	})
	if err != nil {
		log.Printf("Error: %v processing %s", err, uid)
		return false
	}
	desc := fmt.Sprintf("SignBot: Add signatory '%s' (%s)", name, handle)
	newC, _, err := g.CreateCommit(Config.GithubUser, "neveragaindottech.github.io", &github.Commit{
		Message: &desc,
		Tree:    newT,
		Parents: []github.Commit{*baseC},
	})
	if err != nil {
		log.Printf("Error: %v processing %s", err, uid)
		return false
	}
	if _, _, err = g.UpdateRef(Config.GithubUser, "neveragaindottech.github.io", &github.Reference{
		Ref: &refStr,
		Object: &github.GitObject{
			SHA: newC.SHA,
		},
	}, false); err != nil {
		log.Printf("Error: %v processing %s", err, uid)
		return false
	}
	p := gh.PullRequests
	branchName := fmt.Sprintf("%s:%s", Config.GithubUser, uid)
	master := "master"
	_, _, err = p.Create("neveragaindottech", "neveragaindottech.github.io", &github.NewPullRequest{
		Title: &desc,
		Head:  &branchName,
		Base:  &master,
		Body:  &body,
	})
	if err != nil {
		log.Printf("Error: %v processing %s", err, uid)
		return false
	}
	fmt.Printf("Processed %s (https://twitter.com/%s)\n", uid, handle)
	return true
}
<commit_msg>Put more details in the PR description.<commit_after>package signprocessor

import (
	"encoding/json"
	"fmt"
	"io/ioutil"
	"log"
	"strings"
	"time"

	"github.com/google/go-github/github"
	"github.com/mrjones/oauth"
	"github.com/zabawaba99/fireauth"
	"github.com/zabawaba99/firego"

	"golang.org/x/oauth2"
)

func EventLoop() {
	data := make(fireauth.Data)
	options := fireauth.Option{
		Admin: true,
	}
	token, _ := fireauth.New(Secrets.FireBaseSecret).CreateToken(data, &options)
	f := firego.New(Config.FireBaseDB, nil)
	f.Auth(token)

	t := oauth.NewConsumer(Secrets.TwitterKey, Secrets.TwitterSecret, oauth.ServiceProvider{
		RequestTokenUrl:   "https://api.twitter.com/oauth/request_token",
		AuthorizeTokenUrl: "https://api.twitter.com/oauth/authorize",
		AccessTokenUrl:    "https://api.twitter.com/oauth/access_token",
	})

	ts := oauth2.StaticTokenSource(
		&oauth2.Token{AccessToken: Secrets.GithubToken},
	)
	tc := oauth2.NewClient(oauth2.NoContext, ts)
	gh := github.NewClient(tc)

	notifications := make(chan firego.Event)
	if err := f.Watch(notifications); err != nil {
		log.Fatalf("Error setting up watch: %v", err)
	}

	defer f.StopWatching()
	for event := range notifications {
		if event.Path == "/" && event.Data != nil {
			if users, ok := event.Data.(map[string]interface{})["users"]; ok {
				for uid, d := range users.(map[string]interface{}) {
					details := d.(map[string]interface{})
					if process(uid, details, t, gh) {
						f.Child("users").Child(uid).Remove()
					}
				}
			}
		} else if event.Type == "put" && strings.HasPrefix(event.Path, "/users/") {
			uid := strings.TrimPrefix(event.Path, "/users/")
			if details, ok := event.Data.(map[string]interface{}); ok {
				if process(uid, details, t, gh) {
					f.Child("users").Child(uid).Remove()
				}
			}
		}
	}
	fmt.Printf("Notifications have stopped\n")
}

func process(uid string, details map[string]interface{}, t *oauth.Consumer, gh *github.Client) bool {
	linkProfile := details["linkProfile"].(bool)
	link := details["link"].(string)
	personalPage := details["personalPage"].(string)
	name := details["name"].(string)
	title := details["title"].(string)
	affiliation := details["affiliation"].(string)

	secret := details["twitterSecret"].(string)
	token := details["twitterToken"].(string)
	tclient, err := t.MakeHttpClient(&oauth.AccessToken{Token: token, Secret: secret})
	resp, err := tclient.Get("https://api.twitter.com/1.1/account/verify_credentials.json?skip_status=true&include_entities=false")
	if err != nil {
		log.Printf("Twitter API error: %v processing %s", err, uid)
		return false
	}
	defer resp.Body.Close()

	bits, _ := ioutil.ReadAll(resp.Body)

	var user map[string]interface{}
	json.Unmarshal(bits, &user)
	egg := user["default_profile_image"].(bool)
	created, _ := time.Parse(time.RubyDate, user["created_at"].(string))
	description := user["description"].(string)
	followers := int(user["followers_count"].(float64))
	following := int(user["friends_count"].(float64))
	displayName := user["name"].(string)
	handle := user["screen_name"].(string)
	tweets := int(user["statuses_count"].(float64))
	var url string
	switch user["url"].(type) {
	case string:
		url = user["url"].(string)
	}

	if name == "" {
		log.Printf("No signatory name specified for %s (%s)", uid, handle)
		return false
	}

	if score := Score(handle, displayName, url, created, followers, following, tweets, egg, description, personalPage); score < 0 {
		log.Printf("Not creating pull for %s (%s) due to score %d", uid, handle, score)
		return false
	}

	var linkMd, affiliationMd, titleMd string
	if linkProfile {
		linkMd = fmt.Sprintf("  link: https://twitter.com/%s\n", handle)
	} else if link != "" {
		linkMd = fmt.Sprintf("  link: %s\n", link)
	}
	if affiliation != "" {
		affiliationMd = fmt.Sprintf("  affiliation: \"%s\"\n", affiliation)
	}
	if title != "" {
		titleMd = fmt.Sprintf("  occupation_title: \"%s\"\n", title)
	}
	contents := fmt.Sprintf("---\n  name: \"%s\"\n%s%s%s---", name, linkMd, affiliationMd, titleMd)

	body := fmt.Sprintf(
		"Twitter user: https://twitter.com/%s\n" +
		"Created: %v, Followers: %d, Following: %d, Tweets: %d, Egg: %v\n" +
		"\n" +
		"Twitter profile fields:\n" +
		"Name: %s\n" +
		"Website: %s\n" +
		"Tagline: %s\n" +
		"\n" +
		"Personal page: %s\n" +
		"\n" +
		"Signature file contents:\n" +
		"%s",
		handle,
		created, followers, following, tweets, egg,
		displayName,
		url,
		description,
		personalPage,
		"    " + String.Replace(contents, "\n", "\n    ")
	)

	// Ensure we are forking from a clean state.
	g := gh.Git
	ref, _, err := g.GetRef("neveragaindottech", "neveragaindottech.github.io", "heads/master")
	if err != nil {
		log.Printf("Error: %v processing %s", err, uid)
		return false
	}
	refStr := fmt.Sprintf("heads/%s", uid)
	ref.Ref = &refStr
	if _, _, err = g.UpdateRef(Config.GithubUser, "neveragaindottech.github.io", ref, true); err != nil {
		if _, _, err = g.CreateRef(Config.GithubUser, "neveragaindottech.github.io", ref); err != nil {
			log.Printf("Error: %v processing %s", err, uid)
			return false
		}
	}
	baseC, _, err := g.GetCommit(Config.GithubUser, "neveragaindottech.github.io", *ref.Object.SHA)
	if err != nil {
		log.Printf("Error: %v processing %s", err, uid)
		return false
	}
	path := fmt.Sprintf("_signatures/%s.md", uid)
	mode := "100644"
	kind := "blob"
	newT, _, err := g.CreateTree(Config.GithubUser, "neveragaindottech.github.io", *baseC.Tree.SHA, []github.TreeEntry{
		{Path: &path, Mode: &mode, Type: &kind, Content: &contents},
	})
	if err != nil {
		log.Printf("Error: %v processing %s", err, uid)
		return false
	}
	desc := fmt.Sprintf("SignBot: Add signatory '%s' (%s)", name, handle)
	newC, _, err := g.CreateCommit(Config.GithubUser, "neveragaindottech.github.io", &github.Commit{
		Message: &desc,
		Tree:    newT,
		Parents: []github.Commit{*baseC},
	})
	if err != nil {
		log.Printf("Error: %v processing %s", err, uid)
		return false
	}
	if _, _, err = g.UpdateRef(Config.GithubUser, "neveragaindottech.github.io", &github.Reference{
		Ref: &refStr,
		Object: &github.GitObject{
			SHA: newC.SHA,
		},
	}, false); err != nil {
		log.Printf("Error: %v processing %s", err, uid)
		return false
	}
	p := gh.PullRequests
	branchName := fmt.Sprintf("%s:%s", Config.GithubUser, uid)
	master := "master"
	_, _, err = p.Create("neveragaindottech", "neveragaindottech.github.io", &github.NewPullRequest{
		Title: &desc,
		Head:  &branchName,
		Base:  &master,
		Body:  &body,
	})
	if err != nil {
		log.Printf("Error: %v processing %s", err, uid)
		return false
	}
	fmt.Printf("Processed %s (https://twitter.com/%s)\n", uid, handle)
	return true
}
<|endoftext|>


<commit_before>package main

import (
	"fmt"
	"log"

	"k8s.io/kubernetes/pkg/api"
	"k8s.io/kubernetes/pkg/client/unversioned"
	"k8s.io/kubernetes/pkg/fields"
	"k8s.io/kubernetes/pkg/labels"
)

func pollKubernetes() {
	debugf("polling kubernetes API")
	go pollRCs()
	go pollSVCs()
	go pollNodes()
}

type kubeFunc func(*unversioned.Client) error

// Downstream func can ignore API errors
//   because API return structs on error are initialized properly
//   and we log if there was an error in the returned error
func withKube(f kubeFunc) {
	client, err := unversioned.New(&unversioned.Config{
		Host:    kubeAddr,
		Version: "v1",
	})

	if err != nil {
		debugf("client not created: %v\n", err)
		return
	}

	err = f(client)
	if err != nil {
		debugf("kubeFunc returned error: %v\n", err)
		incr("kubestats.errors")
	}
}

// find all RCs and report replica count
func pollRCs() {
	debugf("polling RCs")
	withKube(func(client *unversioned.Client) error {
		rcs, err := client.ReplicationControllers(api.NamespaceAll).List(labels.Everything())
		for _, i := range rcs.Items {
			gauge(fmt.Sprintf("rc.%s.%s", i.ObjectMeta.Namespace, i.ObjectMeta.Name), i.Status.Replicas)
		}
		return err
	})
}

// report endpoint addr count for a service
func showSvc(svc api.Service) {
	withKube(func(client *unversioned.Client) error {
		e := 0
		eps, err := client.Endpoints(svc.ObjectMeta.Namespace).List(labels.Set(svc.Spec.Selector).AsSelector())
		for _, ep := range eps.Items {
			for _, ss := range ep.Subsets {
				e = e + len(ss.Addresses)
			}
		}
		gauge(fmt.Sprintf("svc.%s.%s", svc.ObjectMeta.Namespace, svc.ObjectMeta.Name), e)
		return err
	})
}

// find all services
func pollSVCs() {
	debugf("polling services")
	withKube(func(client *unversioned.Client) error {
		svcs, err := client.Services(api.NamespaceAll).List(labels.Everything())
		for _, i := range svcs.Items {
			go showSvc(i)
		}
		return err
	})
}

// find all nodes and record gauge of statuses
// save map of node names to ips
func pollNodes() {
	debugf("polling nodes")
	withKube(func(client *unversioned.Client) error {
		nodes, err := client.Nodes().List(labels.Everything(), fields.Everything())
		status := make(map[string]int)
		names := make(map[string]string)
		for _, i := range nodes.Items {
			names[i.Status.Addresses[0].Address] = i.ObjectMeta.Name
			for _, c := range i.Status.Conditions {
				if c.Type == "Ready" {
					var key string
					switch c.Status {
					case "True":
						key = "ready"
					case "False":
						key = "notready"
					default:
						key = "notready"
						log.Printf("Unknown ready status: %s\n", c.Status)
					}
					status[key]++
					continue
				}
				log.Printf("Node condition %s: %s\n", c.Type, c.Status)
			}
		}
		for st, num := range status {
			gauge(fmt.Sprintf("nodes.status.%s", st), num)
		}
		// If all nodes are ready, reset the "notready" gauge
		if ready, ok := status["ready"]; ok {
			if ready == len(nodes.Items) {
				gauge("nodes.status.notready", 0)
			}
		}
		pollPods(names)
		return err
	})
}

// get all pods and count per node
func pollPods(names map[string]string) {
	// with ips to names, look for all pods and count by IP and report by name
	withKube(func(client *unversioned.Client) error {
		ppn := make(map[string]int)
		pods, err := client.Pods(api.NamespaceAll).List(labels.Everything(), fields.Everything())
		for _, i := range pods.Items {
			ppn[i.Status.HostIP]++
		}
		for ip, p := range ppn {
			gauge(fmt.Sprintf("nodes.pods.%s", names[ip]), p)
		}
		return err
	})
}
<commit_msg>Don't gauge pods with unknown host ips<commit_after>package main

import (
	"fmt"
	"log"

	"k8s.io/kubernetes/pkg/api"
	"k8s.io/kubernetes/pkg/client/unversioned"
	"k8s.io/kubernetes/pkg/fields"
	"k8s.io/kubernetes/pkg/labels"
)

func pollKubernetes() {
	debugf("polling kubernetes API")
	go pollRCs()
	go pollSVCs()
	go pollNodes()
}

type kubeFunc func(*unversioned.Client) error

// Downstream func can ignore API errors
//   because API return structs on error are initialized properly
//   and we log if there was an error in the returned error
func withKube(f kubeFunc) {
	client, err := unversioned.New(&unversioned.Config{
		Host:    kubeAddr,
		Version: "v1",
	})

	if err != nil {
		debugf("client not created: %v\n", err)
		return
	}

	err = f(client)
	if err != nil {
		debugf("kubeFunc returned error: %v\n", err)
		incr("kubestats.errors")
	}
}

// find all RCs and report replica count
func pollRCs() {
	debugf("polling RCs")
	withKube(func(client *unversioned.Client) error {
		rcs, err := client.ReplicationControllers(api.NamespaceAll).List(labels.Everything())
		for _, i := range rcs.Items {
			gauge(fmt.Sprintf("rc.%s.%s", i.ObjectMeta.Namespace, i.ObjectMeta.Name), i.Status.Replicas)
		}
		return err
	})
}

// report endpoint addr count for a service
func showSvc(svc api.Service) {
	withKube(func(client *unversioned.Client) error {
		e := 0
		eps, err := client.Endpoints(svc.ObjectMeta.Namespace).List(labels.Set(svc.Spec.Selector).AsSelector())
		for _, ep := range eps.Items {
			for _, ss := range ep.Subsets {
				e = e + len(ss.Addresses)
			}
		}
		gauge(fmt.Sprintf("svc.%s.%s", svc.ObjectMeta.Namespace, svc.ObjectMeta.Name), e)
		return err
	})
}

// find all services
func pollSVCs() {
	debugf("polling services")
	withKube(func(client *unversioned.Client) error {
		svcs, err := client.Services(api.NamespaceAll).List(labels.Everything())
		for _, i := range svcs.Items {
			go showSvc(i)
		}
		return err
	})
}

// find all nodes and record gauge of statuses
// save map of node names to ips
func pollNodes() {
	debugf("polling nodes")
	withKube(func(client *unversioned.Client) error {
		nodes, err := client.Nodes().List(labels.Everything(), fields.Everything())
		status := make(map[string]int)
		names := make(map[string]string)
		for _, i := range nodes.Items {
			names[i.Status.Addresses[0].Address] = i.ObjectMeta.Name
			for _, c := range i.Status.Conditions {
				if c.Type == "Ready" {
					var key string
					switch c.Status {
					case "True":
						key = "ready"
					case "False":
						key = "notready"
					default:
						key = "notready"
						log.Printf("Unknown ready status: %s\n", c.Status)
					}
					status[key]++
					continue
				}
				log.Printf("Node condition %s: %s\n", c.Type, c.Status)
			}
		}
		for st, num := range status {
			gauge(fmt.Sprintf("nodes.status.%s", st), num)
		}
		// If all nodes are ready, reset the "notready" gauge
		if ready, ok := status["ready"]; ok {
			if ready == len(nodes.Items) {
				gauge("nodes.status.notready", 0)
			}
		}
		pollPods(names)
		return err
	})
}

// get all pods and count per node
func pollPods(names map[string]string) {
	// with ips to names, look for all pods and count by IP and report by name
	withKube(func(client *unversioned.Client) error {
		ppn := make(map[string]int)
		pods, err := client.Pods(api.NamespaceAll).List(labels.Everything(), fields.Everything())
		for _, i := range pods.Items {
			ppn[i.Status.HostIP]++
		}
		for ip, p := range ppn {
			if name, ok := names[ip]; ok {
				gauge(fmt.Sprintf("nodes.pods.%s", name), p)
			} else {
				log.Printf("Unknown node ip: %s\n", ip)
			}
		}
		return err
	})
}
<|endoftext|>


<commit_before>package main

import (
	"encoding/json"
	"flag"
	"io/ioutil"
	"log"
	"strings"
	"time"

	"github.com/privacylab/talek/common"
	"github.com/privacylab/talek/libtalek"
)

var trustDomainPath = flag.String("trust", "../keys/leaderpublic.json,../keys/followerpublic.json", "Server keys (comma separated)")

// Consistency acts as a client driver against a talek system to verify that
// consistency guarantees are enforced. It will write into a cell, and then
// perform a set of reads to ensure that all servers expose the same snapshot
// of the database and are synchronized.
func main() {
	flag.Parse()

	// Config
	domainPaths := strings.Split(*trustDomainPath, ",")
	trustDomains := make([]*common.TrustDomainConfig, len(domainPaths))
	for i, path := range domainPaths {
		tdString, err := ioutil.ReadFile(path)
		if err != nil {
			log.Printf("Could not read %s!\n", path)
			return
		}
		trustDomains[i] = new(common.TrustDomainConfig)
		if err := json.Unmarshal(tdString, trustDomains[i]); err != nil {
			log.Printf("Could not parse %s: %v\n", path, err)
			return
		}
	}

	leaderRPC := common.NewFrontendRPC("RPC", trustDomains[0].Address)

	clientConfig := libtalek.ClientConfig{
		ReadInterval:  time.Second,
		WriteInterval: time.Second,
		TrustDomains:  trustDomains,
	}
	c0 := libtalek.NewClient("testClient", clientConfig, leaderRPC)
	log.Println("Created client")
	//time.Sleep(time.Duration(rand.Int()%int(clientConfig.WriteInterval)) * time.Nanosecond)
	//c1 := libtalek.NewClient("c1", clientConfig, leaderRpc)
	//log.Println("Created c1")

	topic, _ := libtalek.NewTopic()
	var handle libtalek.Handle
	handle = topic.Handle

	read := c0.Poll(&handle)

	c0.Publish(topic, []byte("PDB Client Trial"))
	log.Printf("waiting for read.")
	data := <-read

	log.Printf("Read back: %v\n", data)
}
<commit_msg>consistency now does spot-check reads comparing db snapshot of replicas.<commit_after>package main

import (
	"bytes"
	"fmt"
	"log"
	"os"

	"github.com/privacylab/talek/common"
	"github.com/privacylab/talek/drbg"
	"github.com/privacylab/talek/libtalek"
	"github.com/spf13/pflag"
)

func initReadArg(buckets uint64, replicas int) *common.ReadArgs {
	args := common.ReadArgs{}
	args.TD = make([]common.PirArgs, replicas)
	for i := 0; i < replicas; i++ {
		args.TD[i].PadSeed = make([]byte, drbg.SeedLength)
		args.TD[i].RequestVector = make([]byte, buckets/8)
	}
	return &args
}

// Consistency acts as a client driver against a talek system to verify that
// consistency guarantees are enforced. It will write into a cell, and then
// perform a set of reads to ensure that all servers expose the same snapshot
// of the database and are synchronized.
func main() {
	configPath := pflag.String("config", "talek.conf", "Client configuration for talek")
	spotChecks := pflag.Int("spotcheck", 8, "How many cells to check for consistency")
	pflag.Parse()

	// Config
	config := libtalek.ClientConfigFromFile(*configPath)
	if config == nil {
		fmt.Fprintln(os.Stderr, "Talek Client must be run with --config specifying where the server is.")
		os.Exit(1)
	}
	if config.Config == nil {
		fmt.Fprintln(os.Stderr, "Common configuration will be fetched from frontend.")
	}

	leaderRPC := common.NewFrontendRPC("RPC", config.FrontendAddr)

	if config.Config == nil {
		config.Config = new(common.Config)
		err := leaderRPC.GetConfig(nil, config.Config)
		if err != nil {
			panic(err)
		}
	}

	// 1. read through database and make sure replicas are in sync.
	for cell := uint64(0); cell < config.Config.NumBuckets; cell += config.Config.NumBuckets / uint64(*spotChecks) {
		var expectedForCell []byte
		for replica := 0; replica < len(config.TrustDomains); replica++ {
			args := initReadArg(config.Config.NumBuckets, len(config.TrustDomains))
			args.TD[replica].RequestVector[cell/8] ^= (1 << (cell % 8))

			encArgs, _ := args.Encode(config.TrustDomains)
			reply := common.ReadReply{}
			leaderRPC.Read(&encArgs, &reply)
			if replica == 0 {
				expectedForCell = reply.Data
			} else {
				if !bytes.Equal(reply.Data, expectedForCell) {
					fmt.Fprintf(os.Stderr, "Disagreement of value of cell %d between 0 and %d.\n", cell, replica)
				}
			}
		}
		fmt.Fprintf(os.Stderr, ".")
	}
	fmt.Fprintf(os.Stderr, "\n")

	c0 := libtalek.NewClient("testClient", *config, leaderRPC)
	log.Println("Created client")
	//time.Sleep(time.Duration(rand.Int()%int(clientConfig.WriteInterval)) * time.Nanosecond)
	//c1 := libtalek.NewClient("c1", clientConfig, leaderRpc)
	//log.Println("Created c1")

	topic, _ := libtalek.NewTopic()
	var handle libtalek.Handle
	handle = topic.Handle

	read := c0.Poll(&handle)

	c0.Publish(topic, []byte("PDB Client Trial"))
	log.Printf("waiting for read.")
	data := <-read

	log.Printf("Read back: %v\n", data)
}
<|endoftext|>


<commit_before>// Copyright 2012 Aaron Jacobs. All Rights Reserved.
// Author: aaronjjacobs@gmail.com (Aaron Jacobs)
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package state_test

import (
	"errors"
	"github.com/jacobsa/comeback/backup"
	"github.com/jacobsa/comeback/backup/mock"
	"github.com/jacobsa/comeback/blob"
	"github.com/jacobsa/comeback/fs"
	"github.com/jacobsa/comeback/fs/mock"
	"github.com/jacobsa/comeback/state"
	. "github.com/jacobsa/oglematchers"
	"github.com/jacobsa/oglemock"
	. "github.com/jacobsa/ogletest"
	"testing"
	"time"
)

func TestScoreMapSaver(t *testing.T) { RunTests(t) }

////////////////////////////////////////////////////////////////////////
// Helpers
////////////////////////////////////////////////////////////////////////

type ScoreMapSaverTest struct {
	sourceMap  state.ScoreMap
	sinkMap    state.ScoreMap
	fileSystem mock_fs.MockFileSystem
	wrapped    mock_backup.MockFileSaver
	saver      backup.FileSaver

	path   string
	scores []blob.Score
	err    error
}

func init() { RegisterTestSuite(&ScoreMapSaverTest{}) }

func (t *ScoreMapSaverTest) SetUp(i *TestInfo) {
	t.sourceMap = state.NewScoreMap()
	t.sinkMap = state.NewScoreMap()
	t.fileSystem = mock_fs.NewMockFileSystem(i.MockController, "fileSystem")
	t.wrapped = mock_backup.NewMockFileSaver(i.MockController, "wrapped")

	t.saver = state.NewScoreMapFileSaver(
		t.sourceMap,
		t.sinkMap,
		t.fileSystem,
		t.wrapped,
	)
}

func (t *ScoreMapSaverTest) call() {
	t.scores, t.err = t.saver.Save(t.path)
}

////////////////////////////////////////////////////////////////////////
// Tests
////////////////////////////////////////////////////////////////////////

func (t *ScoreMapSaverTest) CallsStat() {
	t.path = "taco"

	// File system
	ExpectCall(t.fileSystem, "Stat")("taco").
		WillOnce(oglemock.Return(fs.DirectoryEntry{}, errors.New("")))

	// Call
	t.call()
}

func (t *ScoreMapSaverTest) StatReturnsError() {
	// File system
	ExpectCall(t.fileSystem, "Stat")(Any()).
		WillOnce(oglemock.Return(fs.DirectoryEntry{}, errors.New("taco")))

	// Call
	t.call()

	ExpectThat(t.err, Error(HasSubstr("Stat")))
	ExpectThat(t.err, Error(HasSubstr("taco")))
}

func (t *ScoreMapSaverTest) ScoreMapContainsEntry() {
	t.path = "taco"

	expectedKey := state.ScoreMapKey{
		Path:        "taco",
		Permissions: 0644,
		Uid:         17,
		Gid:         19,
		MTime:       time.Now(),
		Inode:       23,
		Size:        29,
	}

	// Source map
	expectedScores := []blob.Score{
		blob.ComputeScore([]byte("foo")),
		blob.ComputeScore([]byte("bar")),
	}

	t.sourceMap.Set(expectedKey, expectedScores)

	// File system
	entry := fs.DirectoryEntry{
		Permissions: 0644,
		Uid:         17,
		Gid:         19,
		MTime:       expectedKey.MTime,
		Inode:       23,
		Size:        29,
	}

	ExpectCall(t.fileSystem, "Stat")(Any()).
		WillOnce(oglemock.Return(entry, nil))

	// Call
	t.call()

	AssertEq(nil, t.err)
	ExpectThat(t.scores, DeepEquals(expectedScores))
}

func (t *ScoreMapSaverTest) CallsWrapped() {
	t.path = "taco"

	// File system
	ExpectCall(t.fileSystem, "Stat")(Any()).
		WillOnce(oglemock.Return(fs.DirectoryEntry{}, nil))

	// Wrapped
	ExpectCall(t.wrapped, "Save")("taco").
		WillOnce(oglemock.Return(nil, errors.New("")))

	// Call
	t.call()
}

func (t *ScoreMapSaverTest) WrappedReturnsError() {
	expectedKey := state.ScoreMapKey{
		Path:        "taco",
		Permissions: 0644,
		Uid:         17,
		Gid:         19,
		MTime:       time.Now(),
		Inode:       23,
		Size:        29,
	}

	// File system
	entry := fs.DirectoryEntry{
		Permissions: 0644,
		Uid:         17,
		Gid:         19,
		MTime:       expectedKey.MTime,
		Inode:       23,
		Size:        29,
	}

	ExpectCall(t.fileSystem, "Stat")(Any()).
		WillOnce(oglemock.Return(entry, nil))

	// Wrapped
	ExpectCall(t.wrapped, "Save")(Any()).
		WillOnce(oglemock.Return(nil, errors.New("taco")))

	// Call
	t.call()

	AssertThat(t.err, Error(Equals("taco")))
	ExpectEq(nil, t.sinkMap.Get(expectedKey))
}

func (t *ScoreMapSaverTest) WrappedReturnsScores() {
	expectedKey := state.ScoreMapKey{
		Path:        "taco",
		Permissions: 0644,
		Uid:         17,
		Gid:         19,
		MTime:       time.Now(),
		Inode:       23,
		Size:        29,
	}

	// File system
	entry := fs.DirectoryEntry{
		Permissions: 0644,
		Uid:         17,
		Gid:         19,
		MTime:       expectedKey.MTime,
		Inode:       23,
		Size:        29,
	}

	ExpectCall(t.fileSystem, "Stat")(Any()).
		WillOnce(oglemock.Return(entry, nil))

	// Wrapped
	expectedScores := []blob.Score{
		blob.ComputeScore([]byte("foo")),
		blob.ComputeScore([]byte("bar")),
	}

	ExpectCall(t.wrapped, "Save")(Any()).
		WillOnce(oglemock.Return(expectedScores, nil))

	// Call
	t.call()

	AssertEq(nil, t.err)

	ExpectThat(t.sinkMap.Get(expectedKey), DeepEquals(expectedScores))
}
<commit_msg>Updated a test.<commit_after>// Copyright 2012 Aaron Jacobs. All Rights Reserved.
// Author: aaronjjacobs@gmail.com (Aaron Jacobs)
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package state_test

import (
	"errors"
	"github.com/jacobsa/comeback/backup"
	"github.com/jacobsa/comeback/backup/mock"
	"github.com/jacobsa/comeback/blob"
	"github.com/jacobsa/comeback/fs"
	"github.com/jacobsa/comeback/fs/mock"
	"github.com/jacobsa/comeback/state"
	. "github.com/jacobsa/oglematchers"
	"github.com/jacobsa/oglemock"
	. "github.com/jacobsa/ogletest"
	"testing"
	"time"
)

func TestScoreMapSaver(t *testing.T) { RunTests(t) }

////////////////////////////////////////////////////////////////////////
// Helpers
////////////////////////////////////////////////////////////////////////

type ScoreMapSaverTest struct {
	sourceMap  state.ScoreMap
	sinkMap    state.ScoreMap
	fileSystem mock_fs.MockFileSystem
	wrapped    mock_backup.MockFileSaver
	saver      backup.FileSaver

	path   string
	scores []blob.Score
	err    error
}

func init() { RegisterTestSuite(&ScoreMapSaverTest{}) }

func (t *ScoreMapSaverTest) SetUp(i *TestInfo) {
	t.sourceMap = state.NewScoreMap()
	t.sinkMap = state.NewScoreMap()
	t.fileSystem = mock_fs.NewMockFileSystem(i.MockController, "fileSystem")
	t.wrapped = mock_backup.NewMockFileSaver(i.MockController, "wrapped")

	t.saver = state.NewScoreMapFileSaver(
		t.sourceMap,
		t.sinkMap,
		t.fileSystem,
		t.wrapped,
	)
}

func (t *ScoreMapSaverTest) call() {
	t.scores, t.err = t.saver.Save(t.path)
}

////////////////////////////////////////////////////////////////////////
// Tests
////////////////////////////////////////////////////////////////////////

func (t *ScoreMapSaverTest) CallsStat() {
	t.path = "taco"

	// File system
	ExpectCall(t.fileSystem, "Stat")("taco").
		WillOnce(oglemock.Return(fs.DirectoryEntry{}, errors.New("")))

	// Call
	t.call()
}

func (t *ScoreMapSaverTest) StatReturnsError() {
	// File system
	ExpectCall(t.fileSystem, "Stat")(Any()).
		WillOnce(oglemock.Return(fs.DirectoryEntry{}, errors.New("taco")))

	// Call
	t.call()

	ExpectThat(t.err, Error(HasSubstr("Stat")))
	ExpectThat(t.err, Error(HasSubstr("taco")))
}

func (t *ScoreMapSaverTest) ScoreMapContainsEntry() {
	t.path = "taco"

	expectedKey := state.ScoreMapKey{
		Path:        "taco",
		Permissions: 0644,
		Uid:         17,
		Gid:         19,
		MTime:       time.Now(),
		Inode:       23,
		Size:        29,
	}

	// Source map
	expectedScores := []blob.Score{
		blob.ComputeScore([]byte("foo")),
		blob.ComputeScore([]byte("bar")),
	}

	t.sourceMap.Set(expectedKey, expectedScores)

	// File system
	entry := fs.DirectoryEntry{
		Permissions: 0644,
		Uid:         17,
		Gid:         19,
		MTime:       expectedKey.MTime,
		Inode:       23,
		Size:        29,
	}

	ExpectCall(t.fileSystem, "Stat")(Any()).
		WillOnce(oglemock.Return(entry, nil))

	// Call
	t.call()

	AssertEq(nil, t.err)
	ExpectThat(t.scores, DeepEquals(expectedScores))
	ExpectThat(t.sinkMap.Get(expectedKey), DeepEquals(expectedScores))
}

func (t *ScoreMapSaverTest) CallsWrapped() {
	t.path = "taco"

	// File system
	ExpectCall(t.fileSystem, "Stat")(Any()).
		WillOnce(oglemock.Return(fs.DirectoryEntry{}, nil))

	// Wrapped
	ExpectCall(t.wrapped, "Save")("taco").
		WillOnce(oglemock.Return(nil, errors.New("")))

	// Call
	t.call()
}

func (t *ScoreMapSaverTest) WrappedReturnsError() {
	expectedKey := state.ScoreMapKey{
		Path:        "taco",
		Permissions: 0644,
		Uid:         17,
		Gid:         19,
		MTime:       time.Now(),
		Inode:       23,
		Size:        29,
	}

	// File system
	entry := fs.DirectoryEntry{
		Permissions: 0644,
		Uid:         17,
		Gid:         19,
		MTime:       expectedKey.MTime,
		Inode:       23,
		Size:        29,
	}

	ExpectCall(t.fileSystem, "Stat")(Any()).
		WillOnce(oglemock.Return(entry, nil))

	// Wrapped
	ExpectCall(t.wrapped, "Save")(Any()).
		WillOnce(oglemock.Return(nil, errors.New("taco")))

	// Call
	t.call()

	AssertThat(t.err, Error(Equals("taco")))
	ExpectEq(nil, t.sinkMap.Get(expectedKey))
}

func (t *ScoreMapSaverTest) WrappedReturnsScores() {
	expectedKey := state.ScoreMapKey{
		Path:        "taco",
		Permissions: 0644,
		Uid:         17,
		Gid:         19,
		MTime:       time.Now(),
		Inode:       23,
		Size:        29,
	}

	// File system
	entry := fs.DirectoryEntry{
		Permissions: 0644,
		Uid:         17,
		Gid:         19,
		MTime:       expectedKey.MTime,
		Inode:       23,
		Size:        29,
	}

	ExpectCall(t.fileSystem, "Stat")(Any()).
		WillOnce(oglemock.Return(entry, nil))

	// Wrapped
	expectedScores := []blob.Score{
		blob.ComputeScore([]byte("foo")),
		blob.ComputeScore([]byte("bar")),
	}

	ExpectCall(t.wrapped, "Save")(Any()).
		WillOnce(oglemock.Return(expectedScores, nil))

	// Call
	t.call()

	AssertEq(nil, t.err)
	ExpectThat(t.sinkMap.Get(expectedKey), DeepEquals(expectedScores))
}
<|endoftext|>


<commit_before>package handler

import (
	"encoding/json"
	"net/http"

	"github.com/gilcrest/go-api-basic/controller/moviectl"
	"github.com/gilcrest/errs"
	"github.com/gorilla/mux"
)

// AddMovie handles POST requests for the /movies endpoint
// and creates a movie in the database
func (ah *AppHandler) AddMovie() http.HandlerFunc {
	return func(w http.ResponseWriter, r *http.Request) {
		const op errs.Op = "handler/AppHandler.AddMovie"

		// Declare rqst as an instance of moviectl.MovieRequest
		rqst := new(moviectl.MovieRequest)
		// Decode JSON HTTP request body into a Decoder type
		// and unmarshal that into rqst
		err := json.NewDecoder(r.Body).Decode(&rqst)
		defer r.Body.Close()
		if err != nil {
			err = errs.RE(http.StatusBadRequest, errs.InvalidRequest, errs.E(op, err))
			errs.HTTPError(w, err)
			return
		}

		// retrieve the context from the http.Request
		ctx := r.Context()

		// Initialize the MovieController
		mc := moviectl.NewMovieController(ah.App, ah.StandardResponseFields)

		// Send the request context and request struct to the controller
		// Receive a response or error in return
		resp, err := mc.Add(ctx, rqst)
		if err != nil {
			err = errs.RE(http.StatusBadRequest, err)
			errs.HTTPError(w, err)
			return
		}

		// Encode response struct to JSON for the response body
		json.NewEncoder(w).Encode(*resp)
		if err != nil {
			err = errs.RE(http.StatusInternalServerError, errs.Internal)
			errs.HTTPError(w, err)
			return
		}
	}
}

// FindByID handles GET requests for the /movies/{id} endpoint
// and finds a movie by it's ID
func (ah *AppHandler) FindByID() http.HandlerFunc {
	return func(w http.ResponseWriter, r *http.Request) {
		const op errs.Op = "handler/AppHandler.FindByID"

		vars := mux.Vars(r)
		id := vars["id"]

		// retrieve the context from the http.Request
		ctx := r.Context()

		// Initialize the MovieController
		mc := moviectl.NewMovieController(ah.App, ah.StandardResponseFields)

		// Send the request context and request struct to the controller
		// Receive a response or error in return
		resp, err := mc.FindByID(ctx, id)
		if err != nil {
			err = errs.RE(http.StatusBadRequest, errs.InvalidRequest, err)
			errs.HTTPError(w, err)
			return
		}

		// Encode response struct to JSON for the response body
		json.NewEncoder(w).Encode(resp)
		if err != nil {
			err = errs.RE(http.StatusInternalServerError, errs.Internal)
			errs.HTTPError(w, err)
			return
		}
	}
}

// FindAll handles GET requests for the /movies endpoint
// and finds all movies
func (ah *AppHandler) FindAll() http.HandlerFunc {
	return func(w http.ResponseWriter, r *http.Request) {
		const op errs.Op = "handler/AppHandler.FindAll"

		// retrieve the context from the http.Request
		ctx := r.Context()

		// Initialize the MovieController
		mc := moviectl.NewMovieController(ah.App, ah.StandardResponseFields)

		// Send the request context and request struct to the controller
		// Receive a response or error in return
		resp, err := mc.FindAll(ctx, r)
		if err != nil {
			err = errs.RE(http.StatusBadRequest, errs.InvalidRequest, err)
			errs.HTTPError(w, err)
			return
		}

		// Encode response struct to JSON for the response body
		json.NewEncoder(w).Encode(resp)
		if err != nil {
			err = errs.RE(http.StatusInternalServerError, errs.Internal)
			errs.HTTPError(w, err)
			return
		}
	}
}

// Update handles PUT requests for the /movies/{id} endpoint
// and updates the given movie
func (ah *AppHandler) Update() http.HandlerFunc {
	return func(w http.ResponseWriter, r *http.Request) {
		const op errs.Op = "handler/AppHandler.Update"

		vars := mux.Vars(r)
		id := vars["id"]

		// Declare rqst as an instance of moviectl.AddMovieRequest
		rqst := new(moviectl.MovieRequest)
		// Decode JSON HTTP request body into a Decoder type
		// and unmarshal that into rqst
		err := json.NewDecoder(r.Body).Decode(&rqst)
		defer r.Body.Close()
		if err != nil {
			err = errs.RE(http.StatusBadRequest, errs.InvalidRequest, errs.E(op, err))
			errs.HTTPError(w, err)
			return
		}

		// retrieve the context from the http.Request
		ctx := r.Context()

		// Initialize the MovieController
		mc := moviectl.NewMovieController(ah.App, ah.StandardResponseFields)

		// Send the request context and request struct to the controller
		// Receive a response or error in return
		resp, err := mc.Update(ctx, id, rqst)
		if err != nil {
			err = errs.RE(http.StatusBadRequest, errs.InvalidRequest, err)
			errs.HTTPError(w, err)
			return
		}

		// Encode response struct to JSON for the response body
		json.NewEncoder(w).Encode(resp)
		if err != nil {
			err = errs.RE(http.StatusInternalServerError, errs.Internal)
			errs.HTTPError(w, err)
			return
		}
	}
}

// Delete handles DELETE requests for the /movies/{id} endpoint
// and updates the given movie
func (ah *AppHandler) Delete() http.HandlerFunc {
	return func(w http.ResponseWriter, r *http.Request) {
		const op errs.Op = "handler/AppHandler.Delete"

		vars := mux.Vars(r)
		id := vars["id"]

		// retrieve the context from the http.Request
		ctx := r.Context()

		// Initialize the MovieController
		mc := moviectl.NewMovieController(ah.App, ah.StandardResponseFields)

		// Send the request context and request struct to the controller
		// Receive a response or error in return
		resp, err := mc.Delete(ctx, id)
		if err != nil {
			err = errs.RE(http.StatusBadRequest, errs.InvalidRequest, err)
			errs.HTTPError(w, err)
			return
		}

		// Encode response struct to JSON for the response body
		json.NewEncoder(w).Encode(resp)
		if err != nil {
			err = errs.RE(http.StatusInternalServerError, errs.Internal)
			errs.HTTPError(w, err)
			return
		}
	}
}
<commit_msg>Add err to json Encode<commit_after>package handler

import (
	"encoding/json"
	"net/http"

	"github.com/gilcrest/errs"
	"github.com/gilcrest/go-api-basic/controller/moviectl"
	"github.com/gorilla/mux"
)

// AddMovie handles POST requests for the /movies endpoint
// and creates a movie in the database
func (ah *AppHandler) AddMovie() http.HandlerFunc {
	return func(w http.ResponseWriter, r *http.Request) {
		const op errs.Op = "handler/AppHandler.AddMovie"

		// Declare rqst as an instance of moviectl.MovieRequest
		rqst := new(moviectl.MovieRequest)
		// Decode JSON HTTP request body into a Decoder type
		// and unmarshal that into rqst
		err := json.NewDecoder(r.Body).Decode(&rqst)
		defer r.Body.Close()
		if err != nil {
			err = errs.RE(http.StatusBadRequest, errs.InvalidRequest, errs.E(op, err))
			errs.HTTPError(w, err)
			return
		}

		// retrieve the context from the http.Request
		ctx := r.Context()

		// Initialize the MovieController
		mc := moviectl.NewMovieController(ah.App, ah.StandardResponseFields)

		// Send the request context and request struct to the controller
		// Receive a response or error in return
		resp, err := mc.Add(ctx, rqst)
		if err != nil {
			err = errs.RE(http.StatusBadRequest, err)
			errs.HTTPError(w, err)
			return
		}

		// Encode response struct to JSON for the response body
		err = json.NewEncoder(w).Encode(*resp)
		if err != nil {
			err = errs.RE(http.StatusInternalServerError, errs.Internal)
			errs.HTTPError(w, err)
			return
		}
	}
}

// FindByID handles GET requests for the /movies/{id} endpoint
// and finds a movie by it's ID
func (ah *AppHandler) FindByID() http.HandlerFunc {
	return func(w http.ResponseWriter, r *http.Request) {
		const op errs.Op = "handler/AppHandler.FindByID"

		vars := mux.Vars(r)
		id := vars["id"]

		// retrieve the context from the http.Request
		ctx := r.Context()

		// Initialize the MovieController
		mc := moviectl.NewMovieController(ah.App, ah.StandardResponseFields)

		// Send the request context and request struct to the controller
		// Receive a response or error in return
		resp, err := mc.FindByID(ctx, id)
		if err != nil {
			err = errs.RE(http.StatusBadRequest, errs.InvalidRequest, err)
			errs.HTTPError(w, err)
			return
		}

		// Encode response struct to JSON for the response body
		json.NewEncoder(w).Encode(resp)
		if err != nil {
			err = errs.RE(http.StatusInternalServerError, errs.Internal)
			errs.HTTPError(w, err)
			return
		}
	}
}

// FindAll handles GET requests for the /movies endpoint
// and finds all movies
func (ah *AppHandler) FindAll() http.HandlerFunc {
	return func(w http.ResponseWriter, r *http.Request) {
		const op errs.Op = "handler/AppHandler.FindAll"

		// retrieve the context from the http.Request
		ctx := r.Context()

		// Initialize the MovieController
		mc := moviectl.NewMovieController(ah.App, ah.StandardResponseFields)

		// Send the request context and request struct to the controller
		// Receive a response or error in return
		resp, err := mc.FindAll(ctx, r)
		if err != nil {
			err = errs.RE(http.StatusBadRequest, errs.InvalidRequest, err)
			errs.HTTPError(w, err)
			return
		}

		// Encode response struct to JSON for the response body
		json.NewEncoder(w).Encode(resp)
		if err != nil {
			err = errs.RE(http.StatusInternalServerError, errs.Internal)
			errs.HTTPError(w, err)
			return
		}
	}
}

// Update handles PUT requests for the /movies/{id} endpoint
// and updates the given movie
func (ah *AppHandler) Update() http.HandlerFunc {
	return func(w http.ResponseWriter, r *http.Request) {
		const op errs.Op = "handler/AppHandler.Update"

		vars := mux.Vars(r)
		id := vars["id"]

		// Declare rqst as an instance of moviectl.AddMovieRequest
		rqst := new(moviectl.MovieRequest)
		// Decode JSON HTTP request body into a Decoder type
		// and unmarshal that into rqst
		err := json.NewDecoder(r.Body).Decode(&rqst)
		defer r.Body.Close()
		if err != nil {
			err = errs.RE(http.StatusBadRequest, errs.InvalidRequest, errs.E(op, err))
			errs.HTTPError(w, err)
			return
		}

		// retrieve the context from the http.Request
		ctx := r.Context()

		// Initialize the MovieController
		mc := moviectl.NewMovieController(ah.App, ah.StandardResponseFields)

		// Send the request context and request struct to the controller
		// Receive a response or error in return
		resp, err := mc.Update(ctx, id, rqst)
		if err != nil {
			err = errs.RE(http.StatusBadRequest, errs.InvalidRequest, err)
			errs.HTTPError(w, err)
			return
		}

		// Encode response struct to JSON for the response body
		json.NewEncoder(w).Encode(resp)
		if err != nil {
			err = errs.RE(http.StatusInternalServerError, errs.Internal)
			errs.HTTPError(w, err)
			return
		}
	}
}

// Delete handles DELETE requests for the /movies/{id} endpoint
// and updates the given movie
func (ah *AppHandler) Delete() http.HandlerFunc {
	return func(w http.ResponseWriter, r *http.Request) {
		const op errs.Op = "handler/AppHandler.Delete"

		vars := mux.Vars(r)
		id := vars["id"]

		// retrieve the context from the http.Request
		ctx := r.Context()

		// Initialize the MovieController
		mc := moviectl.NewMovieController(ah.App, ah.StandardResponseFields)

		// Send the request context and request struct to the controller
		// Receive a response or error in return
		resp, err := mc.Delete(ctx, id)
		if err != nil {
			err = errs.RE(http.StatusBadRequest, errs.InvalidRequest, err)
			errs.HTTPError(w, err)
			return
		}

		// Encode response struct to JSON for the response body
		json.NewEncoder(w).Encode(resp)
		if err != nil {
			err = errs.RE(http.StatusInternalServerError, errs.Internal)
			errs.HTTPError(w, err)
			return
		}
	}
}
<|endoftext|>