<commit_before>// Package archiver makes it super easy to create and open .zip,
// .tar.gz, and .tar.bz2 files.
package archiver

import (
	"archive/zip"
	"fmt"
	"io"
	"os"
	"path"
	"path/filepath"
	"runtime"
	"strings"
)

// Zip creates a .zip file in the location zipPath containing
// the contents of files listed in filePaths. File paths
// can be those of regular files or directories. Regular
// files are stored at the 'root' of the archive, and
// directories are recursively added.
//
// Files with an extension for formats that are already
// compressed will be stored only, not compressed.
func Zip(zipPath string, filePaths []string) error {
	out, err := os.Create(zipPath)
	if err != nil {
		return fmt.Errorf("error creating %s: %v", zipPath, err)
	}
	defer out.Close()

	w := zip.NewWriter(out)
	for _, fpath := range filePaths {
		err = zipFile(w, fpath)
		if err != nil {
			w.Close()
			return err
		}
	}

	return w.Close()
}

func zipFile(w *zip.Writer, source string) error {
	sourceInfo, err := os.Stat(source)
	if err != nil {
		return fmt.Errorf("%s: stat: %v", source, err)
	}

	var baseDir string
	if sourceInfo.IsDir() {
		baseDir = filepath.Base(source)
	}

	return filepath.Walk(source, func(fpath string, info os.FileInfo, err error) error {
		if err != nil {
			return fmt.Errorf("walking to %s: %v", fpath, err)
		}

		header, err := zip.FileInfoHeader(info)
		if err != nil {
			return fmt.Errorf("%s: getting header: %v", fpath, err)
		}

		if baseDir != "" {
			header.Name = filepath.Join(baseDir, strings.TrimPrefix(fpath, source))
		}

		if info.IsDir() {
			header.Name += "/"
			header.Method = zip.Store
		} else {
			ext := strings.ToLower(path.Ext(header.Name))
			if _, ok := CompressedFormats[ext]; ok {
				header.Method = zip.Store
			} else {
				header.Method = zip.Deflate
			}
		}

		writer, err := w.CreateHeader(header)
		if err != nil {
			return fmt.Errorf("%s: making header: %v", fpath, err)
		}

		if info.IsDir() {
			return nil
		}

		if header.Mode().IsRegular() {
			file, err := os.Open(fpath)
			if err != nil {
				return fmt.Errorf("%s: opening: %v", fpath, err)
			}
			defer file.Close()

			_, err = io.CopyN(writer, file, info.Size())
			if err != nil && err != io.EOF {
				return fmt.Errorf("%s: copying contents: %v", fpath, err)
			}
		}

		return nil
	})
}

// Unzip unzips the .zip file at source into destination.
func Unzip(source, destination string) error {
	r, err := zip.OpenReader(source)
	if err != nil {
		return err
	}
	defer r.Close()

	for _, zf := range r.File {
		if err := unzipFile(zf, destination); err != nil {
			return err
		}
	}

	return nil
}

func unzipFile(zf *zip.File, destination string) error {
	if strings.HasSuffix(zf.Name, "/") {
		return mkdir(filepath.Join(destination, zf.Name))
	}

	rc, err := zf.Open()
	if err != nil {
		return fmt.Errorf("%s: open compressed file: %v", zf.Name, err)
	}
	defer rc.Close()

	return writeNewFile(filepath.Join(destination, zf.Name), rc, zf.FileInfo().Mode())
}

func writeNewFile(fpath string, in io.Reader, fm os.FileMode) error {
	err := os.MkdirAll(filepath.Dir(fpath), 0755)
	if err != nil {
		return fmt.Errorf("%s: making directory for file: %v", fpath, err)
	}

	out, err := os.Create(fpath)
	if err != nil {
		return fmt.Errorf("%s: creating new file: %v", fpath, err)
	}
	defer out.Close()

	err = out.Chmod(fm)
	if err != nil && runtime.GOOS != "windows" {
		return fmt.Errorf("%s: changing file mode: %v", fpath, err)
	}

	_, err = io.Copy(out, in)
	if err != nil {
		return fmt.Errorf("%s: writing file: %v", fpath, err)
	}
	return nil
}

func writeNewSymbolicLink(fpath string, target string) error {
	err := os.MkdirAll(filepath.Dir(fpath), 0755)
	if err != nil {
		return fmt.Errorf("%s: making directory for file: %v", fpath, err)
	}

	err = os.Symlink(target, fpath)
	if err != nil {
		return fmt.Errorf("%s: making symbolic link for: %v", fpath, err)
	}

	return nil
}

func mkdir(dirPath string) error {
	err := os.MkdirAll(dirPath, 0755)
	if err != nil {
		return fmt.Errorf("%s: making directory: %v", dirPath, err)
	}
	return nil
}

// CompressedFormats is a (non-exhaustive) set of lowercased
// file extensions for formats that are typically already
// compressed. Compressing already-compressed files often
// results in a larger file, so when possible, we check this
// set to avoid that.
var CompressedFormats = map[string]struct{}{
	".7z":   {},
	".avi":  {},
	".bz2":  {},
	".cab":  {},
	".gif":  {},
	".gz":   {},
	".jar":  {},
	".jpeg": {},
	".jpg":  {},
	".lz":   {},
	".lzma": {},
	".mov":  {},
	".mp3":  {},
	".mp4":  {},
	".mpeg": {},
	".mpg":  {},
	".png":  {},
	".rar":  {},
	".tgz":  {},
	".xz":   {},
	".zip":  {},
	".zipx": {},
}

type (
	// CompressFunc is a function that makes an archive.
	CompressFunc func(string, []string) error

	// DecompressFunc is a function that extracts an archive.
	DecompressFunc func(string, string) error
)
<commit_msg>Use path.Join when setting zip.FileHeader.Name.<commit_after>// Package archiver makes it super easy to create and open .zip,
// .tar.gz, and .tar.bz2 files.
package archiver

import (
	"archive/zip"
	"fmt"
	"io"
	"os"
	"path"
	"path/filepath"
	"runtime"
	"strings"
)

// Zip creates a .zip file in the location zipPath containing
// the contents of files listed in filePaths. File paths
// can be those of regular files or directories. Regular
// files are stored at the 'root' of the archive, and
// directories are recursively added.
//
// Files with an extension for formats that are already
// compressed will be stored only, not compressed.
func Zip(zipPath string, filePaths []string) error {
	out, err := os.Create(zipPath)
	if err != nil {
		return fmt.Errorf("error creating %s: %v", zipPath, err)
	}
	defer out.Close()

	w := zip.NewWriter(out)
	for _, fpath := range filePaths {
		err = zipFile(w, fpath)
		if err != nil {
			w.Close()
			return err
		}
	}

	return w.Close()
}

func zipFile(w *zip.Writer, source string) error {
	sourceInfo, err := os.Stat(source)
	if err != nil {
		return fmt.Errorf("%s: stat: %v", source, err)
	}

	var baseDir string
	if sourceInfo.IsDir() {
		baseDir = filepath.Base(source)
	}

	return filepath.Walk(source, func(fpath string, info os.FileInfo, err error) error {
		if err != nil {
			return fmt.Errorf("walking to %s: %v", fpath, err)
		}

		header, err := zip.FileInfoHeader(info)
		if err != nil {
			return fmt.Errorf("%s: getting header: %v", fpath, err)
		}

		if baseDir != "" {
			header.Name = path.Join(baseDir, strings.TrimPrefix(fpath, source))
		}

		if info.IsDir() {
			header.Name += "/"
			header.Method = zip.Store
		} else {
			ext := strings.ToLower(path.Ext(header.Name))
			if _, ok := CompressedFormats[ext]; ok {
				header.Method = zip.Store
			} else {
				header.Method = zip.Deflate
			}
		}

		writer, err := w.CreateHeader(header)
		if err != nil {
			return fmt.Errorf("%s: making header: %v", fpath, err)
		}

		if info.IsDir() {
			return nil
		}

		if header.Mode().IsRegular() {
			file, err := os.Open(fpath)
			if err != nil {
				return fmt.Errorf("%s: opening: %v", fpath, err)
			}
			defer file.Close()

			_, err = io.CopyN(writer, file, info.Size())
			if err != nil && err != io.EOF {
				return fmt.Errorf("%s: copying contents: %v", fpath, err)
			}
		}

		return nil
	})
}

// Unzip unzips the .zip file at source into destination.
func Unzip(source, destination string) error {
	r, err := zip.OpenReader(source)
	if err != nil {
		return err
	}
	defer r.Close()

	for _, zf := range r.File {
		if err := unzipFile(zf, destination); err != nil {
			return err
		}
	}

	return nil
}

func unzipFile(zf *zip.File, destination string) error {
	if strings.HasSuffix(zf.Name, "/") {
		return mkdir(filepath.Join(destination, zf.Name))
	}

	rc, err := zf.Open()
	if err != nil {
		return fmt.Errorf("%s: open compressed file: %v", zf.Name, err)
	}
	defer rc.Close()

	return writeNewFile(filepath.Join(destination, zf.Name), rc, zf.FileInfo().Mode())
}

func writeNewFile(fpath string, in io.Reader, fm os.FileMode) error {
	err := os.MkdirAll(filepath.Dir(fpath), 0755)
	if err != nil {
		return fmt.Errorf("%s: making directory for file: %v", fpath, err)
	}

	out, err := os.Create(fpath)
	if err != nil {
		return fmt.Errorf("%s: creating new file: %v", fpath, err)
	}
	defer out.Close()

	err = out.Chmod(fm)
	if err != nil && runtime.GOOS != "windows" {
		return fmt.Errorf("%s: changing file mode: %v", fpath, err)
	}

	_, err = io.Copy(out, in)
	if err != nil {
		return fmt.Errorf("%s: writing file: %v", fpath, err)
	}
	return nil
}

func writeNewSymbolicLink(fpath string, target string) error {
	err := os.MkdirAll(filepath.Dir(fpath), 0755)
	if err != nil {
		return fmt.Errorf("%s: making directory for file: %v", fpath, err)
	}

	err = os.Symlink(target, fpath)
	if err != nil {
		return fmt.Errorf("%s: making symbolic link for: %v", fpath, err)
	}

	return nil
}

func mkdir(dirPath string) error {
	err := os.MkdirAll(dirPath, 0755)
	if err != nil {
		return fmt.Errorf("%s: making directory: %v", dirPath, err)
	}
	return nil
}

// CompressedFormats is a (non-exhaustive) set of lowercased
// file extensions for formats that are typically already
// compressed. Compressing already-compressed files often
// results in a larger file, so when possible, we check this
// set to avoid that.
var CompressedFormats = map[string]struct{}{
	".7z":   {},
	".avi":  {},
	".bz2":  {},
	".cab":  {},
	".gif":  {},
	".gz":   {},
	".jar":  {},
	".jpeg": {},
	".jpg":  {},
	".lz":   {},
	".lzma": {},
	".mov":  {},
	".mp3":  {},
	".mp4":  {},
	".mpeg": {},
	".mpg":  {},
	".png":  {},
	".rar":  {},
	".tgz":  {},
	".xz":   {},
	".zip":  {},
	".zipx": {},
}

type (
	// CompressFunc is a function that makes an archive.
	CompressFunc func(string, []string) error

	// DecompressFunc is a function that extracts an archive.
	DecompressFunc func(string, string) error
)
<|endoftext|>


<commit_before>// Package zmq provides ZeroMQ bindings for Go.
package zmq

/*

#cgo LDFLAGS: -lzmq

#include <zmq.h>
#include <stdlib.h>
#include <string.h>

static int my_errno() {
	return errno;
}

*/
import "C"

import (
	"errors"
	"unsafe"
)

var (
	// ErrTerminated is returned when a socket's context has been closed.
	ErrTerminated = errors.New("zmq context has been terminated")
	// ErrTimeout is returned when an operation times out or a non-blocking operation cannot run immediately.
	ErrTimeout     = errors.New("zmq timeout")
	ErrInterrupted = errors.New("system call interrupted")
)

type SocketType int

const (
	Req    SocketType = C.ZMQ_REQ
	Rep               = C.ZMQ_REP
	Dealer            = C.ZMQ_DEALER
	Router            = C.ZMQ_ROUTER
	Pub               = C.ZMQ_PUB
	Sub               = C.ZMQ_SUB
	XPub              = C.ZMQ_XPUB
	XSub              = C.ZMQ_XSUB
	Push              = C.ZMQ_PUSH
	Pull              = C.ZMQ_PULL
	Pair              = C.ZMQ_PAIR
)

type DeviceType int

const (
	Queue     DeviceType = C.ZMQ_QUEUE
	Forwarder            = C.ZMQ_FORWARDER
	Streamer             = C.ZMQ_STREAMER
)

/* Context */

// A Context manages multiple Sockets. Contexts are thread-safe.
type Context struct {
	ctx unsafe.Pointer
}

// Creates a new Context with the given number of dedicated IO threads.
func NewContextThreads(nthreads int) (ctx *Context, err error) {
	ptr := C.zmq_init(C.int(nthreads))
	if ptr == nil {
		return nil, zmqerr()
	}
	return &Context{ptr}, nil
}

// Creates a new Context with the default number of IO threads (one).
func NewContext() (*Context, error) {
	return NewContextThreads(1)
}

// Closes the Context. Close will block until all related Sockets are closed, and all pending messages are either
// physically transferred to the network or the socket's linger period expires.
func (c *Context) Close() {
	for {
		r := C.zmq_term(c.ctx)
		if r == -1 {
			if C.my_errno() == C.EINTR {
				continue
			}
			panic(zmqerr())
		}
		break
	}
}

// Creates a new Socket of the specified type.
func (c *Context) Socket(socktype SocketType) (sock *Socket, err error) {
	ptr := C.zmq_socket(c.ctx, C.int(socktype))
	if ptr == nil {
		return nil, zmqerr()
	}
	sock = &Socket{
		ctx:  c,
		sock: ptr,
	}
	return
}

/* Socket */

// A ZeroMQ Socket.
type Socket struct {
	ctx  *Context
	sock unsafe.Pointer
}

// Closes the socket.
func (s *Socket) Close() {
	C.zmq_close(s.sock)
}

// Binds the socket to the specified local endpoint address.
func (s *Socket) Bind(endpoint string) (err error) {
	cstr := C.CString(endpoint)
	defer C.free(unsafe.Pointer(cstr))
	r := C.zmq_bind(s.sock, cstr)
	if r == -1 {
		err = zmqerr()
	}
	return
}

// Connects the socket to the specified remote endpoint.
func (s *Socket) Connect(endpoint string) (err error) {
	cstr := C.CString(endpoint)
	defer C.free(unsafe.Pointer(cstr))
	r := C.zmq_connect(s.sock, cstr)
	if r == -1 {
		err = zmqerr()
	}
	return
}

// Sends a single message part. The `more` flag is used to specify whether this is the last part of the message (false),
// or if there are more parts to follow (true). SendPart is fairly low-level, and usually Send will be the preferred
// method to use.
func (s *Socket) SendPart(part []byte, more bool) (err error) {
	for {
		err = nil
		var msg C.zmq_msg_t
		toMsg(&msg, part)
		flags := C.int(0)
		if more {
			flags = C.ZMQ_SNDMORE
		}
		r := C.zmq_msg_send(&msg, s.sock, flags)
		if r == -1 {
			err = zmqerr()
		}
		C.zmq_msg_close(&msg)
		if err != ErrInterrupted {
			break
		}
	}
	return
}

// Sends a message containing a number of parts.
func (s *Socket) Send(parts [][]byte) (err error) {
	for _, part := range parts[:len(parts)-1] {
		if err = s.SendPart(part, true); err != nil {
			return
		}
	}
	return s.SendPart(parts[len(parts)-1], false)
}

// Receives a single part along with a boolean flag (more) indicating whether more parts of the same message follow
// (true), or this is the last part of the message (false). As with Send/SendPart, this is fairly low-level and Recv
// should generally be used instead.
func (s *Socket) RecvPart() (part []byte, more bool, err error) {
	var msg C.zmq_msg_t
	C.zmq_msg_init(&msg)
	for {
		err = nil
		r := C.zmq_msg_recv(&msg, s.sock, 0)
		if r == -1 {
			err = zmqerr()
		}
		if err != ErrInterrupted {
			break
		}
	}
	if err != nil {
		C.zmq_msg_close(&msg)
		return
	}
	part = fromMsg(&msg)
	// Check for more parts
	more = (s.getInt(C.ZMQ_RCVMORE) != 0)
	return
}

// Receives a multi-part message.
func (s *Socket) Recv() (parts [][]byte, err error) {
	parts = make([][]byte, 0)
	for more := true; more; {
		var part []byte
		if part, more, err = s.RecvPart(); err != nil {
			return
		}
		parts = append(parts, part)
	}
	return
}

// Subscribe sets up a filter for incoming messages on Sub sockets.
func (s *Socket) Subscribe(filter []byte) {
	s.setBinary(C.ZMQ_SUBSCRIBE, filter)
}

// Unsubscribes from a filter on a Sub socket.
func (s *Socket) Unsubscribe(filter []byte) {
	s.setBinary(C.ZMQ_UNSUBSCRIBE, filter)
}

/* Device */

// Creates and runs a ZeroMQ Device. See zmq_device(3) for more details.
func Device(deviceType DeviceType, frontend, backend *Socket) {
	C.zmq_device(C.int(deviceType), frontend.sock, backend.sock)
}

/* Utilities */

func zmqerr() error {
	eno := C.my_errno()
	switch eno {
	case C.ETERM:
		return ErrTerminated
	case C.EAGAIN:
		return ErrTimeout
	case C.EINTR:
		return ErrInterrupted
	}
	str := C.GoString(C.zmq_strerror(eno))
	return errors.New(str)
}

func toMsg(msg *C.zmq_msg_t, data []byte) {
	C.zmq_msg_init_size(msg, C.size_t(len(data)))
	if len(data) > 0 {
		C.memcpy(C.zmq_msg_data(msg), unsafe.Pointer(&data[0]), C.size_t(len(data)))
	}
}
func fromMsg(msg *C.zmq_msg_t) []byte {
	defer C.zmq_msg_close(msg)
	return C.GoBytes(C.zmq_msg_data(msg), C.int(C.zmq_msg_size(msg)))
}
<commit_msg>Added the global context<commit_after>// Package zmq provides ZeroMQ bindings for Go.
package zmq

/*

#cgo LDFLAGS: -lzmq

#include <zmq.h>
#include <stdlib.h>
#include <string.h>

static int my_errno() {
	return errno;
}

*/
import "C"

import (
	"errors"
	"sync"
	"unsafe"
)

var (
	// ErrTerminated is returned when a socket's context has been closed.
	ErrTerminated = errors.New("zmq context has been terminated")
	// ErrTimeout is returned when an operation times out or a non-blocking operation cannot run immediately.
	ErrTimeout     = errors.New("zmq timeout")
	ErrInterrupted = errors.New("system call interrupted")
)

type SocketType int

const (
	Req    SocketType = C.ZMQ_REQ
	Rep               = C.ZMQ_REP
	Dealer            = C.ZMQ_DEALER
	Router            = C.ZMQ_ROUTER
	Pub               = C.ZMQ_PUB
	Sub               = C.ZMQ_SUB
	XPub              = C.ZMQ_XPUB
	XSub              = C.ZMQ_XSUB
	Push              = C.ZMQ_PUSH
	Pull              = C.ZMQ_PULL
	Pair              = C.ZMQ_PAIR
)

type DeviceType int

const (
	Queue     DeviceType = C.ZMQ_QUEUE
	Forwarder            = C.ZMQ_FORWARDER
	Streamer             = C.ZMQ_STREAMER
)

/* Context */

// A Context manages multiple Sockets. Contexts are thread-safe.
type Context struct {
	ctx unsafe.Pointer
}

// Creates a new Context with the given number of dedicated IO threads.
func NewContextThreads(nthreads int) (ctx *Context, err error) {
	ptr := C.zmq_init(C.int(nthreads))
	if ptr == nil {
		return nil, zmqerr()
	}
	return &Context{ptr}, nil
}

// Creates a new Context with the default number of IO threads (one).
func NewContext() (*Context, error) {
	return NewContextThreads(1)
}

// Closes the Context. Close will block until all related Sockets are closed, and all pending messages are either
// physically transferred to the network or the socket's linger period expires.
func (c *Context) Close() {
	for {
		r := C.zmq_term(c.ctx)
		if r == -1 {
			if C.my_errno() == C.EINTR {
				continue
			}
			panic(zmqerr())
		}
		break
	}
}

// Creates a new Socket of the specified type.
func (c *Context) Socket(socktype SocketType) (sock *Socket, err error) {
	ptr := C.zmq_socket(c.ctx, C.int(socktype))
	if ptr == nil {
		return nil, zmqerr()
	}
	sock = &Socket{
		ctx:  c,
		sock: ptr,
	}
	sock.SetLinger(0)
	return
}

/* Global context */

var (
	globalCtx  *Context = nil
	globalLock sync.Mutex
)

// Returns the default Context. Note that the context will not be created until
// the first call to DefaultContext.
func DefaultContext() *Context {
	globalLock.Lock()
	defer globalLock.Unlock()
	if globalCtx == nil {
		var err error
		if globalCtx, err = NewContext(); err != nil {
			panic(err)
		}
	}
	return globalCtx
}

// Creates a new socket using the default context (see DefaultContext).
func NewSocket(socktype SocketType) (*Socket, error) {
	return DefaultContext().Socket(socktype)
}

/* Socket */

// A ZeroMQ Socket.
type Socket struct {
	ctx  *Context
	sock unsafe.Pointer
}

// Closes the socket.
func (s *Socket) Close() {
	C.zmq_close(s.sock)
}

// Binds the socket to the specified local endpoint address.
func (s *Socket) Bind(endpoint string) (err error) {
	cstr := C.CString(endpoint)
	defer C.free(unsafe.Pointer(cstr))
	r := C.zmq_bind(s.sock, cstr)
	if r == -1 {
		err = zmqerr()
	}
	return
}

// Connects the socket to the specified remote endpoint.
func (s *Socket) Connect(endpoint string) (err error) {
	cstr := C.CString(endpoint)
	defer C.free(unsafe.Pointer(cstr))
	r := C.zmq_connect(s.sock, cstr)
	if r == -1 {
		err = zmqerr()
	}
	return
}

// Sends a single message part. The `more` flag is used to specify whether this is the last part of the message (false),
// or if there are more parts to follow (true). SendPart is fairly low-level, and usually Send will be the preferred
// method to use.
func (s *Socket) SendPart(part []byte, more bool) (err error) {
	for {
		err = nil
		var msg C.zmq_msg_t
		toMsg(&msg, part)
		flags := C.int(0)
		if more {
			flags = C.ZMQ_SNDMORE
		}
		r := C.zmq_msg_send(&msg, s.sock, flags)
		if r == -1 {
			err = zmqerr()
		}
		C.zmq_msg_close(&msg)
		if err != ErrInterrupted {
			break
		}
	}
	return
}

// Sends a message containing a number of parts.
func (s *Socket) Send(parts [][]byte) (err error) {
	for _, part := range parts[:len(parts)-1] {
		if err = s.SendPart(part, true); err != nil {
			return
		}
	}
	return s.SendPart(parts[len(parts)-1], false)
}

// Receives a single part along with a boolean flag (more) indicating whether more parts of the same message follow
// (true), or this is the last part of the message (false). As with Send/SendPart, this is fairly low-level and Recv
// should generally be used instead.
func (s *Socket) RecvPart() (part []byte, more bool, err error) {
	var msg C.zmq_msg_t
	C.zmq_msg_init(&msg)
	for {
		err = nil
		r := C.zmq_msg_recv(&msg, s.sock, 0)
		if r == -1 {
			err = zmqerr()
		}
		if err != ErrInterrupted {
			break
		}
	}
	if err != nil {
		C.zmq_msg_close(&msg)
		return
	}
	part = fromMsg(&msg)
	// Check for more parts
	more = (s.getInt(C.ZMQ_RCVMORE) != 0)
	return
}

// Receives a multi-part message.
func (s *Socket) Recv() (parts [][]byte, err error) {
	parts = make([][]byte, 0)
	for more := true; more; {
		var part []byte
		if part, more, err = s.RecvPart(); err != nil {
			return
		}
		parts = append(parts, part)
	}
	return
}

// Subscribe sets up a filter for incoming messages on Sub sockets.
func (s *Socket) Subscribe(filter []byte) {
	s.setBinary(C.ZMQ_SUBSCRIBE, filter)
}

// Unsubscribes from a filter on a Sub socket.
func (s *Socket) Unsubscribe(filter []byte) {
	s.setBinary(C.ZMQ_UNSUBSCRIBE, filter)
}

/* Device */

// Creates and runs a ZeroMQ Device. See zmq_device(3) for more details.
func Device(deviceType DeviceType, frontend, backend *Socket) {
	C.zmq_device(C.int(deviceType), frontend.sock, backend.sock)
}

/* Utilities */

func zmqerr() error {
	eno := C.my_errno()
	switch eno {
	case C.ETERM:
		return ErrTerminated
	case C.EAGAIN:
		return ErrTimeout
	case C.EINTR:
		return ErrInterrupted
	}
	str := C.GoString(C.zmq_strerror(eno))
	return errors.New(str)
}

func toMsg(msg *C.zmq_msg_t, data []byte) {
	C.zmq_msg_init_size(msg, C.size_t(len(data)))
	if len(data) > 0 {
		C.memcpy(C.zmq_msg_data(msg), unsafe.Pointer(&data[0]), C.size_t(len(data)))
	}
}
func fromMsg(msg *C.zmq_msg_t) []byte {
	defer C.zmq_msg_close(msg)
	return C.GoBytes(C.zmq_msg_data(msg), C.int(C.zmq_msg_size(msg)))
}
<|endoftext|>


<commit_before>/* Note on file organization:  tests are invoked in sequence by Go test
 * harness and that fact is used here to affirm a set of assumptions as
 * we test the constructs.  As assumptions are affirmed, they are noted
 * in comments below.  So, do not change the order of the tests.
 */

package contextual

import (
	"fmt"
	"testing"
)

// ============================================================================
// testing: contextual.context
// ============================================================================

// NOP - just feedback for test runs. std. per each construct
func TestContextStructStart_NOP(t *testing.T) {
	fmt.Println("contextual.context")
}

// helper
type emptyStruct struct{}

// helper
func mixedTypeValueSet() []interface{} {
	num := 10
	str := emptyStruct{}
	ptr := &emptyStruct{}
	ch := make(chan emptyStruct)
	fn := func() {} // func() is "uncomparable type" oh, well.
	pfn := &fn
	txt := "hello there"

	return []interface{}{
		num, str, ptr, ch /*fn,*/, pfn, txt,
	}

}

// helper
func genericUniqueIndexNames(n int) []string {
	var names []string = make([]string, n)
	var sanitycheck = make(map[string]int)

	for i := 0; i < n; i++ {
		names[i] = fmt.Sprintf("value[%d]", i)
		sanitycheck[names[i]] = i
	}
	// make sure names are indeed unique
	if len(sanitycheck) != len(names) {
		panic("genericUniqueIndexNames")
	}

	return names
}

/* --- CHECK a1 ---------------------------------------------------------------
 *  a1: correct construction and intial state
 *
 * tests for correct construction and initialization of contexts, and asserts
 * on input params of the associated functions.
 * - Context#Size()    // on init
 * - Context#IsEmpty() // on init
 *
 * assumptions:
 * - none
 */

func TestNewContext(t *testing.T) {
	var ctx Context = NewContext()
	if ctx == nil {
		t.Fatalf("NewContext returned nil ref")
	}

	fmt.Println("\tcreate root context")
}

func TestNewContextChild(t *testing.T) {
	rootCtx := NewContext()

	ctx, e := ChildContext(rootCtx)
	if e != nil {
		t.Fatalf("Unexpected error: %s", e)
	}
	if ctx == nil {
		t.Fatalf("ChildContext returned nil ref")
	}
	ctx, e = ChildContext(nil)
	if e == nil {
		t.Fatalf("Expecting error on ChildContext(nil)")
	}

	fmt.Println("\tcreate child context")
}

func TestNewContextInit(t *testing.T) {
	ctx := NewContext()
	if ctx.IsEmpty() != true {
		t.Fatalf("New Context#IsEmpty returned false")
	}
	if ctx.Size() != 0 {
		t.Fatalf("New Context#Size returned non-zero")
	}

	fmt.Println("\ttest context init - root")
}

func TestNewContextChildInit(t *testing.T) {
	rootCtx := NewContext()

	ctx, _ := ChildContext(rootCtx)
	if ctx.IsEmpty() != true {
		t.Fatalf("New child Context#IsEmpty returned false")
	}
	if ctx.Size() != 0 {
		t.Fatalf("New child Context#Size returned non-zero")
	}

	fmt.Println("\ttest context init - child")
}

/* --- CONFIRMED a1 ----------------------------------------------------------*/

/* --- CHECK a2 ---------------------------------------------------------------
 * - a2: correct parent/child order and IsRoot
 *
 * tests for parent child relationship and correct behavior for
 * Context#IsRoot()
 *
 * assumptions:
 * - a1
 */

// tests contextual.context's compliance with Context#IsRoot()
func TestIsRoot(t *testing.T) {
	// a1 - don't bother with error/nil checks
	rootCtx := NewContext()
	if rootCtx.IsRoot() != true {
		t.Fatalf("IsRoot() for a root context must return true")
	}
	if ctx, _ := ChildContext(rootCtx); ctx.IsRoot() {
		t.Fatalf("IsRoot() for a child context must return false")
	}

	fmt.Println("\tContext#IsRoot()")
}

/* --- CONFIRMED a2 ----------------------------------------------------------*/

/* --- CHECK a3 ---------------------------------------------------------------
 * - a3: correct basic ops for a single (root) context
 *
 * tests for correct behavior of the following Context methods:
 * - Context#Bind()
 * - Context#Lookup()
 * - Context#Size()    // post init
 * - Context#IsEmpty() // post init
 * - Context#Unbind()
 * - Context#Rebind()
 * assumptions:
 * - a1
 * - a2
 */

// Confirm spec compliance for faults
// {Lookup, LookupN, Bind, Unbind}
func TestContextSpecdError(t *testing.T) {
	// setup
	ctx := NewContext()

	// Lookup()

	// test specified errors for Lookup
	//  NilNameError <= zero-value names are not allowed
	if _, e := ctx.Lookup(""); e == nil {
		t.Fatalf("Lookup(nil) expected error: %s", NilNameError)
	}
	v, e := ctx.Lookup("no-such-binding")
	if e != nil {
		t.Fatalf("Unexpected error: %s", e)
	}
	if v != nil {
		t.Fatalf("Lookup(\"\") - expected:%v got:%v", nil, v)
	}

	// LookupN()

	//  NilNameError <= nil names are not allowed
	//  NegativeNArgError <= n is negative
	if _, e := ctx.LookupN("", 0); e == nil {
		t.Fatalf("Lookup(nil) expected error: %s", NilNameError)
	}
	v, e = ctx.LookupN("no-such-binding", 0)
	if e != nil {
		t.Fatalf("Unexpected error: %s", e)
	}
	if v != nil {
		t.Fatalf("Lookup(\"\") - expected:%v got:%v", nil, v)
	}

	// Bind()

	// test specified errors for Bind
	//  NilNameError <= zero-value names are not allowed
	//  NilValueError <= nil values are not allowed
	//  AlreadyBoundError <= a value is already bound to the name
	if e := ctx.Bind("", "some value"); e == nil {
		t.Fatalf("Bind(\"\") expected error: %s", NilNameError)
	}
	if e := ctx.Bind("some key", nil); e == nil {
		t.Fatalf("Bind(\"\") expected error: %s", NilValueError)
	}

	// Unbind()

	// test specified errors for Unbind
	//  NilNameError <= zero-value names are not allowed
	//  NoSuchBindingError <= no values are bound to the name
	wat, e := ctx.Unbind("")
	if e == nil {
		t.Fatalf("Unbind(\"\") expected error: %s", NilNameError)
	}
	if wat != nil {
		t.Fatalf("Unexpected value on faulted return: %s", wat)
	}
	wat, e = ctx.Unbind("some key")
	if e == nil {
		t.Fatalf("Unbind(\"\") expected error: %s", NoSuchBindingError)
	}
	if wat != nil {
		t.Fatalf("Unexpected value on faulted return: %s", wat)
	}

	// Rebind()

	// test specified errors for Rebind
	//  NoSuchBinding <= no values were bound to the name
	//  NilNameError <= zero-value names are not allowed
	//  NilValueError <= nil values are not allowed
	wat, e = ctx.Rebind("", "some value")
	if e == nil {
		t.Fatalf("Rebind(\"\", v) expected error: %s", NilNameError)
	}
	if wat != nil {
		t.Fatalf("Unexpected value on faulted return: %s", wat)
	}
	wat, e = ctx.Rebind("some key", "some value")
	if e == nil {
		t.Fatalf("Rebind(\"\", v) expected error: %s", NoSuchBindingError)
	}
	if wat != nil {
		t.Fatalf("Unexpected value on faulted return: %s", wat)
	}
	if wat, e = ctx.Rebind("doesn't matter", nil); e == nil {
		t.Fatalf("Rebind (nil) expected error: %s", NilValueError)
	}
}

func TestSingleContext(t *testing.T) {
	// setup
	ctx := NewContext()
	values := mixedTypeValueSet()
	names := genericUniqueIndexNames(len(values))

	// Bind()

	for i, name := range names {
		value := values[i]
		if e := ctx.Bind(name, value); e != nil {
			t.Fatalf("Unexpected error: %s", e)
		}
	}

	// Lookup()

	for i, name := range names {
		expv := values[i]
		v, e := ctx.Lookup(name)
		if e != nil {
			t.Fatalf("Unexpected error: %s", e)
		}
		if v != expv {
			t.Fatalf("Lookup(%s) - expected:%s got:%s", name, expv, v)
		}
	}

	// IsEmpty()

	if b := ctx.IsEmpty(); b {
		t.Fatalf("IsEmpty() - expected:%s got:%s", false, b)
	}

	// Size()

	if n := ctx.Size(); n != len(names) {
		t.Fatalf("Size() - expected:%s got:%s", len(names), n)
	}

	// Unbind()

	for i, name := range names {
		expv := values[i]
		v, e := ctx.Unbind(name)
		if e != nil {
			t.Fatalf("Unexpected error: %s", e)
		}
		if v != expv {
			t.Fatalf("Unbind(%s) - expected:%s got:%s", name, expv, v)
		}
	}
	// confirm Size and IsEmpty
	if !ctx.IsEmpty() {
		t.Fatalf("IsEmpty() returned true")
	}
	if ctx.Size() != 0 {
		t.Fatalf("Size() returned non-zero")
	}

	// Rebind()
	ctx.Bind(names[0], values[0]) // setup
	oldv, e := ctx.Rebind(names[0], "on the rebound")
	if e != nil {
		t.Fatalf("unexpected error: %s", e)
	}
	if oldv != values[0] {
		t.Fatalf("Rebind() old value - expected:%v got:%v", values[0], oldv)
	}

	fmt.Println("\tContext compliance - root")
}

/* --- CONFIRMED a3 ----------------------------------------------------------*/

/* --- CHECK a4 ---------------------------------------------------------------
 * - a4: correct basic ops for context hierarchy
 *
 * tests for correct behavior of the following Context methods:
 * - Context#Bind()
 * - Context#Lookup()
 * - Context#Size()    // post init
 * - Context#IsEmpty() // post init
 * - Context#Unbind()
 * - Context#Rebind()
 * assumptions:
 * - a1
 * - a2
 * - a3
 */

func TestContextHierarchy(t *testing.T) {
	// setup
	//	ctx := NewContext()
	//	child1, _ := ChildContext(ctx)
	//	child1_1, _ := ChildContext(child1)
	//	child2, _ := ChildContext(ctx)
	//
	//	values := mixedTypeValueSet()
	//	names := genericUniqueIndexNames(len(values))
}

/* --- CONFIRMED a4 ----------------------------------------------------------*/
<commit_msg>WIP - tests<commit_after>/* Note on file organization:  tests are invoked in sequence by Go test
 * harness and that fact is used here to affirm a set of assumptions as
 * we test the constructs.  As assumptions are affirmed, they are noted
 * in comments below.  So, do not change the order of the tests.
 */

package contextual

import (
	"fmt"
	"testing"
)

// ============================================================================
// testing: contextual.context
// ============================================================================

// NOP - just feedback for test runs. std. per each construct
func TestContextStructStart_NOP(t *testing.T) {
	fmt.Println("contextual.context")
}

// helper
type emptyStruct struct{}

// helper
func mixedTypeValueSet() []interface{} {
	num := 10
	str := emptyStruct{}
	ptr := &emptyStruct{}
	ch := make(chan emptyStruct)
	fn := func() {} // func() is "uncomparable type" oh, well.
	pfn := &fn
	txt := "hello there"

	return []interface{}{
		num, str, ptr, ch /*fn,*/, pfn, txt,
	}

}

// helper
func genericUniqueIndexNames(n int) []string {
	var names []string = make([]string, n)
	var sanitycheck = make(map[string]int)

	for i := 0; i < n; i++ {
		names[i] = fmt.Sprintf("value[%d]", i)
		sanitycheck[names[i]] = i
	}
	// make sure names are indeed unique
	if len(sanitycheck) != len(names) {
		panic("genericUniqueIndexNames")
	}

	return names
}

/* --- CHECK a1 ---------------------------------------------------------------
 *  a1: correct construction and intial state
 *
 * tests for correct construction and initialization of contexts, and asserts
 * on input params of the associated functions.
 * - Context#Size()    // on init
 * - Context#IsEmpty() // on init
 *
 * assumptions:
 * - none
 */

func TestNewContext(t *testing.T) {
	var ctx Context = NewContext()
	if ctx == nil {
		t.Fatalf("NewContext returned nil ref")
	}

	fmt.Println("\tcreate root context")
}

func TestNewContextChild(t *testing.T) {
	rootCtx := NewContext()

	ctx, e := ChildContext(rootCtx)
	if e != nil {
		t.Fatalf("Unexpected error: %s", e)
	}
	if ctx == nil {
		t.Fatalf("ChildContext returned nil ref")
	}
	ctx, e = ChildContext(nil)
	if e == nil {
		t.Fatalf("Expecting error on ChildContext(nil)")
	}

	fmt.Println("\tcreate child context")
}

func TestNewContextInit(t *testing.T) {
	ctx := NewContext()
	if ctx.IsEmpty() != true {
		t.Fatalf("New Context#IsEmpty returned false")
	}
	if ctx.Size() != 0 {
		t.Fatalf("New Context#Size returned non-zero")
	}

	fmt.Println("\ttest context init - root")
}

func TestNewContextChildInit(t *testing.T) {
	rootCtx := NewContext()

	ctx, _ := ChildContext(rootCtx)
	if ctx.IsEmpty() != true {
		t.Fatalf("New child Context#IsEmpty returned false")
	}
	if ctx.Size() != 0 {
		t.Fatalf("New child Context#Size returned non-zero")
	}

	fmt.Println("\ttest context init - child")
}

/* --- CONFIRMED a1 ----------------------------------------------------------*/

/* --- CHECK a2 ---------------------------------------------------------------
 * - a2: correct parent/child order and IsRoot
 *
 * tests for parent child relationship and correct behavior for
 * Context#IsRoot()
 *
 * assumptions:
 * - a1
 */

// tests contextual.context's compliance with Context#IsRoot()
func TestIsRoot(t *testing.T) {
	// a1 - don't bother with error/nil checks
	rootCtx := NewContext()
	if rootCtx.IsRoot() != true {
		t.Fatalf("IsRoot() for a root context must return true")
	}
	if ctx, _ := ChildContext(rootCtx); ctx.IsRoot() {
		t.Fatalf("IsRoot() for a child context must return false")
	}

	fmt.Println("\tContext#IsRoot()")
}

/* --- CONFIRMED a2 ----------------------------------------------------------*/

/* --- CHECK a3 ---------------------------------------------------------------
 * - a3: correct basic ops for a single (root) context
 *
 * tests for correct behavior of the following Context methods:
 * - Context#Bind()
 * - Context#Lookup()
 * - Context#Size()    // post init
 * - Context#IsEmpty() // post init
 * - Context#Unbind()
 * - Context#Rebind()
 * assumptions:
 * - a1
 * - a2
 */

// Confirm spec compliance for faults
// {Lookup, LookupN, Bind, Unbind}
func TestContextSpecdError(t *testing.T) {
	// setup
	ctx := NewContext()

	// Lookup()

	// test specified errors for Lookup
	//  NilNameError <= zero-value names are not allowed
	if _, e := ctx.Lookup(""); e == nil {
		t.Fatalf("Lookup(nil) expected error: %s", NilNameError)
	}
	v, e := ctx.Lookup("no-such-binding")
	if e != nil {
		t.Fatalf("Unexpected error: %s", e)
	}
	if v != nil {
		t.Fatalf("Lookup(\"\") - expected:%v got:%v", nil, v)
	}

	// LookupN()

	//  NilNameError <= nil names are not allowed
	//  NegativeNArgError <= n is negative
	if _, e := ctx.LookupN("", 0); e == nil {
		t.Fatalf("Lookup(nil) expected error: %s", NilNameError)
	}
	v, e = ctx.LookupN("no-such-binding", 0)
	if e != nil {
		t.Fatalf("Unexpected error: %s", e)
	}
	if v != nil {
		t.Fatalf("Lookup(\"\") - expected:%v got:%v", nil, v)
	}

	// Bind()

	// test specified errors for Bind
	//  NilNameError <= zero-value names are not allowed
	//  NilValueError <= nil values are not allowed
	//  AlreadyBoundError <= a value is already bound to the name
	if e := ctx.Bind("", "some value"); e == nil {
		t.Fatalf("Bind(\"\") expected error: %s", NilNameError)
	}
	if e := ctx.Bind("some key", nil); e == nil {
		t.Fatalf("Bind(\"\") expected error: %s", NilValueError)
	}

	// Unbind()

	// test specified errors for Unbind
	//  NilNameError <= zero-value names are not allowed
	//  NoSuchBindingError <= no values are bound to the name
	wat, e := ctx.Unbind("")
	if e == nil {
		t.Fatalf("Unbind(\"\") expected error: %s", NilNameError)
	}
	if wat != nil {
		t.Fatalf("Unexpected value on faulted return: %s", wat)
	}
	wat, e = ctx.Unbind("some key")
	if e == nil {
		t.Fatalf("Unbind(\"\") expected error: %s", NoSuchBindingError)
	}
	if wat != nil {
		t.Fatalf("Unexpected value on faulted return: %s", wat)
	}

	// Rebind()

	// test specified errors for Rebind
	//  NoSuchBinding <= no values were bound to the name
	//  NilNameError <= zero-value names are not allowed
	//  NilValueError <= nil values are not allowed
	wat, e = ctx.Rebind("", "some value")
	if e == nil {
		t.Fatalf("Rebind(\"\", v) expected error: %s", NilNameError)
	}
	if wat != nil {
		t.Fatalf("Unexpected value on faulted return: %s", wat)
	}
	wat, e = ctx.Rebind("some key", "some value")
	if e == nil {
		t.Fatalf("Rebind(\"\", v) expected error: %s", NoSuchBindingError)
	}
	if wat != nil {
		t.Fatalf("Unexpected value on faulted return: %s", wat)
	}
	if wat, e = ctx.Rebind("doesn't matter", nil); e == nil {
		t.Fatalf("Rebind (nil) expected error: %s", NilValueError)
	}
}

func TestSingleContext(t *testing.T) {
	// setup
	ctx := NewContext()
	values := mixedTypeValueSet()
	names := genericUniqueIndexNames(len(values))

	// Bind()

	for i, name := range names {
		value := values[i]
		if e := ctx.Bind(name, value); e != nil {
			t.Fatalf("Unexpected error: %s", e)
		}
	}

	// Lookup()

	for i, name := range names {
		expv := values[i]
		v, e := ctx.Lookup(name)
		if e != nil {
			t.Fatalf("Unexpected error: %s", e)
		}
		if v != expv {
			t.Fatalf("Lookup(%s) - expected:%s got:%s", name, expv, v)
		}
	}

	// IsEmpty()

	if b := ctx.IsEmpty(); b {
		t.Fatalf("IsEmpty() - expected:%s got:%s", false, b)
	}

	// Size()

	if n := ctx.Size(); n != len(names) {
		t.Fatalf("Size() - expected:%s got:%s", len(names), n)
	}

	// Unbind()

	for i, name := range names {
		expv := values[i]
		v, e := ctx.Unbind(name)
		if e != nil {
			t.Fatalf("Unexpected error: %s", e)
		}
		if v != expv {
			t.Fatalf("Unbind(%s) - expected:%s got:%s", name, expv, v)
		}
	}
	// confirm Size and IsEmpty
	if !ctx.IsEmpty() {
		t.Fatalf("IsEmpty() returned true")
	}
	if ctx.Size() != 0 {
		t.Fatalf("Size() returned non-zero")
	}

	// Rebind()
	ctx.Bind(names[0], values[0]) // setup
	oldv, e := ctx.Rebind(names[0], "on the rebound")
	if e != nil {
		t.Fatalf("unexpected error: %s", e)
	}
	if oldv != values[0] {
		t.Fatalf("Rebind() old value - expected:%v got:%v", values[0], oldv)
	}

	fmt.Println("\tContext compliance - root")
}

/* --- CONFIRMED a3 ----------------------------------------------------------*/

/* --- CHECK a4 ---------------------------------------------------------------
 * - a4: correct basic ops for context hierarchy
 *
 * tests for correct behavior of the following Context methods:
 * - Context#Bind()
 * - Context#Lookup()
 * - Context#LookupN()
 * - Context#Size()    // post init
 * - Context#IsEmpty() // post init
 * - Context#Unbind()
 * - Context#Rebind()
 * assumptions:
 * - a1
 * - a2
 * - a3
 */

func TestContextHierarchy(t *testing.T) {
	//	setup
	cr := NewContext()
	c1, _ := ChildContext(cr)
	c2, _ := ChildContext(cr)
	c1_1, _ := ChildContext(c1)
	c1_2, _ := ChildContext(c1)
	c2_1, _ := ChildContext(c2)
	c1_1_1, _ := ChildContext(c1_1)

	children := []Context{c1, c2, c1_1, c1_2, c2_1, c1_1_1}
	childrenL1 := []Context{c1, c2}
	childrenL2 := []Context{c1_1, c1_2, c2_1}
	childrenL3 := []Context{c1_1_1}
	levels := [][]Context{childrenL1, childrenL2, childrenL3}

	values := mixedTypeValueSet()
	names := genericUniqueIndexNames(len(values))

	// Bind, Lookup, LookupN

	cr.Bind(names[0], values[0]) // setup - a single binding in root

	// basic lookup, IsEmpty, and Size for children
	// all should see binding in root
	// size for all is 1, non-empty
	for i, ctx := range children {
		v, e := ctx.Lookup(names[0])
		if e != nil {
			t.Fatalf("Unexpected error: %s", e)
		}
		if v != values[0] {
			t.Fatalf("for children[%d] - Lookup(%s) - expected:%v got:%v", i, names[0], v)
		}
		if ctx.IsEmpty() {
			t.Fatalf("for children[%d] - IsEmpty(%s) - expected:false", i)
		}
		if s := ctx.Size(); s != 1 {
			t.Fatalf("for children[%d] - IsEmpty(%s) - expected:%d, got:%d", i, 1, s)
		}
	}

	// limited lookup with N step up
	// N = 0 : none
	// N = 1 : L1
	// N = 2 : L1, L2
	// N = 3 : L1, L2, L3
	var n int
	for l := 0; l < len(levels); l++ {
		for i, ctx := range levels[l] {
			// all should see
			v, e := ctx.LookupN(names[0], l+1)
			if e != nil {
				t.Fatalf("Unexpected error: %s", e)
			}
			if v != values[0] {
				t.Fatalf("l[%d] i[%d] n[%d]- Lookup(%s) - expected:%v got:%v", l, i, n, names[0], v)
			}
			// none should see
			v, _ = ctx.LookupN(names[0], l)
			if v != nil {
				t.Fatalf("l[%d] i[%d] n[%d]- Lookup(%s) - expected:%v got:%v", l, i, n, nil, v)
			}
		}
	}
}

/* --- CONFIRMED a4 ----------------------------------------------------------*/
<|endoftext|>


<commit_before>/*
Copyright 2019 The Kubernetes Authors.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package git

import (
	"io/ioutil"
	"os"
	"path"
	"runtime"
	"sync"

	"github.com/sirupsen/logrus"
	utilpointer "k8s.io/utils/pointer"
)

// ClientFactory knows how to create clientFactory for repos
type ClientFactory interface {
	// ClientFromDir creates a client that operates on a repo that has already
	// been cloned to the given directory.
	ClientFromDir(org, repo, dir string) (RepoClient, error)
	// ClientFor creates a client that operates on a new clone of the repo.
	ClientFor(org, repo string) (RepoClient, error)

	// Clean removes the caches used to generate clients
	Clean() error
}

// RepoClient exposes interactions with a git repo
type RepoClient interface {
	Publisher
	Interactor
}

type repoClient struct {
	publisher
	interactor
}

type ClientFactoryOpts struct {
	// Host, defaults to "github.com" if unset
	Host string
	// UseSSH, defaults to false
	UseSSH *bool
	// The directory in which the cache should be
	// created. Defaults to the "/var/tmp" on
	// Linux and os.TempDir otherwise
	CacheDirBase *string
	// If unset, publishing action will error
	Username LoginGetter
	// If unset, publishing action will error
	Token TokenGetter
	// The git user to use.
	GitUser GitUserGetter
	// The censor to use. Not needed for anonymous
	// actions.
	Censor Censor
}

// Apply allows to use a ClientFactoryOpts as Opt
func (cfo *ClientFactoryOpts) Apply(target *ClientFactoryOpts) {
	if cfo.Host != "" {
		target.Host = cfo.Host
	}
	if cfo.UseSSH != nil {
		target.UseSSH = cfo.UseSSH
	}
	if cfo.CacheDirBase != nil {
		target.CacheDirBase = cfo.CacheDirBase
	}
	if cfo.Token != nil {
		target.Token = cfo.Token
	}
	if cfo.GitUser != nil {
		target.Token = cfo.Token
	}
	if cfo.Censor != nil {
		target.Censor = cfo.Censor
	}
}

// ClientFactoryOpts allows to manipulate the options for a ClientFactory
type ClientFactoryOpt func(*ClientFactoryOpts)

func defaultClientFactoryOpts(cfo *ClientFactoryOpts) {
	if cfo.Host == "" {
		cfo.Host = "github.com"
	}
	if cfo.CacheDirBase == nil {
		switch runtime.GOOS {
		case "linux":
			cfo.CacheDirBase = utilpointer.StringPtr("/var/tmp")
		default:
			cfo.CacheDirBase = utilpointer.StringPtr("")
		}
	}
	if cfo.Censor == nil {
		cfo.Censor = func(in []byte) []byte { return in }
	}
}

// NewClientFactory allows for the creation of repository clients. It uses github.com
// without authentication by default.
func NewClientFactory(opts ...ClientFactoryOpt) (ClientFactory, error) {
	o := ClientFactoryOpts{}
	defaultClientFactoryOpts(&o)
	for _, opt := range opts {
		opt(&o)
	}

	cacheDir, err := ioutil.TempDir(*o.CacheDirBase, "gitcache")
	if err != nil {
		return nil, err
	}
	var remotes RemoteResolverFactory
	if o.UseSSH != nil && *o.UseSSH {
		remotes = &sshRemoteResolverFactory{
			host:     o.Host,
			username: o.Username,
		}
	} else {
		remotes = &httpResolverFactory{
			host:     o.Host,
			username: o.Username,
			token:    o.Token,
		}
	}
	return &clientFactory{
		cacheDir:     cacheDir,
		cacheDirBase: *o.CacheDirBase,
		remotes:      remotes,
		gitUser:      o.GitUser,
		censor:       o.Censor,
		masterLock:   &sync.Mutex{},
		repoLocks:    map[string]*sync.Mutex{},
		logger:       logrus.WithField("client", "git"),
	}, nil
}

// NewLocalClientFactory allows for the creation of repository clients
// based on a local filepath remote for testing
func NewLocalClientFactory(baseDir string, gitUser GitUserGetter, censor Censor) (ClientFactory, error) {
	cacheDir, err := ioutil.TempDir("", "gitcache")
	if err != nil {
		return nil, err
	}
	return &clientFactory{
		cacheDir:   cacheDir,
		remotes:    &pathResolverFactory{baseDir: baseDir},
		gitUser:    gitUser,
		censor:     censor,
		masterLock: &sync.Mutex{},
		repoLocks:  map[string]*sync.Mutex{},
		logger:     logrus.WithField("client", "git"),
	}, nil
}

type clientFactory struct {
	remotes RemoteResolverFactory
	gitUser GitUserGetter
	censor  Censor
	logger  *logrus.Entry

	// cacheDir is the root under which cached clones of repos are created
	cacheDir string
	// cacheDirBase is the basedir under which create tempdirs
	cacheDirBase string
	// masterLock guards mutations to the repoLocks records
	masterLock *sync.Mutex
	// repoLocks guard mutating access to subdirectories under the cacheDir
	repoLocks map[string]*sync.Mutex
}

// bootstrapClients returns a repository client and cloner for a dir.
func (c *clientFactory) bootstrapClients(org, repo, dir string) (cacher, cloner, RepoClient, error) {
	if dir == "" {
		workdir, err := os.Getwd()
		if err != nil {
			return nil, nil, nil, err
		}
		dir = workdir
	}
	logger := c.logger.WithFields(logrus.Fields{"org": org, "repo": repo})
	logger.WithField("dir", dir).Debug("Creating a pre-initialized client.")
	executor, err := NewCensoringExecutor(dir, c.censor, logger)
	if err != nil {
		return nil, nil, nil, err
	}
	client := &repoClient{
		publisher: publisher{
			remote:   c.remotes.PublishRemote(org, repo),
			executor: executor,
			info:     c.gitUser,
		},
		interactor: interactor{
			dir:      dir,
			remote:   c.remotes.CentralRemote(org, repo),
			executor: executor,
			logger:   logger,
		},
	}
	return client, client, client, nil
}

// ClientFromDir returns a repository client for a directory that's already initialized with content.
// If the directory isn't specified, the current working directory is used.
func (c *clientFactory) ClientFromDir(org, repo, dir string) (RepoClient, error) {
	_, _, client, err := c.bootstrapClients(org, repo, dir)
	return client, err
}

// ClientFor returns a repository client for the specified repository.
// This function may take a long time if it is the first time cloning the repo.
// In that case, it must do a full git mirror clone. For large repos, this can
// take a while. Once that is done, it will do a git fetch instead of a clone,
// which will usually take at most a few seconds.
func (c *clientFactory) ClientFor(org, repo string) (RepoClient, error) {
	cacheDir := path.Join(c.cacheDir, org, repo)
	c.logger.WithFields(logrus.Fields{"org": org, "repo": repo, "dir": cacheDir}).Debug("Creating a client from the cache.")
	cacheClientCacher, _, _, err := c.bootstrapClients(org, repo, cacheDir)
	if err != nil {
		return nil, err
	}

	repoDir, err := ioutil.TempDir(c.cacheDirBase, "gitrepo")
	if err != nil {
		return nil, err
	}
	_, repoClientCloner, repoClient, err := c.bootstrapClients(org, repo, repoDir)
	if err != nil {
		return nil, err
	}
	c.masterLock.Lock()
	if _, exists := c.repoLocks[cacheDir]; !exists {
		c.repoLocks[cacheDir] = &sync.Mutex{}
	}
	c.masterLock.Unlock()
	c.repoLocks[cacheDir].Lock()
	defer c.repoLocks[cacheDir].Unlock()
	if _, err := os.Stat(cacheDir); os.IsNotExist(err) {
		// we have not yet cloned this repo, we need to do a full clone
		if err := os.MkdirAll(cacheDir, os.ModePerm); err != nil && !os.IsExist(err) {
			return nil, err
		}
		if err := cacheClientCacher.MirrorClone(); err != nil {
			return nil, err
		}
	} else if err != nil {
		// something unexpected happened
		return nil, err
	} else {
		// we have cloned the repo previously, but will refresh it
		if err := cacheClientCacher.RemoteUpdate(); err != nil {
			return nil, err
		}
	}

	// initialize the new derivative repo from the cache
	if err := repoClientCloner.Clone(cacheDir); err != nil {
		return nil, err
	}

	return repoClient, nil
}

// Clean removes the caches used to generate clients
func (c *clientFactory) Clean() error {
	return os.RemoveAll(c.cacheDir)
}
<commit_msg>GitV2: Be able to recover from a failled mirror clone<commit_after>/*
Copyright 2019 The Kubernetes Authors.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package git

import (
	"io/ioutil"
	"os"
	"path"
	"runtime"
	"sync"

	"github.com/sirupsen/logrus"
	utilpointer "k8s.io/utils/pointer"
)

// ClientFactory knows how to create clientFactory for repos
type ClientFactory interface {
	// ClientFromDir creates a client that operates on a repo that has already
	// been cloned to the given directory.
	ClientFromDir(org, repo, dir string) (RepoClient, error)
	// ClientFor creates a client that operates on a new clone of the repo.
	ClientFor(org, repo string) (RepoClient, error)

	// Clean removes the caches used to generate clients
	Clean() error
}

// RepoClient exposes interactions with a git repo
type RepoClient interface {
	Publisher
	Interactor
}

type repoClient struct {
	publisher
	interactor
}

type ClientFactoryOpts struct {
	// Host, defaults to "github.com" if unset
	Host string
	// UseSSH, defaults to false
	UseSSH *bool
	// The directory in which the cache should be
	// created. Defaults to the "/var/tmp" on
	// Linux and os.TempDir otherwise
	CacheDirBase *string
	// If unset, publishing action will error
	Username LoginGetter
	// If unset, publishing action will error
	Token TokenGetter
	// The git user to use.
	GitUser GitUserGetter
	// The censor to use. Not needed for anonymous
	// actions.
	Censor Censor
}

// Apply allows to use a ClientFactoryOpts as Opt
func (cfo *ClientFactoryOpts) Apply(target *ClientFactoryOpts) {
	if cfo.Host != "" {
		target.Host = cfo.Host
	}
	if cfo.UseSSH != nil {
		target.UseSSH = cfo.UseSSH
	}
	if cfo.CacheDirBase != nil {
		target.CacheDirBase = cfo.CacheDirBase
	}
	if cfo.Token != nil {
		target.Token = cfo.Token
	}
	if cfo.GitUser != nil {
		target.Token = cfo.Token
	}
	if cfo.Censor != nil {
		target.Censor = cfo.Censor
	}
}

// ClientFactoryOpts allows to manipulate the options for a ClientFactory
type ClientFactoryOpt func(*ClientFactoryOpts)

func defaultClientFactoryOpts(cfo *ClientFactoryOpts) {
	if cfo.Host == "" {
		cfo.Host = "github.com"
	}
	if cfo.CacheDirBase == nil {
		switch runtime.GOOS {
		case "linux":
			cfo.CacheDirBase = utilpointer.StringPtr("/var/tmp")
		default:
			cfo.CacheDirBase = utilpointer.StringPtr("")
		}
	}
	if cfo.Censor == nil {
		cfo.Censor = func(in []byte) []byte { return in }
	}
}

// NewClientFactory allows for the creation of repository clients. It uses github.com
// without authentication by default.
func NewClientFactory(opts ...ClientFactoryOpt) (ClientFactory, error) {
	o := ClientFactoryOpts{}
	defaultClientFactoryOpts(&o)
	for _, opt := range opts {
		opt(&o)
	}

	cacheDir, err := ioutil.TempDir(*o.CacheDirBase, "gitcache")
	if err != nil {
		return nil, err
	}
	var remotes RemoteResolverFactory
	if o.UseSSH != nil && *o.UseSSH {
		remotes = &sshRemoteResolverFactory{
			host:     o.Host,
			username: o.Username,
		}
	} else {
		remotes = &httpResolverFactory{
			host:     o.Host,
			username: o.Username,
			token:    o.Token,
		}
	}
	return &clientFactory{
		cacheDir:     cacheDir,
		cacheDirBase: *o.CacheDirBase,
		remotes:      remotes,
		gitUser:      o.GitUser,
		censor:       o.Censor,
		masterLock:   &sync.Mutex{},
		repoLocks:    map[string]*sync.Mutex{},
		logger:       logrus.WithField("client", "git"),
	}, nil
}

// NewLocalClientFactory allows for the creation of repository clients
// based on a local filepath remote for testing
func NewLocalClientFactory(baseDir string, gitUser GitUserGetter, censor Censor) (ClientFactory, error) {
	cacheDir, err := ioutil.TempDir("", "gitcache")
	if err != nil {
		return nil, err
	}
	return &clientFactory{
		cacheDir:   cacheDir,
		remotes:    &pathResolverFactory{baseDir: baseDir},
		gitUser:    gitUser,
		censor:     censor,
		masterLock: &sync.Mutex{},
		repoLocks:  map[string]*sync.Mutex{},
		logger:     logrus.WithField("client", "git"),
	}, nil
}

type clientFactory struct {
	remotes RemoteResolverFactory
	gitUser GitUserGetter
	censor  Censor
	logger  *logrus.Entry

	// cacheDir is the root under which cached clones of repos are created
	cacheDir string
	// cacheDirBase is the basedir under which create tempdirs
	cacheDirBase string
	// masterLock guards mutations to the repoLocks records
	masterLock *sync.Mutex
	// repoLocks guard mutating access to subdirectories under the cacheDir
	repoLocks map[string]*sync.Mutex
}

// bootstrapClients returns a repository client and cloner for a dir.
func (c *clientFactory) bootstrapClients(org, repo, dir string) (cacher, cloner, RepoClient, error) {
	if dir == "" {
		workdir, err := os.Getwd()
		if err != nil {
			return nil, nil, nil, err
		}
		dir = workdir
	}
	logger := c.logger.WithFields(logrus.Fields{"org": org, "repo": repo})
	logger.WithField("dir", dir).Debug("Creating a pre-initialized client.")
	executor, err := NewCensoringExecutor(dir, c.censor, logger)
	if err != nil {
		return nil, nil, nil, err
	}
	client := &repoClient{
		publisher: publisher{
			remote:   c.remotes.PublishRemote(org, repo),
			executor: executor,
			info:     c.gitUser,
		},
		interactor: interactor{
			dir:      dir,
			remote:   c.remotes.CentralRemote(org, repo),
			executor: executor,
			logger:   logger,
		},
	}
	return client, client, client, nil
}

// ClientFromDir returns a repository client for a directory that's already initialized with content.
// If the directory isn't specified, the current working directory is used.
func (c *clientFactory) ClientFromDir(org, repo, dir string) (RepoClient, error) {
	_, _, client, err := c.bootstrapClients(org, repo, dir)
	return client, err
}

// ClientFor returns a repository client for the specified repository.
// This function may take a long time if it is the first time cloning the repo.
// In that case, it must do a full git mirror clone. For large repos, this can
// take a while. Once that is done, it will do a git fetch instead of a clone,
// which will usually take at most a few seconds.
func (c *clientFactory) ClientFor(org, repo string) (RepoClient, error) {
	cacheDir := path.Join(c.cacheDir, org, repo)
	c.logger.WithFields(logrus.Fields{"org": org, "repo": repo, "dir": cacheDir}).Debug("Creating a client from the cache.")
	cacheClientCacher, _, _, err := c.bootstrapClients(org, repo, cacheDir)
	if err != nil {
		return nil, err
	}

	repoDir, err := ioutil.TempDir(c.cacheDirBase, "gitrepo")
	if err != nil {
		return nil, err
	}
	_, repoClientCloner, repoClient, err := c.bootstrapClients(org, repo, repoDir)
	if err != nil {
		return nil, err
	}
	c.masterLock.Lock()
	if _, exists := c.repoLocks[cacheDir]; !exists {
		c.repoLocks[cacheDir] = &sync.Mutex{}
	}
	c.masterLock.Unlock()
	c.repoLocks[cacheDir].Lock()
	defer c.repoLocks[cacheDir].Unlock()
	if _, err := os.Stat(path.Join(cacheDir, "HEAD")); os.IsNotExist(err) {
		// we have not yet cloned this repo, we need to do a full clone
		if err := os.MkdirAll(cacheDir, os.ModePerm); err != nil && !os.IsExist(err) {
			return nil, err
		}
		if err := cacheClientCacher.MirrorClone(); err != nil {
			return nil, err
		}
	} else if err != nil {
		// something unexpected happened
		return nil, err
	} else {
		// we have cloned the repo previously, but will refresh it
		if err := cacheClientCacher.RemoteUpdate(); err != nil {
			return nil, err
		}
	}

	// initialize the new derivative repo from the cache
	if err := repoClientCloner.Clone(cacheDir); err != nil {
		return nil, err
	}

	return repoClient, nil
}

// Clean removes the caches used to generate clients
func (c *clientFactory) Clean() error {
	return os.RemoveAll(c.cacheDir)
}
<|endoftext|>


<commit_before>package bitsmanager

import (
	"crypto/tls"
	"github.com/orange-cloudfoundry/terraform-provider-cloudfoundry/common"
	"net/http"
	"time"
)

type HttpHandler struct {
	SkipInsecureSSL bool
}

func (h HttpHandler) GetZipFile(path string) (FileHandler, error) {
	client := h.makeHttpClient()
	cleanFunc := func() error {
		return nil
	}
	resp, err := client.Get(path)
	if err != nil {
		return FileHandler{}, err
	}
	return FileHandler{
		ZipFile: resp.Body,
		Size:    resp.ContentLength,
		Clean:   cleanFunc,
	}, nil
}
func (h HttpHandler) Detect(path string) bool {
	return common.IsWebURL(path) && IsZipFile(path)
}
func (h HttpHandler) makeHttpClient() *http.Client {
	tr := &http.Transport{
		Proxy:           http.ProxyFromEnvironment,
		TLSClientConfig: &tls.Config{InsecureSkipVerify: h.SkipInsecureSSL},
	}
	return &http.Client{
		Transport: tr,
		Timeout:   2 * time.Second,
	}
}
func (h HttpHandler) GetSha1File(path string) (string, error) {
	client := h.makeHttpClient()
	req, err := http.NewRequest("GET", path, nil)
	if err != nil {
		return "", err
	}
	resp, err := client.Do(req)
	if err != nil {
		return "", err
	}
	defer resp.Body.Close()
	return GetSha1FromReader(resp.Body)
}
<commit_msg>fix http handler for bits manager, timeout was set to 2 seconds...<commit_after>package bitsmanager

import (
	"crypto/tls"
	"github.com/orange-cloudfoundry/terraform-provider-cloudfoundry/common"
	"net/http"
)

type HttpHandler struct {
	SkipInsecureSSL bool
}

func (h HttpHandler) GetZipFile(path string) (FileHandler, error) {
	client := h.makeHttpClient()
	cleanFunc := func() error {
		return nil
	}
	resp, err := client.Get(path)
	if err != nil {
		return FileHandler{}, err
	}
	return FileHandler{
		ZipFile: resp.Body,
		Size:    resp.ContentLength,
		Clean:   cleanFunc,
	}, nil
}
func (h HttpHandler) Detect(path string) bool {
	return common.IsWebURL(path) && IsZipFile(path)
}
func (h HttpHandler) makeHttpClient() *http.Client {
	tr := &http.Transport{
		Proxy:           http.ProxyFromEnvironment,
		TLSClientConfig: &tls.Config{InsecureSkipVerify: h.SkipInsecureSSL},
	}
	return &http.Client{
		Transport: tr,
		Timeout:   0,
	}
}
func (h HttpHandler) GetSha1File(path string) (string, error) {
	client := h.makeHttpClient()
	req, err := http.NewRequest("GET", path, nil)
	if err != nil {
		return "", err
	}
	resp, err := client.Do(req)
	if err != nil {
		return "", err
	}
	defer resp.Body.Close()
	return GetSha1FromReader(resp.Body)
}
<|endoftext|>


<commit_before>package core

// transfer2go implementation of Central Catalog
// Author: Valentin Kuznetsov <vkuznet@gmail.com>

import (
	"fmt"
	"io/ioutil"
	"os"
	"strconv"
	"time"

	log "github.com/sirupsen/logrus"
)

// CC reprents isntance of CentralCatalog
var CC CentralCatalog

// CentralCatalog represent structure of Central Catalog
// it is represneted by list of tables where each table contains list of records
type CentralCatalog struct {
	Path string `json:"path"` // path to central catalog
}

func findLatestSnapshot(path, table string) string {
	var year, month, day, unix int
	files, _ := ioutil.ReadDir(path)
	for _, f := range files {
		v, _ := strconv.Atoi(f.Name())
		if v > year {
			year = v
		}
	}
	files, _ = ioutil.ReadDir(fmt.Sprintf("%s/%d", path, year))
	for _, f := range files {
		v, _ := strconv.Atoi(f.Name())
		if v > month {
			month = v
		}
	}
	files, _ = ioutil.ReadDir(fmt.Sprintf("%s/%d/%d", path, year, month))
	for _, f := range files {
		v, _ := strconv.Atoi(f.Name())
		if v > day {
			day = v
		}
	}
	files, _ = ioutil.ReadDir(fmt.Sprintf("%s/%d/%d/%d", path, year, month, day))
	for _, f := range files {
		v, _ := strconv.Atoi(f.Name())
		if v > unix {
			unix = v
		}
	}
	return fmt.Sprintf("%s/%d/%d/%d/%d/%s", path, year, month, day, unix, table)
}

// Get method gets records from Central Catalog for a given table
func (c *CentralCatalog) Get(table string) ([]byte, error) {
	if c.Path == "" {
		return []byte{}, nil
	}

	fname := findLatestSnapshot(c.Path, table)
	data, err := ioutil.ReadFile(fname)
	if err != nil {
		log.WithFields(log.Fields{
			"File": fname,
			"Err":  err,
		}).Error("CentralCatalog unable to read table")
		return []byte{}, err
	}
	return data, nil
}

// Put method puts given table-records into Central Catalog
func (c *CentralCatalog) Put(table string, records []string) error {
	if c.Path == "" {
		return nil
	}
	t := time.Now()
	path := fmt.Sprintf("%s/%d/%d/%d/%d", c.Path, t.Year(), t.Month(), t.Day(), t.Unix())
	err := os.MkdirAll(path, os.ModePerm)
	if err != nil {
		log.WithFields(log.Fields{
			"Dir": path,
			"Err": err,
		}).Error("CentralCatalog unable to create directory")
		return err
	}
	fname := fmt.Sprintf("%s/%s", path, table)
	file, err := os.Create(fname)
	defer file.Close()
	if err != nil {
		log.WithFields(log.Fields{
			"File": fname,
			"Err":  err,
		}).Error("CentralCatalog unable to create file")
		return err
	}
	for _, v := range records {
		_, err := file.WriteString(v + "\n")
		if err != nil {
			log.WithFields(log.Fields{
				"File":   fname,
				"Record": v,
				"Err":    err,
			}).Error("CentralCatalog unable to write record")
			return err
		}
	}
	return nil
}
<commit_msg>Change log to logs<commit_after>package core

// transfer2go implementation of Central Catalog
// Author: Valentin Kuznetsov <vkuznet@gmail.com>

import (
	"fmt"
	"io/ioutil"
	"os"
	"strconv"
	"time"

	logs "github.com/sirupsen/logrus"
)

// CC reprents isntance of CentralCatalog
var CC CentralCatalog

// CentralCatalog represent structure of Central Catalog
// it is represneted by list of tables where each table contains list of records
type CentralCatalog struct {
	Path string `json:"path"` // path to central catalog
}

func findLatestSnapshot(path, table string) string {
	var year, month, day, unix int
	files, _ := ioutil.ReadDir(path)
	for _, f := range files {
		v, _ := strconv.Atoi(f.Name())
		if v > year {
			year = v
		}
	}
	files, _ = ioutil.ReadDir(fmt.Sprintf("%s/%d", path, year))
	for _, f := range files {
		v, _ := strconv.Atoi(f.Name())
		if v > month {
			month = v
		}
	}
	files, _ = ioutil.ReadDir(fmt.Sprintf("%s/%d/%d", path, year, month))
	for _, f := range files {
		v, _ := strconv.Atoi(f.Name())
		if v > day {
			day = v
		}
	}
	files, _ = ioutil.ReadDir(fmt.Sprintf("%s/%d/%d/%d", path, year, month, day))
	for _, f := range files {
		v, _ := strconv.Atoi(f.Name())
		if v > unix {
			unix = v
		}
	}
	return fmt.Sprintf("%s/%d/%d/%d/%d/%s", path, year, month, day, unix, table)
}

// Get method gets records from Central Catalog for a given table
func (c *CentralCatalog) Get(table string) ([]byte, error) {
	if c.Path == "" {
		return []byte{}, nil
	}

	fname := findLatestSnapshot(c.Path, table)
	data, err := ioutil.ReadFile(fname)
	if err != nil {
		logs.WithFields(logs.Fields{
			"File": fname,
			"Err":  err,
		}).Error("CentralCatalog unable to read table")
		return []byte{}, err
	}
	return data, nil
}

// Put method puts given table-records into Central Catalog
func (c *CentralCatalog) Put(table string, records []string) error {
	if c.Path == "" {
		return nil
	}
	t := time.Now()
	path := fmt.Sprintf("%s/%d/%d/%d/%d", c.Path, t.Year(), t.Month(), t.Day(), t.Unix())
	err := os.MkdirAll(path, os.ModePerm)
	if err != nil {
		logs.WithFields(logs.Fields{
			"Dir": path,
			"Err": err,
		}).Error("CentralCatalog unable to create directory")
		return err
	}
	fname := fmt.Sprintf("%s/%s", path, table)
	file, err := os.Create(fname)
	defer file.Close()
	if err != nil {
		logs.WithFields(logs.Fields{
			"File": fname,
			"Err":  err,
		}).Error("CentralCatalog unable to create file")
		return err
	}
	for _, v := range records {
		_, err := file.WriteString(v + "\n")
		if err != nil {
			logs.WithFields(logs.Fields{
				"File":   fname,
				"Record": v,
				"Err":    err,
			}).Error("CentralCatalog unable to write record")
			return err
		}
	}
	return nil
}
<|endoftext|>


<commit_before>package main

import (
	"fmt"
	"github.com/bobziuchkovski/writ"
	"github.com/jkomoros/boardgame/boardgame-util/lib/build/api"
	"github.com/jkomoros/boardgame/boardgame-util/lib/build/static"
	"github.com/jkomoros/boardgame/boardgame-util/lib/config"
	"os"
	"os/exec"
	"strings"
	"time"
)

type Serve struct {
	baseSubCommand

	Storage string

	Port       string
	StaticPort string
	Prod       bool

	OfflineDevMode bool
}

func (s *Serve) Run(p writ.Path, positional []string) {

	c := s.Base().GetConfig(false)

	if s.OfflineDevMode {
		c.AddOverride(config.EnableOfflineDevMode())
	}

	mode := c.Dev

	if s.Prod {
		mode = c.Prod
	}

	dir := s.Base().NewTempDir("temp_serve_")

	storage := effectiveStorageType(s.Base(), mode, s.Storage)

	pkgs, err := mode.AllGamePackages()

	if err != nil {
		s.Base().errAndQuit("Not all game packages were valid: " + err.Error())
	}

	apiOptions := &api.Options{}

	if s.OfflineDevMode {
		apiOptions.OverrideOfflineDevMode = true
	}

	fmt.Println("Creating temporary binary")
	apiPath, err := api.Build(dir, pkgs, storage, apiOptions)

	if err != nil {
		s.Base().errAndQuit("Couldn't create api: " + err.Error())
	}

	fmt.Println("Creating temporary static assets folder")
	//TODO: should we allow you to pass CopyFiles? I don't know why you'd want
	//to given this is a temp dir.
	_, err = static.Build(dir, pkgs, c.Client(s.Prod), s.Prod, false, mode.OfflineDevMode)

	if err != nil {
		s.Base().errAndQuit("Couldn't create static directory: " + err.Error())
	}

	staticPort := mode.DefaultStaticPort

	if s.StaticPort != "" {
		staticPort = s.StaticPort
	}

	go func() {
		fmt.Println("Starting up asset server at " + staticPort)
		if err := static.Server(dir, staticPort); err != nil {
			//TODO: when this happens we should quit the whole program
			fmt.Println("ERROR: couldn't start static server: " + err.Error())
		}
	}()

	//TODO: simple serving of staticPath here. Do we need a new parameter for
	//default static serving port?

	port := mode.DefaultPort

	if s.Port != "" {
		port = s.Port
	}

	//cmd will be run as though it's in this directory, which is where
	//config.json is.
	cmd := exec.Command(apiPath)
	cmd.Stderr = os.Stderr
	cmd.Stdout = os.Stdout
	cmd.Env = append(os.Environ(), "PORT="+port)

	err = cmd.Start()

	if err == nil {
		topLine := "************************************************************************"

		//Cheat and wait to print the message until later
		time.Sleep(time.Second * 2)

		fmt.Println(" ")
		fmt.Println(topLine)
		for i := 0; i < 2; i++ {
			fmt.Println("*")
		}
		fmt.Println("*     Server running. Open 'http://localhost:" + staticPort + "' in your browser")
		for i := 0; i < 2; i++ {
			fmt.Println("*")
		}
		fmt.Println(topLine)
		fmt.Println(" ")

		err = cmd.Wait()
	}

	if err != nil {
		exitErr, ok := err.(*exec.ExitError)
		if !ok {
			s.Base().errAndQuit("Couldn't cast exiterror")
		}

		//Programs that are signaled and who responded to it before us (the
		//parent) did (which is a race) will have Exited() false, whereas a
		//program that errored and quit on its own should have true. Only the
		//latter is an err; calling errAndQuit not in an error could prevent
		//our own clean shutdown from happening.
		if exitErr.ProcessState.Exited() {
			s.Base().errAndQuit("Error running command: " + err.Error())
		}
	}
}

func (s *Serve) Name() string {
	return "serve"
}

func (s *Serve) Description() string {
	return "Creates and runs a local development server based on config.json"
}

func (s *Serve) WritOptions() []*writ.Option {
	return []*writ.Option{
		{
			Names:       []string{"storage", "s"},
			Decoder:     writ.NewOptionDecoder(&s.Storage),
			Description: "Which storage subsystem to use. One of {" + strings.Join(api.ValidStorageTypeStrings(), ",") + "}. If not provided, falls back on the DefaultStorageType from config, or as a final fallback just the deafult storage type.",
		},
		{
			Names:       []string{"port", "p"},
			Decoder:     writ.NewOptionDecoder(&s.Port),
			Description: "Port to use for the api server, overriding value in config.json's DefaultPort",
		},
		{
			Names:       []string{"static-port"},
			Decoder:     writ.NewOptionDecoder(&s.StaticPort),
			Description: "Port to use for the static file server, overridig value in config.json's DefaultStaticPort",
		},
		{
			Names:       []string{"prod"},
			Description: "If provided, will created bundled build directory for static resources.",
			Decoder:     writ.NewFlagDecoder(&s.Prod),
			Flag:        true,
		},
		{
			Names:       []string{"offline-dev-mode"},
			Description: "If provided, will override OfflineDevMode to true, no matter what is in config. The effect of this is that the webapp won't make any calls to anything but localhost, allowing development on for example a plane. This is generally the best way to enable offline dev mode.",
			Decoder:     writ.NewFlagDecoder(&s.OfflineDevMode),
			Flag:        true,
		},
	}
}
<commit_msg>Refactor cmd_serve so that we can use it for golden. Part of #648. Part of #667.<commit_after>package main

import (
	"fmt"
	"github.com/bobziuchkovski/writ"
	"github.com/jkomoros/boardgame/boardgame-util/lib/build/api"
	"github.com/jkomoros/boardgame/boardgame-util/lib/build/static"
	"github.com/jkomoros/boardgame/boardgame-util/lib/config"
	"github.com/jkomoros/boardgame/boardgame-util/lib/gamepkg"
	"os"
	"os/exec"
	"strings"
	"time"
)

type Serve struct {
	baseSubCommand

	Storage string

	Port       string
	StaticPort string
	Prod       bool

	OfflineDevMode bool
}

func doServe(base *BoardgameUtil, pkgs []*gamepkg.Pkg, offlineDevMode bool, prod bool, storageArg string, staticPort string, port string) {

	c := base.GetConfig(false)

	if offlineDevMode {
		c.AddOverride(config.EnableOfflineDevMode())
	}

	mode := c.Dev

	if prod {
		mode = c.Prod
	}

	storage := effectiveStorageType(base, mode, storageArg)

	dir := base.NewTempDir("temp_serve_")

	if pkgs == nil {
		var err error
		pkgs, err = mode.AllGamePackages()

		if err != nil {
			base.errAndQuit("Not all game packages were valid: " + err.Error())
		}
	}

	apiOptions := &api.Options{}

	if offlineDevMode {
		apiOptions.OverrideOfflineDevMode = true
	}

	fmt.Println("Creating temporary binary")
	apiPath, err := api.Build(dir, pkgs, storage, apiOptions)

	if err != nil {
		base.errAndQuit("Couldn't create api: " + err.Error())
	}

	fmt.Println("Creating temporary static assets folder")
	//TODO: should we allow you to pass CopyFiles? I don't know why you'd want
	//to given this is a temp dir.
	_, err = static.Build(dir, pkgs, c.Client(prod), prod, false, mode.OfflineDevMode)

	if err != nil {
		base.errAndQuit("Couldn't create static directory: " + err.Error())
	}

	if staticPort == "" {
		staticPort = mode.DefaultStaticPort
	}

	go func() {
		fmt.Println("Starting up asset server at " + staticPort)
		if err := static.Server(dir, staticPort); err != nil {
			//TODO: when this happens we should quit the whole program
			fmt.Println("ERROR: couldn't start static server: " + err.Error())
		}
	}()

	//TODO: simple serving of staticPath here. Do we need a new parameter for
	//default static serving port?

	if port == "" {
		port = mode.DefaultPort
	}

	//cmd will be run as though it's in this directory, which is where
	//config.json is.
	cmd := exec.Command(apiPath)
	cmd.Stderr = os.Stderr
	cmd.Stdout = os.Stdout
	cmd.Env = append(os.Environ(), "PORT="+port)

	err = cmd.Start()

	if err == nil {
		topLine := "************************************************************************"

		//Cheat and wait to print the message until later
		time.Sleep(time.Second * 2)

		fmt.Println(" ")
		fmt.Println(topLine)
		for i := 0; i < 2; i++ {
			fmt.Println("*")
		}
		fmt.Println("*     Server running. Open 'http://localhost:" + staticPort + "' in your browser")
		for i := 0; i < 2; i++ {
			fmt.Println("*")
		}
		fmt.Println(topLine)
		fmt.Println(" ")

		err = cmd.Wait()
	}

	if err != nil {
		exitErr, ok := err.(*exec.ExitError)
		if !ok {
			base.errAndQuit("Couldn't cast exiterror")
		}

		//Programs that are signaled and who responded to it before us (the
		//parent) did (which is a race) will have Exited() false, whereas a
		//program that errored and quit on its own should have true. Only the
		//latter is an err; calling errAndQuit not in an error could prevent
		//our own clean shutdown from happening.
		if exitErr.ProcessState.Exited() {
			base.errAndQuit("Error running command: " + err.Error())
		}
	}
}

func (s *Serve) Run(p writ.Path, positional []string) {
	//Pass nil options ot use mode.Games
	doServe(s.Base(), nil, s.OfflineDevMode, s.Prod, s.Storage, s.StaticPort, s.Port)
}

func (s *Serve) Name() string {
	return "serve"
}

func (s *Serve) Description() string {
	return "Creates and runs a local development server based on config.json"
}

func (s *Serve) WritOptions() []*writ.Option {
	return []*writ.Option{
		{
			Names:       []string{"storage", "s"},
			Decoder:     writ.NewOptionDecoder(&s.Storage),
			Description: "Which storage subsystem to use. One of {" + strings.Join(api.ValidStorageTypeStrings(), ",") + "}. If not provided, falls back on the DefaultStorageType from config, or as a final fallback just the deafult storage type.",
		},
		{
			Names:       []string{"port", "p"},
			Decoder:     writ.NewOptionDecoder(&s.Port),
			Description: "Port to use for the api server, overriding value in config.json's DefaultPort",
		},
		{
			Names:       []string{"static-port"},
			Decoder:     writ.NewOptionDecoder(&s.StaticPort),
			Description: "Port to use for the static file server, overridig value in config.json's DefaultStaticPort",
		},
		{
			Names:       []string{"prod"},
			Description: "If provided, will created bundled build directory for static resources.",
			Decoder:     writ.NewFlagDecoder(&s.Prod),
			Flag:        true,
		},
		{
			Names:       []string{"offline-dev-mode"},
			Description: "If provided, will override OfflineDevMode to true, no matter what is in config. The effect of this is that the webapp won't make any calls to anything but localhost, allowing development on for example a plane. This is generally the best way to enable offline dev mode.",
			Decoder:     writ.NewFlagDecoder(&s.OfflineDevMode),
			Flag:        true,
		},
	}
}
<|endoftext|>


<commit_before>/*
Copyright 2018 The Kubernetes Authors.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package runtimeobjects

import (
	"fmt"
	"strconv"

	appsv1 "k8s.io/api/apps/v1"
	batch "k8s.io/api/batch/v1"
	corev1 "k8s.io/api/core/v1"
	"k8s.io/apimachinery/pkg/api/equality"
	"k8s.io/apimachinery/pkg/api/meta"
	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
	"k8s.io/apimachinery/pkg/apis/meta/v1/unstructured"
	"k8s.io/apimachinery/pkg/labels"
	"k8s.io/apimachinery/pkg/runtime"
	clientset "k8s.io/client-go/kubernetes"
	"k8s.io/perf-tests/clusterloader2/pkg/framework/client"
)

// ListRuntimeObjectsForKind returns objects of given kind that satisfy given namespace, labelSelector and fieldSelector.
// TODO: using dynamic interface rather than clientset interface
func ListRuntimeObjectsForKind(c clientset.Interface, kind, namespace, labelSelector, fieldSelector string) ([]runtime.Object, error) {
	var runtimeObjectsList []runtime.Object
	var listFunc func() error
	listOpts := metav1.ListOptions{
		LabelSelector: labelSelector,
		FieldSelector: fieldSelector,
	}
	switch kind {
	case "ReplicationController":
		listFunc = func() error {
			list, err := c.CoreV1().ReplicationControllers(namespace).List(listOpts)
			if err != nil {
				return err
			}
			runtimeObjectsList = make([]runtime.Object, len(list.Items))
			for i := range list.Items {
				runtimeObjectsList[i] = &list.Items[i]
			}
			return nil
		}
	case "ReplicaSet":
		listFunc = func() error {
			list, err := c.AppsV1().ReplicaSets(namespace).List(listOpts)
			if err != nil {
				return err
			}
			runtimeObjectsList = make([]runtime.Object, len(list.Items))
			for i := range list.Items {
				runtimeObjectsList[i] = &list.Items[i]
			}
			return nil
		}
	case "Deployment":
		listFunc = func() error {
			list, err := c.AppsV1().Deployments(namespace).List(listOpts)
			if err != nil {
				return err
			}
			runtimeObjectsList = make([]runtime.Object, len(list.Items))
			for i := range list.Items {
				runtimeObjectsList[i] = &list.Items[i]
			}
			return nil
		}
	case "DaemonSet":
		listFunc = func() error {
			list, err := c.AppsV1().DaemonSets(namespace).List(listOpts)
			if err != nil {
				return err
			}
			runtimeObjectsList = make([]runtime.Object, len(list.Items))
			for i := range list.Items {
				runtimeObjectsList[i] = &list.Items[i]
			}
			return nil
		}
	case "Job":
		listFunc = func() error {
			list, err := c.BatchV1().Jobs(namespace).List(listOpts)
			if err != nil {
				return err
			}
			runtimeObjectsList = make([]runtime.Object, len(list.Items))
			for i := range list.Items {
				runtimeObjectsList[i] = &list.Items[i]
			}
			return nil
		}
	default:
		return nil, fmt.Errorf("unsupported kind when getting runtime object: %v", kind)
	}

	if err := client.RetryWithExponentialBackOff(client.RetryFunction(listFunc)); err != nil {
		return nil, err
	}
	return runtimeObjectsList, nil
}

// GetNameFromRuntimeObject returns name of given runtime object.
func GetNameFromRuntimeObject(obj runtime.Object) (string, error) {
	switch typed := obj.(type) {
	case *unstructured.Unstructured:
		return typed.GetName(), nil
	default:
		metaObjectAccessor, ok := obj.(metav1.ObjectMetaAccessor)
		if !ok {
			return "", fmt.Errorf("unsupported kind when getting name: %v", obj)
		}
		return metaObjectAccessor.GetObjectMeta().GetName(), nil
	}
}

// GetResourceVersionFromRuntimeObject returns resource version of given runtime object.
func GetResourceVersionFromRuntimeObject(obj runtime.Object) (uint64, error) {
	accessor, err := meta.Accessor(obj)
	if err != nil {
		return 0, fmt.Errorf("accessor error: %v", err)
	}
	version := accessor.GetResourceVersion()
	if len(version) == 0 {
		return 0, nil
	}
	return strconv.ParseUint(version, 10, 64)
}

// GetNamespaceFromRuntimeObject returns namespace of given runtime object.
func GetNamespaceFromRuntimeObject(obj runtime.Object) (string, error) {
	switch typed := obj.(type) {
	case *unstructured.Unstructured:
		return typed.GetNamespace(), nil
	default:
		metaObjectAccessor, ok := obj.(metav1.ObjectMetaAccessor)
		if !ok {
			return "", fmt.Errorf("unsupported kind when getting namespace: %v", obj)
		}
		return metaObjectAccessor.GetObjectMeta().GetNamespace(), nil
	}
}

// GetSelectorFromRuntimeObject returns selector of given runtime object.
func GetSelectorFromRuntimeObject(obj runtime.Object) (labels.Selector, error) {
	switch typed := obj.(type) {
	case *unstructured.Unstructured:
		return getSelectorFromUnstrutured(typed)
	case *corev1.ReplicationController:
		return labels.SelectorFromSet(typed.Spec.Selector), nil
	case *appsv1.ReplicaSet:
		return metav1.LabelSelectorAsSelector(typed.Spec.Selector)
	case *appsv1.Deployment:
		return metav1.LabelSelectorAsSelector(typed.Spec.Selector)
	case *appsv1.DaemonSet:
		return metav1.LabelSelectorAsSelector(typed.Spec.Selector)
	case *batch.Job:
		return metav1.LabelSelectorAsSelector(typed.Spec.Selector)
	default:
		return nil, fmt.Errorf("unsupported kind when getting selector: %v", obj)
	}
}

// Note: This function assumes each controller has field Spec.Selector.
// Moreover, Spec.Selector should be *metav1.LabelSelector, except for RelicationController.
func getSelectorFromUnstrutured(obj *unstructured.Unstructured) (labels.Selector, error) {
	spec, err := getSpecFromUnstrutured(obj)
	if err != nil {
		return nil, err
	}
	switch obj.GetKind() {
	case "ReplicationController":
		selectorMap, found, err := unstructured.NestedStringMap(spec, "selector")
		if err != nil {
			return nil, fmt.Errorf("try to selector failed, %v", err)
		}
		if !found {
			return nil, fmt.Errorf("try to selector failed, field selector not found")
		}
		return labels.SelectorFromSet(selectorMap), nil
	default:
		selectorMap, found, err := unstructured.NestedMap(spec, "selector")
		if err != nil {
			return nil, fmt.Errorf("try to selector failed, %v", err)
		}
		if !found {
			return nil, fmt.Errorf("try to selector failed, field selector not found")
		}
		var selector metav1.LabelSelector
		err = runtime.DefaultUnstructuredConverter.FromUnstructured(selectorMap, &selector)
		if err != nil {
			return nil, fmt.Errorf("try to selector failed, %v", err)
		}
		return metav1.LabelSelectorAsSelector(&selector)
	}
}

// GetSpecFromRuntimeObject returns spec of given runtime object.
func GetSpecFromRuntimeObject(obj runtime.Object) (interface{}, error) {
	if obj == nil {
		return nil, nil
	}
	switch typed := obj.(type) {
	case *unstructured.Unstructured:
		return getSpecFromUnstrutured(typed)
	case *corev1.ReplicationController:
		return typed.Spec, nil
	case *appsv1.ReplicaSet:
		return typed.Spec, nil
	case *appsv1.Deployment:
		return typed.Spec, nil
	case *appsv1.DaemonSet:
		return typed.Spec, nil
	case *batch.Job:
		return typed.Spec, nil
	default:
		return nil, fmt.Errorf("unsupported kind when getting spec: %v", obj)
	}
}

// Note: This function assumes each controller has field Spec.
func getSpecFromUnstrutured(obj *unstructured.Unstructured) (map[string]interface{}, error) {
	spec, ok, err := unstructured.NestedMap(obj.UnstructuredContent(), "spec")
	if err != nil {
		return nil, fmt.Errorf("try to acquire spec failed, %v", err)
	}
	if !ok {
		return nil, fmt.Errorf("try to acquire spec failed, no field spec for obj %s", obj.GetName())
	}
	return spec, nil
}

// GetReplicasFromRuntimeObject returns replicas number from given runtime object.
func GetReplicasFromRuntimeObject(obj runtime.Object) (int32, error) {
	if obj == nil {
		return 0, nil
	}
	switch typed := obj.(type) {
	case *unstructured.Unstructured:
		return getReplicasFromUnstrutured(typed)
	case *corev1.ReplicationController:
		if typed.Spec.Replicas != nil {
			return *typed.Spec.Replicas, nil
		}
		return 0, nil
	case *appsv1.ReplicaSet:
		if typed.Spec.Replicas != nil {
			return *typed.Spec.Replicas, nil
		}
		return 0, nil
	case *appsv1.Deployment:
		if typed.Spec.Replicas != nil {
			return *typed.Spec.Replicas, nil
		}
		return 0, nil
	case *appsv1.DaemonSet:
		return 0, nil
	case *batch.Job:
		if typed.Spec.Parallelism != nil {
			return *typed.Spec.Parallelism, nil
		}
		return 0, nil
	default:
		return -1, fmt.Errorf("unsupported kind when getting number of replicas: %v", obj)
	}
}

// Note: This function assumes each controller has field Spec.Replicas, except Daemonset and Job.
func getReplicasFromUnstrutured(obj *unstructured.Unstructured) (int32, error) {
	spec, err := getSpecFromUnstrutured(obj)
	if err != nil {
		return -1, err
	}

	return tryAcquireReplicasFromUnstructuredSpec(spec, obj.GetKind())
}

func tryAcquireReplicasFromUnstructuredSpec(spec map[string]interface{}, kind string) (int32, error) {
	switch kind {
	case "DaemonSet":
		return 0, nil
	case "Job":
		replicas, found, err := unstructured.NestedInt64(spec, "parallelism")
		if err != nil {
			return -1, fmt.Errorf("try to acquire job parallelism failed, %v", err)
		}
		if !found {
			return 0, nil
		}
		return int32(replicas), nil
	default:
		replicas, found, err := unstructured.NestedInt64(spec, "replicas")
		if err != nil {
			return -1, fmt.Errorf("try to acquire replicas failed, %v", err)
		}
		if !found {
			return 0, nil
		}
		return int32(replicas), nil
	}
}

// IsEqualRuntimeObjectsSpec returns true if given runtime objects have identical specs.
func IsEqualRuntimeObjectsSpec(runtimeObj1, runtimeObj2 runtime.Object) (bool, error) {
	runtimeObj1Spec, err := GetSpecFromRuntimeObject(runtimeObj1)
	if err != nil {
		return false, err
	}
	runtimeObj2Spec, err := GetSpecFromRuntimeObject(runtimeObj2)
	if err != nil {
		return false, err
	}

	return equality.Semantic.DeepEqual(runtimeObj1Spec, runtimeObj2Spec), nil
}

// CreateMetaNamespaceKey returns meta key (namespace/name) for given runtime object.
func CreateMetaNamespaceKey(obj runtime.Object) (string, error) {
	namespace, err := GetNamespaceFromRuntimeObject(obj)
	if err != nil {
		return "", fmt.Errorf("retrieving namespace error: %v", err)
	}
	name, err := GetNameFromRuntimeObject(obj)
	if err != nil {
		return "", fmt.Errorf("retrieving name error: %v", err)
	}
	return namespace + "/" + name, nil
}
<commit_msg>Support StatefulSets in runtimeobjects.go<commit_after>/*
Copyright 2018 The Kubernetes Authors.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package runtimeobjects

import (
	"fmt"
	"strconv"

	appsv1 "k8s.io/api/apps/v1"
	batch "k8s.io/api/batch/v1"
	corev1 "k8s.io/api/core/v1"
	"k8s.io/apimachinery/pkg/api/equality"
	"k8s.io/apimachinery/pkg/api/meta"
	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
	"k8s.io/apimachinery/pkg/apis/meta/v1/unstructured"
	"k8s.io/apimachinery/pkg/labels"
	"k8s.io/apimachinery/pkg/runtime"
	clientset "k8s.io/client-go/kubernetes"
	"k8s.io/perf-tests/clusterloader2/pkg/framework/client"
)

// ListRuntimeObjectsForKind returns objects of given kind that satisfy given namespace, labelSelector and fieldSelector.
// TODO: using dynamic interface rather than clientset interface
func ListRuntimeObjectsForKind(c clientset.Interface, kind, namespace, labelSelector, fieldSelector string) ([]runtime.Object, error) {
	var runtimeObjectsList []runtime.Object
	var listFunc func() error
	listOpts := metav1.ListOptions{
		LabelSelector: labelSelector,
		FieldSelector: fieldSelector,
	}
	switch kind {
	case "ReplicationController":
		listFunc = func() error {
			list, err := c.CoreV1().ReplicationControllers(namespace).List(listOpts)
			if err != nil {
				return err
			}
			runtimeObjectsList = make([]runtime.Object, len(list.Items))
			for i := range list.Items {
				runtimeObjectsList[i] = &list.Items[i]
			}
			return nil
		}
	case "ReplicaSet":
		listFunc = func() error {
			list, err := c.AppsV1().ReplicaSets(namespace).List(listOpts)
			if err != nil {
				return err
			}
			runtimeObjectsList = make([]runtime.Object, len(list.Items))
			for i := range list.Items {
				runtimeObjectsList[i] = &list.Items[i]
			}
			return nil
		}
	case "Deployment":
		listFunc = func() error {
			list, err := c.AppsV1().Deployments(namespace).List(listOpts)
			if err != nil {
				return err
			}
			runtimeObjectsList = make([]runtime.Object, len(list.Items))
			for i := range list.Items {
				runtimeObjectsList[i] = &list.Items[i]
			}
			return nil
		}
	case "StatefulSet":
		listFunc = func() error {
			list, err := c.AppsV1().StatefulSets(namespace).List(listOpts)
			if err != nil {
				return err
			}
			runtimeObjectsList = make([]runtime.Object, len(list.Items))
			for i := range list.Items {
				runtimeObjectsList[i] = &list.Items[i]
			}
			return nil
		}
	case "DaemonSet":
		listFunc = func() error {
			list, err := c.AppsV1().DaemonSets(namespace).List(listOpts)
			if err != nil {
				return err
			}
			runtimeObjectsList = make([]runtime.Object, len(list.Items))
			for i := range list.Items {
				runtimeObjectsList[i] = &list.Items[i]
			}
			return nil
		}
	case "Job":
		listFunc = func() error {
			list, err := c.BatchV1().Jobs(namespace).List(listOpts)
			if err != nil {
				return err
			}
			runtimeObjectsList = make([]runtime.Object, len(list.Items))
			for i := range list.Items {
				runtimeObjectsList[i] = &list.Items[i]
			}
			return nil
		}
	default:
		return nil, fmt.Errorf("unsupported kind when getting runtime object: %v", kind)
	}

	if err := client.RetryWithExponentialBackOff(client.RetryFunction(listFunc)); err != nil {
		return nil, err
	}
	return runtimeObjectsList, nil
}

// GetNameFromRuntimeObject returns name of given runtime object.
func GetNameFromRuntimeObject(obj runtime.Object) (string, error) {
	switch typed := obj.(type) {
	case *unstructured.Unstructured:
		return typed.GetName(), nil
	default:
		metaObjectAccessor, ok := obj.(metav1.ObjectMetaAccessor)
		if !ok {
			return "", fmt.Errorf("unsupported kind when getting name: %v", obj)
		}
		return metaObjectAccessor.GetObjectMeta().GetName(), nil
	}
}

// GetResourceVersionFromRuntimeObject returns resource version of given runtime object.
func GetResourceVersionFromRuntimeObject(obj runtime.Object) (uint64, error) {
	accessor, err := meta.Accessor(obj)
	if err != nil {
		return 0, fmt.Errorf("accessor error: %v", err)
	}
	version := accessor.GetResourceVersion()
	if len(version) == 0 {
		return 0, nil
	}
	return strconv.ParseUint(version, 10, 64)
}

// GetNamespaceFromRuntimeObject returns namespace of given runtime object.
func GetNamespaceFromRuntimeObject(obj runtime.Object) (string, error) {
	switch typed := obj.(type) {
	case *unstructured.Unstructured:
		return typed.GetNamespace(), nil
	default:
		metaObjectAccessor, ok := obj.(metav1.ObjectMetaAccessor)
		if !ok {
			return "", fmt.Errorf("unsupported kind when getting namespace: %v", obj)
		}
		return metaObjectAccessor.GetObjectMeta().GetNamespace(), nil
	}
}

// GetSelectorFromRuntimeObject returns selector of given runtime object.
func GetSelectorFromRuntimeObject(obj runtime.Object) (labels.Selector, error) {
	switch typed := obj.(type) {
	case *unstructured.Unstructured:
		return getSelectorFromUnstrutured(typed)
	case *corev1.ReplicationController:
		return labels.SelectorFromSet(typed.Spec.Selector), nil
	case *appsv1.ReplicaSet:
		return metav1.LabelSelectorAsSelector(typed.Spec.Selector)
	case *appsv1.Deployment:
		return metav1.LabelSelectorAsSelector(typed.Spec.Selector)
	case *appsv1.StatefulSet:
		return metav1.LabelSelectorAsSelector(typed.Spec.Selector)
	case *appsv1.DaemonSet:
		return metav1.LabelSelectorAsSelector(typed.Spec.Selector)
	case *batch.Job:
		return metav1.LabelSelectorAsSelector(typed.Spec.Selector)
	default:
		return nil, fmt.Errorf("unsupported kind when getting selector: %v", obj)
	}
}

// Note: This function assumes each controller has field Spec.Selector.
// Moreover, Spec.Selector should be *metav1.LabelSelector, except for RelicationController.
func getSelectorFromUnstrutured(obj *unstructured.Unstructured) (labels.Selector, error) {
	spec, err := getSpecFromUnstrutured(obj)
	if err != nil {
		return nil, err
	}
	switch obj.GetKind() {
	case "ReplicationController":
		selectorMap, found, err := unstructured.NestedStringMap(spec, "selector")
		if err != nil {
			return nil, fmt.Errorf("try to selector failed, %v", err)
		}
		if !found {
			return nil, fmt.Errorf("try to selector failed, field selector not found")
		}
		return labels.SelectorFromSet(selectorMap), nil
	default:
		selectorMap, found, err := unstructured.NestedMap(spec, "selector")
		if err != nil {
			return nil, fmt.Errorf("try to selector failed, %v", err)
		}
		if !found {
			return nil, fmt.Errorf("try to selector failed, field selector not found")
		}
		var selector metav1.LabelSelector
		err = runtime.DefaultUnstructuredConverter.FromUnstructured(selectorMap, &selector)
		if err != nil {
			return nil, fmt.Errorf("try to selector failed, %v", err)
		}
		return metav1.LabelSelectorAsSelector(&selector)
	}
}

// GetSpecFromRuntimeObject returns spec of given runtime object.
func GetSpecFromRuntimeObject(obj runtime.Object) (interface{}, error) {
	if obj == nil {
		return nil, nil
	}
	switch typed := obj.(type) {
	case *unstructured.Unstructured:
		return getSpecFromUnstrutured(typed)
	case *corev1.ReplicationController:
		return typed.Spec, nil
	case *appsv1.ReplicaSet:
		return typed.Spec, nil
	case *appsv1.Deployment:
		return typed.Spec, nil
	case *appsv1.StatefulSet:
		return typed.Spec, nil
	case *appsv1.DaemonSet:
		return typed.Spec, nil
	case *batch.Job:
		return typed.Spec, nil
	default:
		return nil, fmt.Errorf("unsupported kind when getting spec: %v", obj)
	}
}

// Note: This function assumes each controller has field Spec.
func getSpecFromUnstrutured(obj *unstructured.Unstructured) (map[string]interface{}, error) {
	spec, ok, err := unstructured.NestedMap(obj.UnstructuredContent(), "spec")
	if err != nil {
		return nil, fmt.Errorf("try to acquire spec failed, %v", err)
	}
	if !ok {
		return nil, fmt.Errorf("try to acquire spec failed, no field spec for obj %s", obj.GetName())
	}
	return spec, nil
}

// GetReplicasFromRuntimeObject returns replicas number from given runtime object.
func GetReplicasFromRuntimeObject(obj runtime.Object) (int32, error) {
	if obj == nil {
		return 0, nil
	}
	switch typed := obj.(type) {
	case *unstructured.Unstructured:
		return getReplicasFromUnstrutured(typed)
	case *corev1.ReplicationController:
		if typed.Spec.Replicas != nil {
			return *typed.Spec.Replicas, nil
		}
		return 0, nil
	case *appsv1.ReplicaSet:
		if typed.Spec.Replicas != nil {
			return *typed.Spec.Replicas, nil
		}
		return 0, nil
	case *appsv1.Deployment:
		if typed.Spec.Replicas != nil {
			return *typed.Spec.Replicas, nil
		}
		return 0, nil
	case *appsv1.StatefulSet:
		if typed.Spec.Replicas != nil {
			return *typed.Spec.Replicas, nil
		}
		return 0, nil
	case *appsv1.DaemonSet:
		return 0, nil
	case *batch.Job:
		if typed.Spec.Parallelism != nil {
			return *typed.Spec.Parallelism, nil
		}
		return 0, nil
	default:
		return -1, fmt.Errorf("unsupported kind when getting number of replicas: %v", obj)
	}
}

// Note: This function assumes each controller has field Spec.Replicas, except Daemonset and Job.
func getReplicasFromUnstrutured(obj *unstructured.Unstructured) (int32, error) {
	spec, err := getSpecFromUnstrutured(obj)
	if err != nil {
		return -1, err
	}

	return tryAcquireReplicasFromUnstructuredSpec(spec, obj.GetKind())
}

func tryAcquireReplicasFromUnstructuredSpec(spec map[string]interface{}, kind string) (int32, error) {
	switch kind {
	case "DaemonSet":
		return 0, nil
	case "Job":
		replicas, found, err := unstructured.NestedInt64(spec, "parallelism")
		if err != nil {
			return -1, fmt.Errorf("try to acquire job parallelism failed, %v", err)
		}
		if !found {
			return 0, nil
		}
		return int32(replicas), nil
	default:
		replicas, found, err := unstructured.NestedInt64(spec, "replicas")
		if err != nil {
			return -1, fmt.Errorf("try to acquire replicas failed, %v", err)
		}
		if !found {
			return 0, nil
		}
		return int32(replicas), nil
	}
}

// IsEqualRuntimeObjectsSpec returns true if given runtime objects have identical specs.
func IsEqualRuntimeObjectsSpec(runtimeObj1, runtimeObj2 runtime.Object) (bool, error) {
	runtimeObj1Spec, err := GetSpecFromRuntimeObject(runtimeObj1)
	if err != nil {
		return false, err
	}
	runtimeObj2Spec, err := GetSpecFromRuntimeObject(runtimeObj2)
	if err != nil {
		return false, err
	}

	return equality.Semantic.DeepEqual(runtimeObj1Spec, runtimeObj2Spec), nil
}

// CreateMetaNamespaceKey returns meta key (namespace/name) for given runtime object.
func CreateMetaNamespaceKey(obj runtime.Object) (string, error) {
	namespace, err := GetNamespaceFromRuntimeObject(obj)
	if err != nil {
		return "", fmt.Errorf("retrieving namespace error: %v", err)
	}
	name, err := GetNameFromRuntimeObject(obj)
	if err != nil {
		return "", fmt.Errorf("retrieving name error: %v", err)
	}
	return namespace + "/" + name, nil
}
<|endoftext|>