<commit_before>package garden

import (
	"io"
	"time"
)

//go:generate counterfeiter . Container

type Container interface {
	Handle() string

	// Stop stops a container.
	//
	// If kill is false, garden stops a container by sending the processes running inside it the SIGTERM signal.
	// It then waits for the processes to terminate before returning a response.
	// If one or more processes do not terminate within 10 seconds,
	// garden sends these processes the SIGKILL signal, killing them ungracefully.
	//
	// If kill is true, garden stops a container by sending the processing running inside it a SIGKILL signal.
	//
	// Once a container is stopped, garden does not allow spawning new processes inside the container.
	// It is possible to copy files in to and out of a stopped container.
	// It is only when a container is destroyed that its filesystem is cleaned up.
	//
	// Errors:
	// * None.
	Stop(kill bool) error

	// Returns information about a container.
	Info() (ContainerInfo, error)

	// StreamIn streams data into a file in a container.
	//
	// Errors:
	// *  TODO.
	StreamIn(spec StreamInSpec) error

	// StreamOut streams a file out of a container.
	//
	// Errors:
	// * TODO.
	StreamOut(spec StreamOutSpec) (io.ReadCloser, error)

	// Limits the network bandwidth for a container.
	LimitBandwidth(limits BandwidthLimits) error

	CurrentBandwidthLimits() (BandwidthLimits, error)

	// Limits the CPU shares for a container.
	LimitCPU(limits CPULimits) error

	// Returns the currently set CPU limts for a container.
	CurrentCPULimits() (CPULimits, error)

	// Returns the currently set Disk limts for a container.
	CurrentDiskLimits() (DiskLimits, error)

	// Limits the memory usage for a container.
	//
	// The limit applies to all process in the container. When the limit is
	// exceeded, the container will be automatically stopped.
	//
	// Errors:
	// * The kernel does not support setting memory.memsw.limit_in_bytes.
	LimitMemory(limits MemoryLimits) error

	CurrentMemoryLimits() (MemoryLimits, error)

	// Map a port on the host to a port in the container so that traffic to the
	// host port is forwarded to the container port.
	//
	// If a host port is not given, a port will be acquired from the server's port
	// pool.
	//
	// If a container port is not given, the port will be the same as the
	// container port.
	//
	// The two resulting ports are returned in the response.
	//
	// Errors:
	// * When no port can be acquired from the server's port pool.
	NetIn(hostPort, containerPort uint32) (uint32, uint32, error)

	// Whitelist outbound network traffic.
	//
	// If the configuration directive deny_networks is not used,
	// all networks are already whitelisted and this command is effectively a no-op.
	//
	// Later NetOut calls take precedence over earlier calls, which is
	// significant only in relation to logging.
	//
	// Errors:
	// * An error is returned if the NetOut call fails.
	NetOut(netOutRule NetOutRule) error

	// Run a script inside a container.
	//
	// The root user will be mapped to a non-root UID in the host unless the container (not this process) was created with 'privileged' true.
	//
	// Errors:
	// * TODO.
	Run(ProcessSpec, ProcessIO) (Process, error)

	// Attach starts streaming the output back to the client from a specified process.
	//
	// Errors:
	// * processID does not refer to a running process.
	Attach(processID string, io ProcessIO) (Process, error)

	// Metrics returns the current set of metrics for a container
	Metrics() (Metrics, error)

	// Sets the grace time.
	SetGraceTime(graceTime time.Duration) error

	// Properties returns the current set of properties
	Properties() (Properties, error)

	// Property returns the value of the property with the specified name.
	//
	// Errors:
	// * When the property does not exist on the container.
	Property(name string) (string, error)

	// Set a named property on a container to a specified value.
	//
	// Errors:
	// * None.
	SetProperty(name string, value string) error

	// Remove a property with the specified name from a container.
	//
	// Errors:
	// * None.
	RemoveProperty(name string) error
}

// ProcessSpec contains parameters for running a script inside a container.
type ProcessSpec struct {
	// Path to command to execute.
	Path string `json:"path,omitempty"`

	// Arguments to pass to command.
	Args []string `json:"args,omitempty"`

	// Environment variables.
	Env []string `json:"env,omitempty"`

	// Working directory (default: home directory).
	Dir string `json:"dir,omitempty"`

	// The name of a user in the container to run the process as.
	User string `json:"user,omitempty"`

	// Resource limits
	Limits ResourceLimits `json:"rlimits,omitempty"`

	// Execute with a TTY for stdio.
	TTY *TTYSpec `json:"tty,omitempty"`
}

type TTYSpec struct {
	WindowSize *WindowSize `json:"window_size,omitempty"`
}

type WindowSize struct {
	Columns int `json:"columns,omitempty"`
	Rows    int `json:"rows,omitempty"`
}

type ProcessIO struct {
	Stdin  io.Reader
	Stdout io.Writer
	Stderr io.Writer
}

//go:generate counterfeiter . Process

type Process interface {
	ID() string
	Wait() (int, error)
	SetTTY(TTYSpec) error
	Signal(Signal) error
}

type Signal int

const (
	SignalTerminate Signal = iota
	SignalKill
)

type PortMapping struct {
	HostPort      uint32
	ContainerPort uint32
}

type StreamInSpec struct {
	Path      string
	User      string
	TarStream io.Reader
}

type StreamOutSpec struct {
	Path string
	User string
}

// ContainerInfo holds information about a container.
type ContainerInfo struct {
	State         string        // Either "active" or "stopped".
	Events        []string      // List of events that occurred for the container. It currently includes only "oom" (Out Of Memory) event if it occurred.
	HostIP        string        // The IP address of the gateway which controls the host side of the container's virtual ethernet pair.
	ContainerIP   string        // The IP address of the container side of the container's virtual ethernet pair.
	ExternalIP    string        //
	ContainerPath string        // The path to the directory holding the container's files (both its control scripts and filesystem).
	ProcessIDs    []string      // List of running processes.
	Properties    Properties    // List of properties defined for the container.
	MappedPorts   []PortMapping //
}

type ContainerInfoEntry struct {
	Info ContainerInfo
	Err  *Error
}

type Metrics struct {
	MemoryStat  ContainerMemoryStat
	CPUStat     ContainerCPUStat
	DiskStat    ContainerDiskStat
	NetworkStat ContainerNetworkStat
}

type ContainerMetricsEntry struct {
	Metrics Metrics
	Err     *Error
}

type ContainerMemoryStat struct {
	Cache                   uint64
	Rss                     uint64
	MappedFile              uint64
	Pgpgin                  uint64
	Pgpgout                 uint64
	Swap                    uint64
	Pgfault                 uint64
	Pgmajfault              uint64
	InactiveAnon            uint64
	ActiveAnon              uint64
	InactiveFile            uint64
	ActiveFile              uint64
	Unevictable             uint64
	HierarchicalMemoryLimit uint64
	HierarchicalMemswLimit  uint64
	TotalCache              uint64
	TotalRss                uint64
	TotalMappedFile         uint64
	TotalPgpgin             uint64
	TotalPgpgout            uint64
	TotalSwap               uint64
	TotalPgfault            uint64
	TotalPgmajfault         uint64
	TotalInactiveAnon       uint64
	TotalActiveAnon         uint64
	TotalInactiveFile       uint64
	TotalActiveFile         uint64
	TotalUnevictable        uint64
	// A memory usage total which reports memory usage in the same way that limits are enforced.
	// This value includes memory consumed by nested containers.
	TotalUsageTowardLimit uint64
}

type ContainerCPUStat struct {
	Usage  uint64
	User   uint64
	System uint64
}

type ContainerDiskStat struct {
	TotalBytesUsed      uint64
	TotalInodesUsed     uint64
	ExclusiveBytesUsed  uint64
	ExclusiveInodesUsed uint64
}

type ContainerBandwidthStat struct {
	InRate   uint64
	InBurst  uint64
	OutRate  uint64
	OutBurst uint64
}

type ContainerNetworkStat struct {
	RxBytes uint64
	TxBytes uint64
}

type BandwidthLimits struct {
	RateInBytesPerSecond      uint64 `json:"rate,omitempty"`
	BurstRateInBytesPerSecond uint64 `json:"burst,omitempty"`
}

type DiskLimits struct {
	InodeSoft uint64 `json:"inode_soft,omitempty"`
	InodeHard uint64 `json:"inode_hard,omitempty"`

	ByteSoft uint64 `json:"byte_soft,omitempty"`
	ByteHard uint64 `json:"byte_hard,omitempty"`

	Scope DiskLimitScope `json:"scope,omitempty"`
}

type MemoryLimits struct {
	//	Memory usage limit in bytes.
	LimitInBytes uint64 `json:"limit_in_bytes,omitempty"`
}

type CPULimits struct {
	LimitInShares uint64 `json:"limit_in_shares,omitempty"`
}

// Resource limits.
//
// Please refer to the manual page of getrlimit for a description of the individual fields:
// http://www.kernel.org/doc/man-pages/online/pages/man2/getrlimit.2.html
type ResourceLimits struct {
	As         *uint64 `json:"as,omitempty"`
	Core       *uint64 `json:"core,omitempty"`
	Cpu        *uint64 `json:"cpu,omitempty"`
	Data       *uint64 `json:"data,omitempty"`
	Fsize      *uint64 `json:"fsize,omitempty"`
	Locks      *uint64 `json:"locks,omitempty"`
	Memlock    *uint64 `json:"memlock,omitempty"`
	Msgqueue   *uint64 `json:"msgqueue,omitempty"`
	Nice       *uint64 `json:"nice,omitempty"`
	Nofile     *uint64 `json:"nofile,omitempty"`
	Nproc      *uint64 `json:"nproc,omitempty"`
	Rss        *uint64 `json:"rss,omitempty"`
	Rtprio     *uint64 `json:"rtprio,omitempty"`
	Sigpending *uint64 `json:"sigpending,omitempty"`
	Stack      *uint64 `json:"stack,omitempty"`
}

type DiskLimitScope uint8

const DiskLimitScopeTotal DiskLimitScope = 0
const DiskLimitScopeExclusive DiskLimitScope = 1
<commit_msg>Minor clarification in NetIn godoc comment<commit_after>package garden

import (
	"io"
	"time"
)

//go:generate counterfeiter . Container

type Container interface {
	Handle() string

	// Stop stops a container.
	//
	// If kill is false, garden stops a container by sending the processes running inside it the SIGTERM signal.
	// It then waits for the processes to terminate before returning a response.
	// If one or more processes do not terminate within 10 seconds,
	// garden sends these processes the SIGKILL signal, killing them ungracefully.
	//
	// If kill is true, garden stops a container by sending the processing running inside it a SIGKILL signal.
	//
	// Once a container is stopped, garden does not allow spawning new processes inside the container.
	// It is possible to copy files in to and out of a stopped container.
	// It is only when a container is destroyed that its filesystem is cleaned up.
	//
	// Errors:
	// * None.
	Stop(kill bool) error

	// Returns information about a container.
	Info() (ContainerInfo, error)

	// StreamIn streams data into a file in a container.
	//
	// Errors:
	// *  TODO.
	StreamIn(spec StreamInSpec) error

	// StreamOut streams a file out of a container.
	//
	// Errors:
	// * TODO.
	StreamOut(spec StreamOutSpec) (io.ReadCloser, error)

	// Limits the network bandwidth for a container.
	LimitBandwidth(limits BandwidthLimits) error

	CurrentBandwidthLimits() (BandwidthLimits, error)

	// Limits the CPU shares for a container.
	LimitCPU(limits CPULimits) error

	// Returns the currently set CPU limts for a container.
	CurrentCPULimits() (CPULimits, error)

	// Returns the currently set Disk limts for a container.
	CurrentDiskLimits() (DiskLimits, error)

	// Limits the memory usage for a container.
	//
	// The limit applies to all process in the container. When the limit is
	// exceeded, the container will be automatically stopped.
	//
	// Errors:
	// * The kernel does not support setting memory.memsw.limit_in_bytes.
	LimitMemory(limits MemoryLimits) error

	CurrentMemoryLimits() (MemoryLimits, error)

	// Map a port on the host to a port in the container so that traffic to the
	// host port is forwarded to the container port.
	//
	// If a host port is not given, a port will be acquired from the server's port
	// pool.
	//
	// If a container port is not given, the port will be the same as the
	// container port.
	//
	// The resulting host and container ports are returned in that order.
	//
	// Errors:
	// * When no port can be acquired from the server's port pool.
	NetIn(hostPort, containerPort uint32) (uint32, uint32, error)

	// Whitelist outbound network traffic.
	//
	// If the configuration directive deny_networks is not used,
	// all networks are already whitelisted and this command is effectively a no-op.
	//
	// Later NetOut calls take precedence over earlier calls, which is
	// significant only in relation to logging.
	//
	// Errors:
	// * An error is returned if the NetOut call fails.
	NetOut(netOutRule NetOutRule) error

	// Run a script inside a container.
	//
	// The root user will be mapped to a non-root UID in the host unless the container (not this process) was created with 'privileged' true.
	//
	// Errors:
	// * TODO.
	Run(ProcessSpec, ProcessIO) (Process, error)

	// Attach starts streaming the output back to the client from a specified process.
	//
	// Errors:
	// * processID does not refer to a running process.
	Attach(processID string, io ProcessIO) (Process, error)

	// Metrics returns the current set of metrics for a container
	Metrics() (Metrics, error)

	// Sets the grace time.
	SetGraceTime(graceTime time.Duration) error

	// Properties returns the current set of properties
	Properties() (Properties, error)

	// Property returns the value of the property with the specified name.
	//
	// Errors:
	// * When the property does not exist on the container.
	Property(name string) (string, error)

	// Set a named property on a container to a specified value.
	//
	// Errors:
	// * None.
	SetProperty(name string, value string) error

	// Remove a property with the specified name from a container.
	//
	// Errors:
	// * None.
	RemoveProperty(name string) error
}

// ProcessSpec contains parameters for running a script inside a container.
type ProcessSpec struct {
	// Path to command to execute.
	Path string `json:"path,omitempty"`

	// Arguments to pass to command.
	Args []string `json:"args,omitempty"`

	// Environment variables.
	Env []string `json:"env,omitempty"`

	// Working directory (default: home directory).
	Dir string `json:"dir,omitempty"`

	// The name of a user in the container to run the process as.
	User string `json:"user,omitempty"`

	// Resource limits
	Limits ResourceLimits `json:"rlimits,omitempty"`

	// Execute with a TTY for stdio.
	TTY *TTYSpec `json:"tty,omitempty"`
}

type TTYSpec struct {
	WindowSize *WindowSize `json:"window_size,omitempty"`
}

type WindowSize struct {
	Columns int `json:"columns,omitempty"`
	Rows    int `json:"rows,omitempty"`
}

type ProcessIO struct {
	Stdin  io.Reader
	Stdout io.Writer
	Stderr io.Writer
}

//go:generate counterfeiter . Process

type Process interface {
	ID() string
	Wait() (int, error)
	SetTTY(TTYSpec) error
	Signal(Signal) error
}

type Signal int

const (
	SignalTerminate Signal = iota
	SignalKill
)

type PortMapping struct {
	HostPort      uint32
	ContainerPort uint32
}

type StreamInSpec struct {
	Path      string
	User      string
	TarStream io.Reader
}

type StreamOutSpec struct {
	Path string
	User string
}

// ContainerInfo holds information about a container.
type ContainerInfo struct {
	State         string        // Either "active" or "stopped".
	Events        []string      // List of events that occurred for the container. It currently includes only "oom" (Out Of Memory) event if it occurred.
	HostIP        string        // The IP address of the gateway which controls the host side of the container's virtual ethernet pair.
	ContainerIP   string        // The IP address of the container side of the container's virtual ethernet pair.
	ExternalIP    string        //
	ContainerPath string        // The path to the directory holding the container's files (both its control scripts and filesystem).
	ProcessIDs    []string      // List of running processes.
	Properties    Properties    // List of properties defined for the container.
	MappedPorts   []PortMapping //
}

type ContainerInfoEntry struct {
	Info ContainerInfo
	Err  *Error
}

type Metrics struct {
	MemoryStat  ContainerMemoryStat
	CPUStat     ContainerCPUStat
	DiskStat    ContainerDiskStat
	NetworkStat ContainerNetworkStat
}

type ContainerMetricsEntry struct {
	Metrics Metrics
	Err     *Error
}

type ContainerMemoryStat struct {
	Cache                   uint64
	Rss                     uint64
	MappedFile              uint64
	Pgpgin                  uint64
	Pgpgout                 uint64
	Swap                    uint64
	Pgfault                 uint64
	Pgmajfault              uint64
	InactiveAnon            uint64
	ActiveAnon              uint64
	InactiveFile            uint64
	ActiveFile              uint64
	Unevictable             uint64
	HierarchicalMemoryLimit uint64
	HierarchicalMemswLimit  uint64
	TotalCache              uint64
	TotalRss                uint64
	TotalMappedFile         uint64
	TotalPgpgin             uint64
	TotalPgpgout            uint64
	TotalSwap               uint64
	TotalPgfault            uint64
	TotalPgmajfault         uint64
	TotalInactiveAnon       uint64
	TotalActiveAnon         uint64
	TotalInactiveFile       uint64
	TotalActiveFile         uint64
	TotalUnevictable        uint64
	// A memory usage total which reports memory usage in the same way that limits are enforced.
	// This value includes memory consumed by nested containers.
	TotalUsageTowardLimit uint64
}

type ContainerCPUStat struct {
	Usage  uint64
	User   uint64
	System uint64
}

type ContainerDiskStat struct {
	TotalBytesUsed      uint64
	TotalInodesUsed     uint64
	ExclusiveBytesUsed  uint64
	ExclusiveInodesUsed uint64
}

type ContainerBandwidthStat struct {
	InRate   uint64
	InBurst  uint64
	OutRate  uint64
	OutBurst uint64
}

type ContainerNetworkStat struct {
	RxBytes uint64
	TxBytes uint64
}

type BandwidthLimits struct {
	RateInBytesPerSecond      uint64 `json:"rate,omitempty"`
	BurstRateInBytesPerSecond uint64 `json:"burst,omitempty"`
}

type DiskLimits struct {
	InodeSoft uint64 `json:"inode_soft,omitempty"`
	InodeHard uint64 `json:"inode_hard,omitempty"`

	ByteSoft uint64 `json:"byte_soft,omitempty"`
	ByteHard uint64 `json:"byte_hard,omitempty"`

	Scope DiskLimitScope `json:"scope,omitempty"`
}

type MemoryLimits struct {
	//	Memory usage limit in bytes.
	LimitInBytes uint64 `json:"limit_in_bytes,omitempty"`
}

type CPULimits struct {
	LimitInShares uint64 `json:"limit_in_shares,omitempty"`
}

// Resource limits.
//
// Please refer to the manual page of getrlimit for a description of the individual fields:
// http://www.kernel.org/doc/man-pages/online/pages/man2/getrlimit.2.html
type ResourceLimits struct {
	As         *uint64 `json:"as,omitempty"`
	Core       *uint64 `json:"core,omitempty"`
	Cpu        *uint64 `json:"cpu,omitempty"`
	Data       *uint64 `json:"data,omitempty"`
	Fsize      *uint64 `json:"fsize,omitempty"`
	Locks      *uint64 `json:"locks,omitempty"`
	Memlock    *uint64 `json:"memlock,omitempty"`
	Msgqueue   *uint64 `json:"msgqueue,omitempty"`
	Nice       *uint64 `json:"nice,omitempty"`
	Nofile     *uint64 `json:"nofile,omitempty"`
	Nproc      *uint64 `json:"nproc,omitempty"`
	Rss        *uint64 `json:"rss,omitempty"`
	Rtprio     *uint64 `json:"rtprio,omitempty"`
	Sigpending *uint64 `json:"sigpending,omitempty"`
	Stack      *uint64 `json:"stack,omitempty"`
}

type DiskLimitScope uint8

const DiskLimitScopeTotal DiskLimitScope = 0
const DiskLimitScopeExclusive DiskLimitScope = 1
<|endoftext|>


<commit_before>package iconv

/*
#cgo darwin LDFLAGS: -liconv
#cgo freebsd LDFLAGS: -liconv
#cgo windows LDFLAGS: -liconv
#include <stdlib.h>
#include <iconv.h>
*/
import "C"
import "syscall"
import "unsafe"

type Converter struct {
	context C.iconv_t
	open    bool
}

// Initialize a new Converter. If fromEncoding or toEncoding are not supported by
// iconv then an EINVAL error will be returned. An ENOMEM error maybe returned if
// there is not enough memory to initialize an iconv descriptor
func NewConverter(fromEncoding string, toEncoding string) (converter *Converter, err error) {
	converter = new(Converter)

	// convert to C strings
	toEncodingC := C.CString(toEncoding)
	fromEncodingC := C.CString(fromEncoding)

	// open an iconv descriptor
	converter.context, err = C.iconv_open(toEncodingC, fromEncodingC)

	// free the C Strings
	C.free(unsafe.Pointer(toEncodingC))
	C.free(unsafe.Pointer(fromEncodingC))

	// check err
	if err == nil {
		// no error, mark the context as open
		converter.open = true
	}

	return
}

// destroy is called during garbage collection
func (this *Converter) destroy() {
	this.Close()
}

// Close a Converter's iconv description explicitly
func (this *Converter) Close() (err error) {
	if this.open {
		_, err = C.iconv_close(this.context)
	}

	return
}

// Convert bytes from an input byte slice into a give output byte slice
//
// As many bytes that can converted and fit into the size of output will be
// processed and the number of bytes read for input as well as the number of
// bytes written to output will be returned. If not all converted bytes can fit
// into output and E2BIG error will also be returned. If input contains an invalid
// sequence of bytes for the Converter's fromEncoding an EILSEQ error will be returned
//
// For shift based output encodings, any end shift byte sequences can be generated by
// passing a 0 length byte slice as input. Also passing a 0 length byte slice for output
// will simply reset the iconv descriptor shift state without writing any bytes.
func (this *Converter) Convert(input []byte, output []byte) (bytesRead int, bytesWritten int, err error) {
	// make sure we are still open
	if this.open {
		inputLeft := C.size_t(len(input))
		outputLeft := C.size_t(len(output))

		if inputLeft > 0 && outputLeft > 0 {
			// we have to give iconv a pointer to a pointer of the underlying
			// storage of each byte slice - so far this is the simplest
			// way i've found to do that in Go, but it seems ugly
			inputPointer := (*C.char)(unsafe.Pointer(&input[0]))
			outputPointer := (*C.char)(unsafe.Pointer(&output[0]))

			_, err = C.iconv(this.context, &inputPointer, &inputLeft, &outputPointer, &outputLeft)

			// update byte counters
			bytesRead = len(input) - int(inputLeft)
			bytesWritten = len(output) - int(outputLeft)
		} else if inputLeft == 0 && outputLeft > 0 {
			// inputPointer will be nil, outputPointer is generated as above
			outputPointer := (*C.char)(unsafe.Pointer(&output[0]))

			_, err = C.iconv(this.context, nil, &inputLeft, &outputPointer, &outputLeft)

			// update write byte counter
			bytesWritten = len(output) - int(outputLeft)
		} else {
			// both input and output are zero length, do a shift state reset
			_, err = C.iconv(this.context, nil, &inputLeft, nil, &outputLeft)
		}
	} else {
		err = syscall.EBADF
	}

	return bytesRead, bytesWritten, err
}

// Convert an input string
//
// EILSEQ error may be returned if input contains invalid bytes for the
// Converter's fromEncoding.
func (this *Converter) ConvertString(input string) (output string, err error) {
	// make sure we are still open
	if this.open {
		// construct the buffers
		inputBuffer := []byte(input)
		outputBuffer := make([]byte, len(inputBuffer)*2) // we use a larger buffer to help avoid resizing later

		// call Convert until all input bytes are read or an error occurs
		var bytesRead, totalBytesRead, bytesWritten, totalBytesWritten int

		for totalBytesRead < len(inputBuffer) && err == nil {
			// use the totals to create buffer slices
			bytesRead, bytesWritten, err = this.Convert(inputBuffer[totalBytesRead:], outputBuffer[totalBytesWritten:])

			totalBytesRead += bytesRead
			totalBytesWritten += bytesWritten

			// check for the E2BIG error specifically, we can add to the output
			// buffer to correct for it and then continue
			if err == syscall.E2BIG {
				// increase the size of the output buffer by another input length
				// first, create a new buffer
				tempBuffer := make([]byte, len(outputBuffer)+len(inputBuffer))

				// copy the existing data
				copy(tempBuffer, outputBuffer)

				// switch the buffers
				outputBuffer = tempBuffer

				// forget the error
				err = nil
			}
		}

		if err == nil {
			// perform a final shift state reset
			_, bytesWritten, err = this.Convert([]byte{}, outputBuffer[totalBytesWritten:])

			// update total count
			totalBytesWritten += bytesWritten
		}

		// construct the final output string
		output = string(outputBuffer[:totalBytesWritten])
	} else {
		err = syscall.EBADF
	}

	return output, err
}
<commit_msg>Fixes #23 using call_iconv wrapper<commit_after>package iconv

/*
#cgo darwin LDFLAGS: -liconv
#cgo freebsd LDFLAGS: -liconv
#cgo windows LDFLAGS: -liconv
#include <stdlib.h>
#include <iconv.h>

// As of GO 1.6 passing a pointer to Go pointer, will lead to panic
// Therofore we use this wrapper function, to avoid passing **char directly from go
size_t call_iconv(iconv_t ctx, char *in, size_t *size_in, char *out, size_t *size_out){
	return iconv(ctx, &in, size_in, &out, size_out);
}

*/
import "C"
import "syscall"
import "unsafe"

type Converter struct {
	context C.iconv_t
	open    bool
}

// Initialize a new Converter. If fromEncoding or toEncoding are not supported by
// iconv then an EINVAL error will be returned. An ENOMEM error maybe returned if
// there is not enough memory to initialize an iconv descriptor
func NewConverter(fromEncoding string, toEncoding string) (converter *Converter, err error) {
	converter = new(Converter)

	// convert to C strings
	toEncodingC := C.CString(toEncoding)
	fromEncodingC := C.CString(fromEncoding)

	// open an iconv descriptor
	converter.context, err = C.iconv_open(toEncodingC, fromEncodingC)

	// free the C Strings
	C.free(unsafe.Pointer(toEncodingC))
	C.free(unsafe.Pointer(fromEncodingC))

	// check err
	if err == nil {
		// no error, mark the context as open
		converter.open = true
	}

	return
}

// destroy is called during garbage collection
func (this *Converter) destroy() {
	this.Close()
}

// Close a Converter's iconv description explicitly
func (this *Converter) Close() (err error) {
	if this.open {
		_, err = C.iconv_close(this.context)
	}

	return
}

// Convert bytes from an input byte slice into a give output byte slice
//
// As many bytes that can converted and fit into the size of output will be
// processed and the number of bytes read for input as well as the number of
// bytes written to output will be returned. If not all converted bytes can fit
// into output and E2BIG error will also be returned. If input contains an invalid
// sequence of bytes for the Converter's fromEncoding an EILSEQ error will be returned
//
// For shift based output encodings, any end shift byte sequences can be generated by
// passing a 0 length byte slice as input. Also passing a 0 length byte slice for output
// will simply reset the iconv descriptor shift state without writing any bytes.
func (this *Converter) Convert(input []byte, output []byte) (bytesRead int, bytesWritten int, err error) {
	// make sure we are still open
	if this.open {
		inputLeft := C.size_t(len(input))
		outputLeft := C.size_t(len(output))

		if inputLeft > 0 && outputLeft > 0 {
			// we have to give iconv a pointer to a pointer of the underlying
			// storage of each byte slice - so far this is the simplest
			// way i've found to do that in Go, but it seems ugly
			inputPointer := (*C.char)(unsafe.Pointer(&input[0]))
			outputPointer := (*C.char)(unsafe.Pointer(&output[0]))

			_, err = C.call_iconv(this.context, inputPointer, &inputLeft, outputPointer, &outputLeft)

			// update byte counters
			bytesRead = len(input) - int(inputLeft)
			bytesWritten = len(output) - int(outputLeft)
		} else if inputLeft == 0 && outputLeft > 0 {
			// inputPointer will be nil, outputPointer is generated as above
			outputPointer := (*C.char)(unsafe.Pointer(&output[0]))

			_, err = C.call_iconv(this.context, nil, &inputLeft, outputPointer, &outputLeft)

			// update write byte counter
			bytesWritten = len(output) - int(outputLeft)
		} else {
			// both input and output are zero length, do a shift state reset
			_, err = C.call_iconv(this.context, nil, &inputLeft, nil, &outputLeft)
		}
	} else {
		err = syscall.EBADF
	}

	return bytesRead, bytesWritten, err
}

// Convert an input string
//
// EILSEQ error may be returned if input contains invalid bytes for the
// Converter's fromEncoding.
func (this *Converter) ConvertString(input string) (output string, err error) {
	// make sure we are still open
	if this.open {
		// construct the buffers
		inputBuffer := []byte(input)
		outputBuffer := make([]byte, len(inputBuffer)*2) // we use a larger buffer to help avoid resizing later

		// call Convert until all input bytes are read or an error occurs
		var bytesRead, totalBytesRead, bytesWritten, totalBytesWritten int

		for totalBytesRead < len(inputBuffer) && err == nil {
			// use the totals to create buffer slices
			bytesRead, bytesWritten, err = this.Convert(inputBuffer[totalBytesRead:], outputBuffer[totalBytesWritten:])

			totalBytesRead += bytesRead
			totalBytesWritten += bytesWritten

			// check for the E2BIG error specifically, we can add to the output
			// buffer to correct for it and then continue
			if err == syscall.E2BIG {
				// increase the size of the output buffer by another input length
				// first, create a new buffer
				tempBuffer := make([]byte, len(outputBuffer)+len(inputBuffer))

				// copy the existing data
				copy(tempBuffer, outputBuffer)

				// switch the buffers
				outputBuffer = tempBuffer

				// forget the error
				err = nil
			}
		}

		if err == nil {
			// perform a final shift state reset
			_, bytesWritten, err = this.Convert([]byte{}, outputBuffer[totalBytesWritten:])

			// update total count
			totalBytesWritten += bytesWritten
		}

		// construct the final output string
		output = string(outputBuffer[:totalBytesWritten])
	} else {
		err = syscall.EBADF
	}

	return output, err
}
<|endoftext|>


<commit_before>package log2csv

import (
	"encoding/csv"
	"fmt"
	"io"
	"strconv"
	"strings"
	"time"
)

type Converter struct {
	r         io.Reader
	cw        *csv.Writer
	timestamp bool
}

func (c *Converter) Run() error {
	sc := NewScanner(c.r, formats)
	wroteHeader := false

	for {
		log := sc.Scan()
		if log == nil {
			break
		}

		if !wroteHeader {
			if err := c.writeHeader(log); err != nil {
				return err
			}
			wroteHeader = true
		}

		if err := c.writeLog(log); err != nil {
			return err
		}
		c.cw.Flush()
	}

	if sc.Err() != nil {
		return sc.Err()
	}
	return c.cw.Error()
}

func (c *Converter) writeHeader(log *Log) error {
	header := log.Format.Header
	if c.timestamp {
		header = "unixtime," + header
	}

	return c.cw.Write(strings.Split(header, ","))
}

func (c *Converter) writeLog(log *Log) error {
	if c.timestamp {
		log.Fields = append([]string{fmtFrac(log.Timestamp, 6)}, log.Fields...)
	}

	return c.cw.Write(log.Fields)
}

func fmtFrac(t time.Time, prec int) string {
	unixNano := t.UnixNano()
	fmtStr := "%." + strconv.Itoa(prec) + "f"

	return fmt.Sprintf(fmtStr, float64(unixNano)/10e8)
}

func NewConverter(r io.Reader, w io.Writer, timestamp bool) *Converter {
	c := new(Converter)
	c.r = r
	c.cw = csv.NewWriter(w)
	c.timestamp = timestamp

	return c
}
<commit_msg>do not change the log data<commit_after>package log2csv

import (
	"encoding/csv"
	"fmt"
	"io"
	"strconv"
	"strings"
	"time"
)

type Converter struct {
	r         io.Reader
	cw        *csv.Writer
	timestamp bool
}

func (c *Converter) Run() error {
	sc := NewScanner(c.r, formats)
	wroteHeader := false

	for {
		log := sc.Scan()
		if log == nil {
			break
		}

		if !wroteHeader {
			if err := c.writeHeader(log); err != nil {
				return err
			}
			wroteHeader = true
		}

		if err := c.writeLog(log); err != nil {
			return err
		}
		c.cw.Flush()
	}

	if sc.Err() != nil {
		return sc.Err()
	}
	return c.cw.Error()
}

func (c *Converter) writeHeader(log *Log) error {
	header := log.Format.Header
	if c.timestamp {
		header = "unixtime," + header
	}

	return c.cw.Write(strings.Split(header, ","))
}

func (c *Converter) writeLog(log *Log) error {
	fields := log.Fields
	if c.timestamp {
		fields = append([]string{fmtFrac(log.Timestamp, 6)}, fields...)
	}

	return c.cw.Write(fields)
}

func fmtFrac(t time.Time, prec int) string {
	unixNano := t.UnixNano()
	fmtStr := "%." + strconv.Itoa(prec) + "f"

	return fmt.Sprintf(fmtStr, float64(unixNano)/10e8)
}

func NewConverter(r io.Reader, w io.Writer, timestamp bool) *Converter {
	c := new(Converter)
	c.r = r
	c.cw = csv.NewWriter(w)
	c.timestamp = timestamp

	return c
}
<|endoftext|>


<commit_before>/*
Copyright 2015 The Kubernetes Authors.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package lifecycle

import (
	"context"
	"fmt"
	"io"
	"time"

	"k8s.io/klog"

	v1 "k8s.io/api/core/v1"
	"k8s.io/apimachinery/pkg/api/errors"
	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
	"k8s.io/apimachinery/pkg/runtime/schema"
	utilcache "k8s.io/apimachinery/pkg/util/cache"
	"k8s.io/apimachinery/pkg/util/clock"
	"k8s.io/apimachinery/pkg/util/sets"
	"k8s.io/apiserver/pkg/admission"
	"k8s.io/apiserver/pkg/admission/initializer"
	"k8s.io/client-go/informers"
	"k8s.io/client-go/kubernetes"
	corelisters "k8s.io/client-go/listers/core/v1"
)

const (
	// PluginName indicates the name of admission plug-in
	PluginName = "NamespaceLifecycle"
	// how long a namespace stays in the force live lookup cache before expiration.
	forceLiveLookupTTL = 30 * time.Second
	// how long to wait for a missing namespace before re-checking the cache (and then doing a live lookup)
	// this accomplishes two things:
	// 1. It allows a watch-fed cache time to observe a namespace creation event
	// 2. It allows time for a namespace creation to distribute to members of a storage cluster,
	//    so the live lookup has a better chance of succeeding even if it isn't performed against the leader.
	missingNamespaceWait = 50 * time.Millisecond
)

// Register registers a plugin
func Register(plugins *admission.Plugins) {
	plugins.Register(PluginName, func(config io.Reader) (admission.Interface, error) {
		return NewLifecycle(sets.NewString(metav1.NamespaceDefault, metav1.NamespaceSystem, metav1.NamespacePublic))
	})
}

// Lifecycle is an implementation of admission.Interface.
// It enforces life-cycle constraints around a Namespace depending on its Phase
type Lifecycle struct {
	*admission.Handler
	client             kubernetes.Interface
	immortalNamespaces sets.String
	namespaceLister    corelisters.NamespaceLister
	// forceLiveLookupCache holds a list of entries for namespaces that we have a strong reason to believe are stale in our local cache.
	// if a namespace is in this cache, then we will ignore our local state and always fetch latest from api server.
	forceLiveLookupCache *utilcache.LRUExpireCache
}

var _ = initializer.WantsExternalKubeInformerFactory(&Lifecycle{})
var _ = initializer.WantsExternalKubeClientSet(&Lifecycle{})

// Admit makes an admission decision based on the request attributes
func (l *Lifecycle) Admit(ctx context.Context, a admission.Attributes, o admission.ObjectInterfaces) error {
	// prevent deletion of immortal namespaces
	if a.GetOperation() == admission.Delete && a.GetKind().GroupKind() == v1.SchemeGroupVersion.WithKind("Namespace").GroupKind() && l.immortalNamespaces.Has(a.GetName()) {
		return errors.NewForbidden(a.GetResource().GroupResource(), a.GetName(), fmt.Errorf("this namespace may not be deleted"))
	}

	// always allow non-namespaced resources
	if len(a.GetNamespace()) == 0 && a.GetKind().GroupKind() != v1.SchemeGroupVersion.WithKind("Namespace").GroupKind() {
		return nil
	}

	if a.GetKind().GroupKind() == v1.SchemeGroupVersion.WithKind("Namespace").GroupKind() {
		// if a namespace is deleted, we want to prevent all further creates into it
		// while it is undergoing termination.  to reduce incidences where the cache
		// is slow to update, we add the namespace into a force live lookup list to ensure
		// we are not looking at stale state.
		if a.GetOperation() == admission.Delete {
			l.forceLiveLookupCache.Add(a.GetName(), true, forceLiveLookupTTL)
		}
		// allow all operations to namespaces
		return nil
	}

	// always allow deletion of other resources
	if a.GetOperation() == admission.Delete {
		return nil
	}

	// always allow access review checks.  Returning status about the namespace would be leaking information
	if isAccessReview(a) {
		return nil
	}

	// we need to wait for our caches to warm
	if !l.WaitForReady() {
		return admission.NewForbidden(a, fmt.Errorf("not yet ready to handle request"))
	}

	var (
		exists bool
		err    error
	)

	namespace, err := l.namespaceLister.Get(a.GetNamespace())
	if err != nil {
		if !errors.IsNotFound(err) {
			return errors.NewInternalError(err)
		}
	} else {
		exists = true
	}

	if !exists && a.GetOperation() == admission.Create {
		// give the cache time to observe the namespace before rejecting a create.
		// this helps when creating a namespace and immediately creating objects within it.
		time.Sleep(missingNamespaceWait)
		namespace, err = l.namespaceLister.Get(a.GetNamespace())
		switch {
		case errors.IsNotFound(err):
			// no-op
		case err != nil:
			return errors.NewInternalError(err)
		default:
			exists = true
		}
		if exists {
			klog.V(4).Infof("found %s in cache after waiting", a.GetNamespace())
		}
	}

	// forceLiveLookup if true will skip looking at local cache state and instead always make a live call to server.
	forceLiveLookup := false
	if _, ok := l.forceLiveLookupCache.Get(a.GetNamespace()); ok {
		// we think the namespace was marked for deletion, but our current local cache says otherwise, we will force a live lookup.
		forceLiveLookup = exists && namespace.Status.Phase == v1.NamespaceActive
	}

	// refuse to operate on non-existent namespaces
	if !exists || forceLiveLookup {
		// as a last resort, make a call directly to storage
		namespace, err = l.client.CoreV1().Namespaces().Get(context.TODO(), a.GetNamespace(), metav1.GetOptions{})
		switch {
		case errors.IsNotFound(err):
			return err
		case err != nil:
			return errors.NewInternalError(err)
		}
		klog.V(4).Infof("found %s via storage lookup", a.GetNamespace())
	}

	// ensure that we're not trying to create objects in terminating namespaces
	if a.GetOperation() == admission.Create {
		if namespace.Status.Phase != v1.NamespaceTerminating {
			return nil
		}

		err := admission.NewForbidden(a, fmt.Errorf("unable to create new content in namespace %s because it is being terminated", a.GetNamespace()))
		if apierr, ok := err.(*errors.StatusError); ok {
			apierr.ErrStatus.Details.Causes = append(apierr.ErrStatus.Details.Causes, metav1.StatusCause{
				Type:    v1.NamespaceTerminatingCause,
				Message: fmt.Sprintf("namespace %s is being terminated", a.GetNamespace()),
				Field:   "metadata.namespace",
			})
		}
		return err
	}

	return nil
}

// NewLifecycle creates a new namespace Lifecycle admission control handler
func NewLifecycle(immortalNamespaces sets.String) (*Lifecycle, error) {
	return newLifecycleWithClock(immortalNamespaces, clock.RealClock{})
}

func newLifecycleWithClock(immortalNamespaces sets.String, clock utilcache.Clock) (*Lifecycle, error) {
	forceLiveLookupCache := utilcache.NewLRUExpireCacheWithClock(100, clock)
	return &Lifecycle{
		Handler:              admission.NewHandler(admission.Create, admission.Update, admission.Delete),
		immortalNamespaces:   immortalNamespaces,
		forceLiveLookupCache: forceLiveLookupCache,
	}, nil
}

// SetExternalKubeInformerFactory implements the WantsExternalKubeInformerFactory interface.
func (l *Lifecycle) SetExternalKubeInformerFactory(f informers.SharedInformerFactory) {
	namespaceInformer := f.Core().V1().Namespaces()
	l.namespaceLister = namespaceInformer.Lister()
	l.SetReadyFunc(namespaceInformer.Informer().HasSynced)
}

// SetExternalKubeClientSet implements the WantsExternalKubeClientSet interface.
func (l *Lifecycle) SetExternalKubeClientSet(client kubernetes.Interface) {
	l.client = client
}

// ValidateInitialization implements the InitializationValidator interface.
func (l *Lifecycle) ValidateInitialization() error {
	if l.namespaceLister == nil {
		return fmt.Errorf("missing namespaceLister")
	}
	if l.client == nil {
		return fmt.Errorf("missing client")
	}
	return nil
}

// accessReviewResources are resources which give a view into permissions in a namespace.  Users must be allowed to create these
// resources because returning "not found" errors allows someone to search for the "people I'm going to fire in 2017" namespace.
var accessReviewResources = map[schema.GroupResource]bool{
	{Group: "authorization.k8s.io", Resource: "localsubjectaccessreviews"}: true,
}

func isAccessReview(a admission.Attributes) bool {
	return accessReviewResources[a.GetResource().GroupResource()]
}
<commit_msg>UPSTREAM: <carry>: kube-apiserver: add our immortal namespaces directly to admission plugin<commit_after>/*
Copyright 2015 The Kubernetes Authors.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package lifecycle

import (
	"context"
	"fmt"
	"io"
	"time"

	"k8s.io/klog"

	v1 "k8s.io/api/core/v1"
	"k8s.io/apimachinery/pkg/api/errors"
	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
	"k8s.io/apimachinery/pkg/runtime/schema"
	utilcache "k8s.io/apimachinery/pkg/util/cache"
	"k8s.io/apimachinery/pkg/util/clock"
	"k8s.io/apimachinery/pkg/util/sets"
	"k8s.io/apiserver/pkg/admission"
	"k8s.io/apiserver/pkg/admission/initializer"
	"k8s.io/client-go/informers"
	"k8s.io/client-go/kubernetes"
	corelisters "k8s.io/client-go/listers/core/v1"
)

const (
	// PluginName indicates the name of admission plug-in
	PluginName = "NamespaceLifecycle"
	// how long a namespace stays in the force live lookup cache before expiration.
	forceLiveLookupTTL = 30 * time.Second
	// how long to wait for a missing namespace before re-checking the cache (and then doing a live lookup)
	// this accomplishes two things:
	// 1. It allows a watch-fed cache time to observe a namespace creation event
	// 2. It allows time for a namespace creation to distribute to members of a storage cluster,
	//    so the live lookup has a better chance of succeeding even if it isn't performed against the leader.
	missingNamespaceWait = 50 * time.Millisecond
)

// Register registers a plugin
func Register(plugins *admission.Plugins) {
	plugins.Register(PluginName, func(config io.Reader) (admission.Interface, error) {
		return NewLifecycle(sets.NewString(metav1.NamespaceDefault, metav1.NamespaceSystem, metav1.NamespacePublic,
			// user specified configuration that cannot be rebuilt
			"openshift-config",
			// cluster generated configuration that cannot be rebuilt (etcd encryption keys)
			"openshift-config-managed",
			// the CVO which is the root we use to rebuild all the rest
			"openshift-cluster-version",
			// contains a namespaced list of all nodes in the cluster (yeah, weird.  they do it for multi-tenant management I think?)
			"openshift-machine-api",
		))
	})
}

// Lifecycle is an implementation of admission.Interface.
// It enforces life-cycle constraints around a Namespace depending on its Phase
type Lifecycle struct {
	*admission.Handler
	client             kubernetes.Interface
	immortalNamespaces sets.String
	namespaceLister    corelisters.NamespaceLister
	// forceLiveLookupCache holds a list of entries for namespaces that we have a strong reason to believe are stale in our local cache.
	// if a namespace is in this cache, then we will ignore our local state and always fetch latest from api server.
	forceLiveLookupCache *utilcache.LRUExpireCache
}

var _ = initializer.WantsExternalKubeInformerFactory(&Lifecycle{})
var _ = initializer.WantsExternalKubeClientSet(&Lifecycle{})

// Admit makes an admission decision based on the request attributes
func (l *Lifecycle) Admit(ctx context.Context, a admission.Attributes, o admission.ObjectInterfaces) error {
	// prevent deletion of immortal namespaces
	if a.GetOperation() == admission.Delete && a.GetKind().GroupKind() == v1.SchemeGroupVersion.WithKind("Namespace").GroupKind() && l.immortalNamespaces.Has(a.GetName()) {
		return errors.NewForbidden(a.GetResource().GroupResource(), a.GetName(), fmt.Errorf("this namespace may not be deleted"))
	}

	// always allow non-namespaced resources
	if len(a.GetNamespace()) == 0 && a.GetKind().GroupKind() != v1.SchemeGroupVersion.WithKind("Namespace").GroupKind() {
		return nil
	}

	if a.GetKind().GroupKind() == v1.SchemeGroupVersion.WithKind("Namespace").GroupKind() {
		// if a namespace is deleted, we want to prevent all further creates into it
		// while it is undergoing termination.  to reduce incidences where the cache
		// is slow to update, we add the namespace into a force live lookup list to ensure
		// we are not looking at stale state.
		if a.GetOperation() == admission.Delete {
			l.forceLiveLookupCache.Add(a.GetName(), true, forceLiveLookupTTL)
		}
		// allow all operations to namespaces
		return nil
	}

	// always allow deletion of other resources
	if a.GetOperation() == admission.Delete {
		return nil
	}

	// always allow access review checks.  Returning status about the namespace would be leaking information
	if isAccessReview(a) {
		return nil
	}

	// we need to wait for our caches to warm
	if !l.WaitForReady() {
		return admission.NewForbidden(a, fmt.Errorf("not yet ready to handle request"))
	}

	var (
		exists bool
		err    error
	)

	namespace, err := l.namespaceLister.Get(a.GetNamespace())
	if err != nil {
		if !errors.IsNotFound(err) {
			return errors.NewInternalError(err)
		}
	} else {
		exists = true
	}

	if !exists && a.GetOperation() == admission.Create {
		// give the cache time to observe the namespace before rejecting a create.
		// this helps when creating a namespace and immediately creating objects within it.
		time.Sleep(missingNamespaceWait)
		namespace, err = l.namespaceLister.Get(a.GetNamespace())
		switch {
		case errors.IsNotFound(err):
			// no-op
		case err != nil:
			return errors.NewInternalError(err)
		default:
			exists = true
		}
		if exists {
			klog.V(4).Infof("found %s in cache after waiting", a.GetNamespace())
		}
	}

	// forceLiveLookup if true will skip looking at local cache state and instead always make a live call to server.
	forceLiveLookup := false
	if _, ok := l.forceLiveLookupCache.Get(a.GetNamespace()); ok {
		// we think the namespace was marked for deletion, but our current local cache says otherwise, we will force a live lookup.
		forceLiveLookup = exists && namespace.Status.Phase == v1.NamespaceActive
	}

	// refuse to operate on non-existent namespaces
	if !exists || forceLiveLookup {
		// as a last resort, make a call directly to storage
		namespace, err = l.client.CoreV1().Namespaces().Get(context.TODO(), a.GetNamespace(), metav1.GetOptions{})
		switch {
		case errors.IsNotFound(err):
			return err
		case err != nil:
			return errors.NewInternalError(err)
		}
		klog.V(4).Infof("found %s via storage lookup", a.GetNamespace())
	}

	// ensure that we're not trying to create objects in terminating namespaces
	if a.GetOperation() == admission.Create {
		if namespace.Status.Phase != v1.NamespaceTerminating {
			return nil
		}

		err := admission.NewForbidden(a, fmt.Errorf("unable to create new content in namespace %s because it is being terminated", a.GetNamespace()))
		if apierr, ok := err.(*errors.StatusError); ok {
			apierr.ErrStatus.Details.Causes = append(apierr.ErrStatus.Details.Causes, metav1.StatusCause{
				Type:    v1.NamespaceTerminatingCause,
				Message: fmt.Sprintf("namespace %s is being terminated", a.GetNamespace()),
				Field:   "metadata.namespace",
			})
		}
		return err
	}

	return nil
}

// NewLifecycle creates a new namespace Lifecycle admission control handler
func NewLifecycle(immortalNamespaces sets.String) (*Lifecycle, error) {
	return newLifecycleWithClock(immortalNamespaces, clock.RealClock{})
}

func newLifecycleWithClock(immortalNamespaces sets.String, clock utilcache.Clock) (*Lifecycle, error) {
	forceLiveLookupCache := utilcache.NewLRUExpireCacheWithClock(100, clock)
	return &Lifecycle{
		Handler:              admission.NewHandler(admission.Create, admission.Update, admission.Delete),
		immortalNamespaces:   immortalNamespaces,
		forceLiveLookupCache: forceLiveLookupCache,
	}, nil
}

// SetExternalKubeInformerFactory implements the WantsExternalKubeInformerFactory interface.
func (l *Lifecycle) SetExternalKubeInformerFactory(f informers.SharedInformerFactory) {
	namespaceInformer := f.Core().V1().Namespaces()
	l.namespaceLister = namespaceInformer.Lister()
	l.SetReadyFunc(namespaceInformer.Informer().HasSynced)
}

// SetExternalKubeClientSet implements the WantsExternalKubeClientSet interface.
func (l *Lifecycle) SetExternalKubeClientSet(client kubernetes.Interface) {
	l.client = client
}

// ValidateInitialization implements the InitializationValidator interface.
func (l *Lifecycle) ValidateInitialization() error {
	if l.namespaceLister == nil {
		return fmt.Errorf("missing namespaceLister")
	}
	if l.client == nil {
		return fmt.Errorf("missing client")
	}
	return nil
}

// accessReviewResources are resources which give a view into permissions in a namespace.  Users must be allowed to create these
// resources because returning "not found" errors allows someone to search for the "people I'm going to fire in 2017" namespace.
var accessReviewResources = map[schema.GroupResource]bool{
	{Group: "authorization.k8s.io", Resource: "localsubjectaccessreviews"}: true,
}

func isAccessReview(a admission.Attributes) bool {
	return accessReviewResources[a.GetResource().GroupResource()]
}
<|endoftext|>


<commit_before>package gotumblr

type BasePost struct {
	Blog_name    string
	Id           int64
	Post_url     string
	PostType     string `json:"type"`
	Timestamp    int64
	Date         string
	Format       string
	Reblog_key   string
	Tags         []string
	Bookmarklet  bool
	Mobile       bool
	Source_url   string
	Source_title string
	Liked        bool
	State        string
	Total_Posts  int64
	Notes        []Note
}

type Note struct {
	Type                    string
	Timestamp               int64
	Blog_name               string
	Blog_uuid               string
	Blog_url                string
	Followed                bool
	Avatar_shape            string
	Post_id                 string
	Reblog_parent_blog_name string
}
<commit_msg>Add Note_count field to BasePost<commit_after>package gotumblr

type BasePost struct {
	Blog_name    string
	Id           int64
	Post_url     string
	PostType     string `json:"type"`
	Timestamp    int64
	Date         string
	Format       string
	Reblog_key   string
	Tags         []string
	Bookmarklet  bool
	Mobile       bool
	Source_url   string
	Source_title string
	Liked        bool
	State        string
	Total_Posts  int64
	Note_count   int64
	Notes        []Note
}

type Note struct {
	Type                    string
	Timestamp               int64
	Blog_name               string
	Blog_uuid               string
	Blog_url                string
	Followed                bool
	Avatar_shape            string
	Post_id                 string
	Reblog_parent_blog_name string
}
<|endoftext|>


<commit_before>package core

import (
	"context"
	"encoding/json"
	"expvar"
	"log"
	"net/http"
	"os"
	"os/exec"
	"strings"
	"syscall"
	"time"

	"chain/core/accesstoken"
	"chain/core/fetch"
	"chain/core/leader"
	"chain/core/mockhsm"
	"chain/core/txdb"
	"chain/crypto/ed25519"
	"chain/crypto/ed25519/chainkd"
	"chain/database/pg"
	"chain/database/sql"
	"chain/errors"
	"chain/net/http/httpjson"
	"chain/net/rpc"
	"chain/protocol"
	"chain/protocol/state"
)

var (
	errAlreadyConfigured = errors.New("core is already configured; must reset first")
	errUnconfigured      = errors.New("core is not configured")
	errBadGenerator      = errors.New("generator returned an unsuccessful response")
	errBadBlockXPub      = errors.New("supplied block xpub is invalid")
	errNoClientTokens    = errors.New("cannot enable client auth without client access tokens")
)

// reserved mockhsm key alias
const (
	networkRPCVersion = 1
	autoBlockKeyAlias = "_CHAIN_CORE_AUTO_BLOCK_KEY"
)

func isProduction() bool {
	return expvar.Get("buildtag").String() != `"dev"`
}

// errProdReset is returned when reset is called on a
// production system.
var errProdReset = errors.New("reset called on production system")

// Reset deletes all data, resulting in an unconfigured core.
// It must be called before any other functions in this package.
func Reset(ctx context.Context, db pg.DB) error {
	if isProduction() {
		return errors.Wrap(errProdReset)
	}

	const q = `
		TRUNCATE
			access_tokens,
			account_control_programs,
			account_utxos,
			accounts,
			annotated_accounts,
			annotated_assets,
			annotated_outputs,
			annotated_txs,
			asset_tags,
			assets,
			blocks,
			config,
			generator_pending_block,
			leader,
			mockhsm,
			pool_txs,
			query_blocks,
			reservations,
			signed_blocks,
			signers,
			snapshots,
			submitted_txs
			RESTART IDENTITY;
	`

	_, err := db.Exec(ctx, q)
	return errors.Wrap(err)
}

func (a *api) reset(ctx context.Context) error {
	if isProduction() {
		return errors.Wrap(errProdReset)
	}

	w := httpjson.ResponseWriter(ctx)
	closeConnOK(w)
	execSelf("RESET=true")
	panic("unreached")
}

func (a *api) info(ctx context.Context) (map[string]interface{}, error) {
	if a.config == nil {
		// never configured
		return map[string]interface{}{
			"is_configured": false,
		}, nil
	}
	if leader.IsLeading() {
		return a.leaderInfo(ctx)
	} else {
		var resp map[string]interface{}
		err := callLeader(ctx, "/info", nil, &resp)
		return resp, err
	}
}

func (a *api) leaderInfo(ctx context.Context) (map[string]interface{}, error) {
	localHeight := a.c.Height()
	var (
		generatorHeight  uint64
		generatorFetched time.Time
	)
	if a.config.IsGenerator {
		generatorHeight = localHeight
		generatorFetched = time.Now()
	} else {
		generatorHeight, generatorFetched = fetch.GeneratorHeight()
	}

	buildCommit := json.RawMessage(expvar.Get("buildcommit").String())
	buildDate := json.RawMessage(expvar.Get("builddate").String())

	// Because everything is asynchronous, it's possible for the localHeight to
	// be higher than our cached generator height. In that case, display the
	// generatorHeight as our height.
	if localHeight > generatorHeight {
		generatorHeight = localHeight
	}

	return map[string]interface{}{
		"is_configured":                     true,
		"configured_at":                     a.config.ConfiguredAt,
		"is_signer":                         a.config.IsSigner,
		"is_generator":                      a.config.IsGenerator,
		"generator_url":                     a.config.GeneratorURL,
		"blockchain_id":                     a.config.BlockchainID,
		"block_height":                      localHeight,
		"generator_block_height":            generatorHeight,
		"generator_block_height_fetched_at": generatorFetched,
		"is_production":                     isProduction(),
		"network_rpc_version":               networkRPCVersion,
		"build_commit":                      &buildCommit,
		"build_date":                        &buildDate,
		"require_client_access_tokens":      a.config.isClientAuthed(),
		"require_network_access_tokens":     a.config.isNetworkAuthed(),
	}, nil
}

// Configure configures the core by writing to the database.
// If running in a cored process,
// the caller must ensure that the new configuration is properly reloaded,
// for example by restarting the process.
//
// If c.IsSigner is true, Configure generates a new mockhsm keypair
// for signing blocks, and assigns it to c.BlockXPub.
//
// If c.IsGenerator is true, Configure creates an initial block,
// saves it, and assigns its hash to c.BlockchainID.
// Otherwise, c.IsGenerator is false, and Configure makes a test request
// to GeneratorURL to detect simple configuration mistakes.
func Configure(ctx context.Context, db pg.DB, c *Config) error {
	if c.isClientAuthed() {
		clientTokenExists, err := accesstoken.ClientTokenExists(ctx)
		if err != nil {
			return errors.Wrap(err)
		}
		if !clientTokenExists {
			return errors.Wrap(errNoClientTokens)
		}
	}

	var err error
	if !c.IsGenerator {
		err = tryGenerator(ctx, c.GeneratorURL, c.BlockchainID.String())
		if err != nil {
			return err
		}
	}

	var signingKeys []ed25519.PublicKey
	if c.IsSigner {
		var blockXPub chainkd.XPub
		if c.BlockXPub == "" {
			hsm := mockhsm.New(db)
			coreXPub, created, err := hsm.GetOrCreateKey(ctx, autoBlockKeyAlias)
			if err != nil {
				return err
			}
			blockXPub = coreXPub.XPub
			if created {
				log.Printf("Generated new block-signing key %s\n", blockXPub.String())
			} else {
				log.Printf("Using block-signing key %s\n", blockXPub.String())
			}
			c.BlockXPub = blockXPub.String()
		} else {
			var blockXPub chainkd.XPub
			err = blockXPub.UnmarshalText([]byte(c.BlockXPub))
			if err != nil {
				return errors.Wrap(errBadBlockXPub, err.Error())
			}
		}
		signingKeys = append(signingKeys, blockXPub.PublicKey())
	}

	if c.IsGenerator {
		block, err := protocol.NewInitialBlock(signingKeys, 1, time.Now())
		if err != nil {
			return err
		}
		store, pool := txdb.New(db.(*sql.DB))
		chain, err := protocol.NewChain(ctx, store, pool, nil)
		if err != nil {
			return err
		}

		err = chain.CommitBlock(ctx, block, state.Empty())
		if err != nil {
			return err
		}

		c.BlockchainID = block.Hash()
	}

	const q = `
		INSERT INTO config (is_signer, block_xpub, is_generator, blockchain_id, generator_url,
			network_authed, client_authed, configured_at)
		VALUES ($1, $2, $3, $4, $5, $6, $7, NOW())
	`
	_, err = db.Exec(
		ctx,
		q,
		c.IsSigner,
		c.BlockXPub,
		c.IsGenerator,
		c.BlockchainID,
		c.GeneratorURL,
		c.isNetworkAuthed(),
		c.isClientAuthed(),
	)
	return err
}

func configure(ctx context.Context, x *Config) error {
	err := Configure(ctx, pg.FromContext(ctx), x)
	if err != nil {
		return err
	}

	w := httpjson.ResponseWriter(ctx)
	closeConnOK(w)
	execSelf()
	panic("unreached")
}

// LoadConfig loads the stored configuration, if any, from the database/
func LoadConfig(ctx context.Context, db pg.DB) (*Config, error) {
	const q = `
			SELECT is_signer, is_generator, blockchain_id, generator_url, block_xpub,
			network_authed, client_authed, configured_at
			FROM config
		`

	c := new(Config)
	err := db.QueryRow(ctx, q).Scan(
		&c.IsSigner,
		&c.IsGenerator,
		&c.BlockchainID,
		&c.GeneratorURL,
		&c.BlockXPub,
		&c.NetworkAuthed,
		&c.ClientAuthed,
		&c.ConfiguredAt,
	)
	if err == sql.ErrNoRows {
		return nil, nil
	} else if err != nil {
		return nil, errors.Wrap(err, "fetching Core config")
	}
	return c, nil
}

func (a *api) updateConfig(ctx context.Context, x struct {
	ClientAuthed  *bool `json:"require_client_access_tokens"`
	NetworkAuthed *bool `json:"require_network_access_tokens"`
}) error {

	clientAuthed := a.config.isClientAuthed()
	networkAuthed := a.config.isNetworkAuthed()

	if x.ClientAuthed != nil {
		clientAuthed = *x.ClientAuthed
	}
	if x.NetworkAuthed != nil {
		networkAuthed = *x.NetworkAuthed
	}

	if clientAuthed {
		clientTokenExists, err := accesstoken.ClientTokenExists(ctx)
		if err != nil {
			return errors.Wrap(err)
		}
		if !clientTokenExists {
			return errors.Wrap(errNoClientTokens)
		}
	}

	const q = `UPDATE config SET client_authed=$1, network_authed=$2`
	_, err := pg.Exec(ctx, q, clientAuthed, networkAuthed)

	if err != nil {
		return err
	}

	a.config.setClientAuthed(clientAuthed)
	a.config.setNetworkAuthed(networkAuthed)

	return nil
}

func tryGenerator(ctx context.Context, url, blockchainID string) error {
	client := &rpc.Client{
		BaseURL:      url,
		BlockchainID: blockchainID,
	}
	var x struct {
		BlockHeight uint64 `json:"block_height"`
	}
	err := client.Call(ctx, "/rpc/block-height", nil, &x)
	if err != nil {
		return errors.Wrap(errBadGenerator, err.Error())
	}

	if x.BlockHeight < 1 {
		return errBadGenerator
	}

	return nil
}

func closeConnOK(w http.ResponseWriter) {
	w.Header().Add("Connection", "close")
	w.WriteHeader(http.StatusNoContent)

	hijacker, ok := w.(http.Hijacker)
	if !ok {
		log.Printf("no hijacker")
		return
	}
	conn, buf, err := hijacker.Hijack()
	if err != nil {
		log.Printf("could not hijack connection: %s\n", err)
		return
	}
	err = buf.Flush()
	if err != nil {
		log.Printf("could not flush connection buffer: %s\n", err)
	}
	err = conn.Close()
	if err != nil {
		log.Printf("could not close connection: %s\n", err)
	}
}

// execSelf execs Args with environment values replaced
// by the ones in env.
func execSelf(env ...string) {
	binpath, err := exec.LookPath(os.Args[0])
	if err != nil {
		panic(err)
	}

	env = mergeEnvLists(env, os.Environ())
	err = syscall.Exec(binpath, os.Args, env)
	if err != nil {
		panic(err)
	}
}

// mergeEnvLists merges the two environment lists such that
// variables with the same name in "in" replace those in "out".
// This always returns a newly allocated slice.
func mergeEnvLists(in, out []string) []string {
	out = append([]string(nil), out...)
NextVar:
	for _, inkv := range in {
		k := strings.SplitAfterN(inkv, "=", 2)[0]
		for i, outkv := range out {
			if strings.HasPrefix(outkv, k) {
				out[i] = inkv
				continue NextVar
			}
		}
		out = append(out, inkv)
	}
	return out
}
<commit_msg>core: clear txconsumers on /reset<commit_after>package core

import (
	"context"
	"encoding/json"
	"expvar"
	"log"
	"net/http"
	"os"
	"os/exec"
	"strings"
	"syscall"
	"time"

	"chain/core/accesstoken"
	"chain/core/fetch"
	"chain/core/leader"
	"chain/core/mockhsm"
	"chain/core/txdb"
	"chain/crypto/ed25519"
	"chain/crypto/ed25519/chainkd"
	"chain/database/pg"
	"chain/database/sql"
	"chain/errors"
	"chain/net/http/httpjson"
	"chain/net/rpc"
	"chain/protocol"
	"chain/protocol/state"
)

var (
	errAlreadyConfigured = errors.New("core is already configured; must reset first")
	errUnconfigured      = errors.New("core is not configured")
	errBadGenerator      = errors.New("generator returned an unsuccessful response")
	errBadBlockXPub      = errors.New("supplied block xpub is invalid")
	errNoClientTokens    = errors.New("cannot enable client auth without client access tokens")
)

// reserved mockhsm key alias
const (
	networkRPCVersion = 1
	autoBlockKeyAlias = "_CHAIN_CORE_AUTO_BLOCK_KEY"
)

func isProduction() bool {
	return expvar.Get("buildtag").String() != `"dev"`
}

// errProdReset is returned when reset is called on a
// production system.
var errProdReset = errors.New("reset called on production system")

// Reset deletes all data, resulting in an unconfigured core.
// It must be called before any other functions in this package.
func Reset(ctx context.Context, db pg.DB) error {
	if isProduction() {
		return errors.Wrap(errProdReset)
	}

	const q = `
		TRUNCATE
			access_tokens,
			account_control_programs,
			account_utxos,
			accounts,
			annotated_accounts,
			annotated_assets,
			annotated_outputs,
			annotated_txs,
			asset_tags,
			assets,
			blocks,
			config,
			generator_pending_block,
			leader,
			mockhsm,
			pool_txs,
			query_blocks,
			reservations,
			signed_blocks,
			signers,
			snapshots,
			submitted_txs,
			txconsumers
			RESTART IDENTITY;
	`

	_, err := db.Exec(ctx, q)
	return errors.Wrap(err)
}

func (a *api) reset(ctx context.Context) error {
	if isProduction() {
		return errors.Wrap(errProdReset)
	}

	w := httpjson.ResponseWriter(ctx)
	closeConnOK(w)
	execSelf("RESET=true")
	panic("unreached")
}

func (a *api) info(ctx context.Context) (map[string]interface{}, error) {
	if a.config == nil {
		// never configured
		return map[string]interface{}{
			"is_configured": false,
		}, nil
	}
	if leader.IsLeading() {
		return a.leaderInfo(ctx)
	} else {
		var resp map[string]interface{}
		err := callLeader(ctx, "/info", nil, &resp)
		return resp, err
	}
}

func (a *api) leaderInfo(ctx context.Context) (map[string]interface{}, error) {
	localHeight := a.c.Height()
	var (
		generatorHeight  uint64
		generatorFetched time.Time
	)
	if a.config.IsGenerator {
		generatorHeight = localHeight
		generatorFetched = time.Now()
	} else {
		generatorHeight, generatorFetched = fetch.GeneratorHeight()
	}

	buildCommit := json.RawMessage(expvar.Get("buildcommit").String())
	buildDate := json.RawMessage(expvar.Get("builddate").String())

	// Because everything is asynchronous, it's possible for the localHeight to
	// be higher than our cached generator height. In that case, display the
	// generatorHeight as our height.
	if localHeight > generatorHeight {
		generatorHeight = localHeight
	}

	return map[string]interface{}{
		"is_configured":                     true,
		"configured_at":                     a.config.ConfiguredAt,
		"is_signer":                         a.config.IsSigner,
		"is_generator":                      a.config.IsGenerator,
		"generator_url":                     a.config.GeneratorURL,
		"blockchain_id":                     a.config.BlockchainID,
		"block_height":                      localHeight,
		"generator_block_height":            generatorHeight,
		"generator_block_height_fetched_at": generatorFetched,
		"is_production":                     isProduction(),
		"network_rpc_version":               networkRPCVersion,
		"build_commit":                      &buildCommit,
		"build_date":                        &buildDate,
		"require_client_access_tokens":      a.config.isClientAuthed(),
		"require_network_access_tokens":     a.config.isNetworkAuthed(),
	}, nil
}

// Configure configures the core by writing to the database.
// If running in a cored process,
// the caller must ensure that the new configuration is properly reloaded,
// for example by restarting the process.
//
// If c.IsSigner is true, Configure generates a new mockhsm keypair
// for signing blocks, and assigns it to c.BlockXPub.
//
// If c.IsGenerator is true, Configure creates an initial block,
// saves it, and assigns its hash to c.BlockchainID.
// Otherwise, c.IsGenerator is false, and Configure makes a test request
// to GeneratorURL to detect simple configuration mistakes.
func Configure(ctx context.Context, db pg.DB, c *Config) error {
	if c.isClientAuthed() {
		clientTokenExists, err := accesstoken.ClientTokenExists(ctx)
		if err != nil {
			return errors.Wrap(err)
		}
		if !clientTokenExists {
			return errors.Wrap(errNoClientTokens)
		}
	}

	var err error
	if !c.IsGenerator {
		err = tryGenerator(ctx, c.GeneratorURL, c.BlockchainID.String())
		if err != nil {
			return err
		}
	}

	var signingKeys []ed25519.PublicKey
	if c.IsSigner {
		var blockXPub chainkd.XPub
		if c.BlockXPub == "" {
			hsm := mockhsm.New(db)
			coreXPub, created, err := hsm.GetOrCreateKey(ctx, autoBlockKeyAlias)
			if err != nil {
				return err
			}
			blockXPub = coreXPub.XPub
			if created {
				log.Printf("Generated new block-signing key %s\n", blockXPub.String())
			} else {
				log.Printf("Using block-signing key %s\n", blockXPub.String())
			}
			c.BlockXPub = blockXPub.String()
		} else {
			var blockXPub chainkd.XPub
			err = blockXPub.UnmarshalText([]byte(c.BlockXPub))
			if err != nil {
				return errors.Wrap(errBadBlockXPub, err.Error())
			}
		}
		signingKeys = append(signingKeys, blockXPub.PublicKey())
	}

	if c.IsGenerator {
		block, err := protocol.NewInitialBlock(signingKeys, 1, time.Now())
		if err != nil {
			return err
		}
		store, pool := txdb.New(db.(*sql.DB))
		chain, err := protocol.NewChain(ctx, store, pool, nil)
		if err != nil {
			return err
		}

		err = chain.CommitBlock(ctx, block, state.Empty())
		if err != nil {
			return err
		}

		c.BlockchainID = block.Hash()
	}

	const q = `
		INSERT INTO config (is_signer, block_xpub, is_generator, blockchain_id, generator_url,
			network_authed, client_authed, configured_at)
		VALUES ($1, $2, $3, $4, $5, $6, $7, NOW())
	`
	_, err = db.Exec(
		ctx,
		q,
		c.IsSigner,
		c.BlockXPub,
		c.IsGenerator,
		c.BlockchainID,
		c.GeneratorURL,
		c.isNetworkAuthed(),
		c.isClientAuthed(),
	)
	return err
}

func configure(ctx context.Context, x *Config) error {
	err := Configure(ctx, pg.FromContext(ctx), x)
	if err != nil {
		return err
	}

	w := httpjson.ResponseWriter(ctx)
	closeConnOK(w)
	execSelf()
	panic("unreached")
}

// LoadConfig loads the stored configuration, if any, from the database/
func LoadConfig(ctx context.Context, db pg.DB) (*Config, error) {
	const q = `
			SELECT is_signer, is_generator, blockchain_id, generator_url, block_xpub,
			network_authed, client_authed, configured_at
			FROM config
		`

	c := new(Config)
	err := db.QueryRow(ctx, q).Scan(
		&c.IsSigner,
		&c.IsGenerator,
		&c.BlockchainID,
		&c.GeneratorURL,
		&c.BlockXPub,
		&c.NetworkAuthed,
		&c.ClientAuthed,
		&c.ConfiguredAt,
	)
	if err == sql.ErrNoRows {
		return nil, nil
	} else if err != nil {
		return nil, errors.Wrap(err, "fetching Core config")
	}
	return c, nil
}

func (a *api) updateConfig(ctx context.Context, x struct {
	ClientAuthed  *bool `json:"require_client_access_tokens"`
	NetworkAuthed *bool `json:"require_network_access_tokens"`
}) error {

	clientAuthed := a.config.isClientAuthed()
	networkAuthed := a.config.isNetworkAuthed()

	if x.ClientAuthed != nil {
		clientAuthed = *x.ClientAuthed
	}
	if x.NetworkAuthed != nil {
		networkAuthed = *x.NetworkAuthed
	}

	if clientAuthed {
		clientTokenExists, err := accesstoken.ClientTokenExists(ctx)
		if err != nil {
			return errors.Wrap(err)
		}
		if !clientTokenExists {
			return errors.Wrap(errNoClientTokens)
		}
	}

	const q = `UPDATE config SET client_authed=$1, network_authed=$2`
	_, err := pg.Exec(ctx, q, clientAuthed, networkAuthed)

	if err != nil {
		return err
	}

	a.config.setClientAuthed(clientAuthed)
	a.config.setNetworkAuthed(networkAuthed)

	return nil
}

func tryGenerator(ctx context.Context, url, blockchainID string) error {
	client := &rpc.Client{
		BaseURL:      url,
		BlockchainID: blockchainID,
	}
	var x struct {
		BlockHeight uint64 `json:"block_height"`
	}
	err := client.Call(ctx, "/rpc/block-height", nil, &x)
	if err != nil {
		return errors.Wrap(errBadGenerator, err.Error())
	}

	if x.BlockHeight < 1 {
		return errBadGenerator
	}

	return nil
}

func closeConnOK(w http.ResponseWriter) {
	w.Header().Add("Connection", "close")
	w.WriteHeader(http.StatusNoContent)

	hijacker, ok := w.(http.Hijacker)
	if !ok {
		log.Printf("no hijacker")
		return
	}
	conn, buf, err := hijacker.Hijack()
	if err != nil {
		log.Printf("could not hijack connection: %s\n", err)
		return
	}
	err = buf.Flush()
	if err != nil {
		log.Printf("could not flush connection buffer: %s\n", err)
	}
	err = conn.Close()
	if err != nil {
		log.Printf("could not close connection: %s\n", err)
	}
}

// execSelf execs Args with environment values replaced
// by the ones in env.
func execSelf(env ...string) {
	binpath, err := exec.LookPath(os.Args[0])
	if err != nil {
		panic(err)
	}

	env = mergeEnvLists(env, os.Environ())
	err = syscall.Exec(binpath, os.Args, env)
	if err != nil {
		panic(err)
	}
}

// mergeEnvLists merges the two environment lists such that
// variables with the same name in "in" replace those in "out".
// This always returns a newly allocated slice.
func mergeEnvLists(in, out []string) []string {
	out = append([]string(nil), out...)
NextVar:
	for _, inkv := range in {
		k := strings.SplitAfterN(inkv, "=", 2)[0]
		for i, outkv := range out {
			if strings.HasPrefix(outkv, k) {
				out[i] = inkv
				continue NextVar
			}
		}
		out = append(out, inkv)
	}
	return out
}
<|endoftext|>


<commit_before>/*
// Copyright (c) 2016 Intel Corporation
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//      http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
*/

package main

import (
	"encoding/binary"
	"fmt"
	"net"
	"os"

	"golang.org/x/net/context"

	"github.com/golang/glog"

	"github.com/01org/ciao/networking/libsnnet"
	"github.com/01org/ciao/payloads"
	"github.com/01org/ciao/ssntp"
)

var cnNet *libsnnet.ComputeNode
var hostname string
var nicInfo []*payloads.NetworkStat
var dockerNet *libsnnet.DockerPlugin

func initNetworkPhase1() error {

	cn := &libsnnet.ComputeNode{}
	var mnetList []net.IPNet
	var cnetList []net.IPNet

	if computeNet != "" {
		_, cnet, _ := net.ParseCIDR(computeNet)
		if cnet == nil {
			return fmt.Errorf("Unable to Parse CIDR :" + computeNet)
		}
		cnetList = []net.IPNet{*cnet}
	}

	if mgmtNet != "" {
		_, mnet, _ := net.ParseCIDR(mgmtNet)
		if mnet == nil {
			return fmt.Errorf("Unable to Parse CIDR :" + mgmtNet)
		}
		mnetList = []net.IPNet{*mnet}
	}

	cn.NetworkConfig = &libsnnet.NetworkConfig{
		ManagementNet: mnetList,
		ComputeNet:    cnetList,
		Mode:          libsnnet.GreTunnel,
	}

	if err := cn.Init(); err != nil {
		return err
	}

	cnNet = cn

	return nil
}

func initDockerNetworking(ctx context.Context) error {
	err := checkDockerServerVersion("1.10.0", ctx)
	if err != nil {
		glog.Warningf("Docker Version check failed: %v", err)
		return err
	}

	dockerPlugin := libsnnet.NewDockerPlugin()
	if err := dockerPlugin.Init(); err != nil {
		glog.Warningf("Docker Init failed: %v", err)
		return err
	}

	if err := dockerPlugin.Start(); err != nil {
		dockerPlugin.Close()
		glog.Warningf("Docker start failed: %v ", err)
		return err
	}

	dockerNet = dockerPlugin

	return nil
}

func shutdownNetwork() {
	if dockerNet == nil {
		return
	}

	if err := dockerNet.Stop(); err != nil {
		glog.Warningf("Docker stop failed: %v", err)
	}

	if err := dockerNet.Close(); err != nil {
		glog.Warningf("Docker close failed: %v", err)
	}

	glog.Infof("Docker networking shutdown successfully")
}

func initNetwork(ctx context.Context) error {

	if err := initNetworkPhase1(); err != nil {
		return err
	}

	if err := initDockerNetworking(ctx); err != nil {
		glog.Warning("Unable to initialise docker networking")
	}

	if err := cnNet.DbRebuild(nil); err != nil {
		return err
	}

	limit := len(cnNet.ComputeAddr)
	if len(cnNet.ComputeLink) < limit {
		limit = len(cnNet.ComputeLink)
	}

	for i := 0; i < limit; i++ {
		nicInfo = append(nicInfo, &payloads.NetworkStat{
			NodeIP:  cnNet.ComputeAddr[i].IP.String(),
			NodeMAC: cnNet.ComputeLink[i].Attrs().HardwareAddr.String(),
		})
		glog.Infof("Network card %d Info", i)
		glog.Infof("  IP address of node is %s", nicInfo[i].NodeIP)
		glog.Infof("  MAC address of node is %s", nicInfo[i].NodeMAC)
	}

	if len(nicInfo) == 0 {
		glog.Warning("Unable to determine IP address. Should not happen")
	}

	var err error
	hostname, err = os.Hostname()
	if err == nil {
		glog.Infof("Hostname of node is %s", hostname)
	} else {
		glog.Warning("Unable to determine hostname %s", err)
	}

	return nil
}

func initNetworking(ctx context.Context) chan error {
	ch := make(chan error)
	go func() {
		err := initNetwork(ctx)
		ch <- err
	}()
	return ch
}

func createCNVnicCfg(cfg *vmConfig) (*libsnnet.VnicConfig, error) {

	glog.Info("Creating CN Vnic CFG")

	mac, err := net.ParseMAC(cfg.VnicMAC)
	if err != nil {
		return nil, fmt.Errorf("Invalid mac address %v", err)
	}

	_, vnet, err := net.ParseCIDR(cfg.SubnetIP)
	if err != nil {
		return nil, fmt.Errorf("Invalid vnic subnet %v", err)
	}

	concIP := net.ParseIP(cfg.ConcIP)
	if concIP == nil {
		return nil, fmt.Errorf("Invalid concentrator ip %s", cfg.ConcIP)
	}

	vnicIP := net.ParseIP(cfg.VnicIP)
	if vnicIP == nil {
		return nil, fmt.Errorf("Invalid vnicIP ip %s", cfg.VnicIP)
	}

	subnetKey := binary.LittleEndian.Uint32(vnet.IP)
	var role libsnnet.VnicRole
	if cfg.Container {
		role = libsnnet.TenantContainer
	} else {
		role = libsnnet.TenantVM
	}

	return &libsnnet.VnicConfig{
		VnicRole:   role,
		VnicIP:     vnicIP,
		ConcIP:     concIP,
		VnicMAC:    mac,
		Subnet:     *vnet,
		SubnetKey:  int(subnetKey),
		VnicID:     cfg.VnicUUID,
		InstanceID: cfg.Instance,
		TenantID:   cfg.TennantUUID,
		SubnetID:   cfg.SubnetIP,
		ConcID:     cfg.ConcUUID}, nil
}

func createCNCIVnicCfg(cfg *vmConfig) (*libsnnet.VnicConfig, error) {

	glog.Info("Creating CNCI Vnic CFG")

	mac, err := net.ParseMAC(cfg.VnicMAC)
	if err != nil {
		return nil, fmt.Errorf("Invalid mac address %v", err)
	}

	return &libsnnet.VnicConfig{
		VnicRole:   libsnnet.DataCenter,
		VnicMAC:    mac,
		VnicID:     cfg.VnicUUID,
		InstanceID: cfg.Instance,
		TenantID:   cfg.TennantUUID}, nil
}

func createVnicCfg(cfg *vmConfig) (*libsnnet.VnicConfig, error) {
	if cfg.NetworkNode {
		return createCNCIVnicCfg(cfg)
	}

	return createCNVnicCfg(cfg)
}

func sendNetworkEvent(client *ssntpConn, eventType ssntp.Event,
	event *libsnnet.SsntpEventInfo) {

	if event == nil || !client.isConnected() {
		return
	}

	payload, err := generateNetEventPayload(event, client.UUID())
	if err != nil {
		glog.Warningf("Unable parse ssntpEvent %s", err)
		return
	}

	_, err = client.SendEvent(eventType, payload)
	if err != nil {
		glog.Warningf("Unable to send %s", event)
	}
}

func createVnic(client *ssntpConn, vnicCfg *libsnnet.VnicConfig) (string, string, error) {
	var name string
	var bridge string

	//BUG(markus): This function needs a context parameter

	if vnicCfg.VnicRole != libsnnet.DataCenter {
		var vnic *libsnnet.Vnic
		var event *libsnnet.SsntpEventInfo
		var info *libsnnet.ContainerInfo
		var err error
		if vnicCfg.VnicRole == libsnnet.TenantContainer {
			vnic, event, info, err = createDockerVnicV2(vnicCfg)
			bridge = info.SubnetID
		} else {
			vnic, event, info, err = cnNet.CreateVnicV2(vnicCfg)
		}
		if err != nil {
			glog.Errorf("cn.CreateVnic failed %v", err)
			return "", "", err
		}
		sendNetworkEvent(client, ssntp.TenantAdded, event)
		name = vnic.LinkName
		glog.Infoln("CN VNIC created =", name, info, event)
	} else {
		vnic, err := cnNet.CreateCnciVnic(vnicCfg)
		if err != nil {
			glog.Errorf("cn.CreateCnciVnic failed %v", err)
			return "", "", err
		}
		name = vnic.LinkName
		glog.Infoln("CNCI VNIC created =", name)
	}

	return name, bridge, nil
}

func destroyVnic(client *ssntpConn, vnicCfg *libsnnet.VnicConfig) error {
	if vnicCfg.VnicRole != libsnnet.DataCenter {
		var event *libsnnet.SsntpEventInfo
		var err error

		if vnicCfg.VnicRole == libsnnet.TenantContainer {
			event, err = destroyDockerVnicV2(vnicCfg)
		} else {
			event, _, err = cnNet.DestroyVnicV2(vnicCfg)
		}
		if err != nil {
			glog.Errorf("cn.DestroyVnic failed %v", err)
			return err
		}

		sendNetworkEvent(client, ssntp.TenantRemoved, event)

		glog.Infoln("CN VNIC Destroyed =", vnicCfg.VnicIP, event)
	} else {
		err := cnNet.DestroyCnciVnic(vnicCfg)
		if err != nil {
			glog.Errorf("cn.DestroyCnciVnic failed %v", err)
			return err
		}

		glog.Infoln("CNCI VNIC Destroyed =", vnicCfg.VnicIP)
	}

	return nil
}

func getNodeIPAddress() string {
	if len(nicInfo) == 0 {
		return "127.0.0.1"
	}

	return nicInfo[0].NodeIP
}
<commit_msg>launcher: networking: Set CnMaxAPIConcurrency to 1<commit_after>/*
// Copyright (c) 2016 Intel Corporation
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//      http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
*/

package main

import (
	"encoding/binary"
	"fmt"
	"net"
	"os"

	"golang.org/x/net/context"

	"github.com/golang/glog"

	"github.com/01org/ciao/networking/libsnnet"
	"github.com/01org/ciao/payloads"
	"github.com/01org/ciao/ssntp"
)

var cnNet *libsnnet.ComputeNode
var hostname string
var nicInfo []*payloads.NetworkStat
var dockerNet *libsnnet.DockerPlugin

func initNetworkPhase1() error {

	cn := &libsnnet.ComputeNode{}
	var mnetList []net.IPNet
	var cnetList []net.IPNet

	if computeNet != "" {
		_, cnet, _ := net.ParseCIDR(computeNet)
		if cnet == nil {
			return fmt.Errorf("Unable to Parse CIDR :" + computeNet)
		}
		cnetList = []net.IPNet{*cnet}
	}

	if mgmtNet != "" {
		_, mnet, _ := net.ParseCIDR(mgmtNet)
		if mnet == nil {
			return fmt.Errorf("Unable to Parse CIDR :" + mgmtNet)
		}
		mnetList = []net.IPNet{*mnet}
	}

	cn.NetworkConfig = &libsnnet.NetworkConfig{
		ManagementNet: mnetList,
		ComputeNet:    cnetList,
		Mode:          libsnnet.GreTunnel,
	}

	libsnnet.CnMaxAPIConcurrency = 1
	if err := cn.Init(); err != nil {
		return err
	}

	cnNet = cn

	return nil
}

func initDockerNetworking(ctx context.Context) error {
	err := checkDockerServerVersion("1.10.0", ctx)
	if err != nil {
		glog.Warningf("Docker Version check failed: %v", err)
		return err
	}

	dockerPlugin := libsnnet.NewDockerPlugin()
	if err := dockerPlugin.Init(); err != nil {
		glog.Warningf("Docker Init failed: %v", err)
		return err
	}

	if err := dockerPlugin.Start(); err != nil {
		dockerPlugin.Close()
		glog.Warningf("Docker start failed: %v ", err)
		return err
	}

	dockerNet = dockerPlugin

	return nil
}

func shutdownNetwork() {
	if dockerNet == nil {
		return
	}

	if err := dockerNet.Stop(); err != nil {
		glog.Warningf("Docker stop failed: %v", err)
	}

	if err := dockerNet.Close(); err != nil {
		glog.Warningf("Docker close failed: %v", err)
	}

	glog.Infof("Docker networking shutdown successfully")
}

func initNetwork(ctx context.Context) error {

	if err := initNetworkPhase1(); err != nil {
		return err
	}

	if err := initDockerNetworking(ctx); err != nil {
		glog.Warning("Unable to initialise docker networking")
	}

	if err := cnNet.DbRebuild(nil); err != nil {
		return err
	}

	limit := len(cnNet.ComputeAddr)
	if len(cnNet.ComputeLink) < limit {
		limit = len(cnNet.ComputeLink)
	}

	for i := 0; i < limit; i++ {
		nicInfo = append(nicInfo, &payloads.NetworkStat{
			NodeIP:  cnNet.ComputeAddr[i].IP.String(),
			NodeMAC: cnNet.ComputeLink[i].Attrs().HardwareAddr.String(),
		})
		glog.Infof("Network card %d Info", i)
		glog.Infof("  IP address of node is %s", nicInfo[i].NodeIP)
		glog.Infof("  MAC address of node is %s", nicInfo[i].NodeMAC)
	}

	if len(nicInfo) == 0 {
		glog.Warning("Unable to determine IP address. Should not happen")
	}

	var err error
	hostname, err = os.Hostname()
	if err == nil {
		glog.Infof("Hostname of node is %s", hostname)
	} else {
		glog.Warning("Unable to determine hostname %s", err)
	}

	return nil
}

func initNetworking(ctx context.Context) chan error {
	ch := make(chan error)
	go func() {
		err := initNetwork(ctx)
		ch <- err
	}()
	return ch
}

func createCNVnicCfg(cfg *vmConfig) (*libsnnet.VnicConfig, error) {

	glog.Info("Creating CN Vnic CFG")

	mac, err := net.ParseMAC(cfg.VnicMAC)
	if err != nil {
		return nil, fmt.Errorf("Invalid mac address %v", err)
	}

	_, vnet, err := net.ParseCIDR(cfg.SubnetIP)
	if err != nil {
		return nil, fmt.Errorf("Invalid vnic subnet %v", err)
	}

	concIP := net.ParseIP(cfg.ConcIP)
	if concIP == nil {
		return nil, fmt.Errorf("Invalid concentrator ip %s", cfg.ConcIP)
	}

	vnicIP := net.ParseIP(cfg.VnicIP)
	if vnicIP == nil {
		return nil, fmt.Errorf("Invalid vnicIP ip %s", cfg.VnicIP)
	}

	subnetKey := binary.LittleEndian.Uint32(vnet.IP)
	var role libsnnet.VnicRole
	if cfg.Container {
		role = libsnnet.TenantContainer
	} else {
		role = libsnnet.TenantVM
	}

	return &libsnnet.VnicConfig{
		VnicRole:   role,
		VnicIP:     vnicIP,
		ConcIP:     concIP,
		VnicMAC:    mac,
		Subnet:     *vnet,
		SubnetKey:  int(subnetKey),
		VnicID:     cfg.VnicUUID,
		InstanceID: cfg.Instance,
		TenantID:   cfg.TennantUUID,
		SubnetID:   cfg.SubnetIP,
		ConcID:     cfg.ConcUUID}, nil
}

func createCNCIVnicCfg(cfg *vmConfig) (*libsnnet.VnicConfig, error) {

	glog.Info("Creating CNCI Vnic CFG")

	mac, err := net.ParseMAC(cfg.VnicMAC)
	if err != nil {
		return nil, fmt.Errorf("Invalid mac address %v", err)
	}

	return &libsnnet.VnicConfig{
		VnicRole:   libsnnet.DataCenter,
		VnicMAC:    mac,
		VnicID:     cfg.VnicUUID,
		InstanceID: cfg.Instance,
		TenantID:   cfg.TennantUUID}, nil
}

func createVnicCfg(cfg *vmConfig) (*libsnnet.VnicConfig, error) {
	if cfg.NetworkNode {
		return createCNCIVnicCfg(cfg)
	}

	return createCNVnicCfg(cfg)
}

func sendNetworkEvent(client *ssntpConn, eventType ssntp.Event,
	event *libsnnet.SsntpEventInfo) {

	if event == nil || !client.isConnected() {
		return
	}

	payload, err := generateNetEventPayload(event, client.UUID())
	if err != nil {
		glog.Warningf("Unable parse ssntpEvent %s", err)
		return
	}

	_, err = client.SendEvent(eventType, payload)
	if err != nil {
		glog.Warningf("Unable to send %s", event)
	}
}

func createVnic(client *ssntpConn, vnicCfg *libsnnet.VnicConfig) (string, string, error) {
	var name string
	var bridge string

	//BUG(markus): This function needs a context parameter

	if vnicCfg.VnicRole != libsnnet.DataCenter {
		var vnic *libsnnet.Vnic
		var event *libsnnet.SsntpEventInfo
		var info *libsnnet.ContainerInfo
		var err error
		if vnicCfg.VnicRole == libsnnet.TenantContainer {
			vnic, event, info, err = createDockerVnicV2(vnicCfg)
			bridge = info.SubnetID
		} else {
			vnic, event, info, err = cnNet.CreateVnicV2(vnicCfg)
		}
		if err != nil {
			glog.Errorf("cn.CreateVnic failed %v", err)
			return "", "", err
		}
		sendNetworkEvent(client, ssntp.TenantAdded, event)
		name = vnic.LinkName
		glog.Infoln("CN VNIC created =", name, info, event)
	} else {
		vnic, err := cnNet.CreateCnciVnic(vnicCfg)
		if err != nil {
			glog.Errorf("cn.CreateCnciVnic failed %v", err)
			return "", "", err
		}
		name = vnic.LinkName
		glog.Infoln("CNCI VNIC created =", name)
	}

	return name, bridge, nil
}

func destroyVnic(client *ssntpConn, vnicCfg *libsnnet.VnicConfig) error {
	if vnicCfg.VnicRole != libsnnet.DataCenter {
		var event *libsnnet.SsntpEventInfo
		var err error

		if vnicCfg.VnicRole == libsnnet.TenantContainer {
			event, err = destroyDockerVnicV2(vnicCfg)
		} else {
			event, _, err = cnNet.DestroyVnicV2(vnicCfg)
		}
		if err != nil {
			glog.Errorf("cn.DestroyVnic failed %v", err)
			return err
		}

		sendNetworkEvent(client, ssntp.TenantRemoved, event)

		glog.Infoln("CN VNIC Destroyed =", vnicCfg.VnicIP, event)
	} else {
		err := cnNet.DestroyCnciVnic(vnicCfg)
		if err != nil {
			glog.Errorf("cn.DestroyCnciVnic failed %v", err)
			return err
		}

		glog.Infoln("CNCI VNIC Destroyed =", vnicCfg.VnicIP)
	}

	return nil
}

func getNodeIPAddress() string {
	if len(nicInfo) == 0 {
		return "127.0.0.1"
	}

	return nicInfo[0].NodeIP
}
<|endoftext|>


<commit_before>// (c) 2021, Ava Labs, Inc. All rights reserved.
// See the file LICENSE for licensing terms.

package main

import (
	"fmt"
	"os"
	"syscall"

	"github.com/ava-labs/avalanchego/app/process"
	"github.com/ava-labs/avalanchego/config"
	"github.com/ava-labs/avalanchego/utils"
	"github.com/ava-labs/avalanchego/utils/logging"
)

// GitCommit should be optionally set at compile time.
var GitCommit string

// main is the entry point to AvalancheGo.
func main() {
	// Get the config
	nodeConfig, processConfig, err := config.GetConfigs(GitCommit)
	if err != nil {
		fmt.Printf("couldn't get config: %s", err)
		os.Exit(1)
	}

	if processConfig.DisplayVersionAndExit {
		fmt.Print(processConfig.VersionStr)
		os.Exit(0)
	}

	fmt.Println(process.Header)

	// Set the log directory for this process by adding a suffix
	// "-daemon" to the log directory given in the config
	logConfigCopy := nodeConfig.LoggingConfig
	logConfigCopy.Directory += "-daemon"
	logFactory := logging.NewFactory(logConfigCopy)

	log, err := logFactory.Make("main")
	if err != nil {
		logFactory.Close()

		fmt.Printf("starting logger failed with: %s\n", err)
		os.Exit(1)
	}

	log.Info("using build directory at path '%s'", processConfig.BuildDir)

	nodeManager := newNodeManager(processConfig.BuildDir, log)
	_ = utils.HandleSignals(
		func(os.Signal) {
			// SIGINT and SIGTERM cause all running nodes to stop
			nodeManager.shutdown()
		},
		syscall.SIGINT, syscall.SIGTERM,
	)

	// Migrate the database if necessary
	migrationManager := newMigrationManager(nodeManager, nodeConfig, log)
	if err := migrationManager.migrate(); err != nil {
		log.Error("error while running migration: %s", err)
		nodeManager.shutdown()
	}

	// Run normally
	exitCode, err := nodeManager.runNormal()
	if err != nil {
		log.Debug("running node returned error: %s", err)
	} else {
		log.Debug("node returned exit code %d", exitCode)
	}

	nodeManager.shutdown() // make sure all the nodes are stopped

	logFactory.Close()
	os.Exit(exitCode)
}
<commit_msg>use error instead of debug<commit_after>// (c) 2021, Ava Labs, Inc. All rights reserved.
// See the file LICENSE for licensing terms.

package main

import (
	"fmt"
	"os"
	"syscall"

	"github.com/ava-labs/avalanchego/app/process"
	"github.com/ava-labs/avalanchego/config"
	"github.com/ava-labs/avalanchego/utils"
	"github.com/ava-labs/avalanchego/utils/logging"
)

// GitCommit should be optionally set at compile time.
var GitCommit string

// main is the entry point to AvalancheGo.
func main() {
	// Get the config
	nodeConfig, processConfig, err := config.GetConfigs(GitCommit)
	if err != nil {
		fmt.Printf("couldn't get config: %s", err)
		os.Exit(1)
	}

	if processConfig.DisplayVersionAndExit {
		fmt.Print(processConfig.VersionStr)
		os.Exit(0)
	}

	fmt.Println(process.Header)

	// Set the log directory for this process by adding a suffix
	// "-daemon" to the log directory given in the config
	logConfigCopy := nodeConfig.LoggingConfig
	logConfigCopy.Directory += "-daemon"
	logFactory := logging.NewFactory(logConfigCopy)

	log, err := logFactory.Make("main")
	if err != nil {
		logFactory.Close()

		fmt.Printf("starting logger failed with: %s\n", err)
		os.Exit(1)
	}

	log.Info("using build directory at path '%s'", processConfig.BuildDir)

	nodeManager := newNodeManager(processConfig.BuildDir, log)
	_ = utils.HandleSignals(
		func(os.Signal) {
			// SIGINT and SIGTERM cause all running nodes to stop
			nodeManager.shutdown()
		},
		syscall.SIGINT, syscall.SIGTERM,
	)

	// Migrate the database if necessary
	migrationManager := newMigrationManager(nodeManager, nodeConfig, log)
	if err := migrationManager.migrate(); err != nil {
		log.Error("error while running migration: %s", err)
		nodeManager.shutdown()
	}

	// Run normally
	exitCode, err := nodeManager.runNormal()
	if err != nil {
		log.Error("running node returned error: %s", err)
	} else {
		log.Debug("node returned exit code %d", exitCode)
	}

	nodeManager.shutdown() // make sure all the nodes are stopped

	logFactory.Close()
	os.Exit(exitCode)
}
<|endoftext|>


<commit_before>package main

import (
	_ "expvar"
	"flag"
	"github.com/couchbaselabs/logg"
	"github.com/tleyden/checkerlution"
	cbot "github.com/tleyden/checkers-bot"
	ng "github.com/tleyden/neurgo"
	"net/http"
)

func init() {
	logg.LogKeys["MAIN"] = true
	logg.LogKeys["DEBUG"] = true
	logg.LogKeys["CHECKERLUTION"] = true
	logg.LogKeys["CHECKERLUTION_SCAPE"] = true
	logg.LogKeys["CHECKERSBOT"] = true
	logg.LogKeys["NEURGO"] = false
	logg.LogKeys["NEURVOLVE"] = true
	logg.LogKeys["SENSOR_SYNC"] = false
	logg.LogKeys["ACTUATOR_SYNC"] = false
	logg.LogKeys["NODE_PRE_SEND"] = false
	logg.LogKeys["NODE_POST_SEND"] = false
	logg.LogKeys["NODE_POST_RECV"] = false
	logg.LogKeys["NODE_STATE"] = false
	ng.SeedRandom()
}

func train(checkersBotFlags cbot.CheckersBotFlags) {

	// run a webserver in order to view expvar output
	// at http://localhost:8080/debug/vars
	go http.ListenAndServe(":8080", nil)

	trainer := &checkerlution.CheckerlutionTrainer{}

	// checkerlution.RunTopologyMutatingTrainer()
	trainer.RunPopulationTrainer(checkersBotFlags)

}

func run(checkersBotFlags cbot.CheckersBotFlags) {

	LOAD_CORTEX_FROM_FILE := false

	thinker := &checkerlution.Checkerlution{}
	thinker.SetMode(checkerlution.RUNNING_MODE)

	if LOAD_CORTEX_FROM_FILE {
		filename := "checkerlution_trained.json"
		cortex, err := ng.NewCortexFromJSONFile(filename)
		if err != nil {
			logg.LogPanic("Error reading cortex from: %v.  Err: %v", filename, err)
		}
		thinker.StartWithCortex(cortex, checkersBotFlags.Team)

	} else {
		thinker.Start(checkersBotFlags.Team)
	}

	game := cbot.NewGame(checkersBotFlags.Team, thinker)
	game.SetServerUrl(checkersBotFlags.SyncGatewayUrl)
	game.SetFeedType(checkersBotFlags.FeedType)
	game.SetDelayBeforeMove(checkersBotFlags.RandomDelayBeforeMove)

	logg.LogTo("CHECKERLUTION", "Starting game loop")
	game.GameLoop()
	logg.LogTo("CHECKERLUTION", "Game loop finished")

}

func main() {

	checkersBotRawFlags := cbot.GetCheckersBotRawFlags()

	var mode int

	flag.IntVar(
		&mode,
		"mode",
		0,
		"Run Mode - Run: 0 Train: 1",
	)

	flag.Parse()

	checkersBotFlags := checkersBotRawFlags.GetCheckersBotFlags()

	logg.LogTo("CHECKERLUTION", "Flags: %v", checkersBotFlags)
	logg.LogTo("CHECKERLUTION", "Mode: %v", mode)

	if mode == 0 {
		logg.LogTo("CHECKERLUTION", "Run mode: Run")
		run(checkersBotFlags)
	} else {
		logg.LogTo("CHECKERLUTION", "Run mode: Train")
		train(checkersBotFlags)
	}

}
<commit_msg>load neural net from file or from db if a uuid is passed in<commit_after>package main

import (
	_ "expvar"
	"flag"
	"fmt"
	"github.com/couchbaselabs/logg"
	"github.com/tleyden/checkerlution"
	cbot "github.com/tleyden/checkers-bot"
	"github.com/tleyden/go-couch"
	ng "github.com/tleyden/neurgo"
	"net/http"
)

func init() {
	logg.LogKeys["MAIN"] = true
	logg.LogKeys["DEBUG"] = true
	logg.LogKeys["CHECKERLUTION"] = true
	logg.LogKeys["CHECKERLUTION_SCAPE"] = true
	logg.LogKeys["CHECKERSBOT"] = true
	logg.LogKeys["NEURGO"] = false
	logg.LogKeys["NEURVOLVE"] = true
	logg.LogKeys["SENSOR_SYNC"] = false
	logg.LogKeys["ACTUATOR_SYNC"] = false
	logg.LogKeys["NODE_PRE_SEND"] = false
	logg.LogKeys["NODE_POST_SEND"] = false
	logg.LogKeys["NODE_POST_RECV"] = false
	logg.LogKeys["NODE_STATE"] = false
	ng.SeedRandom()
}

func train(checkersBotFlags cbot.CheckersBotFlags) {

	// run a webserver in order to view expvar output
	// at http://localhost:8080/debug/vars
	go http.ListenAndServe(":8080", nil)

	trainer := &checkerlution.CheckerlutionTrainer{}

	// checkerlution.RunTopologyMutatingTrainer()
	trainer.RunPopulationTrainer(checkersBotFlags)

}

func run(checkersBotFlags cbot.CheckersBotFlags, cortexId string) {

	thinker := &checkerlution.Checkerlution{}
	thinker.SetMode(checkerlution.RUNNING_MODE)

	if len(cortexId) > 0 {
		// first try to load it from a file with that id
		filename := fmt.Sprintf("%v.json", cortexId)
		cortex, err := ng.NewCortexFromJSONFile(filename)
		if err == nil {
			thinker.StartWithCortex(cortex, checkersBotFlags.Team)

		} else {
			// otherwise, load it from db
			db, error := couch.Connect(checkersBotFlags.SyncGatewayUrl)
			if error != nil {
				logg.LogPanic("Error connecting to %v: %v", checkersBotFlags.SyncGatewayUrl, error)
			}

			cortex := &ng.Cortex{}

			error = db.Retrieve(cortexId, cortex)
			logg.LogTo("CHECKERLUTION", "error: %v cortex :%v", error, *cortex)

			if error != nil {
				logg.LogPanic("Could not find cortex: %v", cortexId, error)
			}

			cortex.LinkNodesToCortex()
			thinker.StartWithCortex(cortex, checkersBotFlags.Team)

		}

	} else {
		// start with a random cortex
		thinker.Start(checkersBotFlags.Team)
	}

	game := cbot.NewGame(checkersBotFlags.Team, thinker)
	game.SetServerUrl(checkersBotFlags.SyncGatewayUrl)
	game.SetFeedType(checkersBotFlags.FeedType)
	game.SetDelayBeforeMove(checkersBotFlags.RandomDelayBeforeMove)

	logg.LogTo("CHECKERLUTION", "Starting game loop")
	game.GameLoop()
	logg.LogTo("CHECKERLUTION", "Game loop finished")

}

func main() {

	checkersBotRawFlags := cbot.GetCheckersBotRawFlags()

	var mode int

	flag.IntVar(
		&mode,
		"mode",
		0,
		"Run Mode - Run: 0 Train: 1",
	)

	var cortexId string

	flag.StringVar(
		&cortexId,
		"cortexId",
		"",
		"Cortex Id of saved cortex",
	)

	flag.Parse()

	checkersBotFlags := checkersBotRawFlags.GetCheckersBotFlags()

	logg.LogTo("CHECKERLUTION", "Flags: %v", checkersBotFlags)
	logg.LogTo("CHECKERLUTION", "Mode: %v", mode)
	logg.LogTo("CHECKERLUTION", "CortexId: %v", cortexId)

	if mode == 0 {
		logg.LogTo("CHECKERLUTION", "Run mode: Run")
		run(checkersBotFlags, cortexId)
	} else {
		logg.LogTo("CHECKERLUTION", "Run mode: Train")
		train(checkersBotFlags)
	}

}
<|endoftext|>