<commit_before>package garden

import (
	"io"
)

type Container interface {
	Handle() string

	// Stop stops a container.
	//
	// If kill is false, garden stops a container by sending the processes running inside it the SIGTERM signal.
	// It then waits for the processes to terminate before returning a response.
	// If one or more processes do not terminate within 10 seconds,
	// garden sends these processes the SIGKILL signal, killing them ungracefully.
	//
	// If kill is true, garden stops a container by sending the processing running inside it a SIGKILL signal.
	//
	// Once a container is stopped, garden does not allow spawning new processes inside the container.
	// It is possible to copy files in to and out of a stopped container.
	// It is only when a container is destroyed that its filesystem is cleaned up.
	//
	// Errors:
	// * None.
	Stop(kill bool) error

	// Returns information about a container.
	Info() (ContainerInfo, error)

	// StreamIn streams data into a file in a container.
	//
	// Errors:
	// *  TODO.
	StreamIn(dstPath string, tarStream io.Reader) error

	// StreamOut streams a file out of a container.
	//
	// Errors:
	// * TODO.
	StreamOut(srcPath string) (io.ReadCloser, error)

	// Limits the network bandwidth for a container.
	LimitBandwidth(limits BandwidthLimits) error

	CurrentBandwidthLimits() (BandwidthLimits, error)

	// Limits the CPU shares for a container.
	LimitCPU(limits CPULimits) error

	CurrentCPULimits() (CPULimits, error)

	// Limits the disk usage for a container.
	//
	// The disk limits that are set by this command only have effect for the container's unprivileged user.
	// Files/directories created by its privileged user are not subject to these limits.
	//
	// TODO: explain how disk management works.
	LimitDisk(limits DiskLimits) error
	CurrentDiskLimits() (DiskLimits, error)

	// Limits the memory usage for a container.
	//
	// The limit applies to all process in the container. When the limit is
	// exceeded, the container will be automatically stopped.
	//
	// Errors:
	// * The kernel does not support setting memory.memsw.limit_in_bytes.
	LimitMemory(limits MemoryLimits) error

	CurrentMemoryLimits() (MemoryLimits, error)

	// Map a port on the host to a port in the container so that traffic to the
	// host port is forwarded to the container port.
	//
	// If a host port is not given, a port will be acquired from the server's port
	// pool.
	//
	// If a container port is not given, the port will be the same as the
	// container port.
	//
	// The two resulting ports are returned in the response.
	//
	// Errors:
	// * When no port can be acquired from the server's port pool.
	NetIn(hostPort, containerPort uint32) (uint32, uint32, error)

	// Whitelist outbound network traffic.
	//
	// If the configuration directive deny_networks is not used,
	// all networks are already whitelisted and this command is effectively a no-op.
	//
	// * network: Network to whitelist (in the form 1.2.3.4/8) or a range of IP
	//            addresses to whitelist (separated by -)
	//
	// * port: Port to whitelist.
	//
	// * portRange: Colon separated port range (in the form 8080:9080).
	//
	// * protocol : the protocol to be whitelisted (default TCP)
	//
	// * icmpType: the ICMP type value to be whitelisted when protocol=ICMP (a
	//             value of -1 means all types and is the default)
	//
	// * icmpCode: the ICMP code value to be whitelisted when protocol=ICMP (a
	//             value of -1 means all codes and is the default)
	//
	// * log: Boolean specifying whether or not logging should be enabled. If
	//        logging is enabled, the first packet of a given connection is logged.
	//
	// Errors:
	// * None.
	NetOut(network string, port uint32, portRange string, protocol Protocol, icmpType int32, icmpCode int32, log bool) error

	// Run a script inside a container.
	//
	// The 'privileged' flag remains for backwards compatibility, but the 'user' flag is preferred.
	// The root user will be mapped to a non-root UID in the host unless the container (not this process) was created with 'privileged' true.
	//
	// Errors:
	// * TODO.
	Run(ProcessSpec, ProcessIO) (Process, error)

	// Attach starts streaming the output back to the client from a specified process.
	//
	// Errors:
	// * processID does not refer to a running process.
	Attach(processID uint32, io ProcessIO) (Process, error)

	// GetProperty returns the value of the property with the specified name.
	//
	// Errors:
	// * When the property does not exist on the container.
	GetProperty(name string) (string, error)

	// Set a named property on a container to a specified value.
	//
	// Errors:
	// * None.
	SetProperty(name string, value string) error

	// Remove a property with the specified name from a container.
	//
	// Errors:
	// * None.
	RemoveProperty(name string) error
}

type Protocol uint8

const (
	ProtocolTCP Protocol = 1 << iota
	ProtocolUDP
	ProtocolICMP

	ProtocolAll Protocol = (1 << iota) - 1
)

// ProcessSpec contains parameters for running a script inside a container.
type ProcessSpec struct {
	Path string   // Path to command to execute.
	Args []string // Arguments to pass to command.
	Env  []string // Environment variables.
	Dir  string   // Working directory (default: home directory).

	Privileged bool   // Whether to run the script as root or not. Can be overriden by 'user', if specified.
	User       string // The name of a user in the container to run the process as. If not specified defaults to 'root' for privileged processes, and 'vcap' for unprivileged processes.

	Limits ResourceLimits // Resource limits
	TTY    *TTYSpec       // Execute with a TTY for stdio.
}

type TTYSpec struct {
	WindowSize *WindowSize
}

type WindowSize struct {
	Columns int
	Rows    int
}

type ProcessIO struct {
	Stdin  io.Reader
	Stdout io.Writer
	Stderr io.Writer
}

type Process interface {
	ID() uint32
	Wait() (int, error)
	SetTTY(TTYSpec) error
	Signal(Signal) error
}

type Signal int

const (
	SignalTerminate Signal = iota
	SignalKill
)

type PortMapping struct {
	HostPort      uint32
	ContainerPort uint32
}

// ContainerInfo holds information about a container.
type ContainerInfo struct {
	State         string                 // Either "active" or "stopped".
	Events        []string               // List of events that occurred for the container. It currently includes only "oom" (Out Of Memory) event if it occurred.
	HostIP        string                 // The IP address of the gateway which controls the host side of the container's virtual ethernet pair.
	ContainerIP   string                 // The IP address of the container side of the container's virtual ethernet pair.
	ExternalIP    string                 //
	ContainerPath string                 // The path to the directory holding the container's files (both its control scripts and filesystem).
	ProcessIDs    []uint32               // List of running processes.
	MemoryStat    ContainerMemoryStat    //
	CPUStat       ContainerCPUStat       //
	DiskStat      ContainerDiskStat      //
	BandwidthStat ContainerBandwidthStat //
	Properties    Properties             // List of properties defined for the container.
	MappedPorts   []PortMapping          //
}

type ContainerMemoryStat struct {
	Cache                   uint64
	Rss                     uint64
	MappedFile              uint64
	Pgpgin                  uint64
	Pgpgout                 uint64
	Swap                    uint64
	Pgfault                 uint64
	Pgmajfault              uint64
	InactiveAnon            uint64
	ActiveAnon              uint64
	InactiveFile            uint64
	ActiveFile              uint64
	Unevictable             uint64
	HierarchicalMemoryLimit uint64
	HierarchicalMemswLimit  uint64
	TotalCache              uint64
	TotalRss                uint64
	TotalMappedFile         uint64
	TotalPgpgin             uint64
	TotalPgpgout            uint64
	TotalSwap               uint64
	TotalPgfault            uint64
	TotalPgmajfault         uint64
	TotalInactiveAnon       uint64
	TotalActiveAnon         uint64
	TotalInactiveFile       uint64
	TotalActiveFile         uint64
	TotalUnevictable        uint64
}

type ContainerCPUStat struct {
	Usage  uint64
	User   uint64
	System uint64
}

type ContainerDiskStat struct {
	BytesUsed  uint64
	InodesUsed uint64
}

type ContainerBandwidthStat struct {
	InRate   uint64
	InBurst  uint64
	OutRate  uint64
	OutBurst uint64
}

type BandwidthLimits struct {
	RateInBytesPerSecond      uint64
	BurstRateInBytesPerSecond uint64
}

type DiskLimits struct {
	BlockSoft uint64
	BlockHard uint64

	InodeSoft uint64
	InodeHard uint64

	ByteSoft uint64 // New soft block limit specified in bytes. Only has effect when BlockSoft is not specified.
	ByteHard uint64 // New hard block limit specified in bytes. Only has effect when BlockHard is not specified.
}

type MemoryLimits struct {
	LimitInBytes uint64 //	Memory usage limit in bytes.
}

type CPULimits struct {
	LimitInShares uint64
}

// Resource limits.
//
// Please refer to the manual page of getrlimit for a description of the individual fields:
// http://www.kernel.org/doc/man-pages/online/pages/man2/getrlimit.2.html
type ResourceLimits struct {
	As         *uint64
	Core       *uint64
	Cpu        *uint64
	Data       *uint64
	Fsize      *uint64
	Locks      *uint64
	Memlock    *uint64
	Msgqueue   *uint64
	Nice       *uint64
	Nofile     *uint64
	Nproc      *uint64
	Rss        *uint64
	Rtprio     *uint64
	Sigpending *uint64
	Stack      *uint64
}
<commit_msg>Describe ordering of NetOut<commit_after>package garden

import (
	"io"
)

type Container interface {
	Handle() string

	// Stop stops a container.
	//
	// If kill is false, garden stops a container by sending the processes running inside it the SIGTERM signal.
	// It then waits for the processes to terminate before returning a response.
	// If one or more processes do not terminate within 10 seconds,
	// garden sends these processes the SIGKILL signal, killing them ungracefully.
	//
	// If kill is true, garden stops a container by sending the processing running inside it a SIGKILL signal.
	//
	// Once a container is stopped, garden does not allow spawning new processes inside the container.
	// It is possible to copy files in to and out of a stopped container.
	// It is only when a container is destroyed that its filesystem is cleaned up.
	//
	// Errors:
	// * None.
	Stop(kill bool) error

	// Returns information about a container.
	Info() (ContainerInfo, error)

	// StreamIn streams data into a file in a container.
	//
	// Errors:
	// *  TODO.
	StreamIn(dstPath string, tarStream io.Reader) error

	// StreamOut streams a file out of a container.
	//
	// Errors:
	// * TODO.
	StreamOut(srcPath string) (io.ReadCloser, error)

	// Limits the network bandwidth for a container.
	LimitBandwidth(limits BandwidthLimits) error

	CurrentBandwidthLimits() (BandwidthLimits, error)

	// Limits the CPU shares for a container.
	LimitCPU(limits CPULimits) error

	CurrentCPULimits() (CPULimits, error)

	// Limits the disk usage for a container.
	//
	// The disk limits that are set by this command only have effect for the container's unprivileged user.
	// Files/directories created by its privileged user are not subject to these limits.
	//
	// TODO: explain how disk management works.
	LimitDisk(limits DiskLimits) error
	CurrentDiskLimits() (DiskLimits, error)

	// Limits the memory usage for a container.
	//
	// The limit applies to all process in the container. When the limit is
	// exceeded, the container will be automatically stopped.
	//
	// Errors:
	// * The kernel does not support setting memory.memsw.limit_in_bytes.
	LimitMemory(limits MemoryLimits) error

	CurrentMemoryLimits() (MemoryLimits, error)

	// Map a port on the host to a port in the container so that traffic to the
	// host port is forwarded to the container port.
	//
	// If a host port is not given, a port will be acquired from the server's port
	// pool.
	//
	// If a container port is not given, the port will be the same as the
	// container port.
	//
	// The two resulting ports are returned in the response.
	//
	// Errors:
	// * When no port can be acquired from the server's port pool.
	NetIn(hostPort, containerPort uint32) (uint32, uint32, error)

	// Whitelist outbound network traffic.
	//
	// If the configuration directive deny_networks is not used,
	// all networks are already whitelisted and this command is effectively a no-op.
	//
	// * network: Network to whitelist (in the form 1.2.3.4/8) or a range of IP
	//            addresses to whitelist (separated by -)
	//
	// * port: Port to whitelist.
	//
	// * portRange: Colon separated port range (in the form 8080:9080).
	//
	// * protocol : the protocol to be whitelisted (default TCP)
	//
	// * icmpType: the ICMP type value to be whitelisted when protocol=ICMP (a
	//             value of -1 means all types and is the default)
	//
	// * icmpCode: the ICMP code value to be whitelisted when protocol=ICMP (a
	//             value of -1 means all codes and is the default)
	//
	// * log: Boolean specifying whether or not logging should be enabled. If
	//        logging is enabled, the first packet of a given connection is logged.
	//        Later NetOut calls take precedence over earlier calls, which is
	//        significant only in relation to logging.
	//
	// Errors:
	// * None.
	NetOut(network string, port uint32, portRange string, protocol Protocol, icmpType int32, icmpCode int32, log bool) error

	// Run a script inside a container.
	//
	// The 'privileged' flag remains for backwards compatibility, but the 'user' flag is preferred.
	// The root user will be mapped to a non-root UID in the host unless the container (not this process) was created with 'privileged' true.
	//
	// Errors:
	// * TODO.
	Run(ProcessSpec, ProcessIO) (Process, error)

	// Attach starts streaming the output back to the client from a specified process.
	//
	// Errors:
	// * processID does not refer to a running process.
	Attach(processID uint32, io ProcessIO) (Process, error)

	// GetProperty returns the value of the property with the specified name.
	//
	// Errors:
	// * When the property does not exist on the container.
	GetProperty(name string) (string, error)

	// Set a named property on a container to a specified value.
	//
	// Errors:
	// * None.
	SetProperty(name string, value string) error

	// Remove a property with the specified name from a container.
	//
	// Errors:
	// * None.
	RemoveProperty(name string) error
}

type Protocol uint8

const (
	ProtocolTCP Protocol = 1 << iota
	ProtocolUDP
	ProtocolICMP

	ProtocolAll Protocol = (1 << iota) - 1
)

// ProcessSpec contains parameters for running a script inside a container.
type ProcessSpec struct {
	Path string   // Path to command to execute.
	Args []string // Arguments to pass to command.
	Env  []string // Environment variables.
	Dir  string   // Working directory (default: home directory).

	Privileged bool   // Whether to run the script as root or not. Can be overriden by 'user', if specified.
	User       string // The name of a user in the container to run the process as. If not specified defaults to 'root' for privileged processes, and 'vcap' for unprivileged processes.

	Limits ResourceLimits // Resource limits
	TTY    *TTYSpec       // Execute with a TTY for stdio.
}

type TTYSpec struct {
	WindowSize *WindowSize
}

type WindowSize struct {
	Columns int
	Rows    int
}

type ProcessIO struct {
	Stdin  io.Reader
	Stdout io.Writer
	Stderr io.Writer
}

type Process interface {
	ID() uint32
	Wait() (int, error)
	SetTTY(TTYSpec) error
	Signal(Signal) error
}

type Signal int

const (
	SignalTerminate Signal = iota
	SignalKill
)

type PortMapping struct {
	HostPort      uint32
	ContainerPort uint32
}

// ContainerInfo holds information about a container.
type ContainerInfo struct {
	State         string                 // Either "active" or "stopped".
	Events        []string               // List of events that occurred for the container. It currently includes only "oom" (Out Of Memory) event if it occurred.
	HostIP        string                 // The IP address of the gateway which controls the host side of the container's virtual ethernet pair.
	ContainerIP   string                 // The IP address of the container side of the container's virtual ethernet pair.
	ExternalIP    string                 //
	ContainerPath string                 // The path to the directory holding the container's files (both its control scripts and filesystem).
	ProcessIDs    []uint32               // List of running processes.
	MemoryStat    ContainerMemoryStat    //
	CPUStat       ContainerCPUStat       //
	DiskStat      ContainerDiskStat      //
	BandwidthStat ContainerBandwidthStat //
	Properties    Properties             // List of properties defined for the container.
	MappedPorts   []PortMapping          //
}

type ContainerMemoryStat struct {
	Cache                   uint64
	Rss                     uint64
	MappedFile              uint64
	Pgpgin                  uint64
	Pgpgout                 uint64
	Swap                    uint64
	Pgfault                 uint64
	Pgmajfault              uint64
	InactiveAnon            uint64
	ActiveAnon              uint64
	InactiveFile            uint64
	ActiveFile              uint64
	Unevictable             uint64
	HierarchicalMemoryLimit uint64
	HierarchicalMemswLimit  uint64
	TotalCache              uint64
	TotalRss                uint64
	TotalMappedFile         uint64
	TotalPgpgin             uint64
	TotalPgpgout            uint64
	TotalSwap               uint64
	TotalPgfault            uint64
	TotalPgmajfault         uint64
	TotalInactiveAnon       uint64
	TotalActiveAnon         uint64
	TotalInactiveFile       uint64
	TotalActiveFile         uint64
	TotalUnevictable        uint64
}

type ContainerCPUStat struct {
	Usage  uint64
	User   uint64
	System uint64
}

type ContainerDiskStat struct {
	BytesUsed  uint64
	InodesUsed uint64
}

type ContainerBandwidthStat struct {
	InRate   uint64
	InBurst  uint64
	OutRate  uint64
	OutBurst uint64
}

type BandwidthLimits struct {
	RateInBytesPerSecond      uint64
	BurstRateInBytesPerSecond uint64
}

type DiskLimits struct {
	BlockSoft uint64
	BlockHard uint64

	InodeSoft uint64
	InodeHard uint64

	ByteSoft uint64 // New soft block limit specified in bytes. Only has effect when BlockSoft is not specified.
	ByteHard uint64 // New hard block limit specified in bytes. Only has effect when BlockHard is not specified.
}

type MemoryLimits struct {
	LimitInBytes uint64 //	Memory usage limit in bytes.
}

type CPULimits struct {
	LimitInShares uint64
}

// Resource limits.
//
// Please refer to the manual page of getrlimit for a description of the individual fields:
// http://www.kernel.org/doc/man-pages/online/pages/man2/getrlimit.2.html
type ResourceLimits struct {
	As         *uint64
	Core       *uint64
	Cpu        *uint64
	Data       *uint64
	Fsize      *uint64
	Locks      *uint64
	Memlock    *uint64
	Msgqueue   *uint64
	Nice       *uint64
	Nofile     *uint64
	Nproc      *uint64
	Rss        *uint64
	Rtprio     *uint64
	Sigpending *uint64
	Stack      *uint64
}
<|endoftext|>


<commit_before>package pgx

import (
	"bytes"
	"context"
	"fmt"
	"io"

	"github.com/jackc/pgconn"
	"github.com/jackc/pgio"
	errors "golang.org/x/xerrors"
)

// CopyFromRows returns a CopyFromSource interface over the provided rows slice
// making it usable by *Conn.CopyFrom.
func CopyFromRows(rows [][]interface{}) CopyFromSource {
	return &copyFromRows{rows: rows, idx: -1}
}

type copyFromRows struct {
	rows [][]interface{}
	idx  int
}

func (ctr *copyFromRows) Next() bool {
	ctr.idx++
	return ctr.idx < len(ctr.rows)
}

func (ctr *copyFromRows) Values() ([]interface{}, error) {
	return ctr.rows[ctr.idx], nil
}

func (ctr *copyFromRows) Err() error {
	return nil
}

// CopyFromSlice returns a CopyFromSource interface over a dynamic func
// making it usable by *Conn.CopyFrom.
func CopyFromSlice(length int, next func(int) ([]interface{}, error)) CopyFromSource {
	return &copyFromSlice{next: next, idx: -1, len: length}
}

type copyFromSlice struct {
	next func(int) ([]interface{}, error)
	idx  int
	len  int
	err  error
}

func (cts *copyFromSlice) Next() bool {
	cts.idx++
	return cts.idx < cts.len
}

func (cts *copyFromSlice) Values() ([]interface{}, error) {
	values, err := cts.next(cts.idx)
	if err != nil {
		cts.err = err
	}
	return values, err
}

func (cts *copyFromSlice) Err() error {
	return cts.err
}

// CopyFromSource is the interface used by *Conn.CopyFrom as the source for copy data.
type CopyFromSource interface {
	// Next returns true if there is another row and makes the next row data
	// available to Values(). When there are no more rows available or an error
	// has occurred it returns false.
	Next() bool

	// Values returns the values for the current row.
	Values() ([]interface{}, error)

	// Err returns any error that has been encountered by the CopyFromSource. If
	// this is not nil *Conn.CopyFrom will abort the copy.
	Err() error
}

type copyFrom struct {
	conn          *Conn
	tableName     Identifier
	columnNames   []string
	rowSrc        CopyFromSource
	readerErrChan chan error
}

func (ct *copyFrom) run(ctx context.Context) (int64, error) {
	quotedTableName := ct.tableName.Sanitize()
	cbuf := &bytes.Buffer{}
	for i, cn := range ct.columnNames {
		if i != 0 {
			cbuf.WriteString(", ")
		}
		cbuf.WriteString(quoteIdentifier(cn))
	}
	quotedColumnNames := cbuf.String()

	sd, err := ct.conn.Prepare(ctx, "", fmt.Sprintf("select %s from %s", quotedColumnNames, quotedTableName))
	if err != nil {
		return 0, err
	}

	r, w := io.Pipe()
	doneChan := make(chan struct{})

	go func() {
		defer close(doneChan)

		// Purposely NOT using defer w.Close(). See https://github.com/golang/go/issues/24283.
		buf := ct.conn.wbuf

		buf = append(buf, "PGCOPY\n\377\r\n\000"...)
		buf = pgio.AppendInt32(buf, 0)
		buf = pgio.AppendInt32(buf, 0)

		moreRows := true
		for moreRows {
			var err error
			moreRows, buf, err = ct.buildCopyBuf(buf, sd)
			if err != nil {
				w.CloseWithError(err)
				return
			}

			if ct.rowSrc.Err() != nil {
				w.CloseWithError(ct.rowSrc.Err())
				return
			}

			if len(buf) > 0 {
				_, err = w.Write(buf)
				if err != nil {
					w.Close()
					return
				}
			}

			buf = buf[:0]
		}

		w.Close()
	}()

	commandTag, err := ct.conn.pgConn.CopyFrom(ctx, r, fmt.Sprintf("copy %s ( %s ) from stdin binary;", quotedTableName, quotedColumnNames))

	r.Close()
	<-doneChan

	return commandTag.RowsAffected(), err
}

func (ct *copyFrom) buildCopyBuf(buf []byte, sd *pgconn.StatementDescription) (bool, []byte, error) {

	for ct.rowSrc.Next() {
		values, err := ct.rowSrc.Values()
		if err != nil {
			return false, nil, err
		}
		if len(values) != len(ct.columnNames) {
			return false, nil, errors.Errorf("expected %d values, got %d values", len(ct.columnNames), len(values))
		}

		buf = pgio.AppendInt16(buf, int16(len(ct.columnNames)))
		for i, val := range values {
			buf, err = encodePreparedStatementArgument(ct.conn.connInfo, buf, sd.Fields[i].DataTypeOID, val)
			if err != nil {
				return false, nil, err
			}
		}

		if len(buf) > 65536 {
			return true, buf, nil
		}
	}

	return false, buf, nil
}

// CopyFrom uses the PostgreSQL copy protocol to perform bulk data insertion.
// It returns the number of rows copied and an error.
//
// CopyFrom requires all values use the binary format. Almost all types
// implemented by pgx use the binary format by default. Types implementing
// Encoder can only be used if they encode to the binary format.
func (c *Conn) CopyFrom(ctx context.Context, tableName Identifier, columnNames []string, rowSrc CopyFromSource) (int64, error) {
	ct := &copyFrom{
		conn:          c,
		tableName:     tableName,
		columnNames:   columnNames,
		rowSrc:        rowSrc,
		readerErrChan: make(chan error),
	}

	return ct.run(ctx)
}
<commit_msg>Add logging on CopyFrom<commit_after>package pgx

import (
	"bytes"
	"context"
	"fmt"
	"io"
	"time"

	"github.com/jackc/pgconn"
	"github.com/jackc/pgio"
	errors "golang.org/x/xerrors"
)

// CopyFromRows returns a CopyFromSource interface over the provided rows slice
// making it usable by *Conn.CopyFrom.
func CopyFromRows(rows [][]interface{}) CopyFromSource {
	return &copyFromRows{rows: rows, idx: -1}
}

type copyFromRows struct {
	rows [][]interface{}
	idx  int
}

func (ctr *copyFromRows) Next() bool {
	ctr.idx++
	return ctr.idx < len(ctr.rows)
}

func (ctr *copyFromRows) Values() ([]interface{}, error) {
	return ctr.rows[ctr.idx], nil
}

func (ctr *copyFromRows) Err() error {
	return nil
}

// CopyFromSlice returns a CopyFromSource interface over a dynamic func
// making it usable by *Conn.CopyFrom.
func CopyFromSlice(length int, next func(int) ([]interface{}, error)) CopyFromSource {
	return &copyFromSlice{next: next, idx: -1, len: length}
}

type copyFromSlice struct {
	next func(int) ([]interface{}, error)
	idx  int
	len  int
	err  error
}

func (cts *copyFromSlice) Next() bool {
	cts.idx++
	return cts.idx < cts.len
}

func (cts *copyFromSlice) Values() ([]interface{}, error) {
	values, err := cts.next(cts.idx)
	if err != nil {
		cts.err = err
	}
	return values, err
}

func (cts *copyFromSlice) Err() error {
	return cts.err
}

// CopyFromSource is the interface used by *Conn.CopyFrom as the source for copy data.
type CopyFromSource interface {
	// Next returns true if there is another row and makes the next row data
	// available to Values(). When there are no more rows available or an error
	// has occurred it returns false.
	Next() bool

	// Values returns the values for the current row.
	Values() ([]interface{}, error)

	// Err returns any error that has been encountered by the CopyFromSource. If
	// this is not nil *Conn.CopyFrom will abort the copy.
	Err() error
}

type copyFrom struct {
	conn          *Conn
	tableName     Identifier
	columnNames   []string
	rowSrc        CopyFromSource
	readerErrChan chan error
}

func (ct *copyFrom) run(ctx context.Context) (int64, error) {
	quotedTableName := ct.tableName.Sanitize()
	cbuf := &bytes.Buffer{}
	for i, cn := range ct.columnNames {
		if i != 0 {
			cbuf.WriteString(", ")
		}
		cbuf.WriteString(quoteIdentifier(cn))
	}
	quotedColumnNames := cbuf.String()

	sd, err := ct.conn.Prepare(ctx, "", fmt.Sprintf("select %s from %s", quotedColumnNames, quotedTableName))
	if err != nil {
		return 0, err
	}

	r, w := io.Pipe()
	doneChan := make(chan struct{})

	go func() {
		defer close(doneChan)

		// Purposely NOT using defer w.Close(). See https://github.com/golang/go/issues/24283.
		buf := ct.conn.wbuf

		buf = append(buf, "PGCOPY\n\377\r\n\000"...)
		buf = pgio.AppendInt32(buf, 0)
		buf = pgio.AppendInt32(buf, 0)

		moreRows := true
		for moreRows {
			var err error
			moreRows, buf, err = ct.buildCopyBuf(buf, sd)
			if err != nil {
				w.CloseWithError(err)
				return
			}

			if ct.rowSrc.Err() != nil {
				w.CloseWithError(ct.rowSrc.Err())
				return
			}

			if len(buf) > 0 {
				_, err = w.Write(buf)
				if err != nil {
					w.Close()
					return
				}
			}

			buf = buf[:0]
		}

		w.Close()
	}()

	startTime := time.Now()

	commandTag, err := ct.conn.pgConn.CopyFrom(ctx, r, fmt.Sprintf("copy %s ( %s ) from stdin binary;", quotedTableName, quotedColumnNames))

	r.Close()
	<-doneChan

	rowsAffected := commandTag.RowsAffected()
	if err == nil {
		if ct.conn.shouldLog(LogLevelInfo) {
			endTime := time.Now()
			ct.conn.log(ctx, LogLevelInfo, "Copy", map[string]interface{}{"tableName": ct.tableName, "columnNames": ct.columnNames, "time": endTime.Sub(startTime), "rowCount": rowsAffected})
		}
	} else if ct.conn.shouldLog(LogLevelError) {
		ct.conn.log(ctx, LogLevelError, "Copy", map[string]interface{}{"err": err, "tableName": ct.tableName, "columnNames": ct.columnNames})
	}

	return rowsAffected, err
}

func (ct *copyFrom) buildCopyBuf(buf []byte, sd *pgconn.StatementDescription) (bool, []byte, error) {

	for ct.rowSrc.Next() {
		values, err := ct.rowSrc.Values()
		if err != nil {
			return false, nil, err
		}
		if len(values) != len(ct.columnNames) {
			return false, nil, errors.Errorf("expected %d values, got %d values", len(ct.columnNames), len(values))
		}

		buf = pgio.AppendInt16(buf, int16(len(ct.columnNames)))
		for i, val := range values {
			buf, err = encodePreparedStatementArgument(ct.conn.connInfo, buf, sd.Fields[i].DataTypeOID, val)
			if err != nil {
				return false, nil, err
			}
		}

		if len(buf) > 65536 {
			return true, buf, nil
		}
	}

	return false, buf, nil
}

// CopyFrom uses the PostgreSQL copy protocol to perform bulk data insertion.
// It returns the number of rows copied and an error.
//
// CopyFrom requires all values use the binary format. Almost all types
// implemented by pgx use the binary format by default. Types implementing
// Encoder can only be used if they encode to the binary format.
func (c *Conn) CopyFrom(ctx context.Context, tableName Identifier, columnNames []string, rowSrc CopyFromSource) (int64, error) {
	ct := &copyFrom{
		conn:          c,
		tableName:     tableName,
		columnNames:   columnNames,
		rowSrc:        rowSrc,
		readerErrChan: make(chan error),
	}

	return ct.run(ctx)
}
<|endoftext|>


<commit_before>package core

import (
	"fmt"
	ds "github.com/jbenet/datastore.go"
	config "github.com/jbenet/go-ipfs/config"
	peer "github.com/jbenet/go-ipfs/peer"
)

// IPFS Core module. It represents an IPFS instance.

type IpfsNode struct {

	// the node's configuration
	Config *config.Config

	// the local node's identity
	Identity *peer.Peer

	// the book of other nodes (a map of Peer instances)
	PeerBook *peer.PeerBook

	// the local datastore
	Datastore ds.Datastore

	// the network message stream
	// Network *netmux.Netux

	// the routing system. recommend ipfs-dht
	// Routing *routing.Routing

	// the block exchange + strategy (bitswap)
	// BitSwap *bitswap.BitSwap

	// the block service, get/add blocks.
	// Blocks *blocks.BlockService

	// the path resolution system
	// Resolver *resolver.PathResolver

	// the name system, resolves paths to hashes
	// Namesys *namesys.Namesys
}

func NewIpfsNode(cfg *config.Config) (*IpfsNode, error) {
	if cfg == nil {
		return nil, fmt.Errorf("configuration required.")
	}

	d, err := makeDatastore(cfg.Datastore)
	if err != nil {
		return nil, err
	}

	n := &IpfsNode{
		Config:    cfg,
		PeerBook:  &peer.PeerBook{},
		Datastore: d,
	}

	return n, nil
}
<commit_msg>core block service<commit_after>package core

import (
	"fmt"
	ds "github.com/jbenet/datastore.go"
	blocks "github.com/jbenet/go-ipfs/blocks"
	config "github.com/jbenet/go-ipfs/config"
	merkledag "github.com/jbenet/go-ipfs/merkledag"
	peer "github.com/jbenet/go-ipfs/peer"
	u "github.com/jbenet/go-ipfs/util"
)

// IPFS Core module. It represents an IPFS instance.

type IpfsNode struct {

	// the node's configuration
	Config *config.Config

	// the local node's identity
	Identity *peer.Peer

	// the book of other nodes (a map of Peer instances)
	PeerBook *peer.PeerBook

	// the local datastore
	Datastore ds.Datastore

	// the network message stream
	// Network *netmux.Netux

	// the routing system. recommend ipfs-dht
	// Routing *routing.Routing

	// the block exchange + strategy (bitswap)
	// BitSwap *bitswap.BitSwap

	// the block service, get/add blocks.
	Blocks *blocks.BlockService

	// the path resolution system
	// Resolver *resolver.PathResolver

	// the name system, resolves paths to hashes
	// Namesys *namesys.Namesys
}

func NewIpfsNode(cfg *config.Config) (*IpfsNode, error) {
	if cfg == nil {
		return nil, fmt.Errorf("configuration required.")
	}

	d, err := makeDatastore(cfg.Datastore)
	if err != nil {
		return nil, err
	}

	bs, err := blocks.NewBlockService(d)
	if err != nil {
		return nil, err
	}

	n := &IpfsNode{
		Config:    cfg,
		PeerBook:  &peer.PeerBook{},
		Datastore: d,
		Blocks:    bs,
	}

	return n, nil
}

func (n *IpfsNode) AddDagNode(nd *merkledag.Node) (u.Key, error) {
	d, err := nd.Encoded(false)
	if err != nil {
		return "", err
	}

	b, err := blocks.NewBlock(d)
	if err != nil {
		return "", err
	}

	return n.Blocks.AddBlock(b)
}
<|endoftext|>


<commit_before>package core

import (
	"encoding/json"
	"fmt"
	"log"
	"net/http"
	"pilosa/config"
	"pilosa/db"
	"reflect"
	"runtime"
	"strconv"

	notify "github.com/bitly/go-notify"
	"github.com/davecgh/go-spew/spew"
	"github.com/gorilla/websocket"
	"tux21b.org/v1/gocql/uuid"
)

type WebService struct {
	service *Service
}

func NewWebService(service *Service) *WebService {
	return &WebService{service}
}

func (self *WebService) Run() {
	port_string := strconv.Itoa(config.GetInt("port_http"))
	log.Printf("Serving HTTP on port %s...\n", port_string)
	mux := http.NewServeMux()
	mux.HandleFunc("/message", self.HandleMessage)
	mux.HandleFunc("/query", self.HandleQuery)
	mux.HandleFunc("/stats", self.HandleStats)
	mux.HandleFunc("/info", self.HandleInfo)
	mux.HandleFunc("/processes", self.HandleProcesses)
	mux.HandleFunc("/listen/ws", self.HandleListenWS)
	mux.HandleFunc("/listen/stream", self.HandleListenStream)
	mux.HandleFunc("/listen", self.HandleListen)
	mux.HandleFunc("/test", self.HandleTest)
	mux.HandleFunc("/version", self.HandleVersion)
	mux.HandleFunc("/ping", self.HandlePing)
	mux.HandleFunc("/batch", self.HandleBatch)
	s := &http.Server{
		Addr:    ":" + port_string,
		Handler: mux,
	}
	s.ListenAndServe()
}

func (self *WebService) HandleMessage(w http.ResponseWriter, r *http.Request) {
	if r.Method != "POST" {
		http.Error(w, "Only POST allowed", http.StatusMethodNotAllowed)
		return
	}
	var message db.Message
	decoder := json.NewDecoder(r.Body)
	if decoder.Decode(&message) != nil {
		http.Error(w, "Invalid JSON", http.StatusBadRequest)
		return
	}
	//service.Inbox <- &message
}

func (self *WebService) HandleBatch(w http.ResponseWriter, r *http.Request) {
	if r.Method != "POST" {
		http.Error(w, "Only POST allowed", http.StatusMethodNotAllowed)
		return
	}
	/*   	values.Set("db", database)
	values.Set("id", id)
	values.Set("frame", fragment_type)
	values.Set("slice", fmt.Sprintf("%d", slice))
	values.Set("bitmap", compressed_string)
	*/

	err := r.ParseForm()
	if err != nil {
		http.Error(w, err.Error(), http.StatusBadRequest)
		return
	}

	database_name := r.Form.Get("db")
	if database_name == "" {
		http.Error(w, "Provide a database (db)", http.StatusNotFound)
		return
	}

	bitmap_id_string := r.Form.Get("id")
	if bitmap_id_string == "" {
		http.Error(w, "Provide a bitmap id (id)", http.StatusNotFound)
		return
	}
	bitmap_id, _ := strconv.ParseUint(bitmap_id_string, 10, 64)

	slice_string := r.Form.Get("slice")
	if bitmap_id_string == "" {
		http.Error(w, "Provide a slice (slice)", http.StatusNotFound)
		return
	}
	slice, _ := strconv.ParseInt(slice_string, 10, 32)

	frame := r.Form.Get("frame")
	if bitmap_id_string == "" {
		http.Error(w, "Provide a frame (frame)", http.StatusNotFound)
		return
	}

	compressed_bitmap := r.Form.Get("bitmap")
	if bitmap_id_string == "" {
		http.Error(w, "Provide a compressed base64 bitmap (bitmap)", http.StatusNotFound)
		return
	}

	results := self.service.Batch(database_name, frame, compressed_bitmap, bitmap_id, int(slice))

	encoder := json.NewEncoder(w)
	err = encoder.Encode(results)
	if err != nil {
		log.Println("Error Batch results")
		log.Println(spew.Sdump(r.Form))
		err = encoder.Encode("Bad Batch Request")
	}

}

func (self *WebService) HandleQuery(w http.ResponseWriter, r *http.Request) {
	if r.Method != "POST" {
		http.Error(w, "Only POST allowed", http.StatusMethodNotAllowed)
		return
	}
	err := r.ParseForm()
	if err != nil {
		http.Error(w, err.Error(), http.StatusBadRequest)
		return
	}

	database_name := r.Form.Get("db")
	if database_name == "" {
		database_name = config.GetString("default_db")
	}
	if database_name == "" {
		http.Error(w, "Provide a database (db)", http.StatusNotFound)
		return
	}
	pql := r.Form.Get("pql")
	if pql == "" {
		http.Error(w, "Provide a valid query string (pql)", http.StatusNotFound)
		return
	}

	results := self.service.Executor.RunPQL(database_name, pql)

	encoder := json.NewEncoder(w)
	err = encoder.Encode(results)
	if err != nil {
		log.Fatal("Error encoding stats")
	}

}

func (self *WebService) HandleStats(w http.ResponseWriter, r *http.Request) {
	if r.Method != "GET" {
		http.Error(w, "Only GET allowed", http.StatusMethodNotAllowed)
		return
	}
	encoder := json.NewEncoder(w)
	//stats := service.GetStats()
	//	stats := ""
	m := &runtime.MemStats{}
	runtime.ReadMemStats(m)
	stats := map[string]interface{}{"num_goroutines": runtime.NumGoroutine(), "Memory Aquired": m.Sys, "Memory Used": m.Alloc}

	err := encoder.Encode(stats)
	if err != nil {
		log.Fatal("Error encoding stats")
	}
}

func (self *WebService) HandleInfo(w http.ResponseWriter, r *http.Request) {
	if r.Method != "GET" {
		http.Error(w, "Only GET allowed", http.StatusMethodNotAllowed)
		return
	}
	spew.Fdump(w, self.service.Cluster)
}

func (self *WebService) HandleVersion(w http.ResponseWriter, r *http.Request) {
	if r.Method != "GET" {
		http.Error(w, "Only GET allowed", http.StatusMethodNotAllowed)
		return
	}

	fmt.Fprintf(w, "Pilosa v."+self.service.version+"\n")
}

func (self *WebService) HandleTest(w http.ResponseWriter, r *http.Request) {
	if r.Method != "GET" {
		http.Error(w, "Only GET allowed", http.StatusMethodNotAllowed)
		return
	}
	spew.Dump("TEST!")

	msg := new(db.Message)
	msg.Data = "mystring"
	self.service.Transport.Push(msg)

	msg2 := new(db.Message)
	msg2.Data = 789
	self.service.Transport.Push(msg2)
}

func (self *WebService) HandleProcesses(w http.ResponseWriter, r *http.Request) {
	if r.Method != "GET" {
		http.Error(w, "Only GET allowed", http.StatusMethodNotAllowed)
		return
	}
	encoder := json.NewEncoder(w)
	processes := self.service.ProcessMap.GetMetadata()
	err := encoder.Encode(processes)
	if err != nil {
		log.Fatal("Error encoding stats")
	}
}

func (self *WebService) HandlePing(w http.ResponseWriter, r *http.Request) {
	if r.Method != "GET" {
		http.Error(w, "Only GET allowed", http.StatusMethodNotAllowed)
		return
	}
	err := r.ParseForm()
	if err != nil {
		http.Error(w, err.Error(), http.StatusBadRequest)
		return
	}
	process_string := r.Form.Get("process")
	process_id, err := uuid.ParseUUID(process_string)
	if err != nil {
		http.Error(w, err.Error(), http.StatusBadRequest)
		return
	}
	_, err = self.service.ProcessMap.GetProcess(&process_id)
	if err != nil {
		http.Error(w, err.Error(), http.StatusNotFound)
		return
	}
	duration, err := self.service.Ping(&process_id)
	if err != nil {
		spew.Fdump(w, err)
		return
	}
	encoder := json.NewEncoder(w)
	encoder.Encode(map[string]float64{"duration": duration.Seconds()})
}

func (self *WebService) HandleListen(w http.ResponseWriter, r *http.Request) {
	w.Write([]byte(`
	<title>Pilosa - streaming client</title>

	`))
}

func (self *WebService) HandleListenWS(w http.ResponseWriter, r *http.Request) {
	defer func() {
		err := recover()
		spew.Dump(err)
	}()
	ws, err := websocket.Upgrade(w, r, nil, 1024, 1024)
	if _, ok := err.(websocket.HandshakeError); ok {
		http.Error(w, "Not a websocket handshake", 400)
		return
	} else if err != nil {
		log.Println(err)
		return
	}
	inbox := make(chan interface{}, 10)
	outbox := make(chan interface{}, 10)
	notify.Start("inbox", inbox)
	notify.Start("outbox", outbox)
	var obj interface{}
	var data interface{}
	var inmessage *db.Message
	var outmessage *db.Envelope
	var host string
	for {
		select {
		case obj = <-outbox:
			outmessage = obj.(*db.Envelope)
			data = outmessage.Message.Data
			host = outmessage.Host.String()
		case obj = <-inbox:
			inmessage = obj.(*db.Message)
			data = inmessage.Data
			host = ""
		}

		typ := reflect.TypeOf(data)

		err := ws.WriteJSON(map[string]interface{}{
			"type": typ.String(),
			"dump": spew.Sdump(data),
			"host": host,
		})
		if err != nil {
			log.Println("stopping")
			notify.Stop("inbox", inbox)
			notify.Stop("outbox", outbox)
			drain(inbox)
			drain(outbox)
			return
		}
	}
}

func drain(ch chan interface{}) {
	for {
		switch {
		case <-ch:
		default:
			return
		}
	}
}

func (self *WebService) HandleListenStream(w http.ResponseWriter, r *http.Request) {
	defer func() {
		err := recover()
		spew.Dump(err)
	}()
	inbox := make(chan interface{}, 10)
	outbox := make(chan interface{}, 10)
	notify.Start("inbox", inbox)
	notify.Start("outbox", outbox)
	var obj interface{}
	var data interface{}
	var inmessage *db.Message
	var outmessage *db.Envelope
	var host string
	for {
		select {
		case obj = <-outbox:
			outmessage = obj.(*db.Envelope)
			data = outmessage.Message.Data
			host = outmessage.Host.String()
		case obj = <-inbox:
			inmessage = obj.(*db.Message)
			data = inmessage.Data
			host = ""
		}

		typ := reflect.TypeOf(data)

		writer := json.NewEncoder(w)
		err := writer.Encode(map[string]interface{}{
			"type": typ.String(),
			"dump": spew.Sdump(data),
			"host": host,
		})
		w.(http.Flusher).Flush()
		if err != nil {
			log.Println("stopping")
			notify.Stop("inbox", inbox)
			notify.Stop("outbox", outbox)
			drain(inbox)
			drain(outbox)
			return
		}
	}
}
<commit_msg>Refactor<commit_after>package core

import (
	"encoding/json"
	"fmt"
	"log"
	"net/http"
	"pilosa/config"
	"pilosa/db"
	"reflect"
	"runtime"
	"strconv"

	notify "github.com/bitly/go-notify"
	"github.com/davecgh/go-spew/spew"
	"github.com/gorilla/websocket"
	"tux21b.org/v1/gocql/uuid"
)

type WebService struct {
	service *Service
}

func NewWebService(service *Service) *WebService {
	return &WebService{service}
}

func (self *WebService) Run() {
	port_string := strconv.Itoa(config.GetInt("port_http"))
	log.Printf("Serving HTTP on port %s...\n", port_string)
	mux := http.NewServeMux()
	mux.HandleFunc("/message", self.HandleMessage)
	mux.HandleFunc("/query", self.HandleQuery)
	mux.HandleFunc("/stats", self.HandleStats)
	mux.HandleFunc("/info", self.HandleInfo)
	mux.HandleFunc("/processes", self.HandleProcesses)
	mux.HandleFunc("/listen/ws", self.HandleListenWS)
	mux.HandleFunc("/listen/stream", self.HandleListenStream)
	mux.HandleFunc("/listen", self.HandleListen)
	mux.HandleFunc("/test", self.HandleTest)
	mux.HandleFunc("/version", self.HandleVersion)
	mux.HandleFunc("/ping", self.HandlePing)
	mux.HandleFunc("/batch", self.HandleBatch)
	s := &http.Server{
		Addr:    ":" + port_string,
		Handler: mux,
	}
	s.ListenAndServe()
}

func (self *WebService) HandleMessage(w http.ResponseWriter, r *http.Request) {
	if r.Method != "POST" {
		http.Error(w, "Only POST allowed", http.StatusMethodNotAllowed)
		return
	}
	var message db.Message
	decoder := json.NewDecoder(r.Body)
	if decoder.Decode(&message) != nil {
		http.Error(w, "Invalid JSON", http.StatusBadRequest)
		return
	}
	//service.Inbox <- &message
}

func (self *WebService) HandleBatch(w http.ResponseWriter, r *http.Request) {
	if r.Method != "POST" {
		http.Error(w, "Only POST allowed", http.StatusMethodNotAllowed)
		return
	}
	/*   	values.Set("db", database)
	values.Set("id", id)
	values.Set("frame", fragment_type)
	values.Set("slice", fmt.Sprintf("%d", slice))
	values.Set("bitmap", compressed_string)
	*/

	err := r.ParseForm()
	if err != nil {
		http.Error(w, err.Error(), http.StatusBadRequest)
		return
	}

	database_name := r.Form.Get("db")
	if database_name == "" {
		http.Error(w, "Provide a database (db)", http.StatusNotFound)
		return
	}

	bitmap_id_string := r.Form.Get("id")
	if bitmap_id_string == "" {
		http.Error(w, "Provide a bitmap id (id)", http.StatusNotFound)
		return
	}
	bitmap_id, _ := strconv.ParseUint(bitmap_id_string, 10, 64)

	slice_string := r.Form.Get("slice")
	if bitmap_id_string == "" {
		http.Error(w, "Provide a slice (slice)", http.StatusNotFound)
		return
	}
	slice, _ := strconv.ParseInt(slice_string, 10, 32)

	frame := r.Form.Get("frame")
	if bitmap_id_string == "" {
		http.Error(w, "Provide a frame (frame)", http.StatusNotFound)
		return
	}

	compressed_bitmap := r.Form.Get("bitmap")
	if bitmap_id_string == "" {
		http.Error(w, "Provide a compressed base64 bitmap (bitmap)", http.StatusNotFound)
		return
	}

	results := self.service.Batch(database_name, frame, compressed_bitmap, bitmap_id, int(slice))

	encoder := json.NewEncoder(w)
	err = encoder.Encode(results)
	if err != nil {
		log.Println("Error Batch results")
		log.Println(spew.Sdump(r.Form))
		err = encoder.Encode("Bad Batch Request")
	}

}

func (self *WebService) HandleQuery(w http.ResponseWriter, r *http.Request) {
	if r.Method != "POST" {
		http.Error(w, "Only POST allowed", http.StatusMethodNotAllowed)
		return
	}
	err := r.ParseForm()
	if err != nil {
		http.Error(w, err.Error(), http.StatusBadRequest)
		return
	}

	database_name := r.Form.Get("db")
	if database_name == "" {
		database_name = config.GetString("default_db")
	}
	if database_name == "" {
		http.Error(w, "Provide a database (db)", http.StatusNotFound)
		return
	}
	pql := r.Form.Get("pql")
	if pql == "" {
		http.Error(w, "Provide a valid query string (pql)", http.StatusNotFound)
		return
	}

	results := self.service.Executor.RunPQL(database_name, pql)

	encoder := json.NewEncoder(w)
	err = encoder.Encode(results)
	if err != nil {
		log.Fatal("Error encoding stats")
	}

}

func (self *WebService) HandleStats(w http.ResponseWriter, r *http.Request) {
	if r.Method != "GET" {
		http.Error(w, "Only GET allowed", http.StatusMethodNotAllowed)
		return
	}
	encoder := json.NewEncoder(w)
	//stats := service.GetStats()
	//	stats := ""
	m := &runtime.MemStats{}
	runtime.ReadMemStats(m)
	stats := map[string]interface{}{"num_goroutines": runtime.NumGoroutine(), "Memory Aquired": m.Sys, "Memory Used": m.Alloc}

	err := encoder.Encode(stats)
	if err != nil {
		log.Fatal("Error encoding stats")
	}
}

func (self *WebService) HandleInfo(w http.ResponseWriter, r *http.Request) {
	if r.Method != "GET" {
		http.Error(w, "Only GET allowed", http.StatusMethodNotAllowed)
		return
	}
	spew.Fdump(w, self.service.Cluster)
}

func (self *WebService) HandleVersion(w http.ResponseWriter, r *http.Request) {
	if r.Method != "GET" {
		http.Error(w, "Only GET allowed", http.StatusMethodNotAllowed)
		return
	}

	fmt.Fprintf(w, "Pilosa v."+self.service.version+"\n")
}

func (self *WebService) HandleTest(w http.ResponseWriter, r *http.Request) {
	if r.Method != "GET" {
		http.Error(w, "Only GET allowed", http.StatusMethodNotAllowed)
		return
	}
	spew.Dump("TEST!")

	msg := new(db.Message)
	msg.Data = "mystring"
	self.service.Transport.Push(msg)

	msg2 := new(db.Message)
	msg2.Data = 789
	self.service.Transport.Push(msg2)
}

func (self *WebService) HandleProcesses(w http.ResponseWriter, r *http.Request) {
	if r.Method != "GET" {
		http.Error(w, "Only GET allowed", http.StatusMethodNotAllowed)
		return
	}
	encoder := json.NewEncoder(w)
	processes := self.service.ProcessMap.GetMetadata()
	err := encoder.Encode(processes)
	if err != nil {
		log.Fatal("Error encoding stats")
	}
}

func (self *WebService) HandlePing(w http.ResponseWriter, r *http.Request) {
	if r.Method != "GET" {
		http.Error(w, "Only GET allowed", http.StatusMethodNotAllowed)
		return
	}
	err := r.ParseForm()
	if err != nil {
		http.Error(w, err.Error(), http.StatusBadRequest)
		return
	}
	process_string := r.Form.Get("process")
	process_id, err := uuid.ParseUUID(process_string)
	if err != nil {
		http.Error(w, err.Error(), http.StatusBadRequest)
		return
	}
	_, err = self.service.ProcessMap.GetProcess(&process_id)
	if err != nil {
		http.Error(w, err.Error(), http.StatusNotFound)
		return
	}
	duration, err := self.service.Ping(&process_id)
	if err != nil {
		spew.Fdump(w, err)
		return
	}
	encoder := json.NewEncoder(w)
	encoder.Encode(map[string]float64{"duration": duration.Seconds()})
}

func (self *WebService) HandleListen(w http.ResponseWriter, r *http.Request) {
	w.Write([]byte(`
	<title>Pilosa - streaming client</title>

	`))
}

func (self *WebService) streamer(writer func(map[string]interface{}) error) {
	inbox := make(chan interface{}, 10)
	outbox := make(chan interface{}, 10)
	notify.Start("inbox", inbox)
	notify.Start("outbox", outbox)
	var obj interface{}
	var data interface{}
	var inmessage *db.Message
	var outmessage *db.Envelope
	var host string
	for {
		select {
		case obj = <-outbox:
			outmessage = obj.(*db.Envelope)
			data = outmessage.Message.Data
			host = outmessage.Host.String()
		case obj = <-inbox:
			inmessage = obj.(*db.Message)
			data = inmessage.Data
			host = ""
		}

		typ := reflect.TypeOf(data)

		err := writer(map[string]interface{}{
			"type": typ.String(),
			"dump": spew.Sdump(data),
			"host": host,
		})
		if err != nil {
			log.Println("stopping")
			notify.Stop("inbox", inbox)
			notify.Stop("outbox", outbox)
			drain(inbox)
			drain(outbox)
			return
		}
	}
}

func (self *WebService) HandleListenWS(w http.ResponseWriter, r *http.Request) {
	defer func() {
		err := recover()
		spew.Dump(err)
	}()
	ws, err := websocket.Upgrade(w, r, nil, 1024, 1024)
	if _, ok := err.(websocket.HandshakeError); ok {
		http.Error(w, "Not a websocket handshake", 400)
		return
	} else if err != nil {
		log.Println(err)
		return
	}
	self.streamer(func(data map[string]interface{}) error {
		return ws.WriteJSON(data)
	})
}

func drain(ch chan interface{}) {
	for {
		switch {
		case <-ch:
		default:
			return
		}
	}
}

func (self *WebService) HandleListenStream(w http.ResponseWriter, r *http.Request) {
	defer func() {
		err := recover()
		spew.Dump(err)
	}()
	writer := json.NewEncoder(w)
	flusher := w.(http.Flusher)
	self.streamer(func(data map[string]interface{}) error {
		err := writer.Encode(data)
		flusher.Flush()
		return err
	})
}
<|endoftext|>


<commit_before>package cemi

import (
	"bytes"
	"io"

	"github.com/vapourismo/knx-go/knx/encoding"
)

// MessageCode is used to identify the contents of a CEMI frame.
type MessageCode uint8

// Supported message codes
const (
	LBusmonInd   MessageCode = 0x2B
	LDataReq     MessageCode = 0x11
	LDataInd     MessageCode = 0x29
	LDataCon     MessageCode = 0x2E
	LRawReq      MessageCode = 0x10
	LRawInd      MessageCode = 0x2D
	LRawCon      MessageCode = 0x2F
	LPollDataReq MessageCode = 0x13
	LPollDataCon MessageCode = 0x25
)

// Segment is a protocol segment.
type Segment interface {
	io.WriterTo
}

// An UnsupportedMessage is the raw representation of a CEMI message body.
type UnsupportedMessage []byte

// WriteTo serializes the message and writes it to the given Writer.
func (data UnsupportedMessage) WriteTo(w io.Writer) (int64, error) {
	len, err := w.Write(data)
	return int64(len), err
}

// CEMI is the Common External Message Interface.
type CEMI struct {
	Code MessageCode
	Info []byte
	Body Segment
}

// ReadFrom initializes the structure by reading from the given Reader.
func (cemi *CEMI) ReadFrom(r io.Reader) (n int64, err error) {
	var infoLen uint8
	n, err = encoding.ReadSome(r, &cemi.Code, &infoLen)
	if err != nil {
		return
	}

	cemi.Info = make([]byte, int(infoLen))
	len, err := encoding.Read(r, cemi.Info)
	n += len

	if err != nil {
		return n, err
	}

	switch cemi.Code {
	case LBusmonInd:
		buffer := bytes.Buffer{}
		len, err = buffer.ReadFrom(r)
		n += len

		if err != nil {
			return n, err
		}

		cemi.Body = LBusmon(buffer.Bytes())

		return

	case LDataReq, LDataInd, LDataCon:
		ldata := &LData{}
		len, err = ldata.ReadFrom(r)
		n += len

		if err != nil {
			return n, err
		}

		cemi.Body = ldata

		return

	default:
		buffer := bytes.Buffer{}
		len, err = buffer.ReadFrom(r)
		n += len

		if err != nil {
			return n, err
		}

		cemi.Body = UnsupportedMessage(buffer.Bytes())

		return
	}
}

// WriteTo serializes the CEMI frame and writes it to the given Writer.
func (cemi *CEMI) WriteTo(w io.Writer) (int64, error) {
	var infoLen uint8
	var info []byte

	if len(cemi.Info) > 255 {
		infoLen = 255
		info = cemi.Info[:256]
	} else {
		infoLen = uint8(len(info))
		info = cemi.Info
	}

	return encoding.WriteSome(w, cemi.Code, infoLen, info, cemi.Body)
}
<commit_msg>Simplify CEMI ReadFrom method<commit_after>package cemi

import (
	"io"

	"github.com/vapourismo/knx-go/knx/encoding"
)

// MessageCode is used to identify the contents of a CEMI frame.
type MessageCode uint8

// Supported message codes
const (
	LBusmonInd   MessageCode = 0x2B
	LDataReq     MessageCode = 0x11
	LDataInd     MessageCode = 0x29
	LDataCon     MessageCode = 0x2E
	LRawReq      MessageCode = 0x10
	LRawInd      MessageCode = 0x2D
	LRawCon      MessageCode = 0x2F
	LPollDataReq MessageCode = 0x13
	LPollDataCon MessageCode = 0x25
)

// Segment is a protocol segment.
type Segment interface {
	io.WriterTo
}

// An UnsupportedMessage is the raw representation of a CEMI message body.
type UnsupportedMessage []byte

// WriteTo serializes the message and writes it to the given Writer.
func (data UnsupportedMessage) WriteTo(w io.Writer) (int64, error) {
	len, err := w.Write(data)
	return int64(len), err
}

// CEMI is the Common External Message Interface.
type CEMI struct {
	Code MessageCode
	Info []byte
	Body Segment
}

// ReadFrom initializes the structure by reading from the given Reader.
func (cemi *CEMI) ReadFrom(r io.Reader) (n int64, err error) {
	var infoLen uint8
	n, err = encoding.ReadSome(r, &cemi.Code, &infoLen)
	if err != nil {
		return
	}

	cemi.Info = make([]byte, int(infoLen))
	len, err := encoding.Read(r, cemi.Info)
	n += len

	if err != nil {
		return n, err
	}

	switch cemi.Code {
	case LBusmonInd:
		m, body := encoding.ReadAll(r)
		n += m

		cemi.Body = LBusmon(body)

		return

	case LDataReq, LDataInd, LDataCon:
		ldata := &LData{}
		len, err = ldata.ReadFrom(r)
		n += len

		if err != nil {
			return n, err
		}

		cemi.Body = ldata

		return

	default:
		m, body := encoding.ReadAll(r)
		n += m

		cemi.Body = UnsupportedMessage(body)

		return
	}
}

// WriteTo serializes the CEMI frame and writes it to the given Writer.
func (cemi *CEMI) WriteTo(w io.Writer) (int64, error) {
	var infoLen uint8
	var info []byte

	if len(cemi.Info) > 255 {
		infoLen = 255
		info = cemi.Info[:256]
	} else {
		infoLen = uint8(len(info))
		info = cemi.Info
	}

	return encoding.WriteSome(w, cemi.Code, infoLen, info, cemi.Body)
}
<|endoftext|>


<commit_before>// Copyright 2017 Ole Krüger.
// Licensed under the MIT license which can be found in the LICENSE file.

package dpt

import (
	"fmt"
)

// A DatapointValue is a value of a datapoint.
type DatapointValue interface {
	Pack() []byte
	Unpack(data []byte) error
	Unit() string
}

// DPT 1.001 (Switch)
type DPT_1001 bool

// DatapointValue interface functions
func (d DPT_1001) Pack() []byte {
	return packB1(bool(d))
}

func (d *DPT_1001) Unpack(data []byte) error {
	return unpackB1(data, (*bool)(d))
}

// Returns string representation of unit or nil if value has no unit
func (d DPT_1001) Unit() string {
	return ""
}

// Generates string representation of value
func (d DPT_1001) String() string {
	value_map := map[bool]string{false: "Off", true: "On"}

	return value_map[bool(d)]
}

// DPT 1.002 (Bool)
type DPT_1002 bool

// DatapointValue interface functions
func (d DPT_1002) Pack() []byte {
	return packB1(bool(d))
}

func (d *DPT_1002) Unpack(data []byte) error {
	return unpackB1(data, (*bool)(d))
}

// Returns string representation of unit or nil if value has no unit
func (d DPT_1002) Unit() string {
	return ""
}

// Generates string representation of value
func (d DPT_1002) String() string {
	value_map := map[bool]string{false: "False", true: "True"}

	return value_map[bool(d)]
}

// DPT 1.003 (Enable)
type DPT_1003 bool

// DatapointValue interface functions
func (d DPT_1003) Pack() []byte {
	return packB1(bool(d))
}

func (d *DPT_1003) Unpack(data []byte) error {
	return unpackB1(data, (*bool)(d))
}

// Returns string representation of unit or nil if value has no unit
func (d DPT_1003) Unit() string {
	return ""
}

// Generates string representation of value
func (d DPT_1003) String() string {
	value_map := map[bool]string{false: "Disable", true: "Enable"}

	return value_map[bool(d)]
}

// DPT 1.009 (OpenClose)
type DPT_1009 bool

// DatapointValue interface functions
func (d DPT_1009) Pack() []byte {
	return packB1(bool(d))
}

func (d *DPT_1009) Unpack(data []byte) error {
	return unpackB1(data, (*bool)(d))
}

// Returns string representation of unit or nil if value has no unit
func (d DPT_1009) Unit() string {
	return ""
}

// Generates string representation of value
func (d DPT_1009) String() string {
	value_map := map[bool]string{false: "Open", true: "Close"}

	return value_map[bool(d)]
}

// DPT 1.010 (Start)
type DPT_1010 bool

// DatapointValue interface functions
func (d DPT_1010) Pack() []byte {
	return packB1(bool(d))
}

func (d *DPT_1010) Unpack(data []byte) error {
	return unpackB1(data, (*bool)(d))
}

// Returns string representation of unit or nil if value has no unit
func (d DPT_1010) Unit() string {
	return ""
}

// Generates string representation of value
func (d DPT_1010) String() string {
	value_map := map[bool]string{false: "Stop", true: "Start"}

	return value_map[bool(d)]
}

// DPT 5.001 (Scaling)
type DPT_5001 float32

// DatapointValue interface functions
func (d DPT_5001) Pack() []byte {
	value := d

	// Clip the value to a valid range
	if value < 0 {
		value = 0
	}
	if value > 100 {
		value = 100
	}

	// Scale the value to the target range
	value /= 100
	value *= 255

	return packU8(uint8(value))
}

func (d *DPT_5001) Unpack(data []byte) error {
	var buf uint8

	err := unpackU8(data, &buf)
	if err == nil {
		value := DPT_5001(buf)

		// Scale the value
		value *= 100
		value /= 255

		// Check the value for valid range
		if value > 100 {
			return fmt.Errorf("value \"%.2f\" outside range [0, 100]", value)
		}

		*d = value
	}

	return nil
}

// Returns string representation of unit or nil if value has no unit
func (d DPT_5001) Unit() string {
	return "%"
}

// Generates string representation of value
func (d DPT_5001) String() string {
	return fmt.Sprintf("%.2f %%", float32(d))
}

// DPT 5.003 (Angle)
type DPT_5003 float32

// DatapointValue interface functions
func (d DPT_5003) Pack() []byte {
	value := d

	// Clip the value to a valid range
	if value < 0 {
		value = 0
	}
	if value > 360 {
		value = 360
	}

	// Scale the value to the target range
	value /= 360
	value *= 255

	return packU8(uint8(value))
}

func (d *DPT_5003) Unpack(data []byte) error {
	var buf uint8

	err := unpackU8(data, &buf)
	if err == nil {
		value := DPT_5003(buf)

		// Scale the value
		value *= 360
		value /= 255

		// Check the value for valid range
		if value > 360 {
			return fmt.Errorf("value \"%.2f\" outside range [0, 360]", value)
		}

		*d = value
	}

	return nil
}

// Returns string representation of unit or nil if value has no unit
func (d DPT_5003) Unit() string {
	return "°"
}

// Generates string representation of value
func (d DPT_5003) String() string {
	return fmt.Sprintf("%.2f °", float32(d))
}

// DPT 5.004 (Percept)
type DPT_5004 float32

// DatapointValue interface functions
func (d DPT_5004) Pack() []byte {
	value := d

	// Clip the value to a valid range
	if value < 0 {
		value = 0
	}
	if value > 255 {
		value = 255
	}

	return packU8(uint8(value))
}

func (d *DPT_5004) Unpack(data []byte) error {
	var buf uint8

	err := unpackU8(data, &buf)
	if err == nil {
		value := DPT_5004(buf)

		// Check the value for valid range
		if value > 255 {
			return fmt.Errorf("value \"%.2f\" outside range [0, 255]", value)
		}

		*d = value
	}

	return nil
}

// Returns string representation of unit or nil if value has no unit
func (d DPT_5004) Unit() string {
	return "%"
}

// Generates string representation of value
func (d DPT_5004) String() string {
	return fmt.Sprintf("%.2f %%", float32(d))
}

// DPT 9.001 (Temperature)
type DPT_9001 float32

// DatapointValue interface functions
func (d DPT_9001) Pack() []byte {
	value := d

	// Clip the value to a valid range
	if value < -273 {
		value = -273
	}
	if value > 670760 {
		value = 670760
	}

	return packF16(float32(value))
}

func (d *DPT_9001) Unpack(data []byte) error {
	var buf float32

	err := unpackF16(data, &buf)
	if err == nil {
		value := DPT_9001(buf)

		// Check the value for valid range
		if value < -273 {
			return fmt.Errorf("value \"%.2f\" outside range [-273, 670760]", value)
		}
		if value > 670760 {
			return fmt.Errorf("value \"%.2f\" outside range [-273, 670760]", value)
		}

		*d = value
	}

	return nil
}

// Returns string representation of unit or nil if value has no unit
func (d DPT_9001) Unit() string {
	return "°C"
}

// Generates string representation of value
func (d DPT_9001) String() string {
	return fmt.Sprintf("%.2f °C", float32(d))
}

// DPT 9.004 (Illumination)
type DPT_9004 float32

// DatapointValue interface functions
func (d DPT_9004) Pack() []byte {
	value := d

	// Clip the value to a valid range
	if value < 0 {
		value = 0
	}
	if value > 670760 {
		value = 670760
	}

	return packF16(float32(value))
}

func (d *DPT_9004) Unpack(data []byte) error {
	var buf float32

	err := unpackF16(data, &buf)
	if err == nil {
		value := DPT_9004(buf)

		// Check the value for valid range
		if value > 670760 {
			return fmt.Errorf("value \"%.2f\" outside range [0, 670760]", value)
		}

		*d = value
	}

	return nil
}

// Returns string representation of unit or nil if value has no unit
func (d DPT_9004) Unit() string {
	return "lux"
}

// Generates string representation of value
func (d DPT_9004) String() string {
	return fmt.Sprintf("%.2f lux", float32(d))
}
<commit_msg>Fix documentation for types<commit_after>// Copyright 2017 Ole Krüger.
// Licensed under the MIT license which can be found in the LICENSE file.

package dpt

import (
	"fmt"
)

// A DatapointValue is a value of a datapoint.
type DatapointValue interface {
	// Pack the datapoint to a byte array.
	Pack() []byte

	// Unpack a the datapoint value from a byte array.
	Unpack(data []byte) error
}

// DatapointMeta gives meta information about a datapoint type.
type DatapointMeta interface {
	Unit() string
}

// DPT_1001 represents DPT 1.001 / Switch.
type DPT_1001 bool

func (d DPT_1001) Pack() []byte {
	return packB1(bool(d))
}

func (d *DPT_1001) Unpack(data []byte) error {
	return unpackB1(data, (*bool)(d))
}

func (d DPT_1001) Unit() string {
	return nil
}

func (d DPT_1001) String() string {
	value_map := map[bool]string{false: "Off", true: "On"}

	return value_map[bool(d)]
}

// DPT_1002 represents DPT 1.002 / Bool.
type DPT_1002 bool

func (d DPT_1002) Pack() []byte {
	return packB1(bool(d))
}

func (d *DPT_1002) Unpack(data []byte) error {
	return unpackB1(data, (*bool)(d))
}

func (d DPT_1002) Unit() string {
	return nil
}

func (d DPT_1002) String() string {
	value_map := map[bool]string{false: "False", true: "True"}

	return value_map[bool(d)]
}

// DPT_1003 represents DPT 1.003 / Enable.
type DPT_1003 bool

func (d DPT_1003) Pack() []byte {
	return packB1(bool(d))
}

func (d *DPT_1003) Unpack(data []byte) error {
	return unpackB1(data, (*bool)(d))
}

func (d DPT_1003) Unit() string {
	return nil
}

func (d DPT_1003) String() string {
	value_map := map[bool]string{false: "Disable", true: "Enable"}

	return value_map[bool(d)]
}

// DPT_1009 represents DPT 1.009 / OpenClose.
type DPT_1009 bool

func (d DPT_1009) Pack() []byte {
	return packB1(bool(d))
}

func (d *DPT_1009) Unpack(data []byte) error {
	return unpackB1(data, (*bool)(d))
}

func (d DPT_1009) Unit() string {
	return ""
}

func (d DPT_1009) String() string {
	value_map := map[bool]string{false: "Open", true: "Close"}

	return value_map[bool(d)]
}

// DPT_1010 represents DPT 1.010 / Start.
type DPT_1010 bool

func (d DPT_1010) Pack() []byte {
	return packB1(bool(d))
}

func (d *DPT_1010) Unpack(data []byte) error {
	return unpackB1(data, (*bool)(d))
}

func (d DPT_1010) Unit() string {
	return nil
}

func (d DPT_1010) String() string {
	value_map := map[bool]string{false: "Stop", true: "Start"}

	return value_map[bool(d)]
}

// DPT_5001 represents DPT 5.001 / Scaling.
type DPT_5001 float32

func (d DPT_5001) Pack() []byte {
	value := d

	// Clip the value to a valid range
	if value < 0 {
		value = 0
	}
	if value > 100 {
		value = 100
	}

	// Scale the value to the target range
	value /= 100
	value *= 255

	return packU8(uint8(value))
}

func (d *DPT_5001) Unpack(data []byte) error {
	var buf uint8

	err := unpackU8(data, &buf)
	if err == nil {
		value := DPT_5001(buf)

		// Scale the value
		value *= 100
		value /= 255

		// Check the value for valid range
		if value > 100 {
			return fmt.Errorf("value \"%.2f\" outside range [0, 100]", value)
		}

		*d = value
	}

	return nil
}

func (d DPT_5001) Unit() string {
	return "%"
}

func (d DPT_5001) String() string {
	return fmt.Sprintf("%.2f %%", float32(d))
}

// DPT_5003 represents DPT 5.003 / Angle.
type DPT_5003 float32

func (d DPT_5003) Pack() []byte {
	value := d

	// Clip the value to a valid range
	if value < 0 {
		value = 0
	}
	if value > 360 {
		value = 360
	}

	// Scale the value to the target range
	value /= 360
	value *= 255

	return packU8(uint8(value))
}

func (d *DPT_5003) Unpack(data []byte) error {
	var buf uint8

	err := unpackU8(data, &buf)
	if err == nil {
		value := DPT_5003(buf)

		// Scale the value
		value *= 360
		value /= 255

		// Check the value for valid range
		if value > 360 {
			return fmt.Errorf("value \"%.2f\" outside range [0, 360]", value)
		}

		*d = value
	}

	return nil
}

func (d DPT_5003) Unit() string {
	return "°"
}

func (d DPT_5003) String() string {
	return fmt.Sprintf("%.2f °", float32(d))
}

// DPT_5004 represents DPT 5.004 / Percent_U8.
type DPT_5004 float32

func (d DPT_5004) Pack() []byte {
	value := d

	// Clip the value to a valid range
	if value < 0 {
		value = 0
	}
	if value > 255 {
		value = 255
	}

	return packU8(uint8(value))
}

func (d *DPT_5004) Unpack(data []byte) error {
	var buf uint8

	err := unpackU8(data, &buf)
	if err == nil {
		value := DPT_5004(buf)

		// Check the value for valid range
		if value > 255 {
			return fmt.Errorf("value \"%.2f\" outside range [0, 255]", value)
		}

		*d = value
	}

	return nil
}

func (d DPT_5004) Unit() string {
	return "%"
}

func (d DPT_5004) String() string {
	return fmt.Sprintf("%.2f %%", float32(d))
}

// DPT_9001 represents DPT 9.001 / Temperature.
type DPT_9001 float32

func (d DPT_9001) Pack() []byte {
	value := d

	// Clip the value to a valid range
	if value < -273 {
		value = -273
	}
	if value > 670760 {
		value = 670760
	}

	return packF16(float32(value))
}

func (d *DPT_9001) Unpack(data []byte) error {
	var buf float32

	err := unpackF16(data, &buf)
	if err == nil {
		value := DPT_9001(buf)

		// Check the value for valid range
		if value < -273 {
			return fmt.Errorf("value \"%.2f\" outside range [-273, 670760]", value)
		}
		if value > 670760 {
			return fmt.Errorf("value \"%.2f\" outside range [-273, 670760]", value)
		}

		*d = value
	}

	return nil
}

func (d DPT_9001) Unit() string {
	return "°C"
}

func (d DPT_9001) String() string {
	return fmt.Sprintf("%.2f °C", float32(d))
}

// DPT_9004 represents DPT 9.004 / Illumination.
type DPT_9004 float32

func (d DPT_9004) Pack() []byte {
	value := d

	// Clip the value to a valid range
	if value < 0 {
		value = 0
	}
	if value > 670760 {
		value = 670760
	}

	return packF16(float32(value))
}

func (d *DPT_9004) Unpack(data []byte) error {
	var buf float32

	err := unpackF16(data, &buf)
	if err == nil {
		value := DPT_9004(buf)

		// Check the value for valid range
		if value > 670760 {
			return fmt.Errorf("value \"%.2f\" outside range [0, 670760]", value)
		}

		*d = value
	}

	return nil
}

func (d DPT_9004) Unit() string {
	return "lux"
}

func (d DPT_9004) String() string {
	return fmt.Sprintf("%.2f lux", float32(d))
}
<|endoftext|>


<commit_before>package core

import (
	"encoding/json"
	"fmt"
	"github.com/mattbaird/elastigo/api"
)

// Multi GET API allows to get multiple documents based on an index, type (optional) and id (and possibly routing).
// The response includes a docs array with all the fetched documents, each element similar in structure to a document
// provided by the get API.
// see http://www.elasticsearch.org/guide/reference/api/multi-get.html
func MGet(pretty bool, index string, _type string, mgetRequest MGetRequestContainer) (MGetResponseContainer, error) {
	var url string
	var retval MGetResponseContainer
	if len(index) <= 0 {
		url = fmt.Sprintf("/_mget?%s", api.Pretty(pretty))
	}
	if len(_type) > 0 && len(index) > 0 {
		url = fmt.Sprintf("/%s/%s/_mget?%s", index, _type, api.Pretty(pretty))
	} else if len(index) > 0 {
		url = fmt.Sprintf("/%s/_mget?%s", index, api.Pretty(pretty))
	}
	body, err := api.DoCommand("GET", url, nil)
	if err != nil {
		return retval, err
	}
	if err == nil {
		// marshall into json
		jsonErr := json.Unmarshal([]byte(body), &retval)
		if jsonErr != nil {
			return retval, jsonErr
		}
	}
	fmt.Println(body)
	return retval, err
}

type MGetRequestContainer struct {
	Docs []MGetRequest `json:"docs"`
}

type MGetRequest struct {
	Index  string   `json:"_index"`
	Type   string   `json:"_type"`
	ID     string   `json:"_id"`
	Fields []string `json:"fields,omitifempty"`
}

type MGetResponseContainer struct {
	Docs []api.BaseResponse `json:"docs"`
}
<commit_msg>tweak to support multiple id requests<commit_after>package core

import (
	"encoding/json"
	"fmt"
	"github.com/mattbaird/elastigo/api"
)

// Multi GET API allows to get multiple documents based on an index, type (optional) and id (and possibly routing).
// The response includes a docs array with all the fetched documents, each element similar in structure to a document
// provided by the get API.
// see http://www.elasticsearch.org/guide/reference/api/multi-get.html
func MGet(pretty bool, index string, _type string, mgetRequest MGetRequestContainer) (MGetResponseContainer, error) {
	var url string
	var retval MGetResponseContainer
	if len(index) <= 0 {
		url = fmt.Sprintf("/_mget?%s", api.Pretty(pretty))
	}
	if len(_type) > 0 && len(index) > 0 {
		url = fmt.Sprintf("/%s/%s/_mget?%s", index, _type, api.Pretty(pretty))
	} else if len(index) > 0 {
		url = fmt.Sprintf("/%s/_mget?%s", index, api.Pretty(pretty))
	}
	body, err := api.DoCommand("GET", url, nil)
	if err != nil {
		return retval, err
	}
	if err == nil {
		// marshall into json
		jsonErr := json.Unmarshal([]byte(body), &retval)
		if jsonErr != nil {
			return retval, jsonErr
		}
	}
	fmt.Println(body)
	return retval, err
}

type MGetRequestContainer struct {
	Docs []MGetRequest `json:"docs"`
}

type MGetRequest struct {
	Index  string   `json:"_index"`
	Type   string   `json:"_type"`
	ID     string   `json:"_id"`
	IDS    []string `json:"_ids,omitifempty"`
	Fields []string `json:"fields,omitifempty"`
}

type MGetResponseContainer struct {
	Docs []api.BaseResponse `json:"docs"`
}
<|endoftext|>


<commit_before>// Copyright 2013 Matthew Baird
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//     http://www.apache.org/licenses/LICENSE-2.0
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package core

import (
	"encoding/json"
	"fmt"
	"github.com/mattbaird/elastigo/api"
)

// MGet allows the caller to get multiple documents based on an index, type (optional) and id (and possibly routing).
// The response includes a docs array with all the fetched documents, each element similar in structure to a document
// provided by the get API.
// see http://www.elasticsearch.org/guide/reference/api/multi-get.html
func MGet(index string, _type string, mgetRequest MGetRequestContainer, args map[string]interface{}) (MGetResponseContainer, error) {
	var url string
	var retval MGetResponseContainer
	if len(index) <= 0 {
		url = fmt.Sprintf("/_mget")
	}
	if len(_type) > 0 && len(index) > 0 {
		url = fmt.Sprintf("/%s/%s/_mget", index, _type)
	} else if len(index) > 0 {
		url = fmt.Sprintf("/%s/_mget", index)
	}
	body, err := api.DoCommand("GET", url, args, mgetRequest)
	if err != nil {
		return retval, err
	}
	if err == nil {
		// marshall into json
		jsonErr := json.Unmarshal(body, &retval)
		if jsonErr != nil {
			return retval, jsonErr
		}
	}
	return retval, err
}

type MGetRequestContainer struct {
	Docs []MGetRequest `json:"docs"`
}

type MGetRequest struct {
	Index  string   `json:"_index"`
	Type   string   `json:"_type"`
	ID     string   `json:"_id"`
	IDS    []string `json:"_ids,omitempty"`
	Fields []string `json:"fields,omitempty"`
}

type MGetResponseContainer struct {
	Docs []api.BaseResponse `json:"docs"`
}
<commit_msg>pass in request body<commit_after>// Copyright 2013 Matthew Baird
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//     http://www.apache.org/licenses/LICENSE-2.0
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package core

import (
	"encoding/json"
	"fmt"
	"github.com/mattbaird/elastigo/api"
)

// MGet allows the caller to get multiple documents based on an index, type (optional) and id (and possibly routing).
// The response includes a docs array with all the fetched documents, each element similar in structure to a document
// provided by the get API.
// see http://www.elasticsearch.org/guide/reference/api/multi-get.html
func MGet(index string, _type string, mgetRequest MGetRequestContainer, args map[string]interface{}) (MGetResponseContainer, error) {
	var url string
	var retval MGetResponseContainer
	if len(index) <= 0 {
		url = fmt.Sprintf("/_mget")
	}
	if len(_type) > 0 && len(index) > 0 {
		url = fmt.Sprintf("/%s/%s/_mget", index, _type)
	} else if len(index) > 0 {
		url = fmt.Sprintf("/%s/_mget", index)
	}
	body, err := api.DoCommand("GET", url, args, mgetRequest)
	if err != nil {
		return retval, err
	}
	if err == nil {
		// marshall into json
		jsonErr := json.Unmarshal(body, &retval)
		if jsonErr != nil {
			return retval, jsonErr
		}
	}
	return retval, err
}

type MGetRequestContainer struct {
	Docs []MGetRequest `json:"docs"`
}

type MGetRequest struct {
	Index  string   `json:"_index"`
	Type   string   `json:"_type"`
	ID     string   `json:"_id"`
	IDS    []string `json:"_ids,omitempty"`
	Fields []string `json:"fields,omitempty"`
}

type MGetResponseContainer struct {
	Docs []api.BaseResponse `json:"docs"`
}
<|endoftext|>


<commit_before>package kumoru

import (
	"bytes"
	"crypto/hmac"
	"crypto/md5"
	"crypto/sha256"
	"crypto/tls"
	"encoding/base64"
	"encoding/json"
	"fmt"
	"io/ioutil"
	"net/http"
	"net/http/httputil"
	"net/url"
	"os"
	"os/user"
	"reflect"
	"strings"
	"time"

	log "github.com/Sirupsen/logrus"
)

const (
	POST   = "POST"
	GET    = "GET"
	HEAD   = "HEAD"
	PUT    = "PUT"
	DELETE = "DELETE"
	PATCH  = "PATCH"
)

type Request *http.Request
type Response *http.Response

type KumoruClient struct {
	SliceData         []interface{}
	BasicAuth         struct{ UserName, Password string }
	BounceToRawString bool
	Client            *http.Client
	Data              map[string]interface{}
	Debug             bool
	EndPoint          *Endpoints
	Errors            []error
	FormData          url.Values
	Header            map[string]string
	Logger            *log.Logger
	Method            string
	QueryData         url.Values
	RawString         string
	Sign              bool
	TargetType        string
	Tokens            *Ktokens
	Transport         *http.Transport
	Url               string
}

// Used to create a new KumoruClient Object.
func New() *KumoruClient {

	config := os.Getenv("KUMORU_CONFIG")

	if config == "" {
		usr, err := user.Current()
		if err != nil {
			log.Fatal(err)
		}

		config = usr.HomeDir + "/.kumoru/config"
	}

	c, err := LoadCreds(config, "auth")
	if err != nil {
		log.Fatal(err)
	}

	e, err := LoadEndpoints(config, "endpoints")
	if err != nil {
		log.Fatal(err)
	}

	t, err := LoadTokens(config, "tokens")
	if err != nil {
		log.Fatal(err)
	}

	k := &KumoruClient{
		BasicAuth:         struct{ UserName, Password string }{c.UserName, c.Password},
		BounceToRawString: false,
		Client:            &http.Client{},
		Data:              make(map[string]interface{}),
		Debug:             false,
		SliceData:         []interface{}{},
		EndPoint:          &e,
		Errors:            nil,
		FormData:          url.Values{},
		Header:            make(map[string]string),
		Logger:            log.New(),
		QueryData:         url.Values{},
		RawString:         "",
		Sign:              false,
		TargetType:        "form",
		Tokens:            &t,
		Transport:         &http.Transport{},
		Url:               "",
	}

	return k
}

func (k *KumoruClient) SignRequest(enable bool) *KumoruClient {
	k.Sign = enable
	return k
}

func (k *KumoruClient) SetDebug(enable bool) *KumoruClient {
	k.Debug = enable
	return k
}

func (k *KumoruClient) SetLogger(logger *log.Logger) *KumoruClient {
	k.Logger = logger
	return k
}

// Clear KumoruClient data for a new request
func (k *KumoruClient) ClearKumoruClient() {
	k.BounceToRawString = false
	k.Data = make(map[string]interface{})
	k.Errors = nil
	k.FormData = url.Values{}
	k.Header = make(map[string]string)
	k.Method = ""
	k.QueryData = url.Values{}
	k.RawString = ""
	k.Sign = false
	k.TargetType = "form"
	k.Url = ""
	k.SliceData = []interface{}{}
}

func (k *KumoruClient) Get(targetUrl string) *KumoruClient {
	k.ClearKumoruClient()
	k.Method = GET
	k.Url = targetUrl
	k.Errors = nil

	return k
}

func (k *KumoruClient) Patch(targetUrl string) *KumoruClient {
	k.ClearKumoruClient()
	k.Method = PATCH
	k.Url = targetUrl
	k.Errors = nil

	return k
}

func (k *KumoruClient) Put(targetUrl string) *KumoruClient {
	k.ClearKumoruClient()
	k.Method = PUT
	k.Url = targetUrl
	k.Errors = nil

	return k
}

func (k *KumoruClient) Delete(targetUrl string) *KumoruClient {
	k.ClearKumoruClient()
	k.Method = DELETE
	k.Url = targetUrl
	k.Errors = nil

	return k
}

func (k *KumoruClient) Post(targetUrl string) *KumoruClient {
	k.ClearKumoruClient()
	k.Method = POST
	k.Url = targetUrl
	k.Errors = nil

	return k
}

func (k *KumoruClient) Head(targetUrl string) *KumoruClient {
	k.ClearKumoruClient()
	k.Method = POST
	k.Url = targetUrl
	k.Errors = nil

	return k
}

// SetHeader headers
// kumoru.New().
// POST("/application/B8658129-701E-432C-BD80-5D0F464EC932").
// SetHeader("Accept", "application/x-www-form-urlencoded")
func (k *KumoruClient) SetHeader(param string, value string) *KumoruClient {
	k.Header[param] = value
	return k
}

func (k *KumoruClient) Param(key string, value string) *KumoruClient {
	k.QueryData.Add(key, value)
	return k
}

// Probably can merge in the initialize section
func (k *KumoruClient) SetBasicAuth(username string, password string) *KumoruClient {
	k.BasicAuth = struct{ UserName, Password string }{username, password}
	return k
}

// Query fucntion forms a query-string in the url of GET method or body of POST method.
// Usage Example:
//
// kumoru.New().
// Get("/applications/").
// Query(`{ query: 'myapp' }`).
// Query(`{ limit: '5' }`).
// End()
//
// kumoru.New().
// Get("/applications/").
// Query("query=myapp&limit=5").
// End()
//
// kumoru.New().
// Get("/applications/").
// Query("query=myapp&limit=5").
// Query(`{ sort: 'asc' }`).
// End()

func (k *KumoruClient) Query(content interface{}) *KumoruClient {
	switch v := reflect.ValueOf(content); v.Kind() {

	case reflect.String:
		k.queryString(v.String())
	case reflect.Struct:
		k.queryStruct(v.Interface())
	default:
	}
	return k
}

func (k *KumoruClient) queryString(content string) *KumoruClient {
	var val map[string]string
	if err := json.Unmarshal([]byte(content), &val); err == nil {
		for key, v := range val {
			k.QueryData.Add(key, v)
		}
	} else {
		if queryVal, err := url.ParseQuery(content); err == nil {
			for key, _ := range queryVal {
				k.QueryData.Add(key, queryVal.Get(key))
			}
		} else {
			k.Errors = append(k.Errors, err)
		}
	}
	return k
}

func (k *KumoruClient) queryStruct(content interface{}) *KumoruClient {
	if marchalContent, err := json.Marshal(content); err != nil {
		k.Errors = append(k.Errors, err)
	} else {
		var val map[string]interface{}
		if err := json.Unmarshal(marchalContent, &val); err != nil {
			k.Errors = append(k.Errors, err)
		} else {
			for key, v := range val {
				key = strings.ToLower(key)
				k.QueryData.Add(key, v.(string))
			}
		}
	}
	return k
}

func (k *KumoruClient) TLSClientConfig(config *tls.Config) *KumoruClient {
	k.Transport.TLSClientConfig = config
	return k
}

// End() or EndBytes() must be called to execute the call otherwise it won't do a thing.
func (k *KumoruClient) End(callback ...func(response Response, body string, errs []error)) (Response, string, []error) {
	var bytesCallback []func(response Response, body []byte, errs []error)
	if len(callback) > 0 {
		bytesCallback = []func(response Response, body []byte, errs []error){
			func(response Response, body []byte, errs []error) {
				callback[0](response, string(body), errs)
			},
		}
	}
	resp, body, errs := k.EndBytes(bytesCallback...)
	bodyString := string(body)
	return resp, bodyString, errs
}

// EndBytes should be used when you want the body as bytes.
func (k *KumoruClient) EndBytes(callback ...func(response Response, body []byte, errs []error)) (Response, []byte, []error) {
	// check whether there is an error. if yes, return all errors
	if len(k.Errors) != 0 {
		return nil, nil, k.Errors
	}

	req, err := k.NewRequest()

	if err != nil {
		k.Errors = append(k.Errors, err)
		return nil, nil, k.Errors
	}

	for key, v := range k.Header {
		req.Header.Set(key, v)
	}

	// Add all querystring from Query func
	q := req.URL.Query()
	for key, v := range k.QueryData {
		for _, vv := range v {
			q.Add(key, vv)
		}
	}
	req.URL.RawQuery = q.Encode()

	// Sign Request
	if !k.Sign {
		if k.BasicAuth != struct{ UserName, Password string }{} {
			req.SetBasicAuth(k.BasicAuth.UserName, k.BasicAuth.Password)
		}
	} else {
		b1 := strings.NewReader(k.RawString)
		date := time.Now()
		signingString := k.Method + "\n"

		b, err := ioutil.ReadAll(b1)

		if err != nil {
			log.Fatal(err)
		}
		if len(b) != 0 {
			data := md5.Sum(b)
			newdata := fmt.Sprintf("content-md5:%x", string(data[:16]))

			req.Header.Set("Content-MD5", newdata)

			signingString += fmt.Sprintf("content-type:%v", req.Header.Get("Content-Type")+"\n")
		}
		u, _ := url.Parse(k.Url)
		signingString += "x-kumoru-date:" + date.String() + "\n" + u.Path
		req.Header.Set("X-Kumoru-Date", date.String())

		h := hmac.New(sha256.New, []byte(k.Tokens.Private))
		h.Write([]byte(signingString))
		digest := fmt.Sprintf("%x", h.Sum(nil))

		req.Header.Set("Authorization", base64.StdEncoding.EncodeToString([]byte(k.Tokens.Public+":"+digest)))
	}

	// Set Transport
	k.Client.Transport = k.Transport

	// Log details of this request
	if k.Debug {
		dump, err := httputil.DumpRequest(req, true)
		if err != nil {
			k.Logger.Println("Error:", err)
		} else {
			k.Logger.Printf("HTTP Request: %s", string(dump))
		}
	}

	// Send request
	resp, err := k.Client.Do(req)
	if err != nil {
		k.Errors = append(k.Errors, err)
		return nil, nil, k.Errors
	}
	defer resp.Body.Close()

	// Log details of this response
	if k.Debug {
		dump, err := httputil.DumpResponse(resp, true)
		if nil != err {
			k.Logger.Println("Error:", err)
		} else {
			k.Logger.Printf("HTTP Response: %s", string(dump))
		}
	}

	body, _ := ioutil.ReadAll(resp.Body)
	// Reset resp.Body so it can be use again
	resp.Body = ioutil.NopCloser(bytes.NewBuffer(body))
	// deep copy response to give it to both return and callback func
	respCallback := *resp
	if len(callback) != 0 {
		callback[0](&respCallback, body, k.Errors)
	}
	return resp, body, nil
}

func changeMapToURLValues(data map[string]interface{}) url.Values {
	var newUrlValues = url.Values{}
	for k, v := range data {
		switch val := v.(type) {
		case string:
			newUrlValues.Add(k, val)
		case []string:
			for _, element := range val {
				newUrlValues.Add(k, element)
			}
		// if a number, change to string
		// json.Number used to protect against a wrong (for GoRequest) default conversion
		// which always converts number to float64.
		// This type is caused by using Decoder.UseNumber()
		case json.Number:
			newUrlValues.Add(k, string(val))
		}
	}
	return newUrlValues
}
<commit_msg>replace user.Current() with os.Getenv("HOME")<commit_after>package kumoru

import (
	"bytes"
	"crypto/hmac"
	"crypto/md5"
	"crypto/sha256"
	"crypto/tls"
	"encoding/base64"
	"encoding/json"
	"fmt"
	"io/ioutil"
	"net/http"
	"net/http/httputil"
	"net/url"
	"os"
	"reflect"
	"strings"
	"time"

	log "github.com/Sirupsen/logrus"
)

const (
	POST   = "POST"
	GET    = "GET"
	HEAD   = "HEAD"
	PUT    = "PUT"
	DELETE = "DELETE"
	PATCH  = "PATCH"
)

type Request *http.Request
type Response *http.Response

type KumoruClient struct {
	SliceData         []interface{}
	BasicAuth         struct{ UserName, Password string }
	BounceToRawString bool
	Client            *http.Client
	Data              map[string]interface{}
	Debug             bool
	EndPoint          *Endpoints
	Errors            []error
	FormData          url.Values
	Header            map[string]string
	Logger            *log.Logger
	Method            string
	QueryData         url.Values
	RawString         string
	Sign              bool
	TargetType        string
	Tokens            *Ktokens
	Transport         *http.Transport
	Url               string
}

// Used to create a new KumoruClient Object.
func New() *KumoruClient {

	config := os.Getenv("KUMORU_CONFIG")

	if config == "" {
		usrHome := os.Getenv("HOME")

		config = usrHome + "/.kumoru/config"

	}

	c, err := LoadCreds(config, "auth")
	if err != nil {
		log.Fatal(err)
	}

	e, err := LoadEndpoints(config, "endpoints")
	if err != nil {
		log.Fatal(err)
	}

	t, err := LoadTokens(config, "tokens")
	if err != nil {
		log.Fatal(err)
	}

	k := &KumoruClient{
		BasicAuth:         struct{ UserName, Password string }{c.UserName, c.Password},
		BounceToRawString: false,
		Client:            &http.Client{},
		Data:              make(map[string]interface{}),
		Debug:             false,
		SliceData:         []interface{}{},
		EndPoint:          &e,
		Errors:            nil,
		FormData:          url.Values{},
		Header:            make(map[string]string),
		Logger:            log.New(),
		QueryData:         url.Values{},
		RawString:         "",
		Sign:              false,
		TargetType:        "form",
		Tokens:            &t,
		Transport:         &http.Transport{},
		Url:               "",
	}

	return k
}

func (k *KumoruClient) SignRequest(enable bool) *KumoruClient {
	k.Sign = enable
	return k
}

func (k *KumoruClient) SetDebug(enable bool) *KumoruClient {
	k.Debug = enable
	return k
}

func (k *KumoruClient) SetLogger(logger *log.Logger) *KumoruClient {
	k.Logger = logger
	return k
}

// Clear KumoruClient data for a new request
func (k *KumoruClient) ClearKumoruClient() {
	k.BounceToRawString = false
	k.Data = make(map[string]interface{})
	k.Errors = nil
	k.FormData = url.Values{}
	k.Header = make(map[string]string)
	k.Method = ""
	k.QueryData = url.Values{}
	k.RawString = ""
	k.Sign = false
	k.TargetType = "form"
	k.Url = ""
	k.SliceData = []interface{}{}
}

func (k *KumoruClient) Get(targetUrl string) *KumoruClient {
	k.ClearKumoruClient()
	k.Method = GET
	k.Url = targetUrl
	k.Errors = nil

	return k
}

func (k *KumoruClient) Patch(targetUrl string) *KumoruClient {
	k.ClearKumoruClient()
	k.Method = PATCH
	k.Url = targetUrl
	k.Errors = nil

	return k
}

func (k *KumoruClient) Put(targetUrl string) *KumoruClient {
	k.ClearKumoruClient()
	k.Method = PUT
	k.Url = targetUrl
	k.Errors = nil

	return k
}

func (k *KumoruClient) Delete(targetUrl string) *KumoruClient {
	k.ClearKumoruClient()
	k.Method = DELETE
	k.Url = targetUrl
	k.Errors = nil

	return k
}

func (k *KumoruClient) Post(targetUrl string) *KumoruClient {
	k.ClearKumoruClient()
	k.Method = POST
	k.Url = targetUrl
	k.Errors = nil

	return k
}

func (k *KumoruClient) Head(targetUrl string) *KumoruClient {
	k.ClearKumoruClient()
	k.Method = POST
	k.Url = targetUrl
	k.Errors = nil

	return k
}

// SetHeader headers
// kumoru.New().
// POST("/application/B8658129-701E-432C-BD80-5D0F464EC932").
// SetHeader("Accept", "application/x-www-form-urlencoded")
func (k *KumoruClient) SetHeader(param string, value string) *KumoruClient {
	k.Header[param] = value
	return k
}

func (k *KumoruClient) Param(key string, value string) *KumoruClient {
	k.QueryData.Add(key, value)
	return k
}

// Probably can merge in the initialize section
func (k *KumoruClient) SetBasicAuth(username string, password string) *KumoruClient {
	k.BasicAuth = struct{ UserName, Password string }{username, password}
	return k
}

// Query fucntion forms a query-string in the url of GET method or body of POST method.
// Usage Example:
//
// kumoru.New().
// Get("/applications/").
// Query(`{ query: 'myapp' }`).
// Query(`{ limit: '5' }`).
// End()
//
// kumoru.New().
// Get("/applications/").
// Query("query=myapp&limit=5").
// End()
//
// kumoru.New().
// Get("/applications/").
// Query("query=myapp&limit=5").
// Query(`{ sort: 'asc' }`).
// End()

func (k *KumoruClient) Query(content interface{}) *KumoruClient {
	switch v := reflect.ValueOf(content); v.Kind() {

	case reflect.String:
		k.queryString(v.String())
	case reflect.Struct:
		k.queryStruct(v.Interface())
	default:
	}
	return k
}

func (k *KumoruClient) queryString(content string) *KumoruClient {
	var val map[string]string
	if err := json.Unmarshal([]byte(content), &val); err == nil {
		for key, v := range val {
			k.QueryData.Add(key, v)
		}
	} else {
		if queryVal, err := url.ParseQuery(content); err == nil {
			for key, _ := range queryVal {
				k.QueryData.Add(key, queryVal.Get(key))
			}
		} else {
			k.Errors = append(k.Errors, err)
		}
	}
	return k
}

func (k *KumoruClient) queryStruct(content interface{}) *KumoruClient {
	if marchalContent, err := json.Marshal(content); err != nil {
		k.Errors = append(k.Errors, err)
	} else {
		var val map[string]interface{}
		if err := json.Unmarshal(marchalContent, &val); err != nil {
			k.Errors = append(k.Errors, err)
		} else {
			for key, v := range val {
				key = strings.ToLower(key)
				k.QueryData.Add(key, v.(string))
			}
		}
	}
	return k
}

func (k *KumoruClient) TLSClientConfig(config *tls.Config) *KumoruClient {
	k.Transport.TLSClientConfig = config
	return k
}

// End() or EndBytes() must be called to execute the call otherwise it won't do a thing.
func (k *KumoruClient) End(callback ...func(response Response, body string, errs []error)) (Response, string, []error) {
	var bytesCallback []func(response Response, body []byte, errs []error)
	if len(callback) > 0 {
		bytesCallback = []func(response Response, body []byte, errs []error){
			func(response Response, body []byte, errs []error) {
				callback[0](response, string(body), errs)
			},
		}
	}
	resp, body, errs := k.EndBytes(bytesCallback...)
	bodyString := string(body)
	return resp, bodyString, errs
}

// EndBytes should be used when you want the body as bytes.
func (k *KumoruClient) EndBytes(callback ...func(response Response, body []byte, errs []error)) (Response, []byte, []error) {
	// check whether there is an error. if yes, return all errors
	if len(k.Errors) != 0 {
		return nil, nil, k.Errors
	}

	req, err := k.NewRequest()

	if err != nil {
		k.Errors = append(k.Errors, err)
		return nil, nil, k.Errors
	}

	for key, v := range k.Header {
		req.Header.Set(key, v)
	}

	// Add all querystring from Query func
	q := req.URL.Query()
	for key, v := range k.QueryData {
		for _, vv := range v {
			q.Add(key, vv)
		}
	}
	req.URL.RawQuery = q.Encode()

	// Sign Request
	if !k.Sign {
		if k.BasicAuth != struct{ UserName, Password string }{} {
			req.SetBasicAuth(k.BasicAuth.UserName, k.BasicAuth.Password)
		}
	} else {
		b1 := strings.NewReader(k.RawString)
		date := time.Now()
		signingString := k.Method + "\n"

		b, err := ioutil.ReadAll(b1)

		if err != nil {
			log.Fatal(err)
		}
		if len(b) != 0 {
			data := md5.Sum(b)
			newdata := fmt.Sprintf("content-md5:%x", string(data[:16]))

			req.Header.Set("Content-MD5", newdata)

			signingString += fmt.Sprintf("content-type:%v", req.Header.Get("Content-Type")+"\n")
		}
		u, _ := url.Parse(k.Url)
		signingString += "x-kumoru-date:" + date.String() + "\n" + u.Path
		req.Header.Set("X-Kumoru-Date", date.String())

		h := hmac.New(sha256.New, []byte(k.Tokens.Private))
		h.Write([]byte(signingString))
		digest := fmt.Sprintf("%x", h.Sum(nil))

		req.Header.Set("Authorization", base64.StdEncoding.EncodeToString([]byte(k.Tokens.Public+":"+digest)))
	}

	// Set Transport
	k.Client.Transport = k.Transport

	// Log details of this request
	if k.Debug {
		dump, err := httputil.DumpRequest(req, true)
		if err != nil {
			k.Logger.Println("Error:", err)
		} else {
			k.Logger.Printf("HTTP Request: %s", string(dump))
		}
	}

	// Send request
	resp, err := k.Client.Do(req)
	if err != nil {
		k.Errors = append(k.Errors, err)
		return nil, nil, k.Errors
	}
	defer resp.Body.Close()

	// Log details of this response
	if k.Debug {
		dump, err := httputil.DumpResponse(resp, true)
		if nil != err {
			k.Logger.Println("Error:", err)
		} else {
			k.Logger.Printf("HTTP Response: %s", string(dump))
		}
	}

	body, _ := ioutil.ReadAll(resp.Body)
	// Reset resp.Body so it can be use again
	resp.Body = ioutil.NopCloser(bytes.NewBuffer(body))
	// deep copy response to give it to both return and callback func
	respCallback := *resp
	if len(callback) != 0 {
		callback[0](&respCallback, body, k.Errors)
	}
	return resp, body, nil
}

func changeMapToURLValues(data map[string]interface{}) url.Values {
	var newUrlValues = url.Values{}
	for k, v := range data {
		switch val := v.(type) {
		case string:
			newUrlValues.Add(k, val)
		case []string:
			for _, element := range val {
				newUrlValues.Add(k, element)
			}
		// if a number, change to string
		// json.Number used to protect against a wrong (for GoRequest) default conversion
		// which always converts number to float64.
		// This type is caused by using Decoder.UseNumber()
		case json.Number:
			newUrlValues.Add(k, string(val))
		}
	}
	return newUrlValues
}
<|endoftext|>


<commit_before>package main

import (
	"fmt"
	"github.com/gophergala2016/goad/sqsadaptor"
	"io/ioutil"
	"net/http"
	"os"
	"strconv"
	"sync"
	"time"
)

func main() {
	address := os.Args[1]
	concurrencycount, err := strconv.Atoi(os.Args[2])
	if err != nil {
		fmt.Printf("ERROR %s\n", err)
		return
	}
	requestcount, err := strconv.Atoi(os.Args[3])
	sqsurl := os.Args[4]
	if err != nil {
		fmt.Printf("ERROR %s\n", err)
		return
	}
	client := &http.Client{}
	fmt.Printf("Will spawn %d workers each making %d requests to %s\n", concurrencycount, requestcount, address)
	runLoadTest(client, sqsurl, address, requestcount, concurrencycount)
}

func runLoadTest(client *http.Client, sqsurl string, url string, requestcount int, concurrencycount int) {
	sqsAdaptor := sqsadaptor.NewDummyAdaptor(sqsurl)

	totalRequests := requestcount * concurrencycount
	ch := make(chan sqsadaptor.Result, totalRequests)
	var wg sync.WaitGroup
	for i := 0; i < concurrencycount; i++ {
		wg.Add(1)
		go fetch(client, url, requestcount, ch, &wg)
	}
	fmt.Println("Waiting for results…")

	completedRequests := 0
	for completedRequests < totalRequests {
		r := <-ch
		completedRequests++
		if completedRequests%10 == 0 {
			fmt.Printf("\r%.2f%% done (%d requests out of %d)", (float64(completedRequests)/float64(totalRequests))*100.0, completedRequests, totalRequests)
		}
		sqsAdaptor.SendResult(r)
	}
	wg.Wait()
	fmt.Printf("\nYay🎈\n")

}

func fetch(client *http.Client, address string, requestcount int, ch chan sqsadaptor.Result, wg *sync.WaitGroup) {
	defer wg.Done()
	fmt.Printf("Fetching %s %d times\n", address, requestcount)
	for i := 0; i < requestcount; i++ {
		start := time.Now()
		req, err := http.NewRequest("GET", address, nil)
		req.Header.Add("User-Agent", "GOAD/0.1")

		response, err := client.Do(req)
		if err != nil {
			fmt.Printf("ERROR %s\n", err)
			return
		}
		buf := []byte(" ")
		_, err = response.Body.Read(buf)
		elapsedFirstByte := time.Since(start)
		_, err = ioutil.ReadAll(response.Body)
		elapsedLastByte := time.Since(start)
		if err != nil {
			fmt.Printf("ERROR %s\n", err)
			return
		}
		elapsed := time.Since(start)
		result := sqsadaptor.Result{
			start.Format(time.RFC3339),
			req.URL.Host,
			"Fetch",
			response.StatusCode,
			elapsed.Nanoseconds(),
			elapsedFirstByte.Nanoseconds(),
			elapsedLastByte.Nanoseconds(),
			len(buf),
			"Finished",
		}
		ch <- result
	}

}
<commit_msg>Also send results for failed requests<commit_after>package main

import (
	"fmt"
	"github.com/gophergala2016/goad/sqsadaptor"
	"io/ioutil"
	"net/http"
	"os"
	"strconv"
	"sync"
	"time"
)

func main() {
	address := os.Args[1]
	concurrencycount, err := strconv.Atoi(os.Args[2])
	if err != nil {
		fmt.Printf("ERROR %s\n", err)
		return
	}
	requestcount, err := strconv.Atoi(os.Args[3])
	sqsurl := os.Args[4]
	if err != nil {
		fmt.Printf("ERROR %s\n", err)
		return
	}
	client := &http.Client{}
	fmt.Printf("Will spawn %d workers each making %d requests to %s\n", concurrencycount, requestcount, address)
	runLoadTest(client, sqsurl, address, requestcount, concurrencycount)
}

func runLoadTest(client *http.Client, sqsurl string, url string, requestcount int, concurrencycount int) {
	sqsAdaptor := sqsadaptor.NewDummyAdaptor(sqsurl)

	totalRequests := requestcount * concurrencycount
	ch := make(chan sqsadaptor.Result, totalRequests)
	var wg sync.WaitGroup
	for i := 0; i < concurrencycount; i++ {
		wg.Add(1)
		go fetch(client, url, requestcount, ch, &wg)
	}
	fmt.Println("Waiting for results…")

	completedRequests := 0
	for completedRequests < totalRequests {
		r := <-ch
		completedRequests++
		if completedRequests%10 == 0 {
			fmt.Printf("\r%.2f%% done (%d requests out of %d)", (float64(completedRequests)/float64(totalRequests))*100.0, completedRequests, totalRequests)
		}
		sqsAdaptor.SendResult(r)
	}
	wg.Wait()
	fmt.Printf("\nYay🎈\n")

}

func fetch(client *http.Client, address string, requestcount int, ch chan sqsadaptor.Result, wg *sync.WaitGroup) {
	defer wg.Done()
	fmt.Printf("Fetching %s %d times\n", address, requestcount)
	for i := 0; i < requestcount; i++ {
		start := time.Now()
		req, err := http.NewRequest("GET", address, nil)
		req.Header.Add("User-Agent", "GOAD/0.1")
		response, err := client.Do(req)
		var status string
		var elapsedFirstByte time.Duration
		var elapsedLastByte time.Duration
		var elapsed time.Duration
		buf := []byte(" ")
		if err != nil {
			status = fmt.Sprintf("client.Do() failed: %s\n", err)
		} else {
			_, err = response.Body.Read(buf)
			if err != nil {
				status = fmt.Sprintf("reading first byte failed: %s\n", err)
			}
			elapsedFirstByte = time.Since(start)
			_, err = ioutil.ReadAll(response.Body)
			elapsedLastByte = time.Since(start)
			if err != nil {
				status = fmt.Sprintf("reading response body failed: %s\n", err)
			} else {
				status = "Success"
			}
			elapsed = time.Since(start)
		}
		result := sqsadaptor.Result{
			start.Format(time.RFC3339),
			req.URL.Host,
			req.Method,
			response.StatusCode,
			elapsed.Nanoseconds(),
			elapsedFirstByte.Nanoseconds(),
			elapsedLastByte.Nanoseconds(),
			len(buf),
			status,
		}
		ch <- result
	}

}
<|endoftext|>