<commit_before>package mat

// #include <string.h>
// #include <mat.h>
import "C"

import (
	"errors"
	"reflect"
	"unsafe"
)

// Get reads an object from the file.
func (f *File) Get(name string, object interface{}) error {
	value := reflect.ValueOf(object)
	if value.Kind() != reflect.Ptr {
		return errors.New("expected a pointer")
	}

	cname := C.CString(name)
	defer C.free(unsafe.Pointer(cname))

	return f.readArray(cname, value)
}

func (f *File) readArray(name *C.char, value reflect.Value) error {
	ivalue := reflect.Indirect(value)
	switch ivalue.Kind() {
	case reflect.Struct:
		return f.readStruct(name, ivalue)
	default:
		return f.readMatrix(name, ivalue)
	}
}

func (f *File) readMatrix(name *C.char, ivalue reflect.Value) error {
	var classid C.mxClassID
	var read func(unsafe.Pointer, C.size_t)
	var scalar bool

	if ivalue.Kind() == reflect.Slice {
		classid, read = readSlice(ivalue)
		scalar = false
	} else {
		classid, read = readScalar(ivalue)
		scalar = true
	}

	if classid == C.mxUNKNOWN_CLASS {
		return errors.New("unsupported type")
	}

	array := C.matGetVariable(f.mat, name)
	if array == nil {
		return errors.New("cannot find the variable")
	}
	defer C.mxDestroyArray(array)

	if classid != C.mxGetClassID(array) {
		return errors.New("data type mismatch")
	}

	count := C.mxGetM(array) * C.mxGetN(array)
	if scalar && count != 1 {
		return errors.New("data size mismatch")
	}

	parray := unsafe.Pointer(C.mxGetPr(array))
	if parray == nil {
		return errors.New("cannot read the variable")
	}

	read(parray, count)

	return nil
}

func (f *File) readStruct(name *C.char, ivalue reflect.Value) error {
	return nil
}

func readScalar(iv reflect.Value) (C.mxClassID, func(unsafe.Pointer, C.size_t)) {
	switch iv.Kind() {
	case reflect.Int8:
		return C.mxINT8_CLASS, func(p unsafe.Pointer, _ C.size_t) {
			iv.SetInt(int64(*(*int8)(p)))
		}
	case reflect.Uint8:
		return C.mxUINT8_CLASS, func(p unsafe.Pointer, _ C.size_t) {
			iv.SetUint(uint64(*(*uint8)(p)))
		}
	case reflect.Int16:
		return C.mxINT16_CLASS, func(p unsafe.Pointer, _ C.size_t) {
			iv.SetInt(int64(*(*int16)(p)))
		}
	case reflect.Uint16:
		return C.mxUINT16_CLASS, func(p unsafe.Pointer, _ C.size_t) {
			iv.SetUint(uint64(*(*uint16)(p)))
		}
	case reflect.Int32:
		return C.mxINT32_CLASS, func(p unsafe.Pointer, _ C.size_t) {
			iv.SetInt(int64(*(*int32)(p)))
		}
	case reflect.Uint32:
		return C.mxUINT32_CLASS, func(p unsafe.Pointer, _ C.size_t) {
			iv.SetUint(uint64(*(*uint32)(p)))
		}
	case reflect.Int64:
		return C.mxINT64_CLASS, func(p unsafe.Pointer, _ C.size_t) {
			iv.SetInt(int64(*(*int64)(p)))
		}
	case reflect.Uint64:
		return C.mxUINT64_CLASS, func(p unsafe.Pointer, _ C.size_t) {
			iv.SetUint(uint64(*(*uint64)(p)))
		}
	case reflect.Float32:
		return C.mxSINGLE_CLASS, func(p unsafe.Pointer, _ C.size_t) {
			iv.SetFloat(float64(*(*float32)(p)))
		}
	case reflect.Float64:
		return C.mxDOUBLE_CLASS, func(p unsafe.Pointer, _ C.size_t) {
			iv.SetFloat(float64(*(*float64)(p)))
		}
	default:
		return C.mxUNKNOWN_CLASS, nil
	}
}

func readSlice(iv reflect.Value) (C.mxClassID, func(unsafe.Pointer, C.size_t)) {
	read := func(w interface{}, p unsafe.Pointer, s C.size_t) {
		iw := reflect.Indirect(reflect.ValueOf(w))
		C.memcpy(unsafe.Pointer(iw.Pointer()), p, s)
		iv.Set(iw)
	}

	switch iv.Type().Elem().Kind() {
	case reflect.Int8:
		return C.mxINT8_CLASS, func(p unsafe.Pointer, c C.size_t) {
			w := make([]int8, c)
			read(&w, p, 1*c)
		}
	case reflect.Uint8:
		return C.mxUINT8_CLASS, func(p unsafe.Pointer, c C.size_t) {
			w := make([]uint8, c)
			read(&w, p, 1*c)
		}
	case reflect.Int16:
		return C.mxINT16_CLASS, func(p unsafe.Pointer, c C.size_t) {
			w := make([]int16, c)
			read(&w, p, 2*c)
		}
	case reflect.Uint16:
		return C.mxUINT16_CLASS, func(p unsafe.Pointer, c C.size_t) {
			w := make([]uint16, c)
			read(&w, p, 2*c)
		}
	case reflect.Int32:
		return C.mxINT32_CLASS, func(p unsafe.Pointer, c C.size_t) {
			w := make([]int32, c)
			read(&w, p, 4*c)
		}
	case reflect.Uint32:
		return C.mxUINT32_CLASS, func(p unsafe.Pointer, c C.size_t) {
			w := make([]uint32, c)
			read(&w, p, 4*c)
		}
	case reflect.Int64:
		return C.mxINT64_CLASS, func(p unsafe.Pointer, c C.size_t) {
			w := make([]int64, c)
			read(&w, p, 8*c)
		}
	case reflect.Uint64:
		return C.mxUINT64_CLASS, func(p unsafe.Pointer, c C.size_t) {
			w := make([]uint64, c)
			read(&w, p, 8*c)
		}
	case reflect.Float32:
		return C.mxSINGLE_CLASS, func(p unsafe.Pointer, c C.size_t) {
			w := make([]float32, c)
			read(&w, p, 4*c)
		}
	case reflect.Float64:
		return C.mxDOUBLE_CLASS, func(p unsafe.Pointer, c C.size_t) {
			w := make([]float64, c)
			read(&w, p, 8*c)
		}
	default:
		return C.mxUNKNOWN_CLASS, nil
	}
}
<commit_msg>Implemented the reading of structs<commit_after>package mat

// #include <string.h>
// #include <mat.h>
import "C"

import (
	"errors"
	"reflect"
	"unsafe"
)

// Get reads an object from the file.
func (f *File) Get(name string, object interface{}) error {
	value := reflect.ValueOf(object)
	if value.Kind() != reflect.Ptr {
		return errors.New("expected a pointer")
	}

	cname := C.CString(name)
	defer C.free(unsafe.Pointer(cname))

	array := C.matGetVariable(f.mat, cname)
	if array == nil {
		return errors.New("cannot find the variable")
	}
	defer C.mxDestroyArray(array)

	return f.readArray(array, value)
}

func (f *File) readArray(array *C.mxArray, value reflect.Value) error {
	ivalue := reflect.Indirect(value)
	switch ivalue.Kind() {
	case reflect.Struct:
		return f.readStruct(array, ivalue)
	default:
		return f.readMatrix(array, ivalue)
	}
}

func (f *File) readMatrix(array *C.mxArray, ivalue reflect.Value) error {
	var classid C.mxClassID
	var read func(unsafe.Pointer, C.size_t)
	var scalar bool

	if ivalue.Kind() == reflect.Slice {
		classid, read = readSlice(ivalue)
		scalar = false
	} else {
		classid, read = readScalar(ivalue)
		scalar = true
	}

	if classid == C.mxUNKNOWN_CLASS {
		return errors.New("unsupported type")
	}

	if classid != C.mxGetClassID(array) {
		return errors.New("data type mismatch")
	}

	count := C.mxGetM(array) * C.mxGetN(array)
	if scalar && count != 1 {
		return errors.New("data size mismatch")
	}

	parray := unsafe.Pointer(C.mxGetPr(array))
	if parray == nil {
		return errors.New("cannot read the variable")
	}

	read(parray, count)

	return nil
}

func (f *File) readStruct(array *C.mxArray, ivalue reflect.Value) error {
	if C.mxSTRUCT_CLASS != C.mxGetClassID(array) {
		return errors.New("data type mismatch")
	}

	if C.mxGetM(array) * C.mxGetN(array) != 1 {
		return errors.New("data size mismatch")
	}

	typo := ivalue.Type()
	count := typo.NumField()

	if count != int(C.mxGetNumberOfFields(array)) {
		return errors.New("data structure mismatch")
	}

	for i := 0; i < count; i++ {
		field := typo.Field(i)

		name := C.CString(field.Name)
		defer C.free(unsafe.Pointer(name))

		farray := C.mxGetField(array, 0, name)
		if farray == nil {
			return errors.New("data structure mismatch")
		}

		if err := f.readArray(farray, ivalue.Field(i)); err != nil {
			return err
		}
	}

	return nil
}

func readScalar(iv reflect.Value) (C.mxClassID, func(unsafe.Pointer, C.size_t)) {
	switch iv.Kind() {
	case reflect.Int8:
		return C.mxINT8_CLASS, func(p unsafe.Pointer, _ C.size_t) {
			iv.SetInt(int64(*(*int8)(p)))
		}
	case reflect.Uint8:
		return C.mxUINT8_CLASS, func(p unsafe.Pointer, _ C.size_t) {
			iv.SetUint(uint64(*(*uint8)(p)))
		}
	case reflect.Int16:
		return C.mxINT16_CLASS, func(p unsafe.Pointer, _ C.size_t) {
			iv.SetInt(int64(*(*int16)(p)))
		}
	case reflect.Uint16:
		return C.mxUINT16_CLASS, func(p unsafe.Pointer, _ C.size_t) {
			iv.SetUint(uint64(*(*uint16)(p)))
		}
	case reflect.Int32:
		return C.mxINT32_CLASS, func(p unsafe.Pointer, _ C.size_t) {
			iv.SetInt(int64(*(*int32)(p)))
		}
	case reflect.Uint32:
		return C.mxUINT32_CLASS, func(p unsafe.Pointer, _ C.size_t) {
			iv.SetUint(uint64(*(*uint32)(p)))
		}
	case reflect.Int64:
		return C.mxINT64_CLASS, func(p unsafe.Pointer, _ C.size_t) {
			iv.SetInt(int64(*(*int64)(p)))
		}
	case reflect.Uint64:
		return C.mxUINT64_CLASS, func(p unsafe.Pointer, _ C.size_t) {
			iv.SetUint(uint64(*(*uint64)(p)))
		}
	case reflect.Float32:
		return C.mxSINGLE_CLASS, func(p unsafe.Pointer, _ C.size_t) {
			iv.SetFloat(float64(*(*float32)(p)))
		}
	case reflect.Float64:
		return C.mxDOUBLE_CLASS, func(p unsafe.Pointer, _ C.size_t) {
			iv.SetFloat(float64(*(*float64)(p)))
		}
	default:
		return C.mxUNKNOWN_CLASS, nil
	}
}

func readSlice(iv reflect.Value) (C.mxClassID, func(unsafe.Pointer, C.size_t)) {
	read := func(w interface{}, p unsafe.Pointer, s C.size_t) {
		iw := reflect.Indirect(reflect.ValueOf(w))
		C.memcpy(unsafe.Pointer(iw.Pointer()), p, s)
		iv.Set(iw)
	}

	switch iv.Type().Elem().Kind() {
	case reflect.Int8:
		return C.mxINT8_CLASS, func(p unsafe.Pointer, c C.size_t) {
			w := make([]int8, c)
			read(&w, p, 1*c)
		}
	case reflect.Uint8:
		return C.mxUINT8_CLASS, func(p unsafe.Pointer, c C.size_t) {
			w := make([]uint8, c)
			read(&w, p, 1*c)
		}
	case reflect.Int16:
		return C.mxINT16_CLASS, func(p unsafe.Pointer, c C.size_t) {
			w := make([]int16, c)
			read(&w, p, 2*c)
		}
	case reflect.Uint16:
		return C.mxUINT16_CLASS, func(p unsafe.Pointer, c C.size_t) {
			w := make([]uint16, c)
			read(&w, p, 2*c)
		}
	case reflect.Int32:
		return C.mxINT32_CLASS, func(p unsafe.Pointer, c C.size_t) {
			w := make([]int32, c)
			read(&w, p, 4*c)
		}
	case reflect.Uint32:
		return C.mxUINT32_CLASS, func(p unsafe.Pointer, c C.size_t) {
			w := make([]uint32, c)
			read(&w, p, 4*c)
		}
	case reflect.Int64:
		return C.mxINT64_CLASS, func(p unsafe.Pointer, c C.size_t) {
			w := make([]int64, c)
			read(&w, p, 8*c)
		}
	case reflect.Uint64:
		return C.mxUINT64_CLASS, func(p unsafe.Pointer, c C.size_t) {
			w := make([]uint64, c)
			read(&w, p, 8*c)
		}
	case reflect.Float32:
		return C.mxSINGLE_CLASS, func(p unsafe.Pointer, c C.size_t) {
			w := make([]float32, c)
			read(&w, p, 4*c)
		}
	case reflect.Float64:
		return C.mxDOUBLE_CLASS, func(p unsafe.Pointer, c C.size_t) {
			w := make([]float64, c)
			read(&w, p, 8*c)
		}
	default:
		return C.mxUNKNOWN_CLASS, nil
	}
}
<|endoftext|>


<commit_before>package walnut

import (
	"strings"
)

const whitespace = " \t\n\v\f\r\u0085\u00A0"

// Defines a "key = value" assignment.
type def struct {
	key   string
	value string
	line  int
}

// Generates a map of resolved keys and raw string values from a byte slice.
// If the second return value != 0, an indentation error was detected on
// that line (1 being the first line).
func parseConfig(buf []byte) ([]def, int) {
	lines := strings.Split(string(buf), "\n")
	values := make([]def, 0)

	// collapse lines without any content
	for i, line := range lines {
		lines[i] = collapse(line)
	}

	parents := make([]string, 0)
	indents := make([]string, 0)
	first := true

	for n, line := range lines {
		if line == "" {
			continue
		}

		// line numbers should be 1-indexed
		n++

		k := selectKey(line)
		i := selectIndentation(line)
		d := calculateDepth(indents, i)

		// check for invalid indentation
		if d == -1 || (d == len(indents) && !first) {
			return nil, n
		}

		// trim now redundant levels
		if d < len(indents) {
			parents = parents[:d]
			indents = indents[:d]
		}

		// push the key and indentation onto their respective stacks
		parents = append(parents, k)
		indents = append(indents, i)

		// if the line contains an assignment, record the value
		if strings.ContainsRune(line, '=') {
			values = append(values, def{
				key:   strings.Join(parents, "."),
				value: selectValue(line),
				line:  n,
			})

			first = false
			continue
		}

		first = true
	}

	return values, 0
}

// Trims trailing whitespace or, in the case of comment lines, returns
// an empty string.
func collapse(input string) string {
	s := strings.TrimRight(input, whitespace)

	for _, r := range s {
		if strings.ContainsRune(whitespace, r) {
			continue
		}

		// comment detected, blank this line
		if r == '#' {
			break
		}

		// if the first non-whitespace character @todo
		return input
	}

	return ""
}

// Returns the "key" component from a "key = value" string.
func selectKey(input string) string {
	if eq := strings.IndexRune(input, '='); eq != -1 {
		input = input[:eq]
	}

	return strings.Trim(input, whitespace)
}

// Returns the "value" component from a "key = value" string.
func selectValue(input string) string {
	if eq := strings.IndexRune(input, '='); eq != -1 {
		input = input[eq+1:]
	}

	return strings.Trim(input, whitespace)
}

// Returns the string's whitespace prefix.
func selectIndentation(input string) string {
	end := strings.IndexFunc(input, func(r rune) bool {
		return strings.IndexRune(whitespace, r) == -1
	})

	if end == -1 {
		return ""
	}

	return input[:end]
}

// Given a list of previous indentation levels, finds the provided indentation
// level's depth value. A depth of 0 represents the lowest possible level of
// indentation. Returns -1 on errors caused by illegal indentation.
func calculateDepth(parents []string, current string) int {
	if current == "" {
		return 0
	}

	// the base indentation level must be an empty string
	if len(parents) == 0 {
		return -1
	}

	for i, prefix := range parents {
		switch {
		case current == prefix:
			return i
		case !strings.HasPrefix(current, prefix):
			return -1
		}
	}

	// if we get this far, the current line is further indented
	// than its parent
	return len(parents)
}
<commit_msg>Combine the 'select{Key,Value,Indentation}' functions<commit_after>package walnut

import (
	"strings"
)

const whitespace = " \t\n\v\f\r\u0085\u00A0"

// Defines a "key = value" assignment.
type def struct {
	key   string
	value string
	line  int
}

// Generates a map of resolved keys and raw string values from a byte slice.
// If the second return value != 0, an indentation error was detected on
// that line (1 being the first line).
func parseConfig(buf []byte) ([]def, int) {
	lines := strings.Split(string(buf), "\n")
	values := make([]def, 0)

	// collapse lines without any content
	for i, line := range lines {
		lines[i] = collapse(line)
	}

	parents := make([]string, 0)
	indents := make([]string, 0)
	first := true

	for n, line := range lines {
		if line == "" {
			continue
		}

		// line numbers should be 1-indexed
		n++

		i, k, v := split(line)
		d := calculateDepth(indents, i)

		// check for invalid indentation
		if d == -1 || (d == len(indents) && !first) {
			return nil, n
		}

		// trim now redundant levels
		if d < len(indents) {
			parents = parents[:d]
			indents = indents[:d]
		}

		// push the key and indentation onto their respective stacks
		parents = append(parents, k)
		indents = append(indents, i)

		// if the line contains an assignment, record the value
		if strings.ContainsRune(line, '=') {
			values = append(values, def{
				key:   strings.Join(parents, "."),
				value: v,
				line:  n,
			})

			first = false
			continue
		}

		first = true
	}

	return values, 0
}

// Trims trailing whitespace or, in the case of comment lines, returns
// an empty string.
func collapse(input string) string {
	s := strings.TrimRight(input, whitespace)

	for _, r := range s {
		if strings.ContainsRune(whitespace, r) {
			continue
		}

		// comment detected, blank this line
		if r == '#' {
			break
		}

		// if the first non-whitespace character @todo
		return input
	}

	return ""
}

// Returns the prefix whitespace and "key" and "value" components
// of a "key = value" line.
func split(line string) (i, k, v string) {
	for _, r := range line {
		if strings.IndexRune(whitespace, r) == -1 {
			break
		}
		i += string(r)
	}

	if eq := strings.IndexRune(line, '='); eq != -1 {
		k = strings.Trim(line[:eq], whitespace)
		v = strings.Trim(line[eq+1:], whitespace)
	} else {
		k = strings.Trim(line, whitespace)
	}

	return i, k, v
}

// Given a list of previous indentation levels, finds the provided indentation
// level's depth value. A depth of 0 represents the lowest possible level of
// indentation. Returns -1 on errors caused by illegal indentation.
func calculateDepth(parents []string, current string) int {
	if current == "" {
		return 0
	}

	// the base indentation level must be an empty string
	if len(parents) == 0 {
		return -1
	}

	for i, prefix := range parents {
		switch {
		case current == prefix:
			return i
		case !strings.HasPrefix(current, prefix):
			return -1
		}
	}

	// if we get this far, the current line is further indented
	// than its parent
	return len(parents)
}
<|endoftext|>


<commit_before>package gopcap

import (
	"bytes"
	"errors"
	"io"
	"time"
)

// checkMagicNum checks the first four bytes of a pcap file, searching for the magic number
// and checking the byte order. Returns three values: whether the file is a pcap file, whether
// the byte order needs flipping, and any error that was encountered. If error is returned,
// the other values are invalid.
func checkMagicNum(src io.Reader) (bool, bool, error) {
	// These magic numbers form the header of a pcap file.
	magic := []byte{0xa1, 0xb2, 0xc3, 0xd4}
	magic_reverse := []byte{0xd4, 0xc3, 0xb2, 0xa1}

	buffer := make([]byte, 4)
	read_count, err := src.Read(buffer)

	if err != nil {
		return false, false, err
	}
	if read_count != 4 {
		return false, false, errors.New("Insufficent length.")
	}

	if bytes.Compare(buffer, magic) == 0 {
		return true, false, nil
	} else if bytes.Compare(buffer, magic_reverse) == 0 {
		return true, true, nil
	}

	return false, false, errors.New("Not a pcap file.")
}

// parsePacket parses a full packet out of the pcap file. It returns an error if any problems were
// encountered.
func parsePacket(pkt *Packet, src io.Reader, flipped bool) error {
	err := populatePacketHeader(pkt, src, flipped)

	if err != nil {
		return err
	}

	data := make([]byte, pkt.IncludedLen)
	readlen, err := src.Read(data)
	pkt.Data = data

	if uint32(readlen) != pkt.IncludedLen {
		err = errors.New("Unexpected EOF")
	}

	return err
}

// populateFileHeader reads the next 20 bytes out of the .pcap file and uses it to populate the
// PcapFile structure.
func populateFileHeader(file *PcapFile, src io.Reader, flipped bool) error {
	buffer := make([]byte, 20)
	read_count, err := src.Read(buffer)

	if err != nil {
		return err
	} else if read_count != 20 {
		return errors.New("Insufficient length.")
	}

	// First two bytes are the major version number.
	file.MajorVersion = getUint16(buffer[0:2], flipped)

	// Next two are the minor version number.
	file.MinorVersion = getUint16(buffer[2:4], flipped)

	// GMT to local correction, in seconds east of UTC.
	file.TZCorrection = getInt32(buffer[4:8], flipped)

	// Next is the number of significant figures in the timestamps. Almost always zero.
	file.SigFigs = getUint32(buffer[8:12], flipped)

	// Now the maximum length of the captured packet data.
	file.MaxLen = getUint32(buffer[12:16], flipped)

	// And the link type.
	file.LinkType = Link(getUint32(buffer[16:20], flipped))

	return nil
}

// populatePacketHeader reads the next 16 bytes out of the file and builds it into a
// packet header.
func populatePacketHeader(packet *Packet, src io.Reader, flipped bool) error {
	buffer := make([]byte, 16)
	read_count, err := src.Read(buffer)

	if read_count != 16 {
		return errors.New("Insufficient length.")
	}

	// First is a pair of fields that build up the timestamp.
	ts_seconds := getUint32(buffer[0:4], flipped)
	ts_millis := getUint32(buffer[4:8], flipped)
	packet.Timestamp = (time.Duration(ts_seconds) * time.Second) + (time.Duration(ts_millis) * time.Millisecond)

	// Next is the length of the data segment.
	packet.IncludedLen = getUint32(buffer[8:12], flipped)

	// Then the original length of the packet.
	packet.ActualLen = getUint32(buffer[12:16], flipped)

	return err
}
<commit_msg>Return new error.<commit_after>package gopcap

import (
	"bytes"
	"errors"
	"io"
	"time"
)

// checkMagicNum checks the first four bytes of a pcap file, searching for the magic number
// and checking the byte order. Returns three values: whether the file is a pcap file, whether
// the byte order needs flipping, and any error that was encountered. If error is returned,
// the other values are invalid.
func checkMagicNum(src io.Reader) (bool, bool, error) {
	// These magic numbers form the header of a pcap file.
	magic := []byte{0xa1, 0xb2, 0xc3, 0xd4}
	magic_reverse := []byte{0xd4, 0xc3, 0xb2, 0xa1}

	buffer := make([]byte, 4)
	read_count, err := src.Read(buffer)

	if err != nil {
		return false, false, err
	}
	if read_count != 4 {
		return false, false, errors.New("Insufficent length.")
	}

	if bytes.Compare(buffer, magic) == 0 {
		return true, false, nil
	} else if bytes.Compare(buffer, magic_reverse) == 0 {
		return true, true, nil
	}

	return false, false, NotAPcapFile
}

// parsePacket parses a full packet out of the pcap file. It returns an error if any problems were
// encountered.
func parsePacket(pkt *Packet, src io.Reader, flipped bool) error {
	err := populatePacketHeader(pkt, src, flipped)

	if err != nil {
		return err
	}

	data := make([]byte, pkt.IncludedLen)
	readlen, err := src.Read(data)
	pkt.Data = data

	if uint32(readlen) != pkt.IncludedLen {
		err = errors.New("Unexpected EOF")
	}

	return err
}

// populateFileHeader reads the next 20 bytes out of the .pcap file and uses it to populate the
// PcapFile structure.
func populateFileHeader(file *PcapFile, src io.Reader, flipped bool) error {
	buffer := make([]byte, 20)
	read_count, err := src.Read(buffer)

	if err != nil {
		return err
	} else if read_count != 20 {
		return errors.New("Insufficient length.")
	}

	// First two bytes are the major version number.
	file.MajorVersion = getUint16(buffer[0:2], flipped)

	// Next two are the minor version number.
	file.MinorVersion = getUint16(buffer[2:4], flipped)

	// GMT to local correction, in seconds east of UTC.
	file.TZCorrection = getInt32(buffer[4:8], flipped)

	// Next is the number of significant figures in the timestamps. Almost always zero.
	file.SigFigs = getUint32(buffer[8:12], flipped)

	// Now the maximum length of the captured packet data.
	file.MaxLen = getUint32(buffer[12:16], flipped)

	// And the link type.
	file.LinkType = Link(getUint32(buffer[16:20], flipped))

	return nil
}

// populatePacketHeader reads the next 16 bytes out of the file and builds it into a
// packet header.
func populatePacketHeader(packet *Packet, src io.Reader, flipped bool) error {
	buffer := make([]byte, 16)
	read_count, err := src.Read(buffer)

	if read_count != 16 {
		return errors.New("Insufficient length.")
	}

	// First is a pair of fields that build up the timestamp.
	ts_seconds := getUint32(buffer[0:4], flipped)
	ts_millis := getUint32(buffer[4:8], flipped)
	packet.Timestamp = (time.Duration(ts_seconds) * time.Second) + (time.Duration(ts_millis) * time.Millisecond)

	// Next is the length of the data segment.
	packet.IncludedLen = getUint32(buffer[8:12], flipped)

	// Then the original length of the packet.
	packet.ActualLen = getUint32(buffer[12:16], flipped)

	return err
}
<|endoftext|>


<commit_before>package io

import (
	"bytes"
	"io"
	"os"
	"strings"
)

func SimulateStdin(input string, block func(r io.Reader)) {
	reader, writer := io.Pipe()

	go func() {
		writer.Write([]byte(input))
		defer writer.Close()
	}()

	block(reader)
}

func CaptureOutput(block func()) []string {
	oldSTDOUT := os.Stdout
	r, w, err := os.Pipe()
	if err != nil {
		panic(err)
	}

	os.Stdout = w
	defer func() {
		os.Stdout = oldSTDOUT
	}()

	doneWriting := make(chan bool)
	result := make(chan []string)

	go captureOutputAsyncronously(doneWriting, result, r)

	block()
	w.Close()
	doneWriting <- true
	return <-result
}

/*
 The reason we're doing is that you can't write an infinite amount of bytes into a pipe.
 On some platforms, the limit is fairly high; on other platforms, the limit is infuriatingly small
 (looking at you, Windows). To counteract this, we need to read in a goroutine from one end of
 the pipe and return the result across a channel.
*/
func captureOutputAsyncronously(doneWriting <-chan bool, result chan<- []string, reader io.Reader) {
	var readingString string

	for {
		var buf bytes.Buffer
		io.Copy(&buf, reader)
		readingString += buf.String()

		_, ok := <-doneWriting
		if ok {
			// there is no guarantee that the writer did not
			// write more in between the read above and reading from this channel
			// so we absolute must read once more if we want all the bytes
			var buf bytes.Buffer
			io.Copy(&buf, reader)
			readingString += buf.String()
			break
		}
	}

	result <- strings.Split(readingString, "\n")
}
<commit_msg>Fix capturing terminal output on windows<commit_after>package io

import (
	"bytes"
	"io"
	"os"
	"strings"
	"runtime"
	"github.com/mattn/go-colorable"
	"github.com/fatih/color"
)

func SimulateStdin(input string, block func(r io.Reader)) {
	reader, writer := io.Pipe()

	go func() {
		writer.Write([]byte(input))
		defer writer.Close()
	}()

	block(reader)
}

func CaptureOutput(block func()) []string {
	oldSTDOUT := os.Stdout
	r, w, err := os.Pipe()
	if err != nil {
		panic(err)
	}

	os.Stdout = w
	defer func() {
		os.Stdout = oldSTDOUT
	}()

	//////
	// We use fmt.Fprintf() to write to the "github.com/fatih/color".Output file
	// to get colors on Windows machines.
	// That variable gets initialized with a reference to os.Stdout when that library is imported.
	// That means that when we muck with os.Stdout above, it doesn't get reflected in
	// the printing code for windows.
	// Instead, we can just redeclare that color.Output variable with a colorable version of our
	// redirect pipe.
	if runtime.GOOS == "windows" {
		color.Output = colorable.NewColorable(w)
	}
	//////

	doneWriting := make(chan bool)
	result := make(chan []string)

	go captureOutputAsyncronously(doneWriting, result, r)

	block()
	w.Close()
	doneWriting <- true
	return <-result
}

/*
 The reason we're doing is that you can't write an infinite amount of bytes into a pipe.
 On some platforms, the limit is fairly high; on other platforms, the limit is infuriatingly small
 (looking at you, Windows). To counteract this, we need to read in a goroutine from one end of
 the pipe and return the result across a channel.
*/
func captureOutputAsyncronously(doneWriting <-chan bool, result chan<- []string, reader io.Reader) {
	var readingString string

	for {
		var buf bytes.Buffer
		io.Copy(&buf, reader)
		readingString += buf.String()

		_, ok := <-doneWriting
		if ok {
			// there is no guarantee that the writer did not
			// write more in between the read above and reading from this channel
			// so we absolute must read once more if we want all the bytes
			var buf bytes.Buffer
			io.Copy(&buf, reader)
			readingString += buf.String()
			break
		}
	}

	result <- strings.Split(readingString, "\n")
}
<|endoftext|>


<commit_before>package storage

import (
	"fmt"
	"github.com/viant/toolbox"
	"io"
	"net/url"
	"path"
)

//Service represents abstract way to accessing local or remote storage
type Service interface {
	//List returns a list of object for supplied url
	List(URL string) ([]Object, error)

	//Exists returns true if resource exists
	Exists(URL string) (bool, error)

	//Object returns a Object for supplied url
	StorageObject(URL string) (Object, error)

	//Download returns reader for downloaded storage object
	Download(object Object) (io.Reader, error)

	//Upload uploads provided reader content for supplied storage object.
	Upload(URL string, reader io.Reader) error

	//Delete removes passed in storage object
	Delete(object Object) error

	//Register register schema with provided service
	Register(schema string, service Service) error
}

type storageService struct {
	registry map[string]Service
}

func (s *storageService) getServiceForSchema(URL string) (Service, error) {
	parsedUrl, err := url.Parse(URL)
	if err != nil {
		return nil, err
	}

	if result, found := s.registry[parsedUrl.Scheme]; found {
		return result, nil
	}

	return nil, fmt.Errorf("Failed to lookup url schema %v in %v", parsedUrl.Scheme, URL)
}

//List lists all object for passed in URL
func (s *storageService) List(URL string) ([]Object, error) {
	service, err := s.getServiceForSchema(URL)
	if err != nil {
		return nil, err
	}
	return service.List(URL)
}

//Exists returns true if resource exists
func (s *storageService) Exists(URL string) (bool, error) {
	service, err := s.getServiceForSchema(URL)
	if err != nil {
		return false, err
	}
	return service.Exists(URL)
}

//StorageObject returns storage object for provided URL
func (s *storageService) StorageObject(URL string) (Object, error) {
	service, err := s.getServiceForSchema(URL)
	if err != nil {
		return nil, err
	}
	return service.StorageObject(URL)
}

//Download downloads content for passed in object
func (s *storageService) Download(object Object) (io.Reader, error) {
	service, err := s.getServiceForSchema(object.URL())
	if err != nil {
		return nil, err
	}
	return service.Download(object)
}

//Uploads content for passed in URL
func (s *storageService) Upload(URL string, reader io.Reader) error {
	service, err := s.getServiceForSchema(URL)
	if err != nil {
		return err
	}
	return service.Upload(URL, reader)
}

//Delete remove storage object
func (s *storageService) Delete(object Object) error {
	service, err := s.getServiceForSchema(object.URL())
	if err != nil {
		return err
	}
	return service.Delete(object)
}

//Register register storage schema
func (s *storageService) Register(schema string, service Service) error {
	s.registry[schema] = service
	return nil
}

//NewService creates a new storage service
func NewService() Service {
	var result = &storageService{
		registry: make(map[string]Service),
	}
	result.Register("file", &fileStorageService{})
	return result
}

//NewServiceForURL creates a new storage service for provided URL scheme and optional credential file
func NewServiceForURL(URL, credentialFile string) (Service, error) {
	parsedURL, err := url.Parse(URL)
	if err != nil {
		return nil, err
	}
	service := NewService()
	provider := NewStorageProvider().Get(parsedURL.Scheme)

	if provider != nil {
		serviceForScheme, err := provider(credentialFile)
		if err != nil {
			return nil, fmt.Errorf("Failed to get storage for url %v: %v", URL, err)
		}
		err = service.Register(parsedURL.Scheme, serviceForScheme)
		if err != nil {
			return nil, err
		}
	} else if parsedURL.Scheme != "file" {
		return nil, fmt.Errorf("Unsupported scheme %v", URL)
	}
	return service, nil
}

func copy(sourceService Service, sourceURL string, targetService Service, targetURL string, modifyContentHandler func(reader io.Reader) (io.Reader, error), subPath string) error {
	sourceListURL := sourceURL
	if subPath != "" {
		sourceListURL = toolbox.URLPathJoin(sourceURL, subPath)
	}
	objects, err := sourceService.List(sourceListURL)
	var objectRelativePath string
	for _, object := range objects {

		if object.URL() == sourceURL && object.IsFolder() {
			continue
		}

		if len(object.URL()) > len(sourceURL) {
			objectRelativePath = object.URL()[len(sourceURL):]
		}
		var targetObjectURL = targetURL
		if objectRelativePath != "" {
			targetObjectURL = toolbox.URLPathJoin(targetURL, objectRelativePath)
		}
		var reader io.Reader
		if object.IsContent() {
			reader, err = sourceService.Download(object)
			if err != nil {
				err = fmt.Errorf("Unable download, %v -> %v, %v", object.URL(), targetObjectURL, err)
				return err
			}

			if modifyContentHandler != nil {
				reader, err = modifyContentHandler(reader)
				if err != nil {
					err = fmt.Errorf("Unable modify content, %v %v %v", object.URL(), targetObjectURL, err)
					return err
				}
			}

			targetObjects, err := targetService.List(targetObjectURL)
			if err == nil && len(targetObjects) > 0 {
				if targetObjects[0].IsFolder() {
					_, file := path.Split(object.URL())
					targetObjectURL = toolbox.URLPathJoin(targetObjectURL, file)
				}
			}

			err = targetService.Upload(targetObjectURL, reader)
			if err != nil {
				err = fmt.Errorf("Unable upload, %v %v %v", object.URL(), targetObjectURL, err)
				return err
			}
		} else {
			err = copy(sourceService, sourceURL, targetService, targetURL, modifyContentHandler, objectRelativePath)
			if err != nil {
				return err
			}
		}
	}
	return nil
}

//Copy downloads objects from source URL to upload them to target URL.
func Copy(sourceService Service, sourceURL string, targetService Service, targetURL string, modifyContentHandler func(reader io.Reader) (io.Reader, error)) (err error) {
	err = copy(sourceService, sourceURL, targetService, targetURL, modifyContentHandler, "")
	if err != nil {
		err = fmt.Errorf("Failed to copy %v -> %v: %v", sourceURL, targetURL, err)
	}
	return err
}
<commit_msg>patched copy<commit_after>package storage

import (
	"fmt"
	"github.com/viant/toolbox"
	"io"
	"net/url"
	"path"
)

//Service represents abstract way to accessing local or remote storage
type Service interface {
	//List returns a list of object for supplied url
	List(URL string) ([]Object, error)

	//Exists returns true if resource exists
	Exists(URL string) (bool, error)

	//Object returns a Object for supplied url
	StorageObject(URL string) (Object, error)

	//Download returns reader for downloaded storage object
	Download(object Object) (io.Reader, error)

	//Upload uploads provided reader content for supplied storage object.
	Upload(URL string, reader io.Reader) error

	//Delete removes passed in storage object
	Delete(object Object) error

	//Register register schema with provided service
	Register(schema string, service Service) error
}

type storageService struct {
	registry map[string]Service
}

func (s *storageService) getServiceForSchema(URL string) (Service, error) {
	parsedUrl, err := url.Parse(URL)
	if err != nil {
		return nil, err
	}

	if result, found := s.registry[parsedUrl.Scheme]; found {
		return result, nil
	}

	return nil, fmt.Errorf("Failed to lookup url schema %v in %v", parsedUrl.Scheme, URL)
}

//List lists all object for passed in URL
func (s *storageService) List(URL string) ([]Object, error) {
	service, err := s.getServiceForSchema(URL)
	if err != nil {
		return nil, err
	}
	return service.List(URL)
}

//Exists returns true if resource exists
func (s *storageService) Exists(URL string) (bool, error) {
	service, err := s.getServiceForSchema(URL)
	if err != nil {
		return false, err
	}
	return service.Exists(URL)
}

//StorageObject returns storage object for provided URL
func (s *storageService) StorageObject(URL string) (Object, error) {
	service, err := s.getServiceForSchema(URL)
	if err != nil {
		return nil, err
	}
	return service.StorageObject(URL)
}

//Download downloads content for passed in object
func (s *storageService) Download(object Object) (io.Reader, error) {
	service, err := s.getServiceForSchema(object.URL())
	if err != nil {
		return nil, err
	}
	return service.Download(object)
}

//Uploads content for passed in URL
func (s *storageService) Upload(URL string, reader io.Reader) error {
	service, err := s.getServiceForSchema(URL)
	if err != nil {
		return err
	}
	return service.Upload(URL, reader)
}

//Delete remove storage object
func (s *storageService) Delete(object Object) error {
	service, err := s.getServiceForSchema(object.URL())
	if err != nil {
		return err
	}
	return service.Delete(object)
}

//Register register storage schema
func (s *storageService) Register(schema string, service Service) error {
	s.registry[schema] = service
	return nil
}

//NewService creates a new storage service
func NewService() Service {
	var result = &storageService{
		registry: make(map[string]Service),
	}
	result.Register("file", &fileStorageService{})
	return result
}

//NewServiceForURL creates a new storage service for provided URL scheme and optional credential file
func NewServiceForURL(URL, credentialFile string) (Service, error) {
	parsedURL, err := url.Parse(URL)
	if err != nil {
		return nil, err
	}
	service := NewService()
	provider := NewStorageProvider().Get(parsedURL.Scheme)

	if provider != nil {
		serviceForScheme, err := provider(credentialFile)
		if err != nil {
			return nil, fmt.Errorf("Failed to get storage for url %v: %v", URL, err)
		}
		err = service.Register(parsedURL.Scheme, serviceForScheme)
		if err != nil {
			return nil, err
		}
	} else if parsedURL.Scheme != "file" {
		return nil, fmt.Errorf("Unsupported scheme %v", URL)
	}
	return service, nil
}

func copy(sourceService Service, sourceURL string, targetService Service, targetURL string, modifyContentHandler func(reader io.Reader) (io.Reader, error), subPath string) error {
	sourceListURL := sourceURL
	if subPath != "" {
		sourceListURL = toolbox.URLPathJoin(sourceURL, subPath)
	}
	objects, err := sourceService.List(sourceListURL)
	var objectRelativePath string
	for _, object := range objects {

		if object.URL() == sourceURL && object.IsFolder() {
			continue
		}

		if len(object.URL()) > len(sourceURL) {
			objectRelativePath = object.URL()[len(sourceURL):]
		}
		var targetObjectURL = targetURL
		if objectRelativePath != "" {
			targetObjectURL = toolbox.URLPathJoin(targetURL, objectRelativePath)
		}
		var reader io.Reader
		if object.IsContent() {
			reader, err = sourceService.Download(object)
			if err != nil {
				err = fmt.Errorf("Unable download, %v -> %v, %v", object.URL(), targetObjectURL, err)
				return err
			}

			if modifyContentHandler != nil {
				reader, err = modifyContentHandler(reader)
				if err != nil {
					err = fmt.Errorf("Unable modify content, %v %v %v", object.URL(), targetObjectURL, err)
					return err
				}
			}

			targetObject, err := targetService.StorageObject(targetObjectURL)

			fmt.Printf("%v %v %v\n", targetObjectURL, targetObject,  err)

			if (targetObject != nil && targetObject.IsFolder()) {
				_, file := path.Split(object.URL())
				targetObjectURL = toolbox.URLPathJoin(targetObjectURL, file)
			}

			err = targetService.Upload(targetObjectURL, reader)
			if err != nil {
				err = fmt.Errorf("Unable upload, %v %v %v", object.URL(), targetObjectURL, err)
				return err
			}
		} else {
			err = copy(sourceService, sourceURL, targetService, targetURL, modifyContentHandler, objectRelativePath)
			if err != nil {
				return err
			}
		}
	}
	return nil
}

//Copy downloads objects from source URL to upload them to target URL.
func Copy(sourceService Service, sourceURL string, targetService Service, targetURL string, modifyContentHandler func(reader io.Reader) (io.Reader, error)) (err error) {
	err = copy(sourceService, sourceURL, targetService, targetURL, modifyContentHandler, "")
	if err != nil {
		err = fmt.Errorf("Failed to copy %v -> %v: %v", sourceURL, targetURL, err)
	}
	return err
}
<|endoftext|>


<commit_before>package store_test

import (
	"fmt"
	"testing"
)

func TestGetContributedHistory(t *testing.T) {
	shim := setup(t)
	defer teardown(t, shim.SQLDB())

	ths, err := shim.GetContributedTagHistory("kusubooru")
	if err != nil {
		t.Error("err is:", err)
	}
	if ths == nil {
		t.Error("ths is nil")
	}
	if len(ths) == 0 {
		t.Error("ths are empty")
	}
	fmt.Println(ths)
}

// Setup the test environment.
//func setup() (*DB, error) {
//	//err := withTestDB()
//	//if err != nil {
//	//	return nil, err
//	//}
//
//	// testOptions is a global in this case, but you could easily
//	// create one per-test
//	db, err := openDB(*dbDriver, *dbConfig)
//	if err != nil {
//		return nil, err
//	}
//
//	// Loads our test schema
//	db.MustLoad()
//	return db, nil
//}

// Create our test database.
//func withTestDB() error {
//	db, err := Open()
//	if err != nil {
//		return err
//	}
//	defer db.Close()
//
//	_, err = db.Exec(fmt.Sprintf("CREATE DATABASE %s;", testOptions.Name))
//	if err != nil {
//		return err
//	}
//
//	return nil
//}
<commit_msg>Delete tags_test.go for now<commit_after><|endoftext|>


<commit_before>package main

import (
	"fmt"
)

// Environment is the execution environment of a command. It contains every information
// about the database and migrations to use.
type Environment struct {
	Driver    string `json:"driver"`
	Protocol  string `json:"protocol"`
	Host      string `json:"host"`
	Port      uint64 `json:"port"`
	User      string `json:"user"`
	Password  string `json:"password"`
	Database  string `json:"database"`
	Directory string `json:"directory"`
	Table     string `json:"table"`
}

// DSN return the connection string for the current environment
func (e Environment) DSN() string {
	switch e.Driver {
	case "mysql":
		return fmt.Sprintf("%s:%s@%s(%s:%d)/%s", e.User, e.Password, e.Protocol, e.Host, e.Port, e.Database)
	case "postgresql":
		return fmt.Sprintf("user=%s password=%s host=%s port=%d dbname=%s sslmode=disable", e.User, e.Password, e.Host, e.Port, e.Database)
	case "sqlite":
		return e.Database
	default:
		return ""
	}
}
<commit_msg>fix postgres connnection line<commit_after>package main

import (
	"fmt"
)

// Environment is the execution environment of a command. It contains every information
// about the database and migrations to use.
type Environment struct {
	Driver    string `json:"driver"`
	Protocol  string `json:"protocol"`
	Host      string `json:"host"`
	Port      uint64 `json:"port"`
	User      string `json:"user"`
	Password  string `json:"password"`
	Database  string `json:"database"`
	Directory string `json:"directory"`
	Table     string `json:"table"`
}

// DSN return the connection string for the current environment
func (e Environment) DSN() string {
	switch e.Driver {
	case "mysql":
		return fmt.Sprintf("%s:%s@%s(%s:%d)/%s", e.User, e.Password, e.Protocol, e.Host, e.Port, e.Database)
	case "postgresql":
		return fmt.Sprintf("host=%s port=%d user=%s password=%s dbname=%s sslmode=disable", e.Host, e.Port, e.User, e.Password, e.Database)
	case "sqlite":
		return e.Database
	default:
		return ""
	}
}
<|endoftext|>


<commit_before>package core

import (
	"bytes"
	"fmt"
	"io"
)

type (
	Seq interface {
		Seqable
		Object
		First() Object
		Rest() Seq
		IsEmpty() bool
		Cons(obj Object) Seq
	}
	Seqable interface {
		Seq() Seq
	}
	SeqIterator struct {
		seq Seq
	}
	ConsSeq struct {
		InfoHolder
		MetaHolder
		first Object
		rest  Seq
	}
	ArraySeq struct {
		InfoHolder
		MetaHolder
		arr   []Object
		index int
	}
	LazySeq struct {
		InfoHolder
		MetaHolder
		fn  Callable
		seq Seq
	}
	MappingSeq struct {
		InfoHolder
		MetaHolder
		seq Seq
		fn  func(obj Object) Object
	}
)

func SeqsEqual(seq1, seq2 Seq) bool {
	iter2 := iter(seq2)
	for iter1 := iter(seq1); iter1.HasNext(); {
		if !iter2.HasNext() || !iter2.Next().Equals(iter1.Next()) {
			return false
		}
	}
	return !iter2.HasNext()
}

func IsSeqEqual(seq Seq, other interface{}) bool {
	if seq == other {
		return true
	}
	switch other := other.(type) {
	case Sequential:
		switch other := other.(type) {
		case Seqable:
			return SeqsEqual(seq, other.Seq())
		}
	}
	return false
}

func (seq *MappingSeq) Seq() Seq {
	return seq
}

func (seq *MappingSeq) Equals(other interface{}) bool {
	return IsSeqEqual(seq, other)
}

func (seq *MappingSeq) ToString(escape bool) string {
	return SeqToString(seq, escape)
}

func (seq *MappingSeq) Pprint(w io.Writer, indent int) int {
	return pprintSeq(seq, w, indent)
}

func (seq *MappingSeq) WithMeta(meta Map) Object {
	res := *seq
	res.meta = SafeMerge(res.meta, meta)
	return &res
}

func (seq *MappingSeq) GetType() *Type {
	return TYPE.MappingSeq
}

func (seq *MappingSeq) Hash() uint32 {
	return hashOrdered(seq)
}

func (seq *MappingSeq) First() Object {
	return seq.fn(seq.seq.First())
}

func (seq *MappingSeq) Rest() Seq {
	return &MappingSeq{
		seq: seq.seq.Rest(),
		fn:  seq.fn,
	}
}

func (seq *MappingSeq) IsEmpty() bool {
	return seq.seq.IsEmpty()
}

func (seq *MappingSeq) Cons(obj Object) Seq {
	return &ConsSeq{first: obj, rest: seq}
}

func (seq *MappingSeq) sequential() {}

func (seq *LazySeq) Seq() Seq {
	return seq
}

func (seq *LazySeq) realize() {
	if seq.seq == nil {
		seq.seq = AssertSeqable(seq.fn.Call([]Object{}), "").Seq()
	}
}

func (seq *LazySeq) IsRealized() bool {
	return seq.seq != nil
}

func (seq *LazySeq) Equals(other interface{}) bool {
	return IsSeqEqual(seq, other)
}

func (seq *LazySeq) ToString(escape bool) string {
	return SeqToString(seq, escape)
}

func (seq *LazySeq) Pprint(w io.Writer, indent int) int {
	return pprintSeq(seq, w, indent)
}

func (seq *LazySeq) WithMeta(meta Map) Object {
	res := *seq
	res.meta = SafeMerge(res.meta, meta)
	return &res
}

func (seq *LazySeq) GetType() *Type {
	return TYPE.LazySeq
}

func (seq *LazySeq) Hash() uint32 {
	return hashOrdered(seq)
}

func (seq *LazySeq) First() Object {
	seq.realize()
	return seq.seq.First()
}

func (seq *LazySeq) Rest() Seq {
	seq.realize()
	return seq.seq.Rest()
}

func (seq *LazySeq) IsEmpty() bool {
	seq.realize()
	return seq.seq.IsEmpty()
}

func (seq *LazySeq) Cons(obj Object) Seq {
	return &ConsSeq{first: obj, rest: seq}
}

func (seq *LazySeq) sequential() {}

func (seq *ArraySeq) Seq() Seq {
	return seq
}

func (seq *ArraySeq) Equals(other interface{}) bool {
	return IsSeqEqual(seq, other)
}

func (seq *ArraySeq) ToString(escape bool) string {
	return SeqToString(seq, escape)
}

func (seq *ArraySeq) Pprint(w io.Writer, indent int) int {
	return pprintSeq(seq, w, indent)
}

func (seq *ArraySeq) WithMeta(meta Map) Object {
	res := *seq
	res.meta = SafeMerge(res.meta, meta)
	return &res
}

func (seq *ArraySeq) GetType() *Type {
	return TYPE.ArraySeq
}

func (seq *ArraySeq) Hash() uint32 {
	return hashOrdered(seq)
}

func (seq *ArraySeq) First() Object {
	if seq.IsEmpty() {
		return NIL
	}
	return seq.arr[seq.index]
}

func (seq *ArraySeq) Rest() Seq {
	if seq.index+1 < len(seq.arr) {
		return &ArraySeq{index: seq.index + 1, arr: seq.arr}
	}
	return EmptyList
}

func (seq *ArraySeq) IsEmpty() bool {
	return seq.index >= len(seq.arr)
}

func (seq *ArraySeq) Cons(obj Object) Seq {
	return &ConsSeq{first: obj, rest: seq}
}

func (seq *ArraySeq) sequential() {}

func SeqToString(seq Seq, escape bool) string {
	var b bytes.Buffer
	b.WriteRune('(')
	for iter := iter(seq); iter.HasNext(); {
		b.WriteString(iter.Next().ToString(escape))
		if iter.HasNext() {
			b.WriteRune(' ')
		}
	}
	b.WriteRune(')')
	return b.String()
}

func (seq *ConsSeq) WithMeta(meta Map) Object {
	res := *seq
	res.meta = SafeMerge(res.meta, meta)
	return &res
}

func (seq *ConsSeq) Seq() Seq {
	return seq
}

func (seq *ConsSeq) Equals(other interface{}) bool {
	return IsSeqEqual(seq, other)
}

func (seq *ConsSeq) ToString(escape bool) string {
	return SeqToString(seq, escape)
}

func (seq *ConsSeq) Pprint(w io.Writer, indent int) int {
	return pprintSeq(seq, w, indent)
}

func (seq *ConsSeq) GetType() *Type {
	return TYPE.ConsSeq
}

func (seq *ConsSeq) Hash() uint32 {
	return hashOrdered(seq)
}

func (seq *ConsSeq) First() Object {
	return seq.first
}

func (seq *ConsSeq) Rest() Seq {
	return seq.rest
}

func (seq *ConsSeq) IsEmpty() bool {
	return false
}

func (seq *ConsSeq) Cons(obj Object) Seq {
	return &ConsSeq{first: obj, rest: seq}
}

func (seq *ConsSeq) sequential() {}

func iter(seq Seq) *SeqIterator {
	return &SeqIterator{seq: seq}
}

func (iter *SeqIterator) Next() Object {
	res := iter.seq.First()
	iter.seq = iter.seq.Rest()
	return res
}

func (iter *SeqIterator) HasNext() bool {
	return !iter.seq.IsEmpty()
}

func Second(seq Seq) Object {
	return seq.Rest().First()
}

func Third(seq Seq) Object {
	return seq.Rest().Rest().First()
}

func Fourth(seq Seq) Object {
	return seq.Rest().Rest().Rest().First()
}

func ToSlice(seq Seq) []Object {
	res := make([]Object, 0)
	for !seq.IsEmpty() {
		res = append(res, seq.First())
		seq = seq.Rest()
	}
	return res
}

func SeqCount(seq Seq) int {
	c := 0
	for !seq.IsEmpty() {
		switch obj := seq.(type) {
		case Counted:
			return c + obj.Count()
		}
		c++
		seq = seq.Rest()
	}
	return c
}

func SeqNth(seq Seq, n int) Object {
	if n < 0 {
		panic(RT.NewError(fmt.Sprintf("Negative index: %d", n)))
	}
	i := n
	for !seq.IsEmpty() {
		if i == 0 {
			return seq.First()
		}
		seq = seq.Rest()
		i--
	}
	panic(RT.NewError(fmt.Sprintf("Index %d exceeds seq's length %d", n, (n - i))))
}

func SeqTryNth(seq Seq, n int, d Object) Object {
	if n < 0 {
		return d
	}
	i := n
	for !seq.IsEmpty() {
		if i == 0 {
			return seq.First()
		}
		seq = seq.Rest()
		i--
	}
	return d
}

func hashUnordered(seq Seq, seed uint32) uint32 {
	for !seq.IsEmpty() {
		seed += seq.First().Hash()
		seq = seq.Rest()
	}
	h := getHash()
	h.Write(uint32ToBytes(seed))
	return h.Sum32()
}

func hashOrdered(seq Seq) uint32 {
	h := getHash()
	for !seq.IsEmpty() {
		h.Write(uint32ToBytes(seq.First().Hash()))
		seq = seq.Rest()
	}
	return h.Sum32()
}

func pprintSeq(seq Seq, w io.Writer, indent int) int {
	i := indent + 1
	fmt.Fprint(w, "(")
	for iter := iter(seq); iter.HasNext(); {
		i = pprintObject(iter.Next(), indent, w)
		if iter.HasNext() {
			fmt.Fprint(w, "\n")
			writeIndent(w, indent+1)
		}
	}
	fmt.Fprint(w, ")")
	return i + 1
}
<commit_msg>Fix indentation when pretty-printing seqs.<commit_after>package core

import (
	"bytes"
	"fmt"
	"io"
)

type (
	Seq interface {
		Seqable
		Object
		First() Object
		Rest() Seq
		IsEmpty() bool
		Cons(obj Object) Seq
	}
	Seqable interface {
		Seq() Seq
	}
	SeqIterator struct {
		seq Seq
	}
	ConsSeq struct {
		InfoHolder
		MetaHolder
		first Object
		rest  Seq
	}
	ArraySeq struct {
		InfoHolder
		MetaHolder
		arr   []Object
		index int
	}
	LazySeq struct {
		InfoHolder
		MetaHolder
		fn  Callable
		seq Seq
	}
	MappingSeq struct {
		InfoHolder
		MetaHolder
		seq Seq
		fn  func(obj Object) Object
	}
)

func SeqsEqual(seq1, seq2 Seq) bool {
	iter2 := iter(seq2)
	for iter1 := iter(seq1); iter1.HasNext(); {
		if !iter2.HasNext() || !iter2.Next().Equals(iter1.Next()) {
			return false
		}
	}
	return !iter2.HasNext()
}

func IsSeqEqual(seq Seq, other interface{}) bool {
	if seq == other {
		return true
	}
	switch other := other.(type) {
	case Sequential:
		switch other := other.(type) {
		case Seqable:
			return SeqsEqual(seq, other.Seq())
		}
	}
	return false
}

func (seq *MappingSeq) Seq() Seq {
	return seq
}

func (seq *MappingSeq) Equals(other interface{}) bool {
	return IsSeqEqual(seq, other)
}

func (seq *MappingSeq) ToString(escape bool) string {
	return SeqToString(seq, escape)
}

func (seq *MappingSeq) Pprint(w io.Writer, indent int) int {
	return pprintSeq(seq, w, indent)
}

func (seq *MappingSeq) WithMeta(meta Map) Object {
	res := *seq
	res.meta = SafeMerge(res.meta, meta)
	return &res
}

func (seq *MappingSeq) GetType() *Type {
	return TYPE.MappingSeq
}

func (seq *MappingSeq) Hash() uint32 {
	return hashOrdered(seq)
}

func (seq *MappingSeq) First() Object {
	return seq.fn(seq.seq.First())
}

func (seq *MappingSeq) Rest() Seq {
	return &MappingSeq{
		seq: seq.seq.Rest(),
		fn:  seq.fn,
	}
}

func (seq *MappingSeq) IsEmpty() bool {
	return seq.seq.IsEmpty()
}

func (seq *MappingSeq) Cons(obj Object) Seq {
	return &ConsSeq{first: obj, rest: seq}
}

func (seq *MappingSeq) sequential() {}

func (seq *LazySeq) Seq() Seq {
	return seq
}

func (seq *LazySeq) realize() {
	if seq.seq == nil {
		seq.seq = AssertSeqable(seq.fn.Call([]Object{}), "").Seq()
	}
}

func (seq *LazySeq) IsRealized() bool {
	return seq.seq != nil
}

func (seq *LazySeq) Equals(other interface{}) bool {
	return IsSeqEqual(seq, other)
}

func (seq *LazySeq) ToString(escape bool) string {
	return SeqToString(seq, escape)
}

func (seq *LazySeq) Pprint(w io.Writer, indent int) int {
	return pprintSeq(seq, w, indent)
}

func (seq *LazySeq) WithMeta(meta Map) Object {
	res := *seq
	res.meta = SafeMerge(res.meta, meta)
	return &res
}

func (seq *LazySeq) GetType() *Type {
	return TYPE.LazySeq
}

func (seq *LazySeq) Hash() uint32 {
	return hashOrdered(seq)
}

func (seq *LazySeq) First() Object {
	seq.realize()
	return seq.seq.First()
}

func (seq *LazySeq) Rest() Seq {
	seq.realize()
	return seq.seq.Rest()
}

func (seq *LazySeq) IsEmpty() bool {
	seq.realize()
	return seq.seq.IsEmpty()
}

func (seq *LazySeq) Cons(obj Object) Seq {
	return &ConsSeq{first: obj, rest: seq}
}

func (seq *LazySeq) sequential() {}

func (seq *ArraySeq) Seq() Seq {
	return seq
}

func (seq *ArraySeq) Equals(other interface{}) bool {
	return IsSeqEqual(seq, other)
}

func (seq *ArraySeq) ToString(escape bool) string {
	return SeqToString(seq, escape)
}

func (seq *ArraySeq) Pprint(w io.Writer, indent int) int {
	return pprintSeq(seq, w, indent)
}

func (seq *ArraySeq) WithMeta(meta Map) Object {
	res := *seq
	res.meta = SafeMerge(res.meta, meta)
	return &res
}

func (seq *ArraySeq) GetType() *Type {
	return TYPE.ArraySeq
}

func (seq *ArraySeq) Hash() uint32 {
	return hashOrdered(seq)
}

func (seq *ArraySeq) First() Object {
	if seq.IsEmpty() {
		return NIL
	}
	return seq.arr[seq.index]
}

func (seq *ArraySeq) Rest() Seq {
	if seq.index+1 < len(seq.arr) {
		return &ArraySeq{index: seq.index + 1, arr: seq.arr}
	}
	return EmptyList
}

func (seq *ArraySeq) IsEmpty() bool {
	return seq.index >= len(seq.arr)
}

func (seq *ArraySeq) Cons(obj Object) Seq {
	return &ConsSeq{first: obj, rest: seq}
}

func (seq *ArraySeq) sequential() {}

func SeqToString(seq Seq, escape bool) string {
	var b bytes.Buffer
	b.WriteRune('(')
	for iter := iter(seq); iter.HasNext(); {
		b.WriteString(iter.Next().ToString(escape))
		if iter.HasNext() {
			b.WriteRune(' ')
		}
	}
	b.WriteRune(')')
	return b.String()
}

func (seq *ConsSeq) WithMeta(meta Map) Object {
	res := *seq
	res.meta = SafeMerge(res.meta, meta)
	return &res
}

func (seq *ConsSeq) Seq() Seq {
	return seq
}

func (seq *ConsSeq) Equals(other interface{}) bool {
	return IsSeqEqual(seq, other)
}

func (seq *ConsSeq) ToString(escape bool) string {
	return SeqToString(seq, escape)
}

func (seq *ConsSeq) Pprint(w io.Writer, indent int) int {
	return pprintSeq(seq, w, indent)
}

func (seq *ConsSeq) GetType() *Type {
	return TYPE.ConsSeq
}

func (seq *ConsSeq) Hash() uint32 {
	return hashOrdered(seq)
}

func (seq *ConsSeq) First() Object {
	return seq.first
}

func (seq *ConsSeq) Rest() Seq {
	return seq.rest
}

func (seq *ConsSeq) IsEmpty() bool {
	return false
}

func (seq *ConsSeq) Cons(obj Object) Seq {
	return &ConsSeq{first: obj, rest: seq}
}

func (seq *ConsSeq) sequential() {}

func iter(seq Seq) *SeqIterator {
	return &SeqIterator{seq: seq}
}

func (iter *SeqIterator) Next() Object {
	res := iter.seq.First()
	iter.seq = iter.seq.Rest()
	return res
}

func (iter *SeqIterator) HasNext() bool {
	return !iter.seq.IsEmpty()
}

func Second(seq Seq) Object {
	return seq.Rest().First()
}

func Third(seq Seq) Object {
	return seq.Rest().Rest().First()
}

func Fourth(seq Seq) Object {
	return seq.Rest().Rest().Rest().First()
}

func ToSlice(seq Seq) []Object {
	res := make([]Object, 0)
	for !seq.IsEmpty() {
		res = append(res, seq.First())
		seq = seq.Rest()
	}
	return res
}

func SeqCount(seq Seq) int {
	c := 0
	for !seq.IsEmpty() {
		switch obj := seq.(type) {
		case Counted:
			return c + obj.Count()
		}
		c++
		seq = seq.Rest()
	}
	return c
}

func SeqNth(seq Seq, n int) Object {
	if n < 0 {
		panic(RT.NewError(fmt.Sprintf("Negative index: %d", n)))
	}
	i := n
	for !seq.IsEmpty() {
		if i == 0 {
			return seq.First()
		}
		seq = seq.Rest()
		i--
	}
	panic(RT.NewError(fmt.Sprintf("Index %d exceeds seq's length %d", n, (n - i))))
}

func SeqTryNth(seq Seq, n int, d Object) Object {
	if n < 0 {
		return d
	}
	i := n
	for !seq.IsEmpty() {
		if i == 0 {
			return seq.First()
		}
		seq = seq.Rest()
		i--
	}
	return d
}

func hashUnordered(seq Seq, seed uint32) uint32 {
	for !seq.IsEmpty() {
		seed += seq.First().Hash()
		seq = seq.Rest()
	}
	h := getHash()
	h.Write(uint32ToBytes(seed))
	return h.Sum32()
}

func hashOrdered(seq Seq) uint32 {
	h := getHash()
	for !seq.IsEmpty() {
		h.Write(uint32ToBytes(seq.First().Hash()))
		seq = seq.Rest()
	}
	return h.Sum32()
}

func pprintSeq(seq Seq, w io.Writer, indent int) int {
	i := indent + 1
	fmt.Fprint(w, "(")
	for iter := iter(seq); iter.HasNext(); {
		i = pprintObject(iter.Next(), indent+1, w)
		if iter.HasNext() {
			fmt.Fprint(w, "\n")
			writeIndent(w, indent+1)
		}
	}
	fmt.Fprint(w, ")")
	return i + 1
}
<|endoftext|>


<commit_before>/*
Copyright 2015 The Kubernetes Authors All rights reserved.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package e2e

import (
	"fmt"
	"io"
	"io/ioutil"
	"net"
	"os/exec"
	"regexp"
	"strconv"
	"strings"
	"syscall"
	"time"

	"k8s.io/kubernetes/pkg/api"
	"k8s.io/kubernetes/pkg/util/wait"
	"k8s.io/kubernetes/pkg/version"
	"k8s.io/kubernetes/test/e2e/framework"

	. "github.com/onsi/ginkgo"
)

const (
	podName = "pfpod"
)

// TODO support other ports besides 80
var (
	portForwardRegexp        = regexp.MustCompile("Forwarding from 127.0.0.1:([0-9]+) -> 80")
	portForwardPortToStdOutV = version.MustParse("v1.3.0-alpha.4")
)

func pfPod(expectedClientData, chunks, chunkSize, chunkIntervalMillis string) *api.Pod {
	return &api.Pod{
		ObjectMeta: api.ObjectMeta{
			Name:   podName,
			Labels: map[string]string{"name": podName},
		},
		Spec: api.PodSpec{
			Containers: []api.Container{
				{
					Name:  "portforwardtester",
					Image: "gcr.io/google_containers/portforwardtester:1.0",
					Env: []api.EnvVar{
						{
							Name:  "BIND_PORT",
							Value: "80",
						},
						{
							Name:  "EXPECTED_CLIENT_DATA",
							Value: expectedClientData,
						},
						{
							Name:  "CHUNKS",
							Value: chunks,
						},
						{
							Name:  "CHUNK_SIZE",
							Value: chunkSize,
						},
						{
							Name:  "CHUNK_INTERVAL",
							Value: chunkIntervalMillis,
						},
					},
				},
			},
			RestartPolicy: api.RestartPolicyNever,
		},
	}
}

type portForwardCommand struct {
	cmd  *exec.Cmd
	port int
}

// Stop attempts to gracefully stop `kubectl port-forward`, only killing it if necessary.
// This helps avoid spdy goroutine leaks in the Kubelet.
func (c *portForwardCommand) Stop() {
	// SIGINT signals that kubectl port-forward should gracefully terminate
	if err := c.cmd.Process.Signal(syscall.SIGINT); err != nil {
		framework.Logf("error sending SIGINT to kubectl port-forward: %v", err)
	}

	// try to wait for a clean exit
	done := make(chan error)
	go func() {
		done <- c.cmd.Wait()
	}()

	expired := time.NewTimer(wait.ForeverTestTimeout)
	defer expired.Stop()

	select {
	case err := <-done:
		if err == nil {
			// success
			return
		}
		framework.Logf("error waiting for kubectl port-forward to exit: %v", err)
	case <-expired.C:
		framework.Logf("timed out waiting for kubectl port-forward to exit")
	}

	framework.Logf("trying to forcibly kill kubectl port-forward")
	framework.TryKill(c.cmd)
}

func runPortForward(ns, podName string, port int) *portForwardCommand {
	cmd := framework.KubectlCmd("port-forward", fmt.Sprintf("--namespace=%v", ns), podName, fmt.Sprintf(":%d", port))
	// This is somewhat ugly but is the only way to retrieve the port that was picked
	// by the port-forward command. We don't want to hard code the port as we have no
	// way of guaranteeing we can pick one that isn't in use, particularly on Jenkins.
	framework.Logf("starting port-forward command and streaming output")
	stdout, stderr, err := framework.StartCmdAndStreamOutput(cmd)
	if err != nil {
		framework.Failf("Failed to start port-forward command: %v", err)
	}

	buf := make([]byte, 128)

	// After v1.3.0-alpha.4 (#17030), kubectl port-forward outputs port
	// info to stdout, not stderr, so for version-skewed tests, look there
	// instead.
	var portOutput io.ReadCloser
	if useStdOut, err := framework.KubectlVersionGTE(portForwardPortToStdOutV); err != nil {
		framework.Failf("Failed to get kubectl version: %v", err)
	} else if useStdOut {
		portOutput = stdout
	} else {
		portOutput = stderr
	}

	var n int
	framework.Logf("reading from `kubectl port-forward` command's stdout")
	if n, err = portOutput.Read(buf); err != nil {
		framework.Failf("Failed to read from kubectl port-forward stdout: %v", err)
	}
	portForwardOutput := string(buf[:n])
	match := portForwardRegexp.FindStringSubmatch(portForwardOutput)
	if len(match) != 2 {
		framework.Failf("Failed to parse kubectl port-forward output: %s", portForwardOutput)
	}

	listenPort, err := strconv.Atoi(match[1])
	if err != nil {
		framework.Failf("Error converting %s to an int: %v", match[1], err)
	}

	return &portForwardCommand{
		cmd:  cmd,
		port: listenPort,
	}
}

var _ = framework.KubeDescribe("Port forwarding", func() {
	f := framework.NewDefaultFramework("port-forwarding")

	framework.KubeDescribe("With a server that expects a client request", func() {
		It("should support a client that connects, sends no data, and disconnects [Conformance]", func() {
			By("creating the target pod")
			pod := pfPod("abc", "1", "1", "1")
			if _, err := f.Client.Pods(f.Namespace.Name).Create(pod); err != nil {
				framework.Failf("Couldn't create pod: %v", err)
			}
			if err := f.WaitForPodRunning(pod.Name); err != nil {
				framework.Failf("Pod did not start running: %v", err)
			}

			By("Running 'kubectl port-forward'")
			cmd := runPortForward(f.Namespace.Name, pod.Name, 80)
			defer cmd.Stop()

			By("Dialing the local port")
			conn, err := net.Dial("tcp", fmt.Sprintf("127.0.0.1:%d", cmd.port))
			if err != nil {
				framework.Failf("Couldn't connect to port %d: %v", cmd.port, err)
			}

			By("Closing the connection to the local port")
			conn.Close()

			By("Waiting for the target pod to stop running")
			waitErr := f.WaitForPodNoLongerRunning(pod.Name)

			By("Retrieving logs from the target pod")
			logOutput, err := framework.GetPodLogs(f.Client, f.Namespace.Name, pod.Name, "portforwardtester")
			if err != nil {
				framework.Failf("Error retrieving logs: %v", err)
			}

			if waitErr != nil {
				framework.Logf("Pod log:\n%s", logOutput)
				framework.Failf("Pod did not stop running: %v", waitErr)
			}

			By("Verifying logs")
			verifyLogMessage(logOutput, "Accepted client connection")
			verifyLogMessage(logOutput, "Expected to read 3 bytes from client, but got 0 instead")
		})

		It("should support a client that connects, sends data, and disconnects [Conformance]", func() {
			By("creating the target pod")
			pod := pfPod("abc", "10", "10", "100")
			if _, err := f.Client.Pods(f.Namespace.Name).Create(pod); err != nil {
				framework.Failf("Couldn't create pod: %v", err)
			}
			if err := f.WaitForPodRunning(pod.Name); err != nil {
				framework.Failf("Pod did not start running: %v", err)
			}

			By("Running 'kubectl port-forward'")
			cmd := runPortForward(f.Namespace.Name, pod.Name, 80)
			defer cmd.Stop()

			By("Dialing the local port")
			addr, err := net.ResolveTCPAddr("tcp", fmt.Sprintf("127.0.0.1:%d", cmd.port))
			if err != nil {
				framework.Failf("Error resolving tcp addr: %v", err)
			}
			conn, err := net.DialTCP("tcp", nil, addr)
			if err != nil {
				framework.Failf("Couldn't connect to port %d: %v", cmd.port, err)
			}
			defer func() {
				By("Closing the connection to the local port")
				conn.Close()
			}()

			By("Sending the expected data to the local port")
			fmt.Fprint(conn, "abc")

			By("Closing the write half of the client's connection")
			conn.CloseWrite()

			By("Reading data from the local port")
			fromServer, err := ioutil.ReadAll(conn)
			if err != nil {
				framework.Failf("Unexpected error reading data from the server: %v", err)
			}

			if e, a := strings.Repeat("x", 100), string(fromServer); e != a {
				framework.Failf("Expected %q from server, got %q", e, a)
			}

			By("Waiting for the target pod to stop running")
			waitErr := f.WaitForPodNoLongerRunning(pod.Name)

			By("Retrieving logs from the target pod")
			logOutput, err := framework.GetPodLogs(f.Client, f.Namespace.Name, pod.Name, "portforwardtester")
			if err != nil {
				framework.Failf("Error retrieving logs: %v", err)
			}

			if waitErr != nil {
				framework.Logf("Pod log:\n%s", logOutput)
				framework.Failf("Pod did not stop running: %v", waitErr)
			}

			By("Verifying logs")
			verifyLogMessage(logOutput, "^Accepted client connection$")
			verifyLogMessage(logOutput, "^Received expected client data$")
			verifyLogMessage(logOutput, "^Done$")
		})
	})
	framework.KubeDescribe("With a server that expects no client request", func() {
		It("should support a client that connects, sends no data, and disconnects [Conformance]", func() {
			By("creating the target pod")
			pod := pfPod("", "10", "10", "100")
			if _, err := f.Client.Pods(f.Namespace.Name).Create(pod); err != nil {
				framework.Failf("Couldn't create pod: %v", err)
			}
			if err := f.WaitForPodRunning(pod.Name); err != nil {
				framework.Failf("Pod did not start running: %v", err)
			}

			By("Running 'kubectl port-forward'")
			cmd := runPortForward(f.Namespace.Name, pod.Name, 80)
			defer cmd.Stop()

			By("Dialing the local port")
			conn, err := net.Dial("tcp", fmt.Sprintf("127.0.0.1:%d", cmd.port))
			if err != nil {
				framework.Failf("Couldn't connect to port %d: %v", cmd.port, err)
			}
			defer func() {
				By("Closing the connection to the local port")
				conn.Close()
			}()

			By("Reading data from the local port")
			fromServer, err := ioutil.ReadAll(conn)
			if err != nil {
				framework.Failf("Unexpected error reading data from the server: %v", err)
			}

			if e, a := strings.Repeat("x", 100), string(fromServer); e != a {
				framework.Failf("Expected %q from server, got %q", e, a)
			}

			By("Waiting for the target pod to stop running")
			waitErr := f.WaitForPodNoLongerRunning(pod.Name)

			By("Retrieving logs from the target pod")
			logOutput, err := framework.GetPodLogs(f.Client, f.Namespace.Name, pod.Name, "portforwardtester")
			if err != nil {
				framework.Failf("Error retrieving logs: %v", err)
			}

			if waitErr != nil {
				framework.Logf("Pod log:\n%s", logOutput)
				framework.Failf("Pod did not stop running: %v", waitErr)
			}

			By("Verifying logs")
			verifyLogMessage(logOutput, "Accepted client connection")
			verifyLogMessage(logOutput, "Done")
		})
	})
})

func verifyLogMessage(log, expected string) {
	re := regexp.MustCompile(expected)
	lines := strings.Split(log, "\n")
	for i := range lines {
		if re.MatchString(lines[i]) {
			return
		}
	}
	framework.Failf("Missing %q from log: %s", expected, log)
}
<commit_msg>portforward e2e: extend log for flake hunting<commit_after>/*
Copyright 2015 The Kubernetes Authors All rights reserved.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package e2e

import (
	"fmt"
	"io"
	"io/ioutil"
	"net"
	"os/exec"
	"regexp"
	"strconv"
	"strings"
	"syscall"
	"time"

	"k8s.io/kubernetes/pkg/api"
	"k8s.io/kubernetes/pkg/util/wait"
	"k8s.io/kubernetes/pkg/version"
	"k8s.io/kubernetes/test/e2e/framework"

	. "github.com/onsi/ginkgo"
)

const (
	podName = "pfpod"
)

// TODO support other ports besides 80
var (
	portForwardRegexp        = regexp.MustCompile("Forwarding from 127.0.0.1:([0-9]+) -> 80")
	portForwardPortToStdOutV = version.MustParse("v1.3.0-alpha.4")
)

func pfPod(expectedClientData, chunks, chunkSize, chunkIntervalMillis string) *api.Pod {
	return &api.Pod{
		ObjectMeta: api.ObjectMeta{
			Name:   podName,
			Labels: map[string]string{"name": podName},
		},
		Spec: api.PodSpec{
			Containers: []api.Container{
				{
					Name:  "portforwardtester",
					Image: "gcr.io/google_containers/portforwardtester:1.0",
					Env: []api.EnvVar{
						{
							Name:  "BIND_PORT",
							Value: "80",
						},
						{
							Name:  "EXPECTED_CLIENT_DATA",
							Value: expectedClientData,
						},
						{
							Name:  "CHUNKS",
							Value: chunks,
						},
						{
							Name:  "CHUNK_SIZE",
							Value: chunkSize,
						},
						{
							Name:  "CHUNK_INTERVAL",
							Value: chunkIntervalMillis,
						},
					},
				},
			},
			RestartPolicy: api.RestartPolicyNever,
		},
	}
}

type portForwardCommand struct {
	cmd  *exec.Cmd
	port int
}

// Stop attempts to gracefully stop `kubectl port-forward`, only killing it if necessary.
// This helps avoid spdy goroutine leaks in the Kubelet.
func (c *portForwardCommand) Stop() {
	// SIGINT signals that kubectl port-forward should gracefully terminate
	if err := c.cmd.Process.Signal(syscall.SIGINT); err != nil {
		framework.Logf("error sending SIGINT to kubectl port-forward: %v", err)
	}

	// try to wait for a clean exit
	done := make(chan error)
	go func() {
		done <- c.cmd.Wait()
	}()

	expired := time.NewTimer(wait.ForeverTestTimeout)
	defer expired.Stop()

	select {
	case err := <-done:
		if err == nil {
			// success
			return
		}
		framework.Logf("error waiting for kubectl port-forward to exit: %v", err)
	case <-expired.C:
		framework.Logf("timed out waiting for kubectl port-forward to exit")
	}

	framework.Logf("trying to forcibly kill kubectl port-forward")
	framework.TryKill(c.cmd)
}

func runPortForward(ns, podName string, port int) *portForwardCommand {
	cmd := framework.KubectlCmd("port-forward", fmt.Sprintf("--namespace=%v", ns), podName, fmt.Sprintf(":%d", port))
	// This is somewhat ugly but is the only way to retrieve the port that was picked
	// by the port-forward command. We don't want to hard code the port as we have no
	// way of guaranteeing we can pick one that isn't in use, particularly on Jenkins.
	framework.Logf("starting port-forward command and streaming output")
	stdout, stderr, err := framework.StartCmdAndStreamOutput(cmd)
	if err != nil {
		framework.Failf("Failed to start port-forward command: %v", err)
	}

	buf := make([]byte, 128)

	// After v1.3.0-alpha.4 (#17030), kubectl port-forward outputs port
	// info to stdout, not stderr, so for version-skewed tests, look there
	// instead.
	var portOutput io.ReadCloser
	if useStdOut, err := framework.KubectlVersionGTE(portForwardPortToStdOutV); err != nil {
		framework.Failf("Failed to get kubectl version: %v", err)
	} else if useStdOut {
		portOutput = stdout
	} else {
		portOutput = stderr
	}

	var n int
	framework.Logf("reading from `kubectl port-forward` command's stdout")
	if n, err = portOutput.Read(buf); err != nil {
		framework.Failf("Failed to read from kubectl port-forward stdout: %v", err)
	}
	portForwardOutput := string(buf[:n])
	match := portForwardRegexp.FindStringSubmatch(portForwardOutput)
	if len(match) != 2 {
		framework.Failf("Failed to parse kubectl port-forward output: %s", portForwardOutput)
	}

	listenPort, err := strconv.Atoi(match[1])
	if err != nil {
		framework.Failf("Error converting %s to an int: %v", match[1], err)
	}

	return &portForwardCommand{
		cmd:  cmd,
		port: listenPort,
	}
}

var _ = framework.KubeDescribe("Port forwarding", func() {
	f := framework.NewDefaultFramework("port-forwarding")

	framework.KubeDescribe("With a server that expects a client request", func() {
		It("should support a client that connects, sends no data, and disconnects [Conformance]", func() {
			By("creating the target pod")
			pod := pfPod("abc", "1", "1", "1")
			if _, err := f.Client.Pods(f.Namespace.Name).Create(pod); err != nil {
				framework.Failf("Couldn't create pod: %v", err)
			}
			if err := f.WaitForPodRunning(pod.Name); err != nil {
				framework.Failf("Pod did not start running: %v", err)
			}
			defer func() {
				logs, err := framework.GetPodLogs(f.Client, f.Namespace.Name, pod.Name, "portforwardtester")
				if err != nil {
					framework.Logf("Error getting pod log: %v", err)
				} else {
					framework.Logf("Pod log:\n%s", logs)
				}
			}()

			By("Running 'kubectl port-forward'")
			cmd := runPortForward(f.Namespace.Name, pod.Name, 80)
			defer cmd.Stop()

			By("Dialing the local port")
			conn, err := net.Dial("tcp", fmt.Sprintf("127.0.0.1:%d", cmd.port))
			if err != nil {
				framework.Failf("Couldn't connect to port %d: %v", cmd.port, err)
			}

			By("Closing the connection to the local port")
			conn.Close()

			By("Waiting for the target pod to stop running")
			if err := f.WaitForPodNoLongerRunning(pod.Name); err != nil {
				framework.Failf("Pod did not stop running: %v", err)
			}

			By("Verifying logs")
			logOutput, err := framework.GetPodLogs(f.Client, f.Namespace.Name, pod.Name, "portforwardtester")
			if err != nil {
				framework.Failf("Error retrieving pod logs: %v", err)
			}
			verifyLogMessage(logOutput, "Accepted client connection")
			verifyLogMessage(logOutput, "Expected to read 3 bytes from client, but got 0 instead")
		})

		It("should support a client that connects, sends data, and disconnects [Conformance]", func() {
			By("creating the target pod")
			pod := pfPod("abc", "10", "10", "100")
			if _, err := f.Client.Pods(f.Namespace.Name).Create(pod); err != nil {
				framework.Failf("Couldn't create pod: %v", err)
			}
			if err := f.WaitForPodRunning(pod.Name); err != nil {
				framework.Failf("Pod did not start running: %v", err)
			}
			defer func() {
				logs, err := framework.GetPodLogs(f.Client, f.Namespace.Name, pod.Name, "portforwardtester")
				if err != nil {
					framework.Logf("Error getting pod log: %v", err)
				} else {
					framework.Logf("Pod log:\n%s", logs)
				}
			}()

			By("Running 'kubectl port-forward'")
			cmd := runPortForward(f.Namespace.Name, pod.Name, 80)
			defer cmd.Stop()

			By("Dialing the local port")
			addr, err := net.ResolveTCPAddr("tcp", fmt.Sprintf("127.0.0.1:%d", cmd.port))
			if err != nil {
				framework.Failf("Error resolving tcp addr: %v", err)
			}
			conn, err := net.DialTCP("tcp", nil, addr)
			if err != nil {
				framework.Failf("Couldn't connect to port %d: %v", cmd.port, err)
			}
			defer func() {
				By("Closing the connection to the local port")
				conn.Close()
			}()

			By("Sending the expected data to the local port")
			fmt.Fprint(conn, "abc")

			By("Closing the write half of the client's connection")
			conn.CloseWrite()

			By("Reading data from the local port")
			fromServer, err := ioutil.ReadAll(conn)
			if err != nil {
				framework.Failf("Unexpected error reading data from the server: %v", err)
			}

			if e, a := strings.Repeat("x", 100), string(fromServer); e != a {
				framework.Failf("Expected %q from server, got %q", e, a)
			}

			By("Waiting for the target pod to stop running")
			if err := f.WaitForPodNoLongerRunning(pod.Name); err != nil {
				framework.Failf("Pod did not stop running: %v", err)
			}

			By("Verifying logs")
			logOutput, err := framework.GetPodLogs(f.Client, f.Namespace.Name, pod.Name, "portforwardtester")
			if err != nil {
				framework.Failf("Error retrieving pod logs: %v", err)
			}
			verifyLogMessage(logOutput, "^Accepted client connection$")
			verifyLogMessage(logOutput, "^Received expected client data$")
			verifyLogMessage(logOutput, "^Done$")
		})
	})
	framework.KubeDescribe("With a server that expects no client request", func() {
		It("should support a client that connects, sends no data, and disconnects [Conformance]", func() {
			By("creating the target pod")
			pod := pfPod("", "10", "10", "100")
			if _, err := f.Client.Pods(f.Namespace.Name).Create(pod); err != nil {
				framework.Failf("Couldn't create pod: %v", err)
			}
			if err := f.WaitForPodRunning(pod.Name); err != nil {
				framework.Failf("Pod did not start running: %v", err)
			}
			defer func() {
				logs, err := framework.GetPodLogs(f.Client, f.Namespace.Name, pod.Name, "portforwardtester")
				if err != nil {
					framework.Logf("Error getting pod log: %v", err)
				} else {
					framework.Logf("Pod log:\n%s", logs)
				}
			}()

			By("Running 'kubectl port-forward'")
			cmd := runPortForward(f.Namespace.Name, pod.Name, 80)
			defer cmd.Stop()

			By("Dialing the local port")
			conn, err := net.Dial("tcp", fmt.Sprintf("127.0.0.1:%d", cmd.port))
			if err != nil {
				framework.Failf("Couldn't connect to port %d: %v", cmd.port, err)
			}
			defer func() {
				By("Closing the connection to the local port")
				conn.Close()
			}()

			By("Reading data from the local port")
			fromServer, err := ioutil.ReadAll(conn)
			if err != nil {
				framework.Failf("Unexpected error reading data from the server: %v", err)
			}

			if e, a := strings.Repeat("x", 100), string(fromServer); e != a {
				framework.Failf("Expected %q from server, got %q", e, a)
			}

			By("Waiting for the target pod to stop running")
			if err := f.WaitForPodNoLongerRunning(pod.Name); err != nil {
				framework.Failf("Pod did not stop running: %v", err)
			}

			By("Verifying logs")
			logOutput, err := framework.GetPodLogs(f.Client, f.Namespace.Name, pod.Name, "portforwardtester")
			if err != nil {
				framework.Failf("Error retrieving pod logs: %v", err)
			}
			verifyLogMessage(logOutput, "Accepted client connection")
			verifyLogMessage(logOutput, "Done")
		})
	})
})

func verifyLogMessage(log, expected string) {
	re := regexp.MustCompile(expected)
	lines := strings.Split(log, "\n")
	for i := range lines {
		if re.MatchString(lines[i]) {
			return
		}
	}
	framework.Failf("Missing %q from log: %s", expected, log)
}
<|endoftext|>


<commit_before>/*
Copyright 2016 The Kubernetes Authors.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package e2e

import (
	"fmt"
	"time"

	"k8s.io/kubernetes/pkg/api"
	"k8s.io/kubernetes/pkg/labels"
	"k8s.io/kubernetes/test/e2e/framework"

	. "github.com/onsi/ginkgo"
	. "github.com/onsi/gomega"
)

// This test requires Rescheduler to be enabled.
var _ = framework.KubeDescribe("Rescheduler [Serial]", func() {
	f := framework.NewDefaultFramework("rescheduler")
	var ns string
	var totalMillicores int

	BeforeEach(func() {
		framework.SkipUnlessProviderIs("gce")
		ns = f.Namespace.Name
		nodes := framework.GetReadySchedulableNodesOrDie(f.Client)
		nodeCount := len(nodes.Items)
		Expect(nodeCount).NotTo(BeZero())

		cpu := nodes.Items[0].Status.Capacity[api.ResourceCPU]
		totalMillicores = int((&cpu).MilliValue()) * nodeCount
	})

	It("should ensure that critical pod is scheduled in case there is no resources available", func() {
		By("reserving all available cpu")
		err := reserveAllCpu(f, "reserve-all-cpu", totalMillicores)
		defer framework.DeleteRCAndPods(f.Client, ns, "reserve-all-cpu")
		framework.ExpectNoError(err)

		By("creating a new instance of DNS and waiting for DNS to be scheduled")
		label := labels.SelectorFromSet(labels.Set(map[string]string{"k8s-app": "kube-dns"}))
		listOpts := api.ListOptions{LabelSelector: label}
		rcs, err := f.Client.ReplicationControllers(api.NamespaceSystem).List(listOpts)
		framework.ExpectNoError(err)
		Expect(len(rcs.Items)).Should(Equal(1))

		rc := rcs.Items[0]
		replicas := uint(rc.Spec.Replicas)

		err = framework.ScaleRC(f.Client, api.NamespaceSystem, rc.Name, replicas+1, true)
		defer framework.ExpectNoError(framework.ScaleRC(f.Client, api.NamespaceSystem, rc.Name, replicas, true))
		framework.ExpectNoError(err)
	})
})

func reserveAllCpu(f *framework.Framework, id string, millicores int) error {
	timeout := 5 * time.Minute
	replicas := millicores / 100

	ReserveCpu(f, id, 1, 100)
	framework.ExpectNoError(framework.ScaleRC(f.Client, f.Namespace.Name, id, uint(replicas), false))

	for start := time.Now(); time.Since(start) < timeout; time.Sleep(10 * time.Second) {
		pods, err := framework.GetPodsInNamespace(f.Client, f.Namespace.Name, framework.ImagePullerLabels)
		if err != nil {
			return err
		}

		if len(pods) != replicas {
			continue
		}

		allRunningOrUnschedulable := true
		for _, pod := range pods {
			if !podRunningOrUnschedulable(pod) {
				allRunningOrUnschedulable = false
				break
			}
		}
		if allRunningOrUnschedulable {
			return nil
		}
	}
	return fmt.Errorf("Pod name %s: Gave up waiting %v for %d pods to come up", id, timeout, replicas)
}

func podRunningOrUnschedulable(pod *api.Pod) bool {
	_, cond := api.GetPodCondition(&pod.Status, api.PodScheduled)
	if cond != nil && cond.Status == api.ConditionFalse && cond.Reason == "Unschedulable" {
		return true
	}
	running, _ := framework.PodRunningReady(pod)
	return running
}
<commit_msg>Enabled Rescheduler e2e for gke<commit_after>/*
Copyright 2016 The Kubernetes Authors.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package e2e

import (
	"fmt"
	"time"

	"k8s.io/kubernetes/pkg/api"
	"k8s.io/kubernetes/pkg/labels"
	"k8s.io/kubernetes/test/e2e/framework"

	. "github.com/onsi/ginkgo"
	. "github.com/onsi/gomega"
)

// This test requires Rescheduler to be enabled.
var _ = framework.KubeDescribe("Rescheduler [Serial]", func() {
	f := framework.NewDefaultFramework("rescheduler")
	var ns string
	var totalMillicores int

	BeforeEach(func() {
		framework.SkipUnlessProviderIs("gce", "gke")
		ns = f.Namespace.Name
		nodes := framework.GetReadySchedulableNodesOrDie(f.Client)
		nodeCount := len(nodes.Items)
		Expect(nodeCount).NotTo(BeZero())

		cpu := nodes.Items[0].Status.Capacity[api.ResourceCPU]
		totalMillicores = int((&cpu).MilliValue()) * nodeCount
	})

	It("should ensure that critical pod is scheduled in case there is no resources available", func() {
		By("reserving all available cpu")
		err := reserveAllCpu(f, "reserve-all-cpu", totalMillicores)
		defer framework.DeleteRCAndPods(f.Client, ns, "reserve-all-cpu")
		framework.ExpectNoError(err)

		By("creating a new instance of DNS and waiting for DNS to be scheduled")
		label := labels.SelectorFromSet(labels.Set(map[string]string{"k8s-app": "kube-dns"}))
		listOpts := api.ListOptions{LabelSelector: label}
		rcs, err := f.Client.ReplicationControllers(api.NamespaceSystem).List(listOpts)
		framework.ExpectNoError(err)
		Expect(len(rcs.Items)).Should(Equal(1))

		rc := rcs.Items[0]
		replicas := uint(rc.Spec.Replicas)

		err = framework.ScaleRC(f.Client, api.NamespaceSystem, rc.Name, replicas+1, true)
		defer framework.ExpectNoError(framework.ScaleRC(f.Client, api.NamespaceSystem, rc.Name, replicas, true))
		framework.ExpectNoError(err)
	})
})

func reserveAllCpu(f *framework.Framework, id string, millicores int) error {
	timeout := 5 * time.Minute
	replicas := millicores / 100

	ReserveCpu(f, id, 1, 100)
	framework.ExpectNoError(framework.ScaleRC(f.Client, f.Namespace.Name, id, uint(replicas), false))

	for start := time.Now(); time.Since(start) < timeout; time.Sleep(10 * time.Second) {
		pods, err := framework.GetPodsInNamespace(f.Client, f.Namespace.Name, framework.ImagePullerLabels)
		if err != nil {
			return err
		}

		if len(pods) != replicas {
			continue
		}

		allRunningOrUnschedulable := true
		for _, pod := range pods {
			if !podRunningOrUnschedulable(pod) {
				allRunningOrUnschedulable = false
				break
			}
		}
		if allRunningOrUnschedulable {
			return nil
		}
	}
	return fmt.Errorf("Pod name %s: Gave up waiting %v for %d pods to come up", id, timeout, replicas)
}

func podRunningOrUnschedulable(pod *api.Pod) bool {
	_, cond := api.GetPodCondition(&pod.Status, api.PodScheduled)
	if cond != nil && cond.Status == api.ConditionFalse && cond.Reason == "Unschedulable" {
		return true
	}
	running, _ := framework.PodRunningReady(pod)
	return running
}
<|endoftext|>