<commit_before>// Copyright 2010 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

/*
Package zip provides support for reading and writing ZIP archives.

See: https://www.pkware.com/appnote

This package does not support disk spanning.

A note about ZIP64:

To be backwards compatible the FileHeader has both 32 and 64 bit Size
fields. The 64 bit fields will always contain the correct value and
for normal archives both fields will be the same. For files requiring
the ZIP64 format the 32 bit fields will be 0xffffffff and the 64 bit
fields must be used instead.
*/
package zip

import (
	"os"
	"path"
	"time"
)

// Compression methods.
const (
	Store   uint16 = 0 // no compression
	Deflate uint16 = 8 // DEFLATE compressed
)

const (
	fileHeaderSignature      = 0x04034b50
	directoryHeaderSignature = 0x02014b50
	directoryEndSignature    = 0x06054b50
	directory64LocSignature  = 0x07064b50
	directory64EndSignature  = 0x06064b50
	dataDescriptorSignature  = 0x08074b50 // de-facto standard; required by OS X Finder
	fileHeaderLen            = 30         // + filename + extra
	directoryHeaderLen       = 46         // + filename + extra + comment
	directoryEndLen          = 22         // + comment
	dataDescriptorLen        = 16         // four uint32: descriptor signature, crc32, compressed size, size
	dataDescriptor64Len      = 24         // descriptor with 8 byte sizes
	directory64LocLen        = 20         //
	directory64EndLen        = 56         // + extra

	// Constants for the first byte in CreatorVersion.
	creatorFAT    = 0
	creatorUnix   = 3
	creatorNTFS   = 11
	creatorVFAT   = 14
	creatorMacOSX = 19

	// Version numbers.
	zipVersion20 = 20 // 2.0
	zipVersion45 = 45 // 4.5 (reads and writes zip64 archives)

	// Limits for non zip64 files.
	uint16max = (1 << 16) - 1
	uint32max = (1 << 32) - 1

	// Extra header IDs.
	//
	// IDs 0..31 are reserved for official use by PKWARE.
	// IDs above that range are defined by third-party vendors.
	// Since ZIP lacked high precision timestamps (nor a official specification
	// of the timezone used for the date fields), many competing extra fields
	// have been invented. Pervasive use effectively makes them "official".
	//
	// See http://mdfs.net/Docs/Comp/Archiving/Zip/ExtraField
	zip64ExtraID       = 0x0001 // Zip64 extended information
	ntfsExtraID        = 0x000a // NTFS
	unixExtraID        = 0x000d // UNIX
	extTimeExtraID     = 0x5455 // Extended timestamp
	infoZipUnixExtraID = 0x5855 // Info-ZIP Unix extension
)

// FileHeader describes a file within a zip file.
// See the zip spec for details.
type FileHeader struct {
	// Name is the name of the file.
	// It must be a relative path, not start with a drive letter (e.g. C:),
	// and must use forward slashes instead of back slashes. A trailing slash
	// indicates that this file is a directory and should have no data.
	Name string

	// Comment is any arbitrary user-defined string shorter than 64KiB.
	Comment string

	// NonUTF8 indicates that Name and Comment are not encoded in UTF-8.
	//
	// By specification, the only other encoding permitted should be CP-437,
	// but historically many ZIP readers interpret Name and Comment as whatever
	// the system's local character encoding happens to be.
	//
	// This flag should only be set if the user intends to encode a non-portable
	// ZIP file for a specific localized region. Otherwise, the Writer
	// automatically sets the ZIP format's UTF-8 flag for valid UTF-8 strings.
	NonUTF8 bool

	CreatorVersion uint16
	ReaderVersion  uint16
	Flags          uint16

	// Method is the compression method. If zero, Store is used.
	Method uint16

	// Modified is the modified time of the file.
	//
	// When reading, an extended timestamp is preferred over the legacy MS-DOS
	// date field, and the offset between the times is used as the timezone.
	// If only the MS-DOS date is present, the timezone is assumed to be UTC.
	//
	// When writing, an extended timestamp (which is timezone-agnostic) is
	// always emitted. The legacy MS-DOS date field is encoded according to the
	// location of the Modified time.
	Modified     time.Time
	ModifiedTime uint16 // Deprecated: Legacy MS-DOS date; use Modified instead.
	ModifiedDate uint16 // Deprecated: Legacy MS-DOS time; use Modified instead.

	CRC32              uint32
	CompressedSize     uint32 // Deprecated: Use CompressedSize64 instead.
	UncompressedSize   uint32 // Deprecated: Use UncompressedSize64 instead.
	CompressedSize64   uint64
	UncompressedSize64 uint64
	Extra              []byte
	ExternalAttrs      uint32 // Meaning depends on CreatorVersion
}

// FileInfo returns an os.FileInfo for the FileHeader.
func (h *FileHeader) FileInfo() os.FileInfo {
	return headerFileInfo{h}
}

// headerFileInfo implements os.FileInfo.
type headerFileInfo struct {
	fh *FileHeader
}

func (fi headerFileInfo) Name() string { return path.Base(fi.fh.Name) }
func (fi headerFileInfo) Size() int64 {
	if fi.fh.UncompressedSize64 > 0 {
		return int64(fi.fh.UncompressedSize64)
	}
	return int64(fi.fh.UncompressedSize)
}
func (fi headerFileInfo) IsDir() bool        { return fi.Mode().IsDir() }
func (fi headerFileInfo) ModTime() time.Time { return fi.fh.ModTime() }
func (fi headerFileInfo) Mode() os.FileMode  { return fi.fh.Mode() }
func (fi headerFileInfo) Sys() interface{}   { return fi.fh }

// FileInfoHeader creates a partially-populated FileHeader from an
// os.FileInfo.
// Because os.FileInfo's Name method returns only the base name of
// the file it describes, it may be necessary to modify the Name field
// of the returned header to provide the full path name of the file.
// If compression is desired, callers should set the FileHeader.Method
// field; it is unset by default.
func FileInfoHeader(fi os.FileInfo) (*FileHeader, error) {
	size := fi.Size()
	fh := &FileHeader{
		Name:               fi.Name(),
		UncompressedSize64: uint64(size),
	}
	fh.SetModTime(fi.ModTime())
	fh.SetMode(fi.Mode())
	if fh.UncompressedSize64 > uint32max {
		fh.UncompressedSize = uint32max
	} else {
		fh.UncompressedSize = uint32(fh.UncompressedSize64)
	}
	return fh, nil
}

type directoryEnd struct {
	diskNbr            uint32 // unused
	dirDiskNbr         uint32 // unused
	dirRecordsThisDisk uint64 // unused
	directoryRecords   uint64
	directorySize      uint64
	directoryOffset    uint64 // relative to file
	commentLen         uint16
	comment            string
}

// timeZone returns a *time.Location based on the provided offset.
// If the offset is non-sensible, then this uses an offset of zero.
func timeZone(offset time.Duration) *time.Location {
	const (
		minOffset   = -12 * time.Hour  // E.g., Baker island at -12:00
		maxOffset   = +14 * time.Hour  // E.g., Line island at +14:00
		offsetAlias = 15 * time.Minute // E.g., Nepal at +5:45
	)
	offset = offset.Round(offsetAlias)
	if offset < minOffset || maxOffset < offset {
		offset = 0
	}
	return time.FixedZone("", int(offset/time.Second))
}

// msDosTimeToTime converts an MS-DOS date and time into a time.Time.
// The resolution is 2s.
// See: https://msdn.microsoft.com/en-us/library/ms724247(v=VS.85).aspx
func msDosTimeToTime(dosDate, dosTime uint16) time.Time {
	return time.Date(
		// date bits 0-4: day of month; 5-8: month; 9-15: years since 1980
		int(dosDate>>9+1980),
		time.Month(dosDate>>5&0xf),
		int(dosDate&0x1f),

		// time bits 0-4: second/2; 5-10: minute; 11-15: hour
		int(dosTime>>11),
		int(dosTime>>5&0x3f),
		int(dosTime&0x1f*2),
		0, // nanoseconds

		time.UTC,
	)
}

// timeToMsDosTime converts a time.Time to an MS-DOS date and time.
// The resolution is 2s.
// See: https://msdn.microsoft.com/en-us/library/ms724274(v=VS.85).aspx
func timeToMsDosTime(t time.Time) (fDate uint16, fTime uint16) {
	fDate = uint16(t.Day() + int(t.Month())<<5 + (t.Year()-1980)<<9)
	fTime = uint16(t.Second()/2 + t.Minute()<<5 + t.Hour()<<11)
	return
}

// ModTime returns the modification time in UTC using the legacy
// ModifiedDate and ModifiedTime fields.
//
// Deprecated: Use Modified instead.
func (h *FileHeader) ModTime() time.Time {
	return msDosTimeToTime(h.ModifiedDate, h.ModifiedTime)
}

// SetModTime sets the Modified, ModifiedTime, and ModifiedDate fields
// to the given time in UTC.
//
// Deprecated: Use Modified instead.
func (h *FileHeader) SetModTime(t time.Time) {
	t = t.UTC() // Convert to UTC for compatibility
	h.Modified = t
	h.ModifiedDate, h.ModifiedTime = timeToMsDosTime(t)
}

const (
	// Unix constants. The specification doesn't mention them,
	// but these seem to be the values agreed on by tools.
	s_IFMT   = 0xf000
	s_IFSOCK = 0xc000
	s_IFLNK  = 0xa000
	s_IFREG  = 0x8000
	s_IFBLK  = 0x6000
	s_IFDIR  = 0x4000
	s_IFCHR  = 0x2000
	s_IFIFO  = 0x1000
	s_ISUID  = 0x800
	s_ISGID  = 0x400
	s_ISVTX  = 0x200

	msdosDir      = 0x10
	msdosReadOnly = 0x01
)

// Mode returns the permission and mode bits for the FileHeader.
func (h *FileHeader) Mode() (mode os.FileMode) {
	switch h.CreatorVersion >> 8 {
	case creatorUnix, creatorMacOSX:
		mode = unixModeToFileMode(h.ExternalAttrs >> 16)
	case creatorNTFS, creatorVFAT, creatorFAT:
		mode = msdosModeToFileMode(h.ExternalAttrs)
	}
	if len(h.Name) > 0 && h.Name[len(h.Name)-1] == '/' {
		mode |= os.ModeDir
	}
	return mode
}

// SetMode changes the permission and mode bits for the FileHeader.
func (h *FileHeader) SetMode(mode os.FileMode) {
	h.CreatorVersion = h.CreatorVersion&0xff | creatorUnix<<8
	h.ExternalAttrs = fileModeToUnixMode(mode) << 16

	// set MSDOS attributes too, as the original zip does.
	if mode&os.ModeDir != 0 {
		h.ExternalAttrs |= msdosDir
	}
	if mode&0200 == 0 {
		h.ExternalAttrs |= msdosReadOnly
	}
}

// isZip64 reports whether the file size exceeds the 32 bit limit
func (fh *FileHeader) isZip64() bool {
	return fh.CompressedSize64 >= uint32max || fh.UncompressedSize64 >= uint32max
}

func msdosModeToFileMode(m uint32) (mode os.FileMode) {
	if m&msdosDir != 0 {
		mode = os.ModeDir | 0777
	} else {
		mode = 0666
	}
	if m&msdosReadOnly != 0 {
		mode &^= 0222
	}
	return mode
}

func fileModeToUnixMode(mode os.FileMode) uint32 {
	var m uint32
	switch mode & os.ModeType {
	default:
		m = s_IFREG
	case os.ModeDir:
		m = s_IFDIR
	case os.ModeSymlink:
		m = s_IFLNK
	case os.ModeNamedPipe:
		m = s_IFIFO
	case os.ModeSocket:
		m = s_IFSOCK
	case os.ModeDevice:
		if mode&os.ModeCharDevice != 0 {
			m = s_IFCHR
		} else {
			m = s_IFBLK
		}
	}
	if mode&os.ModeSetuid != 0 {
		m |= s_ISUID
	}
	if mode&os.ModeSetgid != 0 {
		m |= s_ISGID
	}
	if mode&os.ModeSticky != 0 {
		m |= s_ISVTX
	}
	return m | uint32(mode&0777)
}

func unixModeToFileMode(m uint32) os.FileMode {
	mode := os.FileMode(m & 0777)
	switch m & s_IFMT {
	case s_IFBLK:
		mode |= os.ModeDevice
	case s_IFCHR:
		mode |= os.ModeDevice | os.ModeCharDevice
	case s_IFDIR:
		mode |= os.ModeDir
	case s_IFIFO:
		mode |= os.ModeNamedPipe
	case s_IFLNK:
		mode |= os.ModeSymlink
	case s_IFREG:
		// nothing to do
	case s_IFSOCK:
		mode |= os.ModeSocket
	}
	if m&s_ISGID != 0 {
		mode |= os.ModeSetgid
	}
	if m&s_ISUID != 0 {
		mode |= os.ModeSetuid
	}
	if m&s_ISVTX != 0 {
		mode |= os.ModeSticky
	}
	return mode
}
<commit_msg>archive/zip: warn about FileHeader.Name being unvalidated on read<commit_after>// Copyright 2010 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

/*
Package zip provides support for reading and writing ZIP archives.

See: https://www.pkware.com/appnote

This package does not support disk spanning.

A note about ZIP64:

To be backwards compatible the FileHeader has both 32 and 64 bit Size
fields. The 64 bit fields will always contain the correct value and
for normal archives both fields will be the same. For files requiring
the ZIP64 format the 32 bit fields will be 0xffffffff and the 64 bit
fields must be used instead.
*/
package zip

import (
	"os"
	"path"
	"time"
)

// Compression methods.
const (
	Store   uint16 = 0 // no compression
	Deflate uint16 = 8 // DEFLATE compressed
)

const (
	fileHeaderSignature      = 0x04034b50
	directoryHeaderSignature = 0x02014b50
	directoryEndSignature    = 0x06054b50
	directory64LocSignature  = 0x07064b50
	directory64EndSignature  = 0x06064b50
	dataDescriptorSignature  = 0x08074b50 // de-facto standard; required by OS X Finder
	fileHeaderLen            = 30         // + filename + extra
	directoryHeaderLen       = 46         // + filename + extra + comment
	directoryEndLen          = 22         // + comment
	dataDescriptorLen        = 16         // four uint32: descriptor signature, crc32, compressed size, size
	dataDescriptor64Len      = 24         // descriptor with 8 byte sizes
	directory64LocLen        = 20         //
	directory64EndLen        = 56         // + extra

	// Constants for the first byte in CreatorVersion.
	creatorFAT    = 0
	creatorUnix   = 3
	creatorNTFS   = 11
	creatorVFAT   = 14
	creatorMacOSX = 19

	// Version numbers.
	zipVersion20 = 20 // 2.0
	zipVersion45 = 45 // 4.5 (reads and writes zip64 archives)

	// Limits for non zip64 files.
	uint16max = (1 << 16) - 1
	uint32max = (1 << 32) - 1

	// Extra header IDs.
	//
	// IDs 0..31 are reserved for official use by PKWARE.
	// IDs above that range are defined by third-party vendors.
	// Since ZIP lacked high precision timestamps (nor a official specification
	// of the timezone used for the date fields), many competing extra fields
	// have been invented. Pervasive use effectively makes them "official".
	//
	// See http://mdfs.net/Docs/Comp/Archiving/Zip/ExtraField
	zip64ExtraID       = 0x0001 // Zip64 extended information
	ntfsExtraID        = 0x000a // NTFS
	unixExtraID        = 0x000d // UNIX
	extTimeExtraID     = 0x5455 // Extended timestamp
	infoZipUnixExtraID = 0x5855 // Info-ZIP Unix extension
)

// FileHeader describes a file within a zip file.
// See the zip spec for details.
type FileHeader struct {
	// Name is the name of the file.
	//
	// It must be a relative path, not start with a drive letter (such as "C:"),
	// and must use forward slashes instead of back slashes. A trailing slash
	// indicates that this file is a directory and should have no data.
	//
	// When reading zip files, the Name field is populated from
	// the zip file directly and is not validated for correctness.
	// It is the caller's responsibility to sanitize it as
	// appropriate, including canonicalizing slash directions,
	// validating that paths are relative, and preventing path
	// traversal through filenames ("../../../").
	Name string

	// Comment is any arbitrary user-defined string shorter than 64KiB.
	Comment string

	// NonUTF8 indicates that Name and Comment are not encoded in UTF-8.
	//
	// By specification, the only other encoding permitted should be CP-437,
	// but historically many ZIP readers interpret Name and Comment as whatever
	// the system's local character encoding happens to be.
	//
	// This flag should only be set if the user intends to encode a non-portable
	// ZIP file for a specific localized region. Otherwise, the Writer
	// automatically sets the ZIP format's UTF-8 flag for valid UTF-8 strings.
	NonUTF8 bool

	CreatorVersion uint16
	ReaderVersion  uint16
	Flags          uint16

	// Method is the compression method. If zero, Store is used.
	Method uint16

	// Modified is the modified time of the file.
	//
	// When reading, an extended timestamp is preferred over the legacy MS-DOS
	// date field, and the offset between the times is used as the timezone.
	// If only the MS-DOS date is present, the timezone is assumed to be UTC.
	//
	// When writing, an extended timestamp (which is timezone-agnostic) is
	// always emitted. The legacy MS-DOS date field is encoded according to the
	// location of the Modified time.
	Modified     time.Time
	ModifiedTime uint16 // Deprecated: Legacy MS-DOS date; use Modified instead.
	ModifiedDate uint16 // Deprecated: Legacy MS-DOS time; use Modified instead.

	CRC32              uint32
	CompressedSize     uint32 // Deprecated: Use CompressedSize64 instead.
	UncompressedSize   uint32 // Deprecated: Use UncompressedSize64 instead.
	CompressedSize64   uint64
	UncompressedSize64 uint64
	Extra              []byte
	ExternalAttrs      uint32 // Meaning depends on CreatorVersion
}

// FileInfo returns an os.FileInfo for the FileHeader.
func (h *FileHeader) FileInfo() os.FileInfo {
	return headerFileInfo{h}
}

// headerFileInfo implements os.FileInfo.
type headerFileInfo struct {
	fh *FileHeader
}

func (fi headerFileInfo) Name() string { return path.Base(fi.fh.Name) }
func (fi headerFileInfo) Size() int64 {
	if fi.fh.UncompressedSize64 > 0 {
		return int64(fi.fh.UncompressedSize64)
	}
	return int64(fi.fh.UncompressedSize)
}
func (fi headerFileInfo) IsDir() bool        { return fi.Mode().IsDir() }
func (fi headerFileInfo) ModTime() time.Time { return fi.fh.ModTime() }
func (fi headerFileInfo) Mode() os.FileMode  { return fi.fh.Mode() }
func (fi headerFileInfo) Sys() interface{}   { return fi.fh }

// FileInfoHeader creates a partially-populated FileHeader from an
// os.FileInfo.
// Because os.FileInfo's Name method returns only the base name of
// the file it describes, it may be necessary to modify the Name field
// of the returned header to provide the full path name of the file.
// If compression is desired, callers should set the FileHeader.Method
// field; it is unset by default.
func FileInfoHeader(fi os.FileInfo) (*FileHeader, error) {
	size := fi.Size()
	fh := &FileHeader{
		Name:               fi.Name(),
		UncompressedSize64: uint64(size),
	}
	fh.SetModTime(fi.ModTime())
	fh.SetMode(fi.Mode())
	if fh.UncompressedSize64 > uint32max {
		fh.UncompressedSize = uint32max
	} else {
		fh.UncompressedSize = uint32(fh.UncompressedSize64)
	}
	return fh, nil
}

type directoryEnd struct {
	diskNbr            uint32 // unused
	dirDiskNbr         uint32 // unused
	dirRecordsThisDisk uint64 // unused
	directoryRecords   uint64
	directorySize      uint64
	directoryOffset    uint64 // relative to file
	commentLen         uint16
	comment            string
}

// timeZone returns a *time.Location based on the provided offset.
// If the offset is non-sensible, then this uses an offset of zero.
func timeZone(offset time.Duration) *time.Location {
	const (
		minOffset   = -12 * time.Hour  // E.g., Baker island at -12:00
		maxOffset   = +14 * time.Hour  // E.g., Line island at +14:00
		offsetAlias = 15 * time.Minute // E.g., Nepal at +5:45
	)
	offset = offset.Round(offsetAlias)
	if offset < minOffset || maxOffset < offset {
		offset = 0
	}
	return time.FixedZone("", int(offset/time.Second))
}

// msDosTimeToTime converts an MS-DOS date and time into a time.Time.
// The resolution is 2s.
// See: https://msdn.microsoft.com/en-us/library/ms724247(v=VS.85).aspx
func msDosTimeToTime(dosDate, dosTime uint16) time.Time {
	return time.Date(
		// date bits 0-4: day of month; 5-8: month; 9-15: years since 1980
		int(dosDate>>9+1980),
		time.Month(dosDate>>5&0xf),
		int(dosDate&0x1f),

		// time bits 0-4: second/2; 5-10: minute; 11-15: hour
		int(dosTime>>11),
		int(dosTime>>5&0x3f),
		int(dosTime&0x1f*2),
		0, // nanoseconds

		time.UTC,
	)
}

// timeToMsDosTime converts a time.Time to an MS-DOS date and time.
// The resolution is 2s.
// See: https://msdn.microsoft.com/en-us/library/ms724274(v=VS.85).aspx
func timeToMsDosTime(t time.Time) (fDate uint16, fTime uint16) {
	fDate = uint16(t.Day() + int(t.Month())<<5 + (t.Year()-1980)<<9)
	fTime = uint16(t.Second()/2 + t.Minute()<<5 + t.Hour()<<11)
	return
}

// ModTime returns the modification time in UTC using the legacy
// ModifiedDate and ModifiedTime fields.
//
// Deprecated: Use Modified instead.
func (h *FileHeader) ModTime() time.Time {
	return msDosTimeToTime(h.ModifiedDate, h.ModifiedTime)
}

// SetModTime sets the Modified, ModifiedTime, and ModifiedDate fields
// to the given time in UTC.
//
// Deprecated: Use Modified instead.
func (h *FileHeader) SetModTime(t time.Time) {
	t = t.UTC() // Convert to UTC for compatibility
	h.Modified = t
	h.ModifiedDate, h.ModifiedTime = timeToMsDosTime(t)
}

const (
	// Unix constants. The specification doesn't mention them,
	// but these seem to be the values agreed on by tools.
	s_IFMT   = 0xf000
	s_IFSOCK = 0xc000
	s_IFLNK  = 0xa000
	s_IFREG  = 0x8000
	s_IFBLK  = 0x6000
	s_IFDIR  = 0x4000
	s_IFCHR  = 0x2000
	s_IFIFO  = 0x1000
	s_ISUID  = 0x800
	s_ISGID  = 0x400
	s_ISVTX  = 0x200

	msdosDir      = 0x10
	msdosReadOnly = 0x01
)

// Mode returns the permission and mode bits for the FileHeader.
func (h *FileHeader) Mode() (mode os.FileMode) {
	switch h.CreatorVersion >> 8 {
	case creatorUnix, creatorMacOSX:
		mode = unixModeToFileMode(h.ExternalAttrs >> 16)
	case creatorNTFS, creatorVFAT, creatorFAT:
		mode = msdosModeToFileMode(h.ExternalAttrs)
	}
	if len(h.Name) > 0 && h.Name[len(h.Name)-1] == '/' {
		mode |= os.ModeDir
	}
	return mode
}

// SetMode changes the permission and mode bits for the FileHeader.
func (h *FileHeader) SetMode(mode os.FileMode) {
	h.CreatorVersion = h.CreatorVersion&0xff | creatorUnix<<8
	h.ExternalAttrs = fileModeToUnixMode(mode) << 16

	// set MSDOS attributes too, as the original zip does.
	if mode&os.ModeDir != 0 {
		h.ExternalAttrs |= msdosDir
	}
	if mode&0200 == 0 {
		h.ExternalAttrs |= msdosReadOnly
	}
}

// isZip64 reports whether the file size exceeds the 32 bit limit
func (fh *FileHeader) isZip64() bool {
	return fh.CompressedSize64 >= uint32max || fh.UncompressedSize64 >= uint32max
}

func msdosModeToFileMode(m uint32) (mode os.FileMode) {
	if m&msdosDir != 0 {
		mode = os.ModeDir | 0777
	} else {
		mode = 0666
	}
	if m&msdosReadOnly != 0 {
		mode &^= 0222
	}
	return mode
}

func fileModeToUnixMode(mode os.FileMode) uint32 {
	var m uint32
	switch mode & os.ModeType {
	default:
		m = s_IFREG
	case os.ModeDir:
		m = s_IFDIR
	case os.ModeSymlink:
		m = s_IFLNK
	case os.ModeNamedPipe:
		m = s_IFIFO
	case os.ModeSocket:
		m = s_IFSOCK
	case os.ModeDevice:
		if mode&os.ModeCharDevice != 0 {
			m = s_IFCHR
		} else {
			m = s_IFBLK
		}
	}
	if mode&os.ModeSetuid != 0 {
		m |= s_ISUID
	}
	if mode&os.ModeSetgid != 0 {
		m |= s_ISGID
	}
	if mode&os.ModeSticky != 0 {
		m |= s_ISVTX
	}
	return m | uint32(mode&0777)
}

func unixModeToFileMode(m uint32) os.FileMode {
	mode := os.FileMode(m & 0777)
	switch m & s_IFMT {
	case s_IFBLK:
		mode |= os.ModeDevice
	case s_IFCHR:
		mode |= os.ModeDevice | os.ModeCharDevice
	case s_IFDIR:
		mode |= os.ModeDir
	case s_IFIFO:
		mode |= os.ModeNamedPipe
	case s_IFLNK:
		mode |= os.ModeSymlink
	case s_IFREG:
		// nothing to do
	case s_IFSOCK:
		mode |= os.ModeSocket
	}
	if m&s_ISGID != 0 {
		mode |= os.ModeSetgid
	}
	if m&s_ISUID != 0 {
		mode |= os.ModeSetuid
	}
	if m&s_ISVTX != 0 {
		mode |= os.ModeSticky
	}
	return mode
}
<|endoftext|>


<commit_before>/*
Copyright 2019 The Kubernetes Authors.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package log_test

import (
	// "errors"
	"regexp"
	"sort"
	"strings"
	"testing"

	"github.com/onsi/ginkgo"
	"github.com/onsi/ginkgo/config"
	"github.com/onsi/ginkgo/reporters"
	"github.com/onsi/gomega"

	// "k8s.io/kubernetes/test/e2e/framework"
	"k8s.io/kubernetes/test/e2e/framework/log"
)

var _ = ginkgo.Describe("log", func() {
	ginkgo.BeforeEach(func() {
		log.Logf("before")
	})
	ginkgo.It("fails", func() {
		func() {
			log.Failf("I'm failing.")
		}()
	})
	ginkgo.It("asserts", func() {
		gomega.Expect(false).To(gomega.Equal(true), "false is never true")
	})
	// ginkgo.It("error", func() { // TODO(pohly): enable again, see comment below.
	// 	err := errors.New("an error with a long, useless description")
	// 	framework.ExpectNoError(err, "hard-coded error")
	// })
	ginkgo.AfterEach(func() {
		log.Logf("after")
		gomega.Expect(true).To(gomega.Equal(false), "true is never false either")
	})
})

func TestFailureOutput(t *testing.T) {
	// Run the Ginkgo suite with output collected by a custom
	// reporter in adddition to the default one. To see what the full
	// Ginkgo report looks like, run this test with "go test -v".
	config.DefaultReporterConfig.FullTrace = true
	gomega.RegisterFailHandler(log.Fail)
	fakeT := &testing.T{}
	reporter := reporters.NewFakeReporter()
	ginkgo.RunSpecsWithDefaultAndCustomReporters(fakeT, "Logging Suite", []ginkgo.Reporter{reporter})

	// Now check the output.
	// TODO: all of the stacks are currently broken because Ginkgo doesn't properly skip
	// over the initial entries returned by runtime.Stack. Fix is pending in
	// https://github.com/onsi/ginkgo/pull/590, "stack" texts need to be updated
	// when updating to a fixed Ginkgo.
	g := gomega.NewGomegaWithT(t)
	actual := normalizeReport(*reporter)
	expected := suiteResults{
		testResult{
			name:    "[Top Level] log asserts",
			output:  "INFO: before\nFAIL: false is never true\nExpected\n    <bool>: false\nto equal\n    <bool>: true\nINFO: after\nFAIL: true is never false either\nExpected\n    <bool>: true\nto equal\n    <bool>: false\n",
			failure: "false is never true\nExpected\n    <bool>: false\nto equal\n    <bool>: true",
			// TODO: should start with k8s.io/kubernetes/test/e2e/framework/log_test.glob..func1.3()
			stack: "\tassertion.go:75\nk8s.io/kubernetes/vendor/github.com/onsi/gomega/internal/assertion.(*Assertion).To()\n\tassertion.go:38\nk8s.io/kubernetes/test/e2e/framework/log_test.glob..func1.3()\n\tlogger_test.go:45\nk8s.io/kubernetes/vendor/github.com/onsi/ginkgo/internal/leafnodes.(*runner).runSync()\n\tlogger_test.go:65\n",
		},
		// That util.go appears in the output is a bug (https://github.com/kubernetes/kubernetes/issues/82013).
		// Because it currently appears, this test case is brittle and breaks when someome makes unrelated
		// changes in util.go which change the line number. Therefore it is commented out.
		// testResult{
		// 	name:    "[Top Level] log error",
		// 	output:  "INFO: before\nFAIL: hard-coded error\nUnexpected error:\n    <*errors.errorString>: {\n        s: \"an error with a long, useless description\",\n    }\n    an error with a long, useless description\noccurred\nINFO: after\nFAIL: true is never false either\nExpected\n    <bool>: true\nto equal\n    <bool>: false\n",
		// 	failure: "hard-coded error\nUnexpected error:\n    <*errors.errorString>: {\n        s: \"an error with a long, useless description\",\n    }\n    an error with a long, useless description\noccurred",
		// 	// TODO: should start with k8s.io/kubernetes/test/e2e/framework/log_test.glob..func1.4()
		// 	stack: "\tutil.go:1362\nk8s.io/kubernetes/test/e2e/framework.ExpectNoError()\n\tutil.go:1356\nk8s.io/kubernetes/test/e2e/framework/log_test.glob..func1.4()\n\tlogger_test.go:49\nk8s.io/kubernetes/vendor/github.com/onsi/ginkgo/internal/leafnodes.(*runner).runSync()\n\tlogger_test.go:65\n",
		// },
		testResult{
			name:    "[Top Level] log fails",
			output:  "INFO: before\nFAIL: I'm failing.\nINFO: after\nFAIL: true is never false either\nExpected\n    <bool>: true\nto equal\n    <bool>: false\n",
			failure: "I'm failing.",
			// TODO: should start with k8s.io/kubernetes/test/e2e/framework/log_test.glob..func1.2.1(...)
			stack: "\tlogger.go:52\nk8s.io/kubernetes/test/e2e/framework/log.Failf()\n\tlogger.go:44\nk8s.io/kubernetes/test/e2e/framework/log_test.glob..func1.2.1(...)\n\tlogger_test.go:41\nk8s.io/kubernetes/test/e2e/framework/log_test.glob..func1.2()\n\tlogger_test.go:42\nk8s.io/kubernetes/vendor/github.com/onsi/ginkgo/internal/leafnodes.(*runner).runSync()\n\tlogger_test.go:65\n",
		},
	}
	// Compare individual fields. Comparing the slices leads to unreadable error output when there is any mismatch.
	g.Expect(len(actual)).To(gomega.Equal(len(expected)), "%d entries in %v", len(expected), actual)
	for i, a := range actual {
		b := expected[i]
		g.Expect(a.name).To(gomega.Equal(b.name), "name in %d", i)
		g.Expect(a.output).To(gomega.Equal(b.output), "output in %d", i)
		g.Expect(a.failure).To(gomega.Equal(b.failure), "failure in %d", i)
		g.Expect(a.stack).To(gomega.Equal(b.stack), "stack in %d", i)
	}
}

type testResult struct {
	name string
	// output written to GinkgoWriter during test.
	output string
	// failure is SpecSummary.Failure.Message with varying parts stripped.
	failure string
	// stack is a normalized version (just file names, function parametes stripped) of
	// Ginkgo's FullStackTrace of a failure. Empty if no failure.
	stack string
}

type suiteResults []testResult

func normalizeReport(report reporters.FakeReporter) suiteResults {
	var results suiteResults
	for _, spec := range report.SpecSummaries {
		results = append(results, testResult{
			name:    strings.Join(spec.ComponentTexts, " "),
			output:  stripAddresses(stripTimes(spec.CapturedOutput)),
			failure: stripAddresses(stripTimes(spec.Failure.Message)),
			stack:   normalizeLocation(spec.Failure.Location.FullStackTrace),
		})
	}
	sort.Slice(results, func(i, j int) bool {
		return strings.Compare(results[i].name, results[j].name) < 0
	})
	return results
}

// timePrefix matches "Jul 17 08:08:25.950: " at the beginning of each line.
var timePrefix = regexp.MustCompile(`(?m)^[[:alpha:]]{3} [[:digit:]]{1,2} [[:digit:]]{2}:[[:digit:]]{2}:[[:digit:]]{2}.[[:digit:]]{3}: `)

func stripTimes(in string) string {
	return timePrefix.ReplaceAllString(in, "")
}

// instanceAddr matches " | 0xc0003dec60>"
var instanceAddr = regexp.MustCompile(` \| 0x[0-9a-fA-F]+>`)

func stripAddresses(in string) string {
	return instanceAddr.ReplaceAllString(in, ">")
}

// stackLocation matches "<some path>/<file>.go:75 +0x1f1" after a slash (built
// locally) or one of a few relative paths (built in the Kubernetes CI).
var stackLocation = regexp.MustCompile(`(?:/|vendor/|test/|GOROOT/).*/([[:^space:]]+.go:[[:digit:]]+)( \+0x[0-9a-fA-F]+)?`)

// functionArgs matches "<function name>(...)".
var functionArgs = regexp.MustCompile(`([[:alpha:]]+)\(.*\)`)

// testingStackEntries matches "testing.tRunner" and "created by" entries.
var testingStackEntries = regexp.MustCompile(`(?m)(?:testing\.|created by).*\n\t.*\n`)

// normalizeLocation removes path prefix and function parameters and certain stack entries
// that we don't care about.
func normalizeLocation(in string) string {
	out := in
	out = stackLocation.ReplaceAllString(out, "$1")
	out = functionArgs.ReplaceAllString(out, "$1()")
	out = testingStackEntries.ReplaceAllString(out, "")
	return out
}
<commit_msg>e2e log: fix time stamp normalization in unit test<commit_after>/*
Copyright 2019 The Kubernetes Authors.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package log_test

import (
	// "errors"
	"regexp"
	"sort"
	"strings"
	"testing"

	"github.com/onsi/ginkgo"
	"github.com/onsi/ginkgo/config"
	"github.com/onsi/ginkgo/reporters"
	"github.com/onsi/gomega"

	// "k8s.io/kubernetes/test/e2e/framework"
	"k8s.io/kubernetes/test/e2e/framework/log"
)

var _ = ginkgo.Describe("log", func() {
	ginkgo.BeforeEach(func() {
		log.Logf("before")
	})
	ginkgo.It("fails", func() {
		func() {
			log.Failf("I'm failing.")
		}()
	})
	ginkgo.It("asserts", func() {
		gomega.Expect(false).To(gomega.Equal(true), "false is never true")
	})
	// ginkgo.It("error", func() { // TODO(pohly): enable again, see comment below.
	// 	err := errors.New("an error with a long, useless description")
	// 	framework.ExpectNoError(err, "hard-coded error")
	// })
	ginkgo.AfterEach(func() {
		log.Logf("after")
		gomega.Expect(true).To(gomega.Equal(false), "true is never false either")
	})
})

func TestFailureOutput(t *testing.T) {
	// Run the Ginkgo suite with output collected by a custom
	// reporter in adddition to the default one. To see what the full
	// Ginkgo report looks like, run this test with "go test -v".
	config.DefaultReporterConfig.FullTrace = true
	gomega.RegisterFailHandler(log.Fail)
	fakeT := &testing.T{}
	reporter := reporters.NewFakeReporter()
	ginkgo.RunSpecsWithDefaultAndCustomReporters(fakeT, "Logging Suite", []ginkgo.Reporter{reporter})

	// Now check the output.
	// TODO: all of the stacks are currently broken because Ginkgo doesn't properly skip
	// over the initial entries returned by runtime.Stack. Fix is pending in
	// https://github.com/onsi/ginkgo/pull/590, "stack" texts need to be updated
	// when updating to a fixed Ginkgo.
	g := gomega.NewGomegaWithT(t)
	actual := normalizeReport(*reporter)
	expected := suiteResults{
		testResult{
			name:    "[Top Level] log asserts",
			output:  "INFO: before\nFAIL: false is never true\nExpected\n    <bool>: false\nto equal\n    <bool>: true\nINFO: after\nFAIL: true is never false either\nExpected\n    <bool>: true\nto equal\n    <bool>: false\n",
			failure: "false is never true\nExpected\n    <bool>: false\nto equal\n    <bool>: true",
			// TODO: should start with k8s.io/kubernetes/test/e2e/framework/log_test.glob..func1.3()
			stack: "\tassertion.go:75\nk8s.io/kubernetes/vendor/github.com/onsi/gomega/internal/assertion.(*Assertion).To()\n\tassertion.go:38\nk8s.io/kubernetes/test/e2e/framework/log_test.glob..func1.3()\n\tlogger_test.go:45\nk8s.io/kubernetes/vendor/github.com/onsi/ginkgo/internal/leafnodes.(*runner).runSync()\n\tlogger_test.go:65\n",
		},
		// That util.go appears in the output is a bug (https://github.com/kubernetes/kubernetes/issues/82013).
		// Because it currently appears, this test case is brittle and breaks when someome makes unrelated
		// changes in util.go which change the line number. Therefore it is commented out.
		// testResult{
		// 	name:    "[Top Level] log error",
		// 	output:  "INFO: before\nFAIL: hard-coded error\nUnexpected error:\n    <*errors.errorString>: {\n        s: \"an error with a long, useless description\",\n    }\n    an error with a long, useless description\noccurred\nINFO: after\nFAIL: true is never false either\nExpected\n    <bool>: true\nto equal\n    <bool>: false\n",
		// 	failure: "hard-coded error\nUnexpected error:\n    <*errors.errorString>: {\n        s: \"an error with a long, useless description\",\n    }\n    an error with a long, useless description\noccurred",
		// 	// TODO: should start with k8s.io/kubernetes/test/e2e/framework/log_test.glob..func1.4()
		// 	stack: "\tutil.go:1362\nk8s.io/kubernetes/test/e2e/framework.ExpectNoError()\n\tutil.go:1356\nk8s.io/kubernetes/test/e2e/framework/log_test.glob..func1.4()\n\tlogger_test.go:49\nk8s.io/kubernetes/vendor/github.com/onsi/ginkgo/internal/leafnodes.(*runner).runSync()\n\tlogger_test.go:65\n",
		// },
		testResult{
			name:    "[Top Level] log fails",
			output:  "INFO: before\nFAIL: I'm failing.\nINFO: after\nFAIL: true is never false either\nExpected\n    <bool>: true\nto equal\n    <bool>: false\n",
			failure: "I'm failing.",
			// TODO: should start with k8s.io/kubernetes/test/e2e/framework/log_test.glob..func1.2.1(...)
			stack: "\tlogger.go:52\nk8s.io/kubernetes/test/e2e/framework/log.Failf()\n\tlogger.go:44\nk8s.io/kubernetes/test/e2e/framework/log_test.glob..func1.2.1(...)\n\tlogger_test.go:41\nk8s.io/kubernetes/test/e2e/framework/log_test.glob..func1.2()\n\tlogger_test.go:42\nk8s.io/kubernetes/vendor/github.com/onsi/ginkgo/internal/leafnodes.(*runner).runSync()\n\tlogger_test.go:65\n",
		},
	}
	// Compare individual fields. Comparing the slices leads to unreadable error output when there is any mismatch.
	g.Expect(len(actual)).To(gomega.Equal(len(expected)), "%d entries in %v", len(expected), actual)
	for i, a := range actual {
		b := expected[i]
		g.Expect(a.name).To(gomega.Equal(b.name), "name in %d", i)
		g.Expect(a.output).To(gomega.Equal(b.output), "output in %d", i)
		g.Expect(a.failure).To(gomega.Equal(b.failure), "failure in %d", i)
		g.Expect(a.stack).To(gomega.Equal(b.stack), "stack in %d", i)
	}
}

type testResult struct {
	name string
	// output written to GinkgoWriter during test.
	output string
	// failure is SpecSummary.Failure.Message with varying parts stripped.
	failure string
	// stack is a normalized version (just file names, function parametes stripped) of
	// Ginkgo's FullStackTrace of a failure. Empty if no failure.
	stack string
}

type suiteResults []testResult

func normalizeReport(report reporters.FakeReporter) suiteResults {
	var results suiteResults
	for _, spec := range report.SpecSummaries {
		results = append(results, testResult{
			name:    strings.Join(spec.ComponentTexts, " "),
			output:  stripAddresses(stripTimes(spec.CapturedOutput)),
			failure: stripAddresses(stripTimes(spec.Failure.Message)),
			stack:   normalizeLocation(spec.Failure.Location.FullStackTrace),
		})
	}
	sort.Slice(results, func(i, j int) bool {
		return strings.Compare(results[i].name, results[j].name) < 0
	})
	return results
}

// timePrefix matches "Jul 17 08:08:25.950: " at the beginning of each line.
var timePrefix = regexp.MustCompile(`(?m)^[[:alpha:]]{3} +[[:digit:]]{1,2} +[[:digit:]]{2}:[[:digit:]]{2}:[[:digit:]]{2}.[[:digit:]]{3}: `)

func stripTimes(in string) string {
	return timePrefix.ReplaceAllString(in, "")
}

// instanceAddr matches " | 0xc0003dec60>"
var instanceAddr = regexp.MustCompile(` \| 0x[0-9a-fA-F]+>`)

func stripAddresses(in string) string {
	return instanceAddr.ReplaceAllString(in, ">")
}

// stackLocation matches "<some path>/<file>.go:75 +0x1f1" after a slash (built
// locally) or one of a few relative paths (built in the Kubernetes CI).
var stackLocation = regexp.MustCompile(`(?:/|vendor/|test/|GOROOT/).*/([[:^space:]]+.go:[[:digit:]]+)( \+0x[0-9a-fA-F]+)?`)

// functionArgs matches "<function name>(...)".
var functionArgs = regexp.MustCompile(`([[:alpha:]]+)\(.*\)`)

// testingStackEntries matches "testing.tRunner" and "created by" entries.
var testingStackEntries = regexp.MustCompile(`(?m)(?:testing\.|created by).*\n\t.*\n`)

// normalizeLocation removes path prefix and function parameters and certain stack entries
// that we don't care about.
func normalizeLocation(in string) string {
	out := in
	out = stackLocation.ReplaceAllString(out, "$1")
	out = functionArgs.ReplaceAllString(out, "$1()")
	out = testingStackEntries.ReplaceAllString(out, "")
	return out
}
<|endoftext|>


<commit_before>package main

import (
	"flag"
	"fmt"
	"io"
	"io/ioutil"
	"log"
	"os"
	"os/exec"
	"path"
	"strings"
	"syscall"
	"time"
	//"user"

	"pty"
)

type cgroupname string

func (c cgroupname) apply(s string, f func(s string)) {
	// range of strings.Split("",",") is 1.
	// not exactly what we might expect.
	if s == "" {
		return
	}
	for _, g := range strings.Split(s, ",") {
		p := path.Join(g)
		f(p)
	}
}

func (c cgroupname) Validate(s string) {
	c.apply(s, func(s string) {
		if st, err := os.Stat(path.Join(string(c), s)); err != nil {
			log.Fatal("%v", err)
		} else if !st.IsDir() {
			log.Fatal("%s: not a directory", s)
		}
	})
}

func (c cgroupname) Create(s, name string) {
	if err := os.MkdirAll(path.Join(string(c), s, name), 0755); err != nil {
		log.Fatal(err)
	}
}

func (c cgroupname) Attach(s, name string, pid int) {
	t := path.Join(string(c), s, name, "tasks")
	b := []byte(fmt.Sprintf("%v", pid))
	if err := ioutil.WriteFile(t, b, 0600); err != nil {
		log.Fatal(err)
	}
}

func (c cgroupname) Destroy(s, n string) {
	if err := os.RemoveAll(path.Join(string(c), s, n)); err != nil {
		log.Fatal(err)
	}
}

func (c cgroupname) Do(groups string, pid int) {
	cgn := fmt.Sprintf("pflask.%d", pid)
	c.apply(groups, func(s string) {
		c.Create(s, cgn)
		c.Attach(s, cgn, pid)
	})
}

type mount struct {
	src, dst, mtype, opts string
	flags                 uintptr
	dir                   bool
}

// Add adds  to teh mouuntlist. Don't know if we need it, but we might for additional volumes?
func Add(src, dst, mtype, opts string, flags uintptr, dir bool) {
	mounts = append(mounts, mount{src: src, dst: dst, mtype: mtype, flags: flags, opts: opts, dir: dir})

}

// One mounts one mountpoint, using base as a prefix for the destination.
// If anything goes wrong, we just bail out; we've privatized the namespace
// so there is nothing more to do.
func (m *mount) One(base string) {
	dst := path.Join(base, m.dst)
	if m.dir {
		if err := os.MkdirAll(dst, 0755); err != nil {
			log.Fatalf("One: mkdirall %v: %v", m.dst, err)
		}
	}
	if err := syscall.Mount(m.src, dst, m.mtype, m.flags, m.opts); err != nil {
		log.Fatalf("Mount :%s: on :%s: type :%s: flags %x: opts :%v: %v\n",
			m.src, m.dst, m.mtype, m.flags, m.opts, err)
	}
}
func MountAll(base string) {
	root.One("")
	for _, m := range mounts {
		m.One(base)
	}
}

func make_console(base, console string) {
	if err := os.Chmod(console, 0600); err != nil {
		log.Printf("%v", err)
	}
	if err := os.Chown(console, 0, 0); err != nil {
		log.Printf("%v", err)
	}

	st, err := os.Stat(console)
	if err != nil {
		log.Printf("%v", err)
	}

	nn := path.Join(base, "/dev/console")
	if err := syscall.Mknod(nn, uint32(st.Mode()), int(st.Sys().(*syscall.Stat_t).Dev)); err != nil {
		log.Printf("%v", err)
	}

	// if any previous steps failed, this one will too, so we can bail here.
	if err := syscall.Mount(console, nn, "", syscall.MS_BIND, ""); err != nil {
		log.Fatalf("Mount :%s: on :%s: flags %v: %v",
			console, nn, syscall.MS_BIND, err)
	}

}
func copy_nodes(base string) {
	nodes := []string{
		"/dev/tty",
		"/dev/full",
		"/dev/null",
		"/dev/zero",
		"/dev/random",
		"/dev/urandom"}

	for _, n := range nodes {
		st, err := os.Stat(n)
		if err != nil {
			log.Printf("%v", err)
		}
		nn := path.Join(base, n)
		if err := syscall.Mknod(nn, uint32(st.Mode()), int(st.Sys().(*syscall.Stat_t).Dev)); err != nil {
			log.Printf("%v", err)
		}

	}
}

func make_ptmx(base string) {
	dst := path.Join(base, "/dev/ptmx")

	if err := os.Symlink("/dev/ptmx", dst); err != nil {
		log.Printf("%v", err)
	}
}

func make_symlinks(base string) {
	linkit := []struct {
		src, dst string
	}{
		{"/dev/pts/ptmx", "/dev/ptmx"},
		{"/proc/kcore", "/dev/core"},
		{"/proc/self/fd", "/dev/fd"},
		{"/proc/self/fd/0", "/dev/stdin"},
		{"/proc/self/fd/1", "/dev/stdout"},
		{"/proc/self/fd/2", "/dev/stderr"},
	}

	for i := range linkit {
		dst := path.Join(base, linkit[i].dst)

		if err := os.Symlink(linkit[i].src, dst); err != nil {
			log.Printf("%v", err)
		}
	}
}

func do_chroot(chroot, chdir string) {
	if chdir == "" {
		chdir = "/"
	}
	if err := syscall.Chroot(chroot); err != nil {
		log.Fatalf("%v", err)
	}
	if err := os.Chdir(chdir); err != nil {
		log.Fatalf("%v", err)
	}
}

var (
	cgpath  = flag.String("cgpath", "/sys/fs/cgroup", "set the cgroups")
	cgroup  = flag.String("cgroup", "", "set the cgroups")
	mnt     = flag.String("mount", "", "define mounts")
	chroot  = flag.String("chroot", "", "where to chroot to")
	chdir   = flag.String("chdir", "", "where to chrdir to in the chroot")
	console = flag.String("console", "/dev/console", "where the root is")
	keepenv = flag.Bool("keepenv", false, "Keep the environment")
	env     = flag.String("env", "", "other environment variables")
	user    = flag.String("user", "root" /*user.User.Username*/, "User name")
	root = &mount {"", "/", "", "", syscall.MS_SLAVE|syscall.MS_REC, false}
	mounts = []mount {
		{"proc", "/proc", "proc", "", syscall.MS_NOSUID|syscall.MS_NOEXEC|syscall.MS_NODEV, true},
		{"/proc/sys", "/proc/sys", "", "", syscall.MS_BIND, true},
		{"", "/proc/sys", "", "", syscall.MS_BIND|syscall.MS_RDONLY|syscall.MS_REMOUNT, true},
		{"sysfs", "/sys", "sysfs", "", syscall.MS_NOSUID|syscall.MS_NOEXEC|syscall.MS_NODEV|syscall.MS_RDONLY, true},
		{"tmpfs", "/dev", "tmpfs", "mode=755", syscall.MS_NOSUID|syscall.MS_STRICTATIME, true},
		{"devpts", "/dev/pts", "devpts", "newinstance,ptmxmode=000,mode=620,gid=5", syscall.MS_NOSUID|syscall.MS_NOEXEC, true},
		{"tmpfs", "/dev/shm", "tmpfs", "mode=1777", syscall.MS_NOSUID|syscall.MS_STRICTATIME|syscall.MS_NODEV, true},
		{"tmpfs", "/run", "tmpfs", "mode=755", syscall.MS_NOSUID|syscall.MS_NODEV|syscall.MS_STRICTATIME, true},
	}

)

func main() {
	flag.Parse()

	if len(flag.Args()) < 1 {
		os.Exit(1)
	}

	// note the unshare system call worketh not for Go.
	// So do it ourselves.
	a := os.Args
	if a[len(a)-1] != "#" {

		a = append(a, "#")
		// spawn ourselves with the right unsharing settings.
		c := exec.Command(a[0], a[1:]...)
		c.SysProcAttr = &syscall.SysProcAttr{Cloneflags: syscall.CLONE_NEWNS | syscall.CLONE_NEWUTS | syscall.CLONE_NEWIPC | syscall.CLONE_NEWPID}
		//		c.SysProcAttr.Cloneflags |= syscall.CLONE_NEWNET

		c.Stdin = os.Stdin
		c.Stdout = os.Stdout
		c.Stderr = os.Stderr
		t, err := getTermios(1)
		if err != nil {
			log.Fatalf("Can't get termios on fd 1: %v", err)
		}
		if err := c.Run(); err != nil {
			log.Printf(err.Error())
		}
		if err := t.set(1); err != nil {
			log.Printf("Can't reset termios on fd1: %v", err)
		}
		os.Exit(1)
	}

	if *chroot == "" {
		log.Fatalf("you are required to set the chroot via --chroot")
	}

	a = flag.Args()
	//log.Printf("greetings %v\n", a)
	a = a[:len(a)-1]

	ptm, pts, sname, err := pty.Open()
	if err != nil {
		log.Fatalf(err.Error())
	}

	// child code. Not really. What really happens here is we set
	// ourselves into the container, and spawn the child. It's a bit odd
	// but we're the master, but we'll run in the container? I don't know
	// how else to do it. This may require we set some things up first,
	// esp. the network. But, it's all fun and games until someone loses
	// an eye.
	MountAll(*chroot)

	copy_nodes(*chroot)

	make_ptmx(*chroot)

	make_symlinks(*chroot)

	make_console(*chroot, sname)

	//do_chroot(*chroot, *chdir)

	//umask(0022);

	/* TODO: drop capabilities */

	//do_user(user);

	e := make(map[string]string)
	if *keepenv {
		for _, v := range os.Environ() {
			k := strings.SplitN(v, "=", 2)
			e[k[0]] = k[1]
		}
	}

	term := os.Getenv("TERM")
	e["TERM"] = term
	e["PATH"] = "/usr/sbin:/usr/bin:/sbin:/bin"
	e["USER"] = *user
	e["LOGNAME"] = *user
	e["HOME"] = "/root"

	if *env != "" {
		for _, c := range strings.Split(*env, ",") {
			k := strings.SplitN(c, "=", 2)
			if len(k) != 2 {
				log.Printf("Bogus environment string %v", c)
				continue
			}
			e[k[0]] = k[1]
		}
	}
	e["container"] = "pflask"

	if len(a) == 0 {
		a = []string{"/bin/bash", "bash"}
	}
	c := exec.Command(a[0]) // , a[1:]...)
	c.Env = nil
	for k, v := range e {
		c.Env = append(c.Env, k+"="+v)
	}

	c.SysProcAttr = &syscall.SysProcAttr{
		Chroot:  *chroot,
		Setctty: true,
		Setsid:  true,
	}
	c.Stdout = pts
	c.Stdin = pts
	c.Stderr = c.Stdout
	c.SysProcAttr.Setctty = true
	c.SysProcAttr.Setsid = true
	c.SysProcAttr.Cloneflags = 0
	c.SysProcAttr.Ptrace = true
	c.Dir = "/"
	err = c.Start()
	if err != nil {
		panic(err)
	}
	kid := c.Process.Pid
	fmt.Printf("Started %d\n", kid)

	// set up the containers, then resume the process.
	// Its children will get the containers as it clones.

	cg := cgroupname(*cgpath)
	cg.Do(*cgroup, kid)

	// sometimes the detach fails. Looks like a race condition: we're
	// sending the detach before the child has hit the TRACE_ME point.
	// Experimentally, when it fails, even one seconds it too short to
	// sleep. Sleep for 5 seconds.

	for {
		if err = syscall.PtraceDetach(kid); err != nil {
			log.Printf("Could not detach %v, sleeping five seconds", kid)
			time.Sleep(5*time.Second)
			continue
		}
		break
	}

	// we don't set raw until the very last, so if they see an issue they can hit ^C
	t, err := getTermios(1)
	if err != nil {
		log.Fatalf(err.Error())
	}
	if err = t.setRaw(1); err != nil {
		log.Fatalf(err.Error())
	}

	go func() {
		io.Copy(os.Stdout, ptm)
		os.Exit(1)
	}()
	io.Copy(ptm, os.Stdin)
}
<commit_msg>do_chdir is not needed.<commit_after>package main

import (
	"flag"
	"fmt"
	"io"
	"io/ioutil"
	"log"
	"os"
	"os/exec"
	"path"
	"strings"
	"syscall"
	"time"
	//"user"

	"pty"
)

type cgroupname string

func (c cgroupname) apply(s string, f func(s string)) {
	// range of strings.Split("",",") is 1.
	// not exactly what we might expect.
	if s == "" {
		return
	}
	for _, g := range strings.Split(s, ",") {
		p := path.Join(g)
		f(p)
	}
}

func (c cgroupname) Validate(s string) {
	c.apply(s, func(s string) {
		if st, err := os.Stat(path.Join(string(c), s)); err != nil {
			log.Fatal("%v", err)
		} else if !st.IsDir() {
			log.Fatal("%s: not a directory", s)
		}
	})
}

func (c cgroupname) Create(s, name string) {
	if err := os.MkdirAll(path.Join(string(c), s, name), 0755); err != nil {
		log.Fatal(err)
	}
}

func (c cgroupname) Attach(s, name string, pid int) {
	t := path.Join(string(c), s, name, "tasks")
	b := []byte(fmt.Sprintf("%v", pid))
	if err := ioutil.WriteFile(t, b, 0600); err != nil {
		log.Fatal(err)
	}
}

func (c cgroupname) Destroy(s, n string) {
	if err := os.RemoveAll(path.Join(string(c), s, n)); err != nil {
		log.Fatal(err)
	}
}

func (c cgroupname) Do(groups string, pid int) {
	cgn := fmt.Sprintf("pflask.%d", pid)
	c.apply(groups, func(s string) {
		c.Create(s, cgn)
		c.Attach(s, cgn, pid)
	})
}

type mount struct {
	src, dst, mtype, opts string
	flags                 uintptr
	dir                   bool
}

// Add adds  to teh mouuntlist. Don't know if we need it, but we might for additional volumes?
func Add(src, dst, mtype, opts string, flags uintptr, dir bool) {
	mounts = append(mounts, mount{src: src, dst: dst, mtype: mtype, flags: flags, opts: opts, dir: dir})

}

// One mounts one mountpoint, using base as a prefix for the destination.
// If anything goes wrong, we just bail out; we've privatized the namespace
// so there is nothing more to do.
func (m *mount) One(base string) {
	dst := path.Join(base, m.dst)
	if m.dir {
		if err := os.MkdirAll(dst, 0755); err != nil {
			log.Fatalf("One: mkdirall %v: %v", m.dst, err)
		}
	}
	if err := syscall.Mount(m.src, dst, m.mtype, m.flags, m.opts); err != nil {
		log.Fatalf("Mount :%s: on :%s: type :%s: flags %x: opts :%v: %v\n",
			m.src, m.dst, m.mtype, m.flags, m.opts, err)
	}
}
func MountAll(base string) {
	root.One("")
	for _, m := range mounts {
		m.One(base)
	}
}

func make_console(base, console string) {
	if err := os.Chmod(console, 0600); err != nil {
		log.Printf("%v", err)
	}
	if err := os.Chown(console, 0, 0); err != nil {
		log.Printf("%v", err)
	}

	st, err := os.Stat(console)
	if err != nil {
		log.Printf("%v", err)
	}

	nn := path.Join(base, "/dev/console")
	if err := syscall.Mknod(nn, uint32(st.Mode()), int(st.Sys().(*syscall.Stat_t).Dev)); err != nil {
		log.Printf("%v", err)
	}

	// if any previous steps failed, this one will too, so we can bail here.
	if err := syscall.Mount(console, nn, "", syscall.MS_BIND, ""); err != nil {
		log.Fatalf("Mount :%s: on :%s: flags %v: %v",
			console, nn, syscall.MS_BIND, err)
	}

}
func copy_nodes(base string) {
	nodes := []string{
		"/dev/tty",
		"/dev/full",
		"/dev/null",
		"/dev/zero",
		"/dev/random",
		"/dev/urandom"}

	for _, n := range nodes {
		st, err := os.Stat(n)
		if err != nil {
			log.Printf("%v", err)
		}
		nn := path.Join(base, n)
		if err := syscall.Mknod(nn, uint32(st.Mode()), int(st.Sys().(*syscall.Stat_t).Dev)); err != nil {
			log.Printf("%v", err)
		}

	}
}

func make_ptmx(base string) {
	dst := path.Join(base, "/dev/ptmx")

	if err := os.Symlink("/dev/ptmx", dst); err != nil {
		log.Printf("%v", err)
	}
}

func make_symlinks(base string) {
	linkit := []struct {
		src, dst string
	}{
		{"/dev/pts/ptmx", "/dev/ptmx"},
		{"/proc/kcore", "/dev/core"},
		{"/proc/self/fd", "/dev/fd"},
		{"/proc/self/fd/0", "/dev/stdin"},
		{"/proc/self/fd/1", "/dev/stdout"},
		{"/proc/self/fd/2", "/dev/stderr"},
	}

	for i := range linkit {
		dst := path.Join(base, linkit[i].dst)

		if err := os.Symlink(linkit[i].src, dst); err != nil {
			log.Printf("%v", err)
		}
	}
}

var (
	cgpath  = flag.String("cgpath", "/sys/fs/cgroup", "set the cgroups")
	cgroup  = flag.String("cgroup", "", "set the cgroups")
	mnt     = flag.String("mount", "", "define mounts")
	chroot  = flag.String("chroot", "", "where to chroot to")
	chdir   = flag.String("chdir", "", "where to chrdir to in the chroot")
	console = flag.String("console", "/dev/console", "where the root is")
	keepenv = flag.Bool("keepenv", false, "Keep the environment")
	env     = flag.String("env", "", "other environment variables")
	user    = flag.String("user", "root" /*user.User.Username*/, "User name")
	root = &mount {"", "/", "", "", syscall.MS_SLAVE|syscall.MS_REC, false}
	mounts = []mount {
		{"proc", "/proc", "proc", "", syscall.MS_NOSUID|syscall.MS_NOEXEC|syscall.MS_NODEV, true},
		{"/proc/sys", "/proc/sys", "", "", syscall.MS_BIND, true},
		{"", "/proc/sys", "", "", syscall.MS_BIND|syscall.MS_RDONLY|syscall.MS_REMOUNT, true},
		{"sysfs", "/sys", "sysfs", "", syscall.MS_NOSUID|syscall.MS_NOEXEC|syscall.MS_NODEV|syscall.MS_RDONLY, true},
		{"tmpfs", "/dev", "tmpfs", "mode=755", syscall.MS_NOSUID|syscall.MS_STRICTATIME, true},
		{"devpts", "/dev/pts", "devpts", "newinstance,ptmxmode=000,mode=620,gid=5", syscall.MS_NOSUID|syscall.MS_NOEXEC, true},
		{"tmpfs", "/dev/shm", "tmpfs", "mode=1777", syscall.MS_NOSUID|syscall.MS_STRICTATIME|syscall.MS_NODEV, true},
		{"tmpfs", "/run", "tmpfs", "mode=755", syscall.MS_NOSUID|syscall.MS_NODEV|syscall.MS_STRICTATIME, true},
	}

)

func main() {
	flag.Parse()

	if len(flag.Args()) < 1 {
		os.Exit(1)
	}

	// note the unshare system call worketh not for Go.
	// So do it ourselves.
	a := os.Args
	if a[len(a)-1] != "#" {

		a = append(a, "#")
		// spawn ourselves with the right unsharing settings.
		c := exec.Command(a[0], a[1:]...)
		c.SysProcAttr = &syscall.SysProcAttr{Cloneflags: syscall.CLONE_NEWNS | syscall.CLONE_NEWUTS | syscall.CLONE_NEWIPC | syscall.CLONE_NEWPID}
		//		c.SysProcAttr.Cloneflags |= syscall.CLONE_NEWNET

		c.Stdin = os.Stdin
		c.Stdout = os.Stdout
		c.Stderr = os.Stderr
		t, err := getTermios(1)
		if err != nil {
			log.Fatalf("Can't get termios on fd 1: %v", err)
		}
		if err := c.Run(); err != nil {
			log.Printf(err.Error())
		}
		if err := t.set(1); err != nil {
			log.Printf("Can't reset termios on fd1: %v", err)
		}
		os.Exit(1)
	}

	if *chroot == "" {
		log.Fatalf("you are required to set the chroot via --chroot")
	}

	a = flag.Args()
	//log.Printf("greetings %v\n", a)
	a = a[:len(a)-1]

	ptm, pts, sname, err := pty.Open()
	if err != nil {
		log.Fatalf(err.Error())
	}

	// child code. Not really. What really happens here is we set
	// ourselves into the container, and spawn the child. It's a bit odd
	// but we're the master, but we'll run in the container? I don't know
	// how else to do it. This may require we set some things up first,
	// esp. the network. But, it's all fun and games until someone loses
	// an eye.
	MountAll(*chroot)

	copy_nodes(*chroot)

	make_ptmx(*chroot)

	make_symlinks(*chroot)

	make_console(*chroot, sname)

	//umask(0022);

	/* TODO: drop capabilities */

	//do_user(user);

	e := make(map[string]string)
	if *keepenv {
		for _, v := range os.Environ() {
			k := strings.SplitN(v, "=", 2)
			e[k[0]] = k[1]
		}
	}

	term := os.Getenv("TERM")
	e["TERM"] = term
	e["PATH"] = "/usr/sbin:/usr/bin:/sbin:/bin"
	e["USER"] = *user
	e["LOGNAME"] = *user
	e["HOME"] = "/root"

	if *env != "" {
		for _, c := range strings.Split(*env, ",") {
			k := strings.SplitN(c, "=", 2)
			if len(k) != 2 {
				log.Printf("Bogus environment string %v", c)
				continue
			}
			e[k[0]] = k[1]
		}
	}
	e["container"] = "pflask"

	if len(a) == 0 {
		a = []string{"/bin/bash", "bash"}
	}
	c := exec.Command(a[0]) // , a[1:]...)
	c.Env = nil
	for k, v := range e {
		c.Env = append(c.Env, k+"="+v)
	}

	c.SysProcAttr = &syscall.SysProcAttr{
		Chroot:  *chroot,
		Setctty: true,
		Setsid:  true,
	}
	c.Stdout = pts
	c.Stdin = pts
	c.Stderr = c.Stdout
	c.SysProcAttr.Setctty = true
	c.SysProcAttr.Setsid = true
	c.SysProcAttr.Cloneflags = 0
	c.SysProcAttr.Ptrace = true
	c.Dir = "/"
	err = c.Start()
	if err != nil {
		panic(err)
	}
	kid := c.Process.Pid
	fmt.Printf("Started %d\n", kid)

	// set up the containers, then resume the process.
	// Its children will get the containers as it clones.

	cg := cgroupname(*cgpath)
	cg.Do(*cgroup, kid)

	// sometimes the detach fails. Looks like a race condition: we're
	// sending the detach before the child has hit the TRACE_ME point.
	// Experimentally, when it fails, even one seconds it too short to
	// sleep. Sleep for 5 seconds.

	for {
		if err = syscall.PtraceDetach(kid); err != nil {
			log.Printf("Could not detach %v, sleeping five seconds", kid)
			time.Sleep(5*time.Second)
			continue
		}
		break
	}

	// we don't set raw until the very last, so if they see an issue they can hit ^C
	t, err := getTermios(1)
	if err != nil {
		log.Fatalf(err.Error())
	}
	if err = t.setRaw(1); err != nil {
		log.Fatalf(err.Error())
	}

	go func() {
		io.Copy(os.Stdout, ptm)
		os.Exit(1)
	}()
	io.Copy(ptm, os.Stdin)
}
<|endoftext|>


<commit_before>package checkerlution

import (
	"fmt"
	"github.com/couchbaselabs/logg"
	cbot "github.com/tleyden/checkers-bot"
	core "github.com/tleyden/checkers-core"
	ng "github.com/tleyden/neurgo"
)

type OperationMode int

const (
	RUNNING_MODE = iota
	TRAINING_MODE
)

type Checkerlution struct {
	ourTeamId            cbot.TeamType
	cortex               *ng.Cortex
	currentGameState     GameStateVector
	currentBoard         core.Board
	currentPossibleMove  ValidMoveCortexInput
	latestActuatorOutput []float64
	mode                 OperationMode
	latestFitnessScore   float64
}

type CheckerlutionFlags struct {
	CheckersBotFlags cbot.CheckersBotFlags
	PopulationName   string
}

func (c *Checkerlution) Start(ourTeamId cbot.TeamType) {
	c.ourTeamId = ourTeamId
	c.CreateNeurgoCortex()
	cortex := c.cortex
	cortex.Run()

}

func (c *Checkerlution) StartWithCortex(cortex *ng.Cortex, ourTeamId cbot.TeamType) {
	c.ourTeamId = ourTeamId
	c.setSensorActuatorFunctions(cortex)
	c.cortex = cortex
	cortex.Run()
}

func (c *Checkerlution) Think(gameState cbot.GameState) (bestMove cbot.ValidMove, ok bool) {

	ok = true
	ourTeam := gameState.Teams[c.ourTeamId]
	allValidMoves := ourTeam.AllValidMoves()
	if len(allValidMoves) > 0 {

		// convert into core.board representation
		board := gameState.Export()
		logg.LogTo("DEBUG", "Before team %v move %v", c.ourTeamId.String(), board.CompactString(true))

		// generate best move (will be a core.move) -- initially, pick random
		move := c.generateBestMove(board)

		// search allValidMoves to find corresponding valid move
		found, bestValidMoveIndex := cbot.CorrespondingValidMoveIndex(move, allValidMoves)

		if !found {
			msg := "Could not find corresponding valid move: %v in %v"
			logg.LogPanic(msg, move, allValidMoves)
		} else {
			bestMove = allValidMoves[bestValidMoveIndex]
		}

		// this is just for debugging purposes
		player := cbot.GetCorePlayer(c.ourTeamId)
		boardPostMove := board.ApplyMove(player, move)
		logg.LogTo("DEBUG", "After team %v move %v", c.ourTeamId.String(), boardPostMove.CompactString(true))

		return

	} else {
		ok = false
	}

	return

}

func (c *Checkerlution) GameFinished(gameState cbot.GameState) (shouldQuit bool) {
	switch c.mode {
	case TRAINING_MODE:
		shouldQuit = true
		c.latestFitnessScore = c.calculateFitness(gameState)
	case RUNNING_MODE:
		shouldQuit = false
	}
	return
}

func (c Checkerlution) Cortex() *ng.Cortex {
	return c.cortex
}

func (c *Checkerlution) SetMode(mode OperationMode) {
	c.mode = mode
}

func (c Checkerlution) calculateFitness(gameState cbot.GameState) (fitness float64) {

	switch gameState.WinningTeam {
	case c.ourTeamId:
		logg.LogTo("CHECKERLUTION", "calculateFitness based on winning.  Turn: %v", gameState.Turn)
		fitness = 1.0
	case c.ourTeamId.Opponent():
		logg.LogTo("CHECKERLUTION", "calculateFitness based on losing.  Turn: %v", gameState.Turn)
		fitness = -2.0
	default:
		logg.LogTo("CHECKERLUTION", "calculateFitness based on draw.  Turn: %v", gameState.Turn)
		fitness = 0.0
	}

	return
}

func (c *Checkerlution) CreateNeurgoCortex() {

	uuid := ng.NewUuid()
	cortexUuid := fmt.Sprintf("cortex-%s", uuid)
	nodeId := ng.NewCortexId(cortexUuid)

	c.cortex = &ng.Cortex{
		NodeId: nodeId,
	}

	c.cortex.Init()

	c.CreateSensors()

	outputNeuron := c.CreateOutputNeuron()
	layer1Neurons := c.CreateHiddenLayer1Neurons(outputNeuron)
	layer2Neurons := c.CreateHiddenLayer2Neurons(layer1Neurons, outputNeuron)

	// combine all into single slice and add neurons to cortex
	neurons := []*ng.Neuron{}
	neurons = append(neurons, layer1Neurons...)
	neurons = append(neurons, layer2Neurons...)
	neurons = append(neurons, outputNeuron)
	c.cortex.SetNeurons(neurons)

	actuator := c.CreateActuator()

	// workaround for error
	// Cannot make outbound connection, dataChan == nil [recovered]
	c.cortex.Init()

	outputNeuron.ConnectOutbound(actuator)
	actuator.ConnectInbound(outputNeuron)

}

func (c *Checkerlution) LoadNeurgoCortex(filename string) {

	cortex, err := ng.NewCortexFromJSONFile(filename)
	if err != nil {
		logg.LogPanic("Error reading cortex from: %v.  Err: %v", filename, err)
	}

	c.setSensorActuatorFunctions(cortex)

	c.cortex = cortex
}

func (c *Checkerlution) setSensorActuatorFunctions(cortex *ng.Cortex) {

	sensor := cortex.FindSensor(ng.NewSensorId("SensorGameState", 0))
	sensor.SensorFunction = c.sensorFuncGameState()

	actuator := cortex.FindActuator(ng.NewActuatorId("Actuator", 0))
	actuator.ActuatorFunction = c.actuatorFunc()

}

func (c *Checkerlution) CreateHiddenLayer1Neurons(outputNeuron *ng.Neuron) []*ng.Neuron {

	cortex := c.cortex
	neurons := []*ng.Neuron{}
	layerIndex := 0.25

	for i := 0; i < 40; i++ {
		name := fmt.Sprintf("hidden-layer-%f-n-%d", layerIndex, i)
		neuron := &ng.Neuron{
			ActivationFunction: ng.EncodableTanh(),
			NodeId:             ng.NewNeuronId(name, layerIndex),
			Bias:               0.0,
		}

		// Workaround for error:
		// Cannot make outbound connection, dataChan == nil [recovered]
		neuron.Init()

		for _, sensor := range cortex.Sensors {
			sensor.ConnectOutbound(neuron)
			weights := ng.RandomWeights(sensor.VectorLength)
			neuron.ConnectInboundWeighted(sensor, weights)
		}

		neurons = append(neurons, neuron)

	}
	return neurons

}

func (c *Checkerlution) CreateHiddenLayer2Neurons(layer1Neurons []*ng.Neuron, outputNeuron *ng.Neuron) []*ng.Neuron {

	neurons := []*ng.Neuron{}
	layerIndex := 0.35

	for i := 0; i < 10; i++ {
		name := fmt.Sprintf("hidden-layer-%f-n-%d", layerIndex, i)
		neuron := &ng.Neuron{
			ActivationFunction: ng.EncodableTanh(),
			NodeId:             ng.NewNeuronId(name, layerIndex),
			Bias:               0.0,
		}

		// Workaround for error:
		// Cannot make outbound connection, dataChan == nil [recovered]
		neuron.Init()

		for _, layer1Neuron := range layer1Neurons {
			layer1Neuron.ConnectOutbound(neuron)
			weights := ng.RandomWeights(1)
			neuron.ConnectInboundWeighted(layer1Neuron, weights)
		}

		// connect to output neuron
		neuron.ConnectOutbound(outputNeuron)
		weights := ng.RandomWeights(1)
		outputNeuron.ConnectInboundWeighted(neuron, weights)

		neurons = append(neurons, neuron)

	}
	return neurons

}

func (c *Checkerlution) CreateOutputNeuron() *ng.Neuron {

	layerIndex := 0.45
	neuron := &ng.Neuron{
		ActivationFunction: ng.EncodableTanh(),
		NodeId:             ng.NewNeuronId("OutputNeuron", layerIndex),
		Bias:               0.0,
	}

	// Workaround for error:
	// Cannot make outbound connection, dataChan == nil [recovered]
	// The best fix is to just load nn from json
	neuron.Init()

	// connect sensors directly to output neuron
	for _, sensor := range c.cortex.Sensors {
		sensor.ConnectOutbound(neuron)
		weights := ng.RandomWeights(sensor.VectorLength)
		neuron.ConnectInboundWeighted(sensor, weights)
	}

	return neuron

}

func (c *Checkerlution) CreateActuator() *ng.Actuator {

	actuatorNodeId := ng.NewActuatorId("Actuator", 0.5)
	actuator := &ng.Actuator{
		NodeId:           actuatorNodeId,
		VectorLength:     1,
		ActuatorFunction: c.actuatorFunc(),
	}
	c.cortex.SetActuators([]*ng.Actuator{actuator})
	return actuator

}

func (c *Checkerlution) actuatorFunc() ng.ActuatorFunction {
	return func(outputs []float64) {
		c.latestActuatorOutput = outputs
	}
}

func (c *Checkerlution) CreateSensors() {

	sensorLayer := 0.0

	sensorGameStateNodeId := ng.NewSensorId("SensorGameState", sensorLayer)
	sensorGameState := &ng.Sensor{
		NodeId:         sensorGameStateNodeId,
		VectorLength:   32,
		SensorFunction: c.sensorFuncGameState(),
	}

	pieceDifferentialNodeId := ng.NewSensorId("SensorPieceDifferential", sensorLayer)
	sensorPieceDifferential := &ng.Sensor{
		NodeId:         pieceDifferentialNodeId,
		VectorLength:   1,
		SensorFunction: c.sensorFuncPieceDifferential(),
	}

	c.cortex.SetSensors([]*ng.Sensor{sensorGameState, sensorPieceDifferential})

}

func (c *Checkerlution) sensorFuncGameState() ng.SensorFunction {
	return func(syncCounter int) []float64 {
		if len(c.currentGameState) == 0 {
			logg.LogPanic("sensor would return invalid gamestate")
		}
		return c.currentGameState
	}
}

func (c *Checkerlution) sensorFuncPieceDifferential() ng.SensorFunction {
	return func(syncCounter int) []float64 {
		player := cbot.GetCorePlayer(c.ourTeamId)
		pieceDifferential := c.currentBoard.WeightedScore(player)
		return []float64{pieceDifferential}
	}
}

func (c *Checkerlution) sensorFuncPossibleMove() ng.SensorFunction {
	return func(syncCounter int) []float64 {
		return c.currentPossibleMove.VectorRepresentation()
	}
}

func (c Checkerlution) Stop() {

}

func (c *Checkerlution) generateBestMove(board core.Board) core.Move {

	counter := 0
	evalFunc := c.getEvaluationFunction(&counter)
	player := cbot.GetCorePlayer(c.ourTeamId)

	// with depth = 5, not working too well on first move.  when
	// there are only a few pieces on the board it seems to work,
	// but with full board .. taking a long time.

	depth := 4 // TODO: crank this up higher
	bestMove, scorePostMove := board.Minimax(player, depth, evalFunc)
	logg.LogTo("DEBUG", "scorePostMove: %v.  boards eval'd: %v", scorePostMove, counter)
	return bestMove
}

func (c *Checkerlution) getEvaluationFunction(counter *int) core.EvaluationFunction {

	evalFunc := func(currentPlayer core.Player, board core.Board) float64 {

		*counter += 1

		// convert the board into inputs for the neural net (32 elt vector)
		// taking into account whether this player is "us" or not
		gameStateVector := NewGameStateVector()
		gameStateVector.loadFromBoard(board, currentPlayer)

		// send input to the neural net
		c.currentGameState = gameStateVector
		c.currentBoard = board
		c.cortex.SyncSensors()
		c.cortex.SyncActuators()

		// return output
		return c.latestActuatorOutput[0]

	}
	return evalFunc

}
<commit_msg>experiment with two sensors that measure piece differential and no hidden neurons.<commit_after>package checkerlution

import (
	"fmt"
	"github.com/couchbaselabs/logg"
	cbot "github.com/tleyden/checkers-bot"
	core "github.com/tleyden/checkers-core"
	ng "github.com/tleyden/neurgo"
)

type OperationMode int

const (
	RUNNING_MODE = iota
	TRAINING_MODE
)

type Checkerlution struct {
	ourTeamId            cbot.TeamType
	cortex               *ng.Cortex
	currentGameState     GameStateVector
	currentBoard         core.Board
	currentPossibleMove  ValidMoveCortexInput
	latestActuatorOutput []float64
	mode                 OperationMode
	latestFitnessScore   float64
}

type CheckerlutionFlags struct {
	CheckersBotFlags cbot.CheckersBotFlags
	PopulationName   string
}

func (c *Checkerlution) Start(ourTeamId cbot.TeamType) {
	c.ourTeamId = ourTeamId
	c.CreateNeurgoCortex()
	cortex := c.cortex
	cortex.Run()

}

func (c *Checkerlution) StartWithCortex(cortex *ng.Cortex, ourTeamId cbot.TeamType) {
	c.ourTeamId = ourTeamId
	c.setSensorActuatorFunctions(cortex)
	c.cortex = cortex
	cortex.Run()
}

func (c *Checkerlution) Think(gameState cbot.GameState) (bestMove cbot.ValidMove, ok bool) {

	ok = true
	ourTeam := gameState.Teams[c.ourTeamId]
	allValidMoves := ourTeam.AllValidMoves()
	if len(allValidMoves) > 0 {

		// convert into core.board representation
		board := gameState.Export()
		logg.LogTo("DEBUG", "Before team %v move %v", c.ourTeamId.String(), board.CompactString(true))

		// generate best move (will be a core.move) -- initially, pick random
		move := c.generateBestMove(board)

		// search allValidMoves to find corresponding valid move
		found, bestValidMoveIndex := cbot.CorrespondingValidMoveIndex(move, allValidMoves)

		if !found {
			msg := "Could not find corresponding valid move: %v in %v"
			logg.LogPanic(msg, move, allValidMoves)
		} else {
			bestMove = allValidMoves[bestValidMoveIndex]
		}

		// this is just for debugging purposes
		player := cbot.GetCorePlayer(c.ourTeamId)
		boardPostMove := board.ApplyMove(player, move)
		logg.LogTo("DEBUG", "After team %v move %v", c.ourTeamId.String(), boardPostMove.CompactString(true))

		return

	} else {
		ok = false
	}

	return

}

func (c *Checkerlution) GameFinished(gameState cbot.GameState) (shouldQuit bool) {
	switch c.mode {
	case TRAINING_MODE:
		shouldQuit = true
		c.latestFitnessScore = c.calculateFitness(gameState)
	case RUNNING_MODE:
		shouldQuit = false
	}
	return
}

func (c Checkerlution) Cortex() *ng.Cortex {
	return c.cortex
}

func (c *Checkerlution) SetMode(mode OperationMode) {
	c.mode = mode
}

func (c Checkerlution) calculateFitness(gameState cbot.GameState) (fitness float64) {

	switch gameState.WinningTeam {
	case c.ourTeamId:
		logg.LogTo("CHECKERLUTION", "calculateFitness based on winning.  Turn: %v", gameState.Turn)
		fitness = 1.0
	case c.ourTeamId.Opponent():
		logg.LogTo("CHECKERLUTION", "calculateFitness based on losing.  Turn: %v", gameState.Turn)
		fitness = -2.0
	default:
		logg.LogTo("CHECKERLUTION", "calculateFitness based on draw.  Turn: %v", gameState.Turn)
		fitness = 0.0
	}

	return
}

func (c *Checkerlution) CreateNeurgoCortex() {

	uuid := ng.NewUuid()
	cortexUuid := fmt.Sprintf("cortex-%s", uuid)
	nodeId := ng.NewCortexId(cortexUuid)

	c.cortex = &ng.Cortex{
		NodeId: nodeId,
	}

	c.cortex.Init()

	c.CreateSensors()

	outputNeuron := c.CreateOutputNeuron()

	// layer1Neurons := c.CreateHiddenLayer1Neurons(outputNeuron)
	// layer2Neurons := c.CreateHiddenLayer2Neurons(layer1Neurons, outputNeuron)

	// combine all into single slice and add neurons to cortex
	neurons := []*ng.Neuron{}
	// neurons = append(neurons, layer1Neurons...)
	// neurons = append(neurons, layer2Neurons...)
	neurons = append(neurons, outputNeuron)
	c.cortex.SetNeurons(neurons)

	actuator := c.CreateActuator()

	// workaround for error
	// Cannot make outbound connection, dataChan == nil [recovered]
	c.cortex.Init()

	outputNeuron.ConnectOutbound(actuator)
	actuator.ConnectInbound(outputNeuron)

}

func (c *Checkerlution) LoadNeurgoCortex(filename string) {

	cortex, err := ng.NewCortexFromJSONFile(filename)
	if err != nil {
		logg.LogPanic("Error reading cortex from: %v.  Err: %v", filename, err)
	}

	c.setSensorActuatorFunctions(cortex)

	c.cortex = cortex
}

func (c *Checkerlution) setSensorActuatorFunctions(cortex *ng.Cortex) {

	// DO we still need this for anything??

	// sensor := cortex.FindSensor(ng.NewSensorId("SensorGameState", 0))
	// sensor.SensorFunction = c.sensorFuncGameState()

	actuator := cortex.FindActuator(ng.NewActuatorId("Actuator", 0))
	actuator.ActuatorFunction = c.actuatorFunc()

}

func (c *Checkerlution) CreateHiddenLayer1Neurons(outputNeuron *ng.Neuron) []*ng.Neuron {

	cortex := c.cortex
	neurons := []*ng.Neuron{}
	layerIndex := 0.25

	for i := 0; i < 40; i++ {
		name := fmt.Sprintf("hidden-layer-%f-n-%d", layerIndex, i)
		neuron := &ng.Neuron{
			ActivationFunction: ng.EncodableTanh(),
			NodeId:             ng.NewNeuronId(name, layerIndex),
			Bias:               0.0,
		}

		// Workaround for error:
		// Cannot make outbound connection, dataChan == nil [recovered]
		neuron.Init()

		for _, sensor := range cortex.Sensors {
			sensor.ConnectOutbound(neuron)
			weights := ng.RandomWeights(sensor.VectorLength)
			neuron.ConnectInboundWeighted(sensor, weights)
		}

		neurons = append(neurons, neuron)

	}
	return neurons

}

func (c *Checkerlution) CreateHiddenLayer2Neurons(layer1Neurons []*ng.Neuron, outputNeuron *ng.Neuron) []*ng.Neuron {

	neurons := []*ng.Neuron{}
	layerIndex := 0.35

	for i := 0; i < 10; i++ {
		name := fmt.Sprintf("hidden-layer-%f-n-%d", layerIndex, i)
		neuron := &ng.Neuron{
			ActivationFunction: ng.EncodableTanh(),
			NodeId:             ng.NewNeuronId(name, layerIndex),
			Bias:               0.0,
		}

		// Workaround for error:
		// Cannot make outbound connection, dataChan == nil [recovered]
		neuron.Init()

		for _, layer1Neuron := range layer1Neurons {
			layer1Neuron.ConnectOutbound(neuron)
			weights := ng.RandomWeights(1)
			neuron.ConnectInboundWeighted(layer1Neuron, weights)
		}

		// connect to output neuron
		neuron.ConnectOutbound(outputNeuron)
		weights := ng.RandomWeights(1)
		outputNeuron.ConnectInboundWeighted(neuron, weights)

		neurons = append(neurons, neuron)

	}
	return neurons

}

func (c *Checkerlution) CreateOutputNeuron() *ng.Neuron {

	layerIndex := 0.45
	neuron := &ng.Neuron{
		ActivationFunction: ng.EncodableTanh(),
		NodeId:             ng.NewNeuronId("OutputNeuron", layerIndex),
		Bias:               0.0,
	}

	// Workaround for error:
	// Cannot make outbound connection, dataChan == nil [recovered]
	// The best fix is to just load nn from json
	neuron.Init()

	// connect sensors directly to output neuron
	for _, sensor := range c.cortex.Sensors {
		sensor.ConnectOutbound(neuron)
		weights := ng.RandomWeights(sensor.VectorLength)
		neuron.ConnectInboundWeighted(sensor, weights)
	}

	return neuron

}

func (c *Checkerlution) CreateActuator() *ng.Actuator {

	actuatorNodeId := ng.NewActuatorId("Actuator", 0.5)
	actuator := &ng.Actuator{
		NodeId:           actuatorNodeId,
		VectorLength:     1,
		ActuatorFunction: c.actuatorFunc(),
	}
	c.cortex.SetActuators([]*ng.Actuator{actuator})
	return actuator

}

func (c *Checkerlution) actuatorFunc() ng.ActuatorFunction {
	return func(outputs []float64) {
		c.latestActuatorOutput = outputs
	}
}

func (c *Checkerlution) CreateSensors() {

	sensorLayer := 0.0
	/*
		sensorGameStateNodeId := ng.NewSensorId("SensorGameState", sensorLayer)
		sensorGameState := &ng.Sensor{
			NodeId:         sensorGameStateNodeId,
			VectorLength:   32,
			SensorFunction: c.sensorFuncGameState(),
		}
	*/

	pieceDifferentialNodeId := ng.NewSensorId("SensorPieceDifferential", sensorLayer)
	sensorPieceDifferential := &ng.Sensor{
		NodeId:         pieceDifferentialNodeId,
		VectorLength:   1,
		SensorFunction: c.sensorFuncPieceDifferential(),
	}

	kingsDifferentialNodeId := ng.NewSensorId("SensorKingsDifferential", sensorLayer)
	sensorKingsDifferential := &ng.Sensor{
		NodeId:         kingsDifferentialNodeId,
		VectorLength:   1,
		SensorFunction: c.sensorFuncKingsDifferential(),
	}

	sensors := []*ng.Sensor{
		// sensorGameState,
		sensorPieceDifferential,
		sensorKingsDifferential,
	}
	c.cortex.SetSensors(sensors)

}

func (c *Checkerlution) sensorFuncGameState() ng.SensorFunction {
	return func(syncCounter int) []float64 {
		if len(c.currentGameState) == 0 {
			logg.LogPanic("sensor would return invalid gamestate")
		}
		return c.currentGameState
	}
}

func (c *Checkerlution) sensorFuncPieceDifferential() ng.SensorFunction {
	return func(syncCounter int) []float64 {
		player := cbot.GetCorePlayer(c.ourTeamId)
		pieceDifferential := c.currentBoard.WeightedScorePiecesOnly(player)
		return []float64{pieceDifferential}
	}
}

func (c *Checkerlution) sensorFuncKingsDifferential() ng.SensorFunction {
	return func(syncCounter int) []float64 {
		player := cbot.GetCorePlayer(c.ourTeamId)
		pieceDifferential := c.currentBoard.WeightedScoreKingsOnly(player)
		return []float64{pieceDifferential}
	}
}

func (c *Checkerlution) sensorFuncPossibleMove() ng.SensorFunction {
	return func(syncCounter int) []float64 {
		return c.currentPossibleMove.VectorRepresentation()
	}
}

func (c Checkerlution) Stop() {

}

func (c *Checkerlution) generateBestMove(board core.Board) core.Move {

	counter := 0
	evalFunc := c.getEvaluationFunction(&counter)
	player := cbot.GetCorePlayer(c.ourTeamId)

	// with depth = 5, not working too well on first move.  when
	// there are only a few pieces on the board it seems to work,
	// but with full board .. taking a long time.

	depth := 4 // TODO: crank this up higher
	bestMove, scorePostMove := board.Minimax(player, depth, evalFunc)
	logg.LogTo("DEBUG", "scorePostMove: %v.  boards eval'd: %v", scorePostMove, counter)
	return bestMove
}

func (c *Checkerlution) getEvaluationFunction(counter *int) core.EvaluationFunction {

	evalFunc := func(currentPlayer core.Player, board core.Board) float64 {

		*counter += 1

		// convert the board into inputs for the neural net (32 elt vector)
		// taking into account whether this player is "us" or not
		gameStateVector := NewGameStateVector()
		gameStateVector.loadFromBoard(board, currentPlayer)

		// send input to the neural net
		c.currentGameState = gameStateVector
		c.currentBoard = board
		c.cortex.SyncSensors()
		c.cortex.SyncActuators()

		// return output
		return c.latestActuatorOutput[0]

	}
	return evalFunc

}
<|endoftext|>


<commit_before>package chyle

import (
	pkgDecorators "github.com/antham/chyle/chyle/decorators"
	pkgExtractors "github.com/antham/chyle/chyle/extractors"
	pkgMatchers "github.com/antham/chyle/chyle/matchers"
	pkgSenders "github.com/antham/chyle/chyle/senders"

	"srcd.works/go-git.v4/plumbing/object"
)

// process represents all configuration operations defined
// needed to create a changelog
type process struct {
	matchers   *[]pkgMatchers.Matcher
	extractors *[]pkgExtractors.Extracter
	decorators *map[string][]pkgDecorators.Decorater
	senders    *[]pkgSenders.Sender
}

// buildProcess creates process entity from defined configuration
func buildProcess() *process {
	matchers := &[]pkgMatchers.Matcher{}
	extractors := &[]pkgExtractors.Extracter{}
	decorators := &map[string][]pkgDecorators.Decorater{}
	senders := &[]pkgSenders.Sender{}

	if chyleConfig.FEATURES.HASMATCHERS {
		matchers = pkgMatchers.CreateMatchers(chyleConfig.MATCHERS)
	}

	if chyleConfig.FEATURES.HASEXTRACTORS {
		extractors = pkgExtractors.CreateExtractors(chyleConfig.EXTRACTORS)
	}

	if chyleConfig.FEATURES.HASDECORATORS {
		decorators = pkgDecorators.CreateDecorators(map[string]bool{"jiraIssueDecorator": chyleConfig.FEATURES.HASJIRAISSUEDECORATOR, "githubIssueDecorator": chyleConfig.FEATURES.HASGITHUBISSUEDECORATOR, "envDecorator": chyleConfig.FEATURES.HASENVDECORATOR}, chyleConfig.DECORATORS)
	}

	if chyleConfig.FEATURES.HASSENDERS {
		senders = pkgSenders.CreateSenders(map[string]bool{"githubReleaseSender": chyleConfig.FEATURES.HASGITHUBRELEASESENDER, "stdoutSender": chyleConfig.FEATURES.HASSTDOUTSENDER}, chyleConfig.SENDERS)
	}

	return &process{
		matchers,
		extractors,
		decorators,
		senders,
	}
}

// proceed extracts datas from a set of commits
func proceed(process *process, commits *[]object.Commit) error {
	changelog := pkgExtractors.Extract(process.extractors, pkgMatchers.Filter(process.matchers, commits))

	changelog, err := pkgDecorators.Decorate(process.decorators, changelog)

	if err != nil {
		return err
	}

	return pkgSenders.Send(process.senders, changelog)
}
<commit_msg>ref(chyle/process) : rewrite<commit_after>package chyle

import (
	"github.com/antham/chyle/chyle/decorators"
	"github.com/antham/chyle/chyle/extractors"
	"github.com/antham/chyle/chyle/matchers"
	"github.com/antham/chyle/chyle/senders"

	"srcd.works/go-git.v4/plumbing/object"
)

// process represents all configuration operations defined
// needed to create a changelog
type process struct {
	matchers   *[]matchers.Matcher
	extractors *[]extractors.Extracter
	decorators *map[string][]decorators.Decorater
	senders    *[]senders.Sender
}

// buildProcess creates process entity from defined configuration
func buildProcess() *process {
	p := &process{
		&[]matchers.Matcher{},
		&[]extractors.Extracter{},
		&map[string][]decorators.Decorater{},
		&[]senders.Sender{},
	}

	if chyleConfig.FEATURES.HASMATCHERS {
		p.matchers = matchers.CreateMatchers(chyleConfig.MATCHERS)
	}

	if chyleConfig.FEATURES.HASEXTRACTORS {
		p.extractors = extractors.CreateExtractors(chyleConfig.EXTRACTORS)
	}

	if chyleConfig.FEATURES.HASDECORATORS {
		p.decorators = decorators.CreateDecorators(map[string]bool{"jiraIssueDecorator": chyleConfig.FEATURES.HASJIRAISSUEDECORATOR, "githubIssueDecorator": chyleConfig.FEATURES.HASGITHUBISSUEDECORATOR, "envDecorator": chyleConfig.FEATURES.HASENVDECORATOR}, chyleConfig.DECORATORS)
	}

	if chyleConfig.FEATURES.HASSENDERS {
		p.senders = senders.CreateSenders(map[string]bool{"githubReleaseSender": chyleConfig.FEATURES.HASGITHUBRELEASESENDER, "stdoutSender": chyleConfig.FEATURES.HASSTDOUTSENDER}, chyleConfig.SENDERS)
	}

	return p
}

// proceed extracts datas from a set of commits
func proceed(process *process, commits *[]object.Commit) error {
	changelog := extractors.Extract(process.extractors, matchers.Filter(process.matchers, commits))

	changelog, err := decorators.Decorate(process.decorators, changelog)

	if err != nil {
		return err
	}

	return senders.Send(process.senders, changelog)
}
<|endoftext|>


<commit_before>package main

import (
	"bufio"
	"flag"
	"fmt"
	"github.com/golang/glog"
	"github.com/heidi-ann/hydra/api/interactive"
	"github.com/heidi-ann/hydra/api/rest"
	"github.com/heidi-ann/hydra/config"
	"github.com/heidi-ann/hydra/msgs"
	"github.com/heidi-ann/hydra/test"
	"io"
	"net"
	"os"
	"time"
)

type API interface {
	Next() (string, bool)
	Return(string)
}

var config_file = flag.String("config", "example.conf", "Client configuration file")
var auto_file = flag.String("auto", "../test/workload.conf", "If workload is automatically generated, configure file for workload")
var stat_file = flag.String("stat", "latency.csv", "File to write stats to")
var mode = flag.String("mode", "interactive", "interactive, rest or test")
var id = flag.Int("id", -1, "ID of client (must be unique)")

func connect(addrs []string, tries int) (net.Conn, error) {
	var conn net.Conn
	var err error

	for i := range addrs {
		for t := tries; t > 0; t-- {
			conn, err = net.Dial("tcp", addrs[i])

			// if successful
			if err == nil {
				glog.Infof("Connected to %s", addrs[i])
				return conn, err
			}

			//if unsuccessful
			glog.Warning(err)
			time.Sleep(2 * time.Second)
		}
	}

	return conn, err
}

func main() {
	// set up logging
	flag.Parse()
	defer glog.Flush()

	// parse config files
	conf := config.Parse(*config_file)
	// TODO: find a better way to handle required flags
	if *id == -1 {
		glog.Fatal("ID must be provided")
	}

	glog.Info("Starting up client ", *id)
	defer glog.Info("Shutting down client ", *id)

	// set up stats collection
	filename := *stat_file
	glog.Info("Opening file: ", filename)
	file, err := os.OpenFile(filename, os.O_RDWR|os.O_CREATE|os.O_APPEND, 0777)
	if err != nil {
		glog.Fatal(err)
	}
	stats := bufio.NewWriter(file)
	defer stats.Flush()

	// set up request id
	// TODO: write this value to disk
	requestID := 0

	// connecting to server
	conn, err := connect(conf.Addresses.Address, 3)
	if err != nil {
		glog.Fatal(err)
	}

	// setup API
	var ioapi API
	switch *mode {
	case "interactive":
		ioapi = interactive.Create()
	case "test":
		ioapi = test.Generate(test.ParseAuto(*auto_file))
	case "rest":
		ioapi = rest.Create()
	default:
		glog.Fatal("Invalid mode: ", mode)
	}

	// setup network reader
	net_reader := bufio.NewReader(conn)

	glog.Info("Client is ready to process incoming requests")
	for {

		// get next command
		text, ok := ioapi.Next()
		if !ok {
			glog.Fatal("No more commands")
		}
		glog.Info("API produced ", text)

		// encode as request
		req := msgs.ClientRequest{
			*id, requestID, text}
		requestID++
		b, err := msgs.Marshal(req)
		if err != nil {
			glog.Fatal(err)
		}
		glog.Info(string(b))

		// send to server
		startTime := time.Now()
		_, err = conn.Write(b)
		_, err = conn.Write([]byte("\n"))
		if err != nil {
			if err == io.EOF {
				continue
			}
			glog.Warning(err)
			// reconnecting to server
			conn, err = connect(conf.Addresses.Address, 3)
			if err != nil {
				glog.Fatal(err)
			}
		}

		glog.Info("Sent ", text)

		// read response
		r, err := net_reader.ReadBytes('\n')
		if err != nil {
			if err == io.EOF {
				continue
			}
			glog.Warning(err)
			// reconnecting to server
			conn, err = connect(conf.Addresses.Address, 3)
			if err != nil {
				glog.Fatal(err)
			}
		}
		reply := new(msgs.ClientResponse)
		msgs.Unmarshal(r, reply)

		// write to latency to log
		str := fmt.Sprintf("%d\n", time.Since(startTime).Nanoseconds())
		n, err := stats.WriteString(str)
		if err != nil {
			glog.Fatal(err)
		}
		glog.Warningf("Written %d bytes to log", n)
		_ = stats.Flush()

		// writing result to user
		// time.Since(startTime)
		ioapi.Return(reply.Response)

	}

}
<commit_msg>client reply when TPC connection fails<commit_after>package main

import (
	"bufio"
	"flag"
	"fmt"
	"github.com/golang/glog"
	"github.com/heidi-ann/hydra/api/interactive"
	"github.com/heidi-ann/hydra/api/rest"
	"github.com/heidi-ann/hydra/config"
	"github.com/heidi-ann/hydra/msgs"
	"github.com/heidi-ann/hydra/test"
	"io"
	"net"
	"os"
	"time"
)

type API interface {
	Next() (string, bool)
	Return(string)
}

var config_file = flag.String("config", "example.conf", "Client configuration file")
var auto_file = flag.String("auto", "../test/workload.conf", "If workload is automatically generated, configure file for workload")
var stat_file = flag.String("stat", "latency.csv", "File to write stats to")
var mode = flag.String("mode", "interactive", "interactive, rest or test")
var id = flag.Int("id", -1, "ID of client (must be unique)")

func connect(addrs []string, tries int) (net.Conn, error) {
	var conn net.Conn
	var err error

	for i := range addrs {
		for t := tries; t > 0; t-- {
			conn, err = net.Dial("tcp", addrs[i])

			// if successful
			if err == nil {
				glog.Infof("Connected to %s", addrs[i])
				return conn, err
			}

			//if unsuccessful
			glog.Warning(err)
			time.Sleep(2 * time.Second)
		}
	}

	return conn, err
}

// send bytes and wait for reply, return bytes returned if succussful or error otherwise
func dispatcher(b []byte, conn net.Conn, r *bufio.Reader) ([]byte, error) {
	// send request
	_, err := conn.Write(b)
	_, err = conn.Write([]byte("\n"))
	if err != nil && err != io.EOF {
		glog.Warning(err)
		return nil, err
	}

	glog.Info("Sent")

	// read response
	reply, err := r.ReadBytes('\n')
	if err != nil && err != io.EOF {
		glog.Warning(err)
		return nil, err
	}

	// success, return reply
	return reply, nil
}

func main() {
	// set up logging
	flag.Parse()
	defer glog.Flush()

	// parse config files
	conf := config.Parse(*config_file)
	// TODO: find a better way to handle required flags
	if *id == -1 {
		glog.Fatal("ID must be provided")
	}

	glog.Info("Starting up client ", *id)
	defer glog.Info("Shutting down client ", *id)

	// set up stats collection
	filename := *stat_file
	glog.Info("Opening file: ", filename)
	file, err := os.OpenFile(filename, os.O_RDWR|os.O_CREATE|os.O_APPEND, 0777)
	if err != nil {
		glog.Fatal(err)
	}
	stats := bufio.NewWriter(file)
	defer stats.Flush()

	// set up request id
	// TODO: write this value to disk
	requestID := 0

	// connecting to server
	conn, err := connect(conf.Addresses.Address, 3)
	if err != nil {
		glog.Fatal(err)
	}
	rd := bufio.NewReader(conn)

	// setup API
	var ioapi API
	switch *mode {
	case "interactive":
		ioapi = interactive.Create()
	case "test":
		ioapi = test.Generate(test.ParseAuto(*auto_file))
	case "rest":
		ioapi = rest.Create()
	default:
		glog.Fatal("Invalid mode: ", mode)
	}

	glog.Info("Client is ready to process incoming requests")
	for {

		// get next command
		text, ok := ioapi.Next()
		if !ok {
			glog.Fatal("No more commands")
		}
		glog.Info("API produced ", text)

		// encode as request
		req := msgs.ClientRequest{
			*id, requestID, text}
		requestID++
		b, err := msgs.Marshal(req)
		if err != nil {
			glog.Fatal(err)
		}
		glog.Info(string(b))

		startTime := time.Now()

		// dispatch request until successfull
		var replyBytes []byte
		for {
			replyBytes, err = dispatcher(b, conn, rd)
			if err == nil {
				break
			}
			conn, err = connect(conf.Addresses.Address, 3)
			if err != nil {
				glog.Fatal(err)
			}
			rd = bufio.NewReader(conn)

		}

		//handle reply
		reply := new(msgs.ClientResponse)
		msgs.Unmarshal(replyBytes, reply)

		// write to latency to log
		str := fmt.Sprintf("%d\n", time.Since(startTime).Nanoseconds())
		n, err := stats.WriteString(str)
		if err != nil {
			glog.Fatal(err)
		}
		glog.Warningf("Written %d bytes to log", n)
		_ = stats.Flush()

		// writing result to user
		// time.Since(startTime)
		ioapi.Return(reply.Response)

	}

}
<|endoftext|>


<commit_before>package main

import (
	"bufio"
	"bytes"
	"context"
	"errors"
	"fmt"
	"io/ioutil"
	"net/http"
	"net/url"
	"os"
	"strings"
	"time"

	kingpin "gopkg.in/alecthomas/kingpin.v2"

	"github.com/ShowMax/go-fqdn"
	"github.com/go-kit/kit/log"
	"github.com/go-kit/kit/log/level"
	"github.com/prometheus/common/promlog"
	"github.com/prometheus/common/promlog/flag"
	"github.com/robustperception/pushprox/util"
)

var (
	myFqdn   = kingpin.Flag("fqdn", "FQDN to register with").Default(fqdn.Get()).String()
	proxyURL = kingpin.Flag("proxy-url", "Push proxy to talk to.").Required().String()
)

type Coordinator struct {
	logger log.Logger
}

func (c *Coordinator) doScrape(request *http.Request, client *http.Client) {
	logger := log.With(c.logger, "scrape_id", request.Header.Get("id"))
	ctx, _ := context.WithTimeout(request.Context(), util.GetScrapeTimeout(request.Header))
	request = request.WithContext(ctx)
	// We cannot handle https requests at the proxy, as we would only
	// see a CONNECT, so use a URL parameter to trigger it.
	params := request.URL.Query()
	if params.Get("_scheme") == "https" {
		request.URL.Scheme = "https"
		params.Del("_scheme")
		request.URL.RawQuery = params.Encode()
	}

	scrapeResp, err := client.Do(request)
	if err != nil {
		msg := fmt.Sprintf("Failed to scrape %s: %s", request.URL.String(), err)
		level.Warn(logger).Log("msg", "Failed to scrape", "Request URL", request.URL.String(), "err", err)
		resp := &http.Response{
			StatusCode: 500,
			Header:     http.Header{},
			Body:       ioutil.NopCloser(strings.NewReader(msg)),
		}
		err = c.doPush(resp, request, client)
		if err != nil {
			level.Warn(logger).Log("msg", "Failed to push failed scrape response:", "err", err)
			return
		}
		level.Info(logger).Log("msg", "Pushed failed scrape response")
		return
	}
	level.Info(logger).Log("msg", "Retrieved scrape response")
	err = c.doPush(scrapeResp, request, client)
	if err != nil {
		level.Warn(logger).Log("msg", "Failed to push scrape response:", "err", err)
		return
	}
	level.Info(logger).Log("msg", "Pushed scrape result")
}

// Report the result of the scrape back up to the proxy.
func (c *Coordinator) doPush(resp *http.Response, origRequest *http.Request, client *http.Client) error {
	resp.Header.Set("id", origRequest.Header.Get("id")) // Link the request and response
	// Remaining scrape deadline.
	deadline, _ := origRequest.Context().Deadline()
	resp.Header.Set("X-Prometheus-Scrape-Timeout", fmt.Sprintf("%f", float64(time.Until(deadline))/1e9))

	base, err := url.Parse(*proxyURL)
	if err != nil {
		return err
	}
	u, err := url.Parse("push")
	if err != nil {
		return err
	}
	url := base.ResolveReference(u)

	buf := &bytes.Buffer{}
	resp.Write(buf)
	request := &http.Request{
		Method:        "POST",
		URL:           url,
		Body:          ioutil.NopCloser(buf),
		ContentLength: int64(buf.Len()),
	}
	request = request.WithContext(origRequest.Context())
	_, err = client.Do(request)
	if err != nil {
		return err
	}
	return nil
}

func loop(c Coordinator) error {
	client := &http.Client{}
	base, err := url.Parse(*proxyURL)
	if err != nil {
		level.Error(c.logger).Log("msg", "Error parsing url:", "err", err)
		return errors.New("error parsing url")
	}
	u, err := url.Parse("poll")
	if err != nil {
		level.Error(c.logger).Log("msg", "Error parsing url:", "err", err)
		return errors.New("error parsing url poll")
	}
	url := base.ResolveReference(u)
	resp, err := client.Post(url.String(), "", strings.NewReader(*myFqdn))
	if err != nil {
		level.Error(c.logger).Log("msg", "Error polling:", "err", err)
		return errors.New("error polling")
	}
	defer resp.Body.Close()
	request, err := http.ReadRequest(bufio.NewReader(resp.Body))
	if err != nil {
		level.Error(c.logger).Log("msg", "Error reading request:", "err", err)
		return errors.New("error reading request")
	}
	level.Info(c.logger).Log("msg", "Got scrape request", "scrape_id", request.Header.Get("id"), "url", request.URL)

	request.RequestURI = ""

	go c.doScrape(request, client)

	return nil
}

func main() {
	allowedLevel := promlog.AllowedLevel{}
	flag.AddFlags(kingpin.CommandLine, &allowedLevel)
	kingpin.HelpFlag.Short('h')
	kingpin.Parse()
	logger := promlog.New(allowedLevel)
	coordinator := Coordinator{logger: logger}
	if *proxyURL == "" {
		level.Error(coordinator.logger).Log("msg", "-proxy-url flag must be specified.")
		os.Exit(1)
	}
	level.Info(coordinator.logger).Log("msg", "URL and FQDN info", "proxy_url", *proxyURL, "Using FQDN of", *myFqdn)
	for {
		err := loop(coordinator)
		if err != nil {
			time.Sleep(time.Second) // Don't pound the server. TODO: Randomised exponential backoff.
		}
	}
}
<commit_msg> Added TLS support in Client (#27)<commit_after>package main

import (
	"bufio"
	"bytes"
	"context"
	"crypto/tls"
	"crypto/x509"
	"errors"
	"fmt"
	"io/ioutil"
	"net/http"
	"net/url"
	"os"
	"strings"
	"time"

	kingpin "gopkg.in/alecthomas/kingpin.v2"

	"github.com/ShowMax/go-fqdn"
	"github.com/go-kit/kit/log"
	"github.com/go-kit/kit/log/level"
	"github.com/prometheus/common/promlog"
	"github.com/prometheus/common/promlog/flag"
	"github.com/robustperception/pushprox/util"
)

var (
	myFqdn     = kingpin.Flag("fqdn", "FQDN to register with").Default(fqdn.Get()).String()
	proxyURL   = kingpin.Flag("proxy-url", "Push proxy to talk to.").Required().String()
	caCertFile = kingpin.Flag("tls.cacert", "<file> CA certificate to verify peer against").String()
	tlsCert    = kingpin.Flag("tls.cert", "<cert> Client certificate file").String()
	tlsKey     = kingpin.Flag("tls.key", "<key> Private key file").String()
)

type Coordinator struct {
	logger log.Logger
}

func (c *Coordinator) doScrape(request *http.Request, client *http.Client) {
	logger := log.With(c.logger, "scrape_id", request.Header.Get("id"))
	ctx, _ := context.WithTimeout(request.Context(), util.GetScrapeTimeout(request.Header))
	request = request.WithContext(ctx)
	// We cannot handle https requests at the proxy, as we would only
	// see a CONNECT, so use a URL parameter to trigger it.
	params := request.URL.Query()
	if params.Get("_scheme") == "https" {
		request.URL.Scheme = "https"
		params.Del("_scheme")
		request.URL.RawQuery = params.Encode()
	}

	scrapeResp, err := client.Do(request)
	if err != nil {
		msg := fmt.Sprintf("Failed to scrape %s: %s", request.URL.String(), err)
		level.Warn(logger).Log("msg", "Failed to scrape", "Request URL", request.URL.String(), "err", err)
		resp := &http.Response{
			StatusCode: 500,
			Header:     http.Header{},
			Body:       ioutil.NopCloser(strings.NewReader(msg)),
		}
		err = c.doPush(resp, request, client)
		if err != nil {
			level.Warn(logger).Log("msg", "Failed to push failed scrape response:", "err", err)
			return
		}
		level.Info(logger).Log("msg", "Pushed failed scrape response")
		return
	}
	level.Info(logger).Log("msg", "Retrieved scrape response")
	err = c.doPush(scrapeResp, request, client)
	if err != nil {
		level.Warn(logger).Log("msg", "Failed to push scrape response:", "err", err)
		return
	}
	level.Info(logger).Log("msg", "Pushed scrape result")
}

// Report the result of the scrape back up to the proxy.
func (c *Coordinator) doPush(resp *http.Response, origRequest *http.Request, client *http.Client) error {
	resp.Header.Set("id", origRequest.Header.Get("id")) // Link the request and response
	// Remaining scrape deadline.
	deadline, _ := origRequest.Context().Deadline()
	resp.Header.Set("X-Prometheus-Scrape-Timeout", fmt.Sprintf("%f", float64(time.Until(deadline))/1e9))

	base, err := url.Parse(*proxyURL)
	if err != nil {
		return err
	}
	u, err := url.Parse("push")
	if err != nil {
		return err
	}
	url := base.ResolveReference(u)

	buf := &bytes.Buffer{}
	resp.Write(buf)
	request := &http.Request{
		Method:        "POST",
		URL:           url,
		Body:          ioutil.NopCloser(buf),
		ContentLength: int64(buf.Len()),
	}
	request = request.WithContext(origRequest.Context())
	_, err = client.Do(request)
	if err != nil {
		return err
	}
	return nil
}

func loop(c Coordinator, t *http.Transport) error {
	client := &http.Client{Transport: t}
	base, err := url.Parse(*proxyURL)
	if err != nil {
		level.Error(c.logger).Log("msg", "Error parsing url:", "err", err)
		return errors.New("error parsing url")
	}
	u, err := url.Parse("poll")
	if err != nil {
		level.Error(c.logger).Log("msg", "Error parsing url:", "err", err)
		return errors.New("error parsing url poll")
	}
	url := base.ResolveReference(u)
	resp, err := client.Post(url.String(), "", strings.NewReader(*myFqdn))
	if err != nil {
		level.Error(c.logger).Log("msg", "Error polling:", "err", err)
		return errors.New("error polling")
	}
	defer resp.Body.Close()

	request, err := http.ReadRequest(bufio.NewReader(resp.Body))
	if err != nil {
		level.Error(c.logger).Log("msg", "Error reading request:", "err", err)
		return errors.New("error reading request")
	}
	level.Info(c.logger).Log("msg", "Got scrape request", "scrape_id", request.Header.Get("id"), "url", request.URL)

	request.RequestURI = ""

	go c.doScrape(request, client)

	return nil
}

func main() {
	allowedLevel := promlog.AllowedLevel{}
	flag.AddFlags(kingpin.CommandLine, &allowedLevel)
	kingpin.HelpFlag.Short('h')
	kingpin.Parse()
	logger := promlog.New(allowedLevel)
	coordinator := Coordinator{logger: logger}
	if *proxyURL == "" {
		level.Error(coordinator.logger).Log("msg", "-proxy-url flag must be specified.")
		os.Exit(1)
	}
	level.Info(coordinator.logger).Log("msg", "URL and FQDN info", "proxy_url", *proxyURL, "Using FQDN of", *myFqdn)

	tlsConfig := &tls.Config{}
	if *tlsCert != "" {
		cert, err := tls.LoadX509KeyPair(*tlsCert, *tlsKey)
		if err != nil {
			level.Error(coordinator.logger).Log("msg", "Certificate or Key is invalid", "err", err)
			os.Exit(1)
		}

		// Setup HTTPS client
		tlsConfig.Certificates = []tls.Certificate{cert}

		tlsConfig.BuildNameToCertificate()
	}

	if *caCertFile != "" {
		caCert, err := ioutil.ReadFile(*caCertFile)
		if err != nil {
			level.Error(coordinator.logger).Log("msg", "Not able to read cacert file", "err", err)
			os.Exit(1)
		}
		caCertPool := x509.NewCertPool()
		caCertPool.AppendCertsFromPEM(caCert)

		tlsConfig.RootCAs = caCertPool
	}

	transport := &http.Transport{TLSClientConfig: tlsConfig}

	for {
		err := loop(coordinator, transport)
		if err != nil {
			time.Sleep(time.Second) // Don't pound the server. TODO: Randomised exponential backoff.
		}
	}
}
<|endoftext|>