<commit_before>package tiff

import (
	"bytes"
	"encoding/binary"
	"fmt"
	"math"
	"math/big"
	"reflect"
)

/* FieldTypeRepr */
// These functions provide string representations of values based on field types.
func reprByte(in []byte, bo binary.ByteOrder) string   { return fmt.Sprintf("%d", in[0]) }
func reprSByte(in []byte, bo binary.ByteOrder) string  { return fmt.Sprintf("%d", int8(in[0])) }
func reprASCII(in []byte, bo binary.ByteOrder) string  { return string(in) }
func reprShort(in []byte, bo binary.ByteOrder) string  { return fmt.Sprintf("%d", bo.Uint16(in)) }
func reprSShort(in []byte, bo binary.ByteOrder) string { return fmt.Sprintf("%d", int16(bo.Uint16(in))) }
func reprLong(in []byte, bo binary.ByteOrder) string   { return fmt.Sprintf("%d", bo.Uint32(in)) }
func reprSLong(in []byte, bo binary.ByteOrder) string  { return fmt.Sprintf("%d", int32(bo.Uint32(in))) }
func reprRational(in []byte, bo binary.ByteOrder) string {
	// Print the representation directly to prevent panics from divide by
	// zero errors when using big.NewRat().
	return fmt.Sprintf("%d/%d", int64(bo.Uint32(in)), int64(bo.Uint32(in[4:])))
}
func reprSRational(in []byte, bo binary.ByteOrder) string {
	// Print the representation directly to prevent panics from divide by
	// zero errors when using big.NewRat()
	return fmt.Sprintf("%d/%d", int64(int32(bo.Uint32(in))), int64(int32(bo.Uint32(in[4:]))))
}
func reprFloat(in []byte, bo binary.ByteOrder) string {
	return fmt.Sprintf("%f", math.Float32frombits(bo.Uint32(in)))
}
func reprDouble(in []byte, bo binary.ByteOrder) string {
	return fmt.Sprintf("%f", math.Float64frombits(bo.Uint64(in)))
}

/* FieldTypeValuer */
func rvalByte(in []byte, bo binary.ByteOrder) reflect.Value  { return reflect.ValueOf(in[0]) }
func rvalSByte(in []byte, bo binary.ByteOrder) reflect.Value { return reflect.ValueOf(int8(in[0])) }
func rvalASCII(in []byte, bo binary.ByteOrder) reflect.Value {
	return reflect.ValueOf(string(bytes.TrimRight(in, "\x00")))
}
func rvalShort(in []byte, bo binary.ByteOrder) reflect.Value { return reflect.ValueOf(bo.Uint16(in)) }
func rvalSShort(in []byte, bo binary.ByteOrder) reflect.Value {
	return reflect.ValueOf(int16(bo.Uint16(in)))
}
func rvalLong(in []byte, bo binary.ByteOrder) reflect.Value { return reflect.ValueOf(bo.Uint32(in)) }
func rvalSLong(in []byte, bo binary.ByteOrder) reflect.Value {
	return reflect.ValueOf(int32(bo.Uint32(in)))
}
func rvalRational(in []byte, bo binary.ByteOrder) reflect.Value {
	denom := int64(bo.Uint32(in[4:]))
	if denom == 0 {
		// Prevent panics due to poorly written Rational fields with a
		// denominator of 0.
		return reflect.New(reflect.TypeOf(big.Rat{}))
	}
	numer := int64(bo.Uint32(in))
	return reflect.ValueOf(big.NewRat(numer, denom))
}
func rvalSRational(in []byte, bo binary.ByteOrder) reflect.Value {
	denom := int64(int32(bo.Uint32(in[4:])))
	if denom == 0 {
		// Prevent panics due to poorly written Rational fields with a
		// denominator of 0.  Their usable value would likely be 0.
		return reflect.New(reflect.TypeOf(big.Rat{}))
	}
	numer := int64(int32(bo.Uint32(in)))
	return reflect.ValueOf(big.NewRat(numer, denom))
}
func rvalFloat(in []byte, bo binary.ByteOrder) reflect.Value {
	return reflect.ValueOf(math.Float32frombits(bo.Uint32(in)))
}
func rvalDouble(in []byte, bo binary.ByteOrder) reflect.Value {
	return reflect.ValueOf(math.Float64frombits(bo.Uint64(in)))
}

/* reflect.Type */
var (
	typByte   = reflect.TypeOf(byte(0))         // BYTE, UNDEFINED
	typString = reflect.TypeOf(string(""))      // ASCII
	typU16    = reflect.TypeOf(uint16(0))       // SHORT
	typU32    = reflect.TypeOf(uint32(0))       // LONG, IFD
	typBigRat = reflect.TypeOf((*big.Rat)(nil)) // RATIONAL, SRATIONAL
	typI8     = reflect.TypeOf(int8(0))         // SBYTE
	typI16    = reflect.TypeOf(int16(0))        // SSHORT
	typI32    = reflect.TypeOf(int32(0))        // SLONG
	typF32    = reflect.TypeOf(float32(0))      // FLOAT
	typF64    = reflect.TypeOf(float64(0))      // DOUBLE
)

/* Field type definitions
From [TIFF6]:
	1-12: Field types 1 - 12 are described in [TIFF6].

	1  = BYTE	8-bit unsigned integer.
	2  = ASCII	8-bit byte that contains a 7-bit ASCII code; the last byte must be NUL (binary zero).
	3  = SHORT	16-bit (2-byte) unsigned integer.
	4  = LONG	32-bit (4-byte) unsigned integer.
	5  = RATIONAL	Two LONGs: the first represents the numerator of a fraction; the second, the denominator.
	6  = SBYTE	An 8-bit signed (twos-complement) integer.
	7  = UNDEFINED	An 8-bit byte that may contain anything, depending on the definition of the field.
	8  = SSHORT	A 16-bit (2-byte) signed (twos-complement) integer.
	9  = SLONG	A 32-bit (4-byte) signed (twos-complement) integer.
	10 = SRATIONAL	Two SLONGs: the first represents the numerator of a fraction, the second the denominator.
	11 = FLOAT	Single precision (4-byte) IEEE format.
	12 = DOUBLE	Double precision (8-byte) IEEE format.
From [BIGTIFFDESIGN]:
	13-15: Field Type IDs 13 - 15 are mentioned in [BIGTIFFDESIGN], but only
		to explain why values 13 - 15 were skipped when identifying new
		Field Types for BigTIFF. These are meant to be used with regular
		TIFF, but were apparently not properly documented prior to the
		BigTIFF design discussion.

	13 = IFD	?? 32-bit unsigned integer offset value ??
	14 = UNICODE	??
	15 = COMPLEX	??
*/

// Default set of Field types.  These are exported for others to use in
// registering custom tags.
var (
	FTByte      = NewFieldType(1, "Byte", 1, false, reprByte, rvalByte, typByte)
	FTAscii     = NewFieldType(2, "ASCII", 1, false, reprASCII, rvalASCII, typString)
	FTShort     = NewFieldType(3, "Short", 2, false, reprShort, rvalShort, typU16)
	FTLong      = NewFieldType(4, "Long", 4, false, reprLong, rvalLong, typU32)
	FTRational  = NewFieldType(5, "Rational", 8, false, reprRational, rvalRational, typBigRat)
	FTSByte     = NewFieldType(6, "SByte", 1, true, reprSByte, rvalSByte, typI8)
	FTUndefined = NewFieldType(7, "Undefined", 1, false, reprByte, rvalByte, typByte)
	FTSShort    = NewFieldType(8, "SShort", 2, true, reprSShort, rvalSShort, typI16)
	FTSLong     = NewFieldType(9, "SLong", 4, true, reprSLong, rvalSLong, typI32)
	FTSRational = NewFieldType(10, "SRational", 8, true, reprSRational, rvalSRational, typBigRat)
	FTFloat     = NewFieldType(11, "Float", 4, true, reprFloat, rvalFloat, typF32)
	FTDouble    = NewFieldType(12, "Double", 8, true, reprDouble, rvalDouble, typF64)
	FTIFD       = NewFieldType(13, "IFD", 4, false, reprLong, rvalLong, typU32)

	// TODO: These two are not complete.  Get the details and finish them.
	FTUnicode = NewFieldType(14, "Unicode", 2, false, reprByte, rvalByte, typByte)
	FTComplex = NewFieldType(15, "Complex", 8, true, reprByte, rvalByte, typByte)
)

/*
Regarding UNICODE and COMPLEX field types:
  UNICODE:  In dng_sdk_1_4/dng_sdk/source/dng_tag_types.cpp and in
  dng_sdk_1_4/dng_sdk/source/dng_image_writer.cpp, ttUnicode is defined to
  have a size of 2.  It is not clear if this was based on the original
  16 bit Unicode or should be treated as UTF-16 to handle Unicode 2.0.

  COMPLEX:  In dng_sdk_1_4/dng_sdk/source/dng_tag_types.cpp, ttComplex is
  defined to have a size of 8.  However, in
  dng_sdk_1_4/dng_sdk/source/dng_image_writer.cpp ttComplex is said to have a
  size of 4 bytes. The file dng_image_writer.cpp also indicates a size of 4
  bytes for Rational and SRational, which we know to be made up of two 4 byte
  parts.  Since a complex is most likely made up of two 32bit floating point
  values, we are going with 8 for the size of a complex.  It is thought that it
  is intended for COMPLEX to be represented by two float32 values with the real
  part followed by the imaginary part in the byte stream.  In go terms this
  would mirror a complex64.  In tiff terms this would be two FLOAT types in a
  similar way that a RATIONAL is two LONGs.
*/

// DefaultFieldTypeSet is the default set of field types supported by this
// package.  A user is free to create their own FieldTypeSet from which to
// support extended functionality or to provide a substitute representation for
// known types.  Most users will be fine with the default set defined here.
var DefaultFieldTypeSet = NewFieldTypeSet("Default")

func init() {
	DefaultFieldTypeSet.Register(FTByte)
	DefaultFieldTypeSet.Register(FTAscii)
	DefaultFieldTypeSet.Register(FTShort)
	DefaultFieldTypeSet.Register(FTLong)
	DefaultFieldTypeSet.Register(FTRational)
	DefaultFieldTypeSet.Register(FTSByte)
	DefaultFieldTypeSet.Register(FTUndefined)
	DefaultFieldTypeSet.Register(FTSShort)
	DefaultFieldTypeSet.Register(FTSLong)
	DefaultFieldTypeSet.Register(FTSRational)
	DefaultFieldTypeSet.Register(FTFloat)
	DefaultFieldTypeSet.Register(FTDouble)
	DefaultFieldTypeSet.Register(FTIFD)
	DefaultFieldTypeSet.Register(FTUnicode)
	DefaultFieldTypeSet.Register(FTComplex)

	// Prevent further registration in the DefaultFieldTypeSet.  Others should
	// add to the DefaultFieldTypeSpace instead of the core set.
	DefaultFieldTypeSet.Lock()

	DefaultFieldTypeSpace.RegisterFieldTypeSet(DefaultFieldTypeSet)
}
<commit_msg>Clarify that Unicode is encoded via UTF-16.<commit_after>package tiff

import (
	"bytes"
	"encoding/binary"
	"fmt"
	"math"
	"math/big"
	"reflect"
)

/* FieldTypeRepr */
// These functions provide string representations of values based on field types.
func reprByte(in []byte, bo binary.ByteOrder) string   { return fmt.Sprintf("%d", in[0]) }
func reprSByte(in []byte, bo binary.ByteOrder) string  { return fmt.Sprintf("%d", int8(in[0])) }
func reprASCII(in []byte, bo binary.ByteOrder) string  { return string(in) }
func reprShort(in []byte, bo binary.ByteOrder) string  { return fmt.Sprintf("%d", bo.Uint16(in)) }
func reprSShort(in []byte, bo binary.ByteOrder) string { return fmt.Sprintf("%d", int16(bo.Uint16(in))) }
func reprLong(in []byte, bo binary.ByteOrder) string   { return fmt.Sprintf("%d", bo.Uint32(in)) }
func reprSLong(in []byte, bo binary.ByteOrder) string  { return fmt.Sprintf("%d", int32(bo.Uint32(in))) }
func reprRational(in []byte, bo binary.ByteOrder) string {
	// Print the representation directly to prevent panics from divide by
	// zero errors when using big.NewRat().
	return fmt.Sprintf("%d/%d", int64(bo.Uint32(in)), int64(bo.Uint32(in[4:])))
}
func reprSRational(in []byte, bo binary.ByteOrder) string {
	// Print the representation directly to prevent panics from divide by
	// zero errors when using big.NewRat()
	return fmt.Sprintf("%d/%d", int64(int32(bo.Uint32(in))), int64(int32(bo.Uint32(in[4:]))))
}
func reprFloat(in []byte, bo binary.ByteOrder) string {
	return fmt.Sprintf("%f", math.Float32frombits(bo.Uint32(in)))
}
func reprDouble(in []byte, bo binary.ByteOrder) string {
	return fmt.Sprintf("%f", math.Float64frombits(bo.Uint64(in)))
}

/* FieldTypeValuer */
func rvalByte(in []byte, bo binary.ByteOrder) reflect.Value  { return reflect.ValueOf(in[0]) }
func rvalSByte(in []byte, bo binary.ByteOrder) reflect.Value { return reflect.ValueOf(int8(in[0])) }
func rvalASCII(in []byte, bo binary.ByteOrder) reflect.Value {
	return reflect.ValueOf(string(bytes.TrimRight(in, "\x00")))
}
func rvalShort(in []byte, bo binary.ByteOrder) reflect.Value { return reflect.ValueOf(bo.Uint16(in)) }
func rvalSShort(in []byte, bo binary.ByteOrder) reflect.Value {
	return reflect.ValueOf(int16(bo.Uint16(in)))
}
func rvalLong(in []byte, bo binary.ByteOrder) reflect.Value { return reflect.ValueOf(bo.Uint32(in)) }
func rvalSLong(in []byte, bo binary.ByteOrder) reflect.Value {
	return reflect.ValueOf(int32(bo.Uint32(in)))
}
func rvalRational(in []byte, bo binary.ByteOrder) reflect.Value {
	denom := int64(bo.Uint32(in[4:]))
	if denom == 0 {
		// Prevent panics due to poorly written Rational fields with a
		// denominator of 0.
		return reflect.New(reflect.TypeOf(big.Rat{}))
	}
	numer := int64(bo.Uint32(in))
	return reflect.ValueOf(big.NewRat(numer, denom))
}
func rvalSRational(in []byte, bo binary.ByteOrder) reflect.Value {
	denom := int64(int32(bo.Uint32(in[4:])))
	if denom == 0 {
		// Prevent panics due to poorly written Rational fields with a
		// denominator of 0.  Their usable value would likely be 0.
		return reflect.New(reflect.TypeOf(big.Rat{}))
	}
	numer := int64(int32(bo.Uint32(in)))
	return reflect.ValueOf(big.NewRat(numer, denom))
}
func rvalFloat(in []byte, bo binary.ByteOrder) reflect.Value {
	return reflect.ValueOf(math.Float32frombits(bo.Uint32(in)))
}
func rvalDouble(in []byte, bo binary.ByteOrder) reflect.Value {
	return reflect.ValueOf(math.Float64frombits(bo.Uint64(in)))
}

/* reflect.Type */
var (
	typByte   = reflect.TypeOf(byte(0))         // BYTE, UNDEFINED
	typString = reflect.TypeOf(string(""))      // ASCII
	typU16    = reflect.TypeOf(uint16(0))       // SHORT
	typU32    = reflect.TypeOf(uint32(0))       // LONG, IFD
	typBigRat = reflect.TypeOf((*big.Rat)(nil)) // RATIONAL, SRATIONAL
	typI8     = reflect.TypeOf(int8(0))         // SBYTE
	typI16    = reflect.TypeOf(int16(0))        // SSHORT
	typI32    = reflect.TypeOf(int32(0))        // SLONG
	typF32    = reflect.TypeOf(float32(0))      // FLOAT
	typF64    = reflect.TypeOf(float64(0))      // DOUBLE
)

/* Field type definitions
From [TIFF6]:
	1-12: Field types 1 - 12 are described in [TIFF6].

	1  = BYTE	8-bit unsigned integer.
	2  = ASCII	8-bit byte that contains a 7-bit ASCII code; the last byte must be NUL (binary zero).
	3  = SHORT	16-bit (2-byte) unsigned integer.
	4  = LONG	32-bit (4-byte) unsigned integer.
	5  = RATIONAL	Two LONGs: the first represents the numerator of a fraction; the second, the denominator.
	6  = SBYTE	An 8-bit signed (twos-complement) integer.
	7  = UNDEFINED	An 8-bit byte that may contain anything, depending on the definition of the field.
	8  = SSHORT	A 16-bit (2-byte) signed (twos-complement) integer.
	9  = SLONG	A 32-bit (4-byte) signed (twos-complement) integer.
	10 = SRATIONAL	Two SLONGs: the first represents the numerator of a fraction, the second the denominator.
	11 = FLOAT	Single precision (4-byte) IEEE format.
	12 = DOUBLE	Double precision (8-byte) IEEE format.
From [BIGTIFFDESIGN]:
	13-15: Field Type IDs 13 - 15 are mentioned in [BIGTIFFDESIGN], but only
		to explain why values 13 - 15 were skipped when identifying new
		Field Types for BigTIFF. These are meant to be used with regular
		TIFF, but were apparently not properly documented prior to the
		BigTIFF design discussion.

	13 = IFD	?? 32-bit unsigned integer offset value ??
	14 = UNICODE	??
	15 = COMPLEX	??
*/

// Default set of Field types.  These are exported for others to use in
// registering custom tags.
var (
	FTByte      = NewFieldType(1, "Byte", 1, false, reprByte, rvalByte, typByte)
	FTAscii     = NewFieldType(2, "ASCII", 1, false, reprASCII, rvalASCII, typString)
	FTShort     = NewFieldType(3, "Short", 2, false, reprShort, rvalShort, typU16)
	FTLong      = NewFieldType(4, "Long", 4, false, reprLong, rvalLong, typU32)
	FTRational  = NewFieldType(5, "Rational", 8, false, reprRational, rvalRational, typBigRat)
	FTSByte     = NewFieldType(6, "SByte", 1, true, reprSByte, rvalSByte, typI8)
	FTUndefined = NewFieldType(7, "Undefined", 1, false, reprByte, rvalByte, typByte)
	FTSShort    = NewFieldType(8, "SShort", 2, true, reprSShort, rvalSShort, typI16)
	FTSLong     = NewFieldType(9, "SLong", 4, true, reprSLong, rvalSLong, typI32)
	FTSRational = NewFieldType(10, "SRational", 8, true, reprSRational, rvalSRational, typBigRat)
	FTFloat     = NewFieldType(11, "Float", 4, true, reprFloat, rvalFloat, typF32)
	FTDouble    = NewFieldType(12, "Double", 8, true, reprDouble, rvalDouble, typF64)
	FTIFD       = NewFieldType(13, "IFD", 4, false, reprLong, rvalLong, typU32)

	// TODO: These two are not complete.  Get the details and finish them.
	FTUnicode = NewFieldType(14, "Unicode", 2, false, reprByte, rvalByte, typByte)
	FTComplex = NewFieldType(15, "Complex", 8, true, reprByte, rvalByte, typByte)
)

/*
Regarding UNICODE and COMPLEX field types:
  UNICODE:  In dng_sdk_1_4/dng_sdk/source/dng_tag_types.cpp and in
  dng_sdk_1_4/dng_sdk/source/dng_image_writer.cpp, ttUnicode is defined to
  have a size of 2. In dng_image_writer.cpp, it appears unicode text is encoded
  with UTF-16.

  COMPLEX:  In dng_sdk_1_4/dng_sdk/source/dng_tag_types.cpp, ttComplex is
  defined to have a size of 8.  However, in
  dng_sdk_1_4/dng_sdk/source/dng_image_writer.cpp ttComplex is said to have a
  size of 4 bytes. The file dng_image_writer.cpp also indicates a size of 4
  bytes for Rational and SRational, which we know to be made up of two 4 byte
  parts.  Since a complex is most likely made up of two 32bit floating point
  values, we are going with 8 for the size of a complex.  It is thought that it
  is intended for COMPLEX to be represented by two float32 values with the real
  part followed by the imaginary part in the byte stream.  In go terms this
  would mirror a complex64.  In tiff terms this would be two FLOAT types in a
  similar way that a RATIONAL is two LONGs.
*/

// DefaultFieldTypeSet is the default set of field types supported by this
// package.  A user is free to create their own FieldTypeSet from which to
// support extended functionality or to provide a substitute representation for
// known types.  Most users will be fine with the default set defined here.
var DefaultFieldTypeSet = NewFieldTypeSet("Default")

func init() {
	DefaultFieldTypeSet.Register(FTByte)
	DefaultFieldTypeSet.Register(FTAscii)
	DefaultFieldTypeSet.Register(FTShort)
	DefaultFieldTypeSet.Register(FTLong)
	DefaultFieldTypeSet.Register(FTRational)
	DefaultFieldTypeSet.Register(FTSByte)
	DefaultFieldTypeSet.Register(FTUndefined)
	DefaultFieldTypeSet.Register(FTSShort)
	DefaultFieldTypeSet.Register(FTSLong)
	DefaultFieldTypeSet.Register(FTSRational)
	DefaultFieldTypeSet.Register(FTFloat)
	DefaultFieldTypeSet.Register(FTDouble)
	DefaultFieldTypeSet.Register(FTIFD)
	DefaultFieldTypeSet.Register(FTUnicode)
	DefaultFieldTypeSet.Register(FTComplex)

	// Prevent further registration in the DefaultFieldTypeSet.  Others should
	// add to the DefaultFieldTypeSpace instead of the core set.
	DefaultFieldTypeSet.Lock()

	DefaultFieldTypeSpace.RegisterFieldTypeSet(DefaultFieldTypeSet)
}
<|endoftext|>


<commit_before>// 文件数据记录类
// create by gloomy 2017-04-06 10:11:35
package common

import (
	"fmt"
	"os"
	"path/filepath"
	"strconv"
	"strings"
	"sync"
	"time"
)

const maxFileDataRecordingBytes = 1000000 // 默认文件大小

// 文件数据记录对象
// create by gloomy 2017-04-06 10:15:00
type FileDataRecording struct {
	sync.Mutex                         // 锁
	F                         *os.File // 文件对象
	FilePre                   string   // 文件开头字符串
	Fn                        string   // 文件路径
	Bytes                     int      // 文件大小
	Seq                       int      // 第几个
	FileProgram               string   // 文件存放路径
	MaxFileDataRecordingBytes int      // 文件大小
}

// 打开文件数据记录
// create by gloomy 2017-04-06 10:17:38
// 文件存放目录地址 文件开头字符串 文件大小
// 文件数据对象
func OpenLoadFile(fileProgram, filePre string, maxSize int) *FileDataRecording {
	if maxSize == 0 {
		maxSize = maxFileDataRecordingBytes
	}
	lf := &FileDataRecording{
		FilePre:                   filePre,
		FileProgram:               fileProgram,
		MaxFileDataRecordingBytes: maxSize,
	}
	lf.Rotate()
	return lf
}

// 文件退出
// create by gloomy 2017-04-06 10:27:58
func (f *FileDataRecording) Exit() {
	f.Lock()
	f.Close()
	f.F = nil
	f.Unlock()
}

// 文件关闭
// create by gloomy 2017-04-06 10:22:14
func (f *FileDataRecording) Close() {
	if f.F != nil {
		f.F.Close()
		os.Rename(f.Fn, f.Fn[0:len(f.Fn)-4]) //去掉末尾的.tmp
	}
}

// 文件切换
// create by gloomy 2017-04-06 10:30:05
func (f *FileDataRecording) Rotate() {
	f.Lock()
	f.Seq = 0
	f.Close()
	f.CreateNewFile()
	f.Unlock()
}

// 创建新文件
// create by gloomy 2017-04-06 10:33:11
// 错误对象
func (f *FileDataRecording) CreateNewFile() (err error) {
	f.Bytes = 0
	if !strings.HasSuffix(f.FileProgram, "/") {
		f.FileProgram += "/"
	}
	f.Fn = fmt.Sprintf("%s%s-%d-%d.tmp", f.FileProgram, f.FilePre, time.Now().UnixNano(), f.Seq)
	f.Seq++
	f.F, err = os.OpenFile(f.Fn, os.O_APPEND|os.O_CREATE|os.O_WRONLY, 0644)
	if err != nil {
		fmt.Printf("create file %s failed: %s \n", f.Fn, err.Error())
	}
	return
}

// 写入数据
// create by gloomy 2017-04-06 11:40:55
// 需要写入的数据
// 错误对象
func (f *FileDataRecording) WriteData(dataStr string) (err error) {
	f.Lock()
	defer f.Unlock()
	if f.F == nil {
		err = f.CreateNewFile()
		if err != nil {
			return
		}
	}
	dataStrLen := len(dataStr)
	if f.Bytes+dataStrLen > f.MaxFileDataRecordingBytes {
		f.Close()
		if err = f.CreateNewFile(); err != nil {
			return
		}
	}
	f.Bytes += dataStrLen
	_, err = f.F.WriteString(dataStr)
	return
}

// 获取所有完成的文件列表
// create by gloomy 2017-04-06 13:46:51
// 文件列表
func (f *FileDataRecording) FileList() *[]string {
	var (
		fileArray []string
	)
	filepath.Walk(f.FileProgram, func(path string, info os.FileInfo, err error) error {
		if err != nil {
			return nil
		}
		if ext := filepath.Ext(path); ext == ".tmp" {
			return nil
		}
		if info.Size() == 0 {
			os.Remove(path)
			return nil
		}
		if strings.HasPrefix(path, f.FilePre) {
			fileArray = append(fileArray, path)
		}
		return nil
	})
	return &fileArray
}

// 删除过期文件
// create by gloomy 2017-04-06 22:53:17
// 几天前
func (f *FileDataRecording) FileListRemoveOld(days int) {
	var (
		timeDate   = time.Now().AddDate(0, 0, 0-days).UnixNano()
		unixNumber int
		fileName   []string
	)
	filepath.Walk(f.FileProgram, func(path string, info os.FileInfo, err error) error {
		if err != nil {
			return nil
		}
		if ext := filepath.Ext(path); ext == ".tmp" {
			return nil
		}
		if info.Size() == 0 {
			os.Remove(path)
			return nil
		}
		if strings.HasPrefix(path, f.FilePre) {
			fileName = strings.Split(path, "-")
			if len(fileName) == 3 {
				unixNumber, err = strconv.Atoi(fileName[1])
				if err != nil {
					fmt.Printf("FileListRemoveOld fileProgram: %s path: %s err: %s \n", f.FileProgram, path, err.Error())
					return nil
				}
				if timeDate >= int64(unixNumber) {
					os.Remove(path)
				}
			}
		}
		return nil
	})
}
<commit_msg>修正时间增加<commit_after>// 文件数据记录类
// create by gloomy 2017-04-06 10:11:35
package common

import (
	"fmt"
	"os"
	"path/filepath"
	"strconv"
	"strings"
	"sync"
	"time"
)

const maxFileDataRecordingBytes = 1000000 // 默认文件大小

// 文件数据记录对象
// create by gloomy 2017-04-06 10:15:00
type FileDataRecording struct {
	sync.Mutex                         // 锁
	F                         *os.File // 文件对象
	FilePre                   string   // 文件开头字符串
	Fn                        string   // 文件路径
	Bytes                     int      // 文件大小
	Seq                       int      // 第几个
	FileProgram               string   // 文件存放路径
	MaxFileDataRecordingBytes int      // 文件大小
}

// 打开文件数据记录
// create by gloomy 2017-04-06 10:17:38
// 文件存放目录地址 文件开头字符串 文件大小
// 文件数据对象
func OpenLoadFile(fileProgram, filePre string, maxSize int) *FileDataRecording {
	if maxSize == 0 {
		maxSize = maxFileDataRecordingBytes
	}
	lf := &FileDataRecording{
		FilePre:                   filePre,
		FileProgram:               fileProgram,
		MaxFileDataRecordingBytes: maxSize,
	}
	lf.Rotate()
	return lf
}

// 文件退出
// create by gloomy 2017-04-06 10:27:58
func (f *FileDataRecording) Exit() {
	f.Lock()
	f.Close()
	f.F = nil
	f.Unlock()
}

// 文件关闭
// create by gloomy 2017-04-06 10:22:14
func (f *FileDataRecording) Close() {
	if f.F != nil {
		f.F.Close()
		os.Rename(f.Fn, f.Fn[0:len(f.Fn)-4]) //去掉末尾的.tmp
	}
}

// 文件切换
// create by gloomy 2017-04-06 10:30:05
func (f *FileDataRecording) Rotate() {
	f.Lock()
	f.Seq = 0
	f.Close()
	f.CreateNewFile()
	f.Unlock()
}

// 创建新文件
// create by gloomy 2017-04-06 10:33:11
// 错误对象
func (f *FileDataRecording) CreateNewFile() (err error) {
	f.Bytes = 0
	if !strings.HasSuffix(f.FileProgram, "/") {
		f.FileProgram += "/"
	}
	f.Fn = fmt.Sprintf("%s%s-%d-%d.tmp", f.FileProgram, f.FilePre, time.Now().UnixNano(), f.Seq)
	f.Seq++
	f.F, err = os.OpenFile(f.Fn, os.O_APPEND|os.O_CREATE|os.O_WRONLY, 0644)
	if err != nil {
		fmt.Printf("create file %s failed: %s \n", f.Fn, err.Error())
	}
	return
}

// 写入数据
// create by gloomy 2017-04-06 11:40:55
// 需要写入的数据
// 错误对象
func (f *FileDataRecording) WriteData(dataStr string) (err error) {
	f.Lock()
	defer f.Unlock()
	if f.F == nil {
		err = f.CreateNewFile()
		if err != nil {
			return
		}
	}
	dataStrLen := len(dataStr)
	if f.Bytes+dataStrLen > f.MaxFileDataRecordingBytes {
		f.Close()
		if err = f.CreateNewFile(); err != nil {
			return
		}
	}
	f.Bytes += dataStrLen
	_, err = f.F.WriteString(dataStr)
	return
}

// 获取所有完成的文件列表
// create by gloomy 2017-04-06 13:46:51
// 文件列表
func (f *FileDataRecording) FileList() *[]string {
	var (
		fileArray []string
	)
	filepath.Walk(f.FileProgram, func(path string, info os.FileInfo, err error) error {
		if err != nil {
			return nil
		}
		if ext := filepath.Ext(path); ext == ".tmp" {
			return nil
		}
		if info.Size() == 0 {
			os.Remove(path)
			return nil
		}
		if strings.HasPrefix(path, f.FilePre) {
			fileArray = append(fileArray, path)
		}
		return nil
	})
	return &fileArray
}

// 删除过期文件
// create by gloomy 2017-04-06 22:53:17
// 几天前
func (f *FileDataRecording) FileListRemoveOld(days int) {
	var (
		timeDate   = time.Now().AddDate(0, 0, days).UnixNano()
		unixNumber int
		fileName   []string
	)
	filepath.Walk(f.FileProgram, func(path string, info os.FileInfo, err error) error {
		if err != nil {
			return nil
		}
		if ext := filepath.Ext(path); ext == ".tmp" {
			return nil
		}
		if info.Size() == 0 {
			os.Remove(path)
			return nil
		}
		if strings.HasPrefix(path, f.FilePre) {
			fileName = strings.Split(path, "-")
			if len(fileName) == 3 {
				unixNumber, err = strconv.Atoi(fileName[1])
				if err != nil {
					fmt.Printf("FileListRemoveOld fileProgram: %s path: %s err: %s \n", f.FileProgram, path, err.Error())
					return nil
				}
				if timeDate >= int64(unixNumber) {
					os.Remove(path)
				}
			}
		}
		return nil
	})
}
<|endoftext|>


<commit_before>// Copyright 2016 Jacob Taylor jacob@ablox.io
// License: Apache2 - http://www.apache.org/licenses/LICENSE-2.0
package main

import (
	"fmt"
	"github.com/rjeczalik/notify"
	"log"
	"os"
	"path/filepath"
	"sort"
	"sync"
	"syscall"
)

type ChangeHandler interface {
	FolderCreated(name string) (err error)
	FolderDeleted(name string) (err error)
}

type StorageTracker interface {
	CreateFolder(name string) (err error)
	DeleteFolder(name string) (err error)
	ListFolders() (folders []string, err error)
}

type LogOnlyChangeHandler struct {
}

func (self *LogOnlyChangeHandler) FolderCreated(name string) (err error) {
	fmt.Printf("FolderCreated: %s\n", name)
	return nil
}

func (self *LogOnlyChangeHandler) FolderDeleted(name string) (err error) {
	fmt.Printf("FolderDeleted: %s\n", name)
	return nil
}

type FilesystemTracker struct {
	directory       string
	contents        DirTreeMap
	setup           bool
	watcher         *ChangeHandler
	fsEventsChannel chan notify.EventInfo
	fsLock          sync.RWMutex
}

func (self *FilesystemTracker) init() {
	self.fsLock.Lock()
	defer self.fsLock.Unlock()

	if self.setup {
		return
	}

	fmt.Println("Setting up filesystemTracker!")
	self.contents = make(DirTreeMap)
	self.setup = true
}

func (self *FilesystemTracker) cleanup() {
	self.fsLock.Lock()
	defer self.fsLock.Unlock()

	if !self.setup {
		panic("cleanup called when not yet setup")
	}

	notify.Stop(self.fsEventsChannel)
}

func (self *FilesystemTracker) watchDirectory(directory string, watcher *ChangeHandler) {
	self.fsLock.Lock()
	defer self.fsLock.Unlock()

	if !self.setup {
		panic("watchDirectory called when not yet setup")
	}

	fmt.Printf("watchDirectory called with %s\n", directory)

	self.directory = directory
	self.watcher = watcher

	// Make the channel buffered to ensure no event is dropped. Notify will drop
	// an event if the receiver is not able to keep up the sending pace.
	self.fsEventsChannel = make(chan notify.EventInfo, 1)

	// Update the path that traffic is served from to be the filesystem canonical path. This will allow the event folders that come in to match what we have.
	fullPath := validatePath(directory)
	self.directory = fullPath

	if fullPath != globalSettings.Directory {
		fmt.Printf("Updating serving directory to: %s\n", fullPath)
		self.directory = fullPath
	}

	go self.monitorLoop(self.fsEventsChannel)

	// Set up a watch point listening for events within a directory tree rooted at the specified folder
	err := notify.Watch(self.directory+"/...", self.fsEventsChannel, notify.All)
	if err != nil {
		log.Panic(err)
	}
}

func validatePath(directory string) (fullPath string) {
	fullPath, err := filepath.EvalSymlinks(directory)
	if err != nil {
		var err2, err3 error
		// We have an error. If the directory does not exist, then try to create it. Fail if we cannot create it and return the original error
		if os.IsNotExist(err) {
			err2 = os.Mkdir(directory, os.ModeDir+os.ModePerm)
			fullPath, err3 = filepath.EvalSymlinks(directory)
			if err2 != nil || err3 != nil {
				panic(fmt.Sprintf("err: %v\nerr2: %v\nerr3: %v\n", err, err2, err3))
			}
		} else {
			panic(err)
		}
	}

	return
}

func (self *FilesystemTracker) CreateFolder(name string) (err error) {
	self.init()

	self.fsLock.Lock()
	defer self.fsLock.Unlock()

	_, exists := self.contents[name]
	fmt.Printf("CreateFolder: '%s' (%v)\n", name, exists)

	if !exists {
		self.contents[name] = make([]string, 0, 10)
	}

	return nil
}

func (self *FilesystemTracker) DeleteFolder(name string) (err error) {
	self.init()

	self.fsLock.Lock()
	defer self.fsLock.Unlock()

	fmt.Printf("%s before delete of: %s\n", len(self.contents), name)
	fmt.Printf("DeleteFolder: '%s'\n", name)
	delete(self.contents, name)
	fmt.Printf("%s after delete of: %s\n", len(self.contents), name)

	return nil
}

func (self *FilesystemTracker) ListFolders() (list []string) {
	self.init()

	self.fsLock.Lock()
	defer self.fsLock.Unlock()

	folderList := make([]string, len(self.contents))
	index := 0

	for k, _ := range self.contents {
		folderList[index] = k
		index++
	}

	sort.Strings(folderList)
	return folderList
}

// Monitor the filesystem looking for changes to files we are keeping track of.
func (self *FilesystemTracker) monitorLoop(c chan notify.EventInfo) {
	directoryLength := len(self.directory)
	for {
		ei := <-c

		fmt.Println("We have an event")
		fullPath := string(ei.Path())

		path := fullPath
		if len(fullPath) >= directoryLength && self.directory == fullPath[:directoryLength] {
			if len(fullPath) == directoryLength {
				path = "."
			} else {
				// update the path to not have this prefix
				path = fullPath[directoryLength+1:]
			}
		}

		event := Event{Name: ei.Event().String(), Message: path}
		log.Printf("Event captured name: %s location: %s", event.Name, event.Message)

		// It looks like there is some sort of reversal of paths now. where adds are being removed from the other side.
		isDirectory := self.checkIfDirectory(event, path, fullPath)

		if isDirectory {
			self.processEvent(event, path)
		}

	}
}

func (self *FilesystemTracker) checkIfDirectory(event Event, path, fullPath string) bool {
	// Check to see if this was a path we knew about
	_, isDirectory := self.contents[path]
	var iNode uint64
	// if we have not found it yet, check to see if it can be stated
	info, err := os.Stat(fullPath)
	if err == nil {
		isDirectory = info.IsDir()
		sysInterface := info.Sys()
		fmt.Printf("sysInterface: %v\n", sysInterface)
		if sysInterface != nil {
			foo := sysInterface.(*syscall.Stat_t)
			iNode = foo.Ino
		}
	}

	fmt.Printf("checkIfDirectory: event raw data: %s with path: %s fullPath: %s isDirectory: %v iNode: %v\n", event.Name, path, fullPath, isDirectory, iNode)

	return isDirectory

}

func (self *FilesystemTracker) processEvent(event Event, pathName string) {
	log.Printf("handleFilsystemEvent name: %s pathName: %s serverMap: %v\n", event.Name, pathName, serverMap)

	currentValue, exists := self.contents[pathName]
	//fmt.Printf("Before: %s: Existing value for %s: %v (%v)\n", event.Name, pathName, currentValue, exists)

	switch event.Name {
	case "notify.Create":
		fmt.Printf("processEvent: About to assign from one path to the next. Original: %s Map: %s\n", self.contents[pathName], self.contents)
		// make sure there is an entry in the DirTreeMap for this folder. Since and empty list will always be returned, we can use that
		_, exists := self.contents[pathName]
		if !exists {
			self.contents[pathName] = make([]string, 0)
		}

		updated_value, exists := self.contents[pathName]
		fmt.Printf("notify.Create: Updated  value for %s: %v (%s)\n", pathName, updated_value, exists)

	case "notify.Remove":
		// clean out the entry in the DirTreMap for this folder
		delete(self.contents, pathName)

		updated_value, exists := self.contents[pathName]
		fmt.Printf("notify.Remove: Updated  value for %s: %v (%s)\n", pathName, updated_value, exists)

	// todo fix this to handle the two rename events to be one event
	//case "notify.Rename":
	//	err = os.Remove(pathName)
	//	if err != nil && !os.IsNotExist(err) {
	//		panic(fmt.Sprintf("Error deleting folder that was renamed %s: %v\n", pathName, err))
	//	}
	//	fmt.Printf("notify.Rename: %s\n", pathName)
	default:
	}

	currentValue, exists = self.contents[pathName]
	fmt.Printf("After: %s: Existing value for %s: %v (%v)\n", event.Name, pathName, currentValue, exists)

	// sendEvent to manager
	sendEvent(&event, globalSettings.ManagerAddress, globalSettings.ManagerCredentials)

	// todo - make this actually send to the peers
	log.Println("TODO Send to peers here")
	//log.Println("We are NodeA send to our peers")
	//// SendEvent to all peers
	//for k, v := range serverMap {
	//	fmt.Printf("Considering sending to: %s\n", k)
	//	if k != globalSettings.Name {
	//		fmt.Printf("sending to peer %s at %s\n", k, v.Address)
	//		sendEvent(&event, v.Address, globalSettings.ManagerCredentials)
	//	}
	//}
}
<commit_msg>adding print message in default event case.<commit_after>// Copyright 2016 Jacob Taylor jacob@ablox.io
// License: Apache2 - http://www.apache.org/licenses/LICENSE-2.0
package main

import (
	"fmt"
	"github.com/rjeczalik/notify"
	"log"
	"os"
	"path/filepath"
	"sort"
	"sync"
	"syscall"
)

type ChangeHandler interface {
	FolderCreated(name string) (err error)
	FolderDeleted(name string) (err error)
}

type StorageTracker interface {
	CreateFolder(name string) (err error)
	DeleteFolder(name string) (err error)
	ListFolders() (folders []string, err error)
}

type LogOnlyChangeHandler struct {
}

func (self *LogOnlyChangeHandler) FolderCreated(name string) (err error) {
	fmt.Printf("FolderCreated: %s\n", name)
	return nil
}

func (self *LogOnlyChangeHandler) FolderDeleted(name string) (err error) {
	fmt.Printf("FolderDeleted: %s\n", name)
	return nil
}

type FilesystemTracker struct {
	directory       string
	contents        DirTreeMap
	setup           bool
	watcher         *ChangeHandler
	fsEventsChannel chan notify.EventInfo
	fsLock          sync.RWMutex
}

func (self *FilesystemTracker) init() {
	self.fsLock.Lock()
	defer self.fsLock.Unlock()

	if self.setup {
		return
	}

	fmt.Println("Setting up filesystemTracker!")
	self.contents = make(DirTreeMap)
	self.setup = true
}

func (self *FilesystemTracker) cleanup() {
	self.fsLock.Lock()
	defer self.fsLock.Unlock()

	if !self.setup {
		panic("cleanup called when not yet setup")
	}

	notify.Stop(self.fsEventsChannel)
}

func (self *FilesystemTracker) watchDirectory(directory string, watcher *ChangeHandler) {
	self.fsLock.Lock()
	defer self.fsLock.Unlock()

	if !self.setup {
		panic("watchDirectory called when not yet setup")
	}

	fmt.Printf("watchDirectory called with %s\n", directory)

	self.directory = directory
	self.watcher = watcher

	// Make the channel buffered to ensure no event is dropped. Notify will drop
	// an event if the receiver is not able to keep up the sending pace.
	self.fsEventsChannel = make(chan notify.EventInfo, 1)

	// Update the path that traffic is served from to be the filesystem canonical path. This will allow the event folders that come in to match what we have.
	fullPath := validatePath(directory)
	self.directory = fullPath

	if fullPath != globalSettings.Directory {
		fmt.Printf("Updating serving directory to: %s\n", fullPath)
		self.directory = fullPath
	}

	go self.monitorLoop(self.fsEventsChannel)

	// Set up a watch point listening for events within a directory tree rooted at the specified folder
	err := notify.Watch(self.directory+"/...", self.fsEventsChannel, notify.All)
	if err != nil {
		log.Panic(err)
	}
}

func validatePath(directory string) (fullPath string) {
	fullPath, err := filepath.EvalSymlinks(directory)
	if err != nil {
		var err2, err3 error
		// We have an error. If the directory does not exist, then try to create it. Fail if we cannot create it and return the original error
		if os.IsNotExist(err) {
			err2 = os.Mkdir(directory, os.ModeDir+os.ModePerm)
			fullPath, err3 = filepath.EvalSymlinks(directory)
			if err2 != nil || err3 != nil {
				panic(fmt.Sprintf("err: %v\nerr2: %v\nerr3: %v\n", err, err2, err3))
			}
		} else {
			panic(err)
		}
	}

	return
}

func (self *FilesystemTracker) CreateFolder(name string) (err error) {
	self.init()

	self.fsLock.Lock()
	defer self.fsLock.Unlock()

	_, exists := self.contents[name]
	fmt.Printf("CreateFolder: '%s' (%v)\n", name, exists)

	if !exists {
		self.contents[name] = make([]string, 0, 10)
	}

	return nil
}

func (self *FilesystemTracker) DeleteFolder(name string) (err error) {
	self.init()

	self.fsLock.Lock()
	defer self.fsLock.Unlock()

	fmt.Printf("%s before delete of: %s\n", len(self.contents), name)
	fmt.Printf("DeleteFolder: '%s'\n", name)
	delete(self.contents, name)
	fmt.Printf("%s after delete of: %s\n", len(self.contents), name)

	return nil
}

func (self *FilesystemTracker) ListFolders() (list []string) {
	self.init()

	self.fsLock.Lock()
	defer self.fsLock.Unlock()

	folderList := make([]string, len(self.contents))
	index := 0

	for k, _ := range self.contents {
		folderList[index] = k
		index++
	}

	sort.Strings(folderList)
	return folderList
}

// Monitor the filesystem looking for changes to files we are keeping track of.
func (self *FilesystemTracker) monitorLoop(c chan notify.EventInfo) {
	directoryLength := len(self.directory)
	for {
		ei := <-c

		fmt.Println("We have an event")
		fullPath := string(ei.Path())

		path := fullPath
		if len(fullPath) >= directoryLength && self.directory == fullPath[:directoryLength] {
			if len(fullPath) == directoryLength {
				path = "."
			} else {
				// update the path to not have this prefix
				path = fullPath[directoryLength+1:]
			}
		}

		event := Event{Name: ei.Event().String(), Message: path}
		log.Printf("Event captured name: %s location: %s", event.Name, event.Message)

		// It looks like there is some sort of reversal of paths now. where adds are being removed from the other side.
		isDirectory := self.checkIfDirectory(event, path, fullPath)

		if isDirectory {
			self.processEvent(event, path)
		}

	}
}

func (self *FilesystemTracker) checkIfDirectory(event Event, path, fullPath string) bool {
	// Check to see if this was a path we knew about
	_, isDirectory := self.contents[path]
	var iNode uint64
	// if we have not found it yet, check to see if it can be stated
	info, err := os.Stat(fullPath)
	if err == nil {
		isDirectory = info.IsDir()
		sysInterface := info.Sys()
		fmt.Printf("sysInterface: %v\n", sysInterface)
		if sysInterface != nil {
			foo := sysInterface.(*syscall.Stat_t)
			iNode = foo.Ino
		}
	}

	fmt.Printf("checkIfDirectory: event raw data: %s with path: %s fullPath: %s isDirectory: %v iNode: %v\n", event.Name, path, fullPath, isDirectory, iNode)

	return isDirectory

}

func (self *FilesystemTracker) processEvent(event Event, pathName string) {
	log.Printf("handleFilsystemEvent name: %s pathName: %s serverMap: %v\n", event.Name, pathName, serverMap)

	currentValue, exists := self.contents[pathName]
	//fmt.Printf("Before: %s: Existing value for %s: %v (%v)\n", event.Name, pathName, currentValue, exists)

	switch event.Name {
	case "notify.Create":
		fmt.Printf("processEvent: About to assign from one path to the next. Original: %s Map: %s\n", self.contents[pathName], self.contents)
		// make sure there is an entry in the DirTreeMap for this folder. Since and empty list will always be returned, we can use that
		_, exists := self.contents[pathName]
		if !exists {
			self.contents[pathName] = make([]string, 0)
		}

		updated_value, exists := self.contents[pathName]
		fmt.Printf("notify.Create: Updated  value for %s: %v (%s)\n", pathName, updated_value, exists)

	case "notify.Remove":
		// clean out the entry in the DirTreMap for this folder
		delete(self.contents, pathName)

		updated_value, exists := self.contents[pathName]
		fmt.Printf("notify.Remove: Updated  value for %s: %v (%s)\n", pathName, updated_value, exists)

	// todo fix this to handle the two rename events to be one event
	//case "notify.Rename":
	//	err = os.Remove(pathName)
	//	if err != nil && !os.IsNotExist(err) {
	//		panic(fmt.Sprintf("Error deleting folder that was renamed %s: %v\n", pathName, err))
	//	}
	//	fmt.Printf("notify.Rename: %s\n", pathName)
	default:
		fmt.Printf("%s: %s not known, skipping\n", event.Name, pathName)
	}

	currentValue, exists = self.contents[pathName]
	fmt.Printf("After: %s: Existing value for %s: %v (%v)\n", event.Name, pathName, currentValue, exists)

	// sendEvent to manager
	sendEvent(&event, globalSettings.ManagerAddress, globalSettings.ManagerCredentials)

	// todo - make this actually send to the peers
	log.Println("TODO Send to peers here")
	//log.Println("We are NodeA send to our peers")
	//// SendEvent to all peers
	//for k, v := range serverMap {
	//	fmt.Printf("Considering sending to: %s\n", k)
	//	if k != globalSettings.Name {
	//		fmt.Printf("sending to peer %s at %s\n", k, v.Address)
	//		sendEvent(&event, v.Address, globalSettings.ManagerCredentials)
	//	}
	//}
}
<|endoftext|>


<commit_before>package fileutil

import (
	"errors"
	"fmt"
	"io/ioutil"
	"log"
	"os"

	"github.com/bitrise-io/go-utils/pathutil"
)

// GetFileModeOfFile ...
//  this is the "permissions" info, which can be passed directly to
//  functions like WriteBytesToFileWithPermission or os.OpenFile
func GetFileModeOfFile(pth string) (os.FileMode, error) {
	finfo, err := os.Stat(pth)
	if err != nil {
		return 0, err
	}
	return finfo.Mode(), nil
}

// WriteStringToFile ...
func WriteStringToFile(pth string, fileCont string) error {
	return WriteBytesToFile(pth, []byte(fileCont))
}

// WriteBytesToFileWithPermission ...
func WriteBytesToFileWithPermission(pth string, fileCont []byte, perm os.FileMode) error {
	if pth == "" {
		return errors.New("No path provided")
	}

	var file *os.File
	var err error
	if perm == 0 {
		file, err = os.Create(pth)
	} else {
		// same as os.Create, but with a specified permission
		//  the flags are copy-pasted from the official
		//  os.Create func: https://golang.org/src/os/file.go?s=7327:7366#L244
		file, err = os.OpenFile(pth, os.O_RDWR|os.O_CREATE|os.O_TRUNC, perm)
	}
	if err != nil {
		return err
	}
	defer func() {
		if err := file.Close(); err != nil {
			log.Println(" [!] Failed to close file:", err)
		}
	}()

	if _, err := file.Write(fileCont); err != nil {
		return err
	}

	return nil
}

// WriteBytesToFile ...
func WriteBytesToFile(pth string, fileCont []byte) error {
	return WriteBytesToFileWithPermission(pth, fileCont, 0)
}

// AppendStringToFile ...
func AppendStringToFile(pth string, fileCont string) error {
	return AppendBytesToFile(pth, []byte(fileCont))
}

// AppendBytesToFile ...
func AppendBytesToFile(pth string, fileCont []byte) error {
	if pth == "" {
		return errors.New("No path provided")
	}

	var file *os.File
	filePerm, err := GetFilePermissions(pth)
	if err != nil {
		// create the file
		file, err = os.Create(pth)
	} else {
		// open for append
		file, err = os.OpenFile(pth, os.O_APPEND|os.O_CREATE|os.O_WRONLY, filePerm)
	}
	if err != nil {
		// failed to create or open-for-append the file
		return err
	}
	defer func() {
		if err := file.Close(); err != nil {
			log.Println(" [!] Failed to close file:", err)
		}
	}()

	if _, err := file.Write(fileCont); err != nil {
		return err
	}

	return nil
}

// ReadBytesFromFile ...
func ReadBytesFromFile(pth string) ([]byte, error) {
	if isExists, err := pathutil.IsPathExists(pth); err != nil {
		return []byte{}, err
	} else if !isExists {
		return []byte{}, fmt.Errorf("No file found at path: %s", pth)
	}

	bytes, err := ioutil.ReadFile(pth)
	if err != nil {
		return []byte{}, err
	}
	return bytes, nil
}

// ReadStringFromFile ...
func ReadStringFromFile(pth string) (string, error) {
	contBytes, err := ReadBytesFromFile(pth)
	if err != nil {
		return "", err
	}
	return string(contBytes), nil
}

// GetFilePermissions ...
func GetFilePermissions(filePth string) (os.FileMode, error) {
	info, err := os.Stat(filePth)
	if err != nil {
		return 0, err
	}
	mode := info.Mode()
	return mode, nil
}
<commit_msg>GetFilePermissions and GetFileModeOfFile unified - was implemented twice, separately - now it's just an alias<commit_after>package fileutil

import (
	"errors"
	"fmt"
	"io/ioutil"
	"log"
	"os"

	"github.com/bitrise-io/go-utils/pathutil"
)

// WriteStringToFile ...
func WriteStringToFile(pth string, fileCont string) error {
	return WriteBytesToFile(pth, []byte(fileCont))
}

// WriteBytesToFileWithPermission ...
func WriteBytesToFileWithPermission(pth string, fileCont []byte, perm os.FileMode) error {
	if pth == "" {
		return errors.New("No path provided")
	}

	var file *os.File
	var err error
	if perm == 0 {
		file, err = os.Create(pth)
	} else {
		// same as os.Create, but with a specified permission
		//  the flags are copy-pasted from the official
		//  os.Create func: https://golang.org/src/os/file.go?s=7327:7366#L244
		file, err = os.OpenFile(pth, os.O_RDWR|os.O_CREATE|os.O_TRUNC, perm)
	}
	if err != nil {
		return err
	}
	defer func() {
		if err := file.Close(); err != nil {
			log.Println(" [!] Failed to close file:", err)
		}
	}()

	if _, err := file.Write(fileCont); err != nil {
		return err
	}

	return nil
}

// WriteBytesToFile ...
func WriteBytesToFile(pth string, fileCont []byte) error {
	return WriteBytesToFileWithPermission(pth, fileCont, 0)
}

// AppendStringToFile ...
func AppendStringToFile(pth string, fileCont string) error {
	return AppendBytesToFile(pth, []byte(fileCont))
}

// AppendBytesToFile ...
func AppendBytesToFile(pth string, fileCont []byte) error {
	if pth == "" {
		return errors.New("No path provided")
	}

	var file *os.File
	filePerm, err := GetFilePermissions(pth)
	if err != nil {
		// create the file
		file, err = os.Create(pth)
	} else {
		// open for append
		file, err = os.OpenFile(pth, os.O_APPEND|os.O_CREATE|os.O_WRONLY, filePerm)
	}
	if err != nil {
		// failed to create or open-for-append the file
		return err
	}
	defer func() {
		if err := file.Close(); err != nil {
			log.Println(" [!] Failed to close file:", err)
		}
	}()

	if _, err := file.Write(fileCont); err != nil {
		return err
	}

	return nil
}

// ReadBytesFromFile ...
func ReadBytesFromFile(pth string) ([]byte, error) {
	if isExists, err := pathutil.IsPathExists(pth); err != nil {
		return []byte{}, err
	} else if !isExists {
		return []byte{}, fmt.Errorf("No file found at path: %s", pth)
	}

	bytes, err := ioutil.ReadFile(pth)
	if err != nil {
		return []byte{}, err
	}
	return bytes, nil
}

// ReadStringFromFile ...
func ReadStringFromFile(pth string) (string, error) {
	contBytes, err := ReadBytesFromFile(pth)
	if err != nil {
		return "", err
	}
	return string(contBytes), nil
}

// GetFileModeOfFile ...
//  this is the "permissions" info, which can be passed directly to
//  functions like WriteBytesToFileWithPermission or os.OpenFile
func GetFileModeOfFile(pth string) (os.FileMode, error) {
	finfo, err := os.Stat(pth)
	if err != nil {
		return 0, err
	}
	return finfo.Mode(), nil
}

// GetFilePermissions ...
// - alias of: GetFileModeOfFile
//  this is the "permissions" info, which can be passed directly to
//  functions like WriteBytesToFileWithPermission or os.OpenFile
func GetFilePermissions(filePth string) (os.FileMode, error) {
	return GetFileModeOfFile(filePth)
}
<|endoftext|>


<commit_before>package tail

import (
	"fmt"
	"io"
)

const rbufSize = 4096

type readSeekCloser interface {
	io.Reader
	io.Seeker
	io.Closer
}

type RTail struct {
	raw     readSeekCloser
	limit   int
	bufsize int64
	closed  bool
	ft      bool
}

func NewRTail(r readSeekCloser, limit int, bufsize int) *RTail {
	return &RTail{
		raw:     r,
		limit:   limit,
		bufsize: int64(bufsize),
	}
}

func (rt *RTail) Read(buf []byte) (int, error) {
	if !rt.ft {
		rt.ft = true
		err := rt.findTop()
		if err != nil {
			return 0, nil
		}
	}
	return rt.raw.Read(buf)
}

func (rt *RTail) Close() error {
	if rt.closed {
		return nil
	}
	rt.closed = true
	return rt.raw.Close()
}

func (rt *RTail) findTop() error {
	sz, err := rt.raw.Seek(0, io.SeekEnd)
	if err != nil {
		return err
	}
	var (
		cnt  = 0
		buf  = make([]byte, rt.bufsize)
		wbuf []byte
		tail = true
	)
	for curr := sz; curr > 0; {
		if curr < rt.bufsize {
			wbuf = buf[0:curr]
		} else {
			wbuf = buf[0:rt.bufsize]
		}
		curr -= int64(len(wbuf))
		err := rt.seekStart(curr)
		if err != nil {
			return err
		}
		n, err := rt.raw.Read(wbuf)
		if err != nil {
			return err
		}
		if n != len(wbuf) {
			return fmt.Errorf("tail requires %d bytes but got %d", len(wbuf), n)
		}
		for off := n - 1; off >= 0; off -= 1 {
			if wbuf[off] != '\n' || tail {
				tail = false
				continue
			}
			cnt++
			if cnt < rt.limit {
				continue
			}
			return rt.seekStart(curr + int64(off) + 1)
		}
	}
	return rt.seekStart(0)
}

func (rt *RTail) seekStart(off int64) error {
	_, err := rt.raw.Seek(off, io.SeekStart)
	return err
}
<commit_msg>use magic number for 1.6<commit_after>package tail

import (
	"fmt"
	"io"
)

const rbufSize = 4096

type readSeekCloser interface {
	io.Reader
	io.Seeker
	io.Closer
}

type RTail struct {
	raw     readSeekCloser
	limit   int
	bufsize int64
	closed  bool
	ft      bool
}

func NewRTail(r readSeekCloser, limit int, bufsize int) *RTail {
	return &RTail{
		raw:     r,
		limit:   limit,
		bufsize: int64(bufsize),
	}
}

func (rt *RTail) Read(buf []byte) (int, error) {
	if !rt.ft {
		rt.ft = true
		err := rt.findTop()
		if err != nil {
			return 0, nil
		}
	}
	return rt.raw.Read(buf)
}

func (rt *RTail) Close() error {
	if rt.closed {
		return nil
	}
	rt.closed = true
	return rt.raw.Close()
}

func (rt *RTail) findTop() error {
	sz, err := rt.raw.Seek(0, 2)
	if err != nil {
		return err
	}
	var (
		cnt  = 0
		buf  = make([]byte, rt.bufsize)
		wbuf []byte
		tail = true
	)
	for curr := sz; curr > 0; {
		if curr < rt.bufsize {
			wbuf = buf[0:curr]
		} else {
			wbuf = buf[0:rt.bufsize]
		}
		curr -= int64(len(wbuf))
		err := rt.seekStart(curr)
		if err != nil {
			return err
		}
		n, err := rt.raw.Read(wbuf)
		if err != nil {
			return err
		}
		if n != len(wbuf) {
			return fmt.Errorf("tail requires %d bytes but got %d", len(wbuf), n)
		}
		for off := n - 1; off >= 0; off -= 1 {
			if wbuf[off] != '\n' || tail {
				tail = false
				continue
			}
			cnt++
			if cnt < rt.limit {
				continue
			}
			return rt.seekStart(curr + int64(off) + 1)
		}
	}
	return rt.seekStart(0)
}

func (rt *RTail) seekStart(off int64) error {
	_, err := rt.raw.Seek(off, 0)
	return err
}
<|endoftext|>


<commit_before>/*
 * csvcol.go
 * Program to select columns (and rows) from a CSV file.
 * by J. Stuart McMurray
 * Created 20141119
 * Last modified 20141120
 *
 * Copyright (c) 2014 J. Stuart McMurray <kd5pbo@gmail.com>
 *
 * Permission to use, copy, modify, and distribute this software for any
 * purpose with or without fee is hereby granted, provided that the above
 * copyright notice and this permission notice appear in all copies.
 *
 * THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES
 * WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF
 * MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR
 * ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES
 * WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN
 * ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF
 * OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
 */

package main

import (
	"bufio"
	"encoding/csv"
	"flag"
	"fmt"
	"os"
	"strings"

	"github.com/magisterquis/ranges"
)

/* Global config */
var gc struct {
	csvfile     *string
	rows        *string
	rowfile     *string
	cols        *string
	colfile     *string
	verbose     *bool
	v           *bool
	debug       *bool
	d           *bool
	commentChar *string
}

func main() {
	/* Set flags and parse */
	gc.csvfile = flag.String("csvfile", "", "CSV file to read.  CSV-formatted data will be also be read from the file(s) listed on the command line (in the order listed).  If -csvfile is - or no files are listed on the command line and -csvfile is not specified, CSV-formatted data will be read from standard input (in which case, neither rowfile nor colfile may be -).  If both -csvfile and additional files are given, the file named by -csvfile will be read first (even if it is -).")
	gc.rows = flag.String("rows", "", "The row(-number)s to output.  This is given as a comma-separated list of row numbers or ranges.  Either the starting or ending number may be omitted in a range to indicate the first or last row, respectively.  Example: -3,5-7,9,11-, which outputs rows 1, 2, 3, 5, 6, 7, 9, and all rows from the 11th row to the end of the data (inclusive of the 11th row).  By default, all rows are output if neither -ros nor -rowfile are specified.  The row counter is not reset between each file.  It is as if all the files were concatenated.")
	gc.rowfile = flag.String("rowfile", "", "If specified, 1-indexed row numbers to to indicate rows to output will be read from this file.  The format is the nearly the same as for -rows, but may be given on multiple lines.  May be - to read from the standard input (in which case, neither csvfile nor colfile may be -).  If both this and -rows are specified, rows specified by either this file or -rows will be output.")
	gc.cols = flag.String("cols", "", "The column(-number)s to output.  This is given as a comma-separated list of column numbers or ranges.  Either the starting or ending number may be omitted in a range to indicate the first or last column, respectively.  Example: -3,5-7,9,11-, which outputs columns 1, 2, 3, 5, 6, 7, 9, and all columns from the 11th column to the end of the data (inclusive of the 11th column).  By default, all columns are output if neither -cols nor -colfile are specified.")
	gc.colfile = flag.String("colfile", "", "If specified, 1-indexed column numbers to to indicate columns to output will be read from this file.  The format is the nearly the same as for -columns, but may be given on multiple lines.  May be - to read from the standard input (in which case, neither csvfile nor rowfile may be -).  If both this and -cols are specified, columns specified by either this file or -cols will be output.")
	gc.commentChar = flag.String("commentchar", "#", "Comment character.  If a line starts with this character, it will be ignored.  Set to \"\" to disable ignoring comments.")
	gc.verbose = flag.Bool("verbose", false, "Print informational messages to the standard error stream.")
	gc.v = flag.Bool("v", false, "Same as -verbose")
	gc.debug = flag.Bool("debug", false, "Print debugging messages to the standard error stream.")
	gc.d = flag.Bool("d", false, "Same as debug")
	flag.Parse()

	/* Handle -v and -d */
	*gc.verbose = *gc.verbose || *gc.v
	*gc.debug = *gc.debug || *gc.d

	/* Ensure that only one of the files is stdin */
	s := false /* Using stdin */
	checkStdin(&s, "-" == *gc.rowfile)
	checkStdin(&s, "-" == *gc.colfile)
	checkStdin(&s, ("-" == *gc.csvfile) ||
		("" == *gc.csvfile && 0 == flag.NArg()))

	/* Work out which rows to print */
	rFilter := mkFilter(*gc.rows, *gc.rowfile, "row")
	/* Work out which columns to print */
	cFilter := mkFilter(*gc.cols, *gc.colfile, "column")

	debug("Row Filter: %v", rFilter)
	debug("Colunm Filter: %v", cFilter)

	/* Make an array of filenames to read. */
	csvfile := []string{}
	/* Only stdin */
	if "" == *gc.csvfile && 0 == flag.NArg() {
		csvfile = []string{"-"}
	} else {
		/* First -csvfile */
		if "" != *gc.csvfile {
			csvfile = append(csvfile, *gc.csvfile)
		}
		/* Then command-line files */
		if flag.NArg() > 0 {
			csvfile = append(csvfile, flag.Args()...)
		}
	}

	/* Set up stdout as a CSV writer */
	w := csv.NewWriter(os.Stdout)

	lineNumber := 1    /* Current line number */
	ldone := false     /* Above the filter */
	orsize := 1        /* Size of previous output record */
	comment := rune(0) /* CSV comment character */
	if len(*gc.commentChar) > 0 {
		comment = []rune(*gc.commentChar)[0]
	}

	/* Read data from each file */
	for _, f := range csvfile {
		/* Printable name */
		var fp *os.File
		fname := f
		if "-" == fname {
			fname = "standard input"
			fp = os.Stdin
		} else {
			fpl, err := os.Open(f)
			if err != nil {
				inform("Unable to open %v: %v", f, err)
				os.Exit(-5)
			}
			fp = fpl
		}
		verbose("Parsing %v", fname)
		/* Make a CSV reader */
		r := csv.NewReader(fp)
		/* Reader settings */
		r.Comment = comment
		r.FieldsPerRecord = -1
		r.LazyQuotes = true

		/* Parse lines until the file is done */
		for ; ; lineNumber++ {
			/* Get a line */
			record, e := r.Read()
			if nil != record {
				debug("%v) Got %v fields: %#v", lineNumber,
					len(record), record)
			}
			/* Give up if we have an error */
			if e != nil {
				/* If it's EOF, go to the next file */
				if "EOF" == e.Error() {
					break
				}
				debug("Got error reading %v (%T): %v", fname,
					e, e)
				break
			}
			/* Work out whether to ignore it */
			if !ldone {
				a, y := rFilter.AllowsOut(lineNumber)
				/* Read the next one if not allowed */
				if !a {
					continue
				}
				/* Done checking lines if all are allowed or
				if we're past the upper limit */
				if ranges.AllMatch == y || ranges.Above == y {
					ldone = true
				}
			}
			/* Roll an output slice */
			orec := make([]string, 0, orsize)
			cdone := false /* Done worrying about columns */
			/* Add the right columns */
			for i := 1; i <= len(record); i++ {
				/* Work out whether to add this column */
				if !cdone {
					a, y := cFilter.AllowsOut(i)
					if !a {
						continue
					}
					/* Done checking if upper limit or
					all allowed */
					if ranges.AllMatch == y ||
						ranges.Above == y {
						cdone = true
					}
				}
				orec = append(orec, record[i-1])
			}
			orsize = len(orec)

			/* Actually output line */
			if err := w.Write(orec); err != nil {
				inform("Error writing %v: %v", orec, err)
				os.Exit(-8)
			}
		}
		/* TODO: Finish this */
		/* Flush output after each file */
		w.Flush()
		if err := w.Error(); err != nil {
			inform("Error flushing output: %v", err)
			os.Exit(-6)
		}
	}

}

/* Check if is is true.  If it is and s is true, die with an error.  If is is
true and s isn't, make s true.  If it's false all is good. */
func checkStdin(s *bool, is bool) {
	/* Don't care if is isn't */
	if !is {
		return
	}
	/* Set s if it's not set */
	if !*s {
		*s = true
		return
	}
	/* If both are set, die with an error. */
	inform("Only one of -csvfile, -rowfile, or -colfile may come from " +
		"the standard input.\n")
	os.Exit(-1)
}

/* mkFilter makes a filter from the specified flagfile (i.e. rowfile) and flag
(i.e. rows).  Name is passed in for error reporting. */
func mkFilter(flag, flagfile, name string) ranges.Filter {
	debug("Making %v filter from flag [%v] and file [%v]", name, flag,
		flagfile)
	/* Filter to return */
	f := ranges.New(verbose, debug)
	/* If we have nothing to set, return a permissive filter */
	if "" == flag && "" == flagfile {
		f.All = true
		return f
	}

	/* Process ranges on the command line */
	if "" != flag {
		verbose("Processing %v ranges from the commandline (%v)", name, flag)
		if err := f.Update(flag); err != nil {
			inform("Unable to process %v ranges (%v): %v", name,
				flag, err)
			os.Exit(-3)
		}

	}

	/* Will be what we read from */
	var in *os.File
	/* Populate in with something */
	if "" != flagfile {
		if "-" == flagfile {
			in = os.Stdin
		} else {
			i, err := os.Open(flagfile)
			if err != nil {
				inform("Unable to open %v file %v: %v",
					name, flagfile, err)
				os.Exit(-2)
			}
			in = i
		}
		/* Read lines from the file */
		scanner := bufio.NewScanner(in)
		fname := flagfile
		if "-" == fname {
			fname = "standard input"
		}
		for scanner.Scan() {
			t := scanner.Text()
			verbose("Processing %v from %v", t, fname)
			if err := f.Update(t); err != nil {
				inform("Unable to process %v ranges from "+
					"%v: %v", name, fname, err)
				os.Exit(-7)
			}
		}
		if err := scanner.Err(); err != nil {
			inform("Error reading from %v: %v", fname, err)
			os.Exit(-4)
		}
	}

	return f
}

/* verbose prints a message if -v */
func verbose(f string, a ...interface{}) {
	if *gc.verbose || *gc.debug {
		inform(f, a...)
	}
}

/* debug prints a message if -v or -d */
func debug(f string, a ...interface{}) {
	f = "D: " + f /* Note as a debug string */
	if *gc.debug {
		inform(f, a...)
	}
}

/* inform prints informational mesages to stderr */
func inform(f string, a ...interface{}) {
	if !strings.HasSuffix(f, "\n") {
		f += "\n"
	}
	fmt.Fprintf(os.Stderr, f, a...)
}
<commit_msg>Updated modified date<commit_after>/*
 * csvcol.go
 * Program to select columns (and rows) from a CSV file.
 * by J. Stuart McMurray
 * Created 20141119
 * Last modified 20170501
 *
 * Copyright (c) 2014-2017 J. Stuart McMurray <kd5pbo@gmail.com>
 *
 * Permission to use, copy, modify, and distribute this software for any
 * purpose with or without fee is hereby granted, provided that the above
 * copyright notice and this permission notice appear in all copies.
 *
 * THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES
 * WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF
 * MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR
 * ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES
 * WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN
 * ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF
 * OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
 */

package main

import (
	"bufio"
	"encoding/csv"
	"flag"
	"fmt"
	"os"
	"strings"

	"github.com/magisterquis/ranges"
)

/* Global config */
var gc struct {
	csvfile     *string
	rows        *string
	rowfile     *string
	cols        *string
	colfile     *string
	verbose     *bool
	v           *bool
	debug       *bool
	d           *bool
	commentChar *string
}

func main() {
	/* Set flags and parse */
	gc.csvfile = flag.String("csvfile", "", "CSV file to read.  CSV-formatted data will be also be read from the file(s) listed on the command line (in the order listed).  If -csvfile is - or no files are listed on the command line and -csvfile is not specified, CSV-formatted data will be read from standard input (in which case, neither rowfile nor colfile may be -).  If both -csvfile and additional files are given, the file named by -csvfile will be read first (even if it is -).")
	gc.rows = flag.String("rows", "", "The row(-number)s to output.  This is given as a comma-separated list of row numbers or ranges.  Either the starting or ending number may be omitted in a range to indicate the first or last row, respectively.  Example: -3,5-7,9,11-, which outputs rows 1, 2, 3, 5, 6, 7, 9, and all rows from the 11th row to the end of the data (inclusive of the 11th row).  By default, all rows are output if neither -ros nor -rowfile are specified.  The row counter is not reset between each file.  It is as if all the files were concatenated.")
	gc.rowfile = flag.String("rowfile", "", "If specified, 1-indexed row numbers to to indicate rows to output will be read from this file.  The format is the nearly the same as for -rows, but may be given on multiple lines.  May be - to read from the standard input (in which case, neither csvfile nor colfile may be -).  If both this and -rows are specified, rows specified by either this file or -rows will be output.")
	gc.cols = flag.String("cols", "", "The column(-number)s to output.  This is given as a comma-separated list of column numbers or ranges.  Either the starting or ending number may be omitted in a range to indicate the first or last column, respectively.  Example: -3,5-7,9,11-, which outputs columns 1, 2, 3, 5, 6, 7, 9, and all columns from the 11th column to the end of the data (inclusive of the 11th column).  By default, all columns are output if neither -cols nor -colfile are specified.")
	gc.colfile = flag.String("colfile", "", "If specified, 1-indexed column numbers to to indicate columns to output will be read from this file.  The format is the nearly the same as for -columns, but may be given on multiple lines.  May be - to read from the standard input (in which case, neither csvfile nor rowfile may be -).  If both this and -cols are specified, columns specified by either this file or -cols will be output.")
	gc.commentChar = flag.String("commentchar", "#", "Comment character.  If a line starts with this character, it will be ignored.  Set to \"\" to disable ignoring comments.")
	gc.verbose = flag.Bool("verbose", false, "Print informational messages to the standard error stream.")
	gc.v = flag.Bool("v", false, "Same as -verbose")
	gc.debug = flag.Bool("debug", false, "Print debugging messages to the standard error stream.")
	gc.d = flag.Bool("d", false, "Same as debug")
	flag.Parse()

	/* Handle -v and -d */
	*gc.verbose = *gc.verbose || *gc.v
	*gc.debug = *gc.debug || *gc.d

	/* Ensure that only one of the files is stdin */
	s := false /* Using stdin */
	checkStdin(&s, "-" == *gc.rowfile)
	checkStdin(&s, "-" == *gc.colfile)
	checkStdin(&s, ("-" == *gc.csvfile) ||
		("" == *gc.csvfile && 0 == flag.NArg()))

	/* Work out which rows to print */
	rFilter := mkFilter(*gc.rows, *gc.rowfile, "row")
	/* Work out which columns to print */
	cFilter := mkFilter(*gc.cols, *gc.colfile, "column")

	debug("Row Filter: %v", rFilter)
	debug("Colunm Filter: %v", cFilter)

	/* Make an array of filenames to read. */
	csvfile := []string{}
	/* Only stdin */
	if "" == *gc.csvfile && 0 == flag.NArg() {
		csvfile = []string{"-"}
	} else {
		/* First -csvfile */
		if "" != *gc.csvfile {
			csvfile = append(csvfile, *gc.csvfile)
		}
		/* Then command-line files */
		if flag.NArg() > 0 {
			csvfile = append(csvfile, flag.Args()...)
		}
	}

	/* Set up stdout as a CSV writer */
	w := csv.NewWriter(os.Stdout)

	lineNumber := 1    /* Current line number */
	ldone := false     /* Above the filter */
	orsize := 1        /* Size of previous output record */
	comment := rune(0) /* CSV comment character */
	if len(*gc.commentChar) > 0 {
		comment = []rune(*gc.commentChar)[0]
	}

	/* Read data from each file */
	for _, f := range csvfile {
		/* Printable name */
		var fp *os.File
		fname := f
		if "-" == fname {
			fname = "standard input"
			fp = os.Stdin
		} else {
			fpl, err := os.Open(f)
			if err != nil {
				inform("Unable to open %v: %v", f, err)
				os.Exit(-5)
			}
			fp = fpl
		}
		verbose("Parsing %v", fname)
		/* Make a CSV reader */
		r := csv.NewReader(fp)
		/* Reader settings */
		r.Comment = comment
		r.FieldsPerRecord = -1
		r.LazyQuotes = true

		/* Parse lines until the file is done */
		for ; ; lineNumber++ {
			/* Get a line */
			record, e := r.Read()
			if nil != record {
				debug("%v) Got %v fields: %#v", lineNumber,
					len(record), record)
			}
			/* Give up if we have an error */
			if e != nil {
				/* If it's EOF, go to the next file */
				if "EOF" == e.Error() {
					break
				}
				debug("Got error reading %v (%T): %v", fname,
					e, e)
				break
			}
			/* Work out whether to ignore it */
			if !ldone {
				a, y := rFilter.AllowsOut(lineNumber)
				/* Read the next one if not allowed */
				if !a {
					continue
				}
				/* Done checking lines if all are allowed or
				if we're past the upper limit */
				if ranges.AllMatch == y || ranges.Above == y {
					ldone = true
				}
			}
			/* Roll an output slice */
			orec := make([]string, 0, orsize)
			cdone := false /* Done worrying about columns */
			/* Add the right columns */
			for i := 1; i <= len(record); i++ {
				/* Work out whether to add this column */
				if !cdone {
					a, y := cFilter.AllowsOut(i)
					if !a {
						continue
					}
					/* Done checking if upper limit or
					all allowed */
					if ranges.AllMatch == y ||
						ranges.Above == y {
						cdone = true
					}
				}
				orec = append(orec, record[i-1])
			}
			orsize = len(orec)

			/* Actually output line */
			if err := w.Write(orec); err != nil {
				inform("Error writing %v: %v", orec, err)
				os.Exit(-8)
			}
		}
		/* TODO: Finish this */
		/* Flush output after each file */
		w.Flush()
		if err := w.Error(); err != nil {
			inform("Error flushing output: %v", err)
			os.Exit(-6)
		}
	}

}

/* Check if is is true.  If it is and s is true, die with an error.  If is is
true and s isn't, make s true.  If it's false all is good. */
func checkStdin(s *bool, is bool) {
	/* Don't care if is isn't */
	if !is {
		return
	}
	/* Set s if it's not set */
	if !*s {
		*s = true
		return
	}
	/* If both are set, die with an error. */
	inform("Only one of -csvfile, -rowfile, or -colfile may come from " +
		"the standard input.\n")
	os.Exit(-1)
}

/* mkFilter makes a filter from the specified flagfile (i.e. rowfile) and flag
(i.e. rows).  Name is passed in for error reporting. */
func mkFilter(flag, flagfile, name string) ranges.Filter {
	debug("Making %v filter from flag [%v] and file [%v]", name, flag,
		flagfile)
	/* Filter to return */
	f := ranges.New(verbose, debug)
	/* If we have nothing to set, return a permissive filter */
	if "" == flag && "" == flagfile {
		f.All = true
		return f
	}

	/* Process ranges on the command line */
	if "" != flag {
		verbose("Processing %v ranges from the commandline (%v)", name, flag)
		if err := f.Update(flag); err != nil {
			inform("Unable to process %v ranges (%v): %v", name,
				flag, err)
			os.Exit(-3)
		}

	}

	/* Will be what we read from */
	var in *os.File
	/* Populate in with something */
	if "" != flagfile {
		if "-" == flagfile {
			in = os.Stdin
		} else {
			i, err := os.Open(flagfile)
			if err != nil {
				inform("Unable to open %v file %v: %v",
					name, flagfile, err)
				os.Exit(-2)
			}
			in = i
		}
		/* Read lines from the file */
		scanner := bufio.NewScanner(in)
		fname := flagfile
		if "-" == fname {
			fname = "standard input"
		}
		for scanner.Scan() {
			t := scanner.Text()
			verbose("Processing %v from %v", t, fname)
			if err := f.Update(t); err != nil {
				inform("Unable to process %v ranges from "+
					"%v: %v", name, fname, err)
				os.Exit(-7)
			}
		}
		if err := scanner.Err(); err != nil {
			inform("Error reading from %v: %v", fname, err)
			os.Exit(-4)
		}
	}

	return f
}

/* verbose prints a message if -v */
func verbose(f string, a ...interface{}) {
	if *gc.verbose || *gc.debug {
		inform(f, a...)
	}
}

/* debug prints a message if -v or -d */
func debug(f string, a ...interface{}) {
	f = "D: " + f /* Note as a debug string */
	if *gc.debug {
		inform(f, a...)
	}
}

/* inform prints informational mesages to stderr */
func inform(f string, a ...interface{}) {
	if !strings.HasSuffix(f, "\n") {
		f += "\n"
	}
	fmt.Fprintf(os.Stderr, f, a...)
}
<|endoftext|>


<commit_before>package foursquarego_test

import (
	"fmt"
	"testing"

	"github.com/peppage/foursquarego"
)

const CLIENT_ID = ""
const CLIENT_SECRET = ""

var api *foursquarego.FoursquareApi

func init() {
	api = foursquarego.NewFoursquareApi(CLIENT_ID, CLIENT_SECRET)
}

func Test_FoursquareApi_GetVenue(t *testing.T) {
	api = foursquarego.NewFoursquareApi(CLIENT_ID, CLIENT_SECRET)
	v, _ := api.GetVenue("40a55d80f964a52020f31ee3")
}

func Test_FoursquareApi_GetCategories(t *testing.T) {
	api = foursquarego.NewFoursquareApi(CLIENT_ID, CLIENT_SECRET)
	_ = "breakpoint"
	c, _ := api.GetCategories()
	fmt.Println(c)
}
<commit_msg>rewrite tests<commit_after>package foursquarego_test

import (
	"os"
	"testing"

	"github.com/peppage/foursquarego"
)

var CLIENT_ID = os.Getenv("CLIENT_ID")
var CLIENT_SECRET = os.Getenv("CLIENT_SECRET")

var api *foursquarego.FoursquareApi

func init() {
	api = foursquarego.NewFoursquareApi(CLIENT_ID, CLIENT_SECRET)
}

// Test cna fetch venu & marshalling is OK
func Test_FoursquareApi_Venue(t *testing.T) {
	const venueId = "40a55d80f964a52020f31ee3"
	const venueName = "Clinton St. Baking Co. & Restaurant"
	api = foursquarego.NewFoursquareApi(CLIENT_ID, CLIENT_SECRET)
	v, err := api.Venue(venueId)
	if err != nil {
		t.Errorf("Getting a Venue returned error: %s", err.Error())
	}
	if v.Name != venueName {
		t.Errorf("Venue %s contained incorrect text. Recieved %s", venueId, v.Name)
	}
}

func Test_FoursquareApi_VenuPhotos(t *testing.T) {
	const venueId = "40a55d80f964a52020f31ee3"
	api = foursquarego.NewFoursquareApi(CLIENT_ID, CLIENT_SECRET)
	p, err := api.VenuePhotos(venueId, nil)
	if err != nil {
		t.Errorf("Getting venue photos returned error: %s", err.Error())
	}
	if len(p) < 1 {
		t.Errorf("Expected 1 or more photo, found %d", len(p))
	}
}

// Events are very
func Test_FoursquareApi_VenueEvents(t *testing.T) {
	const venueId = "451d2920f964a5208d3a1fe3"
	api = foursquarego.NewFoursquareApi(CLIENT_ID, CLIENT_SECRET)
	e, err := api.VenueEvents(venueId)
	if err != nil {
		t.Errorf("Getting venue events returned error: %s", err.Error())
	}
	if len(e) < 1 {
		t.Errorf("Exepected 1 or more event, found %d", len(e))
	}
}

func Test_FoursquareApi_Categories(t *testing.T) {
	api = foursquarego.NewFoursquareApi(CLIENT_ID, CLIENT_SECRET)
	c, err := api.Categories()
	if err != nil {
		t.Errorf("Getting categories returned error: %s", err.Error())
	}
	if len(c) == 0 {
		t.Errorf("Get categories returned no categories")
	}
}
<|endoftext|>


<commit_before>package main

import (
	"encoding/json"
	"fmt"
	"html/template"
	"net/http"
	"path"
)

//Container a single container
type Container struct {
	ID         string `json:"id"`
	Name       string `json:"name"`
	RunningCmd string `json:"command"`
	Port       string `json:"port"`
}

//Agent post data from an agent with ID and containers it has.
type Agent struct {
	AgentID            string      `json:"agentid"`
	ExpireAfterSeconds int         `json:"expireAfterSeconds"`
	Containers         []Container `json:"containers"`
}

func index(w http.ResponseWriter, r *http.Request) {
	resp, _ := http.Get("http://localhost:8989/api/1/list")
	agents := []Agent{}
	decoder := json.NewDecoder(resp.Body)
	err := decoder.Decode(&agents)
	if err != nil {
		fmt.Fprintf(w, "error occured:"+err.Error())
		return
	}

	lp := path.Join("templates", "layout.html")
	tmpl, err := template.ParseFiles(lp)
	if err != nil {
		fmt.Fprintf(w, "error occured:"+err.Error())
		return
	}
	tmpl.Execute(w, agents)
}

func main() {
	http.HandleFunc("/", index)
	fs := http.FileServer(http.Dir("static"))
	http.Handle("/static/", http.StripPrefix("/static/", fs))
	http.ListenAndServe(":9191", nil)
}
<commit_msg>Added server variable to startup.<commit_after>package main

import (
	"encoding/json"
	"flag"
	"fmt"
	"html/template"
	"net/http"
	"path"
)

//Container a single container
type Container struct {
	ID         string `json:"id"`
	Name       string `json:"name"`
	RunningCmd string `json:"command"`
	Port       string `json:"port"`
}

//Agent post data from an agent with ID and containers it has.
type Agent struct {
	AgentID            string      `json:"agentid"`
	ExpireAfterSeconds int         `json:"expireAfterSeconds"`
	Containers         []Container `json:"containers"`
}

var server string

func init() {
	flag.StringVar(&server, "server", "http://localhost:8989", "Full URL for the server.")
	flag.Parse()
}

func index(w http.ResponseWriter, r *http.Request) {
	resp, _ := http.Get(server + "/api/1/list")
	agents := []Agent{}
	decoder := json.NewDecoder(resp.Body)
	err := decoder.Decode(&agents)
	if err != nil {
		fmt.Fprintf(w, "error occured:"+err.Error())
		return
	}

	lp := path.Join("templates", "layout.html")
	tmpl, err := template.ParseFiles(lp)
	if err != nil {
		fmt.Fprintf(w, "error occured:"+err.Error())
		return
	}
	tmpl.Execute(w, agents)
}

func main() {
	http.HandleFunc("/", index)
	fs := http.FileServer(http.Dir("static"))
	http.Handle("/static/", http.StripPrefix("/static/", fs))
	http.ListenAndServe(":9191", nil)
}
<|endoftext|>


<commit_before>package main_test

import (
	"bytes"
	"fmt"
	"io"
	"io/ioutil"
	"log"
	"math/rand"
	"os"
	"os/exec"
	"time"

	. "github.com/onsi/ginkgo"
	. "github.com/onsi/gomega"
	"github.com/onsi/gomega/gbytes"
	"github.com/onsi/gomega/gexec"
)

var _ = Describe("Tee", func() {

	Context("when presented with a normal file", func() {
		var command *exec.Cmd
		var fname string = "test_tee.out"

		BeforeEach(func() {
			command = exec.Command(pathToTee, fname)
		})

		AfterEach(func() {
			err := os.Remove(fname)
			Expect(err).NotTo(HaveOccurred())
		})

		It("should return no error when run on a normal file", func() {
			stdin, e := command.StdinPipe()
			Expect(e).NotTo(HaveOccurred())
			defer stdin.Close()

			session, err := gexec.Start(command, GinkgoWriter, GinkgoWriter)
			Expect(err).NotTo(HaveOccurred())

			fmt.Fprintf(stdin, "test string\n")
			Eventually(session.Out).Should(gbytes.Say("test string.*"))
		})
	})

	Context("when appending an existing file", func() {
		var command *exec.Cmd
		var fname = "test_append.out"

		BeforeEach(func() {
			command = exec.Command(pathToTee,
				"-a",
				fname)

			// Create an empty file
			f, _ := os.OpenFile(fname, os.O_CREATE|os.O_WRONLY,
				os.FileMode(0666))
			f.Close()
		})

		AfterEach(func() {
			err := os.Remove(fname)
			Expect(err).NotTo(HaveOccurred())
		})

		It("should append to an empty file", func() {
			stdin, e := command.StdinPipe()
			Expect(e).NotTo(HaveOccurred())
			defer stdin.Close()

			session, err := gexec.Start(command, GinkgoWriter, GinkgoWriter)
			Expect(err).NotTo(HaveOccurred())

			fmt.Fprintf(stdin, "test\n")
			Eventually(session).Should(gbytes.Say(".*test.*"))

			buf, _ := ioutil.ReadFile(fname)
			benchmark := []byte("test\n")
			Expect(bytes.Equal(buf, benchmark)).To(Equal(true))
		})

		FIt("should append to a file with content", func() {
			stdin, e := command.StdinPipe()
			Expect(e).NotTo(HaveOccurred())
			defer stdin.Close()

			e = ioutil.WriteFile(fname, []byte("test\n"), os.FileMode(0666))
			Expect(e).NotTo(HaveOccurred())

			session, err := gexec.Start(command, GinkgoWriter, GinkgoWriter)
			Expect(err).NotTo(HaveOccurred())

			fmt.Fprintf(stdin, "another test\n")
			Eventually(session).Should(gbytes.Say("another test\n"))

			buf, _ := ioutil.ReadFile(fname)
			benchmark := []byte("test\nanother test\n")
			Expect(bytes.Equal(buf, benchmark)).To(Equal(true))
		})
	})

	Context("when presented with an invalid file", func() {
		var command *exec.Cmd
		var session *gexec.Session
		var stdin io.WriteCloser

		rand.Seed(time.Now().UTC().UnixNano())
		const chars = "abcdefghijklmnopqrstuvwxyx0123456789"
		nameByte := make([]byte, 64)
		for i := 0; i < 64; i++ {
			nameByte[i] = chars[rand.Intn(len(chars))]
		}
		name := string(nameByte)

		JustBeforeEach(func() {
			var err error

			f, ferr := os.OpenFile(name, os.O_CREATE, os.FileMode(0000))
			if ferr != nil {
				log.Fatalf("Error: %v\n", ferr)
			}
			f.Close()

			command = exec.Command(pathToTee, name)
			stdin, err = command.StdinPipe()
			Expect(err).NotTo(HaveOccurred())
			defer stdin.Close()

			session, _ = gexec.Start(command, GinkgoWriter,
				GinkgoWriter)
		})

		AfterEach(func() {
			err := os.Remove(name)
			Expect(err).NotTo(HaveOccurred())
		})

		It("should return a useful error message", func() {
			Eventually(session.Err).Should(gbytes.Say(".*Error:.*"))
			fmt.Fprintf(stdin, "test")
			session.Wait()
		})

		It("should return a non zero return code", func() {
			Eventually(session).ShouldNot(gexec.Exit(0))
			session.Wait()
		})

	})

})
<commit_msg>Remove focus from tee test<commit_after>package main_test

import (
	"bytes"
	"fmt"
	"io"
	"io/ioutil"
	"log"
	"math/rand"
	"os"
	"os/exec"
	"time"

	. "github.com/onsi/ginkgo"
	. "github.com/onsi/gomega"
	"github.com/onsi/gomega/gbytes"
	"github.com/onsi/gomega/gexec"
)

var _ = Describe("Tee", func() {

	Context("when presented with a normal file", func() {
		var command *exec.Cmd
		var fname string = "test_tee.out"

		BeforeEach(func() {
			command = exec.Command(pathToTee, fname)
		})

		AfterEach(func() {
			err := os.Remove(fname)
			Expect(err).NotTo(HaveOccurred())
		})

		It("should return no error when run on a normal file", func() {
			stdin, e := command.StdinPipe()
			Expect(e).NotTo(HaveOccurred())
			defer stdin.Close()

			session, err := gexec.Start(command, GinkgoWriter, GinkgoWriter)
			Expect(err).NotTo(HaveOccurred())

			fmt.Fprintf(stdin, "test string\n")
			Eventually(session.Out).Should(gbytes.Say("test string.*"))
		})
	})

	Context("when appending an existing file", func() {
		var command *exec.Cmd
		var fname = "test_append.out"

		BeforeEach(func() {
			command = exec.Command(pathToTee,
				"-a",
				fname)

			// Create an empty file
			f, _ := os.OpenFile(fname, os.O_CREATE|os.O_WRONLY,
				os.FileMode(0666))
			f.Close()
		})

		AfterEach(func() {
			err := os.Remove(fname)
			Expect(err).NotTo(HaveOccurred())
		})

		It("should append to an empty file", func() {
			stdin, e := command.StdinPipe()
			Expect(e).NotTo(HaveOccurred())
			defer stdin.Close()

			session, err := gexec.Start(command, GinkgoWriter, GinkgoWriter)
			Expect(err).NotTo(HaveOccurred())

			fmt.Fprintf(stdin, "test\n")
			Eventually(session).Should(gbytes.Say(".*test.*"))

			buf, _ := ioutil.ReadFile(fname)
			benchmark := []byte("test\n")
			Expect(bytes.Equal(buf, benchmark)).To(Equal(true))
		})

		It("should append to a file with content", func() {
			stdin, e := command.StdinPipe()
			Expect(e).NotTo(HaveOccurred())
			defer stdin.Close()

			e = ioutil.WriteFile(fname, []byte("test\n"), os.FileMode(0666))
			Expect(e).NotTo(HaveOccurred())

			session, err := gexec.Start(command, GinkgoWriter, GinkgoWriter)
			Expect(err).NotTo(HaveOccurred())

			fmt.Fprintf(stdin, "another test\n")
			Eventually(session).Should(gbytes.Say("another test\n"))

			buf, _ := ioutil.ReadFile(fname)
			benchmark := []byte("test\nanother test\n")
			Expect(bytes.Equal(buf, benchmark)).To(Equal(true))
		})
	})

	Context("when presented with an invalid file", func() {
		var command *exec.Cmd
		var session *gexec.Session
		var stdin io.WriteCloser

		rand.Seed(time.Now().UTC().UnixNano())
		const chars = "abcdefghijklmnopqrstuvwxyx0123456789"
		nameByte := make([]byte, 64)
		for i := 0; i < 64; i++ {
			nameByte[i] = chars[rand.Intn(len(chars))]
		}
		name := string(nameByte)

		JustBeforeEach(func() {
			var err error

			f, ferr := os.OpenFile(name, os.O_CREATE, os.FileMode(0000))
			if ferr != nil {
				log.Fatalf("Error: %v\n", ferr)
			}
			f.Close()

			command = exec.Command(pathToTee, name)
			stdin, err = command.StdinPipe()
			Expect(err).NotTo(HaveOccurred())
			defer stdin.Close()

			session, _ = gexec.Start(command, GinkgoWriter,
				GinkgoWriter)
		})

		AfterEach(func() {
			err := os.Remove(name)
			Expect(err).NotTo(HaveOccurred())
		})

		It("should return a useful error message", func() {
			Eventually(session.Err).Should(gbytes.Say(".*Error:.*"))
			fmt.Fprintf(stdin, "test")
			session.Wait()
		})

		It("should return a non zero return code", func() {
			Eventually(session).ShouldNot(gexec.Exit(0))
			session.Wait()
		})

	})

})
<|endoftext|>


<commit_before>// Copyright 2011 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

// +build windows

// Package terminal provides support functions for dealing with terminals, as
// commonly found on UNIX systems.
//
// Putting a terminal into raw mode is the most common requirement:
//
// 	oldState, err := terminal.MakeRaw(0)
// 	if err != nil {
// 	        panic(err)
// 	}
// 	defer terminal.Restore(0, oldState)
package readline

import (
	"io"
	"syscall"
	"unsafe"
)

const (
	enableLineInput       = 2
	enableEchoInput       = 4
	enableProcessedInput  = 1
	enableWindowInput     = 8
	enableMouseInput      = 16
	enableInsertMode      = 32
	enableQuickEditMode   = 64
	enableExtendedFlags   = 128
	enableAutoPosition    = 256
	enableProcessedOutput = 1
	enableWrapAtEolOutput = 2
)

var kernel32 = syscall.NewLazyDLL("kernel32.dll")

var (
	procGetConsoleMode             = kernel32.NewProc("GetConsoleMode")
	procSetConsoleMode             = kernel32.NewProc("SetConsoleMode")
	procGetConsoleScreenBufferInfo = kernel32.NewProc("GetConsoleScreenBufferInfo")
)

type (
	short int16
	word  uint16

	coord struct {
		x short
		y short
	}
	smallRect struct {
		left   short
		top    short
		right  short
		bottom short
	}
	consoleScreenBufferInfo struct {
		size              coord
		cursorPosition    coord
		attributes        word
		window            smallRect
		maximumWindowSize coord
	}
)

type State struct {
	mode uint32
}

// IsTerminal returns true if the given file descriptor is a terminal.
func IsTerminal(fd int) bool {
	var st uint32
	r, _, e := syscall.Syscall(procGetConsoleMode.Addr(), 2, uintptr(fd), uintptr(unsafe.Pointer(&st)), 0)
	return r != 0 && e == 0
}

// MakeRaw put the terminal connected to the given file descriptor into raw
// mode and returns the previous state of the terminal so that it can be
// restored.
func MakeRaw(fd int) (*State, error) {
	var st uint32
	_, _, e := syscall.Syscall(procGetConsoleMode.Addr(), 2, uintptr(fd), uintptr(unsafe.Pointer(&st)), 0)
	if e != 0 {
		return nil, error(e)
	}
	raw := st &^ (enableEchoInput | enableProcessedInput | enableLineInput | enableProcessedOutput)
	_, _, e = syscall.Syscall(procSetConsoleMode.Addr(), 2, uintptr(fd), uintptr(raw), 0)
	if e != 0 {
		return nil, error(e)
	}
	return &State{st}, nil
}

// GetState returns the current state of a terminal which may be useful to
// restore the terminal after a signal.
func GetState(fd int) (*State, error) {
	var st uint32
	_, _, e := syscall.Syscall(procGetConsoleMode.Addr(), 2, uintptr(fd), uintptr(unsafe.Pointer(&st)), 0)
	if e != 0 {
		return nil, error(e)
	}
	return &State{st}, nil
}

// Restore restores the terminal connected to the given file descriptor to a
// previous state.
func restoreTerm(fd int, state *State) error {
	_, _, err := syscall.Syscall(procSetConsoleMode.Addr(), 2, uintptr(fd), uintptr(state.mode), 0)
	return err
}

// GetSize returns the dimensions of the given terminal.
func GetSize(fd int) (width, height int, err error) {
	var info consoleScreenBufferInfo
	_, _, e := syscall.Syscall(procGetConsoleScreenBufferInfo.Addr(), 2, uintptr(fd), uintptr(unsafe.Pointer(&info)), 0)
	if e != 0 {
		return 0, 0, error(e)
	}
	return int(info.size.x), int(info.size.y), nil
}

// ReadPassword reads a line of input from a terminal without local echo.  This
// is commonly used for inputting passwords and other sensitive data. The slice
// returned does not include the \n.
func ReadPassword(fd int) ([]byte, error) {
	var st uint32
	_, _, e := syscall.Syscall(procGetConsoleMode.Addr(), 2, uintptr(fd), uintptr(unsafe.Pointer(&st)), 0)
	if e != 0 {
		return nil, error(e)
	}
	old := st

	st &^= (enableEchoInput)
	st |= (enableProcessedInput | enableLineInput | enableProcessedOutput)
	_, _, e = syscall.Syscall(procSetConsoleMode.Addr(), 2, uintptr(fd), uintptr(st), 0)
	if e != 0 {
		return nil, error(e)
	}

	defer func() {
		syscall.Syscall(procSetConsoleMode.Addr(), 2, uintptr(fd), uintptr(old), 0)
	}()

	var buf [16]byte
	var ret []byte
	for {
		n, err := syscall.Read(syscall.Handle(fd), buf[:])
		if err != nil {
			return nil, err
		}
		if n == 0 {
			if len(ret) == 0 {
				return nil, io.EOF
			}
			break
		}
		if buf[n-1] == '\n' {
			n--
		}
		if n > 0 && buf[n-1] == '\r' {
			n--
		}
		ret = append(ret, buf[:n]...)
		if n < len(buf) {
			break
		}
	}

	return ret, nil
}
<commit_msg>fix windows compile error<commit_after>// Copyright 2011 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

// +build windows

// Package terminal provides support functions for dealing with terminals, as
// commonly found on UNIX systems.
//
// Putting a terminal into raw mode is the most common requirement:
//
// 	oldState, err := terminal.MakeRaw(0)
// 	if err != nil {
// 	        panic(err)
// 	}
// 	defer terminal.Restore(0, oldState)
package readline

import (
	"io"
	"syscall"
	"unsafe"
)

const (
	enableLineInput       = 2
	enableEchoInput       = 4
	enableProcessedInput  = 1
	enableWindowInput     = 8
	enableMouseInput      = 16
	enableInsertMode      = 32
	enableQuickEditMode   = 64
	enableExtendedFlags   = 128
	enableAutoPosition    = 256
	enableProcessedOutput = 1
	enableWrapAtEolOutput = 2
)

var kernel32 = syscall.NewLazyDLL("kernel32.dll")

var (
	procGetConsoleMode             = kernel32.NewProc("GetConsoleMode")
	procSetConsoleMode             = kernel32.NewProc("SetConsoleMode")
	procGetConsoleScreenBufferInfo = kernel32.NewProc("GetConsoleScreenBufferInfo")
)

type (
	coord struct {
		x short
		y short
	}
	smallRect struct {
		left   short
		top    short
		right  short
		bottom short
	}
	consoleScreenBufferInfo struct {
		size              coord
		cursorPosition    coord
		attributes        word
		window            smallRect
		maximumWindowSize coord
	}
)

type State struct {
	mode uint32
}

// IsTerminal returns true if the given file descriptor is a terminal.
func IsTerminal(fd int) bool {
	var st uint32
	r, _, e := syscall.Syscall(procGetConsoleMode.Addr(), 2, uintptr(fd), uintptr(unsafe.Pointer(&st)), 0)
	return r != 0 && e == 0
}

// MakeRaw put the terminal connected to the given file descriptor into raw
// mode and returns the previous state of the terminal so that it can be
// restored.
func MakeRaw(fd int) (*State, error) {
	var st uint32
	_, _, e := syscall.Syscall(procGetConsoleMode.Addr(), 2, uintptr(fd), uintptr(unsafe.Pointer(&st)), 0)
	if e != 0 {
		return nil, error(e)
	}
	raw := st &^ (enableEchoInput | enableProcessedInput | enableLineInput | enableProcessedOutput)
	_, _, e = syscall.Syscall(procSetConsoleMode.Addr(), 2, uintptr(fd), uintptr(raw), 0)
	if e != 0 {
		return nil, error(e)
	}
	return &State{st}, nil
}

// GetState returns the current state of a terminal which may be useful to
// restore the terminal after a signal.
func GetState(fd int) (*State, error) {
	var st uint32
	_, _, e := syscall.Syscall(procGetConsoleMode.Addr(), 2, uintptr(fd), uintptr(unsafe.Pointer(&st)), 0)
	if e != 0 {
		return nil, error(e)
	}
	return &State{st}, nil
}

// Restore restores the terminal connected to the given file descriptor to a
// previous state.
func restoreTerm(fd int, state *State) error {
	_, _, err := syscall.Syscall(procSetConsoleMode.Addr(), 2, uintptr(fd), uintptr(state.mode), 0)
	return err
}

// GetSize returns the dimensions of the given terminal.
func GetSize(fd int) (width, height int, err error) {
	var info consoleScreenBufferInfo
	_, _, e := syscall.Syscall(procGetConsoleScreenBufferInfo.Addr(), 2, uintptr(fd), uintptr(unsafe.Pointer(&info)), 0)
	if e != 0 {
		return 0, 0, error(e)
	}
	return int(info.size.x), int(info.size.y), nil
}

// ReadPassword reads a line of input from a terminal without local echo.  This
// is commonly used for inputting passwords and other sensitive data. The slice
// returned does not include the \n.
func ReadPassword(fd int) ([]byte, error) {
	var st uint32
	_, _, e := syscall.Syscall(procGetConsoleMode.Addr(), 2, uintptr(fd), uintptr(unsafe.Pointer(&st)), 0)
	if e != 0 {
		return nil, error(e)
	}
	old := st

	st &^= (enableEchoInput)
	st |= (enableProcessedInput | enableLineInput | enableProcessedOutput)
	_, _, e = syscall.Syscall(procSetConsoleMode.Addr(), 2, uintptr(fd), uintptr(st), 0)
	if e != 0 {
		return nil, error(e)
	}

	defer func() {
		syscall.Syscall(procSetConsoleMode.Addr(), 2, uintptr(fd), uintptr(old), 0)
	}()

	var buf [16]byte
	var ret []byte
	for {
		n, err := syscall.Read(syscall.Handle(fd), buf[:])
		if err != nil {
			return nil, err
		}
		if n == 0 {
			if len(ret) == 0 {
				return nil, io.EOF
			}
			break
		}
		if buf[n-1] == '\n' {
			n--
		}
		if n > 0 && buf[n-1] == '\r' {
			n--
		}
		ret = append(ret, buf[:n]...)
		if n < len(buf) {
			break
		}
	}

	return ret, nil
}
<|endoftext|>


<commit_before>/*
Copyright 2015 The Kubernetes Authors.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
    http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package e2e

import (
	"fmt"
	"regexp"
	"time"

	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
	"k8s.io/client-go/kubernetes"
	"k8s.io/kubernetes/test/e2e/framework"

	. "github.com/onsi/ginkgo"
	. "github.com/onsi/gomega"
)

var _ = framework.KubeDescribe("GPU job processing", func() {
	f := framework.NewDefaultFramework("gpu")
	var cs kubernetes.Interface

	BeforeEach(func() {
		cs = f.ClientSet
	})

	It("Should run a job on a gpu node [Slow] [Zalando] [GPU]", func() {
		ns := f.Namespace.Name
		nameprefix := "gpu-test-"
		labels := map[string]string{
			"application": "vector-add",
		}

		By("Creating a vector pod which runs on a GPU node")
		pod := createVectorPod(nameprefix, ns, labels)
		_, err := cs.CoreV1().Pods(ns).Create(pod)
		framework.ExpectNoError(err, fmt.Errorf("Could not create POD %s", pod.Name))
		framework.ExpectNoError(f.WaitForPodRunning(pod.Name))
		for {
			p, err := cs.CoreV1().Pods(ns).Get(pod.Name, metav1.GetOptions{})
			if err != nil {
				framework.ExpectNoError(err, fmt.Errorf("Could not get POD %s", pod.Name))
				return
			}
			if p.Status.ContainerStatuses[0].State.Terminated == nil {
				time.Sleep(10 * time.Second)
				continue
			}
			n := p.Status.ContainerStatuses[0].State.Terminated.ExitCode
			if n < 1 {
				logs, err := getPodLogs(cs, ns, pod.Name, "cuda-vector-add", false)
				framework.ExpectNoError(err, "Should be able to get logs for pod %v", pod.Name)
				regex := regexp.MustCompile("PASSED")
				if regex.MatchString(logs) {
					return
				}
				framework.ExpectNoError(err, "Expected vector job to succeed")
				return
			}
			framework.ExpectNoError(fmt.Errorf("Expected POD %s to terminate with exit code 0", pod.Name))
			return
		}
	})
})
<commit_msg>Cleanup imports<commit_after>/*
Copyright 2015 The Kubernetes Authors.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
    http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package e2e

import (
	"fmt"
	"regexp"
	"time"

	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
	"k8s.io/client-go/kubernetes"
	"k8s.io/kubernetes/test/e2e/framework"

	. "github.com/onsi/ginkgo"
)

var _ = framework.KubeDescribe("GPU job processing", func() {
	f := framework.NewDefaultFramework("gpu")
	var cs kubernetes.Interface

	BeforeEach(func() {
		cs = f.ClientSet
	})

	It("Should run a job on a gpu node [Slow] [Zalando] [GPU]", func() {
		ns := f.Namespace.Name
		nameprefix := "gpu-test-"
		labels := map[string]string{
			"application": "vector-add",
		}

		By("Creating a vector pod which runs on a GPU node")
		pod := createVectorPod(nameprefix, ns, labels)
		_, err := cs.CoreV1().Pods(ns).Create(pod)
		framework.ExpectNoError(err, fmt.Errorf("Could not create POD %s", pod.Name))
		framework.ExpectNoError(f.WaitForPodRunning(pod.Name))
		for {
			p, err := cs.CoreV1().Pods(ns).Get(pod.Name, metav1.GetOptions{})
			if err != nil {
				framework.ExpectNoError(err, fmt.Errorf("Could not get POD %s", pod.Name))
				return
			}
			if p.Status.ContainerStatuses[0].State.Terminated == nil {
				time.Sleep(10 * time.Second)
				continue
			}
			n := p.Status.ContainerStatuses[0].State.Terminated.ExitCode
			if n < 1 {
				logs, err := getPodLogs(cs, ns, pod.Name, "cuda-vector-add", false)
				framework.ExpectNoError(err, "Should be able to get logs for pod %v", pod.Name)
				regex := regexp.MustCompile("PASSED")
				if regex.MatchString(logs) {
					return
				}
				framework.ExpectNoError(err, "Expected vector job to succeed")
				return
			}
			framework.ExpectNoError(fmt.Errorf("Expected POD %s to terminate with exit code 0", pod.Name))
			return
		}
	})
})
<|endoftext|>