<commit_before>// Copyright (c) 2012, Suryandaru Triandana <syndtr@gmail.com>
// All rights reserved.
//
// Use of this source code is governed by a BSD-style license that can be
// found in the LICENSE file.

// Package opt provides sets of options used by LevelDB.
package opt

import (
	"errors"
	"sync"

	"github.com/syndtr/goleveldb/leveldb/cache"
	"github.com/syndtr/goleveldb/leveldb/comparer"
	"github.com/syndtr/goleveldb/leveldb/filter"
)

var (
	ErrInvalid    = errors.New("invalid value")
	ErrNotSet     = errors.New("not set")
	ErrNotAllowed = errors.New("not allowed")
)

const (
	DefaultWriteBuffer          = 4 << 20
	DefaultMaxOpenFiles         = 1000
	DefaultBlockCacheSize       = 8 << 20
	DefaultBlockSize            = 4096
	DefaultBlockRestartInterval = 16
	DefaultCompressionType      = SnappyCompression
)

type OptionsFlag uint

const (
	// If set, the database will be created if it is missing.
	OFCreateIfMissing OptionsFlag = 1 << iota

	// If set, an error is raised if the database already exists.
	OFErrorIfExist

	// If set, the implementation will do aggressive checking of the
	// data it is processing and will stop early if it detects any
	// errors.  This may have unforeseen ramifications: for example, a
	// corruption of one DB entry may cause a large number of entries to
	// become unreadable or for the entire DB to become unopenable.
	OFParanoidCheck
)

// Database compression type
type Compression uint

func (c Compression) String() string {
	switch c {
	case DefaultCompression:
		return "default"
	case NoCompression:
		return "none"
	case SnappyCompression:
		return "snappy"
	}
	return "unknown"
}

const (
	DefaultCompression Compression = iota
	NoCompression
	SnappyCompression
	nCompression
)

// Options represent sets of LevelDB options.
type Options struct {
	// Comparer used to define the order of keys in the table.
	// Default: a comparer that uses lexicographic byte-wise ordering
	//
	// REQUIRES: The client must ensure that the comparer supplied
	// here has the same name and orders keys *exactly* the same as the
	// comparer provided to previous open calls on the same DB.
	Comparer comparer.Comparer

	// Specify the database flag.
	Flag OptionsFlag

	// Amount of data to build up in memory (backed by an unsorted journal
	// on disk) before converting to a sorted on-disk file.
	//
	// Larger values increase performance, especially during bulk loads.
	// Up to two write buffers may be held in memory at the same time,
	// so you may wish to adjust this parameter to control memory usage.
	// Also, a larger write buffer will result in a longer recovery time
	// the next time the database is opened.
	//
	// Default: 4MB
	WriteBuffer int

	// Number of open files that can be used by the DB.  You may need to
	// increase this if your database has a large working set (budget
	// one open file per 2MB of working set).
	//
	// Default: 1000
	MaxOpenFiles int

	// Control over blocks (user data is stored in a set of blocks, and
	// a block is the unit of reading from disk).

	// If non-NULL, use the specified cache for blocks.
	// If NULL, leveldb will automatically create and use an 8MB internal cache.
	// Default: NULL
	BlockCache cache.Cache

	// Approximate size of user data packed per block.  Note that the
	// block size specified here corresponds to uncompressed data.  The
	// actual size of the unit read from disk may be smaller if
	// compression is enabled.  This parameter can be changed dynamically.
	//
	// Default: 4K
	BlockSize int

	// Number of keys between restart points for delta encoding of keys.
	// This parameter can be changed dynamically.  Most clients should
	// leave this parameter alone.
	//
	// Default: 16
	BlockRestartInterval int

	// Compress blocks using the specified compression algorithm.  This
	// parameter can be changed dynamically.
	//
	// Default: kSnappyCompression, which gives lightweight but fast
	// compression.
	//
	// Typical speeds of kSnappyCompression on an Intel(R) Core(TM)2 2.4GHz:
	//    ~200-500MB/s compression
	//    ~400-800MB/s decompression
	// Note that these speeds are significantly faster than most
	// persistent storage speeds, and therefore it is typically never
	// worth switching to kNoCompression.  Even if the input data is
	// incompressible, the kSnappyCompression implementation will
	// efficiently detect that and will switch to uncompressed mode.
	CompressionType Compression

	// If non-NULL, use the specified filter policy to reduce disk reads.
	// Many applications will benefit from passing the result of
	// NewBloomFilter() here.
	//
	// Default: NULL
	Filter filter.Filter

	mu sync.RWMutex
}

// OptionsGetter wraps methods used to get sanitized options.
type OptionsGetter interface {
	GetComparer() comparer.Comparer
	HasFlag(flag OptionsFlag) bool
	GetWriteBuffer() int
	GetMaxOpenFiles() int
	GetBlockCache() cache.Cache
	GetBlockSize() int
	GetBlockRestartInterval() int
	GetCompressionType() Compression
	GetFilter() filter.Filter
}

// OptionsSetter wraps methods used to set options.
type OptionsSetter interface {
	SetComparer(cmp comparer.Comparer) error
	SetFlag(flag OptionsFlag) error
	ClearFlag(flag OptionsFlag) error
	SetWriteBuffer(size int) error
	SetMaxOpenFiles(max int) error
	SetBlockCache(cache cache.Cache) error
	SetBlockCacheCapacity(capacity int) error
	SetBlockSize(size int) error
	SetBlockRestartInterval(interval int) error
	SetCompressionType(compression Compression) error
	SetFilter(p filter.Filter) error
}

// Getter

func (o *Options) GetComparer() comparer.Comparer {
	if o == nil {
		return comparer.DefaultComparer
	}
	o.mu.RLock()
	defer o.mu.RUnlock()
	if o.Comparer == nil {
		return comparer.DefaultComparer
	}
	return o.Comparer
}

func (o *Options) HasFlag(flag OptionsFlag) bool {
	if o == nil {
		return false
	}
	o.mu.RLock()
	defer o.mu.RUnlock()
	return (o.Flag & flag) != 0
}

func (o *Options) GetWriteBuffer() int {
	if o == nil {
		return DefaultWriteBuffer
	}
	o.mu.RLock()
	defer o.mu.RUnlock()
	if o.WriteBuffer <= 0 {
		return DefaultWriteBuffer
	}
	return o.WriteBuffer
}

func (o *Options) GetMaxOpenFiles() int {
	if o == nil {
		return DefaultMaxOpenFiles
	}
	o.mu.RLock()
	defer o.mu.RUnlock()
	if o.MaxOpenFiles <= 0 {
		return DefaultMaxOpenFiles
	}
	return o.MaxOpenFiles
}

func (o *Options) GetBlockCache() cache.Cache {
	if o == nil {
		return nil
	}
	o.mu.RLock()
	defer o.mu.RUnlock()
	return o.BlockCache
}

func (o *Options) GetBlockSize() int {
	if o == nil {
		return DefaultBlockSize
	}
	o.mu.RLock()
	defer o.mu.RUnlock()
	if o.BlockSize <= 0 {
		return DefaultBlockSize
	}
	return o.BlockSize
}

func (o *Options) GetBlockRestartInterval() int {
	if o == nil {
		return DefaultBlockRestartInterval
	}
	o.mu.RLock()
	defer o.mu.RUnlock()
	if o.BlockRestartInterval <= 0 {
		return DefaultBlockRestartInterval
	}
	return o.BlockRestartInterval
}

func (o *Options) GetCompressionType() Compression {
	if o == nil {
		return DefaultCompressionType
	}
	o.mu.RLock()
	defer o.mu.RUnlock()
	if o.CompressionType <= DefaultCompression || o.CompressionType >= nCompression {
		return DefaultCompressionType
	}
	return o.CompressionType
}

func (o *Options) GetFilter() filter.Filter {
	if o == nil {
		return nil
	}
	o.mu.RLock()
	defer o.mu.RUnlock()
	return o.Filter
}

// Setter

func (o *Options) SetComparer(cmp comparer.Comparer) error {
	if o == nil {
		return ErrNotSet
	}
	if cmp == nil {
		return ErrInvalid
	}
	o.mu.Lock()
	o.Comparer = cmp
	o.mu.Unlock()
	return nil
}

func (o *Options) SetFlag(flag OptionsFlag) error {
	if o == nil {
		return ErrNotSet
	}
	o.mu.Lock()
	o.Flag |= flag
	o.mu.Unlock()
	return nil
}

func (o *Options) ClearFlag(flag OptionsFlag) error {
	if o == nil {
		return ErrNotSet
	}
	o.mu.Lock()
	o.Flag &= ^flag
	o.mu.Unlock()
	return nil
}

func (o *Options) SetWriteBuffer(size int) error {
	if o == nil {
		return ErrNotSet
	}
	if size <= 0 {
		return ErrInvalid
	}
	o.mu.Lock()
	o.WriteBuffer = size
	o.mu.Unlock()
	return nil
}

func (o *Options) SetMaxOpenFiles(max int) error {
	if o == nil {
		return ErrNotSet
	}
	if max <= 0 {
		return ErrInvalid
	}
	o.mu.Lock()
	o.MaxOpenFiles = max
	o.mu.Unlock()
	return nil
}

func (o *Options) SetBlockCache(cache cache.Cache) error {
	if o == nil {
		return ErrNotSet
	}
	o.mu.Lock()
	o.BlockCache = cache
	o.mu.Unlock()
	return nil
}

func (o *Options) SetBlockCacheCapacity(capacity int) error {
	if o == nil {
		return ErrNotSet
	}
	o.mu.Lock()
	if o.BlockCache == nil {
		return ErrNotSet
	}
	o.BlockCache.SetCapacity(capacity)
	o.mu.Unlock()
	return nil
}

func (o *Options) SetBlockSize(size int) error {
	if o == nil {
		return ErrNotSet
	}
	if size <= 0 {
		return ErrInvalid
	}
	o.mu.Lock()
	o.BlockSize = size
	o.mu.Unlock()
	return nil
}

func (o *Options) SetBlockRestartInterval(interval int) error {
	if o == nil {
		return ErrNotSet
	}
	if interval <= 0 {
		return ErrInvalid
	}
	o.mu.Lock()
	o.BlockRestartInterval = interval
	o.mu.Unlock()
	return nil
}

func (o *Options) SetCompressionType(compression Compression) error {
	if o == nil {
		return ErrNotSet
	}
	if o.CompressionType >= nCompression {
		return ErrInvalid
	}
	o.mu.Lock()
	o.CompressionType = compression
	o.mu.Unlock()
	return nil
}

func (o *Options) SetFilter(p filter.Filter) error {
	if o == nil {
		return ErrNotSet
	}
	o.mu.Lock()
	o.Filter = p
	o.mu.Unlock()
	return nil
}

type ReadOptionsFlag uint

const (
	// If true, all data read from underlying storage will be
	// verified against corresponding checksums.
	RFVerifyChecksums ReadOptionsFlag = 1 << iota

	// Should the data read for this iteration be cached in memory?
	// If set iteration chaching will be disabled.
	// Callers may wish to set this flag for bulk scans.
	RFDontFillCache
)

// ReadOptions represent sets of options used by LevelDB during read
// operations.
type ReadOptions struct {
	// Specify the read flag.
	Flag ReadOptionsFlag
}

type ReadOptionsGetter interface {
	HasFlag(flag ReadOptionsFlag) bool
}

func (o *ReadOptions) HasFlag(flag ReadOptionsFlag) bool {
	if o == nil {
		return false
	}
	return (o.Flag & flag) != 0
}

type WriteOptionsFlag uint

const (
	// If set, the write will be flushed from the operating system
	// buffer cache (by calling WritableFile::Sync()) before the write
	// is considered complete.  If this flag is true, writes will be
	// slower.
	//
	// If this flag is false, and the machine crashes, some recent
	// writes may be lost.  Note that if it is just the process that
	// crashes (i.e., the machine does not reboot), no writes will be
	// lost even if sync==false.
	//
	// In other words, a DB write with sync==false has similar
	// crash semantics as the "write()" system call.  A DB write
	// with sync==true has similar crash semantics to a "write()"
	// system call followed by "fsync()".
	WFSync WriteOptionsFlag = 1 << iota
)

// WriteOptions represent sets of options used by LevelDB during write
// operations.
type WriteOptions struct {
	// Specify the write flag.
	Flag WriteOptionsFlag
}

type WriteOptionsGetter interface {
	HasFlag(flag WriteOptionsFlag) bool
}

func (o *WriteOptions) HasFlag(flag WriteOptionsFlag) bool {
	if o == nil {
		return false
	}
	return (o.Flag & flag) != 0
}
<commit_msg>Documenting opt.Option.filter behaviour<commit_after>// Copyright (c) 2012, Suryandaru Triandana <syndtr@gmail.com>
// All rights reserved.
//
// Use of this source code is governed by a BSD-style license that can be
// found in the LICENSE file.

// Package opt provides sets of options used by LevelDB.
package opt

import (
	"errors"
	"sync"

	"github.com/syndtr/goleveldb/leveldb/cache"
	"github.com/syndtr/goleveldb/leveldb/comparer"
	"github.com/syndtr/goleveldb/leveldb/filter"
)

var (
	ErrInvalid    = errors.New("invalid value")
	ErrNotSet     = errors.New("not set")
	ErrNotAllowed = errors.New("not allowed")
)

const (
	DefaultWriteBuffer          = 4 << 20
	DefaultMaxOpenFiles         = 1000
	DefaultBlockCacheSize       = 8 << 20
	DefaultBlockSize            = 4096
	DefaultBlockRestartInterval = 16
	DefaultCompressionType      = SnappyCompression
)

type OptionsFlag uint

const (
	// If set, the database will be created if it is missing.
	OFCreateIfMissing OptionsFlag = 1 << iota

	// If set, an error is raised if the database already exists.
	OFErrorIfExist

	// If set, the implementation will do aggressive checking of the
	// data it is processing and will stop early if it detects any
	// errors.  This may have unforeseen ramifications: for example, a
	// corruption of one DB entry may cause a large number of entries to
	// become unreadable or for the entire DB to become unopenable.
	OFParanoidCheck
)

// Database compression type
type Compression uint

func (c Compression) String() string {
	switch c {
	case DefaultCompression:
		return "default"
	case NoCompression:
		return "none"
	case SnappyCompression:
		return "snappy"
	}
	return "unknown"
}

const (
	DefaultCompression Compression = iota
	NoCompression
	SnappyCompression
	nCompression
)

// Options represent sets of LevelDB options.
type Options struct {
	// Comparer used to define the order of keys in the table.
	// Default: a comparer that uses lexicographic byte-wise ordering
	//
	// REQUIRES: The client must ensure that the comparer supplied
	// here has the same name and orders keys *exactly* the same as the
	// comparer provided to previous open calls on the same DB.
	Comparer comparer.Comparer

	// Specify the database flag.
	Flag OptionsFlag

	// Amount of data to build up in memory (backed by an unsorted journal
	// on disk) before converting to a sorted on-disk file.
	//
	// Larger values increase performance, especially during bulk loads.
	// Up to two write buffers may be held in memory at the same time,
	// so you may wish to adjust this parameter to control memory usage.
	// Also, a larger write buffer will result in a longer recovery time
	// the next time the database is opened.
	//
	// Default: 4MB
	WriteBuffer int

	// Number of open files that can be used by the DB.  You may need to
	// increase this if your database has a large working set (budget
	// one open file per 2MB of working set).
	//
	// Default: 1000
	MaxOpenFiles int

	// Control over blocks (user data is stored in a set of blocks, and
	// a block is the unit of reading from disk).

	// If non-NULL, use the specified cache for blocks.
	// If NULL, leveldb will automatically create and use an 8MB internal cache.
	// Default: NULL
	BlockCache cache.Cache

	// Approximate size of user data packed per block.  Note that the
	// block size specified here corresponds to uncompressed data.  The
	// actual size of the unit read from disk may be smaller if
	// compression is enabled.  This parameter can be changed dynamically.
	//
	// Default: 4K
	BlockSize int

	// Number of keys between restart points for delta encoding of keys.
	// This parameter can be changed dynamically.  Most clients should
	// leave this parameter alone.
	//
	// Default: 16
	BlockRestartInterval int

	// Compress blocks using the specified compression algorithm.  This
	// parameter can be changed dynamically.
	//
	// Default: kSnappyCompression, which gives lightweight but fast
	// compression.
	//
	// Typical speeds of kSnappyCompression on an Intel(R) Core(TM)2 2.4GHz:
	//    ~200-500MB/s compression
	//    ~400-800MB/s decompression
	// Note that these speeds are significantly faster than most
	// persistent storage speeds, and therefore it is typically never
	// worth switching to kNoCompression.  Even if the input data is
	// incompressible, the kSnappyCompression implementation will
	// efficiently detect that and will switch to uncompressed mode.
	CompressionType Compression

	// If non-NULL, use the specified filter policy to reduce disk reads.
	// Many applications will benefit from passing the result of
	// NewBloomFilter() here.
	//
	// As long as the same filter (name) was used as last time the
	// database was opened, the previous filter is reused. That is,
	// the filter does not need to be rebuilt. This is made possible
	// since each filter is persisted to disk on a per sstable
	// basis.
	//
	// As opposed to the comparer, a filter can be replaced after a
	// database has been created. If this is done, the previous
	// persisted filter will be ignored for every old sstable.
	// Every new table will use the newly introduced filter. This
	// means that all/some sstables will lack a filter during a
	// transition period. Note that this might have an impact on
	// performance. Rewriting every single key/value will force
	// introduction of the new filter.
	//
	// Default: NULL
	Filter filter.Filter

	mu sync.RWMutex
}

// OptionsGetter wraps methods used to get sanitized options.
type OptionsGetter interface {
	GetComparer() comparer.Comparer
	HasFlag(flag OptionsFlag) bool
	GetWriteBuffer() int
	GetMaxOpenFiles() int
	GetBlockCache() cache.Cache
	GetBlockSize() int
	GetBlockRestartInterval() int
	GetCompressionType() Compression
	GetFilter() filter.Filter
}

// OptionsSetter wraps methods used to set options.
type OptionsSetter interface {
	SetComparer(cmp comparer.Comparer) error
	SetFlag(flag OptionsFlag) error
	ClearFlag(flag OptionsFlag) error
	SetWriteBuffer(size int) error
	SetMaxOpenFiles(max int) error
	SetBlockCache(cache cache.Cache) error
	SetBlockCacheCapacity(capacity int) error
	SetBlockSize(size int) error
	SetBlockRestartInterval(interval int) error
	SetCompressionType(compression Compression) error
	SetFilter(p filter.Filter) error
}

// Getter

func (o *Options) GetComparer() comparer.Comparer {
	if o == nil {
		return comparer.DefaultComparer
	}
	o.mu.RLock()
	defer o.mu.RUnlock()
	if o.Comparer == nil {
		return comparer.DefaultComparer
	}
	return o.Comparer
}

func (o *Options) HasFlag(flag OptionsFlag) bool {
	if o == nil {
		return false
	}
	o.mu.RLock()
	defer o.mu.RUnlock()
	return (o.Flag & flag) != 0
}

func (o *Options) GetWriteBuffer() int {
	if o == nil {
		return DefaultWriteBuffer
	}
	o.mu.RLock()
	defer o.mu.RUnlock()
	if o.WriteBuffer <= 0 {
		return DefaultWriteBuffer
	}
	return o.WriteBuffer
}

func (o *Options) GetMaxOpenFiles() int {
	if o == nil {
		return DefaultMaxOpenFiles
	}
	o.mu.RLock()
	defer o.mu.RUnlock()
	if o.MaxOpenFiles <= 0 {
		return DefaultMaxOpenFiles
	}
	return o.MaxOpenFiles
}

func (o *Options) GetBlockCache() cache.Cache {
	if o == nil {
		return nil
	}
	o.mu.RLock()
	defer o.mu.RUnlock()
	return o.BlockCache
}

func (o *Options) GetBlockSize() int {
	if o == nil {
		return DefaultBlockSize
	}
	o.mu.RLock()
	defer o.mu.RUnlock()
	if o.BlockSize <= 0 {
		return DefaultBlockSize
	}
	return o.BlockSize
}

func (o *Options) GetBlockRestartInterval() int {
	if o == nil {
		return DefaultBlockRestartInterval
	}
	o.mu.RLock()
	defer o.mu.RUnlock()
	if o.BlockRestartInterval <= 0 {
		return DefaultBlockRestartInterval
	}
	return o.BlockRestartInterval
}

func (o *Options) GetCompressionType() Compression {
	if o == nil {
		return DefaultCompressionType
	}
	o.mu.RLock()
	defer o.mu.RUnlock()
	if o.CompressionType <= DefaultCompression || o.CompressionType >= nCompression {
		return DefaultCompressionType
	}
	return o.CompressionType
}

func (o *Options) GetFilter() filter.Filter {
	if o == nil {
		return nil
	}
	o.mu.RLock()
	defer o.mu.RUnlock()
	return o.Filter
}

// Setter

func (o *Options) SetComparer(cmp comparer.Comparer) error {
	if o == nil {
		return ErrNotSet
	}
	if cmp == nil {
		return ErrInvalid
	}
	o.mu.Lock()
	o.Comparer = cmp
	o.mu.Unlock()
	return nil
}

func (o *Options) SetFlag(flag OptionsFlag) error {
	if o == nil {
		return ErrNotSet
	}
	o.mu.Lock()
	o.Flag |= flag
	o.mu.Unlock()
	return nil
}

func (o *Options) ClearFlag(flag OptionsFlag) error {
	if o == nil {
		return ErrNotSet
	}
	o.mu.Lock()
	o.Flag &= ^flag
	o.mu.Unlock()
	return nil
}

func (o *Options) SetWriteBuffer(size int) error {
	if o == nil {
		return ErrNotSet
	}
	if size <= 0 {
		return ErrInvalid
	}
	o.mu.Lock()
	o.WriteBuffer = size
	o.mu.Unlock()
	return nil
}

func (o *Options) SetMaxOpenFiles(max int) error {
	if o == nil {
		return ErrNotSet
	}
	if max <= 0 {
		return ErrInvalid
	}
	o.mu.Lock()
	o.MaxOpenFiles = max
	o.mu.Unlock()
	return nil
}

func (o *Options) SetBlockCache(cache cache.Cache) error {
	if o == nil {
		return ErrNotSet
	}
	o.mu.Lock()
	o.BlockCache = cache
	o.mu.Unlock()
	return nil
}

func (o *Options) SetBlockCacheCapacity(capacity int) error {
	if o == nil {
		return ErrNotSet
	}
	o.mu.Lock()
	if o.BlockCache == nil {
		return ErrNotSet
	}
	o.BlockCache.SetCapacity(capacity)
	o.mu.Unlock()
	return nil
}

func (o *Options) SetBlockSize(size int) error {
	if o == nil {
		return ErrNotSet
	}
	if size <= 0 {
		return ErrInvalid
	}
	o.mu.Lock()
	o.BlockSize = size
	o.mu.Unlock()
	return nil
}

func (o *Options) SetBlockRestartInterval(interval int) error {
	if o == nil {
		return ErrNotSet
	}
	if interval <= 0 {
		return ErrInvalid
	}
	o.mu.Lock()
	o.BlockRestartInterval = interval
	o.mu.Unlock()
	return nil
}

func (o *Options) SetCompressionType(compression Compression) error {
	if o == nil {
		return ErrNotSet
	}
	if o.CompressionType >= nCompression {
		return ErrInvalid
	}
	o.mu.Lock()
	o.CompressionType = compression
	o.mu.Unlock()
	return nil
}

func (o *Options) SetFilter(p filter.Filter) error {
	if o == nil {
		return ErrNotSet
	}
	o.mu.Lock()
	o.Filter = p
	o.mu.Unlock()
	return nil
}

type ReadOptionsFlag uint

const (
	// If true, all data read from underlying storage will be
	// verified against corresponding checksums.
	RFVerifyChecksums ReadOptionsFlag = 1 << iota

	// Should the data read for this iteration be cached in memory?
	// If set iteration chaching will be disabled.
	// Callers may wish to set this flag for bulk scans.
	RFDontFillCache
)

// ReadOptions represent sets of options used by LevelDB during read
// operations.
type ReadOptions struct {
	// Specify the read flag.
	Flag ReadOptionsFlag
}

type ReadOptionsGetter interface {
	HasFlag(flag ReadOptionsFlag) bool
}

func (o *ReadOptions) HasFlag(flag ReadOptionsFlag) bool {
	if o == nil {
		return false
	}
	return (o.Flag & flag) != 0
}

type WriteOptionsFlag uint

const (
	// If set, the write will be flushed from the operating system
	// buffer cache (by calling WritableFile::Sync()) before the write
	// is considered complete.  If this flag is true, writes will be
	// slower.
	//
	// If this flag is false, and the machine crashes, some recent
	// writes may be lost.  Note that if it is just the process that
	// crashes (i.e., the machine does not reboot), no writes will be
	// lost even if sync==false.
	//
	// In other words, a DB write with sync==false has similar
	// crash semantics as the "write()" system call.  A DB write
	// with sync==true has similar crash semantics to a "write()"
	// system call followed by "fsync()".
	WFSync WriteOptionsFlag = 1 << iota
)

// WriteOptions represent sets of options used by LevelDB during write
// operations.
type WriteOptions struct {
	// Specify the write flag.
	Flag WriteOptionsFlag
}

type WriteOptionsGetter interface {
	HasFlag(flag WriteOptionsFlag) bool
}

func (o *WriteOptions) HasFlag(flag WriteOptionsFlag) bool {
	if o == nil {
		return false
	}
	return (o.Flag & flag) != 0
}
<|endoftext|>


<commit_before>package main

import (
	"database/sql"
	"flag"
	"fmt"
	"io"
	"io/ioutil"
	"log"
	"os"
	"os/signal"
	"runtime/pprof"
	"strings"
	"time"
	"unicode/utf8"

	"golang.org/x/crypto/ssh/terminal"

	_ "github.com/go-sql-driver/mysql"
)

const (
	// Amount of CSV write data to buffer between flushes
	flushSize = 26214400 // 25MB

	// Timeout length where ctrl+c is ignored
	// If a second ctrl+c is sent before the timeout the program exits
	signalTimeout = 3 // Seconds

	// Timeout length to wait for a query string sent via stdin
	stdinTimeout = 10 // Milliseconds
)

type (
	// dbInfo contains information necessary to connect to a database
	dbInfo struct {
		user string
		pass string
		host string
		port string
		TLS  bool
	}
)

// ShowUsage prints a help screen
func showUsage() {
	fmt.Println(`
	mycsv usage:
	mycsv DB_COMMANDS [CSV OUTPUT FLAGS] [DEBUG FLAGS] [CSV OUTFILE] query

	EXAMPLES:
	mycsv -user=jprunier -pass= -file=my.csv -query="select * from jjp.example_table where filter in ('1', 'test', 'another')"
	echo "select * from mysql.plugin" | mycsv -user=jprunier -pass=mypass -host=remotedb -tls > my.csv
	mycsv -user=jprunier -pass= -file=my.csv -d="|" -q="'" < queryfile

	DATABASE FLAGS
	==============
	-user: Username (required)
	-pass: Password (interactive prompt if blank)
	-host: Database Host (localhost assumed if blank)
	-port: Port (3306 default)
	-tls:  Use TLS/SSL for database connection (false default)

	CSV FLAGS
	=========
	-file: CSV output filename (Write to stdout if not supplied)
	-query: MySQL query (required, can be sent via stdin redirection)
	-header: Print initial column name header line (true default)
	-d: CSV field delimiter ("," default)
	-q: CSV quote character ("\"" default)
	-e: CSV escape character ("\\" default)
	-t: CSV line terminator ("\n" default)
	-v: Print more information (false default)

	DEBUG FLAGS
	===========
	-debug_cpu: CPU debugging filename
	-debug_mem: Memory debugging filename

	`)
}

func main() {
	start := time.Now()

	// Profiling flags
	var cpuprofile = flag.String("debug_cpu", "", "CPU debugging filename")
	var memprofile = flag.String("debug_mem", "", "Memory debugging filename")

	// Database flags
	dbUser := flag.String("user", "", "Username (required)")
	dbPass := flag.String("pass", "", "Password (interactive prompt if blank)")
	dbHost := flag.String("host", "", "Database Host (localhost assumed if blank)")
	dbPort := flag.String("port", "3306", "Port")
	dbTLS := flag.Bool("tls", false, "Use TLS/SSL for database connection")

	// CSV format flags
	csvDelimiter := flag.String("d", `,`, "CSV field delimiter")
	csvQuote := flag.String("q", `"`, "CSV quote character")
	csvEscape := flag.String("e", `\`, "CSV escape character")
	csvTerminator := flag.String("t", "\n", "CSV line terminator")

	csvHeader := flag.Bool("header", true, "Print initial column name header line")
	csvFile := flag.String("file", "", "CSV output filename")
	sqlQuery := flag.String("query", "", "MySQL query")
	verbose := flag.Bool("v", false, "Print more information")

	// Parse flags
	flag.Parse()

	// Print usage
	if flag.NFlag() == 0 {
		showUsage()

		os.Exit(0)
	}

	// If query not provided read from standard in
	var query string
	queryChan := make(chan string)
	defer close(queryChan)
	if *sqlQuery == "" {
		go func() {
			b, err := ioutil.ReadAll(os.Stdin)
			checkErr(err)

			queryChan <- string(b)
		}()

		select {
		case q := <-queryChan:
			query = q
		case <-time.After(time.Millisecond * stdinTimeout):
			fmt.Fprintln(os.Stderr, "You must supply a query")
			os.Exit(1)
		}
	} else {
		query = *sqlQuery
	}

	// Make sure the query is a select
	if strings.ToLower(query[0:6]) != "select" {
		fmt.Fprintln(os.Stderr, "Query must be a select!")
		os.Exit(1)
	}

	// Check if an output file was supplied otherwise use standard out
	var writeTo string
	var writerDest io.Writer
	var err error
	if *csvFile == "" {
		writeTo = "standard out"
		writerDest = os.Stdout
	} else {
		f, err := os.Open(*csvFile)
		if err == nil {
			fmt.Fprintln(os.Stderr, *csvFile, "already exists!")
			fmt.Fprintln(os.Stderr, "Please remove it or use a different filename")
			os.Exit(1)
		}
		f.Close()
		writerDest, err = os.Create(*csvFile)
		writeTo = *csvFile
	}

	// Create a new CSV writer
	var i int
	CSVWriter := NewWriter(writerDest)
	CSVWriter.Delimiter, i = utf8.DecodeLastRuneInString(*csvDelimiter)
	if i == 0 {
		fmt.Fprintln(os.Stderr, "You must supply a valid delimiter character")
		os.Exit(1)
	}
	CSVWriter.Quote, i = utf8.DecodeLastRuneInString(*csvQuote)
	if i == 0 {
		fmt.Fprintln(os.Stderr, "You must supply a valid quote character")
		os.Exit(1)
	}
	CSVWriter.Escape, i = utf8.DecodeLastRuneInString(*csvEscape)
	if i == 0 {
		fmt.Fprintln(os.Stderr, "You must supply a valid escape character")
		os.Exit(1)
	}

	// Need literal string check here to see all 4 bytes instead of 2 (ascii 13 & 10)
	// Newline is default but check here in case it is manually passed in
	if *csvTerminator == `\r\n` {
		CSVWriter.Terminator = "\r\n"
	} else if *csvTerminator == `\n` {
		CSVWriter.Terminator = "\n"
	} else {
		CSVWriter.Terminator = *csvTerminator
	}

	if *verbose {
		fmt.Println("CSV output will be written to", writeTo)
	}

	// Check if Stdin has been redirected and reset so the user can be prompted for a password
	checkStdin()

	// Listen for SIGINT (ctrl+c)
	catchNotifications()

	// CPU Profiling
	if *cpuprofile != "" {
		f, err := os.Create(*cpuprofile)
		checkErr(err)
		pprof.StartCPUProfile(f)
		defer pprof.StopCPUProfile()
	}

	// Default to localhost if no host or socket provided
	if *dbHost == "" {
		*dbHost = "127.0.0.1"
	}

	// Need to provide a target
	if *dbUser == "" {
		fmt.Fprintln(os.Stderr, "You must provide a user name!")
		os.Exit(1)
	}

	// If password is blank prompt user
	if *dbPass == "" {
		fmt.Println("Enter password: ")
		pwd, err := terminal.ReadPassword(int(os.Stdin.Fd()))
		if err != nil {
			if err != io.EOF {
				checkErr(err)
			}
		}
		*dbPass = string(pwd)
	}

	// Populate dbInfo struct with cli flags
	dbi := dbInfo{user: *dbUser, pass: *dbPass, host: *dbHost, port: *dbPort, TLS: *dbTLS}

	// Create a *sql.DB connection to the source database
	db, err := dbi.Connect()
	defer db.Close()
	if err != nil {
		fmt.Fprintln(os.Stderr, err)
		os.Exit(1)
	}

	// Start reading & writing
	dataChan := make(chan []sql.RawBytes)
	quitChan := make(chan bool)
	goChan := make(chan bool)
	go readRows(db, query, dataChan, quitChan, goChan, *csvHeader)
	rowCount := writeCSV(CSVWriter, dataChan, goChan, *verbose)

	<-quitChan
	close(quitChan)
	close(goChan)

	// Memory Profiling
	if *memprofile != "" {
		f, err := os.Create(*memprofile)
		checkErr(err)
		pprof.WriteHeapProfile(f)
		defer f.Close()
	}

	if *verbose {
		fmt.Println()
		fmt.Println(rowCount, "rows written")
		fmt.Println("Total runtime =", time.Since(start))
	}
}

// Pass the buck error catching
func checkErr(e error) {
	if e != nil {
		log.Panic(e)
	}
}

func catchNotifications() {
	state, err := terminal.GetState(int(os.Stdin.Fd()))
	checkErr(err)

	// Trap for SIGINT
	sigChan := make(chan os.Signal, 1)
	signal.Notify(sigChan, os.Interrupt)

	var timer time.Time
	go func() {
		for sig := range sigChan {
			if time.Now().Sub(timer) < time.Second*signalTimeout {
				terminal.Restore(int(os.Stdin.Fd()), state)
				os.Exit(0)
			}

			fmt.Fprintln(os.Stderr, "")
			fmt.Fprintln(os.Stderr, "")
			fmt.Fprintln(os.Stderr, "")
			fmt.Fprintln(os.Stderr, sig, "signal caught!")
			fmt.Fprintf(os.Stderr, "Send signal again within %v seconds to exit\n", signalTimeout)
			fmt.Fprintln(os.Stderr, "")
			fmt.Fprintln(os.Stderr, "")
			fmt.Fprintln(os.Stderr, "")

			timer = time.Now()
		}
	}()
}

// Create a database connection object
func (dbi *dbInfo) Connect() (*sql.DB, error) {
	var db *sql.DB
	var err error
	if dbi.TLS {
		db, err = sql.Open("mysql", dbi.user+":"+dbi.pass+"@tcp("+dbi.host+":"+dbi.port+")/?allowCleartextPasswords=1&tls=skip-verify")
	} else {
		db, err = sql.Open("mysql", dbi.user+":"+dbi.pass+"@tcp("+dbi.host+":"+dbi.port+")/?allowCleartextPasswords=1")
	}
	checkErr(err)

	// Ping database to verify credentials
	err = db.Ping()

	return db, err
}

// readRows executes a query and returns the rows
func readRows(db *sql.DB, query string, dataChan chan []sql.RawBytes, quitChan chan bool, goChan chan bool, csvHeader bool) {
	rows, err := db.Query(query)
	defer rows.Close()
	if err != nil {
		log.Print(err)
		os.Exit(1)
	}

	cols, err := rows.Columns()
	checkErr(err)

	if csvHeader {
		headers := make([]sql.RawBytes, len(cols))
		for i, col := range cols {
			headers[i] = []byte(col)
		}
		dataChan <- headers
		<-goChan
	}

	// Need to scan into empty interface since we don't know how many columns or their types
	scanVals := make([]interface{}, len(cols))
	vals := make([]sql.RawBytes, len(cols))
	//cpy := make([]sql.RawBytes, len(cols))
	for i := range vals {
		scanVals[i] = &vals[i]
	}

	for rows.Next() {
		err := rows.Scan(scanVals...)
		checkErr(err)

		//copy(cpy, vals)
		//dataChan <- cpy
		dataChan <- vals

		// Block until writeRows() signals it is safe to proceed
		// This is necessary because sql.RawBytes is a memory pointer and rows.Next() will loop and change the memory address before writeRows can properly process the values
		<-goChan
	}

	err = rows.Err()
	checkErr(err)

	close(dataChan)
	quitChan <- true
}

// writeCSV writes csv output
func writeCSV(w *Writer, dataChan chan []sql.RawBytes, goChan chan bool, verbose bool) uint {
	var rowsWritten uint
	var verboseCount uint

	if verbose {
		fmt.Println("A '.' will be shown for every 1000 CSV rows written")
	}

	// Range over row results from readRows()
	for data := range dataChan {
		// Write CSV
		size, err := w.Write(data)
		checkErr(err)

		rowsWritten++
		if verbose {
			verboseCount++
			if verboseCount == 1000 {
				fmt.Printf(".")
				verboseCount = 0
			}
		}

		// Flush CSV writer contents
		if size > flushSize {
			w.Flush()
			err = w.Error()
			checkErr(err)
		}

		// Allow read function to unblock and loop over rows
		goChan <- true
	}

	// Flush CSV writer contents
	w.Flush()
	err := w.Error()
	checkErr(err)

	return rowsWritten
}
<commit_msg>Added character set flag. Default is binary which is used for MySQL outfiles if no character set is provided.<commit_after>package main

import (
	"database/sql"
	"flag"
	"fmt"
	"io"
	"io/ioutil"
	"log"
	"os"
	"os/signal"
	"runtime/pprof"
	"strings"
	"time"
	"unicode/utf8"

	"golang.org/x/crypto/ssh/terminal"

	_ "github.com/go-sql-driver/mysql"
)

const (
	// Amount of CSV write data to buffer between flushes.
	flushSize = 26214400 // 25MB

	// Timeout length where ctrl+c is ignored.
	signalTimeout = 3 // Seconds

	// Timeout length to wait for a query string sent via stdin.
	stdinTimeout = 10 // Milliseconds
)

type (
	// dbInfo contains information necessary to connect to a database
	dbInfo struct {
		user    string
		pass    string
		host    string
		port    string
		tls     bool
		charset string
	}
)

// ShowUsage prints a help screen
func showUsage() {
	fmt.Println(`
	mycsv usage:
	mycsv DB_COMMANDS [CSV OUTPUT FLAGS] [DEBUG FLAGS] [CSV OUTFILE] query

	EXAMPLES:
	mycsv -user=jprunier -pass= -file=my.csv -charset=utf8 -query="select * from jjp.example_table where filter in ('1', 'test', 'another')"
	echo "select * from mysql.plugin" | mycsv -user=jprunier -pass=mypass -host=remotedb -tls > my.csv
	mycsv -user=jprunier -pass= -file=my.csv -d="|" -q="'" < queryfile

	DATABASE FLAGS
	==============
	-user: Database Username (required)
	-pass: Database Password (interactive prompt if blank)
	-host: Database Host (localhost assumed if blank)
	-port: Database Port (3306 default)
	-tls:  Use TLS/SSL for database connection (false default)
	-charset: Database character set (binary default)

	CSV FLAGS
	=========
	-file: CSV output filename (Write to stdout if not supplied)
	-query: MySQL query (required, can be sent via stdin redirection)
	-header: Print initial column name header line (true default)
	-d: CSV field delimiter ("," default)
	-q: CSV quote character ("\"" default)
	-e: CSV escape character ("\\" default)
	-t: CSV line terminator ("\n" default)
	-v: Print more information (false default)

	DEBUG FLAGS
	===========
	-debug_cpu: CPU debugging filename
	-debug_mem: Memory debugging filename

	`)
}

func main() {
	start := time.Now()

	// Profiling flags
	var cpuprofile = flag.String("debug_cpu", "", "CPU debugging filename")
	var memprofile = flag.String("debug_mem", "", "Memory debugging filename")

	// Database flags
	dbUser := flag.String("user", "", "Database Username (required)")
	dbPass := flag.String("pass", "", "Database Password (interactive prompt if blank)")
	dbHost := flag.String("host", "", "Database Host (localhost assumed if blank)")
	dbPort := flag.String("port", "3306", "Database Port")
	dbTLS := flag.Bool("tls", false, "Use TLS/SSL for database connection")
	dbCharset := flag.String("charset", "binary", "Database character set")

	// CSV format flags
	csvDelimiter := flag.String("d", `,`, "CSV field delimiter")
	csvQuote := flag.String("q", `"`, "CSV quote character")
	csvEscape := flag.String("e", `\`, "CSV escape character")
	csvTerminator := flag.String("t", "\n", "CSV line terminator")

	csvHeader := flag.Bool("header", true, "Print initial column name header line")
	csvFile := flag.String("file", "", "CSV output filename")
	sqlQuery := flag.String("query", "", "MySQL query")
	verbose := flag.Bool("v", false, "Print more information")

	// Parse flags
	flag.Parse()

	// Print usage
	if flag.NFlag() == 0 {
		showUsage()

		os.Exit(0)
	}

	// If query not provided read from standard in
	var query string
	queryChan := make(chan string)
	defer close(queryChan)
	if *sqlQuery == "" {
		go func() {
			b, err := ioutil.ReadAll(os.Stdin)
			checkErr(err)

			queryChan <- string(b)
		}()

		select {
		case q := <-queryChan:
			query = q
		case <-time.After(time.Millisecond * stdinTimeout):
			fmt.Fprintln(os.Stderr, "You must supply a query")
			os.Exit(1)
		}
	} else {
		query = *sqlQuery
	}

	// Make sure the query is a select
	if strings.ToLower(query[0:6]) != "select" {
		fmt.Fprintln(os.Stderr, "Query must be a select!")
		os.Exit(1)
	}

	// Check if an output file was supplied otherwise use standard out
	var writeTo string
	var writerDest io.Writer
	var err error
	if *csvFile == "" {
		writeTo = "standard out"
		writerDest = os.Stdout
	} else {
		f, err := os.Open(*csvFile)
		if err == nil {
			fmt.Fprintln(os.Stderr, *csvFile, "already exists!")
			fmt.Fprintln(os.Stderr, "Please remove it or use a different filename")
			os.Exit(1)
		}
		f.Close()
		writerDest, err = os.Create(*csvFile)
		writeTo = *csvFile
	}

	// Create a new CSV writer
	var i int
	CSVWriter := NewWriter(writerDest)
	CSVWriter.Delimiter, i = utf8.DecodeLastRuneInString(*csvDelimiter)
	if i == 0 {
		fmt.Fprintln(os.Stderr, "You must supply a valid delimiter character")
		os.Exit(1)
	}
	CSVWriter.Quote, i = utf8.DecodeLastRuneInString(*csvQuote)
	if i == 0 {
		fmt.Fprintln(os.Stderr, "You must supply a valid quote character")
		os.Exit(1)
	}
	CSVWriter.Escape, i = utf8.DecodeLastRuneInString(*csvEscape)
	if i == 0 {
		fmt.Fprintln(os.Stderr, "You must supply a valid escape character")
		os.Exit(1)
	}

	// Need literal string check here to see all 4 bytes instead of 2 (ascii 13 & 10)
	// Newline is default but check here in case it is manually passed in
	if *csvTerminator == `\r\n` {
		CSVWriter.Terminator = "\r\n"
	} else if *csvTerminator == `\n` {
		CSVWriter.Terminator = "\n"
	} else {
		CSVWriter.Terminator = *csvTerminator
	}

	if *verbose {
		fmt.Println("CSV output will be written to", writeTo)
	}

	// Check if Stdin has been redirected and reset so the user can be prompted for a password
	checkStdin()

	// Listen for SIGINT (ctrl+c)
	catchNotifications()

	// CPU Profiling
	if *cpuprofile != "" {
		f, err := os.Create(*cpuprofile)
		checkErr(err)
		pprof.StartCPUProfile(f)
		defer pprof.StopCPUProfile()
	}

	// Default to localhost if no host or socket provided
	if *dbHost == "" {
		*dbHost = "127.0.0.1"
	}

	// Need to provide a target
	if *dbUser == "" {
		fmt.Fprintln(os.Stderr, "You must provide a user name!")
		os.Exit(1)
	}

	// If password is blank prompt user
	if *dbPass == "" {
		fmt.Println("Enter password: ")
		pwd, err := terminal.ReadPassword(int(os.Stdin.Fd()))
		if err != nil {
			if err != io.EOF {
				checkErr(err)
			}
		}
		*dbPass = string(pwd)
	}

	// Populate dbInfo struct with cli flags
	dbi := dbInfo{user: *dbUser, pass: *dbPass, host: *dbHost, port: *dbPort, tls: *dbTLS, charset: *dbCharset}

	// Create a *sql.DB connection to the source database
	db, err := dbi.Connect()
	defer db.Close()
	if err != nil {
		fmt.Fprintln(os.Stderr, err)
		os.Exit(1)
	}

	// Start reading & writing
	dataChan := make(chan []sql.RawBytes)
	quitChan := make(chan bool)
	goChan := make(chan bool)
	go readRows(db, query, dataChan, quitChan, goChan, *csvHeader)
	rowCount := writeCSV(CSVWriter, dataChan, goChan, *verbose)

	<-quitChan
	close(quitChan)
	close(goChan)

	// Memory Profiling
	if *memprofile != "" {
		f, err := os.Create(*memprofile)
		checkErr(err)
		pprof.WriteHeapProfile(f)
		defer f.Close()
	}

	if *verbose {
		fmt.Println()
		fmt.Println(rowCount, "rows written")
		fmt.Println("Total runtime =", time.Since(start))
	}
}

// Pass the buck error catching
func checkErr(e error) {
	if e != nil {
		log.Panic(e)
	}
}

func catchNotifications() {
	state, err := terminal.GetState(int(os.Stdin.Fd()))
	checkErr(err)

	// Trap for SIGINT
	sigChan := make(chan os.Signal, 1)
	signal.Notify(sigChan, os.Interrupt)

	var timer time.Time
	go func() {
		for sig := range sigChan {
			if time.Now().Sub(timer) < time.Second*signalTimeout {
				terminal.Restore(int(os.Stdin.Fd()), state)
				os.Exit(0)
			}

			fmt.Fprintln(os.Stderr, "")
			fmt.Fprintln(os.Stderr, "")
			fmt.Fprintln(os.Stderr, "")
			fmt.Fprintln(os.Stderr, sig, "signal caught!")
			fmt.Fprintf(os.Stderr, "Send signal again within %v seconds to exit\n", signalTimeout)
			fmt.Fprintln(os.Stderr, "")
			fmt.Fprintln(os.Stderr, "")
			fmt.Fprintln(os.Stderr, "")

			timer = time.Now()
		}
	}()
}

// Create a database connection object
func (dbi *dbInfo) Connect() (*sql.DB, error) {
	var db *sql.DB
	var err error
	if dbi.tls {
		db, err = sql.Open("mysql", dbi.user+":"+dbi.pass+"@tcp("+dbi.host+":"+dbi.port+")/?allowCleartextPasswords=1&tls=skip-verify&charset="+dbi.charset)
	} else {
		db, err = sql.Open("mysql", dbi.user+":"+dbi.pass+"@tcp("+dbi.host+":"+dbi.port+")/?allowCleartextPasswords=1&charset="+dbi.charset)
	}
	checkErr(err)

	// Ping database to verify credentials
	err = db.Ping()

	return db, err
}

// readRows executes a query and returns the rows
func readRows(db *sql.DB, query string, dataChan chan []sql.RawBytes, quitChan chan bool, goChan chan bool, csvHeader bool) {
	rows, err := db.Query(query)
	defer rows.Close()
	if err != nil {
		log.Print(err)
		os.Exit(1)
	}

	cols, err := rows.Columns()
	checkErr(err)

	if csvHeader {
		headers := make([]sql.RawBytes, len(cols))
		for i, col := range cols {
			headers[i] = []byte(col)
		}
		dataChan <- headers
		<-goChan
	}

	// Need to scan into empty interface since we don't know how many columns or their types
	scanVals := make([]interface{}, len(cols))
	vals := make([]sql.RawBytes, len(cols))
	//cpy := make([]sql.RawBytes, len(cols))
	for i := range vals {
		scanVals[i] = &vals[i]
	}

	for rows.Next() {
		err := rows.Scan(scanVals...)
		checkErr(err)

		//copy(cpy, vals)
		//dataChan <- cpy
		dataChan <- vals

		// Block until writeRows() signals it is safe to proceed
		// This is necessary because sql.RawBytes is a memory pointer and rows.Next() will loop and change the memory address before writeRows can properly process the values
		<-goChan
	}

	err = rows.Err()
	checkErr(err)

	close(dataChan)
	quitChan <- true
}

// writeCSV writes csv output
func writeCSV(w *Writer, dataChan chan []sql.RawBytes, goChan chan bool, verbose bool) uint {
	var rowsWritten uint
	var verboseCount uint

	if verbose {
		fmt.Println("A '.' will be shown for every 1000 CSV rows written")
	}

	// Range over row results from readRows()
	for data := range dataChan {
		// Write CSV
		size, err := w.Write(data)
		checkErr(err)

		rowsWritten++
		if verbose {
			verboseCount++
			if verboseCount == 1000 {
				fmt.Printf(".")
				verboseCount = 0
			}
		}

		// Flush CSV writer contents
		if size > flushSize {
			w.Flush()
			err = w.Error()
			checkErr(err)
		}

		// Allow read function to unblock and loop over rows
		goChan <- true
	}

	// Flush CSV writer contents
	w.Flush()
	err := w.Error()
	checkErr(err)

	return rowsWritten
}
<|endoftext|>


<commit_before>package hammer

import (
	"bytes"
	"errors"
	"github.com/Sirupsen/logrus"
	shlex "github.com/anmitsu/go-shlex"
	"github.com/spf13/viper"
	"gopkg.in/yaml.v2"
	"io/ioutil"
	"os"
	"os/exec"
	"path"
	"runtime"
	"strings"
	"text/template"
)

var (
	ErrInvalidScriptName = errors.New("invalid script name")
)

type Target struct {
	Src      string `yaml:"src"`
	Dest     string `yaml:"dest"`
	Template bool   `yaml:"template"`
	Config   bool   `yaml:"config"`
}

type Package struct {
	Name         string     `yaml:"name"`
	Version      string     `yaml:"version"`
	Iteration    string     `yaml:"iteration"`
	Epoch        string     `yaml:"epoch"`
	License      string     `yaml:"license"`
	Vendor       string     `yaml:"vendor"`
	URL          string     `yaml:"url"`
	Description  string     `yaml:"description"`
	Architecture string     `yaml:"architecture"`
	Depends      []string   `yaml:"depends"`
	Resources    []Resource `yaml:"resources"`
	Targets      []Target   `yaml:"targets"`
	Scripts      Scripts    `yaml:"scripts"`
	ExtraArgs    string     `yaml:"extra-args"`

	// various roots
	BuildRoot  string `yaml:"-"`
	OutputRoot string `yaml:"-"`
	ScriptRoot string `yaml:"-"`
	SpecRoot   string `yaml:"-"`
	TargetRoot string `yaml:"-"`

	// information about the machine doing the building
	CPUs int `yaml:"-"`

	logger *logrus.Entry
}

func NewPackageFromYAML(content []byte) (*Package, error) {
	p := new(Package)
	err := yaml.Unmarshal(content, p)
	p.logger = logrus.WithField("name", p.Name)

	if err != nil {
		return p, err
	}

	// machine information
	p.CPUs = runtime.NumCPU()

	return p, nil
}

func (p *Package) Cleanup() error {
	err := os.RemoveAll(p.BuildRoot)
	if err != nil {
		p.logger.WithField("error", err).Error("could not remove build root during cleanup")
		return err
	}

	err = os.RemoveAll(p.ScriptRoot)
	if err != nil {
		p.logger.WithField("error", err).Error("could not remove script root during cleanup")
		return err
	}

	return nil
}

func (p *Package) Build() error {
	// create temporary directory for building
	buildDir, err := ioutil.TempDir("", "hammer-"+p.Name)
	if err != nil {
		p.logger.WithField("error", err).Error("could not create build directory")
		return err
	}
	p.BuildRoot = buildDir

	// get the sources and store them in the temporary directory
	for _, s := range p.Resources {
		body, err := s.Download(p)
		if err != nil {
			return err
		}
		ioutil.WriteFile(
			path.Join(buildDir, s.Name(p)),
			body,
			0777,
		)
	}

	// perform the build
	p.logger.Info("building")
	out, err := p.Scripts.BuildSources(p, buildDir)
	if err != nil {
		p.logger.WithFields(logrus.Fields{
			"error":  err,
			"output": string(out),
		}).Error("failed to build")
		return err
	}

	p.logger.Info("finished building")
	return nil
}

func (p *Package) Render(in string) (bytes.Buffer, error) {
	var buf bytes.Buffer
	tmpl, err := template.New(p.Name + "-" + in).Parse(in)
	if err != nil {
		return buf, err
	}

	err = tmpl.Execute(&buf, p)
	return buf, err
}

func (p *Package) Package() error {
	fpmArgs, err := p.fpmArgs()
	if err != nil {
		p.logger.WithField("error", err).Error("failed to get package args")
		return err
	}

	// prepend extra args
	extra, err := shlex.Split(p.ExtraArgs, true)
	if err != nil {
		p.logger.WithField("error", err).Error("failed to parse extra args")
		return err
	}
	fpmArgs = append(extra, fpmArgs...)

	for _, outType := range strings.Split(viper.GetString("type"), ",") {
		out, err := p.FPM(fpmArgs, outType)
		if err != nil {
			p.logger.WithFields(logrus.Fields{
				"error": err,
				"out":   string(out),
			}).Error("failed to package")
			return err
		}
	}

	p.logger.Info("finished packaging")
	return nil
}

func (p *Package) FPM(args []string, pkgType string) ([]byte, error) {
	logger := p.logger.WithField("type", pkgType)
	logger.Info("packaging with FPM")

	// prepend source and dest arguments
	prefixArgs := []string{
		"-s", "dir",
		"-t", pkgType,
		"-p", p.OutputRoot,
	}
	prefixArgs = append(prefixArgs, p.typeArgs(pkgType)...)
	args = append(prefixArgs, args...)

	logrus.WithField("args", args).Debug("running FPM with args")
	fpm := exec.Command("fpm", args...)
	out, err := fpm.CombinedOutput()

	if err == nil && !fpm.ProcessState.Success() {
		err = errors.New("package command exited with a non-zero exit code")
	}

	if fpm.ProcessState != nil {
		logger.WithFields(logrus.Fields{
			"systemTime": fpm.ProcessState.SystemTime(),
			"userTime":   fpm.ProcessState.UserTime(),
			"success":    fpm.ProcessState.Success(),
		}).Debug("package command exited")
	} else {
		logger.Debug("package command exited")
	}

	return out, err
}

func (p *Package) typeArgs(t string) []string {
	args := []string{}

	// specific fixes for different output types
	switch t {
	case "rpm":
		args = append(
			args,
			"--rpm-auto-add-directories", // TODO: document this somewhere
		)
	}

	return args
}

func (p *Package) fpmArgs() ([]string, error) {
	// create the arguments for FPM
	args := []string{}

	// name
	name, err := p.Render(p.Name)
	if err != nil {
		p.logger.WithField("error", err).Error("failed to render name as template")
		return args, err
	}
	args = append(args, "--name", name.String())

	// version
	version, err := p.Render(p.Version)
	if err != nil {
		p.logger.WithField("error", err).Error("failed to render version as template")
		return args, err
	}
	args = append(args, "--version", version.String())

	// iteration
	iteration, err := p.Render(p.Iteration)
	if err != nil {
		p.logger.WithField("error", err).Error("failed to render iteration as template")
		return args, err
	}
	args = append(args, "--iteration", iteration.String())

	// epoch
	if p.Epoch != "" {
		epoch, err := p.Render(p.Epoch)
		if err != nil {
			p.logger.WithField("error", err).Error("failed to render epoch as template")
			return args, err
		}
		args = append(args, "--epoch", epoch.String())
	}

	// license
	if p.License != "" {
		license, err := p.Render(p.License)
		if err != nil {
			p.logger.WithField("error", err).Error("failed to render license as template")
			return args, err
		}
		args = append(args, "--license", license.String())
	}

	// vendor
	if p.Vendor != "" {
		vendor, err := p.Render(p.Vendor)
		if err != nil {
			p.logger.WithField("error", err).Error("failed to render vendor as template")
			return args, err
		}
		args = append(args, "--vendor", vendor.String())
	}

	// description
	if p.Description != "" {
		description, err := p.Render(p.Description)
		if err != nil {
			p.logger.WithField("error", err).Error("failed to render description as template")
			return args, err
		}
		args = append(args, "--description", description.String())
	}

	// url
	if p.URL != "" {
		url, err := p.Render(p.URL)
		if err != nil {
			p.logger.WithField("error", err).Error("failed to render url as template")
			return args, err
		}
		args = append(args, "--url", url.String())
	}

	for _, depend := range p.Depends {
		args = append(args, "--depends", depend)
	}

	// architecture
	if p.Architecture != "" {
		architecture, err := p.Render(p.Architecture)
		if err != nil {
			p.logger.WithField("error", err).Error("failed to render architecture as template")
			return args, err
		}
		args = append(args, "--architecture", architecture.String())
	}

	scriptDir, err := ioutil.TempDir("", "hammer-scripts-"+p.Name)
	if err != nil {
		p.logger.WithField("error", err).Error("could not create script directory")
		return args, err
	}
	p.ScriptRoot = scriptDir

	for name, value := range p.Scripts {
		if name == "build" {
			continue
		}
		scriptLogger := p.logger.WithField("script", name)

		// validate
		if name != "before-install" && name != "after-install" && name != "before-remove" && name != "after-remove" && name != "before-upgrade" && name != "after-upgrade" {
			scriptLogger.Error("invalid script name")
			return args, ErrInvalidScriptName
		}

		content, err := p.Render(value)
		if err != nil {
			scriptLogger.WithField("error", err).Error("error in templating script")
			return args, err
		}

		// write
		loc := path.Join(scriptDir, name)
		err = ioutil.WriteFile(loc, content.Bytes(), 0755)
		if err != nil {
			scriptLogger.WithField("error", err).Error("could not write script")
			return args, err
		}

		scriptLogger.Debug("wrote script")
		args = append(args, "--"+name, loc)
	}

	// config files
	for i, target := range p.Targets {
		if !target.Config {
			continue
		}

		dest, err := p.Render(target.Dest)
		if err != nil {
			p.logger.WithField("index", i).Error("error templating target destination")
			return args, err
		}

		args = append(args, "--config-files", dest.String())
	}

	// targets
	targetDir, err := ioutil.TempDir("", "hammer-targets-"+p.Name)
	if err != nil {
		p.logger.WithField("error", err).Error("could not create target directory")
		return args, err
	}
	p.TargetRoot = targetDir

	for i, target := range p.Targets {
		srcBuf, err := p.Render(target.Src)
		if err != nil {
			p.logger.WithField("index", i).Error("error templating target source name")
			return args, err
		}
		src := srcBuf.String()

		dest, err := p.Render(target.Dest)
		if err != nil {
			p.logger.WithField("index", i).Error("error templating target destination")
			return args, err
		}

		// opt-in templating. We don't want to template *every* file because some
		// things look like Go templates and aren't (see for example every other
		// kind of mustache template)
		if !target.Template {
			args = append(args, src+"="+dest.String())
		} else {
			var content []byte
			rawContent, err := ioutil.ReadFile(src)
			if err != nil {
				p.logger.WithFields(logrus.Fields{
					"name":  src,
					"error": err,
				}).Error("error reading content")
				return args, err
			}

			contentBuf, err := p.Render(string(rawContent))
			if err != nil {
				p.logger.WithFields(logrus.Fields{
					"name":  src,
					"error": err,
				}).Error("error templating content")
				return args, err
			}
			content = contentBuf.Bytes()

			_, name := path.Split(src)
			contentDest := path.Join(p.TargetRoot, name)
			err = ioutil.WriteFile(contentDest, content, 0777)
			if err != nil {
				p.logger.WithFields(logrus.Fields{
					"name":  src,
					"error": err,
				}).Error("error writing content")
				return args, err
			}

			args = append(args, contentDest+"="+dest.String())
		}
	}

	return args, nil
}
<commit_msg>package: render dependencies<commit_after>package hammer

import (
	"bytes"
	"errors"
	"github.com/Sirupsen/logrus"
	shlex "github.com/anmitsu/go-shlex"
	"github.com/spf13/viper"
	"gopkg.in/yaml.v2"
	"io/ioutil"
	"os"
	"os/exec"
	"path"
	"runtime"
	"strings"
	"text/template"
)

var (
	ErrInvalidScriptName = errors.New("invalid script name")
)

type Target struct {
	Src      string `yaml:"src"`
	Dest     string `yaml:"dest"`
	Template bool   `yaml:"template"`
	Config   bool   `yaml:"config"`
}

type Package struct {
	Name         string     `yaml:"name"`
	Version      string     `yaml:"version"`
	Iteration    string     `yaml:"iteration"`
	Epoch        string     `yaml:"epoch"`
	License      string     `yaml:"license"`
	Vendor       string     `yaml:"vendor"`
	URL          string     `yaml:"url"`
	Description  string     `yaml:"description"`
	Architecture string     `yaml:"architecture"`
	Depends      []string   `yaml:"depends"`
	Resources    []Resource `yaml:"resources"`
	Targets      []Target   `yaml:"targets"`
	Scripts      Scripts    `yaml:"scripts"`
	ExtraArgs    string     `yaml:"extra-args"`

	// various roots
	BuildRoot  string `yaml:"-"`
	OutputRoot string `yaml:"-"`
	ScriptRoot string `yaml:"-"`
	SpecRoot   string `yaml:"-"`
	TargetRoot string `yaml:"-"`

	// information about the machine doing the building
	CPUs int `yaml:"-"`

	logger *logrus.Entry
}

func NewPackageFromYAML(content []byte) (*Package, error) {
	p := new(Package)
	err := yaml.Unmarshal(content, p)
	p.logger = logrus.WithField("name", p.Name)

	if err != nil {
		return p, err
	}

	// machine information
	p.CPUs = runtime.NumCPU()

	return p, nil
}

func (p *Package) Cleanup() error {
	err := os.RemoveAll(p.BuildRoot)
	if err != nil {
		p.logger.WithField("error", err).Error("could not remove build root during cleanup")
		return err
	}

	err = os.RemoveAll(p.ScriptRoot)
	if err != nil {
		p.logger.WithField("error", err).Error("could not remove script root during cleanup")
		return err
	}

	return nil
}

func (p *Package) Build() error {
	// create temporary directory for building
	buildDir, err := ioutil.TempDir("", "hammer-"+p.Name)
	if err != nil {
		p.logger.WithField("error", err).Error("could not create build directory")
		return err
	}
	p.BuildRoot = buildDir

	// get the sources and store them in the temporary directory
	for _, s := range p.Resources {
		body, err := s.Download(p)
		if err != nil {
			return err
		}
		ioutil.WriteFile(
			path.Join(buildDir, s.Name(p)),
			body,
			0777,
		)
	}

	// perform the build
	p.logger.Info("building")
	out, err := p.Scripts.BuildSources(p, buildDir)
	if err != nil {
		p.logger.WithFields(logrus.Fields{
			"error":  err,
			"output": string(out),
		}).Error("failed to build")
		return err
	}

	p.logger.Info("finished building")
	return nil
}

func (p *Package) Render(in string) (bytes.Buffer, error) {
	var buf bytes.Buffer
	tmpl, err := template.New(p.Name + "-" + in).Parse(in)
	if err != nil {
		return buf, err
	}

	err = tmpl.Execute(&buf, p)
	return buf, err
}

func (p *Package) Package() error {
	fpmArgs, err := p.fpmArgs()
	if err != nil {
		p.logger.WithField("error", err).Error("failed to get package args")
		return err
	}

	// prepend extra args
	extra, err := shlex.Split(p.ExtraArgs, true)
	if err != nil {
		p.logger.WithField("error", err).Error("failed to parse extra args")
		return err
	}
	fpmArgs = append(extra, fpmArgs...)

	for _, outType := range strings.Split(viper.GetString("type"), ",") {
		out, err := p.FPM(fpmArgs, outType)
		if err != nil {
			p.logger.WithFields(logrus.Fields{
				"error": err,
				"out":   string(out),
			}).Error("failed to package")
			return err
		}
	}

	p.logger.Info("finished packaging")
	return nil
}

func (p *Package) FPM(args []string, pkgType string) ([]byte, error) {
	logger := p.logger.WithField("type", pkgType)
	logger.Info("packaging with FPM")

	// prepend source and dest arguments
	prefixArgs := []string{
		"-s", "dir",
		"-t", pkgType,
		"-p", p.OutputRoot,
	}
	prefixArgs = append(prefixArgs, p.typeArgs(pkgType)...)
	args = append(prefixArgs, args...)

	logrus.WithField("args", args).Debug("running FPM with args")
	fpm := exec.Command("fpm", args...)
	out, err := fpm.CombinedOutput()

	if err == nil && !fpm.ProcessState.Success() {
		err = errors.New("package command exited with a non-zero exit code")
	}

	if fpm.ProcessState != nil {
		logger.WithFields(logrus.Fields{
			"systemTime": fpm.ProcessState.SystemTime(),
			"userTime":   fpm.ProcessState.UserTime(),
			"success":    fpm.ProcessState.Success(),
		}).Debug("package command exited")
	} else {
		logger.Debug("package command exited")
	}

	return out, err
}

func (p *Package) typeArgs(t string) []string {
	args := []string{}

	// specific fixes for different output types
	switch t {
	case "rpm":
		args = append(
			args,
			"--rpm-auto-add-directories", // TODO: document this somewhere
		)
	}

	return args
}

func (p *Package) fpmArgs() ([]string, error) {
	// create the arguments for FPM
	args := []string{}

	// name
	name, err := p.Render(p.Name)
	if err != nil {
		p.logger.WithField("error", err).Error("failed to render name as template")
		return args, err
	}
	args = append(args, "--name", name.String())

	// version
	version, err := p.Render(p.Version)
	if err != nil {
		p.logger.WithField("error", err).Error("failed to render version as template")
		return args, err
	}
	args = append(args, "--version", version.String())

	// iteration
	iteration, err := p.Render(p.Iteration)
	if err != nil {
		p.logger.WithField("error", err).Error("failed to render iteration as template")
		return args, err
	}
	args = append(args, "--iteration", iteration.String())

	// epoch
	if p.Epoch != "" {
		epoch, err := p.Render(p.Epoch)
		if err != nil {
			p.logger.WithField("error", err).Error("failed to render epoch as template")
			return args, err
		}
		args = append(args, "--epoch", epoch.String())
	}

	// license
	if p.License != "" {
		license, err := p.Render(p.License)
		if err != nil {
			p.logger.WithField("error", err).Error("failed to render license as template")
			return args, err
		}
		args = append(args, "--license", license.String())
	}

	// vendor
	if p.Vendor != "" {
		vendor, err := p.Render(p.Vendor)
		if err != nil {
			p.logger.WithField("error", err).Error("failed to render vendor as template")
			return args, err
		}
		args = append(args, "--vendor", vendor.String())
	}

	// description
	if p.Description != "" {
		description, err := p.Render(p.Description)
		if err != nil {
			p.logger.WithField("error", err).Error("failed to render description as template")
			return args, err
		}
		args = append(args, "--description", description.String())
	}

	// url
	if p.URL != "" {
		url, err := p.Render(p.URL)
		if err != nil {
			p.logger.WithField("error", err).Error("failed to render url as template")
			return args, err
		}
		args = append(args, "--url", url.String())
	}

	for _, rawDepend := range p.Depends {
		depend, err := p.Render(rawDepend)
		if err != nil {
			p.logger.WithFields(logrus.Fields{
				"error": err,
				"raw":   rawDepend,
			}).Error("failed to render dependency as template")
		}
		args = append(args, "--depends", depend.String())
	}

	// architecture
	if p.Architecture != "" {
		architecture, err := p.Render(p.Architecture)
		if err != nil {
			p.logger.WithField("error", err).Error("failed to render architecture as template")
			return args, err
		}
		args = append(args, "--architecture", architecture.String())
	}

	scriptDir, err := ioutil.TempDir("", "hammer-scripts-"+p.Name)
	if err != nil {
		p.logger.WithField("error", err).Error("could not create script directory")
		return args, err
	}
	p.ScriptRoot = scriptDir

	for name, value := range p.Scripts {
		if name == "build" {
			continue
		}
		scriptLogger := p.logger.WithField("script", name)

		// validate
		if name != "before-install" && name != "after-install" && name != "before-remove" && name != "after-remove" && name != "before-upgrade" && name != "after-upgrade" {
			scriptLogger.Error("invalid script name")
			return args, ErrInvalidScriptName
		}

		content, err := p.Render(value)
		if err != nil {
			scriptLogger.WithField("error", err).Error("error in templating script")
			return args, err
		}

		// write
		loc := path.Join(scriptDir, name)
		err = ioutil.WriteFile(loc, content.Bytes(), 0755)
		if err != nil {
			scriptLogger.WithField("error", err).Error("could not write script")
			return args, err
		}

		scriptLogger.Debug("wrote script")
		args = append(args, "--"+name, loc)
	}

	// config files
	for i, target := range p.Targets {
		if !target.Config {
			continue
		}

		dest, err := p.Render(target.Dest)
		if err != nil {
			p.logger.WithField("index", i).Error("error templating target destination")
			return args, err
		}

		args = append(args, "--config-files", dest.String())
	}

	// targets
	targetDir, err := ioutil.TempDir("", "hammer-targets-"+p.Name)
	if err != nil {
		p.logger.WithField("error", err).Error("could not create target directory")
		return args, err
	}
	p.TargetRoot = targetDir

	for i, target := range p.Targets {
		srcBuf, err := p.Render(target.Src)
		if err != nil {
			p.logger.WithField("index", i).Error("error templating target source name")
			return args, err
		}
		src := srcBuf.String()

		dest, err := p.Render(target.Dest)
		if err != nil {
			p.logger.WithField("index", i).Error("error templating target destination")
			return args, err
		}

		// opt-in templating. We don't want to template *every* file because some
		// things look like Go templates and aren't (see for example every other
		// kind of mustache template)
		if !target.Template {
			args = append(args, src+"="+dest.String())
		} else {
			var content []byte
			rawContent, err := ioutil.ReadFile(src)
			if err != nil {
				p.logger.WithFields(logrus.Fields{
					"name":  src,
					"error": err,
				}).Error("error reading content")
				return args, err
			}

			contentBuf, err := p.Render(string(rawContent))
			if err != nil {
				p.logger.WithFields(logrus.Fields{
					"name":  src,
					"error": err,
				}).Error("error templating content")
				return args, err
			}
			content = contentBuf.Bytes()

			_, name := path.Split(src)
			contentDest := path.Join(p.TargetRoot, name)
			err = ioutil.WriteFile(contentDest, content, 0777)
			if err != nil {
				p.logger.WithFields(logrus.Fields{
					"name":  src,
					"error": err,
				}).Error("error writing content")
				return args, err
			}

			args = append(args, contentDest+"="+dest.String())
		}
	}

	return args, nil
}
<|endoftext|>


<commit_before>// Copyright 2017 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

package runtime

import "unsafe"

type mOS struct {
	waitsema uintptr // semaphore for parking on locks
}

//extern malloc
func libc_malloc(uintptr) unsafe.Pointer

//go:noescape
//extern sem_init
func sem_init(sem *semt, pshared int32, value uint32) int32

//go:noescape
//extern sem_wait
func sem_wait(sem *semt) int32

//go:noescape
//extern sem_post
func sem_post(sem *semt) int32

//go:noescape
//extern sem_timedwait
func sem_timedwait(sem *semt, timeout *timespec) int32

//go:noescape
//extern clock_gettime
func clock_gettime(clock_id int64, timeout *timespec) int32

//go:nosplit
func semacreate(mp *m) {
	if mp.mos.waitsema != 0 {
		return
	}

	var sem *semt

	// Call libc's malloc rather than malloc. This will
	// allocate space on the C heap. We can't call malloc
	// here because it could cause a deadlock.
	sem = (*semt)(libc_malloc(unsafe.Sizeof(*sem)))
	if sem_init(sem, 0, 0) != 0 {
		throw("sem_init")
	}
	mp.mos.waitsema = uintptr(unsafe.Pointer(sem))
}

//go:nosplit
func semasleep(ns int64) int32 {
	_m_ := getg().m
	if ns >= 0 {
		const CLOCK_REALTIME int64 = 9
		var ts timespec

		if clock_gettime(CLOCK_REALTIME, &ts) != 0 {
			throw("clock_gettime")
		}
		ts.tv_sec += timespec_sec_t(ns / 1000000000)
		ts.tv_nsec += timespec_nsec_t((int64(ts.tv_nsec) + ns) % 1000000000)
		if ts.tv_nsec >= 1000000000 {
			ts.tv_sec += timespec_sec_t(1)
			ts.tv_nsec -= timespec_nsec_t(1000000000)
		}

		if sem_timedwait((*semt)(unsafe.Pointer(_m_.mos.waitsema)), &ts) != 0 {
			err := errno()
			if err == _ETIMEDOUT || err == _EAGAIN || err == _EINTR {
				return -1
			}
			throw("sem_timedwait")
		}
		return 0
	}
	for {
		r1 := sem_wait((*semt)(unsafe.Pointer(_m_.mos.waitsema)))
		if r1 == 0 {
			break
		}
		if errno() == _EINTR {
			continue
		}
		throw("sem_wait")
	}
	return 0
}

//go:nosplit
func semawakeup(mp *m) {
	if sem_post((*semt)(unsafe.Pointer(mp.mos.waitsema))) != 0 {
		throw("sem_post")
	}
}
<commit_msg>runtime: fix wrong time calculation in semasleep<commit_after>// Copyright 2017 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

package runtime

import "unsafe"

type mOS struct {
	waitsema uintptr // semaphore for parking on locks
}

//extern malloc
func libc_malloc(uintptr) unsafe.Pointer

//go:noescape
//extern sem_init
func sem_init(sem *semt, pshared int32, value uint32) int32

//go:noescape
//extern sem_wait
func sem_wait(sem *semt) int32

//go:noescape
//extern sem_post
func sem_post(sem *semt) int32

//go:noescape
//extern sem_timedwait
func sem_timedwait(sem *semt, timeout *timespec) int32

//go:noescape
//extern clock_gettime
func clock_gettime(clock_id int64, timeout *timespec) int32

//go:nosplit
func semacreate(mp *m) {
	if mp.mos.waitsema != 0 {
		return
	}

	var sem *semt

	// Call libc's malloc rather than malloc. This will
	// allocate space on the C heap. We can't call malloc
	// here because it could cause a deadlock.
	sem = (*semt)(libc_malloc(unsafe.Sizeof(*sem)))
	if sem_init(sem, 0, 0) != 0 {
		throw("sem_init")
	}
	mp.mos.waitsema = uintptr(unsafe.Pointer(sem))
}

//go:nosplit
func semasleep(ns int64) int32 {
	_m_ := getg().m
	if ns >= 0 {
		const CLOCK_REALTIME int64 = 9
		var ts timespec

		if clock_gettime(CLOCK_REALTIME, &ts) != 0 {
			throw("clock_gettime")
		}
		ts.tv_sec += timespec_sec_t(ns / 1000000000)
		ts.tv_nsec += timespec_nsec_t(ns % 1000000000)
		if ts.tv_nsec >= 1000000000 {
			ts.tv_sec += timespec_sec_t(1)
			ts.tv_nsec -= timespec_nsec_t(1000000000)
		}

		if sem_timedwait((*semt)(unsafe.Pointer(_m_.mos.waitsema)), &ts) != 0 {
			err := errno()
			if err == _ETIMEDOUT || err == _EAGAIN || err == _EINTR {
				return -1
			}
			throw("sem_timedwait")
		}
		return 0
	}
	for {
		r1 := sem_wait((*semt)(unsafe.Pointer(_m_.mos.waitsema)))
		if r1 == 0 {
			break
		}
		if errno() == _EINTR {
			continue
		}
		throw("sem_wait")
	}
	return 0
}

//go:nosplit
func semawakeup(mp *m) {
	if sem_post((*semt)(unsafe.Pointer(mp.mos.waitsema))) != 0 {
		throw("sem_post")
	}
}
<|endoftext|>


<commit_before>// Copyright 2015 The Gorilla WebSocket Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

// +build ignore

package main

import (
	"flag"
	"html/template"
	"log"
	"net/http"

	"github.com/gorilla/websocket"
)

var addr = flag.String("addr", "localhost:8080", "http service address")

var upgrader = websocket.Upgrader{} // use default options

func echo(w http.ResponseWriter, r *http.Request) {
	c, err := upgrader.Upgrade(w, r, nil)
	if err != nil {
		log.Print("upgrade:", err)
		return
	}
	defer c.Close()
	for {
		mt, message, err := c.ReadMessage()
		if err != nil {
			log.Println("read:", err)
			break
		}
		log.Printf("recv: %s", message)
		err = c.WriteMessage(mt, message)
		if err != nil {
			log.Println("write:", err)
			break
		}
	}
}

func home(w http.ResponseWriter, r *http.Request) {
	homeTemplate.Execute(w, "ws://"+r.Host+"/echo")
}

func main() {
	flag.Parse()
	log.SetFlags(0)
	http.HandleFunc("/echo", echo)
	http.HandleFunc("/", home)
	log.Fatal(http.ListenAndServe(*addr, nil))
}

var homeTemplate = template.Must(template.New("").Parse(`
<!DOCTYPE html>

<meta charset="utf-8">

Click "Open" to create a connection to the server,
"Send" to send a message to the server and "Close" to close the connection.
You can change the message and send multiple times.

<form>
<button id="open">Open</button>
<button id="close">Close</button>

<input id="input" type="text" value="Hello world!">
<button id="send">Send</button>
</form>

`))
<commit_msg>Add missing html tag in example/echo<commit_after>// Copyright 2015 The Gorilla WebSocket Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

// +build ignore

package main

import (
	"flag"
	"html/template"
	"log"
	"net/http"

	"github.com/gorilla/websocket"
)

var addr = flag.String("addr", "localhost:8080", "http service address")

var upgrader = websocket.Upgrader{} // use default options

func echo(w http.ResponseWriter, r *http.Request) {
	c, err := upgrader.Upgrade(w, r, nil)
	if err != nil {
		log.Print("upgrade:", err)
		return
	}
	defer c.Close()
	for {
		mt, message, err := c.ReadMessage()
		if err != nil {
			log.Println("read:", err)
			break
		}
		log.Printf("recv: %s", message)
		err = c.WriteMessage(mt, message)
		if err != nil {
			log.Println("write:", err)
			break
		}
	}
}

func home(w http.ResponseWriter, r *http.Request) {
	homeTemplate.Execute(w, "ws://"+r.Host+"/echo")
}

func main() {
	flag.Parse()
	log.SetFlags(0)
	http.HandleFunc("/echo", echo)
	http.HandleFunc("/", home)
	log.Fatal(http.ListenAndServe(*addr, nil))
}

var homeTemplate = template.Must(template.New("").Parse(`
<!DOCTYPE html>

<meta charset="utf-8">

Click "Open" to create a connection to the server,
"Send" to send a message to the server and "Close" to close the connection.
You can change the message and send multiple times.

<form>
<button id="open">Open</button>
<button id="close">Close</button>

<input id="input" type="text" value="Hello world!">
<button id="send">Send</button>
</form>

`))
<|endoftext|>


<commit_before>// This package is used to implement Key Derivation Functions (KDF)
// based on the recommendations of NIST SP 800-108. These are useful
// for generating unique-per-transaction keys, or situations in which
// a key hierarchy may be useful.
package kdf

import (
	"crypto/hmac"
	"crypto/sha256"
	"encoding/binary"
	"fmt"
)

// PRF is a psuedo-random function that takes a key or seed,
// as well as additional binary data and generates output that is
// indistinguishable from random. Examples are cryptographic hash
// functions or block ciphers.
type PRF func([]byte, []byte) ([]byte, error)

// CounterMode implements the counter mode KDF that uses a psuedo-random-function (PRF)
// along with a counter to generate derived keys. The KDF takes a base key
// a derivation context, and the requried number of output bits.
func CounterMode(prf PRF, prfLen uint32, base []byte, context []byte, bits uint32) ([]byte, error) {
	// Ensure the PRF is byte aligned
	if prfLen%8 != 0 {
		return nil, fmt.Errorf("PRF must be byte aligned")
	}

	// Ensure the bits required are byte aligned
	if bits%8 != 0 {
		return nil, fmt.Errorf("bits required must be byte aligned")
	}

	// Determine the number of rounds required
	rounds := bits / prfLen
	if bits%prfLen != 0 {
		rounds++
	}

	// Allocate and setup the input
	input := make([]byte, 4+len(context)+4)
	copy(input[4:], context)
	binary.BigEndian.PutUint32(input[4+len(context):], bits)

	// Iteratively generate more key material
	var out []byte
	var i uint32
	for i = 0; i < rounds; i++ {
		// Update the counter in the input string
		binary.BigEndian.PutUint32(input[:4], i)

		// Compute a more key material
		part, err := prf(base, input)
		if err != nil {
			return nil, err
		}
		if uint32(len(part)*8) != prfLen {
			return nil, fmt.Errorf("PRF length mis-match (%d vs %d)", len(part)*8, prfLen)
		}
		out = append(out, part...)
	}

	// Return the desired number of output bytes
	return out[:bits/8], nil
}

const (
	// HMACSHA256PRFLen is the length of output from HMACSHA256PRF
	HMACSHA256PRFLen uint32 = 256
)

// HMACSHA256PRF is a pseudo-random-function (PRF) that uses an HMAC-SHA256
func HMACSHA256PRF(key []byte, data []byte) ([]byte, error) {
	hash := hmac.New(sha256.New, key)
	hash.Write(data)
	return hash.Sum(nil), nil
}
<commit_msg>helper/kdf: changing argument name for clarity<commit_after>// This package is used to implement Key Derivation Functions (KDF)
// based on the recommendations of NIST SP 800-108. These are useful
// for generating unique-per-transaction keys, or situations in which
// a key hierarchy may be useful.
package kdf

import (
	"crypto/hmac"
	"crypto/sha256"
	"encoding/binary"
	"fmt"
)

// PRF is a psuedo-random function that takes a key or seed,
// as well as additional binary data and generates output that is
// indistinguishable from random. Examples are cryptographic hash
// functions or block ciphers.
type PRF func([]byte, []byte) ([]byte, error)

// CounterMode implements the counter mode KDF that uses a psuedo-random-function (PRF)
// along with a counter to generate derived keys. The KDF takes a base key
// a derivation context, and the requried number of output bits.
func CounterMode(prf PRF, prfLen uint32, key []byte, context []byte, bits uint32) ([]byte, error) {
	// Ensure the PRF is byte aligned
	if prfLen%8 != 0 {
		return nil, fmt.Errorf("PRF must be byte aligned")
	}

	// Ensure the bits required are byte aligned
	if bits%8 != 0 {
		return nil, fmt.Errorf("bits required must be byte aligned")
	}

	// Determine the number of rounds required
	rounds := bits / prfLen
	if bits%prfLen != 0 {
		rounds++
	}

	// Allocate and setup the input
	input := make([]byte, 4+len(context)+4)
	copy(input[4:], context)
	binary.BigEndian.PutUint32(input[4+len(context):], bits)

	// Iteratively generate more key material
	var out []byte
	var i uint32
	for i = 0; i < rounds; i++ {
		// Update the counter in the input string
		binary.BigEndian.PutUint32(input[:4], i)

		// Compute a more key material
		part, err := prf(key, input)
		if err != nil {
			return nil, err
		}
		if uint32(len(part)*8) != prfLen {
			return nil, fmt.Errorf("PRF length mis-match (%d vs %d)", len(part)*8, prfLen)
		}
		out = append(out, part...)
	}

	// Return the desired number of output bytes
	return out[:bits/8], nil
}

const (
	// HMACSHA256PRFLen is the length of output from HMACSHA256PRF
	HMACSHA256PRFLen uint32 = 256
)

// HMACSHA256PRF is a pseudo-random-function (PRF) that uses an HMAC-SHA256
func HMACSHA256PRF(key []byte, data []byte) ([]byte, error) {
	hash := hmac.New(sha256.New, key)
	hash.Write(data)
	return hash.Sum(nil), nil
}
<|endoftext|>


<commit_before>/*
 Copyright 2016 Padduck, LLC

 Licensed under the Apache License, Version 2.0 (the "License");
 you may not use this file except in compliance with the License.
 You may obtain a copy of the License at

 	http://www.apache.org/licenses/LICENSE-2.0

 Unless required by applicable law or agreed to in writing, software
 distributed under the License is distributed on an "AS IS" BASIS,
 WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 See the License for the specific language governing permissions and
 limitations under the License.
*/

package server

import (
	"encoding/json"
	"io"
	"io/ioutil"
	"net/http"
	"os"
	"path/filepath"
	"time"

	"github.com/gin-gonic/gin"
	"github.com/gorilla/websocket"
	"github.com/itsjamie/gin-cors"
	"github.com/pufferpanel/pufferd/httphandlers"
	"github.com/pufferpanel/pufferd/logging"
	"github.com/pufferpanel/pufferd/programs"
	"github.com/pufferpanel/pufferd/utils"
	"github.com/pkg/errors"
	"strings"
	"strconv"
)

var wsupgrader = websocket.Upgrader{
	ReadBufferSize:  1024,
	WriteBufferSize: 1024,
	CheckOrigin: func(r *http.Request) bool {
		return true
	},
}

func RegisterRoutes(e *gin.Engine) {
	l := e.Group("/server")
	{
		e.Handle("CONNECT", "/:id/console", func(c *gin.Context) {
			c.Header("Access-Control-Allow-Origin", "*")
			c.Header("Access-Control-Allow-Credentials", "false")
		})
		l.Use(httphandlers.OAuth2Handler)
		l.PUT("/:id", CreateServer)
		l.DELETE("/:id", DeleteServer)
		l.POST("/:id", EditServer)
		l.GET("/:id/start", StartServer)
		l.GET("/:id/stop", StopServer)
		l.POST("/:id/install", InstallServer)
		l.GET("/:id/file/*filename", GetFile)
		l.PUT("/:id/file/*filename", PutFile)
		l.POST("/:id/console", PostConsole)
		l.GET("/:id/stats", GetStats)
		l.POST("/:id/reload", ReloadServer)
		l.GET("/:id/console", cors.Middleware(cors.Config{
			Origins:     "*",
			Credentials: true,
		}), GetConsole)
		l.GET("/:id/logs", GetLogs)
	}
	e.GET("/network", httphandlers.OAuth2Handler, NetworkServer)
}

func StartServer(c *gin.Context) {
	valid, existing := handleInitialCallServer(c, "server.start", true)

	if !valid {
		c.Status(404)
		return
	}

	existing.Start()
}

func StopServer(c *gin.Context) {
	valid, existing := handleInitialCallServer(c, "server.stop", true)
	wait := c.Param("wait")
	if wait == "" || (wait != "true" && wait != "false") {
		wait = "true"
	}

	if !valid {
		return
	}

	err := existing.Stop()
	if err != nil {
		c.Error(err)
	}

	if wait == "true" {
		err = existing.GetEnvironment().WaitForMainProcess()
		if err != nil {
			c.Error(err)
		}
	}
}

func CreateServer(c *gin.Context) {
	serverId := c.Param("id")
	handleInitialCallServer(c, "server.create", false)

	existing := programs.GetFromCache(serverId)

	if existing != nil {
		c.AbortWithStatus(409)
		return
	}

	data := make(map[string]interface{}, 0)
	err := json.NewDecoder(c.Request.Body).Decode(&data)

	if err != nil {
		logging.Error("Error decoding JSON body", err)
		c.AbortWithError(400, err)
		return
	}

	serverType := data["type"].(string)

	if !programs.Create(serverId, serverType, data) {
		c.AbortWithStatus(500)
	}
}

func DeleteServer(c *gin.Context) {
	valid, existing := handleInitialCallServer(c, "server.delete", true)

	if !valid {
		return
	}

	programs.Delete(existing.Id())
	c.Status(204)
}

func InstallServer(c *gin.Context) {
	valid, existing := handleInitialCallServer(c, "server.install", true)

	if !valid {
		return
	}

	c.Status(200)
	go func() {
		existing.Install()
	}()
}

func EditServer(c *gin.Context) {
	valid, existing := handleInitialCallServer(c, "server.edit", true)

	if !valid {
		return
	}

	data := make(map[string]interface{}, 0)
	json.NewDecoder(c.Request.Body).Decode(&data)

	c.Status(200)
	existing.Edit(data)
}

func GetFile(c *gin.Context) {

	valid, server := handleInitialCallServer(c, "server.file.get", true)

	if !valid {
		return
	}

	targetPath := c.Param("filename")

	targetFile := utils.JoinPath(server.GetEnvironment().GetRootDirectory(), targetPath)

	if !utils.EnsureAccess(targetFile, server.GetEnvironment().GetRootDirectory()) {
		return
	}

	info, err := os.Stat(targetFile)

	if os.IsNotExist(err) {
		c.Status(404)
		return
	}

	if info.IsDir() {

		type FileDesc struct {
			Name      string    `json:"name"`
			Modified  time.Time `json:"modifyTime"`
			Size      int64     `json:"size,omitempty"`
			File      bool      `json:"isFile"`
			Extension string    `json:"extension,omitempty"`
		}

		files, _ := ioutil.ReadDir(targetFile)
		fileNames := make([]interface{}, 0)
		if targetFile != "." {
			newFile := &FileDesc{
				Name:      "..",
				File:      false,
			}
			fileNames = append(fileNames, newFile)
		}
		for _, file := range files {
			newFile := &FileDesc{
				Name:      file.Name(),
				File:      !file.IsDir(),
			}

			if newFile.File {
				newFile.Size = file.Size()
				newFile.Modified = file.ModTime()
				newFile.Extension = filepath.Ext(file.Name())
			}

			fileNames = append(fileNames, newFile)
		}
		c.JSON(200, fileNames)
	} else {
		c.File(targetFile)
	}
}

func PutFile(c *gin.Context) {
	valid, server := handleInitialCallServer(c, "server.file.put", true)

	if !valid {
		return
	}

	targetPath := c.Param("filename")

	if targetPath == "" {
		c.Status(404)
		return
	}

	targetFile := utils.JoinPath(server.GetEnvironment().GetRootDirectory(), targetPath)

	if !utils.EnsureAccess(targetFile, server.GetEnvironment().GetRootDirectory()) {
		return
	}

	file, err := os.Create(targetFile)

	if err != nil {
		logging.Error("Error writing file", err)
		return
	}

	_, err = io.Copy(file, c.Request.Body)

	if err != nil {
		logging.Error("Error writing file", err)
	}
}

func PostConsole(c *gin.Context) {
	valid, program := handleInitialCallServer(c, "server.console.send", true)
	if !valid {
		return
	}
	d, _ := ioutil.ReadAll(c.Request.Body)
	cmd := string(d)
	err := program.Execute(cmd)
	if err != nil {
		c.Error(err)
	} else {
		c.Status(200)
	}
}

func GetConsole(c *gin.Context) {
	valid, program := handleInitialCallServer(c, "server.console", true)
	if !valid {
		return
	}
	conn, err := wsupgrader.Upgrade(c.Writer, c.Request, nil)
	if err != nil {
		logging.Error("Error creating websocket", err)
		c.AbortWithError(500, err)
		return
	}
	console, _ := program.GetEnvironment().GetConsole()
	for _, v := range console {
		conn.WriteMessage(websocket.TextMessage, []byte(v))
	}
	program.GetEnvironment().AddListener(conn)
}

func GetStats(c *gin.Context) {
	valid, server := handleInitialCallServer(c, "server.stats", true)

	if !valid {
		return
	}

	results, err := server.GetEnvironment().GetStats()
	if err != nil {
		result := make(map[string]interface{})
		result["error"] = err.Error()
		c.JSON(200, result)
	} else {
		c.JSON(200, results)
	}
}

func ReloadServer(c *gin.Context) {
	valid, existing := handleInitialCallServer(c, "server.reload", true)

	if !valid {
		c.Status(404)
		return
	}

	programs.Reload(existing.Id())
}

func NetworkServer(c *gin.Context) {

	scopes, _ := c.Get("scopes")
	valid := false
	for _, v := range scopes.([]string) {
		if v == "server.network"{
			valid = true
		}
	}
	if !valid {
		c.AbortWithStatus(401)
		return
	}

	servers := c.DefaultQuery("ids", "")
	if servers == "" {
		c.AbortWithError(400, errors.New("Server ids required"))
		return
	}
	serverIds := strings.Split(servers, ",")
	result := make(map[string]string)
	for _, v := range serverIds {
		program, _ := programs.Get(v)
		if program == nil {
			continue
		}
		result[program.Id()] = program.GetNetwork()
	}
	c.JSON(200, result)
}

func GetLogs (c *gin.Context) {
	valid, program := handleInitialCallServer(c, "server.console", true)
	if !valid {
		return
	}

	time := c.DefaultQuery("time", "0")

	castedTime, ok := strconv.ParseInt(time, 10, 64)

	if ok != nil {
		c.AbortWithError(400, errors.New("Time provided is not a valid UNIX time"))
		return
	}

	console, epoch := program.GetEnvironment().GetConsoleFrom(castedTime)
	msg := ""
	for _, k := range console {
		msg += k + "\n"
	}
	result := make(map[string]interface{})
	result["epoch"] = epoch
	result["logs"] = msg;
	c.JSON(200, result)
}

func handleInitialCallServer(c *gin.Context, perm string, requireServer bool) (valid bool, program programs.Program) {
	valid = false

	serverId := c.Param("id")
	canAccessId, _ := c.Get("server_id")

	accessId := canAccessId.(string)

	if accessId != serverId && accessId != "*" {
		c.AbortWithStatus(401)
		return
	}

	if accessId == "*" {
		program, _ = programs.Get(serverId)
	} else {
		program, _ = programs.Get(accessId)
	}

	if requireServer && program == nil {
		c.AbortWithStatus(404)
		valid = false
		return
	}

	scopes, _ := c.Get("scopes")

	for _, v := range scopes.([]string) {
		if v == perm {
			valid = true
		}
	}

	valid = true

	return
}
<commit_msg>Better HTTP response codes for file manager<commit_after>/*
 Copyright 2016 Padduck, LLC

 Licensed under the Apache License, Version 2.0 (the "License");
 you may not use this file except in compliance with the License.
 You may obtain a copy of the License at

 	http://www.apache.org/licenses/LICENSE-2.0

 Unless required by applicable law or agreed to in writing, software
 distributed under the License is distributed on an "AS IS" BASIS,
 WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 See the License for the specific language governing permissions and
 limitations under the License.
*/

package server

import (
	"encoding/json"
	"io"
	"io/ioutil"
	"net/http"
	"os"
	"path/filepath"
	"time"

	"github.com/gin-gonic/gin"
	"github.com/gorilla/websocket"
	"github.com/itsjamie/gin-cors"
	"github.com/pufferpanel/pufferd/httphandlers"
	"github.com/pufferpanel/pufferd/logging"
	"github.com/pufferpanel/pufferd/programs"
	"github.com/pufferpanel/pufferd/utils"
	"github.com/pkg/errors"
	"strings"
	"strconv"
)

var wsupgrader = websocket.Upgrader{
	ReadBufferSize:  1024,
	WriteBufferSize: 1024,
	CheckOrigin: func(r *http.Request) bool {
		return true
	},
}

func RegisterRoutes(e *gin.Engine) {
	l := e.Group("/server")
	{
		e.Handle("CONNECT", "/:id/console", func(c *gin.Context) {
			c.Header("Access-Control-Allow-Origin", "*")
			c.Header("Access-Control-Allow-Credentials", "false")
		})
		l.Use(httphandlers.OAuth2Handler)
		l.PUT("/:id", CreateServer)
		l.DELETE("/:id", DeleteServer)
		l.POST("/:id", EditServer)
		l.GET("/:id/start", StartServer)
		l.GET("/:id/stop", StopServer)
		l.POST("/:id/install", InstallServer)
		l.GET("/:id/file/*filename", GetFile)
		l.PUT("/:id/file/*filename", PutFile)
		l.POST("/:id/console", PostConsole)
		l.GET("/:id/stats", GetStats)
		l.POST("/:id/reload", ReloadServer)
		l.GET("/:id/console", cors.Middleware(cors.Config{
			Origins:     "*",
			Credentials: true,
		}), GetConsole)
		l.GET("/:id/logs", GetLogs)
	}
	e.GET("/network", httphandlers.OAuth2Handler, NetworkServer)
}

func StartServer(c *gin.Context) {
	valid, existing := handleInitialCallServer(c, "server.start", true)

	if !valid {
		c.Status(404)
		return
	}

	existing.Start()
}

func StopServer(c *gin.Context) {
	valid, existing := handleInitialCallServer(c, "server.stop", true)
	wait := c.Param("wait")
	if wait == "" || (wait != "true" && wait != "false") {
		wait = "true"
	}

	if !valid {
		return
	}

	err := existing.Stop()
	if err != nil {
		c.Error(err)
	}

	if wait == "true" {
		err = existing.GetEnvironment().WaitForMainProcess()
		if err != nil {
			c.Error(err)
		}
	}
}

func CreateServer(c *gin.Context) {
	serverId := c.Param("id")
	handleInitialCallServer(c, "server.create", false)

	existing := programs.GetFromCache(serverId)

	if existing != nil {
		c.AbortWithStatus(409)
		return
	}

	data := make(map[string]interface{}, 0)
	err := json.NewDecoder(c.Request.Body).Decode(&data)

	if err != nil {
		logging.Error("Error decoding JSON body", err)
		c.AbortWithError(400, err)
		return
	}

	serverType := data["type"].(string)

	if !programs.Create(serverId, serverType, data) {
		c.AbortWithStatus(500)
	}
}

func DeleteServer(c *gin.Context) {
	valid, existing := handleInitialCallServer(c, "server.delete", true)

	if !valid {
		return
	}

	programs.Delete(existing.Id())
	c.Status(204)
}

func InstallServer(c *gin.Context) {
	valid, existing := handleInitialCallServer(c, "server.install", true)

	if !valid {
		return
	}

	c.Status(200)
	go func() {
		existing.Install()
	}()
}

func EditServer(c *gin.Context) {
	valid, existing := handleInitialCallServer(c, "server.edit", true)

	if !valid {
		return
	}

	data := make(map[string]interface{}, 0)
	json.NewDecoder(c.Request.Body).Decode(&data)

	c.Status(200)
	existing.Edit(data)
}

func GetFile(c *gin.Context) {

	valid, server := handleInitialCallServer(c, "server.file.get", true)

	if !valid {
		return
	}

	targetPath := c.Param("filename")

	targetFile := utils.JoinPath(server.GetEnvironment().GetRootDirectory(), targetPath)

	if !utils.EnsureAccess(targetFile, server.GetEnvironment().GetRootDirectory()) {
		return
	}

	info, err := os.Stat(targetFile)

	if os.IsNotExist(err) {
		c.Status(404)
		return
	}

	if info.IsDir() {

		type FileDesc struct {
			Name      string    `json:"name"`
			Modified  time.Time `json:"modifyTime"`
			Size      int64     `json:"size,omitempty"`
			File      bool      `json:"isFile"`
			Extension string    `json:"extension,omitempty"`
		}

		files, _ := ioutil.ReadDir(targetFile)
		fileNames := make([]interface{}, 0)
		if targetFile != "." {
			newFile := &FileDesc{
				Name:      "..",
				File:      false,
			}
			fileNames = append(fileNames, newFile)
		}
		for _, file := range files {
			newFile := &FileDesc{
				Name:      file.Name(),
				File:      !file.IsDir(),
			}

			if newFile.File {
				newFile.Size = file.Size()
				newFile.Modified = file.ModTime()
				newFile.Extension = filepath.Ext(file.Name())
			}

			fileNames = append(fileNames, newFile)
		}
		c.JSON(200, fileNames)
	} else {
		_, err := os.Open(targetFile)
		if err != nil {
			if err == os.ErrNotExist {
				c.AbortWithStatus(404)
			} else {
				c.AbortWithStatus(500)
			}
		}
		c.File(targetFile)
	}
}

func PutFile(c *gin.Context) {
	valid, server := handleInitialCallServer(c, "server.file.put", true)

	if !valid {
		return
	}

	targetPath := c.Param("filename")

	if targetPath == "" {
		c.Status(404)
		return
	}

	targetFile := utils.JoinPath(server.GetEnvironment().GetRootDirectory(), targetPath)

	if !utils.EnsureAccess(targetFile, server.GetEnvironment().GetRootDirectory()) {
		return
	}

	file, err := os.Create(targetFile)

	if err != nil {
		logging.Error("Error writing file", err)
		return
	}

	_, err = io.Copy(file, c.Request.Body)

	if err != nil {
		logging.Error("Error writing file", err)
	}
}

func PostConsole(c *gin.Context) {
	valid, program := handleInitialCallServer(c, "server.console.send", true)
	if !valid {
		return
	}
	d, _ := ioutil.ReadAll(c.Request.Body)
	cmd := string(d)
	err := program.Execute(cmd)
	if err != nil {
		c.Error(err)
	} else {
		c.Status(200)
	}
}

func GetConsole(c *gin.Context) {
	valid, program := handleInitialCallServer(c, "server.console", true)
	if !valid {
		return
	}
	conn, err := wsupgrader.Upgrade(c.Writer, c.Request, nil)
	if err != nil {
		logging.Error("Error creating websocket", err)
		c.AbortWithError(500, err)
		return
	}
	console, _ := program.GetEnvironment().GetConsole()
	for _, v := range console {
		conn.WriteMessage(websocket.TextMessage, []byte(v))
	}
	program.GetEnvironment().AddListener(conn)
}

func GetStats(c *gin.Context) {
	valid, server := handleInitialCallServer(c, "server.stats", true)

	if !valid {
		return
	}

	results, err := server.GetEnvironment().GetStats()
	if err != nil {
		result := make(map[string]interface{})
		result["error"] = err.Error()
		c.JSON(200, result)
	} else {
		c.JSON(200, results)
	}
}

func ReloadServer(c *gin.Context) {
	valid, existing := handleInitialCallServer(c, "server.reload", true)

	if !valid {
		c.Status(404)
		return
	}

	programs.Reload(existing.Id())
}

func NetworkServer(c *gin.Context) {

	scopes, _ := c.Get("scopes")
	valid := false
	for _, v := range scopes.([]string) {
		if v == "server.network"{
			valid = true
		}
	}
	if !valid {
		c.AbortWithStatus(401)
		return
	}

	servers := c.DefaultQuery("ids", "")
	if servers == "" {
		c.AbortWithError(400, errors.New("Server ids required"))
		return
	}
	serverIds := strings.Split(servers, ",")
	result := make(map[string]string)
	for _, v := range serverIds {
		program, _ := programs.Get(v)
		if program == nil {
			continue
		}
		result[program.Id()] = program.GetNetwork()
	}
	c.JSON(200, result)
}

func GetLogs (c *gin.Context) {
	valid, program := handleInitialCallServer(c, "server.console", true)
	if !valid {
		return
	}

	time := c.DefaultQuery("time", "0")

	castedTime, ok := strconv.ParseInt(time, 10, 64)

	if ok != nil {
		c.AbortWithError(400, errors.New("Time provided is not a valid UNIX time"))
		return
	}

	console, epoch := program.GetEnvironment().GetConsoleFrom(castedTime)
	msg := ""
	for _, k := range console {
		msg += k + "\n"
	}
	result := make(map[string]interface{})
	result["epoch"] = epoch
	result["logs"] = msg;
	c.JSON(200, result)
}

func handleInitialCallServer(c *gin.Context, perm string, requireServer bool) (valid bool, program programs.Program) {
	valid = false

	serverId := c.Param("id")
	canAccessId, _ := c.Get("server_id")

	accessId := canAccessId.(string)

	if accessId != serverId && accessId != "*" {
		c.AbortWithStatus(401)
		return
	}

	if accessId == "*" {
		program, _ = programs.Get(serverId)
	} else {
		program, _ = programs.Get(accessId)
	}

	if requireServer && program == nil {
		c.AbortWithStatus(404)
		valid = false
		return
	}

	scopes, _ := c.Get("scopes")

	for _, v := range scopes.([]string) {
		if v == perm {
			valid = true
		}
	}

	valid = true

	return
}
<|endoftext|>


<commit_before>package logo

import (
	"bytes"
	"encoding/json"
	"io"
	"log"
	"os"
)

//NewStdoutAppender creates an Appender that logs to standard output. Messages
//are formatted as follows: "[LEVEL]: [TIMESTAMP] [MESSAGE]"
func NewStdoutAppender() Appender {
	return &stdoutAppender{
		log.New(os.Stdout, "", 0), //use std logger for its buffering
	}
}

//NewLogstashAppender creates an Appender that emits logstash formatted messages
//to the specified writer.
func NewLogstashAppender(w io.Writer, v LogstashVersion, pretty bool) Appender {
	return &logstashAppender{
		log.New(w, "", 0),
		v,
		pretty,
	}
}

type stdoutAppender struct {
	logger *log.Logger
}

func (s *stdoutAppender) Write(message *LogMessage) {
	const layout = "2006-01-02T15:04:05.000Z"
	var buffer bytes.Buffer

	for key, value := range message.MDC {
		buffer.WriteString(key);
		buffer.WriteString("=");
		buffer.WriteString(value);
		buffer.WriteString(" ");
	}
	s.logger.Printf("%s: [%v] %s %s", message.Level, message.Timestamp.Format(layout), message.Message, buffer.String())
}

func (s *stdoutAppender) Close() { /* noop */
}

type LogstashVersion uint8

const (
	//logstash version 0
	LSV0 LogstashVersion = iota

	//logstash version 1
	LSV1
)

type logstashAppender struct {
	logger  *log.Logger
	version LogstashVersion
	pretty  bool
}

const logstashLayout = "2006-01-02T15:04:05.000000Z"

func (l *logstashAppender) Write(message *LogMessage) {
	var out interface{}
	switch l.version {
	case LSV0:
		msg := make(map[string]interface{})
		message.MDC["level"] = message.Level.String()
		msg["@fields"] = message.MDC
		msg["@message"] = message.Message
		msg["@timestamp"] = message.Timestamp.Format(logstashLayout)
		out = msg
	default: //use version 1
		msg := message.MDC
		msg["level"] = message.Level.String()
		msg["@message"] = message.Message
		msg["@timestamp"] = message.Timestamp.Format(logstashLayout)
		out = msg
	}

	var logData []byte
	if l.pretty {
		logData, _ = json.MarshalIndent(out, "", "  ")
	} else {
		logData, _ = json.Marshal(out)
	}

	l.logger.Print(string(logData))
}

func (l *logstashAppender) Close() { /* noop */
}
<commit_msg>[ES-1017]: do not clobber mdc<commit_after>package logo

import (
	"bytes"
	"encoding/json"
	"io"
	"log"
	"os"
)

//NewStdoutAppender creates an Appender that logs to standard output. Messages
//are formatted as follows: "[LEVEL]: [TIMESTAMP] [MESSAGE]"
func NewStdoutAppender() Appender {
	return &stdoutAppender{
		log.New(os.Stdout, "", 0), //use std logger for its buffering
	}
}

//NewLogstashAppender creates an Appender that emits logstash formatted messages
//to the specified writer.
func NewLogstashAppender(w io.Writer, v LogstashVersion, pretty bool) Appender {
	return &logstashAppender{
		log.New(w, "", 0),
		v,
		pretty,
	}
}

type stdoutAppender struct {
	logger *log.Logger
}

func (s *stdoutAppender) Write(message *LogMessage) {
	const layout = "2006-01-02T15:04:05.000Z"
	var buffer bytes.Buffer

	for key, value := range message.MDC {
		buffer.WriteString(key);
		buffer.WriteString("=");
		buffer.WriteString(value);
		buffer.WriteString(" ");
	}
	s.logger.Printf("%s: [%v] %s %s", message.Level, message.Timestamp.Format(layout), message.Message, buffer.String())
}

func (s *stdoutAppender) Close() { /* noop */
}

type LogstashVersion uint8

const (
	//logstash version 0
	LSV0 LogstashVersion = iota

	//logstash version 1
	LSV1
)

type logstashAppender struct {
	logger  *log.Logger
	version LogstashVersion
	pretty  bool
}

const logstashLayout = "2006-01-02T15:04:05.000000Z"

func (l *logstashAppender) Write(message *LogMessage) {
	var out interface{}
	switch l.version {
	case LSV0:
		msg := make(map[string]interface{})
		message.MDC["level"] = message.Level.String()
		msg["@fields"] = message.MDC
		msg["@message"] = message.Message
		msg["@timestamp"] = message.Timestamp.Format(logstashLayout)
		out = msg
	default: //use version 1
		msg := make(map[string]interface{})
		for k,v := range message.MDC {
			msg[k] = v
		}
		msg["level"] = message.Level.String()
		msg["@message"] = message.Message
		msg["@timestamp"] = message.Timestamp.Format(logstashLayout)
		out = msg
	}

	var logData []byte
	if l.pretty {
		logData, _ = json.MarshalIndent(out, "", "  ")
	} else {
		logData, _ = json.Marshal(out)
	}

	l.logger.Print(string(logData))
}

func (l *logstashAppender) Close() { /* noop */
}
<|endoftext|>