<commit_before>package sq

import (
	"encoding/binary"
	"fmt"
	"os"
	"path"
	"sync"
	"syscall"
	"unsafe"
)

const (
	MESSAGE_OVERHEAD = 5 //length + termination
)

var (
	encoder            = binary.LittleEndian
	blank              = struct{}{}
	ChannelNameLenErr  = fmt.Errorf("channel name cannot exceed %d characters", MAX_CHANNEL_NAME_SIZE)
	ChannelExistsErr   = fmt.Errorf("channel already exists")
	ChannelCreateErr   = fmt.Errorf("channel count not be created")
	MessageTooLargeErr = fmt.Errorf("message too large")
	pageSize           = os.Getpagesize()
)

type addChannelWork struct {
	err     error
	channel *Channel
	config  *ChannelConfiguration
	c       chan *addChannelWork
}

type Topic struct {
	path         string
	state        *State
	states       *States
	segmentSize  int
	channelsLock sync.RWMutex
	channels     map[string]*Channel
	segment      *Segment
	segmentsLock sync.RWMutex
	segments     map[uint64]*Segment
	addChannel   chan *addChannelWork
	messageAdded chan struct{}
	dataLock     sync.RWMutex
}

func OpenTopic(name string, config *TopicConfiguration) (*Topic, error) {
	t := &Topic{
		path:         path.Join(config.path, name),
		channels:     make(map[string]*Channel),
		segments:     make(map[uint64]*Segment),
		addChannel:   make(chan *addChannelWork),
		segmentSize:  config.segmentSize,
		messageAdded: make(chan struct{}, 64),
	}
	err := loadStates(t)
	if err != nil {
		return nil, err
	}

	if id := t.state.segmentId; id == 0 {
		if err := t.expand(); err != nil {
			return nil, err
		}
		if err := t.states.syncTopic(); err != nil {
			return nil, err
		}
	} else {
		t.findWritePosition(openSegment(t, id, false))
		t.segments[id] = t.segment
	}
	go t.worker()
	return t, nil
}

func (t *Topic) Write(data []byte) error {
	length := len(data)

	t.dataLock.Lock()
	start := int(t.state.offset)
	dataStart := start + 4
	dataEnd := dataStart + length

	// do we have enough space in the current segment?
	if dataEnd > t.segmentSize {
		if length+MESSAGE_OVERHEAD+int(SEGMENT_HEADER_SIZE) > t.segmentSize {
			return MessageTooLargeErr
		}
		if err := t.expand(); err != nil {
			return err
		}
		start = int(SEGMENT_HEADER_SIZE)
		dataStart = start + 4
		dataEnd = dataStart + length
	}

	// write the message length
	encoder.PutUint32(t.segment.data[start:], uint32(length))
	// write the message
	copy(t.segment.data[dataStart:], data)
	t.segment.data[dataEnd] = 255
	// location of the next write
	t.state.offset = uint32(dataEnd) + 1
	t.dataLock.Unlock()

	// sync the part of the data file we just wrote
	from := start / pageSize * pageSize
	to := dataStart + length + 1 - from
	_, _, errno := syscall.Syscall(syscall.SYS_MSYNC, uintptr(unsafe.Pointer(&t.segment.data[from])), uintptr(to), syscall.MS_SYNC)
	if errno != 0 {
		return syscall.Errno(errno)
	}

	// notify channels that a new message is waiting
	t.messageAdded <- blank

	return nil
}

func (t *Topic) Channel(name string, config *ChannelConfiguration) (*Channel, error) {
	if len(name) > MAX_CHANNEL_NAME_SIZE {
		return nil, ChannelNameLenErr
	}
	if config == nil {
		config = ConfigureChannel()
	}
	config.name = name
	res := &addChannelWork{
		c:      make(chan *addChannelWork),
		config: config,
	}
	t.addChannel <- res
	<-res.c
	return res.channel, res.err
}

func (t *Topic) expand() error {
	segment := newSegment(t)
	t.segmentsLock.Lock()
	t.segments[segment.id] = segment
	t.segmentsLock.Unlock()
	if t.segment != nil {
		// create a pointer to the next segment id
		t.segment.nextId = segment.id
		t.segment.size = t.state.offset
		if err := t.segment.syncHeader(); err != nil {
			return err
		}
		t.channelsLock.RLock()
		noChannels := len(t.states.channels) == 0
		t.channelsLock.RUnlock()
		if noChannels {
			t.deleteSegment(t.segment.id)
		}
	}
	t.segment = segment
	t.state.offset = SEGMENT_HEADER_SIZE
	t.state.segmentId = segment.id
	return nil
}

func (t *Topic) worker() {
	for {
		select {
		case work := <-t.addChannel:
			work.channel, work.err = t.createChannel(work.config)
			work.c <- work
		case <-t.messageAdded:
			t.channelsLock.RLock()
			for _, c := range t.channels {
				c.notify()
			}
			t.channelsLock.RUnlock()
		}
	}
}

func (t *Topic) createChannel(config *ChannelConfiguration) (*Channel, error) {
	t.channelsLock.Lock()
	_, exists := t.channels[config.name]
	if exists {
		t.channelsLock.Unlock()
		if config.temp {
			// try again until we've created one
			return t.createChannel(config)
		}
		return nil, ChannelExistsErr
	}
	defer t.channelsLock.Unlock()

	c := newChannel(t, config)
	if config.temp {
		c.state = new(State)
	} else {
		c.state = t.states.getOrCreate(c.name)
		if c.state == nil {
			return nil, ChannelCreateErr
		}
	}

	// if we have a temp channel, or a new channel, set the position to the
	// writer's current position
	if config.temp || c.state.segmentId == 0 {
		t.dataLock.RLock()
		c.state.offset = t.state.offset
		c.state.segmentId = t.state.segmentId
		t.dataLock.RUnlock()
	}
	t.channels[c.name] = c
	return c, nil
}

func (t *Topic) read(channel *Channel) []byte {
	state := channel.state

	// no need to lock the channel here since the state of the channel is only
	// ever changed after this point (either to move to the next segment or message)
	t.dataLock.RLock()
	isCurrentSegment := state.segmentId == t.state.segmentId
	isCurrentOffset := state.offset == t.state.offset
	t.dataLock.RUnlock()

	// are we fully caught up with the topic?
	if isCurrentSegment && isCurrentOffset {
		return nil
	}

	segment := t.loadSegment(state.segmentId)
	// If we aren't on the current segment, we might be at the end of an old
	// segment. We can safely read old segments without locking since the only
	// writer (the topic), is done with it
	if !isCurrentSegment {
		if state.offset >= segment.size {
			previousId := state.segmentId
			segment = t.loadSegment(segment.nextId)
			channel.changeSegment(segment)
			if t.isSegmentUsable(previousId) == false {
				t.deleteSegment(previousId)
			}
		}
	}

	l := encoder.Uint32(segment.data[state.offset:])
	start := state.offset + 4
	end := start + uint32(l)
	return segment.data[start:end]
}

func (t *Topic) loadSegment(id uint64) *Segment {
	t.segmentsLock.RLock()
	segment := t.segments[id]
	t.segmentsLock.RUnlock()
	if segment != nil {
		return segment
	}

	t.segmentsLock.Lock()
	defer t.segmentsLock.Unlock()
	if segment := t.segments[id]; segment != nil {
		return segment
	}
	segment = openSegment(t, id, false)
	t.segments[id] = segment
	return segment
}

// A segment is usable if any channel references it or an earlier one
// How we check this depends on whether or not the channel is active
// (currently consuming message). For an active channel, we ask the channel,
// for an inactive channel, we can query the state direclty knowing that
// no one can be writing to it at the same time
func (t *Topic) isSegmentUsable(id uint64) bool {
	t.channelsLock.RLock()
	defer t.channelsLock.RUnlock()
	for name, state := range t.states.channels {
		if channel, active := t.channels[name]; active {
			if channel.isSegmentUsable(id) == true {
				return true
			}
		} else if state.isSegmentUsable(id) == true {
			return true
		}
	}
	return false
}

// The topic's persisted state could be behind from the actual state. However,
// we can determine the actual state based on the data which is guaranteed to be
// up to date (the next reference id and the data itself)
func (t *Topic) findWritePosition(segment *Segment) {
	for segment.nextId != 0 {
		segment = openSegment(t, segment.nextId, false)
	}
	t.segment = segment
	t.state.segmentId = segment.id

	offset := SEGMENT_HEADER_SIZE
	for offset < uint32(t.segmentSize) {
		l := encoder.Uint32(segment.data[offset:])
		if l == 0 {
			break
		}
		if segment.data[offset+l+4] != 255 {
			break
		}
		offset += l + MESSAGE_OVERHEAD
	}
	t.state.offset = offset
}

func (t *Topic) deleteSegment(id uint64) {
	t.segmentsLock.Lock()
	segment := t.segments[id]
	delete(t.segments, id)
	t.segmentsLock.Unlock()
	if segment != nil {
		segment.delete()
	}
}
<commit_msg>we can't safely reference t.segment outside the lock<commit_after>package sq

import (
	"encoding/binary"
	"fmt"
	"os"
	"path"
	"sync"
	"syscall"
	"unsafe"
)

const (
	MESSAGE_OVERHEAD = 5 //length + termination
)

var (
	encoder            = binary.LittleEndian
	blank              = struct{}{}
	ChannelNameLenErr  = fmt.Errorf("channel name cannot exceed %d characters", MAX_CHANNEL_NAME_SIZE)
	ChannelExistsErr   = fmt.Errorf("channel already exists")
	ChannelCreateErr   = fmt.Errorf("channel count not be created")
	MessageTooLargeErr = fmt.Errorf("message too large")
	pageSize           = os.Getpagesize()
)

type addChannelWork struct {
	err     error
	channel *Channel
	config  *ChannelConfiguration
	c       chan *addChannelWork
}

type Topic struct {
	path         string
	state        *State
	states       *States
	segmentSize  int
	channelsLock sync.RWMutex
	channels     map[string]*Channel
	segment      *Segment
	segmentsLock sync.RWMutex
	segments     map[uint64]*Segment
	addChannel   chan *addChannelWork
	messageAdded chan struct{}
	dataLock     sync.RWMutex
}

func OpenTopic(name string, config *TopicConfiguration) (*Topic, error) {
	t := &Topic{
		path:         path.Join(config.path, name),
		channels:     make(map[string]*Channel),
		segments:     make(map[uint64]*Segment),
		addChannel:   make(chan *addChannelWork),
		segmentSize:  config.segmentSize,
		messageAdded: make(chan struct{}, 64),
	}
	err := loadStates(t)
	if err != nil {
		return nil, err
	}

	if id := t.state.segmentId; id == 0 {
		if err := t.expand(); err != nil {
			return nil, err
		}
		if err := t.states.syncTopic(); err != nil {
			return nil, err
		}
	} else {
		t.findWritePosition(openSegment(t, id, false))
		t.segments[id] = t.segment
	}
	go t.worker()
	return t, nil
}

func (t *Topic) Write(data []byte) error {
	length := len(data)

	t.dataLock.Lock()
	start := int(t.state.offset)
	dataStart := start + 4
	dataEnd := dataStart + length

	// do we have enough space in the current segment?
	if dataEnd > t.segmentSize {
		if length+MESSAGE_OVERHEAD+int(SEGMENT_HEADER_SIZE) > t.segmentSize {
			return MessageTooLargeErr
		}
		if err := t.expand(); err != nil {
			return err
		}
		start = int(SEGMENT_HEADER_SIZE)
		dataStart = start + 4
		dataEnd = dataStart + length
	}

	// write the message length
	encoder.PutUint32(t.segment.data[start:], uint32(length))
	// write the message
	copy(t.segment.data[dataStart:], data)
	t.segment.data[dataEnd] = 255
	// location of the next write
	t.state.offset = uint32(dataEnd) + 1
	segment := t.segment
	t.dataLock.Unlock()

	// sync the part of the data file we just wrote
	from := start / pageSize * pageSize
	to := dataStart + length + 1 - from
	_, _, errno := syscall.Syscall(syscall.SYS_MSYNC, uintptr(unsafe.Pointer(&segment.data[from])), uintptr(to), syscall.MS_SYNC)
	if errno != 0 {
		return syscall.Errno(errno)
	}

	// notify channels that a new message is waiting
	t.messageAdded <- blank

	return nil
}

func (t *Topic) Channel(name string, config *ChannelConfiguration) (*Channel, error) {
	if len(name) > MAX_CHANNEL_NAME_SIZE {
		return nil, ChannelNameLenErr
	}
	if config == nil {
		config = ConfigureChannel()
	}
	config.name = name
	res := &addChannelWork{
		c:      make(chan *addChannelWork),
		config: config,
	}
	t.addChannel <- res
	<-res.c
	return res.channel, res.err
}

func (t *Topic) expand() error {
	segment := newSegment(t)
	t.segmentsLock.Lock()
	t.segments[segment.id] = segment
	t.segmentsLock.Unlock()
	if t.segment != nil {
		// create a pointer to the next segment id
		t.segment.nextId = segment.id
		t.segment.size = t.state.offset
		if err := t.segment.syncHeader(); err != nil {
			return err
		}
		t.channelsLock.RLock()
		noChannels := len(t.states.channels) == 0
		t.channelsLock.RUnlock()
		if noChannels {
			t.deleteSegment(t.segment.id)
		}
	}
	t.segment = segment
	t.state.offset = SEGMENT_HEADER_SIZE
	t.state.segmentId = segment.id
	return nil
}

func (t *Topic) worker() {
	for {
		select {
		case work := <-t.addChannel:
			work.channel, work.err = t.createChannel(work.config)
			work.c <- work
		case <-t.messageAdded:
			t.channelsLock.RLock()
			for _, c := range t.channels {
				c.notify()
			}
			t.channelsLock.RUnlock()
		}
	}
}

func (t *Topic) createChannel(config *ChannelConfiguration) (*Channel, error) {
	t.channelsLock.Lock()
	_, exists := t.channels[config.name]
	if exists {
		t.channelsLock.Unlock()
		if config.temp {
			// try again until we've created one
			return t.createChannel(config)
		}
		return nil, ChannelExistsErr
	}
	defer t.channelsLock.Unlock()

	c := newChannel(t, config)
	if config.temp {
		c.state = new(State)
	} else {
		c.state = t.states.getOrCreate(c.name)
		if c.state == nil {
			return nil, ChannelCreateErr
		}
	}

	// if we have a temp channel, or a new channel, set the position to the
	// writer's current position
	if config.temp || c.state.segmentId == 0 {
		t.dataLock.RLock()
		c.state.offset = t.state.offset
		c.state.segmentId = t.state.segmentId
		t.dataLock.RUnlock()
	}
	t.channels[c.name] = c
	return c, nil
}

func (t *Topic) read(channel *Channel) []byte {
	state := channel.state

	// no need to lock the channel here since the state of the channel is only
	// ever changed after this point (either to move to the next segment or message)
	t.dataLock.RLock()
	isCurrentSegment := state.segmentId == t.state.segmentId
	isCurrentOffset := state.offset == t.state.offset
	t.dataLock.RUnlock()

	// are we fully caught up with the topic?
	if isCurrentSegment && isCurrentOffset {
		return nil
	}

	segment := t.loadSegment(state.segmentId)
	// If we aren't on the current segment, we might be at the end of an old
	// segment. We can safely read old segments without locking since the only
	// writer (the topic), is done with it
	if !isCurrentSegment {
		if state.offset >= segment.size {
			previousId := state.segmentId
			segment = t.loadSegment(segment.nextId)
			channel.changeSegment(segment)
			if t.isSegmentUsable(previousId) == false {
				t.deleteSegment(previousId)
			}
		}
	}

	l := encoder.Uint32(segment.data[state.offset:])
	start := state.offset + 4
	end := start + uint32(l)
	return segment.data[start:end]
}

func (t *Topic) loadSegment(id uint64) *Segment {
	t.segmentsLock.RLock()
	segment := t.segments[id]
	t.segmentsLock.RUnlock()
	if segment != nil {
		return segment
	}

	t.segmentsLock.Lock()
	defer t.segmentsLock.Unlock()
	if segment := t.segments[id]; segment != nil {
		return segment
	}
	segment = openSegment(t, id, false)
	t.segments[id] = segment
	return segment
}

// A segment is usable if any channel references it or an earlier one
// How we check this depends on whether or not the channel is active
// (currently consuming message). For an active channel, we ask the channel,
// for an inactive channel, we can query the state direclty knowing that
// no one can be writing to it at the same time
func (t *Topic) isSegmentUsable(id uint64) bool {
	t.channelsLock.RLock()
	defer t.channelsLock.RUnlock()
	for name, state := range t.states.channels {
		if channel, active := t.channels[name]; active {
			if channel.isSegmentUsable(id) == true {
				return true
			}
		} else if state.isSegmentUsable(id) == true {
			return true
		}
	}
	return false
}

// The topic's persisted state could be behind from the actual state. However,
// we can determine the actual state based on the data which is guaranteed to be
// up to date (the next reference id and the data itself)
func (t *Topic) findWritePosition(segment *Segment) {
	for segment.nextId != 0 {
		segment = openSegment(t, segment.nextId, false)
	}
	t.segment = segment
	t.state.segmentId = segment.id

	offset := SEGMENT_HEADER_SIZE
	for offset < uint32(t.segmentSize) {
		l := encoder.Uint32(segment.data[offset:])
		if l == 0 {
			break
		}
		if segment.data[offset+l+4] != 255 {
			break
		}
		offset += l + MESSAGE_OVERHEAD
	}
	t.state.offset = offset
}

func (t *Topic) deleteSegment(id uint64) {
	t.segmentsLock.Lock()
	segment := t.segments[id]
	delete(t.segments, id)
	t.segmentsLock.Unlock()
	if segment != nil {
		segment.delete()
	}
}
<|endoftext|>


<commit_before>package main

import (
	"fmt"
	"github.com/wtolson/go-taglib"
	"io"
	"log"
	"os"
	"path/filepath"
)

// Track info
type Track struct {
	Title  string
	Artist string
	Album  string
	Ext    string
	Number int
	Path   string
}

// NewTrack returns Track and error
func NewTrack(fileName string) (Track, error) {
	f, err := taglib.Read(fileName)
	if err != nil {
		log.Fatal("can't " + fileName + " file read")
		return Track{}, err
	}

	fileAbsPath, err := filepath.Abs(fileName)
	if err != nil {
		log.Fatal("can't get " + fileName + "absolute file path")
		return Track{}, err
	}

	t := Track{
		Title:  f.Title(),
		Artist: f.Artist(),
		Album:  f.Album(),
		Number: f.Track(),
		Ext:    filepath.Ext(fileName),
		Path:   fileAbsPath,
	}

	return t, nil
}

// TransferTo transfer track to dir
func (t Track) TransferTo(dir string) error {
	dst := fmt.Sprintf("%s/%s/%s/%02d-%s%s", dir, t.Artist, t.Album, t.Number, t.Title, t.Ext)
	if err := os.MkdirAll(filepath.Dir(dst), 0777); err != nil {
		log.Fatal("can't create " + filepath.Dir(dst) + "dir")
		return err
	}
	if err := copy(t.Path, dst); err != nil {
		log.Fatal("can't copy " + t.Path + " to " + dst)
		return err
	}

	return nil
}

func copy(src, dst string) error {
	s, err := os.Open(src)
	if err != nil {
		return err
	}

	defer s.Close()
	d, err := os.Create(dst)
	if err != nil {
		return err
	}
	if _, err := io.Copy(d, s); err != nil {
		d.Close()
		return err
	}
	return d.Close()
}
<commit_msg>:art: refactored<commit_after>package main

import (
	"fmt"
	"github.com/wtolson/go-taglib"
	"io"
	"log"
	"os"
	"path/filepath"
)

// Track info
type Track struct {
	Title  string
	Artist string
	Album  string
	Ext    string
	Number int
	Path   string
}

// NewTrack returns Track and error
func NewTrack(fileName string) (Track, error) {
	f, err := taglib.Read(fileName)
	if err != nil {
		log.Fatal("can't " + fileName + " file read")
		return Track{}, err
	}

	fileAbsPath, err := filepath.Abs(fileName)
	if err != nil {
		log.Fatal("can't get " + fileName + "absolute file path")
		return Track{}, err
	}

	t := Track{
		Title:  f.Title(),
		Artist: f.Artist(),
		Album:  f.Album(),
		Number: f.Track(),
		Ext:    filepath.Ext(fileName),
		Path:   fileAbsPath,
	}

	return t, nil
}

// TransferTo transfer track to dir
func (t Track) TransferTo(dir string) error {
	dst := assemblePath(dir, t)
	if err := os.MkdirAll(filepath.Dir(dst), 0777); err != nil {
		log.Fatal("can't create " + filepath.Dir(dst) + "dir")
		return err
	}
	if err := copy(t.Path, dst); err != nil {
		log.Fatal("can't copy " + t.Path + " to " + dst)
		return err
	}

	return nil
}

func assemblePath(dir string, t Track) string {
	return fmt.Sprintf("%s/%s/%s/%02d-%s%s", dir, t.Artist, t.Album, t.Number, t.Title, t.Ext)
}

func copy(src, dst string) error {
	s, err := os.Open(src)
	if err != nil {
		return err
	}

	defer s.Close()
	d, err := os.Create(dst)
	if err != nil {
		return err
	}
	if _, err := io.Copy(d, s); err != nil {
		d.Close()
		return err
	}
	return d.Close()
}
<|endoftext|>


<commit_before>package main

import (
	"fmt"
	"strconv"

	"github.com/sjwhitworth/golearn/base"
	. "github.com/sjwhitworth/golearn/linear_models"
)

func errexit(err error) {
	if err != nil {
		panic(err)
	}
}

func main() {
	rawData, err := base.ParseCSVToInstances("data2.csv", true)
	errexit(err)

	lr := NewLinearRegression()

	//Do a training-test split
	trainData, testData := base.InstancesTrainTestSplit(rawData, 0.60)

	err1 := lr.Fit(trainData)
	errexit(err1)

	predictions, err2 := lr.Predict(testData)
	errexit(err2)

	_, rows := predictions.Size()

	total := 0.0
	m := 0.0
	//n := 0.0
	for i := 0; i < rows; i++ {
		actualValue, _ := strconv.ParseFloat(base.GetClass(testData, i), 64)
		expectedValue, _ := strconv.ParseFloat(base.GetClass(predictions, i), 64)

		if expectedValue <= 0 && actualValue == 0 {
			continue
		}
		if expectedValue < 0 {
			expectedValue = 0
		}
		if expectedValue > 200 || actualValue > 200 {
			d := expectedValue / actualValue
			if d > 1 {
				d = 1 / d
			}
			total += d
			m++
			fmt.Println(expectedValue, actualValue)
		}
	}
	fmt.Println(total / m)
}
<commit_msg>split train,test<commit_after>package main

import (
	"fmt"
	"strconv"

	"github.com/sjwhitworth/golearn/base"
	. "github.com/sjwhitworth/golearn/linear_models"
)

func errexit(err error) {
	if err != nil {
		panic(err)
	}
}

func main() {
	trainData1, err := base.ParseCSVToInstances("datatrain1.csv", true)
	errexit(err)
	testData, err := base.ParseCSVToInstances("data1.csv", true)
	errexit(err)

	lr := NewLinearRegression()

	err1 := lr.Fit(trainData1)
	errexit(err1)

	predictions, err2 := lr.Predict(testData)
	errexit(err2)

	_, rows := predictions.Size()

	total := 0.0
	m := 0.0
	//n := 0.0
	for i := 0; i < rows; i++ {
		actualValue, _ := strconv.ParseFloat(base.GetClass(testData, i), 64)
		expectedValue, _ := strconv.ParseFloat(base.GetClass(predictions, i), 64)

		if expectedValue <= 0 && actualValue == 0 {
			continue
		}
		if expectedValue < 0 {
			expectedValue = 0
		}
		if expectedValue > 20 || actualValue > 20 {
			d := expectedValue / actualValue
			if d > 1 {
				d = 1 / d
			}
			total += d
			m++
			fmt.Println(expectedValue, actualValue)
		}
	}
	fmt.Println(total / m)
}
<|endoftext|>


<commit_before>package main

import (
	"fmt"
	"github.com/codegangsta/cli"
	"os"
)

var VERSION = "0.1.0-beta"

type boolmap map[string]bool

var ctRequired = []string{
	"marathon-host",
	"mesos-host",
}

var csRequired = []string{
	"marathon-host",
	"mesos-host",
}

func buildApp() *cli.App {
	app := cli.NewApp()
	app.Name = "notadash-mon"
	app.Usage = "Monitoring utility for the Mesos/Marathon/Docker stack --> decidedly not-a-dash"
	app.EnableBashCompletion = true
	app.Version = VERSION

	app.Flags = []cli.Flag{
		cli.BoolFlag{
			Name:  "verbose",
			Usage: "Show more output",
		},
		cli.StringFlag{
			Name:   "marathon-host",
			Usage:  "URL to use for Marathon cluster discovery.",
			EnvVar: "NOTADASH_MARATHON_URL",
		},
		cli.StringFlag{
			Name:   "mesos-host",
			Usage:  "URL to use for Mesos cluster discovery.",
			EnvVar: "NOTADASH_MESOS_URL",
		},
		cli.BoolFlag{
			Name:  "ignore-deploys",
			Usage: "Ignore active deployments when checking consensus",
		},
		//        cli.StringFlag{
		//            Name:  "c, config",
		//            Usage: "Specify a config file (default: ~/.notadash.gcfg)",
		//            Value: filepath.Join(os.Getenv("HOME"), ".notadash.gcfg"),
		//            EnvVar: "NOTADASH_CONFIG",
		//        },
	}

	app.Commands = []cli.Command{
		{
			Name:   "resources",
			Usage:  "Show resource allocation per node across the cluster.",
			Action: showAllocation,
		},
		{
			Name:   "tasks",
			Usage:  "Cross-check all tasks registered with Mesos and Marathon.",
			Action: checkTasks,
		},
		{
			Name: "slave",
			Flags: []cli.Flag{
				cli.BoolFlag{
					Name:  "kill-stragglers",
					Usage: "Kill containers which are still running and registered with Mesos, but Marathon has inconveniently forgotten.",
				},
			},
			Usage:  "Verify all tasks registered for mesos slave are running as expected. Must be run on target mesos slave.",
			Action: checkSlave,
		},
	}

	return app
}

func showAllocation(ctx *cli.Context) {
	if missing, err := validateContext(ctx, ctRequired); err != nil {
		fmt.Println(err)
		fmt.Printf("The following parameters must be defined: %s\n", missing)
		os.Exit(2)
	}

	exitStatus := runShowAllocation(ctx)
	os.Exit(exitStatus)
}

func checkTasks(ctx *cli.Context) {
	if missing, err := validateContext(ctx, ctRequired); err != nil {
		fmt.Println(err)
		fmt.Printf("The following parameters must be defined: %s\n", missing)
		os.Exit(2)
	}

	exitStatus := runCheckTasks(ctx)
	os.Exit(exitStatus)
}

func checkSlave(ctx *cli.Context) {
	if missing, err := validateContext(ctx, csRequired); err != nil {
		fmt.Println(err)
		fmt.Printf("The following parameters must be defined: %s\n", missing)
		os.Exit(1)
	}

	exitStatus := runCheckSlave(ctx)
	os.Exit(exitStatus)
}
<commit_msg>remove marathon-host requirement from resources cli cmd<commit_after>package main

import (
	"fmt"
	"github.com/codegangsta/cli"
	"os"
)

var VERSION = "0.1.0-beta"

type boolmap map[string]bool

var saRequired = []string{
	"mesos-host",
}

var ctRequired = []string{
	"marathon-host",
	"mesos-host",
}

var csRequired = []string{
	"marathon-host",
	"mesos-host",
}

func buildApp() *cli.App {
	app := cli.NewApp()
	app.Name = "notadash-mon"
	app.Usage = "Monitoring utility for the Mesos/Marathon/Docker stack --> decidedly not-a-dash"
	app.EnableBashCompletion = true
	app.Version = VERSION

	app.Flags = []cli.Flag{
		cli.BoolFlag{
			Name:  "verbose",
			Usage: "Show more output",
		},
		cli.StringFlag{
			Name:   "marathon-host",
			Usage:  "URL to use for Marathon cluster discovery.",
			EnvVar: "NOTADASH_MARATHON_URL",
		},
		cli.StringFlag{
			Name:   "mesos-host",
			Usage:  "URL to use for Mesos cluster discovery.",
			EnvVar: "NOTADASH_MESOS_URL",
		},
		cli.BoolFlag{
			Name:  "ignore-deploys",
			Usage: "Ignore active deployments when checking consensus",
		},
		//        cli.StringFlag{
		//            Name:  "c, config",
		//            Usage: "Specify a config file (default: ~/.notadash.gcfg)",
		//            Value: filepath.Join(os.Getenv("HOME"), ".notadash.gcfg"),
		//            EnvVar: "NOTADASH_CONFIG",
		//        },
	}

	app.Commands = []cli.Command{
		{
			Name:   "resources",
			Usage:  "Show resource allocation per node across the cluster.",
			Action: showAllocation,
		},
		{
			Name:   "tasks",
			Usage:  "Cross-check all tasks registered with Mesos and Marathon.",
			Action: checkTasks,
		},
		{
			Name: "slave",
			Flags: []cli.Flag{
				cli.BoolFlag{
					Name:  "kill-stragglers",
					Usage: "Kill containers which are still running and registered with Mesos, but Marathon has inconveniently forgotten.",
				},
			},
			Usage:  "Verify all tasks registered for mesos slave are running as expected. Must be run on target mesos slave.",
			Action: checkSlave,
		},
	}

	return app
}

func showAllocation(ctx *cli.Context) {
	if missing, err := validateContext(ctx, saRequired); err != nil {
		fmt.Println(err)
		fmt.Printf("The following parameters must be defined: %s\n", missing)
		os.Exit(2)
	}

	exitStatus := runShowAllocation(ctx)
	os.Exit(exitStatus)
}

func checkTasks(ctx *cli.Context) {
	if missing, err := validateContext(ctx, ctRequired); err != nil {
		fmt.Println(err)
		fmt.Printf("The following parameters must be defined: %s\n", missing)
		os.Exit(2)
	}

	exitStatus := runCheckTasks(ctx)
	os.Exit(exitStatus)
}

func checkSlave(ctx *cli.Context) {
	if missing, err := validateContext(ctx, csRequired); err != nil {
		fmt.Println(err)
		fmt.Printf("The following parameters must be defined: %s\n", missing)
		os.Exit(1)
	}

	exitStatus := runCheckSlave(ctx)
	os.Exit(exitStatus)
}
<|endoftext|>


<commit_before>package main

import (
	"flag"
	"fmt"
	"github.com/kisielk/raven-go/raven"
)

func main() {
	dsn := flag.String("dsn", "", "Sentry dsn")
	flag.Parse()

	if *dsn == "" {
		fmt.Printf("You need to use the --dsn flag to specify the Sentry server's dsn\n")
		return
	}

	fmt.Printf("Connecting to dsn: %v\n", *dsn)
	client, err := raven.NewRavenClient(*dsn)
	if err != nil {
		fmt.Printf("Could not connect: %v", dsn)
	}
	client.CaptureMessage("Hello world")
}
<commit_msg>Made the example a little more like the Python client<commit_after>package main

import (
	"fmt"
	"github.com/kisielk/raven-go/raven"
	"os"
	"strings"
)

func main() {

	var dsn string
	if len(os.Args) > 2 {
		dsn = strings.Join(os.Args[2:], " ")
	} else {
		dsn = os.Getenv("SENTRY_DSN")
	}

	if dsn == "" {
		fmt.Printf("Error: No configuration detected!\n")
		fmt.Printf("You must either pass a DSN to the command, or set the SENTRY_DSN environment variable\n")
		return
	}

	fmt.Printf("Using DSN configuration:\n %v\n", dsn)
	client, err := raven.NewRavenClient(dsn)

	if err != nil {
		fmt.Printf("could not connect: %v", dsn)
	}

	fmt.Printf("Sending a test message...\n")
	id, err := client.CaptureMessage("This is a test message generated using ``goraven test``")

	if err != nil {
		fmt.Printf("failed: %v\n", err)
		return
	}

	fmt.Printf("Message captured, id: %v", id)
}
<|endoftext|>


<commit_before>package main

import (
	"net/http"
	"io"
	"os"
	"strconv"

	"github.com/philips/go-namespace/net"
)

func HelloPid(w http.ResponseWriter, req *http.Request) {
	io.WriteString(w, "hello pid namespace!\n")
}

func HelloServer(w http.ResponseWriter, req *http.Request) {
	io.WriteString(w, "hello original namespace!\n")
}

// This example application creates an http listening inside of the namespace
// of the process given in os.Args[0] on port 8080 and an http server in the
// original namespace each with different messages.
func main() {
	args := os.Args

	pid, _ := strconv.Atoi(args[1])
	l, err := net.ListenNamespace(uintptr(pid), "tcp", ":8080")
	if err != nil {
		panic(err)
	}
	http.HandleFunc("/", HelloPid)
	go http.Serve(l, nil)

	out := http.NewServeMux()
	out.HandleFunc("/", HelloServer)
	srv := &http.Server{
		Addr:           ":8080",
		Handler:        out,
	}
	srv.ListenAndServe()
}
<commit_msg>chore(example): remove this silly example<commit_after><|endoftext|>


<commit_before>package main

import (
	"runtime"
	"time"

	"github.com/veandco/go-sdl2/sdl"

	"github.com/kasworld/actionstat"
	"github.com/kasworld/htmlcolors"
	"github.com/kasworld/log"

	"github.com/kasworld/go-sdlgui"
)

func main() {
	runtime.LockOSThread()

	app := App{
		Stat:  actionstat.NewActionStat(),
		SdlCh: make(chan interface{}, 1),
		Keys:  make(map[sdl.Scancode]bool),
	}
	app.Run()
	runtime.UnlockOSThread()
}

type App struct {
	Quit  bool
	SdlCh chan interface{}
	Keys  sdlgui.KeyState
	Win   *sdlgui.Window
	Stat  *actionstat.ActionStat

	controls []sdlgui.ControlI
	msgtexts *sdlgui.TextBoxControl
	barctrl  *sdlgui.TextControl
}

func (g *App) addControls() {
	g.Win = sdlgui.NewWindow("", 1024, 800, true)

	g.msgtexts = sdlgui.NewTextBoxControl(
		0, 0, 0,
		1024, 720, 60,
		sdlgui.LoadFont("DejaVuSerif.ttf", 12))
	g.msgtexts.SetBG(htmlcolors.Gray.ToRGBA())
	g.Win.AddControl(g.msgtexts)

	g.barctrl = sdlgui.NewTextControl(
		0, 720, 0,
		1024, 80, "hello",
		sdlgui.LoadFont("DejaVuSerif.ttf", 36))
	g.barctrl.SetBG(htmlcolors.Pink.ToRGBA())
	g.Win.AddControl(g.barctrl)

	g.Win.UpdateAll()
}

func (g *App) Run() {
	g.addControls()
	sdlgui.SDLEvent2Ch(g.SdlCh)
	timerInfoCh := time.Tick(time.Duration(1000) * time.Millisecond)
	timerDrawCh := time.Tick(time.Duration(1000/60) * time.Millisecond)
	barlen := 0.0
	for !g.Quit {
		select {
		case data := <-g.SdlCh:
			if g.Win.ProcessSDLMouseEvent(data) ||
				g.Keys.ProcessSDLKeyEvent(data) {
				g.Quit = true
			}
			g.msgtexts.AddText("data %v", data)
			g.barctrl.SetBar(barlen)
			barlen += 0.01
			if barlen > 1 {
				barlen = 0
			}
			g.Stat.Inc()

		case <-timerDrawCh:
			g.msgtexts.DrawSurface()
			g.barctrl.DrawSurface()
			g.Win.Update()

		case <-timerInfoCh:
			log.Info("stat %v", g.Stat)
			g.Stat.UpdateLap()
		}
	}
}
<commit_msg>more structured example<commit_after>package main

import (
	"runtime"
	"time"

	"github.com/veandco/go-sdl2/sdl"

	"github.com/kasworld/actionstat"
	"github.com/kasworld/htmlcolors"
	"github.com/kasworld/log"

	"github.com/kasworld/go-sdlgui"
)

func main() {
	NewApp().Run()
}

type App struct {
	Quit     bool
	SdlCh    chan interface{}
	Keys     sdlgui.KeyState
	Win      *sdlgui.Window
	Controls sdlgui.ControlIList

	Stat    *actionstat.ActionStat
	msgtext *sdlgui.TextBoxControl
	barctrl *sdlgui.TextControl
}

func NewApp() *App {
	app := App{
		SdlCh: make(chan interface{}, 1),
		Keys:  make(map[sdl.Scancode]bool),
		Win:   sdlgui.NewWindow("SDL GUI Example", 1024, 800, true),

		Stat: actionstat.NewActionStat(),
	}
	app.addControls()
	app.Win.UpdateAll()
	return &app
}

func (app *App) AddControl(c sdlgui.ControlI) {
	app.Controls = append(app.Controls, c)
	app.Win.AddControl(c)
}

// changed for every app

func (g *App) addControls() {
	g.msgtext = sdlgui.NewTextBoxControl(
		0, 0, 0,
		1024, 720, 60,
		sdlgui.LoadFont("DejaVuSerif.ttf", 12))
	g.msgtext.SetBG(htmlcolors.Gray.ToRGBA())
	g.AddControl(g.msgtext)

	g.barctrl = sdlgui.NewTextControl(
		0, 720, 0,
		1024, 80, "hello",
		sdlgui.LoadFont("DejaVuSerif.ttf", 36))
	g.barctrl.SetBG(htmlcolors.Pink.ToRGBA())
	g.AddControl(g.barctrl)

}

func (app *App) Run() {
	// need to co-exist sdl lib
	runtime.LockOSThread()
	defer runtime.UnlockOSThread()

	// start sdl event loop
	sdlgui.SDLEvent2Ch(app.SdlCh)
	timerInfoCh := time.Tick(time.Duration(1000) * time.Millisecond)
	timerDrawCh := time.Tick(time.Duration(1000/60) * time.Millisecond)
	barlen := 0.0

	for !app.Quit {
		select {
		case data := <-app.SdlCh:
			if app.Win.ProcessSDLMouseEvent(data) ||
				app.Keys.ProcessSDLKeyEvent(data) {
				app.Quit = true
			}
			app.msgtext.AddText("data %v", data)
			app.barctrl.SetBar(barlen)
			barlen += 0.01
			if barlen > 1 {
				barlen = 0
			}
			app.Stat.Inc()

		case <-timerDrawCh:
			for _, v := range app.Controls {
				v.DrawSurface()
			}
			app.Win.Update()

		case <-timerInfoCh:
			log.Info("stat %v", app.Stat)
			app.Stat.UpdateLap()

		}
	}
}
<|endoftext|>


<commit_before>package sanitiser

import (
	"fmt"
	"reflect"
	"strings"
)

type Logger func(format string, v ...interface{})

var dbg Logger

func init() {

	dbg = func(string, ...interface{}) {}
}

func parseTag(tag string) []string {

	return strings.Split(tag, ",")
}

func contains(contexts []string, context string) bool {

	for _, d := range contexts {

		if d == context {

			return true
		}
	}

	return false
}

func traverseObjects(obj interface{}, context string, hierarchy string) error {

	var v reflect.Value
	var t reflect.Type
	var ok bool

	dbg("%v.%v(type %T)\n", hierarchy, obj, obj)

	// make sure this is a pointer, so that we can update the contents if needed
	if v, ok = obj.(reflect.Value); !ok {

		v = reflect.ValueOf(obj)
	}

	for v.Kind() == reflect.Ptr || v.Kind() == reflect.Interface {

		dbg("object is a pointer or an interface, calling Elem()\n")
		v = v.Elem()
	}

	if !v.IsValid() {

		return nil
	}

	dbg("%v.%v(type %T)\n", hierarchy, v, v)

	t = reflect.TypeOf(v.Interface())
	k := t.Kind()

	if k == reflect.Map {

		keys := v.MapKeys()
		for _, key := range keys {

			dbg("Processing object %v.%v[%v]\n", hierarchy, t.Name(), key)
			if err := traverseObjects(v.MapIndex(key), context, hierarchy+"["+fmt.Sprintf("%v", key)+"]"); err != nil {

				return err
			}
		}
	} else if k == reflect.Struct {

		for i := 0; i < t.NumField(); i++ {

			dbg("Processing field %v.%v(%v)\n", hierarchy, t.Field(i).Name, t.Field(i).Type)
			field := t.Field(i)
			field_kind := field.Type.Kind()

			if tag := field.Tag.Get("sanitise"); len(tag) > 0 {

				// the sanitise tag's value should be a comma-separated list of
				// contexts
				dbg("Field %v.%v(type %T) has a sanitise tag\n", hierarchy, field.Name, v.Field(i))
				contexts := parseTag(tag)
				if contains(contexts, context) || contains(contexts, "*") {
					// sanitise this field
					if !v.Field(i).CanSet() {

						return fmt.Errorf("Unable to set zero value for %v.%v", hierarchy, t.Field(i).Name)
					}

					dbg("Sanitising field %v.%v\n", hierarchy, t.Field(i).Name)
					v.Field(i).Set(reflect.New(t.Field(i).Type).Elem())
				}
			}

			if field_kind == reflect.Struct || field_kind == reflect.Interface || field_kind == reflect.Ptr || field_kind == reflect.Map {

				sv := v.Field(i)
				dbg("Processing object %v.%v(type %T)\n", hierarchy, sv, sv)

				if err := traverseObjects(sv, context, hierarchy+"."+t.Field(i).Name); err != nil {

					return err
				}
			}
		}
	}

	return nil
}

func Sanitise(obj interface{}, context string) error {

	return traverseObjects(obj, context, "")
}

func SetLogger(f Logger) {

	dbg = f
}
<commit_msg>+ TODO<commit_after>package sanitiser

import (
	"fmt"
	"reflect"
	"strings"
)

type Logger func(format string, v ...interface{})

var dbg Logger

func init() {

	dbg = func(string, ...interface{}) {}
}

func parseTag(tag string) []string {

	return strings.Split(tag, ",")
}

func contains(contexts []string, context string) bool {

	for _, d := range contexts {

		if d == context {

			return true
		}
	}

	return false
}

func traverseObjects(obj interface{}, context string, hierarchy string) error {

	// TODO: improve debug messages

	var v reflect.Value
	var t reflect.Type
	var ok bool

	dbg("%v.%v(type %T)\n", hierarchy, obj, obj)

	// make sure this is a pointer, so that we can update the contents if needed
	if v, ok = obj.(reflect.Value); !ok {

		v = reflect.ValueOf(obj)
	}

	for v.Kind() == reflect.Ptr || v.Kind() == reflect.Interface {

		dbg("object is a pointer or an interface, calling Elem()\n")
		v = v.Elem()
	}

	if !v.IsValid() {

		return nil
	}

	dbg("%v.%v(type %T)\n", hierarchy, v, v)

	t = reflect.TypeOf(v.Interface())
	k := t.Kind()

	if k == reflect.Map {

		keys := v.MapKeys()
		for _, key := range keys {

			dbg("Processing object %v.%v[%v]\n", hierarchy, t.Name(), key)
			if err := traverseObjects(v.MapIndex(key), context, hierarchy+"["+fmt.Sprintf("%v", key)+"]"); err != nil {

				return err
			}
		}
	} else if k == reflect.Struct {

		for i := 0; i < t.NumField(); i++ {

			dbg("Processing field %v.%v(%v)\n", hierarchy, t.Field(i).Name, t.Field(i).Type)
			field := t.Field(i)
			field_kind := field.Type.Kind()

			if tag := field.Tag.Get("sanitise"); len(tag) > 0 {

				// the sanitise tag's value should be a comma-separated list of
				// contexts
				dbg("Field %v.%v(type %T) has a sanitise tag\n", hierarchy, field.Name, v.Field(i))
				contexts := parseTag(tag)
				if contains(contexts, context) || contains(contexts, "*") {
					// sanitise this field
					if !v.Field(i).CanSet() {

						return fmt.Errorf("Unable to set zero value for %v.%v", hierarchy, t.Field(i).Name)
					}

					dbg("Sanitising field %v.%v\n", hierarchy, t.Field(i).Name)
					v.Field(i).Set(reflect.New(t.Field(i).Type).Elem())
				}
			}

			if field_kind == reflect.Struct || field_kind == reflect.Interface || field_kind == reflect.Ptr || field_kind == reflect.Map {

				sv := v.Field(i)
				dbg("Processing object %v.%v(type %T)\n", hierarchy, sv, sv)

				if err := traverseObjects(sv, context, hierarchy+"."+t.Field(i).Name); err != nil {

					return err
				}
			}
		}
	}

	return nil
}

func Sanitise(obj interface{}, context string) error {

	return traverseObjects(obj, context, "")
}

func SetLogger(f Logger) {

	dbg = f
}
<|endoftext|>


<commit_before>// Copyright 2011 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

package ssh_test

import (
	"bytes"
	"fmt"
	"io/ioutil"
	"log"
	"net"
	"net/http"

	"golang.org/x/crypto/ssh"
	"golang.org/x/crypto/ssh/terminal"
)

func ExampleNewServerConn() {
	// Public key authentication is done by comparing
	// the public key of a received connection
	// with the entries in the authorized_keys file.
	authorizedKeysBytes, err := ioutil.ReadFile("authorized_keys")
	if err != nil {
		log.Fatalf("Failed to load authorized_keys, err: %v", err)
	}

	authorizedKeysMap := map[string]bool{}
	for len(authorizedKeysBytes) > 0 {
		pubKey, _, _, rest, err := ssh.ParseAuthorizedKey(authorizedKeysBytes)
		if err != nil {
			log.Fatal(err)
		}

		authorizedKeysMap[string(pubKey.Marshal())] = true
		authorizedKeysBytes = rest
	}

	// An SSH server is represented by a ServerConfig, which holds
	// certificate details and handles authentication of ServerConns.
	config := &ssh.ServerConfig{
		// Remove to disable password auth.
		PasswordCallback: func(c ssh.ConnMetadata, pass []byte) (*ssh.Permissions, error) {
			// Should use constant-time compare (or better, salt+hash) in
			// a production setting.
			if c.User() == "testuser" && string(pass) == "tiger" {
				return nil, nil
			}
			return nil, fmt.Errorf("password rejected for %q", c.User())
		},

		// Remove to disable public key auth.
		PublicKeyCallback: func(c ssh.ConnMetadata, pubKey ssh.PublicKey) (*ssh.Permissions, error) {
			if authorizedKeysMap[string(pubKey.Marshal())] {
				return nil, nil
			}
			return nil, fmt.Errorf("unknown public key for %q", c.User())
		},
	}

	privateBytes, err := ioutil.ReadFile("id_rsa")
	if err != nil {
		log.Fatal("Failed to load private key: ", err)
	}

	private, err := ssh.ParsePrivateKey(privateBytes)
	if err != nil {
		log.Fatal("Failed to parse private key: ", err)
	}

	config.AddHostKey(private)

	// Once a ServerConfig has been configured, connections can be
	// accepted.
	listener, err := net.Listen("tcp", "0.0.0.0:2022")
	if err != nil {
		log.Fatal("failed to listen for connection: ", err)
	}
	nConn, err := listener.Accept()
	if err != nil {
		log.Fatal("failed to accept incoming connection: ", err)
	}

	// Before use, a handshake must be performed on the incoming
	// net.Conn.
	_, chans, reqs, err := ssh.NewServerConn(nConn, config)
	if err != nil {
		log.Fatal("failed to handshake: ", err)
	}
	// The incoming Request channel must be serviced.
	go ssh.DiscardRequests(reqs)

	// Service the incoming Channel channel.

	// Service the incoming Channel channel.
	for newChannel := range chans {
		// Channels have a type, depending on the application level
		// protocol intended. In the case of a shell, the type is
		// "session" and ServerShell may be used to present a simple
		// terminal interface.
		if newChannel.ChannelType() != "session" {
			newChannel.Reject(ssh.UnknownChannelType, "unknown channel type")
			continue
		}
		channel, requests, err := newChannel.Accept()
		if err != nil {
			log.Fatalf("Could not accept channel: %v", err)
		}

		// Sessions have out-of-band requests such as "shell",
		// "pty-req" and "env".  Here we handle only the
		// "shell" request.
		go func(in <-chan *ssh.Request) {
			for req := range in {
				req.Reply(req.Type == "shell", nil)
			}
		}(requests)

		term := terminal.NewTerminal(channel, "> ")

		go func() {
			defer channel.Close()
			for {
				line, err := term.ReadLine()
				if err != nil {
					break
				}
				fmt.Println(line)
			}
		}()
	}
}

func ExampleDial() {
	// An SSH client is represented with a ClientConn.
	//
	// To authenticate with the remote server you must pass at least one
	// implementation of AuthMethod via the Auth field in ClientConfig.
	config := &ssh.ClientConfig{
		User: "username",
		Auth: []ssh.AuthMethod{
			ssh.Password("yourpassword"),
		},
	}
	client, err := ssh.Dial("tcp", "yourserver.com:22", config)
	if err != nil {
		log.Fatal("Failed to dial: ", err)
	}

	// Each ClientConn can support multiple interactive sessions,
	// represented by a Session.
	session, err := client.NewSession()
	if err != nil {
		log.Fatal("Failed to create session: ", err)
	}
	defer session.Close()

	// Once a Session is created, you can execute a single command on
	// the remote side using the Run method.
	var b bytes.Buffer
	session.Stdout = &b
	if err := session.Run("/usr/bin/whoami"); err != nil {
		log.Fatal("Failed to run: " + err.Error())
	}
	fmt.Println(b.String())
}

func ExamplePublicKeys() {
	// A public key may be used to authenticate against the remote
	// server by using an unencrypted PEM-encoded private key file.
	//
	// If you have an encrypted private key, the crypto/x509 package
	// can be used to decrypt it.
	key, err := ioutil.ReadFile("/home/user/.ssh/id_rsa")
	if err != nil {
		log.Fatalf("unable to read private key: %v", err)
	}

	// Create the Signer for this private key.
	signer, err := ssh.ParsePrivateKey(key)
	if err != nil {
		log.Fatalf("unable to parse private key: %v", err)
	}

	config := &ssh.ClientConfig{
		User: "user",
		Auth: []ssh.AuthMethod{
			// Use the PublicKeys method for remote authentication.
			ssh.PublicKeys(signer),
		},
	}

	// Connect to the remote server and perform the SSH handshake.
	client, err := ssh.Dial("tcp", "host.com:22", config)
	if err != nil {
		log.Fatalf("unable to connect: %v", err)
	}
	defer client.Close()
}

func ExampleClient_Listen() {
	config := &ssh.ClientConfig{
		User: "username",
		Auth: []ssh.AuthMethod{
			ssh.Password("password"),
		},
	}
	// Dial your ssh server.
	conn, err := ssh.Dial("tcp", "localhost:22", config)
	if err != nil {
		log.Fatal("unable to connect: ", err)
	}
	defer conn.Close()

	// Request the remote side to open port 8080 on all interfaces.
	l, err := conn.Listen("tcp", "0.0.0.0:8080")
	if err != nil {
		log.Fatal("unable to register tcp forward: ", err)
	}
	defer l.Close()

	// Serve HTTP with your SSH server acting as a reverse proxy.
	http.Serve(l, http.HandlerFunc(func(resp http.ResponseWriter, req *http.Request) {
		fmt.Fprintf(resp, "Hello world!\n")
	}))
}

func ExampleSession_RequestPty() {
	// Create client config
	config := &ssh.ClientConfig{
		User: "username",
		Auth: []ssh.AuthMethod{
			ssh.Password("password"),
		},
	}
	// Connect to ssh server
	conn, err := ssh.Dial("tcp", "localhost:22", config)
	if err != nil {
		log.Fatal("unable to connect: ", err)
	}
	defer conn.Close()
	// Create a session
	session, err := conn.NewSession()
	if err != nil {
		log.Fatal("unable to create session: ", err)
	}
	defer session.Close()
	// Set up terminal modes
	modes := ssh.TerminalModes{
		ssh.ECHO:          0,     // disable echoing
		ssh.TTY_OP_ISPEED: 14400, // input speed = 14.4kbaud
		ssh.TTY_OP_OSPEED: 14400, // output speed = 14.4kbaud
	}
	// Request pseudo terminal
	if err := session.RequestPty("xterm", 80, 40, modes); err != nil {
		log.Fatal("request for pseudo terminal failed: ", err)
	}
	// Start remote shell
	if err := session.Shell(); err != nil {
		log.Fatal("failed to start shell: ", err)
	}
}
<commit_msg>ssh: fix height/width order in RequestPty example<commit_after>// Copyright 2011 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

package ssh_test

import (
	"bytes"
	"fmt"
	"io/ioutil"
	"log"
	"net"
	"net/http"

	"golang.org/x/crypto/ssh"
	"golang.org/x/crypto/ssh/terminal"
)

func ExampleNewServerConn() {
	// Public key authentication is done by comparing
	// the public key of a received connection
	// with the entries in the authorized_keys file.
	authorizedKeysBytes, err := ioutil.ReadFile("authorized_keys")
	if err != nil {
		log.Fatalf("Failed to load authorized_keys, err: %v", err)
	}

	authorizedKeysMap := map[string]bool{}
	for len(authorizedKeysBytes) > 0 {
		pubKey, _, _, rest, err := ssh.ParseAuthorizedKey(authorizedKeysBytes)
		if err != nil {
			log.Fatal(err)
		}

		authorizedKeysMap[string(pubKey.Marshal())] = true
		authorizedKeysBytes = rest
	}

	// An SSH server is represented by a ServerConfig, which holds
	// certificate details and handles authentication of ServerConns.
	config := &ssh.ServerConfig{
		// Remove to disable password auth.
		PasswordCallback: func(c ssh.ConnMetadata, pass []byte) (*ssh.Permissions, error) {
			// Should use constant-time compare (or better, salt+hash) in
			// a production setting.
			if c.User() == "testuser" && string(pass) == "tiger" {
				return nil, nil
			}
			return nil, fmt.Errorf("password rejected for %q", c.User())
		},

		// Remove to disable public key auth.
		PublicKeyCallback: func(c ssh.ConnMetadata, pubKey ssh.PublicKey) (*ssh.Permissions, error) {
			if authorizedKeysMap[string(pubKey.Marshal())] {
				return nil, nil
			}
			return nil, fmt.Errorf("unknown public key for %q", c.User())
		},
	}

	privateBytes, err := ioutil.ReadFile("id_rsa")
	if err != nil {
		log.Fatal("Failed to load private key: ", err)
	}

	private, err := ssh.ParsePrivateKey(privateBytes)
	if err != nil {
		log.Fatal("Failed to parse private key: ", err)
	}

	config.AddHostKey(private)

	// Once a ServerConfig has been configured, connections can be
	// accepted.
	listener, err := net.Listen("tcp", "0.0.0.0:2022")
	if err != nil {
		log.Fatal("failed to listen for connection: ", err)
	}
	nConn, err := listener.Accept()
	if err != nil {
		log.Fatal("failed to accept incoming connection: ", err)
	}

	// Before use, a handshake must be performed on the incoming
	// net.Conn.
	_, chans, reqs, err := ssh.NewServerConn(nConn, config)
	if err != nil {
		log.Fatal("failed to handshake: ", err)
	}
	// The incoming Request channel must be serviced.
	go ssh.DiscardRequests(reqs)

	// Service the incoming Channel channel.

	// Service the incoming Channel channel.
	for newChannel := range chans {
		// Channels have a type, depending on the application level
		// protocol intended. In the case of a shell, the type is
		// "session" and ServerShell may be used to present a simple
		// terminal interface.
		if newChannel.ChannelType() != "session" {
			newChannel.Reject(ssh.UnknownChannelType, "unknown channel type")
			continue
		}
		channel, requests, err := newChannel.Accept()
		if err != nil {
			log.Fatalf("Could not accept channel: %v", err)
		}

		// Sessions have out-of-band requests such as "shell",
		// "pty-req" and "env".  Here we handle only the
		// "shell" request.
		go func(in <-chan *ssh.Request) {
			for req := range in {
				req.Reply(req.Type == "shell", nil)
			}
		}(requests)

		term := terminal.NewTerminal(channel, "> ")

		go func() {
			defer channel.Close()
			for {
				line, err := term.ReadLine()
				if err != nil {
					break
				}
				fmt.Println(line)
			}
		}()
	}
}

func ExampleDial() {
	// An SSH client is represented with a ClientConn.
	//
	// To authenticate with the remote server you must pass at least one
	// implementation of AuthMethod via the Auth field in ClientConfig.
	config := &ssh.ClientConfig{
		User: "username",
		Auth: []ssh.AuthMethod{
			ssh.Password("yourpassword"),
		},
	}
	client, err := ssh.Dial("tcp", "yourserver.com:22", config)
	if err != nil {
		log.Fatal("Failed to dial: ", err)
	}

	// Each ClientConn can support multiple interactive sessions,
	// represented by a Session.
	session, err := client.NewSession()
	if err != nil {
		log.Fatal("Failed to create session: ", err)
	}
	defer session.Close()

	// Once a Session is created, you can execute a single command on
	// the remote side using the Run method.
	var b bytes.Buffer
	session.Stdout = &b
	if err := session.Run("/usr/bin/whoami"); err != nil {
		log.Fatal("Failed to run: " + err.Error())
	}
	fmt.Println(b.String())
}

func ExamplePublicKeys() {
	// A public key may be used to authenticate against the remote
	// server by using an unencrypted PEM-encoded private key file.
	//
	// If you have an encrypted private key, the crypto/x509 package
	// can be used to decrypt it.
	key, err := ioutil.ReadFile("/home/user/.ssh/id_rsa")
	if err != nil {
		log.Fatalf("unable to read private key: %v", err)
	}

	// Create the Signer for this private key.
	signer, err := ssh.ParsePrivateKey(key)
	if err != nil {
		log.Fatalf("unable to parse private key: %v", err)
	}

	config := &ssh.ClientConfig{
		User: "user",
		Auth: []ssh.AuthMethod{
			// Use the PublicKeys method for remote authentication.
			ssh.PublicKeys(signer),
		},
	}

	// Connect to the remote server and perform the SSH handshake.
	client, err := ssh.Dial("tcp", "host.com:22", config)
	if err != nil {
		log.Fatalf("unable to connect: %v", err)
	}
	defer client.Close()
}

func ExampleClient_Listen() {
	config := &ssh.ClientConfig{
		User: "username",
		Auth: []ssh.AuthMethod{
			ssh.Password("password"),
		},
	}
	// Dial your ssh server.
	conn, err := ssh.Dial("tcp", "localhost:22", config)
	if err != nil {
		log.Fatal("unable to connect: ", err)
	}
	defer conn.Close()

	// Request the remote side to open port 8080 on all interfaces.
	l, err := conn.Listen("tcp", "0.0.0.0:8080")
	if err != nil {
		log.Fatal("unable to register tcp forward: ", err)
	}
	defer l.Close()

	// Serve HTTP with your SSH server acting as a reverse proxy.
	http.Serve(l, http.HandlerFunc(func(resp http.ResponseWriter, req *http.Request) {
		fmt.Fprintf(resp, "Hello world!\n")
	}))
}

func ExampleSession_RequestPty() {
	// Create client config
	config := &ssh.ClientConfig{
		User: "username",
		Auth: []ssh.AuthMethod{
			ssh.Password("password"),
		},
	}
	// Connect to ssh server
	conn, err := ssh.Dial("tcp", "localhost:22", config)
	if err != nil {
		log.Fatal("unable to connect: ", err)
	}
	defer conn.Close()
	// Create a session
	session, err := conn.NewSession()
	if err != nil {
		log.Fatal("unable to create session: ", err)
	}
	defer session.Close()
	// Set up terminal modes
	modes := ssh.TerminalModes{
		ssh.ECHO:          0,     // disable echoing
		ssh.TTY_OP_ISPEED: 14400, // input speed = 14.4kbaud
		ssh.TTY_OP_OSPEED: 14400, // output speed = 14.4kbaud
	}
	// Request pseudo terminal
	if err := session.RequestPty("xterm", 40, 80, modes); err != nil {
		log.Fatal("request for pseudo terminal failed: ", err)
	}
	// Start remote shell
	if err := session.Shell(); err != nil {
		log.Fatal("failed to start shell: ", err)
	}
}
<|endoftext|>


<commit_before>package main

import (
	"fmt"
	"io/ioutil"
	"testing"
)

// TestParseExiftool tests the ParseExiftoolOutput function.
func TestParseExiftool(t *testing.T) {
	b, err := ioutil.ReadFile("test/exiftool.out")
	if err != nil {
		fmt.Print(err)
	}

	results := ParseExiftoolOutput(string(b), nil)

	if err != nil {
		t.Log(err)
	}

	if true {
		t.Log("results: ", results)
	}
}

// TestParseTRiD tests the ParseTRiDOutput function.
func TestParseTRiD(t *testing.T) {
	b, err := ioutil.ReadFile("test/trid.out") // just pass the file name
	if err != nil {
		fmt.Print(err)
	}

	trid := ParseTRiDOutput(string(b), nil)

	if true {
		t.Log("trid: ", trid)
	}
}

// ParseSsdeepOutput tests the ParseSsdeepOutput function.
func TestParseTRiDSsdeep(t *testing.T) {
	b, err := ioutil.ReadFile("test/ssdeep.out") // just pass the file name
	if err != nil {
		fmt.Print(err)
	}

	ssdeep := ParseSsdeepOutput(string(b), nil)

	if true {
		t.Log("ssdeep: ", ssdeep)
	}
}
<commit_msg>test markdown generator<commit_after>package main

import (
	"fmt"
	"io/ioutil"
	"testing"
)

// TestParseExiftool tests the ParseExiftoolOutput function.
func TestParseExiftool(t *testing.T) {
	b, err := ioutil.ReadFile("test/exiftool.out")
	if err != nil {
		fmt.Print(err)
	}

	results := ParseExiftoolOutput(string(b), nil)

	if err != nil {
		t.Log(err)
	}

	if true {
		t.Log("results: ", results)
	}
}

// TestParseTRiD tests the ParseTRiDOutput function.
func TestParseTRiD(t *testing.T) {
	b, err := ioutil.ReadFile("test/trid.out") // just pass the file name
	if err != nil {
		fmt.Print(err)
	}

	trid := ParseTRiDOutput(string(b), nil)

	if true {
		t.Log("trid: ", trid)
	}
}

// TestParseTRiDSsdeep tests the ParseSsdeepOutput function.
func TestParseTRiDSsdeep(t *testing.T) {
	b, err := ioutil.ReadFile("test/ssdeep.out") // just pass the file name
	if err != nil {
		fmt.Print(err)
	}

	ssdeep := ParseSsdeepOutput(string(b), nil)

	if true {
		t.Log("ssdeep: ", ssdeep)
	}
}

// TestGenerateMarkDownTable tests the ParseSsdeepOutput function.
func TestGenerateMarkDownTable(t *testing.T) {
	exifOut, err := ioutil.ReadFile("test/exiftool.out")
	if err != nil {
		fmt.Print(err)
	}

	tridOut, err := ioutil.ReadFile("test/trid.out")
	if err != nil {
		fmt.Print(err)
	}

	ssdeepOut, err := ioutil.ReadFile("test/ssdeep.out")
	if err != nil {
		fmt.Print(err)
	}

	fileInfo := FileInfo{
		// Magic:    fi.Magic,
		SSDeep:   ParseSsdeepOutput(string(ssdeepOut), nil),
		TRiD:     ParseTRiDOutput(string(tridOut), nil),
		Exiftool: ParseExiftoolOutput(string(exifOut), nil),
	}
	fileInfo.MarkDown = generateMarkDownTable(fileInfo)

	markDown := generateMarkDownTable(fileInfo)

	if true {
		t.Log("markDown: ", markDown)
	}
}
<|endoftext|>


<commit_before>package build

const helpText = `
Usage: packer build [options] TEMPLATE

  Will execute multiple builds in parallel as defined in the template.
  The various artifacts created by the template will be outputted.

Options:

  -debug                     Debug mode enabled for builds
  -force                     Force a build to continue if artifacts exist, deletes existing artifacts
  -except=foo,bar,baz        Build all builds other than these
  -only=foo,bar,baz          Only build the given builds by name
  -var 'key=value'           Variable for templates, can be used multiple times.
  -var-file=path         JSON file containing user variables.
`
<commit_msg>command/bulid: cosmetic, align help text<commit_after>package build

const helpText = `
Usage: packer build [options] TEMPLATE

  Will execute multiple builds in parallel as defined in the template.
  The various artifacts created by the template will be outputted.

Options:

  -debug                     Debug mode enabled for builds
  -force                     Force a build to continue if artifacts exist, deletes existing artifacts
  -except=foo,bar,baz        Build all builds other than these
  -only=foo,bar,baz          Only build the given builds by name
  -var 'key=value'           Variable for templates, can be used multiple times.
  -var-file=path             JSON file containing user variables.
`
<|endoftext|>


<commit_before>// Copyright © 2013 Steve Francia <spf@spf13.com>.
//
// Licensed under the Simple Public License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
// http://opensource.org/licenses/Simple-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package commands

import (
	"github.com/spf13/cobra"
	"os"
	"runtime/pprof"
)

var cpuProfilefile string
var benchmarkTimes int

var benchmark = &cobra.Command{
	Use:   "benchmark",
	Short: "Benchmark hugo by building a site a number of times",
	Long: `Hugo can build a site many times over and anlyze the
    running process creating a `,
	Run: bench,
}

func init() {
	benchmark.Flags().StringVar(&cpuProfilefile, "outputfile", "/tmp/hugo-cpuprofile", "path/filename for the profile file")
	benchmark.Flags().IntVarP(&benchmarkTimes, "count", "n", 13, "number of times to build the site")
}

func bench(cmd *cobra.Command, args []string) {
	f, err := os.Create(cpuProfilefile)

	if err != nil {
		panic(err)
	}

	pprof.StartCPUProfile(f)
	defer pprof.StopCPUProfile()

	for i := 0; i < benchmarkTimes; i++ {
		_ = buildSite()
	}
}
<commit_msg>Fix benchmark panic<commit_after>// Copyright © 2013 Steve Francia <spf@spf13.com>.
//
// Licensed under the Simple Public License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
// http://opensource.org/licenses/Simple-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package commands

import (
	"github.com/spf13/cobra"
	"os"
	"runtime/pprof"
)

var cpuProfilefile string
var benchmarkTimes int

var benchmark = &cobra.Command{
	Use:   "benchmark",
	Short: "Benchmark hugo by building a site a number of times",
	Long: `Hugo can build a site many times over and anlyze the
    running process creating a `,
	Run: func(cmd *cobra.Command, args []string) {
		InitializeConfig()
		bench(cmd, args)
	},
}

func init() {
	benchmark.Flags().StringVar(&cpuProfilefile, "outputfile", "/tmp/hugo-cpuprofile", "path/filename for the profile file")
	benchmark.Flags().IntVarP(&benchmarkTimes, "count", "n", 13, "number of times to build the site")
}

func bench(cmd *cobra.Command, args []string) {
	f, err := os.Create(cpuProfilefile)

	if err != nil {
		panic(err)
	}

	pprof.StartCPUProfile(f)
	defer pprof.StopCPUProfile()

	for i := 0; i < benchmarkTimes; i++ {
		_ = buildSite()
	}
}
<|endoftext|>


<commit_before>package state

//
// DefinitionSelect postgres specific query for definitions
//
const DefinitionSelect = `
select
  coalesce(td.arn,'')       as arn,
  td.definition_id          as definitionid,
  td.adaptive_resource_allocation as adaptiveresourceallocation,
  td.image                  as image,
  td.group_name             as groupname,
  td.container_name         as containername,
  coalesce(td.user,'')      as "user",
  td.alias                  as alias,
  td.memory                 as memory,
  coalesce(td.command,'')   as command,
  coalesce(td.task_type,'') as tasktype,
  env::TEXT                 as env,
  ports                     as ports,
  tags                      as tags,
  td.privileged             as privileged,
  td.cpu                    as cpu,
  td.gpu                    as gpu
  from (select * from task_def) td left outer join
    (select task_def_id,
      array_to_json(array_agg(port))::TEXT as ports
        from task_def_ports group by task_def_id
    ) tdp
  on td.definition_id = tdp.task_def_id left outer join
    (select task_def_id,
      array_to_json(array_agg(tag_id))::TEXT as tags
        from task_def_tags group by task_def_id
    ) tdt
  on td.definition_id = tdt.task_def_id
`

//
// ListDefinitionsSQL postgres specific query for listing definitions
//
const ListDefinitionsSQL = DefinitionSelect + "\n%s %s limit $1 offset $2"

//
// GetDefinitionSQL postgres specific query for getting a single definition
//
const GetDefinitionSQL = DefinitionSelect + "\nwhere definition_id = $1"

//
// GetDefinitionByAliasSQL get definition by alias
//
const GetDefinitionByAliasSQL = DefinitionSelect + "\nwhere alias = $1"

const TaskResourcesSelectCommandSQL = `
SELECT cast((percentile_disc(0.99) within GROUP (ORDER BY A.max_memory_used)) * 1.75 as int) as memory,
       cast((percentile_disc(0.99) within GROUP (ORDER BY A.max_cpu_used)) * 1.25  as int)  as cpu
FROM (SELECT max_memory_used, max_cpu_used
      FROM TASK
      WHERE
           queued_at >= CURRENT_TIMESTAMP - INTERVAL '7 days'
           AND exit_code = 0
           AND max_memory_used is not null
           AND max_cpu_used is not null
           AND engine = 'eks'
           AND definition_id = $1
           AND command_hash = (SELECT command_hash FROM task WHERE run_id = $2)
      LIMIT 30) A
`
const TaskExecutionRuntimeCommandSQL = `
SELECT percentile_disc(0.95) within GROUP (ORDER BY A.minutes) as minutes
FROM (SELECT EXTRACT(epoch from finished_at - started_at) / 60 as minutes
      FROM TASK
      WHERE definition_id = $1
        AND exit_code = 0
        AND engine = 'eks'
        AND queued_at >= CURRENT_TIMESTAMP - INTERVAL '7 days'
        AND command_hash = (SELECT command_hash FROM task WHERE run_id = $2)
      LIMIT 30) A
`

const ListFailingNodesSQL = `
SELECT instance_dns_name
      FROM TASK
      WHERE (exit_code = 128 OR
            pod_events @> '[{"reason": "Failed"}]' OR
            pod_events @> '[{"reason": "FailedSync"}]' OR
            pod_events @> '[{"reason": "FailedCreatePodSandBox"}]' OR
            (exit_code = 1 AND exit_reason is null))
           AND engine = 'eks'
           AND queued_at >= NOW() - INTERVAL '12 HOURS'
      GROUP BY 1
`

const PodReAttemptRate = `
SELECT (multiple_attempts / (CASE WHEN single_attempts = 0 THEN 1 ELSE single_attempts END)) AS attempts
FROM (
      SELECT COUNT(CASE WHEN attempt_count = 1 THEN 1 END) * 1.0 AS single_attempts,
             COUNT(CASE WHEN attempt_count != 1 THEN 1 END) * 1.0 AS multiple_attempts
      FROM task
      WHERE engine = 'eks' AND
            queued_at >= NOW() - INTERVAL '30 MINUTES' AND
            node_lifecycle = 'spot') A
`

//
// RunSelect postgres specific query for runs
//
const RunSelect = `
select
  coalesce(t.task_arn,'')                    as taskarn,
  t.run_id                                   as runid,
  coalesce(t.definition_id,'')               as definitionid,
  coalesce(t.alias,'')                       as alias,
  coalesce(t.image,'')                       as image,
  coalesce(t.cluster_name,'')                as clustername,
  t.exit_code                                as exitcode,
  t.exit_reason                              as exitreason,
  coalesce(t.status,'')                      as status,
  queued_at                                  as queuedat,
  started_at                                 as startedat,
  finished_at                                as finishedat,
  coalesce(t.instance_id,'')                 as instanceid,
  coalesce(t.instance_dns_name,'')           as instancednsname,
  coalesce(t.group_name,'')                  as groupname,
  coalesce(t.user,'')                        as "user",
  coalesce(t.task_type,'')                   as tasktype,
  env::TEXT                                  as env,
  command,
  memory,
  cpu,
  gpu,
  engine,
  ephemeral_storage as ephemeralstorage,
  node_lifecycle as nodelifecycle,
  container_name as containername,
  pod_name as podname,
  namespace,
  max_cpu_used as maxcpuused,
  max_memory_used as maxmemoryused,
  pod_events::TEXT as podevents,
  command_hash as commandhash,
  cloudtrail_notifications::TEXT as cloudtrailnotifications,
  coalesce(executable_id,'') as executableid,
  coalesce(executable_type,'') as executabletype,
  execution_request_custom::TEXT as executionrequestcustom,
  cpu_limit as cpulimit,
  memory_limit as memorylimit,
  attempt_count as attemptcount,
  spawned_runs::TEXT as spawnedruns,
  run_exceptions::TEXT as runexceptions
from task t
`

//
// ListRunsSQL postgres specific query for listing runs
//
const ListRunsSQL = RunSelect + "\n%s %s limit $1 offset $2"

//
// GetRunSQL postgres specific query for getting a single run
//
const GetRunSQL = RunSelect + "\nwhere run_id = $1"

//
// GetRunSQLForUpdate postgres specific query for getting a single run
// for update
//
const GetRunSQLForUpdate = GetRunSQL + " for update"

//
// GroupsSelect postgres specific query for getting existing definition
// group_names
//
const GroupsSelect = `
select distinct group_name from task_def
`

//
// TagsSelect postgres specific query for getting existing definition tags
//
const TagsSelect = `
select distinct text from tags
`

//
// ListGroupsSQL postgres specific query for listing definition group_names
//
const ListGroupsSQL = GroupsSelect + "\n%s order by group_name asc limit $1 offset $2"

//
// ListTagsSQL postgres specific query for listing definition tags
//
const ListTagsSQL = TagsSelect + "\n%s order by text asc limit $1 offset $2"

//
// WorkerSelect postgres specific query for workers
//
const WorkerSelect = `
  select
    worker_type        as workertype,
    count_per_instance as countperinstance,
    engine
  from worker
`

//
// ListWorkersSQL postgres specific query for listing workers
//
const ListWorkersSQL = WorkerSelect

const GetWorkerEngine = WorkerSelect + "\nwhere engine = $1"

//
// GetWorkerSQL postgres specific query for retrieving data for a specific
// worker type.
//
const GetWorkerSQL = WorkerSelect + "\nwhere worker_type = $1 and engine = $2"

//
// GetWorkerSQLForUpdate postgres specific query for retrieving data for a specific
// worker type; locks the row.
//
const GetWorkerSQLForUpdate = GetWorkerSQL + " for update"

// TemplateSelect selects a template
const TemplateSelect = `
SELECT
  template_id as templateid,
  template_name as templatename,
  version,
  schema,
  command_template as commandtemplate,
  adaptive_resource_allocation as adaptiveresourceallocation,
  image,
  container_name as containername,
  memory,
  env::TEXT as env,
  privileged,
  cpu,
  gpu,
  defaults,
  coalesce(avatar_uri, '') as avataruri
FROM template
`

// ListTemplatesSQL postgres specific query for listing templates
const ListTemplatesSQL = TemplateSelect + "\n%s limit $1 offset $2"

// GetTemplateByIDSQL postgres specific query for getting a single template
const GetTemplateByIDSQL = TemplateSelect + "\nwhere template_id = $1"

// ListTemplatesLatestOnlySQL lists the latest version of each distinct
// template name.
const ListTemplatesLatestOnlySQL = `
  SELECT DISTINCT ON (template_name)
    template_id as templateid,
    template_name as templatename,
    version,
    schema,
    command_template as commandtemplate,
    adaptive_resource_allocation as adaptiveresourceallocation,
    image,
    container_name as containername,
    memory,
    env::TEXT as env,
    privileged,
    cpu,
    gpu,
    defaults,
    coalesce(avatar_uri, '') as avataruri
  FROM template
  ORDER BY template_name, version DESC, template_id
  LIMIT $1 OFFSET $2
`

// GetTemplateLatestOnlySQL get the latest version of a specific template name.
const GetTemplateLatestOnlySQL = TemplateSelect + "\nWHERE template_name = $1 ORDER BY version DESC LIMIT 1;"
const GetTemplateByVersionSQL = TemplateSelect + "\nWHERE template_name = $1 AND version = $2 ORDER BY version DESC LIMIT 1;"
<commit_msg>updating query for ARA to increase baseline memory for OOM (#362)<commit_after>package state

//
// DefinitionSelect postgres specific query for definitions
//
const DefinitionSelect = `
select
  coalesce(td.arn,'')       as arn,
  td.definition_id          as definitionid,
  td.adaptive_resource_allocation as adaptiveresourceallocation,
  td.image                  as image,
  td.group_name             as groupname,
  td.container_name         as containername,
  coalesce(td.user,'')      as "user",
  td.alias                  as alias,
  td.memory                 as memory,
  coalesce(td.command,'')   as command,
  coalesce(td.task_type,'') as tasktype,
  env::TEXT                 as env,
  ports                     as ports,
  tags                      as tags,
  td.privileged             as privileged,
  td.cpu                    as cpu,
  td.gpu                    as gpu
  from (select * from task_def) td left outer join
    (select task_def_id,
      array_to_json(array_agg(port))::TEXT as ports
        from task_def_ports group by task_def_id
    ) tdp
  on td.definition_id = tdp.task_def_id left outer join
    (select task_def_id,
      array_to_json(array_agg(tag_id))::TEXT as tags
        from task_def_tags group by task_def_id
    ) tdt
  on td.definition_id = tdt.task_def_id
`

//
// ListDefinitionsSQL postgres specific query for listing definitions
//
const ListDefinitionsSQL = DefinitionSelect + "\n%s %s limit $1 offset $2"

//
// GetDefinitionSQL postgres specific query for getting a single definition
//
const GetDefinitionSQL = DefinitionSelect + "\nwhere definition_id = $1"

//
// GetDefinitionByAliasSQL get definition by alias
//
const GetDefinitionByAliasSQL = DefinitionSelect + "\nwhere alias = $1"

const TaskResourcesSelectCommandSQL = `
SELECT cast((percentile_disc(0.99) within GROUP (ORDER BY A.max_memory_used)) * 1.75 as int) as memory,
       cast((percentile_disc(0.99) within GROUP (ORDER BY A.max_cpu_used)) * 1.25  as int)  as cpu
FROM (SELECT CASE WHEN exit_code = 137 THEN memory * 2 ELSE max_memory_used END as max_memory_used, max_cpu_used
      FROM TASK
      WHERE
           queued_at >= CURRENT_TIMESTAMP - INTERVAL '7 days'
           AND (exit_code = 0 or exit_code = 137)
           AND max_memory_used is not null
           AND max_cpu_used is not null
           AND engine = 'eks'
           AND definition_id = $1
           AND command_hash = (SELECT command_hash FROM task WHERE run_id = $2)
      LIMIT 30) A

`
const TaskExecutionRuntimeCommandSQL = `
SELECT percentile_disc(0.95) within GROUP (ORDER BY A.minutes) as minutes
FROM (SELECT EXTRACT(epoch from finished_at - started_at) / 60 as minutes
      FROM TASK
      WHERE definition_id = $1
        AND exit_code = 0
        AND engine = 'eks'
        AND queued_at >= CURRENT_TIMESTAMP - INTERVAL '7 days'
        AND command_hash = (SELECT command_hash FROM task WHERE run_id = $2)
      LIMIT 30) A
`

const ListFailingNodesSQL = `
SELECT instance_dns_name
      FROM TASK
      WHERE (exit_code = 128 OR
            pod_events @> '[{"reason": "Failed"}]' OR
            pod_events @> '[{"reason": "FailedSync"}]' OR
            pod_events @> '[{"reason": "FailedCreatePodSandBox"}]' OR
            (exit_code = 1 AND exit_reason is null))
           AND engine = 'eks'
           AND queued_at >= NOW() - INTERVAL '12 HOURS'
      GROUP BY 1
`

const PodReAttemptRate = `
SELECT (multiple_attempts / (CASE WHEN single_attempts = 0 THEN 1 ELSE single_attempts END)) AS attempts
FROM (
      SELECT COUNT(CASE WHEN attempt_count = 1 THEN 1 END) * 1.0 AS single_attempts,
             COUNT(CASE WHEN attempt_count != 1 THEN 1 END) * 1.0 AS multiple_attempts
      FROM task
      WHERE engine = 'eks' AND
            queued_at >= NOW() - INTERVAL '30 MINUTES' AND
            node_lifecycle = 'spot') A
`

//
// RunSelect postgres specific query for runs
//
const RunSelect = `
select
  coalesce(t.task_arn,'')                    as taskarn,
  t.run_id                                   as runid,
  coalesce(t.definition_id,'')               as definitionid,
  coalesce(t.alias,'')                       as alias,
  coalesce(t.image,'')                       as image,
  coalesce(t.cluster_name,'')                as clustername,
  t.exit_code                                as exitcode,
  t.exit_reason                              as exitreason,
  coalesce(t.status,'')                      as status,
  queued_at                                  as queuedat,
  started_at                                 as startedat,
  finished_at                                as finishedat,
  coalesce(t.instance_id,'')                 as instanceid,
  coalesce(t.instance_dns_name,'')           as instancednsname,
  coalesce(t.group_name,'')                  as groupname,
  coalesce(t.user,'')                        as "user",
  coalesce(t.task_type,'')                   as tasktype,
  env::TEXT                                  as env,
  command,
  memory,
  cpu,
  gpu,
  engine,
  ephemeral_storage as ephemeralstorage,
  node_lifecycle as nodelifecycle,
  container_name as containername,
  pod_name as podname,
  namespace,
  max_cpu_used as maxcpuused,
  max_memory_used as maxmemoryused,
  pod_events::TEXT as podevents,
  command_hash as commandhash,
  cloudtrail_notifications::TEXT as cloudtrailnotifications,
  coalesce(executable_id,'') as executableid,
  coalesce(executable_type,'') as executabletype,
  execution_request_custom::TEXT as executionrequestcustom,
  cpu_limit as cpulimit,
  memory_limit as memorylimit,
  attempt_count as attemptcount,
  spawned_runs::TEXT as spawnedruns,
  run_exceptions::TEXT as runexceptions
from task t
`

//
// ListRunsSQL postgres specific query for listing runs
//
const ListRunsSQL = RunSelect + "\n%s %s limit $1 offset $2"

//
// GetRunSQL postgres specific query for getting a single run
//
const GetRunSQL = RunSelect + "\nwhere run_id = $1"

//
// GetRunSQLForUpdate postgres specific query for getting a single run
// for update
//
const GetRunSQLForUpdate = GetRunSQL + " for update"

//
// GroupsSelect postgres specific query for getting existing definition
// group_names
//
const GroupsSelect = `
select distinct group_name from task_def
`

//
// TagsSelect postgres specific query for getting existing definition tags
//
const TagsSelect = `
select distinct text from tags
`

//
// ListGroupsSQL postgres specific query for listing definition group_names
//
const ListGroupsSQL = GroupsSelect + "\n%s order by group_name asc limit $1 offset $2"

//
// ListTagsSQL postgres specific query for listing definition tags
//
const ListTagsSQL = TagsSelect + "\n%s order by text asc limit $1 offset $2"

//
// WorkerSelect postgres specific query for workers
//
const WorkerSelect = `
  select
    worker_type        as workertype,
    count_per_instance as countperinstance,
    engine
  from worker
`

//
// ListWorkersSQL postgres specific query for listing workers
//
const ListWorkersSQL = WorkerSelect

const GetWorkerEngine = WorkerSelect + "\nwhere engine = $1"

//
// GetWorkerSQL postgres specific query for retrieving data for a specific
// worker type.
//
const GetWorkerSQL = WorkerSelect + "\nwhere worker_type = $1 and engine = $2"

//
// GetWorkerSQLForUpdate postgres specific query for retrieving data for a specific
// worker type; locks the row.
//
const GetWorkerSQLForUpdate = GetWorkerSQL + " for update"

// TemplateSelect selects a template
const TemplateSelect = `
SELECT
  template_id as templateid,
  template_name as templatename,
  version,
  schema,
  command_template as commandtemplate,
  adaptive_resource_allocation as adaptiveresourceallocation,
  image,
  container_name as containername,
  memory,
  env::TEXT as env,
  privileged,
  cpu,
  gpu,
  defaults,
  coalesce(avatar_uri, '') as avataruri
FROM template
`

// ListTemplatesSQL postgres specific query for listing templates
const ListTemplatesSQL = TemplateSelect + "\n%s limit $1 offset $2"

// GetTemplateByIDSQL postgres specific query for getting a single template
const GetTemplateByIDSQL = TemplateSelect + "\nwhere template_id = $1"

// ListTemplatesLatestOnlySQL lists the latest version of each distinct
// template name.
const ListTemplatesLatestOnlySQL = `
  SELECT DISTINCT ON (template_name)
    template_id as templateid,
    template_name as templatename,
    version,
    schema,
    command_template as commandtemplate,
    adaptive_resource_allocation as adaptiveresourceallocation,
    image,
    container_name as containername,
    memory,
    env::TEXT as env,
    privileged,
    cpu,
    gpu,
    defaults,
    coalesce(avatar_uri, '') as avataruri
  FROM template
  ORDER BY template_name, version DESC, template_id
  LIMIT $1 OFFSET $2
`

// GetTemplateLatestOnlySQL get the latest version of a specific template name.
const GetTemplateLatestOnlySQL = TemplateSelect + "\nWHERE template_name = $1 ORDER BY version DESC LIMIT 1;"
const GetTemplateByVersionSQL = TemplateSelect + "\nWHERE template_name = $1 AND version = $2 ORDER BY version DESC LIMIT 1;"
<|endoftext|>


<commit_before>package erpel

import (
	"bytes"
	"io/ioutil"
	"os"
	"strings"
	"testing"
)

func tempfile(t *testing.T) string {
	f, err := ioutil.TempFile("", "erpel-test-")
	if err != nil {
		t.Fatalf("TempFile(): %v", err)
	}

	name := f.Name()

	if err = f.Close(); err != nil {
		t.Fatalf("Close(): %v", err)
	}

	return name
}

func log(t *testing.T, filename string, data string) Marker {
	f, err := os.OpenFile(filename, os.O_WRONLY|os.O_APPEND|os.O_CREATE, 0644)
	if err != nil {
		t.Fatalf("open(): %v", err)
	}

	if _, err = f.Write([]byte(data)); err != nil {
		t.Errorf("write(): %v", err)
	}

	m, err := Position(f)
	if err != nil {
		t.Errorf("Position(): %v", err)
	}

	if err = f.Close(); err != nil {
		t.Fatalf("close(): %v", err)
	}

	return m
}

func rm(t *testing.T, filename string) {
	if err := os.Remove(filename); err != nil {
		t.Errorf("removing %q failed: %v", filename, err)
	}
}

func writeMessages(t *testing.T, filename string, m []string) []Marker {
	var markers []Marker
	for _, msg := range messages {
		m := log(t, filename, msg)
		markers = append(markers, m)
	}

	return markers
}

func readRemainingData(t *testing.T, filename string, m Marker) []byte {
	fd, err := os.Open(filename)
	if err != nil {
		t.Fatalf("open(%v): %v", filename, err)
	}

	if err = m.Seek(fd); err != nil {
		t.Fatalf("Marker.Seek(): %v", err)
	}

	buf, err := ioutil.ReadAll(fd)
	if err != nil {
		t.Fatalf("read: %v", err)
	}

	if err = fd.Close(); err != nil {
		t.Fatalf("close(%v): %v", filename, err)
	}

	return buf
}

var messages = []string{
	"foobar baz message\n",
	"Jun 30 21:57:10 mopped sudo[19517]: pam_unix(sudo:session): session opened for user root by fd0(uid=0)\n",
	"foobar message2\n",
	"Jun 30 21:57:15 mopped sudo[19517]: pam_unix(sudo:session): session closed for user root\n",
}

func TestMarker(t *testing.T) {
	f := tempfile(t)
	t.Logf("using tempfile %v", f)

	markers := writeMessages(t, f, messages)
	for i, m := range markers {
		var data []byte

		for j := i + 1; j < len(messages); j++ {
			data = append(data, []byte(messages[j])...)
		}

		buf := readRemainingData(t, f, m)

		if !bytes.Equal(buf, data) {
			t.Errorf("marker %d returned wrong data, want:\n  %q\ngot:\n  %q", i, data, buf)
		}
	}

	rm(t, f)
}

func writeFile(t *testing.T, filename string, data []byte) {
	fd, err := os.Create(filename)
	if err != nil {
		t.Fatalf("create() %v", err)
	}

	_, err = fd.Write(data)
	if err != nil {
		t.Fatalf("write() %v", err)
	}

	if err = fd.Close(); err != nil {
		t.Fatalf("Close(): %v", err)
	}
}

func TestMarkerNewFile(t *testing.T) {
	f := tempfile(t)
	t.Logf("using tempfile %v", f)

	markers := writeMessages(t, f, messages)
	data := []byte(strings.Join(messages, ""))

	rm(t, f)
	writeFile(t, f, data)

	fi, err := os.Stat(f)
	if err != nil {
		t.Fatalf("stat(): %v", err)
	}
	t.Logf("stat(%v): %#v", f, fi)

	for i, m := range markers {
		buf := readRemainingData(t, f, m)

		t.Logf("marker %d: %v", i, m)

		if !bytes.Equal(buf, data) {
			t.Errorf("marker %d returned wrong data, want:\n  %q\ngot:\n  %q", i, data, buf)
		}
	}

}
<commit_msg>Make TestMarkerNewFile more realistic<commit_after>package erpel

import (
	"bytes"
	"io/ioutil"
	"os"
	"strings"
	"testing"
)

func tempfile(t *testing.T) string {
	f, err := ioutil.TempFile("", "erpel-test-")
	if err != nil {
		t.Fatalf("TempFile(): %v", err)
	}

	name := f.Name()

	if err = f.Close(); err != nil {
		t.Fatalf("Close(): %v", err)
	}

	return name
}

func log(t *testing.T, filename string, data string) Marker {
	f, err := os.OpenFile(filename, os.O_WRONLY|os.O_APPEND|os.O_CREATE, 0644)
	if err != nil {
		t.Fatalf("open(): %v", err)
	}

	if _, err = f.Write([]byte(data)); err != nil {
		t.Errorf("write(): %v", err)
	}

	m, err := Position(f)
	if err != nil {
		t.Errorf("Position(): %v", err)
	}

	if err = f.Close(); err != nil {
		t.Fatalf("close(): %v", err)
	}

	return m
}

func rm(t *testing.T, filename string) {
	if err := os.Remove(filename); err != nil {
		t.Errorf("removing %q failed: %v", filename, err)
	}
}

func writeMessages(t *testing.T, filename string, m []string) []Marker {
	var markers []Marker
	for _, msg := range messages {
		m := log(t, filename, msg)
		markers = append(markers, m)
	}

	return markers
}

func readRemainingData(t *testing.T, filename string, m Marker) []byte {
	fd, err := os.Open(filename)
	if err != nil {
		t.Fatalf("open(%v): %v", filename, err)
	}

	if err = m.Seek(fd); err != nil {
		t.Fatalf("Marker.Seek(): %v", err)
	}

	buf, err := ioutil.ReadAll(fd)
	if err != nil {
		t.Fatalf("read: %v", err)
	}

	if err = fd.Close(); err != nil {
		t.Fatalf("close(%v): %v", filename, err)
	}

	return buf
}

var messages = []string{
	"foobar baz message\n",
	"Jun 30 21:57:10 mopped sudo[19517]: pam_unix(sudo:session): session opened for user root by fd0(uid=0)\n",
	"foobar message2\n",
	"Jun 30 21:57:15 mopped sudo[19517]: pam_unix(sudo:session): session closed for user root\n",
}

func TestMarker(t *testing.T) {
	f := tempfile(t)
	t.Logf("using tempfile %v", f)

	markers := writeMessages(t, f, messages)
	for i, m := range markers {
		var data []byte

		for j := i + 1; j < len(messages); j++ {
			data = append(data, []byte(messages[j])...)
		}

		buf := readRemainingData(t, f, m)

		if !bytes.Equal(buf, data) {
			t.Errorf("marker %d returned wrong data, want:\n  %q\ngot:\n  %q", i, data, buf)
		}
	}

	rm(t, f)
}

func writeFile(t *testing.T, filename string, data []byte) {
	fd, err := os.Create(filename)
	if err != nil {
		t.Fatalf("create() %v", err)
	}

	_, err = fd.Write(data)
	if err != nil {
		t.Fatalf("write() %v", err)
	}

	if err = fd.Close(); err != nil {
		t.Fatalf("Close(): %v", err)
	}
}

func mv(t *testing.T, from, to string) {
	if err := os.Rename(from, to); err != nil {
		t.Errorf("move %v -> %v failed: %v", from, to, err)
	}
}

func TestMarkerNewFile(t *testing.T) {
	f := tempfile(t)
	t.Logf("using tempfile %v", f)

	markers := writeMessages(t, f, messages)
	data := []byte(strings.Join(messages, ""))

	mv(t, f, f+".1")
	writeFile(t, f, data)

	fi, err := os.Stat(f)
	if err != nil {
		t.Fatalf("stat(): %v", err)
	}
	t.Logf("stat(%v): %#v", f, fi)

	for i, m := range markers {
		buf := readRemainingData(t, f, m)

		t.Logf("marker %d: %v", i, m)

		if !bytes.Equal(buf, data) {
			t.Errorf("marker %d returned wrong data, want:\n  %q\ngot:\n  %q", i, data, buf)
		}
	}

}
<|endoftext|>


<commit_before>// +build linux freebsd darwin

package common

import (
	"os/exec"
	"strconv"
	"strings"
)

func CallLsof(invoke Invoker, pid int32, args ...string) ([]string, error) {
	var cmd []string
	if pid == 0 { // will get from all processes.
		cmd = []string{"-a", "-n", "-P"}
	} else {
		cmd = []string{"-a", "-n", "-P", "-p", strconv.Itoa(int(pid))}
	}
	cmd = append(cmd, args...)
	lsof, err := exec.LookPath("lsof")
	if err != nil {
		return []string{}, err
	}
	out, err := invoke.Command(lsof, cmd...)
	if err != nil {
		// if no pid found, lsof returnes code 1
		if err.Error() == "exit status 1" && len(out) == 0 {
			return []string{}, nil
		}
		return []string{}, err
	}
	lines := strings.Split(string(out), "\n")

	var ret []string
	for _, l := range lines[1:] {
		if len(l) == 0 {
			continue
		}
		ret = append(ret, l)
	}
	return ret, nil
}
<commit_msg>net[linux]: fix lsof output in linux when no pid outputed.<commit_after>// +build linux freebsd darwin

package common

import (
	"os/exec"
	"strconv"
	"strings"
)

func CallLsof(invoke Invoker, pid int32, args ...string) ([]string, error) {
	var cmd []string
	if pid == 0 { // will get from all processes.
		cmd = []string{"-a", "-n", "-P"}
	} else {
		cmd = []string{"-a", "-n", "-P", "-p", strconv.Itoa(int(pid))}
	}
	cmd = append(cmd, args...)
	lsof, err := exec.LookPath("lsof")
	if err != nil {
		return []string{}, err
	}
	out, err := invoke.Command(lsof, cmd...)
	if err != nil {
		// if no pid found, lsof returnes code 1 but have output.
		if err.Error() == "exit status 1" && len(out) == 0 {
			return []string{}, err
		}
	}
	lines := strings.Split(string(out), "\n")

	var ret []string
	for _, l := range lines[1:] {
		if len(l) == 0 {
			continue
		}
		ret = append(ret, l)
	}
	return ret, nil
}
<|endoftext|>


<commit_before>// Package ini provides functions for parsing INI configuration files.
package ini

import (
	"bufio"
	"fmt"
	"io"
	"os"
	"regexp"
	"strings"
)

var (
	sectionRegex = regexp.MustCompile(`^\[(.*)\]$`)
	assignRegex  = regexp.MustCompile(`([^=]+)=(.*)$`)
)

// ErrSyntax is returned when there is a syntax error in an INI file.
type ErrSyntax struct {
	Line   int
	Source string
}

func (e ErrSyntax) Error() string {
	return fmt.Sprintf("invalid INI syntax on line %d: %s", e.Line, e.Source)
}

// A File represents a parsed INI file.
type File map[string]Section

// A Section represents a single section of an INI file.
type Section map[string]string

// Returns a named Section. A Section will be created if one does not already exist for the given name.
func (f File) Section(name string) Section {
	section := f[name]
	if section == nil {
		section = make(Section)
		f[name] = section
	}
	return section
}

// Looks up a value for a key in a section and returns that value, along with a boolean result similar to a map lookup.
func (f File) Get(section, key string) (value string, ok bool) {
	if s := f[section]; s != nil {
		value, ok = s[key]
	}
	return
}

// Loads INI data from a reader and stores the data in the File.
func (f File) Load(in io.Reader) (err error) {
	bufin, ok := in.(*bufio.Reader)
	if !ok {
		bufin = bufio.NewReader(in)
	}
	return parseFile(bufin, f)
}

// Loads INI data from a named file and stores the data in the File.
func (f File) LoadFile(file string) (err error) {
	in, err := os.Open(file)
	if err != nil {
		return
	}
	return f.Load(in)
}

func readLine(in *bufio.Reader) (line string, err error) {
	reading := true
	for reading {
		var part []byte
		if part, reading, err = in.ReadLine(); err != nil {
			return
		}
		line += string(part)
	}
	return
}

func parseFile(in *bufio.Reader, file File) (err error) {
	section := file.Section("")
	lineNum := 0
	for {
		var line string
		if line, err = readLine(in); err != nil {
			if err == io.EOF {
				break
			}
			return
		}
		lineNum++
		line = strings.TrimSpace(line)
		if len(line) == 0 {
			// Skip blank lines
			continue
		}
		if line[0] == ';' || line[0] == '#' {
			// Skip comments
			continue
		}

		if groups := assignRegex.FindStringSubmatch(line); groups != nil {
			key, val := groups[1], groups[2]
			key, val = strings.TrimSpace(key), strings.TrimSpace(val)
			section[key] = val
		} else if groups := sectionRegex.FindStringSubmatch(line); groups != nil {
			name := strings.TrimSpace(groups[1])
			section = file.Section(name)
		} else {
			return ErrSyntax{lineNum, line}
		}

	}
	return nil
}

// Loads and returns a File from a reader.
func Load(in io.Reader) (File, error) {
	file := make(File)
	err := file.Load(in)
	return file, err
}

// Loads and returns an ini File from a file on disk.
func LoadFile(filename string) (File, error) {
	file := make(File)
	err := file.LoadFile(filename)
	return file, err
}
<commit_msg>Use ReadString('\n') instead of ReadLine()<commit_after>// Package ini provides functions for parsing INI configuration files.
package ini

import (
	"bufio"
	"fmt"
	"io"
	"os"
	"regexp"
	"strings"
)

var (
	sectionRegex = regexp.MustCompile(`^\[(.*)\]$`)
	assignRegex  = regexp.MustCompile(`([^=]+)=(.*)$`)
)

// ErrSyntax is returned when there is a syntax error in an INI file.
type ErrSyntax struct {
	Line   int
	Source string
}

func (e ErrSyntax) Error() string {
	return fmt.Sprintf("invalid INI syntax on line %d: %s", e.Line, e.Source)
}

// A File represents a parsed INI file.
type File map[string]Section

// A Section represents a single section of an INI file.
type Section map[string]string

// Returns a named Section. A Section will be created if one does not already exist for the given name.
func (f File) Section(name string) Section {
	section := f[name]
	if section == nil {
		section = make(Section)
		f[name] = section
	}
	return section
}

// Looks up a value for a key in a section and returns that value, along with a boolean result similar to a map lookup.
func (f File) Get(section, key string) (value string, ok bool) {
	if s := f[section]; s != nil {
		value, ok = s[key]
	}
	return
}

// Loads INI data from a reader and stores the data in the File.
func (f File) Load(in io.Reader) (err error) {
	bufin, ok := in.(*bufio.Reader)
	if !ok {
		bufin = bufio.NewReader(in)
	}
	return parseFile(bufin, f)
}

// Loads INI data from a named file and stores the data in the File.
func (f File) LoadFile(file string) (err error) {
	in, err := os.Open(file)
	if err != nil {
		return
	}
	return f.Load(in)
}

func parseFile(in *bufio.Reader, file File) (err error) {
	section := file.Section("")
	lineNum := 0
	for {
		var line string
		if line, err = in.ReadString('\n'); err != nil {
			if err == io.EOF {
				break
			}
			return
		}
		lineNum++
		line = strings.TrimSpace(line)
		if len(line) == 0 {
			// Skip blank lines
			continue
		}
		if line[0] == ';' || line[0] == '#' {
			// Skip comments
			continue
		}

		if groups := assignRegex.FindStringSubmatch(line); groups != nil {
			key, val := groups[1], groups[2]
			key, val = strings.TrimSpace(key), strings.TrimSpace(val)
			section[key] = val
		} else if groups := sectionRegex.FindStringSubmatch(line); groups != nil {
			name := strings.TrimSpace(groups[1])
			section = file.Section(name)
		} else {
			return ErrSyntax{lineNum, line}
		}

	}
	return nil
}

// Loads and returns a File from a reader.
func Load(in io.Reader) (File, error) {
	file := make(File)
	err := file.Load(in)
	return file, err
}

// Loads and returns an ini File from a file on disk.
func LoadFile(filename string) (File, error) {
	file := make(File)
	err := file.LoadFile(filename)
	return file, err
}
<|endoftext|>