<commit_before>// Runtime provides runtime instrumentations
package runtime

import (
	"runtime"
	"sync"
	"sync/atomic"

	"github.com/heroku/instruments"
)

type Allocated struct {
	g   *instruments.Gauge
	mem runtime.MemStats
	m   sync.Mutex
}

func NewAllocated() *Allocated {
	return &Allocated{
		g: instruments.NewGauge(0),
	}
}

func (a *Allocated) Update() {
	a.m.Lock()
	defer a.m.Unlock()

	runtime.ReadMemStats(&a.mem)
	a.g.Update(int64(a.mem.Alloc))
}

func (a *Allocated) Snapshot() int64 {
	return a.Snapshot()
}

type HeapAllocated struct {
	g   *instruments.Gauge
	mem runtime.MemStats
	m   sync.Mutex
}

func NewHeapAllocated() *HeapAllocated {
	return &HeapAllocated{
		g: instruments.NewGauge(0),
	}
}

func (ha *HeapAllocated) Update() {
	ha.m.Lock()
	defer ha.m.Unlock()

	runtime.ReadMemStats(&ha.mem)
	ha.g.Update(int64(ha.mem.HeapAlloc))
}

func (ha *HeapAllocated) Snapshot() int64 {
	return ha.g.Snapshot()
}

type StackInUse struct {
	g   *instruments.Gauge
	mem runtime.MemStats
	m   sync.Mutex
}

func (su *StackInUse) Update() {
	su.m.Lock()
	defer su.m.Unlock()

	runtime.ReadMemStats(&su.mem)
	su.g.Update(int64(su.mem.StackInuse))
}

func (su *StackInUse) Snapshot() int64 {
	return su.g.Snapshot()
}

type Goroutine struct {
	g *instruments.Gauge
}

func NewGoroutine() *Goroutine {
	return &Goroutine{
		g: instruments.NewGauge(0),
	}
}

func (gr *Goroutine) Update() {
	gr.g.Update(int64(runtime.NumGoroutine()))
}

func (gr *Goroutine) Snapshot() int64 {
	return gr.Snapshot()
}

type Cgo struct {
	g *instruments.Gauge
}

func NewCgo() *Cgo {
	return &Cgo{
		g: instruments.NewGauge(0),
	}
}

func (c *Cgo) Update() {
	c.g.Update(runtime.NumCgoCall())
}

func (c *Cgo) Snapshot() int64 {
	return c.Snapshot()
}

type Frees struct {
	d   *instruments.Derive
	mem runtime.MemStats
	m   sync.Mutex
}

func NewFrees() *Frees {
	return &Frees{
		d: instruments.NewDerive(0),
	}
}

func (f *Frees) Update() {
	f.m.Lock()
	defer f.m.Unlock()

	runtime.ReadMemStats(&f.mem)
	f.d.Update(int64(f.mem.Frees))
}

func (f *Frees) Snapshot() int64 {
	return f.Snapshot()
}

type Lookups struct {
	d   *instruments.Derive
	mem runtime.MemStats
	m   sync.Mutex
}

func NewLookups() *Lookups {
	return &Lookups{
		d: instruments.NewDerive(0),
	}
}

func (l *Lookups) Update() {
	l.m.Lock()
	defer l.m.Unlock()

	runtime.ReadMemStats(&l.mem)
	l.d.Update(int64(l.mem.Lookups))
}

func (l *Lookups) Snapshot() int64 {
	return l.Snapshot()
}

type Mallocs struct {
	d   *instruments.Derive
	mem runtime.MemStats
	m   sync.Mutex
}

func NewMallocs() *Mallocs {
	return &Mallocs{
		d: instruments.NewDerive(0),
	}
}

func (m *Mallocs) Update() {
	m.m.Lock()
	defer m.m.Unlock()

	runtime.ReadMemStats(&m.mem)
	m.d.Update(int64(m.mem.Mallocs))
}

func (m *Mallocs) Snapshot() int64 {
	return m.Snapshot()
}

type NumGC struct {
	d   *instruments.Derive
	mem runtime.MemStats
	m   sync.Mutex
}

func NewNumGC() *NumGC {
	return &NumGC{
		d: instruments.NewDerive(0),
	}
}

func (ng *NumGC) Update() {
	ng.m.Lock()
	defer ng.m.Unlock()

	runtime.ReadMemStats(&ng.mem)
	ng.d.Update(int64(ng.mem.NumGC))
}

func (ng *NumGC) Snapshot() int64 {
	return ng.Snapshot()
}

type Pauses struct {
	r   *instruments.Reservoir
	n   uint32
	mem runtime.MemStats
	m   sync.Mutex
}

func NewPauses(size int64) *Pauses {
	return &Pauses{
		r: instruments.NewReservoir(size),
	}
}

func (p *Pauses) Update() {
	p.m.Lock()
	defer p.m.Unlock()

	runtime.ReadMemStats(&p.mem)
	numGC := atomic.SwapUint32(&p.n, p.mem.NumGC)
	i := numGC % uint32(len(p.mem.PauseNs))
	j := p.mem.NumGC % uint32(len(p.mem.PauseNs))
	if p.mem.NumGC-numGC >= uint32(len(p.mem.PauseNs)) {
		for i = 0; i < uint32(len(p.mem.PauseNs)); i++ {
			p.r.Update(int64(p.mem.PauseNs[i]))
		}
	} else {
		if i > j {
			for ; i < uint32(len(p.mem.PauseNs)); i++ {
				p.r.Update(int64(p.mem.PauseNs[i]))
			}
			i = 0
		}
		for ; i < j; i++ {
			p.r.Update(int64(p.mem.PauseNs[i]))
		}
	}
}

func (p *Pauses) Snapshot() []int64 {
	return p.r.Snapshot()
}

type TotalPause struct {
	g   *instruments.Gauge
	mem runtime.MemStats
	m   sync.Mutex
}

func NewTotalPause() *TotalPause {
	return &TotalPause{
		g: instruments.NewGauge(0),
	}
}

func (tp *TotalPause) Update() {
	tp.m.Lock()
	defer tp.m.Unlock()

	runtime.ReadMemStats(&tp.mem)
	tp.g.Update(int64(tp.mem.PauseTotalNs))
}

func (tp *TotalPause) Snapshot() int64 {
	return tp.Snapshot()
}
<commit_msg>add missing function<commit_after>// Runtime provides runtime instrumentations
package runtime

import (
	"runtime"
	"sync"
	"sync/atomic"

	"github.com/heroku/instruments"
)

type Allocated struct {
	g   *instruments.Gauge
	mem runtime.MemStats
	m   sync.Mutex
}

func NewAllocated() *Allocated {
	return &Allocated{
		g: instruments.NewGauge(0),
	}
}

func (a *Allocated) Update() {
	a.m.Lock()
	defer a.m.Unlock()

	runtime.ReadMemStats(&a.mem)
	a.g.Update(int64(a.mem.Alloc))
}

func (a *Allocated) Snapshot() int64 {
	return a.Snapshot()
}

type HeapAllocated struct {
	g   *instruments.Gauge
	mem runtime.MemStats
	m   sync.Mutex
}

func NewHeapAllocated() *HeapAllocated {
	return &HeapAllocated{
		g: instruments.NewGauge(0),
	}
}

func (ha *HeapAllocated) Update() {
	ha.m.Lock()
	defer ha.m.Unlock()

	runtime.ReadMemStats(&ha.mem)
	ha.g.Update(int64(ha.mem.HeapAlloc))
}

func (ha *HeapAllocated) Snapshot() int64 {
	return ha.g.Snapshot()
}

type StackInUse struct {
	g   *instruments.Gauge
	mem runtime.MemStats
	m   sync.Mutex
}

func NewStackInUse() *StackInUse {
	return &StackInUse{
		g: instruments.NewGauge(0),
	}
}

func (su *StackInUse) Update() {
	su.m.Lock()
	defer su.m.Unlock()

	runtime.ReadMemStats(&su.mem)
	su.g.Update(int64(su.mem.StackInuse))
}

func (su *StackInUse) Snapshot() int64 {
	return su.g.Snapshot()
}

type Goroutine struct {
	g *instruments.Gauge
}

func NewGoroutine() *Goroutine {
	return &Goroutine{
		g: instruments.NewGauge(0),
	}
}

func (gr *Goroutine) Update() {
	gr.g.Update(int64(runtime.NumGoroutine()))
}

func (gr *Goroutine) Snapshot() int64 {
	return gr.Snapshot()
}

type Cgo struct {
	g *instruments.Gauge
}

func NewCgo() *Cgo {
	return &Cgo{
		g: instruments.NewGauge(0),
	}
}

func (c *Cgo) Update() {
	c.g.Update(runtime.NumCgoCall())
}

func (c *Cgo) Snapshot() int64 {
	return c.Snapshot()
}

type Frees struct {
	d   *instruments.Derive
	mem runtime.MemStats
	m   sync.Mutex
}

func NewFrees() *Frees {
	return &Frees{
		d: instruments.NewDerive(0),
	}
}

func (f *Frees) Update() {
	f.m.Lock()
	defer f.m.Unlock()

	runtime.ReadMemStats(&f.mem)
	f.d.Update(int64(f.mem.Frees))
}

func (f *Frees) Snapshot() int64 {
	return f.Snapshot()
}

type Lookups struct {
	d   *instruments.Derive
	mem runtime.MemStats
	m   sync.Mutex
}

func NewLookups() *Lookups {
	return &Lookups{
		d: instruments.NewDerive(0),
	}
}

func (l *Lookups) Update() {
	l.m.Lock()
	defer l.m.Unlock()

	runtime.ReadMemStats(&l.mem)
	l.d.Update(int64(l.mem.Lookups))
}

func (l *Lookups) Snapshot() int64 {
	return l.Snapshot()
}

type Mallocs struct {
	d   *instruments.Derive
	mem runtime.MemStats
	m   sync.Mutex
}

func NewMallocs() *Mallocs {
	return &Mallocs{
		d: instruments.NewDerive(0),
	}
}

func (m *Mallocs) Update() {
	m.m.Lock()
	defer m.m.Unlock()

	runtime.ReadMemStats(&m.mem)
	m.d.Update(int64(m.mem.Mallocs))
}

func (m *Mallocs) Snapshot() int64 {
	return m.Snapshot()
}

type NumGC struct {
	d   *instruments.Derive
	mem runtime.MemStats
	m   sync.Mutex
}

func NewNumGC() *NumGC {
	return &NumGC{
		d: instruments.NewDerive(0),
	}
}

func (ng *NumGC) Update() {
	ng.m.Lock()
	defer ng.m.Unlock()

	runtime.ReadMemStats(&ng.mem)
	ng.d.Update(int64(ng.mem.NumGC))
}

func (ng *NumGC) Snapshot() int64 {
	return ng.Snapshot()
}

type Pauses struct {
	r   *instruments.Reservoir
	n   uint32
	mem runtime.MemStats
	m   sync.Mutex
}

func NewPauses(size int64) *Pauses {
	return &Pauses{
		r: instruments.NewReservoir(size),
	}
}

func (p *Pauses) Update() {
	p.m.Lock()
	defer p.m.Unlock()

	runtime.ReadMemStats(&p.mem)
	numGC := atomic.SwapUint32(&p.n, p.mem.NumGC)
	i := numGC % uint32(len(p.mem.PauseNs))
	j := p.mem.NumGC % uint32(len(p.mem.PauseNs))
	if p.mem.NumGC-numGC >= uint32(len(p.mem.PauseNs)) {
		for i = 0; i < uint32(len(p.mem.PauseNs)); i++ {
			p.r.Update(int64(p.mem.PauseNs[i]))
		}
	} else {
		if i > j {
			for ; i < uint32(len(p.mem.PauseNs)); i++ {
				p.r.Update(int64(p.mem.PauseNs[i]))
			}
			i = 0
		}
		for ; i < j; i++ {
			p.r.Update(int64(p.mem.PauseNs[i]))
		}
	}
}

func (p *Pauses) Snapshot() []int64 {
	return p.r.Snapshot()
}

type TotalPause struct {
	g   *instruments.Gauge
	mem runtime.MemStats
	m   sync.Mutex
}

func NewTotalPause() *TotalPause {
	return &TotalPause{
		g: instruments.NewGauge(0),
	}
}

func (tp *TotalPause) Update() {
	tp.m.Lock()
	defer tp.m.Unlock()

	runtime.ReadMemStats(&tp.mem)
	tp.g.Update(int64(tp.mem.PauseTotalNs))
}

func (tp *TotalPause) Snapshot() int64 {
	return tp.Snapshot()
}
<|endoftext|>


<commit_before>package runtime

// Version is the current version of the buffalo binary
const Version = "v0.14.0"
<commit_msg>reset development<commit_after>package runtime

// Version is the current version of the buffalo binary
const Version = "development"
<|endoftext|>


<commit_before>package algoliaconnector

import (
	"encoding/json"
	"errors"
	"fmt"
	"socialapi/models"
	"strconv"

	"github.com/algolia/algoliasearch-client-go/algoliasearch"
	"github.com/koding/logging"
	"github.com/streadway/amqp"
)

var (
	ErrAlgoliaObjectIdNotFoundMsg = "ObjectID does not exist"
	ErrAlgoliaIndexNotExistMsg    = "Index messages.test does not exist"
)

type IndexSet map[string]*algoliasearch.Index

type Controller struct {
	log     logging.Logger
	client  *algoliasearch.Client
	indexes *IndexSet
}

// IsAlgoliaError checks if the given algolia error string and given messages
// are same according their data structure
func IsAlgoliaError(err error, message string) bool {
	if err == nil {
		return false
	}

	v := &algoliaErrorRes{}

	if err := json.Unmarshal([]byte(err.Error()), v); err != nil {
		return false
	}

	if v.Message == message {
		return true
	}

	return false
}

type algoliaErrorRes struct {
	Message string `json:"message"`
	Status  int    `json:"status"`
}

func (i *IndexSet) Get(name string) (*algoliasearch.Index, error) {
	index, ok := (*i)[name]
	if !ok {
		return nil, fmt.Errorf("Unknown index: '%s'", name)
	}
	return index, nil
}

func (c *Controller) DefaultErrHandler(delivery amqp.Delivery, err error) bool {
	c.log.Error(err.Error())
	return false
}

func New(log logging.Logger, client *algoliasearch.Client, indexSuffix string) *Controller {
	return &Controller{
		log:    log,
		client: client,
		indexes: &IndexSet{
			"topics":   client.InitIndex("topics" + indexSuffix),
			"accounts": client.InitIndex("accounts" + indexSuffix),
			"messages": client.InitIndex("messages" + indexSuffix),
		},
	}
}

func (f *Controller) TopicSaved(data *models.Channel) error {
	if data.TypeConstant != models.Channel_TYPE_TOPIC {
		return nil
	}
	return f.insert("topics", map[string]interface{}{
		"objectID": strconv.FormatInt(data.Id, 10),
		"name":     data.Name,
		"purpose":  data.Purpose,
	})
}

// TopicUpdated handles the channel update events, for now only handles the
// channels that are topic channels, we can link channels together in any point
// of time, after linking, leaf channel should be removed from search engine
func (f *Controller) TopicUpdated(data *models.Channel) error {
	if data.TypeConstant != models.Channel_TYPE_LINKED_TOPIC {
		f.log.Debug("unsuported channel for topic update type: %s id: %d", data.TypeConstant, data.Id)
		return nil
	}

	return f.delete("topics", strconv.FormatInt(data.Id, 10))
}

func (f *Controller) AccountSaved(data *models.Account) error {
	return f.insert("accounts", map[string]interface{}{
		"objectID": data.OldId,
		"nick":     data.Nick,
	})
}

func (f *Controller) MessageListSaved(listing *models.ChannelMessageList) error {
	message := models.NewChannelMessage()

	if err := message.ById(listing.MessageId); err != nil {
		return err
	}

	objectId := strconv.FormatInt(message.Id, 10)
	channelId := strconv.FormatInt(listing.ChannelId, 10)

	record, err := f.get("messages", objectId)
	if err != nil &&
		!IsAlgoliaError(err, ErrAlgoliaObjectIdNotFoundMsg) &&
		!IsAlgoliaError(err, ErrAlgoliaIndexNotExistMsg) {
		return err
	}

	if record == nil {
		return f.insert("messages", map[string]interface{}{
			"objectID": objectId,
			"body":     message.Body,
			"_tags":    []string{channelId},
		})
	}

	return f.partialUpdate("messages", map[string]interface{}{
		"objectID": objectId,
		"_tags":    appendMessageTag(record, channelId),
	})
}

func (f *Controller) MessageListDeleted(listing *models.ChannelMessageList) error {
	index, err := f.indexes.Get("messages")
	if err != nil {
		return err
	}

	objectId := strconv.FormatInt(listing.MessageId, 10)

	record, err := f.get("messages", objectId)
	if err != nil &&
		!IsAlgoliaError(err, ErrAlgoliaObjectIdNotFoundMsg) &&
		!IsAlgoliaError(err, ErrAlgoliaIndexNotExistMsg) {
		return err
	}

	if tags, ok := record["_tags"]; ok {
		if t, ok := tags.([]interface{}); ok && len(t) == 1 {
			if _, err = index.DeleteObject(objectId); err != nil {
				return err
			}
			return nil
		}
	}

	return f.partialUpdate("messages", map[string]interface{}{
		"objectID": objectId,
		"_tags":    removeMessageTag(record, strconv.FormatInt(listing.ChannelId, 10)),
	})
}

func (f *Controller) MessageUpdated(message *models.ChannelMessage) error {
	return f.partialUpdate("messages", map[string]interface{}{
		"objectID": strconv.FormatInt(message.Id, 10),
		"body":     message.Body,
	})
}

func (f *Controller) CreateSynonym(cl *models.ChannelLink) error {
	if err := f.validateSynonymRequest(cl); err != nil {
		f.log.Error("CreateSynonym validateSynonymRequest err:", err.Error())
		return nil
	}

	// check channel types
	rootChannel, err := models.ChannelById(cl.RootId)
	if err != nil {
		return err
	}

	if !isValidChannelType(rootChannel) {
		return errors.New("root is not valid type for synonym")
	}

	leafChannel, err := models.ChannelById(cl.LeafId)
	if err != nil {
		return err
	}

	return f.addSynonym("messages", rootChannel.Name, leafChannel.Name)
}

// addSynonym adds given sysnonym pairs to the given index. do not worry about
// duplicate synonyms, algolia handles them perfectly
func (f *Controller) addSynonym(indexName string, synonyms ...string) error {
	// TODO - this get & use pattern is very prone to race conditions
	synonymsSlice, err := f.getSynonyms(indexName)

	// append it to the previous ones, if there is any
	settings := make(map[string]interface{})
	settings["synonyms"] = append(synonymsSlice, synonyms)

	index, err := f.indexes.Get(indexName)
	if err != nil {
		return err
	}

	_, err = index.SetSettings(settings)
	return err
}

func (f *Controller) getSynonyms(indexName string) ([][]string, error) {
	index, err := f.indexes.Get(indexName)
	if err != nil {
		return nil, err
	}

	settingsinter, err := index.GetSettings()
	if err != nil {
		return nil, err
	}

	settings, ok := settingsinter.(map[string]interface{})
	if !ok {
		settings = make(map[string]interface{})
	}

	// define the initial synonymns
	synonymsSlice := make([][]string, 0)

	if sint, ok := settings["synonyms"]; ok {
		if sslice, ok := sint.([][]string); ok {
			// if we have previous ones, use it
			synonymsSlice = sslice
		}
	}

	return synonymsSlice, nil
}

func (f *Controller) validateSynonymRequest(cl *models.ChannelLink) error {
	// check required variables
	if cl == nil {
		return errors.New("channel link is not set (nil)")
	}

	if cl.Id == 0 {
		return errors.New("id is not set")
	}

	if cl.RootId == 0 {
		return errors.New("root id is not set")
	}

	if cl.LeafId == 0 {
		return errors.New("leaf id is not set")
	}

	// check channel types
	rootChannel, err := models.ChannelById(cl.RootId)
	if err != nil {
		return err
	}

	if !isValidChannelType(rootChannel) {
		return errors.New("root is not valid type for synonym")
	}

	leafChannel, err := models.ChannelById(cl.LeafId)
	if err != nil {
		return err
	}

	if !isValidChannelType(leafChannel) {
		return errors.New("leaf is not valid type for synonym")
	}

	return nil
}

func isValidChannelType(c *models.Channel) bool {
	return models.IsIn(
		c.TypeConstant,
		// type constant should be one of followings
		models.Channel_TYPE_TOPIC,
		models.Channel_TYPE_LINKED_TOPIC,
	)
}
<commit_msg>Socialapi: instead of using last two channels, do fetch the old ones and inject them to algolia<commit_after>package algoliaconnector

import (
	"encoding/json"
	"errors"
	"fmt"
	"socialapi/models"
	"strconv"

	"github.com/algolia/algoliasearch-client-go/algoliasearch"
	"github.com/koding/logging"
	"github.com/streadway/amqp"
)

var (
	ErrAlgoliaObjectIdNotFoundMsg = "ObjectID does not exist"
	ErrAlgoliaIndexNotExistMsg    = "Index messages.test does not exist"
)

type IndexSet map[string]*algoliasearch.Index

type Controller struct {
	log     logging.Logger
	client  *algoliasearch.Client
	indexes *IndexSet
}

// IsAlgoliaError checks if the given algolia error string and given messages
// are same according their data structure
func IsAlgoliaError(err error, message string) bool {
	if err == nil {
		return false
	}

	v := &algoliaErrorRes{}

	if err := json.Unmarshal([]byte(err.Error()), v); err != nil {
		return false
	}

	if v.Message == message {
		return true
	}

	return false
}

type algoliaErrorRes struct {
	Message string `json:"message"`
	Status  int    `json:"status"`
}

func (i *IndexSet) Get(name string) (*algoliasearch.Index, error) {
	index, ok := (*i)[name]
	if !ok {
		return nil, fmt.Errorf("Unknown index: '%s'", name)
	}
	return index, nil
}

func (c *Controller) DefaultErrHandler(delivery amqp.Delivery, err error) bool {
	c.log.Error(err.Error())
	return false
}

func New(log logging.Logger, client *algoliasearch.Client, indexSuffix string) *Controller {
	return &Controller{
		log:    log,
		client: client,
		indexes: &IndexSet{
			"topics":   client.InitIndex("topics" + indexSuffix),
			"accounts": client.InitIndex("accounts" + indexSuffix),
			"messages": client.InitIndex("messages" + indexSuffix),
		},
	}
}

func (f *Controller) TopicSaved(data *models.Channel) error {
	if data.TypeConstant != models.Channel_TYPE_TOPIC {
		return nil
	}
	return f.insert("topics", map[string]interface{}{
		"objectID": strconv.FormatInt(data.Id, 10),
		"name":     data.Name,
		"purpose":  data.Purpose,
	})
}

// TopicUpdated handles the channel update events, for now only handles the
// channels that are topic channels, we can link channels together in any point
// of time, after linking, leaf channel should be removed from search engine
func (f *Controller) TopicUpdated(data *models.Channel) error {
	if data.TypeConstant != models.Channel_TYPE_LINKED_TOPIC {
		f.log.Debug("unsuported channel for topic update type: %s id: %d", data.TypeConstant, data.Id)
		return nil
	}

	return f.delete("topics", strconv.FormatInt(data.Id, 10))
}

func (f *Controller) AccountSaved(data *models.Account) error {
	return f.insert("accounts", map[string]interface{}{
		"objectID": data.OldId,
		"nick":     data.Nick,
	})
}

func (f *Controller) MessageListSaved(listing *models.ChannelMessageList) error {
	message := models.NewChannelMessage()

	if err := message.ById(listing.MessageId); err != nil {
		return err
	}

	objectId := strconv.FormatInt(message.Id, 10)
	channelId := strconv.FormatInt(listing.ChannelId, 10)

	record, err := f.get("messages", objectId)
	if err != nil &&
		!IsAlgoliaError(err, ErrAlgoliaObjectIdNotFoundMsg) &&
		!IsAlgoliaError(err, ErrAlgoliaIndexNotExistMsg) {
		return err
	}

	if record == nil {
		return f.insert("messages", map[string]interface{}{
			"objectID": objectId,
			"body":     message.Body,
			"_tags":    []string{channelId},
		})
	}

	return f.partialUpdate("messages", map[string]interface{}{
		"objectID": objectId,
		"_tags":    appendMessageTag(record, channelId),
	})
}

func (f *Controller) MessageListDeleted(listing *models.ChannelMessageList) error {
	index, err := f.indexes.Get("messages")
	if err != nil {
		return err
	}

	objectId := strconv.FormatInt(listing.MessageId, 10)

	record, err := f.get("messages", objectId)
	if err != nil &&
		!IsAlgoliaError(err, ErrAlgoliaObjectIdNotFoundMsg) &&
		!IsAlgoliaError(err, ErrAlgoliaIndexNotExistMsg) {
		return err
	}

	if tags, ok := record["_tags"]; ok {
		if t, ok := tags.([]interface{}); ok && len(t) == 1 {
			if _, err = index.DeleteObject(objectId); err != nil {
				return err
			}
			return nil
		}
	}

	return f.partialUpdate("messages", map[string]interface{}{
		"objectID": objectId,
		"_tags":    removeMessageTag(record, strconv.FormatInt(listing.ChannelId, 10)),
	})
}

func (f *Controller) MessageUpdated(message *models.ChannelMessage) error {
	return f.partialUpdate("messages", map[string]interface{}{
		"objectID": strconv.FormatInt(message.Id, 10),
		"body":     message.Body,
	})
}

func (f *Controller) CreateSynonym(cl *models.ChannelLink) error {
	if err := f.validateSynonymRequest(cl); err != nil {
		f.log.Error("CreateSynonym validateSynonymRequest err:", err.Error())
		return nil
	}

	// check channel types
	rootChannel, err := models.ChannelById(cl.RootId)
	if err != nil {
		return err
	}

	leafChannels, err := rootChannel.FetchLeaves()
	if err != nil {
		return err
	}

	leafNames := make([]string, len(leafChannels)+1) // +1 for root channel
	leafNames[0] = rootChannel.Name

	for i, leafChannel := range leafChannels {
		leafNames[i+1] = leafChannel.Name
	}

	return f.addSynonym("messages", leafNames...)
}

// addSynonym adds given sysnonym pairs to the given index. do not worry about
// duplicate synonyms, algolia handles them perfectly
func (f *Controller) addSynonym(indexName string, synonyms ...string) error {
	// TODO - this get & use pattern is very prone to race conditions
	synonymsSlice, err := f.getSynonyms(indexName)

	// append it to the previous ones, if there is any
	settings := make(map[string]interface{})
	settings["synonyms"] = append(synonymsSlice, synonyms)

	index, err := f.indexes.Get(indexName)
	if err != nil {
		return err
	}

	_, err = index.SetSettings(settings)
	return err
}

func (f *Controller) getSynonyms(indexName string) ([][]string, error) {
	index, err := f.indexes.Get(indexName)
	if err != nil {
		return nil, err
	}

	settingsinter, err := index.GetSettings()
	if err != nil {
		return nil, err
	}

	settings, ok := settingsinter.(map[string]interface{})
	if !ok {
		settings = make(map[string]interface{})
	}

	// define the initial synonymns
	synonymsSlice := make([][]string, 0)

	if sint, ok := settings["synonyms"]; ok {
		if sslice, ok := sint.([][]string); ok {
			// if we have previous ones, use it
			synonymsSlice = sslice
		}
	}

	return synonymsSlice, nil
}

func (f *Controller) validateSynonymRequest(cl *models.ChannelLink) error {
	// check required variables
	if cl == nil {
		return errors.New("channel link is not set (nil)")
	}

	if cl.Id == 0 {
		return errors.New("id is not set")
	}

	if cl.RootId == 0 {
		return errors.New("root id is not set")
	}

	if cl.LeafId == 0 {
		return errors.New("leaf id is not set")
	}

	// check channel types
	rootChannel, err := models.ChannelById(cl.RootId)
	if err != nil {
		return err
	}

	if !isValidChannelType(rootChannel) {
		return errors.New("root is not valid type for synonym")
	}

	leafChannel, err := models.ChannelById(cl.LeafId)
	if err != nil {
		return err
	}

	if !isValidChannelType(leafChannel) {
		return errors.New("leaf is not valid type for synonym")
	}

	return nil
}

func isValidChannelType(c *models.Channel) bool {
	return models.IsIn(
		c.TypeConstant,
		// type constant should be one of followings
		models.Channel_TYPE_TOPIC,
		models.Channel_TYPE_LINKED_TOPIC,
	)
}
<|endoftext|>


<commit_before>// Copyright 2015 Google Inc. All Rights Reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package samples

import (
	"fmt"
	"io/ioutil"
	"log"
	"strings"
	"time"

	"github.com/googlecloudplatform/gcsfuse/timeutil"
	"github.com/jacobsa/fuse"
	"golang.org/x/net/context"
)

// A struct that implements common behavior needed by tests in the samples/
// directory. Use it as an anonymous member of your test fixture, calling its
// Initialize method from your SetUp method and its Destroy method from your
// TearDown method.
type SampleTest struct {
	// A context object that can be used for long-running operations.
	Ctx context.Context

	// A clock with a fixed initial time. The test's set up method may use this
	// to wire the file system with a clock, if desired.
	Clock timeutil.SimulatedClock

	// The directory at which the file system is mounted.
	Dir string

	mfs *fuse.MountedFileSystem
}

// Mount the supplied file system and initialize the exported fields of the
// struct. Panics on error.
func (t *SampleTest) Initialize(fs fuse.FileSystem, config *fuse.MountConfig) {
	err := t.initialize(fs, config)
	if err != nil {
		panic(err)
	}
}

// Like Initialize, but doens't panic.
func (t *SampleTest) initialize(
	fs fuse.FileSystem,
	config *fuse.MountConfig) (err error) {
	// Initialize the context.
	t.Ctx = context.Background()

	// Initialize the clock.
	t.Clock.SetTime(time.Date(2012, 8, 15, 22, 56, 0, 0, time.Local))

	// Set up a temporary directory.
	t.Dir, err = ioutil.TempDir("", "sample_test")
	if err != nil {
		err = fmt.Errorf("TempDir: %v", err)
		return
	}

	// Mount the file system.
	t.mfs, err = fuse.Mount(t.Dir, fs, config)
	if err != nil {
		err = fmt.Errorf("Mount: %v", err)
		return
	}

	return
}

// Unmount the file system and clean up. Panics on error.
func (t *SampleTest) Destroy() {
	// Was the file system mounted?
	if t.mfs == nil {
		return
	}

	// Unmount the file system. Try again on "resource busy" errors.
	delay := 10 * time.Millisecond
	for {
		err := t.mfs.Unmount()
		if err == nil {
			break
		}

		if strings.Contains(err.Error(), "resource busy") {
			log.Println("Resource busy error while unmounting; trying again")
			time.Sleep(delay)
			delay = time.Duration(1.3 * float64(delay))
			continue
		}

		panic("MountedFileSystem.Unmount: " + err.Error())
	}

	if err := t.mfs.Join(context.Background()); err != nil {
		panic("MountedFileSystem.Join: " + err.Error())
	}
}
<commit_msg>Oops, use t.Ctx.<commit_after>// Copyright 2015 Google Inc. All Rights Reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package samples

import (
	"fmt"
	"io/ioutil"
	"log"
	"strings"
	"time"

	"github.com/googlecloudplatform/gcsfuse/timeutil"
	"github.com/jacobsa/fuse"
	"golang.org/x/net/context"
)

// A struct that implements common behavior needed by tests in the samples/
// directory. Use it as an anonymous member of your test fixture, calling its
// Initialize method from your SetUp method and its Destroy method from your
// TearDown method.
type SampleTest struct {
	// A context object that can be used for long-running operations.
	Ctx context.Context

	// A clock with a fixed initial time. The test's set up method may use this
	// to wire the file system with a clock, if desired.
	Clock timeutil.SimulatedClock

	// The directory at which the file system is mounted.
	Dir string

	mfs *fuse.MountedFileSystem
}

// Mount the supplied file system and initialize the exported fields of the
// struct. Panics on error.
func (t *SampleTest) Initialize(fs fuse.FileSystem, config *fuse.MountConfig) {
	err := t.initialize(fs, config)
	if err != nil {
		panic(err)
	}
}

// Like Initialize, but doens't panic.
func (t *SampleTest) initialize(
	fs fuse.FileSystem,
	config *fuse.MountConfig) (err error) {
	// Initialize the context.
	t.Ctx = context.Background()

	// Initialize the clock.
	t.Clock.SetTime(time.Date(2012, 8, 15, 22, 56, 0, 0, time.Local))

	// Set up a temporary directory.
	t.Dir, err = ioutil.TempDir("", "sample_test")
	if err != nil {
		err = fmt.Errorf("TempDir: %v", err)
		return
	}

	// Mount the file system.
	t.mfs, err = fuse.Mount(t.Dir, fs, config)
	if err != nil {
		err = fmt.Errorf("Mount: %v", err)
		return
	}

	return
}

// Unmount the file system and clean up. Panics on error.
func (t *SampleTest) Destroy() {
	// Was the file system mounted?
	if t.mfs == nil {
		return
	}

	// Unmount the file system. Try again on "resource busy" errors.
	delay := 10 * time.Millisecond
	for {
		err := t.mfs.Unmount()
		if err == nil {
			break
		}

		if strings.Contains(err.Error(), "resource busy") {
			log.Println("Resource busy error while unmounting; trying again")
			time.Sleep(delay)
			delay = time.Duration(1.3 * float64(delay))
			continue
		}

		panic("MountedFileSystem.Unmount: " + err.Error())
	}

	if err := t.mfs.Join(t.Ctx); err != nil {
		panic("MountedFileSystem.Join: " + err.Error())
	}
}
<|endoftext|>


<commit_before>// Copyright 2014 The Go Authors.  All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

// TODO(adg): add logging
// TODO(proppy): restrict memory use
// TODO(adg): send exit code to user

// Command sandbox is an HTTP server that takes requests containing go
// source files, and builds and executes them in a NaCl sanbox.
package main

import (
	"encoding/json"
	"errors"
	"fmt"
	"io/ioutil"
	"log"
	"net/http"
	"os"
	"os/exec"
	"path/filepath"
	"strings"
	"time"
)

const maxRunTime = 500 * time.Millisecond

type Request struct {
	Body string
}

type Response struct {
	Errors string
	Events []Event
}

func main() {
	if len(os.Args) > 1 && os.Args[1] == "test" {
		test()
		return
	}
	http.HandleFunc("/compile", compileHandler)
	http.HandleFunc("/_ah/health", healthHandler)
	log.Fatal(http.ListenAndServe(":8080", nil))
}

func compileHandler(w http.ResponseWriter, r *http.Request) {
	var req Request
	if err := json.NewDecoder(r.Body).Decode(&req); err != nil {
		http.Error(w, fmt.Sprintf("error decoding request: %v", err), http.StatusBadRequest)
		return
	}
	resp, err := compileAndRun(&req)
	if err != nil {
		http.Error(w, err.Error(), http.StatusInternalServerError)
		return
	}
	if err := json.NewEncoder(w).Encode(resp); err != nil {
		http.Error(w, fmt.Sprintf("error encoding response: %v", err), http.StatusInternalServerError)
		return
	}
}

func compileAndRun(req *Request) (*Response, error) {
	tmpDir, err := ioutil.TempDir("", "sandbox")
	if err != nil {
		return nil, fmt.Errorf("error creating temp directory: %v", err)
	}
	defer os.RemoveAll(tmpDir)

	in := filepath.Join(tmpDir, "main.go")
	if err := ioutil.WriteFile(in, []byte(req.Body), 0400); err != nil {
		return nil, fmt.Errorf("error creating temp file %q: %v", in, err)
	}
	exe := filepath.Join(tmpDir, "a.out")
	cmd := exec.Command("go", "build", "-o", exe, in)
	cmd.Env = []string{"GOOS=nacl", "GOARCH=amd64p32", "GOPATH=" + os.Getenv("GOPATH")}
	if out, err := cmd.CombinedOutput(); err != nil {
		if _, ok := err.(*exec.ExitError); ok {
			// Return compile errors to the user.

			// Rewrite compiler errors to refer to 'prog.go'
			// instead of '/tmp/sandbox1234/main.go'.
			errs := strings.Replace(string(out), in, "prog.go", -1)

			// "go build", invoked with a file name, puts this odd
			// message before any compile errors; strip it.
			errs = strings.Replace(errs, "# command-line-arguments\n", "", 1)

			return &Response{Errors: errs}, nil
		}
		return nil, fmt.Errorf("error building go source: %v", err)
	}
	cmd = exec.Command("sel_ldr_x86_64", "-l", "/dev/null", "-S", "-e", exe)
	rec := new(Recorder)
	cmd.Stdout = rec.Stdout()
	cmd.Stderr = rec.Stderr()
	if err := runTimeout(cmd, maxRunTime); err != nil {
		if err == timeoutErr {
			return &Response{Errors: "process took too long"}, nil
		}
		if _, ok := err.(*exec.ExitError); !ok {
			return nil, fmt.Errorf("error running sandbox: %v", err)
		}
	}
	events, err := rec.Events()
	if err != nil {
		return nil, fmt.Errorf("error decoding events: %v", err)
	}
	return &Response{Events: events}, nil
}

var timeoutErr = errors.New("process timed out")

func runTimeout(cmd *exec.Cmd, d time.Duration) error {
	if err := cmd.Start(); err != nil {
		return err
	}
	errc := make(chan error, 1)
	go func() {
		errc <- cmd.Wait()
	}()
	t := time.NewTimer(d)
	select {
	case err := <-errc:
		t.Stop()
		return err
	case <-t.C:
		cmd.Process.Kill()
		return timeoutErr
	}
}

func healthHandler(w http.ResponseWriter, r *http.Request) {
	if err := healthCheck(); err != nil {
		http.Error(w, "Health check failed: "+err.Error(), http.StatusInternalServerError)
		return
	}
	fmt.Fprint(w, "ok")
}

func healthCheck() error {
	resp, err := compileAndRun(&Request{Body: healthProg})
	if err != nil {
		return err
	}
	if resp.Errors != "" {
		return fmt.Errorf("compile error: %v", resp.Errors)
	}
	if len(resp.Events) != 1 || resp.Events[0].Message != "ok" {
		return fmt.Errorf("unexpected output: %v", resp.Events)
	}
	return nil
}

const healthProg = `
package main

import "fmt"

func main() { fmt.Print("ok") }
`

func test() {
	if err := healthCheck(); err != nil {
		log.Fatal(err)
	}
	for _, t := range tests {
		resp, err := compileAndRun(&Request{Body: t.prog})
		if err != nil {
			log.Fatal(err)
		}
		if resp.Errors != "" {
			log.Fatal(resp.Errors)
		}
		if len(resp.Events) != 1 || !strings.Contains(resp.Events[0].Message, t.want) {
			log.Fatalf("unexpected output: %v, want %q", resp.Events, t.want)
		}
	}
	fmt.Println("OK")
}

var tests = []struct {
	prog, want string
}{
	{`
package main

import "time"

func main() {
	loc, err := time.LoadLocation("America/New_York")
	if err != nil {
		panic(err.Error())
	}
	println(loc.String())
}
	`, "America/New_York"},

	{`
package main

import (
	"fmt"
	"time"
)

func main() {
	fmt.Println(time.Now())
}
	`, "2009-11-10 23:00:00 +0000 UTC"},

	{`
package main

import (
	"fmt"
	"time"
)

func main() {
	t1 := time.Tick(time.Second * 3)
	t2 := time.Tick(time.Second * 7)
	t3 := time.Tick(time.Second * 11)
	end := time.After(time.Second * 19)
	want := "112131211"
	var got []byte
	for {
		var c byte
		select {
		case <-t1:
			c = '1'
		case <-t2:
			c = '2'
		case <-t3:
			c = '3'
		case <-end:
			if g := string(got); g != want {
				fmt.Printf("got %q, want %q\n", g, want)
			} else {
				fmt.Println("timers fired as expected")
			}
			return
		}
		got = append(got, c)
	}
}
	`, "timers fired as expected"},

	{`
package main

import (
	"code.google.com/p/go-tour/pic"
	"code.google.com/p/go-tour/reader"
	"code.google.com/p/go-tour/tree"
	"code.google.com/p/go-tour/wc"
)

var (
	_ = pic.Show
	_ = reader.Validate
	_ = tree.New
	_ = wc.Test
)

func main() {
	println("ok")
}
	`, "ok"},
}
<commit_msg>playground: improve error message when package != 'main'<commit_after>// Copyright 2014 The Go Authors.  All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

// TODO(adg): add logging
// TODO(proppy): restrict memory use
// TODO(adg): send exit code to user

// Command sandbox is an HTTP server that takes requests containing go
// source files, and builds and executes them in a NaCl sanbox.
package main

import (
	"encoding/json"
	"errors"
	"fmt"
	"go/parser"
	"go/token"
	"io/ioutil"
	"log"
	"net/http"
	"os"
	"os/exec"
	"path/filepath"
	"strings"
	"time"
)

const maxRunTime = 500 * time.Millisecond

type Request struct {
	Body string
}

type Response struct {
	Errors string
	Events []Event
}

func main() {
	if len(os.Args) > 1 && os.Args[1] == "test" {
		test()
		return
	}
	http.HandleFunc("/compile", compileHandler)
	http.HandleFunc("/_ah/health", healthHandler)
	log.Fatal(http.ListenAndServe(":8080", nil))
}

func compileHandler(w http.ResponseWriter, r *http.Request) {
	var req Request
	if err := json.NewDecoder(r.Body).Decode(&req); err != nil {
		http.Error(w, fmt.Sprintf("error decoding request: %v", err), http.StatusBadRequest)
		return
	}
	resp, err := compileAndRun(&req)
	if err != nil {
		http.Error(w, err.Error(), http.StatusInternalServerError)
		return
	}
	if err := json.NewEncoder(w).Encode(resp); err != nil {
		http.Error(w, fmt.Sprintf("error encoding response: %v", err), http.StatusInternalServerError)
		return
	}
}

func compileAndRun(req *Request) (*Response, error) {
	tmpDir, err := ioutil.TempDir("", "sandbox")
	if err != nil {
		return nil, fmt.Errorf("error creating temp directory: %v", err)
	}
	defer os.RemoveAll(tmpDir)

	in := filepath.Join(tmpDir, "main.go")
	if err := ioutil.WriteFile(in, []byte(req.Body), 0400); err != nil {
		return nil, fmt.Errorf("error creating temp file %q: %v", in, err)
	}

	fset := token.NewFileSet()

	f, err := parser.ParseFile(fset, in, nil, parser.PackageClauseOnly)
	if err == nil && f.Name.Name != "main" {
		return &Response{Errors: "package name must be main"}, nil
	}

	exe := filepath.Join(tmpDir, "a.out")
	cmd := exec.Command("go", "build", "-o", exe, in)
	cmd.Env = []string{"GOOS=nacl", "GOARCH=amd64p32", "GOPATH=" + os.Getenv("GOPATH")}
	if out, err := cmd.CombinedOutput(); err != nil {
		if _, ok := err.(*exec.ExitError); ok {
			// Return compile errors to the user.

			// Rewrite compiler errors to refer to 'prog.go'
			// instead of '/tmp/sandbox1234/main.go'.
			errs := strings.Replace(string(out), in, "prog.go", -1)

			// "go build", invoked with a file name, puts this odd
			// message before any compile errors; strip it.
			errs = strings.Replace(errs, "# command-line-arguments\n", "", 1)

			return &Response{Errors: errs}, nil
		}
		return nil, fmt.Errorf("error building go source: %v", err)
	}
	cmd = exec.Command("sel_ldr_x86_64", "-l", "/dev/null", "-S", "-e", exe)
	rec := new(Recorder)
	cmd.Stdout = rec.Stdout()
	cmd.Stderr = rec.Stderr()
	if err := runTimeout(cmd, maxRunTime); err != nil {
		if err == timeoutErr {
			return &Response{Errors: "process took too long"}, nil
		}
		if _, ok := err.(*exec.ExitError); !ok {
			return nil, fmt.Errorf("error running sandbox: %v", err)
		}
	}
	events, err := rec.Events()
	if err != nil {
		return nil, fmt.Errorf("error decoding events: %v", err)
	}
	return &Response{Events: events}, nil
}

var timeoutErr = errors.New("process timed out")

func runTimeout(cmd *exec.Cmd, d time.Duration) error {
	if err := cmd.Start(); err != nil {
		return err
	}
	errc := make(chan error, 1)
	go func() {
		errc <- cmd.Wait()
	}()
	t := time.NewTimer(d)
	select {
	case err := <-errc:
		t.Stop()
		return err
	case <-t.C:
		cmd.Process.Kill()
		return timeoutErr
	}
}

func healthHandler(w http.ResponseWriter, r *http.Request) {
	if err := healthCheck(); err != nil {
		http.Error(w, "Health check failed: "+err.Error(), http.StatusInternalServerError)
		return
	}
	fmt.Fprint(w, "ok")
}

func healthCheck() error {
	resp, err := compileAndRun(&Request{Body: healthProg})
	if err != nil {
		return err
	}
	if resp.Errors != "" {
		return fmt.Errorf("compile error: %v", resp.Errors)
	}
	if len(resp.Events) != 1 || resp.Events[0].Message != "ok" {
		return fmt.Errorf("unexpected output: %v", resp.Events)
	}
	return nil
}

const healthProg = `
package main

import "fmt"

func main() { fmt.Print("ok") }
`

func test() {
	if err := healthCheck(); err != nil {
		log.Fatal(err)
	}
	for _, t := range tests {
		resp, err := compileAndRun(&Request{Body: t.prog})
		if err != nil {
			log.Fatal(err)
		}
		if t.errors != "" {
			if resp.Errors != t.errors {
				log.Fatalf("resp.Errors = %q, want %q", resp.Errors, t.errors)
			}
			continue
		}
		if resp.Errors != "" {
			log.Fatal(resp.Errors)
		}
		if len(resp.Events) != 1 || !strings.Contains(resp.Events[0].Message, t.want) {
			log.Fatalf("unexpected output: %v, want %q", resp.Events, t.want)
		}
	}
	fmt.Println("OK")
}

var tests = []struct {
	prog, want, errors string
}{
	{prog: `
package main

import "time"

func main() {
	loc, err := time.LoadLocation("America/New_York")
	if err != nil {
		panic(err.Error())
	}
	println(loc.String())
}
`, want: "America/New_York"},

	{prog: `
package main

import (
	"fmt"
	"time"
)

func main() {
	fmt.Println(time.Now())
}
`, want: "2009-11-10 23:00:00 +0000 UTC"},

	{prog: `
package main

import (
	"fmt"
	"time"
)

func main() {
	t1 := time.Tick(time.Second * 3)
	t2 := time.Tick(time.Second * 7)
	t3 := time.Tick(time.Second * 11)
	end := time.After(time.Second * 19)
	want := "112131211"
	var got []byte
	for {
		var c byte
		select {
		case <-t1:
			c = '1'
		case <-t2:
			c = '2'
		case <-t3:
			c = '3'
		case <-end:
			if g := string(got); g != want {
				fmt.Printf("got %q, want %q\n", g, want)
			} else {
				fmt.Println("timers fired as expected")
			}
			return
		}
		got = append(got, c)
	}
}
`, want: "timers fired as expected"},

	{prog: `
package main

import (
	"code.google.com/p/go-tour/pic"
	"code.google.com/p/go-tour/reader"
	"code.google.com/p/go-tour/tree"
	"code.google.com/p/go-tour/wc"
)

var (
	_ = pic.Show
	_ = reader.Validate
	_ = tree.New
	_ = wc.Test
)

func main() {
	println("ok")
}
`, want: "ok"},
	{prog: `
package test

func main() {
    println("test")
}
`, want: "", errors: "package name must be main"},
}
<|endoftext|>


<commit_before>package schema

import (
	"context"
	"testing"

	"github.com/go-ggz/ggz/model"

	"github.com/graphql-go/graphql"
	"github.com/stretchr/testify/assert"
)

func TestQueryShortenURL(t *testing.T) {
	assert.NoError(t, model.PrepareTestDatabase())
	user, _ := model.GetUserByID(1)
	ctx := newContextWithUser(context.TODO(), user)
	test := T{
		Query: `
		{
      QueryShortenURL(slug: "abcdef") {
        url
      }
    }
	  `,
		Schema: Schema,
		Expected: &graphql.Result{
			Data: map[string]interface{}{
				"QueryShortenURL": map[string]interface{}{
					"url": "http://example.com",
				},
			},
		},
	}
	params := graphql.Params{
		Schema:        test.Schema,
		RequestString: test.Query,
		Context:       ctx,
	}
	testGraphql(test, params, t)
}

func TestQueryShortenURLNotFound(t *testing.T) {
	assert.NoError(t, model.PrepareTestDatabase())
	user, _ := model.GetUserByID(1)
	ctx := newContextWithUser(context.TODO(), user)
	test := T{
		Query: `
{
  QueryShortenURL(slug: "1234567890") {
    url
  }
}
`,
		Schema: Schema,
	}
	params := graphql.Params{
		Schema:        test.Schema,
		RequestString: test.Query,
		Context:       ctx,
	}
	testGraphqlErr(test, params, t)
}
<commit_msg>test: update query url<commit_after>package schema

import (
	"context"
	"testing"

	"github.com/go-ggz/ggz/model"

	"github.com/graphql-go/graphql"
	"github.com/stretchr/testify/assert"
)

func TestQueryShortenURL(t *testing.T) {
	assert.NoError(t, model.PrepareTestDatabase())
	user, _ := model.GetUserByID(1)
	ctx := newContextWithUser(context.TODO(), user)
	t.Run("shorten url exist", func(t *testing.T) {
		test := T{
			Query: `
{
  QueryShortenURL(slug: "abcdef") {
    url
  }
}
	  `,
			Schema: Schema,
			Expected: &graphql.Result{
				Data: map[string]interface{}{
					"QueryShortenURL": map[string]interface{}{
						"url": "http://example.com",
					},
				},
			},
		}
		params := graphql.Params{
			Schema:        test.Schema,
			RequestString: test.Query,
			Context:       ctx,
		}
		testGraphql(test, params, t)
	})

	t.Run("shorten url not exist", func(t *testing.T) {
		test := T{
			Query: `
{
  QueryShortenURL(slug: "1234567890") {
    url
  }
}
`,
			Schema: Schema,
		}
		params := graphql.Params{
			Schema:        test.Schema,
			RequestString: test.Query,
			Context:       ctx,
		}
		testGraphqlErr(test, params, t)
	})

}
<|endoftext|>


<commit_before>package scraper

import (
	"encoding/json"
	"errors"
	"fmt"
	"io"
	"io/ioutil"
	"net/http"
	"strings"

	"github.com/PuerkitoBio/goquery"
)

//a single result
type result map[string]string

//the configuration file
type Config map[string]*Endpoint

type Handler struct {
	Config Config `opts:"-"`
	Auth   string `help:"Basic auth credentials <user>:<pass>"`
	Log    bool   `opts:"-"`
	Debug  bool   `help:"Enable debug output"`
}

func (h *Handler) LoadConfigFile(path string) error {
	b, err := ioutil.ReadFile(path)
	if err != nil {
		return err
	}
	return h.LoadConfig(b)
}

func (h *Handler) LoadConfig(b []byte) error {
	c := Config{}
	//json unmarshal performs selector validation
	if err := json.Unmarshal(b, &c); err != nil {
		return err
	}
	if h.Log {
		for k, e := range c {
			if strings.HasPrefix(k, "/") {
				delete(c, k)
				k = strings.TrimPrefix(k, "/")
				c[k] = e
			}
			logf("Loaded endpoint: /%s", k)
			e.debug = h.Debug
		}
	}
	if h.Debug {
		logf("Enabled debug mode")
	}
	//replace config
	h.Config = c
	return nil
}

func (h *Handler) ServeHTTP(w http.ResponseWriter, r *http.Request) {
	defer r.Body.Close()

	//basic auth
	if h.Auth != "" {
		u, p, _ := r.BasicAuth()
		if h.Auth != u+":"+p {
			w.WriteHeader(http.StatusUnauthorized)
			w.Write([]byte("Access Denied"))
			return
		}
	}

	//always JSON!
	w.Header().Set("Content-Type", "application/json")

	//admin actions
	if r.URL.Path == "" || r.URL.Path == "/" {
		get := false
		if r.Method == "GET" {
			get = true
		} else if r.Method == "POST" {
			b, err := ioutil.ReadAll(r.Body)
			if err != nil {
				w.WriteHeader(http.StatusBadRequest)
				w.Write(jsonerr(err))
				return
			}
			if err := h.LoadConfig(b); err != nil {
				w.WriteHeader(http.StatusBadRequest)
				w.Write(jsonerr(err))
				return
			}
			get = true
		}

		if !get {
			w.WriteHeader(http.StatusMethodNotAllowed)
			w.Write(jsonerr(errors.New("Use GET or POST")))
		}
		b, _ := json.MarshalIndent(h.Config, "", "  ")
		w.Write(b)
		return
	}
	//search actions
	id := r.URL.Path[1:] //exclude root slash
	if e, ok := h.Config[id]; ok {
		h.execute(e, w, r)
		return
	}
	w.WriteHeader(404)
	w.Write(jsonerr(fmt.Errorf("Endpoint /%s not found", id)))
}

func (h *Handler) execute(e *Endpoint, w http.ResponseWriter, r *http.Request) {

	values := r.URL.Query()

	url, err := template(true, e.URL, values)
	if err != nil {
		w.WriteHeader(http.StatusBadRequest)
		w.Write(jsonerr(err))
		return
	}

	method := e.Method
	if method == "" {
		method = "GET"
	}

	body := io.Reader(nil)
	if e.Body != "" {
		if s, err := template(true, e.Body, values); err != nil {
			w.WriteHeader(http.StatusInternalServerError)
			w.Write(jsonerr(err))
			return
		} else {
			body = strings.NewReader(s)
		}
	}

	req, err := http.NewRequest(method, url, body)
	if err != nil {
		w.WriteHeader(http.StatusInternalServerError)
		w.Write(jsonerr(err))
		return
	}

	if e.Headers != nil {
		for k, v := range e.Headers {
			req.Header.Set(k, v)
		}
	}

	resp, err := http.DefaultClient.Do(req)
	if err != nil {
		w.WriteHeader(http.StatusInternalServerError)
		w.Write(jsonerr(err))
		return
	}

	if h.Log {
		logf("%s %s => %s", method, url, resp.Status)
	}

	doc, err := goquery.NewDocumentFromReader(resp.Body)
	if err != nil {
		w.WriteHeader(http.StatusInternalServerError)
		w.Write(jsonerr(err))
	}
	sel := doc.Selection

	var out interface{}
	//out will be either a list of results, or a single result
	if e.List != "" {
		var results []result
		sels := sel.Find(e.List)
		if h.Debug {
			logf("list: %s => #%d elements", e.List, sels.Length())
		}
		sels.Each(func(i int, sel *goquery.Selection) {
			r := e.extract(sel)
			if len(r) == len(e.Result) {
				results = append(results, r)
			} else if h.Debug {
				logf("excluded #%d: has %d fields, expected %d", i, len(r), len(e.Result))
			}
		})
		out = results
	} else {
		out = e.extract(sel)
	}
	b, _ := json.MarshalIndent(out, "", "  ")
	w.Write(b)
}
<commit_msg>disable JSON HTML encoding<commit_after>package scraper

import (
	"encoding/json"
	"errors"
	"fmt"
	"io"
	"io/ioutil"
	"net/http"
	"strings"

	"github.com/PuerkitoBio/goquery"
)

//a single result
type result map[string]string

//the configuration file
type Config map[string]*Endpoint

type Handler struct {
	Config Config `opts:"-"`
	Auth   string `help:"Basic auth credentials <user>:<pass>"`
	Log    bool   `opts:"-"`
	Debug  bool   `help:"Enable debug output"`
}

func (h *Handler) LoadConfigFile(path string) error {
	b, err := ioutil.ReadFile(path)
	if err != nil {
		return err
	}
	return h.LoadConfig(b)
}

func (h *Handler) LoadConfig(b []byte) error {
	c := Config{}
	//json unmarshal performs selector validation
	if err := json.Unmarshal(b, &c); err != nil {
		return err
	}
	if h.Log {
		for k, e := range c {
			if strings.HasPrefix(k, "/") {
				delete(c, k)
				k = strings.TrimPrefix(k, "/")
				c[k] = e
			}
			logf("Loaded endpoint: /%s", k)
			e.debug = h.Debug
		}
	}
	if h.Debug {
		logf("Enabled debug mode")
	}
	//replace config
	h.Config = c
	return nil
}

func (h *Handler) ServeHTTP(w http.ResponseWriter, r *http.Request) {
	defer r.Body.Close()

	//basic auth
	if h.Auth != "" {
		u, p, _ := r.BasicAuth()
		if h.Auth != u+":"+p {
			w.WriteHeader(http.StatusUnauthorized)
			w.Write([]byte("Access Denied"))
			return
		}
	}

	//always JSON!
	w.Header().Set("Content-Type", "application/json")

	//admin actions
	if r.URL.Path == "" || r.URL.Path == "/" {
		get := false
		if r.Method == "GET" {
			get = true
		} else if r.Method == "POST" {
			b, err := ioutil.ReadAll(r.Body)
			if err != nil {
				w.WriteHeader(http.StatusBadRequest)
				w.Write(jsonerr(err))
				return
			}
			if err := h.LoadConfig(b); err != nil {
				w.WriteHeader(http.StatusBadRequest)
				w.Write(jsonerr(err))
				return
			}
			get = true
		}

		if !get {
			w.WriteHeader(http.StatusMethodNotAllowed)
			w.Write(jsonerr(errors.New("Use GET or POST")))
		}
		b, _ := json.MarshalIndent(h.Config, "", "  ")
		w.Write(b)
		return
	}
	//search actions
	id := r.URL.Path[1:] //exclude root slash
	if e, ok := h.Config[id]; ok {
		h.execute(e, w, r)
		return
	}
	w.WriteHeader(404)
	w.Write(jsonerr(fmt.Errorf("Endpoint /%s not found", id)))
}

func (h *Handler) execute(e *Endpoint, w http.ResponseWriter, r *http.Request) {

	values := r.URL.Query()

	url, err := template(true, e.URL, values)
	if err != nil {
		w.WriteHeader(http.StatusBadRequest)
		w.Write(jsonerr(err))
		return
	}

	method := e.Method
	if method == "" {
		method = "GET"
	}

	body := io.Reader(nil)
	if e.Body != "" {
		if s, err := template(true, e.Body, values); err != nil {
			w.WriteHeader(http.StatusInternalServerError)
			w.Write(jsonerr(err))
			return
		} else {
			body = strings.NewReader(s)
		}
	}

	req, err := http.NewRequest(method, url, body)
	if err != nil {
		w.WriteHeader(http.StatusInternalServerError)
		w.Write(jsonerr(err))
		return
	}

	if e.Headers != nil {
		for k, v := range e.Headers {
			req.Header.Set(k, v)
		}
	}

	resp, err := http.DefaultClient.Do(req)
	if err != nil {
		w.WriteHeader(http.StatusInternalServerError)
		w.Write(jsonerr(err))
		return
	}

	if h.Log {
		logf("%s %s => %s", method, url, resp.Status)
	}

	doc, err := goquery.NewDocumentFromReader(resp.Body)
	if err != nil {
		w.WriteHeader(http.StatusInternalServerError)
		w.Write(jsonerr(err))
	}
	sel := doc.Selection

	var out interface{}
	//out will be either a list of results, or a single result
	if e.List != "" {
		var results []result
		sels := sel.Find(e.List)
		if h.Debug {
			logf("list: %s => #%d elements", e.List, sels.Length())
		}
		sels.Each(func(i int, sel *goquery.Selection) {
			r := e.extract(sel)
			if len(r) == len(e.Result) {
				results = append(results, r)
			} else if h.Debug {
				logf("excluded #%d: has %d fields, expected %d", i, len(r), len(e.Result))
			}
		})
		out = results
	} else {
		out = e.extract(sel)
	}

	enc := json.NewEncoder(w)
	enc.SetEscapeHTML(false)
	enc.SetIndent("", "  ")
	if err := enc.Encode(out); err != nil {
		w.Write([]byte("JSON Error: " + err.Error()))
	}
}
<|endoftext|>


<commit_before>/*
Copyright 2017 Gravitational, Inc.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package monitoring

import (
	"context"
	"fmt"

	"github.com/gravitational/satellite/agent/health"
	pb "github.com/gravitational/satellite/agent/proto/agentpb"

	"github.com/gravitational/trace"
)

// NewPortChecker returns a new port range checker
func NewPortChecker(ranges ...PortRange) health.Checker {
	return &portChecker{
		ranges:   ranges,
		getPorts: realGetPorts,
	}
}

// PortRange defines ports and protocol family to check
type PortRange struct {
	// Protocol of the port
	Protocol string
	// Port range.
	// A single port is defined as From == To
	From, To uint64
	// Description specifies the user-friendly range description
	Description string
}

// portChecker will validate that all required ports are in fact unoccupied
type portChecker struct {
	ranges   []PortRange
	getPorts portCollectorFunc
}

// Name returns this checker name
// Implements health.Checker
func (c *portChecker) Name() string {
	return portCheckerID
}

// Check will scan current open ports and report every conflict detected
// Implements health.Checker
func (c *portChecker) Check(ctx context.Context, reporter health.Reporter) {
	processes, err := c.getPorts()
	if err != nil {
		reporter.Add(NewProbeFromErr(portCheckerID, "failed to query socket connections", trace.Wrap(err)))
		return
	}

	type conn struct {
		pid      pid
		port     int
		protocol string
	}
	// Group processes on the pid/port/protocol to avoid duplicates
	unique := make(map[conn]process)
	for _, process := range processes {
		conn := conn{
			pid:      process.pid,
			port:     process.localAddr().port,
			protocol: process.proto(),
		}
		if _, exists := unique[conn]; !exists {
			unique[conn] = process
		}
	}

	conflicts := false
	for _, proc := range unique {
		if c.checkProcess(proc, reporter) {
			conflicts = true
		}
	}

	if conflicts {
		return
	}
	reporter.Add(NewSuccessProbe(c.Name()))
}

func (c *portChecker) checkProcess(proc process, reporter health.Reporter) bool {
	conflicts := false
	for _, r := range c.ranges {
		if r.Protocol != proc.socket.proto() {
			continue
		}
		if uint64(proc.localAddr().port) >= r.From && uint64(proc.localAddr().port) <= r.To {
			conflicts = true
			reporter.Add(&pb.Probe{
				Checker: portCheckerID,
				Detail: fmt.Sprintf("conflicting program %q(pid=%v) is occupying port %v/%d(%v)",
					proc.name, proc.pid, proc.socket.proto(), proc.localAddr().port, proc.state()),
				Status: pb.Probe_Failed})
		}
	}
	return conflicts
}

const (
	protoTCP      = "tcp"
	protoUDP      = "udp"
	portCheckerID = "port-checker"
)
<commit_msg>Ignore time-wait sockets when checking ports (forward-port).<commit_after>/*
Copyright 2017 Gravitational, Inc.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package monitoring

import (
	"context"
	"fmt"

	"github.com/gravitational/satellite/agent/health"
	pb "github.com/gravitational/satellite/agent/proto/agentpb"

	"github.com/gravitational/trace"
	log "github.com/sirupsen/logrus"
)

// NewPortChecker returns a new port range checker
func NewPortChecker(ranges ...PortRange) health.Checker {
	return &portChecker{
		ranges:   ranges,
		getPorts: realGetPorts,
	}
}

// PortRange defines ports and protocol family to check
type PortRange struct {
	// Protocol of the port
	Protocol string
	// Port range.
	// A single port is defined as From == To
	From, To uint64
	// Description specifies the user-friendly range description
	Description string
}

// portChecker will validate that all required ports are in fact unoccupied
type portChecker struct {
	ranges   []PortRange
	getPorts portCollectorFunc
}

// Name returns this checker name
// Implements health.Checker
func (c *portChecker) Name() string {
	return portCheckerID
}

// Check will scan current open ports and report every conflict detected
// Implements health.Checker
func (c *portChecker) Check(ctx context.Context, reporter health.Reporter) {
	processes, err := c.getPorts()
	if err != nil {
		reporter.Add(NewProbeFromErr(portCheckerID, "failed to query socket connections", trace.Wrap(err)))
		return
	}

	type conn struct {
		pid      pid
		port     int
		protocol string
	}
	// Group processes on the pid/port/protocol to avoid duplicates
	unique := make(map[conn]process)
	for _, process := range processes {
		conn := conn{
			pid:      process.pid,
			port:     process.localAddr().port,
			protocol: process.proto(),
		}
		if _, exists := unique[conn]; !exists {
			unique[conn] = process
		}
	}

	conflicts := false
	for _, proc := range unique {
		if c.checkProcess(proc, reporter) {
			conflicts = true
		}
	}

	if conflicts {
		return
	}
	reporter.Add(NewSuccessProbe(c.Name()))
}

func (c *portChecker) checkProcess(proc process, reporter health.Reporter) bool {
	conflicts := false
	for _, r := range c.ranges {
		if r.Protocol != proc.socket.proto() {
			continue
		}
		// ignore sockets in time-wait and closed states since they're going
		// away soon
		switch proc.socket.state() {
		case TimeWait, Close:
			log.Debugf("Ignoring socket in %q state for program %q(pid=%v).", proc.socket.state(), proc.name, proc.pid)
			continue
		}
		if uint64(proc.localAddr().port) >= r.From && uint64(proc.localAddr().port) <= r.To {
			conflicts = true
			reporter.Add(&pb.Probe{
				Checker: portCheckerID,
				Detail: fmt.Sprintf("conflicting program %q(pid=%v) is occupying port %v/%d(%v)",
					proc.name, proc.pid, proc.socket.proto(), proc.localAddr().port, proc.state()),
				Status: pb.Probe_Failed})
		}
	}
	return conflicts
}

const (
	protoTCP      = "tcp"
	protoUDP      = "udp"
	portCheckerID = "port-checker"
)
<|endoftext|>


<commit_before>// Copyright (c) 2014-2017 Ludovic Fauvet
// Licensed under the MIT license

package process

import (
	"errors"
	"fmt"
	"io/ioutil"
	"net"
	"os"
	"os/exec"
	"strconv"
	"syscall"

	"github.com/etix/mirrorbits/core"
	"github.com/op/go-logging"
)

var (
	// Compile time variable
	defaultPidFile string
)

var (
	// ErrInvalidfd is returned when the given file descriptor is invalid
	ErrInvalidfd = errors.New("invalid file descriptor")

	log = logging.MustGetLogger("main")
)

// Relaunch launches {self} as a child process passing listener details
// to provide a seamless binary upgrade.
func Relaunch(l net.Listener) error {
	argv0, err := exec.LookPath(os.Args[0])
	if err != nil {
		return err
	}
	if _, err := os.Stat(argv0); err != nil {
		return err
	}
	wd, err := os.Getwd()
	if err != nil {
		return err
	}

	var file *os.File

	switch t := l.(type) {
	case *net.TCPListener:
		file, err = t.File()
	case *net.UnixListener:
		file, err = t.File()
	default:
		return ErrInvalidfd
	}
	if err != nil {
		return err
	}

	fd := file.Fd()
	sysfile := file.Name()

	listener, ok := l.(*net.TCPListener)
	if ok {
		listenerFile, err := listener.File()
		if err != nil {
			return err
		}
		fd = listenerFile.Fd()
		sysfile = listenerFile.Name()
	}

	if fd < uintptr(syscall.Stderr) {
		return ErrInvalidfd
	}

	if err := os.Setenv("OLD_FD", fmt.Sprint(fd)); err != nil {
		return err
	}
	if err := os.Setenv("OLD_NAME", fmt.Sprintf("tcp:%s->", l.Addr().String())); err != nil {
		return err
	}
	if err := os.Setenv("OLD_PPID", fmt.Sprint(syscall.Getpid())); err != nil {
		return err
	}

	files := make([]*os.File, fd+1)
	files[syscall.Stdin] = os.Stdin
	files[syscall.Stdout] = os.Stdout
	files[syscall.Stderr] = os.Stderr
	files[fd] = os.NewFile(fd, sysfile)
	p, err := os.StartProcess(argv0, os.Args, &os.ProcAttr{
		Dir:   wd,
		Env:   os.Environ(),
		Files: files,
		Sys:   &syscall.SysProcAttr{},
	})
	if err != nil {
		return err
	}
	log.Infof("Spawned child %d\n", p.Pid)
	return nil
}

// Recover from a seamless binary upgrade and use an already
// existing listener to take over the connections
func Recover() (l net.Listener, ppid int, err error) {
	var fd uintptr
	_, err = fmt.Sscan(os.Getenv("OLD_FD"), &fd)
	if err != nil {
		return
	}
	var i net.Listener
	i, err = net.FileListener(os.NewFile(fd, os.Getenv("OLD_NAME")))
	if err != nil {
		return
	}
	switch i.(type) {
	case *net.TCPListener:
		l = i.(*net.TCPListener)
	case *net.UnixListener:
		l = i.(*net.UnixListener)
	default:
		err = fmt.Errorf("file descriptor is %T not *net.TCPListener or *net.UnixListener", i)
		return
	}
	if err = syscall.Close(int(fd)); err != nil {
		return
	}
	_, err = fmt.Sscan(os.Getenv("OLD_PPID"), &ppid)
	if err != nil {
		return
	}
	return
}

// KillParent sends a signal to make the parent exit gracefully with SIGQUIT
func KillParent(ppid int) error {
	log.Info("Asking parent to quit")
	return syscall.Kill(ppid, syscall.SIGQUIT)
}

// GetPidLocation finds the location to store our pid file
// and fallback to /var/run if none found
func GetPidLocation() string {
	if core.PidFile == "" { // Runtime
		if defaultPidFile == "" { // Compile time
			rdir := os.Getenv("XDG_RUNTIME_DIR")
			if rdir == "" {
				return "/var/run/mirrorbits.pid" // Fallback
			}
			return rdir + "/mirrorbits.pid"
		}
		return defaultPidFile
	}
	return core.PidFile
}

// WritePidFile writes the current pid file to disk
func WritePidFile() {
	pid := fmt.Sprintf("%d", os.Getpid())
	if err := ioutil.WriteFile(GetPidLocation(), []byte(pid), 0644); err != nil {
		log.Errorf("Unable to write pid file: %v", err)
	}
}

// RemovePidFile removes the current pid file
func RemovePidFile() {
	pidFile := GetPidLocation()
	if _, err := os.Stat(pidFile); !os.IsNotExist(err) {
		// Ensures we don't remove our forked process pid file
		// This can happen during seamless binary upgrade
		if GetRemoteProcPid() == os.Getpid() {
			if err = os.Remove(pidFile); err != nil {
				log.Errorf("Unable to remove pid file: %v", err)
			}
		}
	}
}

// GetRemoteProcPid gets the pid as it appears in the pid file (maybe not ours)
func GetRemoteProcPid() int {
	b, err := ioutil.ReadFile(GetPidLocation())
	if err != nil {
		return -1
	}
	i, err := strconv.ParseInt(string(b), 10, 0)
	if err != nil {
		return -1
	}
	return int(i)
}
<commit_msg>process: move the pidfile to its own /run directory<commit_after>// Copyright (c) 2014-2017 Ludovic Fauvet
// Licensed under the MIT license

package process

import (
	"errors"
	"fmt"
	"io/ioutil"
	"net"
	"os"
	"os/exec"
	"strconv"
	"syscall"

	"github.com/etix/mirrorbits/core"
	"github.com/op/go-logging"
)

var (
	// Compile time variable
	defaultPidFile string
)

var (
	// ErrInvalidfd is returned when the given file descriptor is invalid
	ErrInvalidfd = errors.New("invalid file descriptor")

	log = logging.MustGetLogger("main")
)

// Relaunch launches {self} as a child process passing listener details
// to provide a seamless binary upgrade.
func Relaunch(l net.Listener) error {
	argv0, err := exec.LookPath(os.Args[0])
	if err != nil {
		return err
	}
	if _, err := os.Stat(argv0); err != nil {
		return err
	}
	wd, err := os.Getwd()
	if err != nil {
		return err
	}

	var file *os.File

	switch t := l.(type) {
	case *net.TCPListener:
		file, err = t.File()
	case *net.UnixListener:
		file, err = t.File()
	default:
		return ErrInvalidfd
	}
	if err != nil {
		return err
	}

	fd := file.Fd()
	sysfile := file.Name()

	listener, ok := l.(*net.TCPListener)
	if ok {
		listenerFile, err := listener.File()
		if err != nil {
			return err
		}
		fd = listenerFile.Fd()
		sysfile = listenerFile.Name()
	}

	if fd < uintptr(syscall.Stderr) {
		return ErrInvalidfd
	}

	if err := os.Setenv("OLD_FD", fmt.Sprint(fd)); err != nil {
		return err
	}
	if err := os.Setenv("OLD_NAME", fmt.Sprintf("tcp:%s->", l.Addr().String())); err != nil {
		return err
	}
	if err := os.Setenv("OLD_PPID", fmt.Sprint(syscall.Getpid())); err != nil {
		return err
	}

	files := make([]*os.File, fd+1)
	files[syscall.Stdin] = os.Stdin
	files[syscall.Stdout] = os.Stdout
	files[syscall.Stderr] = os.Stderr
	files[fd] = os.NewFile(fd, sysfile)
	p, err := os.StartProcess(argv0, os.Args, &os.ProcAttr{
		Dir:   wd,
		Env:   os.Environ(),
		Files: files,
		Sys:   &syscall.SysProcAttr{},
	})
	if err != nil {
		return err
	}
	log.Infof("Spawned child %d\n", p.Pid)
	return nil
}

// Recover from a seamless binary upgrade and use an already
// existing listener to take over the connections
func Recover() (l net.Listener, ppid int, err error) {
	var fd uintptr
	_, err = fmt.Sscan(os.Getenv("OLD_FD"), &fd)
	if err != nil {
		return
	}
	var i net.Listener
	i, err = net.FileListener(os.NewFile(fd, os.Getenv("OLD_NAME")))
	if err != nil {
		return
	}
	switch i.(type) {
	case *net.TCPListener:
		l = i.(*net.TCPListener)
	case *net.UnixListener:
		l = i.(*net.UnixListener)
	default:
		err = fmt.Errorf("file descriptor is %T not *net.TCPListener or *net.UnixListener", i)
		return
	}
	if err = syscall.Close(int(fd)); err != nil {
		return
	}
	_, err = fmt.Sscan(os.Getenv("OLD_PPID"), &ppid)
	if err != nil {
		return
	}
	return
}

// KillParent sends a signal to make the parent exit gracefully with SIGQUIT
func KillParent(ppid int) error {
	log.Info("Asking parent to quit")
	return syscall.Kill(ppid, syscall.SIGQUIT)
}

// GetPidLocation finds the location to store our pid file
// and fallback to /var/run if none found
func GetPidLocation() string {
	if core.PidFile == "" { // Runtime
		if defaultPidFile == "" { // Compile time
			rdir := os.Getenv("XDG_RUNTIME_DIR")
			if rdir == "" {
				return "/run/mirrorbits/mirrorbits.pid" // Fallback
			}
			return rdir + "/mirrorbits.pid"
		}
		return defaultPidFile
	}
	return core.PidFile
}

// WritePidFile writes the current pid file to disk
func WritePidFile() {
	pid := fmt.Sprintf("%d", os.Getpid())
	if err := ioutil.WriteFile(GetPidLocation(), []byte(pid), 0644); err != nil {
		log.Errorf("Unable to write pid file: %v", err)
	}
}

// RemovePidFile removes the current pid file
func RemovePidFile() {
	pidFile := GetPidLocation()
	if _, err := os.Stat(pidFile); !os.IsNotExist(err) {
		// Ensures we don't remove our forked process pid file
		// This can happen during seamless binary upgrade
		if GetRemoteProcPid() == os.Getpid() {
			if err = os.Remove(pidFile); err != nil {
				log.Errorf("Unable to remove pid file: %v", err)
			}
		}
	}
}

// GetRemoteProcPid gets the pid as it appears in the pid file (maybe not ours)
func GetRemoteProcPid() int {
	b, err := ioutil.ReadFile(GetPidLocation())
	if err != nil {
		return -1
	}
	i, err := strconv.ParseInt(string(b), 10, 0)
	if err != nil {
		return -1
	}
	return int(i)
}
<|endoftext|>


<commit_before>package progress

import (
	"bytes"
	"fmt"
	"io"
	"strings"

	"github.com/containerd/console"
)

// Writer buffers writes until flush, at which time the last screen is cleared
// and the current buffer contents are written. This is useful for
// implementing progress displays, such as those implemented in docker and
// git.
type Writer struct {
	buf   bytes.Buffer
	w     io.Writer
	lines int
}

// NewWriter returns a writer
func NewWriter(w io.Writer) *Writer {
	return &Writer{
		w: w,
	}
}

// Write the provided bytes
func (w *Writer) Write(p []byte) (n int, err error) {
	return w.buf.Write(p)
}

// Flush should be called when refreshing the current display.
func (w *Writer) Flush() error {
	if w.buf.Len() == 0 {
		return nil
	}

	if err := w.clear(); err != nil {
		return err
	}

	ws, err := console.Current().Size()
	if err != nil {
		return fmt.Errorf("failed to get terminal width: %v", err)
	}
	strlines := strings.Split(w.buf.String(), "\n")
	w.lines = -1
	for _, line := range strlines {
		w.lines += len(line)/int(ws.Width) + 1
	}

	if _, err := w.w.Write(w.buf.Bytes()); err != nil {
		return err
	}

	w.buf.Reset()
	return nil
}

// TODO(stevvooe): The following are system specific. Break these out if we
// decide to build this package further.

func (w *Writer) clear() error {
	for i := 0; i < w.lines; i++ {
		if _, err := fmt.Fprintf(w.w, "\x1b[1A\x1b[2K\r"); err != nil {
			return err
		}
	}

	return nil
}
<commit_msg>Handle progress line length properly for formatting<commit_after>package progress

import (
	"bytes"
	"fmt"
	"io"
	"regexp"
	"strings"

	"github.com/containerd/console"
)

var (
	regexCleanLine = regexp.MustCompile("\x1b\\[[0-9]+m[\x1b]?")
)

// Writer buffers writes until flush, at which time the last screen is cleared
// and the current buffer contents are written. This is useful for
// implementing progress displays, such as those implemented in docker and
// git.
type Writer struct {
	buf   bytes.Buffer
	w     io.Writer
	lines int
}

// NewWriter returns a writer
func NewWriter(w io.Writer) *Writer {
	return &Writer{
		w: w,
	}
}

// Write the provided bytes
func (w *Writer) Write(p []byte) (n int, err error) {
	return w.buf.Write(p)
}

// Flush should be called when refreshing the current display.
func (w *Writer) Flush() error {
	if w.buf.Len() == 0 {
		return nil
	}

	if err := w.clear(); err != nil {
		return err
	}

	ws, err := console.Current().Size()
	if err != nil {
		return fmt.Errorf("failed to get terminal width: %v", err)
	}
	strlines := strings.Split(w.buf.String(), "\n")
	w.lines = -1
	for _, line := range strlines {
		w.lines += (len(stripLine(line))-1)/int(ws.Width) + 1
	}

	if _, err := w.w.Write(w.buf.Bytes()); err != nil {
		return err
	}

	w.buf.Reset()
	return nil
}

// TODO(stevvooe): The following are system specific. Break these out if we
// decide to build this package further.

func (w *Writer) clear() error {
	for i := 0; i < w.lines; i++ {
		if _, err := fmt.Fprintf(w.w, "\x1b[1A\x1b[2K\r"); err != nil {
			return err
		}
	}

	return nil
}

func stripLine(line string) string {
	return string(regexCleanLine.ReplaceAll([]byte(line), []byte{}))
}
<|endoftext|>


<commit_before>// Copyright 2016 go-mxnet-predictor Authors. All Rights Reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//      http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package mxnet

/*
// go preamble
#cgo pkg-config: mxnet
#include <mxnet/c_predict_api.h>
#include <stdlib.h>
*/
import "C"
import (
	"fmt"
	"unsafe"
)

const (
	CPU_DEVICE = iota + 1 // cpu device type
	GPU_DEVICE            // gpu device type
)

//TODO higher level api like context
type Device struct {
	Type int // device type
	Id   int // device id
}

type InputNode struct {
	Key   string   // name
	Shape []uint32 // shape of ndarray
}

// predictor for inference
type Predictor struct {
	handle C.PredictorHandle // C handle of predictor
}

// Create a Predictor
// go binding for MXPredCreate
// param symbol The JSON string of the symbol
// param params In-memory raw bytes of parameter ndarray file
// param device Device to run predictor
// param nodes An array of InputNode which stored the name and shape data of ndarray item
func CreatePredictor(symbol []byte,
	params []byte,
	device Device,
	nodes []InputNode,
) (*Predictor, error) {

	var (
		pc        *C.char
		shapeIdx  = []uint32{0}
		shapeData = []uint32{}
	)

	// malloc a **char which like []string to store node keys
	keys := C.malloc(C.size_t(len(nodes)) * C.size_t(unsafe.Sizeof(pc))) // c gc
	for i := 0; i < len(nodes); i++ {
		// get memory address
		p := (**C.char)(unsafe.Pointer(uintptr(keys) + uintptr(i)*unsafe.Sizeof(pc)))
		// c gc
		*p = C.CString(nodes[i].Key)

		// shapeIdx for next node
		shapeIdx = append(shapeIdx, uint32(len(nodes[i].Shape)))
		// shape data for current node
		shapeData = append(shapeData, nodes[i].Shape...)
	}

	var handle C.PredictorHandle

	success := C.MXPredCreate((*C.char)(unsafe.Pointer(&symbol[0])),
		unsafe.Pointer(&params[0]),
		C.int(len(params)),
		C.int(device.Type),
		C.int(device.Id),
		C.mx_uint(len(nodes)),
		(**C.char)(keys),
		(*C.mx_uint)(unsafe.Pointer(&shapeIdx[0])),
		(*C.mx_uint)(unsafe.Pointer(&shapeData[0])),
		&handle,
	)

	// free mem we created before return, go gc won't do that for us
	for i := 0; i < len(nodes); i++ {
		p := (**C.char)(unsafe.Pointer(uintptr(keys) + uintptr(i)*unsafe.Sizeof(pc)))
		C.free(unsafe.Pointer(*p))
	}
	C.free(unsafe.Pointer(keys))

	if success < 0 {
		return nil, GetLastError()
	}
	return &Predictor{handle: handle}, nil
}

// set the input data of predictor
// go binding for MXPredSetInput
// param key The name of input node to set
// param data The float data to be set
func (s *Predictor) SetInput(key string, data []float32) error {
	// check input
	if data == nil || len(data) < 1 {
		return fmt.Errorf("intput data nil or empty")
	}

	// c gc
	k := C.CString(key)
	// free mem before return
	defer C.free(unsafe.Pointer(k))

	success, err := C.MXPredSetInput(s.handle,
		k,
		(*C.mx_float)(unsafe.Pointer(&data[0])),
		C.mx_uint(len(data)),
	)

	if err != nil {
		return err
	} else if success < 0 {
		return GetLastError()
	}
	return nil
}

// run a forward pass after SetInput
// go binding for MXPredForward
func (s *Predictor) Forward() error {
	success, err := C.MXPredForward(s.handle)
	if err != nil {
		return err
	} else if success < 0 {
		return GetLastError()
	}
	return nil
}

// get the shape of output node
// go binding for MXPredGetOutputShape
// param index The index of output node, set to 0 if there is only one output
func (s *Predictor) GetOutputShape(index uint32) ([]uint32, error) {
	var (
		shapeData *C.mx_uint
		shapeDim  C.mx_uint
	)
	success, err := C.MXPredGetOutputShape(s.handle,
		C.mx_uint(index),
		&shapeData,
		&shapeDim,
	)
	if err != nil {
		return nil, err
	} else if success < 0 {
		return nil, GetLastError()
	}
	// c array to go
	shape := (*[1 << 32]uint32)(unsafe.Pointer(shapeData))[:shapeDim:shapeDim]
	return shape, nil
}

// get the output value of prediction
// go binding for MXPredGetOutput
// param index The index of output node, set to 0 if there is only one output
func (s *Predictor) GetOutput(index uint32) ([]float32, error) {
	shape, err := s.GetOutputShape(index)
	if err != nil {
		return nil, err
	}
	size := uint32(1)
	for _, v := range shape {
		size *= v
	}
	data := make([]float32, size)
	success, err := C.MXPredGetOutput(s.handle,
		C.mx_uint(index),
		(*C.mx_float)(unsafe.Pointer(&data[0])),
		C.mx_uint(size),
	)
	if err != nil {
		return nil, err
	} else if success < 0 {
		return nil, GetLastError()
	}
	return data, nil
}

// free this predictor's C handle
// go binding for MXPredFree
func (s *Predictor) Free() error {
	success, err := C.MXPredFree(s.handle)
	if err != nil {
		return err
	} else if success < 0 {
		return GetLastError()
	}
	return nil
}
<commit_msg>add CreatePredictorPartial<commit_after>// Copyright 2016 go-mxnet-predictor Authors. All Rights Reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//      http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package mxnet

/*
// go preamble
#cgo pkg-config: mxnet
#include <mxnet/c_predict_api.h>
#include <stdlib.h>
*/
import "C"
import (
	"fmt"
	"unsafe"
)

const (
	CPU_DEVICE = iota + 1 // cpu device type
	GPU_DEVICE            // gpu device type
)

//TODO higher level api like context
type Device struct {
	Type int // device type
	Id   int // device id
}

type InputNode struct {
	Key   string   // name
	Shape []uint32 // shape of ndarray
}

// predictor for inference
type Predictor struct {
	handle C.PredictorHandle // C handle of predictor
}

// Create a Predictor
// go binding for MXPredCreate
// param symbol The JSON string of the symbol
// param params In-memory raw bytes of parameter ndarray file
// param device Device to run predictor
// param nodes An array of InputNode which stored the name and shape data of ndarray item
func CreatePredictor(symbol []byte,
	params []byte,
	device Device,
	nodes []InputNode,
) (*Predictor, error) {

	var (
		pc        *C.char
		shapeIdx  = []uint32{0}
		shapeData = []uint32{}
	)

	// malloc a **char which like []string to store node keys
	keys := C.malloc(C.size_t(len(nodes)) * C.size_t(unsafe.Sizeof(pc))) // c gc
	for i := 0; i < len(nodes); i++ {
		// get memory address
		p := (**C.char)(unsafe.Pointer(uintptr(keys) + uintptr(i)*unsafe.Sizeof(pc)))
		// c gc
		*p = C.CString(nodes[i].Key)

		// shapeIdx for next node
		shapeIdx = append(shapeIdx, uint32(len(nodes[i].Shape)))
		// shape data for current node
		shapeData = append(shapeData, nodes[i].Shape...)
	}

	var handle C.PredictorHandle

	success := C.MXPredCreate((*C.char)(unsafe.Pointer(&symbol[0])),
		unsafe.Pointer(&params[0]),
		C.int(len(params)),
		C.int(device.Type),
		C.int(device.Id),
		C.mx_uint(len(nodes)),
		(**C.char)(keys),
		(*C.mx_uint)(unsafe.Pointer(&shapeIdx[0])),
		(*C.mx_uint)(unsafe.Pointer(&shapeData[0])),
		&handle,
	)

	// free mem we created before return, go gc won't do that for us
	for i := 0; i < len(nodes); i++ {
		p := (**C.char)(unsafe.Pointer(uintptr(keys) + uintptr(i)*unsafe.Sizeof(pc)))
		C.free(unsafe.Pointer(*p))
	}
	C.free(unsafe.Pointer(keys))

	if success < 0 {
		return nil, GetLastError()
	}
	return &Predictor{handle: handle}, nil
}

// CreatePredictorPartial Creates a Predictor wich customized outputs [layer]
// go binding for MXPredCreate
// param symbol The JSON string of the symbol
// param params In-memory raw bytes of parameter ndarray file
// param device Device to run predictor
// param nodes An array of InputNode which stored the name and shape data of ndarray item
// param outputKey the name of the output layer/key
func CreatePredictorPartial(symbol []byte,
	params []byte,
	device Device,
	nodes []InputNode,
	outputKey string,
) (*Predictor, error) {

	var (
		pc        *C.char
		shapeIdx  = []uint32{0}
		shapeData = []uint32{}
	)

	// malloc a **char which like []string to store node keys
	keys := C.malloc(C.size_t(len(nodes)) * C.size_t(unsafe.Sizeof(pc))) // c gc
	for i := 0; i < len(nodes); i++ {
		// get memory address
		p := (**C.char)(unsafe.Pointer(uintptr(keys) + uintptr(i)*unsafe.Sizeof(pc)))
		// c gc
		*p = C.CString(nodes[i].Key)

		// shapeIdx for next node
		shapeIdx = append(shapeIdx, uint32(len(nodes[i].Shape)))
		// shape data for current node
		shapeData = append(shapeData, nodes[i].Shape...)
	}

	oKeys := C.malloc(C.size_t(len(nodes)) * C.size_t(unsafe.Sizeof(pc)))
	p := (**C.char)(unsafe.Pointer(uintptr(oKeys)))
	*p = C.CString(outputKey)

	var handle C.PredictorHandle

	success := C.MXPredCreatePartialOut((*C.char)(unsafe.Pointer(&symbol[0])),
		unsafe.Pointer(&params[0]),
		C.int(len(params)),
		C.int(device.Type),
		C.int(device.Id),
		C.mx_uint(len(nodes)),
		(**C.char)(keys),
		(*C.mx_uint)(unsafe.Pointer(&shapeIdx[0])),
		(*C.mx_uint)(unsafe.Pointer(&shapeData[0])),
		C.mx_uint(1),
		(**C.char)(oKeys),
		&handle,
	)

	// free mem we created before return, go gc won't do that for us
	for i := 0; i < len(nodes); i++ {
		p := (**C.char)(unsafe.Pointer(uintptr(keys) + uintptr(i)*unsafe.Sizeof(pc)))
		C.free(unsafe.Pointer(*p))
	}
	C.free(unsafe.Pointer(keys))

	if success < 0 {
		return nil, GetLastError()
	}
	return &Predictor{handle: handle}, nil
}

// set the input data of predictor
// go binding for MXPredSetInput
// param key The name of input node to set
// param data The float data to be set
func (s *Predictor) SetInput(key string, data []float32) error {
	// check input
	if data == nil || len(data) < 1 {
		return fmt.Errorf("intput data nil or empty")
	}

	// c gc
	k := C.CString(key)
	// free mem before return
	defer C.free(unsafe.Pointer(k))

	success, err := C.MXPredSetInput(s.handle,
		k,
		(*C.mx_float)(unsafe.Pointer(&data[0])),
		C.mx_uint(len(data)),
	)

	if err != nil {
		return err
	} else if success < 0 {
		return GetLastError()
	}
	return nil
}

// run a forward pass after SetInput
// go binding for MXPredForward
func (s *Predictor) Forward() error {
	success, err := C.MXPredForward(s.handle)
	if err != nil {
		return err
	} else if success < 0 {
		return GetLastError()
	}
	return nil
}

// get the shape of output node
// go binding for MXPredGetOutputShape
// param index The index of output node, set to 0 if there is only one output
func (s *Predictor) GetOutputShape(index uint32) ([]uint32, error) {
	var (
		shapeData *C.mx_uint
		shapeDim  C.mx_uint
	)
	success, err := C.MXPredGetOutputShape(s.handle,
		C.mx_uint(index),
		&shapeData,
		&shapeDim,
	)
	if err != nil {
		return nil, err
	} else if success < 0 {
		return nil, GetLastError()
	}
	// c array to go
	shape := (*[1 << 32]uint32)(unsafe.Pointer(shapeData))[:shapeDim:shapeDim]
	return shape, nil
}

// get the output value of prediction
// go binding for MXPredGetOutput
// param index The index of output node, set to 0 if there is only one output
func (s *Predictor) GetOutput(index uint32) ([]float32, error) {
	shape, err := s.GetOutputShape(index)
	if err != nil {
		return nil, err
	}
	size := uint32(1)
	for _, v := range shape {
		size *= v
	}
	data := make([]float32, size)
	success, err := C.MXPredGetOutput(s.handle,
		C.mx_uint(index),
		(*C.mx_float)(unsafe.Pointer(&data[0])),
		C.mx_uint(size),
	)
	if err != nil {
		return nil, err
	} else if success < 0 {
		return nil, GetLastError()
	}
	return data, nil
}

// free this predictor's C handle
// go binding for MXPredFree
func (s *Predictor) Free() error {
	success, err := C.MXPredFree(s.handle)
	if err != nil {
		return err
	} else if success < 0 {
		return GetLastError()
	}
	return nil
}
<|endoftext|>


<commit_before>package ec2

import (
	"fmt"

	"github.com/aws/aws-sdk-go/aws"
	"github.com/aws/aws-sdk-go/service/ec2"
	"github.com/spf13/cobra"
)

func Start(cmd *cobra.Command, args []string) {
	client := newEC2Client()

	fmt.Println(args)
	params := &ec2.StartInstancesInput{
		InstanceIds: aws.StringSlice(args),
	}

	response, err := client.StartInstances(params)
	if err != nil {
		panic(err)
	}

	fmt.Println(response)
}
<commit_msg>Remove debug print<commit_after>package ec2

import (
	"fmt"

	"github.com/aws/aws-sdk-go/aws"
	"github.com/aws/aws-sdk-go/service/ec2"
	"github.com/spf13/cobra"
)

func Start(cmd *cobra.Command, args []string) {
	client := newEC2Client()

	params := &ec2.StartInstancesInput{
		InstanceIds: aws.StringSlice(args),
	}

	response, err := client.StartInstances(params)
	if err != nil {
		panic(err)
	}

	fmt.Println(response)
}
<|endoftext|>


<commit_before>package main

import (
	"bufio"
	"encoding/json"
	"flag"
	"fmt"
	"github.com/APTrust/easy-store/bagit"
	"github.com/APTrust/easy-store/util"
	"github.com/APTrust/easy-store/util/fileutil"
	"io"
	"os"
	"path/filepath"
	"strings"
	"sync"
	"time"
)

func main() {
	job := loadJob()
	bagPath, err := createBag(job)
	if err != nil {
		fmt.Fprintln(os.Stderr, err.Error())
		os.Exit(1)
	} else {
		fmt.Println("Created", bagPath)
	}
}

func createBag(job *bagit.Job) (string, error) {
	bagPath := filepath.Join(job.BaggingDirectory, job.BagName)
	bagger, err := bagit.NewBagger(bagPath, job.BagItProfile)
	if err != nil {
		return "", err
	}

	// Add files
	// TODO: *** PRESERVE TIMESTAMPS ON COPY ***
	for _, fpath := range job.Files {
		if fileutil.IsFile(fpath) {
			addFile(bagger, job, fpath)
		} else if fileutil.IsDir(fpath) {
			err := filepath.Walk(fpath, func(filePath string, f os.FileInfo, err error) error {
				var e error
				if f != nil && f.Mode().IsRegular() {
					e = addFile(bagger, job, filePath)
				}
				return e
			})
			if err != nil {
				fmt.Fprintln(os.Stderr, err.Error())
				os.Exit(1)
			}
		}
	}

	// Add tags
	for _, tagDef := range job.BagItProfile.RequiredTags {
		kvp := bagit.NewKeyValuePair(tagDef.TagName, tagDef.UserValue)
		if tagDef.TagFile == "bag-info.txt" {
			if tagDef.TagName == "Bagging-Date" {
				kvp.Value = time.Now().Format("2006-01-02")
			} else if tagDef.TagName == "Payload-Oxum" {
				kvp.Value = bagger.GetPayloadOxum()
			}
		}
		bagger.AddTag(tagDef.TagFile, &kvp)
	}

	// Write bag
	bagger.WriteBag(true, true)

	errors := bagger.Errors()
	for _, errMsg := range errors {
		fmt.Fprintln(os.Stderr, errMsg)
	}
	if len(errors) > 0 {
		os.Exit(1)
	}

	// Tar the bag, if required
	// TODO: This will have to be more flexible in the future.
	// TODO: Clean this up
	// TODO: *** TAR WRITER MUST PRESERVE USER/GROUP ***
	validationPath := bagPath
	canTar := util.StringListContains(job.BagItProfile.AcceptSerialization, "application/tar")
	if canTar && job.BagItProfile.Serialization == "required" {
		algorithms := make([]string, 0)
		tarPath := bagPath + ".tar"
		validationPath = tarPath
		fmt.Println("Tarring bag to", tarPath)
		writer := fileutil.NewTarWriter(tarPath)
		writer.Open()
		defer writer.Close()

		var wg sync.WaitGroup

		err := filepath.Walk(bagPath, func(filePath string, f os.FileInfo, err error) error {
			wg.Add(1)
			var e error
			if f != nil && f.Mode().IsRegular() {
				relPath := strings.Replace(filePath, job.BaggingDirectory+"/", "", 1)
				_, e := writer.AddToArchive(filePath, relPath, algorithms)
				if e != nil {
					fmt.Fprintf(os.Stderr, e.Error())
				}
			}
			wg.Done()
			return e
		})

		wg.Wait()

		if err != nil {
			fmt.Fprintf(os.Stderr, err.Error())
		}
	}

	// Validate bag
	bag := bagit.NewBag(validationPath)
	validator := bagit.NewValidator(bag, job.BagItProfile)
	fmt.Println("Validating bag at", validationPath)
	if !validator.Validate() {
		fmt.Fprintln(os.Stderr, "Bag failed validation with the following errors:")
		for _, errMsg := range validator.Errors() {
			fmt.Fprintln(os.Stderr, errMsg)
		}
		os.Exit(1)
	} else {
		fmt.Println("Bag at", validationPath, "is valid")
	}

	// Delete the bag directory that we just tarred up
	if strings.HasSuffix(validationPath, ".tar") && validationPath != bagPath {
		if fileutil.LooksSafeToDelete(bagPath, 12, 3) {
			fmt.Println("Deleting bag directory", bagPath)
			fmt.Println("Bag is in", validationPath)
			os.RemoveAll(bagPath)
		}
	}

	return validationPath, nil
}

func addFile(bagger *bagit.Bagger, job *bagit.Job, sourcePath string) error {
	if job.ShouldIncludeFile(sourcePath) {
		relPath := "data" + sourcePath
		fmt.Println("Adding", sourcePath, "at", relPath)
		if !bagger.AddFile(sourcePath, relPath) {
			errors := bagger.Errors()
			lastError := errors[len(errors)-1]
			return fmt.Errorf(lastError)
		}
	} else {
		fmt.Println("Skipping", sourcePath)
	}
	return nil
}

func loadJob() *bagit.Job {
	var stdin bool
	flag.BoolVar(&stdin, "stdin", false, "Load job from stdin instead of reading from file.")
	flag.Parse()
	if stdin {
		job, err := loadJobFromStdin()
		if err != nil {
			fmt.Fprintln(os.Stderr, err.Error())
			os.Exit(1)
		}
		return job
	}

	if len(os.Args) < 2 {
		fmt.Fprintln(os.Stderr, "You must specify a job file, or pass the job JSON in through STDIN.")
		os.Exit(1)
	}
	job, err := bagit.LoadJobFromFile(os.Args[1])
	if err != nil {
		fmt.Fprintln(os.Stderr, err.Error())
		os.Exit(1)
	}
	errors := job.Validate()
	if len(errors) > 0 {
		for _, err := range errors {
			fmt.Fprintln(os.Stderr, err.Error())
		}
		os.Exit(1)
	}
	return job
}

func loadJobFromStdin() (*bagit.Job, error) {
	jsonBytes := make([]byte, 0)
	data := make([]byte, 4096)
	reader := bufio.NewReader(os.Stdin)
	for {
		bytesRead, err := reader.Read(data)
		jsonBytes = append(jsonBytes, data[0:bytesRead]...)
		if err != nil {
			if err == io.EOF {
				break
			}
			fmt.Fprintln(os.Stderr, err.Error())
			os.Exit(1)
		}
		if bytesRead < len(data) {
			break
		}
	}
	job := &bagit.Job{}
	err := json.Unmarshal(jsonBytes, job)
	return job, err
}
<commit_msg>Shortened file add message<commit_after>package main

import (
	"bufio"
	"encoding/json"
	"flag"
	"fmt"
	"github.com/APTrust/easy-store/bagit"
	"github.com/APTrust/easy-store/util"
	"github.com/APTrust/easy-store/util/fileutil"
	"io"
	"os"
	"path/filepath"
	"strings"
	"sync"
	"time"
)

func main() {
	job := loadJob()
	bagPath, err := createBag(job)
	if err != nil {
		fmt.Fprintln(os.Stderr, err.Error())
		os.Exit(1)
	} else {
		fmt.Println("Created", bagPath)
	}
}

func createBag(job *bagit.Job) (string, error) {
	bagPath := filepath.Join(job.BaggingDirectory, job.BagName)
	bagger, err := bagit.NewBagger(bagPath, job.BagItProfile)
	if err != nil {
		return "", err
	}

	// Add files
	// TODO: *** PRESERVE TIMESTAMPS ON COPY ***
	for _, fpath := range job.Files {
		if fileutil.IsFile(fpath) {
			addFile(bagger, job, fpath)
		} else if fileutil.IsDir(fpath) {
			err := filepath.Walk(fpath, func(filePath string, f os.FileInfo, err error) error {
				var e error
				if f != nil && f.Mode().IsRegular() {
					e = addFile(bagger, job, filePath)
				}
				return e
			})
			if err != nil {
				fmt.Fprintln(os.Stderr, err.Error())
				os.Exit(1)
			}
		}
	}

	// Add tags
	for _, tagDef := range job.BagItProfile.RequiredTags {
		kvp := bagit.NewKeyValuePair(tagDef.TagName, tagDef.UserValue)
		if tagDef.TagFile == "bag-info.txt" {
			if tagDef.TagName == "Bagging-Date" {
				kvp.Value = time.Now().Format("2006-01-02")
			} else if tagDef.TagName == "Payload-Oxum" {
				kvp.Value = bagger.GetPayloadOxum()
			}
		}
		bagger.AddTag(tagDef.TagFile, &kvp)
	}

	// Write bag
	bagger.WriteBag(true, true)

	errors := bagger.Errors()
	for _, errMsg := range errors {
		fmt.Fprintln(os.Stderr, errMsg)
	}
	if len(errors) > 0 {
		os.Exit(1)
	}

	// Tar the bag, if required
	// TODO: This will have to be more flexible in the future.
	// TODO: Clean this up
	// TODO: *** TAR WRITER MUST PRESERVE USER/GROUP ***
	validationPath := bagPath
	canTar := util.StringListContains(job.BagItProfile.AcceptSerialization, "application/tar")
	if canTar && job.BagItProfile.Serialization == "required" {
		algorithms := make([]string, 0)
		tarPath := bagPath + ".tar"
		validationPath = tarPath
		fmt.Println("Tarring bag to", tarPath)
		writer := fileutil.NewTarWriter(tarPath)
		writer.Open()
		defer writer.Close()

		var wg sync.WaitGroup

		err := filepath.Walk(bagPath, func(filePath string, f os.FileInfo, err error) error {
			wg.Add(1)
			var e error
			if f != nil && f.Mode().IsRegular() {
				relPath := strings.Replace(filePath, job.BaggingDirectory+"/", "", 1)
				_, e := writer.AddToArchive(filePath, relPath, algorithms)
				if e != nil {
					fmt.Fprintf(os.Stderr, e.Error())
				}
			}
			wg.Done()
			return e
		})

		wg.Wait()

		if err != nil {
			fmt.Fprintf(os.Stderr, err.Error())
		}
	}

	// Validate bag
	bag := bagit.NewBag(validationPath)
	validator := bagit.NewValidator(bag, job.BagItProfile)
	fmt.Println("Validating bag at", validationPath)
	if !validator.Validate() {
		fmt.Fprintln(os.Stderr, "Bag failed validation with the following errors:")
		for _, errMsg := range validator.Errors() {
			fmt.Fprintln(os.Stderr, errMsg)
		}
		os.Exit(1)
	} else {
		fmt.Println("Bag at", validationPath, "is valid")
	}

	// Delete the bag directory that we just tarred up
	if strings.HasSuffix(validationPath, ".tar") && validationPath != bagPath {
		if fileutil.LooksSafeToDelete(bagPath, 12, 3) {
			fmt.Println("Deleting bag directory", bagPath)
			fmt.Println("Bag is in", validationPath)
			os.RemoveAll(bagPath)
		}
	}

	return validationPath, nil
}

func addFile(bagger *bagit.Bagger, job *bagit.Job, sourcePath string) error {
	if job.ShouldIncludeFile(sourcePath) {
		relPath := "data" + sourcePath
		fmt.Println("Adding", sourcePath)
		if !bagger.AddFile(sourcePath, relPath) {
			errors := bagger.Errors()
			lastError := errors[len(errors)-1]
			return fmt.Errorf(lastError)
		}
	} else {
		fmt.Println("Skipping", sourcePath)
	}
	return nil
}

func loadJob() *bagit.Job {
	var stdin bool
	flag.BoolVar(&stdin, "stdin", false, "Load job from stdin instead of reading from file.")
	flag.Parse()
	if stdin {
		job, err := loadJobFromStdin()
		if err != nil {
			fmt.Fprintln(os.Stderr, err.Error())
			os.Exit(1)
		}
		return job
	}

	if len(os.Args) < 2 {
		fmt.Fprintln(os.Stderr, "You must specify a job file, or pass the job JSON in through STDIN.")
		os.Exit(1)
	}
	job, err := bagit.LoadJobFromFile(os.Args[1])
	if err != nil {
		fmt.Fprintln(os.Stderr, err.Error())
		os.Exit(1)
	}
	errors := job.Validate()
	if len(errors) > 0 {
		for _, err := range errors {
			fmt.Fprintln(os.Stderr, err.Error())
		}
		os.Exit(1)
	}
	return job
}

func loadJobFromStdin() (*bagit.Job, error) {
	jsonBytes := make([]byte, 0)
	data := make([]byte, 4096)
	reader := bufio.NewReader(os.Stdin)
	for {
		bytesRead, err := reader.Read(data)
		jsonBytes = append(jsonBytes, data[0:bytesRead]...)
		if err != nil {
			if err == io.EOF {
				break
			}
			fmt.Fprintln(os.Stderr, err.Error())
			os.Exit(1)
		}
		if bytesRead < len(data) {
			break
		}
	}
	job := &bagit.Job{}
	err := json.Unmarshal(jsonBytes, job)
	return job, err
}
<|endoftext|>