<commit_before>package zenodb

import (
	"bytes"
	"context"
	"errors"
	"fmt"
	"math"
	"strconv"
	"strings"
	"sync"
	"sync/atomic"
	"time"

	"github.com/getlantern/bytemap"
	"github.com/getlantern/mtime"
	"github.com/getlantern/zenodb/common"
	"github.com/getlantern/zenodb/core"
	"github.com/getlantern/zenodb/planner"
)

var (
	ErrMissingQueryHandler = errors.New("Missing query handler for partition")
)

func (db *DB) RegisterQueryHandler(partition int, query planner.QueryClusterFN) {
	db.tablesMutex.Lock()
	handlersCh := db.remoteQueryHandlers[partition]
	if handlersCh == nil {
		handlersCh = make(chan planner.QueryClusterFN, db.opts.ClusterQueryConcurrency)
	}
	db.remoteQueryHandlers[partition] = handlersCh
	db.tablesMutex.Unlock()
	handlersCh <- query
}

func (db *DB) remoteQueryHandlerForPartition(partition int) planner.QueryClusterFN {
	db.tablesMutex.RLock()
	defer db.tablesMutex.RUnlock()
	select {
	case handler := <-db.remoteQueryHandlers[partition]:
		return handler
	default:
		return nil
	}
}

func (db *DB) queryForRemote(ctx context.Context, sqlString string, isSubQuery bool, subQueryResults [][]interface{}, unflat bool, onFields core.OnFields, onRow core.OnRow, onFlatRow core.OnFlatRow) (result interface{}, err error) {
	source, prepareErr := db.Query(sqlString, isSubQuery, subQueryResults, common.ShouldIncludeMemStore(ctx))
	if prepareErr != nil {
		log.Errorf("Error on preparing query for remote: %v", prepareErr)
		return nil, prepareErr
	}
	elapsed := mtime.Stopwatch()
	defer func() {
		log.Debugf("Processed query in %v, error?: %v : %v", elapsed(), err, sqlString)
	}()
	if unflat {
		result, err = core.UnflattenOptimized(source).Iterate(ctx, onFields, onRow)
	} else {
		result, err = source.Iterate(ctx, onFields, onFlatRow)
	}
	return
}

type remoteResult struct {
	partition     int
	fields        core.Fields
	key           bytemap.ByteMap
	vals          core.Vals
	flatRow       *core.FlatRow
	totalRows     int
	elapsed       time.Duration
	highWaterMark int64
	err           error
}

func (db *DB) queryCluster(ctx context.Context, sqlString string, isSubQuery bool, subQueryResults [][]interface{}, includeMemStore bool, unflat bool, onFields core.OnFields, onRow core.OnRow, onFlatRow core.OnFlatRow) (interface{}, error) {
	ctx = common.WithIncludeMemStore(ctx, includeMemStore)
	numPartitions := db.opts.NumPartitions
	results := make(chan *remoteResult, numPartitions*100000) // TODO: make this tunable
	resultsByPartition := make(map[int]*int64)

	stats := &common.QueryStats{NumPartitions: numPartitions}
	missingPartitions := make(map[int]bool, numPartitions)
	var _finalErr error
	var finalMx sync.RWMutex

	finalStats := func() *common.QueryStats {
		finalMx.RLock()
		defer finalMx.RUnlock()
		mps := make([]string, 0, len(missingPartitions))
		for partition := range missingPartitions {
			mps = append(mps, strconv.Itoa(partition))
		}
		stats.MissingPartitions = strings.Join(mps, ",")
		return stats
	}

	finalErr := func() error {
		finalMx.RLock()
		defer finalMx.RUnlock()
		result := _finalErr
		return result
	}

	fail := func(partition int, err error) {
		finalMx.Lock()
		defer finalMx.Unlock()
		if _finalErr != nil {
			_finalErr = err
		}
		missingPartitions[partition] = true
	}

	finish := func(result *remoteResult) {
		finalMx.Lock()
		defer finalMx.Unlock()
		if result.err == nil {
			stats.NumSuccessfulPartitions++
			if stats.LowestHighWaterMark == 0 || stats.LowestHighWaterMark > result.highWaterMark {
				stats.LowestHighWaterMark = result.highWaterMark
			}
			if stats.HighestHighWaterMark < result.highWaterMark {
				stats.HighestHighWaterMark = result.highWaterMark
			}
		}
	}

	_stopped := int64(0)
	stopped := func() bool {
		return atomic.LoadInt64(&_stopped) == 1
	}
	stop := func() {
		atomic.StoreInt64(&_stopped, 1)
	}

	subCtx := ctx
	ctxDeadline, ctxHasDeadline := subCtx.Deadline()
	if ctxHasDeadline {
		// Halve timeout for sub-contexts
		now := time.Now()
		timeout := ctxDeadline.Sub(now)
		var cancel context.CancelFunc
		ctxDeadline = now.Add(timeout / 2)
		subCtx, cancel = context.WithDeadline(subCtx, ctxDeadline)
		defer cancel()
	}

	for i := 0; i < numPartitions; i++ {
		partition := i
		_resultsForPartition := int64(0)
		resultsForPartition := &_resultsForPartition
		resultsByPartition[partition] = resultsForPartition
		go func() {
			for attempt := 0; attempt < 10; attempt++ {
				elapsed := mtime.Stopwatch()
				query := db.remoteQueryHandlerForPartition(partition)
				if query == nil {
					log.Errorf("No query handler for partition %d, ignoring", partition)
					results <- &remoteResult{
						partition: partition,
						totalRows: 0,
						elapsed:   elapsed(),
						err:       ErrMissingQueryHandler,
					}
					break
				}

				var partOnRow func(key bytemap.ByteMap, vals core.Vals) (bool, error)
				var partOnFlatRow func(row *core.FlatRow) (bool, error)
				if unflat {
					partOnRow = func(key bytemap.ByteMap, vals core.Vals) (bool, error) {
						err := finalErr()
						if err != nil {
							return false, err
						}
						if stopped() {
							return false, nil
						}
						results <- &remoteResult{
							partition: partition,
							key:       key,
							vals:      vals,
						}
						atomic.AddInt64(resultsForPartition, 1)
						return true, nil
					}
				} else {
					partOnFlatRow = func(row *core.FlatRow) (bool, error) {
						err := finalErr()
						if err != nil {
							return false, err
						}
						if stopped() {
							return false, nil
						}
						results <- &remoteResult{
							partition: partition,
							flatRow:   row,
						}
						atomic.AddInt64(resultsForPartition, 1)
						return true, nil
					}
				}

				qstats, err := query(subCtx, sqlString, isSubQuery, subQueryResults, unflat, func(fields core.Fields) error {
					results <- &remoteResult{
						partition: partition,
						fields:    fields,
					}
					return nil
				}, partOnRow, partOnFlatRow)
				if err != nil {
					switch err.(type) {
					case common.Retriable:
						wait := time.Duration(math.Pow(1.5, float64(attempt))) * time.Second
						log.Debugf("Failed on partition %d but error is retriable, continuing after %v: %v", partition, wait, err)
						time.Sleep(wait)
						continue
					default:
						log.Debugf("Failed on partition %d and error is not retriable, will abort: %v", partition, err)
					}
				}
				var highWaterMark int64
				qs, ok := qstats.(*common.QueryStats)
				if ok && qs != nil {
					highWaterMark = qs.HighestHighWaterMark
				}
				results <- &remoteResult{
					partition:     partition,
					totalRows:     int(atomic.LoadInt64(resultsForPartition)),
					elapsed:       elapsed(),
					highWaterMark: highWaterMark,
					err:           err,
				}
				break
			}
		}()
	}

	start := time.Now()
	deadline := start.Add(db.opts.ClusterQueryTimeout)
	if ctxHasDeadline {
		deadline = ctxDeadline
	}
	log.Debugf("Deadline for results from partitions: %v (T - %v)", deadline, deadline.Sub(time.Now()))

	timeout := time.NewTimer(deadline.Sub(time.Now()))
	var canonicalFields core.Fields
	fieldsByPartition := make([]core.Fields, db.opts.NumPartitions)
	partitionRowMappers := make([]func(core.Vals) core.Vals, db.opts.NumPartitions)
	resultCount := 0
	for pendingPartitions := numPartitions; pendingPartitions > 0; {
		select {
		case result := <-results:
			// first handle fields
			partitionFields := result.fields
			if partitionFields != nil {
				if canonicalFields == nil {
					log.Debugf("fields: %v", partitionFields)
					err := onFields(partitionFields)
					if err != nil {
						fail(result.partition, err)
					}
					canonicalFields = partitionFields
				}

				// Each partition can theoretically have different field definitions.
				// To accomodate this, we track the fields separate for each partition
				// and convert into the canonical form before sending onward.
				fieldsByPartition[result.partition] = partitionFields
				partitionRowMappers[result.partition] = partitionRowMapper(canonicalFields, partitionFields)
				continue
			}

			// handle unflat rows
			if result.key != nil {
				if stopped() || finalErr() != nil {
					continue
				}
				more, err := onRow(result.key, partitionRowMappers[result.partition](result.vals))
				if err == nil && !more {
					fail(result.partition, err)
					stop()
				}
				continue
			}

			// handle flat rows
			flatRow := result.flatRow
			if flatRow != nil {
				if stopped() || finalErr() != nil {
					continue
				}
				flatRow.SetFields(fieldsByPartition[result.partition])
				more, err := onFlatRow(flatRow)
				if err != nil {
					fail(result.partition, err)
					return finalStats(), err
				} else if !more {
					stop()
				}
				continue
			}

			// final results for partition
			resultCount++
			pendingPartitions--
			if result.err != nil {
				log.Errorf("Error from partition %d: %v", result.partition, result.err)
				fail(result.partition, result.err)
			}
			finish(result)
			log.Debugf("%d/%d got %d results from partition %d in %v", resultCount, db.opts.NumPartitions, result.totalRows, result.partition, result.elapsed)
			delete(resultsByPartition, result.partition)
		case <-timeout.C:
			log.Errorf("Failed to get results by deadline, %d of %d partitions reporting", resultCount, numPartitions)
			msg := bytes.NewBuffer([]byte("Missing partitions: "))
			first := true
			for partition, results := range resultsByPartition {
				if !first {
					msg.WriteString(" | ")
				}
				first = false
				msg.WriteString(fmt.Sprintf("%d (%d)", partition, results))
				fail(partition, core.ErrDeadlineExceeded)
			}
			log.Debug(msg.String())
			return finalStats(), finalErr()
		}
	}

	return finalStats(), finalErr()
}

func partitionRowMapper(canonicalFields core.Fields, partitionFields core.Fields) func(core.Vals) core.Vals {
	if canonicalFields.Equals(partitionFields) {
		return func(vals core.Vals) core.Vals { return vals }
	}

	scratch := make(core.Vals, len(canonicalFields))
	idxs := make([]int, 0, len(canonicalFields))
	for _, canonicalField := range canonicalFields {
		i := -1
		for _i, partitionField := range partitionFields {
			if canonicalField.Equals(partitionField) {
				i = _i
				break
			}
		}
		idxs = append(idxs, i)
	}

	return func(vals core.Vals) core.Vals {
		for o, i := range idxs {
			if i < 0 || i >= len(vals) {
				scratch[o] = nil
			} else {
				scratch[o] = vals[i]
			}
		}

		for i := range vals {
			vals[i] = scratch[i]
		}

		return vals
	}
}
<commit_msg>Took out exponential backoff on retries in querying cluster<commit_after>package zenodb

import (
	"bytes"
	"context"
	"errors"
	"fmt"
	"strconv"
	"strings"
	"sync"
	"sync/atomic"
	"time"

	"github.com/getlantern/bytemap"
	"github.com/getlantern/mtime"
	"github.com/getlantern/zenodb/common"
	"github.com/getlantern/zenodb/core"
	"github.com/getlantern/zenodb/planner"
)

var (
	ErrMissingQueryHandler = errors.New("Missing query handler for partition")
)

func (db *DB) RegisterQueryHandler(partition int, query planner.QueryClusterFN) {
	db.tablesMutex.Lock()
	handlersCh := db.remoteQueryHandlers[partition]
	if handlersCh == nil {
		handlersCh = make(chan planner.QueryClusterFN, db.opts.ClusterQueryConcurrency)
	}
	db.remoteQueryHandlers[partition] = handlersCh
	db.tablesMutex.Unlock()
	handlersCh <- query
}

func (db *DB) remoteQueryHandlerForPartition(partition int) planner.QueryClusterFN {
	db.tablesMutex.RLock()
	defer db.tablesMutex.RUnlock()
	select {
	case handler := <-db.remoteQueryHandlers[partition]:
		return handler
	default:
		return nil
	}
}

func (db *DB) queryForRemote(ctx context.Context, sqlString string, isSubQuery bool, subQueryResults [][]interface{}, unflat bool, onFields core.OnFields, onRow core.OnRow, onFlatRow core.OnFlatRow) (result interface{}, err error) {
	source, prepareErr := db.Query(sqlString, isSubQuery, subQueryResults, common.ShouldIncludeMemStore(ctx))
	if prepareErr != nil {
		log.Errorf("Error on preparing query for remote: %v", prepareErr)
		return nil, prepareErr
	}
	elapsed := mtime.Stopwatch()
	defer func() {
		log.Debugf("Processed query in %v, error?: %v : %v", elapsed(), err, sqlString)
	}()
	if unflat {
		result, err = core.UnflattenOptimized(source).Iterate(ctx, onFields, onRow)
	} else {
		result, err = source.Iterate(ctx, onFields, onFlatRow)
	}
	return
}

type remoteResult struct {
	partition     int
	fields        core.Fields
	key           bytemap.ByteMap
	vals          core.Vals
	flatRow       *core.FlatRow
	totalRows     int
	elapsed       time.Duration
	highWaterMark int64
	err           error
}

func (db *DB) queryCluster(ctx context.Context, sqlString string, isSubQuery bool, subQueryResults [][]interface{}, includeMemStore bool, unflat bool, onFields core.OnFields, onRow core.OnRow, onFlatRow core.OnFlatRow) (interface{}, error) {
	ctx = common.WithIncludeMemStore(ctx, includeMemStore)
	numPartitions := db.opts.NumPartitions
	results := make(chan *remoteResult, numPartitions*100000) // TODO: make this tunable
	resultsByPartition := make(map[int]*int64)

	stats := &common.QueryStats{NumPartitions: numPartitions}
	missingPartitions := make(map[int]bool, numPartitions)
	var _finalErr error
	var finalMx sync.RWMutex

	finalStats := func() *common.QueryStats {
		finalMx.RLock()
		defer finalMx.RUnlock()
		mps := make([]string, 0, len(missingPartitions))
		for partition := range missingPartitions {
			mps = append(mps, strconv.Itoa(partition))
		}
		stats.MissingPartitions = strings.Join(mps, ",")
		return stats
	}

	finalErr := func() error {
		finalMx.RLock()
		defer finalMx.RUnlock()
		result := _finalErr
		return result
	}

	fail := func(partition int, err error) {
		finalMx.Lock()
		defer finalMx.Unlock()
		if _finalErr != nil {
			_finalErr = err
		}
		missingPartitions[partition] = true
	}

	finish := func(result *remoteResult) {
		finalMx.Lock()
		defer finalMx.Unlock()
		if result.err == nil {
			stats.NumSuccessfulPartitions++
			if stats.LowestHighWaterMark == 0 || stats.LowestHighWaterMark > result.highWaterMark {
				stats.LowestHighWaterMark = result.highWaterMark
			}
			if stats.HighestHighWaterMark < result.highWaterMark {
				stats.HighestHighWaterMark = result.highWaterMark
			}
		}
	}

	_stopped := int64(0)
	stopped := func() bool {
		return atomic.LoadInt64(&_stopped) == 1
	}
	stop := func() {
		atomic.StoreInt64(&_stopped, 1)
	}

	subCtx := ctx
	ctxDeadline, ctxHasDeadline := subCtx.Deadline()
	if ctxHasDeadline {
		// Halve timeout for sub-contexts
		now := time.Now()
		timeout := ctxDeadline.Sub(now)
		var cancel context.CancelFunc
		ctxDeadline = now.Add(timeout / 2)
		subCtx, cancel = context.WithDeadline(subCtx, ctxDeadline)
		defer cancel()
	}

	for i := 0; i < numPartitions; i++ {
		partition := i
		_resultsForPartition := int64(0)
		resultsForPartition := &_resultsForPartition
		resultsByPartition[partition] = resultsForPartition
		go func() {
			for {
				elapsed := mtime.Stopwatch()
				query := db.remoteQueryHandlerForPartition(partition)
				if query == nil {
					log.Errorf("No query handler for partition %d, ignoring", partition)
					results <- &remoteResult{
						partition: partition,
						totalRows: 0,
						elapsed:   elapsed(),
						err:       ErrMissingQueryHandler,
					}
					break
				}

				var partOnRow func(key bytemap.ByteMap, vals core.Vals) (bool, error)
				var partOnFlatRow func(row *core.FlatRow) (bool, error)
				if unflat {
					partOnRow = func(key bytemap.ByteMap, vals core.Vals) (bool, error) {
						err := finalErr()
						if err != nil {
							return false, err
						}
						if stopped() {
							return false, nil
						}
						results <- &remoteResult{
							partition: partition,
							key:       key,
							vals:      vals,
						}
						atomic.AddInt64(resultsForPartition, 1)
						return true, nil
					}
				} else {
					partOnFlatRow = func(row *core.FlatRow) (bool, error) {
						err := finalErr()
						if err != nil {
							return false, err
						}
						if stopped() {
							return false, nil
						}
						results <- &remoteResult{
							partition: partition,
							flatRow:   row,
						}
						atomic.AddInt64(resultsForPartition, 1)
						return true, nil
					}
				}

				qstats, err := query(subCtx, sqlString, isSubQuery, subQueryResults, unflat, func(fields core.Fields) error {
					results <- &remoteResult{
						partition: partition,
						fields:    fields,
					}
					return nil
				}, partOnRow, partOnFlatRow)
				if err != nil {
					switch err.(type) {
					case common.Retriable:
						log.Debugf("Failed on partition %d but error is retriable, continuing: %v", partition, err)
						continue
					default:
						log.Debugf("Failed on partition %d and error is not retriable, will abort: %v", partition, err)
					}
				}
				var highWaterMark int64
				qs, ok := qstats.(*common.QueryStats)
				if ok && qs != nil {
					highWaterMark = qs.HighestHighWaterMark
				}
				results <- &remoteResult{
					partition:     partition,
					totalRows:     int(atomic.LoadInt64(resultsForPartition)),
					elapsed:       elapsed(),
					highWaterMark: highWaterMark,
					err:           err,
				}
				break
			}
		}()
	}

	start := time.Now()
	deadline := start.Add(db.opts.ClusterQueryTimeout)
	if ctxHasDeadline {
		deadline = ctxDeadline
	}
	log.Debugf("Deadline for results from partitions: %v (T - %v)", deadline, deadline.Sub(time.Now()))

	timeout := time.NewTimer(deadline.Sub(time.Now()))
	var canonicalFields core.Fields
	fieldsByPartition := make([]core.Fields, db.opts.NumPartitions)
	partitionRowMappers := make([]func(core.Vals) core.Vals, db.opts.NumPartitions)
	resultCount := 0
	for pendingPartitions := numPartitions; pendingPartitions > 0; {
		select {
		case result := <-results:
			// first handle fields
			partitionFields := result.fields
			if partitionFields != nil {
				if canonicalFields == nil {
					log.Debugf("fields: %v", partitionFields)
					err := onFields(partitionFields)
					if err != nil {
						fail(result.partition, err)
					}
					canonicalFields = partitionFields
				}

				// Each partition can theoretically have different field definitions.
				// To accomodate this, we track the fields separate for each partition
				// and convert into the canonical form before sending onward.
				fieldsByPartition[result.partition] = partitionFields
				partitionRowMappers[result.partition] = partitionRowMapper(canonicalFields, partitionFields)
				continue
			}

			// handle unflat rows
			if result.key != nil {
				if stopped() || finalErr() != nil {
					continue
				}
				more, err := onRow(result.key, partitionRowMappers[result.partition](result.vals))
				if err == nil && !more {
					fail(result.partition, err)
					stop()
				}
				continue
			}

			// handle flat rows
			flatRow := result.flatRow
			if flatRow != nil {
				if stopped() || finalErr() != nil {
					continue
				}
				flatRow.SetFields(fieldsByPartition[result.partition])
				more, err := onFlatRow(flatRow)
				if err != nil {
					fail(result.partition, err)
					return finalStats(), err
				} else if !more {
					stop()
				}
				continue
			}

			// final results for partition
			resultCount++
			pendingPartitions--
			if result.err != nil {
				log.Errorf("Error from partition %d: %v", result.partition, result.err)
				fail(result.partition, result.err)
			}
			finish(result)
			log.Debugf("%d/%d got %d results from partition %d in %v", resultCount, db.opts.NumPartitions, result.totalRows, result.partition, result.elapsed)
			delete(resultsByPartition, result.partition)
		case <-timeout.C:
			log.Errorf("Failed to get results by deadline, %d of %d partitions reporting", resultCount, numPartitions)
			msg := bytes.NewBuffer([]byte("Missing partitions: "))
			first := true
			for partition, results := range resultsByPartition {
				if !first {
					msg.WriteString(" | ")
				}
				first = false
				msg.WriteString(fmt.Sprintf("%d (%d)", partition, results))
				fail(partition, core.ErrDeadlineExceeded)
			}
			log.Debug(msg.String())
			return finalStats(), finalErr()
		}
	}

	return finalStats(), finalErr()
}

func partitionRowMapper(canonicalFields core.Fields, partitionFields core.Fields) func(core.Vals) core.Vals {
	if canonicalFields.Equals(partitionFields) {
		return func(vals core.Vals) core.Vals { return vals }
	}

	scratch := make(core.Vals, len(canonicalFields))
	idxs := make([]int, 0, len(canonicalFields))
	for _, canonicalField := range canonicalFields {
		i := -1
		for _i, partitionField := range partitionFields {
			if canonicalField.Equals(partitionField) {
				i = _i
				break
			}
		}
		idxs = append(idxs, i)
	}

	return func(vals core.Vals) core.Vals {
		for o, i := range idxs {
			if i < 0 || i >= len(vals) {
				scratch[o] = nil
			} else {
				scratch[o] = vals[i]
			}
		}

		for i := range vals {
			vals[i] = scratch[i]
		}

		return vals
	}
}
<|endoftext|>


<commit_before>package main

import (
	"encoding/json"
	"errors"
	"fmt"
	"log"
	"os"
	"path"
	"time"

	"github.com/PonyvilleFM/aura/bot"
	"github.com/PonyvilleFM/aura/recording"
	"github.com/bwmarrin/discordgo"
	_ "github.com/joho/godotenv/autoload"
)

type aura struct {
	cs *bot.CommandSet
	s  *discordgo.Session

	guildRecordings map[string]*recording.Recording
	state           *state
}

type state struct {
	DownloadURLs map[string]string // Guild ID -> URL
	PermRoles    map[string]string // Guild ID -> needed role ID
}

func (s *state) Save() error {
	fout, err := os.Create(path.Join(dataPrefix, "state.json"))
	if err != nil {
		return err
	}
	defer fout.Close()

	return json.NewEncoder(fout).Encode(s)
}

func (s *state) Load() error {
	fin, err := os.Open(path.Join(dataPrefix, "state.json"))
	if err != nil {
		return err
	}
	defer fin.Close()

	return json.NewDecoder(fin).Decode(s)
}

const (
	djonHelp  = ``
	djoffHelp = ``
	setupHelp = ``
)

func (a *aura) Permissons(s *discordgo.Session, m *discordgo.Message, parv []string) error {
	ch, err := s.Channel(m.ChannelID)
	if err != nil {
		return err
	}

	gid := ch.GuildID
	role := a.state.PermRoles[gid]

	gu, err := s.GuildMember(gid, m.Author.ID)
	if err != nil {
		return err
	}

	found := false
	for _, r := range gu.Roles {
		if r == role {
			found = true
			break
		}
	}

	if !found {
		return errors.New("aura: no permissions")
	}

	return nil
}

func (a *aura) roles(s *discordgo.Session, m *discordgo.Message, parv []string) error {
	log.Println("got here")
	ch, err := s.Channel(m.ChannelID)
	if err != nil {
		return err
	}

	gid := ch.GuildID

	result := "Roles in this group:\n"

	roles, err := s.GuildRoles(gid)
	if err != nil {
		return err
	}

	for _, r := range roles {
		result += fmt.Sprintf("- %s: %s\n", r.ID, r.Name)
	}

	s.ChannelMessageSend(m.ChannelID, result)
	return nil
}

func (a *aura) setup(s *discordgo.Session, m *discordgo.Message, parv []string) error {
	if len(parv) != 3 {
		return errors.New("aura: wrong number of params for setup")
	}

	role := parv[1]
	url := parv[2]

	ch, err := s.Channel(m.ChannelID)
	if err != nil {
		return err
	}

	gid := ch.GuildID

	roles, err := s.GuildRoles(gid)
	if err != nil {
		return err
	}

	found := false
	for _, r := range roles {
		if r.ID == role {
			found = true
			break
		}
	}

	if !found {
		return errors.New("aura: Role not found")
	}

	a.state.PermRoles[gid] = role
	a.state.DownloadURLs[gid] = url

	s.ChannelMessageSend(m.ChannelID, fmt.Sprintf("Guild %s set up for recording url %s controlled by role %s", gid, url, role))

	a.state.Save()
	return nil
}

func (a *aura) djon(s *discordgo.Session, m *discordgo.Message, parv []string) error {
	fname := fmt.Sprintf("%s - %s.mp3", m.Author.Username, time.Now().Format(time.ANSIC))

	ch, err := s.Channel(m.ChannelID)
	if err != nil {
		return err
	}

	gid := ch.GuildID

	_, ok := a.guildRecordings[gid]
	if ok {
		return errors.New("aura: another recording is already in progress")
	}

	r, err := recording.New(a.state.DownloadURLs[gid], path.Join(dataPrefix, fname))
	if err != nil {
		return err
	}

	a.guildRecordings[gid] = r
	go r.Start()

	s.ChannelMessageSend(m.ChannelID, fmt.Sprintf("Now recording: %s", fname))

	return nil
}

func (a *aura) djoff(s *discordgo.Session, m *discordgo.Message, parv []string) error {
	ch, err := s.Channel(m.ChannelID)
	if err != nil {
		return err
	}

	gid := ch.GuildID

	r, ok := a.guildRecordings[gid]
	if !ok {
		return errors.New("aura: no recording is currently in progress")
	}

	r.Cancel()
	<-r.Done()

	s.ChannelMessageSend(m.ChannelID, fmt.Sprintf("Recording complete: %s", r.OutputFilename()))

	return nil
}

func (a *aura) Handle(s *discordgo.Session, m *discordgo.MessageCreate) {
	err := a.cs.Run(s, m.Message)
	if err != nil {
		log.Println(err)
	}
}

var (
	token      = os.Getenv("TOKEN")
	dataPrefix = os.Getenv("DATA_PREFIX")
)

func main() {
	dg, err := discordgo.New("Bot " + token)
	if err != nil {
		log.Fatal(err)
	}

	a := &aura{
		cs:              bot.NewCommandSet(),
		s:               dg,
		guildRecordings: map[string]*recording.Recording{},

		state: &state{
			DownloadURLs: map[string]string{},
			PermRoles:    map[string]string{},
		},
	}

	err = a.state.Load()
	if err != nil {
		log.Println(err)
	}

	a.cs.Add(bot.NewBasicCommand("roles", "", bot.NoPermissions, a.roles))
	a.cs.Add(bot.NewBasicCommand("setup", setupHelp, bot.NoPermissions, a.setup))
	a.cs.Add(bot.NewBasicCommand("djon", djonHelp, a.Permissons, a.djon))
	a.cs.Add(bot.NewBasicCommand("djoff", djoffHelp, a.Permissons, a.djoff))

	dg.AddHandler(a.Handle)
	dg.AddHandler(messageCreate)

	err = dg.Open()
	if err != nil {
		log.Fatal(err)
	}

	log.Println("ready")
	<-make(chan struct{})
}

// This function will be called (due to AddHandler above) every time a new
// message is created on any channel that the autenticated bot has access to.
func messageCreate(s *discordgo.Session, m *discordgo.MessageCreate) {
	// Print message to stdout.
	fmt.Printf("%20s %20s %20s > %s\n", m.ChannelID, time.Now().Format(time.Stamp), m.Author.Username, m.Content)
}
<commit_msg>cmd/aura: url-escape the filename for download announcements<commit_after>package main

import (
	"encoding/json"
	"errors"
	"fmt"
	"log"
	"net/url"
	"os"
	"path"
	"strings"
	"time"

	"github.com/PonyvilleFM/aura/bot"
	"github.com/PonyvilleFM/aura/recording"
	"github.com/bwmarrin/discordgo"
	_ "github.com/joho/godotenv/autoload"
)

type aura struct {
	cs *bot.CommandSet
	s  *discordgo.Session

	guildRecordings map[string]*recording.Recording
	state           *state
}

type state struct {
	DownloadURLs map[string]string // Guild ID -> URL
	PermRoles    map[string]string // Guild ID -> needed role ID
}

func (s *state) Save() error {
	fout, err := os.Create(path.Join(dataPrefix, "state.json"))
	if err != nil {
		return err
	}
	defer fout.Close()

	return json.NewEncoder(fout).Encode(s)
}

func (s *state) Load() error {
	fin, err := os.Open(path.Join(dataPrefix, "state.json"))
	if err != nil {
		return err
	}
	defer fin.Close()

	return json.NewDecoder(fin).Decode(s)
}

const (
	djonHelp  = ``
	djoffHelp = ``
	setupHelp = ``
)

func (a *aura) Permissons(s *discordgo.Session, m *discordgo.Message, parv []string) error {
	ch, err := s.Channel(m.ChannelID)
	if err != nil {
		return err
	}

	gid := ch.GuildID
	role := a.state.PermRoles[gid]

	gu, err := s.GuildMember(gid, m.Author.ID)
	if err != nil {
		return err
	}

	found := false
	for _, r := range gu.Roles {
		if r == role {
			found = true
			break
		}
	}

	if !found {
		return errors.New("aura: no permissions")
	}

	return nil
}

func (a *aura) roles(s *discordgo.Session, m *discordgo.Message, parv []string) error {
	log.Println("got here")
	ch, err := s.Channel(m.ChannelID)
	if err != nil {
		return err
	}

	gid := ch.GuildID

	result := "Roles in this group:\n"

	roles, err := s.GuildRoles(gid)
	if err != nil {
		return err
	}

	for _, r := range roles {
		result += fmt.Sprintf("- %s: %s\n", r.ID, r.Name)
	}

	s.ChannelMessageSend(m.ChannelID, result)
	return nil
}

func (a *aura) setup(s *discordgo.Session, m *discordgo.Message, parv []string) error {
	if len(parv) != 3 {
		return errors.New("aura: wrong number of params for setup")
	}

	role := parv[1]
	url := parv[2]

	ch, err := s.Channel(m.ChannelID)
	if err != nil {
		return err
	}

	gid := ch.GuildID

	roles, err := s.GuildRoles(gid)
	if err != nil {
		return err
	}

	found := false
	for _, r := range roles {
		if r.ID == role {
			found = true
			break
		}
	}

	if !found {
		return errors.New("aura: Role not found")
	}

	a.state.PermRoles[gid] = role
	a.state.DownloadURLs[gid] = url

	s.ChannelMessageSend(m.ChannelID, fmt.Sprintf("Guild %s set up for recording url %s controlled by role %s", gid, url, role))

	a.state.Save()
	return nil
}

func (a *aura) djon(s *discordgo.Session, m *discordgo.Message, parv []string) error {
	fname := fmt.Sprintf("%s - %s.mp3", m.Author.Username, time.Now().Format(time.ANSIC))

	ch, err := s.Channel(m.ChannelID)
	if err != nil {
		return err
	}

	gid := ch.GuildID

	_, ok := a.guildRecordings[gid]
	if ok {
		return errors.New("aura: another recording is already in progress")
	}

	os.Mkdir(path.Join(dataPrefix, gid), 0775)

	r, err := recording.New(a.state.DownloadURLs[gid], path.Join(dataPrefix, gid, fname))
	if err != nil {
		return err
	}

	a.guildRecordings[gid] = r
	go r.Start()

	s.ChannelMessageSend(m.ChannelID, fmt.Sprintf("Now recording: %s", fname))

	return nil
}

func (a *aura) djoff(s *discordgo.Session, m *discordgo.Message, parv []string) error {
	ch, err := s.Channel(m.ChannelID)
	if err != nil {
		return err
	}

	gid := ch.GuildID

	r, ok := a.guildRecordings[gid]
	if !ok {
		return errors.New("aura: no recording is currently in progress")
	}

	r.Cancel()
	<-r.Done()

	fname := r.OutputFilename()
	parts := strings.Split(fname, "/")

	s.ChannelMessageSend(m.ChannelID, fmt.Sprintf("Recording complete: https://pvfmrecordings.greedo.xeserv.us/var/%s/%s", parts[1], url.QueryEscape(parts[2])))

	return nil
}

func (a *aura) Handle(s *discordgo.Session, m *discordgo.MessageCreate) {
	err := a.cs.Run(s, m.Message)
	if err != nil {
		log.Println(err)
	}
}

var (
	token      = os.Getenv("TOKEN")
	dataPrefix = os.Getenv("DATA_PREFIX")
)

func main() {
	dg, err := discordgo.New("Bot " + token)
	if err != nil {
		log.Fatal(err)
	}

	a := &aura{
		cs:              bot.NewCommandSet(),
		s:               dg,
		guildRecordings: map[string]*recording.Recording{},

		state: &state{
			DownloadURLs: map[string]string{},
			PermRoles:    map[string]string{},
		},
	}

	err = a.state.Load()
	if err != nil {
		log.Println(err)
	}

	a.cs.Add(bot.NewBasicCommand("roles", "", bot.NoPermissions, a.roles))
	a.cs.Add(bot.NewBasicCommand("setup", setupHelp, bot.NoPermissions, a.setup))
	a.cs.Add(bot.NewBasicCommand("djon", djonHelp, a.Permissons, a.djon))
	a.cs.Add(bot.NewBasicCommand("djoff", djoffHelp, a.Permissons, a.djoff))

	dg.AddHandler(a.Handle)
	dg.AddHandler(messageCreate)

	err = dg.Open()
	if err != nil {
		log.Fatal(err)
	}

	log.Println("ready")
	<-make(chan struct{})
}

// This function will be called (due to AddHandler above) every time a new
// message is created on any channel that the autenticated bot has access to.
func messageCreate(s *discordgo.Session, m *discordgo.MessageCreate) {
	// Print message to stdout.
	fmt.Printf("%20s %20s %20s > %s\n", m.ChannelID, time.Now().Format(time.Stamp), m.Author.Username, m.Content)
}
<|endoftext|>


<commit_before>// Copyright 2013 The Go Authors.  All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

// This is a blog server for articles written in present format.
// It powers blog.golang.org.
package main

import (
	"bytes"
	"encoding/json"
	"encoding/xml"
	"fmt"
	"html/template"
	"log"
	"net/http"
	"os"
	"path/filepath"
	"regexp"
	"sort"
	"time"

	"code.google.com/p/go.blog/pkg/atom"
	_ "code.google.com/p/go.talks/pkg/playground"
	"code.google.com/p/go.talks/pkg/present"
)

const (
	hostname     = "blog.golang.org"
	baseURL      = "http://" + hostname
	homeArticles = 5  // articles to display on the home page
	feedArticles = 10 // articles to include in Atom and JSON feeds
)

var validJSONPFunc = regexp.MustCompile(`(?i)^[a-z_][a-z0-9_.]*$`)

// Doc represents an article, adorned with presentation data:
// its absolute path and related articles.
type Doc struct {
	*present.Doc
	Permalink    string
	Path         string
	Related      []*Doc
	Newer, Older *Doc
	HTML         template.HTML // rendered article
}

// Server implements an http.Handler that serves blog articles.
type Server struct {
	docs     []*Doc
	tags     []string
	docPaths map[string]*Doc
	docTags  map[string][]*Doc
	template struct {
		home, index, article, doc *template.Template
	}
	atomFeed []byte // pre-rendered Atom feed
	jsonFeed []byte // pre-rendered JSON feed
	content  http.Handler
}

// NewServer constructs a new Server, serving articles from the specified
// contentPath generated from templates from templatePath.
func NewServer(contentPath, templatePath string) (*Server, error) {
	present.PlayEnabled = true

	root := filepath.Join(templatePath, "root.tmpl")
	parse := func(name string) (*template.Template, error) {
		t := template.New("").Funcs(funcMap)
		return t.ParseFiles(root, filepath.Join(templatePath, name))
	}

	s := &Server{}

	// Parse templates.
	var err error
	s.template.home, err = parse("home.tmpl")
	if err != nil {
		return nil, err
	}
	s.template.index, err = parse("index.tmpl")
	if err != nil {
		return nil, err
	}
	s.template.article, err = parse("article.tmpl")
	if err != nil {
		return nil, err
	}
	p := present.Template().Funcs(funcMap)
	s.template.doc, err = p.ParseFiles(filepath.Join(templatePath, "doc.tmpl"))
	if err != nil {
		return nil, err
	}

	// Load content.
	err = s.loadDocs(filepath.Clean(contentPath))
	if err != nil {
		return nil, err
	}

	err = s.renderAtomFeed()
	if err != nil {
		return nil, err
	}

	err = s.renderJSONFeed()
	if err != nil {
		return nil, err
	}

	// Set up content file server.
	s.content = http.FileServer(http.Dir(contentPath))

	return s, nil
}

var funcMap = template.FuncMap{
	"sectioned": sectioned,
	"authors":   authors,
}

// sectioned returns true if the provided Doc contains more than one section.
// This is used to control whether to display the table of contents and headings.
func sectioned(d *present.Doc) bool {
	return len(d.Sections) > 1
}

// authors returns a comma-separated list of author names.
func authors(authors []present.Author) string {
	var b bytes.Buffer
	last := len(authors) - 1
	for i, a := range authors {
		if i > 0 {
			if i == last {
				b.WriteString(" and ")
			} else {
				b.WriteString(", ")
			}
		}
		b.WriteString(authorName(a))
	}
	return b.String()
}

// authorName returns the first line of the Author text: the author's name.
func authorName(a present.Author) string {
	el := a.TextElem()
	if len(el) == 0 {
		return ""
	}
	text, ok := el[0].(present.Text)
	if !ok || len(text.Lines) == 0 {
		return ""
	}
	return text.Lines[0]
}

// loadDocs reads all content from the provided file system root, renders all
// the articles it finds, adds them to the Server's docs field, computes the
// denormalized docPaths, docTags, and tags fields, and populates the various
// helper fields (Next, Previous, Related) for each Doc.
func (s *Server) loadDocs(root string) error {
	// Read content into docs field.
	const ext = ".article"
	fn := func(p string, info os.FileInfo, err error) error {
		if filepath.Ext(p) != ext {
			return nil
		}
		f, err := os.Open(p)
		if err != nil {
			return err
		}
		defer f.Close()
		d, err := present.Parse(f, p, 0)
		if err != nil {
			return err
		}
		html := new(bytes.Buffer)
		err = d.Render(html, s.template.doc)
		if err != nil {
			return err
		}
		p = p[len(root) : len(p)-len(ext)] // trim root and extension
		s.docs = append(s.docs, &Doc{
			Doc:       d,
			Path:      p,
			Permalink: baseURL + p,
			HTML:      template.HTML(html.String()),
		})
		return nil
	}
	err := filepath.Walk(root, fn)
	if err != nil {
		return err
	}
	sort.Sort(docsByTime(s.docs))

	// Pull out doc paths and tags and put in reverse-associating maps.
	s.docPaths = make(map[string]*Doc)
	s.docTags = make(map[string][]*Doc)
	for _, d := range s.docs {
		s.docPaths[d.Path] = d
		for _, t := range d.Tags {
			s.docTags[t] = append(s.docTags[t], d)
		}
	}

	// Pull out unique sorted list of tags.
	for t := range s.docTags {
		s.tags = append(s.tags, t)
	}
	sort.Strings(s.tags)

	// Set up presentation-related fields, Newer, Older, and Related.
	for _, doc := range s.docs {
		// Newer, Older: docs adjacent to doc
		for i := range s.docs {
			if s.docs[i] != doc {
				continue
			}
			if i > 0 {
				doc.Newer = s.docs[i-1]
			}
			if i+1 < len(s.docs) {
				doc.Older = s.docs[i+1]
			}
			break
		}

		// Related: all docs that share tags with doc.
		related := make(map[*Doc]bool)
		for _, t := range doc.Tags {
			for _, d := range s.docTags[t] {
				if d != doc {
					related[d] = true
				}
			}
		}
		for d := range related {
			doc.Related = append(doc.Related, d)
		}
		sort.Sort(docsByTime(doc.Related))
	}

	return nil
}

// renderAtomFeed generates an XML Atom feed and stores it in the Server's
// atomFeed field.
func (s *Server) renderAtomFeed() error {
	var updated time.Time
	if len(s.docs) > 1 {
		updated = s.docs[0].Time
	}
	feed := atom.Feed{
		Title:   "The Go Programming Language Blog",
		ID:      "tag:" + hostname + ",2013:" + hostname,
		Updated: atom.Time(updated),
		Link: []atom.Link{{
			Rel:  "self",
			Href: baseURL + "/feed.atom",
		}},
	}
	for i, doc := range s.docs {
		if i >= feedArticles {
			break
		}
		e := &atom.Entry{
			Title: doc.Title,
			ID:    feed.ID + doc.Path,
			Link: []atom.Link{{
				Rel:  "alternate",
				Href: baseURL + doc.Path,
			}},
			Published: atom.Time(doc.Time),
			Updated:   atom.Time(doc.Time),
			Summary: &atom.Text{
				Type: "html",
				Body: summary(doc),
			},
			Content: &atom.Text{
				Type: "html",
				Body: string(doc.HTML),
			},
			Author: &atom.Person{
				Name: authors(doc.Authors),
			},
		}
		feed.Entry = append(feed.Entry, e)
	}
	data, err := xml.Marshal(&feed)
	if err != nil {
		return err
	}
	s.atomFeed = data
	return nil
}

type jsonItem struct {
	Title   string
	Link    string
	Time    time.Time
	Summary string
	Content string
	Author  string
}

// renderJSONFeed generates a JSON feed and stores it in the Server's jsonFeed
// field.
func (s *Server) renderJSONFeed() error {
	var feed []jsonItem
	for i, doc := range s.docs {
		if i >= feedArticles {
			break
		}
		item := jsonItem{
			Title:   doc.Title,
			Link:    baseURL + doc.Path,
			Time:    doc.Time,
			Summary: summary(doc),
			Content: string(doc.HTML),
			Author:  authors(doc.Authors),
		}
		feed = append(feed, item)
	}
	data, err := json.Marshal(feed)
	if err != nil {
		return err
	}
	s.jsonFeed = data
	return nil
}

// summary returns the first paragraph of text from the provided Doc.
func summary(d *Doc) string {
	if len(d.Sections) == 0 {
		return ""
	}
	for _, elem := range d.Sections[0].Elem {
		text, ok := elem.(present.Text)
		if !ok || text.Pre {
			// skip everything but non-text elements
			continue
		}
		var buf bytes.Buffer
		for _, s := range text.Lines {
			buf.WriteString(string(present.Style(s)))
			buf.WriteByte('\n')
		}
		return buf.String()
	}
	return ""
}

// rootData encapsulates data destined for the root template.
type rootData struct {
	Doc  *Doc
	Data interface{}
}

// ServeHTTP servers either an article list or a single article.
func (s *Server) ServeHTTP(w http.ResponseWriter, r *http.Request) {
	var (
		d rootData
		t *template.Template
	)
	switch p := r.URL.Path; p {
	case "/":
		d.Data = s.docs
		if len(s.docs) > homeArticles {
			d.Data = s.docs[:homeArticles]
		}
		t = s.template.home
	case "/index":
		d.Data = s.docs
		t = s.template.index
	case "/feed.atom", "/feeds/posts/default":
		w.Header().Set("Content-type", "application/atom+xml; charset=utf-8")
		w.Write(s.atomFeed)
		return
	case "/.json":
		if p := r.FormValue("jsonp"); validJSONPFunc.MatchString(p) {
			w.Header().Set("Content-type", "application/javascript; charset=utf-8")
			fmt.Fprintf(w, "%v(%s)", p, s.jsonFeed)
			return
		}
		w.Header().Set("Content-type", "application/json")
		w.Write(s.jsonFeed)
		return
	default:
		doc, ok := s.docPaths[p]
		if !ok {
			// Not a doc; try to just serve static content.
			s.content.ServeHTTP(w, r)
			return
		}
		d.Doc = doc
		t = s.template.article
	}
	err := t.ExecuteTemplate(w, "root", d)
	if err != nil {
		log.Println(err)
	}
}

// docsByTime implements sort.Interface, sorting Docs by their Time field.
type docsByTime []*Doc

func (s docsByTime) Len() int           { return len(s) }
func (s docsByTime) Swap(i, j int)      { s[i], s[j] = s[j], s[i] }
func (s docsByTime) Less(i, j int) bool { return s[i].Time.After(s[j].Time) }
<commit_msg>go.blog/cmd/blog: fix JSON content-type<commit_after>// Copyright 2013 The Go Authors.  All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

// This is a blog server for articles written in present format.
// It powers blog.golang.org.
package main

import (
	"bytes"
	"encoding/json"
	"encoding/xml"
	"fmt"
	"html/template"
	"log"
	"net/http"
	"os"
	"path/filepath"
	"regexp"
	"sort"
	"time"

	"code.google.com/p/go.blog/pkg/atom"
	_ "code.google.com/p/go.talks/pkg/playground"
	"code.google.com/p/go.talks/pkg/present"
)

const (
	hostname     = "blog.golang.org"
	baseURL      = "http://" + hostname
	homeArticles = 5  // articles to display on the home page
	feedArticles = 10 // articles to include in Atom and JSON feeds
)

var validJSONPFunc = regexp.MustCompile(`(?i)^[a-z_][a-z0-9_.]*$`)

// Doc represents an article, adorned with presentation data:
// its absolute path and related articles.
type Doc struct {
	*present.Doc
	Permalink    string
	Path         string
	Related      []*Doc
	Newer, Older *Doc
	HTML         template.HTML // rendered article
}

// Server implements an http.Handler that serves blog articles.
type Server struct {
	docs     []*Doc
	tags     []string
	docPaths map[string]*Doc
	docTags  map[string][]*Doc
	template struct {
		home, index, article, doc *template.Template
	}
	atomFeed []byte // pre-rendered Atom feed
	jsonFeed []byte // pre-rendered JSON feed
	content  http.Handler
}

// NewServer constructs a new Server, serving articles from the specified
// contentPath generated from templates from templatePath.
func NewServer(contentPath, templatePath string) (*Server, error) {
	present.PlayEnabled = true

	root := filepath.Join(templatePath, "root.tmpl")
	parse := func(name string) (*template.Template, error) {
		t := template.New("").Funcs(funcMap)
		return t.ParseFiles(root, filepath.Join(templatePath, name))
	}

	s := &Server{}

	// Parse templates.
	var err error
	s.template.home, err = parse("home.tmpl")
	if err != nil {
		return nil, err
	}
	s.template.index, err = parse("index.tmpl")
	if err != nil {
		return nil, err
	}
	s.template.article, err = parse("article.tmpl")
	if err != nil {
		return nil, err
	}
	p := present.Template().Funcs(funcMap)
	s.template.doc, err = p.ParseFiles(filepath.Join(templatePath, "doc.tmpl"))
	if err != nil {
		return nil, err
	}

	// Load content.
	err = s.loadDocs(filepath.Clean(contentPath))
	if err != nil {
		return nil, err
	}

	err = s.renderAtomFeed()
	if err != nil {
		return nil, err
	}

	err = s.renderJSONFeed()
	if err != nil {
		return nil, err
	}

	// Set up content file server.
	s.content = http.FileServer(http.Dir(contentPath))

	return s, nil
}

var funcMap = template.FuncMap{
	"sectioned": sectioned,
	"authors":   authors,
}

// sectioned returns true if the provided Doc contains more than one section.
// This is used to control whether to display the table of contents and headings.
func sectioned(d *present.Doc) bool {
	return len(d.Sections) > 1
}

// authors returns a comma-separated list of author names.
func authors(authors []present.Author) string {
	var b bytes.Buffer
	last := len(authors) - 1
	for i, a := range authors {
		if i > 0 {
			if i == last {
				b.WriteString(" and ")
			} else {
				b.WriteString(", ")
			}
		}
		b.WriteString(authorName(a))
	}
	return b.String()
}

// authorName returns the first line of the Author text: the author's name.
func authorName(a present.Author) string {
	el := a.TextElem()
	if len(el) == 0 {
		return ""
	}
	text, ok := el[0].(present.Text)
	if !ok || len(text.Lines) == 0 {
		return ""
	}
	return text.Lines[0]
}

// loadDocs reads all content from the provided file system root, renders all
// the articles it finds, adds them to the Server's docs field, computes the
// denormalized docPaths, docTags, and tags fields, and populates the various
// helper fields (Next, Previous, Related) for each Doc.
func (s *Server) loadDocs(root string) error {
	// Read content into docs field.
	const ext = ".article"
	fn := func(p string, info os.FileInfo, err error) error {
		if filepath.Ext(p) != ext {
			return nil
		}
		f, err := os.Open(p)
		if err != nil {
			return err
		}
		defer f.Close()
		d, err := present.Parse(f, p, 0)
		if err != nil {
			return err
		}
		html := new(bytes.Buffer)
		err = d.Render(html, s.template.doc)
		if err != nil {
			return err
		}
		p = p[len(root) : len(p)-len(ext)] // trim root and extension
		s.docs = append(s.docs, &Doc{
			Doc:       d,
			Path:      p,
			Permalink: baseURL + p,
			HTML:      template.HTML(html.String()),
		})
		return nil
	}
	err := filepath.Walk(root, fn)
	if err != nil {
		return err
	}
	sort.Sort(docsByTime(s.docs))

	// Pull out doc paths and tags and put in reverse-associating maps.
	s.docPaths = make(map[string]*Doc)
	s.docTags = make(map[string][]*Doc)
	for _, d := range s.docs {
		s.docPaths[d.Path] = d
		for _, t := range d.Tags {
			s.docTags[t] = append(s.docTags[t], d)
		}
	}

	// Pull out unique sorted list of tags.
	for t := range s.docTags {
		s.tags = append(s.tags, t)
	}
	sort.Strings(s.tags)

	// Set up presentation-related fields, Newer, Older, and Related.
	for _, doc := range s.docs {
		// Newer, Older: docs adjacent to doc
		for i := range s.docs {
			if s.docs[i] != doc {
				continue
			}
			if i > 0 {
				doc.Newer = s.docs[i-1]
			}
			if i+1 < len(s.docs) {
				doc.Older = s.docs[i+1]
			}
			break
		}

		// Related: all docs that share tags with doc.
		related := make(map[*Doc]bool)
		for _, t := range doc.Tags {
			for _, d := range s.docTags[t] {
				if d != doc {
					related[d] = true
				}
			}
		}
		for d := range related {
			doc.Related = append(doc.Related, d)
		}
		sort.Sort(docsByTime(doc.Related))
	}

	return nil
}

// renderAtomFeed generates an XML Atom feed and stores it in the Server's
// atomFeed field.
func (s *Server) renderAtomFeed() error {
	var updated time.Time
	if len(s.docs) > 1 {
		updated = s.docs[0].Time
	}
	feed := atom.Feed{
		Title:   "The Go Programming Language Blog",
		ID:      "tag:" + hostname + ",2013:" + hostname,
		Updated: atom.Time(updated),
		Link: []atom.Link{{
			Rel:  "self",
			Href: baseURL + "/feed.atom",
		}},
	}
	for i, doc := range s.docs {
		if i >= feedArticles {
			break
		}
		e := &atom.Entry{
			Title: doc.Title,
			ID:    feed.ID + doc.Path,
			Link: []atom.Link{{
				Rel:  "alternate",
				Href: baseURL + doc.Path,
			}},
			Published: atom.Time(doc.Time),
			Updated:   atom.Time(doc.Time),
			Summary: &atom.Text{
				Type: "html",
				Body: summary(doc),
			},
			Content: &atom.Text{
				Type: "html",
				Body: string(doc.HTML),
			},
			Author: &atom.Person{
				Name: authors(doc.Authors),
			},
		}
		feed.Entry = append(feed.Entry, e)
	}
	data, err := xml.Marshal(&feed)
	if err != nil {
		return err
	}
	s.atomFeed = data
	return nil
}

type jsonItem struct {
	Title   string
	Link    string
	Time    time.Time
	Summary string
	Content string
	Author  string
}

// renderJSONFeed generates a JSON feed and stores it in the Server's jsonFeed
// field.
func (s *Server) renderJSONFeed() error {
	var feed []jsonItem
	for i, doc := range s.docs {
		if i >= feedArticles {
			break
		}
		item := jsonItem{
			Title:   doc.Title,
			Link:    baseURL + doc.Path,
			Time:    doc.Time,
			Summary: summary(doc),
			Content: string(doc.HTML),
			Author:  authors(doc.Authors),
		}
		feed = append(feed, item)
	}
	data, err := json.Marshal(feed)
	if err != nil {
		return err
	}
	s.jsonFeed = data
	return nil
}

// summary returns the first paragraph of text from the provided Doc.
func summary(d *Doc) string {
	if len(d.Sections) == 0 {
		return ""
	}
	for _, elem := range d.Sections[0].Elem {
		text, ok := elem.(present.Text)
		if !ok || text.Pre {
			// skip everything but non-text elements
			continue
		}
		var buf bytes.Buffer
		for _, s := range text.Lines {
			buf.WriteString(string(present.Style(s)))
			buf.WriteByte('\n')
		}
		return buf.String()
	}
	return ""
}

// rootData encapsulates data destined for the root template.
type rootData struct {
	Doc  *Doc
	Data interface{}
}

// ServeHTTP servers either an article list or a single article.
func (s *Server) ServeHTTP(w http.ResponseWriter, r *http.Request) {
	var (
		d rootData
		t *template.Template
	)
	switch p := r.URL.Path; p {
	case "/":
		d.Data = s.docs
		if len(s.docs) > homeArticles {
			d.Data = s.docs[:homeArticles]
		}
		t = s.template.home
	case "/index":
		d.Data = s.docs
		t = s.template.index
	case "/feed.atom", "/feeds/posts/default":
		w.Header().Set("Content-type", "application/atom+xml; charset=utf-8")
		w.Write(s.atomFeed)
		return
	case "/.json":
		if p := r.FormValue("jsonp"); validJSONPFunc.MatchString(p) {
			w.Header().Set("Content-type", "application/javascript; charset=utf-8")
			fmt.Fprintf(w, "%v(%s)", p, s.jsonFeed)
			return
		}
		w.Header().Set("Content-type", "application/json; charset=utf-8")
		w.Write(s.jsonFeed)
		return
	default:
		doc, ok := s.docPaths[p]
		if !ok {
			// Not a doc; try to just serve static content.
			s.content.ServeHTTP(w, r)
			return
		}
		d.Doc = doc
		t = s.template.article
	}
	err := t.ExecuteTemplate(w, "root", d)
	if err != nil {
		log.Println(err)
	}
}

// docsByTime implements sort.Interface, sorting Docs by their Time field.
type docsByTime []*Doc

func (s docsByTime) Len() int           { return len(s) }
func (s docsByTime) Swap(i, j int)      { s[i], s[j] = s[j], s[i] }
func (s docsByTime) Less(i, j int) bool { return s[i].Time.After(s[j].Time) }
<|endoftext|>


<commit_before>// Copyright 2017 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

package main

import (
	"bytes"
	"flag"
	"io/ioutil"
	"log"
	"os"
	"path/filepath"
	"sort"
	"strings"

	"github.com/golang/dep"
	"github.com/golang/dep/gps"
	"github.com/golang/dep/gps/pkgtree"
	"github.com/golang/dep/internal/fs"
	"github.com/pkg/errors"
)

const pruneShortHelp = `Pruning is now performed automatically by dep ensure.`
const pruneLongHelp = `
Prune was merged into the ensure command.
Set prune options in the manifest and it will be applied after every ensure.
dep prune will be removed in a future version of dep, causing this command to exit non-0.
`

type pruneCommand struct {
}

func (cmd *pruneCommand) Name() string      { return "prune" }
func (cmd *pruneCommand) Args() string      { return "" }
func (cmd *pruneCommand) ShortHelp() string { return pruneShortHelp }
func (cmd *pruneCommand) LongHelp() string  { return pruneLongHelp }
func (cmd *pruneCommand) Hidden() bool      { return false }

func (cmd *pruneCommand) Register(fs *flag.FlagSet) {
}

func (cmd *pruneCommand) Run(ctx *dep.Ctx, args []string) error {
	ctx.Out.Printf("Pruning is now performed automatically by dep ensure.\n")
	ctx.Out.Printf("Set prune settings in %s and it it will be applied when running ensure.\n", dep.ManifestName)
	ctx.Out.Printf("\ndep prune will be removed in a future version, and this command will exit non-0.\nPlease update your scripts.\n")

	p, err := ctx.LoadProject()
	if err != nil {
		return err
	}

	sm, err := ctx.SourceManager()
	if err != nil {
		return err
	}
	sm.UseDefaultSignalHandling()
	defer sm.Release()

	// While the network churns on ListVersions() requests, statically analyze
	// code from the current project.
	ptree, err := pkgtree.ListPackages(p.ResolvedAbsRoot, string(p.ImportRoot))
	if err != nil {
		return errors.Wrap(err, "analysis of local packages failed: %v")
	}

	// Set up a solver in order to check the InputHash.
	params := p.MakeParams()
	params.RootPackageTree = ptree

	if ctx.Verbose {
		params.TraceLogger = ctx.Err
	}

	s, err := gps.Prepare(params, sm)
	if err != nil {
		return errors.Wrap(err, "could not set up solver for input hashing")
	}

	if p.Lock == nil {
		return errors.Errorf("Gopkg.lock must exist for prune to know what files are safe to remove.")
	}

	if !bytes.Equal(s.HashInputs(), p.Lock.SolveMeta.InputsDigest) {
		return errors.Errorf("Gopkg.lock is out of sync; run dep ensure before pruning.")
	}

	pruneLogger := ctx.Err
	if !ctx.Verbose {
		pruneLogger = log.New(ioutil.Discard, "", 0)
	}
	return pruneProject(p, sm, pruneLogger)
}

// pruneProject removes unused packages from a project.
func pruneProject(p *dep.Project, sm gps.SourceManager, logger *log.Logger) error {
	td, err := ioutil.TempDir(os.TempDir(), "dep")
	if err != nil {
		return errors.Wrap(err, "error while creating temp dir for writing manifest/lock/vendor")
	}
	defer os.RemoveAll(td)

	if err := gps.WriteDepTree(td, p.Lock, sm, gps.CascadingPruneOptions{DefaultOptions: gps.PruneNestedVendorDirs}, logger); err != nil {
		return err
	}

	var toKeep []string
	for _, project := range p.Lock.Projects() {
		projectRoot := string(project.Ident().ProjectRoot)
		for _, pkg := range project.Packages() {
			toKeep = append(toKeep, filepath.Join(projectRoot, pkg))
		}
	}

	toDelete, err := calculatePrune(td, toKeep, logger)
	if err != nil {
		return err
	}

	if len(toDelete) > 0 {
		logger.Println("Calculated the following directories to prune:")
		for _, d := range toDelete {
			logger.Printf("  %s\n", d)
		}
	} else {
		logger.Println("No directories found to prune")
	}

	if err := deleteDirs(toDelete); err != nil {
		return err
	}

	vpath := filepath.Join(p.AbsRoot, "vendor")
	vendorbak := vpath + ".orig"
	var failerr error
	if _, err := os.Stat(vpath); err == nil {
		// Move out the old vendor dir. just do it into an adjacent dir, to
		// try to mitigate the possibility of a pointless cross-filesystem
		// move with a temp directory.
		if _, err := os.Stat(vendorbak); err == nil {
			// If the adjacent dir already exists, bite the bullet and move
			// to a proper tempdir.
			vendorbak = filepath.Join(td, "vendor.orig")
		}
		failerr = fs.RenameWithFallback(vpath, vendorbak)
		if failerr != nil {
			goto fail
		}
	}

	// Move in the new one.
	failerr = fs.RenameWithFallback(td, vpath)
	if failerr != nil {
		goto fail
	}

	os.RemoveAll(vendorbak)

	return nil

fail:
	fs.RenameWithFallback(vendorbak, vpath)
	return failerr
}

func calculatePrune(vendorDir string, keep []string, logger *log.Logger) ([]string, error) {
	logger.Println("Calculating prune. Checking the following packages:")
	sort.Strings(keep)
	toDelete := []string{}
	err := filepath.Walk(vendorDir, func(path string, info os.FileInfo, err error) error {
		if _, err := os.Lstat(path); err != nil {
			return nil
		}
		if !info.IsDir() {
			return nil
		}
		if path == vendorDir {
			return nil
		}

		name := strings.TrimPrefix(path, vendorDir+string(filepath.Separator))
		logger.Printf("  %s", name)
		i := sort.Search(len(keep), func(i int) bool {
			return name <= keep[i]
		})
		if i >= len(keep) || !strings.HasPrefix(keep[i], name) {
			toDelete = append(toDelete, path)
		}
		return nil
	})
	return toDelete, err
}

func deleteDirs(toDelete []string) error {
	// sort by length so we delete sub dirs first
	sort.Sort(byLen(toDelete))
	for _, path := range toDelete {
		if err := os.RemoveAll(path); err != nil {
			return err
		}
	}
	return nil
}

type byLen []string

func (a byLen) Len() int           { return len(a) }
func (a byLen) Swap(i, j int)      { a[i], a[j] = a[j], a[i] }
func (a byLen) Less(i, j int) bool { return len(a[i]) > len(a[j]) }
<commit_msg>dep: Sharpen `dep prune` warning message<commit_after>// Copyright 2017 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

package main

import (
	"bytes"
	"flag"
	"io/ioutil"
	"log"
	"os"
	"path/filepath"
	"sort"
	"strings"

	"github.com/golang/dep"
	"github.com/golang/dep/gps"
	"github.com/golang/dep/gps/pkgtree"
	"github.com/golang/dep/internal/fs"
	"github.com/pkg/errors"
)

const pruneShortHelp = `Pruning is now performed automatically by dep ensure.`
const pruneLongHelp = `
Prune was merged into the ensure command.
Set prune options in the manifest and it will be applied after every ensure.
dep prune will be removed in a future version of dep, causing this command to exit non-0.
`

type pruneCommand struct {
}

func (cmd *pruneCommand) Name() string      { return "prune" }
func (cmd *pruneCommand) Args() string      { return "" }
func (cmd *pruneCommand) ShortHelp() string { return pruneShortHelp }
func (cmd *pruneCommand) LongHelp() string  { return pruneLongHelp }
func (cmd *pruneCommand) Hidden() bool      { return false }

func (cmd *pruneCommand) Register(fs *flag.FlagSet) {
}

func (cmd *pruneCommand) Run(ctx *dep.Ctx, args []string) error {
	ctx.Err.Printf("Pruning is now performed automatically by dep ensure.\n")
	ctx.Err.Printf("Set prune settings in %s and it it will be applied when running ensure.\n", dep.ManifestName)
	ctx.Err.Printf("\nThis command currently still prunes as it always has, to ease the transition.\n")
	ctx.Err.Printf("However, it will be removed in a future version of dep.\n")
	ctx.Err.Printf("\nNow is the time to update your Gopkg.toml and remove `dep prune` from any scripts.\n")

	p, err := ctx.LoadProject()
	if err != nil {
		return err
	}

	sm, err := ctx.SourceManager()
	if err != nil {
		return err
	}
	sm.UseDefaultSignalHandling()
	defer sm.Release()

	// While the network churns on ListVersions() requests, statically analyze
	// code from the current project.
	ptree, err := pkgtree.ListPackages(p.ResolvedAbsRoot, string(p.ImportRoot))
	if err != nil {
		return errors.Wrap(err, "analysis of local packages failed: %v")
	}

	// Set up a solver in order to check the InputHash.
	params := p.MakeParams()
	params.RootPackageTree = ptree

	if ctx.Verbose {
		params.TraceLogger = ctx.Err
	}

	s, err := gps.Prepare(params, sm)
	if err != nil {
		return errors.Wrap(err, "could not set up solver for input hashing")
	}

	if p.Lock == nil {
		return errors.Errorf("Gopkg.lock must exist for prune to know what files are safe to remove.")
	}

	if !bytes.Equal(s.HashInputs(), p.Lock.SolveMeta.InputsDigest) {
		return errors.Errorf("Gopkg.lock is out of sync; run dep ensure before pruning.")
	}

	pruneLogger := ctx.Err
	if !ctx.Verbose {
		pruneLogger = log.New(ioutil.Discard, "", 0)
	}
	return pruneProject(p, sm, pruneLogger)
}

// pruneProject removes unused packages from a project.
func pruneProject(p *dep.Project, sm gps.SourceManager, logger *log.Logger) error {
	td, err := ioutil.TempDir(os.TempDir(), "dep")
	if err != nil {
		return errors.Wrap(err, "error while creating temp dir for writing manifest/lock/vendor")
	}
	defer os.RemoveAll(td)

	if err := gps.WriteDepTree(td, p.Lock, sm, gps.CascadingPruneOptions{DefaultOptions: gps.PruneNestedVendorDirs}, logger); err != nil {
		return err
	}

	var toKeep []string
	for _, project := range p.Lock.Projects() {
		projectRoot := string(project.Ident().ProjectRoot)
		for _, pkg := range project.Packages() {
			toKeep = append(toKeep, filepath.Join(projectRoot, pkg))
		}
	}

	toDelete, err := calculatePrune(td, toKeep, logger)
	if err != nil {
		return err
	}

	if len(toDelete) > 0 {
		logger.Println("Calculated the following directories to prune:")
		for _, d := range toDelete {
			logger.Printf("  %s\n", d)
		}
	} else {
		logger.Println("No directories found to prune")
	}

	if err := deleteDirs(toDelete); err != nil {
		return err
	}

	vpath := filepath.Join(p.AbsRoot, "vendor")
	vendorbak := vpath + ".orig"
	var failerr error
	if _, err := os.Stat(vpath); err == nil {
		// Move out the old vendor dir. just do it into an adjacent dir, to
		// try to mitigate the possibility of a pointless cross-filesystem
		// move with a temp directory.
		if _, err := os.Stat(vendorbak); err == nil {
			// If the adjacent dir already exists, bite the bullet and move
			// to a proper tempdir.
			vendorbak = filepath.Join(td, "vendor.orig")
		}
		failerr = fs.RenameWithFallback(vpath, vendorbak)
		if failerr != nil {
			goto fail
		}
	}

	// Move in the new one.
	failerr = fs.RenameWithFallback(td, vpath)
	if failerr != nil {
		goto fail
	}

	os.RemoveAll(vendorbak)

	return nil

fail:
	fs.RenameWithFallback(vendorbak, vpath)
	return failerr
}

func calculatePrune(vendorDir string, keep []string, logger *log.Logger) ([]string, error) {
	logger.Println("Calculating prune. Checking the following packages:")
	sort.Strings(keep)
	toDelete := []string{}
	err := filepath.Walk(vendorDir, func(path string, info os.FileInfo, err error) error {
		if _, err := os.Lstat(path); err != nil {
			return nil
		}
		if !info.IsDir() {
			return nil
		}
		if path == vendorDir {
			return nil
		}

		name := strings.TrimPrefix(path, vendorDir+string(filepath.Separator))
		logger.Printf("  %s", name)
		i := sort.Search(len(keep), func(i int) bool {
			return name <= keep[i]
		})
		if i >= len(keep) || !strings.HasPrefix(keep[i], name) {
			toDelete = append(toDelete, path)
		}
		return nil
	})
	return toDelete, err
}

func deleteDirs(toDelete []string) error {
	// sort by length so we delete sub dirs first
	sort.Sort(byLen(toDelete))
	for _, path := range toDelete {
		if err := os.RemoveAll(path); err != nil {
			return err
		}
	}
	return nil
}

type byLen []string

func (a byLen) Len() int           { return len(a) }
func (a byLen) Swap(i, j int)      { a[i], a[j] = a[j], a[i] }
func (a byLen) Less(i, j int) bool { return len(a[i]) > len(a[j]) }
<|endoftext|>


<commit_before>package main

import (
	"flag"
	"os"
	"sync"
	"time"

	"github.com/domainr/dnsr"
	"github.com/miekg/dns"
	"github.com/wsxiaoys/terminal/color"
	"golang.org/x/net/idna"
)

const (
	timeout = 2000 * time.Millisecond
)

var (
	verbose  bool
	resolver = dnsr.New(10000)
)

func init() {
	flag.BoolVar(
		&verbose,
		"v",
		false,
		"print verbose info to the console",
	)
}

func logV(fmt string, args ...interface{}) {
	if !verbose {
		return
	}
	color.Printf(fmt, args...)
}

func main() {
	flag.Usage = func() {
		color.Fprintf(os.Stderr, "Usage: %s [arguments] <name> [type]\n\nAvailable arguments:\n", os.Args[0])
		flag.PrintDefaults()
		os.Exit(1)
	}
	flag.Parse()
	qtype := ""
	args := flag.Args()
	if len(args) == 0 {
		flag.Usage()
	} else if _, isType := dns.StringToType[args[len(args)-1]]; len(args) > 1 && isType {
		qtype, args = args[len(args)-1], args[:len(args)-1]
	}
	if verbose {
		dnsr.DebugLogger = os.Stderr
	}
	var wg sync.WaitGroup
	start := time.Now()
	for _, name := range args {
		wg.Add(1)
		go func(name string, qtype string) {
			query(name, qtype)
			wg.Done()
		}(name, qtype)
	}
	wg.Wait()
	logV("\n@{w};; Total elapsed: %s\n", time.Since(start).String())
}

func query(name, qtype string) {
	start := time.Now()
	qname, err := idna.ToASCII(name)
	if err != nil {
		color.Fprintf(os.Stderr, "Invalid IDN domain name: %s\n", name)
		os.Exit(1)
	}

	rrs := resolver.Resolve(qname, qtype)

	color.Printf("\n")
	if len(rrs) > 0 {
		color.Printf("@{g};; RESULTS:\n")
	}
	for _, rr := range rrs {
		color.Printf("@{g}%s\n", rr.String())
	}

	if rrs == nil {
		color.Printf("@{y};; NIL\t%s\t%s\n", name, qtype)
	} else if len(rrs) > 0 {
		color.Printf("@{g};; TRUE\t%s\t%s\n", name, qtype)
	} else {
		color.Printf("@{r};; FALSE\t%s\t%s\n", name, qtype)
	}

	color.Printf("@{.w};; Elapsed: %s\n", time.Since(start).String())
}
<commit_msg>cmd/dnsr: Use ResolveErr and log total time elapsed.<commit_after>package main

import (
	"flag"
	"os"
	"sync"
	"time"

	"github.com/domainr/dnsr"
	"github.com/miekg/dns"
	"github.com/wsxiaoys/terminal/color"
	"golang.org/x/net/idna"
)

var (
	verbose  bool
	resolver = dnsr.New(10000)
)

func init() {
	flag.BoolVar(
		&verbose,
		"v",
		false,
		"print verbose info to the console",
	)
}

func logV(fmt string, args ...interface{}) {
	if !verbose {
		return
	}
	color.Printf(fmt, args...)
}

func main() {
	flag.Usage = func() {
		color.Fprintf(os.Stderr, "Usage: %s [arguments] <name> [type]\n\nAvailable arguments:\n", os.Args[0])
		flag.PrintDefaults()
		os.Exit(1)
	}
	flag.Parse()
	qtype := ""
	args := flag.Args()
	if len(args) == 0 {
		flag.Usage()
	} else if _, isType := dns.StringToType[args[len(args)-1]]; len(args) > 1 && isType {
		qtype, args = args[len(args)-1], args[:len(args)-1]
	}
	if verbose {
		dnsr.DebugLogger = os.Stderr
	}
	var wg sync.WaitGroup
	start := time.Now()
	for _, name := range args {
		wg.Add(1)
		go func(name string, qtype string) {
			query(name, qtype)
			wg.Done()
		}(name, qtype)
	}
	wg.Wait()
	if len(args) > 1 {
		color.Printf("\n@{.w};; Total elapsed: %s\n", time.Since(start).String())
	}
}

func query(name, qtype string) {
	start := time.Now()
	qname, err := idna.ToASCII(name)
	if err != nil {
		color.Fprintf(os.Stderr, "Invalid IDN domain name: %s\n", name)
		os.Exit(1)
	}

	rrs, err := resolver.ResolveErr(qname, qtype)

	color.Printf("\n")
	if len(rrs) > 0 {
		color.Printf("@{g};; RESULTS:\n")
	}
	for _, rr := range rrs {
		color.Printf("@{g}%s\n", rr.String())
	}

	if err != nil {
		color.Printf("@{r};; %s\t%s\t%s\n", err, name, qtype)
	} else if rrs == nil {
		color.Printf("@{y};; NIL\t%s\t%s\n", name, qtype)
	} else if len(rrs) > 0 {
		color.Printf("@{g};; TRUE\t%s\t%s\n", name, qtype)
	} else {
		color.Printf("@{r};; FALSE\t%s\t%s\n", name, qtype)
	}

	color.Printf("@{.w};; Elapsed: %s\n", time.Since(start).String())
}
<|endoftext|>


<commit_before>package main

import (
	"flag"
	"fmt"
	"log"

	"bitbucket.org/cihangirsavas/gene/generators/modules"
)

var (
	flagSchemaFile = flag.String("schema", "", "schema content file")
	flagFolder     = flag.String("target", "./", "target directory name")
)

func main() {
	flag.Parse()

	m, err := modules.NewFromFile(*flagSchemaFile)
	if err != nil {
		log.Fatalf(err.Error())
		return
	}

	m.TargetFolderName = *flagFolder
	fmt.Println("m-->", m.Create())
}
<commit_msg>Gene added: created module with success log<commit_after>package main

import (
	"flag"
	"log"

	"bitbucket.org/cihangirsavas/gene/generators/modules"
)

var (
	flagSchemaFile = flag.String("schema", "", "schema content file")
	flagFolder     = flag.String("target", "./", "target directory name")
)

func main() {
	flag.Parse()

	m, err := modules.NewFromFile(*flagSchemaFile)
	if err != nil {
		log.Fatalf(err.Error())
		return
	}

	m.TargetFolderName = *flagFolder
	if err := m.Create(); err != nil {
		log.Fatalf(err.Error())
		return
	}

	log.Println("module created with success")
}
<|endoftext|>


<commit_before>package main

import (
	"flag"
	"fmt"
	"net/http"
	"os"
	"sync"
	"time"

	"github.com/howeyc/ledger"

	"github.com/BurntSushi/toml"
	"github.com/go-martini/martini"
	"github.com/martini-contrib/gzip"
	"github.com/martini-contrib/staticbin"
)

var ledgerFileName string
var reportConfigFileName string
var stockConfigFileName string
var ledgerLock sync.Mutex

func getTransactions() ([]*ledger.Transaction, error) {
	ledgerLock.Lock()
	defer ledgerLock.Unlock()

	ledgerFileReader, err := os.Open(ledgerFileName)
	if err != nil {
		return nil, err

	}

	trans, terr := ledger.ParseLedger(ledgerFileReader)
	if terr != nil {
		return nil, fmt.Errorf("%s:%s", ledgerFileName, terr.Error())
	}
	ledgerFileReader.Close()

	return trans, nil
}

type accountOp struct {
	Name      string `toml:"name"`
	Operation string `toml:"operation"` // +, -
}

type calculatedAccount struct {
	Name              string      `toml:"name"`
	AccountOperations []accountOp `toml:"account_operation"`
}

type reportConfig struct {
	Name                   string
	Chart                  string
	DateRange              string `toml:"date_range"`
	DateFreq               string `toml:"date_freq"`
	Accounts               []string
	ExcludeAccountTrans    []string            `toml:"exclude_account_trans"`
	ExcludeAccountsSummary []string            `toml:"exclude_account_summary"`
	CalculatedAccounts     []calculatedAccount `toml:"calculated_account"`
}

type reportConfigStruct struct {
	Reports []reportConfig `toml:"report"`
}

var reportConfigData reportConfigStruct

type stockConfig struct {
	Name    string
	Section string
	Ticker  string
	Account string
	Shares  float64
}

type stockInfo struct {
	Name                  string
	Section               string
	Type                  string
	Ticker                string
	Account               string
	Shares                float64
	Price                 float64
	PriceChangeDay        float64
	PriceChangePctDay     float64
	PriceChangeOverall    float64
	PriceChangePctOverall float64
	Cost                  float64
	MarketValue           float64
	GainLossDay           float64
	GainLossOverall       float64
}

type stockConfigStruct struct {
	Stocks []stockConfig `toml:"stock"`
}

var stockConfigData stockConfigStruct

type pageData struct {
	Reports      []reportConfig
	Transactions []*ledger.Transaction
	Accounts     []*ledger.Account
	Stocks       []stockInfo
}

func main() {
	var serverPort int
	var localhost bool

	flag.StringVar(&ledgerFileName, "f", "", "Ledger file name (*Required).")
	flag.StringVar(&reportConfigFileName, "r", "", "Report config file name (*Required).")
	flag.StringVar(&stockConfigFileName, "s", "", "Stock config file name (*Optional).")
	flag.IntVar(&serverPort, "port", 8056, "Port to listen on.")
	flag.BoolVar(&localhost, "localhost", false, "Listen on localhost only.")

	flag.Parse()

	if len(ledgerFileName) == 0 || len(reportConfigFileName) == 0 {
		flag.Usage()
		return
	}

	go func() {
		for {
			var rLoadData reportConfigStruct
			toml.DecodeFile(reportConfigFileName, &rLoadData)
			reportConfigData = rLoadData
			time.Sleep(time.Minute * 5)
		}
	}()

	if len(stockConfigFileName) > 0 {
		var sLoadData stockConfigStruct
		toml.DecodeFile(stockConfigFileName, &sLoadData)
		stockConfigData = sLoadData
	}

	m := martini.Classic()
	m.Use(gzip.All())
	m.Use(staticbin.Static("public", Asset))

	m.Get("/ledger", ledgerHandler)
	m.Get("/accounts", accountsHandler)
	m.Get("/portfolio", portfolioHandler)
	m.Get("/account/:accountName", accountHandler)
	m.Get("/report/:reportName", reportHandler)
	m.Get("/", func(w http.ResponseWriter, r *http.Request) {
		http.Redirect(w, r, "/accounts", http.StatusFound)
	})

	fmt.Println("Listening on port", serverPort)
	listenAddress := ""
	if localhost {
		listenAddress = fmt.Sprintf("127.0.0.1:%d", serverPort)
	} else {
		listenAddress = fmt.Sprintf(":%d", serverPort)
	}
	http.ListenAndServe(listenAddress, m)
}
<commit_msg>cache transactions in web process<commit_after>package main

import (
	"bytes"
	"crypto/sha256"
	"flag"
	"fmt"
	"io"
	"net/http"
	"os"
	"sync"
	"time"

	"github.com/howeyc/ledger"

	"github.com/BurntSushi/toml"
	"github.com/go-martini/martini"
	"github.com/martini-contrib/gzip"
	"github.com/martini-contrib/staticbin"
)

var ledgerFileName string
var reportConfigFileName string
var stockConfigFileName string
var ledgerLock sync.Mutex
var currentSum []byte
var currentTrans []*ledger.Transaction

func getTransactions() ([]*ledger.Transaction, error) {
	ledgerLock.Lock()
	defer ledgerLock.Unlock()

	var buf bytes.Buffer
	h := sha256.New()

	ledgerFileReader, err := os.Open(ledgerFileName)
	if err != nil {
		return nil, err

	}
	tr := io.TeeReader(ledgerFileReader, h)
	io.Copy(&buf, tr)
	ledgerFileReader.Close()

	sum := h.Sum(nil)
	if bytes.Compare(currentSum, sum) == 0 {
		return currentTrans, nil
	}

	trans, terr := ledger.ParseLedger(&buf)
	if terr != nil {
		return nil, fmt.Errorf("%s:%s", ledgerFileName, terr.Error())
	}
	currentSum = sum
	currentTrans = trans

	return trans, nil
}

type accountOp struct {
	Name      string `toml:"name"`
	Operation string `toml:"operation"` // +, -
}

type calculatedAccount struct {
	Name              string      `toml:"name"`
	AccountOperations []accountOp `toml:"account_operation"`
}

type reportConfig struct {
	Name                   string
	Chart                  string
	DateRange              string `toml:"date_range"`
	DateFreq               string `toml:"date_freq"`
	Accounts               []string
	ExcludeAccountTrans    []string            `toml:"exclude_account_trans"`
	ExcludeAccountsSummary []string            `toml:"exclude_account_summary"`
	CalculatedAccounts     []calculatedAccount `toml:"calculated_account"`
}

type reportConfigStruct struct {
	Reports []reportConfig `toml:"report"`
}

var reportConfigData reportConfigStruct

type stockConfig struct {
	Name    string
	Section string
	Ticker  string
	Account string
	Shares  float64
}

type stockInfo struct {
	Name                  string
	Section               string
	Type                  string
	Ticker                string
	Account               string
	Shares                float64
	Price                 float64
	PriceChangeDay        float64
	PriceChangePctDay     float64
	PriceChangeOverall    float64
	PriceChangePctOverall float64
	Cost                  float64
	MarketValue           float64
	GainLossDay           float64
	GainLossOverall       float64
}

type stockConfigStruct struct {
	Stocks []stockConfig `toml:"stock"`
}

var stockConfigData stockConfigStruct

type pageData struct {
	Reports      []reportConfig
	Transactions []*ledger.Transaction
	Accounts     []*ledger.Account
	Stocks       []stockInfo
}

func main() {
	var serverPort int
	var localhost bool

	flag.StringVar(&ledgerFileName, "f", "", "Ledger file name (*Required).")
	flag.StringVar(&reportConfigFileName, "r", "", "Report config file name (*Required).")
	flag.StringVar(&stockConfigFileName, "s", "", "Stock config file name (*Optional).")
	flag.IntVar(&serverPort, "port", 8056, "Port to listen on.")
	flag.BoolVar(&localhost, "localhost", false, "Listen on localhost only.")

	flag.Parse()

	if len(ledgerFileName) == 0 || len(reportConfigFileName) == 0 {
		flag.Usage()
		return
	}

	go func() {
		for {
			var rLoadData reportConfigStruct
			toml.DecodeFile(reportConfigFileName, &rLoadData)
			reportConfigData = rLoadData
			time.Sleep(time.Minute * 5)
		}
	}()

	if len(stockConfigFileName) > 0 {
		var sLoadData stockConfigStruct
		toml.DecodeFile(stockConfigFileName, &sLoadData)
		stockConfigData = sLoadData
	}

	// initialize cache
	getTransactions()

	m := martini.Classic()
	m.Use(gzip.All())
	m.Use(staticbin.Static("public", Asset))

	m.Get("/ledger", ledgerHandler)
	m.Get("/accounts", accountsHandler)
	m.Get("/portfolio", portfolioHandler)
	m.Get("/account/:accountName", accountHandler)
	m.Get("/report/:reportName", reportHandler)
	m.Get("/", func(w http.ResponseWriter, r *http.Request) {
		http.Redirect(w, r, "/accounts", http.StatusFound)
	})

	fmt.Println("Listening on port", serverPort)
	listenAddress := ""
	if localhost {
		listenAddress = fmt.Sprintf("127.0.0.1:%d", serverPort)
	} else {
		listenAddress = fmt.Sprintf(":%d", serverPort)
	}
	http.ListenAndServe(listenAddress, m)
}
<|endoftext|>


<commit_before>package main

import (
	"time"

	"github.com/cortesi/modd"
	"github.com/cortesi/termlog"
	"gopkg.in/alecthomas/kingpin.v2"
)

const batchTime = time.Millisecond * 200

func main() {
	paths := kingpin.Arg(
		"path",
		"Paths to monitor for changes.",
	).Required().Strings()

	excludes := kingpin.Flag("exclude", "Glob pattern for files to exclude from livereload").
		PlaceHolder("PATTERN").
		Short('x').
		Strings()

	debug := kingpin.Flag("debug", "Debugging for devd development").
		Default("false").
		Bool()

	kingpin.Version(modd.Version)
	kingpin.Parse()
	log := termlog.NewLog()

	if *debug {
		log.Enable("debug")
		modd.Logger = log
	}

	modchan := make(chan modd.Mod)
	err := modd.Watch(*paths, *excludes, batchTime, modchan)
	if err != nil {
		kingpin.Fatalf("Fatal error: %s", err)
	}
	for mod := range modchan {
		if len(mod.Added) > 0 {
			log.Say("Added: %v\n", mod.Added)
		}
		if len(mod.Changed) > 0 {
			log.Say("Changed: %v\n", mod.Changed)
		}
		if len(mod.Deleted) > 0 {
			log.Say("Deleted: %v\n", mod.Deleted)
		}
	}
}
<commit_msg>Print a little something to show that modd is running<commit_after>package main

import (
	"time"

	"github.com/cortesi/modd"
	"github.com/cortesi/termlog"
	"gopkg.in/alecthomas/kingpin.v2"
)

const batchTime = time.Millisecond * 200

func main() {
	paths := kingpin.Arg(
		"path",
		"Paths to monitor for changes.",
	).Required().Strings()

	excludes := kingpin.Flag("exclude", "Glob pattern for files to exclude from livereload").
		PlaceHolder("PATTERN").
		Short('x').
		Strings()

	debug := kingpin.Flag("debug", "Debugging for devd development").
		Default("false").
		Bool()

	kingpin.Version(modd.Version)
	kingpin.Parse()
	log := termlog.NewLog()
	log.Notice("modd v%s", modd.Version)

	if *debug {
		log.Enable("debug")
		modd.Logger = log
	}

	modchan := make(chan modd.Mod)
	err := modd.Watch(*paths, *excludes, batchTime, modchan)
	if err != nil {
		kingpin.Fatalf("Fatal error: %s", err)
	}
	for mod := range modchan {
		if len(mod.Added) > 0 {
			log.Say("Added: %v\n", mod.Added)
		}
		if len(mod.Changed) > 0 {
			log.Say("Changed: %v\n", mod.Changed)
		}
		if len(mod.Deleted) > 0 {
			log.Say("Deleted: %v\n", mod.Deleted)
		}
	}
}
<|endoftext|>


<commit_before>package main

func getAddon(args []string) error {
	return nil
}
<commit_msg>getAddon(), comprising go get and copy routines<commit_after>package main

import (
	"errors"
	"os"
	"os/exec"
	"path/filepath"
	"strings"
)

// Use `go get` to download addon and add to $GOPATH/src, useful
// for IDE auto-import and code completion, then copy entire directory
// tree to project's ./addons folder
func getAddon(args []string) error {

	// error return
	errorFunc := func(err error) error {
		return errors.New("Ponzu add failed. " + "\n" + err.Error())
	}

	var cmdOptions []string
	var addonPath = args[1]

	// Go get
	cmdOptions = append(cmdOptions, addonPath)
	get := exec.Command(gocmd, cmdOptions...)
	get.Stderr = os.Stderr
	get.Stdout = os.Stdout

	err := get.Start()
	if err != nil {
		errorFunc(err)
	}
	err = get.Wait()
	if err != nil {
		errorFunc(err)
	}

	// Copy to ./addons folder

	// GOPATH can be a list delimited by ":" on Linux or ";" on Windows
	// go get uses the first, this should parse out the first whatever the OS
	envGOPATH := os.Getenv("GOPATH")
	gopaths := strings.Split(envGOPATH, ":")
	gopath := gopaths[0]
	gopaths = strings.Split(envGOPATH, ";")
	gopath = gopaths[0]

	src := filepath.Join(gopath, addonPath)
	dest := filepath.Join("./addons", addonPath)

	err = copyAll(src, dest)
	if err != nil {
		errorFunc(err)
	}
	return nil
}
<|endoftext|>


<commit_before>package main

import (
	"errors"
	"fmt"
	"os"
	"os/signal"
	"syscall"

	"github.com/spf13/cobra"

	"github.com/flant/werf/cmd/werf/build"
	"github.com/flant/werf/cmd/werf/build_and_publish"
	"github.com/flant/werf/cmd/werf/cleanup"
	"github.com/flant/werf/cmd/werf/publish"
	"github.com/flant/werf/cmd/werf/purge"

	helm_secret_edit "github.com/flant/werf/cmd/werf/helm/secret/edit"
	helm_secret_extract "github.com/flant/werf/cmd/werf/helm/secret/extract"
	helm_secret_generate "github.com/flant/werf/cmd/werf/helm/secret/generate"
	helm_secret_key_generate "github.com/flant/werf/cmd/werf/helm/secret/key_generate"
	helm_secret_regenerate "github.com/flant/werf/cmd/werf/helm/secret/regenerate"

	"github.com/flant/werf/cmd/werf/tools/slugify"

	images_cleanup "github.com/flant/werf/cmd/werf/images/cleanup"
	images_publish "github.com/flant/werf/cmd/werf/images/publish"
	images_purge "github.com/flant/werf/cmd/werf/images/purge"

	stages_build "github.com/flant/werf/cmd/werf/stages/build"
	stages_cleanup "github.com/flant/werf/cmd/werf/stages/cleanup"
	stages_purge "github.com/flant/werf/cmd/werf/stages/purge"

	host_cleanup "github.com/flant/werf/cmd/werf/host/cleanup"
	host_purge "github.com/flant/werf/cmd/werf/host/purge"

	helm_get_service_values "github.com/flant/werf/cmd/werf/helm/get_service_values"

	"github.com/flant/werf/cmd/werf/completion"
	"github.com/flant/werf/cmd/werf/docs"
	"github.com/flant/werf/cmd/werf/version"

	"github.com/flant/werf/cmd/werf/common"
	"github.com/flant/werf/cmd/werf/common/templates"
	"github.com/flant/werf/pkg/logger"
	"github.com/flant/werf/pkg/process_exterminator"
)

func main() {
	trapTerminationSignals()

	logger.Init(logger.Options{})

	if err := process_exterminator.Init(); err != nil {
		logger.LogError(fmt.Errorf("process exterminator initialization error: %s", err))
		os.Exit(1)
	}

	rootCmd := &cobra.Command{
		Use:   "werf",
		Short: "Werf helps to implement and support Continuous Integration and Continuous Delivery",
		Long: common.GetLongCommandDescription(`Werf helps to implement and support Continuous Integration and Continuous Delivery.

Find more information at https://flant.github.io/werf`),
		SilenceUsage:  true,
		SilenceErrors: true,
	}

	groups := templates.CommandGroups{
		{
			Message: "Main Commands:",
			Commands: []*cobra.Command{
				build.NewCmd(),
				publish.NewCmd(),
				build_and_publish.NewCmd(),
				cleanup.NewCmd(),
				purge.NewCmd(),
			},
		},
		{
			Message: "Tools Commands:",
			Commands: []*cobra.Command{
				slugify.NewCmd(),
			},
		},
		{
			Message: "Lowlevel Management Commands:",
			Commands: []*cobra.Command{
				stagesCmd(),
				imagesCmd(),
				helmCmd(),
				hostCmd(),
			},
		},
	}
	groups.Add(rootCmd)

	templates.ActsAsRootCommand(rootCmd, groups...)

	rootCmd.AddCommand(
		completion.NewCmd(rootCmd),
		version.NewCmd(),
		docs.NewCmd(),
	)

	if err := rootCmd.Execute(); err != nil {
		logger.LogError(err)
		os.Exit(1)
	}
}

func imagesCmd() *cobra.Command {
	cmd := &cobra.Command{
		Use:   "images",
		Short: "Commands to work with images",
	}
	cmd.AddCommand(
		images_publish.NewCmd(),
		images_cleanup.NewCmd(),
		images_purge.NewCmd(),
	)

	return cmd
}

func stagesCmd() *cobra.Command {
	cmd := &cobra.Command{
		Use:   "stages",
		Short: "Commands to work with stages, which are cache for images",
	}
	cmd.AddCommand(
		stages_build.NewCmd(),
		stages_cleanup.NewCmd(),
		stages_purge.NewCmd(),
	)

	return cmd
}

func helmCmd() *cobra.Command {
	cmd := &cobra.Command{
		Use:   "helm",
		Short: "Commands to manage deployment with helm",
	}
	cmd.AddCommand(
		helm_get_service_values.NewCmd(),
		secretCmd(),
	)

	return cmd
}

func hostCmd() *cobra.Command {
	cmd := &cobra.Command{
		Use: "host",
	}
	cmd.AddCommand(
		host_cleanup.NewCmd(),
		host_purge.NewCmd(),
	)

	return cmd
}

func secretCmd() *cobra.Command {
	cmd := &cobra.Command{
		Use:   "secret",
		Short: "Commands to work with secrets",
	}
	cmd.AddCommand(
		helm_secret_key_generate.NewCmd(),
		helm_secret_generate.NewCmd(),
		helm_secret_extract.NewCmd(),
		helm_secret_edit.NewCmd(),
		helm_secret_regenerate.NewCmd(),
	)

	return cmd
}

func trapTerminationSignals() {
	c := make(chan os.Signal, 1)
	signals := []os.Signal{os.Interrupt, syscall.SIGTERM, syscall.SIGQUIT}
	signal.Notify(c, signals...)
	go func() {
		<-c

		logger.LogError(errors.New("interrupted"))
		os.Exit(17)
	}()
}
<commit_msg>dismiss<commit_after>package main

import (
	"errors"
	"fmt"
	"os"
	"os/signal"
	"syscall"

	"github.com/spf13/cobra"

	"github.com/flant/werf/cmd/werf/build"
	"github.com/flant/werf/cmd/werf/build_and_publish"
	"github.com/flant/werf/cmd/werf/cleanup"
	"github.com/flant/werf/cmd/werf/dismiss"
	"github.com/flant/werf/cmd/werf/publish"
	"github.com/flant/werf/cmd/werf/purge"

	helm_secret_edit "github.com/flant/werf/cmd/werf/helm/secret/edit"
	helm_secret_extract "github.com/flant/werf/cmd/werf/helm/secret/extract"
	helm_secret_generate "github.com/flant/werf/cmd/werf/helm/secret/generate"
	helm_secret_key_generate "github.com/flant/werf/cmd/werf/helm/secret/key_generate"
	helm_secret_regenerate "github.com/flant/werf/cmd/werf/helm/secret/regenerate"

	"github.com/flant/werf/cmd/werf/tools/slugify"

	images_cleanup "github.com/flant/werf/cmd/werf/images/cleanup"
	images_publish "github.com/flant/werf/cmd/werf/images/publish"
	images_purge "github.com/flant/werf/cmd/werf/images/purge"

	stages_build "github.com/flant/werf/cmd/werf/stages/build"
	stages_cleanup "github.com/flant/werf/cmd/werf/stages/cleanup"
	stages_purge "github.com/flant/werf/cmd/werf/stages/purge"

	host_cleanup "github.com/flant/werf/cmd/werf/host/cleanup"
	host_purge "github.com/flant/werf/cmd/werf/host/purge"

	helm_get_service_values "github.com/flant/werf/cmd/werf/helm/get_service_values"

	"github.com/flant/werf/cmd/werf/completion"
	"github.com/flant/werf/cmd/werf/docs"
	"github.com/flant/werf/cmd/werf/version"

	"github.com/flant/werf/cmd/werf/common"
	"github.com/flant/werf/cmd/werf/common/templates"
	"github.com/flant/werf/pkg/logger"
	"github.com/flant/werf/pkg/process_exterminator"
)

func main() {
	trapTerminationSignals()

	logger.Init(logger.Options{})

	if err := process_exterminator.Init(); err != nil {
		logger.LogError(fmt.Errorf("process exterminator initialization error: %s", err))
		os.Exit(1)
	}

	rootCmd := &cobra.Command{
		Use:   "werf",
		Short: "Werf helps to implement and support Continuous Integration and Continuous Delivery",
		Long: common.GetLongCommandDescription(`Werf helps to implement and support Continuous Integration and Continuous Delivery.

Find more information at https://flant.github.io/werf`),
		SilenceUsage:  true,
		SilenceErrors: true,
	}

	groups := templates.CommandGroups{
		{
			Message: "Main Commands:",
			Commands: []*cobra.Command{
				build.NewCmd(),
				publish.NewCmd(),
				build_and_publish.NewCmd(),
				dismiss.NewCmd(),
				cleanup.NewCmd(),
				purge.NewCmd(),
			},
		},
		{
			Message: "Tools Commands:",
			Commands: []*cobra.Command{
				slugify.NewCmd(),
			},
		},
		{
			Message: "Lowlevel Management Commands:",
			Commands: []*cobra.Command{
				stagesCmd(),
				imagesCmd(),
				helmCmd(),
				hostCmd(),
			},
		},
	}
	groups.Add(rootCmd)

	templates.ActsAsRootCommand(rootCmd, groups...)

	rootCmd.AddCommand(
		completion.NewCmd(rootCmd),
		version.NewCmd(),
		docs.NewCmd(),
	)

	if err := rootCmd.Execute(); err != nil {
		logger.LogError(err)
		os.Exit(1)
	}
}

func imagesCmd() *cobra.Command {
	cmd := &cobra.Command{
		Use:   "images",
		Short: "Commands to work with images",
	}
	cmd.AddCommand(
		images_publish.NewCmd(),
		images_cleanup.NewCmd(),
		images_purge.NewCmd(),
	)

	return cmd
}

func stagesCmd() *cobra.Command {
	cmd := &cobra.Command{
		Use:   "stages",
		Short: "Commands to work with stages, which are cache for images",
	}
	cmd.AddCommand(
		stages_build.NewCmd(),
		stages_cleanup.NewCmd(),
		stages_purge.NewCmd(),
	)

	return cmd
}

func helmCmd() *cobra.Command {
	cmd := &cobra.Command{
		Use:   "helm",
		Short: "Commands to manage deployment with helm",
	}
	cmd.AddCommand(
		helm_get_service_values.NewCmd(),
		secretCmd(),
	)

	return cmd
}

func hostCmd() *cobra.Command {
	cmd := &cobra.Command{
		Use: "host",
	}
	cmd.AddCommand(
		host_cleanup.NewCmd(),
		host_purge.NewCmd(),
	)

	return cmd
}

func secretCmd() *cobra.Command {
	cmd := &cobra.Command{
		Use:   "secret",
		Short: "Commands to work with secrets",
	}
	cmd.AddCommand(
		helm_secret_key_generate.NewCmd(),
		helm_secret_generate.NewCmd(),
		helm_secret_extract.NewCmd(),
		helm_secret_edit.NewCmd(),
		helm_secret_regenerate.NewCmd(),
	)

	return cmd
}

func trapTerminationSignals() {
	c := make(chan os.Signal, 1)
	signals := []os.Signal{os.Interrupt, syscall.SIGTERM, syscall.SIGQUIT}
	signal.Notify(c, signals...)
	go func() {
		<-c

		logger.LogError(errors.New("interrupted"))
		os.Exit(17)
	}()
}
<|endoftext|>


<commit_before>package cmd

import (
	"fmt"

	"github.com/couchbaselabs/sgload/sgload"
	"github.com/spf13/cobra"
)

var (
	numWriters    *int
	createWriters *bool
	writerDelayMs *int
)

// writeloadCmd respresents the writeload command
var writeloadCmd = &cobra.Command{
	Use:   "writeload",
	Short: "Generate a write load",
	Long:  `Generate a write load`,
	Run: func(cmd *cobra.Command, args []string) {

		// Setup logger
		logger = sgload.Logger()

		loadSpec := createLoadSpecFromArgs()
		sgload.SetLogLevel(loadSpec.LogLevel)

		writeLoadSpec := sgload.WriteLoadSpec{
			LoadSpec:      loadSpec,
			NumWriters:    *numWriters,
			CreateWriters: *createWriters,
		}
		if err := writeLoadSpec.Validate(); err != nil {

			panic(fmt.Sprintf("Invalid parameters: %+v. Error: %v", writeLoadSpec, err))
		}
		writeLoadRunner := sgload.NewWriteLoadRunner(writeLoadSpec)
		if err := writeLoadRunner.Run(); err != nil {
			panic(fmt.Sprintf("Writeload.Run() failed with: %v", err))
		}

	},
}

func init() {

	RootCmd.AddCommand(writeloadCmd)

	numWriters = writeloadCmd.PersistentFlags().Int(
		NUM_WRITERS_CMD_NAME,
		NUM_WRITERS_CMD_DEFAULT,
		NUM_WRITERS_CMD_DESC,
	)

	createWriters = writeloadCmd.PersistentFlags().Bool(
		CREATE_WRITERS_CMD_NAME,
		CREATE_WRITERS_CMD_DEFAULT,
		CREATE_WRITERS_CMD_DESC,
	)

	writerDelayMs = writeloadCmd.PersistentFlags().Int(
		WRITER_DELAY_CMD_NAME,
		WRITER_DELAY_CMD_DEFAULT,
		WRITER_DELAY_CMD_DESC,
	)

}
<commit_msg>Fixes #70 writerdelayms parameter is not taken into account while running sgload.<commit_after>package cmd

import (
	"fmt"

	"github.com/couchbaselabs/sgload/sgload"
	"github.com/spf13/cobra"
	"time"
)

var (
	numWriters    *int
	createWriters *bool
	writerDelayMs *int
)

// writeloadCmd respresents the writeload command
var writeloadCmd = &cobra.Command{
	Use:   "writeload",
	Short: "Generate a write load",
	Long:  `Generate a write load`,
	Run: func(cmd *cobra.Command, args []string) {

		// Setup logger
		logger = sgload.Logger()

		loadSpec := createLoadSpecFromArgs()
		sgload.SetLogLevel(loadSpec.LogLevel)

		delayBetweenWrites := time.Millisecond * time.Duration(*writerDelayMs)

		writeLoadSpec := sgload.WriteLoadSpec{
			LoadSpec:      loadSpec,
			NumWriters:    *numWriters,
			CreateWriters: *createWriters,
			DelayBetweenWrites: delayBetweenWrites,
		}
		if err := writeLoadSpec.Validate(); err != nil {

			panic(fmt.Sprintf("Invalid parameters: %+v. Error: %v", writeLoadSpec, err))
		}
		writeLoadRunner := sgload.NewWriteLoadRunner(writeLoadSpec)
		if err := writeLoadRunner.Run(); err != nil {
			panic(fmt.Sprintf("Writeload.Run() failed with: %v", err))
		}

	},
}

func init() {

	RootCmd.AddCommand(writeloadCmd)

	numWriters = writeloadCmd.PersistentFlags().Int(
		NUM_WRITERS_CMD_NAME,
		NUM_WRITERS_CMD_DEFAULT,
		NUM_WRITERS_CMD_DESC,
	)

	createWriters = writeloadCmd.PersistentFlags().Bool(
		CREATE_WRITERS_CMD_NAME,
		CREATE_WRITERS_CMD_DEFAULT,
		CREATE_WRITERS_CMD_DESC,
	)

	writerDelayMs = writeloadCmd.PersistentFlags().Int(
		WRITER_DELAY_CMD_NAME,
		WRITER_DELAY_CMD_DEFAULT,
		WRITER_DELAY_CMD_DESC,
	)

}
<|endoftext|>


<commit_before>package flite

import (
	"testing"
	"time"
)

func TestTextToSpeech(t *testing.T) {
	voice, err := VoiceSelect("awb")
	if err != nil {
		t.Fatal(err)
	}

	s := TextToSpeech("Hello World", voice, "play")
	if s == 0 {
		t.Fatalf("0 seconds of speech generated")
	}
}

func TestTextToWave(t *testing.T) {
	voice, err := VoiceSelect("awb")
	if err != nil {
		t.Fatal(err)
	}

	wav := TextToWave("Hello World", voice)
	if wav.NumChannels != 1 {
		t.Fatalf("Wave number of channels != 1")
	}

	if wav.SampleRate != 16000 {
		t.Fatalf("Wave sample rate != 16000")
	}
}

func TestVoiceSelect(t *testing.T) {
	var voices = []string{"awb", "kal16", "kal", "rms", "slt"}

	for _, v := range voices {
		voice, err := VoiceSelect(v)
		if err != nil {
			t.Fatal(err)
		}

		TextToSpeech("Testing "+v, voice, "play")
		time.Sleep(1 * time.Second)

		s := TextToSpeech("Hello World", voice, "play")
		if s == 0 {
			t.Fatalf("%s: 0 seconds of speech generated", v)
		}

		time.Sleep(1 * time.Second)
	}
}
<commit_msg>Fix test<commit_after>package flite

import (
	"testing"
	"time"
)

func TestTextToSpeech(t *testing.T) {
	voice, err := VoiceSelect("awb")
	if err != nil {
		t.Fatal(err)
	}

	s := TextToSpeech("Hello World", voice, "play")
	if s == 0 {
		t.Fatalf("0 seconds of speech generated")
	}
}

func TestTextToWave(t *testing.T) {
	voice, err := VoiceSelect("awb")
	if err != nil {
		t.Fatal(err)
	}

	wav := TextToWave("Hello World", voice)
	if wav.NumChannels != 1 {
		t.Fatalf("Wave number of channels != 1")
	}

	if wav.SampleRate != 16000 {
		t.Fatalf("Wave sample rate != 16000")
	}
}

func TestVoiceSelect(t *testing.T) {
	var voices = []string{"awb", "kal16", "kal", "rms", "slt"}

	for _, v := range voices {
		voice, err := VoiceSelect(v)
		if err != nil {
			t.Fatal(err)
		}

		TextToSpeech("Testing "+v, voice, "play")
		time.Sleep(1 * time.Second)

		TextToSpeech("Hello World", voice, "play")
		time.Sleep(1 * time.Second)
	}
}
<|endoftext|>


<commit_before>package zmqOutput_test

import (
	. "github.com/CapillarySoftware/goiostat/diskStat"
	. "github.com/CapillarySoftware/goiostat/zmqOutput"
	. "github.com/onsi/ginkgo"
	// . "github.com/CapillarySoftware/goiostat/protoStat"
	// "fmt"
	. "github.com/onsi/gomega"
)

var _ = Describe("ZmqOutput", func() {

	It("Testing basic send stats", func() {
		output := ZmqOutput{}
		output.Connect("ipc:///tmp/zmqOutput.ipc")
		// defer output.Close()
		stats := ExtendedIoStats{
			"Device",
			float64(0),
			float64(0),
			float64(0),
			float64(0),
			float64(0),
			float64(0),
			float64(0),
			float64(0),
			float64(0),
			float64(0),
			float64(0),
			float64(0),
			float64(0),
		}
		err := output.SendStats(&stats)
		Expect(err).Should(BeNil())
	})

	It("Call sendStats without initializing socket", func() {
		output := ZmqOutput{}
		stats := ExtendedIoStats{
			"Device",
			float64(0),
			float64(0),
			float64(0),
			float64(0),
			float64(0),
			float64(0),
			float64(0),
			float64(0),
			float64(0),
			float64(0),
			float64(0),
			float64(0),
			float64(0),
		}
		err := output.SendStats(&stats)
		Expect(err).ShouldNot(BeNil())
	})
})
<commit_msg>woo data<commit_after>package zmqOutput_test

import (
	. "github.com/CapillarySoftware/goiostat/diskStat"
	. "github.com/CapillarySoftware/goiostat/zmqOutput"
	. "github.com/onsi/ginkgo"
	// . "github.com/CapillarySoftware/goiostat/protoStat"
	"fmt"
	. "github.com/onsi/gomega"
	zmq "github.com/pebbe/zmq3"
)

func sendStats(output ZmqOutput, eStat *ExtendedIoStats, sendCount int) {
	for i := 0; i <= sendCount; i++ {
		output.SendStats(eStat)
	}
}

var _ = Describe("ZmqOutput", func() {
	eStat := ExtendedIoStats{
		"Device",
		float64(0),
		float64(0),
		float64(0),
		float64(0),
		float64(0),
		float64(0),
		float64(0),
		float64(0),
		float64(0),
		float64(0),
		float64(0),
		float64(0),
		float64(0),
	}

	url := "ipc:///tmp/testOutput.ipc"

	It("Testing basic send stats", func() {
		output := ZmqOutput{}
		output.Connect(url)
		defer output.Close()

		err := output.SendStats(&eStat)
		Expect(err).Should(BeNil())
	})

	It("Call sendStats without initializing socket", func() {
		output := ZmqOutput{}
		defer output.Close()
		err := output.SendStats(&eStat)
		Expect(err).ShouldNot(BeNil())
	})

	It("Send to recv socket and validate we get what we expect", func() {
		output := ZmqOutput{}
		defer output.Close()
		output.Connect(url)

		recv, err := zmq.NewSocket(zmq.PULL)
		Expect(err).Should(BeNil())
		defer recv.Close()

		recv.Bind(url)
		go sendStats(output, &eStat, 1)

		for i := 0; i <= 12; i++ {
			s, err := recv.RecvBytes(0)
			fmt.Println("bytes: ", s)
			Expect(err).Should(BeNil())
		}
		s, err := recv.RecvBytes(0)
		Expect(err).Should(BeNil())
		fmt.Println("last: ", s)

	})
})
<|endoftext|>


<commit_before>package form

import (
	"html/template"
	"net/http"

	"github.com/janekolszak/idp/core"
	"github.com/janekolszak/idp/userdb"
)

type Config struct {
	LoginForm          string
	LoginUsernameField string
	LoginPasswordField string
	UserStore          userdb.Store
}

type FormAuth struct {
	Config
}

func NewFormAuth(c Config) (*FormAuth, error) {
	if c.LoginUsernameField == "" ||
		c.LoginPasswordField == "" ||
		c.LoginUsernameField == c.LoginPasswordField {
		return nil, core.ErrorInvalidConfig
	}
	auth := FormAuth{Config: c}
	return &auth, nil
}

func (f *FormAuth) Check(r *http.Request) (user string, err error) {
	r.ParseForm()
	user = r.Form.Get(f.LoginUsernameField)
	password := r.Form.Get(f.LoginPasswordField)

	err = f.UserStore.Check(user, password)
	if err != nil {
		user = ""
		err = core.ErrorAuthenticationFailure
	}

	return
}

func (a *FormAuth) WriteError(w http.ResponseWriter, r *http.Request, err error) error {
	msg := ""
	if r.Method == "POST" && err != nil {
		switch err {
		case core.ErrorAuthenticationFailure:
			msg = "Authentication failed"

		default:
			msg = "An error occurred"
		}
	}
	t := template.Must(template.New("tmpl").Parse(a.LoginForm))
	t.Execute(w, msg)
	return nil
}

func (a *FormAuth) Write(w http.ResponseWriter, r *http.Request) error {
	return nil
}
<commit_msg>providers/form-auth: return template parse error<commit_after>package form

import (
	"html/template"
	"net/http"

	"github.com/janekolszak/idp/core"
	"github.com/janekolszak/idp/userdb"
)

type Config struct {
	LoginForm          string
	LoginUsernameField string
	LoginPasswordField string
	UserStore          userdb.Store
}

type FormAuth struct {
	Config
}

func NewFormAuth(c Config) (*FormAuth, error) {
	if c.LoginUsernameField == "" ||
		c.LoginPasswordField == "" ||
		c.LoginUsernameField == c.LoginPasswordField {
		return nil, core.ErrorInvalidConfig
	}
	auth := FormAuth{Config: c}
	return &auth, nil
}

func (f *FormAuth) Check(r *http.Request) (user string, err error) {
	r.ParseForm()
	user = r.Form.Get(f.LoginUsernameField)
	password := r.Form.Get(f.LoginPasswordField)

	err = f.UserStore.Check(user, password)
	if err != nil {
		user = ""
		err = core.ErrorAuthenticationFailure
	}

	return
}

func (a *FormAuth) WriteError(w http.ResponseWriter, r *http.Request, err error) error {
	msg := ""
	if r.Method == "POST" && err != nil {
		switch err {
		case core.ErrorAuthenticationFailure:
			msg = "Authentication failed"

		default:
			msg = "An error occurred"
		}
	}
	t := template.Must(template.New("tmpl").Parse(a.LoginForm))
	return t.Execute(w, msg)
}

func (a *FormAuth) Write(w http.ResponseWriter, r *http.Request) error {
	return nil
}
<|endoftext|>