<commit_before>package consumer

import (
	"context"
	"errors"
	"fmt"
	"io/ioutil"
	"log"
	"sync"
	"time"

	"github.com/aws/aws-sdk-go/aws"
	"github.com/aws/aws-sdk-go/aws/awserr"
	"github.com/aws/aws-sdk-go/aws/session"
	"github.com/aws/aws-sdk-go/service/kinesis"
	"github.com/aws/aws-sdk-go/service/kinesis/kinesisiface"
)

// Record wraps the record returned from the Kinesis library and
// extends to include the shard id.
type Record struct {
	*kinesis.Record
	ShardID string
}

// New creates a kinesis consumer with default settings. Use Option to override
// any of the optional attributes.
func New(streamName string, opts ...Option) (*Consumer, error) {
	if streamName == "" {
		return nil, fmt.Errorf("must provide stream name")
	}

	// new consumer with noop storage, counter, and logger
	c := &Consumer{
		streamName:               streamName,
		initialShardIteratorType: kinesis.ShardIteratorTypeLatest,
		store:                    &noopStore{},
		counter:                  &noopCounter{},
		logger: &noopLogger{
			logger: log.New(ioutil.Discard, "", log.LstdFlags),
		},
		scanInterval: 250 * time.Millisecond,
		maxRecords:   10000,
	}

	// override defaults
	for _, opt := range opts {
		opt(c)
	}

	// default client
	if c.client == nil {
		newSession, err := session.NewSession(aws.NewConfig())
		if err != nil {
			return nil, err
		}
		c.client = kinesis.New(newSession)
	}

	// default group consumes all shards
	if c.group == nil {
		c.group = NewAllGroup(c.client, c.store, streamName, c.logger)
	}

	return c, nil
}

// Consumer wraps the interaction with the Kinesis stream
type Consumer struct {
	streamName               string
	initialShardIteratorType string
	initialTimestamp         *time.Time
	client                   kinesisiface.KinesisAPI
	counter                  Counter
	group                    Group
	logger                   Logger
	store                    Store
	scanInterval             time.Duration
	maxRecords               int64
}

// ScanFunc is the type of the function called for each message read
// from the stream. The record argument contains the original record
// returned from the AWS Kinesis library.
// If an error is returned, scanning stops. The sole exception is when the
// function returns the special value ErrSkipCheckpoint.
type ScanFunc func(*Record) error

// ErrSkipCheckpoint is used as a return value from ScanFunc to indicate that
// the current checkpoint should be skipped skipped. It is not returned
// as an error by any function.
var ErrSkipCheckpoint = errors.New("skip checkpoint")

// Scan launches a goroutine to process each of the shards in the stream. The ScanFunc
// is passed through to each of the goroutines and called with each message pulled from
// the stream.
func (c *Consumer) Scan(ctx context.Context, fn ScanFunc) error {
	ctx, cancel := context.WithCancel(ctx)
	defer cancel()

	var (
		errc   = make(chan error, 1)
		shardc = make(chan *kinesis.Shard, 1)
	)

	go func() {
		c.group.Start(ctx, shardc)
		<-ctx.Done()
		close(shardc)
	}()

	wg := new(sync.WaitGroup)
	// process each of the shards
	for shard := range shardc {
		wg.Add(1)
		go func(shardID string) {
			defer wg.Done()
			if err := c.ScanShard(ctx, shardID, fn); err != nil {
				select {
				case errc <- fmt.Errorf("shard %s error: %v", shardID, err):
					// first error to occur
					cancel()
				default:
					// error has already occurred
				}
			}
		}(aws.StringValue(shard.ShardId))
	}

	go func() {
		wg.Wait()
		close(errc)
	}()

	return <-errc
}

// ScanShard loops over records on a specific shard, calls the callback func
// for each record and checkpoints the progress of scan.
func (c *Consumer) ScanShard(ctx context.Context, shardID string, fn ScanFunc) error {
	// get last seq number from checkpoint
	lastSeqNum, err := c.group.GetCheckpoint(c.streamName, shardID)
	if err != nil {
		return fmt.Errorf("get checkpoint error: %v", err)
	}

	// get shard iterator
	shardIterator, err := c.getShardIterator(ctx, c.streamName, shardID, lastSeqNum)
	if err != nil {
		return fmt.Errorf("get shard iterator error: %v", err)
	}

	c.logger.Log("[CONSUMER] start scan:", shardID, lastSeqNum)
	defer func() {
		c.logger.Log("[CONSUMER] stop scan:", shardID)
	}()
	scanTicker := time.NewTicker(c.scanInterval)
	defer scanTicker.Stop()

	for {
		resp, err := c.client.GetRecords(&kinesis.GetRecordsInput{
			Limit:         aws.Int64(c.maxRecords),
			ShardIterator: shardIterator,
		})

		// attempt to recover from GetRecords error when expired iterator
		if err != nil {
			c.logger.Log("[CONSUMER] get records error:", err.Error())

			if awserr, ok := err.(awserr.Error); ok {
				if _, ok := retriableErrors[awserr.Code()]; !ok {
					return fmt.Errorf("get records error: %v", awserr.Message())
				}
			}

			shardIterator, err = c.getShardIterator(ctx, c.streamName, shardID, lastSeqNum)
			if err != nil {
				return fmt.Errorf("get shard iterator error: %v", err)
			}
		} else {
			// loop over records, call callback func
			for _, r := range resp.Records {
				select {
				case <-ctx.Done():
					return nil
				default:
					err := fn(&Record{r, shardID})
					if err != nil && err != ErrSkipCheckpoint {
						return err
					}

					if err != ErrSkipCheckpoint {
						if err := c.group.SetCheckpoint(c.streamName, shardID, *r.SequenceNumber); err != nil {
							return err
						}
					}

					c.counter.Add("records", 1)
					lastSeqNum = *r.SequenceNumber
				}
			}

			if isShardClosed(resp.NextShardIterator, shardIterator) {
				c.logger.Log("[CONSUMER] shard closed:", shardID)
				return nil
			}

			shardIterator = resp.NextShardIterator
		}

		// Wait for next scan
		select {
		case <-ctx.Done():
			return nil
		case <-scanTicker.C:
			continue
		}
	}
}

var retriableErrors = map[string]struct{}{
	kinesis.ErrCodeExpiredIteratorException:               struct{}{},
	kinesis.ErrCodeProvisionedThroughputExceededException: struct{}{},
}

func isShardClosed(nextShardIterator, currentShardIterator *string) bool {
	return nextShardIterator == nil || currentShardIterator == nextShardIterator
}

func (c *Consumer) getShardIterator(ctx context.Context, streamName, shardID, seqNum string) (*string, error) {
	params := &kinesis.GetShardIteratorInput{
		ShardId:    aws.String(shardID),
		StreamName: aws.String(streamName),
	}

	if seqNum != "" {
		params.ShardIteratorType = aws.String(kinesis.ShardIteratorTypeAfterSequenceNumber)
		params.StartingSequenceNumber = aws.String(seqNum)
	} else if c.initialTimestamp != nil {
		params.ShardIteratorType = aws.String(kinesis.ShardIteratorTypeAtTimestamp)
		params.Timestamp = c.initialTimestamp
	} else {
		params.ShardIteratorType = aws.String(c.initialShardIteratorType)
	}

	res, err := c.client.GetShardIteratorWithContext(aws.Context(ctx), params)
	return res.ShardIterator, err
}
<commit_msg>Include MillisBehindLatest in Record for ScanFunc (#124)<commit_after>package consumer

import (
	"context"
	"errors"
	"fmt"
	"io/ioutil"
	"log"
	"sync"
	"time"

	"github.com/aws/aws-sdk-go/aws"
	"github.com/aws/aws-sdk-go/aws/awserr"
	"github.com/aws/aws-sdk-go/aws/session"
	"github.com/aws/aws-sdk-go/service/kinesis"
	"github.com/aws/aws-sdk-go/service/kinesis/kinesisiface"
)

// Record wraps the record returned from the Kinesis library and
// extends to include the shard id.
type Record struct {
	*kinesis.Record
	ShardID            string
	MillisBehindLatest *int64
}

// New creates a kinesis consumer with default settings. Use Option to override
// any of the optional attributes.
func New(streamName string, opts ...Option) (*Consumer, error) {
	if streamName == "" {
		return nil, fmt.Errorf("must provide stream name")
	}

	// new consumer with noop storage, counter, and logger
	c := &Consumer{
		streamName:               streamName,
		initialShardIteratorType: kinesis.ShardIteratorTypeLatest,
		store:                    &noopStore{},
		counter:                  &noopCounter{},
		logger: &noopLogger{
			logger: log.New(ioutil.Discard, "", log.LstdFlags),
		},
		scanInterval: 250 * time.Millisecond,
		maxRecords:   10000,
	}

	// override defaults
	for _, opt := range opts {
		opt(c)
	}

	// default client
	if c.client == nil {
		newSession, err := session.NewSession(aws.NewConfig())
		if err != nil {
			return nil, err
		}
		c.client = kinesis.New(newSession)
	}

	// default group consumes all shards
	if c.group == nil {
		c.group = NewAllGroup(c.client, c.store, streamName, c.logger)
	}

	return c, nil
}

// Consumer wraps the interaction with the Kinesis stream
type Consumer struct {
	streamName               string
	initialShardIteratorType string
	initialTimestamp         *time.Time
	client                   kinesisiface.KinesisAPI
	counter                  Counter
	group                    Group
	logger                   Logger
	store                    Store
	scanInterval             time.Duration
	maxRecords               int64
}

// ScanFunc is the type of the function called for each message read
// from the stream. The record argument contains the original record
// returned from the AWS Kinesis library.
// If an error is returned, scanning stops. The sole exception is when the
// function returns the special value ErrSkipCheckpoint.
type ScanFunc func(*Record) error

// ErrSkipCheckpoint is used as a return value from ScanFunc to indicate that
// the current checkpoint should be skipped skipped. It is not returned
// as an error by any function.
var ErrSkipCheckpoint = errors.New("skip checkpoint")

// Scan launches a goroutine to process each of the shards in the stream. The ScanFunc
// is passed through to each of the goroutines and called with each message pulled from
// the stream.
func (c *Consumer) Scan(ctx context.Context, fn ScanFunc) error {
	ctx, cancel := context.WithCancel(ctx)
	defer cancel()

	var (
		errc   = make(chan error, 1)
		shardc = make(chan *kinesis.Shard, 1)
	)

	go func() {
		c.group.Start(ctx, shardc)
		<-ctx.Done()
		close(shardc)
	}()

	wg := new(sync.WaitGroup)
	// process each of the shards
	for shard := range shardc {
		wg.Add(1)
		go func(shardID string) {
			defer wg.Done()
			if err := c.ScanShard(ctx, shardID, fn); err != nil {
				select {
				case errc <- fmt.Errorf("shard %s error: %v", shardID, err):
					// first error to occur
					cancel()
				default:
					// error has already occurred
				}
			}
		}(aws.StringValue(shard.ShardId))
	}

	go func() {
		wg.Wait()
		close(errc)
	}()

	return <-errc
}

// ScanShard loops over records on a specific shard, calls the callback func
// for each record and checkpoints the progress of scan.
func (c *Consumer) ScanShard(ctx context.Context, shardID string, fn ScanFunc) error {
	// get last seq number from checkpoint
	lastSeqNum, err := c.group.GetCheckpoint(c.streamName, shardID)
	if err != nil {
		return fmt.Errorf("get checkpoint error: %v", err)
	}

	// get shard iterator
	shardIterator, err := c.getShardIterator(ctx, c.streamName, shardID, lastSeqNum)
	if err != nil {
		return fmt.Errorf("get shard iterator error: %v", err)
	}

	c.logger.Log("[CONSUMER] start scan:", shardID, lastSeqNum)
	defer func() {
		c.logger.Log("[CONSUMER] stop scan:", shardID)
	}()
	scanTicker := time.NewTicker(c.scanInterval)
	defer scanTicker.Stop()

	for {
		resp, err := c.client.GetRecords(&kinesis.GetRecordsInput{
			Limit:         aws.Int64(c.maxRecords),
			ShardIterator: shardIterator,
		})

		// attempt to recover from GetRecords error when expired iterator
		if err != nil {
			c.logger.Log("[CONSUMER] get records error:", err.Error())

			if awserr, ok := err.(awserr.Error); ok {
				if _, ok := retriableErrors[awserr.Code()]; !ok {
					return fmt.Errorf("get records error: %v", awserr.Message())
				}
			}

			shardIterator, err = c.getShardIterator(ctx, c.streamName, shardID, lastSeqNum)
			if err != nil {
				return fmt.Errorf("get shard iterator error: %v", err)
			}
		} else {
			// loop over records, call callback func
			for _, r := range resp.Records {
				select {
				case <-ctx.Done():
					return nil
				default:
					err := fn(&Record{r, shardID, resp.MillisBehindLatest})
					if err != nil && err != ErrSkipCheckpoint {
						return err
					}

					if err != ErrSkipCheckpoint {
						if err := c.group.SetCheckpoint(c.streamName, shardID, *r.SequenceNumber); err != nil {
							return err
						}
					}

					c.counter.Add("records", 1)
					lastSeqNum = *r.SequenceNumber
				}
			}

			if isShardClosed(resp.NextShardIterator, shardIterator) {
				c.logger.Log("[CONSUMER] shard closed:", shardID)
				return nil
			}

			shardIterator = resp.NextShardIterator
		}

		// Wait for next scan
		select {
		case <-ctx.Done():
			return nil
		case <-scanTicker.C:
			continue
		}
	}
}

var retriableErrors = map[string]struct{}{
	kinesis.ErrCodeExpiredIteratorException:               struct{}{},
	kinesis.ErrCodeProvisionedThroughputExceededException: struct{}{},
}

func isShardClosed(nextShardIterator, currentShardIterator *string) bool {
	return nextShardIterator == nil || currentShardIterator == nextShardIterator
}

func (c *Consumer) getShardIterator(ctx context.Context, streamName, shardID, seqNum string) (*string, error) {
	params := &kinesis.GetShardIteratorInput{
		ShardId:    aws.String(shardID),
		StreamName: aws.String(streamName),
	}

	if seqNum != "" {
		params.ShardIteratorType = aws.String(kinesis.ShardIteratorTypeAfterSequenceNumber)
		params.StartingSequenceNumber = aws.String(seqNum)
	} else if c.initialTimestamp != nil {
		params.ShardIteratorType = aws.String(kinesis.ShardIteratorTypeAtTimestamp)
		params.Timestamp = c.initialTimestamp
	} else {
		params.ShardIteratorType = aws.String(c.initialShardIteratorType)
	}

	res, err := c.client.GetShardIteratorWithContext(aws.Context(ctx), params)
	return res.ShardIterator, err
}
<|endoftext|>


<commit_before>package funnel

import (
	"bufio"
	"fmt"
	"io"
	"os"
	"os/signal"
	"path"
	"sync"
	"syscall"
	"time"
)

// Consumer is the main struct which holds all the stuff
// necessary to run the code
type Consumer struct {
	Config        *Config
	LineProcessor LineProcessor

	// internal stuff
	currFile *os.File
	writer   *bufio.Writer
	feed     chan string

	// channel signallers
	done         chan struct{}
	rolloverChan chan struct{}
	signalChan   chan os.Signal
	errChan      chan error
	wg           sync.WaitGroup

	// variable to track write progress
	linesWritten int
	bytesWritten uint64
}

// Start takes the input stream and begins reading line by line
// buffering the output to a file and flushing at set intervals
func (c *Consumer) Start(inputStream io.Reader) {
	c.setupSignalHandling()
	c.done = make(chan struct{})
	c.rolloverChan = make(chan struct{})
	c.errChan = make(chan error, 1)

	// Make the dir along with parents
	if err := os.MkdirAll(c.Config.DirName, 0775); err != nil {
		fmt.Fprintln(os.Stderr, err)
		return
	}

	// Create the file
	if err := c.createNewFile(); err != nil {
		fmt.Fprintln(os.Stderr, err)
		return
	}

	// Create the line feed channel and start the feed goroutine
	c.feed = make(chan string)
	go c.startFeed()

	// Get the reader to the input stream and set initial counters
	reader := bufio.NewReader(inputStream)
	c.linesWritten = 0
	c.bytesWritten = 0

	// start a for-select loop to wait until main loop is done, or catch errors
outer:
	for {
		select {
		case err := <-c.errChan: // error channel to get any errors happening
			// elsewhere. After printing to stderr, it breaks from the loop
			fmt.Println(os.Stderr, err)
			break outer
		default:
			// This will return a line until delimiter
			// If delimiter is not found, it returns the line with error
			// so line will always be available
			// Then we check for error and quit
			line, err := reader.ReadString('\n')
			// Send to feed
			c.feed <- line

			// Update counters
			c.linesWritten++
			c.bytesWritten += uint64(len(line))

			// Check for rollover
			if c.rollOverCondition() {
				c.rolloverChan <- struct{}{}
				c.linesWritten = 0
				c.bytesWritten = 0
			}

			if err != nil {
				if err != io.EOF {
					fmt.Fprintln(os.Stderr, err)
				}
				break outer
			}
		}
	}
	// work is done, signalling done channel
	c.wg.Add(1)
	c.done <- struct{}{}
	c.wg.Wait()
	// quitting from signal handler
	close(c.signalChan)
}

func (c *Consumer) cleanUp() {
	var err error
	// Close file handle
	if err = c.currFile.Sync(); err != nil {
		fmt.Fprintln(os.Stderr, err)
		return
	}

	if err = c.currFile.Close(); err != nil {
		fmt.Fprintln(os.Stderr, err)
		return
	}

	// Rename the currfile to a rolled up one
	var fileName string
	if err, fileName = c.rename(); err != nil {
		fmt.Fprintln(os.Stderr, err)
		return
	}

	if err = c.compress(fileName); err != nil {
		fmt.Fprintln(os.Stderr, err)
		return
	}
}

func (c *Consumer) createNewFile() error {
	f, err := os.OpenFile(path.Join(c.Config.DirName, c.Config.ActiveFileName),
		os.O_CREATE|os.O_WRONLY|os.O_TRUNC|os.O_APPEND|os.O_EXCL,
		0644)
	if err != nil {
		return err
	}
	c.currFile = f
	c.writer = bufio.NewWriter(c.currFile)
	return nil
}

func (c *Consumer) rollOverCondition() bool {
	// Return true if either lines written has exceeded
	// or bytes written has exceeded
	return c.linesWritten >= c.Config.RotationMaxLines ||
		c.bytesWritten >= c.Config.RotationMaxBytes
}

func (c *Consumer) rollOver() error {
	var err error
	// Flush writer
	if err = c.writer.Flush(); err != nil {
		return err
	}

	// Close file handle
	if err = c.currFile.Sync(); err != nil {
		return err
	}
	if err = c.currFile.Close(); err != nil {
		return err
	}

	var fileName string
	if err, fileName = c.rename(); err != nil {
		return err
	}

	if err = c.compress(fileName); err != nil {
		return err
	}

	if err = c.deleteFiles(); err != nil {
		return err
	}

	if err = c.createNewFile(); err != nil {
		return err
	}
	return nil
}

func (c *Consumer) rename() (error, string) {
	var fileName string
	var err error
	if c.Config.FileRenamePolicy == "timestamp" {
		err, fileName = renameFileTimestamp(c.Config)
		if err != nil {
			return err, ""
		}
	} else {
		err, fileName = renameFileSerial(c.Config)
		if err != nil {
			return err, ""
		}
	}
	return nil, fileName
}

func (c *Consumer) compress(fileName string) error {
	// Check config and compress if yes
	if c.Config.Gzip {
		err := gzipFile(path.Join(c.Config.DirName, fileName))
		return err
	}
	return nil
}

func (c *Consumer) deleteFiles() error {
	return deleteOldFiles(c.Config)
}

func (c *Consumer) startFeed() {
	// Will flush the writer at some intervals
	ticker := time.NewTicker(time.Duration(c.Config.FlushingTimeIntervalSecs) * time.Second)
	for {
		select {
		case line := <-c.feed: // Write to buffered writer
			err := c.LineProcessor.Write(c.writer, line)
			if err != nil {
				c.errChan <- err
			}
		case <-c.rolloverChan: // Rollover file to new one
			if err := c.rollOver(); err != nil {
				c.errChan <- err
			}
		case <-c.done: // Done signal received, close shop
			ticker.Stop()
			if err := c.writer.Flush(); err != nil {
				fmt.Fprintln(os.Stderr, err)
			}
			c.cleanUp()
			c.wg.Done()
			return
		case <-ticker.C: // If tick happens, flush the writer
			if err := c.writer.Flush(); err != nil {
				c.errChan <- err
			}
		}
	}
}

func (c *Consumer) setupSignalHandling() {
	c.signalChan = make(chan os.Signal, 1)
	signal.Notify(c.signalChan,
		os.Interrupt, syscall.SIGTERM)

	// Block until a signal is received.
	go func() {
		for range c.signalChan {
			// work is done, signalling done channel
			c.wg.Add(1)
			c.done <- struct{}{}
			c.wg.Wait()
			// Everything taken care of, goodbye
			os.Exit(1)

		}
	}()
}
<commit_msg>Fixing minor printing issue<commit_after>package funnel

import (
	"bufio"
	"fmt"
	"io"
	"os"
	"os/signal"
	"path"
	"sync"
	"syscall"
	"time"
)

// Consumer is the main struct which holds all the stuff
// necessary to run the code
type Consumer struct {
	Config        *Config
	LineProcessor LineProcessor

	// internal stuff
	currFile *os.File
	writer   *bufio.Writer
	feed     chan string

	// channel signallers
	done         chan struct{}
	rolloverChan chan struct{}
	signalChan   chan os.Signal
	errChan      chan error
	wg           sync.WaitGroup

	// variable to track write progress
	linesWritten int
	bytesWritten uint64
}

// Start takes the input stream and begins reading line by line
// buffering the output to a file and flushing at set intervals
func (c *Consumer) Start(inputStream io.Reader) {
	c.setupSignalHandling()
	c.done = make(chan struct{})
	c.rolloverChan = make(chan struct{})
	c.errChan = make(chan error, 1)

	// Make the dir along with parents
	if err := os.MkdirAll(c.Config.DirName, 0775); err != nil {
		fmt.Fprintln(os.Stderr, err)
		return
	}

	// Create the file
	if err := c.createNewFile(); err != nil {
		fmt.Fprintln(os.Stderr, err)
		return
	}

	// Create the line feed channel and start the feed goroutine
	c.feed = make(chan string)
	go c.startFeed()

	// Get the reader to the input stream and set initial counters
	reader := bufio.NewReader(inputStream)
	c.linesWritten = 0
	c.bytesWritten = 0

	// start a for-select loop to wait until main loop is done, or catch errors
outer:
	for {
		select {
		case err := <-c.errChan: // error channel to get any errors happening
			// elsewhere. After printing to stderr, it breaks from the loop
			fmt.Fprintln(os.Stderr, err)
			break outer
		default:
			// This will return a line until delimiter
			// If delimiter is not found, it returns the line with error
			// so line will always be available
			// Then we check for error and quit
			line, err := reader.ReadString('\n')
			// Send to feed
			c.feed <- line

			// Update counters
			c.linesWritten++
			c.bytesWritten += uint64(len(line))

			// Check for rollover
			if c.rollOverCondition() {
				c.rolloverChan <- struct{}{}
				c.linesWritten = 0
				c.bytesWritten = 0
			}

			if err != nil {
				if err != io.EOF {
					fmt.Fprintln(os.Stderr, err)
				}
				break outer
			}
		}
	}
	// work is done, signalling done channel
	c.wg.Add(1)
	c.done <- struct{}{}
	c.wg.Wait()
	// quitting from signal handler
	close(c.signalChan)
}

func (c *Consumer) cleanUp() {
	var err error
	// Close file handle
	if err = c.currFile.Sync(); err != nil {
		fmt.Fprintln(os.Stderr, err)
		return
	}

	if err = c.currFile.Close(); err != nil {
		fmt.Fprintln(os.Stderr, err)
		return
	}

	// Rename the currfile to a rolled up one
	var fileName string
	if err, fileName = c.rename(); err != nil {
		fmt.Fprintln(os.Stderr, err)
		return
	}

	if err = c.compress(fileName); err != nil {
		fmt.Fprintln(os.Stderr, err)
		return
	}
}

func (c *Consumer) createNewFile() error {
	f, err := os.OpenFile(path.Join(c.Config.DirName, c.Config.ActiveFileName),
		os.O_CREATE|os.O_WRONLY|os.O_TRUNC|os.O_APPEND|os.O_EXCL,
		0644)
	if err != nil {
		return err
	}
	c.currFile = f
	c.writer = bufio.NewWriter(c.currFile)
	return nil
}

func (c *Consumer) rollOverCondition() bool {
	// Return true if either lines written has exceeded
	// or bytes written has exceeded
	return c.linesWritten >= c.Config.RotationMaxLines ||
		c.bytesWritten >= c.Config.RotationMaxBytes
}

func (c *Consumer) rollOver() error {
	var err error
	// Flush writer
	if err = c.writer.Flush(); err != nil {
		return err
	}

	// Close file handle
	if err = c.currFile.Sync(); err != nil {
		return err
	}
	if err = c.currFile.Close(); err != nil {
		return err
	}

	var fileName string
	if err, fileName = c.rename(); err != nil {
		return err
	}

	if err = c.compress(fileName); err != nil {
		return err
	}

	if err = c.deleteFiles(); err != nil {
		return err
	}

	if err = c.createNewFile(); err != nil {
		return err
	}
	return nil
}

func (c *Consumer) rename() (error, string) {
	var fileName string
	var err error
	if c.Config.FileRenamePolicy == "timestamp" {
		err, fileName = renameFileTimestamp(c.Config)
		if err != nil {
			return err, ""
		}
	} else {
		err, fileName = renameFileSerial(c.Config)
		if err != nil {
			return err, ""
		}
	}
	return nil, fileName
}

func (c *Consumer) compress(fileName string) error {
	// Check config and compress if yes
	if c.Config.Gzip {
		err := gzipFile(path.Join(c.Config.DirName, fileName))
		return err
	}
	return nil
}

func (c *Consumer) deleteFiles() error {
	return deleteOldFiles(c.Config)
}

func (c *Consumer) startFeed() {
	// Will flush the writer at some intervals
	ticker := time.NewTicker(time.Duration(c.Config.FlushingTimeIntervalSecs) * time.Second)
	for {
		select {
		case line := <-c.feed: // Write to buffered writer
			err := c.LineProcessor.Write(c.writer, line)
			if err != nil {
				c.errChan <- err
			}
		case <-c.rolloverChan: // Rollover file to new one
			if err := c.rollOver(); err != nil {
				c.errChan <- err
			}
		case <-c.done: // Done signal received, close shop
			ticker.Stop()
			if err := c.writer.Flush(); err != nil {
				fmt.Fprintln(os.Stderr, err)
			}
			c.cleanUp()
			c.wg.Done()
			return
		case <-ticker.C: // If tick happens, flush the writer
			if err := c.writer.Flush(); err != nil {
				c.errChan <- err
			}
		}
	}
}

func (c *Consumer) setupSignalHandling() {
	c.signalChan = make(chan os.Signal, 1)
	signal.Notify(c.signalChan,
		os.Interrupt, syscall.SIGTERM)

	// Block until a signal is received.
	go func() {
		for range c.signalChan {
			// work is done, signalling done channel
			c.wg.Add(1)
			c.done <- struct{}{}
			c.wg.Wait()
			// Everything taken care of, goodbye
			os.Exit(1)

		}
	}()
}
<|endoftext|>


<commit_before>// Copyright 2016 Koichi Shiraishi. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

package commands

import (
	"bytes"
	"go/build"
	"go/format"
	"go/parser"
	"go/types"
	"io/ioutil"
	"os"
	"path/filepath"
	"time"

	"nvim-go/context"
	"nvim-go/nvim"
	"nvim-go/nvim/profile"

	"github.com/garyburd/neovim-go/vim"
	"github.com/garyburd/neovim-go/vim/plugin"
	"github.com/motemen/go-iferr"
	"golang.org/x/tools/go/loader"
)

func init() {
	plugin.HandleCommand("GoIferr", &plugin.CommandOptions{Eval: "expand('%:p')"}, cmdIferr)
}

func cmdIferr(v *vim.Vim, file string) {
	go Iferr(v, file)
}

// Iferr automatically insert 'if err' Go idiom by parse the current buffer's Go abstract syntax tree(AST).
func Iferr(v *vim.Vim, file string) error {
	defer profile.Start(time.Now(), "GoIferr")
	ctxt := new(context.Context)
	dir, _ := filepath.Split(file)
	defer ctxt.Build.SetContext(dir)()

	b, err := v.CurrentBuffer()
	if err != nil {
		return err
	}

	bufline, err := v.BufferLines(b, 0, -1, true)
	if err != nil {
		return err
	}

	var buf string
	for _, bufstr := range bufline {
		buf += "\n" + string(bufstr)
	}

	conf := loader.Config{
		ParserMode:  parser.ParseComments,
		TypeChecker: types.Config{FakeImportC: true, DisableUnusedImportCheck: false},
		Build:       &build.Default,
		Cwd:         dir,
		AllowErrors: true,
	}

	f, err := conf.ParseFile(file, buf)
	if err != nil {
		return nvim.Echoerr(v, "GoIferr: %v", err)
	}

	conf.CreateFromFiles(file, f)
	prog, err := conf.Load()
	if err != nil {
		return err
	}

	saveStdout := os.Stdout
	r, w, _ := os.Pipe()
	os.Stdout = w

	for _, pkg := range prog.InitialPackages() {
		for _, f := range pkg.Files {
			iferr.RewriteFile(prog.Fset, f, pkg.Info)
			format.Node(w, prog.Fset, f)
		}
	}

	w.Close()
	os.Stdout = saveStdout

	out, err := ioutil.ReadAll(r)
	if err != nil {
		return err
	}

	return v.SetBufferLines(b, 0, -1, true, bytes.Split(out, []byte{'\n'}))
}
<commit_msg>cmds/iferr: Remove use os.Stdout and Enable DisableUnusedImportCheck<commit_after>// Copyright 2016 Koichi Shiraishi. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

package commands

import (
	"bytes"
	"go/build"
	"go/format"
	"go/parser"
	"go/types"
	"path/filepath"
	"time"

	"nvim-go/context"
	"nvim-go/nvim"
	"nvim-go/nvim/profile"

	"github.com/garyburd/neovim-go/vim"
	"github.com/garyburd/neovim-go/vim/plugin"
	"github.com/motemen/go-iferr"
	"golang.org/x/tools/go/loader"
)

func init() {
	plugin.HandleCommand("GoIferr", &plugin.CommandOptions{Eval: "expand('%:p')"}, cmdIferr)
}

func cmdIferr(v *vim.Vim, file string) {
	go Iferr(v, file)
}

// Iferr automatically insert 'if err' Go idiom by parse the current buffer's Go abstract syntax tree(AST).
func Iferr(v *vim.Vim, file string) error {
	defer profile.Start(time.Now(), "GoIferr")

	dir := filepath.Dir(file)
	ctxt := new(context.Context)
	defer ctxt.Build.SetContext(dir)()

	b, err := v.CurrentBuffer()
	if err != nil {
		return err
	}

	buflines, err := v.BufferLines(b, 0, -1, true)
	if err != nil {
		return err
	}

	conf := loader.Config{
		ParserMode:  parser.ParseComments,
		TypeChecker: types.Config{FakeImportC: true, DisableUnusedImportCheck: true},
		Build:       &build.Default,
		Cwd:         dir,
		AllowErrors: true,
	}

	var src bytes.Buffer
	src.Write(nvim.ToByteSlice(buflines))

	f, err := conf.ParseFile(file, src.Bytes())
	if err != nil {
		return nvim.Echoerr(v, "GoIferr: %v", err)
	}

	conf.CreateFromFiles(file, f)
	prog, err := conf.Load()
	if err != nil {
		return err
	}

	// Reuse src variable
	src.Reset()

	for _, pkg := range prog.InitialPackages() {
		for _, f := range pkg.Files {
			iferr.RewriteFile(prog.Fset, f, pkg.Info)
			format.Node(&src, prog.Fset, f)
		}
	}

	// format.Node() will added pointless newline
	buf := bytes.TrimSuffix(src.Bytes(), []byte{'\n'})
	return v.SetBufferLines(b, 0, -1, true, nvim.ToBufferLines(buf))
}
<|endoftext|>


<commit_before>package core

import (
	"errors"
	"fmt"
	"image"
	"image/color"
	"image/png"
	"io/ioutil"
	"math"
	"os"
	"time"
)

// The Pxl struct definition
type Pxl struct {
	IsEncodeMode   bool
	IsDecodeMode   bool
	IsDebugMode    bool
	Source         string
	Target         string
	encodedPayload image.Image
	decodedPayload []byte
}

// Checks the context on the Pxl struct
// Encode the Source if Pxl.IsEncodeMode
// Decode the Source if Pxl.IsDecodeMode
func (p Pxl) Process() (bool, error) {

	if p.IsEncodeMode {
		if err := p.Encode(); err != nil {
			return false, err
		}
		p.DebugString()

		f, err := os.OpenFile(p.Target, os.O_WRONLY|os.O_CREATE, 0600)

		if err != nil {
			panic(err)
		}

		defer f.Close()
		//******************************************
		start := time.Now()
		//==========================================
		png.Encode(f, p.encodedPayload)
		//******************************************
		elapsed := time.Since(start)
		fmt.Printf("encoding to png: %s\n", elapsed)
		//==========================================
	}

	if p.IsDecodeMode {
		if err := p.Decode(); err != nil {
			return false, err
		}

		err := ioutil.WriteFile(p.Target, p.decodedPayload, 0644)
		if err != nil {
			return false, err
		}
	}

	DisplayDebug(p.DebugString("DONE"))

	return true, nil
}

// Encodes the Pxl.Source and stores it to Pxl.encodedPayload
func (p *Pxl) Encode() error {

	f, err := os.OpenFile(p.Source, os.O_RDONLY, 0444)
	defer f.Close()
	if err != nil {
		return err
	}

	finfo, err := os.Stat(p.Source)
	if err != nil {
		return err
	}

	dimensions := int(math.Sqrt(float64(finfo.Size()/4))) + 1

	//******************************************
	start := time.Now()
	//==========================================

	x := 0
	y := 0

	//create image with dimensions x dimensions
	img := image.NewNRGBA((image.Rect(0, 0, dimensions, dimensions)))

	//fillPx := color.NRGBA{0, 0, 0, 255}
	//draw.Draw(img, img.Bounds(), &image.Uniform{fillPx}, image.ZP, draw.Src)

	var buffer = make([]byte, 838860800)
	tmp := make([]byte, 4)

	for {
		num, err := f.Read(buffer)

		if err != errors.New("EOF") && num == 0 {
			break
		} else if num == 0 {
			break
		} else if err != nil {
			return err
		}

		//loop msg bytes
		for pos := 0; pos < num; pos += 4 {
			for p := 0; p < 4; p++ {
				if len(buffer) < pos+p {
					tmp[p] = byte(255)
				} else {
					tmp[p] = buffer[pos+p]
				}
			}
			img.Set(x, y, color.NRGBA{tmp[pos%4], tmp[pos%4+1], tmp[pos%4+2], tmp[pos%4+3]})
			x++
			if x >= dimensions {
				y++
				x = 0
			}
		}
	}

	for posY := y; posY < dimensions; posY++ {
		for posX := x; posX < dimensions; posX++ {
			img.Set(posX, posY, color.NRGBA{0, 0, 0, 255})
		}
	}

	//******************************************
	elapsed := time.Since(start)
	fmt.Printf("generating image: %s\n", elapsed)
	//==========================================

	p.encodedPayload = img
	return nil
}

// Decoded the Pxl.Source and stores it to Pxl.decodedPayload
func (p *Pxl) Decode() error {
	img, err := loadImage(p.Source)
	if err != nil {
		return err
	}

	for _, i := range img.Pix {
		p.decodedPayload = append(p.decodedPayload, i)
	}
	return nil
}

// Load image from filesystem
func loadImage(path string) (*image.NRGBA, error) {
	file, err := os.Open(path)
	if err != nil {
		return nil, err
	}
	defer file.Close()

	img, err := png.Decode(file)
	if err != nil {
		return nil, err
	}
	return img.(*image.NRGBA), nil
}

// Outputs a petty presentation of the Pxl struct
func (p Pxl) DebugString(msg ...string) string {
	x, y := 0, 0
	if p.encodedPayload != nil {
		x = p.encodedPayload.Bounds().Max.X
		y = p.encodedPayload.Bounds().Max.Y
	}

	if len(msg) <= 0 {
		msg = append(msg, time.Now().String())
	}

	return fmt.Sprintf(`%s
isDebugMode     : %t
isEncode        : %t
isDecode        : %t
source          : %s
target          : %s
encodedPayload  : %d x %d
decodedPayload  : %d`,
		msg,
		p.IsEncodeMode,
		p.IsDecodeMode,
		p.IsDebugMode,
		p.Source,
		p.Target,
		x, y,
		len(p.decodedPayload),
	)
}
<commit_msg>removed useless benchmarking<commit_after>package core

import (
	"errors"
	"fmt"
	"image"
	"image/color"
	"image/png"
	"io/ioutil"
	"math"
	"os"
	"time"
)

// The Pxl struct definition
type Pxl struct {
	IsEncodeMode   bool
	IsDecodeMode   bool
	IsDebugMode    bool
	Source         string
	Target         string
	encodedPayload image.Image
	decodedPayload []byte
}

// Checks the context on the Pxl struct
// Encode the Source if Pxl.IsEncodeMode
// Decode the Source if Pxl.IsDecodeMode
func (p Pxl) Process() (bool, error) {

	if p.IsEncodeMode {
		if err := p.Encode(); err != nil {
			return false, err
		}
		p.DebugString()

		f, err := os.OpenFile(p.Target, os.O_WRONLY|os.O_CREATE, 0600)

		if err != nil {
			panic(err)
		}

		defer f.Close()
		//******************************************
		start := time.Now()
		//==========================================
		png.Encode(f, p.encodedPayload)
		//******************************************
		elapsed := time.Since(start)
		fmt.Printf("encoding to png: %s\n", elapsed)
		//==========================================
	}

	if p.IsDecodeMode {
		if err := p.Decode(); err != nil {
			return false, err
		}

		err := ioutil.WriteFile(p.Target, p.decodedPayload, 0644)
		if err != nil {
			return false, err
		}
	}

	DisplayDebug(p.DebugString("DONE"))

	return true, nil
}

// Encodes the Pxl.Source and stores it to Pxl.encodedPayload
func (p *Pxl) Encode() error {

	f, err := os.OpenFile(p.Source, os.O_RDONLY, 0444)
	defer f.Close()
	if err != nil {
		return err
	}

	finfo, err := os.Stat(p.Source)
	if err != nil {
		return err
	}

	dimensions := int(math.Sqrt(float64(finfo.Size()/4))) + 1

	x := 0
	y := 0

	//create image with dimensions x dimensions
	img := image.NewNRGBA((image.Rect(0, 0, dimensions, dimensions)))

	//fillPx := color.NRGBA{0, 0, 0, 255}
	//draw.Draw(img, img.Bounds(), &image.Uniform{fillPx}, image.ZP, draw.Src)

	var buffer = make([]byte, 838860800)
	tmp := make([]byte, 4)

	for {
		num, err := f.Read(buffer)

		if err != errors.New("EOF") && num == 0 {
			break
		} else if num == 0 {
			break
		} else if err != nil {
			return err
		}

		//loop msg bytes
		for pos := 0; pos < num; pos += 4 {
			for p := 0; p < 4; p++ {
				if len(buffer) < pos+p {
					tmp[p] = byte(255)
				} else {
					tmp[p] = buffer[pos+p]
				}
			}
			img.Set(x, y, color.NRGBA{tmp[pos%4], tmp[pos%4+1], tmp[pos%4+2], tmp[pos%4+3]})
			x++
			if x >= dimensions {
				y++
				x = 0
			}
		}
	}

	for posY := y; posY < dimensions; posY++ {
		for posX := x; posX < dimensions; posX++ {
			img.Set(posX, posY, color.NRGBA{0, 0, 0, 255})
		}
	}

	p.encodedPayload = img
	return nil
}

// Decoded the Pxl.Source and stores it to Pxl.decodedPayload
func (p *Pxl) Decode() error {
	img, err := loadImage(p.Source)
	if err != nil {
		return err
	}

	for _, i := range img.Pix {
		p.decodedPayload = append(p.decodedPayload, i)
	}
	return nil
}

// Load image from filesystem
func loadImage(path string) (*image.NRGBA, error) {
	file, err := os.Open(path)
	if err != nil {
		return nil, err
	}
	defer file.Close()

	img, err := png.Decode(file)
	if err != nil {
		return nil, err
	}
	return img.(*image.NRGBA), nil
}

// Outputs a petty presentation of the Pxl struct
func (p Pxl) DebugString(msg ...string) string {
	x, y := 0, 0
	if p.encodedPayload != nil {
		x = p.encodedPayload.Bounds().Max.X
		y = p.encodedPayload.Bounds().Max.Y
	}

	if len(msg) <= 0 {
		msg = append(msg, time.Now().String())
	}

	return fmt.Sprintf(`%s
isDebugMode     : %t
isEncode        : %t
isDecode        : %t
source          : %s
target          : %s
encodedPayload  : %d x %d
decodedPayload  : %d`,
		msg,
		p.IsEncodeMode,
		p.IsDecodeMode,
		p.IsDebugMode,
		p.Source,
		p.Target,
		x, y,
		len(p.decodedPayload),
	)
}
<|endoftext|>


<commit_before>package faroo

import (
	"appengine"
	"appengine/urlfetch"

 	"bytes"
	"encoding/json"
	"fmt"
	"io/ioutil"
	"net/http"
	"strings"
)

type Related struct {
	Title	string `json:"title"`
	Url	string `json:"url"`
	Domain	string `json:"domain"`
}

type Entry struct {
	Title	string `json:"title"`
	Kwic	string `json:"kwic"`
	Content string `json:"content"`
	Url	string `json:"url"`
	Iurl	string `json:"iurl"`
	Domain	string `json:"domain"`
	Authoor string `json:"author"`
	Date	int64 `json:"date"`
}

type EntryList struct {
	Results []Entry `json:"results"`
	Query	string `json:"query"`
	Suggestions []string `json:"suggestions"`
	Count int `json:"count"`
	Start int `json:"start"`
	Length int `json:"length"`
	Time string `json:"time"`
}

type PushToken struct {
	CreatedAt string `json:"createdAt"`
	CustomizedTopic1 string  `json:"mCustomizedTopic1"`
	CustomizedTopic2 string `json:"mCustomizedTopic2"`
	CustomizedTopic3 string `json:"mCustomizedTopic3"`
	Language string  `json:"mLanguage"`
	DeviceId string `json:"mDeviceId"`
	GoogleId string `json:"mGoogleId"`
	PushToken string `json:"mPushToken"`
	ObjectId string `json:"objectId"`
	UpdatedAt string `json:"updatedAt"`
}

type PushTokenList struct {
	Results []PushToken  `json:"results"`
}

type Error string

func (e Error) Error() string {
	return string(e)
}

func init() {
		http.HandleFunc("/pushEn", pushHandleEnglish)
		http.HandleFunc("/pushDe", pushHandleGerman)
		http.HandleFunc("/pushZh", pushHandleChinese)
		http.HandleFunc("/pushCus", pushHandleCus)
}

func pushHandleEnglish(w http.ResponseWriter, r *http.Request ) {
	pushHandle(w, r, TOPIC_LIST_EN)
}

func pushHandleGerman(w http.ResponseWriter, r *http.Request ) {
	pushHandle(w, r, TOPIC_LIST_DE)
}

func pushHandleChinese(w http.ResponseWriter, r *http.Request ) {
	pushHandle(w, r, TOPIC_LIST_ZH)
}

func pushHandleCus(w http.ResponseWriter, r *http.Request ) {
	defer func() {
		if err := recover(); err != nil {
			s := fmt.Sprintf(`{"status":%d }`, 300)
			w.Header().Set("Content-Type", API_RESTYPE)
			fmt.Fprintf(w, s)
		}
	}()

	ch := make(chan string)
	go doCus(r, ch)
	<-ch
}

//A push handler on all languages which will be used by calling http://your-app.appspot.com/push
func pushHandle(w http.ResponseWriter, r *http.Request, topicList map[string]Topic ) {
	defer func() {
		if err := recover(); err != nil {
			s := fmt.Sprintf(`{"status":%d }`, 300)
			w.Header().Set("Content-Type", API_RESTYPE)
			fmt.Fprintf(w, s)
		}
	}()

	topicCount := len(topicList)
	ch := make(chan string, topicCount)
    for k, v := range topicList {
        go doPush(r,k,v, ch)
    }
	var pushedResult string = ""
	for i:=0; i < topicCount; i++ {
		pushedResult += (<-ch)
	}
	if pushedResult != "" {
		pushedResult = pushedResult[:len(pushedResult)-1]
	}
	output := fmt.Sprintf(`{"status":%d, "pushed:":[%s] }`, 200, pushedResult)
	w.Header().Set("Content-Type", API_RESTYPE)
	fmt.Fprintf(w, output)
}

func doPush(r *http.Request, api string, topic Topic, ch chan string) {
	var pushedContent string = ""
	pRes := getEntryList(r, topic.LocalName, topic.Language)
	topicApi := (TOPICS + api)
	if pRes != nil && pRes.Results != nil &&  len(pRes.Results) > 0 {
		entry := pRes.Results[0]

		entry.Title = strings.Replace(entry.Title, "\"", "'", -1)
		entry.Title = strings.Replace(entry.Title, "%", "％", -1)
		entry.Kwic = strings.Replace(entry.Kwic, "\"", "'", -1)
		entry.Kwic = strings.Replace(entry.Kwic, "%", "％", -1)

		resBytes, _ := json.Marshal(entry)
		data := string(resBytes)
		pushed := push(r, topicApi, data, true)
		pushedContent += pushed
	} else {
		pushedContent += (fmt.Sprintf(`"failed in %s"`, topicApi))
	}
	pushedContent += ","
	ch <- pushedContent
}

func doCus(r *http.Request, ch chan string) {
	getPushTokens(r)
	ch<-"end"
}

//Get all push-tokens of clients inc. the customized topics user subscribed.
func getPushTokens(r *http.Request)(pRes *PushTokenList) {
	cxt := appengine.NewContext(r)
	url := fmt.Sprintf(FAROO_API, query, lang)
	if req, err := http.NewRequest("GET", url, nil); err == nil {
		httpClient := urlfetch.Client(cxt)
		r, err := httpClient.Do(req)
		if r != nil {
			defer r.Body.Close()
		}
		if err == nil {
			if bytes, err := ioutil.ReadAll(r.Body); err == nil {
				pRes = new(PushTokenList)
				json.Unmarshal(bytes, pRes)
				cxt.Infof("json: %s", string(bytes))
			} else {
				cxt.Errorf("getPushTokens unmarshal: %v", err)
				pRes = nil
			}
		} else {
			cxt.Errorf("getPushTokens doing: %v", err)
			pRes = nil
		}
	} else {
		cxt.Errorf("getPushTokens: %v", err)
		pRes = nil
	}
	return
}

//Get a news-entry and channel it.
func getEntryList(r *http.Request, query string, lang string)(pRes *EntryList) {
	cxt := appengine.NewContext(r)
	if strings.Contains(query, "global") {
		query = ""
	}
	url := fmt.Sprintf(FAROO_API, query, lang)
	if req, err := http.NewRequest("GET", url + "&rlength=0", nil); err == nil {
		httpClient := urlfetch.Client(cxt)
		r, err := httpClient.Do(req)
		if r != nil {
			defer r.Body.Close()
		}
		if err == nil {
			if bytes, err := ioutil.ReadAll(r.Body); err == nil {
				pRes = new(EntryList)
				json.Unmarshal(bytes, pRes)
			} else {
				cxt.Errorf("getEntryList unmarshal: %v", err)
				pRes = nil
			}
		} else {
			cxt.Errorf("getEntryList doing: %v", err)
			pRes = nil
		}
	} else {
		cxt.Errorf("getEntryList: %v", err)
		pRes = nil
	}
	return
}

//Make push
func push(r *http.Request, topicApi string, data string, scheduledTask bool) (push string) {
	body := fmt.Sprintf(`{"to" : "%s","data" : %s}`, topicApi, data)
	bodyBytes := bytes.NewBufferString(body)
	cxt := appengine.NewContext(r)
	if req, err := http.NewRequest("POST", PUSH_SENDER, bodyBytes); err == nil {
		req.Header.Add("Authorization", PUSH_KEY)
		req.Header.Add("Content-Type", API_RESTYPE)
		if scheduledTask {
			req.Header.Add("X-AppEngine-Cron", "true")
		}
		client := urlfetch.Client(cxt)
		res, _ := client.Do(req)
		if res != nil {
			defer res.Body.Close()
		}
		if err != nil {
			cxt.Errorf("Push summary doing: %v", err)
			push =  ""
		} else {
			cxt.Infof(body)
			push = body
		}
	} else {
		cxt.Errorf("Push summary: %v", err)
		push = ""
	}
	return
}

 <commit_msg>config cus-api.<commit_after>package faroo

import (
	"appengine"
	"appengine/urlfetch"

 	"bytes"
	"encoding/json"
	"fmt"
	"io/ioutil"
	"net/http"
	"strings"
)

type Related struct {
	Title	string `json:"title"`
	Url	string `json:"url"`
	Domain	string `json:"domain"`
}

type Entry struct {
	Title	string `json:"title"`
	Kwic	string `json:"kwic"`
	Content string `json:"content"`
	Url	string `json:"url"`
	Iurl	string `json:"iurl"`
	Domain	string `json:"domain"`
	Authoor string `json:"author"`
	Date	int64 `json:"date"`
}

type EntryList struct {
	Results []Entry `json:"results"`
	Query	string `json:"query"`
	Suggestions []string `json:"suggestions"`
	Count int `json:"count"`
	Start int `json:"start"`
	Length int `json:"length"`
	Time string `json:"time"`
}

type PushToken struct {
	CreatedAt string `json:"createdAt"`
	CustomizedTopic1 string  `json:"mCustomizedTopic1"`
	CustomizedTopic2 string `json:"mCustomizedTopic2"`
	CustomizedTopic3 string `json:"mCustomizedTopic3"`
	Language string  `json:"mLanguage"`
	DeviceId string `json:"mDeviceId"`
	GoogleId string `json:"mGoogleId"`
	PushToken string `json:"mPushToken"`
	ObjectId string `json:"objectId"`
	UpdatedAt string `json:"updatedAt"`
}

type PushTokenList struct {
	Results []PushToken  `json:"results"`
}

type Error string

func (e Error) Error() string {
	return string(e)
}

func init() {
		http.HandleFunc("/pushEn", pushHandleEnglish)
		http.HandleFunc("/pushDe", pushHandleGerman)
		http.HandleFunc("/pushZh", pushHandleChinese)
		http.HandleFunc("/pushCus", pushHandleCus)
}

func pushHandleEnglish(w http.ResponseWriter, r *http.Request ) {
	pushHandle(w, r, TOPIC_LIST_EN)
}

func pushHandleGerman(w http.ResponseWriter, r *http.Request ) {
	pushHandle(w, r, TOPIC_LIST_DE)
}

func pushHandleChinese(w http.ResponseWriter, r *http.Request ) {
	pushHandle(w, r, TOPIC_LIST_ZH)
}

func pushHandleCus(w http.ResponseWriter, r *http.Request ) {
	defer func() {
		if err := recover(); err != nil {
			s := fmt.Sprintf(`{"status":%d }`, 300)
			w.Header().Set("Content-Type", API_RESTYPE)
			fmt.Fprintf(w, s)
		}
	}()

	ch := make(chan string)
	go doCus(r, ch)
	<-ch
}

//A push handler on all languages which will be used by calling http://your-app.appspot.com/push
func pushHandle(w http.ResponseWriter, r *http.Request, topicList map[string]Topic ) {
	defer func() {
		if err := recover(); err != nil {
			s := fmt.Sprintf(`{"status":%d }`, 300)
			w.Header().Set("Content-Type", API_RESTYPE)
			fmt.Fprintf(w, s)
		}
	}()

	topicCount := len(topicList)
	ch := make(chan string, topicCount)
    for k, v := range topicList {
        go doPush(r,k,v, ch)
    }
	var pushedResult string = ""
	for i:=0; i < topicCount; i++ {
		pushedResult += (<-ch)
	}
	if pushedResult != "" {
		pushedResult = pushedResult[:len(pushedResult)-1]
	}
	output := fmt.Sprintf(`{"status":%d, "pushed:":[%s] }`, 200, pushedResult)
	w.Header().Set("Content-Type", API_RESTYPE)
	fmt.Fprintf(w, output)
}

func doPush(r *http.Request, api string, topic Topic, ch chan string) {
	var pushedContent string = ""
	pRes := getEntryList(r, topic.LocalName, topic.Language)
	topicApi := (TOPICS + api)
	if pRes != nil && pRes.Results != nil &&  len(pRes.Results) > 0 {
		entry := pRes.Results[0]

		entry.Title = strings.Replace(entry.Title, "\"", "'", -1)
		entry.Title = strings.Replace(entry.Title, "%", "％", -1)
		entry.Kwic = strings.Replace(entry.Kwic, "\"", "'", -1)
		entry.Kwic = strings.Replace(entry.Kwic, "%", "％", -1)

		resBytes, _ := json.Marshal(entry)
		data := string(resBytes)
		pushed := push(r, topicApi, data, true)
		pushedContent += pushed
	} else {
		pushedContent += (fmt.Sprintf(`"failed in %s"`, topicApi))
	}
	pushedContent += ","
	ch <- pushedContent
}

func doCus(r *http.Request, ch chan string) {
	getPushTokens(r)
	ch<-"end"
}

//Get all push-tokens of clients inc. the customized topics user subscribed.
func getPushTokens(r *http.Request)(pRes *PushTokenList) {
	cxt := appengine.NewContext(r)
	url := DB_PATH + DB_PUSH_TOKEN_TAB
	if req, err := http.NewRequest("GET", url, nil); err == nil {
		req.Header.Add(DB_HEADER_APP_ID, DB_APP_ID)
		req.Header.Add(DB_HEADER_API_KEY, DB_API_KEY)
		httpClient := urlfetch.Client(cxt)
		r, err := httpClient.Do(req)
		if r != nil {
			defer r.Body.Close()
		}
		if err == nil {
			if bytes, err := ioutil.ReadAll(r.Body); err == nil {
				pRes = new(PushTokenList)
				json.Unmarshal(bytes, pRes)
				cxt.Infof("json: %s", string(bytes))
			} else {
				cxt.Errorf("getPushTokens unmarshal: %v", err)
				pRes = nil
			}
		} else {
			cxt.Errorf("getPushTokens doing: %v", err)
			pRes = nil
		}
	} else {
		cxt.Errorf("getPushTokens: %v", err)
		pRes = nil
	}
	return
}

//Get a news-entry and channel it.
func getEntryList(r *http.Request, query string, lang string)(pRes *EntryList) {
	cxt := appengine.NewContext(r)
	if strings.Contains(query, "global") {
		query = ""
	}
	url := fmt.Sprintf(FAROO_API, query, lang)
	if req, err := http.NewRequest("GET", url + "&rlength=0", nil); err == nil {
		httpClient := urlfetch.Client(cxt)
		r, err := httpClient.Do(req)
		if r != nil {
			defer r.Body.Close()
		}
		if err == nil {
			if bytes, err := ioutil.ReadAll(r.Body); err == nil {
				pRes = new(EntryList)
				json.Unmarshal(bytes, pRes)
			} else {
				cxt.Errorf("getEntryList unmarshal: %v", err)
				pRes = nil
			}
		} else {
			cxt.Errorf("getEntryList doing: %v", err)
			pRes = nil
		}
	} else {
		cxt.Errorf("getEntryList: %v", err)
		pRes = nil
	}
	return
}

//Make push
func push(r *http.Request, topicApi string, data string, scheduledTask bool) (push string) {
	body := fmt.Sprintf(`{"to" : "%s","data" : %s}`, topicApi, data)
	bodyBytes := bytes.NewBufferString(body)
	cxt := appengine.NewContext(r)
	if req, err := http.NewRequest("POST", PUSH_SENDER, bodyBytes); err == nil {
		req.Header.Add("Authorization", PUSH_KEY)
		req.Header.Add("Content-Type", API_RESTYPE)
		if scheduledTask {
			req.Header.Add("X-AppEngine-Cron", "true")
		}
		client := urlfetch.Client(cxt)
		res, _ := client.Do(req)
		if res != nil {
			defer res.Body.Close()
		}
		if err != nil {
			cxt.Errorf("Push summary doing: %v", err)
			push =  ""
		} else {
			cxt.Infof(body)
			push = body
		}
	} else {
		cxt.Errorf("Push summary: %v", err)
		push = ""
	}
	return
}

 <|endoftext|>


<commit_before>package main

/*
 * Copyright 2014 Albert P. Tobey <atobey@datastax.com> @AlTobey
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 *     http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 *
 * cqlstore.go: a quick & dirty Cassandra CQL backend for Gorilla sessions
 *
 */

import (
	"encoding/gob"
	"fmt"
	"github.com/gocql/gocql"
	"github.com/gorilla/securecookie"
	"github.com/gorilla/sessions"
	"log"
	"net/http"
	"time"
)

type CQLStore struct {
	Cass    *gocql.Session       // connected gocql cassandra session
	Codecs  []securecookie.Codec // session codecs
	Options *sessions.Options    // default configuration
}

func init() {
	gob.Register(time.Now())
}

func NewCQLStore(cass *gocql.Session, keyPairs ...[]byte) *CQLStore {
	return &CQLStore{
		Cass:   cass,
		Codecs: securecookie.CodecsFromPairs(keyPairs...),
		Options: &sessions.Options{
			Path:   "/",
			MaxAge: 86400 * 30,
		},
	}
}

func (cs *CQLStore) Get(r *http.Request, name string) (*sessions.Session, error) {
	return sessions.GetRegistry(r).Get(cs, name)
}

func (cs *CQLStore) New(r *http.Request, name string) (sess *sessions.Session, err error) {
	sess = sessions.NewSession(cs, name)
	sess.IsNew = true
	opts := *cs.Options // make a copy
	sess.Options = &opts

	// load the cookie (if it exists)
	c, err := r.Cookie(name)
	if err == nil {
		err = securecookie.DecodeMulti(name, c.Value, &sess.ID, cs.Codecs...)
		if err != nil {
			log.Printf("Cookie decode failed: %s\n", err)
			return
		}
	}

	var data []byte
	var created, modified time.Time
	query := `SELECT data, created, modified FROM sessions WHERE id=?`
	iq := cs.Cass.Query(query, sess.ID).Iter()
	ok := iq.Scan(&data, &created, &modified)
	if ok {
		sess.IsNew = false
		sess.Values["created"] = created
		sess.Values["modified"] = modified
	}

	return
}

func (cs *CQLStore) Save(r *http.Request, w http.ResponseWriter, sess *sessions.Session) (err error) {
	now := time.Now()

	// generate a uuid if there isn't an id already
	if sess.ID == "" {
		var id gocql.UUID
		id, err = gocql.RandomUUID()
		if err != nil {
			return
		}
		sess.ID = id.String()
	}

	// serialize the session for storage in cassandra
	blob, err := securecookie.EncodeMulti(sess.Name(), sess.Values, cs.Codecs...)
	if err != nil {
		log.Printf("Failed to encode session for storage in Cassandra: %s\n", err)
		return
	}

	var created, modified time.Time
	if sess.IsNew {
		created = now
	} else {
		created = sess.Values["created"].(time.Time)
	}
	modified = now

	query := `INSERT INTO sessions (id, data, created, modified) VALUES (?, ?, ?, ?)`
	err = cs.Cass.Query(query, sess.ID, blob, created, modified).Exec()
	if err != nil {
		fmt.Printf("Failed to save session to Cassandra: %s\n", err)
		return
	}

	// update the cookie
	cdata, err := securecookie.EncodeMulti(sess.Name(), sess.ID, cs.Codecs...)
	if err != nil {
		log.Printf("Failed to encode session for the cookie: %s\n", err)
		return
	}
	http.SetCookie(w, sessions.NewCookie(sess.Name(), cdata, sess.Options))

	return nil
}

func (cs *CQLStore) Delete(r *http.Request, w http.ResponseWriter, sess *sessions.Session) (err error) {
	// overwrite the cookie with a negative max age so the browser expires it immediately
	opts := *sess.Options
	opts.MaxAge = -1
	http.SetCookie(w, sessions.NewCookie(sess.Name(), "", &opts))

	// delete the session from the DB
	err = cs.Cass.Query(`DELETE FROM sessions WHERE id=?`, sess.ID).Exec()
	return
}
<commit_msg>remove superflous cookie encoding<commit_after>package main

/*
 * Copyright 2014 Albert P. Tobey <atobey@datastax.com> @AlTobey
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 *     http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 *
 * cqlstore.go: a quick & dirty Cassandra CQL backend for Gorilla sessions
 *
 */

import (
	"encoding/gob"
	"fmt"
	"github.com/gocql/gocql"
	"github.com/gorilla/securecookie"
	"github.com/gorilla/sessions"
	"log"
	"net/http"
	"time"
)

type CQLStore struct {
	Cass    *gocql.Session       // connected gocql cassandra session
	Codecs  []securecookie.Codec // session codecs
	Options *sessions.Options    // default configuration
}

func init() {
	gob.Register(time.Now())
}

func NewCQLStore(cass *gocql.Session, keyPairs ...[]byte) *CQLStore {
	return &CQLStore{
		Cass:   cass,
		Codecs: securecookie.CodecsFromPairs(keyPairs...),
		Options: &sessions.Options{
			Path:   "/",
			MaxAge: 86400 * 30,
		},
	}
}

func (cs *CQLStore) Get(r *http.Request, name string) (*sessions.Session, error) {
	return sessions.GetRegistry(r).Get(cs, name)
}

func (cs *CQLStore) New(r *http.Request, name string) (sess *sessions.Session, err error) {
	sess = sessions.NewSession(cs, name)
	sess.IsNew = true
	opts := *cs.Options // make a copy
	sess.Options = &opts

	// load the cookie (if it exists)
	c, err := r.Cookie(name)
	if err == nil {
		err = securecookie.DecodeMulti(name, c.Value, &sess.ID, cs.Codecs...)
		if err != nil {
			log.Printf("Cookie decode failed: %s\n", err)
			return
		}
	}

	var data []byte
	var created, modified time.Time
	query := `SELECT data, created, modified FROM sessions WHERE id=?`
	iq := cs.Cass.Query(query, sess.ID).Iter()
	ok := iq.Scan(&data, &created, &modified)
	if ok {
		sess.IsNew = false
		sess.Values["created"] = created
		sess.Values["modified"] = modified
	}

	return
}

func (cs *CQLStore) Save(r *http.Request, w http.ResponseWriter, sess *sessions.Session) (err error) {
	now := time.Now()

	// generate a uuid if there isn't an id already
	if sess.ID == "" {
		var id gocql.UUID
		id, err = gocql.RandomUUID()
		if err != nil {
			return
		}
		sess.ID = id.String()
	}

	// serialize the session for storage in cassandra
	blob, err := securecookie.EncodeMulti(sess.Name(), sess.Values, cs.Codecs...)
	if err != nil {
		log.Printf("Failed to encode session for storage in Cassandra: %s\n", err)
		return
	}

	var created, modified time.Time
	if sess.IsNew {
		created = now
	} else {
		created = sess.Values["created"].(time.Time)
	}
	modified = now

	query := `INSERT INTO sessions (id, data, created, modified) VALUES (?, ?, ?, ?)`
	err = cs.Cass.Query(query, sess.ID, blob, created, modified).Exec()
	if err != nil {
		fmt.Printf("Failed to save session to Cassandra: %s\n", err)
		return
	}

	// update the cookie
	http.SetCookie(w, sessions.NewCookie(sess.Name(), blob, sess.Options))

	return nil
}

func (cs *CQLStore) Delete(r *http.Request, w http.ResponseWriter, sess *sessions.Session) (err error) {
	// overwrite the cookie with a negative max age so the browser expires it immediately
	opts := *sess.Options
	opts.MaxAge = -1
	http.SetCookie(w, sessions.NewCookie(sess.Name(), "", &opts))

	// delete the session from the DB
	err = cs.Cass.Query(`DELETE FROM sessions WHERE id=?`, sess.ID).Exec()
	return
}
<|endoftext|>


<commit_before>package gzip

import (
	"compress/gzip"
	"net/http"
	"path/filepath"
	"strings"

	"github.com/gin-gonic/gin"
)

const (
	BestCompression    = gzip.BestCompression
	BestSpeed          = gzip.BestSpeed
	DefaultCompression = gzip.DefaultCompression
	NoCompression      = gzip.NoCompression
)

func Gzip(level int) gin.HandlerFunc {
	return func(c *gin.Context) {
		if !shouldCompress(c.Request) {
			return
		}
		gz, err := gzip.NewWriterLevel(c.Writer, level)
		if err != nil {
			return
		}

		c.Header("Content-Encoding", "gzip")
		c.Header("Vary", "Accept-Encoding")
		c.Writer = &gzipWriter{c.Writer, gz}
		defer func() {
			c.Header("Content-Length", "")
			gz.Close()
		}()
		c.Next()
	}
}

type gzipWriter struct {
	gin.ResponseWriter
	writer *gzip.Writer
}

func (g *gzipWriter) Write(data []byte) (int, error) {
	return g.writer.Write(data)
}

func shouldCompress(req *http.Request) bool {
	if !strings.Contains(req.Header.Get("Accept-Encoding"), "gzip") {
		return false
	}
	extension := filepath.Ext(req.URL.Path)
	if len(extension) < 4 { // fast path
		return true
	}

	switch extension {
	case ".png", ".gif", ".jpeg", ".jpg":
		return false
	default:
		return true
	}
}
<commit_msg>Fix "invalid response" error in Gzip. Closes #66<commit_after>package gzip

import (
	"compress/gzip"
	"net/http"
	"path/filepath"
	"strings"

	"github.com/gin-gonic/gin"
)

const (
	BestCompression    = gzip.BestCompression
	BestSpeed          = gzip.BestSpeed
	DefaultCompression = gzip.DefaultCompression
	NoCompression      = gzip.NoCompression
)

func Gzip(level int) gin.HandlerFunc {
	return func(c *gin.Context) {
		if !shouldCompress(c.Request) {
			return
		}
		gz, err := gzip.NewWriterLevel(c.Writer, level)
		if err != nil {
			return
		}

		c.Header("Content-Encoding", "gzip")
		c.Header("Vary", "Accept-Encoding")
		c.Writer = &gzipWriter{c.Writer, gz}
		defer func() {
			c.Header("Content-Length", "0")
			gz.Close()
		}()
		c.Next()
	}
}

type gzipWriter struct {
	gin.ResponseWriter
	writer *gzip.Writer
}

func (g *gzipWriter) Write(data []byte) (int, error) {
	return g.writer.Write(data)
}

func shouldCompress(req *http.Request) bool {
	if !strings.Contains(req.Header.Get("Accept-Encoding"), "gzip") {
		return false
	}
	extension := filepath.Ext(req.URL.Path)
	if len(extension) < 4 { // fast path
		return true
	}

	switch extension {
	case ".png", ".gif", ".jpeg", ".jpg":
		return false
	default:
		return true
	}
}
<|endoftext|>


<commit_before>//
// go-sonos
// ========
//
// Copyright (c) 2012, Ian T. Richards <ianr@panix.com>
// All rights reserved.
//
// Redistribution and use in source and binary forms, with or without
// modification, are permitted provided that the following conditions
// are met:
//
//   * Redistributions of source code must retain the above copyright notice,
//     this list of conditions and the following disclaimer.
//   * Redistributions in binary form must reproduce the above copyright
//     notice, this list of conditions and the following disclaimer in the
//     documentation and/or other materials provided with the distribution.
//
// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
// HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED
// TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR
// PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF
// LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING
// NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
// SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
//

package model

import (
	"encoding/xml"
	"github.com/ianr0bkny/go-sonos/didl"
	"github.com/ianr0bkny/go-sonos/upnp"
	_ "log"
	"strings"
	"time"
)

type PositionInfo struct {
	Track               uint32
	TrackDuration       time.Duration
	TrackURI            string
	RelTime             time.Duration
	ProtocolInfo        string
	Title               string
	Class               string
	Creator             string
	Album               string
	OriginalTrackNumber string
}

func getDuration(in string) (d time.Duration, err error) {
	in = strings.Replace(in, ":", "h", 1)
	in = strings.Replace(in, ":", "m", 1)
	in += "s"
	return time.ParseDuration(in)
}

func GetPositionInfoMessage(in *upnp.PositionInfo) *PositionInfo {
	var trackDuration, relTime time.Duration
	trackDuration, err := getDuration(in.TrackDuration)
	if nil == err {
		trackDuration /= time.Second
	}

	relTime, err = getDuration(in.RelTime)
	if nil == err {
		relTime /= time.Second
	}

	out := &PositionInfo{
		Track:         in.Track,
		TrackDuration: trackDuration,
		TrackURI:      in.TrackURI,
		RelTime:       relTime,
	}

	metadata := &didl.Lite{}
	xml.Unmarshal([]byte(in.TrackMetaData), metadata)
	metadata.Validate()

	for _, item := range metadata.Item {
		for _, res := range item.Res {
			out.ProtocolInfo = res.ProtocolInfo
			break
		}
		for _, title := range item.Title {
			out.Title = title.Value
			break
		}
		for _, class := range item.Class {
			out.Class = class.Value
			break
		}
		for _, creator := range item.Creator {
			out.Creator = creator.Value
			break
		}
		for _, album := range item.Album {
			out.Album = album.Value
			break
		}
		for _, originalTrackNumber := range item.OriginalTrackNumber {
			out.OriginalTrackNumber = originalTrackNumber.Value
			break
		}
		break
	}
	return out
}

type QueueElement struct {
	ID                  string
	ParentID            string
	TrackURI            string
	Title               string
	Class               string
	AlbumArtURI         string
	Creator             string
	Album               string
	OriginalTrackNumber string
}

func GetQueueContentsMessage(in []Object) []QueueElement {
	var out []QueueElement
	for _, obj := range in {
		out = append(out, QueueElement{
			ID:                  obj.ID(),
			ParentID:            obj.ParentID(),
			TrackURI:            obj.Res(),
			Title:               obj.Title(),
			Class:               obj.Class(),
			AlbumArtURI:         obj.AlbumArtURI(),
			Creator:             obj.Creator(),
			Album:               obj.Album(),
			OriginalTrackNumber: obj.OriginalTrackNumber(),
		})
	}
	return out
}

type TransportInfo *upnp.TransportInfo
<commit_msg>Protecting encoding<commit_after>//
// go-sonos
// ========
//
// Copyright (c) 2012, Ian T. Richards <ianr@panix.com>
// All rights reserved.
//
// Redistribution and use in source and binary forms, with or without
// modification, are permitted provided that the following conditions
// are met:
//
//   * Redistributions of source code must retain the above copyright notice,
//     this list of conditions and the following disclaimer.
//   * Redistributions in binary form must reproduce the above copyright
//     notice, this list of conditions and the following disclaimer in the
//     documentation and/or other materials provided with the distribution.
//
// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
// HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED
// TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR
// PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF
// LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING
// NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
// SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
//

package model

import (
	"encoding/xml"
	"github.com/ianr0bkny/go-sonos/didl"
	"github.com/ianr0bkny/go-sonos/upnp"
	"strings"
	"time"
)

type PositionInfo struct {
	Track               uint32
	TrackDuration       time.Duration
	TrackURI            string
	RelTime             time.Duration
	ProtocolInfo        string
	Title               string
	Class               string
	Creator             string
	Album               string
	OriginalTrackNumber string
}

func getDuration(in string) (d time.Duration, err error) {
	in = strings.Replace(in, ":", "h", 1)
	in = strings.Replace(in, ":", "m", 1)
	in += "s"
	return time.ParseDuration(in)
}

func GetPositionInfoMessage(in *upnp.PositionInfo) *PositionInfo {
	var trackDuration, relTime time.Duration
	trackDuration, err := getDuration(in.TrackDuration)
	if nil == err {
		trackDuration /= time.Second
	}

	relTime, err = getDuration(in.RelTime)
	if nil == err {
		relTime /= time.Second
	}

	out := &PositionInfo{
		Track:         in.Track,
		TrackDuration: trackDuration,
		TrackURI:      in.TrackURI,
		RelTime:       relTime,
	}

	metadata := &didl.Lite{}
	xml.Unmarshal([]byte(in.TrackMetaData), metadata)
	metadata.Validate()

	for _, item := range metadata.Item {
		for _, res := range item.Res {
			out.ProtocolInfo = res.ProtocolInfo
			break
		}
		for _, title := range item.Title {
			out.Title = title.Value
			break
		}
		for _, class := range item.Class {
			out.Class = class.Value
			break
		}
		for _, creator := range item.Creator {
			out.Creator = creator.Value
			break
		}
		for _, album := range item.Album {
			out.Album = album.Value
			break
		}
		for _, originalTrackNumber := range item.OriginalTrackNumber {
			out.OriginalTrackNumber = originalTrackNumber.Value
			break
		}
		break
	}
	return out
}

type QueueElement struct {
	ID                  string
	ParentID            string
	TrackURI            string
	Title               string
	Class               string
	AlbumArtURI         string
	Creator             string
	Album               string
	OriginalTrackNumber string
}

func protectEncoding(s string) string {
	return strings.Replace(s, "%", "%25", -1)
}

func GetQueueContentsMessage(in []Object) []QueueElement {
	var out []QueueElement
	for _, obj := range in {
		out = append(out, QueueElement{
			ID:                  protectEncoding(obj.ID()),
			ParentID:            obj.ParentID(),
			TrackURI:            obj.Res(),
			Title:               obj.Title(),
			Class:               obj.Class(),
			AlbumArtURI:         obj.AlbumArtURI(),
			Creator:             obj.Creator(),
			Album:               obj.Album(),
			OriginalTrackNumber: obj.OriginalTrackNumber(),
		})
	}
	return out
}

type TransportInfo *upnp.TransportInfo
<|endoftext|>


<commit_before>package models

import (
	"errors"
	"log"
	"os"

	"github.com/jinzhu/gorm"
	"github.com/jordan-wright/gophish/config"
	_ "github.com/mattn/go-sqlite3" // Blank import needed to import sqlite3
)

var db gorm.DB
var err error

// ErrUsernameTaken is thrown when a user attempts to register a username that is taken.
var ErrUsernameTaken = errors.New("username already taken")

// Logger is a global logger used to show informational, warning, and error messages
var Logger = log.New(os.Stdout, " ", log.Ldate|log.Ltime|log.Lshortfile)

const (
	CAMPAIGN_IN_PROGRESS string = "In progress"
	CAMPAIGN_QUEUED      string = "Queued"
	CAMPAIGN_COMPLETE    string = "Completed"
	EVENT_SENT           string = "Email Sent"
	EVENT_OPENED         string = "Email Opened"
	EVENT_CLICKED        string = "Clicked Link"
	STATUS_SUCCESS       string = "Success"
	STATUS_UNKNOWN       string = "Unknown"
	ERROR                string = "Error"
)

// Flash is used to hold flash information for use in templates.
type Flash struct {
	Type    string
	Message string
}

// Response contains the attributes found in an API response
type Response struct {
	Message string      `json:"message"`
	Success bool        `json:"success"`
	Data    interface{} `json:"data"`
}

// Setup initializes the Conn object
// It also populates the Gophish Config object
func Setup() error {
	db, err = gorm.Open("sqlite3", config.Conf.DBPath)
	db.LogMode(false)
	db.SetLogger(Logger)
	if err != nil {
		Logger.Println(err)
		return err
	}
	//If the file already exists, delete it and recreate it
	_, err = os.Stat(config.Conf.DBPath)
	if err != nil {
		Logger.Printf("Database not found... creating db at %s\n", config.Conf.DBPath)
		db.CreateTable(User{})
		db.CreateTable(Target{})
		db.CreateTable(Result{})
		db.CreateTable(Group{})
		db.CreateTable(GroupTarget{})
		db.CreateTable(Template{})
		db.CreateTable(Attachment{})
		db.CreateTable(Page{})
		db.CreateTable(SMTP{})
		db.CreateTable(Event{})
		db.CreateTable(Campaign{})
		//Create the default user
		initUser := User{
			Username: "admin",
			Hash:     "$2a$10$IYkPp0.QsM81lYYPrQx6W.U6oQGw7wMpozrKhKAHUBVL4mkm/EvAS", //gophish
			ApiKey:   "12345678901234567890123456789012",
		}
		err = db.Save(&initUser).Error
		if err != nil {
			Logger.Println(err)
		}
	}
	return nil
}
<commit_msg>Fixed issue where database wasn't getting created properly on Linux distros.<commit_after>package models

import (
	"errors"
	"log"
	"os"

	"github.com/jinzhu/gorm"
	"github.com/jordan-wright/gophish/config"
	_ "github.com/mattn/go-sqlite3" // Blank import needed to import sqlite3
)

var db gorm.DB
var err error

// ErrUsernameTaken is thrown when a user attempts to register a username that is taken.
var ErrUsernameTaken = errors.New("username already taken")

// Logger is a global logger used to show informational, warning, and error messages
var Logger = log.New(os.Stdout, " ", log.Ldate|log.Ltime|log.Lshortfile)

const (
	CAMPAIGN_IN_PROGRESS string = "In progress"
	CAMPAIGN_QUEUED      string = "Queued"
	CAMPAIGN_COMPLETE    string = "Completed"
	EVENT_SENT           string = "Email Sent"
	EVENT_OPENED         string = "Email Opened"
	EVENT_CLICKED        string = "Clicked Link"
	STATUS_SUCCESS       string = "Success"
	STATUS_UNKNOWN       string = "Unknown"
	ERROR                string = "Error"
)

// Flash is used to hold flash information for use in templates.
type Flash struct {
	Type    string
	Message string
}

// Response contains the attributes found in an API response
type Response struct {
	Message string      `json:"message"`
	Success bool        `json:"success"`
	Data    interface{} `json:"data"`
}

// Setup initializes the Conn object
// It also populates the Gophish Config object
func Setup() error {
	create_db := false
	if _, err = os.Stat(config.Conf.DBPath); err != nil || config.Conf.DBPath == ":memory:" {
		create_db = true
	}
	db, err = gorm.Open("sqlite3", config.Conf.DBPath)
	db.LogMode(false)
	db.SetLogger(Logger)
	if err != nil {
		Logger.Println(err)
		return err
	}
	//If the file already exists, delete it and recreate it
	if create_db {
		Logger.Printf("Database not found... creating db at %s\n", config.Conf.DBPath)
		db.CreateTable(User{})
		db.CreateTable(Target{})
		db.CreateTable(Result{})
		db.CreateTable(Group{})
		db.CreateTable(GroupTarget{})
		db.CreateTable(Template{})
		db.CreateTable(Attachment{})
		db.CreateTable(Page{})
		db.CreateTable(SMTP{})
		db.CreateTable(Event{})
		db.CreateTable(Campaign{})
		//Create the default user
		initUser := User{
			Username: "admin",
			Hash:     "$2a$10$IYkPp0.QsM81lYYPrQx6W.U6oQGw7wMpozrKhKAHUBVL4mkm/EvAS", //gophish
			ApiKey:   "12345678901234567890123456789012",
		}
		err = db.Save(&initUser).Error
		if err != nil {
			Logger.Println(err)
		}
	}
	return nil
}
<|endoftext|>


<commit_before>package hal

/*
 * Copyright 2016 Albert P. Tobey <atobey@netflix.com>
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 *     http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */

import (
	"bytes"
	"fmt"
	"log"
	"strings"
)

// provides a persistent configuration store

// Order of precendence for prefs:
// user -> room -> broker -> plugin -> global -> default

// PREFS_TABLE contains the SQL to create the prefs table
// key field is called pkey because key is a reserved word
const PREFS_TABLE = `
CREATE TABLE IF NOT EXISTS prefs (
	 id      INT NOT NULL AUTO_INCREMENT, -- only used for deleting/updating by id
	 user    VARCHAR(191) DEFAULT "",
	 room    VARCHAR(191) DEFAULT "",
	 broker  VARCHAR(191) DEFAULT "",
	 plugin  VARCHAR(191) DEFAULT "",
	 pkey    VARCHAR(191) NOT NULL,
	 value   MEDIUMTEXT,
	 INDEX(id), -- required by mysql for non-PK auto_increment
	 -- InnoDB limits indexes to 767 bytes so have the PK only index the first
	 -- 32 characters of each column as a compromise
	 -- (5 cols * 4 bytes * 32 chars = 640)
	 PRIMARY KEY(user(32), room(32), broker(32), plugin(32), pkey(32))
)`

/*
   -- test data, will remove once there are automated tests
   INSERT INTO prefs (user,room,broker,plugin,pkey,value) VALUES ("tobert", "", "", "", "foo", "user");
   INSERT INTO prefs (user,room,broker,plugin,pkey,value) VALUES ("tobert", "CORE", "", "", "foo",
   "user-room");
   INSERT INTO prefs (user,room,broker,plugin,pkey,value) VALUES ("tobert", "CORE", "slack", "", "foo",
   "user-room-broker");
   INSERT INTO prefs (user,room,broker,plugin,pkey,value) VALUES ("tobert", "CORE", "slack", "uptime",
   "foo", "user-room-broker-plugin");
   INSERT INTO prefs (user,room,broker,plugin,pkey,value) VALUES ("tobert", "", "slack", "uptime", "foo",
   "user-broker-plugin");
   INSERT INTO prefs (user,room,broker,plugin,pkey,value) VALUES ("tobert", "CORE", "", "uptime",
   "foo", "user-room-plugin");
   INSERT INTO prefs (user,room,broker,plugin,pkey,value) VALUES ("tobert", "", "", "uptime", "foo",
   "user-plugin");
*/

// !prefs list --scope plugin --plugin autoresponder
// !prefs get --scope room --plugin autoresponder --room CORE --key timezone
// !prefs set --scope user --plugin autoresponder --room CORE

// Pref is a key/value pair associated with a combination of user, plugin,
// borker, or room.
type Pref struct {
	User    string
	Plugin  string
	Broker  string
	Room    string
	Key     string
	Value   string
	Default string
	Success bool
	Error   error
	Id      int
}

type Prefs []Pref

// GetPref will retreive the most-specific preference from pref
// storage using the parameters provided. This is a bit like pattern
// matching. If no match is found, the provided default is returned.
// TODO: explain this better
func GetPref(user, broker, room, plugin, key, def string) Pref {
	pref := Pref{
		User:    user,
		Room:    room,
		Broker:  broker,
		Plugin:  plugin,
		Key:     key,
		Default: def,
	}

	up := pref.Get()
	if up.Success {
		return up
	}

	// no match, return the default
	pref.Value = def
	return pref
}

// SetPref sets a preference and is shorthand for Pref{}.Set().
func SetPref(user, broker, room, plugin, key, value string) error {
	pref := Pref{
		User:   user,
		Room:   room,
		Broker: broker,
		Plugin: plugin,
		Key:    key,
		Value:  value,
	}

	return pref.Set()
}

// GetPrefs retrieves a set of preferences from the database. The
// settings are matched exactly on user,broker,room,plugin.
// e.g. GetPrefs("", "", "", "uptime") would get only records that
// have user/broker/room set to the empty string and room
// set to "uptime". A record with user "pford" and plugin "uptime"
// would not be included.
func GetPrefs(user, broker, room, plugin string) Prefs {
	pref := Pref{
		User:   user,
		Broker: broker,
		Room:   room,
		Plugin: plugin,
	}
	return pref.get()
}

// FindPrefs gets all records that match any of the inputs that are
// not empty strings. (hint: user="x", broker="y"; WHERE user=? OR broker=?)
func FindPrefs(user, broker, room, plugin, key string) Prefs {
	pref := Pref{
		User:   user,
		Broker: broker,
		Room:   room,
		Plugin: plugin,
		Key:    key,
	}
	return pref.Find()
}

// RmPrefId removes a preference from the database by its numeric id.
func RmPrefId(id int) error {
	db := SqlDB()
	SqlInit(PREFS_TABLE)

	_, err := db.Exec("DELETE FROM prefs WHERE id=?", &id)
	return err
}

// Get retrieves a value from the database. If the database returns
// an error, Success will be false and the Error field will be populated.
func (in *Pref) Get() Pref {
	prefs := in.get()

	if len(prefs) == 1 {
		return prefs[0]
	} else if len(prefs) > 1 {
		panic("TOO MANY PREFS")
	} else if len(prefs) == 0 {
		out := *in
		// only set success to false if there is also an error
		// queries with 0 rows are successful
		if out.Error != nil {
			out.Success = false
		} else {
			out.Success = true
			out.Value = out.Default
		}
		return out
	}

	panic("BUG: should be impossible to reach this point")
}

// GetPrefs returns all preferences that match the fields set in the handle.
func (in *Pref) GetPrefs() Prefs {
	return in.get()
}

func (in *Pref) get() Prefs {
	db := SqlDB()
	SqlInit(PREFS_TABLE)

	sql := `SELECT user,room,broker,plugin,pkey,value,id
	        FROM prefs
	        WHERE user=?
			  AND room=?
			  AND broker=?
			  AND plugin=?`
	params := []interface{}{&in.User, &in.Room, &in.Broker, &in.Plugin}

	// only query by key if it's specified, otherwise get all keys for the selection
	if in.Key != "" {
		sql += " AND pkey=?"
		params = append(params, &in.Key)
	}

	rows, err := db.Query(sql, params...)
	if err != nil {
		log.Printf("Returning default due to SQL query failure: %s", err)
		return Prefs{}
	}

	defer rows.Close()

	out := make(Prefs, 0)

	for rows.Next() {
		p := *in

		err := rows.Scan(&p.User, &p.Room, &p.Broker, &p.Plugin, &p.Key, &p.Value, &p.Id)

		if err != nil {
			log.Printf("Returning default due to row iteration failure: %s", err)
			p.Success = false
			p.Value = in.Default
			p.Error = err
		} else {
			p.Success = true
			p.Error = nil
		}

		out = append(out, p)
	}

	return out
}

// Set writes the value and returns a new struct with the new value.
func (in *Pref) Set() error {
	db := SqlDB()
	err := SqlInit(PREFS_TABLE)
	if err != nil {
		log.Printf("Failed to initialize the prefs table: %s", err)
		return err
	}

	sql := `INSERT INTO prefs
						(value,user,room,broker,plugin,pkey)
			VALUES (?,?,?,?,?,?)
			ON DUPLICATE KEY
			UPDATE value=?, user=?, room=?, broker=?, plugin=?, pkey=?`

	params := []interface{}{
		&in.Value, &in.User, &in.Room, &in.Broker, &in.Plugin, &in.Key,
		&in.Value, &in.User, &in.Room, &in.Broker, &in.Plugin, &in.Key,
	}

	_, err = db.Exec(sql, params...)
	if err != nil {
		log.Printf("Pref.Set() write failed: %s", err)
		return err
	}

	return nil
}

// Set writes the value and returns a new struct with the new value.
func (in *Pref) Delete() error {
	db := SqlDB()

	err := SqlInit(PREFS_TABLE)
	if err != nil {
		log.Printf("Failed to initialize the prefs table: %s", err)
		return err
	}

	sql := `DELETE FROM prefs
			WHERE user=?
			  AND room=?
			  AND broker=?
			  AND plugin=?
			  AND pkey=?`

	// TODO: verify only one row was deleted
	_, err = db.Exec(sql, &in.User, &in.Room, &in.Broker, &in.Plugin, &in.Key)
	if err != nil {
		log.Printf("Pref.Delete() write failed: %s", err)
		return err
	}

	return nil
}

// Find retrieves all preferences from the database that match any field in the
// handle's fields.
// Unlike Get(), empty string fields are not included in the (generated) query
// so it can potentially match a lot of rows.
// Returns an empty list and logs upon errors.
func (p Pref) Find() Prefs {
	db := SqlDB()
	SqlInit(PREFS_TABLE)

	fields := make([]string, 0)
	params := make([]interface{}, 0)

	if p.User != "" {
		fields = append(fields, "user=?")
		params = append(params, p.User)
	}

	if p.Room != "" {
		fields = append(fields, "room=?")
		params = append(params, p.Room)
	}

	if p.Broker != "" {
		fields = append(fields, "broker=?")
		params = append(params, p.Broker)
	}

	if p.Plugin != "" {
		fields = append(fields, "plugin=?")
		params = append(params, p.Plugin)
	}

	if p.Key != "" {
		fields = append(fields, "pkey=?")
		params = append(params, p.Key)
	}

	q := bytes.NewBufferString("SELECT user,room,broker,plugin,pkey,value,id\n")
	q.WriteString("FROM prefs\n")

	// TODO: maybe it's silly to make it easy for Find() to get all preferences
	// but let's cross that bridge when we come to it
	if len(fields) > 0 {
		q.WriteString("\nWHERE ")
		// might make sense to add a param to this func to make it easy to
		// switch this between AND/OR for unions/intersections
		q.WriteString(strings.Join(fields, "\n  OR "))
	}

	// TODO: add deterministic ordering at query time

	out := make(Prefs, 0)
	rows, err := db.Query(q.String(), params...)
	if err != nil {
		log.Println(q.String())
		log.Printf("Query failed: %s", err)
		return out
	}
	defer rows.Close()

	for rows.Next() {
		row := Pref{}
		err = rows.Scan(&row.User, &row.Room, &row.Broker, &row.Plugin, &row.Key, &row.Value, &row.Id)
		// improbable in practice - follows previously mentioned conventions for errors
		if err != nil {
			log.Printf("Fetching a row failed: %s\n", err)
			row.Error = err
			row.Success = false
			row.Value = p.Default
		} else {
			row.Error = nil
			row.Success = true
		}

		out = append(out, row)
	}

	return out
}

// Clone returns a full/deep copy of the Prefs list.
func (prefs Prefs) Clone() Prefs {
	out := make(Prefs, len(prefs))

	for i, pref := range prefs {
		copy := pref
		out[i] = copy
	}

	return out
}

// User filters the preference list by user, returning a new Prefs
// e.g. uprefs = prefs.User("adent")
func (prefs Prefs) User(user string) Prefs {
	out := make(Prefs, 0)

	for _, pref := range prefs {
		if pref.User == user {
			out = append(out, pref)
		}
	}

	return out
}

// Room filters the preference list by room, returning a new Prefs
// e.g. instprefs = prefs.Room("magrathea").Plugin("uptime").Broker("slack")
func (prefs Prefs) Room(room string) Prefs {
	out := make(Prefs, 0)

	for _, pref := range prefs {
		if pref.Room == room {
			out = append(out, pref)
		}
	}

	return out
}

// Broker filters the preference list by broker, returning a new Prefs
func (prefs Prefs) Broker(broker string) Prefs {
	out := make(Prefs, 0)

	for _, pref := range prefs {
		if pref.Broker == broker {
			out = append(out, pref)
		}
	}

	return out
}

// Plugin filters the preference list by plugin, returning a new Prefs
func (prefs Prefs) Plugin(plugin string) Prefs {
	out := make(Prefs, 0)

	for _, pref := range prefs {
		if pref.Plugin == plugin {
			out = append(out, pref)
		}
	}

	return out
}

// Key filters the preference list by key, returning a new Prefs
func (prefs Prefs) Key(key string) Prefs {
	out := make(Prefs, 0)

	for _, pref := range prefs {
		if pref.Key == key {
			out = append(out, pref)
		}
	}

	return out
}

// ready to hand off to e.g. hal.AsciiTable()
func (prefs Prefs) Table() [][]string {
	out := make([][]string, 1)
	out[0] = []string{"User", "Room", "Broker", "Plugin", "Key", "Value", "ID"}

	for _, pref := range prefs {
		m := []string{
			pref.User,
			pref.Room,
			pref.Broker,
			pref.Plugin,
			pref.Key,
			pref.Value,
			fmt.Sprintf("%d", pref.Id),
		}

		out = append(out, m)
	}

	return out
}

func (p *Pref) String() string {
	return fmt.Sprintf(`Pref{
	User:    %q,
	Room:    %q,
	Broker:  %q,
	Plugin:  %q,
	Key:     %q,
	Value:   %q,
	Default: %q,
	Success: %t,
	Error:   %v,
	Id:      %d,
}`, p.User, p.Room, p.Broker, p.Plugin, p.Key, p.Value, p.Default, p.Success, p.Error, p.Id)

}

func (p *Prefs) String() string {
	data := p.Table()
	return AsciiTable(data[0], data[1:])
}
<commit_msg>update constant name to go standards<commit_after>package hal

/*
 * Copyright 2016 Albert P. Tobey <atobey@netflix.com>
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 *     http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */

import (
	"bytes"
	"fmt"
	"log"
	"strings"
)

// provides a persistent configuration store

// Order of precendence for prefs:
// user -> room -> broker -> plugin -> global -> default

// PrefsTable contains the SQL to create the prefs table
// key field is called pkey because key is a reserved word
const PrefsTable = `
CREATE TABLE IF NOT EXISTS prefs (
	 id      INT NOT NULL AUTO_INCREMENT, -- only used for deleting/updating by id
	 user    VARCHAR(191) DEFAULT "",
	 room    VARCHAR(191) DEFAULT "",
	 broker  VARCHAR(191) DEFAULT "",
	 plugin  VARCHAR(191) DEFAULT "",
	 pkey    VARCHAR(191) NOT NULL,
	 value   MEDIUMTEXT,
	 INDEX(id), -- required by mysql for non-PK auto_increment
	 -- InnoDB limits indexes to 767 bytes so have the PK only index the first
	 -- 32 characters of each column as a compromise
	 -- (5 cols * 4 bytes * 32 chars = 640)
	 PRIMARY KEY(user(32), room(32), broker(32), plugin(32), pkey(32))
)`

/*
   -- test data, will remove once there are automated tests
   INSERT INTO prefs (user,room,broker,plugin,pkey,value) VALUES ("tobert", "", "", "", "foo", "user");
   INSERT INTO prefs (user,room,broker,plugin,pkey,value) VALUES ("tobert", "CORE", "", "", "foo",
   "user-room");
   INSERT INTO prefs (user,room,broker,plugin,pkey,value) VALUES ("tobert", "CORE", "slack", "", "foo",
   "user-room-broker");
   INSERT INTO prefs (user,room,broker,plugin,pkey,value) VALUES ("tobert", "CORE", "slack", "uptime",
   "foo", "user-room-broker-plugin");
   INSERT INTO prefs (user,room,broker,plugin,pkey,value) VALUES ("tobert", "", "slack", "uptime", "foo",
   "user-broker-plugin");
   INSERT INTO prefs (user,room,broker,plugin,pkey,value) VALUES ("tobert", "CORE", "", "uptime",
   "foo", "user-room-plugin");
   INSERT INTO prefs (user,room,broker,plugin,pkey,value) VALUES ("tobert", "", "", "uptime", "foo",
   "user-plugin");
*/

// !prefs list --scope plugin --plugin autoresponder
// !prefs get --scope room --plugin autoresponder --room CORE --key timezone
// !prefs set --scope user --plugin autoresponder --room CORE

// Pref is a key/value pair associated with a combination of user, plugin,
// borker, or room.
type Pref struct {
	User    string
	Plugin  string
	Broker  string
	Room    string
	Key     string
	Value   string
	Default string
	Success bool
	Error   error
	Id      int
}

type Prefs []Pref

// GetPref will retreive the most-specific preference from pref
// storage using the parameters provided. This is a bit like pattern
// matching. If no match is found, the provided default is returned.
// TODO: explain this better
func GetPref(user, broker, room, plugin, key, def string) Pref {
	pref := Pref{
		User:    user,
		Room:    room,
		Broker:  broker,
		Plugin:  plugin,
		Key:     key,
		Default: def,
	}

	up := pref.Get()
	if up.Success {
		return up
	}

	// no match, return the default
	pref.Value = def
	return pref
}

// SetPref sets a preference and is shorthand for Pref{}.Set().
func SetPref(user, broker, room, plugin, key, value string) error {
	pref := Pref{
		User:   user,
		Room:   room,
		Broker: broker,
		Plugin: plugin,
		Key:    key,
		Value:  value,
	}

	return pref.Set()
}

// GetPrefs retrieves a set of preferences from the database. The
// settings are matched exactly on user,broker,room,plugin.
// e.g. GetPrefs("", "", "", "uptime") would get only records that
// have user/broker/room set to the empty string and room
// set to "uptime". A record with user "pford" and plugin "uptime"
// would not be included.
func GetPrefs(user, broker, room, plugin string) Prefs {
	pref := Pref{
		User:   user,
		Broker: broker,
		Room:   room,
		Plugin: plugin,
	}
	return pref.get()
}

// FindPrefs gets all records that match any of the inputs that are
// not empty strings. (hint: user="x", broker="y"; WHERE user=? OR broker=?)
func FindPrefs(user, broker, room, plugin, key string) Prefs {
	pref := Pref{
		User:   user,
		Broker: broker,
		Room:   room,
		Plugin: plugin,
		Key:    key,
	}
	return pref.Find()
}

// RmPrefId removes a preference from the database by its numeric id.
func RmPrefId(id int) error {
	db := SqlDB()
	SqlInit(PrefsTable)

	_, err := db.Exec("DELETE FROM prefs WHERE id=?", &id)
	return err
}

// Get retrieves a value from the database. If the database returns
// an error, Success will be false and the Error field will be populated.
func (in *Pref) Get() Pref {
	prefs := in.get()

	if len(prefs) == 1 {
		return prefs[0]
	} else if len(prefs) > 1 {
		panic("TOO MANY PREFS")
	} else if len(prefs) == 0 {
		out := *in
		// only set success to false if there is also an error
		// queries with 0 rows are successful
		if out.Error != nil {
			out.Success = false
		} else {
			out.Success = true
			out.Value = out.Default
		}
		return out
	}

	panic("BUG: should be impossible to reach this point")
}

// GetPrefs returns all preferences that match the fields set in the handle.
func (in *Pref) GetPrefs() Prefs {
	return in.get()
}

func (in *Pref) get() Prefs {
	db := SqlDB()
	SqlInit(PrefsTable)

	sql := `SELECT user,room,broker,plugin,pkey,value,id
	        FROM prefs
	        WHERE user=?
			  AND room=?
			  AND broker=?
			  AND plugin=?`
	params := []interface{}{&in.User, &in.Room, &in.Broker, &in.Plugin}

	// only query by key if it's specified, otherwise get all keys for the selection
	if in.Key != "" {
		sql += " AND pkey=?"
		params = append(params, &in.Key)
	}

	rows, err := db.Query(sql, params...)
	if err != nil {
		log.Printf("Returning default due to SQL query failure: %s", err)
		return Prefs{}
	}

	defer rows.Close()

	out := make(Prefs, 0)

	for rows.Next() {
		p := *in

		err := rows.Scan(&p.User, &p.Room, &p.Broker, &p.Plugin, &p.Key, &p.Value, &p.Id)

		if err != nil {
			log.Printf("Returning default due to row iteration failure: %s", err)
			p.Success = false
			p.Value = in.Default
			p.Error = err
		} else {
			p.Success = true
			p.Error = nil
		}

		out = append(out, p)
	}

	return out
}

// Set writes the value and returns a new struct with the new value.
func (in *Pref) Set() error {
	db := SqlDB()
	err := SqlInit(PrefsTable)
	if err != nil {
		log.Printf("Failed to initialize the prefs table: %s", err)
		return err
	}

	sql := `INSERT INTO prefs
						(value,user,room,broker,plugin,pkey)
			VALUES (?,?,?,?,?,?)
			ON DUPLICATE KEY
			UPDATE value=?, user=?, room=?, broker=?, plugin=?, pkey=?`

	params := []interface{}{
		&in.Value, &in.User, &in.Room, &in.Broker, &in.Plugin, &in.Key,
		&in.Value, &in.User, &in.Room, &in.Broker, &in.Plugin, &in.Key,
	}

	_, err = db.Exec(sql, params...)
	if err != nil {
		log.Printf("Pref.Set() write failed: %s", err)
		return err
	}

	return nil
}

// Set writes the value and returns a new struct with the new value.
func (in *Pref) Delete() error {
	db := SqlDB()

	err := SqlInit(PrefsTable)
	if err != nil {
		log.Printf("Failed to initialize the prefs table: %s", err)
		return err
	}

	sql := `DELETE FROM prefs
			WHERE user=?
			  AND room=?
			  AND broker=?
			  AND plugin=?
			  AND pkey=?`

	// TODO: verify only one row was deleted
	_, err = db.Exec(sql, &in.User, &in.Room, &in.Broker, &in.Plugin, &in.Key)
	if err != nil {
		log.Printf("Pref.Delete() write failed: %s", err)
		return err
	}

	return nil
}

// Find retrieves all preferences from the database that match any field in the
// handle's fields.
// Unlike Get(), empty string fields are not included in the (generated) query
// so it can potentially match a lot of rows.
// Returns an empty list and logs upon errors.
func (p Pref) Find() Prefs {
	db := SqlDB()
	SqlInit(PrefsTable)

	fields := make([]string, 0)
	params := make([]interface{}, 0)

	if p.User != "" {
		fields = append(fields, "user=?")
		params = append(params, p.User)
	}

	if p.Room != "" {
		fields = append(fields, "room=?")
		params = append(params, p.Room)
	}

	if p.Broker != "" {
		fields = append(fields, "broker=?")
		params = append(params, p.Broker)
	}

	if p.Plugin != "" {
		fields = append(fields, "plugin=?")
		params = append(params, p.Plugin)
	}

	if p.Key != "" {
		fields = append(fields, "pkey=?")
		params = append(params, p.Key)
	}

	q := bytes.NewBufferString("SELECT user,room,broker,plugin,pkey,value,id\n")
	q.WriteString("FROM prefs\n")

	// TODO: maybe it's silly to make it easy for Find() to get all preferences
	// but let's cross that bridge when we come to it
	if len(fields) > 0 {
		q.WriteString("\nWHERE ")
		// might make sense to add a param to this func to make it easy to
		// switch this between AND/OR for unions/intersections
		q.WriteString(strings.Join(fields, "\n  OR "))
	}

	// TODO: add deterministic ordering at query time

	out := make(Prefs, 0)
	rows, err := db.Query(q.String(), params...)
	if err != nil {
		log.Println(q.String())
		log.Printf("Query failed: %s", err)
		return out
	}
	defer rows.Close()

	for rows.Next() {
		row := Pref{}
		err = rows.Scan(&row.User, &row.Room, &row.Broker, &row.Plugin, &row.Key, &row.Value, &row.Id)
		// improbable in practice - follows previously mentioned conventions for errors
		if err != nil {
			log.Printf("Fetching a row failed: %s\n", err)
			row.Error = err
			row.Success = false
			row.Value = p.Default
		} else {
			row.Error = nil
			row.Success = true
		}

		out = append(out, row)
	}

	return out
}

// Clone returns a full/deep copy of the Prefs list.
func (prefs Prefs) Clone() Prefs {
	out := make(Prefs, len(prefs))

	for i, pref := range prefs {
		copy := pref
		out[i] = copy
	}

	return out
}

// User filters the preference list by user, returning a new Prefs
// e.g. uprefs = prefs.User("adent")
func (prefs Prefs) User(user string) Prefs {
	out := make(Prefs, 0)

	for _, pref := range prefs {
		if pref.User == user {
			out = append(out, pref)
		}
	}

	return out
}

// Room filters the preference list by room, returning a new Prefs
// e.g. instprefs = prefs.Room("magrathea").Plugin("uptime").Broker("slack")
func (prefs Prefs) Room(room string) Prefs {
	out := make(Prefs, 0)

	for _, pref := range prefs {
		if pref.Room == room {
			out = append(out, pref)
		}
	}

	return out
}

// Broker filters the preference list by broker, returning a new Prefs
func (prefs Prefs) Broker(broker string) Prefs {
	out := make(Prefs, 0)

	for _, pref := range prefs {
		if pref.Broker == broker {
			out = append(out, pref)
		}
	}

	return out
}

// Plugin filters the preference list by plugin, returning a new Prefs
func (prefs Prefs) Plugin(plugin string) Prefs {
	out := make(Prefs, 0)

	for _, pref := range prefs {
		if pref.Plugin == plugin {
			out = append(out, pref)
		}
	}

	return out
}

// Key filters the preference list by key, returning a new Prefs
func (prefs Prefs) Key(key string) Prefs {
	out := make(Prefs, 0)

	for _, pref := range prefs {
		if pref.Key == key {
			out = append(out, pref)
		}
	}

	return out
}

// ready to hand off to e.g. hal.AsciiTable()
func (prefs Prefs) Table() [][]string {
	out := make([][]string, 1)
	out[0] = []string{"User", "Room", "Broker", "Plugin", "Key", "Value", "ID"}

	for _, pref := range prefs {
		m := []string{
			pref.User,
			pref.Room,
			pref.Broker,
			pref.Plugin,
			pref.Key,
			pref.Value,
			fmt.Sprintf("%d", pref.Id),
		}

		out = append(out, m)
	}

	return out
}

func (p *Pref) String() string {
	return fmt.Sprintf(`Pref{
	User:    %q,
	Room:    %q,
	Broker:  %q,
	Plugin:  %q,
	Key:     %q,
	Value:   %q,
	Default: %q,
	Success: %t,
	Error:   %v,
	Id:      %d,
}`, p.User, p.Room, p.Broker, p.Plugin, p.Key, p.Value, p.Default, p.Success, p.Error, p.Id)

}

func (p *Prefs) String() string {
	data := p.Table()
	return AsciiTable(data[0], data[1:])
}
<|endoftext|>