<commit_before>package workers

import (
	"fmt"
	"github.com/APTrust/exchange/constants"
	"github.com/APTrust/exchange/context"
	"github.com/APTrust/exchange/models"
	"github.com/APTrust/exchange/network"
	"github.com/APTrust/exchange/stats"
	"github.com/APTrust/exchange/util"
	"github.com/aws/aws-sdk-go/service/s3"
	"net/url"
	"strconv"
	"strings"
	"time"
)

// If Config.BucketReaderCacheHours isn't set to something
// sensible, cache Ingest WorkItems up to this many hours old.
const DEFAULT_CACHE_HOURS = 24

// How many S3 keys should we fetch in each batch when
// we're getting the contents of a bucket?
const MAX_KEYS = 1000

// APTBucketReader scans APTrust receiving buckets for items
// that need to be ingested. It creates a WorkItem record and
// an NSQ entry for each qualifying bag, and is responsible for
// knowing whether items in the receiving buckets actually need
// to be queued.
type APTBucketReader struct {
	Context           *context.Context
	Institutions      map[string]*models.Institution
	RecentIngestItems map[string]*models.WorkItem
	stats             *stats.APTBucketReaderStats
	statsEnabled      bool
}

// Creates a new bucket reader with the given context.
// Param enableStats is generally used for integration tests.
// Enabling stats in production can cause high memory usage,
// so keep that off unless you're trying to diagnose specific problems.
func NewAPTBucketReader(context *context.Context, enableStats bool) *APTBucketReader {
	reader := &APTBucketReader{
		Context:           context,
		Institutions:      make(map[string]*models.Institution),
		RecentIngestItems: make(map[string]*models.WorkItem),
		statsEnabled:      enableStats,
	}
	if enableStats {
		reader.stats = stats.NewAPTBucketReaderStats()
	}
	return reader
}

func (reader *APTBucketReader) Run() error {
	err := reader.cacheInstitutions()
	if err != nil {
		return err
	}
	err = reader.cacheRecentIngestItems()
	if err != nil {
		return err
	}
	reader.readAllBuckets()
	return nil
}

// Cache a list of all institutions. There are < 20.
// Exit on failure.
func (reader *APTBucketReader) cacheInstitutions() error {
	params := url.Values{}
	params.Add("page", "1")
	params.Add("per_page", "100")
	resp := reader.Context.PharosClient.InstitutionList(params)
	if resp.Error != nil {
		if reader.stats != nil {
			reader.stats.AddError(resp.Error.Error())
		}
		return resp.Error
	}
	if resp.Response.StatusCode != 200 {
		return reader.processPharosError(resp)
	}
	for _, inst := range resp.Institutions() {
		reader.Institutions[inst.Identifier] = inst
		if reader.stats != nil {
			reader.stats.AddToInstitutionsCached(inst)
		}
	}
	reader.Context.MessageLog.Info("Loaded %d institutions", len(reader.Institutions))
	return nil
}

// Cache a list of Ingest items that have been added to
// the list of WorkItems in the past X hours, so we won't
// have to do 1000 lookups. (X is usually around 24, but
// check Config.BucketReaderCacheHours for the exact value.
// This function exits on failure.
func (reader *APTBucketReader) cacheRecentIngestItems() error {
	hours := reader.Context.Config.BucketReaderCacheHours
	if hours < 1 {
		hours = DEFAULT_CACHE_HOURS
	}
	createdAfter := time.Now().Add(time.Duration(-1*hours) * time.Hour).UTC()
	params := url.Values{}
	params.Add("page", "1")
	params.Add("per_page", "100") // change to 4 to test hasMoreResults loop
	params.Add("item_action", constants.ActionIngest)
	params.Add("created_after", createdAfter.Format(time.RFC3339))
	hasMoreResults := true
	for hasMoreResults {
		resp := reader.Context.PharosClient.WorkItemList(params)
		if resp.Error != nil {
			if reader.stats != nil {
				reader.stats.AddError(resp.Error.Error())
			}
			return resp.Error
		}
		if resp.Response.StatusCode != 200 {
			return reader.processPharosError(resp)
		}
		reader.Context.MessageLog.Debug("%s", resp.Request.URL.String())
		for _, workItem := range resp.WorkItems() {
			hashKey := reader.makeHashKey(workItem.Name, workItem.ETag)
			reader.RecentIngestItems[hashKey] = workItem
			if reader.stats != nil {
				reader.stats.AddWorkItem("WorkItemsCached", workItem)
			}
		}
		if resp.Next != nil {
			pageNum, err := strconv.Atoi(params.Get("page"))
			if err != nil {
				msg := fmt.Sprintf("Page number '%s' doesn't look like a number!", params.Get("page"))
				if reader.stats != nil {
					reader.stats.AddError(msg)
				}
				return fmt.Errorf(msg)
			}
			params.Set("page", strconv.Itoa(pageNum+1))
		} else {
			hasMoreResults = false
		}
	}
	reader.Context.MessageLog.Info("Loaded %d recent ingest WorkItems", len(reader.RecentIngestItems))
	return nil
}

func (reader *APTBucketReader) makeHashKey(key, etag string) string {
	return fmt.Sprintf("%s|%s", key, etag)
}

func (reader *APTBucketReader) readAllBuckets() {
	for _, bucketName := range reader.Context.Config.ReceivingBuckets {
		reader.processBucket(bucketName)
	}
}

func (reader *APTBucketReader) processBucket(bucketName string) {
	reader.Context.MessageLog.Debug("Checking bucket %s", bucketName)
	s3ObjList := network.NewS3ObjectList(reader.Context.Config.APTrustS3Region,
		bucketName, MAX_KEYS)
	keepFetching := true
	for keepFetching {
		s3ObjList.GetList("")
		if s3ObjList.ErrorMessage != "" {
			if reader.stats != nil {
				reader.stats.AddError(s3ObjList.ErrorMessage)
			}
			reader.Context.MessageLog.Error(s3ObjList.ErrorMessage)
			break
		}
		for _, s3Object := range s3ObjList.Response.Contents {
			reader.stats.AddS3Item(fmt.Sprintf("%s/%s", bucketName, *s3Object.Key))
			reader.processS3Object(s3Object, bucketName)
		}
		keepFetching = *s3ObjList.Response.IsTruncated
	}
}

func (reader *APTBucketReader) processS3Object(s3Object *s3.Object, bucketName string) {
	if reader.Context.Config.MaxFileSize > int64(0) && *s3Object.Size > reader.Context.Config.MaxFileSize {
		msg := fmt.Sprintf("Skipping %s/%s because size %d is greater than "+
			"current max file size %d", bucketName, *s3Object.Key, *s3Object.Size,
			reader.Context.Config.MaxFileSize)
		if reader.stats != nil {
			reader.stats.AddWarning(msg)
		}
		reader.Context.MessageLog.Debug(msg)
		return
	}
	workItem, err := reader.findWorkItem(*s3Object.Key, *s3Object.ETag)
	if err != nil {
		// Don't create a work item, because one may already exist.
		// Error will be logged and added to stats at source.
		msg := fmt.Sprintf("Not creating WorkItem for %s/%s because "+
			"we can't tell if one already exists.", bucketName, *s3Object.Key)
		reader.Context.MessageLog.Warning(msg)
		if reader.stats != nil {
			reader.stats.AddWarning(msg)
		}
		return
	}
	if workItem == nil {
		workItem = reader.createWorkItem(bucketName, s3Object)
		if workItem == nil {
			// Error logged and statted at source.
			return
		}
	}
	if workItem.QueuedAt == nil || workItem.QueuedAt.IsZero() {
		reader.addToNSQ(workItem)
		reader.markAsQueued(workItem)
	}
}

func (reader *APTBucketReader) findWorkItem(key, etag string) (*models.WorkItem, error) {
	hashKey := reader.makeHashKey(key, etag)
	if workItem, ok := reader.RecentIngestItems[hashKey]; ok {
		reader.Context.MessageLog.Debug("Found hash key '%s' in cache", hashKey)
		return workItem, nil
	}
	reader.Context.MessageLog.Debug("Looking up hash key '%s' in Pharos", hashKey)
	params := url.Values{}
	params.Add("page", "1")
	params.Add("per_page", "1")
	params.Add("item_action", constants.ActionIngest)
	params.Add("name", key)
	params.Add("etag", etag)
	//params.Add("bag_date", lastModified.Format(time.RFC3339))
	resp := reader.Context.PharosClient.WorkItemList(params)
	if resp.Error != nil {
		errMsg := fmt.Sprintf("Error getting WorkItem for name '%s', etag '%s', time '%s': %v",
			params.Get("name"), params.Get("etag"), params.Get("bag_date"), resp.Error)
		reader.Context.MessageLog.Debug("%s", resp.Request.URL.String())
		reader.Context.MessageLog.Error(errMsg)
		if reader.stats != nil {
			reader.stats.AddError(errMsg)
		}
		return nil, fmt.Errorf(errMsg)
	}
	if resp.Response.StatusCode != 200 {
		err := reader.processPharosError(resp)
		return nil, err
	}
	workItem := resp.WorkItem()
	if workItem != nil {
		reader.Context.MessageLog.Debug("Found WorkItem for hash key '%s' in Pharos", hashKey)
		if reader.stats != nil {
			reader.stats.AddWorkItem("WorkItemsFetched", workItem)
		}
	} else {
		reader.Context.MessageLog.Debug("Did not find WorkItem for hash key '%s' in Pharos", hashKey)
	}
	return workItem, nil
}

func (reader *APTBucketReader) createWorkItem(bucket string, s3Object *s3.Object) *models.WorkItem {
	// Create a WorkItem in Pharos
	institution := reader.Institutions[util.OwnerOf(bucket)]
	if institution == nil {
		errMsg := fmt.Sprintf("Cannot find institution record for item %s/%s. "+
			"Owner computes to '%s'", bucket, *s3Object.Key, util.OwnerOf(bucket))
		reader.Context.MessageLog.Error(errMsg)
		if reader.stats != nil {
			reader.stats.AddError(errMsg)
		}
		return nil
	}
	workItem := &models.WorkItem{}
	workItem.Id = 0
	workItem.Name = *s3Object.Key
	workItem.Bucket = bucket
	workItem.ETag = strings.Replace(*s3Object.ETag, "\"", "", -1)
	workItem.Size = *s3Object.Size
	workItem.BagDate = *s3Object.LastModified
	workItem.InstitutionId = institution.Id
	workItem.User = constants.APTrustSystemUser
	workItem.Date = time.Now().UTC()
	workItem.Note = "Bag is in receiving bucket"
	workItem.Action = constants.ActionIngest
	workItem.Stage = constants.StageReceive
	workItem.Status = constants.StatusPending
	workItem.Outcome = "Item is pending ingest"
	workItem.Retry = true
	resp := reader.Context.PharosClient.WorkItemSave(workItem)

	if resp.Error != nil {
		errMsg := fmt.Sprintf("Error creating WorkItem for name '%s', etag '%s', time '%s': %v",
			workItem.Name, workItem.ETag, workItem.BagDate, resp.Error)
		reader.Context.MessageLog.Debug("%s", resp.Request.URL.String())
		reader.Context.MessageLog.Error(errMsg)
		if reader.stats != nil {
			reader.stats.AddError(errMsg)
		}
		return nil
	}
	if resp.Response.StatusCode != 201 {
		reader.processPharosError(resp)
		return nil
	}

	savedWorkItem := resp.WorkItem()
	reader.Context.MessageLog.Debug("Created WorkItem with id %d for %s/%s in Pharos",
		savedWorkItem.Id, bucket, *s3Object.Key)
	if reader.stats != nil {
		reader.stats.AddWorkItem("WorkItemsCreated", savedWorkItem)
	}
	return savedWorkItem
}

func (reader *APTBucketReader) addToNSQ(workItem *models.WorkItem) {
	client := network.NewNSQClient(reader.Context.Config.NsqdHttpAddress)
	err := client.Enqueue(reader.Context.Config.FetchWorker.NsqTopic, workItem.Id)
	if err != nil {
		msg := fmt.Sprintf("Error sending WorkItem %d to NSQ: %v", workItem.Id, err)
		if reader.stats != nil {
			reader.stats.AddError(msg)
		}
		reader.Context.MessageLog.Error(msg)
		return
	}
	reader.Context.MessageLog.Info("Added WorkItem id %d to NSQ (%s/%s)",
		workItem.Id, workItem.Bucket, workItem.Name)
	if reader.stats != nil {
		reader.stats.AddWorkItem("WorkItemsQueued", workItem)
	}
	return
}

func (reader *APTBucketReader) markAsQueued(workItem *models.WorkItem) *models.WorkItem {
	utcNow := time.Now().UTC()
	workItem.QueuedAt = &utcNow
	resp := reader.Context.PharosClient.WorkItemSave(workItem)
	if resp.Error != nil {
		errMsg := fmt.Sprintf("Error setting QueuedAt for WorkItem with id %d: %v",
			workItem.Id, resp.Error)
		reader.Context.MessageLog.Debug("%s", resp.Request.URL.String())
		reader.Context.MessageLog.Error(errMsg)
		if reader.stats != nil {
			reader.stats.AddError(errMsg)
		}
		return nil
	}
	if resp.Response.StatusCode != 200 {
		reader.processPharosError(resp)
		return nil
	}
	if reader.stats != nil {
		reader.stats.AddWorkItem("WorkItemsMarkedAsQueued", workItem)
	}
	return resp.WorkItem()
}

func (reader *APTBucketReader) processPharosError(resp *network.PharosResponse) error {
	respBody := ""
	bytesRead, readErr := resp.RawResponseData()
	if readErr == nil {
		respBody = string(bytesRead)
	} else {
		respBody = fmt.Sprintf("[Could not read response body: %v]", readErr)
	}
	msg := fmt.Sprintf("%s %s returned status code %d. Response body: %s",
		resp.Request.Method, resp.Request.URL, resp.Response.StatusCode, respBody)
	reader.Context.MessageLog.Error(msg)
	if reader.stats != nil {
		reader.stats.AddError(msg)
	}
	return fmt.Errorf(msg)
}

func (reader *APTBucketReader) GetStats() *stats.APTBucketReaderStats {
	return reader.stats
}
<commit_msg>Remove quotes from etag for lookups<commit_after>package workers

import (
	"fmt"
	"github.com/APTrust/exchange/constants"
	"github.com/APTrust/exchange/context"
	"github.com/APTrust/exchange/models"
	"github.com/APTrust/exchange/network"
	"github.com/APTrust/exchange/stats"
	"github.com/APTrust/exchange/util"
	"github.com/aws/aws-sdk-go/service/s3"
	"net/url"
	"strconv"
	"strings"
	"time"
)

// If Config.BucketReaderCacheHours isn't set to something
// sensible, cache Ingest WorkItems up to this many hours old.
const DEFAULT_CACHE_HOURS = 24

// How many S3 keys should we fetch in each batch when
// we're getting the contents of a bucket?
const MAX_KEYS = 1000

// APTBucketReader scans APTrust receiving buckets for items
// that need to be ingested. It creates a WorkItem record and
// an NSQ entry for each qualifying bag, and is responsible for
// knowing whether items in the receiving buckets actually need
// to be queued.
type APTBucketReader struct {
	Context           *context.Context
	Institutions      map[string]*models.Institution
	RecentIngestItems map[string]*models.WorkItem
	stats             *stats.APTBucketReaderStats
	statsEnabled      bool
}

// Creates a new bucket reader with the given context.
// Param enableStats is generally used for integration tests.
// Enabling stats in production can cause high memory usage,
// so keep that off unless you're trying to diagnose specific problems.
func NewAPTBucketReader(context *context.Context, enableStats bool) *APTBucketReader {
	reader := &APTBucketReader{
		Context:           context,
		Institutions:      make(map[string]*models.Institution),
		RecentIngestItems: make(map[string]*models.WorkItem),
		statsEnabled:      enableStats,
	}
	if enableStats {
		reader.stats = stats.NewAPTBucketReaderStats()
	}
	return reader
}

func (reader *APTBucketReader) Run() error {
	err := reader.cacheInstitutions()
	if err != nil {
		return err
	}
	err = reader.cacheRecentIngestItems()
	if err != nil {
		return err
	}
	reader.readAllBuckets()
	return nil
}

// Cache a list of all institutions. There are < 20.
// Exit on failure.
func (reader *APTBucketReader) cacheInstitutions() error {
	params := url.Values{}
	params.Add("page", "1")
	params.Add("per_page", "100")
	resp := reader.Context.PharosClient.InstitutionList(params)
	if resp.Error != nil {
		if reader.stats != nil {
			reader.stats.AddError(resp.Error.Error())
		}
		return resp.Error
	}
	if resp.Response.StatusCode != 200 {
		return reader.processPharosError(resp)
	}
	for _, inst := range resp.Institutions() {
		reader.Institutions[inst.Identifier] = inst
		if reader.stats != nil {
			reader.stats.AddToInstitutionsCached(inst)
		}
	}
	reader.Context.MessageLog.Info("Loaded %d institutions", len(reader.Institutions))
	return nil
}

// Cache a list of Ingest items that have been added to
// the list of WorkItems in the past X hours, so we won't
// have to do 1000 lookups. (X is usually around 24, but
// check Config.BucketReaderCacheHours for the exact value.
// This function exits on failure.
func (reader *APTBucketReader) cacheRecentIngestItems() error {
	hours := reader.Context.Config.BucketReaderCacheHours
	if hours < 1 {
		hours = DEFAULT_CACHE_HOURS
	}
	createdAfter := time.Now().Add(time.Duration(-1*hours) * time.Hour).UTC()
	params := url.Values{}
	params.Add("page", "1")
	params.Add("per_page", "100") // change to 4 to test hasMoreResults loop
	params.Add("item_action", constants.ActionIngest)
	params.Add("created_after", createdAfter.Format(time.RFC3339))
	hasMoreResults := true
	for hasMoreResults {
		resp := reader.Context.PharosClient.WorkItemList(params)
		if resp.Error != nil {
			if reader.stats != nil {
				reader.stats.AddError(resp.Error.Error())
			}
			return resp.Error
		}
		if resp.Response.StatusCode != 200 {
			return reader.processPharosError(resp)
		}
		reader.Context.MessageLog.Debug("%s", resp.Request.URL.String())
		for _, workItem := range resp.WorkItems() {
			hashKey := reader.makeHashKey(workItem.Name, workItem.ETag)
			reader.RecentIngestItems[hashKey] = workItem
			if reader.stats != nil {
				reader.stats.AddWorkItem("WorkItemsCached", workItem)
			}
		}
		if resp.Next != nil {
			pageNum, err := strconv.Atoi(params.Get("page"))
			if err != nil {
				msg := fmt.Sprintf("Page number '%s' doesn't look like a number!", params.Get("page"))
				if reader.stats != nil {
					reader.stats.AddError(msg)
				}
				return fmt.Errorf(msg)
			}
			params.Set("page", strconv.Itoa(pageNum+1))
		} else {
			hasMoreResults = false
		}
	}
	reader.Context.MessageLog.Info("Loaded %d recent ingest WorkItems", len(reader.RecentIngestItems))
	return nil
}

func (reader *APTBucketReader) makeHashKey(key, etag string) string {
	return fmt.Sprintf("%s|%s", key, strings.Replace(etag, "\"", "", -1))
}

func (reader *APTBucketReader) readAllBuckets() {
	for _, bucketName := range reader.Context.Config.ReceivingBuckets {
		reader.processBucket(bucketName)
	}
}

func (reader *APTBucketReader) processBucket(bucketName string) {
	reader.Context.MessageLog.Debug("Checking bucket %s", bucketName)
	s3ObjList := network.NewS3ObjectList(reader.Context.Config.APTrustS3Region,
		bucketName, MAX_KEYS)
	keepFetching := true
	for keepFetching {
		s3ObjList.GetList("")
		if s3ObjList.ErrorMessage != "" {
			if reader.stats != nil {
				reader.stats.AddError(s3ObjList.ErrorMessage)
			}
			reader.Context.MessageLog.Error(s3ObjList.ErrorMessage)
			break
		}
		for _, s3Object := range s3ObjList.Response.Contents {
			reader.stats.AddS3Item(fmt.Sprintf("%s/%s", bucketName, *s3Object.Key))
			reader.processS3Object(s3Object, bucketName)
		}
		keepFetching = *s3ObjList.Response.IsTruncated
	}
}

func (reader *APTBucketReader) processS3Object(s3Object *s3.Object, bucketName string) {
	if reader.Context.Config.MaxFileSize > int64(0) && *s3Object.Size > reader.Context.Config.MaxFileSize {
		msg := fmt.Sprintf("Skipping %s/%s because size %d is greater than "+
			"current max file size %d", bucketName, *s3Object.Key, *s3Object.Size,
			reader.Context.Config.MaxFileSize)
		if reader.stats != nil {
			reader.stats.AddWarning(msg)
		}
		reader.Context.MessageLog.Debug(msg)
		return
	}
	workItem, err := reader.findWorkItem(*s3Object.Key, *s3Object.ETag)
	if err != nil {
		// Don't create a work item, because one may already exist.
		// Error will be logged and added to stats at source.
		msg := fmt.Sprintf("Not creating WorkItem for %s/%s because "+
			"we can't tell if one already exists.", bucketName, *s3Object.Key)
		reader.Context.MessageLog.Warning(msg)
		if reader.stats != nil {
			reader.stats.AddWarning(msg)
		}
		return
	}
	if workItem == nil {
		workItem = reader.createWorkItem(bucketName, s3Object)
		if workItem == nil {
			// Error logged and statted at source.
			return
		}
	}
	if workItem.QueuedAt == nil || workItem.QueuedAt.IsZero() {
		reader.addToNSQ(workItem)
		reader.markAsQueued(workItem)
	}
}

func (reader *APTBucketReader) findWorkItem(key, etag string) (*models.WorkItem, error) {
	etag = strings.Replace(etag, "\"", "", -1)
	hashKey := reader.makeHashKey(key, etag)
	if workItem, ok := reader.RecentIngestItems[hashKey]; ok {
		reader.Context.MessageLog.Debug("Found hash key '%s' in cache", hashKey)
		return workItem, nil
	}
	reader.Context.MessageLog.Debug("Looking up hash key '%s' in Pharos", hashKey)
	params := url.Values{}
	params.Add("page", "1")
	params.Add("per_page", "1")
	params.Add("item_action", constants.ActionIngest)
	params.Add("name", key)
	params.Add("etag", etag)
	//params.Add("bag_date", lastModified.Format(time.RFC3339))
	resp := reader.Context.PharosClient.WorkItemList(params)
	if resp.Error != nil {
		errMsg := fmt.Sprintf("Error getting WorkItem for name '%s', etag '%s': %v",
			params.Get("name"), params.Get("etag"), resp.Error)
		reader.Context.MessageLog.Debug("%s", resp.Request.URL.String())
		reader.Context.MessageLog.Error(errMsg)
		if reader.stats != nil {
			reader.stats.AddError(errMsg)
		}
		return nil, fmt.Errorf(errMsg)
	}
	if resp.Response.StatusCode != 200 {
		err := reader.processPharosError(resp)
		return nil, err
	}
	workItem := resp.WorkItem()
	if workItem != nil {
		reader.Context.MessageLog.Debug("Found WorkItem for hash key '%s' in Pharos", hashKey)
		if reader.stats != nil {
			reader.stats.AddWorkItem("WorkItemsFetched", workItem)
		}
	} else {
		reader.Context.MessageLog.Debug("Did not find WorkItem for hash key '%s' in Pharos", hashKey)
	}
	return workItem, nil
}

func (reader *APTBucketReader) createWorkItem(bucket string, s3Object *s3.Object) *models.WorkItem {
	// Create a WorkItem in Pharos
	institution := reader.Institutions[util.OwnerOf(bucket)]
	if institution == nil {
		errMsg := fmt.Sprintf("Cannot find institution record for item %s/%s. "+
			"Owner computes to '%s'", bucket, *s3Object.Key, util.OwnerOf(bucket))
		reader.Context.MessageLog.Error(errMsg)
		if reader.stats != nil {
			reader.stats.AddError(errMsg)
		}
		return nil
	}
	workItem := &models.WorkItem{}
	workItem.Id = 0
	workItem.Name = *s3Object.Key
	workItem.Bucket = bucket
	workItem.ETag = strings.Replace(*s3Object.ETag, "\"", "", -1)
	workItem.Size = *s3Object.Size
	workItem.BagDate = *s3Object.LastModified
	workItem.InstitutionId = institution.Id
	workItem.User = constants.APTrustSystemUser
	workItem.Date = time.Now().UTC()
	workItem.Note = "Bag is in receiving bucket"
	workItem.Action = constants.ActionIngest
	workItem.Stage = constants.StageReceive
	workItem.Status = constants.StatusPending
	workItem.Outcome = "Item is pending ingest"
	workItem.Retry = true
	resp := reader.Context.PharosClient.WorkItemSave(workItem)

	if resp.Error != nil {
		errMsg := fmt.Sprintf("Error creating WorkItem for name '%s', etag '%s', time '%s': %v",
			workItem.Name, workItem.ETag, workItem.BagDate, resp.Error)
		reader.Context.MessageLog.Debug("%s", resp.Request.URL.String())
		reader.Context.MessageLog.Error(errMsg)
		if reader.stats != nil {
			reader.stats.AddError(errMsg)
		}
		return nil
	}
	if resp.Response.StatusCode != 201 {
		reader.processPharosError(resp)
		return nil
	}

	savedWorkItem := resp.WorkItem()
	reader.Context.MessageLog.Debug("Created WorkItem with id %d for %s/%s in Pharos",
		savedWorkItem.Id, bucket, *s3Object.Key)
	if reader.stats != nil {
		reader.stats.AddWorkItem("WorkItemsCreated", savedWorkItem)
	}
	return savedWorkItem
}

func (reader *APTBucketReader) addToNSQ(workItem *models.WorkItem) {
	client := network.NewNSQClient(reader.Context.Config.NsqdHttpAddress)
	err := client.Enqueue(reader.Context.Config.FetchWorker.NsqTopic, workItem.Id)
	if err != nil {
		msg := fmt.Sprintf("Error sending WorkItem %d to NSQ: %v", workItem.Id, err)
		if reader.stats != nil {
			reader.stats.AddError(msg)
		}
		reader.Context.MessageLog.Error(msg)
		return
	}
	reader.Context.MessageLog.Info("Added WorkItem id %d to NSQ (%s/%s)",
		workItem.Id, workItem.Bucket, workItem.Name)
	if reader.stats != nil {
		reader.stats.AddWorkItem("WorkItemsQueued", workItem)
	}
	return
}

func (reader *APTBucketReader) markAsQueued(workItem *models.WorkItem) *models.WorkItem {
	utcNow := time.Now().UTC()
	workItem.QueuedAt = &utcNow
	resp := reader.Context.PharosClient.WorkItemSave(workItem)
	if resp.Error != nil {
		errMsg := fmt.Sprintf("Error setting QueuedAt for WorkItem with id %d: %v",
			workItem.Id, resp.Error)
		reader.Context.MessageLog.Debug("%s", resp.Request.URL.String())
		reader.Context.MessageLog.Error(errMsg)
		if reader.stats != nil {
			reader.stats.AddError(errMsg)
		}
		return nil
	}
	if resp.Response.StatusCode != 200 {
		reader.processPharosError(resp)
		return nil
	}
	if reader.stats != nil {
		reader.stats.AddWorkItem("WorkItemsMarkedAsQueued", workItem)
	}
	return resp.WorkItem()
}

func (reader *APTBucketReader) processPharosError(resp *network.PharosResponse) error {
	respBody := ""
	bytesRead, readErr := resp.RawResponseData()
	if readErr == nil {
		respBody = string(bytesRead)
	} else {
		respBody = fmt.Sprintf("[Could not read response body: %v]", readErr)
	}
	msg := fmt.Sprintf("%s %s returned status code %d. Response body: %s",
		resp.Request.Method, resp.Request.URL, resp.Response.StatusCode, respBody)
	reader.Context.MessageLog.Error(msg)
	if reader.stats != nil {
		reader.stats.AddError(msg)
	}
	return fmt.Errorf(msg)
}

func (reader *APTBucketReader) GetStats() *stats.APTBucketReaderStats {
	return reader.stats
}
<|endoftext|>


<commit_before>package workflows

import (
	"bytes"
	"fmt"
	"regexp"
	"strconv"
	"strings"

	"github.com/stelligent/mu/common"
	"github.com/stelligent/mu/templates"
)

// NewPipelineUpserter create a new workflow for upserting a pipeline
func NewPipelineUpserter(ctx *common.Context, tokenProvider func(bool) string) Executor {

	workflow := new(pipelineWorkflow)
	workflow.codeRevision = ctx.Config.Repo.Revision
	workflow.repoName = ctx.Config.Repo.Slug

	if ctx.Config.Repo.Branch != "" {
		workflow.codeBranch = ctx.Config.Repo.Branch
	} else {
		workflow.codeBranch = ctx.Config.Service.Pipeline.Source.Branch
	}

	stackParams := make(map[string]string)

	return newPipelineExecutor(
		workflow.serviceFinder("", ctx),
		workflow.pipelineBucket(ctx.Config.Namespace, ctx.StackManager, ctx.StackManager),
		workflow.pipelineRolesetUpserter(ctx.RolesetManager, ctx.RolesetManager, stackParams),
		workflow.pipelineUpserter(ctx.Config.Namespace, tokenProvider, ctx.StackManager, ctx.StackManager, stackParams),
	)
}

// Setup the artifact bucket
func (workflow *pipelineWorkflow) pipelineBucket(namespace string, stackUpserter common.StackUpserter, stackWaiter common.StackWaiter) Executor {

	return func() error {
		bucketStackName := common.CreateStackName(namespace, common.StackTypeBucket, "codepipeline")
		overrides := common.GetStackOverrides(bucketStackName)
		template, err := templates.NewTemplate("bucket.yml", nil, overrides)
		if err != nil {
			return err
		}
		log.Noticef("Upserting Bucket for CodePipeline")
		bucketParams := make(map[string]string)
		bucketParams["Namespace"] = namespace
		bucketParams["BucketPrefix"] = "codepipeline"

		var pipeTags TagInterface = &PipelineTags{
			Type: common.StackTypeBucket,
		}
		tags, err := concatTags(workflow.pipelineConfig.Tags, pipeTags)
		if err != nil {
			return err
		}

		err = stackUpserter.UpsertStack(bucketStackName, template, bucketParams, tags, "")
		if err != nil {
			// ignore error if stack is in progress already
			if !strings.Contains(err.Error(), "_IN_PROGRESS state and can not be updated") {
				return err
			}
		}

		log.Debugf("Waiting for stack '%s' to complete", bucketStackName)
		stack := stackWaiter.AwaitFinalStatus(bucketStackName)
		if stack == nil {
			return fmt.Errorf("Unable to create stack %s", bucketStackName)
		}
		if strings.HasSuffix(stack.Status, "ROLLBACK_COMPLETE") || !strings.HasSuffix(stack.Status, "_COMPLETE") {
			return fmt.Errorf("Ended in failed status %s %s", stack.Status, stack.StatusReason)
		}

		return nil
	}
}

func (workflow *pipelineWorkflow) pipelineRolesetUpserter(rolesetUpserter common.RolesetUpserter, rolesetGetter common.RolesetGetter, params map[string]string) Executor {
	return func() error {
		err := rolesetUpserter.UpsertCommonRoleset()
		if err != nil {
			return err
		}

		if !workflow.pipelineConfig.Acceptance.Disabled {
			envName := workflow.pipelineConfig.Acceptance.Environment
			if envName == "" {
				envName = "acceptance"
			}
			err := rolesetUpserter.UpsertEnvironmentRoleset(envName)
			if err != nil {
				return err
			}

			err = rolesetUpserter.UpsertServiceRoleset(envName, workflow.serviceName)
			if err != nil {
				return err
			}

		}

		if !workflow.pipelineConfig.Production.Disabled {
			envName := workflow.pipelineConfig.Production.Environment
			if envName == "" {
				envName = "production"
			}
			err := rolesetUpserter.UpsertEnvironmentRoleset(envName)
			if err != nil {
				return err
			}

			err = rolesetUpserter.UpsertServiceRoleset(envName, workflow.serviceName)
			if err != nil {
				return err
			}
		}

		err = rolesetUpserter.UpsertPipelineRoleset(workflow.serviceName)
		if err != nil {
			return err
		}

		pipelineRoleset, err := rolesetGetter.GetPipelineRoleset(workflow.serviceName)
		if err != nil {
			return err
		}

		for roleType, roleArn := range pipelineRoleset {
			if roleArn != "" {
				params[roleType] = roleArn
			}
		}

		return nil
	}
}

func (workflow *pipelineWorkflow) pipelineUpserter(namespace string, tokenProvider func(bool) string, stackUpserter common.StackUpserter, stackWaiter common.StackWaiter, params map[string]string) Executor {
	return func() error {
		pipelineStackName := common.CreateStackName(namespace, common.StackTypePipeline, workflow.serviceName)
		pipelineStack := stackWaiter.AwaitFinalStatus(pipelineStackName)
		overrides := common.GetStackOverrides(pipelineStackName)

		log.Noticef("Upserting Pipeline for service '%s' ...", workflow.serviceName)
		template, err := templates.NewTemplate("pipeline.yml", nil, overrides)
		if err != nil {
			return err
		}
		pipelineParams := params

		pipelineParams["Namespace"] = namespace
		pipelineParams["ServiceName"] = workflow.serviceName
		pipelineParams["MuFile"] = workflow.muFilegca
		pipelineParams["SourceProvider"] = workflow.pipelineConfig.Source.Provider
		pipelineParams["SourceRepo"] = workflow.pipelineConfig.Source.Repo

		if workflow.codeBranch != "" {
			pipelineParams["SourceBranch"] = workflow.codeBranch
		}

		if workflow.pipelineConfig.Source.Provider == "S3" {
			repoParts := strings.Split(workflow.pipelineConfig.Source.Repo, "/")
			pipelineParams["SourceBucket"] = repoParts[0]
			pipelineParams["SourceObjectKey"] = strings.Join(repoParts[1:], "/")
		}

		if workflow.pipelineConfig.Source.Provider == "GitHub" {
			pipelineParams["GitHubToken"] = tokenProvider(pipelineStack == nil)
		}

		if workflow.pipelineConfig.Build.Type != "" {
			pipelineParams["BuildType"] = workflow.pipelineConfig.Build.Type
		}
		if workflow.pipelineConfig.Build.ComputeType != "" {
			pipelineParams["BuildComputeType"] = workflow.pipelineConfig.Build.ComputeType
		}

		if workflow.pipelineConfig.Build.Image != "" {
			pipelineParams["BuildImage"] = workflow.pipelineConfig.Build.Image
		}

		if workflow.pipelineConfig.Acceptance.Type != "" {
			pipelineParams["TestType"] = workflow.pipelineConfig.Acceptance.Type
		}
		if workflow.pipelineConfig.Acceptance.ComputeType != "" {
			pipelineParams["TestComputeType"] = workflow.pipelineConfig.Acceptance.ComputeType
		}

		if workflow.pipelineConfig.Acceptance.Image != "" {
			pipelineParams["TestImage"] = workflow.pipelineConfig.Acceptance.Image
		}

		if workflow.pipelineConfig.Acceptance.Environment != "" {
			pipelineParams["AcptEnv"] = workflow.pipelineConfig.Acceptance.Environment
		}

		if workflow.pipelineConfig.Production.Environment != "" {
			pipelineParams["ProdEnv"] = workflow.pipelineConfig.Production.Environment
		}

		if workflow.pipelineConfig.MuBaseurl != "" {
			pipelineParams["MuDownloadBaseurl"] = workflow.pipelineConfig.MuBaseurl
		}

		pipelineParams["EnableBuildStage"] = strconv.FormatBool(!workflow.pipelineConfig.Build.Disabled)
		pipelineParams["EnableAcptStage"] = strconv.FormatBool(!workflow.pipelineConfig.Acceptance.Disabled)
		pipelineParams["EnableProdStage"] = strconv.FormatBool(!workflow.pipelineConfig.Production.Disabled)

		// get default buildspec
		buildspec, err := templates.NewTemplate("buildspec.yml", nil, nil)
		if err != nil {
			return err
		}
		buildspecBytes := new(bytes.Buffer)
		buildspecBytes.ReadFrom(buildspec)
		newlineRegexp := regexp.MustCompile(`\r?\n`)
		buildspecString := newlineRegexp.ReplaceAllString(buildspecBytes.String(), "\\n")
		pipelineParams["DefaultBuildspec"] = buildspecString

		version := workflow.pipelineConfig.MuVersion
		if version == "" {
			version = common.GetVersion()
			if version == "0.0.0-local" {
				version = ""
			}
		}
		if version != "" {
			pipelineParams["MuDownloadVersion"] = version
		}
		var pipeTags TagInterface = &PipelineTags{
			Type:     common.StackTypePipeline,
			Service:  workflow.serviceName,
			Revision: workflow.codeRevision,
			Repo:     workflow.repoName,
		}
		tags, err := concatTags(workflow.pipelineConfig.Tags, pipeTags)
		if err != nil {
			return err
		}

		err = stackUpserter.UpsertStack(pipelineStackName, template, pipelineParams, tags, "")
		if err != nil {
			return err
		}

		log.Debugf("Waiting for stack '%s' to complete", pipelineStackName)
		stack := stackWaiter.AwaitFinalStatus(pipelineStackName)
		if stack == nil {
			return fmt.Errorf("Unable to create stack %s", pipelineStackName)
		}
		if strings.HasSuffix(stack.Status, "ROLLBACK_COMPLETE") || !strings.HasSuffix(stack.Status, "_COMPLETE") {
			return fmt.Errorf("Ended in failed status %s %s", stack.Status, stack.StatusReason)
		}

		return nil
	}
}
<commit_msg>remove typo<commit_after>package workflows

import (
	"bytes"
	"fmt"
	"regexp"
	"strconv"
	"strings"

	"github.com/stelligent/mu/common"
	"github.com/stelligent/mu/templates"
)

// NewPipelineUpserter create a new workflow for upserting a pipeline
func NewPipelineUpserter(ctx *common.Context, tokenProvider func(bool) string) Executor {

	workflow := new(pipelineWorkflow)
	workflow.codeRevision = ctx.Config.Repo.Revision
	workflow.repoName = ctx.Config.Repo.Slug

	if ctx.Config.Repo.Branch != "" {
		workflow.codeBranch = ctx.Config.Repo.Branch
	} else {
		workflow.codeBranch = ctx.Config.Service.Pipeline.Source.Branch
	}

	stackParams := make(map[string]string)

	return newPipelineExecutor(
		workflow.serviceFinder("", ctx),
		workflow.pipelineBucket(ctx.Config.Namespace, ctx.StackManager, ctx.StackManager),
		workflow.pipelineRolesetUpserter(ctx.RolesetManager, ctx.RolesetManager, stackParams),
		workflow.pipelineUpserter(ctx.Config.Namespace, tokenProvider, ctx.StackManager, ctx.StackManager, stackParams),
	)
}

// Setup the artifact bucket
func (workflow *pipelineWorkflow) pipelineBucket(namespace string, stackUpserter common.StackUpserter, stackWaiter common.StackWaiter) Executor {

	return func() error {
		bucketStackName := common.CreateStackName(namespace, common.StackTypeBucket, "codepipeline")
		overrides := common.GetStackOverrides(bucketStackName)
		template, err := templates.NewTemplate("bucket.yml", nil, overrides)
		if err != nil {
			return err
		}
		log.Noticef("Upserting Bucket for CodePipeline")
		bucketParams := make(map[string]string)
		bucketParams["Namespace"] = namespace
		bucketParams["BucketPrefix"] = "codepipeline"

		var pipeTags TagInterface = &PipelineTags{
			Type: common.StackTypeBucket,
		}
		tags, err := concatTags(workflow.pipelineConfig.Tags, pipeTags)
		if err != nil {
			return err
		}

		err = stackUpserter.UpsertStack(bucketStackName, template, bucketParams, tags, "")
		if err != nil {
			// ignore error if stack is in progress already
			if !strings.Contains(err.Error(), "_IN_PROGRESS state and can not be updated") {
				return err
			}
		}

		log.Debugf("Waiting for stack '%s' to complete", bucketStackName)
		stack := stackWaiter.AwaitFinalStatus(bucketStackName)
		if stack == nil {
			return fmt.Errorf("Unable to create stack %s", bucketStackName)
		}
		if strings.HasSuffix(stack.Status, "ROLLBACK_COMPLETE") || !strings.HasSuffix(stack.Status, "_COMPLETE") {
			return fmt.Errorf("Ended in failed status %s %s", stack.Status, stack.StatusReason)
		}

		return nil
	}
}

func (workflow *pipelineWorkflow) pipelineRolesetUpserter(rolesetUpserter common.RolesetUpserter, rolesetGetter common.RolesetGetter, params map[string]string) Executor {
	return func() error {
		err := rolesetUpserter.UpsertCommonRoleset()
		if err != nil {
			return err
		}

		if !workflow.pipelineConfig.Acceptance.Disabled {
			envName := workflow.pipelineConfig.Acceptance.Environment
			if envName == "" {
				envName = "acceptance"
			}
			err := rolesetUpserter.UpsertEnvironmentRoleset(envName)
			if err != nil {
				return err
			}

			err = rolesetUpserter.UpsertServiceRoleset(envName, workflow.serviceName)
			if err != nil {
				return err
			}

		}

		if !workflow.pipelineConfig.Production.Disabled {
			envName := workflow.pipelineConfig.Production.Environment
			if envName == "" {
				envName = "production"
			}
			err := rolesetUpserter.UpsertEnvironmentRoleset(envName)
			if err != nil {
				return err
			}

			err = rolesetUpserter.UpsertServiceRoleset(envName, workflow.serviceName)
			if err != nil {
				return err
			}
		}

		err = rolesetUpserter.UpsertPipelineRoleset(workflow.serviceName)
		if err != nil {
			return err
		}

		pipelineRoleset, err := rolesetGetter.GetPipelineRoleset(workflow.serviceName)
		if err != nil {
			return err
		}

		for roleType, roleArn := range pipelineRoleset {
			if roleArn != "" {
				params[roleType] = roleArn
			}
		}

		return nil
	}
}

func (workflow *pipelineWorkflow) pipelineUpserter(namespace string, tokenProvider func(bool) string, stackUpserter common.StackUpserter, stackWaiter common.StackWaiter, params map[string]string) Executor {
	return func() error {
		pipelineStackName := common.CreateStackName(namespace, common.StackTypePipeline, workflow.serviceName)
		pipelineStack := stackWaiter.AwaitFinalStatus(pipelineStackName)
		overrides := common.GetStackOverrides(pipelineStackName)

		log.Noticef("Upserting Pipeline for service '%s' ...", workflow.serviceName)
		template, err := templates.NewTemplate("pipeline.yml", nil, overrides)
		if err != nil {
			return err
		}
		pipelineParams := params

		pipelineParams["Namespace"] = namespace
		pipelineParams["ServiceName"] = workflow.serviceName
		pipelineParams["MuFile"] = workflow.muFile
		pipelineParams["SourceProvider"] = workflow.pipelineConfig.Source.Provider
		pipelineParams["SourceRepo"] = workflow.pipelineConfig.Source.Repo

		if workflow.codeBranch != "" {
			pipelineParams["SourceBranch"] = workflow.codeBranch
		}

		if workflow.pipelineConfig.Source.Provider == "S3" {
			repoParts := strings.Split(workflow.pipelineConfig.Source.Repo, "/")
			pipelineParams["SourceBucket"] = repoParts[0]
			pipelineParams["SourceObjectKey"] = strings.Join(repoParts[1:], "/")
		}

		if workflow.pipelineConfig.Source.Provider == "GitHub" {
			pipelineParams["GitHubToken"] = tokenProvider(pipelineStack == nil)
		}

		if workflow.pipelineConfig.Build.Type != "" {
			pipelineParams["BuildType"] = workflow.pipelineConfig.Build.Type
		}
		if workflow.pipelineConfig.Build.ComputeType != "" {
			pipelineParams["BuildComputeType"] = workflow.pipelineConfig.Build.ComputeType
		}

		if workflow.pipelineConfig.Build.Image != "" {
			pipelineParams["BuildImage"] = workflow.pipelineConfig.Build.Image
		}

		if workflow.pipelineConfig.Acceptance.Type != "" {
			pipelineParams["TestType"] = workflow.pipelineConfig.Acceptance.Type
		}
		if workflow.pipelineConfig.Acceptance.ComputeType != "" {
			pipelineParams["TestComputeType"] = workflow.pipelineConfig.Acceptance.ComputeType
		}

		if workflow.pipelineConfig.Acceptance.Image != "" {
			pipelineParams["TestImage"] = workflow.pipelineConfig.Acceptance.Image
		}

		if workflow.pipelineConfig.Acceptance.Environment != "" {
			pipelineParams["AcptEnv"] = workflow.pipelineConfig.Acceptance.Environment
		}

		if workflow.pipelineConfig.Production.Environment != "" {
			pipelineParams["ProdEnv"] = workflow.pipelineConfig.Production.Environment
		}

		if workflow.pipelineConfig.MuBaseurl != "" {
			pipelineParams["MuDownloadBaseurl"] = workflow.pipelineConfig.MuBaseurl
		}

		pipelineParams["EnableBuildStage"] = strconv.FormatBool(!workflow.pipelineConfig.Build.Disabled)
		pipelineParams["EnableAcptStage"] = strconv.FormatBool(!workflow.pipelineConfig.Acceptance.Disabled)
		pipelineParams["EnableProdStage"] = strconv.FormatBool(!workflow.pipelineConfig.Production.Disabled)

		// get default buildspec
		buildspec, err := templates.NewTemplate("buildspec.yml", nil, nil)
		if err != nil {
			return err
		}
		buildspecBytes := new(bytes.Buffer)
		buildspecBytes.ReadFrom(buildspec)
		newlineRegexp := regexp.MustCompile(`\r?\n`)
		buildspecString := newlineRegexp.ReplaceAllString(buildspecBytes.String(), "\\n")
		pipelineParams["DefaultBuildspec"] = buildspecString

		version := workflow.pipelineConfig.MuVersion
		if version == "" {
			version = common.GetVersion()
			if version == "0.0.0-local" {
				version = ""
			}
		}
		if version != "" {
			pipelineParams["MuDownloadVersion"] = version
		}
		var pipeTags TagInterface = &PipelineTags{
			Type:     common.StackTypePipeline,
			Service:  workflow.serviceName,
			Revision: workflow.codeRevision,
			Repo:     workflow.repoName,
		}
		tags, err := concatTags(workflow.pipelineConfig.Tags, pipeTags)
		if err != nil {
			return err
		}

		err = stackUpserter.UpsertStack(pipelineStackName, template, pipelineParams, tags, "")
		if err != nil {
			return err
		}

		log.Debugf("Waiting for stack '%s' to complete", pipelineStackName)
		stack := stackWaiter.AwaitFinalStatus(pipelineStackName)
		if stack == nil {
			return fmt.Errorf("Unable to create stack %s", pipelineStackName)
		}
		if strings.HasSuffix(stack.Status, "ROLLBACK_COMPLETE") || !strings.HasSuffix(stack.Status, "_COMPLETE") {
			return fmt.Errorf("Ended in failed status %s %s", stack.Status, stack.StatusReason)
		}

		return nil
	}
}
<|endoftext|>


<commit_before>package handler

import (
	"encoding/json"
	"fmt"
	"fullerite/metric"

	l "github.com/Sirupsen/logrus"
)

// Log type
type Log struct {
	BaseHandler
}

// NewLog returns a new Debug handler.
func NewLog(
	channel chan metric.Metric,
	initialInterval int,
	initialBufferSize int,
	log *l.Entry) *Log {

	inst := new(Log)
	inst.name = "Log"

	inst.interval = initialInterval
	inst.maxBufferSize = initialBufferSize
	inst.log = log
	inst.channel = channel

	return inst
}

// Configure accepts the different configuration options for the Log handler
func (h *Log) Configure(configMap map[string]interface{}) {
	h.configureCommonParams(configMap)
}

// Run runs the handler main loop
func (h *Log) Run() {
	h.run(h.emitMetrics)
}

func (h *Log) convertToLog(incomingMetric metric.Metric) (string, error) {
	jsonOut, err := json.Marshal(incomingMetric)
	return string(jsonOut), err
}

func (h *Log) emitMetrics(metrics []metric.Metric) bool {
	h.log.Info("Starting to emit ", len(metrics), " metrics")

	if len(metrics) == 0 {
		h.log.Warn("Skipping send because of an empty payload")
		return false
	}

	for _, m := range metrics {
		if dpString, err := h.convertToLog(m); err == nil {
			h.log.Error(fmt.Sprintf("Cannot convert metric %q to JSON: %s", m, err))
			continue
		} else {
			h.log.Info(dpString)
		}
	}
	return true
}
<commit_msg>Fix typo in error check for Log handler<commit_after>package handler

import (
	"encoding/json"
	"fmt"
	"fullerite/metric"

	l "github.com/Sirupsen/logrus"
)

// Log type
type Log struct {
	BaseHandler
}

// NewLog returns a new Debug handler.
func NewLog(
	channel chan metric.Metric,
	initialInterval int,
	initialBufferSize int,
	log *l.Entry) *Log {

	inst := new(Log)
	inst.name = "Log"

	inst.interval = initialInterval
	inst.maxBufferSize = initialBufferSize
	inst.log = log
	inst.channel = channel

	return inst
}

// Configure accepts the different configuration options for the Log handler
func (h *Log) Configure(configMap map[string]interface{}) {
	h.configureCommonParams(configMap)
}

// Run runs the handler main loop
func (h *Log) Run() {
	h.run(h.emitMetrics)
}

func (h *Log) convertToLog(incomingMetric metric.Metric) (string, error) {
	jsonOut, err := json.Marshal(incomingMetric)
	return string(jsonOut), err
}

func (h *Log) emitMetrics(metrics []metric.Metric) bool {
	h.log.Info("Starting to emit ", len(metrics), " metrics")

	if len(metrics) == 0 {
		h.log.Warn("Skipping send because of an empty payload")
		return false
	}

	for _, m := range metrics {
		if dpString, err := h.convertToLog(m); err != nil {
			h.log.Error(fmt.Sprintf("Cannot convert metric %q to JSON: %s", m, err))
			continue
		} else {
			h.log.Info(dpString)
		}
	}
	return true
}
<|endoftext|>


<commit_before>package stripe

import "encoding/json"

// InvoiceLineType is the list of allowed values for the invoice line's type.
type InvoiceLineType string

// List of values that InvoiceLineType can take.
const (
	InvoiceLineTypeInvoiceItem  InvoiceLineType = "invoiceitem"
	InvoiceLineTypeSubscription InvoiceLineType = "subscription"
)

// InvoiceBilling is the type of billing method for this invoice.
type InvoiceBilling string

// List of values that InvoiceBilling can take.
const (
	InvoiceBillingChargeAutomatically InvoiceBilling = "charge_automatically"
	InvoiceBillingSendInvoice         InvoiceBilling = "send_invoice"
)

// InvoiceBillingReason is the reason why a given invoice was created
type InvoiceBillingReason string

// List of values that InvoiceBillingReason can take.
const (
	InvoiceBillingReasonManual             InvoiceBillingReason = "manual"
	InvoiceBillingReasonSubscription       InvoiceBillingReason = "subscription"
	InvoiceBillingReasonSubscriptionCycle  InvoiceBillingReason = "subscription_cycle"
	InvoiceBillingReasonSubscriptionUpdate InvoiceBillingReason = "subscription_update"
	InvoiceBillingReasonUpcoming           InvoiceBillingReason = "upcoming"
)

// InvoiceUpcomingInvoiceItemParams is the set of parameters that can be used when adding or modifying
// invoice items on an upcoming invoice.
// For more details see https://stripe.com/docs/api#upcoming_invoice-invoice_items.
type InvoiceUpcomingInvoiceItemParams struct {
	Amount       *int64  `form:"amount"`
	Currency     *string `form:"currency"`
	Description  *string `form:"description"`
	Discountable *bool   `form:"discountable"`
	InvoiceItem  *string `form:"invoiceitem"`
}

// InvoiceParams is the set of parameters that can be used when creating or updating an invoice.
// For more details see https://stripe.com/docs/api#create_invoice, https://stripe.com/docs/api#update_invoice.
type InvoiceParams struct {
	Params              `form:"*"`
	ApplicationFee      *int64   `form:"application_fee"`
	Billing             *string  `form:"billing"`
	Closed              *bool    `form:"closed"`
	Customer            *string  `form:"customer"`
	DaysUntilDue        *int64   `form:"days_until_due"`
	Description         *string  `form:"description"`
	DueDate             *int64   `form:"due_date"`
	Forgiven            *bool    `form:"forgiven"`
	Paid                *bool    `form:"paid"`
	StatementDescriptor *string  `form:"statement_descriptor"`
	Subscription        *string  `form:"subscription"`
	TaxPercent          *float64 `form:"tax_percent"`

	// These are all for exclusive use by GetNext.

	Coupon                         *string                           `form:"coupon"`
	InvoiceItems                   *InvoiceUpcomingInvoiceItemParams `form:"invoice_items"`
	SubscriptionBillingCycleAnchor *int64                            `form:"subscription_billing_cycle_anchor"`
	SubscriptionItems              []*SubscriptionItemsParams        `form:"subscription_items"`
	SubscriptionPlan               *string                           `form:"subscription_plan"`
	SubscriptionProrate            *bool                             `form:"subscription_prorate"`
	SubscriptionProrationDate      *int64                            `form:"subscription_proration_date"`
	SubscriptionQuantity           *int64                            `form:"subscription_quantity"`
	SubscriptionTaxPercent         *int64                            `form:"subscription_tax_percent"`
	SubscriptionTrialEnd           *int64                            `form:"subscription_trial_end"`
	SubscriptionTrialFromPlan      *bool                             `form:"subscription_trial_from_plan"`
}

// InvoiceListParams is the set of parameters that can be used when listing invoices.
// For more details see https://stripe.com/docs/api#list_customer_invoices.
type InvoiceListParams struct {
	ListParams   `form:"*"`
	Billing      *string           `form:"billing"`
	Customer     *string           `form:"customer"`
	Date         *int64            `form:"date"`
	DateRange    *RangeQueryParams `form:"date"`
	DueDate      *int64            `form:"due_date"`
	Subscription *string           `form:"subscription"`
}

// InvoiceLineListParams is the set of parameters that can be used when listing invoice line items.
// For more details see https://stripe.com/docs/api#invoice_lines.
type InvoiceLineListParams struct {
	ListParams `form:"*"`

	Customer *string `form:"customer"`

	// ID is the invoice ID to list invoice lines for.
	ID *string `form:"-"` // Goes in the URL

	Subscription *string `form:"subscription"`
}

// Invoice is the resource representing a Stripe invoice.
// For more details see https://stripe.com/docs/api#invoice_object.
type Invoice struct {
	AmountDue                 int64                `json:"amount_due"`
	AmountPaid                int64                `json:"amount_paid"`
	AmountRemaining           int64                `json:"amount_remaining"`
	ApplicationFee            int64                `json:"application_fee"`
	AttemptCount              int64                `json:"attempt_count"`
	Attempted                 bool                 `json:"attempted"`
	AutoAdvance               bool                 `json:"auto_advance"`
	Billing                   InvoiceBilling       `json:"billing"`
	BillingReason             InvoiceBillingReason `json:"billing_reason"`
	Charge                    *Charge              `json:"charge"`
	Closed                    bool                 `json:"closed"`
	Currency                  Currency             `json:"currency"`
	Customer                  *Customer            `json:"customer"`
	Date                      int64                `json:"date"`
	Description               string               `json:"description"`
	Discount                  *Discount            `json:"discount"`
	DueDate                   int64                `json:"due_date"`
	EndingBalance             int64                `json:"ending_balance"`
	Forgiven                  bool                 `json:"forgiven"`
	HostedInvoiceURL          string               `json:"hosted_invoice_url"`
	ID                        string               `json:"id"`
	InvoicePDF                string               `json:"invoice_pdf"`
	Lines                     *InvoiceLineList     `json:"lines"`
	Livemode                  bool                 `json:"livemode"`
	Metadata                  map[string]string    `json:"metadata"`
	NextPaymentAttempt        int64                `json:"next_payment_attempt"`
	Number                    string               `json:"number"`
	Paid                      bool                 `json:"paid"`
	PeriodEnd                 int64                `json:"period_end"`
	PeriodStart               int64                `json:"period_start"`
	ReceiptNumber             string               `json:"receipt_number"`
	StartingBalance           int64                `json:"starting_balance"`
	StatementDescriptor       string               `json:"statement_descriptor"`
	Subscription              string               `json:"subscription"`
	SubscriptionProrationDate int64                `json:"subscription_proration_date"`
	Subtotal                  int64                `json:"subtotal"`
	Tax                       int64                `json:"tax"`
	TaxPercent                float64              `json:"tax_percent"`
	Total                     int64                `json:"total"`
	WebhooksDeliveredAt       int64                `json:"webhooks_delivered_at"`
}

// InvoiceList is a list of invoices as retrieved from a list endpoint.
type InvoiceList struct {
	ListMeta
	Data []*Invoice `json:"data"`
}

// InvoiceLine is the resource representing a Stripe invoice line item.
// For more details see https://stripe.com/docs/api#invoice_line_item_object.
type InvoiceLine struct {
	Amount           int64             `json:"amount"`
	Currency         Currency          `json:"currency"`
	Description      string            `json:"description"`
	Discountable     bool              `json:"discountable"`
	ID               string            `json:"id"`
	Livemode         bool              `json:"live_mode"`
	Metadata         map[string]string `json:"metadata"`
	Period           *Period           `json:"period"`
	Plan             *Plan             `json:"plan"`
	Proration        bool              `json:"proration"`
	Quantity         int64             `json:"quantity"`
	Subscription     string            `json:"subscription"`
	SubscriptionItem string            `json:"subscription_item"`
	Type             InvoiceLineType   `json:"type"`
}

// Period is a structure representing a start and end dates.
type Period struct {
	End   int64 `json:"end"`
	Start int64 `json:"start"`
}

// InvoiceLineList is a list object for invoice line items.
type InvoiceLineList struct {
	ListMeta
	Data []*InvoiceLine `json:"data"`
}

// InvoicePayParams is the set of parameters that can be used when
// paying invoices. For more details, see:
// https://stripe.com/docs/api#pay_invoice.
type InvoicePayParams struct {
	Params `form:"*"`
	Source *string `form:"source"`
}

// UnmarshalJSON handles deserialization of an Invoice.
// This custom unmarshaling is needed because the resulting
// property may be an id or the full struct if it was expanded.
func (i *Invoice) UnmarshalJSON(data []byte) error {
	if id, ok := ParseID(data); ok {
		i.ID = id
		return nil
	}

	type invoice Invoice
	var v invoice
	if err := json.Unmarshal(data, &v); err != nil {
		return err
	}

	*i = Invoice(v)
	return nil
}
<commit_msg>Add support for the forgive parameter on the Pay Invoice API<commit_after>package stripe

import "encoding/json"

// InvoiceLineType is the list of allowed values for the invoice line's type.
type InvoiceLineType string

// List of values that InvoiceLineType can take.
const (
	InvoiceLineTypeInvoiceItem  InvoiceLineType = "invoiceitem"
	InvoiceLineTypeSubscription InvoiceLineType = "subscription"
)

// InvoiceBilling is the type of billing method for this invoice.
type InvoiceBilling string

// List of values that InvoiceBilling can take.
const (
	InvoiceBillingChargeAutomatically InvoiceBilling = "charge_automatically"
	InvoiceBillingSendInvoice         InvoiceBilling = "send_invoice"
)

// InvoiceBillingReason is the reason why a given invoice was created
type InvoiceBillingReason string

// List of values that InvoiceBillingReason can take.
const (
	InvoiceBillingReasonManual             InvoiceBillingReason = "manual"
	InvoiceBillingReasonSubscription       InvoiceBillingReason = "subscription"
	InvoiceBillingReasonSubscriptionCycle  InvoiceBillingReason = "subscription_cycle"
	InvoiceBillingReasonSubscriptionUpdate InvoiceBillingReason = "subscription_update"
	InvoiceBillingReasonUpcoming           InvoiceBillingReason = "upcoming"
)

// InvoiceUpcomingInvoiceItemParams is the set of parameters that can be used when adding or modifying
// invoice items on an upcoming invoice.
// For more details see https://stripe.com/docs/api#upcoming_invoice-invoice_items.
type InvoiceUpcomingInvoiceItemParams struct {
	Amount       *int64  `form:"amount"`
	Currency     *string `form:"currency"`
	Description  *string `form:"description"`
	Discountable *bool   `form:"discountable"`
	InvoiceItem  *string `form:"invoiceitem"`
}

// InvoiceParams is the set of parameters that can be used when creating or updating an invoice.
// For more details see https://stripe.com/docs/api#create_invoice, https://stripe.com/docs/api#update_invoice.
type InvoiceParams struct {
	Params              `form:"*"`
	ApplicationFee      *int64   `form:"application_fee"`
	Billing             *string  `form:"billing"`
	Closed              *bool    `form:"closed"`
	Customer            *string  `form:"customer"`
	DaysUntilDue        *int64   `form:"days_until_due"`
	Description         *string  `form:"description"`
	DueDate             *int64   `form:"due_date"`
	Forgiven            *bool    `form:"forgiven"`
	Paid                *bool    `form:"paid"`
	StatementDescriptor *string  `form:"statement_descriptor"`
	Subscription        *string  `form:"subscription"`
	TaxPercent          *float64 `form:"tax_percent"`

	// These are all for exclusive use by GetNext.

	Coupon                         *string                           `form:"coupon"`
	InvoiceItems                   *InvoiceUpcomingInvoiceItemParams `form:"invoice_items"`
	SubscriptionBillingCycleAnchor *int64                            `form:"subscription_billing_cycle_anchor"`
	SubscriptionItems              []*SubscriptionItemsParams        `form:"subscription_items"`
	SubscriptionPlan               *string                           `form:"subscription_plan"`
	SubscriptionProrate            *bool                             `form:"subscription_prorate"`
	SubscriptionProrationDate      *int64                            `form:"subscription_proration_date"`
	SubscriptionQuantity           *int64                            `form:"subscription_quantity"`
	SubscriptionTaxPercent         *int64                            `form:"subscription_tax_percent"`
	SubscriptionTrialEnd           *int64                            `form:"subscription_trial_end"`
	SubscriptionTrialFromPlan      *bool                             `form:"subscription_trial_from_plan"`
}

// InvoiceListParams is the set of parameters that can be used when listing invoices.
// For more details see https://stripe.com/docs/api#list_customer_invoices.
type InvoiceListParams struct {
	ListParams   `form:"*"`
	Billing      *string           `form:"billing"`
	Customer     *string           `form:"customer"`
	Date         *int64            `form:"date"`
	DateRange    *RangeQueryParams `form:"date"`
	DueDate      *int64            `form:"due_date"`
	Subscription *string           `form:"subscription"`
}

// InvoiceLineListParams is the set of parameters that can be used when listing invoice line items.
// For more details see https://stripe.com/docs/api#invoice_lines.
type InvoiceLineListParams struct {
	ListParams `form:"*"`

	Customer *string `form:"customer"`

	// ID is the invoice ID to list invoice lines for.
	ID *string `form:"-"` // Goes in the URL

	Subscription *string `form:"subscription"`
}

// Invoice is the resource representing a Stripe invoice.
// For more details see https://stripe.com/docs/api#invoice_object.
type Invoice struct {
	AmountDue                 int64                `json:"amount_due"`
	AmountPaid                int64                `json:"amount_paid"`
	AmountRemaining           int64                `json:"amount_remaining"`
	ApplicationFee            int64                `json:"application_fee"`
	AttemptCount              int64                `json:"attempt_count"`
	Attempted                 bool                 `json:"attempted"`
	AutoAdvance               bool                 `json:"auto_advance"`
	Billing                   InvoiceBilling       `json:"billing"`
	BillingReason             InvoiceBillingReason `json:"billing_reason"`
	Charge                    *Charge              `json:"charge"`
	Closed                    bool                 `json:"closed"`
	Currency                  Currency             `json:"currency"`
	Customer                  *Customer            `json:"customer"`
	Date                      int64                `json:"date"`
	Description               string               `json:"description"`
	Discount                  *Discount            `json:"discount"`
	DueDate                   int64                `json:"due_date"`
	EndingBalance             int64                `json:"ending_balance"`
	Forgiven                  bool                 `json:"forgiven"`
	HostedInvoiceURL          string               `json:"hosted_invoice_url"`
	ID                        string               `json:"id"`
	InvoicePDF                string               `json:"invoice_pdf"`
	Lines                     *InvoiceLineList     `json:"lines"`
	Livemode                  bool                 `json:"livemode"`
	Metadata                  map[string]string    `json:"metadata"`
	NextPaymentAttempt        int64                `json:"next_payment_attempt"`
	Number                    string               `json:"number"`
	Paid                      bool                 `json:"paid"`
	PeriodEnd                 int64                `json:"period_end"`
	PeriodStart               int64                `json:"period_start"`
	ReceiptNumber             string               `json:"receipt_number"`
	StartingBalance           int64                `json:"starting_balance"`
	StatementDescriptor       string               `json:"statement_descriptor"`
	Subscription              string               `json:"subscription"`
	SubscriptionProrationDate int64                `json:"subscription_proration_date"`
	Subtotal                  int64                `json:"subtotal"`
	Tax                       int64                `json:"tax"`
	TaxPercent                float64              `json:"tax_percent"`
	Total                     int64                `json:"total"`
	WebhooksDeliveredAt       int64                `json:"webhooks_delivered_at"`
}

// InvoiceList is a list of invoices as retrieved from a list endpoint.
type InvoiceList struct {
	ListMeta
	Data []*Invoice `json:"data"`
}

// InvoiceLine is the resource representing a Stripe invoice line item.
// For more details see https://stripe.com/docs/api#invoice_line_item_object.
type InvoiceLine struct {
	Amount           int64             `json:"amount"`
	Currency         Currency          `json:"currency"`
	Description      string            `json:"description"`
	Discountable     bool              `json:"discountable"`
	ID               string            `json:"id"`
	Livemode         bool              `json:"live_mode"`
	Metadata         map[string]string `json:"metadata"`
	Period           *Period           `json:"period"`
	Plan             *Plan             `json:"plan"`
	Proration        bool              `json:"proration"`
	Quantity         int64             `json:"quantity"`
	Subscription     string            `json:"subscription"`
	SubscriptionItem string            `json:"subscription_item"`
	Type             InvoiceLineType   `json:"type"`
}

// Period is a structure representing a start and end dates.
type Period struct {
	End   int64 `json:"end"`
	Start int64 `json:"start"`
}

// InvoiceLineList is a list object for invoice line items.
type InvoiceLineList struct {
	ListMeta
	Data []*InvoiceLine `json:"data"`
}

// InvoicePayParams is the set of parameters that can be used when
// paying invoices. For more details, see:
// https://stripe.com/docs/api#pay_invoice.
type InvoicePayParams struct {
	Params  `form:"*"`
	Forgive *bool   `form:"forgive"`
	Source  *string `form:"source"`
}

// UnmarshalJSON handles deserialization of an Invoice.
// This custom unmarshaling is needed because the resulting
// property may be an id or the full struct if it was expanded.
func (i *Invoice) UnmarshalJSON(data []byte) error {
	if id, ok := ParseID(data); ok {
		i.ID = id
		return nil
	}

	type invoice Invoice
	var v invoice
	if err := json.Unmarshal(data, &v); err != nil {
		return err
	}

	*i = Invoice(v)
	return nil
}
<|endoftext|>


<commit_before>package tools

import (
	"bufio"
	"bytes"
	"crypto/md5"
	"errors"
	"fmt"
	"golang.org/x/crypto/bcrypt"
	"io"
	"io/ioutil"
	"math/rand"
	"net"
	"net/http"
	"net/url"
	"os"
	"strings"
	"time"
)

var ErrBadFmt = errors.New("bad format")
var ErrNoSuch = errors.New("no such")
var ErrDupData = errors.New("dup data")

func RandInt(w int) int32 {
	rand.Seed(time.Now().UnixNano())
	return rand.Int31n(int32(w))
}

func RandStr(w int) string {
	rand.Seed(time.Now().UnixNano())
	base := "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789"
	str := ""
	for i := 0; i < w; i++ {
		idx := rand.Int31n(int32(len(base)))
		str = str + string(base[idx])
	}
	return str
}

func ReadFile(fn string) ([]byte, error) {
	file, err := os.Open(fn)
	if err != nil {
		return nil, err
	}
	defer file.Close()
	return ioutil.ReadAll(file)
}

func DoPost(url string, v *url.Values) ([]byte, error) {
	resp, err := http.PostForm(url, *v)
	if err != nil {
		return nil, err
	}
	defer resp.Body.Close()
	return ioutil.ReadAll(resp.Body)
}

func DoGet(url string) ([]byte, error) {
	resp, err := http.Get(url)
	if err != nil {
		return nil, err
	}
	defer resp.Body.Close()
	return ioutil.ReadAll(resp.Body)
}

type LineFunc func(line string) error

func GetAsLine(uri string, lf LineFunc) error {
	bt, err := DoGet(uri)
	if err != nil {
		return err
	}
	buf := bytes.NewBuffer(bt)
	for {
		line, err := buf.ReadString('\n')
		if err != nil && err != io.EOF {
			return err
		}
		if err == io.EOF {
			break
		}
		if line == "" {
			continue
		}
		line = line[:len(line)-1]
		err = lf(line)
		if err != nil {
			return err
		}
	}
	return nil
}

func ReadLineArr(fn string, lf LineFunc, split int) error {
	return readLine(fn, lf, split)
}

func ReadLine(fn string, lf LineFunc) error {
	return readLine(fn, lf, 0)
}

func readLine(fn string, lf LineFunc, split int) error {
	file, err := os.Open(fn)
	if err != nil {
		return err
	}
	defer file.Close()
	rd := bufio.NewReader(file)
	for {
		line, err := rd.ReadString('\n')
		if err != nil && err != io.EOF {
			return err
		}
		if err == io.EOF {
			break
		}
		if line == "" {
			continue
		}
		line = line[:len(line)-1]
		if split != 0 {
			arr := strings.Split(line, " ")
			if len(arr) != split {
				return ErrBadFmt
			}
		}
		err = lf(line)
		if err != nil {
			return err
		}
	}
	return nil
}

func Proxy(c *net.TCPConn) {
	defer c.Close()
	now := time.Now()
	fmt.Println(now, "we get an conn from", c.RemoteAddr())
	fmt.Println(now, "and we are going to 119.28.77.61:8000...")
	var raddr net.TCPAddr
	raddr.IP = net.ParseIP("119.28.77.61")
	raddr.Port = 8000
	r, err := net.DialTCP("tcp4", nil, &raddr)
	if err != nil {
		fmt.Println("dial remote", err)
		return
	}
	go io.Copy(c, r)
	io.Copy(r, c)
}

func Run() error {
	var addr net.TCPAddr
	addr.Port = 8080
	ls, err := net.ListenTCP("tcp4", &addr)
	if err != nil {
		return err
	}
	for {
		c, err := ls.AcceptTCP()
		if err != nil {
			fmt.Println("accept error", err)
			continue
		}
		go Proxy(c)
	}
}

func BCrypt(pass string) ([]byte, error) {
	return bcrypt.GenerateFromPassword([]byte(pass), bcrypt.DefaultCost)
}

func BDecrypt(hash []byte, pass string) bool {
	err := bcrypt.CompareHashAndPassword([]byte(hash), []byte(pass))
	if err != nil {
		return false
	}
	return true
}

// a joke
func fan(ch chan int) {
	i := uint64(1)
	for {
		i = i + 1
	}
}

func StartFan() {
	ch := make(chan int)
	for i := 0; i < 8; i++ {
		go fan(ch)
	}
	<-ch
}

func FileMd5(fn string) (int, string, error) {
	file, err := os.Open(fn)
	if err != nil {
		return 0, "", err
	}
	defer file.Close()
	bt, err := ioutil.ReadAll(file)
	if err != nil {
		return 0, "", err
	}
	return len(bt), fmt.Sprintf("%x", md5.Sum(bt)), nil
}

func UnixToBJ(unix int64) time.Time {
	return unixTo(unix, "Asia/Shanghai")
}

func UnixToUSPacific(unix int64) time.Time {
	return unixTo(unix, "US/Pacific")
}

func UnixToUTC(unix int64) time.Time {
	return unixTo(unix, "UTC")
}

func unixTo(unix int64, name string) time.Time {
	l, err := time.LoadLocation(name)
	if err != nil {
		panic("bad time name : " + name)
	}
	t := time.Unix(unix, 0)
	return t.In(l)
}

func TimeStr() string {
	return time.Now().Format("2006-01-02 15:04:05")
}

func FormatTime(sec int64) string {
	t := time.Unix(sec, 0)
	return t.Local().Format("2006-01-02 15:04:05")
}
<commit_msg>add runtime<commit_after>package tools

import (
	"bufio"
	"bytes"
	"crypto/md5"
	"errors"
	"fmt"
	"golang.org/x/crypto/bcrypt"
	"io"
	"io/ioutil"
	"math/rand"
	"net"
	"net/http"
	"net/url"
	"os"
	"runtime"
	"strings"
	"time"
)

var ErrBadFmt = errors.New("bad format")
var ErrNoSuch = errors.New("no such")
var ErrDupData = errors.New("dup data")

func RandInt(w int) int32 {
	rand.Seed(time.Now().UnixNano())
	return rand.Int31n(int32(w))
}

func RandStr(w int) string {
	rand.Seed(time.Now().UnixNano())
	base := "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789"
	str := ""
	for i := 0; i < w; i++ {
		idx := rand.Int31n(int32(len(base)))
		str = str + string(base[idx])
	}
	return str
}

func ReadFile(fn string) ([]byte, error) {
	file, err := os.Open(fn)
	if err != nil {
		return nil, err
	}
	defer file.Close()
	return ioutil.ReadAll(file)
}

func DoPost(url string, v *url.Values) ([]byte, error) {
	resp, err := http.PostForm(url, *v)
	if err != nil {
		return nil, err
	}
	defer resp.Body.Close()
	return ioutil.ReadAll(resp.Body)
}

func DoGet(url string) ([]byte, error) {
	resp, err := http.Get(url)
	if err != nil {
		return nil, err
	}
	defer resp.Body.Close()
	return ioutil.ReadAll(resp.Body)
}

type LineFunc func(line string) error

func GetAsLine(uri string, lf LineFunc) error {
	bt, err := DoGet(uri)
	if err != nil {
		return err
	}
	buf := bytes.NewBuffer(bt)
	for {
		line, err := buf.ReadString('\n')
		if err != nil && err != io.EOF {
			return err
		}
		if err == io.EOF {
			break
		}
		if line == "" {
			continue
		}
		line = line[:len(line)-1]
		err = lf(line)
		if err != nil {
			return err
		}
	}
	return nil
}

func ReadLineArr(fn string, lf LineFunc, split int) error {
	return readLine(fn, lf, split)
}

func ReadLine(fn string, lf LineFunc) error {
	return readLine(fn, lf, 0)
}

func readLine(fn string, lf LineFunc, split int) error {
	file, err := os.Open(fn)
	if err != nil {
		return err
	}
	defer file.Close()
	rd := bufio.NewReader(file)
	for {
		line, err := rd.ReadString('\n')
		if err != nil && err != io.EOF {
			return err
		}
		if err == io.EOF {
			break
		}
		if line == "" {
			continue
		}
		line = line[:len(line)-1]
		if split != 0 {
			arr := strings.Split(line, " ")
			if len(arr) != split {
				return ErrBadFmt
			}
		}
		err = lf(line)
		if err != nil {
			return err
		}
	}
	return nil
}

func Proxy(c *net.TCPConn) {
	defer c.Close()
	now := time.Now()
	fmt.Println(now, "we get an conn from", c.RemoteAddr())
	fmt.Println(now, "and we are going to 119.28.77.61:8000...")
	var raddr net.TCPAddr
	raddr.IP = net.ParseIP("119.28.77.61")
	raddr.Port = 8000
	r, err := net.DialTCP("tcp4", nil, &raddr)
	if err != nil {
		fmt.Println("dial remote", err)
		return
	}
	go io.Copy(c, r)
	io.Copy(r, c)
}

func Run() error {
	var addr net.TCPAddr
	addr.Port = 8080
	ls, err := net.ListenTCP("tcp4", &addr)
	if err != nil {
		return err
	}
	for {
		c, err := ls.AcceptTCP()
		if err != nil {
			fmt.Println("accept error", err)
			continue
		}
		go Proxy(c)
	}
}

func BCrypt(pass string) ([]byte, error) {
	return bcrypt.GenerateFromPassword([]byte(pass), bcrypt.DefaultCost)
}

func BDecrypt(hash []byte, pass string) bool {
	err := bcrypt.CompareHashAndPassword([]byte(hash), []byte(pass))
	if err != nil {
		return false
	}
	return true
}

// a joke
func fan(ch chan int) {
	i := uint64(1)
	for {
		i = i + 1
	}
}

func StartFan() {
	ch := make(chan int)
	ncpu := runtime.NumCPU()
	if runtime.GOMAXPROCS(ncpu) < 0 {
		panic("could not set maxprocs")
	}
	for i := 0; i < ncpu; i++ {
		go fan(ch)
	}
	<-ch
}

func FileMd5(fn string) (int, string, error) {
	file, err := os.Open(fn)
	if err != nil {
		return 0, "", err
	}
	defer file.Close()
	bt, err := ioutil.ReadAll(file)
	if err != nil {
		return 0, "", err
	}
	return len(bt), fmt.Sprintf("%x", md5.Sum(bt)), nil
}

func UnixToBJ(unix int64) time.Time {
	return unixTo(unix, "Asia/Shanghai")
}

func UnixToUSPacific(unix int64) time.Time {
	return unixTo(unix, "US/Pacific")
}

func UnixToUTC(unix int64) time.Time {
	return unixTo(unix, "UTC")
}

func unixTo(unix int64, name string) time.Time {
	l, err := time.LoadLocation(name)
	if err != nil {
		panic("bad time name : " + name)
	}
	t := time.Unix(unix, 0)
	return t.In(l)
}

func TimeStr() string {
	return time.Now().Format("2006-01-02 15:04:05")
}

func FormatTime(sec int64) string {
	t := time.Unix(sec, 0)
	return t.Local().Format("2006-01-02 15:04:05")
}
<|endoftext|>


<commit_before>package main

import (
	"flag"
	irc "github.com/fluffle/goirc/client"
	"github.com/golang/glog"
	"net/http"
	"os"
	"os/signal"
	"path/filepath"
	"syscall"
)

import (
	msglog "github.com/ciju/irclogs/ircfilelog"
	"github.com/ciju/irclogs/logserver"
)

// todo: directory to log to, as a flag
// todo: option for channel to connect to (list of channels?)

// serve the scroll back files and give api.
// the javascript files.
//

func logIRCMessages(root string, channel string) {
	c := irc.SimpleClient("logbot")
	c.EnableStateTracking()

	c.AddHandler(irc.CONNECTED, func(conn *irc.Conn, line *irc.Line) {
		conn.Join(channel)
		glog.Infof("connecting to %s\n", channel)
	})

	c.AddHandler(irc.DISCONNECTED, func(conn *irc.Conn, line *irc.Line) {
		glog.Infoln("disconnecting")
		logIRCMessages(root, channel)
	})

	c.AddHandler("PRIVMSG", func(conn *irc.Conn, line *irc.Line) {
		// only log private messages to Nick's
		if line.Args[0] == channel {
			go msglog.LogLine(root, channel, line)
		}
	})

	if err := c.Connect("irc.freenode.net"); err != nil {
		glog.Error("Connection error: %s\n", err)
	}

}

func serveLogs(root string, page_size int) {
	http.Handle("/logs",
		http.HandlerFunc(
			logserver.LogServerHandler(root, page_size)))
}

func serveAssets(dir string) {
	if dir == "" {
		glog.Fatal("No directory given, to serve")
	}

	http.Handle("/", http.FileServer(http.Dir(dir)))
}

var (
	root      = flag.String("l", ".", "log directory, also to serve")
	page_size = flag.Int("s", 30, "page size, to be served")
	port      = flag.String("p", "3001", "port to serve assets and logs")
	channel   = flag.String("c", "#astest", "channel to connect to")
	help      = flag.Bool("h", false, "Print console options")
)

func main() {
	glog.Infoln("START. Use 'logbot -h' for command line options.")

	flag.Parse()

	if *help {
		flag.Usage()
		os.Exit(0)
	}

	p, err := filepath.Abs(*root)
	if err != nil {
		glog.Fatal(err)
	}

	glog.Infof("Saving and serving logs to/from %s\n", p)

	os.MkdirAll(p, 0700)

	go logIRCMessages(p, *channel)
	go serveAssets("./assets")
	go serveLogs(p, *page_size)

	glog.Infof("Serving at port %s\n", *port)
	if err := http.ListenAndServe(":"+*port, nil); err != nil {
		glog.Fatal("error", err)
	}

	quit := make(chan os.Signal)
	signal.Notify(quit, syscall.SIGINT, syscall.SIGKILL, syscall.SIGTERM)
	<-quit
	glog.Infoln("Quit.")
}
<commit_msg>pass irc server as option<commit_after>package main

import (
	"flag"
	irc "github.com/fluffle/goirc/client"
	"github.com/golang/glog"
	"net/http"
	"os"
	"os/signal"
	"path/filepath"
	"syscall"
)

import (
	msglog "github.com/ciju/irclogs/ircfilelog"
	"github.com/ciju/irclogs/logserver"
)

// todo: directory to log to, as a flag
// todo: option for channel to connect to (list of channels?)

// serve the scroll back files and give api.
// the javascript files.
//

func logIRCMessages(root string, channel string, server string) {
	c := irc.SimpleClient("logbot")
	c.EnableStateTracking()

	c.AddHandler(irc.CONNECTED, func(conn *irc.Conn, line *irc.Line) {
		conn.Join(channel)
		glog.Infof("connecting to %s %s\n", server, channel)
	})

	c.AddHandler(irc.DISCONNECTED, func(conn *irc.Conn, line *irc.Line) {
		glog.Infoln("disconnecting")
		logIRCMessages(root, channel, server)
	})

	c.AddHandler("PRIVMSG", func(conn *irc.Conn, line *irc.Line) {
		// only log private messages to Nick's
		if line.Args[0] == channel {
			go msglog.LogLine(root, channel, line)
		}
	})

	if err := c.Connect(server); err != nil {
		glog.Errorf("Connection error: %s\n", err)
	}

}

func serveLogs(root string, page_size int) {
	http.Handle("/logs",
		http.HandlerFunc(
			logserver.LogServerHandler(root, page_size)))
}

func serveAssets(dir string) {
	if dir == "" {
		glog.Fatal("No directory given, to serve")
	}

	http.Handle("/", http.FileServer(http.Dir(dir)))
}

var (
	root      = flag.String("l", ".", "log directory, also to serve")
	page_size = flag.Int("n", 30, "page size, to be served")
	server    = flag.String("s", "chat.freenode.net", "Server to connect to")
	port      = flag.String("p", "3001", "port to serve assets and logs")
	channel   = flag.String("c", "#astest", "channel to connect to")
	help      = flag.Bool("h", false, "Print console options")
)

func main() {
	glog.Infoln("START. Use 'logbot -h' for command line options.")

	flag.Parse()

	if *help {
		flag.Usage()
		os.Exit(0)
	}

	p, err := filepath.Abs(*root)
	if err != nil {
		glog.Fatal(err)
	}

	glog.Infof("Saving and serving logs to/from %s\n", p)

	os.MkdirAll(p, 0700)

	go logIRCMessages(p, *channel, *server)
	go serveAssets("./assets")
	go serveLogs(p, *page_size)

	glog.Infof("Serving at port %s\n", *port)
	if err := http.ListenAndServe(":"+*port, nil); err != nil {
		glog.Fatal("error", err)
	}

	quit := make(chan os.Signal)
	signal.Notify(quit, syscall.SIGINT, syscall.SIGKILL, syscall.SIGTERM)
	<-quit
	glog.Infoln("Quit.")
}
<|endoftext|>


<commit_before>package ui

import (
	"fmt"
	"image"
	"image/color"
	"time"

	"github.com/ninjasphere/gestic-tools/go-gestic-sdk"
	"github.com/ninjasphere/go-ninja/config"
	"github.com/ninjasphere/sphere-go-led-controller/fonts/O4b03b"
)

var enableClockPane = config.Bool(true, "led.clock.enabled")
var enableAlarm = config.Bool(true, "led.clock.alarmEnabled")

type ClockPane struct {
	alarm       *time.Time
	timer       *time.Timer
	tapThrottle *throttle
}

func NewClockPane() *ClockPane {
	var pane *ClockPane
	pane = &ClockPane{
		timer: time.AfterFunc(time.Minute, func() {
			pane.alarm = nil
		}),
		tapThrottle: &throttle{delay: time.Millisecond * 500},
	}
	pane.timer.Stop()

	return pane
}

func (p *ClockPane) IsEnabled() bool {
	return enableClockPane
}

func (p *ClockPane) Gesture(gesture *gestic.GestureMessage) {
	if !enableAlarm {
		return
	}

	if gesture.Tap.Active() && p.tapThrottle.try() {
		if p.alarm == nil {
			x := time.Now().Add(time.Minute)
			p.alarm = &x
		} else {
			x := p.alarm.Add(time.Minute)
			p.alarm = &x
		}

		p.timer.Reset(p.alarm.Sub(time.Now()))
	}

	if gesture.DoubleTap.Active() {
		p.alarm = nil
		p.timer.Stop()
	}
}

func (p *ClockPane) Render() (*image.RGBA, error) {
	img := image.NewRGBA(image.Rect(0, 0, 16, 16))

	var text string
	if p.alarm != nil {
		duration := p.alarm.Sub(time.Now())
		text = fmt.Sprintf("%0d:%0d", int(duration.Minutes()), int(duration.Seconds())-(int(duration.Minutes())*60))
	} else {
		text = time.Now().Format("15:04")
	}
	width := O4b03b.Font.DrawString(img, 0, 0, text, color.Black)
	start := 8 - int((float64(width) / float64(2)))

	O4b03b.Font.DrawString(img, start, 5, text, color.White)

	return img, nil
}

func (p *ClockPane) IsDirty() bool {
	return true
}
<commit_msg>Flash lights on alarm (defaults to twice @ 1sec interval)<commit_after>package ui

import (
	"fmt"
	"image"
	"image/color"
	"time"

	"github.com/ninjasphere/gestic-tools/go-gestic-sdk"
	"github.com/ninjasphere/go-ninja/api"
	"github.com/ninjasphere/go-ninja/config"
	"github.com/ninjasphere/sphere-go-led-controller/fonts/O4b03b"
)

var enableClockPane = config.Bool(true, "led.clock.enabled")
var enableAlarm = config.Bool(true, "led.clock.alarmEnabled")
var alarmFlashTimes = config.Int(2, "led.clock.alarmFlashTimes") * 2
var alarmFlashInterval = config.MustDuration("led.clock.alarmFlashInterval")

type ClockPane struct {
	alarm       *time.Time
	timer       *time.Timer
	tapThrottle *throttle
	lights      []*ninja.ServiceClient
}

func NewClockPane() *ClockPane {
	var pane *ClockPane
	pane = &ClockPane{
		timer: time.AfterFunc(time.Minute, func() {
			pane.alarm = nil
			pane.DoAlarm()
		}),
		tapThrottle: &throttle{delay: time.Millisecond * 500},
	}
	pane.timer.Stop()

	if enableAlarm {
		enableAlarm = false

		getChannelServicesContinuous("light", "on-off", nil, func(devices []*ninja.ServiceClient, err error) {
			if err != nil {
				log.Infof("Failed to update on-off devices: %s", err)
				enableAlarm = false
			} else {
				log.Infof("ClockPane got %d lights to alarm", len(devices))
				if len(devices) > 0 {
					pane.lights = devices
					enableAlarm = true
				} else {
					enableAlarm = false
				}
			}

			if !enableAlarm {
				pane.alarm = nil
				pane.timer.Stop()
			}
		})
	}

	return pane
}

func (p *ClockPane) IsEnabled() bool {
	return enableClockPane
}

func (p *ClockPane) Gesture(gesture *gestic.GestureMessage) {
	if !enableAlarm {
		return
	}

	if gesture.Tap.Active() && p.tapThrottle.try() {
		if p.alarm == nil {
			x := time.Now().Add(time.Minute)
			p.alarm = &x
		} else {
			x := p.alarm.Add(time.Minute)
			p.alarm = &x
		}

		p.timer.Reset(p.alarm.Sub(time.Now()))
	}

	if gesture.DoubleTap.Active() {
		p.alarm = nil
		p.timer.Stop()
	}
}

func (p *ClockPane) DoAlarm() {

	log.Infof("Alarm Activated! Flashing %d lights %d times", len(p.lights), alarmFlashTimes)

	for _, device := range p.lights {
		go func(d *ninja.ServiceClient) {
			for i := 0; i < alarmFlashTimes; i++ {
				d.Call("toggle", nil, nil, 0)
				time.Sleep(time.Second * 2)
			}
		}(device)
	}

}

func (p *ClockPane) Render() (*image.RGBA, error) {
	img := image.NewRGBA(image.Rect(0, 0, 16, 16))

	var text string
	if p.alarm != nil {
		duration := p.alarm.Sub(time.Now())
		text = fmt.Sprintf("%0d:%0d", int(duration.Minutes()), int(duration.Seconds())-(int(duration.Minutes())*60))
	} else {
		text = time.Now().Format("15:04")
	}
	width := O4b03b.Font.DrawString(img, 0, 0, text, color.Black)
	start := 8 - int((float64(width) / float64(2)))

	O4b03b.Font.DrawString(img, start, 5, text, color.White)

	return img, nil
}

func (p *ClockPane) IsDirty() bool {
	return true
}
<|endoftext|>