<commit_before>package deletion

import (
	"context"
	"fmt"
	"sort"
	"sync"
	"time"

	"github.com/grafana/loki/pkg/storage/stores/indexshipper/compactor/deletionmode"

	"github.com/go-kit/log/level"
	"github.com/prometheus/client_golang/prometheus"
	"github.com/prometheus/common/model"

	"github.com/grafana/loki/pkg/storage/stores/indexshipper/compactor/retention"
	util_log "github.com/grafana/loki/pkg/util/log"
)

const (
	statusSuccess = "success"
	statusFail    = "fail"
)

type userDeleteRequests struct {
	requests []DeleteRequest
	// requestsInterval holds the earliest start time and latest end time considering all the delete requests
	requestsInterval model.Interval
}

type DeleteRequestsManager struct {
	deleteRequestsStore       DeleteRequestsStore
	deleteRequestCancelPeriod time.Duration

	deleteRequestsToProcess map[string]*userDeleteRequests
	chunkIntervalsToRetain  []retention.IntervalFilter
	// WARN: If by any chance we change deleteRequestsToProcessMtx to sync.RWMutex to be able to check multiple chunks at a time,
	// please take care of chunkIntervalsToRetain which should be unique per chunk.
	deleteRequestsToProcessMtx sync.Mutex
	metrics                    *deleteRequestsManagerMetrics
	wg                         sync.WaitGroup
	done                       chan struct{}
	batchSize                  int
	limits                     Limits
}

func NewDeleteRequestsManager(store DeleteRequestsStore, deleteRequestCancelPeriod time.Duration, batchSize int, limits Limits, registerer prometheus.Registerer) *DeleteRequestsManager {
	dm := &DeleteRequestsManager{
		deleteRequestsStore:       store,
		deleteRequestCancelPeriod: deleteRequestCancelPeriod,
		deleteRequestsToProcess:   map[string]*userDeleteRequests{},
		metrics:                   newDeleteRequestsManagerMetrics(registerer),
		done:                      make(chan struct{}),
		batchSize:                 batchSize,
		limits:                    limits,
	}

	go dm.loop()

	return dm
}

func (d *DeleteRequestsManager) loop() {
	ticker := time.NewTicker(5 * time.Minute)
	defer ticker.Stop()

	d.wg.Add(1)
	defer d.wg.Done()

	for {
		select {
		case <-ticker.C:
			if err := d.updateMetrics(); err != nil {
				level.Error(util_log.Logger).Log("msg", "failed to update metrics", "err", err)
			}
		case <-d.done:
			return
		}
	}
}

func (d *DeleteRequestsManager) Stop() {
	close(d.done)
	d.wg.Wait()
}

func (d *DeleteRequestsManager) updateMetrics() error {
	deleteRequests, err := d.deleteRequestsStore.GetDeleteRequestsByStatus(context.Background(), StatusReceived)
	if err != nil {
		return err
	}

	pendingDeleteRequestsCount := 0
	oldestPendingRequestCreatedAt := model.Time(0)

	for _, deleteRequest := range deleteRequests {
		// adding an extra minute here to avoid a race between cancellation of request and picking up the request for processing
		if deleteRequest.Status != StatusReceived || deleteRequest.CreatedAt.Add(d.deleteRequestCancelPeriod).Add(time.Minute).After(model.Now()) {
			continue
		}

		pendingDeleteRequestsCount++
		if oldestPendingRequestCreatedAt == 0 || deleteRequest.CreatedAt.Before(oldestPendingRequestCreatedAt) {
			oldestPendingRequestCreatedAt = deleteRequest.CreatedAt
		}
	}

	// track age of oldest delete request since they became eligible for processing
	oldestPendingRequestAge := time.Duration(0)
	if oldestPendingRequestCreatedAt != 0 {
		oldestPendingRequestAge = model.Now().Sub(oldestPendingRequestCreatedAt.Add(d.deleteRequestCancelPeriod))
	}
	d.metrics.oldestPendingDeleteRequestAgeSeconds.Set(float64(oldestPendingRequestAge / time.Second))
	d.metrics.pendingDeleteRequestsCount.Set(float64(pendingDeleteRequestsCount))

	return nil
}

func (d *DeleteRequestsManager) loadDeleteRequestsToProcess() error {
	d.deleteRequestsToProcessMtx.Lock()
	defer d.deleteRequestsToProcessMtx.Unlock()

	// Reset this first so any errors result in a clear map
	d.deleteRequestsToProcess = map[string]*userDeleteRequests{}

	deleteRequests, err := d.filteredSortedDeleteRequests()
	if err != nil {
		return err
	}

	for i, deleteRequest := range deleteRequests {
		if i >= d.batchSize {
			logBatchTruncation(i, len(deleteRequests))
			break
		}

		level.Info(util_log.Logger).Log(
			"msg", "Started processing delete request for user",
			"delete_request_id", deleteRequest.RequestID,
			"user", deleteRequest.UserID,
		)

		deleteRequest.Metrics = d.metrics

		ur := d.requestsForUser(deleteRequest)
		ur.requests = append(ur.requests, deleteRequest)
		if deleteRequest.StartTime < ur.requestsInterval.Start {
			ur.requestsInterval.Start = deleteRequest.StartTime
		}
		if deleteRequest.EndTime > ur.requestsInterval.End {
			ur.requestsInterval.End = deleteRequest.EndTime
		}
	}

	return nil
}

func (d *DeleteRequestsManager) filteredSortedDeleteRequests() ([]DeleteRequest, error) {
	deleteRequests, err := d.deleteRequestsStore.GetDeleteRequestsByStatus(context.Background(), StatusReceived)
	if err != nil {
		return nil, err
	}

	deleteRequests, err = d.filteredRequests(deleteRequests)
	if err != nil {
		return nil, err
	}

	sort.Slice(deleteRequests, func(i, j int) bool {
		return deleteRequests[i].StartTime < deleteRequests[j].StartTime
	})

	return deleteRequests, nil
}

func (d *DeleteRequestsManager) filteredRequests(reqs []DeleteRequest) ([]DeleteRequest, error) {
	filtered := make([]DeleteRequest, 0, len(reqs))
	for _, deleteRequest := range reqs {
		// adding an extra minute here to avoid a race between cancellation of request and picking up the request for processing
		if deleteRequest.CreatedAt.Add(d.deleteRequestCancelPeriod).Add(time.Minute).After(model.Now()) {
			continue
		}

		processRequest, err := d.shouldProcessRequest(deleteRequest)
		if err != nil {
			return nil, err
		}

		if !processRequest {
			continue
		}

		filtered = append(filtered, deleteRequest)
	}

	return filtered, nil
}

func (d *DeleteRequestsManager) requestsForUser(dr DeleteRequest) *userDeleteRequests {
	ur, ok := d.deleteRequestsToProcess[dr.UserID]
	if !ok {
		ur = &userDeleteRequests{
			requestsInterval: model.Interval{
				Start: dr.StartTime,
				End:   dr.EndTime,
			},
		}
		d.deleteRequestsToProcess[dr.UserID] = ur
	}
	return ur
}

func logBatchTruncation(size, total int) {
	if size < total {
		level.Info(util_log.Logger).Log(
			"msg", fmt.Sprintf("Processing %d of %d delete requests. More requests will be processed in subsequent compactions", size, total),
		)
	}
}

func (d *DeleteRequestsManager) shouldProcessRequest(dr DeleteRequest) (bool, error) {
	mode, err := deleteModeFromLimits(d.limits, dr.UserID)
	if err != nil {
		level.Error(util_log.Logger).Log(
			"msg", "unable to determine deletion mode for user",
			"user", dr.UserID,
		)
		return false, err
	}

	return mode == deletionmode.FilterAndDelete, nil
}

func (d *DeleteRequestsManager) Expired(ref retention.ChunkEntry, _ model.Time) (bool, []retention.IntervalFilter) {
	d.deleteRequestsToProcessMtx.Lock()
	defer d.deleteRequestsToProcessMtx.Unlock()

	userIDStr := unsafeGetString(ref.UserID)
	if d.deleteRequestsToProcess[userIDStr] == nil || !intervalsOverlap(d.deleteRequestsToProcess[userIDStr].requestsInterval, model.Interval{
		Start: ref.From,
		End:   ref.Through,
	}) {
		return false, nil
	}

	d.chunkIntervalsToRetain = d.chunkIntervalsToRetain[:0]
	d.chunkIntervalsToRetain = append(d.chunkIntervalsToRetain, retention.IntervalFilter{
		Interval: model.Interval{
			Start: ref.From,
			End:   ref.Through,
		},
	})

	for _, deleteRequest := range d.deleteRequestsToProcess[userIDStr].requests {
		rebuiltIntervals := make([]retention.IntervalFilter, 0, len(d.chunkIntervalsToRetain))
		for _, ivf := range d.chunkIntervalsToRetain {
			entry := ref
			entry.From = ivf.Interval.Start
			entry.Through = ivf.Interval.End
			isDeleted, newIntervalsToRetain := deleteRequest.IsDeleted(entry)
			if !isDeleted {
				rebuiltIntervals = append(rebuiltIntervals, ivf)
			} else {
				rebuiltIntervals = append(rebuiltIntervals, newIntervalsToRetain...)
			}
		}

		d.chunkIntervalsToRetain = rebuiltIntervals
		if len(d.chunkIntervalsToRetain) == 0 {
			level.Info(util_log.Logger).Log(
				"msg", "no chunks to retain: the whole chunk is deleted",
				"delete_request_id", deleteRequest.RequestID,
				"user", deleteRequest.UserID,
				"chunkID", string(ref.ChunkID),
			)
			d.metrics.deleteRequestsChunksSelectedTotal.WithLabelValues(string(ref.UserID)).Inc()
			return true, nil
		}
	}

	if len(d.chunkIntervalsToRetain) == 1 && d.chunkIntervalsToRetain[0].Interval.Start == ref.From && d.chunkIntervalsToRetain[0].Interval.End == ref.Through {
		return false, nil
	}

	d.metrics.deleteRequestsChunksSelectedTotal.WithLabelValues(string(ref.UserID)).Inc()
	return true, d.chunkIntervalsToRetain
}

func (d *DeleteRequestsManager) MarkPhaseStarted() {
	status := statusSuccess
	if err := d.loadDeleteRequestsToProcess(); err != nil {
		status = statusFail
		level.Error(util_log.Logger).Log("msg", "failed to load delete requests to process", "err", err)
	}
	d.metrics.loadPendingRequestsAttemptsTotal.WithLabelValues(status).Inc()
}

func (d *DeleteRequestsManager) MarkPhaseFailed() {
	d.deleteRequestsToProcessMtx.Lock()
	defer d.deleteRequestsToProcessMtx.Unlock()

	d.metrics.deletionFailures.WithLabelValues("error").Inc()
	d.deleteRequestsToProcess = map[string]*userDeleteRequests{}
}

func (d *DeleteRequestsManager) MarkPhaseTimedOut() {
	d.deleteRequestsToProcessMtx.Lock()
	defer d.deleteRequestsToProcessMtx.Unlock()

	d.metrics.deletionFailures.WithLabelValues("timeout").Inc()
	d.deleteRequestsToProcess = map[string]*userDeleteRequests{}
}

func (d *DeleteRequestsManager) MarkPhaseFinished() {
	d.deleteRequestsToProcessMtx.Lock()
	defer d.deleteRequestsToProcessMtx.Unlock()

	for _, userDeleteRequests := range d.deleteRequestsToProcess {
		if userDeleteRequests == nil {
			continue
		}

		for _, deleteRequest := range userDeleteRequests.requests {
			if err := d.deleteRequestsStore.UpdateStatus(context.Background(), deleteRequest, StatusProcessed); err != nil {
				level.Error(util_log.Logger).Log(
					"msg", "failed to mark delete request for user as processed",
					"delete_request_id", deleteRequest.RequestID,
					"user", deleteRequest.UserID,
					"err", err,
					"deleted_lines", deleteRequest.DeletedLines,
				)
			} else {
				level.Info(util_log.Logger).Log(
					"msg", "delete request for user marked as processed",
					"delete_request_id", deleteRequest.RequestID,
					"user", deleteRequest.UserID,
					"deleted_lines", deleteRequest.DeletedLines,
				)
			}
			d.metrics.deleteRequestsProcessedTotal.WithLabelValues(deleteRequest.UserID).Inc()
		}
	}
}

func (d *DeleteRequestsManager) IntervalMayHaveExpiredChunks(_ model.Interval, userID string) bool {
	d.deleteRequestsToProcessMtx.Lock()
	defer d.deleteRequestsToProcessMtx.Unlock()

	// We can't do the overlap check between the passed interval and delete requests interval from a user because
	// if a request is issued just for today and there are chunks spanning today and yesterday then
	// the overlap check would skip processing yesterday's index which would result in the index pointing to deleted chunks.
	if userID != "" {
		return d.deleteRequestsToProcess[userID] != nil
	}

	return len(d.deleteRequestsToProcess) != 0
}

func (d *DeleteRequestsManager) DropFromIndex(_ retention.ChunkEntry, _ model.Time, _ model.Time) bool {
	return false
}
<commit_msg>Add sequence num to log output for delete requests (#7087)<commit_after>package deletion

import (
	"context"
	"fmt"
	"sort"
	"sync"
	"time"

	"github.com/grafana/loki/pkg/storage/stores/indexshipper/compactor/deletionmode"

	"github.com/go-kit/log/level"
	"github.com/prometheus/client_golang/prometheus"
	"github.com/prometheus/common/model"

	"github.com/grafana/loki/pkg/storage/stores/indexshipper/compactor/retention"
	util_log "github.com/grafana/loki/pkg/util/log"
)

const (
	statusSuccess = "success"
	statusFail    = "fail"
)

type userDeleteRequests struct {
	requests []DeleteRequest
	// requestsInterval holds the earliest start time and latest end time considering all the delete requests
	requestsInterval model.Interval
}

type DeleteRequestsManager struct {
	deleteRequestsStore       DeleteRequestsStore
	deleteRequestCancelPeriod time.Duration

	deleteRequestsToProcess map[string]*userDeleteRequests
	chunkIntervalsToRetain  []retention.IntervalFilter
	// WARN: If by any chance we change deleteRequestsToProcessMtx to sync.RWMutex to be able to check multiple chunks at a time,
	// please take care of chunkIntervalsToRetain which should be unique per chunk.
	deleteRequestsToProcessMtx sync.Mutex
	metrics                    *deleteRequestsManagerMetrics
	wg                         sync.WaitGroup
	done                       chan struct{}
	batchSize                  int
	limits                     Limits
}

func NewDeleteRequestsManager(store DeleteRequestsStore, deleteRequestCancelPeriod time.Duration, batchSize int, limits Limits, registerer prometheus.Registerer) *DeleteRequestsManager {
	dm := &DeleteRequestsManager{
		deleteRequestsStore:       store,
		deleteRequestCancelPeriod: deleteRequestCancelPeriod,
		deleteRequestsToProcess:   map[string]*userDeleteRequests{},
		metrics:                   newDeleteRequestsManagerMetrics(registerer),
		done:                      make(chan struct{}),
		batchSize:                 batchSize,
		limits:                    limits,
	}

	go dm.loop()

	return dm
}

func (d *DeleteRequestsManager) loop() {
	ticker := time.NewTicker(5 * time.Minute)
	defer ticker.Stop()

	d.wg.Add(1)
	defer d.wg.Done()

	for {
		select {
		case <-ticker.C:
			if err := d.updateMetrics(); err != nil {
				level.Error(util_log.Logger).Log("msg", "failed to update metrics", "err", err)
			}
		case <-d.done:
			return
		}
	}
}

func (d *DeleteRequestsManager) Stop() {
	close(d.done)
	d.wg.Wait()
}

func (d *DeleteRequestsManager) updateMetrics() error {
	deleteRequests, err := d.deleteRequestsStore.GetDeleteRequestsByStatus(context.Background(), StatusReceived)
	if err != nil {
		return err
	}

	pendingDeleteRequestsCount := 0
	oldestPendingRequestCreatedAt := model.Time(0)

	for _, deleteRequest := range deleteRequests {
		// adding an extra minute here to avoid a race between cancellation of request and picking up the request for processing
		if deleteRequest.Status != StatusReceived || deleteRequest.CreatedAt.Add(d.deleteRequestCancelPeriod).Add(time.Minute).After(model.Now()) {
			continue
		}

		pendingDeleteRequestsCount++
		if oldestPendingRequestCreatedAt == 0 || deleteRequest.CreatedAt.Before(oldestPendingRequestCreatedAt) {
			oldestPendingRequestCreatedAt = deleteRequest.CreatedAt
		}
	}

	// track age of oldest delete request since they became eligible for processing
	oldestPendingRequestAge := time.Duration(0)
	if oldestPendingRequestCreatedAt != 0 {
		oldestPendingRequestAge = model.Now().Sub(oldestPendingRequestCreatedAt.Add(d.deleteRequestCancelPeriod))
	}
	d.metrics.oldestPendingDeleteRequestAgeSeconds.Set(float64(oldestPendingRequestAge / time.Second))
	d.metrics.pendingDeleteRequestsCount.Set(float64(pendingDeleteRequestsCount))

	return nil
}

func (d *DeleteRequestsManager) loadDeleteRequestsToProcess() error {
	d.deleteRequestsToProcessMtx.Lock()
	defer d.deleteRequestsToProcessMtx.Unlock()

	// Reset this first so any errors result in a clear map
	d.deleteRequestsToProcess = map[string]*userDeleteRequests{}

	deleteRequests, err := d.filteredSortedDeleteRequests()
	if err != nil {
		return err
	}

	for i, deleteRequest := range deleteRequests {
		if i >= d.batchSize {
			logBatchTruncation(i, len(deleteRequests))
			break
		}

		level.Info(util_log.Logger).Log(
			"msg", "Started processing delete request for user",
			"delete_request_id", deleteRequest.RequestID,
			"user", deleteRequest.UserID,
		)

		deleteRequest.Metrics = d.metrics

		ur := d.requestsForUser(deleteRequest)
		ur.requests = append(ur.requests, deleteRequest)
		if deleteRequest.StartTime < ur.requestsInterval.Start {
			ur.requestsInterval.Start = deleteRequest.StartTime
		}
		if deleteRequest.EndTime > ur.requestsInterval.End {
			ur.requestsInterval.End = deleteRequest.EndTime
		}
	}

	return nil
}

func (d *DeleteRequestsManager) filteredSortedDeleteRequests() ([]DeleteRequest, error) {
	deleteRequests, err := d.deleteRequestsStore.GetDeleteRequestsByStatus(context.Background(), StatusReceived)
	if err != nil {
		return nil, err
	}

	deleteRequests, err = d.filteredRequests(deleteRequests)
	if err != nil {
		return nil, err
	}

	sort.Slice(deleteRequests, func(i, j int) bool {
		return deleteRequests[i].StartTime < deleteRequests[j].StartTime
	})

	return deleteRequests, nil
}

func (d *DeleteRequestsManager) filteredRequests(reqs []DeleteRequest) ([]DeleteRequest, error) {
	filtered := make([]DeleteRequest, 0, len(reqs))
	for _, deleteRequest := range reqs {
		// adding an extra minute here to avoid a race between cancellation of request and picking up the request for processing
		if deleteRequest.CreatedAt.Add(d.deleteRequestCancelPeriod).Add(time.Minute).After(model.Now()) {
			continue
		}

		processRequest, err := d.shouldProcessRequest(deleteRequest)
		if err != nil {
			return nil, err
		}

		if !processRequest {
			continue
		}

		filtered = append(filtered, deleteRequest)
	}

	return filtered, nil
}

func (d *DeleteRequestsManager) requestsForUser(dr DeleteRequest) *userDeleteRequests {
	ur, ok := d.deleteRequestsToProcess[dr.UserID]
	if !ok {
		ur = &userDeleteRequests{
			requestsInterval: model.Interval{
				Start: dr.StartTime,
				End:   dr.EndTime,
			},
		}
		d.deleteRequestsToProcess[dr.UserID] = ur
	}
	return ur
}

func logBatchTruncation(size, total int) {
	if size < total {
		level.Info(util_log.Logger).Log(
			"msg", fmt.Sprintf("Processing %d of %d delete requests. More requests will be processed in subsequent compactions", size, total),
		)
	}
}

func (d *DeleteRequestsManager) shouldProcessRequest(dr DeleteRequest) (bool, error) {
	mode, err := deleteModeFromLimits(d.limits, dr.UserID)
	if err != nil {
		level.Error(util_log.Logger).Log(
			"msg", "unable to determine deletion mode for user",
			"user", dr.UserID,
		)
		return false, err
	}

	return mode == deletionmode.FilterAndDelete, nil
}

func (d *DeleteRequestsManager) Expired(ref retention.ChunkEntry, _ model.Time) (bool, []retention.IntervalFilter) {
	d.deleteRequestsToProcessMtx.Lock()
	defer d.deleteRequestsToProcessMtx.Unlock()

	userIDStr := unsafeGetString(ref.UserID)
	if d.deleteRequestsToProcess[userIDStr] == nil || !intervalsOverlap(d.deleteRequestsToProcess[userIDStr].requestsInterval, model.Interval{
		Start: ref.From,
		End:   ref.Through,
	}) {
		return false, nil
	}

	d.chunkIntervalsToRetain = d.chunkIntervalsToRetain[:0]
	d.chunkIntervalsToRetain = append(d.chunkIntervalsToRetain, retention.IntervalFilter{
		Interval: model.Interval{
			Start: ref.From,
			End:   ref.Through,
		},
	})

	for _, deleteRequest := range d.deleteRequestsToProcess[userIDStr].requests {
		rebuiltIntervals := make([]retention.IntervalFilter, 0, len(d.chunkIntervalsToRetain))
		for _, ivf := range d.chunkIntervalsToRetain {
			entry := ref
			entry.From = ivf.Interval.Start
			entry.Through = ivf.Interval.End
			isDeleted, newIntervalsToRetain := deleteRequest.IsDeleted(entry)
			if !isDeleted {
				rebuiltIntervals = append(rebuiltIntervals, ivf)
			} else {
				rebuiltIntervals = append(rebuiltIntervals, newIntervalsToRetain...)
			}
		}

		d.chunkIntervalsToRetain = rebuiltIntervals
		if len(d.chunkIntervalsToRetain) == 0 {
			level.Info(util_log.Logger).Log(
				"msg", "no chunks to retain: the whole chunk is deleted",
				"delete_request_id", deleteRequest.RequestID,
				"sequence_num", deleteRequest.SequenceNum,
				"user", deleteRequest.UserID,
				"chunkID", string(ref.ChunkID),
			)
			d.metrics.deleteRequestsChunksSelectedTotal.WithLabelValues(string(ref.UserID)).Inc()
			return true, nil
		}
	}

	if len(d.chunkIntervalsToRetain) == 1 && d.chunkIntervalsToRetain[0].Interval.Start == ref.From && d.chunkIntervalsToRetain[0].Interval.End == ref.Through {
		return false, nil
	}

	d.metrics.deleteRequestsChunksSelectedTotal.WithLabelValues(string(ref.UserID)).Inc()
	return true, d.chunkIntervalsToRetain
}

func (d *DeleteRequestsManager) MarkPhaseStarted() {
	status := statusSuccess
	if err := d.loadDeleteRequestsToProcess(); err != nil {
		status = statusFail
		level.Error(util_log.Logger).Log("msg", "failed to load delete requests to process", "err", err)
	}
	d.metrics.loadPendingRequestsAttemptsTotal.WithLabelValues(status).Inc()
}

func (d *DeleteRequestsManager) MarkPhaseFailed() {
	d.deleteRequestsToProcessMtx.Lock()
	defer d.deleteRequestsToProcessMtx.Unlock()

	d.metrics.deletionFailures.WithLabelValues("error").Inc()
	d.deleteRequestsToProcess = map[string]*userDeleteRequests{}
}

func (d *DeleteRequestsManager) MarkPhaseTimedOut() {
	d.deleteRequestsToProcessMtx.Lock()
	defer d.deleteRequestsToProcessMtx.Unlock()

	d.metrics.deletionFailures.WithLabelValues("timeout").Inc()
	d.deleteRequestsToProcess = map[string]*userDeleteRequests{}
}

func (d *DeleteRequestsManager) MarkPhaseFinished() {
	d.deleteRequestsToProcessMtx.Lock()
	defer d.deleteRequestsToProcessMtx.Unlock()

	for _, userDeleteRequests := range d.deleteRequestsToProcess {
		if userDeleteRequests == nil {
			continue
		}

		for _, deleteRequest := range userDeleteRequests.requests {
			if err := d.deleteRequestsStore.UpdateStatus(context.Background(), deleteRequest, StatusProcessed); err != nil {
				level.Error(util_log.Logger).Log(
					"msg", "failed to mark delete request for user as processed",
					"delete_request_id", deleteRequest.RequestID,
					"sequence_num", deleteRequest.SequenceNum,
					"user", deleteRequest.UserID,
					"err", err,
					"deleted_lines", deleteRequest.DeletedLines,
				)
			} else {
				level.Info(util_log.Logger).Log(
					"msg", "delete request for user marked as processed",
					"delete_request_id", deleteRequest.RequestID,
					"sequence_num", deleteRequest.SequenceNum,
					"user", deleteRequest.UserID,
					"deleted_lines", deleteRequest.DeletedLines,
				)
			}
			d.metrics.deleteRequestsProcessedTotal.WithLabelValues(deleteRequest.UserID).Inc()
		}
	}
}

func (d *DeleteRequestsManager) IntervalMayHaveExpiredChunks(_ model.Interval, userID string) bool {
	d.deleteRequestsToProcessMtx.Lock()
	defer d.deleteRequestsToProcessMtx.Unlock()

	// We can't do the overlap check between the passed interval and delete requests interval from a user because
	// if a request is issued just for today and there are chunks spanning today and yesterday then
	// the overlap check would skip processing yesterday's index which would result in the index pointing to deleted chunks.
	if userID != "" {
		return d.deleteRequestsToProcess[userID] != nil
	}

	return len(d.deleteRequestsToProcess) != 0
}

func (d *DeleteRequestsManager) DropFromIndex(_ retention.ChunkEntry, _ model.Time, _ model.Time) bool {
	return false
}
<|endoftext|>


<commit_before>package ufop

import (
	"errors"
	"fmt"
	"io"
	"io/ioutil"
	"log"
	"net/http"
	"os"
	"os/exec"
	"path/filepath"
	"regexp"
	"strconv"
	"strings"
	"time"
)

const (
	HTML2IMAGE_MAX_PAGE_SIZE = 10 * 1024 * 1024
)

type Html2Imager struct {
	maxPageSize int64
}

type Html2ImageOptions struct {
	CropH   int
	CropW   int
	CropX   int
	CropY   int
	Format  string
	Height  int
	Width   int
	Quality int
	Force   bool
}

func (this *Html2Imager) parse(cmd string) (options *Html2ImageOptions, err error) {
	pattern := `^html2image(/croph/\d+|/cropw/\d+|/cropx/\d+|/cropy/\d+|/format/(png|jpg|jpeg)|/height/\d+|/quality/\d+|/width/\d+|/force/[0|1]){0,9}$`
	matched, _ := regexp.Match(pattern, []byte(cmd))
	if !matched {
		err = errors.New("invalid html2image command format")
		return
	}

	options = &Html2ImageOptions{
		Format: "jpg",
	}

	//croph
	cropHStr := getParam(cmd, `croph/\d+`, "croph")
	if cropHStr != "" {
		cropH, _ := strconv.Atoi(cropHStr)
		if cropH <= 0 {
			err = errors.New("invalid html2image parameter 'croph'")
			return
		} else {
			options.CropH = cropH
		}
	}

	//cropw
	cropWStr := getParam(cmd, `cropw/\d+`, "cropw")
	if cropWStr != "" {
		cropW, _ := strconv.Atoi(cropWStr)
		if cropW <= 0 {
			err = errors.New("invalid html2image parameter 'cropw'")
			return
		} else {
			options.CropW = cropW
		}
	}

	//cropx
	cropXStr := getParam(cmd, `cropx/\d+`, "cropx")
	fmt.Println(cropXStr)
	if cropXStr != "" {
		cropX, _ := strconv.Atoi(cropXStr)
		if cropX <= 0 {
			err = errors.New("invalid html2image parameter 'cropx'")
			return
		} else {
			options.CropX = cropX
		}
	}

	//cropy
	cropYStr := getParam(cmd, `cropy/\d+`, "cropy")
	if cropYStr != "" {
		cropY, _ := strconv.Atoi(cropYStr)
		if cropY <= 0 {
			err = errors.New("invalid html2image parameter 'cropy'")
			return
		} else {
			options.CropY = cropY
		}
	}

	//format
	formatStr := getParam(cmd, "format/(png|jpg|jpeg)", "format")
	if formatStr != "" {
		options.Format = formatStr
	}

	//height
	heightStr := getParam(cmd, `height/\d+`, "height")
	if heightStr != "" {
		height, _ := strconv.Atoi(heightStr)
		if height <= 0 {
			err = errors.New("invalid html2image parameter 'height'")
			return
		} else {
			options.Height = height
		}
	}

	//width
	widthStr := getParam(cmd, `width/\d+`, "width")
	if widthStr != "" {
		width, _ := strconv.Atoi(widthStr)
		if width <= 0 {
			err = errors.New("invalid html2image parameter 'width'")
			return
		} else {
			options.Width = width
		}
	}

	//quality
	qualityStr := getParam(cmd, `quality/\d+`, "quality")
	if qualityStr != "" {
		quality, _ := strconv.Atoi(qualityStr)
		if quality > 100 || quality <= 0 {
			err = errors.New("invalid html2image parameter 'quality'")
			return
		} else {
			options.Quality = quality
		}
	}

	//force
	forceStr := getParam(cmd, "force/[0|1]", "force")
	if forceStr != "" {
		force, _ := strconv.Atoi(forceStr)
		if force == 1 {
			options.Force = true
		}
	}

	return

}

func (this *Html2Imager) Do(req UfopRequest) (result interface{}, contentType string, err error) {
	if this.maxPageSize <= 0 {
		this.maxPageSize = HTML2IMAGE_MAX_PAGE_SIZE
	}

	//if not text format, error it
	if !strings.HasPrefix(req.Src.MimeType, "text/") {
		err = errors.New("unsupported file mime type, only text/* allowed")
		return
	}

	//if file size exceeds, error it
	if req.Src.Fsize > this.maxPageSize {
		err = errors.New("page file length exceeds the limit")
		return
	}

	options, pErr := this.parse(req.Cmd)
	if pErr != nil {
		err = pErr
		return
	}

	//get page file content save it into temp dir
	resp, respErr := http.Get(req.Src.Url)
	if respErr != nil || resp.StatusCode != 200 {
		if respErr != nil {
			err = errors.New(fmt.Sprintf("retrieve page file resource data failed, %s", respErr.Error()))
		} else {
			err = errors.New(fmt.Sprintf("retrieve page file resource data failed, %s", resp.Status))
			if resp.Body != nil {
				resp.Body.Close()
			}
		}
		return
	}

	jobPrefix := md5Hex(req.Src.Url)

	pageSuffix := "txt"
	if req.Src.MimeType == "text/html" {
		pageSuffix = "html"
	}

	localPageTmpFname := fmt.Sprintf("%s%d.page.%s", jobPrefix, time.Now().UnixNano(), pageSuffix)
	localPageTmpFpath := filepath.Join(os.TempDir(), localPageTmpFname)
	defer os.Remove(localPageTmpFpath)

	localPageTmpFp, openErr := os.OpenFile(localPageTmpFpath, os.O_CREATE|os.O_WRONLY|os.O_TRUNC, 0655)
	if openErr != nil {
		err = errors.New(fmt.Sprintf("open page file temp file failed, %s", openErr.Error()))
		return
	}
	_, cpErr := io.Copy(localPageTmpFp, resp.Body)
	if cpErr != nil {
		err = errors.New(fmt.Sprintf("save page file content to tmp file failed, %s", cpErr.Error()))
		return
	}

	localPageTmpFp.Close()
	resp.Body.Close()

	//prepare command
	cmdParams := make([]string, 0)
	cmdParams = append(cmdParams, "-q")

	if options.CropH > 0 {
		cmdParams = append(cmdParams, "--crop-h", fmt.Sprintf("%d", options.CropH))
	}

	if options.CropW > 0 {
		cmdParams = append(cmdParams, "--crop-w", fmt.Sprintf("%s", options.CropW))
	}

	if options.CropX > 0 {
		cmdParams = append(cmdParams, "--crop-x", fmt.Sprintf("%d", options.CropX))
	}

	if options.CropY > 0 {
		cmdParams = append(cmdParams, "--crop-y", fmt.Sprintf("%d", options.CropY))
	}

	if options.Format != "" {
		cmdParams = append(cmdParams, "--format", options.Format)
	}

	if options.Quality > 0 {
		cmdParams = append(cmdParams, "--quality", fmt.Sprintf("%d", options.Quality))
	}

	if options.Height > 0 {
		cmdParams = append(cmdParams, "--height", fmt.Sprintf("%d", options.Height))
	}

	if options.Width > 0 {
		cmdParams = append(cmdParams, "--width", fmt.Sprintf("%d", options.Width))
	}

	if options.Force {
		cmdParams = append(cmdParams, "--disable-smart-width")
	}

	//result tmp file
	resultTmpFname := fmt.Sprintf("%s%d.result.%s", jobPrefix, time.Now().UnixNano(), options.Format)
	resultTmpFpath := filepath.Join(os.TempDir(), resultTmpFname)
	defer os.Remove(resultTmpFpath)

	cmdParams = append(cmdParams, localPageTmpFpath, resultTmpFpath)

	//cmd
	convertCmd := exec.Command("wkhtmltoimage", cmdParams...)
	log.Println(convertCmd.Path, convertCmd.Args)

	stdErrPipe, pipeErr := convertCmd.StderrPipe()
	if pipeErr != nil {
		err = errors.New(fmt.Sprintf("open exec stderr pipe error, %s", pipeErr.Error()))
		return
	}

	if startErr := convertCmd.Start(); startErr != nil {
		err = errors.New(fmt.Sprintf("start html2image command error, %s", startErr.Error()))
		return
	}

	stdErrData, readErr := ioutil.ReadAll(stdErrPipe)
	if readErr != nil {
		err = errors.New(fmt.Sprintf("read html2image command stderr error, %s", readErr.Error()))
		return
	}

	//check stderr output & output file
	if string(stdErrData) != "" {
		log.Println(string(stdErrData))
	}

	if waitErr := convertCmd.Wait(); waitErr != nil {
		err = errors.New(fmt.Sprintf("wait html2image to exit error, %s", waitErr.Error()))
		return
	}

	if _, statErr := os.Stat(resultTmpFpath); statErr == nil {
		oTmpFp, openErr := os.Open(resultTmpFpath)
		if openErr != nil {
			err = errors.New(fmt.Sprintf("open html2image output result error, %s", openErr.Error()))
			return
		}
		defer oTmpFp.Close()

		outputBytes, readErr := ioutil.ReadAll(oTmpFp)
		if readErr != nil {
			err = errors.New(fmt.Sprintf("read html2image output result error, %s", readErr.Error()))
			return
		}
		result = outputBytes
	} else {
		err = errors.New("html2image with no valid output result")
	}

	if options.Format == "png" {
		contentType = "image/png"
	} else {
		contentType = "image/jpeg"
	}

	return
}
<commit_msg>Fix bug in html2image.<commit_after>package ufop

import (
	"errors"
	"fmt"
	"io"
	"io/ioutil"
	"log"
	"net/http"
	"os"
	"os/exec"
	"path/filepath"
	"regexp"
	"strconv"
	"strings"
	"time"
)

const (
	HTML2IMAGE_MAX_PAGE_SIZE = 10 * 1024 * 1024
)

type Html2Imager struct {
	maxPageSize int64
}

type Html2ImageOptions struct {
	CropH   int
	CropW   int
	CropX   int
	CropY   int
	Format  string
	Height  int
	Width   int
	Quality int
	Force   bool
}

func (this *Html2Imager) parse(cmd string) (options *Html2ImageOptions, err error) {
	pattern := `^html2image(/croph/\d+|/cropw/\d+|/cropx/\d+|/cropy/\d+|/format/(png|jpg|jpeg)|/height/\d+|/quality/\d+|/width/\d+|/force/[0|1]){0,9}$`
	matched, _ := regexp.Match(pattern, []byte(cmd))
	if !matched {
		err = errors.New("invalid html2image command format")
		return
	}

	options = &Html2ImageOptions{
		Format: "jpg",
	}

	//croph
	cropHStr := getParam(cmd, `croph/\d+`, "croph")
	if cropHStr != "" {
		cropH, _ := strconv.Atoi(cropHStr)
		if cropH <= 0 {
			err = errors.New("invalid html2image parameter 'croph'")
			return
		} else {
			options.CropH = cropH
		}
	}

	//cropw
	cropWStr := getParam(cmd, `cropw/\d+`, "cropw")
	if cropWStr != "" {
		cropW, _ := strconv.Atoi(cropWStr)
		if cropW <= 0 {
			err = errors.New("invalid html2image parameter 'cropw'")
			return
		} else {
			options.CropW = cropW
		}
	}

	//cropx
	cropXStr := getParam(cmd, `cropx/\d+`, "cropx")
	fmt.Println(cropXStr)
	if cropXStr != "" {
		cropX, _ := strconv.Atoi(cropXStr)
		if cropX <= 0 {
			err = errors.New("invalid html2image parameter 'cropx'")
			return
		} else {
			options.CropX = cropX
		}
	}

	//cropy
	cropYStr := getParam(cmd, `cropy/\d+`, "cropy")
	if cropYStr != "" {
		cropY, _ := strconv.Atoi(cropYStr)
		if cropY <= 0 {
			err = errors.New("invalid html2image parameter 'cropy'")
			return
		} else {
			options.CropY = cropY
		}
	}

	//format
	formatStr := getParam(cmd, "format/(png|jpg|jpeg)", "format")
	if formatStr != "" {
		options.Format = formatStr
	}

	//height
	heightStr := getParam(cmd, `height/\d+`, "height")
	if heightStr != "" {
		height, _ := strconv.Atoi(heightStr)
		if height <= 0 {
			err = errors.New("invalid html2image parameter 'height'")
			return
		} else {
			options.Height = height
		}
	}

	//width
	widthStr := getParam(cmd, `width/\d+`, "width")
	if widthStr != "" {
		width, _ := strconv.Atoi(widthStr)
		if width <= 0 {
			err = errors.New("invalid html2image parameter 'width'")
			return
		} else {
			options.Width = width
		}
	}

	//quality
	qualityStr := getParam(cmd, `quality/\d+`, "quality")
	if qualityStr != "" {
		quality, _ := strconv.Atoi(qualityStr)
		if quality > 100 || quality <= 0 {
			err = errors.New("invalid html2image parameter 'quality'")
			return
		} else {
			options.Quality = quality
		}
	}

	//force
	forceStr := getParam(cmd, "force/[0|1]", "force")
	if forceStr != "" {
		force, _ := strconv.Atoi(forceStr)
		if force == 1 {
			options.Force = true
		}
	}

	return

}

func (this *Html2Imager) Do(req UfopRequest) (result interface{}, contentType string, err error) {
	if this.maxPageSize <= 0 {
		this.maxPageSize = HTML2IMAGE_MAX_PAGE_SIZE
	}

	//if not text format, error it
	if !strings.HasPrefix(req.Src.MimeType, "text/") {
		err = errors.New("unsupported file mime type, only text/* allowed")
		return
	}

	//if file size exceeds, error it
	if req.Src.Fsize > this.maxPageSize {
		err = errors.New("page file length exceeds the limit")
		return
	}

	options, pErr := this.parse(req.Cmd)
	if pErr != nil {
		err = pErr
		return
	}

	//get page file content save it into temp dir
	resp, respErr := http.Get(req.Src.Url)
	if respErr != nil || resp.StatusCode != 200 {
		if respErr != nil {
			err = errors.New(fmt.Sprintf("retrieve page file resource data failed, %s", respErr.Error()))
		} else {
			err = errors.New(fmt.Sprintf("retrieve page file resource data failed, %s", resp.Status))
			if resp.Body != nil {
				resp.Body.Close()
			}
		}
		return
	}

	jobPrefix := md5Hex(req.Src.Url)

	pageSuffix := "txt"
	if req.Src.MimeType == "text/html" {
		pageSuffix = "html"
	}

	localPageTmpFname := fmt.Sprintf("%s%d.page.%s", jobPrefix, time.Now().UnixNano(), pageSuffix)
	localPageTmpFpath := filepath.Join(os.TempDir(), localPageTmpFname)
	defer os.Remove(localPageTmpFpath)

	localPageTmpFp, openErr := os.OpenFile(localPageTmpFpath, os.O_CREATE|os.O_WRONLY|os.O_TRUNC, 0655)
	if openErr != nil {
		err = errors.New(fmt.Sprintf("open page file temp file failed, %s", openErr.Error()))
		return
	}
	_, cpErr := io.Copy(localPageTmpFp, resp.Body)
	if cpErr != nil {
		err = errors.New(fmt.Sprintf("save page file content to tmp file failed, %s", cpErr.Error()))
		return
	}

	localPageTmpFp.Close()
	resp.Body.Close()

	//prepare command
	cmdParams := make([]string, 0)
	cmdParams = append(cmdParams, "-q")

	if options.CropH > 0 {
		cmdParams = append(cmdParams, "--crop-h", fmt.Sprintf("%d", options.CropH))
	}

	if options.CropW > 0 {
		cmdParams = append(cmdParams, "--crop-w", fmt.Sprintf("%d", options.CropW))
	}

	if options.CropX > 0 {
		cmdParams = append(cmdParams, "--crop-x", fmt.Sprintf("%d", options.CropX))
	}

	if options.CropY > 0 {
		cmdParams = append(cmdParams, "--crop-y", fmt.Sprintf("%d", options.CropY))
	}

	if options.Format != "" {
		cmdParams = append(cmdParams, "--format", options.Format)
	}

	if options.Quality > 0 {
		cmdParams = append(cmdParams, "--quality", fmt.Sprintf("%d", options.Quality))
	}

	if options.Height > 0 {
		cmdParams = append(cmdParams, "--height", fmt.Sprintf("%d", options.Height))
	}

	if options.Width > 0 {
		cmdParams = append(cmdParams, "--width", fmt.Sprintf("%d", options.Width))
	}

	if options.Force {
		cmdParams = append(cmdParams, "--disable-smart-width")
	}

	//result tmp file
	resultTmpFname := fmt.Sprintf("%s%d.result.%s", jobPrefix, time.Now().UnixNano(), options.Format)
	resultTmpFpath := filepath.Join(os.TempDir(), resultTmpFname)
	defer os.Remove(resultTmpFpath)

	cmdParams = append(cmdParams, localPageTmpFpath, resultTmpFpath)

	//cmd
	convertCmd := exec.Command("wkhtmltoimage", cmdParams...)
	log.Println(convertCmd.Path, convertCmd.Args)

	stdErrPipe, pipeErr := convertCmd.StderrPipe()
	if pipeErr != nil {
		err = errors.New(fmt.Sprintf("open exec stderr pipe error, %s", pipeErr.Error()))
		return
	}

	if startErr := convertCmd.Start(); startErr != nil {
		err = errors.New(fmt.Sprintf("start html2image command error, %s", startErr.Error()))
		return
	}

	stdErrData, readErr := ioutil.ReadAll(stdErrPipe)
	if readErr != nil {
		err = errors.New(fmt.Sprintf("read html2image command stderr error, %s", readErr.Error()))
		return
	}

	//check stderr output & output file
	if string(stdErrData) != "" {
		log.Println(string(stdErrData))
	}

	if waitErr := convertCmd.Wait(); waitErr != nil {
		err = errors.New(fmt.Sprintf("wait html2image to exit error, %s", waitErr.Error()))
		return
	}

	if _, statErr := os.Stat(resultTmpFpath); statErr == nil {
		oTmpFp, openErr := os.Open(resultTmpFpath)
		if openErr != nil {
			err = errors.New(fmt.Sprintf("open html2image output result error, %s", openErr.Error()))
			return
		}
		defer oTmpFp.Close()

		outputBytes, readErr := ioutil.ReadAll(oTmpFp)
		if readErr != nil {
			err = errors.New(fmt.Sprintf("read html2image output result error, %s", readErr.Error()))
			return
		}
		result = outputBytes
	} else {
		err = errors.New("html2image with no valid output result")
	}

	if options.Format == "png" {
		contentType = "image/png"
	} else {
		contentType = "image/jpeg"
	}

	return
}
<|endoftext|>


<commit_before>package api

import "fmt"

//AzureEnvironmentSpecConfig is the overall configuration differences in different cloud environments.
type AzureEnvironmentSpecConfig struct {
	CloudName            string
	DockerSpecConfig     DockerSpecConfig
	KubernetesSpecConfig KubernetesSpecConfig
	DCOSSpecConfig       DCOSSpecConfig
	EndpointConfig       AzureEndpointConfig
	OSImageConfig        map[Distro]AzureOSImageConfig
}

//DockerSpecConfig is the configurations of docker
type DockerSpecConfig struct {
	DockerEngineRepo         string
	DockerComposeDownloadURL string
}

//DCOSSpecConfig is the configurations of DCOS
type DCOSSpecConfig struct {
	DCOS188BootstrapDownloadURL     string
	DCOS190BootstrapDownloadURL     string
	DCOS198BootstrapDownloadURL     string
	DCOS110BootstrapDownloadURL     string
	DCOS111BootstrapDownloadURL     string
	DCOSWindowsBootstrapDownloadURL string
	DcosRepositoryURL               string // For custom install, for example CI, need these three addributes
	DcosClusterPackageListID        string // the id of the package list file
	DcosProviderPackageID           string // the id of the dcos-provider-xxx package
}

//KubernetesSpecConfig is the kubernetes container images used.
type KubernetesSpecConfig struct {
	KubernetesImageBase              string
	TillerImageBase                  string
	ACIConnectorImageBase            string
	NVIDIAImageBase                  string
	AzureCNIImageBase                string
	EtcdDownloadURLBase              string
	KubeBinariesSASURLBase           string
	WindowsPackageSASURLBase         string
	WindowsTelemetryGUID             string
	CNIPluginsDownloadURL            string
	VnetCNILinuxPluginsDownloadURL   string
	VnetCNIWindowsPluginsDownloadURL string
	ContainerdDownloadURLBase        string
}

//AzureEndpointConfig describes an Azure endpoint
type AzureEndpointConfig struct {
	ResourceManagerVMDNSSuffix string
}

//AzureOSImageConfig describes an Azure OS image
type AzureOSImageConfig struct {
	ImageOffer     string
	ImageSku       string
	ImagePublisher string
	ImageVersion   string
}

var (
	//DefaultKubernetesSpecConfig is the default Docker image source of Kubernetes
	DefaultKubernetesSpecConfig = KubernetesSpecConfig{
		KubernetesImageBase:              "k8s.gcr.io/",
		TillerImageBase:                  "gcr.io/kubernetes-helm/",
		ACIConnectorImageBase:            "microsoft/",
		NVIDIAImageBase:                  "nvidia/",
		AzureCNIImageBase:                "containernetworking/",
		EtcdDownloadURLBase:              "https://acs-mirror.azureedge.net/github-coreos",
		KubeBinariesSASURLBase:           "https://acs-mirror.azureedge.net/wink8s/",
		WindowsPackageSASURLBase:         "https://acs-mirror.azureedge.net/wink8s/",
		WindowsTelemetryGUID:             "fb801154-36b9-41bc-89c2-f4d4f05472b0",
		CNIPluginsDownloadURL:            "https://acs-mirror.azureedge.net/cni/cni-plugins-amd64-" + CNIPluginVer + ".tgz",
		VnetCNILinuxPluginsDownloadURL:   "https://acs-mirror.azureedge.net/cni/azure-vnet-cni-linux-amd64-" + AzureCniPluginVerLinux + ".tgz",
		VnetCNIWindowsPluginsDownloadURL: "https://acs-mirror.azureedge.net/cni/azure-vnet-cni-windows-amd64-" + AzureCniPluginVerWindows + ".zip",
		ContainerdDownloadURLBase:        "https://storage.googleapis.com/cri-containerd-release/",
	}

	//DefaultDCOSSpecConfig is the default DC/OS binary download URL.
	DefaultDCOSSpecConfig = DCOSSpecConfig{
		DCOS188BootstrapDownloadURL:     fmt.Sprintf(AzureEdgeDCOSBootstrapDownloadURL, "stable", "5df43052907c021eeb5de145419a3da1898c58a5"),
		DCOS190BootstrapDownloadURL:     fmt.Sprintf(AzureEdgeDCOSBootstrapDownloadURL, "stable", "58fd0833ce81b6244fc73bf65b5deb43217b0bd7"),
		DCOS198BootstrapDownloadURL:     fmt.Sprintf(AzureEdgeDCOSBootstrapDownloadURL, "stable/1.9.8", "f4ae0d20665fc68ee25282d6f78681b2773c6e10"),
		DCOS110BootstrapDownloadURL:     fmt.Sprintf(AzureEdgeDCOSBootstrapDownloadURL, "stable/1.10.0", "4d92536e7381176206e71ee15b5ffe454439920c"),
		DCOS111BootstrapDownloadURL:     fmt.Sprintf(AzureEdgeDCOSBootstrapDownloadURL, "stable/1.11.0", "a0654657903fb68dff60f6e522a7f241c1bfbf0f"),
		DCOSWindowsBootstrapDownloadURL: "http://dcos-win.westus.cloudapp.azure.com/dcos-windows/stable/",
		DcosRepositoryURL:               "https://dcosio.azureedge.net/dcos/stable/1.11.0",
		DcosClusterPackageListID:        "248a66388bba1adbcb14a52fd3b7b424ab06fa76",
	}

	//DefaultDockerSpecConfig is the default Docker engine repo.
	DefaultDockerSpecConfig = DockerSpecConfig{
		DockerEngineRepo:         "https://aptdocker.azureedge.net/repo",
		DockerComposeDownloadURL: "https://github.com/docker/compose/releases/download",
	}

	//DefaultUbuntuImageConfig is the default Linux distribution.
	DefaultUbuntuImageConfig = AzureOSImageConfig{
		ImageOffer:     "UbuntuServer",
		ImageSku:       "16.04-LTS",
		ImagePublisher: "Canonical",
		ImageVersion:   "latest",
	}

	//SovereignCloudsUbuntuImageConfig is the Linux distribution for Azure Sovereign Clouds.
	SovereignCloudsUbuntuImageConfig = AzureOSImageConfig{
		ImageOffer:     "UbuntuServer",
		ImageSku:       "16.04-LTS",
		ImagePublisher: "Canonical",
		ImageVersion:   "latest",
	}

	//GermanCloudUbuntuImageConfig is the Linux distribution for Azure Sovereign Clouds.
	GermanCloudUbuntuImageConfig = AzureOSImageConfig{
		ImageOffer:     "UbuntuServer",
		ImageSku:       "16.04-LTS",
		ImagePublisher: "Canonical",
		ImageVersion:   "16.04.201801050",
	}

	//DefaultRHELOSImageConfig is the RHEL Linux distribution.
	DefaultRHELOSImageConfig = AzureOSImageConfig{
		ImageOffer:     "RHEL",
		ImageSku:       "7.3",
		ImagePublisher: "RedHat",
		ImageVersion:   "latest",
	}

	//DefaultCoreOSImageConfig is the CoreOS Linux distribution.
	DefaultCoreOSImageConfig = AzureOSImageConfig{
		ImageOffer:     "CoreOS",
		ImageSku:       "Stable",
		ImagePublisher: "CoreOS",
		ImageVersion:   "latest",
	}

	// DefaultAKSOSImageConfig is the AKS image based on Ubuntu 16.04.
	DefaultAKSOSImageConfig = AzureOSImageConfig{
		ImageOffer:     "aks",
		ImageSku:       "aks-ubuntu-1604-201810",
		ImagePublisher: "microsoft-aks",
		ImageVersion:   "2018.10.26",
	}

	// DefaultAKSDockerEngineOSImageConfig is the AKS image based on Ubuntu 16.04.
	DefaultAKSDockerEngineOSImageConfig = AzureOSImageConfig{
		ImageOffer:     "aks",
		ImageSku:       "aks-ubuntu-1604-docker-engine",
		ImagePublisher: "microsoft-aks",
		ImageVersion:   "2018.10.26",
	}

	//DefaultOpenShift39RHELImageConfig is the OpenShift on RHEL distribution.
	DefaultOpenShift39RHELImageConfig = AzureOSImageConfig{
		ImageOffer:     "acsengine-preview",
		ImageSku:       "rhel74",
		ImagePublisher: "redhat",
		ImageVersion:   "latest",
	}

	//DefaultOpenShift39CentOSImageConfig is the OpenShift on CentOS distribution.
	DefaultOpenShift39CentOSImageConfig = AzureOSImageConfig{
		ImageOffer:     "origin-acsengine-preview",
		ImageSku:       "centos7",
		ImagePublisher: "redhat",
		ImageVersion:   "latest",
	}

	//AzureCloudSpec is the default configurations for global azure.
	AzureCloudSpec = AzureEnvironmentSpecConfig{
		CloudName: azurePublicCloud,
		//DockerSpecConfig specify the docker engine download repo
		DockerSpecConfig: DefaultDockerSpecConfig,
		//KubernetesSpecConfig is the default kubernetes container image url.
		KubernetesSpecConfig: DefaultKubernetesSpecConfig,
		DCOSSpecConfig:       DefaultDCOSSpecConfig,

		EndpointConfig: AzureEndpointConfig{
			ResourceManagerVMDNSSuffix: "cloudapp.azure.com",
		},

		OSImageConfig: map[Distro]AzureOSImageConfig{
			Ubuntu:          DefaultUbuntuImageConfig,
			RHEL:            DefaultRHELOSImageConfig,
			CoreOS:          DefaultCoreOSImageConfig,
			AKS:             DefaultAKSOSImageConfig,
			AKSDockerEngine: DefaultAKSDockerEngineOSImageConfig,
			// Image config supported for OpenShift
			OpenShift39RHEL: DefaultOpenShift39RHELImageConfig,
			OpenShiftCentOS: DefaultOpenShift39CentOSImageConfig,
		},
	}

	//AzureGermanCloudSpec is the German cloud config.
	AzureGermanCloudSpec = AzureEnvironmentSpecConfig{
		CloudName:            azureGermanCloud,
		DockerSpecConfig:     DefaultDockerSpecConfig,
		KubernetesSpecConfig: DefaultKubernetesSpecConfig,
		DCOSSpecConfig:       DefaultDCOSSpecConfig,
		EndpointConfig: AzureEndpointConfig{
			ResourceManagerVMDNSSuffix: "cloudapp.microsoftazure.de",
		},
		OSImageConfig: map[Distro]AzureOSImageConfig{
			Ubuntu: GermanCloudUbuntuImageConfig,
			RHEL:   DefaultRHELOSImageConfig,
			CoreOS: DefaultCoreOSImageConfig,
			AKS:    GermanCloudUbuntuImageConfig,
		},
	}

	//AzureUSGovernmentCloud is the US government config.
	AzureUSGovernmentCloud = AzureEnvironmentSpecConfig{
		CloudName:            azureUSGovernmentCloud,
		DockerSpecConfig:     DefaultDockerSpecConfig,
		KubernetesSpecConfig: DefaultKubernetesSpecConfig,
		DCOSSpecConfig:       DefaultDCOSSpecConfig,
		EndpointConfig: AzureEndpointConfig{
			ResourceManagerVMDNSSuffix: "cloudapp.usgovcloudapi.net",
		},
		OSImageConfig: map[Distro]AzureOSImageConfig{
			Ubuntu: SovereignCloudsUbuntuImageConfig,
			RHEL:   DefaultRHELOSImageConfig,
			CoreOS: DefaultCoreOSImageConfig,
			AKS:    SovereignCloudsUbuntuImageConfig,
		},
	}

	//AzureChinaCloudSpec is the configurations for Azure China (Mooncake)
	AzureChinaCloudSpec = AzureEnvironmentSpecConfig{
		CloudName: azureChinaCloud,
		//DockerSpecConfig specify the docker engine download repo
		DockerSpecConfig: DockerSpecConfig{
			DockerEngineRepo:         "https://mirror.azure.cn/docker-engine/apt/repo/",
			DockerComposeDownloadURL: "https://mirror.azure.cn/docker-toolbox/linux/compose",
		},
		//KubernetesSpecConfig - Due to Chinese firewall issue, the default containers from google is blocked, use the Chinese local mirror instead
		KubernetesSpecConfig: KubernetesSpecConfig{
			KubernetesImageBase:              "gcr.azk8s.cn/google_containers/",
			TillerImageBase:                  "gcr.azk8s.cn/kubernetes-helm/",
			ACIConnectorImageBase:            "dockerhub.azk8s.cn/microsoft/",
			NVIDIAImageBase:                  "dockerhub.azk8s.cn/nvidia/",
			AzureCNIImageBase:                "dockerhub.azk8s.cn/containernetworking/",
			EtcdDownloadURLBase:              DefaultKubernetesSpecConfig.EtcdDownloadURLBase,
			KubeBinariesSASURLBase:           DefaultKubernetesSpecConfig.KubeBinariesSASURLBase,
			WindowsPackageSASURLBase:         DefaultKubernetesSpecConfig.WindowsPackageSASURLBase,
			WindowsTelemetryGUID:             DefaultKubernetesSpecConfig.WindowsTelemetryGUID,
			CNIPluginsDownloadURL:            DefaultKubernetesSpecConfig.CNIPluginsDownloadURL,
			VnetCNILinuxPluginsDownloadURL:   DefaultKubernetesSpecConfig.VnetCNILinuxPluginsDownloadURL,
			VnetCNIWindowsPluginsDownloadURL: DefaultKubernetesSpecConfig.VnetCNIWindowsPluginsDownloadURL,
			ContainerdDownloadURLBase:        "https://mirror.azure.cn/kubernetes/containerd/",
		},
		DCOSSpecConfig: DCOSSpecConfig{
			DCOS188BootstrapDownloadURL:     fmt.Sprintf(AzureChinaCloudDCOSBootstrapDownloadURL, "5df43052907c021eeb5de145419a3da1898c58a5"),
			DCOSWindowsBootstrapDownloadURL: "https://dcosdevstorage.blob.core.windows.net/dcos-windows",
			DCOS190BootstrapDownloadURL:     fmt.Sprintf(AzureChinaCloudDCOSBootstrapDownloadURL, "58fd0833ce81b6244fc73bf65b5deb43217b0bd7"),
			DCOS198BootstrapDownloadURL:     fmt.Sprintf(AzureChinaCloudDCOSBootstrapDownloadURL, "f4ae0d20665fc68ee25282d6f78681b2773c6e10"),
		},

		EndpointConfig: AzureEndpointConfig{
			ResourceManagerVMDNSSuffix: "cloudapp.chinacloudapi.cn",
		},
		OSImageConfig: map[Distro]AzureOSImageConfig{
			Ubuntu: SovereignCloudsUbuntuImageConfig,
			RHEL:   DefaultRHELOSImageConfig,
			CoreOS: DefaultCoreOSImageConfig,
			AKS:    SovereignCloudsUbuntuImageConfig,
		},
	}

	// AzureCloudSpecEnvMap is the environment configuration map for all the Azure cloid environments.
	AzureCloudSpecEnvMap = map[string]AzureEnvironmentSpecConfig{
		azureChinaCloud:        AzureChinaCloudSpec,
		azureGermanCloud:       AzureGermanCloudSpec,
		azureUSGovernmentCloud: AzureUSGovernmentCloud,
		azurePublicCloud:       AzureCloudSpec,
	}
)
<commit_msg>use china mirror in binary downloading (#4137)<commit_after>package api

import "fmt"

//AzureEnvironmentSpecConfig is the overall configuration differences in different cloud environments.
type AzureEnvironmentSpecConfig struct {
	CloudName            string
	DockerSpecConfig     DockerSpecConfig
	KubernetesSpecConfig KubernetesSpecConfig
	DCOSSpecConfig       DCOSSpecConfig
	EndpointConfig       AzureEndpointConfig
	OSImageConfig        map[Distro]AzureOSImageConfig
}

//DockerSpecConfig is the configurations of docker
type DockerSpecConfig struct {
	DockerEngineRepo         string
	DockerComposeDownloadURL string
}

//DCOSSpecConfig is the configurations of DCOS
type DCOSSpecConfig struct {
	DCOS188BootstrapDownloadURL     string
	DCOS190BootstrapDownloadURL     string
	DCOS198BootstrapDownloadURL     string
	DCOS110BootstrapDownloadURL     string
	DCOS111BootstrapDownloadURL     string
	DCOSWindowsBootstrapDownloadURL string
	DcosRepositoryURL               string // For custom install, for example CI, need these three addributes
	DcosClusterPackageListID        string // the id of the package list file
	DcosProviderPackageID           string // the id of the dcos-provider-xxx package
}

//KubernetesSpecConfig is the kubernetes container images used.
type KubernetesSpecConfig struct {
	KubernetesImageBase              string
	TillerImageBase                  string
	ACIConnectorImageBase            string
	NVIDIAImageBase                  string
	AzureCNIImageBase                string
	EtcdDownloadURLBase              string
	KubeBinariesSASURLBase           string
	WindowsPackageSASURLBase         string
	WindowsTelemetryGUID             string
	CNIPluginsDownloadURL            string
	VnetCNILinuxPluginsDownloadURL   string
	VnetCNIWindowsPluginsDownloadURL string
	ContainerdDownloadURLBase        string
}

//AzureEndpointConfig describes an Azure endpoint
type AzureEndpointConfig struct {
	ResourceManagerVMDNSSuffix string
}

//AzureOSImageConfig describes an Azure OS image
type AzureOSImageConfig struct {
	ImageOffer     string
	ImageSku       string
	ImagePublisher string
	ImageVersion   string
}

var (
	//DefaultKubernetesSpecConfig is the default Docker image source of Kubernetes
	DefaultKubernetesSpecConfig = KubernetesSpecConfig{
		KubernetesImageBase:              "k8s.gcr.io/",
		TillerImageBase:                  "gcr.io/kubernetes-helm/",
		ACIConnectorImageBase:            "microsoft/",
		NVIDIAImageBase:                  "nvidia/",
		AzureCNIImageBase:                "containernetworking/",
		EtcdDownloadURLBase:              "https://acs-mirror.azureedge.net/github-coreos",
		KubeBinariesSASURLBase:           "https://acs-mirror.azureedge.net/wink8s/",
		WindowsPackageSASURLBase:         "https://acs-mirror.azureedge.net/wink8s/",
		WindowsTelemetryGUID:             "fb801154-36b9-41bc-89c2-f4d4f05472b0",
		CNIPluginsDownloadURL:            "https://acs-mirror.azureedge.net/cni/cni-plugins-amd64-" + CNIPluginVer + ".tgz",
		VnetCNILinuxPluginsDownloadURL:   "https://acs-mirror.azureedge.net/cni/azure-vnet-cni-linux-amd64-" + AzureCniPluginVerLinux + ".tgz",
		VnetCNIWindowsPluginsDownloadURL: "https://acs-mirror.azureedge.net/cni/azure-vnet-cni-windows-amd64-" + AzureCniPluginVerWindows + ".zip",
		ContainerdDownloadURLBase:        "https://storage.googleapis.com/cri-containerd-release/",
	}

	//DefaultDCOSSpecConfig is the default DC/OS binary download URL.
	DefaultDCOSSpecConfig = DCOSSpecConfig{
		DCOS188BootstrapDownloadURL:     fmt.Sprintf(AzureEdgeDCOSBootstrapDownloadURL, "stable", "5df43052907c021eeb5de145419a3da1898c58a5"),
		DCOS190BootstrapDownloadURL:     fmt.Sprintf(AzureEdgeDCOSBootstrapDownloadURL, "stable", "58fd0833ce81b6244fc73bf65b5deb43217b0bd7"),
		DCOS198BootstrapDownloadURL:     fmt.Sprintf(AzureEdgeDCOSBootstrapDownloadURL, "stable/1.9.8", "f4ae0d20665fc68ee25282d6f78681b2773c6e10"),
		DCOS110BootstrapDownloadURL:     fmt.Sprintf(AzureEdgeDCOSBootstrapDownloadURL, "stable/1.10.0", "4d92536e7381176206e71ee15b5ffe454439920c"),
		DCOS111BootstrapDownloadURL:     fmt.Sprintf(AzureEdgeDCOSBootstrapDownloadURL, "stable/1.11.0", "a0654657903fb68dff60f6e522a7f241c1bfbf0f"),
		DCOSWindowsBootstrapDownloadURL: "http://dcos-win.westus.cloudapp.azure.com/dcos-windows/stable/",
		DcosRepositoryURL:               "https://dcosio.azureedge.net/dcos/stable/1.11.0",
		DcosClusterPackageListID:        "248a66388bba1adbcb14a52fd3b7b424ab06fa76",
	}

	//DefaultDockerSpecConfig is the default Docker engine repo.
	DefaultDockerSpecConfig = DockerSpecConfig{
		DockerEngineRepo:         "https://aptdocker.azureedge.net/repo",
		DockerComposeDownloadURL: "https://github.com/docker/compose/releases/download",
	}

	//DefaultUbuntuImageConfig is the default Linux distribution.
	DefaultUbuntuImageConfig = AzureOSImageConfig{
		ImageOffer:     "UbuntuServer",
		ImageSku:       "16.04-LTS",
		ImagePublisher: "Canonical",
		ImageVersion:   "latest",
	}

	//SovereignCloudsUbuntuImageConfig is the Linux distribution for Azure Sovereign Clouds.
	SovereignCloudsUbuntuImageConfig = AzureOSImageConfig{
		ImageOffer:     "UbuntuServer",
		ImageSku:       "16.04-LTS",
		ImagePublisher: "Canonical",
		ImageVersion:   "latest",
	}

	//GermanCloudUbuntuImageConfig is the Linux distribution for Azure Sovereign Clouds.
	GermanCloudUbuntuImageConfig = AzureOSImageConfig{
		ImageOffer:     "UbuntuServer",
		ImageSku:       "16.04-LTS",
		ImagePublisher: "Canonical",
		ImageVersion:   "16.04.201801050",
	}

	//DefaultRHELOSImageConfig is the RHEL Linux distribution.
	DefaultRHELOSImageConfig = AzureOSImageConfig{
		ImageOffer:     "RHEL",
		ImageSku:       "7.3",
		ImagePublisher: "RedHat",
		ImageVersion:   "latest",
	}

	//DefaultCoreOSImageConfig is the CoreOS Linux distribution.
	DefaultCoreOSImageConfig = AzureOSImageConfig{
		ImageOffer:     "CoreOS",
		ImageSku:       "Stable",
		ImagePublisher: "CoreOS",
		ImageVersion:   "latest",
	}

	// DefaultAKSOSImageConfig is the AKS image based on Ubuntu 16.04.
	DefaultAKSOSImageConfig = AzureOSImageConfig{
		ImageOffer:     "aks",
		ImageSku:       "aks-ubuntu-1604-201810",
		ImagePublisher: "microsoft-aks",
		ImageVersion:   "2018.10.26",
	}

	// DefaultAKSDockerEngineOSImageConfig is the AKS image based on Ubuntu 16.04.
	DefaultAKSDockerEngineOSImageConfig = AzureOSImageConfig{
		ImageOffer:     "aks",
		ImageSku:       "aks-ubuntu-1604-docker-engine",
		ImagePublisher: "microsoft-aks",
		ImageVersion:   "2018.10.26",
	}

	//DefaultOpenShift39RHELImageConfig is the OpenShift on RHEL distribution.
	DefaultOpenShift39RHELImageConfig = AzureOSImageConfig{
		ImageOffer:     "acsengine-preview",
		ImageSku:       "rhel74",
		ImagePublisher: "redhat",
		ImageVersion:   "latest",
	}

	//DefaultOpenShift39CentOSImageConfig is the OpenShift on CentOS distribution.
	DefaultOpenShift39CentOSImageConfig = AzureOSImageConfig{
		ImageOffer:     "origin-acsengine-preview",
		ImageSku:       "centos7",
		ImagePublisher: "redhat",
		ImageVersion:   "latest",
	}

	//AzureCloudSpec is the default configurations for global azure.
	AzureCloudSpec = AzureEnvironmentSpecConfig{
		CloudName: azurePublicCloud,
		//DockerSpecConfig specify the docker engine download repo
		DockerSpecConfig: DefaultDockerSpecConfig,
		//KubernetesSpecConfig is the default kubernetes container image url.
		KubernetesSpecConfig: DefaultKubernetesSpecConfig,
		DCOSSpecConfig:       DefaultDCOSSpecConfig,

		EndpointConfig: AzureEndpointConfig{
			ResourceManagerVMDNSSuffix: "cloudapp.azure.com",
		},

		OSImageConfig: map[Distro]AzureOSImageConfig{
			Ubuntu:          DefaultUbuntuImageConfig,
			RHEL:            DefaultRHELOSImageConfig,
			CoreOS:          DefaultCoreOSImageConfig,
			AKS:             DefaultAKSOSImageConfig,
			AKSDockerEngine: DefaultAKSDockerEngineOSImageConfig,
			// Image config supported for OpenShift
			OpenShift39RHEL: DefaultOpenShift39RHELImageConfig,
			OpenShiftCentOS: DefaultOpenShift39CentOSImageConfig,
		},
	}

	//AzureGermanCloudSpec is the German cloud config.
	AzureGermanCloudSpec = AzureEnvironmentSpecConfig{
		CloudName:            azureGermanCloud,
		DockerSpecConfig:     DefaultDockerSpecConfig,
		KubernetesSpecConfig: DefaultKubernetesSpecConfig,
		DCOSSpecConfig:       DefaultDCOSSpecConfig,
		EndpointConfig: AzureEndpointConfig{
			ResourceManagerVMDNSSuffix: "cloudapp.microsoftazure.de",
		},
		OSImageConfig: map[Distro]AzureOSImageConfig{
			Ubuntu: GermanCloudUbuntuImageConfig,
			RHEL:   DefaultRHELOSImageConfig,
			CoreOS: DefaultCoreOSImageConfig,
			AKS:    GermanCloudUbuntuImageConfig,
		},
	}

	//AzureUSGovernmentCloud is the US government config.
	AzureUSGovernmentCloud = AzureEnvironmentSpecConfig{
		CloudName:            azureUSGovernmentCloud,
		DockerSpecConfig:     DefaultDockerSpecConfig,
		KubernetesSpecConfig: DefaultKubernetesSpecConfig,
		DCOSSpecConfig:       DefaultDCOSSpecConfig,
		EndpointConfig: AzureEndpointConfig{
			ResourceManagerVMDNSSuffix: "cloudapp.usgovcloudapi.net",
		},
		OSImageConfig: map[Distro]AzureOSImageConfig{
			Ubuntu: SovereignCloudsUbuntuImageConfig,
			RHEL:   DefaultRHELOSImageConfig,
			CoreOS: DefaultCoreOSImageConfig,
			AKS:    SovereignCloudsUbuntuImageConfig,
		},
	}

	//AzureChinaCloudSpec is the configurations for Azure China (Mooncake)
	AzureChinaCloudSpec = AzureEnvironmentSpecConfig{
		CloudName: azureChinaCloud,
		//DockerSpecConfig specify the docker engine download repo
		DockerSpecConfig: DockerSpecConfig{
			DockerEngineRepo:         "https://mirror.azk8s.cn/docker-engine/apt/repo/",
			DockerComposeDownloadURL: "https://mirror.azk8s.cn/docker-toolbox/linux/compose",
		},
		//KubernetesSpecConfig - Due to Chinese firewall issue, the default containers from google is blocked, use the Chinese local mirror instead
		KubernetesSpecConfig: KubernetesSpecConfig{
			KubernetesImageBase:              "gcr.azk8s.cn/google_containers/",
			TillerImageBase:                  "gcr.azk8s.cn/kubernetes-helm/",
			ACIConnectorImageBase:            "dockerhub.azk8s.cn/microsoft/",
			NVIDIAImageBase:                  "dockerhub.azk8s.cn/nvidia/",
			AzureCNIImageBase:                "dockerhub.azk8s.cn/containernetworking/",
			EtcdDownloadURLBase:              "https://mirror.azk8s.cn/kubernetes/etcd",
			KubeBinariesSASURLBase:           DefaultKubernetesSpecConfig.KubeBinariesSASURLBase,
			WindowsPackageSASURLBase:         DefaultKubernetesSpecConfig.WindowsPackageSASURLBase,
			WindowsTelemetryGUID:             DefaultKubernetesSpecConfig.WindowsTelemetryGUID,
			CNIPluginsDownloadURL:            "https://mirror.azk8s.cn/kubernetes/containernetworking-plugins/cni-plugins-amd64-" + CNIPluginVer + ".tgz",
			VnetCNILinuxPluginsDownloadURL:   "https://mirror.azk8s.cn/kubernetes/azure-container-networking/azure-vnet-cni-linux-amd64-" + AzureCniPluginVerLinux + ".tgz",
			VnetCNIWindowsPluginsDownloadURL: "https://mirror.azk8s.cn/kubernetes/azure-container-networking/azure-vnet-cni-windows-amd64-" + AzureCniPluginVerWindows + ".zip",
			ContainerdDownloadURLBase:        "https://mirror.azk8s.cn/kubernetes/containerd/",
		},
		DCOSSpecConfig: DCOSSpecConfig{
			DCOS188BootstrapDownloadURL:     fmt.Sprintf(AzureChinaCloudDCOSBootstrapDownloadURL, "5df43052907c021eeb5de145419a3da1898c58a5"),
			DCOSWindowsBootstrapDownloadURL: "https://dcosdevstorage.blob.core.windows.net/dcos-windows",
			DCOS190BootstrapDownloadURL:     fmt.Sprintf(AzureChinaCloudDCOSBootstrapDownloadURL, "58fd0833ce81b6244fc73bf65b5deb43217b0bd7"),
			DCOS198BootstrapDownloadURL:     fmt.Sprintf(AzureChinaCloudDCOSBootstrapDownloadURL, "f4ae0d20665fc68ee25282d6f78681b2773c6e10"),
		},

		EndpointConfig: AzureEndpointConfig{
			ResourceManagerVMDNSSuffix: "cloudapp.chinacloudapi.cn",
		},
		OSImageConfig: map[Distro]AzureOSImageConfig{
			Ubuntu: SovereignCloudsUbuntuImageConfig,
			RHEL:   DefaultRHELOSImageConfig,
			CoreOS: DefaultCoreOSImageConfig,
			AKS:    SovereignCloudsUbuntuImageConfig,
		},
	}

	// AzureCloudSpecEnvMap is the environment configuration map for all the Azure cloid environments.
	AzureCloudSpecEnvMap = map[string]AzureEnvironmentSpecConfig{
		azureChinaCloud:        AzureChinaCloudSpec,
		azureGermanCloud:       AzureGermanCloudSpec,
		azureUSGovernmentCloud: AzureUSGovernmentCloud,
		azurePublicCloud:       AzureCloudSpec,
	}
)
<|endoftext|>


<commit_before>package stns

import (
	"fmt"
	"path/filepath"
	"reflect"

	"github.com/BurntSushi/toml"
)

type Config struct {
	Port       int    `toml:"port"`
	Include    string `toml:"include"`
	Salt       bool   `toml:"salt_enable"`
	Stretching int    `toml:"stretching_number"`
	User       string `toml:"user"`
	Password   string `toml:"password"`
	HashType   string `toml:"hash_type" json:"hash_type"`
	Users      Attributes
	Groups     Attributes
	Sudoers    Attributes
}

var MinUserId, MinGroupId int

func LoadConfig(configFile string) (Config, error) {
	var config Config
	defaultConfig(&config)

	_, err := toml.DecodeFile(configFile, &config)
	if err != nil {
		return Config{}, err
	}

	if config.Include != "" {
		if err := includeConfigFile(&config, config.Include); err != nil {
			return Config{}, err
		}
	}
	setMinId(&MinUserId, config.Users)
	setMinId(&MinGroupId, config.Groups)
	mergeLinkAttribute("user", config.Users)
	mergeLinkAttribute("group", config.Groups)
	return config, nil
}

func defaultConfig(config *Config) {
	config.Port = 1104
	config.Salt = false
	config.Stretching = 0
	config.HashType = "sha256"
}

func includeConfigFile(config *Config, include string) error {
	files, err := filepath.Glob(include)
	if err != nil {
		return err
	}

	for _, file := range files {
		_, err := toml.DecodeFile(file, &config)
		if err != nil {
			return fmt.Errorf("while loading included config file %s: %s", file, err)
		}
	}
	return nil
}

func setMinId(min *int, attrs Attributes) {
	if len(attrs) > 0 {
		for _, a := range attrs {
			switch {
			case *min == 0:
				*min = a.Id
			case *min > a.Id:
				*min = a.Id
			}
		}
	}
}

func mergeLinkAttribute(rtype string, attr Attributes) {
	for k, v := range attr {
		mergeValue := []string{}
		linker := getLinker(rtype, v)

		if linker != nil && !reflect.ValueOf(linker).IsNil() &&
			linker.LinkParams() != nil && !reflect.ValueOf(linker.LinkParams()).IsNil() {
			for _, linkValue := range linker.LinkParams() {
				linkValues := map[string][]string{k: linker.LinkValue()}

				recursiveSetLinkValue(attr, rtype, linkValue, linkValues)
				for _, val := range linkValues {
					mergeValue = append(mergeValue, val...)
				}
				linker.SetLinkValue(RemoveDuplicates(mergeValue))
			}
		}
	}
}

func getLinker(rtype string, attr *Attribute) Linker {
	if attr != nil && !reflect.ValueOf(attr).IsNil() {
		if rtype == "user" {
			return attr.User
		} else if rtype == "group" {
			return attr.Group
		}
	}
	return nil
}

func recursiveSetLinkValue(attr Attributes, rtype, name string, result map[string][]string) {
	if result[name] != nil {
		return
	}

	linker := getLinker(rtype, attr[name])

	if linker != nil && !reflect.ValueOf(linker).IsNil() && len(linker.LinkValue()) > 0 {
		result[name] = linker.LinkValue()
		if linker.LinkParams() != nil || !reflect.ValueOf(linker.LinkParams()).IsNil() {
			for _, next_name := range linker.LinkParams() {
				recursiveSetLinkValue(attr, rtype, next_name, result)
			}
		}
	}
}

func member(n string, xs []string) bool {
	for _, x := range xs {
		if n == x {
			return true
		}
	}
	return false
}

func RemoveDuplicates(xs []string) []string {
	ys := make([]string, 0, len(xs))
	for _, x := range xs {
		if !member(x, ys) {
			ys = append(ys, x)
		}
	}
	return ys
}
<commit_msg>reset minid<commit_after>package stns

import (
	"fmt"
	"path/filepath"
	"reflect"

	"github.com/BurntSushi/toml"
)

type Config struct {
	Port       int    `toml:"port"`
	Include    string `toml:"include"`
	Salt       bool   `toml:"salt_enable"`
	Stretching int    `toml:"stretching_number"`
	User       string `toml:"user"`
	Password   string `toml:"password"`
	HashType   string `toml:"hash_type" json:"hash_type"`
	Users      Attributes
	Groups     Attributes
	Sudoers    Attributes
}

var MinUserId, MinGroupId int

func LoadConfig(configFile string) (Config, error) {
	var config Config
	defaultConfig(&config)

	_, err := toml.DecodeFile(configFile, &config)
	if err != nil {
		return Config{}, err
	}

	if config.Include != "" {
		if err := includeConfigFile(&config, config.Include); err != nil {
			return Config{}, err
		}
	}
	setMinId(&MinUserId, config.Users)
	setMinId(&MinGroupId, config.Groups)
	mergeLinkAttribute("user", config.Users)
	mergeLinkAttribute("group", config.Groups)
	return config, nil
}

func defaultConfig(config *Config) {
	config.Port = 1104
	config.Salt = false
	config.Stretching = 0
	config.HashType = "sha256"
}

func includeConfigFile(config *Config, include string) error {
	files, err := filepath.Glob(include)
	if err != nil {
		return err
	}

	for _, file := range files {
		_, err := toml.DecodeFile(file, &config)
		if err != nil {
			return fmt.Errorf("while loading included config file %s: %s", file, err)
		}
	}
	return nil
}

func setMinId(min *int, attrs Attributes) {
	*min = 0
	if len(attrs) > 0 {
		for _, a := range attrs {
			switch {
			case *min == 0:
				*min = a.Id
			case *min > a.Id:
				*min = a.Id
			}
		}
	}
}

func mergeLinkAttribute(rtype string, attr Attributes) {
	for k, v := range attr {
		mergeValue := []string{}
		linker := getLinker(rtype, v)

		if linker != nil && !reflect.ValueOf(linker).IsNil() &&
			linker.LinkParams() != nil && !reflect.ValueOf(linker.LinkParams()).IsNil() {
			for _, linkValue := range linker.LinkParams() {
				linkValues := map[string][]string{k: linker.LinkValue()}

				recursiveSetLinkValue(attr, rtype, linkValue, linkValues)
				for _, val := range linkValues {
					mergeValue = append(mergeValue, val...)
				}
				linker.SetLinkValue(RemoveDuplicates(mergeValue))
			}
		}
	}
}

func getLinker(rtype string, attr *Attribute) Linker {
	if attr != nil && !reflect.ValueOf(attr).IsNil() {
		if rtype == "user" {
			return attr.User
		} else if rtype == "group" {
			return attr.Group
		}
	}
	return nil
}

func recursiveSetLinkValue(attr Attributes, rtype, name string, result map[string][]string) {
	if result[name] != nil {
		return
	}

	linker := getLinker(rtype, attr[name])

	if linker != nil && !reflect.ValueOf(linker).IsNil() && len(linker.LinkValue()) > 0 {
		result[name] = linker.LinkValue()
		if linker.LinkParams() != nil || !reflect.ValueOf(linker.LinkParams()).IsNil() {
			for _, next_name := range linker.LinkParams() {
				recursiveSetLinkValue(attr, rtype, next_name, result)
			}
		}
	}
}

func member(n string, xs []string) bool {
	for _, x := range xs {
		if n == x {
			return true
		}
	}
	return false
}

func RemoveDuplicates(xs []string) []string {
	ys := make([]string, 0, len(xs))
	for _, x := range xs {
		if !member(x, ys) {
			ys = append(ys, x)
		}
	}
	return ys
}
<|endoftext|>


<commit_before><commit_msg>More ticme in the ticker, the interval was too short for real loads<commit_after><|endoftext|>


<commit_before>// Copyright 2018 The etcd Authors
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

// +build !cov

package e2e

import (
	"fmt"
	"testing"

	"github.com/coreos/etcd/version"
)

func TestV3CurlCipherSuitesValid(t *testing.T)    { testV3CurlCipherSuites(t, true) }
func TestV3CurlCipherSuitesMismatch(t *testing.T) { testV3CurlCipherSuites(t, false) }
func testV3CurlCipherSuites(t *testing.T, valid bool) {
	cc := configClientTLS
	cc.clusterSize = 1
	cc.cipherSuites = []string{
		"TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256",
		"TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384",
		"TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256",
		"TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384",
		"TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305",
		"TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305",
	}
	testFunc := cipherSuiteTestValid
	if !valid {
		testFunc = cipherSuiteTestMismatch
	}
	testCtl(t, testFunc, withCfg(cc))
}

func cipherSuiteTestValid(cx ctlCtx) {
	if err := cURLGet(cx.epc, cURLReq{
		endpoint:         "/metrics",
		expected:         fmt.Sprintf(`etcd_server_version{server_version="%s"} 1`, version.Version),
		metricsURLScheme: cx.cfg.metricsURLScheme,
		ciphers:          "ECDHE-RSA-AES128-GCM-SHA256", // TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256
	}); err != nil {
		cx.t.Fatalf("failed get with curl (%v)", err)
	}
}

func cipherSuiteTestMismatch(cx ctlCtx) {
	if err := cURLGet(cx.epc, cURLReq{
		endpoint:         "/metrics",
		expected:         "alert handshake failure",
		metricsURLScheme: cx.cfg.metricsURLScheme,
		ciphers:          "ECDHE-RSA-DES-CBC3-SHA", // TLS_ECDHE_RSA_WITH_3DES_EDE_CBC_SHA
	}); err != nil {
		cx.t.Fatalf("failed get with curl (%v)", err)
	}
}
<commit_msg>tests/e2e: do not run cipher suite tests for gRPC proxy<commit_after>// Copyright 2018 The etcd Authors
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

// +build !cov,!cluster_proxy

package e2e

import (
	"fmt"
	"testing"

	"github.com/coreos/etcd/version"
)

func TestV3CurlCipherSuitesValid(t *testing.T)    { testV3CurlCipherSuites(t, true) }
func TestV3CurlCipherSuitesMismatch(t *testing.T) { testV3CurlCipherSuites(t, false) }
func testV3CurlCipherSuites(t *testing.T, valid bool) {
	cc := configClientTLS
	cc.clusterSize = 1
	cc.cipherSuites = []string{
		"TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256",
		"TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384",
		"TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256",
		"TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384",
		"TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305",
		"TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305",
	}
	testFunc := cipherSuiteTestValid
	if !valid {
		testFunc = cipherSuiteTestMismatch
	}
	testCtl(t, testFunc, withCfg(cc))
}

func cipherSuiteTestValid(cx ctlCtx) {
	if err := cURLGet(cx.epc, cURLReq{
		endpoint:         "/metrics",
		expected:         fmt.Sprintf(`etcd_server_version{server_version="%s"} 1`, version.Version),
		metricsURLScheme: cx.cfg.metricsURLScheme,
		ciphers:          "ECDHE-RSA-AES128-GCM-SHA256", // TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256
	}); err != nil {
		cx.t.Fatalf("failed get with curl (%v)", err)
	}
}

func cipherSuiteTestMismatch(cx ctlCtx) {
	if err := cURLGet(cx.epc, cURLReq{
		endpoint:         "/metrics",
		expected:         "alert handshake failure",
		metricsURLScheme: cx.cfg.metricsURLScheme,
		ciphers:          "ECDHE-RSA-DES-CBC3-SHA", // TLS_ECDHE_RSA_WITH_3DES_EDE_CBC_SHA
	}); err != nil {
		cx.t.Fatalf("failed get with curl (%v)", err)
	}
}
<|endoftext|>


<commit_before>package values

import (
	"github.com/stretchr/testify/assert"
	"testing"
	"time"
	. "github.com/zubairhamed/go-commons/typeval"
)

func TestStringValue(t *testing.T) {
	val := String("this is a string")
	assert.Equal(t, VALUETYPE_STRING, val.GetType())
	assert.Equal(t, "this is a string", val.GetValue())
	assert.Equal(t, "this is a string", val.GetStringValue())
	assert.Equal(t, 16, len(val.GetBytes()))
}

func TestIntegerValue(t *testing.T) {
	val := Integer(42)
	assert.Equal(t, VALUETYPE_INTEGER, val.GetType())
	assert.Equal(t, 42, val.GetValue())
	assert.Equal(t, "42", val.GetStringValue())
	assert.Equal(t, 2, len(val.GetBytes()))
}

func TestTimeValue(t *testing.T) {
	tv := time.Unix(1433767779, 0)
	val := Time(tv)
	assert.Equal(t, VALUETYPE_TIME, val.GetType())
	assert.Equal(t, tv, val.GetValue())
	assert.Equal(t, "1433767779", val.GetStringValue())
	assert.Equal(t, 10, len(val.GetBytes()))
}

func TestFloatValue(t *testing.T) {
	val := Float(4.2)
	assert.Equal(t, VALUETYPE_FLOAT, val.GetType())
	assert.Equal(t, 4.2, val.GetValue())
	assert.Equal(t, "4", val.GetStringValue())
	assert.Equal(t, 8, len(val.GetBytes()))
}

func TestBooleanValue(t *testing.T) {
	val := Boolean(true)
	assert.Equal(t, VALUETYPE_BOOLEAN, val.GetType())
	assert.Equal(t, true, val.GetValue())
	assert.Equal(t, "1", val.GetStringValue())
	assert.Equal(t, 0, len(val.GetBytes()))
}

func TestEmptyValue(t *testing.T) {
	val := Empty()
	assert.Equal(t, VALUETYPE_EMPTY, val.GetType())
	assert.Equal(t, "", val.GetValue())
	assert.Equal(t, "", val.GetStringValue())
	assert.Equal(t, 0, len(val.GetBytes()))
}

/*
func TestTlvValue(t *testing.T) {
	val := Tlv([]byte{0, 1, 2})
	assert.Equal(t, VALUETYPE_TLV, val.GetType())
	assert.Equal(t, []byte{0, 1, 2}, val.GetValue())
	assert.Equal(t, "", val.GetStringValue())
	assert.Equal(t, 3, len(val.GetBytes()))
}

func TestMultipleResourceInstanceValue(t *testing.T) {

}
*/
<commit_msg>fixing values test<commit_after>package values

import (
	"github.com/stretchr/testify/assert"
	"testing"
	"time"
	. "github.com/zubairhamed/go-commons/typeval"
)

func TestStringValue(t *testing.T) {
	val := String("this is a string")
	assert.Equal(t, VALUETYPE_STRING, val.GetType())
	assert.Equal(t, "this is a string", val.GetValue())
	assert.Equal(t, "this is a string", val.GetStringValue())
	assert.Equal(t, 16, len(val.GetBytes()))
}

func TestIntegerValue(t *testing.T) {
	val := Integer(42)
	assert.Equal(t, VALUETYPE_INTEGER, val.GetType())
	assert.Equal(t, 42, val.GetValue())
	assert.Equal(t, "42", val.GetStringValue())
	assert.Equal(t, 2, len(val.GetBytes()))
}

func TestTimeValue(t *testing.T) {
	tv := time.Unix(1433767779, 0)
	val := Time(tv)
	assert.Equal(t, VALUETYPE_TIME, val.GetType())
	assert.Equal(t, tv, val.GetValue())
	assert.Equal(t, "1433767779", val.GetStringValue())
	assert.Equal(t, 10, len(val.GetBytes()))
}

func TestFloatValue(t *testing.T) {
	val := Float(float32(4.2))
	assert.Equal(t, VALUETYPE_FLOAT, val.GetType())
	assert.Equal(t, float32(4.2), val.GetValue())
	assert.Equal(t, "4", val.GetStringValue())
	assert.Equal(t, 4, len(val.GetBytes()))
}

func TestBooleanValue(t *testing.T) {
	val := Boolean(true)
	assert.Equal(t, VALUETYPE_BOOLEAN, val.GetType())
	assert.Equal(t, true, val.GetValue())
	assert.Equal(t, "1", val.GetStringValue())
	assert.Equal(t, 0, len(val.GetBytes()))
}

func TestEmptyValue(t *testing.T) {
	val := Empty()
	assert.Equal(t, VALUETYPE_EMPTY, val.GetType())
	assert.Equal(t, "", val.GetValue())
	assert.Equal(t, "", val.GetStringValue())
	assert.Equal(t, 0, len(val.GetBytes()))
}

/*
func TestTlvValue(t *testing.T) {
	val := Tlv([]byte{0, 1, 2})
	assert.Equal(t, VALUETYPE_TLV, val.GetType())
	assert.Equal(t, []byte{0, 1, 2}, val.GetValue())
	assert.Equal(t, "", val.GetStringValue())
	assert.Equal(t, 3, len(val.GetBytes()))
}

func TestMultipleResourceInstanceValue(t *testing.T) {

}
*/
<|endoftext|>


<commit_before>package facebook_test

import (
	"github.com/Lunchr/luncher-api/db/model"
	"github.com/Lunchr/luncher-api/facebook"
	"github.com/Lunchr/luncher-api/facebook/mocks"

	. "github.com/onsi/ginkgo"
	. "github.com/onsi/gomega"
)

var _ = Describe("Post", func() {
	var (
		facebookPost facebook.Post

		groupPosts *mocks.OfferGroupPosts
		offers     *mocks.Offers
		regions    *mocks.Regions
		fbAuth     *mocks.Authenticator

		user       *model.User
		restaurant *model.Restaurant
	)

	BeforeEach(func() {
		groupPosts = new(mocks.OfferGroupPosts)
		offers = new(mocks.Offers)
		regions = new(mocks.Regions)
		fbAuth = new(mocks.Authenticator)

		facebookPost = facebook.NewPost(groupPosts, offers, regions, fbAuth)
	})

	Describe("Update", func() {
		var date model.DateWithoutTime

		Context("for restaurants without an associated FB page", func() {
			BeforeEach(func() {
				restaurant = &model.Restaurant{
					FacebookPageID: "",
				}
			})

			It("does nothing", func() {
				err := facebookPost.Update(date, user, restaurant)
				Expect(err).NotTo(HaveOccurred())
			})
		})
	})
})
<commit_msg>Add first full test<commit_after>package facebook_test

import (
	"time"

	"github.com/Lunchr/luncher-api/db/model"
	"github.com/Lunchr/luncher-api/facebook"
	"github.com/Lunchr/luncher-api/facebook/mocks"
	fbmodel "github.com/deiwin/facebook/model"
	"golang.org/x/oauth2"
	"gopkg.in/mgo.v2/bson"

	. "github.com/onsi/ginkgo"
	. "github.com/onsi/gomega"
)

var _ = Describe("Post", func() {
	var (
		facebookPost facebook.Post

		groupPosts       *mocks.OfferGroupPosts
		offersCollection *mocks.Offers
		regions          *mocks.Regions
		fbAuth           *mocks.Authenticator

		user       *model.User
		restaurant *model.Restaurant
	)

	BeforeEach(func() {
		groupPosts = new(mocks.OfferGroupPosts)
		offersCollection = new(mocks.Offers)
		regions = new(mocks.Regions)
		fbAuth = new(mocks.Authenticator)

		facebookPost = facebook.NewPost(groupPosts, offersCollection, regions, fbAuth)
	})

	JustBeforeEach(func() {
	})

	Describe("Update", func() {
		var date model.DateWithoutTime

		Context("for restaurants without an associated FB page", func() {
			BeforeEach(func() {
				restaurant = &model.Restaurant{
					FacebookPageID: "",
				}
				date = model.DateWithoutTime("2011-04-24")
			})

			It("does nothing", func() {
				err := facebookPost.Update(date, user, restaurant)
				Expect(err).NotTo(HaveOccurred())
			})
		})

		Context("for a restaurant with an associated FB page", func() {
			var (
				restaurantID      bson.ObjectId
				facebookPageID    string
				facebookUserToken *oauth2.Token
				facebookPageToken string
				fbAPI             *mocks.API
			)

			BeforeEach(func() {
				restaurantID = bson.NewObjectId()
				facebookPageID = "a page ID"
				facebookUserToken = &oauth2.Token{
					AccessToken: "a user token",
				}
				facebookPageToken = "a page token"

				regionName := "a region"
				region := &model.Region{
					Name:     regionName,
					Location: "UTC",
				}
				regions.On("GetName", regionName).Return(region, nil)

				startTime := time.Date(2011, 04, 24, 0, 0, 0, 0, time.UTC)
				endTime := time.Date(2011, 04, 25, 0, 0, 0, 0, time.UTC)
				offers := []*model.Offer{
					&model.Offer{
						CommonOfferFields: model.CommonOfferFields{
							Title: "atitle",
							Price: 5.670000000000,
						},
					},
					&model.Offer{
						CommonOfferFields: model.CommonOfferFields{
							Title: "btitle",
							Price: 4.670000000000,
						},
					},
				}
				offersCollection.On("GetForRestaurantWithinTimeBounds", restaurantID, startTime, endTime).Return(offers, nil)

				restaurant = &model.Restaurant{
					ID:             restaurantID,
					FacebookPageID: facebookPageID,
					Region:         regionName,
				}
				user = &model.User{
					Session: &model.UserSession{
						FacebookUserToken: *facebookUserToken,
						FacebookPageToken: facebookPageToken,
					},
				}
				fbAPI = new(mocks.API)
				fbAuth.On("APIConnection", facebookUserToken).Return(fbAPI)
			})

			Context("with an existing OfferGroupPost", func() {
				var (
					messageTemplate string
					offerGroupPost  *model.OfferGroupPost
					facebookPostID  string
				)

				BeforeEach(func() {
					id := bson.NewObjectId()
					messageTemplate = "a message template"
					offerGroupPost = &model.OfferGroupPost{
						ID:              id,
						Date:            date,
						MessageTemplate: messageTemplate,
					}
					groupPosts.On("GetByDate", date, restaurantID).Return(offerGroupPost, nil)

					facebookPostID = "fb post id"
					fbAPI.On("PagePublish", facebookPageToken, facebookPageID,
						messageTemplate+"\n\natitle - 5.67€\nbtitle - 4.67€").Return(&fbmodel.Post{
						ID: facebookPostID,
					}, nil)
					groupPosts.On("UpdateByID", id, &model.OfferGroupPost{
						ID:              id,
						Date:            date,
						MessageTemplate: messageTemplate,
						FBPostID:        facebookPostID,
					}).Return(nil)
				})

				Context("without a previous associated FB post", func() {
					AfterEach(func() {
						groupPosts.AssertExpectations(GinkgoT())
						offersCollection.AssertExpectations(GinkgoT())
						regions.AssertExpectations(GinkgoT())
						fbAuth.AssertExpectations(GinkgoT())
					})

					It("should succeed", func() {
						err := facebookPost.Update(date, user, restaurant)
						Expect(err).To(BeNil())
					})
				})
			})
		})
	})
})
<|endoftext|>


<commit_before>package templates

var LoggedInHome = `
<!doctype html>
  <html lang="en">
  <head>
    {{template "header" . }}

    <link rel="stylesheet" href="/a/p/p/{{.Version}}/kd.css" />
    <link rel="stylesheet" href="/a/p/p/{{.Version}}/app.css" />
  </head>

  <body class='logged-in'>
    {{.UnsupportedHTML}}

    <script>
      var _globals = {
        config: {{.Runtime}},
        isLoggedInOnLoad: true,
        userId: {{.User.GetWithDefaultStr "UserId" }},
        userAccount: {{.User.GetWithDefaultHash "Account"  }},
        currentGroup: {{.User.GetWithDefaultHash "Group" }},
        userEnvironmentData: {{.User.GetWithDefaultHash "EnvData" }},
        socialApiData: {{.User.GetWithDefaultHash "SocialApiData" }}
      };
    </script>

    <script src="/a/p/p/{{.Version}}/thirdparty/pubnub.min.js"></script>
    <script src="/a/p/p/{{.Version}}/bundle.js"></script>
    <script>require('app')();</script>

    <script>
      (function(d) {
        var config = {
          kitId: 'rbd0tum',
          scriptTimeout: 3000
        },
        h=d.documentElement,t=setTimeout(function(){h.className=h.className.replace(/\bwf-loading\b/g,"")+" wf-inactive";},config.scriptTimeout),tk=d.createElement("script"),f=false,s=d.getElementsByTagName("script")[0],a;h.className+=" wf-loading";tk.src='//use.typekit.net/'+config.kitId+'.js';tk.async=true;tk.onload=tk.onreadystatechange=function(){a=this.readyState;if(f||a&&a!="complete"&&a!="loaded")return;f=true;clearTimeout(t);try{Typekit.load(config)}catch(e){}};s.parentNode.insertBefore(tk,s)
      })(document);
    </script>

    {{if not .Impersonating }}
      <script type="text/javascript">
        (function () {
        var _user_id = '{{.User.GetWithDefaultStr "Username" }}'; var _session_id = '{{.User.GetWithDefaultStr "SessionId" }}'; var _sift = _sift || []; _sift.push(['_setAccount', 'f270274999']); _sift.push(['_setUserId', _user_id]); _sift.push(['_setSessionId', _session_id]); _sift.push(['_trackPageview']); (function() { function ls() { var e = document.createElement('script'); e.type = 'text/javascript'; e.async = true; e.src = ('https:' == document.location.protocol ? 'https://' : 'http://') + 'cdn.siftscience.com/s.js'; var s = document.getElementsByTagName('script')[0]; s.parentNode.insertBefore(e, s); } if (window.attachEvent) { window.attachEvent('onload', ls); } else { window.addEventListener('load', ls, false); } })();
        })();
      </script>
    {{end}}

    <script>
      (function(h,o,t,j,a,r){
        h.hj=h.hj||function(){(h.hj.q=h.hj.q||[]).push(arguments)};
        h._hjSettings={hjid:156048,hjsv:5};
        a=o.getElementsByTagName('head')[0];
        r=o.createElement('script');r.async=1;
        r.src=t+h._hjSettings.hjid+j+h._hjSettings.hjsv;
        a.appendChild(r);
      })(window,document,'//static.hotjar.com/c/hotjar-','.js?sv=');
    </script>
</body>
</html>
`
<commit_msg>gowebserver: added userRoles and userPermissions<commit_after>package templates

var LoggedInHome = `
<!doctype html>
  <html lang="en">
  <head>
    {{template "header" . }}

    <link rel="stylesheet" href="/a/p/p/{{.Version}}/kd.css" />
    <link rel="stylesheet" href="/a/p/p/{{.Version}}/app.css" />
  </head>

  <body class='logged-in'>
    {{.UnsupportedHTML}}

    <script>
      var _globals = {
        config: {{.Runtime}},
        isLoggedInOnLoad: true,
        userId: {{.User.GetWithDefaultStr "UserId" }},
        userAccount: {{.User.GetWithDefaultHash "Account"  }},
        currentGroup: {{.User.GetWithDefaultHash "Group" }},
        userEnvironmentData: {{.User.GetWithDefaultHash "EnvData" }},
        socialApiData: {{.User.GetWithDefaultHash "SocialApiData" }},
        userRoles: {{.User.GetWithDefaultHash "Roles" }}
        userPermissions: {{.User.GetWithDefaultHash "Permissions" }}
      };
    </script>

    <script src="/a/p/p/{{.Version}}/thirdparty/pubnub.min.js"></script>
    <script src="/a/p/p/{{.Version}}/bundle.js"></script>
    <script>require('app')();</script>

    <script>
      (function(d) {
        var config = {
          kitId: 'rbd0tum',
          scriptTimeout: 3000
        },
        h=d.documentElement,t=setTimeout(function(){h.className=h.className.replace(/\bwf-loading\b/g,"")+" wf-inactive";},config.scriptTimeout),tk=d.createElement("script"),f=false,s=d.getElementsByTagName("script")[0],a;h.className+=" wf-loading";tk.src='//use.typekit.net/'+config.kitId+'.js';tk.async=true;tk.onload=tk.onreadystatechange=function(){a=this.readyState;if(f||a&&a!="complete"&&a!="loaded")return;f=true;clearTimeout(t);try{Typekit.load(config)}catch(e){}};s.parentNode.insertBefore(tk,s)
      })(document);
    </script>

    {{if not .Impersonating }}
      <script type="text/javascript">
        (function () {
        var _user_id = '{{.User.GetWithDefaultStr "Username" }}'; var _session_id = '{{.User.GetWithDefaultStr "SessionId" }}'; var _sift = _sift || []; _sift.push(['_setAccount', 'f270274999']); _sift.push(['_setUserId', _user_id]); _sift.push(['_setSessionId', _session_id]); _sift.push(['_trackPageview']); (function() { function ls() { var e = document.createElement('script'); e.type = 'text/javascript'; e.async = true; e.src = ('https:' == document.location.protocol ? 'https://' : 'http://') + 'cdn.siftscience.com/s.js'; var s = document.getElementsByTagName('script')[0]; s.parentNode.insertBefore(e, s); } if (window.attachEvent) { window.attachEvent('onload', ls); } else { window.addEventListener('load', ls, false); } })();
        })();
      </script>
    {{end}}

    <script>
      (function(h,o,t,j,a,r){
        h.hj=h.hj||function(){(h.hj.q=h.hj.q||[]).push(arguments)};
        h._hjSettings={hjid:156048,hjsv:5};
        a=o.getElementsByTagName('head')[0];
        r=o.createElement('script');r.async=1;
        r.src=t+h._hjSettings.hjid+j+h._hjSettings.hjsv;
        a.appendChild(r);
      })(window,document,'//static.hotjar.com/c/hotjar-','.js?sv=');
    </script>
</body>
</html>
`
<|endoftext|>


<commit_before>package functions

import (
	"fmt"
	"math/rand"
	"strconv"
	"time"
)

func defaults(robot Robot) {
	robot.HandleFunc(robot.Direct("ping"), func(s State) {
		s.Chat().Send(s.Message().ChannelID(), "pong!")
	})

	robot.HandleFunc(robot.Direct("roll(\\s(\\d+))?"), func(s State) {
		defer recover()

		bound := 100
		val := s.Params()[1]

		if val != "" {
			var err error
			bound, err = strconv.Atoi(val)

			if err != nil {
				return
			}
		}

		rand.Seed(time.Now().UTC().UnixNano())
		random := rand.Intn(bound)

		msg := fmt.Sprintf("%s rolled a %d of %d", s.Message().UserName(), random, bound)
		s.Chat().Send(s.Message().ChannelID(), msg)
	})
}
<commit_msg>Fixed defaults example<commit_after>package functions

import (
	"fmt"
	"math/rand"
	"strconv"
	"time"

	"github.com/brettbuddin/victor"
)

func Defaults(robot victor.Robot) {
	robot.HandleFunc(robot.Direct("ping"), func(s victor.State) {
		s.Chat().Send(s.Message().ChannelID(), "pong!")
	})

	robot.HandleFunc(robot.Direct("roll(\\s(\\d+))?"), func(s victor.State) {
		defer recover()

		bound := 100
		val := s.Params()[1]

		if val != "" {
			var err error
			bound, err = strconv.Atoi(val)

			if err != nil {
				return
			}
		}

		rand.Seed(time.Now().UTC().UnixNano())
		random := rand.Intn(bound)

		msg := fmt.Sprintf("%s rolled a %d of %d", s.Message().UserName(), random, bound)
		s.Chat().Send(s.Message().ChannelID(), msg)
	})
}
<|endoftext|>


<commit_before>/*
 Copyright 2016 Padduck, LLC

 Licensed under the Apache License, Version 2.0 (the "License");
 you may not use this file except in compliance with the License.
 You may obtain a copy of the License at

 	http://www.apache.org/licenses/LICENSE-2.0

 Unless required by applicable law or agreed to in writing, software
 distributed under the License is distributed on an "AS IS" BASIS,
 WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 See the License for the specific language governing permissions and
 limitations under the License.
*/

package templates

const SRCDS = `{
  "pufferd": {
    "type": "srcds",
    "install": {
      "commands": [
        {
          "files": "https://steamcdn-a.akamaihd.net/client/installer/steamcmd_linux.tar.gz",
          "type": "download"
        },
        {
          "commands": [
            "mkdir steamcmd",
            "tar --no-same-owner -xzvf steamcmd_linux.tar.gz -C steamcmd",
            "steamcmd/steamcmd.sh +login anonymous +force_install_dir ${rootdir} +app_update ${appid} +quit",
            "mkdir -p .steam/sdk32",
            "cp steamcmd/linux32/steamclient.so .steam/sdk32/steamclient.so"
          ],
          "type": "command"
        }
      ]
    },
    "run": {
      "stop": "exit",
      "pre": [],
      "post": [],
      "arguments": [
      	"+ip",
      	"${ip}",
      	"+port",
      	"${port}",
      	"-game ${gametype}",
      	"-console",
        "+map ${map}",
      	"-norestart"
      ],
      "program": "./srcds_run"
    },
    "data": {
      "appid": {
        "value": "232250",
        "required": true,
        "desc": "App ID",
        "display": "Application ID",
        "internal": false
      },
      "gametype": {
        "value": "tf",
        "required": true,
        "desc": "Game Type",
        "display": "tf, csgo, etc.",
        "internal": false
      },
      "map": {
      	"value": "ctf_2fort",
      	"required": false,
      	"desc": "Map",
      	"display": "Map to load",
      	"internal": false
      },
      "ip": {
        "value": "0.0.0.0",
        "required": true,
        "desc": "What IP to bind the server to",
        "display": "IP",
        "internal": false
      },
      "port": {
        "value": "27015",
        "required": true,
        "desc": "What port to bind the server to",
        "display": "Port",
        "internal": false
      }
    }
  }
}`

const TF2 = `{
  "pufferd": {
    "type": "srcds",
    "display": "Team Fortress 2",
    "install": {
      "commands": [
        {
          "files": "https://steamcdn-a.akamaihd.net/client/installer/steamcmd_linux.tar.gz",
          "type": "download"
        },
        {
          "commands": [
            "mkdir steamcmd",
            "tar --no-same-owner -xzvf steamcmd_linux.tar.gz -C steamcmd",
            "steamcmd/steamcmd.sh +login anonymous +force_install_dir ${rootdir} +app_update 232250 +quit",
            "mkdir -p .steam/sdk32",
            "cp steamcmd/linux32/steamclient.so .steam/sdk32/steamclient.so"
          ],
          "type": "command"
        }
      ]
    },
    "run": {
      "stop": "exit",
      "pre": [],
      "post": [],
      "arguments": [
      	"+ip",
      	"${ip}",
      	"+port",
      	"${port}",
      	"-game tf",
      	"-console",
        "+map ${map}",
      	"-norestart"
      ],
      "program": "./srcds_run"
    },
    "environment": {
      "type": "tty"
    },
    "data": {
      "map": {
      	"value": "ctf_2fort",
      	"required": true,
      	"desc": "TF2 Map",
      	"display": "Team Fortess 2 Map to load",
      	"internal": false
      },
      "ip": {
        "value": "0.0.0.0",
        "required": true,
        "desc": "What IP to bind the server to",
        "display": "IP",
        "internal": false
      },
      "port": {
        "value": "27015",
        "required": true,
        "desc": "What port to bind the server to",
        "display": "Port",
        "internal": false
      }
    }
  }
}`
<commit_msg>Update SRCDS template<commit_after>/*
 Copyright 2016 Padduck, LLC

 Licensed under the Apache License, Version 2.0 (the "License");
 you may not use this file except in compliance with the License.
 You may obtain a copy of the License at

 	http://www.apache.org/licenses/LICENSE-2.0

 Unless required by applicable law or agreed to in writing, software
 distributed under the License is distributed on an "AS IS" BASIS,
 WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 See the License for the specific language governing permissions and
 limitations under the License.
*/

package templates

const SRCDS = `{
  "pufferd": {
    "type": "srcds",
    "install": {
      "commands": [
        {
          "files": "https://steamcdn-a.akamaihd.net/client/installer/steamcmd_linux.tar.gz",
          "type": "download"
        },
        {
          "commands": [
            "mkdir steamcmd",
            "tar --no-same-owner -xzvf steamcmd_linux.tar.gz -C steamcmd",
            "steamcmd/steamcmd.sh +login anonymous +force_install_dir ${rootdir} +app_update ${appid} +quit",
            "mkdir -p .steam/sdk32",
            "cp steamcmd/linux32/steamclient.so .steam/sdk32/steamclient.so"
          ],
          "type": "command"
        }
      ]
    },
    "run": {
      "stop": "exit",
      "pre": [],
      "post": [],
      "arguments": [
      	"+ip",
      	"${ip}",
      	"+port",
      	"${port}",
      	"-game ${gametype}",
      	"-console",
        "+map ${map}",
      	"-norestart"
      ],
      "program": "./srcds_run"
    },
    "environment": {
      "type": "tty"
    },
    "data": {
      "appid": {
        "value": "232250",
        "required": true,
        "desc": "App ID",
        "display": "Application ID",
        "internal": false
      },
      "gametype": {
        "value": "tf",
        "required": false,
        "desc": "Game Type",
        "display": "tf, csgo, etc.",
        "internal": false
      },
      "map": {
      	"value": "ctf_2fort",
      	"required": false,
      	"desc": "Map",
      	"display": "Map to load",
      	"internal": false
      },
      "ip": {
        "value": "0.0.0.0",
        "required": true,
        "desc": "What IP to bind the server to",
        "display": "IP",
        "internal": false
      },
      "port": {
        "value": "27015",
        "required": true,
        "desc": "What port to bind the server to",
        "display": "Port",
        "internal": false
      }
    }
  }
}`

const TF2 = `{
  "pufferd": {
    "type": "srcds",
    "display": "Team Fortress 2",
    "install": {
      "commands": [
        {
          "files": "https://steamcdn-a.akamaihd.net/client/installer/steamcmd_linux.tar.gz",
          "type": "download"
        },
        {
          "commands": [
            "mkdir steamcmd",
            "tar --no-same-owner -xzvf steamcmd_linux.tar.gz -C steamcmd",
            "steamcmd/steamcmd.sh +login anonymous +force_install_dir ${rootdir} +app_update 232250 +quit",
            "mkdir -p .steam/sdk32",
            "cp steamcmd/linux32/steamclient.so .steam/sdk32/steamclient.so"
          ],
          "type": "command"
        }
      ]
    },
    "run": {
      "stop": "exit",
      "pre": [],
      "post": [],
      "arguments": [
      	"+ip",
      	"${ip}",
      	"+port",
      	"${port}",
      	"-game tf",
      	"-console",
        "+map ${map}",
      	"-norestart"
      ],
      "program": "./srcds_run"
    },
    "environment": {
      "type": "tty"
    },
    "data": {
      "map": {
      	"value": "ctf_2fort",
      	"required": true,
      	"desc": "TF2 Map",
      	"display": "Team Fortess 2 Map to load",
      	"internal": false
      },
      "ip": {
        "value": "0.0.0.0",
        "required": true,
        "desc": "What IP to bind the server to",
        "display": "IP",
        "internal": false
      },
      "port": {
        "value": "27015",
        "required": true,
        "desc": "What port to bind the server to",
        "display": "Port",
        "internal": false
      }
    }
  }
}`
<|endoftext|>