<commit_before>package utils

import (
	"fmt"
	"strconv"
	"time"
)

//
// Calculations involving schedule hours and start-of-day.
//

// The hour of the day (local time) at which the scheduled day begins.
const startHour = 6

// StartOfDayOn gets the schedule start-of-day on a given date.
// This is in terms of startHour.
func StartOfDayOn(date time.Time) time.Time {
	y, m, d := date.Date()
	return time.Date(y, m, d, startHour, 0, 0, 0, time.Local)
}

// StartOffset is the type of offsets from the start hour of a schedule.
type StartOffset int

// Valid returns whether a StartOffset is within the 0-23 range required for it to index a schedule hour.
func (h StartOffset) Valid() bool {
	return 0 <= h && h <= 23
}

// ToHour takes a number of hours h since the last day start (0-23) and gives the actual hour.
// It returns an error if the hour is invalid.
func (h StartOffset) ToHour() (int, error) {
	if 23 < h || h < 0 {
		return 0, fmt.Errorf("StartOffset.ToHour: offset %d not between 0 and 23", h)
	}
	return (int(h) + startHour) % 24, nil
}

// HourToStartOffset takes an hour (0-23) and gives the number of hours elapsed since the last day start.
// It returns an error if the hour is invalid.
func HourToStartOffset(hour int) (StartOffset, error) {
	if 23 < hour || hour < 0 {
		return 0, fmt.Errorf("HourToStartOffset: hour %d not between 0 and 23", hour)
	}
	// Adding 24 to ensure we don't go negative.  Negative modulo is scary.
	return StartOffset(((hour + 24) - startHour) % 24), nil
}

//
// Conversions from ISO year-week to more amenable formats.
//

// ParseIsoWeek parses an ISO weekday from year, week, and weekday strings.
// It performs bounds checking.
// weekday must be an integer from 1 (Monday) to 7 (Sunday).
func ParseIsoWeek(isoyear, isoweek, isoweekday string) (year int, week int, weekday time.Weekday, err error) {
	if year, err = strconv.Atoi(isoyear); err != nil {
		return
	}
	if year < 0 {
		err = fmt.Errorf("Invalid year: %d", year)
		return
	}

	if week, err = strconv.Atoi(isoweek); err != nil {
		return
	}
	if week < 1 || 53 < week {
		err = fmt.Errorf("Invalid week: %d", week)
		return
	}

	// Two-stage conversion: first to int, then to Weekday.
	// Go treats Sunday as day 0: we must correct this grave mistake.
	var di int
	if di, err = strconv.Atoi(isoweekday); err != nil {
		return
	}
	if di < 1 || 7 < di {
		err = fmt.Errorf("Invalid day: %d", di)
		return
	}

	if di == 7 {
		weekday = time.Sunday
	} else {
		weekday = time.Weekday(di)
	}

	return
}

// IsoWeekToDate interprets year, week, and weekday strings as an ISO weekday.
// The time is set to local midnight.
func IsoWeekToDate(year, week int, weekday time.Weekday) (time.Time, error) {
	// This is based on the calculation given at:
	// https://en.wikipedia.org/wiki/ISO_week_date#Calculating_a_date_given_the_year.2C_week_number_and_weekday

	// We need to find the first week in the year.
	// This always contains the 4th of January, so find that, and get
	// ISOWeek on it.
	fj := time.Date(year, time.January, 4, 0, 0, 0, 0, time.Local)

	// Correct Go's stupid Sunday is 0 decision, making the weekdays ISO 8601 compliant
	intWeekday := int(weekday)
	if intWeekday == 0 {
		intWeekday = 7
	}
	fjWeekday := int(fj.Weekday())
	if fjWeekday == 0 {
		fjWeekday = 7
	}

	// Sanity check to make sure time (and our intuition) is still working.
	fjYear, fjWeek := fj.ISOWeek()
	if fjYear != year {
		return time.Time{}, fmt.Errorf("ISO weekday year %d != calendar year %d", fjYear, year)
	}
	if fjWeek != 1 {
		return time.Time{}, fmt.Errorf("ISO weekday week of 4 Jan (%d) not week 1", fjWeek)
	}

	// The ISO 8601 ordinal date, which may belong to the next or previous
	// year.
	ord := (week * 7) + intWeekday - (fjWeekday + 3)

	// The ordinal date is just the number of days since 1 Jan y plus one,
	// so calculate the year from that.
	oj := time.Date(year, time.January, 1, 0, 0, 0, 0, time.Local)
	return oj.AddDate(0, 0, ord-1), nil
}

// MostRecentMonday returns the most recent Monday before d.
func MostRecentMonday(d time.Time) time.Time {
	/* The weekday is the number of days since the most recent Sunday, so
	   shifting it by 1 modulo 7 gives us the correct result for Monday. */
	dmon := int(d.Weekday()) - 1
	if dmon < 0 {
		// Correct for Sunday
		dmon = 6
	}

	return d.AddDate(0, 0, -dmon)
}

// FormatWeekRelative pretty-prints the name of a week starting on start.
// start must be a Monday.
func FormatWeekRelative(start time.Time) string {
	/* If we're on the same week, or the week either end of current, we can (and
	   should) use short, human-friendly week names. */

	// To work out which week we're in, get the boundaries of last, this, and next week.
	tm := MostRecentMonday(time.Now())
	lm := tm.AddDate(0, 0, -7)
	nm := tm.AddDate(0, 0, 7)
	fm := tm.AddDate(0, 0, 14)

	switch {
	case start.Before(lm):
		break
	case start.Before(tm):
		return "last week"
	case start.Before(nm):
		return "this week"
	case start.Before(fm):
		return "next week"
	default:
		break
	}

	// If we got here, we can't give a fancy name to this week.
	sun := start.AddDate(0, 0, 6)
	return start.Format("02 Jan 2006") + " to " + sun.Format("02 Jan 2006")
}
<commit_msg>Split FormatWeekRelative<commit_after>package utils

import (
	"fmt"
	"strconv"
	"time"
)

//
// Calculations involving schedule hours and start-of-day.
//

// The hour of the day (local time) at which the scheduled day begins.
const startHour = 6

// StartOfDayOn gets the schedule start-of-day on a given date.
// This is in terms of startHour.
func StartOfDayOn(date time.Time) time.Time {
	y, m, d := date.Date()
	return time.Date(y, m, d, startHour, 0, 0, 0, time.Local)
}

// StartOffset is the type of offsets from the start hour of a schedule.
type StartOffset int

// Valid returns whether a StartOffset is within the 0-23 range required for it to index a schedule hour.
func (h StartOffset) Valid() bool {
	return 0 <= h && h <= 23
}

// ToHour takes a number of hours h since the last day start (0-23) and gives the actual hour.
// It returns an error if the hour is invalid.
func (h StartOffset) ToHour() (int, error) {
	if 23 < h || h < 0 {
		return 0, fmt.Errorf("StartOffset.ToHour: offset %d not between 0 and 23", h)
	}
	return (int(h) + startHour) % 24, nil
}

// HourToStartOffset takes an hour (0-23) and gives the number of hours elapsed since the last day start.
// It returns an error if the hour is invalid.
func HourToStartOffset(hour int) (StartOffset, error) {
	if 23 < hour || hour < 0 {
		return 0, fmt.Errorf("HourToStartOffset: hour %d not between 0 and 23", hour)
	}
	// Adding 24 to ensure we don't go negative.  Negative modulo is scary.
	return StartOffset(((hour + 24) - startHour) % 24), nil
}

//
// Conversions from ISO year-week to more amenable formats.
//

// ParseIsoWeek parses an ISO weekday from year, week, and weekday strings.
// It performs bounds checking.
// weekday must be an integer from 1 (Monday) to 7 (Sunday).
func ParseIsoWeek(isoyear, isoweek, isoweekday string) (year int, week int, weekday time.Weekday, err error) {
	if year, err = strconv.Atoi(isoyear); err != nil {
		return
	}
	if year < 0 {
		err = fmt.Errorf("Invalid year: %d", year)
		return
	}

	if week, err = strconv.Atoi(isoweek); err != nil {
		return
	}
	if week < 1 || 53 < week {
		err = fmt.Errorf("Invalid week: %d", week)
		return
	}

	// Two-stage conversion: first to int, then to Weekday.
	// Go treats Sunday as day 0: we must correct this grave mistake.
	var di int
	if di, err = strconv.Atoi(isoweekday); err != nil {
		return
	}
	if di < 1 || 7 < di {
		err = fmt.Errorf("Invalid day: %d", di)
		return
	}

	if di == 7 {
		weekday = time.Sunday
	} else {
		weekday = time.Weekday(di)
	}

	return
}

// IsoWeekToDate interprets year, week, and weekday strings as an ISO weekday.
// The time is set to local midnight.
func IsoWeekToDate(year, week int, weekday time.Weekday) (time.Time, error) {
	// This is based on the calculation given at:
	// https://en.wikipedia.org/wiki/ISO_week_date#Calculating_a_date_given_the_year.2C_week_number_and_weekday

	// We need to find the first week in the year.
	// This always contains the 4th of January, so find that, and get
	// ISOWeek on it.
	fj := time.Date(year, time.January, 4, 0, 0, 0, 0, time.Local)

	// Correct Go's stupid Sunday is 0 decision, making the weekdays ISO 8601 compliant
	intWeekday := int(weekday)
	if intWeekday == 0 {
		intWeekday = 7
	}
	fjWeekday := int(fj.Weekday())
	if fjWeekday == 0 {
		fjWeekday = 7
	}

	// Sanity check to make sure time (and our intuition) is still working.
	fjYear, fjWeek := fj.ISOWeek()
	if fjYear != year {
		return time.Time{}, fmt.Errorf("ISO weekday year %d != calendar year %d", fjYear, year)
	}
	if fjWeek != 1 {
		return time.Time{}, fmt.Errorf("ISO weekday week of 4 Jan (%d) not week 1", fjWeek)
	}

	// The ISO 8601 ordinal date, which may belong to the next or previous
	// year.
	ord := (week * 7) + intWeekday - (fjWeekday + 3)

	// The ordinal date is just the number of days since 1 Jan y plus one,
	// so calculate the year from that.
	oj := time.Date(year, time.January, 1, 0, 0, 0, 0, time.Local)
	return oj.AddDate(0, 0, ord-1), nil
}

// MostRecentMonday returns the most recent Monday before d.
func MostRecentMonday(d time.Time) time.Time {
	/* The weekday is the number of days since the most recent Sunday, so
	   shifting it by 1 modulo 7 gives us the correct result for Monday. */
	dmon := int(d.Weekday()) - 1
	if dmon < 0 {
		// Correct for Sunday
		dmon = 6
	}

	return d.AddDate(0, 0, -dmon)
}

// FormatWeekRelative pretty-prints the name of a week starting on start, relative to today.
// start must be a Monday.
func FormatWeekRelative(start time.Time) string {
	return FormatWeekRelativeTo(start, MostRecentMonday(time.Now()))
}

// FormatWeekRelativeTo pretty-prints the name of a week starting on start, relative to the week starting on now.
// start and now must be a Monday.
func FormatWeekRelativeTo(start, now time.Time) string {
	/* If we're on the same week, or the week either end of current, we can (and
	   should) use short, human-friendly week names. */

	// To work out which week we're in, get the boundaries of last, this, and next week.
	lm := now.AddDate(0, 0, -7)
	nm := now.AddDate(0, 0, 7)
	fm := now.AddDate(0, 0, 14)

	switch {
	case start.Before(lm):
		break
	case start.Before(now):
		return "last week"
	case start.Before(nm):
		return "this week"
	case start.Before(fm):
		return "next week"
	default:
		break
	}

	// If we got here, we can't give a fancy name to this week.
	sun := start.AddDate(0, 0, 6)
	return start.Format("02 Jan 2006") + " to " + sun.Format("02 Jan 2006")
}
<|endoftext|>


<commit_before>package sarama

type MetadataRequest struct {
	Version                int16
	Topics                 []string
	AllowAutoTopicCreation bool
}

func (r *MetadataRequest) encode(pe packetEncoder) error {
	if r.Version < 0 || r.Version > 5 {
		return PacketEncodingError{"invalid or unsupported MetadataRequest version field"}
	}
	if r.Version == 0 || r.Topics != nil || len(r.Topics) > 0 {
		err := pe.putArrayLength(len(r.Topics))
		if err != nil {
			return err
		}

		for i := range r.Topics {
			err = pe.putString(r.Topics[i])
			if err != nil {
				return err
			}
		}
	} else {
		pe.putInt32(-1)
	}
	if r.Version > 3 {
		pe.putBool(r.AllowAutoTopicCreation)
	}
	return nil
}

func (r *MetadataRequest) decode(pd packetDecoder, version int16) error {
	r.Version = version
	size, err := pd.getInt32()
	if err != nil {
		return err
	}
	if size < 0 {
		return nil
	} else {
		topicCount := size
		if topicCount == 0 {
			return nil
		}

		r.Topics = make([]string, topicCount)
		for i := range r.Topics {
			topic, err := pd.getString()
			if err != nil {
				return err
			}
			r.Topics[i] = topic
		}
	}
	if r.Version > 3 {
		autoCreation, err := pd.getBool()
		if err != nil {
			return err
		}
		r.AllowAutoTopicCreation = autoCreation
	}
	return nil
}

func (r *MetadataRequest) key() int16 {
	return 3
}

func (r *MetadataRequest) version() int16 {
	return r.Version
}

func (r *MetadataRequest) requiredVersion() KafkaVersion {
	switch r.Version {
	case 1:
		return V0_10_0_0
	case 2:
		return V0_10_1_0
	case 3, 4:
		return V0_11_0_0
	case 5:
		return V1_0_0_0
	default:
		return MinVersion
	}
}
<commit_msg>Fix metadata requests on later versions<commit_after>package sarama

type MetadataRequest struct {
	Version                int16
	Topics                 []string
	AllowAutoTopicCreation bool
}

func (r *MetadataRequest) encode(pe packetEncoder) error {
	if r.Version < 0 || r.Version > 5 {
		return PacketEncodingError{"invalid or unsupported MetadataRequest version field"}
	}
	if r.Version == 0 || len(r.Topics) > 0 {
		err := pe.putArrayLength(len(r.Topics))
		if err != nil {
			return err
		}

		for i := range r.Topics {
			err = pe.putString(r.Topics[i])
			if err != nil {
				return err
			}
		}
	} else {
		pe.putInt32(-1)
	}
	if r.Version > 3 {
		pe.putBool(r.AllowAutoTopicCreation)
	}
	return nil
}

func (r *MetadataRequest) decode(pd packetDecoder, version int16) error {
	r.Version = version
	size, err := pd.getInt32()
	if err != nil {
		return err
	}
	if size < 0 {
		return nil
	} else {
		topicCount := size
		if topicCount == 0 {
			return nil
		}

		r.Topics = make([]string, topicCount)
		for i := range r.Topics {
			topic, err := pd.getString()
			if err != nil {
				return err
			}
			r.Topics[i] = topic
		}
	}
	if r.Version > 3 {
		autoCreation, err := pd.getBool()
		if err != nil {
			return err
		}
		r.AllowAutoTopicCreation = autoCreation
	}
	return nil
}

func (r *MetadataRequest) key() int16 {
	return 3
}

func (r *MetadataRequest) version() int16 {
	return r.Version
}

func (r *MetadataRequest) requiredVersion() KafkaVersion {
	switch r.Version {
	case 1:
		return V0_10_0_0
	case 2:
		return V0_10_1_0
	case 3, 4:
		return V0_11_0_0
	case 5:
		return V1_0_0_0
	default:
		return MinVersion
	}
}
<|endoftext|>


<commit_before>package datadog

import (
	"time"

	"github.com/DataDog/datadog-go/statsd"
)

const (
	Error   = "error"
	Info    = "info"
	Success = "success"
)

// EventCreator lol
type EventCreator func(title, text string) *statsd.Event

// TrackTranscationArgs lol
type TrackTranscationArgs struct {
	Tags            []string
	MetricName      string
	Client          *statsd.Client
	StartTime       time.Time
	AlertType       string
	EventInfo       []string
	CreateEvent     EventCreator
	CustomEventName string
}
<commit_msg>added comments<commit_after>package datadog

import (
	"time"

	"github.com/DataDog/datadog-go/statsd"
)

// Constants used to references against statsd's unexported
// AlertType.
const (
	Error   = "error"
	Info    = "info"
	Success = "success"
)

// EventCreator is a interface for passing a statsd DataDog NewEvent
// and responsible - Google Search for creating new metric events.
type EventCreator func(title, text string) *statsd.Event

// TrackTranscationArgs is the args structs for tracking transactions
// to DataDog.
type TrackTranscationArgs struct {
	Tags            []string
	MetricName      string
	Client          *statsd.Client
	StartTime       time.Time
	AlertType       string
	EventInfo       []string
	CreateEvent     EventCreator
	CustomEventName string
}
<|endoftext|>


<commit_before>package filehost

import (
	"context"
	"html/template"
	"io"
	"net/http"
	"os"
	"path/filepath"
	"strconv"
	"strings"
	"time"

	"github.com/pressly/chi"
	"github.com/sirupsen/logrus"
)

var log logrus.FieldLogger = logrus.StandardLogger()
var cfg *Config
var templ *template.Template

// Database is some sort of database that stores FileInfo
type Database interface {
	// GetFileInfo returns the FileInfo associated with the given
	// File Name, it returns nil if the file was not found
	GetFileInfo(string) *FileInfo
	SaveFileInfo(*FileInfo)
}

// Config contains the needed information to call New
type Config struct {
	AllowedMimeTypes []string
	BasePath         string
	BaseURL          string
	UploadPage       bool
	ExposedPassword  string
	NewFileName      func() string
	DB               Database
	Authenticate     func(*http.Request) bool
	AllowFileName    func(*http.Request) bool
	Logger           logrus.FieldLogger
}

// FileInfo contains additional information about the File
type FileInfo struct {
	Name     string // filename without extension
	Path     string
	MimeType string
	Uploader interface{} // information about the person who uploaded it
	Time     time.Time
	Expire   time.Duration
	Clicks   int
}

// New initializes a http Handler and returns it
func New(conf *Config) http.Handler {
	go ratelimit()
	cfg = conf
	log = cfg.Logger
	err := os.MkdirAll(cfg.BasePath, 644)
	if err != nil {
		log.Fatal(err)
	}
	r := chi.NewRouter()
	r.Use(func(next http.Handler) http.Handler {
		return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
			r = r.WithContext(context.WithValue(r.Context(), "logger",
				log.WithFields(logrus.Fields{
					"ip":         r.RemoteAddr,
					"user-agent": r.UserAgent(),
				})))
			next.ServeHTTP(w, r)
		})
	})
	r.Post("/upload", upload)
	r.Get("/:file", serveFile)
	if cfg.UploadPage {
		templ = template.Must(template.ParseFiles("upload.html"))
		r.Get("/", uploadPage)
	} else {
		r.Get("/", http.NotFound)
	}

	return r
}

func uploadPage(w http.ResponseWriter, r *http.Request) {
	data := struct {
		Key     string
		BaseURL string
	}{
		Key:     cfg.ExposedPassword,
		BaseURL: cfg.BaseURL,
	}
	if err := templ.Execute(w, data); err != nil {
		r.Context().Value("logger").(logrus.FieldLogger).
			WithError(err).Error("cannot execute template")
	}
}

func upload(w http.ResponseWriter, r *http.Request) {
	l := r.Context().Value("logger").(logrus.FieldLogger)
	defer r.Body.Close()
	if !cfg.Authenticate(r) {
		http.Error(w, "Not Authenticated", http.StatusUnauthorized)
		return
	}
	if ratelimited(r.RemoteAddr) {
		l.Warning("ratelimited")
		http.Error(w, "Rate Limit Exceeded", 429)
		return
	}
	l.Debug(r.Header)
	err := r.ParseMultipartForm(1024 * 1024 * 64)
	if err != nil {
		l.WithError(err).Error("cannot read multi part form")
		http.Error(w, "Bad Request", http.StatusBadRequest)
		return
	}
	if r.MultipartForm == nil {
		http.Error(w, "Bad Request", http.StatusBadRequest)
		return
	}
	files := r.MultipartForm.File
	l.Debug(files)
	if len(files) < 1 {
		http.Error(w, "Bad Request", http.StatusBadRequest)
		return
	}
	for _, headers := range files {
		for _, h := range headers {
			file, err := h.Open()
			if err != nil {
				l.Error(err)
			}
			name := h.Filename
			if !cfg.AllowFileName(r) || name == "" {
				if cfg.NewFileName != nil {
					name = cfg.NewFileName()
				} else {
					name = RandString(5)
				}
			}
			l = l.WithField("file", name)
			l.Info("uploading...")
			l.Debug(r.Header)
			l.Debug(h.Header)
			mimeType := h.Header.Get("Content-Type")
			if mimeType == octetStream {
				mimeType = "text/plain"
			}
			l = l.WithField("mime-type", mimeType)
			if !whiteListed(cfg.AllowedMimeTypes, mimeType) {
				l.Warning("mime type not allowed")
				http.Error(w, "Unsupported Media Type", 415)
				return
			}
			extension := ExtensionFromMime(mimeType)
			if extension != "" {
				extension = "." + extension
			}

			fullName := name + extension
			dstPath := filepath.Join(cfg.BasePath, fullName)
			// TODO: check if file exists
			dst, err := os.Create(dstPath)
			if err != nil {
				l.WithError(err).Error("cannot create file")
				http.Error(w, "Internal Server Error", 500)
				return
			}
			_, err = io.Copy(dst, file)
			if err != nil {
				l.WithError(err).Error("failed to save file")
				http.Error(w, "Internal Server Error", 500)
				return
			}
			w.Write([]byte(cfg.BaseURL + fullName))
			l.Info("uploaded to ", cfg.BaseURL+fullName)
			if cfg.DB != nil {
				info := &FileInfo{
					Name: name,
					Path: dstPath,
					Uploader: map[string]interface{}{
						"ip":         r.RemoteAddr,
						"user-agent": r.UserAgent(),
					},
					Time:     time.Now(),
					MimeType: mimeType,
				}
				cfg.DB.SaveFileInfo(info)
			}
		}
	}
}

const octetStream = "application/octet-stream"

func serveFile(w http.ResponseWriter, r *http.Request) {
	l := r.Context().Value("logger").(logrus.FieldLogger)
	name := chi.URLParam(r, "file")
	l = l.WithField("file", name)
	if ratelimited(r.RemoteAddr) {
		l.Warning("ratelimited")
		http.Error(w, "Rate Limit Exceeded", 429)
		return
	}
	file, err := os.Open(filepath.Join(cfg.BasePath, name))
	if err != nil {
		l.WithError(err).Warning("not found")
		http.Error(w, "404 Not Found", 404)
		return
	}
	defer file.Close()
	rateAdd(r.RemoteAddr)
	spl := strings.Split(name, ".")
	id := spl[0]
	extension := ""
	if len(spl) > 1 {
		extension = spl[len(spl)-1]
	}
	mimeType := ""
	if cfg.DB != nil {
		info := cfg.DB.GetFileInfo(id)
		if info != nil {
			if info.Expire != 0 {
				if time.Since(time.Now().Add(info.Expire)) > info.Expire {
					l.Info("expired")
					http.Error(w, "404 Not Found", 404)
					return
				}
			}
			mimeType = info.MimeType
			info.Clicks++
		}
	}

	if mimeType == "" {
		sniffData := make([]byte, 512)
		n, err := file.Read(sniffData)
		if err != nil {
			l.WithError(err).Error("cannot read from file")
			http.Error(w, "Internal Server Error", 500)
			return
		}
		sniffData = sniffData[:n]
		mimeType = http.DetectContentType(sniffData)
		_, err = file.Seek(0, 0)
		if err != nil {
			l.WithError(err).Error("cannot seek file")
			http.Error(w, "Internal Server Error", 500)
			return
		}
	}
	if mimeType == octetStream {
		switch extension {
		case "mp3":
			mimeType = "audio/mpeg"
		case "wav":
			mimeType = "audio/wav"
		default:
			mimeType = "text/plain"
		}
	}
	if dl, _ := strconv.ParseBool(r.URL.Query().Get("download")); dl {
		mimeType = octetStream
	}
	w.Header().Set("X-Content-Type-Options", "nosniff")
	w.Header().Set("Content-Type", mimeType)

	http.ServeContent(w, r, "", time.Time{}, file)
}
<commit_msg>:thinking:<commit_after>package filehost

import (
	"context"
	"html/template"
	"io"
	"net/http"
	"os"
	"path/filepath"
	"strconv"
	"strings"
	"time"

	"mime"

	"github.com/pressly/chi"
	"github.com/sirupsen/logrus"
)

var log logrus.FieldLogger = logrus.StandardLogger()
var cfg *Config
var templ *template.Template

// Database is some sort of database that stores FileInfo
type Database interface {
	// GetFileInfo returns the FileInfo associated with the given
	// File Name, it returns nil if the file was not found
	GetFileInfo(string) *FileInfo
	SaveFileInfo(*FileInfo)
}

// Config contains the needed information to call New
type Config struct {
	AllowedMimeTypes []string
	BasePath         string
	BaseURL          string
	UploadPage       bool
	ExposedPassword  string
	NewFileName      func() string
	DB               Database
	Authenticate     func(*http.Request) bool
	AllowFileName    func(*http.Request) bool
	Logger           logrus.FieldLogger
}

// FileInfo contains additional information about the File
type FileInfo struct {
	Name     string // filename without extension
	Path     string
	MimeType string
	Uploader interface{} // information about the person who uploaded it
	Time     time.Time
	Expire   time.Duration
	Clicks   int
}

// New initializes a http Handler and returns it
func New(conf *Config) http.Handler {
	go ratelimit()
	cfg = conf
	log = cfg.Logger
	err := os.MkdirAll(cfg.BasePath, 644)
	if err != nil {
		log.Fatal(err)
	}
	r := chi.NewRouter()
	r.Use(func(next http.Handler) http.Handler {
		return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
			r = r.WithContext(context.WithValue(r.Context(), "logger",
				log.WithFields(logrus.Fields{
					"ip":         r.RemoteAddr,
					"user-agent": r.UserAgent(),
				})))
			next.ServeHTTP(w, r)
		})
	})
	r.Post("/upload", upload)
	r.Get("/:file", serveFile)
	if cfg.UploadPage {
		templ = template.Must(template.ParseFiles("upload.html"))
		r.Get("/", uploadPage)
	} else {
		r.Get("/", http.NotFound)
	}

	return r
}

func uploadPage(w http.ResponseWriter, r *http.Request) {
	data := struct {
		Key     string
		BaseURL string
	}{
		Key:     cfg.ExposedPassword,
		BaseURL: cfg.BaseURL,
	}
	if err := templ.Execute(w, data); err != nil {
		r.Context().Value("logger").(logrus.FieldLogger).
			WithError(err).Error("cannot execute template")
	}
}

func upload(w http.ResponseWriter, r *http.Request) {
	l := r.Context().Value("logger").(logrus.FieldLogger)
	defer r.Body.Close()
	if !cfg.Authenticate(r) {
		http.Error(w, "Not Authenticated", http.StatusUnauthorized)
		return
	}
	if ratelimited(r.RemoteAddr) {
		l.Warning("ratelimited")
		http.Error(w, "Rate Limit Exceeded", 429)
		return
	}
	l.Debug(r.Header)
	err := r.ParseMultipartForm(1024 * 1024 * 64)
	if err != nil {
		l.WithError(err).Error("cannot read multi part form")
		http.Error(w, "Bad Request", http.StatusBadRequest)
		return
	}
	if r.MultipartForm == nil {
		http.Error(w, "Bad Request", http.StatusBadRequest)
		return
	}
	files := r.MultipartForm.File
	l.Debug(files)
	if len(files) < 1 {
		http.Error(w, "Bad Request", http.StatusBadRequest)
		return
	}
	for _, headers := range files {
		for _, h := range headers {
			file, err := h.Open()
			if err != nil {
				l.Error(err)
			}
			name := h.Filename
			if !cfg.AllowFileName(r) || name == "" {
				if cfg.NewFileName != nil {
					name = cfg.NewFileName()
				} else {
					name = RandString(5)
				}
			}
			l = l.WithField("file", name)
			l.Info("uploading...")
			l.Debug(r.Header)
			l.Debug(h.Header)
			mimeType := h.Header.Get("Content-Type")
			if !whiteListed(cfg.AllowedMimeTypes, mimeType) {
				spl := strings.Split(name, ".")
				if len(spl) > 1 {
					ext := spl[len(spl)-1]
					mimeType = mime.TypeByExtension("." + ext)
					l.WithField("mime-type", mimeType).Debug("type from ext")
				}
			}
			if mimeType == octetStream || mimeType == "" {
				mimeType = "text/plain"
			}
			l = l.WithField("mime-type", mimeType)
			if !whiteListed(cfg.AllowedMimeTypes, mimeType) {
				l.Warning("mime type not allowed")
				http.Error(w, "Unsupported Media Type", 415)
				return
			}
			extension := ExtensionFromMime(mimeType)
			if extension != "" {
				extension = "." + extension
			}

			fullName := name + extension
			dstPath := filepath.Join(cfg.BasePath, fullName)
			// TODO: check if file exists
			dst, err := os.Create(dstPath)
			if err != nil {
				l.WithError(err).Error("cannot create file")
				http.Error(w, "Internal Server Error", 500)
				return
			}
			_, err = io.Copy(dst, file)
			if err != nil {
				l.WithError(err).Error("failed to save file")
				http.Error(w, "Internal Server Error", 500)
				return
			}
			w.Write([]byte(cfg.BaseURL + fullName))
			l.Info("uploaded to ", cfg.BaseURL+fullName)
			if cfg.DB != nil {
				info := &FileInfo{
					Name: name,
					Path: dstPath,
					Uploader: map[string]interface{}{
						"ip":         r.RemoteAddr,
						"user-agent": r.UserAgent(),
					},
					Time:     time.Now(),
					MimeType: mimeType,
				}
				cfg.DB.SaveFileInfo(info)
			}
		}
	}
}

const octetStream = "application/octet-stream"

func serveFile(w http.ResponseWriter, r *http.Request) {
	l := r.Context().Value("logger").(logrus.FieldLogger)
	name := chi.URLParam(r, "file")
	l = l.WithField("file", name)
	if ratelimited(r.RemoteAddr) {
		l.Warning("ratelimited")
		http.Error(w, "Rate Limit Exceeded", 429)
		return
	}
	file, err := os.Open(filepath.Join(cfg.BasePath, name))
	if err != nil {
		l.WithError(err).Warning("not found")
		http.Error(w, "404 Not Found", 404)
		return
	}
	defer file.Close()
	rateAdd(r.RemoteAddr)
	spl := strings.Split(name, ".")
	id := spl[0]
	extension := ""
	if len(spl) > 1 {
		extension = spl[len(spl)-1]
	}
	mimeType := ""
	if cfg.DB != nil {
		info := cfg.DB.GetFileInfo(id)
		if info != nil {
			if info.Expire != 0 {
				if time.Since(time.Now().Add(info.Expire)) > info.Expire {
					l.Info("expired")
					http.Error(w, "404 Not Found", 404)
					return
				}
			}
			mimeType = info.MimeType
			info.Clicks++
		}
	}

	if mimeType == "" {
		sniffData := make([]byte, 512)
		n, err := file.Read(sniffData)
		if err != nil {
			l.WithError(err).Error("cannot read from file")
			http.Error(w, "Internal Server Error", 500)
			return
		}
		sniffData = sniffData[:n]
		mimeType = http.DetectContentType(sniffData)
		_, err = file.Seek(0, 0)
		if err != nil {
			l.WithError(err).Error("cannot seek file")
			http.Error(w, "Internal Server Error", 500)
			return
		}
	}
	if mimeType == octetStream {
		switch extension {
		case "mp3":
			mimeType = "audio/mpeg"
		case "wav":
			mimeType = "audio/wav"
		default:
			mimeType = "text/plain"
		}
	}
	if dl, _ := strconv.ParseBool(r.URL.Query().Get("download")); dl {
		mimeType = octetStream
	}
	w.Header().Set("X-Content-Type-Options", "nosniff")
	w.Header().Set("Content-Type", mimeType)

	http.ServeContent(w, r, "", time.Time{}, file)
}
<|endoftext|>


<commit_before>package elkrem

import (
	"testing"

	"github.com/btcsuite/btcd/chaincfg/chainhash"
)

// TestElkremBig tries 10K hashes
func TestElkremBig(t *testing.T) {
	sndr := NewElkremSender(chainhash.DoubleHashH([]byte("elktest")))
	var rcv ElkremReceiver
	//	SenderSerdesTest(t, sndr)
	for n := uint64(0); n < 10000; n++ {
		sha, err := sndr.AtIndex(n)
		if err != nil {
			t.Fatal(err)
		}
		err = rcv.AddNext(sha)
		if err != nil {
			t.Fatal(err)
		}
		if n%1000 == 999 {
			t.Logf("stack with %d received hashes\n", n+1)
			for i, n := range rcv.s {
				t.Logf("Stack element %d: index %d height %d %s\n",
					i, n.i, n.h, n.sha.String())
			}
		}
	}
	//	SenderSerdesTest(t, sndr)
	ReceiverSerdesTest(t, &rcv)
	for n := uint64(0); n < 10000; n += 500 {
		sha, err := rcv.AtIndex(n)
		if err != nil {
			t.Fatal(err)
		}
		t.Logf("Retreived index %d %s\n", n, sha.String())
	}
}

// TestElkremLess tries 10K hashes
func TestElkremLess(t *testing.T) {
	sndr := NewElkremSender(chainhash.DoubleHashH([]byte("elktest2")))
	var rcv ElkremReceiver
	for n := uint64(0); n < 5000; n++ {
		sha, err := sndr.AtIndex(n)
		if err != nil {
			t.Fatal(err)
		}
		err = rcv.AddNext(sha)
		if err != nil {
			t.Fatal(err)
		}
		if n%1000 == 999 {
			t.Logf("stack with %d received hashes\n", n+1)
			for i, n := range rcv.s {
				t.Logf("Stack element %d: index %d height %d %s\n",
					i, n.i, n.h, n.sha.String())
			}
		}
	}
	for n := uint64(0); n < 5000; n += 500 {
		sha, err := rcv.AtIndex(n)
		if err != nil {
			t.Fatal(err)
		}
		t.Logf("Retreived index %d %s\n",
			n, sha.String())
	}
}
<commit_msg>add test cases to elkrem<commit_after>package elkrem

import (
	"testing"

	"github.com/btcsuite/btcd/chaincfg/chainhash"
)

// TestElkremBig tries 10K hashes
func TestElkremBig(t *testing.T) {
	sndr := NewElkremSender(chainhash.DoubleHashH([]byte("elktest")))
	var rcv ElkremReceiver
	//	SenderSerdesTest(t, sndr)
	for n := uint64(0); n < 10000; n++ {
		sha, err := sndr.AtIndex(n)
		if err != nil {
			t.Fatal(err)
		}
		err = rcv.AddNext(sha)
		if err != nil {
			t.Fatal(err)
		}
		if n%1000 == 999 {
			t.Logf("stack with %d received hashes\n", n+1)
			for i, n := range rcv.s {
				t.Logf("Stack element %d: index %d height %d %s\n",
					i, n.i, n.h, n.sha.String())
			}
		}
	}
	//	SenderSerdesTest(t, sndr)
	ReceiverSerdesTest(t, &rcv)
	for n := uint64(0); n < 10000; n += 500 {
		sha, err := rcv.AtIndex(n)
		if err != nil {
			t.Fatal(err)
		}
		t.Logf("Retreived index %d %s\n", n, sha.String())
	}
}

// TestElkremLess tries 10K hashes
func TestElkremLess(t *testing.T) {
	sndr := NewElkremSender(chainhash.DoubleHashH([]byte("elktest2")))
	var rcv ElkremReceiver
	for n := uint64(0); n < 5000; n++ {
		sha, err := sndr.AtIndex(n)
		if err != nil {
			t.Fatal(err)
		}
		err = rcv.AddNext(sha)
		if err != nil {
			t.Fatal(err)
		}
		if n%1000 == 999 {
			t.Logf("stack with %d received hashes\n", n+1)
			for i, n := range rcv.s {
				t.Logf("Stack element %d: index %d height %d %s\n",
					i, n.i, n.h, n.sha.String())
			}
		}
	}
	for n := uint64(0); n < 5000; n += 500 {
		sha, err := rcv.AtIndex(n)
		if err != nil {
			t.Fatal(err)
		}
		t.Logf("Retreived index %d %s\n",
			n, sha.String())
	}
}

func TestFixed(t *testing.T) {
	root, _ := chainhash.NewHashFromStr(
		"b43614f251760d689adf84211148a40d7dee13967b7109e13c8d1437a4966d58")

	sndr := NewElkremSender(*root)

	zero, _ := chainhash.NewHashFromStr(
		"2a124935e0713149b71ff17cb43465e9828bacd1e833f0dc08460783a6a42cb4")

	thousand, _ := chainhash.NewHashFromStr(
		"0151a39169940cdd8ccf1ba619f254ddbf16ce260a243528839b2634eaa63d0a")

	for n := uint64(0); n < 5000; n += 500 {
		sha, err := sndr.AtIndex(n)
		if err != nil {
			t.Fatal(err)
		}
		t.Logf("elk %d: %s\n", n, sha.String())

		if n == 0 && !sha.IsEqual(zero) {
			t.Fatalf("Elk %d expected %s, got %s", n, zero.String(), sha.String())
		}
		if n == 1000 && !sha.IsEqual(thousand) {
			t.Fatalf("Elk %d expected %s, got %s", n, zero.String(), sha.String())
		}

	}

}
<|endoftext|>


<commit_before>/* This Source Code Form is subject to the terms of the Mozilla Public
 * License, v. 2.0. If a copy of the MPL was not distributed with this
 * file, You can obtain one at http://mozilla.org/MPL/2.0/. */

package main

import (
	"encoding/json"
	"errors"
	"io/ioutil"
	"net/http"
	"strings"
	"time"

	"github.com/labstack/echo"
)

// Service holds the service response from service-store
type Service struct {
	ID           string                 `json:"id"`
	GroupID      int                    `json:"group_id"`
	DatacenterID int                    `json:"datacenter_id"`
	Name         string                 `json:"name"`
	Type         string                 `json:"type"`
	Version      time.Time              `json:"version"`
	Options      map[string]interface{} `json:"options"`
	Status       string                 `json:"status"`
	Endpoint     string                 `json:"endpoint"`
	Definition   interface{}            `json:"definition"`
}

// Validate the service
func (d *Service) Validate() error {
	if d.Name == "" {
		return errors.New("Service name is empty")
	}

	if d.DatacenterID == 0 {
		return errors.New("Service group is empty")
	}

	if d.Type == "" {
		return errors.New("Service type is empty")
	}

	return nil
}

// Map : maps a service from a request's body and validates the input
func (d *Service) Map(c echo.Context) *echo.HTTPError {
	body := c.Request().Body()
	data, err := ioutil.ReadAll(body)
	if err != nil {
		return ErrBadReqBody
	}

	err = json.Unmarshal(data, &d)
	if err != nil {
		return ErrBadReqBody
	}

	err = d.Validate()
	if err != nil {
		return ErrBadReqBody
	}

	return nil
}

func getServicesHandler(c echo.Context) (err error) {
	var list []OutputService
	au := authenticatedUser(c)

	query := getSearchFilter(c)
	if au.Admin != true {
		query["group_id"] = au.GroupID
	}

	if list, err = getServicesOutput(query); err != nil {
		return c.JSONBlob(500, []byte(err.Error()))
	}

	return c.JSON(http.StatusOK, list)
}

func getServiceBuildsHandler(c echo.Context) error {
	// get the service name
	au := authenticatedUser(c)

	// Get all builds for service name
	qb := getParamFilter(c)
	if au.Admin != true {
		qb["group_id"] = au.GroupID
	}

	list, err := getServicesOutput(qb)
	if err != nil {
		return c.JSONBlob(500, []byte(err.Error()))
	}

	return c.JSON(http.StatusOK, list)
}

func getServiceHandler(c echo.Context) (err error) {
	var list []OutputService

	au := authenticatedUser(c)
	query := getParamFilter(c)
	if au.Admin != true {
		query["group_id"] = au.GroupID
	}

	if list, err = getServicesOutput(query); err != nil {
		return c.JSONBlob(500, []byte(err.Error()))
	}

	if len(list) > 0 {
		return c.JSON(http.StatusOK, list[0])
	}
	return c.JSON(http.StatusOK, nil)
}

func getServiceBuildHandler(c echo.Context) (err error) {
	var list []OutputService

	au := authenticatedUser(c)
	query := getParamFilter(c)
	if au.Admin != true {
		query["group_id"] = au.GroupID
	}

	if list, err = getServicesOutput(query); err != nil {
		return c.JSONBlob(500, []byte(err.Error()))
	}

	return c.JSON(http.StatusOK, list[0])
}

func searchServicesHandler(c echo.Context) error {
	au := authenticatedUser(c)

	query := getSearchFilter(c)
	if au.Admin != true {
		query["group_id"] = au.GroupID
	}

	list, err := getServicesOutput(query)
	if err != nil {
		return ErrInternal
	}

	return c.JSON(http.StatusOK, list)
}

func resetServiceHandler(c echo.Context) error {
	au := authenticatedUser(c)
	if status, err := resetService(au, c.Param("service")); err != nil {
		return c.JSONBlob(status, []byte(err.Error()))
	}
	return c.String(200, "success")
}

func createUUIDHandler(c echo.Context) error {
	var s struct {
		ID string `json:"id"`
	}
	req := c.Request()
	body, err := ioutil.ReadAll(req.Body())
	if err != nil {
		return c.JSONBlob(500, []byte("Invalid input"))
	}

	json.Unmarshal(body, &s)
	id := generateStreamID(s.ID)

	return c.JSONBlob(http.StatusOK, []byte(`{"uuid":"`+id+`"}`))
}

// createServiceHandler : Will receive a service application
func createServiceHandler(c echo.Context) error {
	var s ServiceInput
	var err error
	var body []byte
	var definition []byte
	var datacenter []byte
	var group []byte

	payload := ServicePayload{}
	au := authenticatedUser(c)

	if s, definition, body, err = mapInputService(c); err != nil {
		return c.JSONBlob(400, []byte(err.Error()))
	}
	payload.Service = (*json.RawMessage)(&body)

	// Get datacenter
	if datacenter, err = getDatacenter(s.Datacenter, au.GroupID); err != nil {
		return c.JSONBlob(404, []byte(err.Error()))
	}
	payload.Datacenter = (*json.RawMessage)(&datacenter)

	// Get group
	if group, err = getGroup(au.GroupID); err != nil {
		return c.JSONBlob(http.StatusNotFound, []byte(err.Error()))
	}
	payload.Group = (*json.RawMessage)(&group)

	// Generate service ID
	payload.ID = generateServiceID(s.Name + "-" + s.Datacenter)

	// Get previous service if exists
	if previous, err := getService(s.Name, au.GroupID); err != nil {
		return c.JSONBlob(http.StatusNotFound, []byte(err.Error()))
	} else {
		if previous != nil {
			payload.PrevID = previous.ID
			if previous.Status == "errored" {
				prev, err := n.Request("service.get.mapping", []byte(`{"id":"`+previous.ID+`"}`), time.Second*3)
				if err != nil {
					return c.JSONBlob(http.StatusNotFound, []byte(`"We found a problem reexecuting your service, please try again"`))
				}
				body := []byte(strings.Replace(string(prev.Data), "\"service.create\"", "\"service.patch\"", -1))
				n.Publish("service.patch", body)
				return c.JSONBlob(http.StatusOK, []byte(`{"id":"`+payload.ID+`"}`))
			}
			if previous.Status == "in_progress" {
				return c.JSONBlob(http.StatusNotFound, []byte(`"Your service process is 'in progress' if your're sure you want to fix it please reset it first"`))
			}
		}
	}

	var service []byte
	if service, err = mapCreateDefinition(payload); err != nil {
		return echo.NewHTTPError(400, err.Error())
	}

	var datacenterStruct struct {
		ID   uint   `json:"id"`
		Type string `json:"type"`
	}
	json.Unmarshal(datacenter, &datacenterStruct)

	version := time.Now()
	status := "in_progress"
	options := "{}"
	mapping := string(service)
	saveService(payload.ID, s.Name, datacenterStruct.Type, version, status, options, string(definition), mapping, uint(au.GroupID), datacenterStruct.ID)

	// Apply changes
	n.Publish("service.create", service)

	return c.JSONBlob(http.StatusOK, []byte(`{"id":"`+payload.ID+`"}`))
}

func updateServiceHandler(c echo.Context) error {
	return echo.NewHTTPError(405, "Not implemented")
}

// Deletes a service by name
func deleteServiceHandler(c echo.Context) error {
	var raw []byte
	var err error

	au := authenticatedUser(c)

	if raw, err = getServiceRaw(c.Param("name"), au.GroupID); err != nil {
		return echo.NewHTTPError(404, err.Error())
	}

	s := Service{}
	json.Unmarshal(raw, &s)

	if s.Status == "in_progress" {
		return c.JSONBlob(400, []byte(`"Service is already applying some changes, please wait until they are done"`))
	}

	query := []byte(`{"previous_id":"` + s.ID + `"}`)
	if msg, err := n.Request("definition.map.deletion", query, 1*time.Second); err != nil {
		return c.JSONBlob(500, []byte(`"Couldn't map the service"`))
	} else {
		n.Publish("service.delete", msg.Data)
	}

	parts := strings.Split(s.ID, "-")
	stream := parts[len(parts)-1]

	return c.JSONBlob(http.StatusOK, []byte(`{"id":"`+s.ID+`","stream_id":"`+stream+`"}`))
}
<commit_msg>Provide datacenter type on deletion mapping<commit_after>/* This Source Code Form is subject to the terms of the Mozilla Public
 * License, v. 2.0. If a copy of the MPL was not distributed with this
 * file, You can obtain one at http://mozilla.org/MPL/2.0/. */

package main

import (
	"encoding/json"
	"errors"
	"io/ioutil"
	"net/http"
	"strings"
	"time"

	"github.com/labstack/echo"
)

// Service holds the service response from service-store
type Service struct {
	ID           string                 `json:"id"`
	GroupID      int                    `json:"group_id"`
	DatacenterID int                    `json:"datacenter_id"`
	Name         string                 `json:"name"`
	Type         string                 `json:"type"`
	Version      time.Time              `json:"version"`
	Options      map[string]interface{} `json:"options"`
	Status       string                 `json:"status"`
	Endpoint     string                 `json:"endpoint"`
	Definition   interface{}            `json:"definition"`
}

// Validate the service
func (d *Service) Validate() error {
	if d.Name == "" {
		return errors.New("Service name is empty")
	}

	if d.DatacenterID == 0 {
		return errors.New("Service group is empty")
	}

	if d.Type == "" {
		return errors.New("Service type is empty")
	}

	return nil
}

// Map : maps a service from a request's body and validates the input
func (d *Service) Map(c echo.Context) *echo.HTTPError {
	body := c.Request().Body()
	data, err := ioutil.ReadAll(body)
	if err != nil {
		return ErrBadReqBody
	}

	err = json.Unmarshal(data, &d)
	if err != nil {
		return ErrBadReqBody
	}

	err = d.Validate()
	if err != nil {
		return ErrBadReqBody
	}

	return nil
}

func getServicesHandler(c echo.Context) (err error) {
	var list []OutputService
	au := authenticatedUser(c)

	query := getSearchFilter(c)
	if au.Admin != true {
		query["group_id"] = au.GroupID
	}

	if list, err = getServicesOutput(query); err != nil {
		return c.JSONBlob(500, []byte(err.Error()))
	}

	return c.JSON(http.StatusOK, list)
}

func getServiceBuildsHandler(c echo.Context) error {
	// get the service name
	au := authenticatedUser(c)

	// Get all builds for service name
	qb := getParamFilter(c)
	if au.Admin != true {
		qb["group_id"] = au.GroupID
	}

	list, err := getServicesOutput(qb)
	if err != nil {
		return c.JSONBlob(500, []byte(err.Error()))
	}

	return c.JSON(http.StatusOK, list)
}

func getServiceHandler(c echo.Context) (err error) {
	var list []OutputService

	au := authenticatedUser(c)
	query := getParamFilter(c)
	if au.Admin != true {
		query["group_id"] = au.GroupID
	}

	if list, err = getServicesOutput(query); err != nil {
		return c.JSONBlob(500, []byte(err.Error()))
	}

	if len(list) > 0 {
		return c.JSON(http.StatusOK, list[0])
	}
	return c.JSON(http.StatusOK, nil)
}

func getServiceBuildHandler(c echo.Context) (err error) {
	var list []OutputService

	au := authenticatedUser(c)
	query := getParamFilter(c)
	if au.Admin != true {
		query["group_id"] = au.GroupID
	}

	if list, err = getServicesOutput(query); err != nil {
		return c.JSONBlob(500, []byte(err.Error()))
	}

	return c.JSON(http.StatusOK, list[0])
}

func searchServicesHandler(c echo.Context) error {
	au := authenticatedUser(c)

	query := getSearchFilter(c)
	if au.Admin != true {
		query["group_id"] = au.GroupID
	}

	list, err := getServicesOutput(query)
	if err != nil {
		return ErrInternal
	}

	return c.JSON(http.StatusOK, list)
}

func resetServiceHandler(c echo.Context) error {
	au := authenticatedUser(c)
	if status, err := resetService(au, c.Param("service")); err != nil {
		return c.JSONBlob(status, []byte(err.Error()))
	}
	return c.String(200, "success")
}

func createUUIDHandler(c echo.Context) error {
	var s struct {
		ID string `json:"id"`
	}
	req := c.Request()
	body, err := ioutil.ReadAll(req.Body())
	if err != nil {
		return c.JSONBlob(500, []byte("Invalid input"))
	}

	json.Unmarshal(body, &s)
	id := generateStreamID(s.ID)

	return c.JSONBlob(http.StatusOK, []byte(`{"uuid":"`+id+`"}`))
}

// createServiceHandler : Will receive a service application
func createServiceHandler(c echo.Context) error {
	var s ServiceInput
	var err error
	var body []byte
	var definition []byte
	var datacenter []byte
	var group []byte

	payload := ServicePayload{}
	au := authenticatedUser(c)

	if s, definition, body, err = mapInputService(c); err != nil {
		return c.JSONBlob(400, []byte(err.Error()))
	}
	payload.Service = (*json.RawMessage)(&body)

	// Get datacenter
	if datacenter, err = getDatacenter(s.Datacenter, au.GroupID); err != nil {
		return c.JSONBlob(404, []byte(err.Error()))
	}
	payload.Datacenter = (*json.RawMessage)(&datacenter)

	// Get group
	if group, err = getGroup(au.GroupID); err != nil {
		return c.JSONBlob(http.StatusNotFound, []byte(err.Error()))
	}
	payload.Group = (*json.RawMessage)(&group)

	// Generate service ID
	payload.ID = generateServiceID(s.Name + "-" + s.Datacenter)

	// Get previous service if exists
	if previous, err := getService(s.Name, au.GroupID); err != nil {
		return c.JSONBlob(http.StatusNotFound, []byte(err.Error()))
	} else {
		if previous != nil {
			payload.PrevID = previous.ID
			if previous.Status == "errored" {
				prev, err := n.Request("service.get.mapping", []byte(`{"id":"`+previous.ID+`"}`), time.Second*3)
				if err != nil {
					return c.JSONBlob(http.StatusNotFound, []byte(`"We found a problem reexecuting your service, please try again"`))
				}
				body := []byte(strings.Replace(string(prev.Data), "\"service.create\"", "\"service.patch\"", -1))
				n.Publish("service.patch", body)
				return c.JSONBlob(http.StatusOK, []byte(`{"id":"`+payload.ID+`"}`))
			}
			if previous.Status == "in_progress" {
				return c.JSONBlob(http.StatusNotFound, []byte(`"Your service process is 'in progress' if your're sure you want to fix it please reset it first"`))
			}
		}
	}

	var service []byte
	if service, err = mapCreateDefinition(payload); err != nil {
		return echo.NewHTTPError(400, err.Error())
	}

	var datacenterStruct struct {
		ID   uint   `json:"id"`
		Type string `json:"type"`
	}
	json.Unmarshal(datacenter, &datacenterStruct)

	version := time.Now()
	status := "in_progress"
	options := "{}"
	mapping := string(service)
	saveService(payload.ID, s.Name, datacenterStruct.Type, version, status, options, string(definition), mapping, uint(au.GroupID), datacenterStruct.ID)

	// Apply changes
	n.Publish("service.create", service)

	return c.JSONBlob(http.StatusOK, []byte(`{"id":"`+payload.ID+`"}`))
}

func updateServiceHandler(c echo.Context) error {
	return echo.NewHTTPError(405, "Not implemented")
}

// Deletes a service by name
func deleteServiceHandler(c echo.Context) error {
	var raw []byte
	var err error

	au := authenticatedUser(c)

	if raw, err = getServiceRaw(c.Param("name"), au.GroupID); err != nil {
		return echo.NewHTTPError(404, err.Error())
	}

	s := Service{}
	json.Unmarshal(raw, &s)

	if s.Status == "in_progress" {
		return c.JSONBlob(400, []byte(`"Service is already applying some changes, please wait until they are done"`))
	}

	query := []byte(`{"previous_id":"` + s.ID + `","datacenter":{"type":"` + s.Type + `"}}`)
	if msg, err := n.Request("definition.map.deletion", query, 1*time.Second); err != nil {
		return c.JSONBlob(500, []byte(`"Couldn't map the service"`))
	} else {
		n.Publish("service.delete", msg.Data)
	}

	parts := strings.Split(s.ID, "-")
	stream := parts[len(parts)-1]

	return c.JSONBlob(http.StatusOK, []byte(`{"id":"`+s.ID+`","stream_id":"`+stream+`"}`))
}
<|endoftext|>


<commit_before>package gist7576154

import (
	"io"
	"os/exec"

	. "github.com/shurcooL/go/gists/gist7729255"
	. "github.com/shurcooL/go/gists/gist7802150"

	"gopkg.in/pipe.v2"
)

// CmdFactory is an interface to create new commands.
type CmdFactory interface {
	NewCommand() *exec.Cmd
}

// CmdTemplate is a command template.
type CmdTemplate struct {
	NameArgs []string
	Dir      string
	Stdin    func() io.Reader
}

// NewCmdTemplate returns a CmdTemplate.
func NewCmdTemplate(name string, arg ...string) CmdTemplate {
	return CmdTemplate{
		NameArgs: append([]string{name}, arg...),
	}
}

// NewCommand generates a new *exec.Cmd from the template.
func (ct CmdTemplate) NewCommand() *exec.Cmd {
	cmd := exec.Command(ct.NameArgs[0], ct.NameArgs[1:]...)
	cmd.Dir = ct.Dir
	if ct.Stdin != nil {
		cmd.Stdin = ct.Stdin()
	}
	return cmd
}

// ---

type CmdTemplateDynamic struct {
	NameArgs Strings
	Dir      String
	Stdin    func() io.Reader
}

func NewCmdTemplateDynamic(nameArgs Strings) CmdTemplateDynamic {
	return CmdTemplateDynamic{
		NameArgs: nameArgs,
	}
}

func (ct CmdTemplateDynamic) NewCommand() *exec.Cmd {
	nameArgs := ct.NameArgs.Get()
	cmd := exec.Command(nameArgs[0], nameArgs[1:]...)
	if ct.Dir != nil {
		cmd.Dir = ct.Dir.Get()
	}
	if ct.Stdin != nil {
		cmd.Stdin = ct.Stdin()
	}
	return cmd
}

// ---

type CmdTemplateDynamic2 struct {
	Template CmdTemplate

	DepNode2Func
}

// TODO: See if there's some way to initialize DepNode2Func.UpdateFunc through NewCmdTemplateDynamic2().
func NewCmdTemplateDynamic2() *CmdTemplateDynamic2 {
	return &CmdTemplateDynamic2{}
}

func (this *CmdTemplateDynamic2) NewCommand() *exec.Cmd {
	MakeUpdated(this)
	return this.Template.NewCommand()
}

// =====

type PipeFactory interface {
	NewPipe(stdout, stderr io.Writer) (*pipe.State, pipe.Pipe)
}

// ---

type PipeStatic pipe.Pipe

func (this PipeStatic) NewPipe(stdout, stderr io.Writer) (*pipe.State, pipe.Pipe) {
	return pipe.NewState(stdout, stderr), (pipe.Pipe)(this)
}

// ---

type pipeTemplate struct {
	Pipe  pipe.Pipe
	Dir   string
	Stdin func() io.Reader
}

func NewPipeTemplate(pipe pipe.Pipe) *pipeTemplate {
	return &pipeTemplate{Pipe: pipe}
}

func (this *pipeTemplate) NewPipe(stdout, stderr io.Writer) (*pipe.State, pipe.Pipe) {
	s := pipe.NewState(stdout, stderr)
	s.Dir = this.Dir
	if this.Stdin != nil {
		s.Stdin = this.Stdin()
	}
	return s, this.Pipe
}

// ---

type pipeTemplateDynamic struct {
	Template *pipeTemplate

	DepNode2Func
}

func NewPipeTemplateDynamic() *pipeTemplateDynamic {
	return &pipeTemplateDynamic{}
}

func (this *pipeTemplateDynamic) NewPipe(stdout, stderr io.Writer) (*pipe.State, pipe.Pipe) {
	MakeUpdated(this)
	return this.Template.NewPipe(stdout, stderr)
}
<commit_msg>Avoid using dot imports.<commit_after>package gist7576154

import (
	"io"
	"os/exec"

	"github.com/shurcooL/go/gists/gist7729255"
	"github.com/shurcooL/go/gists/gist7802150"

	"gopkg.in/pipe.v2"
)

// CmdFactory is an interface to create new commands.
type CmdFactory interface {
	NewCommand() *exec.Cmd
}

// CmdTemplate is a command template.
type CmdTemplate struct {
	NameArgs []string
	Dir      string
	Stdin    func() io.Reader
}

// NewCmdTemplate returns a CmdTemplate.
func NewCmdTemplate(name string, arg ...string) CmdTemplate {
	return CmdTemplate{
		NameArgs: append([]string{name}, arg...),
	}
}

// NewCommand generates a new *exec.Cmd from the template.
func (ct CmdTemplate) NewCommand() *exec.Cmd {
	cmd := exec.Command(ct.NameArgs[0], ct.NameArgs[1:]...)
	cmd.Dir = ct.Dir
	if ct.Stdin != nil {
		cmd.Stdin = ct.Stdin()
	}
	return cmd
}

// ---

type CmdTemplateDynamic struct {
	NameArgs gist7729255.Strings
	Dir      gist7729255.String
	Stdin    func() io.Reader
}

func NewCmdTemplateDynamic(nameArgs gist7729255.Strings) CmdTemplateDynamic {
	return CmdTemplateDynamic{
		NameArgs: nameArgs,
	}
}

func (ct CmdTemplateDynamic) NewCommand() *exec.Cmd {
	nameArgs := ct.NameArgs.Get()
	cmd := exec.Command(nameArgs[0], nameArgs[1:]...)
	if ct.Dir != nil {
		cmd.Dir = ct.Dir.Get()
	}
	if ct.Stdin != nil {
		cmd.Stdin = ct.Stdin()
	}
	return cmd
}

// ---

type CmdTemplateDynamic2 struct {
	Template CmdTemplate

	gist7802150.DepNode2Func
}

// TODO: See if there's some way to initialize DepNode2Func.UpdateFunc through NewCmdTemplateDynamic2().
func NewCmdTemplateDynamic2() *CmdTemplateDynamic2 {
	return &CmdTemplateDynamic2{}
}

func (this *CmdTemplateDynamic2) NewCommand() *exec.Cmd {
	gist7802150.MakeUpdated(this)
	return this.Template.NewCommand()
}

// =====

type PipeFactory interface {
	NewPipe(stdout, stderr io.Writer) (*pipe.State, pipe.Pipe)
}

// ---

type PipeStatic pipe.Pipe

func (this PipeStatic) NewPipe(stdout, stderr io.Writer) (*pipe.State, pipe.Pipe) {
	return pipe.NewState(stdout, stderr), (pipe.Pipe)(this)
}

// ---

type pipeTemplate struct {
	Pipe  pipe.Pipe
	Dir   string
	Stdin func() io.Reader
}

func NewPipeTemplate(pipe pipe.Pipe) *pipeTemplate {
	return &pipeTemplate{Pipe: pipe}
}

func (this *pipeTemplate) NewPipe(stdout, stderr io.Writer) (*pipe.State, pipe.Pipe) {
	s := pipe.NewState(stdout, stderr)
	s.Dir = this.Dir
	if this.Stdin != nil {
		s.Stdin = this.Stdin()
	}
	return s, this.Pipe
}

// ---

type pipeTemplateDynamic struct {
	Template *pipeTemplate

	gist7802150.DepNode2Func
}

func NewPipeTemplateDynamic() *pipeTemplateDynamic {
	return &pipeTemplateDynamic{}
}

func (this *pipeTemplateDynamic) NewPipe(stdout, stderr io.Writer) (*pipe.State, pipe.Pipe) {
	gist7802150.MakeUpdated(this)
	return this.Template.NewPipe(stdout, stderr)
}
<|endoftext|>


<commit_before>package shimmie

import (
	"context"
	"crypto/md5"
	"database/sql"
	"fmt"
	"log"
	"net"
	"net/http"
	"strings"
	"time"
)

type contextKey int

const (
	userContextKey contextKey = iota
)

// Hash returns the MD5 checksum of a string s as type string.
func Hash(s string) string {
	return fmt.Sprintf("%x", md5.Sum([]byte(s)))
}

// PasswordHash returns the password hash of a username and password the same
// way that shimmie2 does it.
func PasswordHash(username, password string) string {
	hash := md5.Sum([]byte(strings.ToLower(username) + password))
	return fmt.Sprintf("%x", hash)
}

// Auth is a handler wrapper that checks if a user is authenticated to Shimmie.
// It checks for two cookies "shm_user" and "shm_session". The first
// contains the username which is used to query the database and the get user's
// password hash. Then it attempts to recreate the "shm_session" cookie value
// by using the username, user IP and password hash. If the recreated value
// does not match the "shm_session" cookie value then it redirects to
// redirectPath. If redirectURL is empty then "/user_admin/login" is used
// instead which is the default login URL for Shimmie.
func (shim *Shimmie) Auth(fn http.HandlerFunc, redirectURL string) http.HandlerFunc {
	return func(w http.ResponseWriter, r *http.Request) {
		const defaultLoginURL = "/user_admin/login"
		if redirectURL == "" {
			redirectURL = defaultLoginURL
		}
		usernameCookie, err := r.Cookie("shm_user")
		if err != nil || usernameCookie.Value == "" {
			http.Redirect(w, r, redirectURL, http.StatusFound)
			return
		}
		sessionCookie, err := r.Cookie("shm_session")
		if err != nil {
			log.Print("shimmie: no session cookie")
			http.Redirect(w, r, redirectURL, http.StatusFound)
			return
		}
		username := usernameCookie.Value
		user, err := shim.Store.GetUserByName(username)
		if err != nil {
			if err == sql.ErrNoRows {
				log.Printf("shimmie: user %q does not exist", username)
				http.Redirect(w, r, redirectURL, http.StatusFound)
				return
			}
			msg := fmt.Sprintf("shimmie: could not authenticate: get user %q failed: %v", username, err.Error())
			log.Print(msg)
			http.Error(w, msg, http.StatusInternalServerError)
			return
		}
		passwordHash := user.Pass
		userIP := GetOriginalIP(r)
		sessionCookieValue := CookieValue(passwordHash, userIP)
		if sessionCookieValue != sessionCookie.Value {
			http.Redirect(w, r, redirectURL, http.StatusFound)
			return
		}
		ctx := NewContextWithUser(r.Context(), user)
		fn(w, r.WithContext(ctx))
	}
}

// FromContextGetUser gets User from context. If User does not exist in context,
// nil and false are returned instead.
func FromContextGetUser(ctx context.Context) (*User, bool) {
	user, ok := ctx.Value(userContextKey).(*User)
	return user, ok
}

// NewContextWithUser adds user to context.
func NewContextWithUser(ctx context.Context, user *User) context.Context {
	return context.WithValue(ctx, userContextKey, user)
}

// GetOriginalIP gets the original IP of the HTTP for the case of being behind
// a proxy. It searches for the X-Forwarded-For header.
func GetOriginalIP(r *http.Request) string {
	x := r.Header.Get("X-Forwarded-For")
	if x != "" && strings.Contains(r.RemoteAddr, "127.0.0.1") {
		// format is comma separated
		return strings.Split(x, ",")[0]
	}
	// it also contains the port
	return strings.Split(r.RemoteAddr, ":")[0]
}

// CookieValue recreates the Shimmie session cookie value based on the user
// password hash and the user IP.
//
// Shimmie creates a cookie "shm_session" containing an md5 digest value of the
// user password hash concatenated with the user IP masked with the 255.255.0.0
// mask. That's essentially:
//
//   md5(password_hash + masked_ip)
//
func CookieValue(passwordHash, userIP string) string {
	addr := net.ParseIP(strings.Split(userIP, ":")[0])
	mask := net.IPv4Mask(255, 255, 0, 0)
	addr = addr.Mask(mask)
	sessionHash := md5.Sum([]byte(fmt.Sprintf("%s%s", passwordHash, addr)))
	return fmt.Sprintf("%x", sessionHash)
}

const loginMemory = 365

// SetCookie creates a cookie on path "/" with 1 year expiration and other
// flags set to false mimicking the cookies that Shimmie creates.
func SetCookie(w http.ResponseWriter, name, value string) {
	expires := time.Now().Add(time.Second * 60 * 60 * 24 * loginMemory)
	c := http.Cookie{
		Name:    name,
		Value:   value,
		Expires: expires,
		Path:    "/",
	}
	http.SetCookie(w, &c)
}
<commit_msg>Change Auth to chain handlers, add AuthFunc<commit_after>package shimmie

import (
	"context"
	"crypto/md5"
	"database/sql"
	"fmt"
	"log"
	"net"
	"net/http"
	"strings"
	"time"
)

type contextKey int

const (
	userContextKey contextKey = iota
)

// Hash returns the MD5 checksum of a string s as type string.
func Hash(s string) string {
	return fmt.Sprintf("%x", md5.Sum([]byte(s)))
}

// PasswordHash returns the password hash of a username and password the same
// way that shimmie2 does it.
func PasswordHash(username, password string) string {
	hash := md5.Sum([]byte(strings.ToLower(username) + password))
	return fmt.Sprintf("%x", hash)
}

// Auth is a handler wrapper that checks if a user is authenticated to Shimmie.
// It checks for two cookies "shm_user" and "shm_session". The first
// contains the username which is used to query the database and the get user's
// password hash. Then it attempts to recreate the "shm_session" cookie value
// by using the username, user IP and password hash. If the recreated value
// does not match the "shm_session" cookie value then it redirects to
// redirectPath. If redirectURL is empty then "/user_admin/login" is used
// instead which is the default login URL for Shimmie.
func (shim *Shimmie) Auth(h http.Handler, redirectURL string) http.Handler {
	return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
		const defaultLoginURL = "/user_admin/login"
		if redirectURL == "" {
			redirectURL = defaultLoginURL
		}
		usernameCookie, err := r.Cookie("shm_user")
		if err != nil || usernameCookie.Value == "" {
			http.Redirect(w, r, redirectURL, http.StatusFound)
			return
		}
		sessionCookie, err := r.Cookie("shm_session")
		if err != nil {
			log.Print("shimmie: no session cookie")
			http.Redirect(w, r, redirectURL, http.StatusFound)
			return
		}
		username := usernameCookie.Value
		user, err := shim.Store.GetUserByName(username)
		if err != nil {
			if err == sql.ErrNoRows {
				log.Printf("shimmie: user %q does not exist", username)
				http.Redirect(w, r, redirectURL, http.StatusFound)
				return
			}
			msg := fmt.Sprintf("shimmie: could not authenticate: get user %q failed: %v", username, err.Error())
			log.Print(msg)
			http.Error(w, msg, http.StatusInternalServerError)
			return
		}
		passwordHash := user.Pass
		userIP := GetOriginalIP(r)
		sessionCookieValue := CookieValue(passwordHash, userIP)
		if sessionCookieValue != sessionCookie.Value {
			http.Redirect(w, r, redirectURL, http.StatusFound)
			return
		}
		ctx := NewContextWithUser(r.Context(), user)
		h.ServeHTTP(w, r.WithContext(ctx))
	})
}

func (shim *Shimmie) AuthFunc(fn http.HandlerFunc, redirectURL string) http.Handler {
	return shim.Auth(fn, redirectURL)
}

// FromContextGetUser gets User from context. If User does not exist in context,
// nil and false are returned instead.
func FromContextGetUser(ctx context.Context) (*User, bool) {
	user, ok := ctx.Value(userContextKey).(*User)
	return user, ok
}

// NewContextWithUser adds user to context.
func NewContextWithUser(ctx context.Context, user *User) context.Context {
	return context.WithValue(ctx, userContextKey, user)
}

// GetOriginalIP gets the original IP of the HTTP for the case of being behind
// a proxy. It searches for the X-Forwarded-For header.
func GetOriginalIP(r *http.Request) string {
	x := r.Header.Get("X-Forwarded-For")
	if x != "" && strings.Contains(r.RemoteAddr, "127.0.0.1") {
		// format is comma separated
		return strings.Split(x, ",")[0]
	}
	// it also contains the port
	return strings.Split(r.RemoteAddr, ":")[0]
}

// CookieValue recreates the Shimmie session cookie value based on the user
// password hash and the user IP.
//
// Shimmie creates a cookie "shm_session" containing an md5 digest value of the
// user password hash concatenated with the user IP masked with the 255.255.0.0
// mask. That's essentially:
//
//   md5(password_hash + masked_ip)
//
func CookieValue(passwordHash, userIP string) string {
	addr := net.ParseIP(strings.Split(userIP, ":")[0])
	mask := net.IPv4Mask(255, 255, 0, 0)
	addr = addr.Mask(mask)
	sessionHash := md5.Sum([]byte(fmt.Sprintf("%s%s", passwordHash, addr)))
	return fmt.Sprintf("%x", sessionHash)
}

const loginMemory = 365

// SetCookie creates a cookie on path "/" with 1 year expiration and other
// flags set to false mimicking the cookies that Shimmie creates.
func SetCookie(w http.ResponseWriter, name, value string) {
	expires := time.Now().Add(time.Second * 60 * 60 * 24 * loginMemory)
	c := http.Cookie{
		Name:    name,
		Value:   value,
		Expires: expires,
		Path:    "/",
	}
	http.SetCookie(w, &c)
}
<|endoftext|>


<commit_before>package main

import "github.com/google/go-github/github"

type reposCreate struct {
	Name     string `cli:"arg required"`
	Public   bool   `cli:"opt --public"`
	Orga     string `cli:"opt --orga"`
	Teams    []int  `cli:"opt --teams"`
	ReadOnly bool   `cli:"opt --read-only desc='Add teams with read only'"`
}

func (r *reposCreate) Run() error {
	cl, err := newClient()
	if err != nil {
		return err
	}
	repo := &github.Repository{Name: s2p(r.Name), Private: b2p(!r.Public)}

	if _, _, err := cl.Repositories.Create(r.Orga, repo); err != nil {
		return err
	}
	permission := "push"
	if r.ReadOnly {
		permission = "pull"
	}
	for _, t := range r.Teams {
		_, err = cl.Organizations.AddTeamRepo(t, r.Orga, r.Name, &github.OrganizationAddTeamRepoOptions{Permission: permission})
		if err != nil {
			return err
		}
	}
	return nil
}
<commit_msg>gh#repo/create: add flag to also clone created repository<commit_after>package main

import (
	"os"

	"github.com/google/go-github/github"
)

type reposCreate struct {
	Name     string `cli:"arg required"`
	Public   bool   `cli:"opt --public"`
	Orga     string `cli:"opt --orga"`
	Teams    []int  `cli:"opt --teams"`
	ReadOnly bool   `cli:"opt --read-only desc='Add teams with read only'"`
	Clone    bool   `cli:"opt --clone"`
}

func (r *reposCreate) Run() error {
	cl, err := newClient()
	if err != nil {
		return err
	}
	repo := &github.Repository{Name: s2p(r.Name), Private: b2p(!r.Public)}

	if _, _, err := cl.Repositories.Create(r.Orga, repo); err != nil {
		return err
	}
	permission := "push"
	if r.ReadOnly {
		permission = "pull"
	}
	for _, t := range r.Teams {
		_, err = cl.Organizations.AddTeamRepo(t, r.Orga, r.Name, &github.OrganizationAddTeamRepoOptions{Permission: permission})
		if err != nil {
			return err
		}
	}
	if r.Clone {
		repo := "git@github.com:" + r.Orga + "/" + r.Name + ".git"
		return cloneRepo(repo, os.ExpandEnv("$HOME/src/github.com/"+r.Orga+"/"+repo))
	}
	return nil
}
<|endoftext|>


<commit_before>// Copyright 2015 Netflix, Inc.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package metrics

import (
	"fmt"
	"net/http"
	"runtime"
	"sort"
)

var prefix = ""

func SetPrefix(p string) {
	prefix = p
}

var memstats = new(runtime.MemStats)

func init() {
	http.Handle("/metrics", http.HandlerFunc(printMetrics))
}

func printMetrics(w http.ResponseWriter, r *http.Request) {
	w.Header().Set("Content-Type", "text/plain; charset=utf-8")

	//////////////////////////
	// Runtime memory stats
	//////////////////////////
	runtime.ReadMemStats(memstats)
	// General statistics.
	fmt.Fprintf(w, "%smem_alloc %d\n", prefix, memstats.Alloc)            // bytes allocated and not yet freed
	fmt.Fprintf(w, "%smem_alloc_total %d\n", prefix, memstats.TotalAlloc) // bytes allocated (even if freed)
	fmt.Fprintf(w, "%smem_sys %d\n", prefix, memstats.Sys)                // bytes obtained from system (sum of XxxSys below)
	fmt.Fprintf(w, "%smem_ptr_lookups %d\n", prefix, memstats.Lookups)    // number of pointer lookups
	fmt.Fprintf(w, "%smem_mallocs %d\n", prefix, memstats.Mallocs)        // number of mallocs
	fmt.Fprintf(w, "%smem_frees %d\n", prefix, memstats.Frees)            // number of frees

	// Main allocation heap statistics.
	fmt.Fprintf(w, "%smem_heap_alloc %d\n", prefix, memstats.HeapAlloc)       // bytes allocated and not yet freed (same as Alloc above)
	fmt.Fprintf(w, "%smem_heap_sys %d\n", prefix, memstats.HeapSys)           // bytes obtained from system
	fmt.Fprintf(w, "%smem_heap_idle %d\n", prefix, memstats.HeapIdle)         // bytes in idle spans
	fmt.Fprintf(w, "%smem_heap_in_use %d\n", prefix, memstats.HeapInuse)      // bytes in non-idle span
	fmt.Fprintf(w, "%smem_heap_released %d\n", prefix, memstats.HeapReleased) // bytes released to the OS
	fmt.Fprintf(w, "%smem_heap_objects %d\n", prefix, memstats.HeapObjects)   // total number of allocated objects

	fmt.Fprintf(w, "%smem_stack_in_use %d\n", prefix, memstats.StackInuse) // bytes used by stack allocator
	fmt.Fprintf(w, "%smem_stack_sys %d\n", prefix, memstats.StackSys)
	fmt.Fprintf(w, "%smem_mspan_in_use %d\n", prefix, memstats.MSpanInuse) // mspan structures
	fmt.Fprintf(w, "%smem_mspan_sys %d\n", prefix, memstats.MSpanSys)
	fmt.Fprintf(w, "%smem_mcache_in_use %d\n", prefix, memstats.MCacheInuse) // mcache structures
	fmt.Fprintf(w, "%smem_mcache_sys %d\n", prefix, memstats.MCacheSys)
	fmt.Fprintf(w, "%smem_buck_hash_sys %d\n", prefix, memstats.BuckHashSys) // profiling bucket hash table
	fmt.Fprintf(w, "%smem_gc_sys %d\n", prefix, memstats.GCSys)              // GC metadata
	fmt.Fprintf(w, "%smem_other_sys %d\n", prefix, memstats.OtherSys)        // other system allocations

	fmt.Fprintf(w, "%sgc_next_gc_heap_alloc %d\n", prefix, memstats.NextGC) // next collection will happen when HeapAlloc ≥ this amount
	fmt.Fprintf(w, "%sgc_last_gc_time %d\n", prefix, memstats.LastGC)       // end time of last collection (nanoseconds since 1970)
	fmt.Fprintf(w, "%sgc_pause_total %d\n", prefix, memstats.PauseTotalNs)
	fmt.Fprintf(w, "%sgc_num_gc %d\n", prefix, memstats.NumGC)
	fmt.Fprintf(w, "%sgc_gc_cpu_frac %f\n", prefix, memstats.GCCPUFraction)

	// circular buffer of recent GC pause durations, most recent at [(NumGC+255)%256]
	pctls := pausePercentiles(memstats.PauseNs[:], memstats.NumGC)
	for i := 0; i < 20; i++ {
		p := pctls[i]
		fmt.Fprintf(w, "%sgc_pause_pctl_%d %d\n", prefix, i*5, p)
	}
	fmt.Fprintf(w, "%sgc_pause_pctl_%d %d\n", prefix, 99, pctls[20])
	fmt.Fprintf(w, "%sgc_pause_pctl_%d %d\n", prefix, 100, pctls[21])

	// Per-size allocation statistics.
	for _, b := range memstats.BySize {
		fmt.Fprintf(w, "%salloc_size_%d_mallocs %d\n", prefix, b.Size, b.Mallocs)
		fmt.Fprintf(w, "%salloc_size_%d_frees %d\n", prefix, b.Size, b.Frees)
	}

	//////////////////////////
	// Histograms
	//////////////////////////
	hists := getAllHistograms()
	for name, dat := range hists {
		fmt.Fprintf(w, "%shist_%s_count %d\n", prefix, name, dat.count)
		fmt.Fprintf(w, "%shist_%s_kept %d\n", prefix, name, dat.kept)

		if dat.total > 0 && dat.count > 0 {
			avg := float64(dat.total) / float64(dat.count)
			fmt.Fprintf(w, "%shist_%s_avg %f\n", prefix, name, avg)
		}

		pctls, ok := hdatPercentiles(dat)
		// Assume if the max is 0 that there were no recorded observations
		if !ok {
			/*println(name)
			println(dat.count)
			println(dat.kept)
			println(dat.min)
			println(dat.max)
			println(dat.total)*/
			continue
		}
		for i := 0; i < 20; i++ {
			p := pctls[i]
			fmt.Fprintf(w, "%shist_%s_pctl_%d %d\n", prefix, name, i*5, p)
		}
		fmt.Fprintf(w, "%shist_%s_pctl_%d %d\n", prefix, name, 99, pctls[20])
		fmt.Fprintf(w, "%shist_%s_pctl_%d %d\n", prefix, name, 100, pctls[21])
	}

	//////////////////////////
	// Bucketized histograms
	//////////////////////////
	// Buckets are based on the generated buckets from the Spectator library
	// https://github.com/Netflix/spectator/blob/master/spectator-api/src/main/java/com/netflix/spectator/api/histogram/PercentileBuckets.java#L64
	bhists := getAllBucketHistograms()
	for name, bh := range bhists {
		for i := 0; i < numAtlasBuckets; i++ {
			fmt.Fprintf(w, "%sbhist_%s_bucket_%d %d\n", prefix, name, bucketValues[i], bh[i])
		}
	}

	//////////////////////////
	// Counters
	//////////////////////////
	ctrs := getAllCounters()
	for name, val := range ctrs {
		fmt.Fprintf(w, "%s%s %d\n", prefix, name, val)
	}

	//////////////////////////
	// Gauges
	//////////////////////////
	intg, floatg := getAllGauges()
	for name, val := range intg {
		fmt.Fprintf(w, "%s%s %d\n", prefix, name, val)
	}
	for name, val := range floatg {
		fmt.Fprintf(w, "%s%s %f\n", prefix, name, val)
	}

	//////////////////////////
	// Gauge Callbacks
	//////////////////////////
	intg, floatg = getAllCallbackGauges()
	for name, val := range intg {
		fmt.Fprintf(w, "%s%s %d\n", prefix, name, val)
	}
	for name, val := range floatg {
		fmt.Fprintf(w, "%s%s %f\n", prefix, name, val)
	}
}

// Percentiles go by 5% percentile steps from min to max. We report all of them even though it's
// likely only min, 25th, 50th, 75th, 95th, 99th, and max will be used. It's assumed the metric
// poller that is consuming this output will choose to only report to the metrics system what it
// considers useful information.
//
// Slice layout:
//  [0]: min (0th)
//  [1]: 5th
//  [n]: 5n
//  [19]: 95th
//  [20]: 99th
//  [21]: max (100th)
func hdatPercentiles(dat hdat) ([22]uint64, bool) {
	buf := dat.buf
	kept := dat.kept

	var pctls [22]uint64

	if kept == 0 {
		return pctls, false
	}
	if kept < uint64(len(buf)) {
		buf = buf[:kept]
	}

	sort.Sort(uint64slice(buf))

	// Take care of 0th and 100th specially
	pctls[0] = dat.min
	pctls[21] = dat.max

	// 5th - 95th
	for i := 1; i < 20; i++ {
		idx := len(buf) * i / 20
		pctls[i] = buf[idx]
	}

	// Add 99th
	idx := len(buf) * 99 / 100
	pctls[20] = buf[idx]

	return pctls, true
}

func pausePercentiles(pauses []uint64, ngc uint32) []uint64 {
	if ngc < uint32(len(pauses)) {
		pauses = pauses[:ngc]
	}

	sort.Sort(uint64slice(pauses))

	pctls := make([]uint64, 22)

	// Take care of 0th and 100th specially
	pctls[0] = pauses[0]
	pctls[21] = pauses[len(pauses)-1]

	// 5th - 95th
	for i := 1; i < 20; i++ {
		idx := len(pauses) * i / 20
		pctls[i] = pauses[idx]
	}

	// Add 99th
	idx := len(pauses) * 99 / 100
	pctls[20] = pauses[idx]

	return pctls
}

type uint64slice []uint64

func (u uint64slice) Len() int           { return len(u) }
func (u uint64slice) Swap(i, j int)      { u[i], u[j] = u[j], u[i] }
func (u uint64slice) Less(i, j int) bool { return u[i] < u[j] }
<commit_msg>Prevent concurrent access to metrics endpoint.<commit_after>// Copyright 2015 Netflix, Inc.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package metrics

import (
	"fmt"
	"net/http"
	"runtime"
	"sort"
	"sync"
)

var prefix = ""

func SetPrefix(p string) {
	prefix = p
}

var (
	memstats        = new(runtime.MemStats)
	metricsReadLock = new(sync.Mutex)
)

func init() {
	http.Handle("/metrics", http.HandlerFunc(printMetrics))
}

func printMetrics(w http.ResponseWriter, r *http.Request) {
	// prevent concurrent access to metrics. This is an assumption helpd by much of
	// the code that retrieves the metrics for printing.
	metricsReadLock.Lock()
	defer metricsReadLock.Unlock()

	w.Header().Set("Content-Type", "text/plain; charset=utf-8")

	//////////////////////////
	// Runtime memory stats
	//////////////////////////
	runtime.ReadMemStats(memstats)
	// General statistics.
	fmt.Fprintf(w, "%smem_alloc %d\n", prefix, memstats.Alloc)            // bytes allocated and not yet freed
	fmt.Fprintf(w, "%smem_alloc_total %d\n", prefix, memstats.TotalAlloc) // bytes allocated (even if freed)
	fmt.Fprintf(w, "%smem_sys %d\n", prefix, memstats.Sys)                // bytes obtained from system (sum of XxxSys below)
	fmt.Fprintf(w, "%smem_ptr_lookups %d\n", prefix, memstats.Lookups)    // number of pointer lookups
	fmt.Fprintf(w, "%smem_mallocs %d\n", prefix, memstats.Mallocs)        // number of mallocs
	fmt.Fprintf(w, "%smem_frees %d\n", prefix, memstats.Frees)            // number of frees

	// Main allocation heap statistics.
	fmt.Fprintf(w, "%smem_heap_alloc %d\n", prefix, memstats.HeapAlloc)       // bytes allocated and not yet freed (same as Alloc above)
	fmt.Fprintf(w, "%smem_heap_sys %d\n", prefix, memstats.HeapSys)           // bytes obtained from system
	fmt.Fprintf(w, "%smem_heap_idle %d\n", prefix, memstats.HeapIdle)         // bytes in idle spans
	fmt.Fprintf(w, "%smem_heap_in_use %d\n", prefix, memstats.HeapInuse)      // bytes in non-idle span
	fmt.Fprintf(w, "%smem_heap_released %d\n", prefix, memstats.HeapReleased) // bytes released to the OS
	fmt.Fprintf(w, "%smem_heap_objects %d\n", prefix, memstats.HeapObjects)   // total number of allocated objects

	fmt.Fprintf(w, "%smem_stack_in_use %d\n", prefix, memstats.StackInuse) // bytes used by stack allocator
	fmt.Fprintf(w, "%smem_stack_sys %d\n", prefix, memstats.StackSys)
	fmt.Fprintf(w, "%smem_mspan_in_use %d\n", prefix, memstats.MSpanInuse) // mspan structures
	fmt.Fprintf(w, "%smem_mspan_sys %d\n", prefix, memstats.MSpanSys)
	fmt.Fprintf(w, "%smem_mcache_in_use %d\n", prefix, memstats.MCacheInuse) // mcache structures
	fmt.Fprintf(w, "%smem_mcache_sys %d\n", prefix, memstats.MCacheSys)
	fmt.Fprintf(w, "%smem_buck_hash_sys %d\n", prefix, memstats.BuckHashSys) // profiling bucket hash table
	fmt.Fprintf(w, "%smem_gc_sys %d\n", prefix, memstats.GCSys)              // GC metadata
	fmt.Fprintf(w, "%smem_other_sys %d\n", prefix, memstats.OtherSys)        // other system allocations

	fmt.Fprintf(w, "%sgc_next_gc_heap_alloc %d\n", prefix, memstats.NextGC) // next collection will happen when HeapAlloc ≥ this amount
	fmt.Fprintf(w, "%sgc_last_gc_time %d\n", prefix, memstats.LastGC)       // end time of last collection (nanoseconds since 1970)
	fmt.Fprintf(w, "%sgc_pause_total %d\n", prefix, memstats.PauseTotalNs)
	fmt.Fprintf(w, "%sgc_num_gc %d\n", prefix, memstats.NumGC)
	fmt.Fprintf(w, "%sgc_gc_cpu_frac %f\n", prefix, memstats.GCCPUFraction)

	// circular buffer of recent GC pause durations, most recent at [(NumGC+255)%256]
	pctls := pausePercentiles(memstats.PauseNs[:], memstats.NumGC)
	for i := 0; i < 20; i++ {
		p := pctls[i]
		fmt.Fprintf(w, "%sgc_pause_pctl_%d %d\n", prefix, i*5, p)
	}
	fmt.Fprintf(w, "%sgc_pause_pctl_%d %d\n", prefix, 99, pctls[20])
	fmt.Fprintf(w, "%sgc_pause_pctl_%d %d\n", prefix, 100, pctls[21])

	// Per-size allocation statistics.
	for _, b := range memstats.BySize {
		fmt.Fprintf(w, "%salloc_size_%d_mallocs %d\n", prefix, b.Size, b.Mallocs)
		fmt.Fprintf(w, "%salloc_size_%d_frees %d\n", prefix, b.Size, b.Frees)
	}

	//////////////////////////
	// Histograms
	//////////////////////////
	hists := getAllHistograms()
	for name, dat := range hists {
		fmt.Fprintf(w, "%shist_%s_count %d\n", prefix, name, dat.count)
		fmt.Fprintf(w, "%shist_%s_kept %d\n", prefix, name, dat.kept)

		if dat.total > 0 && dat.count > 0 {
			avg := float64(dat.total) / float64(dat.count)
			fmt.Fprintf(w, "%shist_%s_avg %f\n", prefix, name, avg)
		}

		pctls, ok := hdatPercentiles(dat)
		// Assume if the max is 0 that there were no recorded observations
		if !ok {
			/*println(name)
			println(dat.count)
			println(dat.kept)
			println(dat.min)
			println(dat.max)
			println(dat.total)*/
			continue
		}
		for i := 0; i < 20; i++ {
			p := pctls[i]
			fmt.Fprintf(w, "%shist_%s_pctl_%d %d\n", prefix, name, i*5, p)
		}
		fmt.Fprintf(w, "%shist_%s_pctl_%d %d\n", prefix, name, 99, pctls[20])
		fmt.Fprintf(w, "%shist_%s_pctl_%d %d\n", prefix, name, 100, pctls[21])
	}

	//////////////////////////
	// Bucketized histograms
	//////////////////////////
	// Buckets are based on the generated buckets from the Spectator library
	// https://github.com/Netflix/spectator/blob/master/spectator-api/src/main/java/com/netflix/spectator/api/histogram/PercentileBuckets.java#L64
	bhists := getAllBucketHistograms()
	for name, bh := range bhists {
		for i := 0; i < numAtlasBuckets; i++ {
			fmt.Fprintf(w, "%sbhist_%s_bucket_%d %d\n", prefix, name, bucketValues[i], bh[i])
		}
	}

	//////////////////////////
	// Counters
	//////////////////////////
	ctrs := getAllCounters()
	for name, val := range ctrs {
		fmt.Fprintf(w, "%s%s %d\n", prefix, name, val)
	}

	//////////////////////////
	// Gauges
	//////////////////////////
	intg, floatg := getAllGauges()
	for name, val := range intg {
		fmt.Fprintf(w, "%s%s %d\n", prefix, name, val)
	}
	for name, val := range floatg {
		fmt.Fprintf(w, "%s%s %f\n", prefix, name, val)
	}

	//////////////////////////
	// Gauge Callbacks
	//////////////////////////
	intg, floatg = getAllCallbackGauges()
	for name, val := range intg {
		fmt.Fprintf(w, "%s%s %d\n", prefix, name, val)
	}
	for name, val := range floatg {
		fmt.Fprintf(w, "%s%s %f\n", prefix, name, val)
	}
}

// Percentiles go by 5% percentile steps from min to max. We report all of them even though it's
// likely only min, 25th, 50th, 75th, 95th, 99th, and max will be used. It's assumed the metric
// poller that is consuming this output will choose to only report to the metrics system what it
// considers useful information.
//
// Slice layout:
//  [0]: min (0th)
//  [1]: 5th
//  [n]: 5n
//  [19]: 95th
//  [20]: 99th
//  [21]: max (100th)
func hdatPercentiles(dat hdat) ([22]uint64, bool) {
	buf := dat.buf
	kept := dat.kept

	var pctls [22]uint64

	if kept == 0 {
		return pctls, false
	}
	if kept < uint64(len(buf)) {
		buf = buf[:kept]
	}

	sort.Sort(uint64slice(buf))

	// Take care of 0th and 100th specially
	pctls[0] = dat.min
	pctls[21] = dat.max

	// 5th - 95th
	for i := 1; i < 20; i++ {
		idx := len(buf) * i / 20
		pctls[i] = buf[idx]
	}

	// Add 99th
	idx := len(buf) * 99 / 100
	pctls[20] = buf[idx]

	return pctls, true
}

func pausePercentiles(pauses []uint64, ngc uint32) []uint64 {
	if ngc < uint32(len(pauses)) {
		pauses = pauses[:ngc]
	}

	sort.Sort(uint64slice(pauses))

	pctls := make([]uint64, 22)

	// Take care of 0th and 100th specially
	pctls[0] = pauses[0]
	pctls[21] = pauses[len(pauses)-1]

	// 5th - 95th
	for i := 1; i < 20; i++ {
		idx := len(pauses) * i / 20
		pctls[i] = pauses[idx]
	}

	// Add 99th
	idx := len(pauses) * 99 / 100
	pctls[20] = pauses[idx]

	return pctls
}

type uint64slice []uint64

func (u uint64slice) Len() int           { return len(u) }
func (u uint64slice) Swap(i, j int)      { u[i], u[j] = u[j], u[i] }
func (u uint64slice) Less(i, j int) bool { return u[i] < u[j] }
<|endoftext|>


<commit_before>// Tideland Go REST Server Library - REST
//
// Copyright (C) 2009-2016 Frank Mueller / Tideland / Oldenburg / Germany
//
// All rights reserved. Use of this source code is governed
// by the new BSD license.

// The Tideland Go REST Server Library provides the package rest for the
// implementation of servers with a RESTful API. The business has to
// be implemented in types fullfilling the ResourceHandler interface.
// This basic interface only allows the initialization of the handler.
// More interesting are the other interfaces like GetResourceHandler
// which defines the Get() method for the HTTP request method GET.
// Others are for PUT, POST, HEAD, PATCH, DELETE, and OPTIONS. Their
// according methods get a Job as argument. It provides convenient
// helpers for the processing of the job.
//
// The processing methods return two values: a boolean and an error.
// The latter is pretty clear, it signals a job processing error. The
// boolean is more interesting. Registering a handler is based on a
// domain and a resource. The URL
//
//     /<DOMAIN>/<RESOURCE>
//
// leads to a handler, or even better, to a list of handlers. All
// are used as long as the returned boolean value is true. E.g. the
// first handler can check the authentication, the second one the
// authorization, and the third one does the business. Additionally
// the URL
//
//     /<DOMAIN>/<RESOURCE>/<ID>
//
// provides the resource identifier via Job.ResourceID().
//
// The handlers then are deployed to the Multiplexer which implements
// the Handler interface of the net/http package. So the typical order
// is
//
//     mux := rest.NewMultiplexer()
//     mux.Register("domain", "resource-type-a", NewTypeAHandler("foo"))
//     mux.Register("domain", "resource-type-b", NewTypeBHandler("bar"))
//     mux.Register("admin", "user", NewUserManagementHandler())
//     http.ListenAndServe(":8000", mux)
//
// Additionally further handlers can be registered or running once
// removed during runtime.
package rest

//--------------------
// IMPORTS
//--------------------

import (
	"github.com/tideland/golib/version"
)

//--------------------
// VERSION
//--------------------

// PackageVersion returns the version of the version package.
func PackageVersion() version.Version {
	return version.New(2, 0, 0, "beta", "2016-09-12")
}

// EOF
<commit_msg>Added more documentation<commit_after>// Tideland Go REST Server Library - REST
//
// Copyright (C) 2009-2016 Frank Mueller / Tideland / Oldenburg / Germany
//
// All rights reserved. Use of this source code is governed
// by the new BSD license.

// The Tideland Go REST Server Library provides the package rest for the
// implementation of servers with a RESTful API. The business has to
// be implemented in types fullfilling the ResourceHandler interface.
// This basic interface only allows the initialization of the handler.
// More interesting are the other interfaces like GetResourceHandler
// which defines the Get() method for the HTTP request method GET.
// Others are for PUT, POST, HEAD, PATCH, DELETE, and OPTIONS. Their
// according methods get a Job as argument. It provides convenient
// helpers for the processing of the job.
//
//     type myHandler struct {
//         id string
//     }
//
//     func NewMyHandler(id string) rest.ResourceHandler {
//         return &myHandler{id}
//     }
//
//     func (h *myHandler) ID() string {
//         return h.id
//     }
//
//     func (h *myHandler) Init(env rest.Environment, domain, resource string) error {
//         // Nothing to do in this example.
//         return nil
//     }
//
//     // Get handles reading of resources, here simplified w/o
//     // error handling.
//     func (h *myHandler) Get(job rest.Job) (bool, error) {
//         id := job.ResourceID()
//         if id == "" {
//            all := model.GetAllData()
//            job.JSON(true).Write(all)
//            return true, nil
//         }
//         one := model.GetOneData(id)
//         job.JSON(true).Write(one)
//         return true, nil
//     }
//
// The processing methods return two values: a boolean and an error.
// The latter is pretty clear, it signals a job processing error. The
// boolean is more interesting. Registering a handler is based on a
// domain and a resource. The URL
//
//     /<DOMAIN>/<RESOURCE>
//
// leads to a handler, or even better, to a list of handlers. All
// are used as long as the returned boolean value is true. E.g. the
// first handler can check the authentication, the second one the
// authorization, and the third one does the business. Additionally
// the URL
//
//     /<DOMAIN>/<RESOURCE>/<ID>
//
// provides the resource identifier via Job.ResourceID().
//
// The handlers then are deployed to the Multiplexer which implements
// the Handler interface of the net/http package. So the typical order
// is
//
//     mux := rest.NewMultiplexer()
//     mux.Register("domain", "resource-type-a", NewTypeAHandler("foo"))
//     mux.Register("domain", "resource-type-b", NewTypeBHandler("bar"))
//     mux.Register("admin", "user", NewUserManagementHandler())
//     http.ListenAndServe(":8000", mux)
//
// Additionally further handlers can be registered or running once
// removed during runtime.
package rest

//--------------------
// IMPORTS
//--------------------

import (
	"github.com/tideland/golib/version"
)

//--------------------
// VERSION
//--------------------

// PackageVersion returns the version of the version package.
func PackageVersion() version.Version {
	return version.New(2, 0, 0, "beta", "2016-09-12")
}

// EOF
<|endoftext|>


<commit_before>package main

import "testing"

func TestExistPath(t *testing.T) {
	ep := existsPath(".git")
	if !ep {
		t.Error("Expected true, got ", ep)
	}

	ep = existsPath("foobar")
	if ep {
		t.Error("Expected false, got ", ep)
	}
}
<commit_msg>separated test<commit_after>package main

import "testing"

func TestExistPath(t *testing.T) {
	ep := existsPath(".git")
	if !ep {
		t.Error("Expected true, got ", ep)
	}
}

func TestDoNotExistPath(t *testing.T) {
	ep := existsPath("foobar")
	if ep {
		t.Error("Expected false, got ", ep)
	}
}
<|endoftext|>


<commit_before>package todolite

import (
	"encoding/json"
	"fmt"
	"io"
	"net/http"

	"github.com/couchbaselabs/logg"
	"github.com/tleyden/go-couch"
	openocr "github.com/tleyden/open-ocr-client"
)

type TodoLiteApp struct {
	DatabaseURL string
	OpenOCRURL  string
	Database    couch.Database
}

func NewTodoLiteApp(DatabaseURL, openOCRURL string) *TodoLiteApp {
	return &TodoLiteApp{
		DatabaseURL: DatabaseURL,
		OpenOCRURL:  openOCRURL,
	}
}

func (t *TodoLiteApp) InitApp() error {
	db, err := couch.Connect(t.DatabaseURL)
	if err != nil {
		logg.LogPanic("Error connecting to db: %v", err)
		return err
	}
	t.Database = db
	return nil
}

func (t TodoLiteApp) FollowChangesFeed(since interface{}) {

	handleChange := func(reader io.Reader) interface{} {
		logg.LogTo("TODOLITE", "handleChange() callback called")
		changes, err := decodeChanges(reader)
		if err == nil {
			logg.LogTo("TODOLITE", "changes: %v", changes)

			t.processChanges(changes)

			since = changes.LastSequence

		} else {
			logg.LogTo("TODOLITE", "error decoding changes: %v", err)

		}

		logg.LogTo("TODOLITE", "returning since: %v", since)
		return since

	}

	options := changes{"since": since}
	options["feed"] = "longpoll"
	t.Database.Changes(handleChange, options)

}

func (t TodoLiteApp) processChanges(changes couch.Changes) {

	for _, change := range changes.Results {
		logg.LogTo("TODOLITE", "change: %v", change)

		if change.Deleted {
			logg.LogTo("TODOLITE", "change was deleted, skipping")
			continue
		}

		todoItem := TodoItem{}
		err := t.Database.Retrieve(change.Id, &todoItem)
		if err != nil {
			errMsg := fmt.Errorf("Didn't retrieve: %v - %v", change.Id, err)
			logg.LogError(errMsg)
			continue
		}
		logg.LogTo("TODOLITE", "todo item: %+v", todoItem)

		if todoItem.OcrDecoded != "" {
			logg.LogTo("TODOLITE", "%v already ocr decoded, skipping", change.Id)
			continue
		}

		attachmentUrl := todoItem.AttachmentUrl(t.Database.DBURL())
		if attachmentUrl == "" {
			logg.LogTo("TODOLITE", "todo item has no attachment, skipping")
			continue
		}
		logg.LogTo("TODOLITE", "OCR Decoding: %v", attachmentUrl)

		ocrDecoded, err := t.ocrDecode(attachmentUrl)
		if err != nil {
			errMsg := fmt.Errorf("OCR failed: %+v - %v", todoItem, err)
			logg.LogError(errMsg)
			ocrDecoded = "failed"
		}
		err = t.updateTodoItemWithOcr(todoItem, ocrDecoded)
		if err != nil {
			errMsg := fmt.Errorf("Update failed: %+v - %v", todoItem, err)
			logg.LogError(errMsg)
			continue
		}

	}

}

func (t TodoLiteApp) ocrDecode(attachmentUrl string) (string, error) {

	openOcrClient := openocr.NewHttpClient(t.OpenOCRURL)

	res, err := http.Get(attachmentUrl)
	if err != nil {
		errMsg := fmt.Errorf("Unable to open reader for %s: %s", attachmentUrl, err)
		logg.LogError(errMsg)
		return "", errMsg
	}
	defer res.Body.Close()

	ocrRequest := openocr.OcrRequest{
		EngineType: openocr.ENGINE_TESSERACT,
	}

	ocrDecoded, err := openOcrClient.DecodeImageReader(res.Body, ocrRequest)

	if err != nil {
		return "", err
	}
	return ocrDecoded, nil
}

func (t TodoLiteApp) updateTodoItemWithOcr(i TodoItem, ocrDecoded string) error {
	i.OcrDecoded = ocrDecoded
	revid, err := t.Database.Edit(i)
	logg.LogTo("TODOLITE", "new revid: %v", revid)
	return err

}

func decodeChanges(reader io.Reader) (couch.Changes, error) {

	changes := couch.Changes{}
	decoder := json.NewDecoder(reader)
	err := decoder.Decode(&changes)
	if err != nil {
		logg.LogTo("TODOLITE", "Err decoding changes: %v", err)
	}
	return changes, err

}
<commit_msg>Follow changes feed from the end if no since given<commit_after>package todolite

import (
	"encoding/json"
	"fmt"
	"io"
	"net/http"

	"github.com/couchbaselabs/logg"
	"github.com/tleyden/go-couch"
	openocr "github.com/tleyden/open-ocr-client"
)

type TodoLiteApp struct {
	DatabaseURL string
	OpenOCRURL  string
	Database    couch.Database
}

func NewTodoLiteApp(DatabaseURL, openOCRURL string) *TodoLiteApp {
	return &TodoLiteApp{
		DatabaseURL: DatabaseURL,
		OpenOCRURL:  openOCRURL,
	}
}

func (t *TodoLiteApp) InitApp() error {
	db, err := couch.Connect(t.DatabaseURL)
	if err != nil {
		logg.LogPanic("Error connecting to db: %v", err)
		return err
	}
	t.Database = db
	return nil
}

func (t TodoLiteApp) FollowChangesFeed(startingSince string) {

	handleChange := func(reader io.Reader) interface{} {
		logg.LogTo("TODOLITE", "handleChange() callback called")
		changes, err := decodeChanges(reader)
		if err != nil {
			logg.LogTo("TODOLITE", "error decoding changes: %v", err)
			return nil // stop changes feed
		}

		logg.LogTo("TODOLITE", "changes: %v", changes)

		t.processChanges(changes)

		since := changes.LastSequence
		logg.LogTo("TODOLITE", "returning since: %v", since)

		return since

	}

	options := changes{}
	if startingSince != "" {
		logg.LogTo("TODOLITE", "startingSince not empty: %v", startingSince)
		options["since"] = startingSince
	} else {
		// find the sequence of most recent change
		lastSequence, err := t.Database.LastSequence()
		if err != nil {
			logg.LogPanic("Error getting LastSequence: %v", err)
			return
		}
		options["since"] = lastSequence
	}

	options["feed"] = "longpoll"
	logg.LogTo("TODOLITE", "Following changes feed: %+v", options)
	t.Database.Changes(handleChange, options)

}

func (t TodoLiteApp) processChanges(changes couch.Changes) {

	for _, change := range changes.Results {
		logg.LogTo("TODOLITE", "change: %v", change)

		if change.Deleted {
			logg.LogTo("TODOLITE", "change was deleted, skipping")
			continue
		}

		todoItem := TodoItem{}
		err := t.Database.Retrieve(change.Id, &todoItem)
		if err != nil {
			errMsg := fmt.Errorf("Didn't retrieve: %v - %v", change.Id, err)
			logg.LogError(errMsg)
			continue
		}
		logg.LogTo("TODOLITE", "todo item: %+v", todoItem)

		if todoItem.OcrDecoded != "" {
			logg.LogTo("TODOLITE", "%v already ocr decoded, skipping", change.Id)
			continue
		}

		attachmentUrl := todoItem.AttachmentUrl(t.Database.DBURL())
		if attachmentUrl == "" {
			logg.LogTo("TODOLITE", "todo item has no attachment, skipping")
			continue
		}
		logg.LogTo("TODOLITE", "OCR Decoding: %v", attachmentUrl)

		ocrDecoded, err := t.ocrDecode(attachmentUrl)
		if err != nil {
			errMsg := fmt.Errorf("OCR failed: %+v - %v", todoItem, err)
			logg.LogError(errMsg)
			ocrDecoded = "failed"
		}
		err = t.updateTodoItemWithOcr(todoItem, ocrDecoded)
		if err != nil {
			errMsg := fmt.Errorf("Update failed: %+v - %v", todoItem, err)
			logg.LogError(errMsg)
			continue
		}

	}

}

func (t TodoLiteApp) ocrDecode(attachmentUrl string) (string, error) {

	openOcrClient := openocr.NewHttpClient(t.OpenOCRURL)

	res, err := http.Get(attachmentUrl)
	if err != nil {
		errMsg := fmt.Errorf("Unable to open reader for %s: %s", attachmentUrl, err)
		logg.LogError(errMsg)
		return "", errMsg
	}
	defer res.Body.Close()

	ocrRequest := openocr.OcrRequest{
		EngineType: openocr.ENGINE_TESSERACT,
	}

	ocrDecoded, err := openOcrClient.DecodeImageReader(res.Body, ocrRequest)

	if err != nil {
		return "", err
	}
	return ocrDecoded, nil
}

func (t TodoLiteApp) updateTodoItemWithOcr(i TodoItem, ocrDecoded string) error {
	i.OcrDecoded = ocrDecoded
	revid, err := t.Database.Edit(i)
	logg.LogTo("TODOLITE", "new revid: %v", revid)
	return err

}

func decodeChanges(reader io.Reader) (couch.Changes, error) {

	changes := couch.Changes{}
	decoder := json.NewDecoder(reader)
	err := decoder.Decode(&changes)
	if err != nil {
		logg.LogTo("TODOLITE", "Err decoding changes: %v", err)
	}
	return changes, err

}
<|endoftext|>


<commit_before>package runner

import (
    "net/http"
    "net/url"
    "log"
    "flag"
)

type ReportPayload struct {
    path string
    data url.Values
}

type Reporter struct {
    publishUri string
    publishChannel chan ReportPayload
    shutdownChannel chan bool
}

func transportSend(r *Reporter) {
    for req := range r.publishChannel {
        path := r.publishUri + req.path
        log.Printf("[reporter] POST %s data: %s", path, req.data)
        _, err := http.PostForm(path, req.data)
        // TODO: Retry on error
        if err != nil {
            log.Printf("[reporter] POST %s failed, err: %s", path, err)
        }
    }
    r.shutdownChannel <- true
}

func NewReporter(publishUri string) *Reporter {
    log.Printf("[reporter] Construct reporter with publish uri: %s", publishUri)
    r := &Reporter{}
    r.publishUri = publishUri
    maxPendingReports := 64
    if f := flag.Lookup("max_pending_reports"); f != nil {
        newVal, ok := f.Value.(flag.Getter)
        if ok {
            maxPendingReports = newVal.Get().(int)
        }
    }
    r.publishChannel = make(chan ReportPayload, maxPendingReports)
    r.shutdownChannel = make(chan bool)
    go transportSend(r)
    return r
}

func (r *Reporter) PushStatus(cId string, status string) {
    form := make(url.Values)
    form.Add("status", status)
    r.publishChannel <- ReportPayload {cId + "/status", form}
}

func (r *Reporter) PushLogChunk(cId string, l LogChunk) {
    form := make(url.Values)
    form.Add("source", l.Source)
    form.Add("offset", string(l.Offset))
    form.Add("text", string(l.Payload))
    r.publishChannel <- ReportPayload {cId + "/logappend", form}
}

func (r *Reporter) Shutdown() {
    log.Print("[reporter] Shutdown")
    close(r.publishChannel)
    <-r.shutdownChannel
    close(r.shutdownChannel)
}
<commit_msg>Update comment, fix detached head<commit_after>package runner

import (
    "net/http"
    "net/url"
    "log"
    "flag"
)

type ReportPayload struct {
    path string
    data url.Values
}

type Reporter struct {
    publishUri string
    publishChannel chan ReportPayload
    shutdownChannel chan bool
}

func transportSend(r *Reporter) {
    for req := range r.publishChannel {
        path := r.publishUri + req.path
        log.Printf("[reporter] POST %s data: %s", path, req.data)
        _, err := http.PostForm(path, req.data)
        // TODO: Retry on error
        // Add a time based retry login, try @ now + x_ms
        if err != nil {
            log.Printf("[reporter] POST %s failed, err: %s", path, err)
        }
    }
    r.shutdownChannel <- true
}

func NewReporter(publishUri string) *Reporter {
    log.Printf("[reporter] Construct reporter with publish uri: %s", publishUri)
    r := &Reporter{}
    r.publishUri = publishUri
    maxPendingReports := 64
    if f := flag.Lookup("max_pending_reports"); f != nil {
        newVal, ok := f.Value.(flag.Getter)
        if ok {
            maxPendingReports = newVal.Get().(int)
        }
    }
    r.publishChannel = make(chan ReportPayload, maxPendingReports)
    r.shutdownChannel = make(chan bool)
    go transportSend(r)
    return r
}

func (r *Reporter) PushStatus(cId string, status string) {
    form := make(url.Values)
    form.Add("status", status)
    r.publishChannel <- ReportPayload {cId + "/status", form}
}

func (r *Reporter) PushLogChunk(cId string, l LogChunk) {
    form := make(url.Values)
    form.Add("source", l.Source)
    form.Add("offset", string(l.Offset))
    form.Add("text", string(l.Payload))
    r.publishChannel <- ReportPayload {cId + "/logappend", form}
}

func (r *Reporter) Shutdown() {
    log.Print("[reporter] Shutdown")
    close(r.publishChannel)
    <-r.shutdownChannel
    close(r.shutdownChannel)
}
<|endoftext|>