<commit_before>/*
Copyright 2019 The HAProxy Ingress Controller Authors.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package utils

import (
	"fmt"
	"strings"
	"time"
)

// Timer ...
type Timer struct {
	Start time.Time
	Ticks []*Tick
}

// Tick ...
type Tick struct {
	Event string
	When  time.Time
}

// NewTimer ...
func NewTimer() *Timer {
	return &Timer{
		Start: time.Now(),
	}
}

// Tick ...
func (t *Timer) Tick(eventLabel string) {
	t.Ticks = append(t.Ticks, &Tick{
		Event: eventLabel,
		When:  time.Now(),
	})
}

// AsString ...
func (t *Timer) AsString(totalLabel string) string {
	out := make([]string, 0, len(t.Ticks)+1)
	last := t.Start
	for _, tick := range t.Ticks {
		out = append(out, fmt.Sprintf("%s=%v", tick.Event, tick.When.Sub(last)))
		last = tick.When
	}
	if totalLabel != "" {
		out = append(out, fmt.Sprintf("%s=%v", totalLabel, last.Sub(t.Start)))
	}
	return strings.Join(out, " ")
}
<commit_msg>Change elapsed time unit to always use millisecond<commit_after>/*
Copyright 2019 The HAProxy Ingress Controller Authors.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package utils

import (
	"fmt"
	"strings"
	"time"
)

// Timer ...
type Timer struct {
	Start time.Time
	Ticks []*Tick
}

// Tick ...
type Tick struct {
	Event string
	When  time.Time
}

// NewTimer ...
func NewTimer() *Timer {
	return &Timer{
		Start: time.Now(),
	}
}

// Tick ...
func (t *Timer) Tick(eventLabel string) {
	t.Ticks = append(t.Ticks, &Tick{
		Event: eventLabel,
		When:  time.Now(),
	})
}

// AsString ...
func (t *Timer) AsString(totalLabel string) string {
	out := make([]string, 0, len(t.Ticks)+1)
	last := t.Start
	for _, tick := range t.Ticks {
		out = append(out, fmt.Sprintf("%s=%fms", tick.Event, tick.When.Sub(last).Seconds()*1000))
		last = tick.When
	}
	if totalLabel != "" {
		out = append(out, fmt.Sprintf("%s=%fms", totalLabel, last.Sub(t.Start).Seconds()*1000))
	}
	return strings.Join(out, " ")
}
<|endoftext|>


<commit_before>package password

import (
	"crypto/rand"
	"encoding/base64"
	"encoding/json"
	"errors"
	"net/http"
	"time"

	jwt "github.com/dgrijalva/jwt-go"
	"golang.org/x/crypto/bcrypt"
	"golang.org/x/net/context"
)

var (
	signingKey = genRandBytes()
	// ErrInvalidSigningMethod is the error returned when a token's signature
	// does match the signature used to sign the token header.
	ErrInvalidSigningMethod = errors.New("Invalid signing method")
	// ErrTokenInvalid means the signature didn't match.
	ErrTokenInvalid = errors.New("Token isn't valid")
)

// Authenticator is an interface for storing and retrieving hashed passwords
type Authenticator interface {
	Store(id string, hashedPassword string) (string, error)
	Retrieve(id string) (string, error)
}

// New hashes and salts a given plaintext password using the bcrypt algorithm,
// and stores it. The returned string is the generated key used to identify
// that id/secret combination. It is typically the primary used to retrieve
// that entry in the database.
func New(username string, password string, a Authenticator) (string, error) {
	hashedPassword, err := bcrypt.GenerateFromPassword([]byte(password), bcrypt.DefaultCost)
	if err != nil {
		return "", err // couldn't run bcrypt
	}

	id, err := a.Store(username, string(hashedPassword))
	if err != nil {
		return "", err // couldn't store pwd in db
	}

	return id, nil
}

// Compare compares the stored hashed password with the password provided by
// the user. If they match, it returns a JSON web token.
func Compare(id string, password string, a Authenticator) (string, error) {
	hashedPassword, err := a.Retrieve(id)
	if err != nil {
		return "", err // failed to retrieve password
	}

	err = bcrypt.CompareHashAndPassword([]byte(hashedPassword), []byte(password))
	if err != nil {
		return "", err // passwords didn't match
	}

	return genToken(id)
}

// Authenticate runs `Compare` against an authenticator interface, and responds
// with a JSON web token in the body of the request.
func Authenticate(id string, password string, w http.ResponseWriter, a Authenticator) {
	tokStr, err := Compare(id, password, a)
	if err != nil {
		http.Error(w, err.Error(), http.StatusInternalServerError)
		return
	}
	json.NewEncoder(w).Encode(map[string]string{"token": tokStr})
}

// Protected is middleware that checks to see if the incoming request has a
// valid JSON web token. If it does, it executes the next `http.HandlerFunc`,
// and passes it a `context.Context` containing a way to identify the current
// user.
type Protected func(ctx context.Context, w http.ResponseWriter, r *http.Request)

func (fn Protected) ServeHTTP(w http.ResponseWriter, r *http.Request) {
	tok, err := jwt.ParseFromRequest(r, func(token *jwt.Token) (interface{}, error) {
		_, ok := token.Method.(*jwt.SigningMethodHMAC)
		if ok == false {
			return nil, ErrInvalidSigningMethod
		}
		return signingKey, nil
	})
	if err != nil {
		// might wanna use switch statement
		http.Error(w, err.Error(), http.StatusInternalServerError)
		return
	}

	if tok.Valid != true {
		http.Error(w, ErrTokenInvalid.Error(), http.StatusUnauthorized)
	}

	id := tok.Claims["sub"]
	ctx := context.WithValue(context.Background(), "id", id)

	fn(ctx, w, r)
}

// SetSigningKey allows you to override the default HMAC signing key with one
// of your own. Every time this package is imported, a signing key is set
// randomly. That means that in between restarts, a new key is set, so you'd
// no longer be able to verify JSON web tokens created with that key. In order
// to reuse the signing key, you must set it yourself. Just call this function
// before creating any tokens, and you'll be good to go.
func SetSigningKey(key []byte) {
	signingKey = key
}

func genToken(id string) (string, error) {
	jwt := jwt.New(jwt.SigningMethodHS256)
	expTime := time.Now().Add(time.Hour * 72).Unix()

	jwt.Claims["sub"] = id
	jwt.Claims["exp"] = expTime
	jwt.Claims["iat"] = time.Now().Unix()

	tokStr, err := jwt.SignedString(signingKey)
	if err != nil {
		return "", err // failed to sign token
	}

	return tokStr, nil
}

func genRandBytes() []byte {
	// Use 32 bytes (256 bits) to satisfy the requirement for the HMAC key
	// length.
	b := make([]byte, 32)
	_, err := rand.Read(b)
	if err != nil {
		// If this errors, it means that something is wrong the system's
		// CSPRNG, which indicates a critical operating system failure. Panic
		// and crash here
		panic(err)
	}
	return []byte(base64.URLEncoding.EncodeToString(b))
}
<commit_msg>Improve GoDoc documentation<commit_after>/*
Package password implements a simple JSON web token based authentication
system.

Background

The package revolves around the `password.Authenticator` interface. This
interface implements only two methods: one for storing passwords, and one for
retrieving them. This lets you use any backend to store your users, whether
that be an in-memory store, Redis, Postgres, or something else altogether.

Usage

The functions defined in this library are designed to make it as easy as
possible to create and authenticate users. They are all designed to be used
with HTTP handlers:

	// Grab the username and password from the request, and create a new user in the user store
	// with those values
	http.HandleFunc("/signup", func(w http.ResponseWriter, r *http.Request) {
		username := r.FormValue("Username")
		password := r.FormValue("Password")
		id, _ := password.New(username, password, UserStore)
		w.Write([]byte("New user: "+id))
	})
	...
	// Sign in using a username and password. This will respond with a JSON web token if the user
	// authenticates successfully
	http.HandleFunc("/signin", func(w http.ResponseWriter, r *http.Request) {
		username := r.FormValue("Username")
		password := r.FormValue("Password")
		password.Authenticate(username, password, w, UserStore)
	})
	...
	// Respond with the user's username. If they don't have a valid JSON web token, then this
	// request will fail, saying the client is authorized
	http.Handle("/whoami", password.Protect(func(ctx context.Context, w http.ResponseWriter, r *http.Request) {
		username := ctx.Value("id")
		w.Write([]byte("Your username is "+username))
	})

For a reference implementation of the `password.Authenticator` interface, see
the example in the GitHub repository.
*/
package password

import (
	"crypto/rand"
	"encoding/base64"
	"encoding/json"
	"errors"
	"net/http"
	"time"

	jwt "github.com/dgrijalva/jwt-go"
	"golang.org/x/crypto/bcrypt"
	"golang.org/x/net/context"
)

var (
	signingKey = genRandBytes()
	// ErrInvalidSigningMethod is the error returned when a token's signature
	// does match the signature used to sign the token header.
	ErrInvalidSigningMethod = errors.New("Invalid signing method")
	// ErrTokenInvalid means the signature didn't match.
	ErrTokenInvalid = errors.New("Token isn't valid")
)

// Authenticator is an interface for storing and retrieving hashed passwords
type Authenticator interface {
	Store(id string, hashedPassword string) (string, error)
	Retrieve(id string) (string, error)
}

// New hashes and salts a given plaintext password using the bcrypt algorithm,
// and stores it. The returned string is the generated key used to identify
// that id/secret combination. It is typically the primary used to retrieve
// that entry in the database.
func New(username string, password string, a Authenticator) (string, error) {
	hashedPassword, err := bcrypt.GenerateFromPassword([]byte(password), bcrypt.DefaultCost)
	if err != nil {
		return "", err // couldn't run bcrypt
	}

	id, err := a.Store(username, string(hashedPassword))
	if err != nil {
		return "", err // couldn't store pwd in db
	}

	return id, nil
}

// Compare compares the stored hashed password with the password provided by
// the user. If they match, it returns a JSON web token.
func Compare(id string, password string, a Authenticator) (string, error) {
	hashedPassword, err := a.Retrieve(id)
	if err != nil {
		return "", err // failed to retrieve password
	}

	err = bcrypt.CompareHashAndPassword([]byte(hashedPassword), []byte(password))
	if err != nil {
		return "", err // passwords didn't match
	}

	return genToken(id)
}

// Authenticate runs `Compare` against an authenticator interface, and responds
// with a JSON web token in the body of the request.
func Authenticate(id string, password string, w http.ResponseWriter, a Authenticator) {
	tokStr, err := Compare(id, password, a)
	if err != nil {
		http.Error(w, err.Error(), http.StatusInternalServerError)
		return
	}
	json.NewEncoder(w).Encode(map[string]string{"token": tokStr})
}

// Protected is middleware that checks to see if the incoming request has a
// valid JSON web token. If it does, it executes the next `http.HandlerFunc`,
// and passes it a `context.Context` containing a way to identify the current
// user.
type Protected func(ctx context.Context, w http.ResponseWriter, r *http.Request)

func (fn Protected) ServeHTTP(w http.ResponseWriter, r *http.Request) {
	tok, err := jwt.ParseFromRequest(r, func(token *jwt.Token) (interface{}, error) {
		_, ok := token.Method.(*jwt.SigningMethodHMAC)
		if ok == false {
			return nil, ErrInvalidSigningMethod
		}
		return signingKey, nil
	})
	if err != nil {
		// might wanna use switch statement
		http.Error(w, err.Error(), http.StatusInternalServerError)
		return
	}

	if tok.Valid != true {
		http.Error(w, ErrTokenInvalid.Error(), http.StatusUnauthorized)
	}

	id := tok.Claims["sub"]
	ctx := context.WithValue(context.Background(), "id", id)

	fn(ctx, w, r)
}

// SetSigningKey allows you to override the default HMAC signing key with one
// of your own. Every time this package is imported, a signing key is set
// randomly. That means that in between restarts, a new key is set, so you'd
// no longer be able to verify JSON web tokens created with that key. In order
// to reuse the signing key, you must set it yourself. Just call this function
// before creating any tokens, and you'll be good to go.
func SetSigningKey(key []byte) {
	signingKey = key
}

func genToken(id string) (string, error) {
	jwt := jwt.New(jwt.SigningMethodHS256)
	expTime := time.Now().Add(time.Hour * 72).Unix()

	jwt.Claims["sub"] = id
	jwt.Claims["exp"] = expTime
	jwt.Claims["iat"] = time.Now().Unix()

	tokStr, err := jwt.SignedString(signingKey)
	if err != nil {
		return "", err // failed to sign token
	}

	return tokStr, nil
}

func genRandBytes() []byte {
	// Use 32 bytes (256 bits) to satisfy the requirement for the HMAC key
	// length.
	b := make([]byte, 32)
	_, err := rand.Read(b)
	if err != nil {
		// If this errors, it means that something is wrong the system's
		// CSPRNG, which indicates a critical operating system failure. Panic
		// and crash here
		panic(err)
	}
	return []byte(base64.URLEncoding.EncodeToString(b))
}
<|endoftext|>


<commit_before>/* Copyright (c) 2014, Daniel Martí <mvdan@mvdan.cc> */
/* See LICENSE for licensing information */

package main

import (
	"errors"
	"flag"
	"fmt"
	"html/template"
	"io"
	"log"
	"math/rand"
	"net/http"
	"os"
	"path"
	"path/filepath"
	"regexp"
	"strconv"
	"strings"
	"sync"
	"time"
)

const (
	indexTmpl = "index.html"
	formTmpl  = "form.html"
	chars     = "abcdefghijklmnopqrstuvwxyz0123456789"
	randTries = 10

	// GET error messages
	invalidId     = "Invalid paste id."
	pasteNotFound = "Paste doesn't exist."
	unknownError  = "Something went terribly wrong."
	// POST error messages
	missingForm = "Paste could not be found inside the posted form."
)

var (
	siteUrl, listen, dataDir string
	lifeTime                 time.Duration
	maxSizeStr               string
	idSize                   int
	maxSize                  ByteSize

	validId       *regexp.Regexp
	regexByteSize = regexp.MustCompile(`^([\d\.]+)\s*([KM]?B|[BKM])$`)
	indexTemplate *template.Template
	formTemplate  *template.Template
	data          = struct {
		sync.RWMutex
		m map[Id]PasteInfo
	}{m: make(map[Id]PasteInfo)}
	customRand *rand.Rand
)

func init() {
	flag.StringVar(&siteUrl, "u", "http://localhost:8080", "URL of the site")
	flag.StringVar(&listen, "l", "localhost:8080", "Host and port to listen to")
	flag.StringVar(&dataDir, "d", "data", "Directory to store all the pastes in")
	flag.DurationVar(&lifeTime, "t", 12*time.Hour, "Lifetime of the pastes (units: s,m,h)")
	flag.StringVar(&maxSizeStr, "s", "1M", "Maximum size of POSTs in bytes (units: B,K,M)")
	flag.IntVar(&idSize, "i", 8, "Size of the paste ids (between 6 and 256)")
	validId = regexp.MustCompile("^[a-zA-Z0-9]{" + strconv.Itoa(idSize) + "}$")
	customRand = rand.New(rand.NewSource(time.Now().UnixNano()))
}

type PasteInfo struct {
	ModTime     time.Time
	DeathTime   time.Time
	Size        ByteSize
	Etag        string
	ContentType string
}

type Id string

func IdFromPath(idPath string) (Id, error) {
	parts := strings.Split(idPath, string(filepath.Separator))
	if len(parts) != 3 {
		return "", errors.New("Found invalid number of directories at " + idPath)
	}
	rawId := parts[0] + parts[1] + parts[2]
	if !validId.MatchString(rawId) {
		return "", errors.New("Found invalid id " + rawId)
	}
	return Id(rawId), nil
}

func RandomId() (Id, error) {
	s := make([]byte, idSize)
	var id Id
	data.RLock()
	defer data.RUnlock()
	for try := 0; try < randTries; try++ {
		var offset int = 0
	RandLoop:
		for {
			r := customRand.Int63()
			for i := 0; i < 8; i++ {
				randbyte := int(r&0xff) % len(chars)
				s[offset] = chars[randbyte]
				offset++
				if offset == idSize {
					break RandLoop
				}
				r >>= 8
			}
		}
		id = Id(s)
		if _, e := data.m[id]; !e {
			return id, nil
		}
	}
	return id, fmt.Errorf("Gave up trying to find an unused random id after %d tries", randTries)
}

func (id Id) String() string {
	return string(id)
}

func (id Id) Path() string {
	return path.Join(string(id[0:2]), string(id[2:4]), string(id[4:]))
}

func (id Id) EndLife() {
	data.Lock()
	defer data.Unlock()
	err := os.Remove(id.Path())
	if err == nil {
		delete(data.m, id)
		log.Printf("Removed paste: %s", id)
	} else {
		log.Printf("Could not end the life of %s: %s", id, err)
		id.EndLifeAfter(2 * time.Minute)
	}
}

func (id Id) EndLifeAfter(duration time.Duration) {
	timer := time.NewTimer(duration)
	go func() {
		<-timer.C
		id.EndLife()
	}()
}

type ByteSize int64

const (
	B ByteSize = 1 << (10 * iota)
	KB
	MB
)

func parseByteSize(str string) (ByteSize, error) {
	if !regexByteSize.MatchString(str) {
		return 0, errors.New("Could not parse size in bytes")
	}
	parts := regexByteSize.FindStringSubmatch(str)
	size, _ := strconv.ParseFloat(string(parts[1]), 64)

	switch string(parts[2]) {
	case "KB", "K":
		size *= float64(KB)
	case "MB", "M":
		size *= float64(MB)
	}
	return ByteSize(size), nil
}

func (b ByteSize) String() string {
	switch {
	case b >= MB:
		return fmt.Sprintf("%.2f MB", float64(b)/float64(MB))
	case b >= KB:
		return fmt.Sprintf("%.2f KB", float64(b)/float64(KB))
	}
	return fmt.Sprintf("%d B", b)
}

func handler(w http.ResponseWriter, r *http.Request) {
	var err error
	switch r.Method {
	case "GET":
		switch r.URL.Path {
		case "/":
			indexTemplate.Execute(w, struct {
				SiteUrl, LifeTime string
			}{
				siteUrl,
				fmt.Sprintf("%g hours and %g minutes", lifeTime.Hours(),
					lifeTime.Minutes()-lifeTime.Hours()*60),
			})
			return
		case "/form":
			formTemplate.Execute(w, struct{ SiteUrl string }{siteUrl})
			return
		}
		rawId := r.URL.Path[1:]
		if !validId.MatchString(rawId) {
			w.WriteHeader(http.StatusBadRequest)
			fmt.Fprintf(w, "%s\n", invalidId)
			return
		}
		id := Id(strings.ToLower(rawId))
		data.RLock()
		defer data.RUnlock()
		pasteInfo, e := data.m[id]
		if !e {
			w.WriteHeader(http.StatusNotFound)
			fmt.Fprintf(w, "%s\n", pasteNotFound)
			return
		}
		if inm := r.Header.Get("If-None-Match"); inm != "" {
			if pasteInfo.Etag == inm || inm == "*" {
				w.WriteHeader(http.StatusNotModified)
				return
			}
		}
		pastePath := id.Path()
		pasteFile, err := os.Open(pastePath)
		if err != nil {
			w.WriteHeader(http.StatusInternalServerError)
			fmt.Fprintf(w, "%s\n", unknownError)
			return
		}
		defer pasteFile.Close()
		w.Header().Set("Etag", pasteInfo.Etag)
		w.Header().Set("Content-Type", pasteInfo.ContentType)
		http.ServeContent(w, r, "", pasteInfo.ModTime, pasteFile)

	case "POST":
		r.Body = http.MaxBytesReader(w, r.Body, int64(maxSize))
		var content string
		if err = r.ParseMultipartForm(int64(maxSize)); err != nil {
			log.Printf("Could not parse POST multipart form: %s", err)
			w.WriteHeader(http.StatusBadRequest)
			fmt.Fprintf(w, "%s\n", err)
			return
		}
		if vs, found := r.Form["paste"]; found && len(vs[0]) > 0 {
			content = vs[0]
		} else {
			w.WriteHeader(http.StatusBadRequest)
			fmt.Fprintf(w, "%s\n", missingForm)
			return
		}
		var id Id
		if id, err = RandomId(); err == nil {
			data.Lock()
			defer data.Unlock()
		} else {
			log.Println(err)
			w.WriteHeader(http.StatusInternalServerError)
			fmt.Fprintf(w, "%s\n", unknownError)
			return
		}
		pastePath := id.Path()
		dir, _ := path.Split(pastePath)
		if err = os.MkdirAll(dir, 0700); err != nil {
			log.Printf("Could not create directories leading to %s: %s", pastePath, err)
			w.WriteHeader(http.StatusInternalServerError)
			fmt.Fprintf(w, "%s\n", unknownError)
			return
		}
		deathTime := time.Now().Add(lifeTime)
		id.EndLifeAfter(lifeTime)
		pasteFile, err := os.OpenFile(pastePath, os.O_WRONLY|os.O_CREATE|os.O_EXCL, 0600)
		if err != nil {
			log.Printf("Could not create new paste file %s: %s", pastePath, err)
			w.WriteHeader(http.StatusInternalServerError)
			fmt.Fprintf(w, "%s\n", unknownError)
			return
		}
		defer pasteFile.Close()
		b, err := io.WriteString(pasteFile, content)
		if err != nil {
			log.Printf("Could not write data into %s: %s", pastePath, err)
			w.WriteHeader(http.StatusInternalServerError)
			fmt.Fprintf(w, "%s\n", unknownError)
			return
		}
		writtenSize := ByteSize(b)
		pasteInfo := PasteInfo{
			ModTime:   time.Now(),
			DeathTime: deathTime,
			Size:      writtenSize,
		}
		pasteInfo.Etag = fmt.Sprintf("%d-%s", pasteInfo.ModTime.Unix(), id)
		pasteInfo.ContentType = http.DetectContentType([]byte(content))
		if pasteInfo.ContentType == "application/octet-stream" {
			pasteInfo.ContentType = "text-plain; charset=utf-8"
		}
		data.m[id] = pasteInfo
		log.Printf("Created new paste %s (%s %s) to die at %s",
			id, pasteInfo.ContentType, pasteInfo.Size, pasteInfo.DeathTime)
		fmt.Fprintf(w, "%s/%s\n", siteUrl, id)
	}
}

func walkFunc(filePath string, fileInfo os.FileInfo, err error) error {
	if err != nil {
		return err
	}
	if fileInfo.IsDir() {
		return nil
	}
	id, err := IdFromPath(filePath)
	if err != nil {
		return err
	}
	modTime := fileInfo.ModTime()
	deathTime := modTime.Add(lifeTime)
	now := time.Now()
	if deathTime.Before(now) {
		go id.EndLife()
		return nil
	}
	var lifeLeft time.Duration
	if deathTime.After(now.Add(lifeTime)) {
		lifeLeft = lifeTime
	} else {
		lifeLeft = deathTime.Sub(now)
	}
	size := ByteSize(fileInfo.Size())
	pasteFile, err := os.Open(filePath)
	if err != nil {
		return err
	}
	defer pasteFile.Close()
	read := make([]byte, 512)
	_, err = pasteFile.Read(read)
	if err != nil {
		return err
	}
	pasteInfo := PasteInfo{
		ModTime:   modTime,
		DeathTime: deathTime,
		Size:      size,
		Etag:      fmt.Sprintf("%d-%s", modTime.Unix(), id),
	}
	pasteInfo.ContentType = http.DetectContentType(read)
	if pasteInfo.ContentType == "application/octet-stream" {
		pasteInfo.ContentType = "text-plain; charset=utf-8"
	}
	data.m[id] = pasteInfo
	log.Printf("Recovered paste %s (%s %s) from %s has %s left",
		id, pasteInfo.ContentType, pasteInfo.Size, pasteInfo.ModTime, lifeLeft)
	id.EndLifeAfter(lifeLeft)
	return nil
}

func main() {
	var err error
	flag.Parse()
	if idSize < 6 || idSize > 256 {
		log.Fatalf("Provided id size %d is not between 6 and 256", idSize)
	}
	if maxSize, err = parseByteSize(maxSizeStr); err != nil {
		log.Fatalf("Invalid max size '%s': %s", maxSizeStr, err)
	}
	if indexTemplate, err = template.ParseFiles(indexTmpl); err != nil {
		log.Fatalf("Could not load template %s: %s", indexTmpl, err)
	}
	if formTemplate, err = template.ParseFiles(formTmpl); err != nil {
		log.Fatalf("Could not load template %s: %s", formTmpl, err)
	}
	if err = os.MkdirAll(dataDir, 0700); err != nil {
		log.Fatalf("Could not create data directory %s: %s", dataDir, err)
	}
	if err = os.Chdir(dataDir); err != nil {
		log.Fatalf("Could not enter data directory %s: %s", dataDir, err)
	}
	log.Printf("idSize   = %d", idSize)
	log.Printf("maxSize  = %s", maxSize)
	log.Printf("siteUrl  = %s", siteUrl)
	log.Printf("listen   = %s", listen)
	log.Printf("dataDir  = %s", dataDir)
	log.Printf("lifeTime = %s", lifeTime)
	if err = filepath.Walk(".", walkFunc); err != nil {
		log.Fatalf("Could not recover data directory %s: %s", dataDir, err)
	}
	http.HandleFunc("/", handler)
	log.Printf("Up and running!")
	log.Fatal(http.ListenAndServe(listen, nil))
}
<commit_msg>Use only one directory level in the fs, not two<commit_after>/* Copyright (c) 2014, Daniel Martí <mvdan@mvdan.cc> */
/* See LICENSE for licensing information */

package main

import (
	"errors"
	"flag"
	"fmt"
	"html/template"
	"io"
	"log"
	"math/rand"
	"net/http"
	"os"
	"path"
	"path/filepath"
	"regexp"
	"strconv"
	"strings"
	"sync"
	"time"
)

const (
	indexTmpl = "index.html"
	formTmpl  = "form.html"
	chars     = "abcdefghijklmnopqrstuvwxyz0123456789"
	randTries = 10

	// GET error messages
	invalidId     = "Invalid paste id."
	pasteNotFound = "Paste doesn't exist."
	unknownError  = "Something went terribly wrong."
	// POST error messages
	missingForm = "Paste could not be found inside the posted form."
)

var (
	siteUrl, listen, dataDir string
	lifeTime                 time.Duration
	maxSizeStr               string
	idSize                   int
	maxSize                  ByteSize

	validId       *regexp.Regexp
	regexByteSize = regexp.MustCompile(`^([\d\.]+)\s*([KM]?B|[BKM])$`)
	indexTemplate *template.Template
	formTemplate  *template.Template
	data          = struct {
		sync.RWMutex
		m map[Id]PasteInfo
	}{m: make(map[Id]PasteInfo)}
	customRand *rand.Rand
)

func init() {
	flag.StringVar(&siteUrl, "u", "http://localhost:8080", "URL of the site")
	flag.StringVar(&listen, "l", "localhost:8080", "Host and port to listen to")
	flag.StringVar(&dataDir, "d", "data", "Directory to store all the pastes in")
	flag.DurationVar(&lifeTime, "t", 12*time.Hour, "Lifetime of the pastes (units: s,m,h)")
	flag.StringVar(&maxSizeStr, "s", "1M", "Maximum size of POSTs in bytes (units: B,K,M)")
	flag.IntVar(&idSize, "i", 8, "Size of the paste ids (between 6 and 256)")
	validId = regexp.MustCompile("^[a-zA-Z0-9]{" + strconv.Itoa(idSize) + "}$")
	customRand = rand.New(rand.NewSource(time.Now().UnixNano()))
}

type PasteInfo struct {
	ModTime     time.Time
	DeathTime   time.Time
	Size        ByteSize
	Etag        string
	ContentType string
}

type Id string

func IdFromPath(idPath string) (Id, error) {
	parts := strings.Split(idPath, string(filepath.Separator))
	if len(parts) != 2 {
		return "", errors.New("Found invalid number of directories at " + idPath)
	}
	rawId := parts[0] + parts[1]
	if !validId.MatchString(rawId) {
		return "", errors.New("Found invalid id " + rawId)
	}
	return Id(rawId), nil
}

func RandomId() (Id, error) {
	s := make([]byte, idSize)
	var id Id
	data.RLock()
	defer data.RUnlock()
	for try := 0; try < randTries; try++ {
		var offset int = 0
	RandLoop:
		for {
			r := customRand.Int63()
			for i := 0; i < 8; i++ {
				randbyte := int(r&0xff) % len(chars)
				s[offset] = chars[randbyte]
				offset++
				if offset == idSize {
					break RandLoop
				}
				r >>= 8
			}
		}
		id = Id(s)
		if _, e := data.m[id]; !e {
			return id, nil
		}
	}
	return id, fmt.Errorf("Gave up trying to find an unused random id after %d tries", randTries)
}

func (id Id) String() string {
	return string(id)
}

func (id Id) Path() string {
	return path.Join(string(id[0:2]), string(id[2:]))
}

func (id Id) EndLife() {
	data.Lock()
	defer data.Unlock()
	err := os.Remove(id.Path())
	if err == nil {
		delete(data.m, id)
		log.Printf("Removed paste: %s", id)
	} else {
		log.Printf("Could not end the life of %s: %s", id, err)
		id.EndLifeAfter(2 * time.Minute)
	}
}

func (id Id) EndLifeAfter(duration time.Duration) {
	timer := time.NewTimer(duration)
	go func() {
		<-timer.C
		id.EndLife()
	}()
}

type ByteSize int64

const (
	B ByteSize = 1 << (10 * iota)
	KB
	MB
)

func parseByteSize(str string) (ByteSize, error) {
	if !regexByteSize.MatchString(str) {
		return 0, errors.New("Could not parse size in bytes")
	}
	parts := regexByteSize.FindStringSubmatch(str)
	size, _ := strconv.ParseFloat(string(parts[1]), 64)

	switch string(parts[2]) {
	case "KB", "K":
		size *= float64(KB)
	case "MB", "M":
		size *= float64(MB)
	}
	return ByteSize(size), nil
}

func (b ByteSize) String() string {
	switch {
	case b >= MB:
		return fmt.Sprintf("%.2f MB", float64(b)/float64(MB))
	case b >= KB:
		return fmt.Sprintf("%.2f KB", float64(b)/float64(KB))
	}
	return fmt.Sprintf("%d B", b)
}

func handler(w http.ResponseWriter, r *http.Request) {
	var err error
	switch r.Method {
	case "GET":
		switch r.URL.Path {
		case "/":
			indexTemplate.Execute(w, struct {
				SiteUrl, LifeTime string
			}{
				siteUrl,
				fmt.Sprintf("%g hours and %g minutes", lifeTime.Hours(),
					lifeTime.Minutes()-lifeTime.Hours()*60),
			})
			return
		case "/form":
			formTemplate.Execute(w, struct{ SiteUrl string }{siteUrl})
			return
		}
		rawId := r.URL.Path[1:]
		if !validId.MatchString(rawId) {
			w.WriteHeader(http.StatusBadRequest)
			fmt.Fprintf(w, "%s\n", invalidId)
			return
		}
		id := Id(strings.ToLower(rawId))
		data.RLock()
		defer data.RUnlock()
		pasteInfo, e := data.m[id]
		if !e {
			w.WriteHeader(http.StatusNotFound)
			fmt.Fprintf(w, "%s\n", pasteNotFound)
			return
		}
		if inm := r.Header.Get("If-None-Match"); inm != "" {
			if pasteInfo.Etag == inm || inm == "*" {
				w.WriteHeader(http.StatusNotModified)
				return
			}
		}
		pastePath := id.Path()
		pasteFile, err := os.Open(pastePath)
		if err != nil {
			w.WriteHeader(http.StatusInternalServerError)
			fmt.Fprintf(w, "%s\n", unknownError)
			return
		}
		defer pasteFile.Close()
		w.Header().Set("Etag", pasteInfo.Etag)
		w.Header().Set("Content-Type", pasteInfo.ContentType)
		http.ServeContent(w, r, "", pasteInfo.ModTime, pasteFile)

	case "POST":
		r.Body = http.MaxBytesReader(w, r.Body, int64(maxSize))
		var content string
		if err = r.ParseMultipartForm(int64(maxSize)); err != nil {
			log.Printf("Could not parse POST multipart form: %s", err)
			w.WriteHeader(http.StatusBadRequest)
			fmt.Fprintf(w, "%s\n", err)
			return
		}
		if vs, found := r.Form["paste"]; found && len(vs[0]) > 0 {
			content = vs[0]
		} else {
			w.WriteHeader(http.StatusBadRequest)
			fmt.Fprintf(w, "%s\n", missingForm)
			return
		}
		var id Id
		if id, err = RandomId(); err == nil {
			data.Lock()
			defer data.Unlock()
		} else {
			log.Println(err)
			w.WriteHeader(http.StatusInternalServerError)
			fmt.Fprintf(w, "%s\n", unknownError)
			return
		}
		pastePath := id.Path()
		dir, _ := path.Split(pastePath)
		if err = os.Mkdir(dir, 0700); err != nil {
			log.Printf("Could not create directories leading to %s: %s", pastePath, err)
			w.WriteHeader(http.StatusInternalServerError)
			fmt.Fprintf(w, "%s\n", unknownError)
			return
		}
		deathTime := time.Now().Add(lifeTime)
		id.EndLifeAfter(lifeTime)
		pasteFile, err := os.OpenFile(pastePath, os.O_WRONLY|os.O_CREATE|os.O_EXCL, 0600)
		if err != nil {
			log.Printf("Could not create new paste file %s: %s", pastePath, err)
			w.WriteHeader(http.StatusInternalServerError)
			fmt.Fprintf(w, "%s\n", unknownError)
			return
		}
		defer pasteFile.Close()
		b, err := io.WriteString(pasteFile, content)
		if err != nil {
			log.Printf("Could not write data into %s: %s", pastePath, err)
			w.WriteHeader(http.StatusInternalServerError)
			fmt.Fprintf(w, "%s\n", unknownError)
			return
		}
		writtenSize := ByteSize(b)
		pasteInfo := PasteInfo{
			ModTime:   time.Now(),
			DeathTime: deathTime,
			Size:      writtenSize,
		}
		pasteInfo.Etag = fmt.Sprintf("%d-%s", pasteInfo.ModTime.Unix(), id)
		pasteInfo.ContentType = http.DetectContentType([]byte(content))
		if pasteInfo.ContentType == "application/octet-stream" {
			pasteInfo.ContentType = "text-plain; charset=utf-8"
		}
		data.m[id] = pasteInfo
		log.Printf("Created new paste %s (%s %s) to die at %s",
			id, pasteInfo.ContentType, pasteInfo.Size, pasteInfo.DeathTime)
		fmt.Fprintf(w, "%s/%s\n", siteUrl, id)
	}
}

func walkFunc(filePath string, fileInfo os.FileInfo, err error) error {
	if err != nil {
		return err
	}
	if fileInfo.IsDir() {
		return nil
	}
	id, err := IdFromPath(filePath)
	if err != nil {
		return err
	}
	modTime := fileInfo.ModTime()
	deathTime := modTime.Add(lifeTime)
	now := time.Now()
	if deathTime.Before(now) {
		go id.EndLife()
		return nil
	}
	var lifeLeft time.Duration
	if deathTime.After(now.Add(lifeTime)) {
		lifeLeft = lifeTime
	} else {
		lifeLeft = deathTime.Sub(now)
	}
	size := ByteSize(fileInfo.Size())
	pasteFile, err := os.Open(filePath)
	if err != nil {
		return err
	}
	defer pasteFile.Close()
	read := make([]byte, 512)
	_, err = pasteFile.Read(read)
	if err != nil {
		return err
	}
	pasteInfo := PasteInfo{
		ModTime:   modTime,
		DeathTime: deathTime,
		Size:      size,
		Etag:      fmt.Sprintf("%d-%s", modTime.Unix(), id),
	}
	pasteInfo.ContentType = http.DetectContentType(read)
	if pasteInfo.ContentType == "application/octet-stream" {
		pasteInfo.ContentType = "text-plain; charset=utf-8"
	}
	data.m[id] = pasteInfo
	log.Printf("Recovered paste %s (%s %s) from %s has %s left",
		id, pasteInfo.ContentType, pasteInfo.Size, pasteInfo.ModTime, lifeLeft)
	id.EndLifeAfter(lifeLeft)
	return nil
}

func main() {
	var err error
	flag.Parse()
	if idSize < 6 || idSize > 256 {
		log.Fatalf("Provided id size %d is not between 6 and 256", idSize)
	}
	if maxSize, err = parseByteSize(maxSizeStr); err != nil {
		log.Fatalf("Invalid max size '%s': %s", maxSizeStr, err)
	}
	if indexTemplate, err = template.ParseFiles(indexTmpl); err != nil {
		log.Fatalf("Could not load template %s: %s", indexTmpl, err)
	}
	if formTemplate, err = template.ParseFiles(formTmpl); err != nil {
		log.Fatalf("Could not load template %s: %s", formTmpl, err)
	}
	if err = os.MkdirAll(dataDir, 0700); err != nil {
		log.Fatalf("Could not create data directory %s: %s", dataDir, err)
	}
	if err = os.Chdir(dataDir); err != nil {
		log.Fatalf("Could not enter data directory %s: %s", dataDir, err)
	}
	log.Printf("idSize   = %d", idSize)
	log.Printf("maxSize  = %s", maxSize)
	log.Printf("siteUrl  = %s", siteUrl)
	log.Printf("listen   = %s", listen)
	log.Printf("dataDir  = %s", dataDir)
	log.Printf("lifeTime = %s", lifeTime)
	if err = filepath.Walk(".", walkFunc); err != nil {
		log.Fatalf("Could not recover data directory %s: %s", dataDir, err)
	}
	http.HandleFunc("/", handler)
	log.Printf("Up and running!")
	log.Fatal(http.ListenAndServe(listen, nil))
}
<|endoftext|>


<commit_before>package httpd

import (
	"bufio"
	"fmt"
	"github.com/Symantec/Dominator/lib/format"
	"github.com/Symantec/Dominator/lib/image"
	"io"
	"net/http"
	"time"
)

var timeFormat string = "02 Jan 2006 15:04:05.99 MST"

func (s state) showImageHandler(w http.ResponseWriter, req *http.Request) {
	writer := bufio.NewWriter(w)
	defer writer.Flush()
	imageName := req.URL.RawQuery
	fmt.Fprintf(writer, "<title>image %s</title>\n", imageName)
	fmt.Fprintln(writer, "")
	fmt.Fprintln(writer, "
")
	image := s.imageDataBase.GetImage(imageName)
	if image == nil {
		fmt.Fprintf(writer, "Image: %s UNKNOWN!\n", imageName)
		return
	}
	fmt.Fprintf(writer, "Information for image: %s
\n", imageName)
	fmt.Fprintln(writer, "
")
	fmt.Fprintf(writer, "Data size: %s
\n",
		imageName, format.FormatBytes(image.FileSystem.TotalDataBytes))
	fmt.Fprintf(writer, "Number of data inodes: %d
\n",
		image.FileSystem.NumRegularInodes)
	if numInodes := image.FileSystem.NumComputedRegularInodes(); numInodes > 0 {
		fmt.Fprintf(writer,
			"Number of computed inodes: %d
\n",
			imageName, numInodes)
	}
	if image.Filter == nil {
		fmt.Fprintln(writer, "Image has no filter: sparse image
")
	} else if len(image.Filter.FilterLines) < 1 {
		fmt.Fprintln(writer,
			"Filter has 0 lines (empty filter: full coverage)
")
	} else {
		fmt.Fprintf(writer,
			"Filter has %d lines
\n",
			imageName, len(image.Filter.FilterLines))
	}
	if image.Triggers == nil || len(image.Triggers.Triggers) < 1 {
		fmt.Fprintln(writer, "Image has no triggers
")
	} else {
		fmt.Fprintf(writer,
			"Number of triggers: %d
\n",
			imageName, len(image.Triggers.Triggers))
	}
	if !image.ExpiresAt.IsZero() {
		fmt.Fprintf(writer, "Expires at: %s
\n",
			image.ExpiresAt.In(time.Local).Format(timeFormat))
	}
	showAnnotation(writer, image.ReleaseNotes, imageName, "Release notes",
		"listReleaseNotes")
	showAnnotation(writer, image.BuildLog, imageName, "Build log",
		"listBuildLog")
	if image.CreatedBy != "" {
		fmt.Fprintf(writer, "Created by: %s\n
", image.CreatedBy)
	}
	if !image.CreatedOn.IsZero() {
		fmt.Fprintf(writer, "Created on: %s\n
", image.CreatedOn)
	}
	fmt.Fprintln(writer, "")
}

func showAnnotation(writer io.Writer, annotation *image.Annotation,
	imageName string, linkName string, baseURL string) {
	if annotation == nil {
		return
	}
	var url string
	if annotation.URL != "" {
		url = annotation.URL
	} else {
		url = baseURL + "?" + imageName
	}
	fmt.Fprintf(writer, "%s
\n", url, linkName)
}
<commit_msg>Also show relative times in imageserver image page.<commit_after>package httpd

import (
	"bufio"
	"fmt"
	"github.com/Symantec/Dominator/lib/format"
	"github.com/Symantec/Dominator/lib/image"
	"io"
	"net/http"
	"time"
)

var timeFormat string = "02 Jan 2006 15:04:05.99 MST"

func (s state) showImageHandler(w http.ResponseWriter, req *http.Request) {
	writer := bufio.NewWriter(w)
	defer writer.Flush()
	imageName := req.URL.RawQuery
	fmt.Fprintf(writer, "<title>image %s</title>\n", imageName)
	fmt.Fprintln(writer, "")
	fmt.Fprintln(writer, "
")
	image := s.imageDataBase.GetImage(imageName)
	if image == nil {
		fmt.Fprintf(writer, "Image: %s UNKNOWN!\n", imageName)
		return
	}
	fmt.Fprintf(writer, "Information for image: %s
\n", imageName)
	fmt.Fprintln(writer, "
")
	fmt.Fprintf(writer, "Data size: %s
\n",
		imageName, format.FormatBytes(image.FileSystem.TotalDataBytes))
	fmt.Fprintf(writer, "Number of data inodes: %d
\n",
		image.FileSystem.NumRegularInodes)
	if numInodes := image.FileSystem.NumComputedRegularInodes(); numInodes > 0 {
		fmt.Fprintf(writer,
			"Number of computed inodes: %d
\n",
			imageName, numInodes)
	}
	if image.Filter == nil {
		fmt.Fprintln(writer, "Image has no filter: sparse image
")
	} else if len(image.Filter.FilterLines) < 1 {
		fmt.Fprintln(writer,
			"Filter has 0 lines (empty filter: full coverage)
")
	} else {
		fmt.Fprintf(writer,
			"Filter has %d lines
\n",
			imageName, len(image.Filter.FilterLines))
	}
	if image.Triggers == nil || len(image.Triggers.Triggers) < 1 {
		fmt.Fprintln(writer, "Image has no triggers
")
	} else {
		fmt.Fprintf(writer,
			"Number of triggers: %d
\n",
			imageName, len(image.Triggers.Triggers))
	}
	if !image.ExpiresAt.IsZero() {
		fmt.Fprintf(writer, "Expires at: %s (in %s)
\n",
			image.ExpiresAt.In(time.Local).Format(timeFormat),
			format.Duration(time.Until(image.ExpiresAt)))
	}
	showAnnotation(writer, image.ReleaseNotes, imageName, "Release notes",
		"listReleaseNotes")
	showAnnotation(writer, image.BuildLog, imageName, "Build log",
		"listBuildLog")
	if image.CreatedBy != "" {
		fmt.Fprintf(writer, "Created by: %s\n
", image.CreatedBy)
	}
	if !image.CreatedOn.IsZero() {
		fmt.Fprintf(writer, "Created on: %s (%s old)\n
",
			image.CreatedOn.In(time.Local).Format(timeFormat),
			format.Duration(time.Since(image.CreatedOn)))
	}
	fmt.Fprintln(writer, "")
}

func showAnnotation(writer io.Writer, annotation *image.Annotation,
	imageName string, linkName string, baseURL string) {
	if annotation == nil {
		return
	}
	var url string
	if annotation.URL != "" {
		url = annotation.URL
	} else {
		url = baseURL + "?" + imageName
	}
	fmt.Fprintf(writer, "%s
\n", url, linkName)
}
<|endoftext|>


<commit_before>//go:generate statik -src=../assets

// Package web Cozy Stack API.
//
// Cozy is a personal platform as a service with a focus on data.
//
// Terms Of Service:
//
// there are no TOS at this moment, use at your own risk we take no responsibility
//
//     Schemes: https
//     Host: localhost
//     BasePath: /
//     Version: 0.0.1
//     License: AGPL-3.0 https://opensource.org/licenses/agpl-3.0
//     Contact: Bruno Michel <bruno@cozycloud.cc> https://cozy.io/
//
//     Consumes:
//     - application/json
//
//     Produces:
//     - application/json
//
// swagger:meta
package web

import (
	"encoding/json"
	"fmt"
	"html/template"
	"io"
	"io/ioutil"
	"net/http"
	"path"
	"strings"

	"github.com/labstack/echo"
	"github.com/rakyll/statik/fs"

	"github.com/cozy/cozy-stack/couchdb"
	"github.com/cozy/cozy-stack/web/jsonapi"
	_ "github.com/cozy/cozy-stack/web/statik" // Generated file with the packed assets
)

var templatesList = []string{
	"login.html",
}

// Config represents the configuration
type Config struct {
	Router    *echo.Echo
	Domain    string
	Assets    string
	ServeApps func(c echo.Context, domain, slug string) error
}

type renderer struct {
	t *template.Template
	h http.Handler
}

func (r *renderer) Render(w io.Writer, name string, data interface{}, c echo.Context) error {
	return r.t.ExecuteTemplate(w, name, data)
}

func (r *renderer) ServeHTTP(res http.ResponseWriter, req *http.Request) {
	r.h.ServeHTTP(res, req)
}

func createRenderer(conf *Config) (*renderer, error) {
	// By default, use the assets packed in the binary
	if conf.Assets != "" {
		if !path.IsAbs(conf.Assets) {
			return nil, fmt.Errorf("path %s is not absolute", conf.Assets)
		}
		list := make([]string, len(templatesList))
		for i, name := range templatesList {
			list[i] = path.Join(conf.Assets, "templates", name)
		}
		t, err := template.ParseFiles(list...)
		if err != nil {
			return nil, err
		}
		h := http.FileServer(http.Dir(conf.Assets))
		r := &renderer{t, h}
		return r, nil
	}

	statikFS, err := fs.New()
	if err != nil {
		return nil, err
	}

	var t, tmpl *template.Template
	for _, name := range templatesList {
		if t == nil {
			t = template.New(name)
			tmpl = t
		} else {
			tmpl = t.New(name)
		}
		f, err := statikFS.Open("/templates/" + name)
		if err != nil {
			return nil, err
		}
		b, err := ioutil.ReadAll(f)
		if err != nil {
			return nil, err
		}
		_, err = tmpl.Parse(string(b))
		if err != nil {
			return nil, err
		}
	}

	h := http.FileServer(statikFS)
	r := &renderer{t, h}
	return r, nil
}

func slugFromHost(host, domain string) (parent, slug string) {
	parts := strings.SplitN(host, ".", 3)
	if len(parts) != 3 {
		return host, ""
	}
	if parts[2] != domain {
		return host, ""
	}
	return parts[1] + "." + parts[2], parts[0]
}

// Create returns a new web server that will handle that apps routing given the
// host of the request. It also adds that the asset handler in /assets/ as well
// as a template rendering to use c.Render.
func Create(conf *Config) (*echo.Echo, error) {
	appsRouter := echo.New()
	apisRouter := conf.Router

	r, err := createRenderer(conf)
	if err != nil {
		return nil, err
	}

	apisRouter.GET("/assets/*", echo.WrapHandler(http.StripPrefix("/assets/", r)))
	apisRouter.Renderer = r
	apisRouter.HTTPErrorHandler = ErrorHandler

	appsRouter.Any("/*", func(c echo.Context) error {
		req := c.Request()
		parent, slug := slugFromHost(req.Host, conf.Domain)
		if slug == "" {
			apisRouter.ServeHTTP(c.Response(), req)
			return nil
		}
		if conf.ServeApps != nil {
			return conf.ServeApps(c, parent, slug)
		}
		return nil
	})

	return appsRouter, nil
}

// ErrorHandler is the default error handler of our server. It always write a
// jsonapi compatible error.
func ErrorHandler(err error, c echo.Context) {
	var je *jsonapi.Error
	var ce *couchdb.Error
	var he *echo.HTTPError
	var ok bool

	if he, ok = err.(*echo.HTTPError); ok {
		c.String(he.Code, he.Message)
		return
	}

	if ce, ok = err.(*couchdb.Error); ok {
		je = &jsonapi.Error{
			Status: ce.StatusCode,
			Title:  ce.Name,
			Detail: ce.Reason,
		}
	} else if je, ok = err.(*jsonapi.Error); !ok {
		je = &jsonapi.Error{
			Status: http.StatusInternalServerError,
			Title:  "Unqualified error",
			Detail: err.Error(),
		}
	}

	resp := c.Response()
	if !resp.Committed {
		if c.Request().Method == http.MethodHead {
			c.NoContent(je.Status)
		} else {
			resp.Header().Set("Content-Type", jsonapi.ContentType)
			resp.WriteHeader(je.Status)
			json.NewEncoder(resp).Encode(je)
		}
	}
}
<commit_msg>Add error loggin in dev mode<commit_after>//go:generate statik -src=../assets

// Package web Cozy Stack API.
//
// Cozy is a personal platform as a service with a focus on data.
//
// Terms Of Service:
//
// there are no TOS at this moment, use at your own risk we take no responsibility
//
//     Schemes: https
//     Host: localhost
//     BasePath: /
//     Version: 0.0.1
//     License: AGPL-3.0 https://opensource.org/licenses/agpl-3.0
//     Contact: Bruno Michel <bruno@cozycloud.cc> https://cozy.io/
//
//     Consumes:
//     - application/json
//
//     Produces:
//     - application/json
//
// swagger:meta
package web

import (
	"encoding/json"
	"fmt"
	"html/template"
	"io"
	"io/ioutil"
	"net/http"
	"path"
	"strings"

	log "github.com/Sirupsen/logrus"
	"github.com/cozy/cozy-stack/config"
	"github.com/cozy/cozy-stack/couchdb"
	"github.com/cozy/cozy-stack/web/jsonapi"
	_ "github.com/cozy/cozy-stack/web/statik" // Generated file with the packed assets
	"github.com/labstack/echo"
	"github.com/rakyll/statik/fs"
)

var templatesList = []string{
	"login.html",
}

// Config represents the configuration
type Config struct {
	Router    *echo.Echo
	Domain    string
	Assets    string
	ServeApps func(c echo.Context, domain, slug string) error
}

type renderer struct {
	t *template.Template
	h http.Handler
}

func (r *renderer) Render(w io.Writer, name string, data interface{}, c echo.Context) error {
	return r.t.ExecuteTemplate(w, name, data)
}

func (r *renderer) ServeHTTP(res http.ResponseWriter, req *http.Request) {
	r.h.ServeHTTP(res, req)
}

func createRenderer(conf *Config) (*renderer, error) {
	// By default, use the assets packed in the binary
	if conf.Assets != "" {
		if !path.IsAbs(conf.Assets) {
			return nil, fmt.Errorf("path %s is not absolute", conf.Assets)
		}
		list := make([]string, len(templatesList))
		for i, name := range templatesList {
			list[i] = path.Join(conf.Assets, "templates", name)
		}
		t, err := template.ParseFiles(list...)
		if err != nil {
			return nil, err
		}
		h := http.FileServer(http.Dir(conf.Assets))
		r := &renderer{t, h}
		return r, nil
	}

	statikFS, err := fs.New()
	if err != nil {
		return nil, err
	}

	var t, tmpl *template.Template
	for _, name := range templatesList {
		if t == nil {
			t = template.New(name)
			tmpl = t
		} else {
			tmpl = t.New(name)
		}
		f, err := statikFS.Open("/templates/" + name)
		if err != nil {
			return nil, err
		}
		b, err := ioutil.ReadAll(f)
		if err != nil {
			return nil, err
		}
		_, err = tmpl.Parse(string(b))
		if err != nil {
			return nil, err
		}
	}

	h := http.FileServer(statikFS)
	r := &renderer{t, h}
	return r, nil
}

func slugFromHost(host, domain string) (parent, slug string) {
	parts := strings.SplitN(host, ".", 3)
	if len(parts) != 3 {
		return host, ""
	}
	if parts[2] != domain {
		return host, ""
	}
	return parts[1] + "." + parts[2], parts[0]
}

// Create returns a new web server that will handle that apps routing given the
// host of the request. It also adds that the asset handler in /assets/ as well
// as a template rendering to use c.Render.
func Create(conf *Config) (*echo.Echo, error) {
	appsRouter := echo.New()
	apisRouter := conf.Router

	r, err := createRenderer(conf)
	if err != nil {
		return nil, err
	}

	apisRouter.GET("/assets/*", echo.WrapHandler(http.StripPrefix("/assets/", r)))
	apisRouter.Renderer = r
	apisRouter.HTTPErrorHandler = ErrorHandler

	appsRouter.Any("/*", func(c echo.Context) error {
		req := c.Request()
		parent, slug := slugFromHost(req.Host, conf.Domain)
		if slug == "" {
			apisRouter.ServeHTTP(c.Response(), req)
			return nil
		}
		if conf.ServeApps != nil {
			return conf.ServeApps(c, parent, slug)
		}
		return nil
	})

	return appsRouter, nil
}

// ErrorHandler is the default error handler of our server. It always write a
// jsonapi compatible error.
func ErrorHandler(err error, c echo.Context) {
	var je *jsonapi.Error
	var ce *couchdb.Error
	var he *echo.HTTPError
	var ok bool

	res := c.Response()
	req := c.Request()

	if he, ok = err.(*echo.HTTPError); ok {
		if !res.Committed {
			if c.Request().Method == http.MethodHead {
				c.NoContent(he.Code)
			} else {
				c.String(he.Code, he.Message)
			}
		}
		if config.IsDevRelease() {
			log.Errorf("[HTTP %s %s] %s", req.Method, req.URL.Path, err)
		}
		return
	}

	if ce, ok = err.(*couchdb.Error); ok {
		je = &jsonapi.Error{
			Status: ce.StatusCode,
			Title:  ce.Name,
			Detail: ce.Reason,
		}
	} else if je, ok = err.(*jsonapi.Error); !ok {
		je = &jsonapi.Error{
			Status: http.StatusInternalServerError,
			Title:  "Unqualified error",
			Detail: err.Error(),
		}
	}

	if !res.Committed {
		if c.Request().Method == http.MethodHead {
			c.NoContent(je.Status)
		} else {
			res.Header().Set("Content-Type", jsonapi.ContentType)
			res.WriteHeader(je.Status)
			json.NewEncoder(res).Encode(je)
		}
	}

	if config.IsDevRelease() {
		log.Errorf("[HTTP %s %s] %s", req.Method, req.URL.Path, err)
	}
}
<|endoftext|>


<commit_before>package tris

import (
	// "errors"
	"fmt"
	"github.com/fvbock/trie"
	"time"
)

var (
	TrisCommands []Command
)

func init() {
}

// make those "singletons"?

/*
CommandInfo sets the actuve database on the server client (the connection)
*/
type CommandInfo struct{}

func (cmd *CommandInfo) Name() string       { return "INFO" }
func (cmd *CommandInfo) Flags() int         { return COMMAND_FLAG_ADMIN }
func (cmd *CommandInfo) ResponseFlags() int { return COMMAND_REPLY_SINGLE }
func (cmd *CommandInfo) Function(s *Server, c *Client, args ...interface{}) (reply *Reply) {
	reply = NewReply([][]byte{[]byte(fmt.Sprintf("%v Tris 0.0.1 says Hello and INFO.\nServer: %v\nClient: %v\n", time.Now(), s, c))}, COMMAND_OK)
	return
}

/*
CommandSelect sets the actuve database on the server client (the connection)
*/
type CommandSelect struct{}

func (cmd *CommandSelect) Name() string       { return "SELECT" }
func (cmd *CommandSelect) Flags() int         { return COMMAND_FLAG_ADMIN }
func (cmd *CommandSelect) ResponseFlags() int { return COMMAND_REPLY_EMPTY }
func (cmd *CommandSelect) Function(s *Server, c *Client, args ...interface{}) (reply *Reply) {
	name := string(args[0].([]byte))
	if _, exists := s.Databases[name]; exists {
		err := fmt.Sprintf("Databases %s does not exist.", name)
		return NewReply([][]byte{[]byte(err)}, COMMAND_FAIL)
	}
	c.ActiveDb = s.Databases[name]
	c.ActiveDbName = name
	return NewReply([][]byte{}, COMMAND_OK)
}

/*
CommandCreateTrie maps to trie.NewRefCountTrie()
*/
type CommandCreateTrie struct{}

func (cmd *CommandCreateTrie) Name() string       { return "CREATE" }
func (cmd *CommandCreateTrie) Flags() int         { return COMMAND_FLAG_ADMIN | COMMAND_FLAG_WRITE }
func (cmd *CommandCreateTrie) ResponseFlags() int { return COMMAND_REPLY_EMPTY }
func (cmd *CommandCreateTrie) Function(s *Server, c *Client, args ...interface{}) (reply *Reply) {
	name := string(args[0].(uint8))
	if _, exists := s.Databases[name]; exists {
		err := fmt.Sprintf("Databases %s has already been registered.", name)
		return NewReply([][]byte{[]byte(err)}, COMMAND_FAIL)
	}
	s.Databases[name] = trie.NewRefCountTrie()
	return NewReply([][]byte{}, COMMAND_OK)
}

/*
CommandAdd maps to RefCountTrie.Add()
*/
type CommandAdd struct{}

func (cmd *CommandAdd) Name() string       { return "ADD" }
func (cmd *CommandAdd) Flags() int         { return COMMAND_FLAG_ADMIN | COMMAND_FLAG_WRITE }
func (cmd *CommandAdd) ResponseFlags() int { return COMMAND_REPLY_SINGLE }
func (cmd *CommandAdd) Function(s *Server, c *Client, args ...interface{}) (reply *Reply) {
	key := string(args[0].([]uint8))
	b := c.ActiveDb.Add(key)
	return NewReply([][]byte{[]byte(string(b.Count))}, COMMAND_OK)
}

/*
CommandDel maps to RefCountTrie.Del()
*/
type CommandDel struct{}

func (cmd *CommandDel) Name() string       { return "DEL" }
func (cmd *CommandDel) Flags() int         { return COMMAND_FLAG_ADMIN | COMMAND_FLAG_WRITE }
func (cmd *CommandDel) ResponseFlags() int { return COMMAND_REPLY_SINGLE }
func (cmd *CommandDel) Function(s *Server, c *Client, args ...interface{}) (reply *Reply) {
	key := string(args[0].([]uint8))
	if c.ActiveDb.Delete(key) {
		return NewReply([][]byte{[]byte("TRUE")}, COMMAND_OK)
	}
	return NewReply([][]byte{[]byte("FALSE")}, COMMAND_OK)
}

/*
CommandHas maps to RefCountTrie.Has()
*/
type CommandHas struct{}

func (cmd *CommandHas) Name() string       { return "HAS" }
func (cmd *CommandHas) Flags() int         { return COMMAND_FLAG_READ }
func (cmd *CommandHas) ResponseFlags() int { return COMMAND_REPLY_SINGLE }
func (cmd *CommandHas) Function(s *Server, c *Client, args ...interface{}) (reply *Reply) {
	key := string(args[0].([]uint8))
	if c.ActiveDb.Has(key) {
		return NewReply([][]byte{[]byte("TRUE")}, COMMAND_OK)
	}
	return NewReply([][]byte{[]byte("FALSE")}, COMMAND_OK)
}

/*
CommandHasPrefix maps to RefCountTrie.HasPrefix()
*/
type CommandHasPrefix struct{}

func (cmd *CommandHasPrefix) Name() string       { return "HASPREFIX" }
func (cmd *CommandHasPrefix) Flags() int         { return COMMAND_FLAG_READ }
func (cmd *CommandHasPrefix) ResponseFlags() int { return COMMAND_REPLY_SINGLE }
func (cmd *CommandHasPrefix) Function(s *Server, c *Client, args ...interface{}) (reply *Reply) {
	key := string(args[0].([]uint8))
	if c.ActiveDb.HasPrefix(key) {
		return NewReply([][]byte{[]byte("TRUE")}, COMMAND_OK)
	}
	return NewReply([][]byte{[]byte("FALSE")}, COMMAND_OK)
}

/*
CommandTree maps to RefCountTrie.Dump()
*/
type CommandTree struct{}

func (cmd *CommandTree) Name() string       { return "TREE" }
func (cmd *CommandTree) Flags() int         { return COMMAND_FLAG_READ }
func (cmd *CommandTree) ResponseFlags() int { return COMMAND_REPLY_SINGLE }
func (cmd *CommandTree) Function(s *Server, c *Client, args ...interface{}) (reply *Reply) {
	return NewReply([][]byte{[]byte(c.ActiveDb.Dump())}, COMMAND_OK)
}

/*
CommandMembers maps to RefCountTrie.Members()
*/
type CommandMembers struct{}

func (cmd *CommandMembers) Name() string       { return "MEMBERS" }
func (cmd *CommandMembers) Flags() int         { return COMMAND_FLAG_READ }
func (cmd *CommandMembers) ResponseFlags() int { return COMMAND_REPLY_MULTI }
func (cmd *CommandMembers) Function(s *Server, c *Client, args ...interface{}) (reply *Reply) {
	var mrep [][]byte
	for _, m := range c.ActiveDb.Members() {
		mrep = append(mrep, []byte(m.Value))
	}

	return NewReply(mrep, COMMAND_OK)
}

/*
CommandPrefixMembers maps to RefCountTrie.PrefixMembers()
*/
type CommandPrefixMembers struct{}

func (cmd *CommandPrefixMembers) Name() string       { return "PREFIXMEMBERS" }
func (cmd *CommandPrefixMembers) Flags() int         { return COMMAND_FLAG_READ }
func (cmd *CommandPrefixMembers) ResponseFlags() int { return COMMAND_REPLY_MULTI }
func (cmd *CommandPrefixMembers) Function(s *Server, c *Client, args ...interface{}) (reply *Reply) {
	key := string(args[0].([]uint8))
	var mrep [][]byte
	for _, m := range c.ActiveDb.PrefixMembers(key) {
		mrep = append(mrep, []byte(m.Value))
	}

	return NewReply(mrep, COMMAND_OK)
}
<commit_msg>add HasCount cmd mapping<commit_after>package tris

import (
	// "errors"
	"fmt"
	"github.com/fvbock/trie"
	"strconv"
	"time"
)

var (
	TrisCommands []Command
)

func init() {
}

// make those "singletons"?

/*
CommandInfo sets the actuve database on the server client (the connection)
*/
type CommandInfo struct{}

func (cmd *CommandInfo) Name() string       { return "INFO" }
func (cmd *CommandInfo) Flags() int         { return COMMAND_FLAG_ADMIN }
func (cmd *CommandInfo) ResponseFlags() int { return COMMAND_REPLY_SINGLE }
func (cmd *CommandInfo) Function(s *Server, c *Client, args ...interface{}) (reply *Reply) {
	reply = NewReply([][]byte{[]byte(fmt.Sprintf("%v Tris 0.0.1 says Hello and INFO.\nServer: %v\nClient: %v\n", time.Now(), s, c))}, COMMAND_OK)
	return
}

/*
CommandSelect sets the actuve database on the server client (the connection)
*/
type CommandSelect struct{}

func (cmd *CommandSelect) Name() string       { return "SELECT" }
func (cmd *CommandSelect) Flags() int         { return COMMAND_FLAG_ADMIN }
func (cmd *CommandSelect) ResponseFlags() int { return COMMAND_REPLY_EMPTY }
func (cmd *CommandSelect) Function(s *Server, c *Client, args ...interface{}) (reply *Reply) {
	name := string(args[0].([]byte))
	if _, exists := s.Databases[name]; exists {
		err := fmt.Sprintf("Databases %s does not exist.", name)
		return NewReply([][]byte{[]byte(err)}, COMMAND_FAIL)
	}
	c.ActiveDb = s.Databases[name]
	c.ActiveDbName = name
	return NewReply([][]byte{}, COMMAND_OK)
}

/*
CommandCreateTrie maps to trie.NewRefCountTrie()
*/
type CommandCreateTrie struct{}

func (cmd *CommandCreateTrie) Name() string       { return "CREATE" }
func (cmd *CommandCreateTrie) Flags() int         { return COMMAND_FLAG_ADMIN | COMMAND_FLAG_WRITE }
func (cmd *CommandCreateTrie) ResponseFlags() int { return COMMAND_REPLY_EMPTY }
func (cmd *CommandCreateTrie) Function(s *Server, c *Client, args ...interface{}) (reply *Reply) {
	name := string(args[0].(uint8))
	if _, exists := s.Databases[name]; exists {
		err := fmt.Sprintf("Databases %s has already been registered.", name)
		return NewReply([][]byte{[]byte(err)}, COMMAND_FAIL)
	}
	s.Databases[name] = trie.NewRefCountTrie()
	return NewReply([][]byte{}, COMMAND_OK)
}

/*
CommandAdd maps to RefCountTrie.Add()
*/
type CommandAdd struct{}

func (cmd *CommandAdd) Name() string       { return "ADD" }
func (cmd *CommandAdd) Flags() int         { return COMMAND_FLAG_ADMIN | COMMAND_FLAG_WRITE }
func (cmd *CommandAdd) ResponseFlags() int { return COMMAND_REPLY_SINGLE }
func (cmd *CommandAdd) Function(s *Server, c *Client, args ...interface{}) (reply *Reply) {
	key := string(args[0].([]uint8))
	b := c.ActiveDb.Add(key)
	return NewReply([][]byte{[]byte(string(b.Count))}, COMMAND_OK)
}

/*
CommandDel maps to RefCountTrie.Del()
*/
type CommandDel struct{}

func (cmd *CommandDel) Name() string       { return "DEL" }
func (cmd *CommandDel) Flags() int         { return COMMAND_FLAG_ADMIN | COMMAND_FLAG_WRITE }
func (cmd *CommandDel) ResponseFlags() int { return COMMAND_REPLY_SINGLE }
func (cmd *CommandDel) Function(s *Server, c *Client, args ...interface{}) (reply *Reply) {
	key := string(args[0].([]uint8))
	if c.ActiveDb.Delete(key) {
		return NewReply([][]byte{[]byte("TRUE")}, COMMAND_OK)
	}
	return NewReply([][]byte{[]byte("FALSE")}, COMMAND_OK)
}

/*
CommandHas maps to RefCountTrie.Has()
*/
type CommandHas struct{}

func (cmd *CommandHas) Name() string       { return "HAS" }
func (cmd *CommandHas) Flags() int         { return COMMAND_FLAG_READ }
func (cmd *CommandHas) ResponseFlags() int { return COMMAND_REPLY_SINGLE }
func (cmd *CommandHas) Function(s *Server, c *Client, args ...interface{}) (reply *Reply) {
	key := string(args[0].([]uint8))
	if c.ActiveDb.Has(key) {
		return NewReply([][]byte{[]byte("TRUE")}, COMMAND_OK)
	}
	return NewReply([][]byte{[]byte("FALSE")}, COMMAND_OK)
}

/*
CommandHasCount maps to RefCountTrie.HasCount()
*/
type CommandHasCount struct{}

func (cmd *CommandHasCount) Name() string       { return "HASCOUNT" }
func (cmd *CommandHasCount) Flags() int         { return COMMAND_FLAG_READ }
func (cmd *CommandHasCount) ResponseFlags() int { return COMMAND_REPLY_SINGLE }
func (cmd *CommandHasCount) Function(s *Server, c *Client, args ...interface{}) (reply *Reply) {
	key := string(args[0].([]uint8))
	has, count := c.ActiveDb.HasCount(key)
	s.Log.Println(has, count, string(count))
	// return NewReply([][]byte{[]byte(string(count))}, COMMAND_OK)
	return NewReply([][]byte{[]byte(strconv.FormatInt(count, 10))}, COMMAND_OK)
}

/*
CommandHasPrefix maps to RefCountTrie.HasPrefix()
*/
type CommandHasPrefix struct{}

func (cmd *CommandHasPrefix) Name() string       { return "HASPREFIX" }
func (cmd *CommandHasPrefix) Flags() int         { return COMMAND_FLAG_READ }
func (cmd *CommandHasPrefix) ResponseFlags() int { return COMMAND_REPLY_SINGLE }
func (cmd *CommandHasPrefix) Function(s *Server, c *Client, args ...interface{}) (reply *Reply) {
	key := string(args[0].([]uint8))
	if c.ActiveDb.HasPrefix(key) {
		return NewReply([][]byte{[]byte("TRUE")}, COMMAND_OK)
	}
	return NewReply([][]byte{[]byte("FALSE")}, COMMAND_OK)
}

/*
CommandTree maps to RefCountTrie.Dump()
*/
type CommandTree struct{}

func (cmd *CommandTree) Name() string       { return "TREE" }
func (cmd *CommandTree) Flags() int         { return COMMAND_FLAG_READ }
func (cmd *CommandTree) ResponseFlags() int { return COMMAND_REPLY_SINGLE }
func (cmd *CommandTree) Function(s *Server, c *Client, args ...interface{}) (reply *Reply) {
	return NewReply([][]byte{[]byte(c.ActiveDb.Dump())}, COMMAND_OK)
}

/*
CommandMembers maps to RefCountTrie.Members()
*/
type CommandMembers struct{}

func (cmd *CommandMembers) Name() string       { return "MEMBERS" }
func (cmd *CommandMembers) Flags() int         { return COMMAND_FLAG_READ }
func (cmd *CommandMembers) ResponseFlags() int { return COMMAND_REPLY_MULTI }
func (cmd *CommandMembers) Function(s *Server, c *Client, args ...interface{}) (reply *Reply) {
	var mrep [][]byte
	for _, m := range c.ActiveDb.Members() {
		mrep = append(mrep, []byte(m.Value))
	}

	return NewReply(mrep, COMMAND_OK)
}

/*
CommandPrefixMembers maps to RefCountTrie.PrefixMembers()
*/
type CommandPrefixMembers struct{}

func (cmd *CommandPrefixMembers) Name() string       { return "PREFIXMEMBERS" }
func (cmd *CommandPrefixMembers) Flags() int         { return COMMAND_FLAG_READ }
func (cmd *CommandPrefixMembers) ResponseFlags() int { return COMMAND_REPLY_MULTI }
func (cmd *CommandPrefixMembers) Function(s *Server, c *Client, args ...interface{}) (reply *Reply) {
	key := string(args[0].([]uint8))
	var mrep [][]byte
	for _, m := range c.ActiveDb.PrefixMembers(key) {
		mrep = append(mrep, []byte(m.Value))
	}

	return NewReply(mrep, COMMAND_OK)
}
<|endoftext|>


<commit_before>package server

import (
	"errors"
	"log"
	"os"
	"sync/atomic"
	"time"

	"github.com/privacylab/talek/common"
)

// Frontend terminates client connections to the leader server.
// It is the point of global serialization, and establishes sequence numbers.
type Frontend struct {
	// Private State
	log  *log.Logger
	name string
	*Config

	proposedSeqNo uint64 // Use atomic.AddUint64, atomic.LoadUint64
	readChan      chan *readRequest

	replicas []common.ReplicaInterface
	dead     int32
}

// readRequest is the grouped request and reply memory used for batching
// incoming reads onto a single thread.
type readRequest struct {
	Args  *common.EncodedReadArgs
	Reply *common.ReadReply
	Done  chan bool
}

// NewFrontend creates a new Frontend for a provided configuration.
func NewFrontend(name string, config *Config, replicas []common.ReplicaInterface) *Frontend {
	fe := &Frontend{}
	fe.log = log.New(os.Stdout, "[Frontend:"+name+"] ", log.Ldate|log.Ltime|log.Lshortfile)
	fe.name = name
	fe.Config = config
	fe.replicas = replicas
	fe.readChan = make(chan *readRequest, 10)

	// Periodically serialize database epoch advances.
	go fe.periodicWrite()
	// Batch incoming reads into combined requests to replicas.
	go fe.batchReads()

	return fe
}

/** PUBLIC METHODS (threadsafe) **/

// Close goroutines associated with this object.
func (fe *Frontend) Close() {
	atomic.StoreInt32(&fe.dead, 1)
}

// GetName exports the name of the server.
func (fe *Frontend) GetName(args *interface{}, reply *string) error {
	*reply = fe.name
	return nil
}

// GetConfig returns the current common configuration from the server.
func (fe *Frontend) GetConfig(args *interface{}, reply *common.Config) error {
	config := *fe.Config.Config
	*reply = config
	return nil
}

func (fe *Frontend) Write(args *common.WriteArgs, reply *common.WriteReply) error {
	seqNo := atomic.AddUint64(&fe.proposedSeqNo, 1)
	args.GlobalSeqNo = seqNo

	replicaWrite := &common.ReplicaWriteArgs{
		WriteArgs: *args,
	}
	replicaReply := common.ReplicaWriteReply{}
	//@todo writes in parallel
	for i, r := range fe.replicas {
		err := r.Write(replicaWrite, &replicaReply)
		if err != nil {
			reply.Err = err.Error()
			fe.log.Fatalf("Error writing to replica %d: %v", i, err)
		} else if len(replicaReply.Err) > 0 {
			reply.Err = replicaReply.Err
		}
	}

	return nil
}

func (fe *Frontend) Read(args *common.EncodedReadArgs, reply *common.ReadReply) error {
	ready := make(chan bool, 1)
	fe.readChan <- &readRequest{Args: args, Reply: reply, Done: ready}
	<-ready

	return nil
}

// GetUpdates provides the most recent global interest vector deltas.
func (fe *Frontend) GetUpdates(args *common.GetUpdatesArgs, reply *common.GetUpdatesReply) error {
	fe.log.Println("GetUpdates: ")
	// @TODO
	return nil
}

// periodicWrite runs until the dead flag is set, and periodically send a write
// request to all replicas telling them to advance their write epoch.
func (fe *Frontend) periodicWrite() {
	for atomic.LoadInt32(&fe.dead) == 0 {
		tick := time.After(fe.WriteInterval)
		select {
		case <-tick:
			args := &common.ReplicaWriteArgs{
				EpochFlag: true,
			}
			var rep common.ReplicaWriteReply
			for _, r := range fe.replicas {
				r.Write(args, &rep)
			}
		}
	}
}

func (fe *Frontend) batchReads() {
	batch := make([]*readRequest, 0, fe.Config.ReadBatch)
	var readReq *readRequest
	tick := time.After(fe.Config.ReadInterval)
	for atomic.LoadInt32(&fe.dead) == 0 {
		select {
		case readReq = <-fe.readChan:
			batch = append(batch, readReq)
			if len(batch) >= fe.Config.ReadBatch {
				go fe.triggerBatchRead(batch)
				batch = make([]*readRequest, 0, fe.Config.ReadBatch)
			} else {
				fe.log.Printf("Read: add to batch, size=%v\n", len(batch))
			}
			continue
		case <-tick:
			if len(batch) > 0 {
				go fe.triggerBatchRead(batch)
				batch = make([]*readRequest, 0, fe.Config.ReadBatch)
			}
			tick = time.After(fe.Config.ReadInterval)
			continue
		}
	}
}

func (fe *Frontend) triggerBatchRead(batch []*readRequest) error {
	args := &common.BatchReadRequest{}
	// Copy args
	args.Args = make([]common.EncodedReadArgs, len(batch), len(batch))
	for i, val := range batch {
		if val.Args != nil {
			args.Args[i] = *val.Args
		}
	}

	// Choose a SeqNoRange
	currSeqNo := atomic.LoadUint64(&fe.proposedSeqNo) + 1
	if currSeqNo <= uint64(fe.Config.WindowSize()) {
		args.SeqNoRange.Start = 1 // Minimum of 1
	} else {
		args.SeqNoRange.Start = currSeqNo - uint64(fe.Config.WindowSize()) // Inclusive
	}
	args.SeqNoRange.End = currSeqNo // Exclusive
	args.SeqNoRange.Aborted = make([]uint64, 0, 0)

	// Start computation
	// @todo reads in parallel
	var replicaErr error
	replies := make([]common.BatchReadReply, len(fe.replicas))
	for i, r := range fe.replicas {
		err := r.BatchRead(args, &replies[i])
		if err != nil || replies[i].Err != "" {
			replicaErr = err
			fe.log.Fatalf("Error making read to replica %d: %v%v", i, err, replies[i].Err)
		}
		if len(replies[i].Replies) != len(batch) {
			replicaErr = errors.New("failure from Replica " + i)
			fe.log.Fatalf("Replica %d gave the wrong number of replies (%d instead of %d)", i, len(replies[i].Replies), len(batch))
		}
	}

	// Respond to clients
	// @todo propagate errors back to clients.
	replyLength := len(replies[0].Replies[0].Data)
	for i, val := range batch {
		val.Reply.Data = make([]byte, replyLength)
		for _, rp := range replies {
			val.Reply.Combine(rp.Replies[i].Data)
		}
		if replicaErr != nil {
			val.Reply.Err = replicaErr.Error()
		}
		val.Reply.GlobalSeqNo = args.SeqNoRange
		val.Done <- true
	}

	return nil
}
<commit_msg>fix bug in frontend<commit_after>package server

import (
	"errors"
	"log"
	"os"
	"sync/atomic"
	"time"

	"github.com/privacylab/talek/common"
)

// Frontend terminates client connections to the leader server.
// It is the point of global serialization, and establishes sequence numbers.
type Frontend struct {
	// Private State
	log  *log.Logger
	name string
	*Config

	proposedSeqNo uint64 // Use atomic.AddUint64, atomic.LoadUint64
	readChan      chan *readRequest

	replicas []common.ReplicaInterface
	dead     int32
}

// readRequest is the grouped request and reply memory used for batching
// incoming reads onto a single thread.
type readRequest struct {
	Args  *common.EncodedReadArgs
	Reply *common.ReadReply
	Done  chan bool
}

// NewFrontend creates a new Frontend for a provided configuration.
func NewFrontend(name string, config *Config, replicas []common.ReplicaInterface) *Frontend {
	fe := &Frontend{}
	fe.log = log.New(os.Stdout, "[Frontend:"+name+"] ", log.Ldate|log.Ltime|log.Lshortfile)
	fe.name = name
	fe.Config = config
	fe.replicas = replicas
	fe.readChan = make(chan *readRequest, 10)

	// Periodically serialize database epoch advances.
	go fe.periodicWrite()
	// Batch incoming reads into combined requests to replicas.
	go fe.batchReads()

	return fe
}

/** PUBLIC METHODS (threadsafe) **/

// Close goroutines associated with this object.
func (fe *Frontend) Close() {
	atomic.StoreInt32(&fe.dead, 1)
}

// GetName exports the name of the server.
func (fe *Frontend) GetName(args *interface{}, reply *string) error {
	*reply = fe.name
	return nil
}

// GetConfig returns the current common configuration from the server.
func (fe *Frontend) GetConfig(args *interface{}, reply *common.Config) error {
	config := *fe.Config.Config
	*reply = config
	return nil
}

func (fe *Frontend) Write(args *common.WriteArgs, reply *common.WriteReply) error {
	seqNo := atomic.AddUint64(&fe.proposedSeqNo, 1)
	args.GlobalSeqNo = seqNo

	replicaWrite := &common.ReplicaWriteArgs{
		WriteArgs: *args,
	}
	replicaReply := common.ReplicaWriteReply{}
	//@todo writes in parallel
	for i, r := range fe.replicas {
		err := r.Write(replicaWrite, &replicaReply)
		if err != nil {
			reply.Err = err.Error()
			fe.log.Fatalf("Error writing to replica %d: %v", i, err)
		} else if len(replicaReply.Err) > 0 {
			reply.Err = replicaReply.Err
		}
	}

	return nil
}

func (fe *Frontend) Read(args *common.EncodedReadArgs, reply *common.ReadReply) error {
	ready := make(chan bool, 1)
	fe.readChan <- &readRequest{Args: args, Reply: reply, Done: ready}
	<-ready

	return nil
}

// GetUpdates provides the most recent global interest vector deltas.
func (fe *Frontend) GetUpdates(args *common.GetUpdatesArgs, reply *common.GetUpdatesReply) error {
	fe.log.Println("GetUpdates: ")
	// @TODO
	return nil
}

// periodicWrite runs until the dead flag is set, and periodically send a write
// request to all replicas telling them to advance their write epoch.
func (fe *Frontend) periodicWrite() {
	for atomic.LoadInt32(&fe.dead) == 0 {
		tick := time.After(fe.WriteInterval)
		select {
		case <-tick:
			args := &common.ReplicaWriteArgs{
				EpochFlag: true,
			}
			var rep common.ReplicaWriteReply
			for _, r := range fe.replicas {
				r.Write(args, &rep)
			}
		}
	}
}

func (fe *Frontend) batchReads() {
	batch := make([]*readRequest, 0, fe.Config.ReadBatch)
	var readReq *readRequest
	tick := time.After(fe.Config.ReadInterval)
	for atomic.LoadInt32(&fe.dead) == 0 {
		select {
		case readReq = <-fe.readChan:
			batch = append(batch, readReq)
			if len(batch) >= fe.Config.ReadBatch {
				go fe.triggerBatchRead(batch)
				batch = make([]*readRequest, 0, fe.Config.ReadBatch)
			} else {
				fe.log.Printf("Read: add to batch, size=%v\n", len(batch))
			}
			continue
		case <-tick:
			if len(batch) > 0 {
				go fe.triggerBatchRead(batch)
				batch = make([]*readRequest, 0, fe.Config.ReadBatch)
			}
			tick = time.After(fe.Config.ReadInterval)
			continue
		}
	}
}

func (fe *Frontend) triggerBatchRead(batch []*readRequest) error {
	args := &common.BatchReadRequest{}
	// Copy args
	args.Args = make([]common.EncodedReadArgs, len(batch), len(batch))
	for i, val := range batch {
		if val.Args != nil {
			args.Args[i] = *val.Args
		}
	}

	// Choose a SeqNoRange
	currSeqNo := atomic.LoadUint64(&fe.proposedSeqNo) + 1
	if currSeqNo <= uint64(fe.Config.WindowSize()) {
		args.SeqNoRange.Start = 1 // Minimum of 1
	} else {
		args.SeqNoRange.Start = currSeqNo - uint64(fe.Config.WindowSize()) // Inclusive
	}
	args.SeqNoRange.End = currSeqNo // Exclusive
	args.SeqNoRange.Aborted = make([]uint64, 0, 0)

	// Start computation
	// @todo reads in parallel
	var replicaErr error
	replies := make([]common.BatchReadReply, len(fe.replicas))
	for i, r := range fe.replicas {
		err := r.BatchRead(args, &replies[i])
		if err != nil || replies[i].Err != "" {
			replicaErr = err
			fe.log.Fatalf("Error making read to replica %d: %v%v", i, err, replies[i].Err)
		}
		if len(replies[i].Replies) != len(batch) {
			replicaErr = errors.New("failure from Replica " + string(i))
			fe.log.Fatalf("Replica %d gave the wrong number of replies (%d instead of %d)", i, len(replies[i].Replies), len(batch))
		}
	}

	// Respond to clients
	// @todo propagate errors back to clients.
	replyLength := len(replies[0].Replies[0].Data)
	for i, val := range batch {
		val.Reply.Data = make([]byte, replyLength)
		for _, rp := range replies {
			val.Reply.Combine(rp.Replies[i].Data)
		}
		if replicaErr != nil {
			val.Reply.Err = replicaErr.Error()
		}
		val.Reply.GlobalSeqNo = args.SeqNoRange
		val.Done <- true
	}

	return nil
}
<|endoftext|>


<commit_before>/*
Copyright 2016 Google Inc. All Rights Reserved.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
    http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

// The gooserve binary is used to serve GooGet repositories.
package main

import (
	"encoding/json"
	"flag"
	"fmt"
	"io/ioutil"
	"net/http"
	"path"
	"path/filepath"
	"strings"
	"sync"
	"time"

	"github.com/google/googet/goolib"
	"github.com/google/googet/oswrap"
	"github.com/google/logger"
)

var (
	root      = flag.String("root", "", "root location")
	interval  = flag.Duration("interval", 5*time.Minute, "duration between refresh runs")
	verbose   = flag.Bool("verbose", false, "print info level logs to stdout")
	systemLog = flag.Bool("system_log", false, "log to Linux Syslog or Windows Event Log")
	port      = flag.Int("port", 8000, "listen port")
	repoName  = flag.String("repo_name", "repo", "name of the repo to setup")

	repoContents *repoPackages
)

// repoPackages describes a repository of packages.
type repoPackages struct {
	rs []goolib.RepoSpec
	mu sync.Mutex
}

// add provides a thread safe way to add a package to repoPackages.
func (r *repoPackages) add(src, chksum string, spec *goolib.PkgSpec) {
	r.mu.Lock()
	defer r.mu.Unlock()
	r.rs = append(r.rs, goolib.RepoSpec{
		Source:      src,
		Checksum:    chksum,
		PackageSpec: spec,
	})
}

func packageInfo(pkgPath, packageDir string) error {
	pkg := filepath.Base(pkgPath)
	pi := goolib.PkgNameSplit(strings.TrimSuffix(pkg, ".goo"))

	spec, err := extractSpec(pkgPath)
	if err != nil {
		return err
	}
	if spec.Name != pi.Name {
		return fmt.Errorf("%s: name in spec does not match package file name", pkgPath)
	}
	if spec.Arch != pi.Arch {
		return fmt.Errorf("%s: arch in spec does not match package file name", pkgPath)
	}
	if spec.Version != pi.Ver {
		return fmt.Errorf("%s: version in spec does not match package version", pkgPath)
	}

	f, err := oswrap.Open(pkgPath)
	if err != nil {
		return err
	}
	defer f.Close()

	repoContents.add(path.Join(packageDir, pkg), goolib.Checksum(f), spec)
	return nil
}

func runSync(packageDir string) error {
	logger.Info("Beginning sync run")
	if err := oswrap.MkdirAll(packageDir, 0774); err != nil {
		return err
	}

	pkgs, err := filepath.Glob(filepath.Join(packageDir, "*.goo"))
	if err != nil {
		return err
	}

	repoContents = &repoPackages{}
	var wg sync.WaitGroup
	for _, pkg := range pkgs {
		wg.Add(1)
		go func(pkg string) {
			defer wg.Done()
			if err := packageInfo(pkg, packageDir); err != nil {
				logger.Error(err)
			}
		}(pkg)
	}
	wg.Wait()
	logger.Info("Sync run completed successfully")
	return nil
}

// extractSpec takes a goopkg file and returns the unmarshalled spec file.
func extractSpec(pkgPath string) (*goolib.PkgSpec, error) {
	f, err := oswrap.Open(pkgPath)
	if err != nil {
		return nil, err
	}
	defer f.Close()
	return goolib.ExtractPkgSpec(f)
}

func serve(w http.ResponseWriter, r *http.Request) {
	out, err := json.MarshalIndent(repoContents.rs, "", "  ")
	if err != nil {
		logger.Fatal(err)
	}
	w.Header().Set("Content-Type", "application/json")
	w.Write(out)
}

func main() {
	flag.Parse()

	logger.Init("GooServe", *verbose, *systemLog, ioutil.Discard)

	packageDir := filepath.Join(*root, "packages")
	if err := runSync(packageDir); err != nil {
		logger.Error(err)
	}

	http.HandleFunc(fmt.Sprintf("/%s/index", *repoName), serve)
	http.Handle("/packages/", http.StripPrefix("/packages/", http.FileServer(http.Dir(packageDir))))
	go func() {
		err := http.ListenAndServe(fmt.Sprintf(":%d", *port), nil)
		if err != nil {
			logger.Fatal(err)
		}
	}()

	for range time.Tick(*interval) {
		if err := runSync(packageDir); err != nil {
			logger.Error(err)
		}
	}
}
<commit_msg>Corrected 'Source' path in packages with -root (#35)<commit_after>/*
Copyright 2016 Google Inc. All Rights Reserved.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
    http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

// The gooserve binary is used to serve GooGet repositories.
package main

import (
	"encoding/json"
	"flag"
	"fmt"
	"io/ioutil"
	"net/http"
	"path"
	"path/filepath"
	"strings"
	"sync"
	"time"

	"github.com/google/googet/goolib"
	"github.com/google/googet/oswrap"
	"github.com/google/logger"
)

var (
	root        = flag.String("root", "", "root location")
	interval    = flag.Duration("interval", 5*time.Minute, "duration between refresh runs")
	verbose     = flag.Bool("verbose", false, "print info level logs to stdout")
	systemLog   = flag.Bool("system_log", false, "log to Linux Syslog or Windows Event Log")
	port        = flag.Int("port", 8000, "listen port")
	repoName    = flag.String("repo_name", "repo", "name of the repo to setup")
	packagePath = flag.String("package_path", "packages", "path under both the filesystem (-root flag) and webserver root where packages are located")

	repoContents *repoPackages
)

// repoPackages describes a repository of packages.
type repoPackages struct {
	rs []goolib.RepoSpec
	mu sync.Mutex
}

// add provides a thread safe way to add a package to repoPackages.
func (r *repoPackages) add(src, chksum string, spec *goolib.PkgSpec) {
	r.mu.Lock()
	defer r.mu.Unlock()
	r.rs = append(r.rs, goolib.RepoSpec{
		Source:      src,
		Checksum:    chksum,
		PackageSpec: spec,
	})
}

func packageInfo(pkgPath string) error {
	pkg := filepath.Base(pkgPath)
	pi := goolib.PkgNameSplit(strings.TrimSuffix(pkg, ".goo"))

	spec, err := extractSpec(pkgPath)
	if err != nil {
		return err
	}
	if spec.Name != pi.Name {
		return fmt.Errorf("%s: name in spec does not match package file name", pkgPath)
	}
	if spec.Arch != pi.Arch {
		return fmt.Errorf("%s: arch in spec does not match package file name", pkgPath)
	}
	if spec.Version != pi.Ver {
		return fmt.Errorf("%s: version in spec does not match package version", pkgPath)
	}

	f, err := oswrap.Open(pkgPath)
	if err != nil {
		return err
	}
	defer f.Close()

	repoContents.add(path.Join(*packagePath, pkg), goolib.Checksum(f), spec)
	return nil
}

func runSync(packageDir string) error {
	logger.Info("Beginning sync run")
	if err := oswrap.MkdirAll(packageDir, 0774); err != nil {
		return err
	}

	pkgs, err := filepath.Glob(filepath.Join(packageDir, "*.goo"))
	if err != nil {
		return err
	}

	repoContents = &repoPackages{}
	var wg sync.WaitGroup
	for _, pkg := range pkgs {
		wg.Add(1)
		go func(pkg string) {
			defer wg.Done()
			if err := packageInfo(pkg); err != nil {
				logger.Error(err)
			}
		}(pkg)
	}
	wg.Wait()
	logger.Info("Sync run completed successfully")
	return nil
}

// extractSpec takes a goopkg file and returns the unmarshalled spec file.
func extractSpec(pkgPath string) (*goolib.PkgSpec, error) {
	f, err := oswrap.Open(pkgPath)
	if err != nil {
		return nil, err
	}
	defer f.Close()
	return goolib.ExtractPkgSpec(f)
}

func serve(w http.ResponseWriter, r *http.Request) {
	out, err := json.MarshalIndent(repoContents.rs, "", "  ")
	if err != nil {
		logger.Fatal(err)
	}
	w.Header().Set("Content-Type", "application/json")
	w.Write(out)
}

func main() {
	flag.Parse()

	logger.Init("GooServe", *verbose, *systemLog, ioutil.Discard)

	packageDir := filepath.Join(*root, *packagePath)
	if err := runSync(packageDir); err != nil {
		logger.Error(err)
	}

	http.HandleFunc(fmt.Sprintf("/%s/index", *repoName), serve)
	prefix := "/" + *packagePath + "/"
	http.Handle(prefix, http.StripPrefix(prefix, http.FileServer(http.Dir(packageDir))))
	go func() {
		err := http.ListenAndServe(fmt.Sprintf(":%d", *port), nil)
		if err != nil {
			logger.Fatal(err)
		}
	}()

	for range time.Tick(*interval) {
		if err := runSync(packageDir); err != nil {
			logger.Error(err)
		}
	}
}
<|endoftext|>


<commit_before>package minify // import "github.com/tdewolff/minify"

import (
	"bufio"
	"bytes"
	"errors"
	"fmt"
	"io"
	"net/http"
	"os"
	"os/exec"
	"regexp"
	"strings"
	"testing"

	"github.com/stretchr/testify/assert"
)

var errDummy = errors.New("dummy error")

// from os/exec/exec_test.go
func helperCommand(t *testing.T, s ...string) *exec.Cmd {
	cs := []string{"-test.run=TestHelperProcess", "--"}
	cs = append(cs, s...)
	cmd := exec.Command(os.Args[0], cs...)
	cmd.Env = []string{"GO_WANT_HELPER_PROCESS=1"}
	return cmd
}

func helperMinifyString(t *testing.T, m *M, mediatype string) string {
	s, err := m.String(mediatype, "")
	assert.Nil(t, err, "minifier must not return error for '"+mediatype+"'")
	return s
}

////////////////////////////////////////////////////////////////

var m *M

func init() {
	m = New()
	m.AddFunc("dummy/copy", func(m *M, w io.Writer, r io.Reader, _ map[string]string) error {
		io.Copy(w, r)
		return nil
	})
	m.AddFunc("dummy/nil", func(m *M, w io.Writer, r io.Reader, _ map[string]string) error {
		return nil
	})
	m.AddFunc("dummy/err", func(m *M, w io.Writer, r io.Reader, _ map[string]string) error {
		return errDummy
	})
	m.AddFunc("dummy/charset", func(m *M, w io.Writer, r io.Reader, params map[string]string) error {
		w.Write([]byte(params["charset"]))
		return nil
	})
	m.AddFunc("dummy/params", func(m *M, w io.Writer, r io.Reader, params map[string]string) error {
		return m.Minify(params["type"]+"/"+params["sub"], w, r)
	})
	m.AddFunc("type/sub", func(m *M, w io.Writer, r io.Reader, _ map[string]string) error {
		w.Write([]byte("type/sub"))
		return nil
	})
	m.AddFuncRegexp(regexp.MustCompile("^type/.+$"), func(m *M, w io.Writer, r io.Reader, _ map[string]string) error {
		w.Write([]byte("type/*"))
		return nil
	})
	m.AddFuncRegexp(regexp.MustCompile("^.+/.+$"), func(m *M, w io.Writer, r io.Reader, _ map[string]string) error {
		w.Write([]byte("*/*"))
		return nil
	})
}

func TestMinify(t *testing.T) {
	assert.Equal(t, ErrNotExist, m.Minify("?", nil, nil), "must return ErrNotExist when minifier doesn't exist")
	assert.Nil(t, m.Minify("dummy/nil", nil, nil), "must return nil for dummy/nil")
	assert.Equal(t, errDummy, m.Minify("dummy/err", nil, nil), "must return errDummy for dummy/err")

	b := []byte("test")
	out, err := m.Bytes("dummy/nil", b)
	assert.Nil(t, err, "must not return error for dummy/nil")
	assert.Equal(t, []byte{}, out, "must return empty byte array for dummy/nil")
	out, err = m.Bytes("?", b)
	assert.Equal(t, ErrNotExist, err, "must return ErrNotExist when minifier doesn't exist")
	assert.Equal(t, b, out, "must return input byte array when minifier doesn't exist")

	s := "test"
	out2, err := m.String("dummy/nil", s)
	assert.Nil(t, err, "must not return error for dummy/nil")
	assert.Equal(t, "", out2, "must return empty string for dummy/nil")
	out2, err = m.String("?", s)
	assert.Equal(t, ErrNotExist, err, "must return ErrNotExist when minifier doesn't exist")
	assert.Equal(t, s, out2, "must return input string when minifier doesn't exist")
}

func TestAdd(t *testing.T) {
	m := New()
	w := &bytes.Buffer{}
	r := bytes.NewBufferString("test")
	m.AddFunc("dummy/err", func(m *M, w io.Writer, r io.Reader, _ map[string]string) error {
		return errDummy
	})
	assert.Equal(t, errDummy, m.Minify("dummy/err", nil, nil), "must return errDummy for dummy/err")

	m.AddCmd("dummy/copy", helperCommand(t, "dummy/copy"))
	m.AddCmd("dummy/err", helperCommand(t, "dummy/err"))
	m.AddCmdRegexp(regexp.MustCompile("err$"), helperCommand(t, "werr"))
	assert.Nil(t, m.Minify("dummy/copy", w, r), "must return nil for dummy/copy command")
	assert.Equal(t, "test", w.String(), "must return input string for dummy/copy command")
	assert.Equal(t, "exit status 1", m.Minify("dummy/err", w, r).Error(), "must return proper exit status when command encounters error")
	assert.Equal(t, "exit status 2", m.Minify("werr", w, r).Error(), "must return proper exit status when command encounters error")
	assert.Equal(t, "exit status 2", m.Minify("stderr", w, r).Error(), "must return proper exit status when command encounters error")
}

func TestWildcard(t *testing.T) {
	assert.Equal(t, "type/sub", helperMinifyString(t, m, "type/sub"), "must return type/sub for type/sub")
	assert.Equal(t, "type/*", helperMinifyString(t, m, "type/*"), "must return type/* for type/*")
	assert.Equal(t, "*/*", helperMinifyString(t, m, "*/*"), "must return */* for */*")
	assert.Equal(t, "type/*", helperMinifyString(t, m, "type/sub2"), "must return type/* for type/sub2")
	assert.Equal(t, "*/*", helperMinifyString(t, m, "type2/sub"), "must return */* for type2/sub")
	assert.Equal(t, "UTF-8", helperMinifyString(t, m, "dummy/charset;charset=UTF-8"), "must return UTF-8 for dummy/charset;charset=UTF-8")
	assert.Equal(t, "UTF-8", helperMinifyString(t, m, "dummy/charset; charset = UTF-8 "), "must return UTF-8 for ' dummy/charset; charset = UTF-8 '")
	assert.Equal(t, "type/sub", helperMinifyString(t, m, "dummy/params;type=type;sub=sub"), "must return type/sub for dummy/params;type=type;sub=sub")
}

func TestHelperProcess(*testing.T) {
	if os.Getenv("GO_WANT_HELPER_PROCESS") != "1" {
		return
	}
	args := os.Args
	for len(args) > 0 {
		if args[0] == "--" {
			args = args[1:]
			break
		}
		args = args[1:]
	}
	if len(args) == 0 {
		fmt.Fprintf(os.Stderr, "No command\n")
		os.Exit(2)
	}
	cmd, args := args[0], args[1:]
	switch cmd {
	case "dummy/copy":
		io.Copy(os.Stdout, os.Stdin)
	case "dummy/err":
		os.Exit(1)
	default:
		os.Exit(2)
	}
	os.Exit(0)
}

////////////////////////////////////////////////////////////////

func ExampleM_Minify_custom() {
	m := New()
	m.AddFunc("text/plain", func(m *M, w io.Writer, r io.Reader, _ map[string]string) error {
		// remove all newlines and spaces
		rb := bufio.NewReader(r)
		for {
			line, err := rb.ReadString('\n')
			if err != nil && err != io.EOF {
				return err
			}
			if _, errws := io.WriteString(w, strings.Replace(line, " ", "", -1)); errws != nil {
				return errws
			}
			if err == io.EOF {
				break
			}
		}
		return nil
	})

	in := "Because my coffee was too cold, I heated it in the microwave."
	out, err := m.String("text/plain", in)
	if err != nil {
		panic(err)
	}
	fmt.Println(out)
	// Output: Becausemycoffeewastoocold,Iheateditinthemicrowave.
}

func ExampleM_Reader() {
	b := bytes.NewReader([]byte("input"))

	m := New()
	// add minfiers

	r := m.Reader("mime/type", b)
	if _, err := io.Copy(os.Stdout, r); err != nil {
		if _, err := io.Copy(os.Stdout, b); err != nil {
			panic(err)
		}
	}
}

func ExampleM_Writer() {
	m := New()
	// add minfiers

	w := m.Writer("mime/type", os.Stdout)
	if _, err := w.Write([]byte("input")); err != nil {
		panic(err)
	}
	if err := w.Close(); err != nil {
		panic(err)
	}
}

type MinifierResponseWriter struct {
	http.ResponseWriter
	io.Writer
}

func (m MinifierResponseWriter) Write(b []byte) (int, error) {
	return m.Writer.Write(b)
}

func ExampleM_Minify_responseWriter(res http.ResponseWriter) http.ResponseWriter {
	m := New()
	// add minfiers

	pr, pw := io.Pipe()
	go func(w io.Writer) {
		if err := m.Minify("mime/type", w, pr); err != nil {
			panic(err)
		}
	}(res)
	return MinifierResponseWriter{res, pw}
}
<commit_msg>Commentary added to example<commit_after>package minify // import "github.com/tdewolff/minify"

import (
	"bufio"
	"bytes"
	"errors"
	"fmt"
	"io"
	"net/http"
	"os"
	"os/exec"
	"regexp"
	"strings"
	"testing"

	"github.com/stretchr/testify/assert"
)

var errDummy = errors.New("dummy error")

// from os/exec/exec_test.go
func helperCommand(t *testing.T, s ...string) *exec.Cmd {
	cs := []string{"-test.run=TestHelperProcess", "--"}
	cs = append(cs, s...)
	cmd := exec.Command(os.Args[0], cs...)
	cmd.Env = []string{"GO_WANT_HELPER_PROCESS=1"}
	return cmd
}

func helperMinifyString(t *testing.T, m *M, mediatype string) string {
	s, err := m.String(mediatype, "")
	assert.Nil(t, err, "minifier must not return error for '"+mediatype+"'")
	return s
}

////////////////////////////////////////////////////////////////

var m *M

func init() {
	m = New()
	m.AddFunc("dummy/copy", func(m *M, w io.Writer, r io.Reader, _ map[string]string) error {
		io.Copy(w, r)
		return nil
	})
	m.AddFunc("dummy/nil", func(m *M, w io.Writer, r io.Reader, _ map[string]string) error {
		return nil
	})
	m.AddFunc("dummy/err", func(m *M, w io.Writer, r io.Reader, _ map[string]string) error {
		return errDummy
	})
	m.AddFunc("dummy/charset", func(m *M, w io.Writer, r io.Reader, params map[string]string) error {
		w.Write([]byte(params["charset"]))
		return nil
	})
	m.AddFunc("dummy/params", func(m *M, w io.Writer, r io.Reader, params map[string]string) error {
		return m.Minify(params["type"]+"/"+params["sub"], w, r)
	})
	m.AddFunc("type/sub", func(m *M, w io.Writer, r io.Reader, _ map[string]string) error {
		w.Write([]byte("type/sub"))
		return nil
	})
	m.AddFuncRegexp(regexp.MustCompile("^type/.+$"), func(m *M, w io.Writer, r io.Reader, _ map[string]string) error {
		w.Write([]byte("type/*"))
		return nil
	})
	m.AddFuncRegexp(regexp.MustCompile("^.+/.+$"), func(m *M, w io.Writer, r io.Reader, _ map[string]string) error {
		w.Write([]byte("*/*"))
		return nil
	})
}

func TestMinify(t *testing.T) {
	assert.Equal(t, ErrNotExist, m.Minify("?", nil, nil), "must return ErrNotExist when minifier doesn't exist")
	assert.Nil(t, m.Minify("dummy/nil", nil, nil), "must return nil for dummy/nil")
	assert.Equal(t, errDummy, m.Minify("dummy/err", nil, nil), "must return errDummy for dummy/err")

	b := []byte("test")
	out, err := m.Bytes("dummy/nil", b)
	assert.Nil(t, err, "must not return error for dummy/nil")
	assert.Equal(t, []byte{}, out, "must return empty byte array for dummy/nil")
	out, err = m.Bytes("?", b)
	assert.Equal(t, ErrNotExist, err, "must return ErrNotExist when minifier doesn't exist")
	assert.Equal(t, b, out, "must return input byte array when minifier doesn't exist")

	s := "test"
	out2, err := m.String("dummy/nil", s)
	assert.Nil(t, err, "must not return error for dummy/nil")
	assert.Equal(t, "", out2, "must return empty string for dummy/nil")
	out2, err = m.String("?", s)
	assert.Equal(t, ErrNotExist, err, "must return ErrNotExist when minifier doesn't exist")
	assert.Equal(t, s, out2, "must return input string when minifier doesn't exist")
}

func TestAdd(t *testing.T) {
	m := New()
	w := &bytes.Buffer{}
	r := bytes.NewBufferString("test")
	m.AddFunc("dummy/err", func(m *M, w io.Writer, r io.Reader, _ map[string]string) error {
		return errDummy
	})
	assert.Equal(t, errDummy, m.Minify("dummy/err", nil, nil), "must return errDummy for dummy/err")

	m.AddCmd("dummy/copy", helperCommand(t, "dummy/copy"))
	m.AddCmd("dummy/err", helperCommand(t, "dummy/err"))
	m.AddCmdRegexp(regexp.MustCompile("err$"), helperCommand(t, "werr"))
	assert.Nil(t, m.Minify("dummy/copy", w, r), "must return nil for dummy/copy command")
	assert.Equal(t, "test", w.String(), "must return input string for dummy/copy command")
	assert.Equal(t, "exit status 1", m.Minify("dummy/err", w, r).Error(), "must return proper exit status when command encounters error")
	assert.Equal(t, "exit status 2", m.Minify("werr", w, r).Error(), "must return proper exit status when command encounters error")
	assert.Equal(t, "exit status 2", m.Minify("stderr", w, r).Error(), "must return proper exit status when command encounters error")
}

func TestWildcard(t *testing.T) {
	assert.Equal(t, "type/sub", helperMinifyString(t, m, "type/sub"), "must return type/sub for type/sub")
	assert.Equal(t, "type/*", helperMinifyString(t, m, "type/*"), "must return type/* for type/*")
	assert.Equal(t, "*/*", helperMinifyString(t, m, "*/*"), "must return */* for */*")
	assert.Equal(t, "type/*", helperMinifyString(t, m, "type/sub2"), "must return type/* for type/sub2")
	assert.Equal(t, "*/*", helperMinifyString(t, m, "type2/sub"), "must return */* for type2/sub")
	assert.Equal(t, "UTF-8", helperMinifyString(t, m, "dummy/charset;charset=UTF-8"), "must return UTF-8 for dummy/charset;charset=UTF-8")
	assert.Equal(t, "UTF-8", helperMinifyString(t, m, "dummy/charset; charset = UTF-8 "), "must return UTF-8 for ' dummy/charset; charset = UTF-8 '")
	assert.Equal(t, "type/sub", helperMinifyString(t, m, "dummy/params;type=type;sub=sub"), "must return type/sub for dummy/params;type=type;sub=sub")
}

func TestHelperProcess(*testing.T) {
	if os.Getenv("GO_WANT_HELPER_PROCESS") != "1" {
		return
	}
	args := os.Args
	for len(args) > 0 {
		if args[0] == "--" {
			args = args[1:]
			break
		}
		args = args[1:]
	}
	if len(args) == 0 {
		fmt.Fprintf(os.Stderr, "No command\n")
		os.Exit(2)
	}
	cmd, args := args[0], args[1:]
	switch cmd {
	case "dummy/copy":
		io.Copy(os.Stdout, os.Stdin)
	case "dummy/err":
		os.Exit(1)
	default:
		os.Exit(2)
	}
	os.Exit(0)
}

////////////////////////////////////////////////////////////////

func ExampleM_Minify_custom() {
	m := New()
	m.AddFunc("text/plain", func(m *M, w io.Writer, r io.Reader, _ map[string]string) error {
		// remove all newlines and spaces
		rb := bufio.NewReader(r)
		for {
			line, err := rb.ReadString('\n')
			if err != nil && err != io.EOF {
				return err
			}
			if _, errws := io.WriteString(w, strings.Replace(line, " ", "", -1)); errws != nil {
				return errws
			}
			if err == io.EOF {
				break
			}
		}
		return nil
	})

	in := "Because my coffee was too cold, I heated it in the microwave."
	out, err := m.String("text/plain", in)
	if err != nil {
		panic(err)
	}
	fmt.Println(out)
	// Output: Becausemycoffeewastoocold,Iheateditinthemicrowave.
}

func ExampleM_Reader() {
	b := bytes.NewReader([]byte("input"))

	m := New()
	// add minfiers

	r := m.Reader("mime/type", b)
	if _, err := io.Copy(os.Stdout, r); err != nil {
		if _, err := io.Copy(os.Stdout, b); err != nil {
			panic(err)
		}
	}
}

func ExampleM_Writer() {
	m := New()
	// add minfiers

	w := m.Writer("mime/type", os.Stdout)
	if _, err := w.Write([]byte("input")); err != nil {
		panic(err)
	}
	if err := w.Close(); err != nil {
		panic(err)
	}
}

type MinifierResponseWriter struct {
	http.ResponseWriter
	io.Writer
}

func (m MinifierResponseWriter) Write(b []byte) (int, error) {
	return m.Writer.Write(b)
}

func ExampleM_Minify_responseWriter(res http.ResponseWriter) http.ResponseWriter {
	// Define the accompanying struct:
	// type MinifierResponseWriter struct {
	// 	http.ResponseWriter
	// 	io.Writer
	// }

	// func (m MinifierResponseWriter) Write(b []byte) (int, error) {
	// 	return m.Writer.Write(b)
	// }

	m := New()
	// add minfiers

	pr, pw := io.Pipe()
	go func(w io.Writer) {
		if err := m.Minify("mime/type", w, pr); err != nil {
			panic(err)
		}
	}(res)
	return MinifierResponseWriter{res, pw}
}
<|endoftext|>


<commit_before>package testutil

import (
	"fmt"
	"io/ioutil"
	"os"
	"testing"
)

// tmpdir is the base directory for all temporary directories
// and files created with TempDir and TempFile. This could be
// achieved by setting a system environment variable but then
// the test execution would depend on whether or not the
// environment variable is set.
//
// On macOS the temp base directory is quite long and that
// triggers a problem with some tests that bind to UNIX sockets
// where the filename seems to be too long. Using a shorter name
// fixes this and makes the paths more readable.
//
// It also provides a single base directory for cleanup.
var tmpdir = "/tmp/consul-test"

func init() {
	if err := os.MkdirAll(tmpdir, 0755); err != nil {
		fmt.Println("Cannot create %s. Reverting to /tmp", tmpdir)
		tmpdir = "/tmp"
	}
}

// TempDir creates a temporary directory within tmpdir
// with the name 'testname-name'. If the directory cannot
// be created t.Fatal is called.
func TempDir(t *testing.T, name string) string {
	if t != nil && t.Name() != "" {
		name = t.Name() + "-" + name
	}
	d, err := ioutil.TempDir(tmpdir, name)
	if err != nil {
		t.Fatalf("err: %s", err)
	}
	return d
}

// TempFile creates a temporary file within tmpdir
// with the name 'testname-name'. If the file cannot
// be created t.Fatal is called. If a temporary directory
// has been created before consider storing the file
// inside this directory to avoid double cleanup.
func TempFile(t *testing.T, name string) *os.File {
	if t != nil && t.Name() != "" {
		name = t.Name() + "-" + name
	}
	f, err := ioutil.TempFile(tmpdir, name)
	if err != nil {
		t.Fatalf("err: %s", err)
	}
	return f
}
<commit_msg>test: fix go vet issue<commit_after>package testutil

import (
	"fmt"
	"io/ioutil"
	"os"
	"testing"
)

// tmpdir is the base directory for all temporary directories
// and files created with TempDir and TempFile. This could be
// achieved by setting a system environment variable but then
// the test execution would depend on whether or not the
// environment variable is set.
//
// On macOS the temp base directory is quite long and that
// triggers a problem with some tests that bind to UNIX sockets
// where the filename seems to be too long. Using a shorter name
// fixes this and makes the paths more readable.
//
// It also provides a single base directory for cleanup.
var tmpdir = "/tmp/consul-test"

func init() {
	if err := os.MkdirAll(tmpdir, 0755); err != nil {
		fmt.Printf("Cannot create %s. Reverting to /tmp\n", tmpdir)
		tmpdir = "/tmp"
	}
}

// TempDir creates a temporary directory within tmpdir
// with the name 'testname-name'. If the directory cannot
// be created t.Fatal is called.
func TempDir(t *testing.T, name string) string {
	if t != nil && t.Name() != "" {
		name = t.Name() + "-" + name
	}
	d, err := ioutil.TempDir(tmpdir, name)
	if err != nil {
		t.Fatalf("err: %s", err)
	}
	return d
}

// TempFile creates a temporary file within tmpdir
// with the name 'testname-name'. If the file cannot
// be created t.Fatal is called. If a temporary directory
// has been created before consider storing the file
// inside this directory to avoid double cleanup.
func TempFile(t *testing.T, name string) *os.File {
	if t != nil && t.Name() != "" {
		name = t.Name() + "-" + name
	}
	f, err := ioutil.TempFile(tmpdir, name)
	if err != nil {
		t.Fatalf("err: %s", err)
	}
	return f
}
<|endoftext|>