<commit_before>// Copyright 2015, Klaus Post, see LICENSE for details.

// Dictionary Password Validation for Go
//
// For usage and examples see: https://github.com/klauspost/password
// (or open the README.md)
//
// This library will help you import a password dictionary and will allow you
// to validate new/changed passwords against the dictionary.
//
// You are able to use your own database and password dictionary.
// Currently the package supports importing dictionaries similar to
// CrackStation's Password Cracking Dictionary: https://crackstation.net/buy-crackstation-wordlist-password-cracking-dictionary.htm
//
// It and has "drivers" for various backends, see the "drivers"  directory, where there are
// implementations and a test framework that will help you test your own drivers.
package password

import (
	"errors"
	"io"
	"log"
	"os"
	"strings"
	"time"
	"unicode/utf8"

	"golang.org/x/text/unicode/norm"
)

// Logger used for output during Import.
// This can be exchanged with your own.
var Logger = log.New(os.Stdout, "", log.LstdFlags)

// A DbWriter is used for adding passwords to a database.
// Items sent to Add has always been sanitized, however
// the same passwords can be sent multiple times.
type DbWriter interface {
	Add(string) error
}

// A DB should check the database for the supplied password.
// The password sent to the interface has always been sanitized.
type DB interface {
	Has(string) (bool, error)
}

// A Sanitizer should prepare a password, and check
// the basic properties that should be satisfied.
// For an example, see DefaultSanitizer
type Sanitizer interface {
	Sanitize(string) (string, error)
}

// Tokenizer delivers input tokens (passwords).
// Calling Next() should return the next password, and when
// finished io.EOF should be returned.
//
// It is ok for the Tokenizer to send empty strings and duplicate
// values.
type Tokenizer interface {
	Next() (string, error)
}

// DefaultSanitizer should be used for adding passwords
// to the database.
// Assumes input is UTF8.
//
// DefaultSanitizer performs the following sanitazion:
//
//  - Trim space, tab and newlines from start+end of input
//  - Check that there is at least 8 runes. Return ErrSanitizeTooShort if not.
//  - Check that the input is valid utf8. Return ErrInvalidString if not.
//  - Normalize input using Unicode Normalization Form KD
//
// If input is less than 8 runes ErrSanitizeTooShort is returned.
var DefaultSanitizer Sanitizer

func init() {
	DefaultSanitizer = &defaultSanitizer{}
}

// ErrSanitizeTooShort is returned by the default sanitizer,
// if the input password is less than 8 runes.
var ErrSanitizeTooShort = errors.New("password too short")

// ErrInvalidString is returned by the default sanitizer
// if the string contains an invalid utf8 character sequence.
var ErrInvalidString = errors.New("invalid utf8 sequence")

// ErrPasswordInDB is returedn by Check, if the password is in the
// database.
var ErrPasswordInDB = errors.New("password found in database")

// doc at DefaultSanitizer
type defaultSanitizer struct{}

// doc at DefaultSanitizer
func (d defaultSanitizer) Sanitize(in string) (string, error) {
	in = strings.TrimSpace(in)
	if utf8.RuneCountInString(in) < 8 {
		return "", ErrSanitizeTooShort
	}
	if !utf8.ValidString(in) {
		return "", ErrInvalidString
	}
	in = norm.NFKD.String(in)
	return in, nil
}

type initer interface {
	Init() error
}

// Import will populate a database with common passwords.
//
// You must supply a Tokenizer (see tokenizer package for default tokenizers)
// that will deliver the passwords,
// a DbWriter, where the passwords will be sent,
// and finally a Sanitizer to clean up the passwords -
// - if you send nil DefaultSanitizer will be used.
func Import(in Tokenizer, out DbWriter, san Sanitizer) (err error) {
	bulk, ok := out.(BulkWriter)
	if ok {
		initer, ok := out.(initer)
		if ok {
			err := initer.Init()
			if err != nil {
				return err
			}
		}
		closer, ok := out.(io.Closer)
		if ok {
			defer func() {
				e := closer.Close()
				if e != nil && err == nil {
					err = e
				}
			}()
		}
		out = bulkWrap(bulk)
	}

	initer, ok := out.(initer)
	if ok {
		err := initer.Init()
		if err != nil {
			return err
		}
	}

	closer, ok := out.(io.Closer)
	if ok {
		defer func() {
			e := closer.Close()
			if e != nil && err == nil {
				err = e
			}
		}()
	}

	if san == nil {
		san = DefaultSanitizer
	}

	start := time.Now()
	i := 0
	added := 0
	for {
		record, err := in.Next()
		if err == io.EOF {
			break
		}
		if err != nil {
			return err
		}

		valstring, err := san.Sanitize(record)
		if err == nil {
			valstring = strings.ToLower(valstring)
			err = out.Add(valstring)
			if err != nil {
				return err
			}
			added++
		}
		i++
		if i%10000 == 0 {
			elapsed := time.Since(start)
			Logger.Printf("Read %d, (%0.0f per sec). Added: %d (%d%%)\n", i, float64(i)/elapsed.Seconds(), added, (added*100)/i)
		}
	}
	elapsed := time.Since(start)
	Logger.Printf("Processing took %s, processing %d entries.\n", elapsed, i)
	Logger.Printf("%0.2f entries/sec.", float64(i)/elapsed.Seconds())
	return nil
}

// Check a password against the database.
// It will return an error if:
//  - Sanitazition fails.
//  - DB lookup returns an error
//  - Password is in database (ErrPasswordInDB)
// If nil is passed as Sanitizer, DefaultSanitizer will be used.
func Check(password string, db DB, san Sanitizer) error {
	if san == nil {
		san = DefaultSanitizer
	}
	p, err := san.Sanitize(password)
	if err != nil {
		return err
	}
	p = strings.ToLower(p)
	has, err := db.Has(p)
	if err != nil {
		return err
	}
	if has {
		return ErrPasswordInDB
	}
	return nil
}

// Sanitize will sanitize a password, useful before hashing
// and storing it.
//
// If the sanitizer is nil, DefaultSanitizer will be used.
func Sanitize(password string, san Sanitizer) (string, error) {
	if san == nil {
		san = DefaultSanitizer
	}
	p, err := san.Sanitize(password)
	return p, err
}

// SanitizeOK can be used to check if a password passes the sanitizer.
//
// If the sanitizer is nil, DefaultSanitizer will be used.
func SanitizeOK(password string, san Sanitizer) error {
	if san == nil {
		san = DefaultSanitizer
	}
	_, err := san.Sanitize(password)
	return err
}
<commit_msg>Trim space after normalize, just in case something changed.<commit_after>// Copyright 2015, Klaus Post, see LICENSE for details.

// Dictionary Password Validation for Go
//
// For usage and examples see: https://github.com/klauspost/password
// (or open the README.md)
//
// This library will help you import a password dictionary and will allow you
// to validate new/changed passwords against the dictionary.
//
// You are able to use your own database and password dictionary.
// Currently the package supports importing dictionaries similar to
// CrackStation's Password Cracking Dictionary: https://crackstation.net/buy-crackstation-wordlist-password-cracking-dictionary.htm
//
// It and has "drivers" for various backends, see the "drivers"  directory, where there are
// implementations and a test framework that will help you test your own drivers.
package password

import (
	"errors"
	"io"
	"log"
	"os"
	"strings"
	"time"
	"unicode/utf8"

	"golang.org/x/text/unicode/norm"
)

// Logger used for output during Import.
// This can be exchanged with your own.
var Logger = log.New(os.Stdout, "", log.LstdFlags)

// A DbWriter is used for adding passwords to a database.
// Items sent to Add has always been sanitized, however
// the same passwords can be sent multiple times.
type DbWriter interface {
	Add(string) error
}

// A DB should check the database for the supplied password.
// The password sent to the interface has always been sanitized.
type DB interface {
	Has(string) (bool, error)
}

// A Sanitizer should prepare a password, and check
// the basic properties that should be satisfied.
// For an example, see DefaultSanitizer
type Sanitizer interface {
	Sanitize(string) (string, error)
}

// Tokenizer delivers input tokens (passwords).
// Calling Next() should return the next password, and when
// finished io.EOF should be returned.
//
// It is ok for the Tokenizer to send empty strings and duplicate
// values.
type Tokenizer interface {
	Next() (string, error)
}

// DefaultSanitizer should be used for adding passwords
// to the database.
// Assumes input is UTF8.
//
// DefaultSanitizer performs the following sanitazion:
//
//  - Trim space, tab and newlines from start+end of input
//  - Check that there is at least 8 runes. Return ErrSanitizeTooShort if not.
//  - Check that the input is valid utf8. Return ErrInvalidString if not.
//  - Normalize input using Unicode Normalization Form KD
//
// If input is less than 8 runes ErrSanitizeTooShort is returned.
var DefaultSanitizer Sanitizer

func init() {
	DefaultSanitizer = &defaultSanitizer{}
}

// ErrSanitizeTooShort is returned by the default sanitizer,
// if the input password is less than 8 runes.
var ErrSanitizeTooShort = errors.New("password too short")

// ErrInvalidString is returned by the default sanitizer
// if the string contains an invalid utf8 character sequence.
var ErrInvalidString = errors.New("invalid utf8 sequence")

// ErrPasswordInDB is returedn by Check, if the password is in the
// database.
var ErrPasswordInDB = errors.New("password found in database")

// doc at DefaultSanitizer
type defaultSanitizer struct{}

// doc at DefaultSanitizer
func (d defaultSanitizer) Sanitize(in string) (string, error) {
	in = strings.TrimSpace(in)
	if utf8.RuneCountInString(in) < 8 {
		return "", ErrSanitizeTooShort
	}
	if !utf8.ValidString(in) {
		return "", ErrInvalidString
	}
	in = norm.NFKD.String(in)
	in = strings.TrimSpace(in)
	return in, nil
}

type initer interface {
	Init() error
}

// Import will populate a database with common passwords.
//
// You must supply a Tokenizer (see tokenizer package for default tokenizers)
// that will deliver the passwords,
// a DbWriter, where the passwords will be sent,
// and finally a Sanitizer to clean up the passwords -
// - if you send nil DefaultSanitizer will be used.
func Import(in Tokenizer, out DbWriter, san Sanitizer) (err error) {
	bulk, ok := out.(BulkWriter)
	if ok {
		initer, ok := out.(initer)
		if ok {
			err := initer.Init()
			if err != nil {
				return err
			}
		}
		closer, ok := out.(io.Closer)
		if ok {
			defer func() {
				e := closer.Close()
				if e != nil && err == nil {
					err = e
				}
			}()
		}
		out = bulkWrap(bulk)
	}

	initer, ok := out.(initer)
	if ok {
		err := initer.Init()
		if err != nil {
			return err
		}
	}

	closer, ok := out.(io.Closer)
	if ok {
		defer func() {
			e := closer.Close()
			if e != nil && err == nil {
				err = e
			}
		}()
	}

	if san == nil {
		san = DefaultSanitizer
	}

	start := time.Now()
	i := 0
	added := 0
	for {
		record, err := in.Next()
		if err == io.EOF {
			break
		}
		if err != nil {
			return err
		}

		valstring, err := san.Sanitize(record)
		if err == nil {
			valstring = strings.ToLower(valstring)
			err = out.Add(valstring)
			if err != nil {
				return err
			}
			added++
		}
		i++
		if i%10000 == 0 {
			elapsed := time.Since(start)
			Logger.Printf("Read %d, (%0.0f per sec). Added: %d (%d%%)\n", i, float64(i)/elapsed.Seconds(), added, (added*100)/i)
		}
	}
	elapsed := time.Since(start)
	Logger.Printf("Processing took %s, processing %d entries.\n", elapsed, i)
	Logger.Printf("%0.2f entries/sec.", float64(i)/elapsed.Seconds())
	return nil
}

// Check a password against the database.
// It will return an error if:
//  - Sanitazition fails.
//  - DB lookup returns an error
//  - Password is in database (ErrPasswordInDB)
// If nil is passed as Sanitizer, DefaultSanitizer will be used.
func Check(password string, db DB, san Sanitizer) error {
	if san == nil {
		san = DefaultSanitizer
	}
	p, err := san.Sanitize(password)
	if err != nil {
		return err
	}
	p = strings.ToLower(p)
	has, err := db.Has(p)
	if err != nil {
		return err
	}
	if has {
		return ErrPasswordInDB
	}
	return nil
}

// Sanitize will sanitize a password, useful before hashing
// and storing it.
//
// If the sanitizer is nil, DefaultSanitizer will be used.
func Sanitize(password string, san Sanitizer) (string, error) {
	if san == nil {
		san = DefaultSanitizer
	}
	p, err := san.Sanitize(password)
	return p, err
}

// SanitizeOK can be used to check if a password passes the sanitizer.
//
// If the sanitizer is nil, DefaultSanitizer will be used.
func SanitizeOK(password string, san Sanitizer) error {
	if san == nil {
		san = DefaultSanitizer
	}
	_, err := san.Sanitize(password)
	return err
}
<|endoftext|>


<commit_before>// Copyright 2015 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

package main

import (
	"bytes"
	"crypto/sha1"
	"flag"
	"fmt"
	"io/ioutil"
	"log"
	"os"
	"os/exec"
	"path/filepath"
	"runtime"
	"strings"
)

var (
	verbose = flag.Bool("v", false, "print commands being run")
)

var goroot = runtime.GOROOT()

var binTools = []string{"go", "godoc", "gofmt"}

func main() {
	flag.Usage = func() {
		fmt.Fprintf(os.Stderr, "Usage:\n  %s [flags] save [name]\n  %s [flags] run name command...\n\nFlags:\n", os.Args[0], os.Args[0])
		flag.PrintDefaults()
	}

	flag.Parse()
	if flag.NArg() < 1 {
		flag.Usage()
		os.Exit(2)
	}

	switch flag.Arg(0) {
	case "save":
		if flag.NArg() > 2 {
			flag.Usage()
			os.Exit(2)
		}
		hash, diff := getHash()
		name := ""
		if flag.NArg() >= 2 {
			name = flag.Arg(1)
		}
		doSave(name, hash, diff)

	case "run":
		if flag.NArg() < 3 {
			flag.Usage()
			os.Exit(2)
		}
		doRun(flag.Arg(1), flag.Args()[2:])

	default:
		flag.Usage()
		os.Exit(2)
	}
}

func getHash() (string, []byte) {
	c := exec.Command("git", "-C", goroot, "rev-parse", "--short", "HEAD")
	out, err := c.CombinedOutput()
	if err != nil {
		log.Fatalf("git error %s: %s", err, out)
	}

	rev := strings.TrimSpace(string(out))

	c = exec.Command("git", "-C", goroot, "diff", "HEAD")
	out, err = c.CombinedOutput()
	if err != nil {
		log.Fatal("git error %s: %s", err, out)
	}

	if len(bytes.TrimSpace(out)) > 0 {
		diffHash := fmt.Sprintf("%x", sha1.Sum(out))
		return rev + "+" + diffHash[:10], out
	}
	return rev, nil
}

func doSave(name string, hash string, diff []byte) {
	// Create a minimal GOROOT at $GOROOT/gover/hash.
	savePath := filepath.Join(goroot, "gover", hash)
	goos, goarch := runtime.GOOS, runtime.GOARCH
	if x := os.Getenv("GOOS"); x != "" {
		goos = x
	}
	if x := os.Getenv("GOARCH"); x != "" {
		goarch = x
	}
	osArch := goos + "_" + goarch

	for _, binTool := range binTools {
		src := filepath.Join(goroot, "bin", binTool)
		if _, err := os.Stat(src); err == nil {
			cp(src, filepath.Join(savePath, "bin", binTool))
		}
	}
	cpR(filepath.Join(goroot, "pkg", osArch), filepath.Join(savePath, "pkg", osArch))
	cpR(filepath.Join(goroot, "pkg", "tool", osArch), filepath.Join(savePath, "pkg", "tool", osArch))
	cpR(filepath.Join(goroot, "pkg", "include"), filepath.Join(savePath, "pkg", "include"))
	cpR(filepath.Join(goroot, "src"), filepath.Join(savePath, "src"))

	if diff != nil {
		if err := ioutil.WriteFile(filepath.Join(savePath, "diff"), diff, 0666); err != nil {
			log.Fatal(err)
		}
	}

	// If there's a name, symlink it under that name.
	if name != "" {
		err := os.Symlink(hash, filepath.Join(goroot, "gover", name))
		if err != nil {
			log.Fatal(err)
		}
	}
}

func doRun(name string, cmd []string) {
	savePath := filepath.Join(goroot, "gover", name)

	c := exec.Command(filepath.Join(savePath, "bin", cmd[0]), cmd[1:]...)
	c.Env = append([]string(nil), os.Environ()...)
	c.Env = append(c.Env, "GOROOT="+savePath)

	c.Stdin, c.Stdout, c.Stderr = os.Stdin, os.Stdout, os.Stderr
	if err := c.Run(); err != nil {
		fmt.Printf("command failed: %s\n", err)
		os.Exit(1)
	}
}

func cp(src, dst string) {
	if *verbose {
		fmt.Printf("cp %s %s\n", src, dst)
	}
	if err := os.MkdirAll(filepath.Dir(dst), 0777); err != nil {
		log.Fatal(err)
	}
	data, err := ioutil.ReadFile(src)
	if err != nil {
		log.Fatal(err)
	}
	st, err := os.Stat(src)
	if err != nil {
		log.Fatal(err)
	}
	if err := ioutil.WriteFile(dst, data, st.Mode()); err != nil {
		log.Fatal(err)
	}
}

func cpR(src, dst string) {
	filepath.Walk(src, func(path string, info os.FileInfo, err error) error {
		if info.IsDir() {
			return nil
		}
		base := filepath.Base(path)
		if base == "core" || strings.HasSuffix(base, ".test") {
			return nil
		}

		cp(path, dst+path[len(src):])
		return nil
	})
}
<commit_msg>gover: retain mtime to avoid unnecessary rebuilds<commit_after>// Copyright 2015 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

package main

import (
	"bytes"
	"crypto/sha1"
	"flag"
	"fmt"
	"io/ioutil"
	"log"
	"os"
	"os/exec"
	"path/filepath"
	"runtime"
	"strings"
)

var (
	verbose = flag.Bool("v", false, "print commands being run")
)

var goroot = runtime.GOROOT()

var binTools = []string{"go", "godoc", "gofmt"}

func main() {
	flag.Usage = func() {
		fmt.Fprintf(os.Stderr, "Usage:\n  %s [flags] save [name]\n  %s [flags] run name command...\n\nFlags:\n", os.Args[0], os.Args[0])
		flag.PrintDefaults()
	}

	flag.Parse()
	if flag.NArg() < 1 {
		flag.Usage()
		os.Exit(2)
	}

	switch flag.Arg(0) {
	case "save":
		if flag.NArg() > 2 {
			flag.Usage()
			os.Exit(2)
		}
		hash, diff := getHash()
		name := ""
		if flag.NArg() >= 2 {
			name = flag.Arg(1)
		}
		doSave(name, hash, diff)

	case "run":
		if flag.NArg() < 3 {
			flag.Usage()
			os.Exit(2)
		}
		doRun(flag.Arg(1), flag.Args()[2:])

	default:
		flag.Usage()
		os.Exit(2)
	}
}

func getHash() (string, []byte) {
	c := exec.Command("git", "-C", goroot, "rev-parse", "--short", "HEAD")
	out, err := c.CombinedOutput()
	if err != nil {
		log.Fatalf("git error %s: %s", err, out)
	}

	rev := strings.TrimSpace(string(out))

	c = exec.Command("git", "-C", goroot, "diff", "HEAD")
	out, err = c.CombinedOutput()
	if err != nil {
		log.Fatal("git error %s: %s", err, out)
	}

	if len(bytes.TrimSpace(out)) > 0 {
		diffHash := fmt.Sprintf("%x", sha1.Sum(out))
		return rev + "+" + diffHash[:10], out
	}
	return rev, nil
}

func doSave(name string, hash string, diff []byte) {
	// Create a minimal GOROOT at $GOROOT/gover/hash.
	savePath := filepath.Join(goroot, "gover", hash)
	goos, goarch := runtime.GOOS, runtime.GOARCH
	if x := os.Getenv("GOOS"); x != "" {
		goos = x
	}
	if x := os.Getenv("GOARCH"); x != "" {
		goarch = x
	}
	osArch := goos + "_" + goarch

	for _, binTool := range binTools {
		src := filepath.Join(goroot, "bin", binTool)
		if _, err := os.Stat(src); err == nil {
			cp(src, filepath.Join(savePath, "bin", binTool))
		}
	}
	cpR(filepath.Join(goroot, "pkg", osArch), filepath.Join(savePath, "pkg", osArch))
	cpR(filepath.Join(goroot, "pkg", "tool", osArch), filepath.Join(savePath, "pkg", "tool", osArch))
	cpR(filepath.Join(goroot, "pkg", "include"), filepath.Join(savePath, "pkg", "include"))
	cpR(filepath.Join(goroot, "src"), filepath.Join(savePath, "src"))

	if diff != nil {
		if err := ioutil.WriteFile(filepath.Join(savePath, "diff"), diff, 0666); err != nil {
			log.Fatal(err)
		}
	}

	// If there's a name, symlink it under that name.
	if name != "" {
		err := os.Symlink(hash, filepath.Join(goroot, "gover", name))
		if err != nil {
			log.Fatal(err)
		}
	}
}

func doRun(name string, cmd []string) {
	savePath := filepath.Join(goroot, "gover", name)

	c := exec.Command(filepath.Join(savePath, "bin", cmd[0]), cmd[1:]...)
	c.Env = append([]string(nil), os.Environ()...)
	c.Env = append(c.Env, "GOROOT="+savePath)

	c.Stdin, c.Stdout, c.Stderr = os.Stdin, os.Stdout, os.Stderr
	if err := c.Run(); err != nil {
		fmt.Printf("command failed: %s\n", err)
		os.Exit(1)
	}
}

func cp(src, dst string) {
	if *verbose {
		fmt.Printf("cp %s %s\n", src, dst)
	}
	if err := os.MkdirAll(filepath.Dir(dst), 0777); err != nil {
		log.Fatal(err)
	}
	data, err := ioutil.ReadFile(src)
	if err != nil {
		log.Fatal(err)
	}
	st, err := os.Stat(src)
	if err != nil {
		log.Fatal(err)
	}
	if err := ioutil.WriteFile(dst, data, st.Mode()); err != nil {
		log.Fatal(err)
	}
	if err := os.Chtimes(dst, st.ModTime(), st.ModTime()); err != nil {
		log.Fatal(err)
	}
}

func cpR(src, dst string) {
	filepath.Walk(src, func(path string, info os.FileInfo, err error) error {
		if info.IsDir() {
			return nil
		}
		base := filepath.Base(path)
		if base == "core" || strings.HasSuffix(base, ".test") {
			return nil
		}

		cp(path, dst+path[len(src):])
		return nil
	})
}
<|endoftext|>


<commit_before>/* Copyright (c) 2014, Daniel Martí <mvdan@mvdan.cc> */
/* See LICENSE for licensing information */

package main

import (
	"errors"
	"flag"
	"fmt"
	"html/template"
	"io"
	"log"
	"math/rand"
	"net/http"
	"os"
	"path"
	"path/filepath"
	"regexp"
	"strconv"
	"strings"
	"sync"
	"time"
)

const (
	indexTmpl = "index.html"
	formTmpl  = "form.html"
	chars     = "abcdefghijklmnopqrstuvwxyz0123456789"
	randTries = 10

	// GET error messages
	invalidId     = "Invalid paste id."
	pasteNotFound = "Paste doesn't exist."
	unknownError  = "Something went terribly wrong."
	// POST error messages
	missingForm = "Paste could not be found inside the posted form."
)

var (
	siteUrl, listen, dataDir string
	lifeTime                 time.Duration
	maxSizeStr               string
	idSize                   int
	maxSize                  ByteSize

	validId       *regexp.Regexp
	regexByteSize = regexp.MustCompile(`^([\d\.]+)\s*([KM]?B|[BKM])$`)
	indexTemplate *template.Template
	formTemplate  *template.Template
	data          = struct {
		sync.RWMutex
		m map[Id]PasteInfo
	}{m: make(map[Id]PasteInfo)}
	customRand *rand.Rand
)

func init() {
	flag.StringVar(&siteUrl, "u", "http://localhost:8080", "URL of the site")
	flag.StringVar(&listen, "l", "localhost:8080", "Host and port to listen to")
	flag.StringVar(&dataDir, "d", "data", "Directory to store all the pastes in")
	flag.DurationVar(&lifeTime, "t", 12*time.Hour, "Lifetime of the pastes (units: s,m,h)")
	flag.StringVar(&maxSizeStr, "s", "1M", "Maximum size of POSTs in bytes (units: B,K,M)")
	flag.IntVar(&idSize, "i", 8, "Size of the paste ids (between 6 and 256)")
	validId = regexp.MustCompile("^[a-zA-Z0-9]{" + strconv.Itoa(idSize) + "}$")
	customRand = rand.New(rand.NewSource(time.Now().UnixNano()))
}

type PasteInfo struct {
	ModTime     time.Time
	DeathTime   time.Time
	Size        ByteSize
	Etag        string
	ContentType string
}

type Id string

func IdFromPath(idPath string) (Id, error) {
	parts := strings.Split(idPath, string(filepath.Separator))
	if len(parts) != 3 {
		return "", errors.New("Found invalid number of directories at " + idPath)
	}
	rawId := parts[0] + parts[1] + parts[2]
	if !validId.MatchString(rawId) {
		return "", errors.New("Found invalid id " + rawId)
	}
	return Id(rawId), nil
}

func RandomId() (Id, error) {
	s := make([]byte, idSize)
	var id Id
	data.RLock()
	for try := 0; try < randTries; try++ {
		var offset int = 0
	RandLoop:
		for {
			r := customRand.Int63()
			for i := 0; i < 8; i++ {
				randbyte := int(r&0xff) % len(chars)
				s[offset] = chars[randbyte]
				offset++
				if offset == idSize {
					break RandLoop
				}
				r >>= 8
			}
		}
		id = Id(s)
		if _, e := data.m[id]; !e {
			data.RUnlock()
			return id, nil
		}
	}
	return id, fmt.Errorf("Gave up trying to find an unused random id after %d tries", randTries)
}

func (id Id) String() string {
	return string(id)
}

func (id Id) Path() string {
	return path.Join(string(id[0:2]), string(id[2:4]), string(id[4:]))
}

func (id Id) EndLife() {
	data.Lock()
	err := os.Remove(id.Path())
	if err == nil {
		delete(data.m, id)
		data.Unlock()
		log.Printf("Removed paste: %s", id)
	} else {
		data.Unlock()
		log.Printf("Could not end the life of %s: %s", id, err)
		id.EndLifeAfter(2 * time.Minute)
	}
}

func (id Id) EndLifeAfter(duration time.Duration) {
	timer := time.NewTimer(duration)
	go func() {
		<-timer.C
		id.EndLife()
	}()
}

type ByteSize int64

const (
	B ByteSize = 1 << (10 * iota)
	KB
	MB
)

func parseByteSize(str string) (ByteSize, error) {
	if !regexByteSize.MatchString(str) {
		return 0, errors.New("Could not parse size in bytes")
	}
	parts := regexByteSize.FindStringSubmatch(str)
	size, _ := strconv.ParseFloat(string(parts[1]), 64)

	switch string(parts[2]) {
	case "KB", "K":
		size *= float64(KB)
	case "MB", "M":
		size *= float64(MB)
	}
	return ByteSize(size), nil
}

func (b ByteSize) String() string {
	switch {
	case b >= MB:
		return fmt.Sprintf("%.2f MB", float64(b)/float64(MB))
	case b >= KB:
		return fmt.Sprintf("%.2f KB", float64(b)/float64(KB))
	}
	return fmt.Sprintf("%d B", b)
}

func handler(w http.ResponseWriter, r *http.Request) {
	var err error
	switch r.Method {
	case "GET":
		switch r.URL.Path {
		case "/":
			indexTemplate.Execute(w, struct {
				SiteUrl, LifeTime string
			}{
				siteUrl,
				fmt.Sprintf("%g hours and %g minutes", lifeTime.Hours(),
					lifeTime.Minutes()-lifeTime.Hours()*60),
			})
			return
		case "/form":
			formTemplate.Execute(w, struct{ SiteUrl string }{siteUrl})
			return
		}
		rawId := r.URL.Path[1:]
		if !validId.MatchString(rawId) {
			w.WriteHeader(http.StatusBadRequest)
			fmt.Fprintf(w, "%s\n", invalidId)
			return
		}
		id := Id(strings.ToLower(rawId))
		data.RLock()
		pasteInfo, e := data.m[id]
		if !e {
			w.WriteHeader(http.StatusNotFound)
			fmt.Fprintf(w, "%s\n", pasteNotFound)
			return
		}
		if inm := r.Header.Get("If-None-Match"); inm != "" {
			if pasteInfo.Etag == inm || inm == "*" {
				w.WriteHeader(http.StatusNotModified)
				return
			}
		}
		pastePath := id.Path()
		pasteFile, err := os.Open(pastePath)
		if err != nil {
			w.WriteHeader(http.StatusInternalServerError)
			fmt.Fprintf(w, "%s\n", unknownError)
			return
		}
		defer pasteFile.Close()
		w.Header().Set("Etag", pasteInfo.Etag)
		w.Header().Set("Content-Type", pasteInfo.ContentType)
		http.ServeContent(w, r, "", pasteInfo.ModTime, pasteFile)
		data.RUnlock()

	case "POST":
		r.Body = http.MaxBytesReader(w, r.Body, int64(maxSize))
		var id Id
		var content string
		id, err = RandomId()
		if err != nil {
			log.Println(err)
			w.WriteHeader(http.StatusInternalServerError)
			fmt.Fprintf(w, "%s\n", unknownError)
			return
		}
		if err = r.ParseMultipartForm(int64(maxSize)); err != nil {
			log.Printf("Could not parse POST multipart form: %s", err)
			w.WriteHeader(http.StatusBadRequest)
			fmt.Fprintf(w, "%s\n", err)
			return
		}
		if vs, found := r.Form["paste"]; found && len(vs[0]) > 0 {
			content = vs[0]
		} else {
			w.WriteHeader(http.StatusBadRequest)
			fmt.Fprintf(w, "%s\n", missingForm)
			return
		}
		pastePath := id.Path()
		dir, _ := path.Split(pastePath)
		if err = os.MkdirAll(dir, 0700); err != nil {
			log.Printf("Could not create directories leading to %s: %s", pastePath, err)
			w.WriteHeader(http.StatusInternalServerError)
			fmt.Fprintf(w, "%s\n", unknownError)
			return
		}
		deathTime := time.Now().Add(lifeTime)
		id.EndLifeAfter(lifeTime)
		pasteFile, err := os.OpenFile(pastePath, os.O_WRONLY|os.O_CREATE|os.O_EXCL, 0600)
		if err != nil {
			log.Printf("Could not create new paste pasteFile %s: %s", pastePath, err)
			w.WriteHeader(http.StatusInternalServerError)
			fmt.Fprintf(w, "%s\n", unknownError)
			return
		}
		defer pasteFile.Close()
		b, err := io.WriteString(pasteFile, content)
		if err != nil {
			log.Printf("Could not write data into %s: %s", pastePath, err)
			w.WriteHeader(http.StatusInternalServerError)
			fmt.Fprintf(w, "%s\n", unknownError)
			return
		}
		writtenSize := ByteSize(b)
		pasteInfo := PasteInfo{
			ModTime:   time.Now(),
			DeathTime: deathTime,
			Size:      writtenSize,
		}
		pasteInfo.Etag = fmt.Sprintf("%d-%s", pasteInfo.ModTime.Unix(), id)
		pasteInfo.ContentType = http.DetectContentType([]byte(content))
		if pasteInfo.ContentType == "application/octet-stream" {
			pasteInfo.ContentType = "text-plain; charset=utf-8"
		}
		data.Lock()
		data.m[id] = pasteInfo
		data.Unlock()
		log.Printf("Created new paste %s (%s %s) to die at %s",
			id, pasteInfo.ContentType, pasteInfo.Size, pasteInfo.DeathTime)
		fmt.Fprintf(w, "%s/%s\n", siteUrl, id)
	}
}

func walkFunc(filePath string, fileInfo os.FileInfo, err error) error {
	if err != nil {
		return err
	}
	if fileInfo.IsDir() {
		return nil
	}
	id, err := IdFromPath(filePath)
	if err != nil {
		return err
	}
	modTime := fileInfo.ModTime()
	deathTime := modTime.Add(lifeTime)
	now := time.Now()
	if deathTime.Before(now) {
		go id.EndLife()
		return nil
	}
	var lifeLeft time.Duration
	if deathTime.After(now.Add(lifeTime)) {
		lifeLeft = lifeTime
	} else {
		lifeLeft = deathTime.Sub(now)
	}
	size := ByteSize(fileInfo.Size())
	pasteFile, err := os.Open(filePath)
	if err != nil {
		return err
	}
	defer pasteFile.Close()
	read := make([]byte, 512)
	_, err = pasteFile.Read(read)
	if err != nil {
		return err
	}
	pasteInfo := PasteInfo{
		ModTime:   modTime,
		DeathTime: deathTime,
		Size:      size,
		Etag:      fmt.Sprintf("%d-%s", modTime.Unix(), id),
	}
	pasteInfo.ContentType = http.DetectContentType(read)
	if pasteInfo.ContentType == "application/octet-stream" {
		pasteInfo.ContentType = "text-plain; charset=utf-8"
	}
	data.m[id] = pasteInfo
	log.Printf("Recovered paste %s (%s %s) from %s has %s left",
		id, pasteInfo.ContentType, pasteInfo.Size, pasteInfo.ModTime, lifeLeft)
	id.EndLifeAfter(lifeLeft)
	return nil
}

func main() {
	var err error
	flag.Parse()
	if idSize < 6 || idSize > 256 {
		log.Fatalf("Provided id size %d is not between 6 and 256", idSize)
	}
	if maxSize, err = parseByteSize(maxSizeStr); err != nil {
		log.Fatalf("Invalid max size '%s': %s", maxSizeStr, err)
	}
	if indexTemplate, err = template.ParseFiles(indexTmpl); err != nil {
		log.Fatalf("Could not load template %s: %s", indexTmpl, err)
	}
	if formTemplate, err = template.ParseFiles(formTmpl); err != nil {
		log.Fatalf("Could not load template %s: %s", formTmpl, err)
	}
	if err = os.MkdirAll(dataDir, 0700); err != nil {
		log.Fatalf("Could not create data directory %s: %s", dataDir, err)
	}
	if err = os.Chdir(dataDir); err != nil {
		log.Fatalf("Could not enter data directory %s: %s", dataDir, err)
	}
	log.Printf("idSize   = %d", idSize)
	log.Printf("maxSize  = %s", maxSize)
	log.Printf("siteUrl  = %s", siteUrl)
	log.Printf("listen   = %s", listen)
	log.Printf("dataDir  = %s", dataDir)
	log.Printf("lifeTime = %s", lifeTime)
	if err = filepath.Walk(".", walkFunc); err != nil {
		log.Fatalf("Could not recover data directory %s: %s", dataDir, err)
	}
	http.HandleFunc("/", handler)
	log.Printf("Up and running!")
	log.Fatal(http.ListenAndServe(listen, nil))
}
<commit_msg>Lock db as soon as we decide on an unused id<commit_after>/* Copyright (c) 2014, Daniel Martí <mvdan@mvdan.cc> */
/* See LICENSE for licensing information */

package main

import (
	"errors"
	"flag"
	"fmt"
	"html/template"
	"io"
	"log"
	"math/rand"
	"net/http"
	"os"
	"path"
	"path/filepath"
	"regexp"
	"strconv"
	"strings"
	"sync"
	"time"
)

const (
	indexTmpl = "index.html"
	formTmpl  = "form.html"
	chars     = "abcdefghijklmnopqrstuvwxyz0123456789"
	randTries = 10

	// GET error messages
	invalidId     = "Invalid paste id."
	pasteNotFound = "Paste doesn't exist."
	unknownError  = "Something went terribly wrong."
	// POST error messages
	missingForm = "Paste could not be found inside the posted form."
)

var (
	siteUrl, listen, dataDir string
	lifeTime                 time.Duration
	maxSizeStr               string
	idSize                   int
	maxSize                  ByteSize

	validId       *regexp.Regexp
	regexByteSize = regexp.MustCompile(`^([\d\.]+)\s*([KM]?B|[BKM])$`)
	indexTemplate *template.Template
	formTemplate  *template.Template
	data          = struct {
		sync.RWMutex
		m map[Id]PasteInfo
	}{m: make(map[Id]PasteInfo)}
	customRand *rand.Rand
)

func init() {
	flag.StringVar(&siteUrl, "u", "http://localhost:8080", "URL of the site")
	flag.StringVar(&listen, "l", "localhost:8080", "Host and port to listen to")
	flag.StringVar(&dataDir, "d", "data", "Directory to store all the pastes in")
	flag.DurationVar(&lifeTime, "t", 12*time.Hour, "Lifetime of the pastes (units: s,m,h)")
	flag.StringVar(&maxSizeStr, "s", "1M", "Maximum size of POSTs in bytes (units: B,K,M)")
	flag.IntVar(&idSize, "i", 8, "Size of the paste ids (between 6 and 256)")
	validId = regexp.MustCompile("^[a-zA-Z0-9]{" + strconv.Itoa(idSize) + "}$")
	customRand = rand.New(rand.NewSource(time.Now().UnixNano()))
}

type PasteInfo struct {
	ModTime     time.Time
	DeathTime   time.Time
	Size        ByteSize
	Etag        string
	ContentType string
}

type Id string

func IdFromPath(idPath string) (Id, error) {
	parts := strings.Split(idPath, string(filepath.Separator))
	if len(parts) != 3 {
		return "", errors.New("Found invalid number of directories at " + idPath)
	}
	rawId := parts[0] + parts[1] + parts[2]
	if !validId.MatchString(rawId) {
		return "", errors.New("Found invalid id " + rawId)
	}
	return Id(rawId), nil
}

func RandomId() (Id, error) {
	s := make([]byte, idSize)
	var id Id
	data.RLock()
	for try := 0; try < randTries; try++ {
		var offset int = 0
	RandLoop:
		for {
			r := customRand.Int63()
			for i := 0; i < 8; i++ {
				randbyte := int(r&0xff) % len(chars)
				s[offset] = chars[randbyte]
				offset++
				if offset == idSize {
					break RandLoop
				}
				r >>= 8
			}
		}
		id = Id(s)
		if _, e := data.m[id]; !e {
			data.RUnlock()
			return id, nil
		}
	}
	return id, fmt.Errorf("Gave up trying to find an unused random id after %d tries", randTries)
}

func (id Id) String() string {
	return string(id)
}

func (id Id) Path() string {
	return path.Join(string(id[0:2]), string(id[2:4]), string(id[4:]))
}

func (id Id) EndLife() {
	data.Lock()
	err := os.Remove(id.Path())
	if err == nil {
		delete(data.m, id)
		data.Unlock()
		log.Printf("Removed paste: %s", id)
	} else {
		data.Unlock()
		log.Printf("Could not end the life of %s: %s", id, err)
		id.EndLifeAfter(2 * time.Minute)
	}
}

func (id Id) EndLifeAfter(duration time.Duration) {
	timer := time.NewTimer(duration)
	go func() {
		<-timer.C
		id.EndLife()
	}()
}

type ByteSize int64

const (
	B ByteSize = 1 << (10 * iota)
	KB
	MB
)

func parseByteSize(str string) (ByteSize, error) {
	if !regexByteSize.MatchString(str) {
		return 0, errors.New("Could not parse size in bytes")
	}
	parts := regexByteSize.FindStringSubmatch(str)
	size, _ := strconv.ParseFloat(string(parts[1]), 64)

	switch string(parts[2]) {
	case "KB", "K":
		size *= float64(KB)
	case "MB", "M":
		size *= float64(MB)
	}
	return ByteSize(size), nil
}

func (b ByteSize) String() string {
	switch {
	case b >= MB:
		return fmt.Sprintf("%.2f MB", float64(b)/float64(MB))
	case b >= KB:
		return fmt.Sprintf("%.2f KB", float64(b)/float64(KB))
	}
	return fmt.Sprintf("%d B", b)
}

func handler(w http.ResponseWriter, r *http.Request) {
	var err error
	switch r.Method {
	case "GET":
		switch r.URL.Path {
		case "/":
			indexTemplate.Execute(w, struct {
				SiteUrl, LifeTime string
			}{
				siteUrl,
				fmt.Sprintf("%g hours and %g minutes", lifeTime.Hours(),
					lifeTime.Minutes()-lifeTime.Hours()*60),
			})
			return
		case "/form":
			formTemplate.Execute(w, struct{ SiteUrl string }{siteUrl})
			return
		}
		rawId := r.URL.Path[1:]
		if !validId.MatchString(rawId) {
			w.WriteHeader(http.StatusBadRequest)
			fmt.Fprintf(w, "%s\n", invalidId)
			return
		}
		id := Id(strings.ToLower(rawId))
		data.RLock()
		pasteInfo, e := data.m[id]
		if !e {
			w.WriteHeader(http.StatusNotFound)
			fmt.Fprintf(w, "%s\n", pasteNotFound)
			return
		}
		if inm := r.Header.Get("If-None-Match"); inm != "" {
			if pasteInfo.Etag == inm || inm == "*" {
				w.WriteHeader(http.StatusNotModified)
				return
			}
		}
		pastePath := id.Path()
		pasteFile, err := os.Open(pastePath)
		if err != nil {
			w.WriteHeader(http.StatusInternalServerError)
			fmt.Fprintf(w, "%s\n", unknownError)
			return
		}
		defer pasteFile.Close()
		w.Header().Set("Etag", pasteInfo.Etag)
		w.Header().Set("Content-Type", pasteInfo.ContentType)
		http.ServeContent(w, r, "", pasteInfo.ModTime, pasteFile)
		data.RUnlock()

	case "POST":
		r.Body = http.MaxBytesReader(w, r.Body, int64(maxSize))
		var content string
		if err = r.ParseMultipartForm(int64(maxSize)); err != nil {
			log.Printf("Could not parse POST multipart form: %s", err)
			w.WriteHeader(http.StatusBadRequest)
			fmt.Fprintf(w, "%s\n", err)
			return
		}
		if vs, found := r.Form["paste"]; found && len(vs[0]) > 0 {
			content = vs[0]
		} else {
			w.WriteHeader(http.StatusBadRequest)
			fmt.Fprintf(w, "%s\n", missingForm)
			return
		}
		var id Id
		if id, err = RandomId(); err == nil {
			data.Lock()
		} else {
			log.Println(err)
			w.WriteHeader(http.StatusInternalServerError)
			fmt.Fprintf(w, "%s\n", unknownError)
			return
		}
		pastePath := id.Path()
		dir, _ := path.Split(pastePath)
		if err = os.MkdirAll(dir, 0700); err != nil {
			log.Printf("Could not create directories leading to %s: %s", pastePath, err)
			w.WriteHeader(http.StatusInternalServerError)
			fmt.Fprintf(w, "%s\n", unknownError)
			data.Unlock()
			return
		}
		deathTime := time.Now().Add(lifeTime)
		id.EndLifeAfter(lifeTime)
		pasteFile, err := os.OpenFile(pastePath, os.O_WRONLY|os.O_CREATE|os.O_EXCL, 0600)
		if err != nil {
			log.Printf("Could not create new paste file %s: %s", pastePath, err)
			w.WriteHeader(http.StatusInternalServerError)
			fmt.Fprintf(w, "%s\n", unknownError)
			data.Unlock()
			return
		}
		defer pasteFile.Close()
		b, err := io.WriteString(pasteFile, content)
		if err != nil {
			log.Printf("Could not write data into %s: %s", pastePath, err)
			w.WriteHeader(http.StatusInternalServerError)
			fmt.Fprintf(w, "%s\n", unknownError)
			data.Unlock()
			return
		}
		writtenSize := ByteSize(b)
		pasteInfo := PasteInfo{
			ModTime:   time.Now(),
			DeathTime: deathTime,
			Size:      writtenSize,
		}
		pasteInfo.Etag = fmt.Sprintf("%d-%s", pasteInfo.ModTime.Unix(), id)
		pasteInfo.ContentType = http.DetectContentType([]byte(content))
		if pasteInfo.ContentType == "application/octet-stream" {
			pasteInfo.ContentType = "text-plain; charset=utf-8"
		}
		data.m[id] = pasteInfo
		data.Unlock()
		log.Printf("Created new paste %s (%s %s) to die at %s",
			id, pasteInfo.ContentType, pasteInfo.Size, pasteInfo.DeathTime)
		fmt.Fprintf(w, "%s/%s\n", siteUrl, id)
	}
}

func walkFunc(filePath string, fileInfo os.FileInfo, err error) error {
	if err != nil {
		return err
	}
	if fileInfo.IsDir() {
		return nil
	}
	id, err := IdFromPath(filePath)
	if err != nil {
		return err
	}
	modTime := fileInfo.ModTime()
	deathTime := modTime.Add(lifeTime)
	now := time.Now()
	if deathTime.Before(now) {
		go id.EndLife()
		return nil
	}
	var lifeLeft time.Duration
	if deathTime.After(now.Add(lifeTime)) {
		lifeLeft = lifeTime
	} else {
		lifeLeft = deathTime.Sub(now)
	}
	size := ByteSize(fileInfo.Size())
	pasteFile, err := os.Open(filePath)
	if err != nil {
		return err
	}
	defer pasteFile.Close()
	read := make([]byte, 512)
	_, err = pasteFile.Read(read)
	if err != nil {
		return err
	}
	pasteInfo := PasteInfo{
		ModTime:   modTime,
		DeathTime: deathTime,
		Size:      size,
		Etag:      fmt.Sprintf("%d-%s", modTime.Unix(), id),
	}
	pasteInfo.ContentType = http.DetectContentType(read)
	if pasteInfo.ContentType == "application/octet-stream" {
		pasteInfo.ContentType = "text-plain; charset=utf-8"
	}
	data.m[id] = pasteInfo
	log.Printf("Recovered paste %s (%s %s) from %s has %s left",
		id, pasteInfo.ContentType, pasteInfo.Size, pasteInfo.ModTime, lifeLeft)
	id.EndLifeAfter(lifeLeft)
	return nil
}

func main() {
	var err error
	flag.Parse()
	if idSize < 6 || idSize > 256 {
		log.Fatalf("Provided id size %d is not between 6 and 256", idSize)
	}
	if maxSize, err = parseByteSize(maxSizeStr); err != nil {
		log.Fatalf("Invalid max size '%s': %s", maxSizeStr, err)
	}
	if indexTemplate, err = template.ParseFiles(indexTmpl); err != nil {
		log.Fatalf("Could not load template %s: %s", indexTmpl, err)
	}
	if formTemplate, err = template.ParseFiles(formTmpl); err != nil {
		log.Fatalf("Could not load template %s: %s", formTmpl, err)
	}
	if err = os.MkdirAll(dataDir, 0700); err != nil {
		log.Fatalf("Could not create data directory %s: %s", dataDir, err)
	}
	if err = os.Chdir(dataDir); err != nil {
		log.Fatalf("Could not enter data directory %s: %s", dataDir, err)
	}
	log.Printf("idSize   = %d", idSize)
	log.Printf("maxSize  = %s", maxSize)
	log.Printf("siteUrl  = %s", siteUrl)
	log.Printf("listen   = %s", listen)
	log.Printf("dataDir  = %s", dataDir)
	log.Printf("lifeTime = %s", lifeTime)
	if err = filepath.Walk(".", walkFunc); err != nil {
		log.Fatalf("Could not recover data directory %s: %s", dataDir, err)
	}
	http.HandleFunc("/", handler)
	log.Printf("Up and running!")
	log.Fatal(http.ListenAndServe(listen, nil))
}
<|endoftext|>


<commit_before>package registry

import (
	"errors"
	"path"
	"time"

	"code.google.com/p/go-uuid/uuid"

	"github.com/spesnova/iruka/schema"
)

const (
	containerPrefix = "containers"
)

type Containers []schema.Container

// CreateContainer creates etcd key-value to store container options
func (r *Registry) CreateContainer(appIdOrName string, opts schema.ContainerCreateOpts) (schema.Container, error) {
	// Validate opts
	if opts.Image == "" {
		return schema.Container{}, errors.New("image parameter is required, but missing")
	}
	if opts.Size == "" {
		return schema.Container{}, errors.New("size parameter is required, but missing")
	}
	if opts.Command == "" {
		return schema.Container{}, errors.New("command parameter is required, but missing")
	}
	if opts.Type == "" {
		return schema.Container{}, errors.New("type parameter is required, but missing")
	}

	id := uuid.NewUUID()
	app, err := r.App(appIdOrName)
	if err != nil {
		return schema.Container{}, err
	}

	// TODO (spesnova): Add a varidation container size
	// Size takes supported value only like "1X", "2X", "3X" ...
	// But I haven't define the values yet.

	// TODO (spesnova): Add a varidation for container type

	container := schema.Container{
		ID:          id,
		AppID:       app.ID,
		Name:        app.Name + "." + opts.Type, // TODO (spesnova): add number like `examle.run.1`
		Image:       opts.Image,
		Size:        opts.Size,
		Command:     opts.Command,
		Type:        opts.Type,
		Ports:       opts.Ports,
		DesireState: "up",
		State:       "",
		CreatedAt:   time.Now(),
		UpdatedAt:   time.Now(),
	}

	j, err := marshal(container)
	if err != nil {
		return schema.Container{}, err
	}

	key := path.Join(r.keyPrefix, containerPrefix, container.ID.String())

	_, err = r.etcd.Create(key, string(j), 0)
	if err != nil {
		return schema.Container{}, err
	}

	return container, nil
}

// Container returns a container
func (r *Registry) Container(idOrName string) (schema.Container, error) {
	var container schema.Container

	cs, err := r.Containers()
	if err != nil {
		return schema.Container{}, err
	}

	if uuid.Parse(idOrName) == nil {
		for _, c := range cs {
			if c.Name == idOrName {
				return c, nil
			}
		}
	} else {
		for _, c := range cs {
			if uuid.Equal(c.ID, uuid.Parse(idOrName)) {
				return c, nil
			}
		}
	}

	return container, errors.New("No such container: " + idOrName)
}

// Container returns a container of an app
func (r *Registry) ContainerFilteredByApp(appIdOrName, idOrName string) (schema.Container, error) {
	container, err := r.Container(idOrName)
	if err != nil {
		return schema.Container{}, err
	}

	app, err := r.App(appIdOrName)
	if err != nil {
		return schema.Container{}, err
	}

	if uuid.Equal(container.AppID, app.ID) {
		return container, nil
	}

	return container, errors.New("No such container: " + idOrName)
}

// Containers returns container collection
func (r *Registry) Containers() ([]schema.Container, error) {
	key := path.Join(r.keyPrefix, containerPrefix)
	res, err := r.etcd.Get(key, false, true)
	if err != nil {
		return nil, err
	}

	containers := make([]schema.Container, len(res.Node.Nodes))
	for i, node := range res.Node.Nodes {
		var container schema.Container
		err = unmarshal(node.Value, &container)
		if err != nil {
			return nil, err
		}

		containers[i] = container
	}

	return containers, nil
}

// ContainersFilteredByApp returns containers of an app
func (r *Registry) ContainersFilteredByApp(appIdOrName string) ([]schema.Container, error) {
	var containers []schema.Container

	app, err := r.App(appIdOrName)
	if err != nil {
		return nil, err
	}

	cs, err := r.Containers()
	if err != nil {
		return nil, err
	}

	for _, c := range cs {
		if uuid.Equal(c.AppID, app.ID) {
			containers = append(containers, c)
		}
	}

	return containers, nil
}
<commit_msg>Identifying by container name is not supported yet<commit_after>package registry

import (
	"errors"
	"path"
	"time"

	"code.google.com/p/go-uuid/uuid"

	"github.com/spesnova/iruka/schema"
)

const (
	containerPrefix = "containers"
)

type Containers []schema.Container

// CreateContainer creates etcd key-value to store container options
func (r *Registry) CreateContainer(appIdOrName string, opts schema.ContainerCreateOpts) (schema.Container, error) {
	app, err := r.App(appIdOrName)
	if err != nil {
		return schema.Container{}, err
	}

	// Validate opts
	if opts.Image == "" {
		return schema.Container{}, errors.New("image parameter is required, but missing")
	}
	if opts.Size == "" {
		return schema.Container{}, errors.New("size parameter is required, but missing")
	}
	if opts.Command == "" {
		return schema.Container{}, errors.New("command parameter is required, but missing")
	}
	if opts.Type == "" {
		return schema.Container{}, errors.New("type parameter is required, but missing")
	}

	id := uuid.NewUUID()

	// TODO (spesnova): Add a varidation container size
	// Size takes supported value only like "1X", "2X", "3X" ...
	// But I haven't define the values yet.

	// TODO (spesnova): Add a varidation for container type

	container := schema.Container{
		ID:          id,
		AppID:       app.ID,
		Name:        app.Name + "." + opts.Type, // TODO (spesnova): add number like `examle.run.1`
		Image:       opts.Image,
		Size:        opts.Size,
		Command:     opts.Command,
		Type:        opts.Type,
		Ports:       opts.Ports,
		DesireState: "up",
		State:       "",
		CreatedAt:   time.Now(),
		UpdatedAt:   time.Now(),
	}

	j, err := marshal(container)
	if err != nil {
		return schema.Container{}, err
	}

	key := path.Join(r.keyPrefix, containerPrefix, container.ID.String())

	_, err = r.etcd.Create(key, string(j), 0)
	if err != nil {
		return schema.Container{}, err
	}

	return container, nil
}

// Container returns a container
func (r *Registry) Container(idOrName string) (schema.Container, error) {
	var container schema.Container

	cs, err := r.Containers()
	if err != nil {
		return schema.Container{}, err
	}

	if uuid.Parse(idOrName) == nil {
		for _, c := range cs {
			if c.Name == idOrName {
				// TODO (spesnova): support this
				return schema.Container{}, errors.New("identifying a container by a container name is not supported yet")
			}
		}
	} else {
		for _, c := range cs {
			if uuid.Equal(c.ID, uuid.Parse(idOrName)) {
				return c, nil
			}
		}
	}

	return container, errors.New("No such container: " + idOrName)
}

// Container returns a container of an app
func (r *Registry) ContainerFilteredByApp(appIdOrName, idOrName string) (schema.Container, error) {
	container, err := r.Container(idOrName)
	if err != nil {
		return schema.Container{}, err
	}

	app, err := r.App(appIdOrName)
	if err != nil {
		return schema.Container{}, err
	}

	if uuid.Equal(container.AppID, app.ID) {
		return container, nil
	}

	return container, errors.New("No such container: " + idOrName)
}

// Containers returns container collection
func (r *Registry) Containers() ([]schema.Container, error) {
	key := path.Join(r.keyPrefix, containerPrefix)
	res, err := r.etcd.Get(key, false, true)
	if err != nil {
		return nil, err
	}

	containers := make([]schema.Container, len(res.Node.Nodes))
	for i, node := range res.Node.Nodes {
		var container schema.Container
		err = unmarshal(node.Value, &container)
		if err != nil {
			return nil, err
		}

		containers[i] = container
	}

	return containers, nil
}

// ContainersFilteredByApp returns containers of an app
func (r *Registry) ContainersFilteredByApp(appIdOrName string) ([]schema.Container, error) {
	var containers []schema.Container

	app, err := r.App(appIdOrName)
	if err != nil {
		return nil, err
	}

	cs, err := r.Containers()
	if err != nil {
		return nil, err
	}

	for _, c := range cs {
		if uuid.Equal(c.AppID, app.ID) {
			containers = append(containers, c)
		}
	}

	return containers, nil
}
<|endoftext|>


<commit_before>package jspb

import "github.com/gopherjs/gopherjs/js"

// Reader encapsulates the jspb.BinaryReader
type Reader interface {
	Next() bool
	Err() error

	GetFieldNumber() int
	SkipField()

	// Scalars
	ReadInt32() int32
	ReadInt64() int64
	ReadUint32() uint32
	ReadUint64() uint64
	ReadSint32() int32
	ReadSint64() int64
	ReadFixed32() uint32
	ReadFixed64() uint64
	ReadSfixed32() int32
	ReadSfixed64() int64
	ReadFloat32() float32
	ReadFloat64() float64
	ReadEnum() int
	ReadBool() bool
	ReadString() string
	ReadBytes() []byte

	// Scalar Slices
	ReadInt32Slice() []int32
	ReadInt64Slice() []int64
	ReadUint32Slice() []uint32
	ReadUint64Slice() []uint64
	ReadSint32Slice() []int32
	ReadSint64Slice() []int64
	ReadFixed32Slice() []uint32
	ReadFixed64Slice() []uint64
	ReadSfixed32Slice() []int32
	ReadSfixed64Slice() []int64
	ReadFloat32Slice() []float32
	ReadFloat64Slice() []float64
	ReadEnumSlice() []int
	ReadBoolSlice() []bool

	// Specials
	ReadMessage(func())
}

// NewReader returns a new Reader ready for writing
func NewReader(data []byte) Reader {
	return &reader{
		Object: js.Global.Get("BinaryReader").New(data),
	}
}

// reader implements the Reader interface
type reader struct {
	*js.Object
	err error
}

// Reads the next field header in the stream if there is one, returns true if
// we saw a valid field header or false if we've read the whole stream.
// Sets err if we encountered a deprecated START_GROUP/END_GROUP field.
func (r *reader) Next() bool {
	defer catchException(&r.err)
	return r.err == nil && r.Call("nextField").Bool() && !r.Call("isEndGroup").Bool()
}

// Err returns the error state of the Reader. It must
// be called after Next() has returned false.
func (r reader) Err() error {
	return r.err
}

// The field number of the next field in the buffer, or
// -1 if there is no next field.
func (r reader) GetFieldNumber() int {
	return r.Call("getFieldNumber").Int()
}

// Skips over the next field in the binary stream.
func (r reader) SkipField() {
	r.Call("skipField")
}

// ReadInt32 reads a signed 32-bit integer field from the binary
// stream, or sets err if the next field in the
// stream is not of the correct wire type.
func (r *reader) ReadInt32() int32 {
	defer catchException(&r.err)
	return int32(r.Call("readInt32").Int())
}

// ReadInt64 reads a signed 64-bit integer field from the binary
// stream, or sets err if the next field in the
// stream is not of the correct wire type.
func (r *reader) ReadInt64() int64 {
	defer catchException(&r.err)
	return int64(r.Call("readInt64").Int())
}

// ReadUint32 reads an unsigned 32-bit integer field from the binary
// stream, or sets err if the next field in the
// stream is not of the correct wire type.
func (r *reader) ReadUint32() uint32 {
	defer catchException(&r.err)
	return uint32(r.Call("readUint32").Int())
}

// ReadUint64 reads an unsigned 64-bit integer field from the binary
// stream, or sets err if the next field in the
// stream is not of the correct wire type.
func (r *reader) ReadUint64() uint64 {
	defer catchException(&r.err)
	return uint64(r.Call("readUint64").Int())
}

// ReadSint32 reads a signed 32-bit integer field from the binary
// stream, or sets err if the next field in the
// stream is not of the correct wire type.
func (r *reader) ReadSint32() int32 {
	defer catchException(&r.err)
	return int32(r.Call("readSint32").Int())
}

// ReadSint64 reads a signed 64-bit integer field from the binary
// stream, or sets err if the next field in the
// stream is not of the correct wire type.
func (r *reader) ReadSint64() int64 {
	defer catchException(&r.err)
	return int64(r.Call("readSint64").Int())
}

// ReadFixed32 reads an unsigned 32-bit integer field from the binary
// stream, or sets err if the next field in the
// stream is not of the correct wire type.
func (r *reader) ReadFixed32() uint32 {
	defer catchException(&r.err)
	return uint32(r.Call("readFixed32").Int())
}

// ReadFixed64 reads an unsigned 64-bit integer field from the binary
// stream, or sets err if the next field in the
// stream is not of the correct wire type.
func (r *reader) ReadFixed64() uint64 {
	defer catchException(&r.err)
	return uint64(r.Call("readFixed64").Int())
}

// ReadSfixed32 reads a signed 32-bit integer field from the binary
// stream, or sets err if the next field in the
// stream is not of the correct wire type.
func (r *reader) ReadSfixed32() int32 {
	defer catchException(&r.err)
	return int32(r.Call("readSfixed32").Int())
}

// ReadSfixed64 reads a signed 64-bit integer field from the binary
// stream, or sets err if the next field in the
// stream is not of the correct wire type.
func (r *reader) ReadSfixed64() int64 {
	defer catchException(&r.err)
	return int64(r.Call("readSfixed64").Int())
}

// ReadFloat32 reads a 32-bit floating point field from the binary
// stream, or sets err if the next field in the
// stream is not of the correct wire type.
func (r *reader) ReadFloat32() float32 {
	defer catchException(&r.err)
	return float32(r.Call("readFloat").Float())
}

// ReadFloat64 reads a 64-bit floating point field from the binary
// stream, or sets err if the next field in the
// stream is not of the correct wire type.
func (r *reader) ReadFloat64() float64 {
	defer catchException(&r.err)
	return r.Call("readDouble").Float()
}

// ReadEnum reads an enum field from the binary stream,
// or sets err if the next field in the stream
// is not of the correct wire type.
func (r *reader) ReadEnum() int {
	defer catchException(&r.err)
	return r.Call("readEnum").Int()
}

// ReadBool reads a bool field from the binary stream, or sets err
// if the next field in the stream is not of the correct wire type.
func (r *reader) ReadBool() bool {
	defer catchException(&r.err)
	return r.Call("readBool").Bool()
}

// ReadString reads a string field from the binary stream, or sets err
// if the next field in the stream is not of the correct wire type.
func (r *reader) ReadString() string {
	defer catchException(&r.err)
	return r.Call("readString").String()
}

// ReadBytes reads a bytes field from the binary stream, or sets err
// if the next field in the stream is not of the correct wire type.
func (r *reader) ReadBytes() []byte {
	defer catchException(&r.err)
	return r.Call("readBytes").Interface().([]byte)
}

// ReadMessage deserializes a proto using
// the provided reader function.
func (r *reader) ReadMessage(readFunc func()) {
	defer catchException(&r.err)
	r.Call("readMessage", js.Undefined /* Unused */, readFunc)
}

// ReadInt32Slice reads a repeated signed 32-bit integer field from the binary
// stream, or sets err if the next field in the
// stream is not of the correct wire type.
func (r *reader) ReadInt32Slice() (ret []int32) {
	defer catchException(&r.err)
	values := r.Call("readPackedInt32").Interface().([]interface{})
	for _, value := range values {
		ret = append(ret, int32(value.(float64)))
	}

	return ret
}

// ReadInt64Slice reads a repeated signed 64-bit integer field from the binary
// stream, or sets err if the next field in the
// stream is not of the correct wire type.
func (r *reader) ReadInt64Slice() (ret []int64) {
	defer catchException(&r.err)
	values := r.Call("readPackedInt64").Interface().([]interface{})
	for _, value := range values {
		ret = append(ret, int64(value.(float64)))
	}

	return ret
}

// ReadUint32Slice reads a repeated unsigned 32-bit integer field from the binary
// stream, or sets err if the next field in the
// stream is not of the correct wire type.
func (r *reader) ReadUint32Slice() (ret []uint32) {
	defer catchException(&r.err)
	values := r.Call("readPackedUint32").Interface().([]interface{})
	for _, value := range values {
		ret = append(ret, uint32(value.(float64)))
	}

	return ret
}

// ReadUint64Slice reads a repeated unsigned 64-bit integer field from the binary
// stream, or sets err if the next field in the
// stream is not of the correct wire type.
func (r *reader) ReadUint64Slice() (ret []uint64) {
	defer catchException(&r.err)
	values := r.Call("readPackedUint64").Interface().([]interface{})
	for _, value := range values {
		ret = append(ret, uint64(value.(float64)))
	}

	return ret
}

// ReadSint32Slice reads a repeated signed 32-bit integer field from the binary
// stream, or sets err if the next field in the
// stream is not of the correct wire type.
func (r *reader) ReadSint32Slice() (ret []int32) {
	defer catchException(&r.err)
	values := r.Call("readPackedSint32").Interface().([]interface{})
	for _, value := range values {
		ret = append(ret, int32(value.(float64)))
	}

	return ret
}

// ReadSint64Slice reads a repeated signed 64-bit integer field from the binary
// stream, or sets err if the next field in the
// stream is not of the correct wire type.
func (r *reader) ReadSint64Slice() (ret []int64) {
	defer catchException(&r.err)
	values := r.Call("readPackedSint64").Interface().([]interface{})
	for _, value := range values {
		ret = append(ret, int64(value.(float64)))
	}

	return ret
}

// ReadFixed32Slice reads a repeated unsigned 32-bit integer field from the binary
// stream, or sets err if the next field in the
// stream is not of the correct wire type.
func (r *reader) ReadFixed32Slice() (ret []uint32) {
	defer catchException(&r.err)
	values := r.Call("readPackedFixed32").Interface().([]interface{})
	for _, value := range values {
		ret = append(ret, uint32(value.(float64)))
	}

	return ret
}

// ReadFixed64Slice reads a repeated unsigned 64-bit integer field from the binary
// stream, or sets err if the next field in the
// stream is not of the correct wire type.
func (r *reader) ReadFixed64Slice() (ret []uint64) {
	defer catchException(&r.err)
	values := r.Call("readPackedFixed64").Interface().([]interface{})
	for _, value := range values {
		ret = append(ret, uint64(value.(float64)))
	}

	return ret
}

// ReadSfixed32Slice reads a repeated signed 32-bit integer field from the binary
// stream, or sets err if the next field in the
// stream is not of the correct wire type.
func (r *reader) ReadSfixed32Slice() (ret []int32) {
	defer catchException(&r.err)
	values := r.Call("readPackedSfixed32").Interface().([]interface{})
	for _, value := range values {
		ret = append(ret, int32(value.(float64)))
	}

	return ret
}

// ReadSfixed64Slice reads a repeated signed 64-bit integer field from the binary
// stream, or sets err if the next field in the
// stream is not of the correct wire type.
func (r *reader) ReadSfixed64Slice() (ret []int64) {
	defer catchException(&r.err)
	values := r.Call("readPackedSfixed64").Interface().([]interface{})
	for _, value := range values {
		ret = append(ret, int64(value.(float64)))
	}

	return ret
}

// ReadFloat32Slice reads a repeated 32-bit floating point field from the binary
// stream, or sets err if the next field in the
// stream is not of the correct wire type.
func (r *reader) ReadFloat32Slice() (ret []float32) {
	defer catchException(&r.err)
	values := r.Call("readPackedFloat").Interface().([]interface{})
	for _, value := range values {
		ret = append(ret, float32(value.(float64)))
	}

	return ret
}

// ReadFloat64Slice reads a repeated 64-bit floating point field from the binary
// stream, or sets err if the next field in the
// stream is not of the correct wire type.
func (r *reader) ReadFloat64Slice() (ret []float64) {
	defer catchException(&r.err)
	values := r.Call("readPackedDouble").Interface().([]interface{})
	for _, value := range values {
		ret = append(ret, value.(float64))
	}

	return ret
}

// ReadEnumSlice reads a repeated enum field from the binary stream,
// or sets err if the next field in the stream
// is not of the correct wire type.
func (r *reader) ReadEnumSlice() (ret []int) {
	defer catchException(&r.err)
	values := r.Call("readPackedEnum").Interface().([]interface{})
	for _, value := range values {
		ret = append(ret, int(value.(float64)))
	}

	return ret
}

// ReadBoolSlice reads a repeated bool field from the binary stream, or sets err
// if the next field in the stream is not of the correct wire type.
func (r *reader) ReadBoolSlice() (ret []bool) {
	defer catchException(&r.err)
	values := r.Call("readPackedBool").Interface().([]interface{})
	for _, value := range values {
		ret = append(ret, value.(bool))
	}

	return ret
}
<commit_msg>Omg add a dot to JSPB reader comment<commit_after>package jspb

import "github.com/gopherjs/gopherjs/js"

// Reader encapsulates the jspb.BinaryReader.
type Reader interface {
	Next() bool
	Err() error

	GetFieldNumber() int
	SkipField()

	// Scalars
	ReadInt32() int32
	ReadInt64() int64
	ReadUint32() uint32
	ReadUint64() uint64
	ReadSint32() int32
	ReadSint64() int64
	ReadFixed32() uint32
	ReadFixed64() uint64
	ReadSfixed32() int32
	ReadSfixed64() int64
	ReadFloat32() float32
	ReadFloat64() float64
	ReadEnum() int
	ReadBool() bool
	ReadString() string
	ReadBytes() []byte

	// Scalar Slices
	ReadInt32Slice() []int32
	ReadInt64Slice() []int64
	ReadUint32Slice() []uint32
	ReadUint64Slice() []uint64
	ReadSint32Slice() []int32
	ReadSint64Slice() []int64
	ReadFixed32Slice() []uint32
	ReadFixed64Slice() []uint64
	ReadSfixed32Slice() []int32
	ReadSfixed64Slice() []int64
	ReadFloat32Slice() []float32
	ReadFloat64Slice() []float64
	ReadEnumSlice() []int
	ReadBoolSlice() []bool

	// Specials
	ReadMessage(func())
}

// NewReader returns a new Reader ready for writing
func NewReader(data []byte) Reader {
	return &reader{
		Object: js.Global.Get("BinaryReader").New(data),
	}
}

// reader implements the Reader interface
type reader struct {
	*js.Object
	err error
}

// Reads the next field header in the stream if there is one, returns true if
// we saw a valid field header or false if we've read the whole stream.
// Sets err if we encountered a deprecated START_GROUP/END_GROUP field.
func (r *reader) Next() bool {
	defer catchException(&r.err)
	return r.err == nil && r.Call("nextField").Bool() && !r.Call("isEndGroup").Bool()
}

// Err returns the error state of the Reader. It must
// be called after Next() has returned false.
func (r reader) Err() error {
	return r.err
}

// The field number of the next field in the buffer, or
// -1 if there is no next field.
func (r reader) GetFieldNumber() int {
	return r.Call("getFieldNumber").Int()
}

// Skips over the next field in the binary stream.
func (r reader) SkipField() {
	r.Call("skipField")
}

// ReadInt32 reads a signed 32-bit integer field from the binary
// stream, or sets err if the next field in the
// stream is not of the correct wire type.
func (r *reader) ReadInt32() int32 {
	defer catchException(&r.err)
	return int32(r.Call("readInt32").Int())
}

// ReadInt64 reads a signed 64-bit integer field from the binary
// stream, or sets err if the next field in the
// stream is not of the correct wire type.
func (r *reader) ReadInt64() int64 {
	defer catchException(&r.err)
	return int64(r.Call("readInt64").Int())
}

// ReadUint32 reads an unsigned 32-bit integer field from the binary
// stream, or sets err if the next field in the
// stream is not of the correct wire type.
func (r *reader) ReadUint32() uint32 {
	defer catchException(&r.err)
	return uint32(r.Call("readUint32").Int())
}

// ReadUint64 reads an unsigned 64-bit integer field from the binary
// stream, or sets err if the next field in the
// stream is not of the correct wire type.
func (r *reader) ReadUint64() uint64 {
	defer catchException(&r.err)
	return uint64(r.Call("readUint64").Int())
}

// ReadSint32 reads a signed 32-bit integer field from the binary
// stream, or sets err if the next field in the
// stream is not of the correct wire type.
func (r *reader) ReadSint32() int32 {
	defer catchException(&r.err)
	return int32(r.Call("readSint32").Int())
}

// ReadSint64 reads a signed 64-bit integer field from the binary
// stream, or sets err if the next field in the
// stream is not of the correct wire type.
func (r *reader) ReadSint64() int64 {
	defer catchException(&r.err)
	return int64(r.Call("readSint64").Int())
}

// ReadFixed32 reads an unsigned 32-bit integer field from the binary
// stream, or sets err if the next field in the
// stream is not of the correct wire type.
func (r *reader) ReadFixed32() uint32 {
	defer catchException(&r.err)
	return uint32(r.Call("readFixed32").Int())
}

// ReadFixed64 reads an unsigned 64-bit integer field from the binary
// stream, or sets err if the next field in the
// stream is not of the correct wire type.
func (r *reader) ReadFixed64() uint64 {
	defer catchException(&r.err)
	return uint64(r.Call("readFixed64").Int())
}

// ReadSfixed32 reads a signed 32-bit integer field from the binary
// stream, or sets err if the next field in the
// stream is not of the correct wire type.
func (r *reader) ReadSfixed32() int32 {
	defer catchException(&r.err)
	return int32(r.Call("readSfixed32").Int())
}

// ReadSfixed64 reads a signed 64-bit integer field from the binary
// stream, or sets err if the next field in the
// stream is not of the correct wire type.
func (r *reader) ReadSfixed64() int64 {
	defer catchException(&r.err)
	return int64(r.Call("readSfixed64").Int())
}

// ReadFloat32 reads a 32-bit floating point field from the binary
// stream, or sets err if the next field in the
// stream is not of the correct wire type.
func (r *reader) ReadFloat32() float32 {
	defer catchException(&r.err)
	return float32(r.Call("readFloat").Float())
}

// ReadFloat64 reads a 64-bit floating point field from the binary
// stream, or sets err if the next field in the
// stream is not of the correct wire type.
func (r *reader) ReadFloat64() float64 {
	defer catchException(&r.err)
	return r.Call("readDouble").Float()
}

// ReadEnum reads an enum field from the binary stream,
// or sets err if the next field in the stream
// is not of the correct wire type.
func (r *reader) ReadEnum() int {
	defer catchException(&r.err)
	return r.Call("readEnum").Int()
}

// ReadBool reads a bool field from the binary stream, or sets err
// if the next field in the stream is not of the correct wire type.
func (r *reader) ReadBool() bool {
	defer catchException(&r.err)
	return r.Call("readBool").Bool()
}

// ReadString reads a string field from the binary stream, or sets err
// if the next field in the stream is not of the correct wire type.
func (r *reader) ReadString() string {
	defer catchException(&r.err)
	return r.Call("readString").String()
}

// ReadBytes reads a bytes field from the binary stream, or sets err
// if the next field in the stream is not of the correct wire type.
func (r *reader) ReadBytes() []byte {
	defer catchException(&r.err)
	return r.Call("readBytes").Interface().([]byte)
}

// ReadMessage deserializes a proto using
// the provided reader function.
func (r *reader) ReadMessage(readFunc func()) {
	defer catchException(&r.err)
	r.Call("readMessage", js.Undefined /* Unused */, readFunc)
}

// ReadInt32Slice reads a repeated signed 32-bit integer field from the binary
// stream, or sets err if the next field in the
// stream is not of the correct wire type.
func (r *reader) ReadInt32Slice() (ret []int32) {
	defer catchException(&r.err)
	values := r.Call("readPackedInt32").Interface().([]interface{})
	for _, value := range values {
		ret = append(ret, int32(value.(float64)))
	}

	return ret
}

// ReadInt64Slice reads a repeated signed 64-bit integer field from the binary
// stream, or sets err if the next field in the
// stream is not of the correct wire type.
func (r *reader) ReadInt64Slice() (ret []int64) {
	defer catchException(&r.err)
	values := r.Call("readPackedInt64").Interface().([]interface{})
	for _, value := range values {
		ret = append(ret, int64(value.(float64)))
	}

	return ret
}

// ReadUint32Slice reads a repeated unsigned 32-bit integer field from the binary
// stream, or sets err if the next field in the
// stream is not of the correct wire type.
func (r *reader) ReadUint32Slice() (ret []uint32) {
	defer catchException(&r.err)
	values := r.Call("readPackedUint32").Interface().([]interface{})
	for _, value := range values {
		ret = append(ret, uint32(value.(float64)))
	}

	return ret
}

// ReadUint64Slice reads a repeated unsigned 64-bit integer field from the binary
// stream, or sets err if the next field in the
// stream is not of the correct wire type.
func (r *reader) ReadUint64Slice() (ret []uint64) {
	defer catchException(&r.err)
	values := r.Call("readPackedUint64").Interface().([]interface{})
	for _, value := range values {
		ret = append(ret, uint64(value.(float64)))
	}

	return ret
}

// ReadSint32Slice reads a repeated signed 32-bit integer field from the binary
// stream, or sets err if the next field in the
// stream is not of the correct wire type.
func (r *reader) ReadSint32Slice() (ret []int32) {
	defer catchException(&r.err)
	values := r.Call("readPackedSint32").Interface().([]interface{})
	for _, value := range values {
		ret = append(ret, int32(value.(float64)))
	}

	return ret
}

// ReadSint64Slice reads a repeated signed 64-bit integer field from the binary
// stream, or sets err if the next field in the
// stream is not of the correct wire type.
func (r *reader) ReadSint64Slice() (ret []int64) {
	defer catchException(&r.err)
	values := r.Call("readPackedSint64").Interface().([]interface{})
	for _, value := range values {
		ret = append(ret, int64(value.(float64)))
	}

	return ret
}

// ReadFixed32Slice reads a repeated unsigned 32-bit integer field from the binary
// stream, or sets err if the next field in the
// stream is not of the correct wire type.
func (r *reader) ReadFixed32Slice() (ret []uint32) {
	defer catchException(&r.err)
	values := r.Call("readPackedFixed32").Interface().([]interface{})
	for _, value := range values {
		ret = append(ret, uint32(value.(float64)))
	}

	return ret
}

// ReadFixed64Slice reads a repeated unsigned 64-bit integer field from the binary
// stream, or sets err if the next field in the
// stream is not of the correct wire type.
func (r *reader) ReadFixed64Slice() (ret []uint64) {
	defer catchException(&r.err)
	values := r.Call("readPackedFixed64").Interface().([]interface{})
	for _, value := range values {
		ret = append(ret, uint64(value.(float64)))
	}

	return ret
}

// ReadSfixed32Slice reads a repeated signed 32-bit integer field from the binary
// stream, or sets err if the next field in the
// stream is not of the correct wire type.
func (r *reader) ReadSfixed32Slice() (ret []int32) {
	defer catchException(&r.err)
	values := r.Call("readPackedSfixed32").Interface().([]interface{})
	for _, value := range values {
		ret = append(ret, int32(value.(float64)))
	}

	return ret
}

// ReadSfixed64Slice reads a repeated signed 64-bit integer field from the binary
// stream, or sets err if the next field in the
// stream is not of the correct wire type.
func (r *reader) ReadSfixed64Slice() (ret []int64) {
	defer catchException(&r.err)
	values := r.Call("readPackedSfixed64").Interface().([]interface{})
	for _, value := range values {
		ret = append(ret, int64(value.(float64)))
	}

	return ret
}

// ReadFloat32Slice reads a repeated 32-bit floating point field from the binary
// stream, or sets err if the next field in the
// stream is not of the correct wire type.
func (r *reader) ReadFloat32Slice() (ret []float32) {
	defer catchException(&r.err)
	values := r.Call("readPackedFloat").Interface().([]interface{})
	for _, value := range values {
		ret = append(ret, float32(value.(float64)))
	}

	return ret
}

// ReadFloat64Slice reads a repeated 64-bit floating point field from the binary
// stream, or sets err if the next field in the
// stream is not of the correct wire type.
func (r *reader) ReadFloat64Slice() (ret []float64) {
	defer catchException(&r.err)
	values := r.Call("readPackedDouble").Interface().([]interface{})
	for _, value := range values {
		ret = append(ret, value.(float64))
	}

	return ret
}

// ReadEnumSlice reads a repeated enum field from the binary stream,
// or sets err if the next field in the stream
// is not of the correct wire type.
func (r *reader) ReadEnumSlice() (ret []int) {
	defer catchException(&r.err)
	values := r.Call("readPackedEnum").Interface().([]interface{})
	for _, value := range values {
		ret = append(ret, int(value.(float64)))
	}

	return ret
}

// ReadBoolSlice reads a repeated bool field from the binary stream, or sets err
// if the next field in the stream is not of the correct wire type.
func (r *reader) ReadBoolSlice() (ret []bool) {
	defer catchException(&r.err)
	values := r.Call("readPackedBool").Interface().([]interface{})
	for _, value := range values {
		ret = append(ret, value.(bool))
	}

	return ret
}
<|endoftext|>


<commit_before>package vault

import (
	"github.com/hashicorp/vault/api"
	. "github.com/smartystreets/goconvey/convey"
	"testing"

	"bytes"
	"github.com/pkg/errors"
	"io/ioutil"
	"net/http"
	"net/url"
	"strings"
)

func Test_DecryptEnvs(t *testing.T) {
	envVault := EnvVault{client: &mockVaultAPI{}}

	Convey("EnvVault", t, func() {
		Convey("isVaultPath() returns false for a normal env value", func() {
			value := isVaultPath("value")
			So(value, ShouldBeFalse)

			value = isVaultPath("1234")
			So(value, ShouldBeFalse)

			value = isVaultPath("http://url")
			So(value, ShouldBeFalse)

		})

		Convey("isVaultPath() returns true for a vault key value", func() {
			value := isVaultPath("vault://test")
			So(value, ShouldBeTrue)

			value = isVaultPath("vault://secret/key1")
			So(value, ShouldBeTrue)
		})

		Convey("ReadSecretValue fails for an invalid key", func() {
			value, err := envVault.ReadSecretValue("invalid path")
			So(value, ShouldBeEmpty)
			So(err.Error(), ShouldContainSubstring, "invalid path")

		})

		Convey("ReadSecretValue fails for key not found", func() {
			value, err := envVault.ReadSecretValue("vault://secure/notFoundKey")
			So(value, ShouldBeEmpty)
			So(err.Error(), ShouldContainSubstring, "Path 'secure/notFoundKey' not found")
		})

		Convey("ReadSecretValue returns a secured value for a valid key", func() {
			securedValue, err := envVault.ReadSecretValue("vault://secure/validKey")
			So(err, ShouldBeNil)
			So(securedValue, ShouldEqual, "secret_value")

		})

		Convey("ReadSecretValue returns a secured value for a valid sub-key", func() {
			securedValue, err := envVault.ReadSecretValue("vault://secure/validKeyWithSub?key=subkey")
			So(err, ShouldBeNil)
			So(securedValue, ShouldEqual, "some_sub_value")
		})

		Convey("Process all environment vars with Vault path", func() {
			envs := []string{"Key1=Value1", "Key2=Value2", "Key3=vault://secure/validKey"}
			expected := []string{"Key1=Value1", "Key2=Value2", "Key3=secret_value"}
			securedValue, err := envVault.DecryptAllEnv(envs)
			So(err, ShouldBeNil)
			So(securedValue, ShouldResemble, expected)
		})
	})
}

type mockVaultAPI struct {
	VaultAPI
}

func (m mockVaultAPI) Address() string {
	return "http://test"
}

func (m mockVaultAPI) NewRequest(method, path string) *api.Request {
	url, _ := url.Parse(path)
	return &api.Request{URL: url}
}

func (m mockVaultAPI) RawRequest(r *api.Request) (*api.Response, error) {
	if strings.Contains(r.URL.Path, "secure/notFoundKey") {
		return &api.Response{
			Response: &http.Response{
				StatusCode: 404,
				Body:       ioutil.NopCloser(bytes.NewReader([]byte(""))),
			},
		}, nil
	}

	if strings.Contains(r.URL.Path, "secure/validKeyWithSub") {
		return &api.Response{
			Response: &http.Response{
				StatusCode: 200,
				Body:       ioutil.NopCloser(bytes.NewReader([]byte(`{ "data":{ "subkey": "some_sub_value" }}`))),
			},
		}, nil
	}

	if strings.Contains(r.URL.Path, "secure/validKey") {
		return &api.Response{
			Response: &http.Response{
				StatusCode: 200,
				Body:       ioutil.NopCloser(bytes.NewReader([]byte(`{ "data":{ "value": "secret_value" }}`))),
			},
		}, nil
	}

	return nil, errors.New("Unexpected")
}
<commit_msg>Add a negative test case as well<commit_after>package vault

import (
	"github.com/hashicorp/vault/api"
	. "github.com/smartystreets/goconvey/convey"
	"testing"

	"bytes"
	"github.com/pkg/errors"
	"io/ioutil"
	"net/http"
	"net/url"
	"strings"
)

func Test_DecryptEnvs(t *testing.T) {
	envVault := EnvVault{client: &mockVaultAPI{}}

	Convey("EnvVault", t, func() {
		Convey("isVaultPath() returns false for a normal env value", func() {
			value := isVaultPath("value")
			So(value, ShouldBeFalse)

			value = isVaultPath("1234")
			So(value, ShouldBeFalse)

			value = isVaultPath("http://url")
			So(value, ShouldBeFalse)

		})

		Convey("isVaultPath() returns true for a vault key value", func() {
			value := isVaultPath("vault://test")
			So(value, ShouldBeTrue)

			value = isVaultPath("vault://secret/key1")
			So(value, ShouldBeTrue)
		})

		Convey("ReadSecretValue fails for an invalid key", func() {
			value, err := envVault.ReadSecretValue("invalid path")
			So(value, ShouldBeEmpty)
			So(err.Error(), ShouldContainSubstring, "invalid path")

		})

		Convey("ReadSecretValue fails for key not found", func() {
			value, err := envVault.ReadSecretValue("vault://secure/notFoundKey")
			So(value, ShouldBeEmpty)
			So(err.Error(), ShouldContainSubstring, "Path 'secure/notFoundKey' not found")
		})

		Convey("ReadSecretValue returns a secured value for a valid key", func() {
			securedValue, err := envVault.ReadSecretValue("vault://secure/validKey")
			So(err, ShouldBeNil)
			So(securedValue, ShouldEqual, "secret_value")

		})

		Convey("ReadSecretValue returns a secured value for a valid sub-key", func() {
			securedValue, err := envVault.ReadSecretValue("vault://secure/validKeyWithSub?key=subkey")
			So(err, ShouldBeNil)
			So(securedValue, ShouldEqual, "some_sub_value")
		})

		Convey("ReadSecretValue returns an error on a missing sub-key", func() {
			securedValue, err := envVault.ReadSecretValue("vault://secure/validKeyWithSub?key=missing")
			So(err.Error(), ShouldContainSubstring, "Value for path 'secure/validKeyWithSub' not found")
			So(securedValue, ShouldEqual, "")
		})

		Convey("Process all environment vars with Vault path", func() {
			envs := []string{"Key1=Value1", "Key2=Value2", "Key3=vault://secure/validKey"}
			expected := []string{"Key1=Value1", "Key2=Value2", "Key3=secret_value"}
			securedValue, err := envVault.DecryptAllEnv(envs)
			So(err, ShouldBeNil)
			So(securedValue, ShouldResemble, expected)
		})
	})
}

type mockVaultAPI struct {
	VaultAPI
}

func (m mockVaultAPI) Address() string {
	return "http://test"
}

func (m mockVaultAPI) NewRequest(method, path string) *api.Request {
	url, _ := url.Parse(path)
	return &api.Request{URL: url}
}

func (m mockVaultAPI) RawRequest(r *api.Request) (*api.Response, error) {
	if strings.Contains(r.URL.Path, "secure/notFoundKey") {
		return &api.Response{
			Response: &http.Response{
				StatusCode: 404,
				Body:       ioutil.NopCloser(bytes.NewReader([]byte(""))),
			},
		}, nil
	}

	if strings.Contains(r.URL.Path, "secure/validKeyWithSub") {
		return &api.Response{
			Response: &http.Response{
				StatusCode: 200,
				Body:       ioutil.NopCloser(bytes.NewReader([]byte(`{ "data":{ "subkey": "some_sub_value" }}`))),
			},
		}, nil
	}

	if strings.Contains(r.URL.Path, "secure/validKey") {
		return &api.Response{
			Response: &http.Response{
				StatusCode: 200,
				Body:       ioutil.NopCloser(bytes.NewReader([]byte(`{ "data":{ "value": "secret_value" }}`))),
			},
		}, nil
	}

	return nil, errors.New("Unexpected")
}
<|endoftext|>


<commit_before>package metrics

import (
	"encoding/json"
	"fmt"
	"io/ioutil"
	"strconv"
	"strings"
	"time"

	"bytes"

	"github.com/Sirupsen/logrus"
	"github.com/feedhenry/mcp-standalone/pkg/mobile"
	"github.com/pkg/errors"
)

// TODO: Add multi-worker support?
type statsResponse struct {
	Metrics struct {
		CPUUsage struct {
			Master struct {
				Current         string    `json:"current"`
				Max             string    `json:"max"`
				Min             string    `json:"min"`
				Average         string    `json:"average"`
				NumberOfRecords int64     `json:"numberOfRecords"`
				From            time.Time `json:"from"`
				End             time.Time `json:"end"`
			} `json:"master"`
		} `json:"CPU usage"`
		RSSMemoryUsage struct {
			Master struct {
				Current         string    `json:"current"`
				Max             string    `json:"max"`
				Min             string    `json:"min"`
				Average         string    `json:"average"`
				NumberOfRecords int64     `json:"numberOfRecords"`
				From            time.Time `json:"from"`
				End             time.Time `json:"end"`
			} `json:"master"`
		} `json:"RSS Memory Usage"`
		JobProcessTime struct {
			SyncWorker struct {
				Current         string    `json:"current"`
				Max             string    `json:"max"`
				Min             string    `json:"min"`
				Average         string    `json:"average"`
				NumberOfRecords int64     `json:"numberOfRecords"`
				From            time.Time `json:"from"`
				End             time.Time `json:"end"`
			} `json:"sync_worker"`
			AckWorker struct {
				Current         string    `json:"current"`
				Max             string    `json:"max"`
				Min             string    `json:"min"`
				Average         string    `json:"average"`
				NumberOfRecords int64     `json:"numberOfRecords"`
				From            time.Time `json:"from"`
				End             time.Time `json:"end"`
			} `json:"ack_worker"`
			PendingWorker struct {
				Current         string    `json:"current"`
				Max             string    `json:"max"`
				Min             string    `json:"min"`
				Average         string    `json:"average"`
				NumberOfRecords int64     `json:"numberOfRecords"`
				From            time.Time `json:"from"`
				End             time.Time `json:"end"`
			} `json:"pending_worker"`
		} `json:"Job Process Time"`
		JobQueueSize struct {
			PendingWorker struct {
				Current         int64     `json:"current"`
				Max             int64     `json:"max"`
				Min             int64     `json:"min"`
				Average         float64   `json:"average"`
				NumberOfRecords int64     `json:"numberOfRecords"`
				From            time.Time `json:"from"`
				End             time.Time `json:"end"`
			} `json:"pending_worker"`
			AckWorker struct {
				Current         int64     `json:"current"`
				Max             int64     `json:"max"`
				Min             int64     `json:"min"`
				Average         float64   `json:"average"`
				NumberOfRecords int64     `json:"numberOfRecords"`
				From            time.Time `json:"from"`
				End             time.Time `json:"end"`
			} `json:"ack_worker"`
			SyncWorker struct {
				Current         int64     `json:"current"`
				Max             int64     `json:"max"`
				Min             int64     `json:"min"`
				Average         float64   `json:"average"`
				NumberOfRecords int64     `json:"numberOfRecords"`
				From            time.Time `json:"from"`
				End             time.Time `json:"end"`
			} `json:"sync_worker"`
		} `json:"Job Queue Size"`
		APIProcessTime struct {
			Sync struct {
				Current         string    `json:"current"`
				Max             string    `json:"max"`
				Min             string    `json:"min"`
				Average         string    `json:"average"`
				NumberOfRecords int64     `json:"numberOfRecords"`
				From            time.Time `json:"from"`
				End             time.Time `json:"end"`
			} `json:"sync"`
		} `json:"API Process Time"`
		MongodbOperationTime struct {
			DoUpdateManyDatasetClients struct {
				Current         string    `json:"current"`
				Max             string    `json:"max"`
				Min             string    `json:"min"`
				Average         string    `json:"average"`
				NumberOfRecords int64     `json:"numberOfRecords"`
				From            time.Time `json:"from"`
				End             time.Time `json:"end"`
			} `json:"doUpdateManyDatasetClients"`
			DoListDatasetClients struct {
				Current         string    `json:"current"`
				Max             string    `json:"max"`
				Min             string    `json:"min"`
				Average         string    `json:"average"`
				NumberOfRecords int64     `json:"numberOfRecords"`
				From            time.Time `json:"from"`
				End             time.Time `json:"end"`
			} `json:"doListDatasetClients"`
		} `json:"Mongodb Operation Time"`
	} `json:"metrics"`
}

type FhSyncServer struct {
	requestBuilder     mobile.HTTPRequesterBuilder
	tokenClientBuilder mobile.TokenScopedClientBuilder
	ServiceName        string
	logger             *logrus.Logger
}

func NewFhSyncServer(rbuilder mobile.HTTPRequesterBuilder, tokenCBuilder mobile.TokenScopedClientBuilder, l *logrus.Logger) *FhSyncServer {
	return &FhSyncServer{requestBuilder: rbuilder, tokenClientBuilder: tokenCBuilder, ServiceName: "fh-sync-server", logger: l}
}

// Gather will retrieve varous metrics from fh-sync-server
func (ss *FhSyncServer) Gather() ([]*metric, error) {
	svc, err := ss.tokenClientBuilder.UseDefaultSAToken().MobileServiceCruder("")
	if err != nil {
		return nil, errors.Wrap(err, "fh-sync-server gather failed to create svcruder")
	}
	ssServices, err := svc.List(func(attrs mobile.Attributer) bool {
		return attrs.GetName() == ss.ServiceName
	})
	if err != nil {
		return nil, errors.Wrap(err, "fh-sync-server gather failed to list existing services")
	}
	if len(ssServices) == 0 {
		return nil, errors.New(" no fh-sync-server service present")
	}
	ssService := ssServices[0] //TODO deal with more than one
	//TODO get protocol from secret
	host := ssService.Host
	now := time.Now()

	stats, err := ss.getStats(host)
	if err != nil {
		return nil, errors.Wrap(err, "fh-sync-server gather failed to list existing services")
	}

	var ssMetrics = []*metric{}
	if nil != stats {
		// sync worker stats
		ssMetrics = append(ssMetrics, &metric{Type: "sync_worker_queue_count", XValue: now.Format("2006-01-02 15:04:05"), YValue: stats.Metrics.JobQueueSize.SyncWorker.Current})
		syncProcessTimeStr := strings.Split(stats.Metrics.JobProcessTime.SyncWorker.Current, ".")[0]
		syncProcessTime, _ := strconv.ParseInt(syncProcessTimeStr, 10, 64)
		ssMetrics = append(ssMetrics, &metric{Type: "sync_worker_process_time_ms", XValue: now.Format("2006-01-02 15:04:05"), YValue: syncProcessTime})

		// pending worker stats
		ssMetrics = append(ssMetrics, &metric{Type: "pending_worker_queue_count", XValue: now.Format("2006-01-02 15:04:05"), YValue: stats.Metrics.JobQueueSize.PendingWorker.Current})
		pendingProcessTimeStr := strings.Split(stats.Metrics.JobProcessTime.PendingWorker.Current, ".")[0]
		pendingProcessTime, _ := strconv.ParseInt(pendingProcessTimeStr, 10, 64)
		ssMetrics = append(ssMetrics, &metric{Type: "pending_worker_process_time_ms", XValue: now.Format("2006-01-02 15:04:05"), YValue: pendingProcessTime})

		// ack worker stats
		ssMetrics = append(ssMetrics, &metric{Type: "ack_worker_queue_count", XValue: now.Format("2006-01-02 15:04:05"), YValue: stats.Metrics.JobQueueSize.AckWorker.Current})
		ackProcessTimeStr := strings.Split(stats.Metrics.JobProcessTime.AckWorker.Current, ".")[0]
		ackProcessTime, _ := strconv.ParseInt(ackProcessTimeStr, 10, 64)
		ssMetrics = append(ssMetrics, &metric{Type: "ack_worker_process_time_ms", XValue: now.Format("2006-01-02 15:04:05"), YValue: ackProcessTime})
	}

	return ssMetrics, nil
}

func (ss *FhSyncServer) getStats(host string) (*statsResponse, error) {
	statsURL := fmt.Sprintf("%s/sys/info/stats", host)
	requester := ss.requestBuilder.Insecure(true).Build()
	res, err := requester.Get(statsURL)
	if err != nil {
		return &statsResponse{}, err
	}
	defer res.Body.Close()

	data, err := ioutil.ReadAll(res.Body)
	if err != nil {
		return &statsResponse{}, errors.Wrap(err, "error reading stats response body")
	}
	fmt.Printf("\n\ndata %s ", string(data))

	var stats statsResponse
	decoder := json.NewDecoder(bytes.NewBuffer(data))
	if err := decoder.Decode(&stats); err != nil {
		return &statsResponse{}, errors.Wrap(err, "failed to decode stats response")
	}

	fmt.Printf("\n\nstats %v ", stats)
	fmt.Printf("\n\nstats.JobQueueSize.SyncWorker.NumberOfRecords (%v) ", stats.Metrics.JobQueueSize.SyncWorker.NumberOfRecords)

	return &stats, nil
}
<commit_msg>Add sync metrics debug logging<commit_after>package metrics

import (
	"encoding/json"
	"fmt"
	"io/ioutil"
	"strconv"
	"strings"
	"time"

	"bytes"

	"github.com/Sirupsen/logrus"
	"github.com/feedhenry/mcp-standalone/pkg/mobile"
	"github.com/pkg/errors"
)

// TODO: Add multi-worker support?
type statsResponse struct {
	Metrics struct {
		CPUUsage struct {
			Master struct {
				Current         string    `json:"current"`
				Max             string    `json:"max"`
				Min             string    `json:"min"`
				Average         string    `json:"average"`
				NumberOfRecords int64     `json:"numberOfRecords"`
				From            time.Time `json:"from"`
				End             time.Time `json:"end"`
			} `json:"master"`
		} `json:"CPU usage"`
		RSSMemoryUsage struct {
			Master struct {
				Current         string    `json:"current"`
				Max             string    `json:"max"`
				Min             string    `json:"min"`
				Average         string    `json:"average"`
				NumberOfRecords int64     `json:"numberOfRecords"`
				From            time.Time `json:"from"`
				End             time.Time `json:"end"`
			} `json:"master"`
		} `json:"RSS Memory Usage"`
		JobProcessTime struct {
			SyncWorker struct {
				Current         string    `json:"current"`
				Max             string    `json:"max"`
				Min             string    `json:"min"`
				Average         string    `json:"average"`
				NumberOfRecords int64     `json:"numberOfRecords"`
				From            time.Time `json:"from"`
				End             time.Time `json:"end"`
			} `json:"sync_worker"`
			AckWorker struct {
				Current         string    `json:"current"`
				Max             string    `json:"max"`
				Min             string    `json:"min"`
				Average         string    `json:"average"`
				NumberOfRecords int64     `json:"numberOfRecords"`
				From            time.Time `json:"from"`
				End             time.Time `json:"end"`
			} `json:"ack_worker"`
			PendingWorker struct {
				Current         string    `json:"current"`
				Max             string    `json:"max"`
				Min             string    `json:"min"`
				Average         string    `json:"average"`
				NumberOfRecords int64     `json:"numberOfRecords"`
				From            time.Time `json:"from"`
				End             time.Time `json:"end"`
			} `json:"pending_worker"`
		} `json:"Job Process Time"`
		JobQueueSize struct {
			PendingWorker struct {
				Current         int64     `json:"current"`
				Max             int64     `json:"max"`
				Min             int64     `json:"min"`
				Average         float64   `json:"average"`
				NumberOfRecords int64     `json:"numberOfRecords"`
				From            time.Time `json:"from"`
				End             time.Time `json:"end"`
			} `json:"pending_worker"`
			AckWorker struct {
				Current         int64     `json:"current"`
				Max             int64     `json:"max"`
				Min             int64     `json:"min"`
				Average         float64   `json:"average"`
				NumberOfRecords int64     `json:"numberOfRecords"`
				From            time.Time `json:"from"`
				End             time.Time `json:"end"`
			} `json:"ack_worker"`
			SyncWorker struct {
				Current         int64     `json:"current"`
				Max             int64     `json:"max"`
				Min             int64     `json:"min"`
				Average         float64   `json:"average"`
				NumberOfRecords int64     `json:"numberOfRecords"`
				From            time.Time `json:"from"`
				End             time.Time `json:"end"`
			} `json:"sync_worker"`
		} `json:"Job Queue Size"`
		APIProcessTime struct {
			Sync struct {
				Current         string    `json:"current"`
				Max             string    `json:"max"`
				Min             string    `json:"min"`
				Average         string    `json:"average"`
				NumberOfRecords int64     `json:"numberOfRecords"`
				From            time.Time `json:"from"`
				End             time.Time `json:"end"`
			} `json:"sync"`
		} `json:"API Process Time"`
		MongodbOperationTime struct {
			DoUpdateManyDatasetClients struct {
				Current         string    `json:"current"`
				Max             string    `json:"max"`
				Min             string    `json:"min"`
				Average         string    `json:"average"`
				NumberOfRecords int64     `json:"numberOfRecords"`
				From            time.Time `json:"from"`
				End             time.Time `json:"end"`
			} `json:"doUpdateManyDatasetClients"`
			DoListDatasetClients struct {
				Current         string    `json:"current"`
				Max             string    `json:"max"`
				Min             string    `json:"min"`
				Average         string    `json:"average"`
				NumberOfRecords int64     `json:"numberOfRecords"`
				From            time.Time `json:"from"`
				End             time.Time `json:"end"`
			} `json:"doListDatasetClients"`
		} `json:"Mongodb Operation Time"`
	} `json:"metrics"`
}

type FhSyncServer struct {
	requestBuilder     mobile.HTTPRequesterBuilder
	tokenClientBuilder mobile.TokenScopedClientBuilder
	ServiceName        string
	logger             *logrus.Logger
}

func NewFhSyncServer(rbuilder mobile.HTTPRequesterBuilder, tokenCBuilder mobile.TokenScopedClientBuilder, l *logrus.Logger) *FhSyncServer {
	return &FhSyncServer{requestBuilder: rbuilder, tokenClientBuilder: tokenCBuilder, ServiceName: "fh-sync-server", logger: l}
}

// Gather will retrieve varous metrics from fh-sync-server
func (ss *FhSyncServer) Gather() ([]*metric, error) {
	svc, err := ss.tokenClientBuilder.UseDefaultSAToken().MobileServiceCruder("")
	if err != nil {
		return nil, errors.Wrap(err, "fh-sync-server gather failed to create svcruder")
	}
	ssServices, err := svc.List(func(attrs mobile.Attributer) bool {
		return attrs.GetName() == ss.ServiceName
	})
	if err != nil {
		return nil, errors.Wrap(err, "fh-sync-server gather failed to list existing services")
	}
	if len(ssServices) == 0 {
		return nil, errors.New(" no fh-sync-server service present")
	}
	ssService := ssServices[0] //TODO deal with more than one
	//TODO get protocol from secret
	host := ssService.Host
	now := time.Now()

	stats, err := ss.getStats(host)
	if err != nil {
		return nil, errors.Wrap(err, "fh-sync-server gather failed to list existing services")
	}

	var ssMetrics = []*metric{}
	if nil != stats {
		// sync worker stats
		ssMetrics = append(ssMetrics, &metric{Type: "sync_worker_queue_count", XValue: now.Format("2006-01-02 15:04:05"), YValue: stats.Metrics.JobQueueSize.SyncWorker.Current})
		syncProcessTimeStr := strings.Split(stats.Metrics.JobProcessTime.SyncWorker.Current, ".")[0]
		syncProcessTime, _ := strconv.ParseInt(syncProcessTimeStr, 10, 64)
		ssMetrics = append(ssMetrics, &metric{Type: "sync_worker_process_time_ms", XValue: now.Format("2006-01-02 15:04:05"), YValue: syncProcessTime})

		// pending worker stats
		ssMetrics = append(ssMetrics, &metric{Type: "pending_worker_queue_count", XValue: now.Format("2006-01-02 15:04:05"), YValue: stats.Metrics.JobQueueSize.PendingWorker.Current})
		pendingProcessTimeStr := strings.Split(stats.Metrics.JobProcessTime.PendingWorker.Current, ".")[0]
		pendingProcessTime, _ := strconv.ParseInt(pendingProcessTimeStr, 10, 64)
		ssMetrics = append(ssMetrics, &metric{Type: "pending_worker_process_time_ms", XValue: now.Format("2006-01-02 15:04:05"), YValue: pendingProcessTime})

		// ack worker stats
		ssMetrics = append(ssMetrics, &metric{Type: "ack_worker_queue_count", XValue: now.Format("2006-01-02 15:04:05"), YValue: stats.Metrics.JobQueueSize.AckWorker.Current})
		ackProcessTimeStr := strings.Split(stats.Metrics.JobProcessTime.AckWorker.Current, ".")[0]
		ackProcessTime, _ := strconv.ParseInt(ackProcessTimeStr, 10, 64)
		ssMetrics = append(ssMetrics, &metric{Type: "ack_worker_process_time_ms", XValue: now.Format("2006-01-02 15:04:05"), YValue: ackProcessTime})
	}
	ss.logger.Debugf("ssMetrics %v", ssMetrics)

	return ssMetrics, nil
}

func (ss *FhSyncServer) getStats(host string) (*statsResponse, error) {
	statsURL := fmt.Sprintf("%s/sys/info/stats", host)
	requester := ss.requestBuilder.Insecure(true).Build()
	res, err := requester.Get(statsURL)
	if err != nil {
		return &statsResponse{}, err
	}
	defer res.Body.Close()

	data, err := ioutil.ReadAll(res.Body)
	if err != nil {
		return &statsResponse{}, errors.Wrap(err, "error reading stats response body")
	}
	ss.logger.Debugf("raw stats response %v", data)

	var stats statsResponse
	decoder := json.NewDecoder(bytes.NewBuffer(data))
	if err := decoder.Decode(&stats); err != nil {
		return &statsResponse{}, errors.Wrap(err, "failed to decode stats response")
	}
	ss.logger.Debugf("decoded stats response %v", stats)

	return &stats, nil
}
<|endoftext|>