<commit_before>package lazycache

import (
	"fmt"
	flag "github.com/spf13/pflag"
	"github.com/spf13/viper"
	"strings"
)

func ViperConfiguration() {

	// Configuration
	viper.SetDefault("port", 8080)
	viper.SetDefault("bind", "0.0.0.0")
	viper.SetDefault("imagestore", "")
	viper.SetDefault("imagestore.bucket", "camhd-image-cache")

	viper.SetDefault("quicktimestore", "")
	viper.SetDefault("directorystore", "")
	viper.SetDefault("redishost", "localhost:6379")

	viper.SetConfigName("lazycache")
	viper.AddConfigPath("/etc/lazycache")
	viper.AddConfigPath(".")
	err := viper.ReadInConfig() // Find and read the config file
	if err != nil {             // Handle errors reading the config file
		switch err.(type) {
		case viper.ConfigFileNotFoundError:
			// ignore
		default:
			panic(fmt.Errorf("Fatal error config file: %s \n", err))
		}
	}

	viper.SetEnvPrefix("lazycache")
	viper.AutomaticEnv()
	// Convert '.' to '_' in configuration variable names
	viper.SetEnvKeyReplacer(strings.NewReplacer(".", "_"))

	// var (
	// 	bindFlag          = flag.String("bind", "0.0.0.0", "Network interface to bind to (defaults to 0.0.0.0)")
	// 	ImageStoreFlag   = flag.String("image-store", "", "Type of image store (none, google)")
	// 	ImageBucketFlag = flag.String("image-store-bucket", "", "Bucket used for Google image store")
	// )
	flag.Int("port", 80, "Network port to listen on (default: 8080)")
	flag.String("bind", "0.0.0.0", "Network interface to bind to (defaults to 0.0.0.0)")

	flag.String("image-store", "", "Type of image store (none, local, google)")
	flag.String("image-store-bucket", "camhd-image-cache", "Bucket used for Google image store")
	flag.String("image-local-root", "", "Bucket used for Google image store")
	flag.String("image-url-root", "", "Bucket used for Google image store")

	flag.String("quicktime-store", "", "Type of quicktime store (none, redis)")
	flag.String("directory-store", "", "Type of directory store (none, redis)")
	flag.String("redis-host", "localhost:6379", "Host used for redis store")

	viper.BindPFlag("port", flag.Lookup("port"))
	viper.BindPFlag("bind", flag.Lookup("bind"))

	viper.BindPFlag("imagestore", flag.Lookup("image-store"))
	viper.BindPFlag("imagestore.bucket", flag.Lookup("image-store-bucket"))
	viper.BindPFlag("imagestore.localroot", flag.Lookup("image-local-root"))

	viper.BindPFlag("directorystore", flag.Lookup("directory-store"))
	viper.BindPFlag("quicktimestore", flag.Lookup("quicktime-store"))
	viper.BindPFlag("redishost", flag.Lookup("redis-host"))

	flag.Parse()
}

func ConfigureImageStoreFromViper() {
	storeKey := viper.GetString("imagestore")
	DefaultLogger.Log("msg", fmt.Sprintf("Configuring image store with type \"%s\"", storeKey))
	switch strings.ToLower(storeKey) {
	default:
		DefaultLogger.Log("msg", fmt.Sprintf("Unable to determine type of image store from \"%s\"", storeKey))
		DefaultImageStore = NullImageStore{}
	case "", "none":
		DefaultLogger.Log("msg", "No image store configured.")
		DefaultImageStore = NullImageStore{}
	case "local":
		DefaultImageStore = CreateLocalStore(viper.GetString("imagestore.localRoot"),
			viper.GetString("imagestore.bind"))
	case "google":
		DefaultImageStore = CreateGoogleStore(viper.GetString("imagestore.bucket"))
	}
}

func ConfigureQuicktimeStoreFromViper() {
	storeKey := viper.GetString("quicktimestore")
	DefaultLogger.Log("msg", fmt.Sprintf("Configuring quicktime store with type \"%s\"", storeKey))

	switch strings.ToLower(storeKey) {
	default:
		DefaultLogger.Log("msg", fmt.Sprintf("Unable to determine type of image store from \"%s\"", storeKey))
		QTMetadataStore = CreateMapJSONStore()
	case "", "none":
		DefaultLogger.Log("msg", "Using default QuicktimeStore.")
		QTMetadataStore = CreateMapJSONStore()
	case "redis":
		hostname := viper.GetString("redishost")
		DefaultLogger.Log("msg", fmt.Sprintf("Connecting to redis host \"%s\"", hostname))
		redis, err := CreateRedisJSONStore(hostname, "qt")
		if err != nil {
			DefaultLogger.Log("msg", fmt.Sprintf("Failed to configure Redis Quicktime store to host \"%s\"", hostname))
		}

		DefaultLogger.Log("msg", fmt.Sprintf("Logging movie metadata to Redis at %s", hostname))
		QTMetadataStore = redis
	}
}

func ConfigureDirectoryStoreFromViper() {
	storeKey := viper.GetString("directorystore")
	DefaultLogger.Log("msg", fmt.Sprintf("Configuring directory store with type \"%s\"", storeKey))

	switch strings.ToLower(storeKey) {
	default:
		DefaultLogger.Log("msg", fmt.Sprintf("Unable to determine type of directory store from \"%s\"", storeKey))
		DirKeyStore = CreateMapJSONStore()
	case "", "none":
		DefaultLogger.Log("msg", "Using default directory store.")
		DirKeyStore = CreateMapJSONStore()
	case "redis":
		hostname := viper.GetString("redishost")
		DefaultLogger.Log("msg", fmt.Sprintf("Connecting to redis host \"%s\"", hostname))
		redis, err := CreateRedisJSONStore(hostname, "dir")
		if err != nil {
			DefaultLogger.Log("msg", fmt.Sprintf("Failed to configure Redis directory store to host \"%s\"", hostname))
		}

		DefaultLogger.Log("msg", fmt.Sprintf("Logging directory metadata to Redis at %s", hostname))
		DirKeyStore = redis
	}
}

func ConfigureFromViper() {
	ViperConfiguration()

	DefaultLogger.Log("msg", "In ConfigureFromViper")
	ConfigureImageStoreFromViper()
	ConfigureDirectoryStoreFromViper()
	ConfigureQuicktimeStoreFromViper()
}
<commit_msg>Fixed naming error with image store configuration variables.<commit_after>package lazycache

import (
	"fmt"
	flag "github.com/spf13/pflag"
	"github.com/spf13/viper"
	"strings"
)

func ViperConfiguration() {

	// Configuration
	viper.SetDefault("port", 8080)
	viper.SetDefault("bind", "0.0.0.0")
	viper.SetDefault("imagestore", "")
	viper.SetDefault("imagestore.bucket", "camhd-image-cache")

	viper.SetDefault("quicktimestore", "")
	viper.SetDefault("directorystore", "")
	viper.SetDefault("redishost", "localhost:6379")

	viper.SetConfigName("lazycache")
	viper.AddConfigPath("/etc/lazycache")
	viper.AddConfigPath(".")
	err := viper.ReadInConfig() // Find and read the config file
	if err != nil {             // Handle errors reading the config file
		switch err.(type) {
		case viper.ConfigFileNotFoundError:
			// ignore
		default:
			panic(fmt.Errorf("Fatal error config file: %s \n", err))
		}
	}

	viper.SetEnvPrefix("lazycache")
	viper.AutomaticEnv()
	// Convert '.' to '_' in configuration variable names
	viper.SetEnvKeyReplacer(strings.NewReplacer(".", "_"))

	flag.Int("port", 80, "Network port to listen on (default: 8080)")
	flag.String("bind", "0.0.0.0", "Network interface to bind to (defaults to 0.0.0.0)")

	flag.String("image-store", "", "Type of image store (none, local, google)")
	flag.String("image-store-bucket", "camhd-image-cache", "Bucket used for Google image store")
	flag.String("image-store-root", "", "Path to local image store directory (must be writable)")
	flag.String("image-store-url", "", "Root URL for webserver which serves image store directory")

	flag.String("quicktime-store", "", "Type of quicktime store (none, redis)")
	flag.String("directory-store", "", "Type of directory store (none, redis)")
	flag.String("redis-host", "localhost:6379", "Host used for redis store")

	viper.BindPFlag("port", flag.Lookup("port"))
	viper.BindPFlag("bind", flag.Lookup("bind"))

	viper.BindPFlag("imagestore", flag.Lookup("image-store"))
	viper.BindPFlag("imagestore.bucket", flag.Lookup("image-store-bucket"))
	viper.BindPFlag("imagestore.root", flag.Lookup("image-store-root"))
	viper.BindPFlag("imagestore.url", flag.Lookup("image-store-url"))

	viper.BindPFlag("directorystore", flag.Lookup("directory-store"))
	viper.BindPFlag("quicktimestore", flag.Lookup("quicktime-store"))
	viper.BindPFlag("redishost", flag.Lookup("redis-host"))

	flag.Parse()
}

func ConfigureImageStoreFromViper() {
	storeKey := viper.GetString("imagestore")
	DefaultLogger.Log("msg", fmt.Sprintf("Configuring image store with type \"%s\"", storeKey))
	switch strings.ToLower(storeKey) {
	default:
		DefaultLogger.Log("msg", fmt.Sprintf("Unable to determine type of image store from \"%s\"", storeKey))
		DefaultImageStore = NullImageStore{}
	case "", "none":
		DefaultLogger.Log("msg", "No image store configured.")
		DefaultImageStore = NullImageStore{}
	case "local":
		DefaultImageStore = CreateLocalStore(viper.GetString("imagestore.root"), viper.GetString("imagestore.url"))
	case "google":
		DefaultImageStore = CreateGoogleStore(viper.GetString("imagestore.bucket"))
	}
}

func ConfigureQuicktimeStoreFromViper() {
	storeKey := viper.GetString("quicktimestore")
	DefaultLogger.Log("msg", fmt.Sprintf("Configuring quicktime store with type \"%s\"", storeKey))

	switch strings.ToLower(storeKey) {
	default:
		DefaultLogger.Log("msg", fmt.Sprintf("Unable to determine type of image store from \"%s\"", storeKey))
		QTMetadataStore = CreateMapJSONStore()
	case "", "none":
		DefaultLogger.Log("msg", "Using default QuicktimeStore.")
		QTMetadataStore = CreateMapJSONStore()
	case "redis":
		hostname := viper.GetString("redishost")
		DefaultLogger.Log("msg", fmt.Sprintf("Connecting to redis host \"%s\"", hostname))
		redis, err := CreateRedisJSONStore(hostname, "qt")
		if err != nil {
			DefaultLogger.Log("msg", fmt.Sprintf("Failed to configure Redis Quicktime store to host \"%s\"", hostname))
		}

		DefaultLogger.Log("msg", fmt.Sprintf("Logging movie metadata to Redis at %s", hostname))
		QTMetadataStore = redis
	}
}

func ConfigureDirectoryStoreFromViper() {
	storeKey := viper.GetString("directorystore")
	DefaultLogger.Log("msg", fmt.Sprintf("Configuring directory store with type \"%s\"", storeKey))

	switch strings.ToLower(storeKey) {
	default:
		DefaultLogger.Log("msg", fmt.Sprintf("Unable to determine type of directory store from \"%s\"", storeKey))
		DirKeyStore = CreateMapJSONStore()
	case "", "none":
		DefaultLogger.Log("msg", "Using default directory store.")
		DirKeyStore = CreateMapJSONStore()
	case "redis":
		hostname := viper.GetString("redishost")
		DefaultLogger.Log("msg", fmt.Sprintf("Connecting to redis host \"%s\"", hostname))
		redis, err := CreateRedisJSONStore(hostname, "dir")
		if err != nil {
			DefaultLogger.Log("msg", fmt.Sprintf("Failed to configure Redis directory store to host \"%s\"", hostname))
		}

		DefaultLogger.Log("msg", fmt.Sprintf("Logging directory metadata to Redis at %s", hostname))
		DirKeyStore = redis
	}
}

func ConfigureFromViper() {
	ViperConfiguration()

	DefaultLogger.Log("msg", "In ConfigureFromViper")
	ConfigureImageStoreFromViper()
	ConfigureDirectoryStoreFromViper()
	ConfigureQuicktimeStoreFromViper()
}
<|endoftext|>


<commit_before>package version

import (
	"bytes"
	"fmt"
)

var (
	// The git commit that was compiled. This will be filled in by the compiler.
	GitCommit   string
	GitDescribe string

	// The main version number that is being run at the moment.
	Version = "0.12.0"

	// A pre-release marker for the version. If this is "" (empty string)
	// then it means that it is a final release. Otherwise, this is a pre-release
	// such as "dev" (in development), "beta", "rc1", etc.
	VersionPrerelease = "rc1"

	// VersionMetadata is metadata further describing the build type.
	VersionMetadata = ""
)

// VersionInfo
type VersionInfo struct {
	Revision          string
	Version           string
	VersionPrerelease string
	VersionMetadata   string
}

func GetVersion() *VersionInfo {
	ver := Version
	rel := VersionPrerelease
	md := VersionMetadata
	if GitDescribe != "" {
		ver = GitDescribe
	}
	if GitDescribe == "" && rel == "" && VersionPrerelease != "" {
		rel = "dev"
	}

	return &VersionInfo{
		Revision:          GitCommit,
		Version:           ver,
		VersionPrerelease: rel,
		VersionMetadata:   md,
	}
}

func (c *VersionInfo) VersionNumber() string {
	version := fmt.Sprintf("%s", c.Version)

	if c.VersionPrerelease != "" {
		version = fmt.Sprintf("%s-%s", version, c.VersionPrerelease)
	}

	if c.VersionMetadata != "" {
		version = fmt.Sprintf("%s+%s", version, c.VersionMetadata)
	}

	return version
}

func (c *VersionInfo) FullVersionNumber(rev bool) string {
	var versionString bytes.Buffer

	fmt.Fprintf(&versionString, "Nomad v%s", c.Version)
	if c.VersionPrerelease != "" {
		fmt.Fprintf(&versionString, "-%s", c.VersionPrerelease)
	}

	if c.VersionMetadata != "" {
		fmt.Fprintf(&versionString, "+%s", c.VersionMetadata)
	}

	if rev && c.Revision != "" {
		fmt.Fprintf(&versionString, " (%s)", c.Revision)
	}

	return versionString.String()
}
<commit_msg>Generate files for 0.12.0 release<commit_after>package version

import (
	"bytes"
	"fmt"
)

var (
	// The git commit that was compiled. This will be filled in by the compiler.
	GitCommit   string
	GitDescribe string

	// The main version number that is being run at the moment.
	Version = "0.12.0"

	// A pre-release marker for the version. If this is "" (empty string)
	// then it means that it is a final release. Otherwise, this is a pre-release
	// such as "dev" (in development), "beta", "rc1", etc.
	VersionPrerelease = ""

	// VersionMetadata is metadata further describing the build type.
	VersionMetadata = ""
)

// VersionInfo
type VersionInfo struct {
	Revision          string
	Version           string
	VersionPrerelease string
	VersionMetadata   string
}

func GetVersion() *VersionInfo {
	ver := Version
	rel := VersionPrerelease
	md := VersionMetadata
	if GitDescribe != "" {
		ver = GitDescribe
	}
	if GitDescribe == "" && rel == "" && VersionPrerelease != "" {
		rel = "dev"
	}

	return &VersionInfo{
		Revision:          GitCommit,
		Version:           ver,
		VersionPrerelease: rel,
		VersionMetadata:   md,
	}
}

func (c *VersionInfo) VersionNumber() string {
	version := fmt.Sprintf("%s", c.Version)

	if c.VersionPrerelease != "" {
		version = fmt.Sprintf("%s-%s", version, c.VersionPrerelease)
	}

	if c.VersionMetadata != "" {
		version = fmt.Sprintf("%s+%s", version, c.VersionMetadata)
	}

	return version
}

func (c *VersionInfo) FullVersionNumber(rev bool) string {
	var versionString bytes.Buffer

	fmt.Fprintf(&versionString, "Nomad v%s", c.Version)
	if c.VersionPrerelease != "" {
		fmt.Fprintf(&versionString, "-%s", c.VersionPrerelease)
	}

	if c.VersionMetadata != "" {
		fmt.Fprintf(&versionString, "+%s", c.VersionMetadata)
	}

	if rev && c.Revision != "" {
		fmt.Fprintf(&versionString, " (%s)", c.Revision)
	}

	return versionString.String()
}
<|endoftext|>


<commit_before>package main

import (
	"encoding/json"
	"errors"
	"io"
)

var InvalidCredentials = errors.New("Invalid credentials provided. Must have a username/ password or none at all.")

type Credential struct {
	Username string `json:"username"`
	Password string `json:"password"`
}

type Config struct {
	Credentials       []Credential `json:"credentials"`
	StripProxyHeaders bool         `json:"strip_proxy_headers"`
	Port              int
}

func (config *Config) AuthenticationRequired() bool {
	return len(config.Credentials) > 0
}

func validCredentials(username, password string) bool {
	if username == "" && password == "" {
		return true
	}
	if username != "" && password != "" {
		return true
	}
	return false
}

func (config *Config) Validate() error {
	for _, credential := range config.Credentials {
		if !validCredentials(credential.Username, credential.Password) {
			return InvalidCredentials
		}
	}

	return nil
}

func (config *Config) IsAuthenticated(username, password string) bool {
	for _, credential := range config.Credentials {
		if credential.Username == username && credential.Password == password {
			return true
		}
	}

	return false
}

func NewConfigFromReader(reader io.Reader) (*Config, error) {
	config := new(Config)
	decoder := json.NewDecoder(reader)
	err := decoder.Decode(&config)
	if err != nil {
		return nil, err
	}
	return config, nil
}
<commit_msg>We only need the strip proxy headers json annotation<commit_after>package main

import (
	"encoding/json"
	"errors"
	"io"
)

var InvalidCredentials = errors.New("Invalid credentials provided. Must have a username/ password or none at all.")

type Credential struct {
	Username string
	Password string
}

type Config struct {
	Credentials       []Credential
	StripProxyHeaders bool `json:"strip_proxy_headers"`
	Port              int
}

func (config *Config) AuthenticationRequired() bool {
	return len(config.Credentials) > 0
}

func validCredentials(username, password string) bool {
	if username == "" && password == "" {
		return true
	}
	if username != "" && password != "" {
		return true
	}
	return false
}

func (config *Config) Validate() error {
	for _, credential := range config.Credentials {
		if !validCredentials(credential.Username, credential.Password) {
			return InvalidCredentials
		}
	}

	return nil
}

func (config *Config) IsAuthenticated(username, password string) bool {
	for _, credential := range config.Credentials {
		if credential.Username == username && credential.Password == password {
			return true
		}
	}

	return false
}

func NewConfigFromReader(reader io.Reader) (*Config, error) {
	config := new(Config)
	decoder := json.NewDecoder(reader)
	err := decoder.Decode(&config)
	if err != nil {
		return nil, err
	}
	return config, nil
}
<|endoftext|>


<commit_before>package version

import "fmt"

const (
	// VersionMajor is for an API incompatible changes
	VersionMajor = 5
	// VersionMinor is for functionality in a backwards-compatible manner
	VersionMinor = 13
	// VersionPatch is for backwards-compatible bug fixes
	VersionPatch = 2

	// VersionDev indicates development branch. Releases will be empty string.
	VersionDev = "-dev"
)

// Version is the specification version that the package types support.
var Version = fmt.Sprintf("%d.%d.%d%s", VersionMajor, VersionMinor, VersionPatch, VersionDev)
<commit_msg>Bump to v5.13.2<commit_after>package version

import "fmt"

const (
	// VersionMajor is for an API incompatible changes
	VersionMajor = 5
	// VersionMinor is for functionality in a backwards-compatible manner
	VersionMinor = 13
	// VersionPatch is for backwards-compatible bug fixes
	VersionPatch = 2

	// VersionDev indicates development branch. Releases will be empty string.
	VersionDev = ""
)

// Version is the specification version that the package types support.
var Version = fmt.Sprintf("%d.%d.%d%s", VersionMajor, VersionMinor, VersionPatch, VersionDev)
<|endoftext|>


<commit_before>package vals

import (
	"fmt"
	"reflect"

	"github.com/elves/elvish/pkg/parse"
	"github.com/elves/elvish/pkg/util"
)

// NoPretty can be passed to Repr to suppress pretty-printing.
const NoPretty = util.MinInt

// Reprer wraps the Repr method.
type Reprer interface {
	// Repr returns a string that represents a Value. The string either be a
	// literal of that Value that is preferably deep-equal to it (like `[a b c]`
	// for a list), or a string enclosed in "<>" containing the kind and
	// identity of the Value(like `<fn 0xdeadcafe>`).
	//
	// If indent is at least 0, it should be pretty-printed with the current
	// indentation level of indent; the indent of the first line has already
	// been written and shall not be written in Repr. The returned string
	// should never contain a trailing newline.
	Repr(indent int) string
}

// Repr returns the representation for a value, a string that is preferably (but
// not necessarily) an Elvish expression that evaluates to the argument. If
// indent >= 0, the representation is pretty-printed. It is implemented for the
// builtin types nil, bool and string, the File, List and Map types, StructMap
// types, and types satisfying the Reprer interface. For other types, it uses
// fmt.Sprint with the format "<unknown %v>".
func Repr(v interface{}, indent int) string {
	switch v := v.(type) {
	case nil:
		return "$nil"
	case bool:
		if v {
			return "$true"
		}
		return "$false"
	case string:
		return parse.Quote(v)
	case float64:
		return "(float64 " + formatFloat64(v) + ")"
	case Reprer:
		return v.Repr(indent)
	case File:
		return fmt.Sprintf("<file{%s %d}>", parse.Quote(v.Name()), v.Fd())
	case List:
		b := NewListReprBuilder(indent)
		for it := v.Iterator(); it.HasElem(); it.Next() {
			b.WriteElem(Repr(it.Elem(), indent+1))
		}
		return b.String()
	case Map:
		builder := NewMapReprBuilder(indent)
		for it := v.Iterator(); it.HasElem(); it.Next() {
			k, v := it.Elem()
			builder.WritePair(Repr(k, indent+1), indent+2, Repr(v, indent+2))
		}
		return builder.String()
	case StructMap:
		return reprStructMap(v, indent)
	case PseudoStructMap:
		return reprStructMap(v.Fields(), indent)
	default:
		return fmt.Sprintf("<unknown %v>", v)
	}
}

func reprStructMap(v StructMap, indent int) string {
	vValue := reflect.ValueOf(v)
	vType := vValue.Type()
	builder := NewMapReprBuilder(indent)
	it := iterateStructMap(vType)
	for it.Next() {
		k, v := it.Get(vValue)
		builder.WritePair(Repr(k, indent+1), indent+2, Repr(v, indent+2))
	}
	return builder.String()
}
<commit_msg>pkg/eval/vals: Simply use math.MinInt32 for NoPretty.<commit_after>package vals

import (
	"fmt"
	"math"
	"reflect"

	"github.com/elves/elvish/pkg/parse"
)

// NoPretty can be passed to Repr to suppress pretty-printing.
const NoPretty = math.MinInt32

// Reprer wraps the Repr method.
type Reprer interface {
	// Repr returns a string that represents a Value. The string either be a
	// literal of that Value that is preferably deep-equal to it (like `[a b c]`
	// for a list), or a string enclosed in "<>" containing the kind and
	// identity of the Value(like `<fn 0xdeadcafe>`).
	//
	// If indent is at least 0, it should be pretty-printed with the current
	// indentation level of indent; the indent of the first line has already
	// been written and shall not be written in Repr. The returned string
	// should never contain a trailing newline.
	Repr(indent int) string
}

// Repr returns the representation for a value, a string that is preferably (but
// not necessarily) an Elvish expression that evaluates to the argument. If
// indent >= 0, the representation is pretty-printed. It is implemented for the
// builtin types nil, bool and string, the File, List and Map types, StructMap
// types, and types satisfying the Reprer interface. For other types, it uses
// fmt.Sprint with the format "<unknown %v>".
func Repr(v interface{}, indent int) string {
	switch v := v.(type) {
	case nil:
		return "$nil"
	case bool:
		if v {
			return "$true"
		}
		return "$false"
	case string:
		return parse.Quote(v)
	case float64:
		return "(float64 " + formatFloat64(v) + ")"
	case Reprer:
		return v.Repr(indent)
	case File:
		return fmt.Sprintf("<file{%s %d}>", parse.Quote(v.Name()), v.Fd())
	case List:
		b := NewListReprBuilder(indent)
		for it := v.Iterator(); it.HasElem(); it.Next() {
			b.WriteElem(Repr(it.Elem(), indent+1))
		}
		return b.String()
	case Map:
		builder := NewMapReprBuilder(indent)
		for it := v.Iterator(); it.HasElem(); it.Next() {
			k, v := it.Elem()
			builder.WritePair(Repr(k, indent+1), indent+2, Repr(v, indent+2))
		}
		return builder.String()
	case StructMap:
		return reprStructMap(v, indent)
	case PseudoStructMap:
		return reprStructMap(v.Fields(), indent)
	default:
		return fmt.Sprintf("<unknown %v>", v)
	}
}

func reprStructMap(v StructMap, indent int) string {
	vValue := reflect.ValueOf(v)
	vType := vValue.Type()
	builder := NewMapReprBuilder(indent)
	it := iterateStructMap(vType)
	for it.Next() {
		k, v := it.Get(vValue)
		builder.WritePair(Repr(k, indent+1), indent+2, Repr(v, indent+2))
	}
	return builder.String()
}
<|endoftext|>


<commit_before>package version

// VERSION ...
const VERSION = "1.1.68"
<commit_msg>v1.1.69 [skip ci]<commit_after>package version

// VERSION ...
const VERSION = "1.1.69"
<|endoftext|>


<commit_before>package glob

import (
	"os"
	"reflect"
	"sort"
	"strings"
	"testing"

	"src.elv.sh/pkg/testutil"
)

var (
	mkdirs = []string{"a", "b", "c", "d1", "d1/e", "d1/e/f", "d1/e/f/g",
		"d2", "d2/e", "d2/e/f", "d2/e/f/g"}
	mkdirDots = []string{".el"}
	creates   = []string{"a/X", "a/Y", "b/X", "c/Y",
		"dX", "dXY",
		"lorem", "ipsum",
		"d1/e/f/g/X", "d2/e/f/g/X"}
	createDots = []string{".x", ".el/x"}
)

type globCase struct {
	pattern string
	want    []string
}

var globCases = []globCase{
	{"*", []string{"a", "b", "c", "d1", "d2", "dX", "dXY", "lorem", "ipsum"}},
	{".", []string{"."}},
	{"./*", []string{"./a", "./b", "./c", "./d1", "./d2", "./dX", "./dXY", "./lorem", "./ipsum"}},
	{"..", []string{".."}},
	{"a/..", []string{"a/.."}},
	{"a/../*", []string{"a/../a", "a/../b", "a/../c", "a/../d1", "a/../d2", "a/../dX", "a/../dXY", "a/../lorem", "a/../ipsum"}},
	{"*/", []string{"a/", "b/", "c/", "d1/", "d2/"}},
	{"**", append(mkdirs, creates...)},
	{"*/X", []string{"a/X", "b/X"}},
	{"**X", []string{"a/X", "b/X", "dX", "d1/e/f/g/X", "d2/e/f/g/X"}},
	{"*/*/*", []string{"d1/e/f", "d2/e/f"}},
	{"l*m", []string{"lorem"}},
	{"d*", []string{"d1", "d2", "dX", "dXY"}},
	{"d*/", []string{"d1/", "d2/"}},
	{"d**", []string{"d1", "d1/e", "d1/e/f", "d1/e/f/g", "d1/e/f/g/X",
		"d2", "d2/e", "d2/e/f", "d2/e/f/g", "d2/e/f/g/X", "dX", "dXY"}},
	{"?", []string{"a", "b", "c"}},
	{"??", []string{"d1", "d2", "dX"}},

	// Nonexistent paths.
	{"xxxx", []string{}},
	{"xxxx/*", []string{}},
	{"a/*/", []string{}},

	// TODO Test cases against dotfiles.
}

func TestGlob_Relative(t *testing.T) {
	testGlob(t, false)
}

func TestGlob_Absolute(t *testing.T) {
	testGlob(t, true)
}

func testGlob(t *testing.T, abs bool) {
	dir, cleanup := testutil.InTestDir()
	dir = strings.ReplaceAll(dir, string(os.PathSeparator), "/")
	defer cleanup()

	for _, dir := range append(mkdirs, mkdirDots...) {
		err := os.Mkdir(dir, 0755)
		if err != nil {
			panic(err)
		}
	}
	for _, file := range append(creates, createDots...) {
		f, err := os.Create(file)
		if err != nil {
			panic(err)
		}
		f.Close()
	}
	for _, tc := range globCases {
		pattern := tc.pattern
		if abs {
			pattern = dir + "/" + pattern
		}
		wantResults := make([]string, len(tc.want))
		for i, result := range tc.want {
			if abs {
				wantResults[i] = dir + "/" + result
			} else {
				wantResults[i] = result
			}
		}
		sort.Strings(wantResults)

		results := globPaths(pattern)

		if !reflect.DeepEqual(results, wantResults) {
			t.Errorf(`Glob(%q) => %v, want %v`, pattern, results, wantResults)
		}
	}
}

// Regression test for b.elv.sh/1220
func TestGlob_InvalidUTF8InFilename(t *testing.T) {
	_, cleanup := testutil.InTestDir()
	defer cleanup()

	name := string([]byte{255}) + ".txt"
	f, err := os.Create(name)
	if err != nil {
		t.Skip("create: ", err)
	}
	f.Close()

	paths := globPaths("*.txt")
	wantPaths := []string{name}
	if !reflect.DeepEqual(paths, wantPaths) {
		t.Errorf("got %v, want %v", paths, wantPaths)
	}
}

func globPaths(pattern string) []string {
	paths := []string{}
	Glob(pattern, func(pathInfo PathInfo) bool {
		paths = append(paths, pathInfo.Path)
		return true
	})
	sort.Strings(paths)
	return paths
}
<commit_msg>Make test case crashing too.<commit_after>package glob

import (
	"os"
	"reflect"
	"sort"
	"strings"
	"testing"

	"src.elv.sh/pkg/testutil"
)

var (
	mkdirs = []string{"a", "b", "c", "d1", "d1/e", "d1/e/f", "d1/e/f/g",
		"d2", "d2/e", "d2/e/f", "d2/e/f/g"}
	mkdirDots = []string{".el"}
	creates   = []string{"a/X", "a/Y", "b/X", "c/Y",
		"dX", "dXY",
		"lorem", "ipsum",
		"d1/e/f/g/X", "d2/e/f/g/X"}
	createDots = []string{".x", ".el/x"}
)

type globCase struct {
	pattern string
	want    []string
}

var globCases = []globCase{
	{"*", []string{"a", "b", "c", "d1", "d2", "dX", "dXY", "lorem", "ipsum"}},
	{".", []string{"."}},
	{"./*", []string{"./a", "./b", "./c", "./d1", "./d2", "./dX", "./dXY", "./lorem", "./ipsum"}},
	{"..", []string{".."}},
	{"a/..", []string{"a/.."}},
	{"a/../*", []string{"a/../a", "a/../b", "a/../c", "a/../d1", "a/../d2", "a/../dX", "a/../dXY", "a/../lorem", "a/../ipsum"}},
	{"*/", []string{"a/", "b/", "c/", "d1/", "d2/"}},
	{"**", append(mkdirs, creates...)},
	{"*/X", []string{"a/X", "b/X"}},
	{"**X", []string{"a/X", "b/X", "dX", "d1/e/f/g/X", "d2/e/f/g/X"}},
	{"*/*/*", []string{"d1/e/f", "d2/e/f"}},
	{"l*m", []string{"lorem"}},
	{"d*", []string{"d1", "d2", "dX", "dXY"}},
	{"d*/", []string{"d1/", "d2/"}},
	{"d**", []string{"d1", "d1/e", "d1/e/f", "d1/e/f/g", "d1/e/f/g/X",
		"d2", "d2/e", "d2/e/f", "d2/e/f/g", "d2/e/f/g/X", "dX", "dXY"}},
	{"?", []string{"a", "b", "c"}},
	{"??", []string{"d1", "d2", "dX"}},

	// Nonexistent paths.
	{"xxxx", []string{}},
	{"xxxx/*", []string{}},
	{"a/*/", []string{}},

	// TODO Test cases against dotfiles.
}

func TestGlob_Relative(t *testing.T) {
	testGlob(t, false)
}

func TestGlob_Absolute(t *testing.T) {
	testGlob(t, true)
}

func testGlob(t *testing.T, abs bool) {
	dir, cleanup := testutil.InTestDir()
	dir = strings.ReplaceAll(dir, string(os.PathSeparator), "/")
	defer cleanup()

	for _, dir := range append(mkdirs, mkdirDots...) {
		err := os.Mkdir(dir, 0755)
		if err != nil {
			panic(err)
		}
	}
	for _, file := range append(creates, createDots...) {
		f, err := os.Create(file)
		if err != nil {
			panic(err)
		}
		f.Close()
	}
	for _, tc := range globCases {
		pattern := tc.pattern
		if abs {
			pattern = dir + "/" + pattern
		}
		wantResults := make([]string, len(tc.want))
		for i, result := range tc.want {
			if abs {
				wantResults[i] = dir + "/" + result
			} else {
				wantResults[i] = result
			}
		}
		sort.Strings(wantResults)

		results := globPaths(pattern)

		if !reflect.DeepEqual(results, wantResults) {
			t.Errorf(`Glob(%q) => %v, want %v`, pattern, results, wantResults)
		}
	}
}

// Regression test for b.elv.sh/1220
func TestGlob_InvalidUTF8InFilename(t *testing.T) {
	_, cleanup := testutil.InTestDir()
	defer cleanup()

	name := string([]byte{255}) + ".c"
	f, err := os.Create(name)
	if err != nil {
		t.Skip("create: ", err)
	}
	f.Close()

	paths := globPaths("*.c")
	wantPaths := []string{name}
	if !reflect.DeepEqual(paths, wantPaths) {
		t.Errorf("got %v, want %v", paths, wantPaths)
	}
}

func globPaths(pattern string) []string {
	paths := []string{}
	Glob(pattern, func(pathInfo PathInfo) bool {
		paths = append(paths, pathInfo.Path)
		return true
	})
	sort.Strings(paths)
	return paths
}
<|endoftext|>


<commit_before>package english

import (
	"github.com/kljensen/snowball/snowballword"
)

// Step 2 is the stemming of various endings found in
// R1 including "al", "ness", and "li".
//
func step2(w *snowballword.SnowballWord) bool {

	// Possible sufficies for this step, longest first.
	suffix, suffixRunes := w.FirstSuffix(
		"ational", "fulness", "iveness", "ization", "ousness",
		"biliti", "lessli", "tional", "alism", "aliti", "ation",
		"entli", "fulli", "iviti", "ousli", "anci", "abli",
		"alli", "ator", "enci", "izer", "bli", "ogi", "li",
	)

	// If it is not in R1, do nothing
	if suffix == "" || len(suffixRunes) > len(w.RS)-w.R1start {
		return false
	}

	// Handle special cases where we're not just going to
	// replace the suffix with another suffix: there are
	// other things we need to do.
	//
	switch suffix {

	case "li":

		// Delete if preceded by a valid li-ending. Valid li-endings inlude the
		// following charaters: cdeghkmnrt. (Note, the unicode code points for
		// these characters are, repectively, as follows:
		// 99 100 101 103 104 107 109 110 114 116)
		//
		rsLen := len(w.RS)
		if rsLen >= 3 {
			switch w.RS[rsLen-3] {
			case 99, 100, 101, 103, 104, 107, 109, 110, 114, 116:
				w.RemoveLastNRunes(len(suffixRunes))
				return true
			}
		}
		return false

	case "ogi":

		// Replace by og if preceded by l.
		// (Note, the unicode code point for l is 108)
		//
		rsLen := len(w.RS)
		if rsLen >= 4 && w.RS[rsLen-4] == 108 {
			w.ReplaceSuffixRunes(suffixRunes, []rune("og"), true)
		}
		return true
	}

	// Handle a suffix that was found, which is going
	// to be replaced with a different suffix.
	//
	var repl string
	switch suffix {
	case "tional":
		repl = "tion"
	case "enci":
		repl = "ence"
	case "anci":
		repl = "ance"
	case "abli":
		repl = "able"
	case "entli":
		repl = "ent"
	case "izer", "ization":
		repl = "ize"
	case "ational", "ation", "ator":
		repl = "ate"
	case "alism", "aliti", "alli":
		repl = "al"
	case "fulness":
		repl = "ful"
	case "ousli", "ousness":
		repl = "ous"
	case "iveness", "iviti":
		repl = "ive"
	case "biliti", "bli":
		repl = "ble"
	case "fulli":
		repl = "ful"
	case "lessli":
		repl = "less"
	}
	w.ReplaceSuffixRunes(suffixRunes, []rune(repl), true)
	return true

}
<commit_msg>fix typo<commit_after>package english

import (
	"github.com/kljensen/snowball/snowballword"
)

// Step 2 is the stemming of various endings found in
// R1 including "al", "ness", and "li".
//
func step2(w *snowballword.SnowballWord) bool {

	// Possible sufficies for this step, longest first.
	suffix, suffixRunes := w.FirstSuffix(
		"ational", "fulness", "iveness", "ization", "ousness",
		"biliti", "lessli", "tional", "alism", "aliti", "ation",
		"entli", "fulli", "iviti", "ousli", "anci", "abli",
		"alli", "ator", "enci", "izer", "bli", "ogi", "li",
	)

	// If it is not in R1, do nothing
	if suffix == "" || len(suffixRunes) > len(w.RS)-w.R1start {
		return false
	}

	// Handle special cases where we're not just going to
	// replace the suffix with another suffix: there are
	// other things we need to do.
	//
	switch suffix {

	case "li":

		// Delete if preceded by a valid li-ending. Valid li-endings inlude the
		// following charaters: cdeghkmnrt. (Note, the unicode code points for
		// these characters are, respectively, as follows:
		// 99 100 101 103 104 107 109 110 114 116)
		//
		rsLen := len(w.RS)
		if rsLen >= 3 {
			switch w.RS[rsLen-3] {
			case 99, 100, 101, 103, 104, 107, 109, 110, 114, 116:
				w.RemoveLastNRunes(len(suffixRunes))
				return true
			}
		}
		return false

	case "ogi":

		// Replace by og if preceded by l.
		// (Note, the unicode code point for l is 108)
		//
		rsLen := len(w.RS)
		if rsLen >= 4 && w.RS[rsLen-4] == 108 {
			w.ReplaceSuffixRunes(suffixRunes, []rune("og"), true)
		}
		return true
	}

	// Handle a suffix that was found, which is going
	// to be replaced with a different suffix.
	//
	var repl string
	switch suffix {
	case "tional":
		repl = "tion"
	case "enci":
		repl = "ence"
	case "anci":
		repl = "ance"
	case "abli":
		repl = "able"
	case "entli":
		repl = "ent"
	case "izer", "ization":
		repl = "ize"
	case "ational", "ation", "ator":
		repl = "ate"
	case "alism", "aliti", "alli":
		repl = "al"
	case "fulness":
		repl = "ful"
	case "ousli", "ousness":
		repl = "ous"
	case "iveness", "iviti":
		repl = "ive"
	case "biliti", "bli":
		repl = "ble"
	case "fulli":
		repl = "ful"
	case "lessli":
		repl = "less"
	}
	w.ReplaceSuffixRunes(suffixRunes, []rune(repl), true)
	return true

}
<|endoftext|>


<commit_before>// Copyright 2015 The Prometheus Authors
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package main

import (
	"bytes"
	"fmt"
	"io/ioutil"
	"os"
	"path/filepath"
	"strings"
	"text/template"

	"gopkg.in/yaml.v2"

	"github.com/prometheus/prometheus/config"
	"github.com/prometheus/prometheus/promql"
	"github.com/prometheus/prometheus/util/cli"
	"github.com/prometheus/prometheus/version"
)

// CheckConfigCmd validates configuration files.
func CheckConfigCmd(t cli.Term, args ...string) int {
	if len(args) == 0 {
		t.Infof("usage: promtool check-config <files>")
		return 2
	}
	failed := false

	for _, arg := range args {
		ruleFiles, err := checkConfig(t, arg)
		if err != nil {
			t.Errorf("  FAILED: %s", err)
			failed = true
		} else {
			t.Infof("  SUCCESS: %d rule files found", len(ruleFiles))
		}
		t.Infof("")

		for _, rf := range ruleFiles {
			if n, err := checkRules(t, rf); err != nil {
				t.Errorf("  FAILED: %s", err)
				failed = true
			} else {
				t.Infof("  SUCCESS: %d rules found", n)
			}
			t.Infof("")
		}
	}
	if failed {
		return 1
	}
	return 0
}

func checkConfig(t cli.Term, filename string) ([]string, error) {
	t.Infof("Checking %s", filename)

	if stat, err := os.Stat(filename); err != nil {
		return nil, fmt.Errorf("cannot get file info")
	} else if stat.IsDir() {
		return nil, fmt.Errorf("is a directory")
	}

	content, err := ioutil.ReadFile(filename)
	if err != nil {
		return nil, err
	}

	var cfg config.Config
	if err := yaml.Unmarshal(content, &cfg); err != nil {
		return nil, err
	}
	var ruleFiles []string
	for _, rf := range cfg.RuleFiles {
		rfs, err := filepath.Glob(rf)
		if err != nil {
			return nil, err
		}
		// If an explicit file was given, error if it doesn't exist.
		if !strings.Contains(rf, "*") && len(rfs) == 0 {
			return nil, fmt.Errorf("%q does not point to an existing file", rf)
		}
		ruleFiles = append(ruleFiles, rfs...)
	}

	return ruleFiles, nil
}

// CheckRulesCmd validates rule files.
func CheckRulesCmd(t cli.Term, args ...string) int {
	if len(args) == 0 {
		t.Infof("usage: promtool check-rules <files>")
		return 2
	}
	failed := false

	for _, arg := range args {
		if n, err := checkRules(t, arg); err != nil {
			t.Errorf("  FAILED: %s", err)
			failed = true
		} else {
			t.Infof("  SUCCESS: %d rules found", n)
		}
		t.Infof("")
	}
	if failed {
		return 1
	}
	return 0
}

func checkRules(t cli.Term, filename string) (int, error) {
	t.Infof("Checking %s", filename)

	if stat, err := os.Stat(filename); err != nil {
		return 0, fmt.Errorf("cannot get file info")
	} else if stat.IsDir() {
		return 0, fmt.Errorf("is a directory")
	}

	content, err := ioutil.ReadFile(filename)
	if err != nil {
		return 0, err
	}

	rules, err := promql.ParseStmts(string(content))
	if err != nil {
		return 0, err
	}
	return len(rules), nil
}

var versionInfoTmpl = `
prometheus, version {{.version}} (branch: {{.branch}}, revision: {{.revision}})
  build user:       {{.buildUser}}
  build date:       {{.buildDate}}
  go version:       {{.goVersion}}
`

// VersionCmd prints the binaries version information.
func VersionCmd(t cli.Term, _ ...string) int {
	tmpl := template.Must(template.New("version").Parse(versionInfoTmpl))

	var buf bytes.Buffer
	if err := tmpl.ExecuteTemplate(&buf, "version", version.Map); err != nil {
		panic(err)
	}
	t.Out(strings.TrimSpace(buf.String()))
	return 0
}

func main() {
	app := cli.NewApp("promtool")

	app.Register("check-config", &cli.Command{
		Desc: "validate configuration files for correctness",
		Run:  CheckConfigCmd,
	})

	app.Register("check-rules", &cli.Command{
		Desc: "validate rule files for correctness",
		Run:  CheckRulesCmd,
	})

	app.Register("version", &cli.Command{
		Desc: "print the version of this binary",
		Run:  VersionCmd,
	})

	t := cli.BasicTerm(os.Stdout, os.Stderr)
	os.Exit(app.Run(t, os.Args[1:]...))
}
<commit_msg>cmd/promtool: resolve rule files relative to config file<commit_after>// Copyright 2015 The Prometheus Authors
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package main

import (
	"bytes"
	"fmt"
	"io/ioutil"
	"os"
	"path/filepath"
	"strings"
	"text/template"

	"gopkg.in/yaml.v2"

	"github.com/prometheus/prometheus/config"
	"github.com/prometheus/prometheus/promql"
	"github.com/prometheus/prometheus/util/cli"
	"github.com/prometheus/prometheus/version"
)

// CheckConfigCmd validates configuration files.
func CheckConfigCmd(t cli.Term, args ...string) int {
	if len(args) == 0 {
		t.Infof("usage: promtool check-config <files>")
		return 2
	}
	failed := false

	for _, arg := range args {
		ruleFiles, err := checkConfig(t, arg)
		if err != nil {
			t.Errorf("  FAILED: %s", err)
			failed = true
		} else {
			t.Infof("  SUCCESS: %d rule files found", len(ruleFiles))
		}
		t.Infof("")

		for _, rf := range ruleFiles {
			if n, err := checkRules(t, rf); err != nil {
				t.Errorf("  FAILED: %s", err)
				failed = true
			} else {
				t.Infof("  SUCCESS: %d rules found", n)
			}
			t.Infof("")
		}
	}
	if failed {
		return 1
	}
	return 0
}

func checkConfig(t cli.Term, filename string) ([]string, error) {
	t.Infof("Checking %s", filename)

	if stat, err := os.Stat(filename); err != nil {
		return nil, fmt.Errorf("cannot get file info")
	} else if stat.IsDir() {
		return nil, fmt.Errorf("is a directory")
	}

	content, err := ioutil.ReadFile(filename)
	if err != nil {
		return nil, err
	}

	var cfg config.Config
	if err := yaml.Unmarshal(content, &cfg); err != nil {
		return nil, err
	}
	var ruleFiles []string
	for _, rf := range cfg.RuleFiles {
		if !filepath.IsAbs(rf) {
			rf = filepath.Join(filepath.Dir(filename), rf)
		}

		rfs, err := filepath.Glob(rf)
		if err != nil {
			return nil, err
		}
		// If an explicit file was given, error if it doesn't exist.
		if !strings.Contains(rf, "*") && len(rfs) == 0 {
			return nil, fmt.Errorf("%q does not point to an existing file", rf)
		}
		ruleFiles = append(ruleFiles, rfs...)
	}

	return ruleFiles, nil
}

// CheckRulesCmd validates rule files.
func CheckRulesCmd(t cli.Term, args ...string) int {
	if len(args) == 0 {
		t.Infof("usage: promtool check-rules <files>")
		return 2
	}
	failed := false

	for _, arg := range args {
		if n, err := checkRules(t, arg); err != nil {
			t.Errorf("  FAILED: %s", err)
			failed = true
		} else {
			t.Infof("  SUCCESS: %d rules found", n)
		}
		t.Infof("")
	}
	if failed {
		return 1
	}
	return 0
}

func checkRules(t cli.Term, filename string) (int, error) {
	t.Infof("Checking %s", filename)

	if stat, err := os.Stat(filename); err != nil {
		return 0, fmt.Errorf("cannot get file info")
	} else if stat.IsDir() {
		return 0, fmt.Errorf("is a directory")
	}

	content, err := ioutil.ReadFile(filename)
	if err != nil {
		return 0, err
	}

	rules, err := promql.ParseStmts(string(content))
	if err != nil {
		return 0, err
	}
	return len(rules), nil
}

var versionInfoTmpl = `
prometheus, version {{.version}} (branch: {{.branch}}, revision: {{.revision}})
  build user:       {{.buildUser}}
  build date:       {{.buildDate}}
  go version:       {{.goVersion}}
`

// VersionCmd prints the binaries version information.
func VersionCmd(t cli.Term, _ ...string) int {
	tmpl := template.Must(template.New("version").Parse(versionInfoTmpl))

	var buf bytes.Buffer
	if err := tmpl.ExecuteTemplate(&buf, "version", version.Map); err != nil {
		panic(err)
	}
	t.Out(strings.TrimSpace(buf.String()))
	return 0
}

func main() {
	app := cli.NewApp("promtool")

	app.Register("check-config", &cli.Command{
		Desc: "validate configuration files for correctness",
		Run:  CheckConfigCmd,
	})

	app.Register("check-rules", &cli.Command{
		Desc: "validate rule files for correctness",
		Run:  CheckRulesCmd,
	})

	app.Register("version", &cli.Command{
		Desc: "print the version of this binary",
		Run:  VersionCmd,
	})

	t := cli.BasicTerm(os.Stdout, os.Stderr)
	os.Exit(app.Run(t, os.Args[1:]...))
}
<|endoftext|>


<commit_before>package forms

import (
	"github.com/kirves/go-form-it/common"
	"github.com/kirves/go-form-it/fields"
	"testing"
	"time"
)

const (
	style = formcommon.BOOTSTRAP
)

var (
	txt, psw, btn fields.FieldInterface
)

func TestFieldRender(t *testing.T) {
	field := fields.TextField("test")
	field.AddClass("test").AddClass("class").SetId("testId").SetParam("param1", "val1").AddCss("css1", "val1").SetStyle(formcommon.BASE).Disabled()
	field.AddLabelClass("LABEL")
	field.SetLabel("This is a label")
	t.Log("Rendered field:", field.Render())
	txt = field
}

func TestPasswordRender(t *testing.T) {
	field := fields.PasswordField("test")
	field.AddClass("test")
	field.AddClass("class")
	field.SetId("testId")
	field.SetStyle(style)
	t.Log("Rendered field:", field.Render())
	psw = field
}

func TestButtonRender(t *testing.T) {
	field := fields.SubmitButton("btn", "Click me!")
	field.SetStyle(style)
	t.Log("Rendered button:", field.Render())
	btn = field
}

func TestRadioButtonRender(t *testing.T) {
	field := fields.RadioField("radio", []fields.InputChoice{
		fields.InputChoice{Id: "choice1", Val: "value1"},
		fields.InputChoice{Id: "choice2", Val: "value2"},
	})
	field.SetStyle(style)
	t.Log("Rendered radio:", field.Render())
}

func TestSelectRender(t *testing.T) {
	field := fields.SelectField("select", map[string][]fields.InputChoice{
		"": []fields.InputChoice{
			fields.InputChoice{"choice1", "value1"},
			fields.InputChoice{"choice2", "value2"},
		},
	})
	field.SetStyle(style)
	t.Log("Rendered select:", field.Render())
}

func TestHiddenRender(t *testing.T) {
	field := fields.HiddenField("hidden")
	field.SetStyle(style)
	t.Log("Rendered hidden:", field.Render())
}

func TestNumberRender(t *testing.T) {
	field := fields.NumberField("number")
	field.SetStyle(style)
	t.Log("Rendered number:", field.Render())
}

func TestFormRender(t *testing.T) {
	form := BootstrapForm(POST, "")
	form.Elements(&FieldSetType{}, txt, psw, btn)
	// form.AddField(psw)
	// form.AddField(btn)
	t.Log("Rendered form:", form.Render())
}

func TestFormFromSimpleModel(t *testing.T) {
	type User struct {
		Username  string
		Password1 string `form_widget:"password" form_label:"Password 1"`
		Password2 string `form_widget:"password" form_label:"Password 2"`
		SkipThis  int    `form_options:"skip"`
	}

	u := User{}

	form := BaseFormFromModel(u, POST, "/action.html")
	t.Log("Rendered form:", form.Render())
}

func TestFormFromModel(t *testing.T) {
	type Model struct {
		User      string    `form_label:"User label test"`
		password  string    `form_widget:"password"`
		Id        int       `form_min:"0" form_max:"5"`
		Ts        time.Time `form_min:"2013-04-22T15:00"`
		RadioTest string    `form_widget:"select" form_choices:"|A|Option A|G1|B|Option B" form_value:"A"`
		BoolTest  bool      //`form_options:"checked"`
	}

	form := BaseFormFromModel(Model{"asd", "lol", 20, time.Now(), "B", false}, POST, "")
	t.Log("Rendered form:", form.Render())
}

func TestBSFormFromModel(t *testing.T) {
	type Model struct {
		User      string    `form_label:"User label test"`
		password  string    `form_widget:"password"`
		Id        int       `form_min:"0" form_max:"5"`
		Ts        time.Time `form_min:"2013-04-22T15:00"`
		RadioTest string    `form_widget:"select" form_choices:"|A|Option A|G1|B|Option B" form_value:"A"`
		BoolTest  bool      //`form_options:"checked"`
	}

	form := BootstrapFormFromModel(Model{"asd", "lol", 20, time.Now(), "B", false}, POST, "")
	t.Log("Rendered form:", form.Render())
}

func TestInlineCreation(t *testing.T) {
	form := BaseForm(POST, "/action.html").Elements(
		fields.TextField("text_field").SetLabel("Username"),
		FieldSet("psw_fieldset",
			fields.PasswordField("psw1").AddClass("password_class").SetLabel("Password 1"),
			fields.PasswordField("psw2").AddClass("password_class").SetLabel("Password 2"),
		),
		fields.SubmitButton("btn1", "Submit"),
	)
	t.Log("Rendered form:", form.Render())
}
<commit_msg>More tests<commit_after>package forms

import (
	"github.com/kirves/go-form-it/common"
	"github.com/kirves/go-form-it/fields"
	"testing"
	"time"
)

const (
	style = formcommon.BOOTSTRAP
)

var (
	txt, psw, btn fields.FieldInterface
)

func TestFieldRender(t *testing.T) {
	field := fields.TextField("test")
	field.AddClass("test").AddClass("class").SetId("testId").SetParam("param1", "val1").AddCss("css1", "val1").SetStyle(style).Disabled()
	field.AddLabelClass("LABEL")
	field.SetLabel("This is a label")
	field.AddError("ERROR")
	t.Log("Rendered field:", field.Render())
	txt = field
}

func TestPasswordRender(t *testing.T) {
	field := fields.PasswordField("test")
	field.AddClass("test")
	field.AddClass("class")
	field.SetId("testId")
	field.SetStyle(style)
	t.Log("Rendered field:", field.Render())
	psw = field
}

func TestButtonRender(t *testing.T) {
	field := fields.SubmitButton("btn", "Click me!")
	field.SetStyle(style)
	t.Log("Rendered button:", field.Render())
	btn = field
}

func TestRadioButtonRender(t *testing.T) {
	field := fields.RadioField("radio", []fields.InputChoice{
		fields.InputChoice{Id: "choice1", Val: "value1"},
		fields.InputChoice{Id: "choice2", Val: "value2"},
	})
	field.SetStyle(style)
	t.Log("Rendered radio:", field.Render())
}

func TestSelectRender(t *testing.T) {
	field := fields.SelectField("select", map[string][]fields.InputChoice{
		"": []fields.InputChoice{
			fields.InputChoice{"choice1", "value1"},
			fields.InputChoice{"choice2", "value2"},
		},
	})
	field.SetStyle(style)
	t.Log("Rendered select:", field.Render())
}

func TestHiddenRender(t *testing.T) {
	field := fields.HiddenField("hidden")
	field.SetStyle(style)
	t.Log("Rendered hidden:", field.Render())
}

func TestNumberRender(t *testing.T) {
	field := fields.NumberField("number")
	field.SetStyle(style)
	t.Log("Rendered number:", field.Render())
}

func TestFormRender(t *testing.T) {
	form := BootstrapForm(POST, "")
	form.Elements(&FieldSetType{}, txt, psw, btn)
	// form.AddField(psw)
	// form.AddField(btn)
	t.Log("Rendered form:", form.Render())
}

func TestFormFromSimpleModel(t *testing.T) {
	type User struct {
		Username  string
		Password1 string `form_widget:"password" form_label:"Password 1"`
		Password2 string `form_widget:"password" form_label:"Password 2"`
		SkipThis  int    `form_options:"skip"`
	}

	u := User{}

	form := BaseFormFromModel(u, POST, "/action.html")
	t.Log("Rendered form:", form.Render())
}

func TestFormFromModel(t *testing.T) {
	type Model struct {
		User      string    `form_label:"User label test"`
		password  string    `form_widget:"password"`
		Id        int       `form_min:"0" form_max:"5"`
		Ts        time.Time `form_min:"2013-04-22T15:00"`
		RadioTest string    `form_widget:"select" form_choices:"|A|Option A|G1|B|Option B" form_value:"A"`
		BoolTest  bool      //`form_options:"checked"`
	}

	form := BaseFormFromModel(Model{"asd", "lol", 20, time.Now(), "B", false}, POST, "")
	t.Log("Rendered form:", form.Render())
}

func TestBSFormFromModel(t *testing.T) {
	type Model struct {
		User      string    `form_label:"User label test"`
		password  string    `form_widget:"password"`
		Id        int       `form_min:"0" form_max:"5"`
		Ts        time.Time `form_min:"2013-04-22T15:00"`
		RadioTest string    `form_widget:"select" form_choices:"|A|Option A|G1|B|Option B" form_value:"A"`
		BoolTest  bool      //`form_options:"checked"`
	}

	form := BootstrapFormFromModel(Model{"asd", "lol", 20, time.Now(), "B", false}, POST, "")
	t.Log("Rendered form:", form.Render())
}

func TestInlineCreation(t *testing.T) {
	form := BaseForm(POST, "/action.html").Elements(
		fields.TextField("text_field").SetLabel("Username"),
		FieldSet("psw_fieldset",
			fields.PasswordField("psw1").AddClass("password_class").SetLabel("Password 1"),
			fields.PasswordField("psw2").AddClass("password_class").SetLabel("Password 2"),
		),
		fields.SubmitButton("btn1", "Submit"),
	)
	t.Log("Rendered form:", form.Render())
}
<|endoftext|>


<commit_before>package html

const style = `
#topbar {
	padding: 5px 10px;
	background: #E0EBF5;
}

#topbar a {
	color: #375EAB;
	text-decoration: none;
}

h1, h2, h3, h4 {
	margin: 0;
	padding: 0;
	color: #375EAB;
	font-weight: bold;
}

table {
	border: 1px solid #ccc;
	margin: 20px 5px;
	border-collapse: collapse;
	white-space: nowrap;
	text-overflow: ellipsis;
	overflow: hidden;
}

table caption {
	font-weight: bold;
}

table td, table th {
	vertical-align: top;
	padding: 2px 8px;
	text-overflow: ellipsis;
	overflow: hidden;
}

.namespace {
	font-weight: bold;
	font-size: large;
	color: #375EAB;
}

.position_table {
	border: 0px;
	margin: 0px;
	width: 100%;
	border-collapse: collapse;
}

.position_table td, .position_table tr {
	vertical-align: center;
	padding: 0px;
}

.position_table .search {
	text-align: right;
}

.list_table td, .list_table th {
	border-left: 1px solid #ccc;
}

.list_table th {
	background: #F4F4F4;
}

.list_table tr:nth-child(2n) {
	background: #F4F4F4;
}

.list_table tr:hover {
	background: #ffff99;
}

.list_table .namespace {
	width: 100pt;
	max-width: 100pt;
}

.list_table .title {
	width: 350pt;
	max-width: 350pt;
}

.list_table .commit_list {
	width: 500pt;
	max-width: 500pt;
}

.list_table .tag {
	font-family: monospace;
	font-size: 8pt;
	width: 40pt;
	max-width: 40pt;
}

.list_table .opts {
	width: 40pt;
	max-width: 40pt;
}

.list_table .status {
	width: 250pt;
	max-width: 250pt;
}

.list_table .patched {
	width: 60pt;
	max-width: 60pt;
	text-align: center;
}

.list_table .kernel {
	width: 80pt;
	max-width: 80pt;
}

.list_table .maintainers {
	width: 150pt;
	max-width: 150pt;
}

.list_table .result {
	width: 60pt;
	max-width: 60pt;
}

.list_table .stat {
	width: 55pt;
	max-width: 55pt;
	font-family: monospace;
	text-align: right;
}

.list_table .date {
	width: 60pt;
	max-width: 60pt;
	font-family: monospace;
	text-align: right;
}

.list_table .stat_name {
	width: 150pt;
	max-width: 150pt;
	font-family: monospace;
}

.list_table .stat_value {
	width: 120pt;
	max-width: 120pt;
	font-family: monospace;
}

.bad {
	color: #f00;
	font-weight: bold;
}

.inactive {
	color: #888;
}

.plain {
	text-decoration: none;
}

textarea {
	width:100%;
	font-family: monospace;
}

.mono {
	font-family: monospace;
}
`
const js = `
// Copyright 2018 syzkaller project authors. All rights reserved.
// Use of this source code is governed by Apache 2 LICENSE that can be found in the LICENSE file.

function sortTable(item, colName, conv, desc = false) {
	table = item.parentNode.parentNode.parentNode.parentNode;
	rows = table.rows;
	col = findColumnByName(rows[0].getElementsByTagName("th"), colName);
	values = [];
	for (i = 1; i < rows.length; i++)
		values.push([conv(rows[i].getElementsByTagName("td")[col].textContent), rows[i]]);
	if (desc)
		desc = !isSorted(values.slice().reverse())
	else
		desc = isSorted(values);
	values.sort(function(a, b) {
		if (a[0] == b[0]) return 0;
		if (desc && a[0] > b[0] || !desc && a[0] < b[0]) return -1;
		return 1;
	});
	for (i = 0; i < values.length; i++)
		table.tBodies[0].appendChild(values[i][1]);
	return false;
}

function findColumnByName(headers, colName) {
	for (i = 0; i < headers.length; i++) {
		if (headers[i].textContent == colName)
			return i;
	}
	return 0;
}

function isSorted(values) {
	for (i = 0; i < values.length - 1; i++) {
		if (values[i][0] > values[i + 1][0])
			return false;
	}
	return true;
}

function textSort(v) { return v.toLowerCase(); }
function numSort(v) { return -parseInt(v); }
function floatSort(v) { return -parseFloat(v); }
function reproSort(v) { return v == "C" ? 0 : v == "syz" ? 1 : 2; }
function patchedSort(v) { return v == "" ? -1 : parseInt(v); }

function timeSort(v) {
	if (v == "now")
		return 0;
	m = v.indexOf('m');
	h = v.indexOf('h');
	d = v.indexOf('d');
	if (m > 0 && h < 0)
		return parseInt(v);
	if (h > 0 && m > 0)
		return parseInt(v) * 60 + parseInt(v.substring(h + 1));
	if (d > 0 && h > 0)
		return parseInt(v) * 60 * 24 + parseInt(v.substring(d + 1)) * 60;
	if (d > 0)
		return parseInt(v) * 60 * 24;
	return 1000000000;
}
`
<commit_msg>pkg/html: regenerate<commit_after>package html

const style = `
#topbar {
	padding: 5px 10px;
	background: #E0EBF5;
}

#topbar a {
	color: #375EAB;
	text-decoration: none;
}

h1, h2, h3, h4 {
	margin: 0;
	padding: 0;
	color: #375EAB;
	font-weight: bold;
}

table {
	border: 1px solid #ccc;
	margin: 20px 5px;
	border-collapse: collapse;
	white-space: nowrap;
	text-overflow: ellipsis;
	overflow: hidden;
}

table caption {
	font-weight: bold;
}

table td, table th {
	vertical-align: top;
	padding: 2px 8px;
	text-overflow: ellipsis;
	overflow: hidden;
}

.namespace {
	font-weight: bold;
	font-size: large;
	color: #375EAB;
}

.position_table {
	border: 0px;
	margin: 0px;
	width: 100%;
	border-collapse: collapse;
}

.position_table td, .position_table tr {
	vertical-align: center;
	padding: 0px;
}

.position_table .search {
	text-align: right;
}

.list_table td, .list_table th {
	border-left: 1px solid #ccc;
}

.list_table th {
	background: #F4F4F4;
}

.list_table tr:nth-child(2n) {
	background: #F4F4F4;
}

.list_table tr:hover {
	background: #ffff99;
}

.list_table .namespace {
	width: 100pt;
	max-width: 100pt;
}

.list_table .title {
	width: 350pt;
	max-width: 350pt;
}

.list_table .commit_list {
	width: 500pt;
	max-width: 500pt;
}

.list_table .tag {
	font-family: monospace;
	font-size: 8pt;
	width: 40pt;
	max-width: 40pt;
}

.list_table .opts {
	width: 40pt;
	max-width: 40pt;
}

.list_table .status {
	width: 250pt;
	max-width: 250pt;
}

.list_table .patched {
	width: 60pt;
	max-width: 60pt;
	text-align: center;
}

.list_table .kernel {
	width: 80pt;
	max-width: 80pt;
}

.list_table .maintainers {
	width: 150pt;
	max-width: 150pt;
}

.list_table .result {
	width: 60pt;
	max-width: 60pt;
}

.list_table .stat {
	width: 55pt;
	max-width: 55pt;
	font-family: monospace;
	text-align: right;
}

.list_table .date {
	width: 60pt;
	max-width: 60pt;
	font-family: monospace;
	text-align: right;
}

.list_table .stat_name {
	width: 150pt;
	max-width: 150pt;
	font-family: monospace;
}

.list_table .stat_value {
	width: 120pt;
	max-width: 120pt;
	font-family: monospace;
}

.bad {
	color: #f00;
	font-weight: bold;
}

.inactive {
	color: #888;
}

.plain {
	text-decoration: none;
}

textarea {
	width:100%;
	font-family: monospace;
}

.mono {
	font-family: monospace;
}

.info_link {
	color: #25a7db;
	text-decoration: none;
}
`
const js = `
// Copyright 2018 syzkaller project authors. All rights reserved.
// Use of this source code is governed by Apache 2 LICENSE that can be found in the LICENSE file.

function sortTable(item, colName, conv, desc = false) {
	table = item.parentNode.parentNode.parentNode.parentNode;
	rows = table.rows;
	col = findColumnByName(rows[0].getElementsByTagName("th"), colName);
	values = [];
	for (i = 1; i < rows.length; i++)
		values.push([conv(rows[i].getElementsByTagName("td")[col].textContent), rows[i]]);
	if (desc)
		desc = !isSorted(values.slice().reverse())
	else
		desc = isSorted(values);
	values.sort(function(a, b) {
		if (a[0] == b[0]) return 0;
		if (desc && a[0] > b[0] || !desc && a[0] < b[0]) return -1;
		return 1;
	});
	for (i = 0; i < values.length; i++)
		table.tBodies[0].appendChild(values[i][1]);
	return false;
}

function findColumnByName(headers, colName) {
	for (i = 0; i < headers.length; i++) {
		if (headers[i].textContent == colName)
			return i;
	}
	return 0;
}

function isSorted(values) {
	for (i = 0; i < values.length - 1; i++) {
		if (values[i][0] > values[i + 1][0])
			return false;
	}
	return true;
}

function textSort(v) { return v.toLowerCase(); }
function numSort(v) { return -parseInt(v); }
function floatSort(v) { return -parseFloat(v); }
function reproSort(v) { return v == "C" ? 0 : v == "syz" ? 1 : 2; }
function patchedSort(v) { return v == "" ? -1 : parseInt(v); }

function timeSort(v) {
	if (v == "now")
		return 0;
	m = v.indexOf('m');
	h = v.indexOf('h');
	d = v.indexOf('d');
	if (m > 0 && h < 0)
		return parseInt(v);
	if (h > 0 && m > 0)
		return parseInt(v) * 60 + parseInt(v.substring(h + 1));
	if (d > 0 && h > 0)
		return parseInt(v) * 60 * 24 + parseInt(v.substring(d + 1)) * 60;
	if (d > 0)
		return parseInt(v) * 60 * 24;
	return 1000000000;
}
`
<|endoftext|>


<commit_before>package main

import (
	"encoding/json"
	"encoding/xml"
	"fmt"
	"io/ioutil"
	"log"
	"net/http"
	"os"
	"path"
	"path/filepath"
	"regexp"
	"runtime"
	"sort"
	"strings"
	"sync"

	"github.com/gorilla/mux"
	"github.com/stts-se/rbg2p"
)

type g2pMutex struct {
	g2ps  map[string]rbg2p.RuleSet
	sylls map[string]rbg2p.Syllabifier
	mutex *sync.RWMutex
}

var g2pM = g2pMutex{
	g2ps:  make(map[string]rbg2p.RuleSet),
	sylls: make(map[string]rbg2p.Syllabifier),
	mutex: &sync.RWMutex{},
}

func g2pMain_Handler(w http.ResponseWriter, r *http.Request) {
	_, cmdFileName, _, _ := runtime.Caller(0)
	fString := path.Join(path.Dir(cmdFileName), "static/g2p_demo.html")
	http.ServeFile(w, r, fString)
}

var wSplitRe = regexp.MustCompile(" *, *")

// Word internal struct for json
type Word struct {
	Orth    string   `json:"orth"`
	Transes []string `json:"transes"`
}

func syllabify(lang string, trans string) (string, int, error) {
	g2pM.mutex.RLock()
	defer g2pM.mutex.RUnlock()
	syller, ok := g2pM.sylls[lang]
	if !ok {
		ruleSet, ok := g2pM.g2ps[lang]
		if !ok {
			msg := "unknown 'lang': " + lang
			langs, err := listSyllLanguages()
			if err != nil {
				return "", http.StatusInternalServerError, err
			}
			msg = fmt.Sprintf("%s. Known 'lang' values: %s", msg, strings.Join(langs, ", "))
			return "", http.StatusBadRequest, fmt.Errorf(msg)
		}

		if !ruleSet.Syllabifier.IsDefined() {
			msg := fmt.Sprintf("no syllabifier defined for language %s", lang)
			return "", http.StatusInternalServerError, fmt.Errorf(msg)
		}
		syller = ruleSet.Syllabifier
	}
	phns, err := syller.PhonemeSet.SplitTranscription(trans)
	if err != nil {
		msg := fmt.Sprintf("couldn't split input transcription /%s/ : %s", trans, err)
		return "", http.StatusInternalServerError, fmt.Errorf(msg)
	}
	sylled := syller.SyllabifyFromPhonemes(phns)
	return sylled, http.StatusOK, nil
}

func syllabify_Handler(w http.ResponseWriter, r *http.Request) {

	vars := mux.Vars(r)
	lang := vars["lang"]
	if "" == lang {
		msg := "no value for the expected 'lang' parameter"
		log.Println(msg)
		http.Error(w, msg, http.StatusBadRequest)
		return
	}

	trans := vars["trans"]
	if "" == trans {
		msg := "no value for the expected 'trans' parameter"
		log.Println(msg)
		http.Error(w, msg, http.StatusBadRequest)
		return
	}
	res, status, err := syllabify(lang, trans)
	if err != nil {
		log.Printf("%s\n", err)
		http.Error(w, fmt.Sprintf("%s", err), status)
		return
	}

	//w.Header().Set("Content-Type", "application/json; charset=utf-8")
	// j, err := json.Marshal(res)
	// if err != nil {
	// 	msg := fmt.Sprintf("failed json marshalling : %v", err)
	// 	log.Println(msg)
	// 	http.Error(w, msg, http.StatusInternalServerError)
	// 	return
	// }
	// fmt.Fprintf(w, string(j))
	fmt.Fprintf(w, string(res))
}

func transcribe(lang string, word string) (Word, int, error) {
	g2pM.mutex.RLock()
	defer g2pM.mutex.RUnlock()
	ruleSet, ok := g2pM.g2ps[lang]
	if !ok {
		msg := "unknown 'lang': " + lang
		langs := listG2PLanguages()
		msg = fmt.Sprintf("%s. Known 'lang' values: %s", msg, strings.Join(langs, ", "))
		return Word{}, http.StatusBadRequest, fmt.Errorf(msg)
	}

	transes, err := ruleSet.Apply(word)
	if err != nil {
		msg := fmt.Sprintf("couldn't transcribe word : %v", err)
		return Word{}, http.StatusInternalServerError, fmt.Errorf(msg)
	}
	tRes := []string{}
	for _, trans := range transes {
		tRes = append(tRes, trans)
	}
	res := Word{word, tRes}
	return res, http.StatusOK, nil
}

func transcribe_Handler(w http.ResponseWriter, r *http.Request) {

	format := r.FormValue("format")
	if "xml" == format {
		transcribe_AsXml_Handler(w, r)
		return
	}

	vars := mux.Vars(r)
	lang := vars["lang"]
	if "" == lang {
		msg := "no value for the expected 'lang' parameter"
		log.Println(msg)
		http.Error(w, msg, http.StatusBadRequest)
		return
	}

	word := vars["word"]
	if "" == word {
		msg := "no value for the expected 'word' parameter"
		log.Println(msg)
		http.Error(w, msg, http.StatusBadRequest)
		return
	}
	word = strings.ToLower(word)

	res, status, err := transcribe(lang, word)
	if err != nil {
		log.Printf("%s\n", err)
		http.Error(w, fmt.Sprintf("%s", err), status)
		return
	}

	w.Header().Set("Content-Type", "application/json; charset=utf-8")
	j, err := json.Marshal(res)
	if err != nil {
		msg := fmt.Sprintf("failed json marshalling : %v", err)
		log.Println(msg)
		http.Error(w, msg, http.StatusInternalServerError)
		return
	}
	fmt.Fprintf(w, string(j))
}

// XMLWords container go generate xml from http request, for legacy calls from ltool/yalt
type XMLWords struct {
	XMLName xml.Name `xml:"words"`
	Words   []XMLWord
}

// XMLWord container go generate xml from http request, for legacy calls from ltool/yalt
type XMLWord struct {
	XMLName xml.Name `xml:"word"`
	Orth    string   `xml:"orth,attr"`
	Trans   string   `xml:"trans"`
}

// for legacy calls from ltool/yalt
func transcribe_AsXml_Handler(w http.ResponseWriter, r *http.Request) {
	vars := mux.Vars(r)
	lang := vars["lang"]
	if "" == lang {
		msg := "no value for the expected 'lang' parameter"
		log.Println(msg)
		http.Error(w, msg, http.StatusBadRequest)
		return
	}
	word := vars["word"]
	if "" == word {
		msg := "no value for the expected 'word' parameter"
		log.Println(msg)
		http.Error(w, msg, http.StatusBadRequest)
		return
	}
	word = strings.ToLower(word)
	res, status, err := transcribe(lang, word)
	if err != nil {
		log.Printf("%s\n", err)
		http.Error(w, fmt.Sprintf("%s", err), status)
		return
	}
	//<words>
	//<word orth='apa' word_lang='mk' trans_lang='mk' >" a p a</word>
	//</words>

	// words := XMLWords{
	// 	Words: []XMLWord{
	// 		XMLWord{Orth: word, Trans: res.Transes[0]},
	// 	},
	// }
	words := XMLWords{}
	for _, t := range res.Transes {
		words.Words = append(words.Words, XMLWord{Orth: word, Trans: t})
	}

	w.Header().Set("Content-Type", "application/xml; charset=utf-8")
	xml, err := xml.Marshal(words)
	if err != nil {
		msg := fmt.Sprintf("failed xml marshalling : %v", err)
		log.Println(msg)
		http.Error(w, msg, http.StatusInternalServerError)
		return
	}
	fmt.Fprintf(w, string(xml))
	//fmt.Fprintf(w, string(res.Transes[0]))
}

func listG2PLanguages() []string {
	var res []string
	for name := range g2pM.g2ps {
		res = append(res, name)
	}
	return res
}

func listSyllLanguages() ([]string, error) {
	var res []string
	for name, g2p := range g2pM.g2ps {
		if g2p.Syllabifier.IsDefined() {
			res = append(res, name)
		}
	}
	for name := range g2pM.sylls {
		if rbg2p.Contains(res, name) {
			return res, fmt.Errorf("name conflict for '%s' (g2p and syllabifier cannot share the same name)", name)
		}
		res = append(res, name)
	}
	return res, nil
}

func g2pList_Handler(w http.ResponseWriter, r *http.Request) {
	g2pM.mutex.RLock()
	res := listG2PLanguages()
	g2pM.mutex.RUnlock()

	sort.Strings(res)
	j, err := json.Marshal(res)
	if err != nil {
		msg := fmt.Sprintf("failed json marshalling : %v", err)
		log.Println(msg)
		http.Error(w, msg, http.StatusInternalServerError)
		return
	}
	fmt.Fprintf(w, string(j))
}

func syllList_Handler(w http.ResponseWriter, r *http.Request) {
	g2pM.mutex.RLock()
	res, err := listSyllLanguages()
	g2pM.mutex.RUnlock()
	if err != nil {
		msg := fmt.Sprintf("%s", err)
		log.Println(msg)
		http.Error(w, msg, http.StatusInternalServerError)
		return
	}

	sort.Strings(res)
	j, err := json.Marshal(res)
	if err != nil {
		msg := fmt.Sprintf("failed json marshalling : %v", err)
		log.Println(msg)
		http.Error(w, msg, http.StatusInternalServerError)
		return
	}
	fmt.Fprintf(w, string(j))
}

// langFromFilePath returns the base file name stripped from any '.g2p' extension
func langFromFilePath(p string) string {
	b := filepath.Base(p)
	if strings.HasSuffix(b, ".g2p") {
		b = b[0 : len(b)-4]
	} else if strings.HasSuffix(b, ".syll") {
		b = b[0 : len(b)-5]
	}
	return b
}

func main() {

	if len(os.Args) != 2 {
		fmt.Fprintf(os.Stderr, "server <G2P FILES DIR>\n")
		os.Exit(0)
	}

	// g2p file dir. Each file in dir with .g2p extension
	// is treated as a g2p file
	var dir = os.Args[1]

	files, err := ioutil.ReadDir(dir)
	if err != nil {
		fmt.Fprintf(os.Stderr, "%v\n", err)
		os.Exit(0)
	}

	// populate map of g2p rules from files.
	// The base file name minus '.g2p' is the language name.
	var fn string
	for _, f := range files {
		fn = filepath.Join(dir, f.Name())
		if strings.HasSuffix(fn, ".g2p") {

			ruleSet, err := rbg2p.LoadFile(fn)
			if err != nil {
				fmt.Fprintf(os.Stderr, "%v\n", err)
				fmt.Fprintf(os.Stderr, "server: skipping file: '%s'\n", fn)
				continue
			}

			lang := langFromFilePath(fn)
			g2pM.mutex.Lock()
			g2pM.g2ps[lang] = ruleSet
			g2pM.mutex.Unlock()
			fmt.Fprintf(os.Stderr, "server: loaded file '%s'\n", fn)

		} else if strings.HasSuffix(fn, ".syll") {

			syll, err := rbg2p.LoadSyllFile(fn)
			if err != nil {
				fmt.Fprintf(os.Stderr, "%v\n", err)
				fmt.Fprintf(os.Stderr, "server: skipping file: '%s'\n", fn)
				continue
			}

			lang := langFromFilePath(fn)
			g2pM.mutex.Lock()
			g2pM.sylls[lang] = syll
			g2pM.mutex.Unlock()
			fmt.Fprintf(os.Stderr, "server: loaded file '%s'\n", fn)

		} else {
			fmt.Fprintf(os.Stderr, "server: skipping file: '%s'\n", fn)
			continue
		}

	}

	if _, err := listSyllLanguages(); err != nil {
		log.Fatalf("server init error : %s", err)
	}

	r := mux.NewRouter().StrictSlash(true)

	r.HandleFunc("/rbg2p", g2pMain_Handler) //.Methods("get")
	r.HandleFunc("/", g2pMain_Handler)      //.Methods("get")

	s := r.PathPrefix("/rbg2p").Subrouter()

	s.HandleFunc("/g2p/list", g2pList_Handler)
	s.HandleFunc("/syll/list", syllList_Handler)

	s.HandleFunc("/transcribe/{lang}/{word}", transcribe_Handler) //.Methods("get", "post")
	s.HandleFunc("/syllabify/{lang}/{trans}", syllabify_Handler)  //.Methods("get", "post")

	// for legacy calls from ltool/yalt
	s.HandleFunc("/xmltranscribe/{lang}/{word}", transcribe_AsXml_Handler)

	fmt.Println("Serving urls:")
	r.Walk(func(route *mux.Route, router *mux.Router, ancestors []*mux.Route) error {
		t, err := route.GetPathTemplate()
		if err != nil {
			return err
		}
		fmt.Println(t)
		return nil
	})

	port := ":6771"
	log.Printf("starting g2p server at port %s\n", port)
	err = http.ListenAndServe(port, r)
	if err != nil {

		log.Fatalf("no fun: %v\n", err)
	}

}
<commit_msg>/ping implemented<commit_after>package main

import (
	"encoding/json"
	"encoding/xml"
	"fmt"
	"io/ioutil"
	"log"
	"net/http"
	"os"
	"path"
	"path/filepath"
	"regexp"
	"runtime"
	"sort"
	"strings"
	"sync"

	"github.com/gorilla/mux"
	"github.com/stts-se/rbg2p"
)

type g2pMutex struct {
	g2ps  map[string]rbg2p.RuleSet
	sylls map[string]rbg2p.Syllabifier
	mutex *sync.RWMutex
}

var g2pM = g2pMutex{
	g2ps:  make(map[string]rbg2p.RuleSet),
	sylls: make(map[string]rbg2p.Syllabifier),
	mutex: &sync.RWMutex{},
}

func g2pMain_Handler(w http.ResponseWriter, r *http.Request) {
	_, cmdFileName, _, _ := runtime.Caller(0)
	fString := path.Join(path.Dir(cmdFileName), "static/g2p_demo.html")
	http.ServeFile(w, r, fString)
}

func ping_Handler(w http.ResponseWriter, r *http.Request) {
	fmt.Fprintf(w, "rbg2p")
}

var wSplitRe = regexp.MustCompile(" *, *")

// Word internal struct for json
type Word struct {
	Orth    string   `json:"orth"`
	Transes []string `json:"transes"`
}

func syllabify(lang string, trans string) (string, int, error) {
	g2pM.mutex.RLock()
	defer g2pM.mutex.RUnlock()
	syller, ok := g2pM.sylls[lang]
	if !ok {
		ruleSet, ok := g2pM.g2ps[lang]
		if !ok {
			msg := "unknown 'lang': " + lang
			langs, err := listSyllLanguages()
			if err != nil {
				return "", http.StatusInternalServerError, err
			}
			msg = fmt.Sprintf("%s. Known 'lang' values: %s", msg, strings.Join(langs, ", "))
			return "", http.StatusBadRequest, fmt.Errorf(msg)
		}

		if !ruleSet.Syllabifier.IsDefined() {
			msg := fmt.Sprintf("no syllabifier defined for language %s", lang)
			return "", http.StatusInternalServerError, fmt.Errorf(msg)
		}
		syller = ruleSet.Syllabifier
	}
	phns, err := syller.PhonemeSet.SplitTranscription(trans)
	if err != nil {
		msg := fmt.Sprintf("couldn't split input transcription /%s/ : %s", trans, err)
		return "", http.StatusInternalServerError, fmt.Errorf(msg)
	}
	sylled := syller.SyllabifyFromPhonemes(phns)
	return sylled, http.StatusOK, nil
}

func syllabify_Handler(w http.ResponseWriter, r *http.Request) {

	vars := mux.Vars(r)
	lang := vars["lang"]
	if "" == lang {
		msg := "no value for the expected 'lang' parameter"
		log.Println(msg)
		http.Error(w, msg, http.StatusBadRequest)
		return
	}

	trans := vars["trans"]
	if "" == trans {
		msg := "no value for the expected 'trans' parameter"
		log.Println(msg)
		http.Error(w, msg, http.StatusBadRequest)
		return
	}
	res, status, err := syllabify(lang, trans)
	if err != nil {
		log.Printf("%s\n", err)
		http.Error(w, fmt.Sprintf("%s", err), status)
		return
	}

	//w.Header().Set("Content-Type", "application/json; charset=utf-8")
	// j, err := json.Marshal(res)
	// if err != nil {
	// 	msg := fmt.Sprintf("failed json marshalling : %v", err)
	// 	log.Println(msg)
	// 	http.Error(w, msg, http.StatusInternalServerError)
	// 	return
	// }
	// fmt.Fprintf(w, string(j))
	fmt.Fprintf(w, string(res))
}

func transcribe(lang string, word string) (Word, int, error) {
	g2pM.mutex.RLock()
	defer g2pM.mutex.RUnlock()
	ruleSet, ok := g2pM.g2ps[lang]
	if !ok {
		msg := "unknown 'lang': " + lang
		langs := listG2PLanguages()
		msg = fmt.Sprintf("%s. Known 'lang' values: %s", msg, strings.Join(langs, ", "))
		return Word{}, http.StatusBadRequest, fmt.Errorf(msg)
	}

	transes, err := ruleSet.Apply(word)
	if err != nil {
		msg := fmt.Sprintf("couldn't transcribe word : %v", err)
		return Word{}, http.StatusInternalServerError, fmt.Errorf(msg)
	}
	tRes := []string{}
	for _, trans := range transes {
		tRes = append(tRes, trans)
	}
	res := Word{word, tRes}
	return res, http.StatusOK, nil
}

func transcribe_Handler(w http.ResponseWriter, r *http.Request) {

	format := r.FormValue("format")
	if "xml" == format {
		transcribe_AsXml_Handler(w, r)
		return
	}

	vars := mux.Vars(r)
	lang := vars["lang"]
	if "" == lang {
		msg := "no value for the expected 'lang' parameter"
		log.Println(msg)
		http.Error(w, msg, http.StatusBadRequest)
		return
	}

	word := vars["word"]
	if "" == word {
		msg := "no value for the expected 'word' parameter"
		log.Println(msg)
		http.Error(w, msg, http.StatusBadRequest)
		return
	}
	word = strings.ToLower(word)

	res, status, err := transcribe(lang, word)
	if err != nil {
		log.Printf("%s\n", err)
		http.Error(w, fmt.Sprintf("%s", err), status)
		return
	}

	w.Header().Set("Content-Type", "application/json; charset=utf-8")
	j, err := json.Marshal(res)
	if err != nil {
		msg := fmt.Sprintf("failed json marshalling : %v", err)
		log.Println(msg)
		http.Error(w, msg, http.StatusInternalServerError)
		return
	}
	fmt.Fprintf(w, string(j))
}

// XMLWords container go generate xml from http request, for legacy calls from ltool/yalt
type XMLWords struct {
	XMLName xml.Name `xml:"words"`
	Words   []XMLWord
}

// XMLWord container go generate xml from http request, for legacy calls from ltool/yalt
type XMLWord struct {
	XMLName xml.Name `xml:"word"`
	Orth    string   `xml:"orth,attr"`
	Trans   string   `xml:"trans"`
}

// for legacy calls from ltool/yalt
func transcribe_AsXml_Handler(w http.ResponseWriter, r *http.Request) {
	vars := mux.Vars(r)
	lang := vars["lang"]
	if "" == lang {
		msg := "no value for the expected 'lang' parameter"
		log.Println(msg)
		http.Error(w, msg, http.StatusBadRequest)
		return
	}
	word := vars["word"]
	if "" == word {
		msg := "no value for the expected 'word' parameter"
		log.Println(msg)
		http.Error(w, msg, http.StatusBadRequest)
		return
	}
	word = strings.ToLower(word)
	res, status, err := transcribe(lang, word)
	if err != nil {
		log.Printf("%s\n", err)
		http.Error(w, fmt.Sprintf("%s", err), status)
		return
	}
	//<words>
	//<word orth='apa' word_lang='mk' trans_lang='mk' >" a p a</word>
	//</words>

	// words := XMLWords{
	// 	Words: []XMLWord{
	// 		XMLWord{Orth: word, Trans: res.Transes[0]},
	// 	},
	// }
	words := XMLWords{}
	for _, t := range res.Transes {
		words.Words = append(words.Words, XMLWord{Orth: word, Trans: t})
	}

	w.Header().Set("Content-Type", "application/xml; charset=utf-8")
	xml, err := xml.Marshal(words)
	if err != nil {
		msg := fmt.Sprintf("failed xml marshalling : %v", err)
		log.Println(msg)
		http.Error(w, msg, http.StatusInternalServerError)
		return
	}
	fmt.Fprintf(w, string(xml))
	//fmt.Fprintf(w, string(res.Transes[0]))
}

func listG2PLanguages() []string {
	var res []string
	for name := range g2pM.g2ps {
		res = append(res, name)
	}
	return res
}

func listSyllLanguages() ([]string, error) {
	var res []string
	for name, g2p := range g2pM.g2ps {
		if g2p.Syllabifier.IsDefined() {
			res = append(res, name)
		}
	}
	for name := range g2pM.sylls {
		if rbg2p.Contains(res, name) {
			return res, fmt.Errorf("name conflict for '%s' (g2p and syllabifier cannot share the same name)", name)
		}
		res = append(res, name)
	}
	return res, nil
}

func g2pList_Handler(w http.ResponseWriter, r *http.Request) {
	g2pM.mutex.RLock()
	res := listG2PLanguages()
	g2pM.mutex.RUnlock()

	sort.Strings(res)
	j, err := json.Marshal(res)
	if err != nil {
		msg := fmt.Sprintf("failed json marshalling : %v", err)
		log.Println(msg)
		http.Error(w, msg, http.StatusInternalServerError)
		return
	}
	fmt.Fprintf(w, string(j))
}

func syllList_Handler(w http.ResponseWriter, r *http.Request) {
	g2pM.mutex.RLock()
	res, err := listSyllLanguages()
	g2pM.mutex.RUnlock()
	if err != nil {
		msg := fmt.Sprintf("%s", err)
		log.Println(msg)
		http.Error(w, msg, http.StatusInternalServerError)
		return
	}

	sort.Strings(res)
	j, err := json.Marshal(res)
	if err != nil {
		msg := fmt.Sprintf("failed json marshalling : %v", err)
		log.Println(msg)
		http.Error(w, msg, http.StatusInternalServerError)
		return
	}
	fmt.Fprintf(w, string(j))
}

// langFromFilePath returns the base file name stripped from any '.g2p' extension
func langFromFilePath(p string) string {
	b := filepath.Base(p)
	if strings.HasSuffix(b, ".g2p") {
		b = b[0 : len(b)-4]
	} else if strings.HasSuffix(b, ".syll") {
		b = b[0 : len(b)-5]
	}
	return b
}

func main() {

	if len(os.Args) != 2 {
		fmt.Fprintf(os.Stderr, "server <G2P FILES DIR>\n")
		os.Exit(0)
	}

	// g2p file dir. Each file in dir with .g2p extension
	// is treated as a g2p file
	var dir = os.Args[1]

	files, err := ioutil.ReadDir(dir)
	if err != nil {
		fmt.Fprintf(os.Stderr, "%v\n", err)
		os.Exit(0)
	}

	// populate map of g2p rules from files.
	// The base file name minus '.g2p' is the language name.
	var fn string
	for _, f := range files {
		fn = filepath.Join(dir, f.Name())
		if strings.HasSuffix(fn, ".g2p") {

			ruleSet, err := rbg2p.LoadFile(fn)
			if err != nil {
				fmt.Fprintf(os.Stderr, "%v\n", err)
				fmt.Fprintf(os.Stderr, "server: skipping file: '%s'\n", fn)
				continue
			}

			lang := langFromFilePath(fn)
			g2pM.mutex.Lock()
			g2pM.g2ps[lang] = ruleSet
			g2pM.mutex.Unlock()
			fmt.Fprintf(os.Stderr, "server: loaded file '%s'\n", fn)

		} else if strings.HasSuffix(fn, ".syll") {

			syll, err := rbg2p.LoadSyllFile(fn)
			if err != nil {
				fmt.Fprintf(os.Stderr, "%v\n", err)
				fmt.Fprintf(os.Stderr, "server: skipping file: '%s'\n", fn)
				continue
			}

			lang := langFromFilePath(fn)
			g2pM.mutex.Lock()
			g2pM.sylls[lang] = syll
			g2pM.mutex.Unlock()
			fmt.Fprintf(os.Stderr, "server: loaded file '%s'\n", fn)

		} else {
			fmt.Fprintf(os.Stderr, "server: skipping file: '%s'\n", fn)
			continue
		}

	}

	if _, err := listSyllLanguages(); err != nil {
		log.Fatalf("server init error : %s", err)
	}

	r := mux.NewRouter().StrictSlash(true)

	r.HandleFunc("/rbg2p", g2pMain_Handler)
	r.HandleFunc("/", g2pMain_Handler)

	r.HandleFunc("/ping", ping_Handler)

	s := r.PathPrefix("/rbg2p").Subrouter()

	s.HandleFunc("/g2p/list", g2pList_Handler)
	s.HandleFunc("/syll/list", syllList_Handler)

	s.HandleFunc("/transcribe/{lang}/{word}", transcribe_Handler)
	s.HandleFunc("/syllabify/{lang}/{trans}", syllabify_Handler)

	// for legacy calls from ltool/yalt
	s.HandleFunc("/xmltranscribe/{lang}/{word}", transcribe_AsXml_Handler)

	fmt.Println("Serving urls:")
	r.Walk(func(route *mux.Route, router *mux.Router, ancestors []*mux.Route) error {
		t, err := route.GetPathTemplate()
		if err != nil {
			return err
		}
		fmt.Println(t)
		return nil
	})

	port := ":6771"
	log.Printf("starting g2p server at port %s\n", port)
	err = http.ListenAndServe(port, r)
	if err != nil {

		log.Fatalf("no fun: %v\n", err)
	}

}
<|endoftext|>


<commit_before>// Copyright 2016 DeepFabric, Inc.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// See the License for the specific language governing permissions and
// limitations under the License.

package raftstore

import (
	"fmt"
	"io"
	"os"
	"path/filepath"
	"sync"
	"time"

	"github.com/deepfabric/elasticell/pkg/log"
	"github.com/deepfabric/elasticell/pkg/pb/mraft"
	"github.com/deepfabric/elasticell/pkg/storage"
	"github.com/deepfabric/elasticell/pkg/util"
	"github.com/fagongzi/goetty"
	"github.com/pkg/errors"
	"golang.org/x/net/context"
	"golang.org/x/time/rate"
)

var (
	creating = 1
	sending  = 2
)

// SnapshotManager manager snapshot
type SnapshotManager interface {
	Register(msg *mraft.SnapshotMessage, step int) bool
	Deregister(msg *mraft.SnapshotMessage, step int)
	Create(msg *mraft.SnapshotMessage) error
	Exists(msg *mraft.SnapshotMessage) bool
	WriteTo(msg *mraft.SnapshotMessage, conn goetty.IOSession) (uint64, error)
	CleanSnap(msg *mraft.SnapshotMessage) error
	ReceiveSnapData(msg *mraft.SnapshotMessage) error
	Apply(msg *mraft.SnapshotMessage) error
}

type defaultSnapshotManager struct {
	sync.RWMutex

	limiter *rate.Limiter

	cfg *Cfg
	db  storage.DataEngine
	dir string

	registry map[string]struct{}
}

func newDefaultSnapshotManager(cfg *Cfg, db storage.DataEngine) SnapshotManager {
	dir := cfg.getSnapDir()

	if !exist(dir) {
		if err := os.Mkdir(dir, 0750); err != nil {
			log.Fatalf("raftstore-snap: cannot create dir for snapshot, errors:\n %+v",
				err)
		}
	}

	go func() {
		interval := time.Hour * 2

		for {
			log.Infof("raftstore-snap: start scan gc snap files")

			var paths []string

			err := filepath.Walk(dir, func(path string, f os.FileInfo, err error) error {
				if f == nil {
					return nil
				}

				if f.IsDir() && f.Name() == getSnapDirName() {
					return nil
				}

				var skip error
				if f.IsDir() && f.Name() != getSnapDirName() {
					skip = filepath.SkipDir
				}

				now := time.Now()
				if now.Sub(f.ModTime()) > interval {
					paths = append(paths, path)
				}

				return skip
			})

			if err != nil {
				log.Errorf("raftstore-snap: scan snap file failed, errors:\n%+v",
					err)
			}

			for _, path := range paths {
				err := os.RemoveAll(path)
				if err != nil {
					log.Errorf("raftstore-snap: scan snap file failed, file=<%s>, errors:\n%+v",
						path,
						err)
				}
			}

			time.Sleep(interval)
		}
	}()

	return &defaultSnapshotManager{
		cfg:      cfg,
		limiter:  rate.NewLimiter(rate.Every(time.Second), int(cfg.LimitSnapChunkRate)),
		dir:      dir,
		db:       db,
		registry: make(map[string]struct{}),
	}
}

func formatKey(msg *mraft.SnapshotMessage) string {
	return fmt.Sprintf("%d_%d_%d", msg.Header.Cell.ID, msg.Header.Term, msg.Header.Index)
}

func formatKeyStep(msg *mraft.SnapshotMessage, step int) string {
	return fmt.Sprintf("%s_%d", formatKey(msg), step)
}

func (m *defaultSnapshotManager) getPathOfSnapKey(msg *mraft.SnapshotMessage) string {
	return fmt.Sprintf("%s/%s", m.dir, formatKey(msg))
}

func (m *defaultSnapshotManager) getPathOfSnapKeyGZ(msg *mraft.SnapshotMessage) string {
	return fmt.Sprintf("%s.gz", m.getPathOfSnapKey(msg))
}

func (m *defaultSnapshotManager) getTmpPathOfSnapKeyGZ(msg *mraft.SnapshotMessage) string {
	return fmt.Sprintf("%s.tmp", m.getPathOfSnapKey(msg))
}

func (m *defaultSnapshotManager) Register(msg *mraft.SnapshotMessage, step int) bool {
	m.Lock()
	defer m.Unlock()

	fkey := formatKeyStep(msg, step)

	if _, ok := m.registry[fkey]; ok {
		return false
	}

	m.registry[fkey] = emptyStruct
	return true
}

func (m *defaultSnapshotManager) Deregister(msg *mraft.SnapshotMessage, step int) {
	m.Lock()
	defer m.Unlock()

	fkey := formatKeyStep(msg, step)
	delete(m.registry, fkey)
}

func (m *defaultSnapshotManager) inRegistry(msg *mraft.SnapshotMessage, step int) bool {
	m.RLock()
	defer m.RUnlock()

	fkey := formatKeyStep(msg, step)
	_, ok := m.registry[fkey]

	return ok
}

func (m *defaultSnapshotManager) Create(msg *mraft.SnapshotMessage) error {
	path := m.getPathOfSnapKey(msg)
	gzPath := m.getPathOfSnapKeyGZ(msg)
	start := encStartKey(&msg.Header.Cell)
	end := encEndKey(&msg.Header.Cell)

	if !exist(gzPath) {
		if !exist(path) {
			err := m.db.CreateSnapshot(path, start, end)
			if err != nil {
				return errors.Wrapf(err, "")
			}
		}

		err := util.GZIP(path)
		if err != nil {
			return errors.Wrapf(err, "")
		}
	}

	info, err := os.Stat(fmt.Sprintf("%s.gz", path))
	if err != nil {
		return errors.Wrapf(err, "")
	}

	fileSize := uint64(info.Size())
	snapshotSizeHistogram.Observe(float64(fileSize))
	return nil
}

func (m *defaultSnapshotManager) Exists(msg *mraft.SnapshotMessage) bool {
	file := m.getPathOfSnapKeyGZ(msg)
	return exist(file)
}

func (m *defaultSnapshotManager) WriteTo(msg *mraft.SnapshotMessage, conn goetty.IOSession) (uint64, error) {
	file := m.getPathOfSnapKeyGZ(msg)

	if !m.Exists(msg) {
		return 0, fmt.Errorf("missing snapshot file: %s", file)
	}

	info, err := os.Stat(file)
	if err != nil {
		return 0, errors.Wrapf(err, "")
	}
	fileSize := info.Size()

	f, err := os.Open(file)
	if err != nil {
		return 0, err
	}
	defer f.Close()

	var written int64
	buf := make([]byte, m.cfg.LimitSnapChunkBytes)
	ctx := context.TODO()

	log.Infof("raftstore-snap[cell-%d]: try to send snap, header=<%s>,size=<%d>",
		msg.Header.Cell.ID,
		msg.Header.String(),
		fileSize)

	for {
		nr, er := f.Read(buf)
		if nr > 0 {
			dst := &mraft.SnapshotMessage{}
			dst.Header = msg.Header
			dst.Chunk = &mraft.SnapshotChunkMessage{
				Data:     buf[0:nr],
				FileSize: uint64(fileSize),
				First:    0 == written,
				Last:     fileSize == written+int64(nr),
			}

			written += int64(nr)
			err := m.limiter.Wait(ctx)
			if err != nil {
				return 0, err
			}

			err = conn.Write(dst)
			if err != nil {
				return 0, err
			}
		}
		if er != nil {
			if er != io.EOF {
				return 0, er
			}
			break
		}
	}

	log.Infof("raftstore-snap[cell-%d]: send snap complete",
		msg.Header.Cell.ID)
	return uint64(written), nil
}

func (m *defaultSnapshotManager) CleanSnap(msg *mraft.SnapshotMessage) error {
	var err error

	tmpFile := m.getTmpPathOfSnapKeyGZ(msg)
	if exist(tmpFile) {
		log.Infof("raftstore-snap[cell-%d]: delete exists snap tmp file, file=<%s>, header=<%s>",
			msg.Header.Cell.ID,
			tmpFile,
			msg.Header.String())
		err = os.RemoveAll(tmpFile)
	}

	if err != nil {
		return err
	}

	file := m.getPathOfSnapKeyGZ(msg)
	if exist(file) {
		log.Infof("raftstore-snap[cell-%d]: delete exists snap gz file, file=<%s>, header=<%s>",
			msg.Header.Cell.ID,
			file,
			msg.Header.String())
		err = os.RemoveAll(file)
	}

	if err != nil {
		return err
	}

	dir := m.getPathOfSnapKey(msg)
	if exist(dir) {
		log.Infof("raftstore-snap[cell-%d]: delete exists snap dir, file=<%s>, header=<%s>",
			msg.Header.Cell.ID,
			dir,
			msg.Header.String())
		err = os.RemoveAll(dir)
	}

	return err
}

func (m *defaultSnapshotManager) ReceiveSnapData(msg *mraft.SnapshotMessage) error {
	var err error
	var f *os.File

	if msg.Chunk.First {
		err = m.cleanTmp(msg)
	}

	if err != nil {
		return err
	}

	file := m.getTmpPathOfSnapKeyGZ(msg)
	if exist(file) {
		f, err = os.OpenFile(file, os.O_APPEND|os.O_WRONLY, 0600)
		if err != nil {
			f.Close()
			return err
		}
	} else {
		f, err = os.Create(file)
		if err != nil {
			f.Close()
			return err
		}
	}

	n, err := f.Write(msg.Chunk.Data)
	if err != nil {
		f.Close()
		return err
	}

	if n != len(msg.Chunk.Data) {
		f.Close()
		return fmt.Errorf("write snapshot file failed, expect=<%d> actual=<%d>",
			len(msg.Chunk.Data),
			n)
	}

	f.Close()

	if msg.Chunk.Last {
		return m.check(msg)
	}

	return nil
}

func (m *defaultSnapshotManager) Apply(msg *mraft.SnapshotMessage) error {
	file := m.getPathOfSnapKeyGZ(msg)
	if !m.Exists(msg) {
		return fmt.Errorf("missing snapshot file, path=%s", file)
	}

	defer m.CleanSnap(msg)

	err := util.UnGZIP(file, m.dir)
	if err != nil {
		return err
	}
	defer os.RemoveAll(m.getPathOfSnapKey(msg))

	return m.db.ApplySnapshot(m.getPathOfSnapKey(msg))
}

func (m *defaultSnapshotManager) cleanTmp(msg *mraft.SnapshotMessage) error {
	var err error
	tmpFile := m.getTmpPathOfSnapKeyGZ(msg)
	if exist(tmpFile) {
		log.Infof("raftstore-snap[cell-%d]: delete exists snap tmp file, file=<%s>, header=<%s>",
			msg.Header.Cell.ID,
			tmpFile,
			msg.Header.String())
		err = os.RemoveAll(tmpFile)
	}

	if err != nil {
		return err
	}

	return nil
}

func (m *defaultSnapshotManager) check(msg *mraft.SnapshotMessage) error {
	file := m.getTmpPathOfSnapKeyGZ(msg)
	if exist(file) {
		info, err := os.Stat(file)
		if err != nil {
			return errors.Wrapf(err, "")
		}

		if msg.Chunk.FileSize != uint64(info.Size()) {
			return fmt.Errorf("snap file size not match, got=<%d> expect=<%d> path=<%s>",
				info.Size(),
				msg.Chunk.FileSize,
				file)
		}

		return os.Rename(file, m.getPathOfSnapKeyGZ(msg))
	}

	return fmt.Errorf("missing snapshot file, path=%s", file)
}

func exist(name string) bool {
	_, err := os.Stat(name)
	return err == nil
}
<commit_msg>fix: rate limiter produce one token per second<commit_after>// Copyright 2016 DeepFabric, Inc.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// See the License for the specific language governing permissions and
// limitations under the License.

package raftstore

import (
	"fmt"
	"io"
	"os"
	"path/filepath"
	"sync"
	"time"

	"github.com/deepfabric/elasticell/pkg/log"
	"github.com/deepfabric/elasticell/pkg/pb/mraft"
	"github.com/deepfabric/elasticell/pkg/storage"
	"github.com/deepfabric/elasticell/pkg/util"
	"github.com/fagongzi/goetty"
	"github.com/pkg/errors"
	"golang.org/x/net/context"
	"golang.org/x/time/rate"
)

var (
	creating = 1
	sending  = 2
)

// SnapshotManager manager snapshot
type SnapshotManager interface {
	Register(msg *mraft.SnapshotMessage, step int) bool
	Deregister(msg *mraft.SnapshotMessage, step int)
	Create(msg *mraft.SnapshotMessage) error
	Exists(msg *mraft.SnapshotMessage) bool
	WriteTo(msg *mraft.SnapshotMessage, conn goetty.IOSession) (uint64, error)
	CleanSnap(msg *mraft.SnapshotMessage) error
	ReceiveSnapData(msg *mraft.SnapshotMessage) error
	Apply(msg *mraft.SnapshotMessage) error
}

type defaultSnapshotManager struct {
	sync.RWMutex

	limiter *rate.Limiter

	cfg *Cfg
	db  storage.DataEngine
	dir string

	registry map[string]struct{}
}

func newDefaultSnapshotManager(cfg *Cfg, db storage.DataEngine) SnapshotManager {
	dir := cfg.getSnapDir()

	if !exist(dir) {
		if err := os.Mkdir(dir, 0750); err != nil {
			log.Fatalf("raftstore-snap: cannot create dir for snapshot, errors:\n %+v",
				err)
		}
	}

	go func() {
		interval := time.Hour * 2

		for {
			log.Infof("raftstore-snap: start scan gc snap files")

			var paths []string

			err := filepath.Walk(dir, func(path string, f os.FileInfo, err error) error {
				if f == nil {
					return nil
				}

				if f.IsDir() && f.Name() == getSnapDirName() {
					return nil
				}

				var skip error
				if f.IsDir() && f.Name() != getSnapDirName() {
					skip = filepath.SkipDir
				}

				now := time.Now()
				if now.Sub(f.ModTime()) > interval {
					paths = append(paths, path)
				}

				return skip
			})

			if err != nil {
				log.Errorf("raftstore-snap: scan snap file failed, errors:\n%+v",
					err)
			}

			for _, path := range paths {
				err := os.RemoveAll(path)
				if err != nil {
					log.Errorf("raftstore-snap: scan snap file failed, file=<%s>, errors:\n%+v",
						path,
						err)
				}
			}

			time.Sleep(interval)
		}
	}()

	return &defaultSnapshotManager{
		cfg:      cfg,
		limiter:  rate.NewLimiter(rate.Every(time.Second/time.Duration(cfg.LimitSnapChunkRate)), int(cfg.LimitSnapChunkRate)),
		dir:      dir,
		db:       db,
		registry: make(map[string]struct{}),
	}
}

func formatKey(msg *mraft.SnapshotMessage) string {
	return fmt.Sprintf("%d_%d_%d", msg.Header.Cell.ID, msg.Header.Term, msg.Header.Index)
}

func formatKeyStep(msg *mraft.SnapshotMessage, step int) string {
	return fmt.Sprintf("%s_%d", formatKey(msg), step)
}

func (m *defaultSnapshotManager) getPathOfSnapKey(msg *mraft.SnapshotMessage) string {
	return fmt.Sprintf("%s/%s", m.dir, formatKey(msg))
}

func (m *defaultSnapshotManager) getPathOfSnapKeyGZ(msg *mraft.SnapshotMessage) string {
	return fmt.Sprintf("%s.gz", m.getPathOfSnapKey(msg))
}

func (m *defaultSnapshotManager) getTmpPathOfSnapKeyGZ(msg *mraft.SnapshotMessage) string {
	return fmt.Sprintf("%s.tmp", m.getPathOfSnapKey(msg))
}

func (m *defaultSnapshotManager) Register(msg *mraft.SnapshotMessage, step int) bool {
	m.Lock()
	defer m.Unlock()

	fkey := formatKeyStep(msg, step)

	if _, ok := m.registry[fkey]; ok {
		return false
	}

	m.registry[fkey] = emptyStruct
	return true
}

func (m *defaultSnapshotManager) Deregister(msg *mraft.SnapshotMessage, step int) {
	m.Lock()
	defer m.Unlock()

	fkey := formatKeyStep(msg, step)
	delete(m.registry, fkey)
}

func (m *defaultSnapshotManager) inRegistry(msg *mraft.SnapshotMessage, step int) bool {
	m.RLock()
	defer m.RUnlock()

	fkey := formatKeyStep(msg, step)
	_, ok := m.registry[fkey]

	return ok
}

func (m *defaultSnapshotManager) Create(msg *mraft.SnapshotMessage) error {
	path := m.getPathOfSnapKey(msg)
	gzPath := m.getPathOfSnapKeyGZ(msg)
	start := encStartKey(&msg.Header.Cell)
	end := encEndKey(&msg.Header.Cell)

	if !exist(gzPath) {
		if !exist(path) {
			err := m.db.CreateSnapshot(path, start, end)
			if err != nil {
				return errors.Wrapf(err, "")
			}
		}

		err := util.GZIP(path)
		if err != nil {
			return errors.Wrapf(err, "")
		}
	}

	info, err := os.Stat(fmt.Sprintf("%s.gz", path))
	if err != nil {
		return errors.Wrapf(err, "")
	}

	fileSize := uint64(info.Size())
	snapshotSizeHistogram.Observe(float64(fileSize))
	return nil
}

func (m *defaultSnapshotManager) Exists(msg *mraft.SnapshotMessage) bool {
	file := m.getPathOfSnapKeyGZ(msg)
	return exist(file)
}

func (m *defaultSnapshotManager) WriteTo(msg *mraft.SnapshotMessage, conn goetty.IOSession) (uint64, error) {
	file := m.getPathOfSnapKeyGZ(msg)

	if !m.Exists(msg) {
		return 0, fmt.Errorf("missing snapshot file: %s", file)
	}

	info, err := os.Stat(file)
	if err != nil {
		return 0, errors.Wrapf(err, "")
	}
	fileSize := info.Size()

	f, err := os.Open(file)
	if err != nil {
		return 0, err
	}
	defer f.Close()

	var written int64
	buf := make([]byte, m.cfg.LimitSnapChunkBytes)
	ctx := context.TODO()

	log.Infof("raftstore-snap[cell-%d]: try to send snap, header=<%s>,size=<%d>",
		msg.Header.Cell.ID,
		msg.Header.String(),
		fileSize)

	for {
		nr, er := f.Read(buf)
		if nr > 0 {
			dst := &mraft.SnapshotMessage{}
			dst.Header = msg.Header
			dst.Chunk = &mraft.SnapshotChunkMessage{
				Data:     buf[0:nr],
				FileSize: uint64(fileSize),
				First:    0 == written,
				Last:     fileSize == written+int64(nr),
			}

			written += int64(nr)
			err := m.limiter.Wait(ctx)
			if err != nil {
				return 0, err
			}

			err = conn.Write(dst)
			if err != nil {
				return 0, err
			}
		}
		if er != nil {
			if er != io.EOF {
				return 0, er
			}
			break
		}
	}

	log.Infof("raftstore-snap[cell-%d]: send snap complete",
		msg.Header.Cell.ID)
	return uint64(written), nil
}

func (m *defaultSnapshotManager) CleanSnap(msg *mraft.SnapshotMessage) error {
	var err error

	tmpFile := m.getTmpPathOfSnapKeyGZ(msg)
	if exist(tmpFile) {
		log.Infof("raftstore-snap[cell-%d]: delete exists snap tmp file, file=<%s>, header=<%s>",
			msg.Header.Cell.ID,
			tmpFile,
			msg.Header.String())
		err = os.RemoveAll(tmpFile)
	}

	if err != nil {
		return err
	}

	file := m.getPathOfSnapKeyGZ(msg)
	if exist(file) {
		log.Infof("raftstore-snap[cell-%d]: delete exists snap gz file, file=<%s>, header=<%s>",
			msg.Header.Cell.ID,
			file,
			msg.Header.String())
		err = os.RemoveAll(file)
	}

	if err != nil {
		return err
	}

	dir := m.getPathOfSnapKey(msg)
	if exist(dir) {
		log.Infof("raftstore-snap[cell-%d]: delete exists snap dir, file=<%s>, header=<%s>",
			msg.Header.Cell.ID,
			dir,
			msg.Header.String())
		err = os.RemoveAll(dir)
	}

	return err
}

func (m *defaultSnapshotManager) ReceiveSnapData(msg *mraft.SnapshotMessage) error {
	var err error
	var f *os.File

	if msg.Chunk.First {
		err = m.cleanTmp(msg)
	}

	if err != nil {
		return err
	}

	file := m.getTmpPathOfSnapKeyGZ(msg)
	if exist(file) {
		f, err = os.OpenFile(file, os.O_APPEND|os.O_WRONLY, 0600)
		if err != nil {
			f.Close()
			return err
		}
	} else {
		f, err = os.Create(file)
		if err != nil {
			f.Close()
			return err
		}
	}

	n, err := f.Write(msg.Chunk.Data)
	if err != nil {
		f.Close()
		return err
	}

	if n != len(msg.Chunk.Data) {
		f.Close()
		return fmt.Errorf("write snapshot file failed, expect=<%d> actual=<%d>",
			len(msg.Chunk.Data),
			n)
	}

	f.Close()

	if msg.Chunk.Last {
		return m.check(msg)
	}

	return nil
}

func (m *defaultSnapshotManager) Apply(msg *mraft.SnapshotMessage) error {
	file := m.getPathOfSnapKeyGZ(msg)
	if !m.Exists(msg) {
		return fmt.Errorf("missing snapshot file, path=%s", file)
	}

	defer m.CleanSnap(msg)

	err := util.UnGZIP(file, m.dir)
	if err != nil {
		return err
	}
	defer os.RemoveAll(m.getPathOfSnapKey(msg))

	return m.db.ApplySnapshot(m.getPathOfSnapKey(msg))
}

func (m *defaultSnapshotManager) cleanTmp(msg *mraft.SnapshotMessage) error {
	var err error
	tmpFile := m.getTmpPathOfSnapKeyGZ(msg)
	if exist(tmpFile) {
		log.Infof("raftstore-snap[cell-%d]: delete exists snap tmp file, file=<%s>, header=<%s>",
			msg.Header.Cell.ID,
			tmpFile,
			msg.Header.String())
		err = os.RemoveAll(tmpFile)
	}

	if err != nil {
		return err
	}

	return nil
}

func (m *defaultSnapshotManager) check(msg *mraft.SnapshotMessage) error {
	file := m.getTmpPathOfSnapKeyGZ(msg)
	if exist(file) {
		info, err := os.Stat(file)
		if err != nil {
			return errors.Wrapf(err, "")
		}

		if msg.Chunk.FileSize != uint64(info.Size()) {
			return fmt.Errorf("snap file size not match, got=<%d> expect=<%d> path=<%s>",
				info.Size(),
				msg.Chunk.FileSize,
				file)
		}

		return os.Rename(file, m.getPathOfSnapKeyGZ(msg))
	}

	return fmt.Errorf("missing snapshot file, path=%s", file)
}

func exist(name string) bool {
	_, err := os.Stat(name)
	return err == nil
}
<|endoftext|>