<commit_before>package air

import (
	"fmt"
	"io/ioutil"
	"net"
	"os"
	"time"

	yaml "gopkg.in/yaml.v2"
)

// Config is a global set of configs that for an instance of the `Air` for customization.
type Config struct {
	// AppName represens the name of the `Air` instance.
	//
	// The default Value is "air".
	//
	// It's called "app_name" in the config file.
	AppName string

	// DebugMode represents the state of the debug mode enabled of the `Air`. It works only with
	// the default `Logger`.
	//
	// The default value is false.
	//
	// It's called "debug_mode" in the config file.
	DebugMode bool

	// LogEnabled represents the state of the enabled of the `Logger`. It will be forced to the
	// true if the `DebugMode` is true. It works only with the default `Logger`.
	//
	// The default value is false.
	//
	// It's called "log_enabled" in the config file.
	LogEnabled bool

	// LogFormat represents the format of the output content of the `Logger`. It works only with
	// the default `Logger`.
	//
	// The default value is:
	// `{"app_name":"{{.app_name}}","time":"{{.time_rfc3339}}","level":"{{.level}}",` +
	// `"file":"{{.short_file}}","line":"{{.line}}"}`
	//
	// It's called "log_format" in the config file.
	LogFormat string

	// Address represents the TCP address that the HTTP server to listen on.
	//
	// The default value is "localhost:2333".
	//
	// It's called "address" in the config file.
	Address string

	// Listener represens the custom `net.Listener`. If set, the HTTP server accepts connections
	// on it.
	//
	// The default value is nil.
	Listener net.Listener

	// DisableHTTP2 represens the state of the HTTP/2 disabled of the `Air`.
	//
	// The default value is false.
	//
	// It's called "disable_http2" in the config file.
	DisableHTTP2 bool

	// TLSCertFile represents the path of the TLS certificate file.
	//
	// The default value is "".
	//
	// It's called "tls_cert_file" in the config file.
	TLSCertFile string

	// TLSKeyFile represents the path of the TLS key file.
	//
	// The default value is "".
	//
	// It's called "tls_key_file" in the config file.
	TLSKeyFile string

	// ReadTimeout represents the maximum duration before timing out read of the HTTP request.
	//
	// The default value is 0.
	//
	// It's called "read_timeout" in the config file.
	//
	// **It's unit in the config file is SECONDS.**
	ReadTimeout time.Duration

	// WriteTimeout represents the maximum duration before timing out write of the HTTP
	// response.
	//
	// The default value is 0.
	//
	// It's called "write_timeout" in the config file.
	//
	// **It's unit in the config file is SECONDS.**
	WriteTimeout time.Duration

	// TemplateRoot represents the root directory of the HTML templates. It will be parsed into
	// the `Renderer`. It works only with the default `Renderer`.
	//
	// The default value is "templates" that means a subdirectory of the runtime directory.
	//
	// It's called "template_root" in the config file.
	TemplateRoot string

	// TemplateExt represents the file name extension of the HTML templates. It will be used
	// when parsing the HTML templates. It works only with the default `Renderer`.
	//
	// The default value is ".html".
	//
	// It's called "template_ext" in the config file.
	TemplateExt string

	// TemplateLeftDelim represents the left side of the HTML template delimiter. It will be
	// used when parsing the HTML templates. It works only with the default `Renderer`.
	//
	// The default value is "{{".
	//
	// It's called "template_left_delim" in the config file.
	TemplateLeftDelim string

	// TemplateRightDelim represents the right side of the HTML template delimiter. It will be
	// used when parsing the HTML templates. It works only with the default `Renderer`.
	//
	// The default value is "}}".
	//
	// It's called "template_right_delim" in the config file.
	TemplateRightDelim string

	// MinifyTemplate indicates whether to minify the HTML templates before they being parsed
	// into the `Renderer`. It works only with the default `Renderer`. The minify feature
	// powered by the Minify project that can be found at "https://github.com/tdewolff/minify".
	//
	// The default value is false.
	//
	// It's called "minify_template" in the config file.
	MinifyTemplate bool

	// Data represents the data that parsing from the config file. You can use it to access the
	// values in the config file.
	//
	// e.g. Data["foobar"] will accesses the value in the config file called "foobar".
	Data JSONMap
}

// defaultConfig is the default instance of the `Config`.
var defaultConfig = Config{
	AppName: "air",
	LogFormat: `{"app_name":"{{.app_name}}","time":"{{.time_rfc3339}}","level":"{{.level}}",` +
		`"file":"{{.short_file}}","line":"{{.line}}"}`,
	Address:            "localhost:2333",
	TemplateRoot:       "templates",
	TemplateExt:        ".html",
	TemplateLeftDelim:  "{{",
	TemplateRightDelim: "}}",
}

// newConfig returns a pointer of a new instance of the `Config` by parsing the config file that in
// the rumtime directory named "config.yml" or "config.json". It returns the defaultConfig if the
// config file does not exist.
func newConfig() *Config {
	c := defaultConfig
	cfn := "config.yml"
	cfnJSON := "config.json"
	if _, err := os.Stat(cfn); err == nil || os.IsExist(err) {
		c.ParseFile(cfn)
	} else if _, err := os.Stat(cfnJSON); err == nil || os.IsExist(err) {
		c.ParseFile(cfnJSON)
	}
	return &c
}

// Parse parses the src into the c.
func (c *Config) Parse(src string) {
	if err := yaml.Unmarshal([]byte(src), &c.Data); err != nil {
		panic(err)
	}
	c.fillData()
}

// ParseFile parses the config file found in the filename path into the c.
func (c *Config) ParseFile(filename string) {
	if _, err := os.Stat(filename); err != nil && !os.IsExist(err) {
		panic(fmt.Sprintf("the config file %s does not exist", filename))
	}
	b, err := ioutil.ReadFile(filename)
	if err != nil {
		panic(err)
	}
	c.Parse(string(b))
}

// fillData fills the values of the fields from the field `Data` of the c.
func (c *Config) fillData() {
	if an, ok := c.Data["app_name"]; ok {
		c.AppName = an.(string)
	}
	if dm, ok := c.Data["debug_mode"]; ok {
		c.DebugMode = dm.(bool)
	}
	if le, ok := c.Data["log_enabled"]; ok {
		c.LogEnabled = le.(bool)
	}
	if lf, ok := c.Data["log_format"]; ok {
		c.LogFormat = lf.(string)
	}
	if addr, ok := c.Data["address"]; ok {
		c.Address = addr.(string)
	}
	if dh, ok := c.Data["disable_http2"]; ok {
		c.DisableHTTP2 = dh.(bool)
	}
	if tlscf, ok := c.Data["tls_cert_file"]; ok {
		c.TLSCertFile = tlscf.(string)
	}
	if tlskf, ok := c.Data["tls_key_file"]; ok {
		c.TLSKeyFile = tlskf.(string)
	}
	if rt, ok := c.Data["read_timeout"]; ok {
		c.ReadTimeout = time.Duration(rt.(int)) * time.Second
	}
	if wt, ok := c.Data["write_timeout"]; ok {
		c.WriteTimeout = time.Duration(wt.(int)) * time.Second
	}
	if tr, ok := c.Data["template_root"]; ok {
		c.TemplateRoot = tr.(string)
	}
	if te, ok := c.Data["template_ext"]; ok {
		c.TemplateExt = te.(string)
	}
	if tld, ok := c.Data["template_left_delim"]; ok {
		c.TemplateLeftDelim = tld.(string)
	}
	if trd, ok := c.Data["template_right_delim"]; ok {
		c.TemplateRightDelim = trd.(string)
	}
	if mt, ok := c.Data["minify_template"]; ok {
		c.MinifyTemplate = mt.(bool)
	}
}
<commit_msg>refactor: simplify the `newConfig()`<commit_after>package air

import (
	"fmt"
	"io/ioutil"
	"net"
	"os"
	"time"

	yaml "gopkg.in/yaml.v2"
)

// Config is a global set of configs that for an instance of the `Air` for customization.
type Config struct {
	// AppName represens the name of the `Air` instance.
	//
	// The default Value is "air".
	//
	// It's called "app_name" in the config file.
	AppName string

	// DebugMode represents the state of the debug mode enabled of the `Air`. It works only with
	// the default `Logger`.
	//
	// The default value is false.
	//
	// It's called "debug_mode" in the config file.
	DebugMode bool

	// LogEnabled represents the state of the enabled of the `Logger`. It will be forced to the
	// true if the `DebugMode` is true. It works only with the default `Logger`.
	//
	// The default value is false.
	//
	// It's called "log_enabled" in the config file.
	LogEnabled bool

	// LogFormat represents the format of the output content of the `Logger`. It works only with
	// the default `Logger`.
	//
	// The default value is:
	// `{"app_name":"{{.app_name}}","time":"{{.time_rfc3339}}","level":"{{.level}}",` +
	// `"file":"{{.short_file}}","line":"{{.line}}"}`
	//
	// It's called "log_format" in the config file.
	LogFormat string

	// Address represents the TCP address that the HTTP server to listen on.
	//
	// The default value is "localhost:2333".
	//
	// It's called "address" in the config file.
	Address string

	// Listener represens the custom `net.Listener`. If set, the HTTP server accepts connections
	// on it.
	//
	// The default value is nil.
	Listener net.Listener

	// DisableHTTP2 represens the state of the HTTP/2 disabled of the `Air`.
	//
	// The default value is false.
	//
	// It's called "disable_http2" in the config file.
	DisableHTTP2 bool

	// TLSCertFile represents the path of the TLS certificate file.
	//
	// The default value is "".
	//
	// It's called "tls_cert_file" in the config file.
	TLSCertFile string

	// TLSKeyFile represents the path of the TLS key file.
	//
	// The default value is "".
	//
	// It's called "tls_key_file" in the config file.
	TLSKeyFile string

	// ReadTimeout represents the maximum duration before timing out read of the HTTP request.
	//
	// The default value is 0.
	//
	// It's called "read_timeout" in the config file.
	//
	// **It's unit in the config file is SECONDS.**
	ReadTimeout time.Duration

	// WriteTimeout represents the maximum duration before timing out write of the HTTP
	// response.
	//
	// The default value is 0.
	//
	// It's called "write_timeout" in the config file.
	//
	// **It's unit in the config file is SECONDS.**
	WriteTimeout time.Duration

	// TemplateRoot represents the root directory of the HTML templates. It will be parsed into
	// the `Renderer`. It works only with the default `Renderer`.
	//
	// The default value is "templates" that means a subdirectory of the runtime directory.
	//
	// It's called "template_root" in the config file.
	TemplateRoot string

	// TemplateExt represents the file name extension of the HTML templates. It will be used
	// when parsing the HTML templates. It works only with the default `Renderer`.
	//
	// The default value is ".html".
	//
	// It's called "template_ext" in the config file.
	TemplateExt string

	// TemplateLeftDelim represents the left side of the HTML template delimiter. It will be
	// used when parsing the HTML templates. It works only with the default `Renderer`.
	//
	// The default value is "{{".
	//
	// It's called "template_left_delim" in the config file.
	TemplateLeftDelim string

	// TemplateRightDelim represents the right side of the HTML template delimiter. It will be
	// used when parsing the HTML templates. It works only with the default `Renderer`.
	//
	// The default value is "}}".
	//
	// It's called "template_right_delim" in the config file.
	TemplateRightDelim string

	// MinifyTemplate indicates whether to minify the HTML templates before they being parsed
	// into the `Renderer`. It works only with the default `Renderer`. The minify feature
	// powered by the Minify project that can be found at "https://github.com/tdewolff/minify".
	//
	// The default value is false.
	//
	// It's called "minify_template" in the config file.
	MinifyTemplate bool

	// Data represents the data that parsing from the config file. You can use it to access the
	// values in the config file.
	//
	// e.g. Data["foobar"] will accesses the value in the config file called "foobar".
	Data JSONMap
}

// defaultConfig is the default instance of the `Config`.
var defaultConfig = Config{
	AppName: "air",
	LogFormat: `{"app_name":"{{.app_name}}","time":"{{.time_rfc3339}}","level":"{{.level}}",` +
		`"file":"{{.short_file}}","line":"{{.line}}"}`,
	Address:            "localhost:2333",
	TemplateRoot:       "templates",
	TemplateExt:        ".html",
	TemplateLeftDelim:  "{{",
	TemplateRightDelim: "}}",
}

// newConfig returns a pointer of a new instance of the `Config` by parsing the config file that in
// the rumtime directory named "config.yml". It returns the defaultConfig if the config file does
// not exist.
func newConfig() *Config {
	c := defaultConfig
	cfn := "config.yml"
	if _, err := os.Stat(cfn); err == nil || os.IsExist(err) {
		c.ParseFile(cfn)
	}
	return &c
}

// Parse parses the src into the c.
func (c *Config) Parse(src string) {
	if err := yaml.Unmarshal([]byte(src), &c.Data); err != nil {
		panic(err)
	}
	c.fillData()
}

// ParseFile parses the config file found in the filename path into the c.
func (c *Config) ParseFile(filename string) {
	if _, err := os.Stat(filename); err != nil && !os.IsExist(err) {
		panic(fmt.Sprintf("the config file %s does not exist", filename))
	}
	b, err := ioutil.ReadFile(filename)
	if err != nil {
		panic(err)
	}
	c.Parse(string(b))
}

// fillData fills the values of the fields from the field `Data` of the c.
func (c *Config) fillData() {
	if an, ok := c.Data["app_name"]; ok {
		c.AppName = an.(string)
	}
	if dm, ok := c.Data["debug_mode"]; ok {
		c.DebugMode = dm.(bool)
	}
	if le, ok := c.Data["log_enabled"]; ok {
		c.LogEnabled = le.(bool)
	}
	if lf, ok := c.Data["log_format"]; ok {
		c.LogFormat = lf.(string)
	}
	if addr, ok := c.Data["address"]; ok {
		c.Address = addr.(string)
	}
	if dh, ok := c.Data["disable_http2"]; ok {
		c.DisableHTTP2 = dh.(bool)
	}
	if tlscf, ok := c.Data["tls_cert_file"]; ok {
		c.TLSCertFile = tlscf.(string)
	}
	if tlskf, ok := c.Data["tls_key_file"]; ok {
		c.TLSKeyFile = tlskf.(string)
	}
	if rt, ok := c.Data["read_timeout"]; ok {
		c.ReadTimeout = time.Duration(rt.(int)) * time.Second
	}
	if wt, ok := c.Data["write_timeout"]; ok {
		c.WriteTimeout = time.Duration(wt.(int)) * time.Second
	}
	if tr, ok := c.Data["template_root"]; ok {
		c.TemplateRoot = tr.(string)
	}
	if te, ok := c.Data["template_ext"]; ok {
		c.TemplateExt = te.(string)
	}
	if tld, ok := c.Data["template_left_delim"]; ok {
		c.TemplateLeftDelim = tld.(string)
	}
	if trd, ok := c.Data["template_right_delim"]; ok {
		c.TemplateRightDelim = trd.(string)
	}
	if mt, ok := c.Data["minify_template"]; ok {
		c.MinifyTemplate = mt.(bool)
	}
}
<|endoftext|>


<commit_before>package main

import (
	"encoding/json"
	"io/ioutil"
	"os"
	"time"
)

type PoolConfig struct {
	Image    string `json:"image"`
	Capacity int    `json:"capacity"`
}

type Config struct {
	DockerHost          string        `json:"docker_host"`
	SharedPath          string        `json:"shared_path"`
	RunDuration         time.Duration `json:"run_duration"`
	ThrottleQuota       int           `json:"throttle_quota"`
	ThrottleConcurrency int           `json:"throttle_concurrency"`
	NetworkDisabled     bool          `json:"network_disabled"`
	MemoryLimit         int64         `json:"memory_limit"`
	Pools               []PoolConfig  `json:"pools"`
}

func NewConfig() *Config {
	cfg := Config{
		DockerHost: os.Getenv("DOCKER_HOST"),
		SharedPath: os.Getenv("SHARED_PATH"),
	}

	cfg.SharedPath = expandPath(cfg.SharedPath)
	cfg.RunDuration = time.Second * 10
	cfg.ThrottleQuota = 5
	cfg.ThrottleConcurrency = 1
	cfg.NetworkDisabled = true
	cfg.MemoryLimit = 67108864
	cfg.Pools = []PoolConfig{}

	return &cfg
}

func NewConfigFromFile(path string) (*Config, error) {
	data, err := ioutil.ReadFile(path)
	if err != nil {
		return nil, err
	}

	config := Config{}

	err = json.Unmarshal(data, &config)
	return &config, err
}
<commit_msg>Tweak config loaded from file<commit_after>package main

import (
	"encoding/json"
	"io/ioutil"
	"os"
	"time"
)

type PoolConfig struct {
	Image    string `json:"image"`
	Capacity int    `json:"capacity"`
}

type Config struct {
	DockerHost          string        `json:"docker_host"`
	SharedPath          string        `json:"shared_path"`
	RunDuration         time.Duration `json:"run_duration"`
	ThrottleQuota       int           `json:"throttle_quota"`
	ThrottleConcurrency int           `json:"throttle_concurrency"`
	NetworkDisabled     bool          `json:"network_disabled"`
	MemoryLimit         int64         `json:"memory_limit"`
	Pools               []PoolConfig  `json:"pools"`
}

func NewConfig() *Config {
	cfg := Config{
		DockerHost: os.Getenv("DOCKER_HOST"),
		SharedPath: os.Getenv("SHARED_PATH"),
	}

	cfg.SharedPath = expandPath(cfg.SharedPath)
	cfg.RunDuration = time.Second * 10
	cfg.ThrottleQuota = 5
	cfg.ThrottleConcurrency = 1
	cfg.NetworkDisabled = true
	cfg.MemoryLimit = 67108864
	cfg.Pools = []PoolConfig{}

	return &cfg
}

func NewConfigFromFile(path string) (*Config, error) {
	data, err := ioutil.ReadFile(path)
	if err != nil {
		return nil, err
	}

	config := Config{}

	err = json.Unmarshal(data, &config)

	if err == nil {
		config.SharedPath = expandPath(config.SharedPath)
		config.RunDuration = config.RunDuration * time.Second
	}

	return &config, err
}
<|endoftext|>


<commit_before>package google

import (
	"encoding/json"
	"fmt"
	"log"
	"net/http"
	"os"

	"code.google.com/p/google-api-go-client/compute/v1"
	// oauth2 "github.com/rasa/oauth2-fork-b3f9a68"
	"github.com/rasa/oauth2-fork-b3f9a68"

	// oauth2 "github.com/rasa/oauth2-fork-b3f9a68/google"
	"github.com/rasa/oauth2-fork-b3f9a68/google"
)

const clientScopes string = "https://www.googleapis.com/auth/compute"

// Config is the configuration structure used to instantiate the Google
// provider.
type Config struct {
	AccountFile string
	Project     string
	Region      string

	clientCompute *compute.Service
}

func (c *Config) loadAndValidate() error {
	var account accountFile

	// TODO: validation that it isn't blank
	if c.AccountFile == "" {
		c.AccountFile = os.Getenv("GOOGLE_ACCOUNT_FILE")
	}
	if c.Project == "" {
		c.Project = os.Getenv("GOOGLE_PROJECT")
	}
	if c.Region == "" {
		c.Region = os.Getenv("GOOGLE_REGION")
	}

	var f *oauth2.Options
	var err error

	if c.AccountFile != "" {
		if err := loadJSON(&account, c.AccountFile); err != nil {
			return fmt.Errorf(
				"Error loading account file '%s': %s",
				c.AccountFile,
				err)
		}

		// Get the token for use in our requests
		log.Printf("[INFO] Requesting Google token...")
		log.Printf("[INFO]   -- Email: %s", account.ClientEmail)
		log.Printf("[INFO]   -- Scopes: %s", clientScopes)
		log.Printf("[INFO]   -- Private Key Length: %d", len(account.PrivateKey))

		f, err = oauth2.New(
			oauth2.JWTClient(account.ClientEmail, []byte(account.PrivateKey)),
			oauth2.Scope(clientScopes),
			google.JWTEndpoint())

	} else {
		log.Printf("[INFO] Requesting Google token via GCE Service Role...")
		f, err = oauth2.New(google.ComputeEngineAccount(""))

	}

	if err != nil {
		return fmt.Errorf("Error retrieving auth token: %s", err)
	}

	log.Printf("[INFO] Instantiating GCE client...")
	c.clientCompute, err = compute.New(&http.Client{Transport: f.NewTransport()})
	if err != nil {
		return err
	}

	return nil
}

// accountFile represents the structure of the account file JSON file.
type accountFile struct {
	PrivateKeyId string `json:"private_key_id"`
	PrivateKey   string `json:"private_key"`
	ClientEmail  string `json:"client_email"`
	ClientId     string `json:"client_id"`
}

func loadJSON(result interface{}, path string) error {
	f, err := os.Open(path)
	if err != nil {
		return err
	}
	defer f.Close()

	dec := json.NewDecoder(f)
	return dec.Decode(result)
}
<commit_msg>Revert to upstream oauth2<commit_after>package google

import (
	"encoding/json"
	"fmt"
	"log"
	"net/http"
	"os"

	"code.google.com/p/google-api-go-client/compute/v1"
	// oauth2 "github.com/rasa/oauth2-fork-b3f9a68"
	"github.com/golang/oauth2"

	// oauth2 "github.com/rasa/oauth2-fork-b3f9a68/google"
	"github.com/golang/oauth2/google"
)

const clientScopes string = "https://www.googleapis.com/auth/compute"

// Config is the configuration structure used to instantiate the Google
// provider.
type Config struct {
	AccountFile string
	Project     string
	Region      string

	clientCompute *compute.Service
}

func (c *Config) loadAndValidate() error {
	var account accountFile

	// TODO: validation that it isn't blank
	if c.AccountFile == "" {
		c.AccountFile = os.Getenv("GOOGLE_ACCOUNT_FILE")
	}
	if c.Project == "" {
		c.Project = os.Getenv("GOOGLE_PROJECT")
	}
	if c.Region == "" {
		c.Region = os.Getenv("GOOGLE_REGION")
	}

	var f *oauth2.Options
	var err error

	if c.AccountFile != "" {
		if err := loadJSON(&account, c.AccountFile); err != nil {
			return fmt.Errorf(
				"Error loading account file '%s': %s",
				c.AccountFile,
				err)
		}

		// Get the token for use in our requests
		log.Printf("[INFO] Requesting Google token...")
		log.Printf("[INFO]   -- Email: %s", account.ClientEmail)
		log.Printf("[INFO]   -- Scopes: %s", clientScopes)
		log.Printf("[INFO]   -- Private Key Length: %d", len(account.PrivateKey))

		f, err = oauth2.New(
			oauth2.JWTClient(account.ClientEmail, []byte(account.PrivateKey)),
			oauth2.Scope(clientScopes),
			google.JWTEndpoint())

	} else {
		log.Printf("[INFO] Requesting Google token via GCE Service Role...")
		f, err = oauth2.New(google.ComputeEngineAccount(""))

	}

	if err != nil {
		return fmt.Errorf("Error retrieving auth token: %s", err)
	}

	log.Printf("[INFO] Instantiating GCE client...")
	c.clientCompute, err = compute.New(&http.Client{Transport: f.NewTransport()})
	if err != nil {
		return err
	}

	return nil
}

// accountFile represents the structure of the account file JSON file.
type accountFile struct {
	PrivateKeyId string `json:"private_key_id"`
	PrivateKey   string `json:"private_key"`
	ClientEmail  string `json:"client_email"`
	ClientId     string `json:"client_id"`
}

func loadJSON(result interface{}, path string) error {
	f, err := os.Open(path)
	if err != nil {
		return err
	}
	defer f.Close()

	dec := json.NewDecoder(f)
	return dec.Decode(result)
}
<|endoftext|>


<commit_before>package specs

// Spec is the base configuration for the container.  It specifies platform
// independent configuration. This information must be included when the
// bundle is packaged for distribution.
type Spec struct {
	// Version is the version of the specification that is supported.
	Version string `json:"version"`
	// Platform is the host information for OS and Arch.
	Platform Platform `json:"platform"`
	// Process is the container's main process.
	Process Process `json:"process"`
	// Root is the root information for the container's filesystem.
	Root Root `json:"root"`
	// Hostname is the container's host name.
	Hostname string `json:"hostname,omitempty"`
	// Mounts profile configuration for adding mounts to the container's filesystem.
	Mounts []MountPoint `json:"mounts"`
}

// Process contains information to start a specific application inside the container.
type Process struct {
	// Terminal creates an interactive terminal for the container.
	Terminal bool `json:"terminal"`
	// User specifies user information for the process.
	User User `json:"user"`
	// Args specifies the binary and arguments for the application to execute.
	Args []string `json:"args"`
	// Env populates the process environment for the process.
	Env []string `json:"env,omitempty"`
	// Cwd is the current working directory for the process and must be
	// relative to the container's root.
	Cwd string `json:"cwd,omitempty"`
}

// Root contains information about the container's root filesystem on the host.
type Root struct {
	// Path is the absolute path to the container's root filesystem.
	Path string `json:"path"`
	// Readonly makes the root filesystem for the container readonly before the process is executed.
	Readonly bool `json:"readonly"`
}

// Platform specifies OS and arch information for the host system that the container
// is created for.
type Platform struct {
	// OS is the operating system.
	OS string `json:"os"`
	// Arch is the architecture
	Arch string `json:"arch"`
}

// MountPoint describes a directory that may be fullfilled by a mount in the runtime.json.
type MountPoint struct {
	// Name is a unique descriptive identifier for this mount point.
	Name string `json:"name"`
	// Path specifies the path of the mount. The path and child directories MUST exist, a runtime MUST NOT create directories automatically to a mount point.
	Path string `json:"path"`
}
<commit_msg>config: corresponding change for required field<commit_after>package specs

// Spec is the base configuration for the container.  It specifies platform
// independent configuration. This information must be included when the
// bundle is packaged for distribution.
type Spec struct {
	// Version is the version of the specification that is supported.
	Version string `json:"version"`
	// Platform is the host information for OS and Arch.
	Platform Platform `json:"platform"`
	// Process is the container's main process.
	Process Process `json:"process"`
	// Root is the root information for the container's filesystem.
	Root Root `json:"root"`
	// Hostname is the container's host name.
	Hostname string `json:"hostname,omitempty"`
	// Mounts profile configuration for adding mounts to the container's filesystem.
	Mounts []MountPoint `json:"mounts"`
}

// Process contains information to start a specific application inside the container.
type Process struct {
	// Terminal creates an interactive terminal for the container.
	Terminal bool `json:"terminal"`
	// User specifies user information for the process.
	User User `json:"user"`
	// Args specifies the binary and arguments for the application to execute.
	Args []string `json:"args"`
	// Env populates the process environment for the process.
	Env []string `json:"env,omitempty"`
	// Cwd is the current working directory for the process and must be
	// relative to the container's root.
	Cwd string `json:"cwd"`
}

// Root contains information about the container's root filesystem on the host.
type Root struct {
	// Path is the absolute path to the container's root filesystem.
	Path string `json:"path"`
	// Readonly makes the root filesystem for the container readonly before the process is executed.
	Readonly bool `json:"readonly"`
}

// Platform specifies OS and arch information for the host system that the container
// is created for.
type Platform struct {
	// OS is the operating system.
	OS string `json:"os"`
	// Arch is the architecture
	Arch string `json:"arch"`
}

// MountPoint describes a directory that may be fullfilled by a mount in the runtime.json.
type MountPoint struct {
	// Name is a unique descriptive identifier for this mount point.
	Name string `json:"name"`
	// Path specifies the path of the mount. The path and child directories MUST exist, a runtime MUST NOT create directories automatically to a mount point.
	Path string `json:"path"`
}
<|endoftext|>


<commit_before>package vervet

type Config interface {
    GetUrlPattern() (string, error)
    GetLogIdLiteral() (string, error)
    GetErrorCodeLiteral() (string, error)
    GetErrorMessageLiteral() (string, error)
    GetTimeCostLiteral() () (string, error)
    GetRequestUrlLiteral() (string, error)
}
<commit_msg>fix literal error<commit_after>package vervet

type Config interface {
    GetUrlPattern() (string, error)
    GetLogIdLiteral() (string, error)
    GetErrorCodeLiteral() (string, error)
    GetErrorMessageLiteral() (string, error)
    GetTimeCostLiteral() (string, error)
    GetRequestUrlLiteral() (string, error)
}
<|endoftext|>


<commit_before>package config

import (
    "encoding/json"
    "io/ioutil"
)

type Config struct {
    Nick        string
    Host        string
    Networks    []string
    Servers     map[string] []string
    Channels    map[string] []string
    Passwords   map[string] string
    Plugins     []string
    Ignore      []string
    Logpath     string
}

func ReadConfig(path string) (Config, error) {
    var config Config

    data, err := ioutil.ReadFile(path)
    if err != nil {
        return config, err
    }

    err = json.Unmarshal(data, &config)
    if err != nil {
        return config, err
    }

    return config, nil
}
<commit_msg>Expose irc "realname" to configuration<commit_after>package config

import (
    "encoding/json"
    "io/ioutil"
)

type Config struct {
    Nick        string
    Host        string
    RealName    string
    Networks    []string
    Servers     map[string] []string
    Channels    map[string] []string
    Passwords   map[string] string
    Plugins     []string
    Ignore      []string
    Logpath     string
}

func ReadConfig(path string) (Config, error) {
    var config Config

    data, err := ioutil.ReadFile(path)
    if err != nil {
        return config, err
    }

    err = json.Unmarshal(data, &config)
    if err != nil {
        return config, err
    }

    return config, nil
}
<|endoftext|>


<commit_before>package wellington

import (
	"bufio"
	"bytes"
	"errors"
	"io"
	"io/ioutil"
	"os"
	"path/filepath"
	"regexp"
	"strings"

	"github.com/wellington/wellington/context"
)

// ImportPath accepts a directory and file path to find partials for importing.
// Returning a new pwd and string of the file contents.
// File can contain a directory and should be evaluated if
// successfully found.
// Dir is used to provide relative context to the importee.  If no file is found
// pwd is echoed back.
//
// Paths are looked up in the following order:
// {includepath}/_file.scss
// {includePath}/_file.sass
// {includepath}/file.scss
// {includePath}/file.sass
// {Dir{dir+file}}/_{Base{file}}.scss
// {Dir{dir+file}}/_{Base{file}}.sass
// {Dir{dir+file}}/{Base{file}}.scss
// {Dir{dir+file}}/{Base{file}}.sass
func (p *Parser) ImportPath(dir, file string) (string, string, error) {
	baseerr := ""
	r, fpath, err := importPath(dir, file)
	if err == nil {
		p.PartialMap.AddRelation(p.MainFile, fpath)
		contents, _ := ioutil.ReadAll(r)
		return filepath.Dir(fpath), string(contents), nil
	}
	rel, _ := filepath.Rel(p.SassDir, fpath)
	if rel == "" {
		rel = "./"
	}
	baseerr += rel + "\n"
	if os.IsNotExist(err) {
		// Look through the import path for the file
		for _, lib := range p.Includes {
			r, pwd, err := importPath(lib, file)
			if err == nil {
				p.PartialMap.AddRelation(p.MainFile, fpath)
				bs, _ := ioutil.ReadAll(r)
				return pwd, string(bs), nil
			}
		}
	}
	// Ignore failures on compass
	re := regexp.MustCompile("compass\\/?")
	if re.Match([]byte(file)) {
		return filepath.Dir(fpath), "", nil //errors.New("compass")
	}
	if file == "images" {
		return filepath.Dir(fpath), "", nil
	}

	baseerr += strings.Join(p.Includes, "\n")
	return filepath.Dir(fpath), "",
		errors.New("Could not import: " + file + "\nTried:\n" + baseerr)
}

// Attempt _{}.scss, _{}.sass, {}.scss, {}.sass paths and return
// reader if found
func importPath(dir, file string) (io.Reader, string, error) {
	spath, _ := filepath.Abs(dir + "/" + file)
	pwd := filepath.Dir(spath)
	base := filepath.Base(spath)

	fpath := filepath.Join(pwd, "_"+base+".scss")
	if r, err := readSass(fpath); err == nil {
		return r, fpath, err
	}

	fpath = filepath.Join(pwd, base+".scss")
	if r, err := readSass(fpath); err == nil {
		return r, fpath, err
	}

	fpath = filepath.Join(pwd, "_"+base+".sass")
	if r, err := readSass(fpath); err == nil {
		return r, fpath, err
	}

	fpath = filepath.Join(pwd, base+".sass")
	if r, err := readSass(fpath); err == nil {
		return r, fpath, err
	}

	return nil, pwd, os.ErrNotExist
}

func readSassBytes(path string) ([]byte, error) {
	reader, err := readSass(path)
	if err != nil {
		return nil, err
	}
	return ioutil.ReadAll(reader)
}

// readSass retrives a file from path. If found, it converts Sass
// to Scss or returns found Scss;
func readSass(path string) (io.Reader, error) {
	file, err := os.Open(path)
	if err != nil {
		return nil, err
	}
	defer file.Close()
	return ToScssReader(file)
}

// ToScssReader ...
func ToScssReader(r io.Reader) (io.Reader, error) {
	var (
		buf bytes.Buffer
	)
	tr := io.TeeReader(r, &buf)

	if IsSass(&tr) {
		pr, w := io.Pipe()
		go func() {
			context.ToScss(io.MultiReader(&buf, r), w)
			w.Close()
		}()
		return pr, nil
	}
	mr := io.MultiReader(&buf, r)

	return mr, nil
}

// IsSass determines if the given reader is Sass (not Scss).
// This is predicted by the presence of semicolons
func IsSass(ir *io.Reader) bool {
	r := bufio.NewReader(*ir)
	for {
		line, err := r.ReadString('\n')
		clean := strings.TrimSpace(line)
		// Errors, empty file probably
		if err != nil {
			return false
		}
		if strings.HasSuffix(clean, "{") ||
			strings.HasSuffix(clean, "}") {
			continue
		}
		if strings.HasSuffix(clean, ";") {
			return false
		}
		// Probably Sass, say so
		return true
	}
}
<commit_msg>resolve race condition<commit_after>package wellington

import (
	"bufio"
	"bytes"
	"errors"
	"io"
	"io/ioutil"
	"os"
	"path/filepath"
	"regexp"
	"strings"

	"github.com/wellington/wellington/context"
)

// ImportPath accepts a directory and file path to find partials for importing.
// Returning a new pwd and string of the file contents.
// File can contain a directory and should be evaluated if
// successfully found.
// Dir is used to provide relative context to the importee.  If no file is found
// pwd is echoed back.
//
// Paths are looked up in the following order:
// {includepath}/_file.scss
// {includePath}/_file.sass
// {includepath}/file.scss
// {includePath}/file.sass
// {Dir{dir+file}}/_{Base{file}}.scss
// {Dir{dir+file}}/_{Base{file}}.sass
// {Dir{dir+file}}/{Base{file}}.scss
// {Dir{dir+file}}/{Base{file}}.sass
func (p *Parser) ImportPath(dir, file string) (string, string, error) {
	baseerr := ""
	r, fpath, err := importPath(dir, file)
	if err == nil {
		p.PartialMap.AddRelation(p.MainFile, fpath)
		contents, _ := ioutil.ReadAll(r)
		return filepath.Dir(fpath), string(contents), nil
	}
	rel, _ := filepath.Rel(p.SassDir, fpath)
	if rel == "" {
		rel = "./"
	}
	baseerr += rel + "\n"
	if os.IsNotExist(err) {
		// Look through the import path for the file
		for _, lib := range p.Includes {
			r, pwd, err := importPath(lib, file)
			if err == nil {
				p.PartialMap.AddRelation(p.MainFile, fpath)
				bs, _ := ioutil.ReadAll(r)
				return pwd, string(bs), nil
			}
		}
	}
	// Ignore failures on compass
	re := regexp.MustCompile("compass\\/?")
	if re.Match([]byte(file)) {
		return filepath.Dir(fpath), "", nil //errors.New("compass")
	}
	if file == "images" {
		return filepath.Dir(fpath), "", nil
	}

	baseerr += strings.Join(p.Includes, "\n")
	return filepath.Dir(fpath), "",
		errors.New("Could not import: " + file + "\nTried:\n" + baseerr)
}

// Attempt _{}.scss, _{}.sass, {}.scss, {}.sass paths and return
// reader if found
func importPath(dir, file string) (io.Reader, string, error) {
	spath, _ := filepath.Abs(dir + "/" + file)
	pwd := filepath.Dir(spath)
	base := filepath.Base(spath)

	fpath := filepath.Join(pwd, "_"+base+".scss")
	if r, err := readSass(fpath); err == nil {
		return r, fpath, err
	}

	fpath = filepath.Join(pwd, base+".scss")
	if r, err := readSass(fpath); err == nil {
		return r, fpath, err
	}

	fpath = filepath.Join(pwd, "_"+base+".sass")
	if r, err := readSass(fpath); err == nil {
		return r, fpath, err
	}

	fpath = filepath.Join(pwd, base+".sass")
	if r, err := readSass(fpath); err == nil {
		return r, fpath, err
	}

	return nil, pwd, os.ErrNotExist
}

func readSassBytes(path string) ([]byte, error) {
	reader, err := readSass(path)
	if err != nil {
		return nil, err
	}
	return ioutil.ReadAll(reader)
}

// readSass retrives a file from path. If found, it converts Sass
// to Scss or returns found Scss;
func readSass(path string) (io.Reader, error) {
	file, err := os.Open(path)
	if err != nil {
		return nil, err
	}
	defer file.Close()
	return ToScssReader(file)
}

// ToScssReader ...
func ToScssReader(r io.Reader) (io.Reader, error) {
	var (
		buf bytes.Buffer
	)
	tr := io.TeeReader(r, &buf)

	if IsSass(&tr) {

		//This code causes race conditions, buffer until problem resolved
		// pr, w := io.Pipe()
		// go func(r io.Reader, w *io.PipeWriter, buf bytes.Buffer) {
		// 	context.ToScss(io.MultiReader(&buf, r), w)
		// 	w.Close()
		// }(r, w, buf)
		// return pr, nil

		var ibuf bytes.Buffer
		context.ToScss(io.MultiReader(&buf, r), &ibuf)
		return &ibuf, nil
	}
	mr := io.MultiReader(&buf, r)

	return mr, nil
}

// IsSass determines if the given reader is Sass (not Scss).
// This is predicted by the presence of semicolons
func IsSass(ir *io.Reader) bool {
	r := bufio.NewReader(*ir)
	for {
		line, err := r.ReadString('\n')
		clean := strings.TrimSpace(line)
		// Errors, empty file probably
		if err != nil {
			return false
		}
		if strings.HasSuffix(clean, "{") ||
			strings.HasSuffix(clean, "}") {
			continue
		}
		if strings.HasSuffix(clean, ";") {
			return false
		}
		// Probably Sass, say so
		return true
	}
}
<|endoftext|>


<commit_before>package main

import (
	"github.com/spf13/viper"
	"logger/stderr"
	"reflect"
)

type Config struct {
	Servers map[string]Server
}

func init() {
	viper.SetConfigName("config")
	viper.AddConfigPath("/etc/mouse/")
	viper.AddConfigPath("$HOME/.mouse")
	viper.AddConfigPath(".")

	if err := viper.ReadInConfig(); err != nil {
		stderr.Fatalf("Could not read config file:", err)
	}

	// Temporary(?) fix for HCL problems.
	//
	// As of me writing this, the problem is that currently the HCL parser adds
	// extra arrays for some weird reason.
	//
	// This:
	//
	// servers "fc00" {
	//   ...
	// }
	//
	// Is converted to this [[map:[]]] when it should be converted to [map:[]]
	// on every map.
	//
	// @see https://github.com/hashicorp/hcl/pull/24#issuecomment-69821965
	servers := viper.Get("servers")
	if reflect.ValueOf(servers).Kind() == reflect.Slice {
		servers = servers.([]map[string]interface{})[0]
		for key, s := range servers.(map[string]interface{}) {
			if reflect.ValueOf(s).Kind() == reflect.Slice {
				servers.(map[string]interface{})[key] = s.([]map[string]interface{})[0]
				for k, v := range servers.(map[string]interface{})[key].(map[string]interface{}) {
					if reflect.ValueOf(v).Kind() == reflect.Slice {
						switch v.(type) {
						case []interface{}:
							continue
						case []map[string]interface{}:
							for p, b := range v.([]map[string]interface{})[0] {
								v.([]map[string]interface{})[0][p] = b.([]map[string]interface{})[0]
							}

							servers.(map[string]interface{})[key].(map[string]interface{})[k] = v.([]map[string]interface{})[0]
						}
					}
				}
			}
		}

		viper.Set("servers", servers)
	}

	if err := viper.Unmarshal(&config); err != nil {
		stderr.Fatalf("Could not unmarshal config:", err)
	}
}
<commit_msg>Remove ugly code<commit_after>package main

import (
	"github.com/spf13/viper"
	"logger/stderr"
)

type Config struct {
	Servers map[string]Server
}

func init() {
	viper.SetConfigName("config")
	viper.AddConfigPath("/etc/mouse/")
	viper.AddConfigPath("$HOME/.mouse")
	viper.AddConfigPath(".")

	if err := viper.ReadInConfig(); err != nil {
		stderr.Fatalf("Could not read config file:", err)
	}

	if err := viper.Unmarshal(&config); err != nil {
		stderr.Fatalf("Could not unmarshal config:", err)
	}
}
<|endoftext|>


<commit_before>package main

import (
	"net/http"
	"html/template"
	"github.com/julienschmidt/httprouter"
	"os"
	"bytes"
	"io"
	"fmt"
	"encoding/json"
	"io/ioutil"
	"net"
	"strconv"
	"github.com/russross/blackfriday"
	"path/filepath"
	"github.com/nfnt/resize"
	"path"
	"image/jpeg"
	"strings"
)

func Template(path string) string {
	return gApplicationState.Configuration.Templates + path;
}

func BaseContext(r *http.Request) *Page {
	stat, _ := os.Stat(gApplicationState.Configuration.Assets)
	if stat.ModTime().After(gApplicationState.AssetModificationTime) {
		gApplicationState.AssetModificationTime = stat.ModTime()
		gApplicationState.Page.UnsafeTemplateData,
			gApplicationState.Page.SafeTemplateJs,
			gApplicationState.Page.SafeTemplateCss =
			loadResources(gApplicationState.Configuration.Assets)
	}
	page := gApplicationState.Page
	page.Platform = getPlatform(r.UserAgent())
	page.Route = r.URL.Path
	page.Parameters = map[string]string{}
	page.Parameters["ExplicitRuntimeMode"] =
		gApplicationState.Configuration.Mode
	return &page
}

func LazyLoadTemplate(templateName string) {
	if (gApplicationState.Templates[templateName] == nil) {
		gApplicationState.Templates[templateName] =
			LoadTemplate(templateName, template.New(templateName))
	}
}

func LoadTemplate(templateName string, t *template.Template) *template.Template {
	file, load := readFileMemoized(Template(templateName))
	if (load) {
		return template.Must(t.New(templateName).Parse(file))
	}
	return t;
}

func LazyLoadLayout() {
	if (gApplicationState.Templates["template"] == nil) {
		t := template.New("template")
		LoadTemplate("layout/components/head.gohtml", t)
		LoadTemplate("layout/components/header.gohtml", t)
		LoadTemplate("layout/components/main.gohtml", t)
		LoadTemplate("layout/components/footer.gohtml", t)
		LoadTemplate("layout/layout.gohtml", t)
		gApplicationState.Templates["template"] = t
	}
}

func Render(w io.Writer, templateName string, page *Page) {
	LazyLoadLayout()
	LazyLoadTemplate(templateName)

	buffer := &bytes.Buffer{}
	gApplicationState.Templates[templateName].Execute(buffer, page)
	page.InheritedHTML = template.HTML(buffer.Bytes())
	err := gApplicationState.Templates["template"].ExecuteTemplate(w, "layout/layout.gohtml", page)
	if err != nil {
		fmt.Println(err)
	}
}

func getIndex(w http.ResponseWriter, r *http.Request, p httprouter.Params) {
	context := BaseContext(r)
	context.NavbarSelected = 0
	all := getParadisePackages(
		gApplicationState.Configuration.Data,
	)
	context.Packages = []Package{}
	for i := 0; i < len(all); i++ {
		if(all[i].ShowOnIndexPage) {
			context.Packages = append(context.Packages, all[i])
		}
	}
	Render(w, "index.gohtml", context)
}

func getPrices(w http.ResponseWriter, r *http.Request, p httprouter.Params) {
	context := BaseContext(r)
	context.NavbarSelected = 1

	file, _ := readFileBytesMemoized(
		gApplicationState.Configuration.Data + "prices/prices.md",
	)
	html := blackfriday.MarkdownBasic(
		file,
	)
	context.RenderedPricesMarkdown =
		template.HTML(
			html,
		)
	context.Parameters["markdownHTML"] = string(html)
	Render(w, "prices.gohtml", context);
}
func getPackages(w http.ResponseWriter, r *http.Request, p httprouter.Params) {
	context := BaseContext(r)
	context.NavbarSelected = 2
	all := getParadisePackages(
		gApplicationState.Configuration.Data,
	)
	context.Packages = []Package{}
	for i := 0; i < len(all); i++ {
		if(all[i].ShowOnPackagePage) {
			context.Packages = append(context.Packages, all[i])
		}
	}
	Render(w, "packages.gohtml", context)
}

func getPackage(w http.ResponseWriter, r *http.Request, p httprouter.Params) {
	context := BaseContext(r)
	context.NavbarSelected = 0
	context.PackageDetails = getParadisePackageByUrl(
		gApplicationState.Configuration.Data,
		p.ByName("url"),
	)
	context.Route = "/package/:url"

	if (context.PackageDetails != nil) {
		file, _ := readFileBytesMemoized(
			gApplicationState.Configuration.Data + context.PackageDetails.PageDetailsMarkdown,
		)

		html := blackfriday.MarkdownBasic(
			file,
		);

		context.RenderedPackageMarkdown =
			template.HTML(
				html,
			)

		context.RenderedPackageCover = template.HTMLAttr(
			context.PackageDetails.PageDetailsCover,
		)

		context.Parameters["url"] = context.PackageDetails.Url;
		context.Parameters["id"] = strconv.Itoa(context.PackageDetails.Id)
		context.Parameters["markdownHTML"] = string(html)
		context.Parameters["cover"] = context.PackageDetails.PageDetailsCover
	}

	Render(w, "package.gohtml", context)
}

func getRestaurant(w http.ResponseWriter, r *http.Request, p httprouter.Params) {
	context := BaseContext(r)
	context.NavbarSelected = 3
	Render(w, "restaurant.gohtml", context)
}
func getLocation(w http.ResponseWriter, r *http.Request, p httprouter.Params) {
	context := BaseContext(r)
	context.NavbarSelected = 4
	file, _ := readFileBytesMemoized(
		gApplicationState.Configuration.Data + "location/location.md",
	)
	html := blackfriday.MarkdownBasic(
		file,
	)
	context.RenderedLocationMarkdown =
		template.HTML(
			html,
		)
	context.Parameters["markdownHTML"] = string(html)
	if (gApplicationState.Configuration.GoogleApiKey != nil) {
		context.Parameters["GoogleApiKey"] = *gApplicationState.Configuration.GoogleApiKey
	}
	Render(w, "location.gohtml", context)
}

func getGallery(w http.ResponseWriter, r *http.Request, p httprouter.Params) {
	context := BaseContext(r)
	context.NavbarSelected = 5
	Render(w, "gallery.gohtml", context)
}

func getApiPackage(w http.ResponseWriter, _ *http.Request, p httprouter.Params) {
	id, err := strconv.Atoi(p.ByName("id"));
	pack := (*Package)(nil)
	if (err == nil) {
		pack = getParadisePackage(
			gApplicationState.Configuration.Data,
			id,
		)
	}

	jData, _ := json.Marshal(pack)
	w.Header().Set("Content-Type", "application/json")
	w.Write(jData)
}

var gApplicationState *ApplicationState

func loadResources(filename string) (UnsafeTemplateData, SafeTemplateJs, SafeTemplateCss) {
	assets, err := ioutil.ReadFile(filename)
	runtimeAssert(err)
	m := make(map[string]VersionedScript)
	n := make(UnsafeTemplateData)
	o := make(SafeTemplateJs)
	p := make(SafeTemplateCss)
	err = json.Unmarshal(assets, &m)
	runtimeAssert(err)

	if (m["inline_sync_top"].Js != "") {
		file, _ := readFileMemoized("public/" + m["inline_sync_top"].Js)
		o["inline_sync_js_top"] =
			template.JS(file)
	}
	if (m["inline_sync_top"].Css != "") {
		file, _ := readFileMemoized("public/" + m["inline_sync_top"].Css)
		p["inline_sync_css_top"] =
			template.CSS(file)
	}

	if (m["async"].Js != "") {
		n["async_js"] = "/public/" + m["async"].Js
	}

	if (m["async"].Css != "") {
		n["async_css"] = "/public/" + m["async"].Css
	}
	return n, o, p
}

func getApiPackages(w http.ResponseWriter, r *http.Request, _ httprouter.Params) {
	jData, _ := json.Marshal(
		getParadisePackages(
			gApplicationState.Configuration.Data,
		),
	)
	w.Header().Set("Content-Type", "application/json")
	w.Write(jData)
}

func getApiPhotos(w http.ResponseWriter, r *http.Request, _ httprouter.Params) {
	photos := []Photo{}
	filepath.Walk(
		gApplicationState.Configuration.Data + "gallery/images/",
		func(stringPath string, info os.FileInfo, err error) error {
			stringPath = path.Clean(filepath.ToSlash(stringPath))

			if (err != nil) {
				return err
			}

			if (info.IsDir()) {
				return nil
			}
			_, file := path.Split(stringPath)
			ext := strings.ToLower(path.Ext(stringPath))

			if (ext != ".jpg" && ext != ".jpeg") {
				return nil
			}

			if _, err := os.Stat(gApplicationState.Configuration.Data + "gallery/full/" + file); os.IsNotExist(err) {
				photo, err := os.Open(stringPath)
				runtimeAssert(err)
				img, err := jpeg.Decode(photo)
				photo.Close()
				m := resize.Resize(1200, 0, img, resize.Lanczos3)
				out, err := os.Create(gApplicationState.Configuration.Data + "gallery/full/" + file)
				runtimeAssert(err)
				defer out.Close()
				jpeg.Encode(out, m, nil)
			}

			if _, err := os.Stat(gApplicationState.Configuration.Data + "gallery/thumbnails/" + file); os.IsNotExist(err) {
				photo, err := os.Open(stringPath)
				runtimeAssert(err)
				img, err := jpeg.Decode(photo)
				photo.Close()
				m := resize.Resize(400, 0, img, resize.Lanczos3)
				out, err := os.Create(gApplicationState.Configuration.Data + "gallery/thumbnails/" + file)
				runtimeAssert(err)
				defer out.Close()
				jpeg.Encode(out, m, nil)
			}

			photos = append(photos, Photo{
				Thumbnail: "/static/gallery/thumbnails/" + file,
				FullPicture: "/static/gallery/full/" + file,
			})
			return err
		},
	)
	jData, _ := json.Marshal(
		photos,
	)
	w.Header().Set("Content-Type", "application/json")
	w.Write(jData)
}

func redirectToHTTPS(w http.ResponseWriter, r *http.Request) {
	ssl := gApplicationState.Configuration.SSL;
	port := ssl.Port;
	host := gApplicationState.Configuration.Host;
	toURL := "https://" + net.JoinHostPort(host, strconv.Itoa(port));
	toURL += r.URL.RequestURI()
	w.Header().Set("Connection", "close")
	http.Redirect(w, r, toURL, http.StatusMovedPermanently)
}

func runApplicationSimple(applicationState *ApplicationState) {
	gApplicationState = applicationState
	router := httprouter.New();

	router.GET("/", getIndex)
	router.GET("/prices", getPrices)
	router.GET("/packages", getPackages)
	router.GET("/package/:url", getPackage)
	router.GET("/restaurant", getRestaurant)
	router.GET("/location", getLocation)
	router.GET("/gallery", getGallery)

	router.GET("/api/package", getApiPackages)
	router.GET("/api/package/:id", getApiPackage)

	router.GET("/api/photo", getApiPhotos)

	router.ServeFiles("/public/*filepath", http.Dir(applicationState.Configuration.Public))
	router.ServeFiles("/static/*filepath", http.Dir(applicationState.Configuration.Data))
	router.NotFound = http.FileServer(http.Dir(applicationState.Configuration.Data + "public/"))

	configuration := applicationState.Configuration;
	ssl := configuration.SSL;
	httpAddress := net.JoinHostPort(configuration.Host, strconv.Itoa(configuration.Port))

	if ssl != nil {
		tlsAddress := net.JoinHostPort(configuration.Host, strconv.Itoa(ssl.Port))
		fmt.Fprintf(os.Stdout, "Listening on %s...\n", tlsAddress)
		go http.ListenAndServeTLS(tlsAddress, ssl.Cert, ssl.Key, router)

		fmt.Fprintf(os.Stdout, "Listening on %s...\n", httpAddress)
		http.ListenAndServe(httpAddress, http.HandlerFunc(redirectToHTTPS));
	} else {
		fmt.Fprintf(os.Stdout, "Listening on %s...\n", httpAddress)
		http.ListenAndServe(httpAddress, router)
	}
}
<commit_msg>Gzipped content<commit_after>package main

import (
	"net/http"
	"html/template"
	"github.com/julienschmidt/httprouter"
	"os"
	"bytes"
	"io"
	"fmt"
	"encoding/json"
	"io/ioutil"
	"net"
	"strconv"
	"github.com/russross/blackfriday"
	"path/filepath"
	"github.com/nfnt/resize"
	"path"
	"image/jpeg"
	"strings"
	"compress/gzip"
)

func Template(path string) string {
	return gApplicationState.Configuration.Templates + path;
}

func BaseContext(r *http.Request) *Page {
	stat, _ := os.Stat(gApplicationState.Configuration.Assets)
	if stat.ModTime().After(gApplicationState.AssetModificationTime) {
		gApplicationState.AssetModificationTime = stat.ModTime()
		gApplicationState.Page.UnsafeTemplateData,
			gApplicationState.Page.SafeTemplateJs,
			gApplicationState.Page.SafeTemplateCss =
			loadResources(gApplicationState.Configuration.Assets)
	}
	page := gApplicationState.Page
	page.Platform = getPlatform(r.UserAgent())
	page.Route = r.URL.Path
	page.Parameters = map[string]string{}
	page.Parameters["ExplicitRuntimeMode"] =
		gApplicationState.Configuration.Mode
	return &page
}

func LazyLoadTemplate(templateName string) {
	if (gApplicationState.Templates[templateName] == nil) {
		gApplicationState.Templates[templateName] =
			LoadTemplate(templateName, template.New(templateName))
	}
}

func LoadTemplate(templateName string, t *template.Template) *template.Template {
	file, load := readFileMemoized(Template(templateName))
	if (load) {
		return template.Must(t.New(templateName).Parse(file))
	}
	return t;
}

func LazyLoadLayout() {
	if (gApplicationState.Templates["template"] == nil) {
		t := template.New("template")
		LoadTemplate("layout/components/head.gohtml", t)
		LoadTemplate("layout/components/header.gohtml", t)
		LoadTemplate("layout/components/main.gohtml", t)
		LoadTemplate("layout/components/footer.gohtml", t)
		LoadTemplate("layout/layout.gohtml", t)
		gApplicationState.Templates["template"] = t
	}
}

func Render(w io.Writer, templateName string, page *Page) {
	LazyLoadLayout()
	LazyLoadTemplate(templateName)

	buffer := &bytes.Buffer{}
	gApplicationState.Templates[templateName].Execute(buffer, page)
	page.InheritedHTML = template.HTML(buffer.Bytes())
	err := gApplicationState.Templates["template"].ExecuteTemplate(w, "layout/layout.gohtml", page)
	if err != nil {
		fmt.Println(err)
	}
}

type gzipResponseWriter struct {
	io.Writer
	http.ResponseWriter
}

func (w gzipResponseWriter) Write(b []byte) (int, error) {
	if "" == w.Header().Get("Content-Type") {
		// If no content type, apply sniffing algorithm to un-gzipped body.
		w.Header().Set("Content-Type", http.DetectContentType(b))
	}
	return w.Writer.Write(b)
}

func makeGzipHandler(fn httprouter.Handle) httprouter.Handle {
	return func(w http.ResponseWriter, r *http.Request, p httprouter.Params) {
		if !strings.Contains(r.Header.Get("Accept-Encoding"), "gzip") {
			fn(w, r, p)
			return
		}
		w.Header().Set("Content-Encoding", "gzip")
		gz := gzip.NewWriter(w)
		defer gz.Close()
		gzr := gzipResponseWriter{Writer: gz, ResponseWriter: w}
		fn(gzr, r, p)
	}
}

func getIndex(w http.ResponseWriter, r *http.Request, p httprouter.Params) {
	context := BaseContext(r)
	context.NavbarSelected = 0
	all := getParadisePackages(
		gApplicationState.Configuration.Data,
	)
	context.Packages = []Package{}
	for i := 0; i < len(all); i++ {
		if(all[i].ShowOnIndexPage) {
			context.Packages = append(context.Packages, all[i])
		}
	}
	Render(w, "index.gohtml", context)
}

func getPrices(w http.ResponseWriter, r *http.Request, p httprouter.Params) {
	context := BaseContext(r)
	context.NavbarSelected = 1

	file, _ := readFileBytesMemoized(
		gApplicationState.Configuration.Data + "prices/prices.md",
	)
	html := blackfriday.MarkdownBasic(
		file,
	)
	context.RenderedPricesMarkdown =
		template.HTML(
			html,
		)
	context.Parameters["markdownHTML"] = string(html)
	Render(w, "prices.gohtml", context);
}
func getPackages(w http.ResponseWriter, r *http.Request, p httprouter.Params) {
	context := BaseContext(r)
	context.NavbarSelected = 2
	all := getParadisePackages(
		gApplicationState.Configuration.Data,
	)
	context.Packages = []Package{}
	for i := 0; i < len(all); i++ {
		if(all[i].ShowOnPackagePage) {
			context.Packages = append(context.Packages, all[i])
		}
	}
	Render(w, "packages.gohtml", context)
}

func getPackage(w http.ResponseWriter, r *http.Request, p httprouter.Params) {
	context := BaseContext(r)
	context.NavbarSelected = 0
	context.PackageDetails = getParadisePackageByUrl(
		gApplicationState.Configuration.Data,
		p.ByName("url"),
	)
	context.Route = "/package/:url"

	if (context.PackageDetails != nil) {
		file, _ := readFileBytesMemoized(
			gApplicationState.Configuration.Data + context.PackageDetails.PageDetailsMarkdown,
		)

		html := blackfriday.MarkdownBasic(
			file,
		);

		context.RenderedPackageMarkdown =
			template.HTML(
				html,
			)

		context.RenderedPackageCover = template.HTMLAttr(
			context.PackageDetails.PageDetailsCover,
		)

		context.Parameters["url"] = context.PackageDetails.Url;
		context.Parameters["id"] = strconv.Itoa(context.PackageDetails.Id)
		context.Parameters["markdownHTML"] = string(html)
		context.Parameters["cover"] = context.PackageDetails.PageDetailsCover
	}

	Render(w, "package.gohtml", context)
}

func getRestaurant(w http.ResponseWriter, r *http.Request, p httprouter.Params) {
	context := BaseContext(r)
	context.NavbarSelected = 3
	Render(w, "restaurant.gohtml", context)
}
func getLocation(w http.ResponseWriter, r *http.Request, p httprouter.Params) {
	context := BaseContext(r)
	context.NavbarSelected = 4
	file, _ := readFileBytesMemoized(
		gApplicationState.Configuration.Data + "location/location.md",
	)
	html := blackfriday.MarkdownBasic(
		file,
	)
	context.RenderedLocationMarkdown =
		template.HTML(
			html,
		)
	context.Parameters["markdownHTML"] = string(html)
	if (gApplicationState.Configuration.GoogleApiKey != nil) {
		context.Parameters["GoogleApiKey"] = *gApplicationState.Configuration.GoogleApiKey
	}
	Render(w, "location.gohtml", context)
}

func getGallery(w http.ResponseWriter, r *http.Request, p httprouter.Params) {
	context := BaseContext(r)
	context.NavbarSelected = 5
	Render(w, "gallery.gohtml", context)
}

func getApiPackage(w http.ResponseWriter, _ *http.Request, p httprouter.Params) {
	id, err := strconv.Atoi(p.ByName("id"));
	pack := (*Package)(nil)
	if (err == nil) {
		pack = getParadisePackage(
			gApplicationState.Configuration.Data,
			id,
		)
	}

	jData, _ := json.Marshal(pack)
	w.Header().Set("Content-Type", "application/json")
	w.Write(jData)
}

var gApplicationState *ApplicationState

func loadResources(filename string) (UnsafeTemplateData, SafeTemplateJs, SafeTemplateCss) {
	assets, err := ioutil.ReadFile(filename)
	runtimeAssert(err)
	m := make(map[string]VersionedScript)
	n := make(UnsafeTemplateData)
	o := make(SafeTemplateJs)
	p := make(SafeTemplateCss)
	err = json.Unmarshal(assets, &m)
	runtimeAssert(err)

	if (m["inline_sync_top"].Js != "") {
		file, _ := readFileMemoized("public/" + m["inline_sync_top"].Js)
		o["inline_sync_js_top"] =
			template.JS(file)
	}
	if (m["inline_sync_top"].Css != "") {
		file, _ := readFileMemoized("public/" + m["inline_sync_top"].Css)
		p["inline_sync_css_top"] =
			template.CSS(file)
	}

	if (m["async"].Js != "") {
		n["async_js"] = "/public/" + m["async"].Js
	}

	if (m["async"].Css != "") {
		n["async_css"] = "/public/" + m["async"].Css
	}
	return n, o, p
}

func getApiPackages(w http.ResponseWriter, r *http.Request, _ httprouter.Params) {
	jData, _ := json.Marshal(
		getParadisePackages(
			gApplicationState.Configuration.Data,
		),
	)
	w.Header().Set("Content-Type", "application/json")
	w.Write(jData)
}

func getApiPhotos(w http.ResponseWriter, r *http.Request, _ httprouter.Params) {
	photos := []Photo{}
	filepath.Walk(
		gApplicationState.Configuration.Data + "gallery/images/",
		func(stringPath string, info os.FileInfo, err error) error {
			stringPath = path.Clean(filepath.ToSlash(stringPath))

			if (err != nil) {
				return err
			}

			if (info.IsDir()) {
				return nil
			}
			_, file := path.Split(stringPath)
			ext := strings.ToLower(path.Ext(stringPath))

			if (ext != ".jpg" && ext != ".jpeg") {
				return nil
			}

			if _, err := os.Stat(gApplicationState.Configuration.Data + "gallery/full/" + file); os.IsNotExist(err) {
				photo, err := os.Open(stringPath)
				runtimeAssert(err)
				img, err := jpeg.Decode(photo)
				photo.Close()
				m := resize.Resize(1200, 0, img, resize.Lanczos3)
				out, err := os.Create(gApplicationState.Configuration.Data + "gallery/full/" + file)
				runtimeAssert(err)
				defer out.Close()
				jpeg.Encode(out, m, nil)
			}

			if _, err := os.Stat(gApplicationState.Configuration.Data + "gallery/thumbnails/" + file); os.IsNotExist(err) {
				photo, err := os.Open(stringPath)
				runtimeAssert(err)
				img, err := jpeg.Decode(photo)
				photo.Close()
				m := resize.Resize(400, 0, img, resize.Lanczos3)
				out, err := os.Create(gApplicationState.Configuration.Data + "gallery/thumbnails/" + file)
				runtimeAssert(err)
				defer out.Close()
				jpeg.Encode(out, m, nil)
			}

			photos = append(photos, Photo{
				Thumbnail: "/static/gallery/thumbnails/" + file,
				FullPicture: "/static/gallery/full/" + file,
			})
			return err
		},
	)
	jData, _ := json.Marshal(
		photos,
	)
	w.Header().Set("Content-Type", "application/json")
	w.Write(jData)
}

func redirectToHTTPS(w http.ResponseWriter, r *http.Request) {
	ssl := gApplicationState.Configuration.SSL;
	port := ssl.Port;
	host := gApplicationState.Configuration.Host;
	toURL := "https://" + net.JoinHostPort(host, strconv.Itoa(port));
	toURL += r.URL.RequestURI()
	w.Header().Set("Connection", "close")
	http.Redirect(w, r, toURL, http.StatusMovedPermanently)
}

func ServeFilesGzipped(r *httprouter.Router, path string, root http.FileSystem) {
	if len(path) < 10 || path[len(path)-10:] != "/*filepath" {
		panic("path must end with /*filepath in path '" + path + "'")
	}
	fileServer := http.FileServer(root)

	r.GET(path, makeGzipHandler(func(w http.ResponseWriter, req *http.Request, ps httprouter.Params) {
		req.URL.Path = ps.ByName("filepath")
		fileServer.ServeHTTP(w, req)
	}))
}

func runApplicationSimple(applicationState *ApplicationState) {
	gApplicationState = applicationState
	router := httprouter.New();

	router.GET("/", makeGzipHandler(getIndex))
	router.GET("/prices", makeGzipHandler(getPrices))
	router.GET("/packages", makeGzipHandler(getPackages))
	router.GET("/package/:url", makeGzipHandler(getPackage))
	router.GET("/restaurant", makeGzipHandler(getRestaurant))
	router.GET("/location", makeGzipHandler(getLocation))
	router.GET("/gallery", makeGzipHandler(getGallery))

	router.GET("/api/package", getApiPackages)
	router.GET("/api/package/:id", getApiPackage)
	router.GET("/api/photo", getApiPhotos)

	ServeFilesGzipped(router, "/public/*filepath", http.Dir(applicationState.Configuration.Public));
	ServeFilesGzipped(router, "/static/*filepath", http.Dir(applicationState.Configuration.Data));
	router.NotFound = http.FileServer(http.Dir(applicationState.Configuration.Data + "public/"))

	configuration := applicationState.Configuration;
	ssl := configuration.SSL;
	httpAddress := net.JoinHostPort(configuration.Host, strconv.Itoa(configuration.Port))

	if ssl != nil {
		tlsAddress := net.JoinHostPort(configuration.Host, strconv.Itoa(ssl.Port))
		fmt.Fprintf(os.Stdout, "Listening on %s...\n", tlsAddress)
		go http.ListenAndServeTLS(tlsAddress, ssl.Cert, ssl.Key, router)

		fmt.Fprintf(os.Stdout, "Listening on %s...\n", httpAddress)
		http.ListenAndServe(httpAddress, http.HandlerFunc(redirectToHTTPS));
	} else {
		fmt.Fprintf(os.Stdout, "Listening on %s...\n", httpAddress)
		http.ListenAndServe(httpAddress, router)
	}
}
<|endoftext|>


<commit_before>// +build rocksdb

package rocksdb

import (
	"bytes"
	"context"
	"crypto/md5"
	"fmt"
	"github.com/chrislusf/seaweedfs/weed/filer"
	"github.com/chrislusf/seaweedfs/weed/glog"
	"github.com/chrislusf/seaweedfs/weed/pb/filer_pb"
	weed_util "github.com/chrislusf/seaweedfs/weed/util"
	"github.com/tecbot/gorocksdb"
	"io"
)

func init() {
	filer.Stores = append(filer.Stores, &RocksDBStore{})
}

type RocksDBStore struct {
	path string
	db   *gorocksdb.DB
}

func (store *RocksDBStore) GetName() string {
	return "rocksdb"
}

func (store *RocksDBStore) Initialize(configuration weed_util.Configuration, prefix string) (err error) {
	dir := configuration.GetString(prefix + "dir")
	return store.initialize(dir)
}

func (store *RocksDBStore) initialize(dir string) (err error) {
	glog.Infof("filer store rocksdb dir: %s", dir)
	if err := weed_util.TestFolderWritable(dir); err != nil {
		return fmt.Errorf("Check Level Folder %s Writable: %s", dir, err)
	}

	options := gorocksdb.NewDefaultOptions()
	options.SetCreateIfMissing(true)
	store.db, err = gorocksdb.OpenDb(options, dir)

	return
}

func (store *RocksDBStore) BeginTransaction(ctx context.Context) (context.Context, error) {
	return ctx, nil
}
func (store *RocksDBStore) CommitTransaction(ctx context.Context) error {
	return nil
}
func (store *RocksDBStore) RollbackTransaction(ctx context.Context) error {
	return nil
}

func (store *RocksDBStore) InsertEntry(ctx context.Context, entry *filer.Entry) (err error) {
	dir, name := entry.DirAndName()
	key := genKey(dir, name)

	value, err := entry.EncodeAttributesAndChunks()
	if err != nil {
		return fmt.Errorf("encoding %s %+v: %v", entry.FullPath, entry.Attr, err)
	}

	wo := gorocksdb.NewDefaultWriteOptions()
	err = store.db.Put(wo, key, value)

	if err != nil {
		return fmt.Errorf("persisting %s : %v", entry.FullPath, err)
	}

	// println("saved", entry.FullPath, "chunks", len(entry.Chunks))

	return nil
}

func (store *RocksDBStore) UpdateEntry(ctx context.Context, entry *filer.Entry) (err error) {

	return store.InsertEntry(ctx, entry)
}

func (store *RocksDBStore) FindEntry(ctx context.Context, fullpath weed_util.FullPath) (entry *filer.Entry, err error) {
	dir, name := fullpath.DirAndName()
	key := genKey(dir, name)

	ro := gorocksdb.NewDefaultReadOptions()
	data, err := store.db.GetBytes(ro, key)

	if data == nil {
		return nil, filer_pb.ErrNotFound
	}
	if err != nil {
		return nil, fmt.Errorf("get %s : %v", entry.FullPath, err)
	}

	entry = &filer.Entry{
		FullPath: fullpath,
	}
	err = entry.DecodeAttributesAndChunks(weed_util.MaybeDecompressData(data))
	if err != nil {
		return entry, fmt.Errorf("decode %s : %v", entry.FullPath, err)
	}

	// println("read", entry.FullPath, "chunks", len(entry.Chunks), "data", len(data), string(data))

	return entry, nil
}

func (store *RocksDBStore) DeleteEntry(ctx context.Context, fullpath weed_util.FullPath) (err error) {
	dir, name := fullpath.DirAndName()
	key := genKey(dir, name)

	wo := gorocksdb.NewDefaultWriteOptions()
	err = store.db.Delete(wo, key)
	if err != nil {
		return fmt.Errorf("delete %s : %v", fullpath, err)
	}

	return nil
}

func (store *RocksDBStore) DeleteFolderChildren(ctx context.Context, fullpath weed_util.FullPath) (err error) {
	directoryPrefix := genDirectoryKeyPrefix(fullpath, "")

	batch := new(gorocksdb.WriteBatch)

	ro := gorocksdb.NewDefaultReadOptions()
	ro.SetFillCache(false)
	iter := store.db.NewIterator(ro)
	defer iter.Close()
	err = enumerate(iter, directoryPrefix, nil, false, -1, func(key, value []byte) bool {
		batch.Delete(key)
		return true
	})
	if err != nil {
		return fmt.Errorf("delete list %s : %v", fullpath, err)
	}

	wo := gorocksdb.NewDefaultWriteOptions()
	err = store.db.Write(wo, batch)

	if err != nil {
		return fmt.Errorf("delete %s : %v", fullpath, err)
	}

	return nil
}

func enumerate(iter *gorocksdb.Iterator, prefix, lastKey []byte, includeLastKey bool, limit int, fn func(key, value []byte) bool) error {

	if len(lastKey) == 0 {
		iter.Seek(prefix)
	} else {
		iter.Seek(lastKey)

		if !includeLastKey {
			k := iter.Key()
			v := iter.Value()
			key := k.Data()
			defer k.Free()
			defer v.Free()

			if !bytes.HasPrefix(key, prefix) {
				return nil
			}

			if bytes.Equal(key, lastKey) {
				iter.Next()
			}

		}
	}

	i := 0
	for ; iter.Valid(); iter.Next() {

		if limit > 0 {
			i++
			if i > limit {
				break
			}
		}

		k := iter.Key()
		v := iter.Value()
		key := k.Data()
		value := v.Data()

		if !bytes.HasPrefix(key, prefix) {
			k.Free()
			v.Free()
			break
		}

		ret := fn(key, value)

		k.Free()
		v.Free()

		if !ret {
			break
		}

	}

	if err := iter.Err(); err != nil {
		return fmt.Errorf("prefix scan iterator: %v", err)
	}
	return nil
}

func (store *RocksDBStore) ListDirectoryEntries(ctx context.Context, fullpath weed_util.FullPath, startFileName string, inclusive bool,
	limit int) (entries []*filer.Entry, err error) {
	return store.ListDirectoryPrefixedEntries(ctx, fullpath, startFileName, inclusive, limit, "")
}

func (store *RocksDBStore) ListDirectoryPrefixedEntries(ctx context.Context, fullpath weed_util.FullPath, startFileName string, inclusive bool, limit int, prefix string) (entries []*filer.Entry, err error) {

	directoryPrefix := genDirectoryKeyPrefix(fullpath, prefix)
	lastFileStart := directoryPrefix
	if startFileName != "" {
		lastFileStart = genDirectoryKeyPrefix(fullpath, startFileName)
	}

	ro := gorocksdb.NewDefaultReadOptions()
	ro.SetFillCache(false)
	iter := store.db.NewIterator(ro)
	defer iter.Close()
	err = enumerate(iter, directoryPrefix, lastFileStart, inclusive, limit, func(key, value []byte) bool {
		fileName := getNameFromKey(key)
		if fileName == "" {
			return true
		}
		limit--
		if limit < 0 {
			return false
		}
		entry := &filer.Entry{
			FullPath: weed_util.NewFullPath(string(fullpath), fileName),
		}

		// println("list", entry.FullPath, "chunks", len(entry.Chunks))
		if decodeErr := entry.DecodeAttributesAndChunks(weed_util.MaybeDecompressData(value)); decodeErr != nil {
			err = decodeErr
			glog.V(0).Infof("list %s : %v", entry.FullPath, err)
			return false
		}
		entries = append(entries, entry)
		return true
	})
	if err != nil {
		return entries, fmt.Errorf("prefix list %s : %v", fullpath, err)
	}

	return entries, err
}

func genKey(dirPath, fileName string) (key []byte) {
	key = hashToBytes(dirPath)
	key = append(key, []byte(fileName)...)
	return key
}

func genDirectoryKeyPrefix(fullpath weed_util.FullPath, startFileName string) (keyPrefix []byte) {
	keyPrefix = hashToBytes(string(fullpath))
	if len(startFileName) > 0 {
		keyPrefix = append(keyPrefix, []byte(startFileName)...)
	}
	return keyPrefix
}

func getNameFromKey(key []byte) string {

	return string(key[md5.Size:])

}

// hash directory, and use last byte for partitioning
func hashToBytes(dir string) []byte {
	h := md5.New()
	io.WriteString(h, dir)

	b := h.Sum(nil)

	return b
}

func (store *RocksDBStore) Shutdown() {
	store.db.Close()
}
<commit_msg>fix #1726<commit_after>// +build rocksdb

package rocksdb

import (
	"bytes"
	"context"
	"crypto/md5"
	"fmt"
	"github.com/chrislusf/seaweedfs/weed/filer"
	"github.com/chrislusf/seaweedfs/weed/glog"
	"github.com/chrislusf/seaweedfs/weed/pb/filer_pb"
	weed_util "github.com/chrislusf/seaweedfs/weed/util"
	rocksdb "github.com/tecbot/gorocksdb"
	"io"
)

func init() {
	filer.Stores = append(filer.Stores, &RocksDBStore{})
}

type options struct {
	opt *rocksdb.Options
	ro  *rocksdb.ReadOptions
	wo  *rocksdb.WriteOptions
}

func (opt *options) init() {
	opt.opt = rocksdb.NewDefaultOptions()
	opt.ro = rocksdb.NewDefaultReadOptions()
	opt.wo = rocksdb.NewDefaultWriteOptions()
}

func (opt *options) close() {
	opt.opt.Destroy()
	opt.ro.Destroy()
	opt.wo.Destroy()
}

type RocksDBStore struct {
	path string
	db   *rocksdb.DB
	options
}

func (store *RocksDBStore) GetName() string {
	return "rocksdb"
}

func (store *RocksDBStore) Initialize(configuration weed_util.Configuration, prefix string) (err error) {
	dir := configuration.GetString(prefix + "dir")
	return store.initialize(dir)
}

func (store *RocksDBStore) initialize(dir string) (err error) {
	glog.Infof("filer store rocksdb dir: %s", dir)
	if err := weed_util.TestFolderWritable(dir); err != nil {
		return fmt.Errorf("Check Level Folder %s Writable: %s", dir, err)
	}
	store.options.init()
	store.opt.SetCreateIfMissing(true)
	store.db, err = rocksdb.OpenDb(store.opt, dir)

	return
}

func (store *RocksDBStore) BeginTransaction(ctx context.Context) (context.Context, error) {
	return ctx, nil
}
func (store *RocksDBStore) CommitTransaction(ctx context.Context) error {
	return nil
}
func (store *RocksDBStore) RollbackTransaction(ctx context.Context) error {
	return nil
}

func (store *RocksDBStore) InsertEntry(ctx context.Context, entry *filer.Entry) (err error) {
	dir, name := entry.DirAndName()
	key := genKey(dir, name)

	value, err := entry.EncodeAttributesAndChunks()
	if err != nil {
		return fmt.Errorf("encoding %s %+v: %v", entry.FullPath, entry.Attr, err)
	}

	err = store.db.Put(store.wo, key, value)

	if err != nil {
		return fmt.Errorf("persisting %s : %v", entry.FullPath, err)
	}

	// println("saved", entry.FullPath, "chunks", len(entry.Chunks))

	return nil
}

func (store *RocksDBStore) UpdateEntry(ctx context.Context, entry *filer.Entry) (err error) {

	return store.InsertEntry(ctx, entry)
}

func (store *RocksDBStore) FindEntry(ctx context.Context, fullpath weed_util.FullPath) (entry *filer.Entry, err error) {
	dir, name := fullpath.DirAndName()
	key := genKey(dir, name)
	data, err := store.db.Get(store.ro, key)

	if data == nil {
		return nil, filer_pb.ErrNotFound
	}
	defer data.Free()

	if err != nil {
		return nil, fmt.Errorf("get %s : %v", fullpath, err)
	}

	entry = &filer.Entry{
		FullPath: fullpath,
	}
	err = entry.DecodeAttributesAndChunks(weed_util.MaybeDecompressData(data.Data()))
	if err != nil {
		return entry, fmt.Errorf("decode %s : %v", entry.FullPath, err)
	}

	// println("read", entry.FullPath, "chunks", len(entry.Chunks), "data", len(data), string(data))

	return entry, nil
}

func (store *RocksDBStore) DeleteEntry(ctx context.Context, fullpath weed_util.FullPath) (err error) {
	dir, name := fullpath.DirAndName()
	key := genKey(dir, name)

	err = store.db.Delete(store.wo, key)
	if err != nil {
		return fmt.Errorf("delete %s : %v", fullpath, err)
	}

	return nil
}

func (store *RocksDBStore) DeleteFolderChildren(ctx context.Context, fullpath weed_util.FullPath) (err error) {
	directoryPrefix := genDirectoryKeyPrefix(fullpath, "")

	batch := rocksdb.NewWriteBatch()
	defer batch.Destroy()

	ro := rocksdb.NewDefaultReadOptions()
	defer ro.Destroy()
	ro.SetFillCache(false)

	iter := store.db.NewIterator(ro)
	defer iter.Close()
	err = enumerate(iter, directoryPrefix, nil, false, -1, func(key, value []byte) bool {
		batch.Delete(key)
		return true
	})
	if err != nil {
		return fmt.Errorf("delete list %s : %v", fullpath, err)
	}

	err = store.db.Write(store.wo, batch)

	if err != nil {
		return fmt.Errorf("delete %s : %v", fullpath, err)
	}

	return nil
}

func enumerate(iter *rocksdb.Iterator, prefix, lastKey []byte, includeLastKey bool, limit int, fn func(key, value []byte) bool) error {

	if len(lastKey) == 0 {
		iter.Seek(prefix)
	} else {
		iter.Seek(lastKey)

		if !includeLastKey {
			key := iter.Key().Data()

			if !bytes.HasPrefix(key, prefix) {
				return nil
			}

			if bytes.Equal(key, lastKey) {
				iter.Next()
			}

		}
	}

	i := 0
	for ; iter.Valid(); iter.Next() {

		if limit > 0 {
			i++
			if i > limit {
				break
			}
		}

		key := iter.Key().Data()

		if !bytes.HasPrefix(key, prefix) {
			break
		}

		ret := fn(key, iter.Value().Data())

		if !ret {
			break
		}

	}

	if err := iter.Err(); err != nil {
		return fmt.Errorf("prefix scan iterator: %v", err)
	}
	return nil
}

func (store *RocksDBStore) ListDirectoryEntries(ctx context.Context, fullpath weed_util.FullPath, startFileName string, inclusive bool,
	limit int) (entries []*filer.Entry, err error) {
	return store.ListDirectoryPrefixedEntries(ctx, fullpath, startFileName, inclusive, limit, "")
}

func (store *RocksDBStore) ListDirectoryPrefixedEntries(ctx context.Context, fullpath weed_util.FullPath, startFileName string, inclusive bool, limit int, prefix string) (entries []*filer.Entry, err error) {

	directoryPrefix := genDirectoryKeyPrefix(fullpath, prefix)
	lastFileStart := directoryPrefix
	if startFileName != "" {
		lastFileStart = genDirectoryKeyPrefix(fullpath, startFileName)
	}

	ro := rocksdb.NewDefaultReadOptions()
	defer ro.Destroy()
	ro.SetFillCache(false)

	iter := store.db.NewIterator(ro)
	defer iter.Close()
	err = enumerate(iter, directoryPrefix, lastFileStart, inclusive, limit, func(key, value []byte) bool {
		fileName := getNameFromKey(key)
		if fileName == "" {
			return true
		}
		limit--
		if limit < 0 {
			return false
		}
		entry := &filer.Entry{
			FullPath: weed_util.NewFullPath(string(fullpath), fileName),
		}

		// println("list", entry.FullPath, "chunks", len(entry.Chunks))
		if decodeErr := entry.DecodeAttributesAndChunks(weed_util.MaybeDecompressData(value)); decodeErr != nil {
			err = decodeErr
			glog.V(0).Infof("list %s : %v", entry.FullPath, err)
			return false
		}
		entries = append(entries, entry)
		return true
	})
	if err != nil {
		return entries, fmt.Errorf("prefix list %s : %v", fullpath, err)
	}

	return entries, err
}

func genKey(dirPath, fileName string) (key []byte) {
	key = hashToBytes(dirPath)
	key = append(key, []byte(fileName)...)
	return key
}

func genDirectoryKeyPrefix(fullpath weed_util.FullPath, startFileName string) (keyPrefix []byte) {
	keyPrefix = hashToBytes(string(fullpath))
	if len(startFileName) > 0 {
		keyPrefix = append(keyPrefix, []byte(startFileName)...)
	}
	return keyPrefix
}

func getNameFromKey(key []byte) string {

	return string(key[md5.Size:])

}

// hash directory, and use last byte for partitioning
func hashToBytes(dir string) []byte {
	h := md5.New()
	io.WriteString(h, dir)

	b := h.Sum(nil)

	return b
}

func (store *RocksDBStore) Shutdown() {
	store.db.Close()
	store.options.close()
}
<|endoftext|>