<commit_before>package config

import (
	"encoding/json"
	"fmt"
	"io/ioutil"
	"os"
	"path/filepath"

	"github.com/satori/go.uuid"
	"k8s.io/client-go/tools/clientcmd"
)

const configEnvVar = "PACH_CONFIG"
const contextEnvVar = "PACH_CONTEXT"

var defaultConfigDir = filepath.Join(os.Getenv("HOME"), ".pachyderm")
var defaultConfigPath = filepath.Join(defaultConfigDir, "config.json")

func configPath() string {
	if env, ok := os.LookupEnv(configEnvVar); ok {
		return env
	}
	return defaultConfigPath
}

// ActiveContext gets the active context in the config
func (c *Config) ActiveContext() (*Context, error) {
	if env, ok := os.LookupEnv(contextEnvVar); ok {
		context := c.V2.Contexts[env]
		if context == nil {
			return nil, fmt.Errorf("`%s` refers to a context that does not exist", contextEnvVar)
		}
		return context, nil
	}
	context := c.V2.Contexts[c.V2.ActiveContext]
	if context == nil {
		return nil, fmt.Errorf("the active context references one that does exist; change the active context first")
	}
	return context, nil
}

// Read loads the Pachyderm config on this machine.
// If an existing configuration cannot be found, it sets up the defaults. Read
// returns a nil Config if and only if it returns a non-nil error.
func Read() (*Config, error) {
	var c *Config

	// Read json file
	p := configPath()
	if raw, err := ioutil.ReadFile(p); err == nil {
		err = json.Unmarshal(raw, &c)
		if err != nil {
			return nil, err
		}
	} else if os.IsNotExist(err) {
		// File doesn't exist, so create a new config
		fmt.Fprintf(os.Stderr, "No config detected at %q. Generating new config...\n", p)
		c = &Config{}
	} else {
		return nil, fmt.Errorf("fatal: could not read config at %q: %v", p, err)
	}

	updated := false

	if c.UserID == "" {
		updated = true
		fmt.Fprintln(os.Stderr, "No UserID present in config - generating new one.")
		uuid, err := uuid.NewV4()
		if err != nil {
			return nil, fmt.Errorf("could not generate new user ID: %v", err)
		}
		c.UserID = uuid.String()
	}

	if c.V2 == nil {
		updated = true
		if err := c.initV2(); err != nil {
			return nil, err
		}
	}

	if updated {
		fmt.Fprintf(os.Stderr, "Rewriting config at %q.\n", p)

		if err := c.Write(); err != nil {
			return nil, fmt.Errorf("could not rewrite config at %q: %v", p, err)
		}
	}

	return c, nil
}

func (c *Config) initV2() error {
	rules := clientcmd.NewDefaultClientConfigLoadingRules()
	overrides := &clientcmd.ConfigOverrides{}
	kubeConfig := clientcmd.NewNonInteractiveDeferredLoadingClientConfig(rules, overrides)
	rawKubeConfig, err := kubeConfig.RawConfig()
	if err != nil {
		return err
	}
	kubeContext := rawKubeConfig.CurrentContext
	fmt.Fprintf(os.Stderr, "No config V2 present in config - generating a new one off of the kube context '%s'.\n", kubeContext)

	c.V2 = &ConfigV2{
		ActiveContext: "default",
		Contexts:      map[string]*Context{},
	}

	if c.V1 != nil {
		c.V2.Contexts["default"] = &Context{
			Source:            ContextSource_CONFIG_V1,
			PachdAddress:      c.V1.PachdAddress,
			ServerCAs:         c.V1.ServerCAs,
			SessionToken:      c.V1.SessionToken,
			ActiveTransaction: c.V1.ActiveTransaction,
			KubeContext:       kubeContext,
		}

		c.V1 = nil
	} else {
		c.V2.Contexts["default"] = &Context{
			Source:      ContextSource_NONE,
			KubeContext: kubeContext,
		}
	}
	return nil
}

// Write writes the configuration in 'c' to this machine's Pachyderm config
// file.
func (c *Config) Write() error {
	if c.V1 != nil {
		panic("v1 config included, implying a bug")
	}

	rawConfig, err := json.MarshalIndent(c, "", "  ")
	if err != nil {
		return err
	}

	// If we're not using a custom config path, create the default config path
	p := configPath()
	if _, ok := os.LookupEnv(configEnvVar); ok {
		// using overridden config path -- just make sure the parent dir exists
		d := filepath.Dir(p)
		if _, err := os.Stat(d); err != nil {
			return fmt.Errorf("cannot use config at %s: could not stat parent directory (%v)", p, err)
		}
	} else {
		// using the default config path, create the config directory
		err = os.MkdirAll(defaultConfigDir, 0755)
		if err != nil {
			return err
		}
	}
	return ioutil.WriteFile(p, rawConfig, 0644)
}
<commit_msg>Ensure config is read only once<commit_after>package config

import (
	"encoding/json"
	"fmt"
	"io/ioutil"
	"os"
	"path/filepath"
	"sync"

	"github.com/satori/go.uuid"
	"k8s.io/client-go/tools/clientcmd"
)

const configEnvVar = "PACH_CONFIG"
const contextEnvVar = "PACH_CONTEXT"

var defaultConfigDir = filepath.Join(os.Getenv("HOME"), ".pachyderm")
var defaultConfigPath = filepath.Join(defaultConfigDir, "config.json")

var readerOnce sync.Once
var value *Config
var readErr error

func configPath() string {
	if env, ok := os.LookupEnv(configEnvVar); ok {
		return env
	}
	return defaultConfigPath
}

// ActiveContext gets the active context in the config
func (c *Config) ActiveContext() (*Context, error) {
	if env, ok := os.LookupEnv(contextEnvVar); ok {
		context := c.V2.Contexts[env]
		if context == nil {
			return nil, fmt.Errorf("`%s` refers to a context that does not exist", contextEnvVar)
		}
		return context, nil
	}
	context := c.V2.Contexts[c.V2.ActiveContext]
	if context == nil {
		return nil, fmt.Errorf("the active context references one that does exist; change the active context first")
	}
	return context, nil
}

// Read loads the Pachyderm config on this machine.
// If an existing configuration cannot be found, it sets up the defaults. Read
// returns a nil Config if and only if it returns a non-nil error.
func Read() (*Config, error) {
	readerOnce.Do(func() {
		// Read json file
		p := configPath()
		if raw, err := ioutil.ReadFile(p); err == nil {
			err = json.Unmarshal(raw, &value)
			if err != nil {
				readErr = err
				return
			}
		} else if os.IsNotExist(err) {
			// File doesn't exist, so create a new config
			fmt.Fprintf(os.Stderr, "No config detected at %q. Generating new config...\n", p)
			value = &Config{}
		} else {
			readErr = fmt.Errorf("fatal: could not read config at %q: %v", p, err)
			return
		}

		updated := false

		if value.UserID == "" {
			updated = true
			fmt.Fprintln(os.Stderr, "No UserID present in config - generating new one.")
			uuid, err := uuid.NewV4()
			if err != nil {
				readErr = fmt.Errorf("could not generate new user ID: %v", err)
				return
			}
			value.UserID = uuid.String()
		}

		if value.V2 == nil {
			updated = true
			if err := value.initV2(); err != nil {
				readErr = err
				return
			}
		}

		if updated {
			fmt.Fprintf(os.Stderr, "Rewriting config at %q.\n", p)

			if err := value.Write(); err != nil {
				readErr = fmt.Errorf("could not rewrite config at %q: %v", p, err)
				return
			}
		}

	})

	return value, readErr
}

func (c *Config) initV2() error {
	rules := clientcmd.NewDefaultClientConfigLoadingRules()
	overrides := &clientcmd.ConfigOverrides{}
	kubeConfig := clientcmd.NewNonInteractiveDeferredLoadingClientConfig(rules, overrides)
	rawKubeConfig, err := kubeConfig.RawConfig()
	if err != nil {
		return err
	}
	kubeContext := rawKubeConfig.CurrentContext
	fmt.Fprintf(os.Stderr, "No config V2 present in config - generating a new one off of the kube context '%s'.\n", kubeContext)

	c.V2 = &ConfigV2{
		ActiveContext: "default",
		Contexts:      map[string]*Context{},
	}

	if c.V1 != nil {
		c.V2.Contexts["default"] = &Context{
			Source:            ContextSource_CONFIG_V1,
			PachdAddress:      c.V1.PachdAddress,
			ServerCAs:         c.V1.ServerCAs,
			SessionToken:      c.V1.SessionToken,
			ActiveTransaction: c.V1.ActiveTransaction,
			KubeContext:       kubeContext,
		}

		c.V1 = nil
	} else {
		c.V2.Contexts["default"] = &Context{
			Source:      ContextSource_NONE,
			KubeContext: kubeContext,
		}
	}
	return nil
}

// Write writes the configuration in 'c' to this machine's Pachyderm config
// file.
func (c *Config) Write() error {
	if c.V1 != nil {
		panic("v1 config included, implying a bug")
	}

	rawConfig, err := json.MarshalIndent(c, "", "  ")
	if err != nil {
		return err
	}

	// If we're not using a custom config path, create the default config path
	p := configPath()
	if _, ok := os.LookupEnv(configEnvVar); ok {
		// using overridden config path -- just make sure the parent dir exists
		d := filepath.Dir(p)
		if _, err := os.Stat(d); err != nil {
			return fmt.Errorf("cannot use config at %s: could not stat parent directory (%v)", p, err)
		}
	} else {
		// using the default config path, create the config directory
		err = os.MkdirAll(defaultConfigDir, 0755)
		if err != nil {
			return err
		}
	}
	return ioutil.WriteFile(p, rawConfig, 0644)
}
<|endoftext|>


<commit_before>/*
 *  Bulldozer Framework
 *  Copyright (C) DesertBit
 */

package firewall

import (
	"code.desertbit.com/bulldozer/bulldozer/settings"
	"code.desertbit.com/bulldozer/bulldozer/utils"
	"net/http"
	"sync"
	"time"
)

var (
	blockList      map[string]int64 = make(map[string]int64)
	blockListMutex sync.Mutex

	requestCounter      map[string]int = make(map[string]int)
	requestCounterMutex sync.Mutex
)

func init() {
	// Start the loops in an new goroutine
	go cleanupRequestsCountLoop()
	go releaseBlockedRemoteAddrLoop()
}

//##############//
//### Public ###//
//##############//

// NewRequest tells the firewall, that a new request happened.
// False is returned, if this request should be blocked,
// because the IP is on the block list.
// The remote address is always returned for logging purpose.
func NewRequest(req *http.Request) (bool, string) {
	// Get the remote addresse of the request
	remoteAddr, _ := utils.RemoteAddress(req)

	// Check if this remote address is blocked
	if isBlocked(remoteAddr) {
		return false, remoteAddr
	}

	// Register the new request in a new goroutine
	go addRequest(remoteAddr)

	return true, remoteAddr
}

//###############//
//### Private ###//
//###############//

// isBlocked checks if the remote addres is blocked
func isBlocked(remoteAddr string) bool {
	// Lock the mutex
	blockListMutex.Lock()
	defer blockListMutex.Unlock()

	// Check if the remote addresse exists in the blocked map
	_, exists := blockList[remoteAddr]
	return exists
}

func addRequest(remoteAddr string) {
	// Lock the mutex
	requestCounterMutex.Lock()
	defer requestCounterMutex.Unlock()

	count, ok := requestCounter[remoteAddr]
	if !ok {
		count = 1
	}

	// Add the remote address to the block map, if the count
	// reached the limit.
	if count > settings.Settings.FirewallMaxRequestsPerMinute {
		// Remove the remote address from the request counter map
		delete(requestCounter, remoteAddr)

		// Get the current timestamp
		timestamp := time.Now().Unix()

		// Lock the mutex
		blockListMutex.Lock()
		defer blockListMutex.Unlock()

		// Add the remote address with a timestamp to the block map
		blockList[remoteAddr] = timestamp

		return
	}

	// Save the incremented count to the map
	requestCounter[remoteAddr] = count + 1
}

func cleanupRequestsCountLoop() {
	for {
		// Sleep one minute
		time.Sleep(time.Minute)

		func() {
			// Lock the mutex
			requestCounterMutex.Lock()
			defer requestCounterMutex.Unlock()

			// Clear the map if not emtpy
			if len(requestCounter) > 0 {
				requestCounter = make(map[string]int)
			}
		}()
	}
}

func releaseBlockedRemoteAddrLoop() {
	for {
		// Sleep one minute
		time.Sleep(time.Minute)

		func() {
			// Lock the mutex
			blockListMutex.Lock()
			defer blockListMutex.Unlock()

			// Return if nothing to do
			if len(blockList) == 0 {
				return
			}

			// Create the release timestamp
			releaseTimestamp := time.Now().Unix() - int64(settings.Settings.FirewallReleaseBlockAfter)

			// Remove expired blocks
			for key, timestamp := range blockList {
				if timestamp < releaseTimestamp {
					delete(blockList, key)
				}
			}
		}()
	}
}
<commit_msg>fixed spelling mistake<commit_after>/*
 *  Bulldozer Framework
 *  Copyright (C) DesertBit
 */

package firewall

import (
	"code.desertbit.com/bulldozer/bulldozer/settings"
	"code.desertbit.com/bulldozer/bulldozer/utils"
	"net/http"
	"sync"
	"time"
)

var (
	blockList      map[string]int64 = make(map[string]int64)
	blockListMutex sync.Mutex

	requestCounter      map[string]int = make(map[string]int)
	requestCounterMutex sync.Mutex
)

func init() {
	// Start the loops in an new goroutine
	go cleanupRequestsCountLoop()
	go releaseBlockedRemoteAddrLoop()
}

//##############//
//### Public ###//
//##############//

// NewRequest tells the firewall, that a new request happened.
// False is returned, if this request should be blocked,
// because the IP is on the block list.
// The remote address is always returned for logging purpose.
func NewRequest(req *http.Request) (bool, string) {
	// Get the remote addresse of the request
	remoteAddr, _ := utils.RemoteAddress(req)

	// Check if this remote address is blocked
	if isBlocked(remoteAddr) {
		return false, remoteAddr
	}

	// Register the new request in a new goroutine
	go addRequest(remoteAddr)

	return true, remoteAddr
}

//###############//
//### Private ###//
//###############//

// isBlocked checks if the remote addres is blocked
func isBlocked(remoteAddr string) bool {
	// Lock the mutex
	blockListMutex.Lock()
	defer blockListMutex.Unlock()

	// Check if the remote addresse exists in the blocked map
	_, exists := blockList[remoteAddr]
	return exists
}

func addRequest(remoteAddr string) {
	// Lock the mutex
	requestCounterMutex.Lock()
	defer requestCounterMutex.Unlock()

	count, ok := requestCounter[remoteAddr]
	if !ok {
		count = 1
	}

	// Add the remote address to the block map, if the count
	// reached the limit.
	if count > settings.Settings.FirewallMaxRequestsPerMinute {
		// Remove the remote address from the request counter map
		delete(requestCounter, remoteAddr)

		// Get the current timestamp
		timestamp := time.Now().Unix()

		// Lock the mutex
		blockListMutex.Lock()
		defer blockListMutex.Unlock()

		// Add the remote address with the timestamp to the block map
		blockList[remoteAddr] = timestamp

		return
	}

	// Save the incremented count to the map
	requestCounter[remoteAddr] = count + 1
}

func cleanupRequestsCountLoop() {
	for {
		// Sleep one minute
		time.Sleep(time.Minute)

		func() {
			// Lock the mutex
			requestCounterMutex.Lock()
			defer requestCounterMutex.Unlock()

			// Clear the map if not emtpy
			if len(requestCounter) > 0 {
				requestCounter = make(map[string]int)
			}
		}()
	}
}

func releaseBlockedRemoteAddrLoop() {
	for {
		// Sleep one minute
		time.Sleep(time.Minute)

		func() {
			// Lock the mutex
			blockListMutex.Lock()
			defer blockListMutex.Unlock()

			// Return if nothing to do
			if len(blockList) == 0 {
				return
			}

			// Create the release timestamp
			releaseTimestamp := time.Now().Unix() - int64(settings.Settings.FirewallReleaseBlockAfter)

			// Remove expired blocks
			for key, timestamp := range blockList {
				if timestamp < releaseTimestamp {
					delete(blockList, key)
				}
			}
		}()
	}
}
<|endoftext|>


<commit_before>package main

import (
	"fmt"
	"strconv"
	"strings"
)

// Val is a supporting type to hold and represent enumerated value
type Val struct {
	i int
	s []string
}

// String pretty printer for Val type. It outputs either space-joined V.s,
// or V.i in string form.
func (v Val) String() string {
	s := strconv.Itoa(v.i)
	if len(v.s) > 0 {
		s = strings.Join(v.s, " ")
	}
	return s
}

// Start of the program
func main() {
	// Create filtering pipeline
	in := createPipeline(
		makeGenerator(),
		makeLimitFilter(45),
		makeValueFilter(3, "fizz"),
		makeValueFilter(5, "buzz"),
		makeValueFilter(6, "boom"),
		makeValueFilter(9, "bang"),
	)
	// Display all values that are passed through the pipeline
	for v := range in {
		fmt.Println(v)
	}
}

// Pipeline factory
func createPipeline(gen Generator, filters ...Filter) <-chan Val {
	// Start generator
	ch := gen()
	for _, f := range filters {
		// Add filter to the chain
		ch = f(ch)
	}
	// Return resulting channel
	return ch
}

// Generator type
type Generator func() <-chan Val

// Create Val generator
func makeGenerator() Generator {
	return func() <-chan Val {
		ch := make(chan Val)
		go func() {
			for i := 1; ; i++ {
				ch <- Val{i, make([]string, 0, 1)}
			}
			close(ch)
		}()
		return ch
	}
}

// Filter is generic interface to a filtering function
type Filter func(in <-chan Val) <-chan Val

// Logic function that does the actual filtering
type FilterLogicFn func(out chan<- Val, in <-chan Val)

// apply creats Filter by applying specified FilterLogicFn
func apply(fl FilterLogicFn) Filter {
	return func(in <-chan Val) <-chan Val {
		out := make(chan Val)
		go func() {
			defer close(out) // Always close channel, even if fl() panics
			fl(out, in)
		}()
		return out
	}
}

// Limit filter generated implementation.
// Must use function to capture parameter in a closure
func makeLimitFilter(limit int) Filter {
	return apply(func(limit int) FilterLogicFn {
		return func(out chan<- Val, in <-chan Val) {
			for i := 0; i < limit; i++ {
				v, ok := <-in
				if !ok {
					break
				}
				// passthrough
				out <- v
			}
		}
	}(limit))
}

// Value filter generator implementaion
// Must use function to capture parameter in a closure
func makeValueFilter(num int, msg string) Filter {
	return apply(func(div int, msg string) FilterLogicFn {
		return func(out chan<- Val, in <-chan Val) {
			for {
				v, ok := <-in
				if !ok {
					break
				}
				// Check if number if divisible by our divizor
				if v.i%div == 0 {
					v.s = append(v.s, msg)
				}
				out <- v
			}
		}
	}(num, msg))
}
<commit_msg>Renamed filter operation type<commit_after>package main

import (
	"fmt"
	"log"
	"strconv"
	"strings"
)

// Val is a supporting type to hold and represent enumerated value
type Val struct {
	i int
	s []string
}

// String pretty printer for Val type. It outputs either space-joined V.s,
// or V.i in string form.
func (v Val) String() string {
	s := strconv.Itoa(v.i)
	if len(v.s) > 0 {
		s = strings.Join(v.s, " ")
	}
	return s
}

// Start of the program
func main() {
	// Create filtering pipeline
	in := createPipeline(
		makeGenerator(),
		makeLimitFilter(45),
		makeValueFilter(3, "fizz"),
		makeValueFilter(5, "buzz"),
		makeValueFilter(6, "boom"),
		makeValueFilter(9, "bang"),
	)
	// Display all values that are passed through the pipeline
	for v := range in {
		fmt.Println(v)
	}
}

// Pipeline factory
func createPipeline(gen Generator, filters ...Filter) <-chan Val {
	// Start generator
	ch := gen()
	for _, f := range filters {
		// Add filter to the chain
		ch = f(ch)
	}
	// Return resulting channel
	return ch
}

// Generator type
type Generator func() <-chan Val

// Create Val generator
func makeGenerator() Generator {
	return func() <-chan Val {
		ch := make(chan Val)
		go func() {
			for i := 1; ; i++ {
				ch <- Val{i, make([]string, 0, 1)}
			}
		}()
		return ch
	}
}

// Filter is generic interface to a filtering function
type Filter func(in <-chan Val) <-chan Val

// Logic function that does the actual filtering
type FilterOpFn func(out chan<- Val, in <-chan Val)

// apply creats Filter by applying specified FilterOpFn
func apply(fl FilterOpFn) Filter {
	return func(in <-chan Val) <-chan Val {
		out := make(chan Val)
		go func() {
			defer close(out) // Always close channel, even if fl() panics
			fl(out, in)
		}()
		return out
	}
}

// Limit filter generated implementation.
// Must use function to capture parameter in a closure
func makeLimitFilter(limit int) Filter {
	return apply(func(limit int) FilterOpFn {
		return func(out chan<- Val, in <-chan Val) {
			for i := 0; i < limit; i++ {
				v, ok := <-in
				if !ok {
					break
				}
				// passthrough
				out <- v
			}
		}
	}(limit))
}

// Value filter generator implementaion
// Must use function to capture parameter in a closure
func makeValueFilter(num int, msg string) Filter {
	return apply(func(div int, msg string) FilterOpFn {
		return func(out chan<- Val, in <-chan Val) {
			for {
				v, ok := <-in
				if !ok {
					break
				}
				// Check if number if divisible by our divizor
				if v.i%div == 0 {
					v.s = append(v.s, msg)
				}
				out <- v
			}
		}
	}(num, msg))
}
<|endoftext|>


<commit_before>package flagfile

import (
	"flag"
	"fmt"
	"io"
	"os"
	"path/filepath"
	"reflect"
)

// FlagSet is a subset of methods of *flag.FlagSet
type FlagSet interface {
	Parse([]string) error
	Lookup(name string) *flag.Flag
}

// Decoder can decode a stream of bytes into structured values.
// The root value for flagfile.File must be of type:
//     []string
//     []interface{}
//     map[string]interface{}
//
// Example: encoding/json.Decoder
type Decoder interface {
	Decode(interface{}) error
}

type DecoderBuilder func(r io.Reader) Decoder

type file struct {
	flagset    FlagSet
	decoder    DecoderBuilder
	contextDir []string
}

func File(flagset FlagSet, decoder DecoderBuilder) flag.Value {
	return &file{
		flagset: flagset,
		decoder: decoder,
	}
}

func (file) String() string { return "" }

func (f *file) Set(path string) error {
	if !filepath.IsAbs(path) {
		var ctxDir string
		if len(f.contextDir) == 0 {
			var e error
			ctxDir, e = os.Getwd()
			if e != nil {
				return e
			}
		} else {
			ctxDir = f.contextDir[len(f.contextDir)-1]
		}
		path = filepath.Join(ctxDir, path)
	}

	r, err := os.Open(path)
	if err != nil {
		// TODO return a structured error to allow l18n
		return fmt.Errorf("%s: %s", path, err)
	}

	v, err := func(r *os.File) (interface{}, error) {
		defer r.Close()
		dec := f.decoder(r)
		var v interface{}
		err = dec.Decode(&v)
		return v, err
	}(r)
	if err != nil {
		// TODO return a structured error to allow l18n
		return fmt.Errorf("%s: can't decode: %s", path, err)
	}

	// push directory of the path
	f.contextDir = append(f.contextDir, filepath.Dir(path))
	defer func() {
		// pop contextDir
		f.contextDir = f.contextDir[:len(f.contextDir)-1]
	}()

	switch v := v.(type) {
	case []interface{}:
		args := make([]string, 0, len(v))
		for _, arg := range v {
			args = append(args, fmt.Sprint(arg))
		}
		err = f.flagset.Parse(args)
	case []string:
		err = f.flagset.Parse(v)
	case map[string]interface{}:
		err = parseObject(f.flagset, v)
		// TODO map[string]string
		// TODO map[string][]string
	default:
		return fmt.Errorf("unexpected type %T", v)
	}
	if err != nil {
		err = fmt.Errorf("%s#%s", path, err)
	}
	return err
}

func parseObject(flagset FlagSet, m map[string]interface{}) error {
	for k, v := range m {
		f := flagset.Lookup(k)
		if f == nil {
			return fmt.Errorf("/%s: unknown flag", k)
		}
		var err error
		switch v := v.(type) {
		case string:
			err = f.Value.Set(v)
		case []string:
			for i, v := range v {
				err = f.Value.Set(v)
				if err != nil {
					return fmt.Errorf("/%s/%d: %s", k, i, err)
				}
			}
		default:
			rv := reflect.ValueOf(v)
			if rv.Kind() == reflect.Array || rv.Kind() == reflect.Slice {
				for i := 0; i < rv.Len(); i++ {
					err = f.Value.Set(fmt.Sprint(rv.Index(i).Interface()))
					if err != nil {
						return fmt.Errorf("/%s/%d: %s", k, i, err)
					}
				}
			} else {
				err = f.Value.Set(fmt.Sprint(v))
			}
		}
		if err != nil {
			return fmt.Errorf("/%s: %s", k, err)
		}
	}

	return nil
}
<commit_msg>flagfile: add doc<commit_after>package flagfile

import (
	"flag"
	"fmt"
	"io"
	"os"
	"path/filepath"
	"reflect"
)

// FlagSet is a subset of methods of *flag.FlagSet
type FlagSet interface {
	Parse([]string) error
	Lookup(name string) *flag.Flag
}

// Decoder can decode a stream of bytes into structured values.
// The root value for flagfile.File must be of type:
//     []string
//     []interface{}
//     map[string]interface{}
//
// Example: encoding/json.Decoder
type Decoder interface {
	Decode(interface{}) error
}

type DecoderBuilder func(r io.Reader) Decoder

type file struct {
	flagset    FlagSet
	decoder    DecoderBuilder
	contextDir []string
}

// File allows to define a command-line flag that gives a path to a structured
// file whose content will be expanded as command-line arguments and injected into flagset.
//
// The structured data may be:
// - an array of strings given to flagset.Parse
// - a map where keys are argument names (without leading '-') and keys are values
//   or arrays of values
//
// This flag is reentrant, so the file may refer to (include) other files by reusing the
// same flag.
func File(flagset FlagSet, decoder DecoderBuilder) flag.Value {
	return &file{
		flagset: flagset,
		decoder: decoder,
	}
}

func (file) String() string { return "" }

// Set loads the given file, decodes it and interprets
// its structured data as a list of arguments.
//
// Set is part of the flag.Value interface.
func (f *file) Set(path string) error {
	if !filepath.IsAbs(path) {
		var ctxDir string
		if len(f.contextDir) == 0 {
			var e error
			ctxDir, e = os.Getwd()
			if e != nil {
				return e
			}
		} else {
			ctxDir = f.contextDir[len(f.contextDir)-1]
		}
		path = filepath.Join(ctxDir, path)
	}

	r, err := os.Open(path)
	if err != nil {
		// TODO return a structured error to allow l18n
		return fmt.Errorf("%s: %s", path, err)
	}

	v, err := func(r *os.File) (interface{}, error) {
		defer r.Close()
		dec := f.decoder(r)
		var v interface{}
		err = dec.Decode(&v)
		return v, err
	}(r)
	if err != nil {
		// TODO return a structured error to allow l18n
		return fmt.Errorf("%s: can't decode: %s", path, err)
	}

	// push directory of the path
	f.contextDir = append(f.contextDir, filepath.Dir(path))
	defer func() {
		// pop contextDir
		f.contextDir = f.contextDir[:len(f.contextDir)-1]
	}()

	switch v := v.(type) {
	case []interface{}:
		args := make([]string, 0, len(v))
		for _, arg := range v {
			args = append(args, fmt.Sprint(arg))
		}
		err = f.flagset.Parse(args)
	case []string:
		err = f.flagset.Parse(v)
	case map[string]interface{}:
		err = parseObject(f.flagset, v)
		// TODO map[string]string
		// TODO map[string][]string
	default:
		return fmt.Errorf("unexpected type %T", v)
	}
	if err != nil {
		err = fmt.Errorf("%s#%s", path, err)
	}
	return err
}

func parseObject(flagset FlagSet, m map[string]interface{}) error {
	for k, v := range m {
		f := flagset.Lookup(k)
		if f == nil {
			return fmt.Errorf("/%s: unknown flag", k)
		}
		var err error
		switch v := v.(type) {
		case string:
			err = f.Value.Set(v)
		case []string:
			for i, v := range v {
				err = f.Value.Set(v)
				if err != nil {
					return fmt.Errorf("/%s/%d: %s", k, i, err)
				}
			}
		default:
			rv := reflect.ValueOf(v)
			if rv.Kind() == reflect.Array || rv.Kind() == reflect.Slice {
				for i := 0; i < rv.Len(); i++ {
					err = f.Value.Set(fmt.Sprint(rv.Index(i).Interface()))
					if err != nil {
						return fmt.Errorf("/%s/%d: %s", k, i, err)
					}
				}
			} else {
				err = f.Value.Set(fmt.Sprint(v))
			}
		}
		if err != nil {
			return fmt.Errorf("/%s: %s", k, err)
		}
	}

	return nil
}
<|endoftext|>


<commit_before>package db

import (
	"fmt"
	"os"
	"testing"
)

func TestL0Compaction(t *testing.T) {
	path := "/tmp/compactor_test/TestL0Compaction"
	os.RemoveAll(path)
	err := os.MkdirAll(path, os.ModePerm)
	if err != nil {
		t.Error("Fails to create testing dir")
	}

	opt := DbOption{
		path:         path,
		env:          MakeNativeEnv(),
		comp:         ByteOrder(0),
		numTblCache:  8,
		minLogSize:   64,
		maxL0Levels:  4,
		minTableSize: 4 * 1024 * 1024,
	}

	db := MakeDb(opt)
	if db == nil {
		t.Error("Fails to create a DB!")
	}

	bigVal := make([]byte, 64)
	for idx, _ := range bigVal {
		bigVal[idx] = 'a'
	}

	var wopt WriteOptions
	var ropt ReadOptions

	db.Put(wopt, []byte("hello"), []byte("world"))

	for i := 1000; i < 1002; i++ {
		key := fmt.Sprintf("%d", i)
		status, finish := db.PutMore(wopt, []byte(key), bigVal)

		if !status.Ok() {
			t.Error("Fails to put an item!")
		}

		// Wait until L0 compaction completes.
		for v := range finish {
			if v != true {
				t.Error("Does not finish compaction successfully")
			}
		}
	}

	val, status := db.Get(ropt, []byte("hello"))
	if !status.Ok() || string(val) != "world" {
		t.Error("Fails to get a key")
	}
}
<commit_msg>Extend L0 compaction test case<commit_after>package db

import (
	"fmt"
	"os"
	"testing"
)

func TestL0Compaction(t *testing.T) {
	path := "/tmp/compactor_test/TestL0Compaction"
	os.RemoveAll(path)
	err := os.MkdirAll(path, os.ModePerm)
	if err != nil {
		t.Error("Fails to create testing dir")
	}

	opt := DbOption{
		path:         path,
		env:          MakeNativeEnv(),
		comp:         ByteOrder(0),
		numTblCache:  8,
		minLogSize:   64,
		maxL0Levels:  4,
		minTableSize: 4 * 1024 * 1024,
	}

	db := MakeDb(opt)
	if db == nil {
		t.Error("Fails to create a DB!")
	}

	bigVal := make([]byte, 64)
	for idx, _ := range bigVal {
		bigVal[idx] = 'a'
	}

	var wopt WriteOptions
	var ropt ReadOptions

	db.Put(wopt, []byte("hello"), []byte("world"))

	for i := 1000; i < 1002; i++ {
		key := fmt.Sprintf("%d", i)
		status, finish := db.PutMore(wopt, []byte(key), bigVal)

		if !status.Ok() {
			t.Error("Fails to put an item!")
		}

		// Wait until L0 compaction completes.
		for v := range finish {
			if v != true {
				t.Error("Does not finish compaction successfully")
			}
		}
	}

	// Verify a particular key is present.
	val, status := db.Get(ropt, []byte("hello"))
	if !status.Ok() || string(val) != "world" {
		t.Error("Fails to get a key")
	}

	// Verify that all keys are present.
	it := db.NewIterator(ropt)
	it.SeekToFirst()

	if !it.Valid() || string(it.Key()) != "1000" {
		t.Error("Fails to get expected key")
	}

	it.Next()

	if !it.Valid() || string(it.Key()) != "1001" {
		t.Error("Fails to get expected key")
	}

	it.Next()

	if !it.Valid() || string(it.Key()) != "hello" {
		t.Error("Fails to get expected key")
	}
}
<|endoftext|>


<commit_before>// Copyright 2015 Google Inc. All Rights Reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package cachingfs

import (
	"fmt"
	"os"
	"time"

	"github.com/jacobsa/fuse"
	"github.com/jacobsa/fuse/fuseops"
	"github.com/jacobsa/gcloud/syncutil"
	"golang.org/x/net/context"
)

const (
	// Sizes of the files according to the file system.
	FooSize = 123
	BarSize = 456
)

// A file system with a fixed structure that looks like this:
//
//     foo
//     dir/
//         bar
//
// The file system is configured with durations that specify how long to allow
// inode entries and attributes to be cached, used when responding to fuse
// requests. It also exposes methods for renumbering inodes and updating mtimes
// that are useful in testing that these durations are honored.
type CachingFS interface {
	fuse.Server

	// Return the current inode ID of the file/directory with the given name.
	FooID() fuseops.InodeID
	DirID() fuseops.InodeID
	BarID() fuseops.InodeID

	// Cause the inode IDs to change to values that have never before been used.
	RenumberInodes()

	// Cause further queries for the attributes of inodes to use the supplied
	// time as the inode's mtime.
	SetMtime(mtime time.Time)
}

// Create a file system that issues cacheable responses according to the
// following rules:
//
//  *  LookUpInodeResponse.Entry.EntryExpiration is set according to
//     lookupEntryTimeout.
//
//  *  GetInodeAttributesResponse.AttributesExpiration is set according to
//     getattrTimeout.
//
//  *  Nothing else is marked cacheable. (In particular, the attributes
//     returned by LookUpInode are not cacheable.)
//
func NewCachingFS(
	lookupEntryTimeout time.Duration,
	getattrTimeout time.Duration) (fs CachingFS, err error) {
	roundUp := func(n fuseops.InodeID) fuseops.InodeID {
		return numInodes * ((n + numInodes - 1) / numInodes)
	}

	cfs := &cachingFS{
		lookupEntryTimeout: lookupEntryTimeout,
		getattrTimeout:     getattrTimeout,
		baseID:             roundUp(fuse.RootInodeID + 1),
		mtime:              time.Now(),
	}

	cfs.mu = syncutil.NewInvariantMutex(cfs.checkInvariants)

	fs = cfs
	return
}

const (
	// Inode IDs are issued such that "foo" always receives an ID that is
	// congruent to fooOffset modulo numInodes, etc.
	fooOffset = iota
	dirOffset
	barOffset
	numInodes
)

type cachingFS struct {
	/////////////////////////
	// Constant data
	/////////////////////////

	lookupEntryTimeout time.Duration
	getattrTimeout     time.Duration

	/////////////////////////
	// Mutable state
	/////////////////////////

	mu syncutil.InvariantMutex

	// The current ID of the lowest numbered non-root inode.
	//
	// INVARIANT: baseID > fuse.RootInodeID
	// INVARIANT: baseID % numInodes == 0
	//
	// GUARDED_BY(mu)
	baseID fuseops.InodeID

	// GUARDED_BY(mu)
	mtime time.Time
}

////////////////////////////////////////////////////////////////////////
// Helpers
////////////////////////////////////////////////////////////////////////

func (fs *cachingFS) checkInvariants() {
	// INVARIANT: baseID > fuse.RootInodeID
	// INVARIANT: baseID % numInodes == 0
	if fs.baseID <= fuse.RootInodeID || fs.baseID%numInodes != 0 {
		panic(fmt.Sprintf("Bad baseID: %v", fs.baseID))
	}
}

// LOCKS_REQUIRED(fs.mu)
func (fs *cachingFS) fooID() fuseops.InodeID {
	return fs.baseID + fooOffset
}

// LOCKS_REQUIRED(fs.mu)
func (fs *cachingFS) dirID() fuseops.InodeID {
	return fs.baseID + dirOffset
}

// LOCKS_REQUIRED(fs.mu)
func (fs *cachingFS) barID() fuseops.InodeID {
	return fs.baseID + barOffset
}

// LOCKS_REQUIRED(fs.mu)
func (fs *cachingFS) rootAttrs() fuseops.InodeAttributes {
	return fuseops.InodeAttributes{
		Mode:  os.ModeDir | 0777,
		Mtime: fs.mtime,
	}
}

// LOCKS_REQUIRED(fs.mu)
func (fs *cachingFS) fooAttrs() fuseops.InodeAttributes {
	return fuseops.InodeAttributes{
		Nlink: 1,
		Size:  FooSize,
		Mode:  0777,
		Mtime: fs.mtime,
	}
}

// LOCKS_REQUIRED(fs.mu)
func (fs *cachingFS) dirAttrs() fuseops.InodeAttributes {
	return fuseops.InodeAttributes{
		Nlink: 1,
		Mode:  os.ModeDir | 0777,
		Mtime: fs.mtime,
	}
}

// LOCKS_REQUIRED(fs.mu)
func (fs *cachingFS) barAttrs() fuseops.InodeAttributes {
	return fuseops.InodeAttributes{
		Nlink: 1,
		Size:  BarSize,
		Mode:  0777,
		Mtime: fs.mtime,
	}
}

////////////////////////////////////////////////////////////////////////
// Public interface
////////////////////////////////////////////////////////////////////////

// LOCKS_EXCLUDED(fs.mu)
func (fs *cachingFS) FooID() fuseops.InodeID {
	fs.mu.Lock()
	defer fs.mu.Unlock()

	return fs.fooID()
}

// LOCKS_EXCLUDED(fs.mu)
func (fs *cachingFS) DirID() fuseops.InodeID {
	fs.mu.Lock()
	defer fs.mu.Unlock()

	return fs.dirID()
}

// LOCKS_EXCLUDED(fs.mu)
func (fs *cachingFS) BarID() fuseops.InodeID {
	fs.mu.Lock()
	defer fs.mu.Unlock()

	return fs.barID()
}

// LOCKS_EXCLUDED(fs.mu)
func (fs *cachingFS) RenumberInodes() {
	fs.mu.Lock()
	defer fs.mu.Unlock()

	fs.baseID += numInodes
}

// LOCKS_EXCLUDED(fs.mu)
func (fs *cachingFS) SetMtime(mtime time.Time) {
	fs.mu.Lock()
	defer fs.mu.Unlock()

	fs.mtime = mtime
}

////////////////////////////////////////////////////////////////////////
// FileSystem methods
////////////////////////////////////////////////////////////////////////

func (fs *cachingFS) Init(
	ctx context.Context,
	req *fuse.InitRequest) (resp *fuse.InitResponse, err error) {
	resp = &fuse.InitResponse{}
	return
}

// LOCKS_EXCLUDED(fs.mu)
func (fs *cachingFS) LookUpInode(
	ctx context.Context,
	req *fuse.LookUpInodeRequest) (resp *fuse.LookUpInodeResponse, err error) {
	resp = &fuse.LookUpInodeResponse{}

	fs.mu.Lock()
	defer fs.mu.Unlock()

	// Find the ID and attributes.
	var id fuseops.InodeID
	var attrs fuseops.InodeAttributes

	switch req.Name {
	case "foo":
		// Parent must be the root.
		if req.Parent != fuse.RootInodeID {
			err = fuse.ENOENT
			return
		}

		id = fs.fooID()
		attrs = fs.fooAttrs()

	case "dir":
		// Parent must be the root.
		if req.Parent != fuse.RootInodeID {
			err = fuse.ENOENT
			return
		}

		id = fs.dirID()
		attrs = fs.dirAttrs()

	case "bar":
		// Parent must be dir.
		if req.Parent == fuse.RootInodeID || req.Parent%numInodes != dirOffset {
			err = fuse.ENOENT
			return
		}

		id = fs.barID()
		attrs = fs.barAttrs()

	default:
		err = fuse.ENOENT
		return
	}

	// Fill in the response.
	resp.Entry.Child = id
	resp.Entry.Attributes = attrs
	resp.Entry.EntryExpiration = time.Now().Add(fs.lookupEntryTimeout)

	return
}

// LOCKS_EXCLUDED(fs.mu)
func (fs *cachingFS) GetInodeAttributes(
	ctx context.Context,
	req *fuse.GetInodeAttributesRequest) (
	resp *fuse.GetInodeAttributesResponse, err error) {
	resp = &fuse.GetInodeAttributesResponse{}

	fs.mu.Lock()
	defer fs.mu.Unlock()

	// Figure out which inode the request is for.
	var attrs fuseops.InodeAttributes

	switch {
	case req.Inode == fuse.RootInodeID:
		attrs = fs.rootAttrs()

	case req.Inode%numInodes == fooOffset:
		attrs = fs.fooAttrs()

	case req.Inode%numInodes == dirOffset:
		attrs = fs.dirAttrs()

	case req.Inode%numInodes == barOffset:
		attrs = fs.barAttrs()
	}

	// Fill in the response.
	resp.Attributes = attrs
	resp.AttributesExpiration = time.Now().Add(fs.getattrTimeout)

	return
}

func (fs *cachingFS) OpenDir(
	ctx context.Context,
	req *fuse.OpenDirRequest) (
	resp *fuse.OpenDirResponse, err error) {
	resp = &fuse.OpenDirResponse{}
	return
}

func (fs *cachingFS) OpenFile(
	ctx context.Context,
	req *fuse.OpenFileRequest) (
	resp *fuse.OpenFileResponse, err error) {
	resp = &fuse.OpenFileResponse{}
	return
}
<commit_msg>Fixed more errors.<commit_after>// Copyright 2015 Google Inc. All Rights Reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package cachingfs

import (
	"fmt"
	"os"
	"time"

	"github.com/jacobsa/fuse"
	"github.com/jacobsa/fuse/fuseops"
	"github.com/jacobsa/gcloud/syncutil"
	"golang.org/x/net/context"
)

const (
	// Sizes of the files according to the file system.
	FooSize = 123
	BarSize = 456
)

// A file system with a fixed structure that looks like this:
//
//     foo
//     dir/
//         bar
//
// The file system is configured with durations that specify how long to allow
// inode entries and attributes to be cached, used when responding to fuse
// requests. It also exposes methods for renumbering inodes and updating mtimes
// that are useful in testing that these durations are honored.
type CachingFS interface {
	fuse.Server

	// Return the current inode ID of the file/directory with the given name.
	FooID() fuseops.InodeID
	DirID() fuseops.InodeID
	BarID() fuseops.InodeID

	// Cause the inode IDs to change to values that have never before been used.
	RenumberInodes()

	// Cause further queries for the attributes of inodes to use the supplied
	// time as the inode's mtime.
	SetMtime(mtime time.Time)
}

// Create a file system that issues cacheable responses according to the
// following rules:
//
//  *  LookUpInodeResponse.Entry.EntryExpiration is set according to
//     lookupEntryTimeout.
//
//  *  GetInodeAttributesResponse.AttributesExpiration is set according to
//     getattrTimeout.
//
//  *  Nothing else is marked cacheable. (In particular, the attributes
//     returned by LookUpInode are not cacheable.)
//
func NewCachingFS(
	lookupEntryTimeout time.Duration,
	getattrTimeout time.Duration) (fs CachingFS, err error) {
	roundUp := func(n fuseops.InodeID) fuseops.InodeID {
		return numInodes * ((n + numInodes - 1) / numInodes)
	}

	cfs := &cachingFS{
		lookupEntryTimeout: lookupEntryTimeout,
		getattrTimeout:     getattrTimeout,
		baseID:             roundUp(fuseops.RootInodeID + 1),
		mtime:              time.Now(),
	}

	cfs.mu = syncutil.NewInvariantMutex(cfs.checkInvariants)

	fs = cfs
	return
}

const (
	// Inode IDs are issued such that "foo" always receives an ID that is
	// congruent to fooOffset modulo numInodes, etc.
	fooOffset = iota
	dirOffset
	barOffset
	numInodes
)

type cachingFS struct {
	/////////////////////////
	// Constant data
	/////////////////////////

	lookupEntryTimeout time.Duration
	getattrTimeout     time.Duration

	/////////////////////////
	// Mutable state
	/////////////////////////

	mu syncutil.InvariantMutex

	// The current ID of the lowest numbered non-root inode.
	//
	// INVARIANT: baseID > fuseops.RootInodeID
	// INVARIANT: baseID % numInodes == 0
	//
	// GUARDED_BY(mu)
	baseID fuseops.InodeID

	// GUARDED_BY(mu)
	mtime time.Time
}

////////////////////////////////////////////////////////////////////////
// Helpers
////////////////////////////////////////////////////////////////////////

func (fs *cachingFS) checkInvariants() {
	// INVARIANT: baseID > fuseops.RootInodeID
	// INVARIANT: baseID % numInodes == 0
	if fs.baseID <= fuseops.RootInodeID || fs.baseID%numInodes != 0 {
		panic(fmt.Sprintf("Bad baseID: %v", fs.baseID))
	}
}

// LOCKS_REQUIRED(fs.mu)
func (fs *cachingFS) fooID() fuseops.InodeID {
	return fs.baseID + fooOffset
}

// LOCKS_REQUIRED(fs.mu)
func (fs *cachingFS) dirID() fuseops.InodeID {
	return fs.baseID + dirOffset
}

// LOCKS_REQUIRED(fs.mu)
func (fs *cachingFS) barID() fuseops.InodeID {
	return fs.baseID + barOffset
}

// LOCKS_REQUIRED(fs.mu)
func (fs *cachingFS) rootAttrs() fuseops.InodeAttributes {
	return fuseops.InodeAttributes{
		Mode:  os.ModeDir | 0777,
		Mtime: fs.mtime,
	}
}

// LOCKS_REQUIRED(fs.mu)
func (fs *cachingFS) fooAttrs() fuseops.InodeAttributes {
	return fuseops.InodeAttributes{
		Nlink: 1,
		Size:  FooSize,
		Mode:  0777,
		Mtime: fs.mtime,
	}
}

// LOCKS_REQUIRED(fs.mu)
func (fs *cachingFS) dirAttrs() fuseops.InodeAttributes {
	return fuseops.InodeAttributes{
		Nlink: 1,
		Mode:  os.ModeDir | 0777,
		Mtime: fs.mtime,
	}
}

// LOCKS_REQUIRED(fs.mu)
func (fs *cachingFS) barAttrs() fuseops.InodeAttributes {
	return fuseops.InodeAttributes{
		Nlink: 1,
		Size:  BarSize,
		Mode:  0777,
		Mtime: fs.mtime,
	}
}

////////////////////////////////////////////////////////////////////////
// Public interface
////////////////////////////////////////////////////////////////////////

// LOCKS_EXCLUDED(fs.mu)
func (fs *cachingFS) FooID() fuseops.InodeID {
	fs.mu.Lock()
	defer fs.mu.Unlock()

	return fs.fooID()
}

// LOCKS_EXCLUDED(fs.mu)
func (fs *cachingFS) DirID() fuseops.InodeID {
	fs.mu.Lock()
	defer fs.mu.Unlock()

	return fs.dirID()
}

// LOCKS_EXCLUDED(fs.mu)
func (fs *cachingFS) BarID() fuseops.InodeID {
	fs.mu.Lock()
	defer fs.mu.Unlock()

	return fs.barID()
}

// LOCKS_EXCLUDED(fs.mu)
func (fs *cachingFS) RenumberInodes() {
	fs.mu.Lock()
	defer fs.mu.Unlock()

	fs.baseID += numInodes
}

// LOCKS_EXCLUDED(fs.mu)
func (fs *cachingFS) SetMtime(mtime time.Time) {
	fs.mu.Lock()
	defer fs.mu.Unlock()

	fs.mtime = mtime
}

////////////////////////////////////////////////////////////////////////
// Op methods
////////////////////////////////////////////////////////////////////////

func (fs *cachingFS) init(op *fuseops.InitOp) {
	op.Respond(nil)
}

// LOCKS_EXCLUDED(fs.mu)
func (fs *cachingFS) LookUpInode(
	ctx context.Context,
	req *fuse.LookUpInodeRequest) (resp *fuse.LookUpInodeResponse, err error) {
	resp = &fuse.LookUpInodeResponse{}

	fs.mu.Lock()
	defer fs.mu.Unlock()

	// Find the ID and attributes.
	var id fuseops.InodeID
	var attrs fuseops.InodeAttributes

	switch req.Name {
	case "foo":
		// Parent must be the root.
		if req.Parent != fuseops.RootInodeID {
			err = fuse.ENOENT
			return
		}

		id = fs.fooID()
		attrs = fs.fooAttrs()

	case "dir":
		// Parent must be the root.
		if req.Parent != fuseops.RootInodeID {
			err = fuse.ENOENT
			return
		}

		id = fs.dirID()
		attrs = fs.dirAttrs()

	case "bar":
		// Parent must be dir.
		if req.Parent == fuseops.RootInodeID || req.Parent%numInodes != dirOffset {
			err = fuse.ENOENT
			return
		}

		id = fs.barID()
		attrs = fs.barAttrs()

	default:
		err = fuse.ENOENT
		return
	}

	// Fill in the response.
	resp.Entry.Child = id
	resp.Entry.Attributes = attrs
	resp.Entry.EntryExpiration = time.Now().Add(fs.lookupEntryTimeout)

	return
}

// LOCKS_EXCLUDED(fs.mu)
func (fs *cachingFS) GetInodeAttributes(
	ctx context.Context,
	req *fuse.GetInodeAttributesRequest) (
	resp *fuse.GetInodeAttributesResponse, err error) {
	resp = &fuse.GetInodeAttributesResponse{}

	fs.mu.Lock()
	defer fs.mu.Unlock()

	// Figure out which inode the request is for.
	var attrs fuseops.InodeAttributes

	switch {
	case req.Inode == fuseops.RootInodeID:
		attrs = fs.rootAttrs()

	case req.Inode%numInodes == fooOffset:
		attrs = fs.fooAttrs()

	case req.Inode%numInodes == dirOffset:
		attrs = fs.dirAttrs()

	case req.Inode%numInodes == barOffset:
		attrs = fs.barAttrs()
	}

	// Fill in the response.
	resp.Attributes = attrs
	resp.AttributesExpiration = time.Now().Add(fs.getattrTimeout)

	return
}

func (fs *cachingFS) OpenDir(
	ctx context.Context,
	req *fuse.OpenDirRequest) (
	resp *fuse.OpenDirResponse, err error) {
	resp = &fuse.OpenDirResponse{}
	return
}

func (fs *cachingFS) OpenFile(
	ctx context.Context,
	req *fuse.OpenFileRequest) (
	resp *fuse.OpenFileResponse, err error) {
	resp = &fuse.OpenFileResponse{}
	return
}
<|endoftext|>


<commit_before>package volume

import (
	"errors"
	"math"

	"github.com/go-ole/go-ole"
	"github.com/moutend/go-wca"
)

// GetVolume returns the current volume (0 to 100).
func GetVolume() (int, error) {
	vol, err := invoke(func(aev *wca.IAudioEndpointVolume) (interface{}, error) {
		var level float32
		err := aev.GetMasterVolumeLevelScalar(&level)
		vol := math.Floor(float64(level*100.0 + 0.5))
		return vol, err
	})
	if vol == nil {
		return 0, err
	}
	return vol.(int), err
}

// SetVolume sets the sound volume to the specified value.
func SetVolume(volume int) error {
	if volume < 0 || 100 < volume {
		return errors.New("out of valid volume range")
	}
	_, err := invoke(func(aev *wca.IAudioEndpointVolume) (interface{}, error) {
		err := aev.SetMasterVolumeLevelScalar(float32(volume)/100, nil)
		return nil, err
	})
	return err
}

// IncreaseVolume increases (or decreases) the audio volume by the specified value.
func IncreaseVolume(diff int) error {
	panic("not implemented on Windows")
	return nil
}

// GetMuted returns the current muted status.
func GetMuted() (bool, error) {
	panic("not implemented on Windows")
	return false, nil
}

// Mute mutes the audio.
func Mute() error {
	panic("not implemented on Windows")
	return nil
}

// Unmute unmutes the audio.
func Unmute() error {
	panic("not implemented on Windows")
	return nil
}

func invoke(f func(aev *wca.IAudioEndpointVolume) (interface{}, error)) (ret interface{}, err error) {
	if err = ole.CoInitializeEx(0, ole.COINIT_APARTMENTTHREADED); err != nil {
		return
	}
	defer ole.CoUninitialize()

	var mmde *wca.IMMDeviceEnumerator
	if err = wca.CoCreateInstance(wca.CLSID_MMDeviceEnumerator, 0, wca.CLSCTX_ALL, wca.IID_IMMDeviceEnumerator, &mmde); err != nil {
		return
	}
	defer mmde.Release()

	var mmd *wca.IMMDevice
	if err = mmde.GetDefaultAudioEndpoint(wca.ERender, wca.EConsole, &mmd); err != nil {
		return
	}
	defer mmd.Release()

	var ps *wca.IPropertyStore
	if err = mmd.OpenPropertyStore(wca.STGM_READ, &ps); err != nil {
		return
	}
	defer ps.Release()

	var pv wca.PROPVARIANT
	if err = ps.GetValue(&wca.PKEY_Device_FriendlyName, &pv); err != nil {
		return
	}

	var aev *wca.IAudioEndpointVolume
	if err = mmd.Activate(wca.IID_IAudioEndpointVolume, wca.CLSCTX_ALL, nil, &aev); err != nil {
		return
	}
	defer aev.Release()

	ret, err = f(aev)
	return
}
<commit_msg>implement IncreaseVolume for Windows<commit_after>package volume

import (
	"errors"
	"math"

	"github.com/go-ole/go-ole"
	"github.com/moutend/go-wca"
)

// GetVolume returns the current volume (0 to 100).
func GetVolume() (int, error) {
	vol, err := invoke(func(aev *wca.IAudioEndpointVolume) (interface{}, error) {
		var level float32
		err := aev.GetMasterVolumeLevelScalar(&level)
		vol := math.Floor(float64(level*100.0 + 0.5))
		return vol, err
	})
	if vol == nil {
		return 0, err
	}
	return vol.(int), err
}

// SetVolume sets the sound volume to the specified value.
func SetVolume(volume int) error {
	if volume < 0 || 100 < volume {
		return errors.New("out of valid volume range")
	}
	_, err := invoke(func(aev *wca.IAudioEndpointVolume) (interface{}, error) {
		err := aev.SetMasterVolumeLevelScalar(float32(volume)/100, nil)
		return nil, err
	})
	return err
}

// IncreaseVolume increases (or decreases) the audio volume by the specified value.
func IncreaseVolume(diff int) error {
	_, err := invoke(func(aev *wca.IAudioEndpointVolume) (interface{}, error) {
		var level float32
		err := aev.GetMasterVolumeLevelScalar(&level)
		if err != nil {
			return nil, err
		}
		vol := math.Min(math.Max(math.Floor(float64(level*100.0+0.5))+float64(diff), 0.0), 100.0)
		err = aev.SetMasterVolumeLevelScalar(float32(vol)/100, nil)
		return nil, err
	})
	return err
}

// GetMuted returns the current muted status.
func GetMuted() (bool, error) {
	panic("not implemented on Windows")
	return false, nil
}

// Mute mutes the audio.
func Mute() error {
	panic("not implemented on Windows")
	return nil
}

// Unmute unmutes the audio.
func Unmute() error {
	panic("not implemented on Windows")
	return nil
}

func invoke(f func(aev *wca.IAudioEndpointVolume) (interface{}, error)) (ret interface{}, err error) {
	if err = ole.CoInitializeEx(0, ole.COINIT_APARTMENTTHREADED); err != nil {
		return
	}
	defer ole.CoUninitialize()

	var mmde *wca.IMMDeviceEnumerator
	if err = wca.CoCreateInstance(wca.CLSID_MMDeviceEnumerator, 0, wca.CLSCTX_ALL, wca.IID_IMMDeviceEnumerator, &mmde); err != nil {
		return
	}
	defer mmde.Release()

	var mmd *wca.IMMDevice
	if err = mmde.GetDefaultAudioEndpoint(wca.ERender, wca.EConsole, &mmd); err != nil {
		return
	}
	defer mmd.Release()

	var ps *wca.IPropertyStore
	if err = mmd.OpenPropertyStore(wca.STGM_READ, &ps); err != nil {
		return
	}
	defer ps.Release()

	var pv wca.PROPVARIANT
	if err = ps.GetValue(&wca.PKEY_Device_FriendlyName, &pv); err != nil {
		return
	}

	var aev *wca.IAudioEndpointVolume
	if err = mmd.Activate(wca.IID_IAudioEndpointVolume, wca.CLSCTX_ALL, nil, &aev); err != nil {
		return
	}
	defer aev.Release()

	ret, err = f(aev)
	return
}
<|endoftext|>


<commit_before>//  Copyright (c) 2014 Marty Schoch
//  Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file
//  except in compliance with the License. You may obtain a copy of the License at
//    http://www.apache.org/licenses/LICENSE-2.0
//  Unless required by applicable law or agreed to in writing, software distributed under the
//  License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND,
//  either express or implied. See the License for the specific language governing permissions
//  and limitations under the License.

package gouchstore

import (
	"hash/crc32"

	"code.google.com/p/snappy-go/snappy"
)

const gs_CHUNK_LENGTH_SIZE int64 = 4
const gs_CHUNK_CRC_SIZE int64 = 4

// attempt to read a chunk at the specified location
func (g *Gouchstore) readChunkAt(pos int64, header bool) ([]byte, error) {
	// chunk starts with 8 bytes (32bit length, 32bit crc)
	chunkPrefix := make([]byte, gs_CHUNK_LENGTH_SIZE+gs_CHUNK_CRC_SIZE)
	n, err := g.readAt(chunkPrefix, pos)
	if err != nil {
		return nil, err
	}
	if n < gs_CHUNK_LENGTH_SIZE+gs_CHUNK_CRC_SIZE {
		return nil, gs_ERROR_INVALID_CHUNK_SHORT_PREFIX
	}

	size := decode_raw31(chunkPrefix[0:gs_CHUNK_LENGTH_SIZE])
	crc := decode_raw32(chunkPrefix[gs_CHUNK_LENGTH_SIZE : gs_CHUNK_LENGTH_SIZE+gs_CHUNK_CRC_SIZE])

	// size should at least be the size of the length field + 1
	if size < uint32(gs_CHUNK_LENGTH_SIZE+1) {
		return nil, gs_ERROR_INVALID_CHUNK_SIZE_TOO_SMALL
	}
	if header {
		size -= uint32(gs_CHUNK_LENGTH_SIZE) // headers include the length of the hash, data does not
	}

	data := make([]byte, size)
	pos += n // skip the actual number of bytes read for the header (may be more than header size if we crossed a block boundary)
	n, err = g.readAt(data, pos)
	if uint32(n) < size {
		return nil, gs_ERROR_INVALID_CHUNK_DATA_LESS_THAN_SIZE
	}

	// validate crc
	actualCRC := crc32.ChecksumIEEE(data)
	if actualCRC != crc {
		return nil, gs_ERROR_INVALID_CHUNK_BAD_CRC
	}

	return data, nil
}

func (g *Gouchstore) readCompressedDataChunkAt(pos int64) ([]byte, error) {
	chunk, err := g.readChunkAt(pos, false)
	if err != nil {
		return nil, err
	}

	decompressedChunk, err := snappy.Decode(nil, chunk)
	if err != nil {
		return nil, err
	}
	return decompressedChunk, nil
}

func (g *Gouchstore) writeChunk(buf []byte, header bool) (int64, int64, error) {
	// always write to the end of the file
	startPos := g.pos
	pos := startPos
	endpos := pos

	// if we're writing a header, advance to the next block size boundary
	if header {
		if pos%gs_BLOCK_SIZE != 0 {
			pos += (gs_BLOCK_SIZE - (pos % gs_BLOCK_SIZE))
		}
	}

	// chunk starts with 8 bytes (32bit length, 32bit crc)
	size := uint32(len(buf))
	if header {
		size += uint32(gs_CHUNK_CRC_SIZE) // header chunks include the length of the hash
	}
	crc := crc32.ChecksumIEEE(buf)

	var sizeBytes []byte
	if header {
		sizeBytes = encode_raw32(size)
	} else {
		sizeBytes = encode_raw31_highestbiton(size)
	}
	crcBytes := encode_raw32(crc)
	written, err := g.writeAt(sizeBytes, pos, header)
	if err != nil {
		return pos, written, err
	}
	g.pos += written
	pos += written
	endpos += written
	written, err = g.writeAt(crcBytes, pos, header)
	if err != nil {
		return pos, written, err
	}
	g.pos += written
	pos += written
	endpos += written
	written, err = g.writeAt(buf, pos, header)
	if err != nil {
		return pos, written, err
	}
	g.pos += written
	endpos += written

	return startPos, endpos - startPos, nil
}

func (g *Gouchstore) writeCompressedChunk(buf []byte) (int64, int64, error) {
	compressed, err := snappy.Encode(nil, buf)
	if err != nil {
		return -1, -1, err
	}
	return g.writeChunk(compressed, false)
}
<commit_msg>fix major bug overwriting file header<commit_after>//  Copyright (c) 2014 Marty Schoch
//  Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file
//  except in compliance with the License. You may obtain a copy of the License at
//    http://www.apache.org/licenses/LICENSE-2.0
//  Unless required by applicable law or agreed to in writing, software distributed under the
//  License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND,
//  either express or implied. See the License for the specific language governing permissions
//  and limitations under the License.

package gouchstore

import (
	"hash/crc32"

	"code.google.com/p/snappy-go/snappy"
)

const gs_CHUNK_LENGTH_SIZE int64 = 4
const gs_CHUNK_CRC_SIZE int64 = 4

// attempt to read a chunk at the specified location
func (g *Gouchstore) readChunkAt(pos int64, header bool) ([]byte, error) {
	// chunk starts with 8 bytes (32bit length, 32bit crc)
	chunkPrefix := make([]byte, gs_CHUNK_LENGTH_SIZE+gs_CHUNK_CRC_SIZE)
	n, err := g.readAt(chunkPrefix, pos)
	if err != nil {
		return nil, err
	}
	if n < gs_CHUNK_LENGTH_SIZE+gs_CHUNK_CRC_SIZE {
		return nil, gs_ERROR_INVALID_CHUNK_SHORT_PREFIX
	}

	size := decode_raw31(chunkPrefix[0:gs_CHUNK_LENGTH_SIZE])
	crc := decode_raw32(chunkPrefix[gs_CHUNK_LENGTH_SIZE : gs_CHUNK_LENGTH_SIZE+gs_CHUNK_CRC_SIZE])

	// size should at least be the size of the length field + 1
	if size < uint32(gs_CHUNK_LENGTH_SIZE+1) {
		return nil, gs_ERROR_INVALID_CHUNK_SIZE_TOO_SMALL
	}
	if header {
		size -= uint32(gs_CHUNK_LENGTH_SIZE) // headers include the length of the hash, data does not
	}

	data := make([]byte, size)
	pos += n // skip the actual number of bytes read for the header (may be more than header size if we crossed a block boundary)
	n, err = g.readAt(data, pos)
	if uint32(n) < size {
		return nil, gs_ERROR_INVALID_CHUNK_DATA_LESS_THAN_SIZE
	}

	// validate crc
	actualCRC := crc32.ChecksumIEEE(data)
	if actualCRC != crc {
		return nil, gs_ERROR_INVALID_CHUNK_BAD_CRC
	}

	return data, nil
}

func (g *Gouchstore) readCompressedDataChunkAt(pos int64) ([]byte, error) {
	chunk, err := g.readChunkAt(pos, false)
	if err != nil {
		return nil, err
	}

	decompressedChunk, err := snappy.Decode(nil, chunk)
	if err != nil {
		return nil, err
	}
	return decompressedChunk, nil
}

func (g *Gouchstore) writeChunk(buf []byte, header bool) (int64, int64, error) {
	// always write to the end of the file
	startPos := g.pos
	pos := startPos
	endpos := pos

	// if we're writing a header, advance to the next block size boundary
	if header {
		if pos%gs_BLOCK_SIZE != 0 {
			pos += (gs_BLOCK_SIZE - (pos % gs_BLOCK_SIZE))
			g.pos += (gs_BLOCK_SIZE - (pos % gs_BLOCK_SIZE))
		}
	}

	// chunk starts with 8 bytes (32bit length, 32bit crc)
	size := uint32(len(buf))
	if header {
		size += uint32(gs_CHUNK_CRC_SIZE) // header chunks include the length of the hash
	}
	crc := crc32.ChecksumIEEE(buf)

	var sizeBytes []byte
	if header {
		sizeBytes = encode_raw32(size)
	} else {
		sizeBytes = encode_raw31_highestbiton(size)
	}
	crcBytes := encode_raw32(crc)
	written, err := g.writeAt(sizeBytes, pos, header)
	if err != nil {
		return pos, written, err
	}
	g.pos += written
	pos += written
	endpos += written
	written, err = g.writeAt(crcBytes, pos, header)
	if err != nil {
		return pos, written, err
	}
	g.pos += written
	pos += written
	endpos += written
	written, err = g.writeAt(buf, pos, header)
	if err != nil {
		return pos, written, err
	}
	g.pos += written
	endpos += written

	return startPos, endpos - startPos, nil
}

func (g *Gouchstore) writeCompressedChunk(buf []byte) (int64, int64, error) {
	compressed, err := snappy.Encode(nil, buf)
	if err != nil {
		return -1, -1, err
	}
	return g.writeChunk(compressed, false)
}
<|endoftext|>


<commit_before>package jobs

import (
	"encoding/json"
	"net/http"
	"time"

	"github.com/cozy/cozy-stack/pkg/consts"
	"github.com/cozy/cozy-stack/pkg/couchdb"
	"github.com/cozy/cozy-stack/pkg/globals"
	"github.com/cozy/cozy-stack/pkg/jobs"
	"github.com/cozy/cozy-stack/pkg/scheduler"
	"github.com/cozy/cozy-stack/web/jsonapi"
	"github.com/cozy/cozy-stack/web/middlewares"
	"github.com/cozy/cozy-stack/web/permissions"
	multierror "github.com/hashicorp/go-multierror"
	"github.com/labstack/echo"

	// exec is needed for bad triggers cleanup

	// import workers
	_ "github.com/cozy/cozy-stack/pkg/workers/exec"
	_ "github.com/cozy/cozy-stack/pkg/workers/log"
	_ "github.com/cozy/cozy-stack/pkg/workers/mails"
	_ "github.com/cozy/cozy-stack/pkg/workers/sharings"
	_ "github.com/cozy/cozy-stack/pkg/workers/thumbnail"
	_ "github.com/cozy/cozy-stack/pkg/workers/unzip"
)

type (
	apiJob struct {
		j *jobs.JobInfos
	}
	apiJobRequest struct {
		Arguments json.RawMessage  `json:"arguments"`
		Options   *jobs.JobOptions `json:"options"`
	}
	apiQueue struct {
		Count      int `json:"count"`
		workerType string
	}
	apiTrigger struct {
		t scheduler.Trigger
	}
	apiTriggerRequest struct {
		Type            string           `json:"type"`
		Arguments       string           `json:"arguments"`
		WorkerType      string           `json:"worker"`
		WorkerArguments json.RawMessage  `json:"worker_arguments"`
		Debounce        string           `json:"debounce"`
		Options         *jobs.JobOptions `json:"options"`
	}
)

func (j *apiJob) ID() string                             { return j.j.ID() }
func (j *apiJob) Rev() string                            { return j.j.Rev() }
func (j *apiJob) DocType() string                        { return consts.Jobs }
func (j *apiJob) Clone() couchdb.Doc                     { return j }
func (j *apiJob) SetID(_ string)                         {}
func (j *apiJob) SetRev(_ string)                        {}
func (j *apiJob) Relationships() jsonapi.RelationshipMap { return nil }
func (j *apiJob) Included() []jsonapi.Object             { return nil }
func (j *apiJob) Links() *jsonapi.LinksList {
	return &jsonapi.LinksList{Self: "/jobs/" + j.j.WorkerType + "/" + j.j.ID()}
}
func (j *apiJob) MarshalJSON() ([]byte, error) {
	return json.Marshal(j.j)
}

func (q *apiQueue) ID() string                             { return q.workerType }
func (q *apiQueue) Rev() string                            { return "" }
func (q *apiQueue) DocType() string                        { return consts.Queues }
func (q *apiQueue) Clone() couchdb.Doc                     { return q }
func (q *apiQueue) SetID(_ string)                         {}
func (q *apiQueue) SetRev(_ string)                        {}
func (q *apiQueue) Relationships() jsonapi.RelationshipMap { return nil }
func (q *apiQueue) Included() []jsonapi.Object             { return nil }
func (q *apiQueue) Links() *jsonapi.LinksList {
	return &jsonapi.LinksList{Self: "/jobs/queue/" + q.workerType}
}
func (q *apiQueue) Valid(key, value string) bool {
	switch key {
	case "worker":
		return q.workerType == value
	}
	return false
}

func (t *apiTrigger) ID() string                             { return t.t.Infos().TID }
func (t *apiTrigger) Rev() string                            { return "" }
func (t *apiTrigger) DocType() string                        { return consts.Triggers }
func (t *apiTrigger) Clone() couchdb.Doc                     { return t }
func (t *apiTrigger) SetID(_ string)                         {}
func (t *apiTrigger) SetRev(_ string)                        {}
func (t *apiTrigger) Relationships() jsonapi.RelationshipMap { return nil }
func (t *apiTrigger) Included() []jsonapi.Object             { return nil }
func (t *apiTrigger) Links() *jsonapi.LinksList {
	return &jsonapi.LinksList{Self: "/jobs/triggers/" + t.ID()}
}
func (t *apiTrigger) MarshalJSON() ([]byte, error) {
	return json.Marshal(t.t.Infos())
}

func getQueue(c echo.Context) error {
	workerType := c.Param("worker-type")
	count, err := globals.GetBroker().QueueLen(workerType)
	if err != nil {
		return wrapJobsError(err)
	}
	o := &apiQueue{
		workerType: workerType,
		Count:      count,
	}

	if err := permissions.Allow(c, permissions.GET, o); err != nil {
		return err
	}

	return jsonapi.Data(c, http.StatusOK, o, nil)
}

func pushJob(c echo.Context) error {
	instance := middlewares.GetInstance(c)

	req := &apiJobRequest{}
	if _, err := jsonapi.Bind(c.Request(), &req); err != nil {
		return wrapJobsError(err)
	}

	jr := &jobs.JobRequest{
		Domain:     instance.Domain,
		WorkerType: c.Param("worker-type"),
		Options:    req.Options,
		Message: &jobs.Message{
			Type: jobs.JSONEncoding,
			Data: req.Arguments,
		},
	}
	if err := permissions.Allow(c, permissions.POST, jr); err != nil {
		return err
	}

	job, err := globals.GetBroker().PushJob(jr)
	if err != nil {
		return wrapJobsError(err)
	}

	return jsonapi.Data(c, http.StatusAccepted, &apiJob{job}, nil)
}

func newTrigger(c echo.Context) error {
	instance := middlewares.GetInstance(c)
	sched := globals.GetScheduler()
	req := &apiTriggerRequest{}
	if _, err := jsonapi.Bind(c.Request(), &req); err != nil {
		return wrapJobsError(err)
	}

	if req.Debounce != "" {
		if _, err := time.ParseDuration(req.Debounce); err != nil {
			return jsonapi.InvalidAttribute("debounce", err)
		}
	}

	t, err := scheduler.NewTrigger(&scheduler.TriggerInfos{
		Type:       req.Type,
		WorkerType: req.WorkerType,
		Domain:     instance.Domain,
		Arguments:  req.Arguments,
		Debounce:   req.Debounce,
		Options:    req.Options,
		Message: &jobs.Message{
			Type: jobs.JSONEncoding,
			Data: req.WorkerArguments,
		},
	})
	if err != nil {
		return wrapJobsError(err)
	}

	if err = permissions.Allow(c, permissions.POST, t); err != nil {
		return err
	}

	if err = sched.Add(t); err != nil {
		return wrapJobsError(err)
	}
	return jsonapi.Data(c, http.StatusCreated, &apiTrigger{t}, nil)
}

func getTrigger(c echo.Context) error {
	instance := middlewares.GetInstance(c)
	sched := globals.GetScheduler()
	t, err := sched.Get(instance.Domain, c.Param("trigger-id"))
	if err != nil {
		return wrapJobsError(err)
	}
	if err := permissions.Allow(c, permissions.GET, t); err != nil {
		return err
	}
	return jsonapi.Data(c, http.StatusOK, &apiTrigger{t}, nil)
}

func deleteTrigger(c echo.Context) error {
	instance := middlewares.GetInstance(c)
	sched := globals.GetScheduler()
	t, err := sched.Get(instance.Domain, c.Param("trigger-id"))
	if err != nil {
		return wrapJobsError(err)
	}
	if err := permissions.Allow(c, permissions.DELETE, t); err != nil {
		return err
	}
	if err := sched.Delete(instance.Domain, c.Param("trigger-id")); err != nil {
		return wrapJobsError(err)
	}
	return c.NoContent(http.StatusNoContent)
}

func getAllTriggers(c echo.Context) error {
	instance := middlewares.GetInstance(c)
	workerFilter := c.QueryParam("Worker")
	sched := globals.GetScheduler()
	if err := permissions.AllowWholeType(c, permissions.GET, consts.Triggers); err != nil {
		return err
	}
	ts, err := sched.GetAll(instance.Domain)
	if err != nil {
		return wrapJobsError(err)
	}
	objs := make([]jsonapi.Object, 0, len(ts))
	for _, t := range ts {
		if workerFilter == "" || t.Infos().WorkerType == workerFilter {
			objs = append(objs, &apiTrigger{t})
		}
	}
	return jsonapi.DataList(c, http.StatusOK, objs, nil)
}

func getJob(c echo.Context) error {
	instance := middlewares.GetInstance(c)
	job, err := globals.GetBroker().GetJobInfos(instance.Domain, c.Param("job-id"))
	if err != nil {
		return err
	}
	if err := permissions.Allow(c, permissions.GET, job); err != nil {
		return err
	}
	return jsonapi.Data(c, http.StatusOK, &apiJob{job}, nil)
}

func cleanJobs(c echo.Context) error {
	instance := middlewares.GetInstance(c)
	if err := permissions.AllowWholeType(c, permissions.GET, consts.Jobs); err != nil {
		return err
	}
	var ups []*jobs.JobInfos
	now := time.Now()
	err := couchdb.ForeachDocs(instance, consts.Jobs, func(data []byte) error {
		var job *jobs.JobInfos
		if err := json.Unmarshal(data, &job); err != nil {
			return err
		}
		if job.State != jobs.Running {
			return nil
		}
		if job.StartedAt.Add(1 * time.Hour).Before(now) {
			ups = append(ups, job)
		}
		return nil
	})
	if err != nil && !couchdb.IsNoDatabaseError(err) {
		return err
	}
	var errf error
	for _, j := range ups {
		j.State = jobs.Done
		err := couchdb.UpdateDoc(instance, j)
		if err != nil {
			errf = multierror.Append(errf, err)
		}
	}
	if errf != nil {
		return errf
	}
	return c.JSON(200, map[string]int{"deleted": len(ups)})
}

// Routes sets the routing for the jobs service
func Routes(router *echo.Group) {
	router.GET("/queue/:worker-type", getQueue)
	router.POST("/queue/:worker-type", pushJob)

	router.GET("/triggers", getAllTriggers)
	router.POST("/triggers", newTrigger)
	router.GET("/triggers/:trigger-id", getTrigger)
	router.DELETE("/triggers/:trigger-id", deleteTrigger)

	router.POST("/clean", cleanJobs)
	router.GET("/:job-id", getJob)
}

func wrapJobsError(err error) error {
	switch err {
	case scheduler.ErrNotFoundTrigger,
		jobs.ErrNotFoundJob,
		jobs.ErrUnknownWorker:
		return jsonapi.NotFound(err)
	case scheduler.ErrUnknownTrigger:
		return jsonapi.InvalidAttribute("Type", err)
	}
	return err
}
<commit_msg>Remove outdated comment<commit_after>package jobs

import (
	"encoding/json"
	"net/http"
	"time"

	"github.com/cozy/cozy-stack/pkg/consts"
	"github.com/cozy/cozy-stack/pkg/couchdb"
	"github.com/cozy/cozy-stack/pkg/globals"
	"github.com/cozy/cozy-stack/pkg/jobs"
	"github.com/cozy/cozy-stack/pkg/scheduler"
	"github.com/cozy/cozy-stack/web/jsonapi"
	"github.com/cozy/cozy-stack/web/middlewares"
	"github.com/cozy/cozy-stack/web/permissions"
	multierror "github.com/hashicorp/go-multierror"
	"github.com/labstack/echo"

	// import workers
	_ "github.com/cozy/cozy-stack/pkg/workers/exec"
	_ "github.com/cozy/cozy-stack/pkg/workers/log"
	_ "github.com/cozy/cozy-stack/pkg/workers/mails"
	_ "github.com/cozy/cozy-stack/pkg/workers/sharings"
	_ "github.com/cozy/cozy-stack/pkg/workers/thumbnail"
	_ "github.com/cozy/cozy-stack/pkg/workers/unzip"
)

type (
	apiJob struct {
		j *jobs.JobInfos
	}
	apiJobRequest struct {
		Arguments json.RawMessage  `json:"arguments"`
		Options   *jobs.JobOptions `json:"options"`
	}
	apiQueue struct {
		Count      int `json:"count"`
		workerType string
	}
	apiTrigger struct {
		t scheduler.Trigger
	}
	apiTriggerRequest struct {
		Type            string           `json:"type"`
		Arguments       string           `json:"arguments"`
		WorkerType      string           `json:"worker"`
		WorkerArguments json.RawMessage  `json:"worker_arguments"`
		Debounce        string           `json:"debounce"`
		Options         *jobs.JobOptions `json:"options"`
	}
)

func (j *apiJob) ID() string                             { return j.j.ID() }
func (j *apiJob) Rev() string                            { return j.j.Rev() }
func (j *apiJob) DocType() string                        { return consts.Jobs }
func (j *apiJob) Clone() couchdb.Doc                     { return j }
func (j *apiJob) SetID(_ string)                         {}
func (j *apiJob) SetRev(_ string)                        {}
func (j *apiJob) Relationships() jsonapi.RelationshipMap { return nil }
func (j *apiJob) Included() []jsonapi.Object             { return nil }
func (j *apiJob) Links() *jsonapi.LinksList {
	return &jsonapi.LinksList{Self: "/jobs/" + j.j.WorkerType + "/" + j.j.ID()}
}
func (j *apiJob) MarshalJSON() ([]byte, error) {
	return json.Marshal(j.j)
}

func (q *apiQueue) ID() string                             { return q.workerType }
func (q *apiQueue) Rev() string                            { return "" }
func (q *apiQueue) DocType() string                        { return consts.Queues }
func (q *apiQueue) Clone() couchdb.Doc                     { return q }
func (q *apiQueue) SetID(_ string)                         {}
func (q *apiQueue) SetRev(_ string)                        {}
func (q *apiQueue) Relationships() jsonapi.RelationshipMap { return nil }
func (q *apiQueue) Included() []jsonapi.Object             { return nil }
func (q *apiQueue) Links() *jsonapi.LinksList {
	return &jsonapi.LinksList{Self: "/jobs/queue/" + q.workerType}
}
func (q *apiQueue) Valid(key, value string) bool {
	switch key {
	case "worker":
		return q.workerType == value
	}
	return false
}

func (t *apiTrigger) ID() string                             { return t.t.Infos().TID }
func (t *apiTrigger) Rev() string                            { return "" }
func (t *apiTrigger) DocType() string                        { return consts.Triggers }
func (t *apiTrigger) Clone() couchdb.Doc                     { return t }
func (t *apiTrigger) SetID(_ string)                         {}
func (t *apiTrigger) SetRev(_ string)                        {}
func (t *apiTrigger) Relationships() jsonapi.RelationshipMap { return nil }
func (t *apiTrigger) Included() []jsonapi.Object             { return nil }
func (t *apiTrigger) Links() *jsonapi.LinksList {
	return &jsonapi.LinksList{Self: "/jobs/triggers/" + t.ID()}
}
func (t *apiTrigger) MarshalJSON() ([]byte, error) {
	return json.Marshal(t.t.Infos())
}

func getQueue(c echo.Context) error {
	workerType := c.Param("worker-type")
	count, err := globals.GetBroker().QueueLen(workerType)
	if err != nil {
		return wrapJobsError(err)
	}
	o := &apiQueue{
		workerType: workerType,
		Count:      count,
	}

	if err := permissions.Allow(c, permissions.GET, o); err != nil {
		return err
	}

	return jsonapi.Data(c, http.StatusOK, o, nil)
}

func pushJob(c echo.Context) error {
	instance := middlewares.GetInstance(c)

	req := &apiJobRequest{}
	if _, err := jsonapi.Bind(c.Request(), &req); err != nil {
		return wrapJobsError(err)
	}

	jr := &jobs.JobRequest{
		Domain:     instance.Domain,
		WorkerType: c.Param("worker-type"),
		Options:    req.Options,
		Message: &jobs.Message{
			Type: jobs.JSONEncoding,
			Data: req.Arguments,
		},
	}
	if err := permissions.Allow(c, permissions.POST, jr); err != nil {
		return err
	}

	job, err := globals.GetBroker().PushJob(jr)
	if err != nil {
		return wrapJobsError(err)
	}

	return jsonapi.Data(c, http.StatusAccepted, &apiJob{job}, nil)
}

func newTrigger(c echo.Context) error {
	instance := middlewares.GetInstance(c)
	sched := globals.GetScheduler()
	req := &apiTriggerRequest{}
	if _, err := jsonapi.Bind(c.Request(), &req); err != nil {
		return wrapJobsError(err)
	}

	if req.Debounce != "" {
		if _, err := time.ParseDuration(req.Debounce); err != nil {
			return jsonapi.InvalidAttribute("debounce", err)
		}
	}

	t, err := scheduler.NewTrigger(&scheduler.TriggerInfos{
		Type:       req.Type,
		WorkerType: req.WorkerType,
		Domain:     instance.Domain,
		Arguments:  req.Arguments,
		Debounce:   req.Debounce,
		Options:    req.Options,
		Message: &jobs.Message{
			Type: jobs.JSONEncoding,
			Data: req.WorkerArguments,
		},
	})
	if err != nil {
		return wrapJobsError(err)
	}

	if err = permissions.Allow(c, permissions.POST, t); err != nil {
		return err
	}

	if err = sched.Add(t); err != nil {
		return wrapJobsError(err)
	}
	return jsonapi.Data(c, http.StatusCreated, &apiTrigger{t}, nil)
}

func getTrigger(c echo.Context) error {
	instance := middlewares.GetInstance(c)
	sched := globals.GetScheduler()
	t, err := sched.Get(instance.Domain, c.Param("trigger-id"))
	if err != nil {
		return wrapJobsError(err)
	}
	if err := permissions.Allow(c, permissions.GET, t); err != nil {
		return err
	}
	return jsonapi.Data(c, http.StatusOK, &apiTrigger{t}, nil)
}

func deleteTrigger(c echo.Context) error {
	instance := middlewares.GetInstance(c)
	sched := globals.GetScheduler()
	t, err := sched.Get(instance.Domain, c.Param("trigger-id"))
	if err != nil {
		return wrapJobsError(err)
	}
	if err := permissions.Allow(c, permissions.DELETE, t); err != nil {
		return err
	}
	if err := sched.Delete(instance.Domain, c.Param("trigger-id")); err != nil {
		return wrapJobsError(err)
	}
	return c.NoContent(http.StatusNoContent)
}

func getAllTriggers(c echo.Context) error {
	instance := middlewares.GetInstance(c)
	workerFilter := c.QueryParam("Worker")
	sched := globals.GetScheduler()
	if err := permissions.AllowWholeType(c, permissions.GET, consts.Triggers); err != nil {
		return err
	}
	ts, err := sched.GetAll(instance.Domain)
	if err != nil {
		return wrapJobsError(err)
	}
	objs := make([]jsonapi.Object, 0, len(ts))
	for _, t := range ts {
		if workerFilter == "" || t.Infos().WorkerType == workerFilter {
			objs = append(objs, &apiTrigger{t})
		}
	}
	return jsonapi.DataList(c, http.StatusOK, objs, nil)
}

func getJob(c echo.Context) error {
	instance := middlewares.GetInstance(c)
	job, err := globals.GetBroker().GetJobInfos(instance.Domain, c.Param("job-id"))
	if err != nil {
		return err
	}
	if err := permissions.Allow(c, permissions.GET, job); err != nil {
		return err
	}
	return jsonapi.Data(c, http.StatusOK, &apiJob{job}, nil)
}

func cleanJobs(c echo.Context) error {
	instance := middlewares.GetInstance(c)
	if err := permissions.AllowWholeType(c, permissions.GET, consts.Jobs); err != nil {
		return err
	}
	var ups []*jobs.JobInfos
	now := time.Now()
	err := couchdb.ForeachDocs(instance, consts.Jobs, func(data []byte) error {
		var job *jobs.JobInfos
		if err := json.Unmarshal(data, &job); err != nil {
			return err
		}
		if job.State != jobs.Running {
			return nil
		}
		if job.StartedAt.Add(1 * time.Hour).Before(now) {
			ups = append(ups, job)
		}
		return nil
	})
	if err != nil && !couchdb.IsNoDatabaseError(err) {
		return err
	}
	var errf error
	for _, j := range ups {
		j.State = jobs.Done
		err := couchdb.UpdateDoc(instance, j)
		if err != nil {
			errf = multierror.Append(errf, err)
		}
	}
	if errf != nil {
		return errf
	}
	return c.JSON(200, map[string]int{"deleted": len(ups)})
}

// Routes sets the routing for the jobs service
func Routes(router *echo.Group) {
	router.GET("/queue/:worker-type", getQueue)
	router.POST("/queue/:worker-type", pushJob)

	router.GET("/triggers", getAllTriggers)
	router.POST("/triggers", newTrigger)
	router.GET("/triggers/:trigger-id", getTrigger)
	router.DELETE("/triggers/:trigger-id", deleteTrigger)

	router.POST("/clean", cleanJobs)
	router.GET("/:job-id", getJob)
}

func wrapJobsError(err error) error {
	switch err {
	case scheduler.ErrNotFoundTrigger,
		jobs.ErrNotFoundJob,
		jobs.ErrUnknownWorker:
		return jsonapi.NotFound(err)
	case scheduler.ErrUnknownTrigger:
		return jsonapi.InvalidAttribute("Type", err)
	}
	return err
}
<|endoftext|>


<commit_before>package main

import (
	"context"
	"crypto/tls"
	"flag"
	"net"
	"net/http"
	"os"
	"os/signal"
	"strings"
	"syscall"
	"time"

	"github.com/coreos/go-semver/semver"
	"github.com/fnproject/fn/fnlb/lb"
	"github.com/sirupsen/logrus"
)

const VERSION = "0.0.256"

func main() {
	// XXX (reed): normalize
	level, err := logrus.ParseLevel(os.Getenv("LOG_LEVEL"))
	if err != nil {
		level = logrus.InfoLevel
	}
	logrus.SetLevel(level)

	fnodes := flag.String("nodes", "", "comma separated list of functions nodes")
	minAPIVersion := flag.String("min-api-version", "0.0.223", "minimal node API to accept")

	var conf lb.Config
	flag.StringVar(&conf.DBurl, "db", "sqlite3://:memory:", "backend to store nodes, default to in memory; use k8s for kuberneted")
	flag.StringVar(&conf.Listen, "listen", ":8081", "port to run on")
	flag.StringVar(&conf.MgmtListen, "mgmt-listen", ":8081", "management port to run on")
	flag.IntVar(&conf.ShutdownTimeout, "shutdown-timeout", 0, "graceful shutdown timeout")
	flag.IntVar(&conf.HealthcheckInterval, "hc-interval", 3, "how often to check f(x) nodes, in seconds")
	flag.StringVar(&conf.HealthcheckEndpoint, "hc-path", "/version", "endpoint to determine node health")
	flag.IntVar(&conf.HealthcheckUnhealthy, "hc-unhealthy", 2, "threshold of failed checks to declare node unhealthy")
	flag.IntVar(&conf.HealthcheckHealthy, "hc-healthy", 1, "threshold of success checks to declare node healthy")
	flag.IntVar(&conf.HealthcheckTimeout, "hc-timeout", 5, "timeout of healthcheck endpoint, in seconds")
	flag.StringVar(&conf.ZipkinURL, "zipkin", "", "zipkin endpoint to send traces")
	flag.StringVar(&conf.Namespace, "namespace", "", "kubernetes namespace to monitor")
	flag.StringVar(&conf.LabelSelector, "label-selector", "", "kubernetes label selector to monitor")
	flag.IntVar(&conf.TargetPort, "target-port", 8080, "kubernetes port to target on selected pods")

	flag.Parse()

	conf.MinAPIVersion = semver.New(*minAPIVersion)

	if len(*fnodes) > 0 {
		// starting w/o nodes is fine too
		conf.Nodes = strings.Split(*fnodes, ",")
	}

	conf.Transport = &http.Transport{
		Proxy: http.ProxyFromEnvironment,
		Dial: (&net.Dialer{
			Timeout:   10 * time.Second,
			KeepAlive: 120 * time.Second,
		}).Dial,
		MaxIdleConnsPerHost: 512,
		TLSHandshakeTimeout: 10 * time.Second,
		TLSClientConfig: &tls.Config{
			ClientSessionCache: tls.NewLRUClientSessionCache(4096),
		},
	}

	db, err := lb.NewDB(conf) // Handles case where DBurl == "k8s"
	if err != nil {
		logrus.WithError(err).Fatal("error setting up database")
	}
	defer db.Close()

	g, err := lb.NewAllGrouper(conf, db)
	if err != nil {
		logrus.WithError(err).Fatal("error setting up grouper")
	}

	r := lb.NewConsistentRouter(conf)
	k := func(r *http.Request) (string, error) {
		return r.URL.Path, nil
	}

	servers := make([]*http.Server, 0, 1)
	handler := lb.NewProxy(k, g, r, conf)

	// a separate mgmt listener is requested? then let's create a LB traffic only server
	if conf.Listen != conf.MgmtListen {
		servers = append(servers, &http.Server{Addr: conf.Listen, Handler: handler})
		handler = lb.NullHandler()
	}

	// add mgmt endpoints to the handler
	handler = g.Wrap(handler) // add/del/list endpoints
	handler = r.Wrap(handler) // stats / dash endpoint

	servers = append(servers, &http.Server{Addr: conf.MgmtListen, Handler: handler})
	serve(servers, &conf)
}

func serve(servers []*http.Server, conf *lb.Config) {

	ch := make(chan os.Signal, 1)
	signal.Notify(ch, syscall.SIGQUIT, syscall.SIGINT)

	for i := 0; i < len(servers); i++ {
		go func(idx int) {
			err := servers[idx].ListenAndServe()
			if err != nil && err != http.ErrServerClosed {
				logrus.WithFields(logrus.Fields{"server_id": idx}).WithError(err).Fatal("server error")
			} else {
				logrus.WithFields(logrus.Fields{"server_id": idx}).Info("server stopped")
			}
		}(i)
	}

	sig := <-ch
	logrus.WithFields(logrus.Fields{"signal": sig}).Info("received signal")

	for i := 0; i < len(servers); i++ {

		ctx := context.Background()

		if conf.ShutdownTimeout > 0 {
			tmpCtx, cancel := context.WithTimeout(context.Background(), time.Duration(conf.ShutdownTimeout)*time.Second)
			ctx = tmpCtx
			defer cancel()
		}

		err := servers[i].Shutdown(ctx) // safe shutdown
		if err != nil {
			logrus.WithFields(logrus.Fields{"server_id": i}).WithError(err).Fatal("server shutdown error")
		} else {
			logrus.WithFields(logrus.Fields{"server_id": i}).Info("server shutdown")
		}
	}
}
<commit_msg>fnlb: 0.0.257 release [skip ci]<commit_after>package main

import (
	"context"
	"crypto/tls"
	"flag"
	"net"
	"net/http"
	"os"
	"os/signal"
	"strings"
	"syscall"
	"time"

	"github.com/coreos/go-semver/semver"
	"github.com/fnproject/fn/fnlb/lb"
	"github.com/sirupsen/logrus"
)

const VERSION = "0.0.257"

func main() {
	// XXX (reed): normalize
	level, err := logrus.ParseLevel(os.Getenv("LOG_LEVEL"))
	if err != nil {
		level = logrus.InfoLevel
	}
	logrus.SetLevel(level)

	fnodes := flag.String("nodes", "", "comma separated list of functions nodes")
	minAPIVersion := flag.String("min-api-version", "0.0.224", "minimal node API to accept")

	var conf lb.Config
	flag.StringVar(&conf.DBurl, "db", "sqlite3://:memory:", "backend to store nodes, default to in memory; use k8s for kuberneted")
	flag.StringVar(&conf.Listen, "listen", ":8081", "port to run on")
	flag.StringVar(&conf.MgmtListen, "mgmt-listen", ":8081", "management port to run on")
	flag.IntVar(&conf.ShutdownTimeout, "shutdown-timeout", 0, "graceful shutdown timeout")
	flag.IntVar(&conf.HealthcheckInterval, "hc-interval", 3, "how often to check f(x) nodes, in seconds")
	flag.StringVar(&conf.HealthcheckEndpoint, "hc-path", "/version", "endpoint to determine node health")
	flag.IntVar(&conf.HealthcheckUnhealthy, "hc-unhealthy", 2, "threshold of failed checks to declare node unhealthy")
	flag.IntVar(&conf.HealthcheckHealthy, "hc-healthy", 1, "threshold of success checks to declare node healthy")
	flag.IntVar(&conf.HealthcheckTimeout, "hc-timeout", 5, "timeout of healthcheck endpoint, in seconds")
	flag.StringVar(&conf.ZipkinURL, "zipkin", "", "zipkin endpoint to send traces")
	flag.StringVar(&conf.Namespace, "namespace", "", "kubernetes namespace to monitor")
	flag.StringVar(&conf.LabelSelector, "label-selector", "", "kubernetes label selector to monitor")
	flag.IntVar(&conf.TargetPort, "target-port", 8080, "kubernetes port to target on selected pods")

	flag.Parse()

	conf.MinAPIVersion = semver.New(*minAPIVersion)

	if len(*fnodes) > 0 {
		// starting w/o nodes is fine too
		conf.Nodes = strings.Split(*fnodes, ",")
	}

	conf.Transport = &http.Transport{
		Proxy: http.ProxyFromEnvironment,
		Dial: (&net.Dialer{
			Timeout:   10 * time.Second,
			KeepAlive: 120 * time.Second,
		}).Dial,
		MaxIdleConnsPerHost: 512,
		TLSHandshakeTimeout: 10 * time.Second,
		TLSClientConfig: &tls.Config{
			ClientSessionCache: tls.NewLRUClientSessionCache(4096),
		},
	}

	db, err := lb.NewDB(conf) // Handles case where DBurl == "k8s"
	if err != nil {
		logrus.WithError(err).Fatal("error setting up database")
	}
	defer db.Close()

	g, err := lb.NewAllGrouper(conf, db)
	if err != nil {
		logrus.WithError(err).Fatal("error setting up grouper")
	}

	r := lb.NewConsistentRouter(conf)
	k := func(r *http.Request) (string, error) {
		return r.URL.Path, nil
	}

	servers := make([]*http.Server, 0, 1)
	handler := lb.NewProxy(k, g, r, conf)

	// a separate mgmt listener is requested? then let's create a LB traffic only server
	if conf.Listen != conf.MgmtListen {
		servers = append(servers, &http.Server{Addr: conf.Listen, Handler: handler})
		handler = lb.NullHandler()
	}

	// add mgmt endpoints to the handler
	handler = g.Wrap(handler) // add/del/list endpoints
	handler = r.Wrap(handler) // stats / dash endpoint

	servers = append(servers, &http.Server{Addr: conf.MgmtListen, Handler: handler})
	serve(servers, &conf)
}

func serve(servers []*http.Server, conf *lb.Config) {

	ch := make(chan os.Signal, 1)
	signal.Notify(ch, syscall.SIGQUIT, syscall.SIGINT)

	for i := 0; i < len(servers); i++ {
		go func(idx int) {
			err := servers[idx].ListenAndServe()
			if err != nil && err != http.ErrServerClosed {
				logrus.WithFields(logrus.Fields{"server_id": idx}).WithError(err).Fatal("server error")
			} else {
				logrus.WithFields(logrus.Fields{"server_id": idx}).Info("server stopped")
			}
		}(i)
	}

	sig := <-ch
	logrus.WithFields(logrus.Fields{"signal": sig}).Info("received signal")

	for i := 0; i < len(servers); i++ {

		ctx := context.Background()

		if conf.ShutdownTimeout > 0 {
			tmpCtx, cancel := context.WithTimeout(context.Background(), time.Duration(conf.ShutdownTimeout)*time.Second)
			ctx = tmpCtx
			defer cancel()
		}

		err := servers[i].Shutdown(ctx) // safe shutdown
		if err != nil {
			logrus.WithFields(logrus.Fields{"server_id": i}).WithError(err).Fatal("server shutdown error")
		} else {
			logrus.WithFields(logrus.Fields{"server_id": i}).Info("server shutdown")
		}
	}
}
<|endoftext|>


<commit_before>package main

import (
	"context"
	"crypto/tls"
	"flag"
	"net"
	"net/http"
	"os"
	"os/signal"
	"strings"
	"syscall"
	"time"

	"github.com/coreos/go-semver/semver"
	"github.com/fnproject/fn/fnlb/lb"
	"github.com/sirupsen/logrus"
)

const VERSION = "0.0.257"

func main() {
	// XXX (reed): normalize
	level, err := logrus.ParseLevel(os.Getenv("LOG_LEVEL"))
	if err != nil {
		level = logrus.InfoLevel
	}
	logrus.SetLevel(level)

	fnodes := flag.String("nodes", "", "comma separated list of functions nodes")
	minAPIVersion := flag.String("min-api-version", "0.0.224", "minimal node API to accept")

	var conf lb.Config
	flag.StringVar(&conf.DBurl, "db", "sqlite3://:memory:", "backend to store nodes, default to in memory; use k8s for kuberneted")
	flag.StringVar(&conf.Listen, "listen", ":8081", "port to run on")
	flag.StringVar(&conf.MgmtListen, "mgmt-listen", ":8081", "management port to run on")
	flag.IntVar(&conf.ShutdownTimeout, "shutdown-timeout", 0, "graceful shutdown timeout")
	flag.IntVar(&conf.HealthcheckInterval, "hc-interval", 3, "how often to check f(x) nodes, in seconds")
	flag.StringVar(&conf.HealthcheckEndpoint, "hc-path", "/version", "endpoint to determine node health")
	flag.IntVar(&conf.HealthcheckUnhealthy, "hc-unhealthy", 2, "threshold of failed checks to declare node unhealthy")
	flag.IntVar(&conf.HealthcheckHealthy, "hc-healthy", 1, "threshold of success checks to declare node healthy")
	flag.IntVar(&conf.HealthcheckTimeout, "hc-timeout", 5, "timeout of healthcheck endpoint, in seconds")
	flag.StringVar(&conf.ZipkinURL, "zipkin", "", "zipkin endpoint to send traces")
	flag.StringVar(&conf.Namespace, "namespace", "", "kubernetes namespace to monitor")
	flag.StringVar(&conf.LabelSelector, "label-selector", "", "kubernetes label selector to monitor")
	flag.IntVar(&conf.TargetPort, "target-port", 8080, "kubernetes port to target on selected pods")

	flag.Parse()

	conf.MinAPIVersion = semver.New(*minAPIVersion)

	if len(*fnodes) > 0 {
		// starting w/o nodes is fine too
		conf.Nodes = strings.Split(*fnodes, ",")
	}

	conf.Transport = &http.Transport{
		Proxy: http.ProxyFromEnvironment,
		Dial: (&net.Dialer{
			Timeout:   10 * time.Second,
			KeepAlive: 120 * time.Second,
		}).Dial,
		MaxIdleConnsPerHost: 512,
		TLSHandshakeTimeout: 10 * time.Second,
		TLSClientConfig: &tls.Config{
			ClientSessionCache: tls.NewLRUClientSessionCache(4096),
		},
	}

	db, err := lb.NewDB(conf) // Handles case where DBurl == "k8s"
	if err != nil {
		logrus.WithError(err).Fatal("error setting up database")
	}
	defer db.Close()

	g, err := lb.NewAllGrouper(conf, db)
	if err != nil {
		logrus.WithError(err).Fatal("error setting up grouper")
	}

	r := lb.NewConsistentRouter(conf)
	k := func(r *http.Request) (string, error) {
		return r.URL.Path, nil
	}

	servers := make([]*http.Server, 0, 1)
	handler := lb.NewProxy(k, g, r, conf)

	// a separate mgmt listener is requested? then let's create a LB traffic only server
	if conf.Listen != conf.MgmtListen {
		servers = append(servers, &http.Server{Addr: conf.Listen, Handler: handler})
		handler = lb.NullHandler()
	}

	// add mgmt endpoints to the handler
	handler = g.Wrap(handler) // add/del/list endpoints
	handler = r.Wrap(handler) // stats / dash endpoint

	servers = append(servers, &http.Server{Addr: conf.MgmtListen, Handler: handler})
	serve(servers, &conf)
}

func serve(servers []*http.Server, conf *lb.Config) {

	ch := make(chan os.Signal, 1)
	signal.Notify(ch, syscall.SIGQUIT, syscall.SIGINT)

	for i := 0; i < len(servers); i++ {
		go func(idx int) {
			err := servers[idx].ListenAndServe()
			if err != nil && err != http.ErrServerClosed {
				logrus.WithFields(logrus.Fields{"server_id": idx}).WithError(err).Fatal("server error")
			} else {
				logrus.WithFields(logrus.Fields{"server_id": idx}).Info("server stopped")
			}
		}(i)
	}

	sig := <-ch
	logrus.WithFields(logrus.Fields{"signal": sig}).Info("received signal")

	for i := 0; i < len(servers); i++ {

		ctx := context.Background()

		if conf.ShutdownTimeout > 0 {
			tmpCtx, cancel := context.WithTimeout(context.Background(), time.Duration(conf.ShutdownTimeout)*time.Second)
			ctx = tmpCtx
			defer cancel()
		}

		err := servers[i].Shutdown(ctx) // safe shutdown
		if err != nil {
			logrus.WithFields(logrus.Fields{"server_id": i}).WithError(err).Fatal("server shutdown error")
		} else {
			logrus.WithFields(logrus.Fields{"server_id": i}).Info("server shutdown")
		}
	}
}
<commit_msg>fnlb: 0.0.258 release [skip ci]<commit_after>package main

import (
	"context"
	"crypto/tls"
	"flag"
	"net"
	"net/http"
	"os"
	"os/signal"
	"strings"
	"syscall"
	"time"

	"github.com/coreos/go-semver/semver"
	"github.com/fnproject/fn/fnlb/lb"
	"github.com/sirupsen/logrus"
)

const VERSION = "0.0.258"

func main() {
	// XXX (reed): normalize
	level, err := logrus.ParseLevel(os.Getenv("LOG_LEVEL"))
	if err != nil {
		level = logrus.InfoLevel
	}
	logrus.SetLevel(level)

	fnodes := flag.String("nodes", "", "comma separated list of functions nodes")
	minAPIVersion := flag.String("min-api-version", "0.0.225", "minimal node API to accept")

	var conf lb.Config
	flag.StringVar(&conf.DBurl, "db", "sqlite3://:memory:", "backend to store nodes, default to in memory; use k8s for kuberneted")
	flag.StringVar(&conf.Listen, "listen", ":8081", "port to run on")
	flag.StringVar(&conf.MgmtListen, "mgmt-listen", ":8081", "management port to run on")
	flag.IntVar(&conf.ShutdownTimeout, "shutdown-timeout", 0, "graceful shutdown timeout")
	flag.IntVar(&conf.HealthcheckInterval, "hc-interval", 3, "how often to check f(x) nodes, in seconds")
	flag.StringVar(&conf.HealthcheckEndpoint, "hc-path", "/version", "endpoint to determine node health")
	flag.IntVar(&conf.HealthcheckUnhealthy, "hc-unhealthy", 2, "threshold of failed checks to declare node unhealthy")
	flag.IntVar(&conf.HealthcheckHealthy, "hc-healthy", 1, "threshold of success checks to declare node healthy")
	flag.IntVar(&conf.HealthcheckTimeout, "hc-timeout", 5, "timeout of healthcheck endpoint, in seconds")
	flag.StringVar(&conf.ZipkinURL, "zipkin", "", "zipkin endpoint to send traces")
	flag.StringVar(&conf.Namespace, "namespace", "", "kubernetes namespace to monitor")
	flag.StringVar(&conf.LabelSelector, "label-selector", "", "kubernetes label selector to monitor")
	flag.IntVar(&conf.TargetPort, "target-port", 8080, "kubernetes port to target on selected pods")

	flag.Parse()

	conf.MinAPIVersion = semver.New(*minAPIVersion)

	if len(*fnodes) > 0 {
		// starting w/o nodes is fine too
		conf.Nodes = strings.Split(*fnodes, ",")
	}

	conf.Transport = &http.Transport{
		Proxy: http.ProxyFromEnvironment,
		Dial: (&net.Dialer{
			Timeout:   10 * time.Second,
			KeepAlive: 120 * time.Second,
		}).Dial,
		MaxIdleConnsPerHost: 512,
		TLSHandshakeTimeout: 10 * time.Second,
		TLSClientConfig: &tls.Config{
			ClientSessionCache: tls.NewLRUClientSessionCache(4096),
		},
	}

	db, err := lb.NewDB(conf) // Handles case where DBurl == "k8s"
	if err != nil {
		logrus.WithError(err).Fatal("error setting up database")
	}
	defer db.Close()

	g, err := lb.NewAllGrouper(conf, db)
	if err != nil {
		logrus.WithError(err).Fatal("error setting up grouper")
	}

	r := lb.NewConsistentRouter(conf)
	k := func(r *http.Request) (string, error) {
		return r.URL.Path, nil
	}

	servers := make([]*http.Server, 0, 1)
	handler := lb.NewProxy(k, g, r, conf)

	// a separate mgmt listener is requested? then let's create a LB traffic only server
	if conf.Listen != conf.MgmtListen {
		servers = append(servers, &http.Server{Addr: conf.Listen, Handler: handler})
		handler = lb.NullHandler()
	}

	// add mgmt endpoints to the handler
	handler = g.Wrap(handler) // add/del/list endpoints
	handler = r.Wrap(handler) // stats / dash endpoint

	servers = append(servers, &http.Server{Addr: conf.MgmtListen, Handler: handler})
	serve(servers, &conf)
}

func serve(servers []*http.Server, conf *lb.Config) {

	ch := make(chan os.Signal, 1)
	signal.Notify(ch, syscall.SIGQUIT, syscall.SIGINT)

	for i := 0; i < len(servers); i++ {
		go func(idx int) {
			err := servers[idx].ListenAndServe()
			if err != nil && err != http.ErrServerClosed {
				logrus.WithFields(logrus.Fields{"server_id": idx}).WithError(err).Fatal("server error")
			} else {
				logrus.WithFields(logrus.Fields{"server_id": idx}).Info("server stopped")
			}
		}(i)
	}

	sig := <-ch
	logrus.WithFields(logrus.Fields{"signal": sig}).Info("received signal")

	for i := 0; i < len(servers); i++ {

		ctx := context.Background()

		if conf.ShutdownTimeout > 0 {
			tmpCtx, cancel := context.WithTimeout(context.Background(), time.Duration(conf.ShutdownTimeout)*time.Second)
			ctx = tmpCtx
			defer cancel()
		}

		err := servers[i].Shutdown(ctx) // safe shutdown
		if err != nil {
			logrus.WithFields(logrus.Fields{"server_id": i}).WithError(err).Fatal("server shutdown error")
		} else {
			logrus.WithFields(logrus.Fields{"server_id": i}).Info("server shutdown")
		}
	}
}
<|endoftext|>


<commit_before>package main

import (
	"context"
	"crypto/tls"
	"flag"
	"net"
	"net/http"
	"os"
	"os/signal"
	"strings"
	"syscall"
	"time"

	"github.com/coreos/go-semver/semver"
	"github.com/fnproject/fn/fnlb/lb"
	"github.com/sirupsen/logrus"
)

const VERSION = "0.0.58"

func main() {
	// XXX (reed): normalize
	fnodes := flag.String("nodes", "", "comma separated list of functions nodes")
	minAPIVersion := flag.String("min-api-version", "0.0.25", "minimal node API to accept")

	var conf lb.Config
	flag.StringVar(&conf.DBurl, "db", "sqlite3://:memory:", "backend to store nodes, default to in memory")
	flag.StringVar(&conf.Listen, "listen", ":8081", "port to run on")
	flag.IntVar(&conf.HealthcheckInterval, "hc-interval", 3, "how often to check f(x) nodes, in seconds")
	flag.StringVar(&conf.HealthcheckEndpoint, "hc-path", "/version", "endpoint to determine node health")
	flag.IntVar(&conf.HealthcheckUnhealthy, "hc-unhealthy", 2, "threshold of failed checks to declare node unhealthy")
	flag.IntVar(&conf.HealthcheckTimeout, "hc-timeout", 5, "timeout of healthcheck endpoint, in seconds")
	flag.StringVar(&conf.ZipkinURL, "zipkin", "", "zipkin endpoint to send traces")
	flag.Parse()

	conf.MinAPIVersion = semver.New(*minAPIVersion)

	if len(*fnodes) > 0 {
		// starting w/o nodes is fine too
		conf.Nodes = strings.Split(*fnodes, ",")
	}

	conf.Transport = &http.Transport{
		Proxy: http.ProxyFromEnvironment,
		Dial: (&net.Dialer{
			Timeout:   10 * time.Second,
			KeepAlive: 120 * time.Second,
		}).Dial,
		MaxIdleConnsPerHost: 512,
		TLSHandshakeTimeout: 10 * time.Second,
		TLSClientConfig: &tls.Config{
			ClientSessionCache: tls.NewLRUClientSessionCache(4096),
		},
	}

	g, err := lb.NewAllGrouper(conf)
	if err != nil {
		logrus.WithError(err).Fatal("error setting up grouper")
	}

	r := lb.NewConsistentRouter(conf)
	k := func(r *http.Request) (string, error) {
		return r.URL.Path, nil
	}

	h := lb.NewProxy(k, g, r, conf)
	h = g.Wrap(h) // add/del/list endpoints
	h = r.Wrap(h) // stats / dash endpoint

	err = serve(conf.Listen, h)
	if err != nil {
		logrus.WithError(err).Fatal("server error")
	}
}

func serve(addr string, handler http.Handler) error {
	server := &http.Server{Addr: addr, Handler: handler}

	ch := make(chan os.Signal, 1)
	signal.Notify(ch, syscall.SIGQUIT, syscall.SIGINT)
	go func() {
		for sig := range ch {
			logrus.WithFields(logrus.Fields{"signal": sig}).Info("received signal")
			server.Shutdown(context.Background()) // safe shutdown
			return
		}
	}()
	return server.ListenAndServe()
}
<commit_msg>fnlb: 0.0.59 release [skip ci]<commit_after>package main

import (
	"context"
	"crypto/tls"
	"flag"
	"net"
	"net/http"
	"os"
	"os/signal"
	"strings"
	"syscall"
	"time"

	"github.com/coreos/go-semver/semver"
	"github.com/fnproject/fn/fnlb/lb"
	"github.com/sirupsen/logrus"
)

const VERSION = "0.0.59"

func main() {
	// XXX (reed): normalize
	fnodes := flag.String("nodes", "", "comma separated list of functions nodes")
	minAPIVersion := flag.String("min-api-version", "0.0.26", "minimal node API to accept")

	var conf lb.Config
	flag.StringVar(&conf.DBurl, "db", "sqlite3://:memory:", "backend to store nodes, default to in memory")
	flag.StringVar(&conf.Listen, "listen", ":8081", "port to run on")
	flag.IntVar(&conf.HealthcheckInterval, "hc-interval", 3, "how often to check f(x) nodes, in seconds")
	flag.StringVar(&conf.HealthcheckEndpoint, "hc-path", "/version", "endpoint to determine node health")
	flag.IntVar(&conf.HealthcheckUnhealthy, "hc-unhealthy", 2, "threshold of failed checks to declare node unhealthy")
	flag.IntVar(&conf.HealthcheckTimeout, "hc-timeout", 5, "timeout of healthcheck endpoint, in seconds")
	flag.StringVar(&conf.ZipkinURL, "zipkin", "", "zipkin endpoint to send traces")
	flag.Parse()

	conf.MinAPIVersion = semver.New(*minAPIVersion)

	if len(*fnodes) > 0 {
		// starting w/o nodes is fine too
		conf.Nodes = strings.Split(*fnodes, ",")
	}

	conf.Transport = &http.Transport{
		Proxy: http.ProxyFromEnvironment,
		Dial: (&net.Dialer{
			Timeout:   10 * time.Second,
			KeepAlive: 120 * time.Second,
		}).Dial,
		MaxIdleConnsPerHost: 512,
		TLSHandshakeTimeout: 10 * time.Second,
		TLSClientConfig: &tls.Config{
			ClientSessionCache: tls.NewLRUClientSessionCache(4096),
		},
	}

	g, err := lb.NewAllGrouper(conf)
	if err != nil {
		logrus.WithError(err).Fatal("error setting up grouper")
	}

	r := lb.NewConsistentRouter(conf)
	k := func(r *http.Request) (string, error) {
		return r.URL.Path, nil
	}

	h := lb.NewProxy(k, g, r, conf)
	h = g.Wrap(h) // add/del/list endpoints
	h = r.Wrap(h) // stats / dash endpoint

	err = serve(conf.Listen, h)
	if err != nil {
		logrus.WithError(err).Fatal("server error")
	}
}

func serve(addr string, handler http.Handler) error {
	server := &http.Server{Addr: addr, Handler: handler}

	ch := make(chan os.Signal, 1)
	signal.Notify(ch, syscall.SIGQUIT, syscall.SIGINT)
	go func() {
		for sig := range ch {
			logrus.WithFields(logrus.Fields{"signal": sig}).Info("received signal")
			server.Shutdown(context.Background()) // safe shutdown
			return
		}
	}()
	return server.ListenAndServe()
}
<|endoftext|>