<commit_before>package babble

import (
	"fmt"
	"os"

	"github.com/mosaicnetworks/babble/src/crypto/keys"
	h "github.com/mosaicnetworks/babble/src/hashgraph"
	"github.com/mosaicnetworks/babble/src/net"
	"github.com/mosaicnetworks/babble/src/node"
	"github.com/mosaicnetworks/babble/src/peers"
	"github.com/mosaicnetworks/babble/src/service"
	"github.com/sirupsen/logrus"
)

type Babble struct {
	Config       *BabbleConfig
	Node         *node.Node
	Transport    net.Transport
	Store        h.Store
	Peers        *peers.PeerSet
	GenesisPeers *peers.PeerSet
	Service      *service.Service
}

func NewBabble(config *BabbleConfig) *Babble {
	engine := &Babble{
		Config: config,
	}

	return engine
}

func (b *Babble) initTransport() error {
	transport, err := net.NewTCPTransport(
		b.Config.BindAddr,
		nil,
		b.Config.MaxPool,
		b.Config.NodeConfig.TCPTimeout,
		b.Config.NodeConfig.JoinTimeout,
		b.Config.Logger,
	)

	if err != nil {
		return err
	}

	b.Transport = transport

	return nil
}

func (b *Babble) initPeers() error {
	if !b.Config.LoadPeers {
		if b.Peers == nil {
			return fmt.Errorf("Did not load peers but none was present")
		}

		return nil
	}

	// peers.json
	peerStore := peers.NewJSONPeerSet(b.Config.DataDir, true)

	participants, err := peerStore.PeerSet()
	if err != nil {
		return err
	}

	b.Peers = participants

	// Set Genesis Peer Set from peers.genesis.json

	genesisPeerStore := peers.NewJSONPeerSet(b.Config.DataDir, false)

	genesisParticipants, err := genesisPeerStore.PeerSet()
	if err != nil { // If there is any error, the current peer set is used as the genesis peer set
		b.Config.Logger.Debugf("could not read peers.genesis.json: %v", err)
		b.GenesisPeers = participants
	} else {
		b.GenesisPeers = genesisParticipants
	}

	b.Peers = participants

	return nil
}

func (b *Babble) initStore() error {
	if !b.Config.Store {
		b.Config.Logger.Debug("Creating InmemStore")
		b.Store = h.NewInmemStore(b.Config.NodeConfig.CacheSize)
	} else {
		b.Config.Logger.WithField("path", b.Config.BadgerDir()).Debug("BadgerDB")

		bootstrap := b.Config.NodeConfig.Bootstrap
		dbpath := b.Config.BadgerDir()
		i := 1

		for {
			if _, err := os.Stat(dbpath); err == nil {
				b.Config.Logger.Debugf("%s already exists", dbpath)

				if bootstrap {
					break
				}

				dbpath = fmt.Sprintf("%s(%d)", b.Config.BadgerDir(), i)
				b.Config.Logger.Debugf("No Bootstrap - using new db %s", dbpath)
				i++
			} else {
				break
			}
		}

		b.Config.Logger.WithField("path", dbpath).Debug("Creating BadgerStore")

		dbStore, err := h.NewBadgerStore(b.Config.NodeConfig.CacheSize, dbpath)
		if err != nil {
			return err
		}

		b.Store = dbStore
	}

	return nil
}

func (b *Babble) initKey() error {
	if b.Config.Key == nil {
		simpleKeyfile := keys.NewSimpleKeyfile(b.Config.Keyfile())

		privKey, err := simpleKeyfile.ReadKey()

		if err != nil {
			b.Config.Logger.Warn(fmt.Sprintf("Cannot read private key from file: %v", err))

			privKey, err = keys.GenerateECDSAKey()
			if err != nil {
				b.Config.Logger.Error("Error generating a new ECDSA key")
				return err
			}

			if err := simpleKeyfile.WriteKey(privKey); err != nil {
				b.Config.Logger.Error("Error saving private key", err)
				return err
			}

			b.Config.Logger.Debug("Generated a new private key")
		}

		b.Config.Key = privKey
	}
	return nil
}

func (b *Babble) initNode() error {

	validator := node.NewValidator(b.Config.Key, b.Config.Moniker)

	p, ok := b.Peers.ByID[validator.ID()]
	if ok {
		if p.Moniker != validator.Moniker {
			b.Config.Logger.WithFields(logrus.Fields{
				"json_moniker": p.Moniker,
				"cli_moniker":  validator.Moniker,
			}).Debugf("Using moniker from peers.json file")
			validator.Moniker = p.Moniker
		}
	}

	b.Config.Logger.WithFields(logrus.Fields{
		"genesis_peers": len(b.GenesisPeers.Peers),
		"peers":         len(b.Peers.Peers),
		"id":            validator.ID(),
		"moniker":       validator.Moniker,
	}).Debug("PARTICIPANTS")

	b.Node = node.NewNode(
		&b.Config.NodeConfig,
		validator,
		b.Peers,
		b.GenesisPeers,
		b.Store,
		b.Transport,
		b.Config.Proxy,
	)

	if err := b.Node.Init(); err != nil {
		return fmt.Errorf("failed to initialize node: %s", err)
	}

	return nil
}

func (b *Babble) initService() error {
	if b.Config.ServiceAddr != "" {
		b.Service = service.NewService(b.Config.ServiceAddr, b.Node, b.Config.Logger)
	}
	return nil
}

func (b *Babble) Init() error {

	if err := b.initPeers(); err != nil {
		return err
	}

	if err := b.initStore(); err != nil {
		return err
	}

	if err := b.initTransport(); err != nil {
		return err
	}

	if err := b.initKey(); err != nil {
		return err
	}

	if err := b.initNode(); err != nil {
		return err
	}

	if err := b.initService(); err != nil {
		return err
	}

	return nil
}

func (b *Babble) Run() {
	if b.Service != nil {
		go b.Service.Serve()
	}

	b.Node.Run(true)
}
<commit_msg>Add debug messages to babble.go init function<commit_after>package babble

import (
	"fmt"
	"os"

	"github.com/mosaicnetworks/babble/src/crypto/keys"
	h "github.com/mosaicnetworks/babble/src/hashgraph"
	"github.com/mosaicnetworks/babble/src/net"
	"github.com/mosaicnetworks/babble/src/node"
	"github.com/mosaicnetworks/babble/src/peers"
	"github.com/mosaicnetworks/babble/src/service"
	"github.com/sirupsen/logrus"
)

type Babble struct {
	Config       *BabbleConfig
	Node         *node.Node
	Transport    net.Transport
	Store        h.Store
	Peers        *peers.PeerSet
	GenesisPeers *peers.PeerSet
	Service      *service.Service
}

func NewBabble(config *BabbleConfig) *Babble {
	engine := &Babble{
		Config: config,
	}

	return engine
}

func (b *Babble) initTransport() error {
	transport, err := net.NewTCPTransport(
		b.Config.BindAddr,
		nil,
		b.Config.MaxPool,
		b.Config.NodeConfig.TCPTimeout,
		b.Config.NodeConfig.JoinTimeout,
		b.Config.Logger,
	)

	if err != nil {
		return err
	}

	b.Transport = transport

	return nil
}

func (b *Babble) initPeers() error {
	if !b.Config.LoadPeers {
		if b.Peers == nil {
			return fmt.Errorf("Did not load peers but none was present")
		}

		return nil
	}

	// peers.json
	peerStore := peers.NewJSONPeerSet(b.Config.DataDir, true)

	participants, err := peerStore.PeerSet()
	if err != nil {
		return err
	}

	b.Peers = participants

	// Set Genesis Peer Set from peers.genesis.json

	genesisPeerStore := peers.NewJSONPeerSet(b.Config.DataDir, false)

	genesisParticipants, err := genesisPeerStore.PeerSet()
	if err != nil { // If there is any error, the current peer set is used as the genesis peer set
		b.Config.Logger.Debugf("could not read peers.genesis.json: %v", err)
		b.GenesisPeers = participants
	} else {
		b.GenesisPeers = genesisParticipants
	}

	b.Peers = participants

	return nil
}

func (b *Babble) initStore() error {
	if !b.Config.Store {
		b.Config.Logger.Debug("Creating InmemStore")
		b.Store = h.NewInmemStore(b.Config.NodeConfig.CacheSize)
	} else {
		b.Config.Logger.WithField("path", b.Config.BadgerDir()).Debug("BadgerDB")

		bootstrap := b.Config.NodeConfig.Bootstrap
		dbpath := b.Config.BadgerDir()
		i := 1

		for {
			if _, err := os.Stat(dbpath); err == nil {
				b.Config.Logger.Debugf("%s already exists", dbpath)

				if bootstrap {
					break
				}

				dbpath = fmt.Sprintf("%s(%d)", b.Config.BadgerDir(), i)
				b.Config.Logger.Debugf("No Bootstrap - using new db %s", dbpath)
				i++
			} else {
				break
			}
		}

		b.Config.Logger.WithField("path", dbpath).Debug("Creating BadgerStore")

		dbStore, err := h.NewBadgerStore(b.Config.NodeConfig.CacheSize, dbpath)
		if err != nil {
			return err
		}

		b.Store = dbStore
	}

	return nil
}

func (b *Babble) initKey() error {
	if b.Config.Key == nil {
		simpleKeyfile := keys.NewSimpleKeyfile(b.Config.Keyfile())

		privKey, err := simpleKeyfile.ReadKey()

		if err != nil {
			b.Config.Logger.Warn(fmt.Sprintf("Cannot read private key from file: %v", err))

			privKey, err = keys.GenerateECDSAKey()
			if err != nil {
				b.Config.Logger.Error("Error generating a new ECDSA key")
				return err
			}

			if err := simpleKeyfile.WriteKey(privKey); err != nil {
				b.Config.Logger.Error("Error saving private key", err)
				return err
			}

			b.Config.Logger.Debug("Generated a new private key")
		}

		b.Config.Key = privKey
	}
	return nil
}

func (b *Babble) initNode() error {

	validator := node.NewValidator(b.Config.Key, b.Config.Moniker)

	p, ok := b.Peers.ByID[validator.ID()]
	if ok {
		if p.Moniker != validator.Moniker {
			b.Config.Logger.WithFields(logrus.Fields{
				"json_moniker": p.Moniker,
				"cli_moniker":  validator.Moniker,
			}).Debugf("Using moniker from peers.json file")
			validator.Moniker = p.Moniker
		}
	}

	b.Config.Logger.WithFields(logrus.Fields{
		"genesis_peers": len(b.GenesisPeers.Peers),
		"peers":         len(b.Peers.Peers),
		"id":            validator.ID(),
		"moniker":       validator.Moniker,
	}).Debug("PARTICIPANTS")

	b.Node = node.NewNode(
		&b.Config.NodeConfig,
		validator,
		b.Peers,
		b.GenesisPeers,
		b.Store,
		b.Transport,
		b.Config.Proxy,
	)

	if err := b.Node.Init(); err != nil {
		return fmt.Errorf("failed to initialize node: %s", err)
	}

	return nil
}

func (b *Babble) initService() error {
	if b.Config.ServiceAddr != "" {
		b.Service = service.NewService(b.Config.ServiceAddr, b.Node, b.Config.Logger)
	}
	return nil
}

func (b *Babble) Init() error {

	if err := b.initPeers(); err != nil {
		b.Config.Logger.Debug("babble.go:Init() initPeers")
		return err
	}

	if err := b.initStore(); err != nil {
		b.Config.Logger.Debug("babble.go:Init() initStore")
		return err
	}

	if err := b.initTransport(); err != nil {
		b.Config.Logger.Debug("babble.go:Init() initTransport")
		return err
	}

	if err := b.initKey(); err != nil {
		b.Config.Logger.Debug("babble.go:Init() initKey")
		return err
	}

	if err := b.initNode(); err != nil {
		b.Config.Logger.Debug("babble.go:Init() initNode")
		return err
	}

	if err := b.initService(); err != nil {
		b.Config.Logger.Debug("babble.go:Init() initService")
		return err
	}

	return nil
}

func (b *Babble) Run() {
	if b.Service != nil {
		go b.Service.Serve()
	}

	b.Node.Run(true)
}
<|endoftext|>


<commit_before>package webhooks

import (
	"encoding/json"
	"errors"
	"fmt"
	"io"
	"io/ioutil"
	"net/url"
	"os"
	"path"
	"strings"

	"github.com/ParsePlatform/parse-cli/parsecli"
	"github.com/facebookgo/stackerr"
)

var (
	errInvalidFormat = errors.New(
		`
invalid format.
valid formats should look like:
{"hooks": [OPERATION]}

OPERATION ->
{"op": "put", "function": {"functionName": "name", "url": "https_url"}}
{"op": "post", "function": {"functionName": "name", "url": "https_url"}}
{"op": "delete", "function": {"functionName": "name"}}

{"op": "put", "trigger": {"className": "cname", "triggerName": "tname", "url":"https_url"}}
{"op": "post", "trigger": {"className": "cname", "triggerName": "tname", "url":"https_url"}}
{"op": "delete", "trigger": {"className": "cname", "triggerName": "tname"}}
`)

	errPostToPut = errors.New(
		`a hook with given name already exists: cannot create a new one.
`)

	errPutToPost = errors.New(
		`a hook with the given name does not exist yet: cannot update the url.
`)

	errNotExist = errors.New(
		`a hook with the given name does not exist. cannot delete it.
	`)
)

type hookOperation struct {
	Method   string        `json:"op,omitempty"`
	Function *functionHook `json:"function,omitempty"`
	Trigger  *triggerHook  `json:"trigger,omitempty"`
}

func validateURL(urlStr string) error {
	netURL, err := url.Parse(urlStr)
	if err != nil {
		return stackerr.Wrap(err)
	}

	if netURL.Scheme != "https" {
		return errors.New("Please enter a valid https url")
	}
	return nil
}

func checkTriggerName(s string) error {
	switch strings.ToLower(s) {
	case "beforesave", "beforedelete", "aftersave", "afterdelete":
		return nil
	}
	return stackerr.Newf(
		`invalid trigger name: %v.
	This is the list of valid trigger names:
		beforeSave
		afterSave
		beforeDelete
		afterDelete
`,
		s,
	)
}

type Hooks struct {
	HooksStrict    bool
	BaseURL        string
	baseWebhookURL *url.URL
}

func (h *Hooks) checkTriggerName(s string) error {
	switch strings.ToLower(s) {
	case "beforesave", "beforedelete", "aftersave", "afterdelete":
		return nil
	}
	return stackerr.Newf(
		`invalid trigger name: %v.
	This is the list of valid trigger names:
		beforeSave
		afterSave
		beforeDelete
		afterDelete
`,
		s,
	)
}

func (h *Hooks) appendHookOperation(
	e *parsecli.Env,
	hookOp *hookOperation,
	hooksOps []*hookOperation,
) (bool, []*hookOperation, error) {
	if hookOp == nil || (hookOp.Function == nil && hookOp.Trigger == nil) ||
		(hookOp.Function != nil && hookOp.Trigger != nil) {
		return false, hooksOps, nil
	}

	method := strings.ToUpper(hookOp.Method)
	if method != "POST" && method != "PUT" && method != "DELETE" {
		return false, nil, stackerr.Wrap(errInvalidFormat)
	}

	hookOp.Method = method
	if hookOp.Trigger != nil {
		if err := h.checkTriggerName(hookOp.Trigger.TriggerName); err != nil {
			return false, nil, err
		}
	}

	hooksOps = append(hooksOps, hookOp)
	return true, hooksOps, nil
}

func (h *Hooks) createHooksOperations(
	e *parsecli.Env,
	reader io.Reader,
) ([]*hookOperation, error) {
	var input struct {
		HooksOps []*hookOperation `json:"hooks,omitempty"`
	}
	err := json.NewDecoder(ioutil.NopCloser(reader)).Decode(&input)
	if err != nil {
		return nil, stackerr.Wrap(err)
	}

	var (
		hooksOps []*hookOperation
		added    bool
	)
	for _, hookOp := range input.HooksOps {
		added, hooksOps, err = h.appendHookOperation(e, hookOp, hooksOps)
		if err != nil {
			return nil, err
		}
		if !added {
			op, err := json.MarshalIndent(hookOp, "", " ")
			if err == nil {
				fmt.Fprintf(e.Out, "Ignoring hook operation: \n%s\n", op)
			}
		}
	}

	return hooksOps, nil
}

func (h *Hooks) checkStrictMode(restOp string, exists bool) (string, bool, error) {
	restOp = strings.ToUpper(restOp)
	if !exists {
		if restOp == "PUT" {
			if h.HooksStrict {
				return "", false, stackerr.Wrap(errPutToPost)
			}
			return "POST", true, nil
		}
		if restOp == "DELETE" {
			if h.HooksStrict {
				return "", false, stackerr.Wrap(errNotExist)
			}
			return "DELETE", true, nil
		}
	} else if restOp == "POST" {
		if h.HooksStrict {
			return "", false, stackerr.Wrap(errPostToPut)
		}
		return "PUT", true, nil
	}
	return restOp, false, nil
}

func (h *Hooks) functionHookExists(e *parsecli.Env, name string) (bool, error) {
	functionsURL, err := url.Parse(path.Join(defaultFunctionsURL, name))
	if err != nil {
		return false, stackerr.Wrap(err)
	}
	var results struct {
		Results []*functionHook `json:"results,omitempty"`
	}
	_, err = e.ParseAPIClient.Get(functionsURL, &results)
	if err != nil {
		if strings.Contains(err.Error(), "is defined") {
			return false, nil
		}
		return false, stackerr.Wrap(err)
	}
	for _, result := range results.Results {
		if result.URL != "" && result.FunctionName == name {
			return true, nil
		}
	}
	return false, nil
}

func (h *Hooks) deployFunctionHook(e *parsecli.Env, op *hookOperation) error {
	if op.Function == nil {
		return stackerr.New("cannot deploy nil function hook")
	}
	exists, err := h.functionHookExists(e, op.Function.FunctionName)
	if err != nil {
		return err
	}

	restOp, suppressed, err := h.checkStrictMode(op.Method, exists)
	if err != nil {
		return err
	}

	function := &functionHooksCmd{Function: op.Function}
	switch restOp {
	case "POST":
		return function.functionHooksCreate(e, nil)
	case "PUT":
		return function.functionHooksUpdate(e, nil)
	case "DELETE":
		if suppressed {
			return nil
		}
		return function.functionHooksDelete(e, nil)
	}
	return stackerr.Wrap(errInvalidFormat)
}

func (h *Hooks) triggerHookExists(e *parsecli.Env, className, triggerName string) (bool, error) {
	triggersURL, err := url.Parse(path.Join(defaultTriggersURL, className, triggerName))
	if err != nil {
		return false, stackerr.Wrap(err)
	}
	var results struct {
		Results []*triggerHook `json:"results,omitempty"`
	}
	_, err = e.ParseAPIClient.Get(triggersURL, &results)
	if err != nil {
		if strings.Contains(err.Error(), "is defined") {
			return false, nil
		}
		return false, stackerr.Wrap(err)
	}
	for _, result := range results.Results {
		if result.URL != "" && result.ClassName == className && result.TriggerName == triggerName {
			return true, nil
		}
	}
	return false, nil
}

func (h *Hooks) deployTriggerHook(e *parsecli.Env, op *hookOperation) error {
	if op.Trigger == nil {
		return stackerr.New("cannot deploy nil trigger hook")
	}

	exists, err := h.triggerHookExists(e, op.Trigger.ClassName, op.Trigger.TriggerName)
	if err != nil {
		return err
	}
	restOp, suppressed, err := h.checkStrictMode(op.Method, exists)
	if err != nil {
		return err
	}

	trigger := &triggerHooksCmd{Trigger: op.Trigger, All: false}
	switch restOp {
	case "POST":
		return trigger.triggerHooksCreate(e, nil)
	case "PUT":
		return trigger.triggerHooksUpdate(e, nil)
	case "DELETE":
		if suppressed {
			return nil
		}
		return trigger.triggerHooksDelete(e, nil)
	}
	return stackerr.Wrap(errInvalidFormat)

}

func (h *Hooks) deployWebhooksConfig(e *parsecli.Env, hooksOps []*hookOperation) error {
	for _, op := range hooksOps {
		if op.Function == nil && op.Trigger == nil {
			return stackerr.New("hook operation is neither a function, not a trigger.")
		}
		if op.Function != nil && op.Trigger != nil {
			return stackerr.New("a hook cannot be both a function and a trigger.")
		}
		if op.Function != nil {
			if err := h.deployFunctionHook(e, op); err != nil {
				return err
			}
		} else {
			if err := h.deployTriggerHook(e, op); err != nil {
				return err
			}
		}
		fmt.Fprintln(e.Out)
	}
	return nil
}

func (h *Hooks) parseBaseURL(e *parsecli.Env) error {
	if h.BaseURL != "" {
		u, err := url.Parse(h.BaseURL)
		if err != nil {
			fmt.Fprintln(e.Err, "Invalid base webhook url provided")
			return stackerr.Wrap(err)
		}
		if u.Scheme != "https" {
			return stackerr.New("Please provide a valid https url")
		}
		h.baseWebhookURL = u
	}
	return nil
}

func (h *Hooks) HooksCmd(e *parsecli.Env, ctx *parsecli.Context, args []string) error {
	if len(args) > 1 {
		return fmt.Errorf("Invalid args: %v, only an optional hooks config file is expected.", args)
	}
	if err := h.parseBaseURL(e); err != nil {
		return err
	}

	reader := e.In
	if len(args) == 1 {
		file, err := os.Open(args[0])
		if err != nil {
			return stackerr.Wrap(err)
		}
		reader = file
	} else {
		fmt.Fprintln(e.Out, "Since a webhooks config file was not provided reading from stdin.")
	}
	hooksOps, err := h.createHooksOperations(e, reader)
	if err != nil {
		return err
	}
	err = h.deployWebhooksConfig(e, hooksOps)
	if err != nil {
		fmt.Fprintln(
			e.Out,
			"Failed to deploy the webhooks config. Please try again...",
		)
		return err
	}
	fmt.Fprintln(e.Out, "Successfully configured the given webhooks for the app.")
	return nil
}
<commit_msg>[hooks] fix error check for trigger hook exists<commit_after>package webhooks

import (
	"encoding/json"
	"errors"
	"fmt"
	"io"
	"io/ioutil"
	"net/url"
	"os"
	"path"
	"strings"

	"github.com/ParsePlatform/parse-cli/parsecli"
	"github.com/facebookgo/stackerr"
)

var (
	errInvalidFormat = errors.New(
		`
invalid format.
valid formats should look like:
{"hooks": [OPERATION]}

OPERATION ->
{"op": "put", "function": {"functionName": "name", "url": "https_url"}}
{"op": "post", "function": {"functionName": "name", "url": "https_url"}}
{"op": "delete", "function": {"functionName": "name"}}

{"op": "put", "trigger": {"className": "cname", "triggerName": "tname", "url":"https_url"}}
{"op": "post", "trigger": {"className": "cname", "triggerName": "tname", "url":"https_url"}}
{"op": "delete", "trigger": {"className": "cname", "triggerName": "tname"}}
`)

	errPostToPut = errors.New(
		`a hook with given name already exists: cannot create a new one.
`)

	errPutToPost = errors.New(
		`a hook with the given name does not exist yet: cannot update the url.
`)

	errNotExist = errors.New(
		`a hook with the given name does not exist. cannot delete it.
	`)
)

type hookOperation struct {
	Method   string        `json:"op,omitempty"`
	Function *functionHook `json:"function,omitempty"`
	Trigger  *triggerHook  `json:"trigger,omitempty"`
}

func validateURL(urlStr string) error {
	netURL, err := url.Parse(urlStr)
	if err != nil {
		return stackerr.Wrap(err)
	}

	if netURL.Scheme != "https" {
		return errors.New("Please enter a valid https url")
	}
	return nil
}

func checkTriggerName(s string) error {
	switch strings.ToLower(s) {
	case "beforesave", "beforedelete", "aftersave", "afterdelete":
		return nil
	}
	return stackerr.Newf(
		`invalid trigger name: %v.
	This is the list of valid trigger names:
		beforeSave
		afterSave
		beforeDelete
		afterDelete
`,
		s,
	)
}

type Hooks struct {
	HooksStrict    bool
	BaseURL        string
	baseWebhookURL *url.URL
}

func (h *Hooks) checkTriggerName(s string) error {
	switch strings.ToLower(s) {
	case "beforesave", "beforedelete", "aftersave", "afterdelete":
		return nil
	}
	return stackerr.Newf(
		`invalid trigger name: %v.
	This is the list of valid trigger names:
		beforeSave
		afterSave
		beforeDelete
		afterDelete
`,
		s,
	)
}

func (h *Hooks) appendHookOperation(
	e *parsecli.Env,
	hookOp *hookOperation,
	hooksOps []*hookOperation,
) (bool, []*hookOperation, error) {
	if hookOp == nil || (hookOp.Function == nil && hookOp.Trigger == nil) ||
		(hookOp.Function != nil && hookOp.Trigger != nil) {
		return false, hooksOps, nil
	}

	method := strings.ToUpper(hookOp.Method)
	if method != "POST" && method != "PUT" && method != "DELETE" {
		return false, nil, stackerr.Wrap(errInvalidFormat)
	}

	hookOp.Method = method
	if hookOp.Trigger != nil {
		if err := h.checkTriggerName(hookOp.Trigger.TriggerName); err != nil {
			return false, nil, err
		}
	}

	hooksOps = append(hooksOps, hookOp)
	return true, hooksOps, nil
}

func (h *Hooks) createHooksOperations(
	e *parsecli.Env,
	reader io.Reader,
) ([]*hookOperation, error) {
	var input struct {
		HooksOps []*hookOperation `json:"hooks,omitempty"`
	}
	err := json.NewDecoder(ioutil.NopCloser(reader)).Decode(&input)
	if err != nil {
		return nil, stackerr.Wrap(err)
	}

	var (
		hooksOps []*hookOperation
		added    bool
	)
	for _, hookOp := range input.HooksOps {
		added, hooksOps, err = h.appendHookOperation(e, hookOp, hooksOps)
		if err != nil {
			return nil, err
		}
		if !added {
			op, err := json.MarshalIndent(hookOp, "", " ")
			if err == nil {
				fmt.Fprintf(e.Out, "Ignoring hook operation: \n%s\n", op)
			}
		}
	}

	return hooksOps, nil
}

func (h *Hooks) checkStrictMode(restOp string, exists bool) (string, bool, error) {
	restOp = strings.ToUpper(restOp)
	if !exists {
		if restOp == "PUT" {
			if h.HooksStrict {
				return "", false, stackerr.Wrap(errPutToPost)
			}
			return "POST", true, nil
		}
		if restOp == "DELETE" {
			if h.HooksStrict {
				return "", false, stackerr.Wrap(errNotExist)
			}
			return "DELETE", true, nil
		}
	} else if restOp == "POST" {
		if h.HooksStrict {
			return "", false, stackerr.Wrap(errPostToPut)
		}
		return "PUT", true, nil
	}
	return restOp, false, nil
}

func (h *Hooks) functionHookExists(e *parsecli.Env, name string) (bool, error) {
	functionsURL, err := url.Parse(path.Join(defaultFunctionsURL, name))
	if err != nil {
		return false, stackerr.Wrap(err)
	}
	var results struct {
		Results []*functionHook `json:"results,omitempty"`
	}
	_, err = e.ParseAPIClient.Get(functionsURL, &results)
	if err != nil {
		if strings.Contains(err.Error(), "is defined") {
			return false, nil
		}
		return false, stackerr.Wrap(err)
	}
	for _, result := range results.Results {
		if result.URL != "" && result.FunctionName == name {
			return true, nil
		}
	}
	return false, nil
}

func (h *Hooks) deployFunctionHook(e *parsecli.Env, op *hookOperation) error {
	if op.Function == nil {
		return stackerr.New("cannot deploy nil function hook")
	}
	exists, err := h.functionHookExists(e, op.Function.FunctionName)
	if err != nil {
		return err
	}

	restOp, suppressed, err := h.checkStrictMode(op.Method, exists)
	if err != nil {
		return err
	}

	function := &functionHooksCmd{Function: op.Function}
	switch restOp {
	case "POST":
		return function.functionHooksCreate(e, nil)
	case "PUT":
		return function.functionHooksUpdate(e, nil)
	case "DELETE":
		if suppressed {
			return nil
		}
		return function.functionHooksDelete(e, nil)
	}
	return stackerr.Wrap(errInvalidFormat)
}

func (h *Hooks) triggerHookExists(e *parsecli.Env, className, triggerName string) (bool, error) {
	triggersURL, err := url.Parse(path.Join(defaultTriggersURL, className, triggerName))
	if err != nil {
		return false, stackerr.Wrap(err)
	}
	var results struct {
		Results []*triggerHook `json:"results,omitempty"`
	}
	_, err = e.ParseAPIClient.Get(triggersURL, &results)
	if err != nil {
		if strings.Contains(err.Error(), "defined for") {
			return false, nil
		}
		return false, stackerr.Wrap(err)
	}
	for _, result := range results.Results {
		if result.URL != "" && result.ClassName == className && result.TriggerName == triggerName {
			return true, nil
		}
	}
	return false, nil
}

func (h *Hooks) deployTriggerHook(e *parsecli.Env, op *hookOperation) error {
	if op.Trigger == nil {
		return stackerr.New("cannot deploy nil trigger hook")
	}

	exists, err := h.triggerHookExists(e, op.Trigger.ClassName, op.Trigger.TriggerName)
	if err != nil {
		return err
	}
	restOp, suppressed, err := h.checkStrictMode(op.Method, exists)
	if err != nil {
		return err
	}

	trigger := &triggerHooksCmd{Trigger: op.Trigger, All: false}
	switch restOp {
	case "POST":
		return trigger.triggerHooksCreate(e, nil)
	case "PUT":
		return trigger.triggerHooksUpdate(e, nil)
	case "DELETE":
		if suppressed {
			return nil
		}
		return trigger.triggerHooksDelete(e, nil)
	}
	return stackerr.Wrap(errInvalidFormat)

}

func (h *Hooks) deployWebhooksConfig(e *parsecli.Env, hooksOps []*hookOperation) error {
	for _, op := range hooksOps {
		if op.Function == nil && op.Trigger == nil {
			return stackerr.New("hook operation is neither a function, not a trigger.")
		}
		if op.Function != nil && op.Trigger != nil {
			return stackerr.New("a hook cannot be both a function and a trigger.")
		}
		if op.Function != nil {
			if err := h.deployFunctionHook(e, op); err != nil {
				return err
			}
		} else {
			if err := h.deployTriggerHook(e, op); err != nil {
				return err
			}
		}
		fmt.Fprintln(e.Out)
	}
	return nil
}

func (h *Hooks) parseBaseURL(e *parsecli.Env) error {
	if h.BaseURL != "" {
		u, err := url.Parse(h.BaseURL)
		if err != nil {
			fmt.Fprintln(e.Err, "Invalid base webhook url provided")
			return stackerr.Wrap(err)
		}
		if u.Scheme != "https" {
			return stackerr.New("Please provide a valid https url")
		}
		h.baseWebhookURL = u
	}
	return nil
}

func (h *Hooks) HooksCmd(e *parsecli.Env, ctx *parsecli.Context, args []string) error {
	if len(args) > 1 {
		return fmt.Errorf("Invalid args: %v, only an optional hooks config file is expected.", args)
	}
	if err := h.parseBaseURL(e); err != nil {
		return err
	}

	reader := e.In
	if len(args) == 1 {
		file, err := os.Open(args[0])
		if err != nil {
			return stackerr.Wrap(err)
		}
		reader = file
	} else {
		fmt.Fprintln(e.Out, "Since a webhooks config file was not provided reading from stdin.")
	}
	hooksOps, err := h.createHooksOperations(e, reader)
	if err != nil {
		return err
	}
	err = h.deployWebhooksConfig(e, hooksOps)
	if err != nil {
		fmt.Fprintln(
			e.Out,
			"Failed to deploy the webhooks config. Please try again...",
		)
		return err
	}
	fmt.Fprintln(e.Out, "Successfully configured the given webhooks for the app.")
	return nil
}
<|endoftext|>


<commit_before>package unionfs

import (
	"fmt"
	"github.com/hanwen/go-fuse/fuse"
	"os"
	"strings"
)

var _ = fmt.Println

const _XATTRSEP = "@XATTR@"

type attrResponse struct {
	*os.FileInfo
	fuse.Status
}

type xattrResponse struct {
	data []byte
	fuse.Status
}

type dirResponse struct {
	entries []fuse.DirEntry
	fuse.Status
}

type linkResponse struct {
	linkContent string
	fuse.Status
}

// Caches filesystem metadata.
type CachingFileSystem struct {
	fuse.FileSystem

	attributes *TimedCache
	dirs       *TimedCache
	links      *TimedCache
	xattr      *TimedCache
	files      *TimedCache
}

func readDir(fs fuse.FileSystem, name string) *dirResponse {
	origStream, code := fs.OpenDir(name)

	r := &dirResponse{nil, code}
	if code != fuse.OK {
		return r
	}

	for {
		d, ok := <-origStream
		if !ok {
			break
		}
		r.entries = append(r.entries, d)
	}
	return r
}

func getAttr(fs fuse.FileSystem, name string) *attrResponse {
	a, code := fs.GetAttr(name)
	return &attrResponse{
		FileInfo: a,
		Status:   code,
	}
}

func getXAttr(fs fuse.FileSystem, nameAttr string) *xattrResponse {
	ns := strings.SplitN(nameAttr, _XATTRSEP, 2)
	a, code := fs.GetXAttr(ns[0], ns[1])
	return &xattrResponse{
		data:   a,
		Status: code,
	}
}

func readLink(fs fuse.FileSystem, name string) *linkResponse {
	a, code := fs.Readlink(name)
	return &linkResponse{
		linkContent: a,
		Status:      code,
	}
}

func NewCachingFileSystem(fs fuse.FileSystem, ttlNs int64) *CachingFileSystem {
	c := new(CachingFileSystem)
	c.FileSystem = fs
	c.attributes = NewTimedCache(func(n string) interface{} { return getAttr(fs, n) }, ttlNs)
	c.dirs = NewTimedCache(func(n string) interface{} { return readDir(fs, n) }, ttlNs)
	c.links = NewTimedCache(func(n string) interface{} { return readLink(fs, n) }, ttlNs)
	c.xattr = NewTimedCache(func(n string) interface{} {
		return getXAttr(fs, n)
	}, ttlNs)
	return c
}

func (me *CachingFileSystem) DropCache() {
	for _, c := range []*TimedCache{me.attributes, me.dirs, me.links, me.xattr} {
		c.DropAll()
	}
}

func (me *CachingFileSystem) GetAttr(name string) (*os.FileInfo, fuse.Status) {
	if name == _DROP_CACHE {
		return &os.FileInfo{
			Mode: fuse.S_IFREG | 0777,
		}, fuse.OK
	}

	r := me.attributes.Get(name).(*attrResponse)
	return r.FileInfo, r.Status
}

func (me *CachingFileSystem) GetXAttr(name string, attr string) ([]byte, fuse.Status) {
	key := name + _XATTRSEP + attr
	r := me.xattr.Get(key).(*xattrResponse)
	return r.data, r.Status
}

func (me *CachingFileSystem) Readlink(name string) (string, fuse.Status) {
	r := me.links.Get(name).(*linkResponse)
	return r.linkContent, r.Status
}

func (me *CachingFileSystem) OpenDir(name string) (stream chan fuse.DirEntry, status fuse.Status) {
	r := me.dirs.Get(name).(*dirResponse)
	if r.Status.Ok() {
		stream = make(chan fuse.DirEntry, len(r.entries))
		for _, d := range r.entries {
			stream <- d
		}
		close(stream)
		return stream, r.Status
	}

	return nil, r.Status
}

func (me *CachingFileSystem) Name() string {
	return fmt.Sprintf("CachingFileSystem(%s)", me.FileSystem.Name())
}

func (me *CachingFileSystem) Open(name string, flags uint32) (f fuse.File, status fuse.Status) {
	if flags&fuse.O_ANYWRITE != 0 && name == _DROP_CACHE {
		me.DropCache()
	}
	return me.FileSystem.Open(name, flags)
}
<commit_msg>Print message when dropping cache in CachingFileSystem.<commit_after>package unionfs

import (
	"fmt"
	"github.com/hanwen/go-fuse/fuse"
	"log"
	"os"
	"strings"
)

var _ = fmt.Println

const _XATTRSEP = "@XATTR@"

type attrResponse struct {
	*os.FileInfo
	fuse.Status
}

type xattrResponse struct {
	data []byte
	fuse.Status
}

type dirResponse struct {
	entries []fuse.DirEntry
	fuse.Status
}

type linkResponse struct {
	linkContent string
	fuse.Status
}

// Caches filesystem metadata.
type CachingFileSystem struct {
	fuse.FileSystem

	attributes *TimedCache
	dirs       *TimedCache
	links      *TimedCache
	xattr      *TimedCache
	files      *TimedCache
}

func readDir(fs fuse.FileSystem, name string) *dirResponse {
	origStream, code := fs.OpenDir(name)

	r := &dirResponse{nil, code}
	if code != fuse.OK {
		return r
	}

	for {
		d, ok := <-origStream
		if !ok {
			break
		}
		r.entries = append(r.entries, d)
	}
	return r
}

func getAttr(fs fuse.FileSystem, name string) *attrResponse {
	a, code := fs.GetAttr(name)
	return &attrResponse{
		FileInfo: a,
		Status:   code,
	}
}

func getXAttr(fs fuse.FileSystem, nameAttr string) *xattrResponse {
	ns := strings.SplitN(nameAttr, _XATTRSEP, 2)
	a, code := fs.GetXAttr(ns[0], ns[1])
	return &xattrResponse{
		data:   a,
		Status: code,
	}
}

func readLink(fs fuse.FileSystem, name string) *linkResponse {
	a, code := fs.Readlink(name)
	return &linkResponse{
		linkContent: a,
		Status:      code,
	}
}

func NewCachingFileSystem(fs fuse.FileSystem, ttlNs int64) *CachingFileSystem {
	c := new(CachingFileSystem)
	c.FileSystem = fs
	c.attributes = NewTimedCache(func(n string) interface{} { return getAttr(fs, n) }, ttlNs)
	c.dirs = NewTimedCache(func(n string) interface{} { return readDir(fs, n) }, ttlNs)
	c.links = NewTimedCache(func(n string) interface{} { return readLink(fs, n) }, ttlNs)
	c.xattr = NewTimedCache(func(n string) interface{} {
		return getXAttr(fs, n)
	}, ttlNs)
	return c
}

func (me *CachingFileSystem) DropCache() {
	for _, c := range []*TimedCache{me.attributes, me.dirs, me.links, me.xattr} {
		c.DropAll()
	}
}

func (me *CachingFileSystem) GetAttr(name string) (*os.FileInfo, fuse.Status) {
	if name == _DROP_CACHE {
		return &os.FileInfo{
			Mode: fuse.S_IFREG | 0777,
		}, fuse.OK
	}

	r := me.attributes.Get(name).(*attrResponse)
	return r.FileInfo, r.Status
}

func (me *CachingFileSystem) GetXAttr(name string, attr string) ([]byte, fuse.Status) {
	key := name + _XATTRSEP + attr
	r := me.xattr.Get(key).(*xattrResponse)
	return r.data, r.Status
}

func (me *CachingFileSystem) Readlink(name string) (string, fuse.Status) {
	r := me.links.Get(name).(*linkResponse)
	return r.linkContent, r.Status
}

func (me *CachingFileSystem) OpenDir(name string) (stream chan fuse.DirEntry, status fuse.Status) {
	r := me.dirs.Get(name).(*dirResponse)
	if r.Status.Ok() {
		stream = make(chan fuse.DirEntry, len(r.entries))
		for _, d := range r.entries {
			stream <- d
		}
		close(stream)
		return stream, r.Status
	}

	return nil, r.Status
}

func (me *CachingFileSystem) Name() string {
	return fmt.Sprintf("CachingFileSystem(%s)", me.FileSystem.Name())
}

func (me *CachingFileSystem) Open(name string, flags uint32) (f fuse.File, status fuse.Status) {
	if flags&fuse.O_ANYWRITE != 0 && name == _DROP_CACHE {
		log.Println("Dropping cache for", me.Name())
		me.DropCache()
	}
	return me.FileSystem.Open(name, flags)
}
<|endoftext|>


<commit_before>// Copyright 2016-2020 The grok_exporter Authors
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package exporter

import (
	configuration "github.com/fstab/grok_exporter/config/v3"
	"github.com/fstab/grok_exporter/oniguruma"
	"github.com/prometheus/client_golang/prometheus"
	"github.com/prometheus/client_model/go"
	"reflect"
	"testing"
)

func TestCounterVec(t *testing.T) {
	regex := initCounterRegex(t)
	counterCfg := newMetricConfig(t, &configuration.MetricConfig{
		Name: "exim_rejected_rcpt_total",
		Labels: map[string]string{
			"error_message": "{{.message}}",
		},
	})
	counter := NewCounterMetric(counterCfg, regex, nil)
	counter.ProcessMatch("some unrelated line", nil)
	counter.ProcessMatch("2016-04-26 10:19:57 H=(85.214.241.101) [36.224.138.227] F=<z2007tw@yahoo.com.tw> rejected RCPT <alan.a168@msa.hinet.net>: relay not permitted", nil)
	counter.ProcessMatch("2016-04-26 12:31:39 H=(186-90-8-31.genericrev.cantv.net) [186.90.8.31] F=<Hans.Krause9@cantv.net> rejected RCPT <ug2seeng-admin@example.com>: Unrouteable address", nil)
	counter.ProcessMatch("2016-04-26 10:19:57 H=(85.214.241.101) [36.224.138.227] F=<z2007tw@yahoo.com.tw> rejected RCPT <alan.a168@msa.hinet.net>: relay not permitted", nil)

	switch c := counter.Collector().(type) {
	case *prometheus.CounterVec:
		m := io_prometheus_client.Metric{}
		c.WithLabelValues("relay not permitted").Write(&m)
		if *m.Counter.Value != float64(2) {
			t.Errorf("Expected 2 matches, but got %v matches.", *m.Counter.Value)
		}
		c.WithLabelValues("Unrouteable address").Write(&m)
		if *m.Counter.Value != float64(1) {
			t.Errorf("Expected 1 match, but got %v matches.", *m.Counter.Value)
		}
	default:
		t.Errorf("Unexpected type of metric: %v", reflect.TypeOf(c))
	}
}

func TestCounter(t *testing.T) {
	regex := initCounterRegex(t)
	counterCfg := newMetricConfig(t, &configuration.MetricConfig{
		Name: "exim_rejected_rcpt_total",
	})
	counter := NewCounterMetric(counterCfg, regex, nil)

	counter.ProcessMatch("some unrelated line", nil)
	counter.ProcessMatch("2016-04-26 10:19:57 H=(85.214.241.101) [36.224.138.227] F=<z2007tw@yahoo.com.tw> rejected RCPT <alan.a168@msa.hinet.net>: relay not permitted", nil)
	counter.ProcessMatch("2016-04-26 12:31:39 H=(186-90-8-31.genericrev.cantv.net) [186.90.8.31] F=<Hans.Krause9@cantv.net> rejected RCPT <ug2seeng-admin@example.com>: Unrouteable address", nil)
	counter.ProcessMatch("2016-04-26 10:19:57 H=(85.214.241.101) [36.224.138.227] F=<z2007tw@yahoo.com.tw> rejected RCPT <alan.a168@msa.hinet.net>: relay not permitted", nil)

	switch c := counter.Collector().(type) {
	case prometheus.Counter:
		m := io_prometheus_client.Metric{}
		c.Write(&m)
		if *m.Counter.Value != float64(3) {
			t.Errorf("Expected 3 matches, but got %v matches.", *m.Counter.Value)
		}
	default:
		t.Errorf("Unexpected type of metric: %v", reflect.TypeOf(c))
	}
}

func TestLogfileLabel(t *testing.T) {
	regex := initCounterRegex(t)
	counterCfg := newMetricConfig(t, &configuration.MetricConfig{
		Name: "exim_rejected_rcpt_total",
		Labels: map[string]string{
			"error_message": "{{.message}}",
			"logfile":       "{{.logfile}}",
		},
	})
	logfile1 := map[string]string{
		"logfile": "/var/log/exim-1.log",
	}
	logfile2 := map[string]string{
		"logfile": "/var/log/exim-2.log",
	}
	counter := NewCounterMetric(counterCfg, regex, nil)
	counter.ProcessMatch("2016-04-26 10:19:57 H=(85.214.241.101) [36.224.138.227] F=<z2007tw@yahoo.com.tw> rejected RCPT <alan.a168@msa.hinet.net>: relay not permitted", logfile1)
	counter.ProcessMatch("2016-04-26 12:31:39 H=(186-90-8-31.genericrev.cantv.net) [186.90.8.31] F=<Hans.Krause9@cantv.net> rejected RCPT <ug2seeng-admin@example.com>: Unrouteable address", logfile1)
	counter.ProcessMatch("2016-04-26 10:19:57 H=(85.214.241.101) [36.224.138.227] F=<z2007tw@yahoo.com.tw> rejected RCPT <alan.a168@msa.hinet.net>: relay not permitted", logfile2)

	switch c := counter.Collector().(type) {
	case *prometheus.CounterVec:
		m := io_prometheus_client.Metric{}
		c.With(map[string]string{
			"error_message": "relay not permitted",
			"logfile":       "/var/log/exim-1.log",
		}).Write(&m)
		if *m.Counter.Value != float64(1) {
			t.Errorf("Expected 1 match, but got %v matches.", *m.Counter.Value)
		}
		c.With(map[string]string{
			"error_message": "Unrouteable address",
			"logfile":       "/var/log/exim-1.log",
		}).Write(&m)
		if *m.Counter.Value != float64(1) {
			t.Errorf("Expected 1 match, but got %v matches.", *m.Counter.Value)
		}
		c.With(map[string]string{
			"error_message": "relay not permitted",
			"logfile":       "/var/log/exim-2.log",
		}).Write(&m)
		if *m.Counter.Value != float64(1) {
			t.Errorf("Expected 1 match, but got %v matches.", *m.Counter.Value)
		}
	default:
		t.Errorf("Unexpected type of metric: %v", reflect.TypeOf(c))
	}
}

func initCounterRegex(t *testing.T) *oniguruma.Regex {
	patterns := loadPatternDir(t)
	err := patterns.AddPattern("EXIM_MESSAGE [a-zA-Z ]*")
	if err != nil {
		t.Error(err)
	}
	regex, err := Compile("%{EXIM_DATE} %{EXIM_REMOTE_HOST} F=<%{EMAILADDRESS}> rejected RCPT <%{EMAILADDRESS}>: %{EXIM_MESSAGE:message}", patterns)
	if err != nil {
		t.Error(err)
	}
	return regex
}

func TestGauge(t *testing.T) {
	regex := initGaugeRegex(t)
	gaugeCfg := newMetricConfig(t, &configuration.MetricConfig{
		Name:  "temperature",
		Value: "{{.temperature}}",
	})
	gauge := NewGaugeMetric(gaugeCfg, regex, nil)

	gauge.ProcessMatch("Temperature in Berlin: 32", nil)
	gauge.ProcessMatch("Temperature in Moscow: -5", nil)

	switch c := gauge.Collector().(type) {
	case prometheus.Gauge:
		m := io_prometheus_client.Metric{}
		c.Write(&m)
		if *m.Gauge.Value != float64(-5) {
			t.Errorf("Expected -5 as last observed value, but got %v.", *m.Gauge.Value)
		}
	default:
		t.Errorf("Unexpected type of metric: %v", reflect.TypeOf(c))
	}
}

func TestGaugeCumulative(t *testing.T) {
	regex := initGaugeRegex(t)
	gaugeCfg := newMetricConfig(t, &configuration.MetricConfig{
		Name:       "temperature",
		Value:      "{{.temperature}}",
		Cumulative: true,
	})
	gauge := NewGaugeMetric(gaugeCfg, regex, nil)

	gauge.ProcessMatch("Temperature in Berlin: 32", nil)
	gauge.ProcessMatch("Temperature in Moscow: -5", nil)

	switch c := gauge.Collector().(type) {
	case prometheus.Gauge:
		m := io_prometheus_client.Metric{}
		c.Write(&m)
		if *m.Gauge.Value != float64(27) {
			t.Errorf("Expected 27 as cumulative value, but got %v.", *m.Gauge.Value)
		}
	default:
		t.Errorf("Unexpected type of metric: %v", reflect.TypeOf(c))
	}
}

func TestGaugeVec(t *testing.T) {
	regex := initGaugeRegex(t)
	gaugeCfg := newMetricConfig(t, &configuration.MetricConfig{
		Name:  "temperature",
		Value: "{{.temperature}}",
		Labels: map[string]string{
			"city": "{{.city}}",
		},
	})
	gauge := NewGaugeMetric(gaugeCfg, regex, nil)

	gauge.ProcessMatch("Temperature in Berlin: 32", nil)
	gauge.ProcessMatch("Temperature in Moscow: -5", nil)
	gauge.ProcessMatch("Temperature in Berlin: 31", nil)

	switch c := gauge.Collector().(type) {
	case *prometheus.GaugeVec:
		m := io_prometheus_client.Metric{}
		c.WithLabelValues("Berlin").Write(&m)
		if *m.Gauge.Value != float64(31) {
			t.Errorf("Expected 31 as last observed value in Berlin, but got %v.", *m.Gauge.Value)
		}
		c.WithLabelValues("Moscow").Write(&m)
		if *m.Gauge.Value != float64(-5) {
			t.Errorf("Expected -5 as last observed value in Moscow, but got %v.", *m.Gauge.Value)
		}
	default:
		t.Errorf("Unexpected type of metric: %v", reflect.TypeOf(c))
	}
}

func initGaugeRegex(t *testing.T) *oniguruma.Regex {
	patterns := loadPatternDir(t)
	regex, err := Compile("Temperature in %{WORD:city}: %{INT:temperature}", patterns)
	if err != nil {
		t.Error(err)
	}
	return regex
}

func newMetricConfig(t *testing.T, cfg *configuration.MetricConfig) *configuration.MetricConfig {
	// Handle default for counter's value
	if cfg.Type == "counter" && len(cfg.Value) == 0 {
		cfg.Value = "1.0"
	}
	err := cfg.InitTemplates()
	if err != nil {
		t.Fatal(err)
	}
	return cfg
}
<commit_msg>Added cumulative counter test<commit_after>// Copyright 2016-2020 The grok_exporter Authors
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package exporter

import (
	configuration "github.com/fstab/grok_exporter/config/v3"
	"github.com/fstab/grok_exporter/oniguruma"
	"github.com/prometheus/client_golang/prometheus"
	"github.com/prometheus/client_model/go"
	"reflect"
	"testing"
)

func TestCounterVec(t *testing.T) {
	regex := initCounterRegex(t)
	counterCfg := newMetricConfig(t, &configuration.MetricConfig{
		Name: "exim_rejected_rcpt_total",
		Labels: map[string]string{
			"error_message": "{{.message}}",
		},
	})
	counter := NewCounterMetric(counterCfg, regex, nil)
	counter.ProcessMatch("some unrelated line", nil)
	counter.ProcessMatch("2016-04-26 10:19:57 H=(85.214.241.101) [36.224.138.227] F=<z2007tw@yahoo.com.tw> rejected RCPT <alan.a168@msa.hinet.net>: relay not permitted", nil)
	counter.ProcessMatch("2016-04-26 12:31:39 H=(186-90-8-31.genericrev.cantv.net) [186.90.8.31] F=<Hans.Krause9@cantv.net> rejected RCPT <ug2seeng-admin@example.com>: Unrouteable address", nil)
	counter.ProcessMatch("2016-04-26 10:19:57 H=(85.214.241.101) [36.224.138.227] F=<z2007tw@yahoo.com.tw> rejected RCPT <alan.a168@msa.hinet.net>: relay not permitted", nil)

	switch c := counter.Collector().(type) {
	case *prometheus.CounterVec:
		m := io_prometheus_client.Metric{}
		c.WithLabelValues("relay not permitted").Write(&m)
		if *m.Counter.Value != float64(2) {
			t.Errorf("Expected 2 matches, but got %v matches.", *m.Counter.Value)
		}
		c.WithLabelValues("Unrouteable address").Write(&m)
		if *m.Counter.Value != float64(1) {
			t.Errorf("Expected 1 match, but got %v matches.", *m.Counter.Value)
		}
	default:
		t.Errorf("Unexpected type of metric: %v", reflect.TypeOf(c))
	}
}

func TestCounter(t *testing.T) {
	regex := initCounterRegex(t)
	counterCfg := newMetricConfig(t, &configuration.MetricConfig{
		Name: "exim_rejected_rcpt_total",
	})
	counter := NewCounterMetric(counterCfg, regex, nil)

	counter.ProcessMatch("some unrelated line", nil)
	counter.ProcessMatch("2016-04-26 10:19:57 H=(85.214.241.101) [36.224.138.227] F=<z2007tw@yahoo.com.tw> rejected RCPT <alan.a168@msa.hinet.net>: relay not permitted", nil)
	counter.ProcessMatch("2016-04-26 12:31:39 H=(186-90-8-31.genericrev.cantv.net) [186.90.8.31] F=<Hans.Krause9@cantv.net> rejected RCPT <ug2seeng-admin@example.com>: Unrouteable address", nil)
	counter.ProcessMatch("2016-04-26 10:19:57 H=(85.214.241.101) [36.224.138.227] F=<z2007tw@yahoo.com.tw> rejected RCPT <alan.a168@msa.hinet.net>: relay not permitted", nil)

	switch c := counter.Collector().(type) {
	case prometheus.Counter:
		m := io_prometheus_client.Metric{}
		c.Write(&m)
		if *m.Counter.Value != float64(3) {
			t.Errorf("Expected 3 matches, but got %v matches.", *m.Counter.Value)
		}
	default:
		t.Errorf("Unexpected type of metric: %v", reflect.TypeOf(c))
	}
}

func TestCounterValue(t *testing.T) {
	regex := initCumulativeRegex(t)
	counterCfg := newMetricConfig(t, &configuration.MetricConfig{
		Name:       "rainfall",
		Value:      "{{.rainfall}}",
	})
	counter := NewCounterMetric(counterCfg, regex, nil)

	counter.ProcessMatch("Rainfall in Berlin: 32", nil)
	counter.ProcessMatch("Rainfall in Berlin: 5", nil)

	switch c := counter.Collector().(type) {
	case prometheus.Counter:
		m := io_prometheus_client.Metric{}
		c.Write(&m)
		if *m.Counter.Value != float64(37) {
			t.Errorf("Expected 37 as counter value, but got %v.", *m.Counter.Value)
		}
	default:
		t.Errorf("Unexpected type of metric: %v", reflect.TypeOf(c))
	}
}

func TestLogfileLabel(t *testing.T) {
	regex := initCounterRegex(t)
	counterCfg := newMetricConfig(t, &configuration.MetricConfig{
		Name: "exim_rejected_rcpt_total",
		Labels: map[string]string{
			"error_message": "{{.message}}",
			"logfile":       "{{.logfile}}",
		},
	})
	logfile1 := map[string]string{
		"logfile": "/var/log/exim-1.log",
	}
	logfile2 := map[string]string{
		"logfile": "/var/log/exim-2.log",
	}
	counter := NewCounterMetric(counterCfg, regex, nil)
	counter.ProcessMatch("2016-04-26 10:19:57 H=(85.214.241.101) [36.224.138.227] F=<z2007tw@yahoo.com.tw> rejected RCPT <alan.a168@msa.hinet.net>: relay not permitted", logfile1)
	counter.ProcessMatch("2016-04-26 12:31:39 H=(186-90-8-31.genericrev.cantv.net) [186.90.8.31] F=<Hans.Krause9@cantv.net> rejected RCPT <ug2seeng-admin@example.com>: Unrouteable address", logfile1)
	counter.ProcessMatch("2016-04-26 10:19:57 H=(85.214.241.101) [36.224.138.227] F=<z2007tw@yahoo.com.tw> rejected RCPT <alan.a168@msa.hinet.net>: relay not permitted", logfile2)

	switch c := counter.Collector().(type) {
	case *prometheus.CounterVec:
		m := io_prometheus_client.Metric{}
		c.With(map[string]string{
			"error_message": "relay not permitted",
			"logfile":       "/var/log/exim-1.log",
		}).Write(&m)
		if *m.Counter.Value != float64(1) {
			t.Errorf("Expected 1 match, but got %v matches.", *m.Counter.Value)
		}
		c.With(map[string]string{
			"error_message": "Unrouteable address",
			"logfile":       "/var/log/exim-1.log",
		}).Write(&m)
		if *m.Counter.Value != float64(1) {
			t.Errorf("Expected 1 match, but got %v matches.", *m.Counter.Value)
		}
		c.With(map[string]string{
			"error_message": "relay not permitted",
			"logfile":       "/var/log/exim-2.log",
		}).Write(&m)
		if *m.Counter.Value != float64(1) {
			t.Errorf("Expected 1 match, but got %v matches.", *m.Counter.Value)
		}
	default:
		t.Errorf("Unexpected type of metric: %v", reflect.TypeOf(c))
	}
}

func initCounterRegex(t *testing.T) *oniguruma.Regex {
	patterns := loadPatternDir(t)
	err := patterns.AddPattern("EXIM_MESSAGE [a-zA-Z ]*")
	if err != nil {
		t.Error(err)
	}
	regex, err := Compile("%{EXIM_DATE} %{EXIM_REMOTE_HOST} F=<%{EMAILADDRESS}> rejected RCPT <%{EMAILADDRESS}>: %{EXIM_MESSAGE:message}", patterns)
	if err != nil {
		t.Error(err)
	}
	return regex
}

func TestGauge(t *testing.T) {
	regex := initGaugeRegex(t)
	gaugeCfg := newMetricConfig(t, &configuration.MetricConfig{
		Name:  "temperature",
		Value: "{{.temperature}}",
	})
	gauge := NewGaugeMetric(gaugeCfg, regex, nil)

	gauge.ProcessMatch("Temperature in Berlin: 32", nil)
	gauge.ProcessMatch("Temperature in Moscow: -5", nil)

	switch c := gauge.Collector().(type) {
	case prometheus.Gauge:
		m := io_prometheus_client.Metric{}
		c.Write(&m)
		if *m.Gauge.Value != float64(-5) {
			t.Errorf("Expected -5 as last observed value, but got %v.", *m.Gauge.Value)
		}
	default:
		t.Errorf("Unexpected type of metric: %v", reflect.TypeOf(c))
	}
}

func TestGaugeCumulative(t *testing.T) {
	regex := initCumulativeRegex(t)
	gaugeCfg := newMetricConfig(t, &configuration.MetricConfig{
		Name:       "rainfall",
		Value:      "{{.rainfall}}",
		Cumulative: true,
	})
	gauge := NewGaugeMetric(gaugeCfg, regex, nil)

	gauge.ProcessMatch("Rainfall in Berlin: 32", nil)
	gauge.ProcessMatch("Rainfall in Moscow: 5", nil)

	switch c := gauge.Collector().(type) {
	case prometheus.Gauge:
		m := io_prometheus_client.Metric{}
		c.Write(&m)
		if *m.Gauge.Value != float64(37) {
			t.Errorf("Expected 37 as cumulative value, but got %v.", *m.Gauge.Value)
		}
	default:
		t.Errorf("Unexpected type of metric: %v", reflect.TypeOf(c))
	}
}

func TestGaugeVec(t *testing.T) {
	regex := initGaugeRegex(t)
	gaugeCfg := newMetricConfig(t, &configuration.MetricConfig{
		Name:  "temperature",
		Value: "{{.temperature}}",
		Labels: map[string]string{
			"city": "{{.city}}",
		},
	})
	gauge := NewGaugeMetric(gaugeCfg, regex, nil)

	gauge.ProcessMatch("Temperature in Berlin: 32", nil)
	gauge.ProcessMatch("Temperature in Moscow: -5", nil)
	gauge.ProcessMatch("Temperature in Berlin: 31", nil)

	switch c := gauge.Collector().(type) {
	case *prometheus.GaugeVec:
		m := io_prometheus_client.Metric{}
		c.WithLabelValues("Berlin").Write(&m)
		if *m.Gauge.Value != float64(31) {
			t.Errorf("Expected 31 as last observed value in Berlin, but got %v.", *m.Gauge.Value)
		}
		c.WithLabelValues("Moscow").Write(&m)
		if *m.Gauge.Value != float64(-5) {
			t.Errorf("Expected -5 as last observed value in Moscow, but got %v.", *m.Gauge.Value)
		}
	default:
		t.Errorf("Unexpected type of metric: %v", reflect.TypeOf(c))
	}
}

func initGaugeRegex(t *testing.T) *oniguruma.Regex {
	patterns := loadPatternDir(t)
	regex, err := Compile("Temperature in %{WORD:city}: %{INT:temperature}", patterns)
	if err != nil {
		t.Error(err)
	}
	return regex
}

func initCumulativeRegex(t *testing.T) *oniguruma.Regex {
	patterns := loadPatternDir(t)
	regex, err := Compile("Rainfall in %{WORD:city}: %{INT:rainfall}", patterns)
	if err != nil {
		t.Error(err)
	}
	return regex
}

func newMetricConfig(t *testing.T, cfg *configuration.MetricConfig) *configuration.MetricConfig {
	// Handle default for counter's value
	if cfg.Type == "counter" && len(cfg.Value) == 0 {
		cfg.Value = "1.0"
	}
	err := cfg.InitTemplates()
	if err != nil {
		t.Fatal(err)
	}
	return cfg
}
<|endoftext|>


<commit_before>package main

import (
	"errors"
	"fmt"
	"io"
	"log"
	"net/http"
	"os"
	"path"
	"regexp"
	"strconv"
)

// fileRoute matches /files/<id>. Go seems to use \r to terminate header
// values, so to ease bash scripting, the route ignores a trailing \r in the
// route. Better ideas are welcome.
var fileRoute = regexp.MustCompile("^/files/([^/\r\n]+)\r?$")

var filesRoute = regexp.MustCompile("^/files/?$")
var dataStore *DataStore

func init() {
	wd, err := os.Getwd()
	if err != nil {
		panic(err)
	}

	dataDir := path.Join(wd, "tus_data")
	if configDir := os.Getenv("TUSD_DATA_DIR"); configDir != "" {
		dataDir = configDir
	}

	// dataStoreSize limits the storage used by the data store. If exceeded, the
	// data store will start garbage collection old files until enough storage is
	// available again.
	var dataStoreSize int64
	dataStoreSize = 1024 * 1024 * 1024
	if configStoreSize := os.Getenv("TUSD_DATA_STORE_MAXSIZE"); configStoreSize != "" {
		parsed, err := strconv.ParseInt(configStoreSize, 10, 64)
		if err != nil {
			panic(errors.New("Invalid data store max size configured"))
		}
		dataStoreSize = parsed
	}

	log.Print("Datastore directory: ", dataDir)
	log.Print("Datastore max size: ", dataStoreSize)

	if err := os.MkdirAll(dataDir, 0777); err != nil {
		panic(err)
	}
	dataStore = NewDataStore(dataDir, dataStoreSize)
}

func serveHttp() error {
	http.HandleFunc("/", route)

	addr := ":1080"
	if port := os.Getenv("TUSD_PORT"); port != "" {
		addr = ":" + port
	}
	log.Printf("serving clients at %s", addr)

	return http.ListenAndServe(addr, nil)
}

func route(w http.ResponseWriter, r *http.Request) {
	log.Printf("request: %s %s", r.Method, r.URL.RequestURI())

	w.Header().Set("Server", "tusd")

	// Allow CORS for almost everything. This needs to be revisted / limited to
	// routes and methods that need it.
	w.Header().Add("Access-Control-Allow-Origin", "*")
	w.Header().Add("Access-Control-Allow-Methods", "HEAD,GET,PUT,POST,DELETE")
	w.Header().Add("Access-Control-Allow-Headers", "Origin, X-Requested-With, Content-Type, Accept, Content-Range, Content-Disposition")
	w.Header().Add("Access-Control-Expose-Headers", "Location, Range, Content-Disposition")

	if r.Method == "OPTIONS" {
		reply(w, http.StatusOK, "")
		return
	}

	if r.Method == "POST" && filesRoute.Match([]byte(r.URL.Path)) {
		postFiles(w, r)
	} else if match := fileRoute.FindStringSubmatch(r.URL.Path); match != nil {
		id := match[1]
		switch r.Method {
		case "HEAD":
			headFile(w, r, id)
		case "GET":
			getFile(w, r, id)
		case "PUT":
			putFile(w, r, id)
		default:
			reply(w, http.StatusMethodNotAllowed, "Invalid http method")
		}
	} else {
		reply(w, http.StatusNotFound, "No matching route")
	}
}

func reply(w http.ResponseWriter, code int, message string) {
	w.WriteHeader(code)
	fmt.Fprintf(w, "%d - %s: %s\n", code, http.StatusText(code), message)
}

func postFiles(w http.ResponseWriter, r *http.Request) {
	contentRange, err := parseContentRange(r.Header.Get("Content-Range"))
	if err != nil {
		reply(w, http.StatusBadRequest, err.Error())
		return
	}

	if contentRange.Size == -1 {
		reply(w, http.StatusBadRequest, "Content-Range must indicate total file size.")
		return
	}

	contentType := r.Header.Get("Content-Type")
	if contentType == "" {
		contentType = "application/octet-stream"
	}

	contentDisposition := r.Header.Get("Content-Disposition")

	id := uid()
	if err := dataStore.CreateFile(id, contentRange.Size, contentType, contentDisposition); err != nil {
		reply(w, http.StatusInternalServerError, err.Error())
		return
	}

	if contentRange.End != -1 {
		err := dataStore.WriteFileChunk(id, contentRange.Start, contentRange.End, r.Body)
		if os.IsNotExist(err) {
			reply(w, http.StatusNotFound, err.Error())
			return
		} else if err != nil {
			reply(w, http.StatusInternalServerError, err.Error())
			return
		}

	}

	w.Header().Set("Location", "http://"+r.Host+"/files/"+id)
	setFileHeaders(w, id)
	w.WriteHeader(http.StatusCreated)
}

func headFile(w http.ResponseWriter, r *http.Request, fileId string) {
	// Work around a bug in Go that would cause HEAD responses to hang. Should be
	// fixed in future release, see:
	// http://code.google.com/p/go/issues/detail?id=4126
	w.Header().Set("Content-Length", "0")
	setFileHeaders(w, fileId)
}

func getFile(w http.ResponseWriter, r *http.Request, fileId string) {
	meta, err := dataStore.GetFileMeta(fileId)
	if os.IsNotExist(err) {
		reply(w, http.StatusNotFound, err.Error())
		return
	} else if err != nil {
		reply(w, http.StatusInternalServerError, err.Error())
		return
	}

	data, err := dataStore.ReadFile(fileId)
	if os.IsNotExist(err) {
		reply(w, http.StatusNotFound, err.Error())
		return
	} else if err != nil {
		reply(w, http.StatusInternalServerError, err.Error())
		return
	}

	defer data.Close()

	setFileHeaders(w, fileId)
	w.Header().Set("Content-Length", strconv.FormatInt(meta.Size, 10))

	if _, err := io.CopyN(w, data, meta.Size); err != nil {
		log.Printf("getFile: CopyN of fileId %s failed with: %s. Is the upload complete yet?", fileId, err.Error())
		return
	}
}

func putFile(w http.ResponseWriter, r *http.Request, fileId string) {
	var start int64 = 0
	var end int64 = 0

	contentRange, err := parseContentRange(r.Header.Get("Content-Range"))
	if err != nil {
		contentLength := r.Header.Get("Content-Length")
		end, err = strconv.ParseInt(contentLength, 10, 64)
		if err != nil {
			reply(w, http.StatusBadRequest, "Invalid content length provided")
		}

		// we are zero-indexed
		end = end - 1

		// @TODO: Make sure contentLength matches the content length of the initial
		// POST request
	} else {

		// @TODO: Make sure contentRange.Size matches file size

		start = contentRange.Start
		end = contentRange.End
	}

	// @TODO: Check that file exists

	err = dataStore.WriteFileChunk(fileId, start, end, r.Body)
	if os.IsNotExist(err) {
		reply(w, http.StatusNotFound, err.Error())
		return
	} else if err != nil {
		reply(w, http.StatusInternalServerError, err.Error())
		return
	}

	setFileHeaders(w, fileId)
}

func setFileHeaders(w http.ResponseWriter, fileId string) {
	meta, err := dataStore.GetFileMeta(fileId)
	if os.IsNotExist(err) {
		reply(w, http.StatusNotFound, err.Error())
		return
	} else if err != nil {
		reply(w, http.StatusInternalServerError, err.Error())
		return
	}

	rangeHeader := ""
	for i, chunk := range meta.Chunks {
		rangeHeader += fmt.Sprintf("%d-%d", chunk.Start, chunk.End)
		if i+1 < len(meta.Chunks) {
			rangeHeader += ","
		}
	}

	if rangeHeader != "" {
		w.Header().Set("Range", "bytes="+rangeHeader)
	}

	w.Header().Set("Content-Type", meta.ContentType)
	w.Header().Set("Content-Disposition", meta.ContentDisposition)
}
<commit_msg>Add reply logging<commit_after>package main

import (
	"errors"
	"fmt"
	"io"
	"log"
	"net/http"
	"os"
	"path"
	"regexp"
	"strconv"
	"time"
)

// fileRoute matches /files/<id>. Go seems to use \r to terminate header
// values, so to ease bash scripting, the route ignores a trailing \r in the
// route. Better ideas are welcome.
var fileRoute = regexp.MustCompile("^/files/([^/\r\n]+)\r?$")

var filesRoute = regexp.MustCompile("^/files/?$")
var dataStore *DataStore

func init() {
	wd, err := os.Getwd()
	if err != nil {
		panic(err)
	}

	dataDir := path.Join(wd, "tus_data")
	if configDir := os.Getenv("TUSD_DATA_DIR"); configDir != "" {
		dataDir = configDir
	}

	// dataStoreSize limits the storage used by the data store. If exceeded, the
	// data store will start garbage collection old files until enough storage is
	// available again.
	var dataStoreSize int64
	dataStoreSize = 1024 * 1024 * 1024
	if configStoreSize := os.Getenv("TUSD_DATA_STORE_MAXSIZE"); configStoreSize != "" {
		parsed, err := strconv.ParseInt(configStoreSize, 10, 64)
		if err != nil {
			panic(errors.New("Invalid data store max size configured"))
		}
		dataStoreSize = parsed
	}

	log.Print("Datastore directory: ", dataDir)
	log.Print("Datastore max size: ", dataStoreSize)

	if err := os.MkdirAll(dataDir, 0777); err != nil {
		panic(err)
	}
	dataStore = NewDataStore(dataDir, dataStoreSize)
}

func serveHttp() error {
	http.HandleFunc("/", route)

	addr := ":1080"
	if port := os.Getenv("TUSD_PORT"); port != "" {
		addr = ":" + port
	}
	log.Printf("serving clients at %s", addr)

	return http.ListenAndServe(addr, nil)
}

func route(w http.ResponseWriter, r *http.Request) {
	start := time.Now()
	log.Printf("request: %s %s", r.Method, r.URL.RequestURI())

	w.Header().Set("Server", "tusd")

	// Allow CORS for almost everything. This needs to be revisted / limited to
	// routes and methods that need it.
	w.Header().Add("Access-Control-Allow-Origin", "*")
	w.Header().Add("Access-Control-Allow-Methods", "HEAD,GET,PUT,POST,DELETE")
	w.Header().Add("Access-Control-Allow-Headers", "Origin, X-Requested-With, Content-Type, Accept, Content-Range, Content-Disposition")
	w.Header().Add("Access-Control-Expose-Headers", "Location, Range, Content-Disposition")

	if r.Method == "OPTIONS" {
		reply(w, http.StatusOK, "")
		return
	}

	if r.Method == "POST" && filesRoute.Match([]byte(r.URL.Path)) {
		postFiles(w, r)
	} else if match := fileRoute.FindStringSubmatch(r.URL.Path); match != nil {
		id := match[1]
		switch r.Method {
		case "HEAD":
			headFile(w, r, id)
		case "GET":
			getFile(w, r, id)
		case "PUT":
			putFile(w, r, id)
		default:
			reply(w, http.StatusMethodNotAllowed, "Invalid http method")
		}
	} else {
		reply(w, http.StatusNotFound, "No matching route")
	}

	duration := time.Since(start)
	log.Printf("finished: %s %s (took %s)", r.Method, r.URL.RequestURI(), duration)
}

func reply(w http.ResponseWriter, code int, message string) {
	w.WriteHeader(code)
	fmt.Fprintf(w, "%d - %s: %s\n", code, http.StatusText(code), message)
}

func postFiles(w http.ResponseWriter, r *http.Request) {
	contentRange, err := parseContentRange(r.Header.Get("Content-Range"))
	if err != nil {
		reply(w, http.StatusBadRequest, err.Error())
		return
	}

	if contentRange.Size == -1 {
		reply(w, http.StatusBadRequest, "Content-Range must indicate total file size.")
		return
	}

	contentType := r.Header.Get("Content-Type")
	if contentType == "" {
		contentType = "application/octet-stream"
	}

	contentDisposition := r.Header.Get("Content-Disposition")

	id := uid()
	if err := dataStore.CreateFile(id, contentRange.Size, contentType, contentDisposition); err != nil {
		reply(w, http.StatusInternalServerError, err.Error())
		return
	}

	if contentRange.End != -1 {
		err := dataStore.WriteFileChunk(id, contentRange.Start, contentRange.End, r.Body)
		if os.IsNotExist(err) {
			reply(w, http.StatusNotFound, err.Error())
			return
		} else if err != nil {
			reply(w, http.StatusInternalServerError, err.Error())
			return
		}

	}

	w.Header().Set("Location", "http://"+r.Host+"/files/"+id)
	setFileHeaders(w, id)
	w.WriteHeader(http.StatusCreated)
}

func headFile(w http.ResponseWriter, r *http.Request, fileId string) {
	// Work around a bug in Go that would cause HEAD responses to hang. Should be
	// fixed in future release, see:
	// http://code.google.com/p/go/issues/detail?id=4126
	w.Header().Set("Content-Length", "0")
	setFileHeaders(w, fileId)
}

func getFile(w http.ResponseWriter, r *http.Request, fileId string) {
	meta, err := dataStore.GetFileMeta(fileId)
	if os.IsNotExist(err) {
		reply(w, http.StatusNotFound, err.Error())
		return
	} else if err != nil {
		reply(w, http.StatusInternalServerError, err.Error())
		return
	}

	data, err := dataStore.ReadFile(fileId)
	if os.IsNotExist(err) {
		reply(w, http.StatusNotFound, err.Error())
		return
	} else if err != nil {
		reply(w, http.StatusInternalServerError, err.Error())
		return
	}

	defer data.Close()

	setFileHeaders(w, fileId)
	w.Header().Set("Content-Length", strconv.FormatInt(meta.Size, 10))

	if _, err := io.CopyN(w, data, meta.Size); err != nil {
		log.Printf("getFile: CopyN of fileId %s failed with: %s. Is the upload complete yet?", fileId, err.Error())
		return
	}
}

func putFile(w http.ResponseWriter, r *http.Request, fileId string) {
	var start int64 = 0
	var end int64 = 0

	contentRange, err := parseContentRange(r.Header.Get("Content-Range"))
	if err != nil {
		contentLength := r.Header.Get("Content-Length")
		end, err = strconv.ParseInt(contentLength, 10, 64)
		if err != nil {
			reply(w, http.StatusBadRequest, "Invalid content length provided")
		}

		// we are zero-indexed
		end = end - 1

		// @TODO: Make sure contentLength matches the content length of the initial
		// POST request
	} else {

		// @TODO: Make sure contentRange.Size matches file size

		start = contentRange.Start
		end = contentRange.End
	}

	// @TODO: Check that file exists

	err = dataStore.WriteFileChunk(fileId, start, end, r.Body)
	if os.IsNotExist(err) {
		reply(w, http.StatusNotFound, err.Error())
		return
	} else if err != nil {
		reply(w, http.StatusInternalServerError, err.Error())
		return
	}

	setFileHeaders(w, fileId)
}

func setFileHeaders(w http.ResponseWriter, fileId string) {
	meta, err := dataStore.GetFileMeta(fileId)
	if os.IsNotExist(err) {
		reply(w, http.StatusNotFound, err.Error())
		return
	} else if err != nil {
		reply(w, http.StatusInternalServerError, err.Error())
		return
	}

	rangeHeader := ""
	for i, chunk := range meta.Chunks {
		rangeHeader += fmt.Sprintf("%d-%d", chunk.Start, chunk.End)
		if i+1 < len(meta.Chunks) {
			rangeHeader += ","
		}
	}

	if rangeHeader != "" {
		w.Header().Set("Range", "bytes="+rangeHeader)
	}

	w.Header().Set("Content-Type", meta.ContentType)
	w.Header().Set("Content-Disposition", meta.ContentDisposition)
}
<|endoftext|>


<commit_before>package common

import (
	"fmt"

	"github.com/satori/go.uuid"
)

const (
	MajorVersion      = 0
	MinorVersion      = 10
	MicroVersion      = 0
	AdditionalVersion = "dev"
)

func VersionString() string {
	return fmt.Sprintf("%d.%d.%d%s", MajorVersion, MinorVersion, MicroVersion, AdditionalVersion)
}

func NewUUID() string {
	return uuid.NewV4().String()
}
<commit_msg>register protolog logger<commit_after>package common

import (
	"fmt"

	"go.pedge.io/protolog/logrus"

	"github.com/satori/go.uuid"
)

const (
	MajorVersion      = 0
	MinorVersion      = 10
	MicroVersion      = 0
	AdditionalVersion = "dev"
)

func init() {
	logrus.Register()
}

func VersionString() string {
	return fmt.Sprintf("%d.%d.%d%s", MajorVersion, MinorVersion, MicroVersion, AdditionalVersion)
}

func NewUUID() string {
	return uuid.NewV4().String()
}
<|endoftext|>


<commit_before>package pool

import "net"
import "error"
import "fmt"
import "sync"

//blockingPool implements the Pool interface.
//Connestions from blockingPool offer a kind of blocking mechanism that is derived from buffered channel.
type blockingPool struct {
	//mutex is to make closing the pool and recycling the connection an atomic operation
	mutex sync.Mutex

	//storage for net.Conn connections
	conns chan net.Conn

	//net.Conn generator
	factory Factory
}

//Factory is a function to create new connections
//which is provided by the user
type Factory func() (net.Conn, error)

func NewBlockingPool(initCap, maxCap int, factory Factory) (net.Conn, error) {
	if initCap < 0 || maxCap < 1 || initCap > maxCap {
		return nil, errors.New("invalid capacity settings")
	}
}

func (p *blockingPool) Get() (net.Conn, error) {}

func (p *blockingPool) Close() {}

func (p *blockingPool) Len() {}
<commit_msg>blockingPool in process<commit_after>package pool

import "net"
import "errors"
import "sync"
import "time"

//blockingPool implements the Pool interface.
//Connestions from blockingPool offer a kind of blocking mechanism that is derived from buffered channel.
type blockingPool struct {
	//mutex is to make closing the pool and recycling the connection an atomic operation
	mutex sync.Mutex

	//timeout to Get, default to 3
	timeout int

	//storage for net.Conn connections
	conns chan net.Conn

	//net.Conn generator
	factory Factory
}

//Factory is a function to create new connections
//which is provided by the user
type Factory func() (net.Conn, error)

//Create a new blocking pool.
//As no new connections would be made when the pool is busy, maxCap does not make sense yet.
func NewBlockingPool(initCap, maxCap int, factory Factory) (Pool, error) {
	if initCap < 0 || maxCap < 1 || initCap > maxCap {
		return nil, errors.New("invalid capacity settings")
	}

	newPool := &blockingPool{
		timeout: 3,
		conns: make(chan net.Conn, maxCap),
		factory: factory
	}

	for i := 0; i < initCap; i++ {
		conn, err := factory()
		if err != nil {
			newPool.Close()
			return nil, fmt.Errorf("error counted when calling factory: %s", err)
		}
		newPool.conns <- conn
	}
	return newPool, nil
}

func (p *blockingPool) Get() (net.Conn, error) {
	//in case that pool is closed and pool.conns is set to nil
	conns := p.conns
	if conns == nil {
		return nil, ErrClosed
	}

	select {
	case conn := <-conns:
		return conn, nil/*not wrapped yet*/
	case <-time.After(time.Second*p.timeout) {
		return nil, errors.New("timeout")
	}
}

//put puts the connection back to the pool. If the pool is closed, put simply close
//any connections received and return immediately. A nil net.Conn is illegal and will be rejected.
func (p *blockingPool) put(conn net.Conn) error {
	if conn == nil {
		return errors.New("connection is nil.")
	}

	//in case that pool is closed and pool.conns is set to nil
	conns := p.conns
	if conns == nil {
		return conn.Close()
	}

	//It is impossible to block as number of connections is never more than length of channel
	conns <-conn
	return nil
}

func (p *blockingPool) Close() {}

func (p *blockingPool) Len() {}
<|endoftext|>


<commit_before>package ppserver

import (
	"appengine"
	"appengine/datastore"
	"encoding/json"
	"log"
	"net/http"

	"github.com/nictuku/webpprof/ppcommon"
)

func init() {
	http.HandleFunc("/profile", HandlePostProfile)

}

func HandlePostProfile(w http.ResponseWriter, r *http.Request) {
	c := appengine.NewContext(r)

	p := r.FormValue("p")
	if p == "" {
		http.NotFound(w, r)
		return
	}
	var profile ppcommon.Profile
	defer r.Body.Close()
	dec := json.NewDecoder(r.Body)
	err := dec.Decode(&profile)
	if err != nil {
		log.Printf("handle post profile error parsing JSON for %v: %v", p, err)
		http.Error(w, "invalid profile content", http.StatusBadRequest)
		return
	}

	//fmt.Fprintf(w, "Hello, %q", html.EscapeString(r.URL.String()))
	/*	body, err := ioutil.ReadAll(r.Body)
		if err != nil {
			log.Printf("handleProfile read body error: %v", err)
			return
		}
	*/
	_, err = datastore.Put(c, datastore.NewIncompleteKey(c, "Profile", nil), &profile)
	if err != nil {
		http.Error(w, err.Error(), http.StatusInternalServerError)
		return
	}
	log.Printf("decoded profile: %+q", profile)
	return
}
<commit_msg>cznic/ql for datastore<commit_after>package ppserver

import (
	"database/sql"
	"encoding/json"
	"fmt"
	"log"
	"net/http"
	"sync"

	// Install the QL SQL driver.
	_ "github.com/cznic/ql/driver"
	"github.com/nictuku/webpprof/ppcommon"
)

var (
	db   *sql.DB
	once sync.Once
)

// HandlePostProfile receives a pprof profile and stores it.
func HandlePostProfile(w http.ResponseWriter, r *http.Request) {
	once.Do(func() {
		var err error
		db, err = sql.Open("ql", "ql.db")
		if err != nil {
			log.Fatalf("db opening error: %v", err)
		}
	})

	p := r.FormValue("p")
	if p == "" {
		http.NotFound(w, r)
		return
	}
	var profile ppcommon.Profile
	defer r.Body.Close()
	dec := json.NewDecoder(r.Body)
	if err := dec.Decode(&profile); err != nil {
		log.Printf("handle post profile error parsing JSON for %v: %v", p, err)
		http.Error(w, "invalid profile content", http.StatusBadRequest)
		return
	}
	if err := saveProfile(&profile); err != nil {
		http.Error(w, err.Error(), http.StatusInternalServerError)
		return
	}
}

// Schema:
//
// CREATE TABLE profiles (user string, profile blob, t time);

func saveProfile(p *Profile) (err error) {
	tx, err := db.Begin()
	if err != nil {
		return
	}
	result, err := tx.Exec(`
		INSERT INTO profiles VALUES ($1, $2, now());`,
		"yves.junqueira@gmail.com", profile.Content)

	if err != nil {
		return
	}
	if err := tx.Commit(); err != nil {
		return
	}
	id, err := result.LastInsertId()
	if err != nil {
		return
	}
	aff, err := result.RowsAffected()
	if err != nil {
		return
	}

	fmt.Printf("LastInsertId %d, RowsAffected %d\n", id, aff)

	log.Printf("decoded profile: %+q", profile)
	return
}
<|endoftext|>


<commit_before>// Package realtime - Fetch realtime stock data info http://mis.tse.com.tw/
// 擷取盤中即時股價資訊
//
package realtime

import (
	"encoding/json"
	"fmt"
	"net/http"
	"strconv"
	"strings"
	"time"

	"github.com/toomore/gogrs/utils"
)

//STOCKPATH = '/stock/api/getStockInfo.jsp?ex_ch=%(exchange)s_%(no)s.tw_%(date)s&json=1&delay=%(delay)s&_=%(timestamp)s'

type msgArray []map[string]string
type unixMapData map[int64]msgArray

// StockRealTime start with No, Timestamp, Date.
type StockRealTime struct {
	No          string
	Timestamp   int64
	Date        time.Time
	UnixMapData unixMapData
	Exchange    string
}

var exchangeMap = map[string]bool{"tse": true, "otc": true}

// StockBlob return map data.
type StockBlob struct {
	Rtcode    string
	UserDelay int
	Rtmessage string
	Referer   string
	MsgArray  msgArray
	QueryTime map[string]interface{}
}

// URL return realtime url path.
func (stock StockRealTime) URL() string {
	if exchangeMap[stock.Exchange] {
		return fmt.Sprintf("%s%s", utils.TWSEURL,
			fmt.Sprintf(utils.TWSEREAL,
				stock.Exchange,
				stock.No,
				fmt.Sprintf(
					"%d%02d%02d",
					stock.Date.Year(),
					int(stock.Date.Month()),
					stock.Date.Day(),
				),
				stock.Timestamp,
			))
	}
	return ""
}

// StockInfo is base stock info.
type StockInfo struct {
	Exchange string // tse or otc
	FullName string // Full company name.
	Name     string // Stock name.
	No       string // Stock no
	Ticker   string // Ticker symbol（股票代號）
}

// Data is realtime return formated data.
type Data struct {
	BestAskPrice   []float64
	BestBidPrice   []float64
	BestAskVolume  []int64
	BestBidVolume  []int64
	Open           float64
	Highest        float64
	Lowest         float64
	Price          float64
	LimitUp        float64
	LimitDown      float64
	Volume         float64
	VolumeAcc      float64
	YesterdayPrice float64
	Info           StockInfo
}

func (stock *StockRealTime) get() (StockBlob, error) {
	var value StockBlob
	url := stock.URL()
	resp, err := http.Get(url)
	if err != nil {
		return value, fmt.Errorf("Network fail: %s", err)
	}
	defer resp.Body.Close()
	json.NewDecoder(resp.Body).Decode(&value)

	if len(value.MsgArray) != 0 {
		unixTime, _ := strconv.ParseInt(value.MsgArray[0]["tlong"], 10, 64)
		if stock.UnixMapData == nil {
			stock.UnixMapData = make(unixMapData)
		}

		// Should format data.
		stock.UnixMapData[unixTime/1000] = value.MsgArray
	}

	return value, nil
}

// Get return stock realtime map data.
func (stock *StockRealTime) Get() (Data, error) {
	value, err := stock.get()

	if err != nil {
		return Data{}, err
	}

	if len(value.MsgArray) != 0 {
		var result Data
		aList := strings.Split(value.MsgArray[0]["a"], "_")
		result.BestAskPrice = make([]float64, len(aList)-1)
		for i, v := range aList[:len(aList)-1] {
			result.BestAskPrice[i], _ = strconv.ParseFloat(v, 10)
		}

		bList := strings.Split(value.MsgArray[0]["b"], "_")
		result.BestBidPrice = make([]float64, len(bList)-1)
		for i, v := range bList[:len(bList)-1] {
			result.BestBidPrice[i], _ = strconv.ParseFloat(v, 10)
		}

		fList := strings.Split(value.MsgArray[0]["f"], "_")
		result.BestAskVolume = make([]int64, len(fList)-1)
		for i, v := range fList[:len(fList)-1] {
			result.BestAskVolume[i], _ = strconv.ParseInt(v, 10, 64)
		}

		gList := strings.Split(value.MsgArray[0]["g"], "_")
		result.BestBidVolume = make([]int64, len(gList)-1)
		for i, v := range gList[:len(gList)-1] {
			result.BestBidVolume[i], _ = strconv.ParseInt(v, 10, 64)
		}

		result.Open, _ = strconv.ParseFloat(value.MsgArray[0]["o"], 10)
		result.Highest, _ = strconv.ParseFloat(value.MsgArray[0]["h"], 10)
		result.Lowest, _ = strconv.ParseFloat(value.MsgArray[0]["l"], 10)
		result.Price, _ = strconv.ParseFloat(value.MsgArray[0]["z"], 10)
		result.LimitUp, _ = strconv.ParseFloat(value.MsgArray[0]["u"], 10)
		result.LimitDown, _ = strconv.ParseFloat(value.MsgArray[0]["w"], 10)
		result.Volume, _ = strconv.ParseFloat(value.MsgArray[0]["tv"], 10)
		result.VolumeAcc, _ = strconv.ParseFloat(value.MsgArray[0]["v"], 10)
		result.YesterdayPrice, _ = strconv.ParseFloat(value.MsgArray[0]["y"], 10)

		result.Info.No = value.MsgArray[0]["n"]
		result.Info.FullName = value.MsgArray[0]["nf"]
		result.Info.No = value.MsgArray[0]["n"]
		result.Info.Ticker = value.MsgArray[0]["ch"]
		result.Info.Exchange = value.MsgArray[0]["ex"]

		return result, nil
	}

	return Data{}, nil
}
<commit_msg>Add some note.<commit_after>// Package realtime - Fetch realtime stock data info http://mis.tse.com.tw/
// 擷取盤中即時股價資訊
//
package realtime

import (
	"encoding/json"
	"fmt"
	"net/http"
	"strconv"
	"strings"
	"time"

	"github.com/toomore/gogrs/utils"
)

//STOCKPATH = '/stock/api/getStockInfo.jsp?ex_ch=%(exchange)s_%(no)s.tw_%(date)s&json=1&delay=%(delay)s&_=%(timestamp)s'

type msgArray []map[string]string
type unixMapData map[int64]msgArray

// StockRealTime start with No, Timestamp, Date.
type StockRealTime struct {
	No          string      // 股票代碼
	Timestamp   int64       // 時間戳記
	Date        time.Time   // 擷取時間
	UnixMapData unixMapData // 時間資料暫存
	Exchange    string      // tse, otc
}

var exchangeMap = map[string]bool{"tse": true, "otc": true}

// StockBlob return map data.
type StockBlob struct {
	Rtcode    string
	UserDelay int
	Rtmessage string
	Referer   string
	MsgArray  msgArray
	QueryTime map[string]interface{}
}

// URL return realtime url path.
func (stock StockRealTime) URL() string {
	if exchangeMap[stock.Exchange] {
		return fmt.Sprintf("%s%s", utils.TWSEURL,
			fmt.Sprintf(utils.TWSEREAL,
				stock.Exchange,
				stock.No,
				fmt.Sprintf(
					"%d%02d%02d",
					stock.Date.Year(),
					int(stock.Date.Month()),
					stock.Date.Day(),
				),
				stock.Timestamp,
			))
	}
	return ""
}

// StockInfo is base stock info.
type StockInfo struct {
	Exchange string // tse or otc
	FullName string // Full company name.
	Name     string // Stock name.
	No       string // Stock no
	Ticker   string // Ticker symbol（股票代號）
}

// Data is realtime return formated data.
type Data struct {
	BestAskPrice   []float64 // 最佳五檔賣出價資訊
	BestBidPrice   []float64 // 最佳五檔買進價資訊
	BestAskVolume  []int64   // 最佳五檔賣出量資訊
	BestBidVolume  []int64   // 最佳五檔買進量資訊
	Open           float64   // 開盤價格
	Highest        float64   // 最高價
	Lowest         float64   // 最低價
	Price          float64   // 該盤成交價格
	LimitUp        float64   // 漲停價
	LimitDown      float64   // 跌停價
	Volume         float64   // 該盤成交量
	VolumeAcc      float64   // 累計成交量
	YesterdayPrice float64   // 昨日收盤價格
	Info           StockInfo // 相關資訊
}

func (stock *StockRealTime) get() (StockBlob, error) {
	var value StockBlob
	url := stock.URL()
	resp, err := http.Get(url)
	if err != nil {
		return value, fmt.Errorf("Network fail: %s", err)
	}
	defer resp.Body.Close()
	json.NewDecoder(resp.Body).Decode(&value)

	if len(value.MsgArray) != 0 {
		unixTime, _ := strconv.ParseInt(value.MsgArray[0]["tlong"], 10, 64)
		if stock.UnixMapData == nil {
			stock.UnixMapData = make(unixMapData)
		}

		// Should format data.
		stock.UnixMapData[unixTime/1000] = value.MsgArray
	}

	return value, nil
}

// Get return stock realtime map data.
func (stock *StockRealTime) Get() (Data, error) {
	value, err := stock.get()

	if err != nil {
		return Data{}, err
	}

	if len(value.MsgArray) != 0 {
		var result Data
		aList := strings.Split(value.MsgArray[0]["a"], "_")
		result.BestAskPrice = make([]float64, len(aList)-1)
		for i, v := range aList[:len(aList)-1] {
			result.BestAskPrice[i], _ = strconv.ParseFloat(v, 10)
		}

		bList := strings.Split(value.MsgArray[0]["b"], "_")
		result.BestBidPrice = make([]float64, len(bList)-1)
		for i, v := range bList[:len(bList)-1] {
			result.BestBidPrice[i], _ = strconv.ParseFloat(v, 10)
		}

		fList := strings.Split(value.MsgArray[0]["f"], "_")
		result.BestAskVolume = make([]int64, len(fList)-1)
		for i, v := range fList[:len(fList)-1] {
			result.BestAskVolume[i], _ = strconv.ParseInt(v, 10, 64)
		}

		gList := strings.Split(value.MsgArray[0]["g"], "_")
		result.BestBidVolume = make([]int64, len(gList)-1)
		for i, v := range gList[:len(gList)-1] {
			result.BestBidVolume[i], _ = strconv.ParseInt(v, 10, 64)
		}

		result.Open, _ = strconv.ParseFloat(value.MsgArray[0]["o"], 10)
		result.Highest, _ = strconv.ParseFloat(value.MsgArray[0]["h"], 10)
		result.Lowest, _ = strconv.ParseFloat(value.MsgArray[0]["l"], 10)
		result.Price, _ = strconv.ParseFloat(value.MsgArray[0]["z"], 10)
		result.LimitUp, _ = strconv.ParseFloat(value.MsgArray[0]["u"], 10)
		result.LimitDown, _ = strconv.ParseFloat(value.MsgArray[0]["w"], 10)
		result.Volume, _ = strconv.ParseFloat(value.MsgArray[0]["tv"], 10)
		result.VolumeAcc, _ = strconv.ParseFloat(value.MsgArray[0]["v"], 10)
		result.YesterdayPrice, _ = strconv.ParseFloat(value.MsgArray[0]["y"], 10)

		result.Info.No = value.MsgArray[0]["n"]
		result.Info.FullName = value.MsgArray[0]["nf"]
		result.Info.No = value.MsgArray[0]["n"]
		result.Info.Ticker = value.MsgArray[0]["ch"]
		result.Info.Exchange = value.MsgArray[0]["ex"]

		return result, nil
	}

	return Data{}, nil
}
<|endoftext|>


<commit_before>package pager

import (
	"strconv"
	"sync"
)

// Pager is a tool to help paging
type Pager struct {
	BeginPage  int
	BeginIndex int
	PageSize   int

	MaxPage int
}

func (p *Pager) Begin(page int) int {
	if page <= p.BeginPage {
		return p.BeginIndex
	}

	return (page-p.BeginPage)*p.PageSize + p.BeginIndex
}

func (p *Pager) End(page int) int {
	return p.Begin(page) + p.PageSize
}

func (p *Pager) BeginByString(page string) int {
	if page == "" {
		return p.BeginIndex
	}

	val, err := strconv.Atoi(page)
	if err != nil {
		return p.BeginIndex
	}

	return p.Begin(val)
}

func (p *Pager) IsOverRange(start, count int) bool {
	if p.MaxPage > 0 {
		return (start + count) >= p.PageSize*p.MaxPage
	}
	return false
}

func (p *Pager) EndByString(page string) int {
	return p.BeginByString(page) + p.PageSize
}

type PagerGroup struct {
	pagers []Pager
	lock   sync.Mutex
}

func (pg *PagerGroup) Add(beginPage, beginIndex, pageSize, maxPage int) *Pager {
	if beginPage < 0 {
		beginPage = 1
	}

	if beginIndex < 0 {
		beginIndex = 0
	}

	pg.lock.Lock()
	l := len(pg.pagers)
	pg.pagers = append(pg.pagers, Pager{
		BeginPage:  beginPage,
		BeginIndex: beginIndex,
		PageSize:   pageSize,
		MaxPage:    maxPage,
	})
	p := &pg.pagers[l]
	pg.lock.Unlock()
	return p
}
<commit_msg>utils/pager: add IsReachBottom method<commit_after>package pager

import (
	"strconv"
	"sync"
)

// Pager is a tool to help paging
type Pager struct {
	BeginPage  int
	BeginIndex int
	PageSize   int

	MaxPage int
}

func (p *Pager) Begin(page int) int {
	if page <= p.BeginPage {
		return p.BeginIndex
	}

	return (page-p.BeginPage)*p.PageSize + p.BeginIndex
}

func (p *Pager) End(page int) int {
	return p.Begin(page) + p.PageSize
}

func (p *Pager) BeginByString(page string) int {
	if page == "" {
		return p.BeginIndex
	}

	val, err := strconv.Atoi(page)
	if err != nil {
		return p.BeginIndex
	}

	return p.Begin(val)
}

func (p *Pager) IsOverRange(start, count int) bool {
	if p.MaxPage > 0 {
		return (start + count) > p.PageSize*p.MaxPage
	}
	return false
}

func (p *Pager) IsReachBottom(start, count, maxPage int) bool {
	if maxPage <= 0 {
		maxPage = p.MaxPage
	} else if p.MaxPage > 0 && p.MaxPage < maxPage {
		maxPage = p.MaxPage
	}
	if maxPage > 0 {
		return (start + count) >= p.PageSize*maxPage
	}
	return false
}

func (p *Pager) EndByString(page string) int {
	return p.BeginByString(page) + p.PageSize
}

type PagerGroup struct {
	pagers []Pager
	lock   sync.Mutex
}

func (pg *PagerGroup) Add(beginPage, beginIndex, pageSize, maxPage int) *Pager {
	if beginPage < 0 {
		beginPage = 1
	}

	if beginIndex < 0 {
		beginIndex = 0
	}

	pg.lock.Lock()
	l := len(pg.pagers)
	pg.pagers = append(pg.pagers, Pager{
		BeginPage:  beginPage,
		BeginIndex: beginIndex,
		PageSize:   pageSize,
		MaxPage:    maxPage,
	})
	p := &pg.pagers[l]
	pg.lock.Unlock()
	return p
}
<|endoftext|>