<commit_before>package cloudflare

import (
	"context"
	"encoding/json"
	"fmt"
	"net/http"

	"github.com/pkg/errors"
)

// DevicePostureRule represents a device posture rule.
type DevicePostureRule struct {
	ID          string                   `json:"id,omitempty"`
	Type        string                   `json:"type"`
	Name        string                   `json:"name"`
	Description string                   `json:"description,omitempty"`
	Schedule    string                   `json:"schedule,omitempty"`
	Match       []DevicePostureRuleMatch `json:"match,omitempty"`
	Input       DevicePostureRuleInput   `json:"input,omitempty"`
}

// DevicePostureRuleMatch represents the conditions that the client must match to run the rule.
type DevicePostureRuleMatch struct {
	Platform string `json:"platform,omitempty"`
}

// DevicePostureRuleInput represents the value to be checked against.
type DevicePostureRuleInput struct {
	ID string `json:"id"`
}

// DevicePostureRuleListResponse represents the response from the list
// device posture rules endpoint.
type DevicePostureRuleListResponse struct {
	Result []DevicePostureRule `json:"result"`
	Response
	ResultInfo `json:"result_info"`
}

// DevicePostureRuleDetailResponse is the API response, containing a single
// device posture rule.
type DevicePostureRuleDetailResponse struct {
	Success  bool              `json:"success"`
	Errors   []string          `json:"errors"`
	Messages []string          `json:"messages"`
	Result   DevicePostureRule `json:"result"`
}

// DevicePostureRules returns all device posture rules within an account.
//
// API reference: TODO
func (api *API) DevicePostureRules(ctx context.Context, accountID string) ([]DevicePostureRule, ResultInfo, error) {
	uri := fmt.Sprintf("/%s/%s/devices/posture", AccountRouteRoot, accountID)

	res, err := api.makeRequestContext(ctx, http.MethodGet, uri, nil)
	if err != nil {
		return []DevicePostureRule{}, ResultInfo{}, err
	}

	var devicePostureRuleListResponse DevicePostureRuleListResponse
	err = json.Unmarshal(res, &devicePostureRuleListResponse)
	if err != nil {
		return []DevicePostureRule{}, ResultInfo{}, errors.Wrap(err, errUnmarshalError)
	}

	return devicePostureRuleListResponse.Result, devicePostureRuleListResponse.ResultInfo, nil
}

// DevicePostureRule returns a single device posture rule based on the rule ID.
//
// API reference: TODO
func (api *API) DevicePostureRule(ctx context.Context, accountID, ruleID string) (DevicePostureRule, error) {
	uri := fmt.Sprintf(
		"/%s/%s/devices/posture/%s",
		AccountRouteRoot,
		accountID,
		ruleID,
	)

	res, err := api.makeRequestContext(ctx, http.MethodGet, uri, nil)
	if err != nil {
		return DevicePostureRule{}, err
	}

	var devicePostureRuleDetailResponse DevicePostureRuleDetailResponse
	err = json.Unmarshal(res, &devicePostureRuleDetailResponse)
	if err != nil {
		return DevicePostureRule{}, errors.Wrap(err, errUnmarshalError)
	}

	return devicePostureRuleDetailResponse.Result, nil
}

// CreateDevicePostureRule creates a new device posture rule.
//
// API reference: TODO
func (api *API) CreateDevicePostureRule(ctx context.Context, accountID string, rule DevicePostureRule) (DevicePostureRule, error) {
	uri := fmt.Sprintf("/%s/%s/devices/posture", AccountRouteRoot, accountID)

	res, err := api.makeRequestContext(ctx, http.MethodPost, uri, rule)
	if err != nil {
		return DevicePostureRule{}, err
	}

	var devicePostureRuleDetailResponse DevicePostureRuleDetailResponse
	err = json.Unmarshal(res, &devicePostureRuleDetailResponse)
	if err != nil {
		return DevicePostureRule{}, errors.Wrap(err, errUnmarshalError)
	}

	return devicePostureRuleDetailResponse.Result, nil
}

// UpdateDevicePostureRule updates an existing device posture rule.
//
// API reference: TODO
func (api *API) UpdateDevicePostureRule(ctx context.Context, accountID string, rule DevicePostureRule) (DevicePostureRule, error) {
	if rule.ID == "" {
		return DevicePostureRule{}, errors.Errorf("device posture rule ID cannot be empty")
	}

	uri := fmt.Sprintf(
		"/%s/%s/devices/posture/%s",
		AccountRouteRoot,
		accountID,
		rule.ID,
	)

	res, err := api.makeRequestContext(ctx, http.MethodPut, uri, rule)
	if err != nil {
		return DevicePostureRule{}, err
	}

	var devicePostureRuleDetailResponse DevicePostureRuleDetailResponse
	err = json.Unmarshal(res, &devicePostureRuleDetailResponse)
	if err != nil {
		return DevicePostureRule{}, errors.Wrap(err, errUnmarshalError)
	}

	return devicePostureRuleDetailResponse.Result, nil
}

// DeleteDevicePostureRule deletes a device posture rule.
//
// API reference: TODO
func (api *API) DeleteDevicePostureRule(ctx context.Context, accountID, ruleID string) error {
	uri := fmt.Sprintf(
		"/%s/%s/devices/posture/%s",
		AccountRouteRoot,
		accountID,
		ruleID,
	)

	_, err := api.makeRequestContext(ctx, http.MethodDelete, uri, nil)
	if err != nil {
		return err
	}

	return nil
}
<commit_msg>add documentation URLs for device posture<commit_after>package cloudflare

import (
	"context"
	"encoding/json"
	"fmt"
	"net/http"

	"github.com/pkg/errors"
)

// DevicePostureRule represents a device posture rule.
type DevicePostureRule struct {
	ID          string                   `json:"id,omitempty"`
	Type        string                   `json:"type"`
	Name        string                   `json:"name"`
	Description string                   `json:"description,omitempty"`
	Schedule    string                   `json:"schedule,omitempty"`
	Match       []DevicePostureRuleMatch `json:"match,omitempty"`
	Input       DevicePostureRuleInput   `json:"input,omitempty"`
}

// DevicePostureRuleMatch represents the conditions that the client must match to run the rule.
type DevicePostureRuleMatch struct {
	Platform string `json:"platform,omitempty"`
}

// DevicePostureRuleInput represents the value to be checked against.
type DevicePostureRuleInput struct {
	ID string `json:"id"`
}

// DevicePostureRuleListResponse represents the response from the list
// device posture rules endpoint.
type DevicePostureRuleListResponse struct {
	Result []DevicePostureRule `json:"result"`
	Response
	ResultInfo `json:"result_info"`
}

// DevicePostureRuleDetailResponse is the API response, containing a single
// device posture rule.
type DevicePostureRuleDetailResponse struct {
	Response
	Result DevicePostureRule `json:"result"`
}

// DevicePostureRules returns all device posture rules within an account.
//
// API reference: https://api.cloudflare.com/#device-posture-rules-list-device-posture-rules
func (api *API) DevicePostureRules(ctx context.Context, accountID string) ([]DevicePostureRule, ResultInfo, error) {
	uri := fmt.Sprintf("/%s/%s/devices/posture", AccountRouteRoot, accountID)

	res, err := api.makeRequestContext(ctx, http.MethodGet, uri, nil)
	if err != nil {
		return []DevicePostureRule{}, ResultInfo{}, err
	}

	var devicePostureRuleListResponse DevicePostureRuleListResponse
	err = json.Unmarshal(res, &devicePostureRuleListResponse)
	if err != nil {
		return []DevicePostureRule{}, ResultInfo{}, errors.Wrap(err, errUnmarshalError)
	}

	return devicePostureRuleListResponse.Result, devicePostureRuleListResponse.ResultInfo, nil
}

// DevicePostureRule returns a single device posture rule based on the rule ID.
//
// API reference: https://api.cloudflare.com/#device-posture-rules-device-posture-rules-details
func (api *API) DevicePostureRule(ctx context.Context, accountID, ruleID string) (DevicePostureRule, error) {
	uri := fmt.Sprintf(
		"/%s/%s/devices/posture/%s",
		AccountRouteRoot,
		accountID,
		ruleID,
	)

	res, err := api.makeRequestContext(ctx, http.MethodGet, uri, nil)
	if err != nil {
		return DevicePostureRule{}, err
	}

	var devicePostureRuleDetailResponse DevicePostureRuleDetailResponse
	err = json.Unmarshal(res, &devicePostureRuleDetailResponse)
	if err != nil {
		return DevicePostureRule{}, errors.Wrap(err, errUnmarshalError)
	}

	return devicePostureRuleDetailResponse.Result, nil
}

// CreateDevicePostureRule creates a new device posture rule.
//
// API reference: https://api.cloudflare.com/#device-posture-rules-create-device-posture-rule
func (api *API) CreateDevicePostureRule(ctx context.Context, accountID string, rule DevicePostureRule) (DevicePostureRule, error) {
	uri := fmt.Sprintf("/%s/%s/devices/posture", AccountRouteRoot, accountID)

	res, err := api.makeRequestContext(ctx, http.MethodPost, uri, rule)
	if err != nil {
		return DevicePostureRule{}, err
	}

	var devicePostureRuleDetailResponse DevicePostureRuleDetailResponse
	err = json.Unmarshal(res, &devicePostureRuleDetailResponse)
	if err != nil {
		return DevicePostureRule{}, errors.Wrap(err, errUnmarshalError)
	}

	return devicePostureRuleDetailResponse.Result, nil
}

// UpdateDevicePostureRule updates an existing device posture rule.
//
// API reference: https://api.cloudflare.com/#device-posture-rules-update-device-posture-rule
func (api *API) UpdateDevicePostureRule(ctx context.Context, accountID string, rule DevicePostureRule) (DevicePostureRule, error) {
	if rule.ID == "" {
		return DevicePostureRule{}, errors.Errorf("device posture rule ID cannot be empty")
	}

	uri := fmt.Sprintf(
		"/%s/%s/devices/posture/%s",
		AccountRouteRoot,
		accountID,
		rule.ID,
	)

	res, err := api.makeRequestContext(ctx, http.MethodPut, uri, rule)
	if err != nil {
		return DevicePostureRule{}, err
	}

	var devicePostureRuleDetailResponse DevicePostureRuleDetailResponse
	err = json.Unmarshal(res, &devicePostureRuleDetailResponse)
	if err != nil {
		return DevicePostureRule{}, errors.Wrap(err, errUnmarshalError)
	}

	return devicePostureRuleDetailResponse.Result, nil
}

// DeleteDevicePostureRule deletes a device posture rule.
//
// API reference: https://api.cloudflare.com/#device-posture-rules-delete-device-posture-rule
func (api *API) DeleteDevicePostureRule(ctx context.Context, accountID, ruleID string) error {
	uri := fmt.Sprintf(
		"/%s/%s/devices/posture/%s",
		AccountRouteRoot,
		accountID,
		ruleID,
	)

	_, err := api.makeRequestContext(ctx, http.MethodDelete, uri, nil)
	if err != nil {
		return err
	}

	return nil
}
<|endoftext|>


<commit_before>package docker

import (
	"fmt"
	"io"

	"github.com/fsouza/go-dockerclient"
	"github.com/hashicorp/errwrap"
)

const exportBufSize = 32768

type Client struct {
	docker dockerclient
}

func NewClient() (*Client, error) {
	client, err := newDockerClient()
	if err != nil {
		return nil, err
	}
	return &Client{
		docker: client,
	}, nil
}

func (c *Client) ExportImage(imageID string) (io.ReadCloser, error) {
	container, err := c.docker.CreateContainer(docker.CreateContainerOptions{
		Config: &docker.Config{
			Image: imageID,
		},
	})
	if err != nil {
		return nil, errwrap.Wrapf(fmt.Sprintf("Failed to create container (imageID:%s): {{err}}", imageID), err)
	}
	defer func(containerID string) error {
		return c.docker.RemoveContainer(docker.RemoveContainerOptions{
			ID:    containerID,
			Force: true,
		})
	}(container.ID)

	pReader, pWriter := io.Pipe()

	go func() {
		err := c.docker.ExportContainer(docker.ExportContainerOptions{
			ID:           container.ID,
			OutputStream: pWriter,
		})
		if err != nil {
			err = errwrap.Wrapf(fmt.Sprintf("Failed to export container %s: {{err}}", container.ID), err)
			pWriter.CloseWithError(err)
		} else {
			pWriter.Close()
		}
	}()

	return pReader, nil
}
<commit_msg>Fix failure export image with push command because it already removed container before exporting container<commit_after>package docker

import (
	"fmt"
	"io"

	"github.com/fsouza/go-dockerclient"
	"github.com/hashicorp/errwrap"
)

const exportBufSize = 32768

type Client struct {
	docker dockerclient
}

func NewClient() (*Client, error) {
	client, err := newDockerClient()
	if err != nil {
		return nil, err
	}
	return &Client{
		docker: client,
	}, nil
}

func (c *Client) ExportImage(imageID string) (io.ReadCloser, error) {
	container, err := c.docker.CreateContainer(docker.CreateContainerOptions{
		Config: &docker.Config{
			Image: imageID,
		},
	})
	if err != nil {
		return nil, errwrap.Wrapf(fmt.Sprintf("Failed to create container (imageID:%s): {{err}}", imageID), err)
	}

	pReader, pWriter := io.Pipe()

	go func() {
		defer func() {
			c.docker.RemoveContainer(docker.RemoveContainerOptions{
				ID:    container.ID,
				Force: true,
			})
		}()

		err := c.docker.ExportContainer(docker.ExportContainerOptions{
			ID:           container.ID,
			OutputStream: pWriter,
		})
		if err != nil {
			err = errwrap.Wrapf(fmt.Sprintf("Failed to export container %s: {{err}}", container.ID), err)
			pWriter.CloseWithError(err)
		} else {
			pWriter.Close()
		}
	}()

	return pReader, nil
}
<|endoftext|>


<commit_before>// Copyright 2012 The Gorilla Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

package mux

import (
	"fmt"
	"net/http"
	"path"
	"regexp"

	"github.com/gorilla/context"
)

// NewRouter returns a new router instance.
func NewRouter() *Router {
	return &Router{namedRoutes: make(map[string]*Route), KeepContext: false}
}

// Router registers routes to be matched and dispatches a handler.
//
// It implements the http.Handler interface, so it can be registered to serve
// requests:
//
//     var router = mux.NewRouter()
//
//     func main() {
//         http.Handle("/", router)
//     }
//
// Or, for Google App Engine, register it in a init() function:
//
//     func init() {
//         http.Handle("/", router)
//     }
//
// This will send all incoming requests to the router.
type Router struct {
	// Configurable Handler to be used when no route matches.
	NotFoundHandler http.Handler
	// Parent route, if this is a subrouter.
	parent parentRoute
	// Routes to be matched, in order.
	routes []*Route
	// Routes by name for URL building.
	namedRoutes map[string]*Route
	// See Router.StrictSlash(). This defines the flag for new routes.
	strictSlash bool
	// If true, do not clear the request context after handling the request
	KeepContext bool
}

// Match matches registered routes against the request.
func (r *Router) Match(req *http.Request, match *RouteMatch) bool {
	for _, route := range r.routes {
		if route.Match(req, match) {
			return true
		}
	}
	return false
}

// ServeHTTP dispatches the handler registered in the matched route.
//
// When there is a match, the route variables can be retrieved calling
// mux.Vars(request).
func (r *Router) ServeHTTP(w http.ResponseWriter, req *http.Request) {
	// Clean path to canonical form and redirect.
	if p := cleanPath(req.URL.Path); p != req.URL.Path {

		// Added 3 lines (Philip Schlump) - It was droping the query string and #whatever from query.
		// This matches with fix in go 1.2 r.c. 4 for same problem.  Go Issue:
		// http://code.google.com/p/go/issues/detail?id=5252
		url := *req.URL
		url.Path = p
		p = url.String()

		w.Header().Set("Location", p)
		w.WriteHeader(http.StatusMovedPermanently)
		return
	}
	var match RouteMatch
	var handler http.Handler
	if r.Match(req, &match) {
		handler = match.Handler
		setVars(req, match.Vars)
		setCurrentRoute(req, match.Route)
	}
	if handler == nil {
		handler = r.NotFoundHandler
		if handler == nil {
			handler = http.NotFoundHandler()
		}
	}
	if !r.KeepContext {
		defer context.Clear(req)
	}
	handler.ServeHTTP(w, req)
}

// Get returns a route registered with the given name.
func (r *Router) Get(name string) *Route {
	return r.getNamedRoutes()[name]
}

// GetRoute returns a route registered with the given name. This method
// was renamed to Get() and remains here for backwards compatibility.
func (r *Router) GetRoute(name string) *Route {
	return r.getNamedRoutes()[name]
}

// StrictSlash defines the trailing slash behavior for new routes. The initial
// value is false.
//
// When true, if the route path is "/path/", accessing "/path" will redirect
// to the former and vice versa. In other words, your application will always
// see the path as specified in the route.
//
// When false, if the route path is "/path", accessing "/path/" will not match
// this route and vice versa.
//
// Special case: when a route sets a path prefix using the PathPrefix() method,
// strict slash is ignored for that route because the redirect behavior can't
// be determined from a prefix alone. However, any subrouters created from that
// route inherit the original StrictSlash setting.
func (r *Router) StrictSlash(value bool) *Router {
	r.strictSlash = value
	return r
}

// ----------------------------------------------------------------------------
// parentRoute
// ----------------------------------------------------------------------------

// getNamedRoutes returns the map where named routes are registered.
func (r *Router) getNamedRoutes() map[string]*Route {
	if r.namedRoutes == nil {
		if r.parent != nil {
			r.namedRoutes = r.parent.getNamedRoutes()
		} else {
			r.namedRoutes = make(map[string]*Route)
		}
	}
	return r.namedRoutes
}

// getRegexpGroup returns regexp definitions from the parent route, if any.
func (r *Router) getRegexpGroup() *routeRegexpGroup {
	if r.parent != nil {
		return r.parent.getRegexpGroup()
	}
	return nil
}

func (r *Router) buildVars(m map[string]string) map[string]string {
	if r.parent != nil {
		m = r.parent.buildVars(m)
	}
	return m
}

// ----------------------------------------------------------------------------
// Route factories
// ----------------------------------------------------------------------------

// NewRoute registers an empty route.
func (r *Router) NewRoute() *Route {
	route := &Route{parent: r, strictSlash: r.strictSlash}
	r.routes = append(r.routes, route)
	return route
}

// Handle registers a new route with a matcher for the URL path.
// See Route.Path() and Route.Handler().
func (r *Router) Handle(path string, handler http.Handler) *Route {
	return r.NewRoute().Path(path).Handler(handler)
}

// HandleFunc registers a new route with a matcher for the URL path.
// See Route.Path() and Route.HandlerFunc().
func (r *Router) HandleFunc(path string, f func(http.ResponseWriter,
	*http.Request)) *Route {
	return r.NewRoute().Path(path).HandlerFunc(f)
}

// Headers registers a new route with a matcher for request header values.
// See Route.Headers().
func (r *Router) Headers(pairs ...string) *Route {
	return r.NewRoute().Headers(pairs...)
}

// Host registers a new route with a matcher for the URL host.
// See Route.Host().
func (r *Router) Host(tpl string) *Route {
	return r.NewRoute().Host(tpl)
}

// MatcherFunc registers a new route with a custom matcher function.
// See Route.MatcherFunc().
func (r *Router) MatcherFunc(f MatcherFunc) *Route {
	return r.NewRoute().MatcherFunc(f)
}

// Methods registers a new route with a matcher for HTTP methods.
// See Route.Methods().
func (r *Router) Methods(methods ...string) *Route {
	return r.NewRoute().Methods(methods...)
}

// Path registers a new route with a matcher for the URL path.
// See Route.Path().
func (r *Router) Path(tpl string) *Route {
	return r.NewRoute().Path(tpl)
}

// PathPrefix registers a new route with a matcher for the URL path prefix.
// See Route.PathPrefix().
func (r *Router) PathPrefix(tpl string) *Route {
	return r.NewRoute().PathPrefix(tpl)
}

// Queries registers a new route with a matcher for URL query values.
// See Route.Queries().
func (r *Router) Queries(pairs ...string) *Route {
	return r.NewRoute().Queries(pairs...)
}

// Schemes registers a new route with a matcher for URL schemes.
// See Route.Schemes().
func (r *Router) Schemes(schemes ...string) *Route {
	return r.NewRoute().Schemes(schemes...)
}

// BuildVars registers a new route with a custom function for modifying
// route variables before building a URL.
func (r *Router) BuildVarsFunc(f BuildVarsFunc) *Route {
	return r.NewRoute().BuildVarsFunc(f)
}

// ----------------------------------------------------------------------------
// Context
// ----------------------------------------------------------------------------

// RouteMatch stores information about a matched route.
type RouteMatch struct {
	Route   *Route
	Handler http.Handler
	Vars    map[string]string
}

type contextKey int

const (
	varsKey contextKey = iota
	routeKey
)

// Vars returns the route variables for the current request, if any.
func Vars(r *http.Request) map[string]string {
	if rv := context.Get(r, varsKey); rv != nil {
		return rv.(map[string]string)
	}
	return nil
}

// CurrentRoute returns the matched route for the current request, if any.
func CurrentRoute(r *http.Request) *Route {
	if rv := context.Get(r, routeKey); rv != nil {
		return rv.(*Route)
	}
	return nil
}

func setVars(r *http.Request, val interface{}) {
	context.Set(r, varsKey, val)
}

func setCurrentRoute(r *http.Request, val interface{}) {
	context.Set(r, routeKey, val)
}

// ----------------------------------------------------------------------------
// Helpers
// ----------------------------------------------------------------------------

// cleanPath returns the canonical path for p, eliminating . and .. elements.
// Borrowed from the net/http package.
func cleanPath(p string) string {
	if p == "" {
		return "/"
	}
	if p[0] != '/' {
		p = "/" + p
	}
	np := path.Clean(p)
	// path.Clean removes trailing slash except for root;
	// put the trailing slash back if necessary.
	if p[len(p)-1] == '/' && np != "/" {
		np += "/"
	}
	return np
}

// uniqueVars returns an error if two slices contain duplicated strings.
func uniqueVars(s1, s2 []string) error {
	for _, v1 := range s1 {
		for _, v2 := range s2 {
			if v1 == v2 {
				return fmt.Errorf("mux: duplicated route variable %q", v2)
			}
		}
	}
	return nil
}

func checkPairs(pairs ...string) (int, error) {
	length := len(pairs)
	if length%2 != 0 {
		return length, fmt.Errorf(
			"mux: number of parameters must be multiple of 2, got %v", pairs)
	}
	return length, nil
}

// mapFromPairs converts variadic string parameters to a string map.
func mapFromPairsToString(pairs ...string) (map[string]string, error) {
	length, err := checkPairs(pairs...)
	if err != nil {
		return nil, err
	}
	m := make(map[string]string, length/2)
	for i := 0; i < length; i += 2 {
		m[pairs[i]] = pairs[i+1]
	}
	return m, nil
}

func mapFromPairsToRegex(pairs ...string) (map[string]*regexp.Regexp, error) {
	length, err := checkPairs(pairs...)
	if err != nil {
		return nil, err
	}
	m := make(map[string]*regexp.Regexp, length/2)
	for i := 0; i < length; i += 2 {
		regex, _ := regexp.Compile(pairs[i+1])
		m[pairs[i]] = regex
	}
	return m, nil
}

// matchInArray returns true if the given string value is in the array.
func matchInArray(arr []string, value string) bool {
	for _, v := range arr {
		if v == value {
			return true
		}
	}
	return false
}

// matchMapWithString returns true if the given key/value pairs exist in a given map.
func matchMapWithString(toCheck map[string]string, toMatch map[string][]string, canonicalKey bool) bool {
	for k, v := range toCheck {
		// Check if key exists.
		if canonicalKey {
			k = http.CanonicalHeaderKey(k)
		}
		if values := toMatch[k]; values == nil {
			return false
		} else if v != "" {
			// If value was defined as an empty string we only check that the
			// key exists. Otherwise we also check for equality.
			valueExists := false
			for _, value := range values {
				if v == value {
					valueExists = true
					break
				}
			}
			if !valueExists {
				return false
			}
		}
	}
	return true
}

// matchMapWithRegex returns true if the given key/value pairs exist in a given map compiled against
// the given regex
func matchMapWithRegex(toCheck map[string]*regexp.Regexp, toMatch map[string][]string, canonicalKey bool) bool {
	for k, v := range toCheck {
		// Check if key exists.
		if canonicalKey {
			k = http.CanonicalHeaderKey(k)
		}
		if values := toMatch[k]; values == nil {
			return false
		} else if v != nil {
			// If value was defined as an empty string we only check that the
			// key exists. Otherwise we also check for equality.
			valueExists := false
			for _, value := range values {
				if v.MatchString(value) {
					valueExists = true
					break
				}
			}
			if !valueExists {
				return false
			}
		}
	}
	return true
}
<commit_msg>Issue 16: Return the regexp compile error<commit_after>// Copyright 2012 The Gorilla Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

package mux

import (
	"fmt"
	"net/http"
	"path"
	"regexp"

	"github.com/gorilla/context"
)

// NewRouter returns a new router instance.
func NewRouter() *Router {
	return &Router{namedRoutes: make(map[string]*Route), KeepContext: false}
}

// Router registers routes to be matched and dispatches a handler.
//
// It implements the http.Handler interface, so it can be registered to serve
// requests:
//
//     var router = mux.NewRouter()
//
//     func main() {
//         http.Handle("/", router)
//     }
//
// Or, for Google App Engine, register it in a init() function:
//
//     func init() {
//         http.Handle("/", router)
//     }
//
// This will send all incoming requests to the router.
type Router struct {
	// Configurable Handler to be used when no route matches.
	NotFoundHandler http.Handler
	// Parent route, if this is a subrouter.
	parent parentRoute
	// Routes to be matched, in order.
	routes []*Route
	// Routes by name for URL building.
	namedRoutes map[string]*Route
	// See Router.StrictSlash(). This defines the flag for new routes.
	strictSlash bool
	// If true, do not clear the request context after handling the request
	KeepContext bool
}

// Match matches registered routes against the request.
func (r *Router) Match(req *http.Request, match *RouteMatch) bool {
	for _, route := range r.routes {
		if route.Match(req, match) {
			return true
		}
	}
	return false
}

// ServeHTTP dispatches the handler registered in the matched route.
//
// When there is a match, the route variables can be retrieved calling
// mux.Vars(request).
func (r *Router) ServeHTTP(w http.ResponseWriter, req *http.Request) {
	// Clean path to canonical form and redirect.
	if p := cleanPath(req.URL.Path); p != req.URL.Path {

		// Added 3 lines (Philip Schlump) - It was droping the query string and #whatever from query.
		// This matches with fix in go 1.2 r.c. 4 for same problem.  Go Issue:
		// http://code.google.com/p/go/issues/detail?id=5252
		url := *req.URL
		url.Path = p
		p = url.String()

		w.Header().Set("Location", p)
		w.WriteHeader(http.StatusMovedPermanently)
		return
	}
	var match RouteMatch
	var handler http.Handler
	if r.Match(req, &match) {
		handler = match.Handler
		setVars(req, match.Vars)
		setCurrentRoute(req, match.Route)
	}
	if handler == nil {
		handler = r.NotFoundHandler
		if handler == nil {
			handler = http.NotFoundHandler()
		}
	}
	if !r.KeepContext {
		defer context.Clear(req)
	}
	handler.ServeHTTP(w, req)
}

// Get returns a route registered with the given name.
func (r *Router) Get(name string) *Route {
	return r.getNamedRoutes()[name]
}

// GetRoute returns a route registered with the given name. This method
// was renamed to Get() and remains here for backwards compatibility.
func (r *Router) GetRoute(name string) *Route {
	return r.getNamedRoutes()[name]
}

// StrictSlash defines the trailing slash behavior for new routes. The initial
// value is false.
//
// When true, if the route path is "/path/", accessing "/path" will redirect
// to the former and vice versa. In other words, your application will always
// see the path as specified in the route.
//
// When false, if the route path is "/path", accessing "/path/" will not match
// this route and vice versa.
//
// Special case: when a route sets a path prefix using the PathPrefix() method,
// strict slash is ignored for that route because the redirect behavior can't
// be determined from a prefix alone. However, any subrouters created from that
// route inherit the original StrictSlash setting.
func (r *Router) StrictSlash(value bool) *Router {
	r.strictSlash = value
	return r
}

// ----------------------------------------------------------------------------
// parentRoute
// ----------------------------------------------------------------------------

// getNamedRoutes returns the map where named routes are registered.
func (r *Router) getNamedRoutes() map[string]*Route {
	if r.namedRoutes == nil {
		if r.parent != nil {
			r.namedRoutes = r.parent.getNamedRoutes()
		} else {
			r.namedRoutes = make(map[string]*Route)
		}
	}
	return r.namedRoutes
}

// getRegexpGroup returns regexp definitions from the parent route, if any.
func (r *Router) getRegexpGroup() *routeRegexpGroup {
	if r.parent != nil {
		return r.parent.getRegexpGroup()
	}
	return nil
}

func (r *Router) buildVars(m map[string]string) map[string]string {
	if r.parent != nil {
		m = r.parent.buildVars(m)
	}
	return m
}

// ----------------------------------------------------------------------------
// Route factories
// ----------------------------------------------------------------------------

// NewRoute registers an empty route.
func (r *Router) NewRoute() *Route {
	route := &Route{parent: r, strictSlash: r.strictSlash}
	r.routes = append(r.routes, route)
	return route
}

// Handle registers a new route with a matcher for the URL path.
// See Route.Path() and Route.Handler().
func (r *Router) Handle(path string, handler http.Handler) *Route {
	return r.NewRoute().Path(path).Handler(handler)
}

// HandleFunc registers a new route with a matcher for the URL path.
// See Route.Path() and Route.HandlerFunc().
func (r *Router) HandleFunc(path string, f func(http.ResponseWriter,
	*http.Request)) *Route {
	return r.NewRoute().Path(path).HandlerFunc(f)
}

// Headers registers a new route with a matcher for request header values.
// See Route.Headers().
func (r *Router) Headers(pairs ...string) *Route {
	return r.NewRoute().Headers(pairs...)
}

// Host registers a new route with a matcher for the URL host.
// See Route.Host().
func (r *Router) Host(tpl string) *Route {
	return r.NewRoute().Host(tpl)
}

// MatcherFunc registers a new route with a custom matcher function.
// See Route.MatcherFunc().
func (r *Router) MatcherFunc(f MatcherFunc) *Route {
	return r.NewRoute().MatcherFunc(f)
}

// Methods registers a new route with a matcher for HTTP methods.
// See Route.Methods().
func (r *Router) Methods(methods ...string) *Route {
	return r.NewRoute().Methods(methods...)
}

// Path registers a new route with a matcher for the URL path.
// See Route.Path().
func (r *Router) Path(tpl string) *Route {
	return r.NewRoute().Path(tpl)
}

// PathPrefix registers a new route with a matcher for the URL path prefix.
// See Route.PathPrefix().
func (r *Router) PathPrefix(tpl string) *Route {
	return r.NewRoute().PathPrefix(tpl)
}

// Queries registers a new route with a matcher for URL query values.
// See Route.Queries().
func (r *Router) Queries(pairs ...string) *Route {
	return r.NewRoute().Queries(pairs...)
}

// Schemes registers a new route with a matcher for URL schemes.
// See Route.Schemes().
func (r *Router) Schemes(schemes ...string) *Route {
	return r.NewRoute().Schemes(schemes...)
}

// BuildVars registers a new route with a custom function for modifying
// route variables before building a URL.
func (r *Router) BuildVarsFunc(f BuildVarsFunc) *Route {
	return r.NewRoute().BuildVarsFunc(f)
}

// ----------------------------------------------------------------------------
// Context
// ----------------------------------------------------------------------------

// RouteMatch stores information about a matched route.
type RouteMatch struct {
	Route   *Route
	Handler http.Handler
	Vars    map[string]string
}

type contextKey int

const (
	varsKey contextKey = iota
	routeKey
)

// Vars returns the route variables for the current request, if any.
func Vars(r *http.Request) map[string]string {
	if rv := context.Get(r, varsKey); rv != nil {
		return rv.(map[string]string)
	}
	return nil
}

// CurrentRoute returns the matched route for the current request, if any.
func CurrentRoute(r *http.Request) *Route {
	if rv := context.Get(r, routeKey); rv != nil {
		return rv.(*Route)
	}
	return nil
}

func setVars(r *http.Request, val interface{}) {
	context.Set(r, varsKey, val)
}

func setCurrentRoute(r *http.Request, val interface{}) {
	context.Set(r, routeKey, val)
}

// ----------------------------------------------------------------------------
// Helpers
// ----------------------------------------------------------------------------

// cleanPath returns the canonical path for p, eliminating . and .. elements.
// Borrowed from the net/http package.
func cleanPath(p string) string {
	if p == "" {
		return "/"
	}
	if p[0] != '/' {
		p = "/" + p
	}
	np := path.Clean(p)
	// path.Clean removes trailing slash except for root;
	// put the trailing slash back if necessary.
	if p[len(p)-1] == '/' && np != "/" {
		np += "/"
	}
	return np
}

// uniqueVars returns an error if two slices contain duplicated strings.
func uniqueVars(s1, s2 []string) error {
	for _, v1 := range s1 {
		for _, v2 := range s2 {
			if v1 == v2 {
				return fmt.Errorf("mux: duplicated route variable %q", v2)
			}
		}
	}
	return nil
}

func checkPairs(pairs ...string) (int, error) {
	length := len(pairs)
	if length%2 != 0 {
		return length, fmt.Errorf(
			"mux: number of parameters must be multiple of 2, got %v", pairs)
	}
	return length, nil
}

// mapFromPairs converts variadic string parameters to a string map.
func mapFromPairsToString(pairs ...string) (map[string]string, error) {
	length, err := checkPairs(pairs...)
	if err != nil {
		return nil, err
	}
	m := make(map[string]string, length/2)
	for i := 0; i < length; i += 2 {
		m[pairs[i]] = pairs[i+1]
	}
	return m, nil
}

func mapFromPairsToRegex(pairs ...string) (map[string]*regexp.Regexp, error) {
	length, err := checkPairs(pairs...)
	if err != nil {
		return nil, err
	}
	m := make(map[string]*regexp.Regexp, length/2)
	for i := 0; i < length; i += 2 {
		regex, err := regexp.Compile(pairs[i+1])
		if err != nil {
			return nil, err
		}
		m[pairs[i]] = regex
	}
	return m, nil
}

// matchInArray returns true if the given string value is in the array.
func matchInArray(arr []string, value string) bool {
	for _, v := range arr {
		if v == value {
			return true
		}
	}
	return false
}

// matchMapWithString returns true if the given key/value pairs exist in a given map.
func matchMapWithString(toCheck map[string]string, toMatch map[string][]string, canonicalKey bool) bool {
	for k, v := range toCheck {
		// Check if key exists.
		if canonicalKey {
			k = http.CanonicalHeaderKey(k)
		}
		if values := toMatch[k]; values == nil {
			return false
		} else if v != "" {
			// If value was defined as an empty string we only check that the
			// key exists. Otherwise we also check for equality.
			valueExists := false
			for _, value := range values {
				if v == value {
					valueExists = true
					break
				}
			}
			if !valueExists {
				return false
			}
		}
	}
	return true
}

// matchMapWithRegex returns true if the given key/value pairs exist in a given map compiled against
// the given regex
func matchMapWithRegex(toCheck map[string]*regexp.Regexp, toMatch map[string][]string, canonicalKey bool) bool {
	for k, v := range toCheck {
		// Check if key exists.
		if canonicalKey {
			k = http.CanonicalHeaderKey(k)
		}
		if values := toMatch[k]; values == nil {
			return false
		} else if v != nil {
			// If value was defined as an empty string we only check that the
			// key exists. Otherwise we also check for equality.
			valueExists := false
			for _, value := range values {
				if v.MatchString(value) {
					valueExists = true
					break
				}
			}
			if !valueExists {
				return false
			}
		}
	}
	return true
}
<|endoftext|>


<commit_before>// Ogo

package ogo

/* {{{ import
 */
import (
	"fmt"
	"os"
	"path/filepath"
	"strings"
	"sync"
	"time"

	"github.com/Odinman/ogo/libs/config"
	"github.com/Odinman/ogo/libs/logs"
	"github.com/Odinman/ogo/utils"
)

/* }}} */

/* {{{ type Environ struct
 */
type Environ struct {
	lock          *sync.RWMutex
	WorkPath      string         // working path(abs)
	AppPath       string         // application path
	ProcName      string         // proc name
	Worker        string         // worker name
	AppConfigPath string         // config file path
	RunMode       string         // run mode, "dev" or "prod"
	Daemonize     bool           // daemonize or not
	EnableGzip    bool           // enable gzip or not
	DebugLevel    int            // debug level
	PidFile       string         // pidfile abs path
	Port          string         // http port
	IndentJSON    bool           // indent JSON
	MaxMemory     int64          //max memory(form-data)
	Location      *time.Location // location
	initErr       error
}

/* }}} */

/* {{{ type Mux struct
 */
type Mux struct {
	env      *Environ               //环境参数
	cfg      config.ConfigContainer //配置信息
	logger   *logs.OLogger          //debug日志记录
	accessor *logs.OLogger          //日志
	Workers  map[string]*Worker
	Routes   map[string]*Route
	Hooks    HStack
}

/* }}} */

/* {{{ func New() *Mux
 */
func New() *Mux {
	return &Mux{
		Workers: make(map[string]*Worker),
		Routes:  make(map[string]*Route),
		Hooks: HStack{
			preHooks:  make([]OgoHook, 0),
			postHooks: make([]OgoHook, 0),
		},
	}
}

/* }}} */

/* {{{ func (mux *Mux) PreHook(hook OgoHook)
 * 正式程序之前的钩子
 */
func (mux *Mux) PreHook(hook OgoHook) {
	mux.Hooks.preHooks = append(mux.Hooks.preHooks, hook)
}

/* }}} */

/* {{{ func (mux *Mux) PostHook(hook OgoHook)
 * 正式程序之后的钩子
 */
func (mux *Mux) PostHook(hook OgoHook) {
	mux.Hooks.postHooks = append(mux.Hooks.postHooks, hook)
}

/* }}} */

/* {{{ func (mux *Mux) NewController(c ControllerInterface, endpoint string)
 * 这样做的目的是给controller设置mux(mux可多个) -- mux=multiplexer,复用器
 */
func (mux *Mux) NewController(c ControllerInterface, endpoint string) ControllerInterface {
	c.New(mux, endpoint)
	return c
}

/* }}} */

/* {{{ func (mux *Mux) Env(path string) *Environ, error
 * 获取env变量
 */
func (mux *Mux) Env() (*Environ, error) {

	if mux.env == nil {
		env = &Environ{
			lock: new(sync.RWMutex),
		}

		// default value
		env.RunMode = "dev"     //default runmod
		env.Port = DEFAULT_PORT //default is 8001
		env.Daemonize = false
		env.DebugLevel = logs.LevelTrace //默认debug等级
		env.IndentJSON = false
		env.MaxMemory = 1 << 26                              //64MB
		env.Location, _ = time.LoadLocation("Asia/Shanghai") //默认上海时区

		workPath, _ := os.Getwd()
		env.WorkPath, _ = filepath.Abs(workPath)
		env.AppPath, _ = filepath.Abs(filepath.Dir(os.Args[0]))
		env.ProcName = filepath.Base(os.Args[0])   //程序名字
		env.Worker = strings.ToLower(env.ProcName) //worker默认为procname,小写

		//默认配置文件是 conf/{ProcName}.conf
		env.AppConfigPath = filepath.Join(env.AppPath, "conf", env.ProcName+".conf")
		if !utils.FileExists(env.AppConfigPath) {
			//不存在时指定为app.conf
			tp := filepath.Join(env.AppPath, "conf", "app.conf")
			if utils.FileExists(tp) {
				env.AppConfigPath = tp
			}
		}

		// pidfile depend AppPath
		env.PidFile = filepath.Join(env.AppPath, "run", env.ProcName+".pid")

		if env.WorkPath != env.AppPath {
			//切换工作目录
			os.Chdir(env.AppPath)
		}

		//配置文件必须存在
		if !utils.FileExists(env.AppConfigPath) {
			return nil, fmt.Errorf("config file not found: %s", env.AppConfigPath)
		}

		mux.env = env
	}

	return mux.env, nil
}

/* }}} */

/* {{{ func (mux *Mux) initEnv(cfg config.ConfigContainer) error
 * 设置环境变量
 */
func (mux *Mux) initEnv() (err error) {
	if env, err = mux.Env(); err != nil {
		return err
	}
	if cfg, err = mux.Config(); err != nil {
		return err
	}

	//根据配置设置环境变量
	env.lock.Lock()
	defer env.lock.Unlock()

	// location
	if tz := cfg.String("TimeZone"); tz != "" {
		if loc, err := time.LoadLocation(tz); err == nil {
			env.Location = loc
		}
	}

	if port := cfg.String("Port"); port != "" {
		env.Port = port
	}
	os.Setenv("PORT", env.Port) // pass to bind

	if runmode := cfg.String("RunMode"); runmode != "" {
		env.RunMode = runmode
	}

	if workerName := cfg.String("Worker"); workerName != "" {
		env.Worker = workerName
	}

	if daemonize, err := cfg.Bool("Daemonize"); err == nil {
		env.Daemonize = daemonize
	}
	if enablegzip, err := cfg.Bool("EnableGzip"); err == nil {
		env.EnableGzip = enablegzip
	}
	if indentJson, err := cfg.Bool("IndentJson"); err == nil {
		env.IndentJSON = indentJson
	}
	// 自定义pidfile
	if pidfile := cfg.String("PidFile"); pidfile != "" {
		// make sure pidfile is abs path
		if filepath.IsAbs(pidfile) {
			env.PidFile = pidfile
		} else {
			env.PidFile = filepath.Join(env.AppPath, pidfile)
		}
	}
	if level, err := cfg.Int("DebugLevel"); err == nil {
		env.DebugLevel = level
	}

	// logger init
	if logger, err = mux.Logger(); err != nil {
		return err
	}

	//access init
	if accessor, err = mux.Accessor(); err != nil {
		return err
	}

	return nil
}

/* }}} */

/* {{{ func (mux *Mux) Config() (config.ConfigContainer, error)
 * 获取配置信息
 */
func (mux *Mux) Config() (config.ConfigContainer, error) {
	if mux.cfg == nil {
		if env, err := mux.Env(); err != nil {
			return nil, err
		} else {
			cp := env.AppConfigPath
			if cfg, err := config.NewConfig("ini", cp); err != nil {
				return nil, err
			} else {
				mux.cfg = cfg
			}
		}
	}
	return mux.cfg, nil
}

/* }}} */

/* {{{ func (mux *Mux) Logger() config.LoggerContainer
 *
 */
func (mux *Mux) Logger() (*logs.OLogger, error) {
	if mux.logger == nil {
		// init logger
		logger := logs.NewLogger(2046)
		var err error
		if mux.env.Daemonize {
			err = logger.SetLogger("file", `{"filename":"logs/debug.log"}`)
		} else {
			err = logger.SetLogger("console", "")
		}

		if err != nil {
			return nil, err
		}

		logger.EnableFuncCallDepth(true)
		logger.SetLogFuncCallDepth(4)
		logger.SetLevel(mux.env.DebugLevel)
		mux.logger = logger
	}

	return mux.logger, nil
}

/* }}} */

/* {{{ func (mux *Mux) Accessor() config.LoggerContainer
 *
 */
func (mux *Mux) Accessor() (*logs.OLogger, error) {
	if mux.accessor == nil {
		// init logger
		logger := logs.NewLogger(2046)
		var err error
		err = logger.SetLogger("access", `{"filename":"logs/access.log"}`)

		if err != nil {
			return nil, err
		}

		mux.accessor = logger
	}

	return mux.accessor, nil
}

/* }}} */
<commit_msg>access日志位置可配置<commit_after>// Ogo

package ogo

/* {{{ import
 */
import (
	"fmt"
	"os"
	"path/filepath"
	"strings"
	"sync"
	"time"

	"github.com/Odinman/ogo/libs/config"
	"github.com/Odinman/ogo/libs/logs"
	"github.com/Odinman/ogo/utils"
)

/* }}} */

/* {{{ type Environ struct
 */
type Environ struct {
	lock          *sync.RWMutex
	WorkPath      string         // working path(abs)
	AppPath       string         // application path
	ProcName      string         // proc name
	Worker        string         // worker name
	AppConfigPath string         // config file path
	RunMode       string         // run mode, "dev" or "prod"
	AccessPath    string         //acces log file path
	Daemonize     bool           // daemonize or not
	EnableGzip    bool           // enable gzip or not
	DebugLevel    int            // debug level
	PidFile       string         // pidfile abs path
	Port          string         // http port
	IndentJSON    bool           // indent JSON
	MaxMemory     int64          //max memory(form-data)
	Location      *time.Location // location
	initErr       error
}

/* }}} */

/* {{{ type Mux struct
 */
type Mux struct {
	env      *Environ               //环境参数
	cfg      config.ConfigContainer //配置信息
	logger   *logs.OLogger          //debug日志记录
	accessor *logs.OLogger          //日志
	Workers  map[string]*Worker
	Routes   map[string]*Route
	Hooks    HStack
}

/* }}} */

/* {{{ func New() *Mux
 */
func New() *Mux {
	return &Mux{
		Workers: make(map[string]*Worker),
		Routes:  make(map[string]*Route),
		Hooks: HStack{
			preHooks:  make([]OgoHook, 0),
			postHooks: make([]OgoHook, 0),
		},
	}
}

/* }}} */

/* {{{ func (mux *Mux) PreHook(hook OgoHook)
 * 正式程序之前的钩子
 */
func (mux *Mux) PreHook(hook OgoHook) {
	mux.Hooks.preHooks = append(mux.Hooks.preHooks, hook)
}

/* }}} */

/* {{{ func (mux *Mux) PostHook(hook OgoHook)
 * 正式程序之后的钩子
 */
func (mux *Mux) PostHook(hook OgoHook) {
	mux.Hooks.postHooks = append(mux.Hooks.postHooks, hook)
}

/* }}} */

/* {{{ func (mux *Mux) NewController(c ControllerInterface, endpoint string)
 * 这样做的目的是给controller设置mux(mux可多个) -- mux=multiplexer,复用器
 */
func (mux *Mux) NewController(c ControllerInterface, endpoint string) ControllerInterface {
	c.New(mux, endpoint)
	return c
}

/* }}} */

/* {{{ func (mux *Mux) Env(path string) *Environ, error
 * 获取env变量
 */
func (mux *Mux) Env() (*Environ, error) {

	if mux.env == nil {
		env = &Environ{
			lock: new(sync.RWMutex),
		}

		// default value
		env.RunMode = "dev"     //default runmod
		env.Port = DEFAULT_PORT //default is 8001
		env.Daemonize = false
		env.DebugLevel = logs.LevelTrace //默认debug等级
		env.IndentJSON = false
		env.MaxMemory = 1 << 26                              //64MB
		env.Location, _ = time.LoadLocation("Asia/Shanghai") //默认上海时区

		workPath, _ := os.Getwd()
		env.WorkPath, _ = filepath.Abs(workPath)
		env.AppPath, _ = filepath.Abs(filepath.Dir(os.Args[0]))
		env.ProcName = filepath.Base(os.Args[0])   //程序名字
		env.Worker = strings.ToLower(env.ProcName) //worker默认为procname,小写
		env.AccessPath = "logs/access.log"         //默认access日志是程序目录下的logs/access.log

		//默认配置文件是 conf/{ProcName}.conf
		env.AppConfigPath = filepath.Join(env.AppPath, "conf", env.ProcName+".conf")
		if !utils.FileExists(env.AppConfigPath) {
			//不存在时指定为app.conf
			tp := filepath.Join(env.AppPath, "conf", "app.conf")
			if utils.FileExists(tp) {
				env.AppConfigPath = tp
			}
		}

		// pidfile depend AppPath
		env.PidFile = filepath.Join(env.AppPath, "run", env.ProcName+".pid")

		if env.WorkPath != env.AppPath {
			//切换工作目录
			os.Chdir(env.AppPath)
		}

		//配置文件必须存在
		if !utils.FileExists(env.AppConfigPath) {
			return nil, fmt.Errorf("config file not found: %s", env.AppConfigPath)
		}

		mux.env = env
	}

	return mux.env, nil
}

/* }}} */

/* {{{ func (mux *Mux) initEnv(cfg config.ConfigContainer) error
 * 设置环境变量
 */
func (mux *Mux) initEnv() (err error) {
	if env, err = mux.Env(); err != nil {
		return err
	}
	if cfg, err = mux.Config(); err != nil {
		return err
	}

	//根据配置设置环境变量
	env.lock.Lock()
	defer env.lock.Unlock()

	// location
	if tz := cfg.String("TimeZone"); tz != "" {
		if loc, err := time.LoadLocation(tz); err == nil {
			env.Location = loc
		}
	}

	if port := cfg.String("Port"); port != "" {
		env.Port = port
	}
	os.Setenv("PORT", env.Port) // pass to bind

	if runmode := cfg.String("RunMode"); runmode != "" {
		env.RunMode = runmode
	}

	if workerName := cfg.String("Worker"); workerName != "" {
		env.Worker = workerName
	}

	if daemonize, err := cfg.Bool("Daemonize"); err == nil {
		env.Daemonize = daemonize
	}
	if enablegzip, err := cfg.Bool("EnableGzip"); err == nil {
		env.EnableGzip = enablegzip
	}
	if indentJson, err := cfg.Bool("IndentJson"); err == nil {
		env.IndentJSON = indentJson
	}
	// 自定义pidfile
	if pidfile := cfg.String("PidFile"); pidfile != "" {
		// make sure pidfile is abs path
		if filepath.IsAbs(pidfile) {
			env.PidFile = pidfile
		} else {
			env.PidFile = filepath.Join(env.AppPath, pidfile)
		}
	}
	//自定义access 日志位置
	if apath := cfg.String("AccessPath"); apath != "" {
		env.AccessPath = apath
	}
	if level, err := cfg.Int("DebugLevel"); err == nil {
		env.DebugLevel = level
	}

	// logger init
	if logger, err = mux.Logger(); err != nil {
		return err
	}

	//access init
	if accessor, err = mux.Accessor(); err != nil {
		return err
	}

	return nil
}

/* }}} */

/* {{{ func (mux *Mux) Config() (config.ConfigContainer, error)
 * 获取配置信息
 */
func (mux *Mux) Config() (config.ConfigContainer, error) {
	if mux.cfg == nil {
		if env, err := mux.Env(); err != nil {
			return nil, err
		} else {
			cp := env.AppConfigPath
			if cfg, err := config.NewConfig("ini", cp); err != nil {
				return nil, err
			} else {
				mux.cfg = cfg
			}
		}
	}
	return mux.cfg, nil
}

/* }}} */

/* {{{ func (mux *Mux) Logger() config.LoggerContainer
 *
 */
func (mux *Mux) Logger() (*logs.OLogger, error) {
	if mux.logger == nil {
		// init logger
		logger := logs.NewLogger(2046)
		var err error
		if mux.env.Daemonize {
			err = logger.SetLogger("file", `{"filename":"logs/debug.log"}`)
		} else {
			err = logger.SetLogger("console", "")
		}

		if err != nil {
			return nil, err
		}

		logger.EnableFuncCallDepth(true)
		logger.SetLogFuncCallDepth(4)
		logger.SetLevel(mux.env.DebugLevel)
		mux.logger = logger
	}

	return mux.logger, nil
}

/* }}} */

/* {{{ func (mux *Mux) Accessor() config.LoggerContainer
 *
 */
func (mux *Mux) Accessor() (*logs.OLogger, error) {
	if mux.accessor == nil {
		// init logger
		logger := logs.NewLogger(2046)
		var err error
		err = logger.SetLogger("access", fmt.Sprintf(`{"filename":"%s"}`, mux.env.AccessPath))

		if err != nil {
			return nil, err
		}

		mux.accessor = logger
	}

	return mux.accessor, nil
}

/* }}} */
<|endoftext|>


<commit_before>package sugar

import (
	"bytes"
	"context"
	"database/sql"
	"errors"
	"fmt"
	"reflect"
	"strings"
)

const (
	// Space is a single space.
	Space = " "
	// FieldSep is a single field separator.
	FieldSep = ", "
)

var (
	// ErrNoRecord means that the record was not found.
	ErrNoRecord = errors.New("no record found")

	errEmptyQuery = errors.New("query is empty")
)

// Executor is an interface for an opaque query executor.
type Executor interface {
	ExecContext(context.Context, string, ...interface{}) (sql.Result, error)
	QueryContext(context.Context, string, ...interface{}) (*sql.Rows, error)
}

// DeferFunc runs when the query is finished.
type DeferFunc func(*Querier)

// ScanFunc is called for each row in the result set.
type ScanFunc func(*Querier, *sql.Rows) error

// Formatter formats a query or a part of a query.
type Formatter func(*Querier, int) string

// Querier can build and execute queries.
type Querier struct {
	ex Executor

	// Formatters
	bindVar Formatter

	// Query builder
	query    bytes.Buffer
	sep      string
	preWrite string
	params   []interface{}

	// For deffered functions.
	err          error
	lastInsertID int64
	rowsAffected int64
	deferred     []DeferFunc
}

func NewQuerier(ex Executor, bindVar Formatter) *Querier {
	return &Querier{ex: ex, bindVar: bindVar, sep: Space}
}

// Write writes a string (query) to the Querier. A single space is appended after query.
func (q *Querier) Write(query string, params ...interface{}) *Querier {
	q.writeSep()
	q.query.WriteString(query)
	q.params = append(q.params, params...)
	return q
}

// Writef writes a formatted string (format) to the Querier. A single space is appended after query.
func (q *Querier) Writef(format string, args ...interface{}) *Querier {
	q.writeSep()
	q.query.WriteString(fmt.Sprintf(format, args...))
	return q
}

func (q *Querier) WriteFields(format, sep string, fields ...Field) *Querier {
	q.writeSep()
	q.writeFormat(format, sep, fields, len(fields))
	return q
}

func (q *Querier) WriteValues(format, sep string, values ...interface{}) *Querier {
	q.writeSep()
	q.writeFormat(format, sep, nil, len(values))
	q.params = append(q.params, values...)
	return q
}

func (q *Querier) WriteValueMap(format, sep string, valueMap ValueMap, fields ...Field) *Querier {
	q.writeSep()
	q.writeFormat(format, sep, fields, len(fields))
	q.params = append(q.params, valueMap.MapToFields(fields, nil)...)
	return q
}

func (q *Querier) WriteRaw(s string) *Querier {
	q.query.WriteString(s)
	return q
}

func (q *Querier) PreWrite() *Querier {
	if q.preWrite != "" {
		q.writeSep()
		q.query.WriteString(q.preWrite)
	}
	return q
}

func (q *Querier) SetPreWrite(s string) *Querier {
	q.preWrite = s
	return q
}

func (q *Querier) SetSeparator(sep string) *Querier {
	q.sep = sep
	return q
}

func (q *Querier) AddParams(params ...interface{}) *Querier {
	q.params = append(q.params, params...)
	return q
}

func (q *Querier) Params() []interface{} {
	return q.params
}

func (q *Querier) String() string {
	return q.query.String()
}

func (q *Querier) Defer(fn DeferFunc) *Querier {
	q.deferred = append(q.deferred, fn)
	return q
}

func (q *Querier) ExecContext(ctx context.Context) error {
	if q.query.Len() == 0 {
		panic(errEmptyQuery)
	}
	defer q.runDeferred()

	result, err := q.ex.ExecContext(ctx, q.query.String(), q.params...)
	if err != nil {
		return q.returnErr(err)
	}
	if q.rowsAffected, err = result.RowsAffected(); err != nil {
		return q.returnErr(err)
	}
	q.lastInsertID, err = result.LastInsertId()

	return q.returnErr(err)
}

func (q *Querier) Exec() error {
	return q.ExecContext(context.Background())
}

func (q *Querier) FirstContext(ctx context.Context, i interface{}) error {
	if q.query.Len() == 0 {
		panic(errEmptyQuery)
	}
	valueMap := Values(i)
	defer q.runDeferred()

	rows, err := q.ex.QueryContext(ctx, q.query.String(), q.params...)
	if err != nil {
		return q.returnErr(err)
	}
	defer rows.Close()

	if !rows.Next() {
		return q.returnErr(ErrNoRecord)
	}
	columns, err := rows.Columns()
	if err != nil {
		return q.returnErr(err)
	}
	err = rows.Scan(valueMap.MapToColumns(columns, nil))
	return q.returnErr(err)
}

func (q *Querier) First(i interface{}) error {
	return q.FirstContext(context.Background(), i)
}

func (q *Querier) FindContext(ctx context.Context, i interface{}) error {
	if q.query.Len() == 0 {
		panic(errEmptyQuery)
	}
	v, elemType, elemIsPtr := extractStructSliceInfo(i)
	defer q.runDeferred()

	rows, err := q.ex.QueryContext(ctx, q.query.String(), q.params...)
	if err != nil {
		return q.returnErr(err)
	}
	defer rows.Close()
	columns, err := rows.Columns()
	if err != nil {
		return q.returnErr(err)
	}

	var fields []interface{}
	var valueMap ValueMap
	for rows.Next() {
		element := reflect.New(elemType).Elem()
		valueMap = makeValueMap(element, valueMap)
		fields = valueMap.MapToColumns(columns, fields)
		err = rows.Scan(fields...)
		if err != nil {
			return q.returnErr(err)
		}
		if elemIsPtr {
			element = element.Addr()
		}
		v.Set(reflect.Append(v, element))
		fields = fields[:0]
	}

	return nil
}

func (q *Querier) Find(i interface{}) error {
	return q.FindContext(context.Background(), i)
}

func (q *Querier) ScanContext(ctx context.Context, dest ...interface{}) error {
	if q.query.Len() == 0 {
		panic(errEmptyQuery)
	}
	defer q.runDeferred()

	rows, err := q.ex.QueryContext(ctx, q.query.String(), q.params...)
	if err != nil {
		return q.returnErr(err)
	}
	defer rows.Close()

	if !rows.Next() {
		return q.returnErr(ErrNoRecord)
	}
	err = rows.Scan(dest...)
	return q.returnErr(err)
}

func (q *Querier) Scan(dest ...interface{}) error {
	return q.ScanContext(context.Background(), dest...)
}

func (q *Querier) ForEachContext(ctx context.Context, fn ScanFunc) error {
	if q.query.Len() == 0 {
		panic(errEmptyQuery)
	}
	defer q.runDeferred()

	rows, err := q.ex.QueryContext(ctx, q.query.String(), q.params...)
	if err != nil {
		return q.returnErr(err)
	}
	defer rows.Close()

	for rows.Next() {
		if err = fn(q, rows); err != nil {
			return q.returnErr(err)
		}
	}

	return nil
}

func (q *Querier) ForEach(fn ScanFunc) error {
	return q.ForEachContext(context.Background(), fn)
}

func (q *Querier) RowsAffected() int64 {
	return q.rowsAffected
}

func (q *Querier) LastInsertID() int64 {
	return q.lastInsertID
}

func (q *Querier) New() *Querier {
	return NewQuerier(q.ex, q.bindVar)
}

func (q *Querier) Reset() *Querier {
	q.query.Reset()
	if q.params != nil {
		q.params = q.params[:0]
	}
	q.sep = Space
	q.err = nil
	q.lastInsertID, q.rowsAffected = 0, 0
	if q.deferred != nil {
		q.deferred = q.deferred[:0]
	}
	return q
}

func (q *Querier) writeSep() {
	if q.query.Len() > 0 {
		q.query.WriteString(q.sep)
	}
}

func (q *Querier) returnErr(err error) error {
	q.err = err
	return err
}

func (q *Querier) runDeferred() {
	for _, fn := range q.deferred {
		fn(q)
	}
}

const (
	phName     = "{name}"
	phDataType = "{dataType}"
	phBindVar  = "{bindVar}"
)

func (q *Querier) writeFormat(format, sep string, fields []Field, n int) {
	if n < 1 {
		return
	}

	var (
		hasName     = strings.Contains(format, phName)
		hasDataType = strings.Contains(format, phDataType)
		hasBindVar  = strings.Contains(format, phBindVar)
	)

	if fields == nil && (hasName || hasDataType) {
		panic("format contains placeholder {name} or {dataType}, this is not allowed when only formatting values")
	}

	fmtr := func(i int, f *Field) {
		part := format
		if hasName {
			part = strings.Replace(part, phName, f.Name, -1)
		}
		if hasDataType {
			part = strings.Replace(part, phDataType, f.DataType, -1)
		}
		if hasBindVar {
			part = strings.Replace(part, phBindVar, q.bindVar(q, i), -1)
		}
		q.query.WriteString(part)
	}

	fmtr(0, &fields[0])
	for i := 1; i < n; i++ {
		q.query.WriteString(sep)
		fmtr(i, &fields[i])
	}
}
<commit_msg>querier: Add DeferSuccess function<commit_after>package sugar

import (
	"bytes"
	"context"
	"database/sql"
	"errors"
	"fmt"
	"reflect"
	"strings"
)

const (
	// Space is a single space.
	Space = " "
	// FieldSep is a single field separator.
	FieldSep = ", "
)

var (
	// ErrNoRecord means that the record was not found.
	ErrNoRecord = errors.New("no record found")

	errEmptyQuery = errors.New("query is empty")
)

// Executor is an interface for an opaque query executor.
type Executor interface {
	ExecContext(context.Context, string, ...interface{}) (sql.Result, error)
	QueryContext(context.Context, string, ...interface{}) (*sql.Rows, error)
}

// DeferFunc runs when the query is finished.
type DeferFunc func(*Querier)

// ScanFunc is called for each row in the result set.
type ScanFunc func(*Querier, *sql.Rows) error

// Formatter formats a query or a part of a query.
type Formatter func(*Querier, int) string

// Querier can build and execute queries.
type Querier struct {
	ex Executor

	// Formatters
	bindVar Formatter

	// Query builder
	query    bytes.Buffer
	sep      string
	preWrite string
	params   []interface{}

	// For deffered functions.
	err          error
	lastInsertID int64
	rowsAffected int64
	deferred     []DeferFunc
}

func NewQuerier(ex Executor, bindVar Formatter) *Querier {
	return &Querier{ex: ex, bindVar: bindVar, sep: Space}
}

// Write writes a string (query) to the Querier. A single space is appended after query.
func (q *Querier) Write(query string, params ...interface{}) *Querier {
	q.writeSep()
	q.query.WriteString(query)
	q.params = append(q.params, params...)
	return q
}

// Writef writes a formatted string (format) to the Querier. A single space is appended after query.
func (q *Querier) Writef(format string, args ...interface{}) *Querier {
	q.writeSep()
	q.query.WriteString(fmt.Sprintf(format, args...))
	return q
}

func (q *Querier) WriteFields(format, sep string, fields ...Field) *Querier {
	q.writeSep()
	q.writeFormat(format, sep, fields, len(fields))
	return q
}

func (q *Querier) WriteValues(format, sep string, values ...interface{}) *Querier {
	q.writeSep()
	q.writeFormat(format, sep, nil, len(values))
	q.params = append(q.params, values...)
	return q
}

func (q *Querier) WriteValueMap(format, sep string, valueMap ValueMap, fields ...Field) *Querier {
	q.writeSep()
	q.writeFormat(format, sep, fields, len(fields))
	q.params = append(q.params, valueMap.MapToFields(fields, nil)...)
	return q
}

func (q *Querier) WriteRaw(s string) *Querier {
	q.query.WriteString(s)
	return q
}

func (q *Querier) PreWrite() *Querier {
	if q.preWrite != "" {
		q.writeSep()
		q.query.WriteString(q.preWrite)
	}
	return q
}

func (q *Querier) SetPreWrite(s string) *Querier {
	q.preWrite = s
	return q
}

func (q *Querier) SetSeparator(sep string) *Querier {
	q.sep = sep
	return q
}

func (q *Querier) AddParams(params ...interface{}) *Querier {
	q.params = append(q.params, params...)
	return q
}

func (q *Querier) Params() []interface{} {
	return q.params
}

func (q *Querier) String() string {
	return q.query.String()
}

func (q *Querier) Defer(fn DeferFunc) *Querier {
	q.deferred = append(q.deferred, fn)
	return q
}

func (q *Querier) DeferSuccess(fn DeferFunc) *Querier {
	q.deferred = append(q.deferred, func(q *Querier) {
		if q.err == nil {
			fn(q)
		}
	})
	return q
}

func (q *Querier) ExecContext(ctx context.Context) error {
	if q.query.Len() == 0 {
		panic(errEmptyQuery)
	}
	defer q.runDeferred()

	result, err := q.ex.ExecContext(ctx, q.query.String(), q.params...)
	if err != nil {
		return q.returnErr(err)
	}
	if q.rowsAffected, err = result.RowsAffected(); err != nil {
		return q.returnErr(err)
	}
	q.lastInsertID, err = result.LastInsertId()

	return q.returnErr(err)
}

func (q *Querier) Exec() error {
	return q.ExecContext(context.Background())
}

func (q *Querier) FirstContext(ctx context.Context, i interface{}) error {
	if q.query.Len() == 0 {
		panic(errEmptyQuery)
	}
	valueMap := Values(i)
	defer q.runDeferred()

	rows, err := q.ex.QueryContext(ctx, q.query.String(), q.params...)
	if err != nil {
		return q.returnErr(err)
	}
	defer rows.Close()

	if !rows.Next() {
		return q.returnErr(ErrNoRecord)
	}
	columns, err := rows.Columns()
	if err != nil {
		return q.returnErr(err)
	}
	err = rows.Scan(valueMap.MapToColumns(columns, nil))
	return q.returnErr(err)
}

func (q *Querier) First(i interface{}) error {
	return q.FirstContext(context.Background(), i)
}

func (q *Querier) FindContext(ctx context.Context, i interface{}) error {
	if q.query.Len() == 0 {
		panic(errEmptyQuery)
	}
	v, elemType, elemIsPtr := extractStructSliceInfo(i)
	defer q.runDeferred()

	rows, err := q.ex.QueryContext(ctx, q.query.String(), q.params...)
	if err != nil {
		return q.returnErr(err)
	}
	defer rows.Close()
	columns, err := rows.Columns()
	if err != nil {
		return q.returnErr(err)
	}

	var fields []interface{}
	var valueMap ValueMap
	for rows.Next() {
		element := reflect.New(elemType).Elem()
		valueMap = makeValueMap(element, valueMap)
		fields = valueMap.MapToColumns(columns, fields)
		err = rows.Scan(fields...)
		if err != nil {
			return q.returnErr(err)
		}
		if elemIsPtr {
			element = element.Addr()
		}
		v.Set(reflect.Append(v, element))
		fields = fields[:0]
	}

	return nil
}

func (q *Querier) Find(i interface{}) error {
	return q.FindContext(context.Background(), i)
}

func (q *Querier) ScanContext(ctx context.Context, dest ...interface{}) error {
	if q.query.Len() == 0 {
		panic(errEmptyQuery)
	}
	defer q.runDeferred()

	rows, err := q.ex.QueryContext(ctx, q.query.String(), q.params...)
	if err != nil {
		return q.returnErr(err)
	}
	defer rows.Close()

	if !rows.Next() {
		return q.returnErr(ErrNoRecord)
	}
	err = rows.Scan(dest...)
	return q.returnErr(err)
}

func (q *Querier) Scan(dest ...interface{}) error {
	return q.ScanContext(context.Background(), dest...)
}

func (q *Querier) ForEachContext(ctx context.Context, fn ScanFunc) error {
	if q.query.Len() == 0 {
		panic(errEmptyQuery)
	}
	defer q.runDeferred()

	rows, err := q.ex.QueryContext(ctx, q.query.String(), q.params...)
	if err != nil {
		return q.returnErr(err)
	}
	defer rows.Close()

	for rows.Next() {
		if err = fn(q, rows); err != nil {
			return q.returnErr(err)
		}
	}

	return nil
}

func (q *Querier) ForEach(fn ScanFunc) error {
	return q.ForEachContext(context.Background(), fn)
}

func (q *Querier) RowsAffected() int64 {
	return q.rowsAffected
}

func (q *Querier) LastInsertID() int64 {
	return q.lastInsertID
}

func (q *Querier) Error() error {
	return q.err
}

func (q *Querier) New() *Querier {
	return NewQuerier(q.ex, q.bindVar)
}

func (q *Querier) Reset() *Querier {
	q.query.Reset()
	if q.params != nil {
		q.params = q.params[:0]
	}
	q.sep = Space
	q.err = nil
	q.lastInsertID, q.rowsAffected = 0, 0
	if q.deferred != nil {
		q.deferred = q.deferred[:0]
	}
	return q
}

func (q *Querier) writeSep() {
	if q.query.Len() > 0 {
		q.query.WriteString(q.sep)
	}
}

func (q *Querier) returnErr(err error) error {
	q.err = err
	return err
}

func (q *Querier) runDeferred() {
	for _, fn := range q.deferred {
		fn(q)
	}
}

const (
	phName     = "{name}"
	phDataType = "{dataType}"
	phBindVar  = "{bindVar}"
)

func (q *Querier) writeFormat(format, sep string, fields []Field, n int) {
	if n < 1 {
		return
	}

	var (
		hasName     = strings.Contains(format, phName)
		hasDataType = strings.Contains(format, phDataType)
		hasBindVar  = strings.Contains(format, phBindVar)
	)

	if fields == nil && (hasName || hasDataType) {
		panic("format contains placeholder {name} or {dataType}, this is not allowed when only formatting values")
	}

	fmtr := func(i int, f *Field) {
		part := format
		if hasName {
			part = strings.Replace(part, phName, f.Name, -1)
		}
		if hasDataType {
			part = strings.Replace(part, phDataType, f.DataType, -1)
		}
		if hasBindVar {
			part = strings.Replace(part, phBindVar, q.bindVar(q, i), -1)
		}
		q.query.WriteString(part)
	}

	fmtr(0, &fields[0])
	for i := 1; i < n; i++ {
		q.query.WriteString(sep)
		fmtr(i, &fields[i])
	}
}
<|endoftext|>


<commit_before>package protocol

import (
	"context"
	"encoding/json"
	"fmt"
	"io"
	"strings"
	"sync"
	"time"

	"golang.org/x/tools/internal/jsonrpc2"
)

type loggingStream struct {
	stream jsonrpc2.Stream
	log    io.Writer
}

// LoggingStream returns a stream that does LSP protocol logging too
func LoggingStream(str jsonrpc2.Stream, w io.Writer) jsonrpc2.Stream {
	return &loggingStream{str, w}
}

func (s *loggingStream) Read(ctx context.Context) ([]byte, int64, error) {
	data, count, err := s.stream.Read(ctx)
	if err == nil {
		logIn(s.log, data)
	}
	return data, count, err
}

func (s *loggingStream) Write(ctx context.Context, data []byte) (int64, error) {
	logOut(s.log, data)
	count, err := s.stream.Write(ctx, data)
	return count, err
}

// Combined has all the fields of both Request and Response.
// We can decode this and then work out which it is.
type Combined struct {
	VersionTag jsonrpc2.VersionTag `json:"jsonrpc"`
	ID         *jsonrpc2.ID        `json:"id,omitempty"`
	Method     string              `json:"method"`
	Params     *json.RawMessage    `json:"params,omitempty"`
	Result     *json.RawMessage    `json:"result,omitempty"`
	Error      *jsonrpc2.Error     `json:"error,omitempty"`
}

type req struct {
	method string
	start  time.Time
}

type mapped struct {
	mu          sync.Mutex
	clientCalls map[string]req
	serverCalls map[string]req
}

var maps = &mapped{
	sync.Mutex{},
	make(map[string]req),
	make(map[string]req),
}

// these 4 methods are each used exactly once, but it seemed
// better to have the encapsulation rather than ad hoc mutex
// code in 4 places
func (m *mapped) client(id string, del bool) req {
	m.mu.Lock()
	defer m.mu.Unlock()
	v := m.clientCalls[id]
	if del {
		delete(m.clientCalls, id)
	}
	return v
}

func (m *mapped) server(id string, del bool) req {
	m.mu.Lock()
	defer m.mu.Unlock()
	v := m.serverCalls[id]
	if del {
		delete(m.serverCalls, id)
	}
	return v
}

func (m *mapped) setClient(id string, r req) {
	m.mu.Lock()
	defer m.mu.Unlock()
	m.clientCalls[id] = r
}

func (m *mapped) setServer(id string, r req) {
	m.mu.Lock()
	defer m.mu.Unlock()
	m.serverCalls[id] = r
}

const eor = "\r\n\r\n\r\n"

func strID(x *jsonrpc2.ID) string {
	if x == nil {
		// should never happen, but we need a number
		return "999999999"
	}
	if x.Name != "" {
		return x.Name
	}
	return fmt.Sprintf("%d", x.Number)
}

func logCommon(outfd io.Writer, data []byte) (*Combined, time.Time, string) {
	if outfd == nil {
		return nil, time.Time{}, ""
	}
	var v Combined
	err := json.Unmarshal(data, &v)
	if err != nil {
		fmt.Fprintf(outfd, "Unmarshal %v\n", err)
		panic(err) // do better
	}
	tm := time.Now()
	tmfmt := tm.Format("15:04:05.000 PM")
	return &v, tm, tmfmt
}

// logOut and logIn could be combined. "received"<->"Sending", serverCalls<->clientCalls
// but it wouldn't be a lot shorter or clearer and "shutdown" is a special case

// Writing a message to the client, log it
func logOut(outfd io.Writer, data []byte) {
	v, tm, tmfmt := logCommon(outfd, data)
	if v == nil {
		return
	}
	if v.Error != nil {
		id := strID(v.ID)
		fmt.Fprintf(outfd, "[Error - %s] Received #%s %s%s", tmfmt, id, v.Error, eor)
		return
	}
	buf := strings.Builder{}
	id := strID(v.ID)
	fmt.Fprintf(&buf, "[Trace - %s] ", tmfmt) // common beginning
	if v.ID != nil && v.Method != "" && v.Params != nil {
		fmt.Fprintf(&buf, "Received request '%s - (%s)'.\n", v.Method, id)
		fmt.Fprintf(&buf, "Params: %s%s", *v.Params, eor)
		maps.setServer(id, req{method: v.Method, start: tm})
	} else if v.ID != nil && v.Method == "" && v.Params == nil {
		cc := maps.client(id, true)
		elapsed := tm.Sub(cc.start)
		fmt.Fprintf(&buf, "Received response '%s - (%s)' in %dms.\n",
			cc.method, id, elapsed/time.Millisecond)
		if v.Result == nil {
			fmt.Fprintf(&buf, "Result: {}%s", eor)
		} else {
			fmt.Fprintf(&buf, "Result: %s%s", string(*v.Result), eor)
		}
	} else if v.ID == nil && v.Method != "" && v.Params != nil {
		p := "null"
		if v.Params != nil {
			p = string(*v.Params)
		}
		fmt.Fprintf(&buf, "Received notification '%s'.\n", v.Method)
		fmt.Fprintf(&buf, "Params: %s%s", p, eor)
	} else { // for completeness, as it should never happen
		buf = strings.Builder{} // undo common Trace
		fmt.Fprintf(&buf, "[Error - %s] on write ID?%v method:%q Params:%v Result:%v Error:%v%s",
			tmfmt, v.ID != nil, v.Method, v.Params != nil,
			v.Result != nil, v.Error != nil, eor)
		p := "null"
		if v.Params != nil {
			p = string(*v.Params)
		}
		r := "null"
		if v.Result != nil {
			r = string(*v.Result)
		}
		fmt.Fprintf(&buf, "%s\n%s\n%s%s", p, r, v.Error, eor)
	}
	outfd.Write([]byte(buf.String()))
}

// Got a message from the client, log it
func logIn(outfd io.Writer, data []byte) {
	v, tm, tmfmt := logCommon(outfd, data)
	if v == nil {
		return
	}
	// ID Method Params => Sending request
	// ID !Method Result(might be null, but !Params) => Sending response (could we get an Error?)
	// !ID Method Params => Sending notification
	if v.Error != nil { // does this ever happen?
		id := strID(v.ID)
		fmt.Fprintf(outfd, "[Error - %s] Sent #%s %s%s", tmfmt, id, v.Error, eor)
		return
	}
	buf := strings.Builder{}
	id := strID(v.ID)
	fmt.Fprintf(&buf, "[Trace - %s] ", tmfmt) // common beginning
	if v.ID != nil && v.Method != "" && (v.Params != nil || v.Method == "shutdown") {
		fmt.Fprintf(&buf, "Sending request '%s - (%s)'.\n", v.Method, id)
		x := "{}"
		if v.Params != nil {
			x = string(*v.Params)
		}
		fmt.Fprintf(&buf, "Params: %s%s", x, eor)
		maps.setClient(id, req{method: v.Method, start: tm})
	} else if v.ID != nil && v.Method == "" && v.Params == nil {
		sc := maps.server(id, true)
		elapsed := tm.Sub(sc.start)
		fmt.Fprintf(&buf, "Sending response '%s - (%s)' in %dms.\n",
			sc.method, id, elapsed/time.Millisecond)
		if v.Result == nil {
			fmt.Fprintf(&buf, "Result: {}%s", eor)
		} else {
			fmt.Fprintf(&buf, "Result: %s%s", string(*v.Result), eor)
		}
	} else if v.ID == nil && v.Method != "" && v.Params != nil {
		fmt.Fprintf(&buf, "Sending notification '%s'.\n", v.Method)
		fmt.Fprintf(&buf, "Params: %s%s", *v.Params, eor)
	} else { // for completeness, as it should never happen
		buf = strings.Builder{} // undo common Trace
		fmt.Fprintf(&buf, "[Error - %s] on read ID?%v method:%q Params:%v Result:%v Error:%v%s",
			tmfmt, v.ID != nil, v.Method, v.Params != nil,
			v.Result != nil, v.Error != nil, eor)
		p := "null"
		if v.Params != nil {
			p = string(*v.Params)
		}
		r := "null"
		if v.Result != nil {
			r = string(*v.Result)
		}
		fmt.Fprintf(&buf, "%s\n%s\n%s%s", p, r, v.Error, eor)
	}
	outfd.Write([]byte(buf.String()))
}
<commit_msg>internal/lsp/protocol: handle case of an otherwise empty notification<commit_after>package protocol

import (
	"context"
	"encoding/json"
	"fmt"
	"io"
	"strings"
	"sync"
	"time"

	"golang.org/x/tools/internal/jsonrpc2"
)

type loggingStream struct {
	stream jsonrpc2.Stream
	log    io.Writer
}

// LoggingStream returns a stream that does LSP protocol logging too
func LoggingStream(str jsonrpc2.Stream, w io.Writer) jsonrpc2.Stream {
	return &loggingStream{str, w}
}

func (s *loggingStream) Read(ctx context.Context) ([]byte, int64, error) {
	data, count, err := s.stream.Read(ctx)
	if err == nil {
		logIn(s.log, data)
	}
	return data, count, err
}

func (s *loggingStream) Write(ctx context.Context, data []byte) (int64, error) {
	logOut(s.log, data)
	count, err := s.stream.Write(ctx, data)
	return count, err
}

// Combined has all the fields of both Request and Response.
// We can decode this and then work out which it is.
type Combined struct {
	VersionTag jsonrpc2.VersionTag `json:"jsonrpc"`
	ID         *jsonrpc2.ID        `json:"id,omitempty"`
	Method     string              `json:"method"`
	Params     *json.RawMessage    `json:"params,omitempty"`
	Result     *json.RawMessage    `json:"result,omitempty"`
	Error      *jsonrpc2.Error     `json:"error,omitempty"`
}

type req struct {
	method string
	start  time.Time
}

type mapped struct {
	mu          sync.Mutex
	clientCalls map[string]req
	serverCalls map[string]req
}

var maps = &mapped{
	sync.Mutex{},
	make(map[string]req),
	make(map[string]req),
}

// these 4 methods are each used exactly once, but it seemed
// better to have the encapsulation rather than ad hoc mutex
// code in 4 places
func (m *mapped) client(id string, del bool) req {
	m.mu.Lock()
	defer m.mu.Unlock()
	v := m.clientCalls[id]
	if del {
		delete(m.clientCalls, id)
	}
	return v
}

func (m *mapped) server(id string, del bool) req {
	m.mu.Lock()
	defer m.mu.Unlock()
	v := m.serverCalls[id]
	if del {
		delete(m.serverCalls, id)
	}
	return v
}

func (m *mapped) setClient(id string, r req) {
	m.mu.Lock()
	defer m.mu.Unlock()
	m.clientCalls[id] = r
}

func (m *mapped) setServer(id string, r req) {
	m.mu.Lock()
	defer m.mu.Unlock()
	m.serverCalls[id] = r
}

const eor = "\r\n\r\n\r\n"

func strID(x *jsonrpc2.ID) string {
	if x == nil {
		// should never happen, but we need a number
		return "999999999"
	}
	if x.Name != "" {
		return x.Name
	}
	return fmt.Sprintf("%d", x.Number)
}

func logCommon(outfd io.Writer, data []byte) (*Combined, time.Time, string) {
	if outfd == nil {
		return nil, time.Time{}, ""
	}
	var v Combined
	err := json.Unmarshal(data, &v)
	if err != nil {
		fmt.Fprintf(outfd, "Unmarshal %v\n", err)
		panic(err) // do better
	}
	tm := time.Now()
	tmfmt := tm.Format("15:04:05.000 PM")
	return &v, tm, tmfmt
}

// logOut and logIn could be combined. "received"<->"Sending", serverCalls<->clientCalls
// but it wouldn't be a lot shorter or clearer and "shutdown" is a special case

// Writing a message to the client, log it
func logOut(outfd io.Writer, data []byte) {
	v, tm, tmfmt := logCommon(outfd, data)
	if v == nil {
		return
	}
	if v.Error != nil {
		id := strID(v.ID)
		fmt.Fprintf(outfd, "[Error - %s] Received #%s %s%s", tmfmt, id, v.Error, eor)
		return
	}
	buf := strings.Builder{}
	id := strID(v.ID)
	fmt.Fprintf(&buf, "[Trace - %s] ", tmfmt) // common beginning
	if v.ID != nil && v.Method != "" && v.Params != nil {
		fmt.Fprintf(&buf, "Received request '%s - (%s)'.\n", v.Method, id)
		fmt.Fprintf(&buf, "Params: %s%s", *v.Params, eor)
		maps.setServer(id, req{method: v.Method, start: tm})
	} else if v.ID != nil && v.Method == "" && v.Params == nil {
		cc := maps.client(id, true)
		elapsed := tm.Sub(cc.start)
		fmt.Fprintf(&buf, "Received response '%s - (%s)' in %dms.\n",
			cc.method, id, elapsed/time.Millisecond)
		if v.Result == nil {
			fmt.Fprintf(&buf, "Result: {}%s", eor)
		} else {
			fmt.Fprintf(&buf, "Result: %s%s", string(*v.Result), eor)
		}
	} else if v.ID == nil && v.Method != "" && v.Params != nil {
		p := "null"
		if v.Params != nil {
			p = string(*v.Params)
		}
		fmt.Fprintf(&buf, "Received notification '%s'.\n", v.Method)
		fmt.Fprintf(&buf, "Params: %s%s", p, eor)
	} else { // for completeness, as it should never happen
		buf = strings.Builder{} // undo common Trace
		fmt.Fprintf(&buf, "[Error - %s] on write ID?%v method:%q Params:%v Result:%v Error:%v%s",
			tmfmt, v.ID != nil, v.Method, v.Params != nil,
			v.Result != nil, v.Error != nil, eor)
		p := "null"
		if v.Params != nil {
			p = string(*v.Params)
		}
		r := "null"
		if v.Result != nil {
			r = string(*v.Result)
		}
		fmt.Fprintf(&buf, "%s\n%s\n%s%s", p, r, v.Error, eor)
	}
	outfd.Write([]byte(buf.String()))
}

// Got a message from the client, log it
func logIn(outfd io.Writer, data []byte) {
	v, tm, tmfmt := logCommon(outfd, data)
	if v == nil {
		return
	}
	// ID Method Params => Sending request
	// ID !Method Result(might be null, but !Params) => Sending response (could we get an Error?)
	// !ID Method Params => Sending notification
	if v.Error != nil { // does this ever happen?
		id := strID(v.ID)
		fmt.Fprintf(outfd, "[Error - %s] Sent #%s %s%s", tmfmt, id, v.Error, eor)
		return
	}
	buf := strings.Builder{}
	id := strID(v.ID)
	fmt.Fprintf(&buf, "[Trace - %s] ", tmfmt) // common beginning
	if v.ID != nil && v.Method != "" && (v.Params != nil || v.Method == "shutdown") {
		fmt.Fprintf(&buf, "Sending request '%s - (%s)'.\n", v.Method, id)
		x := "{}"
		if v.Params != nil {
			x = string(*v.Params)
		}
		fmt.Fprintf(&buf, "Params: %s%s", x, eor)
		maps.setClient(id, req{method: v.Method, start: tm})
	} else if v.ID != nil && v.Method == "" && v.Params == nil {
		sc := maps.server(id, true)
		elapsed := tm.Sub(sc.start)
		fmt.Fprintf(&buf, "Sending response '%s - (%s)' in %dms.\n",
			sc.method, id, elapsed/time.Millisecond)
		if v.Result == nil {
			fmt.Fprintf(&buf, "Result: {}%s", eor)
		} else {
			fmt.Fprintf(&buf, "Result: %s%s", string(*v.Result), eor)
		}
	} else if v.ID == nil && v.Method != "" {
		p := "null"
		if v.Params != nil {
			p = string(*v.Params)
		}
		fmt.Fprintf(&buf, "Sending notification '%s'.\n", v.Method)
		fmt.Fprintf(&buf, "Params: %s%s", p, eor)
	} else { // for completeness, as it should never happen
		buf = strings.Builder{} // undo common Trace
		fmt.Fprintf(&buf, "[Error - %s] on read ID?%v method:%q Params:%v Result:%v Error:%v%s",
			tmfmt, v.ID != nil, v.Method, v.Params != nil,
			v.Result != nil, v.Error != nil, eor)
		p := "null"
		if v.Params != nil {
			p = string(*v.Params)
		}
		r := "null"
		if v.Result != nil {
			r = string(*v.Result)
		}
		fmt.Fprintf(&buf, "%s\n%s\n%s%s", p, r, v.Error, eor)
	}
	outfd.Write([]byte(buf.String()))
}
<|endoftext|>


<commit_before>// Copyright 2020 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

package postgres

import (
	"context"
	"database/sql"
	"fmt"
	"net/http"
	"strconv"

	"golang.org/x/pkgsite/internal"
	"golang.org/x/pkgsite/internal/derrors"
	"golang.org/x/pkgsite/internal/log"
)

// UpdateModuleVersionStatesForReprocessing marks modules to be reprocessed
// that were processed prior to the provided appVersion.
func (db *DB) UpdateModuleVersionStatesForReprocessing(ctx context.Context, appVersion string) (err error) {
	defer derrors.Wrap(&err, "UpdateModuleVersionStatesForReprocessing(ctx, %q)", appVersion)

	for _, status := range []int{
		http.StatusOK,
		derrors.ToHTTPStatus(derrors.HasIncompletePackages),
		derrors.ToHTTPStatus(derrors.BadModule),
		derrors.ToHTTPStatus(derrors.AlternativeModule),
	} {
		query := `UPDATE module_version_states
			SET
				status = $2,
				next_processed_after = CURRENT_TIMESTAMP,
				last_processed_at = NULL
			WHERE
				app_version < $1
				AND status = $3;`
		result, err := db.db.Exec(ctx, query, appVersion,
			derrors.ToReprocessStatus(status), status)
		if err != nil {
			return err
		}
		affected, err := result.RowsAffected()
		if err != nil {
			return fmt.Errorf("result.RowsAffected(): %v", err)
		}
		log.Infof(ctx,
			"Updated %d module version states with status %v to be reprocessed for app_version <= %q; new status = %v",
			affected, status, appVersion,
			derrors.ToReprocessStatus(status))
	}
	return nil
}

// Modules with more than this number of packages are generally different
// versions or forks of kubernetes, aws-sdk-go, azure-sdk-go, and bilibili.
var isBigModulePackageThreshold = 1500

// GetNextModulesToFetch returns the next batch of modules that need to be
// processed. We prioritize modules based on (1) whether it is the latest version,
// (2) if it is an alternative module, and (3) the number of packages it has.
// We want to leave time-consuming modules until the end and process them at
// a slower rate to reduce database load and timeouts. We also want to leave
// alternative modules towards the end, since these will incur unnecessary
// deletes otherwise.
func (db *DB) GetNextModulesToFetch(ctx context.Context, limit int) (_ []*internal.ModuleVersionState, err error) {
	defer derrors.Wrap(&err, "GetNextModulesToFetch(ctx, %d)", limit)

	var mvs []*internal.ModuleVersionState
	for _, next := range []struct {
		query    string
		statuses []int
	}{
		{
			query: getLatestModuleVersionStates,
			statuses: []int{
				derrors.ToHTTPStatus(derrors.ReprocessStatusOK),
				derrors.ToHTTPStatus(derrors.ReprocessHasIncompletePackages),
			},
		},
		{
			query: getLatestModuleVersionStates,
			statuses: []int{
				derrors.ToHTTPStatus(derrors.ReprocessBadModule),
				derrors.ToHTTPStatus(derrors.ReprocessAlternative),
			},
		},
		{
			query: getModuleVersionStates,
			statuses: []int{
				derrors.ToHTTPStatus(derrors.ReprocessStatusOK),
				derrors.ToHTTPStatus(derrors.ReprocessHasIncompletePackages),
			},
		},
		{
			query: getModuleVersionStates,
			statuses: []int{
				derrors.ToHTTPStatus(derrors.ReprocessBadModule),
				derrors.ToHTTPStatus(derrors.ReprocessAlternative),
			},
		},
		{
			query: getModuleVersionStatesRemainder,
		},
	} {
		query := constructRequeueQuery(next.query, next.statuses)
		collect := func(rows *sql.Rows) error {
			mv, err := scanModuleVersionState(rows.Scan)
			if err != nil {
				return err
			}
			mvs = append(mvs, mv)
			return nil
		}
		if err := db.db.RunQuery(ctx, query, collect, limit); err != nil {
			return nil, err
		}
		if len(mvs) > 0 {
			var msg string
			switch next.query {
			case getModuleVersionStatesRemainder:
				msg = fmt.Sprintf("modules with status=0 or status=500 or num_packages > %d", isBigModulePackageThreshold)
			case getLatestModuleVersionStates:
				msg = "latest version of modules"
			default:
				msg = "non-latest version of modules"
			}
			fmtIntp := func(p *int) string {
				if p == nil {
					return "NULL"
				}
				return strconv.Itoa(*p)
			}
			start := mvs[0]
			end := mvs[len(mvs)-1]
			pkgRange := fmt.Sprintf("%s <= num_packages <= %s", fmtIntp(start.NumPackages), fmtIntp(end.NumPackages))
			log.Infof(ctx, fmt.Sprintf("GetNextModulesToFetch (%s): num_modules=%d; statuses=%v; %s; start_module=%q; end_module=%q",
				msg, len(mvs), next.statuses, pkgRange,
				fmt.Sprintf("%s/@v/%s", start.ModulePath, start.Version),
				fmt.Sprintf("%s/@v/%s", end.ModulePath, end.Version)))
			return mvs, nil
		}
	}
	log.Infof(ctx, "No modules to requeue")
	return mvs, nil
}

func constructRequeueQuery(baseQuery string, statuses []int) string {
	where := "WHERE next_processed_after < CURRENT_TIMESTAMP"
	if baseQuery != getModuleVersionStatesRemainder {
		where += fmt.Sprintf(" AND COALESCE(num_packages, 0) < %d", isBigModulePackageThreshold)
		var s string
		for i, status := range statuses {
			s += fmt.Sprintf("status=%d", status)
			if i < len(statuses)-1 {
				s += " OR "
			}
		}
		where += fmt.Sprintf(" AND (%s)", s)
	}
	query := fmt.Sprintf(baseQuery, moduleVersionStateColumns, where)
	return query
}

// Get the latest versions of modules that previously
// returned a 20x status; process them in order of
// number of packages.
//
// We also want to prefer release to non-release
// versions. A sort_version will end in '~' if it is a
// release, and that is larger than any other character
// that can occur in a sort_version.
// So if we sort first by the last character in
// sort_version, then by sort_version itself, we will
// get releases before non-releases.  To implement that
// two-level ordering in a MAX, we construct an array
// of the two strings.
// Arrays are ordered lexicographically, so MAX will do
// just what we want.
const getLatestModuleVersionStates = `
SELECT %s
FROM (
    SELECT s.*
    FROM module_version_states s
    INNER JOIN (
        SELECT module_path,
        MAX(ARRAY[right(sort_version, 1), sort_version]) AS mv
        FROM module_version_states
        GROUP BY 1
    ) m
    ON
        s.module_path = m.module_path
        AND s.sort_version = m.mv[2]

    -- WHERE clause
    %s

    ORDER BY
        num_packages,
        sort_version DESC,
        module_path
    LIMIT $1
) foo`

// Get non-latest versions to be reprocessed.
// Start with modules that previously succeeded, then
// move onto alternative modules.
const getModuleVersionStates = `
SELECT %s
FROM module_version_states

-- WHERE clause
%s

ORDER BY
    num_packages,
    sort_version DESC,
    module_path
LIMIT $1`

const getModuleVersionStatesRemainder = `
SELECT %s
FROM module_version_states

-- WHERE clause
%s

ORDER BY
    CASE WHEN status=0 THEN 0
         WHEN (status=520 OR status=521) THEN 1
         WHEN (status=540 OR status=541) THEN 2
         ELSE 3 END,
    COALESCE(num_packages, 0),
    sort_version DESC,
    module_path
LIMIT $1`
<commit_msg>internal/postgres: tweak log for reprocessing<commit_after>// Copyright 2020 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

package postgres

import (
	"context"
	"database/sql"
	"fmt"
	"net/http"
	"strconv"

	"golang.org/x/pkgsite/internal"
	"golang.org/x/pkgsite/internal/derrors"
	"golang.org/x/pkgsite/internal/log"
)

// UpdateModuleVersionStatesForReprocessing marks modules to be reprocessed
// that were processed prior to the provided appVersion.
func (db *DB) UpdateModuleVersionStatesForReprocessing(ctx context.Context, appVersion string) (err error) {
	defer derrors.Wrap(&err, "UpdateModuleVersionStatesForReprocessing(ctx, %q)", appVersion)

	for _, status := range []int{
		http.StatusOK,
		derrors.ToHTTPStatus(derrors.HasIncompletePackages),
		derrors.ToHTTPStatus(derrors.BadModule),
		derrors.ToHTTPStatus(derrors.AlternativeModule),
	} {
		query := `UPDATE module_version_states
			SET
				status = $2,
				next_processed_after = CURRENT_TIMESTAMP,
				last_processed_at = NULL
			WHERE
				app_version < $1
				AND status = $3;`
		result, err := db.db.Exec(ctx, query, appVersion,
			derrors.ToReprocessStatus(status), status)
		if err != nil {
			return err
		}
		affected, err := result.RowsAffected()
		if err != nil {
			return fmt.Errorf("result.RowsAffected(): %v", err)
		}
		log.Infof(ctx,
			"Updated module_version_states with status=%d and app_version < %q to status=%d; %d affected",
			status, appVersion, derrors.ToReprocessStatus(status), affected)
	}
	return nil
}

// Modules with more than this number of packages are generally different
// versions or forks of kubernetes, aws-sdk-go, azure-sdk-go, and bilibili.
var isBigModulePackageThreshold = 1500

// GetNextModulesToFetch returns the next batch of modules that need to be
// processed. We prioritize modules based on (1) whether it is the latest version,
// (2) if it is an alternative module, and (3) the number of packages it has.
// We want to leave time-consuming modules until the end and process them at
// a slower rate to reduce database load and timeouts. We also want to leave
// alternative modules towards the end, since these will incur unnecessary
// deletes otherwise.
func (db *DB) GetNextModulesToFetch(ctx context.Context, limit int) (_ []*internal.ModuleVersionState, err error) {
	defer derrors.Wrap(&err, "GetNextModulesToFetch(ctx, %d)", limit)

	var mvs []*internal.ModuleVersionState
	for _, next := range []struct {
		query    string
		statuses []int
	}{
		{
			query: getLatestModuleVersionStates,
			statuses: []int{
				derrors.ToHTTPStatus(derrors.ReprocessStatusOK),
				derrors.ToHTTPStatus(derrors.ReprocessHasIncompletePackages),
			},
		},
		{
			query: getLatestModuleVersionStates,
			statuses: []int{
				derrors.ToHTTPStatus(derrors.ReprocessBadModule),
				derrors.ToHTTPStatus(derrors.ReprocessAlternative),
			},
		},
		{
			query: getModuleVersionStates,
			statuses: []int{
				derrors.ToHTTPStatus(derrors.ReprocessStatusOK),
				derrors.ToHTTPStatus(derrors.ReprocessHasIncompletePackages),
			},
		},
		{
			query: getModuleVersionStates,
			statuses: []int{
				derrors.ToHTTPStatus(derrors.ReprocessBadModule),
				derrors.ToHTTPStatus(derrors.ReprocessAlternative),
			},
		},
		{
			query: getModuleVersionStatesRemainder,
		},
	} {
		query := constructRequeueQuery(next.query, next.statuses)
		collect := func(rows *sql.Rows) error {
			mv, err := scanModuleVersionState(rows.Scan)
			if err != nil {
				return err
			}
			mvs = append(mvs, mv)
			return nil
		}
		if err := db.db.RunQuery(ctx, query, collect, limit); err != nil {
			return nil, err
		}
		if len(mvs) > 0 {
			var msg string
			switch next.query {
			case getModuleVersionStatesRemainder:
				msg = fmt.Sprintf("modules with status=0 or status=500 or num_packages > %d", isBigModulePackageThreshold)
			case getLatestModuleVersionStates:
				msg = "latest version of modules"
			default:
				msg = "non-latest version of modules"
			}
			fmtIntp := func(p *int) string {
				if p == nil {
					return "NULL"
				}
				return strconv.Itoa(*p)
			}
			start := mvs[0]
			end := mvs[len(mvs)-1]
			pkgRange := fmt.Sprintf("%s <= num_packages <= %s", fmtIntp(start.NumPackages), fmtIntp(end.NumPackages))
			log.Infof(ctx, fmt.Sprintf("GetNextModulesToFetch (%s): num_modules=%d; statuses=%v; %s; start_module=%q; end_module=%q",
				msg, len(mvs), next.statuses, pkgRange,
				fmt.Sprintf("%s/@v/%s", start.ModulePath, start.Version),
				fmt.Sprintf("%s/@v/%s", end.ModulePath, end.Version)))
			return mvs, nil
		}
	}
	log.Infof(ctx, "No modules to requeue")
	return mvs, nil
}

func constructRequeueQuery(baseQuery string, statuses []int) string {
	where := "WHERE next_processed_after < CURRENT_TIMESTAMP"
	if baseQuery != getModuleVersionStatesRemainder {
		where += fmt.Sprintf(" AND COALESCE(num_packages, 0) < %d", isBigModulePackageThreshold)
		var s string
		for i, status := range statuses {
			s += fmt.Sprintf("status=%d", status)
			if i < len(statuses)-1 {
				s += " OR "
			}
		}
		where += fmt.Sprintf(" AND (%s)", s)
	}
	query := fmt.Sprintf(baseQuery, moduleVersionStateColumns, where)
	return query
}

// Get the latest versions of modules that previously
// returned a 20x status; process them in order of
// number of packages.
//
// We also want to prefer release to non-release
// versions. A sort_version will end in '~' if it is a
// release, and that is larger than any other character
// that can occur in a sort_version.
// So if we sort first by the last character in
// sort_version, then by sort_version itself, we will
// get releases before non-releases.  To implement that
// two-level ordering in a MAX, we construct an array
// of the two strings.
// Arrays are ordered lexicographically, so MAX will do
// just what we want.
const getLatestModuleVersionStates = `
SELECT %s
FROM (
    SELECT s.*
    FROM module_version_states s
    INNER JOIN (
        SELECT module_path,
        MAX(ARRAY[right(sort_version, 1), sort_version]) AS mv
        FROM module_version_states
        GROUP BY 1
    ) m
    ON
        s.module_path = m.module_path
        AND s.sort_version = m.mv[2]

    -- WHERE clause
    %s

    ORDER BY
        num_packages,
        sort_version DESC,
        module_path
    LIMIT $1
) foo`

// Get non-latest versions to be reprocessed.
// Start with modules that previously succeeded, then
// move onto alternative modules.
const getModuleVersionStates = `
SELECT %s
FROM module_version_states

-- WHERE clause
%s

ORDER BY
    num_packages,
    sort_version DESC,
    module_path
LIMIT $1`

const getModuleVersionStatesRemainder = `
SELECT %s
FROM module_version_states

-- WHERE clause
%s

ORDER BY
    CASE WHEN status=0 THEN 0
         WHEN (status=520 OR status=521) THEN 1
         WHEN (status=540 OR status=541) THEN 2
         ELSE 3 END,
    COALESCE(num_packages, 0),
    sort_version DESC,
    module_path
LIMIT $1`
<|endoftext|>