<commit_before>package cmd

import (
	"crypto/tls"
	"fmt"
	"net"
	"net/http"
	"os"

	"github.com/codegangsta/cli"
	"gopkg.in/macaron.v1"

	"github.com/containerops/dockyard/backend"
	"github.com/containerops/dockyard/web"
	webgen "github.com/containerops/generator/web"
	"github.com/containerops/wrench/setting"
	"github.com/containerops/wrench/utils"
)

var CmdWeb = cli.Command{
	Name:        "web",
	Usage:       "start dockyard web service",
	Description: "dockyard is the module of handler docker and rkt image.",
	Action:      runWeb,
	Flags: []cli.Flag{
		cli.StringFlag{
			Name:  "address",
			Value: "0.0.0.0",
			Usage: "web service listen ip, default is 0.0.0.0; if listen with Unix Socket, the value is sock file path.",
		},
		cli.IntFlag{
			Name:  "port",
			Value: 80,
			Usage: "web service listen at port 80; if run with https will be 443.",
		},
	},
}

func runWeb(c *cli.Context) {
	m := macaron.New()

	//Set Macaron Web Middleware And Routers
	web.SetDockyardMacaron(m)
	webgen.SetGeneratorMacaron(m)

	backend.InitBackend()

	switch setting.ListenMode {
	case "http":
		listenaddr := fmt.Sprintf("%s:%d", c.String("address"), c.Int("port"))
		if err := http.ListenAndServe(listenaddr, m); err != nil {
			fmt.Printf("Start Dockyard http service error: %v", err.Error())
		}
		break
	case "https":
		listenaddr := fmt.Sprintf("%s:443", c.String("address"))
		server := &http.Server{Addr: listenaddr, TLSConfig: &tls.Config{MinVersion: tls.VersionTLS10}, Handler: m}
		if err := server.ListenAndServeTLS(setting.HttpsCertFile, setting.HttpsKeyFile); err != nil {
			fmt.Printf("Start Dockyard https service error: %v", err.Error())
		}
		break
	case "unix":
		listenaddr := fmt.Sprintf("%s", c.String("address"))
		if utils.IsFileExist(listenaddr) {
			os.Remove(listenaddr)
		}

		if listener, err := net.Listen("unix", listenaddr); err != nil {
			fmt.Printf("Start Dockyard unix socket error: %v", err.Error())
		} else {
			server := &http.Server{Handler: m}
			if err := server.Serve(listener); err != nil {
				fmt.Printf("Start Dockyard unix socket error: %v", err.Error())
			}
		}
		break
	default:
		break
	}
}
<commit_msg>Get rid of the dependence on Generator project<commit_after>package cmd

import (
	"crypto/tls"
	"fmt"
	"net"
	"net/http"
	"os"

	"github.com/codegangsta/cli"
	"gopkg.in/macaron.v1"

	"github.com/containerops/dockyard/backend"
	"github.com/containerops/dockyard/web"
	"github.com/containerops/wrench/setting"
	"github.com/containerops/wrench/utils"
)

var CmdWeb = cli.Command{
	Name:        "web",
	Usage:       "start dockyard web service",
	Description: "dockyard is the module of handler docker and rkt image.",
	Action:      runWeb,
	Flags: []cli.Flag{
		cli.StringFlag{
			Name:  "address",
			Value: "0.0.0.0",
			Usage: "web service listen ip, default is 0.0.0.0; if listen with Unix Socket, the value is sock file path.",
		},
		cli.IntFlag{
			Name:  "port",
			Value: 80,
			Usage: "web service listen at port 80; if run with https will be 443.",
		},
	},
}

func runWeb(c *cli.Context) {
	m := macaron.New()

	//Set Macaron Web Middleware And Routers
	web.SetDockyardMacaron(m)

	backend.InitBackend()

	switch setting.ListenMode {
	case "http":
		listenaddr := fmt.Sprintf("%s:%d", c.String("address"), c.Int("port"))
		if err := http.ListenAndServe(listenaddr, m); err != nil {
			fmt.Printf("Start Dockyard http service error: %v", err.Error())
		}
		break
	case "https":
		listenaddr := fmt.Sprintf("%s:443", c.String("address"))
		server := &http.Server{Addr: listenaddr, TLSConfig: &tls.Config{MinVersion: tls.VersionTLS10}, Handler: m}
		if err := server.ListenAndServeTLS(setting.HttpsCertFile, setting.HttpsKeyFile); err != nil {
			fmt.Printf("Start Dockyard https service error: %v", err.Error())
		}
		break
	case "unix":
		listenaddr := fmt.Sprintf("%s", c.String("address"))
		if utils.IsFileExist(listenaddr) {
			os.Remove(listenaddr)
		}

		if listener, err := net.Listen("unix", listenaddr); err != nil {
			fmt.Printf("Start Dockyard unix socket error: %v", err.Error())
		} else {
			server := &http.Server{Handler: m}
			if err := server.Serve(listener); err != nil {
				fmt.Printf("Start Dockyard unix socket error: %v", err.Error())
			}
		}
		break
	default:
		break
	}
}
<|endoftext|>


<commit_before>package registry

import (
	"fmt"
	"io/ioutil"
	"net/http"
	"net/url"
	"strings"
	"time"

	"github.com/Sirupsen/logrus"
	"github.com/docker/distribution/registry/client/auth"
	"github.com/docker/distribution/registry/client/transport"
	"github.com/docker/engine-api/types"
	registrytypes "github.com/docker/engine-api/types/registry"
)

const (
	// AuthClientID is used the ClientID used for the token server
	AuthClientID = "docker"
)

// loginV1 tries to register/login to the v1 registry server.
func loginV1(authConfig *types.AuthConfig, apiEndpoint APIEndpoint, userAgent string) (string, string, error) {
	registryEndpoint, err := apiEndpoint.ToV1Endpoint(userAgent, nil)
	if err != nil {
		return "", "", err
	}

	serverAddress := registryEndpoint.String()

	logrus.Debugf("attempting v1 login to registry endpoint %s", registryEndpoint)

	if serverAddress == "" {
		return "", "", fmt.Errorf("Server Error: Server Address not set.")
	}

	loginAgainstOfficialIndex := serverAddress == IndexServer

	req, err := http.NewRequest("GET", serverAddress+"users/", nil)
	if err != nil {
		return "", "", err
	}
	req.SetBasicAuth(authConfig.Username, authConfig.Password)
	resp, err := registryEndpoint.client.Do(req)
	if err != nil {
		// fallback when request could not be completed
		return "", "", fallbackError{
			err: err,
		}
	}
	defer resp.Body.Close()
	body, err := ioutil.ReadAll(resp.Body)
	if err != nil {
		return "", "", err
	}
	if resp.StatusCode == http.StatusOK {
		return "Login Succeeded", "", nil
	} else if resp.StatusCode == http.StatusUnauthorized {
		if loginAgainstOfficialIndex {
			return "", "", fmt.Errorf("Wrong login/password, please try again. Haven't got a Docker ID? Create one at https://hub.docker.com")
		}
		return "", "", fmt.Errorf("Wrong login/password, please try again")
	} else if resp.StatusCode == http.StatusForbidden {
		if loginAgainstOfficialIndex {
			return "", "", fmt.Errorf("Login: Account is not active. Please check your e-mail for a confirmation link.")
		}
		// *TODO: Use registry configuration to determine what this says, if anything?
		return "", "", fmt.Errorf("Login: Account is not active. Please see the documentation of the registry %s for instructions how to activate it.", serverAddress)
	} else if resp.StatusCode == http.StatusInternalServerError { // Issue #14326
		logrus.Errorf("%s returned status code %d. Response Body :\n%s", req.URL.String(), resp.StatusCode, body)
		return "", "", fmt.Errorf("Internal Server Error")
	}
	return "", "", fmt.Errorf("Login: %s (Code: %d; Headers: %s)", body,
		resp.StatusCode, resp.Header)
}

type loginCredentialStore struct {
	authConfig *types.AuthConfig
}

func (lcs loginCredentialStore) Basic(*url.URL) (string, string) {
	return lcs.authConfig.Username, lcs.authConfig.Password
}

func (lcs loginCredentialStore) RefreshToken(*url.URL, string) string {
	return lcs.authConfig.IdentityToken
}

func (lcs loginCredentialStore) SetRefreshToken(u *url.URL, service, token string) {
	lcs.authConfig.IdentityToken = token
}

type fallbackError struct {
	err error
}

func (err fallbackError) Error() string {
	return err.err.Error()
}

// loginV2 tries to login to the v2 registry server. The given registry
// endpoint will be pinged to get authorization challenges. These challenges
// will be used to authenticate against the registry to validate credentials.
func loginV2(authConfig *types.AuthConfig, endpoint APIEndpoint, userAgent string) (string, string, error) {
	logrus.Debugf("attempting v2 login to registry endpoint %s", endpoint)

	modifiers := DockerHeaders(userAgent, nil)
	authTransport := transport.NewTransport(NewTransport(endpoint.TLSConfig), modifiers...)

	challengeManager, foundV2, err := PingV2Registry(endpoint, authTransport)
	if err != nil {
		if !foundV2 {
			err = fallbackError{err: err}
		}
		return "", "", err
	}

	credentialAuthConfig := *authConfig
	creds := loginCredentialStore{
		authConfig: &credentialAuthConfig,
	}

	tokenHandlerOptions := auth.TokenHandlerOptions{
		Transport:     authTransport,
		Credentials:   creds,
		OfflineAccess: true,
		ClientID:      AuthClientID,
	}
	tokenHandler := auth.NewTokenHandlerWithOptions(tokenHandlerOptions)
	basicHandler := auth.NewBasicHandler(creds)
	modifiers = append(modifiers, auth.NewAuthorizer(challengeManager, tokenHandler, basicHandler))
	tr := transport.NewTransport(authTransport, modifiers...)

	loginClient := &http.Client{
		Transport: tr,
		Timeout:   15 * time.Second,
	}

	endpointStr := strings.TrimRight(endpoint.URL.String(), "/") + "/v2/"
	req, err := http.NewRequest("GET", endpointStr, nil)
	if err != nil {
		if !foundV2 {
			err = fallbackError{err: err}
		}
		return "", "", err
	}

	resp, err := loginClient.Do(req)
	if err != nil {
		if !foundV2 {
			err = fallbackError{err: err}
		}
		return "", "", err
	}
	defer resp.Body.Close()

	if resp.StatusCode != http.StatusOK {
		// TODO(dmcgowan): Attempt to further interpret result, status code and error code string
		err := fmt.Errorf("login attempt to %s failed with status: %d %s", endpointStr, resp.StatusCode, http.StatusText(resp.StatusCode))
		if !foundV2 {
			err = fallbackError{err: err}
		}
		return "", "", err
	}

	return "Login Succeeded", credentialAuthConfig.IdentityToken, nil

}

// ResolveAuthConfig matches an auth configuration to a server address or a URL
func ResolveAuthConfig(authConfigs map[string]types.AuthConfig, index *registrytypes.IndexInfo) types.AuthConfig {
	configKey := GetAuthConfigKey(index)
	// First try the happy case
	if c, found := authConfigs[configKey]; found || index.Official {
		return c
	}

	convertToHostname := func(url string) string {
		stripped := url
		if strings.HasPrefix(url, "http://") {
			stripped = strings.Replace(url, "http://", "", 1)
		} else if strings.HasPrefix(url, "https://") {
			stripped = strings.Replace(url, "https://", "", 1)
		}

		nameParts := strings.SplitN(stripped, "/", 2)

		return nameParts[0]
	}

	// Maybe they have a legacy config file, we will iterate the keys converting
	// them to the new format and testing
	for registry, ac := range authConfigs {
		if configKey == convertToHostname(registry) {
			return ac
		}
	}

	// When all else fails, return an empty auth config
	return types.AuthConfig{}
}

// PingResponseError is used when the response from a ping
// was received but invalid.
type PingResponseError struct {
	Err error
}

func (err PingResponseError) Error() string {
	return err.Error()
}

// PingV2Registry attempts to ping a v2 registry and on success return a
// challenge manager for the supported authentication types and
// whether v2 was confirmed by the response. If a response is received but
// cannot be interpreted a PingResponseError will be returned.
func PingV2Registry(endpoint APIEndpoint, transport http.RoundTripper) (auth.ChallengeManager, bool, error) {
	var (
		foundV2   = false
		v2Version = auth.APIVersion{
			Type:    "registry",
			Version: "2.0",
		}
	)

	pingClient := &http.Client{
		Transport: transport,
		Timeout:   15 * time.Second,
	}
	endpointStr := strings.TrimRight(endpoint.URL.String(), "/") + "/v2/"
	req, err := http.NewRequest("GET", endpointStr, nil)
	if err != nil {
		return nil, false, err
	}
	resp, err := pingClient.Do(req)
	if err != nil {
		return nil, false, err
	}
	defer resp.Body.Close()

	versions := auth.APIVersions(resp, DefaultRegistryVersionHeader)
	for _, pingVersion := range versions {
		if pingVersion == v2Version {
			// The version header indicates we're definitely
			// talking to a v2 registry. So don't allow future
			// fallbacks to the v1 protocol.

			foundV2 = true
			break
		}
	}

	challengeManager := auth.NewSimpleChallengeManager()
	if err := challengeManager.AddResponse(resp); err != nil {
		return nil, foundV2, PingResponseError{
			Err: err,
		}
	}

	return challengeManager, foundV2, nil
}
<commit_msg>Correct login debug log message<commit_after>package registry

import (
	"fmt"
	"io/ioutil"
	"net/http"
	"net/url"
	"strings"
	"time"

	"github.com/Sirupsen/logrus"
	"github.com/docker/distribution/registry/client/auth"
	"github.com/docker/distribution/registry/client/transport"
	"github.com/docker/engine-api/types"
	registrytypes "github.com/docker/engine-api/types/registry"
)

const (
	// AuthClientID is used the ClientID used for the token server
	AuthClientID = "docker"
)

// loginV1 tries to register/login to the v1 registry server.
func loginV1(authConfig *types.AuthConfig, apiEndpoint APIEndpoint, userAgent string) (string, string, error) {
	registryEndpoint, err := apiEndpoint.ToV1Endpoint(userAgent, nil)
	if err != nil {
		return "", "", err
	}

	serverAddress := registryEndpoint.String()

	logrus.Debugf("attempting v1 login to registry endpoint %s", serverAddress)

	if serverAddress == "" {
		return "", "", fmt.Errorf("Server Error: Server Address not set.")
	}

	loginAgainstOfficialIndex := serverAddress == IndexServer

	req, err := http.NewRequest("GET", serverAddress+"users/", nil)
	if err != nil {
		return "", "", err
	}
	req.SetBasicAuth(authConfig.Username, authConfig.Password)
	resp, err := registryEndpoint.client.Do(req)
	if err != nil {
		// fallback when request could not be completed
		return "", "", fallbackError{
			err: err,
		}
	}
	defer resp.Body.Close()
	body, err := ioutil.ReadAll(resp.Body)
	if err != nil {
		return "", "", err
	}
	if resp.StatusCode == http.StatusOK {
		return "Login Succeeded", "", nil
	} else if resp.StatusCode == http.StatusUnauthorized {
		if loginAgainstOfficialIndex {
			return "", "", fmt.Errorf("Wrong login/password, please try again. Haven't got a Docker ID? Create one at https://hub.docker.com")
		}
		return "", "", fmt.Errorf("Wrong login/password, please try again")
	} else if resp.StatusCode == http.StatusForbidden {
		if loginAgainstOfficialIndex {
			return "", "", fmt.Errorf("Login: Account is not active. Please check your e-mail for a confirmation link.")
		}
		// *TODO: Use registry configuration to determine what this says, if anything?
		return "", "", fmt.Errorf("Login: Account is not active. Please see the documentation of the registry %s for instructions how to activate it.", serverAddress)
	} else if resp.StatusCode == http.StatusInternalServerError { // Issue #14326
		logrus.Errorf("%s returned status code %d. Response Body :\n%s", req.URL.String(), resp.StatusCode, body)
		return "", "", fmt.Errorf("Internal Server Error")
	}
	return "", "", fmt.Errorf("Login: %s (Code: %d; Headers: %s)", body,
		resp.StatusCode, resp.Header)
}

type loginCredentialStore struct {
	authConfig *types.AuthConfig
}

func (lcs loginCredentialStore) Basic(*url.URL) (string, string) {
	return lcs.authConfig.Username, lcs.authConfig.Password
}

func (lcs loginCredentialStore) RefreshToken(*url.URL, string) string {
	return lcs.authConfig.IdentityToken
}

func (lcs loginCredentialStore) SetRefreshToken(u *url.URL, service, token string) {
	lcs.authConfig.IdentityToken = token
}

type fallbackError struct {
	err error
}

func (err fallbackError) Error() string {
	return err.err.Error()
}

// loginV2 tries to login to the v2 registry server. The given registry
// endpoint will be pinged to get authorization challenges. These challenges
// will be used to authenticate against the registry to validate credentials.
func loginV2(authConfig *types.AuthConfig, endpoint APIEndpoint, userAgent string) (string, string, error) {
	logrus.Debugf("attempting v2 login to registry endpoint %s", strings.TrimRight(endpoint.URL.String(), "/")+"/v2/")

	modifiers := DockerHeaders(userAgent, nil)
	authTransport := transport.NewTransport(NewTransport(endpoint.TLSConfig), modifiers...)

	challengeManager, foundV2, err := PingV2Registry(endpoint, authTransport)
	if err != nil {
		if !foundV2 {
			err = fallbackError{err: err}
		}
		return "", "", err
	}

	credentialAuthConfig := *authConfig
	creds := loginCredentialStore{
		authConfig: &credentialAuthConfig,
	}

	tokenHandlerOptions := auth.TokenHandlerOptions{
		Transport:     authTransport,
		Credentials:   creds,
		OfflineAccess: true,
		ClientID:      AuthClientID,
	}
	tokenHandler := auth.NewTokenHandlerWithOptions(tokenHandlerOptions)
	basicHandler := auth.NewBasicHandler(creds)
	modifiers = append(modifiers, auth.NewAuthorizer(challengeManager, tokenHandler, basicHandler))
	tr := transport.NewTransport(authTransport, modifiers...)

	loginClient := &http.Client{
		Transport: tr,
		Timeout:   15 * time.Second,
	}

	endpointStr := strings.TrimRight(endpoint.URL.String(), "/") + "/v2/"
	req, err := http.NewRequest("GET", endpointStr, nil)
	if err != nil {
		if !foundV2 {
			err = fallbackError{err: err}
		}
		return "", "", err
	}

	resp, err := loginClient.Do(req)
	if err != nil {
		if !foundV2 {
			err = fallbackError{err: err}
		}
		return "", "", err
	}
	defer resp.Body.Close()

	if resp.StatusCode != http.StatusOK {
		// TODO(dmcgowan): Attempt to further interpret result, status code and error code string
		err := fmt.Errorf("login attempt to %s failed with status: %d %s", endpointStr, resp.StatusCode, http.StatusText(resp.StatusCode))
		if !foundV2 {
			err = fallbackError{err: err}
		}
		return "", "", err
	}

	return "Login Succeeded", credentialAuthConfig.IdentityToken, nil

}

// ResolveAuthConfig matches an auth configuration to a server address or a URL
func ResolveAuthConfig(authConfigs map[string]types.AuthConfig, index *registrytypes.IndexInfo) types.AuthConfig {
	configKey := GetAuthConfigKey(index)
	// First try the happy case
	if c, found := authConfigs[configKey]; found || index.Official {
		return c
	}

	convertToHostname := func(url string) string {
		stripped := url
		if strings.HasPrefix(url, "http://") {
			stripped = strings.Replace(url, "http://", "", 1)
		} else if strings.HasPrefix(url, "https://") {
			stripped = strings.Replace(url, "https://", "", 1)
		}

		nameParts := strings.SplitN(stripped, "/", 2)

		return nameParts[0]
	}

	// Maybe they have a legacy config file, we will iterate the keys converting
	// them to the new format and testing
	for registry, ac := range authConfigs {
		if configKey == convertToHostname(registry) {
			return ac
		}
	}

	// When all else fails, return an empty auth config
	return types.AuthConfig{}
}

// PingResponseError is used when the response from a ping
// was received but invalid.
type PingResponseError struct {
	Err error
}

func (err PingResponseError) Error() string {
	return err.Error()
}

// PingV2Registry attempts to ping a v2 registry and on success return a
// challenge manager for the supported authentication types and
// whether v2 was confirmed by the response. If a response is received but
// cannot be interpreted a PingResponseError will be returned.
func PingV2Registry(endpoint APIEndpoint, transport http.RoundTripper) (auth.ChallengeManager, bool, error) {
	var (
		foundV2   = false
		v2Version = auth.APIVersion{
			Type:    "registry",
			Version: "2.0",
		}
	)

	pingClient := &http.Client{
		Transport: transport,
		Timeout:   15 * time.Second,
	}
	endpointStr := strings.TrimRight(endpoint.URL.String(), "/") + "/v2/"
	req, err := http.NewRequest("GET", endpointStr, nil)
	if err != nil {
		return nil, false, err
	}
	resp, err := pingClient.Do(req)
	if err != nil {
		return nil, false, err
	}
	defer resp.Body.Close()

	versions := auth.APIVersions(resp, DefaultRegistryVersionHeader)
	for _, pingVersion := range versions {
		if pingVersion == v2Version {
			// The version header indicates we're definitely
			// talking to a v2 registry. So don't allow future
			// fallbacks to the v1 protocol.

			foundV2 = true
			break
		}
	}

	challengeManager := auth.NewSimpleChallengeManager()
	if err := challengeManager.AddResponse(resp); err != nil {
		return nil, foundV2, PingResponseError{
			Err: err,
		}
	}

	return challengeManager, foundV2, nil
}
<|endoftext|>


<commit_before>/*
Copyright 2015 The Kubernetes Authors All rights reserved.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package pulls

import (
	"fmt"
	"strings"

	"k8s.io/contrib/mungegithub/opts"

	"github.com/golang/glog"
	"github.com/google/go-github/github"
)

type PRSizeMunger struct{}

func init() {
	RegisterMungerOrDie(PRSizeMunger{})
}

func (PRSizeMunger) Name() string { return "size" }

const labelSizePrefix = "size/"

func (PRSizeMunger) MungePullRequest(client *github.Client, pr *github.PullRequest, issue *github.Issue, commits []github.RepositoryCommit, events []github.IssueEvent, opts opts.MungeOptions) {
	if pr.Number == nil {
		glog.Warningf("PR has no Number: %+v", *pr)
		return
	}
	if pr.Additions == nil {
		glog.Warningf("PR %d has nil Additions", *pr.Number)
		return
	}
	if pr.Deletions == nil {
		glog.Warningf("PR %d has nil Deletions", *pr.Number)
		return
	}

	adds := *pr.Additions
	dels := *pr.Deletions

	for _, c := range commits {
		for _, f := range c.Files {
			if strings.HasPrefix(*f.Filename, "Godeps/") {
				adds = adds - *f.Additions
				dels = dels - *f.Deletions
				continue
			}
		}
	}

	newSize := calculateSize(adds, dels)
	newLabel := labelSizePrefix + newSize

	existing := GetLabelsWithPrefix(issue.Labels, labelSizePrefix)
	needsUpdate := true
	for _, l := range existing {
		if l == newLabel {
			needsUpdate = false
			continue
		}
		if opts.Dryrun {
			glog.Infof("PR #%d: would have removed label %s", *pr.Number, l)
		} else {
			if _, err := client.Issues.RemoveLabelForIssue(opts.Org, opts.Project, *pr.Number, l); err != nil {
				glog.Errorf("PR #%d: error removing label %q: %v", *pr.Number, l, err)
			}
		}
	}
	if needsUpdate {
		if opts.Dryrun {
			glog.Infof("PR #%d: would have added label %s", *pr.Number, newLabel)
		} else {
			if _, _, err := client.Issues.AddLabelsToIssue(opts.Org, opts.Project, *pr.Number, []string{newLabel}); err != nil {
				glog.Errorf("PR #%d: error adding label %q: %v", *pr.Number, newLabel, err)
			}
			body := fmt.Sprintf("Labelling this PR as %s", newLabel)
			if _, _, err := client.Issues.CreateComment(opts.Org, opts.Project, *pr.Number, &github.IssueComment{Body: &body}); err != nil {
				glog.Errorf("PR #%d: error adding comment: %v", *pr.Number, err)
			}
		}
	}
}

const (
	sizeXS  = "XS"
	sizeS   = "S"
	sizeM   = "M"
	sizeL   = "L"
	sizeXL  = "XL"
	sizeXXL = "XXL"
)

func calculateSize(adds, dels int) string {
	lines := adds + dels

	// This is a totally arbitrary heuristic and is open for tweaking.
	if lines < 10 {
		return sizeXS
	}
	if lines < 30 {
		return sizeS
	}
	if lines < 100 {
		return sizeM
	}
	if lines < 500 {
		return sizeL
	}
	if lines < 1000 {
		return sizeXL
	}
	return sizeXXL
}
<commit_msg>mungegithub: sizer: Ignore files which are automatically generated when sizing<commit_after>/*
Copyright 2015 The Kubernetes Authors All rights reserved.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package pulls

import (
	"fmt"
	"strings"

	"k8s.io/contrib/mungegithub/opts"

	"github.com/golang/glog"
	"github.com/google/go-github/github"
)

type PRSizeMunger struct{}

func init() {
	RegisterMungerOrDie(PRSizeMunger{})
}

func (PRSizeMunger) Name() string { return "size" }

const labelSizePrefix = "size/"

// getGeneratedFiles returns a list of all automatically generated files in the repo. These include
// docs, deep_copy, and conversions
func getGeneratedFiles(client *github.Client, opts opts.MungeOptions, c github.RepositoryCommit) []string {
	getOpts := &github.RepositoryContentGetOptions{Ref: *c.SHA}
	genFiles := []string{
		"pkg/api/v1/deep_copy_generated.go",
		"pkg/api/deep_copy_generated.go",
		"pkg/expapi/v1/deep_copy_generated.go",
		"pkg/expapi/deep_copy_generated.go",
		"pkg/api/v1/conversion_generated.go",
		"pkg/expapi/v1/conversion_generated.go",
		"api/swagger-spec/resourceListing.json",
		"api/swagger-spec/version.json",
		"api/swagger-spec/api.json",
		"api/swagger-spec/v1.json",
	}
	genDocs, _, _, err := client.Repositories.GetContents(opts.Org, opts.Project, ".generated_docs", getOpts)
	if err == nil && genDocs != nil {
		if b, err := genDocs.Decode(); err == nil {
			docs := strings.Split(string(b), "\n")
			genFiles = append(genFiles, docs...)
		}
	}
	return genFiles
}

func (PRSizeMunger) MungePullRequest(client *github.Client, pr *github.PullRequest, issue *github.Issue, commits []github.RepositoryCommit, events []github.IssueEvent, opts opts.MungeOptions) {
	if pr.Number == nil {
		glog.Warningf("PR has no Number: %+v", *pr)
		return
	}
	if pr.Additions == nil {
		glog.Warningf("PR %d has nil Additions", *pr.Number)
		return
	}
	if pr.Deletions == nil {
		glog.Warningf("PR %d has nil Deletions", *pr.Number)
		return
	}

	adds := *pr.Additions
	dels := *pr.Deletions

	// It would be 'better' to call this for every commit but that takes
	// a whole lot of time for almost always the same information, and if
	// our results are slightly wrong, who cares? Instead look for the
	// generated files once per PR and if someone changed both what files
	// are generated and then undid that change in an intermediate commit
	// we might call this PR bigger than we "should."
	genFiles := getGeneratedFiles(client, opts, commits[len(commits)-1])

	for _, c := range commits {
		for _, f := range c.Files {
			if strings.HasPrefix(*f.Filename, "Godeps/") {
				adds = adds - *f.Additions
				dels = dels - *f.Deletions
				continue
			}
			found := false
			for _, genFile := range genFiles {
				if *f.Filename == genFile {
					adds = adds - *f.Additions
					dels = dels - *f.Deletions
					found = true
					break
				}
			}
			if found {
				continue
			}
		}
	}

	newSize := calculateSize(adds, dels)
	newLabel := labelSizePrefix + newSize

	existing := GetLabelsWithPrefix(issue.Labels, labelSizePrefix)
	needsUpdate := true
	for _, l := range existing {
		if l == newLabel {
			if opts.Dryrun {
				glog.Infof("PR #%d: has label %s which is correct", *pr.Number, l)
			}
			needsUpdate = false
			continue
		}
		if opts.Dryrun {
			glog.Infof("PR #%d: would have removed label %s", *pr.Number, l)
		} else {
			if _, err := client.Issues.RemoveLabelForIssue(opts.Org, opts.Project, *pr.Number, l); err != nil {
				glog.Errorf("PR #%d: error removing label %q: %v", *pr.Number, l, err)
			}
		}
	}
	if needsUpdate {
		if opts.Dryrun {
			glog.Infof("PR #%d: would have added label %s", *pr.Number, newLabel)
		} else {
			if _, _, err := client.Issues.AddLabelsToIssue(opts.Org, opts.Project, *pr.Number, []string{newLabel}); err != nil {
				glog.Errorf("PR #%d: error adding label %q: %v", *pr.Number, newLabel, err)
			}
			body := fmt.Sprintf("Labelling this PR as %s", newLabel)
			if _, _, err := client.Issues.CreateComment(opts.Org, opts.Project, *pr.Number, &github.IssueComment{Body: &body}); err != nil {
				glog.Errorf("PR #%d: error adding comment: %v", *pr.Number, err)
			}
		}
	}
}

const (
	sizeXS  = "XS"
	sizeS   = "S"
	sizeM   = "M"
	sizeL   = "L"
	sizeXL  = "XL"
	sizeXXL = "XXL"
)

func calculateSize(adds, dels int) string {
	lines := adds + dels

	// This is a totally arbitrary heuristic and is open for tweaking.
	if lines < 10 {
		return sizeXS
	}
	if lines < 30 {
		return sizeS
	}
	if lines < 100 {
		return sizeM
	}
	if lines < 500 {
		return sizeL
	}
	if lines < 1000 {
		return sizeXL
	}
	return sizeXXL
}
<|endoftext|>


<commit_before>package rel

type SelectManager struct {
	Engine Engine
	Ast    SelectStatementNode
	Ctx    *SelectCoreNode
	BaseNode
}

func NewSelectManager(t *Table) SelectManager {
	stmt := SelectStatementNode{Cores: make([]*SelectCoreNode, 0)}
	core := NewSelectCoreNode()
	stmt.Cores = append(stmt.Cores, &core)
	ctx := stmt.Cores[len(stmt.Cores)-1]
	manager := SelectManager{
		Engine: t.Engine,
		Ast:    stmt,
		Ctx:    ctx,
	}
	manager.From(t)
	return manager
}

func (s *SelectManager) ToSql() string {
	return s.Engine.Visitor().Accept(s.Ast)
}

func (s *SelectManager) Project(projections ...AstNode) *SelectManager {
	for _, projection := range projections {
		if s.Ctx.Projections == nil {
			nodeslice := make([]AstNode, 0)
			s.Ctx.Projections = &nodeslice
		}

		if s.Ctx.Projections != nil {
			*s.Ctx.Projections = append(*s.Ctx.Projections, projection)
		}
	}
	return s
}

func (s *SelectManager) From(t *Table) *SelectManager {
	s.Ctx.Source.Left = t
	return s
}

func (s *SelectManager) Join(a ...AstNode) *SelectManager {
	return s
}

func (s *SelectManager) Take(i int) *SelectManager {
	limit := NewLimitNode(Sql(i))
	s.Ast.Limit = &limit
	return s
}

func (s *SelectManager) Exists() ExistsNode {
	return NewExistsNode(s.Ast)
}

func (s *SelectManager) Order(exprs ...string) *SelectManager {
	if len(exprs) > 0 {
		if s.Ast.Orders == nil {
			orders := make([]AstNode, 0)
			s.Ast.Orders = &orders
		}
		for _, expr := range exprs {
			order := NewSqlLiteralNode(expr)
			*s.Ast.Orders = append(*s.Ast.Orders, order)
		}
	}
	return s
}

func (s *SelectManager) Where(n AstNode) *SelectManager {
	if s.Ctx.Wheres == nil {
		wheres := make([]AstNode, 0)
		s.Ctx.Wheres = &wheres
	}
	*s.Ctx.Wheres = append(*s.Ctx.Wheres, n)
	return s
}

func (s *SelectManager) Group(columns ...AstNode) *SelectManager {
	var group GroupNode
	if len(columns) > 0 {
		if s.Ctx.Groups == nil {
			groups := make([]GroupNode, 0)
			s.Ctx.Groups = &groups
		}
		for _, column := range columns {
			group = NewGroupNode(column)
			*s.Ctx.Groups = append(*s.Ctx.Groups, group)
		}
	}
	return s
}

func (s *SelectManager) Skip(i int) *SelectManager {
	offset := NewOffsetNode(Sql(i))
	s.Ast.Offset = &offset
	return s
}

func (s *SelectManager) Offset(i int) *SelectManager {
	return s.Skip(i)
}

func (s *SelectManager) Having(a ...AstNode) *SelectManager {
	var b AstNode

	// use the first Node if there is only one
	// else create and And node
	if len(a) == 1 {
		b = a[0]
	} else {
		b = s.NewAndNode(a...)
	}

	// pass in an AstNode
	having := NewHavingNode(b)
	s.Ctx.Having = &having
	return s
}
<commit_msg>Allow a SelectManager to be passed into a SelectManager#Where<commit_after>package rel

type SelectManager struct {
	Engine Engine
	Ast    SelectStatementNode
	Ctx    *SelectCoreNode
	BaseNode
}

func NewSelectManager(t *Table) SelectManager {
	stmt := SelectStatementNode{Cores: make([]*SelectCoreNode, 0)}
	core := NewSelectCoreNode()
	stmt.Cores = append(stmt.Cores, &core)
	ctx := stmt.Cores[len(stmt.Cores)-1]
	manager := SelectManager{
		Engine: t.Engine,
		Ast:    stmt,
		Ctx:    ctx,
	}
	manager.From(t)
	return manager
}

func (s *SelectManager) ToSql() string {
	return s.Engine.Visitor().Accept(s.Ast)
}

func (s *SelectManager) Project(projections ...AstNode) *SelectManager {
	for _, projection := range projections {
		if s.Ctx.Projections == nil {
			nodeslice := make([]AstNode, 0)
			s.Ctx.Projections = &nodeslice
		}

		if s.Ctx.Projections != nil {
			*s.Ctx.Projections = append(*s.Ctx.Projections, projection)
		}
	}
	return s
}

func (s *SelectManager) From(t *Table) *SelectManager {
	s.Ctx.Source.Left = t
	return s
}

func (s *SelectManager) Join(a ...AstNode) *SelectManager {
	return s
}

func (s *SelectManager) Take(i int) *SelectManager {
	limit := NewLimitNode(Sql(i))
	s.Ast.Limit = &limit
	return s
}

func (s *SelectManager) Exists() ExistsNode {
	return NewExistsNode(s.Ast)
}

func (s *SelectManager) Order(exprs ...string) *SelectManager {
	if len(exprs) > 0 {
		if s.Ast.Orders == nil {
			orders := make([]AstNode, 0)
			s.Ast.Orders = &orders
		}
		for _, expr := range exprs {
			order := NewSqlLiteralNode(expr)
			*s.Ast.Orders = append(*s.Ast.Orders, order)
		}
	}
	return s
}

func (s *SelectManager) Where(n AstNode) *SelectManager {
	if s.Ctx.Wheres == nil {
		wheres := make([]AstNode, 0)
		s.Ctx.Wheres = &wheres
	}

	if expr, ok := n.(SelectManager); ok {
		*s.Ctx.Wheres = append(*s.Ctx.Wheres, expr.Ast)
	} else {
		*s.Ctx.Wheres = append(*s.Ctx.Wheres, n)
	}

	return s
}

func (s *SelectManager) Group(columns ...AstNode) *SelectManager {
	var group GroupNode
	if len(columns) > 0 {
		if s.Ctx.Groups == nil {
			groups := make([]GroupNode, 0)
			s.Ctx.Groups = &groups
		}
		for _, column := range columns {
			group = NewGroupNode(column)
			*s.Ctx.Groups = append(*s.Ctx.Groups, group)
		}
	}
	return s
}

func (s *SelectManager) Skip(i int) *SelectManager {
	offset := NewOffsetNode(Sql(i))
	s.Ast.Offset = &offset
	return s
}

func (s *SelectManager) Offset(i int) *SelectManager {
	return s.Skip(i)
}

func (s *SelectManager) Having(a ...AstNode) *SelectManager {
	var b AstNode

	// use the first Node if there is only one
	// else create and And node
	if len(a) == 1 {
		b = a[0]
	} else {
		b = s.NewAndNode(a...)
	}

	// pass in an AstNode
	having := NewHavingNode(b)
	s.Ctx.Having = &having
	return s
}
<|endoftext|>


<commit_before>// Copyright 2015 Google Inc. All Rights Reserved.
// Author: jacobsa@google.com (Aaron Jacobs)

package fuseutil

import (
	"time"

	"golang.org/x/net/context"
)

// An interface that must be implemented by file systems to be mounted with
// FUSE. See also the comments on request and response structs.
//
// Not all methods need to have interesting implementations. Embed a field of
// type NotImplementedFileSystem to inherit defaults that return ENOSYS to the
// kernel.
type FileSystem interface {
	// Look up a child by name within a parent directory. The kernel calls this
	// when resolving user paths to dentry structs, which are then cached.
	Lookup(
		ctx context.Context,
		req *LookupRequest) (*LookupResponse, error)

	// Forget an inode ID previously issued (e.g. by Lookup). The kernel calls
	// this when removing an inode from its internal caches.
	//
	// The kernel guarantees that the node ID will not be used in further calls
	// to the file system (unless it is reissued by the file system).
	Forget(
		ctx context.Context,
		req *ForgetRequest) (*ForgetResponse, error)
}

////////////////////////////////////////////////////////////////////////
// Simple types
////////////////////////////////////////////////////////////////////////

// A 64-bit number used to uniquely identify a file or directory in the file
// system.
//
// This corresponds to struct inode::i_no in the VFS layer.
// (Cf. http://goo.gl/tvYyQt)
type InodeID uint64

// A generation number for an inode. Irrelevant for file systems that won't be
// exported over NFS. For those that will and that reuse inode IDs when they
// become free, the generation number must change when an ID is reused.
//
// This corresponds to struct inode::i_generation in the VFS layer.
// (Cf. http://goo.gl/tvYyQt)
//
// Some related reading:
//
//     http://fuse.sourceforge.net/doxygen/structfuse__entry__param.html
//     http://stackoverflow.com/q/11071996/1505451
//     http://goo.gl/CqvwyX
//     http://julipedia.meroh.net/2005/09/nfs-file-handles.html
//     http://goo.gl/wvo3MB
//
type GenerationNumber uint64

// Attributes for a file or directory inode. Corresponds to struct inode (cf.
// http://goo.gl/tvYyQt).
type InodeAttributes struct {
	// The size of the file in bytes.
	Size uint64
}

////////////////////////////////////////////////////////////////////////
// Requests and responses
////////////////////////////////////////////////////////////////////////

type LookupRequest struct {
	// The ID of the directory inode to which the child belongs.
	Parent InodeID

	// The name of the child of interest, relative to the parent. For example, in
	// this directory structure:
	//
	//     foo/
	//         bar/
	//             baz
	//
	// the file system may receive a request to look up the child named "bar" for
	// the parent foo/.
	Name string
}

type LookupResponse struct {
	// The ID of the child inode. The file system must ensure that the returned
	// inode ID remains valid until a later call to Forget.
	Child InodeID

	// A generation number for this incarnation of the inode with the given ID.
	// See comments on type GenerationNumber for more.
	Generation GenerationNumber

	// Current ttributes for the child inode.
	Attributes InodeAttributes

	// The FUSE VFS layer in the kernel maintains a cache of file attributes,
	// used whenever up to date information about size, mode, etc. is needed.
	//
	// For example, this is the abridged call chain for fstat(2):
	//
	//  *  (http://goo.gl/tKBH1p) fstat calls vfs_fstat.
	//  *  (http://goo.gl/3HeITq) vfs_fstat eventuall calls vfs_getattr_nosec.
	//  *  (http://goo.gl/DccFQr) vfs_getattr_nosec calls i_op->getattr.
	//  *  (http://goo.gl/dpKkst) fuse_getattr calls fuse_update_attributes.
	//  *  (http://goo.gl/yNlqPw) fuse_update_attributes uses the values in the
	//     struct inode if allowed, otherwise calling out to the user-space code.
	//
	// In addition to obvious cases like fstat, this is also used in more subtle
	// cases like updating size information before seeking (http://goo.gl/2nnMFa)
	// or reading (http://goo.gl/FQSWs8).
	//
	// Most 'real' file systems do not set inode_operations::getattr, and
	// therefore vfs_getattr_nosec calls generic_fillattr which simply grabs the
	// information from the inode struct. This makes sense because these file
	// systems cannot spontaneously change; all modifications go through the
	// kernel which can update the inode struct as appropriate.
	//
	// In contrast, a FUSE file system may have spontaneous changes, so it calls
	// out to user space to fetch attributes. However this is expensive, so the
	// FUSE layer in the kernel caches the attributes if requested.
	//
	// This field controls when the attributes returned in this response and
	// stashed in the struct inode should be re-queried. Leave at the zero value
	// to disable caching.
	//
	// More reading:
	//     http://stackoverflow.com/q/21540315/1505451
	AttributesExpiration time.Time

	// The time until which the kernel may maintain an entry for this name to
	// inode mapping in its dentry cache. After this time, it will revalidate the
	// dentry.
	//
	// As in the discussion of attribute caching above, unlike real file systems,
	// FUSE file systems may spontaneously change their name -> inode mapping.
	// Therefore the FUSE VFS layer uses dentry_operations::d_revalidate
	// (http://goo.gl/dVea0h) to intercept lookups and revalidate by calling the
	// user-space Lookup method. However the latter may be slow, so it caches the
	// entries until the time defined by this field.
	//
	// Example code walk:
	//
	//     * (http://goo.gl/M2G3tO) lookup_dcache calls d_revalidate if enabled.
	//     * (http://goo.gl/ef0Elu) fuse_dentry_revalidate just uses the dentry's
	//     inode if fuse_dentry_time(entry) hasn't passed. Otherwise it sends a
	//     lookup request.
	//
	// Leave at the zero value to disable caching.
	EntryExpiration time.Time
}

type ForgetRequest struct {
	// The inode to be forgotten. The kernel guarantees that the node ID will not
	// be used in further calls to the file system (unless it is reissued by the
	// file system).
	ID InodeID
}

type ForgetResponse struct {
}
<commit_msg>Added concurrency requirements.<commit_after>// Copyright 2015 Google Inc. All Rights Reserved.
// Author: jacobsa@google.com (Aaron Jacobs)

package fuseutil

import (
	"time"

	"golang.org/x/net/context"
)

// An interface that must be implemented by file systems to be mounted with
// FUSE. See also the comments on request and response structs.
//
// Not all methods need to have interesting implementations. Embed a field of
// type NotImplementedFileSystem to inherit defaults that return ENOSYS to the
// kernel.
//
// Must be safe for concurrent access via all methods.
type FileSystem interface {
	// Look up a child by name within a parent directory. The kernel calls this
	// when resolving user paths to dentry structs, which are then cached.
	Lookup(
		ctx context.Context,
		req *LookupRequest) (*LookupResponse, error)

	// Forget an inode ID previously issued (e.g. by Lookup). The kernel calls
	// this when removing an inode from its internal caches.
	//
	// The kernel guarantees that the node ID will not be used in further calls
	// to the file system (unless it is reissued by the file system).
	Forget(
		ctx context.Context,
		req *ForgetRequest) (*ForgetResponse, error)
}

////////////////////////////////////////////////////////////////////////
// Simple types
////////////////////////////////////////////////////////////////////////

// A 64-bit number used to uniquely identify a file or directory in the file
// system.
//
// This corresponds to struct inode::i_no in the VFS layer.
// (Cf. http://goo.gl/tvYyQt)
type InodeID uint64

// A generation number for an inode. Irrelevant for file systems that won't be
// exported over NFS. For those that will and that reuse inode IDs when they
// become free, the generation number must change when an ID is reused.
//
// This corresponds to struct inode::i_generation in the VFS layer.
// (Cf. http://goo.gl/tvYyQt)
//
// Some related reading:
//
//     http://fuse.sourceforge.net/doxygen/structfuse__entry__param.html
//     http://stackoverflow.com/q/11071996/1505451
//     http://goo.gl/CqvwyX
//     http://julipedia.meroh.net/2005/09/nfs-file-handles.html
//     http://goo.gl/wvo3MB
//
type GenerationNumber uint64

// Attributes for a file or directory inode. Corresponds to struct inode (cf.
// http://goo.gl/tvYyQt).
type InodeAttributes struct {
	// The size of the file in bytes.
	Size uint64
}

////////////////////////////////////////////////////////////////////////
// Requests and responses
////////////////////////////////////////////////////////////////////////

type LookupRequest struct {
	// The ID of the directory inode to which the child belongs.
	Parent InodeID

	// The name of the child of interest, relative to the parent. For example, in
	// this directory structure:
	//
	//     foo/
	//         bar/
	//             baz
	//
	// the file system may receive a request to look up the child named "bar" for
	// the parent foo/.
	Name string
}

type LookupResponse struct {
	// The ID of the child inode. The file system must ensure that the returned
	// inode ID remains valid until a later call to Forget.
	Child InodeID

	// A generation number for this incarnation of the inode with the given ID.
	// See comments on type GenerationNumber for more.
	Generation GenerationNumber

	// Current ttributes for the child inode.
	Attributes InodeAttributes

	// The FUSE VFS layer in the kernel maintains a cache of file attributes,
	// used whenever up to date information about size, mode, etc. is needed.
	//
	// For example, this is the abridged call chain for fstat(2):
	//
	//  *  (http://goo.gl/tKBH1p) fstat calls vfs_fstat.
	//  *  (http://goo.gl/3HeITq) vfs_fstat eventuall calls vfs_getattr_nosec.
	//  *  (http://goo.gl/DccFQr) vfs_getattr_nosec calls i_op->getattr.
	//  *  (http://goo.gl/dpKkst) fuse_getattr calls fuse_update_attributes.
	//  *  (http://goo.gl/yNlqPw) fuse_update_attributes uses the values in the
	//     struct inode if allowed, otherwise calling out to the user-space code.
	//
	// In addition to obvious cases like fstat, this is also used in more subtle
	// cases like updating size information before seeking (http://goo.gl/2nnMFa)
	// or reading (http://goo.gl/FQSWs8).
	//
	// Most 'real' file systems do not set inode_operations::getattr, and
	// therefore vfs_getattr_nosec calls generic_fillattr which simply grabs the
	// information from the inode struct. This makes sense because these file
	// systems cannot spontaneously change; all modifications go through the
	// kernel which can update the inode struct as appropriate.
	//
	// In contrast, a FUSE file system may have spontaneous changes, so it calls
	// out to user space to fetch attributes. However this is expensive, so the
	// FUSE layer in the kernel caches the attributes if requested.
	//
	// This field controls when the attributes returned in this response and
	// stashed in the struct inode should be re-queried. Leave at the zero value
	// to disable caching.
	//
	// More reading:
	//     http://stackoverflow.com/q/21540315/1505451
	AttributesExpiration time.Time

	// The time until which the kernel may maintain an entry for this name to
	// inode mapping in its dentry cache. After this time, it will revalidate the
	// dentry.
	//
	// As in the discussion of attribute caching above, unlike real file systems,
	// FUSE file systems may spontaneously change their name -> inode mapping.
	// Therefore the FUSE VFS layer uses dentry_operations::d_revalidate
	// (http://goo.gl/dVea0h) to intercept lookups and revalidate by calling the
	// user-space Lookup method. However the latter may be slow, so it caches the
	// entries until the time defined by this field.
	//
	// Example code walk:
	//
	//     * (http://goo.gl/M2G3tO) lookup_dcache calls d_revalidate if enabled.
	//     * (http://goo.gl/ef0Elu) fuse_dentry_revalidate just uses the dentry's
	//     inode if fuse_dentry_time(entry) hasn't passed. Otherwise it sends a
	//     lookup request.
	//
	// Leave at the zero value to disable caching.
	EntryExpiration time.Time
}

type ForgetRequest struct {
	// The inode to be forgotten. The kernel guarantees that the node ID will not
	// be used in further calls to the file system (unless it is reissued by the
	// file system).
	ID InodeID
}

type ForgetResponse struct {
}
<|endoftext|>


<commit_before>// Copyright 2015 The LUCI Authors. All rights reserved.
// Use of this source code is governed under the Apache License, Version 2.0
// that can be found in the LICENSE file.

package memory

import (
	"fmt"
	"regexp"

	"github.com/luci/gae/impl/dummy"
	"github.com/luci/gae/service/info"
	"golang.org/x/net/context"
)

type giContextKeyType int

var giContextKey giContextKeyType

// validNamespace matches valid namespace names.
var validNamespace = regexp.MustCompile(`^[0-9A-Za-z._-]{0,100}$`)

var defaultGlobalInfoData = globalInfoData{
	// versionID returns X.Y where Y is autogenerated by appengine, and X is
	// whatever's in app.yaml.
	versionID: "testVersionID.1",
	requestID: "test-request-id",
}

type globalInfoData struct {
	appID     string
	fqAppID   string
	namespace *string
	versionID string
	requestID string
}

func (gid *globalInfoData) getNamespace() (string, bool) {
	if ns := gid.namespace; ns != nil {
		return *ns, true
	}
	return "", false
}

func curGID(c context.Context) *globalInfoData {
	if gid, ok := c.Value(giContextKey).(*globalInfoData); ok {
		return gid
	}
	return &defaultGlobalInfoData
}

func useGID(c context.Context, f func(mod *globalInfoData)) context.Context {
	cur := curGID(c)
	if cur == nil {
		cur = &defaultGlobalInfoData
	}

	clone := *cur
	f(&clone)
	return context.WithValue(c, giContextKey, &clone)
}

// useGI adds a gae.GlobalInfo context, accessible
// by gae.GetGI(c)
func useGI(c context.Context) context.Context {
	return info.SetFactory(c, func(ic context.Context) info.Interface {
		return &giImpl{dummy.Info(), curGID(ic), ic}
	})
}

type giImpl struct {
	info.Interface
	*globalInfoData
	c context.Context
}

var _ = info.Interface((*giImpl)(nil))

func (gi *giImpl) GetNamespace() (string, bool) {
	return gi.getNamespace()
}

func (gi *giImpl) Namespace(ns string) (ret context.Context, err error) {
	if !validNamespace.MatchString(ns) {
		return nil, fmt.Errorf("appengine: namespace %q does not match /%s/", ns, validNamespace)
	}

	return useGID(gi.c, func(mod *globalInfoData) {
		mod.namespace = &ns
	}), nil
}

func (gi *giImpl) MustNamespace(ns string) context.Context {
	ret, err := gi.Namespace(ns)
	if err != nil {
		panic(err)
	}
	return ret
}

func (gi *giImpl) AppID() string {
	return gi.appID
}

func (gi *giImpl) FullyQualifiedAppID() string {
	return gi.fqAppID
}

func (gi *giImpl) IsDevAppServer() bool {
	return true
}

func (gi *giImpl) VersionID() string {
	return curGID(gi.c).versionID
}

func (gi *giImpl) RequestID() string {
	return curGID(gi.c).requestID
}

func (gi *giImpl) Testable() info.Testable {
	return gi
}

func (gi *giImpl) SetVersionID(v string) context.Context {
	return useGID(gi.c, func(mod *globalInfoData) {
		mod.versionID = v
	})
}

func (gi *giImpl) SetRequestID(v string) context.Context {
	return useGID(gi.c, func(mod *globalInfoData) {
		mod.requestID = v
	})
}
<commit_msg>Add DefaultVersionHostname to impl/memory/info.<commit_after>// Copyright 2015 The LUCI Authors. All rights reserved.
// Use of this source code is governed under the Apache License, Version 2.0
// that can be found in the LICENSE file.

package memory

import (
	"fmt"
	"regexp"

	"github.com/luci/gae/impl/dummy"
	"github.com/luci/gae/service/info"
	"golang.org/x/net/context"
)

type giContextKeyType int

var giContextKey giContextKeyType

// validNamespace matches valid namespace names.
var validNamespace = regexp.MustCompile(`^[0-9A-Za-z._-]{0,100}$`)

var defaultGlobalInfoData = globalInfoData{
	// versionID returns X.Y where Y is autogenerated by appengine, and X is
	// whatever's in app.yaml.
	versionID: "testVersionID.1",
	requestID: "test-request-id",
}

type globalInfoData struct {
	appID     string
	fqAppID   string
	namespace *string
	versionID string
	requestID string
}

func (gid *globalInfoData) getNamespace() (string, bool) {
	if ns := gid.namespace; ns != nil {
		return *ns, true
	}
	return "", false
}

func curGID(c context.Context) *globalInfoData {
	if gid, ok := c.Value(giContextKey).(*globalInfoData); ok {
		return gid
	}
	return &defaultGlobalInfoData
}

func useGID(c context.Context, f func(mod *globalInfoData)) context.Context {
	cur := curGID(c)
	if cur == nil {
		cur = &defaultGlobalInfoData
	}

	clone := *cur
	f(&clone)
	return context.WithValue(c, giContextKey, &clone)
}

// useGI adds a gae.GlobalInfo context, accessible
// by gae.GetGI(c)
func useGI(c context.Context) context.Context {
	return info.SetFactory(c, func(ic context.Context) info.Interface {
		return &giImpl{dummy.Info(), curGID(ic), ic}
	})
}

type giImpl struct {
	info.Interface
	*globalInfoData
	c context.Context
}

var _ = info.Interface((*giImpl)(nil))

func (gi *giImpl) GetNamespace() (string, bool) {
	return gi.getNamespace()
}

func (gi *giImpl) Namespace(ns string) (ret context.Context, err error) {
	if !validNamespace.MatchString(ns) {
		return nil, fmt.Errorf("appengine: namespace %q does not match /%s/", ns, validNamespace)
	}

	return useGID(gi.c, func(mod *globalInfoData) {
		mod.namespace = &ns
	}), nil
}

func (gi *giImpl) MustNamespace(ns string) context.Context {
	ret, err := gi.Namespace(ns)
	if err != nil {
		panic(err)
	}
	return ret
}

func (gi *giImpl) AppID() string {
	return gi.appID
}

func (gi *giImpl) FullyQualifiedAppID() string {
	return gi.fqAppID
}

func (gi *giImpl) DefaultVersionHostname() string {
	return fmt.Sprintf("%s.example.com", gi.appID)
}

func (gi *giImpl) IsDevAppServer() bool {
	return true
}

func (gi *giImpl) VersionID() string {
	return curGID(gi.c).versionID
}

func (gi *giImpl) RequestID() string {
	return curGID(gi.c).requestID
}

func (gi *giImpl) Testable() info.Testable {
	return gi
}

func (gi *giImpl) SetVersionID(v string) context.Context {
	return useGID(gi.c, func(mod *globalInfoData) {
		mod.versionID = v
	})
}

func (gi *giImpl) SetRequestID(v string) context.Context {
	return useGID(gi.c, func(mod *globalInfoData) {
		mod.requestID = v
	})
}
<|endoftext|>


<commit_before>package namesys

import (
	"context"
	"strings"
	"time"

	opts "github.com/ipfs/go-ipfs/namesys/opts"
	pb "github.com/ipfs/go-ipfs/namesys/pb"
	path "github.com/ipfs/go-ipfs/path"

	u "gx/ipfs/QmNiJuT8Ja3hMVpBHXv3Q6dwmperaQ6JjLtpMQgMCD7xvx/go-ipfs-util"
	logging "gx/ipfs/QmRb5jh8z2E8hMGN2tkvs1yHynUanqnZ3UeKwgN1i9P1F8/go-log"
	routing "gx/ipfs/QmTiWLZ6Fo5j4KcTVutZJ5KWRRJrbxzmxA4td8NfEdrPh7/go-libp2p-routing"
	lru "gx/ipfs/QmVYxfoJQiZijTgPNHCHgHELvQpbsJNTg6Crmc3dQkj3yy/golang-lru"
	proto "gx/ipfs/QmZ4Qi3GaRbjcx28Sme5eMH7RQjGkt8wHxt2a65oLaeFEV/gogo-protobuf/proto"
	peer "gx/ipfs/QmZoWKhxUmZ2seW4BzX6fJkNR8hh9PsGModr7q171yq2SS/go-libp2p-peer"
	mh "gx/ipfs/QmZyZDi491cCNTLfAhwcaDii2Kg4pwKRkhqQzURGDvY6ua/go-multihash"
	cid "gx/ipfs/QmcZfnkapfECQGcLZaf9B79NRg7cRa9EnZh4LSbkCzwNvY/go-cid"
)

var log = logging.Logger("namesys")

// routingResolver implements NSResolver for the main IPFS SFS-like naming
type routingResolver struct {
	routing routing.ValueStore

	cache *lru.Cache
}

func (r *routingResolver) cacheGet(name string) (path.Path, bool) {
	if r.cache == nil {
		return "", false
	}

	ientry, ok := r.cache.Get(name)
	if !ok {
		return "", false
	}

	entry, ok := ientry.(cacheEntry)
	if !ok {
		// should never happen, purely for sanity
		log.Panicf("unexpected type %T in cache for %q.", ientry, name)
	}

	if time.Now().Before(entry.eol) {
		return entry.val, true
	}

	r.cache.Remove(name)

	return "", false
}

func (r *routingResolver) cacheSet(name string, val path.Path, rec *pb.IpnsEntry) {
	if r.cache == nil {
		return
	}

	// if completely unspecified, just use one minute
	ttl := DefaultResolverCacheTTL
	if rec.Ttl != nil {
		recttl := time.Duration(rec.GetTtl())
		if recttl >= 0 {
			ttl = recttl
		}
	}

	cacheTil := time.Now().Add(ttl)
	eol, ok := checkEOL(rec)
	if ok && eol.Before(cacheTil) {
		cacheTil = eol
	}

	r.cache.Add(name, cacheEntry{
		val: val,
		eol: cacheTil,
	})
}

type cacheEntry struct {
	val path.Path
	eol time.Time
}

// NewRoutingResolver constructs a name resolver using the IPFS Routing system
// to implement SFS-like naming on top.
// cachesize is the limit of the number of entries in the lru cache. Setting it
// to '0' will disable caching.
func NewRoutingResolver(route routing.ValueStore, cachesize int) *routingResolver {
	if route == nil {
		panic("attempt to create resolver with nil routing system")
	}

	var cache *lru.Cache
	if cachesize > 0 {
		cache, _ = lru.New(cachesize)
	}

	return &routingResolver{
		routing: route,
		cache:   cache,
	}
}

// Resolve implements Resolver.
func (r *routingResolver) Resolve(ctx context.Context, name string, options ...opts.ResolveOpt) (path.Path, error) {
	return resolve(ctx, r, name, opts.ProcessOpts(options), "/ipns/")
}

// resolveOnce implements resolver. Uses the IPFS routing system to
// resolve SFS-like names.
func (r *routingResolver) resolveOnce(ctx context.Context, name string, options *opts.ResolveOpts) (path.Path, error) {
	log.Debugf("RoutingResolver resolving %s", name)
	cached, ok := r.cacheGet(name)
	if ok {
		return cached, nil
	}

	if options.DhtTimeout != 0 {
		// Resolution must complete within the timeout
		var cancel context.CancelFunc
		ctx, cancel = context.WithTimeout(ctx, options.DhtTimeout)
		defer cancel()
	}

	name = strings.TrimPrefix(name, "/ipns/")
	hash, err := mh.FromB58String(name)
	if err != nil {
		// name should be a multihash. if it isn't, error out here.
		log.Debugf("RoutingResolver: bad input hash: [%s]\n", name)
		return "", err
	}

	// Name should be the hash of a public key retrievable from ipfs.
	// We retrieve the public key here to make certain that it's in the peer
	// store before calling GetValue() on the DHT - the DHT will call the
	// ipns validator, which in turn will get the public key from the peer
	// store to verify the record signature
	_, err = routing.GetPublicKey(r.routing, ctx, hash)
	if err != nil {
		log.Debugf("RoutingResolver: could not retrieve public key %s: %s\n", name, err)
		return "", err
	}

	pid, err := peer.IDFromBytes(hash)
	if err != nil {
		log.Debugf("RoutingResolver: could not convert public key hash %s to peer ID: %s\n", name, err)
		return "", err
	}

	// Use the routing system to get the name.
	// Note that the DHT will call the ipns validator when retrieving
	// the value, which in turn verifies the ipns record signature
	_, ipnsKey := IpnsKeysForID(pid)
	val, err := r.getValue(ctx, ipnsKey, options)
	if err != nil {
		log.Debugf("RoutingResolver: dht get for name %s failed: %s", name, err)
		return "", err
	}

	entry := new(pb.IpnsEntry)
	err = proto.Unmarshal(val, entry)
	if err != nil {
		log.Debugf("RoutingResolver: could not unmarshal value for name %s: %s", name, err)
		return "", err
	}

	// check for old style record:
	valh, err := mh.Cast(entry.GetValue())
	if err != nil {
		// Not a multihash, probably a new record
		p, err := path.ParsePath(string(entry.GetValue()))
		if err != nil {
			return "", err
		}

		r.cacheSet(name, p, entry)
		return p, nil
	} else {
		// Its an old style multihash record
		log.Debugf("encountered CIDv0 ipns entry: %s", valh)
		p := path.FromCid(cid.NewCidV0(valh))
		r.cacheSet(name, p, entry)
		return p, nil
	}
}

func (r *routingResolver) getValue(ctx context.Context, ipnsKey string, options *opts.ResolveOpts) ([]byte, error) {
	// Get specified number of values from the DHT
	vals, err := r.routing.GetValues(ctx, ipnsKey, int(options.DhtRecordCount))
	if err != nil {
		return nil, err
	}

	// Select the best value
	recs := make([][]byte, 0, len(vals))
	for _, v := range vals {
		if v.Val != nil {
			recs = append(recs, v.Val)
		}
	}

	i, err := IpnsSelectorFunc(ipnsKey, recs)
	if err != nil {
		return nil, err
	}

	best := recs[i]
	if best == nil {
		log.Errorf("GetValues %s yielded record with nil value", ipnsKey)
		return nil, routing.ErrNotFound
	}

	return best, nil
}

func checkEOL(e *pb.IpnsEntry) (time.Time, bool) {
	if e.GetValidityType() == pb.IpnsEntry_EOL {
		eol, err := u.ParseRFC3339(string(e.GetValidity()))
		if err != nil {
			return time.Time{}, false
		}
		return eol, true
	}
	return time.Time{}, false
}
<commit_msg>Return ErrNotFound when zero values are returned from DHT namesys resolve<commit_after>package namesys

import (
	"context"
	"strings"
	"time"

	opts "github.com/ipfs/go-ipfs/namesys/opts"
	pb "github.com/ipfs/go-ipfs/namesys/pb"
	path "github.com/ipfs/go-ipfs/path"

	u "gx/ipfs/QmNiJuT8Ja3hMVpBHXv3Q6dwmperaQ6JjLtpMQgMCD7xvx/go-ipfs-util"
	logging "gx/ipfs/QmRb5jh8z2E8hMGN2tkvs1yHynUanqnZ3UeKwgN1i9P1F8/go-log"
	routing "gx/ipfs/QmTiWLZ6Fo5j4KcTVutZJ5KWRRJrbxzmxA4td8NfEdrPh7/go-libp2p-routing"
	lru "gx/ipfs/QmVYxfoJQiZijTgPNHCHgHELvQpbsJNTg6Crmc3dQkj3yy/golang-lru"
	proto "gx/ipfs/QmZ4Qi3GaRbjcx28Sme5eMH7RQjGkt8wHxt2a65oLaeFEV/gogo-protobuf/proto"
	peer "gx/ipfs/QmZoWKhxUmZ2seW4BzX6fJkNR8hh9PsGModr7q171yq2SS/go-libp2p-peer"
	mh "gx/ipfs/QmZyZDi491cCNTLfAhwcaDii2Kg4pwKRkhqQzURGDvY6ua/go-multihash"
	cid "gx/ipfs/QmcZfnkapfECQGcLZaf9B79NRg7cRa9EnZh4LSbkCzwNvY/go-cid"
)

var log = logging.Logger("namesys")

// routingResolver implements NSResolver for the main IPFS SFS-like naming
type routingResolver struct {
	routing routing.ValueStore

	cache *lru.Cache
}

func (r *routingResolver) cacheGet(name string) (path.Path, bool) {
	if r.cache == nil {
		return "", false
	}

	ientry, ok := r.cache.Get(name)
	if !ok {
		return "", false
	}

	entry, ok := ientry.(cacheEntry)
	if !ok {
		// should never happen, purely for sanity
		log.Panicf("unexpected type %T in cache for %q.", ientry, name)
	}

	if time.Now().Before(entry.eol) {
		return entry.val, true
	}

	r.cache.Remove(name)

	return "", false
}

func (r *routingResolver) cacheSet(name string, val path.Path, rec *pb.IpnsEntry) {
	if r.cache == nil {
		return
	}

	// if completely unspecified, just use one minute
	ttl := DefaultResolverCacheTTL
	if rec.Ttl != nil {
		recttl := time.Duration(rec.GetTtl())
		if recttl >= 0 {
			ttl = recttl
		}
	}

	cacheTil := time.Now().Add(ttl)
	eol, ok := checkEOL(rec)
	if ok && eol.Before(cacheTil) {
		cacheTil = eol
	}

	r.cache.Add(name, cacheEntry{
		val: val,
		eol: cacheTil,
	})
}

type cacheEntry struct {
	val path.Path
	eol time.Time
}

// NewRoutingResolver constructs a name resolver using the IPFS Routing system
// to implement SFS-like naming on top.
// cachesize is the limit of the number of entries in the lru cache. Setting it
// to '0' will disable caching.
func NewRoutingResolver(route routing.ValueStore, cachesize int) *routingResolver {
	if route == nil {
		panic("attempt to create resolver with nil routing system")
	}

	var cache *lru.Cache
	if cachesize > 0 {
		cache, _ = lru.New(cachesize)
	}

	return &routingResolver{
		routing: route,
		cache:   cache,
	}
}

// Resolve implements Resolver.
func (r *routingResolver) Resolve(ctx context.Context, name string, options ...opts.ResolveOpt) (path.Path, error) {
	return resolve(ctx, r, name, opts.ProcessOpts(options), "/ipns/")
}

// resolveOnce implements resolver. Uses the IPFS routing system to
// resolve SFS-like names.
func (r *routingResolver) resolveOnce(ctx context.Context, name string, options *opts.ResolveOpts) (path.Path, error) {
	log.Debugf("RoutingResolver resolving %s", name)
	cached, ok := r.cacheGet(name)
	if ok {
		return cached, nil
	}

	if options.DhtTimeout != 0 {
		// Resolution must complete within the timeout
		var cancel context.CancelFunc
		ctx, cancel = context.WithTimeout(ctx, options.DhtTimeout)
		defer cancel()
	}

	name = strings.TrimPrefix(name, "/ipns/")
	hash, err := mh.FromB58String(name)
	if err != nil {
		// name should be a multihash. if it isn't, error out here.
		log.Debugf("RoutingResolver: bad input hash: [%s]\n", name)
		return "", err
	}

	// Name should be the hash of a public key retrievable from ipfs.
	// We retrieve the public key here to make certain that it's in the peer
	// store before calling GetValue() on the DHT - the DHT will call the
	// ipns validator, which in turn will get the public key from the peer
	// store to verify the record signature
	_, err = routing.GetPublicKey(r.routing, ctx, hash)
	if err != nil {
		log.Debugf("RoutingResolver: could not retrieve public key %s: %s\n", name, err)
		return "", err
	}

	pid, err := peer.IDFromBytes(hash)
	if err != nil {
		log.Debugf("RoutingResolver: could not convert public key hash %s to peer ID: %s\n", name, err)
		return "", err
	}

	// Use the routing system to get the name.
	// Note that the DHT will call the ipns validator when retrieving
	// the value, which in turn verifies the ipns record signature
	_, ipnsKey := IpnsKeysForID(pid)
	val, err := r.getValue(ctx, ipnsKey, options)
	if err != nil {
		log.Debugf("RoutingResolver: dht get for name %s failed: %s", name, err)
		return "", err
	}

	entry := new(pb.IpnsEntry)
	err = proto.Unmarshal(val, entry)
	if err != nil {
		log.Debugf("RoutingResolver: could not unmarshal value for name %s: %s", name, err)
		return "", err
	}

	// check for old style record:
	valh, err := mh.Cast(entry.GetValue())
	if err != nil {
		// Not a multihash, probably a new record
		p, err := path.ParsePath(string(entry.GetValue()))
		if err != nil {
			return "", err
		}

		r.cacheSet(name, p, entry)
		return p, nil
	} else {
		// Its an old style multihash record
		log.Debugf("encountered CIDv0 ipns entry: %s", valh)
		p := path.FromCid(cid.NewCidV0(valh))
		r.cacheSet(name, p, entry)
		return p, nil
	}
}

func (r *routingResolver) getValue(ctx context.Context, ipnsKey string, options *opts.ResolveOpts) ([]byte, error) {
	// Get specified number of values from the DHT
	vals, err := r.routing.GetValues(ctx, ipnsKey, int(options.DhtRecordCount))
	if err != nil {
		return nil, err
	}

	// Select the best value
	recs := make([][]byte, 0, len(vals))
	for _, v := range vals {
		if v.Val != nil {
			recs = append(recs, v.Val)
		}
	}

	if len(recs) == 0 {
		return nil, routing.ErrNotFound
	}

	i, err := IpnsSelectorFunc(ipnsKey, recs)
	if err != nil {
		return nil, err
	}

	best := recs[i]
	if best == nil {
		log.Errorf("GetValues %s yielded record with nil value", ipnsKey)
		return nil, routing.ErrNotFound
	}

	return best, nil
}

func checkEOL(e *pb.IpnsEntry) (time.Time, bool) {
	if e.GetValidityType() == pb.IpnsEntry_EOL {
		eol, err := u.ParseRFC3339(string(e.GetValidity()))
		if err != nil {
			return time.Time{}, false
		}
		return eol, true
	}
	return time.Time{}, false
}
<|endoftext|>


<commit_before>// Copyright 2016 Joel Scoble and The JoeFriday authors.
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

// Package usage calculates network usage.  Usage is calculated by taking the
// difference in two /proc/net/dev snapshots and reflect bytes received and
// transmitted since the prior snapshot.
package usage

import (
	"fmt"
	"io"
	"sync"
	"time"

	"github.com/SermoDigital/helpers"
	joe "github.com/mohae/joefriday"
	"github.com/mohae/joefriday/net/info"
	"github.com/mohae/joefriday/net/structs"
)

// Profiler is used to process the network usage..
type Profiler struct {
	*info.Profiler
	prior *structs.Info
}

// Returns an initialized Profiler; ready to use.
func New() (prof *Profiler, err error) {
	p, err := info.New()
	if err != nil {
		return nil, err
	}
	return &Profiler{Profiler: p, prior: &structs.Info{}}, nil
}

// Get returns the current network usage.
// TODO: should this be changed so that this calculates usage since the last
// time the network info was obtained.  If there aren't pre-existing info
// it would get current usage (which may be a separate method (or should be?))
func (prof *Profiler) Get() (u *structs.Info, err error) {
	prof.prior, err = prof.Profiler.Get()
	if err != nil {
		return nil, err
	}
	time.Sleep(time.Second)
	infCur, err := prof.Profiler.Get()
	if err != nil {
		return nil, err
	}
	return prof.CalculateUsage(infCur), nil
}

var std *Profiler
var stdMu sync.Mutex

// Get returns the current network usage using the package's global Profiler..
func Get() (u *structs.Info, err error) {
	stdMu.Lock()
	defer stdMu.Unlock()
	if std == nil {
		std, err = New()
		if err != nil {
			return nil, err
		}
	}
	return std.Get()
}

// Ticker calculates network usage on a ticker.  The generated data is sent
// to the out channel.  Any errors encountered are sent to the errs channel.
//  Processing ends when a done signal is received.
//
// It is the callers responsibility to close the done and errs channels.
//
// TODO: better handle errors, e.g. restore cur from prior so that there
// isn't the possibility of temporarily having bad data, just a missed
// collection interval.
func (prof *Profiler) Ticker(interval time.Duration, out chan *structs.Info, done chan struct{}, errs chan error) {
	ticker := time.NewTicker(interval)
	defer ticker.Stop()
	defer close(out)
	// predeclare some vars
	var (
		i, l, pos, fieldNum int
		n                   uint64
		v                   byte
		iInfo               structs.Interface
	)
	// first get Info as the baseline
	cur, err := prof.Profiler.Get()
	if err != nil {
		errs <- err
		return
	}
	// ticker
tick:
	for {
		select {
		case <-done:
			return
		case <-ticker.C:
			prof.prior.Timestamp = cur.Timestamp
			if len(prof.prior.Interfaces) != len(cur.Interfaces) {
				prof.prior.Interfaces = make([]structs.Interface, len(cur.Interfaces))
			}
			copy(prof.prior.Interfaces, cur.Interfaces)
			cur.Timestamp = time.Now().UTC().UnixNano()
			err = prof.Reset()
			if err != nil {
				errs <- joe.Error{Type: "net", Op: "usage ticker", Err: err}
				continue tick
			}
			cur.Interfaces = cur.Interfaces[:0]
			// read each line until eof
			for {
				prof.Line, err = prof.Buf.ReadSlice('\n')
				if err != nil {
					if err == io.EOF {
						break
					}
					errs <- fmt.Errorf("/proc/mem/dev: read output bytes: %s", err)
					break
				}
				l++
				if l < 3 {
					continue
				}

				// skip leading spaces
				for i, v = range prof.Line {
					if v != 0x20 {
						pos = i
						break
					}
				}
				// first grab the interface name (everything up to the ':')
				for i, v = range prof.Line[pos:] {
					if v == 0x3A {
						iInfo.Name = string(prof.Line[pos : pos+i])
						pos += i + 1
						break
					}
				}
				fieldNum = 0
				// process the rest of the line
				for {
					fieldNum++
					// skip all spaces
					for i, v = range prof.Line[pos:] {
						if v != 0x20 {
							pos += i
							break
						}
					}

					// grab the numbers
					for i, v = range prof.Line[pos:] {
						if v == 0x20 || v == '\n' {
							prof.Val = prof.Line[pos : pos+i]
							pos += i + 1
							break
						}
					}
					// any conversion error results in 0
					n, err = helpers.ParseUint(prof.Val[:])
					if err != nil {
						errs <- fmt.Errorf("/proc/net/dev ticker: %s: %s", iInfo.Name, err)
						continue
					}
					prof.Val = prof.Val[:0]
					if fieldNum == 1 {
						iInfo.RBytes = int64(n)
						continue
					}
					if fieldNum == 2 {
						iInfo.RPackets = int64(n)
						continue
					}
					if fieldNum == 3 {
						iInfo.RErrs = int64(n)
						continue
					}
					if fieldNum == 4 {
						iInfo.RDrop = int64(n)
						continue
					}
					if fieldNum == 5 {
						iInfo.RFIFO = int64(n)
						continue
					}
					if fieldNum == 6 {
						iInfo.RFrame = int64(n)
						continue
					}
					if fieldNum == 7 {
						iInfo.RCompressed = int64(n)
						continue
					}
					if fieldNum == 8 {
						iInfo.RMulticast = int64(n)
						continue
					}
					if fieldNum == 9 {
						iInfo.TBytes = int64(n)
						continue
					}
					if fieldNum == 10 {
						iInfo.TPackets = int64(n)
						continue
					}
					if fieldNum == 11 {
						iInfo.TErrs = int64(n)
						continue
					}
					if fieldNum == 12 {
						iInfo.TDrop = int64(n)
						continue
					}
					if fieldNum == 13 {
						iInfo.TFIFO = int64(n)
						continue
					}
					if fieldNum == 14 {
						iInfo.TColls = int64(n)
						continue
					}
					if fieldNum == 15 {
						iInfo.TCarrier = int64(n)
						continue
					}
					if fieldNum == 16 {
						iInfo.TCompressed = int64(n)
						break
					}
				}
				cur.Interfaces = append(cur.Interfaces, iInfo)
			}
			out <- prof.CalculateUsage(cur)
			l = 0
		}
	}
}

// Ticker gathers information on a ticker using the specified interval.
// This uses a local Profiler as using the global doesn't make sense for
// an ongoing ticker.
func Ticker(interval time.Duration, out chan *structs.Info, done chan struct{}, errs chan error) {
	prof, err := New()
	if err != nil {
		errs <- err
		close(out)
		return
	}
	prof.Ticker(interval, out, done, errs)
}

// CalculateUsage returns the difference between the current /proc/net/dev
// data and the prior one.
func (prof *Profiler) CalculateUsage(cur *structs.Info) *structs.Info {
	u := &structs.Info{Timestamp: cur.Timestamp, Interfaces: make([]structs.Interface, len(cur.Interfaces))}
	for i := 0; i < len(cur.Interfaces); i++ {
		u.Interfaces[i].Name = cur.Interfaces[i].Name
		u.Interfaces[i].RBytes = cur.Interfaces[i].RBytes - prof.prior.Interfaces[i].RBytes
		u.Interfaces[i].RPackets = cur.Interfaces[i].RPackets - prof.prior.Interfaces[i].RPackets
		u.Interfaces[i].RErrs = cur.Interfaces[i].RErrs - prof.prior.Interfaces[i].RErrs
		u.Interfaces[i].RDrop = cur.Interfaces[i].RDrop - prof.prior.Interfaces[i].RDrop
		u.Interfaces[i].RFIFO = cur.Interfaces[i].RFIFO - prof.prior.Interfaces[i].RFIFO
		u.Interfaces[i].RFrame = cur.Interfaces[i].RFrame - prof.prior.Interfaces[i].RFrame
		u.Interfaces[i].RCompressed = cur.Interfaces[i].RCompressed - prof.prior.Interfaces[i].RCompressed
		u.Interfaces[i].RMulticast = cur.Interfaces[i].RMulticast - prof.prior.Interfaces[i].RMulticast
		u.Interfaces[i].TBytes = cur.Interfaces[i].TBytes - prof.prior.Interfaces[i].TBytes
		u.Interfaces[i].TPackets = cur.Interfaces[i].TPackets - prof.prior.Interfaces[i].TPackets
		u.Interfaces[i].TErrs = cur.Interfaces[i].TErrs - prof.prior.Interfaces[i].TErrs
		u.Interfaces[i].TDrop = cur.Interfaces[i].TDrop - prof.prior.Interfaces[i].TDrop
		u.Interfaces[i].TFIFO = cur.Interfaces[i].TFIFO - prof.prior.Interfaces[i].TFIFO
		u.Interfaces[i].TColls = cur.Interfaces[i].TColls - prof.prior.Interfaces[i].TColls
		u.Interfaces[i].TCarrier = cur.Interfaces[i].TCarrier - prof.prior.Interfaces[i].TCarrier
		u.Interfaces[i].TCompressed = cur.Interfaces[i].TCompressed - prof.prior.Interfaces[i].TCompressed
	}
	return u
}
<commit_msg>refactor usage processing; esp field evaluations<commit_after>// Copyright 2016 Joel Scoble and The JoeFriday authors.
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

// Package usage calculates network usage.  Usage is calculated by taking the
// difference in two /proc/net/dev snapshots and reflect bytes received and
// transmitted since the prior snapshot.
package usage

import (
	"fmt"
	"io"
	"sync"
	"time"

	"github.com/SermoDigital/helpers"
	joe "github.com/mohae/joefriday"
	"github.com/mohae/joefriday/net/info"
	"github.com/mohae/joefriday/net/structs"
)

// Profiler is used to process the network usage..
type Profiler struct {
	*info.Profiler
	prior *structs.Info
}

// Returns an initialized Profiler; ready to use.
func New() (prof *Profiler, err error) {
	p, err := info.New()
	if err != nil {
		return nil, err
	}
	return &Profiler{Profiler: p, prior: &structs.Info{}}, nil
}

// Get returns the current network usage.
// TODO: should this be changed so that this calculates usage since the last
// time the network info was obtained.  If there aren't pre-existing info
// it would get current usage (which may be a separate method (or should be?))
func (prof *Profiler) Get() (u *structs.Info, err error) {
	prof.prior, err = prof.Profiler.Get()
	if err != nil {
		return nil, err
	}
	time.Sleep(time.Second)
	infCur, err := prof.Profiler.Get()
	if err != nil {
		return nil, err
	}
	return prof.CalculateUsage(infCur), nil
}

var std *Profiler
var stdMu sync.Mutex

// Get returns the current network usage using the package's global Profiler..
func Get() (u *structs.Info, err error) {
	stdMu.Lock()
	defer stdMu.Unlock()
	if std == nil {
		std, err = New()
		if err != nil {
			return nil, err
		}
	}
	return std.Get()
}

// Ticker calculates network usage on a ticker.  The generated data is sent
// to the out channel.  Any errors encountered are sent to the errs channel.
//  Processing ends when a done signal is received.
//
// It is the callers responsibility to close the done and errs channels.
//
// TODO: better handle errors, e.g. restore cur from prior so that there
// isn't the possibility of temporarily having bad data, just a missed
// collection interval.
func (prof *Profiler) Ticker(interval time.Duration, out chan *structs.Info, done chan struct{}, errs chan error) {
	ticker := time.NewTicker(interval)
	defer ticker.Stop()
	defer close(out)
	// predeclare some vars
	var (
		i, l, pos, fieldNum int
		n                   uint64
		v                   byte
		iInfo               structs.Interface
	)
	// first get Info as the baseline
	cur, err := prof.Profiler.Get()
	if err != nil {
		errs <- err
		return
	}
	// ticker
tick:
	for {
		select {
		case <-done:
			return
		case <-ticker.C:
			prof.prior.Timestamp = cur.Timestamp
			if len(prof.prior.Interfaces) != len(cur.Interfaces) {
				prof.prior.Interfaces = make([]structs.Interface, len(cur.Interfaces))
			}
			copy(prof.prior.Interfaces, cur.Interfaces)
			cur.Timestamp = time.Now().UTC().UnixNano()
			err = prof.Reset()
			if err != nil {
				errs <- joe.Error{Type: "net", Op: "usage ticker", Err: err}
				continue tick
			}
			cur.Interfaces = cur.Interfaces[:0]
			// read each line until eof
			for {
				prof.Val = prof.Val[:0]
				prof.Line, err = prof.Buf.ReadSlice('\n')
				if err != nil {
					if err == io.EOF {
						break
					}
					errs <- fmt.Errorf("/proc/mem/dev: read output bytes: %s", err)
					break
				}
				l++
				if l < 3 {
					continue
				}
				// first grab the interface name (everything up to the ':')
				for i, v = range prof.Line {
					if v == 0x3A {
						pos = i + 1
						break
					}
					// skip spaces
					if v != 0x20 {
						continue
					}
					prof.Val = append(prof.Val, v)
				}
				iInfo.Name = string(prof.Val[:])
				fieldNum = 0
				// process the rest of the line
				for {
					fieldNum++
					// skip all spaces
					for i, v = range prof.Line[pos:] {
						if v != 0x20 {
							pos += i
							break
						}
					}
					// grab the numbers
					for i, v = range prof.Line[pos:] {
						if v == 0x20 || v == '\n' {
							break
						}
					}
					// any conversion error results in 0
					n, err = helpers.ParseUint(prof.Line[pos : pos+1])
					pos += i
					if err != nil {
						errs <- fmt.Errorf("/proc/net/dev ticker: %s: %s", iInfo.Name, err)
						continue
					}
					if fieldNum < 9 {
						if fieldNum < 5 {
							if fieldNum < 3 {
								if fieldNum == 1 {
									iInfo.RBytes = int64(n)
									continue
								}
								iInfo.RPackets = int64(n) // must be 2
								continue
							}
							if fieldNum == 3 {
								iInfo.RErrs = int64(n)
								continue
							}
							iInfo.RDrop = int64(n) // must be 4
							continue
						}
						if fieldNum < 7 {
							if fieldNum == 5 {
								iInfo.RFIFO = int64(n)
								continue
							}
							iInfo.RFrame = int64(n) // must be 6
							continue
						}
						if fieldNum == 7 {
							iInfo.RCompressed = int64(n)
							continue
						}
						iInfo.RMulticast = int64(n) // must be 8
						continue
					}
					if fieldNum < 13 {
						if fieldNum < 11 {
							if fieldNum == 9 {
								iInfo.TBytes = int64(n)
								continue
							}
							iInfo.TPackets = int64(n) // must be 10
							continue
						}
						if fieldNum == 11 {
							iInfo.TErrs = int64(n)
							continue
						}
						iInfo.TDrop = int64(n) // must be 12
						continue
					}
					if fieldNum < 15 {
						if fieldNum == 13 {
							iInfo.TFIFO = int64(n)
							continue
						}
						iInfo.TColls = int64(n) // must be 14
						continue
					}
					if fieldNum == 15 {
						iInfo.TCarrier = int64(n)
						continue
					}
					if fieldNum == 16 {
						iInfo.TCompressed = int64(n)
						break
					}
				}
				cur.Interfaces = append(cur.Interfaces, iInfo)
			}
			out <- prof.CalculateUsage(cur)
			l = 0
		}
	}
}

// Ticker gathers information on a ticker using the specified interval.
// This uses a local Profiler as using the global doesn't make sense for
// an ongoing ticker.
func Ticker(interval time.Duration, out chan *structs.Info, done chan struct{}, errs chan error) {
	prof, err := New()
	if err != nil {
		errs <- err
		close(out)
		return
	}
	prof.Ticker(interval, out, done, errs)
}

// CalculateUsage returns the difference between the current /proc/net/dev
// data and the prior one.
func (prof *Profiler) CalculateUsage(cur *structs.Info) *structs.Info {
	u := &structs.Info{Timestamp: cur.Timestamp, Interfaces: make([]structs.Interface, len(cur.Interfaces))}
	for i := 0; i < len(cur.Interfaces); i++ {
		u.Interfaces[i].Name = cur.Interfaces[i].Name
		u.Interfaces[i].RBytes = cur.Interfaces[i].RBytes - prof.prior.Interfaces[i].RBytes
		u.Interfaces[i].RPackets = cur.Interfaces[i].RPackets - prof.prior.Interfaces[i].RPackets
		u.Interfaces[i].RErrs = cur.Interfaces[i].RErrs - prof.prior.Interfaces[i].RErrs
		u.Interfaces[i].RDrop = cur.Interfaces[i].RDrop - prof.prior.Interfaces[i].RDrop
		u.Interfaces[i].RFIFO = cur.Interfaces[i].RFIFO - prof.prior.Interfaces[i].RFIFO
		u.Interfaces[i].RFrame = cur.Interfaces[i].RFrame - prof.prior.Interfaces[i].RFrame
		u.Interfaces[i].RCompressed = cur.Interfaces[i].RCompressed - prof.prior.Interfaces[i].RCompressed
		u.Interfaces[i].RMulticast = cur.Interfaces[i].RMulticast - prof.prior.Interfaces[i].RMulticast
		u.Interfaces[i].TBytes = cur.Interfaces[i].TBytes - prof.prior.Interfaces[i].TBytes
		u.Interfaces[i].TPackets = cur.Interfaces[i].TPackets - prof.prior.Interfaces[i].TPackets
		u.Interfaces[i].TErrs = cur.Interfaces[i].TErrs - prof.prior.Interfaces[i].TErrs
		u.Interfaces[i].TDrop = cur.Interfaces[i].TDrop - prof.prior.Interfaces[i].TDrop
		u.Interfaces[i].TFIFO = cur.Interfaces[i].TFIFO - prof.prior.Interfaces[i].TFIFO
		u.Interfaces[i].TColls = cur.Interfaces[i].TColls - prof.prior.Interfaces[i].TColls
		u.Interfaces[i].TCarrier = cur.Interfaces[i].TCarrier - prof.prior.Interfaces[i].TCarrier
		u.Interfaces[i].TCompressed = cur.Interfaces[i].TCompressed - prof.prior.Interfaces[i].TCompressed
	}
	return u
}
<|endoftext|>


<commit_before>// Package newmath is a trivial example package.
package newmath

// Sqrt returns an approximation to the square root of x.
func Sqrt(x float64) float64 {
	z := 1.0
	for i := 0; i < 1000; i++ {
		z -= (z*z - x) / (2 * z)
	}
	return z
}<commit_msg>Update newmath.go<commit_after>// Package newmath is a trivial example package.
package newmath

// Sqrt returns an approximation to the square root of x.
func Sqrt(x float64) float64 {
	z := 1.0
	for i := 0; i < 1000; i++ {
		z -= (z*z - x) / (2 * x)
	}
	return z
}
<|endoftext|>


<commit_before>package distribution

import (
	"fmt"
	"net"
	"net/http"
	"net/url"
	"strings"
	"time"

	"github.com/docker/distribution"
	"github.com/docker/distribution/registry/api/errcode"
	"github.com/docker/distribution/registry/client"
	"github.com/docker/distribution/registry/client/auth"
	"github.com/docker/distribution/registry/client/transport"
	"github.com/docker/docker/distribution/xfer"
	"github.com/docker/docker/registry"
	"github.com/docker/engine-api/types"
	"golang.org/x/net/context"
)

// fallbackError wraps an error that can possibly allow fallback to a different
// endpoint.
type fallbackError struct {
	// err is the error being wrapped.
	err error
	// confirmedV2 is set to true if it was confirmed that the registry
	// supports the v2 protocol. This is used to limit fallbacks to the v1
	// protocol.
	confirmedV2 bool
}

// Error renders the FallbackError as a string.
func (f fallbackError) Error() string {
	return f.err.Error()
}

type dumbCredentialStore struct {
	auth *types.AuthConfig
}

func (dcs dumbCredentialStore) Basic(*url.URL) (string, string) {
	return dcs.auth.Username, dcs.auth.Password
}

// NewV2Repository returns a repository (v2 only). It creates a HTTP transport
// providing timeout settings and authentication support, and also verifies the
// remote API version.
func NewV2Repository(ctx context.Context, repoInfo *registry.RepositoryInfo, endpoint registry.APIEndpoint, metaHeaders http.Header, authConfig *types.AuthConfig, actions ...string) (repo distribution.Repository, foundVersion bool, err error) {
	repoName := repoInfo.FullName()
	// If endpoint does not support CanonicalName, use the RemoteName instead
	if endpoint.TrimHostname {
		repoName = repoInfo.RemoteName()
	}

	// TODO(dmcgowan): Call close idle connections when complete, use keep alive
	base := &http.Transport{
		Proxy: http.ProxyFromEnvironment,
		Dial: (&net.Dialer{
			Timeout:   30 * time.Second,
			KeepAlive: 30 * time.Second,
			DualStack: true,
		}).Dial,
		TLSHandshakeTimeout: 10 * time.Second,
		TLSClientConfig:     endpoint.TLSConfig,
		// TODO(dmcgowan): Call close idle connections when complete and use keep alive
		DisableKeepAlives: true,
	}

	modifiers := registry.DockerHeaders(metaHeaders)
	authTransport := transport.NewTransport(base, modifiers...)
	pingClient := &http.Client{
		Transport: authTransport,
		Timeout:   15 * time.Second,
	}
	endpointStr := strings.TrimRight(endpoint.URL, "/") + "/v2/"
	req, err := http.NewRequest("GET", endpointStr, nil)
	if err != nil {
		return nil, false, err
	}
	resp, err := pingClient.Do(req)
	if err != nil {
		return nil, false, err
	}
	defer resp.Body.Close()

	v2Version := auth.APIVersion{
		Type:    "registry",
		Version: "2.0",
	}

	versions := auth.APIVersions(resp, registry.DefaultRegistryVersionHeader)
	for _, pingVersion := range versions {
		if pingVersion == v2Version {
			// The version header indicates we're definitely
			// talking to a v2 registry. So don't allow future
			// fallbacks to the v1 protocol.

			foundVersion = true
			break
		}
	}

	challengeManager := auth.NewSimpleChallengeManager()
	if err := challengeManager.AddResponse(resp); err != nil {
		return nil, foundVersion, err
	}

	if authConfig.RegistryToken != "" {
		passThruTokenHandler := &existingTokenHandler{token: authConfig.RegistryToken}
		modifiers = append(modifiers, auth.NewAuthorizer(challengeManager, passThruTokenHandler))
	} else {
		creds := dumbCredentialStore{auth: authConfig}
		tokenHandler := auth.NewTokenHandler(authTransport, creds, repoName, actions...)
		basicHandler := auth.NewBasicHandler(creds)
		modifiers = append(modifiers, auth.NewAuthorizer(challengeManager, tokenHandler, basicHandler))
	}
	tr := transport.NewTransport(base, modifiers...)

	repo, err = client.NewRepository(ctx, repoName, endpoint.URL, tr)
	return repo, foundVersion, err
}

type existingTokenHandler struct {
	token string
}

func (th *existingTokenHandler) Scheme() string {
	return "bearer"
}

func (th *existingTokenHandler) AuthorizeRequest(req *http.Request, params map[string]string) error {
	req.Header.Set("Authorization", fmt.Sprintf("Bearer %s", th.token))
	return nil
}

// retryOnError wraps the error in xfer.DoNotRetry if we should not retry the
// operation after this error.
func retryOnError(err error) error {
	switch v := err.(type) {
	case errcode.Errors:
		return retryOnError(v[0])
	case errcode.Error:
		switch v.Code {
		case errcode.ErrorCodeUnauthorized, errcode.ErrorCodeUnsupported, errcode.ErrorCodeDenied:
			return xfer.DoNotRetry{Err: err}
		}
	case *client.UnexpectedHTTPResponseError:
		return xfer.DoNotRetry{Err: err}
	}
	// let's be nice and fallback if the error is a completely
	// unexpected one.
	// If new errors have to be handled in some way, please
	// add them to the switch above.
	return err
}
<commit_msg>Don't retry downloads when disk is full<commit_after>package distribution

import (
	"fmt"
	"net"
	"net/http"
	"net/url"
	"strings"
	"syscall"
	"time"

	"github.com/docker/distribution"
	"github.com/docker/distribution/registry/api/errcode"
	"github.com/docker/distribution/registry/client"
	"github.com/docker/distribution/registry/client/auth"
	"github.com/docker/distribution/registry/client/transport"
	"github.com/docker/docker/distribution/xfer"
	"github.com/docker/docker/registry"
	"github.com/docker/engine-api/types"
	"golang.org/x/net/context"
)

// fallbackError wraps an error that can possibly allow fallback to a different
// endpoint.
type fallbackError struct {
	// err is the error being wrapped.
	err error
	// confirmedV2 is set to true if it was confirmed that the registry
	// supports the v2 protocol. This is used to limit fallbacks to the v1
	// protocol.
	confirmedV2 bool
}

// Error renders the FallbackError as a string.
func (f fallbackError) Error() string {
	return f.err.Error()
}

type dumbCredentialStore struct {
	auth *types.AuthConfig
}

func (dcs dumbCredentialStore) Basic(*url.URL) (string, string) {
	return dcs.auth.Username, dcs.auth.Password
}

// NewV2Repository returns a repository (v2 only). It creates a HTTP transport
// providing timeout settings and authentication support, and also verifies the
// remote API version.
func NewV2Repository(ctx context.Context, repoInfo *registry.RepositoryInfo, endpoint registry.APIEndpoint, metaHeaders http.Header, authConfig *types.AuthConfig, actions ...string) (repo distribution.Repository, foundVersion bool, err error) {
	repoName := repoInfo.FullName()
	// If endpoint does not support CanonicalName, use the RemoteName instead
	if endpoint.TrimHostname {
		repoName = repoInfo.RemoteName()
	}

	// TODO(dmcgowan): Call close idle connections when complete, use keep alive
	base := &http.Transport{
		Proxy: http.ProxyFromEnvironment,
		Dial: (&net.Dialer{
			Timeout:   30 * time.Second,
			KeepAlive: 30 * time.Second,
			DualStack: true,
		}).Dial,
		TLSHandshakeTimeout: 10 * time.Second,
		TLSClientConfig:     endpoint.TLSConfig,
		// TODO(dmcgowan): Call close idle connections when complete and use keep alive
		DisableKeepAlives: true,
	}

	modifiers := registry.DockerHeaders(metaHeaders)
	authTransport := transport.NewTransport(base, modifiers...)
	pingClient := &http.Client{
		Transport: authTransport,
		Timeout:   15 * time.Second,
	}
	endpointStr := strings.TrimRight(endpoint.URL, "/") + "/v2/"
	req, err := http.NewRequest("GET", endpointStr, nil)
	if err != nil {
		return nil, false, err
	}
	resp, err := pingClient.Do(req)
	if err != nil {
		return nil, false, err
	}
	defer resp.Body.Close()

	v2Version := auth.APIVersion{
		Type:    "registry",
		Version: "2.0",
	}

	versions := auth.APIVersions(resp, registry.DefaultRegistryVersionHeader)
	for _, pingVersion := range versions {
		if pingVersion == v2Version {
			// The version header indicates we're definitely
			// talking to a v2 registry. So don't allow future
			// fallbacks to the v1 protocol.

			foundVersion = true
			break
		}
	}

	challengeManager := auth.NewSimpleChallengeManager()
	if err := challengeManager.AddResponse(resp); err != nil {
		return nil, foundVersion, err
	}

	if authConfig.RegistryToken != "" {
		passThruTokenHandler := &existingTokenHandler{token: authConfig.RegistryToken}
		modifiers = append(modifiers, auth.NewAuthorizer(challengeManager, passThruTokenHandler))
	} else {
		creds := dumbCredentialStore{auth: authConfig}
		tokenHandler := auth.NewTokenHandler(authTransport, creds, repoName, actions...)
		basicHandler := auth.NewBasicHandler(creds)
		modifiers = append(modifiers, auth.NewAuthorizer(challengeManager, tokenHandler, basicHandler))
	}
	tr := transport.NewTransport(base, modifiers...)

	repo, err = client.NewRepository(ctx, repoName, endpoint.URL, tr)
	return repo, foundVersion, err
}

type existingTokenHandler struct {
	token string
}

func (th *existingTokenHandler) Scheme() string {
	return "bearer"
}

func (th *existingTokenHandler) AuthorizeRequest(req *http.Request, params map[string]string) error {
	req.Header.Set("Authorization", fmt.Sprintf("Bearer %s", th.token))
	return nil
}

// retryOnError wraps the error in xfer.DoNotRetry if we should not retry the
// operation after this error.
func retryOnError(err error) error {
	switch v := err.(type) {
	case errcode.Errors:
		return retryOnError(v[0])
	case errcode.Error:
		switch v.Code {
		case errcode.ErrorCodeUnauthorized, errcode.ErrorCodeUnsupported, errcode.ErrorCodeDenied:
			return xfer.DoNotRetry{Err: err}
		}
	case *client.UnexpectedHTTPResponseError:
		return xfer.DoNotRetry{Err: err}
	case error:
		if strings.Contains(err.Error(), strings.ToLower(syscall.ENOSPC.Error())) {
			return xfer.DoNotRetry{Err: err}
		}
	}
	// let's be nice and fallback if the error is a completely
	// unexpected one.
	// If new errors have to be handled in some way, please
	// add them to the switch above.
	return err
}
<|endoftext|>


<commit_before>// Brief: Scripture API
// Primary responsibility: Query calls for Scripture functionality

package main

import (
	"fmt"
	"log"
	"strings"

	bmul "github.com/julwrites/BotMultiplexer"
	"golang.org/x/net/html"
)

func GetReference(doc *html.Node, env *bmul.SessionData) string {
	refNode, err := FindByClass(doc, "bcv")
	if err != nil {
		log.Printf("Error parsing for reference: %v", err)
		return ""
	}

	return refNode.FirstChild.Data
}

func ParseNodesForPassage(node *html.Node) string {
	var text string
	var parts []string

	for child := node.FirstChild; child != nil; child = child.NextSibling {
		parts = append(parts, text)

		switch tag := child.Data; tag {
		case "span":
			childText := strings.Trim(ParseNodesForPassage(child), " ")
			if len(childText) > 0 {
				parts = append(parts, childText)
			}
		case "sup":
			isFootnote := func(node *html.Node) bool {
				for _, attr := range node.Attr {
					if attr.Key == "class" && attr.Val == "footnote" {
						return true
					}
				}
				return false
			}
			if isFootnote(child) {
				break
			}
			childText := strings.Trim(ParseNodesForPassage(child), " ")
			if len(childText) > 0 {
				parts = append(parts, fmt.Sprintf("^%s^", childText))
			}
			break
		default:
			parts = append(parts, child.Data)
		}
	}

	text = strings.Join(parts, "")

	if node.Data == "h1" || node.Data == "h2" || node.Data == "h3" {
		text = fmt.Sprintf("*%s*", text)
	}
	return text
}

func GetPassage(doc *html.Node, env *bmul.SessionData) string {
	passageNode, startErr := FindByClass(doc, fmt.Sprintf("version-%s result-text-style-normal text-html", GetUserConfig(&env.User).Version))
	if startErr != nil {
		log.Printf("Error parsing for passage: %v", startErr)
		return ""
	}

	filtNodes := FilterChildren(passageNode, func(child *html.Node) bool {
		switch tag := child.Data; tag {
		case "h1":
			fallthrough
		case "h2":
			fallthrough
		case "h3":
			fallthrough
		case "p":
			return true
		}
		return false
	})

	log.Printf("Candidate nodes number %d", len(filtNodes))

	textBlocks := MapNodeList(filtNodes, ParseNodesForPassage)

	var passage strings.Builder

	for _, block := range textBlocks {
		passage.WriteString(block)
		passage.WriteString("\n\n")
	}

	return passage.String()
}

func GetBiblePassage(env *bmul.SessionData) {
	if len(env.Msg.Message) > 0 {

		doc := QueryBibleGateway(env.Msg.Message, env)

		ref := GetReference(doc, env)
		log.Printf("Reference retrieved: %s", ref)

		if len(ref) > 0 {
			log.Printf("Getting passage")
			env.Res.Message = GetPassage(doc, env)
		}
	}
}

// func main() {
// 	var env bmul.SessionData
// 	var config UserConfig
// 	config.Version = "NIV"
// 	UpdateUserConfig(&env.User, config)
// 	env.Msg.Message = "gal 1"
// 	GetBiblePassage(&env)
// }
<commit_msg>Fixing formatting with MarkdownV2<commit_after>// Brief: Scripture API
// Primary responsibility: Query calls for Scripture functionality

package main

import (
	"fmt"
	"log"
	"strings"

	bmul "github.com/julwrites/BotMultiplexer"
	"golang.org/x/net/html"
)

func GetReference(doc *html.Node, env *bmul.SessionData) string {
	refNode, err := FindByClass(doc, "bcv")
	if err != nil {
		log.Printf("Error parsing for reference: %v", err)
		return ""
	}

	return refNode.FirstChild.Data
}

func ParseNodesForPassage(node *html.Node) string {
	var text string
	var parts []string

	for child := node.FirstChild; child != nil; child = child.NextSibling {
		parts = append(parts, text)

		switch tag := child.Data; tag {
		case "span":
			childText := strings.Trim(ParseNodesForPassage(child), " ")
			if len(childText) > 0 {
				parts = append(parts, childText)
			}
		case "sup":
			isFootnote := func(node *html.Node) bool {
				for _, attr := range node.Attr {
					if attr.Key == "class" && attr.Val == "footnote" {
						return true
					}
				}
				return false
			}
			if isFootnote(child) {
				break
			}
			childText := strings.Trim(ParseNodesForPassage(child), " ")
			if len(childText) > 0 {
				parts = append(parts, fmt.Sprintf("^%s^", childText))
			}
			break
		default:
			parts = append(parts, child.Data)
		}
	}

	text = strings.Join(parts, "")

	if node.Data == "h1" || node.Data == "h2" || node.Data == "h3" {
		text = fmt.Sprintf("*%s*", text)
	}
	return text
}

func GetPassage(doc *html.Node, env *bmul.SessionData) string {
	passageNode, startErr := FindByClass(doc, fmt.Sprintf("version-%s result-text-style-normal text-html", GetUserConfig(&env.User).Version))
	if startErr != nil {
		log.Printf("Error parsing for passage: %v", startErr)
		return ""
	}

	filtNodes := FilterChildren(passageNode, func(child *html.Node) bool {
		switch tag := child.Data; tag {
		case "h1":
			fallthrough
		case "h2":
			fallthrough
		case "h3":
			fallthrough
		case "p":
			return true
		}
		return false
	})

	textBlocks := MapNodeList(filtNodes, ParseNodesForPassage)

	var passage strings.Builder

	for _, block := range textBlocks {
		passage.WriteString(block)
		passage.WriteString("\n")
	}

	return passage.String()
}

func GetBiblePassage(env *bmul.SessionData) {
	if len(env.Msg.Message) > 0 {

		doc := QueryBibleGateway(env.Msg.Message, env)

		ref := GetReference(doc, env)
		log.Printf("Reference retrieved: %s", ref)

		if len(ref) > 0 {
			log.Printf("Getting passage")
			env.Res.Message = GetPassage(doc, env)
		}
	}
}

// func main() {
// 	var env bmul.SessionData
// 	var config UserConfig
// 	config.Version = "NIV"
// 	UpdateUserConfig(&env.User, config)
// 	env.Msg.Message = "gal 1"
// 	GetBiblePassage(&env)
// }
<|endoftext|>