<commit_before>package commands

import (
	"errors"
	"net/http"
	"os"
	"path"
	"path/filepath"
	"runtime"
	"strings"

	"github.com/codegangsta/cli"
	"github.com/jfrog/jfrog-cli-core/utils/coreutils"
	pluginsutils "github.com/jfrog/jfrog-cli/plugins/utils"
	"github.com/jfrog/jfrog-cli/utils/cliutils"
	logUtils "github.com/jfrog/jfrog-cli/utils/log"
	"github.com/jfrog/jfrog-cli/utils/progressbar"
	"github.com/jfrog/jfrog-client-go/http/httpclient"
	"github.com/jfrog/jfrog-client-go/utils"
	"github.com/jfrog/jfrog-client-go/utils/errorutils"
	"github.com/jfrog/jfrog-client-go/utils/io/fileutils"
	"github.com/jfrog/jfrog-client-go/utils/io/httputils"
	"github.com/jfrog/jfrog-client-go/utils/log"
)

const pluginsRegistryUrl = "https://releases.jfrog.io/artifactory"
const pluginsRegistryRepo = "jfrog-cli-plugins"
const latestVersionName = "latest"

func InstallCmd(c *cli.Context) error {
	if c.NArg() != 1 {
		return cliutils.PrintHelpAndReturnError("Wrong number of arguments.", c)
	}
	return runInstallCmd(c.Args().Get(0))
}

func runInstallCmd(requestedPlugin string) error {
	pluginName, version, err := getNameAndVersion(requestedPlugin)
	if err != nil {
		return err
	}
	srcPath, err := buildSrcPath(pluginName, version)
	if err != nil {
		return err
	}
	downloadUrl := utils.AddTrailingSlashIfNeeded(pluginsRegistryUrl) + srcPath

	pluginsDir, err := coreutils.GetJfrogPluginsDir()
	if err != nil {
		return err
	}

	exists, err := fileutils.IsDirExists(pluginsDir, false)
	if err != nil {
		return err
	}
	if exists {
		should, err := shouldDownloadPlugin(pluginsDir, pluginName, downloadUrl)
		if err != nil {
			return err
		}
		if !should {
			return errors.New("requested plugin already exists locally")
		}
	} else {
		err = createPluginsDir(pluginsDir)
		if err != nil {
			return err
		}
	}

	return downloadPlugin(pluginsDir, pluginName, downloadUrl)
}

func shouldDownloadPlugin(pluginsDir, pluginName, downloadUrl string) (bool, error) {
	log.Debug("Verifying plugin download is needed...")
	client, err := httpclient.ClientBuilder().Build()
	if err != nil {
		return false, err
	}
	log.Debug("Fetching plugin details from: ", downloadUrl)

	details, resp, err := client.GetRemoteFileDetails(downloadUrl, httputils.HttpClientDetails{})
	if err != nil {
		return false, err
	}
	log.Debug("Artifactory response: ", resp.Status)
	err = errorutils.CheckResponseStatus(resp, http.StatusOK)
	if err != nil {
		return false, err
	}
	isEqual, err := fileutils.IsEqualToLocalFile(filepath.Join(pluginsDir, pluginName), details.Checksum.Md5, details.Checksum.Sha1)
	return !isEqual, err
}

func buildSrcPath(pluginName, version string) (string, error) {
	arc, err := getArchitecture()
	if err != nil {
		return "", err
	}
	return path.Join(pluginsRegistryRepo, pluginName, version, arc, pluginsutils.GetPluginExecutableName(pluginName)), nil
}

func createPluginsDir(pluginsDir string) error {
	return os.MkdirAll(pluginsDir, 0777)
}

func downloadPlugin(pluginsDir, pluginName, downloadUrl string) error {
	exeName := pluginsutils.GetPluginExecutableName(pluginName)
	log.Debug("Downloading plugin from: ", downloadUrl)
	downloadDetails := &httpclient.DownloadFileDetails{
		FileName:      pluginName,
		DownloadPath:  downloadUrl,
		LocalPath:     pluginsDir,
		LocalFileName: exeName,
		RelativePath:  exeName,
	}

	client, err := httpclient.ClientBuilder().Build()
	if err != nil {
		return err
	}
	// Init progress bar.
	progressMgr, logFile, err := progressbar.InitProgressBarIfPossible()
	if err != nil {
		return err
	}
	if progressMgr != nil {
		progressMgr.IncGeneralProgressTotalBy(1)
		defer logUtils.CloseLogFile(logFile)
		defer progressMgr.Quit()
	}
	log.Info("Downloading plugin: " + pluginName)

	resp, err := client.DownloadFileWithProgress(downloadDetails, "", httputils.HttpClientDetails{}, 3, false, progressMgr)
	if err != nil {
		return err
	}
	log.Debug("Artifactory response: ", resp.Status)
	err = errorutils.CheckResponseStatus(resp, http.StatusOK)
	if err != nil {
		return err
	}
	log.Debug("Plugin downloaded successfully.")
	return os.Chmod(filepath.Join(pluginsDir, exeName), 0777)
}

func getNameAndVersion(requested string) (name, version string, err error) {
	split := strings.Split(requested, "@")
	if len(split) == 1 || (len(split) == 2 && split[1] == "") {
		return split[0], latestVersionName, nil
	}
	if len(split) > 2 {
		return "", "", errors.New("unexpected number of '@' separators in provided argument")
	}
	return split[0], split[1], nil
}

// Get the architecture name corresponding to the architectures that exist in registry.
func getArchitecture() (string, error) {
	switch runtime.GOOS {
	case "windows":
		return "windows-amd64", nil
	case "darwin":
		return "mac-386", nil
	}
	// Assuming linux.
	switch runtime.GOARCH {
	case "amd64":
		return "linux-amd64", nil
	case "arm64":
		return "linux-arm64", nil
	case "arm":
		return "linux-arm", nil
	case "386":
		return "linux-386", nil
	}
	return "", errors.New("no compatible plugin architecture was found for the architecture of this machine")
}
<commit_msg>Support plugins for s390x linux arch<commit_after>package commands

import (
	"errors"
	"net/http"
	"os"
	"path"
	"path/filepath"
	"runtime"
	"strings"

	"github.com/codegangsta/cli"
	"github.com/jfrog/jfrog-cli-core/utils/coreutils"
	pluginsutils "github.com/jfrog/jfrog-cli/plugins/utils"
	"github.com/jfrog/jfrog-cli/utils/cliutils"
	logUtils "github.com/jfrog/jfrog-cli/utils/log"
	"github.com/jfrog/jfrog-cli/utils/progressbar"
	"github.com/jfrog/jfrog-client-go/http/httpclient"
	"github.com/jfrog/jfrog-client-go/utils"
	"github.com/jfrog/jfrog-client-go/utils/errorutils"
	"github.com/jfrog/jfrog-client-go/utils/io/fileutils"
	"github.com/jfrog/jfrog-client-go/utils/io/httputils"
	"github.com/jfrog/jfrog-client-go/utils/log"
)

const pluginsRegistryUrl = "https://releases.jfrog.io/artifactory"
const pluginsRegistryRepo = "jfrog-cli-plugins"
const latestVersionName = "latest"

func InstallCmd(c *cli.Context) error {
	if c.NArg() != 1 {
		return cliutils.PrintHelpAndReturnError("Wrong number of arguments.", c)
	}
	return runInstallCmd(c.Args().Get(0))
}

func runInstallCmd(requestedPlugin string) error {
	pluginName, version, err := getNameAndVersion(requestedPlugin)
	if err != nil {
		return err
	}
	srcPath, err := buildSrcPath(pluginName, version)
	if err != nil {
		return err
	}
	downloadUrl := utils.AddTrailingSlashIfNeeded(pluginsRegistryUrl) + srcPath

	pluginsDir, err := coreutils.GetJfrogPluginsDir()
	if err != nil {
		return err
	}

	exists, err := fileutils.IsDirExists(pluginsDir, false)
	if err != nil {
		return err
	}
	if exists {
		should, err := shouldDownloadPlugin(pluginsDir, pluginName, downloadUrl)
		if err != nil {
			return err
		}
		if !should {
			return errors.New("requested plugin already exists locally")
		}
	} else {
		err = createPluginsDir(pluginsDir)
		if err != nil {
			return err
		}
	}

	return downloadPlugin(pluginsDir, pluginName, downloadUrl)
}

func shouldDownloadPlugin(pluginsDir, pluginName, downloadUrl string) (bool, error) {
	log.Debug("Verifying plugin download is needed...")
	client, err := httpclient.ClientBuilder().Build()
	if err != nil {
		return false, err
	}
	log.Debug("Fetching plugin details from: ", downloadUrl)

	details, resp, err := client.GetRemoteFileDetails(downloadUrl, httputils.HttpClientDetails{})
	if err != nil {
		return false, err
	}
	log.Debug("Artifactory response: ", resp.Status)
	err = errorutils.CheckResponseStatus(resp, http.StatusOK)
	if err != nil {
		return false, err
	}
	isEqual, err := fileutils.IsEqualToLocalFile(filepath.Join(pluginsDir, pluginName), details.Checksum.Md5, details.Checksum.Sha1)
	return !isEqual, err
}

func buildSrcPath(pluginName, version string) (string, error) {
	arc, err := getArchitecture()
	if err != nil {
		return "", err
	}
	return path.Join(pluginsRegistryRepo, pluginName, version, arc, pluginsutils.GetPluginExecutableName(pluginName)), nil
}

func createPluginsDir(pluginsDir string) error {
	return os.MkdirAll(pluginsDir, 0777)
}

func downloadPlugin(pluginsDir, pluginName, downloadUrl string) error {
	exeName := pluginsutils.GetPluginExecutableName(pluginName)
	log.Debug("Downloading plugin from: ", downloadUrl)
	downloadDetails := &httpclient.DownloadFileDetails{
		FileName:      pluginName,
		DownloadPath:  downloadUrl,
		LocalPath:     pluginsDir,
		LocalFileName: exeName,
		RelativePath:  exeName,
	}

	client, err := httpclient.ClientBuilder().Build()
	if err != nil {
		return err
	}
	// Init progress bar.
	progressMgr, logFile, err := progressbar.InitProgressBarIfPossible()
	if err != nil {
		return err
	}
	if progressMgr != nil {
		progressMgr.IncGeneralProgressTotalBy(1)
		defer logUtils.CloseLogFile(logFile)
		defer progressMgr.Quit()
	}
	log.Info("Downloading plugin: " + pluginName)

	resp, err := client.DownloadFileWithProgress(downloadDetails, "", httputils.HttpClientDetails{}, 3, false, progressMgr)
	if err != nil {
		return err
	}
	log.Debug("Artifactory response: ", resp.Status)
	err = errorutils.CheckResponseStatus(resp, http.StatusOK)
	if err != nil {
		return err
	}
	log.Debug("Plugin downloaded successfully.")
	return os.Chmod(filepath.Join(pluginsDir, exeName), 0777)
}

func getNameAndVersion(requested string) (name, version string, err error) {
	split := strings.Split(requested, "@")
	if len(split) == 1 || (len(split) == 2 && split[1] == "") {
		return split[0], latestVersionName, nil
	}
	if len(split) > 2 {
		return "", "", errors.New("unexpected number of '@' separators in provided argument")
	}
	return split[0], split[1], nil
}

// Get the architecture name corresponding to the architectures that exist in registry.
func getArchitecture() (string, error) {
	switch runtime.GOOS {
	case "windows":
		return "windows-amd64", nil
	case "darwin":
		return "mac-386", nil
	}
	// Assuming linux.
	switch runtime.GOARCH {
	case "amd64":
		return "linux-amd64", nil
	case "arm64":
		return "linux-arm64", nil
	case "arm":
		return "linux-arm", nil
	case "386":
		return "linux-386", nil
	case "s390x":
		return "linux-s390x", nil
	}
	return "", errors.New("no compatible plugin architecture was found for the architecture of this machine")
}
<|endoftext|>


<commit_before>// Copyright 2017 Keybase Inc. All rights reserved.
// Use of this source code is governed by a BSD
// license that can be found in the LICENSE file.

package libpages

import (
	"net"
	"strings"

	"go.uber.org/zap"
	"go.uber.org/zap/zapcore"
)

// RootLoader is the interface for loading a site root. This interface exists
// for instrumenting tests. In real instances, only the DNSRootLoader should be
// used.
type RootLoader interface {
	LoadRoot(domain string) (root Root, err error)
}

// DNSRootLoader is an implementation of RootLoader loads a root from DNS. This
// is the RootLoader that should be used in all non-test scenarios. See doc for
// LoadRoot for more.
type DNSRootLoader struct {
	log *zap.Logger
}

const (
	keybasePagesPrefix = "kbp="
)

// ErrKeybasePagesRecordNotFound is returned when a domain requested doesn't
// have a kbp= record configured.
type ErrKeybasePagesRecordNotFound struct{}

// Error implements the error interface.
func (ErrKeybasePagesRecordNotFound) Error() string {
	return "no TXT record is found for " + keybasePagesPrefix
}

// ErrKeybasePagesRecordTooMany is returned when a domain requested has more
// than one kbp= record configured.
type ErrKeybasePagesRecordTooMany struct{}

// Error implements the error interface.
func (ErrKeybasePagesRecordTooMany) Error() string {
	return "more than 1 TXT record are found for " + keybasePagesPrefix
}

// kbpRecordPrefixes specifies the TXT record prefixes that we look at to
// locate the root of these Keybase pages. We have 2 records since some
// registrars don't support underscores in the middle of a domain. This order
// must remain fixed since it reflects the order the strings are evaluated in.
var kbpRecordPrefixes []string = []string{"_keybase_pages.", "_keybasepages."}

// LoadRoot loads the root path configured for domain from DNS, with following
// steps:
//   1. Construct a domain name by prefixing the `domain` parameter with
//      "_keybase_pages." or "_keybasepages". So for example,
//      "static.keybase.io" turns into "_keybase_pages.static.keybase.io" or
//      "_keybasepages.static.keybase.io".
//   2. Load TXT record(s) from the domain constructed in step 1, and look for
//      one starting with "kbp=". If exactly one exists, parse it into a `Root`
//      and return it.
//
// There must be exactly one "kbp=" TXT record configured for domain. If more
// than one exists, an ErrKeybasePagesRecordTooMany{} is returned. If none is
// found, an ErrKeybasePagesRecordNotFound{} is returned. In case user has some
// configuration that requires other records that we can't foresee for now,
// other records (TXT or not) can co-exist with the "kbp=" record (as long as
// no CNAME record exists on the "_keybase_pages." or "_keybasepages." prefixed
// domain of course).
//
// If the given domain is invalid, it would cause the domain name constructed
// in step will be invalid too, which causes Go's DNS resolver to return a
// net.DNSError typed "no such host" error.
//
// Examples for "static.keybase.io", "meatball.gao.io", "song.gao.io",
// "blah.strib.io", and "kbp.jzila.com" respectively:
//
// _keybase_pages.static.keybase.io TXT "kbp=/keybase/team/keybase.bots/static.keybase.io"
// _keybase_pages.meatball.gao.io   TXT "kbp=/keybase/public/songgao/meatball/"
// _keybase_pages.song.gao.io       TXT "kbp=/keybase/private/songgao,kb_bot/blah"
// _keybase_pages.blah.strib.io     TXT "kbp=/keybase/private/strib#kb_bot/blahblahb" "lah/blah/"
// _keybase_pages.kbp.jzila.com     TXT "kbp=git@keybase:private/jzila,kb_bot/kbp.git"
func (l DNSRootLoader) LoadRoot(domain string) (root Root, err error) {
	var rootPath string

	defer func() {
		zapFields := []zapcore.Field{
			zap.String("domain", domain),
			zap.String("kbp_record", rootPath),
		}
		if err == nil {
			l.log.Info("LoadRootFromDNS", zapFields...)
		} else {
			l.log.Warn("LoadRootFromDNS", append(zapFields, zap.Error(err))...)
		}
	}()

	// Check all possible kbp record prefixes.
	var txtRecords []string
	for _, kbpRecordPrefix := range kbpRecordPrefixes {
		txtRecords, err = net.LookupTXT(kbpRecordPrefix + domain)
		if err == nil {
			break
		}
	}
	if err != nil {
		return Root{}, err
	}

	for _, r := range txtRecords {
		r = strings.TrimSpace(r)

		if strings.HasPrefix(r, keybasePagesPrefix) {
			if len(rootPath) != 0 {
				return Root{}, ErrKeybasePagesRecordTooMany{}
			}
			rootPath = r[len(keybasePagesPrefix):]
		}
	}

	if len(rootPath) == 0 {
		return Root{}, ErrKeybasePagesRecordNotFound{}
	}

	return ParseRoot(rootPath)
}
<commit_msg>libpages: Fix lint.<commit_after>// Copyright 2017 Keybase Inc. All rights reserved.
// Use of this source code is governed by a BSD
// license that can be found in the LICENSE file.

package libpages

import (
	"net"
	"strings"

	"go.uber.org/zap"
	"go.uber.org/zap/zapcore"
)

// RootLoader is the interface for loading a site root. This interface exists
// for instrumenting tests. In real instances, only the DNSRootLoader should be
// used.
type RootLoader interface {
	LoadRoot(domain string) (root Root, err error)
}

// DNSRootLoader is an implementation of RootLoader loads a root from DNS. This
// is the RootLoader that should be used in all non-test scenarios. See doc for
// LoadRoot for more.
type DNSRootLoader struct {
	log *zap.Logger
}

const (
	keybasePagesPrefix = "kbp="
)

// ErrKeybasePagesRecordNotFound is returned when a domain requested doesn't
// have a kbp= record configured.
type ErrKeybasePagesRecordNotFound struct{}

// Error implements the error interface.
func (ErrKeybasePagesRecordNotFound) Error() string {
	return "no TXT record is found for " + keybasePagesPrefix
}

// ErrKeybasePagesRecordTooMany is returned when a domain requested has more
// than one kbp= record configured.
type ErrKeybasePagesRecordTooMany struct{}

// Error implements the error interface.
func (ErrKeybasePagesRecordTooMany) Error() string {
	return "more than 1 TXT record are found for " + keybasePagesPrefix
}

// kbpRecordPrefixes specifies the TXT record prefixes that we look at to
// locate the root of these Keybase pages. We have 2 records since some
// registrars don't support underscores in the middle of a domain. This order
// must remain fixed since it reflects the order the strings are evaluated in.
var kbpRecordPrefixes = []string{"_keybase_pages.", "_keybasepages."}

// LoadRoot loads the root path configured for domain from DNS, with following
// steps:
//   1. Construct a domain name by prefixing the `domain` parameter with
//      "_keybase_pages." or "_keybasepages". So for example,
//      "static.keybase.io" turns into "_keybase_pages.static.keybase.io" or
//      "_keybasepages.static.keybase.io".
//   2. Load TXT record(s) from the domain constructed in step 1, and look for
//      one starting with "kbp=". If exactly one exists, parse it into a `Root`
//      and return it.
//
// There must be exactly one "kbp=" TXT record configured for domain. If more
// than one exists, an ErrKeybasePagesRecordTooMany{} is returned. If none is
// found, an ErrKeybasePagesRecordNotFound{} is returned. In case user has some
// configuration that requires other records that we can't foresee for now,
// other records (TXT or not) can co-exist with the "kbp=" record (as long as
// no CNAME record exists on the "_keybase_pages." or "_keybasepages." prefixed
// domain of course).
//
// If the given domain is invalid, it would cause the domain name constructed
// in step will be invalid too, which causes Go's DNS resolver to return a
// net.DNSError typed "no such host" error.
//
// Examples for "static.keybase.io", "meatball.gao.io", "song.gao.io",
// "blah.strib.io", and "kbp.jzila.com" respectively:
//
// _keybase_pages.static.keybase.io TXT "kbp=/keybase/team/keybase.bots/static.keybase.io"
// _keybase_pages.meatball.gao.io   TXT "kbp=/keybase/public/songgao/meatball/"
// _keybase_pages.song.gao.io       TXT "kbp=/keybase/private/songgao,kb_bot/blah"
// _keybase_pages.blah.strib.io     TXT "kbp=/keybase/private/strib#kb_bot/blahblahb" "lah/blah/"
// _keybase_pages.kbp.jzila.com     TXT "kbp=git@keybase:private/jzila,kb_bot/kbp.git"
func (l DNSRootLoader) LoadRoot(domain string) (root Root, err error) {
	var rootPath string

	defer func() {
		zapFields := []zapcore.Field{
			zap.String("domain", domain),
			zap.String("kbp_record", rootPath),
		}
		if err == nil {
			l.log.Info("LoadRootFromDNS", zapFields...)
		} else {
			l.log.Warn("LoadRootFromDNS", append(zapFields, zap.Error(err))...)
		}
	}()

	// Check all possible kbp record prefixes.
	var txtRecords []string
	for _, kbpRecordPrefix := range kbpRecordPrefixes {
		txtRecords, err = net.LookupTXT(kbpRecordPrefix + domain)
		if err == nil {
			break
		}
	}
	if err != nil {
		return Root{}, err
	}

	for _, r := range txtRecords {
		r = strings.TrimSpace(r)

		if strings.HasPrefix(r, keybasePagesPrefix) {
			if len(rootPath) != 0 {
				return Root{}, ErrKeybasePagesRecordTooMany{}
			}
			rootPath = r[len(keybasePagesPrefix):]
		}
	}

	if len(rootPath) == 0 {
		return Root{}, ErrKeybasePagesRecordNotFound{}
	}

	return ParseRoot(rootPath)
}
<|endoftext|>


<commit_before>package imjasonh

import (
	"appengine"
	"html/template"
	"math/rand"
	"net/http"
	"strconv"
)

const (
	warHTML = `

<meta http-equiv="Content-Type" content="text/html; charset=ISO-8859-1" />
<title>100 Games of War</title>

{{.NumGames}} Games of War

    {{.P1Wins}}

    {{.P2Wins}}

    {{.AllP1Wins}}
    « All time wins »
    {{.AllP2Wins}}

    refresh | what? | src

`
)

func init() {
	http.HandleFunc("/war", war)
}

func defaultAndInt(val string, def int) int {
	if val == "" {
		i := int64(0)
		i, _ = strconv.ParseInt(val, 0, 16)
		return int(i)
	} else {
		return def
	}
	return -1
}

// war simulates playing a number of games of the card game War.
func war(w http.ResponseWriter, r *http.Request) {
	c := appengine.NewContext(r)

	games := defaultAndInt(r.FormValue("games"), 100)
	//warCards := defaultAndInt(r.FormValue("warCards"), 3)
	values := defaultAndInt(r.FormValue("values"), 13)
	suits := defaultAndInt(r.FormValue("suits"), 4)
	numCards := values * suits
	var p1wins, p2wins, ties int

	// TODO: Actually implement card-playing logic.
	allP1wins := 100
	allP2wins := 200

	for game := 0; game < games; game++ {
		c.Infof("Game", game)
		allCards := rand.Perm(numCards)
		half := numCards / 2
		p1cards := allCards[:half]
		p2cards := allCards[half:]

		// TODO: Actually implement War logic.
		if p1cards[0] > p2cards[0] {
			p1wins++
		} else if p2cards[0] > p1cards[0] {
			p2wins++
		} else {
			ties++
		}
	}

	t := template.Must(template.New("war").Parse(warHTML))
	t.Execute(w, map[string]int{
		"P1Wins":    p1wins,
		"P2Wins":    p2wins,
		"Ties":      ties,
		"NumGames":  games,
		"AllP1Wins": allP1wins,
		"AllP2Wins": allP2wins,
	})
}
<commit_msg>Implement most of the card game<commit_after>package imjasonh

import (
	"html/template"
	"math/rand"
	"net/http"
	"strconv"
)

var games, warCards, values, suits, numCards int

const warHTML = `

<title>{{.NumGames}} Games of War</title>

{{.NumGames}} Games of War

    {{.P1Wins}}

    {{.P2Wins}}

    {{.AllP1Wins}}
    « All time wins »
    {{.AllP2Wins}}

    refresh | what? | src

`

type gameResult int

const (
	// Results of games
	p1win gameResult = iota
	p2win
	tie
)

func init() {
	http.HandleFunc("/war", war)
}

func defaultAndInt(val string, def int) int {
	if val != "" {
		i, _ := strconv.ParseInt(val, 0, 16)
		return int(i)
	} else {
		return def
	}
	panic("Unreachable")
}

func pop(in []int) (int, []int) {
	if len(in) == 1 {
		return in[0], []int{}
	} else {
		return in[0], in[1:]
	}
	panic("Unreachable")
}

func playGame() gameResult {
	values := 13
	suits := 3
	numCards := values * suits

	allCards := rand.Perm(numCards)
	p1deck := allCards[numCards/2:]
	p2deck := allCards[:numCards/2]

	for len(p1deck) > 0 && len(p2deck) > 0 {
		var p1card, p2card int
		p1card, p1deck = pop(p1deck)
		p2card, p2deck = pop(p2deck)

		p1val := p1card % values
		p2val := p2card % values

		switch {
		case p1val == p2val:
			// TODO: Actually do war
		case p1val > p2val:
			p1deck = append(p1deck, p1card, p2card)
		case p2val > p1val:
			p2deck = append(p2deck, p1card, p2card)
		}
	}

	switch {
	case len(p1deck) > 0 && len(p2deck) == 0:
		return p1win
	case len(p2deck) > 0 && len(p1deck) == 0:
		return p2win
	default:
		return tie
	}
	panic("Unreachable")
}

// war simulates playing a number of games of the card game War.
func war(w http.ResponseWriter, r *http.Request) {
	games = defaultAndInt(r.FormValue("games"), 100)
	//warCards = defaultAndInt(r.FormValue("warCards"), 3)
	values = defaultAndInt(r.FormValue("values"), 13)
	suits = defaultAndInt(r.FormValue("suits"), 4)
	numCards = values * suits
	var p1wins, p2wins, ties int

	// TODO: Actually implement total score counter in datastore.
	allP1wins := -1
	allP2wins := -1

	for game := 0; game < games; game++ {
		// TODO: Do this in a goroutine.
		gameResult := playGame()
		switch {
		case gameResult == p1win:
			p1wins++
		case gameResult == p2win:
			p2wins++
		case gameResult == tie:
			ties++
		}
	}

	t := template.Must(template.New("war").Parse(warHTML))
	t.Execute(w, map[string]int{
		"P1Wins":    p1wins,
		"P2Wins":    p2wins,
		"Ties":      ties,
		"NumGames":  games,
		"AllP1Wins": allP1wins,
		"AllP2Wins": allP2wins,
	})
}
<|endoftext|>


<commit_before>// Copyright 2019 The go-hep Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

package ntup_test

import (
	"fmt"
	"log"

	"go-hep.org/x/hep/hbook/ntup/ntcsv"
)

func ExampleNtuple_scanH2D() {
	nt, err := ntcsv.Open(
		"ntcsv/testdata/simple-with-header.csv",
		ntcsv.Comma(';'),
		ntcsv.Header(),
		ntcsv.Columns("v1", "v2", "v3"),
	)
	if err != nil {
		log.Fatal(err)
	}
	defer nt.DB().Close()

	h, err := nt.ScanH2D("v1, v2", nil)

	if err != nil {
		log.Fatal(err)
	}
	fmt.Printf("XMean:      %f\n", h.XMean())
	fmt.Printf("YMean:      %f\n", h.YMean())
	fmt.Printf("XRMS:       %f\n", h.XRMS())
	fmt.Printf("YRMS:       %f\n", h.YRMS())
	fmt.Printf("XStdDev:    %f\n", h.XStdDev())
	fmt.Printf("YStdDev:    %f\n", h.YStdDev())
	fmt.Printf("XStdErr:    %f\n", h.XStdErr())
	fmt.Printf("YStdErr:    %f\n", h.YStdErr())

	// Output:
	// XMean:      4.500000
	// YMean:      4.500000
	// XRMS:       5.338539
	// YRMS:       5.338539
	// XStdDev:    3.027650
	// YStdDev:    3.027650
	// XStdErr:    0.957427
	// YStdErr:    0.957427
}
<commit_msg>hbook/ntup: add examples for Scan and ScanH1D methods<commit_after>// Copyright 2019 The go-hep Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

package ntup_test

import (
	"fmt"
	"log"
	"math"

	"go-hep.org/x/hep/hbook/ntup/ntcsv"
)

func ExampleNtuple_scanH2D() {
	nt, err := ntcsv.Open(
		"ntcsv/testdata/simple-with-header.csv",
		ntcsv.Comma(';'),
		ntcsv.Header(),
		ntcsv.Columns("v1", "v2", "v3"),
	)
	if err != nil {
		log.Fatal(err)
	}
	defer nt.DB().Close()

	h, err := nt.ScanH2D("v1, v2", nil)

	if err != nil {
		log.Fatal(err)
	}
	fmt.Printf("XMean:      %f\n", h.XMean())
	fmt.Printf("YMean:      %f\n", h.YMean())
	fmt.Printf("XRMS:       %f\n", h.XRMS())
	fmt.Printf("YRMS:       %f\n", h.YRMS())
	fmt.Printf("XStdDev:    %f\n", h.XStdDev())
	fmt.Printf("YStdDev:    %f\n", h.YStdDev())
	fmt.Printf("XStdErr:    %f\n", h.XStdErr())
	fmt.Printf("YStdErr:    %f\n", h.YStdErr())

	// Output:
	// XMean:      4.500000
	// YMean:      4.500000
	// XRMS:       5.338539
	// YRMS:       5.338539
	// XStdDev:    3.027650
	// YStdDev:    3.027650
	// XStdErr:    0.957427
	// YStdErr:    0.957427
}

func ExampleNtuple_scan() {
	nt, err := ntcsv.Open(
		"ntcsv/testdata/simple-with-header.csv",
		ntcsv.Comma(';'),
		ntcsv.Header(),
		ntcsv.Columns("v1", "v2", "v3"),
	)
	if err != nil {
		log.Fatal(err)
	}
	defer nt.DB().Close()
	var (
		v1min = +math.MaxFloat64
		v1max = -math.MaxFloat64
		v2min = +math.MaxFloat64
		v2max = -math.MaxFloat64
	)
	err = nt.Scan("v1, v2", func(v1, v2 float64) error {
		v1min = math.Min(v1min, v1)
		v1max = math.Max(v1max, v1)
		v2min = math.Min(v2min, v2)
		v2max = math.Max(v2max, v2)
		return nil
	})
	if err != nil {
		log.Fatal(err)
	}

	fmt.Printf("V1Min  %v\n", v1min)
	fmt.Printf("V1Max  %v\n", v1max)
	fmt.Printf("V2Min  %v\n", v2min)
	fmt.Printf("V2Max  %v\n", v2max)

	//Output:
	// V1Min  0
	// V1Max  9
	// V2Min  0
	// V2Max  9

}

func ExampleNtuple_scanH1D() {
	nt, err := ntcsv.Open(
		"ntcsv/testdata/simple-with-header.csv",
		ntcsv.Comma(';'),
		ntcsv.Header(),
		ntcsv.Columns("v1", "v2", "v3"),
	)
	if err != nil {
		log.Fatal(err)
	}
	defer nt.DB().Close()

	h, err := nt.ScanH1D("v1", nil)
	if err != nil {
		log.Fatal(err)
	}
	fmt.Printf("V1Mean:      %f\n", h.XMean())
	fmt.Printf("V1RMS:       %f\n", h.XRMS())
	fmt.Printf("V1StdDev:    %f\n", h.XStdDev())
	fmt.Printf("V1StdErr:    %f\n", h.XStdErr())

	// Output:
	// V1Mean:      4.500000
	// V1RMS:       5.338539
	// V1StdDev:    3.027650
	// V1StdErr:    0.957427
}
<|endoftext|>


<commit_before>package hclsyntax

import (
	"fmt"
	"testing"

	"reflect"

	"github.com/davecgh/go-spew/spew"
	"github.com/hashicorp/hcl2/hcl"
	"github.com/zclconf/go-cty/cty"
)

func TestVariables(t *testing.T) {
	tests := []struct {
		Expr Expression
		Want []hcl.Traversal
	}{
		{
			&LiteralValueExpr{
				Val: cty.True,
			},
			nil,
		},
		{
			&ScopeTraversalExpr{
				Traversal: hcl.Traversal{
					hcl.TraverseRoot{
						Name: "foo",
					},
				},
			},
			[]hcl.Traversal{
				{
					hcl.TraverseRoot{
						Name: "foo",
					},
				},
			},
		},
		{
			&BinaryOpExpr{
				LHS: &ScopeTraversalExpr{
					Traversal: hcl.Traversal{
						hcl.TraverseRoot{
							Name: "foo",
						},
					},
				},
				Op: OpAdd,
				RHS: &ScopeTraversalExpr{
					Traversal: hcl.Traversal{
						hcl.TraverseRoot{
							Name: "bar",
						},
					},
				},
			},
			[]hcl.Traversal{
				{
					hcl.TraverseRoot{
						Name: "foo",
					},
				},
				{
					hcl.TraverseRoot{
						Name: "bar",
					},
				},
			},
		},
		{
			&UnaryOpExpr{
				Val: &ScopeTraversalExpr{
					Traversal: hcl.Traversal{
						hcl.TraverseRoot{
							Name: "foo",
						},
					},
				},
				Op: OpNegate,
			},
			[]hcl.Traversal{
				{
					hcl.TraverseRoot{
						Name: "foo",
					},
				},
			},
		},
		{
			&ConditionalExpr{
				Condition: &ScopeTraversalExpr{
					Traversal: hcl.Traversal{
						hcl.TraverseRoot{
							Name: "foo",
						},
					},
				},
				TrueResult: &ScopeTraversalExpr{
					Traversal: hcl.Traversal{
						hcl.TraverseRoot{
							Name: "bar",
						},
					},
				},
				FalseResult: &ScopeTraversalExpr{
					Traversal: hcl.Traversal{
						hcl.TraverseRoot{
							Name: "baz",
						},
					},
				},
			},
			[]hcl.Traversal{
				{
					hcl.TraverseRoot{
						Name: "foo",
					},
				},
				{
					hcl.TraverseRoot{
						Name: "bar",
					},
				},
				{
					hcl.TraverseRoot{
						Name: "baz",
					},
				},
			},
		},
		{
			&ForExpr{
				KeyVar: "k",
				ValVar: "v",

				CollExpr: &ScopeTraversalExpr{
					Traversal: hcl.Traversal{
						hcl.TraverseRoot{
							Name: "foo",
						},
					},
				},
				KeyExpr: &BinaryOpExpr{
					LHS: &ScopeTraversalExpr{
						Traversal: hcl.Traversal{
							hcl.TraverseRoot{
								Name: "k",
							},
						},
					},
					Op: OpAdd,
					RHS: &ScopeTraversalExpr{
						Traversal: hcl.Traversal{
							hcl.TraverseRoot{
								Name: "bar",
							},
						},
					},
				},
				ValExpr: &BinaryOpExpr{
					LHS: &ScopeTraversalExpr{
						Traversal: hcl.Traversal{
							hcl.TraverseRoot{
								Name: "v",
							},
						},
					},
					Op: OpAdd,
					RHS: &ScopeTraversalExpr{
						Traversal: hcl.Traversal{
							hcl.TraverseRoot{
								Name: "baz",
							},
						},
					},
				},
				CondExpr: &BinaryOpExpr{
					LHS: &ScopeTraversalExpr{
						Traversal: hcl.Traversal{
							hcl.TraverseRoot{
								Name: "k",
							},
						},
					},
					Op: OpLessThan,
					RHS: &ScopeTraversalExpr{
						Traversal: hcl.Traversal{
							hcl.TraverseRoot{
								Name: "limit",
							},
						},
					},
				},
			},
			[]hcl.Traversal{
				{
					hcl.TraverseRoot{
						Name: "foo",
					},
				},
				{
					hcl.TraverseRoot{
						Name: "bar",
					},
				},
				{
					hcl.TraverseRoot{
						Name: "baz",
					},
				},
				{
					hcl.TraverseRoot{
						Name: "limit",
					},
				},
			},
		},
	}

	for _, test := range tests {
		t.Run(fmt.Sprintf("%#v", test.Expr), func(t *testing.T) {
			got := Variables(test.Expr)

			if !reflect.DeepEqual(got, test.Want) {
				t.Errorf("wrong result\ngot:  %s\nwant: %s", spew.Sdump(got), spew.Sdump(test.Want))
			}
		})
	}
}
<commit_msg>hclsyntax: Add test for idx'd func outputs<commit_after>package hclsyntax

import (
	"fmt"
	"reflect"
	"testing"

	"github.com/hashicorp/hcl2/hcl"
	"github.com/kr/pretty"
	"github.com/zclconf/go-cty/cty"
)

func TestVariables(t *testing.T) {
	tests := []struct {
		Expr Expression
		Want []hcl.Traversal
	}{
		{
			&LiteralValueExpr{
				Val: cty.True,
			},
			nil,
		},
		{
			&ScopeTraversalExpr{
				Traversal: hcl.Traversal{
					hcl.TraverseRoot{
						Name: "foo",
					},
				},
			},
			[]hcl.Traversal{
				{
					hcl.TraverseRoot{
						Name: "foo",
					},
				},
			},
		},
		{
			&BinaryOpExpr{
				LHS: &ScopeTraversalExpr{
					Traversal: hcl.Traversal{
						hcl.TraverseRoot{
							Name: "foo",
						},
					},
				},
				Op: OpAdd,
				RHS: &ScopeTraversalExpr{
					Traversal: hcl.Traversal{
						hcl.TraverseRoot{
							Name: "bar",
						},
					},
				},
			},
			[]hcl.Traversal{
				{
					hcl.TraverseRoot{
						Name: "foo",
					},
				},
				{
					hcl.TraverseRoot{
						Name: "bar",
					},
				},
			},
		},
		{
			&UnaryOpExpr{
				Val: &ScopeTraversalExpr{
					Traversal: hcl.Traversal{
						hcl.TraverseRoot{
							Name: "foo",
						},
					},
				},
				Op: OpNegate,
			},
			[]hcl.Traversal{
				{
					hcl.TraverseRoot{
						Name: "foo",
					},
				},
			},
		},
		{
			&ConditionalExpr{
				Condition: &ScopeTraversalExpr{
					Traversal: hcl.Traversal{
						hcl.TraverseRoot{
							Name: "foo",
						},
					},
				},
				TrueResult: &ScopeTraversalExpr{
					Traversal: hcl.Traversal{
						hcl.TraverseRoot{
							Name: "bar",
						},
					},
				},
				FalseResult: &ScopeTraversalExpr{
					Traversal: hcl.Traversal{
						hcl.TraverseRoot{
							Name: "baz",
						},
					},
				},
			},
			[]hcl.Traversal{
				{
					hcl.TraverseRoot{
						Name: "foo",
					},
				},
				{
					hcl.TraverseRoot{
						Name: "bar",
					},
				},
				{
					hcl.TraverseRoot{
						Name: "baz",
					},
				},
			},
		},
		{
			&ForExpr{
				KeyVar: "k",
				ValVar: "v",

				CollExpr: &ScopeTraversalExpr{
					Traversal: hcl.Traversal{
						hcl.TraverseRoot{
							Name: "foo",
						},
					},
				},
				KeyExpr: &BinaryOpExpr{
					LHS: &ScopeTraversalExpr{
						Traversal: hcl.Traversal{
							hcl.TraverseRoot{
								Name: "k",
							},
						},
					},
					Op: OpAdd,
					RHS: &ScopeTraversalExpr{
						Traversal: hcl.Traversal{
							hcl.TraverseRoot{
								Name: "bar",
							},
						},
					},
				},
				ValExpr: &BinaryOpExpr{
					LHS: &ScopeTraversalExpr{
						Traversal: hcl.Traversal{
							hcl.TraverseRoot{
								Name: "v",
							},
						},
					},
					Op: OpAdd,
					RHS: &ScopeTraversalExpr{
						Traversal: hcl.Traversal{
							hcl.TraverseRoot{
								Name: "baz",
							},
						},
					},
				},
				CondExpr: &BinaryOpExpr{
					LHS: &ScopeTraversalExpr{
						Traversal: hcl.Traversal{
							hcl.TraverseRoot{
								Name: "k",
							},
						},
					},
					Op: OpLessThan,
					RHS: &ScopeTraversalExpr{
						Traversal: hcl.Traversal{
							hcl.TraverseRoot{
								Name: "limit",
							},
						},
					},
				},
			},
			[]hcl.Traversal{
				{
					hcl.TraverseRoot{
						Name: "foo",
					},
				},
				{
					hcl.TraverseRoot{
						Name: "bar",
					},
				},
				{
					hcl.TraverseRoot{
						Name: "baz",
					},
				},
				{
					hcl.TraverseRoot{
						Name: "limit",
					},
				},
			},
		},
		{
			&ScopeTraversalExpr{
				Traversal: hcl.Traversal{
					hcl.TraverseRoot{
						Name: "data",
					},
					hcl.TraverseAttr{
						Name: "null_data_source",
					},
					hcl.TraverseAttr{
						Name: "multi",
					},
					hcl.TraverseIndex{
						Key: cty.NumberFloatVal(0),
					},
				},
			},
			[]hcl.Traversal{
				{
					hcl.TraverseRoot{
						Name: "data",
					},
					hcl.TraverseAttr{
						Name: "null_data_source",
					},
					hcl.TraverseAttr{
						Name: "multi",
					},
					hcl.TraverseIndex{
						Key: cty.NumberFloatVal(0),
					},
				},
			},
		},
		{
			&RelativeTraversalExpr{
				Source: &FunctionCallExpr{
					Name: "sort",
					Args: []Expression{
						&ScopeTraversalExpr{
							Traversal: hcl.Traversal{
								hcl.TraverseRoot{
									Name: "data",
								},
								hcl.TraverseAttr{
									Name: "null_data_source",
								},
								hcl.TraverseAttr{
									Name: "multi",
								},
							},
						},
					},
				},
				Traversal: hcl.Traversal{
					hcl.TraverseIndex{
						Key: cty.NumberFloatVal(0),
					},
				},
			},
			[]hcl.Traversal{
				{
					hcl.TraverseRoot{
						Name: "data",
					},
					hcl.TraverseAttr{
						Name: "null_data_source",
					},
					hcl.TraverseAttr{
						Name: "multi",
					},
				},
			},
		},
	}

	for _, test := range tests {
		t.Run(fmt.Sprintf("%#v", test.Expr), func(t *testing.T) {
			got := Variables(test.Expr)

			if !reflect.DeepEqual(got, test.Want) {
				t.Errorf("wrong result\ngot:  %s\nwant: %s", pretty.Sprint(got), pretty.Sprint(test.Want))
			}
		})
	}
}
<|endoftext|>


<commit_before>package clickhouse

import (
	"context"
	"encoding/binary"
	"errors"
	"fmt"
	"html"
	"io"
	"io/ioutil"
	"math/rand"
	"net"
	"net/http"
	"net/url"
	"strings"
	"time"

	"github.com/lomik/graphite-clickhouse/pkg/scope"

	"go.uber.org/zap"
)

type ErrDataParse struct {
	err  string
	data string
}

func NewErrDataParse(err string, data string) error {
	return &ErrDataParse{err, data}
}

func (e *ErrDataParse) Error() string {
	return fmt.Sprintf("%s: %s", e.err, e.data)
}

func (e *ErrDataParse) PrependDescription(test string) {
	e.data = test + e.data
}

type ErrorWithCode struct {
	err  string
	Code int // error code
}

func NewErrorWithCode(err string, code int) error {
	return &ErrorWithCode{err, code}
}

func (e *ErrorWithCode) Error() string { return e.err }

var ErrUvarintRead = errors.New("ReadUvarint: Malformed array")
var ErrUvarintOverflow = errors.New("ReadUvarint: varint overflows a 64-bit integer")
var ErrClickHouseResponse = errors.New("Malformed response from clickhouse")

func HandleError(w http.ResponseWriter, err error) {
	if errors.Is(err, context.Canceled) {
		http.Error(w, "Storage read context canceled", http.StatusGatewayTimeout)
		return
	}
	netErr, ok := err.(net.Error)
	if ok {
		if netErr.Timeout() {
			http.Error(w, "Storage read timeout", http.StatusGatewayTimeout)
		} else if strings.HasSuffix(err.Error(), "connect: no route to host") ||
			strings.HasSuffix(err.Error(), "connect: connection refused") ||
			strings.HasSuffix(err.Error(), ": connection reset by peer") ||
			strings.HasPrefix(err.Error(), "dial tcp: lookup ") { // DNS lookup
			http.Error(w, "Storage error", http.StatusServiceUnavailable)
		} else {
			http.Error(w, err.Error(), http.StatusInternalServerError)
		}
		return
	}
	errCode, ok := err.(*ErrorWithCode)
	if ok {
		if (errCode.Code > 500 && errCode.Code < 512) ||
			errCode.Code == http.StatusBadRequest || errCode.Code == http.StatusForbidden {
			http.Error(w, html.EscapeString(errCode.Error()), errCode.Code)
		} else {
			http.Error(w, err.Error(), http.StatusInternalServerError)
		}
		return
	}
	_, ok = err.(*ErrDataParse)
	if ok || strings.HasPrefix(err.Error(), "clickhouse response status 500: Code:") {
		if strings.Contains(err.Error(), ": Limit for ") {
			//logger.Info("limit", zap.Error(err))
			http.Error(w, "Storage read limit", http.StatusForbidden)
		} else if !ok && strings.HasPrefix(err.Error(), "clickhouse response status 500: Code: 170,") {
			// distributed table configuration error
			// clickhouse response status 500: Code: 170, e.displayText() = DB::Exception: Requested cluster 'cluster' not found
			http.Error(w, "Storage configuration error", http.StatusServiceUnavailable)
		}
	} else {
		//logger.Debug("query", zap.Error(err))
		http.Error(w, err.Error(), http.StatusInternalServerError)
	}
}

type Options struct {
	Timeout        time.Duration
	ConnectTimeout time.Duration
}

type loggedReader struct {
	reader   io.ReadCloser
	logger   *zap.Logger
	start    time.Time
	finished bool
	queryID  string
}

func (r *loggedReader) Read(p []byte) (int, error) {
	n, err := r.reader.Read(p)
	if err != nil && !r.finished {
		r.finished = true
		r.logger.Info("query", zap.String("query_id", r.queryID), zap.Duration("time", time.Since(r.start)))
	}
	return n, err
}

func (r *loggedReader) Close() error {
	err := r.reader.Close()
	if !r.finished {
		r.finished = true
		r.logger.Info("query", zap.String("query_id", r.queryID), zap.Duration("time", time.Since(r.start)))
	}
	return err
}

func formatSQL(q string) string {
	s := strings.Split(q, "\n")
	for i := 0; i < len(s); i++ {
		s[i] = strings.TrimSpace(s[i])
	}

	return strings.Join(s, " ")
}

func Query(ctx context.Context, dsn string, query string, opts Options, extData *ExternalData) ([]byte, error) {
	return Post(ctx, dsn, query, nil, opts, extData)
}

func Post(ctx context.Context, dsn string, query string, postBody io.Reader, opts Options, extData *ExternalData) ([]byte, error) {
	return do(ctx, dsn, query, postBody, false, opts, extData)
}

func PostGzip(ctx context.Context, dsn string, query string, postBody io.Reader, opts Options, extData *ExternalData) ([]byte, error) {
	return do(ctx, dsn, query, postBody, true, opts, extData)
}

func Reader(ctx context.Context, dsn string, query string, opts Options, extData *ExternalData) (io.ReadCloser, error) {
	return reader(ctx, dsn, query, nil, false, opts, extData)
}

func reader(ctx context.Context, dsn string, query string, postBody io.Reader, gzip bool, opts Options, extData *ExternalData) (bodyReader io.ReadCloser, err error) {
	if postBody != nil && extData != nil {
		err = fmt.Errorf("postBody and extData could not be passed in one request")
		return
	}

	var chQueryID string

	start := time.Now()

	requestID := scope.RequestID(ctx)

	queryForLogger := query
	if len(queryForLogger) > 500 {
		queryForLogger = queryForLogger[:395] + "<...>" + queryForLogger[len(queryForLogger)-100:]
	}
	logger := scope.Logger(ctx).With(zap.String("query", formatSQL(queryForLogger)))

	defer func() {
		// fmt.Println(time.Since(start), formatSQL(queryForLogger))
		if err != nil {
			logger.Error("query", zap.Error(err), zap.Duration("time", time.Since(start)))
		}
	}()

	p, err := url.Parse(dsn)
	if err != nil {
		return
	}

	var b [8]byte
	binary.LittleEndian.PutUint64(b[:], rand.Uint64())
	queryID := fmt.Sprintf("%x", b)

	q := p.Query()
	q.Set("query_id", fmt.Sprintf("%s::%s", requestID, queryID))
	p.RawQuery = q.Encode()

	var contentHeader string
	if postBody != nil {
		q := p.Query()
		q.Set("query", query)
		p.RawQuery = q.Encode()
	} else if extData != nil {
		q := p.Query()
		q.Set("query", query)
		p.RawQuery = q.Encode()
		postBody, contentHeader, err = extData.buildBody(ctx, p)
		if err != nil {
			return
		}
	} else {
		postBody = strings.NewReader(query)
	}

	url := p.String()

	req, err := http.NewRequest("POST", url, postBody)
	if err != nil {
		return
	}

	req.Header.Add("User-Agent", scope.ClickhouseUserAgent(ctx))
	if contentHeader != "" {
		req.Header.Add("Content-Type", contentHeader)
	}

	if gzip {
		req.Header.Add("Content-Encoding", "gzip")
	}

	client := &http.Client{
		Timeout: opts.Timeout,
		Transport: &http.Transport{
			Dial: (&net.Dialer{
				Timeout: opts.ConnectTimeout,
			}).Dial,
			DisableKeepAlives: true,
		},
	}
	resp, err := client.Do(req)
	if err != nil {
		return
	}

	// chproxy overwrite our query id. So read it again
	chQueryID = resp.Header.Get("X-ClickHouse-Query-Id")

	// check for return 5xx error, may be 502 code if clickhouse accesed via reverse proxy
	if resp.StatusCode > 500 && resp.StatusCode < 512 {
		body, _ := ioutil.ReadAll(resp.Body)
		resp.Body.Close()
		err = NewErrorWithCode(string(body), resp.StatusCode)
		return
	} else if resp.StatusCode != 200 {
		body, _ := ioutil.ReadAll(resp.Body)
		resp.Body.Close()
		err = fmt.Errorf("clickhouse response status %d: %s", resp.StatusCode, string(body))
		return
	}

	bodyReader = &loggedReader{
		reader:  resp.Body,
		logger:  logger,
		start:   start,
		queryID: chQueryID,
	}

	return
}

func do(ctx context.Context, dsn string, query string, postBody io.Reader, gzip bool, opts Options, extData *ExternalData) ([]byte, error) {
	bodyReader, err := reader(ctx, dsn, query, postBody, gzip, opts, extData)
	if err != nil {
		return nil, err
	}

	body, err := ioutil.ReadAll(bodyReader)
	bodyReader.Close()
	if err != nil {
		return nil, err
	}

	return body, nil
}

func ReadUvarint(array []byte) (uint64, int, error) {
	var x uint64
	var s uint
	l := len(array) - 1
	for i := 0; ; i++ {
		if i > l {
			return x, i + 1, ErrUvarintRead
		}
		if array[i] < 0x80 {
			if i > 9 || i == 9 && array[i] > 1 {
				return x, i + 1, ErrUvarintOverflow
			}
			return x | uint64(array[i])<<s, i + 1, nil
		}
		x |= uint64(array[i]&0x7f) << s
		s += 7
	}
}
<commit_msg>Add logging for clickhouse query summary<commit_after>package clickhouse

import (
	"context"
	"encoding/binary"
	"encoding/json"
	"errors"
	"fmt"
	"html"
	"io"
	"io/ioutil"
	"math/rand"
	"net"
	"net/http"
	"net/url"
	"strings"
	"time"

	"github.com/lomik/graphite-clickhouse/pkg/scope"

	"go.uber.org/zap"
	"go.uber.org/zap/zapcore"
)

type ErrDataParse struct {
	err  string
	data string
}

func NewErrDataParse(err string, data string) error {
	return &ErrDataParse{err, data}
}

func (e *ErrDataParse) Error() string {
	return fmt.Sprintf("%s: %s", e.err, e.data)
}

func (e *ErrDataParse) PrependDescription(test string) {
	e.data = test + e.data
}

type ErrorWithCode struct {
	err  string
	Code int // error code
}

func NewErrorWithCode(err string, code int) error {
	return &ErrorWithCode{err, code}
}

func (e *ErrorWithCode) Error() string { return e.err }

var ErrUvarintRead = errors.New("ReadUvarint: Malformed array")
var ErrUvarintOverflow = errors.New("ReadUvarint: varint overflows a 64-bit integer")
var ErrClickHouseResponse = errors.New("Malformed response from clickhouse")

func HandleError(w http.ResponseWriter, err error) {
	if errors.Is(err, context.Canceled) {
		http.Error(w, "Storage read context canceled", http.StatusGatewayTimeout)
		return
	}
	netErr, ok := err.(net.Error)
	if ok {
		if netErr.Timeout() {
			http.Error(w, "Storage read timeout", http.StatusGatewayTimeout)
		} else if strings.HasSuffix(err.Error(), "connect: no route to host") ||
			strings.HasSuffix(err.Error(), "connect: connection refused") ||
			strings.HasSuffix(err.Error(), ": connection reset by peer") ||
			strings.HasPrefix(err.Error(), "dial tcp: lookup ") { // DNS lookup
			http.Error(w, "Storage error", http.StatusServiceUnavailable)
		} else {
			http.Error(w, err.Error(), http.StatusInternalServerError)
		}
		return
	}
	errCode, ok := err.(*ErrorWithCode)
	if ok {
		if (errCode.Code > 500 && errCode.Code < 512) ||
			errCode.Code == http.StatusBadRequest || errCode.Code == http.StatusForbidden {
			http.Error(w, html.EscapeString(errCode.Error()), errCode.Code)
		} else {
			http.Error(w, err.Error(), http.StatusInternalServerError)
		}
		return
	}
	_, ok = err.(*ErrDataParse)
	if ok || strings.HasPrefix(err.Error(), "clickhouse response status 500: Code:") {
		if strings.Contains(err.Error(), ": Limit for ") {
			//logger.Info("limit", zap.Error(err))
			http.Error(w, "Storage read limit", http.StatusForbidden)
		} else if !ok && strings.HasPrefix(err.Error(), "clickhouse response status 500: Code: 170,") {
			// distributed table configuration error
			// clickhouse response status 500: Code: 170, e.displayText() = DB::Exception: Requested cluster 'cluster' not found
			http.Error(w, "Storage configuration error", http.StatusServiceUnavailable)
		}
	} else {
		//logger.Debug("query", zap.Error(err))
		http.Error(w, err.Error(), http.StatusInternalServerError)
	}
}

type Options struct {
	Timeout        time.Duration
	ConnectTimeout time.Duration
}

type loggedReader struct {
	reader   io.ReadCloser
	logger   *zap.Logger
	start    time.Time
	finished bool
	queryID  string
}

func (r *loggedReader) Read(p []byte) (int, error) {
	n, err := r.reader.Read(p)
	if err != nil && !r.finished {
		r.finished = true
		r.logger.Info("query", zap.String("query_id", r.queryID), zap.Duration("time", time.Since(r.start)))
	}
	return n, err
}

func (r *loggedReader) Close() error {
	err := r.reader.Close()
	if !r.finished {
		r.finished = true
		r.logger.Info("query", zap.String("query_id", r.queryID), zap.Duration("time", time.Since(r.start)))
	}
	return err
}

func formatSQL(q string) string {
	s := strings.Split(q, "\n")
	for i := 0; i < len(s); i++ {
		s[i] = strings.TrimSpace(s[i])
	}

	return strings.Join(s, " ")
}

func Query(ctx context.Context, dsn string, query string, opts Options, extData *ExternalData) ([]byte, error) {
	return Post(ctx, dsn, query, nil, opts, extData)
}

func Post(ctx context.Context, dsn string, query string, postBody io.Reader, opts Options, extData *ExternalData) ([]byte, error) {
	return do(ctx, dsn, query, postBody, false, opts, extData)
}

func PostGzip(ctx context.Context, dsn string, query string, postBody io.Reader, opts Options, extData *ExternalData) ([]byte, error) {
	return do(ctx, dsn, query, postBody, true, opts, extData)
}

func Reader(ctx context.Context, dsn string, query string, opts Options, extData *ExternalData) (io.ReadCloser, error) {
	return reader(ctx, dsn, query, nil, false, opts, extData)
}

func reader(ctx context.Context, dsn string, query string, postBody io.Reader, gzip bool, opts Options, extData *ExternalData) (bodyReader io.ReadCloser, err error) {
	if postBody != nil && extData != nil {
		err = fmt.Errorf("postBody and extData could not be passed in one request")
		return
	}

	var chQueryID string

	start := time.Now()

	requestID := scope.RequestID(ctx)

	queryForLogger := query
	if len(queryForLogger) > 500 {
		queryForLogger = queryForLogger[:395] + "<...>" + queryForLogger[len(queryForLogger)-100:]
	}
	logger := scope.Logger(ctx).With(zap.String("query", formatSQL(queryForLogger)))

	defer func() {
		// fmt.Println(time.Since(start), formatSQL(queryForLogger))
		if err != nil {
			logger.Error("query", zap.Error(err), zap.Duration("time", time.Since(start)))
		}
	}()

	p, err := url.Parse(dsn)
	if err != nil {
		return
	}

	var b [8]byte
	binary.LittleEndian.PutUint64(b[:], rand.Uint64())
	queryID := fmt.Sprintf("%x", b)

	q := p.Query()
	q.Set("query_id", fmt.Sprintf("%s::%s", requestID, queryID))
	// Get X-Clickhouse-Summary header
	// TODO: remove when https://github.com/ClickHouse/ClickHouse/issues/16207 is done
	q.Set("send_progress_in_http_headers", "1")
	q.Set("http_headers_progress_interval_ms", "10000")
	p.RawQuery = q.Encode()

	var contentHeader string
	if postBody != nil {
		q := p.Query()
		q.Set("query", query)
		p.RawQuery = q.Encode()
	} else if extData != nil {
		q := p.Query()
		q.Set("query", query)
		p.RawQuery = q.Encode()
		postBody, contentHeader, err = extData.buildBody(ctx, p)
		if err != nil {
			return
		}
	} else {
		postBody = strings.NewReader(query)
	}

	url := p.String()

	req, err := http.NewRequest("POST", url, postBody)
	if err != nil {
		return
	}

	req.Header.Add("User-Agent", scope.ClickhouseUserAgent(ctx))
	if contentHeader != "" {
		req.Header.Add("Content-Type", contentHeader)
	}

	if gzip {
		req.Header.Add("Content-Encoding", "gzip")
	}

	client := &http.Client{
		Timeout: opts.Timeout,
		Transport: &http.Transport{
			Dial: (&net.Dialer{
				Timeout: opts.ConnectTimeout,
			}).Dial,
			DisableKeepAlives: true,
		},
	}
	resp, err := client.Do(req)
	if err != nil {
		return
	}

	// chproxy overwrite our query id. So read it again
	chQueryID = resp.Header.Get("X-ClickHouse-Query-Id")

	summary := make(map[string]string)
	err = json.Unmarshal([]byte(resp.Header.Get("X-Clickhouse-Summary")), &summary)
	if err == nil {
		// TODO: use in carbon metrics sender when it will be implemented
		fields := make([]zapcore.Field, 0, len(summary))
		for k, v := range summary {
			fields = append(fields, zap.String(k, v))
		}
		logger = logger.With(fields...)
	}

	// check for return 5xx error, may be 502 code if clickhouse accesed via reverse proxy
	if resp.StatusCode > 500 && resp.StatusCode < 512 {
		body, _ := ioutil.ReadAll(resp.Body)
		resp.Body.Close()
		err = NewErrorWithCode(string(body), resp.StatusCode)
		return
	} else if resp.StatusCode != 200 {
		body, _ := ioutil.ReadAll(resp.Body)
		resp.Body.Close()
		err = fmt.Errorf("clickhouse response status %d: %s", resp.StatusCode, string(body))
		return
	}

	bodyReader = &loggedReader{
		reader:  resp.Body,
		logger:  logger,
		start:   start,
		queryID: chQueryID,
	}

	return
}

func do(ctx context.Context, dsn string, query string, postBody io.Reader, gzip bool, opts Options, extData *ExternalData) ([]byte, error) {
	bodyReader, err := reader(ctx, dsn, query, postBody, gzip, opts, extData)
	if err != nil {
		return nil, err
	}

	body, err := ioutil.ReadAll(bodyReader)
	bodyReader.Close()
	if err != nil {
		return nil, err
	}

	return body, nil
}

func ReadUvarint(array []byte) (uint64, int, error) {
	var x uint64
	var s uint
	l := len(array) - 1
	for i := 0; ; i++ {
		if i > l {
			return x, i + 1, ErrUvarintRead
		}
		if array[i] < 0x80 {
			if i > 9 || i == 9 && array[i] > 1 {
				return x, i + 1, ErrUvarintOverflow
			}
			return x | uint64(array[i])<<s, i + 1, nil
		}
		x |= uint64(array[i]&0x7f) << s
		s += 7
	}
}
<|endoftext|>


<commit_before>/*
Copyright 2021 The Kubernetes Authors.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package sidecar

import (
	"bytes"
	"io"
	"io/ioutil"
	"os"
	"os/exec"
	"path/filepath"
	"sort"
	"testing"

	"github.com/google/go-cmp/cmp"

	"k8s.io/test-infra/prow/gcsupload"
	"k8s.io/test-infra/prow/pod-utils/wrapper"
	"k8s.io/test-infra/prow/secretutil"
	"k8s.io/test-infra/prow/testutil"
)

func TestCensor(t *testing.T) {
	preamble := func() string {
		return `In my younger and more vulnerable years my father gave me some advice that I’ve been turning over in my mind ever since.`
	}

	var testCases = []struct {
		name          string
		input, output string
		secrets       []string
		bufferSize    int
	}{
		{
			name:       "input smaller than buffer size",
			input:      preamble()[:100],
			secrets:    []string{"younger", "my"},
			output:     "In ** ******* and more vulnerable years ** father gave me some advice that I’ve been turning over ",
			bufferSize: 200,
		},
		{
			name:       "input larger than buffer size, not a multiple",
			input:      preamble()[:100],
			secrets:    []string{"younger", "my"},
			output:     "In ** ******* and more vulnerable years ** father gave me some advice that I’ve been turning over ",
			bufferSize: 16,
		},
	}
	for _, testCase := range testCases {
		t.Run(testCase.name, func(t *testing.T) {
			censorer := secretutil.NewCensorer()
			censorer.Refresh(testCase.secrets...)
			input := ioutil.NopCloser(bytes.NewBufferString(testCase.input))
			outputSink := &bytes.Buffer{}
			output := nopWriteCloser(outputSink)
			if err := censor(input, output, censorer, testCase.bufferSize); err != nil {
				t.Fatalf("expected no error from censor, got %v", err)
			}
			if diff := cmp.Diff(outputSink.String(), testCase.output); diff != "" {
				t.Fatalf("got incorrect output after censoring: %v", diff)
			}
		})
	}

}

func nopWriteCloser(w io.Writer) io.WriteCloser {
	return &nopCloser{Writer: w}
}

type nopCloser struct {
	io.Writer
}

func (nopCloser) Close() error { return nil }

const inputDir = "testdata/input"

func copyTestData(t *testing.T) string {
	tempDir := t.TempDir()
	if err := filepath.Walk(inputDir, func(path string, info os.FileInfo, err error) error {
		if err != nil {
			return err
		}
		relpath, _ := filepath.Rel(inputDir, path) // this errors when it's not relative, but that's known here
		dest := filepath.Join(tempDir, relpath)
		if info.IsDir() {
			return os.MkdirAll(dest, info.Mode())
		}
		if info.Mode()&os.ModeSymlink == os.ModeSymlink {
			link, err := os.Readlink(path)
			if err != nil {
				t.Fatalf("failed to read input link: %v", err)
			}
			return os.Symlink(link, dest)
		}
		if info.Name() == "link" {
			link, err := ioutil.ReadFile(path)
			if err != nil {
				t.Fatalf("failed to read input link: %v", err)
			}
			return os.Symlink(string(link), dest)
		}
		out, err := os.Create(dest)
		if err != nil {
			return err
		}
		defer func() {
			if err := out.Close(); err != nil {
				t.Fatalf("could not close output file: %v", err)
			}
		}()
		in, err := os.Open(path)
		if err != nil {
			return err
		}
		defer func() {
			if err := in.Close(); err != nil {
				t.Fatalf("could not close input file: %v", err)
			}
		}()
		if _, err := io.Copy(out, in); err != nil {
			return err
		}
		return nil
	}); err != nil {
		t.Fatalf("failed to copy input to temp dir: %v", err)
	}
	return tempDir
}

func TestCensorIntegration(t *testing.T) {
	// copy input to a temp dir so we don't touch the golden input files
	tempDir := copyTestData(t)
	// also, tar the input - it's not trivial to diff two tarballs while only caring about
	// file content, not metadata, so this test will tar up the archive from the input and
	// untar it after the fact for simple diffs and updates
	archiveDir := filepath.Join(tempDir, "artifacts/archive")
	archiveFile := filepath.Join(tempDir, "artifacts/archive.tar.gz")
	if err := archive(archiveDir, archiveFile); err != nil {
		t.Fatalf("failed to archive input: %v", err)
	}

	bufferSize := 1
	options := Options{
		GcsOptions: &gcsupload.Options{
			Items: []string{filepath.Join(tempDir, "artifacts")},
		},
		Entries: []wrapper.Options{
			{ProcessLog: filepath.Join(tempDir, "logs/one.log")},
			{ProcessLog: filepath.Join(tempDir, "logs/two.log")},
			{ProcessLog: filepath.Join(tempDir, "logs/three.log")},
		},
		CensoringOptions: &CensoringOptions{
			SecretDirectories: []string{"testdata/secrets"},
			// this will be smaller than the size of a secret, so this tests our buffer calculation
			CensoringBufferSize: &bufferSize,
			ExcludeDirectories:  []string{"**/exclude"},
			IniFilenames:        []string{".awscred"},
		},
	}
	if err := options.censor(); err != nil {
		t.Fatalf("got an error from censoring: %v", err)
	}

	if err := unarchive(archiveFile, archiveDir); err != nil {
		t.Fatalf("failed to unarchive input: %v", err)
	}
	if err := os.Remove(archiveFile); err != nil {
		t.Fatalf("failed to removce archive: %v", err)
	}

	testutil.CompareWithFixtureDir(t, "testdata/output", tempDir)
}

func TestArchiveMatchesTar(t *testing.T) {
	tempDir := t.TempDir()
	archiveOutput := filepath.Join(tempDir, "archive.tar.gz")
	archiveDir := "testdata/archives"
	archiveInputs := filepath.Join(archiveDir, "archive/")
	if err := archive(archiveInputs, archiveOutput); err != nil {
		t.Fatalf("failed to archive input: %v", err)
	}
	tarOutput := t.TempDir()
	cmd := exec.Command("tar", "-C", tarOutput, "-xzvf", archiveOutput)
	if out, err := cmd.CombinedOutput(); err != nil {
		t.Fatalf("could not run tar: %v:\n %s", err, string(out))
	}
	testutil.CompareWithFixtureDir(t, tarOutput, archiveInputs)
}

func TestUnarchive(t *testing.T) {
	unarchiveOutput := t.TempDir()
	archiveDir := "testdata/archives"
	archiveInputs := filepath.Join(archiveDir, "archive/")
	archiveFile := filepath.Join(archiveDir, "archive.tar.gz")
	if err := unarchive(archiveFile, unarchiveOutput); err != nil {
		t.Fatalf("failed to unarchive input: %v", err)
	}
	testutil.CompareWithFixtureDir(t, archiveInputs, unarchiveOutput)
}

func TestUnarchiveMatchesTar(t *testing.T) {
	unarchiveOutput := t.TempDir()
	archiveDir := "testdata/archives"
	archiveFile := filepath.Join(archiveDir, "archive.tar.gz")
	if err := unarchive(archiveFile, unarchiveOutput); err != nil {
		t.Fatalf("failed to unarchive input: %v", err)
	}
	tarOutput := t.TempDir()
	cmd := exec.Command("tar", "-C", tarOutput, "-xzvf", archiveFile)
	if out, err := cmd.CombinedOutput(); err != nil {
		t.Fatalf("could not run tar: %v:\n %s", err, string(out))
	}
	testutil.CompareWithFixtureDir(t, tarOutput, unarchiveOutput)
}

func TestRoundTrip(t *testing.T) {
	tempDir := t.TempDir()
	archiveOutput := filepath.Join(tempDir, "archive.tar.gz")
	unarchiveOutput := filepath.Join(tempDir, "archive/")
	archiveDir := "testdata/archives"
	archiveInputs := filepath.Join(archiveDir, "archive/")
	if err := archive(archiveInputs, archiveOutput); err != nil {
		t.Fatalf("failed to archive input: %v", err)
	}
	if err := unarchive(archiveOutput, unarchiveOutput); err != nil {
		t.Fatalf("failed to unarchive input: %v", err)
	}
	testutil.CompareWithFixtureDir(t, archiveInputs, unarchiveOutput)
}

func TestLoadDockerCredentials(t *testing.T) {
	expected := []string{"a", "b", "c", "d", "e", "f"}
	dockercfgraw := []byte(`{
	"registry": {
		"password": "a",
		"auth": "b"
	},
	"other": {
		"password": "c",
		"auth": "d"
	},
	"third": {
		"auth": "e"
	},
	"fourth": {
		"password": "f"
	}
}`)
	dockerconfigjsonraw := []byte(`{
	"auths": {
		"registry": {
			"password": "a",
			"auth": "b"
		},
		"other": {
			"password": "c",
			"auth": "d"
		},
		"third": {
			"auth": "e"
		},
		"fourth": {
			"password": "f"
		}
	}
}`)
	malformed := []byte(`notreallyjson`)

	if _, err := loadDockercfgAuths(malformed); err == nil {
		t.Error("dockercfg: expected loading malformed data to error, but it did not")
	}
	if _, err := loadDockerconfigJsonAuths(malformed); err == nil {
		t.Error("dockerconfigjson: expected loading malformed data to error, but it did not")
	}

	actual, err := loadDockercfgAuths(dockercfgraw)
	if err != nil {
		t.Errorf("dockercfg: expected loading data not to error, but it did: %v", err)
	}
	sort.Strings(actual)
	if diff := cmp.Diff(actual, expected); diff != "" {
		t.Errorf("dockercfg: got incorrect values: %s", err)
	}

	actual, err = loadDockerconfigJsonAuths(dockerconfigjsonraw)
	if err != nil {
		t.Errorf("dockerconfigjson: expected loading data not to error, but it did: %v", err)
	}
	sort.Strings(actual)
	if diff := cmp.Diff(actual, expected); diff != "" {
		t.Errorf("dockerconfigjson: got incorrect values: %s", err)
	}
}

func TestShouldCensor(t *testing.T) {
	var testCases = []struct {
		name     string
		path     string
		options  CensoringOptions
		expected bool
	}{
		{
			name:     "no options defaults to include",
			options:  CensoringOptions{},
			path:     "/usr/bin/bash",
			expected: true,
		},
		{
			name: "not matching include defaults to false",
			options: CensoringOptions{
				IncludeDirectories: []string{"/tmp/**/*"},
			},
			path:     "/usr/bin/bash",
			expected: false,
		},
		{
			name: "matching include censors",
			options: CensoringOptions{
				IncludeDirectories: []string{"/usr/**/*"},
			},
			path:     "/usr/bin/bash",
			expected: true,
		},
		{
			name: "matching include and exclude does not censor",
			options: CensoringOptions{
				IncludeDirectories: []string{"/usr/**/*"},
				ExcludeDirectories: []string{"/usr/bin/**/*"},
			},
			path:     "/usr/bin/bash",
			expected: false,
		},
		{
			name: "matching exclude does not censor",
			options: CensoringOptions{
				ExcludeDirectories: []string{"/usr/bin/**/*"},
			},
			path:     "/usr/bin/bash",
			expected: false,
		},
	}
	for _, testCase := range testCases {
		t.Run(testCase.name, func(t *testing.T) {
			should, err := shouldCensor(testCase.options, testCase.path)
			if err != nil {
				t.Fatalf("%s: got an error from shouldCensor: %v", testCase.name, err)
			}
			if should != testCase.expected {
				t.Errorf("%s: expected %v, got %v", testCase.name, testCase.expected, should)
			}
		})
	}
}
<commit_msg>Add a testcase showing failed censoring on corrupt archive<commit_after>/*
Copyright 2021 The Kubernetes Authors.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package sidecar

import (
	"bytes"
	"fmt"
	"io"
	"io/ioutil"
	"os"
	"os/exec"
	"path/filepath"
	"sort"
	"testing"

	"github.com/google/go-cmp/cmp"
	"k8s.io/test-infra/prow/gcsupload"
	"k8s.io/test-infra/prow/pod-utils/wrapper"

	"k8s.io/test-infra/prow/secretutil"
	"k8s.io/test-infra/prow/testutil"
)

func TestCensor(t *testing.T) {
	preamble := func() string {
		return `In my younger and more vulnerable years my father gave me some advice that I’ve been turning over in my mind ever since.`
	}

	var testCases = []struct {
		name          string
		input, output string
		secrets       []string
		bufferSize    int
	}{
		{
			name:       "input smaller than buffer size",
			input:      preamble()[:100],
			secrets:    []string{"younger", "my"},
			output:     "In ** ******* and more vulnerable years ** father gave me some advice that I’ve been turning over ",
			bufferSize: 200,
		},
		{
			name:       "input larger than buffer size, not a multiple",
			input:      preamble()[:100],
			secrets:    []string{"younger", "my"},
			output:     "In ** ******* and more vulnerable years ** father gave me some advice that I’ve been turning over ",
			bufferSize: 16,
		},
	}
	for _, testCase := range testCases {
		t.Run(testCase.name, func(t *testing.T) {
			censorer := secretutil.NewCensorer()
			censorer.Refresh(testCase.secrets...)
			input := ioutil.NopCloser(bytes.NewBufferString(testCase.input))
			outputSink := &bytes.Buffer{}
			output := nopWriteCloser(outputSink)
			if err := censor(input, output, censorer, testCase.bufferSize); err != nil {
				t.Fatalf("expected no error from censor, got %v", err)
			}
			if diff := cmp.Diff(outputSink.String(), testCase.output); diff != "" {
				t.Fatalf("got incorrect output after censoring: %v", diff)
			}
		})
	}

}

func nopWriteCloser(w io.Writer) io.WriteCloser {
	return &nopCloser{Writer: w}
}

type nopCloser struct {
	io.Writer
}

func (nopCloser) Close() error { return nil }

const inputDir = "testdata/input"

func copyTestData(t *testing.T) string {
	tempDir := t.TempDir()
	if err := filepath.Walk(inputDir, func(path string, info os.FileInfo, err error) error {
		if err != nil {
			return err
		}
		relpath, _ := filepath.Rel(inputDir, path) // this errors when it's not relative, but that's known here
		dest := filepath.Join(tempDir, relpath)
		if info.IsDir() {
			return os.MkdirAll(dest, info.Mode())
		}
		if info.Mode()&os.ModeSymlink == os.ModeSymlink {
			link, err := os.Readlink(path)
			if err != nil {
				t.Fatalf("failed to read input link: %v", err)
			}
			return os.Symlink(link, dest)
		}
		if info.Name() == "link" {
			link, err := ioutil.ReadFile(path)
			if err != nil {
				t.Fatalf("failed to read input link: %v", err)
			}
			return os.Symlink(string(link), dest)
		}
		out, err := os.Create(dest)
		if err != nil {
			return err
		}
		defer func() {
			if err := out.Close(); err != nil {
				t.Fatalf("could not close output file: %v", err)
			}
		}()
		in, err := os.Open(path)
		if err != nil {
			return err
		}
		defer func() {
			if err := in.Close(); err != nil {
				t.Fatalf("could not close input file: %v", err)
			}
		}()
		if _, err := io.Copy(out, in); err != nil {
			return err
		}
		return nil
	}); err != nil {
		t.Fatalf("failed to copy input to temp dir: %v", err)
	}
	return tempDir
}

const (
	artifactPath = "artifacts"
	logPath      = "logs"
)

func optionsForTestData(location string) Options {
	return Options{
		GcsOptions: &gcsupload.Options{
			Items: []string{filepath.Join(location, artifactPath)},
		},
		Entries: []wrapper.Options{
			{ProcessLog: filepath.Join(location, logPath, "one.log")},
			{ProcessLog: filepath.Join(location, logPath, "two.log")},
			{ProcessLog: filepath.Join(location, logPath, "three.log")},
		},
		CensoringOptions: &CensoringOptions{
			SecretDirectories:  []string{"testdata/secrets"},
			ExcludeDirectories: []string{"**/exclude"},
			IniFilenames:       []string{".awscred"},
		},
	}

}

// TestCensorRobustnessForCorruptArchive tests that all possible artifacts are censored even in
// the presence of a corrupt archive (test that the censoring does not bail out too soon)
func TestCensorRobustnessForCorruptArchive(t *testing.T) {
	// copy input to a temp dir so we don't touch the golden input files
	tempDir := copyTestData(t)
	// also, tar the input - it's not trivial to diff two tarballs while only caring about
	// file content, not metadata, so this test will tar up the archive from the input and
	// untar it after the fact for simple diffs and updates
	archiveDir := filepath.Join(tempDir, artifactPath, "archive")

	// create a corrupt archive as well to test for resiliency
	corruptArchiveFile := filepath.Join(tempDir, artifactPath, "corrupt.tar.gz")
	if err := archive(archiveDir, corruptArchiveFile); err != nil {
		t.Fatalf("failed to archive input: %v", err)
	}
	file, err := os.OpenFile(corruptArchiveFile, os.O_RDWR, 0666)
	if err != nil {
		t.Fatalf("failed to open archived input: %v", err)
	}
	raw, err := ioutil.ReadAll(file)
	if err != nil {
		t.Fatalf("failed to read archived input: %v", err)
	}
	// the third byte in a gzip archive is a flag; some values are
	// reserved - if we set this to be some corrupt value, we expect
	// that the archive will be detected as gzip but that reading this
	// archive to be impossible.
	// ref: https://datatracker.ietf.org/doc/html/rfc1952#page-5
	raw[3] = 0x6
	if n, err := file.WriteAt(raw, 0); err != nil || n != len(raw) {
		t.Fatalf("failed to write corrupted archive: wrote %d (of %d) bytes, err: %v", n, len(raw), err)
	}
	options := optionsForTestData(tempDir)

	// We expect the error to happen
	expectedError := fmt.Sprintf("could not walk items to censor them: could not censor archive %s: could not unpack archive: could not read archive: unexpected EOF", corruptArchiveFile)
	if diff := cmp.Diff(expectedError, options.censor().Error()); diff != "" {
		t.Errorf("censor() did not end with expected error:\n%s", diff)
	}

	if err := os.Remove(corruptArchiveFile); err != nil {
		t.Fatalf("failed to remove archive: %v", err)
	}

	testutil.CompareWithFixtureDir(t, "testdata/output", tempDir)
}

func TestCensorIntegration(t *testing.T) {
	// copy input to a temp dir so we don't touch the golden input files
	tempDir := copyTestData(t)
	// also, tar the input - it's not trivial to diff two tarballs while only caring about
	// file content, not metadata, so this test will tar up the archive from the input and
	// untar it after the fact for simple diffs and updates
	archiveDir := filepath.Join(tempDir, artifactPath, "archive")
	archiveFile := filepath.Join(tempDir, artifactPath, "archive.tar.gz")
	if err := archive(archiveDir, archiveFile); err != nil {
		t.Fatalf("failed to archive input: %v", err)
	}

	bufferSize := 1
	options := optionsForTestData(tempDir)

	// this will be smaller than the size of a secret, so this tests our buffer calculation
	options.CensoringOptions.CensoringBufferSize = &bufferSize

	if err := options.censor(); err != nil {
		t.Fatalf("got an error from censoring: %v", err)
	}

	if err := unarchive(archiveFile, archiveDir); err != nil {
		t.Fatalf("failed to unarchive input: %v", err)
	}
	if err := os.Remove(archiveFile); err != nil {
		t.Fatalf("failed to remove archive: %v", err)
	}

	testutil.CompareWithFixtureDir(t, "testdata/output", tempDir)
}

func TestArchiveMatchesTar(t *testing.T) {
	tempDir := t.TempDir()
	archiveOutput := filepath.Join(tempDir, "archive.tar.gz")
	archiveDir := "testdata/archives"
	archiveInputs := filepath.Join(archiveDir, "archive/")
	if err := archive(archiveInputs, archiveOutput); err != nil {
		t.Fatalf("failed to archive input: %v", err)
	}
	tarOutput := t.TempDir()
	cmd := exec.Command("tar", "-C", tarOutput, "-xzvf", archiveOutput)
	if out, err := cmd.CombinedOutput(); err != nil {
		t.Fatalf("could not run tar: %v:\n %s", err, string(out))
	}
	testutil.CompareWithFixtureDir(t, tarOutput, archiveInputs)
}

func TestUnarchive(t *testing.T) {
	unarchiveOutput := t.TempDir()
	archiveDir := "testdata/archives"
	archiveInputs := filepath.Join(archiveDir, "archive/")
	archiveFile := filepath.Join(archiveDir, "archive.tar.gz")
	if err := unarchive(archiveFile, unarchiveOutput); err != nil {
		t.Fatalf("failed to unarchive input: %v", err)
	}
	testutil.CompareWithFixtureDir(t, archiveInputs, unarchiveOutput)
}

func TestUnarchiveMatchesTar(t *testing.T) {
	unarchiveOutput := t.TempDir()
	archiveDir := "testdata/archives"
	archiveFile := filepath.Join(archiveDir, "archive.tar.gz")
	if err := unarchive(archiveFile, unarchiveOutput); err != nil {
		t.Fatalf("failed to unarchive input: %v", err)
	}
	tarOutput := t.TempDir()
	cmd := exec.Command("tar", "-C", tarOutput, "-xzvf", archiveFile)
	if out, err := cmd.CombinedOutput(); err != nil {
		t.Fatalf("could not run tar: %v:\n %s", err, string(out))
	}
	testutil.CompareWithFixtureDir(t, tarOutput, unarchiveOutput)
}

func TestRoundTrip(t *testing.T) {
	tempDir := t.TempDir()
	archiveOutput := filepath.Join(tempDir, "archive.tar.gz")
	unarchiveOutput := filepath.Join(tempDir, "archive/")
	archiveDir := "testdata/archives"
	archiveInputs := filepath.Join(archiveDir, "archive/")
	if err := archive(archiveInputs, archiveOutput); err != nil {
		t.Fatalf("failed to archive input: %v", err)
	}
	if err := unarchive(archiveOutput, unarchiveOutput); err != nil {
		t.Fatalf("failed to unarchive input: %v", err)
	}
	testutil.CompareWithFixtureDir(t, archiveInputs, unarchiveOutput)
}

func TestLoadDockerCredentials(t *testing.T) {
	expected := []string{"a", "b", "c", "d", "e", "f"}
	dockercfgraw := []byte(`{
	"registry": {
		"password": "a",
		"auth": "b"
	},
	"other": {
		"password": "c",
		"auth": "d"
	},
	"third": {
		"auth": "e"
	},
	"fourth": {
		"password": "f"
	}
}`)
	dockerconfigjsonraw := []byte(`{
	"auths": {
		"registry": {
			"password": "a",
			"auth": "b"
		},
		"other": {
			"password": "c",
			"auth": "d"
		},
		"third": {
			"auth": "e"
		},
		"fourth": {
			"password": "f"
		}
	}
}`)
	malformed := []byte(`notreallyjson`)

	if _, err := loadDockercfgAuths(malformed); err == nil {
		t.Error("dockercfg: expected loading malformed data to error, but it did not")
	}
	if _, err := loadDockerconfigJsonAuths(malformed); err == nil {
		t.Error("dockerconfigjson: expected loading malformed data to error, but it did not")
	}

	actual, err := loadDockercfgAuths(dockercfgraw)
	if err != nil {
		t.Errorf("dockercfg: expected loading data not to error, but it did: %v", err)
	}
	sort.Strings(actual)
	if diff := cmp.Diff(actual, expected); diff != "" {
		t.Errorf("dockercfg: got incorrect values: %s", err)
	}

	actual, err = loadDockerconfigJsonAuths(dockerconfigjsonraw)
	if err != nil {
		t.Errorf("dockerconfigjson: expected loading data not to error, but it did: %v", err)
	}
	sort.Strings(actual)
	if diff := cmp.Diff(actual, expected); diff != "" {
		t.Errorf("dockerconfigjson: got incorrect values: %s", err)
	}
}

func TestShouldCensor(t *testing.T) {
	var testCases = []struct {
		name     string
		path     string
		options  CensoringOptions
		expected bool
	}{
		{
			name:     "no options defaults to include",
			options:  CensoringOptions{},
			path:     "/usr/bin/bash",
			expected: true,
		},
		{
			name: "not matching include defaults to false",
			options: CensoringOptions{
				IncludeDirectories: []string{"/tmp/**/*"},
			},
			path:     "/usr/bin/bash",
			expected: false,
		},
		{
			name: "matching include censors",
			options: CensoringOptions{
				IncludeDirectories: []string{"/usr/**/*"},
			},
			path:     "/usr/bin/bash",
			expected: true,
		},
		{
			name: "matching include and exclude does not censor",
			options: CensoringOptions{
				IncludeDirectories: []string{"/usr/**/*"},
				ExcludeDirectories: []string{"/usr/bin/**/*"},
			},
			path:     "/usr/bin/bash",
			expected: false,
		},
		{
			name: "matching exclude does not censor",
			options: CensoringOptions{
				ExcludeDirectories: []string{"/usr/bin/**/*"},
			},
			path:     "/usr/bin/bash",
			expected: false,
		},
	}
	for _, testCase := range testCases {
		t.Run(testCase.name, func(t *testing.T) {
			should, err := shouldCensor(testCase.options, testCase.path)
			if err != nil {
				t.Fatalf("%s: got an error from shouldCensor: %v", testCase.name, err)
			}
			if should != testCase.expected {
				t.Errorf("%s: expected %v, got %v", testCase.name, testCase.expected, should)
			}
		})
	}
}
<|endoftext|>


<commit_before>// Copyright (C) 2014 Miquel Sabaté Solà <mikisabate@gmail.com>
// This file is licensed under the MIT license.
// See the LICENSE file.

package app

import (
	"encoding/csv"
	"net/http"
	"net/http/httptest"
	"net/url"
	"testing"
	"time"

	"github.com/gorilla/mux"
	"github.com/stretchr/testify/assert"
)

func TestPlayersNew(t *testing.T) {
	InitTest()
	defer CloseDB()

	req, err := http.NewRequest("GET", "/", nil)
	assert.Nil(t, err)
	w := httptest.NewRecorder()
	PlayersNew(w, req)

	assert.Equal(t, w.Code, 200)
	assert.Contains(t, w.Body.String(),
		"Crear jugador")
}

func TestPlayersCreate(t *testing.T) {
	InitTest()
	defer CloseDB()

	param := make(url.Values)

	// "name" is not set.
	req, err := http.NewRequest("POST", "/", nil)
	assert.Nil(t, err)
	req.PostForm = param
	w := httptest.NewRecorder()
	PlayersCreate(w, req)

	assert.Equal(t, w.Code, 302)
	count, err := Db.SelectInt("select count(*) from players")
	assert.Equal(t, count, 0)

	// Creating a player.
	param["name"] = []string{"player"}
	req, err = http.NewRequest("POST", "/", nil)
	assert.Nil(t, err)
	req.PostForm = param
	w = httptest.NewRecorder()
	PlayersCreate(w, req)

	assert.Equal(t, w.Code, 302)
	count, err = Db.SelectInt("select count(*) from players")
	assert.Equal(t, count, 1)

	// You can't create the same player.
	req, err = http.NewRequest("POST", "/", nil)
	assert.Nil(t, err)
	req.PostForm = param
	w = httptest.NewRecorder()
	PlayersCreate(w, req)

	assert.Equal(t, w.Code, 302)
	count, err = Db.SelectInt("select count(*) from players")
	assert.Equal(t, count, 1)
}

func TestPlayersShow(t *testing.T) {
	// TODO
}

func TestPlayersUpdate(t *testing.T) {
	InitTest()
	defer CloseDB()

	createPlayer("one", []int{1, 2, 3})
	var p Player
	err := Db.SelectOne(&p, "select * from players where name=$1", "one")
	assert.Nil(t, err)

	param := make(url.Values)
	param["name"] = []string{"another"}

	req, err := http.NewRequest("POST", "/"+p.Id, nil)
	assert.Nil(t, err)
	req.PostForm = param
	w := httptest.NewRecorder()

	m := mux.NewRouter()
	m.HandleFunc("/{id}", PlayersUpdate)
	m.ServeHTTP(w, req)

	assert.Equal(t, w.Code, 302)
	err = Db.SelectOne(&p, "select * from players")
	assert.Equal(t, p.Name, "another")
}

func TestPlayersDelete(t *testing.T) {
	InitTest()
	defer CloseDB()

	createPlayer("one", []int{1, 2, 3})
	var p Player
	err := Db.SelectOne(&p, "select * from players where name=$1", "one")
	assert.Nil(t, err)

	// Nothing has to happen since we picked the wrong guy.
	param := make(url.Values)
	param["name"] = []string{"another"}

	req, err := http.NewRequest("POST", "/"+p.Id, nil)
	assert.Nil(t, err)
	req.PostForm = param
	w := httptest.NewRecorder()

	m := mux.NewRouter()
	m.HandleFunc("/{id}", PlayersDelete)
	m.ServeHTTP(w, req)

	assert.Equal(t, w.Code, 302)
	err = Db.SelectOne(&p, "select * from players")
	assert.Equal(t, p.Name, "one")

	// Now we pick the right one.
	param["name"] = []string{"one"}
	req, err = http.NewRequest("POST", "/"+p.Id, nil)
	assert.Nil(t, err)
	req.PostForm = param
	w = httptest.NewRecorder()
	m.ServeHTTP(w, req)

	assert.Equal(t, w.Code, 302)
	count, err := Db.SelectInt("select count(*) from players")
	assert.Equal(t, count, 0)
}

func TestPlayersRate(t *testing.T) {
	// TODO
}

func TestPlayersRated(t *testing.T) {
	InitTest()
	defer CloseDB()

	m := mux.NewRouter()
	m.HandleFunc("/{id}", PlayersRated)

	// Ok.
	r, err := http.NewRequest("GET", "/1", nil)
	assert.Nil(t, err)
	w := httptest.NewRecorder()
	m.ServeHTTP(w, r)
	assert.Equal(t, w.Code, 200)
	assert.Contains(t, w.Body.String(), "
Ho tenim !
")

	// Error.
	param := make(url.Values)
	param["error"] = []string{"true"}
	r, err = http.NewRequest("GET", "/1", nil)
	assert.Nil(t, err)
	r.PostForm = param
	w = httptest.NewRecorder()
	m.ServeHTTP(w, r)
	assert.Equal(t, w.Code, 200)
	assert.Contains(t, w.Body.String(), "
Error !
")
}

func TestPlayersCsv(t *testing.T) {
	InitTest()
	defer CloseDB()

	// Someone that doesn't exist.
	m := mux.NewRouter()
	m.HandleFunc("/{id}", PlayersCsv)
	r, err := http.NewRequest("GET", "/1", nil)
	assert.Nil(t, err)
	w := httptest.NewRecorder()
	m.ServeHTTP(w, r)

	assert.Equal(t, w.Code, 302)
	header := w.Header()
	assert.Equal(t, header["Location"], []string{"/"})

	// Let's create a couple of players and some ratings.
	createPlayer("one", []int{1, 2, 3})
	createPlayer("another", []int{0, 3, 8})

	var p Player
	err = Db.SelectOne(&p, "select * from players where name=$1", "one")
	assert.Nil(t, err)

	// Perform the request.
	r, err = http.NewRequest("GET", "/"+p.Id, nil)
	assert.Nil(t, err)
	w = httptest.NewRecorder()
	m.ServeHTTP(w, r)

	// HTTP
	assert.Equal(t, w.Code, 200)
	header = w.Header()
	assert.Equal(t, header["Content-Type"][0], "text/csv")
	assert.Equal(t, header["Content-Disposition"][0],
		"attachment;filename=one.csv")

	dt := fmtDate(time.Now())

	// CSV
	re := csv.NewReader(w.Body)
	testCSV(t, re, 4, "one", "1", "3", "2.00")
	testCSV(t, re, 7, "one", "1", dt, "2", dt, "3", dt)
}
<commit_msg>app: the test suite is now covering more than 90% of the app package.<commit_after>// Copyright (C) 2014 Miquel Sabaté Solà <mikisabate@gmail.com>
// This file is licensed under the MIT license.
// See the LICENSE file.

package app

import (
	"encoding/csv"
	"fmt"
	"net/http"
	"net/http/httptest"
	"net/url"
	"testing"
	"time"

	"github.com/gorilla/mux"
	"github.com/stretchr/testify/assert"
)

func TestPlayersNew(t *testing.T) {
	InitTest()
	defer CloseDB()

	req, err := http.NewRequest("GET", "/", nil)
	assert.Nil(t, err)
	w := httptest.NewRecorder()
	PlayersNew(w, req)

	assert.Equal(t, w.Code, 200)
	assert.Contains(t, w.Body.String(),
		"Crear jugador")
}

func TestPlayersCreate(t *testing.T) {
	InitTest()
	defer CloseDB()

	param := make(url.Values)

	// "name" is not set.
	req, err := http.NewRequest("POST", "/", nil)
	assert.Nil(t, err)
	req.PostForm = param
	w := httptest.NewRecorder()
	PlayersCreate(w, req)

	assert.Equal(t, w.Code, 302)
	count, err := Db.SelectInt("select count(*) from players")
	assert.Equal(t, count, 0)

	// Creating a player.
	param["name"] = []string{"player"}
	req, err = http.NewRequest("POST", "/", nil)
	assert.Nil(t, err)
	req.PostForm = param
	w = httptest.NewRecorder()
	PlayersCreate(w, req)

	assert.Equal(t, w.Code, 302)
	count, err = Db.SelectInt("select count(*) from players")
	assert.Equal(t, count, 1)

	// You can't create the same player.
	req, err = http.NewRequest("POST", "/", nil)
	assert.Nil(t, err)
	req.PostForm = param
	w = httptest.NewRecorder()
	PlayersCreate(w, req)

	assert.Equal(t, w.Code, 302)
	count, err = Db.SelectInt("select count(*) from players")
	assert.Equal(t, count, 1)
}

func TestPlayersShow(t *testing.T) {
	InitTest()
	defer CloseDB()

	m := mux.NewRouter()
	m.HandleFunc("/{id}", PlayersShow)

	// No players.
	req, err := http.NewRequest("GET", "/1", nil)
	assert.Nil(t, err)
	w := httptest.NewRecorder()
	m.ServeHTTP(w, req)
	assert.Equal(t, w.Code, 302)
	errorUrl := []string{"/"}
	assert.Equal(t, w.Header()["Location"], errorUrl)

	// Create some players.
	createPlayer("one", []int{})
	createPlayer("another", []int{1, 2, 3})
	var one, another Player
	err = Db.SelectOne(&one, "select * from players where name=$1", "one")
	assert.Nil(t, err)
	err = Db.SelectOne(&another, "select * from players where name=$1", "another")
	assert.Nil(t, err)
	createUser("user", "1111")

	// Login and perform a couple of requests.
	req, err = http.NewRequest("GET", "/"+one.Id, nil)
	assert.Nil(t, err)
	w = httptest.NewRecorder()
	login(w, req)
	m.ServeHTTP(w, req)
	assert.Equal(t, w.Code, 200)
	assert.Contains(t, w.Body.String(), "Aquest"+
		" jugador encara no ha valorat cap entrenament.")

	// And now the other player.
	req, err = http.NewRequest("GET", "/"+another.Id, nil)
	assert.Nil(t, err)
	w = httptest.NewRecorder()
	login(w, req)
	m.ServeHTTP(w, req)
	assert.Equal(t, w.Code, 200)
	assert.Contains(t, w.Body.String(),
		`

            Mínim
            Màxim
            Mitjana

            1
            3
            2.00

`)
}

func TestPlayersUpdate(t *testing.T) {
	InitTest()
	defer CloseDB()

	createPlayer("one", []int{1, 2, 3})
	var p Player
	err := Db.SelectOne(&p, "select * from players where name=$1", "one")
	assert.Nil(t, err)

	param := make(url.Values)
	param["name"] = []string{"another"}

	req, err := http.NewRequest("POST", "/"+p.Id, nil)
	assert.Nil(t, err)
	req.PostForm = param
	w := httptest.NewRecorder()

	m := mux.NewRouter()
	m.HandleFunc("/{id}", PlayersUpdate)
	m.ServeHTTP(w, req)

	assert.Equal(t, w.Code, 302)
	err = Db.SelectOne(&p, "select * from players")
	assert.Equal(t, p.Name, "another")
}

func TestPlayersDelete(t *testing.T) {
	InitTest()
	defer CloseDB()

	createPlayer("one", []int{1, 2, 3})
	var p Player
	err := Db.SelectOne(&p, "select * from players where name=$1", "one")
	assert.Nil(t, err)

	// Nothing has to happen since we picked the wrong guy.
	param := make(url.Values)
	param["name"] = []string{"another"}

	req, err := http.NewRequest("POST", "/"+p.Id, nil)
	assert.Nil(t, err)
	req.PostForm = param
	w := httptest.NewRecorder()

	m := mux.NewRouter()
	m.HandleFunc("/{id}", PlayersDelete)
	m.ServeHTTP(w, req)

	assert.Equal(t, w.Code, 302)
	err = Db.SelectOne(&p, "select * from players")
	assert.Equal(t, p.Name, "one")

	// Now we pick the right one.
	param["name"] = []string{"one"}
	req, err = http.NewRequest("POST", "/"+p.Id, nil)
	assert.Nil(t, err)
	req.PostForm = param
	w = httptest.NewRecorder()
	m.ServeHTTP(w, req)

	assert.Equal(t, w.Code, 302)
	count, err := Db.SelectInt("select count(*) from players")
	assert.Equal(t, count, 0)
}

func TestPlayersRate(t *testing.T) {
	InitTest()
	defer CloseDB()

	// Invalid Rating (it's not a number).
	m := mux.NewRouter()
	m.HandleFunc("/{id}", PlayersRate)
	r, err := http.NewRequest("POST", "/1", nil)
	assert.Nil(t, err)
	w := httptest.NewRecorder()
	m.ServeHTTP(w, r)
	assert.Equal(t, w.Code, 302)
	errorUrl := []string{"/players/1/rate?error=true"}
	assert.Equal(t, w.Header()["Location"], errorUrl)

	// Invalid Rating (out of range number).
	r, err = http.NewRequest("POST", "/1", nil)
	assert.Nil(t, err)
	param := make(url.Values)
	param["rating"] = []string{"11"}
	r.PostForm = param
	w = httptest.NewRecorder()
	m.ServeHTTP(w, r)
	assert.Equal(t, w.Code, 302)
	assert.Equal(t, w.Header()["Location"], errorUrl)

	// Invalid id parameter.
	r, err = http.NewRequest("POST", "/1", nil)
	assert.Nil(t, err)
	param["rating"] = []string{"5"}
	r.PostForm = param
	w = httptest.NewRecorder()
	m.ServeHTTP(w, r)
	assert.Equal(t, w.Code, 302)
	assert.Equal(t, w.Header()["Location"], errorUrl)

	// Ok.
	createPlayer("user", []int{1, 2, 3})
	var p Player
	err = Db.SelectOne(&p, "select * from players where name=$1", "user")
	assert.Nil(t, err)

	r, err = http.NewRequest("POST", "/"+p.Id, nil)
	assert.Nil(t, err)
	r.PostForm = param
	w = httptest.NewRecorder()
	m.ServeHTTP(w, r)
	assert.Equal(t, w.Code, 302)
	okUrl := []string{fmt.Sprintf("/players/%v/rate", p.Id)}
	assert.Equal(t, w.Header()["Location"], okUrl)
}

func TestPlayersRated(t *testing.T) {
	InitTest()
	defer CloseDB()

	m := mux.NewRouter()
	m.HandleFunc("/{id}", PlayersRated)

	// Ok.
	r, err := http.NewRequest("GET", "/1", nil)
	assert.Nil(t, err)
	w := httptest.NewRecorder()
	m.ServeHTTP(w, r)
	assert.Equal(t, w.Code, 200)
	assert.Contains(t, w.Body.String(), "
Ho tenim !
")

	// Error.
	param := make(url.Values)
	param["error"] = []string{"true"}
	r, err = http.NewRequest("GET", "/1", nil)
	assert.Nil(t, err)
	r.PostForm = param
	w = httptest.NewRecorder()
	m.ServeHTTP(w, r)
	assert.Equal(t, w.Code, 200)
	assert.Contains(t, w.Body.String(), "
Error !
")
}

func TestPlayersCsv(t *testing.T) {
	InitTest()
	defer CloseDB()

	// Someone that doesn't exist.
	m := mux.NewRouter()
	m.HandleFunc("/{id}", PlayersCsv)
	r, err := http.NewRequest("GET", "/1", nil)
	assert.Nil(t, err)
	w := httptest.NewRecorder()
	m.ServeHTTP(w, r)

	assert.Equal(t, w.Code, 302)
	header := w.Header()
	assert.Equal(t, header["Location"], []string{"/"})

	// Let's create a couple of players and some ratings.
	createPlayer("one", []int{1, 2, 3})
	createPlayer("another", []int{0, 3, 8})

	var p Player
	err = Db.SelectOne(&p, "select * from players where name=$1", "one")
	assert.Nil(t, err)

	// Perform the request.
	r, err = http.NewRequest("GET", "/"+p.Id, nil)
	assert.Nil(t, err)
	w = httptest.NewRecorder()
	m.ServeHTTP(w, r)

	// HTTP
	assert.Equal(t, w.Code, 200)
	header = w.Header()
	assert.Equal(t, header["Content-Type"][0], "text/csv")
	assert.Equal(t, header["Content-Disposition"][0],
		"attachment;filename=one.csv")

	dt := fmtDate(time.Now())

	// CSV
	re := csv.NewReader(w.Body)
	testCSV(t, re, 4, "one", "1", "3", "2.00")
	testCSV(t, re, 7, "one", "1", dt, "2", dt, "3", dt)
}
<|endoftext|>


<commit_before>package kafka_test

import (
	"testing"

	kafka "github.com/segmentio/kafka-go"
	"github.com/segmentio/kafka-go/gzip"
	"github.com/segmentio/kafka-go/lz4"
	"github.com/segmentio/kafka-go/snappy"
)

func TestCompression(t *testing.T) {
	msg := kafka.Message{
		Value: []byte("message"),
	}

	testEncodeDecode(t, msg, kafka.CompressionNoneCode)
	testEncodeDecode(t, msg, gzip.Code)
	testEncodeDecode(t, msg, snappy.Code)
	testEncodeDecode(t, msg, lz4.Code)
	testUnknownCodec(t, msg, 42)
}

func testEncodeDecode(t *testing.T, m kafka.Message, codec int8) {
	var r1, r2 kafka.Message
	var err error

	t.Run("encode with "+codecToStr(codec), func(t *testing.T) {
		m.CompressionCodec = codec
		r1, err = m.Encode()
		if err != nil {
			t.Error(err)
		}
	})
	t.Run("encode with "+codecToStr(codec), func(t *testing.T) {
		r2, err = r1.Decode()
		if err != nil {
			t.Error(err)
		}
		if string(r2.Value) != "message" {
			t.Error("bad message")
			t.Log("got: ", r2.Value)
			t.Log("expected: ", []byte("message"))
		}
	})
}

func testUnknownCodec(t *testing.T, m kafka.Message, codec int8) {
	t.Run("unknown codec", func(t *testing.T) {
		expectedErr := "codec 42 not imported."
		m.CompressionCodec = codec
		_, err := m.Encode()
		if err.Error() != expectedErr {
			t.Error("wrong error")
			t.Log("got: ", err)
			t.Error("expected: ", expectedErr)
		}
	})
}

func codecToStr(codec int8) string {
	switch codec {
	case kafka.CompressionNoneCode:
		return "none"
	case gzip.Code:
		return "gzip"
	case snappy.Code:
		return "snappy"
	case lz4.Code:
		return "lz4"
	default:
		return "unknown"
	}
}
<commit_msg>Add compression benchmark<commit_after>package kafka_test

import (
	"testing"

	kafka "github.com/segmentio/kafka-go"
	"github.com/segmentio/kafka-go/gzip"
	"github.com/segmentio/kafka-go/lz4"
	"github.com/segmentio/kafka-go/snappy"
)

func TestCompression(t *testing.T) {
	msg := kafka.Message{
		Value: []byte("message"),
	}

	testEncodeDecode(t, msg, kafka.CompressionNoneCode)
	testEncodeDecode(t, msg, gzip.Code)
	testEncodeDecode(t, msg, snappy.Code)
	testEncodeDecode(t, msg, lz4.Code)
	testUnknownCodec(t, msg, 42)
}

func testEncodeDecode(t *testing.T, m kafka.Message, codec int8) {
	var r1, r2 kafka.Message
	var err error

	t.Run("encode with "+codecToStr(codec), func(t *testing.T) {
		m.CompressionCodec = codec
		r1, err = m.Encode()
		if err != nil {
			t.Error(err)
		}
	})
	t.Run("encode with "+codecToStr(codec), func(t *testing.T) {
		r2, err = r1.Decode()
		if err != nil {
			t.Error(err)
		}
		if string(r2.Value) != "message" {
			t.Error("bad message")
			t.Log("got: ", r2.Value)
			t.Log("expected: ", []byte("message"))
		}
	})
}

func testUnknownCodec(t *testing.T, m kafka.Message, codec int8) {
	t.Run("unknown codec", func(t *testing.T) {
		expectedErr := "codec 42 not imported."
		m.CompressionCodec = codec
		_, err := m.Encode()
		if err.Error() != expectedErr {
			t.Error("wrong error")
			t.Log("got: ", err)
			t.Error("expected: ", expectedErr)
		}
	})
}

func codecToStr(codec int8) string {
	switch codec {
	case kafka.CompressionNoneCode:
		return "none"
	case gzip.Code:
		return "gzip"
	case snappy.Code:
		return "snappy"
	case lz4.Code:
		return "lz4"
	default:
		return "unknown"
	}
}

func BenchmarkCompression(b *testing.B) {
	benchmarks := []struct {
		scenario string
		codec    int8
		function func(*testing.B, int8, []byte)
	}{
		{
			scenario: "None",
			codec:    kafka.CompressionNoneCode,
			function: benchmarkCompression,
		},
		{
			scenario: "GZIP",
			codec:    gzip.Code,
			function: benchmarkCompression,
		},
		{
			scenario: "Snappy",
			codec:    snappy.Code,
			function: benchmarkCompression,
		},
		{
			scenario: "LZ4",
			codec:    lz4.Code,
			function: benchmarkCompression,
		},
	}

	payload := []byte("message")

	for _, benchmark := range benchmarks {
		b.Run(benchmark.scenario, func(b *testing.B) {
			benchmark.function(b, benchmark.codec, payload)
		})
	}

}

func benchmarkCompression(b *testing.B, codec int8, payload []byte) {
	msg := kafka.Message{
		Value:            payload,
		CompressionCodec: codec,
	}

	for i := 0; i < b.N; i++ {
		m1, err := msg.Encode()
		if err != nil {
			b.Fatal(err)
		}

		_, err = m1.Decode()
		if err != nil {
			b.Fatal(err)
		}

	}
}
<|endoftext|>