<commit_before>package git

// everything git and github related

import (
	"encoding/json"
	"errors"
	"fmt"
	"io"
	"log"
	"os"
	"os/exec"
	"path"
	"path/filepath"
	"regexp"
	"strings"
	"time"

	"golang.org/x/net/context"
)

const GIT_BASE_DIR = "repo"

// Invoke a `command` in `workdir` with `args`, connecting up its Stdout and Stderr
func Command(workdir, command string, args ...string) *exec.Cmd {
	// log.Printf("wd = %s cmd = %s, args = %q", workdir, command, append([]string{}, args...))

	cmd := exec.Command(command, args...)
	cmd.Dir = workdir
	cmd.Stdout = os.Stdout
	cmd.Stderr = os.Stderr
	return cmd
}

var (
	ErrEmptyRepoName         = errors.New("Empty repository name")
	ErrEmptyRepoOrganization = errors.New("Empty repository organization")
	ErrUserNotAllowed        = errors.New("User not in the allowed set")
)

type Repository struct {
	Name         string `json:"name"`
	Url          string `json:"url"`
	Organization string `json:"organization"`
}

type Pusher struct {
	Name string `json:"name"`
}

type NonGithub struct {
	NoBuild bool `json:"nobuild"`
	Wait    bool `json:"wait"`
}

type JustNongithub struct {
	NonGithub NonGithub `json:"nongithub"`
}

func ParseJustNongithub(in []byte) (j JustNongithub, err error) {
	err = json.Unmarshal(in, &j)
	return
}

type PushEvent struct {
	Ref        string     `json:"ref"`
	Deleted    bool       `json:"deleted"`
	Repository Repository `json:"repository"`
	After      string     `json:"after"`
	Pusher     Pusher     `json:"pusher"`
	NonGithub  NonGithub  `json:"nongithub"`
	HtmlUrl    string     `json:"html_url"`
}

type GithubStatus struct {
	State       string `json:"state"`
	TargetUrl   string `json:"target_url"`
	Description string `json:"description"`
}

var ErrSkipGithubEndpoint = errors.New("Github endpoint skipped")

// Creates or updates a mirror of `url` at `gitDir` using `git clone --mirror`
func gitLocalMirror(
	url, gitDir, ref string,
	messages io.Writer,
) (err error) {

	// When mirroring, allow up to two minutes before giving up.
	const MirrorTimeout = 2 * time.Minute
	ctx, done := context.WithTimeout(context.Background(), MirrorTimeout)
	defer done()

	if _, err := os.Stat(gitDir); err == nil {
		// Repo already exists, don't need to clone it.

		if gitAlreadyHaveRef(gitDir, ref) {
			// Sha already exists, don't need to fetch.
			// log.Printf("Already have ref: %v %v", gitDir, ref)
			return nil
		}

		return gitFetch(ctx, gitDir, url, messages)
	}

	err = os.MkdirAll(gitDir, 0777)
	if err != nil {
		return fmt.Errorf("gitLocalMirror: %v", err)
	}

	return gitClone(ctx, url, gitDir, messages)
}

func gitClone(
	ctx context.Context,
	url, gitDir string,
	messages io.Writer,
) error {
	cmd := Command(".", "git", "clone", "-q", "--mirror", url, gitDir)
	cmd.Stdout = messages
	cmd.Stderr = messages
	return ContextRun(ctx, cmd)
}

// Run cmd within a net Context.
// If the context is cancelled or times out, the process is killed.
func ContextRun(ctx context.Context, cmd *exec.Cmd) error {
	errc := make(chan error)

	err := cmd.Start()
	if err != nil {
		return err
	}

	go func() { errc <- cmd.Wait() }()

	select {
	case <-ctx.Done():
		cmd.Process.Kill()
		return ctx.Err()
	case err := <-errc:
		return err // err may be nil
	}
	return nil
}

func gitFetch(
	ctx context.Context,
	gitDir, url string,
	messages io.Writer,
) (err error) {

	cmd := Command(gitDir, "git", "fetch", "-f", url, "*:*")
	cmd.Stdout = messages
	cmd.Stderr = messages

	err = ContextRun(ctx, cmd)
	if err != nil {
		// git fetch where there is no update is exit status 1.
		if err.Error() != "exit status 1" {
			return err
		}
	}

	return nil
}

var ShaLike = regexp.MustCompile("[0-9a-zA-Z]{40}")

// Returns true if ref is sha-like and is in the object database.
// The "sha-like" condition ensures that refs like `master` are always
// freshened.
func gitAlreadyHaveRef(gitDir, sha string) bool {
	if !ShaLike.MatchString(sha) {
		return false
	}
	cmd := Command(gitDir, "git", "cat-file", "-t", sha)
	cmd.Stdout = nil
	cmd.Stderr = nil

	err := cmd.Run()
	return err == nil
}

func gitHaveFile(gitDir, ref, path string) (ok bool, err error) {
	cmd := Command(gitDir, "git", "show", fmt.Sprintf("%s:%s", ref, path))
	cmd.Stdout = nil // don't want to see the contents
	err = cmd.Run()
	ok = true
	if err != nil {
		ok = false
		if err.Error() == "exit status 128" {
			// This happens if the file doesn't exist.
			err = nil
		}
	}
	return ok, err
}

func gitRevParse(gitDir, ref string) (sha string, err error) {
	cmd := Command(gitDir, "git", "rev-parse", ref)
	cmd.Stdout = nil // for cmd.Output

	var stdout []byte
	stdout, err = cmd.Output()
	if err != nil {
		return
	}

	sha = strings.TrimSpace(string(stdout))
	return
}

func gitDescribe(gitDir, ref string) (desc string, err error) {
	cmd := Command(gitDir, "git", "describe", "--all", "--tags", "--long", ref)
	cmd.Stdout = nil // for cmd.Output

	var stdout []byte
	stdout, err = cmd.Output()
	if err != nil {
		return
	}

	desc = strings.TrimSpace(string(stdout))
	desc = strings.TrimPrefix(desc, "heads/")
	return
}

func gitCheckout(gitDir, checkoutDir, ref string) error {

	err := os.MkdirAll(checkoutDir, 0777)
	if err != nil {
		return err
	}

	args := []string{"--work-tree", checkoutDir, "checkout", ref, "--", "."}
	err = Command(gitDir, "git", args...).Run()
	if err != nil {
		return err
	}

	// Set mtimes to time file is most recently affected by a commit.
	// This is annoying but unfortunately git sets the timestamps to now,
	// and docker depends on the mtime for cache invalidation.
	err = GitSetMTimes(gitDir, checkoutDir, ref)
	if err != nil {
		return err
	}

	return nil
}

func GitSetMTimes(gitDir, checkoutDir, ref string) error {

	commitTimes, err := GitCommitTimes(gitDir, ref)
	if err != nil {
		return err
	}

	lsFiles := Command(gitDir, "git", "ls-tree", "-r", "--name-only", "-z", ref)
	lsFiles.Stdout = nil
	out, err := lsFiles.Output()
	if err != nil {
		return fmt.Errorf("git ls-files: %v", err)
	}

	dirMTimes := map[string]time.Time{}

	files := strings.Split(strings.TrimRight(string(out), "\x00"), "\x00")
	for _, file := range files {
		mTime, ok := commitTimes[file]
		if !ok {
			return fmt.Errorf("failed to find file in history: %q", file)
		}

		// Loop over each directory in the path to `file`, updating `dirMTimes`
		// to take the most recent time seen.
		dir := filepath.Dir(file)
		for {
			if other, ok := dirMTimes[dir]; ok {
				if mTime.After(other) {
					// file mTime is more recent than previous seen for 'dir'
					dirMTimes[dir] = mTime
				}
			} else {
				// first occurrence of dir
				dirMTimes[dir] = mTime
			}

			// Remove one directory from the path until it isn't changed anymore
			if dir == filepath.Dir(dir) {
				break
			}
			dir = filepath.Dir(dir)
		}

		err = os.Chtimes(filepath.Join(checkoutDir, file), mTime, mTime)
		if err != nil {
			return fmt.Errorf("chtimes: %v", err)
		}

		// fmt.Printf("%s: %s\n", file, mTime)
	}

	for dir, mTime := range dirMTimes {
		err = os.Chtimes(filepath.Join(checkoutDir, dir), mTime, mTime)
		if err != nil {
			return fmt.Errorf("chtimes: %v", err)
		}
		// fmt.Printf("%s: %s\n", dir, mTime)
	}
	return nil
}

type BuildDirectory struct {
	Name, Dir string
	Cleanup   func()
}

func PrepBuildDirectory(
	gitDir, remote, ref string,
) (*BuildDirectory, error) {

	start := time.Now()
	defer func() {
		log.Printf("Took %v to prep %v", time.Since(start), remote)
	}()

	if strings.HasPrefix(remote, "github.com/") {
		remote = "https://" + remote
	}

	gitDir, err := filepath.Abs(gitDir)
	if err != nil {
		return nil, fmt.Errorf("unable to determine abspath: %v", err)
	}

	err = gitLocalMirror(remote, gitDir, ref, os.Stderr)
	if err != nil {
		return nil, fmt.Errorf("unable to gitLocalMirror: %v", err)
	}

	rev, err := gitRevParse(gitDir, ref)
	if err != nil {
		return nil, fmt.Errorf("unable to parse rev: %v", err)
	}

	tagName, err := gitDescribe(gitDir, rev)
	if err != nil {
		return nil, fmt.Errorf("unable to describe %v: %v", rev, err)
	}

	shortRev := rev[:10]
	checkoutPath := path.Join(gitDir, filepath.Join("c/", shortRev))

	err = recursiveCheckout(gitDir, checkoutPath, rev)
	if err != nil {
		return nil, err
	}

	cleanup := func() {
		err := SafeCleanup(checkoutPath)
		if err != nil {
			log.Printf("Error cleaning up path: ", checkoutPath)
		}
	}

	return &BuildDirectory{tagName, checkoutPath, cleanup}, nil
}

func recursiveCheckout(gitDir, checkoutPath, rev string) error {
	err := gitCheckout(gitDir, checkoutPath, rev)
	if err != nil {
		return fmt.Errorf("failed to checkout: %v", err)
	}

	err = PrepSubmodules(gitDir, checkoutPath, rev)
	if err != nil {
		return fmt.Errorf("failed to prep submodules: %v", err)
	}
	return nil
}

func SafeCleanup(path string) error {
	if path == "/" || path == "" || path == "." || strings.Contains(path, "..") {
		return fmt.Errorf("invalid path specified for deletion %q", path)
	}
	return os.RemoveAll(path)
}
<commit_msg>Create only the parent directory of gitDir<commit_after>package git

// everything git and github related

import (
	"encoding/json"
	"errors"
	"fmt"
	"io"
	"log"
	"os"
	"os/exec"
	"path"
	"path/filepath"
	"regexp"
	"strings"
	"time"

	"golang.org/x/net/context"
)

const GIT_BASE_DIR = "repo"

// Invoke a `command` in `workdir` with `args`, connecting up its Stdout and Stderr
func Command(workdir, command string, args ...string) *exec.Cmd {
	// log.Printf("wd = %s cmd = %s, args = %q", workdir, command, append([]string{}, args...))

	cmd := exec.Command(command, args...)
	cmd.Dir = workdir
	cmd.Stdout = os.Stdout
	cmd.Stderr = os.Stderr
	return cmd
}

var (
	ErrEmptyRepoName         = errors.New("Empty repository name")
	ErrEmptyRepoOrganization = errors.New("Empty repository organization")
	ErrUserNotAllowed        = errors.New("User not in the allowed set")
)

type Repository struct {
	Name         string `json:"name"`
	Url          string `json:"url"`
	Organization string `json:"organization"`
}

type Pusher struct {
	Name string `json:"name"`
}

type NonGithub struct {
	NoBuild bool `json:"nobuild"`
	Wait    bool `json:"wait"`
}

type JustNongithub struct {
	NonGithub NonGithub `json:"nongithub"`
}

func ParseJustNongithub(in []byte) (j JustNongithub, err error) {
	err = json.Unmarshal(in, &j)
	return
}

type PushEvent struct {
	Ref        string     `json:"ref"`
	Deleted    bool       `json:"deleted"`
	Repository Repository `json:"repository"`
	After      string     `json:"after"`
	Pusher     Pusher     `json:"pusher"`
	NonGithub  NonGithub  `json:"nongithub"`
	HtmlUrl    string     `json:"html_url"`
}

type GithubStatus struct {
	State       string `json:"state"`
	TargetUrl   string `json:"target_url"`
	Description string `json:"description"`
}

var ErrSkipGithubEndpoint = errors.New("Github endpoint skipped")

// Creates or updates a mirror of `url` at `gitDir` using `git clone --mirror`
func gitLocalMirror(
	url, gitDir, ref string,
	messages io.Writer,
) (err error) {

	// When mirroring, allow up to two minutes before giving up.
	const MirrorTimeout = 2 * time.Minute
	ctx, done := context.WithTimeout(context.Background(), MirrorTimeout)
	defer done()

	if _, err := os.Stat(gitDir); err == nil {
		// Repo already exists, don't need to clone it.

		if gitAlreadyHaveRef(gitDir, ref) {
			// Sha already exists, don't need to fetch.
			// log.Printf("Already have ref: %v %v", gitDir, ref)
			return nil
		}

		return gitFetch(ctx, gitDir, url, messages)
	}

	err = os.MkdirAll(filepath.Dir(gitDir), 0777)
	if err != nil {
		return err
	}

	return gitClone(ctx, url, gitDir, messages)
}

func gitClone(
	ctx context.Context,
	url, gitDir string,
	messages io.Writer,
) error {
	cmd := Command(".", "git", "clone", "-q", "--mirror", url, gitDir)
	cmd.Stdout = messages
	cmd.Stderr = messages
	return ContextRun(ctx, cmd)
}

// Run cmd within a net Context.
// If the context is cancelled or times out, the process is killed.
func ContextRun(ctx context.Context, cmd *exec.Cmd) error {
	errc := make(chan error)

	err := cmd.Start()
	if err != nil {
		return err
	}

	go func() { errc <- cmd.Wait() }()

	select {
	case <-ctx.Done():
		cmd.Process.Kill()
		return ctx.Err()
	case err := <-errc:
		return err // err may be nil
	}
	return nil
}

func gitFetch(
	ctx context.Context,
	gitDir, url string,
	messages io.Writer,
) (err error) {

	cmd := Command(gitDir, "git", "fetch", "-f", url, "*:*")
	cmd.Stdout = messages
	cmd.Stderr = messages

	err = ContextRun(ctx, cmd)
	if err != nil {
		// git fetch where there is no update is exit status 1.
		if err.Error() != "exit status 1" {
			return err
		}
	}

	return nil
}

var ShaLike = regexp.MustCompile("[0-9a-zA-Z]{40}")

// Returns true if ref is sha-like and is in the object database.
// The "sha-like" condition ensures that refs like `master` are always
// freshened.
func gitAlreadyHaveRef(gitDir, sha string) bool {
	if !ShaLike.MatchString(sha) {
		return false
	}
	cmd := Command(gitDir, "git", "cat-file", "-t", sha)
	cmd.Stdout = nil
	cmd.Stderr = nil

	err := cmd.Run()
	return err == nil
}

func gitHaveFile(gitDir, ref, path string) (ok bool, err error) {
	cmd := Command(gitDir, "git", "show", fmt.Sprintf("%s:%s", ref, path))
	cmd.Stdout = nil // don't want to see the contents
	err = cmd.Run()
	ok = true
	if err != nil {
		ok = false
		if err.Error() == "exit status 128" {
			// This happens if the file doesn't exist.
			err = nil
		}
	}
	return ok, err
}

func gitRevParse(gitDir, ref string) (sha string, err error) {
	cmd := Command(gitDir, "git", "rev-parse", ref)
	cmd.Stdout = nil // for cmd.Output

	var stdout []byte
	stdout, err = cmd.Output()
	if err != nil {
		return
	}

	sha = strings.TrimSpace(string(stdout))
	return
}

func gitDescribe(gitDir, ref string) (desc string, err error) {
	cmd := Command(gitDir, "git", "describe", "--all", "--tags", "--long", ref)
	cmd.Stdout = nil // for cmd.Output

	var stdout []byte
	stdout, err = cmd.Output()
	if err != nil {
		return
	}

	desc = strings.TrimSpace(string(stdout))
	desc = strings.TrimPrefix(desc, "heads/")
	return
}

func gitCheckout(gitDir, checkoutDir, ref string) error {

	err := os.MkdirAll(checkoutDir, 0777)
	if err != nil {
		return err
	}

	args := []string{"--work-tree", checkoutDir, "checkout", ref, "--", "."}
	err = Command(gitDir, "git", args...).Run()
	if err != nil {
		return err
	}

	// Set mtimes to time file is most recently affected by a commit.
	// This is annoying but unfortunately git sets the timestamps to now,
	// and docker depends on the mtime for cache invalidation.
	err = GitSetMTimes(gitDir, checkoutDir, ref)
	if err != nil {
		return err
	}

	return nil
}

func GitSetMTimes(gitDir, checkoutDir, ref string) error {

	commitTimes, err := GitCommitTimes(gitDir, ref)
	if err != nil {
		return err
	}

	lsFiles := Command(gitDir, "git", "ls-tree", "-r", "--name-only", "-z", ref)
	lsFiles.Stdout = nil
	out, err := lsFiles.Output()
	if err != nil {
		return fmt.Errorf("git ls-files: %v", err)
	}

	dirMTimes := map[string]time.Time{}

	files := strings.Split(strings.TrimRight(string(out), "\x00"), "\x00")
	for _, file := range files {
		mTime, ok := commitTimes[file]
		if !ok {
			return fmt.Errorf("failed to find file in history: %q", file)
		}

		// Loop over each directory in the path to `file`, updating `dirMTimes`
		// to take the most recent time seen.
		dir := filepath.Dir(file)
		for {
			if other, ok := dirMTimes[dir]; ok {
				if mTime.After(other) {
					// file mTime is more recent than previous seen for 'dir'
					dirMTimes[dir] = mTime
				}
			} else {
				// first occurrence of dir
				dirMTimes[dir] = mTime
			}

			// Remove one directory from the path until it isn't changed anymore
			if dir == filepath.Dir(dir) {
				break
			}
			dir = filepath.Dir(dir)
		}

		err = os.Chtimes(filepath.Join(checkoutDir, file), mTime, mTime)
		if err != nil {
			return fmt.Errorf("chtimes: %v", err)
		}

		// fmt.Printf("%s: %s\n", file, mTime)
	}

	for dir, mTime := range dirMTimes {
		err = os.Chtimes(filepath.Join(checkoutDir, dir), mTime, mTime)
		if err != nil {
			return fmt.Errorf("chtimes: %v", err)
		}
		// fmt.Printf("%s: %s\n", dir, mTime)
	}
	return nil
}

type BuildDirectory struct {
	Name, Dir string
	Cleanup   func()
}

func PrepBuildDirectory(
	gitDir, remote, ref string,
) (*BuildDirectory, error) {

	start := time.Now()
	defer func() {
		log.Printf("Took %v to prep %v", time.Since(start), remote)
	}()

	if strings.HasPrefix(remote, "github.com/") {
		remote = "https://" + remote
	}

	gitDir, err := filepath.Abs(gitDir)
	if err != nil {
		return nil, fmt.Errorf("unable to determine abspath: %v", err)
	}

	err = gitLocalMirror(remote, gitDir, ref, os.Stderr)
	if err != nil {
		return nil, fmt.Errorf("unable to gitLocalMirror: %v", err)
	}

	rev, err := gitRevParse(gitDir, ref)
	if err != nil {
		return nil, fmt.Errorf("unable to parse rev: %v", err)
	}

	tagName, err := gitDescribe(gitDir, rev)
	if err != nil {
		return nil, fmt.Errorf("unable to describe %v: %v", rev, err)
	}

	shortRev := rev[:10]
	checkoutPath := path.Join(gitDir, filepath.Join("c/", shortRev))

	err = recursiveCheckout(gitDir, checkoutPath, rev)
	if err != nil {
		return nil, err
	}

	cleanup := func() {
		err := SafeCleanup(checkoutPath)
		if err != nil {
			log.Printf("Error cleaning up path: ", checkoutPath)
		}
	}

	return &BuildDirectory{tagName, checkoutPath, cleanup}, nil
}

func recursiveCheckout(gitDir, checkoutPath, rev string) error {
	err := gitCheckout(gitDir, checkoutPath, rev)
	if err != nil {
		return fmt.Errorf("failed to checkout: %v", err)
	}

	err = PrepSubmodules(gitDir, checkoutPath, rev)
	if err != nil {
		return fmt.Errorf("failed to prep submodules: %v", err)
	}
	return nil
}

func SafeCleanup(path string) error {
	if path == "/" || path == "" || path == "." || strings.Contains(path, "..") {
		return fmt.Errorf("invalid path specified for deletion %q", path)
	}
	return os.RemoveAll(path)
}
<|endoftext|>


<commit_before>// Copyright (C) 2014-2020 The Syncthing Authors.
//
// This Source Code Form is subject to the terms of the Mozilla Public
// License, v. 2.0. If a copy of the MPL was not distributed with this file,
// You can obtain one at https://mozilla.org/MPL/2.0/.

// Package assets hold utilities for serving static assets.
//
// The actual assets live in auto subpackages instead of here,
// because the set of assets varies per program.
package assets

import (
	"compress/gzip"
	"fmt"
	"io"
	"mime"
	"net/http"
	"path/filepath"
	"strconv"
	"strings"
	"time"
)

// An Asset is an embedded file to be served over HTTP.
type Asset struct {
	Content  string // Contents of asset, possibly gzipped.
	Gzipped  bool
	Length   int       // Length of (decompressed) Content.
	Filename string    // Original filename, determines Content-Type.
	Modified time.Time // Determines ETag and Last-Modified.
}

// Serve writes a gzipped asset to w.
func Serve(w http.ResponseWriter, r *http.Request, asset Asset) {
	header := w.Header()

	mtype := MimeTypeForFile(asset.Filename)
	if mtype != "" {
		header.Set("Content-Type", mtype)
	}

	etag := fmt.Sprintf(`"%x"`, asset.Modified.Unix())
	header.Set("ETag", etag)
	header.Set("Last-Modified", asset.Modified.Format(http.TimeFormat))

	t, err := time.Parse(http.TimeFormat, r.Header.Get("If-Modified-Since"))
	if err == nil && !asset.Modified.After(t) {
		w.WriteHeader(http.StatusNotModified)
		return
	}

	if r.Header.Get("If-None-Match") == etag {
		w.WriteHeader(http.StatusNotModified)
		return
	}

	switch {
	case !asset.Gzipped:
		header.Set("Content-Length", strconv.Itoa(len(asset.Content)))
		io.WriteString(w, asset.Content)
	case strings.Contains(r.Header.Get("Accept-Encoding"), "gzip"):
		header.Set("Content-Encoding", "gzip")
		header.Set("Content-Length", strconv.Itoa(len(asset.Content)))
		io.WriteString(w, asset.Content)
	default:
		header.Set("Content-Length", strconv.Itoa(asset.Length))
		// gunzip for browsers that don't want gzip.
		var gr *gzip.Reader
		gr, _ = gzip.NewReader(strings.NewReader(asset.Content))
		io.Copy(w, gr)
		gr.Close()
	}
}

// MimeTypeForFile returns the appropriate MIME type for an asset,
// based on the filename.
//
// We use a built in table of the common types since the system
// TypeByExtension might be unreliable. But if we don't know, we delegate
// to the system. All our text files are in UTF-8.
func MimeTypeForFile(file string) string {
	ext := filepath.Ext(file)
	switch ext {
	case ".htm", ".html":
		return "text/html; charset=utf-8"
	case ".css":
		return "text/css; charset=utf-8"
	case ".js":
		return "application/javascript; charset=utf-8"
	case ".json":
		return "application/json; charset=utf-8"
	case ".png":
		return "image/png"
	case ".ttf":
		return "application/x-font-ttf"
	case ".woff":
		return "application/x-font-woff"
	case ".svg":
		return "image/svg+xml; charset=utf-8"
	default:
		return mime.TypeByExtension(ext)
	}
}
<commit_msg>lib/assets: MIME types, time formats (#8351)<commit_after>// Copyright (C) 2014-2020 The Syncthing Authors.
//
// This Source Code Form is subject to the terms of the Mozilla Public
// License, v. 2.0. If a copy of the MPL was not distributed with this file,
// You can obtain one at https://mozilla.org/MPL/2.0/.

// Package assets hold utilities for serving static assets.
//
// The actual assets live in auto subpackages instead of here,
// because the set of assets varies per program.
package assets

import (
	"compress/gzip"
	"fmt"
	"io"
	"mime"
	"net/http"
	"path/filepath"
	"strconv"
	"strings"
	"time"
)

// An Asset is an embedded file to be served over HTTP.
type Asset struct {
	Content  string // Contents of asset, possibly gzipped.
	Gzipped  bool
	Length   int       // Length of (decompressed) Content.
	Filename string    // Original filename, determines Content-Type.
	Modified time.Time // Determines ETag and Last-Modified.
}

// Serve writes a gzipped asset to w.
func Serve(w http.ResponseWriter, r *http.Request, asset Asset) {
	header := w.Header()

	mtype := MimeTypeForFile(asset.Filename)
	if mtype != "" {
		header.Set("Content-Type", mtype)
	}

	etag := fmt.Sprintf(`"%x"`, asset.Modified.Unix())
	header.Set("ETag", etag)
	header.Set("Last-Modified", asset.Modified.Format(http.TimeFormat))

	t, err := http.ParseTime(r.Header.Get("If-Modified-Since"))
	if err == nil && !asset.Modified.After(t) {
		w.WriteHeader(http.StatusNotModified)
		return
	}

	if r.Header.Get("If-None-Match") == etag {
		w.WriteHeader(http.StatusNotModified)
		return
	}

	switch {
	case !asset.Gzipped:
		header.Set("Content-Length", strconv.Itoa(len(asset.Content)))
		io.WriteString(w, asset.Content)
	case strings.Contains(r.Header.Get("Accept-Encoding"), "gzip"):
		header.Set("Content-Encoding", "gzip")
		header.Set("Content-Length", strconv.Itoa(len(asset.Content)))
		io.WriteString(w, asset.Content)
	default:
		header.Set("Content-Length", strconv.Itoa(asset.Length))
		// gunzip for browsers that don't want gzip.
		var gr *gzip.Reader
		gr, _ = gzip.NewReader(strings.NewReader(asset.Content))
		io.Copy(w, gr)
		gr.Close()
	}
}

// MimeTypeForFile returns the appropriate MIME type for an asset,
// based on the filename.
//
// We use a built in table of the common types since the system
// TypeByExtension might be unreliable. But if we don't know, we delegate
// to the system. All our text files are in UTF-8.
func MimeTypeForFile(file string) string {
	ext := filepath.Ext(file)
	switch ext {
	case ".htm", ".html":
		return "text/html; charset=utf-8"
	case ".css":
		return "text/css; charset=utf-8"
	case ".eot":
		return "application/vnd.ms-fontobject"
	case ".js":
		return "application/javascript; charset=utf-8"
	case ".json":
		return "application/json; charset=utf-8"
	case ".png":
		return "image/png"
	case ".svg":
		return "image/svg+xml; charset=utf-8"
	case ".ttf":
		return "font/ttf"
	case ".woff":
		return "font/woff"
	case ".woff2":
		return "font/woff2"
	default:
		return mime.TypeByExtension(ext)
	}
}
<|endoftext|>


<commit_before>// Package camoproxy provides an HTTP proxy server with content type
// restrictions as well as regex host allow and deny list support.
package camoproxy

import (
	"crypto/hmac"
	"crypto/sha1"
	"encoding/hex"
	"errors"
	"github.com/cactus/gologit"
	"io"
	"log"
	"net"
	"net/http"
	"net/url"
	"regexp"
	"strings"
	"time"
)

// Headers that are acceptible to pass from the client to the remote
// server. Only those present and true, are forwarded.
var validReqHeaders = map[string]bool{
	"Accept":            true,
	"Accept-Charset":    true,
	"Accept-Encoding":   true,
	"Accept-Language":   true,
	"Cache-Control":     true,
	"If-None-Match":     true,
	"If-Modified-Since": true,
}

// Headers that are acceptible to pass from the remote server to the
// client. Only those present and true, are forwarded. Empty implies
// no filtering.
var validRespHeaders = map[string]bool{}

// A ProxyHandler is a Camo like HTTP proxy, that provides content type
// restrictions as well as regex host allow and deny list support
type ProxyHandler struct {
	Client    *http.Client
	HMacKey   []byte
	Allowlist []*regexp.Regexp
	Denylist  []*regexp.Regexp
	MaxSize   int64
	log       *gologit.DebugLogger
}

// ServerHTTP handles the client request, validates the request is validly
// HMAC signed, filters based on the Allow/Deny list, and then proxies
// valid requests to the desired endpoint. Responses are filtered for
// proper image content types.
func (p *ProxyHandler) ServeHTTP(w http.ResponseWriter, req *http.Request) {
	p.log.Debugln("Request:", req.URL)

	// do fiddly things
	if req.Method != "GET" {
		log.Println("Something other than GET received", req.Method)
		http.Error(w, "Method Not Implemented", http.StatusNotImplemented)
		return
	}

	surl, ok := p.validateURL(req.URL.Path, p.HMacKey)
	if !ok {
		http.Error(w, "Bad Signature", http.StatusForbidden)
		return
	}
	p.log.Debugln("URL:", surl)

	u, err := url.Parse(surl)
	if err != nil {
		log.Println(err)
		http.Error(w, "Bad url", http.StatusBadRequest)
		return
	}

	if u.Host == "" {
		http.Error(w, "Bad url", http.StatusNotFound)
		return
	}

	// if Allowlist is set, require match
	matchFound := true
	if len(p.Allowlist) > 0 {
		matchFound = false
		for _, rgx := range p.Allowlist {
			if rgx.MatchString(u.Host) {
				matchFound = true
			}
		}
	}
	if !matchFound {
		http.Error(w, "Allowlist host failure", http.StatusNotFound)
		return
	}

	// filter out Denylist urls based on regexes. Do this second
	// as Denylist takes precedence
	for _, rgx := range p.Denylist {
		if rgx.MatchString(u.Host) {
			http.Error(w, "Denylist host failure", http.StatusNotFound)
			return
		}
	}

	nreq, err := http.NewRequest("GET", surl, nil)
	if err != nil {
		p.log.Debugln("Could not create NewRequest", err)
		http.Error(w, "Error Fetching Resource", http.StatusBadGateway)
		return
	}

	// filter headers
	p.copyHeader(&nreq.Header, &req.Header, &validReqHeaders)
	nreq.Header.Add("connection", "close")
	nreq.Header.Add("user-agent", "pew pew pew")

	resp, err := p.Client.Do(nreq)
	if err != nil {
		p.log.Debugln("Could not connect to endpoint", err)
		if strings.Contains(err.Error(), "timeout") {
			http.Error(w, "Error Fetching Resource", http.StatusBadGateway)
		} else {
			http.Error(w, "Error Fetching Resource", http.StatusNotFound)
		}
		return
	}
	defer resp.Body.Close()

	// check for too large a response
	if resp.ContentLength > p.MaxSize {
		p.log.Debugln("Content length exceeded", surl)
		http.Error(w, "Content length exceeded", http.StatusNotFound)
		return
	}

	switch resp.StatusCode {
	case 300:
		log.Println("Multiple choices not supported")
		http.Error(w, "Multiple choices not supported", http.StatusNotFound)
		return
	case 301, 302, 303:
		// if we get a redirect here, we either disabled following,
		// or followed until max depth and still got one (redirect loop)
		http.Error(w, "Not Found", http.StatusNotFound)
		return
	case 500, 502, 503, 504:
		// upstream errors should probably just 502. client can try later.
		http.Error(w, "Error Fetching Resource", http.StatusBadGateway)
		return
	case 404:
		http.Error(w, "Not Found", http.StatusNotFound)
		return
	case 200:
		// check content type
		ct, ok := resp.Header[http.CanonicalHeaderKey("content-type")]
		if !ok || ct[0][:6] != "image/" {
			log.Println("Non-Image content-type returned", u)
			http.Error(w, "Non-Image content-type returned",
				http.StatusBadRequest)
			return
		}
	default:
		http.Error(w, "Not Found", http.StatusNotFound)
		return
	}

	h := w.Header()
	p.copyHeader(&h, &resp.Header, &validRespHeaders)
	h.Add("X-Content-Type-Options", "nosniff")
	if resp.StatusCode == 304 && h.Get("Content-Type") != "" {
		h.Del("Content-Type")
	}

	w.WriteHeader(resp.StatusCode)
	io.Copy(w, resp.Body)
	p.log.Debugln(req, resp.StatusCode)
}

// validateURL ensures the url is properly verified via HMAC, and then
// unencodes the url, returning the url (if valid) and whether the
// HMAC was verified.
func (p *ProxyHandler) validateURL(path string, key []byte) (surl string, valid bool) {
	pathParts := strings.SplitN(path[1:], "/", 3)
	valid = false
	if len(pathParts) != 2 {
		p.log.Println("Bad path format", pathParts)
		return
	}
	hexdig, hexurl := pathParts[0], pathParts[1]
	urlBytes, err := hex.DecodeString(hexurl)
	if err != nil {
		p.log.Println("Bad Hex Decode", hexurl)
		return
	}
	surl = string(urlBytes)
	mac := hmac.New(sha1.New, key)
	mac.Write([]byte(surl))
	macSum := hex.EncodeToString(mac.Sum([]byte{}))
	if macSum != hexdig {
		p.log.Debugf("Bad signature: %s != %s\n", macSum, hexdig)
		return
	}
	valid = true
	return
}

// copy headers from src into dst
// empty filter map will result in no filtering being done
func (p *ProxyHandler) copyHeader(dst, src *http.Header, filter *map[string]bool) {
	f := *filter
	filtering := false
	if len(f) > 0 {
		filtering = true
	}

	for k, vv := range *src {
		if x, ok := f[k]; filtering && !ok && !x {
			continue
		}
		for _, v := range vv {
			dst.Add(k, v)
		}
	}
}

type ProxyConfig struct {
	HmacKey string
	AllowList []string
	DenyList []string
	MaxSize int64
	FollowRedirects bool
	RequestTimeout uint
}

func New(pc ProxyConfig, logger *gologit.DebugLogger) *ProxyHandler {
	tr := &http.Transport{
		Dial: func(netw, addr string) (net.Conn, error) {
			// 2 second timeout on requests
			timeout := time.Second * time.Duration(pc.RequestTimeout)
			c, err := net.DialTimeout(netw, addr, timeout)
			if err != nil {
				return nil, err
			}
			// also set time limit on reading
			c.SetDeadline(time.Now().Add(timeout))
			return c, nil
		}}

	// spawn an idle conn trimmer
	go func() {
		time.Sleep(5 * time.Minute)
		tr.CloseIdleConnections()
	}()

	// build/compile regex
	client := &http.Client{Transport: tr}
	if !pc.FollowRedirects {
		client.CheckRedirect = func(req *http.Request, via []*http.Request) error {
			return errors.New("Not following redirect")
		}
	}

	allow := make([]*regexp.Regexp, 0)
	deny := make([]*regexp.Regexp, 0)

	var c *regexp.Regexp
	var err error
	for _, v := range pc.DenyList {
		c, err = regexp.Compile(v)
		if err != nil {
			log.Fatal(err)
		}
		deny = append(deny, c)
	}
	for _, v := range pc.AllowList {
		c, err = regexp.Compile(v)
		if err != nil {
			log.Fatal(err)
		}
		allow = append(allow, c)
	}

	return &ProxyHandler{
		Client:    client,
		HMacKey:   []byte(pc.HmacKey),
		Allowlist: allow,
		Denylist:  deny,
		MaxSize:   pc.MaxSize,
		log:       logger}
}
<commit_msg>reorder cases<commit_after>// Package camoproxy provides an HTTP proxy server with content type
// restrictions as well as regex host allow and deny list support.
package camoproxy

import (
	"crypto/hmac"
	"crypto/sha1"
	"encoding/hex"
	"errors"
	"github.com/cactus/gologit"
	"io"
	"log"
	"net"
	"net/http"
	"net/url"
	"regexp"
	"strings"
	"time"
)

// Headers that are acceptible to pass from the client to the remote
// server. Only those present and true, are forwarded.
var validReqHeaders = map[string]bool{
	"Accept":            true,
	"Accept-Charset":    true,
	"Accept-Encoding":   true,
	"Accept-Language":   true,
	"Cache-Control":     true,
	"If-None-Match":     true,
	"If-Modified-Since": true,
}

// Headers that are acceptible to pass from the remote server to the
// client. Only those present and true, are forwarded. Empty implies
// no filtering.
var validRespHeaders = map[string]bool{}

// A ProxyHandler is a Camo like HTTP proxy, that provides content type
// restrictions as well as regex host allow and deny list support
type ProxyHandler struct {
	Client    *http.Client
	HMacKey   []byte
	Allowlist []*regexp.Regexp
	Denylist  []*regexp.Regexp
	MaxSize   int64
	log       *gologit.DebugLogger
}

// ServerHTTP handles the client request, validates the request is validly
// HMAC signed, filters based on the Allow/Deny list, and then proxies
// valid requests to the desired endpoint. Responses are filtered for
// proper image content types.
func (p *ProxyHandler) ServeHTTP(w http.ResponseWriter, req *http.Request) {
	p.log.Debugln("Request:", req.URL)

	// do fiddly things
	if req.Method != "GET" {
		log.Println("Something other than GET received", req.Method)
		http.Error(w, "Method Not Implemented", http.StatusNotImplemented)
		return
	}

	surl, ok := p.validateURL(req.URL.Path, p.HMacKey)
	if !ok {
		http.Error(w, "Bad Signature", http.StatusForbidden)
		return
	}
	p.log.Debugln("URL:", surl)

	u, err := url.Parse(surl)
	if err != nil {
		log.Println(err)
		http.Error(w, "Bad url", http.StatusBadRequest)
		return
	}

	if u.Host == "" {
		http.Error(w, "Bad url", http.StatusNotFound)
		return
	}

	// if Allowlist is set, require match
	matchFound := true
	if len(p.Allowlist) > 0 {
		matchFound = false
		for _, rgx := range p.Allowlist {
			if rgx.MatchString(u.Host) {
				matchFound = true
			}
		}
	}
	if !matchFound {
		http.Error(w, "Allowlist host failure", http.StatusNotFound)
		return
	}

	// filter out Denylist urls based on regexes. Do this second
	// as Denylist takes precedence
	for _, rgx := range p.Denylist {
		if rgx.MatchString(u.Host) {
			http.Error(w, "Denylist host failure", http.StatusNotFound)
			return
		}
	}

	nreq, err := http.NewRequest("GET", surl, nil)
	if err != nil {
		p.log.Debugln("Could not create NewRequest", err)
		http.Error(w, "Error Fetching Resource", http.StatusBadGateway)
		return
	}

	// filter headers
	p.copyHeader(&nreq.Header, &req.Header, &validReqHeaders)
	nreq.Header.Add("connection", "close")
	nreq.Header.Add("user-agent", "pew pew pew")

	resp, err := p.Client.Do(nreq)
	if err != nil {
		p.log.Debugln("Could not connect to endpoint", err)
		if strings.Contains(err.Error(), "timeout") {
			http.Error(w, "Error Fetching Resource", http.StatusBadGateway)
		} else {
			http.Error(w, "Error Fetching Resource", http.StatusNotFound)
		}
		return
	}
	defer resp.Body.Close()

	// check for too large a response
	if resp.ContentLength > p.MaxSize {
		p.log.Debugln("Content length exceeded", surl)
		http.Error(w, "Content length exceeded", http.StatusNotFound)
		return
	}

	switch resp.StatusCode {
	case 200:
		// check content type
		ct, ok := resp.Header[http.CanonicalHeaderKey("content-type")]
		if !ok || ct[0][:6] != "image/" {
			log.Println("Non-Image content-type returned", u)
			http.Error(w, "Non-Image content-type returned",
				http.StatusBadRequest)
			return
		}
	case 300:
		log.Println("Multiple choices not supported")
		http.Error(w, "Multiple choices not supported", http.StatusNotFound)
		return
	case 301, 302, 303:
		// if we get a redirect here, we either disabled following,
		// or followed until max depth and still got one (redirect loop)
		http.Error(w, "Not Found", http.StatusNotFound)
		return
	case 404:
		http.Error(w, "Not Found", http.StatusNotFound)
		return
	case 500, 502, 503, 504:
		// upstream errors should probably just 502. client can try later.
		http.Error(w, "Error Fetching Resource", http.StatusBadGateway)
		return
	default:
		http.Error(w, "Not Found", http.StatusNotFound)
		return
	}

	h := w.Header()
	p.copyHeader(&h, &resp.Header, &validRespHeaders)
	h.Add("X-Content-Type-Options", "nosniff")
	if resp.StatusCode == 304 && h.Get("Content-Type") != "" {
		h.Del("Content-Type")
	}

	w.WriteHeader(resp.StatusCode)
	io.Copy(w, resp.Body)
	p.log.Debugln(req, resp.StatusCode)
}

// validateURL ensures the url is properly verified via HMAC, and then
// unencodes the url, returning the url (if valid) and whether the
// HMAC was verified.
func (p *ProxyHandler) validateURL(path string, key []byte) (surl string, valid bool) {
	pathParts := strings.SplitN(path[1:], "/", 3)
	valid = false
	if len(pathParts) != 2 {
		p.log.Println("Bad path format", pathParts)
		return
	}
	hexdig, hexurl := pathParts[0], pathParts[1]
	urlBytes, err := hex.DecodeString(hexurl)
	if err != nil {
		p.log.Println("Bad Hex Decode", hexurl)
		return
	}
	surl = string(urlBytes)
	mac := hmac.New(sha1.New, key)
	mac.Write([]byte(surl))
	macSum := hex.EncodeToString(mac.Sum([]byte{}))
	if macSum != hexdig {
		p.log.Debugf("Bad signature: %s != %s\n", macSum, hexdig)
		return
	}
	valid = true
	return
}

// copy headers from src into dst
// empty filter map will result in no filtering being done
func (p *ProxyHandler) copyHeader(dst, src *http.Header, filter *map[string]bool) {
	f := *filter
	filtering := false
	if len(f) > 0 {
		filtering = true
	}

	for k, vv := range *src {
		if x, ok := f[k]; filtering && !ok && !x {
			continue
		}
		for _, v := range vv {
			dst.Add(k, v)
		}
	}
}

type ProxyConfig struct {
	HmacKey string
	AllowList []string
	DenyList []string
	MaxSize int64
	FollowRedirects bool
	RequestTimeout uint
}

func New(pc ProxyConfig, logger *gologit.DebugLogger) *ProxyHandler {
	tr := &http.Transport{
		Dial: func(netw, addr string) (net.Conn, error) {
			// 2 second timeout on requests
			timeout := time.Second * time.Duration(pc.RequestTimeout)
			c, err := net.DialTimeout(netw, addr, timeout)
			if err != nil {
				return nil, err
			}
			// also set time limit on reading
			c.SetDeadline(time.Now().Add(timeout))
			return c, nil
		}}

	// spawn an idle conn trimmer
	go func() {
		time.Sleep(5 * time.Minute)
		tr.CloseIdleConnections()
	}()

	// build/compile regex
	client := &http.Client{Transport: tr}
	if !pc.FollowRedirects {
		client.CheckRedirect = func(req *http.Request, via []*http.Request) error {
			return errors.New("Not following redirect")
		}
	}

	allow := make([]*regexp.Regexp, 0)
	deny := make([]*regexp.Regexp, 0)

	var c *regexp.Regexp
	var err error
	for _, v := range pc.DenyList {
		c, err = regexp.Compile(v)
		if err != nil {
			log.Fatal(err)
		}
		deny = append(deny, c)
	}
	for _, v := range pc.AllowList {
		c, err = regexp.Compile(v)
		if err != nil {
			log.Fatal(err)
		}
		allow = append(allow, c)
	}

	return &ProxyHandler{
		Client:    client,
		HMacKey:   []byte(pc.HmacKey),
		Allowlist: allow,
		Denylist:  deny,
		MaxSize:   pc.MaxSize,
		log:       logger}
}
<|endoftext|>


<commit_before>package testing

var FluxEndToEndSkipList = map[string]map[string]string{
	"universe": {
		// TODO(adam) determine the reason for these test failures.
		"cov":                      "Reason TBD",
		"covariance":               "Reason TBD",
		"cumulative_sum":           "Reason TBD",
		"cumulative_sum_default":   "Reason TBD",
		"cumulative_sum_noop":      "Reason TBD",
		"drop_non_existent":        "Reason TBD",
		"first":                    "Reason TBD",
		"highestAverage":           "Reason TBD",
		"highestMax":               "Reason TBD",
		"histogram":                "Reason TBD",
		"histogram_normalize":      "Reason TBD",
		"histogram_quantile":       "Reason TBD",
		"join":                     "Reason TBD",
		"join_across_measurements": "Reason TBD",
		"join_agg":                 "Reason TBD",
		"keep_non_existent":        "Reason TBD",
		"key_values":               "Reason TBD",
		"key_values_host_name":     "Reason TBD",
		"last":                     "Reason TBD",
		"lowestAverage":            "Reason TBD",
		"max":                      "Reason TBD",
		"min":                      "Reason TBD",
		"sample":                   "Reason TBD",
		"selector_preserve_time":   "Reason TBD",
		"shift":                    "Reason TBD",
		"shift_negative_duration":  "Reason TBD",
		"task_per_line":            "Reason TBD",
		"top":                      "Reason TBD",
		"union":                    "Reason TBD",
		"union_heterogeneous":      "Reason TBD",
		"unique":                   "Reason TBD",
		"distinct":                 "Reason TBD",

		// it appears these occur when writing the input data.  `to` may not be null safe.
		"fill_bool":   "failed to read meta data: panic: interface conversion: interface {} is nil, not uint64",
		"fill_float":  "failed to read meta data: panic: interface conversion: interface {} is nil, not uint64",
		"fill_int":    "failed to read meta data: panic: interface conversion: interface {} is nil, not uint64",
		"fill_string": "failed to read meta data: panic: interface conversion: interface {} is nil, not uint64",
		"fill_time":   "failed to read meta data: panic: interface conversion: interface {} is nil, not uint64",
		"fill_uint":   "failed to read meta data: panic: interface conversion: interface {} is nil, not uint64",
		"window_null": "failed to read meta data: panic: interface conversion: interface {} is nil, not float64",

		// these may just be missing calls to range() in the tests.  easy to fix in a new PR.
		"group_nulls":         "unbounded test",
		"integral":            "unbounded test",
		"integral_columns":    "unbounded test",
		"map":                 "unbounded test",
		"join_missing_on_col": "unbounded test",
		"rowfn_with_import":   "unbounded test",

		// the following tests have a difference between the CSV-decoded input table, and the storage-retrieved version of that table
		"columns":            "group key mismatch",
		"set":                "column order mismatch",
		"simple_max":         "_stop missing from expected output",
		"derivative":         "time bounds mismatch (engine uses now() instead of bounds on input table)",
		"difference_columns": "data write/read path loses columns x and y",
		"keys":               "group key mismatch",

		// failed to read meta data errors: the CSV encoding is incomplete probably due to data schema errors.  needs more detailed investigation to find root cause of error
		"filter_by_regex":             "failed to read metadata",
		"filter_by_tags":              "failed to read metadata",
		"group":                       "failed to read metadata",
		"group_except":                "failed to read metadata",
		"group_ungroup":               "failed to read metadata",
		"pivot_mean":                  "failed to read metadata",
		"histogram_quantile_minvalue": "failed to read meta data: no column with label _measurement exists",
		"increase":                    "failed to read meta data: table has no _value column",

		"string_max":                  "error: invalid use of function: *functions.MaxSelector has no implementation for type string (https://github.com/influxdata/platform/issues/224)",
		"null_as_value":               "null not supported as value in influxql (https://github.com/influxdata/platform/issues/353)",
		"string_interp":               "string interpolation not working as expected in flux (https://github.com/influxdata/platform/issues/404)",
		"to":                          "to functions are not supported in the testing framework (https://github.com/influxdata/flux/issues/77)",
		"covariance_missing_column_1": "need to support known errors in new test framework (https://github.com/influxdata/flux/issues/536)",
		"covariance_missing_column_2": "need to support known errors in new test framework (https://github.com/influxdata/flux/issues/536)",
		"drop_before_rename":          "need to support known errors in new test framework (https://github.com/influxdata/flux/issues/536)",
		"drop_referenced":             "need to support known errors in new test framework (https://github.com/influxdata/flux/issues/536)",
		"yield":                       "yield requires special test case (https://github.com/influxdata/flux/issues/535)",

		"window_group_mean_ungroup": "window trigger optimization modifies sort order of its output tables (https://github.com/influxdata/flux/issues/1067)",

		"median_column": "failing in different ways (https://github.com/influxdata/influxdb/issues/13909)",
		"dynamic_query": "panic when executing",

		"to_int":  "dateTime conversion issue: https://github.com/influxdata/influxdb/issues/14575",
		"to_uint": "dateTime conversion issue: https://github.com/influxdata/influxdb/issues/14575",

		"holt_winters_panic": "Expected output is an empty table which breaks the testing framework (https://github.com/influxdata/influxdb/issues/14749)",
	},
	"experimental": {
		"set": "Reason TBD",
	},
	"regexp": {
		"replaceAllString": "Reason TBD",
	},
	"http": {
		"http_endpoint": "need ability to test side effects in e2e tests: (https://github.com/influxdata/flux/issues/1723)",
	},
	"influxdata/influxdb/monitor": {
		"check": "Cannot see overridden options from inside stdlib functions (https://github.com/influxdata/flux/issues/1720)",
	},
	"influxdata/influxdb/secrets": {
		"secrets": "Cannot inject custom deps into the test framework so the secrets don't lookup correctly",
	},
	"internal/promql": {
		"join": "unbounded test",
	},
	"testing/chronograf": {
		"buckets": "unbounded test",
	},
	"testing/kapacitor": {
		"fill_default": "unknown field type for f1",
	},
	"testing/pandas": {
		"extract_regexp_findStringIndex": "pandas. map does not correctly handled returned arrays (https://github.com/influxdata/flux/issues/1387)",
		"partition_strings_splitN":       "pandas. map does not correctly handled returned arrays (https://github.com/influxdata/flux/issues/1387)",
	},
	// PromQL tests don't work. Missing `range` or `_measurement`.
	"testing/promql": {
		"changes":                        "unbounded or missing _measurement column",
		"dayOfMonth":                     "unbounded or missing _measurement column",
		"dayOfWeek":                      "unbounded or missing _measurement column",
		"daysInMonth":                    "unbounded or missing _measurement column",
		"emptyTable":                     "unbounded or missing _measurement column",
		"extrapolatedRate_counter_rate":  "unbounded or missing _measurement column",
		"extrapolatedRate_nocounter":     "unbounded or missing _measurement column",
		"extrapolatedRate_norate":        "unbounded or missing _measurement column",
		"histogramQuantile":              "unbounded or missing _measurement column",
		"holtWinters":                    "unbounded or missing _measurement column",
		"hour":                           "unbounded or missing _measurement column",
		"instantRate":                    "unbounded or missing _measurement column",
		"labelReplace_empty_dst":         "unbounded or missing _measurement column",
		"labelReplace_full_string_match": "unbounded or missing _measurement column",
		"labelReplace_multiple_groups":   "unbounded or missing _measurement column",
		"labelReplace_src_empty":         "unbounded or missing _measurement column",
		"labelReplace_src_nonexistent":   "unbounded or missing _measurement column",
		"labelReplace_src_not_matched":   "unbounded or missing _measurement column",
		"labelReplace_sub_string_match":  "unbounded or missing _measurement column",
		"linearRegression_nopredict":     "unbounded or missing _measurement column",
		"linearRegression_predict":       "unbounded or missing _measurement column",
		"minute":                         "unbounded or missing _measurement column",
		"month":                          "unbounded or missing _measurement column",
		"resets":                         "unbounded or missing _measurement column",
		"timestamp":                      "unbounded or missing _measurement column",
		"year":                           "unbounded or missing _measurement column",
	},
}
<commit_msg>chore(query/stdlib): skip flaky tests regarding storage meta queries (#15451)<commit_after>package testing

var FluxEndToEndSkipList = map[string]map[string]string{
	"universe": {
		// TODO(adam) determine the reason for these test failures.
		"cov":                      "Reason TBD",
		"covariance":               "Reason TBD",
		"cumulative_sum":           "Reason TBD",
		"cumulative_sum_default":   "Reason TBD",
		"cumulative_sum_noop":      "Reason TBD",
		"drop_non_existent":        "Reason TBD",
		"first":                    "Reason TBD",
		"highestAverage":           "Reason TBD",
		"highestMax":               "Reason TBD",
		"histogram":                "Reason TBD",
		"histogram_normalize":      "Reason TBD",
		"histogram_quantile":       "Reason TBD",
		"join":                     "Reason TBD",
		"join_across_measurements": "Reason TBD",
		"join_agg":                 "Reason TBD",
		"keep_non_existent":        "Reason TBD",
		"key_values":               "Reason TBD",
		"key_values_host_name":     "Reason TBD",
		"last":                     "Reason TBD",
		"lowestAverage":            "Reason TBD",
		"max":                      "Reason TBD",
		"min":                      "Reason TBD",
		"sample":                   "Reason TBD",
		"selector_preserve_time":   "Reason TBD",
		"shift":                    "Reason TBD",
		"shift_negative_duration":  "Reason TBD",
		"task_per_line":            "Reason TBD",
		"top":                      "Reason TBD",
		"union":                    "Reason TBD",
		"union_heterogeneous":      "Reason TBD",
		"unique":                   "Reason TBD",
		"distinct":                 "Reason TBD",

		// it appears these occur when writing the input data.  `to` may not be null safe.
		"fill_bool":   "failed to read meta data: panic: interface conversion: interface {} is nil, not uint64",
		"fill_float":  "failed to read meta data: panic: interface conversion: interface {} is nil, not uint64",
		"fill_int":    "failed to read meta data: panic: interface conversion: interface {} is nil, not uint64",
		"fill_string": "failed to read meta data: panic: interface conversion: interface {} is nil, not uint64",
		"fill_time":   "failed to read meta data: panic: interface conversion: interface {} is nil, not uint64",
		"fill_uint":   "failed to read meta data: panic: interface conversion: interface {} is nil, not uint64",
		"window_null": "failed to read meta data: panic: interface conversion: interface {} is nil, not float64",

		// these may just be missing calls to range() in the tests.  easy to fix in a new PR.
		"group_nulls":         "unbounded test",
		"integral":            "unbounded test",
		"integral_columns":    "unbounded test",
		"map":                 "unbounded test",
		"join_missing_on_col": "unbounded test",
		"rowfn_with_import":   "unbounded test",

		// the following tests have a difference between the CSV-decoded input table, and the storage-retrieved version of that table
		"columns":            "group key mismatch",
		"set":                "column order mismatch",
		"simple_max":         "_stop missing from expected output",
		"derivative":         "time bounds mismatch (engine uses now() instead of bounds on input table)",
		"difference_columns": "data write/read path loses columns x and y",
		"keys":               "group key mismatch",

		// failed to read meta data errors: the CSV encoding is incomplete probably due to data schema errors.  needs more detailed investigation to find root cause of error
		"filter_by_regex":             "failed to read metadata",
		"filter_by_tags":              "failed to read metadata",
		"group":                       "failed to read metadata",
		"group_except":                "failed to read metadata",
		"group_ungroup":               "failed to read metadata",
		"pivot_mean":                  "failed to read metadata",
		"histogram_quantile_minvalue": "failed to read meta data: no column with label _measurement exists",
		"increase":                    "failed to read meta data: table has no _value column",

		"string_max":                  "error: invalid use of function: *functions.MaxSelector has no implementation for type string (https://github.com/influxdata/platform/issues/224)",
		"null_as_value":               "null not supported as value in influxql (https://github.com/influxdata/platform/issues/353)",
		"string_interp":               "string interpolation not working as expected in flux (https://github.com/influxdata/platform/issues/404)",
		"to":                          "to functions are not supported in the testing framework (https://github.com/influxdata/flux/issues/77)",
		"covariance_missing_column_1": "need to support known errors in new test framework (https://github.com/influxdata/flux/issues/536)",
		"covariance_missing_column_2": "need to support known errors in new test framework (https://github.com/influxdata/flux/issues/536)",
		"drop_before_rename":          "need to support known errors in new test framework (https://github.com/influxdata/flux/issues/536)",
		"drop_referenced":             "need to support known errors in new test framework (https://github.com/influxdata/flux/issues/536)",
		"yield":                       "yield requires special test case (https://github.com/influxdata/flux/issues/535)",

		"window_group_mean_ungroup": "window trigger optimization modifies sort order of its output tables (https://github.com/influxdata/flux/issues/1067)",

		"median_column": "failing in different ways (https://github.com/influxdata/influxdb/issues/13909)",
		"dynamic_query": "panic when executing",

		"to_int":  "dateTime conversion issue: https://github.com/influxdata/influxdb/issues/14575",
		"to_uint": "dateTime conversion issue: https://github.com/influxdata/influxdb/issues/14575",

		"holt_winters_panic": "Expected output is an empty table which breaks the testing framework (https://github.com/influxdata/influxdb/issues/14749)",
	},
	"experimental": {
		"set": "Reason TBD",
	},
	"regexp": {
		"replaceAllString": "Reason TBD",
	},
	"http": {
		"http_endpoint": "need ability to test side effects in e2e tests: (https://github.com/influxdata/flux/issues/1723)",
	},
	"influxdata/influxdb/v1": {
		"show_measurements": "flaky test (https://github.com/influxdata/influxdb/issues/15450)",
		"show_tag_values":   "flaky test (https://github.com/influxdata/influxdb/issues/15450)",
		"show_tag_keys":     "flaky test (https://github.com/influxdata/influxdb/issues/15450)",
	},
	"influxdata/influxdb/monitor": {
		"check": "Cannot see overridden options from inside stdlib functions (https://github.com/influxdata/flux/issues/1720)",
	},
	"influxdata/influxdb/secrets": {
		"secrets": "Cannot inject custom deps into the test framework so the secrets don't lookup correctly",
	},
	"internal/promql": {
		"join": "unbounded test",
	},
	"testing/chronograf": {
		"buckets": "unbounded test",
	},
	"testing/kapacitor": {
		"fill_default": "unknown field type for f1",
	},
	"testing/pandas": {
		"extract_regexp_findStringIndex": "pandas. map does not correctly handled returned arrays (https://github.com/influxdata/flux/issues/1387)",
		"partition_strings_splitN":       "pandas. map does not correctly handled returned arrays (https://github.com/influxdata/flux/issues/1387)",
	},
	// PromQL tests don't work. Missing `range` or `_measurement`.
	"testing/promql": {
		"changes":                        "unbounded or missing _measurement column",
		"dayOfMonth":                     "unbounded or missing _measurement column",
		"dayOfWeek":                      "unbounded or missing _measurement column",
		"daysInMonth":                    "unbounded or missing _measurement column",
		"emptyTable":                     "unbounded or missing _measurement column",
		"extrapolatedRate_counter_rate":  "unbounded or missing _measurement column",
		"extrapolatedRate_nocounter":     "unbounded or missing _measurement column",
		"extrapolatedRate_norate":        "unbounded or missing _measurement column",
		"histogramQuantile":              "unbounded or missing _measurement column",
		"holtWinters":                    "unbounded or missing _measurement column",
		"hour":                           "unbounded or missing _measurement column",
		"instantRate":                    "unbounded or missing _measurement column",
		"labelReplace_empty_dst":         "unbounded or missing _measurement column",
		"labelReplace_full_string_match": "unbounded or missing _measurement column",
		"labelReplace_multiple_groups":   "unbounded or missing _measurement column",
		"labelReplace_src_empty":         "unbounded or missing _measurement column",
		"labelReplace_src_nonexistent":   "unbounded or missing _measurement column",
		"labelReplace_src_not_matched":   "unbounded or missing _measurement column",
		"labelReplace_sub_string_match":  "unbounded or missing _measurement column",
		"linearRegression_nopredict":     "unbounded or missing _measurement column",
		"linearRegression_predict":       "unbounded or missing _measurement column",
		"minute":                         "unbounded or missing _measurement column",
		"month":                          "unbounded or missing _measurement column",
		"resets":                         "unbounded or missing _measurement column",
		"timestamp":                      "unbounded or missing _measurement column",
		"year":                           "unbounded or missing _measurement column",
	},
}
<|endoftext|>


<commit_before>package stun

import (
	"testing"
)

func BenchmarkMessage_GetNotFound(b *testing.B) {
	m := New()
	b.ReportAllocs()
	for i := 0; i < b.N; i++ {
		m.Get(AttrRealm)
	}
}

func TestMessage_GetNoAllocs(t *testing.T) {
	m := New()
	NewSoftware("c").AddTo(m)
	m.WriteHeader()

	t.Run("Default", func(t *testing.T) {
		allocs := testing.AllocsPerRun(10, func() {
			m.Get(AttrSoftware)
		})
		if allocs > 0 {
			t.Error("allocated memory, but should not")
		}
	})
	t.Run("Not found", func(t *testing.T) {
		allocs := testing.AllocsPerRun(10, func() {
			m.Get(AttrOrigin)
		})
		if allocs > 0 {
			t.Error("allocated memory, but should not")
		}
	})
}

func TestPadding(t *testing.T) {
	tt := []struct {
		in, out int
	}{
		{4, 4},   // 0
		{2, 4},   // 1
		{5, 8},   // 2
		{8, 8},   // 3
		{11, 12}, // 4
		{1, 4},   // 5
		{3, 4},   // 6
		{6, 8},   // 7
		{7, 8},   // 8
		{0, 0},   // 9
		{40, 40}, // 10
	}
	for i, c := range tt {
		if got := nearestPaddedValueLength(c.in); got != c.out {
			t.Errorf("[%d]: padd(%d) %d (got) != %d (expected)",
				i, c.in, got, c.out,
			)
		}
	}
}

func TestAttrLengthError_Error(t *testing.T) {
	err := AttrOverflowErr{
		Got:  100,
		Max:  50,
		Type: AttrLifetime,
	}
	if err.Error() != "incorrect length of LIFETIME attribute: 100 exceeds maximum 50" {
		t.Error("bad error string", err)
	}
}

func TestAttrLengthErr_Error(t *testing.T) {
	err := AttrLengthErr{
		Attr:     AttrErrorCode,
		Expected: 15,
		Got:      99,
	}
	if err.Error() != "incorrect length of ERROR-CODE attribute: got 99, expected 15" {
		t.Errorf("bad error string: %s", err)
	}
}
<commit_msg>attributes: add BenchmarkMessage_Get<commit_after>package stun

import (
	"testing"
)

func BenchmarkMessage_GetNotFound(b *testing.B) {
	m := New()
	b.ReportAllocs()
	for i := 0; i < b.N; i++ {
		m.Get(AttrRealm)
	}
}

func BenchmarkMessage_Get(b *testing.B) {
	m := New()
	m.Add(AttrUsername, []byte{1, 2, 3, 4, 5, 6, 7})
	b.ReportAllocs()
	for i := 0; i < b.N; i++ {
		m.Get(AttrUsername)
	}
}

func TestMessage_GetNoAllocs(t *testing.T) {
	m := New()
	NewSoftware("c").AddTo(m)
	m.WriteHeader()

	t.Run("Default", func(t *testing.T) {
		allocs := testing.AllocsPerRun(10, func() {
			m.Get(AttrSoftware)
		})
		if allocs > 0 {
			t.Error("allocated memory, but should not")
		}
	})
	t.Run("Not found", func(t *testing.T) {
		allocs := testing.AllocsPerRun(10, func() {
			m.Get(AttrOrigin)
		})
		if allocs > 0 {
			t.Error("allocated memory, but should not")
		}
	})
}

func TestPadding(t *testing.T) {
	tt := []struct {
		in, out int
	}{
		{4, 4},   // 0
		{2, 4},   // 1
		{5, 8},   // 2
		{8, 8},   // 3
		{11, 12}, // 4
		{1, 4},   // 5
		{3, 4},   // 6
		{6, 8},   // 7
		{7, 8},   // 8
		{0, 0},   // 9
		{40, 40}, // 10
	}
	for i, c := range tt {
		if got := nearestPaddedValueLength(c.in); got != c.out {
			t.Errorf("[%d]: padd(%d) %d (got) != %d (expected)",
				i, c.in, got, c.out,
			)
		}
	}
}

func TestAttrLengthError_Error(t *testing.T) {
	err := AttrOverflowErr{
		Got:  100,
		Max:  50,
		Type: AttrLifetime,
	}
	if err.Error() != "incorrect length of LIFETIME attribute: 100 exceeds maximum 50" {
		t.Error("bad error string", err)
	}
}

func TestAttrLengthErr_Error(t *testing.T) {
	err := AttrLengthErr{
		Attr:     AttrErrorCode,
		Expected: 15,
		Got:      99,
	}
	if err.Error() != "incorrect length of ERROR-CODE attribute: got 99, expected 15" {
		t.Errorf("bad error string: %s", err)
	}
}
<|endoftext|>


<commit_before>package dpn_test

import (
	"fmt"
	"github.com/APTrust/bagman/dpn"
	"net/http"
	"testing"
	"time"
)

var skipSyncMessagePrinted = false

const (
	BAG_COUNT     = 6
	REPL_COUNT    = 24
	RESTORE_COUNT = 4
)

func runSyncTests(t *testing.T) bool {
	config := loadConfig(t, configFile)
	_, err := http.Get(config.RestClient.LocalServiceURL)
	if !canRunSyncTests("aptrust", config.RestClient.LocalServiceURL, err) {
		return false
	}
	for nodeNamespace, url := range config.RemoteNodeURLs {
		if url == "" {
			continue
		}
		_, err := http.Get(url)
		if !canRunSyncTests(nodeNamespace, url, err) {
			return false
		}
	}
	return true
}

func canRunSyncTests(nodeNamespace string, url string, err error) (bool) {
	if err != nil {
		if skipSyncMessagePrinted == false {
			skipSyncMessagePrinted = true
			fmt.Printf("**** Skipping DPN sync integration tests: "+
				"%s server is not running at %s\n", nodeNamespace, url)
			fmt.Println("     Run the run_cluster.sh script in " +
				"DPN-REST/dpnode to get a local cluster running.")
		}
		return false
	}
	return true
}

func newDPNSync(t *testing.T) (*dpn.DPNSync) {
	// loadConfig and configFile are defined in dpnrestclient_test.go
	config := loadConfig(t, configFile)

	dpnSync, err := dpn.NewDPNSync(config)
	if err != nil {
		t.Error(err)
		return nil
	}

	for namespace, _ := range config.RemoteNodeTokens {
		if dpnSync.RemoteClients[namespace] == nil {
			t.Errorf("Remote client for node '%s' is missing", namespace)
			return nil
		}
	}
	return dpnSync
}

func TestNewDPNSync(t *testing.T) {
	if runSyncTests(t) == false {
		return  // local test cluster isn't running
	}
	dpnSync := newDPNSync(t)
	if dpnSync == nil {
		return
	}
}

func TestGetAllNodes(t *testing.T) {
	if runSyncTests(t) == false {
		return  // local test cluster isn't running
	}
	dpnSync := newDPNSync(t)
	if dpnSync == nil {
		return
	}
	nodes, err := dpnSync.GetAllNodes()
	if err != nil {
		t.Error(err)
	}
	if len(nodes) != 5 {
		t.Errorf("Expected 5 nodes, got %d", len(nodes))
	}
}

func TestUpdateLastPullDate(t *testing.T) {
	if runSyncTests(t) == false {
		return  // local test cluster isn't running
	}
	dpnSync := newDPNSync(t)
	if dpnSync == nil {
		return
	}
	nodes, err := dpnSync.GetAllNodes()
	if err != nil {
		t.Error(err)
		return
	}
	if len(nodes) != 5 {
		t.Errorf("Expected 5 nodes, got %d", len(nodes))
		return
	}
	someNode := nodes[2]
	origLastPullDate := someNode.LastPullDate
	newLastPullDate := origLastPullDate.Add(-12 * time.Hour)

	updatedNode, err := dpnSync.UpdateLastPullDate(someNode, newLastPullDate)
	if err != nil {
		t.Error(err)
		return
	}
	if updatedNode.LastPullDate != newLastPullDate {
		t.Errorf("Expected LastPullDate %s, got %s",
			newLastPullDate, updatedNode.LastPullDate)
	}
}

func TestSyncBags(t *testing.T) {
	if runSyncTests(t) == false {
		return  // local test cluster isn't running
	}
	dpnSync := newDPNSync(t)
	if dpnSync == nil {
		return
	}
	nodes, err := dpnSync.GetAllNodes()
	if err != nil {
		t.Error(err)
		return
	}
	for _, node := range nodes {
		if node.Namespace == "aptrust" {
			continue
		}
		aLongTimeAgo := time.Date(1999, time.December, 31, 23, 0, 0, 0, time.UTC)
		node.LastPullDate = aLongTimeAgo
		_, err := dpnSync.LocalClient.DPNNodeUpdate(node)
		if err != nil {
			t.Errorf("Error setting last pull date to 1999: %v", err)
			return
		}
		bagsSynched, err := dpnSync.SyncBags(node)
		if err != nil {
			t.Errorf("Error synching bags for node %s: %v", node.Namespace, err)
		}
		if len(bagsSynched) != BAG_COUNT {
			t.Errorf("Synched %d bags for node %s. Expected %d.",
				len(bagsSynched), node.Namespace, BAG_COUNT)
		}
		for _, remoteBag := range(bagsSynched) {
			if remoteBag == nil {
				t.Errorf("Remote bag is nil")
				return
			}
			localBag, _ := dpnSync.LocalClient.DPNBagGet(remoteBag.UUID)
			if localBag == nil {
				t.Errorf("Bag %s didn't make into local registry", remoteBag.UUID)
			}
			if localBag.UpdatedAt != remoteBag.UpdatedAt {
				t.Errorf("Bag %s isn't up to date in local registry", remoteBag.UUID)
			}
		}
	}
}

func TestSyncReplicationRequests(t *testing.T) {
	if runSyncTests(t) == false {
		return  // local test cluster isn't running
	}
	dpnSync := newDPNSync(t)
	if dpnSync == nil {
		return
	}
	nodes, err := dpnSync.GetAllNodes()
	if err != nil {
		t.Error(err)
		return
	}
	for _, node := range nodes {
		if node.Namespace == "aptrust" {
			continue
		}
		aLongTimeAgo := time.Date(1999, time.December, 31, 23, 0, 0, 0, time.UTC)
		node.LastPullDate = aLongTimeAgo
		_, err := dpnSync.LocalClient.DPNNodeUpdate(node)
		if err != nil {
			t.Errorf("Error setting last pull date to 1999: %v", err)
			return
		}
		xfersSynched, err := dpnSync.SyncReplicationRequests(node)
		if err != nil {
			t.Errorf("Error synching replication requests for node %s: %v",
				node.Namespace, err)
		}
		if len(xfersSynched) != REPL_COUNT {
			t.Errorf("Synched %d replication requests for %s. Expected %d.",
				len(xfersSynched), node.Namespace, REPL_COUNT)
		}
		for _, xfer := range(xfersSynched) {
			if xfer == nil {
				t.Errorf("Xfer is nil")
				return
			}
			localCopy, _ := dpnSync.LocalClient.ReplicationTransferGet(xfer.ReplicationId)
			if localCopy == nil {
				t.Errorf("Xfer %s didn't make into local registry", xfer.ReplicationId)
			}
			if xfer.UpdatedAt != localCopy.UpdatedAt {
				t.Errorf("Xfer %s isn't up to date in local registry", xfer.ReplicationId)
			}
		}
	}
}

func TestSyncRestoreRequests(t *testing.T) {
	if runSyncTests(t) == false {
		return  // local test cluster isn't running
	}
	dpnSync := newDPNSync(t)
	if dpnSync == nil {
		return
	}
	nodes, err := dpnSync.GetAllNodes()
	if err != nil {
		t.Error(err)
		return
	}
	for _, node := range nodes {
		if node.Namespace == "aptrust" {
			continue
		}
		aLongTimeAgo := time.Date(1999, time.December, 31, 23, 0, 0, 0, time.UTC)
		node.LastPullDate = aLongTimeAgo
		_, err := dpnSync.LocalClient.DPNNodeUpdate(node)
		if err != nil {
			t.Errorf("Error setting last pull date to 1999: %v", err)
			return
		}
		xfersSynched, err := dpnSync.SyncRestoreRequests(node)
		if err != nil {
			t.Errorf("Error synching restore requests for node %s: %v",
				node.Namespace, err)
		}
		if len(xfersSynched) != RESTORE_COUNT {
			t.Errorf("Synched %d restore requests for %s. Expected %d.",
				len(xfersSynched), node.Namespace, RESTORE_COUNT)
		}
		for _, xfer := range(xfersSynched) {
			localCopy, _ := dpnSync.LocalClient.RestoreTransferGet(xfer.RestoreId)
			if localCopy == nil {
				t.Errorf("Xfer %s didn't make into local registry", xfer.RestoreId)
			}
			if xfer.UpdatedAt != localCopy.UpdatedAt {
				t.Errorf("Xfer %s isn't up to date in local registry", xfer.RestoreId)
			}
		}
	}
}

func TestSyncEverythingFromNode(t *testing.T) {
	if runSyncTests(t) == false {
		return  // local test cluster isn't running
	}
	dpnSync := newDPNSync(t)
	if dpnSync == nil {
		return
	}
	nodes, err := dpnSync.GetAllNodes()
	if err != nil {
		t.Error(err)
		return
	}
	for _, node := range nodes {
		if node.Namespace == "aptrust" {
			continue
		}
		aLongTimeAgo := time.Date(1999, time.December, 31, 23, 0, 0, 0, time.UTC)
		node.LastPullDate = aLongTimeAgo
		_, err := dpnSync.LocalClient.DPNNodeUpdate(node)
		if err != nil {
			t.Errorf("Error setting last pull date to 1999: %v", err)
			return
		}
		syncResult := dpnSync.SyncEverythingFromNode(node)

		// Bags
		if syncResult.BagSyncError != nil {
			t.Errorf("Got unexpected bag-sync error from node %s: %v",
				node.Namespace, syncResult.BagSyncError)
		}
		if len(syncResult.Bags) != BAG_COUNT {
			t.Errorf("Expected %d bags from %s, got %d",
				BAG_COUNT, node.Namespace, len(syncResult.Bags))
		}

		// Replication Transfers
		if syncResult.ReplicationSyncError != nil {
			t.Errorf("Got unexpected replication transfer-sync error from node %s: %v",
				node.Namespace, syncResult.ReplicationSyncError)
		}
		if len(syncResult.ReplicationTransfers) != REPL_COUNT {
			t.Errorf("Expected %d replication transfers from %s, got %d",
				REPL_COUNT, node.Namespace, len(syncResult.ReplicationTransfers))
		}

		// Bags
		if syncResult.RestoreSyncError != nil {
			t.Errorf("Got unexpected restore transfer-sync error from node %s: %v",
				node.Namespace, syncResult.RestoreSyncError)
		}
		if len(syncResult.RestoreTransfers) != RESTORE_COUNT {
			t.Errorf("Expected %d restore transfers from %s, got %d",
				RESTORE_COUNT, node.Namespace, len(syncResult.RestoreTransfers))
		}

		// Timestamp update
		updatedNode, err := dpnSync.LocalClient.DPNNodeGet(node.Namespace)
		if err != nil {
			t.Errorf("Can't check timestamp. Error getting node: %v", err)
		}
		if updatedNode.LastPullDate == aLongTimeAgo {
			t.Errorf("LastPullDate was not updated for %s", node.Namespace)
		}
	}
}

func TestSyncWithError(t *testing.T) {
	if runSyncTests(t) == false {
		return  // local test cluster isn't running
	}
	dpnSync := newDPNSync(t)
	if dpnSync == nil {
		return
	}
	nodes, err := dpnSync.GetAllNodes()
	if err != nil {
		t.Error(err)
		return
	}

	// Pick one node to sync with, and set the API key for that node
	// to a value we know is invalid. This will cause the sync to fail.
	node := nodes[len(nodes) - 1]
	dpnSync.RemoteClients[node.Namespace].APIKey = "0000000000000000"

	aLongTimeAgo := time.Date(1999, time.December, 31, 23, 0, 0, 0, time.UTC)
	node.LastPullDate = aLongTimeAgo
	_, err = dpnSync.LocalClient.DPNNodeUpdate(node)
	if err != nil {
		t.Errorf("Error setting last pull date to 1999: %v", err)
		return
	}

	syncResult := dpnSync.SyncEverythingFromNode(node)
	if syncResult.BagSyncError == nil {
		t.Errorf("BagSyncError should not be nil")
	}
	if syncResult.ReplicationSyncError == nil {
		t.Errorf("ReplicationSyncError should not be nil")
	}
	if syncResult.RestoreSyncError == nil {
		t.Errorf("RestoreSyncError should not be nil")
	}

	// Because the sync failed (due to the bad API Key), the LastPullDate
	// on the node we tried to pull from should NOT be updated.
	updatedNode, err := dpnSync.LocalClient.DPNNodeGet(node.Namespace)
	if err != nil {
		t.Errorf("Can't check timestamp. Error getting node: %v", err)
	}
	if updatedNode.LastPullDate != aLongTimeAgo {
		t.Errorf("LastPullDate was updated when it should not have been")
	}
}

func TestHasSyncErrors(t *testing.T) {
	syncResult := &dpn.SyncResult{}
	if syncResult.HasSyncErrors() == true {
		t.Errorf("HasSyncErrors() returned true. Expected false.")
	}
	syncResult.BagSyncError = fmt.Errorf("Oops.")
	if syncResult.HasSyncErrors() == false {
		t.Errorf("HasSyncErrors() returned false. Expected true.")
	}
	syncResult.BagSyncError = nil
	syncResult.ReplicationSyncError = fmt.Errorf("Oops.")
	if syncResult.HasSyncErrors() == false {
		t.Errorf("HasSyncErrors() returned false. Expected true.")
	}
	syncResult.ReplicationSyncError = nil
	syncResult.RestoreSyncError = fmt.Errorf("Oops.")
	if syncResult.HasSyncErrors() == false {
		t.Errorf("HasSyncErrors() returned false. Expected true.")
	}
}
<commit_msg>Removed test for UpdateLastPullDate<commit_after>package dpn_test

import (
	"fmt"
	"github.com/APTrust/bagman/dpn"
	"net/http"
	"testing"
	"time"
)

var skipSyncMessagePrinted = false

const (
	BAG_COUNT     = 6
	REPL_COUNT    = 24
	RESTORE_COUNT = 4
)

func runSyncTests(t *testing.T) bool {
	config := loadConfig(t, configFile)
	_, err := http.Get(config.RestClient.LocalServiceURL)
	if !canRunSyncTests("aptrust", config.RestClient.LocalServiceURL, err) {
		return false
	}
	for nodeNamespace, url := range config.RemoteNodeURLs {
		if url == "" {
			continue
		}
		_, err := http.Get(url)
		if !canRunSyncTests(nodeNamespace, url, err) {
			return false
		}
	}
	return true
}

func canRunSyncTests(nodeNamespace string, url string, err error) (bool) {
	if err != nil {
		if skipSyncMessagePrinted == false {
			skipSyncMessagePrinted = true
			fmt.Printf("**** Skipping DPN sync integration tests: "+
				"%s server is not running at %s\n", nodeNamespace, url)
			fmt.Println("     Run the run_cluster.sh script in " +
				"DPN-REST/dpnode to get a local cluster running.")
		}
		return false
	}
	return true
}

func newDPNSync(t *testing.T) (*dpn.DPNSync) {
	// loadConfig and configFile are defined in dpnrestclient_test.go
	config := loadConfig(t, configFile)

	dpnSync, err := dpn.NewDPNSync(config)
	if err != nil {
		t.Error(err)
		return nil
	}

	for namespace, _ := range config.RemoteNodeTokens {
		if dpnSync.RemoteClients[namespace] == nil {
			t.Errorf("Remote client for node '%s' is missing", namespace)
			return nil
		}
	}
	return dpnSync
}

func TestNewDPNSync(t *testing.T) {
	if runSyncTests(t) == false {
		return  // local test cluster isn't running
	}
	dpnSync := newDPNSync(t)
	if dpnSync == nil {
		return
	}
}

func TestGetAllNodes(t *testing.T) {
	if runSyncTests(t) == false {
		return  // local test cluster isn't running
	}
	dpnSync := newDPNSync(t)
	if dpnSync == nil {
		return
	}
	nodes, err := dpnSync.GetAllNodes()
	if err != nil {
		t.Error(err)
	}
	if len(nodes) != 5 {
		t.Errorf("Expected 5 nodes, got %d", len(nodes))
	}
}

func TestSyncBags(t *testing.T) {
	if runSyncTests(t) == false {
		return  // local test cluster isn't running
	}
	dpnSync := newDPNSync(t)
	if dpnSync == nil {
		return
	}
	nodes, err := dpnSync.GetAllNodes()
	if err != nil {
		t.Error(err)
		return
	}
	for _, node := range nodes {
		if node.Namespace == "aptrust" {
			continue
		}
		aLongTimeAgo := time.Date(1999, time.December, 31, 23, 0, 0, 0, time.UTC)
		node.LastPullDate = aLongTimeAgo
		_, err := dpnSync.LocalClient.DPNNodeUpdate(node)
		if err != nil {
			t.Errorf("Error setting last pull date to 1999: %v", err)
			return
		}
		bagsSynched, err := dpnSync.SyncBags(node)
		if err != nil {
			t.Errorf("Error synching bags for node %s: %v", node.Namespace, err)
		}
		if len(bagsSynched) != BAG_COUNT {
			t.Errorf("Synched %d bags for node %s. Expected %d.",
				len(bagsSynched), node.Namespace, BAG_COUNT)
		}
		for _, remoteBag := range(bagsSynched) {
			if remoteBag == nil {
				t.Errorf("Remote bag is nil")
				return
			}
			localBag, _ := dpnSync.LocalClient.DPNBagGet(remoteBag.UUID)
			if localBag == nil {
				t.Errorf("Bag %s didn't make into local registry", remoteBag.UUID)
			}
			if localBag.UpdatedAt != remoteBag.UpdatedAt {
				t.Errorf("Bag %s isn't up to date in local registry", remoteBag.UUID)
			}
		}
	}
}

func TestSyncReplicationRequests(t *testing.T) {
	if runSyncTests(t) == false {
		return  // local test cluster isn't running
	}
	dpnSync := newDPNSync(t)
	if dpnSync == nil {
		return
	}
	nodes, err := dpnSync.GetAllNodes()
	if err != nil {
		t.Error(err)
		return
	}
	for _, node := range nodes {
		if node.Namespace == "aptrust" {
			continue
		}
		aLongTimeAgo := time.Date(1999, time.December, 31, 23, 0, 0, 0, time.UTC)
		node.LastPullDate = aLongTimeAgo
		_, err := dpnSync.LocalClient.DPNNodeUpdate(node)
		if err != nil {
			t.Errorf("Error setting last pull date to 1999: %v", err)
			return
		}
		xfersSynched, err := dpnSync.SyncReplicationRequests(node)
		if err != nil {
			t.Errorf("Error synching replication requests for node %s: %v",
				node.Namespace, err)
		}
		if len(xfersSynched) != REPL_COUNT {
			t.Errorf("Synched %d replication requests for %s. Expected %d.",
				len(xfersSynched), node.Namespace, REPL_COUNT)
		}
		for _, xfer := range(xfersSynched) {
			if xfer == nil {
				t.Errorf("Xfer is nil")
				return
			}
			localCopy, _ := dpnSync.LocalClient.ReplicationTransferGet(xfer.ReplicationId)
			if localCopy == nil {
				t.Errorf("Xfer %s didn't make into local registry", xfer.ReplicationId)
			}
			if xfer.UpdatedAt != localCopy.UpdatedAt {
				t.Errorf("Xfer %s isn't up to date in local registry", xfer.ReplicationId)
			}
		}
	}
}

func TestSyncRestoreRequests(t *testing.T) {
	if runSyncTests(t) == false {
		return  // local test cluster isn't running
	}
	dpnSync := newDPNSync(t)
	if dpnSync == nil {
		return
	}
	nodes, err := dpnSync.GetAllNodes()
	if err != nil {
		t.Error(err)
		return
	}
	for _, node := range nodes {
		if node.Namespace == "aptrust" {
			continue
		}
		aLongTimeAgo := time.Date(1999, time.December, 31, 23, 0, 0, 0, time.UTC)
		node.LastPullDate = aLongTimeAgo
		_, err := dpnSync.LocalClient.DPNNodeUpdate(node)
		if err != nil {
			t.Errorf("Error setting last pull date to 1999: %v", err)
			return
		}
		xfersSynched, err := dpnSync.SyncRestoreRequests(node)
		if err != nil {
			t.Errorf("Error synching restore requests for node %s: %v",
				node.Namespace, err)
		}
		if len(xfersSynched) != RESTORE_COUNT {
			t.Errorf("Synched %d restore requests for %s. Expected %d.",
				len(xfersSynched), node.Namespace, RESTORE_COUNT)
		}
		for _, xfer := range(xfersSynched) {
			localCopy, _ := dpnSync.LocalClient.RestoreTransferGet(xfer.RestoreId)
			if localCopy == nil {
				t.Errorf("Xfer %s didn't make into local registry", xfer.RestoreId)
			}
			if xfer.UpdatedAt != localCopy.UpdatedAt {
				t.Errorf("Xfer %s isn't up to date in local registry", xfer.RestoreId)
			}
		}
	}
}

func TestSyncEverythingFromNode(t *testing.T) {
	if runSyncTests(t) == false {
		return  // local test cluster isn't running
	}
	dpnSync := newDPNSync(t)
	if dpnSync == nil {
		return
	}
	nodes, err := dpnSync.GetAllNodes()
	if err != nil {
		t.Error(err)
		return
	}
	for _, node := range nodes {
		if node.Namespace == "aptrust" {
			continue
		}
		aLongTimeAgo := time.Date(1999, time.December, 31, 23, 0, 0, 0, time.UTC)
		node.LastPullDate = aLongTimeAgo
		_, err := dpnSync.LocalClient.DPNNodeUpdate(node)
		if err != nil {
			t.Errorf("Error setting last pull date to 1999: %v", err)
			return
		}
		syncResult := dpnSync.SyncEverythingFromNode(node)

		// Bags
		if syncResult.BagSyncError != nil {
			t.Errorf("Got unexpected bag-sync error from node %s: %v",
				node.Namespace, syncResult.BagSyncError)
		}
		if len(syncResult.Bags) != BAG_COUNT {
			t.Errorf("Expected %d bags from %s, got %d",
				BAG_COUNT, node.Namespace, len(syncResult.Bags))
		}

		// Replication Transfers
		if syncResult.ReplicationSyncError != nil {
			t.Errorf("Got unexpected replication transfer-sync error from node %s: %v",
				node.Namespace, syncResult.ReplicationSyncError)
		}
		if len(syncResult.ReplicationTransfers) != REPL_COUNT {
			t.Errorf("Expected %d replication transfers from %s, got %d",
				REPL_COUNT, node.Namespace, len(syncResult.ReplicationTransfers))
		}

		// Bags
		if syncResult.RestoreSyncError != nil {
			t.Errorf("Got unexpected restore transfer-sync error from node %s: %v",
				node.Namespace, syncResult.RestoreSyncError)
		}
		if len(syncResult.RestoreTransfers) != RESTORE_COUNT {
			t.Errorf("Expected %d restore transfers from %s, got %d",
				RESTORE_COUNT, node.Namespace, len(syncResult.RestoreTransfers))
		}

		// Timestamp update
		updatedNode, err := dpnSync.LocalClient.DPNNodeGet(node.Namespace)
		if err != nil {
			t.Errorf("Can't check timestamp. Error getting node: %v", err)
		}
		if updatedNode.LastPullDate == aLongTimeAgo {
			t.Errorf("LastPullDate was not updated for %s", node.Namespace)
		}
	}
}

func TestSyncWithError(t *testing.T) {
	if runSyncTests(t) == false {
		return  // local test cluster isn't running
	}
	dpnSync := newDPNSync(t)
	if dpnSync == nil {
		return
	}
	nodes, err := dpnSync.GetAllNodes()
	if err != nil {
		t.Error(err)
		return
	}

	// Pick one node to sync with, and set the API key for that node
	// to a value we know is invalid. This will cause the sync to fail.
	node := nodes[len(nodes) - 1]
	dpnSync.RemoteClients[node.Namespace].APIKey = "0000000000000000"

	aLongTimeAgo := time.Date(1999, time.December, 31, 23, 0, 0, 0, time.UTC)
	node.LastPullDate = aLongTimeAgo
	_, err = dpnSync.LocalClient.DPNNodeUpdate(node)
	if err != nil {
		t.Errorf("Error setting last pull date to 1999: %v", err)
		return
	}

	syncResult := dpnSync.SyncEverythingFromNode(node)
	if syncResult.BagSyncError == nil {
		t.Errorf("BagSyncError should not be nil")
	}
	if syncResult.ReplicationSyncError == nil {
		t.Errorf("ReplicationSyncError should not be nil")
	}
	if syncResult.RestoreSyncError == nil {
		t.Errorf("RestoreSyncError should not be nil")
	}

	// Because the sync failed (due to the bad API Key), the LastPullDate
	// on the node we tried to pull from should NOT be updated.
	updatedNode, err := dpnSync.LocalClient.DPNNodeGet(node.Namespace)
	if err != nil {
		t.Errorf("Can't check timestamp. Error getting node: %v", err)
	}
	if updatedNode.LastPullDate != aLongTimeAgo {
		t.Errorf("LastPullDate was updated when it should not have been")
	}
}

func TestHasSyncErrors(t *testing.T) {
	syncResult := &dpn.SyncResult{}
	if syncResult.HasSyncErrors() == true {
		t.Errorf("HasSyncErrors() returned true. Expected false.")
	}
	syncResult.BagSyncError = fmt.Errorf("Oops.")
	if syncResult.HasSyncErrors() == false {
		t.Errorf("HasSyncErrors() returned false. Expected true.")
	}
	syncResult.BagSyncError = nil
	syncResult.ReplicationSyncError = fmt.Errorf("Oops.")
	if syncResult.HasSyncErrors() == false {
		t.Errorf("HasSyncErrors() returned false. Expected true.")
	}
	syncResult.ReplicationSyncError = nil
	syncResult.RestoreSyncError = fmt.Errorf("Oops.")
	if syncResult.HasSyncErrors() == false {
		t.Errorf("HasSyncErrors() returned false. Expected true.")
	}
}
<|endoftext|>


<commit_before>package main

import (
    "flag"
    "bufio"
    "encoding/binary"
    "os"
    "strings"
    "database/sql"
    _ "github.com/lib/pq"
    _ "github.com/go-sql-driver/mysql"
    "fmt"
    "gopkg.in/ini.v1"
    "log"
)

type Config struct {
    Driver string
    Host string
    Port string
    User string
    Pass string
    Dbname string
    Dbargs string
    Table string
    UserField string
    PassField string
    ServerField string
}

func auth(conf *Config, db *sql.DB, user string,
    server string, passwd string) (bool, error) {
    var query string
    var value string
    var err error

    log.Printf("user: %s, server: %s, passwd: %s\n", user, server, passwd)

    if conf.ServerField != "" {
        query = fmt.Sprintf(
            "select %s from %s where %s = $1 and %s = $2 and %s = $3",
            conf.UserField, conf.Table, conf.UserField, conf.PassField,
            conf.ServerField)
        err = db.QueryRow(query, user, passwd, server).Scan(&value)
    } else {
        query = fmt.Sprintf(
            "select %s from %s where %s = $1 and %s = $2",
            conf.UserField, conf.Table, conf.UserField, conf.PassField)
        err = db.QueryRow(query, user, passwd).Scan(&value)
    }
    if err != nil || value == "" {
        return false, err
    }

    return true, nil
}

func isuser(conf *Config, db *sql.DB, user string,
    server string) (bool, error) {
    var query string
    var value string
    var err error

    if conf.ServerField != "" {
        query = fmt.Sprintf(
            "select %s from %s where %s = $1 and %s = $2",
            conf.UserField, conf.Table, conf.UserField, conf.ServerField)
        err = db.QueryRow(query, user, server).Scan(&value)
    } else {
        query = fmt.Sprintf(
            "select %s from %s where %s = $1",
            conf.UserField, conf.Table, conf.UserField)
        err = db.QueryRow(query, user).Scan(&value)
    }
    if err != nil || value == "" {
        return false, err
    }

    return true, nil
}

func GetSqlConnectionString(conf *Config) string {
    return fmt.Sprintf("%s://%s:%s@%s:%s/%s?%s",
        conf.Driver, conf.User, conf.Pass, conf.Host, conf.Port,
        conf.Dbname, conf.Dbargs)
}

func OpenSqlConnection(conf *Config) (*sql.DB, error) {
    var err error

    connectionString := GetSqlConnectionString(conf)
    db, err := sql.Open(conf.Driver, connectionString)
    if err != nil {
        return nil, err
    }

    if err = db.Ping(); err != nil {
        return nil, err
    }

    return db, nil
}

func AuthLoop(conf *Config) {
    db, err := OpenSqlConnection(conf)

    bioIn := bufio.NewReader(os.Stdin)
    bioOut := bufio.NewWriter(os.Stdout)

    var success bool
    var length uint16
    var result uint16

    for {
        _ = binary.Read(bioIn, binary.BigEndian, &length)

        buf := make([]byte, length)

        r, _ := bioIn.Read(buf)
        if r == 0 {
            continue
        }

        if err != nil {
            err = db.Ping()
        }

        if err == nil {
            data := strings.Split(string(buf), ":")
            if data[0] == "auth" {
                success, err = auth(conf, db, data[1], data[2], data[3])
            } else if data[0] == "isuser" {
                success, err = isuser(conf, db, data[1], data[2])
            } else {
                success = false
            }
        } else {
            success = false
        }

        length = 2
        binary.Write(bioOut, binary.BigEndian, &length)

        if success != true {
            result = 0
        } else {
            result = 1
        }

        binary.Write(bioOut, binary.BigEndian, &result)
        bioOut.Flush()
    }
}

func main() {
    filename := flag.String("conf", "/etc/ejabberd-pg-auth.ini",
        "Config file with all the connection infos needed.")
    flag.Parse()

    cfg, err := ini.Load(*filename)
    if err != nil {
        log.Fatal(err)
    }

    conf := new(Config)
    err = cfg.MapTo(conf)
    if err != nil {
        log.Fatal(err)
    }

    log.SetOutput(os.Stderr)

    AuthLoop(conf)
}
<commit_msg>Add 25ms pause to loop solves 100% CPU problem<commit_after>package main

import (
    "flag"
    "bufio"
    "encoding/binary"
    "os"
    "strings"
    "database/sql"
    _ "github.com/lib/pq"
    _ "github.com/go-sql-driver/mysql"
    "fmt"
    "gopkg.in/ini.v1"
    "log"
)

type Config struct {
    Driver string
    Host string
    Port string
    User string
    Pass string
    Dbname string
    Dbargs string
    Table string
    UserField string
    PassField string
    ServerField string
}

func auth(conf *Config, db *sql.DB, user string,
    server string, passwd string) (bool, error) {
    var query string
    var value string
    var err error

    log.Printf("user: %s, server: %s, passwd: %s\n", user, server, passwd)

    if conf.ServerField != "" {
        query = fmt.Sprintf(
            "select %s from %s where %s = $1 and %s = $2 and %s = $3",
            conf.UserField, conf.Table, conf.UserField, conf.PassField,
            conf.ServerField)
        err = db.QueryRow(query, user, passwd, server).Scan(&value)
    } else {
        query = fmt.Sprintf(
            "select %s from %s where %s = $1 and %s = $2",
            conf.UserField, conf.Table, conf.UserField, conf.PassField)
        err = db.QueryRow(query, user, passwd).Scan(&value)
    }
    if err != nil || value == "" {
        return false, err
    }

    return true, nil
}

func isuser(conf *Config, db *sql.DB, user string,
    server string) (bool, error) {
    var query string
    var value string
    var err error

    if conf.ServerField != "" {
        query = fmt.Sprintf(
            "select %s from %s where %s = $1 and %s = $2",
            conf.UserField, conf.Table, conf.UserField, conf.ServerField)
        err = db.QueryRow(query, user, server).Scan(&value)
    } else {
        query = fmt.Sprintf(
            "select %s from %s where %s = $1",
            conf.UserField, conf.Table, conf.UserField)
        err = db.QueryRow(query, user).Scan(&value)
    }
    if err != nil || value == "" {
        return false, err
    }

    return true, nil
}

func GetSqlConnectionString(conf *Config) string {
    return fmt.Sprintf("%s://%s:%s@%s:%s/%s?%s",
        conf.Driver, conf.User, conf.Pass, conf.Host, conf.Port,
        conf.Dbname, conf.Dbargs)
}

func OpenSqlConnection(conf *Config) (*sql.DB, error) {
    var err error

    connectionString := GetSqlConnectionString(conf)
    db, err := sql.Open(conf.Driver, connectionString)
    if err != nil {
        return nil, err
    }

    if err = db.Ping(); err != nil {
        return nil, err
    }

    return db, nil
}

func AuthLoop(conf *Config) {
    db, err := OpenSqlConnection(conf)

    bioIn := bufio.NewReader(os.Stdin)
    bioOut := bufio.NewWriter(os.Stdout)

    var success bool
    var length uint16
    var result uint16

    for {
        _ = binary.Read(bioIn, binary.BigEndian, &length)

        buf := make([]byte, length)

        r, err := bioIn.Read(buf)
        if r == 0 || err != nil {
            time.Sleep(25 * time.Millisecond)
            continue
        }

        if err != nil {
            err = db.Ping()
        }

        if err == nil {
            data := strings.Split(string(buf), ":")
            if data[0] == "auth" {
                success, err = auth(conf, db, data[1], data[2], data[3])
            } else if data[0] == "isuser" {
                success, err = isuser(conf, db, data[1], data[2])
            } else {
                success = false
            }
        } else {
            success = false
        }

        length = 2
        binary.Write(bioOut, binary.BigEndian, &length)

        if success != true {
            result = 0
        } else {
            result = 1
        }

        binary.Write(bioOut, binary.BigEndian, &result)
        bioOut.Flush()
    }
}

func main() {
    filename := flag.String("conf", "/etc/ejabberd-pg-auth.ini",
        "Config file with all the connection infos needed.")
    flag.Parse()

    cfg, err := ini.Load(*filename)
    if err != nil {
        log.Fatal(err)
    }

    conf := new(Config)
    err = cfg.MapTo(conf)
    if err != nil {
        log.Fatal(err)
    }

    log.SetOutput(os.Stderr)

    AuthLoop(conf)
}
<|endoftext|>


<commit_before>package herd

import (
	"fmt"
	"net"
	"net/http"
)

var httpdHerd *Herd

func (herd *Herd) startServer(portNum uint, daemon bool) error {
	listener, err := net.Listen("tcp", fmt.Sprintf(":%d", portNum))
	if err != nil {
		return err
	}
	httpdHerd = herd
	http.HandleFunc("/", statusHandler)
	http.HandleFunc("/listSubs", listSubsHandler)
	http.HandleFunc("/showSubs", showAllSubsHandler)
	http.HandleFunc("/showDeviantSubs", showDeviantSubsHandler)
	http.HandleFunc("/showCompliantSubs", showCompliantSubsHandler)
	if daemon {
		go http.Serve(listener, nil)
	} else {
		http.Serve(listener, nil)
	}
	return nil
}

func (herd *Herd) addHtmlWriter(htmlWriter HtmlWriter) {
	herd.htmlWriters = append(herd.htmlWriters, htmlWriter)
}
<commit_msg>Fix bug: list of all subs did not work.<commit_after>package herd

import (
	"fmt"
	"net"
	"net/http"
)

var httpdHerd *Herd

func (herd *Herd) startServer(portNum uint, daemon bool) error {
	listener, err := net.Listen("tcp", fmt.Sprintf(":%d", portNum))
	if err != nil {
		return err
	}
	httpdHerd = herd
	http.HandleFunc("/", statusHandler)
	http.HandleFunc("/listSubs", listSubsHandler)
	http.HandleFunc("/showAllSubs", showAllSubsHandler)
	http.HandleFunc("/showDeviantSubs", showDeviantSubsHandler)
	http.HandleFunc("/showCompliantSubs", showCompliantSubsHandler)
	if daemon {
		go http.Serve(listener, nil)
	} else {
		http.Serve(listener, nil)
	}
	return nil
}

func (herd *Herd) addHtmlWriter(htmlWriter HtmlWriter) {
	herd.htmlWriters = append(herd.htmlWriters, htmlWriter)
}
<|endoftext|>


<commit_before>package main

import (
	"fmt"
	"log"
	"net/http"
	"net/url"

	"github.com/PuerkitoBio/goquery"
	"github.com/hatstand/shinywaffle/metar"
)

const (
	baseURL = "https://www.ogimet.com/display_metars2.php"
	ICAO    = "EGLC"
)

func main() {
	v := url.Values{}
	v.Set("lang", "en")
	v.Set("lugar", ICAO)
	v.Set("tipo", "SA")
	v.Set("ord", "REV")
	v.Set("nil", "NO")
	v.Set("fmt", "txt")
	v.Set("ano", "2017")
	v.Set("mes", "08")
	v.Set("day", "10")
	v.Set("hora", "00")
	v.Set("anof", "2017")
	v.Set("mesf", "08")
	v.Set("dayf", "31")
	v.Set("horaf", "23")
	v.Set("minf", "59")
	v.Set("send", "send")

	url, _ := url.Parse(baseURL)
	url.RawQuery = v.Encode()

	fmt.Printf("URL: %v\n", url)

	resp, err := http.Get(url.String())
	if err != nil {
		log.Fatalf("Failed to fetch METAR data: %v", err)
	}
	defer resp.Body.Close()

	doc, _ := goquery.NewDocumentFromReader(resp.Body)
	raw := doc.Find("pre").Text()
	METARs, err := metar.ParseMETARs(raw)
	if err != nil {
		log.Fatalf("Failed to parse METARs: %v", err)
	}
	for _, m := range METARs {
		fmt.Printf("Parsed: %v\n", m)
	}
}
<commit_msg>Generate chart of METAR data.<commit_after>package main

import (
	"bufio"
	"log"
	"net/http"
	"net/url"
	"os"
	"time"

	"github.com/PuerkitoBio/goquery"
	"github.com/hatstand/shinywaffle/metar"
	"github.com/wcharczuk/go-chart"
)

const (
	baseURL = "https://www.ogimet.com/display_metars2.php"
	ICAO    = "EGLC"
)

func main() {
	v := url.Values{}
	v.Set("lang", "en")
	v.Set("lugar", ICAO)
	v.Set("tipo", "SA")
	v.Set("ord", "REV")
	v.Set("nil", "NO")
	v.Set("fmt", "txt")
	v.Set("ano", "2017")
	v.Set("mes", "08")
	v.Set("day", "10")
	v.Set("hora", "00")
	v.Set("anof", "2017")
	v.Set("mesf", "08")
	v.Set("dayf", "31")
	v.Set("horaf", "23")
	v.Set("minf", "59")
	v.Set("send", "send")

	url, _ := url.Parse(baseURL)
	url.RawQuery = v.Encode()

	resp, err := http.Get(url.String())
	if err != nil {
		log.Fatalf("Failed to fetch METAR data: %v", err)
	}
	defer resp.Body.Close()

	doc, _ := goquery.NewDocumentFromReader(resp.Body)
	raw := doc.Find("pre").Text()
	METARs, err := metar.ParseMETARs(raw)
	if err != nil {
		log.Fatalf("Failed to parse METARs: %v", err)
	}

	var x []time.Time
	var y []float64
	for _, m := range METARs {
		x = append(x, m.DateTime)
		y = append(y, float64(m.Temperature))
	}

	graph := chart.Chart{
		XAxis: chart.XAxis{
			Style: chart.Style{
				Show: true,
			},
		},
		YAxis: chart.YAxis{
			Style: chart.Style{
				Show: true,
			},
		},
		Series: []chart.Series{
			chart.TimeSeries{
				XValues: x,
				YValues: y,
			},
		},
	}

	w := bufio.NewWriter(os.Stdout)
	err = graph.Render(chart.PNG, w)
}
<|endoftext|>


<commit_before>package admin

import (
	"net/http"
	"net/http/httptest"
	"testing"
)

func TestAdminServerAccess(t *testing.T) {
	type test struct {
		uri  string
		code int
	}

	testAccess := func(access string, tests []test) {
		srv := &Server{Access: access}
		ts := httptest.NewServer(srv.handler())
		defer ts.Close()

		noRedirectClient := &http.Client{
			CheckRedirect: func(req *http.Request, via []*http.Request) error {
				return http.ErrUseLastResponse
			},
		}
		for _, tt := range tests {
			t.Run(access+tt.uri, func(t *testing.T) {
				resp, err := noRedirectClient.Get(ts.URL + tt.uri)
				if err != nil {
					t.Fatalf("got %v want nil", err)
				}
				if got, want := resp.StatusCode, tt.code; got != want {
					t.Fatalf("got code %d want %d", got, want)
				}
			})
		}
	}

	roTests := []test{
		{"/api/manual", 403},
		{"/api/config", 200},
		{"/api/routes", 200},
		{"/api/version", 200},
		{"/manual", 403},
		{"/routes", 200},
		{"/health", 200},
		{"/logo.svg", 200},
		{"/", 303},
	}

	rwTests := []test{
		{"/api/manual", 200},
		{"/api/config", 200},
		{"/api/routes", 200},
		{"/api/version", 200},
		{"/manual", 200},
		{"/routes", 200},
		{"/health", 200},
		{"/logo.svg", 200},
		{"/", 303},
	}

	testAccess("ro", roTests)
	testAccess("rw", rwTests)
}
<commit_msg>fix tests<commit_after>package admin

import (
	"net/http"
	"net/http/httptest"
	"testing"

	"github.com/fabiolb/fabio/config"
)

func TestAdminServerAccess(t *testing.T) {
	type test struct {
		uri  string
		code int
	}

	testAccess := func(access string, tests []test) {
		srv := &Server{
			Access: access,
			Cfg: &config.Config{
				Registry: config.Registry{
					Consul: config.Consul{
						KVPath: "/fabio/config",
					},
				},
			},
		}
		ts := httptest.NewServer(srv.handler())
		defer ts.Close()

		noRedirectClient := &http.Client{
			CheckRedirect: func(req *http.Request, via []*http.Request) error {
				return http.ErrUseLastResponse
			},
		}
		for _, tt := range tests {
			t.Run(access+tt.uri, func(t *testing.T) {
				resp, err := noRedirectClient.Get(ts.URL + tt.uri)
				if err != nil {
					t.Fatalf("got %v want nil", err)
				}
				if got, want := resp.StatusCode, tt.code; got != want {
					t.Fatalf("got code %d want %d", got, want)
				}
			})
		}
	}

	roTests := []test{
		{"/api/manual", 403},
		{"/api/paths", 403},
		{"/api/config", 200},
		{"/api/routes", 200},
		{"/api/version", 200},
		{"/manual", 403},
		{"/routes", 200},
		{"/health", 200},
		{"/logo.svg", 200},
		{"/", 303},
	}

	rwTests := []test{
		{"/api/manual", 200},
		{"/api/paths", 200},
		{"/api/config", 200},
		{"/api/routes", 200},
		{"/api/version", 200},
		{"/manual", 200},
		{"/routes", 200},
		{"/health", 200},
		{"/logo.svg", 200},
		{"/", 303},
	}

	testAccess("ro", roTests)
	testAccess("rw", rwTests)
}
<|endoftext|>