<commit_before>// Copyright 2015 The LUCI Authors.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//      http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package main

import (
	"context"
	"encoding/json"
	"errors"
	"fmt"
	"io"
	"log"
	"os"
	"path/filepath"
	"strings"
	"time"

	"github.com/maruel/subcommands"

	"go.chromium.org/luci/auth"
	"go.chromium.org/luci/client/archiver"
	"go.chromium.org/luci/client/isolate"
	"go.chromium.org/luci/common/data/text/units"
	"go.chromium.org/luci/common/isolated"
	"go.chromium.org/luci/common/isolatedclient"
)

func cmdBatchArchive(defaultAuthOpts auth.Options) *subcommands.Command {
	return &subcommands.Command{
		UsageLine: "batcharchive <options> file1 file2 ...",
		ShortDesc: "archives multiple isolated trees at once.",
		LongDesc: `Archives multiple isolated trees at once.

Using single command instead of multiple sequential invocations allows to cut
redundant work when isolated trees share common files (e.g. file hashes are
checked only once, their presence on the server is checked only once, and
so on).

Takes a list of paths to *.isolated.gen.json files that describe what trees to
isolate. Format of files is:
{
  "version": 1,
  "dir": <absolute path to a directory all other paths are relative to>,
  "args": [list of command line arguments for single 'archive' command]
}`,
		CommandRun: func() subcommands.CommandRun {
			c := batchArchiveRun{}
			c.commonServerFlags.Init(defaultAuthOpts)
			c.loggingFlags.Init(&c.Flags)
			c.Flags.StringVar(&c.dumpJSON, "dump-json", "", "Write isolated digests of archived trees to this file as JSON")
			c.Flags.BoolVar(&c.expArchive, "exparchive", true, "IGNORED (deprecated) Whether to use the new exparchive implementation, which tars small files before uploading them.")
			c.Flags.IntVar(&c.maxConcurrentChecks, "max-concurrent-checks", 1, "The maxiumum number of in-flight check requests.")
			c.Flags.IntVar(&c.maxConcurrentUploads, "max-concurrent-uploads", 8, "The maximum number of in-flight uploads.")
			return &c
		},
	}
}

type batchArchiveRun struct {
	commonServerFlags
	loggingFlags         loggingFlags
	dumpJSON             string
	expArchive           bool // deprecated
	maxConcurrentChecks  int
	maxConcurrentUploads int
}

func (c *batchArchiveRun) Parse(a subcommands.Application, args []string) error {
	if err := c.commonServerFlags.Parse(); err != nil {
		return err
	}
	if len(args) == 0 {
		return errors.New("at least one isolate file required")
	}
	return nil
}

func parseArchiveCMD(args []string, cwd string) (*isolate.ArchiveOptions, error) {
	// Python isolate allows form "--XXXX-variable key value".
	// Golang flag pkg doesn't consider value to be part of --XXXX-variable flag.
	// Therefore, we convert all such "--XXXX-variable key value" to
	// "--XXXX-variable key --XXXX-variable value" form.
	// Note, that key doesn't have "=" in it in either case, but value might.
	// TODO(tandrii): eventually, we want to retire this hack.
	args = convertPyToGoArchiveCMDArgs(args)
	base := subcommands.CommandRunBase{}
	i := isolateFlags{}
	i.Init(&base.Flags)
	if err := base.GetFlags().Parse(args); err != nil {
		return nil, err
	}
	if err := i.Parse(cwd, RequireIsolatedFile); err != nil {
		return nil, err
	}
	if base.GetFlags().NArg() > 0 {
		return nil, fmt.Errorf("no positional arguments expected")
	}
	i.PostProcess(cwd)
	return &i.ArchiveOptions, nil
}

// convertPyToGoArchiveCMDArgs converts kv-args from old python isolate into go variants.
// Essentially converts "--X key value" into "--X key=value".
func convertPyToGoArchiveCMDArgs(args []string) []string {
	kvars := map[string]bool{
		"--path-variable": true, "--config-variable": true, "--extra-variable": true}
	var newArgs []string
	for i := 0; i < len(args); {
		newArgs = append(newArgs, args[i])
		kvar := args[i]
		i++
		if !kvars[kvar] {
			continue
		}
		if i >= len(args) {
			// Ignore unexpected behaviour, it'll be caught by flags.Parse() .
			break
		}
		appendArg := args[i]
		i++
		if !strings.Contains(appendArg, "=") && i < len(args) {
			// appendArg is key, and args[i] is value .
			appendArg = fmt.Sprintf("%s=%s", appendArg, args[i])
			i++
		}
		newArgs = append(newArgs, appendArg)
	}
	return newArgs
}

func (c *batchArchiveRun) main(a subcommands.Application, args []string) error {
	start := time.Now()
	ctx := c.defaultFlags.MakeLoggingContext(os.Stderr)

	authClient, err := c.createAuthClient()
	if err != nil {
		return err
	}
	client := isolatedclient.New(nil, authClient, c.isolatedFlags.ServerURL, c.isolatedFlags.Namespace, nil, nil)

	al := archiveLogger{
		start: start,
		quiet: c.defaultFlags.Quiet,
	}
	return batchArchive(ctx, client, al, c.dumpJSON, c.maxConcurrentChecks, c.maxConcurrentUploads, args)
}

// batchArchive archives a series of isolates specified by genJSONPaths.
func batchArchive(ctx context.Context, client *isolatedclient.Client, al archiveLogger, dumpJSONPath string, concurrentChecks, concurrentUploads int, genJSONPaths []string) error {
	// Set up a checker and uploader. We limit the uploader to one concurrent
	// upload, since the uploads are all coming from disk (with the exception of
	// the isolated JSON itself) and we only want a single goroutine reading from
	// disk at once.
	checker := archiver.NewChecker(ctx, client, concurrentChecks)
	uploader := archiver.NewUploader(ctx, client, concurrentUploads)
	a := archiver.NewTarringArchiver(checker, uploader)

	var errArchive error
	var isolSummaries []archiver.IsolatedSummary
	for _, genJSONPath := range genJSONPaths {
		opts, err := processGenJSON(genJSONPath)
		if err != nil {
			return err
		}
		// Parse the incoming isolate file.
		deps, rootDir, isol, err := isolate.ProcessIsolate(opts)
		if err != nil {
			return fmt.Errorf("isolate %s: failed to process: %v", opts.Isolate, err)
		}
		log.Printf("Isolate %s referenced %d deps", opts.Isolate, len(deps))

		isolSummary, err := a.Archive(deps, rootDir, isol, opts.Blacklist, opts.Isolated)
		if err != nil && errArchive == nil {
			errArchive = fmt.Errorf("isolate %s: %v", opts.Isolate, err)
		} else {
			printSummary(al, isolSummary)
			isolSummaries = append(isolSummaries, isolSummary)
		}
	}
	if errArchive != nil {
		return errArchive
	}
	// Make sure that all pending items have been checked.
	if err := checker.Close(); err != nil {
		return err
	}

	// Make sure that all the uploads have completed successfully.
	if err := uploader.Close(); err != nil {
		return err
	}

	if err := dumpSummaryJSON(dumpJSONPath, isolSummaries...); err != nil {
		return err
	}

	al.LogSummary(ctx, int64(checker.Hit.Count()), int64(checker.Miss.Count()), units.Size(checker.Hit.Bytes()), units.Size(checker.Miss.Bytes()), digests(isolSummaries))
	return nil
}

// digests extracts the digests from the supplied IsolatedSummarys.
func digests(summaries []archiver.IsolatedSummary) []string {
	var result []string
	for _, summary := range summaries {
		result = append(result, string(summary.Digest))
	}
	return result
}

// processGenJSON validates a genJSON file and returns the contents.
func processGenJSON(genJSONPath string) (*isolate.ArchiveOptions, error) {
	f, err := os.Open(genJSONPath)
	if err != nil {
		return nil, fmt.Errorf("opening %s: %s", genJSONPath, err)
	}
	defer f.Close()

	opts, err := processGenJSONData(f)
	if err != nil {
		return nil, fmt.Errorf("processing %s: %s", genJSONPath, err)
	}
	return opts, nil
}

// processGenJSONData performs the function of processGenJSON, but operates on an io.Reader.
func processGenJSONData(r io.Reader) (*isolate.ArchiveOptions, error) {
	data := &struct {
		Args    []string
		Dir     string
		Version int
	}{}
	if err := json.NewDecoder(r).Decode(data); err != nil {
		return nil, fmt.Errorf("failed to decode: %s", err)
	}

	if data.Version != isolate.IsolatedGenJSONVersion {
		return nil, fmt.Errorf("invalid version %d", data.Version)
	}

	if fileInfo, err := os.Stat(data.Dir); err != nil || !fileInfo.IsDir() {
		return nil, fmt.Errorf("invalid dir %s", data.Dir)
	}

	opts, err := parseArchiveCMD(data.Args, data.Dir)
	if err != nil {
		return nil, fmt.Errorf("invalid archive command: %s", err)
	}
	return opts, nil
}

// strippedIsolatedName returns the base name of an isolated path, with the extension (if any) removed.
func strippedIsolatedName(isolated string) string {
	name := filepath.Base(isolated)
	// Strip the extension if there is one.
	if dotIndex := strings.LastIndex(name, "."); dotIndex != -1 {
		return name[0:dotIndex]
	}
	return name
}

func writeJSONDigestFile(filePath string, data map[string]isolated.HexDigest) error {
	digestBytes, err := json.MarshalIndent(data, "", "  ")
	if err != nil {
		return fmt.Errorf("encoding digest JSON: %s", err)
	}
	return writeFile(filePath, digestBytes)
}

// writeFile writes data to filePath. File permission is set to user only.
func writeFile(filePath string, data []byte) error {
	f, err := os.OpenFile(filePath, os.O_RDWR|os.O_CREATE|os.O_TRUNC, 0600)
	if err != nil {
		return fmt.Errorf("opening %s: %s", filePath, err)
	}
	// NOTE: We don't defer f.Close here, because it may return an error.

	_, writeErr := f.Write(data)
	closeErr := f.Close()
	if writeErr != nil {
		return fmt.Errorf("writing %s: %s", filePath, writeErr)
	} else if closeErr != nil {
		return fmt.Errorf("closing %s: %s", filePath, closeErr)
	}
	return nil
}

func (c *batchArchiveRun) Run(a subcommands.Application, args []string, _ subcommands.Env) int {
	if err := c.Parse(a, args); err != nil {
		fmt.Fprintf(a.GetErr(), "%s: %s\n", a.GetName(), err)
		return 1
	}
	cl, err := c.defaultFlags.StartTracing()
	if err != nil {
		fmt.Fprintf(a.GetErr(), "%s: %s\n", a.GetName(), err)
		return 1
	}
	defer cl.Close()
	if err := c.main(a, args); err != nil {
		fmt.Fprintf(a.GetErr(), "%s: %s\n", a.GetName(), err)
		return 1
	}
	return 0
}
<commit_msg>Add support for an explicit blacklist to batch_archive.<commit_after>// Copyright 2015 The LUCI Authors.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//      http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package main

import (
	"context"
	"encoding/json"
	"errors"
	"fmt"
	"io"
	"log"
	"os"
	"path/filepath"
	"strings"
	"time"

	"github.com/maruel/subcommands"

	"go.chromium.org/luci/auth"
	"go.chromium.org/luci/client/archiver"
	"go.chromium.org/luci/client/internal/common"
	"go.chromium.org/luci/client/isolate"
	"go.chromium.org/luci/common/data/text/units"
	"go.chromium.org/luci/common/isolated"
	"go.chromium.org/luci/common/isolatedclient"
)

func cmdBatchArchive(defaultAuthOpts auth.Options) *subcommands.Command {
	return &subcommands.Command{
		UsageLine: "batcharchive <options> file1 file2 ...",
		ShortDesc: "archives multiple isolated trees at once.",
		LongDesc: `Archives multiple isolated trees at once.

Using single command instead of multiple sequential invocations allows to cut
redundant work when isolated trees share common files (e.g. file hashes are
checked only once, their presence on the server is checked only once, and
so on).

Takes a list of paths to *.isolated.gen.json files that describe what trees to
isolate. Format of files is:
{
  "version": 1,
  "dir": <absolute path to a directory all other paths are relative to>,
  "args": [list of command line arguments for single 'archive' command]
}`,
		CommandRun: func() subcommands.CommandRun {
			c := batchArchiveRun{}
			c.commonServerFlags.Init(defaultAuthOpts)
			c.loggingFlags.Init(&c.Flags)
			c.Flags.Var(&c.blacklist, "blacklist", "List of globs to use as blacklist filter when uploading directories")
			c.Flags.StringVar(&c.dumpJSON, "dump-json", "", "Write isolated digests of archived trees to this file as JSON")
			c.Flags.BoolVar(&c.expArchive, "exparchive", true, "IGNORED (deprecated) Whether to use the new exparchive implementation, which tars small files before uploading them.")
			c.Flags.IntVar(&c.maxConcurrentChecks, "max-concurrent-checks", 1, "The maxiumum number of in-flight check requests.")
			c.Flags.IntVar(&c.maxConcurrentUploads, "max-concurrent-uploads", 8, "The maximum number of in-flight uploads.")
			return &c
		},
	}
}

type batchArchiveRun struct {
	commonServerFlags
	loggingFlags         loggingFlags
	dumpJSON             string
	expArchive           bool // deprecated
	maxConcurrentChecks  int
	maxConcurrentUploads int
	// Blacklist is a list of filename regexes describing which files to
	// ignore.
	blacklist common.Strings
}

func (c *batchArchiveRun) Parse(a subcommands.Application, args []string) error {
	if err := c.commonServerFlags.Parse(); err != nil {
		return err
	}
	if len(args) == 0 {
		return errors.New("at least one isolate file required")
	}
	return nil
}

func parseArchiveCMD(args []string, cwd string) (*isolate.ArchiveOptions, error) {
	// Python isolate allows form "--XXXX-variable key value".
	// Golang flag pkg doesn't consider value to be part of --XXXX-variable flag.
	// Therefore, we convert all such "--XXXX-variable key value" to
	// "--XXXX-variable key --XXXX-variable value" form.
	// Note, that key doesn't have "=" in it in either case, but value might.
	// TODO(tandrii): eventually, we want to retire this hack.
	args = convertPyToGoArchiveCMDArgs(args)
	base := subcommands.CommandRunBase{}
	i := isolateFlags{}
	i.Init(&base.Flags)
	if err := base.GetFlags().Parse(args); err != nil {
		return nil, err
	}
	if err := i.Parse(cwd, RequireIsolatedFile); err != nil {
		return nil, err
	}
	if base.GetFlags().NArg() > 0 {
		return nil, fmt.Errorf("no positional arguments expected")
	}
	i.PostProcess(cwd)
	return &i.ArchiveOptions, nil
}

// convertPyToGoArchiveCMDArgs converts kv-args from old python isolate into go variants.
// Essentially converts "--X key value" into "--X key=value".
func convertPyToGoArchiveCMDArgs(args []string) []string {
	kvars := map[string]bool{
		"--path-variable": true, "--config-variable": true, "--extra-variable": true}
	var newArgs []string
	for i := 0; i < len(args); {
		newArgs = append(newArgs, args[i])
		kvar := args[i]
		i++
		if !kvars[kvar] {
			continue
		}
		if i >= len(args) {
			// Ignore unexpected behaviour, it'll be caught by flags.Parse() .
			break
		}
		appendArg := args[i]
		i++
		if !strings.Contains(appendArg, "=") && i < len(args) {
			// appendArg is key, and args[i] is value .
			appendArg = fmt.Sprintf("%s=%s", appendArg, args[i])
			i++
		}
		newArgs = append(newArgs, appendArg)
	}
	return newArgs
}

func (c *batchArchiveRun) main(a subcommands.Application, args []string) error {
	start := time.Now()
	ctx := c.defaultFlags.MakeLoggingContext(os.Stderr)

	authClient, err := c.createAuthClient()
	if err != nil {
		return err
	}
	client := isolatedclient.New(nil, authClient, c.isolatedFlags.ServerURL, c.isolatedFlags.Namespace, nil, nil)

	al := archiveLogger{
		start: start,
		quiet: c.defaultFlags.Quiet,
	}
	blacklistStrings := []string(c.blacklist)
	return batchArchive(ctx, client, al, c.dumpJSON, c.maxConcurrentChecks, c.maxConcurrentUploads, args, blacklistStrings)
}

// batchArchive archives a series of isolates specified by genJSONPaths.
func batchArchive(ctx context.Context, client *isolatedclient.Client, al archiveLogger, dumpJSONPath string, concurrentChecks, concurrentUploads int, genJSONPaths []string, blacklistStrings []string) error {
	// Set up a checker and uploader. We limit the uploader to one concurrent
	// upload, since the uploads are all coming from disk (with the exception of
	// the isolated JSON itself) and we only want a single goroutine reading from
	// disk at once.
	checker := archiver.NewChecker(ctx, client, concurrentChecks)
	uploader := archiver.NewUploader(ctx, client, concurrentUploads)
	a := archiver.NewTarringArchiver(checker, uploader)

	var errArchive error
	var isolSummaries []archiver.IsolatedSummary
	for _, genJSONPath := range genJSONPaths {
		opts, err := processGenJSON(genJSONPath)
		if err != nil {
			return err
		}
		// Parse the incoming isolate file.
		deps, rootDir, isol, err := isolate.ProcessIsolate(opts)
		if err != nil {
			return fmt.Errorf("isolate %s: failed to process: %v", opts.Isolate, err)
		}
		log.Printf("Isolate %s referenced %d deps", opts.Isolate, len(deps))

		// Use the explicit blacklist if it's provided.
		if len(blacklistStrings) > 0 {
			opts.Blacklist = blacklistStrings
		}
		isolSummary, err := a.Archive(deps, rootDir, isol, opts.Blacklist, opts.Isolated)
		if err != nil && errArchive == nil {
			errArchive = fmt.Errorf("isolate %s: %v", opts.Isolate, err)
		} else {
			printSummary(al, isolSummary)
			isolSummaries = append(isolSummaries, isolSummary)
		}
	}
	if errArchive != nil {
		return errArchive
	}
	// Make sure that all pending items have been checked.
	if err := checker.Close(); err != nil {
		return err
	}

	// Make sure that all the uploads have completed successfully.
	if err := uploader.Close(); err != nil {
		return err
	}

	if err := dumpSummaryJSON(dumpJSONPath, isolSummaries...); err != nil {
		return err
	}

	al.LogSummary(ctx, int64(checker.Hit.Count()), int64(checker.Miss.Count()), units.Size(checker.Hit.Bytes()), units.Size(checker.Miss.Bytes()), digests(isolSummaries))
	return nil
}

// digests extracts the digests from the supplied IsolatedSummarys.
func digests(summaries []archiver.IsolatedSummary) []string {
	var result []string
	for _, summary := range summaries {
		result = append(result, string(summary.Digest))
	}
	return result
}

// processGenJSON validates a genJSON file and returns the contents.
func processGenJSON(genJSONPath string) (*isolate.ArchiveOptions, error) {
	f, err := os.Open(genJSONPath)
	if err != nil {
		return nil, fmt.Errorf("opening %s: %s", genJSONPath, err)
	}
	defer f.Close()

	opts, err := processGenJSONData(f)
	if err != nil {
		return nil, fmt.Errorf("processing %s: %s", genJSONPath, err)
	}
	return opts, nil
}

// processGenJSONData performs the function of processGenJSON, but operates on an io.Reader.
func processGenJSONData(r io.Reader) (*isolate.ArchiveOptions, error) {
	data := &struct {
		Args    []string
		Dir     string
		Version int
	}{}
	if err := json.NewDecoder(r).Decode(data); err != nil {
		return nil, fmt.Errorf("failed to decode: %s", err)
	}

	if data.Version != isolate.IsolatedGenJSONVersion {
		return nil, fmt.Errorf("invalid version %d", data.Version)
	}

	if fileInfo, err := os.Stat(data.Dir); err != nil || !fileInfo.IsDir() {
		return nil, fmt.Errorf("invalid dir %s", data.Dir)
	}

	opts, err := parseArchiveCMD(data.Args, data.Dir)
	if err != nil {
		return nil, fmt.Errorf("invalid archive command: %s", err)
	}
	return opts, nil
}

// strippedIsolatedName returns the base name of an isolated path, with the extension (if any) removed.
func strippedIsolatedName(isolated string) string {
	name := filepath.Base(isolated)
	// Strip the extension if there is one.
	if dotIndex := strings.LastIndex(name, "."); dotIndex != -1 {
		return name[0:dotIndex]
	}
	return name
}

func writeJSONDigestFile(filePath string, data map[string]isolated.HexDigest) error {
	digestBytes, err := json.MarshalIndent(data, "", "  ")
	if err != nil {
		return fmt.Errorf("encoding digest JSON: %s", err)
	}
	return writeFile(filePath, digestBytes)
}

// writeFile writes data to filePath. File permission is set to user only.
func writeFile(filePath string, data []byte) error {
	f, err := os.OpenFile(filePath, os.O_RDWR|os.O_CREATE|os.O_TRUNC, 0600)
	if err != nil {
		return fmt.Errorf("opening %s: %s", filePath, err)
	}
	// NOTE: We don't defer f.Close here, because it may return an error.

	_, writeErr := f.Write(data)
	closeErr := f.Close()
	if writeErr != nil {
		return fmt.Errorf("writing %s: %s", filePath, writeErr)
	} else if closeErr != nil {
		return fmt.Errorf("closing %s: %s", filePath, closeErr)
	}
	return nil
}

func (c *batchArchiveRun) Run(a subcommands.Application, args []string, _ subcommands.Env) int {
	if err := c.Parse(a, args); err != nil {
		fmt.Fprintf(a.GetErr(), "%s: %s\n", a.GetName(), err)
		return 1
	}
	cl, err := c.defaultFlags.StartTracing()
	if err != nil {
		fmt.Fprintf(a.GetErr(), "%s: %s\n", a.GetName(), err)
		return 1
	}
	defer cl.Close()
	if err := c.main(a, args); err != nil {
		fmt.Fprintf(a.GetErr(), "%s: %s\n", a.GetName(), err)
		return 1
	}
	return 0
}
<|endoftext|>


<commit_before>// +build fvtests

// Copyright (c) 2017 Tigera, Inc. All rights reserved.

// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package fv_test

// The tests in this file test Felix's and Typha's health endpoints, http://.../liveness and
// http://.../readiness.
//
// Felix should report itself as live, so long as its calc_graph and int_dataplane loops have not
// died or hung; and as ready, so long as it has completed its initial dataplane programming, is
// connected to its datastore, and is not doing a resync (either the initial resync, or a subsequent
// one).
//
// Typha should report itself as live, so long as its Felix-serving loop has not died or hung; and
// as ready, so long as it is connected to its datastore, and is not doing a resync (either the
// initial resync, or a subsequent one).
//
// (These reports are useful because k8s can detect and handle a pod that is consistently non-live,
// by killing and restarting it; and can adjust for a pod that is non-ready, by (a) not routing
// Service traffic to it (when that pod is otherwise one of the possible backends for a Service),
// and (b) not moving on to the next pod, in a rolling upgrade process, until the just-upgraded pod
// says that it is ready.)

import (
	"context"
	"fmt"
	"math/rand"
	"net/http"

	. "github.com/onsi/ginkgo"
	. "github.com/onsi/gomega"
	"github.com/onsi/gomega/types"
	log "github.com/sirupsen/logrus"
	"k8s.io/api/core/v1"
	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"

	"github.com/projectcalico/felix/fv/containers"
	"github.com/projectcalico/felix/fv/k8sapiserver"
	"github.com/projectcalico/felix/fv/utils"
	"github.com/projectcalico/libcalico-go/lib/health"
	"github.com/projectcalico/libcalico-go/lib/options"
)

var _ = Describe("health tests", func() {

	var k8sAPIServer *k8sapiserver.Server

	BeforeEach(func() {
		k8sAPIServer = k8sapiserver.SetUp()
	})

	var felixContainer *containers.Container
	var felixReady, felixLiveness func() int

	// describeCommonFelixTests creates specs for Felix tests that are common between the
	// two scenarios below (with and without Typha).
	describeCommonFelixTests := func() {
		var podsToCleanUp []string

		Describe("with normal Felix startup", func() {

			It("should become ready and stay ready", func() {
				Eventually(felixReady, "5s", "100ms").Should(BeGood())
				Consistently(felixReady, "10s", "1s").Should(BeGood())
			})

			It("should become live and stay live", func() {
				Eventually(felixLiveness, "5s", "100ms").Should(BeGood())
				Consistently(felixLiveness, "10s", "1s").Should(BeGood())
			})
		})

		createLocalPod := func() {
			testPodName := fmt.Sprintf("test-pod-%x", rand.Uint32())
			pod := &v1.Pod{
				ObjectMeta: metav1.ObjectMeta{Name: testPodName},
				Spec: v1.PodSpec{Containers: []v1.Container{{
					Name:  fmt.Sprintf("container-foo"),
					Image: "ignore",
				}},
					NodeName: felixContainer.Hostname,
				},
				Status: v1.PodStatus{
					Phase: v1.PodRunning,
					Conditions: []v1.PodCondition{{
						Type:   v1.PodScheduled,
						Status: v1.ConditionTrue,
					}},
					PodIP: "10.0.0.1",
				},
			}
			var err error
			pod, err = k8sAPIServer.Client.CoreV1().Pods("default").Create(pod)
			Expect(err).NotTo(HaveOccurred())
			pod.Status.PodIP = "10.0.0.1"
			_, err = k8sAPIServer.Client.CoreV1().Pods("default").UpdateStatus(pod)
			Expect(err).NotTo(HaveOccurred())
			podsToCleanUp = append(podsToCleanUp, testPodName)
		}

		AfterEach(func() {
			for _, name := range podsToCleanUp {
				err := k8sAPIServer.Client.CoreV1().Pods("default").Delete(name, &metav1.DeleteOptions{})
				Expect(err).NotTo(HaveOccurred())
			}
			podsToCleanUp = nil
		})

		Describe("after removing iptables-restore", func() {
			BeforeEach(func() {
				// Wait until felix gets into steady state.
				Eventually(felixReady, "5s", "100ms").Should(BeGood())

				// Then remove iptables-restore.
				err := felixContainer.ExecMayFail("rm", "/sbin/iptables-restore")
				Expect(err).NotTo(HaveOccurred())

				// Make an update that will force felix to run iptables-restore.
				createLocalPod()
			})

			It("should become unready, then die", func() {
				Eventually(felixReady, "120s", "10s").ShouldNot(BeGood())
				Eventually(felixContainer.Stopped, "5s").Should(BeTrue())
			})
		})

		Describe("after replacing iptables with a slow version", func() {
			BeforeEach(func() {
				// Wait until felix gets into steady state.
				Eventually(felixReady, "5s", "100ms").Should(BeGood())

				// Then replace iptables-restore with the bad version:

				// We need to delete the file first since it's a symlink and "docker cp"
				// follows the link and overwrites the wrong file if we don't.
				err := felixContainer.ExecMayFail("rm", "/sbin/iptables-restore")
				Expect(err).NotTo(HaveOccurred())

				// Copy in the nobbled iptables command.
				err = felixContainer.CopyFileIntoContainer("slow-iptables-restore",
					"/sbin/iptables-restore")
				Expect(err).NotTo(HaveOccurred())

				// Make it executable.
				err = felixContainer.ExecMayFail("chmod", "+x", "/sbin/iptables-restore")
				Expect(err).NotTo(HaveOccurred())

				// Make an update that will force felix to run iptables-restore.
				createLocalPod()
			})

			It("should detect dataplane pause and become non-ready", func() {
				Eventually(felixReady, "120s", "10s").ShouldNot(BeGood())
			})
		})
	}

	var typhaContainer *containers.Container
	var typhaReady, typhaLiveness func() int

	startTypha := func(endpoint string) {
		typhaContainer = containers.Run("typha",
			containers.RunOpts{AutoRemove: true},
			"--privileged",
			"-e", "CALICO_DATASTORE_TYPE=kubernetes",
			"-e", "TYPHA_HEALTHENABLED=true",
			"-e", "TYPHA_LOGSEVERITYSCREEN=info",
			"-e", "TYPHA_DATASTORETYPE=kubernetes",
			"-e", "TYPHA_PROMETHEUSMETRICSENABLED=true",
			"-e", "TYPHA_USAGEREPORTINGENABLED=false",
			"-e", "TYPHA_DEBUGMEMORYPROFILEPATH=\"heap-<timestamp>\"",
			"-e", "K8S_API_ENDPOINT="+endpoint,
			"-e", "K8S_INSECURE_SKIP_TLS_VERIFY=true",
			"-v", k8sAPIServer.CertFileName+":/tmp/apiserver.crt",
			utils.Config.TyphaImage,
			"calico-typha")
		Expect(typhaContainer).NotTo(BeNil())
		typhaReady = getHealthStatus(typhaContainer.IP, "9098", "readiness")
		typhaLiveness = getHealthStatus(typhaContainer.IP, "9098", "liveness")
	}

	startFelix := func(typhaAddr string, calcGraphHangTime string, dataplaneHangTime string) {
		felixContainer = containers.Run("felix",
			containers.RunOpts{AutoRemove: true},
			"--privileged",
			"-e", "CALICO_DATASTORE_TYPE=kubernetes",
			"-e", "FELIX_IPV6SUPPORT=false",
			"-e", "FELIX_HEALTHENABLED=true",
			"-e", "FELIX_LOGSEVERITYSCREEN=info",
			"-e", "FELIX_DATASTORETYPE=kubernetes",
			"-e", "FELIX_PROMETHEUSMETRICSENABLED=true",
			"-e", "FELIX_USAGEREPORTINGENABLED=false",
			"-e", "FELIX_DEBUGMEMORYPROFILEPATH=\"heap-<timestamp>\"",
			"-e", "FELIX_DebugSimulateCalcGraphHangAfter="+calcGraphHangTime,
			"-e", "FELIX_DebugSimulateDataplaneHangAfter="+dataplaneHangTime,
			"-e", "K8S_API_ENDPOINT="+k8sAPIServer.Endpoint,
			"-e", "K8S_INSECURE_SKIP_TLS_VERIFY=true",
			"-e", "FELIX_TYPHAADDR="+typhaAddr,
			"-v", k8sAPIServer.CertFileName+":/tmp/apiserver.crt",
			"calico/felix:latest",
		)
		Expect(felixContainer).NotTo(BeNil())

		felixReady = getHealthStatus(felixContainer.IP, "9099", "readiness")
		felixLiveness = getHealthStatus(felixContainer.IP, "9099", "liveness")
	}

	Describe("with Felix running (no Typha)", func() {
		BeforeEach(func() {
			startFelix("", "", "")
		})

		AfterEach(func() {
			felixContainer.Stop()
		})

		describeCommonFelixTests()
	})

	Describe("with Felix (no Typha) and Felix calc graph set to hang", func() {
		BeforeEach(func() {
			startFelix("", "5", "")
		})

		AfterEach(func() {
			felixContainer.Stop()
		})

		It("should report live initially, then become non-live", func() {
			Eventually(felixLiveness, "10s", "100ms").Should(BeGood())
			Eventually(felixLiveness, "30s", "100ms").Should(BeBad())
			Consistently(felixLiveness, "10s", "100ms").Should(BeBad())
		})
	})

	Describe("with Felix (no Typha) and Felix dataplane set to hang", func() {
		BeforeEach(func() {
			startFelix("", "", "5")
		})

		AfterEach(func() {
			felixContainer.Stop()
		})

		It("should report live initially, then become non-live", func() {
			Eventually(felixLiveness, "10s", "100ms").Should(BeGood())
			Eventually(felixLiveness, "30s", "100ms").Should(BeBad())
			Consistently(felixLiveness, "10s", "100ms").Should(BeBad())
		})
	})

	Describe("with Felix and Typha running", func() {
		BeforeEach(func() {
			startTypha(k8sAPIServer.Endpoint)
			startFelix(typhaContainer.IP+":5473", "", "")
		})

		AfterEach(func() {
			felixContainer.Stop()
			typhaContainer.Stop()
		})

		describeCommonFelixTests()

		It("typha should report ready", func() {
			Eventually(typhaReady, "5s", "100ms").Should(BeGood())
			Consistently(typhaReady, "10s", "1s").Should(BeGood())
		})

		It("typha should report live", func() {
			Eventually(typhaLiveness, "5s", "100ms").Should(BeGood())
			Consistently(typhaLiveness, "10s", "1s").Should(BeGood())
		})
	})

	Describe("with typha connected to bad API endpoint", func() {
		BeforeEach(func() {
			startTypha(k8sAPIServer.BadEndpoint)
		})

		AfterEach(func() {
			typhaContainer.Stop()
		})

		It("typha should not report ready", func() {
			Consistently(typhaReady, "10s", "1s").ShouldNot(BeGood())
		})

		It("typha should not report live", func() {
			Consistently(typhaLiveness(), "10s", "1s").ShouldNot(BeGood())
		})
	})

	Describe("with datastore not ready", func() {
		BeforeEach(func() {
			info, err := k8sAPIServer.CalicoClient.ClusterInformation().Get(
				context.Background(),
				"default",
				options.GetOptions{},
			)
			Expect(err).NotTo(HaveOccurred())
			log.Infof("info = %#v", info)
			notReady := false
			info.Spec.DatastoreReady = &notReady
			_, err = k8sAPIServer.CalicoClient.ClusterInformation().Update(
				context.Background(),
				info,
				options.SetOptions{},
			)
			Expect(err).NotTo(HaveOccurred())
			startFelix("", "", "")
		})

		AfterEach(func() {
			felixContainer.Stop()
		})

		It("felix should not report ready", func() {
			Consistently(felixReady, "10s", "1s").ShouldNot(BeGood())
		})

		It("felix should report live", func() {
			Eventually(felixLiveness, "5s", "100ms").Should(BeGood())
			Consistently(felixLiveness, "10s", "1s").Should(BeGood())
		})
	})
})

const statusErr = -1

func getHealthStatus(ip, port, endpoint string) func() int {
	return func() int {
		resp, err := http.Get("http://" + ip + ":" + port + "/" + endpoint)
		if err != nil {
			log.WithError(err).WithField("resp", resp).Warn("HTTP GET failed")
			return statusErr
		}
		defer resp.Body.Close()
		log.WithField("resp", resp).Info("Health response")
		return resp.StatusCode
	}
}

func BeErr() types.GomegaMatcher {
	return BeNumerically("==", statusErr)
}

func BeBad() types.GomegaMatcher {
	return BeNumerically("==", health.StatusBad)
}

func BeGood() types.GomegaMatcher {
	return BeNumerically("==", health.StatusGood)
}
<commit_msg>Fix up FVs.<commit_after>// +build fvtests

// Copyright (c) 2017 Tigera, Inc. All rights reserved.

// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package fv_test

// The tests in this file test Felix's and Typha's health endpoints, http://.../liveness and
// http://.../readiness.
//
// Felix should report itself as live, so long as its calc_graph and int_dataplane loops have not
// died or hung; and as ready, so long as it has completed its initial dataplane programming, is
// connected to its datastore, and is not doing a resync (either the initial resync, or a subsequent
// one).
//
// Typha should report itself as live, so long as its Felix-serving loop has not died or hung; and
// as ready, so long as it is connected to its datastore, and is not doing a resync (either the
// initial resync, or a subsequent one).
//
// (These reports are useful because k8s can detect and handle a pod that is consistently non-live,
// by killing and restarting it; and can adjust for a pod that is non-ready, by (a) not routing
// Service traffic to it (when that pod is otherwise one of the possible backends for a Service),
// and (b) not moving on to the next pod, in a rolling upgrade process, until the just-upgraded pod
// says that it is ready.)

import (
	"context"
	"fmt"
	"math/rand"
	"net/http"

	. "github.com/onsi/ginkgo"
	. "github.com/onsi/gomega"
	"github.com/onsi/gomega/types"
	log "github.com/sirupsen/logrus"
	"k8s.io/api/core/v1"
	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"

	"time"

	"github.com/projectcalico/felix/fv/containers"
	"github.com/projectcalico/felix/fv/k8sapiserver"
	"github.com/projectcalico/felix/fv/utils"
	"github.com/projectcalico/libcalico-go/lib/health"
	"github.com/projectcalico/libcalico-go/lib/options"
)

var _ = Describe("health tests", func() {

	var k8sAPIServer *k8sapiserver.Server

	BeforeEach(func() {
		k8sAPIServer = k8sapiserver.SetUp()
	})

	JustBeforeEach(func() {
		// Felix can now flap ready/non-ready while loading its config.  Delay until that
		// is done.
		time.Sleep(1 * time.Second)
	})

	var felixContainer *containers.Container
	var felixReady, felixLiveness func() int

	// describeCommonFelixTests creates specs for Felix tests that are common between the
	// two scenarios below (with and without Typha).
	describeCommonFelixTests := func() {
		var podsToCleanUp []string

		Describe("with normal Felix startup", func() {

			It("should become ready and stay ready", func() {
				Eventually(felixReady, "5s", "100ms").Should(BeGood())
				Consistently(felixReady, "10s", "1s").Should(BeGood())
			})

			It("should become live and stay live", func() {
				Eventually(felixLiveness, "5s", "100ms").Should(BeGood())
				Consistently(felixLiveness, "10s", "1s").Should(BeGood())
			})
		})

		createLocalPod := func() {
			testPodName := fmt.Sprintf("test-pod-%x", rand.Uint32())
			pod := &v1.Pod{
				ObjectMeta: metav1.ObjectMeta{Name: testPodName},
				Spec: v1.PodSpec{Containers: []v1.Container{{
					Name:  fmt.Sprintf("container-foo"),
					Image: "ignore",
				}},
					NodeName: felixContainer.Hostname,
				},
				Status: v1.PodStatus{
					Phase: v1.PodRunning,
					Conditions: []v1.PodCondition{{
						Type:   v1.PodScheduled,
						Status: v1.ConditionTrue,
					}},
					PodIP: "10.0.0.1",
				},
			}
			var err error
			pod, err = k8sAPIServer.Client.CoreV1().Pods("default").Create(pod)
			Expect(err).NotTo(HaveOccurred())
			pod.Status.PodIP = "10.0.0.1"
			_, err = k8sAPIServer.Client.CoreV1().Pods("default").UpdateStatus(pod)
			Expect(err).NotTo(HaveOccurred())
			podsToCleanUp = append(podsToCleanUp, testPodName)
		}

		AfterEach(func() {
			for _, name := range podsToCleanUp {
				err := k8sAPIServer.Client.CoreV1().Pods("default").Delete(name, &metav1.DeleteOptions{})
				Expect(err).NotTo(HaveOccurred())
			}
			podsToCleanUp = nil
		})

		Describe("after removing iptables-restore", func() {
			BeforeEach(func() {
				// Wait until felix gets into steady state.
				Eventually(felixReady, "5s", "100ms").Should(BeGood())

				// Then remove iptables-restore.
				err := felixContainer.ExecMayFail("rm", "/sbin/iptables-restore")
				Expect(err).NotTo(HaveOccurred())

				// Make an update that will force felix to run iptables-restore.
				createLocalPod()
			})

			It("should become unready, then die", func() {
				Eventually(felixReady, "120s", "10s").ShouldNot(BeGood())
				Eventually(felixContainer.Stopped, "5s").Should(BeTrue())
			})
		})

		Describe("after replacing iptables with a slow version", func() {
			BeforeEach(func() {
				// Wait until felix gets into steady state.
				Eventually(felixReady, "5s", "100ms").Should(BeGood())

				// Then replace iptables-restore with the bad version:

				// We need to delete the file first since it's a symlink and "docker cp"
				// follows the link and overwrites the wrong file if we don't.
				err := felixContainer.ExecMayFail("rm", "/sbin/iptables-restore")
				Expect(err).NotTo(HaveOccurred())

				// Copy in the nobbled iptables command.
				err = felixContainer.CopyFileIntoContainer("slow-iptables-restore",
					"/sbin/iptables-restore")
				Expect(err).NotTo(HaveOccurred())

				// Make it executable.
				err = felixContainer.ExecMayFail("chmod", "+x", "/sbin/iptables-restore")
				Expect(err).NotTo(HaveOccurred())

				// Make an update that will force felix to run iptables-restore.
				createLocalPod()
			})

			It("should detect dataplane pause and become non-ready", func() {
				Eventually(felixReady, "120s", "10s").ShouldNot(BeGood())
			})
		})
	}

	var typhaContainer *containers.Container
	var typhaReady, typhaLiveness func() int

	startTypha := func(endpoint string) {
		typhaContainer = containers.Run("typha",
			containers.RunOpts{AutoRemove: true},
			"--privileged",
			"-e", "CALICO_DATASTORE_TYPE=kubernetes",
			"-e", "TYPHA_HEALTHENABLED=true",
			"-e", "TYPHA_LOGSEVERITYSCREEN=info",
			"-e", "TYPHA_DATASTORETYPE=kubernetes",
			"-e", "TYPHA_PROMETHEUSMETRICSENABLED=true",
			"-e", "TYPHA_USAGEREPORTINGENABLED=false",
			"-e", "TYPHA_DEBUGMEMORYPROFILEPATH=\"heap-<timestamp>\"",
			"-e", "K8S_API_ENDPOINT="+endpoint,
			"-e", "K8S_INSECURE_SKIP_TLS_VERIFY=true",
			"-v", k8sAPIServer.CertFileName+":/tmp/apiserver.crt",
			utils.Config.TyphaImage,
			"calico-typha")
		Expect(typhaContainer).NotTo(BeNil())
		typhaReady = getHealthStatus(typhaContainer.IP, "9098", "readiness")
		typhaLiveness = getHealthStatus(typhaContainer.IP, "9098", "liveness")
	}

	startFelix := func(typhaAddr string, calcGraphHangTime string, dataplaneHangTime string) {
		felixContainer = containers.Run("felix",
			containers.RunOpts{AutoRemove: true},
			"--privileged",
			"-e", "CALICO_DATASTORE_TYPE=kubernetes",
			"-e", "FELIX_IPV6SUPPORT=false",
			"-e", "FELIX_HEALTHENABLED=true",
			"-e", "FELIX_LOGSEVERITYSCREEN=info",
			"-e", "FELIX_DATASTORETYPE=kubernetes",
			"-e", "FELIX_PROMETHEUSMETRICSENABLED=true",
			"-e", "FELIX_USAGEREPORTINGENABLED=false",
			"-e", "FELIX_DEBUGMEMORYPROFILEPATH=\"heap-<timestamp>\"",
			"-e", "FELIX_DebugSimulateCalcGraphHangAfter="+calcGraphHangTime,
			"-e", "FELIX_DebugSimulateDataplaneHangAfter="+dataplaneHangTime,
			"-e", "K8S_API_ENDPOINT="+k8sAPIServer.Endpoint,
			"-e", "K8S_INSECURE_SKIP_TLS_VERIFY=true",
			"-e", "FELIX_TYPHAADDR="+typhaAddr,
			"-v", k8sAPIServer.CertFileName+":/tmp/apiserver.crt",
			"calico/felix:latest",
		)
		Expect(felixContainer).NotTo(BeNil())

		felixReady = getHealthStatus(felixContainer.IP, "9099", "readiness")
		felixLiveness = getHealthStatus(felixContainer.IP, "9099", "liveness")
	}

	Describe("with Felix running (no Typha)", func() {
		BeforeEach(func() {
			startFelix("", "", "")
		})

		AfterEach(func() {
			felixContainer.Stop()
		})

		describeCommonFelixTests()
	})

	Describe("with Felix (no Typha) and Felix calc graph set to hang", func() {
		BeforeEach(func() {
			startFelix("", "5", "")
		})

		AfterEach(func() {
			felixContainer.Stop()
		})

		It("should report live initially, then become non-live", func() {
			Eventually(felixLiveness, "10s", "100ms").Should(BeGood())
			Eventually(felixLiveness, "30s", "100ms").Should(BeBad())
			Consistently(felixLiveness, "10s", "100ms").Should(BeBad())
		})
	})

	Describe("with Felix (no Typha) and Felix dataplane set to hang", func() {
		BeforeEach(func() {
			startFelix("", "", "5")
		})

		AfterEach(func() {
			felixContainer.Stop()
		})

		It("should report live initially, then become non-live", func() {
			Eventually(felixLiveness, "10s", "100ms").Should(BeGood())
			Eventually(felixLiveness, "30s", "100ms").Should(BeBad())
			Consistently(felixLiveness, "10s", "100ms").Should(BeBad())
		})
	})

	Describe("with Felix and Typha running", func() {
		BeforeEach(func() {
			startTypha(k8sAPIServer.Endpoint)
			startFelix(typhaContainer.IP+":5473", "", "")
		})

		AfterEach(func() {
			felixContainer.Stop()
			typhaContainer.Stop()
		})

		describeCommonFelixTests()

		It("typha should report ready", func() {
			Eventually(typhaReady, "5s", "100ms").Should(BeGood())
			Consistently(typhaReady, "10s", "1s").Should(BeGood())
		})

		It("typha should report live", func() {
			Eventually(typhaLiveness, "5s", "100ms").Should(BeGood())
			Consistently(typhaLiveness, "10s", "1s").Should(BeGood())
		})
	})

	Describe("with typha connected to bad API endpoint", func() {
		BeforeEach(func() {
			startTypha(k8sAPIServer.BadEndpoint)
		})

		AfterEach(func() {
			typhaContainer.Stop()
		})

		It("typha should not report ready", func() {
			Consistently(typhaReady, "10s", "1s").ShouldNot(BeGood())
		})

		It("typha should not report live", func() {
			Consistently(typhaLiveness(), "10s", "1s").ShouldNot(BeGood())
		})
	})

	Describe("with datastore not ready", func() {
		BeforeEach(func() {
			info, err := k8sAPIServer.CalicoClient.ClusterInformation().Get(
				context.Background(),
				"default",
				options.GetOptions{},
			)
			Expect(err).NotTo(HaveOccurred())
			log.Infof("info = %#v", info)
			notReady := false
			info.Spec.DatastoreReady = &notReady
			_, err = k8sAPIServer.CalicoClient.ClusterInformation().Update(
				context.Background(),
				info,
				options.SetOptions{},
			)
			Expect(err).NotTo(HaveOccurred())
			startFelix("", "", "")
		})

		AfterEach(func() {
			felixContainer.Stop()
		})

		It("felix should report ready", func() {
			Eventually(felixReady, "5s", "100ms").Should(BeGood())
			Consistently(felixReady, "10s", "1s").Should(BeGood())
		})

		It("felix should report live", func() {
			Eventually(felixLiveness, "5s", "100ms").Should(BeGood())
			Consistently(felixLiveness, "10s", "1s").Should(BeGood())
		})
	})
})

const statusErr = -1

func getHealthStatus(ip, port, endpoint string) func() int {
	return func() int {
		resp, err := http.Get("http://" + ip + ":" + port + "/" + endpoint)
		if err != nil {
			log.WithError(err).WithField("resp", resp).Warn("HTTP GET failed")
			return statusErr
		}
		defer resp.Body.Close()
		log.WithField("resp", resp).Info("Health response")
		return resp.StatusCode
	}
}

func BeErr() types.GomegaMatcher {
	return BeNumerically("==", statusErr)
}

func BeBad() types.GomegaMatcher {
	return BeNumerically("==", health.StatusBad)
}

func BeGood() types.GomegaMatcher {
	return BeNumerically("==", health.StatusGood)
}
<|endoftext|>


<commit_before>package gat

import (
	"testing"
	"time"

	libgit "github.com/libgit2/git2go"
	. "github.com/smartystreets/goconvey/convey"

	"polydawn.net/danggit/api"
	"polydawn.net/danggit/lib/testutil"
)

func TestHeads(t *testing.T) {
	Convey("Given a local git repo", t, testutil.WithTmpdir(func() {
		repo, err := libgit.InitRepository("repo", true)
		maybePanic(err)

		Convey("which is empty", func() {
			Convey("ListHeads should work", func() {
				resp := ListHeads(git.ReqListHeads{Repo: "repo"})
				So(resp.Error, ShouldBeNil)
				So(resp.Heads, ShouldHaveLength, 0)
			})
		})

		Convey("and given some commits and branches", func() {
			// shove in file content
			blobOid, err := repo.CreateBlobFromBuffer([]byte("hello, world!\n"))
			maybePanic(err)

			// assemble tree
			treeBuilder, err := repo.TreeBuilder()
			maybePanic(err)
			defer treeBuilder.Free()
			treeBuilder.Insert("thefile", blobOid, libgit.FilemodeBlob)
			treeOid, err := treeBuilder.Write()
			maybePanic(err)
			// immediately look it up again because silly api
			tree, err := repo.LookupTree(treeOid)
			maybePanic(err)

			// assemble commit
			author := &libgit.Signature{
				Name:  "author",
				Email: "email@domain.wow",
				When:  time.Date(2009, 10, 14, 12, 0, 0, 0, time.UTC),
			}
			commitOid, err := repo.CreateCommit(
				"",
				author,
				author,
				"log message",
				tree,
			)
			maybePanic(err)
			// immediately look it up again because silly api
			commit, err := repo.LookupCommit(commitOid)
			maybePanic(err)

			// create branch
			branch, err := repo.CreateBranch("branchname", commit, false)
			maybePanic(err)
			branch.Free()

			Convey("ListHeads should work", func() {
				resp := ListHeads(git.ReqListHeads{Repo: "repo"})
				So(resp.Error, ShouldBeNil)
				//So(resp.Heads, ShouldHaveLength, 1)
			})
		})
	}))
}
<commit_msg>Attempt to exercise the ListHeads_Remote code.  it blows up *marvellously*.<commit_after>package gat

import (
	"testing"
	"time"

	libgit "github.com/libgit2/git2go"
	. "github.com/smartystreets/goconvey/convey"

	"polydawn.net/danggit/api"
	"polydawn.net/danggit/lib/testutil"
)

func TestHeads(t *testing.T) {
	Convey("Given a local git repo", t, testutil.WithTmpdir(func() {
		repo, err := libgit.InitRepository("repo", true)
		maybePanic(err)

		Convey("which is empty", func() {
			Convey("ListHeads should work", func() {
				resp := ListHeads(git.ReqListHeads{Repo: "repo"})
				So(resp.Error, ShouldBeNil)
				So(resp.Heads, ShouldHaveLength, 0)
			})

			Convey("ListHeads_Remote should work", func() {
				resp := ListHeads_Remote(git.ReqListHeadsRemote{Repo: "repo"})
				So(resp.Error, ShouldBeNil)
				So(resp.Heads, ShouldHaveLength, 0)
			})
		})

		Convey("and given some commits and branches", func() {
			// shove in file content
			blobOid, err := repo.CreateBlobFromBuffer([]byte("hello, world!\n"))
			maybePanic(err)

			// assemble tree
			treeBuilder, err := repo.TreeBuilder()
			maybePanic(err)
			defer treeBuilder.Free()
			treeBuilder.Insert("thefile", blobOid, libgit.FilemodeBlob)
			treeOid, err := treeBuilder.Write()
			maybePanic(err)
			// immediately look it up again because silly api
			tree, err := repo.LookupTree(treeOid)
			maybePanic(err)

			// assemble commit
			author := &libgit.Signature{
				Name:  "author",
				Email: "email@domain.wow",
				When:  time.Date(2009, 10, 14, 12, 0, 0, 0, time.UTC),
			}
			commitOid, err := repo.CreateCommit(
				"",
				author,
				author,
				"log message",
				tree,
			)
			maybePanic(err)
			// immediately look it up again because silly api
			commit, err := repo.LookupCommit(commitOid)
			maybePanic(err)

			// create branch
			branch, err := repo.CreateBranch("branchname", commit, false)
			maybePanic(err)
			branch.Free()

			Convey("ListHeads should work", func() {
				resp := ListHeads(git.ReqListHeads{Repo: "repo"})
				So(resp.Error, ShouldBeNil)
				So(resp.Heads, ShouldHaveLength, 1)
			})

			Convey("ListHeads_Remote should work", func() {
				resp := ListHeads_Remote(git.ReqListHeadsRemote{Repo: "repo"})
				So(resp.Error, ShouldBeNil)
				So(resp.Heads, ShouldHaveLength, 1)
			})
		})
	}))
}
<|endoftext|>


<commit_before>package gclient

import (
	"encoding/json"
	"fmt"
	"github.com/dougEfresh/gtoggl"
)

// Toggl Client Definition
type Client struct {
	Id       uint64 `json:"id"`
	WId      uint64 `json:"wid"`
	Name     string `json:"name"`
	Currency string `json:"currency"`
}

type Clients []Client

const Endpoint = "/clients"

//Return a Toggl Client. An error is also returned when some configuration option is invalid
//    thc,err := gtoggl.NewClient("token")
//    tc,err := gclient.NewClient(tc)
func NewClient(thc *gtoggl.TogglHttpClient, options ...ToggleClientOptionFunc) (*TogglClient, error) {
	tc := &TogglClient{
		thc:             thc,
		listTransport:   defaultClientTransport,
		getTransport:    defaultClientTransport,
		updateTransport: defaultClientTransport,
		createTransport: defaultClientTransport,
		deleteTransport: defaultClientTransport,
	}
	// Run the options on it

	for _, option := range options {
		if err := option(tc); err != nil {
			return nil, err
		}
	}
	tc.endpoint = thc.Url + Endpoint
	return tc, nil
}

type TogglClient struct {
	thc             *gtoggl.TogglHttpClient
	endpoint        string
	listTransport   ClientLister
	getTransport    ClientGetter
	updateTransport ClientUpdater
	createTransport ClientCreater
	deleteTransport ClientDeleter
}

func (tc *TogglClient) List() (Clients, error) {
	return tc.listTransport.List(tc)
}

func (tc *TogglClient) Get(id uint64) (Client, error) {
	return tc.getTransport.Get(tc, id)
}

func (tc *TogglClient) Create(c *Client) (Client, error) {
	return tc.createTransport.Create(tc, c)
}

func (tc *TogglClient) Update(c *Client) (Client, error) {
	return tc.updateTransport.Update(tc, c)
}

func (tc *TogglClient) Delete(id uint64) error {
	return tc.deleteTransport.Delete(tc, id)
}

type ClientLister interface {
	List(tc *TogglClient) (Clients, error)
}
type ClientGetter interface {
	Get(tc *TogglClient, id uint64) (Client, error)
}
type ClientUpdater interface {
	Update(tc *TogglClient, c *Client) (Client, error)
}
type ClientCreater interface {
	Create(tc *TogglClient, c *Client) (Client, error)
}
type ClientDeleter interface {
	Delete(tc *TogglClient, id uint64) error
}

//Configures a Client.
/*
    func SetURL(url string) ToggleClientOptionFunc {
	return func(c *TogglClient) error {
	    c.Url = url
	}
    }
*/
type ToggleClientOptionFunc func(*TogglClient) error
type clientTransport struct{}

var defaultClientTransport = &clientTransport{}

type clientResponse struct {
	Data Client `json:"data"`
}
type clientRequest struct {
	Data Client `json:"data"`
}
type clientCreateRequest struct {
	Client Client `json:"client"`
}

//GET https://www.toggl.com/api/v8/clients/1239455
func (cl *clientTransport) Get(tc *TogglClient, id uint64) (Client, error) {
	body, err := tc.thc.GetRequest(fmt.Sprintf("%s/%d", tc.endpoint, id))
	if err != nil {
		return Client{}, err
	}

	var aux clientResponse
	err = json.Unmarshal(body, &aux)
	return aux.Data, err
}

//DELETE https://www.toggl.com/api/v8/clients/1239455
func (cl *clientTransport) Delete(tc *TogglClient, id uint64) error {
	_, err := tc.thc.DeleteRequest(fmt.Sprintf("%s/%d", tc.endpoint, id), nil)
	return err
}

//GET https://www.toggl.com/api/v8/clients/1239455
func (cl *clientTransport) List(tc *TogglClient) (Clients, error) {
	body, err := tc.thc.GetRequest(tc.endpoint)
	var Clients []Client
	if err != nil {
		return Clients, err
	}
	err = json.Unmarshal(body, &Clients)
	return Clients, err
}

//PUT https://www.toggl.com/api/v8/clients/1239455
func (cl *clientTransport) Update(tc *TogglClient, c *Client) (Client, error) {
	put := clientCreateRequest{Client: *c}
	body, err := json.Marshal(put)
	if err != nil {
		return Client{}, err
	}
	response, err := tc.thc.PutRequest(fmt.Sprintf("%s/%d", tc.endpoint, c.Id), body)
	if err != nil {
		return Client{}, err
	}
	var aux clientResponse
	err = json.Unmarshal(response, &aux)
	return aux.Data, err
}

//POST https://www.toggl.com/api/v8/clients
func (cl *clientTransport) Create(tc *TogglClient, c *Client) (Client, error) {
	put := clientCreateRequest{Client: *c}
	body, err := json.Marshal(put)
	if err != nil {
		return Client{}, err
	}
	response, err := tc.thc.PostRequest(tc.endpoint, body)
	if err != nil {
		return Client{}, err
	}
	var aux clientResponse
	err = json.Unmarshal(response, &aux)
	return aux.Data, err
}
<commit_msg>client updates<commit_after>package gclient

import (
	"encoding/json"
	"fmt"
	"github.com/dougEfresh/gtoggl"
)

// Toggl Client Definition
type Client struct {
	Id       uint64 `json:"id"`
	WId      uint64 `json:"wid"`
	Name     string `json:"name"`
	Currency string `json:"currency"`
}

type Clients []Client

const Endpoint = "/clients"

//Return a Toggl Client. An error is also returned when some configuration option is invalid
//    thc,err := gtoggl.NewClient("token")
//    tc,err := gclient.NewClient(tc)
func NewClient(thc *gtoggl.TogglHttpClient, options ...ToggleClientOptionFunc) (*TogglClient, error) {
	tc := &TogglClient{
		thc:             thc,
	}
	// Run the options on it
	for _, option := range options {
		if err := option(tc); err != nil {
			return nil, err
		}
	}
	tc.endpoint = thc.Url + Endpoint
	return tc, nil
}

type TogglClient struct {
	thc             *gtoggl.TogglHttpClient
	endpoint        string
}

func (tc *TogglClient) List() (Clients, error) {
	body, err := tc.thc.GetRequest(tc.endpoint)
	var clients Clients
	if err != nil {
		return clients, err
	}
	err = json.Unmarshal(*body, &clients)
	return clients, err
}

func (tc *TogglClient) Get(id uint64) (*Client, error) {
	body, err := tc.thc.GetRequest(fmt.Sprintf("%s/%d", tc.endpoint, id))
	if err != nil {
		return nil, err
	}
	return clientResponse(body)
}

func (tc *TogglClient) Create(c *Client) (*Client, error) {
	put := clientCreateRequest{Client: *c}
	body, err := json.Marshal(put)
	if err != nil {
		return nil, err
	}
	response, err := tc.thc.PostRequest(tc.endpoint, body)
	if err != nil {
		return nil, err
	}
	return clientResponse(response)
}

func (tc *TogglClient) Update(c *Client) (*Client, error) {
	put := clientCreateRequest{Client: *c}
	body, err := json.Marshal(put)
	if err != nil {
		return nil, err
	}
	response, err := tc.thc.PutRequest(fmt.Sprintf("%s/%d", tc.endpoint, c.Id), body)
	if err != nil {
		return nil, err
	}
	return clientResponse(response)
}

func (tc *TogglClient) Delete(id uint64) error {
	_, err := tc.thc.DeleteRequest(fmt.Sprintf("%s/%d", tc.endpoint, id), nil)
	return err
}

//Configures a Client.
/*
    func SetURL(url string) ToggleClientOptionFunc {
	return func(c *TogglClient) error {
	    c.Url = url
	}
    }
*/
type ToggleClientOptionFunc func(*TogglClient) error

type clientCreateRequest struct {
	Client Client `json:"client"`
}

func clientResponse(response *json.RawMessage) (*Client, error) {
	var tResp gtoggl.TogglResponse
	err := json.Unmarshal(*response, &tResp)
	if err != nil {
		return nil, err
	}
	var cl Client
	err = json.Unmarshal(*tResp.Data, &cl)
	if err != nil {
		return nil, err
	}
	return &cl, err
}
<|endoftext|>


<commit_before>// Copyright 2010 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

// +build linux

package fsnotify

import (
	"errors"
	"fmt"
	"os"
	"strings"
	"sync"
	"syscall"
	"unsafe"
)

const (
	// Options for inotify_init() are not exported
	// sys_IN_CLOEXEC    uint32 = syscall.IN_CLOEXEC
	// sys_IN_NONBLOCK   uint32 = syscall.IN_NONBLOCK

	// Options for AddWatch
	sys_IN_DONT_FOLLOW uint32 = syscall.IN_DONT_FOLLOW
	sys_IN_ONESHOT     uint32 = syscall.IN_ONESHOT
	sys_IN_ONLYDIR     uint32 = syscall.IN_ONLYDIR

	// The "sys_IN_MASK_ADD" option is not exported, as AddWatch
	// adds it automatically, if there is already a watch for the given path
	// sys_IN_MASK_ADD      uint32 = syscall.IN_MASK_ADD

	// Events
	sys_IN_ACCESS        uint32 = syscall.IN_ACCESS
	sys_IN_ALL_EVENTS    uint32 = syscall.IN_ALL_EVENTS
	sys_IN_ATTRIB        uint32 = syscall.IN_ATTRIB
	sys_IN_CLOSE         uint32 = syscall.IN_CLOSE
	sys_IN_CLOSE_NOWRITE uint32 = syscall.IN_CLOSE_NOWRITE
	sys_IN_CLOSE_WRITE   uint32 = syscall.IN_CLOSE_WRITE
	sys_IN_CREATE        uint32 = syscall.IN_CREATE
	sys_IN_DELETE        uint32 = syscall.IN_DELETE
	sys_IN_DELETE_SELF   uint32 = syscall.IN_DELETE_SELF
	sys_IN_MODIFY        uint32 = syscall.IN_MODIFY
	sys_IN_MOVE          uint32 = syscall.IN_MOVE
	sys_IN_MOVED_FROM    uint32 = syscall.IN_MOVED_FROM
	sys_IN_MOVED_TO      uint32 = syscall.IN_MOVED_TO
	sys_IN_MOVE_SELF     uint32 = syscall.IN_MOVE_SELF
	sys_IN_OPEN          uint32 = syscall.IN_OPEN

	sys_AGNOSTIC_EVENTS = sys_IN_MOVED_TO | sys_IN_MOVED_FROM | sys_IN_CREATE | sys_IN_ATTRIB | sys_IN_MODIFY | sys_IN_MOVE_SELF | sys_IN_DELETE | sys_IN_DELETE_SELF

	// Special events
	sys_IN_ISDIR      uint32 = syscall.IN_ISDIR
	sys_IN_IGNORED    uint32 = syscall.IN_IGNORED
	sys_IN_Q_OVERFLOW uint32 = syscall.IN_Q_OVERFLOW
	sys_IN_UNMOUNT    uint32 = syscall.IN_UNMOUNT
)

type FileEvent struct {
	mask   uint32 // Mask of events
	cookie uint32 // Unique cookie associating related events (for rename(2))
	Name   string // File name (optional)
}

// IsCreate reports whether the FileEvent was triggered by a creation
func (e *FileEvent) IsCreate() bool {
	return (e.mask&sys_IN_CREATE) == sys_IN_CREATE || (e.mask&sys_IN_MOVED_TO) == sys_IN_MOVED_TO
}

// IsDelete reports whether the FileEvent was triggered by a delete
func (e *FileEvent) IsDelete() bool {
	return (e.mask&sys_IN_DELETE_SELF) == sys_IN_DELETE_SELF || (e.mask&sys_IN_DELETE) == sys_IN_DELETE
}

// IsModify reports whether the FileEvent was triggered by a file modification or attribute change
func (e *FileEvent) IsModify() bool {
	return ((e.mask&sys_IN_MODIFY) == sys_IN_MODIFY || (e.mask&sys_IN_ATTRIB) == sys_IN_ATTRIB)
}

// IsRename reports whether the FileEvent was triggered by a change name
func (e *FileEvent) IsRename() bool {
	return ((e.mask&sys_IN_MOVE_SELF) == sys_IN_MOVE_SELF || (e.mask&sys_IN_MOVED_FROM) == sys_IN_MOVED_FROM)
}

// IsAttrib reports whether the FileEvent was triggered by a change in the file metadata.
func (e *FileEvent) IsAttrib() bool {
	return (e.mask & sys_IN_ATTRIB) == sys_IN_ATTRIB
}

type watch struct {
	wd    uint32 // Watch descriptor (as returned by the inotify_add_watch() syscall)
	flags uint32 // inotify flags of this watch (see inotify(7) for the list of valid flags)
}

type Watcher struct {
	mu            sync.Mutex        // Map access
	fd            int               // File descriptor (as returned by the inotify_init() syscall)
	watches       map[string]*watch // Map of inotify watches (key: path)
	fsnFlags      map[string]uint32 // Map of watched files to flags used for filter
	fsnmut        sync.Mutex        // Protects access to fsnFlags.
	paths         map[int]string    // Map of watched paths (key: watch descriptor)
	Error         chan error        // Errors are sent on this channel
	internalEvent chan *FileEvent   // Events are queued on this channel
	Event         chan *FileEvent   // Events are returned on this channel
	done          chan bool         // Channel for sending a "quit message" to the reader goroutine
	isClosed      bool              // Set to true when Close() is first called
}

// NewWatcher creates and returns a new inotify instance using inotify_init(2)
func NewWatcher() (*Watcher, error) {
	fd, errno := syscall.InotifyInit()
	if fd == -1 {
		return nil, os.NewSyscallError("inotify_init", errno)
	}
	w := &Watcher{
		fd:            fd,
		watches:       make(map[string]*watch),
		fsnFlags:      make(map[string]uint32),
		paths:         make(map[int]string),
		internalEvent: make(chan *FileEvent),
		Event:         make(chan *FileEvent),
		Error:         make(chan error),
		done:          make(chan bool, 1),
	}

	go w.readEvents()
	go w.purgeEvents()
	return w, nil
}

// Close closes an inotify watcher instance
// It sends a message to the reader goroutine to quit and removes all watches
// associated with the inotify instance
func (w *Watcher) Close() error {
	if w.isClosed {
		return nil
	}
	w.isClosed = true

	// Remove all watches
	for path := range w.watches {
		w.RemoveWatch(path)
	}

	// Send "quit" message to the reader goroutine
	w.done <- true

	return nil
}

// AddWatch adds path to the watched file set.
// The flags are interpreted as described in inotify_add_watch(2).
func (w *Watcher) addWatch(path string, flags uint32) error {
	if w.isClosed {
		return errors.New("inotify instance already closed")
	}

	w.mu.Lock()
	watchEntry, found := w.watches[path]
	w.mu.Unlock()
	if found {
		watchEntry.flags |= flags
		flags |= syscall.IN_MASK_ADD
	}
	wd, errno := syscall.InotifyAddWatch(w.fd, path, flags)
	if wd == -1 {
		return errno
	}

	w.mu.Lock()
	w.watches[path] = &watch{wd: uint32(wd), flags: flags}
	w.paths[wd] = path
	w.mu.Unlock()

	return nil
}

// Watch adds path to the watched file set, watching all events.
func (w *Watcher) watch(path string) error {
	return w.addWatch(path, sys_AGNOSTIC_EVENTS)
}

// RemoveWatch removes path from the watched file set.
func (w *Watcher) removeWatch(path string) error {
	w.mu.Lock()
	defer w.mu.Unlock()
	watch, ok := w.watches[path]
	if !ok {
		return errors.New(fmt.Sprintf("can't remove non-existent inotify watch for: %s", path))
	}
	success, errno := syscall.InotifyRmWatch(w.fd, watch.wd)
	if success == -1 {
		return os.NewSyscallError("inotify_rm_watch", errno)
	}
	delete(w.watches, path)
	return nil
}

// readEvents reads from the inotify file descriptor, converts the
// received events into Event objects and sends them via the Event channel
func (w *Watcher) readEvents() {
	var (
		buf   [syscall.SizeofInotifyEvent * 4096]byte // Buffer for a maximum of 4096 raw events
		n     int                                     // Number of bytes read with read()
		errno error                                   // Syscall errno
	)

	for {
		// See if there is a message on the "done" channel
		select {
		case <-w.done:
			syscall.Close(w.fd)
			close(w.internalEvent)
			close(w.Error)
			return
		default:
		}

		n, errno = syscall.Read(w.fd, buf[:])

		// If EOF is received
		if n == 0 {
			syscall.Close(w.fd)
			close(w.internalEvent)
			close(w.Error)
			return
		}

		if n < 0 {
			w.Error <- os.NewSyscallError("read", errno)
			continue
		}
		if n < syscall.SizeofInotifyEvent {
			w.Error <- errors.New("inotify: short read in readEvents()")
			continue
		}

		var offset uint32 = 0
		// We don't know how many events we just read into the buffer
		// While the offset points to at least one whole event...
		for offset <= uint32(n-syscall.SizeofInotifyEvent) {
			// Point "raw" to the event in the buffer
			raw := (*syscall.InotifyEvent)(unsafe.Pointer(&buf[offset]))
			event := new(FileEvent)
			event.mask = uint32(raw.Mask)
			event.cookie = uint32(raw.Cookie)
			nameLen := uint32(raw.Len)
			// If the event happened to the watched directory or the watched file, the kernel
			// doesn't append the filename to the event, but we would like to always fill the
			// the "Name" field with a valid filename. We retrieve the path of the watch from
			// the "paths" map.
			w.mu.Lock()
			event.Name = w.paths[int(raw.Wd)]
			w.mu.Unlock()
			watchedName := event.Name
			if nameLen > 0 {
				// Point "bytes" at the first byte of the filename
				bytes := (*[syscall.PathMax]byte)(unsafe.Pointer(&buf[offset+syscall.SizeofInotifyEvent]))
				// The filename is padded with NUL bytes. TrimRight() gets rid of those.
				event.Name += "/" + strings.TrimRight(string(bytes[0:nameLen]), "\000")
			}

			// Send the events that are not ignored on the events channel
			if !event.ignoreLinux() {
				// Setup FSNotify flags (inherit from directory watch)
				w.fsnmut.Lock()
				if _, fsnFound := w.fsnFlags[event.Name]; !fsnFound {
					if fsnFlags, watchFound := w.fsnFlags[watchedName]; watchFound {
						w.fsnFlags[event.Name] = fsnFlags
					} else {
						w.fsnFlags[event.Name] = FSN_ALL
					}
				}
				w.fsnmut.Unlock()

				w.internalEvent <- event
			}

			// Move to the next event in the buffer
			offset += syscall.SizeofInotifyEvent + nameLen
		}
	}
}

// Certain types of events can be "ignored" and not sent over the Event
// channel. Such as events marked ignore by the kernel, or MODIFY events
// against files that do not exist.
func (e *FileEvent) ignoreLinux() bool {
	// Ignore anything the inotify API says to ignore
	if e.mask&sys_IN_IGNORED == sys_IN_IGNORED {
		return true
	}

	// If the event is not a DELETE or RENAME, the file must exist.
	// Otherwise the event is ignored.
	// *Note*: this was put in place because it was seen that a MODIFY
	// event was sent after the DELETE. This ignores that MODIFY and
	// assumes a DELETE will come or has come if the file doesn't exist.
	if !(e.IsDelete() || e.IsRename()) {
		_, statErr := os.Lstat(e.Name)
		return os.IsNotExist(statErr)
	}
	return false
}
<commit_msg>hack in support for close events<commit_after>// Copyright 2010 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

// +build linux

package fsnotify

import (
	"errors"
	"fmt"
	"os"
	"strings"
	"sync"
	"syscall"
	"unsafe"
)

const (
	// Options for inotify_init() are not exported
	// sys_IN_CLOEXEC    uint32 = syscall.IN_CLOEXEC
	// sys_IN_NONBLOCK   uint32 = syscall.IN_NONBLOCK

	// Options for AddWatch
	sys_IN_DONT_FOLLOW uint32 = syscall.IN_DONT_FOLLOW
	sys_IN_ONESHOT     uint32 = syscall.IN_ONESHOT
	sys_IN_ONLYDIR     uint32 = syscall.IN_ONLYDIR

	// The "sys_IN_MASK_ADD" option is not exported, as AddWatch
	// adds it automatically, if there is already a watch for the given path
	// sys_IN_MASK_ADD      uint32 = syscall.IN_MASK_ADD

	// Events
	sys_IN_ACCESS        uint32 = syscall.IN_ACCESS
	sys_IN_ALL_EVENTS    uint32 = syscall.IN_ALL_EVENTS
	sys_IN_ATTRIB        uint32 = syscall.IN_ATTRIB
	sys_IN_CLOSE         uint32 = syscall.IN_CLOSE
	sys_IN_CLOSE_NOWRITE uint32 = syscall.IN_CLOSE_NOWRITE
	sys_IN_CLOSE_WRITE   uint32 = syscall.IN_CLOSE_WRITE
	sys_IN_CREATE        uint32 = syscall.IN_CREATE
	sys_IN_DELETE        uint32 = syscall.IN_DELETE
	sys_IN_DELETE_SELF   uint32 = syscall.IN_DELETE_SELF
	sys_IN_MODIFY        uint32 = syscall.IN_MODIFY
	sys_IN_MOVE          uint32 = syscall.IN_MOVE
	sys_IN_MOVED_FROM    uint32 = syscall.IN_MOVED_FROM
	sys_IN_MOVED_TO      uint32 = syscall.IN_MOVED_TO
	sys_IN_MOVE_SELF     uint32 = syscall.IN_MOVE_SELF
	sys_IN_OPEN          uint32 = syscall.IN_OPEN

	sys_AGNOSTIC_EVENTS = sys_IN_MOVED_TO | sys_IN_MOVED_FROM | sys_IN_CREATE | sys_IN_ATTRIB | sys_IN_MODIFY | sys_IN_MOVE_SELF | sys_IN_DELETE | sys_IN_DELETE_SELF

	// Special events
	sys_IN_ISDIR      uint32 = syscall.IN_ISDIR
	sys_IN_IGNORED    uint32 = syscall.IN_IGNORED
	sys_IN_Q_OVERFLOW uint32 = syscall.IN_Q_OVERFLOW
	sys_IN_UNMOUNT    uint32 = syscall.IN_UNMOUNT
)

type FileEvent struct {
	mask   uint32 // Mask of events
	cookie uint32 // Unique cookie associating related events (for rename(2))
	Name   string // File name (optional)
}

// IsCreate reports whether the FileEvent was triggered by a creation
func (e *FileEvent) IsCreate() bool {
	return (e.mask&sys_IN_CREATE) == sys_IN_CREATE || (e.mask&sys_IN_MOVED_TO) == sys_IN_MOVED_TO
}

// IsDelete reports whether the FileEvent was triggered by a delete
func (e *FileEvent) IsDelete() bool {
	return (e.mask&sys_IN_DELETE_SELF) == sys_IN_DELETE_SELF || (e.mask&sys_IN_DELETE) == sys_IN_DELETE
}

// IsModify reports whether the FileEvent was triggered by a file modification or attribute change
func (e *FileEvent) IsModify() bool {
	return ((e.mask&sys_IN_MODIFY) == sys_IN_MODIFY || (e.mask&sys_IN_ATTRIB) == sys_IN_ATTRIB)
}

// IsRename reports whether the FileEvent was triggered by a change name
func (e *FileEvent) IsRename() bool {
	return ((e.mask&sys_IN_MOVE_SELF) == sys_IN_MOVE_SELF || (e.mask&sys_IN_MOVED_FROM) == sys_IN_MOVED_FROM)
}

// IsAttrib reports whether the FileEvent was triggered by a change in the file metadata.
func (e *FileEvent) IsAttrib() bool {
	return (e.mask & sys_IN_ATTRIB) == sys_IN_ATTRIB
}

// IsClose reports whether the FileEvent was triggered by the closing of a file
func (e *FileEvent) IsClose() bool {
	return ((e.mask&sys_IN_CLOSE_NOWRITE) == sys_IN_CLOSE_NOWRITE || (e.mask&sys_IN_CLOSE_WRITE) == sys_IN_CLOSE_WRITE)
}

type watch struct {
	wd    uint32 // Watch descriptor (as returned by the inotify_add_watch() syscall)
	flags uint32 // inotify flags of this watch (see inotify(7) for the list of valid flags)
}

type Watcher struct {
	mu            sync.Mutex        // Map access
	fd            int               // File descriptor (as returned by the inotify_init() syscall)
	watches       map[string]*watch // Map of inotify watches (key: path)
	fsnFlags      map[string]uint32 // Map of watched files to flags used for filter
	fsnmut        sync.Mutex        // Protects access to fsnFlags.
	paths         map[int]string    // Map of watched paths (key: watch descriptor)
	Error         chan error        // Errors are sent on this channel
	internalEvent chan *FileEvent   // Events are queued on this channel
	Event         chan *FileEvent   // Events are returned on this channel
	done          chan bool         // Channel for sending a "quit message" to the reader goroutine
	isClosed      bool              // Set to true when Close() is first called
}

// NewWatcher creates and returns a new inotify instance using inotify_init(2)
func NewWatcher() (*Watcher, error) {
	fd, errno := syscall.InotifyInit()
	if fd == -1 {
		return nil, os.NewSyscallError("inotify_init", errno)
	}
	w := &Watcher{
		fd:            fd,
		watches:       make(map[string]*watch),
		fsnFlags:      make(map[string]uint32),
		paths:         make(map[int]string),
		internalEvent: make(chan *FileEvent),
		Event:         make(chan *FileEvent),
		Error:         make(chan error),
		done:          make(chan bool, 1),
	}

	go w.readEvents()
	go w.purgeEvents()
	return w, nil
}

// Close closes an inotify watcher instance
// It sends a message to the reader goroutine to quit and removes all watches
// associated with the inotify instance
func (w *Watcher) Close() error {
	if w.isClosed {
		return nil
	}
	w.isClosed = true

	// Remove all watches
	for path := range w.watches {
		w.RemoveWatch(path)
	}

	// Send "quit" message to the reader goroutine
	w.done <- true

	return nil
}

// AddWatch adds path to the watched file set.
// The flags are interpreted as described in inotify_add_watch(2).
func (w *Watcher) addWatch(path string, flags uint32) error {
	if w.isClosed {
		return errors.New("inotify instance already closed")
	}

	w.mu.Lock()
	watchEntry, found := w.watches[path]
	w.mu.Unlock()
	if found {
		watchEntry.flags |= flags
		flags |= syscall.IN_MASK_ADD
	}
	wd, errno := syscall.InotifyAddWatch(w.fd, path, flags)
	if wd == -1 {
		return errno
	}

	w.mu.Lock()
	w.watches[path] = &watch{wd: uint32(wd), flags: flags}
	w.paths[wd] = path
	w.mu.Unlock()

	return nil
}

// Watch adds path to the watched file set, watching all events.
func (w *Watcher) watch(path string) error {
	return w.addWatch(path, sys_AGNOSTIC_EVENTS | sys_IN_CLOSE_WRITE | sys_IN_CLOSE_NOWRITE)
}

// RemoveWatch removes path from the watched file set.
func (w *Watcher) removeWatch(path string) error {
	w.mu.Lock()
	defer w.mu.Unlock()
	watch, ok := w.watches[path]
	if !ok {
		return errors.New(fmt.Sprintf("can't remove non-existent inotify watch for: %s", path))
	}
	success, errno := syscall.InotifyRmWatch(w.fd, watch.wd)
	if success == -1 {
		return os.NewSyscallError("inotify_rm_watch", errno)
	}
	delete(w.watches, path)
	return nil
}

// readEvents reads from the inotify file descriptor, converts the
// received events into Event objects and sends them via the Event channel
func (w *Watcher) readEvents() {
	var (
		buf   [syscall.SizeofInotifyEvent * 4096]byte // Buffer for a maximum of 4096 raw events
		n     int                                     // Number of bytes read with read()
		errno error                                   // Syscall errno
	)

	for {
		// See if there is a message on the "done" channel
		select {
		case <-w.done:
			syscall.Close(w.fd)
			close(w.internalEvent)
			close(w.Error)
			return
		default:
		}

		n, errno = syscall.Read(w.fd, buf[:])

		// If EOF is received
		if n == 0 {
			syscall.Close(w.fd)
			close(w.internalEvent)
			close(w.Error)
			return
		}

		if n < 0 {
			w.Error <- os.NewSyscallError("read", errno)
			continue
		}
		if n < syscall.SizeofInotifyEvent {
			w.Error <- errors.New("inotify: short read in readEvents()")
			continue
		}

		var offset uint32 = 0
		// We don't know how many events we just read into the buffer
		// While the offset points to at least one whole event...
		for offset <= uint32(n-syscall.SizeofInotifyEvent) {
			// Point "raw" to the event in the buffer
			raw := (*syscall.InotifyEvent)(unsafe.Pointer(&buf[offset]))
			event := new(FileEvent)
			event.mask = uint32(raw.Mask)
			event.cookie = uint32(raw.Cookie)
			nameLen := uint32(raw.Len)
			// If the event happened to the watched directory or the watched file, the kernel
			// doesn't append the filename to the event, but we would like to always fill the
			// the "Name" field with a valid filename. We retrieve the path of the watch from
			// the "paths" map.
			w.mu.Lock()
			event.Name = w.paths[int(raw.Wd)]
			w.mu.Unlock()
			watchedName := event.Name
			if nameLen > 0 {
				// Point "bytes" at the first byte of the filename
				bytes := (*[syscall.PathMax]byte)(unsafe.Pointer(&buf[offset+syscall.SizeofInotifyEvent]))
				// The filename is padded with NUL bytes. TrimRight() gets rid of those.
				event.Name += "/" + strings.TrimRight(string(bytes[0:nameLen]), "\000")
			}

			// Send the events that are not ignored on the events channel
			if !event.ignoreLinux() {
				// Setup FSNotify flags (inherit from directory watch)
				w.fsnmut.Lock()
				if _, fsnFound := w.fsnFlags[event.Name]; !fsnFound {
					if fsnFlags, watchFound := w.fsnFlags[watchedName]; watchFound {
						w.fsnFlags[event.Name] = fsnFlags
					} else {
						w.fsnFlags[event.Name] = FSN_ALL
					}
				}
				w.fsnmut.Unlock()

				w.internalEvent <- event
			}

			// Move to the next event in the buffer
			offset += syscall.SizeofInotifyEvent + nameLen
		}
	}
}

// Certain types of events can be "ignored" and not sent over the Event
// channel. Such as events marked ignore by the kernel, or MODIFY events
// against files that do not exist.
func (e *FileEvent) ignoreLinux() bool {
	// Ignore anything the inotify API says to ignore
	if e.mask&sys_IN_IGNORED == sys_IN_IGNORED {
		return true
	}

	// If the event is not a DELETE or RENAME, the file must exist.
	// Otherwise the event is ignored.
	// *Note*: this was put in place because it was seen that a MODIFY
	// event was sent after the DELETE. This ignores that MODIFY and
	// assumes a DELETE will come or has come if the file doesn't exist.
	if !(e.IsDelete() || e.IsRename()) {
		_, statErr := os.Lstat(e.Name)
		return os.IsNotExist(statErr)
	}
	return false
}
<|endoftext|>


<commit_before>// Copyright (C) 2022, Ava Labs, Inc. All rights reserved.
// See the file LICENSE for licensing terms.

package client

import (
	"bytes"
	"context"
	"fmt"
	"net/http"
	"net/url"
	"time"

	rpc "github.com/gorilla/rpc/v2/json2"
)

type Option func(*Options)

type Options struct {
	headers     http.Header
	queryParams url.Values
}

func NewOptions(ops []Option) *Options {
	o := &Options{
		headers:     http.Header{},
		queryParams: url.Values{},
	}
	o.applyOptions(ops)
	return o
}

func (o *Options) applyOptions(ops []Option) {
	for _, op := range ops {
		op(o)
	}
}

func (o *Options) Headers() http.Header {
	return o.headers
}

func (o *Options) QueryParams() url.Values {
	return o.queryParams
}

func WithHeader(key, val string) Option {
	return func(o *Options) {
		o.headers.Set(key, val)
	}
}

func WithQueryParam(key, val string) Option {
	return func(o *Options) {
		o.queryParams.Set(key, val)
	}
}

type EndpointRequester struct {
	cli       *http.Client
	uri, base string
}

// NewEndpointRequester is an extension of AvalancheGo's EndpointRequester with
// [http.Client] reuse.
func NewEndpointRequester(uri, base string) *EndpointRequester {
	t := http.DefaultTransport.(*http.Transport).Clone()
	t.MaxIdleConns = 100_000
	t.MaxConnsPerHost = 100_000
	t.MaxIdleConnsPerHost = 100_000

	return &EndpointRequester{
		cli: &http.Client{
			Timeout:   3600 * time.Second, // allow waiting requests
			Transport: t,
		},
		uri:  uri,
		base: base,
	}
}

func (e *EndpointRequester) SendRequest(
	ctx context.Context,
	method string,
	params interface{},
	reply interface{},
	options ...Option,
) error {
	uri, err := url.Parse(e.uri)
	if err != nil {
		return err
	}
	return SendJSONRequest(
		ctx,
		e.cli,
		uri,
		fmt.Sprintf("%s.%s", e.base, method),
		params,
		reply,
		options...,
	)
}

func SendJSONRequest(
	ctx context.Context,
	cli *http.Client,
	uri *url.URL,
	method string,
	params interface{},
	reply interface{},
	options ...Option,
) error {
	requestBodyBytes, err := rpc.EncodeClientRequest(method, params)
	if err != nil {
		return fmt.Errorf("failed to encode client params: %w", err)
	}

	ops := NewOptions(options)
	uri.RawQuery = ops.queryParams.Encode()

	request, err := http.NewRequestWithContext(
		ctx,
		"POST",
		uri.String(),
		bytes.NewBuffer(requestBodyBytes),
	)
	if err != nil {
		return fmt.Errorf("failed to create request: %w", err)
	}

	request.Header = ops.headers
	request.Header.Set("Content-Type", "application/json")

	resp, err := cli.Do(request)
	if err != nil {
		return fmt.Errorf("failed to issue request: %w", err)
	}

	// Return an error for any non successful status code
	if resp.StatusCode < 200 || resp.StatusCode > 299 {
		// Drop any error during close to report the original error
		_ = resp.Body.Close()
		return fmt.Errorf("received status code: %d", resp.StatusCode)
	}

	if err := rpc.DecodeClientResponse(resp.Body, reply); err != nil {
		// Drop any error during close to report the original error
		_ = resp.Body.Close()
		return fmt.Errorf("failed to decode client response: %w", err)
	}
	return resp.Body.Close()
}
<commit_msg>fix comment<commit_after>// Copyright (C) 2022, Ava Labs, Inc. All rights reserved.
// See the file LICENSE for licensing terms.

package client

import (
	"bytes"
	"context"
	"fmt"
	"net/http"
	"net/url"
	"time"

	rpc "github.com/gorilla/rpc/v2/json2"
)

type Option func(*Options)

type Options struct {
	headers     http.Header
	queryParams url.Values
}

func NewOptions(ops []Option) *Options {
	o := &Options{
		headers:     http.Header{},
		queryParams: url.Values{},
	}
	o.applyOptions(ops)
	return o
}

func (o *Options) applyOptions(ops []Option) {
	for _, op := range ops {
		op(o)
	}
}

func (o *Options) Headers() http.Header {
	return o.headers
}

func (o *Options) QueryParams() url.Values {
	return o.queryParams
}

func WithHeader(key, val string) Option {
	return func(o *Options) {
		o.headers.Set(key, val)
	}
}

func WithQueryParam(key, val string) Option {
	return func(o *Options) {
		o.queryParams.Set(key, val)
	}
}

// EndpointRequester is an extension of AvalancheGo's [EndpointRequester] with
// [http.Client] reuse.
type EndpointRequester struct {
	cli       *http.Client
	uri, base string
}

func NewEndpointRequester(uri, base string) *EndpointRequester {
	t := http.DefaultTransport.(*http.Transport).Clone()
	t.MaxIdleConns = 100_000
	t.MaxConnsPerHost = 100_000
	t.MaxIdleConnsPerHost = 100_000

	return &EndpointRequester{
		cli: &http.Client{
			Timeout:   3600 * time.Second, // allow waiting requests
			Transport: t,
		},
		uri:  uri,
		base: base,
	}
}

func (e *EndpointRequester) SendRequest(
	ctx context.Context,
	method string,
	params interface{},
	reply interface{},
	options ...Option,
) error {
	uri, err := url.Parse(e.uri)
	if err != nil {
		return err
	}
	return SendJSONRequest(
		ctx,
		e.cli,
		uri,
		fmt.Sprintf("%s.%s", e.base, method),
		params,
		reply,
		options...,
	)
}

func SendJSONRequest(
	ctx context.Context,
	cli *http.Client,
	uri *url.URL,
	method string,
	params interface{},
	reply interface{},
	options ...Option,
) error {
	requestBodyBytes, err := rpc.EncodeClientRequest(method, params)
	if err != nil {
		return fmt.Errorf("failed to encode client params: %w", err)
	}

	ops := NewOptions(options)
	uri.RawQuery = ops.queryParams.Encode()

	request, err := http.NewRequestWithContext(
		ctx,
		"POST",
		uri.String(),
		bytes.NewBuffer(requestBodyBytes),
	)
	if err != nil {
		return fmt.Errorf("failed to create request: %w", err)
	}

	request.Header = ops.headers
	request.Header.Set("Content-Type", "application/json")

	resp, err := cli.Do(request)
	if err != nil {
		return fmt.Errorf("failed to issue request: %w", err)
	}

	// Return an error for any non successful status code
	if resp.StatusCode < 200 || resp.StatusCode > 299 {
		// Drop any error during close to report the original error
		_ = resp.Body.Close()
		return fmt.Errorf("received status code: %d", resp.StatusCode)
	}

	if err := rpc.DecodeClientResponse(resp.Body, reply); err != nil {
		// Drop any error during close to report the original error
		_ = resp.Body.Close()
		return fmt.Errorf("failed to decode client response: %w", err)
	}
	return resp.Body.Close()
}
<|endoftext|>