<commit_before>/*
   Copyright 2020 Docker Compose CLI authors

   Licensed under the Apache License, Version 2.0 (the "License");
   you may not use this file except in compliance with the License.
   You may obtain a copy of the License at

       http://www.apache.org/licenses/LICENSE-2.0

   Unless required by applicable law or agreed to in writing, software
   distributed under the License is distributed on an "AS IS" BASIS,
   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
   See the License for the specific language governing permissions and
   limitations under the License.
*/

package main

import (
	"fmt"
	"io/ioutil"
	"net/http"
	"os"
	"path/filepath"
	"strconv"
	"strings"
	"testing"
	"time"

	"gotest.tools/v3/assert"
	"gotest.tools/v3/icmd"
	"gotest.tools/v3/poll"

	. "github.com/docker/compose-cli/tests/framework"
)

var binDir string

func TestMain(m *testing.M) {
	p, cleanup, err := SetupExistingCLI()
	if err != nil {
		fmt.Println(err)
		os.Exit(1)
	}
	binDir = p
	exitCode := m.Run()
	cleanup()
	os.Exit(exitCode)
}

func TestSecrets(t *testing.T) {
	cmd, testID := setupTest(t)
	secretName := "secret" + testID

	t.Run("create secret", func(t *testing.T) {
		secretFile := filepath.Join(cmd.BinDir, "secret.txt")
		err := ioutil.WriteFile(secretFile, []byte("pass1"), 0644)
		assert.Check(t, err == nil)
		res := cmd.RunDockerCmd("secret", "create", secretName, secretFile)
		assert.Check(t, strings.Contains(res.Stdout(), secretName))
	})

	t.Run("list secrets", func(t *testing.T) {
		res := cmd.RunDockerCmd("secret", "list")
		assert.Check(t, strings.Contains(res.Stdout(), secretName))
	})

	t.Run("inspect secret", func(t *testing.T) {
		res := cmd.RunDockerCmd("secret", "inspect", secretName)
		assert.Check(t, strings.Contains(res.Stdout(), `"Name": "`+secretName+`"`))
	})

	t.Run("rm secret", func(t *testing.T) {
		cmd.RunDockerCmd("secret", "rm", secretName)
		res := cmd.RunDockerCmd("secret", "list")
		assert.Check(t, !strings.Contains(res.Stdout(), secretName))
	})
}

func TestCompose(t *testing.T) {
	c, stack := setupTest(t)

	t.Run("compose up", func(t *testing.T) {
		c.RunDockerCmd("compose", "up", "--project-name", stack, "-f", "../composefiles/ecs_e2e/multi_port_secrets.yaml")
	})

	var webURL, wordsURL, secretsURL string
	t.Run("compose ps", func(t *testing.T) {
		res := c.RunDockerCmd("compose", "ps", "--project-name", stack)
		fmt.Println(strings.TrimSpace(res.Stdout()))
		lines := strings.Split(strings.TrimSpace(res.Stdout()), "\n")

		assert.Equal(t, 5, len(lines))

		var dbDisplayed, wordsDisplayed, webDisplayed, secretsDisplayed bool
		for _, line := range lines {
			fields := strings.Fields(line)
			containerID := fields[0]
			serviceName := fields[1]
			switch serviceName {
			case "db":
				dbDisplayed = true
				assert.DeepEqual(t, fields, []string{containerID, serviceName, "Running"})
			case "words":
				wordsDisplayed = true
				assert.Check(t, strings.Contains(fields[3], ":8080->8080/tcp"))
				wordsURL = "http://" + strings.Replace(fields[3], "->8080/tcp", "", 1) + "/noun"
			case "web":
				webDisplayed = true
				assert.Check(t, strings.Contains(fields[3], ":80->80/tcp"))
				webURL = "http://" + strings.Replace(fields[3], "->80/tcp", "", 1)
			case "websecrets":
				secretsDisplayed = true
				assert.Check(t, strings.Contains(fields[3], ":90->90/tcp"))
				secretsURL = "http://" + strings.Replace(fields[3], "->90/tcp", "", 1)
			}
		}

		assert.Check(t, dbDisplayed)
		assert.Check(t, wordsDisplayed)
		assert.Check(t, webDisplayed)
		assert.Check(t, secretsDisplayed)
	})

	t.Run("compose ls", func(t *testing.T) {
		res := c.RunDockerCmd("compose", "ls", "--project-name", stack)
		lines := strings.Split(strings.TrimSpace(res.Stdout()), "\n")

		assert.Equal(t, 2, len(lines))
		fields := strings.Fields(lines[1])
		assert.Equal(t, 2, len(fields))
		assert.Equal(t, fields[0], stack)
		assert.Equal(t, "Running", fields[1])
	})

	t.Run("Words GET validating cross service connection", func(t *testing.T) {
		out := HTTPGetWithRetry(t, wordsURL, http.StatusOK, 5*time.Second, 240*time.Second)
		assert.Assert(t, strings.Contains(out, `"word":`))
	})

	t.Run("web app GET", func(t *testing.T) {
		out := HTTPGetWithRetry(t, webURL, http.StatusOK, 3*time.Second, 120*time.Second)
		assert.Assert(t, strings.Contains(out, "Docker Compose demo"))

		out = HTTPGetWithRetry(t, webURL+"/words/noun", http.StatusOK, 2*time.Second, 60*time.Second)
		assert.Assert(t, strings.Contains(out, `"word":`))
	})

	t.Run("access secret", func(t *testing.T) {
		out := HTTPGetWithRetry(t, secretsURL+"/mysecret1", http.StatusOK, 3*time.Second, 120*time.Second)
		out = strings.ReplaceAll(out, "\r", "")
		assert.Equal(t, out, "myPassword1\n")
	})

	t.Run("compose down", func(t *testing.T) {
		cmd := c.NewDockerCmd("compose", "down", "--project-name", stack)
		res := icmd.StartCmd(cmd)

		checkUp := func(t poll.LogT) poll.Result {
			out := res.Stdout()
			if !strings.Contains(out, "DeleteComplete") {
				return poll.Continue("current status \n%s\n", out)
			}
			return poll.Success()
		}
		poll.WaitOn(t, checkUp, poll.WithDelay(2*time.Second), poll.WithTimeout(60*time.Second))
	})
}

func setupTest(t *testing.T) (*E2eCLI, string) {
	startTime := strconv.Itoa(int(time.Now().UnixNano()))
	c := NewParallelE2eCLI(t, binDir)
	contextName := "e2e" + t.Name() + startTime
	stack := contextName
	t.Run("create context", func(t *testing.T) {
		localTestProfile := os.Getenv("TEST_AWS_PROFILE")
		var res *icmd.Result
		if localTestProfile != "" {
			res = c.RunDockerCmd("context", "create", "ecs", contextName, "--profile", localTestProfile)
		} else {
			region := os.Getenv("AWS_DEFAULT_REGION")
			secretKey := os.Getenv("AWS_SECRET_ACCESS_KEY")
			keyID := os.Getenv("AWS_ACCESS_KEY_ID")
			assert.Check(t, keyID != "")
			assert.Check(t, secretKey != "")
			assert.Check(t, region != "")
			res = c.RunDockerCmd("context", "create", "ecs", contextName, "--from-env")
		}
		res.Assert(t, icmd.Expected{Out: "Successfully created ecs context \"" + contextName + "\""})
		res = c.RunDockerCmd("context", "use", contextName)
		res.Assert(t, icmd.Expected{Out: contextName})
		res = c.RunDockerCmd("context", "ls")
		res.Assert(t, icmd.Expected{Out: contextName + " *"})
	})
	return c, stack
}
<commit_msg>Fix ECS flaky test and add bettre debug info in assertions<commit_after>/*
   Copyright 2020 Docker Compose CLI authors

   Licensed under the Apache License, Version 2.0 (the "License");
   you may not use this file except in compliance with the License.
   You may obtain a copy of the License at

       http://www.apache.org/licenses/LICENSE-2.0

   Unless required by applicable law or agreed to in writing, software
   distributed under the License is distributed on an "AS IS" BASIS,
   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
   See the License for the specific language governing permissions and
   limitations under the License.
*/

package main

import (
	"fmt"
	"io/ioutil"
	"net/http"
	"os"
	"path/filepath"
	"strconv"
	"strings"
	"testing"
	"time"

	"gotest.tools/v3/assert"
	"gotest.tools/v3/icmd"
	"gotest.tools/v3/poll"

	. "github.com/docker/compose-cli/tests/framework"
)

var binDir string

func TestMain(m *testing.M) {
	p, cleanup, err := SetupExistingCLI()
	if err != nil {
		fmt.Println(err)
		os.Exit(1)
	}
	binDir = p
	exitCode := m.Run()
	cleanup()
	os.Exit(exitCode)
}

func TestSecrets(t *testing.T) {
	cmd, testID := setupTest(t)
	secretName := "secret" + testID

	t.Run("create secret", func(t *testing.T) {
		secretFile := filepath.Join(cmd.BinDir, "secret.txt")
		err := ioutil.WriteFile(secretFile, []byte("pass1"), 0644)
		assert.Check(t, err == nil)
		res := cmd.RunDockerCmd("secret", "create", secretName, secretFile)
		assert.Check(t, strings.Contains(res.Stdout(), secretName), res.Stdout())
	})

	t.Run("list secrets", func(t *testing.T) {
		res := cmd.RunDockerCmd("secret", "list")
		if !strings.Contains(res.Stdout(), secretName) { // test sometimes fail, it seems things might need a bit of time on the AWS side, trying once morez
			time.Sleep(1 * time.Second)
			res = cmd.RunDockerCmd("secret", "list")
		}
		assert.Check(t, strings.Contains(res.Stdout(), secretName), res.Stdout())
	})

	t.Run("inspect secret", func(t *testing.T) {
		res := cmd.RunDockerCmd("secret", "inspect", secretName)
		assert.Check(t, strings.Contains(res.Stdout(), `"Name": "`+secretName+`"`), res.Stdout())
	})

	t.Run("rm secret", func(t *testing.T) {
		cmd.RunDockerCmd("secret", "rm", secretName)
		res := cmd.RunDockerCmd("secret", "list")
		assert.Check(t, !strings.Contains(res.Stdout(), secretName), res.Stdout())
	})
}

func TestCompose(t *testing.T) {
	c, stack := setupTest(t)

	t.Run("compose up", func(t *testing.T) {
		c.RunDockerCmd("compose", "up", "--project-name", stack, "-f", "../composefiles/ecs_e2e/multi_port_secrets.yaml")
	})

	var webURL, wordsURL, secretsURL string
	t.Run("compose ps", func(t *testing.T) {
		res := c.RunDockerCmd("compose", "ps", "--project-name", stack)
		fmt.Println(strings.TrimSpace(res.Stdout()))
		lines := strings.Split(strings.TrimSpace(res.Stdout()), "\n")

		assert.Equal(t, 5, len(lines))

		var dbDisplayed, wordsDisplayed, webDisplayed, secretsDisplayed bool
		for _, line := range lines {
			fields := strings.Fields(line)
			containerID := fields[0]
			serviceName := fields[1]
			switch serviceName {
			case "db":
				dbDisplayed = true
				assert.DeepEqual(t, fields, []string{containerID, serviceName, "Running"})
			case "words":
				wordsDisplayed = true
				assert.Check(t, strings.Contains(fields[3], ":8080->8080/tcp"))
				wordsURL = "http://" + strings.Replace(fields[3], "->8080/tcp", "", 1) + "/noun"
			case "web":
				webDisplayed = true
				assert.Check(t, strings.Contains(fields[3], ":80->80/tcp"))
				webURL = "http://" + strings.Replace(fields[3], "->80/tcp", "", 1)
			case "websecrets":
				secretsDisplayed = true
				assert.Check(t, strings.Contains(fields[3], ":90->90/tcp"))
				secretsURL = "http://" + strings.Replace(fields[3], "->90/tcp", "", 1)
			}
		}

		assert.Check(t, dbDisplayed)
		assert.Check(t, wordsDisplayed)
		assert.Check(t, webDisplayed)
		assert.Check(t, secretsDisplayed)
	})

	t.Run("compose ls", func(t *testing.T) {
		res := c.RunDockerCmd("compose", "ls", "--project-name", stack)
		lines := strings.Split(strings.TrimSpace(res.Stdout()), "\n")

		assert.Equal(t, 2, len(lines))
		fields := strings.Fields(lines[1])
		assert.Equal(t, 2, len(fields))
		assert.Equal(t, fields[0], stack)
		assert.Equal(t, "Running", fields[1])
	})

	t.Run("Words GET validating cross service connection", func(t *testing.T) {
		out := HTTPGetWithRetry(t, wordsURL, http.StatusOK, 5*time.Second, 240*time.Second)
		assert.Assert(t, strings.Contains(out, `"word":`))
	})

	t.Run("web app GET", func(t *testing.T) {
		out := HTTPGetWithRetry(t, webURL, http.StatusOK, 3*time.Second, 120*time.Second)
		assert.Assert(t, strings.Contains(out, "Docker Compose demo"))

		out = HTTPGetWithRetry(t, webURL+"/words/noun", http.StatusOK, 2*time.Second, 60*time.Second)
		assert.Assert(t, strings.Contains(out, `"word":`))
	})

	t.Run("access secret", func(t *testing.T) {
		out := HTTPGetWithRetry(t, secretsURL+"/mysecret1", http.StatusOK, 3*time.Second, 120*time.Second)
		out = strings.ReplaceAll(out, "\r", "")
		assert.Equal(t, out, "myPassword1\n")
	})

	t.Run("compose down", func(t *testing.T) {
		cmd := c.NewDockerCmd("compose", "down", "--project-name", stack)
		res := icmd.StartCmd(cmd)

		checkUp := func(t poll.LogT) poll.Result {
			out := res.Stdout()
			if !strings.Contains(out, "DeleteComplete") {
				return poll.Continue("current status \n%s\n", out)
			}
			return poll.Success()
		}
		poll.WaitOn(t, checkUp, poll.WithDelay(2*time.Second), poll.WithTimeout(60*time.Second))
	})
}

func setupTest(t *testing.T) (*E2eCLI, string) {
	startTime := strconv.Itoa(int(time.Now().UnixNano()))
	c := NewParallelE2eCLI(t, binDir)
	contextName := "e2e" + t.Name() + startTime
	stack := contextName
	t.Run("create context", func(t *testing.T) {
		localTestProfile := os.Getenv("TEST_AWS_PROFILE")
		var res *icmd.Result
		if localTestProfile != "" {
			res = c.RunDockerCmd("context", "create", "ecs", contextName, "--profile", localTestProfile)
		} else {
			region := os.Getenv("AWS_DEFAULT_REGION")
			secretKey := os.Getenv("AWS_SECRET_ACCESS_KEY")
			keyID := os.Getenv("AWS_ACCESS_KEY_ID")
			assert.Check(t, keyID != "")
			assert.Check(t, secretKey != "")
			assert.Check(t, region != "")
			res = c.RunDockerCmd("context", "create", "ecs", contextName, "--from-env")
		}
		res.Assert(t, icmd.Expected{Out: "Successfully created ecs context \"" + contextName + "\""})
		res = c.RunDockerCmd("context", "use", contextName)
		res.Assert(t, icmd.Expected{Out: contextName})
		res = c.RunDockerCmd("context", "ls")
		res.Assert(t, icmd.Expected{Out: contextName + " *"})
	})
	return c, stack
}
<|endoftext|>


<commit_before>package quic

import (
	"bytes"

	. "github.com/onsi/ginkgo"
	. "github.com/onsi/gomega"
)

var _ = Describe("AckFrame", func() {
	Context("when writing", func() {
		It("writes simple frames", func() {
			b := &bytes.Buffer{}
			(&AckFrame{
				Entropy:         2,
				LargestObserved: 1,
			}).Write(b)
			Expect(b.Bytes()).To(Equal([]byte{0x48, 0x02, 0x01, 0, 0, 0, 1, 0, 1, 0, 0, 0, 0, 0}))
		})
	})
})
<commit_msg>add tests for ACK frame parsing<commit_after>package quic

import (
	"bytes"

	. "github.com/onsi/ginkgo"
	. "github.com/onsi/gomega"
)

var _ = Describe("AckFrame", func() {
	Context("when parsing", func() {
		It("accepts sample frame", func() {
			b := bytes.NewReader([]byte{0x40, 0xA4, 0x03, 0x23, 0x45, 0x01, 0x02, 0xFF, 0xEE, 0xDD, 0xCC})
			frame, err := ParseAckFrame(b)
			Expect(err).ToNot(HaveOccurred())
			Expect(frame.Entropy).To(Equal(byte(0xA4)))
			Expect(frame.LargestObserved).To(Equal(uint64(0x03)))
			Expect(frame.DelayTime).To(Equal(uint16(0x4523)))
			Expect(b.Len()).To(Equal(0))
		})

		It("parses a frame with a 48 bit packet number", func() {
			b := bytes.NewReader([]byte{0x4C, 0xA4, 0x37, 0x13, 0xAD, 0xFB, 0xCA, 0xDE, 0x23, 0x45, 0x01, 0x02, 0xFF, 0xEE, 0xDD, 0xCC})
			frame, err := ParseAckFrame(b)
			Expect(err).ToNot(HaveOccurred())
			Expect(frame.LargestObserved).To(Equal(uint64(0xDECAFBAD1337)))
			Expect(b.Len()).To(Equal(0))
		})

		It("completely parses a frame with multiple timestamps", func() {
			b := bytes.NewReader([]byte{0x40, 0xA4, 0x03, 0x23, 0x45, 0x03, 0x02, 0xFF, 0xEE, 0xDD, 0xCC, 0x12, 0x34, 0x56, 0x78, 0x90, 0xA0})
			_, err := ParseAckFrame(b)
			Expect(err).ToNot(HaveOccurred())
			Expect(b.Len()).To(Equal(0))
		})
	})

	Context("when writing", func() {
		It("writes simple frames", func() {
			b := &bytes.Buffer{}
			(&AckFrame{
				Entropy:         2,
				LargestObserved: 1,
			}).Write(b)
			Expect(b.Bytes()).To(Equal([]byte{0x48, 0x02, 0x01, 0, 0, 0, 1, 0, 1, 0, 0, 0, 0, 0}))
		})
	})
})
<|endoftext|>


<commit_before>package acmeagent_test

import (
	"os"
	"path/filepath"
	"testing"

	"github.com/lestrrat/go-cloud-acmeagent"
	"github.com/lestrrat/go-cloud-acmeagent/gcp"
	"github.com/lestrrat/go-cloud-acmeagent/localfs"
	"github.com/stretchr/testify/assert"
	"golang.org/x/net/context"
	"golang.org/x/oauth2/google"
	"google.golang.org/api/dns/v1"
)

func TestAuthorizeGCP(t *testing.T) {
	email := os.Getenv("ACME_AGENT_TEST_EMAIL")
	if email == "" {
		t.Logf("ACME_AGENT_TEST_EMAIL environment variable is required for this test")
		return
	}
	domain := os.Getenv("ACME_AGENT_TEST_DOMAIN")
	if domain == "" {
		t.Logf("ACME_AGENT_TEST_DOMAIN environment variable is required for this test")
		return
	}
	gcpproj := os.Getenv("ACME_AGENT_TEST_GCP_PROJECT_ID")
	if gcpproj == "" {
		t.Logf("ACME_AGENT_TEST_GCP_PROJECT_ID environment variable is required for this test")
		return
	}
	gcpzone := os.Getenv("ACME_AGENT_TEST_GCP_ZONE_NAME")
	if gcpzone == "" {
		t.Logf("ACME_AGENT_TEST_GCP_ZONE_NAME environment variable is required for this test")
		return
	}

	cn := os.Getenv("ACME_AGENT_TEST_COMMON_NAME")
	if cn == "" {
		t.Logf("ACME_AGENT_TEST_COMMON_NAME environment variable is required for this test")
		return
	}

	wd, err := os.Getwd()
	if !assert.NoError(t, err, "Getting working directory should succeed") {
		return
	}

	store, err := localfs.New(localfs.StorageOptions{
		Root: filepath.Join(wd, "acme"),
		ID:   email,
	})
	if !assert.NoError(t, err, "Creating localfs state storage should succeed") {
		return
	}

	var authz acmeagent.Authorization
	if err := store.LoadAuthorization(domain, &authz); err == nil && !authz.IsExpired() {
		return // no auth necessary
	}

	// Get your challange fulfilling strategy ready. Here we're
	// getting an object that can create appropriate DNS entries
	// using Google CloudDNS to respond to dns-01 challenge
	ctx := context.Background()
	httpcl, err := google.DefaultClient(
		ctx,
		dns.NdevClouddnsReadwriteScope, // We need to be able to update CloudDNS
	)
	if !assert.NoError(t, err, "creating new Google oauth'ed client should succeed") {
		panic(err)
	}

	dnssvc, err := dns.New(httpcl)
	if !assert.NoError(t, err, "creating new DNS service should succeed") {
		return
	}

	// Tell the agent which challenges we can accept
	aa, err := acmeagent.New(acmeagent.AgentOptions{
		DNSCompleter: gcp.NewDNS(dnssvc, gcpproj, gcpzone),
		StateStorage: store,
	})

	var acct acmeagent.Account
	if err := store.LoadAccount(&acct); err != nil {
		if !assert.NoError(t, aa.Register(email), "Register should succeed") {
			return
		}
	}

	// With us so far? now fire the request, and let the authorization happen
	if !assert.NoError(t, aa.AuthorizeForDomain(domain), "authorize should succeed") {
		return
	}

	if !assert.NoError(t, aa.IssueCertificate(cn, domain, false), "IssueCertificate should succeed") {
		return
	}
}<commit_msg>see if this works<commit_after>package acmeagent_test

import (
	"os"
	"path/filepath"
	"testing"

	"github.com/lestrrat/go-cloud-acmeagent"
	"github.com/lestrrat/go-cloud-acmeagent/gcp"
	"github.com/lestrrat/go-cloud-acmeagent/localfs"
	"github.com/stretchr/testify/assert"
	"golang.org/x/net/context"
	"golang.org/x/oauth2/google"
	"google.golang.org/api/dns/v1"
)

func TestAuthorizeGCP(t *testing.T) {
	email := os.Getenv("ACME_AGENT_TEST_EMAIL")
	if email == "" {
		t.Logf("ACME_AGENT_TEST_EMAIL environment variable is required for this test")
		return
	}
	domain := os.Getenv("ACME_AGENT_TEST_DOMAIN")
	if domain == "" {
		t.Logf("ACME_AGENT_TEST_DOMAIN environment variable is required for this test")
		return
	}
	gcpproj := os.Getenv("ACME_AGENT_TEST_GCP_PROJECT_ID")
	if gcpproj == "" {
		t.Logf("ACME_AGENT_TEST_GCP_PROJECT_ID environment variable is required for this test")
		return
	}
	gcpzone := os.Getenv("ACME_AGENT_TEST_GCP_ZONE_NAME")
	if gcpzone == "" {
		t.Logf("ACME_AGENT_TEST_GCP_ZONE_NAME environment variable is required for this test")
		return
	}

	cn := os.Getenv("ACME_AGENT_TEST_COMMON_NAME")
	if cn == "" {
		t.Logf("ACME_AGENT_TEST_COMMON_NAME environment variable is required for this test")
		return
	}

	wd, err := os.Getwd()
	if !assert.NoError(t, err, "Getting working directory should succeed") {
		return
	}

	store, err := localfs.New(localfs.StorageOptions{
		Root: filepath.Join(wd, "acme"),
		ID:   email,
	})
	if !assert.NoError(t, err, "Creating localfs state storage should succeed") {
		return
	}

	var authz acmeagent.Authorization
	if err := store.LoadAuthorization(domain, &authz); err == nil && !authz.IsExpired() {
		return // no auth necessary
	}

	// Get your challange fulfilling strategy ready. Here we're
	// getting an object that can create appropriate DNS entries
	// using Google CloudDNS to respond to dns-01 challenge
	ctx := context.Background()
	httpcl, err := google.DefaultClient(
		ctx,
		dns.NdevClouddnsReadwriteScope, // We need to be able to update CloudDNS
	)
	if !assert.NoError(t, err, "creating new Google oauth'ed client should succeed") {
		panic(err)
	}

	dnssvc, err := dns.New(httpcl)
	if !assert.NoError(t, err, "creating new DNS service should succeed") {
		return
	}

	// Tell the agent which challenges we can accept
	aa, err := acmeagent.New(acmeagent.AgentOptions{
		DNSCompleter: gcp.NewDNS(dnssvc, gcpproj, gcpzone),
		StateStorage: store,
	})

	var acct acmeagent.Account
	if err := store.LoadAccount(&acct); err != nil {
		if !assert.NoError(t, aa.Register(email), "Register should succeed") {
			return
		}
	}

	// With us so far? now fire the request, and let the authorization happen
	if !assert.NoError(t, aa.AuthorizeForDomain(cn + "." + domain), "authorize should succeed") {
		return
	}

	if !assert.NoError(t, aa.IssueCertificate(cn, domain, false), "IssueCertificate should succeed") {
		return
	}
}<|endoftext|>


<commit_before>package main

import (
	"encoding/json"
	"flag"
	"fmt"
	"net"
	"os"
	"os/exec"
	"os/signal"
	"sync"
)

// AntConfig contains fields to pass to a sia-ant job runner.
type AntConfig struct {
	Jobs []string
}

// getAddrs returns n free listening addresses on localhost by leveraging the
// behaviour of net.Listen("localhost:0").
func getAddrs(n int) ([]string, error) {
	var addrs []string

	for i := 0; i < n; i++ {
		l, err := net.Listen("tcp", "localhost:0")
		if err != nil {
			return nil, err
		}
		defer l.Close()
		addrs = append(addrs, l.Addr().String())
	}
	return addrs, nil
}

// NewAnt spawns a new sia-ant process using os/exec.  The jobs defined by
// `jobs` are passed as flags to sia-ant.
func NewAnt(jobs []string) (*exec.Cmd, error) {
	var args []string
	for _, job := range jobs {
		args = append(args, "-"+job)
	}
	addrs, err := getAddrs(3)
	if err != nil {
		return nil, err
	}
	args = append(args, "-api-addr", addrs[0], "-rpc-addr", addrs[1], "-host-addr", addrs[2])
	cmd := exec.Command("sia-ant", args...)
	cmd.Stderr = os.Stderr
	if err := cmd.Start(); err != nil {
		return nil, err
	}
	return cmd, nil
}

func main() {
	configPath := flag.String("config", "config.json", "path to the sia-antfarm configuration file")

	flag.Parse()

	// Read and decode the sia-antfarm configuration file.
	var antConfigs []AntConfig
	f, err := os.Open(*configPath)
	if err != nil {
		fmt.Fprintf(os.Stderr, "error opening %v: %v\n", *configPath, err)
		os.Exit(1)
	}

	if err = json.NewDecoder(f).Decode(&antConfigs); err != nil {
		fmt.Fprintf(os.Stderr, "error decoding %v: %v\n", *configPath, err)
		os.Exit(1)
	}
	f.Close()

	// Start each sia-ant process with its assigned jobs from the config file.
	var wg sync.WaitGroup
	var antCommands []*exec.Cmd
	for antindex, config := range antConfigs {
		fmt.Printf("Starting ant %v with jobs %v\n", antindex, config.Jobs)
		antcmd, err := NewAnt(config.Jobs)
		if err != nil {
			fmt.Fprintf(os.Stderr, "error starting ant %v: %v\n", antindex, err)
			os.Exit(1)
		}
		defer func() {
			antcmd.Process.Signal(os.Interrupt)
		}()
		wg.Add(1)
		antCommands = append(antCommands, antcmd)
		go func() {
			antcmd.Wait()
			wg.Done()
		}()
	}

	sigchan := make(chan os.Signal, 1)
	signal.Notify(sigchan, os.Interrupt)

	go func() {
		<-sigchan
		fmt.Println("Caught quit signal, stopping all ants...")
		for _, cmd := range antCommands {
			cmd.Process.Signal(os.Interrupt)
		}
	}()

	fmt.Printf("Finished.  Running sia-antfarm with %v ants.\n", len(antConfigs))

	wg.Wait()
}
<commit_msg>add optional APIAddr, RPCAddr, HostAddr config fields<commit_after>package main

import (
	"encoding/json"
	"flag"
	"fmt"
	"net"
	"os"
	"os/exec"
	"os/signal"
	"sync"
)

// AntConfig contains fields to pass to a sia-ant job runner.
type AntConfig struct {
	APIAddr  string `json:",omitempty"`
	RPCAddr  string `json:",omitempty"`
	HostAddr string `json:",omitempty"`
	Jobs     []string
}

// getAddrs returns n free listening addresses on localhost by leveraging the
// behaviour of net.Listen("localhost:0").
func getAddrs(n int) ([]string, error) {
	var addrs []string

	for i := 0; i < n; i++ {
		l, err := net.Listen("tcp", "localhost:0")
		if err != nil {
			return nil, err
		}
		defer l.Close()
		addrs = append(addrs, l.Addr().String())
	}
	return addrs, nil
}

// NewAnt spawns a new sia-ant process using os/exec.  The jobs defined by
// `jobs` are passed as flags to sia-ant.  If APIAddr, RPCAddr, or HostAddr are
// defined in `config`, they will be passed to the Ant.  Otherwise, the Ant
// will be passed 3 unused addresses.
func NewAnt(config AntConfig) (*exec.Cmd, error) {
	var args []string
	for _, job := range config.Jobs {
		args = append(args, "-"+job)
	}

	// Automatically generate 3 free operating system ports for the Ant's api,
	// rpc, and host addresses
	addrs, err := getAddrs(3)
	if err != nil {
		return nil, err
	}
	apiaddr := addrs[0]
	rpcaddr := addrs[1]
	hostaddr := addrs[2]

	// Override the automatically generated addresses with the ones in AntConfig,
	// if they exist.
	if config.APIAddr != "" {
		apiaddr = config.APIAddr
	}
	if config.RPCAddr != "" {
		rpcaddr = config.RPCAddr
	}
	if config.HostAddr != "" {
		hostaddr = config.HostAddr
	}

	args = append(args, "-api-addr", apiaddr, "-rpc-addr", rpcaddr, "-host-addr", hostaddr)
	cmd := exec.Command("sia-ant", args...)
	cmd.Stderr = os.Stderr
	if err := cmd.Start(); err != nil {
		return nil, err
	}
	return cmd, nil
}

func main() {
	configPath := flag.String("config", "config.json", "path to the sia-antfarm configuration file")

	flag.Parse()

	// Read and decode the sia-antfarm configuration file.
	var antConfigs []AntConfig
	f, err := os.Open(*configPath)
	if err != nil {
		fmt.Fprintf(os.Stderr, "error opening %v: %v\n", *configPath, err)
		os.Exit(1)
	}

	if err = json.NewDecoder(f).Decode(&antConfigs); err != nil {
		fmt.Fprintf(os.Stderr, "error decoding %v: %v\n", *configPath, err)
		os.Exit(1)
	}
	f.Close()

	// Start each sia-ant process with its assigned jobs from the config file.
	var wg sync.WaitGroup
	var antCommands []*exec.Cmd
	for antindex, config := range antConfigs {
		fmt.Printf("Starting ant %v with jobs %v\n", antindex, config.Jobs)
		antcmd, err := NewAnt(config)
		if err != nil {
			fmt.Fprintf(os.Stderr, "error starting ant %v: %v\n", antindex, err)
			os.Exit(1)
		}
		defer func() {
			antcmd.Process.Signal(os.Interrupt)
		}()
		wg.Add(1)
		antCommands = append(antCommands, antcmd)
		go func() {
			antcmd.Wait()
			wg.Done()
		}()
	}

	sigchan := make(chan os.Signal, 1)
	signal.Notify(sigchan, os.Interrupt)

	go func() {
		<-sigchan
		fmt.Println("Caught quit signal, stopping all ants...")
		for _, cmd := range antCommands {
			cmd.Process.Signal(os.Interrupt)
		}
	}()

	fmt.Printf("Finished.  Running sia-antfarm with %v ants.\n", len(antConfigs))

	wg.Wait()
}
<|endoftext|>


<commit_before>package main

import (
	"io/ioutil"

	"github.com/pkg/errors"
	"gopkg.in/yaml.v2"
)

// Config holds global configuration, defaults are provided in main.
// Agent config is populated from startup flag.
type Config struct {
	Environment     string `m:"Environment"`
	LogLevel        string `m:"LogLevel"`
	Port            int    `m:"Port"`
	Nats            string `m:"Nats"`
	CollectorConfig string `m:"CollectorConfig"`
}

type CollectorConfig struct {
	Docker  ApiCollectorConfig `json:"docker" yaml:"docker"`
	Consul  ApiCollectorConfig `json:"consul" yaml:"consul"`
	VSphere ApiCollectorConfig `json:"vsphere" yaml:"vsphere"`
}

type ApiCollectorConfig struct {
	Endpoints []string `json:"endpoints" yaml:"endpoints"`
	Include   []string `json:"include" yaml:"include"`
	Exclude   []string `json:"exclude" yaml:"exclude"`
	Cron      string   `json:"cron" yaml:"cron"`
}

func LoadCollectorConfig(path string) (*CollectorConfig, error) {
	cfg := &CollectorConfig{}

	if len(path) < 1 {
		return nil, errors.Errorf("Collector config %v not found", path)
	}

	data, err := ioutil.ReadFile(path)
	if err != nil {
		return nil, errors.Wrapf(err, "Reading %v failed", path)
	}

	if err := yaml.Unmarshal(data, cfg); err != nil {
		return nil, errors.Wrapf(err, "Parsing %v failed", path)
	}

	return cfg, nil
}
<commit_msg>clustered services collector config<commit_after>package main

import (
	"io/ioutil"

	"github.com/pkg/errors"
	"gopkg.in/yaml.v2"
)

// Config holds global configuration, defaults are provided in main.
// Agent config is populated from startup flag.
type Config struct {
	Environment     string `m:"Environment"`
	LogLevel        string `m:"LogLevel"`
	Port            int    `m:"Port"`
	Nats            string `m:"Nats"`
	CollectorConfig string `m:"CollectorConfig"`
}

type CollectorConfig struct {
	Docker  ApiCollectorConfig     `json:"docker" yaml:"docker"`
	Consul  ApiCollectorConfig     `json:"consul" yaml:"consul"`
	VSphere ApiCollectorConfig     `json:"vsphere" yaml:"vsphere"`
	Cluster ClusterCollectorConfig `json:"cluster" yaml:"cluster"`
}

type ApiCollectorConfig struct {
	Endpoints []string `json:"endpoints" yaml:"endpoints"`
	Include   []string `json:"include" yaml:"include"`
	Exclude   []string `json:"exclude" yaml:"exclude"`
	Cron      string   `json:"cron" yaml:"cron"`
}

type ClusterCollectorConfig struct {
	Cron     string                   `json:"cron" yaml:"cron"`
	Services []ClusteredServiceConfig `json:"services" yaml:"services"`
}

type ClusteredServiceConfig struct {
	Name      string   `json:"name" yaml:"name"`
	Endpoints []string `json:"endpoints" yaml:"endpoints"`
}

func LoadCollectorConfig(path string) (*CollectorConfig, error) {
	cfg := &CollectorConfig{}

	if len(path) < 1 {
		return nil, errors.Errorf("Collector config %v not found", path)
	}

	data, err := ioutil.ReadFile(path)
	if err != nil {
		return nil, errors.Wrapf(err, "Reading %v failed", path)
	}

	if err := yaml.Unmarshal(data, cfg); err != nil {
		return nil, errors.Wrapf(err, "Parsing %v failed", path)
	}

	return cfg, nil
}
<|endoftext|>


<commit_before>/*
Copyright 2017 The Kubernetes Authors.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package main

import (
	"context"
	"flag"
	"net/http"
	"net/url"
	"os"
	"os/signal"
	"strconv"
	"syscall"
	"time"

	"github.com/sirupsen/logrus"

	"k8s.io/test-infra/prow/config"
	"k8s.io/test-infra/prow/flagutil"
	"k8s.io/test-infra/prow/git"
	"k8s.io/test-infra/prow/github"
	"k8s.io/test-infra/prow/kube"
	"k8s.io/test-infra/prow/logrusutil"
	"k8s.io/test-infra/prow/metrics"
	"k8s.io/test-infra/prow/tide"
)

type options struct {
	port int

	dryRun  bool
	runOnce bool
	deckURL string

	configPath    string
	jobConfigPath string
	cluster       string

	githubEndpoint  flagutil.Strings
	githubTokenFile string
}

func gatherOptions() options {
	o := options{
		githubEndpoint: flagutil.NewStrings("https://api.github.com"),
	}
	flag.IntVar(&o.port, "port", 8888, "Port to listen on.")

	flag.BoolVar(&o.dryRun, "dry-run", true, "Whether to mutate any real-world state.")
	flag.BoolVar(&o.runOnce, "run-once", false, "If true, run only once then quit.")
	flag.StringVar(&o.deckURL, "deck-url", "", "Deck URL for read-only access to the cluster.")

	flag.StringVar(&o.configPath, "config-path", "/etc/config/config.yaml", "Path to config.yaml.")
	flag.StringVar(&o.jobConfigPath, "job-config-path", "", "Path to prow job configs.")
	flag.StringVar(&o.cluster, "cluster", "", "Path to kube.Cluster YAML file. If empty, uses the local cluster.")

	flag.Var(&o.githubEndpoint, "github-endpoint", "GitHub's API endpoint.")
	flag.StringVar(&o.githubTokenFile, "github-token-file", "/etc/github/oauth", "Path to the file containing the GitHub OAuth token.")

	flag.Parse()
	return o
}

func main() {
	o := gatherOptions()

	logrus.SetFormatter(
		logrusutil.NewDefaultFieldsFormatter(nil, logrus.Fields{"component": "tide"}),
	)

	configAgent := &config.Agent{}
	if err := configAgent.Start(o.configPath, o.jobConfigPath); err != nil {
		logrus.WithError(err).Fatal("Error starting config agent.")
	}

	var err error
	for _, ep := range o.githubEndpoint.Strings() {
		_, err = url.ParseRequestURI(ep)
		if err != nil {
			logrus.WithError(err).Fatalf("Invalid --endpoint URL %q.", ep)
		}
	}

	secretAgent := &config.SecretAgent{}
	if err := secretAgent.Start([]string{o.githubTokenFile}); err != nil {
		logrus.WithError(err).Fatal("Error starting secrets agent.")
	}

	var ghcSync, ghcStatus *github.Client
	var kc *kube.Client
	if o.dryRun {
		ghcSync = github.NewDryRunClient(secretAgent.GetTokenGenerator(o.githubTokenFile), o.githubEndpoint.Strings()...)
		ghcStatus = github.NewDryRunClient(secretAgent.GetTokenGenerator(o.githubTokenFile), o.githubEndpoint.Strings()...)
		if o.deckURL == "" {
			logrus.Fatal("no deck URL was given for read-only ProwJob access")
		}
		kc = kube.NewFakeClient(o.deckURL)
	} else {
		ghcSync = github.NewClient(secretAgent.GetTokenGenerator(o.githubTokenFile), o.githubEndpoint.Strings()...)
		ghcStatus = github.NewClient(secretAgent.GetTokenGenerator(o.githubTokenFile), o.githubEndpoint.Strings()...)
		if o.cluster == "" {
			kc, err = kube.NewClientInCluster(configAgent.Config().ProwJobNamespace)
			if err != nil {
				logrus.WithError(err).Fatal("Error getting kube client.")
			}
		} else {
			kc, err = kube.NewClientFromFile(o.cluster, configAgent.Config().ProwJobNamespace)
			if err != nil {
				logrus.WithError(err).Fatal("Error getting kube client.")
			}
		}
	}
	// The sync loop should be allowed more tokens than the status loop because
	// it has to list all PRs in the pool every loop while the status loop only
	// has to list changed PRs every loop.
	// The sync loop should have a much lower burst allowance than the status
	// loop which may need to update many statuses upon restarting Tide after
	// changing the context format or starting Tide on a new repo.
	ghcSync.Throttle(800, 20)
	ghcStatus.Throttle(400, 200)

	gc, err := git.NewClient()
	if err != nil {
		logrus.WithError(err).Fatal("Error getting git client.")
	}
	defer gc.Clean()
	// Get the bot's name in order to set credentials for the git client.
	botName, err := ghcSync.BotName()
	if err != nil {
		logrus.WithError(err).Fatal("Error getting bot name.")
	}
	gc.SetCredentials(botName, secretAgent.GetTokenGenerator(o.githubTokenFile))

	c := tide.NewController(ghcSync, ghcStatus, kc, configAgent, gc, nil)
	defer c.Shutdown()
	server := &http.Server{Addr: ":" + strconv.Itoa(o.port), Handler: c}

	// Push metrics to the configured prometheus pushgateway endpoint.
	pushGateway := configAgent.Config().PushGateway
	if pushGateway.Endpoint != "" {
		go metrics.PushMetrics("tide", pushGateway.Endpoint, pushGateway.Interval)
	}

	start := time.Now()
	sync(c)
	if o.runOnce {
		return
	}
	go func() {
		sig := make(chan os.Signal, 1)
		signal.Notify(sig, os.Interrupt, syscall.SIGTERM)
		for {
			select {
			case <-time.After(time.Until(start.Add(configAgent.Config().Tide.SyncPeriod))):
				start = time.Now()
				sync(c)
			case <-sig:
				logrus.Info("Tide is shutting down...")
				// Shutdown the http server with a 10s timeout then return to execute
				// defered c.Shutdown()
				ctx, cancel := context.WithTimeout(context.Background(), time.Second*10)
				defer cancel() // frees ctx resources
				server.Shutdown(ctx)
				return
			}
		}
	}()
	logrus.WithError(server.ListenAndServe()).Warn("Tide HTTP server stopped.")
}

func sync(c *tide.Controller) {
	if err := c.Sync(); err != nil {
		logrus.WithError(err).Error("Error syncing.")
	}
}
<commit_msg>Make Tide GitHub client throttling configurable.<commit_after>/*
Copyright 2017 The Kubernetes Authors.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package main

import (
	"context"
	"flag"
	"net/http"
	"net/url"
	"os"
	"os/signal"
	"strconv"
	"syscall"
	"time"

	"github.com/sirupsen/logrus"

	"k8s.io/test-infra/prow/config"
	"k8s.io/test-infra/prow/flagutil"
	"k8s.io/test-infra/prow/git"
	"k8s.io/test-infra/prow/github"
	"k8s.io/test-infra/prow/kube"
	"k8s.io/test-infra/prow/logrusutil"
	"k8s.io/test-infra/prow/metrics"
	"k8s.io/test-infra/prow/tide"
)

type options struct {
	port int

	dryRun  bool
	runOnce bool
	deckURL string

	configPath    string
	jobConfigPath string
	cluster       string

	syncThrottle   int
	statusThrottle int

	githubEndpoint  flagutil.Strings
	githubTokenFile string
}

func gatherOptions() options {
	o := options{
		githubEndpoint: flagutil.NewStrings("https://api.github.com"),
	}
	flag.IntVar(&o.port, "port", 8888, "Port to listen on.")

	flag.BoolVar(&o.dryRun, "dry-run", true, "Whether to mutate any real-world state.")
	flag.BoolVar(&o.runOnce, "run-once", false, "If true, run only once then quit.")
	flag.StringVar(&o.deckURL, "deck-url", "", "Deck URL for read-only access to the cluster.")

	flag.StringVar(&o.configPath, "config-path", "/etc/config/config.yaml", "Path to config.yaml.")
	flag.StringVar(&o.jobConfigPath, "job-config-path", "", "Path to prow job configs.")
	flag.StringVar(&o.cluster, "cluster", "", "Path to kube.Cluster YAML file. If empty, uses the local cluster.")

	flag.IntVar(&o.syncThrottle, "sync-hourly-tokens", 800, "The maximum number of tokens per hour to be used by the sync controller.")
	flag.IntVar(&o.statusThrottle, "status-hourly-tokens", 400, "The maximum number of tokens per hour to be used by the status controller.")

	flag.Var(&o.githubEndpoint, "github-endpoint", "GitHub's API endpoint.")
	flag.StringVar(&o.githubTokenFile, "github-token-file", "/etc/github/oauth", "Path to the file containing the GitHub OAuth token.")

	flag.Parse()
	return o
}

func main() {
	o := gatherOptions()

	logrus.SetFormatter(
		logrusutil.NewDefaultFieldsFormatter(nil, logrus.Fields{"component": "tide"}),
	)

	configAgent := &config.Agent{}
	if err := configAgent.Start(o.configPath, o.jobConfigPath); err != nil {
		logrus.WithError(err).Fatal("Error starting config agent.")
	}

	var err error
	for _, ep := range o.githubEndpoint.Strings() {
		_, err = url.ParseRequestURI(ep)
		if err != nil {
			logrus.WithError(err).Fatalf("Invalid --endpoint URL %q.", ep)
		}
	}

	secretAgent := &config.SecretAgent{}
	if err := secretAgent.Start([]string{o.githubTokenFile}); err != nil {
		logrus.WithError(err).Fatal("Error starting secrets agent.")
	}

	var ghcSync, ghcStatus *github.Client
	var kc *kube.Client
	if o.dryRun {
		ghcSync = github.NewDryRunClient(secretAgent.GetTokenGenerator(o.githubTokenFile), o.githubEndpoint.Strings()...)
		ghcStatus = github.NewDryRunClient(secretAgent.GetTokenGenerator(o.githubTokenFile), o.githubEndpoint.Strings()...)
		if o.deckURL == "" {
			logrus.Fatal("no deck URL was given for read-only ProwJob access")
		}
		kc = kube.NewFakeClient(o.deckURL)
	} else {
		ghcSync = github.NewClient(secretAgent.GetTokenGenerator(o.githubTokenFile), o.githubEndpoint.Strings()...)
		ghcStatus = github.NewClient(secretAgent.GetTokenGenerator(o.githubTokenFile), o.githubEndpoint.Strings()...)
		if o.cluster == "" {
			kc, err = kube.NewClientInCluster(configAgent.Config().ProwJobNamespace)
			if err != nil {
				logrus.WithError(err).Fatal("Error getting kube client.")
			}
		} else {
			kc, err = kube.NewClientFromFile(o.cluster, configAgent.Config().ProwJobNamespace)
			if err != nil {
				logrus.WithError(err).Fatal("Error getting kube client.")
			}
		}
	}
	// The sync loop should be allowed more tokens than the status loop because
	// it has to list all PRs in the pool every loop while the status loop only
	// has to list changed PRs every loop.
	// The sync loop should have a much lower burst allowance than the status
	// loop which may need to update many statuses upon restarting Tide after
	// changing the context format or starting Tide on a new repo.
	ghcSync.Throttle(o.syncThrottle, 3*tokensPerIteration(o.syncThrottle, configAgent.Config().Tide.SyncPeriod))
	ghcStatus.Throttle(o.statusThrottle, o.statusThrottle/2)

	gc, err := git.NewClient()
	if err != nil {
		logrus.WithError(err).Fatal("Error getting git client.")
	}
	defer gc.Clean()
	// Get the bot's name in order to set credentials for the git client.
	botName, err := ghcSync.BotName()
	if err != nil {
		logrus.WithError(err).Fatal("Error getting bot name.")
	}
	gc.SetCredentials(botName, secretAgent.GetTokenGenerator(o.githubTokenFile))

	c := tide.NewController(ghcSync, ghcStatus, kc, configAgent, gc, nil)
	defer c.Shutdown()
	server := &http.Server{Addr: ":" + strconv.Itoa(o.port), Handler: c}

	// Push metrics to the configured prometheus pushgateway endpoint.
	pushGateway := configAgent.Config().PushGateway
	if pushGateway.Endpoint != "" {
		go metrics.PushMetrics("tide", pushGateway.Endpoint, pushGateway.Interval)
	}

	start := time.Now()
	sync(c)
	if o.runOnce {
		return
	}
	go func() {
		sig := make(chan os.Signal, 1)
		signal.Notify(sig, os.Interrupt, syscall.SIGTERM)
		for {
			select {
			case <-time.After(time.Until(start.Add(configAgent.Config().Tide.SyncPeriod))):
				start = time.Now()
				sync(c)
			case <-sig:
				logrus.Info("Tide is shutting down...")
				// Shutdown the http server with a 10s timeout then return to execute
				// defered c.Shutdown()
				ctx, cancel := context.WithTimeout(context.Background(), time.Second*10)
				defer cancel() // frees ctx resources
				server.Shutdown(ctx)
				return
			}
		}
	}()
	logrus.WithError(server.ListenAndServe()).Warn("Tide HTTP server stopped.")
}

func sync(c *tide.Controller) {
	if err := c.Sync(); err != nil {
		logrus.WithError(err).Error("Error syncing.")
	}
}

func tokensPerIteration(hourlyTokens int, iterPeriod time.Duration) int {
	tokenRate := float64(hourlyTokens) / float64(time.Hour)
	return int(tokenRate * float64(iterPeriod))
}
<|endoftext|>


<commit_before>package agent

import (
	"errors"
	"fmt"
	"os"
	"strings"

	"github.com/aws/aws-sdk-go/aws"
	"github.com/aws/aws-sdk-go/aws/credentials"
	"github.com/aws/aws-sdk-go/aws/credentials/stscreds"
	"github.com/aws/aws-sdk-go/aws/defaults"
	"github.com/aws/aws-sdk-go/aws/session"
	"github.com/aws/aws-sdk-go/service/s3"
	"github.com/aws/aws-sdk-go/service/s3/s3manager"
	"github.com/aws/aws-sdk-go/service/sts"
	"github.com/buildkite/agent/v3/logger"
)

var regionHintEnvVar = "BUILDKITE_S3_DEFAULT_REGION"

type credentialsProvider struct {
	retrieved bool
}

func (e *credentialsProvider) Retrieve() (creds credentials.Value, err error) {
	e.retrieved = false

	creds.AccessKeyID = os.Getenv("BUILDKITE_S3_ACCESS_KEY_ID")
	if creds.AccessKeyID == "" {
		creds.AccessKeyID = os.Getenv("BUILDKITE_S3_ACCESS_KEY")
	}

	creds.SecretAccessKey = os.Getenv("BUILDKITE_S3_SECRET_ACCESS_KEY")
	if creds.SecretAccessKey == "" {
		creds.SecretAccessKey = os.Getenv("BUILDKITE_S3_SECRET_KEY")
	}

	creds.SessionToken = os.Getenv("BUILDKITE_S3_SESSION_TOKEN")

	if creds.AccessKeyID == "" {
		err = errors.New("BUILDKITE_S3_ACCESS_KEY_ID or BUILDKITE_S3_ACCESS_KEY not found in environment")
	}
	if creds.SecretAccessKey == "" {
		err = errors.New("BUILDKITE_S3_SECRET_ACCESS_KEY or BUILDKITE_S3_SECRET_KEY not found in environment")
	}

	e.retrieved = true
	return
}

func (e *credentialsProvider) IsExpired() bool {
	return !e.retrieved
}

func awsS3Session(region string) (*session.Session, error) {
	// Chicken and egg... but this is kinda how they do it in the sdk
	sess, err := session.NewSession()
	if err != nil {
		return nil, err
	}

	sess.Config.Region = aws.String(region)

	sess.Config.Credentials = credentials.NewChainCredentials(
		[]credentials.Provider{
			&credentialsProvider{},
			&credentials.EnvProvider{},
			webIdentityRoleProvider(sess),
			// EC2 and ECS meta-data providers
			defaults.RemoteCredProvider(*sess.Config, sess.Handlers),
		})

	return sess, nil
}

func webIdentityRoleProvider(sess *session.Session) *stscreds.WebIdentityRoleProvider {
	return stscreds.NewWebIdentityRoleProvider(
		sts.New(sess),
		os.Getenv("AWS_ROLE_ARN"),
		os.Getenv("AWS_ROLE_SESSION_NAME"),
		os.Getenv("AWS_WEB_IDENTITY_TOKEN_FILE"),
	)
}

func newS3Client(l logger.Logger, bucket string) (*s3.S3, error) {
	var sess *session.Session

	regionHint := os.Getenv(regionHintEnvVar)
	if regionHint != "" {
		// If there is a region hint provided, we use it unconditionally
		session, err := awsS3Session(regionHint)
		if err != nil {
			return nil, fmt.Errorf("Could not load the AWS SDK config (%v)", err)
		}

		sess = session
	} else {
		// Otherwise, use the current region (or a guess) to dynamically find
		// where the bucket lives.
		region, err := awsRegion()
		if err != nil {
			region = "us-east-1"
		}

		l.Debug("Discovered current region as %q\n", region)

		// Using the guess region, construct a session and ask that region where the
		// bucket lives
		session, err := awsS3Session(region)
		if err != nil {
			return nil, fmt.Errorf("Could not load the AWS SDK config (%v)", err)
		}

		bucketRegion, bucketRegionErr := s3manager.GetBucketRegion(aws.BackgroundContext(), sess, bucket, region)
		if bucketRegionErr == nil && bucketRegion != "" {
			l.Debug("Discovered %q bucket region as %q\n", bucket, bucketRegion)
			session.Config.Region = &bucketRegion
		} else {
			l.Error("Could not discover region for bucket %q. Using the %q region as a fallback, if this is not correct configure a bucket region using the %q environment variable. (%v)\n", bucket, *sess.Config.Region, regionHintEnvVar, err)
		}

		sess = session
	}

	l.Debug("Testing AWS S3 credentials for bucket %q in region %q...", bucket, *sess.Config.Region)

	s3client := s3.New(sess)

	// Test the authentication by trying to list the first 0 objects in the bucket.
	_, err := s3client.ListObjects(&s3.ListObjectsInput{
		Bucket:  aws.String(bucket),
		MaxKeys: aws.Int64(0),
	})
	if err != nil {
		if err == credentials.ErrNoValidProvidersFoundInChain {
			hasProxy := os.Getenv("HTTP_PROXY") != "" || os.Getenv("HTTPS_PROXY") != ""
			hasNoProxyIdmsException := strings.Contains(os.Getenv("NO_PROXY"), "169.254.169.254")

			errorTitle := "Could not authenticate with AWS S3 using any of the included credential providers."

			if hasProxy && !hasNoProxyIdmsException {
				return nil, fmt.Errorf("%s Your HTTP proxy settings do not grant a NO_PROXY=169.254.169.254 exemption for the instance metadata service, instance profile credentials may not be retrievable via your HTTP proxy.", errorTitle)
			}

			return nil, fmt.Errorf("%s You can authenticate by setting Buildkite environment variables (BUILDKITE_S3_ACCESS_KEY_ID, BUILDKITE_S3_SECRET_ACCESS_KEY), AWS environment variables (AWS_ACCESS_KEY_ID, AWS_SECRET_ACCESS_KEY), Web Identity environment variables (AWS_ROLE_ARN, AWS_ROLE_SESSION_NAME, AWS_WEB_IDENTITY_TOKEN_FILE), or if running on AWS EC2 ensuring network access to the EC2 Instance Metadata Service to use an instance profile’s IAM Role credentials.", errorTitle)
		}
		return nil, fmt.Errorf("Could not s3:ListObjects in your AWS S3 bucket %q in region %q: (%s)", bucket, *sess.Config.Region, err.Error())
	}

	return s3client, nil
}
<commit_msg>Fix bucket region discovery to use non-nil variable<commit_after>package agent

import (
	"errors"
	"fmt"
	"os"
	"strings"

	"github.com/aws/aws-sdk-go/aws"
	"github.com/aws/aws-sdk-go/aws/credentials"
	"github.com/aws/aws-sdk-go/aws/credentials/stscreds"
	"github.com/aws/aws-sdk-go/aws/defaults"
	"github.com/aws/aws-sdk-go/aws/session"
	"github.com/aws/aws-sdk-go/service/s3"
	"github.com/aws/aws-sdk-go/service/s3/s3manager"
	"github.com/aws/aws-sdk-go/service/sts"
	"github.com/buildkite/agent/v3/logger"
)

var regionHintEnvVar = "BUILDKITE_S3_DEFAULT_REGION"

type credentialsProvider struct {
	retrieved bool
}

func (e *credentialsProvider) Retrieve() (creds credentials.Value, err error) {
	e.retrieved = false

	creds.AccessKeyID = os.Getenv("BUILDKITE_S3_ACCESS_KEY_ID")
	if creds.AccessKeyID == "" {
		creds.AccessKeyID = os.Getenv("BUILDKITE_S3_ACCESS_KEY")
	}

	creds.SecretAccessKey = os.Getenv("BUILDKITE_S3_SECRET_ACCESS_KEY")
	if creds.SecretAccessKey == "" {
		creds.SecretAccessKey = os.Getenv("BUILDKITE_S3_SECRET_KEY")
	}

	creds.SessionToken = os.Getenv("BUILDKITE_S3_SESSION_TOKEN")

	if creds.AccessKeyID == "" {
		err = errors.New("BUILDKITE_S3_ACCESS_KEY_ID or BUILDKITE_S3_ACCESS_KEY not found in environment")
	}
	if creds.SecretAccessKey == "" {
		err = errors.New("BUILDKITE_S3_SECRET_ACCESS_KEY or BUILDKITE_S3_SECRET_KEY not found in environment")
	}

	e.retrieved = true
	return
}

func (e *credentialsProvider) IsExpired() bool {
	return !e.retrieved
}

func awsS3Session(region string) (*session.Session, error) {
	// Chicken and egg... but this is kinda how they do it in the sdk
	sess, err := session.NewSession()
	if err != nil {
		return nil, err
	}

	sess.Config.Region = aws.String(region)

	sess.Config.Credentials = credentials.NewChainCredentials(
		[]credentials.Provider{
			&credentialsProvider{},
			&credentials.EnvProvider{},
			webIdentityRoleProvider(sess),
			// EC2 and ECS meta-data providers
			defaults.RemoteCredProvider(*sess.Config, sess.Handlers),
		})

	return sess, nil
}

func webIdentityRoleProvider(sess *session.Session) *stscreds.WebIdentityRoleProvider {
	return stscreds.NewWebIdentityRoleProvider(
		sts.New(sess),
		os.Getenv("AWS_ROLE_ARN"),
		os.Getenv("AWS_ROLE_SESSION_NAME"),
		os.Getenv("AWS_WEB_IDENTITY_TOKEN_FILE"),
	)
}

func newS3Client(l logger.Logger, bucket string) (*s3.S3, error) {
	var sess *session.Session

	regionHint := os.Getenv(regionHintEnvVar)
	if regionHint != "" {
		// If there is a region hint provided, we use it unconditionally
		session, err := awsS3Session(regionHint)
		if err != nil {
			return nil, fmt.Errorf("Could not load the AWS SDK config (%v)", err)
		}

		sess = session
	} else {
		// Otherwise, use the current region (or a guess) to dynamically find
		// where the bucket lives.
		region, err := awsRegion()
		if err != nil {
			region = "us-east-1"
		}

		l.Debug("Discovered current region as %q\n", region)

		// Using the guess region, construct a session and ask that region where the
		// bucket lives
		session, err := awsS3Session(region)
		if err != nil {
			return nil, fmt.Errorf("Could not load the AWS SDK config (%v)", err)
		}

		bucketRegion, bucketRegionErr := s3manager.GetBucketRegion(aws.BackgroundContext(), session, bucket, region)
		if bucketRegionErr == nil && bucketRegion != "" {
			l.Debug("Discovered %q bucket region as %q\n", bucket, bucketRegion)
			session.Config.Region = &bucketRegion
		} else {
			l.Error("Could not discover region for bucket %q. Using the %q region as a fallback, if this is not correct configure a bucket region using the %q environment variable. (%v)\n", bucket, *sess.Config.Region, regionHintEnvVar, err)
		}

		sess = session
	}

	l.Debug("Testing AWS S3 credentials for bucket %q in region %q...", bucket, *sess.Config.Region)

	s3client := s3.New(sess)

	// Test the authentication by trying to list the first 0 objects in the bucket.
	_, err := s3client.ListObjects(&s3.ListObjectsInput{
		Bucket:  aws.String(bucket),
		MaxKeys: aws.Int64(0),
	})
	if err != nil {
		if err == credentials.ErrNoValidProvidersFoundInChain {
			hasProxy := os.Getenv("HTTP_PROXY") != "" || os.Getenv("HTTPS_PROXY") != ""
			hasNoProxyIdmsException := strings.Contains(os.Getenv("NO_PROXY"), "169.254.169.254")

			errorTitle := "Could not authenticate with AWS S3 using any of the included credential providers."

			if hasProxy && !hasNoProxyIdmsException {
				return nil, fmt.Errorf("%s Your HTTP proxy settings do not grant a NO_PROXY=169.254.169.254 exemption for the instance metadata service, instance profile credentials may not be retrievable via your HTTP proxy.", errorTitle)
			}

			return nil, fmt.Errorf("%s You can authenticate by setting Buildkite environment variables (BUILDKITE_S3_ACCESS_KEY_ID, BUILDKITE_S3_SECRET_ACCESS_KEY), AWS environment variables (AWS_ACCESS_KEY_ID, AWS_SECRET_ACCESS_KEY), Web Identity environment variables (AWS_ROLE_ARN, AWS_ROLE_SESSION_NAME, AWS_WEB_IDENTITY_TOKEN_FILE), or if running on AWS EC2 ensuring network access to the EC2 Instance Metadata Service to use an instance profile’s IAM Role credentials.", errorTitle)
		}
		return nil, fmt.Errorf("Could not s3:ListObjects in your AWS S3 bucket %q in region %q: (%s)", bucket, *sess.Config.Region, err.Error())
	}

	return s3client, nil
}
<|endoftext|>


<commit_before>// Command line app to process fiddles in bulk.
//
// Example:
//  fiddlecli --input demo/testbulk.json --output /tmp/output.json
package main

import (
	"bytes"
	"encoding/json"
	"flag"
	"fmt"
	"io/ioutil"
	"log"
	"net/http"
	"sync"

	"golang.org/x/sync/errgroup"

	"go.skia.org/infra/fiddlek/go/types"
	"go.skia.org/infra/go/common"
	"go.skia.org/infra/go/httputils"
)

const (
	RETRIES = 5
)

// flags
var (
	domain = flag.String("domain", "https://fiddle.skia.org", "Where to send the JSON request.")
	input  = flag.String("input", "", "The name of the file to read the JSON from.")
	output = flag.String("output", "", "The name of the file to write the JSON results to.")
	procs  = flag.Int("procs", 4, "The number of parallel requests to make to the fiddle server.")
	quiet  = flag.Bool("quiet", false, "Run without a progress bar.")
	force  = flag.Bool("force", false, "Force a compile and run for each fiddle, don't take the fast path.")
)

// chanRequest is sent to each worker in the pool.
type chanRequest struct {
	id  string
	req *types.FiddleContext
}

func main() {
	// Check flags.
	common.Init()
	if *input == "" {
		flag.Usage()
		log.Fatalf("--input is a required flag.")
	}
	if *output == "" {
		flag.Usage()
		log.Fatalf("--output is a required flag.")
	}

	// Read the source JSON file.
	b, err := ioutil.ReadFile(*input)
	if err != nil {
		log.Fatalf("Failed to read %s: %s", *input, err)
	}
	requests := types.BulkRequest{}
	if err := json.Unmarshal(b, &requests); err != nil {
		log.Fatalf("%s does not contain valid JSON: %s", *input, err)
	}
	lastWritten := types.BulkResponse{}
	b, err = ioutil.ReadFile(*output)
	if err == nil {
		if err := json.Unmarshal(b, &lastWritten); err != nil {
			lastWritten = nil
		}
	}
	g := errgroup.Group{}
	requestsCh := make(chan chanRequest, len(requests))

	// mutex protects response.
	mutex := sync.Mutex{}
	response := types.BulkResponse{}

	// Spin up workers.
	for i := 0; i < *procs; i++ {
		g.Go(func() error {
			c := httputils.NewTimeoutClient()
			for req := range requestsCh {
				if !*quiet {
					fmt.Print(".")
				}
				if *force {
					req.req.Fast = false
				} else if lastWritten != nil {
					fiddleHash, err := req.req.Options.ComputeHash(req.req.Code)
					if err == nil {
						if lastWritten[req.id] != nil {
							if lastWritten[req.id].FiddleHash == fiddleHash {
								mutex.Lock()
								response[req.id] = lastWritten[req.id]
								mutex.Unlock()
								continue
							}
						}
					}
				}
				// POST to fiddle.
				b, err = json.Marshal(req.req)
				if err != nil {
					return fmt.Errorf("Failed to encode an individual request: %s", err)
				}
				var resp *http.Response
				success := false
				for tries := 0; tries < RETRIES; tries++ {
					resp, err = c.Post(*domain+"/_/run", "application/json", bytes.NewReader(b))
					if err == nil && resp.StatusCode == 200 {
						success = true
						break
					}
				}
				if !success {
					return fmt.Errorf("Failed to make request: %s %s", err, resp.Status)
				}

				// Decode response and add to all responses.
				runResults := types.RunResults{}
				if err := json.NewDecoder(resp.Body).Decode(&runResults); err != nil {
					return fmt.Errorf("Failed to read response: %s", err)
				}
				mutex.Lock()
				response[req.id] = &runResults
				mutex.Unlock()
			}
			return nil
		})
	}

	// Loop over each entry and queue them up for the workers.
	for id, req := range requests {
		requestsCh <- chanRequest{
			id:  id,
			req: req,
		}
	}
	close(requestsCh)

	// Wait for the workers to finish.
	if err := g.Wait(); err != nil {
		log.Fatalf("Failed to complete all requests: %s", err)
	}

	if !*quiet {
		fmt.Print("\n")
	}
	b, err = json.MarshalIndent(response, "", "  ")
	if err != nil {
		log.Fatalf("Failed to encode response file: %s", err)
	}
	if err := ioutil.WriteFile(*output, b, 0600); err != nil {
		log.Fatalf("Failed to write response file: %s", err)
	}
}
<commit_msg>[fiddlecli] - Include decoding the JSON response in the retry logic.<commit_after>// Command line app to process fiddles in bulk.
//
// Example:
//  fiddlecli --input demo/testbulk.json --output /tmp/output.json
package main

import (
	"bytes"
	"encoding/json"
	"flag"
	"fmt"
	"io/ioutil"
	"log"
	"net/http"
	"sync"
	"time"

	"golang.org/x/sync/errgroup"

	"go.skia.org/infra/fiddlek/go/types"
	"go.skia.org/infra/go/common"
	"go.skia.org/infra/go/httputils"
)

const (
	RETRIES = 5
)

// flags
var (
	domain = flag.String("domain", "https://fiddle.skia.org", "Where to send the JSON request.")
	input  = flag.String("input", "", "The name of the file to read the JSON from.")
	output = flag.String("output", "", "The name of the file to write the JSON results to.")
	procs  = flag.Int("procs", 4, "The number of parallel requests to make to the fiddle server.")
	quiet  = flag.Bool("quiet", false, "Run without a progress bar.")
	force  = flag.Bool("force", false, "Force a compile and run for each fiddle, don't take the fast path.")
)

// chanRequest is sent to each worker in the pool.
type chanRequest struct {
	id  string
	req *types.FiddleContext
}

func main() {
	// Check flags.
	common.Init()
	if *input == "" {
		flag.Usage()
		log.Fatalf("--input is a required flag.")
	}
	if *output == "" {
		flag.Usage()
		log.Fatalf("--output is a required flag.")
	}

	// Read the source JSON file.
	b, err := ioutil.ReadFile(*input)
	if err != nil {
		log.Fatalf("Failed to read %s: %s", *input, err)
	}
	requests := types.BulkRequest{}
	if err := json.Unmarshal(b, &requests); err != nil {
		log.Fatalf("%s does not contain valid JSON: %s", *input, err)
	}
	lastWritten := types.BulkResponse{}
	b, err = ioutil.ReadFile(*output)
	if err == nil {
		if err := json.Unmarshal(b, &lastWritten); err != nil {
			lastWritten = nil
		}
	}
	g := errgroup.Group{}
	requestsCh := make(chan chanRequest, len(requests))

	// mutex protects response.
	mutex := sync.Mutex{}
	response := types.BulkResponse{}

	// Spin up workers.
	for i := 0; i < *procs; i++ {
		g.Go(func() error {
			c := httputils.NewTimeoutClient()
			for req := range requestsCh {
				if !*quiet {
					fmt.Print(".")
				}
				if *force {
					req.req.Fast = false
				} else if lastWritten != nil {
					fiddleHash, err := req.req.Options.ComputeHash(req.req.Code)
					if err == nil {
						if lastWritten[req.id] != nil {
							if lastWritten[req.id].FiddleHash == fiddleHash {
								mutex.Lock()
								response[req.id] = lastWritten[req.id]
								mutex.Unlock()
								continue
							}
						}
					}
				}
				// POST to fiddle.
				b, err = json.Marshal(req.req)
				if err != nil {
					return fmt.Errorf("Failed to encode an individual request: %s", err)
				}
				var resp *http.Response
				runResults := types.RunResults{}
				success := false
				for tries := 0; tries < RETRIES; tries++ {
					resp, err = c.Post(*domain+"/_/run", "application/json", bytes.NewReader(b))
					if err != nil || resp.StatusCode != 200 {
						time.Sleep(time.Second)
						continue
					}

					// Decode response and add to all responses.
					if err := json.NewDecoder(resp.Body).Decode(&runResults); err != nil {
						time.Sleep(time.Second)
						continue
					}
					success = true
					break
				}
				if !success {
					return fmt.Errorf("Failed to make request: %s %s", err, resp.Status)
				}

				mutex.Lock()
				response[req.id] = &runResults
				mutex.Unlock()
			}
			return nil
		})
	}

	// Loop over each entry and queue them up for the workers.
	for id, req := range requests {
		requestsCh <- chanRequest{
			id:  id,
			req: req,
		}
	}
	close(requestsCh)

	// Wait for the workers to finish.
	if err := g.Wait(); err != nil {
		log.Fatalf("Failed to complete all requests: %s", err)
	}

	if !*quiet {
		fmt.Print("\n")
	}
	b, err = json.MarshalIndent(response, "", "  ")
	if err != nil {
		log.Fatalf("Failed to encode response file: %s", err)
	}
	if err := ioutil.WriteFile(*output, b, 0600); err != nil {
		log.Fatalf("Failed to write response file: %s", err)
	}
}
<|endoftext|>


<commit_before>package torrentfs

import (
	"expvar"
	"log"
	"os"
	"strings"
	"sync"
	"time"

	"bazil.org/fuse"
	fusefs "bazil.org/fuse/fs"
	"bitbucket.org/anacrolix/go.torrent"
	"github.com/anacrolix/libtorgo/metainfo"
)

const (
	defaultMode = 0555
)

var (
	torrentfsReadRequests        = expvar.NewInt("torrentfsReadRequests")
	torrentfsDelayedReadRequests = expvar.NewInt("torrentfsDelayedReadRequests")
	interruptedReads             = expvar.NewInt("interruptedReads")
)

type torrentFS struct {
	Client    *torrent.Client
	destroyed chan struct{}
	mu        sync.Mutex
}

var (
	_ fusefs.FSDestroyer = &torrentFS{}
	_ fusefs.FSIniter    = &torrentFS{}
)

func (fs *torrentFS) Init(req *fuse.InitRequest, resp *fuse.InitResponse, intr fusefs.Intr) fuse.Error {
	log.Print(req)
	log.Print(resp)
	resp.MaxReadahead = req.MaxReadahead
	resp.Flags |= fuse.InitAsyncRead
	return nil
}

var _ fusefs.NodeForgetter = rootNode{}

type rootNode struct {
	fs *torrentFS
}

type node struct {
	path     []string
	metadata *metainfo.Info
	FS       *torrentFS
	InfoHash torrent.InfoHash
}

type fileNode struct {
	node
	size          uint64
	TorrentOffset int64
}

func (fn fileNode) Attr() (attr fuse.Attr) {
	attr.Size = fn.size
	attr.Mode = defaultMode
	return
}

func (n *node) fsPath() string {
	return "/" + strings.Join(append([]string{n.metadata.Name}, n.path...), "/")
}

func (fn fileNode) Read(req *fuse.ReadRequest, resp *fuse.ReadResponse, intr fusefs.Intr) fuse.Error {
	torrentfsReadRequests.Add(1)
	started := time.Now()
	if req.Dir {
		panic("read on directory")
	}
	defer func() {
		ms := time.Now().Sub(started).Nanoseconds() / 1000000
		if ms < 20 {
			return
		}
		log.Printf("torrentfs read took %dms", ms)
	}()
	size := req.Size
	fileLeft := int64(fn.size) - req.Offset
	if fileLeft < int64(size) {
		size = int(fileLeft)
	}
	resp.Data = resp.Data[:size]
	if len(resp.Data) == 0 {
		return nil
	}
	infoHash := fn.InfoHash
	torrentOff := fn.TorrentOffset + req.Offset
	go func() {
		if err := fn.FS.Client.PrioritizeDataRegion(infoHash, torrentOff, int64(size)); err != nil {
			panic(err)
		}
	}()
	delayed := false
	for {
		dataWaiter := fn.FS.Client.DataWaiter()
		n, err := fn.FS.Client.TorrentReadAt(infoHash, torrentOff, resp.Data)
		switch err {
		case nil:
			resp.Data = resp.Data[:n]
			return nil
		case torrent.ErrDataNotReady:
			if !delayed {
				torrentfsDelayedReadRequests.Add(1)
				delayed = true
			}
			select {
			case <-dataWaiter:
			case <-fn.FS.destroyed:
				return fuse.EIO
			case <-intr:
				return fuse.EINTR
			}
		default:
			log.Print(err)
			return err // bazil.org/fuse will convert generic errors appropriately.
		}
	}
}

type dirNode struct {
	node
}

var (
	_ fusefs.HandleReadDirer = dirNode{}

	_ fusefs.HandleReader = fileNode{}
)

func isSubPath(parent, child []string) bool {
	if len(child) <= len(parent) {
		return false
	}
	for i := range parent {
		if parent[i] != child[i] {
			return false
		}
	}
	return true
}

func (dn dirNode) ReadDir(intr fusefs.Intr) (des []fuse.Dirent, err fuse.Error) {
	names := map[string]bool{}
	for _, fi := range dn.metadata.Files {
		if !isSubPath(dn.path, fi.Path) {
			continue
		}
		name := fi.Path[len(dn.path)]
		if names[name] {
			continue
		}
		names[name] = true
		de := fuse.Dirent{
			Name: name,
		}
		if len(fi.Path) == len(dn.path)+1 {
			de.Type = fuse.DT_File
		} else {
			de.Type = fuse.DT_Dir
		}
		des = append(des, de)
	}
	return
}

func (dn dirNode) Lookup(name string, intr fusefs.Intr) (_node fusefs.Node, err fuse.Error) {
	var torrentOffset int64
	for _, fi := range dn.metadata.Files {
		if !isSubPath(dn.path, fi.Path) {
			torrentOffset += fi.Length
			continue
		}
		if fi.Path[len(dn.path)] != name {
			torrentOffset += fi.Length
			continue
		}
		__node := dn.node
		__node.path = append(__node.path, name)
		if len(fi.Path) == len(dn.path)+1 {
			_node = fileNode{
				node:          __node,
				size:          uint64(fi.Length),
				TorrentOffset: torrentOffset,
			}
		} else {
			_node = dirNode{__node}
		}
		break
	}
	if _node == nil {
		err = fuse.ENOENT
	}
	return
}

func (dn dirNode) Attr() (attr fuse.Attr) {
	attr.Mode = os.ModeDir | defaultMode
	return
}

func isSingleFileTorrent(md *metainfo.Info) bool {
	return len(md.Files) == 0
}

func (me rootNode) Lookup(name string, intr fusefs.Intr) (_node fusefs.Node, err fuse.Error) {
	for _, t := range me.fs.Client.Torrents() {
		if t.Name() != name || t.Info == nil {
			continue
		}
		__node := node{
			metadata: t.Info,
			FS:       me.fs,
			InfoHash: t.InfoHash,
		}
		if isSingleFileTorrent(t.Info) {
			_node = fileNode{__node, uint64(t.Info.Length), 0}
		} else {
			_node = dirNode{__node}
		}
		break
	}
	if _node == nil {
		err = fuse.ENOENT
	}
	return
}

func (me rootNode) ReadDir(intr fusefs.Intr) (dirents []fuse.Dirent, err fuse.Error) {
	for _, _torrent := range me.fs.Client.Torrents() {
		metaInfo := _torrent.Info
		if metaInfo == nil {
			continue
		}
		dirents = append(dirents, fuse.Dirent{
			Name: metaInfo.Name,
			Type: func() fuse.DirentType {
				if isSingleFileTorrent(metaInfo) {
					return fuse.DT_File
				} else {
					return fuse.DT_Dir
				}
			}(),
		})
	}
	return
}

func (rootNode) Attr() fuse.Attr {
	return fuse.Attr{
		Mode: os.ModeDir,
	}
}

// TODO(anacrolix): Why should rootNode implement this?
func (me rootNode) Forget() {
	me.fs.Destroy()
}

func (tfs *torrentFS) Root() (fusefs.Node, fuse.Error) {
	return rootNode{tfs}, nil
}

func (me *torrentFS) Destroy() {
	me.mu.Lock()
	select {
	case <-me.destroyed:
	default:
		close(me.destroyed)
	}
	me.mu.Unlock()
}

func New(cl *torrent.Client) *torrentFS {
	fs := &torrentFS{
		Client:    cl,
		destroyed: make(chan struct{}),
	}
	return fs
}
<commit_msg>Fix crashes in torrentfs.fileNode.Read<commit_after>package torrentfs

import (
	"expvar"
	"log"
	"os"
	"strings"
	"sync"
	"time"

	"bazil.org/fuse"
	fusefs "bazil.org/fuse/fs"
	"bitbucket.org/anacrolix/go.torrent"
	"github.com/anacrolix/libtorgo/metainfo"
)

const (
	defaultMode = 0555
)

var (
	torrentfsReadRequests        = expvar.NewInt("torrentfsReadRequests")
	torrentfsDelayedReadRequests = expvar.NewInt("torrentfsDelayedReadRequests")
	interruptedReads             = expvar.NewInt("interruptedReads")
)

type torrentFS struct {
	Client    *torrent.Client
	destroyed chan struct{}
	mu        sync.Mutex
}

var (
	_ fusefs.FSDestroyer = &torrentFS{}
	_ fusefs.FSIniter    = &torrentFS{}
)

func (fs *torrentFS) Init(req *fuse.InitRequest, resp *fuse.InitResponse, intr fusefs.Intr) fuse.Error {
	log.Print(req)
	log.Print(resp)
	resp.MaxReadahead = req.MaxReadahead
	resp.Flags |= fuse.InitAsyncRead
	return nil
}

var _ fusefs.NodeForgetter = rootNode{}

type rootNode struct {
	fs *torrentFS
}

type node struct {
	path     []string
	metadata *metainfo.Info
	FS       *torrentFS
	InfoHash torrent.InfoHash
}

type fileNode struct {
	node
	size          uint64
	TorrentOffset int64
}

func (fn fileNode) Attr() (attr fuse.Attr) {
	attr.Size = fn.size
	attr.Mode = defaultMode
	return
}

func (n *node) fsPath() string {
	return "/" + strings.Join(append([]string{n.metadata.Name}, n.path...), "/")
}

func (fn fileNode) Read(req *fuse.ReadRequest, resp *fuse.ReadResponse, intr fusefs.Intr) fuse.Error {
	torrentfsReadRequests.Add(1)
	started := time.Now()
	if req.Dir {
		panic("read on directory")
	}
	defer func() {
		ms := time.Now().Sub(started).Nanoseconds() / 1000000
		if ms < 20 {
			return
		}
		log.Printf("torrentfs read took %dms", ms)
	}()
	size := req.Size
	fileLeft := int64(fn.size) - req.Offset
	if fileLeft < 0 {
		fileLeft = 0
	}
	if fileLeft < int64(size) {
		size = int(fileLeft)
	}
	resp.Data = resp.Data[:size]
	if len(resp.Data) == 0 {
		return nil
	}
	infoHash := fn.InfoHash
	torrentOff := fn.TorrentOffset + req.Offset
	go func() {
		if err := fn.FS.Client.PrioritizeDataRegion(infoHash, torrentOff, int64(size)); err != nil {
			log.Printf("error prioritizing %s: %s", fn.fsPath(), err)
		}
	}()
	delayed := false
	for {
		dataWaiter := fn.FS.Client.DataWaiter()
		n, err := fn.FS.Client.TorrentReadAt(infoHash, torrentOff, resp.Data)
		switch err {
		case nil:
			resp.Data = resp.Data[:n]
			return nil
		case torrent.ErrDataNotReady:
			if !delayed {
				torrentfsDelayedReadRequests.Add(1)
				delayed = true
			}
			select {
			case <-dataWaiter:
			case <-fn.FS.destroyed:
				return fuse.EIO
			case <-intr:
				return fuse.EINTR
			}
		default:
			log.Print(err)
			return err // bazil.org/fuse will convert generic errors appropriately.
		}
	}
}

type dirNode struct {
	node
}

var (
	_ fusefs.HandleReadDirer = dirNode{}

	_ fusefs.HandleReader = fileNode{}
)

func isSubPath(parent, child []string) bool {
	if len(child) <= len(parent) {
		return false
	}
	for i := range parent {
		if parent[i] != child[i] {
			return false
		}
	}
	return true
}

func (dn dirNode) ReadDir(intr fusefs.Intr) (des []fuse.Dirent, err fuse.Error) {
	names := map[string]bool{}
	for _, fi := range dn.metadata.Files {
		if !isSubPath(dn.path, fi.Path) {
			continue
		}
		name := fi.Path[len(dn.path)]
		if names[name] {
			continue
		}
		names[name] = true
		de := fuse.Dirent{
			Name: name,
		}
		if len(fi.Path) == len(dn.path)+1 {
			de.Type = fuse.DT_File
		} else {
			de.Type = fuse.DT_Dir
		}
		des = append(des, de)
	}
	return
}

func (dn dirNode) Lookup(name string, intr fusefs.Intr) (_node fusefs.Node, err fuse.Error) {
	var torrentOffset int64
	for _, fi := range dn.metadata.Files {
		if !isSubPath(dn.path, fi.Path) {
			torrentOffset += fi.Length
			continue
		}
		if fi.Path[len(dn.path)] != name {
			torrentOffset += fi.Length
			continue
		}
		__node := dn.node
		__node.path = append(__node.path, name)
		if len(fi.Path) == len(dn.path)+1 {
			_node = fileNode{
				node:          __node,
				size:          uint64(fi.Length),
				TorrentOffset: torrentOffset,
			}
		} else {
			_node = dirNode{__node}
		}
		break
	}
	if _node == nil {
		err = fuse.ENOENT
	}
	return
}

func (dn dirNode) Attr() (attr fuse.Attr) {
	attr.Mode = os.ModeDir | defaultMode
	return
}

func isSingleFileTorrent(md *metainfo.Info) bool {
	return len(md.Files) == 0
}

func (me rootNode) Lookup(name string, intr fusefs.Intr) (_node fusefs.Node, err fuse.Error) {
	for _, t := range me.fs.Client.Torrents() {
		if t.Name() != name || t.Info == nil {
			continue
		}
		__node := node{
			metadata: t.Info,
			FS:       me.fs,
			InfoHash: t.InfoHash,
		}
		if isSingleFileTorrent(t.Info) {
			_node = fileNode{__node, uint64(t.Info.Length), 0}
		} else {
			_node = dirNode{__node}
		}
		break
	}
	if _node == nil {
		err = fuse.ENOENT
	}
	return
}

func (me rootNode) ReadDir(intr fusefs.Intr) (dirents []fuse.Dirent, err fuse.Error) {
	for _, _torrent := range me.fs.Client.Torrents() {
		metaInfo := _torrent.Info
		if metaInfo == nil {
			continue
		}
		dirents = append(dirents, fuse.Dirent{
			Name: metaInfo.Name,
			Type: func() fuse.DirentType {
				if isSingleFileTorrent(metaInfo) {
					return fuse.DT_File
				} else {
					return fuse.DT_Dir
				}
			}(),
		})
	}
	return
}

func (rootNode) Attr() fuse.Attr {
	return fuse.Attr{
		Mode: os.ModeDir,
	}
}

// TODO(anacrolix): Why should rootNode implement this?
func (me rootNode) Forget() {
	me.fs.Destroy()
}

func (tfs *torrentFS) Root() (fusefs.Node, fuse.Error) {
	return rootNode{tfs}, nil
}

func (me *torrentFS) Destroy() {
	me.mu.Lock()
	select {
	case <-me.destroyed:
	default:
		close(me.destroyed)
	}
	me.mu.Unlock()
}

func New(cl *torrent.Client) *torrentFS {
	fs := &torrentFS{
		Client:    cl,
		destroyed: make(chan struct{}),
	}
	return fs
}
<|endoftext|>


<commit_before>package runc

import (
	"encoding/json"
	"io"
	"io/ioutil"
	"os/exec"
	"path/filepath"

	"github.com/inconshreveable/log15"
	"github.com/polydawn/gosh"
	"github.com/spacemonkeygo/errors"
	"github.com/spacemonkeygo/errors/try"

	"polydawn.net/repeatr/def"
	"polydawn.net/repeatr/executor"
	"polydawn.net/repeatr/executor/basicjob"
	"polydawn.net/repeatr/executor/util"
	"polydawn.net/repeatr/io"
	"polydawn.net/repeatr/io/assets"
	"polydawn.net/repeatr/lib/flak"
	"polydawn.net/repeatr/lib/streamer"
)

// interface assertion
var _ executor.Executor = &Executor{}

type Executor struct {
	workspacePath string
}

func (e *Executor) Configure(workspacePath string) {
	e.workspacePath = workspacePath
}

func (e *Executor) Start(f def.Formula, id def.JobID, stdin io.Reader, journal io.Writer) def.Job {

	// Prepare the forumla for execution on this host
	def.ValidateAll(&f)

	job := basicjob.New(id)
	jobReady := make(chan struct{})

	go func() {
		// Run the formula in a temporary directory
		flak.WithDir(func(dir string) {

			// spool our output to a muxed stream
			var strm streamer.Mux
			strm = streamer.CborFileMux(filepath.Join(dir, "log"))
			outS := strm.Appender(1)
			errS := strm.Appender(2)
			job.Streams = strm
			defer func() {
				// Regardless of how the job ends (or even if it fails the remaining setup), output streams must be terminated.
				outS.Close()
				errS.Close()
			}()

			// Job is ready to stream process output
			close(jobReady)

			// Set up a logger.  Tag all messages with this jobid.
			logger := log15.New(log15.Ctx{"JobID": id})
			logger.SetHandler(log15.StreamHandler(journal, log15.TerminalFormat()))

			job.Result = e.Run(f, job, dir, stdin, outS, errS, logger)
		}, e.workspacePath, "job", string(job.Id()))

		// Directory is clean; job complete
		close(job.WaitChan)
	}()

	<-jobReady
	return job
}

// Executes a job, catching any panics.
func (e *Executor) Run(f def.Formula, j def.Job, d string, stdin io.Reader, outS, errS io.WriteCloser, journal log15.Logger) def.JobResult {
	r := def.JobResult{
		ID:       j.Id(),
		ExitCode: -1,
	}

	try.Do(func() {
		e.Execute(f, j, d, &r, outS, errS, journal)
	}).Catch(executor.Error, func(err *errors.Error) {
		r.Error = err
	}).Catch(integrity.Error, func(err *errors.Error) {
		r.Error = err
	}).CatchAll(func(err error) {
		r.Error = executor.UnknownError.Wrap(err).(*errors.Error)
	}).Done()

	return r
}

// Execute a formula in a specified directory. MAY PANIC.
func (e *Executor) Execute(formula def.Formula, job def.Job, jobPath string, result *def.JobResult, stdout, stderr io.WriteCloser, journal log15.Logger) {
	rootfsPath := filepath.Join(jobPath, "rootfs")
	logPath := filepath.Join(jobPath, "runc-debug.log")

	// Prepare inputs
	transmat := util.DefaultTransmat()
	inputArenas := util.ProvisionInputs(transmat, formula.Inputs, journal)
	util.ProvisionOutputs(formula.Outputs, rootfsPath, journal)

	// Assemble filesystem
	assembly := util.AssembleFilesystem(
		util.BestAssembler(),
		rootfsPath,
		formula.Inputs,
		inputArenas,
		formula.Action.Escapes.Mounts,
		journal,
	)
	defer assembly.Teardown()

	// Emit configs for runc.
	runcConfigJsonPath := filepath.Join(jobPath, "config.json")
	cfg := EmitRuncConfigStruct(formula, rootfsPath)
	buf, err := json.Marshal(cfg)
	if err != nil {
		panic(executor.UnknownError.Wrap(err))
	}
	ioutil.WriteFile(runcConfigJsonPath, buf, 0600)
	runcRuntimeJsonPath := filepath.Join(jobPath, "runtime.json")
	cfg = EmitRuncRuntimeStruct(formula)
	buf, err = json.Marshal(cfg)
	if err != nil {
		panic(executor.UnknownError.Wrap(err))
	}
	ioutil.WriteFile(runcRuntimeJsonPath, buf, 0600)

	// Get handle to invokable runc plugin.
	runcPath := filepath.Join(assets.Get("runc"), "bin/runc")

	// Prepare command to exec
	args := []string{
		"--root", filepath.Join(e.workspacePath, "shared"), // a tmpfs would be appropriate
		"--log", logPath,
		"start",
		"--config-file", runcConfigJsonPath,
		"--runtime-file", runcRuntimeJsonPath,
	}
	cmd := exec.Command(runcPath, args...)
	cmd.Stdin = nil
	cmd.Stdout = stdout
	cmd.Stderr = stderr

	// launch execution.
	// transform gosh's typed errors to repeatr's hierarchical errors.
	// this is... not untroubled code: since we're invoking a helper that's then
	//  proxying the exec even further, most errors are fatal (the mapping here is
	//   very different than in e.g. chroot executor, and provides much less meaning).
	var proc gosh.Proc
	try.Do(func() {
		proc = gosh.ExecProcCmd(cmd)
	}).CatchAll(func(err error) {
		switch err.(type) {
		case gosh.NoSuchCommandError:
			panic(executor.ConfigError.New("runc binary is missing"))
		case gosh.NoArgumentsError:
			panic(executor.UnknownError.Wrap(err))
		case gosh.NoSuchCwdError:
			panic(executor.UnknownError.Wrap(err))
		case gosh.ProcMonitorError:
			panic(executor.TaskExecError.Wrap(err))
		default:
			panic(executor.UnknownError.Wrap(err))
		}
	}).Done()

	// Wait for the job to complete
	result.ExitCode = proc.GetExitCode()

	// Save outputs
	result.Outputs = util.PreserveOutputs(transmat, formula.Outputs, rootfsPath, journal)

	// TODO : additional error detection of runc failure modes
	// Okay, what do we know?
	//  - They use logrus
	//  - I've only seen warns and fatals so far, which is kinda amusing
	//    - But I'm AOK with this: it means we can proxy those messages freely without noise issues
	//  - INVESTIGATE: does logrus even *have* a mechanism to switch formats to something structural like log15 can?
	//    - If so, this would still likely require moving a patch upstream in order to expose usefully
	//      - And while I'm all about moving everyone forward together, that's probably going to get derailed into a spec discussion about lifecycle events, and that's great,
	//        - but I want to use this before that even if it requires ducttape.
	//      - Alternatively!  We *are* building runc from source already; maintaining a formula and patches together in-repo here would be totally reasonable.
	//    - Yes!  It can accept a Formatter and there is a JSON Formatter.  No, there are no flags that currently do this.
	//      - SetFormatter: we can call it.  Runc is currently only using the global logrus instance.
}
<commit_msg>Did I mention our build of runc can emit json?<commit_after>package runc

import (
	"encoding/json"
	"io"
	"io/ioutil"
	"os/exec"
	"path/filepath"

	"github.com/inconshreveable/log15"
	"github.com/polydawn/gosh"
	"github.com/spacemonkeygo/errors"
	"github.com/spacemonkeygo/errors/try"

	"polydawn.net/repeatr/def"
	"polydawn.net/repeatr/executor"
	"polydawn.net/repeatr/executor/basicjob"
	"polydawn.net/repeatr/executor/util"
	"polydawn.net/repeatr/io"
	"polydawn.net/repeatr/io/assets"
	"polydawn.net/repeatr/lib/flak"
	"polydawn.net/repeatr/lib/streamer"
)

// interface assertion
var _ executor.Executor = &Executor{}

type Executor struct {
	workspacePath string
}

func (e *Executor) Configure(workspacePath string) {
	e.workspacePath = workspacePath
}

func (e *Executor) Start(f def.Formula, id def.JobID, stdin io.Reader, journal io.Writer) def.Job {

	// Prepare the forumla for execution on this host
	def.ValidateAll(&f)

	job := basicjob.New(id)
	jobReady := make(chan struct{})

	go func() {
		// Run the formula in a temporary directory
		flak.WithDir(func(dir string) {

			// spool our output to a muxed stream
			var strm streamer.Mux
			strm = streamer.CborFileMux(filepath.Join(dir, "log"))
			outS := strm.Appender(1)
			errS := strm.Appender(2)
			job.Streams = strm
			defer func() {
				// Regardless of how the job ends (or even if it fails the remaining setup), output streams must be terminated.
				outS.Close()
				errS.Close()
			}()

			// Job is ready to stream process output
			close(jobReady)

			// Set up a logger.  Tag all messages with this jobid.
			logger := log15.New(log15.Ctx{"JobID": id})
			logger.SetHandler(log15.StreamHandler(journal, log15.TerminalFormat()))

			job.Result = e.Run(f, job, dir, stdin, outS, errS, logger)
		}, e.workspacePath, "job", string(job.Id()))

		// Directory is clean; job complete
		close(job.WaitChan)
	}()

	<-jobReady
	return job
}

// Executes a job, catching any panics.
func (e *Executor) Run(f def.Formula, j def.Job, d string, stdin io.Reader, outS, errS io.WriteCloser, journal log15.Logger) def.JobResult {
	r := def.JobResult{
		ID:       j.Id(),
		ExitCode: -1,
	}

	try.Do(func() {
		e.Execute(f, j, d, &r, outS, errS, journal)
	}).Catch(executor.Error, func(err *errors.Error) {
		r.Error = err
	}).Catch(integrity.Error, func(err *errors.Error) {
		r.Error = err
	}).CatchAll(func(err error) {
		r.Error = executor.UnknownError.Wrap(err).(*errors.Error)
	}).Done()

	return r
}

// Execute a formula in a specified directory. MAY PANIC.
func (e *Executor) Execute(formula def.Formula, job def.Job, jobPath string, result *def.JobResult, stdout, stderr io.WriteCloser, journal log15.Logger) {
	rootfsPath := filepath.Join(jobPath, "rootfs")

	// Prepare inputs
	transmat := util.DefaultTransmat()
	inputArenas := util.ProvisionInputs(transmat, formula.Inputs, journal)
	util.ProvisionOutputs(formula.Outputs, rootfsPath, journal)

	// Assemble filesystem
	assembly := util.AssembleFilesystem(
		util.BestAssembler(),
		rootfsPath,
		formula.Inputs,
		inputArenas,
		formula.Action.Escapes.Mounts,
		journal,
	)
	defer assembly.Teardown()

	// Emit configs for runc.
	runcConfigJsonPath := filepath.Join(jobPath, "config.json")
	cfg := EmitRuncConfigStruct(formula, rootfsPath)
	buf, err := json.Marshal(cfg)
	if err != nil {
		panic(executor.UnknownError.Wrap(err))
	}
	ioutil.WriteFile(runcConfigJsonPath, buf, 0600)
	runcRuntimeJsonPath := filepath.Join(jobPath, "runtime.json")
	cfg = EmitRuncRuntimeStruct(formula)
	buf, err = json.Marshal(cfg)
	if err != nil {
		panic(executor.UnknownError.Wrap(err))
	}
	ioutil.WriteFile(runcRuntimeJsonPath, buf, 0600)

	// Routing logs through a fifo appears to work, but we're going to use a file as a buffer anyway:
	//  in the event of nasty breakdowns, it's preferable that the runc log remain readable even if repeatr was the process to end first.
	logPath := filepath.Join(jobPath, "runc-debug.log")

	// Get handle to invokable runc plugin.
	runcPath := filepath.Join(assets.Get("runc"), "bin/runc")

	// Prepare command to exec
	args := []string{
		"--root", filepath.Join(e.workspacePath, "shared"), // a tmpfs would be appropriate
		"--log", logPath,
		"--log-format", "json",
		"start",
		"--config-file", runcConfigJsonPath,
		"--runtime-file", runcRuntimeJsonPath,
	}
	cmd := exec.Command(runcPath, args...)
	cmd.Stdin = nil
	cmd.Stdout = stdout
	cmd.Stderr = stderr

	// launch execution.
	// transform gosh's typed errors to repeatr's hierarchical errors.
	// this is... not untroubled code: since we're invoking a helper that's then
	//  proxying the exec even further, most errors are fatal (the mapping here is
	//   very different than in e.g. chroot executor, and provides much less meaning).
	var proc gosh.Proc
	try.Do(func() {
		proc = gosh.ExecProcCmd(cmd)
	}).CatchAll(func(err error) {
		switch err.(type) {
		case gosh.NoSuchCommandError:
			panic(executor.ConfigError.New("runc binary is missing"))
		case gosh.NoArgumentsError:
			panic(executor.UnknownError.Wrap(err))
		case gosh.NoSuchCwdError:
			panic(executor.UnknownError.Wrap(err))
		case gosh.ProcMonitorError:
			panic(executor.TaskExecError.Wrap(err))
		default:
			panic(executor.UnknownError.Wrap(err))
		}
	}).Done()

	// Wait for the job to complete
	result.ExitCode = proc.GetExitCode()

	// Save outputs
	result.Outputs = util.PreserveOutputs(transmat, formula.Outputs, rootfsPath, journal)
}
<|endoftext|>


<commit_before>package main

import (
	"github.com/mehrdadrad/myping/cli"
	"github.com/mehrdadrad/myping/icmp"
	"github.com/mehrdadrad/myping/icmp/telia"
	"net"
	"regexp"
	"strings"
)

func validateProvider(p string) (string, error) {
	p = strings.ToLower(p)
	return p, nil
}

func main() {
	var (
		err       error
		cProvider string = "local"
	)

	rep := make(chan string, 1)
	cmd := make(chan string, 1)
	nxt := make(chan struct{}, 1)

	c := cli.Init("local")
	go c.Run(cmd, nxt)

	r, _ := regexp.Compile("(ping|connect) (.*)")

	for {
		select {
		case req := <-cmd:
			subReq := r.FindStringSubmatch(req)
			if len(subReq) == 0 {
				println("syntax error")
				nxt <- struct{}{}
				continue
			}
			switch {
			case subReq[1] == "ping" && cProvider == "local":
				p := icmp.NewPing()
				ra, err := net.ResolveIPAddr("ip", subReq[2])
				if err != nil {
					println("cannot resolve", subReq[2], ": Unknown host")
					nxt <- struct{}{}
					continue
				}
				p.IP(ra.String())
				for n := 0; n < 4; n++ {
					p.Ping(rep)
					println(<-rep)
				}
				nxt <- struct{}{}
			case subReq[1] == "ping" && cProvider == "telia":
				p := telia.Init(subReq[2], "ipv4", "Los Angeles")
				println(p.Ping())
				nxt <- struct{}{}
			case subReq[1] == "connect":
				var provider string
				if provider, err = validateProvider(subReq[2]); err != nil {
					println("provider not available")
					nxt <- struct{}{}
					continue
				}
				cProvider = provider
				c.SetPrompt(cProvider)
				nxt <- struct{}{}
			}
		}
	}
}
<commit_msg>create provider interface<commit_after>package main

import (
	"github.com/mehrdadrad/myping/cli"
	"github.com/mehrdadrad/myping/icmp"
	"github.com/mehrdadrad/myping/icmp/telia"
	"net"
	"regexp"
	"strings"
)

type Provider interface {
	//Init()
	GetNodes() map[string]string
	Ping() (string, error)
}

var providers = map[string]Provider{"telia": new(telia.Provider)}

func validateProvider(p string) (string, error) {
	p = strings.ToLower(p)
	return p, nil
}

func main() {
	var (
		err       error
		cProvider string = "local"
	)

	rep := make(chan string, 1)
	cmd := make(chan string, 1)
	nxt := make(chan struct{}, 1)

	c := cli.Init("local")
	go c.Run(cmd, nxt)

	r, _ := regexp.Compile("(ping|connect) (.*)")

	for {
		select {
		case req := <-cmd:
			subReq := r.FindStringSubmatch(req)
			if len(subReq) == 0 {
				println("syntax error")
				nxt <- struct{}{}
				continue
			}
			switch {
			case subReq[1] == "ping" && cProvider == "local":
				p := icmp.NewPing()
				ra, err := net.ResolveIPAddr("ip", subReq[2])
				if err != nil {
					println("cannot resolve", subReq[2], ": Unknown host")
					nxt <- struct{}{}
					continue
				}
				p.IP(ra.String())
				for n := 0; n < 4; n++ {
					p.Ping(rep)
					println(<-rep)
				}
				nxt <- struct{}{}
			case subReq[1] == "ping" && cProvider == "telia":
				p := telia.Init(subReq[2], "ipv4", "Los Angeles")
				m, _ := p.Ping()
				println(m)
				nxt <- struct{}{}
			case subReq[1] == "connect":
				var provider string
				if provider, err = validateProvider(subReq[2]); err != nil {
					println("provider not available")
					nxt <- struct{}{}
					continue
				}
				cProvider = provider
				c.SetPrompt(cProvider)
				nxt <- struct{}{}
			}
		}
	}
}
<|endoftext|>