<commit_before>/*
Copyright 2017 Matthew Lord (mattalord@gmail.com)

Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are met:

1. Redistributions of source code must retain the above copyright notice, this list of conditions and the following disclaimer.

2. Redistributions in binary form must reproduce the above copyright notice, this list of conditions and the following disclaimer in the documentation and/or other materials provided with the distribution.

3. Neither the name of the copyright holder nor the names of its contributors may be used to endorse or promote products derived from this software without specific prior written permission.

THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
*/

package main

import (
  "os"
  "fmt"
  "log"
  "github.com/mattlord/myarbitratord/group_replication/instance"
  "time"
// "flag"
  "sort"
)

type MembersByOnlineNodes []instance.Instance

func main(){
  if( len(os.Args) < 3 ){
    fmt.Println( "myarbitratord usage: myarbitratord <seed_host> <seed_port>" )
    os.Exit(1);
  }

  seed_host := os.Args[1]
  seed_port := os.Args[2]
  mysql_user := "root"
  mysql_pass := "!root19M"

  // check out flag.String( "foo" ) for option handling ...

  fmt.Println( "Welcome to the MySQL Group Replication Arbitrator!" )

  fmt.Printf( "Starting operations from seed node: '%s:%s'\n", seed_host, seed_port )

  seed_node := instance.New( seed_host, seed_port, mysql_user, mysql_pass )
  err := MonitorCluster( seed_node )

  if( err != nil ){
    log.Fatal( err )
    os.Exit( 100 )
  } else {
    os.Exit( 0 )
  }
}

func MonitorCluster( seed_node *instance.Instance ) error {
  loop := true
  var err error
  last_view := []instance.Instance{}

  for( loop == true ){
    // let's check the status of the current seed node
    // if the seed node
    err := seed_node.Connect()

    if( err != nil || seed_node.Member_state != "ONLINE" ){
      // if we couldn't connect to the current seed node or it's no longer part of the group
      // let's try and get a new seed node from the last known membership view
      for _, member := range last_view {
        member.Connect()
        if( err == nil && member.Member_state == "ONLINE"){
          seed_node = &member
          fmt.Println( "Updated seed node! New seed node is: ", seed_node.Mysql_host, ":", seed_node.Mysql_port )
          break
        }
      }
    }

    members, err := seed_node.GetMembers()

    if( err != nil ){
      log.Fatal( err )
    }

    // save this view in case the seed node is no longer valid next time
    //last_view = copy( last_view, members )
    last_view = *members

    quorum, err := seed_node.HasQuorum()

    if( err != nil ){
      log.Fatal( err )
    }

    if( quorum ){
      // Let's try and shutdown the nodes NOT in the primary partition if we can reach them from the arbitrator

      for _, member := range *members {
        if( member.Member_state == "ERROR" || member.Member_state == "UNREACHABLE" ){
          fmt.Println( "Shutting down non-healthy node: ", member.Mysql_host, ":", member.Mysql_port )
          err = member.Shutdown()

          if( err != nil ){
            fmt.Println( "Could not shutdown instance: ", member.Mysql_host, ":", member.Mysql_port )
          }
        }
      }
    } else {
      // handling network partitions and split brain scenarios will be much trickier... I'll need to try and
      // contact each member in the last seen view and try to determine which partition should become the
      // primary one we'll then need to contact 1 node in the primary partition and explicitly set the new
      // membership with 'set global group_replication_force_members="<node_list>"' and finally we'll need
      // to try and connect to the nodes on the losing side of the partition and attempt to shutdown mysqld

      // does anyone have a quorum? Let's double check before forcing the membership
      primary_partition := false
      for _, member := range last_view {
        member.Connect()
        quorum, err := member.HasQuorum()
        if( err == nil && quorum ){
          seed_node = &member
          primary_partition = true
          break
        }
      }

      // If noone in fact has a quorum, then let's see which partition has the most
      // online/participating/communicating members; the participants in that partition
      // will then be the ones that we use to force the new membership and unlock the cluster
      if( primary_partition == false ){
        sort.Sort( MembersByOnlineNodes(last_view) )
        // now the last element in the array is the one to use as it's coordinating with the most nodes
        seed_node = &last_view[len(last_view)-1]

        // let's build a string of '<host>:<port>' combinations that we want to use for the new membership view
        members, _ := seed_node.GetMembers()
        force_member_string := ""
        for _, member := range *members {
          if( member.Member_state == "ONLINE" ){
            force_member_string = force_member_string + member.Mysql_host + ":" + member.Mysql_port
          }
        }

        fmt.Println( "Forcing group membership! using: '", force_member_string, "'" )

        err = seed_node.ForceMembers( force_member_string )
      }
    }

    if( err != nil ){
      loop = false
    } else {
      time.Sleep( time.Millisecond * 2000 )
    }
  }

  return err
}

// The remaining functions are used to sort our membership slice
func (a MembersByOnlineNodes) Len() int {
  return len(a)
}

func (a MembersByOnlineNodes) Swap( i, j int ) {
  a[i], a[j] = a[j], a[i]
}

func (a MembersByOnlineNodes) Less( i, j int ) bool {

  return a[i].Online_participants < a[j].Online_participants
}
<commit_msg>minor spacing<commit_after>/*
Copyright 2017 Matthew Lord (mattalord@gmail.com)

Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are met:

1. Redistributions of source code must retain the above copyright notice, this list of conditions and the following disclaimer.

2. Redistributions in binary form must reproduce the above copyright notice, this list of conditions and the following disclaimer in the documentation and/or other materials provided with the distribution.

3. Neither the name of the copyright holder nor the names of its contributors may be used to endorse or promote products derived from this software without specific prior written permission.

THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
*/

package main

import (
  "os"
  "fmt"
  "log"
  "github.com/mattlord/myarbitratord/group_replication/instance"
  "time"
// "flag"
  "sort"
)

type MembersByOnlineNodes []instance.Instance

func main(){
  if( len(os.Args) < 3 ){
    fmt.Println( "myarbitratord usage: myarbitratord <seed_host> <seed_port>" )
    os.Exit(1);
  }

  seed_host := os.Args[1]
  seed_port := os.Args[2]
  mysql_user := "root"
  mysql_pass := "!root19M"

  // check out flag.String( "foo" ) for option handling ...

  fmt.Println( "Welcome to the MySQL Group Replication Arbitrator!" )

  fmt.Printf( "Starting operations from seed node: '%s:%s'\n", seed_host, seed_port )

  seed_node := instance.New( seed_host, seed_port, mysql_user, mysql_pass )
  err := MonitorCluster( seed_node )

  if( err != nil ){
    log.Fatal( err )
    os.Exit( 100 )
  } else {
    os.Exit( 0 )
  }
}

func MonitorCluster( seed_node *instance.Instance ) error {
  loop := true
  var err error
  last_view := []instance.Instance{}

  for( loop == true ){
    // let's check the status of the current seed node
    // if the seed node
    err := seed_node.Connect()

    if( err != nil || seed_node.Member_state != "ONLINE" ){
      // if we couldn't connect to the current seed node or it's no longer part of the group
      // let's try and get a new seed node from the last known membership view
      for _, member := range last_view {
        member.Connect()
        if( err == nil && member.Member_state == "ONLINE"){
          seed_node = &member
          fmt.Println( "Updated seed node! New seed node is: ", seed_node.Mysql_host, ":", seed_node.Mysql_port )
          break
        }
      }
    }

    members, err := seed_node.GetMembers()

    if( err != nil ){
      log.Fatal( err )
    }

    // save this view in case the seed node is no longer valid next time
    //last_view = copy( last_view, members )
    last_view = *members

    quorum, err := seed_node.HasQuorum()

    if( err != nil ){
      log.Fatal( err )
    }

    if( quorum ){
      // Let's try and shutdown the nodes NOT in the primary partition if we can reach them from the arbitrator

      for _, member := range *members {
        if( member.Member_state == "ERROR" || member.Member_state == "UNREACHABLE" ){
          fmt.Println( "Shutting down non-healthy node: ", member.Mysql_host, ":", member.Mysql_port )
          err = member.Shutdown()

          if( err != nil ){
            fmt.Println( "Could not shutdown instance: ", member.Mysql_host, ":", member.Mysql_port )
          }
        }
      }
    } else {
      // handling network partitions and split brain scenarios will be much trickier... I'll need to try and
      // contact each member in the last seen view and try to determine which partition should become the
      // primary one we'll then need to contact 1 node in the primary partition and explicitly set the new
      // membership with 'set global group_replication_force_members="<node_list>"' and finally we'll need
      // to try and connect to the nodes on the losing side of the partition and attempt to shutdown mysqld

      // does anyone have a quorum? Let's double check before forcing the membership
      primary_partition := false
      for _, member := range last_view {
        member.Connect()
        quorum, err := member.HasQuorum()
        if( err == nil && quorum ){
          seed_node = &member
          primary_partition = true
          break
        }
      }

      // If noone in fact has a quorum, then let's see which partition has the most
      // online/participating/communicating members; the participants in that partition
      // will then be the ones that we use to force the new membership and unlock the cluster
      if( primary_partition == false ){
        sort.Sort( MembersByOnlineNodes(last_view) )
        // now the last element in the array is the one to use as it's coordinating with the most nodes
        seed_node = &last_view[len(last_view)-1]

        // let's build a string of '<host>:<port>' combinations that we want to use for the new membership view
        members, _ := seed_node.GetMembers()
        force_member_string := ""
        for _, member := range *members {
          if( member.Member_state == "ONLINE" ){
            force_member_string = force_member_string + member.Mysql_host + ":" + member.Mysql_port
          }
        }

        fmt.Println( "Forcing group membership! using: '", force_member_string, "'" )
        err = seed_node.ForceMembers( force_member_string )
      }
    }

    if( err != nil ){
      loop = false
    } else {
      time.Sleep( time.Millisecond * 2000 )
    }
  }

  return err
}

// The remaining functions are used to sort our membership slice
func (a MembersByOnlineNodes) Len() int {
  return len(a)
}

func (a MembersByOnlineNodes) Swap( i, j int ) {
  a[i], a[j] = a[j], a[i]
}

func (a MembersByOnlineNodes) Less( i, j int ) bool {
  return a[i].Online_participants < a[j].Online_participants
}
<|endoftext|>


<commit_before>/*
Copyright 2015 The Kubernetes Authors All rights reserved.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package e2e

import (
	"time"

	"k8s.io/kubernetes/pkg/api"
	client "k8s.io/kubernetes/pkg/client/unversioned"
	"k8s.io/kubernetes/pkg/util"
	"k8s.io/kubernetes/pkg/util/wait"

	. "github.com/onsi/ginkgo"
	. "github.com/onsi/gomega"
)

var _ = Describe("Probing container", func() {
	framework := Framework{BaseName: "container-probe"}
	var podClient client.PodInterface
	probe := webserverProbeBuilder{}

	BeforeEach(func() {
		framework.beforeEach()
		podClient = framework.Client.Pods(framework.Namespace.Name)
	})

	AfterEach(framework.afterEach)

	It("with readiness probe should not be ready before initial delay and never restart [Conformance]", func() {
		p, err := podClient.Create(makePodSpec(probe.withInitialDelay().build(), nil))
		expectNoError(err)
		startTime := time.Now()

		Expect(wait.Poll(poll, 90*time.Second, func() (bool, error) {
			p, err := podClient.Get(p.Name)
			if err != nil {
				return false, err
			}
			ready := api.IsPodReady(p)
			if !ready {
				Logf("pod is not yet ready; pod has phase %q.", p.Status.Phase)
				return false, nil
			}
			return true, nil
		})).NotTo(HaveOccurred(), "pod never became ready")

		if time.Since(startTime) < 30*time.Second {
			Failf("Pod became ready before it's initial delay")
		}

		p, err = podClient.Get(p.Name)
		expectNoError(err)

		isReady, err := podRunningReady(p)
		expectNoError(err)
		Expect(isReady).To(BeTrue(), "pod should be ready")

		restartCount := getRestartCount(p)
		Expect(restartCount == 0).To(BeTrue(), "pod should have a restart count of 0 but got %v", restartCount)
	})

	It("with readiness probe that fails should never be ready and never restart [Conformance]", func() {
		p, err := podClient.Create(makePodSpec(probe.withFailing().build(), nil))
		expectNoError(err)

		err = wait.Poll(poll, 90*time.Second, func() (bool, error) {
			p, err := podClient.Get(p.Name)
			if err != nil {
				return false, err
			}
			return api.IsPodReady(p), nil
		})
		if err != wait.ErrWaitTimeout {
			Failf("expecting wait timeout error but got: %v", err)
		}

		p, err = podClient.Get(p.Name)
		expectNoError(err)

		isReady, err := podRunningReady(p)
		Expect(isReady).NotTo(BeTrue(), "pod should be not ready")

		restartCount := getRestartCount(p)
		Expect(restartCount == 0).To(BeTrue(), "pod should have a restart count of 0 but got %v", restartCount)
	})

})

func getRestartCount(p *api.Pod) int {
	count := 0
	for _, containerStatus := range p.Status.ContainerStatuses {
		count += containerStatus.RestartCount
	}
	return count
}

func makePodSpec(readinessProbe, livenessProbe *api.Probe) *api.Pod {
	pod := &api.Pod{
		ObjectMeta: api.ObjectMeta{Name: "test-webserver-" + string(util.NewUUID())},
		Spec: api.PodSpec{
			Containers: []api.Container{
				{
					Name:           "test-webserver",
					Image:          "gcr.io/google_containers/test-webserver",
					LivenessProbe:  livenessProbe,
					ReadinessProbe: readinessProbe,
				}, {
					Name:  "test-noprobe",
					Image: "gcr.io/google_containers/pause",
				},
			},
		},
	}
	return pod
}

type webserverProbeBuilder struct {
	failing      bool
	initialDelay bool
}

func (b webserverProbeBuilder) withFailing() webserverProbeBuilder {
	b.failing = true
	return b
}

func (b webserverProbeBuilder) withInitialDelay() webserverProbeBuilder {
	b.initialDelay = true
	return b
}

func (b webserverProbeBuilder) build() *api.Probe {
	probe := &api.Probe{
		Handler: api.Handler{
			HTTPGet: &api.HTTPGetAction{
				Port: util.NewIntOrStringFromInt(80),
				Path: "/",
			},
		},
	}
	if b.initialDelay {
		probe.InitialDelaySeconds = 30
	}
	if b.failing {
		probe.HTTPGet.Port = util.NewIntOrStringFromInt(81)
	}
	return probe
}
<commit_msg>Using right pause image in container_probe test<commit_after>/*
Copyright 2015 The Kubernetes Authors All rights reserved.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package e2e

import (
	"time"

	"k8s.io/kubernetes/pkg/api"
	client "k8s.io/kubernetes/pkg/client/unversioned"
	"k8s.io/kubernetes/pkg/util"
	"k8s.io/kubernetes/pkg/util/wait"

	. "github.com/onsi/ginkgo"
	. "github.com/onsi/gomega"
)

var _ = Describe("Probing container", func() {
	framework := Framework{BaseName: "container-probe"}
	var podClient client.PodInterface
	probe := webserverProbeBuilder{}

	BeforeEach(func() {
		framework.beforeEach()
		podClient = framework.Client.Pods(framework.Namespace.Name)
	})

	AfterEach(framework.afterEach)

	It("with readiness probe should not be ready before initial delay and never restart [Conformance]", func() {
		p, err := podClient.Create(makePodSpec(probe.withInitialDelay().build(), nil))
		expectNoError(err)
		startTime := time.Now()

		Expect(wait.Poll(poll, 90*time.Second, func() (bool, error) {
			p, err := podClient.Get(p.Name)
			if err != nil {
				return false, err
			}
			ready := api.IsPodReady(p)
			if !ready {
				Logf("pod is not yet ready; pod has phase %q.", p.Status.Phase)
				return false, nil
			}
			return true, nil
		})).NotTo(HaveOccurred(), "pod never became ready")

		if time.Since(startTime) < 30*time.Second {
			Failf("Pod became ready before it's initial delay")
		}

		p, err = podClient.Get(p.Name)
		expectNoError(err)

		isReady, err := podRunningReady(p)
		expectNoError(err)
		Expect(isReady).To(BeTrue(), "pod should be ready")

		restartCount := getRestartCount(p)
		Expect(restartCount == 0).To(BeTrue(), "pod should have a restart count of 0 but got %v", restartCount)
	})

	It("with readiness probe that fails should never be ready and never restart [Conformance]", func() {
		p, err := podClient.Create(makePodSpec(probe.withFailing().build(), nil))
		expectNoError(err)

		err = wait.Poll(poll, 90*time.Second, func() (bool, error) {
			p, err := podClient.Get(p.Name)
			if err != nil {
				return false, err
			}
			return api.IsPodReady(p), nil
		})
		if err != wait.ErrWaitTimeout {
			Failf("expecting wait timeout error but got: %v", err)
		}

		p, err = podClient.Get(p.Name)
		expectNoError(err)

		isReady, err := podRunningReady(p)
		Expect(isReady).NotTo(BeTrue(), "pod should be not ready")

		restartCount := getRestartCount(p)
		Expect(restartCount == 0).To(BeTrue(), "pod should have a restart count of 0 but got %v", restartCount)
	})

})

func getRestartCount(p *api.Pod) int {
	count := 0
	for _, containerStatus := range p.Status.ContainerStatuses {
		count += containerStatus.RestartCount
	}
	return count
}

func makePodSpec(readinessProbe, livenessProbe *api.Probe) *api.Pod {
	pod := &api.Pod{
		ObjectMeta: api.ObjectMeta{Name: "test-webserver-" + string(util.NewUUID())},
		Spec: api.PodSpec{
			Containers: []api.Container{
				{
					Name:           "test-webserver",
					Image:          "gcr.io/google_containers/test-webserver",
					LivenessProbe:  livenessProbe,
					ReadinessProbe: readinessProbe,
				}, {
					Name:  "test-noprobe",
					Image: "beta.gcr.io/google_containers/pause:2.0",
				},
			},
		},
	}
	return pod
}

type webserverProbeBuilder struct {
	failing      bool
	initialDelay bool
}

func (b webserverProbeBuilder) withFailing() webserverProbeBuilder {
	b.failing = true
	return b
}

func (b webserverProbeBuilder) withInitialDelay() webserverProbeBuilder {
	b.initialDelay = true
	return b
}

func (b webserverProbeBuilder) build() *api.Probe {
	probe := &api.Probe{
		Handler: api.Handler{
			HTTPGet: &api.HTTPGetAction{
				Port: util.NewIntOrStringFromInt(80),
				Path: "/",
			},
		},
	}
	if b.initialDelay {
		probe.InitialDelaySeconds = 30
	}
	if b.failing {
		probe.HTTPGet.Port = util.NewIntOrStringFromInt(81)
	}
	return probe
}
<|endoftext|>


<commit_before>/*
Copyright 2016 The Kubernetes Authors All rights reserved.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package integration

import (
	"flag"
	"fmt"
	"math"
	"os"
	"runtime"
	"strconv"
	"strings"
	"testing"
	"time"

	"k8s.io/minikube/pkg/minikube/driver"
)

// General configuration: used to set the VM Driver
var startArgs = flag.String("minikube-start-args", "", "Arguments to pass to minikube start")

// Flags for faster local integration testing
var forceProfile = flag.String("profile", "", "force tests to run against a particular profile")
var cleanup = flag.Bool("cleanup", true, "cleanup failed test run")
var enableGvisor = flag.Bool("gvisor", false, "run gvisor integration test (slow)")
var postMortemLogs = flag.Bool("postmortem-logs", true, "show logs after a failed test run")
var timeOutMultiplier = flag.Float64("timeout-multiplier", 1, "multiply the timeout for the tests")

// Paths to files - normally set for CI
var binaryPath = flag.String("binary", "../../out/minikube", "path to minikube binary")
var testdataDir = flag.String("testdata-dir", "testdata", "the directory relative to test/integration where the testdata lives")

// Node names are consistent, let's store these for easy access later
const (
	SecondNodeName = "m02"
	ThirdNodeName  = "m03"
)

// TestMain is the test main
func TestMain(m *testing.M) {
	flag.Parse()
	setMaxParallelism()

	start := time.Now()
	code := m.Run()
	fmt.Printf("Tests completed in %s (result code %d)\n", time.Since(start), code)
	os.Exit(code)
}

// setMaxParallelism caps the max parallelism. Go assumes 1 core per test, whereas minikube needs 2 cores per test.
func setMaxParallelism() {
	procs := runtime.GOMAXPROCS(0)
	pval := flag.Lookup("test.parallel").Value.String()
	pv, err := strconv.Atoi(pval)
	if err != nil {
		fmt.Fprintf(os.Stderr, "unable to parse --test.parallel value: %q\n", pval)
		return
	}

	if procs != pv {
		fmt.Fprintf(os.Stderr, "--test-parallel=%d was set via flags (system has %d cores)\n", pv, procs)
		return
	}

	if procs == 2 {
		fmt.Fprintf(os.Stderr, "Found %d cores, will not round down core count.\n", procs)
		return
	}

	// Each "minikube start" consumes up to 2 cores - during startup, ~1.5 cores is typical
	limit := int(math.Floor(float64(procs) * 0.6))
	fmt.Fprintf(os.Stderr, "Found %d cores, limiting parallelism with --test.parallel=%d\n", procs, limit)
	if err := flag.Set("test.parallel", strconv.Itoa(limit)); err != nil {
		fmt.Fprintf(os.Stderr, "Unable to set test.parallel: %v\n", err)
	}
	runtime.GOMAXPROCS(limit)
}

// StartArgs returns the arguments normally used for starting minikube
func StartArgs() []string {
	return strings.Split(*startArgs, " ")
}

// Target returns where the minikube binary can be found
func Target() string {
	return *binaryPath
}

// NoneDriver returns whether or not this test is using the none driver
func NoneDriver() bool {
	return strings.Contains(*startArgs, "--driver=none") || strings.Contains(*startArgs, "--vm-driver=none")
}

// HyperVDriver returns whether or not this test is using the Hyper-V driver
func HyperVDriver() bool {
	return strings.Contains(*startArgs, "--driver=hyperv") || strings.Contains(*startArgs, "--vm-driver=hyperv")
}

// DockerDriver returns whether or not this test is using the docker or podman driver
func DockerDriver() bool {
	return strings.Contains(*startArgs, "--driver=docker") || strings.Contains(*startArgs, "--vm-driver=docker")
}

// PodmanDriver returns whether or not this test is using the docker or podman driver
func PodmanDriver() bool {
	return strings.Contains(*startArgs, "--vm-driver=podman") || strings.Contains(*startArgs, "driver=podman")
}

// KicDriver returns whether or not this test is using the docker or podman driver
func KicDriver() bool {
	return DockerDriver() || PodmanDriver()
}

// GithubActionRunner returns true if running inside a github action runner
func GithubActionRunner() bool {
	// based on https://help.github.com/en/actions/configuring-and-managing-workflows/using-environment-variables
	return os.Getenv("GITHUB_ACTIONS") == "true"
}

// NeedsPortForward returns access to endpoints with this driver needs port forwarding
// (Docker on non-Linux platforms requires ports to be forwarded to 127.0.0.1)
func NeedsPortForward() bool {
	return KicDriver() && (runtime.GOOS == "windows" || runtime.GOOS == "darwin") || driver.IsMicrosoftWSL()
}

// CanCleanup returns if cleanup is allowed
func CanCleanup() bool {
	return *cleanup
}

// Minutes will return timeout in minutes based on how slow the machine is
func Minutes(n int) time.Duration {
	return time.Duration(*timeOutMultiplier) * time.Duration(n) * time.Minute
}

// Seconds will return timeout in minutes based on how slow the machine is
func Seconds(n int) time.Duration {
	return time.Duration(*timeOutMultiplier) * time.Duration(n) * time.Second
}
<commit_msg>Make the calculation easier to understand<commit_after>/*
Copyright 2016 The Kubernetes Authors All rights reserved.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package integration

import (
	"flag"
	"fmt"
	"math"
	"os"
	"runtime"
	"strconv"
	"strings"
	"testing"
	"time"

	"k8s.io/minikube/pkg/minikube/driver"
)

// General configuration: used to set the VM Driver
var startArgs = flag.String("minikube-start-args", "", "Arguments to pass to minikube start")

// Flags for faster local integration testing
var forceProfile = flag.String("profile", "", "force tests to run against a particular profile")
var cleanup = flag.Bool("cleanup", true, "cleanup failed test run")
var enableGvisor = flag.Bool("gvisor", false, "run gvisor integration test (slow)")
var postMortemLogs = flag.Bool("postmortem-logs", true, "show logs after a failed test run")
var timeOutMultiplier = flag.Float64("timeout-multiplier", 1, "multiply the timeout for the tests")

// Paths to files - normally set for CI
var binaryPath = flag.String("binary", "../../out/minikube", "path to minikube binary")
var testdataDir = flag.String("testdata-dir", "testdata", "the directory relative to test/integration where the testdata lives")

// Node names are consistent, let's store these for easy access later
const (
	SecondNodeName = "m02"
	ThirdNodeName  = "m03"
)

// TestMain is the test main
func TestMain(m *testing.M) {
	flag.Parse()
	setMaxParallelism()

	start := time.Now()
	code := m.Run()
	fmt.Printf("Tests completed in %s (result code %d)\n", time.Since(start), code)
	os.Exit(code)
}

// setMaxParallelism caps the max parallelism. Go assumes 1 core per test, whereas minikube needs 2 cores per test.
func setMaxParallelism() {
	procs := runtime.GOMAXPROCS(0)
	pval := flag.Lookup("test.parallel").Value.String()
	pv, err := strconv.Atoi(pval)
	if err != nil {
		fmt.Fprintf(os.Stderr, "unable to parse --test.parallel value: %q\n", pval)
		return
	}

	if procs != pv {
		fmt.Fprintf(os.Stderr, "--test-parallel=%d was set via flags (system has %d cores)\n", pv, procs)
		return
	}

	if procs == 2 {
		fmt.Fprintf(os.Stderr, "Found %d cores, will not round down core count.\n", procs)
		return
	}

	// Each "minikube start" consumes up to 2 cores, though the average usage is somewhat lower
	limit := int(math.Floor(float64(procs) / 1.75))

	fmt.Fprintf(os.Stderr, "Found %d cores, limiting parallelism with --test.parallel=%d\n", procs, limit)
	if err := flag.Set("test.parallel", strconv.Itoa(limit)); err != nil {
		fmt.Fprintf(os.Stderr, "Unable to set test.parallel: %v\n", err)
	}
	runtime.GOMAXPROCS(limit)
}

// StartArgs returns the arguments normally used for starting minikube
func StartArgs() []string {
	return strings.Split(*startArgs, " ")
}

// Target returns where the minikube binary can be found
func Target() string {
	return *binaryPath
}

// NoneDriver returns whether or not this test is using the none driver
func NoneDriver() bool {
	return strings.Contains(*startArgs, "--driver=none") || strings.Contains(*startArgs, "--vm-driver=none")
}

// HyperVDriver returns whether or not this test is using the Hyper-V driver
func HyperVDriver() bool {
	return strings.Contains(*startArgs, "--driver=hyperv") || strings.Contains(*startArgs, "--vm-driver=hyperv")
}

// DockerDriver returns whether or not this test is using the docker or podman driver
func DockerDriver() bool {
	return strings.Contains(*startArgs, "--driver=docker") || strings.Contains(*startArgs, "--vm-driver=docker")
}

// PodmanDriver returns whether or not this test is using the docker or podman driver
func PodmanDriver() bool {
	return strings.Contains(*startArgs, "--vm-driver=podman") || strings.Contains(*startArgs, "driver=podman")
}

// KicDriver returns whether or not this test is using the docker or podman driver
func KicDriver() bool {
	return DockerDriver() || PodmanDriver()
}

// GithubActionRunner returns true if running inside a github action runner
func GithubActionRunner() bool {
	// based on https://help.github.com/en/actions/configuring-and-managing-workflows/using-environment-variables
	return os.Getenv("GITHUB_ACTIONS") == "true"
}

// NeedsPortForward returns access to endpoints with this driver needs port forwarding
// (Docker on non-Linux platforms requires ports to be forwarded to 127.0.0.1)
func NeedsPortForward() bool {
	return KicDriver() && (runtime.GOOS == "windows" || runtime.GOOS == "darwin") || driver.IsMicrosoftWSL()
}

// CanCleanup returns if cleanup is allowed
func CanCleanup() bool {
	return *cleanup
}

// Minutes will return timeout in minutes based on how slow the machine is
func Minutes(n int) time.Duration {
	return time.Duration(*timeOutMultiplier) * time.Duration(n) * time.Minute
}

// Seconds will return timeout in minutes based on how slow the machine is
func Seconds(n int) time.Duration {
	return time.Duration(*timeOutMultiplier) * time.Duration(n) * time.Second
}
<|endoftext|>


<commit_before>// Copyright 2009 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

package os

import (
	"os";
	"syscall";
)

// ForkExec forks the current process and invokes Exec with the file, arguments,
// and environment specified by argv0, argv, and envv.  It returns the process
// id of the forked process and an Error, if any.  The fd array specifies the
// file descriptors to be set up in the new process: fd[0] will be Unix file
// descriptor 0 (standard input), fd[1] descriptor 1, and so on.  A nil entry
// will cause the child to have no open file descriptor with that index.
// If dir is not empty, the child chdirs into the directory before execing the program.
func ForkExec(argv0 string, argv []string, envv []string, dir string, fd []*File)
	(pid int, err Error)
{
	// Create array of integer (system) fds.
	intfd := make([]int64, len(fd));
	for i, f := range(fd) {
		if f == nil {
			intfd[i] = -1;
		} else {
			intfd[i] = f.Fd();
		}
	}

	p, e := syscall.ForkExec(argv0, argv, envv, dir, intfd);
	return int(p), ErrnoToError(e);
}

// Exec replaces the current process with an execution of the program
// named by argv0, with arguments argv and environment envv.
// If successful, Exec never returns.  If it fails, it returns an Error.
// ForkExec is almost always a better way to execute a program.
func Exec(argv0 string, argv []string, envv []string) Error {
	if envv == nil {
		envv = Environ();
	}
	e := syscall.Exec(argv0, argv, envv);
	return ErrnoToError(e);
}

// TODO(rsc): Should os implement its own syscall.WaitStatus
// wrapper with the methods, or is exposing the underlying one enough?
//
// TODO(rsc): Certainly need to have os.Rusage struct,
// since syscall one might have different field types across
// different OS.

// Waitmsg stores the information about an exited process as reported by Wait.
type Waitmsg struct {
	Pid int;	// The process's id.
	syscall.WaitStatus;	// System-dependent status info.
	Rusage *syscall.Rusage;	// System-dependent resource usage info.
}

// Options for Wait.
const (
	WNOHANG = syscall.WNOHANG;	// Don't wait if no process has exited.
	WSTOPPED = syscall.WSTOPPED;	// If set, status of stopped subprocesses is also reported.
	WUNTRACED = WSTOPPED;
	WRUSAGE = 1<<60;	// Record resource usage.
)

// Wait waits for process pid to exit or stop, and then returns a
// Waitmsg describing its status and an Error, if any. The options
// (WNOHANG etc.) affect the behavior of the Wait call.
func Wait(pid int, options uint64) (w *Waitmsg, err Error) {
	var status syscall.WaitStatus;
	var rusage *syscall.Rusage;
	if options & WRUSAGE != 0 {
		rusage = new(syscall.Rusage);
		options ^= WRUSAGE;
	}
	pid1, e := syscall.Wait4(int64(pid), &status, int64(options), rusage);
	if e != 0 {
		return nil, ErrnoToError(e);
	}
	w = new(Waitmsg);
	w.Pid = pid;
	w.WaitStatus = status;
	w.Rusage = rusage;
	return w, nil;
}

<commit_msg>Add os.Getpid and os.Getppid.<commit_after>// Copyright 2009 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

package os

import (
	"os";
	"syscall";
)

// ForkExec forks the current process and invokes Exec with the file, arguments,
// and environment specified by argv0, argv, and envv.  It returns the process
// id of the forked process and an Error, if any.  The fd array specifies the
// file descriptors to be set up in the new process: fd[0] will be Unix file
// descriptor 0 (standard input), fd[1] descriptor 1, and so on.  A nil entry
// will cause the child to have no open file descriptor with that index.
// If dir is not empty, the child chdirs into the directory before execing the program.
func ForkExec(argv0 string, argv []string, envv []string, dir string, fd []*File)
	(pid int, err Error)
{
	// Create array of integer (system) fds.
	intfd := make([]int64, len(fd));
	for i, f := range(fd) {
		if f == nil {
			intfd[i] = -1;
		} else {
			intfd[i] = f.Fd();
		}
	}

	p, e := syscall.ForkExec(argv0, argv, envv, dir, intfd);
	return int(p), ErrnoToError(e);
}

// Exec replaces the current process with an execution of the program
// named by argv0, with arguments argv and environment envv.
// If successful, Exec never returns.  If it fails, it returns an Error.
// ForkExec is almost always a better way to execute a program.
func Exec(argv0 string, argv []string, envv []string) Error {
	if envv == nil {
		envv = Environ();
	}
	e := syscall.Exec(argv0, argv, envv);
	return ErrnoToError(e);
}

// TODO(rsc): Should os implement its own syscall.WaitStatus
// wrapper with the methods, or is exposing the underlying one enough?
//
// TODO(rsc): Certainly need to have os.Rusage struct,
// since syscall one might have different field types across
// different OS.

// Waitmsg stores the information about an exited process as reported by Wait.
type Waitmsg struct {
	Pid int;	// The process's id.
	syscall.WaitStatus;	// System-dependent status info.
	Rusage *syscall.Rusage;	// System-dependent resource usage info.
}

// Options for Wait.
const (
	WNOHANG = syscall.WNOHANG;	// Don't wait if no process has exited.
	WSTOPPED = syscall.WSTOPPED;	// If set, status of stopped subprocesses is also reported.
	WUNTRACED = WSTOPPED;
	WRUSAGE = 1<<60;	// Record resource usage.
)

// Wait waits for process pid to exit or stop, and then returns a
// Waitmsg describing its status and an Error, if any. The options
// (WNOHANG etc.) affect the behavior of the Wait call.
func Wait(pid int, options uint64) (w *Waitmsg, err Error) {
	var status syscall.WaitStatus;
	var rusage *syscall.Rusage;
	if options & WRUSAGE != 0 {
		rusage = new(syscall.Rusage);
		options ^= WRUSAGE;
	}
	pid1, e := syscall.Wait4(int64(pid), &status, int64(options), rusage);
	if e != 0 {
		return nil, ErrnoToError(e);
	}
	w = new(Waitmsg);
	w.Pid = pid;
	w.WaitStatus = status;
	w.Rusage = rusage;
	return w, nil;
}

// Getpid returns the process id of the caller.
func Getpid() int {
	p, r2, e := syscall.Syscall(syscall.SYS_GETPID, 0, 0, 0);
	return int(p)
}

// Getppid returns the process id of the caller's parent.
func Getppid() int {
	p, r2, e := syscall.Syscall(syscall.SYS_GETPPID, 0, 0, 0);
	return int(p)
}
<|endoftext|>


<commit_before>// Copyright 2020 The LUCI Authors.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//      http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package handler

import (
	"context"
	"fmt"
	"time"

	"google.golang.org/protobuf/types/known/timestamppb"

	"go.chromium.org/luci/common/clock"
	"go.chromium.org/luci/common/errors"
	"go.chromium.org/luci/common/logging"
	"go.chromium.org/luci/gae/service/datastore"

	"go.chromium.org/luci/cv/internal/common"
	"go.chromium.org/luci/cv/internal/configs/prjcfg"
	"go.chromium.org/luci/cv/internal/gerrit"
	"go.chromium.org/luci/cv/internal/migration/migrationcfg"
	"go.chromium.org/luci/cv/internal/run"
	"go.chromium.org/luci/cv/internal/run/eventpb"
	"go.chromium.org/luci/cv/internal/run/impl/state"
	"go.chromium.org/luci/cv/internal/tryjob/requirement"
	"go.chromium.org/luci/cv/internal/usertext"
)

const (
	// maxPostStartMessageDuration is the max time that a Run will be waiting for
	// the starting message to be posted on every CL.
	maxPostStartMessageDuration = 10 * time.Minute

	logEntryLabelPostStartMessage = "Posting Starting Message"
)

// Start implements Handler interface.
func (impl *Impl) Start(ctx context.Context, rs *state.RunState) (*Result, error) {
	switch status := rs.Status; {
	case status == run.Status_STATUS_UNSPECIFIED:
		err := errors.Reason("CRITICAL: can't start a Run %q with unspecified status", rs.ID).Err()
		common.LogError(ctx, err)
		panic(err)
	case status != run.Status_PENDING:
		logging.Debugf(ctx, "Skip starting Run because this Run is %s", status)
		return &Result{State: rs}, nil
	}

	rs = rs.ShallowCopy()
	cg, runCLs, cls, err := loadCLsAndConfig(ctx, rs, rs.CLs)
	if err != nil {
		return nil, err
	}
	rs = rs.ShallowCopy()
	switch ok, err := checkRunCreate(ctx, rs, cg, runCLs, cls); {
	case err != nil:
		return nil, err
	case !ok:
		return &Result{State: rs}, nil
	}

	rs.UseCVTryjobExecutor, err = migrationcfg.IsCVInChargeOfTryjob(ctx, impl.Env, rs.ID.LUCIProject())
	if err != nil {
		return nil, err
	}
	switch result, err := requirement.Compute(ctx, requirement.Input{
		ConfigGroup: cg,
		RunOwner:    rs.Owner,
		CLs:         runCLs,
		RunOptions:  rs.Options,
	}); {
	case err != nil:
		return nil, err
	case !rs.UseCVTryjobExecutor:
		// Let CQDaemon handle Tryjobs.
	case rs.UseCVTryjobExecutor && result.OK():
		rs.Tryjobs = &run.Tryjobs{
			Requirement:          result.Requirement,
			RequirementVersion:   1,
			RequirementComputeAt: timestamppb.New(clock.Now(ctx).UTC()),
		}
		enqueueRequirementChangedTask(ctx, rs)
	case rs.UseCVTryjobExecutor:
		meta := reviewInputMeta{
			message:        fmt.Sprintf("Failed to compute tryjob requirement. Reason:\n\n%s", result.ComputationFailure.Reason()),
			notify:         gerrit.Whoms{gerrit.Owner, gerrit.CQVoters},
			addToAttention: gerrit.Whoms{gerrit.Owner, gerrit.CQVoters},
			reason:         "Computing tryjob requirement failed",
		}
		metas := make(map[common.CLID]reviewInputMeta, len(cls))
		for _, cl := range rs.CLs {
			metas[cl] = meta
		}
		scheduleTriggersCancellation(ctx, rs, metas, run.Status_FAILED)
		rs.LogInfof(ctx, "Tryjob Requirement Computation", "Failed to compute tryjob requirement. Reason: %s", result.ComputationFailure.Reason())
		return &Result{State: rs}, nil
	}

	rs.EnqueueLongOp(&run.OngoingLongOps_Op{
		Deadline: timestamppb.New(clock.Now(ctx).UTC().Add(maxPostStartMessageDuration)),
		Work: &run.OngoingLongOps_Op_PostStartMessage{
			PostStartMessage: true,
		},
	})
	// Note that it is inevitable that duplicate pickup latency metric maybe
	// emitted for the same Run if the state transition fails later that
	// causes a retry.
	rs.Status = run.Status_RUNNING
	rs.StartTime = datastore.RoundTime(clock.Now(ctx).UTC())
	rs.LogEntries = append(rs.LogEntries, &run.LogEntry{
		Time: timestamppb.New(rs.StartTime),
		Kind: &run.LogEntry_Started_{
			Started: &run.LogEntry_Started{},
		},
	})
	recordPickupLatency(ctx, rs, cg)
	return &Result{State: rs}, nil
}

func (impl *Impl) onCompletedPostStartMessage(ctx context.Context, rs *state.RunState, op *run.OngoingLongOps_Op, result *eventpb.LongOpCompleted) (*Result, error) {
	opID := result.GetOperationId()
	rs = rs.ShallowCopy()
	rs.RemoveCompletedLongOp(opID)
	if rs.Status != run.Status_RUNNING {
		logging.Warningf(ctx, "Long operation to post start message completed but Run is %s: %s", rs.Status, result)
		return &Result{State: rs}, nil
	}

	failRunReason := "Failed to post the starting message"
	switch result.GetStatus() {
	case eventpb.LongOpCompleted_FAILED:
		// NOTE: if there are several CLs across different projects, detailed
		// failure may have sensitive info which shouldn't be shared elsewhere.
		// Future improvement: if posting start message is a common problem,
		// record `result` message and render links to CLs on which CV failed to
		// post starting message.
		rs.LogInfo(ctx, logEntryLabelPostStartMessage, failRunReason)
	case eventpb.LongOpCompleted_EXPIRED:
		failRunReason += fmt.Sprintf(" within the %s deadline", maxPostStartMessageDuration)
		rs.LogInfo(ctx, logEntryLabelPostStartMessage, failRunReason)
	case eventpb.LongOpCompleted_SUCCEEDED:
		// TODO(tandrii): simplify once all such events have timestamp.
		if t := result.GetPostStartMessage().GetTime(); t != nil {
			rs.LogInfoAt(t.AsTime(), logEntryLabelPostStartMessage, "posted start message on each CL")
		} else {
			rs.LogInfo(ctx, logEntryLabelPostStartMessage, "posted start message on each CL")
		}
		return &Result{State: rs}, nil
	case eventpb.LongOpCompleted_CANCELLED:
		rs.LogInfo(ctx, logEntryLabelPostStartMessage, "cancelled posting start message on CL(s)")
		return &Result{State: rs}, nil
	default:
		panic(fmt.Errorf("unexpected LongOpCompleted status: %s", result.GetStatus()))
	}

	msgPrefix, attentionReason := usertext.OnRunFailed(rs.Mode)
	meta := reviewInputMeta{
		notify:         gerrit.Whoms{gerrit.Owner, gerrit.CQVoters},
		addToAttention: gerrit.Whoms{gerrit.Owner, gerrit.CQVoters},
		reason:         attentionReason,
		message:        msgPrefix + "\n\n" + failRunReason,
	}
	if err := impl.cancelTriggers(ctx, rs, meta); err != nil {
		return nil, err
	}
	return &Result{
		State:        rs,
		SideEffectFn: impl.endRun(ctx, rs, run.Status_FAILED),
	}, nil
}

func recordPickupLatency(ctx context.Context, rs *state.RunState, cg *prjcfg.ConfigGroup) {
	delay := rs.StartTime.Sub(rs.CreateTime)
	metricPickupLatencyS.Add(ctx, delay.Seconds(), rs.ID.LUCIProject())

	if d := cg.Content.GetCombineCls().GetStabilizationDelay(); d != nil {
		delay -= d.AsDuration()
	}
	metricPickupLatencyAdjustedS.Add(ctx, delay.Seconds(), rs.ID.LUCIProject())

	if delay >= 1*time.Minute {
		logging.Warningf(ctx, "Too large adjusted pickup delay: %s", delay)
	}
}
<commit_msg>cv: follow up for comments on https://crrev.com/c/3738960<commit_after>// Copyright 2020 The LUCI Authors.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//      http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package handler

import (
	"context"
	"fmt"
	"time"

	"google.golang.org/protobuf/types/known/timestamppb"

	"go.chromium.org/luci/common/clock"
	"go.chromium.org/luci/common/errors"
	"go.chromium.org/luci/common/logging"
	"go.chromium.org/luci/gae/service/datastore"

	"go.chromium.org/luci/cv/internal/common"
	"go.chromium.org/luci/cv/internal/configs/prjcfg"
	"go.chromium.org/luci/cv/internal/gerrit"
	"go.chromium.org/luci/cv/internal/migration/migrationcfg"
	"go.chromium.org/luci/cv/internal/run"
	"go.chromium.org/luci/cv/internal/run/eventpb"
	"go.chromium.org/luci/cv/internal/run/impl/state"
	"go.chromium.org/luci/cv/internal/tryjob/requirement"
	"go.chromium.org/luci/cv/internal/usertext"
)

const (
	// maxPostStartMessageDuration is the max time that a Run will be waiting for
	// the starting message to be posted on every CL.
	maxPostStartMessageDuration = 10 * time.Minute

	logEntryLabelPostStartMessage = "Posting Starting Message"
)

// Start implements Handler interface.
func (impl *Impl) Start(ctx context.Context, rs *state.RunState) (*Result, error) {
	switch status := rs.Status; {
	case status == run.Status_STATUS_UNSPECIFIED:
		err := errors.Reason("CRITICAL: can't start a Run %q with unspecified status", rs.ID).Err()
		common.LogError(ctx, err)
		panic(err)
	case status != run.Status_PENDING:
		logging.Debugf(ctx, "Skip starting Run because this Run is %s", status)
		return &Result{State: rs}, nil
	}

	rs = rs.ShallowCopy()
	cg, runCLs, cls, err := loadCLsAndConfig(ctx, rs, rs.CLs)
	if err != nil {
		return nil, err
	}
	rs = rs.ShallowCopy()
	switch ok, err := checkRunCreate(ctx, rs, cg, runCLs, cls); {
	case err != nil:
		return nil, err
	case !ok:
		return &Result{State: rs}, nil
	}

	rs.UseCVTryjobExecutor, err = migrationcfg.IsCVInChargeOfTryjob(ctx, impl.Env, rs.ID.LUCIProject())
	if err != nil {
		return nil, err
	}
	switch result, err := requirement.Compute(ctx, requirement.Input{
		ConfigGroup: cg,
		RunOwner:    rs.Owner,
		CLs:         runCLs,
		RunOptions:  rs.Options,
	}); {
	case err != nil:
		return nil, err
	case !rs.UseCVTryjobExecutor:
		// Let CQDaemon handle Tryjobs.
	case result.OK():
		rs.Tryjobs = &run.Tryjobs{
			Requirement:          result.Requirement,
			RequirementVersion:   1,
			RequirementComputeAt: timestamppb.New(clock.Now(ctx).UTC()),
		}
		enqueueRequirementChangedTask(ctx, rs)
	default:
		meta := reviewInputMeta{
			message:        fmt.Sprintf("Failed to compute tryjob requirement. Reason:\n\n%s", result.ComputationFailure.Reason()),
			notify:         gerrit.Whoms{gerrit.Owner, gerrit.CQVoters},
			addToAttention: gerrit.Whoms{gerrit.Owner, gerrit.CQVoters},
			reason:         "Computing tryjob requirement failed",
		}
		metas := make(map[common.CLID]reviewInputMeta, len(cls))
		for _, cl := range rs.CLs {
			metas[cl] = meta
		}
		scheduleTriggersCancellation(ctx, rs, metas, run.Status_FAILED)
		rs.LogInfof(ctx, "Tryjob Requirement Computation", "Failed to compute tryjob requirement. Reason: %s", result.ComputationFailure.Reason())
		return &Result{State: rs}, nil
	}

	rs.EnqueueLongOp(&run.OngoingLongOps_Op{
		Deadline: timestamppb.New(clock.Now(ctx).UTC().Add(maxPostStartMessageDuration)),
		Work: &run.OngoingLongOps_Op_PostStartMessage{
			PostStartMessage: true,
		},
	})
	// Note that it is inevitable that duplicate pickup latency metric maybe
	// emitted for the same Run if the state transition fails later that
	// causes a retry.
	rs.Status = run.Status_RUNNING
	rs.StartTime = datastore.RoundTime(clock.Now(ctx).UTC())
	rs.LogEntries = append(rs.LogEntries, &run.LogEntry{
		Time: timestamppb.New(rs.StartTime),
		Kind: &run.LogEntry_Started_{
			Started: &run.LogEntry_Started{},
		},
	})
	recordPickupLatency(ctx, rs, cg)
	return &Result{State: rs}, nil
}

func (impl *Impl) onCompletedPostStartMessage(ctx context.Context, rs *state.RunState, op *run.OngoingLongOps_Op, result *eventpb.LongOpCompleted) (*Result, error) {
	opID := result.GetOperationId()
	rs = rs.ShallowCopy()
	rs.RemoveCompletedLongOp(opID)
	if rs.Status != run.Status_RUNNING {
		logging.Warningf(ctx, "Long operation to post start message completed but Run is %s: %s", rs.Status, result)
		return &Result{State: rs}, nil
	}

	failRunReason := "Failed to post the starting message"
	switch result.GetStatus() {
	case eventpb.LongOpCompleted_FAILED:
		// NOTE: if there are several CLs across different projects, detailed
		// failure may have sensitive info which shouldn't be shared elsewhere.
		// Future improvement: if posting start message is a common problem,
		// record `result` message and render links to CLs on which CV failed to
		// post starting message.
		rs.LogInfo(ctx, logEntryLabelPostStartMessage, failRunReason)
	case eventpb.LongOpCompleted_EXPIRED:
		failRunReason += fmt.Sprintf(" within the %s deadline", maxPostStartMessageDuration)
		rs.LogInfo(ctx, logEntryLabelPostStartMessage, failRunReason)
	case eventpb.LongOpCompleted_SUCCEEDED:
		// TODO(tandrii): simplify once all such events have timestamp.
		if t := result.GetPostStartMessage().GetTime(); t != nil {
			rs.LogInfoAt(t.AsTime(), logEntryLabelPostStartMessage, "posted start message on each CL")
		} else {
			rs.LogInfo(ctx, logEntryLabelPostStartMessage, "posted start message on each CL")
		}
		return &Result{State: rs}, nil
	case eventpb.LongOpCompleted_CANCELLED:
		rs.LogInfo(ctx, logEntryLabelPostStartMessage, "cancelled posting start message on CL(s)")
		return &Result{State: rs}, nil
	default:
		panic(fmt.Errorf("unexpected LongOpCompleted status: %s", result.GetStatus()))
	}

	msgPrefix, attentionReason := usertext.OnRunFailed(rs.Mode)
	meta := reviewInputMeta{
		notify:         gerrit.Whoms{gerrit.Owner, gerrit.CQVoters},
		addToAttention: gerrit.Whoms{gerrit.Owner, gerrit.CQVoters},
		reason:         attentionReason,
		message:        msgPrefix + "\n\n" + failRunReason,
	}
	if err := impl.cancelTriggers(ctx, rs, meta); err != nil {
		return nil, err
	}
	return &Result{
		State:        rs,
		SideEffectFn: impl.endRun(ctx, rs, run.Status_FAILED),
	}, nil
}

func recordPickupLatency(ctx context.Context, rs *state.RunState, cg *prjcfg.ConfigGroup) {
	delay := rs.StartTime.Sub(rs.CreateTime)
	metricPickupLatencyS.Add(ctx, delay.Seconds(), rs.ID.LUCIProject())

	if d := cg.Content.GetCombineCls().GetStabilizationDelay(); d != nil {
		delay -= d.AsDuration()
	}
	metricPickupLatencyAdjustedS.Add(ctx, delay.Seconds(), rs.ID.LUCIProject())

	if delay >= 1*time.Minute {
		logging.Warningf(ctx, "Too large adjusted pickup delay: %s", delay)
	}
}
<|endoftext|>


<commit_before>package main

import (
	"encoding/json"
	"flag"
	"fmt"
	"io/ioutil"
	"log"
	"net/http"
	"os/exec"
	"strings"
)

type KeyValue map[string]interface{}

type KeyValueGroup []KeyValue // a slice of KeyValue's

type EtcdMachine struct {
	name      string
	state     string
	clientURL string
	peerURL   string
}

type EtcdMachineGroup []EtcdMachine // a slice of EtcdMachine's

// Modify EtcdMachine
func (e *EtcdMachine) SetProperties(machineData KeyValue) {
	e.name = machineData["name"].(string)
	e.state = machineData["state"].(string)
	e.clientURL = machineData["clientURL"].(string)
	e.peerURL = machineData["peerURL"].(string)
}

// Retrieve string of EtcdMachine
func (e *EtcdMachine) String() string {
	output := fmt.Sprintf("Name: %s | State: %s | ClientURL: %s | PeerURL: %s",
		e.name,
		e.state,
		e.clientURL,
		e.peerURL,
	)
	return output
}

// Check for errors and panic, if found
func checkForErrors(err error) {
	if err != nil {
		log.Fatal("%s", err)
	}
}

// Get the IP address of the docker host as this is run from within container
func getDockerHostIP() string {
	cmd := fmt.Sprintf("netstat -nr | grep '^0\\.0\\.0\\.0' | awk '{print $2}'")
	out, err := exec.Command("sh", "-c", cmd).Output()
	checkForErrors(err)

	ip := string(out)
	ip = strings.Replace(ip, "\n", "", -1)
	return ip
}

// Compose the etcd API host:port location
func getEtcdAPI(host string, port string) string {
	return fmt.Sprintf("http://%s:%s", host, port)
}

func httpGetRequest(url string) []byte {
	resp, err := http.Get(url)
	if err != nil {
		log.Fatal("%s", err)
	}

	defer resp.Body.Close()
	body, err := ioutil.ReadAll(resp.Body)

	if err != nil {
		log.Fatal("%s", err)
	}

	return body
}

func waitForMachines(machineCount int) {
	log.Printf("%d", machineCount)
}

func setupFlags() int {
	machineCount :=
		flag.Int("machine_count", 0, "Number of machines to watch for")
	flag.Parse()

	return *machineCount
}

// Access the CoreOS / docker etcd API to extract machine information
func main() {
	if machineCount := setupFlags(); machineCount > 0 {
		waitForMachines(machineCount)
	}

	// Local etcd API host & port
	port := "7001"
	etcdAPI := getEtcdAPI(getDockerHostIP(), port)

	// Request path listing etcd machines in cluster
	etcdAPIPath := "v2/admin/machines"
	url := fmt.Sprintf("%s/%s", etcdAPI, etcdAPIPath)

	jsonResponse := httpGetRequest(url)

	// Decode the JSON returned
	var machineDataGroup KeyValueGroup
	err := json.Unmarshal(jsonResponse, &machineDataGroup)
	checkForErrors(err)

	// Use machine data to create local objects of the etcd machines
	machines := EtcdMachineGroup{}
	for _, machineData := range machineDataGroup {
		machine := EtcdMachine{}
		machine.SetProperties(machineData)
		machines = append(machines, machine)
	}

	for _, machine := range machines {
		log.Printf("%s\n", machine.String())
	}
}
<commit_msg>Add ability to get state of docker host<commit_after>package main

import (
	"encoding/json"
	"flag"
	"fmt"
	"io/ioutil"
	"log"
	"net/http"
	"os"
	"os/exec"
	"strings"
)

type KeyValue map[string]interface{}

type KeyValueGroup []KeyValue // a slice of KeyValue's

type EtcdMachine struct {
	name      string
	state     string
	clientURL string
	peerURL   string
}

type EtcdMachineGroup []EtcdMachine // a slice of EtcdMachine's

// Modify EtcdMachine
func (e *EtcdMachine) SetProperties(machineData KeyValue) {
	e.name = machineData["name"].(string)
	e.state = machineData["state"].(string)
	e.clientURL = machineData["clientURL"].(string)
	e.peerURL = machineData["peerURL"].(string)
}

// Retrieve string of EtcdMachine
func (e *EtcdMachine) String() string {
	output := fmt.Sprintf("Name: %s | State: %s | ClientURL: %s | PeerURL: %s",
		e.name,
		e.state,
		e.clientURL,
		e.peerURL,
	)
	return output
}

// Check for errors and panic, if found
func checkForErrors(err error) {
	if err != nil {
		log.Fatal("%s", err)
	}
}

// Get the IP address of the docker host as this is run from within container
func getDockerHostIP() string {
	cmd := fmt.Sprintf("netstat -nr | grep '^0\\.0\\.0\\.0' | awk '{print $2}'")
	out, err := exec.Command("sh", "-c", cmd).Output()
	checkForErrors(err)

	ip := string(out)
	ip = strings.Replace(ip, "\n", "", -1)
	return ip
}

// Compose the etcd API host:port location
func getEtcdAPI(host string, port string) string {
	return fmt.Sprintf("http://%s:%s", host, port)
}

func httpGetRequest(url string) []byte {
	resp, err := http.Get(url)
	if err != nil {
		log.Fatal("%s", err)
	}

	defer resp.Body.Close()
	body, err := ioutil.ReadAll(resp.Body)

	if err != nil {
		log.Fatal("%s", err)
	}

	return body
}

func waitForMachines(machineCount int) {
	log.Printf("%d", machineCount)
}

func getState(machines *EtcdMachineGroup) string {
	hostname := os.Getenv("DOCKERHOST_HOSTNAME")
	hostname = strings.Split(hostname, ".")[0]

	for _, machine := range *machines {
		log.Printf("%s -- %s", machine.name, hostname)
		if machine.name == hostname {
			return machine.state
		}
	}

	return ""
}

func setupFlags() int {
	machineCount :=
		flag.Int("machine_count", 0, "Number of machines to watch for")
	flag.Parse()

	return *machineCount
}

// Access the CoreOS / docker etcd API to extract machine information
func main() {
	if machineCount := setupFlags(); machineCount > 0 {
		waitForMachines(machineCount)
	}

	// Local etcd API host & port
	port := "7001"
	etcdAPI := getEtcdAPI(getDockerHostIP(), port)

	// Request path listing etcd machines in cluster
	etcdAPIPath := "v2/admin/machines"
	url := fmt.Sprintf("%s/%s", etcdAPI, etcdAPIPath)

	jsonResponse := httpGetRequest(url)

	// Decode the JSON returned
	var machineDataGroup KeyValueGroup
	err := json.Unmarshal(jsonResponse, &machineDataGroup)
	checkForErrors(err)

	// Use machine data to create local objects of the etcd machines
	machines := EtcdMachineGroup{}
	for _, machineData := range machineDataGroup {
		machine := EtcdMachine{}
		machine.SetProperties(machineData)
		machines = append(machines, machine)
	}

	for _, machine := range machines {
		log.Printf("%s\n", machine.String())
	}

	state := getState(&machines)
	log.Printf("%s", state)
}
<|endoftext|>


<commit_before>// Package udp implements a BitTorrent tracker via the UDP protocol as
// described in BEP 15.
package udp

import (
	"bytes"
	"context"
	"encoding/binary"
	"math/rand"
	"net"
	"sync"
	"time"

	log "github.com/Sirupsen/logrus"
	"github.com/prometheus/client_golang/prometheus"

	"github.com/chihaya/chihaya/bittorrent"
	"github.com/chihaya/chihaya/frontend"
	"github.com/chihaya/chihaya/frontend/udp/bytepool"
)

var allowedGeneratedPrivateKeyRunes = []rune("abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ1234567890")

func init() {
	prometheus.MustRegister(promResponseDurationMilliseconds)
}

// ErrInvalidIP indicates an invalid IP.
var ErrInvalidIP = bittorrent.ClientError("invalid IP")

var promResponseDurationMilliseconds = prometheus.NewHistogramVec(
	prometheus.HistogramOpts{
		Name:    "chihaya_udp_response_duration_milliseconds",
		Help:    "The duration of time it takes to receive and write a response to an API request",
		Buckets: prometheus.ExponentialBuckets(9.375, 2, 10),
	},
	[]string{"action", "address_family", "error"},
)

// recordResponseDuration records the duration of time to respond to a UDP
// Request in milliseconds .
func recordResponseDuration(action string, af *bittorrent.AddressFamily, err error, duration time.Duration) {
	var errString string
	if err != nil {
		if _, ok := err.(bittorrent.ClientError); ok {
			errString = err.Error()
		} else {
			errString = "internal error"
		}
	}

	var afString string
	if af == nil {
		afString = "Unknown"
	} else if *af == bittorrent.IPv4 {
		afString = "IPv4"
	} else if *af == bittorrent.IPv6 {
		afString = "IPv6"
	}

	promResponseDurationMilliseconds.
		WithLabelValues(action, afString, errString).
		Observe(float64(duration.Nanoseconds()) / float64(time.Millisecond))
}

// Config represents all of the configurable options for a UDP BitTorrent
// Tracker.
type Config struct {
	Addr            string        `yaml:"addr"`
	PrivateKey      string        `yaml:"private_key"`
	MaxClockSkew    time.Duration `yaml:"max_clock_skew"`
	AllowIPSpoofing bool          `yaml:"allow_ip_spoofing"`
}

// Frontend holds the state of a UDP BitTorrent Frontend.
type Frontend struct {
	socket  *net.UDPConn
	closing chan struct{}
	wg      sync.WaitGroup

	logic frontend.TrackerLogic
	Config
}

// NewFrontend allocates a new instance of a Frontend.
func NewFrontend(logic frontend.TrackerLogic, cfg Config) *Frontend {
	// Generate a private key if one isn't provided by the user.
	if cfg.PrivateKey == "" {
		rand.Seed(time.Now().UnixNano())
		pkeyRunes := make([]rune, 64)
		for i := range pkeyRunes {
			pkeyRunes[i] = allowedGeneratedPrivateKeyRunes[rand.Intn(len(allowedGeneratedPrivateKeyRunes))]
		}
		cfg.PrivateKey = string(pkeyRunes)

		log.Warn("UDP private key was not provided, using generated key: ", cfg.PrivateKey)
	}

	return &Frontend{
		closing: make(chan struct{}),
		logic:   logic,
		Config:  cfg,
	}
}

// Stop provides a thread-safe way to shutdown a currently running Frontend.
func (t *Frontend) Stop() {
	close(t.closing)
	t.socket.SetReadDeadline(time.Now())
	t.wg.Wait()
}

// ListenAndServe listens on the UDP network address t.Addr and blocks serving
// BitTorrent requests until t.Stop() is called or an error is returned.
func (t *Frontend) ListenAndServe() error {
	udpAddr, err := net.ResolveUDPAddr("udp", t.Addr)
	if err != nil {
		return err
	}

	t.socket, err = net.ListenUDP("udp", udpAddr)
	if err != nil {
		return err
	}
	defer t.socket.Close()

	pool := bytepool.New(2048)

	for {
		// Check to see if we need to shutdown.
		select {
		case <-t.closing:
			return nil
		default:
		}

		// Read a UDP packet into a reusable buffer.
		buffer := pool.Get()
		t.socket.SetReadDeadline(time.Now().Add(time.Second))
		n, addr, err := t.socket.ReadFromUDP(buffer)
		if err != nil {
			pool.Put(buffer)
			if netErr, ok := err.(net.Error); ok && netErr.Temporary() {
				// A temporary failure is not fatal; just pretend it never happened.
				continue
			}
			return err
		}

		// We got nothin'
		if n == 0 {
			pool.Put(buffer)
			continue
		}

		t.wg.Add(1)
		go func() {
			defer t.wg.Done()
			defer pool.Put(buffer)

			if ip := addr.IP.To4(); ip != nil {
				addr.IP = ip
			}

			// Handle the request.
			start := time.Now()
			action, af, err := t.handleRequest(
				// Make sure the IP is copied, not referenced.
				Request{buffer[:n], append([]byte{}, addr.IP...)},
				ResponseWriter{t.socket, addr},
			)
			recordResponseDuration(action, af, err, time.Since(start))
		}()
	}
}

// Request represents a UDP payload received by a Tracker.
type Request struct {
	Packet []byte
	IP     net.IP
}

// ResponseWriter implements the ability to respond to a Request via the
// io.Writer interface.
type ResponseWriter struct {
	socket *net.UDPConn
	addr   *net.UDPAddr
}

// Write implements the io.Writer interface for a ResponseWriter.
func (w ResponseWriter) Write(b []byte) (int, error) {
	w.socket.WriteToUDP(b, w.addr)
	return len(b), nil
}

// handleRequest parses and responds to a UDP Request.
func (t *Frontend) handleRequest(r Request, w ResponseWriter) (actionName string, af *bittorrent.AddressFamily, err error) {
	if len(r.Packet) < 16 {
		// Malformed, no client packets are less than 16 bytes.
		// We explicitly return nothing in case this is a DoS attempt.
		err = errMalformedPacket
		return
	}

	// Parse the headers of the UDP packet.
	connID := r.Packet[0:8]
	actionID := binary.BigEndian.Uint32(r.Packet[8:12])
	txID := r.Packet[12:16]

	// If this isn't requesting a new connection ID and the connection ID is
	// invalid, then fail.
	if actionID != connectActionID && !ValidConnectionID(connID, r.IP, time.Now(), t.MaxClockSkew, t.PrivateKey) {
		err = errBadConnectionID
		WriteError(w, txID, err)
		return
	}

	// Handle the requested action.
	switch actionID {
	case connectActionID:
		actionName = "connect"

		if !bytes.Equal(connID, initialConnectionID) {
			err = errMalformedPacket
			return
		}

		WriteConnectionID(w, txID, NewConnectionID(r.IP, time.Now(), t.PrivateKey))

	case announceActionID, announceV6ActionID:
		actionName = "announce"

		var req *bittorrent.AnnounceRequest
		req, err = ParseAnnounce(r, t.AllowIPSpoofing, actionID == announceV6ActionID)
		if err != nil {
			WriteError(w, txID, err)
			return
		}
		af = new(bittorrent.AddressFamily)
		*af = req.IP.AddressFamily

		var resp *bittorrent.AnnounceResponse
		resp, err = t.logic.HandleAnnounce(context.Background(), req)
		if err != nil {
			WriteError(w, txID, err)
			return
		}

		WriteAnnounce(w, txID, resp, actionID == announceV6ActionID)

		go t.logic.AfterAnnounce(context.Background(), req, resp)

	case scrapeActionID:
		actionName = "scrape"

		var req *bittorrent.ScrapeRequest
		req, err = ParseScrape(r)
		if err != nil {
			WriteError(w, txID, err)
			return
		}

		if r.IP.To4() != nil {
			req.AddressFamily = bittorrent.IPv4
		} else if len(r.IP) == net.IPv6len { // implies r.IP.To4() == nil
			req.AddressFamily = bittorrent.IPv6
		} else {
			log.Errorln("udp: invalid IP: neither v4 nor v6, IP was", r.IP)
			WriteError(w, txID, ErrInvalidIP)
			return
		}
		af = new(bittorrent.AddressFamily)
		*af = req.AddressFamily

		var resp *bittorrent.ScrapeResponse
		resp, err = t.logic.HandleScrape(context.Background(), req)
		if err != nil {
			WriteError(w, txID, err)
			return
		}

		WriteScrape(w, txID, resp)

		go t.logic.AfterScrape(context.Background(), req, resp)

	default:
		err = errUnknownAction
		WriteError(w, txID, err)
	}

	return
}
<commit_msg>udp: remove read deadline<commit_after>// Package udp implements a BitTorrent tracker via the UDP protocol as
// described in BEP 15.
package udp

import (
	"bytes"
	"context"
	"encoding/binary"
	"math/rand"
	"net"
	"sync"
	"time"

	log "github.com/Sirupsen/logrus"
	"github.com/prometheus/client_golang/prometheus"

	"github.com/chihaya/chihaya/bittorrent"
	"github.com/chihaya/chihaya/frontend"
	"github.com/chihaya/chihaya/frontend/udp/bytepool"
)

var allowedGeneratedPrivateKeyRunes = []rune("abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ1234567890")

func init() {
	prometheus.MustRegister(promResponseDurationMilliseconds)
}

// ErrInvalidIP indicates an invalid IP.
var ErrInvalidIP = bittorrent.ClientError("invalid IP")

var promResponseDurationMilliseconds = prometheus.NewHistogramVec(
	prometheus.HistogramOpts{
		Name:    "chihaya_udp_response_duration_milliseconds",
		Help:    "The duration of time it takes to receive and write a response to an API request",
		Buckets: prometheus.ExponentialBuckets(9.375, 2, 10),
	},
	[]string{"action", "address_family", "error"},
)

// recordResponseDuration records the duration of time to respond to a UDP
// Request in milliseconds .
func recordResponseDuration(action string, af *bittorrent.AddressFamily, err error, duration time.Duration) {
	var errString string
	if err != nil {
		if _, ok := err.(bittorrent.ClientError); ok {
			errString = err.Error()
		} else {
			errString = "internal error"
		}
	}

	var afString string
	if af == nil {
		afString = "Unknown"
	} else if *af == bittorrent.IPv4 {
		afString = "IPv4"
	} else if *af == bittorrent.IPv6 {
		afString = "IPv6"
	}

	promResponseDurationMilliseconds.
		WithLabelValues(action, afString, errString).
		Observe(float64(duration.Nanoseconds()) / float64(time.Millisecond))
}

// Config represents all of the configurable options for a UDP BitTorrent
// Tracker.
type Config struct {
	Addr            string        `yaml:"addr"`
	PrivateKey      string        `yaml:"private_key"`
	MaxClockSkew    time.Duration `yaml:"max_clock_skew"`
	AllowIPSpoofing bool          `yaml:"allow_ip_spoofing"`
}

// Frontend holds the state of a UDP BitTorrent Frontend.
type Frontend struct {
	socket  *net.UDPConn
	closing chan struct{}
	wg      sync.WaitGroup

	logic frontend.TrackerLogic
	Config
}

// NewFrontend allocates a new instance of a Frontend.
func NewFrontend(logic frontend.TrackerLogic, cfg Config) *Frontend {
	// Generate a private key if one isn't provided by the user.
	if cfg.PrivateKey == "" {
		rand.Seed(time.Now().UnixNano())
		pkeyRunes := make([]rune, 64)
		for i := range pkeyRunes {
			pkeyRunes[i] = allowedGeneratedPrivateKeyRunes[rand.Intn(len(allowedGeneratedPrivateKeyRunes))]
		}
		cfg.PrivateKey = string(pkeyRunes)

		log.Warn("UDP private key was not provided, using generated key: ", cfg.PrivateKey)
	}

	return &Frontend{
		closing: make(chan struct{}),
		logic:   logic,
		Config:  cfg,
	}
}

// Stop provides a thread-safe way to shutdown a currently running Frontend.
func (t *Frontend) Stop() {
	close(t.closing)
	t.socket.SetReadDeadline(time.Now())
	t.wg.Wait()
}

// ListenAndServe listens on the UDP network address t.Addr and blocks serving
// BitTorrent requests until t.Stop() is called or an error is returned.
func (t *Frontend) ListenAndServe() error {
	udpAddr, err := net.ResolveUDPAddr("udp", t.Addr)
	if err != nil {
		return err
	}

	t.socket, err = net.ListenUDP("udp", udpAddr)
	if err != nil {
		return err
	}
	defer t.socket.Close()

	pool := bytepool.New(2048)

	for {
		// Check to see if we need to shutdown.
		select {
		case <-t.closing:
			return nil
		default:
		}

		// Read a UDP packet into a reusable buffer.
		buffer := pool.Get()
		n, addr, err := t.socket.ReadFromUDP(buffer)
		if err != nil {
			pool.Put(buffer)
			if netErr, ok := err.(net.Error); ok && netErr.Temporary() {
				// A temporary failure is not fatal; just pretend it never happened.
				continue
			}
			return err
		}

		// We got nothin'
		if n == 0 {
			pool.Put(buffer)
			continue
		}

		t.wg.Add(1)
		go func() {
			defer t.wg.Done()
			defer pool.Put(buffer)

			if ip := addr.IP.To4(); ip != nil {
				addr.IP = ip
			}

			// Handle the request.
			start := time.Now()
			action, af, err := t.handleRequest(
				// Make sure the IP is copied, not referenced.
				Request{buffer[:n], append([]byte{}, addr.IP...)},
				ResponseWriter{t.socket, addr},
			)
			recordResponseDuration(action, af, err, time.Since(start))
		}()
	}
}

// Request represents a UDP payload received by a Tracker.
type Request struct {
	Packet []byte
	IP     net.IP
}

// ResponseWriter implements the ability to respond to a Request via the
// io.Writer interface.
type ResponseWriter struct {
	socket *net.UDPConn
	addr   *net.UDPAddr
}

// Write implements the io.Writer interface for a ResponseWriter.
func (w ResponseWriter) Write(b []byte) (int, error) {
	w.socket.WriteToUDP(b, w.addr)
	return len(b), nil
}

// handleRequest parses and responds to a UDP Request.
func (t *Frontend) handleRequest(r Request, w ResponseWriter) (actionName string, af *bittorrent.AddressFamily, err error) {
	if len(r.Packet) < 16 {
		// Malformed, no client packets are less than 16 bytes.
		// We explicitly return nothing in case this is a DoS attempt.
		err = errMalformedPacket
		return
	}

	// Parse the headers of the UDP packet.
	connID := r.Packet[0:8]
	actionID := binary.BigEndian.Uint32(r.Packet[8:12])
	txID := r.Packet[12:16]

	// If this isn't requesting a new connection ID and the connection ID is
	// invalid, then fail.
	if actionID != connectActionID && !ValidConnectionID(connID, r.IP, time.Now(), t.MaxClockSkew, t.PrivateKey) {
		err = errBadConnectionID
		WriteError(w, txID, err)
		return
	}

	// Handle the requested action.
	switch actionID {
	case connectActionID:
		actionName = "connect"

		if !bytes.Equal(connID, initialConnectionID) {
			err = errMalformedPacket
			return
		}

		WriteConnectionID(w, txID, NewConnectionID(r.IP, time.Now(), t.PrivateKey))

	case announceActionID, announceV6ActionID:
		actionName = "announce"

		var req *bittorrent.AnnounceRequest
		req, err = ParseAnnounce(r, t.AllowIPSpoofing, actionID == announceV6ActionID)
		if err != nil {
			WriteError(w, txID, err)
			return
		}
		af = new(bittorrent.AddressFamily)
		*af = req.IP.AddressFamily

		var resp *bittorrent.AnnounceResponse
		resp, err = t.logic.HandleAnnounce(context.Background(), req)
		if err != nil {
			WriteError(w, txID, err)
			return
		}

		WriteAnnounce(w, txID, resp, actionID == announceV6ActionID)

		go t.logic.AfterAnnounce(context.Background(), req, resp)

	case scrapeActionID:
		actionName = "scrape"

		var req *bittorrent.ScrapeRequest
		req, err = ParseScrape(r)
		if err != nil {
			WriteError(w, txID, err)
			return
		}

		if r.IP.To4() != nil {
			req.AddressFamily = bittorrent.IPv4
		} else if len(r.IP) == net.IPv6len { // implies r.IP.To4() == nil
			req.AddressFamily = bittorrent.IPv6
		} else {
			log.Errorln("udp: invalid IP: neither v4 nor v6, IP was", r.IP)
			WriteError(w, txID, ErrInvalidIP)
			return
		}
		af = new(bittorrent.AddressFamily)
		*af = req.AddressFamily

		var resp *bittorrent.ScrapeResponse
		resp, err = t.logic.HandleScrape(context.Background(), req)
		if err != nil {
			WriteError(w, txID, err)
			return
		}

		WriteScrape(w, txID, resp)

		go t.logic.AfterScrape(context.Background(), req, resp)

	default:
		err = errUnknownAction
		WriteError(w, txID, err)
	}

	return
}
<|endoftext|>


<commit_before>// Copyright 2015 Google Inc. All Rights Reserved.
// Author: jacobsa@google.com (Aaron Jacobs)
//
// Tests registered by RegisterFSTests.

package fstesting

import (
	"io/ioutil"
	"math"
	"os"
	"path"
	"time"

	"github.com/jacobsa/gcloud/gcs"
	"github.com/jacobsa/gcloud/gcs/gcsutil"
	"github.com/jacobsa/gcsfuse/fs"
	"github.com/jacobsa/gcsfuse/fuseutil"
	. "github.com/jacobsa/oglematchers"
	. "github.com/jacobsa/ogletest"
	"golang.org/x/net/context"
	"google.golang.org/cloud/storage"
)

////////////////////////////////////////////////////////////////////////
// Common
////////////////////////////////////////////////////////////////////////

type fsTest struct {
	ctx    context.Context
	bucket gcs.Bucket
	mfs    *fuseutil.MountedFileSystem
}

var _ fsTestInterface = &fsTest{}

func (t *fsTest) setUpFsTest(b gcs.Bucket) {
	var err error

	// Record bucket and context information.
	t.bucket = b
	t.ctx = context.Background()

	// Set up a temporary directory for mounting.
	mountPoint, err := ioutil.TempDir("", "fs_test")
	if err != nil {
		panic("ioutil.TempDir: " + err.Error())
	}

	// Mount a file system.
	fileSystem, err := fs.NewFuseFS(b)
	if err != nil {
		panic("NewFuseFS: " + err.Error())
	}

	t.mfs = fuseutil.MountFileSystem(mountPoint, fileSystem)
	if err := t.mfs.WaitForReady(t.ctx); err != nil {
		panic("MountedFileSystem.WaitForReady: " + err.Error())
	}
}

func (t *fsTest) tearDownFsTest() {
	// Unmount the file system.
	if err := t.mfs.Unmount(); err != nil {
		panic("MountedFileSystem.Unmount: " + err.Error())
	}

	if err := t.mfs.Join(t.ctx); err != nil {
		panic("MountedFileSystem.Join: " + err.Error())
	}
}

func (t *fsTest) createObjects(objects []*gcsutil.ObjectInfo) error {
	_, err := gcsutil.CreateObjects(t.ctx, t.bucket, objects)
	return err
}

func (t *fsTest) createEmptyObjects(names []string) error {
	_, err := gcsutil.CreateEmptyObjects(t.ctx, t.bucket, names)
	return err
}

////////////////////////////////////////////////////////////////////////
// Read-only interaction
////////////////////////////////////////////////////////////////////////

type readOnlyTest struct {
	fsTest
}

func (t *readOnlyTest) EmptyRoot() {
	// ReadDir
	entries, err := ioutil.ReadDir(t.mfs.Dir())
	AssertEq(nil, err)

	ExpectThat(entries, ElementsAre())
}

func (t *readOnlyTest) ContentsInRoot() {
	// Set up contents.
	AssertEq(
		nil,
		t.createObjects(
			[]*gcsutil.ObjectInfo{
				// File
				&gcsutil.ObjectInfo{
					Attrs: storage.ObjectAttrs{
						Name: "foo",
					},
					Contents: "taco",
				},

				// Directory
				&gcsutil.ObjectInfo{
					Attrs: storage.ObjectAttrs{
						Name: "bar/",
					},
				},

				// File
				&gcsutil.ObjectInfo{
					Attrs: storage.ObjectAttrs{
						Name: "baz",
					},
					Contents: "burrito",
				},

				// File in sub-directory
				&gcsutil.ObjectInfo{
					Attrs: storage.ObjectAttrs{
						Name: "qux/asdf",
					},
					Contents: "",
				},
			}))

	// ReadDir
	entries, err := ioutil.ReadDir(t.mfs.Dir())
	AssertEq(nil, err)

	AssertEq(4, len(entries))
	var e os.FileInfo

	// bar
	e = entries[0]
	ExpectEq("bar", e.Name())
	ExpectEq(0, e.Size())
	ExpectEq(os.ModeDir|os.FileMode(0500), e.Mode())
	ExpectLt(math.Abs(time.Since(e.ModTime()).Seconds()), 30)
	ExpectTrue(e.IsDir())

	// baz
	e = entries[1]
	ExpectEq("baz", e.Name())
	ExpectEq(len("burrito"), e.Size())
	ExpectEq(os.FileMode(0400), e.Mode())
	ExpectLt(math.Abs(time.Since(e.ModTime()).Seconds()), 30)
	ExpectFalse(e.IsDir())

	// foo
	e = entries[2]
	ExpectEq("foo", e.Name())
	ExpectEq(len("taco"), e.Size())
	ExpectEq(os.FileMode(0400), e.Mode())
	ExpectLt(math.Abs(time.Since(e.ModTime()).Seconds()), 30)
	ExpectFalse(e.IsDir())

	// qux
	e = entries[3]
	ExpectEq("qux", e.Name())
	ExpectEq(0, e.Size())
	ExpectEq(os.ModeDir|os.FileMode(0500), e.Mode())
	ExpectLt(math.Abs(time.Since(e.ModTime()).Seconds()), 30)
	ExpectTrue(e.IsDir())
}

func (t *readOnlyTest) EmptySubDirectory() {
	// Set up an empty directory placeholder called 'bar'.
	AssertEq(nil, t.createEmptyObjects([]string{"bar/"}))

	// ReadDir
	entries, err := ioutil.ReadDir(path.Join(t.mfs.Dir(), "bar"))
	AssertEq(nil, err)

	ExpectThat(entries, ElementsAre())
}

func (t *readOnlyTest) ContentsInSubDirectory_PlaceholderPresent() {
	// Set up contents.
	AssertEq(
		nil,
		t.createObjects(
			[]*gcsutil.ObjectInfo{
				// Placeholder
				&gcsutil.ObjectInfo{
					Attrs: storage.ObjectAttrs{
						Name: "dir/",
					},
					Contents: "",
				},

				// File
				&gcsutil.ObjectInfo{
					Attrs: storage.ObjectAttrs{
						Name: "dir/foo",
					},
					Contents: "taco",
				},

				// Directory
				&gcsutil.ObjectInfo{
					Attrs: storage.ObjectAttrs{
						Name: "dir/bar/",
					},
				},

				// File
				&gcsutil.ObjectInfo{
					Attrs: storage.ObjectAttrs{
						Name: "dir/baz",
					},
					Contents: "burrito",
				},

				// File in sub-directory
				&gcsutil.ObjectInfo{
					Attrs: storage.ObjectAttrs{
						Name: "dir/qux/asdf",
					},
					Contents: "",
				},
			}))

	// ReadDir
	entries, err := ioutil.ReadDir(path.Join(t.mfs.Dir(), "dir"))
	AssertEq(nil, err)

	AssertEq(4, len(entries))
	var e os.FileInfo

	// bar
	e = entries[0]
	ExpectEq("bar", e.Name())
	ExpectEq(0, e.Size())
	ExpectEq(os.ModeDir|os.FileMode(0500), e.Mode())
	ExpectLt(math.Abs(time.Since(e.ModTime()).Seconds()), 30)
	ExpectTrue(e.IsDir())

	// baz
	e = entries[1]
	ExpectEq("baz", e.Name())
	ExpectEq(len("burrito"), e.Size())
	ExpectEq(os.FileMode(0400), e.Mode())
	ExpectLt(math.Abs(time.Since(e.ModTime()).Seconds()), 30)
	ExpectFalse(e.IsDir())

	// foo
	e = entries[2]
	ExpectEq("foo", e.Name())
	ExpectEq(len("taco"), e.Size())
	ExpectEq(os.FileMode(0400), e.Mode())
	ExpectLt(math.Abs(time.Since(e.ModTime()).Seconds()), 30)
	ExpectFalse(e.IsDir())

	// qux
	e = entries[3]
	ExpectEq("qux", e.Name())
	ExpectEq(0, e.Size())
	ExpectEq(os.ModeDir|os.FileMode(0500), e.Mode())
	ExpectLt(math.Abs(time.Since(e.ModTime()).Seconds()), 30)
	ExpectTrue(e.IsDir())
}

func (t *readOnlyTest) ContentsInSubDirectory_PlaceholderNotPresent() {
	// Set up contents.
	AssertEq(
		nil,
		t.createObjects(
			[]*gcsutil.ObjectInfo{
				// File
				&gcsutil.ObjectInfo{
					Attrs: storage.ObjectAttrs{
						Name: "dir/foo",
					},
					Contents: "taco",
				},

				// Directory
				&gcsutil.ObjectInfo{
					Attrs: storage.ObjectAttrs{
						Name: "dir/bar/",
					},
				},

				// File
				&gcsutil.ObjectInfo{
					Attrs: storage.ObjectAttrs{
						Name: "dir/baz",
					},
					Contents: "burrito",
				},

				// File in sub-directory
				&gcsutil.ObjectInfo{
					Attrs: storage.ObjectAttrs{
						Name: "dir/qux/asdf",
					},
					Contents: "",
				},
			}))

	// ReadDir
	entries, err := ioutil.ReadDir(path.Join(t.mfs.Dir(), "dir"))
	AssertEq(nil, err)

	AssertEq(4, len(entries))
	var e os.FileInfo

	// bar
	e = entries[0]
	ExpectEq("bar", e.Name())
	ExpectEq(0, e.Size())
	ExpectEq(os.ModeDir|os.FileMode(0500), e.Mode())
	ExpectLt(math.Abs(time.Since(e.ModTime()).Seconds()), 30)
	ExpectTrue(e.IsDir())

	// baz
	e = entries[1]
	ExpectEq("baz", e.Name())
	ExpectEq(len("burrito"), e.Size())
	ExpectEq(os.FileMode(0400), e.Mode())
	ExpectLt(math.Abs(time.Since(e.ModTime()).Seconds()), 30)
	ExpectFalse(e.IsDir())

	// foo
	e = entries[2]
	ExpectEq("foo", e.Name())
	ExpectEq(len("taco"), e.Size())
	ExpectEq(os.FileMode(0400), e.Mode())
	ExpectLt(math.Abs(time.Since(e.ModTime()).Seconds()), 30)
	ExpectFalse(e.IsDir())

	// qux
	e = entries[3]
	ExpectEq("qux", e.Name())
	ExpectEq(0, e.Size())
	ExpectEq(os.ModeDir|os.FileMode(0500), e.Mode())
	ExpectLt(math.Abs(time.Since(e.ModTime()).Seconds()), 30)
	ExpectTrue(e.IsDir())
}

func (t *readOnlyTest) ListDirectoryTwice_NoChange() {
	// Set up initial contents.
	AssertEq(
		nil,
		t.createEmptyObjects([]string{
			"foo",
			"bar",
		}))

	// List once.
	entries, err := ioutil.ReadDir(t.mfs.Dir())
	AssertEq(nil, err)

	AssertEq(2, len(entries))
	ExpectEq("bar", entries[0].Name())
	ExpectEq("foo", entries[1].Name())

	// List again.
	entries, err = ioutil.ReadDir(t.mfs.Dir())
	AssertEq(nil, err)

	AssertEq(2, len(entries))
	ExpectEq("bar", entries[0].Name())
	ExpectEq("foo", entries[1].Name())
}

func (t *readOnlyTest) ListDirectoryTwice_Changed() {
	// Set up initial contents.
	AssertEq(
		nil,
		t.createEmptyObjects([]string{
			"foo",
			"bar",
		}))

	// List once.
	entries, err := ioutil.ReadDir(t.mfs.Dir())
	AssertEq(nil, err)

	AssertEq(2, len(entries))
	ExpectEq("bar", entries[0].Name())
	ExpectEq("foo", entries[1].Name())

	// Add "baz" and remove "bar".
	AssertEq(nil, t.bucket.DeleteObject(t.ctx, "bar"))
	AssertEq(nil, t.createEmptyObjects([]string{"baz"}))

	// List again.
	entries, err = ioutil.ReadDir(t.mfs.Dir())
	AssertEq(nil, err)

	AssertEq(2, len(entries))
	ExpectEq("baz", entries[0].Name())
	ExpectEq("foo", entries[1].Name())
}

// TODO(jacobsa): Inodes
// TODO(jacobsa): Error conditions
func (t *readOnlyTest) DoesFoo() {
	AssertTrue(false, "TODO")
}
<commit_msg>Added more test names.<commit_after>// Copyright 2015 Google Inc. All Rights Reserved.
// Author: jacobsa@google.com (Aaron Jacobs)
//
// Tests registered by RegisterFSTests.

package fstesting

import (
	"io/ioutil"
	"math"
	"os"
	"path"
	"time"

	"github.com/jacobsa/gcloud/gcs"
	"github.com/jacobsa/gcloud/gcs/gcsutil"
	"github.com/jacobsa/gcsfuse/fs"
	"github.com/jacobsa/gcsfuse/fuseutil"
	. "github.com/jacobsa/oglematchers"
	. "github.com/jacobsa/ogletest"
	"golang.org/x/net/context"
	"google.golang.org/cloud/storage"
)

////////////////////////////////////////////////////////////////////////
// Common
////////////////////////////////////////////////////////////////////////

type fsTest struct {
	ctx    context.Context
	bucket gcs.Bucket
	mfs    *fuseutil.MountedFileSystem
}

var _ fsTestInterface = &fsTest{}

func (t *fsTest) setUpFsTest(b gcs.Bucket) {
	var err error

	// Record bucket and context information.
	t.bucket = b
	t.ctx = context.Background()

	// Set up a temporary directory for mounting.
	mountPoint, err := ioutil.TempDir("", "fs_test")
	if err != nil {
		panic("ioutil.TempDir: " + err.Error())
	}

	// Mount a file system.
	fileSystem, err := fs.NewFuseFS(b)
	if err != nil {
		panic("NewFuseFS: " + err.Error())
	}

	t.mfs = fuseutil.MountFileSystem(mountPoint, fileSystem)
	if err := t.mfs.WaitForReady(t.ctx); err != nil {
		panic("MountedFileSystem.WaitForReady: " + err.Error())
	}
}

func (t *fsTest) tearDownFsTest() {
	// Unmount the file system.
	if err := t.mfs.Unmount(); err != nil {
		panic("MountedFileSystem.Unmount: " + err.Error())
	}

	if err := t.mfs.Join(t.ctx); err != nil {
		panic("MountedFileSystem.Join: " + err.Error())
	}
}

func (t *fsTest) createObjects(objects []*gcsutil.ObjectInfo) error {
	_, err := gcsutil.CreateObjects(t.ctx, t.bucket, objects)
	return err
}

func (t *fsTest) createEmptyObjects(names []string) error {
	_, err := gcsutil.CreateEmptyObjects(t.ctx, t.bucket, names)
	return err
}

////////////////////////////////////////////////////////////////////////
// Read-only interaction
////////////////////////////////////////////////////////////////////////

type readOnlyTest struct {
	fsTest
}

func (t *readOnlyTest) EmptyRoot() {
	// ReadDir
	entries, err := ioutil.ReadDir(t.mfs.Dir())
	AssertEq(nil, err)

	ExpectThat(entries, ElementsAre())
}

func (t *readOnlyTest) ContentsInRoot() {
	// Set up contents.
	AssertEq(
		nil,
		t.createObjects(
			[]*gcsutil.ObjectInfo{
				// File
				&gcsutil.ObjectInfo{
					Attrs: storage.ObjectAttrs{
						Name: "foo",
					},
					Contents: "taco",
				},

				// Directory
				&gcsutil.ObjectInfo{
					Attrs: storage.ObjectAttrs{
						Name: "bar/",
					},
				},

				// File
				&gcsutil.ObjectInfo{
					Attrs: storage.ObjectAttrs{
						Name: "baz",
					},
					Contents: "burrito",
				},

				// File in sub-directory
				&gcsutil.ObjectInfo{
					Attrs: storage.ObjectAttrs{
						Name: "qux/asdf",
					},
					Contents: "",
				},
			}))

	// ReadDir
	entries, err := ioutil.ReadDir(t.mfs.Dir())
	AssertEq(nil, err)

	AssertEq(4, len(entries))
	var e os.FileInfo

	// bar
	e = entries[0]
	ExpectEq("bar", e.Name())
	ExpectEq(0, e.Size())
	ExpectEq(os.ModeDir|os.FileMode(0500), e.Mode())
	ExpectLt(math.Abs(time.Since(e.ModTime()).Seconds()), 30)
	ExpectTrue(e.IsDir())

	// baz
	e = entries[1]
	ExpectEq("baz", e.Name())
	ExpectEq(len("burrito"), e.Size())
	ExpectEq(os.FileMode(0400), e.Mode())
	ExpectLt(math.Abs(time.Since(e.ModTime()).Seconds()), 30)
	ExpectFalse(e.IsDir())

	// foo
	e = entries[2]
	ExpectEq("foo", e.Name())
	ExpectEq(len("taco"), e.Size())
	ExpectEq(os.FileMode(0400), e.Mode())
	ExpectLt(math.Abs(time.Since(e.ModTime()).Seconds()), 30)
	ExpectFalse(e.IsDir())

	// qux
	e = entries[3]
	ExpectEq("qux", e.Name())
	ExpectEq(0, e.Size())
	ExpectEq(os.ModeDir|os.FileMode(0500), e.Mode())
	ExpectLt(math.Abs(time.Since(e.ModTime()).Seconds()), 30)
	ExpectTrue(e.IsDir())
}

func (t *readOnlyTest) EmptySubDirectory() {
	// Set up an empty directory placeholder called 'bar'.
	AssertEq(nil, t.createEmptyObjects([]string{"bar/"}))

	// ReadDir
	entries, err := ioutil.ReadDir(path.Join(t.mfs.Dir(), "bar"))
	AssertEq(nil, err)

	ExpectThat(entries, ElementsAre())
}

func (t *readOnlyTest) ContentsInSubDirectory_PlaceholderPresent() {
	// Set up contents.
	AssertEq(
		nil,
		t.createObjects(
			[]*gcsutil.ObjectInfo{
				// Placeholder
				&gcsutil.ObjectInfo{
					Attrs: storage.ObjectAttrs{
						Name: "dir/",
					},
					Contents: "",
				},

				// File
				&gcsutil.ObjectInfo{
					Attrs: storage.ObjectAttrs{
						Name: "dir/foo",
					},
					Contents: "taco",
				},

				// Directory
				&gcsutil.ObjectInfo{
					Attrs: storage.ObjectAttrs{
						Name: "dir/bar/",
					},
				},

				// File
				&gcsutil.ObjectInfo{
					Attrs: storage.ObjectAttrs{
						Name: "dir/baz",
					},
					Contents: "burrito",
				},

				// File in sub-directory
				&gcsutil.ObjectInfo{
					Attrs: storage.ObjectAttrs{
						Name: "dir/qux/asdf",
					},
					Contents: "",
				},
			}))

	// ReadDir
	entries, err := ioutil.ReadDir(path.Join(t.mfs.Dir(), "dir"))
	AssertEq(nil, err)

	AssertEq(4, len(entries))
	var e os.FileInfo

	// bar
	e = entries[0]
	ExpectEq("bar", e.Name())
	ExpectEq(0, e.Size())
	ExpectEq(os.ModeDir|os.FileMode(0500), e.Mode())
	ExpectLt(math.Abs(time.Since(e.ModTime()).Seconds()), 30)
	ExpectTrue(e.IsDir())

	// baz
	e = entries[1]
	ExpectEq("baz", e.Name())
	ExpectEq(len("burrito"), e.Size())
	ExpectEq(os.FileMode(0400), e.Mode())
	ExpectLt(math.Abs(time.Since(e.ModTime()).Seconds()), 30)
	ExpectFalse(e.IsDir())

	// foo
	e = entries[2]
	ExpectEq("foo", e.Name())
	ExpectEq(len("taco"), e.Size())
	ExpectEq(os.FileMode(0400), e.Mode())
	ExpectLt(math.Abs(time.Since(e.ModTime()).Seconds()), 30)
	ExpectFalse(e.IsDir())

	// qux
	e = entries[3]
	ExpectEq("qux", e.Name())
	ExpectEq(0, e.Size())
	ExpectEq(os.ModeDir|os.FileMode(0500), e.Mode())
	ExpectLt(math.Abs(time.Since(e.ModTime()).Seconds()), 30)
	ExpectTrue(e.IsDir())
}

func (t *readOnlyTest) ContentsInSubDirectory_PlaceholderNotPresent() {
	// Set up contents.
	AssertEq(
		nil,
		t.createObjects(
			[]*gcsutil.ObjectInfo{
				// File
				&gcsutil.ObjectInfo{
					Attrs: storage.ObjectAttrs{
						Name: "dir/foo",
					},
					Contents: "taco",
				},

				// Directory
				&gcsutil.ObjectInfo{
					Attrs: storage.ObjectAttrs{
						Name: "dir/bar/",
					},
				},

				// File
				&gcsutil.ObjectInfo{
					Attrs: storage.ObjectAttrs{
						Name: "dir/baz",
					},
					Contents: "burrito",
				},

				// File in sub-directory
				&gcsutil.ObjectInfo{
					Attrs: storage.ObjectAttrs{
						Name: "dir/qux/asdf",
					},
					Contents: "",
				},
			}))

	// ReadDir
	entries, err := ioutil.ReadDir(path.Join(t.mfs.Dir(), "dir"))
	AssertEq(nil, err)

	AssertEq(4, len(entries))
	var e os.FileInfo

	// bar
	e = entries[0]
	ExpectEq("bar", e.Name())
	ExpectEq(0, e.Size())
	ExpectEq(os.ModeDir|os.FileMode(0500), e.Mode())
	ExpectLt(math.Abs(time.Since(e.ModTime()).Seconds()), 30)
	ExpectTrue(e.IsDir())

	// baz
	e = entries[1]
	ExpectEq("baz", e.Name())
	ExpectEq(len("burrito"), e.Size())
	ExpectEq(os.FileMode(0400), e.Mode())
	ExpectLt(math.Abs(time.Since(e.ModTime()).Seconds()), 30)
	ExpectFalse(e.IsDir())

	// foo
	e = entries[2]
	ExpectEq("foo", e.Name())
	ExpectEq(len("taco"), e.Size())
	ExpectEq(os.FileMode(0400), e.Mode())
	ExpectLt(math.Abs(time.Since(e.ModTime()).Seconds()), 30)
	ExpectFalse(e.IsDir())

	// qux
	e = entries[3]
	ExpectEq("qux", e.Name())
	ExpectEq(0, e.Size())
	ExpectEq(os.ModeDir|os.FileMode(0500), e.Mode())
	ExpectLt(math.Abs(time.Since(e.ModTime()).Seconds()), 30)
	ExpectTrue(e.IsDir())
}

func (t *readOnlyTest) ListDirectoryTwice_NoChange() {
	// Set up initial contents.
	AssertEq(
		nil,
		t.createEmptyObjects([]string{
			"foo",
			"bar",
		}))

	// List once.
	entries, err := ioutil.ReadDir(t.mfs.Dir())
	AssertEq(nil, err)

	AssertEq(2, len(entries))
	ExpectEq("bar", entries[0].Name())
	ExpectEq("foo", entries[1].Name())

	// List again.
	entries, err = ioutil.ReadDir(t.mfs.Dir())
	AssertEq(nil, err)

	AssertEq(2, len(entries))
	ExpectEq("bar", entries[0].Name())
	ExpectEq("foo", entries[1].Name())
}

func (t *readOnlyTest) ListDirectoryTwice_Changed() {
	// Set up initial contents.
	AssertEq(
		nil,
		t.createEmptyObjects([]string{
			"foo",
			"bar",
		}))

	// List once.
	entries, err := ioutil.ReadDir(t.mfs.Dir())
	AssertEq(nil, err)

	AssertEq(2, len(entries))
	ExpectEq("bar", entries[0].Name())
	ExpectEq("foo", entries[1].Name())

	// Add "baz" and remove "bar".
	AssertEq(nil, t.bucket.DeleteObject(t.ctx, "bar"))
	AssertEq(nil, t.createEmptyObjects([]string{"baz"}))

	// List again.
	entries, err = ioutil.ReadDir(t.mfs.Dir())
	AssertEq(nil, err)

	AssertEq(2, len(entries))
	ExpectEq("baz", entries[0].Name())
	ExpectEq("foo", entries[1].Name())
}

func (t *readOnlyTest) Inodes() {
	AssertTrue(false, "TODO")
}

func (t *readOnlyTest) OpenNonExistentFile() {
	AssertTrue(false, "TODO")
}

func (t *readOnlyTest) ReadNonExistentDir() {
	AssertTrue(false, "TODO")
}
<|endoftext|>