<commit_before>// Copyright (c) 2016 Hong Xiaodong. All rights reserved.
// Use of this source code is governed by a BSD-style license that can be
// found in the LICENSE file.

package util

import (
  "testing"
  "encoding/binary"
  "fmt"
)

func EncodeKey(k int) []byte {
  var result []byte = make([]byte, 4)
  binary.LittleEndian.PutUint32(result, uint32(k))
  return result
}

func DecodeKey(k *Slice) int {
  if k.size() != 4 {
    panic("DecodeKey() error")
  }
  return int(binary.LittleEndian.Uint32(k.data()))
}

func DecodeValue(v interface{}) int {
  return v.(int)
}

const kCacheSize = 1000

var current_ *CacheTest = ConstructCacheTest()

type CacheTest struct {
  deleted_keys_   []int
  deleted_values_ []int
  cache_   Cache
}

var current_deleted_keys   []int
var current_deleted_values []int

func Deleter(key *Slice, v interface{}) {
  current_deleted_keys   = append(current_deleted_keys, DecodeKey(key))
  current_deleted_values = append(current_deleted_values, DecodeValue(v))
}

func ConstructCacheTest() *CacheTest {
  var cache_test *CacheTest = new(CacheTest)
  cache_test.cache_ = NewLRUCache(kCacheSize)
  return cache_test
}

func (s *CacheTest) Lookup(key int) int {
  var handle CacheHandle = s.cache_.Lookup(NewSlice(EncodeKey(key)))
  var lru_handle *LRUHandle = handle.(*LRUHandle)
  var r int
  if lru_handle == nil {
    r = -1
  } else {
    r = DecodeValue(s.cache_.Value(handle))
  }
  if lru_handle != nil {
    s.cache_.Release(handle)
  }
  return r
}

func (s *CacheTest) Insert(key int, value int, charge uint64) {
  s.cache_.Release(s.cache_.Insert(NewSlice(EncodeKey(key)), value, charge, Deleter))
}

func (s *CacheTest) Erase(key int) {
  s.cache_.Erase(NewSlice(EncodeKey(key)))
}

func TestCache_HitAndMiss(t *testing.T) {
  fmt.Println("Run TestCache_HitAndMiss()")
  current_deleted_keys   = current_deleted_keys[:0]
  current_deleted_values = current_deleted_values[:0]

  ASSERT_EQ(-1, current_.Lookup(100))

  current_.Insert(100, 101, 1)
  ASSERT_EQ(101, current_.Lookup(100))
  ASSERT_EQ(-1, current_.Lookup(200))
  ASSERT_EQ(-1, current_.Lookup(300))

  current_.Insert(200, 201, 1)
  ASSERT_EQ(101, current_.Lookup(100))
  ASSERT_EQ(201, current_.Lookup(200))
  ASSERT_EQ(-1, current_.Lookup(300))

  current_.Insert(100, 102, 1)
  ASSERT_EQ(102, current_.Lookup(100))
  ASSERT_EQ(201, current_.Lookup(200))
  ASSERT_EQ(-1, current_.Lookup(300))

  ASSERT_EQ(1, len(current_deleted_keys))
  ASSERT_EQ(100, current_deleted_keys[0])
  ASSERT_EQ(101, current_deleted_values[0])
  // fmt.Printf("(%v, %T)\n", current_.deleted_values_, current_.deleted_values_)
}

var current_2 *CacheTest = ConstructCacheTest()

func TestCache_Erase(t *testing.T) {
  current_deleted_keys   = current_deleted_keys[:0]
  current_deleted_values = current_deleted_values[:0]

  current_2.Erase(200)
  ASSERT_EQ(0, len(current_2.deleted_keys_))

  current_2.Insert(100, 101, 1)
  current_2.Insert(200, 201, 1)
  current_2.Erase(100)
  // fmt.Printf("(%v, %T)\n", current_deleted_keys, current_deleted_keys)
  ASSERT_EQ(-1,  current_2.Lookup(100))
  ASSERT_EQ(201, current_2.Lookup(200))
  ASSERT_EQ(1,   len(current_deleted_keys))
  ASSERT_EQ(100, current_deleted_keys[0])
  ASSERT_EQ(101, current_deleted_values[0])

  current_2.Erase(100)
  ASSERT_EQ(-1,  current_2.Lookup(100))
  ASSERT_EQ(201, current_2.Lookup(200))
  ASSERT_EQ(1,   len(current_deleted_keys))
}

<commit_msg>Update cache_test.go<commit_after>// Copyright (c) 2016 Hong Xiaodong. All rights reserved.
// Use of this source code is governed by a BSD-style license that can be
// found in the LICENSE file.

package util

import (
  "testing"
  "encoding/binary"
  "fmt"
)

func EncodeKey(k int) []byte {
  var result []byte = make([]byte, 4)
  binary.LittleEndian.PutUint32(result, uint32(k))
  return result
}

func DecodeKey(k *Slice) int {
  if k.size() != 4 {
    panic("DecodeKey() error")
  }
  return int(binary.LittleEndian.Uint32(k.data()))
}

func DecodeValue(v interface{}) int {
  return v.(int)
}

const kCacheSize = 1000

type CacheTest struct {
  deleted_keys_   []int
  deleted_values_ []int
  cache_   Cache
}

var current_deleted_keys   []int
var current_deleted_values []int

func Deleter(key *Slice, v interface{}) {
  current_deleted_keys   = append(current_deleted_keys, DecodeKey(key))
  current_deleted_values = append(current_deleted_values, DecodeValue(v))
}

func ConstructCacheTest() *CacheTest {
  var cache_test *CacheTest = new(CacheTest)
  cache_test.cache_ = NewLRUCache(kCacheSize)
  return cache_test
}

func (s *CacheTest) Lookup(key int) int {
  var handle CacheHandle = s.cache_.Lookup(NewSlice(EncodeKey(key)))
  var lru_handle *LRUHandle = handle.(*LRUHandle)
  var r int
  if lru_handle == nil {
    r = -1
  } else {
    r = DecodeValue(s.cache_.Value(handle))
  }
  if lru_handle != nil {
    s.cache_.Release(handle)
  }
  return r
}

func (s *CacheTest) Insert(key int, value int, charge uint64) {
  s.cache_.Release(s.cache_.Insert(NewSlice(EncodeKey(key)), value, charge, Deleter))
}

func (s *CacheTest) Erase(key int) {
  s.cache_.Erase(NewSlice(EncodeKey(key)))
}

func TestCache_HitAndMiss(t *testing.T) {
  fmt.Println("Run TestCache_HitAndMiss()")
  var current_ *CacheTest = ConstructCacheTest()
  current_deleted_keys   = current_deleted_keys[:0]
  current_deleted_values = current_deleted_values[:0]

  ASSERT_EQ(-1, current_.Lookup(100))

  current_.Insert(100, 101, 1)
  ASSERT_EQ(101, current_.Lookup(100))
  ASSERT_EQ(-1, current_.Lookup(200))
  ASSERT_EQ(-1, current_.Lookup(300))

  current_.Insert(200, 201, 1)
  ASSERT_EQ(101, current_.Lookup(100))
  ASSERT_EQ(201, current_.Lookup(200))
  ASSERT_EQ(-1, current_.Lookup(300))

  current_.Insert(100, 102, 1)
  ASSERT_EQ(102, current_.Lookup(100))
  ASSERT_EQ(201, current_.Lookup(200))
  ASSERT_EQ(-1, current_.Lookup(300))

  ASSERT_EQ(1, len(current_deleted_keys))
  ASSERT_EQ(100, current_deleted_keys[0])
  ASSERT_EQ(101, current_deleted_values[0])
  // fmt.Printf("(%v, %T)\n", current_.deleted_values_, current_.deleted_values_)
}

func TestCache_Erase(t *testing.T) {
  var current_2 *CacheTest = ConstructCacheTest()
  current_deleted_keys   = current_deleted_keys[:0]
  current_deleted_values = current_deleted_values[:0]

  current_2.Erase(200)
  ASSERT_EQ(0, len(current_2.deleted_keys_))

  current_2.Insert(100, 101, 1)
  current_2.Insert(200, 201, 1)
  current_2.Erase(100)
  // fmt.Printf("(%v, %T)\n", current_deleted_keys, current_deleted_keys)
  ASSERT_EQ(-1,  current_2.Lookup(100))
  ASSERT_EQ(201, current_2.Lookup(200))
  ASSERT_EQ(1,   len(current_deleted_keys))
  ASSERT_EQ(100, current_deleted_keys[0])
  ASSERT_EQ(101, current_deleted_values[0])

  current_2.Erase(100)
  ASSERT_EQ(-1,  current_2.Lookup(100))
  ASSERT_EQ(201, current_2.Lookup(200))
  ASSERT_EQ(1,   len(current_deleted_keys))
}

func TestCache_EntriesArePinned(t *testing.T) {
  var current_3 *CacheTest = ConstructCacheTest()
  current_deleted_keys   = current_deleted_keys[:0]
  current_deleted_values = current_deleted_values[:0]

  current_3.Insert(100, 101, 1)
  var h1 CacheHandle = current_3.cache_.Lookup(NewSlice(EncodeKey(100)))
  ASSERT_EQ(101, DecodeValue(current_3.cache_.Value(h1)))

  current_3.Insert(100, 102, 1)
  var h2 CacheHandle = current_3.cache_.Lookup(NewSlice(EncodeKey(100)))
  ASSERT_EQ(102, DecodeValue(current_3.cache_.Value(h2)))
  ASSERT_EQ(0, len(current_deleted_keys))

  current_3.cache_.Release(h1)
  ASSERT_EQ(1, len(current_deleted_keys))
  ASSERT_EQ(100, current_deleted_keys[0])
  ASSERT_EQ(101, current_deleted_values[0])

  current_3.Erase(100)
  ASSERT_EQ(-1, current_3.Lookup(100))
  ASSERT_EQ(1,  len(current_deleted_keys))

  current_3.cache_.Release(h2)
  ASSERT_EQ(2, len(current_deleted_keys))
  ASSERT_EQ(100, current_deleted_keys[1])
  ASSERT_EQ(102, current_deleted_values[1])
}

func TestCache_EvictionPolicy(t *testing.T) {
  var current_4 *CacheTest = ConstructCacheTest()
  current_deleted_keys = current_deleted_keys[:0]
  current_deleted_values = current_deleted_values[:0]

  current_4.Insert(100, 101, 1)
  current_4.Insert(200, 201, 1)
  current_4.Insert(300, 301, 1)
  var h CacheHandle = current_4.cache_.Lookup(NewSlice(EncodeKey(300)))

  // Frequently used entry must be kept around,
  // as must things that are still in use.
  for i := 0; i < kCacheSize + 100; i++ {
    current_4.Insert(1000+i, 2000+i, 1)
    ASSERT_EQ(2000+i, current_4.Lookup(1000+i))
    ASSERT_EQ(101, current_4.Lookup(100))
  }
  ASSERT_EQ(101, current_4.Lookup(100))
  ASSERT_EQ(-1,  current_4.Lookup(200))
  ASSERT_EQ(301, current_4.Lookup(300))
  current_4.cache_.Release(h)
}

<|endoftext|>


<commit_before>package dex

import (
	"fmt"

	"github.com/ghodss/yaml"

	"github.com/argoproj/argo-cd/common"
	"github.com/argoproj/argo-cd/util/settings"
)

func GenerateDexConfigYAML(settings *settings.ArgoCDSettings) ([]byte, error) {
	if !settings.IsDexConfigured() {
		return nil, nil
	}
	redirectURL, err := settings.RedirectURL()
	if err != nil {
		return nil, fmt.Errorf("failed to infer redirect url from config: %v", err)
	}
	var dexCfg map[string]interface{}
	err = yaml.Unmarshal([]byte(settings.DexConfig), &dexCfg)
	if err != nil {
		return nil, fmt.Errorf("failed to unmarshal dex.config from configmap: %v", err)
	}
	dexCfg["issuer"] = settings.IssuerURL()
	dexCfg["storage"] = map[string]interface{}{
		"type": "memory",
	}
	dexCfg["web"] = map[string]interface{}{
		"http": "0.0.0.0:5556",
	}
	dexCfg["grpc"] = map[string]interface{}{
		"addr": "0.0.0.0:5557",
	}
	dexCfg["telemetry"] = map[string]interface{}{
		"http": "0.0.0.0:5558",
	}
	dexCfg["oauth2"] = map[string]interface{}{
		"skipApprovalScreen": true,
	}

	argoCDStaticClient := map[string]interface{}{
		"id":     common.ArgoCDClientAppID,
		"name":   common.ArgoCDClientAppName,
		"secret": settings.DexOAuth2ClientSecret(),
		"redirectURIs": []string{
			redirectURL,
		},
	}
	argoCDCLIStaticClient := map[string]interface{}{
		"id":     common.ArgoCDCLIClientAppID,
		"name":   common.ArgoCDCLIClientAppName,
		"public": true,
		"redirectURIs": []string{
			"http://localhost",
		},
	}

	staticClients, ok := dexCfg["staticClients"].([]interface{})
	if ok {
		dexCfg["staticClients"] = append([]interface{}{argoCDStaticClient, argoCDCLIStaticClient}, staticClients...)
	} else {
		dexCfg["staticClients"] = []interface{}{argoCDStaticClient, argoCDCLIStaticClient}
	}

	dexRedirectURL, err := settings.DexRedirectURL()
	if err != nil {
		return nil, err
	}
	connectors, ok := dexCfg["connectors"].([]interface{})
	if !ok {
		return nil, fmt.Errorf("malformed Dex configuration found")
	}
	for i, connectorIf := range connectors {
		connector, ok := connectorIf.(map[string]interface{})
		if !ok {
			return nil, fmt.Errorf("malformed Dex configuration found")
		}
		connectorType := connector["type"].(string)
		if !needsRedirectURI(connectorType) {
			continue
		}
		connectorCfg, ok := connector["config"].(map[string]interface{})
		if !ok {
			return nil, fmt.Errorf("malformed Dex configuration found")
		}
		connectorCfg["redirectURI"] = dexRedirectURL
		connector["config"] = connectorCfg
		connectors[i] = connector
	}
	dexCfg["connectors"] = connectors
	dexCfg = replaceMapSecrets(dexCfg, settings.Secrets)
	return yaml.Marshal(dexCfg)
}

// replaceMapSecrets takes a json object and recursively looks for any secret key references in the
// object and replaces the value with the secret value
func replaceMapSecrets(obj map[string]interface{}, secretValues map[string]string) map[string]interface{} {
	newObj := make(map[string]interface{})
	for k, v := range obj {
		switch val := v.(type) {
		case map[string]interface{}:
			newObj[k] = replaceMapSecrets(val, secretValues)
		case []interface{}:
			newObj[k] = replaceListSecrets(val, secretValues)
		case string:
			newObj[k] = settings.ReplaceStringSecret(val, secretValues)
		default:
			newObj[k] = val
		}
	}
	return newObj
}

func replaceListSecrets(obj []interface{}, secretValues map[string]string) []interface{} {
	newObj := make([]interface{}, len(obj))
	for i, v := range obj {
		switch val := v.(type) {
		case map[string]interface{}:
			newObj[i] = replaceMapSecrets(val, secretValues)
		case []interface{}:
			newObj[i] = replaceListSecrets(val, secretValues)
		case string:
			newObj[i] = settings.ReplaceStringSecret(val, secretValues)
		default:
			newObj[i] = val
		}
	}
	return newObj
}

// needsRedirectURI returns whether or not the given connector type needs a redirectURI
// Update this list as necessary, as new connectors are added
// https://github.com/dexidp/dex/tree/master/Documentation/connectors
func needsRedirectURI(connectorType string) bool {
	switch connectorType {
	case "oidc", "saml", "microsoft", "linkedin", "gitlab", "github", "bitbucket-cloud":
		return true
	}
	return false
}
<commit_msg>fix: Add openshift as a dex connector type which requires a redirectURI (#4222)<commit_after>package dex

import (
	"fmt"

	"github.com/ghodss/yaml"

	"github.com/argoproj/argo-cd/common"
	"github.com/argoproj/argo-cd/util/settings"
)

func GenerateDexConfigYAML(settings *settings.ArgoCDSettings) ([]byte, error) {
	if !settings.IsDexConfigured() {
		return nil, nil
	}
	redirectURL, err := settings.RedirectURL()
	if err != nil {
		return nil, fmt.Errorf("failed to infer redirect url from config: %v", err)
	}
	var dexCfg map[string]interface{}
	err = yaml.Unmarshal([]byte(settings.DexConfig), &dexCfg)
	if err != nil {
		return nil, fmt.Errorf("failed to unmarshal dex.config from configmap: %v", err)
	}
	dexCfg["issuer"] = settings.IssuerURL()
	dexCfg["storage"] = map[string]interface{}{
		"type": "memory",
	}
	dexCfg["web"] = map[string]interface{}{
		"http": "0.0.0.0:5556",
	}
	dexCfg["grpc"] = map[string]interface{}{
		"addr": "0.0.0.0:5557",
	}
	dexCfg["telemetry"] = map[string]interface{}{
		"http": "0.0.0.0:5558",
	}
	dexCfg["oauth2"] = map[string]interface{}{
		"skipApprovalScreen": true,
	}

	argoCDStaticClient := map[string]interface{}{
		"id":     common.ArgoCDClientAppID,
		"name":   common.ArgoCDClientAppName,
		"secret": settings.DexOAuth2ClientSecret(),
		"redirectURIs": []string{
			redirectURL,
		},
	}
	argoCDCLIStaticClient := map[string]interface{}{
		"id":     common.ArgoCDCLIClientAppID,
		"name":   common.ArgoCDCLIClientAppName,
		"public": true,
		"redirectURIs": []string{
			"http://localhost",
		},
	}

	staticClients, ok := dexCfg["staticClients"].([]interface{})
	if ok {
		dexCfg["staticClients"] = append([]interface{}{argoCDStaticClient, argoCDCLIStaticClient}, staticClients...)
	} else {
		dexCfg["staticClients"] = []interface{}{argoCDStaticClient, argoCDCLIStaticClient}
	}

	dexRedirectURL, err := settings.DexRedirectURL()
	if err != nil {
		return nil, err
	}
	connectors, ok := dexCfg["connectors"].([]interface{})
	if !ok {
		return nil, fmt.Errorf("malformed Dex configuration found")
	}
	for i, connectorIf := range connectors {
		connector, ok := connectorIf.(map[string]interface{})
		if !ok {
			return nil, fmt.Errorf("malformed Dex configuration found")
		}
		connectorType := connector["type"].(string)
		if !needsRedirectURI(connectorType) {
			continue
		}
		connectorCfg, ok := connector["config"].(map[string]interface{})
		if !ok {
			return nil, fmt.Errorf("malformed Dex configuration found")
		}
		connectorCfg["redirectURI"] = dexRedirectURL
		connector["config"] = connectorCfg
		connectors[i] = connector
	}
	dexCfg["connectors"] = connectors
	dexCfg = replaceMapSecrets(dexCfg, settings.Secrets)
	return yaml.Marshal(dexCfg)
}

// replaceMapSecrets takes a json object and recursively looks for any secret key references in the
// object and replaces the value with the secret value
func replaceMapSecrets(obj map[string]interface{}, secretValues map[string]string) map[string]interface{} {
	newObj := make(map[string]interface{})
	for k, v := range obj {
		switch val := v.(type) {
		case map[string]interface{}:
			newObj[k] = replaceMapSecrets(val, secretValues)
		case []interface{}:
			newObj[k] = replaceListSecrets(val, secretValues)
		case string:
			newObj[k] = settings.ReplaceStringSecret(val, secretValues)
		default:
			newObj[k] = val
		}
	}
	return newObj
}

func replaceListSecrets(obj []interface{}, secretValues map[string]string) []interface{} {
	newObj := make([]interface{}, len(obj))
	for i, v := range obj {
		switch val := v.(type) {
		case map[string]interface{}:
			newObj[i] = replaceMapSecrets(val, secretValues)
		case []interface{}:
			newObj[i] = replaceListSecrets(val, secretValues)
		case string:
			newObj[i] = settings.ReplaceStringSecret(val, secretValues)
		default:
			newObj[i] = val
		}
	}
	return newObj
}

// needsRedirectURI returns whether or not the given connector type needs a redirectURI
// Update this list as necessary, as new connectors are added
// https://github.com/dexidp/dex/tree/master/Documentation/connectors
func needsRedirectURI(connectorType string) bool {
	switch connectorType {
	case "oidc", "saml", "microsoft", "linkedin", "gitlab", "github", "bitbucket-cloud", "openshift":
		return true
	}
	return false
}
<|endoftext|>


<commit_before>package openstack

import (
	"fmt"
	"testing"

	"github.com/hashicorp/terraform/helper/resource"
	"github.com/hashicorp/terraform/terraform"

	"github.com/gophercloud/gophercloud/openstack/networking/v2/extensions/security/groups"
	"github.com/gophercloud/gophercloud/openstack/networking/v2/extensions/security/rules"
)

func TestAccNetworkingV2SecGroupRule_basic(t *testing.T) {
	var secgroup_1 groups.SecGroup
	var secgroup_2 groups.SecGroup
	var secgroup_rule_1 rules.SecGroupRule
	var secgroup_rule_2 rules.SecGroupRule

	resource.Test(t, resource.TestCase{
		PreCheck:     func() { testAccPreCheck(t) },
		Providers:    testAccProviders,
		CheckDestroy: testAccCheckNetworkingV2SecGroupRuleDestroy,
		Steps: []resource.TestStep{
			resource.TestStep{
				Config: testAccNetworkingV2SecGroupRule_basic,
				Check: resource.ComposeTestCheckFunc(
					testAccCheckNetworkingV2SecGroupExists(
						"openstack_networking_secgroup_v2.secgroup_1", &secgroup_1),
					testAccCheckNetworkingV2SecGroupExists(
						"openstack_networking_secgroup_v2.secgroup_2", &secgroup_2),
					testAccCheckNetworkingV2SecGroupRuleExists(
						"openstack_networking_secgroup_rule_v2.secgroup_rule_1", &secgroup_rule_1),
					testAccCheckNetworkingV2SecGroupRuleExists(
						"openstack_networking_secgroup_rule_v2.secgroup_rule_2", &secgroup_rule_2),
				),
			},
		},
	})
}

func TestAccNetworkingV2SecGroupRule_lowerCaseCIDR(t *testing.T) {
	var secgroup_1 groups.SecGroup
	var secgroup_rule_1 rules.SecGroupRule

	resource.Test(t, resource.TestCase{
		PreCheck:     func() { testAccPreCheck(t) },
		Providers:    testAccProviders,
		CheckDestroy: testAccCheckNetworkingV2SecGroupRuleDestroy,
		Steps: []resource.TestStep{
			resource.TestStep{
				Config: testAccNetworkingV2SecGroupRule_lowerCaseCIDR,
				Check: resource.ComposeTestCheckFunc(
					testAccCheckNetworkingV2SecGroupExists(
						"openstack_networking_secgroup_v2.secgroup_1", &secgroup_1),
					testAccCheckNetworkingV2SecGroupRuleExists(
						"openstack_networking_secgroup_rule_v2.secgroup_rule_1", &secgroup_rule_1),
					resource.TestCheckResourceAttr(
						"openstack_networking_secgroup_rule_v2.secgroup_rule_1", "remote_ip_prefix", "2001:558:fc00::/39"),
				),
			},
		},
	})
}

func TestAccNetworkingV2SecGroupRule_timeout(t *testing.T) {
	var secgroup_1 groups.SecGroup
	var secgroup_2 groups.SecGroup

	resource.Test(t, resource.TestCase{
		PreCheck:     func() { testAccPreCheck(t) },
		Providers:    testAccProviders,
		CheckDestroy: testAccCheckNetworkingV2SecGroupRuleDestroy,
		Steps: []resource.TestStep{
			resource.TestStep{
				Config: testAccNetworkingV2SecGroupRule_timeout,
				Check: resource.ComposeTestCheckFunc(
					testAccCheckNetworkingV2SecGroupExists(
						"openstack_networking_secgroup_v2.secgroup_1", &secgroup_1),
					testAccCheckNetworkingV2SecGroupExists(
						"openstack_networking_secgroup_v2.secgroup_2", &secgroup_2),
				),
			},
		},
	})
}

func testAccCheckNetworkingV2SecGroupRuleDestroy(s *terraform.State) error {
	config := testAccProvider.Meta().(*Config)
	networkingClient, err := config.networkingV2Client(OS_REGION_NAME)
	if err != nil {
		return fmt.Errorf("Error creating OpenStack networking client: %s", err)
	}

	for _, rs := range s.RootModule().Resources {
		if rs.Type != "openstack_networking_secgroup_rule_v2" {
			continue
		}

		_, err := rules.Get(networkingClient, rs.Primary.ID).Extract()
		if err == nil {
			return fmt.Errorf("Security group rule still exists")
		}
	}

	return nil
}

func testAccCheckNetworkingV2SecGroupRuleExists(n string, security_group_rule *rules.SecGroupRule) resource.TestCheckFunc {
	return func(s *terraform.State) error {
		rs, ok := s.RootModule().Resources[n]
		if !ok {
			return fmt.Errorf("Not found: %s", n)
		}

		if rs.Primary.ID == "" {
			return fmt.Errorf("No ID is set")
		}

		config := testAccProvider.Meta().(*Config)
		networkingClient, err := config.networkingV2Client(OS_REGION_NAME)
		if err != nil {
			return fmt.Errorf("Error creating OpenStack networking client: %s", err)
		}

		found, err := rules.Get(networkingClient, rs.Primary.ID).Extract()
		if err != nil {
			return err
		}

		if found.ID != rs.Primary.ID {
			return fmt.Errorf("Security group rule not found")
		}

		*security_group_rule = *found

		return nil
	}
}

const testAccNetworkingV2SecGroupRule_basic = `
resource "openstack_networking_secgroup_v2" "secgroup_1" {
  name = "secgroup_1"
  description = "terraform security group rule acceptance test"
}

resource "openstack_networking_secgroup_v2" "secgroup_2" {
  name = "secgroup_2"
  description = "terraform security group rule acceptance test"
}

resource "openstack_networking_secgroup_rule_v2" "secgroup_rule_1" {
  direction = "ingress"
  ethertype = "IPv4"
  port_range_max = 22
  port_range_min = 22
  protocol = "tcp"
  remote_ip_prefix = "0.0.0.0/0"
  security_group_id = "${openstack_networking_secgroup_v2.secgroup_1.id}"
}

resource "openstack_networking_secgroup_rule_v2" "secgroup_rule_2" {
  direction = "ingress"
  ethertype = "IPv4"
  port_range_max = 80
  port_range_min = 80
  protocol = "tcp"
  remote_group_id = "${openstack_networking_secgroup_v2.secgroup_1.id}"
  security_group_id = "${openstack_networking_secgroup_v2.secgroup_2.id}"
}
`

const testAccNetworkingV2SecGroupRule_lowerCaseCIDR = `
resource "openstack_networking_secgroup_v2" "secgroup_1" {
  name = "secgroup_1"
  description = "terraform security group rule acceptance test"
}

resource "openstack_networking_secgroup_rule_v2" "secgroup_rule_1" {
  direction = "ingress"
  ethertype = "IPv6"
  port_range_max = 22
  port_range_min = 22
  protocol = "tcp"
  remote_ip_prefix = "2001:558:FC00::/39"
  security_group_id = "${openstack_networking_secgroup_v2.secgroup_1.id}"
}
`

const testAccNetworkingV2SecGroupRule_timeout = `
resource "openstack_networking_secgroup_v2" "secgroup_1" {
  name = "secgroup_1"
  description = "terraform security group rule acceptance test"
}

resource "openstack_networking_secgroup_v2" "secgroup_2" {
  name = "secgroup_2"
  description = "terraform security group rule acceptance test"
}

resource "openstack_networking_secgroup_rule_v2" "secgroup_rule_1" {
  direction = "ingress"
  ethertype = "IPv4"
  port_range_max = 22
  port_range_min = 22
  protocol = "tcp"
  remote_ip_prefix = "0.0.0.0/0"
  security_group_id = "${openstack_networking_secgroup_v2.secgroup_1.id}"

  timeouts {
    create = "5m"
  }
}

resource "openstack_networking_secgroup_rule_v2" "secgroup_rule_2" {
  direction = "ingress"
  ethertype = "IPv4"
  port_range_max = 80
  port_range_min = 80
  protocol = "tcp"
  remote_group_id = "${openstack_networking_secgroup_v2.secgroup_1.id}"
  security_group_id = "${openstack_networking_secgroup_v2.secgroup_2.id}"

  timeouts {
    delete = "5m"
  }
}
`
<commit_msg>provider/openstack: Fixing typo in secgroup rule timeout test<commit_after>package openstack

import (
	"fmt"
	"testing"

	"github.com/hashicorp/terraform/helper/resource"
	"github.com/hashicorp/terraform/terraform"

	"github.com/gophercloud/gophercloud/openstack/networking/v2/extensions/security/groups"
	"github.com/gophercloud/gophercloud/openstack/networking/v2/extensions/security/rules"
)

func TestAccNetworkingV2SecGroupRule_basic(t *testing.T) {
	var secgroup_1 groups.SecGroup
	var secgroup_2 groups.SecGroup
	var secgroup_rule_1 rules.SecGroupRule
	var secgroup_rule_2 rules.SecGroupRule

	resource.Test(t, resource.TestCase{
		PreCheck:     func() { testAccPreCheck(t) },
		Providers:    testAccProviders,
		CheckDestroy: testAccCheckNetworkingV2SecGroupRuleDestroy,
		Steps: []resource.TestStep{
			resource.TestStep{
				Config: testAccNetworkingV2SecGroupRule_basic,
				Check: resource.ComposeTestCheckFunc(
					testAccCheckNetworkingV2SecGroupExists(
						"openstack_networking_secgroup_v2.secgroup_1", &secgroup_1),
					testAccCheckNetworkingV2SecGroupExists(
						"openstack_networking_secgroup_v2.secgroup_2", &secgroup_2),
					testAccCheckNetworkingV2SecGroupRuleExists(
						"openstack_networking_secgroup_rule_v2.secgroup_rule_1", &secgroup_rule_1),
					testAccCheckNetworkingV2SecGroupRuleExists(
						"openstack_networking_secgroup_rule_v2.secgroup_rule_2", &secgroup_rule_2),
				),
			},
		},
	})
}

func TestAccNetworkingV2SecGroupRule_lowerCaseCIDR(t *testing.T) {
	var secgroup_1 groups.SecGroup
	var secgroup_rule_1 rules.SecGroupRule

	resource.Test(t, resource.TestCase{
		PreCheck:     func() { testAccPreCheck(t) },
		Providers:    testAccProviders,
		CheckDestroy: testAccCheckNetworkingV2SecGroupRuleDestroy,
		Steps: []resource.TestStep{
			resource.TestStep{
				Config: testAccNetworkingV2SecGroupRule_lowerCaseCIDR,
				Check: resource.ComposeTestCheckFunc(
					testAccCheckNetworkingV2SecGroupExists(
						"openstack_networking_secgroup_v2.secgroup_1", &secgroup_1),
					testAccCheckNetworkingV2SecGroupRuleExists(
						"openstack_networking_secgroup_rule_v2.secgroup_rule_1", &secgroup_rule_1),
					resource.TestCheckResourceAttr(
						"openstack_networking_secgroup_rule_v2.secgroup_rule_1", "remote_ip_prefix", "2001:558:fc00::/39"),
				),
			},
		},
	})
}

func TestAccNetworkingV2SecGroupRule_timeout(t *testing.T) {
	var secgroup_1 groups.SecGroup
	var secgroup_2 groups.SecGroup

	resource.Test(t, resource.TestCase{
		PreCheck:     func() { testAccPreCheck(t) },
		Providers:    testAccProviders,
		CheckDestroy: testAccCheckNetworkingV2SecGroupRuleDestroy,
		Steps: []resource.TestStep{
			resource.TestStep{
				Config: testAccNetworkingV2SecGroupRule_timeout,
				Check: resource.ComposeTestCheckFunc(
					testAccCheckNetworkingV2SecGroupExists(
						"openstack_networking_secgroup_v2.secgroup_1", &secgroup_1),
					testAccCheckNetworkingV2SecGroupExists(
						"openstack_networking_secgroup_v2.secgroup_2", &secgroup_2),
				),
			},
		},
	})
}

func testAccCheckNetworkingV2SecGroupRuleDestroy(s *terraform.State) error {
	config := testAccProvider.Meta().(*Config)
	networkingClient, err := config.networkingV2Client(OS_REGION_NAME)
	if err != nil {
		return fmt.Errorf("Error creating OpenStack networking client: %s", err)
	}

	for _, rs := range s.RootModule().Resources {
		if rs.Type != "openstack_networking_secgroup_rule_v2" {
			continue
		}

		_, err := rules.Get(networkingClient, rs.Primary.ID).Extract()
		if err == nil {
			return fmt.Errorf("Security group rule still exists")
		}
	}

	return nil
}

func testAccCheckNetworkingV2SecGroupRuleExists(n string, security_group_rule *rules.SecGroupRule) resource.TestCheckFunc {
	return func(s *terraform.State) error {
		rs, ok := s.RootModule().Resources[n]
		if !ok {
			return fmt.Errorf("Not found: %s", n)
		}

		if rs.Primary.ID == "" {
			return fmt.Errorf("No ID is set")
		}

		config := testAccProvider.Meta().(*Config)
		networkingClient, err := config.networkingV2Client(OS_REGION_NAME)
		if err != nil {
			return fmt.Errorf("Error creating OpenStack networking client: %s", err)
		}

		found, err := rules.Get(networkingClient, rs.Primary.ID).Extract()
		if err != nil {
			return err
		}

		if found.ID != rs.Primary.ID {
			return fmt.Errorf("Security group rule not found")
		}

		*security_group_rule = *found

		return nil
	}
}

const testAccNetworkingV2SecGroupRule_basic = `
resource "openstack_networking_secgroup_v2" "secgroup_1" {
  name = "secgroup_1"
  description = "terraform security group rule acceptance test"
}

resource "openstack_networking_secgroup_v2" "secgroup_2" {
  name = "secgroup_2"
  description = "terraform security group rule acceptance test"
}

resource "openstack_networking_secgroup_rule_v2" "secgroup_rule_1" {
  direction = "ingress"
  ethertype = "IPv4"
  port_range_max = 22
  port_range_min = 22
  protocol = "tcp"
  remote_ip_prefix = "0.0.0.0/0"
  security_group_id = "${openstack_networking_secgroup_v2.secgroup_1.id}"
}

resource "openstack_networking_secgroup_rule_v2" "secgroup_rule_2" {
  direction = "ingress"
  ethertype = "IPv4"
  port_range_max = 80
  port_range_min = 80
  protocol = "tcp"
  remote_group_id = "${openstack_networking_secgroup_v2.secgroup_1.id}"
  security_group_id = "${openstack_networking_secgroup_v2.secgroup_2.id}"
}
`

const testAccNetworkingV2SecGroupRule_lowerCaseCIDR = `
resource "openstack_networking_secgroup_v2" "secgroup_1" {
  name = "secgroup_1"
  description = "terraform security group rule acceptance test"
}

resource "openstack_networking_secgroup_rule_v2" "secgroup_rule_1" {
  direction = "ingress"
  ethertype = "IPv6"
  port_range_max = 22
  port_range_min = 22
  protocol = "tcp"
  remote_ip_prefix = "2001:558:FC00::/39"
  security_group_id = "${openstack_networking_secgroup_v2.secgroup_1.id}"
}
`

const testAccNetworkingV2SecGroupRule_timeout = `
resource "openstack_networking_secgroup_v2" "secgroup_1" {
  name = "secgroup_1"
  description = "terraform security group rule acceptance test"
}

resource "openstack_networking_secgroup_v2" "secgroup_2" {
  name = "secgroup_2"
  description = "terraform security group rule acceptance test"
}

resource "openstack_networking_secgroup_rule_v2" "secgroup_rule_1" {
  direction = "ingress"
  ethertype = "IPv4"
  port_range_max = 22
  port_range_min = 22
  protocol = "tcp"
  remote_ip_prefix = "0.0.0.0/0"
  security_group_id = "${openstack_networking_secgroup_v2.secgroup_1.id}"

  timeouts {
    delete = "5m"
  }
}

resource "openstack_networking_secgroup_rule_v2" "secgroup_rule_2" {
  direction = "ingress"
  ethertype = "IPv4"
  port_range_max = 80
  port_range_min = 80
  protocol = "tcp"
  remote_group_id = "${openstack_networking_secgroup_v2.secgroup_1.id}"
  security_group_id = "${openstack_networking_secgroup_v2.secgroup_2.id}"

  timeouts {
    delete = "5m"
  }
}
`
<|endoftext|>


<commit_before>package gcache

import (
	"errors"
	"fmt"
	"io/ioutil"
	"net/http"
	"time"

	"golang.org/x/oauth2"
	"google.golang.org/api/drive/v3"
	"google.golang.org/api/googleapi"
)

var (
	oauth2TokenSource oauth2.TokenSource // The token is valid for 30 minutes.
)

// GetGDriveService returns the API service of Google Drive.
func GetGDriveService(
	r *http.Request,
) (
	*drive.Service,
	error,
) {
	n := 1
retry:
	service, err := drive.New(createGDriveClient(r))
	if err == nil {
		return service, nil
	}
	_, n, err = triable(n, err)
	if err != nil {
		return nil, err
	}
	goto retry
}

func getGDriveFile(
	r *http.Request,
	name string,
	field googleapi.Field,
) (
	*drive.File,
	*drive.Service,
	error,
) {
	if field == "" {
		field = MinimumField
	}

	var refreshToken bool
	n := 1
refresh:
	service, err := GetGDriveService(r)
	if err != nil {
		return nil, nil, err
	}

retry:
	fileList, err := service.Files.List().PageSize(1).Spaces("drive").Q(fmt.Sprintf("name='%s'", name)).Fields(field).Do()

	if err != nil {
		refreshToken, n, err = triable(n, err)
		if err != nil {
			return nil, nil, err
		}
		if refreshToken {
			goto refresh
		}
		goto retry
	}

	if len(fileList.Files) <= 0 {
		return nil, nil, &DriveFileDoesNotExistError{}
	}
	return fileList.Files[0], service, nil
}

// GetGDriveFile returns a file on Google Drive.
func GetGDriveFile(
	r *http.Request,
	name string,
	field googleapi.Field,
) (
	*drive.File,
	[]byte, // payload
	error,
) {
	if field == "" {
		field = defaultField
	}

	file, service, err := getGDriveFile(r, name, field)
	if err != nil {
		return nil, nil, err
	}
	fileID := file.Id

	var payload []byte

	var refreshToken bool
	n := 1
retry:
	httpResponse, err := service.Files.Export(fileID, mimeTxt).Download()
	if IsFileNotExportableError(err) {
		err = service.Files.Delete(fileID).Do()
		if err != nil {
			// pass
		}
		return nil, nil, &DriveFileDoesNotExistError{}
	}
	if err != nil {
		refreshToken, n, err = triable(n, err)
		if err != nil {
			return nil, nil, err
		}
		if refreshToken {
			service, err = GetGDriveService(r)
			if err != nil {
				return nil, nil, err
			}
		}
		goto retry
	}

	defer httpResponse.Body.Close()
	payload, err = ioutil.ReadAll(httpResponse.Body)
	if err != nil {
		return nil, nil, err
	}
	return file, payload, err
}

func triable(
	retries int,
	err error,
) (
	bool, //refreshToken
	int, //retries
	error,
) {
	if err == nil {
		return false, retries, nil
	}
	if retries < 1 {
		retries = 1
	}
	if IsInvalidSecurityTicket(err) {
		oauth2TokenSource = nil
		return true, retries, nil
	}
	if IsServerError(err) {
		retries, err = sleeping(retries)
		if err != nil {
			return false, retries, err
		}
		return false, retries, nil
	}
	return false, retries, err
}

func sleeping(
	n int,
) (
	int,
	error,
) {
	if n > 16 {
		return 0, errors.New("Sleeping Timeout")
	}
	time.Sleep(time.Duration(n)*time.Second + time.Duration(random.Intn(1000))*time.Millisecond)
	return n * 2, nil
}
<commit_msg>update GetGDriveFile and GetGDriveFileContent<commit_after>package gcache

import (
	"errors"
	"fmt"
	"io/ioutil"
	"net/http"
	"time"

	"golang.org/x/oauth2"
	"google.golang.org/api/drive/v3"
	"google.golang.org/api/googleapi"
)

var (
	oauth2TokenSource oauth2.TokenSource // The token is valid for 30 minutes.
)

// GetGDriveService returns the API service of Google Drive.
func GetGDriveService(
	r *http.Request,
) (
	*drive.Service,
	error,
) {
	n := 1
retry:
	service, err := drive.New(createGDriveClient(r))
	if err == nil {
		return service, nil
	}
	_, n, err = triable(n, err)
	if err != nil {
		return nil, err
	}
	goto retry
}

func getGDriveFile(
	r *http.Request,
	name string,
	field googleapi.Field,
) (
	*drive.File,
	*drive.Service,
	error,
) {
	if field == "" {
		field = MinimumField
	}

	var refreshToken bool
	n := 1
refresh:
	service, err := GetGDriveService(r)
	if err != nil {
		return nil, nil, err
	}

retry:
	fileList, err := service.Files.List().PageSize(1).Spaces("drive").Q(fmt.Sprintf("name='%s'", name)).Fields(field).Do()

	if err != nil {
		refreshToken, n, err = triable(n, err)
		if err != nil {
			return nil, nil, err
		}
		if refreshToken {
			goto refresh
		}
		goto retry
	}

	if len(fileList.Files) <= 0 {
		return nil, nil, &DriveFileDoesNotExistError{}
	}
	return fileList.Files[0], service, nil
}

// GetGDriveFile returns a file on Google Drive.
func GetGDriveFile(
	r *http.Request,
	name string,
	field googleapi.Field,
) (
	*drive.File,
	error,
) {
	if field == "" {
		field = defaultField
	}

	file, _, err := getGDriveFile(r, name, field)
	if err != nil {
		return nil, err
	}

	return file, nil
}

// GetGDriveFileContent returns a file with content on Google Drive.
func GetGDriveFileContent(
	r *http.Request,
	name string,
	field googleapi.Field,
) (
	*drive.File,
	[]byte, // payload
	error,
) {
	if field == "" {
		field = defaultField
	}

	file, service, err := getGDriveFile(r, name, field)
	if err != nil {
		return nil, nil, err
	}
	fileID := file.Id

	var payload []byte

	var refreshToken bool
	n := 1
retry:
	httpResponse, err := service.Files.Export(fileID, mimeTxt).Download()
	if IsFileNotExportableError(err) {
		err = service.Files.Delete(fileID).Do()
		if err != nil {
			// pass
		}
		return nil, nil, &DriveFileDoesNotExistError{}
	}
	if err != nil {
		refreshToken, n, err = triable(n, err)
		if err != nil {
			return nil, nil, err
		}
		if refreshToken {
			service, err = GetGDriveService(r)
			if err != nil {
				return nil, nil, err
			}
		}
		goto retry
	}

	defer httpResponse.Body.Close()
	payload, err = ioutil.ReadAll(httpResponse.Body)
	if err != nil {
		return nil, nil, err
	}
	return file, payload, err
}

func triable(
	retries int,
	err error,
) (
	bool, //refreshToken
	int, //retries
	error,
) {
	if err == nil {
		return false, retries, nil
	}
	if retries < 1 {
		retries = 1
	}
	if IsInvalidSecurityTicket(err) {
		oauth2TokenSource = nil
		return true, retries, nil
	}
	if IsServerError(err) {
		retries, err = sleeping(retries)
		if err != nil {
			return false, retries, err
		}
		return false, retries, nil
	}
	return false, retries, err
}

func sleeping(
	n int,
) (
	int,
	error,
) {
	if n > 16 {
		return 0, errors.New("Sleeping Timeout")
	}
	time.Sleep(time.Duration(n)*time.Second + time.Duration(random.Intn(1000))*time.Millisecond)
	return n * 2, nil
}
<|endoftext|>


<commit_before>package analytics

import (
	"bytes"
	"net/http"
	"net/url"
	"runtime"
	"strconv"

	"github.com/getlantern/flashlight/util"
	"github.com/getlantern/golog"
)

const (
	ApiEndpoint       = `https://ssl.google-analytics.com/collect`
	ProtocolVersion   = "1"
	DefaultInstanceId = "1260961011.1389432370"
)

var (
	log        = golog.LoggerFor("analytics")
	httpClient *http.Client
)

type HitType string

const (
	PageViewType HitType = "pageview"
	EventType    HitType = "event"
)

type PageView struct {
	Hostname string `param:"dh"`
	Pagename string `param:"dp"`
	Title    string `param:"dt"`
}

type Event struct {
	Category string `param:"ec"`
	Action   string `param:"ea"`
	Label    string `param:"el,omitempty"`
	Value    string `param:"ev,omitempty"`
}

type Payload struct {
	InstanceId string `json:"clientId"`

	ClientVersion string `json:"clientVersion,omitempty"`

	ViewPortSize string `json:"viewPortSize,omitempty"`

	TrackingId string `json:"trackingId"`

	Language string `json:"language,omitempty"`

	ScreenColors string `json:"screenColors,omitempty"`

	ScreenResolution string `json:"screenResolution,omitempty"`

	Hostname string `json:"hostname,omitempty"`

	HitType HitType `json:"hitType,omitempty"`

	CustomVars map[string]string

	UserAgent string

	Event *Event
}

func Configure(trackingId string, version string, proxyAddr string) {
	var err error
	go func() {
		httpClient, err = util.HTTPClient("", proxyAddr)
		if err != nil {
			log.Errorf("Could not create HTTP client via %s: %s", proxyAddr, err)
			return
		}
		// Store new session info whenever client proxy is ready
		sessionEvent(version, trackingId)
	}()
}

// assemble list of parameters to send to GA
func collectArgs(payload *Payload) string {
	vals := make(url.Values, 0)

	// Add default payload
	vals.Add("v", ProtocolVersion)
	if payload.ClientVersion != "" {
		vals.Add("_v", payload.ClientVersion)
	}
	if payload.TrackingId != "" {
		vals.Add("tid", payload.TrackingId)
	}
	if payload.InstanceId != "" {
		vals.Add("cid", payload.InstanceId)
	}
	if payload.ScreenResolution != "" {
		vals.Add("sr", payload.ScreenResolution)
	}
	if payload.Language != "" {
		vals.Add("ul", payload.Language)
	}

	vals.Add("dh", payload.Hostname)

	vals.Add("t", string(payload.HitType))

	if payload.HitType == EventType && payload.Event != nil {
		vals.Add("ec", payload.Event.Category)
		vals.Add("ea", payload.Event.Action)
		if payload.Event.Label != "" {
			vals.Add("el", payload.Event.Label)
		}
		if payload.Event.Value != "" {
			vals.Add("ev", payload.Event.Value)
		}
	}

	for dim, customVar := range payload.CustomVars {
		if customVar != "" {
			vals.Add(dim, customVar)
		}
	}

	return vals.Encode()
}

// Makes a tracking request to Google Analytics
func SendRequest(payload *Payload) (status bool, err error) {
	if httpClient == nil {
		log.Error("No HTTP client; could not send HTTP request to GA")
		return false, nil
	}

	args := collectArgs(payload)

	r, err := http.NewRequest("POST", ApiEndpoint, bytes.NewBufferString(args))
	r.Header.Add("Content-Type", "application/x-www-form-urlencoded")
	r.Header.Add("Content-Length", strconv.Itoa(len(args)))

	if err != nil {
		log.Errorf("Error constructing GA request: %s", err)
		return false, err
	}

	resp, err := httpClient.Do(r)
	if err != nil {
		log.Errorf("Could not send HTTP request to GA: %s", err)
		return false, err
	}
	log.Debugf("Successfully sent request to GA: %s", resp.Status)
	defer resp.Body.Close()

	return true, nil
}

// Fired whenever a new Lanern session is initiated
func sessionEvent(trackingId string, version string) (status bool, err error) {

	sessionPayload := &Payload{
		HitType:    EventType,
		TrackingId: trackingId,
		Hostname:   "localhost",
		InstanceId: DefaultInstanceId,
		Event: &Event{
			Category: "Session",
			Action:   "Start",
			Label:    runtime.GOOS,
			Value:    version,
		},
	}

	if version != "" {
		sessionPayload.CustomVars = map[string]string{
			"cd1": version,
		}
	}
	return SendRequest(sessionPayload)
}
<commit_msg>analytics fixes<commit_after>package analytics

import (
	"bytes"
	"net/http"
	"net/url"
	"runtime"
	"strconv"

	"github.com/getlantern/flashlight/util"
	"github.com/getlantern/golog"
)

const (
	ApiEndpoint       = `https://ssl.google-analytics.com/collect`
	ProtocolVersion   = "1"
	DefaultInstanceId = "555"
)

var (
	log        = golog.LoggerFor("analytics")
	httpClient *http.Client
)

type HitType string

const (
	PageViewType HitType = "pageview"
	EventType    HitType = "event"
)

type PageView struct {
	Hostname string `param:"dh"`
	Pagename string `param:"dp"`
	Title    string `param:"dt"`
}

type Event struct {
	Category string `param:"ec"`
	Action   string `param:"ea"`
	Label    string `param:"el,omitempty"`
	Value    string `param:"ev,omitempty"`
}

type Payload struct {
	InstanceId string `json:"clientId"`

	ClientVersion string `json:"clientVersion,omitempty"`

	ViewPortSize string `json:"viewPortSize,omitempty"`

	TrackingId string `json:"trackingId"`

	Language string `json:"language,omitempty"`

	ScreenColors string `json:"screenColors,omitempty"`

	ScreenResolution string `json:"screenResolution,omitempty"`

	Hostname string `json:"hostname,omitempty"`

	HitType HitType `json:"hitType,omitempty"`

	CustomVars map[string]string

	UserAgent string

	Event *Event
}

func Configure(trackingId string, version string, proxyAddr string) {
	var err error
	go func() {
		httpClient, err = util.HTTPClient("", proxyAddr)
		if err != nil {
			log.Errorf("Could not create HTTP client via %s: %s", proxyAddr, err)
			return
		}
		// Store new session info whenever client proxy is ready
		sessionEvent(trackingId, version)
	}()
}

// assemble list of parameters to send to GA
func collectArgs(payload *Payload) string {
	vals := make(url.Values, 0)

	// Add default payload
	vals.Add("v", ProtocolVersion)
	if payload.ClientVersion != "" {
		vals.Add("_v", payload.ClientVersion)
	}
	if payload.TrackingId != "" {
		vals.Add("tid", payload.TrackingId)
	}
	if payload.InstanceId != "" {
		vals.Add("cid", payload.InstanceId)
	}
	if payload.ScreenResolution != "" {
		vals.Add("sr", payload.ScreenResolution)
	}
	if payload.Language != "" {
		vals.Add("ul", payload.Language)
	}

	vals.Add("t", string(payload.HitType))

	if payload.HitType == EventType && payload.Event != nil {
		vals.Add("ec", payload.Event.Category)
		vals.Add("ea", payload.Event.Action)
		if payload.Event.Label != "" {
			vals.Add("el", payload.Event.Label)
		}
		if payload.Event.Value != "" {
			vals.Add("ev", payload.Event.Value)
		}
	}

	for dim, customVar := range payload.CustomVars {
		if customVar != "" {
			vals.Add(dim, customVar)
		}
	}

	return vals.Encode()
}

// Makes a tracking request to Google Analytics
func SendRequest(payload *Payload) (status bool, err error) {
	if httpClient == nil {
		log.Error("No HTTP client; could not send HTTP request to GA")
		return false, nil
	}

	args := collectArgs(payload)

	r, err := http.NewRequest("POST", ApiEndpoint, bytes.NewBufferString(args))
	r.Header.Add("Content-Type", "application/x-www-form-urlencoded")
	r.Header.Add("Content-Length", strconv.Itoa(len(args)))

	if err != nil {
		log.Errorf("Error constructing GA request: %s", err)
		return false, err
	}

	resp, err := httpClient.Do(r)
	if err != nil {
		log.Errorf("Could not send HTTP request to GA: %s", err)
		return false, err
	}
	log.Debugf("Successfully sent request to GA: %s", resp.Status)
	defer resp.Body.Close()

	return true, nil
}

// Fired whenever a new Lanern session is initiated
func sessionEvent(trackingId string, version string) (status bool, err error) {

	sessionPayload := &Payload{
		HitType:    EventType,
		TrackingId: trackingId,
		InstanceId: DefaultInstanceId,
		Event: &Event{
			Category: "Session",
			Action:   "Start",
			Label:    runtime.GOOS,
		},
	}

	if version != "" {
		sessionPayload.CustomVars = map[string]string{
			"cd1": version,
		}
	}
	return SendRequest(sessionPayload)
}
<|endoftext|>


<commit_before>/*

Provides the mechanism for managing a given OLContainer container-based lambda.

Must be paired with a OLContainerSandboxManager which handles pulling handler
code, initializing containers, etc.

*/

package sandbox

import (
	"fmt"
	"io/ioutil"
	"log"
	"net"
	"net/http"
	"os"
	"os/exec"
	"path"
	"path/filepath"
	"strconv"
	"strings"
	"syscall"
	"time"

	"github.com/open-lambda/open-lambda/worker/config"
	"github.com/open-lambda/open-lambda/worker/handler/state"
)

type OLContainerSandbox struct {
	opts         *config.Config
	cgf          *CgroupFactory
	id           string
	cgId         string
	rootDir      string
	hostDir      string
	status       state.HandlerState
	initPid      string
	initCmd      *exec.Cmd
	startCmd     []string
	unshareFlags []string
	unmounts     []string
	removals     []string
}

func NewOLContainerSandbox(cgf *CgroupFactory, opts *config.Config, rootDir, hostDir, id string, startCmd, unshareFlags, unmounts, removals []string) (*OLContainerSandbox, error) {
	// create container cgroups
	cgId := cgf.GetCg(id)

	sandbox := &OLContainerSandbox{
		cgf:          cgf,
		opts:         opts,
		id:           id,
		cgId:         cgId,
		rootDir:      rootDir,
		hostDir:      hostDir,
		unshareFlags: unshareFlags,
		status:       state.Stopped,
		startCmd:     startCmd,
		unmounts:     unmounts,
		removals:     removals,
	}

	return sandbox, nil
}

func (s *OLContainerSandbox) State() (hstate state.HandlerState, err error) {
	return s.status, nil
}

func (s *OLContainerSandbox) Channel() (channel *SandboxChannel, err error) {
	dial := func(proto, addr string) (net.Conn, error) {
		return net.Dial("unix", filepath.Join(s.hostDir, "ol.sock"))
	}
	tr := http.Transport{Dial: dial}

	// the server name doesn't matter since we have a sock file
	return &SandboxChannel{Url: "http://container/", Transport: tr}, nil
}

func (s *OLContainerSandbox) Start() error {
	initArgs := []string{s.opts.OLContainer_init_path, s.rootDir}
	initArgs = append(initArgs, s.startCmd...)
	initArgs = append(s.unshareFlags, initArgs...)

	s.initCmd = exec.Command(
		"unshare",
		initArgs...,
	)

	s.initCmd.Env = []string{fmt.Sprintf("ol.config=%s", s.opts.SandboxConfJson())}
	if err := s.initCmd.Start(); err != nil {
		return err
	}

	// wait up to 5s for server olcontainer_init to spawn
	start := time.Now()
	for {
		pgrep := exec.Command("pgrep", "-P", strconv.Itoa(s.initCmd.Process.Pid))
		out, err := pgrep.Output()
		if err == nil {
			s.initPid = strings.TrimSpace(string(out[:]))
			break
		}

		if time.Since(start).Seconds() > 5 {
			return fmt.Errorf("olcontainer_init failed to spawn after 5s")
		}
		time.Sleep(10 * time.Microsecond)
	}
	log.Printf("wait for olcontainer_init took %v\n", time.Since(start))

	if err := s.CGroupEnter(s.initPid); err != nil {
		return err
	}

	s.status = state.Running
	return nil
}

func (s *OLContainerSandbox) Stop() error {
	start := time.Now()
	// kill any remaining processes
	procsPath := path.Join("/sys/fs/cgroup/memory", OLCGroupName, s.cgId, "cgroup.procs")
	pids, err := ioutil.ReadFile(procsPath)
	if err != nil {
		return err
	}

	for _, pidStr := range strings.Split(strings.TrimSpace(string(pids[:])), "\n") {
		if pidStr == "" {
			break
		}

		pid, err := strconv.Atoi(pidStr)
		if err != nil {
			log.Printf("read bad pid string: %s :: %v", pidStr, err)
			continue
		}

		proc, err := os.FindProcess(pid)
		if err != nil {
			log.Printf("failed to find process with pid=%d :: %v", pid, err)
			continue
		}

		err = proc.Signal(syscall.SIGKILL)
		if err != nil {
			log.Printf("failed to send kill signal to pid=%d :: %v", pid, err)
		}
	}

	go func(s *OLContainerSandbox, start time.Time) {
		// release unshare process resources
		s.initCmd.Process.Kill()
		s.initCmd.Process.Wait()
	}(s, start)

	s.status = state.Stopped
	return nil
}

func (s *OLContainerSandbox) Pause() error {
	freezerPath := path.Join("/sys/fs/cgroup/freezer", OLCGroupName, s.cgId, "freezer.state")
	err := ioutil.WriteFile(freezerPath, []byte("FROZEN"), os.ModeAppend)
	if err != nil {
		return err
	}

	s.status = state.Paused
	return nil
}

func (s *OLContainerSandbox) Unpause() error {
	freezerPath := path.Join("/sys/fs/cgroup/freezer", OLCGroupName, s.cgId, "freezer.state")
	err := ioutil.WriteFile(freezerPath, []byte("THAWED"), os.ModeAppend)
	if err != nil {
		return err
	}

	s.status = state.Running
	return nil
}

func (s *OLContainerSandbox) Remove() error {
	start := time.Now()

	// remove sockets if they exist
	if err := os.RemoveAll(filepath.Join(s.hostDir, "ol.sock")); err != nil {
		return err
	}
	if err := os.RemoveAll(filepath.Join(s.hostDir, "fs.sock")); err != nil {
		return err
	}

	// remove cgroups
	s.cgf.PutCg(s.id, s.cgId)

	// unmount things
	for _, mnt := range s.unmounts {
		if err := syscall.Unmount(mnt, syscall.MNT_DETACH); err != nil {
			log.Printf("unmount %s failed :: %v\n", mnt, err)
		}
	}

	// remove things
	for _, dir := range s.removals {
		if err := os.RemoveAll(dir); err != nil {
			log.Printf("remove %s failed :: %v\n", dir, err)
		}
	}

	log.Printf("remove took %v\n", time.Since(start))

	return nil
}

func (s *OLContainerSandbox) Logs() (string, error) {
	// TODO(ed)
	return "TODO", nil
}

func (s *OLContainerSandbox) CGroupEnter(pid string) (err error) {
	// put process into each cgroup
	for _, cgroup := range CGroupList {
		tasksPath := path.Join("/sys/fs/cgroup/", cgroup, OLCGroupName, s.cgId, "tasks")

		err := ioutil.WriteFile(tasksPath, []byte(pid), os.ModeAppend)
		if err != nil {
			return err
		}
	}

	return nil
}

func (s *OLContainerSandbox) NSPid() string {
	return s.initPid
}

func (s *OLContainerSandbox) ID() string {
	return s.id
}

func (s *OLContainerSandbox) RunServer() error {
	pid, err := strconv.Atoi(s.initPid)
	if err != nil {
		log.Printf("bad initPid string: %s :: %v", s.initPid, err)
		return err
	}

	proc, err := os.FindProcess(pid)
	if err != nil {
		log.Printf("failed to find initPid process with pid=%d :: %v", pid, err)
		return err
	}

	err = proc.Signal(syscall.SIGURG)
	if err != nil {
		log.Printf("failed to send SIGUSR1 to pid=%d :: %v", pid, err)
		return err
	}

	return nil
}

func (s *OLContainerSandbox) MemoryCGroupPath() string {
	return fmt.Sprintf("/sys/fs/cgroup/memory/%s/%s/", OLCGroupName, s.cgId)
}

func (s *OLContainerSandbox) RootDir() string {
	return s.rootDir
}

func (s *OLContainerSandbox) AddUnmounts(mnts []string) {
	s.unmounts = append(mnts, s.unmounts...)

	return
}

func (s *OLContainerSandbox) AddRemovals(dirs []string) {
	s.removals = append(dirs, s.removals...)

	return
}
<commit_msg>sandbox: cgroups fix failure to be removed<commit_after>/*

Provides the mechanism for managing a given OLContainer container-based lambda.

Must be paired with a OLContainerSandboxManager which handles pulling handler
code, initializing containers, etc.

*/

package sandbox

import (
	"fmt"
	"io/ioutil"
	"log"
	"net"
	"net/http"
	"os"
	"os/exec"
	"path"
	"path/filepath"
	"strconv"
	"strings"
	"syscall"
	"time"

	"github.com/open-lambda/open-lambda/worker/config"
	"github.com/open-lambda/open-lambda/worker/handler/state"
)

type OLContainerSandbox struct {
	opts         *config.Config
	cgf          *CgroupFactory
	id           string
	cgId         string
	rootDir      string
	hostDir      string
	status       state.HandlerState
	initPid      string
	initCmd      *exec.Cmd
	startCmd     []string
	unshareFlags []string
	unmounts     []string
	removals     []string
}

func NewOLContainerSandbox(cgf *CgroupFactory, opts *config.Config, rootDir, hostDir, id string, startCmd, unshareFlags, unmounts, removals []string) (*OLContainerSandbox, error) {
	// create container cgroups
	cgId := cgf.GetCg(id)

	sandbox := &OLContainerSandbox{
		cgf:          cgf,
		opts:         opts,
		id:           id,
		cgId:         cgId,
		rootDir:      rootDir,
		hostDir:      hostDir,
		unshareFlags: unshareFlags,
		status:       state.Stopped,
		startCmd:     startCmd,
		unmounts:     unmounts,
		removals:     removals,
	}

	return sandbox, nil
}

func (s *OLContainerSandbox) State() (hstate state.HandlerState, err error) {
	return s.status, nil
}

func (s *OLContainerSandbox) Channel() (channel *SandboxChannel, err error) {
	dial := func(proto, addr string) (net.Conn, error) {
		return net.Dial("unix", filepath.Join(s.hostDir, "ol.sock"))
	}
	tr := http.Transport{Dial: dial}

	// the server name doesn't matter since we have a sock file
	return &SandboxChannel{Url: "http://container/", Transport: tr}, nil
}

func (s *OLContainerSandbox) Start() error {
	initArgs := []string{s.opts.OLContainer_init_path, s.rootDir}
	initArgs = append(initArgs, s.startCmd...)
	initArgs = append(s.unshareFlags, initArgs...)

	s.initCmd = exec.Command(
		"unshare",
		initArgs...,
	)

	s.initCmd.Env = []string{fmt.Sprintf("ol.config=%s", s.opts.SandboxConfJson())}
	if err := s.initCmd.Start(); err != nil {
		return err
	}

	// wait up to 5s for server olcontainer_init to spawn
	start := time.Now()
	for {
		pgrep := exec.Command("pgrep", "-P", strconv.Itoa(s.initCmd.Process.Pid))
		out, err := pgrep.Output()
		if err == nil {
			s.initPid = strings.TrimSpace(string(out[:]))
			break
		}

		if time.Since(start).Seconds() > 5 {
			return fmt.Errorf("olcontainer_init failed to spawn after 5s")
		}
		time.Sleep(10 * time.Microsecond)
	}
	log.Printf("wait for olcontainer_init took %v\n", time.Since(start))

	if err := s.CGroupEnter(s.initPid); err != nil {
		return err
	}

	s.status = state.Running
	return nil
}

func (s *OLContainerSandbox) Stop() error {
	start := time.Now()
	// kill any remaining processes
	procsPath := path.Join("/sys/fs/cgroup/memory", OLCGroupName, s.cgId, "cgroup.procs")
	pids, err := ioutil.ReadFile(procsPath)
	if err != nil {
		return err
	}

	for _, pidStr := range strings.Split(strings.TrimSpace(string(pids[:])), "\n") {
		if pidStr == "" {
			break
		}

		pid, err := strconv.Atoi(pidStr)
		if err != nil {
			log.Printf("read bad pid string: %s :: %v", pidStr, err)
			continue
		}

		proc, err := os.FindProcess(pid)
		if err != nil {
			log.Printf("failed to find process with pid=%d :: %v", pid, err)
			continue
		}

		err = proc.Signal(syscall.SIGKILL)
		if err != nil {
			log.Printf("failed to send kill signal to pid=%d :: %v", pid, err)
		}
	}

	go func(s *OLContainerSandbox, start time.Time) {
		// release unshare process resources
		s.initCmd.Process.Kill()
		s.initCmd.Process.Wait()
	}(s, start)

	s.status = state.Stopped
	return nil
}

func (s *OLContainerSandbox) Pause() error {
	freezerPath := path.Join("/sys/fs/cgroup/freezer", OLCGroupName, s.cgId, "freezer.state")
	err := ioutil.WriteFile(freezerPath, []byte("FROZEN"), os.ModeAppend)
	if err != nil {
		return err
	}

	s.status = state.Paused
	return nil
}

func (s *OLContainerSandbox) Unpause() error {
	freezerPath := path.Join("/sys/fs/cgroup/freezer", OLCGroupName, s.cgId, "freezer.state")
	err := ioutil.WriteFile(freezerPath, []byte("THAWED"), os.ModeAppend)
	if err != nil {
		return err
	}

	s.status = state.Running
	return nil
}

func (s *OLContainerSandbox) Remove() error {
	start := time.Now()

	// remove sockets if they exist
	if err := os.RemoveAll(filepath.Join(s.hostDir, "ol.sock")); err != nil {
		return err
	}
	if err := os.RemoveAll(filepath.Join(s.hostDir, "fs.sock")); err != nil {
		return err
	}

	// unmount things
	for _, mnt := range s.unmounts {
		if err := syscall.Unmount(mnt, syscall.MNT_DETACH); err != nil {
			log.Printf("unmount %s failed :: %v\n", mnt, err)
		}
	}

	// remove things
	for _, dir := range s.removals {
		if err := os.RemoveAll(dir); err != nil {
			log.Printf("remove %s failed :: %v\n", dir, err)
		}
	}

	// remove cgroups
	if err := s.cgf.PutCg(s.id, s.cgId); err != nil {
		log.Printf("Unable to delete cgroups: %v", err)
	}

	log.Printf("remove took %v\n", time.Since(start))

	return nil
}

func (s *OLContainerSandbox) Logs() (string, error) {
	// TODO(ed)
	return "TODO", nil
}

func (s *OLContainerSandbox) CGroupEnter(pid string) (err error) {
	// put process into each cgroup
	for _, cgroup := range CGroupList {
		tasksPath := path.Join("/sys/fs/cgroup/", cgroup, OLCGroupName, s.cgId, "tasks")

		err := ioutil.WriteFile(tasksPath, []byte(pid), os.ModeAppend)
		if err != nil {
			return err
		}
	}

	return nil
}

func (s *OLContainerSandbox) NSPid() string {
	return s.initPid
}

func (s *OLContainerSandbox) ID() string {
	return s.id
}

func (s *OLContainerSandbox) RunServer() error {
	pid, err := strconv.Atoi(s.initPid)
	if err != nil {
		log.Printf("bad initPid string: %s :: %v", s.initPid, err)
		return err
	}

	proc, err := os.FindProcess(pid)
	if err != nil {
		log.Printf("failed to find initPid process with pid=%d :: %v", pid, err)
		return err
	}

	err = proc.Signal(syscall.SIGURG)
	if err != nil {
		log.Printf("failed to send SIGUSR1 to pid=%d :: %v", pid, err)
		return err
	}

	return nil
}

func (s *OLContainerSandbox) MemoryCGroupPath() string {
	return fmt.Sprintf("/sys/fs/cgroup/memory/%s/%s/", OLCGroupName, s.cgId)
}

func (s *OLContainerSandbox) RootDir() string {
	return s.rootDir
}

func (s *OLContainerSandbox) AddUnmounts(mnts []string) {
	s.unmounts = append(mnts, s.unmounts...)

	return
}

func (s *OLContainerSandbox) AddRemovals(dirs []string) {
	s.removals = append(dirs, s.removals...)

	return
}
<|endoftext|>


<commit_before>package gc

import (
	"time"

	"code.cloudfoundry.org/garden"
	"code.cloudfoundry.org/lager"
	"github.com/concourse/atc/db"
	"github.com/concourse/atc/metric"
	"github.com/concourse/atc/worker"
)

const HijackedContainerTimeout = 5 * time.Minute

//go:generate counterfeiter . containerFactory

type containerFactory interface {
	FindContainersForDeletion() ([]db.CreatingContainer, []db.CreatedContainer, []db.DestroyingContainer, error)
}

type containerCollector struct {
	rootLogger       lager.Logger
	containerFactory containerFactory
	jobRunner        WorkerJobRunner
}

func NewContainerCollector(
	logger lager.Logger,
	containerFactory containerFactory,
	jobRunner WorkerJobRunner,
) Collector {
	return &containerCollector{
		rootLogger:       logger,
		containerFactory: containerFactory,
		jobRunner:        jobRunner,
	}
}

type job struct {
	JobName string
	RunFunc func(worker.Worker)
}

func (j *job) Name() string {
	return j.JobName
}

func (j *job) Run(w worker.Worker) {
	j.RunFunc(w)
}

func (c *containerCollector) Run() error {
	logger := c.rootLogger.Session("run")

	logger.Debug("start")
	defer logger.Debug("done")

	creatingContainers, createdContainers, destroyingContainers, err := c.containerFactory.FindContainersForDeletion()
	if err != nil {
		logger.Error("failed-to-get-containers-for-deletion", err)
		return err
	}

	creatingContainerHandles := []string{}
	createdContainerHandles := []string{}
	destroyingContainerHandles := []string{}

	if len(creatingContainers) > 0 {
		for _, container := range creatingContainers {
			creatingContainerHandles = append(creatingContainerHandles, container.Handle())
		}
	}

	if len(createdContainers) > 0 {
		for _, container := range createdContainers {
			createdContainerHandles = append(createdContainerHandles, container.Handle())
		}
	}

	if len(destroyingContainers) > 0 {
		for _, container := range destroyingContainers {
			destroyingContainerHandles = append(destroyingContainerHandles, container.Handle())
		}
	}

	logger.Debug("found-containers-for-deletion", lager.Data{
		"creating-containers":   creatingContainerHandles,
		"created-containers":    createdContainerHandles,
		"destroying-containers": destroyingContainerHandles,
	})

	metric.CreatingContainersToBeGarbageCollected{
		Containers: len(creatingContainerHandles),
	}.Emit(logger)

	metric.CreatedContainersToBeGarbageCollected{
		Containers: len(createdContainerHandles),
	}.Emit(logger)

	metric.DestroyingContainersToBeGarbageCollected{
		Containers: len(destroyingContainerHandles),
	}.Emit(logger)

	for _, createdContainer := range createdContainers {
		// prevent closure from capturing last value of loop
		container := createdContainer

		c.jobRunner.Try(logger,
			container.WorkerName(),
			&job{
				JobName: container.Handle(),
				RunFunc: destroyCreatedContainer(logger, container),
			},
		)
	}

	for _, destroyingContainer := range destroyingContainers {
		// prevent closure from capturing last value of loop
		container := destroyingContainer

		c.jobRunner.Try(logger,
			container.WorkerName(),
			&job{
				JobName: container.Handle(),
				RunFunc: destroyDestroyingContainer(logger, container),
			},
		)
	}

	return nil
}

func destroyCreatedContainer(logger lager.Logger, container db.CreatedContainer) func(worker.Worker) {
	return func(workerClient worker.Worker) {
		var destroyingContainer db.DestroyingContainer
		var cLog lager.Logger

		if container.IsHijacked() {
			cLog = logger.Session("mark-hijacked-container", lager.Data{
				"container": container.Handle(),
				"worker":    workerClient.Name(),
			})

			var err error
			destroyingContainer, err = markHijackedContainerAsDestroying(cLog, container, workerClient.GardenClient())
			if err != nil {
				cLog.Error("failed-to-transition", err)
				return
			}
		} else {
			cLog = logger.Session("mark-created-as-destroying", lager.Data{
				"container": container.Handle(),
				"worker":    workerClient.Name(),
			})

			var err error
			destroyingContainer, err = container.Destroying()
			if err != nil {
				cLog.Error("failed-to-transition", err)
				return
			}
		}

		if destroyingContainer != nil {
			tryToDestroyContainer(cLog, destroyingContainer, workerClient)
		}
	}
}

func destroyDestroyingContainer(logger lager.Logger, container db.DestroyingContainer) func(worker.Worker) {
	return func(workerClient worker.Worker) {
		cLog := logger.Session("destroy-container", lager.Data{
			"container": container.Handle(),
			"worker":    workerClient.Name(),
		})

		tryToDestroyContainer(cLog, container, workerClient)
	}
}

func markHijackedContainerAsDestroying(
	logger lager.Logger,
	hijackedContainer db.CreatedContainer,
	gardenClient garden.Client,
) (db.DestroyingContainer, error) {
	gardenContainer, found, err := findContainer(gardenClient, hijackedContainer.Handle())
	if err != nil {
		logger.Error("failed-to-lookup-container-in-garden", err)
		return nil, err
	}

	if !found {
		logger.Debug("hijacked-container-not-found-in-garden")

		destroyingContainer, err := hijackedContainer.Destroying()
		if err != nil {
			logger.Error("failed-to-mark-container-as-destroying", err)
			return nil, err
		}

		return destroyingContainer, nil
	}

	err = gardenContainer.SetGraceTime(HijackedContainerTimeout)
	if err != nil {
		logger.Error("failed-to-set-grace-time-on-hijacked-container", err)
		return nil, err
	}

	_, err = hijackedContainer.Discontinue()
	if err != nil {
		logger.Error("failed-to-mark-container-as-destroying", err)
		return nil, err
	}

	return nil, nil
}

func tryToDestroyContainer(
	logger lager.Logger,
	container db.DestroyingContainer,
	workerClient worker.Worker,
) {
	logger.Debug("start")
	defer logger.Debug("done")

	gardenClient := workerClient.GardenClient()

	if container.IsDiscontinued() {
		logger.Debug("discontinued")

		_, found, err := findContainer(gardenClient, container.Handle())
		if err != nil {
			logger.Error("failed-to-lookup-container-in-garden", err)
			return
		}

		if found {
			logger.Debug("still-present-in-garden")
			return
		} else {
			logger.Debug("container-no-longer-present-in-garden")
		}
	} else {
		err := gardenClient.Destroy(container.Handle())
		if err != nil {
			if _, ok := err.(garden.ContainerNotFoundError); ok {
				logger.Debug("container-no-longer-present-in-garden")
			} else {
				logger.Error("failed-to-destroy-garden-container", err)
				return
			}
		}

		logger.Debug("destroyed-in-garden")

		metric.ContainersDeleted.Inc()
	}

	ok, err := container.Destroy()
	if err != nil {
		logger.Error("failed-to-destroy-database-container", err)
		return
	}

	if !ok {
		logger.Info("could-not-destroy-database-container")
		return
	}

	logger.Debug("destroyed-in-db")
}

func findContainer(gardenClient garden.Client, handle string) (garden.Container, bool, error) {
	gardenContainer, err := gardenClient.Lookup(handle)
	if err != nil {
		if _, ok := err.(garden.ContainerNotFoundError); ok {
			return nil, false, nil
		} else {
			return nil, false, err
		}
	}

	return gardenContainer, true, nil
}
<commit_msg>Revert "don't remove creating containers, but log them"<commit_after>package gc

import (
	"time"

	"code.cloudfoundry.org/garden"
	"code.cloudfoundry.org/lager"
	"github.com/concourse/atc/db"
	"github.com/concourse/atc/metric"
	"github.com/concourse/atc/worker"
)

const HijackedContainerTimeout = 5 * time.Minute

//go:generate counterfeiter . containerFactory

type containerFactory interface {
	FindContainersForDeletion() ([]db.CreatingContainer, []db.CreatedContainer, []db.DestroyingContainer, error)
}

type containerCollector struct {
	rootLogger       lager.Logger
	containerFactory containerFactory
	jobRunner        WorkerJobRunner
}

func NewContainerCollector(
	logger lager.Logger,
	containerFactory containerFactory,
	jobRunner WorkerJobRunner,
) Collector {
	return &containerCollector{
		rootLogger:       logger,
		containerFactory: containerFactory,
		jobRunner:        jobRunner,
	}
}

type job struct {
	JobName string
	RunFunc func(worker.Worker)
}

func (j *job) Name() string {
	return j.JobName
}

func (j *job) Run(w worker.Worker) {
	j.RunFunc(w)
}

func (c *containerCollector) Run() error {
	logger := c.rootLogger.Session("run")

	logger.Debug("start")
	defer logger.Debug("done")

	creatingContainers, createdContainers, destroyingContainers, err := c.containerFactory.FindContainersForDeletion()
	if err != nil {
		logger.Error("failed-to-get-containers-for-deletion", err)
		return err
	}

	creatingContainerHandles := []string{}
	createdContainerHandles := []string{}
	destroyingContainerHandles := []string{}

	if len(creatingContainers) > 0 {
		for _, container := range creatingContainers {
			creatingContainerHandles = append(creatingContainerHandles, container.Handle())
		}
	}

	if len(createdContainers) > 0 {
		for _, container := range createdContainers {
			createdContainerHandles = append(createdContainerHandles, container.Handle())
		}
	}

	if len(destroyingContainers) > 0 {
		for _, container := range destroyingContainers {
			destroyingContainerHandles = append(destroyingContainerHandles, container.Handle())
		}
	}

	logger.Debug("found-containers-for-deletion", lager.Data{
		"creating-containers":   creatingContainerHandles,
		"created-containers":    createdContainerHandles,
		"destroying-containers": destroyingContainerHandles,
	})

	metric.CreatingContainersToBeGarbageCollected{
		Containers: len(creatingContainerHandles),
	}.Emit(logger)

	metric.CreatedContainersToBeGarbageCollected{
		Containers: len(createdContainerHandles),
	}.Emit(logger)

	metric.DestroyingContainersToBeGarbageCollected{
		Containers: len(destroyingContainerHandles),
	}.Emit(logger)

	for _, creatingContainer := range creatingContainers {
		cLog := logger.Session("mark-creating-as-created", lager.Data{
			"container": creatingContainer.Handle(),
		})

		createdContainer, err := creatingContainer.Created()
		if err != nil {
			cLog.Error("failed-to-transition", err)
			continue
		}

		createdContainers = append(createdContainers, createdContainer)
	}

	for _, createdContainer := range createdContainers {
		// prevent closure from capturing last value of loop
		container := createdContainer

		c.jobRunner.Try(logger,
			container.WorkerName(),
			&job{
				JobName: container.Handle(),
				RunFunc: destroyCreatedContainer(logger, container),
			},
		)
	}

	for _, destroyingContainer := range destroyingContainers {
		// prevent closure from capturing last value of loop
		container := destroyingContainer

		c.jobRunner.Try(logger,
			container.WorkerName(),
			&job{
				JobName: container.Handle(),
				RunFunc: destroyDestroyingContainer(logger, container),
			},
		)
	}

	return nil
}

func destroyCreatedContainer(logger lager.Logger, container db.CreatedContainer) func(worker.Worker) {
	return func(workerClient worker.Worker) {
		var destroyingContainer db.DestroyingContainer
		var cLog lager.Logger

		if container.IsHijacked() {
			cLog = logger.Session("mark-hijacked-container", lager.Data{
				"container": container.Handle(),
				"worker":    workerClient.Name(),
			})

			var err error
			destroyingContainer, err = markHijackedContainerAsDestroying(cLog, container, workerClient.GardenClient())
			if err != nil {
				cLog.Error("failed-to-transition", err)
				return
			}
		} else {
			cLog = logger.Session("mark-created-as-destroying", lager.Data{
				"container": container.Handle(),
				"worker":    workerClient.Name(),
			})

			var err error
			destroyingContainer, err = container.Destroying()
			if err != nil {
				cLog.Error("failed-to-transition", err)
				return
			}
		}

		if destroyingContainer != nil {
			tryToDestroyContainer(cLog, destroyingContainer, workerClient)
		}
	}
}

func destroyDestroyingContainer(logger lager.Logger, container db.DestroyingContainer) func(worker.Worker) {
	return func(workerClient worker.Worker) {
		cLog := logger.Session("destroy-container", lager.Data{
			"container": container.Handle(),
			"worker":    workerClient.Name(),
		})

		tryToDestroyContainer(cLog, container, workerClient)
	}
}

func markHijackedContainerAsDestroying(
	logger lager.Logger,
	hijackedContainer db.CreatedContainer,
	gardenClient garden.Client,
) (db.DestroyingContainer, error) {
	gardenContainer, found, err := findContainer(gardenClient, hijackedContainer.Handle())
	if err != nil {
		logger.Error("failed-to-lookup-container-in-garden", err)
		return nil, err
	}

	if !found {
		logger.Debug("hijacked-container-not-found-in-garden")

		destroyingContainer, err := hijackedContainer.Destroying()
		if err != nil {
			logger.Error("failed-to-mark-container-as-destroying", err)
			return nil, err
		}

		return destroyingContainer, nil
	}

	err = gardenContainer.SetGraceTime(HijackedContainerTimeout)
	if err != nil {
		logger.Error("failed-to-set-grace-time-on-hijacked-container", err)
		return nil, err
	}

	_, err = hijackedContainer.Discontinue()
	if err != nil {
		logger.Error("failed-to-mark-container-as-destroying", err)
		return nil, err
	}

	return nil, nil
}

func tryToDestroyContainer(
	logger lager.Logger,
	container db.DestroyingContainer,
	workerClient worker.Worker,
) {
	logger.Debug("start")
	defer logger.Debug("done")

	gardenClient := workerClient.GardenClient()

	if container.IsDiscontinued() {
		logger.Debug("discontinued")

		_, found, err := findContainer(gardenClient, container.Handle())
		if err != nil {
			logger.Error("failed-to-lookup-container-in-garden", err)
			return
		}

		if found {
			logger.Debug("still-present-in-garden")
			return
		} else {
			logger.Debug("container-no-longer-present-in-garden")
		}
	} else {
		err := gardenClient.Destroy(container.Handle())
		if err != nil {
			if _, ok := err.(garden.ContainerNotFoundError); ok {
				logger.Debug("container-no-longer-present-in-garden")
			} else {
				logger.Error("failed-to-destroy-garden-container", err)
				return
			}
		}

		logger.Debug("destroyed-in-garden")

		metric.ContainersDeleted.Inc()
	}

	ok, err := container.Destroy()
	if err != nil {
		logger.Error("failed-to-destroy-database-container", err)
		return
	}

	if !ok {
		logger.Info("could-not-destroy-database-container")
		return
	}

	logger.Debug("destroyed-in-db")
}

func findContainer(gardenClient garden.Client, handle string) (garden.Container, bool, error) {
	gardenContainer, err := gardenClient.Lookup(handle)
	if err != nil {
		if _, ok := err.(garden.ContainerNotFoundError); ok {
			return nil, false, nil
		} else {
			return nil, false, err
		}
	}

	return gardenContainer, true, nil
}
<|endoftext|>


<commit_before>// Copyright 2012, 2013 Canonical Ltd.
// Licensed under the AGPLv3, see LICENCE file for details.

package tools

import (
	"fmt"

	"launchpad.net/loggo"

	"launchpad.net/juju-core/environs"
	"launchpad.net/juju-core/environs/simplestreams"
	"launchpad.net/juju-core/environs/storage"
	"launchpad.net/juju-core/errors"
	coretools "launchpad.net/juju-core/tools"
	"launchpad.net/juju-core/version"
)

var logger = loggo.GetLogger("juju.environs.tools")

// NewFindTools returns a List containing all tools with a given
// major.minor version number available at the data sources, filtered by filter.
// It is called NewFindTools because the legacy functionality is still present
// but deprecated. Once the legacy find tools is removed, this will be renamed.
// If minorVersion = -1, then only majorVersion is considered.
// At each URL, simplestreams metadata is used to search for the tools.
// If no *available* tools have the supplied major.minor version number, or match the
// supplied filter, the function returns a *NotFoundError.
func NewFindTools(sources []simplestreams.DataSource, cloudSpec simplestreams.CloudSpec,
	majorVersion, minorVersion int, filter coretools.Filter) (list coretools.List, err error) {

	toolsConstraint, err := makeToolsConstraint(cloudSpec, majorVersion, minorVersion, filter)
	if err != nil {
		return nil, err
	}
	toolsMetadata, err := Fetch(sources, simplestreams.DefaultIndexPath, toolsConstraint, false)
	if err != nil {
		return nil, err
	}
	if len(toolsMetadata) == 0 {
		return nil, coretools.ErrNoMatches
	}
	list = make(coretools.List, len(toolsMetadata))
	for i, metadata := range toolsMetadata {
		binary := version.Binary{
			Number: version.MustParse(metadata.Version),
			Arch:   metadata.Arch,
			Series: metadata.Release,
		}
		list[i] = &coretools.Tools{
			Version: binary,
			URL:     metadata.FullPath,
		}
	}
	if filter.Series != "" {
		if err := checkToolsSeries(list, filter.Series); err != nil {
			return nil, err
		}
	}
	return list, err
}

func makeToolsConstraint(cloudSpec simplestreams.CloudSpec, majorVersion, minorVersion int,
	filter coretools.Filter) (*ToolsConstraint, error) {

	var toolsConstraint *ToolsConstraint
	if filter.Number != version.Zero {
		// A specific tools version is required, however, a general match based on major/minor
		// version may also have been requested. This is used to ensure any agent version currently
		// recorded in the environment matches the Juju cli version.
		// We can short circuit any lookup here by checking the major/minor numbers against
		// the filter version and exiting early if there is a mismatch.
		majorMismatch := majorVersion > 0 && majorVersion != filter.Number.Major
		minorMismacth := minorVersion != -1 && minorVersion != filter.Number.Minor
		if majorMismatch || minorMismacth {
			return nil, coretools.ErrNoMatches
		}
		toolsConstraint = NewVersionedToolsConstraint(filter.Number.String(),
			simplestreams.LookupParams{CloudSpec: cloudSpec})
	} else {
		toolsConstraint = NewGeneralToolsConstraint(majorVersion, minorVersion, filter.Released,
			simplestreams.LookupParams{CloudSpec: cloudSpec})
	}
	if filter.Arch != "" {
		toolsConstraint.Arches = []string{filter.Arch}
	} else {
		logger.Debugf("no architecture specified when finding tools, looking for any")
		toolsConstraint.Arches = []string{"amd64", "i386", "arm"}
	}
	// The old tools search allowed finding tools without needing to specify a series.
	// The simplestreams metadata is keyed off series, so series must be specified in
	// the search constraint. If no series is specified, we gather all the series from
	// lucid onwards and add those to the constraint.
	var seriesToSearch []string
	if filter.Series != "" {
		seriesToSearch = []string{filter.Series}
	} else {
		logger.Debugf("no series specified when finding tools, looking for any")
		seriesToSearch = simplestreams.SupportedSeries()
	}
	toolsConstraint.Series = seriesToSearch
	return toolsConstraint, nil
}

// UseLegacyFallback is true is we try loading the tools from the env storage if the
// new lookup using simplestreams fails.
// Tests can turn off this feature.
var UseLegacyFallback = true

// Define some boolean parameter values.
const DoNotAllowRetry = false

// FindTools returns a List containing all tools with a given
// major.minor version number available in the cloud instance, filtered by filter.
// If minorVersion = -1, then only majorVersion is considered.
// If no *available* tools have the supplied major.minor version number, or match the
// supplied filter, the function returns a *NotFoundError.
func FindTools(cloudInst environs.ConfigGetter, majorVersion, minorVersion int,
	filter coretools.Filter, allowRetry bool) (list coretools.List, err error) {

	var cloudSpec simplestreams.CloudSpec
	if inst, ok := cloudInst.(simplestreams.HasRegion); ok {
		if cloudSpec, err = inst.Region(); err != nil {
			return nil, err
		}
	}
	// If only one of region or endpoint is provided, that is a problem.
	if cloudSpec.Region != cloudSpec.Endpoint && (cloudSpec.Region == "" || cloudSpec.Endpoint == "") {
		return nil, fmt.Errorf("cannot find tools without a complete cloud configuration")
	}

	if minorVersion >= 0 {
		logger.Infof("reading tools with major.minor version %d.%d", majorVersion, minorVersion)
	} else {
		logger.Infof("reading tools with major version %d", majorVersion)
	}
	defer convertToolsError(&err)
	// Construct a tools filter.
	// Discard all that are known to be irrelevant.
	if filter.Number != version.Zero {
		logger.Infof("filtering tools by version: %s", filter.Number)
	}
	if filter.Series != "" {
		logger.Infof("filtering tools by series: %s", filter.Series)
	}
	if filter.Arch != "" {
		logger.Infof("filtering tools by architecture: %s", filter.Arch)
	}
	sources, err := GetMetadataSourcesWithRetries(cloudInst, allowRetry)
	if err != nil {
		return nil, err
	}
	list, err = NewFindTools(sources, cloudSpec, majorVersion, minorVersion, filter)
	if UseLegacyFallback && (err != nil || len(list) == 0) {
		logger.Warningf("no tools found using simplestreams metadata, using legacy fallback")
		if env, ok := cloudInst.(environs.Environ); ok {
			list, err = LegacyFindTools(
				[]storage.StorageReader{env.Storage(), env.PublicStorage()}, majorVersion, minorVersion, filter)
		} else {
			return nil, fmt.Errorf("cannot find legacy tools without an environment")
		}
	}
	return list, err
}

// The following allows FindTools, as called by FindBootstrapTools, to be patched for testing.
var bootstrapFindTools = FindTools

type findtoolsfunc func(environs.ConfigGetter, int, int, coretools.Filter, bool) (coretools.List, error)

func TestingPatchBootstrapFindTools(stub findtoolsfunc) func() {
	origFunc := bootstrapFindTools
	bootstrapFindTools = stub
	return func() {
		bootstrapFindTools = origFunc
	}
}

// BootstrapToolsParams contains parameters for FindBootstrapTools
type BootstrapToolsParams struct {
	Version    *version.Number
	Arch       *string
	AllowRetry bool
}

// FindBootstrapTools returns a ToolsList containing only those tools with
// which it would be reasonable to launch an environment's first machine, given the supplied constraints.
// If a specific agent version is not requested, all tools matching the current major.minor version are chosen.
func FindBootstrapTools(cloudInst environs.ConfigGetter, params BootstrapToolsParams) (list coretools.List, err error) {
	// Construct a tools filter.
	cfg := cloudInst.Config()
	cliVersion := version.Current.Number
	filter := coretools.Filter{
		Series: cfg.DefaultSeries(),
		Arch:   stringOrEmpty(params.Arch),
	}
	if params.Version != nil {
		// If we already have an explicit agent version set, we're done.
		filter.Number = *params.Version
		return bootstrapFindTools(cloudInst, cliVersion.Major, cliVersion.Minor, filter, params.AllowRetry)
	}
	if dev := cliVersion.IsDev() || cfg.Development(); !dev {
		logger.Infof("filtering tools by released version")
		filter.Released = true
	}
	return bootstrapFindTools(cloudInst, cliVersion.Major, cliVersion.Minor, filter, params.AllowRetry)
}

func stringOrEmpty(pstr *string) string {
	if pstr == nil {
		return ""
	}
	return *pstr
}

// FindInstanceTools returns a ToolsList containing only those tools with which
// it would be reasonable to start a new instance, given the supplied series and arch.
func FindInstanceTools(cloudInst environs.ConfigGetter,
	vers version.Number, series string, arch *string) (list coretools.List, err error) {

	// Construct a tools filter.
	// Discard all that are known to be irrelevant.
	filter := coretools.Filter{
		Number: vers,
		Series: series,
		Arch:   stringOrEmpty(arch),
	}
	return FindTools(cloudInst, vers.Major, vers.Minor, filter, DoNotAllowRetry)
}

// FindExactTools returns only the tools that match the supplied version.
func FindExactTools(cloudInst environs.ConfigGetter,
	vers version.Number, series string, arch string) (t *coretools.Tools, err error) {

	logger.Infof("finding exact version %s", vers)
	// Construct a tools filter.
	// Discard all that are known to be irrelevant.
	filter := coretools.Filter{
		Number: vers,
		Series: series,
		Arch:   arch,
	}
	availableTools, err := FindTools(cloudInst, vers.Major, vers.Minor, filter, DoNotAllowRetry)
	if err != nil {
		return nil, err
	}
	if len(availableTools) != 1 {
		return nil, fmt.Errorf("expected one tools, got %d tools", len(availableTools))
	}
	return availableTools[0], nil
}

// CheckToolsSeries verifies that all the given possible tools are for the
// given OS series.
func checkToolsSeries(toolsList coretools.List, series string) error {
	toolsSeries := toolsList.AllSeries()
	if len(toolsSeries) != 1 {
		return fmt.Errorf("expected single series, got %v", toolsSeries)
	}
	if toolsSeries[0] != series {
		return fmt.Errorf("tools mismatch: expected series %v, got %v", series, toolsSeries[0])
	}
	return nil
}

func isToolsError(err error) bool {
	switch err {
	case ErrNoTools, coretools.ErrNoMatches:
		return true
	}
	return false
}

func convertToolsError(err *error) {
	if isToolsError(*err) {
		*err = errors.NewNotFoundError(*err, "")
	}
}
<commit_msg>Record checksum for new find tools<commit_after>// Copyright 2012, 2013 Canonical Ltd.
// Licensed under the AGPLv3, see LICENCE file for details.

package tools

import (
	"fmt"

	"launchpad.net/loggo"

	"launchpad.net/juju-core/environs"
	"launchpad.net/juju-core/environs/simplestreams"
	"launchpad.net/juju-core/environs/storage"
	"launchpad.net/juju-core/errors"
	coretools "launchpad.net/juju-core/tools"
	"launchpad.net/juju-core/version"
)

var logger = loggo.GetLogger("juju.environs.tools")

// NewFindTools returns a List containing all tools with a given
// major.minor version number available at the data sources, filtered by filter.
// It is called NewFindTools because the legacy functionality is still present
// but deprecated. Once the legacy find tools is removed, this will be renamed.
// If minorVersion = -1, then only majorVersion is considered.
// At each URL, simplestreams metadata is used to search for the tools.
// If no *available* tools have the supplied major.minor version number, or match the
// supplied filter, the function returns a *NotFoundError.
func NewFindTools(sources []simplestreams.DataSource, cloudSpec simplestreams.CloudSpec,
	majorVersion, minorVersion int, filter coretools.Filter) (list coretools.List, err error) {

	toolsConstraint, err := makeToolsConstraint(cloudSpec, majorVersion, minorVersion, filter)
	if err != nil {
		return nil, err
	}
	toolsMetadata, err := Fetch(sources, simplestreams.DefaultIndexPath, toolsConstraint, false)
	if err != nil {
		return nil, err
	}
	if len(toolsMetadata) == 0 {
		return nil, coretools.ErrNoMatches
	}
	list = make(coretools.List, len(toolsMetadata))
	for i, metadata := range toolsMetadata {
		binary := version.Binary{
			Number: version.MustParse(metadata.Version),
			Arch:   metadata.Arch,
			Series: metadata.Release,
		}
		list[i] = &coretools.Tools{
			Version: binary,
			URL:     metadata.FullPath,
			Size:    metadata.Size,
			SHA256:  metadata.SHA256,
		}
	}
	if filter.Series != "" {
		if err := checkToolsSeries(list, filter.Series); err != nil {
			return nil, err
		}
	}
	return list, err
}

func makeToolsConstraint(cloudSpec simplestreams.CloudSpec, majorVersion, minorVersion int,
	filter coretools.Filter) (*ToolsConstraint, error) {

	var toolsConstraint *ToolsConstraint
	if filter.Number != version.Zero {
		// A specific tools version is required, however, a general match based on major/minor
		// version may also have been requested. This is used to ensure any agent version currently
		// recorded in the environment matches the Juju cli version.
		// We can short circuit any lookup here by checking the major/minor numbers against
		// the filter version and exiting early if there is a mismatch.
		majorMismatch := majorVersion > 0 && majorVersion != filter.Number.Major
		minorMismacth := minorVersion != -1 && minorVersion != filter.Number.Minor
		if majorMismatch || minorMismacth {
			return nil, coretools.ErrNoMatches
		}
		toolsConstraint = NewVersionedToolsConstraint(filter.Number.String(),
			simplestreams.LookupParams{CloudSpec: cloudSpec})
	} else {
		toolsConstraint = NewGeneralToolsConstraint(majorVersion, minorVersion, filter.Released,
			simplestreams.LookupParams{CloudSpec: cloudSpec})
	}
	if filter.Arch != "" {
		toolsConstraint.Arches = []string{filter.Arch}
	} else {
		logger.Debugf("no architecture specified when finding tools, looking for any")
		toolsConstraint.Arches = []string{"amd64", "i386", "arm"}
	}
	// The old tools search allowed finding tools without needing to specify a series.
	// The simplestreams metadata is keyed off series, so series must be specified in
	// the search constraint. If no series is specified, we gather all the series from
	// lucid onwards and add those to the constraint.
	var seriesToSearch []string
	if filter.Series != "" {
		seriesToSearch = []string{filter.Series}
	} else {
		logger.Debugf("no series specified when finding tools, looking for any")
		seriesToSearch = simplestreams.SupportedSeries()
	}
	toolsConstraint.Series = seriesToSearch
	return toolsConstraint, nil
}

// UseLegacyFallback is true is we try loading the tools from the env storage if the
// new lookup using simplestreams fails.
// Tests can turn off this feature.
var UseLegacyFallback = true

// Define some boolean parameter values.
const DoNotAllowRetry = false

// FindTools returns a List containing all tools with a given
// major.minor version number available in the cloud instance, filtered by filter.
// If minorVersion = -1, then only majorVersion is considered.
// If no *available* tools have the supplied major.minor version number, or match the
// supplied filter, the function returns a *NotFoundError.
func FindTools(cloudInst environs.ConfigGetter, majorVersion, minorVersion int,
	filter coretools.Filter, allowRetry bool) (list coretools.List, err error) {

	var cloudSpec simplestreams.CloudSpec
	if inst, ok := cloudInst.(simplestreams.HasRegion); ok {
		if cloudSpec, err = inst.Region(); err != nil {
			return nil, err
		}
	}
	// If only one of region or endpoint is provided, that is a problem.
	if cloudSpec.Region != cloudSpec.Endpoint && (cloudSpec.Region == "" || cloudSpec.Endpoint == "") {
		return nil, fmt.Errorf("cannot find tools without a complete cloud configuration")
	}

	if minorVersion >= 0 {
		logger.Infof("reading tools with major.minor version %d.%d", majorVersion, minorVersion)
	} else {
		logger.Infof("reading tools with major version %d", majorVersion)
	}
	defer convertToolsError(&err)
	// Construct a tools filter.
	// Discard all that are known to be irrelevant.
	if filter.Number != version.Zero {
		logger.Infof("filtering tools by version: %s", filter.Number)
	}
	if filter.Series != "" {
		logger.Infof("filtering tools by series: %s", filter.Series)
	}
	if filter.Arch != "" {
		logger.Infof("filtering tools by architecture: %s", filter.Arch)
	}
	sources, err := GetMetadataSourcesWithRetries(cloudInst, allowRetry)
	if err != nil {
		return nil, err
	}
	list, err = NewFindTools(sources, cloudSpec, majorVersion, minorVersion, filter)
	if UseLegacyFallback && (err != nil || len(list) == 0) {
		logger.Warningf("no tools found using simplestreams metadata, using legacy fallback")
		if env, ok := cloudInst.(environs.Environ); ok {
			list, err = LegacyFindTools(
				[]storage.StorageReader{env.Storage(), env.PublicStorage()}, majorVersion, minorVersion, filter)
		} else {
			return nil, fmt.Errorf("cannot find legacy tools without an environment")
		}
	}
	return list, err
}

// The following allows FindTools, as called by FindBootstrapTools, to be patched for testing.
var bootstrapFindTools = FindTools

type findtoolsfunc func(environs.ConfigGetter, int, int, coretools.Filter, bool) (coretools.List, error)

func TestingPatchBootstrapFindTools(stub findtoolsfunc) func() {
	origFunc := bootstrapFindTools
	bootstrapFindTools = stub
	return func() {
		bootstrapFindTools = origFunc
	}
}

// BootstrapToolsParams contains parameters for FindBootstrapTools
type BootstrapToolsParams struct {
	Version    *version.Number
	Arch       *string
	AllowRetry bool
}

// FindBootstrapTools returns a ToolsList containing only those tools with
// which it would be reasonable to launch an environment's first machine, given the supplied constraints.
// If a specific agent version is not requested, all tools matching the current major.minor version are chosen.
func FindBootstrapTools(cloudInst environs.ConfigGetter, params BootstrapToolsParams) (list coretools.List, err error) {
	// Construct a tools filter.
	cfg := cloudInst.Config()
	cliVersion := version.Current.Number
	filter := coretools.Filter{
		Series: cfg.DefaultSeries(),
		Arch:   stringOrEmpty(params.Arch),
	}
	if params.Version != nil {
		// If we already have an explicit agent version set, we're done.
		filter.Number = *params.Version
		return bootstrapFindTools(cloudInst, cliVersion.Major, cliVersion.Minor, filter, params.AllowRetry)
	}
	if dev := cliVersion.IsDev() || cfg.Development(); !dev {
		logger.Infof("filtering tools by released version")
		filter.Released = true
	}
	return bootstrapFindTools(cloudInst, cliVersion.Major, cliVersion.Minor, filter, params.AllowRetry)
}

func stringOrEmpty(pstr *string) string {
	if pstr == nil {
		return ""
	}
	return *pstr
}

// FindInstanceTools returns a ToolsList containing only those tools with which
// it would be reasonable to start a new instance, given the supplied series and arch.
func FindInstanceTools(cloudInst environs.ConfigGetter,
	vers version.Number, series string, arch *string) (list coretools.List, err error) {

	// Construct a tools filter.
	// Discard all that are known to be irrelevant.
	filter := coretools.Filter{
		Number: vers,
		Series: series,
		Arch:   stringOrEmpty(arch),
	}
	return FindTools(cloudInst, vers.Major, vers.Minor, filter, DoNotAllowRetry)
}

// FindExactTools returns only the tools that match the supplied version.
func FindExactTools(cloudInst environs.ConfigGetter,
	vers version.Number, series string, arch string) (t *coretools.Tools, err error) {

	logger.Infof("finding exact version %s", vers)
	// Construct a tools filter.
	// Discard all that are known to be irrelevant.
	filter := coretools.Filter{
		Number: vers,
		Series: series,
		Arch:   arch,
	}
	availableTools, err := FindTools(cloudInst, vers.Major, vers.Minor, filter, DoNotAllowRetry)
	if err != nil {
		return nil, err
	}
	if len(availableTools) != 1 {
		return nil, fmt.Errorf("expected one tools, got %d tools", len(availableTools))
	}
	return availableTools[0], nil
}

// CheckToolsSeries verifies that all the given possible tools are for the
// given OS series.
func checkToolsSeries(toolsList coretools.List, series string) error {
	toolsSeries := toolsList.AllSeries()
	if len(toolsSeries) != 1 {
		return fmt.Errorf("expected single series, got %v", toolsSeries)
	}
	if toolsSeries[0] != series {
		return fmt.Errorf("tools mismatch: expected series %v, got %v", series, toolsSeries[0])
	}
	return nil
}

func isToolsError(err error) bool {
	switch err {
	case ErrNoTools, coretools.ErrNoMatches:
		return true
	}
	return false
}

func convertToolsError(err *error) {
	if isToolsError(*err) {
		*err = errors.NewNotFoundError(*err, "")
	}
}
<|endoftext|>


<commit_before>// Copyright 2017 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

package cpu

const CacheLineSize = 64
<commit_msg>internal/cpu: fix arm64 build<commit_after>// Copyright 2017 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

package cpu

const CacheLineSize = 64

// TODO: delete this once https://go-review.googlesource.com/c/go/+/76490 lands.
// These will just be false for now.
var ARM64 struct {
	HasSHA1 bool
	HasSHA2 bool
}
<|endoftext|>


<commit_before>package amd64

import (
	"reflect"
	"unsafe"
)

func (a *Assembler) CallFunc(f interface{}) {
	if reflect.TypeOf(f).Kind() != reflect.Func {
		panic("CallFunc: Can't call non-func")
	}
	ival := *(*struct {
		typ uintptr
		fun uintptr
	})(unsafe.Pointer(&f))

	// runtime·cgocallback_gofunc(f, frame, framesize)
	// plan9 ABI

	framesize := calculateFramesize(f)

	a.Sub(Imm{24}, Rsp)
	a.Mov(Imm{int32(framesize)}, Indirect{Rsp, 16, 64})
	a.Lea(Indirect{Rsp, 24, 64}, Rax)
	a.Mov(Rax, Indirect{Rsp, 8, 64})
	a.MovAbs(uint64(ival.fun), Rax)
	a.Mov(Rax, Indirect{Rsp, 0, 64})
	a.MovAbs(uint64(get_runtime_cgocallback_gofunc()), Rax)
	a.Call(Rax)
	a.Add(Imm{24}, Rsp)
}

func calculateFramesize(f interface{}) uintptr {
	t := reflect.TypeOf(f)
	s := uintptr(0)

	for i := 0; i < t.NumIn(); i++ {
		s += t.In(i).Size()
	}
	for i := 0; i < t.NumOut(); i++ {
		s += t.Out(i).Size()
	}
	return s
}

func get_runtime_cgocallback_gofunc() uintptr
<commit_msg>Document CallFunc.<commit_after>package amd64

import (
	"reflect"
	"unsafe"
)

// CallFunc assembles a sequence to call into the go function
// 'f'. Prior to CallFunc, a 6c/6g-layout stack frame should be set up
// containing arguments and return slots for f. The call will be
// effected by way of runtime.cgocallback_gofunc, involving a stack
// switch back to the goroutine stack.
//
// All registers are caller-save in the 6c ABI, and so all registers
// should be assumed clobbered across a CallFunc.
func (a *Assembler) CallFunc(f interface{}) {
	if reflect.TypeOf(f).Kind() != reflect.Func {
		panic("CallFunc: Can't call non-func")
	}
	ival := *(*struct {
		typ uintptr
		fun uintptr
	})(unsafe.Pointer(&f))

	// runtime·cgocallback_gofunc(f, frame, framesize)
	// plan9 ABI

	framesize := calculateFramesize(f)

	a.Sub(Imm{24}, Rsp)
	a.Mov(Imm{int32(framesize)}, Indirect{Rsp, 16, 64})
	a.Lea(Indirect{Rsp, 24, 64}, Rax)
	a.Mov(Rax, Indirect{Rsp, 8, 64})
	a.MovAbs(uint64(ival.fun), Rax)
	a.Mov(Rax, Indirect{Rsp, 0, 64})
	a.MovAbs(uint64(get_runtime_cgocallback_gofunc()), Rax)
	a.Call(Rax)
	a.Add(Imm{24}, Rsp)
}

func calculateFramesize(f interface{}) uintptr {
	t := reflect.TypeOf(f)
	s := uintptr(0)

	for i := 0; i < t.NumIn(); i++ {
		s += t.In(i).Size()
	}
	for i := 0; i < t.NumOut(); i++ {
		s += t.Out(i).Size()
	}
	return s
}

func get_runtime_cgocallback_gofunc() uintptr
<|endoftext|>