<commit_before>// Copyright The OpenTelemetry Authors
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//       http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

// Package tests contains test cases. To run the tests go to tests directory and run:
// RUN_TESTBED=1 go test -v

package tests

import (
	"fmt"
	"path/filepath"
	"testing"
	"time"

	"github.com/stretchr/testify/assert"
	"github.com/stretchr/testify/require"

	"github.com/open-telemetry/opentelemetry-collector-contrib/testbed/testbed"
)

func TestIdleMode(t *testing.T) {

	options := testbed.LoadOptions{DataItemsPerSecond: 10_000, ItemsPerBatch: 10}
	dataProvider := testbed.NewPerfTestDataProvider(options)

	resultDir, err := filepath.Abs(filepath.Join("results", t.Name()))
	require.NoError(t, err)

	sender := testbed.NewOTLPTraceDataSender(testbed.DefaultHost, testbed.GetAvailablePort(t))
	receiver := testbed.NewOTLPDataReceiver(testbed.GetAvailablePort(t))
	cfg := createConfigYaml(t, sender, receiver, resultDir, nil, nil)
	cp := testbed.NewChildProcessCollector()
	cleanup, err := cp.PrepareConfig(cfg)
	require.NoError(t, err)
	t.Cleanup(cleanup)

	tc := testbed.NewTestCase(
		t,
		dataProvider,
		sender,
		receiver,
		cp,
		&testbed.PerfTestValidator{},
		performanceResultsSummary,
		testbed.WithResourceLimits(testbed.ResourceSpec{ExpectedMaxCPU: 20, ExpectedMaxRAM: 83}),
	)
	tc.StartAgent()

	tc.Sleep(tc.Duration)

	tc.Stop()
}

const ballastConfig = `
  memory_ballast:
    size_mib: %d
`

func TestBallastMemory(t *testing.T) {
	tests := []struct {
		ballastSize uint32
		maxRSS      uint32
	}{
		{100, 80},
		{500, 110},
		{1000, 120},
	}

	resultDir, err := filepath.Abs(filepath.Join("results", t.Name()))
	require.NoError(t, err)

	options := testbed.LoadOptions{DataItemsPerSecond: 10_000, ItemsPerBatch: 10}
	dataProvider := testbed.NewPerfTestDataProvider(options)
	for _, test := range tests {
		t.Run(fmt.Sprintf("ballast-size-%d", test.ballastSize), func(t *testing.T) {
			sender := testbed.NewOTLPTraceDataSender(testbed.DefaultHost, testbed.GetAvailablePort(t))
			receiver := testbed.NewOTLPDataReceiver(testbed.GetAvailablePort(t))
			ballastCfg := createConfigYaml(
				t, sender, receiver, resultDir, nil,
				map[string]string{"memory_ballast": fmt.Sprintf(ballastConfig, test.ballastSize)})
			cp := testbed.NewChildProcessCollector()
			cleanup, err := cp.PrepareConfig(ballastCfg)
			require.NoError(t, err)
			tc := testbed.NewTestCase(
				t,
				dataProvider,
				sender,
				receiver,
				cp,
				&testbed.PerfTestValidator{},
				performanceResultsSummary,
				testbed.WithSkipResults(),
				testbed.WithResourceLimits(testbed.ResourceSpec{ExpectedMaxRAM: test.maxRSS}),
			)
			tc.StartAgent()

			var rss, vms uint32
			// It is possible that the process is not ready or the ballast code path
			// is not hit immediately so we give the process up to a couple of seconds
			// to fire up and setup ballast. 2 seconds is a long time for this case but
			// it is short enough to not be annoying if the test fails repeatedly
			tc.WaitForN(func() bool {
				rss, vms, _ = tc.AgentMemoryInfo()
				return vms > test.ballastSize
			}, time.Second*2, fmt.Sprintf("VMS must be greater than %d", test.ballastSize))

			// https://github.com/open-telemetry/opentelemetry-collector/issues/3233
			// given that the maxRSS isn't an absolute maximum and that the actual maximum might be a bit off,
			// we give some room here instead of failing when the memory usage isn't that much higher than the max
			lenientMax := 1.1 * float32(test.maxRSS)

			// https://github.com/open-telemetry/opentelemetry-collector-contrib/issues/6927#issuecomment-1138624098
			// During garbage collection, we may observe the ballast in rss.
			// If this happens, allow a brief window for garbage collection to complete.
			garbageCollectionMax := lenientMax + float32(test.ballastSize)

			rssTooHigh := fmt.Sprintf("The RSS memory usage (%d) is >10%% higher than the limit (%d).", rss, test.maxRSS)

			if rss > test.ballastSize && float32(rss) <= garbageCollectionMax {
				t.Log("Possible garbage collection under way. Remeasuring RSS.")
				tc.WaitForN(func() bool {
					rss, vms, _ = tc.AgentMemoryInfo()
					return float32(rss) <= lenientMax
				}, time.Second, rssTooHigh)
			} else {
				assert.LessOrEqual(t, float32(rss), lenientMax, rssTooHigh)
			}

			cleanup()
			tc.Stop()
		})
	}
}
<commit_msg>[testbed] Wait longer when possible garbage collection is observed (#10601)<commit_after>// Copyright The OpenTelemetry Authors
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//       http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

// Package tests contains test cases. To run the tests go to tests directory and run:
// RUN_TESTBED=1 go test -v

package tests

import (
	"fmt"
	"path/filepath"
	"testing"
	"time"

	"github.com/stretchr/testify/assert"
	"github.com/stretchr/testify/require"

	"github.com/open-telemetry/opentelemetry-collector-contrib/testbed/testbed"
)

func TestIdleMode(t *testing.T) {

	options := testbed.LoadOptions{DataItemsPerSecond: 10_000, ItemsPerBatch: 10}
	dataProvider := testbed.NewPerfTestDataProvider(options)

	resultDir, err := filepath.Abs(filepath.Join("results", t.Name()))
	require.NoError(t, err)

	sender := testbed.NewOTLPTraceDataSender(testbed.DefaultHost, testbed.GetAvailablePort(t))
	receiver := testbed.NewOTLPDataReceiver(testbed.GetAvailablePort(t))
	cfg := createConfigYaml(t, sender, receiver, resultDir, nil, nil)
	cp := testbed.NewChildProcessCollector()
	cleanup, err := cp.PrepareConfig(cfg)
	require.NoError(t, err)
	t.Cleanup(cleanup)

	tc := testbed.NewTestCase(
		t,
		dataProvider,
		sender,
		receiver,
		cp,
		&testbed.PerfTestValidator{},
		performanceResultsSummary,
		testbed.WithResourceLimits(testbed.ResourceSpec{ExpectedMaxCPU: 20, ExpectedMaxRAM: 83}),
	)
	tc.StartAgent()

	tc.Sleep(tc.Duration)

	tc.Stop()
}

const ballastConfig = `
  memory_ballast:
    size_mib: %d
`

func TestBallastMemory(t *testing.T) {
	tests := []struct {
		ballastSize uint32
		maxRSS      uint32
	}{
		{100, 80},
		{500, 110},
		{1000, 120},
	}

	resultDir, err := filepath.Abs(filepath.Join("results", t.Name()))
	require.NoError(t, err)

	options := testbed.LoadOptions{DataItemsPerSecond: 10_000, ItemsPerBatch: 10}
	dataProvider := testbed.NewPerfTestDataProvider(options)
	for _, test := range tests {
		t.Run(fmt.Sprintf("ballast-size-%d", test.ballastSize), func(t *testing.T) {
			sender := testbed.NewOTLPTraceDataSender(testbed.DefaultHost, testbed.GetAvailablePort(t))
			receiver := testbed.NewOTLPDataReceiver(testbed.GetAvailablePort(t))
			ballastCfg := createConfigYaml(
				t, sender, receiver, resultDir, nil,
				map[string]string{"memory_ballast": fmt.Sprintf(ballastConfig, test.ballastSize)})
			cp := testbed.NewChildProcessCollector()
			cleanup, err := cp.PrepareConfig(ballastCfg)
			require.NoError(t, err)
			tc := testbed.NewTestCase(
				t,
				dataProvider,
				sender,
				receiver,
				cp,
				&testbed.PerfTestValidator{},
				performanceResultsSummary,
				testbed.WithSkipResults(),
				testbed.WithResourceLimits(testbed.ResourceSpec{ExpectedMaxRAM: test.maxRSS}),
			)
			tc.StartAgent()

			var rss, vms uint32
			// It is possible that the process is not ready or the ballast code path
			// is not hit immediately so we give the process up to a couple of seconds
			// to fire up and setup ballast. 2 seconds is a long time for this case but
			// it is short enough to not be annoying if the test fails repeatedly
			tc.WaitForN(func() bool {
				rss, vms, _ = tc.AgentMemoryInfo()
				return vms > test.ballastSize
			}, time.Second*2, fmt.Sprintf("VMS must be greater than %d", test.ballastSize))

			// https://github.com/open-telemetry/opentelemetry-collector/issues/3233
			// given that the maxRSS isn't an absolute maximum and that the actual maximum might be a bit off,
			// we give some room here instead of failing when the memory usage isn't that much higher than the max
			lenientMax := 1.1 * float32(test.maxRSS)

			// https://github.com/open-telemetry/opentelemetry-collector-contrib/issues/6927#issuecomment-1138624098
			// During garbage collection, we may observe the ballast in rss.
			// If this happens, allow a brief window for garbage collection to complete.
			garbageCollectionMax := lenientMax + float32(test.ballastSize)

			rssTooHigh := fmt.Sprintf("The RSS memory usage (%d) is >10%% higher than the limit (%d).", rss, test.maxRSS)

			if rss > test.ballastSize && float32(rss) <= garbageCollectionMax {
				t.Log("Possible garbage collection under way. Remeasuring RSS.")
				tc.WaitForN(func() bool {
					rss, vms, _ = tc.AgentMemoryInfo()
					return float32(rss) <= lenientMax
				}, time.Second*5, rssTooHigh)
			} else {
				assert.LessOrEqual(t, float32(rss), lenientMax, rssTooHigh)
			}

			cleanup()
			tc.Stop()
		})
	}
}
<|endoftext|>


<commit_before>package sslLabsClientSupport

import (
	"encoding/json"
	"fmt"
	"net/http"
	"strconv"
	"strings"

	"github.com/mozilla/tls-observatory/connection"
	"github.com/mozilla/tls-observatory/constants"
	"github.com/mozilla/tls-observatory/logger"
	"github.com/mozilla/tls-observatory/worker"
)

var (
	workerName           = "sslLabsClientSupport"
	workerDesc           = "Determines client compatibility with a given target based on server certificate and ciphersuites."
	sslLabsClientDataURL = "https://api.ssllabs.com/api/v3/getClients"
	log                  = logger.GetLogger()
)

func init() {
	runner := new(slabscrunner)
	cs, err := getConffromURL(sslLabsClientDataURL)
	if err != nil {
		log.Printf("Failed to initialize %s: %v", workerName, err)
		return
	}
	runner.Clients = cs
	worker.RegisterWorker(workerName, worker.Info{Runner: runner, Description: workerDesc})
}

// getClientDataFrom retrieves the json containing the sslLabs client data
func getConffromURL(url string) (cs []Client, err error) {
	r, err := http.Get(url)
	if err != nil {
		return
	}
	defer r.Body.Close()
	err = json.NewDecoder(r.Body).Decode(&cs)
	return
}

type slabscrunner struct {
	Clients []Client
}

// Client is a definition of a TLS client with all the parameters it supports
type Client struct {
	ID                       int      `json:"id"`
	Name                     string   `json:"name"`
	Version                  string   `json:"version"`
	HandshakeFormat          string   `json:"handshakeFormat"`
	LowestProtocol           int      `json:"lowestProtocol"`
	HighestProtocol          int      `json:"highestProtocol"`
	UserAgent                string   `json:"userAgent"`
	IsGrade0                 bool     `json:"isGrade0"`
	MaxDhBits                int      `json:"maxDhBits"`
	AbortsOnUnrecognizedName bool     `json:"abortsOnUnrecognizedName"`
	MaxRsaBits               int      `json:"maxRsaBits"`
	MinDhBits                int      `json:"minDhBits"`
	RequiresSha2             bool     `json:"requiresSha2"`
	MinRsaBits               int      `json:"minRsaBits"`
	MinEcdsaBits             int      `json:"minEcdsaBits"`
	SuiteIds                 []int    `json:"suiteIds"`
	SuiteNames               []string `json:"suiteNames"`
	SupportsSni              bool     `json:"supportsSni"`
	SupportsCompression      bool     `json:"supportsCompression"`
	SupportsStapling         bool     `json:"supportsStapling"`
	SupportsTickets          bool     `json:"supportsTickets"`
	SupportsRi               bool     `json:"supportsRi"`
	SignatureAlgorithms      []int    `json:"signatureAlgorithms"`
	EllipticCurves           []int    `json:"ellipticCurves"`
	SupportsNpn              bool     `json:"supportsNpn"`
	NpnProtocols             []string `json:"npnProtocols"`
	AlpnProtocols            []string `json:"alpnProtocols"`
}

// CipherSuite represent a ciphersuite generated and recognised by OpenSSL
type CipherSuite struct {
	IANAName   string     `json:"iana_name"`
	GnuTLSName string     `json:"gnutls_name"`
	NSSName    string     `json:"nss_name"`
	Proto      string     `json:"proto"`
	Kx         string     `json:"kx"`
	Au         string     `json:"au"`
	Enc        Encryption `json:"encryption"`
	Mac        string     `json:"mac"`
	Code       uint64     `json:"code"`
}

//Encryption represents the encryption aspects of a Ciphersuite
type Encryption struct {
	Cipher string `json:"cipher"`
	Bits   int    `json:"key"`
}

type ClientSupport struct {
	IsSupported bool   `json:"is_supported"`
	Ciphersuite string `json:"ciphersuite,omitempty"`
	Code        int    `json:"code,omitempty"`
	Curve       string `json:"curve,omitempty"`
	Protocol    string `json:"protocol,omitempty"`
}

func (w slabscrunner) Run(in worker.Input, res chan worker.Result) {
	ClientsSupport := make(map[string]ClientSupport)
	// Loop over every client defined in the sslLabs document and check if they can
	// negotiate one of the ciphersuite measured on the server
	for _, client := range w.Clients {
		var cs ClientSupport
		for _, clientCiphersuite := range client.SuiteIds {
			for _, serverCiphersuite := range in.Connection.CipherSuite {
				serverCiphersuiteCode := constants.CipherSuites[serverCiphersuite.Cipher].Code
				if clientCiphersuite == int(serverCiphersuiteCode) {
					// if the ciphersuite is DHE, verify that the client support the DH size
					if strings.HasPrefix(serverCiphersuite.Cipher, "DHE-") &&
						client.MaxDhBits > 0 &&
						!clientSupportsDHE(client, serverCiphersuite) {
						continue
					}
					// if the ciphersuite is ECDHE, verify that the client supports the curve
					if strings.HasPrefix(serverCiphersuite.Cipher, "ECDHE-") && len(client.EllipticCurves) > 0 {
						cs.Curve = findClientCurve(client, serverCiphersuite)
						if cs.Curve == "" {
							continue
						}
					}
					cs.Protocol = findClientProtocol(client, serverCiphersuite)
					if cs.Protocol == "" {
						continue
					}
					// if we reached this point, the client is able to establish a connection
					// to the server. we flag it as supported and go to the next client.
					cs.IsSupported = true
					cs.Ciphersuite = serverCiphersuite.Cipher
					cs.Code = clientCiphersuite
					goto nextClient
				}
			}
		}
	nextClient:
		ClientsSupport[fmt.Sprintf("%s %s", client.Name, client.Version)] = cs
	}
	out, err := json.Marshal(ClientsSupport)
	if err != nil {
		w.error(res, "Failed to marshal results: %v", err)
	}
	res <- worker.Result{
		Success:    true,
		WorkerName: workerName,
		Errors:     nil,
		Result:     out,
	}
}

func clientSupportsDHE(client Client, serverCiphersuite connection.Ciphersuite) bool {
	// extract the dhe bits from the pfs string
	split := strings.Split(serverCiphersuite.PFS, ",")
	if len(split) < 2 {
		return false
	}
	split = strings.Split(split[1], "b")
	if len(split) < 2 {
		return false
	}
	dhsize, err := strconv.Atoi(split[0])
	if err != nil {
		return false
	}
	if client.MaxDhBits < dhsize {
		return false
	}
	return true
}

func findClientCurve(client Client, serverCiphersuite connection.Ciphersuite) string {
	for _, code := range client.EllipticCurves {
		// convert curve code to name
		for _, curveRef := range constants.Curves {
			if int(curveRef.Code) == code {
				for _, serverCurves := range serverCiphersuite.Curves {
					if curveRef.Name == serverCurves || curveRef.OpenSSLName == serverCurves {
						return curveRef.Name
					}
				}
			}
		}
	}
	return ""
}

func findClientProtocol(client Client, serverCiphersuite connection.Ciphersuite) string {
	for _, serverProto := range serverCiphersuite.Protocols {
		var spcode int
		for _, proto := range constants.Protocols {
			if proto.OpenSSLName == serverProto {
				spcode = proto.Code
			}
		}
		if client.LowestProtocol <= spcode && client.HighestProtocol >= spcode {
			return serverProto
		}
	}
	return ""
}

func (w slabscrunner) error(res chan worker.Result, messageFormat string, args ...interface{}) {
	out, _ := json.Marshal(fmt.Sprintf(messageFormat, args...))
	res <- worker.Result{
		Success:    false,
		WorkerName: workerName,
		Result:     out,
	}
}
<commit_msg>Improve client support JSON format, add analysis printer<commit_after>package sslLabsClientSupport

import (
	"encoding/json"
	"fmt"
	"net/http"
	"sort"
	"strconv"
	"strings"

	"github.com/mozilla/scribe"
	"github.com/mozilla/tls-observatory/connection"
	"github.com/mozilla/tls-observatory/constants"
	"github.com/mozilla/tls-observatory/logger"
	"github.com/mozilla/tls-observatory/worker"
)

var (
	workerName           = "sslLabsClientSupport"
	workerDesc           = "Determines client compatibility with a given target based on server certificate and ciphersuites."
	sslLabsClientDataURL = "https://api.ssllabs.com/api/v3/getClients"
	log                  = logger.GetLogger()
)

func init() {
	runner := new(slabscrunner)
	cs, err := getConffromURL(sslLabsClientDataURL)
	if err != nil {
		log.Printf("Failed to initialize %s: %v", workerName, err)
		return
	}
	runner.Clients = cs
	worker.RegisterWorker(workerName, worker.Info{Runner: runner, Description: workerDesc})
}

// getClientDataFrom retrieves the json containing the sslLabs client data
func getConffromURL(url string) (cs []Client, err error) {
	r, err := http.Get(url)
	if err != nil {
		return
	}
	defer r.Body.Close()
	err = json.NewDecoder(r.Body).Decode(&cs)
	return
}

type slabscrunner struct {
	Clients []Client
}

// Client is a definition of a TLS client with all the parameters it supports
type Client struct {
	ID                       int      `json:"id"`
	Name                     string   `json:"name"`
	Platform                 string   `json:"platform"`
	Version                  string   `json:"version"`
	HandshakeFormat          string   `json:"handshakeFormat"`
	LowestProtocol           int      `json:"lowestProtocol"`
	HighestProtocol          int      `json:"highestProtocol"`
	UserAgent                string   `json:"userAgent"`
	IsGrade0                 bool     `json:"isGrade0"`
	MaxDhBits                int      `json:"maxDhBits"`
	AbortsOnUnrecognizedName bool     `json:"abortsOnUnrecognizedName"`
	MaxRsaBits               int      `json:"maxRsaBits"`
	MinDhBits                int      `json:"minDhBits"`
	RequiresSha2             bool     `json:"requiresSha2"`
	MinRsaBits               int      `json:"minRsaBits"`
	MinEcdsaBits             int      `json:"minEcdsaBits"`
	SuiteIds                 []int    `json:"suiteIds"`
	SuiteNames               []string `json:"suiteNames"`
	SupportsSni              bool     `json:"supportsSni"`
	SupportsCompression      bool     `json:"supportsCompression"`
	SupportsStapling         bool     `json:"supportsStapling"`
	SupportsTickets          bool     `json:"supportsTickets"`
	SupportsRi               bool     `json:"supportsRi"`
	SignatureAlgorithms      []int    `json:"signatureAlgorithms"`
	EllipticCurves           []int    `json:"ellipticCurves"`
	SupportsNpn              bool     `json:"supportsNpn"`
	NpnProtocols             []string `json:"npnProtocols"`
	AlpnProtocols            []string `json:"alpnProtocols"`
}

// CipherSuite represent a ciphersuite generated and recognised by OpenSSL
type CipherSuite struct {
	IANAName   string     `json:"iana_name"`
	GnuTLSName string     `json:"gnutls_name"`
	NSSName    string     `json:"nss_name"`
	Proto      string     `json:"proto"`
	Kx         string     `json:"kx"`
	Au         string     `json:"au"`
	Enc        Encryption `json:"encryption"`
	Mac        string     `json:"mac"`
	Code       uint64     `json:"code"`
}

//Encryption represents the encryption aspects of a Ciphersuite
type Encryption struct {
	Cipher string `json:"cipher"`
	Bits   int    `json:"key"`
}

type ClientSupport struct {
	Name            string `json:"name"`
	Version         string `json:"version"`
	Platform        string `json:"platform"`
	IsSupported     bool   `json:"is_supported"`
	Ciphersuite     string `json:"ciphersuite,omitempty"`
	CiphersuiteCode uint64 `json:"ciphersuite_code,omitempty"`
	Curve           string `json:"curve,omitempty"`
	CurveCode       uint64 `json:"curve_code"`
	Protocol        string `json:"protocol,omitempty"`
	ProtocolCode    int    `json:"protocol_code"`
}

type ClientsSupport []ClientSupport

func (slice ClientsSupport) Len() int {
	return len(slice)
}

func (slice ClientsSupport) Less(i, j int) bool {
	return slice[i].Name < slice[j].Name
}

func (slice ClientsSupport) Swap(i, j int) {
	slice[i], slice[j] = slice[j], slice[i]
}

func (w slabscrunner) Run(in worker.Input, res chan worker.Result) {
	var clients ClientsSupport
	dedupClients := make(map[string]bool)
	// Loop over every client defined in the sslLabs document and check if they can
	// negotiate one of the ciphersuite measured on the server
	for _, client := range w.Clients {
		// if we already processed a client with this name, version and platform, skip it
		if _, ok := dedupClients[fmt.Sprintf("%s%s%s", client.Name, client.Version, client.Platform)]; ok {
			continue
		}
		for _, clientCiphersuite := range client.SuiteIds {
			for _, serverCiphersuite := range in.Connection.CipherSuite {
				serverCiphersuiteCode := constants.CipherSuites[serverCiphersuite.Cipher].Code
				if clientCiphersuite == int(serverCiphersuiteCode) {
					var (
						curve, protocol string
						curveCode       uint64
						protocolCode    int
					)
					// if the ciphersuite is DHE, verify that the client support the DH size
					if strings.HasPrefix(serverCiphersuite.Cipher, "DHE-") &&
						client.MaxDhBits > 0 &&
						!clientSupportsDHE(client, serverCiphersuite) {
						continue
					}
					// if the ciphersuite is ECDHE, verify that the client supports the curve
					if strings.HasPrefix(serverCiphersuite.Cipher, "ECDHE-") && len(client.EllipticCurves) > 0 {
						curve, curveCode = findClientCurve(client, serverCiphersuite)
						if curve == "" {
							continue
						}
					}
					protocol, protocolCode = findClientProtocol(client, serverCiphersuite)
					if protocol == "" {
						continue
					}
					// if we reached this point, the client is able to establish a connection
					// to the server. we flag it as supported and go to the next client.
					clients = append(clients, ClientSupport{
						Name:            client.Name,
						Version:         client.Version,
						Platform:        client.Platform,
						IsSupported:     true,
						Ciphersuite:     serverCiphersuite.Cipher,
						CiphersuiteCode: serverCiphersuite.Code,
						Curve:           curve,
						CurveCode:       curveCode,
						Protocol:        protocol,
						ProtocolCode:    protocolCode,
					})
					goto nextClient
				}
			}
		}
		// if we reach this point, it means no support was found for this client
		clients = append(clients, ClientSupport{
			Name:        client.Name,
			Version:     client.Version,
			IsSupported: false,
		})
	nextClient:
	}
	out, err := json.Marshal(clients)
	if err != nil {
		w.error(res, "Failed to marshal results: %v", err)
	}
	res <- worker.Result{
		Success:    true,
		WorkerName: workerName,
		Errors:     nil,
		Result:     out,
	}
}

func clientSupportsDHE(client Client, serverCiphersuite connection.Ciphersuite) bool {
	// extract the dhe bits from the pfs string
	split := strings.Split(serverCiphersuite.PFS, ",")
	if len(split) < 2 {
		return false
	}
	split = strings.Split(split[1], "b")
	if len(split) < 2 {
		return false
	}
	dhsize, err := strconv.Atoi(split[0])
	if err != nil {
		return false
	}
	if client.MaxDhBits < dhsize {
		return false
	}
	return true
}

func findClientCurve(client Client, serverCiphersuite connection.Ciphersuite) (string, uint64) {
	for _, code := range client.EllipticCurves {
		// convert curve code to name
		for _, curveRef := range constants.Curves {
			if int(curveRef.Code) == code {
				for _, serverCurves := range serverCiphersuite.Curves {
					if curveRef.Name == serverCurves || curveRef.OpenSSLName == serverCurves {
						return curveRef.Name, curveRef.Code
					}
				}
			}
		}
	}
	return "", 0
}

func findClientProtocol(client Client, serverCiphersuite connection.Ciphersuite) (string, int) {
	for _, serverProto := range serverCiphersuite.Protocols {
		var spcode int
		for _, proto := range constants.Protocols {
			if proto.OpenSSLName == serverProto {
				spcode = proto.Code
			}
		}
		if client.LowestProtocol <= spcode && client.HighestProtocol >= spcode {
			return serverProto, spcode
		}
	}
	return "", 0
}

func (w slabscrunner) error(res chan worker.Result, messageFormat string, args ...interface{}) {
	out, _ := json.Marshal(fmt.Sprintf(messageFormat, args...))
	res <- worker.Result{
		Success:    false,
		WorkerName: workerName,
		Result:     out,
	}
}

type eval struct {
}

func (w slabscrunner) AnalysisPrinter(r []byte, printAll interface{}) (results []string, err error) {
	var cs ClientsSupport
	err = json.Unmarshal(r, &cs)
	if err != nil {
		err = fmt.Errorf("SSLLabs Client Support: failed to parse results: %v", err)
		return
	}
	if printAll != nil && printAll.(bool) == true {
		results = append(results, "* SSLLabs Client Support: showing all clients compatibility")
	} else {
		results = append(results, "* SSLLabs Client Support: showing most recent compatible clients")
	}
	var productsSupport = make(map[string]ClientSupport)
	// sort the list of clients, it's nicer to display
	sort.Sort(cs)
	for _, client := range cs {
		// if we want all clients, store the result and go to next entry
		if printAll != nil && printAll.(bool) == true {
			result := fmt.Sprintf("  - %s %s", client.Name, client.Version)
			if client.Platform != "" {
				result += fmt.Sprintf(" (%s)", client.Platform)
			}
			if client.IsSupported {
				result += fmt.Sprintf(": yes, %s %s %s", client.Protocol, client.Ciphersuite, client.Curve)
			} else {
				result += fmt.Sprintf(": no")
			}
			results = append(results, result)
			continue
		}
		// Once we reach this point, we only want supported clients
		if !client.IsSupported {
			continue
		}
		// if we only want the oldest compatible client, some dark magic is required
		// to parse the version of each client, which varies in format, and figure out
		// the oldest
		if _, ok := productsSupport[client.Name]; !ok {
			// this is the first supported client of this name that we encounter,
			// no comparison needed, simply store it
			productsSupport[client.Name] = client
			continue
		}
		prevClient := productsSupport[client.Name]
		// FIXME: scribe doesn't like single number versions, so add a ".0" to work around it
		cVersion := client.Version
		_, err = strconv.Atoi(cVersion)
		if err == nil {
			cVersion += ".0"
		}
		pVersion := prevClient.Version
		_, err = strconv.Atoi(pVersion)
		if err == nil {
			pVersion += ".0"
		}
		// compare the version of the previous and current clients,
		// if the current client is older, store it instead of the previous one
		isOlder, err := scribe.TestEvrCompare(scribe.EVROP_LESS_THAN, cVersion, pVersion)
		if err != nil {
			log.Printf("Failed to compare version %s with version %s for client %s: %v",
				pVersion, cVersion, client.Name, err)
		}
		if isOlder {
			productsSupport[client.Name] = client
		}
	}
	if printAll != nil && printAll.(bool) == true {
		// if we just want to print all clients, return here
		return
	}
	// if we only want the oldest client, build the list here
	var supportedClients []string
	for _, clientName := range []string{"Firefox", "Chrome", "Edge", "IE", "Safari", "Opera", "Android", "OpenSSL", "Java"} {
		client := productsSupport[clientName]
		result := fmt.Sprintf("%s %s", client.Name, client.Version)
		supportedClients = append(supportedClients, result)
	}
	results = append(results, fmt.Sprintf("  - %s", strings.Join(supportedClients, ", ")))
	return
}
<|endoftext|>


<commit_before>
package spl

import (
	"io"
)

// TODO: Implement INTEGER and BLOB.

type SeqParser struct {
	_end bool
	_current [1]byte
	_reader io.Reader
}

func NewSeqParser(input io.Reader) *SeqParser {
	p := new(SeqParser)
	p._reader = input
	p.shift(1)
	return p
}

func (p *SeqParser) shift(count int) {
	for i := 0; !p._end && i < count; i++ {
		n, err := p._reader.Read(p._current[:])
		if n < 1 || err != nil {
			p._end = true
		}
	}
}

func (p *SeqParser) isEOF() bool {
	return p._end
}

func (p *SeqParser) current() byte {
	if p._end {
		return 0
	}
	return p._current[0]
}

func (p *SeqParser) skipSpace() {
	for p.current() == ' ' || p.current() == '\t' || p.current() == '\r' || p.current() == '\n' {
		p.shift(1)
	}
}

func (p *SeqParser) IsList() bool {
	return p.current() == '('
}

func (p *SeqParser) IsString() bool {
	return p.current() == '"'
}

func (p *SeqParser) IsEnd() bool {
	return p.isEOF() || p.current() == ')'
}

func (p *SeqParser) Down() {
	p.shift(1)
	p.skipSpace()
}

func (p *SeqParser) Up() {
	for !p.IsEnd() {
		p.Next()
	}

	p.shift(1)
	p.skipSpace()
}

func (p *SeqParser) Next() {
	switch {
	case p.IsList():
		p.Down()
		p.Up()

	case p.IsString():
		p.skipString()

	case p.IsEnd():
		// Nothing.

	default:
		// TODO: Remove panic() in favor of returning errors.
		panic("Bad format in SPL file.")
	}
}

func (p *SeqParser) skipString() {
	p.shift(1)

	for {
		if p.isEOF() {
			panic("End of file within a string.")
		}

		c := p.current()
		p.shift(1)

		switch c {
		case '"':
			p.skipSpace()
			return

		case '\\':
			switch p.current() {
			case '"', '\\', 'n', 'r':
				p.shift(1)
			case 'x':
				// TODO: validate escape sequences.
				p.shift(3)
			case 'u':
				p.shift(5)
			case 'U':
				p.shift(9)
			}
		}
	}
}

func unhex(h []byte) (result uint) {
	for _, d := range h {
		switch {
		case d >= '0' && d <= '9':
			result = result * 16 + uint(d - '0')
		case d >= 'a' && d <= 'f':
			result = result * 16 + 10 + uint(d - 'a')
		case d >= 'A' && d <= 'F':
			result = result * 16 + 10 + uint(d - 'A')
		default:
			panic("not a hex digit")
		}
	}

	return result
}

func (p *SeqParser) String() string {
	if p.current() != '"' {
		panic("Not a string")
	}
	p.shift(1)

	str := make([]byte, 0, 8)

	for {
		if p.isEOF() {
			panic("End of file within a string.")
		}

		c := p.current()
		p.shift(1)

		switch c {
		case '"':
			p.skipSpace()
			return string(str)

		case '\\':
			switch p.current() {
			case '"', '\\':
				str = append(str, p.current())
				p.shift(1)
			case 'n':
				str = append(str, '\n')
				p.shift(1)
			case 'r':
				str = append(str, '\r')
				p.shift(1)
			case 'x':
				h := []byte{0, 0}
				p.shift(1)
				h[0] = p.current()
				p.shift(1)
				h[1] = p.current()
				str = append(str, byte(unhex(h)))
			case 'u':
				// TODO
				panic("Not implemented yet")
			case 'U':
				// TODO
				panic("Not implemented yet")
			}
		default:
			str = append(str, c)
		}
	}
}
<commit_msg>Add file position counters, rename Next() to Skip(), and run go fmt (sorry for conflating commits).<commit_after>package spl

import (
	"io"
)

// TODO: Implement INTEGER and BLOB.

type SeqParser struct {
	_line    int
	_column  int
	_end     bool
	_current [1]byte
	_reader  io.Reader
}

func NewSeqParser(input io.Reader) *SeqParser {
	p := new(SeqParser)
	p._reader = input
	p.shift(1)
	return p
}

func (p *SeqParser) shift(count int) {
	for i := 0; !p._end && i < count; i++ {
		if p._current[0] == '\n' {
			p._line++
			p._column = 0
		}

		n, err := p._reader.Read(p._current[:])
		if n < 1 || err != nil {
			p._current[0] = 0
			p._end = true
		} else {
			p._column++
		}
	}
}

func (p *SeqParser) isEOF() bool {
	return p._end
}

func (p *SeqParser) current() byte {
	if p._end {
		return 0
	}
	return p._current[0]
}

func (p *SeqParser) skipSpace() {
	for p.current() == ' ' || p.current() == '\t' || p.current() == '\r' || p.current() == '\n' {
		p.shift(1)
	}
}

func (p *SeqParser) Line() int {
	return p._line
}

func (p *SeqParser) Column() int {
	return p._column
}

func (p *SeqParser) IsList() bool {
	return p.current() == '('
}

func (p *SeqParser) IsString() bool {
	return p.current() == '"'
}

func (p *SeqParser) IsEnd() bool {
	return p.isEOF() || p.current() == ')'
}

func (p *SeqParser) Down() {
	p.shift(1)
	p.skipSpace()
}

func (p *SeqParser) Up() {
	for !p.IsEnd() {
		p.Skip()
	}

	p.shift(1)
	p.skipSpace()
}

func (p *SeqParser) Skip() {
	switch {
	case p.IsList():
		p.Down()
		p.Up()

	case p.IsString():
		p.skipString()

	case p.IsEnd():
		// Nothing.

	default:
		// TODO: Remove panic() in favor of returning errors.
		panic("Bad format in SPL file.")
	}
}

func (p *SeqParser) skipString() {
	p.shift(1)

	for {
		if p.isEOF() {
			panic("End of file within a string.")
		}

		c := p.current()
		p.shift(1)

		switch c {
		case '"':
			p.skipSpace()
			return

		case '\\':
			switch p.current() {
			case '"', '\\', 'n', 'r':
				p.shift(1)
			case 'x':
				// TODO: validate escape sequences.
				p.shift(3)
			case 'u':
				p.shift(5)
			case 'U':
				p.shift(9)
			}
		}
	}
}

func unhex(h []byte) (result uint) {
	for _, d := range h {
		switch {
		case d >= '0' && d <= '9':
			result = result*16 + uint(d-'0')
		case d >= 'a' && d <= 'f':
			result = result*16 + 10 + uint(d-'a')
		case d >= 'A' && d <= 'F':
			result = result*16 + 10 + uint(d-'A')
		default:
			panic("not a hex digit")
		}
	}

	return result
}

func (p *SeqParser) String() string {
	if p.current() != '"' {
		panic("Not a string")
	}
	p.shift(1)

	str := make([]byte, 0, 8)

	for {
		if p.isEOF() {
			panic("End of file within a string.")
		}

		c := p.current()
		p.shift(1)

		switch c {
		case '"':
			p.skipSpace()
			return string(str)

		case '\\':
			switch p.current() {
			case '"', '\\':
				str = append(str, p.current())
				p.shift(1)
			case 'n':
				str = append(str, '\n')
				p.shift(1)
			case 'r':
				str = append(str, '\r')
				p.shift(1)
			case 'x':
				h := []byte{0, 0}
				p.shift(1)
				h[0] = p.current()
				p.shift(1)
				h[1] = p.current()
				str = append(str, byte(unhex(h)))
			case 'u':
				// TODO
				panic("Not implemented yet")
			case 'U':
				// TODO
				panic("Not implemented yet")
			}
		default:
			str = append(str, c)
		}
	}
}
<|endoftext|>


<commit_before>package template

import (
	"archive/tar"
	"bytes"
	"compress/gzip"
	"crypto/md5"
	"encoding/json"
	"fmt"
	"io"
	"net/http"
	"os"
	"strconv"
	"strings"
	"time"

	"github.com/subutai-io/base/agent/log"
	"github.com/subutai-io/gorjun/config"
	"github.com/subutai-io/gorjun/db"
	"github.com/subutai-io/gorjun/download"
	"github.com/subutai-io/gorjun/upload"
)

type Template struct {
	hash    string
	arch    string
	name    string
	parent  string
	version string
}

func readTempl(hash string) bytes.Buffer {
	var configfile bytes.Buffer
	f, err := os.Open(config.Filepath + hash)
	log.Check(log.WarnLevel, "Opening file "+config.Filepath+hash, err)
	defer f.Close()

	gzf, err := gzip.NewReader(f)
	log.Check(log.WarnLevel, "Creating gzip reader", err)

	tr := tar.NewReader(gzf)

	for {
		hdr, err := tr.Next()
		if err == io.EOF {
			break
		}
		log.Check(log.WarnLevel, "Reading tar content", err)

		if hdr.Name == "config" {
			if _, err := io.Copy(&configfile, tr); err != nil {
				log.Warn(err.Error())
			}
			break
		}
	}
	return configfile
}

func getConf(hash string, configfile bytes.Buffer) (t *Template) {
	t = &Template{
		arch:    "lxc.arch",
		name:    "lxc.utsname",
		hash:    hash,
		parent:  "subutai.parent",
		version: "subutai.template.version",
	}

	for _, v := range strings.Split(configfile.String(), "\n") {
		line := strings.Split(v, "=")
		switch strings.Trim(line[0], " ") {
		case t.arch:
			t.arch = strings.Trim(line[1], " ")
		case t.name:
			t.name = strings.Trim(line[1], " ")
		case t.parent:
			t.parent = strings.Trim(line[1], " ")
		case t.version:
			t.version = strings.Trim(line[1], " ")
		}
	}
	return
}

func Upload(w http.ResponseWriter, r *http.Request) {
	var hash, owner string
	var configfile bytes.Buffer
	if r.Method == "POST" {
		if hash, owner = upload.Handler(w, r); len(hash) == 0 {
			return
		}
		if configfile = readTempl(hash); len(configfile.String()) == 0 {
			return
		}
		t := getConf(hash, configfile)
		db.Write(owner, t.hash, t.name+"-subutai-template_"+t.version+"_"+t.arch+".tar.gz",
			map[string]string{
				"arch":    t.arch,
				"version": t.version,
				"parent":  t.parent,
				"type":    "template",
			})
		w.WriteHeader(http.StatusOK)
		w.Write([]byte(t.hash))
	}
}

func Download(w http.ResponseWriter, r *http.Request) {
	download.Handler(w, r)
}

func Show(w http.ResponseWriter, r *http.Request) {
	download.List("template", w, r)
}

func Search(w http.ResponseWriter, r *http.Request) {
	if r.Method != "GET" {
		w.WriteHeader(http.StatusBadRequest)
		w.Write([]byte("Incorrect method"))
		log.Warn("Incorrect method")
		return
	}
	download.Search("template", w, r)
}

func Info(w http.ResponseWriter, r *http.Request) {
	if r.Method != "GET" {
		w.WriteHeader(http.StatusBadRequest)
		w.Write([]byte("Incorrect method"))
		log.Warn("Incorrect method")
		return
	}
	info := download.Info("template", r)
	if len(info) != 0 {
		w.Write(info)
	} else {
		w.Write([]byte("Not found"))
	}
}

func Delete(w http.ResponseWriter, r *http.Request) {
	if r.Method != "DELETE" {
		w.WriteHeader(http.StatusBadRequest)
		w.Write([]byte("Incorrect method"))
		log.Warn("Incorrect method")
		return
	}
	if len(upload.Delete(w, r)) != 0 {
		w.WriteHeader(http.StatusOK)
		w.Write([]byte("Removed"))
	}
}

func Md5(w http.ResponseWriter, r *http.Request) {
	hash := md5.New()
	hash.Write([]byte(time.Now().String()))
	w.Write([]byte(fmt.Sprintf("%x", hash.Sum(nil))))
}

func List(w http.ResponseWriter, r *http.Request) {
	list := make([]download.ListItem, 0)
	for hash, _ := range db.List() {
		var item download.ListItem
		info := db.Info(hash)
		if info["type"] != "template" {
			continue
		}

		name := strings.Split(info["name"], "-")
		if len(name) > 0 {
			item.Name = name[0]
		}
		item.Size, _ = strconv.ParseInt(info["size"], 10, 64)
		item.Architecture = strings.ToUpper(info["arch"])
		item.Version = info["version"]
		item.OwnerFprint = info["owner"]
		item.Parent = info["parent"]
		item.Md5Sum = hash
		item.ID = item.OwnerFprint + "." + item.Md5Sum
		list = append(list, item)
	}
	js, _ := json.Marshal(list)
	w.Header().Set("Access-Control-Allow-Origin", "*")
	w.Header().Set("Access-Control-Allow-Credentials", "true")
	w.Write(js)
}
<commit_msg>Minor refactor<commit_after>package template

import (
	"archive/tar"
	"bytes"
	"compress/gzip"
	"crypto/md5"
	"encoding/json"
	"fmt"
	"io"
	"net/http"
	"os"
	"strconv"
	"strings"
	"time"

	"github.com/subutai-io/base/agent/log"
	"github.com/subutai-io/gorjun/config"
	"github.com/subutai-io/gorjun/db"
	"github.com/subutai-io/gorjun/download"
	"github.com/subutai-io/gorjun/upload"
)

type Template struct {
	hash    string
	arch    string
	name    string
	parent  string
	version string
}

func readTempl(hash string) bytes.Buffer {
	var configfile bytes.Buffer
	f, err := os.Open(config.Filepath + hash)
	log.Check(log.WarnLevel, "Opening file "+config.Filepath+hash, err)
	defer f.Close()

	gzf, err := gzip.NewReader(f)
	log.Check(log.WarnLevel, "Creating gzip reader", err)

	tr := tar.NewReader(gzf)

	for {
		hdr, err := tr.Next()
		if err == io.EOF {
			break
		}
		log.Check(log.WarnLevel, "Reading tar content", err)

		if hdr.Name == "config" {
			if _, err := io.Copy(&configfile, tr); err != nil {
				log.Warn(err.Error())
			}
			break
		}
	}
	return configfile
}

func getConf(hash string, configfile bytes.Buffer) (t *Template) {
	t = &Template{hash: hash}

	for _, v := range strings.Split(configfile.String(), "\n") {
		if line := strings.Split(v, "="); len(line) > 1 {
			line[0] = strings.TrimSpace(line[0])
			line[1] = strings.TrimSpace(line[1])

			switch line[0] {
			case "lxc.arch":
				t.arch = line[1]
			case "lxc.utsname":
				t.name = line[1]
			case "subutai.parent":
				t.parent = line[1]
			case "subutai.template.version":
				t.version = line[1]
			}
		}
	}
	return
}

func Upload(w http.ResponseWriter, r *http.Request) {
	var hash, owner string
	var configfile bytes.Buffer
	if r.Method == "POST" {
		if hash, owner = upload.Handler(w, r); len(hash) == 0 {
			return
		}
		if configfile = readTempl(hash); len(configfile.String()) == 0 {
			return
		}
		t := getConf(hash, configfile)
		db.Write(owner, t.hash, t.name+"-subutai-template_"+t.version+"_"+t.arch+".tar.gz",
			map[string]string{
				"arch":    t.arch,
				"version": t.version,
				"parent":  t.parent,
				"type":    "template",
			})
		w.WriteHeader(http.StatusOK)
		w.Write([]byte(t.hash))
	}
}

func Download(w http.ResponseWriter, r *http.Request) {
	download.Handler(w, r)
}

func Show(w http.ResponseWriter, r *http.Request) {
	download.List("template", w, r)
}

func Search(w http.ResponseWriter, r *http.Request) {
	if r.Method != "GET" {
		w.WriteHeader(http.StatusBadRequest)
		w.Write([]byte("Incorrect method"))
		log.Warn("Incorrect method")
		return
	}
	download.Search("template", w, r)
}

func Info(w http.ResponseWriter, r *http.Request) {
	if r.Method != "GET" {
		w.WriteHeader(http.StatusBadRequest)
		w.Write([]byte("Incorrect method"))
		log.Warn("Incorrect method")
		return
	}
	info := download.Info("template", r)
	if len(info) != 0 {
		w.Write(info)
	} else {
		w.Write([]byte("Not found"))
	}
}

func Delete(w http.ResponseWriter, r *http.Request) {
	if r.Method != "DELETE" {
		w.WriteHeader(http.StatusBadRequest)
		w.Write([]byte("Incorrect method"))
		log.Warn("Incorrect method")
		return
	}
	if len(upload.Delete(w, r)) != 0 {
		w.WriteHeader(http.StatusOK)
		w.Write([]byte("Removed"))
	}
}

func Md5(w http.ResponseWriter, r *http.Request) {
	hash := md5.New()
	hash.Write([]byte(time.Now().String()))
	w.Write([]byte(fmt.Sprintf("%x", hash.Sum(nil))))
}

func List(w http.ResponseWriter, r *http.Request) {
	list := make([]download.ListItem, 0)
	for hash, _ := range db.List() {
		var item download.ListItem
		info := db.Info(hash)
		if info["type"] != "template" {
			continue
		}

		name := strings.Split(info["name"], "-")
		if len(name) > 0 {
			item.Name = name[0]
		}
		item.Size, _ = strconv.ParseInt(info["size"], 10, 64)
		item.Architecture = strings.ToUpper(info["arch"])
		item.Version = info["version"]
		item.OwnerFprint = info["owner"]
		item.Parent = info["parent"]
		item.Md5Sum = hash
		item.ID = item.OwnerFprint + "." + item.Md5Sum
		list = append(list, item)
	}
	js, _ := json.Marshal(list)
	w.Header().Set("Access-Control-Allow-Origin", "*")
	w.Header().Set("Access-Control-Allow-Credentials", "true")
	w.Write(js)
}
<|endoftext|>


<commit_before>package modjulian_test

import (
	"fmt"
	"astrocalc/modjulian"
	"time"
)

func ExampleDayNumber() {
	t := time.Date(2015, 1, 1, 0, 0, 0, 0, time.UTC)
	fmt.Print(modjulian.DayNumber(t))
	// Output:
	// 57023
}
<commit_msg>Update example_test.go<commit_after>package modjulian_test

import (
	"fmt"
	"github.com/spiegel-im-spiegel/astrocalc/modjulian"
	"time"
)

func ExampleDayNumber() {
	t := time.Date(2015, 1, 1, 0, 0, 0, 0, time.UTC)
	fmt.Print(modjulian.DayNumber(t))
	// Output:
	// 57023
}
<|endoftext|>


<commit_before>package maas

import (
	"bytes"
	. "launchpad.net/gocheck"
	"launchpad.net/gomaasapi"
	"launchpad.net/juju-core/environs"
	"launchpad.net/juju-core/environs/config"
	"launchpad.net/juju-core/state"
	"launchpad.net/juju-core/testing"
)

type EnvironSuite struct {
	ProviderSuite
}

var _ = Suite(new(EnvironSuite))

// getTestConfig creates a customized sample MAAS provider configuration.
func getTestConfig(name, server, oauth, secret string) *config.Config {
	ecfg, err := newConfig(map[string]interface{}{
		"name":         name,
		"maas-server":  server,
		"maas-oauth":   oauth,
		"admin-secret": secret,
	})
	if err != nil {
		panic(err)
	}
	return ecfg.Config
}

// makeEnviron creates a functional maasEnviron for a test.  Its configuration
// is a bit arbitrary and none of the test code's business.
func (suite *EnvironSuite) makeEnviron() *maasEnviron {
	config, err := config.New(map[string]interface{}{
		"name":            suite.environ.Name(),
		"type":            "maas",
		"admin-secret":    "local-secret",
		"authorized-keys": "foo",
		"ca-cert":         testing.CACert,
		"ca-private-key":  testing.CAKey,
		"maas-oauth":      "a:b:c",
		"maas-server":     suite.testMAASObject.URL().String(),
	})
	if err != nil {
		panic(err)
	}
	env, err := NewEnviron(config)
	if err != nil {
		panic(err)
	}
	return env
}

func (EnvironSuite) TestSetConfigUpdatesConfig(c *C) {
	cfg := getTestConfig("test env", "http://maas2.example.com", "a:b:c", "secret")
	env, err := NewEnviron(cfg)
	c.Check(err, IsNil)
	c.Check(env.name, Equals, "test env")

	anotherName := "another name"
	anotherServer := "http://maas.example.com"
	anotherOauth := "c:d:e"
	anotherSecret := "secret2"
	cfg2 := getTestConfig(anotherName, anotherServer, anotherOauth, anotherSecret)
	errSetConfig := env.SetConfig(cfg2)
	c.Check(errSetConfig, IsNil)
	c.Check(env.name, Equals, anotherName)
	authClient, _ := gomaasapi.NewAuthenticatedClient(anotherServer, anotherOauth)
	maas := gomaasapi.NewMAAS(*authClient)
	MAASServer := env.maasClientUnlocked
	c.Check(MAASServer, DeepEquals, maas)
}

func (EnvironSuite) TestNewEnvironSetsConfig(c *C) {
	name := "test env"
	cfg := getTestConfig(name, "http://maas.example.com", "a:b:c", "secret")

	env, err := NewEnviron(cfg)

	c.Check(err, IsNil)
	c.Check(env.name, Equals, name)
}

func (suite *EnvironSuite) TestInstancesReturnsInstances(c *C) {
	input := `{"system_id": "test"}`
	node := suite.testMAASObject.TestServer.NewNode(input)
	resourceURI, _ := node.GetField("resource_uri")
	instanceIds := []state.InstanceId{state.InstanceId(resourceURI)}

	instances, err := suite.environ.Instances(instanceIds)

	c.Check(err, IsNil)
	c.Check(len(instances), Equals, 1)
	c.Check(string(instances[0].Id()), Equals, resourceURI)
}

func (suite *EnvironSuite) TestInstancesReturnsNilIfEmptyParameter(c *C) {
	// Instances returns nil if the given parameter is empty.
	input := `{"system_id": "test"}`
	suite.testMAASObject.TestServer.NewNode(input)
	instances, err := suite.environ.Instances([]state.InstanceId{})

	c.Check(err, IsNil)
	c.Check(instances, IsNil)
}

func (suite *EnvironSuite) TestInstancesReturnsNilIfNilParameter(c *C) {
	// Instances returns nil if the given parameter is nil.
	input := `{"system_id": "test"}`
	suite.testMAASObject.TestServer.NewNode(input)
	instances, err := suite.environ.Instances(nil)

	c.Check(err, IsNil)
	c.Check(instances, IsNil)
}

func (suite *EnvironSuite) TestAllInstancesReturnsAllInstances(c *C) {
	input := `{"system_id": "test"}`
	node := suite.testMAASObject.TestServer.NewNode(input)
	resourceURI, _ := node.GetField("resource_uri")

	instances, err := suite.environ.AllInstances()

	c.Check(err, IsNil)
	c.Check(len(instances), Equals, 1)
	c.Check(string(instances[0].Id()), Equals, resourceURI)
}

func (suite *EnvironSuite) TestAllInstancesReturnsEmptySliceIfNoInstance(c *C) {
	instances, err := suite.environ.AllInstances()

	c.Check(err, IsNil)
	c.Check(len(instances), Equals, 0)
}

func (suite *EnvironSuite) TestInstancesReturnsErrorIfPartialInstances(c *C) {
	input1 := `{"system_id": "test"}`
	node1 := suite.testMAASObject.TestServer.NewNode(input1)
	resourceURI1, _ := node1.GetField("resource_uri")
	input2 := `{"system_id": "test2"}`
	suite.testMAASObject.TestServer.NewNode(input2)
	instanceId1 := state.InstanceId(resourceURI1)
	instanceId2 := state.InstanceId("unknown systemID")
	instanceIds := []state.InstanceId{instanceId1, instanceId2}

	instances, err := suite.environ.Instances(instanceIds)

	c.Check(err, Equals, environs.ErrPartialInstances)
	c.Check(len(instances), Equals, 1)
	c.Check(string(instances[0].Id()), Equals, resourceURI1)
}

func (suite *EnvironSuite) TestStorageReturnsStorage(c *C) {
	env := suite.makeEnviron()
	storage := env.Storage()
	c.Check(storage, NotNil)
	specificStorage := storage.(*maasStorage)
	c.Check(specificStorage.environUnlocked, Equals, env)
}

func (suite *EnvironSuite) TestPublicStorageIsNotImplemented(c *C) {
	env := suite.makeEnviron()
	c.Check(env.PublicStorage(), IsNil)
}

func (suite *EnvironSuite) TestStartInstanceStartsInstance(c *C) {
	input := `{"system_id": "test"}`
	node := suite.testMAASObject.TestServer.NewNode(input)
	resourceURI, _ := node.GetField("resource_uri")

	instance, err := suite.environ.StartInstance(resourceURI, nil, nil, nil)

	c.Check(err, IsNil)
	c.Check(string(instance.Id()), Equals, resourceURI)
	operations := suite.testMAASObject.TestServer.NodeOperations()
	actions, found := operations["test"]
	c.Check(found, Equals, true)
	c.Check(actions, DeepEquals, []string{"start"})
}

func (suite *EnvironSuite) getInstance(systemId string) *maasInstance {
	input := `{"system_id": "` + systemId + `"}`
	node := suite.testMAASObject.TestServer.NewNode(input)
	return &maasInstance{&node, suite.environ}
}

func (suite *EnvironSuite) TestStopInstancesReturnsIfParameterEmpty(c *C) {
	suite.getInstance("test1")

	err := suite.environ.StopInstances([]environs.Instance{})
	c.Check(err, IsNil)
	operations := suite.testMAASObject.TestServer.NodeOperations()
	c.Check(operations, DeepEquals, map[string][]string{})
}

func (suite *EnvironSuite) TestStopInstancesStopsInstances(c *C) {
	instance1 := suite.getInstance("test1")
	instance2 := suite.getInstance("test2")
	suite.getInstance("test3")
	instances := []environs.Instance{instance1, instance2}

	err := suite.environ.StopInstances(instances)

	c.Check(err, IsNil)
	operations := suite.testMAASObject.TestServer.NodeOperations()
	expectedOperations := map[string][]string{"test1": {"stop"}, "test2": {"stop"}}
	c.Check(operations, DeepEquals, expectedOperations)
}

func (suite *EnvironSuite) TestQuiesceStateFileIsHappyWithoutStateFile(c *C) {
	err := suite.makeEnviron().quiesceStateFile()
	c.Check(err, IsNil)
}

func (suite *EnvironSuite) TestQuiesceStateFileFailsWithStateFile(c *C) {
	env := suite.makeEnviron()
	err := env.saveState(&bootstrapState{})
	c.Assert(err, IsNil)

	err = env.quiesceStateFile()

	c.Check(err, Not(IsNil))
}

func (suite *EnvironSuite) TestQuiesceStateFileFailsOnBrokenStateFile(c *C) {
	const content = "@#$(*&Y%!"
	reader := bytes.NewReader([]byte(content))
	env := suite.makeEnviron()
	err := env.Storage().Put(stateFile, reader, int64(len(content)))
	c.Assert(err, IsNil)

	err = env.quiesceStateFile()

	c.Check(err, Not(IsNil))
}

func (suite *EnvironSuite) TestBootstrap(c *C) {
	env := suite.makeEnviron()

	err := env.Bootstrap(true, []byte{}, []byte{})
	// TODO: Get this to succeed.
	unused(err)
	// c.Assert(err, IsNil)

	// TODO: Verify a simile of success.
}
<commit_msg>Comments as per review.<commit_after>package maas

import (
	"bytes"
	. "launchpad.net/gocheck"
	"launchpad.net/gomaasapi"
	"launchpad.net/juju-core/environs"
	"launchpad.net/juju-core/environs/config"
	"launchpad.net/juju-core/state"
	"launchpad.net/juju-core/testing"
)

type EnvironSuite struct {
	ProviderSuite
}

var _ = Suite(new(EnvironSuite))

// getTestConfig creates a customized sample MAAS provider configuration.
func getTestConfig(name, server, oauth, secret string) *config.Config {
	ecfg, err := newConfig(map[string]interface{}{
		"name":         name,
		"maas-server":  server,
		"maas-oauth":   oauth,
		"admin-secret": secret,
	})
	if err != nil {
		panic(err)
	}
	return ecfg.Config
}

// makeEnviron creates a functional maasEnviron for a test.  Its configuration
// is a bit arbitrary and none of the test code's business.
func (suite *EnvironSuite) makeEnviron() *maasEnviron {
	config, err := config.New(map[string]interface{}{
		"name":            suite.environ.Name(),
		"type":            "maas",
		"admin-secret":    "local-secret",
		"authorized-keys": "foo",
		"ca-cert":         testing.CACert,
		"ca-private-key":  testing.CAKey,
		"maas-oauth":      "a:b:c",
		"maas-server":     suite.testMAASObject.URL().String(),
	})
	if err != nil {
		panic(err)
	}
	env, err := NewEnviron(config)
	if err != nil {
		panic(err)
	}
	return env
}

func (EnvironSuite) TestSetConfigUpdatesConfig(c *C) {
	cfg := getTestConfig("test env", "http://maas2.example.com", "a:b:c", "secret")
	env, err := NewEnviron(cfg)
	c.Check(err, IsNil)
	c.Check(env.name, Equals, "test env")

	anotherName := "another name"
	anotherServer := "http://maas.example.com"
	anotherOauth := "c:d:e"
	anotherSecret := "secret2"
	cfg2 := getTestConfig(anotherName, anotherServer, anotherOauth, anotherSecret)
	errSetConfig := env.SetConfig(cfg2)
	c.Check(errSetConfig, IsNil)
	c.Check(env.name, Equals, anotherName)
	authClient, _ := gomaasapi.NewAuthenticatedClient(anotherServer, anotherOauth)
	maas := gomaasapi.NewMAAS(*authClient)
	MAASServer := env.maasClientUnlocked
	c.Check(MAASServer, DeepEquals, maas)
}

func (EnvironSuite) TestNewEnvironSetsConfig(c *C) {
	name := "test env"
	cfg := getTestConfig(name, "http://maas.example.com", "a:b:c", "secret")

	env, err := NewEnviron(cfg)

	c.Check(err, IsNil)
	c.Check(env.name, Equals, name)
}

func (suite *EnvironSuite) TestInstancesReturnsInstances(c *C) {
	input := `{"system_id": "test"}`
	node := suite.testMAASObject.TestServer.NewNode(input)
	resourceURI, _ := node.GetField("resource_uri")
	instanceIds := []state.InstanceId{state.InstanceId(resourceURI)}

	instances, err := suite.environ.Instances(instanceIds)

	c.Check(err, IsNil)
	c.Check(len(instances), Equals, 1)
	c.Check(string(instances[0].Id()), Equals, resourceURI)
}

func (suite *EnvironSuite) TestInstancesReturnsNilIfEmptyParameter(c *C) {
	// Instances returns nil if the given parameter is empty.
	input := `{"system_id": "test"}`
	suite.testMAASObject.TestServer.NewNode(input)
	instances, err := suite.environ.Instances([]state.InstanceId{})

	c.Check(err, IsNil)
	c.Check(instances, IsNil)
}

func (suite *EnvironSuite) TestInstancesReturnsNilIfNilParameter(c *C) {
	// Instances returns nil if the given parameter is nil.
	input := `{"system_id": "test"}`
	suite.testMAASObject.TestServer.NewNode(input)
	instances, err := suite.environ.Instances(nil)

	c.Check(err, IsNil)
	c.Check(instances, IsNil)
}

func (suite *EnvironSuite) TestAllInstancesReturnsAllInstances(c *C) {
	input := `{"system_id": "test"}`
	node := suite.testMAASObject.TestServer.NewNode(input)
	resourceURI, _ := node.GetField("resource_uri")

	instances, err := suite.environ.AllInstances()

	c.Check(err, IsNil)
	c.Check(len(instances), Equals, 1)
	c.Check(string(instances[0].Id()), Equals, resourceURI)
}

func (suite *EnvironSuite) TestAllInstancesReturnsEmptySliceIfNoInstance(c *C) {
	instances, err := suite.environ.AllInstances()

	c.Check(err, IsNil)
	c.Check(len(instances), Equals, 0)
}

func (suite *EnvironSuite) TestInstancesReturnsErrorIfPartialInstances(c *C) {
	input1 := `{"system_id": "test"}`
	node1 := suite.testMAASObject.TestServer.NewNode(input1)
	resourceURI1, _ := node1.GetField("resource_uri")
	input2 := `{"system_id": "test2"}`
	suite.testMAASObject.TestServer.NewNode(input2)
	instanceId1 := state.InstanceId(resourceURI1)
	instanceId2 := state.InstanceId("unknown systemID")
	instanceIds := []state.InstanceId{instanceId1, instanceId2}

	instances, err := suite.environ.Instances(instanceIds)

	c.Check(err, Equals, environs.ErrPartialInstances)
	c.Check(len(instances), Equals, 1)
	c.Check(string(instances[0].Id()), Equals, resourceURI1)
}

func (suite *EnvironSuite) TestStorageReturnsStorage(c *C) {
	env := suite.makeEnviron()
	storage := env.Storage()
	c.Check(storage, NotNil)
	// The Storage object is really a maasStorage.
	specificStorage := storage.(*maasStorage)
	// Its environment pointer refers back to its environment.
	c.Check(specificStorage.environUnlocked, Equals, env)
}

func (suite *EnvironSuite) TestPublicStorageIsNotImplemented(c *C) {
	env := suite.makeEnviron()
	c.Check(env.PublicStorage(), IsNil)
}

func (suite *EnvironSuite) TestStartInstanceStartsInstance(c *C) {
	input := `{"system_id": "test"}`
	node := suite.testMAASObject.TestServer.NewNode(input)
	resourceURI, _ := node.GetField("resource_uri")

	instance, err := suite.environ.StartInstance(resourceURI, nil, nil, nil)

	c.Check(err, IsNil)
	c.Check(string(instance.Id()), Equals, resourceURI)
	operations := suite.testMAASObject.TestServer.NodeOperations()
	actions, found := operations["test"]
	c.Check(found, Equals, true)
	c.Check(actions, DeepEquals, []string{"start"})
}

func (suite *EnvironSuite) getInstance(systemId string) *maasInstance {
	input := `{"system_id": "` + systemId + `"}`
	node := suite.testMAASObject.TestServer.NewNode(input)
	return &maasInstance{&node, suite.environ}
}

func (suite *EnvironSuite) TestStopInstancesReturnsIfParameterEmpty(c *C) {
	suite.getInstance("test1")

	err := suite.environ.StopInstances([]environs.Instance{})
	c.Check(err, IsNil)
	operations := suite.testMAASObject.TestServer.NodeOperations()
	c.Check(operations, DeepEquals, map[string][]string{})
}

func (suite *EnvironSuite) TestStopInstancesStopsInstances(c *C) {
	instance1 := suite.getInstance("test1")
	instance2 := suite.getInstance("test2")
	suite.getInstance("test3")
	instances := []environs.Instance{instance1, instance2}

	err := suite.environ.StopInstances(instances)

	c.Check(err, IsNil)
	operations := suite.testMAASObject.TestServer.NodeOperations()
	expectedOperations := map[string][]string{"test1": {"stop"}, "test2": {"stop"}}
	c.Check(operations, DeepEquals, expectedOperations)
}

func (suite *EnvironSuite) TestQuiesceStateFileIsHappyWithoutStateFile(c *C) {
	err := suite.makeEnviron().quiesceStateFile()
	c.Check(err, IsNil)
}

func (suite *EnvironSuite) TestQuiesceStateFileFailsWithStateFile(c *C) {
	env := suite.makeEnviron()
	err := env.saveState(&bootstrapState{})
	c.Assert(err, IsNil)

	err = env.quiesceStateFile()

	c.Check(err, Not(IsNil))
}

func (suite *EnvironSuite) TestQuiesceStateFileFailsOnBrokenStateFile(c *C) {
	const content = "@#$(*&Y%!"
	reader := bytes.NewReader([]byte(content))
	env := suite.makeEnviron()
	err := env.Storage().Put(stateFile, reader, int64(len(content)))
	c.Assert(err, IsNil)

	err = env.quiesceStateFile()

	c.Check(err, Not(IsNil))
}

func (suite *EnvironSuite) TestBootstrap(c *C) {
	env := suite.makeEnviron()

	err := env.Bootstrap(true, []byte{}, []byte{})
	// TODO: Get this to succeed.
	unused(err)
	// c.Assert(err, IsNil)

	// TODO: Verify a simile of success.
}
<|endoftext|>


<commit_before>// Google Storage utility that contains methods for both CT master and worker
// scripts.
package util

import (
	"fmt"
	"io"
	"io/ioutil"
	"net/http"
	"os"
	"path/filepath"
	"strings"
	"sync"

	"github.com/skia-dev/glog"

	storage "code.google.com/p/google-api-go-client/storage/v1"
	"skia.googlesource.com/buildbot.git/go/auth"
	"skia.googlesource.com/buildbot.git/go/gs"
)

const (
	GOROUTINE_POOL_SIZE = 50
	MAX_CHANNEL_SIZE    = 100000
)

type GsUtil struct {
	// The client used to connect to Google Storage.
	client  *http.Client
	service *storage.Service
}

// NewGsUtil initializes and returns a utility for CT interations with Google
// Storage. If client is nil then auth.RunFlow is invoked. if client is nil then
// the client from GetOAuthClient is used.
func NewGsUtil(client *http.Client) (*GsUtil, error) {
	if client == nil {
		oauthClient, err := GetOAuthClient()
		if err != nil {
			return nil, err
		}
		client = oauthClient
	}
	service, err := storage.New(client)
	if err != nil {
		return nil, fmt.Errorf("Failed to create interface to Google Storage: %s", err)
	}
	return &GsUtil{client: client, service: service}, nil
}

func GetOAuthClient() (*http.Client, error) {
	config := auth.OAuthConfig(GSTokenPath, auth.SCOPE_FULL_CONTROL)
	return auth.RunFlow(config)
}

// Returns the response body of the specified GS object. Tries MAX_URI_GET_TRIES
// times if download is unsuccessful. Client must close the response body when
// finished with it.
func getRespBody(res *storage.Object, client *http.Client) (io.ReadCloser, error) {
	for i := 0; i < MAX_URI_GET_TRIES; i++ {
		glog.Infof("Fetching: %s", res.Name)
		request, err := gs.RequestForStorageURL(res.MediaLink)
		if err != nil {
			glog.Warningf("Unable to create Storage MediaURI request: %s\n", err)
			continue
		}

		resp, err := client.Do(request)
		if err != nil {
			glog.Warningf("Unable to retrieve Storage MediaURI: %s", err)
			continue
		}
		if resp.StatusCode != 200 {
			glog.Warningf("Failed to retrieve: %d  %s", resp.StatusCode, resp.Status)
			resp.Body.Close()
			continue
		}
		return resp.Body, nil
	}
	return nil, fmt.Errorf("Failed fetching file after %d attempts", MAX_URI_GET_TRIES)
}

// Returns the response body of the specified GS file. Client must close the
// response body when finished with it.
func (gs *GsUtil) GetRemoteFileContents(filePath string) (io.ReadCloser, error) {
	res, err := gs.service.Objects.Get(GS_BUCKET_NAME, filePath).Do()
	if err != nil {
		return nil, fmt.Errorf("Could not get %s from GS: %s", filePath, err)
	}
	return getRespBody(res, gs.client)
}

// AreTimeStampsEqual checks whether the TIMESTAMP in the local dir matches the
// TIMESTAMP in the remote Google Storage dir.
func (gs *GsUtil) AreTimeStampsEqual(localDir, gsDir string) (bool, error) {
	// Get timestamp from the local directory.
	localTimestampPath := filepath.Join(localDir, TIMESTAMP_FILE_NAME)
	fileContent, err := ioutil.ReadFile(localTimestampPath)
	if err != nil {
		return false, fmt.Errorf("Could not read %s: %s", localTimestampPath, err)
	}
	localTimestamp := strings.Trim(string(fileContent), "\n")

	// Get timestamp from the Google Storage directory.
	gsTimestampPath := filepath.Join(gsDir, TIMESTAMP_FILE_NAME)
	respBody, err := gs.GetRemoteFileContents(gsTimestampPath)
	if err != nil {
		return false, err
	}
	defer respBody.Close()
	resp, err := ioutil.ReadAll(respBody)
	if err != nil {
		return false, err
	}
	gsTimestamp := strings.Trim(string(resp), "\n")

	// Return the comparison of the two timestamps.
	return localTimestamp == gsTimestamp, nil
}

type filePathToStorageObject struct {
	storageObject *storage.Object
	filePath      string
}

// downloadRemoteDir downloads the specified Google Storage dir to the specified
// local dir. The local dir will be emptied and recreated. Handles multiple levels
// of directories.
func (gs *GsUtil) downloadRemoteDir(localDir, gsDir string) error {
	// Empty the local dir.
	os.RemoveAll(localDir)
	// Create the local dir.
	os.MkdirAll(localDir, 0700)
	// The channel where the storage objects to be deleted will be sent to.
	chStorageObjects := make(chan filePathToStorageObject, MAX_CHANNEL_SIZE)
	req := gs.service.Objects.List(GS_BUCKET_NAME).Prefix(gsDir + "/")
	for req != nil {
		resp, err := req.Do()
		if err != nil {
			return fmt.Errorf("Error occured while listing %s: %s", gsDir, err)
		}
		for _, result := range resp.Items {
			fileName := filepath.Base(result.Name)
			// If downloading from subdir then add it to the fileName.
			fileGsDir := filepath.Dir(result.Name)
			subDirs := strings.TrimPrefix(fileGsDir, gsDir)
			if subDirs != "" {
				dirTokens := strings.Split(subDirs, "/")
				for i := range dirTokens {
					fileName = filepath.Join(dirTokens[len(dirTokens)-i-1], fileName)
				}
				// Create the local directory.
				os.MkdirAll(filepath.Join(localDir, filepath.Dir(fileName)), 0700)
			}
			chStorageObjects <- filePathToStorageObject{storageObject: result, filePath: fileName}
		}
		if len(resp.NextPageToken) > 0 {
			req.PageToken(resp.NextPageToken)
		} else {
			req = nil
		}
	}
	close(chStorageObjects)

	// Kick off goroutines to download the storage objects.
	var wg sync.WaitGroup
	for i := 0; i < GOROUTINE_POOL_SIZE; i++ {
		wg.Add(1)
		go func(goroutineNum int) {
			defer wg.Done()
			for obj := range chStorageObjects {
				result := obj.storageObject
				filePath := obj.filePath
				respBody, err := getRespBody(result, gs.client)
				if err != nil {
					glog.Errorf("Could not fetch %s: %s", result.MediaLink, err)
					return
				}
				defer respBody.Close()
				outputFile := filepath.Join(localDir, filePath)
				out, err := os.Create(outputFile)
				if err != nil {
					glog.Errorf("Unable to create file %s: %s", outputFile, err)
					return
				}
				defer out.Close()
				if _, err = io.Copy(out, respBody); err != nil {
					glog.Error(err)
					return
				}
				glog.Infof("Downloaded gs://%s/%s to %s with goroutine#%d", GS_BUCKET_NAME, result.Name, outputFile, goroutineNum)
			}
		}(i + 1)
	}
	wg.Wait()
	return nil
}

// DownloadChromiumBuild downloads the specified Chromium build from Google
// Storage to a local dir.
func (gs *GsUtil) DownloadChromiumBuild(chromiumBuild string) error {
	localDir := filepath.Join(ChromiumBuildsDir, chromiumBuild)
	gsDir := filepath.Join(CHROMIUM_BUILDS_DIR_NAME, chromiumBuild)
	if equal, _ := gs.AreTimeStampsEqual(localDir, gsDir); equal {
		glog.Infof("Not downloading %s because TIMESTAMPS match", gsDir)
		return nil
	}
	glog.Infof("Timestamps between %s and %s are different. Downloading from Google Storage", localDir, gsDir)
	if err := gs.downloadRemoteDir(localDir, gsDir); err != nil {
		return fmt.Errorf("Error downloading %s into %s: %s", gsDir, localDir, err)
	}
	// Downloaded chrome binary needs to be set as an executable.
	os.Chmod(filepath.Join(localDir, "chrome"), 0777)

	return nil
}

// DownloadWorkerArtifacts downloads artifacts from Google Storage to a local dir.
func (gs *GsUtil) DownloadWorkerArtifacts(dirName, pagesetType string, workerNum int) error {
	localDir := filepath.Join(StorageDir, dirName, pagesetType)
	gsDir := filepath.Join(dirName, pagesetType, fmt.Sprintf("slave%d", workerNum))

	if equal, _ := gs.AreTimeStampsEqual(localDir, gsDir); equal {
		// No need to download artifacts they already exist locally.
		glog.Infof("Not downloading %s because TIMESTAMPS match", gsDir)
		return nil
	}
	glog.Infof("Timestamps between %s and %s are different. Downloading from Google Storage", localDir, gsDir)
	return gs.downloadRemoteDir(localDir, gsDir)
}

func (gs *GsUtil) deleteRemoteDir(gsDir string) error {
	// The channel where the GS filepaths to be deleted will be sent to.
	chFilePaths := make(chan string, MAX_CHANNEL_SIZE)
	req := gs.service.Objects.List(GS_BUCKET_NAME).Prefix(gsDir + "/")
	for req != nil {
		resp, err := req.Do()
		if err != nil {
			return fmt.Errorf("Error occured while listing %s: %s", gsDir, err)
		}
		for _, result := range resp.Items {
			chFilePaths <- result.Name
		}
		if len(resp.NextPageToken) > 0 {
			req.PageToken(resp.NextPageToken)
		} else {
			req = nil
		}
	}
	close(chFilePaths)

	// Kick off goroutines to delete the file paths.
	var wg sync.WaitGroup
	for i := 0; i < GOROUTINE_POOL_SIZE; i++ {
		wg.Add(1)
		go func(goroutineNum int) {
			defer wg.Done()
			for filePath := range chFilePaths {
				if err := gs.service.Objects.Delete(GS_BUCKET_NAME, filePath).Do(); err != nil {
					glog.Errorf("Goroutine#%d could not delete %s: %s", goroutineNum, filePath, err)
					return
				}
				glog.Infof("Deleted gs://%s/%s with goroutine#%d", GS_BUCKET_NAME, filePath, goroutineNum)
			}
		}(i + 1)
	}
	wg.Wait()
	return nil
}

// UploadFile uploads the specified file to the remote dir in Google Storage. It
// also sets the appropriate ACLs on the uploaded file.
func (gs *GsUtil) UploadFile(fileName, localDir, gsDir string) error {
	localFile := filepath.Join(localDir, fileName)
	gsFile := filepath.Join(gsDir, fileName)
	object := &storage.Object{Name: gsFile}
	f, err := os.Open(localFile)
	if err != nil {
		return fmt.Errorf("Error opening %s: %s", localFile, err)
	}
	defer f.Close()
	if _, err := gs.service.Objects.Insert(GS_BUCKET_NAME, object).Media(f).Do(); err != nil {
		return fmt.Errorf("Objects.Insert failed: %s", err)
	}
	glog.Infof("Copied %s to %s", localFile, fmt.Sprintf("gs://%s/%s", GS_BUCKET_NAME, gsFile))

	// All objects uploaded to CT's bucket via this util must be readable by
	// the google.com domain. This will be fine tuned later if required.
	objectAcl := &storage.ObjectAccessControl{
		Bucket: GS_BUCKET_NAME, Entity: "domain-google.com", Object: gsFile, Role: "READER",
	}
	if _, err := gs.service.ObjectAccessControls.Insert(GS_BUCKET_NAME, gsFile, objectAcl).Do(); err != nil {
		return fmt.Errorf("Could not update ACL of %s: %s", object.Name, err)
	}
	glog.Infof("Updated ACL of %s", fmt.Sprintf("gs://%s/%s", GS_BUCKET_NAME, gsFile))

	return nil
}

// UploadWorkerArtifacts uploads artifacts from a local dir to Google Storage.
func (gs *GsUtil) UploadWorkerArtifacts(dirName, pagesetType string, workerNum int) error {
	localDir := filepath.Join(StorageDir, dirName, pagesetType)
	gsDir := filepath.Join(dirName, pagesetType, fmt.Sprintf("slave%d", workerNum))

	if equal, _ := gs.AreTimeStampsEqual(localDir, gsDir); equal {
		glog.Infof("Not uploading %s because TIMESTAMPS match", localDir)
		return nil
	}
	glog.Infof("Timestamps between %s and %s are different. Uploading to Google Storage", localDir, gsDir)
	return gs.UploadDir(localDir, gsDir)
}

// UploadDir uploads the specified local dir into the specified Google Storage dir.
func (gs *GsUtil) UploadDir(localDir, gsDir string) error {
	// Empty the remote dir.
	gs.deleteRemoteDir(gsDir)

	// Construct a dictionary of file paths to their file infos.
	pathsToFileInfos := map[string]os.FileInfo{}
	visit := func(path string, f os.FileInfo, err error) error {
		if f.IsDir() {
			return nil
		}
		pathsToFileInfos[path] = f
		return nil
	}
	if err := filepath.Walk(localDir, visit); err != nil {
		return fmt.Errorf("Unable to read the local dir %s: %s", localDir, err)
	}

	// The channel where the filepaths to be uploaded will be sent to.
	chFilePaths := make(chan string, MAX_CHANNEL_SIZE)
	// File filepaths and send it to the above channel.
	for path, fileInfo := range pathsToFileInfos {
		fileName := fileInfo.Name()
		containingDir := strings.TrimSuffix(path, fileName)
		subDirs := strings.TrimPrefix(containingDir, localDir)
		if subDirs != "" {
			dirTokens := strings.Split(subDirs, "/")
			for i := range dirTokens {
				fileName = filepath.Join(dirTokens[len(dirTokens)-i-1], fileName)
			}
		}
		chFilePaths <- fileName
	}
	close(chFilePaths)

	// Kick off goroutines to upload the file paths.
	var wg sync.WaitGroup
	for i := 0; i < GOROUTINE_POOL_SIZE; i++ {
		wg.Add(1)
		go func(goroutineNum int) {
			defer wg.Done()
			for filePath := range chFilePaths {
				glog.Infof("Uploading %s to %s with goroutine#%d", filePath, gsDir, goroutineNum)
				if err := gs.UploadFile(filePath, localDir, gsDir); err != nil {
					glog.Errorf("Goroutine#%d could not upload %s to %s: %s", goroutineNum, filePath, localDir, err)
				}
			}
		}(i + 1)
	}
	wg.Wait()
	return nil
}
<commit_msg>Add one second sleep and reduce number of GS workers<commit_after>// Google Storage utility that contains methods for both CT master and worker
// scripts.
package util

import (
	"fmt"
	"io"
	"io/ioutil"
	"net/http"
	"os"
	"path/filepath"
	"strings"
	"sync"
	"time"

	"github.com/skia-dev/glog"

	storage "code.google.com/p/google-api-go-client/storage/v1"
	"skia.googlesource.com/buildbot.git/go/auth"
	"skia.googlesource.com/buildbot.git/go/gs"
)

const (
	GOROUTINE_POOL_SIZE = 40
	MAX_CHANNEL_SIZE    = 100000
)

type GsUtil struct {
	// The client used to connect to Google Storage.
	client  *http.Client
	service *storage.Service
}

// NewGsUtil initializes and returns a utility for CT interations with Google
// Storage. If client is nil then auth.RunFlow is invoked. if client is nil then
// the client from GetOAuthClient is used.
func NewGsUtil(client *http.Client) (*GsUtil, error) {
	if client == nil {
		oauthClient, err := GetOAuthClient()
		if err != nil {
			return nil, err
		}
		client = oauthClient
	}
	service, err := storage.New(client)
	if err != nil {
		return nil, fmt.Errorf("Failed to create interface to Google Storage: %s", err)
	}
	return &GsUtil{client: client, service: service}, nil
}

func GetOAuthClient() (*http.Client, error) {
	config := auth.OAuthConfig(GSTokenPath, auth.SCOPE_FULL_CONTROL)
	return auth.RunFlow(config)
}

// Returns the response body of the specified GS object. Tries MAX_URI_GET_TRIES
// times if download is unsuccessful. Client must close the response body when
// finished with it.
func getRespBody(res *storage.Object, client *http.Client) (io.ReadCloser, error) {
	for i := 0; i < MAX_URI_GET_TRIES; i++ {
		glog.Infof("Fetching: %s", res.Name)
		request, err := gs.RequestForStorageURL(res.MediaLink)
		if err != nil {
			glog.Warningf("Unable to create Storage MediaURI request: %s\n", err)
			continue
		}

		resp, err := client.Do(request)
		if err != nil {
			glog.Warningf("Unable to retrieve Storage MediaURI: %s", err)
			continue
		}
		if resp.StatusCode != 200 {
			glog.Warningf("Failed to retrieve: %d  %s", resp.StatusCode, resp.Status)
			resp.Body.Close()
			continue
		}
		return resp.Body, nil
	}
	return nil, fmt.Errorf("Failed fetching file after %d attempts", MAX_URI_GET_TRIES)
}

// Returns the response body of the specified GS file. Client must close the
// response body when finished with it.
func (gs *GsUtil) GetRemoteFileContents(filePath string) (io.ReadCloser, error) {
	res, err := gs.service.Objects.Get(GS_BUCKET_NAME, filePath).Do()
	if err != nil {
		return nil, fmt.Errorf("Could not get %s from GS: %s", filePath, err)
	}
	return getRespBody(res, gs.client)
}

// AreTimeStampsEqual checks whether the TIMESTAMP in the local dir matches the
// TIMESTAMP in the remote Google Storage dir.
func (gs *GsUtil) AreTimeStampsEqual(localDir, gsDir string) (bool, error) {
	// Get timestamp from the local directory.
	localTimestampPath := filepath.Join(localDir, TIMESTAMP_FILE_NAME)
	fileContent, err := ioutil.ReadFile(localTimestampPath)
	if err != nil {
		return false, fmt.Errorf("Could not read %s: %s", localTimestampPath, err)
	}
	localTimestamp := strings.Trim(string(fileContent), "\n")

	// Get timestamp from the Google Storage directory.
	gsTimestampPath := filepath.Join(gsDir, TIMESTAMP_FILE_NAME)
	respBody, err := gs.GetRemoteFileContents(gsTimestampPath)
	if err != nil {
		return false, err
	}
	defer respBody.Close()
	resp, err := ioutil.ReadAll(respBody)
	if err != nil {
		return false, err
	}
	gsTimestamp := strings.Trim(string(resp), "\n")

	// Return the comparison of the two timestamps.
	return localTimestamp == gsTimestamp, nil
}

type filePathToStorageObject struct {
	storageObject *storage.Object
	filePath      string
}

// downloadRemoteDir downloads the specified Google Storage dir to the specified
// local dir. The local dir will be emptied and recreated. Handles multiple levels
// of directories.
func (gs *GsUtil) downloadRemoteDir(localDir, gsDir string) error {
	// Empty the local dir.
	os.RemoveAll(localDir)
	// Create the local dir.
	os.MkdirAll(localDir, 0700)
	// The channel where the storage objects to be deleted will be sent to.
	chStorageObjects := make(chan filePathToStorageObject, MAX_CHANNEL_SIZE)
	req := gs.service.Objects.List(GS_BUCKET_NAME).Prefix(gsDir + "/")
	for req != nil {
		resp, err := req.Do()
		if err != nil {
			return fmt.Errorf("Error occured while listing %s: %s", gsDir, err)
		}
		for _, result := range resp.Items {
			fileName := filepath.Base(result.Name)
			// If downloading from subdir then add it to the fileName.
			fileGsDir := filepath.Dir(result.Name)
			subDirs := strings.TrimPrefix(fileGsDir, gsDir)
			if subDirs != "" {
				dirTokens := strings.Split(subDirs, "/")
				for i := range dirTokens {
					fileName = filepath.Join(dirTokens[len(dirTokens)-i-1], fileName)
				}
				// Create the local directory.
				os.MkdirAll(filepath.Join(localDir, filepath.Dir(fileName)), 0700)
			}
			chStorageObjects <- filePathToStorageObject{storageObject: result, filePath: fileName}
		}
		if len(resp.NextPageToken) > 0 {
			req.PageToken(resp.NextPageToken)
		} else {
			req = nil
		}
	}
	close(chStorageObjects)

	// Kick off goroutines to download the storage objects.
	var wg sync.WaitGroup
	for i := 0; i < GOROUTINE_POOL_SIZE; i++ {
		wg.Add(1)
		go func(goroutineNum int) {
			defer wg.Done()
			for obj := range chStorageObjects {
				result := obj.storageObject
				filePath := obj.filePath
				respBody, err := getRespBody(result, gs.client)
				if err != nil {
					glog.Errorf("Could not fetch %s: %s", result.MediaLink, err)
					return
				}
				defer respBody.Close()
				outputFile := filepath.Join(localDir, filePath)
				out, err := os.Create(outputFile)
				if err != nil {
					glog.Errorf("Unable to create file %s: %s", outputFile, err)
					return
				}
				defer out.Close()
				if _, err = io.Copy(out, respBody); err != nil {
					glog.Error(err)
					return
				}
				glog.Infof("Downloaded gs://%s/%s to %s with goroutine#%d", GS_BUCKET_NAME, result.Name, outputFile, goroutineNum)
			}
		}(i + 1)
	}
	wg.Wait()
	return nil
}

// DownloadChromiumBuild downloads the specified Chromium build from Google
// Storage to a local dir.
func (gs *GsUtil) DownloadChromiumBuild(chromiumBuild string) error {
	localDir := filepath.Join(ChromiumBuildsDir, chromiumBuild)
	gsDir := filepath.Join(CHROMIUM_BUILDS_DIR_NAME, chromiumBuild)
	if equal, _ := gs.AreTimeStampsEqual(localDir, gsDir); equal {
		glog.Infof("Not downloading %s because TIMESTAMPS match", gsDir)
		return nil
	}
	glog.Infof("Timestamps between %s and %s are different. Downloading from Google Storage", localDir, gsDir)
	if err := gs.downloadRemoteDir(localDir, gsDir); err != nil {
		return fmt.Errorf("Error downloading %s into %s: %s", gsDir, localDir, err)
	}
	// Downloaded chrome binary needs to be set as an executable.
	os.Chmod(filepath.Join(localDir, "chrome"), 0777)

	return nil
}

// DownloadWorkerArtifacts downloads artifacts from Google Storage to a local dir.
func (gs *GsUtil) DownloadWorkerArtifacts(dirName, pagesetType string, workerNum int) error {
	localDir := filepath.Join(StorageDir, dirName, pagesetType)
	gsDir := filepath.Join(dirName, pagesetType, fmt.Sprintf("slave%d", workerNum))

	if equal, _ := gs.AreTimeStampsEqual(localDir, gsDir); equal {
		// No need to download artifacts they already exist locally.
		glog.Infof("Not downloading %s because TIMESTAMPS match", gsDir)
		return nil
	}
	glog.Infof("Timestamps between %s and %s are different. Downloading from Google Storage", localDir, gsDir)
	return gs.downloadRemoteDir(localDir, gsDir)
}

func (gs *GsUtil) deleteRemoteDir(gsDir string) error {
	// The channel where the GS filepaths to be deleted will be sent to.
	chFilePaths := make(chan string, MAX_CHANNEL_SIZE)
	req := gs.service.Objects.List(GS_BUCKET_NAME).Prefix(gsDir + "/")
	for req != nil {
		resp, err := req.Do()
		if err != nil {
			return fmt.Errorf("Error occured while listing %s: %s", gsDir, err)
		}
		for _, result := range resp.Items {
			chFilePaths <- result.Name
		}
		if len(resp.NextPageToken) > 0 {
			req.PageToken(resp.NextPageToken)
		} else {
			req = nil
		}
	}
	close(chFilePaths)

	// Kick off goroutines to delete the file paths.
	var wg sync.WaitGroup
	for i := 0; i < GOROUTINE_POOL_SIZE; i++ {
		wg.Add(1)
		go func(goroutineNum int) {
			defer wg.Done()
			for filePath := range chFilePaths {
				if err := gs.service.Objects.Delete(GS_BUCKET_NAME, filePath).Do(); err != nil {
					glog.Errorf("Goroutine#%d could not delete %s: %s", goroutineNum, filePath, err)
					return
				}
				glog.Infof("Deleted gs://%s/%s with goroutine#%d", GS_BUCKET_NAME, filePath, goroutineNum)
			}
		}(i + 1)
	}
	wg.Wait()
	return nil
}

// UploadFile uploads the specified file to the remote dir in Google Storage. It
// also sets the appropriate ACLs on the uploaded file.
func (gs *GsUtil) UploadFile(fileName, localDir, gsDir string) error {
	localFile := filepath.Join(localDir, fileName)
	gsFile := filepath.Join(gsDir, fileName)
	object := &storage.Object{Name: gsFile}
	f, err := os.Open(localFile)
	if err != nil {
		return fmt.Errorf("Error opening %s: %s", localFile, err)
	}
	defer f.Close()
	if _, err := gs.service.Objects.Insert(GS_BUCKET_NAME, object).Media(f).Do(); err != nil {
		return fmt.Errorf("Objects.Insert failed: %s", err)
	}
	glog.Infof("Copied %s to %s", localFile, fmt.Sprintf("gs://%s/%s", GS_BUCKET_NAME, gsFile))

	// All objects uploaded to CT's bucket via this util must be readable by
	// the google.com domain. This will be fine tuned later if required.
	objectAcl := &storage.ObjectAccessControl{
		Bucket: GS_BUCKET_NAME, Entity: "domain-google.com", Object: gsFile, Role: "READER",
	}
	if _, err := gs.service.ObjectAccessControls.Insert(GS_BUCKET_NAME, gsFile, objectAcl).Do(); err != nil {
		return fmt.Errorf("Could not update ACL of %s: %s", object.Name, err)
	}
	glog.Infof("Updated ACL of %s", fmt.Sprintf("gs://%s/%s", GS_BUCKET_NAME, gsFile))

	return nil
}

// UploadWorkerArtifacts uploads artifacts from a local dir to Google Storage.
func (gs *GsUtil) UploadWorkerArtifacts(dirName, pagesetType string, workerNum int) error {
	localDir := filepath.Join(StorageDir, dirName, pagesetType)
	gsDir := filepath.Join(dirName, pagesetType, fmt.Sprintf("slave%d", workerNum))

	if equal, _ := gs.AreTimeStampsEqual(localDir, gsDir); equal {
		glog.Infof("Not uploading %s because TIMESTAMPS match", localDir)
		return nil
	}
	glog.Infof("Timestamps between %s and %s are different. Uploading to Google Storage", localDir, gsDir)
	return gs.UploadDir(localDir, gsDir)
}

// UploadDir uploads the specified local dir into the specified Google Storage dir.
func (gs *GsUtil) UploadDir(localDir, gsDir string) error {
	// Empty the remote dir.
	gs.deleteRemoteDir(gsDir)

	// Construct a dictionary of file paths to their file infos.
	pathsToFileInfos := map[string]os.FileInfo{}
	visit := func(path string, f os.FileInfo, err error) error {
		if f.IsDir() {
			return nil
		}
		pathsToFileInfos[path] = f
		return nil
	}
	if err := filepath.Walk(localDir, visit); err != nil {
		return fmt.Errorf("Unable to read the local dir %s: %s", localDir, err)
	}

	// The channel where the filepaths to be uploaded will be sent to.
	chFilePaths := make(chan string, MAX_CHANNEL_SIZE)
	// File filepaths and send it to the above channel.
	for path, fileInfo := range pathsToFileInfos {
		fileName := fileInfo.Name()
		containingDir := strings.TrimSuffix(path, fileName)
		subDirs := strings.TrimPrefix(containingDir, localDir)
		if subDirs != "" {
			dirTokens := strings.Split(subDirs, "/")
			for i := range dirTokens {
				fileName = filepath.Join(dirTokens[len(dirTokens)-i-1], fileName)
			}
		}
		chFilePaths <- fileName
	}
	close(chFilePaths)

	// Kick off goroutines to upload the file paths.
	var wg sync.WaitGroup
	for i := 0; i < GOROUTINE_POOL_SIZE; i++ {
		wg.Add(1)
		go func(goroutineNum int) {
			defer wg.Done()
			for filePath := range chFilePaths {
				glog.Infof("Uploading %s to %s with goroutine#%d", filePath, gsDir, goroutineNum)
				if err := gs.UploadFile(filePath, localDir, gsDir); err != nil {
					glog.Errorf("Goroutine#%d could not upload %s to %s: %s", goroutineNum, filePath, localDir, err)
				}
				// Sleep for a second after uploading file to avoid bombarding Cloud
				// storage.
				time.Sleep(time.Second)
			}
		}(i + 1)
	}
	wg.Wait()
	return nil
}
<|endoftext|>


<commit_before>/*
Copyright 2017 The Kubernetes Authors.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package gce

import (
	"fmt"
	"net/http"
	"path"
	"strings"
	"time"

	"k8s.io/apimachinery/pkg/types"
	"k8s.io/kubernetes/pkg/cloudprovider"

	"github.com/golang/glog"
	compute "google.golang.org/api/compute/v1"
)

func newRoutesMetricContext(request string) *metricContext {
	return &metricContext{
		start:      time.Now(),
		attributes: []string{"routes_" + request, unusedMetricLabel, unusedMetricLabel},
	}
}

func (gce *GCECloud) ListRoutes(clusterName string) ([]*cloudprovider.Route, error) {
	var routes []*cloudprovider.Route
	pageToken := ""
	page := 0
	for ; page == 0 || (pageToken != "" && page < maxPages); page++ {
		mc := newRoutesMetricContext("list_page")
		listCall := gce.service.Routes.List(gce.networkProjectID)

		prefix := truncateClusterName(clusterName)
		listCall = listCall.Filter("name eq " + prefix + "-.*")
		if pageToken != "" {
			listCall = listCall.PageToken(pageToken)
		}
		res, err := listCall.Do()
		mc.Observe(err)
		if err != nil {
			glog.Errorf("Error getting routes from GCE: %v", err)
			return nil, err
		}
		pageToken = res.NextPageToken
		for _, r := range res.Items {
			if r.Network != gce.networkURL {
				continue
			}
			// Not managed if route description != "k8s-node-route"
			if r.Description != k8sNodeRouteTag {
				continue
			}
			// Not managed if route name doesn't start with <clusterName>
			if !strings.HasPrefix(r.Name, prefix) {
				continue
			}

			target := path.Base(r.NextHopInstance)
			// TODO: Should we lastComponent(target) this?
			targetNodeName := types.NodeName(target) // NodeName == Instance Name on GCE
			routes = append(routes, &cloudprovider.Route{Name: r.Name, TargetNode: targetNodeName, DestinationCIDR: r.DestRange})
		}
	}
	if page >= maxPages {
		glog.Errorf("ListRoutes exceeded maxPages=%d for Routes.List; truncating.", maxPages)
	}
	return routes, nil
}

func (gce *GCECloud) CreateRoute(clusterName string, nameHint string, route *cloudprovider.Route) error {
	routeName := truncateClusterName(clusterName) + "-" + nameHint

	instanceName := mapNodeNameToInstanceName(route.TargetNode)
	targetInstance, err := gce.getInstanceByName(instanceName)
	if err != nil {
		return err
	}

	mc := newRoutesMetricContext("create")
	insertOp, err := gce.service.Routes.Insert(gce.networkProjectID, &compute.Route{
		Name:            routeName,
		DestRange:       route.DestinationCIDR,
		NextHopInstance: fmt.Sprintf("zones/%s/instances/%s", targetInstance.Zone, targetInstance.Name),
		Network:         gce.networkURL,
		Priority:        1000,
		Description:     k8sNodeRouteTag,
	}).Do()
	if err != nil {
		if isHTTPErrorCode(err, http.StatusConflict) {
			glog.Info("Route %v already exists.")
			return nil
		} else {
			return mc.Observe(err)
		}
	}
	return gce.waitForGlobalOp(insertOp, mc)
}

func (gce *GCECloud) DeleteRoute(clusterName string, route *cloudprovider.Route) error {
	mc := newRoutesMetricContext("delete")
	deleteOp, err := gce.service.Routes.Delete(gce.networkProjectID, route.Name).Do()
	if err != nil {
		return mc.Observe(err)
	}
	return gce.waitForGlobalOp(deleteOp, mc)
}

func truncateClusterName(clusterName string) string {
	if len(clusterName) > 26 {
		return clusterName[:26]
	}
	return clusterName
}
<commit_msg>Use glog.*f when a format string is passed<commit_after>/*
Copyright 2017 The Kubernetes Authors.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package gce

import (
	"fmt"
	"net/http"
	"path"
	"strings"
	"time"

	"k8s.io/apimachinery/pkg/types"
	"k8s.io/kubernetes/pkg/cloudprovider"

	"github.com/golang/glog"
	compute "google.golang.org/api/compute/v1"
)

func newRoutesMetricContext(request string) *metricContext {
	return &metricContext{
		start:      time.Now(),
		attributes: []string{"routes_" + request, unusedMetricLabel, unusedMetricLabel},
	}
}

func (gce *GCECloud) ListRoutes(clusterName string) ([]*cloudprovider.Route, error) {
	var routes []*cloudprovider.Route
	pageToken := ""
	page := 0
	for ; page == 0 || (pageToken != "" && page < maxPages); page++ {
		mc := newRoutesMetricContext("list_page")
		listCall := gce.service.Routes.List(gce.networkProjectID)

		prefix := truncateClusterName(clusterName)
		listCall = listCall.Filter("name eq " + prefix + "-.*")
		if pageToken != "" {
			listCall = listCall.PageToken(pageToken)
		}
		res, err := listCall.Do()
		mc.Observe(err)
		if err != nil {
			glog.Errorf("Error getting routes from GCE: %v", err)
			return nil, err
		}
		pageToken = res.NextPageToken
		for _, r := range res.Items {
			if r.Network != gce.networkURL {
				continue
			}
			// Not managed if route description != "k8s-node-route"
			if r.Description != k8sNodeRouteTag {
				continue
			}
			// Not managed if route name doesn't start with <clusterName>
			if !strings.HasPrefix(r.Name, prefix) {
				continue
			}

			target := path.Base(r.NextHopInstance)
			// TODO: Should we lastComponent(target) this?
			targetNodeName := types.NodeName(target) // NodeName == Instance Name on GCE
			routes = append(routes, &cloudprovider.Route{Name: r.Name, TargetNode: targetNodeName, DestinationCIDR: r.DestRange})
		}
	}
	if page >= maxPages {
		glog.Errorf("ListRoutes exceeded maxPages=%d for Routes.List; truncating.", maxPages)
	}
	return routes, nil
}

func (gce *GCECloud) CreateRoute(clusterName string, nameHint string, route *cloudprovider.Route) error {
	routeName := truncateClusterName(clusterName) + "-" + nameHint

	instanceName := mapNodeNameToInstanceName(route.TargetNode)
	targetInstance, err := gce.getInstanceByName(instanceName)
	if err != nil {
		return err
	}

	mc := newRoutesMetricContext("create")
	insertOp, err := gce.service.Routes.Insert(gce.networkProjectID, &compute.Route{
		Name:            routeName,
		DestRange:       route.DestinationCIDR,
		NextHopInstance: fmt.Sprintf("zones/%s/instances/%s", targetInstance.Zone, targetInstance.Name),
		Network:         gce.networkURL,
		Priority:        1000,
		Description:     k8sNodeRouteTag,
	}).Do()
	if err != nil {
		if isHTTPErrorCode(err, http.StatusConflict) {
			glog.Infof("Route %v already exists.", routeName)
			return nil
		} else {
			return mc.Observe(err)
		}
	}
	return gce.waitForGlobalOp(insertOp, mc)
}

func (gce *GCECloud) DeleteRoute(clusterName string, route *cloudprovider.Route) error {
	mc := newRoutesMetricContext("delete")
	deleteOp, err := gce.service.Routes.Delete(gce.networkProjectID, route.Name).Do()
	if err != nil {
		return mc.Observe(err)
	}
	return gce.waitForGlobalOp(deleteOp, mc)
}

func truncateClusterName(clusterName string) string {
	if len(clusterName) > 26 {
		return clusterName[:26]
	}
	return clusterName
}
<|endoftext|>