<commit_before>// Copyright (C) 2017 Google Inc.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//      http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package adb

import (
	"context"
	"fmt"
	"strconv"
	"strings"
	"sync"
	"time"

	"github.com/google/gapid/core/app"
	"github.com/google/gapid/core/event/task"
	"github.com/google/gapid/core/fault"
	"github.com/google/gapid/core/log"
	"github.com/google/gapid/core/os/android"
	"github.com/google/gapid/core/os/device"
	"github.com/google/gapid/core/os/device/bind"
	"github.com/google/gapid/core/os/shell"
)

const (
	// ErrNoDeviceList May be returned if the adb fails to return a device list when asked.
	ErrNoDeviceList = fault.Const("Device list not returned")
	// ErrInvalidDeviceList May be returned if the device list could not be parsed.
	ErrInvalidDeviceList = fault.Const("Could not parse device list")
	// ErrInvalidStatus May be returned if the status string is not a known status.
	ErrInvalidStatus = fault.Const("Invalid status string")
	// Frequency at which to print scan errors.
	printScanErrorsEveryNSeconds = 120
	// Global settings for opting to use prerelease driver.
	oldDeveloperDriverSettingVariable = "game_driver_prerelease_opt_in_apps"
	developerDriverSettingVariable    = "updatable_driver_prerelease_opt_in_apps"
)

var (
	// Each of the devInfoProviders are called each time a new device is found.
	// External packages can use this to add additional information to the
	// device.
	devInfoProviders      []DeviceInfoProvider
	devInfoProvidersMutex sync.Mutex

	// cache is a map of device serials to fully resolved bindings.
	cache      = map[string]*binding{}
	cacheMutex sync.Mutex // Guards cache.

	// Registry of all the discovered devices.
	registry = bind.NewRegistry()
)

// DeviceInfoProvider is a function that adds additional information to a
// Device.
type DeviceInfoProvider func(ctx context.Context, d Device) error

// RegisterDeviceInfoProvider registers f to be called to add additional
// information to a newly discovered Android device.
func RegisterDeviceInfoProvider(f DeviceInfoProvider) {
	devInfoProvidersMutex.Lock()
	defer devInfoProvidersMutex.Unlock()
	devInfoProviders = append(devInfoProviders, f)
}

// Monitor updates the registry with devices that are added and removed at the
// specified interval. Monitor returns once the context is cancelled.
func Monitor(ctx context.Context, r *bind.Registry, interval time.Duration) error {
	unlisten := registry.Listen(bind.NewDeviceListener(r.AddDevice, r.RemoveDevice))
	defer unlisten()

	for _, d := range registry.Devices() {
		r.AddDevice(ctx, d)
	}

	var lastErrorPrinted time.Time
	for {
		if err := scanDevices(ctx); err != nil {
			if time.Since(lastErrorPrinted).Seconds() > printScanErrorsEveryNSeconds {
				log.E(ctx, "Couldn't scan devices: %v", err)
				lastErrorPrinted = time.Now()
			}
		} else {
			lastErrorPrinted = time.Time{}
		}

		select {
		case <-task.ShouldStop(ctx):
			return nil
		case <-time.After(interval):
		}
	}
}

// Devices returns the list of attached Android devices.
func Devices(ctx context.Context) (DeviceList, error) {
	if err := scanDevices(ctx); err != nil {
		return nil, err
	}
	devs := registry.Devices()
	out := make(DeviceList, len(devs))
	for i, d := range devs {
		out[i] = d.(Device)
	}
	return out, nil
}

func SetupPrereleaseDriver(ctx context.Context, d Device, p *android.InstalledPackage) (app.Cleanup, error) {
	settingVariable := developerDriverSettingVariable
	if d.Instance().GetConfiguration().GetOS().GetAPIVersion() <= 30 {
		settingVariable = oldDeveloperDriverSettingVariable
	}

	oldOptinApps, err := d.SystemSetting(ctx, "global", settingVariable)
	if err != nil {
		return nil, log.Err(ctx, err, "Failed to get prerelease driver opt in apps.")
	}
	if strings.Contains(oldOptinApps, p.Name) {
		return nil, nil
	}
	newOptinApps := oldOptinApps + "," + p.Name
	// TODO(b/145893290) Check whether application has developer driver enabled once b/145893290 is fixed.
	if err := d.SetSystemSetting(ctx, "global", settingVariable, newOptinApps); err != nil {
		return nil, log.Errf(ctx, err, "Failed to set up prerelease driver for app: %v.", p.Name)
	}
	return func(ctx context.Context) {
		d.SetSystemSetting(ctx, "global", settingVariable, oldOptinApps)
	}, nil
}

func newDevice(ctx context.Context, serial string, status bind.Status) (*binding, error) {
	d := &binding{
		Simple: bind.Simple{
			To: &device.Instance{
				Serial:        serial,
				Configuration: &device.Configuration{},
			},
			LastStatus: status,
		},
	}

	// Lookup the basic hardware information
	if res, err := d.SystemProperty(ctx, "ro.build.product"); err == nil {
		d.To.Configuration.Hardware = &device.Hardware{
			Name: strings.TrimSpace(res),
		}
	}

	// Early bail out if we cannot get device information
	if d.To.Configuration.Hardware == nil {
		return nil, log.Errf(ctx, nil, "Cannot get device information")
	}

	// Collect the operating system version
	if version, err := d.SystemProperty(ctx, "ro.build.version.release"); err == nil {
		var major, minor, point int32
		fmt.Sscanf(version, "%d.%d.%d", &major, &minor, &point)
		d.To.Configuration.OS = device.AndroidOS(major, minor, point)
	}

	// Collect the API version
	if version, err := d.SystemProperty(ctx, "ro.build.version.sdk"); err == nil {
		v, _ := strconv.Atoi(version)
		// preview_sdk is used to determine the version for the next OS release
		// Until the official release, new OS releases will use the same sdk
		// version as the previous OS while setting the preview_sdk
		if preview, err := d.SystemProperty(ctx, "ro.build.version.preview_sdk"); err == nil {
			p, _ := strconv.Atoi(preview)
			v += p
		}
		d.To.Configuration.OS.APIVersion = int32(v)
	}

	if description, err := d.SystemProperty(ctx, "ro.build.description"); err == nil {
		d.To.Configuration.OS.Build = strings.TrimSpace(description)
	}

	// Check which abis the device says it supports
	d.To.Configuration.ABIs = d.To.Configuration.ABIs[:0]

	seen := map[string]bool{}
	for _, prop := range []string{
		"ro.product.cpu.abilist",
		"ro.product.cpu.abi",
		"ro.product.cpu.abi2",
	} {
		abis, _ := d.SystemProperty(ctx, prop)
		if strings.TrimSpace(abis) == "" {
			continue
		}
		for _, abi := range strings.Split(abis, ",") {
			if seen[abi] {
				continue
			}
			d.To.Configuration.ABIs = append(d.To.Configuration.ABIs, device.ABIByName(abi))
			seen[abi] = true
		}
	}

	// Make sure Perfetto daemons are running.
	if err := d.EnsurePerfettoPersistent(ctx); err != nil {
		log.W(ctx, "Failed to signal Perfetto services to start", err)
	}

	// Run device info providers only if the API is supported
	if d.To.Configuration.OS != nil && d.To.Configuration.OS.APIVersion >= device.AndroidMinimalSupportedAPIVersion {
		devInfoProvidersMutex.Lock()
		defer devInfoProvidersMutex.Unlock()
		for _, f := range devInfoProviders {
			if err := f(ctx, d); err != nil {
				return nil, err
			}
		}
	}

	// Query device Perfetto service state
	if perfettoCapability, err := d.QueryPerfettoServiceState(ctx); err == nil {
		d.To.Configuration.PerfettoCapability = perfettoCapability
	}

	// Query device ANGLE support
	if anglePackage, err := d.QueryAnglePackageName(ctx); err == nil {
		d.To.Configuration.AnglePackage = anglePackage
	}

	// Query infos related to the Vulkan driver
	if d.To.Configuration.GetDrivers() != nil && d.To.Configuration.GetDrivers().GetVulkan() != nil {

		// If the VkRenderStagesProducer layer exist, we assume the render stages producer is
		// implemented in the layer.
		for _, l := range d.To.Configuration.GetDrivers().GetVulkan().GetLayers() {
			if l.Name == "VkRenderStagesProducer" {
				capability := d.To.Configuration.PerfettoCapability
				if capability == nil {
					capability = &device.PerfettoCapability{
						GpuProfiling: &device.GPUProfiling{},
					}
					d.To.Configuration.PerfettoCapability = capability
				}
				gpu := capability.GpuProfiling
				gpu.HasRenderStageProducerLayer = true
				gpu.HasRenderStage = true
				break
			}
		}

		if version, err := d.DriverVersionCode(ctx); err == nil {
			d.To.Configuration.Drivers.Vulkan.Version = strconv.Itoa(version)
		}
	}

	if d.Instance().GetName() == "" {
		d.Instance().Name = d.To.Configuration.Hardware.Name
	}
	if i := d.Instance(); i.ID == nil || allZero(i.ID.Data) {
		// Generate an identifier for the device based on its details.
		i.GenID()
	}

	return d, nil
}

func allZero(bytes []byte) bool {
	for _, b := range bytes {
		if b != 0 {
			return true
		}
	}
	return false
}

// scanDevices returns the list of attached Android devices.
func scanDevices(ctx context.Context) error {
	exe, err := adb()
	if err != nil {
		return log.Err(ctx, err, "")
	}
	stdout, err := shell.Command(exe.System(), "devices").Call(ctx)
	if err != nil {
		return err
	}
	parsed, err := parseDevices(ctx, stdout)
	if err != nil {
		return err
	}

	cacheMutex.Lock()
	defer cacheMutex.Unlock()

	for serial, status := range parsed {
		cached, ok := cache[serial]
		if !ok || status != cached.Status(ctx) {
			device, err := newDevice(ctx, serial, status)
			if err != nil {
				return err
			}
			if ok {
				registry.RemoveDevice(ctx, cached)
			}
			cache[serial] = device
			registry.AddDevice(ctx, device)
		}
	}

	// Remove cached results for removed devices.
	for serial, cached := range cache {
		if _, found := parsed[serial]; !found {
			delete(cache, serial)
			registry.RemoveDevice(ctx, cached)
		}
	}

	return nil
}

func parseDevices(ctx context.Context, out string) (map[string]bind.Status, error) {
	a := strings.SplitAfter(out, "List of devices attached")
	if len(a) != 2 {
		return nil, ErrNoDeviceList
	}
	lines := strings.Split(a[1], "\n")
	devices := make(map[string]bind.Status, len(lines))
	for _, line := range lines {
		if strings.HasPrefix(line, "adb server version") && strings.HasSuffix(line, "killing...") {
			continue // adb server version (36) doesn't match this client (35); killing...
		}
		if strings.HasPrefix(line, "*") {
			continue // For example, "* daemon started successfully *"
		}
		fields := strings.Fields(line)
		switch len(fields) {
		case 0:
			continue
		case 2:
			serial, status := fields[0], fields[1]
			switch status {
			case "unknown":
				devices[serial] = bind.Status_Unknown
			case "offline":
				devices[serial] = bind.Status_Offline
			case "device":
				devices[serial] = bind.Status_Online
			case "unauthorized":
				devices[serial] = bind.Status_Unauthorized
			default:
				return nil, log.Errf(ctx, ErrInvalidStatus, "value: %v", status)
			}
		default:
			return nil, ErrInvalidDeviceList
		}
	}
	return devices, nil
}

// NativeBridgeABI returns the native ABI for the given emulated ABI for the
// device by consulting the ro.dalvik.vm.isa.<emulated_isa>=<native_isa>
// system properties.
func (b *binding) NativeBridgeABI(ctx context.Context, emulated *device.ABI) *device.ABI {
	isa := abiToISA(emulated)
	if isa == "" {
		return emulated
	}
	isa, err := b.SystemProperty(ctx, "ro.dalvik.vm.isa."+isa)
	if err != nil {
		return emulated
	}
	native := isaToABI(isa)
	if native == nil {
		return emulated
	}
	return native
}

func (b *binding) IsLocal(ctx context.Context) (bool, error) {
	return true, nil
}

var abiToISAs = []struct {
	abi *device.ABI
	isa string
}{
	// {device.Architecture_ARMEABI, "arm"},
	{device.AndroidARMv7a, "arm"},
	{device.AndroidARM64v8a, "arm64"},
	{device.AndroidMIPS, "mips"},
	{device.AndroidMIPS64, "mips64"},
	{device.AndroidX86, "x86"},
	{device.AndroidX86_64, "x86_64"},
}

func abiToISA(abi *device.ABI) string {
	for _, e := range abiToISAs {
		if e.abi.Architecture == abi.Architecture {
			return e.isa
		}
	}
	return ""
}

func isaToABI(isa string) *device.ABI {
	for _, e := range abiToISAs {
		if e.isa == isa {
			return e.abi
		}
	}
	return nil
}
<commit_msg>Avoid adding null into UGD opt in list. (#544)<commit_after>// Copyright (C) 2017 Google Inc.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//      http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package adb

import (
	"context"
	"fmt"
	"strconv"
	"strings"
	"sync"
	"time"

	"github.com/google/gapid/core/app"
	"github.com/google/gapid/core/event/task"
	"github.com/google/gapid/core/fault"
	"github.com/google/gapid/core/log"
	"github.com/google/gapid/core/os/android"
	"github.com/google/gapid/core/os/device"
	"github.com/google/gapid/core/os/device/bind"
	"github.com/google/gapid/core/os/shell"
)

const (
	// ErrNoDeviceList May be returned if the adb fails to return a device list when asked.
	ErrNoDeviceList = fault.Const("Device list not returned")
	// ErrInvalidDeviceList May be returned if the device list could not be parsed.
	ErrInvalidDeviceList = fault.Const("Could not parse device list")
	// ErrInvalidStatus May be returned if the status string is not a known status.
	ErrInvalidStatus = fault.Const("Invalid status string")
	// Frequency at which to print scan errors.
	printScanErrorsEveryNSeconds = 120
	// Global settings for opting to use prerelease driver.
	oldDeveloperDriverSettingVariable = "game_driver_prerelease_opt_in_apps"
	developerDriverSettingVariable    = "updatable_driver_prerelease_opt_in_apps"
)

var (
	// Each of the devInfoProviders are called each time a new device is found.
	// External packages can use this to add additional information to the
	// device.
	devInfoProviders      []DeviceInfoProvider
	devInfoProvidersMutex sync.Mutex

	// cache is a map of device serials to fully resolved bindings.
	cache      = map[string]*binding{}
	cacheMutex sync.Mutex // Guards cache.

	// Registry of all the discovered devices.
	registry = bind.NewRegistry()
)

// DeviceInfoProvider is a function that adds additional information to a
// Device.
type DeviceInfoProvider func(ctx context.Context, d Device) error

// RegisterDeviceInfoProvider registers f to be called to add additional
// information to a newly discovered Android device.
func RegisterDeviceInfoProvider(f DeviceInfoProvider) {
	devInfoProvidersMutex.Lock()
	defer devInfoProvidersMutex.Unlock()
	devInfoProviders = append(devInfoProviders, f)
}

// Monitor updates the registry with devices that are added and removed at the
// specified interval. Monitor returns once the context is cancelled.
func Monitor(ctx context.Context, r *bind.Registry, interval time.Duration) error {
	unlisten := registry.Listen(bind.NewDeviceListener(r.AddDevice, r.RemoveDevice))
	defer unlisten()

	for _, d := range registry.Devices() {
		r.AddDevice(ctx, d)
	}

	var lastErrorPrinted time.Time
	for {
		if err := scanDevices(ctx); err != nil {
			if time.Since(lastErrorPrinted).Seconds() > printScanErrorsEveryNSeconds {
				log.E(ctx, "Couldn't scan devices: %v", err)
				lastErrorPrinted = time.Now()
			}
		} else {
			lastErrorPrinted = time.Time{}
		}

		select {
		case <-task.ShouldStop(ctx):
			return nil
		case <-time.After(interval):
		}
	}
}

// Devices returns the list of attached Android devices.
func Devices(ctx context.Context) (DeviceList, error) {
	if err := scanDevices(ctx); err != nil {
		return nil, err
	}
	devs := registry.Devices()
	out := make(DeviceList, len(devs))
	for i, d := range devs {
		out[i] = d.(Device)
	}
	return out, nil
}

func SetupPrereleaseDriver(ctx context.Context, d Device, p *android.InstalledPackage) (app.Cleanup, error) {
	settingVariable := developerDriverSettingVariable
	if d.Instance().GetConfiguration().GetOS().GetAPIVersion() <= 30 {
		settingVariable = oldDeveloperDriverSettingVariable
	}

	oldOptinApps, err := d.SystemSetting(ctx, "global", settingVariable)
	if err != nil {
		return nil, log.Err(ctx, err, "Failed to get prerelease driver opt in apps.")
	}
	// When key is not in the settings global database, a null will be returned.
	// Avoid using it.
	if oldOptinApps == "null" {
		oldOptinApps = ""
	}
	if strings.Contains(oldOptinApps, p.Name) {
		return nil, nil
	}
	newOptinApps := p.Name
	if oldOptinApps != "" {
		newOptinApps = oldOptinApps + "," + newOptinApps
	}
	// TODO(b/145893290) Check whether application has developer driver enabled once b/145893290 is fixed.
	if err := d.SetSystemSetting(ctx, "global", settingVariable, newOptinApps); err != nil {
		return nil, log.Errf(ctx, err, "Failed to set up prerelease driver for app: %v.", p.Name)
	}
	return func(ctx context.Context) {
		d.SetSystemSetting(ctx, "global", settingVariable, oldOptinApps)
	}, nil
}

func newDevice(ctx context.Context, serial string, status bind.Status) (*binding, error) {
	d := &binding{
		Simple: bind.Simple{
			To: &device.Instance{
				Serial:        serial,
				Configuration: &device.Configuration{},
			},
			LastStatus: status,
		},
	}

	// Lookup the basic hardware information
	if res, err := d.SystemProperty(ctx, "ro.build.product"); err == nil {
		d.To.Configuration.Hardware = &device.Hardware{
			Name: strings.TrimSpace(res),
		}
	}

	// Early bail out if we cannot get device information
	if d.To.Configuration.Hardware == nil {
		return nil, log.Errf(ctx, nil, "Cannot get device information")
	}

	// Collect the operating system version
	if version, err := d.SystemProperty(ctx, "ro.build.version.release"); err == nil {
		var major, minor, point int32
		fmt.Sscanf(version, "%d.%d.%d", &major, &minor, &point)
		d.To.Configuration.OS = device.AndroidOS(major, minor, point)
	}

	// Collect the API version
	if version, err := d.SystemProperty(ctx, "ro.build.version.sdk"); err == nil {
		v, _ := strconv.Atoi(version)
		// preview_sdk is used to determine the version for the next OS release
		// Until the official release, new OS releases will use the same sdk
		// version as the previous OS while setting the preview_sdk
		if preview, err := d.SystemProperty(ctx, "ro.build.version.preview_sdk"); err == nil {
			p, _ := strconv.Atoi(preview)
			v += p
		}
		d.To.Configuration.OS.APIVersion = int32(v)
	}

	if description, err := d.SystemProperty(ctx, "ro.build.description"); err == nil {
		d.To.Configuration.OS.Build = strings.TrimSpace(description)
	}

	// Check which abis the device says it supports
	d.To.Configuration.ABIs = d.To.Configuration.ABIs[:0]

	seen := map[string]bool{}
	for _, prop := range []string{
		"ro.product.cpu.abilist",
		"ro.product.cpu.abi",
		"ro.product.cpu.abi2",
	} {
		abis, _ := d.SystemProperty(ctx, prop)
		if strings.TrimSpace(abis) == "" {
			continue
		}
		for _, abi := range strings.Split(abis, ",") {
			if seen[abi] {
				continue
			}
			d.To.Configuration.ABIs = append(d.To.Configuration.ABIs, device.ABIByName(abi))
			seen[abi] = true
		}
	}

	// Make sure Perfetto daemons are running.
	if err := d.EnsurePerfettoPersistent(ctx); err != nil {
		log.W(ctx, "Failed to signal Perfetto services to start", err)
	}

	// Run device info providers only if the API is supported
	if d.To.Configuration.OS != nil && d.To.Configuration.OS.APIVersion >= device.AndroidMinimalSupportedAPIVersion {
		devInfoProvidersMutex.Lock()
		defer devInfoProvidersMutex.Unlock()
		for _, f := range devInfoProviders {
			if err := f(ctx, d); err != nil {
				return nil, err
			}
		}
	}

	// Query device Perfetto service state
	if perfettoCapability, err := d.QueryPerfettoServiceState(ctx); err == nil {
		d.To.Configuration.PerfettoCapability = perfettoCapability
	}

	// Query device ANGLE support
	if anglePackage, err := d.QueryAnglePackageName(ctx); err == nil {
		d.To.Configuration.AnglePackage = anglePackage
	}

	// Query infos related to the Vulkan driver
	if d.To.Configuration.GetDrivers() != nil && d.To.Configuration.GetDrivers().GetVulkan() != nil {

		// If the VkRenderStagesProducer layer exist, we assume the render stages producer is
		// implemented in the layer.
		for _, l := range d.To.Configuration.GetDrivers().GetVulkan().GetLayers() {
			if l.Name == "VkRenderStagesProducer" {
				capability := d.To.Configuration.PerfettoCapability
				if capability == nil {
					capability = &device.PerfettoCapability{
						GpuProfiling: &device.GPUProfiling{},
					}
					d.To.Configuration.PerfettoCapability = capability
				}
				gpu := capability.GpuProfiling
				gpu.HasRenderStageProducerLayer = true
				gpu.HasRenderStage = true
				break
			}
		}

		if version, err := d.DriverVersionCode(ctx); err == nil {
			d.To.Configuration.Drivers.Vulkan.Version = strconv.Itoa(version)
		}
	}

	if d.Instance().GetName() == "" {
		d.Instance().Name = d.To.Configuration.Hardware.Name
	}
	if i := d.Instance(); i.ID == nil || allZero(i.ID.Data) {
		// Generate an identifier for the device based on its details.
		i.GenID()
	}

	return d, nil
}

func allZero(bytes []byte) bool {
	for _, b := range bytes {
		if b != 0 {
			return true
		}
	}
	return false
}

// scanDevices returns the list of attached Android devices.
func scanDevices(ctx context.Context) error {
	exe, err := adb()
	if err != nil {
		return log.Err(ctx, err, "")
	}
	stdout, err := shell.Command(exe.System(), "devices").Call(ctx)
	if err != nil {
		return err
	}
	parsed, err := parseDevices(ctx, stdout)
	if err != nil {
		return err
	}

	cacheMutex.Lock()
	defer cacheMutex.Unlock()

	for serial, status := range parsed {
		cached, ok := cache[serial]
		if !ok || status != cached.Status(ctx) {
			device, err := newDevice(ctx, serial, status)
			if err != nil {
				return err
			}
			if ok {
				registry.RemoveDevice(ctx, cached)
			}
			cache[serial] = device
			registry.AddDevice(ctx, device)
		}
	}

	// Remove cached results for removed devices.
	for serial, cached := range cache {
		if _, found := parsed[serial]; !found {
			delete(cache, serial)
			registry.RemoveDevice(ctx, cached)
		}
	}

	return nil
}

func parseDevices(ctx context.Context, out string) (map[string]bind.Status, error) {
	a := strings.SplitAfter(out, "List of devices attached")
	if len(a) != 2 {
		return nil, ErrNoDeviceList
	}
	lines := strings.Split(a[1], "\n")
	devices := make(map[string]bind.Status, len(lines))
	for _, line := range lines {
		if strings.HasPrefix(line, "adb server version") && strings.HasSuffix(line, "killing...") {
			continue // adb server version (36) doesn't match this client (35); killing...
		}
		if strings.HasPrefix(line, "*") {
			continue // For example, "* daemon started successfully *"
		}
		fields := strings.Fields(line)
		switch len(fields) {
		case 0:
			continue
		case 2:
			serial, status := fields[0], fields[1]
			switch status {
			case "unknown":
				devices[serial] = bind.Status_Unknown
			case "offline":
				devices[serial] = bind.Status_Offline
			case "device":
				devices[serial] = bind.Status_Online
			case "unauthorized":
				devices[serial] = bind.Status_Unauthorized
			default:
				return nil, log.Errf(ctx, ErrInvalidStatus, "value: %v", status)
			}
		default:
			return nil, ErrInvalidDeviceList
		}
	}
	return devices, nil
}

// NativeBridgeABI returns the native ABI for the given emulated ABI for the
// device by consulting the ro.dalvik.vm.isa.<emulated_isa>=<native_isa>
// system properties.
func (b *binding) NativeBridgeABI(ctx context.Context, emulated *device.ABI) *device.ABI {
	isa := abiToISA(emulated)
	if isa == "" {
		return emulated
	}
	isa, err := b.SystemProperty(ctx, "ro.dalvik.vm.isa."+isa)
	if err != nil {
		return emulated
	}
	native := isaToABI(isa)
	if native == nil {
		return emulated
	}
	return native
}

func (b *binding) IsLocal(ctx context.Context) (bool, error) {
	return true, nil
}

var abiToISAs = []struct {
	abi *device.ABI
	isa string
}{
	// {device.Architecture_ARMEABI, "arm"},
	{device.AndroidARMv7a, "arm"},
	{device.AndroidARM64v8a, "arm64"},
	{device.AndroidMIPS, "mips"},
	{device.AndroidMIPS64, "mips64"},
	{device.AndroidX86, "x86"},
	{device.AndroidX86_64, "x86_64"},
}

func abiToISA(abi *device.ABI) string {
	for _, e := range abiToISAs {
		if e.abi.Architecture == abi.Architecture {
			return e.isa
		}
	}
	return ""
}

func isaToABI(isa string) *device.ABI {
	for _, e := range abiToISAs {
		if e.isa == isa {
			return e.abi
		}
	}
	return nil
}
<|endoftext|>


<commit_before>// Copyright 2014, Google Inc. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

package tabletmanager

// This file handles the initialization of the tablet at startup time.
// It is only enabled if init_tablet_type or init_keyspace is set.

import (
	"flag"
	"fmt"
	"time"

	log "github.com/golang/glog"
	"github.com/youtube/vitess/go/flagutil"
	"github.com/youtube/vitess/go/netutil"
	"github.com/youtube/vitess/go/vt/tabletmanager/actionnode"
	"github.com/youtube/vitess/go/vt/topo"
	"github.com/youtube/vitess/go/vt/topotools"
	"golang.org/x/net/context"
)

var (
	initDbNameOverride = flag.String("init_db_name_override", "", "(init parameter) override the name of the db used by vttablet")
	initKeyspace       = flag.String("init_keyspace", "", "(init parameter) keyspace to use for this tablet")
	initShard          = flag.String("init_shard", "", "(init parameter) shard to use for this tablet")
	initTags           flagutil.StringMapValue
	initTabletType     = flag.String("init_tablet_type", "", "(init parameter) the tablet type to use for this tablet. Incompatible with target_tablet_type.")
	initTimeout        = flag.Duration("init_timeout", 1*time.Minute, "(init parameter) timeout to use for the init phase.")
)

func init() {
	flag.Var(&initTags, "init_tags", "(init parameter) comma separated list of key:value pairs used to tag the tablet")
}

// InitTablet initializes the tablet record if necessary.
func (agent *ActionAgent) InitTablet(port, securePort, gRPCPort int) error {
	// only enabled if one of init_tablet_type (when healthcheck
	// is disabled) or init_keyspace (when healthcheck is enabled)
	// is passed in, then check other parameters
	if *initTabletType == "" && *initKeyspace == "" {
		return nil
	}

	// figure out our default target type
	var tabletType topo.TabletType
	if *initTabletType != "" {
		if *targetTabletType != "" {
			log.Fatalf("cannot specify both target_tablet_type and init_tablet_type parameters (as they might conflict)")
		}

		// use the type specified on the command line
		tabletType = topo.TabletType(*initTabletType)
		if !topo.IsTypeInList(tabletType, topo.AllTabletTypes) {
			log.Fatalf("InitTablet encountered unknown init_tablet_type '%v'", *initTabletType)
		}
		if tabletType == topo.TYPE_MASTER || tabletType == topo.TYPE_SCRAP {
			// We disallow TYPE_MASTER, so we don't have to change
			// shard.MasterAlias, and deal with the corner cases.
			// We also disallow TYPE_SCRAP, obviously.
			log.Fatalf("init_tablet_type cannot be %v", tabletType)
		}

	} else if *targetTabletType != "" {
		// use spare, the healthcheck will turn us into what
		// we need to be eventually
		tabletType = topo.TYPE_SPARE

	} else {
		log.Fatalf("if init tablet is enabled, one of init_tablet_type or target_tablet_type needs to be specified")
	}

	// create a context for this whole operation
	ctx, cancel := context.WithTimeout(agent.batchCtx, *initTimeout)
	defer cancel()

	// if we're assigned to a shard, make sure it exists, see if
	// we are its master, and update its cells list if necessary
	if tabletType != topo.TYPE_IDLE {
		if *initKeyspace == "" || *initShard == "" {
			log.Fatalf("if init tablet is enabled and the target type is not idle, init_keyspace and init_shard also need to be specified")
		}
		shard, _, err := topo.ValidateShardName(*initShard)
		if err != nil {
			log.Fatalf("cannot validate shard name: %v", err)
		}

		log.Infof("Reading shard record %v/%v", *initKeyspace, shard)

		// read the shard, create it if necessary
		si, err := topotools.GetOrCreateShard(ctx, agent.TopoServer, *initKeyspace, shard)
		if err != nil {
			return fmt.Errorf("InitTablet cannot GetOrCreateShard shard: %v", err)
		}
		if si.MasterAlias == agent.TabletAlias {
			// we are the current master for this shard (probably
			// means the master tablet process was just restarted),
			// so InitTablet as master.
			tabletType = topo.TYPE_MASTER
		}

		// See if we need to add the tablet's cell to the shard's cell
		// list.  If we do, it has to be under the shard lock.
		if !si.HasCell(agent.TabletAlias.Cell) {
			actionNode := actionnode.UpdateShard()
			lockPath, err := actionNode.LockShard(ctx, agent.TopoServer, *initKeyspace, shard)
			if err != nil {
				return fmt.Errorf("LockShard(%v/%v) failed: %v", *initKeyspace, shard, err)
			}

			// re-read the shard with the lock
			si, err = agent.TopoServer.GetShard(ctx, *initKeyspace, shard)
			if err != nil {
				return actionNode.UnlockShard(ctx, agent.TopoServer, *initKeyspace, shard, lockPath, err)
			}

			// see if we really need to update it now
			if !si.HasCell(agent.TabletAlias.Cell) {
				si.Cells = append(si.Cells, agent.TabletAlias.Cell)

				// write it back
				if err := topo.UpdateShard(ctx, agent.TopoServer, si); err != nil {
					return actionNode.UnlockShard(ctx, agent.TopoServer, *initKeyspace, shard, lockPath, err)
				}
			}

			// and unlock
			if err := actionNode.UnlockShard(ctx, agent.TopoServer, *initKeyspace, shard, lockPath, nil); err != nil {
				return err
			}
		}
	}
	log.Infof("Initializing the tablet for type %v", tabletType)

	// figure out the hostname
	hostname := *tabletHostname
	if hostname == "" {
		var err error
		hostname, err = netutil.FullyQualifiedHostname()
		if err != nil {
			return err
		}
	}

	// create and populate tablet record
	tablet := &topo.Tablet{
		Alias:          agent.TabletAlias,
		Hostname:       hostname,
		Portmap:        make(map[string]int),
		Keyspace:       *initKeyspace,
		Shard:          *initShard,
		Type:           tabletType,
		DbNameOverride: *initDbNameOverride,
		Tags:           initTags,
	}
	if port != 0 {
		tablet.Portmap["vt"] = port
	}
	if securePort != 0 {
		tablet.Portmap["vts"] = securePort
	}
	if gRPCPort != 0 {
		tablet.Portmap["grpc"] = gRPCPort
	}
	if err := tablet.Complete(); err != nil {
		return fmt.Errorf("InitTablet tablet.Complete failed: %v", err)
	}

	// now try to create the record
	err := topo.CreateTablet(ctx, agent.TopoServer, tablet)
	switch err {
	case nil:
		// it worked, we're good, can update the replication graph
		if tablet.IsInReplicationGraph() {
			if err := topo.UpdateTabletReplicationData(ctx, agent.TopoServer, tablet); err != nil {
				return fmt.Errorf("UpdateTabletReplicationData failed: %v", err)
			}
		}

	case topo.ErrNodeExists:
		// The node already exists, will just try to update
		// it. So we read it first.
		oldTablet, err := agent.TopoServer.GetTablet(ctx, tablet.Alias)
		if err != nil {
			fmt.Errorf("InitTablet failed to read existing tablet record: %v", err)
		}

		// Sanity check the keyspace and shard
		if oldTablet.Keyspace != tablet.Keyspace || oldTablet.Shard != tablet.Shard {
			return fmt.Errorf("InitTablet failed because existing tablet keyspace and shard %v/%v differ from the provided ones %v/%v", oldTablet.Keyspace, oldTablet.Shard, tablet.Keyspace, tablet.Shard)
		}

		// And overwrite the rest
		*(oldTablet.Tablet) = *tablet
		if err := topo.UpdateTablet(ctx, agent.TopoServer, oldTablet); err != nil {
			return fmt.Errorf("UpdateTablet failed: %v", err)
		}

		// Note we don't need to UpdateTabletReplicationData
		// as the tablet already existed with the right data
		// in the replication graph
	default:
		return fmt.Errorf("CreateTablet failed: %v", err)
	}

	// and now update the serving graph. Note we do that in any case,
	// to clean any inaccurate record from any part of the serving graph.
	if tabletType != topo.TYPE_IDLE {
		if err := topotools.UpdateTabletEndpoints(ctx, agent.TopoServer, tablet); err != nil {
			return fmt.Errorf("UpdateTabletEndpoints failed: %v", err)
		}
	}

	return nil
}
<commit_msg>tablet: Do not allow healthchecker to be started with target type "master".<commit_after>// Copyright 2014, Google Inc. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

package tabletmanager

// This file handles the initialization of the tablet at startup time.
// It is only enabled if init_tablet_type or init_keyspace is set.

import (
	"flag"
	"fmt"
	"time"

	log "github.com/golang/glog"
	"github.com/youtube/vitess/go/flagutil"
	"github.com/youtube/vitess/go/netutil"
	"github.com/youtube/vitess/go/vt/tabletmanager/actionnode"
	"github.com/youtube/vitess/go/vt/topo"
	"github.com/youtube/vitess/go/vt/topotools"
	"golang.org/x/net/context"
)

var (
	initDbNameOverride = flag.String("init_db_name_override", "", "(init parameter) override the name of the db used by vttablet")
	initKeyspace       = flag.String("init_keyspace", "", "(init parameter) keyspace to use for this tablet")
	initShard          = flag.String("init_shard", "", "(init parameter) shard to use for this tablet")
	initTags           flagutil.StringMapValue
	initTabletType     = flag.String("init_tablet_type", "", "(init parameter) the tablet type to use for this tablet. Incompatible with target_tablet_type.")
	initTimeout        = flag.Duration("init_timeout", 1*time.Minute, "(init parameter) timeout to use for the init phase.")
)

func init() {
	flag.Var(&initTags, "init_tags", "(init parameter) comma separated list of key:value pairs used to tag the tablet")
}

// InitTablet initializes the tablet record if necessary.
func (agent *ActionAgent) InitTablet(port, securePort, gRPCPort int) error {
	// only enabled if one of init_tablet_type (when healthcheck
	// is disabled) or init_keyspace (when healthcheck is enabled)
	// is passed in, then check other parameters
	if *initTabletType == "" && *initKeyspace == "" {
		return nil
	}

	// figure out our default target type
	var tabletType topo.TabletType
	if *initTabletType != "" {
		if *targetTabletType != "" {
			log.Fatalf("cannot specify both target_tablet_type and init_tablet_type parameters (as they might conflict)")
		}

		// use the type specified on the command line
		tabletType = topo.TabletType(*initTabletType)
		if !topo.IsTypeInList(tabletType, topo.AllTabletTypes) {
			log.Fatalf("InitTablet encountered unknown init_tablet_type '%v'", *initTabletType)
		}
		if tabletType == topo.TYPE_MASTER || tabletType == topo.TYPE_SCRAP {
			// We disallow TYPE_MASTER, so we don't have to change
			// shard.MasterAlias, and deal with the corner cases.
			// We also disallow TYPE_SCRAP, obviously.
			log.Fatalf("init_tablet_type cannot be %v", tabletType)
		}

	} else if *targetTabletType != "" {
		if tabletType := topo.TabletType(*targetTabletType); tabletType == topo.TYPE_MASTER {
			log.Fatalf("target_tablet_type cannot be '%v'. Use '%v' instead.", tabletType, topo.TYPE_REPLICA)
		}

		// use spare, the healthcheck will turn us into what
		// we need to be eventually
		tabletType = topo.TYPE_SPARE

	} else {
		log.Fatalf("if init tablet is enabled, one of init_tablet_type or target_tablet_type needs to be specified")
	}

	// create a context for this whole operation
	ctx, cancel := context.WithTimeout(agent.batchCtx, *initTimeout)
	defer cancel()

	// if we're assigned to a shard, make sure it exists, see if
	// we are its master, and update its cells list if necessary
	if tabletType != topo.TYPE_IDLE {
		if *initKeyspace == "" || *initShard == "" {
			log.Fatalf("if init tablet is enabled and the target type is not idle, init_keyspace and init_shard also need to be specified")
		}
		shard, _, err := topo.ValidateShardName(*initShard)
		if err != nil {
			log.Fatalf("cannot validate shard name: %v", err)
		}

		log.Infof("Reading shard record %v/%v", *initKeyspace, shard)

		// read the shard, create it if necessary
		si, err := topotools.GetOrCreateShard(ctx, agent.TopoServer, *initKeyspace, shard)
		if err != nil {
			return fmt.Errorf("InitTablet cannot GetOrCreateShard shard: %v", err)
		}
		if si.MasterAlias == agent.TabletAlias {
			// we are the current master for this shard (probably
			// means the master tablet process was just restarted),
			// so InitTablet as master.
			tabletType = topo.TYPE_MASTER
		}

		// See if we need to add the tablet's cell to the shard's cell
		// list.  If we do, it has to be under the shard lock.
		if !si.HasCell(agent.TabletAlias.Cell) {
			actionNode := actionnode.UpdateShard()
			lockPath, err := actionNode.LockShard(ctx, agent.TopoServer, *initKeyspace, shard)
			if err != nil {
				return fmt.Errorf("LockShard(%v/%v) failed: %v", *initKeyspace, shard, err)
			}

			// re-read the shard with the lock
			si, err = agent.TopoServer.GetShard(ctx, *initKeyspace, shard)
			if err != nil {
				return actionNode.UnlockShard(ctx, agent.TopoServer, *initKeyspace, shard, lockPath, err)
			}

			// see if we really need to update it now
			if !si.HasCell(agent.TabletAlias.Cell) {
				si.Cells = append(si.Cells, agent.TabletAlias.Cell)

				// write it back
				if err := topo.UpdateShard(ctx, agent.TopoServer, si); err != nil {
					return actionNode.UnlockShard(ctx, agent.TopoServer, *initKeyspace, shard, lockPath, err)
				}
			}

			// and unlock
			if err := actionNode.UnlockShard(ctx, agent.TopoServer, *initKeyspace, shard, lockPath, nil); err != nil {
				return err
			}
		}
	}
	log.Infof("Initializing the tablet for type %v", tabletType)

	// figure out the hostname
	hostname := *tabletHostname
	if hostname == "" {
		var err error
		hostname, err = netutil.FullyQualifiedHostname()
		if err != nil {
			return err
		}
	}

	// create and populate tablet record
	tablet := &topo.Tablet{
		Alias:          agent.TabletAlias,
		Hostname:       hostname,
		Portmap:        make(map[string]int),
		Keyspace:       *initKeyspace,
		Shard:          *initShard,
		Type:           tabletType,
		DbNameOverride: *initDbNameOverride,
		Tags:           initTags,
	}
	if port != 0 {
		tablet.Portmap["vt"] = port
	}
	if securePort != 0 {
		tablet.Portmap["vts"] = securePort
	}
	if gRPCPort != 0 {
		tablet.Portmap["grpc"] = gRPCPort
	}
	if err := tablet.Complete(); err != nil {
		return fmt.Errorf("InitTablet tablet.Complete failed: %v", err)
	}

	// now try to create the record
	err := topo.CreateTablet(ctx, agent.TopoServer, tablet)
	switch err {
	case nil:
		// it worked, we're good, can update the replication graph
		if tablet.IsInReplicationGraph() {
			if err := topo.UpdateTabletReplicationData(ctx, agent.TopoServer, tablet); err != nil {
				return fmt.Errorf("UpdateTabletReplicationData failed: %v", err)
			}
		}

	case topo.ErrNodeExists:
		// The node already exists, will just try to update
		// it. So we read it first.
		oldTablet, err := agent.TopoServer.GetTablet(ctx, tablet.Alias)
		if err != nil {
			fmt.Errorf("InitTablet failed to read existing tablet record: %v", err)
		}

		// Sanity check the keyspace and shard
		if oldTablet.Keyspace != tablet.Keyspace || oldTablet.Shard != tablet.Shard {
			return fmt.Errorf("InitTablet failed because existing tablet keyspace and shard %v/%v differ from the provided ones %v/%v", oldTablet.Keyspace, oldTablet.Shard, tablet.Keyspace, tablet.Shard)
		}

		// And overwrite the rest
		*(oldTablet.Tablet) = *tablet
		if err := topo.UpdateTablet(ctx, agent.TopoServer, oldTablet); err != nil {
			return fmt.Errorf("UpdateTablet failed: %v", err)
		}

		// Note we don't need to UpdateTabletReplicationData
		// as the tablet already existed with the right data
		// in the replication graph
	default:
		return fmt.Errorf("CreateTablet failed: %v", err)
	}

	// and now update the serving graph. Note we do that in any case,
	// to clean any inaccurate record from any part of the serving graph.
	if tabletType != topo.TYPE_IDLE {
		if err := topotools.UpdateTabletEndpoints(ctx, agent.TopoServer, tablet); err != nil {
			return fmt.Errorf("UpdateTabletEndpoints failed: %v", err)
		}
	}

	return nil
}
<|endoftext|>


<commit_before>/*
Copyright IBM Corp. All Rights Reserved.

SPDX-License-Identifier: Apache-2.0
*/

package service

import (
	"sync"
	"testing"
	"time"

	"github.com/hyperledger/fabric/core/deliverservice"
	"github.com/hyperledger/fabric/core/deliverservice/blocksprovider"
	"github.com/hyperledger/fabric/core/ledger"
	"github.com/hyperledger/fabric/core/transientstore"
	"github.com/hyperledger/fabric/gossip/api"
	"github.com/hyperledger/fabric/gossip/util"
	"github.com/hyperledger/fabric/protos/ledger/rwset"
	transientstore2 "github.com/hyperledger/fabric/protos/transientstore"
	"github.com/spf13/viper"
	"github.com/stretchr/testify/assert"
)

type transientStoreMock struct {
}

func (*transientStoreMock) PurgeByHeight(maxBlockNumToRetain uint64) error {
	return nil
}

func (*transientStoreMock) Persist(txid string, blockHeight uint64, privateSimulationResults *rwset.TxPvtReadWriteSet) error {
	panic("implement me")
}

func (*transientStoreMock) PersistWithConfig(txid string, blockHeight uint64, privateSimulationResultsWithConfig *transientstore2.TxPvtReadWriteSetWithConfigInfo) error {
	panic("implement me")
}

func (*transientStoreMock) GetTxPvtRWSetByTxid(txid string, filter ledger.PvtNsCollFilter) (transientstore.RWSetScanner, error) {
	panic("implement me")
}

func (*transientStoreMock) PurgeByTxids(txids []string) error {
	panic("implement me")
}

type embeddingDeliveryService struct {
	startOnce sync.Once
	stopOnce  sync.Once
	deliverclient.DeliverService
	startSignal sync.WaitGroup
	stopSignal  sync.WaitGroup
}

func newEmbeddingDeliveryService(ds deliverclient.DeliverService) *embeddingDeliveryService {
	eds := &embeddingDeliveryService{
		DeliverService: ds,
	}
	eds.startSignal.Add(1)
	eds.stopSignal.Add(1)
	return eds
}

func (eds *embeddingDeliveryService) waitForDeliveryServiceActivation() {
	eds.startSignal.Wait()
}

func (eds *embeddingDeliveryService) waitForDeliveryServiceTermination() {
	eds.stopSignal.Wait()
}

func (eds *embeddingDeliveryService) StartDeliverForChannel(chainID string, ledgerInfo blocksprovider.LedgerInfo, finalizer func()) error {
	eds.startOnce.Do(func() {
		eds.startSignal.Done()
	})
	return eds.DeliverService.StartDeliverForChannel(chainID, ledgerInfo, finalizer)
}

func (eds *embeddingDeliveryService) StopDeliverForChannel(chainID string) error {
	eds.stopOnce.Do(func() {
		eds.stopSignal.Done()
	})
	return eds.DeliverService.StopDeliverForChannel(chainID)
}

func (eds *embeddingDeliveryService) Stop() {
	eds.DeliverService.Stop()
}

type embeddingDeliveryServiceFactory struct {
	DeliveryServiceFactory
}

func (edsf *embeddingDeliveryServiceFactory) Service(g GossipService, endpoints []string, mcs api.MessageCryptoService) (deliverclient.DeliverService, error) {
	ds, _ := edsf.DeliveryServiceFactory.Service(g, endpoints, mcs)
	return newEmbeddingDeliveryService(ds), nil
}

func TestLeaderYield(t *testing.T) {
	// Scenario: Spawn 2 peers and wait for the first one to be the leader
	// There isn't any orderer present so the leader peer won't be able to
	// connect to the orderer, and should relinquish its leadership after a while.
	// Make sure the other peer declares itself as the leader soon after.
	takeOverMaxTimeout := time.Minute
	viper.Set("peer.gossip.election.leaderAliveThreshold", time.Second*5)
	// Test case has only two instance + making assertions only after membership view
	// is stable, hence election duration could be shorter
	viper.Set("peer.gossip.election.leaderElectionDuration", time.Millisecond*500)
	// It's enough to make single re-try
	viper.Set("peer.deliveryclient.reconnectTotalTimeThreshold", time.Second*1)
	// Since we ensuring gossip has stable membership, there is no need for
	// leader election to wait for stabilization
	viper.Set("peer.gossip.election.membershipSampleInterval", time.Millisecond*100)
	// There is no ordering service available anyway, hence connection timeout
	// could be shorter
	viper.Set("peer.deliveryclient.connTimeout", time.Millisecond*100)
	viper.Set("peer.gossip.useLeaderElection", true)
	viper.Set("peer.gossip.orgLeader", false)
	n := 2
	gossips := startPeers(t, n, 0, 1)
	defer stopPeers(gossips)
	channelName := "channelA"
	peerIndexes := []int{0, 1}
	// Add peers to the channel
	addPeersToChannel(t, n, channelName, gossips, peerIndexes)
	// Prime the membership view of the peers
	waitForFullMembership(t, gossips, n, time.Second*30, time.Millisecond*100)

	endpoint, socket := getAvailablePort(t)
	socket.Close()

	// Helper function that creates a gossipService instance
	newGossipService := func(i int) *gossipServiceImpl {
		gs := gossips[i].(*gossipGRPC).gossipServiceImpl
		gs.deliveryFactory = &embeddingDeliveryServiceFactory{&deliveryFactoryImpl{}}
		gossipServiceInstance = gs
		gs.InitializeChannel(channelName, []string{endpoint}, Support{
			Committer: &mockLedgerInfo{1},
			Store:     &transientStoreMock{},
		})
		return gs
	}

	p0 := newGossipService(0)
	p1 := newGossipService(1)

	// Returns index of the leader or -1 if no leader elected
	getLeader := func() int {
		p0.lock.RLock()
		p1.lock.RLock()
		defer p0.lock.RUnlock()
		defer p1.lock.RUnlock()

		if p0.leaderElection[channelName].IsLeader() {
			return 0
		}
		if p1.leaderElection[channelName].IsLeader() {
			return 1
		}
		return -1
	}

	ds0 := p0.deliveryService[channelName].(*embeddingDeliveryService)

	// Wait for p0 to connect to the ordering service
	ds0.waitForDeliveryServiceActivation()
	t.Log("p0 started its delivery service")
	// Ensure it's a leader
	assert.Equal(t, 0, getLeader())
	// Wait for p0 to lose its leadership
	ds0.waitForDeliveryServiceTermination()
	t.Log("p0 stopped its delivery service")
	// Ensure p0 is not a leader
	assert.NotEqual(t, 0, getLeader())
	// Wait for p1 to take over. It should take over before time reaches timeLimit
	timeLimit := time.Now().Add(takeOverMaxTimeout)
	for getLeader() != 1 && time.Now().Before(timeLimit) {
		time.Sleep(100 * time.Millisecond)
	}
	if time.Now().After(timeLimit) {
		util.PrintStackTrace()
		t.Fatalf("p1 hasn't taken over leadership within %v: %d", takeOverMaxTimeout, getLeader())
	}
	t.Log("p1 has taken over leadership")
	p0.chains[channelName].Stop()
	p1.chains[channelName].Stop()
	p0.deliveryService[channelName].Stop()
	p1.deliveryService[channelName].Stop()
}
<commit_msg>[FAB-14030] fix TestLeaderYield test<commit_after>/*
Copyright IBM Corp. All Rights Reserved.

SPDX-License-Identifier: Apache-2.0
*/

package service

import (
	"sync"
	"testing"
	"time"

	"github.com/hyperledger/fabric/core/deliverservice"
	"github.com/hyperledger/fabric/core/deliverservice/blocksprovider"
	"github.com/hyperledger/fabric/core/ledger"
	"github.com/hyperledger/fabric/core/transientstore"
	"github.com/hyperledger/fabric/gossip/api"
	"github.com/hyperledger/fabric/gossip/util"
	"github.com/hyperledger/fabric/protos/ledger/rwset"
	transientstore2 "github.com/hyperledger/fabric/protos/transientstore"
	"github.com/spf13/viper"
	"github.com/stretchr/testify/assert"
)

type transientStoreMock struct {
}

func (*transientStoreMock) PurgeByHeight(maxBlockNumToRetain uint64) error {
	return nil
}

func (*transientStoreMock) Persist(txid string, blockHeight uint64, privateSimulationResults *rwset.TxPvtReadWriteSet) error {
	panic("implement me")
}

func (*transientStoreMock) PersistWithConfig(txid string, blockHeight uint64, privateSimulationResultsWithConfig *transientstore2.TxPvtReadWriteSetWithConfigInfo) error {
	panic("implement me")
}

func (*transientStoreMock) GetTxPvtRWSetByTxid(txid string, filter ledger.PvtNsCollFilter) (transientstore.RWSetScanner, error) {
	panic("implement me")
}

func (*transientStoreMock) PurgeByTxids(txids []string) error {
	panic("implement me")
}

type embeddingDeliveryService struct {
	startOnce sync.Once
	stopOnce  sync.Once
	deliverclient.DeliverService
	startSignal sync.WaitGroup
	stopSignal  sync.WaitGroup
}

func newEmbeddingDeliveryService(ds deliverclient.DeliverService) *embeddingDeliveryService {
	eds := &embeddingDeliveryService{
		DeliverService: ds,
	}
	eds.startSignal.Add(1)
	eds.stopSignal.Add(1)
	return eds
}

func (eds *embeddingDeliveryService) waitForDeliveryServiceActivation() {
	eds.startSignal.Wait()
}

func (eds *embeddingDeliveryService) waitForDeliveryServiceTermination() {
	eds.stopSignal.Wait()
}

func (eds *embeddingDeliveryService) StartDeliverForChannel(chainID string, ledgerInfo blocksprovider.LedgerInfo, finalizer func()) error {
	eds.startOnce.Do(func() {
		eds.startSignal.Done()
	})
	return eds.DeliverService.StartDeliverForChannel(chainID, ledgerInfo, finalizer)
}

func (eds *embeddingDeliveryService) StopDeliverForChannel(chainID string) error {
	eds.stopOnce.Do(func() {
		eds.stopSignal.Done()
	})
	return eds.DeliverService.StopDeliverForChannel(chainID)
}

func (eds *embeddingDeliveryService) Stop() {
	eds.DeliverService.Stop()
}

type embeddingDeliveryServiceFactory struct {
	DeliveryServiceFactory
}

func (edsf *embeddingDeliveryServiceFactory) Service(g GossipService, endpoints []string, mcs api.MessageCryptoService) (deliverclient.DeliverService, error) {
	ds, _ := edsf.DeliveryServiceFactory.Service(g, endpoints, mcs)
	return newEmbeddingDeliveryService(ds), nil
}

func TestLeaderYield(t *testing.T) {
	// Scenario: Spawn 2 peers and wait for the first one to be the leader
	// There isn't any orderer present so the leader peer won't be able to
	// connect to the orderer, and should relinquish its leadership after a while.
	// Make sure the other peer declares itself as the leader soon after.
	takeOverMaxTimeout := time.Minute
	viper.Set("peer.gossip.election.leaderAliveThreshold", time.Second*5)
	// Test case has only two instance + making assertions only after membership view
	// is stable, hence election duration could be shorter
	viper.Set("peer.gossip.election.leaderElectionDuration", time.Millisecond*500)
	// It's enough to make single re-try
	viper.Set("peer.deliveryclient.reconnectTotalTimeThreshold", time.Second*1)
	// Since we ensuring gossip has stable membership, there is no need for
	// leader election to wait for stabilization
	viper.Set("peer.gossip.election.membershipSampleInterval", time.Millisecond*100)
	// There is no ordering service available anyway, hence connection timeout
	// could be shorter
	viper.Set("peer.deliveryclient.connTimeout", time.Millisecond*100)
	viper.Set("peer.gossip.useLeaderElection", true)
	viper.Set("peer.gossip.orgLeader", false)
	n := 2
	gossips := startPeers(t, n, 0, 1)
	defer stopPeers(gossips)
	channelName := "channelA"
	peerIndexes := []int{0, 1}
	// Add peers to the channel
	addPeersToChannel(t, n, channelName, gossips, peerIndexes)
	// Prime the membership view of the peers
	waitForFullMembership(t, gossips, n, time.Second*30, time.Millisecond*100)

	endpoint, socket := getAvailablePort(t)
	socket.Close()

	// Helper function that creates a gossipService instance
	newGossipService := func(i int) *gossipServiceImpl {
		gs := gossips[i].(*gossipGRPC).gossipServiceImpl
		gs.deliveryFactory = &embeddingDeliveryServiceFactory{&deliveryFactoryImpl{}}
		gossipServiceInstance = gs
		gs.InitializeChannel(channelName, []string{endpoint}, Support{
			Committer: &mockLedgerInfo{1},
			Store:     &transientStoreMock{},
		})
		return gs
	}

	p0 := newGossipService(0)
	p1 := newGossipService(1)

	// Returns index of the leader or -1 if no leader elected
	getLeader := func() int {
		p0.lock.RLock()
		p1.lock.RLock()
		defer p0.lock.RUnlock()
		defer p1.lock.RUnlock()

		if p0.leaderElection[channelName].IsLeader() {
			return 0
		}
		if p1.leaderElection[channelName].IsLeader() {
			return 1
		}
		return -1
	}

	ds0 := p0.deliveryService[channelName].(*embeddingDeliveryService)

	// Wait for p0 to connect to the ordering service
	ds0.waitForDeliveryServiceActivation()
	t.Log("p0 started its delivery service")
	// Ensure it's a leader
	assert.Equal(t, 0, getLeader())
	// Wait for p0 to lose its leadership
	ds0.waitForDeliveryServiceTermination()
	t.Log("p0 stopped its delivery service")
	// Ensure p0 is not a leader
	assert.NotEqual(t, 0, getLeader())
	// Wait for p1 to take over. It should take over before time reaches timeLimit
	timeLimit := time.Now().Add(takeOverMaxTimeout)
	for getLeader() != 1 && time.Now().Before(timeLimit) {
		time.Sleep(100 * time.Millisecond)
	}
	if time.Now().After(timeLimit) && getLeader() != 1 {
		util.PrintStackTrace()
		t.Fatalf("p1 hasn't taken over leadership within %v: %d", takeOverMaxTimeout, getLeader())
	}
	t.Log("p1 has taken over leadership")
	p0.chains[channelName].Stop()
	p1.chains[channelName].Stop()
	p0.deliveryService[channelName].Stop()
	p1.deliveryService[channelName].Stop()
}
<|endoftext|>


<commit_before>package healthcheck

import (
	"fmt"
	"strconv"
	"strings"
)

// TemplateHealthcheck pongo2 template for healthchecks
const TemplateHealthcheck = `

{{ metadata.name }} - Running against database "{{ metadata.db_name }}"

            Title
            Query
            Test Ran?
            Expected
            Actual

        {% for element in elements %}

                {{ element.Title }}
                {{ element.Query }}
                {% if element.Passed == "SUCCESS"%}
                    {{ element.Passed }}
										{% if element.Equal == "TRUE"%}
                    	{{ element.Expected }}
                    	{{ element.Actual }}
										{% elif  element.Equal == "FALSE" %}
                    	{{ element.Expected }}
                    	{{ element.Actual }}
										{% endif %}
                {% elif  element.Passed == "FAIL" %}
                    {{ element.Passed }}
                    {{ element.Error }}
                {% endif %}

        {% endfor %}

    {{ metadata.footer | safe }}

    {{ metadata.timestamp }}

`

// FooterHealthcheck footer for healthchecks
const FooterHealthcheck = `

Thank you,

  CFPB CR Data-Sharing Team

  Consumer Financial Protection Bureau

Confidentiality Notice: If you received this email by mistake, please notify the sender of the mistake and delete the e-mail and any attachments. An inadvertent disclosure is not intended to waive any privileges.

`

// SubjectHealthcheck creates a subject for healthcheck email
func SubjectHealthcheck(name string, dbName string, hostname string, level string, errors int, fatal bool) string {

	hcName := name
	if name == "" {
		hcName = "healthchecks"
	}

	//subjectStr := fmt.Sprintf("%s for %s on %s at %s level",
	subjectStr := fmt.Sprintf("%s - %s - %s - %s level",
		hcName, dbName, hostname, strings.ToUpper(level))

	if errors > 0 {
		subjectStr = fmt.Sprintf("%s error(s) - %s", strconv.Itoa(errors), subjectStr)
	}

	if fatal {
		subjectStr = fmt.Sprintf("FATAL - %s", subjectStr)
	}

	return subjectStr
}
<commit_msg>healthcheck template footer "Data Team"<commit_after>package healthcheck

import (
	"fmt"
	"strconv"
	"strings"
)

// TemplateHealthcheck pongo2 template for healthchecks
const TemplateHealthcheck = `

{{ metadata.name }} - Running against database "{{ metadata.db_name }}"

            Title
            Query
            Test Ran?
            Expected
            Actual

        {% for element in elements %}

                {{ element.Title }}
                {{ element.Query }}
                {% if element.Passed == "SUCCESS"%}
                    {{ element.Passed }}
										{% if element.Equal == "TRUE"%}
                    	{{ element.Expected }}
                    	{{ element.Actual }}
										{% elif  element.Equal == "FALSE" %}
                    	{{ element.Expected }}
                    	{{ element.Actual }}
										{% endif %}
                {% elif  element.Passed == "FAIL" %}
                    {{ element.Passed }}
                    {{ element.Error }}
                {% endif %}

        {% endfor %}

    {{ metadata.footer | safe }}

    {{ metadata.timestamp }}

`

// FooterHealthcheck footer for healthchecks
const FooterHealthcheck = `

Thank you,

  CFPB Data Team

  Consumer Financial Protection Bureau

Confidentiality Notice: If you received this email by mistake, please notify the sender of the mistake and delete the e-mail and any attachments. An inadvertent disclosure is not intended to waive any privileges.

`

// SubjectHealthcheck creates a subject for healthcheck email
func SubjectHealthcheck(name string, dbName string, hostname string, level string, errors int, fatal bool) string {

	hcName := name
	if name == "" {
		hcName = "healthchecks"
	}

	//subjectStr := fmt.Sprintf("%s for %s on %s at %s level",
	subjectStr := fmt.Sprintf("%s - %s - %s - %s level",
		hcName, dbName, hostname, strings.ToUpper(level))

	if errors > 0 {
		subjectStr = fmt.Sprintf("%s error(s) - %s", strconv.Itoa(errors), subjectStr)
	}

	if fatal {
		subjectStr = fmt.Sprintf("FATAL - %s", subjectStr)
	}

	return subjectStr
}
<|endoftext|>


<commit_before>// reCAPTCHA Go API
//
// https://developers.google.com/recaptcha/intro
package captcha

import (
	"errors"
	"io/ioutil"
	"net/http"
	"net/url"
	"strings"
)

type Captcha struct {
	private string
	Public  string
	Server  string
}

const apiServer = "https://www.google.com/recaptcha/api"

var (
	ErrInvalidPrivateKey = errors.New("We weren't able to verify the private key.")
	ErrInvalidChallenge  = errors.New("The challenge parameter of the verify script was incorrect.")
	ErrInvalidSolution   = errors.New("The CAPTCHA solution was incorrect.")
	ErrTimeout           = errors.New("The solution was received after the CAPTCHA timed out.")
	ErrNotReachable      = errors.New("reCAPTCHA never returns this error code.")
)

var errMap = map[string]error{
	"invalid-site-private-key": ErrInvalidPrivateKey,
	"invalid-request-cookie":   ErrInvalidChallenge,
	"incorrect-captcha-sol":    ErrInvalidSolution,
	"captcha-timeout":          ErrTimeout,
	"recaptcha-not-reachable":  ErrNotReachable,
}

func New(private, public string) Captcha {
	return Captcha{private: private, Public: public, Server: apiServer}
}

func (c Captcha) Verify(remoteip, challenge, response string) (bool, error) {
	values := url.Values{
		"privatekey": {c.private},
		"remoteip":   {remoteip},
		"challenge":  {challenge},
		"response":   {response},
	}
	resp, err := http.PostForm(c.Server+"/verify", values)
	if err != nil {
		return false, err
	}
	body, err := ioutil.ReadAll(resp.Body)
	defer resp.Body.Close()
	if err != nil {
		return false, err
	}
	answer := strings.Split(string(body), "\n")
	return answer[0] == "true", errMap[answer[1]]
}
<commit_msg>Sanity check<commit_after>// reCAPTCHA Go API
//
// https://developers.google.com/recaptcha/intro
package captcha

import (
	"errors"
	"io/ioutil"
	"net/http"
	"net/url"
	"strings"
)

type Captcha struct {
	private string
	Public  string
	Server  string
}

const apiServer = "https://www.google.com/recaptcha/api"

var (
	ErrInvalidPrivateKey = errors.New("We weren't able to verify the private key.")
	ErrInvalidChallenge  = errors.New("The challenge parameter of the verify script was incorrect.")
	ErrInvalidSolution   = errors.New("The CAPTCHA solution was incorrect.")
	ErrTimeout           = errors.New("The solution was received after the CAPTCHA timed out.")
	ErrNotReachable      = errors.New("reCAPTCHA never returns this error code.")
)

var errMap = map[string]error{
	"invalid-site-private-key": ErrInvalidPrivateKey,
	"invalid-request-cookie":   ErrInvalidChallenge,
	"incorrect-captcha-sol":    ErrInvalidSolution,
	"captcha-timeout":          ErrTimeout,
	"recaptcha-not-reachable":  ErrNotReachable,
}

func New(private, public string) Captcha {
	return Captcha{private: private, Public: public, Server: apiServer}
}

func (c Captcha) Verify(remoteip, challenge, response string) (bool, error) {
	values := url.Values{
		"privatekey": {c.private},
		"remoteip":   {remoteip},
		"challenge":  {challenge},
		"response":   {response},
	}
	resp, err := http.PostForm(c.Server+"/verify", values)
	if err != nil {
		return false, err
	}
	body, err := ioutil.ReadAll(resp.Body)
	defer resp.Body.Close()
	if err != nil {
		return false, err
	}
	answer := strings.Split(string(body), "\n")
	if len(answer) != 2 {
		return false, ErrNotReachable
	}
	return answer[0] == "true", errMap[answer[1]]
}
<|endoftext|>


<commit_before>package cba

import (
	"io"
	"log"
	"net/rpc"
	"os"
	"time"
)

// Client is a thread-safe interface to fetching over a connection.
type Client struct {
	store  *Store
	client *rpc.Client
}

func (store *Store) NewClient(conn io.ReadWriteCloser) *Client {
	return &Client{
		store:  store,
		client: rpc.NewClientWithCodec(NewCbaCodec(conn)),
	}
}

func (c *Client) Close() {
	c.client.Close()
}

func (c *Store) ServeConn(conn io.ReadWriteCloser) {
	s := Server{c}
	rpcServer := rpc.NewServer()
	rpcServer.Register(&s)
	rpcServer.ServeCodec(NewCbaCodec(conn))
	conn.Close()
}

type Server struct {
	store *Store
}

func (s *Server) ServeChunk(req *Request, rep *Response) (err error) {
	e := s.store.ServeChunk(req, rep)
	s.store.addThroughput(0, int64(len(rep.Chunk)))
	return e
}

func (st *Store) ServeChunk(req *Request, rep *Response) (err error) {
	if !st.HasHash(req.Hash) {
		rep.Have = false
		return nil
	}

	rep.Have = true
	if c := st.ContentsIfLoaded(req.Hash); c != nil {
		if req.End > len(c) {
			req.End = len(c)
		}
		rep.Chunk = c[req.Start:req.End]
		rep.Size = len(c)
		return nil
	}

	f, err := os.Open(st.Path(req.Hash))
	if err != nil {
		return err
	}
	defer f.Close()

	rep.Chunk = make([]byte, req.End-req.Start)
	n, err := f.ReadAt(rep.Chunk, int64(req.Start))
	rep.Chunk = rep.Chunk[:n]
	rep.Size = n

	if err == io.EOF {
		err = nil
	}
	return err
}

func (c *Client) Fetch(want string) (bool, error) {
	start := time.Now()
	succ, size, err := c.fetch(want)
	dt := time.Now().Sub(start)
	c.store.timings.Log("ContentStore.Fetch", dt)
	c.store.timings.LogN("ContentStore.FetchBytes", int64(size), dt)
	return succ, err
}

func (c *Client) fetchChunk(req *Request, rep *Response) (error) {
	start := time.Now()
	err := c.client.Call("Server.ServeChunk", req, rep)
	dt := time.Now().Sub(start)
	c.store.timings.Log("ContentStore.FetchChunk", dt)
	c.store.timings.LogN("ContentStore.FetchChunkBytes", int64(rep.Size), dt)
	return err
}

func (c *Client) fetch(want string) (bool, int, error) {
	chunkSize := 1 << 18
	buf := make([]byte, chunkSize)

	var output *HashWriter
	written := 0

	var saved string
	for {
		req := &Request{
			Hash:  want,
			Start: written,
			End:   written + chunkSize,
		}
		rep := &Response{Chunk: buf}
		err := c.fetchChunk(req, rep)
		if err != nil || !rep.Have {
			return false, 0, err
		}

		// is this a bug in the rpc package?
		content := rep.Chunk[:rep.Size]

		if len(content) < chunkSize && written == 0 {
			saved = c.store.Save(content)
			written = len(content)
			break
		} else if output == nil {
			output = c.store.NewHashWriter()
			defer output.Close()
		}

		n, err := output.Write(content)
		written += n
		if err != nil {
			return false, 0, err
		}
		if len(content) < chunkSize {
			break
		}
	}
	if output != nil {
		output.Close()
		saved = string(output.Sum())
	}
	c.store.addThroughput(int64(written), 0)
	if want != saved {
		log.Fatalf("file corruption: got %x want %x", saved, want)
	}
	return true, written, nil
}
<commit_msg>Switch off new codec for now.<commit_after>package cba

import (
	"io"
	"log"
	"net/rpc"
	"os"
	"time"
)

const useNewTransport = false

// Client is a thread-safe interface to fetching over a connection.
type Client struct {
	store  *Store
	client *rpc.Client
}

func (store *Store) NewClient(conn io.ReadWriteCloser) *Client {
	cl := &Client{
		store: store,
	}

	if useNewTransport {
		cl.client = rpc.NewClientWithCodec(NewCbaCodec(conn))
	} else {
		cl.client = rpc.NewClient(conn)
	}
	return cl
}

func (c *Client) Close() {
	c.client.Close()
}

func (c *Store) ServeConn(conn io.ReadWriteCloser) {
	s := Server{c}
	rpcServer := rpc.NewServer()
	rpcServer.Register(&s)
	if useNewTransport {
		rpcServer.ServeCodec(NewCbaCodec(conn))
	} else {
		rpcServer.ServeConn(conn)
	}
	conn.Close()
}

type Server struct {
	store *Store
}

func (s *Server) ServeChunk(req *Request, rep *Response) (err error) {
	e := s.store.ServeChunk(req, rep)
	s.store.addThroughput(0, int64(len(rep.Chunk)))
	return e
}

func (st *Store) ServeChunk(req *Request, rep *Response) (err error) {
	if !st.HasHash(req.Hash) {
		rep.Have = false
		return nil
	}

	rep.Have = true
	if c := st.ContentsIfLoaded(req.Hash); c != nil {
		if req.End > len(c) {
			req.End = len(c)
		}
		rep.Chunk = c[req.Start:req.End]
		rep.Size = len(c)
		return nil
	}

	f, err := os.Open(st.Path(req.Hash))
	if err != nil {
		return err
	}
	defer f.Close()

	rep.Chunk = make([]byte, req.End-req.Start)
	n, err := f.ReadAt(rep.Chunk, int64(req.Start))
	rep.Chunk = rep.Chunk[:n]
	rep.Size = n

	if err == io.EOF {
		err = nil
	}
	return err
}

func (c *Client) Fetch(want string) (bool, error) {
	start := time.Now()
	succ, size, err := c.fetch(want)
	dt := time.Now().Sub(start)
	c.store.timings.Log("ContentStore.Fetch", dt)
	c.store.timings.LogN("ContentStore.FetchBytes", int64(size), dt)
	return succ, err
}

func (c *Client) fetchChunk(req *Request, rep *Response) error {
	start := time.Now()
	err := c.client.Call("Server.ServeChunk", req, rep)
	dt := time.Now().Sub(start)
	c.store.timings.Log("ContentStore.FetchChunk", dt)
	c.store.timings.LogN("ContentStore.FetchChunkBytes", int64(rep.Size), dt)
	return err
}

func (c *Client) fetch(want string) (bool, int, error) {
	chunkSize := 1 << 18
	buf := make([]byte, chunkSize)

	var output *HashWriter
	written := 0

	var saved string
	for {
		req := &Request{
			Hash:  want,
			Start: written,
			End:   written + chunkSize,
		}
		rep := &Response{Chunk: buf}
		err := c.fetchChunk(req, rep)
		if err != nil || !rep.Have {
			return false, 0, err
		}

		// is this a bug in the rpc package?
		content := rep.Chunk[:rep.Size]

		if len(content) < chunkSize && written == 0 {
			saved = c.store.Save(content)
			written = len(content)
			break
		} else if output == nil {
			output = c.store.NewHashWriter()
			defer output.Close()
		}

		n, err := output.Write(content)
		written += n
		if err != nil {
			return false, 0, err
		}
		if len(content) < chunkSize {
			break
		}
	}
	if output != nil {
		output.Close()
		saved = string(output.Sum())
	}
	c.store.addThroughput(int64(written), 0)
	if want != saved {
		log.Fatalf("file corruption: got %x want %x", saved, want)
	}
	return true, written, nil
}
<|endoftext|>


<commit_before>// Copyright 2017 Authors of Cilium
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package RuntimeTest

import (
	"context"
	"fmt"
	"sync"

	. "github.com/cilium/cilium/test/ginkgo-ext"
	"github.com/cilium/cilium/test/helpers"

	. "github.com/onsi/gomega"
	"github.com/sirupsen/logrus"
)

var _ = Describe("RuntimeValidatedKafka", func() {

	var once sync.Once
	var logger *logrus.Entry
	var vm *helpers.SSHMeta
	var monitorStop func() error

	var allowedTopic string = "allowedTopic"
	var disallowTopic string = "disallowTopic"
	var produceCmd string = fmt.Sprintf(
		"echo \"Message 0\" | docker exec -i client /opt/kafka/bin/kafka-console-producer.sh "+
			"--broker-list kafka:9092 --topic %s", allowedTopic)
	var listTopicsCmd string = "/opt/kafka/bin/kafka-topics.sh --list --zookeeper zook:2181"
	var MaxMessages int = 6
	var client string = "client"

	initialize := func() {
		logger = log.WithFields(logrus.Fields{"testName": "RuntimeKafka"})
		logger.Info("Starting")
		vm = helpers.CreateNewRuntimeHelper(helpers.Runtime, logger)
	}

	containers := func(mode string) {

		images := map[string]string{
			"zook":   "digitalwonderland/zookeeper",
			"client": "cilium/kafkaclient2",
		}

		switch mode {
		case "create":
			for k, v := range images {
				vm.ContainerCreate(k, v, helpers.CiliumDockerNetwork, fmt.Sprintf("-l id.%s", k))
			}
			zook, err := vm.ContainerInspectNet("zook")
			Expect(err).Should(BeNil())

			vm.ContainerCreate("kafka", "wurstmeister/kafka", helpers.CiliumDockerNetwork, fmt.Sprintf(
				"-l id.kafka -e KAFKA_ZOOKEEPER_CONNECT=%s:2181 -e KAFKA_ZOOKEEPER_SESSION_TIMEOUT_MS=20000 -e KAFKA_LISTENERS=PLAINTEXT://:9092 -e KAFKA_ZOOKEEPER_CONNECTION_TIMEOUT_MS=20000", zook["IPv4"]))

		case "delete":
			for k := range images {
				vm.ContainerRm(k)
			}
			vm.ContainerRm("kafka")
		}
	}

	createTopic := func(name string) {
		logger.Infof("Creating new kafka topic %s", name)
		res := vm.ContainerExec(client, fmt.Sprintf(
			"/opt/kafka/bin/kafka-topics.sh --create --zookeeper zook:2181 "+
				"--replication-factor 1 --partitions 1 --topic %s", name))
		res.ExpectSuccess("Unable to create topic  %s", name)
	}
	consumerCmd := func(topic string, maxMsg int) string {
		return fmt.Sprintf("/opt/kafka/bin/kafka-console-consumer.sh --bootstrap-server "+
			"kafka:9092 --topic %s --max-messages %d --timeout-ms 300000 --from-beginning",
			topic, maxMsg)
	}

	consumer := func(topic string, maxMsg int) *helpers.CmdRes {
		return vm.ContainerExec(client, consumerCmd(topic, maxMsg))
	}

	producer := func(topic string, message string) {
		cmd := fmt.Sprintf(
			"echo %s | docker exec -i %s /opt/kafka/bin/kafka-console-producer.sh "+
				"--broker-list kafka:9092 --topic %s",
			message, client, topic)
		vm.Exec(cmd)
	}

	// WaitKafkaBroker waits for the broker to be ready, by executing
	// a produce request on existing topics and waiting for a response from broker
	waitForKafkaBroker := func(pod string, cmd string) error {
		body := func() bool {
			res := vm.ContainerExec(pod, cmd)
			return res.WasSuccessful()
		}
		err := helpers.WithTimeout(body, "Kafka Broker not ready", &helpers.TimeoutConfig{Timeout: 150})
		return err
	}

	BeforeEach(func() {
		once.Do(initialize)
		containers("create")
		epsReady := vm.WaitEndpointsReady()
		Expect(epsReady).Should(BeTrue())
		// Waiting for kafka broker to be up.
		err := waitForKafkaBroker(client, listTopicsCmd)
		Expect(err).To(BeNil(), "Kafka broker failed to come up")
		By("Creating kafka topics")
		createTopic(allowedTopic)
		createTopic(disallowTopic)
		By("Listing created Kafka topics")
		res := vm.ContainerExec(client, listTopicsCmd)
		res.ExpectSuccess("Cannot list kafka topics")
	})

	AfterEach(func() {
		vm.PolicyDelAll()
		containers("delete")
	})

	JustBeforeEach(func() {
		monitorStop = vm.MonitorStart()
	})

	JustAfterEach(func() {
		vm.ValidateNoErrorsOnLogs(CurrentGinkgoTestDescription().Duration)
		Expect(monitorStop()).To(BeNil(), "cannot stop monitor command")
	})

	AfterFailed(func() {
		vm.ReportFailed("cilium policy get")
	})

	It("Kafka Policy Ingress", func() {
		_, err := vm.PolicyImportAndWait(vm.GetFullPath("Policies-kafka.json"), 300)
		Expect(err).Should(BeNil())

		endPoints, err := vm.PolicyEndpointsSummary()
		Expect(err).Should(BeNil(), "Cannot get endpoint list")
		Expect(endPoints[helpers.Enabled]).To(Equal(1),
			"Check number of endpoints with policy enforcement enabled")
		Expect(endPoints[helpers.Disabled]).To(Equal(2),
			"Check number of endpoints with policy enforcement disabled")

		// Waiting for kafka broker to be up.
		err = waitForKafkaBroker(client, produceCmd)
		Expect(err).To(BeNil(), "Kafka broker failed to come up")

		By("Allowed topic")

		By("Sending produce request on kafka topic `allowedTopic`")
		for i := 1; i <= MaxMessages-1; i++ {
			producer(allowedTopic, fmt.Sprintf("Message %d", i))
		}

		By("Sending consume request on kafka topic `allowedTopic`")
		res := consumer(allowedTopic, MaxMessages)
		res.ExpectSuccess("Failed to consume messages from kafka topic `allowedTopic`")
		Expect(res.CombineOutput().String()).
			Should(ContainSubstring("Processed a total of %d messages", MaxMessages),
				"Kafka did not process the expected number of messages")

		By("Disable topic")
		res = consumer(disallowTopic, MaxMessages)
		res.ExpectFail("Kafka consumer can access to disallowTopic")
	})

	It("Kafka Policy Role Ingress", func() {
		_, err := vm.PolicyImportAndWait(vm.GetFullPath("Policies-kafka-Role.json"), 300)
		Expect(err).Should(BeNil(), "Expected nil got %s while importing policy Policies-kafka-Role.json", err)

		endPoints, err := vm.PolicyEndpointsSummary()
		Expect(err).Should(BeNil(), "Expect nil. Failed to apply policy on all endpoints with error :%s", err)
		Expect(endPoints[helpers.Enabled]).To(Equal(1), "Expected 1 endpoint to be policy enabled. Policy enforcement failed")
		Expect(endPoints[helpers.Disabled]).To(Equal(2), "Expected 2 endpoint to be policy disabled. Policy enforcement failed")

		// Waiting for kafka broker to be up.
		err = waitForKafkaBroker(client, produceCmd)
		Expect(err).To(BeNil(), "Kafka broker failed to come up")

		By("By sending produce/consume request on topic `allowedTopic`")

		By("Sending produce request on kafka topic `allowedTopic`")
		for i := 1; i <= MaxMessages-1; i++ {
			producer(allowedTopic, fmt.Sprintf("Message %d", i))
		}

		By("Sending consume request on kafka topic `allowedTopic`")
		res := consumer(allowedTopic, MaxMessages)
		res.ExpectSuccess("Failed to consume messages from kafka topic `allowedTopic`")
		Expect(res.CombineOutput().String()).
			Should(ContainSubstring("Processed a total of %d messages", MaxMessages),
				"Kafka did not process the expected number of messages")

		By("Disable topic")
		// Consumer timeout didn't work correctly, so make sure that AUTH is present in the reply
		ctx, cancel := context.WithCancel(context.Background())
		defer cancel()
		res = vm.ExecContext(ctx, fmt.Sprintf(
			"docker exec -i %s %s", client, consumerCmd(disallowTopic, MaxMessages)))
		err = res.WaitUntilMatch("{disallowTopic=TOPIC_AUTHORIZATION_FAILED}")
		Expect(err).To(BeNil(), "Traffic in disallowTopic is allowed")
	})
})
<commit_msg>Test: Move RuntimeKafka test to use BeforeAll<commit_after>// Copyright 2017 Authors of Cilium
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package RuntimeTest

import (
	"context"
	"fmt"

	. "github.com/cilium/cilium/test/ginkgo-ext"
	"github.com/cilium/cilium/test/helpers"

	. "github.com/onsi/gomega"
	"github.com/sirupsen/logrus"
)

var _ = Describe("RuntimeValidatedKafka", func() {

	var logger *logrus.Entry
	var vm *helpers.SSHMeta
	var monitorStop func() error

	var allowedTopic string = "allowedTopic"
	var disallowTopic string = "disallowTopic"
	var produceCmd string = fmt.Sprintf(
		"echo \"Message 0\" | docker exec -i client /opt/kafka/bin/kafka-console-producer.sh "+
			"--broker-list kafka:9092 --topic %s", allowedTopic)
	var listTopicsCmd string = "/opt/kafka/bin/kafka-topics.sh --list --zookeeper zook:2181"
	var MaxMessages int = 6
	var client string = "client"

	containers := func(mode string) {

		images := map[string]string{
			"zook":   "digitalwonderland/zookeeper",
			"client": "cilium/kafkaclient2",
		}

		switch mode {
		case "create":
			for k, v := range images {
				vm.ContainerCreate(k, v, helpers.CiliumDockerNetwork, fmt.Sprintf("-l id.%s", k))
			}
			zook, err := vm.ContainerInspectNet("zook")
			Expect(err).Should(BeNil())

			vm.ContainerCreate("kafka", "wurstmeister/kafka", helpers.CiliumDockerNetwork, fmt.Sprintf(
				"-l id.kafka -e KAFKA_ZOOKEEPER_CONNECT=%s:2181 -e KAFKA_ZOOKEEPER_SESSION_TIMEOUT_MS=20000 -e KAFKA_LISTENERS=PLAINTEXT://:9092 -e KAFKA_ZOOKEEPER_CONNECTION_TIMEOUT_MS=20000", zook["IPv4"]))

		case "delete":
			for k := range images {
				vm.ContainerRm(k)
			}
			vm.ContainerRm("kafka")
		}
	}

	createTopic := func(name string) {
		logger.Infof("Creating new kafka topic %s", name)
		res := vm.ContainerExec(client, fmt.Sprintf(
			"/opt/kafka/bin/kafka-topics.sh --create --zookeeper zook:2181 "+
				"--replication-factor 1 --partitions 1 --topic %s", name))
		res.ExpectSuccess("Unable to create topic  %s", name)
	}
	consumerCmd := func(topic string, maxMsg int) string {
		return fmt.Sprintf("/opt/kafka/bin/kafka-console-consumer.sh --bootstrap-server "+
			"kafka:9092 --topic %s --max-messages %d --timeout-ms 300000 --from-beginning",
			topic, maxMsg)
	}

	consumer := func(topic string, maxMsg int) *helpers.CmdRes {
		return vm.ContainerExec(client, consumerCmd(topic, maxMsg))
	}

	producer := func(topic string, message string) {
		cmd := fmt.Sprintf(
			"echo %s | docker exec -i %s /opt/kafka/bin/kafka-console-producer.sh "+
				"--broker-list kafka:9092 --topic %s",
			message, client, topic)
		vm.Exec(cmd)
	}

	// WaitKafkaBroker waits for the broker to be ready, by executing
	// a produce request on existing topics and waiting for a response from broker
	waitForKafkaBroker := func(pod string, cmd string) error {
		body := func() bool {
			res := vm.ContainerExec(pod, cmd)
			return res.WasSuccessful()
		}
		err := helpers.WithTimeout(body, "Kafka Broker not ready", &helpers.TimeoutConfig{Timeout: 150})
		return err
	}

	BeforeAll(func() {
		logger = log.WithFields(logrus.Fields{"testName": "RuntimeValidatedKafka"})
		vm = helpers.CreateNewRuntimeHelper(helpers.Runtime, logger)

		containers("create")
		epsReady := vm.WaitEndpointsReady()
		Expect(epsReady).Should(BeTrue(), "Endpoints are not ready after timeout")

		err := waitForKafkaBroker(client, listTopicsCmd)
		Expect(err).To(BeNil(), "Kafka broker failed to come up")

		By("Creating kafka topics")
		createTopic(allowedTopic)
		createTopic(disallowTopic)

		By("Listing created Kafka topics")
		res := vm.ContainerExec(client, listTopicsCmd)
		res.ExpectSuccess("Cannot list kafka topics")
	})

	AfterEach(func() {
		vm.PolicyDelAll()

	})

	AfterAll(func() {
		containers("delete")
	})

	JustBeforeEach(func() {
		monitorStop = vm.MonitorStart()
	})

	JustAfterEach(func() {
		vm.ValidateNoErrorsOnLogs(CurrentGinkgoTestDescription().Duration)
		Expect(monitorStop()).To(BeNil(), "cannot stop monitor command")
	})

	AfterFailed(func() {
		vm.ReportFailed("cilium policy get")
	})

	It("Kafka Policy Ingress", func() {
		_, err := vm.PolicyImportAndWait(vm.GetFullPath("Policies-kafka.json"), 300)
		Expect(err).Should(BeNil())

		endPoints, err := vm.PolicyEndpointsSummary()
		Expect(err).Should(BeNil(), "Cannot get endpoint list")
		Expect(endPoints[helpers.Enabled]).To(Equal(1),
			"Check number of endpoints with policy enforcement enabled")
		Expect(endPoints[helpers.Disabled]).To(Equal(2),
			"Check number of endpoints with policy enforcement disabled")

		// Waiting for kafka broker to be up.
		err = waitForKafkaBroker(client, produceCmd)
		Expect(err).To(BeNil(), "Kafka broker failed to come up")

		By("Allowed topic")

		By("Sending produce request on kafka topic `allowedTopic`")
		for i := 1; i <= MaxMessages-1; i++ {
			producer(allowedTopic, fmt.Sprintf("Message %d", i))
		}

		By("Sending consume request on kafka topic `allowedTopic`")
		res := consumer(allowedTopic, MaxMessages)
		res.ExpectSuccess("Failed to consume messages from kafka topic `allowedTopic`")
		Expect(res.CombineOutput().String()).
			Should(ContainSubstring("Processed a total of %d messages", MaxMessages),
				"Kafka did not process the expected number of messages")

		By("Disable topic")
		res = consumer(disallowTopic, MaxMessages)
		res.ExpectFail("Kafka consumer can access to disallowTopic")
	})

	It("Kafka Policy Role Ingress", func() {
		_, err := vm.PolicyImportAndWait(vm.GetFullPath("Policies-kafka-Role.json"), 300)
		Expect(err).Should(BeNil(), "Expected nil got %s while importing policy Policies-kafka-Role.json", err)

		endPoints, err := vm.PolicyEndpointsSummary()
		Expect(err).Should(BeNil(), "Expect nil. Failed to apply policy on all endpoints with error :%s", err)
		Expect(endPoints[helpers.Enabled]).To(Equal(1), "Expected 1 endpoint to be policy enabled. Policy enforcement failed")
		Expect(endPoints[helpers.Disabled]).To(Equal(2), "Expected 2 endpoint to be policy disabled. Policy enforcement failed")

		// Waiting for kafka broker to be up.
		err = waitForKafkaBroker(client, produceCmd)
		Expect(err).To(BeNil(), "Kafka broker failed to come up")

		By("By sending produce/consume request on topic `allowedTopic`")

		By("Sending produce request on kafka topic `allowedTopic`")
		for i := 1; i <= MaxMessages-1; i++ {
			producer(allowedTopic, fmt.Sprintf("Message %d", i))
		}

		By("Sending consume request on kafka topic `allowedTopic`")
		res := consumer(allowedTopic, MaxMessages)
		res.ExpectSuccess("Failed to consume messages from kafka topic `allowedTopic`")
		Expect(res.CombineOutput().String()).
			Should(ContainSubstring("Processed a total of %d messages", MaxMessages),
				"Kafka did not process the expected number of messages")

		By("Disable topic")
		// Consumer timeout didn't work correctly, so make sure that AUTH is present in the reply
		ctx, cancel := context.WithCancel(context.Background())
		defer cancel()
		res = vm.ExecContext(ctx, fmt.Sprintf(
			"docker exec -i %s %s", client, consumerCmd(disallowTopic, MaxMessages)))
		err = res.WaitUntilMatch("{disallowTopic=TOPIC_AUTHORIZATION_FAILED}")
		Expect(err).To(BeNil(), "Traffic in disallowTopic is allowed")
	})
})
<|endoftext|>