<commit_before>package benchmark

import (
	"fmt"
	"strings"
	"sync"
	"time"

	"github.com/lxc/lxd/client"
	"github.com/lxc/lxd/lxc/config"
	"github.com/lxc/lxd/shared/api"
	"github.com/lxc/lxd/shared/version"
)

const userConfigKey = "user.lxd-benchmark"

// PrintServerInfo prints out information about the server.
func PrintServerInfo(c lxd.ContainerServer) error {
	server, _, err := c.GetServer()
	if err != nil {
		return err
	}
	env := server.Environment
	fmt.Printf("Test environment:\n")
	fmt.Printf("  Server backend: %s\n", env.Server)
	fmt.Printf("  Server version: %s\n", env.ServerVersion)
	fmt.Printf("  Kernel: %s\n", env.Kernel)
	fmt.Printf("  Kernel architecture: %s\n", env.KernelArchitecture)
	fmt.Printf("  Kernel version: %s\n", env.KernelVersion)
	fmt.Printf("  Storage backend: %s\n", env.Storage)
	fmt.Printf("  Storage version: %s\n", env.StorageVersion)
	fmt.Printf("  Container backend: %s\n", env.Driver)
	fmt.Printf("  Container version: %s\n", env.DriverVersion)
	fmt.Printf("\n")
	return nil
}

// SpawnContainers launches a set of containers.
func SpawnContainers(c lxd.ContainerServer, count int, parallel int, image string, privileged bool, start bool, freeze bool) (time.Duration, error) {
	var duration time.Duration

	batchSize, err := getBatchSize(parallel)
	if err != nil {
		return duration, err
	}

	printTestConfig(count, batchSize, image, privileged, freeze)

	fingerprint, err := ensureImage(c, image)
	if err != nil {
		return duration, err
	}

	startContainer := func(index int, wg *sync.WaitGroup) {
		defer wg.Done()

		name := getContainerName(count, index)

		err := createContainer(c, fingerprint, name, privileged)
		if err != nil {
			logf("Failed to spawn container '%s': %s", name, err)
			return
		}

		if start {
			err := startContainer(c, name)
			if err != nil {
				logf("Failed to start container '%s': %s", name, err)
				return
			}

			if freeze {
				err := freezeContainer(c, name)
				if err != nil {
					logf("Failed to freeze container '%s': %s", name, err)
					return
				}
			}
		}
	}

	duration = processBatch(count, batchSize, startContainer)
	return duration, nil
}

// CreateContainers create the specified number of containers.
func CreateContainers(c lxd.ContainerServer, count int, parallel int, fingerprint string, privileged bool) (time.Duration, error) {
	var duration time.Duration

	batchSize, err := getBatchSize(parallel)
	if err != nil {
		return duration, err
	}

	createContainer := func(index int, wg *sync.WaitGroup) {
		defer wg.Done()

		name := getContainerName(count, index)

		err := createContainer(c, fingerprint, name, privileged)
		if err != nil {
			logf("Failed to spawn container '%s': %s", name, err)
			return
		}
	}

	duration = processBatch(count, batchSize, createContainer)

	return duration, nil
}

// GetContainers returns containers created by the benchmark.
func GetContainers(c lxd.ContainerServer) ([]api.Container, error) {
	containers := []api.Container{}

	allContainers, err := c.GetContainers()
	if err != nil {
		return containers, err
	}

	for _, container := range allContainers {
		if container.Config[userConfigKey] == "true" {
			containers = append(containers, container)
		}
	}

	return containers, nil
}

// StartContainers starts containers created by the benchmark.
func StartContainers(c lxd.ContainerServer, containers []api.Container, parallel int) (time.Duration, error) {
	var duration time.Duration

	batchSize, err := getBatchSize(parallel)
	if err != nil {
		return duration, err
	}

	count := len(containers)
	logf("Starting %d containers", count)

	startContainer := func(index int, wg *sync.WaitGroup) {
		defer wg.Done()

		container := containers[index]
		if !container.IsActive() {
			err := startContainer(c, container.Name)
			if err != nil {
				logf("Failed to start container '%s': %s", container.Name, err)
				return
			}
		}
	}

	duration = processBatch(count, batchSize, startContainer)
	return duration, nil
}

// StopContainers stops containers created by the benchmark.
func StopContainers(c lxd.ContainerServer, containers []api.Container, parallel int) (time.Duration, error) {
	var duration time.Duration

	batchSize, err := getBatchSize(parallel)
	if err != nil {
		return duration, err
	}

	count := len(containers)
	logf("Stopping %d containers", count)

	stopContainer := func(index int, wg *sync.WaitGroup) {
		defer wg.Done()

		container := containers[index]
		if container.IsActive() {
			err := stopContainer(c, container.Name)
			if err != nil {
				logf("Failed to stop container '%s': %s", container.Name, err)
				return
			}
		}
	}

	duration = processBatch(count, batchSize, stopContainer)
	return duration, nil
}

// DeleteContainers removes containers created by the benchmark.
func DeleteContainers(c lxd.ContainerServer, containers []api.Container, parallel int) (time.Duration, error) {
	var duration time.Duration

	batchSize, err := getBatchSize(parallel)
	if err != nil {
		return duration, err
	}

	count := len(containers)
	logf("Deleting %d containers", count)

	deleteContainer := func(index int, wg *sync.WaitGroup) {
		defer wg.Done()

		container := containers[index]
		name := container.Name
		if container.IsActive() {
			err := stopContainer(c, name)
			if err != nil {
				logf("Failed to stop container '%s': %s", name, err)
				return
			}
		}

		err = deleteContainer(c, name)
		if err != nil {
			logf("Failed to delete container: %s", name)
			return
		}
	}

	duration = processBatch(count, batchSize, deleteContainer)
	return duration, nil
}

func ensureImage(c lxd.ContainerServer, image string) (string, error) {
	var fingerprint string

	if strings.Contains(image, ":") {
		defaultConfig := config.DefaultConfig
		defaultConfig.UserAgent = version.UserAgent

		remote, fp, err := defaultConfig.ParseRemote(image)
		if err != nil {
			return "", err
		}
		fingerprint = fp

		imageServer, err := defaultConfig.GetImageServer(remote)
		if err != nil {
			return "", err
		}

		if fingerprint == "" {
			fingerprint = "default"
		}

		alias, _, err := imageServer.GetImageAlias(fingerprint)
		if err == nil {
			fingerprint = alias.Target
		}

		_, _, err = c.GetImage(fingerprint)
		if err != nil {
			logf("Importing image into local store: %s", fingerprint)
			image, _, err := imageServer.GetImage(fingerprint)
			if err != nil {
				logf("Failed to import image: %s", err)
				return "", err
			}

			err = copyImage(c, imageServer, *image)
			if err != nil {
				logf("Failed to import image: %s", err)
				return "", err
			}
		}
	} else {
		alias, _, err := c.GetImageAlias(image)
		if err == nil {
			fingerprint = alias.Target
		} else {
			_, _, err = c.GetImage(image)
		}

		if err != nil {
			logf("Image not found in local store: %s", image)
			return "", err
		}
		fingerprint = image
	}

	logf("Found image in local store: %s", fingerprint)
	return fingerprint, nil
}
<commit_msg>lxd-benchmark: Fix local image handling<commit_after>package benchmark

import (
	"fmt"
	"strings"
	"sync"
	"time"

	"github.com/lxc/lxd/client"
	"github.com/lxc/lxd/lxc/config"
	"github.com/lxc/lxd/shared/api"
	"github.com/lxc/lxd/shared/version"
)

const userConfigKey = "user.lxd-benchmark"

// PrintServerInfo prints out information about the server.
func PrintServerInfo(c lxd.ContainerServer) error {
	server, _, err := c.GetServer()
	if err != nil {
		return err
	}
	env := server.Environment
	fmt.Printf("Test environment:\n")
	fmt.Printf("  Server backend: %s\n", env.Server)
	fmt.Printf("  Server version: %s\n", env.ServerVersion)
	fmt.Printf("  Kernel: %s\n", env.Kernel)
	fmt.Printf("  Kernel architecture: %s\n", env.KernelArchitecture)
	fmt.Printf("  Kernel version: %s\n", env.KernelVersion)
	fmt.Printf("  Storage backend: %s\n", env.Storage)
	fmt.Printf("  Storage version: %s\n", env.StorageVersion)
	fmt.Printf("  Container backend: %s\n", env.Driver)
	fmt.Printf("  Container version: %s\n", env.DriverVersion)
	fmt.Printf("\n")
	return nil
}

// SpawnContainers launches a set of containers.
func SpawnContainers(c lxd.ContainerServer, count int, parallel int, image string, privileged bool, start bool, freeze bool) (time.Duration, error) {
	var duration time.Duration

	batchSize, err := getBatchSize(parallel)
	if err != nil {
		return duration, err
	}

	printTestConfig(count, batchSize, image, privileged, freeze)

	fingerprint, err := ensureImage(c, image)
	if err != nil {
		return duration, err
	}

	startContainer := func(index int, wg *sync.WaitGroup) {
		defer wg.Done()

		name := getContainerName(count, index)

		err := createContainer(c, fingerprint, name, privileged)
		if err != nil {
			logf("Failed to spawn container '%s': %s", name, err)
			return
		}

		if start {
			err := startContainer(c, name)
			if err != nil {
				logf("Failed to start container '%s': %s", name, err)
				return
			}

			if freeze {
				err := freezeContainer(c, name)
				if err != nil {
					logf("Failed to freeze container '%s': %s", name, err)
					return
				}
			}
		}
	}

	duration = processBatch(count, batchSize, startContainer)
	return duration, nil
}

// CreateContainers create the specified number of containers.
func CreateContainers(c lxd.ContainerServer, count int, parallel int, fingerprint string, privileged bool) (time.Duration, error) {
	var duration time.Duration

	batchSize, err := getBatchSize(parallel)
	if err != nil {
		return duration, err
	}

	createContainer := func(index int, wg *sync.WaitGroup) {
		defer wg.Done()

		name := getContainerName(count, index)

		err := createContainer(c, fingerprint, name, privileged)
		if err != nil {
			logf("Failed to spawn container '%s': %s", name, err)
			return
		}
	}

	duration = processBatch(count, batchSize, createContainer)

	return duration, nil
}

// GetContainers returns containers created by the benchmark.
func GetContainers(c lxd.ContainerServer) ([]api.Container, error) {
	containers := []api.Container{}

	allContainers, err := c.GetContainers()
	if err != nil {
		return containers, err
	}

	for _, container := range allContainers {
		if container.Config[userConfigKey] == "true" {
			containers = append(containers, container)
		}
	}

	return containers, nil
}

// StartContainers starts containers created by the benchmark.
func StartContainers(c lxd.ContainerServer, containers []api.Container, parallel int) (time.Duration, error) {
	var duration time.Duration

	batchSize, err := getBatchSize(parallel)
	if err != nil {
		return duration, err
	}

	count := len(containers)
	logf("Starting %d containers", count)

	startContainer := func(index int, wg *sync.WaitGroup) {
		defer wg.Done()

		container := containers[index]
		if !container.IsActive() {
			err := startContainer(c, container.Name)
			if err != nil {
				logf("Failed to start container '%s': %s", container.Name, err)
				return
			}
		}
	}

	duration = processBatch(count, batchSize, startContainer)
	return duration, nil
}

// StopContainers stops containers created by the benchmark.
func StopContainers(c lxd.ContainerServer, containers []api.Container, parallel int) (time.Duration, error) {
	var duration time.Duration

	batchSize, err := getBatchSize(parallel)
	if err != nil {
		return duration, err
	}

	count := len(containers)
	logf("Stopping %d containers", count)

	stopContainer := func(index int, wg *sync.WaitGroup) {
		defer wg.Done()

		container := containers[index]
		if container.IsActive() {
			err := stopContainer(c, container.Name)
			if err != nil {
				logf("Failed to stop container '%s': %s", container.Name, err)
				return
			}
		}
	}

	duration = processBatch(count, batchSize, stopContainer)
	return duration, nil
}

// DeleteContainers removes containers created by the benchmark.
func DeleteContainers(c lxd.ContainerServer, containers []api.Container, parallel int) (time.Duration, error) {
	var duration time.Duration

	batchSize, err := getBatchSize(parallel)
	if err != nil {
		return duration, err
	}

	count := len(containers)
	logf("Deleting %d containers", count)

	deleteContainer := func(index int, wg *sync.WaitGroup) {
		defer wg.Done()

		container := containers[index]
		name := container.Name
		if container.IsActive() {
			err := stopContainer(c, name)
			if err != nil {
				logf("Failed to stop container '%s': %s", name, err)
				return
			}
		}

		err = deleteContainer(c, name)
		if err != nil {
			logf("Failed to delete container: %s", name)
			return
		}
	}

	duration = processBatch(count, batchSize, deleteContainer)
	return duration, nil
}

func ensureImage(c lxd.ContainerServer, image string) (string, error) {
	var fingerprint string

	if strings.Contains(image, ":") {
		defaultConfig := config.DefaultConfig
		defaultConfig.UserAgent = version.UserAgent

		remote, fp, err := defaultConfig.ParseRemote(image)
		if err != nil {
			return "", err
		}
		fingerprint = fp

		imageServer, err := defaultConfig.GetImageServer(remote)
		if err != nil {
			return "", err
		}

		if fingerprint == "" {
			fingerprint = "default"
		}

		alias, _, err := imageServer.GetImageAlias(fingerprint)
		if err == nil {
			fingerprint = alias.Target
		}

		_, _, err = c.GetImage(fingerprint)
		if err != nil {
			logf("Importing image into local store: %s", fingerprint)
			image, _, err := imageServer.GetImage(fingerprint)
			if err != nil {
				logf("Failed to import image: %s", err)
				return "", err
			}

			err = copyImage(c, imageServer, *image)
			if err != nil {
				logf("Failed to import image: %s", err)
				return "", err
			}
		}
	} else {
		fingerprint = image
		alias, _, err := c.GetImageAlias(image)
		if err == nil {
			fingerprint = alias.Target
		} else {
			_, _, err = c.GetImage(image)
		}

		if err != nil {
			logf("Image not found in local store: %s", image)
			return "", err
		}
	}

	logf("Found image in local store: %s", fingerprint)
	return fingerprint, nil
}
<|endoftext|>


<commit_before>//go:build linux
// +build linux

/*
   Copyright The docker Authors.
   Copyright The Moby Authors.
   Copyright The containerd Authors.

   Licensed under the Apache License, Version 2.0 (the "License");
   you may not use this file except in compliance with the License.
   You may obtain a copy of the License at

       http://www.apache.org/licenses/LICENSE-2.0

   Unless required by applicable law or agreed to in writing, software
   distributed under the License is distributed on an "AS IS" BASIS,
   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
   See the License for the specific language governing permissions and
   limitations under the License.
*/

package apparmor

import (
	"bufio"
	"fmt"
	"io"
	"os"
	"path"
	"strconv"
	"strings"
	"text/template"

	exec "golang.org/x/sys/execabs"
)

// NOTE: This code is copied from <github.com/docker/docker/profiles/apparmor>.
//       If you plan to make any changes, please make sure they are also sent
//       upstream.

const dir = "/etc/apparmor.d"

const defaultTemplate = `
{{range $value := .Imports}}
{{$value}}
{{end}}

profile {{.Name}} flags=(attach_disconnected,mediate_deleted) {
{{range $value := .InnerImports}}
  {{$value}}
{{end}}

  network,
  capability,
  file,
  umount,
{{if ge .Version 208096}}
  # Host (privileged) processes may send signals to container processes.
  signal (receive) peer=unconfined,
  # Manager may send signals to container processes.
  signal (receive) peer={{.DaemonProfile}},
  # Container processes may send signals amongst themselves.
  signal (send,receive) peer={{.Name}},
{{end}}

  deny @{PROC}/* w,   # deny write for all files directly in /proc (not in a subdir)
  # deny write to files not in /proc/<number>/** or /proc/sys/**
  deny @{PROC}/{[^1-9],[^1-9][^0-9],[^1-9s][^0-9y][^0-9s],[^1-9][^0-9][^0-9][^0-9]*}/** w,
  deny @{PROC}/sys/[^k]** w,  # deny /proc/sys except /proc/sys/k* (effectively /proc/sys/kernel)
  deny @{PROC}/sys/kernel/{?,??,[^s][^h][^m]**} w,  # deny everything except shm* in /proc/sys/kernel/
  deny @{PROC}/sysrq-trigger rwklx,
  deny @{PROC}/mem rwklx,
  deny @{PROC}/kmem rwklx,
  deny @{PROC}/kcore rwklx,

  deny mount,

  deny /sys/[^f]*/** wklx,
  deny /sys/f[^s]*/** wklx,
  deny /sys/fs/[^c]*/** wklx,
  deny /sys/fs/c[^g]*/** wklx,
  deny /sys/fs/cg[^r]*/** wklx,
  deny /sys/firmware/** rwklx,
  deny /sys/kernel/security/** rwklx,

{{if ge .Version 208095}}
  ptrace (trace,read) peer={{.Name}},
{{end}}
}
`

type data struct {
	Name          string
	Imports       []string
	InnerImports  []string
	DaemonProfile string
	Version       int
}

func cleanProfileName(profile string) string {
	// Normally profiles are suffixed by " (enforce)". AppArmor profiles cannot
	// contain spaces so this doesn't restrict daemon profile names.
	if parts := strings.SplitN(profile, " ", 2); len(parts) >= 1 {
		profile = parts[0]
	}
	if profile == "" {
		profile = "unconfined"
	}
	return profile
}

func loadData(name string) (*data, error) {
	p := data{
		Name: name,
	}

	if macroExists("tunables/global") {
		p.Imports = append(p.Imports, "#include <tunables/global>")
	} else {
		p.Imports = append(p.Imports, "@{PROC}=/proc/")
	}
	if macroExists("abstractions/base") {
		p.InnerImports = append(p.InnerImports, "#include <abstractions/base>")
	}
	ver, err := getVersion()
	if err != nil {
		return nil, fmt.Errorf("get apparmor_parser version: %w", err)
	}
	p.Version = ver

	// Figure out the daemon profile.
	currentProfile, err := os.ReadFile("/proc/self/attr/current")
	if err != nil {
		// If we couldn't get the daemon profile, assume we are running
		// unconfined which is generally the default.
		currentProfile = nil
	}
	p.DaemonProfile = cleanProfileName(string(currentProfile))

	return &p, nil
}

func generate(p *data, o io.Writer) error {
	t, err := template.New("apparmor_profile").Parse(defaultTemplate)
	if err != nil {
		return err
	}
	return t.Execute(o, p)
}

func load(path string) error {
	out, err := aaParser("-Kr", path)
	if err != nil {
		return fmt.Errorf("%s: %s", err, out)
	}
	return nil
}

// macrosExists checks if the passed macro exists.
func macroExists(m string) bool {
	_, err := os.Stat(path.Join(dir, m))
	return err == nil
}

func aaParser(args ...string) (string, error) {
	out, err := exec.Command("apparmor_parser", args...).CombinedOutput()
	if err != nil {
		return "", err
	}
	return string(out), nil
}

func getVersion() (int, error) {
	out, err := aaParser("--version")
	if err != nil {
		return -1, err
	}
	return parseVersion(out)
}

// parseVersion takes the output from `apparmor_parser --version` and returns
// a representation of the {major, minor, patch} version as a single number of
// the form MMmmPPP {major, minor, patch}.
func parseVersion(output string) (int, error) {
	// output is in the form of the following:
	// AppArmor parser version 2.9.1
	// Copyright (C) 1999-2008 Novell Inc.
	// Copyright 2009-2012 Canonical Ltd.

	lines := strings.SplitN(output, "\n", 2)
	words := strings.Split(lines[0], " ")
	version := words[len(words)-1]

	// trim "-beta1" suffix from version="3.0.0-beta1" if exists
	version = strings.SplitN(version, "-", 2)[0]
	// also trim tilde
	version = strings.SplitN(version, "~", 2)[0]

	// split by major minor version
	v := strings.Split(version, ".")
	if len(v) == 0 || len(v) > 3 {
		return -1, fmt.Errorf("parsing version failed for output: `%s`", output)
	}

	// Default the versions to 0.
	var majorVersion, minorVersion, patchLevel int

	majorVersion, err := strconv.Atoi(v[0])
	if err != nil {
		return -1, err
	}

	if len(v) > 1 {
		minorVersion, err = strconv.Atoi(v[1])
		if err != nil {
			return -1, err
		}
	}
	if len(v) > 2 {
		patchLevel, err = strconv.Atoi(v[2])
		if err != nil {
			return -1, err
		}
	}

	// major*10^5 + minor*10^3 + patch*10^0
	numericVersion := majorVersion*1e5 + minorVersion*1e3 + patchLevel
	return numericVersion, nil
}

func isLoaded(name string) (bool, error) {
	f, err := os.Open("/sys/kernel/security/apparmor/profiles")
	if err != nil {
		return false, err
	}
	defer f.Close()
	r := bufio.NewReader(f)
	for {
		p, err := r.ReadString('\n')
		if err == io.EOF {
			break
		}
		if err != nil {
			return false, err
		}
		if strings.HasPrefix(p, name+" ") {
			return true, nil
		}
	}
	return false, nil
}
<commit_msg>Update error message for apparmor parser<commit_after>//go:build linux
// +build linux

/*
   Copyright The docker Authors.
   Copyright The Moby Authors.
   Copyright The containerd Authors.

   Licensed under the Apache License, Version 2.0 (the "License");
   you may not use this file except in compliance with the License.
   You may obtain a copy of the License at

       http://www.apache.org/licenses/LICENSE-2.0

   Unless required by applicable law or agreed to in writing, software
   distributed under the License is distributed on an "AS IS" BASIS,
   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
   See the License for the specific language governing permissions and
   limitations under the License.
*/

package apparmor

import (
	"bufio"
	"fmt"
	"io"
	"os"
	"path"
	"strconv"
	"strings"
	"text/template"

	exec "golang.org/x/sys/execabs"
)

// NOTE: This code is copied from <github.com/docker/docker/profiles/apparmor>.
//       If you plan to make any changes, please make sure they are also sent
//       upstream.

const dir = "/etc/apparmor.d"

const defaultTemplate = `
{{range $value := .Imports}}
{{$value}}
{{end}}

profile {{.Name}} flags=(attach_disconnected,mediate_deleted) {
{{range $value := .InnerImports}}
  {{$value}}
{{end}}

  network,
  capability,
  file,
  umount,
{{if ge .Version 208096}}
  # Host (privileged) processes may send signals to container processes.
  signal (receive) peer=unconfined,
  # Manager may send signals to container processes.
  signal (receive) peer={{.DaemonProfile}},
  # Container processes may send signals amongst themselves.
  signal (send,receive) peer={{.Name}},
{{end}}

  deny @{PROC}/* w,   # deny write for all files directly in /proc (not in a subdir)
  # deny write to files not in /proc/<number>/** or /proc/sys/**
  deny @{PROC}/{[^1-9],[^1-9][^0-9],[^1-9s][^0-9y][^0-9s],[^1-9][^0-9][^0-9][^0-9]*}/** w,
  deny @{PROC}/sys/[^k]** w,  # deny /proc/sys except /proc/sys/k* (effectively /proc/sys/kernel)
  deny @{PROC}/sys/kernel/{?,??,[^s][^h][^m]**} w,  # deny everything except shm* in /proc/sys/kernel/
  deny @{PROC}/sysrq-trigger rwklx,
  deny @{PROC}/mem rwklx,
  deny @{PROC}/kmem rwklx,
  deny @{PROC}/kcore rwklx,

  deny mount,

  deny /sys/[^f]*/** wklx,
  deny /sys/f[^s]*/** wklx,
  deny /sys/fs/[^c]*/** wklx,
  deny /sys/fs/c[^g]*/** wklx,
  deny /sys/fs/cg[^r]*/** wklx,
  deny /sys/firmware/** rwklx,
  deny /sys/kernel/security/** rwklx,

{{if ge .Version 208095}}
  ptrace (trace,read) peer={{.Name}},
{{end}}
}
`

type data struct {
	Name          string
	Imports       []string
	InnerImports  []string
	DaemonProfile string
	Version       int
}

func cleanProfileName(profile string) string {
	// Normally profiles are suffixed by " (enforce)". AppArmor profiles cannot
	// contain spaces so this doesn't restrict daemon profile names.
	if parts := strings.SplitN(profile, " ", 2); len(parts) >= 1 {
		profile = parts[0]
	}
	if profile == "" {
		profile = "unconfined"
	}
	return profile
}

func loadData(name string) (*data, error) {
	p := data{
		Name: name,
	}

	if macroExists("tunables/global") {
		p.Imports = append(p.Imports, "#include <tunables/global>")
	} else {
		p.Imports = append(p.Imports, "@{PROC}=/proc/")
	}
	if macroExists("abstractions/base") {
		p.InnerImports = append(p.InnerImports, "#include <abstractions/base>")
	}
	ver, err := getVersion()
	if err != nil {
		return nil, fmt.Errorf("get apparmor_parser version: %w", err)
	}
	p.Version = ver

	// Figure out the daemon profile.
	currentProfile, err := os.ReadFile("/proc/self/attr/current")
	if err != nil {
		// If we couldn't get the daemon profile, assume we are running
		// unconfined which is generally the default.
		currentProfile = nil
	}
	p.DaemonProfile = cleanProfileName(string(currentProfile))

	return &p, nil
}

func generate(p *data, o io.Writer) error {
	t, err := template.New("apparmor_profile").Parse(defaultTemplate)
	if err != nil {
		return err
	}
	return t.Execute(o, p)
}

func load(path string) error {
	out, err := aaParser("-Kr", path)
	if err != nil {
		return fmt.Errorf("parser error(%q): %w", strings.TrimSpace(out), err)
	}
	return nil
}

// macrosExists checks if the passed macro exists.
func macroExists(m string) bool {
	_, err := os.Stat(path.Join(dir, m))
	return err == nil
}

func aaParser(args ...string) (string, error) {
	out, err := exec.Command("apparmor_parser", args...).CombinedOutput()
	return string(out), err
}

func getVersion() (int, error) {
	out, err := aaParser("--version")
	if err != nil {
		return -1, err
	}
	return parseVersion(out)
}

// parseVersion takes the output from `apparmor_parser --version` and returns
// a representation of the {major, minor, patch} version as a single number of
// the form MMmmPPP {major, minor, patch}.
func parseVersion(output string) (int, error) {
	// output is in the form of the following:
	// AppArmor parser version 2.9.1
	// Copyright (C) 1999-2008 Novell Inc.
	// Copyright 2009-2012 Canonical Ltd.

	lines := strings.SplitN(output, "\n", 2)
	words := strings.Split(lines[0], " ")
	version := words[len(words)-1]

	// trim "-beta1" suffix from version="3.0.0-beta1" if exists
	version = strings.SplitN(version, "-", 2)[0]
	// also trim tilde
	version = strings.SplitN(version, "~", 2)[0]

	// split by major minor version
	v := strings.Split(version, ".")
	if len(v) == 0 || len(v) > 3 {
		return -1, fmt.Errorf("parsing version failed for output: `%s`", output)
	}

	// Default the versions to 0.
	var majorVersion, minorVersion, patchLevel int

	majorVersion, err := strconv.Atoi(v[0])
	if err != nil {
		return -1, err
	}

	if len(v) > 1 {
		minorVersion, err = strconv.Atoi(v[1])
		if err != nil {
			return -1, err
		}
	}
	if len(v) > 2 {
		patchLevel, err = strconv.Atoi(v[2])
		if err != nil {
			return -1, err
		}
	}

	// major*10^5 + minor*10^3 + patch*10^0
	numericVersion := majorVersion*1e5 + minorVersion*1e3 + patchLevel
	return numericVersion, nil
}

func isLoaded(name string) (bool, error) {
	f, err := os.Open("/sys/kernel/security/apparmor/profiles")
	if err != nil {
		return false, err
	}
	defer f.Close()
	r := bufio.NewReader(f)
	for {
		p, err := r.ReadString('\n')
		if err == io.EOF {
			break
		}
		if err != nil {
			return false, err
		}
		if strings.HasPrefix(p, name+" ") {
			return true, nil
		}
	}
	return false, nil
}
<|endoftext|>


<commit_before>package entity

import (
	"fmt"

	"github.com/xiaonanln/go-aoi"
	"github.com/xiaonanln/goworld/engine/common"
	"github.com/xiaonanln/goworld/engine/consts"
	"github.com/xiaonanln/goworld/engine/gwlog"
	"github.com/xiaonanln/goworld/engine/gwutils"
)

const (
	_SPACE_ENTITY_TYPE    = "__space__"
	_SPACE_KIND_ATTR_KEY  = "_K"
	_SPACE_ENABLE_AOI_KEY = "_EnableAOI"

	_DEFAULT_AOI_DISTANCE = 100
)

var (
	nilSpace *Space
)

// Space is the entity type of spaces
//
// Spaces are also entities but with space management logics
type Space struct {
	Entity

	entities EntitySet
	Kind     int
	I        ISpace

	aoiMgr aoi.AOIManager
}

func (space *Space) String() string {
	if space == nil {
		return "nil"
	}

	if space.Kind != 0 {
		return fmt.Sprintf("Space<%d|%s>", space.Kind, space.ID)
	} else {
		return fmt.Sprintf("NilSpace<%s>", space.ID)
	}
}

func (space *Space) DescribeEntityType(desc *EntityTypeDesc) {
	desc.DefineAttr(_SPACE_KIND_ATTR_KEY, "AllClients")
}

func (space *Space) GetSpaceRange() (minX, minY, maxX, maxY Coord) {
	return -1000, -1000, 1000, 1000
}

func (space *Space) GetTowerRange() (minX, minY, maxX, maxY Coord) {
	return -1000, -1000, 1000, 1000
}

// OnInit initialize Space entity
func (space *Space) OnInit() {
	space.entities = EntitySet{}
	space.I = space.Entity.I.(ISpace)

	space.I.OnSpaceInit()
	//space.callCompositiveMethod("OnSpaceInit")
}

// OnSpaceInit is a compositive method for initializing space fields
func (space *Space) OnSpaceInit() {

}

// OnCreated is called when Space entity is created
func (space *Space) OnCreated() {
	//dispatcher_client.GetDispatcherClientForSend().SendNotifyCreateEntity(space.ID)
	space.onSpaceCreated()
	if space.IsNil() {
		gwlog.Infof("nil space is created: %s, all games connected: %v", space, allGamesConnected)
		if allGamesConnected {
			space.I.OnGameReady()
		}
		return
	}

	if consts.DEBUG_SPACES {
		gwlog.Debugf("%s.OnCreated", space)
	}
	space.I.OnSpaceCreated()
	//space.callCompositiveMethod("OnSpaceCreated")
}

func (space *Space) EnableAOI() {
	if space.aoiMgr != nil {
		return
	}

	if len(space.entities) > 0 {
		gwlog.Panicf("%s is already using AOI", space)
	}

	space.Attrs.SetBool(_SPACE_ENABLE_AOI_KEY, true)
	space.aoiMgr = aoi.NewXZListAOICalculator()
}

//func (space *Space) UseTowerAOI(minX, maxX, minY, maxY Coord, towerRange Coord) {
//	if space.aoiMgr != nil || len(space.entities) > 0 {
//		gwlog.Panicf("%s is already using AOI", space)
//	}
//
//	space.aoiMgr = aoi.NewTowerAOIManager(aoi.Coord(minX), aoi.Coord(maxX), aoi.Coord(minY), aoi.Coord(maxY), aoi.Coord(towerRange))
//}

// OnRestored is called when space entity is restored
func (space *Space) OnRestored() {
	space.onSpaceCreated()
	if space.GetBool(_SPACE_KIND_ATTR_KEY) {
		space.EnableAOI()
	}
}

func (space *Space) onSpaceCreated() {
	space.Kind = int(space.GetInt(_SPACE_KIND_ATTR_KEY))
	spaceManager.putSpace(space)

	if space.Kind == 0 {
		if nilSpace != nil {
			gwlog.Panicf("duplicate nil space: %s && %s", nilSpace, space)
		}
		nilSpace = space
		nilSpace.Space = nilSpace
		gwlog.Infof("Created nil space: %s", nilSpace)
		return
	}
}

// OnSpaceCreated is called when space is created
//
// Custom space type can override to provide custom logic
func (space *Space) OnSpaceCreated() {
	if consts.DEBUG_SPACES {
		gwlog.Debugf("Space %s created", space)
	}
}

// OnDestroy is called when Space entity is destroyed
func (space *Space) OnDestroy() {
	space.I.OnSpaceDestroy()
	//space.callCompositiveMethod("OnSpaceDestroy")
	// destroy all entities
	for e := range space.entities {
		e.Destroy()
	}

	spaceManager.delSpace(space.ID)
}

// OnSpaceDestroy is called when space is destroying
//
// Custom space type can override to provide custom logic
func (space *Space) OnSpaceDestroy() {
	if consts.DEBUG_SPACES {
		gwlog.Debugf("Space %s created", space)
	}
}

// IsNil checks if the space is the nil space
func (space *Space) IsNil() bool {
	return space.Kind == 0
}

// CreateEntity creates a new local entity in this space
func (space *Space) CreateEntity(typeName string, pos Vector3) {
	createEntity(typeName, space, pos, "", nil, nil, nil, ccCreate)
}

// LoadEntity loads a entity of specified entityID to the space
//
// If the entity already exists on server, this call has no effect
func (space *Space) LoadEntity(typeName string, entityID common.EntityID, pos Vector3) {
	loadEntityLocally(typeName, entityID, space, pos)
}

func (space *Space) enter(entity *Entity, pos Vector3, isRestore bool) {
	if consts.DEBUG_SPACES {
		gwlog.Debugf("%s.enter <<< %s, avatar count=%d, monster count=%d", space, entity, space.CountEntities("Avatar"), space.CountEntities("Monster"))
	}

	if entity.Space != nilSpace {
		gwlog.Panicf("%s.enter(%s): current space is not nil, but %s", space, entity, entity.Space)
	}

	if space.IsNil() { // enter nil space does nothing
		return
	}

	entity.Space = space
	space.entities.Add(entity)
	entity.Position = pos

	entity.syncInfoFlag |= sifSyncOwnClient | sifSyncNeighborClients

	if !isRestore {
		entity.client.sendCreateEntity(&space.Entity, false) // create Space entity before every other entities

		if space.aoiMgr != nil && entity.IsUseAOI() {
			space.aoiMgr.Enter(&entity.aoi, aoi.Coord(pos.X), aoi.Coord(pos.Z))
		}

		gwutils.RunPanicless(func() {
			space.I.OnEntityEnterSpace(entity)
			//space.callCompositiveMethod("OnEntityEnterSpace", entity)
			entity.I.OnEnterSpace()
			//entity.callCompositiveMethod("OnEnterSpace")
		})
	} else {
		// restoring ...
		if space.aoiMgr != nil && entity.IsUseAOI() {
			client := entity.client
			entity.client = nil
			space.aoiMgr.Enter(&entity.aoi, aoi.Coord(pos.X), aoi.Coord(pos.Z))
			entity.client = client
		}

	}
	//space.verifyAOICorrectness(entity)
}

func (space *Space) leave(entity *Entity) {
	if entity.Space != space {
		gwlog.Panicf("%s.leave(%s): entity is not in this Space", space, entity)
	}

	if space.IsNil() {
		// leaving nil space does nothing
		return
	}

	// remove from Space entities
	space.entities.Del(entity)
	entity.Space = nilSpace

	if space.aoiMgr != nil && entity.IsUseAOI() {
		space.aoiMgr.Leave(&entity.aoi)
	}

	entity.client.sendDestroyEntity(&space.Entity)
	gwutils.RunPanicless(func() {
		space.I.OnEntityLeaveSpace(entity)
		entity.I.OnLeaveSpace(space)
	})
}

func (space *Space) move(entity *Entity, newPos Vector3) {
	if space.aoiMgr == nil {
		return
	}

	entity.Position = newPos
	space.aoiMgr.Moved(&entity.aoi, aoi.Coord(newPos.X), aoi.Coord(newPos.Z))
}

// OnEntityEnterSpace is called when entity enters space
//
// Custom space type can override this function
func (space *Space) OnEntityEnterSpace(entity *Entity) {
	if consts.DEBUG_SPACES {
		gwlog.Debugf("%s ENTER SPACE %s", entity, space)
	}
}

// OnEntityLeaveSpace is called when entity leaves space
//
// Custom space type can override this function
func (space *Space) OnEntityLeaveSpace(entity *Entity) {
	if consts.DEBUG_SPACES {
		gwlog.Debugf("%s LEAVE SPACE %s", entity, space)
	}
}

// CountEntities returns the number of entities of specified type in space
func (space *Space) CountEntities(typeName string) int {
	count := 0
	for e := range space.entities {
		if e.TypeName == typeName {
			count += 1
		}
	}
	return count
}

// GetEntityCount returns the total count of entities in space
func (space *Space) GetEntityCount() int {
	return len(space.entities)
}

// ForEachEntity visits all entities in space and call function f with each entity
func (space *Space) ForEachEntity(f func(e *Entity)) {
	for e := range space.entities {
		f(e)
	}
}

// GetEntity returns the entity in space with specified ID, nil otherwise
func (space *Space) GetEntity(entityID common.EntityID) *Entity {
	entity := GetEntity(entityID)
	if entity == nil {
		return nil
	}

	if space.entities.Contains(entity) {
		return entity
	} else {
		return nil
	}
}

// aoi Management
func (space *Space) addToAOI(entity *Entity) {

}

// OnGameReady is called when the game server is ready on NilSpace only
func (space *Space) OnGameReady() {
	gwlog.Warnf("Game server is ready. Override function %T.OnGameReady to write your own game logic!", space.I)
}
<commit_msg>enable AOI after restore space<commit_after>package entity

import (
	"fmt"

	"github.com/xiaonanln/go-aoi"
	"github.com/xiaonanln/goworld/engine/common"
	"github.com/xiaonanln/goworld/engine/consts"
	"github.com/xiaonanln/goworld/engine/gwlog"
	"github.com/xiaonanln/goworld/engine/gwutils"
)

const (
	_SPACE_ENTITY_TYPE    = "__space__"
	_SPACE_KIND_ATTR_KEY  = "_K"
	_SPACE_ENABLE_AOI_KEY = "_EnableAOI"

	_DEFAULT_AOI_DISTANCE = 100
)

var (
	nilSpace *Space
)

// Space is the entity type of spaces
//
// Spaces are also entities but with space management logics
type Space struct {
	Entity

	entities EntitySet
	Kind     int
	I        ISpace

	aoiMgr aoi.AOIManager
}

func (space *Space) String() string {
	if space == nil {
		return "nil"
	}

	if space.Kind != 0 {
		return fmt.Sprintf("Space<%d|%s>", space.Kind, space.ID)
	} else {
		return fmt.Sprintf("NilSpace<%s>", space.ID)
	}
}

func (space *Space) DescribeEntityType(desc *EntityTypeDesc) {
	desc.DefineAttr(_SPACE_KIND_ATTR_KEY, "AllClients")
}

func (space *Space) GetSpaceRange() (minX, minY, maxX, maxY Coord) {
	return -1000, -1000, 1000, 1000
}

func (space *Space) GetTowerRange() (minX, minY, maxX, maxY Coord) {
	return -1000, -1000, 1000, 1000
}

// OnInit initialize Space entity
func (space *Space) OnInit() {
	space.entities = EntitySet{}
	space.I = space.Entity.I.(ISpace)

	space.I.OnSpaceInit()
	//space.callCompositiveMethod("OnSpaceInit")
}

// OnSpaceInit is a compositive method for initializing space fields
func (space *Space) OnSpaceInit() {

}

// OnCreated is called when Space entity is created
func (space *Space) OnCreated() {
	//dispatcher_client.GetDispatcherClientForSend().SendNotifyCreateEntity(space.ID)
	space.onSpaceCreated()
	if space.IsNil() {
		gwlog.Infof("nil space is created: %s, all games connected: %v", space, allGamesConnected)
		if allGamesConnected {
			space.I.OnGameReady()
		}
		return
	}

	if consts.DEBUG_SPACES {
		gwlog.Debugf("%s.OnCreated", space)
	}
	space.I.OnSpaceCreated()
	//space.callCompositiveMethod("OnSpaceCreated")
}

func (space *Space) EnableAOI() {
	if space.aoiMgr != nil {
		return
	}

	if len(space.entities) > 0 {
		gwlog.Panicf("%s is already using AOI", space)
	}

	space.Attrs.SetBool(_SPACE_ENABLE_AOI_KEY, true)
	space.aoiMgr = aoi.NewXZListAOICalculator()
}

//func (space *Space) UseTowerAOI(minX, maxX, minY, maxY Coord, towerRange Coord) {
//	if space.aoiMgr != nil || len(space.entities) > 0 {
//		gwlog.Panicf("%s is already using AOI", space)
//	}
//
//	space.aoiMgr = aoi.NewTowerAOIManager(aoi.Coord(minX), aoi.Coord(maxX), aoi.Coord(minY), aoi.Coord(maxY), aoi.Coord(towerRange))
//}

// OnRestored is called when space entity is restored
func (space *Space) OnRestored() {
	space.onSpaceCreated()
	gwlog.Debugf("space %s restored: atts=%+v", space, space.Attrs)
	if space.GetBool(_SPACE_KIND_ATTR_KEY) {
		space.EnableAOI()
	}
}

func (space *Space) onSpaceCreated() {
	space.Kind = int(space.GetInt(_SPACE_KIND_ATTR_KEY))
	spaceManager.putSpace(space)

	if space.Kind == 0 {
		if nilSpace != nil {
			gwlog.Panicf("duplicate nil space: %s && %s", nilSpace, space)
		}
		nilSpace = space
		nilSpace.Space = nilSpace
		gwlog.Infof("Created nil space: %s", nilSpace)
		return
	}
}

// OnSpaceCreated is called when space is created
//
// Custom space type can override to provide custom logic
func (space *Space) OnSpaceCreated() {
	if consts.DEBUG_SPACES {
		gwlog.Debugf("Space %s created", space)
	}
}

// OnDestroy is called when Space entity is destroyed
func (space *Space) OnDestroy() {
	space.I.OnSpaceDestroy()
	//space.callCompositiveMethod("OnSpaceDestroy")
	// destroy all entities
	for e := range space.entities {
		e.Destroy()
	}

	spaceManager.delSpace(space.ID)
}

// OnSpaceDestroy is called when space is destroying
//
// Custom space type can override to provide custom logic
func (space *Space) OnSpaceDestroy() {
	if consts.DEBUG_SPACES {
		gwlog.Debugf("Space %s created", space)
	}
}

// IsNil checks if the space is the nil space
func (space *Space) IsNil() bool {
	return space.Kind == 0
}

// CreateEntity creates a new local entity in this space
func (space *Space) CreateEntity(typeName string, pos Vector3) {
	createEntity(typeName, space, pos, "", nil, nil, nil, ccCreate)
}

// LoadEntity loads a entity of specified entityID to the space
//
// If the entity already exists on server, this call has no effect
func (space *Space) LoadEntity(typeName string, entityID common.EntityID, pos Vector3) {
	loadEntityLocally(typeName, entityID, space, pos)
}

func (space *Space) enter(entity *Entity, pos Vector3, isRestore bool) {
	if consts.DEBUG_SPACES {
		gwlog.Debugf("%s.enter <<< %s, avatar count=%d, monster count=%d", space, entity, space.CountEntities("Avatar"), space.CountEntities("Monster"))
	}

	if entity.Space != nilSpace {
		gwlog.Panicf("%s.enter(%s): current space is not nil, but %s", space, entity, entity.Space)
	}

	if space.IsNil() { // enter nil space does nothing
		return
	}

	entity.Space = space
	space.entities.Add(entity)
	entity.Position = pos

	entity.syncInfoFlag |= sifSyncOwnClient | sifSyncNeighborClients

	if !isRestore {
		entity.client.sendCreateEntity(&space.Entity, false) // create Space entity before every other entities

		if space.aoiMgr != nil && entity.IsUseAOI() {
			space.aoiMgr.Enter(&entity.aoi, aoi.Coord(pos.X), aoi.Coord(pos.Z))
		}

		gwutils.RunPanicless(func() {
			space.I.OnEntityEnterSpace(entity)
			//space.callCompositiveMethod("OnEntityEnterSpace", entity)
			entity.I.OnEnterSpace()
			//entity.callCompositiveMethod("OnEnterSpace")
		})
	} else {
		// restoring ...
		if space.aoiMgr != nil && entity.IsUseAOI() {
			client := entity.client
			entity.client = nil
			space.aoiMgr.Enter(&entity.aoi, aoi.Coord(pos.X), aoi.Coord(pos.Z))
			entity.client = client
		}

	}
	//space.verifyAOICorrectness(entity)
}

func (space *Space) leave(entity *Entity) {
	if entity.Space != space {
		gwlog.Panicf("%s.leave(%s): entity is not in this Space", space, entity)
	}

	if space.IsNil() {
		// leaving nil space does nothing
		return
	}

	// remove from Space entities
	space.entities.Del(entity)
	entity.Space = nilSpace

	if space.aoiMgr != nil && entity.IsUseAOI() {
		space.aoiMgr.Leave(&entity.aoi)
	}

	entity.client.sendDestroyEntity(&space.Entity)
	gwutils.RunPanicless(func() {
		space.I.OnEntityLeaveSpace(entity)
		entity.I.OnLeaveSpace(space)
	})
}

func (space *Space) move(entity *Entity, newPos Vector3) {
	if space.aoiMgr == nil {
		return
	}

	entity.Position = newPos
	space.aoiMgr.Moved(&entity.aoi, aoi.Coord(newPos.X), aoi.Coord(newPos.Z))
}

// OnEntityEnterSpace is called when entity enters space
//
// Custom space type can override this function
func (space *Space) OnEntityEnterSpace(entity *Entity) {
	if consts.DEBUG_SPACES {
		gwlog.Debugf("%s ENTER SPACE %s", entity, space)
	}
}

// OnEntityLeaveSpace is called when entity leaves space
//
// Custom space type can override this function
func (space *Space) OnEntityLeaveSpace(entity *Entity) {
	if consts.DEBUG_SPACES {
		gwlog.Debugf("%s LEAVE SPACE %s", entity, space)
	}
}

// CountEntities returns the number of entities of specified type in space
func (space *Space) CountEntities(typeName string) int {
	count := 0
	for e := range space.entities {
		if e.TypeName == typeName {
			count += 1
		}
	}
	return count
}

// GetEntityCount returns the total count of entities in space
func (space *Space) GetEntityCount() int {
	return len(space.entities)
}

// ForEachEntity visits all entities in space and call function f with each entity
func (space *Space) ForEachEntity(f func(e *Entity)) {
	for e := range space.entities {
		f(e)
	}
}

// GetEntity returns the entity in space with specified ID, nil otherwise
func (space *Space) GetEntity(entityID common.EntityID) *Entity {
	entity := GetEntity(entityID)
	if entity == nil {
		return nil
	}

	if space.entities.Contains(entity) {
		return entity
	} else {
		return nil
	}
}

// aoi Management
func (space *Space) addToAOI(entity *Entity) {

}

// OnGameReady is called when the game server is ready on NilSpace only
func (space *Space) OnGameReady() {
	gwlog.Warnf("Game server is ready. Override function %T.OnGameReady to write your own game logic!", space.I)
}
<|endoftext|>


<commit_before>/*
** Copyright 2014 Edward Walker
**
** Licensed under the Apache License, Version 2.0 (the "License");
** you may not use this file except in compliance with the License.
** You may obtain a copy of the License at
**
** http ://www.apache.org/licenses/LICENSE-2.0
**
** Unless required by applicable law or agreed to in writing, software
** distributed under the License is distributed on an "AS IS" BASIS,
** WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
** See the License for the specific language governing permissions and
** limitations under the License.
**
** Description: Describes the parameters of the Supper Vector Machine solver
** @author: Ed Walker
 */
package libSvm

const (
	C_SVC       = iota
	NU_SVC      = iota
	ONE_CLASS   = iota
	EPSILON_SVR = iota
	NU_SVR      = iota
)

const (
	LINEAR      = iota
	POLY        = iota
	RBF         = iota
	SIGMOID     = iota
	PRECOMPUTED = iota
)

var svm_type_string = []string{"c_svc", "nu_svc", "one_class", "epsilon_svr", "nu_svr"}
var kernel_type_string = []string{"linear", "polynomial", "rbf", "sigmoid", "precomputed"}

type Parameter struct {
	SvmType    int     // Support vector type
	KernelType int     // Kernel type
	Degree     int     // Degree used in polynomial kernel
	Gamma      float64 // Gamma used in rbf, polynomial, and sigmoid kernel
	Coef0      float64 // Coef0 used in polynomial and sigmoid kernel

	Eps         float64 // stopping criteria
	C           float64 // penality
	NrWeight    int
	WeightLabel []int
	Weight      []float64
	Nu          float64
	P           float64
	Probability bool // Should probability estimation be performed?
	CacheSize   int  // Size of Q matrix cache
	QuietMode   bool // quiet mode
	NumCPU      int  // Number of CPUs to use
}

func NewParameter() *Parameter {
	return &Parameter{SvmType: C_SVC, KernelType: RBF, Degree: 3, Gamma: 0, Coef0: 0, Nu: 0.5, C: 1, Eps: 1e-3, P: 0.1,
		NrWeight: 0, Probability: false, CacheSize: 100, QuietMode: false, NumCPU: -1}
}
<commit_msg>Added version number<commit_after>/*
** Copyright 2014 Edward Walker
**
** Licensed under the Apache License, Version 2.0 (the "License");
** you may not use this file except in compliance with the License.
** You may obtain a copy of the License at
**
** http ://www.apache.org/licenses/LICENSE-2.0
**
** Unless required by applicable law or agreed to in writing, software
** distributed under the License is distributed on an "AS IS" BASIS,
** WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
** See the License for the specific language governing permissions and
** limitations under the License.
**
** Description: Describes the parameters of the Supper Vector Machine solver
** @author: Ed Walker
 */
package libSvm

const LibSvmCudaVersion = 0.318

const (
	C_SVC       = iota
	NU_SVC      = iota
	ONE_CLASS   = iota
	EPSILON_SVR = iota
	NU_SVR      = iota
)

const (
	LINEAR      = iota
	POLY        = iota
	RBF         = iota
	SIGMOID     = iota
	PRECOMPUTED = iota
)

var svm_type_string = []string{"c_svc", "nu_svc", "one_class", "epsilon_svr", "nu_svr"}
var kernel_type_string = []string{"linear", "polynomial", "rbf", "sigmoid", "precomputed"}

type Parameter struct {
	SvmType    int     // Support vector type
	KernelType int     // Kernel type
	Degree     int     // Degree used in polynomial kernel
	Gamma      float64 // Gamma used in rbf, polynomial, and sigmoid kernel
	Coef0      float64 // Coef0 used in polynomial and sigmoid kernel

	Eps         float64 // stopping criteria
	C           float64 // penality
	NrWeight    int
	WeightLabel []int
	Weight      []float64
	Nu          float64
	P           float64
	Probability bool // Should probability estimation be performed?
	CacheSize   int  // Size of Q matrix cache
	QuietMode   bool // quiet mode
	NumCPU      int  // Number of CPUs to use
}

func NewParameter() *Parameter {
	return &Parameter{SvmType: C_SVC, KernelType: RBF, Degree: 3, Gamma: 0, Coef0: 0, Nu: 0.5, C: 1, Eps: 1e-3, P: 0.1,
		NrWeight: 0, Probability: false, CacheSize: 100, QuietMode: false, NumCPU: -1}
}
<|endoftext|>


<commit_before>package gohistogram

// Copyright (c) 2013 VividCortex, Inc. All rights reserved.
// Please see the LICENSE file for applicable license terms.

import (
	"fmt"
)

type NumericHistogram struct {
	bins    []bin
	maxbins int
	total   uint64
}

// NewHistogram returns a new NumericHistogram with a maximum of n bins.
//
// There is no "optimal" bin count, but somewhere between 20 and 80 bins
// should be sufficient.
func NewHistogram(n int) *NumericHistogram {
	return &NumericHistogram{
		bins:    make([]bin, 0),
		maxbins: n,
		total:   0,
	}
}

func (h *NumericHistogram) Add(n float64) {
	defer h.trim()
	h.total++
	for i := range h.bins {
		if h.bins[i].value == n {
			h.bins[i].count++
			return
		}

		if h.bins[i].value > n {

			newbin := bin{value: n, count: 1}
			head := append(make([]bin, 0), h.bins[0:i]...)

			head = append(head, newbin)
			tail := h.bins[i:]
			h.bins = append(head, tail...)
			return
		}
	}

	h.bins = append(h.bins, bin{count: 1, value: n})
}

func (h *NumericHistogram) Quantile(q float64) float64 {
	count := q * float64(h.total)
	for i := range h.bins {
		count -= float64(h.bins[i].count)

		if count <= 0 {
			return h.bins[i].value
		}
	}

	return -1
}

// CDF returns the value of the cumulative distribution function
// at x
func (h *NumericHistogram) CDF(x float64) float64 {
	count := 0.0
	for i := range h.bins {
		if h.bins[i].value <= x {
			count += float64(h.bins[i].count)
		}
	}

	return count / float64(h.total)
}

// Mean returns the sample mean of the distribution
func (h *NumericHistogram) Mean() float64 {
	if h.total == 0 {
		return 0
	}

	sum := 0.0

	for i := range h.bins {
		sum += h.bins[i].value * h.bins[i].count
	}

	return sum / float64(h.total)
}

// Variance returns the variance of the distribution
func (h *NumericHistogram) Variance() float64 {
	if h.total == 0 {
		return 0
	}

	sum := 0.0
	mean := h.Mean()

	for i := range h.bins {
		sum += (h.bins[i].count * (h.bins[i].value - mean) * (h.bins[i].value - mean))
	}

	return sum / float64(h.total)
}

func (h *NumericHistogram) Count() float64 {
	return float64(h.total)
}

// trim merges adjacent bins to decrease the bin count to the maximum value
func (h *NumericHistogram) trim() {
	for len(h.bins) > h.maxbins {
		// Find closest bins in terms of value
		minDelta := 1e99
		minDeltaIndex := 0
		for i := range h.bins {
			if i == 0 {
				continue
			}

			if delta := h.bins[i].value - h.bins[i-1].value; delta < minDelta {
				minDelta = delta
				minDeltaIndex = i
			}
		}

		// We need to merge bins minDeltaIndex-1 and minDeltaIndex
		totalCount := h.bins[minDeltaIndex-1].count + h.bins[minDeltaIndex].count
		mergedbin := bin{
			value: (h.bins[minDeltaIndex-1].value*
				h.bins[minDeltaIndex-1].count +
				h.bins[minDeltaIndex].value*
					h.bins[minDeltaIndex].count) /
				totalCount, // weighted average
			count: totalCount, // summed heights
		}
		head := append(make([]bin, 0), h.bins[0:minDeltaIndex-1]...)
		tail := append([]bin{mergedbin}, h.bins[minDeltaIndex+1:]...)
		h.bins = append(head, tail...)
	}
}

// String returns a string reprentation of the histogram,
// which is useful for printing to a terminal.
func (h *NumericHistogram) String() (str string) {
	str += fmt.Sprintln("Total:", h.total)

	for i := range h.bins {
		var bar string
		for j := 0; j < int(float64(h.bins[i].count)/float64(h.total)*200); j++ {
			bar += "."
		}
		str += fmt.Sprintf("%.3fms \t\t %v\n", h.bins[i].value/1000000, bar)
	}

	return
}
<commit_msg>change dependency for special unit print<commit_after>package gohistogram

// Copyright (c) 2013 VividCortex, Inc. All rights reserved.
// Please see the LICENSE file for applicable license terms.

import (
	"fmt"
)

type NumericHistogram struct {
	bins    []bin
	maxbins int
	total   uint64
}

// NewHistogram returns a new NumericHistogram with a maximum of n bins.
//
// There is no "optimal" bin count, but somewhere between 20 and 80 bins
// should be sufficient.
func NewHistogram(n int) *NumericHistogram {
	return &NumericHistogram{
		bins:    make([]bin, 0),
		maxbins: n,
		total:   0,
	}
}

func (h *NumericHistogram) Add(n float64) {
	defer h.trim()
	h.total++
	for i := range h.bins {
		if h.bins[i].value == n {
			h.bins[i].count++
			return
		}

		if h.bins[i].value > n {

			newbin := bin{value: n, count: 1}
			head := append(make([]bin, 0), h.bins[0:i]...)

			head = append(head, newbin)
			tail := h.bins[i:]
			h.bins = append(head, tail...)
			return
		}
	}

	h.bins = append(h.bins, bin{count: 1, value: n})
}

func (h *NumericHistogram) Quantile(q float64) float64 {
	count := q * float64(h.total)
	for i := range h.bins {
		count -= float64(h.bins[i].count)

		if count <= 0 {
			return h.bins[i].value
		}
	}

	return -1
}

// CDF returns the value of the cumulative distribution function
// at x
func (h *NumericHistogram) CDF(x float64) float64 {
	count := 0.0
	for i := range h.bins {
		if h.bins[i].value <= x {
			count += float64(h.bins[i].count)
		}
	}

	return count / float64(h.total)
}

// Mean returns the sample mean of the distribution
func (h *NumericHistogram) Mean() float64 {
	if h.total == 0 {
		return 0
	}

	sum := 0.0

	for i := range h.bins {
		sum += h.bins[i].value * h.bins[i].count
	}

	return sum / float64(h.total)
}

// Variance returns the variance of the distribution
func (h *NumericHistogram) Variance() float64 {
	if h.total == 0 {
		return 0
	}

	sum := 0.0
	mean := h.Mean()

	for i := range h.bins {
		sum += (h.bins[i].count * (h.bins[i].value - mean) * (h.bins[i].value - mean))
	}

	return sum / float64(h.total)
}

func (h *NumericHistogram) Count() float64 {
	return float64(h.total)
}

// trim merges adjacent bins to decrease the bin count to the maximum value
func (h *NumericHistogram) trim() {
	for len(h.bins) > h.maxbins {
		// Find closest bins in terms of value
		minDelta := 1e99
		minDeltaIndex := 0
		for i := range h.bins {
			if i == 0 {
				continue
			}

			if delta := h.bins[i].value - h.bins[i-1].value; delta < minDelta {
				minDelta = delta
				minDeltaIndex = i
			}
		}

		// We need to merge bins minDeltaIndex-1 and minDeltaIndex
		totalCount := h.bins[minDeltaIndex-1].count + h.bins[minDeltaIndex].count
		mergedbin := bin{
			value: (h.bins[minDeltaIndex-1].value*
				h.bins[minDeltaIndex-1].count +
				h.bins[minDeltaIndex].value*
					h.bins[minDeltaIndex].count) /
				totalCount, // weighted average
			count: totalCount, // summed heights
		}
		head := append(make([]bin, 0), h.bins[0:minDeltaIndex-1]...)
		tail := append([]bin{mergedbin}, h.bins[minDeltaIndex+1:]...)
		h.bins = append(head, tail...)
	}
}

// String returns a string reprentation of the histogram,
// which is useful for printing to a terminal.
func (h *NumericHistogram) String() (str string) {
	str += fmt.Sprintln("Total:", h.total)

	for i := range h.bins {
		var bar string
		for j := 0; j < int(float64(h.bins[i].count)/float64(h.total)*200); j++ {
			bar += "."
		}
		str += fmt.Sprintf("%.3fms\t Count:%v\t %v\n", h.bins[i].value/1000000, h.bins[i].count, bar)
	}

	return
}
<|endoftext|>


<commit_before>package main

import (
  "encoding/xml"
  "os"
  "io/ioutil"
  "log"
  "github.com/bdon/jklmnt/linref"
)

type NextBusVehicleReport struct {
  Id string `xml:"id,attr"`
  DirTag string `xml:"dirTag,attr"`
  Lat float64 `xml:"lat,attr"`
  Lon float64 `xml:"lon,attr"`
  SecsSinceReport int `xml:"secsSinceReport,attr"`
}

type NextBusResponse struct {
  Reports []NextBusVehicleReport `xml:"vehicle"`
}

func main() {
  // For now, let's assume that all Trips for a Route have the same Shape
  // N Judah is Route # 1093, Shape 102909
  nReferencer := linref.NewReferencer("102909")

  file, err := os.Open("N/1377452461.xml")
  if err != nil {
    log.Fatal(err)
  }
  foo := NextBusResponse{}
  body, err := ioutil.ReadAll(file)
  if err != nil {
    log.Fatal(err)
  }
  xml.Unmarshal(body, &foo)
  log.Printf("%d", len(foo.Reports))
  for _, report := range foo.Reports {
    log.Printf("%s %s %f %f %d\n", report.Id, report.DirTag, report.Lat, report.Lon, report.SecsSinceReport)
    log.Printf("%f\n", nReferencer.Reference(report.Lat, report.Lon))
  }

}
<commit_msg>remove parse_xml<commit_after><|endoftext|>


<commit_before>package consumergroup

import (
	"errors"
	"fmt"
	"sync"
	"time"

	"github.com/Shopify/sarama"
)

type partitionConsumer struct {
	owner      *topicConsumer
	group      string
	topic      string
	partition  int32
	offset     int64
	prevOffset int64

	consumer sarama.PartitionConsumer
}

func newPartitionConsumer(owner *topicConsumer, partition int32) *partitionConsumer {
	return &partitionConsumer{
		owner:      owner,
		topic:      owner.name,
		group:      owner.owner.name,
		partition:  partition,
		offset:     0,
		prevOffset: 0,
	}
}

func (pc *partitionConsumer) start() {
	var wg sync.WaitGroup

	cg := pc.owner.owner
	err := pc.claim()
	if err != nil {
		cg.logger.Errorf("Failed to claim topic[%s] partition[%d] and give up, err %s",
			pc.topic, pc.partition, err)
		goto ERROR
	}
	defer func() {
		err = pc.release()
		if err != nil {
			cg.logger.Errorf("Failed to release topic[%s] partition[%d], err %s",
				pc.topic, pc.partition, err)
		} else {
			cg.logger.Infof("Release topic[%s] partition[%d] success", pc.topic, pc.partition)
		}
	}()

	err = pc.loadOffsetFromZk()
	if err != nil {
		cg.logger.Errorf("Failed to load topic[%s] partition[%d] offset from zk, err %s",
			pc.topic, pc.partition, err)
		goto ERROR
	}
	cg.logger.Debugf("Get topic[%s] partition[%d] offset[%d] from offset storage",
		pc.topic, pc.partition, pc.offset)

	pc.consumer, err = cg.getPartitionConsumer(pc.topic, pc.partition, pc.offset)
	if err != nil {
		cg.logger.Errorf("Failed to topic[%s] partition[%d] message, err %s",
			pc.topic, pc.partition, err)
		goto ERROR
	}
	defer pc.consumer.Close()

	if cg.config.OffsetAutoCommitEnable { // start auto commit-offset thread when enable
		wg.Add(1)
		go func() {
			defer cg.callRecover()
			defer wg.Done()
			cg.logger.Infof("Offset auto-commit topic[%s] partition[%d] thread was started",
				pc.topic, pc.partition)
			pc.autoCommitOffset()
		}()
	}

	pc.fetch()
	if cg.config.OffsetAutoCommitEnable {
		err = pc.commitOffset()
		if err != nil {
			cg.logger.Errorf("Failed to commit topic[%s] partition[%d] offset[%d]",
				pc.topic, pc.partition, pc.offset)
		}
		wg.Wait() // Wait for auto-commit-offset thread
		cg.logger.Infof("Offset auto-commit topic[%s] partition[%d] thread was stoped",
			pc.topic, pc.partition)
	}
	return

ERROR:
	cg.ExitGroup()
}

func (pc *partitionConsumer) loadOffsetFromZk() error {
	cg := pc.owner.owner
	offset, err := cg.storage.getOffset(pc.group, pc.topic, pc.partition)
	if err != nil {
		return err
	}
	if offset == -1 {
		offset = cg.config.OffsetAutoReset
	}
	pc.offset = offset
	pc.prevOffset = offset
	return nil
}

func (pc *partitionConsumer) claim() error {
	cg := pc.owner.owner
	timer := time.NewTimer(cg.config.ClaimPartitionRetryInterval)
	defer timer.Stop()
	retry := cg.config.ClaimPartitionRetryTimes
	// Claim partition would retry until success
	for i := 0; i < retry+1 || retry <= 0; i++ {
		err := cg.storage.claimPartition(pc.group, pc.topic, pc.partition, cg.id)
		if err == nil {
			return nil
		}
		if i%3 == 0 || retry > 0 {
			cg.logger.Errorf("Failed to claim topic[%s] partition[%d] after %d retires, err %s",
				pc.topic, pc.partition, i, err)
		}
		select {
		case <-timer.C:
			timer.Reset(cg.config.ClaimPartitionRetryInterval)
		case <-cg.stopper:
			return errors.New("stop signal was received when claim partition")
		}
	}
	return fmt.Errorf("claim partition err, after %d retries", retry)
}

func (pc *partitionConsumer) release() error {
	cg := pc.owner.owner
	owner, err := cg.storage.getPartitionOwner(pc.group, pc.topic, pc.partition)
	if err != nil {
		return err
	}
	if cg.id == owner {
		return cg.storage.releasePartition(pc.group, pc.topic, pc.partition)
	}
	return errors.New("partition wasn't ownered by this consumergroup")
}

func (pc *partitionConsumer) fetch() {
	cg := pc.owner.owner
	messageChan := pc.owner.messages
	errorChan := pc.owner.errors

PARTITION_CONSUMER_LOOP:
	for {
		select {
		case <-cg.stopper:
			break PARTITION_CONSUMER_LOOP
		case err := <-pc.consumer.Errors():
			errorChan <- err
		case message := <-pc.consumer.Messages():
			select {
			case messageChan <- message:
				pc.offset++
			case <-cg.stopper:
				break PARTITION_CONSUMER_LOOP
			}
		}
	}
}

func (pc *partitionConsumer) autoCommitOffset() {
	cg := pc.owner.owner
	defer cg.callRecover()
	timer := time.NewTimer(cg.config.OffsetAutoCommitInterval)
	for {
		select {
		case <-cg.stopper:
			return
		case <-timer.C:
			err := pc.commitOffset()
			if err != nil {
				cg.logger.Errorf("Failed to auto commit topic[%s] partition[%d] offset[%d], err :%s",
					pc.topic, pc.partition, pc.offset, err)
			}
			timer.Reset(cg.config.OffsetAutoCommitInterval)
		}
	}
}

func (pc *partitionConsumer) commitOffset() error {
	cg := pc.owner.owner
	offset := pc.offset
	if pc.prevOffset == offset {
		return nil
	}
	err := cg.storage.commitOffset(pc.group, pc.topic, pc.partition, offset)
	if err != nil {
		return err
	}
	pc.prevOffset = offset
	return nil
}
<commit_msg>MOD: Update partition's offset with message offest, instead of auto increment<commit_after>package consumergroup

import (
	"errors"
	"fmt"
	"sync"
	"time"

	"github.com/Shopify/sarama"
)

type partitionConsumer struct {
	owner      *topicConsumer
	group      string
	topic      string
	partition  int32
	offset     int64
	prevOffset int64

	consumer sarama.PartitionConsumer
}

func newPartitionConsumer(owner *topicConsumer, partition int32) *partitionConsumer {
	return &partitionConsumer{
		owner:      owner,
		topic:      owner.name,
		group:      owner.owner.name,
		partition:  partition,
		offset:     0,
		prevOffset: 0,
	}
}

func (pc *partitionConsumer) start() {
	var wg sync.WaitGroup

	cg := pc.owner.owner
	err := pc.claim()
	if err != nil {
		cg.logger.Errorf("Failed to claim topic[%s] partition[%d] and give up, err %s",
			pc.topic, pc.partition, err)
		goto ERROR
	}
	defer func() {
		err = pc.release()
		if err != nil {
			cg.logger.Errorf("Failed to release topic[%s] partition[%d], err %s",
				pc.topic, pc.partition, err)
		} else {
			cg.logger.Infof("Release topic[%s] partition[%d] success", pc.topic, pc.partition)
		}
	}()

	err = pc.loadOffsetFromZk()
	if err != nil {
		cg.logger.Errorf("Failed to load topic[%s] partition[%d] offset from zk, err %s",
			pc.topic, pc.partition, err)
		goto ERROR
	}
	cg.logger.Debugf("Get topic[%s] partition[%d] offset[%d] from offset storage",
		pc.topic, pc.partition, pc.offset)

	pc.consumer, err = cg.getPartitionConsumer(pc.topic, pc.partition, pc.offset)
	if err != nil {
		cg.logger.Errorf("Failed to topic[%s] partition[%d] message, err %s",
			pc.topic, pc.partition, err)
		goto ERROR
	}
	defer pc.consumer.Close()

	if cg.config.OffsetAutoCommitEnable { // start auto commit-offset thread when enable
		wg.Add(1)
		go func() {
			defer cg.callRecover()
			defer wg.Done()
			cg.logger.Infof("Offset auto-commit topic[%s] partition[%d] thread was started",
				pc.topic, pc.partition)
			pc.autoCommitOffset()
		}()
	}

	pc.fetch()
	if cg.config.OffsetAutoCommitEnable {
		err = pc.commitOffset()
		if err != nil {
			cg.logger.Errorf("Failed to commit topic[%s] partition[%d] offset[%d]",
				pc.topic, pc.partition, pc.offset)
		}
		wg.Wait() // Wait for auto-commit-offset thread
		cg.logger.Infof("Offset auto-commit topic[%s] partition[%d] thread was stoped",
			pc.topic, pc.partition)
	}
	return

ERROR:
	cg.ExitGroup()
}

func (pc *partitionConsumer) loadOffsetFromZk() error {
	cg := pc.owner.owner
	offset, err := cg.storage.getOffset(pc.group, pc.topic, pc.partition)
	if err != nil {
		return err
	}
	if offset == -1 {
		offset = cg.config.OffsetAutoReset
	}
	pc.offset = offset
	pc.prevOffset = offset
	return nil
}

func (pc *partitionConsumer) claim() error {
	cg := pc.owner.owner
	timer := time.NewTimer(cg.config.ClaimPartitionRetryInterval)
	defer timer.Stop()
	retry := cg.config.ClaimPartitionRetryTimes
	// Claim partition would retry until success
	for i := 0; i < retry+1 || retry <= 0; i++ {
		err := cg.storage.claimPartition(pc.group, pc.topic, pc.partition, cg.id)
		if err == nil {
			return nil
		}
		if i%3 == 0 || retry > 0 {
			cg.logger.Errorf("Failed to claim topic[%s] partition[%d] after %d retires, err %s",
				pc.topic, pc.partition, i, err)
		}
		select {
		case <-timer.C:
			timer.Reset(cg.config.ClaimPartitionRetryInterval)
		case <-cg.stopper:
			return errors.New("stop signal was received when claim partition")
		}
	}
	return fmt.Errorf("claim partition err, after %d retries", retry)
}

func (pc *partitionConsumer) release() error {
	cg := pc.owner.owner
	owner, err := cg.storage.getPartitionOwner(pc.group, pc.topic, pc.partition)
	if err != nil {
		return err
	}
	if cg.id == owner {
		return cg.storage.releasePartition(pc.group, pc.topic, pc.partition)
	}
	return errors.New("partition wasn't ownered by this consumergroup")
}

func (pc *partitionConsumer) fetch() {
	cg := pc.owner.owner
	messageChan := pc.owner.messages
	errorChan := pc.owner.errors

PARTITION_CONSUMER_LOOP:
	for {
		select {
		case <-cg.stopper:
			break PARTITION_CONSUMER_LOOP
		case err := <-pc.consumer.Errors():
			errorChan <- err
		case message := <-pc.consumer.Messages():
			select {
			case messageChan <- message:
				pc.offset = message.Offset + 1
			case <-cg.stopper:
				break PARTITION_CONSUMER_LOOP
			}
		}
	}
}

func (pc *partitionConsumer) autoCommitOffset() {
	cg := pc.owner.owner
	defer cg.callRecover()
	timer := time.NewTimer(cg.config.OffsetAutoCommitInterval)
	for {
		select {
		case <-cg.stopper:
			return
		case <-timer.C:
			err := pc.commitOffset()
			if err != nil {
				cg.logger.Errorf("Failed to auto commit topic[%s] partition[%d] offset[%d], err :%s",
					pc.topic, pc.partition, pc.offset, err)
			}
			timer.Reset(cg.config.OffsetAutoCommitInterval)
		}
	}
}

func (pc *partitionConsumer) commitOffset() error {
	cg := pc.owner.owner
	offset := pc.offset
	if pc.prevOffset == offset {
		return nil
	}
	err := cg.storage.commitOffset(pc.group, pc.topic, pc.partition, offset)
	if err != nil {
		return err
	}
	pc.prevOffset = offset
	return nil
}
<|endoftext|>


<commit_before>// Copyright 2013 Canonical Ltd.
// Licensed under the AGPLv3, see LICENCE file for details.

package tools

import (
	"launchpad.net/juju-core/environs"
	"launchpad.net/juju-core/environs/simplestreams"
)

// SupportsCustomSources represents an environment that
// can host tools metadata at provider specific sources.
type SupportsCustomSources interface {
	GetToolsSources() ([]simplestreams.DataSource, error)
}

// GetMetadataSources returns the sources to use when looking for
// simplestreams tools metadata. If env implements SupportsCustomSurces,
// the sources returned from that method will also be considered.
func GetMetadataSources(env environs.ConfigGetter) ([]simplestreams.DataSource, error) {
	return GetMetadataSourcesWithRetries(env, false)
}

// GetMetadataSources returns the sources to use when looking for
// simplestreams tools metadata. If env implements SupportsCustomSurces,
// the sources returned from that method will also be considered.
func GetMetadataSourcesWithRetries(env environs.ConfigGetter, allowRetry bool) ([]simplestreams.DataSource, error) {
	var sources []simplestreams.DataSource
	if userURL, ok := env.Config().ToolsURL(); ok {
		sources = append(sources, simplestreams.NewURLDataSource(userURL))
	}
	if custom, ok := env.(SupportsCustomSources); ok {
		customSources, err := custom.GetToolsSources()
		if err != nil {
			return nil, err
		}
		sources = append(sources, customSources...)
	}

	if DefaultBaseURL != "" {
		sources = append(sources, simplestreams.NewURLDataSource(DefaultBaseURL))
	}
	for _, source := range sources {
		source.SetAllowRetry(allowRetry)
	}
	return sources, nil
}
<commit_msg>FIx comment<commit_after>// Copyright 2013 Canonical Ltd.
// Licensed under the AGPLv3, see LICENCE file for details.

package tools

import (
	"launchpad.net/juju-core/environs"
	"launchpad.net/juju-core/environs/simplestreams"
)

// SupportsCustomSources represents an environment that
// can host tools metadata at provider specific sources.
type SupportsCustomSources interface {
	GetToolsSources() ([]simplestreams.DataSource, error)
}

// GetMetadataSources returns the sources to use when looking for
// simplestreams tools metadata. If env implements SupportsCustomSurces,
// the sources returned from that method will also be considered.
// The sources are configured to not use retries.
func GetMetadataSources(env environs.ConfigGetter) ([]simplestreams.DataSource, error) {
	return GetMetadataSourcesWithRetries(env, false)
}

// GetMetadataSourcesWithRetries returns the sources to use when looking for
// simplestreams tools metadata. If env implements SupportsCustomSurces,
// the sources returned from that method will also be considered.
// The sources are configured to use retries according to the value of allowRetry.
func GetMetadataSourcesWithRetries(env environs.ConfigGetter, allowRetry bool) ([]simplestreams.DataSource, error) {
	var sources []simplestreams.DataSource
	if userURL, ok := env.Config().ToolsURL(); ok {
		sources = append(sources, simplestreams.NewURLDataSource(userURL))
	}
	if custom, ok := env.(SupportsCustomSources); ok {
		customSources, err := custom.GetToolsSources()
		if err != nil {
			return nil, err
		}
		sources = append(sources, customSources...)
	}

	if DefaultBaseURL != "" {
		sources = append(sources, simplestreams.NewURLDataSource(DefaultBaseURL))
	}
	for _, source := range sources {
		source.SetAllowRetry(allowRetry)
	}
	return sources, nil
}
<|endoftext|>


<commit_before>package graphite

import (
	"fmt"
	"strings"

	"github.com/influxdb/influxdb/toml"
	"github.com/influxdb/influxdb/tsdb"
)

const (
	// DefaultBindAddress is the default binding interface if none is specified.
	DefaultBindAddress = ":2003"

	// DefaultDatabase is the default database if none is specified.
	DefaultDatabase = "graphite"

	// DefaultProtocol is the default IP protocol used by the Graphite input.
	DefaultProtocol = "tcp"

	// DefaultConsistencyLevel is the default write consistency for the Graphite input.
	DefaultConsistencyLevel = "one"
)

// Config represents the configuration for Graphite endpoints.
type Config struct {
	BindAddress      string        `toml:"bind-address"`
	Database         string        `toml:"database"`
	Enabled          bool          `toml:"enabled"`
	Protocol         string        `toml:"protocol"`
	BatchSize        int           `toml:"batch-size"`
	BatchTimeout     toml.Duration `toml:"batch-timeout"`
	ConsistencyLevel string        `toml:"consistency-level"`
	Templates        []string      `toml:"templates"`
	Tags             []string      `toml:"tags"`
}

// NewConfig returns a new Config with defaults.
func NewConfig() Config {
	return Config{
		BindAddress:      DefaultBindAddress,
		Database:         DefaultDatabase,
		Protocol:         DefaultProtocol,
		ConsistencyLevel: DefaultConsistencyLevel,
	}
}

// WithDefaults takes the given config and returns a new config with any required
// default values set.
func (c *Config) WithDefaults() *Config {
	d := *c
	if d.BindAddress == "" {
		d.BindAddress = DefaultBindAddress
	}
	if d.Database == "" {
		d.Database = DefaultDatabase
	}
	if d.Protocol == "" {
		d.Protocol = DefaultProtocol
	}
	if d.ConsistencyLevel == "" {
		d.ConsistencyLevel = DefaultConsistencyLevel
	}
	return &d
}

func (c *Config) DefaultTags() tsdb.Tags {
	tags := tsdb.Tags{}
	for _, t := range c.Tags {
		parts := strings.Split(t, "=")
		tags[parts[0]] = parts[1]
	}
	return tags
}

func (c *Config) Validate() error {
	if err := c.validateTemplates(); err != nil {
		return err
	}

	if err := c.validateTags(); err != nil {
		return err
	}

	return nil
}

func (c *Config) validateTemplates() error {
	for i, t := range c.Templates {
		parts := strings.Fields(t)
		// Ensure template string is non-empty
		if len(parts) == 0 {
			return fmt.Errorf("missing template at position: %d", i)
		}
		if len(parts) == 1 && parts[0] == "" {
			return fmt.Errorf("missing template at position: %d", i)
		}

		if len(parts) > 3 {
			return fmt.Errorf("invalid template format: '%s'", t)
		}

		template := t
		filter := ""
		tags := ""
		if len(parts) >= 2 {
			filter = parts[0]
			template = parts[1]
		}

		if len(parts) == 3 {
			tags = parts[2]
		}

		// Validate the template has one and only one measurement
		if err := c.validateTemplate(template); err != nil {
			return err
		}
		// Validate filter expression is valid
		if err := c.validateFilter(filter); err != nil {
			return err
		}

		if tags != "" {
			// Validate tags
			for _, tagStr := range strings.Split(tags, ",") {
				if err := c.validateTag(tagStr); err != nil {
					return err
				}
			}
		}
	}
	return nil
}

func (c *Config) validateTags() error {
	for _, t := range c.Tags {
		if err := c.validateTag(t); err != nil {
			return err
		}
	}
	return nil
}

func (c *Config) validateTemplate(template string) error {
	hasMeasurement := false
	for _, p := range strings.Split(template, ".") {
		if p == "measurement" || p == "measurement*" {
			if hasMeasurement {
				return fmt.Errorf("multiple measurements in template `%s`", template)
			}
			hasMeasurement = true
		}
	}

	if !hasMeasurement {
		return fmt.Errorf("no measurement in template `%s`", template)
	}

	return nil
}

func (c *Config) validateFilter(filter string) error {
	for _, p := range strings.Split(filter, ".") {
		if p == "" {
			return fmt.Errorf("filter contains blank section: %s", filter)
		}

		if strings.Contains(p, "*") && p != "*" {
			return fmt.Errorf("invalid filter wildcard section: %s", filter)
		}
	}
	return nil
}

func (c *Config) validateTag(keyValue string) error {
	parts := strings.Split(keyValue, "=")
	if len(parts) != 2 {
		return fmt.Errorf("invalid template tags: '%s'", keyValue)
	}

	if parts[0] == "" || parts[1] == "" {
		return fmt.Errorf("invalid template tags: %s'", keyValue)
	}

	return nil
}
<commit_msg>Fix validation failing when using a default template<commit_after>package graphite

import (
	"fmt"
	"strings"

	"github.com/influxdb/influxdb/toml"
	"github.com/influxdb/influxdb/tsdb"
)

const (
	// DefaultBindAddress is the default binding interface if none is specified.
	DefaultBindAddress = ":2003"

	// DefaultDatabase is the default database if none is specified.
	DefaultDatabase = "graphite"

	// DefaultProtocol is the default IP protocol used by the Graphite input.
	DefaultProtocol = "tcp"

	// DefaultConsistencyLevel is the default write consistency for the Graphite input.
	DefaultConsistencyLevel = "one"
)

// Config represents the configuration for Graphite endpoints.
type Config struct {
	BindAddress      string        `toml:"bind-address"`
	Database         string        `toml:"database"`
	Enabled          bool          `toml:"enabled"`
	Protocol         string        `toml:"protocol"`
	BatchSize        int           `toml:"batch-size"`
	BatchTimeout     toml.Duration `toml:"batch-timeout"`
	ConsistencyLevel string        `toml:"consistency-level"`
	Templates        []string      `toml:"templates"`
	Tags             []string      `toml:"tags"`
}

// NewConfig returns a new Config with defaults.
func NewConfig() Config {
	return Config{
		BindAddress:      DefaultBindAddress,
		Database:         DefaultDatabase,
		Protocol:         DefaultProtocol,
		ConsistencyLevel: DefaultConsistencyLevel,
	}
}

// WithDefaults takes the given config and returns a new config with any required
// default values set.
func (c *Config) WithDefaults() *Config {
	d := *c
	if d.BindAddress == "" {
		d.BindAddress = DefaultBindAddress
	}
	if d.Database == "" {
		d.Database = DefaultDatabase
	}
	if d.Protocol == "" {
		d.Protocol = DefaultProtocol
	}
	if d.ConsistencyLevel == "" {
		d.ConsistencyLevel = DefaultConsistencyLevel
	}
	return &d
}

func (c *Config) DefaultTags() tsdb.Tags {
	tags := tsdb.Tags{}
	for _, t := range c.Tags {
		parts := strings.Split(t, "=")
		tags[parts[0]] = parts[1]
	}
	return tags
}

func (c *Config) Validate() error {
	if err := c.validateTemplates(); err != nil {
		return err
	}

	if err := c.validateTags(); err != nil {
		return err
	}

	return nil
}

func (c *Config) validateTemplates() error {
	for i, t := range c.Templates {
		parts := strings.Fields(t)
		// Ensure template string is non-empty
		if len(parts) == 0 {
			return fmt.Errorf("missing template at position: %d", i)
		}
		if len(parts) == 1 && parts[0] == "" {
			return fmt.Errorf("missing template at position: %d", i)
		}

		if len(parts) > 3 {
			return fmt.Errorf("invalid template format: '%s'", t)
		}

		template := t
		filter := ""
		tags := ""
		if len(parts) >= 2 {
			filter = parts[0]
			template = parts[1]
		}

		if len(parts) == 3 {
			tags = parts[2]
		}

		// Validate the template has one and only one measurement
		if err := c.validateTemplate(template); err != nil {
			return err
		}

		if filter != "" {
			// Validate filter expression is valid
			if err := c.validateFilter(filter); err != nil {
				return err
			}
		}

		if tags != "" {
			// Validate tags
			for _, tagStr := range strings.Split(tags, ",") {
				if err := c.validateTag(tagStr); err != nil {
					return err
				}
			}
		}
	}
	return nil
}

func (c *Config) validateTags() error {
	for _, t := range c.Tags {
		if err := c.validateTag(t); err != nil {
			return err
		}
	}
	return nil
}

func (c *Config) validateTemplate(template string) error {
	hasMeasurement := false
	for _, p := range strings.Split(template, ".") {
		if p == "measurement" || p == "measurement*" {
			if hasMeasurement {
				return fmt.Errorf("multiple measurements in template `%s`", template)
			}
			hasMeasurement = true
		}
	}

	if !hasMeasurement {
		return fmt.Errorf("no measurement in template `%s`", template)
	}

	return nil
}

func (c *Config) validateFilter(filter string) error {
	for _, p := range strings.Split(filter, ".") {
		if p == "" {
			return fmt.Errorf("filter contains blank section: %s", filter)
		}

		if strings.Contains(p, "*") && p != "*" {
			return fmt.Errorf("invalid filter wildcard section: %s", filter)
		}
	}
	return nil
}

func (c *Config) validateTag(keyValue string) error {
	parts := strings.Split(keyValue, "=")
	if len(parts) != 2 {
		return fmt.Errorf("invalid template tags: '%s'", keyValue)
	}

	if parts[0] == "" || parts[1] == "" {
		return fmt.Errorf("invalid template tags: %s'", keyValue)
	}

	return nil
}
<|endoftext|>


<commit_before>/*
Copyright 2011 Google Inc.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

     http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package schema

import (
	"fmt"
	"io"
	"json"
	"log"
	"os"

	"camli/blobref"
)

var _ = log.Printf

type FileReader struct {
	fetcher blobref.SeekFetcher
	ss      *Superset

	ci     int    // index into contentparts
	ccon   uint64 // bytes into current chunk already consumed
	remain int64  // bytes remaining

	cr   blobref.ReadSeekCloser // cached reader (for blobref chunks)
	crbr *blobref.BlobRef       // the blobref that cr is for

	csubfr *FileReader  // cached sub blobref reader (for subBlobRef chunks)
	ccp    *ContentPart // the content part that csubfr is cached for
}

// TODO: make this take a blobref.FetcherAt instead?
func NewFileReader(fetcher blobref.SeekFetcher, fileBlobRef *blobref.BlobRef) (*FileReader, os.Error) {
	if fileBlobRef == nil {
		return nil, os.NewError("schema/filereader: NewFileReader blobref was nil")
	}
	ss := new(Superset)
	rsc, _, err := fetcher.Fetch(fileBlobRef)
	if err != nil {
		return nil, fmt.Errorf("schema/filereader: fetching file schema blob: %v", err)
	}
	if err = json.NewDecoder(rsc).Decode(ss); err != nil {
		return nil, fmt.Errorf("schema/filereader: decoding file schema blob: %v", err)
	}
	if ss.Type != "file" {
		return nil, fmt.Errorf("schema/filereader: expected \"file\" schema blob, got %q", ss.Type)
	}
	return ss.NewFileReader(fetcher), nil
}

func (ss *Superset) NewFileReader(fetcher blobref.SeekFetcher) *FileReader {
	// TODO: return an error if ss isn't a Type "file"
	//
	return &FileReader{fetcher: fetcher, ss: ss, remain: int64(ss.Size)}
}

// FileSchema returns the reader's schema superset. Don't mutate it.
func (fr *FileReader) FileSchema() *Superset {
	return fr.ss
}

func (fr *FileReader) Skip(skipBytes uint64) uint64 {
	wantedSkipped := skipBytes

	for skipBytes != 0 && fr.ci < len(fr.ss.ContentParts) {
		cp := fr.ss.ContentParts[fr.ci]
		thisChunkSkippable := cp.Size - fr.ccon
		toSkip := minu64(skipBytes, thisChunkSkippable)
		fr.ccon += toSkip
		fr.remain -= int64(toSkip)
		if fr.ccon == cp.Size {
			fr.ci++
			fr.ccon = 0
		}
		skipBytes -= toSkip
	}

	return wantedSkipped - skipBytes
}

func (fr *FileReader) closeOpenBlobs() {
	if fr.cr != nil {
		fr.cr.Close()
		fr.cr = nil
		fr.crbr = nil
	}
}

func (fr *FileReader) readerFor(br *blobref.BlobRef, seekTo int64) (r io.Reader, err os.Error) {
	if fr.crbr == br {
		return fr.cr, nil
	}
	fr.closeOpenBlobs()
	var rsc blobref.ReadSeekCloser
	if br != nil {
		rsc, _, err = fr.fetcher.Fetch(br)
		if err != nil {
			return
		}

		_, serr := rsc.Seek(int64(seekTo), os.SEEK_SET)
		if serr != nil {
			return nil, fmt.Errorf("schema: FileReader.Read seek error on blob %s: %v", br, serr)
		}

	} else {
		rsc = &zeroReader{}
	}
	fr.crbr = br
	fr.cr = rsc
	return rsc, nil
}

func (fr *FileReader) subBlobRefReader(cp *ContentPart) (io.Reader, os.Error) {
	if fr.ccp == cp {
		return fr.csubfr, nil
	}
	subfr, err := NewFileReader(fr.fetcher, cp.SubBlobRef)
	if err == nil {
		subfr.Skip(cp.Offset)
		fr.csubfr = subfr
		fr.ccp = cp
	}
	return subfr, err
}

func (fr *FileReader) currentPart() (*ContentPart, os.Error) {
	for {
		if fr.ci >= len(fr.ss.ContentParts) {
			fr.closeOpenBlobs()
			if fr.remain > 0 {
				return nil, fmt.Errorf("schema: declared file schema size was larger than sum of content parts")
			}
			return nil, os.EOF
		}
		cp := fr.ss.ContentParts[fr.ci]
		thisChunkReadable := cp.Size - fr.ccon
		if thisChunkReadable == 0 {
			fr.ci++
			fr.ccon = 0
			continue
		}
		return cp, nil
	}
	panic("unreachable")
}

func (fr *FileReader) Read(p []byte) (n int, err os.Error) {
	cp, err := fr.currentPart()
	if err != nil {
		return 0, err
	}

	if cp.Size == 0 {
		return 0, fmt.Errorf("blobref content part contained illegal size 0")
	}

	br := cp.BlobRef
	sbr := cp.SubBlobRef
	if br != nil && sbr != nil {
		return 0, fmt.Errorf("content part index %d has both blobRef and subFileBlobRef", fr.ci)
	}

	var r io.Reader

	if sbr != nil {
		r, err = fr.subBlobRefReader(cp)
		if err != nil {
			return 0, fmt.Errorf("schema: FileReader.Read error fetching sub file %s: %v", sbr, err)
		}
	} else {
		seekTo := cp.Offset + fr.ccon
		r, err = fr.readerFor(br, int64(seekTo))
		if err != nil {
			return 0, fmt.Errorf("schema: FileReader.Read error fetching blob %s: %v", br, err)
		}
	}

	readSize := cp.Size - fr.ccon
	if readSize < uint64(len(p)) {
		p = p[:int(readSize)]
	}

	n, err = r.Read(p)
	fr.ccon += uint64(n)
	fr.remain -= int64(n)
	if fr.remain < 0 {
		err = fmt.Errorf("schema: file schema was invalid; content parts sum to over declared size")
	}
	return
}

func minu64(a, b uint64) uint64 {
	if a < b {
		return a
	}
	return b
}

type zeroReader struct{}

func (*zeroReader) Read(p []byte) (int, os.Error) {
	for i := range p {
		p[i] = 0
	}
	return len(p), nil
}

func (*zeroReader) Close() os.Error {
	return nil
}

func (*zeroReader) Seek(offset int64, whence int) (newFilePos int64, err os.Error) {
	// Caller is ignoring our newFilePos return value.
	return 0, nil
}
<commit_msg>let file reader be closed<commit_after>/*
Copyright 2011 Google Inc.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

     http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package schema

import (
	"fmt"
	"io"
	"json"
	"log"
	"os"

	"camli/blobref"
)

var _ = log.Printf

const closedIndex = -1
var errClosed = os.NewError("filereader is closed")

type FileReader struct {
	fetcher blobref.SeekFetcher
	ss      *Superset

	ci     int    // index into contentparts, or -1 on closed
	ccon   uint64 // bytes into current chunk already consumed
	remain int64  // bytes remaining

	cr   blobref.ReadSeekCloser // cached reader (for blobref chunks)
	crbr *blobref.BlobRef       // the blobref that cr is for

	csubfr *FileReader  // cached sub blobref reader (for subBlobRef chunks)
	ccp    *ContentPart // the content part that csubfr is cached for
}

// TODO: make this take a blobref.FetcherAt instead?
func NewFileReader(fetcher blobref.SeekFetcher, fileBlobRef *blobref.BlobRef) (*FileReader, os.Error) {
	if fileBlobRef == nil {
		return nil, os.NewError("schema/filereader: NewFileReader blobref was nil")
	}
	ss := new(Superset)
	rsc, _, err := fetcher.Fetch(fileBlobRef)
	if err != nil {
		return nil, fmt.Errorf("schema/filereader: fetching file schema blob: %v", err)
	}
	if err = json.NewDecoder(rsc).Decode(ss); err != nil {
		return nil, fmt.Errorf("schema/filereader: decoding file schema blob: %v", err)
	}
	if ss.Type != "file" {
		return nil, fmt.Errorf("schema/filereader: expected \"file\" schema blob, got %q", ss.Type)
	}
	return ss.NewFileReader(fetcher), nil
}

func (ss *Superset) NewFileReader(fetcher blobref.SeekFetcher) *FileReader {
	// TODO: return an error if ss isn't a Type "file"
	//
	return &FileReader{fetcher: fetcher, ss: ss, remain: int64(ss.Size)}
}

// FileSchema returns the reader's schema superset. Don't mutate it.
func (fr *FileReader) FileSchema() *Superset {
	return fr.ss
}

func (fr *FileReader) Close() os.Error {
	if fr.ci == closedIndex {
		return errClosed
	}
	fr.closeOpenBlobs()
	fr.ci = closedIndex
	return nil
}

func (fr *FileReader) Skip(skipBytes uint64) uint64 {
	if fr.ci == closedIndex {
		return 0
	}

	wantedSkipped := skipBytes

	for skipBytes != 0 && fr.ci < len(fr.ss.ContentParts) {
		cp := fr.ss.ContentParts[fr.ci]
		thisChunkSkippable := cp.Size - fr.ccon
		toSkip := minu64(skipBytes, thisChunkSkippable)
		fr.ccon += toSkip
		fr.remain -= int64(toSkip)
		if fr.ccon == cp.Size {
			fr.ci++
			fr.ccon = 0
		}
		skipBytes -= toSkip
	}

	return wantedSkipped - skipBytes
}

func (fr *FileReader) closeOpenBlobs() {
	if fr.cr != nil {
		fr.cr.Close()
		fr.cr = nil
		fr.crbr = nil
	}
}

func (fr *FileReader) readerFor(br *blobref.BlobRef, seekTo int64) (r io.Reader, err os.Error) {
	if fr.crbr == br {
		return fr.cr, nil
	}
	fr.closeOpenBlobs()
	var rsc blobref.ReadSeekCloser
	if br != nil {
		rsc, _, err = fr.fetcher.Fetch(br)
		if err != nil {
			return
		}

		_, serr := rsc.Seek(int64(seekTo), os.SEEK_SET)
		if serr != nil {
			return nil, fmt.Errorf("schema: FileReader.Read seek error on blob %s: %v", br, serr)
		}

	} else {
		rsc = &zeroReader{}
	}
	fr.crbr = br
	fr.cr = rsc
	return rsc, nil
}

func (fr *FileReader) subBlobRefReader(cp *ContentPart) (io.Reader, os.Error) {
	if fr.ccp == cp {
		return fr.csubfr, nil
	}
	subfr, err := NewFileReader(fr.fetcher, cp.SubBlobRef)
	if err == nil {
		subfr.Skip(cp.Offset)
		fr.csubfr = subfr
		fr.ccp = cp
	}
	return subfr, err
}

func (fr *FileReader) currentPart() (*ContentPart, os.Error) {
	for {
		if fr.ci >= len(fr.ss.ContentParts) {
			fr.closeOpenBlobs()
			if fr.remain > 0 {
				return nil, fmt.Errorf("schema: declared file schema size was larger than sum of content parts")
			}
			return nil, os.EOF
		}
		cp := fr.ss.ContentParts[fr.ci]
		thisChunkReadable := cp.Size - fr.ccon
		if thisChunkReadable == 0 {
			fr.ci++
			fr.ccon = 0
			continue
		}
		return cp, nil
	}
	panic("unreachable")
}

func (fr *FileReader) Read(p []byte) (n int, err os.Error) {
	if fr.ci == closedIndex {
		return 0, errClosed
	}

	cp, err := fr.currentPart()
	if err != nil {
		return 0, err
	}

	if cp.Size == 0 {
		return 0, fmt.Errorf("blobref content part contained illegal size 0")
	}

	br := cp.BlobRef
	sbr := cp.SubBlobRef
	if br != nil && sbr != nil {
		return 0, fmt.Errorf("content part index %d has both blobRef and subFileBlobRef", fr.ci)
	}

	var r io.Reader

	if sbr != nil {
		r, err = fr.subBlobRefReader(cp)
		if err != nil {
			return 0, fmt.Errorf("schema: FileReader.Read error fetching sub file %s: %v", sbr, err)
		}
	} else {
		seekTo := cp.Offset + fr.ccon
		r, err = fr.readerFor(br, int64(seekTo))
		if err != nil {
			return 0, fmt.Errorf("schema: FileReader.Read error fetching blob %s: %v", br, err)
		}
	}

	readSize := cp.Size - fr.ccon
	if readSize < uint64(len(p)) {
		p = p[:int(readSize)]
	}

	n, err = r.Read(p)
	fr.ccon += uint64(n)
	fr.remain -= int64(n)
	if fr.remain < 0 {
		err = fmt.Errorf("schema: file schema was invalid; content parts sum to over declared size")
	}
	return
}

func minu64(a, b uint64) uint64 {
	if a < b {
		return a
	}
	return b
}

type zeroReader struct{}

func (*zeroReader) Read(p []byte) (int, os.Error) {
	for i := range p {
		p[i] = 0
	}
	return len(p), nil
}

func (*zeroReader) Close() os.Error {
	return nil
}

func (*zeroReader) Seek(offset int64, whence int) (newFilePos int64, err os.Error) {
	// Caller is ignoring our newFilePos return value.
	return 0, nil
}
<|endoftext|>


<commit_before>// Copyright 2020 Google LLC
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     https://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

// Package snippets contains speech examples.
package snippets

// [START speech_context_classes]

import (
	"context"
	"fmt"
	"io"
	"io/ioutil"

	speech "cloud.google.com/go/speech/apiv1"
	speechpb "google.golang.org/genproto/googleapis/cloud/speech/v1"
)

// contextClasses provides "hints" to the speech recognizer
// to favour specific classes of words in the results.
func contextClasses(w io.Writer, gcsURI string) error {
	ctx := context.Background()

	client, err := speech.NewClient(ctx)
	if err != nil {
		return fmt.Errorf("NewClient: %v", err)
	}
	defer client.Close()

	// SpeechContext: to configure your speech_context see:
	// https://cloud.google.com/speech-to-text/docs/reference/rpc/google.cloud.speech.v1#speechcontext
	// Full list of supported phrases (class tokens) here:
	// https://cloud.google.com/speech-to-text/docs/class-tokens
	// In this instance, the use of "$TIME" favours time of day detections.
	SpeechContext := &speechpb.SpeechContext{Phrases: []string{"$TIME"}}

	resp, err := client.Recognize(ctx, &speechpb.RecognizeRequest{
		Config: &speechpb.RecognitionConfig{
			Encoding:                            speechpb.RecognitionConfig_LINEAR16,
			SampleRateHertz:                     8000,
			LanguageCode:                        "en-US",
			SpeechContexts:                      []&speechpb.SpeechContext{SpeechContext},
		},
		Audio: &speechpb.RecognitionAudio{
			AudioSource: &speechpb.RecognitionAudio_Uri{Uri: gcsURI},
		},
	})
	if err != nil {
		return fmt.Errorf("Recognize: %v", err)
	}

	// Print the results.
	for i, result := range resp.Results {
		fmt.Fprintf(w, "%s\n", strings.Repeat("-", 20))
		fmt.Fprintf(w, "Result %d\n", i+1)
		for j, alternative := range result.Alternatives {
			fmt.Fprintf(w, "Alternative %d: %s\n", j+1, alternative.Transcript)
		}
	}
	return nil
}

// [END speech_context_classes]
<commit_msg>speech: fix syntax errors (#1246)<commit_after>// Copyright 2020 Google LLC
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     https://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

// Package snippets contains speech examples.
package snippets

// [START speech_context_classes]

import (
	"context"
	"fmt"
	"io"
	"io/ioutil"

	speech "cloud.google.com/go/speech/apiv1"
	speechpb "google.golang.org/genproto/googleapis/cloud/speech/v1"
)

// contextClasses provides "hints" to the speech recognizer
// to favour specific classes of words in the results.
func contextClasses(w io.Writer, gcsURI string) error {
	ctx := context.Background()

	client, err := speech.NewClient(ctx)
	if err != nil {
		return fmt.Errorf("NewClient: %v", err)
	}
	defer client.Close()

	// SpeechContext: to configure your speech_context see:
	// https://cloud.google.com/speech-to-text/docs/reference/rpc/google.cloud.speech.v1#speechcontext
	// Full list of supported phrases (class tokens) here:
	// https://cloud.google.com/speech-to-text/docs/class-tokens
	// In this instance, the use of "$TIME" favours time of day detections.
	speechContext := &speechpb.SpeechContext{Phrases: []string{"$TIME"}}

	resp, err := client.Recognize(ctx, &speechpb.RecognizeRequest{
		Config: &speechpb.RecognitionConfig{
			Encoding:                            speechpb.RecognitionConfig_LINEAR16,
			SampleRateHertz:                     8000,
			LanguageCode:                        "en-US",
			SpeechContexts:                      []*speechpb.SpeechContext{speechContext},
		},
		Audio: &speechpb.RecognitionAudio{
			AudioSource: &speechpb.RecognitionAudio_Uri{Uri: gcsURI},
		},
	})
	if err != nil {
		return fmt.Errorf("Recognize: %v", err)
	}

	// Print the results.
	for i, result := range resp.Results {
		fmt.Fprintf(w, "%s\n", strings.Repeat("-", 20))
		fmt.Fprintf(w, "Result %d\n", i+1)
		for j, alternative := range result.Alternatives {
			fmt.Fprintf(w, "Alternative %d: %s\n", j+1, alternative.Transcript)
		}
	}
	return nil
}

// [END speech_context_classes]
<|endoftext|>