<commit_before>package fastly

import (
	"fmt"
	"net/http"
	"sort"
)

type ACLEntryConfig config

type ACLEntry struct {
	// Non-writable
	ServiceID string `json:"service_id"`
	ID        string `json:"id"`
	ACLID     string `json:"acl_id"`

	// writable
	IP      string      `json:"ip"`
	Subnet  uint8       `json:"subnet"`
	Comment string      `json:"comment"`
	Negated Compatibool `json:"negated"`
}

// aclEntriesByName is a sortable list of aclEntries.
type aclEntriesByIP []*ACLEntry

// Len, Swap, and Less implement the sortable interface.
func (s aclEntriesByIP) Len() int      { return len(s) }
func (s aclEntriesByIP) Swap(i, j int) { s[i], s[j] = s[j], s[i] }
func (s aclEntriesByIP) Less(i, j int) bool {
	return s[i].IP < s[j].IP
}

// List aclEntries for a specific ACL and service.
func (c *ACLEntryConfig) List(serviceID, aclID string) ([]*ACLEntry, *http.Response, error) {
	u := fmt.Sprintf("/service/%s/acl/%s/entries", serviceID, aclID)

	req, err := c.client.NewRequest("GET", u, nil)
	if err != nil {
		return nil, nil, err
	}

	aclEntries := new([]*ACLEntry)
	resp, err := c.client.Do(req, aclEntries)
	if err != nil {
		return nil, resp, err
	}

	sort.Stable(aclEntriesByIP(*aclEntries))

	return *aclEntries, resp, nil
}

// Get fetches a specific aclEntry by entryID.
func (c *ACLEntryConfig) Get(serviceID, aclID, entryID string) (*ACLEntry, *http.Response, error) {
	u := fmt.Sprintf("/service/%s/acl/%s/entry/%s", serviceID, aclID, entryID)

	req, err := c.client.NewRequest("GET", u, nil)
	if err != nil {
		return nil, nil, err
	}

	aclEntry := new(ACLEntry)
	resp, err := c.client.Do(req, aclEntry)
	if err != nil {
		return nil, resp, err
	}
	return aclEntry, resp, nil
}

// Create a new aclEntry.
func (c *ACLEntryConfig) Create(serviceID, aclID string, aclEntry *ACLEntry) (*ACLEntry, *http.Response, error) {
	u := fmt.Sprintf("/service/%s/acl/%s/entry", serviceID, aclID)

	req, err := c.client.NewJSONRequest("POST", u, aclEntry)
	if err != nil {
		return nil, nil, err
	}

	b := new(ACLEntry)
	resp, err := c.client.Do(req, b)
	if err != nil {
		return nil, resp, err
	}

	return b, resp, nil
}

// Update a aclEntry
func (c *ACLEntryConfig) Update(serviceID, aclID, entryID string, aclEntry *ACLEntry) (*ACLEntry, *http.Response, error) {
	u := fmt.Sprintf("/service/%s/acl/%s/entry/%s", serviceID, aclID, entryID)

	req, err := c.client.NewJSONRequest("PATCH", u, aclEntry)
	if err != nil {
		return nil, nil, err
	}

	b := new(ACLEntry)
	resp, err := c.client.Do(req, b)
	if err != nil {
		return nil, resp, err
	}

	return b, resp, nil
}

// Delete a aclEntry
func (c *ACLEntryConfig) Delete(serviceID, aclID, entryID string) (*http.Response, error) {
	u := fmt.Sprintf("/service/%s/acl/%s/entry/%s", serviceID, aclID, entryID)

	req, err := c.client.NewRequest("DELETE", u, nil)
	if err != nil {
		return nil, err
	}

	resp, err := c.client.Do(req, nil)
	if err != nil {
		return resp, err
	}

	return resp, nil
}

type ACLEntryBatchUpdate struct {
	Entries []ACLEntryUpdate `json:"entries"`
}

type ACLEntryUpdate struct {
	Operation BatchOperation `json:"op,omitempty"`
	ID        string         `json:"id,omitempty"`
	IP        string         `json:"ip,omitempty"`
	Subnet    string         `json:"subnet,omitempty"`
}

func (c *ACLEntryConfig) BatchUpdate(serviceID, aclID string, entries []ACLEntryUpdate) (*http.Response, error) {
	u := fmt.Sprintf("/service/%s/acl/%s/entries", serviceID, aclID)

	var update ACLEntryBatchUpdate
	update.Entries = entries
	req, err := c.client.NewJSONRequest("PATCH", u, update)
	if err != nil {
		return nil, err
	}

	resp, err := c.client.Do(req, nil)
	if err != nil {
		return resp, err
	}

	return resp, nil
}
<commit_msg>Add the Comment field to ACLEntryUpdate.<commit_after>package fastly

import (
	"fmt"
	"net/http"
	"sort"
)

type ACLEntryConfig config

type ACLEntry struct {
	// Non-writable
	ServiceID string `json:"service_id"`
	ID        string `json:"id"`
	ACLID     string `json:"acl_id"`

	// writable
	IP      string      `json:"ip"`
	Subnet  uint8       `json:"subnet"`
	Comment string      `json:"comment"`
	Negated Compatibool `json:"negated"`
}

// aclEntriesByName is a sortable list of aclEntries.
type aclEntriesByIP []*ACLEntry

// Len, Swap, and Less implement the sortable interface.
func (s aclEntriesByIP) Len() int      { return len(s) }
func (s aclEntriesByIP) Swap(i, j int) { s[i], s[j] = s[j], s[i] }
func (s aclEntriesByIP) Less(i, j int) bool {
	return s[i].IP < s[j].IP
}

// List aclEntries for a specific ACL and service.
func (c *ACLEntryConfig) List(serviceID, aclID string) ([]*ACLEntry, *http.Response, error) {
	u := fmt.Sprintf("/service/%s/acl/%s/entries", serviceID, aclID)

	req, err := c.client.NewRequest("GET", u, nil)
	if err != nil {
		return nil, nil, err
	}

	aclEntries := new([]*ACLEntry)
	resp, err := c.client.Do(req, aclEntries)
	if err != nil {
		return nil, resp, err
	}

	sort.Stable(aclEntriesByIP(*aclEntries))

	return *aclEntries, resp, nil
}

// Get fetches a specific aclEntry by entryID.
func (c *ACLEntryConfig) Get(serviceID, aclID, entryID string) (*ACLEntry, *http.Response, error) {
	u := fmt.Sprintf("/service/%s/acl/%s/entry/%s", serviceID, aclID, entryID)

	req, err := c.client.NewRequest("GET", u, nil)
	if err != nil {
		return nil, nil, err
	}

	aclEntry := new(ACLEntry)
	resp, err := c.client.Do(req, aclEntry)
	if err != nil {
		return nil, resp, err
	}
	return aclEntry, resp, nil
}

// Create a new aclEntry.
func (c *ACLEntryConfig) Create(serviceID, aclID string, aclEntry *ACLEntry) (*ACLEntry, *http.Response, error) {
	u := fmt.Sprintf("/service/%s/acl/%s/entry", serviceID, aclID)

	req, err := c.client.NewJSONRequest("POST", u, aclEntry)
	if err != nil {
		return nil, nil, err
	}

	b := new(ACLEntry)
	resp, err := c.client.Do(req, b)
	if err != nil {
		return nil, resp, err
	}

	return b, resp, nil
}

// Update a aclEntry
func (c *ACLEntryConfig) Update(serviceID, aclID, entryID string, aclEntry *ACLEntry) (*ACLEntry, *http.Response, error) {
	u := fmt.Sprintf("/service/%s/acl/%s/entry/%s", serviceID, aclID, entryID)

	req, err := c.client.NewJSONRequest("PATCH", u, aclEntry)
	if err != nil {
		return nil, nil, err
	}

	b := new(ACLEntry)
	resp, err := c.client.Do(req, b)
	if err != nil {
		return nil, resp, err
	}

	return b, resp, nil
}

// Delete a aclEntry
func (c *ACLEntryConfig) Delete(serviceID, aclID, entryID string) (*http.Response, error) {
	u := fmt.Sprintf("/service/%s/acl/%s/entry/%s", serviceID, aclID, entryID)

	req, err := c.client.NewRequest("DELETE", u, nil)
	if err != nil {
		return nil, err
	}

	resp, err := c.client.Do(req, nil)
	if err != nil {
		return resp, err
	}

	return resp, nil
}

type ACLEntryBatchUpdate struct {
	Entries []ACLEntryUpdate `json:"entries"`
}

type ACLEntryUpdate struct {
	Operation BatchOperation `json:"op,omitempty"`
	ID        string         `json:"id,omitempty"`
	IP        string         `json:"ip,omitempty"`
	Subnet    string         `json:"subnet,omitempty"`
	Comment   string         `json:"comment,omitempty"`
}

func (c *ACLEntryConfig) BatchUpdate(serviceID, aclID string, entries []ACLEntryUpdate) (*http.Response, error) {
	u := fmt.Sprintf("/service/%s/acl/%s/entries", serviceID, aclID)

	var update ACLEntryBatchUpdate
	update.Entries = entries
	req, err := c.client.NewJSONRequest("PATCH", u, update)
	if err != nil {
		return nil, err
	}

	resp, err := c.client.Do(req, nil)
	if err != nil {
		return resp, err
	}

	return resp, nil
}
<|endoftext|>


<commit_before>package builder

import (
	"fmt"
	"io"
	"io/ioutil"
	"os"
	"os/exec"
	"path/filepath"
	"strings"
	"time"

	docker "github.com/fsouza/go-dockerclient"
	"github.com/golang/glog"

	s2igit "github.com/openshift/source-to-image/pkg/scm/git"

	"github.com/openshift/origin/pkg/build/api"
	"github.com/openshift/origin/pkg/build/builder/cmd/dockercfg"
	"github.com/openshift/origin/pkg/generate/git"
	"github.com/openshift/source-to-image/pkg/tar"
)

const (
	// urlCheckTimeout is the timeout used to check the source URL
	// If fetching the URL exceeds the timeout, then the build will
	// not proceed further and stop
	urlCheckTimeout = 16 * time.Second
)

type gitAuthError string
type gitNotFoundError string

func (e gitAuthError) Error() string {
	return fmt.Sprintf("failed to fetch requested repository %q with provided credentials", string(e))
}

func (e gitNotFoundError) Error() string {
	return fmt.Sprintf("requested repository %q not found", string(e))
}

// fetchSource retrieves the inputs defined by the build source into the
// provided directory, or returns an error if retrieval is not possible.
func fetchSource(dockerClient DockerClient, dir string, build *api.Build, urlTimeout time.Duration, in io.Reader, gitClient GitClient) (*git.SourceInfo, error) {
	hasGitSource := false

	// expect to receive input from STDIN
	if err := extractInputBinary(in, build.Spec.Source.Binary, dir); err != nil {
		return nil, err
	}

	// may retrieve source from Git
	hasGitSource, err := extractGitSource(gitClient, build.Spec.Source.Git, build.Spec.Revision, dir, urlTimeout)
	if err != nil {
		return nil, err
	}
	var sourceInfo *git.SourceInfo
	if hasGitSource {
		var errs []error
		sourceInfo, errs = gitClient.GetInfo(dir)
		if len(errs) > 0 {
			for _, e := range errs {
				glog.Warningf("Error getting git info: %v", e)
			}
		}
	}

	forcePull := false
	switch {
	case build.Spec.Strategy.SourceStrategy != nil:
		forcePull = build.Spec.Strategy.SourceStrategy.ForcePull
	case build.Spec.Strategy.DockerStrategy != nil:
		forcePull = build.Spec.Strategy.DockerStrategy.ForcePull
	case build.Spec.Strategy.CustomStrategy != nil:
		forcePull = build.Spec.Strategy.CustomStrategy.ForcePull
	}
	// extract source from an Image if specified
	for i, image := range build.Spec.Source.Images {
		imageSecretIndex := i
		if image.PullSecret == nil {
			imageSecretIndex = -1
		}
		err := extractSourceFromImage(dockerClient, image.From.Name, dir, imageSecretIndex, image.Paths, forcePull)
		if err != nil {
			return nil, err
		}
	}

	// a Dockerfile has been specified, create or overwrite into the destination
	if dockerfileSource := build.Spec.Source.Dockerfile; dockerfileSource != nil {
		baseDir := dir
		// if a context dir has been defined and we cloned source, overwrite the destination
		if hasGitSource && len(build.Spec.Source.ContextDir) != 0 {
			baseDir = filepath.Join(baseDir, build.Spec.Source.ContextDir)
		}
		return sourceInfo, ioutil.WriteFile(filepath.Join(baseDir, "Dockerfile"), []byte(*dockerfileSource), 0660)
	}

	return sourceInfo, nil
}

// checkRemoteGit validates the specified Git URL. It returns GitNotFoundError
// when the remote repository not found and GitAuthenticationError when the
// remote repository failed to authenticate.
// Since this is calling the 'git' binary, the proxy settings should be
// available for this command.
func checkRemoteGit(gitClient GitClient, url string, timeout time.Duration) error {
	glog.V(4).Infof("git ls-remote %s --heads", url)

	var (
		out    string
		errOut string
		err    error
	)

	finish := make(chan struct{}, 1)
	go func() {
		out, errOut, err = gitClient.ListRemote(url, "--heads")
		close(finish)
	}()
	select {
	case <-finish:
	case <-time.After(timeout):
		return fmt.Errorf("timeout while waiting for remote repository %q", url)
	}

	if len(out) != 0 {
		glog.V(4).Infof(out)
	}
	if len(errOut) != 0 {
		glog.V(4).Infof(errOut)
	}

	combinedOut := out + errOut
	switch {
	case strings.Contains(combinedOut, "Authentication failed"):
		return gitAuthError(url)
	case strings.Contains(combinedOut, "not found"):
		return gitNotFoundError(url)
	}

	return err
}

// checkSourceURI performs a check on the URI associated with the build
// to make sure that it is valid.
func checkSourceURI(gitClient GitClient, rawurl string, timeout time.Duration) error {
	if !s2igit.New().ValidCloneSpec(rawurl) {
		return fmt.Errorf("Invalid git source url: %s", rawurl)
	}
	return checkRemoteGit(gitClient, rawurl, timeout)
}

// extractInputBinary processes the provided input stream as directed by BinaryBuildSource
// into dir.
func extractInputBinary(in io.Reader, source *api.BinaryBuildSource, dir string) error {
	if source == nil {
		return nil
	}

	var path string
	if len(source.AsFile) > 0 {
		glog.V(2).Infof("Receiving source from STDIN as file %s", source.AsFile)
		path = filepath.Join(dir, source.AsFile)

		f, err := os.OpenFile(path, os.O_CREATE|os.O_EXCL|os.O_WRONLY, 0664)
		if err != nil {
			return err
		}
		defer f.Close()
		n, err := io.Copy(f, os.Stdin)
		if err != nil {
			return err
		}
		glog.V(4).Infof("Received %d bytes into %s", n, path)
		return nil
	}

	glog.V(2).Infof("Receiving source from STDIN as archive")

	cmd := exec.Command("bsdtar", "-x", "-o", "-m", "-f", "-", "-C", dir)
	cmd.Stdin = in
	out, err := cmd.CombinedOutput()
	if err != nil {
		glog.V(2).Infof("Extracting...\n%s", string(out))
		return fmt.Errorf("unable to extract binary build input, must be a zip, tar, or gzipped tar, or specified as a file: %v", err)
	}
	return nil
}

func extractGitSource(gitClient GitClient, gitSource *api.GitBuildSource, revision *api.SourceRevision, dir string, timeout time.Duration) (bool, error) {
	if gitSource == nil {
		return false, nil
	}

	// Check source URI, trying to connect to the server only if not using a proxy.
	if err := checkSourceURI(gitClient, gitSource.URI, timeout); err != nil {
		return true, err
	}

	// check if we specify a commit, ref, or branch to check out
	usingRef := len(gitSource.Ref) != 0 || (revision != nil && revision.Git != nil && len(revision.Git.Commit) != 0)

	// Recursive clone if we're not going to checkout a ref and submodule update later
	glog.V(2).Infof("Cloning source from %s", gitSource.URI)

	// Only use the quiet flag if Verbosity is not 5 or greater
	quiet := !bool(glog.V(5))
	if err := gitClient.CloneWithOptions(dir, gitSource.URI, git.CloneOptions{Recursive: !usingRef, Quiet: quiet}); err != nil {
		return true, err
	}

	// if we specify a commit, ref, or branch to checkout, do so, and update submodules
	if usingRef {
		commit := gitSource.Ref

		if revision != nil && revision.Git != nil && revision.Git.Commit != "" {
			commit = revision.Git.Commit
		}

		if err := gitClient.Checkout(dir, commit); err != nil {
			return true, err
		}

		// Recursively update --init
		if err := gitClient.SubmoduleUpdate(dir, true, true); err != nil {
			return true, err
		}
	}
	return true, nil
}

func copyImageSource(dockerClient DockerClient, containerID, sourceDir, destDir string, tarHelper tar.Tar) error {
	// Setup destination directory
	fi, err := os.Stat(destDir)
	if err != nil {
		if !os.IsNotExist(err) {
			return err
		}
		glog.V(4).Infof("Creating image destination directory: %s", destDir)
		err := os.MkdirAll(destDir, 0644)
		if err != nil {
			return err
		}
	} else {
		if !fi.IsDir() {
			return fmt.Errorf("destination %s must be a directory", destDir)
		}
	}

	tempFile, err := ioutil.TempFile("", "imgsrc")
	if err != nil {
		return err
	}
	glog.V(4).Infof("Downloading source from path %s in container %s to temporary archive %s", sourceDir, containerID, tempFile.Name())
	err = dockerClient.DownloadFromContainer(containerID, docker.DownloadFromContainerOptions{
		OutputStream: tempFile,
		Path:         sourceDir,
	})
	if err != nil {
		return err
	}
	if err := tempFile.Close(); err != nil {
		return err
	}

	// Extract the created tar file to the destination directory
	file, err := os.Open(tempFile.Name())
	if err != nil {
		return err
	}
	defer file.Close()

	glog.V(4).Infof("Extracting temporary tar %s to directory %s", tempFile.Name(), destDir)
	var tarOutput io.Writer
	if glog.V(4) {
		tarOutput = os.Stdout
	}
	return tarHelper.ExtractTarStreamWithLogging(destDir, file, tarOutput)
}

func extractSourceFromImage(dockerClient DockerClient, image, buildDir string, imageSecretIndex int, paths []api.ImageSourcePath, forcePull bool) error {
	glog.V(4).Infof("Extracting image source from %s", image)

	dockerAuth := docker.AuthConfiguration{}
	if imageSecretIndex != -1 {
		pullSecret := os.Getenv(fmt.Sprintf("%s%d", dockercfg.PullSourceAuthType, imageSecretIndex))
		if len(pullSecret) > 0 {
			authPresent := false
			dockerAuth, authPresent = dockercfg.NewHelper().GetDockerAuth(image, fmt.Sprintf("%s%d", dockercfg.PullSourceAuthType, imageSecretIndex))
			if authPresent {
				glog.V(5).Infof("Registry server Address: %s", dockerAuth.ServerAddress)
				glog.V(5).Infof("Registry server User Name: %s", dockerAuth.Username)
				glog.V(5).Infof("Registry server Email: %s", dockerAuth.Email)
				passwordPresent := "<<empty>>"
				if len(dockerAuth.Password) > 0 {
					passwordPresent = "<<non-empty>>"
				}
				glog.V(5).Infof("Registry server Password: %s", passwordPresent)
			}
		}
	}

	exists := true
	if !forcePull {
		_, err := dockerClient.InspectImage(image)
		if err != nil && err == docker.ErrNoSuchImage {
			exists = false
		} else if err != nil {
			return err
		}
	}

	if !exists || forcePull {
		if err := dockerClient.PullImage(docker.PullImageOptions{Repository: image}, dockerAuth); err != nil {
			return fmt.Errorf("error pulling image %v: %v", image, err)
		}

	}

	// Create container to copy from
	container, err := dockerClient.CreateContainer(docker.CreateContainerOptions{
		Config: &docker.Config{
			Image: image,
		},
	})
	if err != nil {
		return fmt.Errorf("error creating source image container: %v", err)
	}
	defer dockerClient.RemoveContainer(docker.RemoveContainerOptions{ID: container.ID})

	tarHelper := tar.New()
	tarHelper.SetExclusionPattern(nil)

	for _, path := range paths {
		glog.V(4).Infof("Extracting path %s from container %s to %s", path.SourcePath, container.ID, path.DestinationDir)
		err := copyImageSource(dockerClient, container.ID, path.SourcePath, filepath.Join(buildDir, path.DestinationDir), tarHelper)
		if err != nil {
			return fmt.Errorf("error copying source path %s to %s: %v", path.SourcePath, path.DestinationDir, err)
		}
	}

	return nil
}
<commit_msg>Display source downloading in build logs by default<commit_after>package builder

import (
	"fmt"
	"io"
	"io/ioutil"
	"os"
	"os/exec"
	"path/filepath"
	"strings"
	"time"

	docker "github.com/fsouza/go-dockerclient"
	"github.com/golang/glog"

	s2igit "github.com/openshift/source-to-image/pkg/scm/git"

	"github.com/openshift/origin/pkg/build/api"
	"github.com/openshift/origin/pkg/build/builder/cmd/dockercfg"
	"github.com/openshift/origin/pkg/generate/git"
	"github.com/openshift/source-to-image/pkg/tar"
)

const (
	// urlCheckTimeout is the timeout used to check the source URL
	// If fetching the URL exceeds the timeout, then the build will
	// not proceed further and stop
	urlCheckTimeout = 16 * time.Second
)

type gitAuthError string
type gitNotFoundError string

func (e gitAuthError) Error() string {
	return fmt.Sprintf("failed to fetch requested repository %q with provided credentials", string(e))
}

func (e gitNotFoundError) Error() string {
	return fmt.Sprintf("requested repository %q not found", string(e))
}

// fetchSource retrieves the inputs defined by the build source into the
// provided directory, or returns an error if retrieval is not possible.
func fetchSource(dockerClient DockerClient, dir string, build *api.Build, urlTimeout time.Duration, in io.Reader, gitClient GitClient) (*git.SourceInfo, error) {
	hasGitSource := false

	// expect to receive input from STDIN
	if err := extractInputBinary(in, build.Spec.Source.Binary, dir); err != nil {
		return nil, err
	}

	// may retrieve source from Git
	hasGitSource, err := extractGitSource(gitClient, build.Spec.Source.Git, build.Spec.Revision, dir, urlTimeout)
	if err != nil {
		return nil, err
	}

	var sourceInfo *git.SourceInfo
	if hasGitSource {
		var errs []error
		sourceInfo, errs = gitClient.GetInfo(dir)
		if len(errs) > 0 {
			for _, e := range errs {
				glog.Warningf("Error getting git info: %v", e)
			}
		}
	}

	forcePull := false
	switch {
	case build.Spec.Strategy.SourceStrategy != nil:
		forcePull = build.Spec.Strategy.SourceStrategy.ForcePull
	case build.Spec.Strategy.DockerStrategy != nil:
		forcePull = build.Spec.Strategy.DockerStrategy.ForcePull
	case build.Spec.Strategy.CustomStrategy != nil:
		forcePull = build.Spec.Strategy.CustomStrategy.ForcePull
	}
	// extract source from an Image if specified
	for i, image := range build.Spec.Source.Images {
		imageSecretIndex := i
		if image.PullSecret == nil {
			imageSecretIndex = -1
		}
		err := extractSourceFromImage(dockerClient, image.From.Name, dir, imageSecretIndex, image.Paths, forcePull)
		if err != nil {
			return nil, err
		}
	}

	// a Dockerfile has been specified, create or overwrite into the destination
	if dockerfileSource := build.Spec.Source.Dockerfile; dockerfileSource != nil {
		baseDir := dir
		// if a context dir has been defined and we cloned source, overwrite the destination
		if hasGitSource && len(build.Spec.Source.ContextDir) != 0 {
			baseDir = filepath.Join(baseDir, build.Spec.Source.ContextDir)
		}
		return sourceInfo, ioutil.WriteFile(filepath.Join(baseDir, "Dockerfile"), []byte(*dockerfileSource), 0660)
	}

	return sourceInfo, nil
}

// checkRemoteGit validates the specified Git URL. It returns GitNotFoundError
// when the remote repository not found and GitAuthenticationError when the
// remote repository failed to authenticate.
// Since this is calling the 'git' binary, the proxy settings should be
// available for this command.
func checkRemoteGit(gitClient GitClient, url string, timeout time.Duration) error {
	glog.V(4).Infof("git ls-remote %s --heads", url)

	var (
		out    string
		errOut string
		err    error
	)

	finish := make(chan struct{}, 1)
	go func() {
		out, errOut, err = gitClient.ListRemote(url, "--heads")
		close(finish)
	}()
	select {
	case <-finish:
	case <-time.After(timeout):
		return fmt.Errorf("timeout while waiting for remote repository %q", url)
	}

	if len(out) != 0 {
		glog.V(4).Infof(out)
	}
	if len(errOut) != 0 {
		glog.V(4).Infof(errOut)
	}

	combinedOut := out + errOut
	switch {
	case strings.Contains(combinedOut, "Authentication failed"):
		return gitAuthError(url)
	case strings.Contains(combinedOut, "not found"):
		return gitNotFoundError(url)
	}

	return err
}

// checkSourceURI performs a check on the URI associated with the build
// to make sure that it is valid.
func checkSourceURI(gitClient GitClient, rawurl string, timeout time.Duration) error {
	if !s2igit.New().ValidCloneSpec(rawurl) {
		return fmt.Errorf("Invalid git source url: %s", rawurl)
	}
	return checkRemoteGit(gitClient, rawurl, timeout)
}

// extractInputBinary processes the provided input stream as directed by BinaryBuildSource
// into dir.
func extractInputBinary(in io.Reader, source *api.BinaryBuildSource, dir string) error {
	if source == nil {
		return nil
	}

	var path string
	if len(source.AsFile) > 0 {
		glog.V(2).Infof("Receiving source from STDIN as file %s", source.AsFile)
		path = filepath.Join(dir, source.AsFile)

		f, err := os.OpenFile(path, os.O_CREATE|os.O_EXCL|os.O_WRONLY, 0664)
		if err != nil {
			return err
		}
		defer f.Close()
		n, err := io.Copy(f, os.Stdin)
		if err != nil {
			return err
		}
		glog.V(4).Infof("Received %d bytes into %s", n, path)
		return nil
	}

	glog.Infof("Receiving source from STDIN as archive ...")

	cmd := exec.Command("bsdtar", "-x", "-o", "-m", "-f", "-", "-C", dir)
	cmd.Stdin = in
	out, err := cmd.CombinedOutput()
	if err != nil {
		glog.V(2).Infof("Extracting...\n%s", string(out))
		return fmt.Errorf("unable to extract binary build input, must be a zip, tar, or gzipped tar, or specified as a file: %v", err)
	}
	return nil
}

func extractGitSource(gitClient GitClient, gitSource *api.GitBuildSource, revision *api.SourceRevision, dir string, timeout time.Duration) (bool, error) {
	if gitSource == nil {
		return false, nil
	}

	glog.Infof("Downloading %q ...", gitSource.URI)

	// Check source URI, trying to connect to the server only if not using a proxy.
	if err := checkSourceURI(gitClient, gitSource.URI, timeout); err != nil {
		return true, err
	}

	// check if we specify a commit, ref, or branch to check out
	usingRef := len(gitSource.Ref) != 0 || (revision != nil && revision.Git != nil && len(revision.Git.Commit) != 0)

	// Recursive clone if we're not going to checkout a ref and submodule update later
	glog.V(2).Infof("Cloning source from %s", gitSource.URI)

	// Only use the quiet flag if Verbosity is not 5 or greater
	quiet := !bool(glog.V(5))
	if err := gitClient.CloneWithOptions(dir, gitSource.URI, git.CloneOptions{Recursive: !usingRef, Quiet: quiet}); err != nil {
		return true, err
	}

	// if we specify a commit, ref, or branch to checkout, do so, and update submodules
	if usingRef {
		commit := gitSource.Ref

		if revision != nil && revision.Git != nil && revision.Git.Commit != "" {
			commit = revision.Git.Commit
		}

		if err := gitClient.Checkout(dir, commit); err != nil {
			return true, err
		}

		// Recursively update --init
		if err := gitClient.SubmoduleUpdate(dir, true, true); err != nil {
			return true, err
		}
	}
	return true, nil
}

func copyImageSource(dockerClient DockerClient, containerID, sourceDir, destDir string, tarHelper tar.Tar) error {
	// Setup destination directory
	fi, err := os.Stat(destDir)
	if err != nil {
		if !os.IsNotExist(err) {
			return err
		}
		glog.V(4).Infof("Creating image destination directory: %s", destDir)
		err := os.MkdirAll(destDir, 0644)
		if err != nil {
			return err
		}
	} else {
		if !fi.IsDir() {
			return fmt.Errorf("destination %s must be a directory", destDir)
		}
	}

	tempFile, err := ioutil.TempFile("", "imgsrc")
	if err != nil {
		return err
	}
	glog.V(4).Infof("Downloading source from path %s in container %s to temporary archive %s", sourceDir, containerID, tempFile.Name())
	err = dockerClient.DownloadFromContainer(containerID, docker.DownloadFromContainerOptions{
		OutputStream: tempFile,
		Path:         sourceDir,
	})
	if err != nil {
		return err
	}
	if err := tempFile.Close(); err != nil {
		return err
	}

	// Extract the created tar file to the destination directory
	file, err := os.Open(tempFile.Name())
	if err != nil {
		return err
	}
	defer file.Close()

	glog.V(4).Infof("Extracting temporary tar %s to directory %s", tempFile.Name(), destDir)
	var tarOutput io.Writer
	if glog.V(4) {
		tarOutput = os.Stdout
	}
	return tarHelper.ExtractTarStreamWithLogging(destDir, file, tarOutput)
}

func extractSourceFromImage(dockerClient DockerClient, image, buildDir string, imageSecretIndex int, paths []api.ImageSourcePath, forcePull bool) error {
	glog.V(4).Infof("Extracting image source from %s", image)

	dockerAuth := docker.AuthConfiguration{}
	if imageSecretIndex != -1 {
		pullSecret := os.Getenv(fmt.Sprintf("%s%d", dockercfg.PullSourceAuthType, imageSecretIndex))
		if len(pullSecret) > 0 {
			authPresent := false
			dockerAuth, authPresent = dockercfg.NewHelper().GetDockerAuth(image, fmt.Sprintf("%s%d", dockercfg.PullSourceAuthType, imageSecretIndex))
			if authPresent {
				glog.V(5).Infof("Registry server Address: %s", dockerAuth.ServerAddress)
				glog.V(5).Infof("Registry server User Name: %s", dockerAuth.Username)
				glog.V(5).Infof("Registry server Email: %s", dockerAuth.Email)
				passwordPresent := "<<empty>>"
				if len(dockerAuth.Password) > 0 {
					passwordPresent = "<<non-empty>>"
				}
				glog.V(5).Infof("Registry server Password: %s", passwordPresent)
			}
		}
	}

	exists := true
	if !forcePull {
		_, err := dockerClient.InspectImage(image)
		if err != nil && err == docker.ErrNoSuchImage {
			exists = false
		} else if err != nil {
			return err
		}
	}

	if !exists || forcePull {
		glog.Infof("Pulling image %q ...", image)
		if err := dockerClient.PullImage(docker.PullImageOptions{Repository: image}, dockerAuth); err != nil {
			return fmt.Errorf("error pulling image %v: %v", image, err)
		}

	}

	// Create container to copy from
	container, err := dockerClient.CreateContainer(docker.CreateContainerOptions{
		Config: &docker.Config{
			Image: image,
		},
	})
	if err != nil {
		return fmt.Errorf("error creating source image container: %v", err)
	}
	defer dockerClient.RemoveContainer(docker.RemoveContainerOptions{ID: container.ID})

	tarHelper := tar.New()
	tarHelper.SetExclusionPattern(nil)

	for _, path := range paths {
		glog.V(4).Infof("Extracting path %s from container %s to %s", path.SourcePath, container.ID, path.DestinationDir)
		err := copyImageSource(dockerClient, container.ID, path.SourcePath, filepath.Join(buildDir, path.DestinationDir), tarHelper)
		if err != nil {
			return fmt.Errorf("error copying source path %s to %s: %v", path.SourcePath, path.DestinationDir, err)
		}
	}

	return nil
}
<|endoftext|>


<commit_before>// Copyright 2016-2019 Authors of Cilium
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package loader

import (
	"bufio"
	"context"
	"fmt"
	"os"
	"path/filepath"
	"strings"

	"github.com/cilium/cilium/common"
	"github.com/cilium/cilium/pkg/bpf"
	"github.com/cilium/cilium/pkg/cgroups"
	"github.com/cilium/cilium/pkg/command/exec"
	"github.com/cilium/cilium/pkg/datapath"
	"github.com/cilium/cilium/pkg/datapath/alignchecker"
	"github.com/cilium/cilium/pkg/datapath/prefilter"
	"github.com/cilium/cilium/pkg/defaults"
	"github.com/cilium/cilium/pkg/logging/logfields"
	"github.com/cilium/cilium/pkg/node"
	"github.com/cilium/cilium/pkg/option"
	"github.com/cilium/cilium/pkg/sysctl"
	"github.com/sirupsen/logrus"
	"github.com/vishvananda/netlink"
)

const (
	initArgLib int = iota
	initArgRundir
	initArgIPv4NodeIP
	initArgIPv6NodeIP
	initArgMode
	initArgDevice
	initArgDevicePreFilter
	initArgModePreFilter
	initArgMTU
	initArgIPSec
	initArgMasquerade
	initArgEncryptInterface
	initArgHostReachableServices
	initArgHostReachableServicesUDP
	initArgCgroupRoot
	initArgBpffsRoot
	initArgNodePort
	initArgMax
)

func (l *Loader) writeNetdevHeader(dir string, o datapath.BaseProgramOwner) error {
	headerPath := filepath.Join(dir, common.NetdevHeaderFileName)
	log.WithField(logfields.Path, headerPath).Debug("writing configuration")

	f, err := os.Create(headerPath)
	if err != nil {
		return fmt.Errorf("failed to open file %s for writing: %s", headerPath, err)

	}
	defer f.Close()

	if err := l.templateCache.WriteNetdevConfig(f, o); err != nil {
		return err
	}
	return nil
}

// Must be called with option.Config.EnablePolicyMU locked.
func writePreFilterHeader(preFilter *prefilter.PreFilter, dir string) error {
	headerPath := filepath.Join(dir, common.PreFilterHeaderFileName)
	log.WithField(logfields.Path, headerPath).Debug("writing configuration")
	f, err := os.Create(headerPath)
	if err != nil {
		return fmt.Errorf("failed to open file %s for writing: %s", headerPath, err)

	}
	defer f.Close()
	fw := bufio.NewWriter(f)
	fmt.Fprint(fw, "/*\n")
	fmt.Fprintf(fw, " * XDP device: %s\n", option.Config.DevicePreFilter)
	fmt.Fprintf(fw, " * XDP mode: %s\n", option.Config.ModePreFilter)
	fmt.Fprint(fw, " */\n\n")
	preFilter.WriteConfig(fw)
	return fw.Flush()
}

// Reinitialize (re-)configures the base datapath configuration including global
// BPF programs, netfilter rule configuration and reserving routes in IPAM for
// locally detected prefixes. It may be run upon initial Cilium startup, after
// restore from a previous Cilium run, or during regular Cilium operation.
func (l *Loader) Reinitialize(ctx context.Context, o datapath.BaseProgramOwner, deviceMTU int, iptMgr datapath.RulesManager, p datapath.Proxy, r datapath.RouteReserver) error {
	var args []string
	var mode string
	var ret error

	type setting struct {
		name      string
		val       string
		ignoreErr bool
	}

	args = make([]string, initArgMax)

	sysSettings := []setting{
		{"net.core.bpf_jit_enable", "1", true},
		{"net.ipv4.conf.all.rp_filter", "0", false},
		{"kernel.unprivileged_bpf_disabled", "1", true},
	}

	// Lock so that endpoints cannot be built while we are compile base programs.
	o.GetCompilationLock().Lock()
	defer o.GetCompilationLock().Unlock()

	l.Init(o.Datapath(), o.LocalConfig())

	if err := l.writeNetdevHeader("./", o); err != nil {
		log.WithError(err).Warn("Unable to write netdev header")
		return err
	}

	scopedLog := log.WithField(logfields.XDPDevice, option.Config.DevicePreFilter)
	if option.Config.DevicePreFilter != "undefined" {
		if err := prefilter.ProbePreFilter(option.Config.DevicePreFilter, option.Config.ModePreFilter); err != nil {
			scopedLog.WithError(err).Warn("Turning off prefilter")
			option.Config.DevicePreFilter = "undefined"
		}
	}
	if option.Config.DevicePreFilter != "undefined" {
		preFilter, err := prefilter.NewPreFilter()
		if err != nil {
			scopedLog.WithError(ret).Warn("Unable to init prefilter")
			return ret
		}

		if err := writePreFilterHeader(preFilter, "./"); err != nil {
			scopedLog.WithError(err).Warn("Unable to write prefilter header")
			return err
		}

		o.SetPrefilter(preFilter)

		args[initArgDevicePreFilter] = option.Config.DevicePreFilter
		args[initArgModePreFilter] = option.Config.ModePreFilter
	}

	args[initArgLib] = option.Config.BpfDir
	args[initArgRundir] = option.Config.StateDir
	args[initArgCgroupRoot] = cgroups.GetCgroupRoot()
	args[initArgBpffsRoot] = bpf.GetMapRoot()

	if option.Config.EnableIPv4 {
		args[initArgIPv4NodeIP] = node.GetInternalIPv4().String()
	} else {
		args[initArgIPv4NodeIP] = "<nil>"
	}

	if option.Config.EnableIPv6 {
		args[initArgIPv6NodeIP] = node.GetIPv6().String()
		// Docker <17.05 has an issue which causes IPv6 to be disabled in the initns for all
		// interface (https://github.com/docker/libnetwork/issues/1720)
		// Enable IPv6 for now
		sysSettings = append(sysSettings,
			setting{"net.ipv6.conf.all.disable_ipv6", "0", false})
	} else {
		args[initArgIPv6NodeIP] = "<nil>"
	}

	args[initArgMTU] = fmt.Sprintf("%d", deviceMTU)

	if option.Config.EnableIPSec {
		args[initArgIPSec] = "true"
	} else {
		args[initArgIPSec] = "false"
	}

	if !option.Config.InstallIptRules && option.Config.Masquerade {
		args[initArgMasquerade] = "true"
	} else {
		args[initArgMasquerade] = "false"
	}

	if option.Config.EnableHostReachableServices {
		args[initArgHostReachableServices] = "true"
		if option.Config.EnableHostServicesUDP {
			args[initArgHostReachableServicesUDP] = "true"
		} else {
			args[initArgHostReachableServicesUDP] = "false"
		}
	} else {
		args[initArgHostReachableServices] = "false"
		args[initArgHostReachableServicesUDP] = "false"
	}

	if option.Config.EncryptInterface != "" {
		args[initArgEncryptInterface] = option.Config.EncryptInterface
	}

	if option.Config.Device != "undefined" {
		_, err := netlink.LinkByName(option.Config.Device)
		if err != nil {
			log.WithError(err).WithField("device", option.Config.Device).Warn("Link does not exist")
			return err
		}

		if option.Config.DatapathMode == option.DatapathModeIpvlan {
			mode = "ipvlan"
		} else {
			mode = "direct"
		}

		args[initArgMode] = mode
		if option.Config.EnableNodePort &&
			strings.ToLower(option.Config.Tunnel) != "disabled" {
			args[initArgMode] = option.Config.Tunnel
		}
		args[initArgDevice] = option.Config.Device
	} else {
		args[initArgMode] = option.Config.Tunnel

		if option.Config.IsFlannelMasterDeviceSet() {
			args[initArgMode] = "flannel"
			args[initArgDevice] = option.Config.FlannelMasterDevice
		}
	}

	if option.Config.EnableEndpointRoutes == true {
		args[initArgMode] = "routed"
	}

	if option.Config.EnableNodePort {
		args[initArgNodePort] = "true"
	}

	log.Info("Setting up base BPF datapath")

	for _, s := range sysSettings {
		log.Infof("Setting sysctl %s=%s", s.name, s.val)
		if err := sysctl.Write(s.name, s.val); err != nil {
			if !s.ignoreErr {
				return fmt.Errorf("Failed to sysctl -w %s=%s: %s", s.name, s.val, err)
			}
			log.WithError(err).WithFields(logrus.Fields{
				logfields.SysParamName:  s.name,
				logfields.SysParamValue: s.val,
			}).Warning("Failed to sysctl -w")
		}
	}

	prog := filepath.Join(option.Config.BpfDir, "init.sh")
	ctx, cancel := context.WithTimeout(ctx, defaults.ExecTimeout)
	defer cancel()
	cmd := exec.CommandContext(ctx, prog, args...)
	cmd.Env = bpf.Environment()
	if _, err := cmd.CombinedOutput(log, true); err != nil {
		return err
	}

	if l.canDisableDwarfRelocations {
		// Validate alignments of C and Go equivalent structs
		if err := alignchecker.CheckStructAlignments(defaults.AlignCheckerName); err != nil {
			log.WithError(err).Fatal("C and Go structs alignment check failed")
		}
	} else {
		log.Warning("Cannot check matching of C and Go common struct alignments due to old LLVM/clang version")
	}

	if !option.Config.IsFlannelMasterDeviceSet() {
		r.ReserveLocalRoutes()
	}

	if err := o.Datapath().Node().NodeConfigurationChanged(*o.LocalConfig()); err != nil {
		return err
	}

	if option.Config.InstallIptRules {
		if err := iptMgr.TransientRulesStart(option.Config.HostDevice); err != nil {
			return err
		}
	}
	// Always remove masquerade rule and then re-add it if required
	iptMgr.RemoveRules()
	if option.Config.InstallIptRules {
		err := iptMgr.InstallRules(option.Config.HostDevice)
		iptMgr.TransientRulesEnd(false)
		if err != nil {
			return err
		}
	}
	// Reinstall proxy rules for any running proxies
	if p != nil {
		p.ReinstallRules()
	}

	return nil
}
<commit_msg>loader: group `var` declarations<commit_after>// Copyright 2016-2019 Authors of Cilium
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package loader

import (
	"bufio"
	"context"
	"fmt"
	"os"
	"path/filepath"
	"strings"

	"github.com/cilium/cilium/common"
	"github.com/cilium/cilium/pkg/bpf"
	"github.com/cilium/cilium/pkg/cgroups"
	"github.com/cilium/cilium/pkg/command/exec"
	"github.com/cilium/cilium/pkg/datapath"
	"github.com/cilium/cilium/pkg/datapath/alignchecker"
	"github.com/cilium/cilium/pkg/datapath/prefilter"
	"github.com/cilium/cilium/pkg/defaults"
	"github.com/cilium/cilium/pkg/logging/logfields"
	"github.com/cilium/cilium/pkg/node"
	"github.com/cilium/cilium/pkg/option"
	"github.com/cilium/cilium/pkg/sysctl"
	"github.com/sirupsen/logrus"
	"github.com/vishvananda/netlink"
)

const (
	initArgLib int = iota
	initArgRundir
	initArgIPv4NodeIP
	initArgIPv6NodeIP
	initArgMode
	initArgDevice
	initArgDevicePreFilter
	initArgModePreFilter
	initArgMTU
	initArgIPSec
	initArgMasquerade
	initArgEncryptInterface
	initArgHostReachableServices
	initArgHostReachableServicesUDP
	initArgCgroupRoot
	initArgBpffsRoot
	initArgNodePort
	initArgMax
)

func (l *Loader) writeNetdevHeader(dir string, o datapath.BaseProgramOwner) error {
	headerPath := filepath.Join(dir, common.NetdevHeaderFileName)
	log.WithField(logfields.Path, headerPath).Debug("writing configuration")

	f, err := os.Create(headerPath)
	if err != nil {
		return fmt.Errorf("failed to open file %s for writing: %s", headerPath, err)

	}
	defer f.Close()

	if err := l.templateCache.WriteNetdevConfig(f, o); err != nil {
		return err
	}
	return nil
}

// Must be called with option.Config.EnablePolicyMU locked.
func writePreFilterHeader(preFilter *prefilter.PreFilter, dir string) error {
	headerPath := filepath.Join(dir, common.PreFilterHeaderFileName)
	log.WithField(logfields.Path, headerPath).Debug("writing configuration")
	f, err := os.Create(headerPath)
	if err != nil {
		return fmt.Errorf("failed to open file %s for writing: %s", headerPath, err)

	}
	defer f.Close()
	fw := bufio.NewWriter(f)
	fmt.Fprint(fw, "/*\n")
	fmt.Fprintf(fw, " * XDP device: %s\n", option.Config.DevicePreFilter)
	fmt.Fprintf(fw, " * XDP mode: %s\n", option.Config.ModePreFilter)
	fmt.Fprint(fw, " */\n\n")
	preFilter.WriteConfig(fw)
	return fw.Flush()
}

// Reinitialize (re-)configures the base datapath configuration including global
// BPF programs, netfilter rule configuration and reserving routes in IPAM for
// locally detected prefixes. It may be run upon initial Cilium startup, after
// restore from a previous Cilium run, or during regular Cilium operation.
func (l *Loader) Reinitialize(ctx context.Context, o datapath.BaseProgramOwner, deviceMTU int, iptMgr datapath.RulesManager, p datapath.Proxy, r datapath.RouteReserver) error {
	var (
		args []string
		mode string
		ret  error
	)

	type setting struct {
		name      string
		val       string
		ignoreErr bool
	}

	args = make([]string, initArgMax)

	sysSettings := []setting{
		{"net.core.bpf_jit_enable", "1", true},
		{"net.ipv4.conf.all.rp_filter", "0", false},
		{"kernel.unprivileged_bpf_disabled", "1", true},
	}

	// Lock so that endpoints cannot be built while we are compile base programs.
	o.GetCompilationLock().Lock()
	defer o.GetCompilationLock().Unlock()

	l.Init(o.Datapath(), o.LocalConfig())

	if err := l.writeNetdevHeader("./", o); err != nil {
		log.WithError(err).Warn("Unable to write netdev header")
		return err
	}

	scopedLog := log.WithField(logfields.XDPDevice, option.Config.DevicePreFilter)
	if option.Config.DevicePreFilter != "undefined" {
		if err := prefilter.ProbePreFilter(option.Config.DevicePreFilter, option.Config.ModePreFilter); err != nil {
			scopedLog.WithError(err).Warn("Turning off prefilter")
			option.Config.DevicePreFilter = "undefined"
		}
	}
	if option.Config.DevicePreFilter != "undefined" {
		preFilter, err := prefilter.NewPreFilter()
		if err != nil {
			scopedLog.WithError(ret).Warn("Unable to init prefilter")
			return ret
		}

		if err := writePreFilterHeader(preFilter, "./"); err != nil {
			scopedLog.WithError(err).Warn("Unable to write prefilter header")
			return err
		}

		o.SetPrefilter(preFilter)

		args[initArgDevicePreFilter] = option.Config.DevicePreFilter
		args[initArgModePreFilter] = option.Config.ModePreFilter
	}

	args[initArgLib] = option.Config.BpfDir
	args[initArgRundir] = option.Config.StateDir
	args[initArgCgroupRoot] = cgroups.GetCgroupRoot()
	args[initArgBpffsRoot] = bpf.GetMapRoot()

	if option.Config.EnableIPv4 {
		args[initArgIPv4NodeIP] = node.GetInternalIPv4().String()
	} else {
		args[initArgIPv4NodeIP] = "<nil>"
	}

	if option.Config.EnableIPv6 {
		args[initArgIPv6NodeIP] = node.GetIPv6().String()
		// Docker <17.05 has an issue which causes IPv6 to be disabled in the initns for all
		// interface (https://github.com/docker/libnetwork/issues/1720)
		// Enable IPv6 for now
		sysSettings = append(sysSettings,
			setting{"net.ipv6.conf.all.disable_ipv6", "0", false})
	} else {
		args[initArgIPv6NodeIP] = "<nil>"
	}

	args[initArgMTU] = fmt.Sprintf("%d", deviceMTU)

	if option.Config.EnableIPSec {
		args[initArgIPSec] = "true"
	} else {
		args[initArgIPSec] = "false"
	}

	if !option.Config.InstallIptRules && option.Config.Masquerade {
		args[initArgMasquerade] = "true"
	} else {
		args[initArgMasquerade] = "false"
	}

	if option.Config.EnableHostReachableServices {
		args[initArgHostReachableServices] = "true"
		if option.Config.EnableHostServicesUDP {
			args[initArgHostReachableServicesUDP] = "true"
		} else {
			args[initArgHostReachableServicesUDP] = "false"
		}
	} else {
		args[initArgHostReachableServices] = "false"
		args[initArgHostReachableServicesUDP] = "false"
	}

	if option.Config.EncryptInterface != "" {
		args[initArgEncryptInterface] = option.Config.EncryptInterface
	}

	if option.Config.Device != "undefined" {
		_, err := netlink.LinkByName(option.Config.Device)
		if err != nil {
			log.WithError(err).WithField("device", option.Config.Device).Warn("Link does not exist")
			return err
		}

		if option.Config.DatapathMode == option.DatapathModeIpvlan {
			mode = "ipvlan"
		} else {
			mode = "direct"
		}

		args[initArgMode] = mode
		if option.Config.EnableNodePort &&
			strings.ToLower(option.Config.Tunnel) != "disabled" {
			args[initArgMode] = option.Config.Tunnel
		}
		args[initArgDevice] = option.Config.Device
	} else {
		args[initArgMode] = option.Config.Tunnel

		if option.Config.IsFlannelMasterDeviceSet() {
			args[initArgMode] = "flannel"
			args[initArgDevice] = option.Config.FlannelMasterDevice
		}
	}

	if option.Config.EnableEndpointRoutes == true {
		args[initArgMode] = "routed"
	}

	if option.Config.EnableNodePort {
		args[initArgNodePort] = "true"
	}

	log.Info("Setting up base BPF datapath")

	for _, s := range sysSettings {
		log.Infof("Setting sysctl %s=%s", s.name, s.val)
		if err := sysctl.Write(s.name, s.val); err != nil {
			if !s.ignoreErr {
				return fmt.Errorf("Failed to sysctl -w %s=%s: %s", s.name, s.val, err)
			}
			log.WithError(err).WithFields(logrus.Fields{
				logfields.SysParamName:  s.name,
				logfields.SysParamValue: s.val,
			}).Warning("Failed to sysctl -w")
		}
	}

	prog := filepath.Join(option.Config.BpfDir, "init.sh")
	ctx, cancel := context.WithTimeout(ctx, defaults.ExecTimeout)
	defer cancel()
	cmd := exec.CommandContext(ctx, prog, args...)
	cmd.Env = bpf.Environment()
	if _, err := cmd.CombinedOutput(log, true); err != nil {
		return err
	}

	if l.canDisableDwarfRelocations {
		// Validate alignments of C and Go equivalent structs
		if err := alignchecker.CheckStructAlignments(defaults.AlignCheckerName); err != nil {
			log.WithError(err).Fatal("C and Go structs alignment check failed")
		}
	} else {
		log.Warning("Cannot check matching of C and Go common struct alignments due to old LLVM/clang version")
	}

	if !option.Config.IsFlannelMasterDeviceSet() {
		r.ReserveLocalRoutes()
	}

	if err := o.Datapath().Node().NodeConfigurationChanged(*o.LocalConfig()); err != nil {
		return err
	}

	if option.Config.InstallIptRules {
		if err := iptMgr.TransientRulesStart(option.Config.HostDevice); err != nil {
			return err
		}
	}
	// Always remove masquerade rule and then re-add it if required
	iptMgr.RemoveRules()
	if option.Config.InstallIptRules {
		err := iptMgr.InstallRules(option.Config.HostDevice)
		iptMgr.TransientRulesEnd(false)
		if err != nil {
			return err
		}
	}
	// Reinstall proxy rules for any running proxies
	if p != nil {
		p.ReinstallRules()
	}

	return nil
}
<|endoftext|>


<commit_before>package ingester

import (
	"testing"

	"github.com/prometheus/common/model"
)

var (
	// cm11, cm12, cm13 are colliding with fp1.
	// cm21, cm22 are colliding with fp2.
	// cm31, cm32 are colliding with fp3, which is below maxMappedFP.
	// Note that fingerprints are set and not actually calculated.
	// The collision detection is independent from the actually used
	// fingerprinting algorithm.
	fp1  = model.Fingerprint(maxMappedFP + 1)
	fp2  = model.Fingerprint(maxMappedFP + 2)
	fp3  = model.Fingerprint(1)
	cm11 = model.Metric{
		"foo":   "bar",
		"dings": "bumms",
	}
	cm12 = model.Metric{
		"bar": "foo",
	}
	cm13 = model.Metric{
		"foo": "bar",
	}
	cm21 = model.Metric{
		"foo":   "bumms",
		"dings": "bar",
	}
	cm22 = model.Metric{
		"dings": "foo",
		"bar":   "bumms",
	}
	cm31 = model.Metric{
		"bumms": "dings",
	}
	cm32 = model.Metric{
		"bumms": "dings",
		"bar":   "foo",
	}
)

func TestFPMapper(t *testing.T) {
	sm := newSeriesMap()

	mapper := newFPMapper(sm)

	// Everything is empty, resolving a FP should do nothing.
	gotFP := mapper.mapFP(fp1, cm11)
	if wantFP := fp1; gotFP != wantFP {
		t.Errorf("got fingerprint %v, want fingerprint %v", gotFP, wantFP)
	}
	gotFP = mapper.mapFP(fp1, cm12)
	if wantFP := fp1; gotFP != wantFP {
		t.Errorf("got fingerprint %v, want fingerprint %v", gotFP, wantFP)
	}

	// cm11 is in sm. Adding cm11 should do nothing. Mapping cm12 should resolve
	// the collision.
	sm.put(fp1, &memorySeries{metric: cm11})
	gotFP = mapper.mapFP(fp1, cm11)
	if wantFP := fp1; gotFP != wantFP {
		t.Errorf("got fingerprint %v, want fingerprint %v", gotFP, wantFP)
	}
	gotFP = mapper.mapFP(fp1, cm12)
	if wantFP := model.Fingerprint(1); gotFP != wantFP {
		t.Errorf("got fingerprint %v, want fingerprint %v", gotFP, wantFP)
	}

	// The mapped cm12 is added to sm, too. That should not change the outcome.
	sm.put(model.Fingerprint(1), &memorySeries{metric: cm12})
	gotFP = mapper.mapFP(fp1, cm11)
	if wantFP := fp1; gotFP != wantFP {
		t.Errorf("got fingerprint %v, want fingerprint %v", gotFP, wantFP)
	}
	gotFP = mapper.mapFP(fp1, cm12)
	if wantFP := model.Fingerprint(1); gotFP != wantFP {
		t.Errorf("got fingerprint %v, want fingerprint %v", gotFP, wantFP)
	}

	// Now map cm13, should reproducibly result in the next mapped FP.
	gotFP = mapper.mapFP(fp1, cm13)
	if wantFP := model.Fingerprint(2); gotFP != wantFP {
		t.Errorf("got fingerprint %v, want fingerprint %v", gotFP, wantFP)
	}
	gotFP = mapper.mapFP(fp1, cm13)
	if wantFP := model.Fingerprint(2); gotFP != wantFP {
		t.Errorf("got fingerprint %v, want fingerprint %v", gotFP, wantFP)
	}

	// Add cm13 to sm. Should not change anything.
	sm.put(model.Fingerprint(2), &memorySeries{metric: cm13})
	gotFP = mapper.mapFP(fp1, cm11)
	if wantFP := fp1; gotFP != wantFP {
		t.Errorf("got fingerprint %v, want fingerprint %v", gotFP, wantFP)
	}
	gotFP = mapper.mapFP(fp1, cm12)
	if wantFP := model.Fingerprint(1); gotFP != wantFP {
		t.Errorf("got fingerprint %v, want fingerprint %v", gotFP, wantFP)
	}
	gotFP = mapper.mapFP(fp1, cm13)
	if wantFP := model.Fingerprint(2); gotFP != wantFP {
		t.Errorf("got fingerprint %v, want fingerprint %v", gotFP, wantFP)
	}

	// Now add cm21 and cm22 in the same way, checking the mapped FPs.
	gotFP = mapper.mapFP(fp2, cm21)
	if wantFP := fp2; gotFP != wantFP {
		t.Errorf("got fingerprint %v, want fingerprint %v", gotFP, wantFP)
	}
	sm.put(fp2, &memorySeries{metric: cm21})
	gotFP = mapper.mapFP(fp2, cm21)
	if wantFP := fp2; gotFP != wantFP {
		t.Errorf("got fingerprint %v, want fingerprint %v", gotFP, wantFP)
	}
	gotFP = mapper.mapFP(fp2, cm22)
	if wantFP := model.Fingerprint(3); gotFP != wantFP {
		t.Errorf("got fingerprint %v, want fingerprint %v", gotFP, wantFP)
	}
	sm.put(model.Fingerprint(3), &memorySeries{metric: cm22})
	gotFP = mapper.mapFP(fp2, cm21)
	if wantFP := fp2; gotFP != wantFP {
		t.Errorf("got fingerprint %v, want fingerprint %v", gotFP, wantFP)
	}
	gotFP = mapper.mapFP(fp2, cm22)
	if wantFP := model.Fingerprint(3); gotFP != wantFP {
		t.Errorf("got fingerprint %v, want fingerprint %v", gotFP, wantFP)
	}

	// Map cm31, resulting in a mapping straight away.
	gotFP = mapper.mapFP(fp3, cm31)
	if wantFP := model.Fingerprint(4); gotFP != wantFP {
		t.Errorf("got fingerprint %v, want fingerprint %v", gotFP, wantFP)
	}
	sm.put(model.Fingerprint(4), &memorySeries{metric: cm31})

	// Map cm32, which is now mapped for two reasons...
	gotFP = mapper.mapFP(fp3, cm32)
	if wantFP := model.Fingerprint(5); gotFP != wantFP {
		t.Errorf("got fingerprint %v, want fingerprint %v", gotFP, wantFP)
	}
	sm.put(model.Fingerprint(5), &memorySeries{metric: cm32})

	// Now check ALL the mappings, just to be sure.
	gotFP = mapper.mapFP(fp1, cm11)
	if wantFP := fp1; gotFP != wantFP {
		t.Errorf("got fingerprint %v, want fingerprint %v", gotFP, wantFP)
	}
	gotFP = mapper.mapFP(fp1, cm12)
	if wantFP := model.Fingerprint(1); gotFP != wantFP {
		t.Errorf("got fingerprint %v, want fingerprint %v", gotFP, wantFP)
	}
	gotFP = mapper.mapFP(fp1, cm13)
	if wantFP := model.Fingerprint(2); gotFP != wantFP {
		t.Errorf("got fingerprint %v, want fingerprint %v", gotFP, wantFP)
	}
	gotFP = mapper.mapFP(fp2, cm21)
	if wantFP := fp2; gotFP != wantFP {
		t.Errorf("got fingerprint %v, want fingerprint %v", gotFP, wantFP)
	}
	gotFP = mapper.mapFP(fp2, cm22)
	if wantFP := model.Fingerprint(3); gotFP != wantFP {
		t.Errorf("got fingerprint %v, want fingerprint %v", gotFP, wantFP)
	}
	gotFP = mapper.mapFP(fp3, cm31)
	if wantFP := model.Fingerprint(4); gotFP != wantFP {
		t.Errorf("got fingerprint %v, want fingerprint %v", gotFP, wantFP)
	}
	gotFP = mapper.mapFP(fp3, cm32)
	if wantFP := model.Fingerprint(5); gotFP != wantFP {
		t.Errorf("got fingerprint %v, want fingerprint %v", gotFP, wantFP)
	}

	// Remove all the fingerprints from sm, which should change nothing, as
	// the existing mappings stay and should be detected.
	sm.del(fp1)
	sm.del(fp2)
	sm.del(fp3)
	gotFP = mapper.mapFP(fp1, cm11)
	if wantFP := fp1; gotFP != wantFP {
		t.Errorf("got fingerprint %v, want fingerprint %v", gotFP, wantFP)
	}
	gotFP = mapper.mapFP(fp1, cm12)
	if wantFP := model.Fingerprint(1); gotFP != wantFP {
		t.Errorf("got fingerprint %v, want fingerprint %v", gotFP, wantFP)
	}
	gotFP = mapper.mapFP(fp1, cm13)
	if wantFP := model.Fingerprint(2); gotFP != wantFP {
		t.Errorf("got fingerprint %v, want fingerprint %v", gotFP, wantFP)
	}
	gotFP = mapper.mapFP(fp2, cm21)
	if wantFP := fp2; gotFP != wantFP {
		t.Errorf("got fingerprint %v, want fingerprint %v", gotFP, wantFP)
	}
	gotFP = mapper.mapFP(fp2, cm22)
	if wantFP := model.Fingerprint(3); gotFP != wantFP {
		t.Errorf("got fingerprint %v, want fingerprint %v", gotFP, wantFP)
	}
	gotFP = mapper.mapFP(fp3, cm31)
	if wantFP := model.Fingerprint(4); gotFP != wantFP {
		t.Errorf("got fingerprint %v, want fingerprint %v", gotFP, wantFP)
	}
	gotFP = mapper.mapFP(fp3, cm32)
	if wantFP := model.Fingerprint(5); gotFP != wantFP {
		t.Errorf("got fingerprint %v, want fingerprint %v", gotFP, wantFP)
	}
}
<commit_msg>Simplifier mapper tests (#651)<commit_after>package ingester

import (
	"testing"

	"github.com/prometheus/common/model"
)

var (
	// cm11, cm12, cm13 are colliding with fp1.
	// cm21, cm22 are colliding with fp2.
	// cm31, cm32 are colliding with fp3, which is below maxMappedFP.
	// Note that fingerprints are set and not actually calculated.
	// The collision detection is independent from the actually used
	// fingerprinting algorithm.
	fp1  = model.Fingerprint(maxMappedFP + 1)
	fp2  = model.Fingerprint(maxMappedFP + 2)
	fp3  = model.Fingerprint(1)
	cm11 = model.Metric{
		"foo":   "bar",
		"dings": "bumms",
	}
	cm12 = model.Metric{
		"bar": "foo",
	}
	cm13 = model.Metric{
		"foo": "bar",
	}
	cm21 = model.Metric{
		"foo":   "bumms",
		"dings": "bar",
	}
	cm22 = model.Metric{
		"dings": "foo",
		"bar":   "bumms",
	}
	cm31 = model.Metric{
		"bumms": "dings",
	}
	cm32 = model.Metric{
		"bumms": "dings",
		"bar":   "foo",
	}
)

func TestFPMapper(t *testing.T) {
	sm := newSeriesMap()

	mapper := newFPMapper(sm)

	// Everything is empty, resolving a FP should do nothing.
	assertFingerprintEqual(t, mapper.mapFP(fp1, cm11), fp1)
	assertFingerprintEqual(t, mapper.mapFP(fp1, cm12), fp1)

	// cm11 is in sm. Adding cm11 should do nothing. Mapping cm12 should resolve
	// the collision.
	sm.put(fp1, &memorySeries{metric: cm11})
	assertFingerprintEqual(t, mapper.mapFP(fp1, cm11), fp1)
	assertFingerprintEqual(t, mapper.mapFP(fp1, cm12), model.Fingerprint(1))

	// The mapped cm12 is added to sm, too. That should not change the outcome.
	sm.put(model.Fingerprint(1), &memorySeries{metric: cm12})
	assertFingerprintEqual(t, mapper.mapFP(fp1, cm11), fp1)
	assertFingerprintEqual(t, mapper.mapFP(fp1, cm12), model.Fingerprint(1))

	// Now map cm13, should reproducibly result in the next mapped FP.
	assertFingerprintEqual(t, mapper.mapFP(fp1, cm13), model.Fingerprint(2))
	assertFingerprintEqual(t, mapper.mapFP(fp1, cm13), model.Fingerprint(2))

	// Add cm13 to sm. Should not change anything.
	sm.put(model.Fingerprint(2), &memorySeries{metric: cm13})
	assertFingerprintEqual(t, mapper.mapFP(fp1, cm11), fp1)
	assertFingerprintEqual(t, mapper.mapFP(fp1, cm12), model.Fingerprint(1))
	assertFingerprintEqual(t, mapper.mapFP(fp1, cm13), model.Fingerprint(2))

	// Now add cm21 and cm22 in the same way, checking the mapped FPs.
	assertFingerprintEqual(t, mapper.mapFP(fp2, cm21), fp2)
	sm.put(fp2, &memorySeries{metric: cm21})
	assertFingerprintEqual(t, mapper.mapFP(fp2, cm21), fp2)
	assertFingerprintEqual(t, mapper.mapFP(fp2, cm22), model.Fingerprint(3))
	sm.put(model.Fingerprint(3), &memorySeries{metric: cm22})
	assertFingerprintEqual(t, mapper.mapFP(fp2, cm21), fp2)
	assertFingerprintEqual(t, mapper.mapFP(fp2, cm22), model.Fingerprint(3))

	// Map cm31, resulting in a mapping straight away.
	assertFingerprintEqual(t, mapper.mapFP(fp3, cm31), model.Fingerprint(4))
	sm.put(model.Fingerprint(4), &memorySeries{metric: cm31})

	// Map cm32, which is now mapped for two reasons...
	assertFingerprintEqual(t, mapper.mapFP(fp3, cm32), model.Fingerprint(5))
	sm.put(model.Fingerprint(5), &memorySeries{metric: cm32})

	// Now check ALL the mappings, just to be sure.
	assertFingerprintEqual(t, mapper.mapFP(fp1, cm11), fp1)
	assertFingerprintEqual(t, mapper.mapFP(fp1, cm12), model.Fingerprint(1))
	assertFingerprintEqual(t, mapper.mapFP(fp1, cm13), model.Fingerprint(2))
	assertFingerprintEqual(t, mapper.mapFP(fp2, cm21), fp2)
	assertFingerprintEqual(t, mapper.mapFP(fp2, cm22), model.Fingerprint(3))
	assertFingerprintEqual(t, mapper.mapFP(fp3, cm31), model.Fingerprint(4))
	assertFingerprintEqual(t, mapper.mapFP(fp3, cm32), model.Fingerprint(5))

	// Remove all the fingerprints from sm, which should change nothing, as
	// the existing mappings stay and should be detected.
	sm.del(fp1)
	sm.del(fp2)
	sm.del(fp3)
	assertFingerprintEqual(t, mapper.mapFP(fp1, cm11), fp1)
	assertFingerprintEqual(t, mapper.mapFP(fp1, cm12), model.Fingerprint(1))
	assertFingerprintEqual(t, mapper.mapFP(fp1, cm13), model.Fingerprint(2))
	assertFingerprintEqual(t, mapper.mapFP(fp2, cm21), fp2)
	assertFingerprintEqual(t, mapper.mapFP(fp2, cm22), model.Fingerprint(3))
	assertFingerprintEqual(t, mapper.mapFP(fp3, cm31), model.Fingerprint(4))
	assertFingerprintEqual(t, mapper.mapFP(fp3, cm32), model.Fingerprint(5))
}

// assertFingerprintEqual asserts that two fingerprints are equal.
func assertFingerprintEqual(t *testing.T, gotFP, wantFP model.Fingerprint) {
	if gotFP != wantFP {
		t.Errorf("got fingerprint %v, want fingerprint %v", gotFP, wantFP)
	}
}
<|endoftext|>


<commit_before>/*
Copyright 2014 Google Inc. All rights reserved.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package kubecfg

import (
	"fmt"
	"net/http"

	"github.com/GoogleCloudPlatform/kubernetes/pkg/api"
	"github.com/GoogleCloudPlatform/kubernetes/pkg/api/latest"
	"github.com/GoogleCloudPlatform/kubernetes/pkg/client"
)

// ProxyServer is a http.Handler which proxies Kubernetes APIs to remote API server.
type ProxyServer struct {
	Client *client.Client
}

func newFileHandler(prefix, base string) http.Handler {
	return http.StripPrefix(prefix, http.FileServer(http.Dir(base)))
}

// NewProxyServer creates and installs a new ProxyServer.
// It automatically registers the created ProxyServer to http.DefaultServeMux.
func NewProxyServer(filebase string, kubeClient *client.Client) *ProxyServer {
	server := &ProxyServer{
		Client: kubeClient,
	}
	http.Handle("/api/", server)
	http.Handle("/static/", newFileHandler("/static/", filebase))
	return server
}

// Serve starts the server (http.DefaultServeMux) on TCP port 8001, loops forever.
func (s *ProxyServer) Serve() error {
	return http.ListenAndServe(":8001", nil)
}

func (s *ProxyServer) doError(w http.ResponseWriter, err error) {
	w.WriteHeader(http.StatusInternalServerError)
	w.Header().Add("Content-type", "application/json")
	data, _ := latest.Codec.Encode(&api.Status{
		Status:  api.StatusFailure,
		Message: fmt.Sprintf("internal error: %#v", err),
	})
	w.Write(data)
}

func (s *ProxyServer) ServeHTTP(w http.ResponseWriter, r *http.Request) {
	result := s.Client.Verb(r.Method).AbsPath(r.URL.Path).Body(r.Body).Do()
	if result.Error() != nil {
		s.doError(w, result.Error())
		return
	}
	data, err := result.Raw()
	if err != nil {
		s.doError(w, err)
		return
	}
	w.Header().Add("Content-type", "application/json")
	w.WriteHeader(http.StatusOK)
	w.Write(data)
}
<commit_msg>Update the proxy server so that it passes labels along too.<commit_after>/*
Copyright 2014 Google Inc. All rights reserved.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package kubecfg

import (
	"fmt"
	"net/http"

	"github.com/GoogleCloudPlatform/kubernetes/pkg/api"
	"github.com/GoogleCloudPlatform/kubernetes/pkg/api/latest"
	"github.com/GoogleCloudPlatform/kubernetes/pkg/client"
)

// ProxyServer is a http.Handler which proxies Kubernetes APIs to remote API server.
type ProxyServer struct {
	Client *client.Client
}

func newFileHandler(prefix, base string) http.Handler {
	return http.StripPrefix(prefix, http.FileServer(http.Dir(base)))
}

// NewProxyServer creates and installs a new ProxyServer.
// It automatically registers the created ProxyServer to http.DefaultServeMux.
func NewProxyServer(filebase string, kubeClient *client.Client) *ProxyServer {
	server := &ProxyServer{
		Client: kubeClient,
	}
	http.Handle("/api/", server)
	http.Handle("/static/", newFileHandler("/static/", filebase))
	return server
}

// Serve starts the server (http.DefaultServeMux) on TCP port 8001, loops forever.
func (s *ProxyServer) Serve() error {
	return http.ListenAndServe(":8001", nil)
}

func (s *ProxyServer) doError(w http.ResponseWriter, err error) {
	w.WriteHeader(http.StatusInternalServerError)
	w.Header().Add("Content-type", "application/json")
	data, _ := latest.Codec.Encode(&api.Status{
		Status:  api.StatusFailure,
		Message: fmt.Sprintf("internal error: %#v", err),
	})
	w.Write(data)
}

func (s *ProxyServer) ServeHTTP(w http.ResponseWriter, r *http.Request) {
	url := r.URL
	selector := url.Query().Get("labels")
	result := s.Client.Verb(r.Method).AbsPath(r.URL.Path).ParseSelectorParam("labels", selector).Body(r.Body).Do()
	if result.Error() != nil {
		s.doError(w, result.Error())
		return
	}
	data, err := result.Raw()
	if err != nil {
		s.doError(w, err)
		return
	}
	w.Header().Add("Content-type", "application/json")
	w.WriteHeader(http.StatusOK)
	w.Write(data)
}
<|endoftext|>


<commit_before>package core

import (
	"math"
	"math/rand"
)

// UniformRandom emits a uniform random between 0 and 1
func UniformRandom() Spec {
	return Spec{
		Name:    "uniform",
		Inputs:  []Pin{Pin{"trigger", ANY}},
		Outputs: []Pin{Pin{"draw", NUMBER}},
		Kernel: func(in, out, internal MessageMap, s Source, i chan Interrupt) Interrupt {
			out[0] = rand.Float64()
			return nil
		},
	}
}

// NormalRandom emits a normally distributed random number with the
// supplied mean and variance
func NormalRandom() Spec {
	return Spec{
		Name:    "normal",
		Inputs:  []Pin{Pin{"mean", NUMBER}, Pin{"variance", NUMBER}},
		Outputs: []Pin{Pin{"draw", NUMBER}},
		Kernel: func(in, out, internal MessageMap, s Source, i chan Interrupt) Interrupt {
			variance, ok := in[1].(float64)
			if !ok {
				out[0] = NewError("variance must be a number")
				return nil
			}
			mean, ok := in[0].(float64)
			if !ok {
				out[0] = NewError("mean must be a number")
				return nil
			}
			out[0] = rand.NormFloat64()*math.Sqrt(variance) + mean
			return nil
		},
	}
}

// the global random number source
var RAND *rand.Rand = rand.New(rand.NewSource(12345))

// ZipfRandom emits a Zipfian distributed random number
// notation follows the wikipedia page http://en.wikipedia.org/wiki/Zipf%E2%80%93Mandelbrot_law not the golang Zipf parameters
func ZipfRandom() Spec {
	return Spec{
		Name: "Zipf",
		Inputs: []Pin{
			Pin{"q", NUMBER}, Pin{"s", NUMBER}, Pin{"N", NUMBER}},
		Outputs: []Pin{Pin{"draw", NUMBER}},
		Kernel: func(in, out, internal MessageMap, ss Source, i chan Interrupt) Interrupt {

			q, ok := in[0].(float64)
			if !ok {
				out[0] = NewError("q must be a number")
				return nil
			}
			s, ok := in[1].(float64)
			if !ok {
				out[0] = NewError("s must be a number")
				return nil
			}
			N, ok := in[2].(float64)
			if !ok {
				out[0] = NewError("N must be an number")
				return nil
			}

			z := rand.NewZipf(RAND, s, q, uint64(N))
			out[0] = z.Uint64()
			return nil
		},
	}
}

// poisson returns an integer (though we actually pretend it's a float) from a Poisson distrbution
func poisson(λ float64) float64 {
	var k float64
	L := math.Exp(-λ)
	k = 0
	p := 1.0
	for {
		k++
		u := RAND.Float64()
		p = p * u
		if p <= L {
			return k - 1
		}
	}
}

// PoissonRandom emits a Poisson distribtued random number
func PoissonRandom() Spec {
	return Spec{
		Name:    "poisson",
		Inputs:  []Pin{Pin{"rate", NUMBER}},
		Outputs: []Pin{Pin{"draw", NUMBER}},
		Kernel: func(in, out, internal MessageMap, ss Source, i chan Interrupt) Interrupt {
			λ, ok := in[0].(float64)
			if !ok {
				out[0] = NewError("rate must be a number")
				return nil
			}
			if λ < 0 {
				out[0] = NewError("rate must be positive")
				return nil
			}
			out[0] = poisson(λ)
			return nil
		},
	}
}

// BernoulliRandom emits a draw from a Bernoulli distribution. This block returns a boolean
func BernoulliRandom() Spec {
	return Spec{
		Name:    "Bernoulli",
		Inputs:  []Pin{Pin{"bias", NUMBER}},
		Outputs: []Pin{Pin{"draw", NUMBER}},
		Kernel: func(in, out, internal MessageMap, s Source, i chan Interrupt) Interrupt {
			r := RAND.Float64()
			p, ok := in[0].(float64)
			if !ok {
				out[0] = NewError("bias must be a number")
				return nil
			}
			if p < 0 || p > 1 {
				out[0] = NewError("bias must be between 0 and 1")
			}
			if r > p {
				out[0] = false
			} else {
				out[0] = true
			}
			return nil
		},
	}
}
<commit_msg>uniform random number now pushes<commit_after>package core

import (
	"math"
	"math/rand"
)

// UniformRandom emits a uniform random between 0 and 1
func UniformRandom() Spec {
	return Spec{
		Name:    "uniform",
		Inputs:  []Pin{},
		Outputs: []Pin{Pin{"draw", NUMBER}},
		Kernel: func(in, out, internal MessageMap, s Source, i chan Interrupt) Interrupt {
			out[0] = rand.Float64()
			return nil
		},
	}
}

// NormalRandom emits a normally distributed random number with the
// supplied mean and variance
func NormalRandom() Spec {
	return Spec{
		Name:    "normal",
		Inputs:  []Pin{Pin{"mean", NUMBER}, Pin{"variance", NUMBER}},
		Outputs: []Pin{Pin{"draw", NUMBER}},
		Kernel: func(in, out, internal MessageMap, s Source, i chan Interrupt) Interrupt {
			variance, ok := in[1].(float64)
			if !ok {
				out[0] = NewError("variance must be a number")
				return nil
			}
			mean, ok := in[0].(float64)
			if !ok {
				out[0] = NewError("mean must be a number")
				return nil
			}
			out[0] = rand.NormFloat64()*math.Sqrt(variance) + mean
			return nil
		},
	}
}

// the global random number source
var RAND *rand.Rand = rand.New(rand.NewSource(12345))

// ZipfRandom emits a Zipfian distributed random number
// notation follows the wikipedia page http://en.wikipedia.org/wiki/Zipf%E2%80%93Mandelbrot_law not the golang Zipf parameters
func ZipfRandom() Spec {
	return Spec{
		Name: "Zipf",
		Inputs: []Pin{
			Pin{"q", NUMBER}, Pin{"s", NUMBER}, Pin{"N", NUMBER}},
		Outputs: []Pin{Pin{"draw", NUMBER}},
		Kernel: func(in, out, internal MessageMap, ss Source, i chan Interrupt) Interrupt {

			q, ok := in[0].(float64)
			if !ok {
				out[0] = NewError("q must be a number")
				return nil
			}
			s, ok := in[1].(float64)
			if !ok {
				out[0] = NewError("s must be a number")
				return nil
			}
			N, ok := in[2].(float64)
			if !ok {
				out[0] = NewError("N must be an number")
				return nil
			}

			z := rand.NewZipf(RAND, s, q, uint64(N))
			out[0] = z.Uint64()
			return nil
		},
	}
}

// poisson returns an integer (though we actually pretend it's a float) from a Poisson distrbution
func poisson(λ float64) float64 {
	var k float64
	L := math.Exp(-λ)
	k = 0
	p := 1.0
	for {
		k++
		u := RAND.Float64()
		p = p * u
		if p <= L {
			return k - 1
		}
	}
}

// PoissonRandom emits a Poisson distribtued random number
func PoissonRandom() Spec {
	return Spec{
		Name:    "poisson",
		Inputs:  []Pin{Pin{"rate", NUMBER}},
		Outputs: []Pin{Pin{"draw", NUMBER}},
		Kernel: func(in, out, internal MessageMap, ss Source, i chan Interrupt) Interrupt {
			λ, ok := in[0].(float64)
			if !ok {
				out[0] = NewError("rate must be a number")
				return nil
			}
			if λ < 0 {
				out[0] = NewError("rate must be positive")
				return nil
			}
			out[0] = poisson(λ)
			return nil
		},
	}
}

// BernoulliRandom emits a draw from a Bernoulli distribution. This block returns a boolean
func BernoulliRandom() Spec {
	return Spec{
		Name:    "Bernoulli",
		Inputs:  []Pin{Pin{"bias", NUMBER}},
		Outputs: []Pin{Pin{"draw", NUMBER}},
		Kernel: func(in, out, internal MessageMap, s Source, i chan Interrupt) Interrupt {
			r := RAND.Float64()
			p, ok := in[0].(float64)
			if !ok {
				out[0] = NewError("bias must be a number")
				return nil
			}
			if p < 0 || p > 1 {
				out[0] = NewError("bias must be between 0 and 1")
			}
			if r > p {
				out[0] = false
			} else {
				out[0] = true
			}
			return nil
		},
	}
}
<|endoftext|>


<commit_before>package agent

import (
	"os"

	"github.com/aws/aws-sdk-go/aws"
	"github.com/aws/aws-sdk-go/aws/ec2metadata"
	"github.com/aws/aws-sdk-go/aws/session"
)

var awsSess *session.Session

// The aws sdk relies on being given a region, which is a breaking change for us
// This applies a heuristic that detects where the agent might be based on the env
// but also the local isntance metadata if available
func awsRegion() (string, error) {
	if r := os.Getenv("AWS_REGION"); r != "" {
		return r, nil
	}

	if r := os.Getenv("AWS_DEFAULT_REGION"); r != "" {
		return r, nil
	}

	// The metadata service seems to want a session
	sess, err := session.NewSession(&aws.Config{
		Region: aws.String("us-east-1"),
	})
	if err != nil {
		return "", err
	}

	meta := ec2metadata.New(sess)
	if meta.Available() {
		return meta.Region()
	}

	return "", aws.ErrMissingRegion
}

func awsSession() (*session.Session, error) {
	region, err := awsRegion()
	if err != nil {
		return nil, err
	}

	if awsSess == nil {
		awsSess, err = session.NewSession(&aws.Config{
			Region: aws.String(region),
		})
		if err != nil {
			return nil, err
		}
	}

	return awsSess, nil
}
<commit_msg>Show debug message when region is detected<commit_after>package agent

import (
	"os"

	"github.com/aws/aws-sdk-go/aws"
	"github.com/aws/aws-sdk-go/aws/ec2metadata"
	"github.com/aws/aws-sdk-go/aws/session"
	"github.com/buildkite/agent/logger"
)

var awsSess *session.Session

// The aws sdk relies on being given a region, which is a breaking change for us
// This applies a heuristic that detects where the agent might be based on the env
// but also the local isntance metadata if available
func awsRegion() (string, error) {
	if r := os.Getenv("AWS_REGION"); r != "" {
		return r, nil
	}

	if r := os.Getenv("AWS_DEFAULT_REGION"); r != "" {
		return r, nil
	}

	// The metadata service seems to want a session
	sess, err := session.NewSession(&aws.Config{
		Region: aws.String("us-east-1"),
	})
	if err != nil {
		return "", err
	}

	meta := ec2metadata.New(sess)
	if meta.Available() {
		region, err := meta.Region()
		if err == nil {
			logger.Debug("Detected AWS region %s", region)
		}
		return region, err
	}

	return "", aws.ErrMissingRegion
}

func awsSession() (*session.Session, error) {
	region, err := awsRegion()
	if err != nil {
		return nil, err
	}

	if awsSess == nil {
		awsSess, err = session.NewSession(&aws.Config{
			Region: aws.String(region),
		})
		if err != nil {
			return nil, err
		}
	}

	return awsSess, nil
}
<|endoftext|>


<commit_before>/*
 * MinIO Cloud Storage, (C) 2017 MinIO, Inc.
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 *     http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 *
 */

package madmin

import (
	"context"
	"encoding/json"
	"io/ioutil"
	"net/http"
	"time"
)

// BackendType - represents different backend types.
type BackendType int

// Enum for different backend types.
const (
	Unknown BackendType = iota
	// Filesystem backend.
	FS
	// Multi disk Erasure (single, distributed) backend.
	Erasure
	// Gateway to other storage
	Gateway

	// Add your own backend.
)

// ItemState - represents the status of any item in offline,init,online state
type ItemState string

const (

	// ItemOffline indicates that the item is offline
	ItemOffline = ItemState("offline")
	// ItemInitializing indicates that the item is still in initialization phase
	ItemInitializing = ItemState("initializing")
	// ItemOnline indicates that the item is online
	ItemOnline = ItemState("online")
)

// StorageInfo - represents total capacity of underlying storage.
type StorageInfo struct {
	Disks []Disk

	// Backend type.
	Backend BackendInfo
}

// BackendInfo - contains info of the underlying backend
type BackendInfo struct {
	// Represents various backend types, currently on FS, Erasure and Gateway
	Type BackendType

	// Following fields are only meaningful if BackendType is Gateway.
	GatewayOnline bool

	// Following fields are only meaningful if BackendType is Erasure.
	OnlineDisks  BackendDisks // Online disks during server startup.
	OfflineDisks BackendDisks // Offline disks during server startup.

	// Following fields are only meaningful if BackendType is Erasure.
	StandardSCData   []int // Data disks for currently configured Standard storage class.
	StandardSCParity int   // Parity disks for currently configured Standard storage class.
	RRSCData         []int // Data disks for currently configured Reduced Redundancy storage class.
	RRSCParity       int   // Parity disks for currently configured Reduced Redundancy storage class.
}

// BackendDisks - represents the map of endpoint-disks.
type BackendDisks map[string]int

// Sum - Return the sum of the disks in the endpoint-disk map.
func (d1 BackendDisks) Sum() (sum int) {
	for _, count := range d1 {
		sum += count
	}
	return sum
}

// Merge - Reduces two endpoint-disk maps.
func (d1 BackendDisks) Merge(d2 BackendDisks) BackendDisks {
	if len(d2) == 0 {
		d2 = make(BackendDisks)
	}
	for i1, v1 := range d1 {
		if v2, ok := d2[i1]; ok {
			d2[i1] = v2 + v1
			continue
		}
		d2[i1] = v1
	}
	return d2
}

// StorageInfo - Connect to a minio server and call Storage Info Management API
// to fetch server's information represented by StorageInfo structure
func (adm *AdminClient) StorageInfo(ctx context.Context) (StorageInfo, error) {
	resp, err := adm.executeMethod(ctx, http.MethodGet, requestData{relPath: adminAPIPrefix + "/storageinfo"})
	defer closeResponse(resp)
	if err != nil {
		return StorageInfo{}, err
	}

	// Check response http status code
	if resp.StatusCode != http.StatusOK {
		return StorageInfo{}, httpRespToErrorResponse(resp)
	}

	// Unmarshal the server's json response
	var storageInfo StorageInfo

	respBytes, err := ioutil.ReadAll(resp.Body)
	if err != nil {
		return StorageInfo{}, err
	}

	err = json.Unmarshal(respBytes, &storageInfo)
	if err != nil {
		return StorageInfo{}, err
	}

	return storageInfo, nil
}

// DataUsageInfo represents data usage of an Object API
type DataUsageInfo struct {
	// LastUpdate is the timestamp of when the data usage info was last updated.
	// This does not indicate a full scan.
	LastUpdate       time.Time `json:"lastUpdate"`
	ObjectsCount     uint64    `json:"objectsCount"`
	ObjectsTotalSize uint64    `json:"objectsTotalSize"`

	// ObjectsSizesHistogram contains information on objects across all buckets.
	// See ObjectsHistogramIntervals.
	ObjectsSizesHistogram map[string]uint64 `json:"objectsSizesHistogram"`

	BucketsCount uint64 `json:"bucketsCount"`

	// BucketsSizes is "bucket name" -> size.
	BucketsSizes map[string]uint64 `json:"bucketsSizes"`
}

// DataUsageInfo - returns data usage of the current object API
func (adm *AdminClient) DataUsageInfo(ctx context.Context) (DataUsageInfo, error) {
	resp, err := adm.executeMethod(ctx, http.MethodGet, requestData{relPath: adminAPIPrefix + "/datausageinfo"})
	defer closeResponse(resp)
	if err != nil {
		return DataUsageInfo{}, err
	}

	// Check response http status code
	if resp.StatusCode != http.StatusOK {
		return DataUsageInfo{}, httpRespToErrorResponse(resp)
	}

	// Unmarshal the server's json response
	var dataUsageInfo DataUsageInfo

	respBytes, err := ioutil.ReadAll(resp.Body)
	if err != nil {
		return DataUsageInfo{}, err
	}

	err = json.Unmarshal(respBytes, &dataUsageInfo)
	if err != nil {
		return DataUsageInfo{}, err
	}

	return dataUsageInfo, nil
}

// InfoMessage container to hold server admin related information.
type InfoMessage struct {
	Mode         string             `json:"mode,omitempty"`
	Domain       []string           `json:"domain,omitempty"`
	Region       string             `json:"region,omitempty"`
	SQSARN       []string           `json:"sqsARN,omitempty"`
	DeploymentID string             `json:"deploymentID,omitempty"`
	Buckets      Buckets            `json:"buckets,omitempty"`
	Objects      Objects            `json:"objects,omitempty"`
	Usage        Usage              `json:"usage,omitempty"`
	Services     Services           `json:"services,omitempty"`
	Backend      interface{}        `json:"backend,omitempty"`
	Servers      []ServerProperties `json:"servers,omitempty"`
}

// Services contains different services information
type Services struct {
	KMS           KMS                           `json:"kms,omitempty"`
	LDAP          LDAP                          `json:"ldap,omitempty"`
	Logger        []Logger                      `json:"logger,omitempty"`
	Audit         []Audit                       `json:"audit,omitempty"`
	Notifications []map[string][]TargetIDStatus `json:"notifications,omitempty"`
}

// Buckets contains the number of buckets
type Buckets struct {
	Count uint64 `json:"count"`
	Error string `json:"error,omitempty"`
}

// Objects contains the number of objects
type Objects struct {
	Count uint64 `json:"count"`
	Error string `json:"error,omitempty"`
}

// Usage contains the total size used
type Usage struct {
	Size  uint64 `json:"size"`
	Error string `json:"error,omitempty"`
}

// KMS contains KMS status information
type KMS struct {
	Status  string `json:"status,omitempty"`
	Encrypt string `json:"encrypt,omitempty"`
	Decrypt string `json:"decrypt,omitempty"`
}

// LDAP contains ldap status
type LDAP struct {
	Status string `json:"status,omitempty"`
}

// Status of endpoint
type Status struct {
	Status string `json:"status,omitempty"`
}

// Audit contains audit logger status
type Audit map[string]Status

// Logger contains logger status
type Logger map[string]Status

// TargetIDStatus containsid and status
type TargetIDStatus map[string]Status

// backendType - indicates the type of backend storage
type backendType string

const (
	// FsType - Backend is FS Type
	FsType = backendType("FS")
	// ErasureType - Backend is Erasure type
	ErasureType = backendType("Erasure")
)

// FSBackend contains specific FS storage information
type FSBackend struct {
	Type backendType `json:"backendType,omitempty"`
}

// ErasureBackend contains specific erasure storage information
type ErasureBackend struct {
	Type         backendType `json:"backendType,omitempty"`
	OnlineDisks  int         `json:"onlineDisks,omitempty"`
	OfflineDisks int         `json:"offlineDisks,omitempty"`
	// Parity disks for currently configured Standard storage class.
	StandardSCParity int `json:"standardSCParity,omitempty"`
	// Parity disks for currently configured Reduced Redundancy storage class.
	RRSCParity int `json:"rrSCParity,omitempty"`
}

// ServerProperties holds server information
type ServerProperties struct {
	State      string            `json:"state,omitempty"`
	Endpoint   string            `json:"endpoint,omitempty"`
	Uptime     int64             `json:"uptime,omitempty"`
	Version    string            `json:"version,omitempty"`
	CommitID   string            `json:"commitID,omitempty"`
	Network    map[string]string `json:"network,omitempty"`
	Disks      []Disk            `json:"drives,omitempty"`
	PoolNumber int               `json:"poolNumber,omitempty"`
}

// Disk holds Disk information
type Disk struct {
	Endpoint        string       `json:"endpoint,omitempty"`
	RootDisk        bool         `json:"rootDisk,omitempty"`
	DrivePath       string       `json:"path,omitempty"`
	Healing         bool         `json:"healing,omitempty"`
	State           string       `json:"state,omitempty"`
	UUID            string       `json:"uuid,omitempty"`
	Model           string       `json:"model,omitempty"`
	TotalSpace      uint64       `json:"totalspace,omitempty"`
	UsedSpace       uint64       `json:"usedspace,omitempty"`
	AvailableSpace  uint64       `json:"availspace,omitempty"`
	ReadThroughput  float64      `json:"readthroughput,omitempty"`
	WriteThroughPut float64      `json:"writethroughput,omitempty"`
	ReadLatency     float64      `json:"readlatency,omitempty"`
	WriteLatency    float64      `json:"writelatency,omitempty"`
	Utilization     float64      `json:"utilization,omitempty"`
	HealInfo        *HealingDisk `json:"heal_info,omitempty"`

	// Indexes, will be -1 until assigned a set.
	PoolIndex int `json:"pool_index"`
	SetIndex  int `json:"set_index"`
	DiskIndex int `json:"disk_index"`
}

// ServerInfo - Connect to a minio server and call Server Admin Info Management API
// to fetch server's information represented by infoMessage structure
func (adm *AdminClient) ServerInfo(ctx context.Context) (InfoMessage, error) {
	resp, err := adm.executeMethod(ctx,
		http.MethodGet,
		requestData{relPath: adminAPIPrefix + "/info"},
	)
	defer closeResponse(resp)
	if err != nil {
		return InfoMessage{}, err
	}

	// Check response http status code
	if resp.StatusCode != http.StatusOK {
		return InfoMessage{}, httpRespToErrorResponse(resp)
	}

	// Unmarshal the server's json response
	var message InfoMessage

	respBytes, err := ioutil.ReadAll(resp.Body)
	if err != nil {
		return InfoMessage{}, err
	}

	err = json.Unmarshal(respBytes, &message)
	if err != nil {
		return InfoMessage{}, err
	}

	return message, nil
}
<commit_msg>fix: Merge() should merge and return a copy (#11714)<commit_after>/*
 * MinIO Cloud Storage, (C) 2017 MinIO, Inc.
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 *     http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 *
 */

package madmin

import (
	"context"
	"encoding/json"
	"io/ioutil"
	"net/http"
	"time"
)

// BackendType - represents different backend types.
type BackendType int

// Enum for different backend types.
const (
	Unknown BackendType = iota
	// Filesystem backend.
	FS
	// Multi disk Erasure (single, distributed) backend.
	Erasure
	// Gateway to other storage
	Gateway

	// Add your own backend.
)

// ItemState - represents the status of any item in offline,init,online state
type ItemState string

const (

	// ItemOffline indicates that the item is offline
	ItemOffline = ItemState("offline")
	// ItemInitializing indicates that the item is still in initialization phase
	ItemInitializing = ItemState("initializing")
	// ItemOnline indicates that the item is online
	ItemOnline = ItemState("online")
)

// StorageInfo - represents total capacity of underlying storage.
type StorageInfo struct {
	Disks []Disk

	// Backend type.
	Backend BackendInfo
}

// BackendInfo - contains info of the underlying backend
type BackendInfo struct {
	// Represents various backend types, currently on FS, Erasure and Gateway
	Type BackendType

	// Following fields are only meaningful if BackendType is Gateway.
	GatewayOnline bool

	// Following fields are only meaningful if BackendType is Erasure.
	OnlineDisks  BackendDisks // Online disks during server startup.
	OfflineDisks BackendDisks // Offline disks during server startup.

	// Following fields are only meaningful if BackendType is Erasure.
	StandardSCData   []int // Data disks for currently configured Standard storage class.
	StandardSCParity int   // Parity disks for currently configured Standard storage class.
	RRSCData         []int // Data disks for currently configured Reduced Redundancy storage class.
	RRSCParity       int   // Parity disks for currently configured Reduced Redundancy storage class.
}

// BackendDisks - represents the map of endpoint-disks.
type BackendDisks map[string]int

// Sum - Return the sum of the disks in the endpoint-disk map.
func (d1 BackendDisks) Sum() (sum int) {
	for _, count := range d1 {
		sum += count
	}
	return sum
}

// Merge - Reduces two endpoint-disk maps.
func (d1 BackendDisks) Merge(d2 BackendDisks) BackendDisks {
	if len(d2) == 0 {
		d2 = make(BackendDisks)
	}
	var merged = make(BackendDisks)
	for i1, v1 := range d1 {
		if v2, ok := d2[i1]; ok {
			merged[i1] = v2 + v1
			continue
		}
		merged[i1] = v1
	}
	return merged
}

// StorageInfo - Connect to a minio server and call Storage Info Management API
// to fetch server's information represented by StorageInfo structure
func (adm *AdminClient) StorageInfo(ctx context.Context) (StorageInfo, error) {
	resp, err := adm.executeMethod(ctx, http.MethodGet, requestData{relPath: adminAPIPrefix + "/storageinfo"})
	defer closeResponse(resp)
	if err != nil {
		return StorageInfo{}, err
	}

	// Check response http status code
	if resp.StatusCode != http.StatusOK {
		return StorageInfo{}, httpRespToErrorResponse(resp)
	}

	// Unmarshal the server's json response
	var storageInfo StorageInfo

	respBytes, err := ioutil.ReadAll(resp.Body)
	if err != nil {
		return StorageInfo{}, err
	}

	err = json.Unmarshal(respBytes, &storageInfo)
	if err != nil {
		return StorageInfo{}, err
	}

	return storageInfo, nil
}

// DataUsageInfo represents data usage of an Object API
type DataUsageInfo struct {
	// LastUpdate is the timestamp of when the data usage info was last updated.
	// This does not indicate a full scan.
	LastUpdate       time.Time `json:"lastUpdate"`
	ObjectsCount     uint64    `json:"objectsCount"`
	ObjectsTotalSize uint64    `json:"objectsTotalSize"`

	// ObjectsSizesHistogram contains information on objects across all buckets.
	// See ObjectsHistogramIntervals.
	ObjectsSizesHistogram map[string]uint64 `json:"objectsSizesHistogram"`

	BucketsCount uint64 `json:"bucketsCount"`

	// BucketsSizes is "bucket name" -> size.
	BucketsSizes map[string]uint64 `json:"bucketsSizes"`
}

// DataUsageInfo - returns data usage of the current object API
func (adm *AdminClient) DataUsageInfo(ctx context.Context) (DataUsageInfo, error) {
	resp, err := adm.executeMethod(ctx, http.MethodGet, requestData{relPath: adminAPIPrefix + "/datausageinfo"})
	defer closeResponse(resp)
	if err != nil {
		return DataUsageInfo{}, err
	}

	// Check response http status code
	if resp.StatusCode != http.StatusOK {
		return DataUsageInfo{}, httpRespToErrorResponse(resp)
	}

	// Unmarshal the server's json response
	var dataUsageInfo DataUsageInfo

	respBytes, err := ioutil.ReadAll(resp.Body)
	if err != nil {
		return DataUsageInfo{}, err
	}

	err = json.Unmarshal(respBytes, &dataUsageInfo)
	if err != nil {
		return DataUsageInfo{}, err
	}

	return dataUsageInfo, nil
}

// InfoMessage container to hold server admin related information.
type InfoMessage struct {
	Mode         string             `json:"mode,omitempty"`
	Domain       []string           `json:"domain,omitempty"`
	Region       string             `json:"region,omitempty"`
	SQSARN       []string           `json:"sqsARN,omitempty"`
	DeploymentID string             `json:"deploymentID,omitempty"`
	Buckets      Buckets            `json:"buckets,omitempty"`
	Objects      Objects            `json:"objects,omitempty"`
	Usage        Usage              `json:"usage,omitempty"`
	Services     Services           `json:"services,omitempty"`
	Backend      interface{}        `json:"backend,omitempty"`
	Servers      []ServerProperties `json:"servers,omitempty"`
}

// Services contains different services information
type Services struct {
	KMS           KMS                           `json:"kms,omitempty"`
	LDAP          LDAP                          `json:"ldap,omitempty"`
	Logger        []Logger                      `json:"logger,omitempty"`
	Audit         []Audit                       `json:"audit,omitempty"`
	Notifications []map[string][]TargetIDStatus `json:"notifications,omitempty"`
}

// Buckets contains the number of buckets
type Buckets struct {
	Count uint64 `json:"count"`
	Error string `json:"error,omitempty"`
}

// Objects contains the number of objects
type Objects struct {
	Count uint64 `json:"count"`
	Error string `json:"error,omitempty"`
}

// Usage contains the total size used
type Usage struct {
	Size  uint64 `json:"size"`
	Error string `json:"error,omitempty"`
}

// KMS contains KMS status information
type KMS struct {
	Status  string `json:"status,omitempty"`
	Encrypt string `json:"encrypt,omitempty"`
	Decrypt string `json:"decrypt,omitempty"`
}

// LDAP contains ldap status
type LDAP struct {
	Status string `json:"status,omitempty"`
}

// Status of endpoint
type Status struct {
	Status string `json:"status,omitempty"`
}

// Audit contains audit logger status
type Audit map[string]Status

// Logger contains logger status
type Logger map[string]Status

// TargetIDStatus containsid and status
type TargetIDStatus map[string]Status

// backendType - indicates the type of backend storage
type backendType string

const (
	// FsType - Backend is FS Type
	FsType = backendType("FS")
	// ErasureType - Backend is Erasure type
	ErasureType = backendType("Erasure")
)

// FSBackend contains specific FS storage information
type FSBackend struct {
	Type backendType `json:"backendType,omitempty"`
}

// ErasureBackend contains specific erasure storage information
type ErasureBackend struct {
	Type         backendType `json:"backendType,omitempty"`
	OnlineDisks  int         `json:"onlineDisks,omitempty"`
	OfflineDisks int         `json:"offlineDisks,omitempty"`
	// Parity disks for currently configured Standard storage class.
	StandardSCParity int `json:"standardSCParity,omitempty"`
	// Parity disks for currently configured Reduced Redundancy storage class.
	RRSCParity int `json:"rrSCParity,omitempty"`
}

// ServerProperties holds server information
type ServerProperties struct {
	State      string            `json:"state,omitempty"`
	Endpoint   string            `json:"endpoint,omitempty"`
	Uptime     int64             `json:"uptime,omitempty"`
	Version    string            `json:"version,omitempty"`
	CommitID   string            `json:"commitID,omitempty"`
	Network    map[string]string `json:"network,omitempty"`
	Disks      []Disk            `json:"drives,omitempty"`
	PoolNumber int               `json:"poolNumber,omitempty"`
}

// Disk holds Disk information
type Disk struct {
	Endpoint        string       `json:"endpoint,omitempty"`
	RootDisk        bool         `json:"rootDisk,omitempty"`
	DrivePath       string       `json:"path,omitempty"`
	Healing         bool         `json:"healing,omitempty"`
	State           string       `json:"state,omitempty"`
	UUID            string       `json:"uuid,omitempty"`
	Model           string       `json:"model,omitempty"`
	TotalSpace      uint64       `json:"totalspace,omitempty"`
	UsedSpace       uint64       `json:"usedspace,omitempty"`
	AvailableSpace  uint64       `json:"availspace,omitempty"`
	ReadThroughput  float64      `json:"readthroughput,omitempty"`
	WriteThroughPut float64      `json:"writethroughput,omitempty"`
	ReadLatency     float64      `json:"readlatency,omitempty"`
	WriteLatency    float64      `json:"writelatency,omitempty"`
	Utilization     float64      `json:"utilization,omitempty"`
	HealInfo        *HealingDisk `json:"heal_info,omitempty"`

	// Indexes, will be -1 until assigned a set.
	PoolIndex int `json:"pool_index"`
	SetIndex  int `json:"set_index"`
	DiskIndex int `json:"disk_index"`
}

// ServerInfo - Connect to a minio server and call Server Admin Info Management API
// to fetch server's information represented by infoMessage structure
func (adm *AdminClient) ServerInfo(ctx context.Context) (InfoMessage, error) {
	resp, err := adm.executeMethod(ctx,
		http.MethodGet,
		requestData{relPath: adminAPIPrefix + "/info"},
	)
	defer closeResponse(resp)
	if err != nil {
		return InfoMessage{}, err
	}

	// Check response http status code
	if resp.StatusCode != http.StatusOK {
		return InfoMessage{}, httpRespToErrorResponse(resp)
	}

	// Unmarshal the server's json response
	var message InfoMessage

	respBytes, err := ioutil.ReadAll(resp.Body)
	if err != nil {
		return InfoMessage{}, err
	}

	err = json.Unmarshal(respBytes, &message)
	if err != nil {
		return InfoMessage{}, err
	}

	return message, nil
}
<|endoftext|>