<commit_before>/*
Copyright 2011 Google Inc.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

     http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package cond

import (
	"bytes"
	"encoding/json"
	"errors"
	"fmt"
	"io"
	"log"
	"net/http"
	"time"

	"camlistore.org/pkg/blobref"
	"camlistore.org/pkg/blobserver"
	"camlistore.org/pkg/jsonconfig"
	"camlistore.org/pkg/schema"
)

const buffered = 8

type storageFunc func(src io.Reader) (dest blobserver.Storage, overRead []byte, err error)

type condStorage struct {
	*blobserver.SimpleBlobHubPartitionMap

	storageForReceive storageFunc
	read              blobserver.Storage
	remove            blobserver.Storage

	ctx *http.Request // optional per-request context
}

var _ blobserver.ContextWrapper = (*condStorage)(nil)

func (sto *condStorage) GetBlobHub() blobserver.BlobHub {
	return sto.SimpleBlobHubPartitionMap.GetBlobHub()
}

func (sto *condStorage) WrapContext(req *http.Request) blobserver.Storage {
	s2 := new(condStorage)
	*s2 = *sto
	s2.ctx = req
	return s2
}

func newFromConfig(ld blobserver.Loader, conf jsonconfig.Obj) (storage blobserver.Storage, err error) {
	sto := &condStorage{
		SimpleBlobHubPartitionMap: &blobserver.SimpleBlobHubPartitionMap{},
	}

	receive := conf.OptionalStringOrObject("write")
	read := conf.RequiredString("read")
	remove := conf.OptionalString("remove", "")
	if err := conf.Validate(); err != nil {
		return nil, err
	}

	if receive != nil {
		sto.storageForReceive, err = buildStorageForReceive(ld, receive)
		if err != nil {
			return
		}
	}

	sto.read, err = ld.GetStorage(read)
	if err != nil {
		return
	}

	if remove != "" {
		sto.remove, err = ld.GetStorage(remove)
		if err != nil {
			return
		}
	}
	return sto, nil
}

func buildStorageForReceive(ld blobserver.Loader, confOrString interface{}) (storageFunc, error) {
	// Static configuration from a string
	if s, ok := confOrString.(string); ok {
		sto, err := ld.GetStorage(s)
		if err != nil {
			return nil, err
		}
		f := func(io.Reader) (blobserver.Storage, []byte, error) {
			return sto, nil, nil
		}
		return f, nil
	}

	conf := jsonconfig.Obj(confOrString.(map[string]interface{}))

	ifStr := conf.RequiredString("if")
	// TODO: let 'then' and 'else' point to not just strings but either
	// a string or a JSON object with another condition, and then
	// call buildStorageForReceive on it recursively
	thenTarget := conf.RequiredString("then")
	elseTarget := conf.RequiredString("else")
	if err := conf.Validate(); err != nil {
		return nil, err
	}
	thenSto, err := ld.GetStorage(thenTarget)
	if err != nil {
		return nil, err
	}
	elseSto, err := ld.GetStorage(elseTarget)
	if err != nil {
		return nil, err
	}

	switch ifStr {
	case "isSchema":
		return isSchemaPicker(thenSto, elseSto), nil
	}
	return nil, fmt.Errorf("cond: unsupported 'if' type of %q", ifStr)
}

func isSchemaPicker(thenSto, elseSto blobserver.Storage) storageFunc {
	return func(src io.Reader) (dest blobserver.Storage, overRead []byte, err error) {
		// TODO: make decision earlier, by parsing JSON as it comes in,
		// not after we have up to 1 MB.
		var buf bytes.Buffer
		_, err = io.CopyN(&buf, src, 1<<20)
		if err != nil && err != io.EOF {
			return
		}
		ss := new(schema.Superset)
		if err = json.NewDecoder(bytes.NewReader(buf.Bytes())).Decode(ss); err != nil {
			log.Printf("cond: json parse failure => not schema => else")
			return elseSto, buf.Bytes(), nil
		}
		if ss.Type == "" {
			log.Printf("cond: json => but not schema => else")
			return elseSto, buf.Bytes(), nil
		}
		log.Printf("cond: json => schema => then")
		return thenSto, buf.Bytes(), nil
	}
}

func (sto *condStorage) ReceiveBlob(b *blobref.BlobRef, source io.Reader) (sb blobref.SizedBlobRef, err error) {
	destSto, overRead, err := sto.storageForReceive(source)
	if err != nil {
		return
	}
	if len(overRead) > 0 {
		source = io.MultiReader(bytes.NewBuffer(overRead), source)
	}
	destSto = blobserver.MaybeWrapContext(destSto, sto.ctx)
	return destSto.ReceiveBlob(b, source)
}

func (sto *condStorage) RemoveBlobs(blobs []*blobref.BlobRef) error {
	if sto.remove != nil {
		rsto := blobserver.MaybeWrapContext(sto.remove, sto.ctx)
		return rsto.RemoveBlobs(blobs)
	}
	return errors.New("cond: Remove not configured")
}

func (sto *condStorage) IsFetcherASeeker() bool {
	_, ok := sto.read.(blobref.SeekFetcher)
	return ok
}

func (sto *condStorage) FetchStreaming(b *blobref.BlobRef) (file io.ReadCloser, size int64, err error) {
	if sto.read != nil {
		rsto := blobserver.MaybeWrapContext(sto.read, sto.ctx)
		return rsto.FetchStreaming(b)
	}
	err = errors.New("cond: Read not configured")
	return
}

func (sto *condStorage) StatBlobs(dest chan<- blobref.SizedBlobRef, blobs []*blobref.BlobRef, wait time.Duration) error {
	if sto.read != nil {
		rsto := blobserver.MaybeWrapContext(sto.read, sto.ctx)
		return rsto.StatBlobs(dest, blobs, wait)
	}
	return errors.New("cond: Read not configured")
}

func (sto *condStorage) EnumerateBlobs(dest chan<- blobref.SizedBlobRef, after string, limit int, wait time.Duration) error {
	if sto.read != nil {
		rsto := blobserver.MaybeWrapContext(sto.read, sto.ctx)
		return rsto.EnumerateBlobs(dest, after, limit, wait)
	}
	return errors.New("cond: Read not configured")
}

func init() {
	blobserver.RegisterStorageConstructor("cond", blobserver.StorageConstructor(newFromConfig))
}
<commit_msg>cond: implement more efficient JSON sniffing.<commit_after>/*
Copyright 2011 Google Inc.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

     http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package cond

import (
	"bytes"
	"encoding/json"
	"errors"
	"fmt"
	"io"
	"net/http"
	"time"

	"camlistore.org/pkg/blobref"
	"camlistore.org/pkg/blobserver"
	"camlistore.org/pkg/jsonconfig"
	"camlistore.org/pkg/schema"
)

const buffered = 8

type storageFunc func(src io.Reader) (dest blobserver.Storage, overRead []byte, err error)

type condStorage struct {
	*blobserver.SimpleBlobHubPartitionMap

	storageForReceive storageFunc
	read              blobserver.Storage
	remove            blobserver.Storage

	ctx *http.Request // optional per-request context
}

var _ blobserver.ContextWrapper = (*condStorage)(nil)

func (sto *condStorage) GetBlobHub() blobserver.BlobHub {
	return sto.SimpleBlobHubPartitionMap.GetBlobHub()
}

func (sto *condStorage) WrapContext(req *http.Request) blobserver.Storage {
	s2 := new(condStorage)
	*s2 = *sto
	s2.ctx = req
	return s2
}

func newFromConfig(ld blobserver.Loader, conf jsonconfig.Obj) (storage blobserver.Storage, err error) {
	sto := &condStorage{
		SimpleBlobHubPartitionMap: &blobserver.SimpleBlobHubPartitionMap{},
	}

	receive := conf.OptionalStringOrObject("write")
	read := conf.RequiredString("read")
	remove := conf.OptionalString("remove", "")
	if err := conf.Validate(); err != nil {
		return nil, err
	}

	if receive != nil {
		sto.storageForReceive, err = buildStorageForReceive(ld, receive)
		if err != nil {
			return
		}
	}

	sto.read, err = ld.GetStorage(read)
	if err != nil {
		return
	}

	if remove != "" {
		sto.remove, err = ld.GetStorage(remove)
		if err != nil {
			return
		}
	}
	return sto, nil
}

func buildStorageForReceive(ld blobserver.Loader, confOrString interface{}) (storageFunc, error) {
	// Static configuration from a string
	if s, ok := confOrString.(string); ok {
		sto, err := ld.GetStorage(s)
		if err != nil {
			return nil, err
		}
		f := func(io.Reader) (blobserver.Storage, []byte, error) {
			return sto, nil, nil
		}
		return f, nil
	}

	conf := jsonconfig.Obj(confOrString.(map[string]interface{}))

	ifStr := conf.RequiredString("if")
	// TODO: let 'then' and 'else' point to not just strings but either
	// a string or a JSON object with another condition, and then
	// call buildStorageForReceive on it recursively
	thenTarget := conf.RequiredString("then")
	elseTarget := conf.RequiredString("else")
	if err := conf.Validate(); err != nil {
		return nil, err
	}
	thenSto, err := ld.GetStorage(thenTarget)
	if err != nil {
		return nil, err
	}
	elseSto, err := ld.GetStorage(elseTarget)
	if err != nil {
		return nil, err
	}

	switch ifStr {
	case "isSchema":
		return isSchemaPicker(thenSto, elseSto), nil
	}
	return nil, fmt.Errorf("cond: unsupported 'if' type of %q", ifStr)
}

func isSchemaPicker(thenSto, elseSto blobserver.Storage) storageFunc {
	return func(src io.Reader) (dest blobserver.Storage, overRead []byte, err error) {
		var buf bytes.Buffer
		var ss schema.Superset
		if err = json.NewDecoder(io.TeeReader(src, &buf)).Decode(&ss); err != nil {
			return elseSto, buf.Bytes(), nil
		}
		if ss.Type == "" {
			// json, but not schema, so use the else path.
			return elseSto, buf.Bytes(), nil
		}
		return thenSto, buf.Bytes(), nil
	}
}

func (sto *condStorage) ReceiveBlob(b *blobref.BlobRef, source io.Reader) (sb blobref.SizedBlobRef, err error) {
	destSto, overRead, err := sto.storageForReceive(source)
	if err != nil {
		return
	}
	if len(overRead) > 0 {
		source = io.MultiReader(bytes.NewBuffer(overRead), source)
	}
	destSto = blobserver.MaybeWrapContext(destSto, sto.ctx)
	return destSto.ReceiveBlob(b, source)
}

func (sto *condStorage) RemoveBlobs(blobs []*blobref.BlobRef) error {
	if sto.remove != nil {
		rsto := blobserver.MaybeWrapContext(sto.remove, sto.ctx)
		return rsto.RemoveBlobs(blobs)
	}
	return errors.New("cond: Remove not configured")
}

func (sto *condStorage) IsFetcherASeeker() bool {
	_, ok := sto.read.(blobref.SeekFetcher)
	return ok
}

func (sto *condStorage) FetchStreaming(b *blobref.BlobRef) (file io.ReadCloser, size int64, err error) {
	if sto.read != nil {
		rsto := blobserver.MaybeWrapContext(sto.read, sto.ctx)
		return rsto.FetchStreaming(b)
	}
	err = errors.New("cond: Read not configured")
	return
}

func (sto *condStorage) StatBlobs(dest chan<- blobref.SizedBlobRef, blobs []*blobref.BlobRef, wait time.Duration) error {
	if sto.read != nil {
		rsto := blobserver.MaybeWrapContext(sto.read, sto.ctx)
		return rsto.StatBlobs(dest, blobs, wait)
	}
	return errors.New("cond: Read not configured")
}

func (sto *condStorage) EnumerateBlobs(dest chan<- blobref.SizedBlobRef, after string, limit int, wait time.Duration) error {
	if sto.read != nil {
		rsto := blobserver.MaybeWrapContext(sto.read, sto.ctx)
		return rsto.EnumerateBlobs(dest, after, limit, wait)
	}
	return errors.New("cond: Read not configured")
}

func init() {
	blobserver.RegisterStorageConstructor("cond", blobserver.StorageConstructor(newFromConfig))
}
<|endoftext|>


<commit_before>package builder

import (
	"fmt"
	"io"
	"io/ioutil"
	"os"
	"os/exec"
	"path/filepath"
	"strings"
	"time"

	docker "github.com/fsouza/go-dockerclient"

	s2igit "github.com/openshift/source-to-image/pkg/scm/git"
	s2iutil "github.com/openshift/source-to-image/pkg/util"

	"github.com/openshift/origin/pkg/build/api"
	"github.com/openshift/origin/pkg/build/builder/cmd/dockercfg"
	"github.com/openshift/origin/pkg/generate/git"
	"github.com/openshift/source-to-image/pkg/tar"
)

const (
	// initialURLCheckTimeout is the initial timeout used to check the
	// source URL.  If fetching the URL exceeds the timeout, then a longer
	// timeout will be tried until the fetch either succeeds or the build
	// itself times out.
	initialURLCheckTimeout = 16 * time.Second

	// timeoutIncrementFactor is the factor to use when increasing
	// the timeout after each unsuccessful try
	timeoutIncrementFactor = 4
)

type gitAuthError string
type gitNotFoundError string
type contextDirNotFoundError string

func (e gitAuthError) Error() string {
	return fmt.Sprintf("failed to fetch requested repository %q with provided credentials", string(e))
}

func (e gitNotFoundError) Error() string {
	return fmt.Sprintf("requested repository %q not found", string(e))
}

func (e contextDirNotFoundError) Error() string {
	return fmt.Sprintf("provided context directory does not exist: %s", string(e))
}

// fetchSource retrieves the inputs defined by the build source into the
// provided directory, or returns an error if retrieval is not possible.
func fetchSource(dockerClient DockerClient, dir string, build *api.Build, urlTimeout time.Duration, in io.Reader, gitClient GitClient) (*git.SourceInfo, error) {
	hasGitSource := false

	// expect to receive input from STDIN
	if err := extractInputBinary(in, build.Spec.Source.Binary, dir); err != nil {
		return nil, err
	}

	// may retrieve source from Git
	hasGitSource, err := extractGitSource(gitClient, build.Spec.Source.Git, build.Spec.Revision, dir, urlTimeout)
	if err != nil {
		return nil, err
	}

	var sourceInfo *git.SourceInfo
	if hasGitSource {
		var errs []error
		sourceInfo, errs = gitClient.GetInfo(dir)
		if len(errs) > 0 {
			for _, e := range errs {
				glog.V(0).Infof("error: Unable to retrieve Git info: %v", e)
			}
		}
	}

	forcePull := false
	switch {
	case build.Spec.Strategy.SourceStrategy != nil:
		forcePull = build.Spec.Strategy.SourceStrategy.ForcePull
	case build.Spec.Strategy.DockerStrategy != nil:
		forcePull = build.Spec.Strategy.DockerStrategy.ForcePull
	case build.Spec.Strategy.CustomStrategy != nil:
		forcePull = build.Spec.Strategy.CustomStrategy.ForcePull
	}
	// extract source from an Image if specified
	for i, image := range build.Spec.Source.Images {
		imageSecretIndex := i
		if image.PullSecret == nil {
			imageSecretIndex = -1
		}
		err := extractSourceFromImage(dockerClient, image.From.Name, dir, imageSecretIndex, image.Paths, forcePull)
		if err != nil {
			return nil, err
		}
	}

	if len(build.Spec.Source.ContextDir) > 0 {
		if _, err := os.Stat(filepath.Join(dir, build.Spec.Source.ContextDir)); os.IsNotExist(err) {
			return sourceInfo, contextDirNotFoundError(build.Spec.Source.ContextDir)
		}
	}

	// a Dockerfile has been specified, create or overwrite into the destination
	if dockerfileSource := build.Spec.Source.Dockerfile; dockerfileSource != nil {
		baseDir := dir
		// if a context dir has been defined and we cloned source, overwrite the destination
		if hasGitSource && len(build.Spec.Source.ContextDir) != 0 {
			baseDir = filepath.Join(baseDir, build.Spec.Source.ContextDir)
		}
		return sourceInfo, ioutil.WriteFile(filepath.Join(baseDir, "Dockerfile"), []byte(*dockerfileSource), 0660)
	}

	return sourceInfo, nil
}

// checkRemoteGit validates the specified Git URL. It returns GitNotFoundError
// when the remote repository not found and GitAuthenticationError when the
// remote repository failed to authenticate.
// Since this is calling the 'git' binary, the proxy settings should be
// available for this command.
func checkRemoteGit(gitClient GitClient, url string, initialTimeout time.Duration) error {

	var (
		out    string
		errOut string
		err    error
	)

	timeout := initialTimeout
	for {
		glog.V(4).Infof("git ls-remote --heads %s", url)
		out, errOut, err = gitClient.TimedListRemote(timeout, url, "--heads")
		if len(out) != 0 {
			glog.V(4).Infof(out)
		}
		if len(errOut) != 0 {
			glog.V(4).Infof(errOut)
		}
		if err != nil {
			if _, ok := err.(*git.TimeoutError); ok {
				timeout = timeout * timeoutIncrementFactor
				glog.Infof("WARNING: timed out waiting for git server, will wait %s", timeout)
				continue
			}
		}
		break
	}
	if err != nil {
		combinedOut := out + errOut
		switch {
		case strings.Contains(combinedOut, "Authentication failed"):
			return gitAuthError(url)
		case strings.Contains(combinedOut, "not found"):
			return gitNotFoundError(url)
		}
	}
	return err
}

// checkSourceURI performs a check on the URI associated with the build
// to make sure that it is valid.
func checkSourceURI(gitClient GitClient, rawurl string, timeout time.Duration) error {
	ok, err := s2igit.New(s2iutil.NewFileSystem()).ValidCloneSpec(rawurl)
	if err != nil {
		return fmt.Errorf("Invalid git source url %q: %v", rawurl, err)
	}
	if !ok {
		return fmt.Errorf("Invalid git source url: %s", rawurl)
	}
	return checkRemoteGit(gitClient, rawurl, timeout)
}

// extractInputBinary processes the provided input stream as directed by BinaryBuildSource
// into dir.
func extractInputBinary(in io.Reader, source *api.BinaryBuildSource, dir string) error {
	if source == nil {
		return nil
	}

	var path string
	if len(source.AsFile) > 0 {
		glog.V(0).Infof("Receiving source from STDIN as file %s", source.AsFile)
		path = filepath.Join(dir, source.AsFile)

		f, err := os.OpenFile(path, os.O_CREATE|os.O_EXCL|os.O_WRONLY, 0664)
		if err != nil {
			return err
		}
		defer f.Close()
		n, err := io.Copy(f, os.Stdin)
		if err != nil {
			return err
		}
		glog.V(4).Infof("Received %d bytes into %s", n, path)
		return nil
	}

	glog.V(0).Infof("Receiving source from STDIN as archive ...")

	cmd := exec.Command("bsdtar", "-x", "-o", "-m", "-f", "-", "-C", dir)
	cmd.Stdin = in
	out, err := cmd.CombinedOutput()
	if err != nil {
		glog.V(0).Infof("Extracting...\n%s", string(out))
		return fmt.Errorf("unable to extract binary build input, must be a zip, tar, or gzipped tar, or specified as a file: %v", err)
	}
	return nil
}

func extractGitSource(gitClient GitClient, gitSource *api.GitBuildSource, revision *api.SourceRevision, dir string, timeout time.Duration) (bool, error) {
	if gitSource == nil {
		return false, nil
	}

	glog.V(0).Infof("Cloning %q ...", gitSource.URI)

	// Check source URI by trying to connect to the server
	if err := checkSourceURI(gitClient, gitSource.URI, timeout); err != nil {
		return true, err
	}

	cloneOptions := []string{}
	usingRevision := revision != nil && revision.Git != nil && len(revision.Git.Commit) != 0
	usingRef := len(gitSource.Ref) != 0 || usingRevision

	// check if we specify a commit, ref, or branch to check out
	// Recursive clone if we're not going to checkout a ref and submodule update later
	if !usingRef {
		cloneOptions = append(cloneOptions, "--recursive")
		cloneOptions = append(cloneOptions, git.Shallow)
	}

	glog.V(3).Infof("Cloning source from %s", gitSource.URI)

	// Only use the quiet flag if Verbosity is not 5 or greater
	if !glog.Is(5) {
		cloneOptions = append(cloneOptions, "--quiet")
	}
	if err := gitClient.CloneWithOptions(dir, gitSource.URI, cloneOptions...); err != nil {
		return true, err
	}

	// if we specify a commit, ref, or branch to checkout, do so, and update submodules
	if usingRef {
		commit := gitSource.Ref

		if usingRevision {
			commit = revision.Git.Commit
		}

		if err := gitClient.Checkout(dir, commit); err != nil {
			return true, err
		}

		// Recursively update --init
		if err := gitClient.SubmoduleUpdate(dir, true, true); err != nil {
			return true, err
		}
	}

	if glog.Is(0) {
		if information, gitErr := gitClient.GetInfo(dir); len(gitErr) == 0 {
			glog.Infof("\tCommit:\t%s (%s)\n", information.CommitID, information.Message)
			glog.Infof("\tAuthor:\t%s <%s>\n", information.AuthorName, information.AuthorEmail)
			glog.Infof("\tDate:\t%s\n", information.Date)
		}
	}

	return true, nil
}

func copyImageSource(dockerClient DockerClient, containerID, sourceDir, destDir string, tarHelper tar.Tar) error {
	// Setup destination directory
	fi, err := os.Stat(destDir)
	if err != nil {
		if !os.IsNotExist(err) {
			return err
		}
		glog.V(4).Infof("Creating image destination directory: %s", destDir)
		err := os.MkdirAll(destDir, 0644)
		if err != nil {
			return err
		}
	} else {
		if !fi.IsDir() {
			return fmt.Errorf("destination %s must be a directory", destDir)
		}
	}

	tempFile, err := ioutil.TempFile("", "imgsrc")
	if err != nil {
		return err
	}
	glog.V(4).Infof("Downloading source from path %s in container %s to temporary archive %s", sourceDir, containerID, tempFile.Name())
	err = dockerClient.DownloadFromContainer(containerID, docker.DownloadFromContainerOptions{
		OutputStream: tempFile,
		Path:         sourceDir,
	})
	if err != nil {
		tempFile.Close()
		return err
	}
	if err := tempFile.Close(); err != nil {
		return err
	}

	// Extract the created tar file to the destination directory
	file, err := os.Open(tempFile.Name())
	if err != nil {
		return err
	}
	defer file.Close()

	glog.V(4).Infof("Extracting temporary tar %s to directory %s", tempFile.Name(), destDir)
	var tarOutput io.Writer
	if glog.Is(4) {
		tarOutput = os.Stdout
	}
	return tarHelper.ExtractTarStreamWithLogging(destDir, file, tarOutput)
}

func extractSourceFromImage(dockerClient DockerClient, image, buildDir string, imageSecretIndex int, paths []api.ImageSourcePath, forcePull bool) error {
	glog.V(4).Infof("Extracting image source from %s", image)

	dockerAuth := docker.AuthConfiguration{}
	if imageSecretIndex != -1 {
		pullSecret := os.Getenv(fmt.Sprintf("%s%d", dockercfg.PullSourceAuthType, imageSecretIndex))
		if len(pullSecret) > 0 {
			authPresent := false
			dockerAuth, authPresent = dockercfg.NewHelper().GetDockerAuth(image, fmt.Sprintf("%s%d", dockercfg.PullSourceAuthType, imageSecretIndex))
			if authPresent {
				glog.V(5).Infof("Registry server Address: %s", dockerAuth.ServerAddress)
				glog.V(5).Infof("Registry server User Name: %s", dockerAuth.Username)
				glog.V(5).Infof("Registry server Email: %s", dockerAuth.Email)
				passwordPresent := "<<empty>>"
				if len(dockerAuth.Password) > 0 {
					passwordPresent = "<<non-empty>>"
				}
				glog.V(5).Infof("Registry server Password: %s", passwordPresent)
			}
		}
	}

	exists := true
	if !forcePull {
		_, err := dockerClient.InspectImage(image)
		if err == docker.ErrNoSuchImage {
			exists = false
		} else if err != nil {
			return err
		}
	}

	if !exists || forcePull {
		glog.V(0).Infof("Pulling image %q ...", image)
		if err := dockerClient.PullImage(docker.PullImageOptions{Repository: image}, dockerAuth); err != nil {
			return fmt.Errorf("error pulling image %v: %v", image, err)
		}
	}

	containerConfig := &docker.Config{Image: image}
	if inspect, err := dockerClient.InspectImage(image); err != nil {
		return err
	} else {
		// In case the Docker image does not specify the entrypoint
		if len(inspect.Config.Entrypoint) == 0 && len(inspect.Config.Cmd) == 0 {
			containerConfig.Entrypoint = []string{"/fake-entrypoint"}
		}
	}

	// Create container to copy from
	container, err := dockerClient.CreateContainer(docker.CreateContainerOptions{Config: containerConfig})
	if err != nil {
		return fmt.Errorf("error creating source image container: %v", err)
	}
	defer dockerClient.RemoveContainer(docker.RemoveContainerOptions{ID: container.ID})

	tarHelper := tar.New(s2iutil.NewFileSystem())
	tarHelper.SetExclusionPattern(nil)

	for _, path := range paths {
		glog.V(4).Infof("Extracting path %s from container %s to %s", path.SourcePath, container.ID, path.DestinationDir)
		err := copyImageSource(dockerClient, container.ID, path.SourcePath, filepath.Join(buildDir, path.DestinationDir), tarHelper)
		if err != nil {
			return fmt.Errorf("error copying source path %s to %s: %v", path.SourcePath, path.DestinationDir, err)
		}
	}

	return nil
}
<commit_msg>Use 0755 in image source nested directory permissions<commit_after>package builder

import (
	"fmt"
	"io"
	"io/ioutil"
	"os"
	"os/exec"
	"path/filepath"
	"strings"
	"time"

	docker "github.com/fsouza/go-dockerclient"

	s2igit "github.com/openshift/source-to-image/pkg/scm/git"
	s2iutil "github.com/openshift/source-to-image/pkg/util"

	"github.com/openshift/origin/pkg/build/api"
	"github.com/openshift/origin/pkg/build/builder/cmd/dockercfg"
	"github.com/openshift/origin/pkg/generate/git"
	"github.com/openshift/source-to-image/pkg/tar"
)

const (
	// initialURLCheckTimeout is the initial timeout used to check the
	// source URL.  If fetching the URL exceeds the timeout, then a longer
	// timeout will be tried until the fetch either succeeds or the build
	// itself times out.
	initialURLCheckTimeout = 16 * time.Second

	// timeoutIncrementFactor is the factor to use when increasing
	// the timeout after each unsuccessful try
	timeoutIncrementFactor = 4
)

type gitAuthError string
type gitNotFoundError string
type contextDirNotFoundError string

func (e gitAuthError) Error() string {
	return fmt.Sprintf("failed to fetch requested repository %q with provided credentials", string(e))
}

func (e gitNotFoundError) Error() string {
	return fmt.Sprintf("requested repository %q not found", string(e))
}

func (e contextDirNotFoundError) Error() string {
	return fmt.Sprintf("provided context directory does not exist: %s", string(e))
}

// fetchSource retrieves the inputs defined by the build source into the
// provided directory, or returns an error if retrieval is not possible.
func fetchSource(dockerClient DockerClient, dir string, build *api.Build, urlTimeout time.Duration, in io.Reader, gitClient GitClient) (*git.SourceInfo, error) {
	hasGitSource := false

	// expect to receive input from STDIN
	if err := extractInputBinary(in, build.Spec.Source.Binary, dir); err != nil {
		return nil, err
	}

	// may retrieve source from Git
	hasGitSource, err := extractGitSource(gitClient, build.Spec.Source.Git, build.Spec.Revision, dir, urlTimeout)
	if err != nil {
		return nil, err
	}

	var sourceInfo *git.SourceInfo
	if hasGitSource {
		var errs []error
		sourceInfo, errs = gitClient.GetInfo(dir)
		if len(errs) > 0 {
			for _, e := range errs {
				glog.V(0).Infof("error: Unable to retrieve Git info: %v", e)
			}
		}
	}

	forcePull := false
	switch {
	case build.Spec.Strategy.SourceStrategy != nil:
		forcePull = build.Spec.Strategy.SourceStrategy.ForcePull
	case build.Spec.Strategy.DockerStrategy != nil:
		forcePull = build.Spec.Strategy.DockerStrategy.ForcePull
	case build.Spec.Strategy.CustomStrategy != nil:
		forcePull = build.Spec.Strategy.CustomStrategy.ForcePull
	}
	// extract source from an Image if specified
	for i, image := range build.Spec.Source.Images {
		imageSecretIndex := i
		if image.PullSecret == nil {
			imageSecretIndex = -1
		}
		err := extractSourceFromImage(dockerClient, image.From.Name, dir, imageSecretIndex, image.Paths, forcePull)
		if err != nil {
			return nil, err
		}
	}

	if len(build.Spec.Source.ContextDir) > 0 {
		if _, err := os.Stat(filepath.Join(dir, build.Spec.Source.ContextDir)); os.IsNotExist(err) {
			return sourceInfo, contextDirNotFoundError(build.Spec.Source.ContextDir)
		}
	}

	// a Dockerfile has been specified, create or overwrite into the destination
	if dockerfileSource := build.Spec.Source.Dockerfile; dockerfileSource != nil {
		baseDir := dir
		// if a context dir has been defined and we cloned source, overwrite the destination
		if hasGitSource && len(build.Spec.Source.ContextDir) != 0 {
			baseDir = filepath.Join(baseDir, build.Spec.Source.ContextDir)
		}
		return sourceInfo, ioutil.WriteFile(filepath.Join(baseDir, "Dockerfile"), []byte(*dockerfileSource), 0660)
	}

	return sourceInfo, nil
}

// checkRemoteGit validates the specified Git URL. It returns GitNotFoundError
// when the remote repository not found and GitAuthenticationError when the
// remote repository failed to authenticate.
// Since this is calling the 'git' binary, the proxy settings should be
// available for this command.
func checkRemoteGit(gitClient GitClient, url string, initialTimeout time.Duration) error {

	var (
		out    string
		errOut string
		err    error
	)

	timeout := initialTimeout
	for {
		glog.V(4).Infof("git ls-remote --heads %s", url)
		out, errOut, err = gitClient.TimedListRemote(timeout, url, "--heads")
		if len(out) != 0 {
			glog.V(4).Infof(out)
		}
		if len(errOut) != 0 {
			glog.V(4).Infof(errOut)
		}
		if err != nil {
			if _, ok := err.(*git.TimeoutError); ok {
				timeout = timeout * timeoutIncrementFactor
				glog.Infof("WARNING: timed out waiting for git server, will wait %s", timeout)
				continue
			}
		}
		break
	}
	if err != nil {
		combinedOut := out + errOut
		switch {
		case strings.Contains(combinedOut, "Authentication failed"):
			return gitAuthError(url)
		case strings.Contains(combinedOut, "not found"):
			return gitNotFoundError(url)
		}
	}
	return err
}

// checkSourceURI performs a check on the URI associated with the build
// to make sure that it is valid.
func checkSourceURI(gitClient GitClient, rawurl string, timeout time.Duration) error {
	ok, err := s2igit.New(s2iutil.NewFileSystem()).ValidCloneSpec(rawurl)
	if err != nil {
		return fmt.Errorf("Invalid git source url %q: %v", rawurl, err)
	}
	if !ok {
		return fmt.Errorf("Invalid git source url: %s", rawurl)
	}
	return checkRemoteGit(gitClient, rawurl, timeout)
}

// extractInputBinary processes the provided input stream as directed by BinaryBuildSource
// into dir.
func extractInputBinary(in io.Reader, source *api.BinaryBuildSource, dir string) error {
	if source == nil {
		return nil
	}

	var path string
	if len(source.AsFile) > 0 {
		glog.V(0).Infof("Receiving source from STDIN as file %s", source.AsFile)
		path = filepath.Join(dir, source.AsFile)

		f, err := os.OpenFile(path, os.O_CREATE|os.O_EXCL|os.O_WRONLY, 0664)
		if err != nil {
			return err
		}
		defer f.Close()
		n, err := io.Copy(f, os.Stdin)
		if err != nil {
			return err
		}
		glog.V(4).Infof("Received %d bytes into %s", n, path)
		return nil
	}

	glog.V(0).Infof("Receiving source from STDIN as archive ...")

	cmd := exec.Command("bsdtar", "-x", "-o", "-m", "-f", "-", "-C", dir)
	cmd.Stdin = in
	out, err := cmd.CombinedOutput()
	if err != nil {
		glog.V(0).Infof("Extracting...\n%s", string(out))
		return fmt.Errorf("unable to extract binary build input, must be a zip, tar, or gzipped tar, or specified as a file: %v", err)
	}
	return nil
}

func extractGitSource(gitClient GitClient, gitSource *api.GitBuildSource, revision *api.SourceRevision, dir string, timeout time.Duration) (bool, error) {
	if gitSource == nil {
		return false, nil
	}

	glog.V(0).Infof("Cloning %q ...", gitSource.URI)

	// Check source URI by trying to connect to the server
	if err := checkSourceURI(gitClient, gitSource.URI, timeout); err != nil {
		return true, err
	}

	cloneOptions := []string{}
	usingRevision := revision != nil && revision.Git != nil && len(revision.Git.Commit) != 0
	usingRef := len(gitSource.Ref) != 0 || usingRevision

	// check if we specify a commit, ref, or branch to check out
	// Recursive clone if we're not going to checkout a ref and submodule update later
	if !usingRef {
		cloneOptions = append(cloneOptions, "--recursive")
		cloneOptions = append(cloneOptions, git.Shallow)
	}

	glog.V(3).Infof("Cloning source from %s", gitSource.URI)

	// Only use the quiet flag if Verbosity is not 5 or greater
	if !glog.Is(5) {
		cloneOptions = append(cloneOptions, "--quiet")
	}
	if err := gitClient.CloneWithOptions(dir, gitSource.URI, cloneOptions...); err != nil {
		return true, err
	}

	// if we specify a commit, ref, or branch to checkout, do so, and update submodules
	if usingRef {
		commit := gitSource.Ref

		if usingRevision {
			commit = revision.Git.Commit
		}

		if err := gitClient.Checkout(dir, commit); err != nil {
			return true, err
		}

		// Recursively update --init
		if err := gitClient.SubmoduleUpdate(dir, true, true); err != nil {
			return true, err
		}
	}

	if glog.Is(0) {
		if information, gitErr := gitClient.GetInfo(dir); len(gitErr) == 0 {
			glog.Infof("\tCommit:\t%s (%s)\n", information.CommitID, information.Message)
			glog.Infof("\tAuthor:\t%s <%s>\n", information.AuthorName, information.AuthorEmail)
			glog.Infof("\tDate:\t%s\n", information.Date)
		}
	}

	return true, nil
}

func copyImageSource(dockerClient DockerClient, containerID, sourceDir, destDir string, tarHelper tar.Tar) error {
	// Setup destination directory
	fi, err := os.Stat(destDir)
	if err != nil {
		if !os.IsNotExist(err) {
			return err
		}
		glog.V(4).Infof("Creating image destination directory: %s", destDir)
		err := os.MkdirAll(destDir, 0755)
		if err != nil {
			return err
		}
	} else {
		if !fi.IsDir() {
			return fmt.Errorf("destination %s must be a directory", destDir)
		}
	}

	tempFile, err := ioutil.TempFile("", "imgsrc")
	if err != nil {
		return err
	}
	glog.V(4).Infof("Downloading source from path %s in container %s to temporary archive %s", sourceDir, containerID, tempFile.Name())
	err = dockerClient.DownloadFromContainer(containerID, docker.DownloadFromContainerOptions{
		OutputStream: tempFile,
		Path:         sourceDir,
	})
	if err != nil {
		tempFile.Close()
		return err
	}
	if err := tempFile.Close(); err != nil {
		return err
	}

	// Extract the created tar file to the destination directory
	file, err := os.Open(tempFile.Name())
	if err != nil {
		return err
	}
	defer file.Close()

	glog.V(4).Infof("Extracting temporary tar %s to directory %s", tempFile.Name(), destDir)
	var tarOutput io.Writer
	if glog.Is(4) {
		tarOutput = os.Stdout
	}
	return tarHelper.ExtractTarStreamWithLogging(destDir, file, tarOutput)
}

func extractSourceFromImage(dockerClient DockerClient, image, buildDir string, imageSecretIndex int, paths []api.ImageSourcePath, forcePull bool) error {
	glog.V(4).Infof("Extracting image source from %s", image)

	dockerAuth := docker.AuthConfiguration{}
	if imageSecretIndex != -1 {
		pullSecret := os.Getenv(fmt.Sprintf("%s%d", dockercfg.PullSourceAuthType, imageSecretIndex))
		if len(pullSecret) > 0 {
			authPresent := false
			dockerAuth, authPresent = dockercfg.NewHelper().GetDockerAuth(image, fmt.Sprintf("%s%d", dockercfg.PullSourceAuthType, imageSecretIndex))
			if authPresent {
				glog.V(5).Infof("Registry server Address: %s", dockerAuth.ServerAddress)
				glog.V(5).Infof("Registry server User Name: %s", dockerAuth.Username)
				glog.V(5).Infof("Registry server Email: %s", dockerAuth.Email)
				passwordPresent := "<<empty>>"
				if len(dockerAuth.Password) > 0 {
					passwordPresent = "<<non-empty>>"
				}
				glog.V(5).Infof("Registry server Password: %s", passwordPresent)
			}
		}
	}

	exists := true
	if !forcePull {
		_, err := dockerClient.InspectImage(image)
		if err == docker.ErrNoSuchImage {
			exists = false
		} else if err != nil {
			return err
		}
	}

	if !exists || forcePull {
		glog.V(0).Infof("Pulling image %q ...", image)
		if err := dockerClient.PullImage(docker.PullImageOptions{Repository: image}, dockerAuth); err != nil {
			return fmt.Errorf("error pulling image %v: %v", image, err)
		}
	}

	containerConfig := &docker.Config{Image: image}
	if inspect, err := dockerClient.InspectImage(image); err != nil {
		return err
	} else {
		// In case the Docker image does not specify the entrypoint
		if len(inspect.Config.Entrypoint) == 0 && len(inspect.Config.Cmd) == 0 {
			containerConfig.Entrypoint = []string{"/fake-entrypoint"}
		}
	}

	// Create container to copy from
	container, err := dockerClient.CreateContainer(docker.CreateContainerOptions{Config: containerConfig})
	if err != nil {
		return fmt.Errorf("error creating source image container: %v", err)
	}
	defer dockerClient.RemoveContainer(docker.RemoveContainerOptions{ID: container.ID})

	tarHelper := tar.New(s2iutil.NewFileSystem())
	tarHelper.SetExclusionPattern(nil)

	for _, path := range paths {
		glog.V(4).Infof("Extracting path %s from container %s to %s", path.SourcePath, container.ID, path.DestinationDir)
		err := copyImageSource(dockerClient, container.ID, path.SourcePath, filepath.Join(buildDir, path.DestinationDir), tarHelper)
		if err != nil {
			return fmt.Errorf("error copying source path %s to %s: %v", path.SourcePath, path.DestinationDir, err)
		}
	}

	return nil
}
<|endoftext|>


<commit_before>package cliconfigmap

import (
	"encoding/base64"
	"encoding/json"
	"fmt"
	"io/ioutil"
	"os"
	"path"
	"strings"

	"github.com/containerum/chkit/pkg/context"
	"github.com/containerum/chkit/pkg/model/configmap"
	"github.com/containerum/chkit/pkg/model/configmap/activeconfigmap"
	"github.com/containerum/chkit/pkg/util/activekit"
	"github.com/containerum/chkit/pkg/util/coblog"
	"github.com/containerum/chkit/pkg/util/namegen"
	"github.com/containerum/kube-client/pkg/model"
	"github.com/sirupsen/logrus"
	"github.com/spf13/cobra"
	flag "github.com/spf13/pflag"
	"gopkg.in/yaml.v2"
)

var aliases = []string{"cm", "confmap", "conf-map", "comap"}

func Create(ctx *context.Context) *cobra.Command {
	comand := &cobra.Command{
		Use:     "configmap",
		Aliases: aliases,
		Run: func(cmd *cobra.Command, args []string) {
			var logger = coblog.Logger(cmd)
			var flags = cmd.Flags()
			var config, err = buildConfigMapFromFlags(flags, logger)
			if err != nil {
				fmt.Println(err)
				os.Exit(1)
			}
			force, _ := flags.GetBool("force")
			if !force {
				config = activeconfigmap.Config{
					EditName:  true,
					ConfigMap: &config,
				}.Wizard()
				fmt.Println(config.RenderTable())
			}
			if force || activekit.YesNo("Are you sure you want to create configmap %s?", config.Name) {
				if err := config.Validate(); err != nil {
					fmt.Println(err)
					os.Exit(1)
				}
				if err := ctx.Client.CreateConfigMap(ctx.Namespace.ID, config); err != nil {
					logger.WithError(err).Errorf("unable to create configmap %q", config.Name)
					fmt.Println(err)
					os.Exit(1)
				}
				fmt.Println("OK")
			} else if !force {
				config = activeconfigmap.Config{
					EditName:  false,
					ConfigMap: &config,
				}.Wizard()
				fmt.Println(config.RenderTable())
			}
		},
	}
	var persistentFlags = comand.PersistentFlags()
	persistentFlags.String("name", namegen.Aster()+"-"+namegen.Physicist(), "configmap name")
	persistentFlags.StringSlice("item-string", nil, "configmap item, KEY:VALUE string pair")
	persistentFlags.StringSlice("item-file", nil, "configmap file, KEY:FILE_PATH or FILE_PATH")
	persistentFlags.String("file", "", "file with configmap data")
	persistentFlags.BoolP("force", "f", false, "suppress confirmation")
	return comand
}

func buildConfigMapFromFlags(flags *flag.FlagSet, logger logrus.FieldLogger) (configmap.ConfigMap, error) {
	var config = configmap.ConfigMap{
		Data: make(model.ConfigMapData, 16),
	}
	if flags.Changed("file") {
		var err error
		var fName, _ = flags.GetString("file")
		data, err := ioutil.ReadFile(fName)
		if err != nil {
			logger.WithError(err).Error("unable to load configmap data from file")
			fmt.Println(err)
			os.Exit(1)
		}
		switch path.Ext(fName) {
		case "json":
			err = json.Unmarshal(data, &config)
		case "yaml":
			err = yaml.Unmarshal(data, &config)
		}
		return config, err
	} else {
		config.Name, _ = flags.GetString("name")
		if flags.Changed("item-string") {
			rawItems, _ := flags.GetStringSlice("item-string")
			items, err := getStringItems(rawItems)
			if err != nil {
				fmt.Println(err)
				os.Exit(1)
			}
			config = config.AddItems(items...)
		}
		if flags.Changed("item-file") {
			rawItems, _ := flags.GetStringSlice("item-file")
			items, err := getFileItems(rawItems)
			if err != nil {
				fmt.Println(err)
				os.Exit(1)
			}
			config = config.AddItems(items...)
		}
		return config, nil
	}
}

func getFileItems(rawItems []string) ([]configmap.Item, error) {
	var items = make([]configmap.Item, 0, len(rawItems))
	for _, rawItem := range rawItems {
		var filepath string
		var key string
		if tokens := strings.SplitN(rawItem, ":", 2); len(tokens) == 2 {
			key = strings.TrimSpace(tokens[0])
			filepath = tokens[1]
		} else if len(tokens) == 1 {
			key = path.Base(tokens[0])
			filepath = tokens[0]
		} else {
			logrus.Panicf("[chkit/pkg/cli/configmap.getFileItems] ivalid token number in raw file item", len(tokens))
		}
		value, err := ioutil.ReadFile(filepath)
		if err != nil {
			return nil, err
		}
		items = append(items, configmap.Item{
			Key:   key,
			Value: base64.StdEncoding.EncodeToString(value),
		})
	}
	return items, nil
}

func getStringItems(rawItems []string) ([]configmap.Item, error) {
	var items = make([]configmap.Item, 0, len(rawItems))
	for _, rawItem := range rawItems {
		var key string
		var value string
		if tokens := strings.SplitN(rawItem, ":", 2); len(tokens) == 2 {
			key = strings.TrimSpace(tokens[0])
			value = strings.TrimSpace(tokens[1])
		} else {
			return nil, fmt.Errorf("invalid token number in raw string item (got %v, required 2)", len(tokens))
		}
		items = append(items, configmap.Item{
			Key:   key,
			Value: base64.StdEncoding.EncodeToString([]byte(value)),
		})
	}
	return items, nil
}
<commit_msg>Fix panic in config map creation from file too<commit_after>package cliconfigmap

import (
	"encoding/base64"
	"encoding/json"
	"fmt"
	"io/ioutil"
	"os"
	"path"
	"strings"

	"github.com/containerum/chkit/pkg/context"
	"github.com/containerum/chkit/pkg/model/configmap"
	"github.com/containerum/chkit/pkg/model/configmap/activeconfigmap"
	"github.com/containerum/chkit/pkg/util/activekit"
	"github.com/containerum/chkit/pkg/util/coblog"
	"github.com/containerum/chkit/pkg/util/namegen"
	"github.com/containerum/kube-client/pkg/model"
	"github.com/sirupsen/logrus"
	"github.com/spf13/cobra"
	flag "github.com/spf13/pflag"
	"gopkg.in/yaml.v2"
)

var aliases = []string{"cm", "confmap", "conf-map", "comap"}

func Create(ctx *context.Context) *cobra.Command {
	comand := &cobra.Command{
		Use:     "configmap",
		Aliases: aliases,
		Run: func(cmd *cobra.Command, args []string) {
			var logger = coblog.Logger(cmd)
			var flags = cmd.Flags()
			var config, err = buildConfigMapFromFlags(flags, logger)
			if err != nil {
				fmt.Println(err)
				os.Exit(1)
			}
			force, _ := flags.GetBool("force")
			if !force {
				config = activeconfigmap.Config{
					EditName:  true,
					ConfigMap: &config,
				}.Wizard()
				fmt.Println(config.RenderTable())
			}
			if force || activekit.YesNo("Are you sure you want to create configmap %s?", config.Name) {
				if err := config.Validate(); err != nil {
					fmt.Println(err)
					os.Exit(1)
				}
				if err := ctx.Client.CreateConfigMap(ctx.Namespace.ID, config); err != nil {
					logger.WithError(err).Errorf("unable to create configmap %q", config.Name)
					fmt.Println(err)
					os.Exit(1)
				}
				fmt.Println("OK")
			} else if !force {
				config = activeconfigmap.Config{
					EditName:  false,
					ConfigMap: &config,
				}.Wizard()
				fmt.Println(config.RenderTable())
			}
		},
	}
	var persistentFlags = comand.PersistentFlags()
	persistentFlags.String("name", namegen.Aster()+"-"+namegen.Physicist(), "configmap name")
	persistentFlags.StringSlice("item-string", nil, "configmap item, KEY:VALUE string pair")
	persistentFlags.StringSlice("item-file", nil, "configmap file, KEY:FILE_PATH or FILE_PATH")
	persistentFlags.String("file", "", "file with configmap data")
	persistentFlags.BoolP("force", "f", false, "suppress confirmation")
	return comand
}

func buildConfigMapFromFlags(flags *flag.FlagSet, logger logrus.FieldLogger) (configmap.ConfigMap, error) {
	var config = configmap.ConfigMap{
		Data: make(model.ConfigMapData, 16),
	}
	if flags.Changed("file") {
		var err error
		var fName, _ = flags.GetString("file")
		data, err := ioutil.ReadFile(fName)
		if err != nil {
			logger.WithError(err).Error("unable to load configmap data from file")
			fmt.Println(err)
			os.Exit(1)
		}
		switch path.Ext(fName) {
		case "json":
			err = json.Unmarshal(data, &config)
		case "yaml":
			err = yaml.Unmarshal(data, &config)
		}
		return config, err
	} else {
		config.Name, _ = flags.GetString("name")
		if flags.Changed("item-string") {
			rawItems, _ := flags.GetStringSlice("item-string")
			items, err := getStringItems(rawItems)
			if err != nil {
				fmt.Println(err)
				os.Exit(1)
			}
			config = config.AddItems(items...)
		}
		if flags.Changed("item-file") {
			rawItems, _ := flags.GetStringSlice("item-file")
			items, err := getFileItems(rawItems)
			if err != nil {
				fmt.Println(err)
				os.Exit(1)
			}
			config = config.AddItems(items...)
		}
		return config, nil
	}
}

func getFileItems(rawItems []string) ([]configmap.Item, error) {
	var items = make([]configmap.Item, 0, len(rawItems))
	for _, rawItem := range rawItems {
		var filepath string
		var key string
		if tokens := strings.SplitN(rawItem, ":", 2); len(tokens) == 2 {
			key = strings.TrimSpace(tokens[0])
			filepath = tokens[1]
		} else if len(tokens) == 1 {
			key = path.Base(tokens[0])
			filepath = tokens[0]
		} else {
			return nil, fmt.Errorf("invalid token number in raw file item (got %v, required 2)", len(tokens))
		}
		value, err := ioutil.ReadFile(filepath)
		if err != nil {
			return nil, err
		}
		items = append(items, configmap.Item{
			Key:   key,
			Value: base64.StdEncoding.EncodeToString(value),
		})
	}
	return items, nil
}

func getStringItems(rawItems []string) ([]configmap.Item, error) {
	var items = make([]configmap.Item, 0, len(rawItems))
	for _, rawItem := range rawItems {
		var key string
		var value string
		if tokens := strings.SplitN(rawItem, ":", 2); len(tokens) == 2 {
			key = strings.TrimSpace(tokens[0])
			value = strings.TrimSpace(tokens[1])
		} else {
			return nil, fmt.Errorf("invalid token number in raw string item (got %v, required 2)", len(tokens))
		}
		items = append(items, configmap.Item{
			Key:   key,
			Value: base64.StdEncoding.EncodeToString([]byte(value)),
		})
	}
	return items, nil
}
<|endoftext|>


<commit_before>package config

import (
	"io/ioutil"

	log "github.com/sirupsen/logrus"
)

var initContainerTemplate = `
- name: init-pytorch
  image: alpine:3.10
  imagePullPolicy: IfNotPresent
  command: ['sh', '-c', 'until nslookup {{.MasterAddr}}; do echo waiting for master; sleep 2; done;']`

func init() {
	bytes, err := ioutil.ReadFile("/etc/config/initContainer.yaml")
	if err != nil {
		log.Info("Using default init container template")
	} else {
		log.Info("Using init container template from /etc/config/initContainer.yaml")
		initContainerTemplate = string(bytes)
	}
}

func GetInitContainerTemplate() string {
	return initContainerTemplate
}
<commit_msg>fix: Add resource limits for init container (#253)<commit_after>package config

import (
	"io/ioutil"

	log "github.com/sirupsen/logrus"
)

var initContainerTemplate = `
- name: init-pytorch
  image: alpine:3.10
  imagePullPolicy: IfNotPresent
  resources:
    limits:
      cpu: 100m
      memory: 10Mi
    requests:
      cpu: 10m
      memory: 1Mi
  command: ['sh', '-c', 'until nslookup {{.MasterAddr}}; do echo waiting for master; sleep 2; done;']`

func init() {
	bytes, err := ioutil.ReadFile("/etc/config/initContainer.yaml")
	if err != nil {
		log.Info("Using default init container template")
	} else {
		log.Info("Using init container template from /etc/config/initContainer.yaml")
		initContainerTemplate = string(bytes)
	}
}

func GetInitContainerTemplate() string {
	return initContainerTemplate
}
<|endoftext|>


<commit_before>// Copyright (c) 2014-2017 Bitmark Inc.
// Use of this source code is governed by an ISC
// license that can be found in the LICENSE file.

package payment

import (
	"encoding/hex"
	"encoding/json"
	"github.com/bitmark-inc/bitmarkd/constants"
	"github.com/bitmark-inc/bitmarkd/currency"
	"github.com/bitmark-inc/bitmarkd/currency/satoshi"
	"github.com/bitmark-inc/bitmarkd/pay"
	"github.com/bitmark-inc/bitmarkd/reservoir"
	"github.com/bitmark-inc/bitmarkd/util"
	"github.com/bitmark-inc/logger"
	"net/http"
	"sync"
	"time"
)

const (
	bitcoinOPReturnHexCode      = "6a30" // op code with 48 byte parameter
	bitcoinOPReturnPrefixLength = len(bitcoinOPReturnHexCode)
	bitcoinOPReturnPayIDOffset  = bitcoinOPReturnPrefixLength
	bitcoinOPReturnRecordLength = bitcoinOPReturnPrefixLength + 2*48
)

type bitcoinScriptPubKey struct {
	Hex       string   `json:"hex"`
	Addresses []string `json:"addresses"`
}

type bitcoinVout struct {
	Value        json.RawMessage     `json:"value"`
	ScriptPubKey bitcoinScriptPubKey `json:"scriptPubKey"`
}

type bitcoinTransaction struct {
	TxId string        `json:"txid"`
	Vout []bitcoinVout `json:"vout"`
}

type bitcoinBlock struct {
	Hash              string               `json:"hash"`
	Confirmations     uint64               `json:"confirmations"`
	Height            uint64               `json:"height"`
	Tx                []bitcoinTransaction `json:"tx"`
	Time              int64                `json:"time"`
	PreviousBlockHash string               `json:"previousblockhash"`
	NextBlockHash     string               `json:"nextblockhash"`
}

type bitcoinBlockHeader struct {
	Hash              string `json:"hash"`
	Confirmations     uint64 `json:"confirmations"`
	Height            uint64 `json:"height"`
	Time              int64  `json:"time"`
	PreviousBlockHash string `json:"previousblockhash"`
	NextBlockHash     string `json:"nextblockhash"`
}

type bitcoinChainInfo struct {
	Blocks uint64 `json:"blocks"`
	Hash   string `json:"bestblockhash"`
}

// bitcoinHandler implements the currencyHandler interface for Bitcoin
type bitcoinHandler struct {
	log   *logger.L
	state *bitcoinState
}

func newBitcoinHandler(useDiscovery bool, conf *currencyConfiguration) (*bitcoinHandler, error) {
	log := logger.New("bitcoin")

	if useDiscovery {
		return &bitcoinHandler{log: log}, nil
	}

	state, err := newBitcoinState(conf.URL)
	if err != nil {
		return nil, err
	}
	return &bitcoinHandler{log, state}, nil
}

func (h *bitcoinHandler) processPastTxs(dat []byte) {
	txs := make([]bitcoinTransaction, 0)
	if err := json.Unmarshal(dat, &txs); err != nil {
		h.log.Errorf("unable to unmarshal txs: %v", err)
		return
	}

	for _, tx := range txs {
		h.log.Debugf("old possible payment tx received: %s\n", tx.TxId)
		inspectBitcoinTx(h.log, &tx)
	}
}

func (h *bitcoinHandler) processIncomingTx(dat []byte) {
	var tx bitcoinTransaction
	if err := json.Unmarshal(dat, &tx); err != nil {
		h.log.Errorf("unable to unmarshal tx: %v", err)
		return
	}

	h.log.Infof("new possible payment tx received: %s\n", tx.TxId)
	inspectBitcoinTx(h.log, &tx)
}

func (h *bitcoinHandler) checkLatestBlock(wg *sync.WaitGroup) {
	defer wg.Done()

	var headers []bitcoinBlockHeader
	if err := util.FetchJSON(h.state.client, h.state.url+"/headers/1/"+h.state.latestBlockHash+".json", &headers); err != nil {
		h.log.Errorf("headers: error: %s", err)
		return
	}

	if len(headers) < 1 {
		return
	}

	h.log.Infof("block number: %d confirmations: %d", headers[0].Height, headers[0].Confirmations)

	if h.state.forward && headers[0].Confirmations <= requiredConfirmations {
		return
	}

	h.state.process(h.log)
}

// bitcoinState maintains the block state and extracts possible payment txs from bitcoin blocks
type bitcoinState struct {
	// connection to bitcoind
	client *http.Client
	url    string

	// latest block info
	latestBlockNumber uint64
	latestBlockHash   string

	// scanning direction
	forward bool
}

func newBitcoinState(url string) (*bitcoinState, error) {
	client := &http.Client{}

	var chain bitcoinChainInfo
	if err := util.FetchJSON(client, url+"/chaininfo.json", &chain); err != nil {
		return nil, err
	}

	return &bitcoinState{
		client:            client,
		url:               url,
		latestBlockNumber: chain.Blocks,
		latestBlockHash:   chain.Hash,
		forward:           false,
	}, nil
}

func (state *bitcoinState) process(log *logger.L) {
	counter := 0                                                 // number of blocks processed
	startTime := time.Now()                                      // used to calculate the elapsed time of the process
	traceStopTime := time.Now().Add(-constants.ReservoirTimeout) // reverse scan stops when the block is older than traceStopTime

	hash := state.latestBlockHash

process_blocks:
	for {
		var block bitcoinBlock
		if err := util.FetchJSON(state.client, state.url+"/block/"+hash+".json", &block); err != nil {
			log.Errorf("failed to get the block by hash: %s", hash)
			return
		}
		log.Infof("height: %d hash: %q number of txs: %d", block.Height, block.Hash, len(block.Tx))
		log.Tracef("block: %#v", block)

		if block.Confirmations <= requiredConfirmations {
			if !state.forward {
				hash = block.PreviousBlockHash
				state.latestBlockHash = hash
				continue process_blocks
			}
			state.latestBlockHash = hash
			break process_blocks
		}

		// extract possible payment txs from the block
		transactionCount := len(block.Tx) // ignore the first tx (coinbase tx)
		if transactionCount > 1 {
			for _, tx := range block.Tx[1:] {
				inspectBitcoinTx(log, &tx)
			}
		}

		// throttle the sync speed
		counter++
		if counter > 10 {
			timeTaken := time.Since(startTime)
			rate := float64(counter) / timeTaken.Seconds()
			if rate > maximumBlockRate {
				log.Infof("the current rate %f exceeds the limit %f", rate, maximumBlockRate)
				time.Sleep(2 * time.Second)
			}
		}

		// move to the next block
		if state.forward {
			hash = block.NextBlockHash
		} else {
			blockTime := time.Unix(block.Time, 0)
			if blockTime.Before(traceStopTime) {
				state.forward = true
				break process_blocks
			}
			hash = block.PreviousBlockHash
		}
	}
}

func inspectBitcoinTx(log *logger.L, tx *bitcoinTransaction) {
	_, err := hex.DecodeString(tx.TxId)
	if err != nil {
		log.Errorf("invalid tx id: %s", tx.TxId)
		return
	}

	var payId pay.PayId
	amounts := make(map[string]uint64)
	found := false

scan_vouts:
	for _, vout := range tx.Vout {
		if len(vout.ScriptPubKey.Hex) == bitcoinOPReturnRecordLength && vout.ScriptPubKey.Hex[0:4] == bitcoinOPReturnHexCode {
			pid := vout.ScriptPubKey.Hex[bitcoinOPReturnPayIDOffset:]
			if err := payId.UnmarshalText([]byte(pid)); err != nil {
				log.Errorf("invalid pay id: %s", pid)
				return
			}

			found = true
			continue scan_vouts
		}

		if len(vout.ScriptPubKey.Addresses) == 1 {
			amounts[vout.ScriptPubKey.Addresses[0]] += satoshi.FromByteString(vout.Value)
		}
	}

	if !found {
		return
	}

	if len(amounts) == 0 {
		log.Warnf("found pay id but no payments in tx id: %s", tx.TxId)
		return
	}

	reservoir.SetTransferVerified(
		payId,
		&reservoir.PaymentDetail{
			Currency: currency.Bitcoin,
			TxID:     tx.TxId,
			Amounts:  amounts,
		},
	)
}
<commit_msg>[payment] reduce log messages<commit_after>// Copyright (c) 2014-2017 Bitmark Inc.
// Use of this source code is governed by an ISC
// license that can be found in the LICENSE file.

package payment

import (
	"encoding/hex"
	"encoding/json"
	"github.com/bitmark-inc/bitmarkd/constants"
	"github.com/bitmark-inc/bitmarkd/currency"
	"github.com/bitmark-inc/bitmarkd/currency/satoshi"
	"github.com/bitmark-inc/bitmarkd/pay"
	"github.com/bitmark-inc/bitmarkd/reservoir"
	"github.com/bitmark-inc/bitmarkd/util"
	"github.com/bitmark-inc/logger"
	"net/http"
	"sync"
	"time"
)

const (
	bitcoinOPReturnHexCode      = "6a30" // op code with 48 byte parameter
	bitcoinOPReturnPrefixLength = len(bitcoinOPReturnHexCode)
	bitcoinOPReturnPayIDOffset  = bitcoinOPReturnPrefixLength
	bitcoinOPReturnRecordLength = bitcoinOPReturnPrefixLength + 2*48
)

type bitcoinScriptPubKey struct {
	Hex       string   `json:"hex"`
	Addresses []string `json:"addresses"`
}

type bitcoinVout struct {
	Value        json.RawMessage     `json:"value"`
	ScriptPubKey bitcoinScriptPubKey `json:"scriptPubKey"`
}

type bitcoinTransaction struct {
	TxId string        `json:"txid"`
	Vout []bitcoinVout `json:"vout"`
}

type bitcoinBlock struct {
	Hash              string               `json:"hash"`
	Confirmations     uint64               `json:"confirmations"`
	Height            uint64               `json:"height"`
	Tx                []bitcoinTransaction `json:"tx"`
	Time              int64                `json:"time"`
	PreviousBlockHash string               `json:"previousblockhash"`
	NextBlockHash     string               `json:"nextblockhash"`
}

type bitcoinBlockHeader struct {
	Hash              string `json:"hash"`
	Confirmations     uint64 `json:"confirmations"`
	Height            uint64 `json:"height"`
	Time              int64  `json:"time"`
	PreviousBlockHash string `json:"previousblockhash"`
	NextBlockHash     string `json:"nextblockhash"`
}

type bitcoinChainInfo struct {
	Blocks uint64 `json:"blocks"`
	Hash   string `json:"bestblockhash"`
}

// bitcoinHandler implements the currencyHandler interface for Bitcoin
type bitcoinHandler struct {
	log   *logger.L
	state *bitcoinState
}

func newBitcoinHandler(useDiscovery bool, conf *currencyConfiguration) (*bitcoinHandler, error) {
	log := logger.New("bitcoin")

	if useDiscovery {
		return &bitcoinHandler{log: log}, nil
	}

	state, err := newBitcoinState(conf.URL)
	if err != nil {
		return nil, err
	}
	return &bitcoinHandler{log, state}, nil
}

func (h *bitcoinHandler) processPastTxs(dat []byte) {
	txs := make([]bitcoinTransaction, 0)
	if err := json.Unmarshal(dat, &txs); err != nil {
		h.log.Errorf("unable to unmarshal txs: %v", err)
		return
	}

	for _, tx := range txs {
		h.log.Debugf("old possible payment tx received: %s\n", tx.TxId)
		inspectBitcoinTx(h.log, &tx)
	}
}

func (h *bitcoinHandler) processIncomingTx(dat []byte) {
	var tx bitcoinTransaction
	if err := json.Unmarshal(dat, &tx); err != nil {
		h.log.Errorf("unable to unmarshal tx: %v", err)
		return
	}

	h.log.Debugf("new possible payment tx received: %s\n", tx.TxId)
	inspectBitcoinTx(h.log, &tx)
}

func (h *bitcoinHandler) checkLatestBlock(wg *sync.WaitGroup) {
	defer wg.Done()

	var headers []bitcoinBlockHeader
	if err := util.FetchJSON(h.state.client, h.state.url+"/headers/1/"+h.state.latestBlockHash+".json", &headers); err != nil {
		h.log.Errorf("headers: error: %s", err)
		return
	}

	if len(headers) < 1 {
		return
	}

	h.log.Infof("block number: %d confirmations: %d", headers[0].Height, headers[0].Confirmations)

	if h.state.forward && headers[0].Confirmations <= requiredConfirmations {
		return
	}

	h.state.process(h.log)
}

// bitcoinState maintains the block state and extracts possible payment txs from bitcoin blocks
type bitcoinState struct {
	// connection to bitcoind
	client *http.Client
	url    string

	// latest block info
	latestBlockNumber uint64
	latestBlockHash   string

	// scanning direction
	forward bool
}

func newBitcoinState(url string) (*bitcoinState, error) {
	client := &http.Client{}

	var chain bitcoinChainInfo
	if err := util.FetchJSON(client, url+"/chaininfo.json", &chain); err != nil {
		return nil, err
	}

	return &bitcoinState{
		client:            client,
		url:               url,
		latestBlockNumber: chain.Blocks,
		latestBlockHash:   chain.Hash,
		forward:           false,
	}, nil
}

func (state *bitcoinState) process(log *logger.L) {
	counter := 0                                                 // number of blocks processed
	startTime := time.Now()                                      // used to calculate the elapsed time of the process
	traceStopTime := time.Now().Add(-constants.ReservoirTimeout) // reverse scan stops when the block is older than traceStopTime

	hash := state.latestBlockHash

process_blocks:
	for {
		var block bitcoinBlock
		if err := util.FetchJSON(state.client, state.url+"/block/"+hash+".json", &block); err != nil {
			log.Errorf("failed to get the block by hash: %s", hash)
			return
		}
		log.Infof("height: %d hash: %q number of txs: %d", block.Height, block.Hash, len(block.Tx))
		log.Tracef("block: %#v", block)

		if block.Confirmations <= requiredConfirmations {
			if !state.forward {
				hash = block.PreviousBlockHash
				state.latestBlockHash = hash
				continue process_blocks
			}
			state.latestBlockHash = hash
			break process_blocks
		}

		// extract possible payment txs from the block
		transactionCount := len(block.Tx) // ignore the first tx (coinbase tx)
		if transactionCount > 1 {
			for _, tx := range block.Tx[1:] {
				inspectBitcoinTx(log, &tx)
			}
		}

		// throttle the sync speed
		counter++
		if counter > 10 {
			timeTaken := time.Since(startTime)
			rate := float64(counter) / timeTaken.Seconds()
			if rate > maximumBlockRate {
				log.Infof("the current rate %f exceeds the limit %f", rate, maximumBlockRate)
				time.Sleep(2 * time.Second)
			}
		}

		// move to the next block
		if state.forward {
			hash = block.NextBlockHash
		} else {
			blockTime := time.Unix(block.Time, 0)
			if blockTime.Before(traceStopTime) {
				state.forward = true
				break process_blocks
			}
			hash = block.PreviousBlockHash
		}
	}
}

func inspectBitcoinTx(log *logger.L, tx *bitcoinTransaction) {
	_, err := hex.DecodeString(tx.TxId)
	if err != nil {
		log.Errorf("invalid tx id: %s", tx.TxId)
		return
	}

	var payId pay.PayId
	amounts := make(map[string]uint64)
	found := false

scan_vouts:
	for _, vout := range tx.Vout {
		if len(vout.ScriptPubKey.Hex) == bitcoinOPReturnRecordLength && vout.ScriptPubKey.Hex[0:4] == bitcoinOPReturnHexCode {
			pid := vout.ScriptPubKey.Hex[bitcoinOPReturnPayIDOffset:]
			if err := payId.UnmarshalText([]byte(pid)); err != nil {
				log.Errorf("invalid pay id: %s", pid)
				return
			}

			found = true
			continue scan_vouts
		}

		if len(vout.ScriptPubKey.Addresses) == 1 {
			amounts[vout.ScriptPubKey.Addresses[0]] += satoshi.FromByteString(vout.Value)
		}
	}

	if !found {
		return
	}

	if len(amounts) == 0 {
		log.Warnf("found pay id but no payments in tx id: %s", tx.TxId)
		return
	}

	reservoir.SetTransferVerified(
		payId,
		&reservoir.PaymentDetail{
			Currency: currency.Bitcoin,
			TxID:     tx.TxId,
			Amounts:  amounts,
		},
	)
}
<|endoftext|>


<commit_before>// Copyright 2017 Authors of Cilium
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package client

import (
	"fmt"
	"io"
	"net"
	"net/http"
	"net/url"
	"os"
	"sort"
	"strings"
	"time"

	clientapi "github.com/cilium/cilium/api/v1/health/client"
	"github.com/cilium/cilium/api/v1/health/models"
	"github.com/cilium/cilium/pkg/health/defaults"

	runtime_client "github.com/go-openapi/runtime/client"
	"github.com/go-openapi/strfmt"
)

// Client is a client for cilium health
type Client struct {
	clientapi.CiliumHealth
}

func configureTransport(tr *http.Transport, proto, addr string) *http.Transport {
	if tr == nil {
		tr = &http.Transport{}
	}

	if proto == "unix" {
		// No need for compression in local communications.
		tr.DisableCompression = true
		tr.Dial = func(_, _ string) (net.Conn, error) {
			return net.Dial(proto, addr)
		}
	} else {
		tr.Proxy = http.ProxyFromEnvironment
		tr.Dial = (&net.Dialer{}).Dial
	}

	return tr
}

// NewDefaultClient creates a client with default parameters connecting to UNIX domain socket.
func NewDefaultClient() (*Client, error) {
	return NewClient("")
}

// NewClient creates a client for the given `host`.
func NewClient(host string) (*Client, error) {
	if host == "" {
		// Check if environment variable points to socket
		e := os.Getenv(defaults.SockPathEnv)
		if e == "" {
			// If unset, fall back to default value
			e = defaults.SockPath
		}
		host = "unix://" + e
	}
	tmp := strings.SplitN(host, "://", 2)
	if len(tmp) != 2 {
		return nil, fmt.Errorf("invalid host format '%s'", host)
	}

	switch tmp[0] {
	case "tcp":
		if _, err := url.Parse("tcp://" + tmp[1]); err != nil {
			return nil, err
		}
		host = "http://" + tmp[1]
	case "unix":
		host = tmp[1]
	}

	transport := configureTransport(nil, tmp[0], host)
	httpClient := &http.Client{Transport: transport}
	clientTrans := runtime_client.NewWithClient(tmp[1], clientapi.DefaultBasePath,
		clientapi.DefaultSchemes, httpClient)
	return &Client{*clientapi.New(clientTrans, strfmt.Default)}, nil
}

// Hint tries to improve the error message displayed to the user.
func Hint(err error) error {
	if err == nil {
		return err
	}
	e, _ := url.PathUnescape(err.Error())
	if strings.Contains(err.Error(), defaults.SockPath) {
		return fmt.Errorf("%s\nIs the agent running?", e)
	}
	return fmt.Errorf("%s", e)
}

func connectivityStatusHealthy(cs *models.ConnectivityStatus) bool {
	return cs != nil && cs.Status == ""
}

func formatConnectivityStatus(w io.Writer, cs *models.ConnectivityStatus, path, indent string) {
	status := cs.Status
	if connectivityStatusHealthy(cs) {
		latency := time.Duration(cs.Latency)
		status = fmt.Sprintf("OK, RTT=%s", latency)
	}
	fmt.Fprintf(w, "%s%s:\t%s\n", indent, path, status)
}

func formatPathStatus(w io.Writer, name string, cp *models.PathStatus, indent string, verbose bool) {
	if cp == nil {
		if verbose {
			fmt.Fprintf(w, "%s%s connectivity:\tnil\n", indent, name)
		}
		return
	}
	fmt.Fprintf(w, "%s%s connectivity to %s:\n", indent, name, cp.IP)
	indent = fmt.Sprintf("%s  ", indent)

	statuses := map[string]*models.ConnectivityStatus{
		"ICMP":        cp.Icmp,
		"HTTP via L3": cp.HTTP,
	}
	for name, status := range statuses {
		if status != nil {
			formatConnectivityStatus(w, status, name, indent)
		}
	}
}

func pathIsHealthy(cp *models.PathStatus) bool {
	if cp == nil {
		return false
	}

	statuses := []*models.ConnectivityStatus{
		cp.Icmp,
		cp.HTTP,
	}
	for _, status := range statuses {
		if !connectivityStatusHealthy(status) {
			return false
		}
	}
	return true
}

func nodeIsHealthy(node *models.NodeStatus) bool {
	return pathIsHealthy(node.Host.PrimaryAddress) &&
		(node.Endpoint == nil || pathIsHealthy(node.Endpoint))
}

func nodeIsLocalhost(node *models.NodeStatus, self *models.SelfStatus) bool {
	return self != nil && node.Name == self.Name
}

func formatNodeStatus(w io.Writer, node *models.NodeStatus, printAll, verbose, succinct, localhost bool) {
	localStr := ""
	if localhost {
		localStr = " (localhost)"
	}
	if succinct {
		if printAll || !nodeIsHealthy(node) {
			fmt.Fprintf(w, "  %s%s\t%s\t%t\t%t\n", node.Name,
				localStr, node.Host.PrimaryAddress.IP,
				pathIsHealthy(node.Host.PrimaryAddress),
				pathIsHealthy(node.Endpoint))
		}
	} else {
		fmt.Fprintf(w, "  %s%s:\n", node.Name, localStr)
		formatPathStatus(w, "Host", node.Host.PrimaryAddress, "    ", verbose)
		if verbose && len(node.Host.SecondaryAddresses) > 0 {
			for _, addr := range node.Host.SecondaryAddresses {
				formatPathStatus(w, "Secondary", addr, "      ", verbose)
			}
		}
		formatPathStatus(w, "Endpoint", node.Endpoint, "    ", verbose)
	}
}

// FormatHealthStatusResponse writes a HealthStatusResponse as a string to the
// writer.
//
// 'printAll', if true, causes all nodes to be printed regardless of status
// 'succinct', if true, causes node health to be output as one line per node
// 'verbose', if true, overrides 'succinct' and prints all information
// 'maxLines', if nonzero, determines the maximum number of lines to print
func FormatHealthStatusResponse(w io.Writer, sr *models.HealthStatusResponse, printAll, succinct, verbose bool, maxLines int) {
	var (
		healthy   int
		localhost *models.NodeStatus
	)
	for _, node := range sr.Nodes {
		if nodeIsHealthy(node) {
			healthy++
		}
		if nodeIsLocalhost(node, sr.Local) {
			localhost = node
		}
	}
	if succinct {
		fmt.Fprintf(w, "Cluster health:\t%d/%d reachable\t(%s)\n",
			healthy, len(sr.Nodes), sr.Timestamp)
		if printAll || healthy < len(sr.Nodes) {
			fmt.Fprintf(w, "  Name\tIP\tReachable\tEndpoints reachable\n")
		}
	} else {
		fmt.Fprintf(w, "Probe time:\t%s\n", sr.Timestamp)
		fmt.Fprintf(w, "Nodes:\n")
	}

	if localhost != nil {
		formatNodeStatus(w, localhost, printAll, succinct, verbose, true)
		maxLines--
	}

	nodes := sr.Nodes
	sort.Slice(nodes, func(i, j int) bool {
		return strings.Compare(nodes[i].Name, nodes[j].Name) < 0
	})
	for n, node := range nodes {
		if maxLines > 0 && n > maxLines {
			break
		}
		if node == localhost {
			continue
		}
		formatNodeStatus(w, node, printAll, succinct, verbose, false)
	}
	if maxLines > 0 && len(sr.Nodes)-healthy > maxLines {
		fmt.Fprintf(w, "  ...")
	}
}

// GetAndFormatHealthStatus fetches the health status from the cilium-health
// daemon via the default channel and formats its output as a string to the
// writer.
//
// 'succinct', 'verbose' and 'maxLines' are handled the same as in
// FormatHealthStatusResponse().
func GetAndFormatHealthStatus(w io.Writer, succinct, verbose bool, maxLines int) {
	client, err := NewClient("")
	if err != nil {
		fmt.Fprintf(w, "Cluster health:\t\t\tClient error: %s\n", err)
		return
	}
	hr, err := client.Connectivity.GetStatus(nil)
	if err != nil {
		// The regular `cilium status` output will print the reason why.
		fmt.Fprintf(w, "Cluster health:\t\t\tWarning\tcilium-health daemon unreachable\n")
		return
	}
	FormatHealthStatusResponse(w, hr.Payload, verbose, succinct, verbose, maxLines)
}
<commit_msg>health: Print ICMP then HTTP results<commit_after>// Copyright 2017 Authors of Cilium
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package client

import (
	"fmt"
	"io"
	"net"
	"net/http"
	"net/url"
	"os"
	"sort"
	"strings"
	"time"

	clientapi "github.com/cilium/cilium/api/v1/health/client"
	"github.com/cilium/cilium/api/v1/health/models"
	"github.com/cilium/cilium/pkg/health/defaults"

	runtime_client "github.com/go-openapi/runtime/client"
	"github.com/go-openapi/strfmt"
)

// Client is a client for cilium health
type Client struct {
	clientapi.CiliumHealth
}

func configureTransport(tr *http.Transport, proto, addr string) *http.Transport {
	if tr == nil {
		tr = &http.Transport{}
	}

	if proto == "unix" {
		// No need for compression in local communications.
		tr.DisableCompression = true
		tr.Dial = func(_, _ string) (net.Conn, error) {
			return net.Dial(proto, addr)
		}
	} else {
		tr.Proxy = http.ProxyFromEnvironment
		tr.Dial = (&net.Dialer{}).Dial
	}

	return tr
}

// NewDefaultClient creates a client with default parameters connecting to UNIX domain socket.
func NewDefaultClient() (*Client, error) {
	return NewClient("")
}

// NewClient creates a client for the given `host`.
func NewClient(host string) (*Client, error) {
	if host == "" {
		// Check if environment variable points to socket
		e := os.Getenv(defaults.SockPathEnv)
		if e == "" {
			// If unset, fall back to default value
			e = defaults.SockPath
		}
		host = "unix://" + e
	}
	tmp := strings.SplitN(host, "://", 2)
	if len(tmp) != 2 {
		return nil, fmt.Errorf("invalid host format '%s'", host)
	}

	switch tmp[0] {
	case "tcp":
		if _, err := url.Parse("tcp://" + tmp[1]); err != nil {
			return nil, err
		}
		host = "http://" + tmp[1]
	case "unix":
		host = tmp[1]
	}

	transport := configureTransport(nil, tmp[0], host)
	httpClient := &http.Client{Transport: transport}
	clientTrans := runtime_client.NewWithClient(tmp[1], clientapi.DefaultBasePath,
		clientapi.DefaultSchemes, httpClient)
	return &Client{*clientapi.New(clientTrans, strfmt.Default)}, nil
}

// Hint tries to improve the error message displayed to the user.
func Hint(err error) error {
	if err == nil {
		return err
	}
	e, _ := url.PathUnescape(err.Error())
	if strings.Contains(err.Error(), defaults.SockPath) {
		return fmt.Errorf("%s\nIs the agent running?", e)
	}
	return fmt.Errorf("%s", e)
}

func connectivityStatusHealthy(cs *models.ConnectivityStatus) bool {
	return cs != nil && cs.Status == ""
}

func formatConnectivityStatus(w io.Writer, cs *models.ConnectivityStatus, path, indent string) {
	status := cs.Status
	if connectivityStatusHealthy(cs) {
		latency := time.Duration(cs.Latency)
		status = fmt.Sprintf("OK, RTT=%s", latency)
	}
	fmt.Fprintf(w, "%s%s:\t%s\n", indent, path, status)
}

func formatPathStatus(w io.Writer, name string, cp *models.PathStatus, indent string, verbose bool) {
	if cp == nil {
		if verbose {
			fmt.Fprintf(w, "%s%s connectivity:\tnil\n", indent, name)
		}
		return
	}
	fmt.Fprintf(w, "%s%s connectivity to %s:\n", indent, name, cp.IP)
	indent = fmt.Sprintf("%s  ", indent)

	if cp.Icmp != nil {
		formatConnectivityStatus(w, cp.Icmp, "ICMP", indent)
	}
	if cp.HTTP != nil {
		formatConnectivityStatus(w, cp.HTTP, "HTTP via L3", indent)
	}
}

func pathIsHealthy(cp *models.PathStatus) bool {
	if cp == nil {
		return false
	}

	statuses := []*models.ConnectivityStatus{
		cp.Icmp,
		cp.HTTP,
	}
	for _, status := range statuses {
		if !connectivityStatusHealthy(status) {
			return false
		}
	}
	return true
}

func nodeIsHealthy(node *models.NodeStatus) bool {
	return pathIsHealthy(node.Host.PrimaryAddress) &&
		(node.Endpoint == nil || pathIsHealthy(node.Endpoint))
}

func nodeIsLocalhost(node *models.NodeStatus, self *models.SelfStatus) bool {
	return self != nil && node.Name == self.Name
}

func formatNodeStatus(w io.Writer, node *models.NodeStatus, printAll, verbose, succinct, localhost bool) {
	localStr := ""
	if localhost {
		localStr = " (localhost)"
	}
	if succinct {
		if printAll || !nodeIsHealthy(node) {
			fmt.Fprintf(w, "  %s%s\t%s\t%t\t%t\n", node.Name,
				localStr, node.Host.PrimaryAddress.IP,
				pathIsHealthy(node.Host.PrimaryAddress),
				pathIsHealthy(node.Endpoint))
		}
	} else {
		fmt.Fprintf(w, "  %s%s:\n", node.Name, localStr)
		formatPathStatus(w, "Host", node.Host.PrimaryAddress, "    ", verbose)
		if verbose && len(node.Host.SecondaryAddresses) > 0 {
			for _, addr := range node.Host.SecondaryAddresses {
				formatPathStatus(w, "Secondary", addr, "      ", verbose)
			}
		}
		formatPathStatus(w, "Endpoint", node.Endpoint, "    ", verbose)
	}
}

// FormatHealthStatusResponse writes a HealthStatusResponse as a string to the
// writer.
//
// 'printAll', if true, causes all nodes to be printed regardless of status
// 'succinct', if true, causes node health to be output as one line per node
// 'verbose', if true, overrides 'succinct' and prints all information
// 'maxLines', if nonzero, determines the maximum number of lines to print
func FormatHealthStatusResponse(w io.Writer, sr *models.HealthStatusResponse, printAll, succinct, verbose bool, maxLines int) {
	var (
		healthy   int
		localhost *models.NodeStatus
	)
	for _, node := range sr.Nodes {
		if nodeIsHealthy(node) {
			healthy++
		}
		if nodeIsLocalhost(node, sr.Local) {
			localhost = node
		}
	}
	if succinct {
		fmt.Fprintf(w, "Cluster health:\t%d/%d reachable\t(%s)\n",
			healthy, len(sr.Nodes), sr.Timestamp)
		if printAll || healthy < len(sr.Nodes) {
			fmt.Fprintf(w, "  Name\tIP\tReachable\tEndpoints reachable\n")
		}
	} else {
		fmt.Fprintf(w, "Probe time:\t%s\n", sr.Timestamp)
		fmt.Fprintf(w, "Nodes:\n")
	}

	if localhost != nil {
		formatNodeStatus(w, localhost, printAll, succinct, verbose, true)
		maxLines--
	}

	nodes := sr.Nodes
	sort.Slice(nodes, func(i, j int) bool {
		return strings.Compare(nodes[i].Name, nodes[j].Name) < 0
	})
	for n, node := range nodes {
		if maxLines > 0 && n > maxLines {
			break
		}
		if node == localhost {
			continue
		}
		formatNodeStatus(w, node, printAll, succinct, verbose, false)
	}
	if maxLines > 0 && len(sr.Nodes)-healthy > maxLines {
		fmt.Fprintf(w, "  ...")
	}
}

// GetAndFormatHealthStatus fetches the health status from the cilium-health
// daemon via the default channel and formats its output as a string to the
// writer.
//
// 'succinct', 'verbose' and 'maxLines' are handled the same as in
// FormatHealthStatusResponse().
func GetAndFormatHealthStatus(w io.Writer, succinct, verbose bool, maxLines int) {
	client, err := NewClient("")
	if err != nil {
		fmt.Fprintf(w, "Cluster health:\t\t\tClient error: %s\n", err)
		return
	}
	hr, err := client.Connectivity.GetStatus(nil)
	if err != nil {
		// The regular `cilium status` output will print the reason why.
		fmt.Fprintf(w, "Cluster health:\t\t\tWarning\tcilium-health daemon unreachable\n")
		return
	}
	FormatHealthStatusResponse(w, hr.Payload, verbose, succinct, verbose, maxLines)
}
<|endoftext|>


<commit_before>package main

import (
	"fmt"
	"os"
	"strconv"
	"strings"
)

type Commit struct {
	num    int    // mercurial revision number
	node   string // mercurial hash
	parent string // hash of commit's parent
	user   string // author's Name <email>
	date   string // date of commit
	desc   string // description
}

// getCommit returns details about the Commit specified by the revision hash
func getCommit(rev string) (c Commit, err os.Error) {
	defer func() {
		if err != nil {
			err = fmt.Errorf("getCommit: %s: %s", rev, err)
		}
	}()
	parts, err := getCommitParts(rev)
	if err != nil {
		return
	}
	num, err := strconv.Atoi(parts[0])
	if err != nil {
		return
	}
	parent := ""
	if num > 0 {
		prev := strconv.Itoa(num - 1)
		if pparts, err := getCommitParts(prev); err == nil {
			parent = pparts[1]
		}
	}
	user := strings.Replace(parts[2], "<", "<", -1)
	user = strings.Replace(user, ">", ">", -1)
	return Commit{num, parts[1], parent, user, parts[3], parts[4]}, nil
}

func getCommitParts(rev string) (parts []string, err os.Error) {
	const format = "{rev}>{node}>{author|escape}>{date}>{desc}"
	s, _, err := runLog(nil, "", goroot,
		"hg", "log", "-r", rev, "-l", "1", "--template", format)
	if err != nil {
		return
	}
	return strings.Split(s, ">", 5), nil
}
<commit_msg>misc/dashboard/builder: talk to hg with utf-8 encoding always.<commit_after>package main

import (
	"fmt"
	"os"
	"strconv"
	"strings"
)

type Commit struct {
	num    int    // mercurial revision number
	node   string // mercurial hash
	parent string // hash of commit's parent
	user   string // author's Name <email>
	date   string // date of commit
	desc   string // description
}

// getCommit returns details about the Commit specified by the revision hash
func getCommit(rev string) (c Commit, err os.Error) {
	defer func() {
		if err != nil {
			err = fmt.Errorf("getCommit: %s: %s", rev, err)
		}
	}()
	parts, err := getCommitParts(rev)
	if err != nil {
		return
	}
	num, err := strconv.Atoi(parts[0])
	if err != nil {
		return
	}
	parent := ""
	if num > 0 {
		prev := strconv.Itoa(num - 1)
		if pparts, err := getCommitParts(prev); err == nil {
			parent = pparts[1]
		}
	}
	user := strings.Replace(parts[2], "<", "<", -1)
	user = strings.Replace(user, ">", ">", -1)
	return Commit{num, parts[1], parent, user, parts[3], parts[4]}, nil
}

func getCommitParts(rev string) (parts []string, err os.Error) {
	const format = "{rev}>{node}>{author|escape}>{date}>{desc}"
	s, _, err := runLog(nil, "", goroot,
		"hg", "log",
		"--encoding", "utf-8",
		"--rev", rev,
		"--limit", "1",
		"--template", format,
	)
	if err != nil {
		return
	}
	return strings.Split(s, ">", 5), nil
}
<|endoftext|>


<commit_before>package mpb

import (
	"io"

	"github.com/acarl005/stripansi"
	"github.com/mattn/go-runewidth"
	"github.com/vbauerster/mpb/v7/decor"
	"github.com/vbauerster/mpb/v7/internal"
)

const (
	iLbound = iota
	iRbound
	iFiller
	iRefiller
	iPadding
	components
)

// BarStyleComposer interface.
type BarStyleComposer interface {
	BarFillerBuilder
	Lbound(string) BarStyleComposer
	Rbound(string) BarStyleComposer
	Filler(string) BarStyleComposer
	Refiller(string) BarStyleComposer
	Padding(string) BarStyleComposer
	TipOnComplete(string) BarStyleComposer
	Tip(frames ...string) BarStyleComposer
	Reverse() BarStyleComposer
}

type bFiller struct {
	components [components]*component
	tip        struct {
		count      uint
		onComplete *component
		frames     []*component
	}
	flush func(dst io.Writer, filling, padding [][]byte)
}

type component struct {
	width int
	bytes []byte
}

type barStyle struct {
	lbound        string
	rbound        string
	filler        string
	refiller      string
	padding       string
	tipOnComplete string
	tipFrames     []string
	rev           bool
}

// BarStyle constructs default bar style which can be altered via
// BarStyleComposer interface.
func BarStyle() BarStyleComposer {
	return &barStyle{
		lbound:    "[",
		rbound:    "]",
		filler:    "=",
		refiller:  "+",
		padding:   "-",
		tipFrames: []string{">"},
	}
}

func (s *barStyle) Lbound(bound string) BarStyleComposer {
	s.lbound = bound
	return s
}

func (s *barStyle) Rbound(bound string) BarStyleComposer {
	s.rbound = bound
	return s
}

func (s *barStyle) Filler(filler string) BarStyleComposer {
	s.filler = filler
	return s
}

func (s *barStyle) Refiller(refiller string) BarStyleComposer {
	s.refiller = refiller
	return s
}

func (s *barStyle) Padding(padding string) BarStyleComposer {
	s.padding = padding
	return s
}

func (s *barStyle) TipOnComplete(tip string) BarStyleComposer {
	s.tipOnComplete = tip
	return s
}

func (s *barStyle) Tip(frames ...string) BarStyleComposer {
	if len(frames) != 0 {
		s.tipFrames = append(s.tipFrames[:0], frames...)
	}
	return s
}

func (s *barStyle) Reverse() BarStyleComposer {
	s.rev = true
	return s
}

func (s *barStyle) Build() BarFiller {
	bf := new(bFiller)
	if s.rev {
		bf.flush = func(dst io.Writer, filling, padding [][]byte) {
			flush(dst, padding, filling)
		}
	} else {
		bf.flush = flush
	}
	bf.components[iLbound] = &component{
		width: runewidth.StringWidth(stripansi.Strip(s.lbound)),
		bytes: []byte(s.lbound),
	}
	bf.components[iRbound] = &component{
		width: runewidth.StringWidth(stripansi.Strip(s.rbound)),
		bytes: []byte(s.rbound),
	}
	bf.components[iFiller] = &component{
		width: runewidth.StringWidth(stripansi.Strip(s.filler)),
		bytes: []byte(s.filler),
	}
	bf.components[iRefiller] = &component{
		width: runewidth.StringWidth(stripansi.Strip(s.refiller)),
		bytes: []byte(s.refiller),
	}
	bf.components[iPadding] = &component{
		width: runewidth.StringWidth(stripansi.Strip(s.padding)),
		bytes: []byte(s.padding),
	}
	bf.tip.onComplete = &component{
		width: runewidth.StringWidth(stripansi.Strip(s.tipOnComplete)),
		bytes: []byte(s.tipOnComplete),
	}
	bf.tip.frames = make([]*component, len(s.tipFrames))
	for i, t := range s.tipFrames {
		bf.tip.frames[i] = &component{
			width: runewidth.StringWidth(stripansi.Strip(t)),
			bytes: []byte(t),
		}
	}
	return bf
}

func (s *bFiller) Fill(w io.Writer, width int, stat decor.Statistics) {
	width = internal.CheckRequestedWidth(width, stat.AvailableWidth)
	brackets := s.components[iLbound].width + s.components[iRbound].width
	// don't count brackets as progress
	width -= brackets
	if width < 0 {
		return
	}

	w.Write(s.components[iLbound].bytes)
	defer w.Write(s.components[iRbound].bytes)

	if width == 0 {
		return
	}

	var filling [][]byte
	var padding [][]byte
	var tip *component
	var filled int
	curWidth := int(internal.PercentageRound(stat.Total, stat.Current, uint(width)))

	if stat.Current >= stat.Total {
		tip = s.tip.onComplete
	} else {
		tip = s.tip.frames[s.tip.count%uint(len(s.tip.frames))]
	}

	if curWidth > 0 {
		filling = append(filling, tip.bytes)
		filled += tip.width
		s.tip.count++
	}

	for filled < curWidth && curWidth-filled >= s.components[iFiller].width {
		filling = append(filling, s.components[iFiller].bytes)
		if s.components[iFiller].width == 0 {
			break
		}
		filled += s.components[iFiller].width
	}

	if stat.Refill > 0 {
		refWidth := int(internal.PercentageRound(stat.Total, stat.Refill, uint(width)))
		available := curWidth
		bound := available - refWidth
		for i := len(filling) - 1; available > bound && i > 0; i-- {
			filling[i] = s.components[iRefiller].bytes
			available -= s.components[iRefiller].width
		}
	}

	padWidth := width - filled
	for padWidth > 0 && padWidth >= s.components[iPadding].width {
		padding = append(padding, s.components[iPadding].bytes)
		if s.components[iPadding].width == 0 {
			break
		}
		padWidth -= s.components[iPadding].width
	}

	for padWidth > 0 {
		padding = append(padding, []byte("…"))
		padWidth--
	}

	s.flush(w, filling, padding)
}

func flush(dst io.Writer, filling, padding [][]byte) {
	for i := len(filling) - 1; i >= 0; i-- {
		dst.Write(filling[i])
	}
	for i := 0; i < len(padding); i++ {
		dst.Write(padding[i])
	}
}
<commit_msg>refill refactoring<commit_after>package mpb

import (
	"io"

	"github.com/acarl005/stripansi"
	"github.com/mattn/go-runewidth"
	"github.com/vbauerster/mpb/v7/decor"
	"github.com/vbauerster/mpb/v7/internal"
)

const (
	iLbound = iota
	iRbound
	iFiller
	iRefiller
	iPadding
	components
)

// BarStyleComposer interface.
type BarStyleComposer interface {
	BarFillerBuilder
	Lbound(string) BarStyleComposer
	Rbound(string) BarStyleComposer
	Filler(string) BarStyleComposer
	Refiller(string) BarStyleComposer
	Padding(string) BarStyleComposer
	TipOnComplete(string) BarStyleComposer
	Tip(frames ...string) BarStyleComposer
	Reverse() BarStyleComposer
}

type bFiller struct {
	components [components]*component
	tip        struct {
		count      uint
		onComplete *component
		frames     []*component
	}
	flush func(dst io.Writer, filling, padding [][]byte)
}

type component struct {
	width int
	bytes []byte
}

type barStyle struct {
	lbound        string
	rbound        string
	filler        string
	refiller      string
	padding       string
	tipOnComplete string
	tipFrames     []string
	rev           bool
}

// BarStyle constructs default bar style which can be altered via
// BarStyleComposer interface.
func BarStyle() BarStyleComposer {
	return &barStyle{
		lbound:    "[",
		rbound:    "]",
		filler:    "=",
		refiller:  "+",
		padding:   "-",
		tipFrames: []string{">"},
	}
}

func (s *barStyle) Lbound(bound string) BarStyleComposer {
	s.lbound = bound
	return s
}

func (s *barStyle) Rbound(bound string) BarStyleComposer {
	s.rbound = bound
	return s
}

func (s *barStyle) Filler(filler string) BarStyleComposer {
	s.filler = filler
	return s
}

func (s *barStyle) Refiller(refiller string) BarStyleComposer {
	s.refiller = refiller
	return s
}

func (s *barStyle) Padding(padding string) BarStyleComposer {
	s.padding = padding
	return s
}

func (s *barStyle) TipOnComplete(tip string) BarStyleComposer {
	s.tipOnComplete = tip
	return s
}

func (s *barStyle) Tip(frames ...string) BarStyleComposer {
	if len(frames) != 0 {
		s.tipFrames = append(s.tipFrames[:0], frames...)
	}
	return s
}

func (s *barStyle) Reverse() BarStyleComposer {
	s.rev = true
	return s
}

func (s *barStyle) Build() BarFiller {
	bf := new(bFiller)
	if s.rev {
		bf.flush = func(dst io.Writer, filling, padding [][]byte) {
			flush(dst, padding, filling)
		}
	} else {
		bf.flush = flush
	}
	bf.components[iLbound] = &component{
		width: runewidth.StringWidth(stripansi.Strip(s.lbound)),
		bytes: []byte(s.lbound),
	}
	bf.components[iRbound] = &component{
		width: runewidth.StringWidth(stripansi.Strip(s.rbound)),
		bytes: []byte(s.rbound),
	}
	bf.components[iFiller] = &component{
		width: runewidth.StringWidth(stripansi.Strip(s.filler)),
		bytes: []byte(s.filler),
	}
	bf.components[iRefiller] = &component{
		width: runewidth.StringWidth(stripansi.Strip(s.refiller)),
		bytes: []byte(s.refiller),
	}
	bf.components[iPadding] = &component{
		width: runewidth.StringWidth(stripansi.Strip(s.padding)),
		bytes: []byte(s.padding),
	}
	bf.tip.onComplete = &component{
		width: runewidth.StringWidth(stripansi.Strip(s.tipOnComplete)),
		bytes: []byte(s.tipOnComplete),
	}
	bf.tip.frames = make([]*component, len(s.tipFrames))
	for i, t := range s.tipFrames {
		bf.tip.frames[i] = &component{
			width: runewidth.StringWidth(stripansi.Strip(t)),
			bytes: []byte(t),
		}
	}
	return bf
}

func (s *bFiller) Fill(w io.Writer, width int, stat decor.Statistics) {
	width = internal.CheckRequestedWidth(width, stat.AvailableWidth)
	brackets := s.components[iLbound].width + s.components[iRbound].width
	// don't count brackets as progress
	width -= brackets
	if width < 0 {
		return
	}

	w.Write(s.components[iLbound].bytes)
	defer w.Write(s.components[iRbound].bytes)

	if width == 0 {
		return
	}

	var filling [][]byte
	var padding [][]byte
	var tip *component
	var filled int
	curWidth := int(internal.PercentageRound(stat.Total, stat.Current, uint(width)))

	if stat.Current >= stat.Total {
		tip = s.tip.onComplete
	} else {
		tip = s.tip.frames[s.tip.count%uint(len(s.tip.frames))]
	}

	if curWidth > 0 {
		filling = append(filling, tip.bytes)
		filled += tip.width
		s.tip.count++
	}

	for filled < curWidth && curWidth-filled >= s.components[iFiller].width {
		filling = append(filling, s.components[iFiller].bytes)
		if s.components[iFiller].width == 0 {
			break
		}
		filled += s.components[iFiller].width
	}

	if stat.Refill > 0 {
		refWidth := int(internal.PercentageRound(stat.Total, stat.Refill, uint(width)))
		if refWidth == curWidth {
			refWidth -= tip.width
		}
		for i := len(filling) - 1; i >= 0; i-- {
			if refWidth < s.components[iRefiller].width {
				break
			}
			filling[i] = s.components[iRefiller].bytes
			refWidth -= s.components[iRefiller].width
		}
	}

	padWidth := width - filled
	for padWidth > 0 && padWidth >= s.components[iPadding].width {
		padding = append(padding, s.components[iPadding].bytes)
		if s.components[iPadding].width == 0 {
			break
		}
		padWidth -= s.components[iPadding].width
	}

	for padWidth > 0 {
		padding = append(padding, []byte("…"))
		padWidth--
	}

	s.flush(w, filling, padding)
}

func flush(dst io.Writer, filling, padding [][]byte) {
	for i := len(filling) - 1; i >= 0; i-- {
		dst.Write(filling[i])
	}
	for i := 0; i < len(padding); i++ {
		dst.Write(padding[i])
	}
}
<|endoftext|>


<commit_before>package people

import (
	"encoding/json"
	"net/http"

	"fmt"
	fthealth "github.com/Financial-Times/go-fthealth/v1_1"
	"github.com/Financial-Times/service-status-go/gtg"
	"github.com/Financial-Times/transactionid-utils-go"
	"github.com/gorilla/mux"
	log "github.com/sirupsen/logrus"
	"regexp"
	"strings"
)

const (
	urlPrefix = "http://api.ft.com/things/"
	validUUID = "([0-9a-fA-F]{8}-[0-9a-fA-F]{4}-[0-9a-fA-F]{4}-[0-9a-fA-F]{4}-[0-9a-fA-F]{12})$"
)

// PeopleDriver for cypher queries
var PeopleDriver Driver
var CacheControlHeader string

//var maxAge = 24 * time.Hour

// HealthCheck does something
func HealthCheck() fthealth.Check {
	return fthealth.Check{
		BusinessImpact: "Unable to respond to Public People API requests",
		Name:           "Public People Heathchecks",
		PanicGuide:     "https://dewey.in.ft.com/view/system/public-people-api",
		Severity:       2,
		TechnicalSummary: `Cannot connect to Neo4j. If this check fails, check that the Neo4J cluster is responding.  `,
		Checker: Checker,
	}
}

// Checker does more stuff
func Checker() (string, error) {
	err := PeopleDriver.CheckConnectivity()
	if err == nil {
		return "Connectivity to neo4j is ok", err
	}
	return "Error connecting to neo4j", err
}

func GTG() gtg.Status {
	statusCheck := func() gtg.Status {
		return gtgCheck(Checker)
	}

	return gtg.FailFastParallelCheck([]gtg.StatusChecker{statusCheck})()
}

func gtgCheck(handler func() (string, error)) gtg.Status {
	if _, err := handler(); err != nil {
		return gtg.Status{GoodToGo: false, Message: err.Error()}
	}
	return gtg.Status{GoodToGo: true}
}

// MethodNotAllowedHandler handles 405
func MethodNotAllowedHandler(w http.ResponseWriter, r *http.Request) {
	w.WriteHeader(http.StatusMethodNotAllowed)
	return
}

// GetPerson is the public API
func GetPerson(w http.ResponseWriter, r *http.Request) {
	vars := mux.Vars(r)
	requestId := vars["uuid"]
	transId := transactionidutils.GetTransactionIDFromRequest(r)
	w.Header().Set("X-Request-Id", transId)
	w.Header().Set("Content-Type", "application/json; charset=UTF-8")

	validRegexp := regexp.MustCompile(validUUID)
	if requestId == "" || !validRegexp.MatchString(requestId) {
		msg := fmt.Sprintf("Invalid request id %s", requestId)
		log.WithFields(log.Fields{"UUID": requestId, "transaction_id": transId}).Error(msg)
		w.WriteHeader(http.StatusBadRequest)
		w.Write([]byte(`"{\"message\":\"` + msg + `\"}"`))
		return
	}

	person, found, err := PeopleDriver.Read(requestId, transId)
	if err != nil {
		w.WriteHeader(http.StatusInternalServerError)
		w.Write([]byte(`"{\"message\": \"Person could not be retrieved\"}"`))
		return
	}
	if !found {
		w.WriteHeader(http.StatusNotFound)
		w.Write([]byte(`"{\"message\":\"Person ` + requestId + ` not found in DB\"}"`))
		return
	}

	canonicalId := strings.TrimPrefix(person.ID, urlPrefix)
	if strings.Compare(canonicalId, requestId) != 0 {
		log.WithFields(log.Fields{"UUID": requestId}).Info("Person " + requestId + " is concorded to " + canonicalId + "; serving redirect")
		redirectURL := strings.Replace(r.URL.String(), requestId, canonicalId, 1)
		w.Header().Set("Location", redirectURL)
		w.WriteHeader(http.StatusMovedPermanently)
		w.Write([]byte(`"{\"message\":\"Person ` + requestId + ` is concorded, redirecting...\"}"`))
		return
	}

	w.Header().Set("Cache-Control", CacheControlHeader)
	w.WriteHeader(http.StatusOK)

	if err = json.NewEncoder(w).Encode(person); err != nil {
		w.WriteHeader(http.StatusInternalServerError)
		w.Write([]byte(`"{\"message\":\"Person could not be retrieved\"}"`))
	}
}
<commit_msg>Updated the healthcheck information- name updated part 2<commit_after>package people

import (
	"encoding/json"
	"net/http"

	"fmt"
	fthealth "github.com/Financial-Times/go-fthealth/v1_1"
	"github.com/Financial-Times/service-status-go/gtg"
	"github.com/Financial-Times/transactionid-utils-go"
	"github.com/gorilla/mux"
	log "github.com/sirupsen/logrus"
	"regexp"
	"strings"
)

const (
	urlPrefix = "http://api.ft.com/things/"
	validUUID = "([0-9a-fA-F]{8}-[0-9a-fA-F]{4}-[0-9a-fA-F]{4}-[0-9a-fA-F]{4}-[0-9a-fA-F]{12})$"
)

// PeopleDriver for cypher queries
var PeopleDriver Driver
var CacheControlHeader string

//var maxAge = 24 * time.Hour

// HealthCheck does something
func HealthCheck() fthealth.Check {
	return fthealth.Check{
		BusinessImpact: "Unable to respond to Public People API requests",
		Name:           "Neo4j Connectivity",
		PanicGuide:     "https://dewey.in.ft.com/view/system/public-people-api",
		Severity:       2,
		TechnicalSummary: `Cannot connect to Neo4j. If this check fails, check that the Neo4J cluster is responding.  `,
		Checker: Checker,
	}
}

// Checker does more stuff
func Checker() (string, error) {
	err := PeopleDriver.CheckConnectivity()
	if err == nil {
		return "Connectivity to neo4j is ok", err
	}
	return "Error connecting to neo4j", err
}

func GTG() gtg.Status {
	statusCheck := func() gtg.Status {
		return gtgCheck(Checker)
	}

	return gtg.FailFastParallelCheck([]gtg.StatusChecker{statusCheck})()
}

func gtgCheck(handler func() (string, error)) gtg.Status {
	if _, err := handler(); err != nil {
		return gtg.Status{GoodToGo: false, Message: err.Error()}
	}
	return gtg.Status{GoodToGo: true}
}

// MethodNotAllowedHandler handles 405
func MethodNotAllowedHandler(w http.ResponseWriter, r *http.Request) {
	w.WriteHeader(http.StatusMethodNotAllowed)
	return
}

// GetPerson is the public API
func GetPerson(w http.ResponseWriter, r *http.Request) {
	vars := mux.Vars(r)
	requestId := vars["uuid"]
	transId := transactionidutils.GetTransactionIDFromRequest(r)
	w.Header().Set("X-Request-Id", transId)
	w.Header().Set("Content-Type", "application/json; charset=UTF-8")

	validRegexp := regexp.MustCompile(validUUID)
	if requestId == "" || !validRegexp.MatchString(requestId) {
		msg := fmt.Sprintf("Invalid request id %s", requestId)
		log.WithFields(log.Fields{"UUID": requestId, "transaction_id": transId}).Error(msg)
		w.WriteHeader(http.StatusBadRequest)
		w.Write([]byte(`"{\"message\":\"` + msg + `\"}"`))
		return
	}

	person, found, err := PeopleDriver.Read(requestId, transId)
	if err != nil {
		w.WriteHeader(http.StatusInternalServerError)
		w.Write([]byte(`"{\"message\": \"Person could not be retrieved\"}"`))
		return
	}
	if !found {
		w.WriteHeader(http.StatusNotFound)
		w.Write([]byte(`"{\"message\":\"Person ` + requestId + ` not found in DB\"}"`))
		return
	}

	canonicalId := strings.TrimPrefix(person.ID, urlPrefix)
	if strings.Compare(canonicalId, requestId) != 0 {
		log.WithFields(log.Fields{"UUID": requestId}).Info("Person " + requestId + " is concorded to " + canonicalId + "; serving redirect")
		redirectURL := strings.Replace(r.URL.String(), requestId, canonicalId, 1)
		w.Header().Set("Location", redirectURL)
		w.WriteHeader(http.StatusMovedPermanently)
		w.Write([]byte(`"{\"message\":\"Person ` + requestId + ` is concorded, redirecting...\"}"`))
		return
	}

	w.Header().Set("Cache-Control", CacheControlHeader)
	w.WriteHeader(http.StatusOK)

	if err = json.NewEncoder(w).Encode(person); err != nil {
		w.WriteHeader(http.StatusInternalServerError)
		w.Write([]byte(`"{\"message\":\"Person could not be retrieved\"}"`))
	}
}
<|endoftext|>