<commit_before>/*
Copyright 2016 Google Inc. All Rights Reserved.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
    http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

// The goopack binary creates a GooGet package using the provided GooSpec file.
package main

import (
	"archive/tar"
	"compress/gzip"
	"flag"
	"fmt"
	"io"
	"io/ioutil"
	"log"
	"os"
	"path/filepath"
	"regexp"
	"runtime"
	"strings"

	"github.com/google/googet/goolib"
	"github.com/google/googet/oswrap"
)

var outputDir = flag.String("output_dir", "", "where to put the built package")

type fileMap map[string][]string

// walkDir returns a list of all files in directory and subdirectories, it is similar
// to filepath.Walk but works even if dir is a symlink, which is the case with blaze Filesets.
func walkDir(dir string) ([]string, error) {
	rl, err := ioutil.ReadDir(dir)
	if err != nil {
		return nil, err
	}
	var wl []string
	for _, fi := range rl {
		path := filepath.Join(dir, fi.Name())

		// follow symlinks
		if (fi.Mode() & os.ModeSymlink) != 0 {
			if fi, err = oswrap.Stat(path); err != nil {
				return nil, err
			}
		}
		if !fi.IsDir() {
			wl = append(wl, path)
			continue
		}
		l, err := walkDir(path)
		if err != nil {
			return nil, err
		}
		wl = append(wl, l...)
	}
	return wl, nil
}

// pathMatch is a simpler filepath.Match but which supports recursive globbing
// (**) and doesn't get any more special than * or **.
func pathMatch(pattern, path string) (bool, error) {
	regex := []rune("^")
	runePattern := []rune(pattern)
	for i := 0; i < len(runePattern); i++ {
		ch := runePattern[i]
		switch ch {
		default:
			regex = append(regex, ch)
		case '%', '\\', '(', ')', '[', ']', '.', '^', '$', '?', '+', '{', '}', '=':
			regex = append(regex, '\\', ch)
		case '*':
			if i+1 < len(runePattern) && runePattern[i+1] == '*' {
				if i+2 < len(runePattern) && runePattern[i+2] == '*' {
					return false, fmt.Errorf("%s: malformed glob", pattern)
				}
				regex = append(regex, []rune(".*")...)
				i++
			} else {
				regex = append(regex, []rune("[^/]*")...)
			}
		}
	}
	regex = append(regex, '$')
	re, err := regexp.Compile(string(regex))
	if err != nil {
		return false, err
	}
	return re.MatchString(path), nil
}

func anyMatch(patterns []string, name string) (bool, error) {
	for _, ex := range patterns {
		m, err := pathMatch(ex, name)
		if err != nil {
			return false, err
		}
		if m {
			return true, nil
		}
	}
	return false, nil
}

func min(i, j int) int {
	if i < j {
		return i
	}
	return j
}

type pathWalk struct {
	parts     [][]string
	firstGlob int
}

// mergeWalks reduces the number of filesystem walks needed. If one walk will
// cover all the paths in another walk, it merges the include patterns, and only
// the larger walk will be performed.
func mergeWalks(walks []pathWalk) []pathWalk {
	for i := len(walks) - 2; i >= 0; i-- {
		wi := &walks[i]
		for j := i + 1; j < len(walks); j++ {
			wj := &walks[j]
			lowGlob := min(wi.firstGlob, wj.firstGlob)
			if lowGlob < 0 {
				continue
			}
			if filepath.Join(wi.parts[0][:lowGlob]...) == filepath.Join(wj.parts[0][:lowGlob]...) {
				wi.parts = append(wi.parts, wj.parts...)
				wi.firstGlob = lowGlob
				if j+1 < len(walks) {
					walks = append(walks[:j], walks[j+1:]...)
				} else {
					walks = walks[:j]
				}
			}
		}
	}
	return walks
}

func glob(base string, includes, excludes []string) ([]string, error) {
	var pathincludes []string
	for _, in := range includes {
		pathincludes = append(pathincludes, filepath.Join(base, in))
	}
	var pathexcludes []string
	for _, ex := range excludes {
		pathexcludes = append(pathexcludes, filepath.Join(base, ex))
	}

	var walks []pathWalk
	for _, pi := range pathincludes {
		parts := [][]string{splitPath(pi)}
		if !strings.Contains(pi, "*") {
			walks = append(walks, pathWalk{parts, -1})
			continue
		}
		firstGlob := -1
		for i, part := range parts[0] {
			if strings.Contains(part, "*") {
				firstGlob = i
				break
			}
		}
		walks = append(walks, pathWalk{parts, firstGlob})
	}

	walks = mergeWalks(walks)

	var out []string
	for _, walk := range walks {
		if walk.firstGlob < 0 {
			out = append(out, filepath.Join(walk.parts[0]...))
			continue
		}
		wd := filepath.Join(walk.parts[0][:walk.firstGlob]...)
		files, err := walkDir(wd)
		if err != nil {
			return nil, fmt.Errorf("walking %s: %v", wd, err)
		}

		var walkincludes []string
		for _, p := range walk.parts {
			walkincludes = append(walkincludes, filepath.Join(p...))
		}
		for _, file := range files {
			keep, err := anyMatch(walkincludes, file)
			if err != nil {
				return nil, err
			}
			remove, err := anyMatch(pathexcludes, file)
			if err != nil {
				return nil, err
			}
			if keep && !remove {
				out = append(out, file)
			}
		}
	}
	return out, nil
}

func globFiles(s goolib.PkgSources) ([]string, error) {
	cr := filepath.Clean(s.Root)
	return glob(cr, s.Include, s.Exclude)
}

func writeFiles(tw *tar.Writer, fm fileMap) error {
	for folder, fl := range fm {
		for _, file := range fl {
			fi, err := oswrap.Stat(file)
			if err != nil {
				return err
			}
			fpath := filepath.Join(folder, filepath.Base(file))
			fih, err := tar.FileInfoHeader(fi, "")
			if err != nil {
				return err
			}
			fih.Name = filepath.ToSlash(fpath)
			if err := tw.WriteHeader(fih); err != nil {
				return err
			}
			f, err := oswrap.Open(file)
			if err != nil {
				return err
			}
			if _, err := io.Copy(tw, f); err != nil {
				f.Close()
				return err
			}
			f.Close()
		}
	}
	return nil
}

func packageFiles(fm fileMap, gs goolib.GooSpec, dir string) (err error) {
	pn := goolib.PackageInfo{gs.PackageSpec.Name, gs.PackageSpec.Arch, gs.PackageSpec.Version}.PkgName()
	f, err := oswrap.Create(filepath.Join(dir, pn))
	if err != nil {
		return err
	}
	defer func() {
		cErr := f.Close()
		if cErr != nil && err == nil {
			err = cErr
		}
	}()
	gw := gzip.NewWriter(f)
	defer func() {
		cErr := gw.Close()
		if cErr != nil && err == nil {
			err = cErr
		}
	}()
	tw := tar.NewWriter(gw)
	defer func() {
		cErr := tw.Close()
		if cErr != nil && err == nil {
			err = cErr
		}
	}()

	if err := writeFiles(tw, fm); err != nil {
		return err
	}

	return goolib.WritePackageSpec(tw, gs.PackageSpec)
}

func mapFiles(sources []goolib.PkgSources) (fileMap, error) {
	fm := make(fileMap)
	for _, s := range sources {
		fl, err := globFiles(s)
		if err != nil {
			return nil, err
		}
		for _, f := range fl {
			dir := strings.TrimPrefix(filepath.Dir(f), s.Root)
			// Ensure leading '/' is trimmed for directories.
			dir = strings.TrimPrefix(dir, string(filepath.Separator))
			tgt := filepath.Join(s.Target, dir)
			fm[tgt] = append(fm[tgt], f)
		}
	}
	return fm, nil
}

func splitPath(path string) []string {
	parts := strings.Split(filepath.Clean(path), string(os.PathSeparator))
	out := []string{}
	for _, part := range parts {
		if part != "" {
			out = append(out, part)
		}
	}
	if len(path) > 0 && path[0] == '/' {
		out = append([]string{"/"}, out...)
	}
	return out
}

func verifyFiles(gs goolib.GooSpec, fm fileMap) error {
	fs := make(map[string]bool)
	for folder, fl := range fm {
		parts := splitPath(folder)
		for i := range parts {
			fs[filepath.Join(parts[:i+1]...)] = true
		}
		folder = filepath.Join(parts...)
		for _, file := range fl {
			fpath := filepath.Join(folder, filepath.Base(file))
			fs[fpath] = true
		}
	}
	var missing []string
	for src := range gs.PackageSpec.Files {
		if !fs[src] {
			missing = append(missing, src)
		}
	}
	if len(missing) > 0 {
		return fmt.Errorf("requested files %v not in package", missing)
	}
	return nil
}

func createPackage(gs goolib.GooSpec, baseDir, outDir string) error {
	switch {
	case gs.Build.Linux != "" && runtime.GOOS == "linux":
		cmd := gs.Build.Linux
		if !filepath.IsAbs(cmd) {
			cmd = filepath.Join(baseDir, cmd)
		}
		if err := goolib.Exec(cmd, gs.Build.LinuxArgs, nil, ioutil.Discard); err != nil {
			return err
		}
	case gs.Build.Windows != "" && runtime.GOOS == "windows":
		cmd := gs.Build.Windows
		if !filepath.IsAbs(cmd) {
			cmd = filepath.Join(baseDir, cmd)
		}
		if err := goolib.Exec(cmd, gs.Build.WindowsArgs, nil, ioutil.Discard); err != nil {
			return err
		}
	}
	fm, err := mapFiles(gs.Sources)
	if err != nil {
		return err
	}
	if err := verifyFiles(gs, fm); err != nil {
		return err
	}
	return packageFiles(fm, gs, outDir)
}

func usage() {
	fmt.Printf("Usage: %s <path/to/goospec>\n", filepath.Base(os.Args[0]))
}

func main() {
	flag.Parse()
	switch len(flag.Args()) {
	case 0:
		fmt.Println("Not enough args.")
		usage()
		os.Exit(1)
	case 1:
	default:
		fmt.Println("Too many args.")
		usage()
		os.Exit(1)
	}
	if flag.Arg(1) == "help" {
		usage()
		os.Exit(0)
	}

	outDir := *outputDir
	if outDir == "" {
		var err error
		outDir, err = os.Getwd()
		if err != nil {
			log.Fatal(err)
		}
	}
	gs, err := goolib.ReadGooSpec(flag.Arg(0))
	if err != nil {
		log.Fatal(err)
	}

	baseDir := filepath.Dir(filepath.Clean(flag.Arg(0)))
	if baseDir == "." {
		baseDir, err = os.Getwd()
		if err != nil {
			log.Fatal(err)
		}
	}
	if err := createPackage(gs, baseDir, outDir); err != nil {
		log.Fatal(err)
	}
}
<commit_msg>Update goopack.go<commit_after>/*
Copyright 2016 Google Inc. All Rights Reserved.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
    http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

// The goopack binary creates a GooGet package using the provided GooSpec file.
package main

import (
	"archive/tar"
	"compress/gzip"
	"flag"
	"fmt"
	"io"
	"io/ioutil"
	"log"
	"os"
	"path/filepath"
	"regexp"
	"runtime"
	"strings"

	"github.com/google/googet/goolib"
	"github.com/google/googet/oswrap"
)

var outputDir = flag.String("output_dir", "", "where to put the built package")

type fileMap map[string][]string

// walkDir returns a list of all files in directory and subdirectories, it is similar
// to filepath.Walk but works even if dir is a symlink.
func walkDir(dir string) ([]string, error) {
	rl, err := ioutil.ReadDir(dir)
	if err != nil {
		return nil, err
	}
	var wl []string
	for _, fi := range rl {
		path := filepath.Join(dir, fi.Name())

		// follow symlinks
		if (fi.Mode() & os.ModeSymlink) != 0 {
			if fi, err = oswrap.Stat(path); err != nil {
				return nil, err
			}
		}
		if !fi.IsDir() {
			wl = append(wl, path)
			continue
		}
		l, err := walkDir(path)
		if err != nil {
			return nil, err
		}
		wl = append(wl, l...)
	}
	return wl, nil
}

// pathMatch is a simpler filepath.Match but which supports recursive globbing
// (**) and doesn't get any more special than * or **.
func pathMatch(pattern, path string) (bool, error) {
	regex := []rune("^")
	runePattern := []rune(pattern)
	for i := 0; i < len(runePattern); i++ {
		ch := runePattern[i]
		switch ch {
		default:
			regex = append(regex, ch)
		case '%', '\\', '(', ')', '[', ']', '.', '^', '$', '?', '+', '{', '}', '=':
			regex = append(regex, '\\', ch)
		case '*':
			if i+1 < len(runePattern) && runePattern[i+1] == '*' {
				if i+2 < len(runePattern) && runePattern[i+2] == '*' {
					return false, fmt.Errorf("%s: malformed glob", pattern)
				}
				regex = append(regex, []rune(".*")...)
				i++
			} else {
				regex = append(regex, []rune("[^/]*")...)
			}
		}
	}
	regex = append(regex, '$')
	re, err := regexp.Compile(string(regex))
	if err != nil {
		return false, err
	}
	return re.MatchString(path), nil
}

func anyMatch(patterns []string, name string) (bool, error) {
	for _, ex := range patterns {
		m, err := pathMatch(ex, name)
		if err != nil {
			return false, err
		}
		if m {
			return true, nil
		}
	}
	return false, nil
}

func min(i, j int) int {
	if i < j {
		return i
	}
	return j
}

type pathWalk struct {
	parts     [][]string
	firstGlob int
}

// mergeWalks reduces the number of filesystem walks needed. If one walk will
// cover all the paths in another walk, it merges the include patterns, and only
// the larger walk will be performed.
func mergeWalks(walks []pathWalk) []pathWalk {
	for i := len(walks) - 2; i >= 0; i-- {
		wi := &walks[i]
		for j := i + 1; j < len(walks); j++ {
			wj := &walks[j]
			lowGlob := min(wi.firstGlob, wj.firstGlob)
			if lowGlob < 0 {
				continue
			}
			if filepath.Join(wi.parts[0][:lowGlob]...) == filepath.Join(wj.parts[0][:lowGlob]...) {
				wi.parts = append(wi.parts, wj.parts...)
				wi.firstGlob = lowGlob
				if j+1 < len(walks) {
					walks = append(walks[:j], walks[j+1:]...)
				} else {
					walks = walks[:j]
				}
			}
		}
	}
	return walks
}

func glob(base string, includes, excludes []string) ([]string, error) {
	var pathincludes []string
	for _, in := range includes {
		pathincludes = append(pathincludes, filepath.Join(base, in))
	}
	var pathexcludes []string
	for _, ex := range excludes {
		pathexcludes = append(pathexcludes, filepath.Join(base, ex))
	}

	var walks []pathWalk
	for _, pi := range pathincludes {
		parts := [][]string{splitPath(pi)}
		if !strings.Contains(pi, "*") {
			walks = append(walks, pathWalk{parts, -1})
			continue
		}
		firstGlob := -1
		for i, part := range parts[0] {
			if strings.Contains(part, "*") {
				firstGlob = i
				break
			}
		}
		walks = append(walks, pathWalk{parts, firstGlob})
	}

	walks = mergeWalks(walks)

	var out []string
	for _, walk := range walks {
		if walk.firstGlob < 0 {
			out = append(out, filepath.Join(walk.parts[0]...))
			continue
		}
		wd := filepath.Join(walk.parts[0][:walk.firstGlob]...)
		files, err := walkDir(wd)
		if err != nil {
			return nil, fmt.Errorf("walking %s: %v", wd, err)
		}

		var walkincludes []string
		for _, p := range walk.parts {
			walkincludes = append(walkincludes, filepath.Join(p...))
		}
		for _, file := range files {
			keep, err := anyMatch(walkincludes, file)
			if err != nil {
				return nil, err
			}
			remove, err := anyMatch(pathexcludes, file)
			if err != nil {
				return nil, err
			}
			if keep && !remove {
				out = append(out, file)
			}
		}
	}
	return out, nil
}

func globFiles(s goolib.PkgSources) ([]string, error) {
	cr := filepath.Clean(s.Root)
	return glob(cr, s.Include, s.Exclude)
}

func writeFiles(tw *tar.Writer, fm fileMap) error {
	for folder, fl := range fm {
		for _, file := range fl {
			fi, err := oswrap.Stat(file)
			if err != nil {
				return err
			}
			fpath := filepath.Join(folder, filepath.Base(file))
			fih, err := tar.FileInfoHeader(fi, "")
			if err != nil {
				return err
			}
			fih.Name = filepath.ToSlash(fpath)
			if err := tw.WriteHeader(fih); err != nil {
				return err
			}
			f, err := oswrap.Open(file)
			if err != nil {
				return err
			}
			if _, err := io.Copy(tw, f); err != nil {
				f.Close()
				return err
			}
			f.Close()
		}
	}
	return nil
}

func packageFiles(fm fileMap, gs goolib.GooSpec, dir string) (err error) {
	pn := goolib.PackageInfo{gs.PackageSpec.Name, gs.PackageSpec.Arch, gs.PackageSpec.Version}.PkgName()
	f, err := oswrap.Create(filepath.Join(dir, pn))
	if err != nil {
		return err
	}
	defer func() {
		cErr := f.Close()
		if cErr != nil && err == nil {
			err = cErr
		}
	}()
	gw := gzip.NewWriter(f)
	defer func() {
		cErr := gw.Close()
		if cErr != nil && err == nil {
			err = cErr
		}
	}()
	tw := tar.NewWriter(gw)
	defer func() {
		cErr := tw.Close()
		if cErr != nil && err == nil {
			err = cErr
		}
	}()

	if err := writeFiles(tw, fm); err != nil {
		return err
	}

	return goolib.WritePackageSpec(tw, gs.PackageSpec)
}

func mapFiles(sources []goolib.PkgSources) (fileMap, error) {
	fm := make(fileMap)
	for _, s := range sources {
		fl, err := globFiles(s)
		if err != nil {
			return nil, err
		}
		for _, f := range fl {
			dir := strings.TrimPrefix(filepath.Dir(f), s.Root)
			// Ensure leading '/' is trimmed for directories.
			dir = strings.TrimPrefix(dir, string(filepath.Separator))
			tgt := filepath.Join(s.Target, dir)
			fm[tgt] = append(fm[tgt], f)
		}
	}
	return fm, nil
}

func splitPath(path string) []string {
	parts := strings.Split(filepath.Clean(path), string(os.PathSeparator))
	out := []string{}
	for _, part := range parts {
		if part != "" {
			out = append(out, part)
		}
	}
	if len(path) > 0 && path[0] == '/' {
		out = append([]string{"/"}, out...)
	}
	return out
}

func verifyFiles(gs goolib.GooSpec, fm fileMap) error {
	fs := make(map[string]bool)
	for folder, fl := range fm {
		parts := splitPath(folder)
		for i := range parts {
			fs[filepath.Join(parts[:i+1]...)] = true
		}
		folder = filepath.Join(parts...)
		for _, file := range fl {
			fpath := filepath.Join(folder, filepath.Base(file))
			fs[fpath] = true
		}
	}
	var missing []string
	for src := range gs.PackageSpec.Files {
		if !fs[src] {
			missing = append(missing, src)
		}
	}
	if len(missing) > 0 {
		return fmt.Errorf("requested files %v not in package", missing)
	}
	return nil
}

func createPackage(gs goolib.GooSpec, baseDir, outDir string) error {
	switch {
	case gs.Build.Linux != "" && runtime.GOOS == "linux":
		cmd := gs.Build.Linux
		if !filepath.IsAbs(cmd) {
			cmd = filepath.Join(baseDir, cmd)
		}
		if err := goolib.Exec(cmd, gs.Build.LinuxArgs, nil, ioutil.Discard); err != nil {
			return err
		}
	case gs.Build.Windows != "" && runtime.GOOS == "windows":
		cmd := gs.Build.Windows
		if !filepath.IsAbs(cmd) {
			cmd = filepath.Join(baseDir, cmd)
		}
		if err := goolib.Exec(cmd, gs.Build.WindowsArgs, nil, ioutil.Discard); err != nil {
			return err
		}
	}
	fm, err := mapFiles(gs.Sources)
	if err != nil {
		return err
	}
	if err := verifyFiles(gs, fm); err != nil {
		return err
	}
	return packageFiles(fm, gs, outDir)
}

func usage() {
	fmt.Printf("Usage: %s <path/to/goospec>\n", filepath.Base(os.Args[0]))
}

func main() {
	flag.Parse()
	switch len(flag.Args()) {
	case 0:
		fmt.Println("Not enough args.")
		usage()
		os.Exit(1)
	case 1:
	default:
		fmt.Println("Too many args.")
		usage()
		os.Exit(1)
	}
	if flag.Arg(1) == "help" {
		usage()
		os.Exit(0)
	}

	outDir := *outputDir
	if outDir == "" {
		var err error
		outDir, err = os.Getwd()
		if err != nil {
			log.Fatal(err)
		}
	}
	gs, err := goolib.ReadGooSpec(flag.Arg(0))
	if err != nil {
		log.Fatal(err)
	}

	baseDir := filepath.Dir(filepath.Clean(flag.Arg(0)))
	if baseDir == "." {
		baseDir, err = os.Getwd()
		if err != nil {
			log.Fatal(err)
		}
	}
	if err := createPackage(gs, baseDir, outDir); err != nil {
		log.Fatal(err)
	}
}
<|endoftext|>


<commit_before>package builder

import (
	"fmt"
	"io"
	"io/ioutil"
	"os"
	"os/exec"
	"path/filepath"
	"strings"
	"time"

	docker "github.com/fsouza/go-dockerclient"

	s2igit "github.com/openshift/source-to-image/pkg/scm/git"

	"github.com/openshift/origin/pkg/build/api"
	"github.com/openshift/origin/pkg/build/builder/cmd/dockercfg"
	"github.com/openshift/origin/pkg/generate/git"
	"github.com/openshift/source-to-image/pkg/tar"
)

const (
	// initialURLCheckTimeout is the initial timeout used to check the
	// source URL.  If fetching the URL exceeds the timeout, then a longer
	// timeout will be tried until the fetch either succeeds or the build
	// itself times out.
	initialURLCheckTimeout = 16 * time.Second

	// timeoutIncrementFactor is the factor to use when increasing
	// the timeout after each unsuccessful try
	timeoutIncrementFactor = 4
)

type gitAuthError string
type gitNotFoundError string

func (e gitAuthError) Error() string {
	return fmt.Sprintf("failed to fetch requested repository %q with provided credentials", string(e))
}

func (e gitNotFoundError) Error() string {
	return fmt.Sprintf("requested repository %q not found", string(e))
}

// fetchSource retrieves the inputs defined by the build source into the
// provided directory, or returns an error if retrieval is not possible.
func fetchSource(dockerClient DockerClient, dir string, build *api.Build, urlTimeout time.Duration, in io.Reader, gitClient GitClient) (*git.SourceInfo, error) {
	hasGitSource := false

	// expect to receive input from STDIN
	if err := extractInputBinary(in, build.Spec.Source.Binary, dir); err != nil {
		return nil, err
	}

	// may retrieve source from Git
	hasGitSource, err := extractGitSource(gitClient, build.Spec.Source.Git, build.Spec.Revision, dir, urlTimeout)
	if err != nil {
		return nil, err
	}

	var sourceInfo *git.SourceInfo
	if hasGitSource {
		var errs []error
		sourceInfo, errs = gitClient.GetInfo(dir)
		if len(errs) > 0 {
			for _, e := range errs {
				glog.V(0).Infof("error: Unable to retrieve Git info: %v", e)
			}
		}
	}

	forcePull := false
	switch {
	case build.Spec.Strategy.SourceStrategy != nil:
		forcePull = build.Spec.Strategy.SourceStrategy.ForcePull
	case build.Spec.Strategy.DockerStrategy != nil:
		forcePull = build.Spec.Strategy.DockerStrategy.ForcePull
	case build.Spec.Strategy.CustomStrategy != nil:
		forcePull = build.Spec.Strategy.CustomStrategy.ForcePull
	}
	// extract source from an Image if specified
	for i, image := range build.Spec.Source.Images {
		imageSecretIndex := i
		if image.PullSecret == nil {
			imageSecretIndex = -1
		}
		err := extractSourceFromImage(dockerClient, image.From.Name, dir, imageSecretIndex, image.Paths, forcePull)
		if err != nil {
			return nil, err
		}
	}

	// a Dockerfile has been specified, create or overwrite into the destination
	if dockerfileSource := build.Spec.Source.Dockerfile; dockerfileSource != nil {
		baseDir := dir
		// if a context dir has been defined and we cloned source, overwrite the destination
		if hasGitSource && len(build.Spec.Source.ContextDir) != 0 {
			baseDir = filepath.Join(baseDir, build.Spec.Source.ContextDir)
		}
		return sourceInfo, ioutil.WriteFile(filepath.Join(baseDir, "Dockerfile"), []byte(*dockerfileSource), 0660)
	}

	return sourceInfo, nil
}

// checkRemoteGit validates the specified Git URL. It returns GitNotFoundError
// when the remote repository not found and GitAuthenticationError when the
// remote repository failed to authenticate.
// Since this is calling the 'git' binary, the proxy settings should be
// available for this command.
func checkRemoteGit(gitClient GitClient, url string, initialTimeout time.Duration) error {

	var (
		out    string
		errOut string
		err    error
	)

	timeout := initialTimeout
	for {
		glog.V(4).Infof("git ls-remote --heads %s", url)
		out, errOut, err = gitClient.TimedListRemote(timeout, url, "--heads")
		if len(out) != 0 {
			glog.V(4).Infof(out)
		}
		if len(errOut) != 0 {
			glog.V(4).Infof(errOut)
		}
		if err != nil {
			if _, ok := err.(*git.TimeoutError); ok {
				timeout = timeout * timeoutIncrementFactor
				glog.Infof("WARNING: timed out waiting for git server, will wait %s", timeout)
				continue
			}
		}
		break
	}
	if err != nil {
		combinedOut := out + errOut
		switch {
		case strings.Contains(combinedOut, "Authentication failed"):
			return gitAuthError(url)
		case strings.Contains(combinedOut, "not found"):
			return gitNotFoundError(url)
		}
	}
	return err
}

// checkSourceURI performs a check on the URI associated with the build
// to make sure that it is valid.
func checkSourceURI(gitClient GitClient, rawurl string, timeout time.Duration) error {
	if !s2igit.New().ValidCloneSpec(rawurl) {
		return fmt.Errorf("Invalid git source url: %s", rawurl)
	}
	return checkRemoteGit(gitClient, rawurl, timeout)
}

// extractInputBinary processes the provided input stream as directed by BinaryBuildSource
// into dir.
func extractInputBinary(in io.Reader, source *api.BinaryBuildSource, dir string) error {
	if source == nil {
		return nil
	}

	var path string
	if len(source.AsFile) > 0 {
		glog.V(0).Infof("Receiving source from STDIN as file %s", source.AsFile)
		path = filepath.Join(dir, source.AsFile)

		f, err := os.OpenFile(path, os.O_CREATE|os.O_EXCL|os.O_WRONLY, 0664)
		if err != nil {
			return err
		}
		defer f.Close()
		n, err := io.Copy(f, os.Stdin)
		if err != nil {
			return err
		}
		glog.V(4).Infof("Received %d bytes into %s", n, path)
		return nil
	}

	glog.V(0).Infof("Receiving source from STDIN as archive ...")

	cmd := exec.Command("bsdtar", "-x", "-o", "-m", "-f", "-", "-C", dir)
	cmd.Stdin = in
	out, err := cmd.CombinedOutput()
	if err != nil {
		glog.V(0).Infof("Extracting...\n%s", string(out))
		return fmt.Errorf("unable to extract binary build input, must be a zip, tar, or gzipped tar, or specified as a file: %v", err)
	}
	return nil
}

func extractGitSource(gitClient GitClient, gitSource *api.GitBuildSource, revision *api.SourceRevision, dir string, timeout time.Duration) (bool, error) {
	if gitSource == nil {
		return false, nil
	}

	glog.V(0).Infof("Downloading %q ...", gitSource.URI)

	// Check source URI, trying to connect to the server only if not using a proxy.
	if err := checkSourceURI(gitClient, gitSource.URI, timeout); err != nil {
		return true, err
	}

	// check if we specify a commit, ref, or branch to check out
	usingRef := len(gitSource.Ref) != 0 || (revision != nil && revision.Git != nil && len(revision.Git.Commit) != 0)

	// Recursive clone if we're not going to checkout a ref and submodule update later
	glog.V(3).Infof("Cloning source from %s", gitSource.URI)

	// Only use the quiet flag if Verbosity is not 5 or greater
	quiet := !glog.Is(5)
	if err := gitClient.CloneWithOptions(dir, gitSource.URI, git.CloneOptions{Recursive: !usingRef, Quiet: quiet, Shallow: !usingRef}); err != nil {
		return true, err
	}

	// if we specify a commit, ref, or branch to checkout, do so, and update submodules
	if usingRef {
		commit := gitSource.Ref

		if revision != nil && revision.Git != nil && revision.Git.Commit != "" {
			commit = revision.Git.Commit
		}

		if err := gitClient.Checkout(dir, commit); err != nil {
			return true, err
		}

		// Recursively update --init
		if err := gitClient.SubmoduleUpdate(dir, true, true); err != nil {
			return true, err
		}
	}

	return true, nil
}

func copyImageSource(dockerClient DockerClient, containerID, sourceDir, destDir string, tarHelper tar.Tar) error {
	// Setup destination directory
	fi, err := os.Stat(destDir)
	if err != nil {
		if !os.IsNotExist(err) {
			return err
		}
		glog.V(4).Infof("Creating image destination directory: %s", destDir)
		err := os.MkdirAll(destDir, 0644)
		if err != nil {
			return err
		}
	} else {
		if !fi.IsDir() {
			return fmt.Errorf("destination %s must be a directory", destDir)
		}
	}

	tempFile, err := ioutil.TempFile("", "imgsrc")
	if err != nil {
		return err
	}
	glog.V(4).Infof("Downloading source from path %s in container %s to temporary archive %s", sourceDir, containerID, tempFile.Name())
	err = dockerClient.DownloadFromContainer(containerID, docker.DownloadFromContainerOptions{
		OutputStream: tempFile,
		Path:         sourceDir,
	})
	if err != nil {
		return err
	}
	if err := tempFile.Close(); err != nil {
		return err
	}

	// Extract the created tar file to the destination directory
	file, err := os.Open(tempFile.Name())
	if err != nil {
		return err
	}
	defer file.Close()

	glog.V(4).Infof("Extracting temporary tar %s to directory %s", tempFile.Name(), destDir)
	var tarOutput io.Writer
	if glog.Is(4) {
		tarOutput = os.Stdout
	}
	return tarHelper.ExtractTarStreamWithLogging(destDir, file, tarOutput)
}

func extractSourceFromImage(dockerClient DockerClient, image, buildDir string, imageSecretIndex int, paths []api.ImageSourcePath, forcePull bool) error {
	glog.V(4).Infof("Extracting image source from %s", image)

	dockerAuth := docker.AuthConfiguration{}
	if imageSecretIndex != -1 {
		pullSecret := os.Getenv(fmt.Sprintf("%s%d", dockercfg.PullSourceAuthType, imageSecretIndex))
		if len(pullSecret) > 0 {
			authPresent := false
			dockerAuth, authPresent = dockercfg.NewHelper().GetDockerAuth(image, fmt.Sprintf("%s%d", dockercfg.PullSourceAuthType, imageSecretIndex))
			if authPresent {
				glog.V(5).Infof("Registry server Address: %s", dockerAuth.ServerAddress)
				glog.V(5).Infof("Registry server User Name: %s", dockerAuth.Username)
				glog.V(5).Infof("Registry server Email: %s", dockerAuth.Email)
				passwordPresent := "<<empty>>"
				if len(dockerAuth.Password) > 0 {
					passwordPresent = "<<non-empty>>"
				}
				glog.V(5).Infof("Registry server Password: %s", passwordPresent)
			}
		}
	}

	exists := true
	if !forcePull {
		_, err := dockerClient.InspectImage(image)
		if err != nil && err == docker.ErrNoSuchImage {
			exists = false
		} else if err != nil {
			return err
		}
	}

	if !exists || forcePull {
		glog.V(0).Infof("Pulling image %q ...", image)
		if err := dockerClient.PullImage(docker.PullImageOptions{Repository: image}, dockerAuth); err != nil {
			return fmt.Errorf("error pulling image %v: %v", image, err)
		}
	}

	containerConfig := &docker.Config{Image: image}
	if inspect, err := dockerClient.InspectImage(image); err != nil {
		return err
	} else {
		// In case the Docker image does not specify the entrypoint
		if len(inspect.Config.Entrypoint) == 0 && len(inspect.Config.Cmd) == 0 {
			containerConfig.Entrypoint = []string{"/fake-entrypoint"}
		}
	}

	// Create container to copy from
	container, err := dockerClient.CreateContainer(docker.CreateContainerOptions{Config: containerConfig})
	if err != nil {
		return fmt.Errorf("error creating source image container: %v", err)
	}
	defer dockerClient.RemoveContainer(docker.RemoveContainerOptions{ID: container.ID})

	tarHelper := tar.New()
	tarHelper.SetExclusionPattern(nil)

	for _, path := range paths {
		glog.V(4).Infof("Extracting path %s from container %s to %s", path.SourcePath, container.ID, path.DestinationDir)
		err := copyImageSource(dockerClient, container.ID, path.SourcePath, filepath.Join(buildDir, path.DestinationDir), tarHelper)
		if err != nil {
			return fmt.Errorf("error copying source path %s to %s: %v", path.SourcePath, path.DestinationDir, err)
		}
	}

	return nil
}
<commit_msg>the tmpfile may be not closed when function DownloadFromContainer returns err<commit_after>package builder

import (
	"fmt"
	"io"
	"io/ioutil"
	"os"
	"os/exec"
	"path/filepath"
	"strings"
	"time"

	docker "github.com/fsouza/go-dockerclient"

	s2igit "github.com/openshift/source-to-image/pkg/scm/git"

	"github.com/openshift/origin/pkg/build/api"
	"github.com/openshift/origin/pkg/build/builder/cmd/dockercfg"
	"github.com/openshift/origin/pkg/generate/git"
	"github.com/openshift/source-to-image/pkg/tar"
)

const (
	// initialURLCheckTimeout is the initial timeout used to check the
	// source URL.  If fetching the URL exceeds the timeout, then a longer
	// timeout will be tried until the fetch either succeeds or the build
	// itself times out.
	initialURLCheckTimeout = 16 * time.Second

	// timeoutIncrementFactor is the factor to use when increasing
	// the timeout after each unsuccessful try
	timeoutIncrementFactor = 4
)

type gitAuthError string
type gitNotFoundError string

func (e gitAuthError) Error() string {
	return fmt.Sprintf("failed to fetch requested repository %q with provided credentials", string(e))
}

func (e gitNotFoundError) Error() string {
	return fmt.Sprintf("requested repository %q not found", string(e))
}

// fetchSource retrieves the inputs defined by the build source into the
// provided directory, or returns an error if retrieval is not possible.
func fetchSource(dockerClient DockerClient, dir string, build *api.Build, urlTimeout time.Duration, in io.Reader, gitClient GitClient) (*git.SourceInfo, error) {
	hasGitSource := false

	// expect to receive input from STDIN
	if err := extractInputBinary(in, build.Spec.Source.Binary, dir); err != nil {
		return nil, err
	}

	// may retrieve source from Git
	hasGitSource, err := extractGitSource(gitClient, build.Spec.Source.Git, build.Spec.Revision, dir, urlTimeout)
	if err != nil {
		return nil, err
	}

	var sourceInfo *git.SourceInfo
	if hasGitSource {
		var errs []error
		sourceInfo, errs = gitClient.GetInfo(dir)
		if len(errs) > 0 {
			for _, e := range errs {
				glog.V(0).Infof("error: Unable to retrieve Git info: %v", e)
			}
		}
	}

	forcePull := false
	switch {
	case build.Spec.Strategy.SourceStrategy != nil:
		forcePull = build.Spec.Strategy.SourceStrategy.ForcePull
	case build.Spec.Strategy.DockerStrategy != nil:
		forcePull = build.Spec.Strategy.DockerStrategy.ForcePull
	case build.Spec.Strategy.CustomStrategy != nil:
		forcePull = build.Spec.Strategy.CustomStrategy.ForcePull
	}
	// extract source from an Image if specified
	for i, image := range build.Spec.Source.Images {
		imageSecretIndex := i
		if image.PullSecret == nil {
			imageSecretIndex = -1
		}
		err := extractSourceFromImage(dockerClient, image.From.Name, dir, imageSecretIndex, image.Paths, forcePull)
		if err != nil {
			return nil, err
		}
	}

	// a Dockerfile has been specified, create or overwrite into the destination
	if dockerfileSource := build.Spec.Source.Dockerfile; dockerfileSource != nil {
		baseDir := dir
		// if a context dir has been defined and we cloned source, overwrite the destination
		if hasGitSource && len(build.Spec.Source.ContextDir) != 0 {
			baseDir = filepath.Join(baseDir, build.Spec.Source.ContextDir)
		}
		return sourceInfo, ioutil.WriteFile(filepath.Join(baseDir, "Dockerfile"), []byte(*dockerfileSource), 0660)
	}

	return sourceInfo, nil
}

// checkRemoteGit validates the specified Git URL. It returns GitNotFoundError
// when the remote repository not found and GitAuthenticationError when the
// remote repository failed to authenticate.
// Since this is calling the 'git' binary, the proxy settings should be
// available for this command.
func checkRemoteGit(gitClient GitClient, url string, initialTimeout time.Duration) error {

	var (
		out    string
		errOut string
		err    error
	)

	timeout := initialTimeout
	for {
		glog.V(4).Infof("git ls-remote --heads %s", url)
		out, errOut, err = gitClient.TimedListRemote(timeout, url, "--heads")
		if len(out) != 0 {
			glog.V(4).Infof(out)
		}
		if len(errOut) != 0 {
			glog.V(4).Infof(errOut)
		}
		if err != nil {
			if _, ok := err.(*git.TimeoutError); ok {
				timeout = timeout * timeoutIncrementFactor
				glog.Infof("WARNING: timed out waiting for git server, will wait %s", timeout)
				continue
			}
		}
		break
	}
	if err != nil {
		combinedOut := out + errOut
		switch {
		case strings.Contains(combinedOut, "Authentication failed"):
			return gitAuthError(url)
		case strings.Contains(combinedOut, "not found"):
			return gitNotFoundError(url)
		}
	}
	return err
}

// checkSourceURI performs a check on the URI associated with the build
// to make sure that it is valid.
func checkSourceURI(gitClient GitClient, rawurl string, timeout time.Duration) error {
	if !s2igit.New().ValidCloneSpec(rawurl) {
		return fmt.Errorf("Invalid git source url: %s", rawurl)
	}
	return checkRemoteGit(gitClient, rawurl, timeout)
}

// extractInputBinary processes the provided input stream as directed by BinaryBuildSource
// into dir.
func extractInputBinary(in io.Reader, source *api.BinaryBuildSource, dir string) error {
	if source == nil {
		return nil
	}

	var path string
	if len(source.AsFile) > 0 {
		glog.V(0).Infof("Receiving source from STDIN as file %s", source.AsFile)
		path = filepath.Join(dir, source.AsFile)

		f, err := os.OpenFile(path, os.O_CREATE|os.O_EXCL|os.O_WRONLY, 0664)
		if err != nil {
			return err
		}
		defer f.Close()
		n, err := io.Copy(f, os.Stdin)
		if err != nil {
			return err
		}
		glog.V(4).Infof("Received %d bytes into %s", n, path)
		return nil
	}

	glog.V(0).Infof("Receiving source from STDIN as archive ...")

	cmd := exec.Command("bsdtar", "-x", "-o", "-m", "-f", "-", "-C", dir)
	cmd.Stdin = in
	out, err := cmd.CombinedOutput()
	if err != nil {
		glog.V(0).Infof("Extracting...\n%s", string(out))
		return fmt.Errorf("unable to extract binary build input, must be a zip, tar, or gzipped tar, or specified as a file: %v", err)
	}
	return nil
}

func extractGitSource(gitClient GitClient, gitSource *api.GitBuildSource, revision *api.SourceRevision, dir string, timeout time.Duration) (bool, error) {
	if gitSource == nil {
		return false, nil
	}

	glog.V(0).Infof("Downloading %q ...", gitSource.URI)

	// Check source URI, trying to connect to the server only if not using a proxy.
	if err := checkSourceURI(gitClient, gitSource.URI, timeout); err != nil {
		return true, err
	}

	// check if we specify a commit, ref, or branch to check out
	usingRef := len(gitSource.Ref) != 0 || (revision != nil && revision.Git != nil && len(revision.Git.Commit) != 0)

	// Recursive clone if we're not going to checkout a ref and submodule update later
	glog.V(3).Infof("Cloning source from %s", gitSource.URI)

	// Only use the quiet flag if Verbosity is not 5 or greater
	quiet := !glog.Is(5)
	if err := gitClient.CloneWithOptions(dir, gitSource.URI, git.CloneOptions{Recursive: !usingRef, Quiet: quiet, Shallow: !usingRef}); err != nil {
		return true, err
	}

	// if we specify a commit, ref, or branch to checkout, do so, and update submodules
	if usingRef {
		commit := gitSource.Ref

		if revision != nil && revision.Git != nil && revision.Git.Commit != "" {
			commit = revision.Git.Commit
		}

		if err := gitClient.Checkout(dir, commit); err != nil {
			return true, err
		}

		// Recursively update --init
		if err := gitClient.SubmoduleUpdate(dir, true, true); err != nil {
			return true, err
		}
	}

	return true, nil
}

func copyImageSource(dockerClient DockerClient, containerID, sourceDir, destDir string, tarHelper tar.Tar) error {
	// Setup destination directory
	fi, err := os.Stat(destDir)
	if err != nil {
		if !os.IsNotExist(err) {
			return err
		}
		glog.V(4).Infof("Creating image destination directory: %s", destDir)
		err := os.MkdirAll(destDir, 0644)
		if err != nil {
			return err
		}
	} else {
		if !fi.IsDir() {
			return fmt.Errorf("destination %s must be a directory", destDir)
		}
	}

	tempFile, err := ioutil.TempFile("", "imgsrc")
	if err != nil {
		return err
	}
	glog.V(4).Infof("Downloading source from path %s in container %s to temporary archive %s", sourceDir, containerID, tempFile.Name())
	err = dockerClient.DownloadFromContainer(containerID, docker.DownloadFromContainerOptions{
		OutputStream: tempFile,
		Path:         sourceDir,
	})
	if err != nil {
		tempFile.Close()
		return err
	}
	if err := tempFile.Close(); err != nil {
		return err
	}

	// Extract the created tar file to the destination directory
	file, err := os.Open(tempFile.Name())
	if err != nil {
		return err
	}
	defer file.Close()

	glog.V(4).Infof("Extracting temporary tar %s to directory %s", tempFile.Name(), destDir)
	var tarOutput io.Writer
	if glog.Is(4) {
		tarOutput = os.Stdout
	}
	return tarHelper.ExtractTarStreamWithLogging(destDir, file, tarOutput)
}

func extractSourceFromImage(dockerClient DockerClient, image, buildDir string, imageSecretIndex int, paths []api.ImageSourcePath, forcePull bool) error {
	glog.V(4).Infof("Extracting image source from %s", image)

	dockerAuth := docker.AuthConfiguration{}
	if imageSecretIndex != -1 {
		pullSecret := os.Getenv(fmt.Sprintf("%s%d", dockercfg.PullSourceAuthType, imageSecretIndex))
		if len(pullSecret) > 0 {
			authPresent := false
			dockerAuth, authPresent = dockercfg.NewHelper().GetDockerAuth(image, fmt.Sprintf("%s%d", dockercfg.PullSourceAuthType, imageSecretIndex))
			if authPresent {
				glog.V(5).Infof("Registry server Address: %s", dockerAuth.ServerAddress)
				glog.V(5).Infof("Registry server User Name: %s", dockerAuth.Username)
				glog.V(5).Infof("Registry server Email: %s", dockerAuth.Email)
				passwordPresent := "<<empty>>"
				if len(dockerAuth.Password) > 0 {
					passwordPresent = "<<non-empty>>"
				}
				glog.V(5).Infof("Registry server Password: %s", passwordPresent)
			}
		}
	}

	exists := true
	if !forcePull {
		_, err := dockerClient.InspectImage(image)
		if err != nil && err == docker.ErrNoSuchImage {
			exists = false
		} else if err != nil {
			return err
		}
	}

	if !exists || forcePull {
		glog.V(0).Infof("Pulling image %q ...", image)
		if err := dockerClient.PullImage(docker.PullImageOptions{Repository: image}, dockerAuth); err != nil {
			return fmt.Errorf("error pulling image %v: %v", image, err)
		}
	}

	containerConfig := &docker.Config{Image: image}
	if inspect, err := dockerClient.InspectImage(image); err != nil {
		return err
	} else {
		// In case the Docker image does not specify the entrypoint
		if len(inspect.Config.Entrypoint) == 0 && len(inspect.Config.Cmd) == 0 {
			containerConfig.Entrypoint = []string{"/fake-entrypoint"}
		}
	}

	// Create container to copy from
	container, err := dockerClient.CreateContainer(docker.CreateContainerOptions{Config: containerConfig})
	if err != nil {
		return fmt.Errorf("error creating source image container: %v", err)
	}
	defer dockerClient.RemoveContainer(docker.RemoveContainerOptions{ID: container.ID})

	tarHelper := tar.New()
	tarHelper.SetExclusionPattern(nil)

	for _, path := range paths {
		glog.V(4).Infof("Extracting path %s from container %s to %s", path.SourcePath, container.ID, path.DestinationDir)
		err := copyImageSource(dockerClient, container.ID, path.SourcePath, filepath.Join(buildDir, path.DestinationDir), tarHelper)
		if err != nil {
			return fmt.Errorf("error copying source path %s to %s: %v", path.SourcePath, path.DestinationDir, err)
		}
	}

	return nil
}
<|endoftext|>


<commit_before>package playback

import (
	"context"
	"fmt"
	"time"

	"github.com/StageAutoControl/controller/pkg/cntl"
	"github.com/StageAutoControl/controller/pkg/internal/logging"

	"github.com/StageAutoControl/controller/pkg/cntl/song"
)

// Player plays various things from a given data store, for example songs or a whole SetList.
type Player struct {
	logger    logging.Logger
	dataStore *cntl.DataStore
	writers   []TransportWriter
	waiters   []Waiter
}

// NewPlayer returns a new Player instance
func NewPlayer(logger logging.Logger, ds *cntl.DataStore, writers []TransportWriter, waiters []Waiter) *Player {
	return &Player{
		logger:    logger,
		dataStore: ds,
		writers:   writers,
		waiters:   waiters,
	}
}

func (p *Player) checkSetList(setList *cntl.SetList) error {
	for _, songID := range setList.Songs {
		if _, ok := p.dataStore.Songs[songID]; !ok {
			return fmt.Errorf("cannot find Process %q", songID)
		}
	}

	return nil
}

// PlaySetList plays a full SetList
func (p *Player) PlaySetList(ctx context.Context, setListID string) error {
	setList, ok := p.dataStore.SetLists[setListID]
	if !ok {
		return fmt.Errorf("cannot find SetList %q", setListID)
	}

	if err := p.checkSetList(setList); err != nil {
		return err
	}

	for _, songID := range setList.Songs {
		select {
		case <-ctx.Done():
			p.logger.Warn("Aborting")
			return nil
		default:
		}

		p.logger.Infof("Playing song %s", songID)

		if err := p.PlaySong(ctx, songID); err != nil {
			return err
		}
	}

	return nil
}

func (p *Player) wait(ctx context.Context) error {
	if len(p.waiters) == 0 {
		return nil
	}

	chanLen := len(p.waiters) + 1
	done := make(chan struct{}, chanLen)
	cancel := make(chan struct{}, chanLen)

	defer func() {
		cancel <- struct{}{}
	}()

	for _, w := range p.waiters {
		go func() {
			if err := w.Wait(done, cancel); err != nil {
				p.logger.Error(err)
			}
		}()
	}

	select {
	case <-ctx.Done():
		return ErrCancelled
	case <-done:
		return nil
	}
}

// PlaySong plays a full song
func (p *Player) PlaySong(ctx context.Context, songID string) error {
	commands, err := song.Render(p.dataStore, songID)
	if err != nil {
		return err
	}

	p.logger.Infof("Waiting for waiters before playing song %s", songID)
	if err := p.wait(ctx); err != nil {
		return err
	}

	p.logger.Infof("Playing song %s", songID)
	return Play(ctx, p.logger, p.writers, commands)
}

// CalcRenderSpeed calculates the render speed of a BarChange to a time.Duration
func CalcRenderSpeed(bc *cntl.BarChange) time.Duration {
	return time.Minute / time.Duration(bc.Speed*uint16(bc.NoteValue)/4) / time.Duration(cntl.RenderFrames/bc.NoteValue)
}

// Play plays a given slice of commands and send it to the given writers
func Play(ctx context.Context, logger logging.Logger, writers []TransportWriter, commands []cntl.Command) error {
	l := len(commands)
	t := time.NewTicker(1 * time.Nanosecond)
	done := ctx.Done()

	var i int
	var cmd cntl.Command
	for {
		select {
		case <-done:
			return ErrCancelled

		case <-t.C:
			if i >= l {
				t.Stop()
				return nil
			}

			cmd = commands[i]
			if cmd.BarChange != nil {
				t.Stop()
				t = time.NewTicker(CalcRenderSpeed(cmd.BarChange))
			}

			for _, w := range writers {
				go func() {
					if err := w.Write(cmd); err != nil {
						logger.Error(err)
					}
				}()
			}

			i++
		}
	}
}

// ToPlayable takes a slice of DMXCommands and combines it with the given BarParams to a playable slice of Commands
func ToPlayable(bp cntl.BarParams, dmxCommands []cntl.DMXCommands) []cntl.Command {
	commands := make([]cntl.Command, len(dmxCommands))
	for i, cmd := range dmxCommands {
		commands[i] = cntl.Command{

			DMXCommands:  cmd,
			MIDICommands: []cntl.MIDICommand{},
		}
	}

	commands[0].BarChange = &cntl.BarChange{
		At:        0,
		BarParams: bp,
	}

	return commands
}
<commit_msg>Improve SetList playback logs<commit_after>package playback

import (
	"context"
	"fmt"
	"time"

	"github.com/StageAutoControl/controller/pkg/cntl"
	"github.com/StageAutoControl/controller/pkg/cntl/song"
	"github.com/StageAutoControl/controller/pkg/internal/logging"
)

// Player plays various things from a given data store, for example songs or a whole SetList.
type Player struct {
	logger    logging.Logger
	dataStore *cntl.DataStore
	writers   []TransportWriter
	waiters   []Waiter
}

// NewPlayer returns a new Player instance
func NewPlayer(logger logging.Logger, ds *cntl.DataStore, writers []TransportWriter, waiters []Waiter) *Player {
	return &Player{
		logger:    logger,
		dataStore: ds,
		writers:   writers,
		waiters:   waiters,
	}
}

func (p *Player) checkSetList(setList *cntl.SetList) error {
	for _, songID := range setList.Songs {
		if _, ok := p.dataStore.Songs[songID]; !ok {
			return fmt.Errorf("cannot find Process %q", songID)
		}
	}

	return nil
}

// PlaySetList plays a full SetList
func (p *Player) PlaySetList(ctx context.Context, setListID string) error {
	setList, ok := p.dataStore.SetLists[setListID]
	if !ok {
		return fmt.Errorf("cannot find SetList %q", setListID)
	}

	if err := p.checkSetList(setList); err != nil {
		return err
	}

	for _, songID := range setList.Songs {
		select {
		case <-ctx.Done():
			p.logger.Warn("Aborting")
			return nil
		default:
		}

		s, ok := p.dataStore.Songs[songID]
		if !ok {
			return fmt.Errorf("failed to find song %v", songID)
		}

		p.logger.Infof("Playing song %v", s.Name)

		if err := p.PlaySong(ctx, songID); err != nil {
			return err
		}
	}

	return nil
}

func (p *Player) wait(ctx context.Context) error {
	if len(p.waiters) == 0 {
		return nil
	}

	chanLen := len(p.waiters) + 1
	done := make(chan struct{}, chanLen)
	cancel := make(chan struct{}, chanLen)

	defer func() {
		cancel <- struct{}{}
	}()

	for _, w := range p.waiters {
		go func() {
			if err := w.Wait(done, cancel); err != nil {
				p.logger.Error(err)
			}
		}()
	}

	select {
	case <-ctx.Done():
		return ErrCancelled
	case <-done:
		return nil
	}
}

// PlaySong plays a full song
func (p *Player) PlaySong(ctx context.Context, songID string) error {
	commands, err := song.Render(p.dataStore, songID)
	if err != nil {
		return err
	}

	s, ok := p.dataStore.Songs[songID]
	if !ok {
		return fmt.Errorf("failed to find song %v", songID)
	}

	p.logger.Infof("Playing song %v", s.Name)

	p.logger.Infof("Waiting for waiters before playing song %v", s.Name)
	if err := p.wait(ctx); err != nil {
		return err
	}

	p.logger.Infof("Playing song %v", s.Name)
	return Play(ctx, p.logger, p.writers, commands)
}

// CalcRenderSpeed calculates the render speed of a BarChange to a time.Duration
func CalcRenderSpeed(bc *cntl.BarChange) time.Duration {
	return time.Minute / time.Duration(bc.Speed*uint16(bc.NoteValue)/4) / time.Duration(cntl.RenderFrames/bc.NoteValue)
}

// Play plays a given slice of commands and send it to the given writers
func Play(ctx context.Context, logger logging.Logger, writers []TransportWriter, commands []cntl.Command) error {
	l := len(commands)
	t := time.NewTicker(1 * time.Nanosecond)
	done := ctx.Done()

	var i int
	var cmd cntl.Command
	for {
		select {
		case <-done:
			return ErrCancelled

		case <-t.C:
			if i >= l {
				t.Stop()
				return nil
			}

			cmd = commands[i]
			if cmd.BarChange != nil {
				t.Stop()
				t = time.NewTicker(CalcRenderSpeed(cmd.BarChange))
			}

			for _, w := range writers {
				go func() {
					if err := w.Write(cmd); err != nil {
						logger.Error(err)
					}
				}()
			}

			i++
		}
	}
}

// ToPlayable takes a slice of DMXCommands and combines it with the given BarParams to a playable slice of Commands
func ToPlayable(bp cntl.BarParams, dmxCommands []cntl.DMXCommands) []cntl.Command {
	commands := make([]cntl.Command, len(dmxCommands))
	for i, cmd := range dmxCommands {
		commands[i] = cntl.Command{

			DMXCommands:  cmd,
			MIDICommands: []cntl.MIDICommand{},
		}
	}

	commands[0].BarChange = &cntl.BarChange{
		At:        0,
		BarParams: bp,
	}

	return commands
}
<|endoftext|>


<commit_before>package main

import (
	"context"
	"encoding/json"
	"flag"
	"fmt"
	"os"
	"path/filepath"

	"github.com/containerd/containerd"
	"github.com/containerd/containerd/cio"
	"github.com/containerd/containerd/namespaces"
	specs "github.com/opencontainers/runtime-spec/specs-go"
	log "github.com/sirupsen/logrus"
)

func parseCmd(ctx context.Context, command string, args []string) (*log.Entry, string, string, string, string) {
	invoked := filepath.Base(os.Args[0])
	flags := flag.NewFlagSet(command, flag.ExitOnError)
	flags.Usage = func() {
		fmt.Printf("USAGE: %s %s [service]\n\n", invoked, command)
		fmt.Printf("Options:\n")
		flags.PrintDefaults()
	}

	sock := flags.String("sock", defaultSocket, "Path to containerd socket")
	path := flags.String("path", defaultPath, "Path to service configs")

	dumpSpec := flags.String("dump-spec", "", "Dump container spec to file before start")

	if err := flags.Parse(args); err != nil {
		log.Fatal("Unable to parse args")
	}
	args = flags.Args()

	if len(args) != 1 {
		fmt.Println("Please specify the service")
		flags.Usage()
		os.Exit(1)
	}

	service := args[0]

	log := log.WithFields(log.Fields{
		"service": service,
	})

	return log, service, *sock, *path, *dumpSpec
}

func stopCmd(ctx context.Context, args []string) {
	log, service, sock, path, _ := parseCmd(ctx, "stop", args)

	log.Infof("Stopping service: %q", service)
	id, pid, msg, err := stop(ctx, service, sock, path)
	if err != nil {
		log.WithError(err).Fatal(msg)
	}

	log.Debugf("Stopped %s pid %d", id, pid)
}

func startCmd(ctx context.Context, args []string) {
	log, service, sock, path, dumpSpec := parseCmd(ctx, "start", args)

	log.Infof("Starting service: %q", service)
	id, pid, msg, err := start(ctx, service, sock, path, dumpSpec)
	if err != nil {
		log.WithError(err).Fatal(msg)
	}

	log.Debugf("Started %s pid %d", id, pid)
}

func restartCmd(ctx context.Context, args []string) {
	// validate arguments with the command as "restart"
	parseCmd(ctx, "restart", args)

	stopCmd(ctx, args)
	startCmd(ctx, args)
}

type logio struct {
	config cio.Config
}

func (c *logio) Config() cio.Config {
	return c.config
}

func (c *logio) Cancel() {
}

func (c *logio) Wait() {
}

func (c *logio) Close() error {
	return nil
}

func stop(ctx context.Context, service, sock, basePath string) (string, uint32, string, error) {
	path := filepath.Join(basePath, service)

	runtimeConfig := getRuntimeConfig(path)

	client, err := containerd.New(sock)
	if err != nil {
		return "", 0, "creating containerd client", err
	}

	if runtimeConfig.Namespace != "" {
		ctx = namespaces.WithNamespace(ctx, runtimeConfig.Namespace)
	}

	ctr, err := client.LoadContainer(ctx, service)
	if err != nil {
		return "", 0, "loading container", err
	}

	task, err := ctr.Task(ctx, nil)
	if err != nil {
		return "", 0, "fetching task", err
	}

	id := ctr.ID()
	pid := task.Pid()

	err = task.Kill(ctx, 9)
	if err != nil {
		return "", 0, "killing task", err
	}

	_, err = task.Wait(ctx)
	if err != nil {
		return "", 0, "waiting for task to exit", err
	}

	_, err = task.Delete(ctx)
	if err != nil {
		return "", 0, "deleting task", err
	}

	err = ctr.Delete(ctx)
	if err != nil {
		return "", 0, "deleting container", err
	}

	return id, pid, "", nil
}

func start(ctx context.Context, service, sock, basePath, dumpSpec string) (string, uint32, string, error) {
	path := filepath.Join(basePath, service)

	runtimeConfig := getRuntimeConfig(path)

	rootfs := filepath.Join(path, "rootfs")

	if err := prepareFilesystem(path, runtimeConfig); err != nil {
		return "", 0, "preparing filesystem", err
	}

	client, err := containerd.New(sock)
	if err != nil {
		return "", 0, "creating containerd client", err
	}

	var spec *specs.Spec
	specf, err := os.Open(filepath.Join(path, "config.json"))
	if err != nil {
		return "", 0, "failed to read service spec", err
	}
	if err := json.NewDecoder(specf).Decode(&spec); err != nil {
		return "", 0, "failed to parse service spec", err
	}

	spec.Root.Path = rootfs

	if dumpSpec != "" {
		d, err := os.Create(dumpSpec)
		if err != nil {
			return "", 0, "failed to open file for spec dump", err
		}
		enc := json.NewEncoder(d)
		enc.SetIndent("", "    ")
		if err := enc.Encode(&spec); err != nil {
			return "", 0, "failed to write spec dump", err
		}

	}

	if runtimeConfig.Namespace != "" {
		ctx = namespaces.WithNamespace(ctx, runtimeConfig.Namespace)
	}

	ctr, err := client.NewContainer(ctx, service, containerd.WithSpec(spec))
	if err != nil {
		return "", 0, "failed to create container", err
	}

	logger := GetLog(varLogDir)

	io := func(id string) (cio.IO, error) {
		stdoutFile := logger.Path(service + ".out")
		stderrFile := logger.Path(service)
		return &logio{
			cio.Config{
				Stdin:    "/dev/null",
				Stdout:   stdoutFile,
				Stderr:   stderrFile,
				Terminal: false,
			},
		}, nil
	}
	task, err := ctr.NewTask(ctx, io)
	if err != nil {
		// Don't bother to destroy the container here.
		return "", 0, "failed to create task", err
	}

	if err := prepareProcess(int(task.Pid()), runtimeConfig); err != nil {
		return "", 0, "preparing process", err
	}

	if err := task.Start(ctx); err != nil {
		// Don't destroy the container here so it can be inspected for debugging.
		return "", 0, "failed to start task", err
	}

	return ctr.ID(), task.Pid(), "", nil
}
<commit_msg>workaround bad containerd bug<commit_after>package main

import (
	"context"
	"encoding/json"
	"flag"
	"fmt"
	"os"
	pathutil "path"
	"path/filepath"

	syscall "golang.org/x/sys/unix"

	"github.com/containerd/containerd"
	"github.com/containerd/containerd/cio"
	"github.com/containerd/containerd/namespaces"
	specs "github.com/opencontainers/runtime-spec/specs-go"
	log "github.com/sirupsen/logrus"
)

func parseCmd(ctx context.Context, command string, args []string) (*log.Entry, string, string, string, string) {
	invoked := filepath.Base(os.Args[0])
	flags := flag.NewFlagSet(command, flag.ExitOnError)
	flags.Usage = func() {
		fmt.Printf("USAGE: %s %s [service]\n\n", invoked, command)
		fmt.Printf("Options:\n")
		flags.PrintDefaults()
	}

	sock := flags.String("sock", defaultSocket, "Path to containerd socket")
	path := flags.String("path", defaultPath, "Path to service configs")

	dumpSpec := flags.String("dump-spec", "", "Dump container spec to file before start")

	if err := flags.Parse(args); err != nil {
		log.Fatal("Unable to parse args")
	}
	args = flags.Args()

	if len(args) != 1 {
		fmt.Println("Please specify the service")
		flags.Usage()
		os.Exit(1)
	}

	service := args[0]

	log := log.WithFields(log.Fields{
		"service": service,
	})

	return log, service, *sock, *path, *dumpSpec
}

func stopCmd(ctx context.Context, args []string) {
	log, service, sock, path, _ := parseCmd(ctx, "stop", args)

	log.Infof("Stopping service: %q", service)
	id, pid, msg, err := stop(ctx, service, sock, path)
	if err != nil {
		log.WithError(err).Fatal(msg)
	}

	log.Debugf("Stopped %s pid %d", id, pid)
}

func startCmd(ctx context.Context, args []string) {
	log, service, sock, path, dumpSpec := parseCmd(ctx, "start", args)

	log.Infof("Starting service: %q", service)
	id, pid, msg, err := start(ctx, service, sock, path, dumpSpec)
	if err != nil {
		log.WithError(err).Fatal(msg)
	}

	log.Debugf("Started %s pid %d", id, pid)
}

func restartCmd(ctx context.Context, args []string) {
	// validate arguments with the command as "restart"
	parseCmd(ctx, "restart", args)

	stopCmd(ctx, args)
	startCmd(ctx, args)
}

type logio struct {
	config cio.Config
}

func (c *logio) Config() cio.Config {
	return c.config
}

func (c *logio) Cancel() {
}

func (c *logio) Wait() {
}

func (c *logio) Close() error {
	return nil
}

func stop(ctx context.Context, service, sock, basePath string) (string, uint32, string, error) {
	path := filepath.Join(basePath, service)

	runtimeConfig := getRuntimeConfig(path)

	client, err := containerd.New(sock)
	if err != nil {
		return "", 0, "creating containerd client", err
	}

	if runtimeConfig.Namespace != "" {
		ctx = namespaces.WithNamespace(ctx, runtimeConfig.Namespace)
	}

	ctr, err := client.LoadContainer(ctx, service)
	if err != nil {
		return "", 0, "loading container", err
	}

	task, err := ctr.Task(ctx, nil)
	if err != nil {
		return "", 0, "fetching task", err
	}

	id := ctr.ID()
	pid := task.Pid()

	err = task.Kill(ctx, 9)
	if err != nil {
		return "", 0, "killing task", err
	}

	_, err = task.Wait(ctx)
	if err != nil {
		return "", 0, "waiting for task to exit", err
	}

	_, err = task.Delete(ctx)
	if err != nil {
		return "", 0, "deleting task", err
	}

	err = ctr.Delete(ctx)
	if err != nil {
		return "", 0, "deleting container", err
	}

	return id, pid, "", nil
}

func start(ctx context.Context, service, sock, basePath, dumpSpec string) (string, uint32, string, error) {
	path := filepath.Join(basePath, service)

	runtimeConfig := getRuntimeConfig(path)

	rootfs := filepath.Join(path, "rootfs")

	if err := prepareFilesystem(path, runtimeConfig); err != nil {
		return "", 0, "preparing filesystem", err
	}

	client, err := containerd.New(sock)
	if err != nil {
		return "", 0, "creating containerd client", err
	}

	var spec *specs.Spec
	specf, err := os.Open(filepath.Join(path, "config.json"))
	if err != nil {
		return "", 0, "failed to read service spec", err
	}
	if err := json.NewDecoder(specf).Decode(&spec); err != nil {
		return "", 0, "failed to parse service spec", err
	}

	spec.Root.Path = rootfs

	if dumpSpec != "" {
		d, err := os.Create(dumpSpec)
		if err != nil {
			return "", 0, "failed to open file for spec dump", err
		}
		enc := json.NewEncoder(d)
		enc.SetIndent("", "    ")
		if err := enc.Encode(&spec); err != nil {
			return "", 0, "failed to write spec dump", err
		}

	}

	if runtimeConfig.Namespace != "" {
		ctx = namespaces.WithNamespace(ctx, runtimeConfig.Namespace)
	}

	ctr, err := client.NewContainer(ctx, service, containerd.WithSpec(spec))
	if err != nil {
		return "", 0, "failed to create container", err
	}

	logger := GetLog(varLogDir)

	// This is silly but necessary due to containerd bug
	// https://github.com/containerd/containerd/issues/4019
	// essentially, when you create a container and then remove it,
	// containerd blows away everything in the parent dir of the first one it finds,
	// in this case, "/dev/null", so it blows away everything in "/dev".
	// This most certainly is a "bad thing".
	//
	// To fix it temporarily, we are creating a tmpdir and creating a null
	// device there, so that it can blow away the tempdir
	stdinDir := pathutil.Join("/run", "containers-stdin", service)
	if err := os.MkdirAll(stdinDir, 0700); err != nil {
		return "", 0, "failed to create stdin directory", err
	}
	stdinFile := pathutil.Join(stdinDir, "null")
	// make a dev null in stdinDir
	if err := syscall.Mknod(stdinFile, uint32(os.FileMode(0660)|syscall.S_IFCHR), int(syscall.Mkdev(1, 3))); err != nil {
		return "", 0, "failed to create stdin null file", err
	}

	io := func(id string) (cio.IO, error) {
		stdoutFile := logger.Path(service + ".out")
		stderrFile := logger.Path(service)
		return &logio{
			cio.Config{
				Stdin:    stdinFile,
				Stdout:   stdoutFile,
				Stderr:   stderrFile,
				Terminal: false,
			},
		}, nil
	}
	task, err := ctr.NewTask(ctx, io)
	if err != nil {
		// Don't bother to destroy the container here.
		return "", 0, "failed to create task", err
	}

	if err := prepareProcess(int(task.Pid()), runtimeConfig); err != nil {
		return "", 0, "preparing process", err
	}

	if err := task.Start(ctx); err != nil {
		// Don't destroy the container here so it can be inspected for debugging.
		return "", 0, "failed to start task", err
	}

	return ctr.ID(), task.Pid(), "", nil
}
<|endoftext|>


<commit_before>/*
Copyright 2020 The Kubernetes Authors All rights reserved.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package node

import (
	"fmt"
	"net"
	"os"
	"os/exec"
	"path/filepath"
	"strconv"
	"strings"

	"github.com/blang/semver"
	"github.com/docker/machine/libmachine"
	"github.com/docker/machine/libmachine/host"
	"github.com/golang/glog"
	"github.com/pkg/errors"
	"github.com/spf13/viper"
	cmdcfg "k8s.io/minikube/cmd/minikube/cmd/config"
	"k8s.io/minikube/pkg/drivers/kic/oci"
	"k8s.io/minikube/pkg/minikube/bootstrapper"
	"k8s.io/minikube/pkg/minikube/cluster"
	"k8s.io/minikube/pkg/minikube/config"
	"k8s.io/minikube/pkg/minikube/constants"
	"k8s.io/minikube/pkg/minikube/cruntime"
	"k8s.io/minikube/pkg/minikube/driver"
	"k8s.io/minikube/pkg/minikube/exit"
	"k8s.io/minikube/pkg/minikube/kubeconfig"
	"k8s.io/minikube/pkg/minikube/localpath"
	"k8s.io/minikube/pkg/minikube/machine"
	"k8s.io/minikube/pkg/minikube/out"
	"k8s.io/minikube/pkg/util/lock"
)

var (
	// DockerEnv contains the environment variables
	DockerEnv []string
	// DockerOpt contains the option parameters
	DockerOpt []string
	// ExtraOptions contains extra options (if any)
	ExtraOptions config.ExtraOptionSlice
	// AddonList contains the list of addons
	AddonList []string
)

// configureRuntimes does what needs to happen to get a runtime going.
func configureRuntimes(runner cruntime.CommandRunner, drvName string, k8s config.KubernetesConfig, kv semver.Version) cruntime.Manager {
	co := cruntime.Config{
		Type:   viper.GetString(containerRuntime),
		Runner: runner, ImageRepository: k8s.ImageRepository,
		KubernetesVersion: kv,
	}
	cr, err := cruntime.New(co)
	if err != nil {
		exit.WithError("Failed runtime", err)
	}

	disableOthers := true
	if driver.BareMetal(drvName) {
		disableOthers = false
	}

	// Preload is overly invasive for bare metal, and caching is not meaningful. KIC handled elsewhere.
	if driver.IsVM(drvName) {
		if err := cr.Preload(k8s); err != nil {
			switch err.(type) {
			case *cruntime.ErrISOFeature:
				out.T(out.Tip, "Existing disk is missing new features ({{.error}}). To upgrade, run 'minikube delete'", out.V{"error": err})
			default:
				glog.Warningf("%s preload failed: %v, falling back to caching images", cr.Name(), err)
			}

			if err := machine.CacheImagesForBootstrapper(k8s.ImageRepository, k8s.KubernetesVersion, viper.GetString(cmdcfg.Bootstrapper)); err != nil {
				exit.WithError("Failed to cache images", err)
			}
		}
	}

	err = cr.Enable(disableOthers)
	if err != nil {
		exit.WithError("Failed to enable container runtime", err)
	}

	return cr
}

func showVersionInfo(k8sVersion string, cr cruntime.Manager) {
	version, _ := cr.Version()
	out.T(cr.Style(), "Preparing Kubernetes {{.k8sVersion}} on {{.runtime}} {{.runtimeVersion}} ...", out.V{"k8sVersion": k8sVersion, "runtime": cr.Name(), "runtimeVersion": version})
	for _, v := range DockerOpt {
		out.T(out.Option, "opt {{.docker_option}}", out.V{"docker_option": v})
	}
	for _, v := range DockerEnv {
		out.T(out.Option, "env {{.docker_env}}", out.V{"docker_env": v})
	}
}

// setupKubeAdm adds any requested files into the VM before Kubernetes is started
func setupKubeAdm(mAPI libmachine.API, cfg config.ClusterConfig, node config.Node) bootstrapper.Bootstrapper {
	bs, err := cluster.Bootstrapper(mAPI, viper.GetString(cmdcfg.Bootstrapper), cfg, node)
	if err != nil {
		exit.WithError("Failed to get bootstrapper", err)
	}
	for _, eo := range ExtraOptions {
		out.T(out.Option, "{{.extra_option_component_name}}.{{.key}}={{.value}}", out.V{"extra_option_component_name": eo.Component, "key": eo.Key, "value": eo.Value})
	}
	// Loads cached images, generates config files, download binaries
	if err := bs.UpdateCluster(cfg); err != nil {
		exit.WithError("Failed to update cluster", err)
	}
	if err := bs.SetupCerts(cfg.KubernetesConfig, node); err != nil {
		exit.WithError("Failed to setup certs", err)
	}
	return bs
}

func setupKubeconfig(h *host.Host, cc *config.ClusterConfig, n *config.Node, clusterName string) (*kubeconfig.Settings, error) {
	addr, err := apiServerURL(*h, *n)
	if err != nil {
		exit.WithError("Failed to get api server URL", err)
	}
	if cc.KubernetesConfig.APIServerName != constants.APIServerName {
		addr = strings.Replace(addr, n.IP, cc.KubernetesConfig.APIServerName, -1)
	}
	kcs := &kubeconfig.Settings{
		ClusterName:          clusterName,
		ClusterServerAddress: addr,
		ClientCertificate:    localpath.MakeMiniPath("client.crt"),
		ClientKey:            localpath.MakeMiniPath("client.key"),
		CertificateAuthority: localpath.MakeMiniPath("ca.crt"),
		KeepContext:          viper.GetBool(keepContext),
		EmbedCerts:           viper.GetBool(embedCerts),
	}

	kcs.SetPath(kubeconfig.PathFromEnv())
	if err := kubeconfig.Update(kcs); err != nil {
		return kcs, err
	}
	return kcs, nil
}

// apiServerURL returns a URL to end user can reach to the api server
func apiServerURL(h host.Host, n config.Node) (string, error) {
	if driver.IsKIC(h.DriverName) {
		p, err := oci.HostPortBinding(h.DriverName, h.Name, n.Port)
		if err != nil {
			return "", errors.Wrap(err, "host port binding")
		}
		return fmt.Sprintf("https://" + net.JoinHostPort("127.0.0.1", strconv.Itoa(p))), nil
	}
	ip, err := h.Driver.GetIP()
	if err != nil {
		return "", errors.Wrap(err, "get ip")
	}
	return fmt.Sprintf("https://" + net.JoinHostPort(ip, strconv.Itoa(n.Port))), nil
}

// configureMounts configures any requested filesystem mounts
func configureMounts() {
	if !viper.GetBool(createMount) {
		return
	}

	out.T(out.Mounting, "Creating mount {{.name}} ...", out.V{"name": viper.GetString(mountString)})
	path := os.Args[0]
	mountDebugVal := 0
	if glog.V(8) {
		mountDebugVal = 1
	}
	mountCmd := exec.Command(path, "mount", fmt.Sprintf("--v=%d", mountDebugVal), viper.GetString(mountString))
	mountCmd.Env = append(os.Environ(), constants.IsMinikubeChildProcess+"=true")
	if glog.V(8) {
		mountCmd.Stdout = os.Stdout
		mountCmd.Stderr = os.Stderr
	}
	if err := mountCmd.Start(); err != nil {
		exit.WithError("Error starting mount", err)
	}
	if err := lock.WriteFile(filepath.Join(localpath.MiniPath(), constants.MountProcessFileName), []byte(strconv.Itoa(mountCmd.Process.Pid)), 0644); err != nil {
		exit.WithError("Error writing mount pid", err)
	}
}
<commit_msg>move replace api server name to apiServerURL<commit_after>/*
Copyright 2020 The Kubernetes Authors All rights reserved.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package node

import (
	"fmt"
	"net"
	"os"
	"os/exec"
	"path/filepath"
	"strconv"
	"strings"

	"github.com/blang/semver"
	"github.com/docker/machine/libmachine"
	"github.com/docker/machine/libmachine/host"
	"github.com/golang/glog"
	"github.com/pkg/errors"
	"github.com/spf13/viper"
	cmdcfg "k8s.io/minikube/cmd/minikube/cmd/config"
	"k8s.io/minikube/pkg/drivers/kic/oci"
	"k8s.io/minikube/pkg/minikube/bootstrapper"
	"k8s.io/minikube/pkg/minikube/cluster"
	"k8s.io/minikube/pkg/minikube/config"
	"k8s.io/minikube/pkg/minikube/constants"
	"k8s.io/minikube/pkg/minikube/cruntime"
	"k8s.io/minikube/pkg/minikube/driver"
	"k8s.io/minikube/pkg/minikube/exit"
	"k8s.io/minikube/pkg/minikube/kubeconfig"
	"k8s.io/minikube/pkg/minikube/localpath"
	"k8s.io/minikube/pkg/minikube/machine"
	"k8s.io/minikube/pkg/minikube/out"
	"k8s.io/minikube/pkg/util/lock"
)

var (
	// DockerEnv contains the environment variables
	DockerEnv []string
	// DockerOpt contains the option parameters
	DockerOpt []string
	// ExtraOptions contains extra options (if any)
	ExtraOptions config.ExtraOptionSlice
	// AddonList contains the list of addons
	AddonList []string
)

// configureRuntimes does what needs to happen to get a runtime going.
func configureRuntimes(runner cruntime.CommandRunner, drvName string, k8s config.KubernetesConfig, kv semver.Version) cruntime.Manager {
	co := cruntime.Config{
		Type:   viper.GetString(containerRuntime),
		Runner: runner, ImageRepository: k8s.ImageRepository,
		KubernetesVersion: kv,
	}
	cr, err := cruntime.New(co)
	if err != nil {
		exit.WithError("Failed runtime", err)
	}

	disableOthers := true
	if driver.BareMetal(drvName) {
		disableOthers = false
	}

	// Preload is overly invasive for bare metal, and caching is not meaningful. KIC handled elsewhere.
	if driver.IsVM(drvName) {
		if err := cr.Preload(k8s); err != nil {
			switch err.(type) {
			case *cruntime.ErrISOFeature:
				out.T(out.Tip, "Existing disk is missing new features ({{.error}}). To upgrade, run 'minikube delete'", out.V{"error": err})
			default:
				glog.Warningf("%s preload failed: %v, falling back to caching images", cr.Name(), err)
			}

			if err := machine.CacheImagesForBootstrapper(k8s.ImageRepository, k8s.KubernetesVersion, viper.GetString(cmdcfg.Bootstrapper)); err != nil {
				exit.WithError("Failed to cache images", err)
			}
		}
	}

	err = cr.Enable(disableOthers)
	if err != nil {
		exit.WithError("Failed to enable container runtime", err)
	}

	return cr
}

func showVersionInfo(k8sVersion string, cr cruntime.Manager) {
	version, _ := cr.Version()
	out.T(cr.Style(), "Preparing Kubernetes {{.k8sVersion}} on {{.runtime}} {{.runtimeVersion}} ...", out.V{"k8sVersion": k8sVersion, "runtime": cr.Name(), "runtimeVersion": version})
	for _, v := range DockerOpt {
		out.T(out.Option, "opt {{.docker_option}}", out.V{"docker_option": v})
	}
	for _, v := range DockerEnv {
		out.T(out.Option, "env {{.docker_env}}", out.V{"docker_env": v})
	}
}

// setupKubeAdm adds any requested files into the VM before Kubernetes is started
func setupKubeAdm(mAPI libmachine.API, cfg config.ClusterConfig, node config.Node) bootstrapper.Bootstrapper {
	bs, err := cluster.Bootstrapper(mAPI, viper.GetString(cmdcfg.Bootstrapper), cfg, node)
	if err != nil {
		exit.WithError("Failed to get bootstrapper", err)
	}
	for _, eo := range ExtraOptions {
		out.T(out.Option, "{{.extra_option_component_name}}.{{.key}}={{.value}}", out.V{"extra_option_component_name": eo.Component, "key": eo.Key, "value": eo.Value})
	}
	// Loads cached images, generates config files, download binaries
	if err := bs.UpdateCluster(cfg); err != nil {
		exit.WithError("Failed to update cluster", err)
	}
	if err := bs.SetupCerts(cfg.KubernetesConfig, node); err != nil {
		exit.WithError("Failed to setup certs", err)
	}
	return bs
}

func setupKubeconfig(h *host.Host, cc *config.ClusterConfig, n *config.Node, clusterName string) (*kubeconfig.Settings, error) {
	addr, err := apiServerURL(*h, *cc, *n)
	if err != nil {
		exit.WithError("Failed to get api server URL", err)
	}

	kcs := &kubeconfig.Settings{
		ClusterName:          clusterName,
		ClusterServerAddress: addr,
		ClientCertificate:    localpath.MakeMiniPath("client.crt"),
		ClientKey:            localpath.MakeMiniPath("client.key"),
		CertificateAuthority: localpath.MakeMiniPath("ca.crt"),
		KeepContext:          viper.GetBool(keepContext),
		EmbedCerts:           viper.GetBool(embedCerts),
	}

	kcs.SetPath(kubeconfig.PathFromEnv())
	if err := kubeconfig.Update(kcs); err != nil {
		return kcs, err
	}
	return kcs, nil
}

// apiServerURL returns a URL to end user can reach to the api server
func apiServerURL(h host.Host, cc config.ClusterConfig, n config.Node) (string, error) {
	if driver.IsKIC(h.DriverName) {
		p, err := oci.HostPortBinding(h.DriverName, h.Name, n.Port)
		if err != nil {
			return "", errors.Wrap(err, "host port binding")
		}
		// for kic drivers we use 127.0.0.1 instead of node IP,
		// because of Docker on MacOs limitations for reaching to container's IP.
		addr := fmt.Sprintf("https://" + net.JoinHostPort(oci.DefaultBindIPV4, strconv.Itoa(p)))
		if cc.KubernetesConfig.APIServerName != constants.APIServerName {
			addr = strings.Replace(addr, oci.DefaultBindIPV4, cc.KubernetesConfig.APIServerName, -1)
		}
		return addr, nil
	}
	ip, err := h.Driver.GetIP()
	if err != nil {
		return "", errors.Wrap(err, "get ip")
	}
	addr := fmt.Sprintf("https://" + net.JoinHostPort(ip, strconv.Itoa(n.Port)))
	if cc.KubernetesConfig.APIServerName != constants.APIServerName {
		addr = strings.Replace(addr, n.IP, cc.KubernetesConfig.APIServerName, -1)
	}
	return addr, nil
}

// configureMounts configures any requested filesystem mounts
func configureMounts() {
	if !viper.GetBool(createMount) {
		return
	}

	out.T(out.Mounting, "Creating mount {{.name}} ...", out.V{"name": viper.GetString(mountString)})
	path := os.Args[0]
	mountDebugVal := 0
	if glog.V(8) {
		mountDebugVal = 1
	}
	mountCmd := exec.Command(path, "mount", fmt.Sprintf("--v=%d", mountDebugVal), viper.GetString(mountString))
	mountCmd.Env = append(os.Environ(), constants.IsMinikubeChildProcess+"=true")
	if glog.V(8) {
		mountCmd.Stdout = os.Stdout
		mountCmd.Stderr = os.Stderr
	}
	if err := mountCmd.Start(); err != nil {
		exit.WithError("Error starting mount", err)
	}
	if err := lock.WriteFile(filepath.Join(localpath.MiniPath(), constants.MountProcessFileName), []byte(strconv.Itoa(mountCmd.Process.Pid)), 0644); err != nil {
		exit.WithError("Error writing mount pid", err)
	}
}
<|endoftext|>


<commit_before>package network

import (
	"errors"
	"fmt"
	"io/ioutil"
	"os"
	"strings"

	"github.com/rancher/os/config"
	httpRetry "github.com/rancher/os/config/cloudinit/pkg"
	"github.com/rancher/os/pkg/log"

	yaml "github.com/cloudfoundry-incubator/candiedyaml"
	composeConfig "github.com/docker/libcompose/config"
)

var (
	ErrNoNetwork = errors.New("Networking not available to load resource")
	ErrNotFound  = errors.New("Failed to find resource")
)

func GetServices(urls []string) ([]string, error) {
	return getServices(urls, "services")
}

func GetConsoles(urls []string) ([]string, error) {
	return getServices(urls, "consoles")
}

func GetEngines(urls []string) ([]string, error) {
	return getServices(urls, "engines")
}

func getServices(urls []string, key string) ([]string, error) {
	result := []string{}

	for _, url := range urls {
		indexURL := fmt.Sprintf("%s/index.yml", url)
		content, err := LoadResource(indexURL, true)
		if err != nil {
			log.Errorf("Failed to load %s: %v", indexURL, err)
			continue
		}

		services := make(map[string][]string)
		err = yaml.Unmarshal(content, &services)
		if err != nil {
			log.Errorf("Failed to unmarshal %s: %v", indexURL, err)
			continue
		}

		if list, ok := services[key]; ok {
			result = append(result, list...)
		}
	}

	return result, nil
}

func SetProxyEnvironmentVariables() {
	cfg := config.LoadConfig()
	if cfg.Rancher.Network.HTTPProxy != "" {
		err := os.Setenv("HTTP_PROXY", cfg.Rancher.Network.HTTPProxy)
		if err != nil {
			log.Errorf("Unable to set HTTP_PROXY: %s", err)
		}
	}
	if cfg.Rancher.Network.HTTPSProxy != "" {
		err := os.Setenv("HTTPS_PROXY", cfg.Rancher.Network.HTTPSProxy)
		if err != nil {
			log.Errorf("Unable to set HTTPS_PROXY: %s", err)
		}
	}
	if cfg.Rancher.Network.NoProxy != "" {
		err := os.Setenv("NO_PROXY", cfg.Rancher.Network.NoProxy)
		if err != nil {
			log.Errorf("Unable to set NO_PROXY: %s", err)
		}
	}
	if cfg.Rancher.Network.HTTPProxy != "" {
		config.Set("rancher.environment.http_proxy", cfg.Rancher.Network.HTTPProxy)
		config.Set("rancher.environment.HTTP_PROXY", cfg.Rancher.Network.HTTPProxy)
	}
	if cfg.Rancher.Network.HTTPSProxy != "" {
		config.Set("rancher.environment.https_proxy", cfg.Rancher.Network.HTTPSProxy)
		config.Set("rancher.environment.HTTPS_PROXY", cfg.Rancher.Network.HTTPSProxy)
	}
	if cfg.Rancher.Network.NoProxy != "" {
		config.Set("rancher.environment.no_proxy", cfg.Rancher.Network.NoProxy)
		config.Set("rancher.environment.NO_PROXY", cfg.Rancher.Network.NoProxy)
	}
}

func LoadFromNetworkWithCache(location string) ([]byte, error) {
	bytes := cacheLookup(location)
	if bytes != nil {
		return bytes, nil
	}
	return LoadFromNetwork(location)
}

func LoadFromNetwork(location string) ([]byte, error) {
	var err error

	if err = AllDefaultGWOK(DefaultRoutesCheckTimeout); err != nil {
		return nil, err
	}
	SetProxyEnvironmentVariables()

	cfg := config.LoadConfig()
	client := httpRetry.NewHTTPClient()
	client.MaxRetries = cfg.Rancher.HTTPLoadRetries
	log.Debugf("start trying LoadFromNetwork(%s)", location)
	bytes, err := client.GetRetry(location)
	if err != nil {
		log.Errorf("failed to LoadFromNetwork: %v", err)
		return nil, err
	}
	log.Debugf("LoadFromNetwork(%s) returned", location)
	cacheAdd(location, bytes)

	return bytes, nil
}

func LoadResource(location string, network bool) ([]byte, error) {
	if strings.HasPrefix(location, "http:/") || strings.HasPrefix(location, "https:/") {
		if !network {
			return nil, ErrNoNetwork
		}
		return LoadFromNetworkWithCache(location)
	} else if strings.HasPrefix(location, "/") {
		return ioutil.ReadFile(location)
	}

	return nil, ErrNotFound
}

func serviceURL(url, name string) string {
	return fmt.Sprintf("%s/%s/%s.yml", url, name[0:1], name)
}

func LoadServiceResource(name string, useNetwork bool, cfg *config.CloudConfig) ([]byte, error) {
	bytes, err := LoadResource(name, useNetwork)
	if err == nil {
		log.Debugf("Loaded %s from %s", name, name)
		return bytes, nil
	}
	if err == ErrNoNetwork || !useNetwork {
		return nil, ErrNoNetwork
	}

	urls := cfg.Rancher.Repositories.ToArray()
	for _, url := range urls {
		serviceURL := serviceURL(url, name)
		bytes, err = LoadResource(serviceURL, useNetwork)
		if err == nil {
			log.Debugf("Loaded %s from %s", name, serviceURL)
			return bytes, nil
		}
	}

	return nil, err
}

func LoadMultiEngineResource(name string) ([]byte, error) {
	composeConfigs := map[string]composeConfig.ServiceConfigV1{}
	if _, err := os.Stat(config.MultiDockerConfFile); err == nil {
		multiEngineBytes, err := ioutil.ReadFile(config.MultiDockerConfFile)
		if err != nil {
			return nil, err
		}
		err = yaml.Unmarshal(multiEngineBytes, &composeConfigs)
		if err != nil {
			return nil, err
		}
	}

	if _, ok := composeConfigs[name]; !ok {
		return nil, errors.New("Failed to found " + name + " from " + config.MultiDockerConfFile + " will load from network")
	}

	foundServiceConfig := map[string]composeConfig.ServiceConfigV1{}
	foundServiceConfig[name] = composeConfigs[name]
	bytes, err := yaml.Marshal(foundServiceConfig)
	if err == nil {
		return bytes, err
	}

	return nil, err
}

func UpdateCaches(urls []string, key string) error {
	for _, url := range urls {
		indexURL := fmt.Sprintf("%s/index.yml", url)
		content, err := UpdateCache(indexURL)
		if err != nil {
			return err
		}

		services := make(map[string][]string)
		err = yaml.Unmarshal(content, &services)
		if err != nil {
			return err
		}

		list := services[key]
		for _, name := range list {
			serviceURL := serviceURL(url, name)
			// no need to handle error
			UpdateCache(serviceURL)
		}
	}
	return nil
}

func UpdateCache(location string) ([]byte, error) {
	// move cache file to temp directory
	tempFile, err := cacheMove(location)
	if err != nil {
		return nil, err
	}

	content, err := LoadResource(location, true)
	if err != nil {
		// move back old cache file
		if err := cacheMoveBack(tempFile); err != nil {
			return nil, err
		}
		return ioutil.ReadFile(location)
	}
	// remove old cache file
	if err := os.Remove(tempFile); err != nil {
		return nil, err
	}

	return content, nil
}
<commit_msg>Prefer to use go dns<commit_after>package network

import (
	"errors"
	"fmt"
	"io/ioutil"
	"net"
	"os"
	"strings"

	"github.com/rancher/os/config"
	httpRetry "github.com/rancher/os/config/cloudinit/pkg"
	"github.com/rancher/os/pkg/log"

	yaml "github.com/cloudfoundry-incubator/candiedyaml"
	composeConfig "github.com/docker/libcompose/config"
)

var (
	ErrNoNetwork = errors.New("Networking not available to load resource")
	ErrNotFound  = errors.New("Failed to find resource")
)

func GetServices(urls []string) ([]string, error) {
	return getServices(urls, "services")
}

func GetConsoles(urls []string) ([]string, error) {
	return getServices(urls, "consoles")
}

func GetEngines(urls []string) ([]string, error) {
	return getServices(urls, "engines")
}

func getServices(urls []string, key string) ([]string, error) {
	result := []string{}

	for _, url := range urls {
		indexURL := fmt.Sprintf("%s/index.yml", url)
		content, err := LoadResource(indexURL, true)
		if err != nil {
			log.Errorf("Failed to load %s: %v", indexURL, err)
			continue
		}

		services := make(map[string][]string)
		err = yaml.Unmarshal(content, &services)
		if err != nil {
			log.Errorf("Failed to unmarshal %s: %v", indexURL, err)
			continue
		}

		if list, ok := services[key]; ok {
			result = append(result, list...)
		}
	}

	return result, nil
}

func SetProxyEnvironmentVariables() {
	cfg := config.LoadConfig()
	if cfg.Rancher.Network.HTTPProxy != "" {
		err := os.Setenv("HTTP_PROXY", cfg.Rancher.Network.HTTPProxy)
		if err != nil {
			log.Errorf("Unable to set HTTP_PROXY: %s", err)
		}
	}
	if cfg.Rancher.Network.HTTPSProxy != "" {
		err := os.Setenv("HTTPS_PROXY", cfg.Rancher.Network.HTTPSProxy)
		if err != nil {
			log.Errorf("Unable to set HTTPS_PROXY: %s", err)
		}
	}
	if cfg.Rancher.Network.NoProxy != "" {
		err := os.Setenv("NO_PROXY", cfg.Rancher.Network.NoProxy)
		if err != nil {
			log.Errorf("Unable to set NO_PROXY: %s", err)
		}
	}
	if cfg.Rancher.Network.HTTPProxy != "" {
		config.Set("rancher.environment.http_proxy", cfg.Rancher.Network.HTTPProxy)
		config.Set("rancher.environment.HTTP_PROXY", cfg.Rancher.Network.HTTPProxy)
	}
	if cfg.Rancher.Network.HTTPSProxy != "" {
		config.Set("rancher.environment.https_proxy", cfg.Rancher.Network.HTTPSProxy)
		config.Set("rancher.environment.HTTPS_PROXY", cfg.Rancher.Network.HTTPSProxy)
	}
	if cfg.Rancher.Network.NoProxy != "" {
		config.Set("rancher.environment.no_proxy", cfg.Rancher.Network.NoProxy)
		config.Set("rancher.environment.NO_PROXY", cfg.Rancher.Network.NoProxy)
	}
}

func LoadFromNetworkWithCache(location string) ([]byte, error) {
	bytes := cacheLookup(location)
	if bytes != nil {
		return bytes, nil
	}
	return LoadFromNetwork(location)
}

func LoadFromNetwork(location string) ([]byte, error) {
	var err error

	if err = AllDefaultGWOK(DefaultRoutesCheckTimeout); err != nil {
		return nil, err
	}
	SetProxyEnvironmentVariables()

	net.DefaultResolver.PreferGo = true
	cfg := config.LoadConfig()
	client := httpRetry.NewHTTPClient()
	client.MaxRetries = cfg.Rancher.HTTPLoadRetries
	log.Debugf("start trying LoadFromNetwork(%s)", location)
	bytes, err := client.GetRetry(location)
	if err != nil {
		log.Errorf("failed to LoadFromNetwork: %v", err)
		return nil, err
	}
	log.Debugf("LoadFromNetwork(%s) returned", location)
	cacheAdd(location, bytes)

	return bytes, nil
}

func LoadResource(location string, network bool) ([]byte, error) {
	if strings.HasPrefix(location, "http:/") || strings.HasPrefix(location, "https:/") {
		if !network {
			return nil, ErrNoNetwork
		}
		return LoadFromNetworkWithCache(location)
	} else if strings.HasPrefix(location, "/") {
		return ioutil.ReadFile(location)
	}

	return nil, ErrNotFound
}

func serviceURL(url, name string) string {
	return fmt.Sprintf("%s/%s/%s.yml", url, name[0:1], name)
}

func LoadServiceResource(name string, useNetwork bool, cfg *config.CloudConfig) ([]byte, error) {
	bytes, err := LoadResource(name, useNetwork)
	if err == nil {
		log.Debugf("Loaded %s from %s", name, name)
		return bytes, nil
	}
	if err == ErrNoNetwork || !useNetwork {
		return nil, ErrNoNetwork
	}

	urls := cfg.Rancher.Repositories.ToArray()
	for _, url := range urls {
		serviceURL := serviceURL(url, name)
		bytes, err = LoadResource(serviceURL, useNetwork)
		if err == nil {
			log.Debugf("Loaded %s from %s", name, serviceURL)
			return bytes, nil
		}
	}

	return nil, err
}

func LoadMultiEngineResource(name string) ([]byte, error) {
	composeConfigs := map[string]composeConfig.ServiceConfigV1{}
	if _, err := os.Stat(config.MultiDockerConfFile); err == nil {
		multiEngineBytes, err := ioutil.ReadFile(config.MultiDockerConfFile)
		if err != nil {
			return nil, err
		}
		err = yaml.Unmarshal(multiEngineBytes, &composeConfigs)
		if err != nil {
			return nil, err
		}
	}

	if _, ok := composeConfigs[name]; !ok {
		return nil, errors.New("Failed to found " + name + " from " + config.MultiDockerConfFile + " will load from network")
	}

	foundServiceConfig := map[string]composeConfig.ServiceConfigV1{}
	foundServiceConfig[name] = composeConfigs[name]
	bytes, err := yaml.Marshal(foundServiceConfig)
	if err == nil {
		return bytes, err
	}

	return nil, err
}

func UpdateCaches(urls []string, key string) error {
	for _, url := range urls {
		indexURL := fmt.Sprintf("%s/index.yml", url)
		content, err := UpdateCache(indexURL)
		if err != nil {
			return err
		}

		services := make(map[string][]string)
		err = yaml.Unmarshal(content, &services)
		if err != nil {
			return err
		}

		list := services[key]
		for _, name := range list {
			serviceURL := serviceURL(url, name)
			// no need to handle error
			UpdateCache(serviceURL)
		}
	}
	return nil
}

func UpdateCache(location string) ([]byte, error) {
	// move cache file to temp directory
	tempFile, err := cacheMove(location)
	if err != nil {
		return nil, err
	}

	content, err := LoadResource(location, true)
	if err != nil {
		// move back old cache file
		if err := cacheMoveBack(tempFile); err != nil {
			return nil, err
		}
		return ioutil.ReadFile(location)
	}
	// remove old cache file
	if err := os.Remove(tempFile); err != nil {
		return nil, err
	}

	return content, nil
}
<|endoftext|>


<commit_before>// Copyright 2016 fatedier, fatedier@gmail.com
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package version

import (
	"strconv"
	"strings"
)

var version string = "0.34.1"

func Full() string {
	return version
}

func getSubVersion(v string, position int) int64 {
	arr := strings.Split(v, ".")
	if len(arr) < 3 {
		return 0
	}
	res, _ := strconv.ParseInt(arr[position], 10, 64)
	return res
}

func Proto(v string) int64 {
	return getSubVersion(v, 0)
}

func Major(v string) int64 {
	return getSubVersion(v, 1)
}

func Minor(v string) int64 {
	return getSubVersion(v, 2)
}

// add every case there if server will not accept client's protocol and return false
func Compat(client string) (ok bool, msg string) {
	if LessThan(client, "0.18.0") {
		return false, "Please upgrade your frpc version to at least 0.18.0"
	}
	return true, ""
}

func LessThan(client string, server string) bool {
	vc := Proto(client)
	vs := Proto(server)
	if vc > vs {
		return false
	} else if vc < vs {
		return true
	}

	vc = Major(client)
	vs = Major(server)
	if vc > vs {
		return false
	} else if vc < vs {
		return true
	}

	vc = Minor(client)
	vs = Minor(server)
	if vc > vs {
		return false
	} else if vc < vs {
		return true
	}
	return false
}
<commit_msg>update version<commit_after>// Copyright 2016 fatedier, fatedier@gmail.com
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package version

import (
	"strconv"
	"strings"
)

var version string = "0.34.2"

func Full() string {
	return version
}

func getSubVersion(v string, position int) int64 {
	arr := strings.Split(v, ".")
	if len(arr) < 3 {
		return 0
	}
	res, _ := strconv.ParseInt(arr[position], 10, 64)
	return res
}

func Proto(v string) int64 {
	return getSubVersion(v, 0)
}

func Major(v string) int64 {
	return getSubVersion(v, 1)
}

func Minor(v string) int64 {
	return getSubVersion(v, 2)
}

// add every case there if server will not accept client's protocol and return false
func Compat(client string) (ok bool, msg string) {
	if LessThan(client, "0.18.0") {
		return false, "Please upgrade your frpc version to at least 0.18.0"
	}
	return true, ""
}

func LessThan(client string, server string) bool {
	vc := Proto(client)
	vs := Proto(server)
	if vc > vs {
		return false
	} else if vc < vs {
		return true
	}

	vc = Major(client)
	vs = Major(server)
	if vc > vs {
		return false
	} else if vc < vs {
		return true
	}

	vc = Minor(client)
	vs = Minor(server)
	if vc > vs {
		return false
	} else if vc < vs {
		return true
	}
	return false
}
<|endoftext|>


<commit_before>/**
 * Copyright 2022 Google LLC
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 *      http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless assertd by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */

// Package golden helps manage goldenfiles.
package golden

import (
	"fmt"
	"io/ioutil"
	"os"
	"path"
	"strings"

	"github.com/GoogleCloudPlatform/cloud-foundation-toolkit/infra/blueprint-test/pkg/gcloud"
	"github.com/GoogleCloudPlatform/cloud-foundation-toolkit/infra/blueprint-test/pkg/utils"
	"github.com/mitchellh/go-testing-interface"
	"github.com/stretchr/testify/assert"
	"github.com/tidwall/gjson"
)

const (
	gfDir          = "testdata"
	gfPerms        = 0755
	gfUpdateEnvVar = "UPDATE_GOLDEN"
)

type GoldenFile struct {
	dir        string
	fileName   string
	sanitizers []Sanitizer
	t          testing.TB
}

type Sanitizer func(string) string

// StringSanitizer replaces all occurrences of old string with new string
func StringSanitizer(old, new string) Sanitizer {
	return func(s string) string {
		return strings.ReplaceAll(s, old, new)
	}
}

// ProjectIDSanitizer replaces all occurrences of current gcloud project ID with PROJECT_ID string
func ProjectIDSanitizer(t testing.TB) Sanitizer {
	return func(s string) string {
		projectID := gcloud.Run(t, "config get-value project")
		if projectID.String() == "[]" {
			t.Logf("no project ID currently set, skipping ProjectIDSanitizer: %s", projectID.String())
			return s
		}
		return strings.ReplaceAll(s, projectID.String(), "PROJECT_ID")
	}
}

type goldenFileOption func(*GoldenFile)

func WithDir(dir string) goldenFileOption {
	return func(g *GoldenFile) {
		g.dir = dir
	}
}

func WithFileName(fn string) goldenFileOption {
	return func(g *GoldenFile) {
		g.fileName = fn
	}
}

func WithSanitizer(s Sanitizer) goldenFileOption {
	return func(g *GoldenFile) {
		g.sanitizers = append(g.sanitizers, s)
	}
}

func NewOrUpdate(t testing.TB, data string, opts ...goldenFileOption) *GoldenFile {
	g := &GoldenFile{
		dir:        gfDir,
		fileName:   fmt.Sprintf("%s.json", strings.ReplaceAll(t.Name(), "/", "-")),
		sanitizers: []Sanitizer{ProjectIDSanitizer(t)},
		t:          t,
	}
	for _, opt := range opts {
		opt(g)
	}
	g.update(data)
	return g
}

// update updates goldenfile data iff gfUpdateEnvVar is true
func (g *GoldenFile) update(data string) {
	// exit early if gfUpdateEnvVar is not set or true
	if strings.ToLower(os.Getenv(gfUpdateEnvVar)) != "true" {
		return
	}
	fp := g.GetName()
	err := os.MkdirAll(path.Dir(fp), gfPerms)
	if err != nil {
		g.t.Fatalf("error updating result: %v", err)
	}
	// apply sanitizers on data
	data = g.ApplySanitizers(data)

	err = ioutil.WriteFile(fp, []byte(data), gfPerms)
	if err != nil {
		g.t.Fatalf("error updating result: %v", err)
	}
}

// GetName return path of the goldenfile
func (g *GoldenFile) GetName() string {
	return path.Join(g.dir, g.fileName)
}

// ApplySanitizers returns sanitized string
func (g *GoldenFile) ApplySanitizers(s string) string {
	for _, sanitizer := range g.sanitizers {
		s = sanitizer(s)
	}
	return s
}

// GetSanitizedJSON returns sanitizes and returns JSON result
func (g *GoldenFile) GetSanitizedJSON(s gjson.Result) gjson.Result {
	resultStr := s.String()
	resultStr = g.ApplySanitizers(resultStr)
	return utils.ParseJSONResult(g.t, resultStr)
}

// GetJSON returns goldenfile as parsed json
func (g *GoldenFile) GetJSON() gjson.Result {
	return utils.LoadJSON(g.t, g.GetName())
}

// JSONEq asserts that json content in jsonPath for got and goldenfile is the same
func (g *GoldenFile) JSONEq(a *assert.Assertions, got gjson.Result, jsonPath string) {
	gf := g.GetJSON()
	gotData := g.ApplySanitizers(got.Get(jsonPath).String())
	gfData := gf.Get(jsonPath).String()
	a.Equal(gfData, gotData, fmt.Sprintf("expected %s to match fixture %s", jsonPath, gfData))
}
<commit_msg>feat: add golden string sanitizer option (#1109)<commit_after>/**
 * Copyright 2022 Google LLC
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 *      http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless assertd by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */

// Package golden helps manage goldenfiles.
package golden

import (
	"fmt"
	"io/ioutil"
	"os"
	"path"
	"strings"

	"github.com/GoogleCloudPlatform/cloud-foundation-toolkit/infra/blueprint-test/pkg/gcloud"
	"github.com/GoogleCloudPlatform/cloud-foundation-toolkit/infra/blueprint-test/pkg/utils"
	"github.com/mitchellh/go-testing-interface"
	"github.com/stretchr/testify/assert"
	"github.com/tidwall/gjson"
)

const (
	gfDir          = "testdata"
	gfPerms        = 0755
	gfUpdateEnvVar = "UPDATE_GOLDEN"
)

type GoldenFile struct {
	dir        string
	fileName   string
	sanitizers []Sanitizer
	t          testing.TB
}

type Sanitizer func(string) string

// StringSanitizer replaces all occurrences of old string with new string
func StringSanitizer(old, new string) Sanitizer {
	return func(s string) string {
		return strings.ReplaceAll(s, old, new)
	}
}

// ProjectIDSanitizer replaces all occurrences of current gcloud project ID with PROJECT_ID string
func ProjectIDSanitizer(t testing.TB) Sanitizer {
	return func(s string) string {
		projectID := gcloud.Run(t, "config get-value project")
		if projectID.String() == "[]" {
			t.Logf("no project ID currently set, skipping ProjectIDSanitizer: %s", projectID.String())
			return s
		}
		return strings.ReplaceAll(s, projectID.String(), "PROJECT_ID")
	}
}

type goldenFileOption func(*GoldenFile)

func WithDir(dir string) goldenFileOption {
	return func(g *GoldenFile) {
		g.dir = dir
	}
}

func WithFileName(fn string) goldenFileOption {
	return func(g *GoldenFile) {
		g.fileName = fn
	}
}

func WithSanitizer(s Sanitizer) goldenFileOption {
	return func(g *GoldenFile) {
		g.sanitizers = append(g.sanitizers, s)
	}
}

func WithStringSanitizer(old, new string) goldenFileOption {
	return func(g *GoldenFile) {
		g.sanitizers = append(g.sanitizers, StringSanitizer(old, new))
	}
}

func NewOrUpdate(t testing.TB, data string, opts ...goldenFileOption) *GoldenFile {
	g := &GoldenFile{
		dir:        gfDir,
		fileName:   fmt.Sprintf("%s.json", strings.ReplaceAll(t.Name(), "/", "-")),
		sanitizers: []Sanitizer{ProjectIDSanitizer(t)},
		t:          t,
	}
	for _, opt := range opts {
		opt(g)
	}
	g.update(data)
	return g
}

// update updates goldenfile data iff gfUpdateEnvVar is true
func (g *GoldenFile) update(data string) {
	// exit early if gfUpdateEnvVar is not set or true
	if strings.ToLower(os.Getenv(gfUpdateEnvVar)) != "true" {
		return
	}
	fp := g.GetName()
	err := os.MkdirAll(path.Dir(fp), gfPerms)
	if err != nil {
		g.t.Fatalf("error updating result: %v", err)
	}
	// apply sanitizers on data
	data = g.ApplySanitizers(data)

	err = ioutil.WriteFile(fp, []byte(data), gfPerms)
	if err != nil {
		g.t.Fatalf("error updating result: %v", err)
	}
}

// GetName return path of the goldenfile
func (g *GoldenFile) GetName() string {
	return path.Join(g.dir, g.fileName)
}

// ApplySanitizers returns sanitized string
func (g *GoldenFile) ApplySanitizers(s string) string {
	for _, sanitizer := range g.sanitizers {
		s = sanitizer(s)
	}
	return s
}

// GetSanitizedJSON returns sanitizes and returns JSON result
func (g *GoldenFile) GetSanitizedJSON(s gjson.Result) gjson.Result {
	resultStr := s.String()
	resultStr = g.ApplySanitizers(resultStr)
	return utils.ParseJSONResult(g.t, resultStr)
}

// GetJSON returns goldenfile as parsed json
func (g *GoldenFile) GetJSON() gjson.Result {
	return utils.LoadJSON(g.t, g.GetName())
}

// JSONEq asserts that json content in jsonPath for got and goldenfile is the same
func (g *GoldenFile) JSONEq(a *assert.Assertions, got gjson.Result, jsonPath string) {
	gf := g.GetJSON()
	gotData := g.ApplySanitizers(got.Get(jsonPath).String())
	gfData := gf.Get(jsonPath).String()
	a.Equal(gfData, gotData, fmt.Sprintf("expected %s to match fixture %s", jsonPath, gfData))
}
<|endoftext|>