<commit_before>package helper

import (
	"math"
	"time"

	"github.com/go-graphite/carbonapi/expr/types"
)

// GCD returns greatest common divisor calculated via Euclidean algorithm
func GCD(a, b int64) int64 {
	for b != 0 {
		t := b
		b = a % b
		a = t
	}
	return a
}

// LCM returns the least common multiple of 2 or more integers via GDB
func LCM(args ...int64) int64 {
	if len(args) <= 1 {
		if len(args) == 0 {
			return 0
		}
		return args[0]
	}
	lcm := args[0] / GCD(args[0], args[1]) * args[1]

	for i := 2; i < len(args); i++ {
		lcm = LCM(lcm, args[i])
	}
	return lcm
}

// GetCommonStep returns LCM(steps) for slice of metrics.
// minStart and maxStop will be set to closest lower or equal multiple of LCM(steps).
func GetCommonStep(args []*types.MetricData) int64 {
	steps := make([]int64, 0, len(args))
	for _, arg := range args {
		steps = append(steps, arg.StepTime)
	}
	commonStep := LCM(steps...)
	return commonStep
}

// GetBuckets returns amount buckets for timeSeries (defined with startTime, stopTime and step (bucket) size.
func GetBuckets(start, stop, bucketSize int64) int64 {
	return int64(math.Ceil(float64(stop-start) / float64(bucketSize)))
}

// AlignStartToInterval aligns start of serie to interval
func AlignStartToInterval(start, stop, bucketSize int64) int64 {
	for _, v := range []int64{86400, 3600, 60} {
		if bucketSize >= v {
			start -= start % v
			break
		}
	}

	return start
}

// AlignToBucketSize aligns start and stop of serie to specified bucket (step) size
func AlignToBucketSize(start, stop, bucketSize int64) (int64, int64) {
	start = time.Unix(start, 0).Truncate(time.Duration(bucketSize) * time.Second).Unix()
	newStop := time.Unix(stop, 0).Truncate(time.Duration(bucketSize) * time.Second).Unix()

	// check if a partial bucket is needed
	if stop != newStop {
		newStop += bucketSize
	}

	return start, newStop
}

// AlignSeries aligns different series together. By default it only prepends and appends NaNs in case of different length, but if ExtrapolatePoints is enabled, it can extrapolate
func AlignSeries(args []*types.MetricData) []*types.MetricData {
	minStart := args[0].StartTime
	maxStop := args[0].StopTime
	maxVals := 0
	minStepTime := args[0].StepTime
	for j := 0; j < 2; j++ {
		if ExtrapolatePoints {
			for _, arg := range args {
				if arg.StepTime < minStepTime {
					minStepTime = arg.StepTime
				}

				if arg.StepTime > minStepTime {
					valsCnt := int(math.Ceil(float64(arg.StopTime-arg.StartTime) / float64(minStepTime)))
					newVals := make([]float64, valsCnt)
					ts := arg.StartTime
					nextTs := arg.StartTime + arg.StepTime
					i := 0
					j := 0
					pointsPerInterval := float64(ts-nextTs) / float64(minStepTime)
					v := arg.Values[0]
					dv := (arg.Values[0] - arg.Values[1]) / pointsPerInterval
					for ts < arg.StopTime {
						newVals[i] = v
						v += dv
						if ts > nextTs {
							j++
							nextTs += arg.StepTime
							v = arg.Values[j]
							dv = (arg.Values[j-1] - v) / pointsPerInterval
						}
						ts += minStepTime
						i++
					}
					arg.Values = newVals
					arg.StepTime = minStepTime
				}
			}
		}

		for _, arg := range args {
			if len(arg.Values) > maxVals {
				maxVals = len(arg.Values)
			}
			if arg.StartTime < minStart {
				minStart = arg.StartTime
			}
			if minStart < arg.StartTime {
				valCnt := (arg.StartTime - minStart) / arg.StepTime
				newVals := make([]float64, valCnt)
				newVals = append(newVals, arg.Values...)
				arg.Values = newVals
				arg.StartTime = minStart
			}

			if arg.StopTime > maxStop {
				maxStop = arg.StopTime
			}
			if maxStop > arg.StopTime {
				valCnt := (maxStop - arg.StopTime) / arg.StepTime
				newVals := make([]float64, valCnt)
				arg.Values = append(arg.Values, newVals...)
				arg.StopTime = maxStop
			}
		}
	}
	return args
}
<commit_msg>Generate NaNs instead of zeros in AlignSeries<commit_after>package helper

import (
	"math"
	"time"

	"github.com/go-graphite/carbonapi/expr/types"
)

// GCD returns greatest common divisor calculated via Euclidean algorithm
func GCD(a, b int64) int64 {
	for b != 0 {
		t := b
		b = a % b
		a = t
	}
	return a
}

// LCM returns the least common multiple of 2 or more integers via GDB
func LCM(args ...int64) int64 {
	if len(args) <= 1 {
		if len(args) == 0 {
			return 0
		}
		return args[0]
	}
	lcm := args[0] / GCD(args[0], args[1]) * args[1]

	for i := 2; i < len(args); i++ {
		lcm = LCM(lcm, args[i])
	}
	return lcm
}

// GetCommonStep returns LCM(steps) for slice of metrics.
// minStart and maxStop will be set to closest lower or equal multiple of LCM(steps).
func GetCommonStep(args []*types.MetricData) int64 {
	steps := make([]int64, 0, len(args))
	for _, arg := range args {
		steps = append(steps, arg.StepTime)
	}
	commonStep := LCM(steps...)
	return commonStep
}

// GetBuckets returns amount buckets for timeSeries (defined with startTime, stopTime and step (bucket) size.
func GetBuckets(start, stop, bucketSize int64) int64 {
	return int64(math.Ceil(float64(stop-start) / float64(bucketSize)))
}

// AlignStartToInterval aligns start of serie to interval
func AlignStartToInterval(start, stop, bucketSize int64) int64 {
	for _, v := range []int64{86400, 3600, 60} {
		if bucketSize >= v {
			start -= start % v
			break
		}
	}

	return start
}

// AlignToBucketSize aligns start and stop of serie to specified bucket (step) size
func AlignToBucketSize(start, stop, bucketSize int64) (int64, int64) {
	start = time.Unix(start, 0).Truncate(time.Duration(bucketSize) * time.Second).Unix()
	newStop := time.Unix(stop, 0).Truncate(time.Duration(bucketSize) * time.Second).Unix()

	// check if a partial bucket is needed
	if stop != newStop {
		newStop += bucketSize
	}

	return start, newStop
}

// AlignSeries aligns different series together. By default it only prepends and appends NaNs in case of different length, but if ExtrapolatePoints is enabled, it can extrapolate
func AlignSeries(args []*types.MetricData) []*types.MetricData {
	minStart := args[0].StartTime
	maxStop := args[0].StopTime
	maxVals := 0
	minStepTime := args[0].StepTime
	for j := 0; j < 2; j++ {
		if ExtrapolatePoints {
			for _, arg := range args {
				if arg.StepTime < minStepTime {
					minStepTime = arg.StepTime
				}

				if arg.StepTime > minStepTime {
					valsCnt := int(math.Ceil(float64(arg.StopTime-arg.StartTime) / float64(minStepTime)))
					newVals := make([]float64, valsCnt)
					ts := arg.StartTime
					nextTs := arg.StartTime + arg.StepTime
					i := 0
					j := 0
					pointsPerInterval := float64(ts-nextTs) / float64(minStepTime)
					v := arg.Values[0]
					dv := (arg.Values[0] - arg.Values[1]) / pointsPerInterval
					for ts < arg.StopTime {
						newVals[i] = v
						v += dv
						if ts > nextTs {
							j++
							nextTs += arg.StepTime
							v = arg.Values[j]
							dv = (arg.Values[j-1] - v) / pointsPerInterval
						}
						ts += minStepTime
						i++
					}
					arg.Values = newVals
					arg.StepTime = minStepTime
				}
			}
		}

		for _, arg := range args {
			if len(arg.Values) > maxVals {
				maxVals = len(arg.Values)
			}
			if arg.StartTime < minStart {
				minStart = arg.StartTime
			}
			if minStart < arg.StartTime {
				valCnt := (arg.StartTime - minStart) / arg.StepTime
				newVals := genNaNs(int(valCnt))
				arg.Values = append(newVals, arg.Values...)
				arg.StartTime = minStart
			}

			if arg.StopTime > maxStop {
				maxStop = arg.StopTime
			}
			if maxStop > arg.StopTime {
				valCnt := (maxStop - arg.StopTime) / arg.StepTime
				newVals := genNaNs(int(valCnt))
				arg.Values = append(arg.Values, newVals...)
				arg.StopTime = maxStop
			}
		}
	}
	return args
}

func genNaNs(length int) []float64 {
	nans := make([]float64, length)
	for i := range nans {
		nans[i] = math.NaN()
	}
	return nans
}
<|endoftext|>


<commit_before>// Simple utility to convert a file into a Go byte array

// Clint Caywood

// http://github.com/cratonica/2goarray
package main

import (
	"fmt"
	"github.com/codegangsta/cli"
	"io"
	"os"
)

const (
	NAME         = "2goarray"
	VERSION      = "0.2.0"
	URL          = "http://github.com/cratonica/2goarray"
	GENERATED_BY = "// File generated by " + NAME + " v" + VERSION + " (" + URL + ")"
)

func main() {
	// the commandline interface
	app := cli.NewApp()
	app.Name = NAME
	app.Version = VERSION
	app.Author = NAME + " contributors"
	app.Email = URL + "/graphs/contributors"
	app.Usage = "A simple utility to encode a file into a Go byte array"
	app.Flags = []cli.Flag{
		cli.StringFlag{
			Name:  "package-name, p",
			Value: "main",
			Usage: "name of the package",
		},
		cli.StringFlag{
			Name:  "array-name, a",
			Value: "DATA",
			Usage: "name of the byte array",
		},
		cli.BoolFlag{
			Name:  "no-generated-info, ni",
			Usage: "no 'generated by' info at the first code line",
		},
		cli.BoolFlag{
			Name:  "no-package, np",
			Usage: "no 'package' code",
		},
	}
	app.Action = func(c *cli.Context) {
		if isTerminal() {
			fmt.Println("\nPlease pipe the file you wish to encode into stdin\n")
			return
		}

		// get the cli flags
		packageName := c.String("package-name")
		arrayName := c.String("array-name")
		noGeneratedInfo := c.Bool("no-generated-info")
		noPackage := c.Bool("no-package")

		// print out the code
		if !noGeneratedInfo {
			fmt.Println(GENERATED_BY + "\n")
		}
		if !noPackage {
			fmt.Printf("package %s\n\n", packageName)
		}
		fmt.Printf("var %s []byte = []byte{", arrayName)
		buf := make([]byte, 1)
		var err error
		var totalBytes uint64
		var n int
		for n, err = os.Stdin.Read(buf); n > 0 && err == nil; {
			if totalBytes%12 == 0 {
				fmt.Printf("\n\t")
			}
			fmt.Printf("0x%02x, ", buf[0])
			totalBytes++
			n, err = os.Stdin.Read(buf)
		}
		if err != nil && err != io.EOF {
			fmt.Errorf("Error: %v", err)
		}
		fmt.Print("\n}\n\n")

	}
	app.Run(os.Args)
}
<commit_msg>add input, output flag<commit_after>// Simple utility to convert a file into a Go byte array

// Clint Caywood

// http://github.com/cratonica/2goarray
package main

import (
	"fmt"
	"github.com/codegangsta/cli"
	"io"
	"os"
)

const (
	NAME         = "2goarray"
	VERSION      = "0.2.0"
	URL          = "http://github.com/cratonica/2goarray"
	GENERATED_BY = "// File generated by " + NAME + " v" + VERSION + " (" + URL + ")"
)

func main() {
	// the commandline interface
	app := cli.NewApp()
	app.Name = NAME
	app.Version = VERSION
	app.Author = NAME + " contributors"
	app.Email = URL + "/graphs/contributors"
	app.Usage = "A simple utility to encode a file into a Go byte array"
	app.Flags = []cli.Flag{
		cli.StringFlag{
			Name:  "input, i",
			Value: "",
			Usage: "path to the input json file",
		},
		cli.StringFlag{
			Name:  "output, o",
			Value: "",
			Usage: "filepath to write to the generated source",
		},
		cli.StringFlag{
			Name:  "package-name, p",
			Value: "main",
			Usage: "name of the package",
		},
		cli.StringFlag{
			Name:  "array-name, a",
			Value: "DATA",
			Usage: "name of the byte array",
		},
		cli.BoolFlag{
			Name:  "no-generated-info, ni",
			Usage: "no 'generated by' info at the first code line",
		},
		cli.BoolFlag{
			Name:  "no-package, np",
			Usage: "no 'package' code",
		},
	}
	app.Action = func(c *cli.Context) {
		if isTerminal() {
			fmt.Println("\nPlease pipe the file you wish to encode into stdin\n")
			return
		}

		// get the cli flags
		packageName := c.String("package-name")
		arrayName := c.String("array-name")
		noGeneratedInfo := c.Bool("no-generated-info")
		noPackage := c.Bool("no-package")

		// print out the code
		if !noGeneratedInfo {
			fmt.Println(GENERATED_BY + "\n")
		}
		if !noPackage {
			fmt.Printf("package %s\n\n", packageName)
		}
		fmt.Printf("var %s []byte = []byte{", arrayName)
		buf := make([]byte, 1)
		var err error
		var totalBytes uint64
		var n int
		for n, err = os.Stdin.Read(buf); n > 0 && err == nil; {
			if totalBytes%12 == 0 {
				fmt.Printf("\n\t")
			}
			fmt.Printf("0x%02x, ", buf[0])
			totalBytes++
			n, err = os.Stdin.Read(buf)
		}
		if err != nil && err != io.EOF {
			fmt.Errorf("Error: %v", err)
		}
		fmt.Print("\n}\n\n")

	}
	app.Run(os.Args)
}
<|endoftext|>


<commit_before>/*
Copyright 2020 The Kubernetes Authors All rights reserved.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package download

import (
	"context"
	"crypto/md5"
	"fmt"
	"io/ioutil"
	"net/http"
	"os"
	"path/filepath"
	"runtime"

	"cloud.google.com/go/storage"
	"google.golang.org/api/option"

	"github.com/golang/glog"
	"github.com/pkg/errors"
	"github.com/spf13/viper"
	"k8s.io/minikube/pkg/minikube/localpath"
	"k8s.io/minikube/pkg/minikube/out"
)

const (
	// PreloadVersion is the current version of the preloaded tarball
	//
	// NOTE: You may need to bump this version up when upgrading auxiliary docker images
	PreloadVersion = "v4"
	// PreloadBucket is the name of the GCS bucket where preloaded volume tarballs exist
	PreloadBucket = "minikube-preloaded-volume-tarballs"
)

// TarballName returns name of the tarball
func TarballName(k8sVersion, containerRuntime string) string {
	var storageDriver string
	if containerRuntime == "cri-o" {
		storageDriver = "overlay"
	} else {
		storageDriver = "overlay2"
	}
	return fmt.Sprintf("preloaded-images-k8s-%s-%s-%s-%s-%s.tar.lz4", PreloadVersion, k8sVersion, containerRuntime, storageDriver, runtime.GOARCH)
}

// returns the name of the checksum file
func checksumName(k8sVersion, containerRuntime string) string {
	return fmt.Sprintf("%s.checksum", TarballName(k8sVersion, containerRuntime))
}

// returns target dir for all cached items related to preloading
func targetDir() string {
	return localpath.MakeMiniPath("cache", "preloaded-tarball")
}

// PreloadChecksumPath returns the local path to the cached checksum file
func PreloadChecksumPath(k8sVersion, containerRuntime string) string {
	return filepath.Join(targetDir(), checksumName(k8sVersion, containerRuntime))
}

// TarballPath returns the local path to the cached preload tarball
func TarballPath(k8sVersion, containerRuntime string) string {
	return filepath.Join(targetDir(), TarballName(k8sVersion, containerRuntime))
}

// remoteTarballURL returns the URL for the remote tarball in GCS
func remoteTarballURL(k8sVersion, containerRuntime string) string {
	return fmt.Sprintf("https://storage.googleapis.com/%s/%s", PreloadBucket, TarballName(k8sVersion, containerRuntime))
}

// PreloadExists returns true if there is a preloaded tarball that can be used
func PreloadExists(k8sVersion, containerRuntime string, forcePreload ...bool) bool {

	// and https://github.com/kubernetes/minikube/issues/6934
	// to track status of adding crio
	if containerRuntime == "crio" {
		glog.Info("crio is not supported yet, skipping preload")
		return false
	}

	// TODO (#8166): Get rid of the need for this and viper at all
	force := false
	if len(forcePreload) > 0 {
		force = forcePreload[0]
	}

	// TODO: debug why this func is being called two times
	glog.Infof("Checking if preload exists for k8s version %s and runtime %s", k8sVersion, containerRuntime)
	if !viper.GetBool("preload") && !force {
		return false
	}

	// Omit remote check if tarball exists locally
	targetPath := TarballPath(k8sVersion, containerRuntime)
	if _, err := os.Stat(targetPath); err == nil {
		glog.Infof("Found local preload: %s", targetPath)
		return true
	}

	url := remoteTarballURL(k8sVersion, containerRuntime)
	resp, err := http.Head(url)
	if err != nil {
		glog.Warningf("%s fetch error: %v", url, err)
		return false
	}

	// note: err won't be set if it's a 404
	if resp.StatusCode != 200 {
		glog.Warningf("%s status code: %d", url, resp.StatusCode)
		return false
	}

	glog.Infof("Found remote preload: %s", url)
	return true
}

// Preload caches the preloaded images tarball on the host machine
func Preload(k8sVersion, containerRuntime string) error {
	targetPath := TarballPath(k8sVersion, containerRuntime)

	if _, err := os.Stat(targetPath); err == nil {
		glog.Infof("Found %s in cache, skipping download", targetPath)
		return nil
	}

	// Make sure we support this k8s version
	if !PreloadExists(k8sVersion, containerRuntime) {
		glog.Infof("Preloaded tarball for k8s version %s does not exist", k8sVersion)
		return nil
	}

	out.T(out.FileDownload, "Downloading Kubernetes {{.version}} preload ...", out.V{"version": k8sVersion})
	url := remoteTarballURL(k8sVersion, containerRuntime)

	if err := download(url, targetPath); err != nil {
		return errors.Wrapf(err, "download failed: %s", url)
	}

	if err := saveChecksumFile(k8sVersion, containerRuntime); err != nil {
		return errors.Wrap(err, "saving checksum file")
	}

	if err := verifyChecksum(k8sVersion, containerRuntime, targetPath); err != nil {
		return errors.Wrap(err, "verify")
	}

	return nil
}

func saveChecksumFile(k8sVersion, containerRuntime string) error {
	glog.Infof("saving checksum for %s ...", TarballName(k8sVersion, containerRuntime))
	ctx := context.Background()
	client, err := storage.NewClient(ctx, option.WithoutAuthentication())
	if err != nil {
		return errors.Wrap(err, "getting storage client")
	}
	attrs, err := client.Bucket(PreloadBucket).Object(TarballName(k8sVersion, containerRuntime)).Attrs(ctx)
	if err != nil {
		return errors.Wrap(err, "getting storage object")
	}
	checksum := attrs.MD5
	return ioutil.WriteFile(PreloadChecksumPath(k8sVersion, containerRuntime), checksum, 0644)
}

// verifyChecksum returns true if the checksum of the local binary matches
// the checksum of the remote binary
func verifyChecksum(k8sVersion, containerRuntime, path string) error {
	glog.Infof("verifying checksumm of %s ...", path)
	// get md5 checksum of tarball path
	contents, err := ioutil.ReadFile(path)
	if err != nil {
		return errors.Wrap(err, "reading tarball")
	}
	checksum := md5.Sum(contents)

	remoteChecksum, err := ioutil.ReadFile(PreloadChecksumPath(k8sVersion, containerRuntime))
	if err != nil {
		return errors.Wrap(err, "reading checksum file")
	}

	// create a slice of checksum, which is [16]byte
	if string(remoteChecksum) != string(checksum[:]) {
		return fmt.Errorf("checksum of %s does not match remote checksum (%s != %s)", path, string(remoteChecksum), string(checksum[:]))
	}
	return nil
}
<commit_msg>Normalize name of the container runtime param<commit_after>/*
Copyright 2020 The Kubernetes Authors All rights reserved.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package download

import (
	"context"
	"crypto/md5"
	"fmt"
	"io/ioutil"
	"net/http"
	"os"
	"path/filepath"
	"runtime"

	"cloud.google.com/go/storage"
	"google.golang.org/api/option"

	"github.com/golang/glog"
	"github.com/pkg/errors"
	"github.com/spf13/viper"
	"k8s.io/minikube/pkg/minikube/localpath"
	"k8s.io/minikube/pkg/minikube/out"
)

const (
	// PreloadVersion is the current version of the preloaded tarball
	//
	// NOTE: You may need to bump this version up when upgrading auxiliary docker images
	PreloadVersion = "v4"
	// PreloadBucket is the name of the GCS bucket where preloaded volume tarballs exist
	PreloadBucket = "minikube-preloaded-volume-tarballs"
)

// TarballName returns name of the tarball
func TarballName(k8sVersion, containerRuntime string) string {
	if containerRuntime == "crio" {
		containerRuntime = "cri-o"
	}
	var storageDriver string
	if containerRuntime == "cri-o" {
		storageDriver = "overlay"
	} else {
		storageDriver = "overlay2"
	}
	return fmt.Sprintf("preloaded-images-k8s-%s-%s-%s-%s-%s.tar.lz4", PreloadVersion, k8sVersion, containerRuntime, storageDriver, runtime.GOARCH)
}

// returns the name of the checksum file
func checksumName(k8sVersion, containerRuntime string) string {
	return fmt.Sprintf("%s.checksum", TarballName(k8sVersion, containerRuntime))
}

// returns target dir for all cached items related to preloading
func targetDir() string {
	return localpath.MakeMiniPath("cache", "preloaded-tarball")
}

// PreloadChecksumPath returns the local path to the cached checksum file
func PreloadChecksumPath(k8sVersion, containerRuntime string) string {
	return filepath.Join(targetDir(), checksumName(k8sVersion, containerRuntime))
}

// TarballPath returns the local path to the cached preload tarball
func TarballPath(k8sVersion, containerRuntime string) string {
	return filepath.Join(targetDir(), TarballName(k8sVersion, containerRuntime))
}

// remoteTarballURL returns the URL for the remote tarball in GCS
func remoteTarballURL(k8sVersion, containerRuntime string) string {
	return fmt.Sprintf("https://storage.googleapis.com/%s/%s", PreloadBucket, TarballName(k8sVersion, containerRuntime))
}

// PreloadExists returns true if there is a preloaded tarball that can be used
func PreloadExists(k8sVersion, containerRuntime string, forcePreload ...bool) bool {

	// and https://github.com/kubernetes/minikube/issues/6934
	// to track status of adding crio
	if containerRuntime == "crio" {
		glog.Info("crio is not supported yet, skipping preload")
		return false
	}

	// TODO (#8166): Get rid of the need for this and viper at all
	force := false
	if len(forcePreload) > 0 {
		force = forcePreload[0]
	}

	// TODO: debug why this func is being called two times
	glog.Infof("Checking if preload exists for k8s version %s and runtime %s", k8sVersion, containerRuntime)
	if !viper.GetBool("preload") && !force {
		return false
	}

	// Omit remote check if tarball exists locally
	targetPath := TarballPath(k8sVersion, containerRuntime)
	if _, err := os.Stat(targetPath); err == nil {
		glog.Infof("Found local preload: %s", targetPath)
		return true
	}

	url := remoteTarballURL(k8sVersion, containerRuntime)
	resp, err := http.Head(url)
	if err != nil {
		glog.Warningf("%s fetch error: %v", url, err)
		return false
	}

	// note: err won't be set if it's a 404
	if resp.StatusCode != 200 {
		glog.Warningf("%s status code: %d", url, resp.StatusCode)
		return false
	}

	glog.Infof("Found remote preload: %s", url)
	return true
}

// Preload caches the preloaded images tarball on the host machine
func Preload(k8sVersion, containerRuntime string) error {
	targetPath := TarballPath(k8sVersion, containerRuntime)

	if _, err := os.Stat(targetPath); err == nil {
		glog.Infof("Found %s in cache, skipping download", targetPath)
		return nil
	}

	// Make sure we support this k8s version
	if !PreloadExists(k8sVersion, containerRuntime) {
		glog.Infof("Preloaded tarball for k8s version %s does not exist", k8sVersion)
		return nil
	}

	out.T(out.FileDownload, "Downloading Kubernetes {{.version}} preload ...", out.V{"version": k8sVersion})
	url := remoteTarballURL(k8sVersion, containerRuntime)

	if err := download(url, targetPath); err != nil {
		return errors.Wrapf(err, "download failed: %s", url)
	}

	if err := saveChecksumFile(k8sVersion, containerRuntime); err != nil {
		return errors.Wrap(err, "saving checksum file")
	}

	if err := verifyChecksum(k8sVersion, containerRuntime, targetPath); err != nil {
		return errors.Wrap(err, "verify")
	}

	return nil
}

func saveChecksumFile(k8sVersion, containerRuntime string) error {
	glog.Infof("saving checksum for %s ...", TarballName(k8sVersion, containerRuntime))
	ctx := context.Background()
	client, err := storage.NewClient(ctx, option.WithoutAuthentication())
	if err != nil {
		return errors.Wrap(err, "getting storage client")
	}
	attrs, err := client.Bucket(PreloadBucket).Object(TarballName(k8sVersion, containerRuntime)).Attrs(ctx)
	if err != nil {
		return errors.Wrap(err, "getting storage object")
	}
	checksum := attrs.MD5
	return ioutil.WriteFile(PreloadChecksumPath(k8sVersion, containerRuntime), checksum, 0644)
}

// verifyChecksum returns true if the checksum of the local binary matches
// the checksum of the remote binary
func verifyChecksum(k8sVersion, containerRuntime, path string) error {
	glog.Infof("verifying checksumm of %s ...", path)
	// get md5 checksum of tarball path
	contents, err := ioutil.ReadFile(path)
	if err != nil {
		return errors.Wrap(err, "reading tarball")
	}
	checksum := md5.Sum(contents)

	remoteChecksum, err := ioutil.ReadFile(PreloadChecksumPath(k8sVersion, containerRuntime))
	if err != nil {
		return errors.Wrap(err, "reading checksum file")
	}

	// create a slice of checksum, which is [16]byte
	if string(remoteChecksum) != string(checksum[:]) {
		return fmt.Errorf("checksum of %s does not match remote checksum (%s != %s)", path, string(remoteChecksum), string(checksum[:]))
	}
	return nil
}
<|endoftext|>


<commit_before>// Copyright 2018 Google LLC All Rights Reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//    http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package transport

import (
	"encoding/json"
	"fmt"
	"io/ioutil"
	"net/http"
	"net/url"
	"strings"
)

// The set of query string keys that we expect to send as part of the registry
// protocol. Anything else is potentially dangerous to leak, as it's probably
// from a redirect. These redirects often included tokens or signed URLs.
var paramWhitelist = map[string]struct{}{
	// Token exchange
	"scope":   struct{}{},
	"service": struct{}{},
	// Cross-repo mounting
	"mount": struct{}{},
	"from":  struct{}{},
	// Layer PUT
	"digest": struct{}{},
	// Listing tags and catalog
	"n":    struct{}{},
	"last": struct{}{},
}

// Error implements error to support the following error specification:
// https://github.com/docker/distribution/blob/master/docs/spec/api.md#errors
type Error struct {
	Errors []Diagnostic `json:"errors,omitempty"`
	// The http status code returned.
	StatusCode int
	// The raw body if we couldn't understand it.
	rawBody string
	// The request that failed.
	request *http.Request
}

// Check that Error implements error
var _ error = (*Error)(nil)

// Error implements error
func (e *Error) Error() string {
	prefix := ""
	if e.request != nil {
		prefix = fmt.Sprintf("%s %s: ", e.request.Method, redact(e.request.URL))
	}
	return prefix + e.responseErr()
}

func (e *Error) responseErr() string {
	switch len(e.Errors) {
	case 0:
		if len(e.rawBody) == 0 {
			return fmt.Sprintf("unsupported status code %d", e.StatusCode)
		}
		return fmt.Sprintf("unsupported status code %d; body: %s", e.StatusCode, e.rawBody)
	case 1:
		return e.Errors[0].String()
	default:
		var errors []string
		for _, d := range e.Errors {
			errors = append(errors, d.String())
		}
		return fmt.Sprintf("multiple errors returned: %s",
			strings.Join(errors, "; "))
	}
}

// Temporary returns whether the request that preceded the error is temporary.
func (e *Error) Temporary() bool {
	if len(e.Errors) == 0 {
		return false
	}
	for _, d := range e.Errors {
		if _, ok := temporaryErrorCodes[d.Code]; !ok {
			return false
		}
	}
	return true
}

func redact(original *url.URL) *url.URL {
	qs := original.Query()
	for k, v := range qs {
		for i := range v {
			if _, ok := paramWhitelist[k]; !ok {
				// key is not in the whitelist
				v[i] = "REDACTED"
			}
		}
	}
	redacted := *original
	redacted.RawQuery = qs.Encode()
	return &redacted
}

// Diagnostic represents a single error returned by a Docker registry interaction.
type Diagnostic struct {
	Code    ErrorCode   `json:"code"`
	Message string      `json:"message,omitempty"`
	Detail  interface{} `json:"detail,omitempty"`
}

// String stringifies the Diagnostic in the form: $Code: $Message[; $Detail]
func (d Diagnostic) String() string {
	msg := fmt.Sprintf("%s: %s", d.Code, d.Message)
	if d.Detail != nil {
		msg = fmt.Sprintf("%s; %v", msg, d.Detail)
	}
	return msg
}

// ErrorCode is an enumeration of supported error codes.
type ErrorCode string

// The set of error conditions a registry may return:
// https://github.com/docker/distribution/blob/master/docs/spec/api.md#errors-2
const (
	BlobUnknownErrorCode         ErrorCode = "BLOB_UNKNOWN"
	BlobUploadInvalidErrorCode   ErrorCode = "BLOB_UPLOAD_INVALID"
	BlobUploadUnknownErrorCode   ErrorCode = "BLOB_UPLOAD_UNKNOWN"
	DigestInvalidErrorCode       ErrorCode = "DIGEST_INVALID"
	ManifestBlobUnknownErrorCode ErrorCode = "MANIFEST_BLOB_UNKNOWN"
	ManifestInvalidErrorCode     ErrorCode = "MANIFEST_INVALID"
	ManifestUnknownErrorCode     ErrorCode = "MANIFEST_UNKNOWN"
	ManifestUnverifiedErrorCode  ErrorCode = "MANIFEST_UNVERIFIED"
	NameInvalidErrorCode         ErrorCode = "NAME_INVALID"
	NameUnknownErrorCode         ErrorCode = "NAME_UNKNOWN"
	SizeInvalidErrorCode         ErrorCode = "SIZE_INVALID"
	TagInvalidErrorCode          ErrorCode = "TAG_INVALID"
	UnauthorizedErrorCode        ErrorCode = "UNAUTHORIZED"
	DeniedErrorCode              ErrorCode = "DENIED"
	UnsupportedErrorCode         ErrorCode = "UNSUPPORTED"
	TooManyRequestsErrorCode     ErrorCode = "TOOMANYREQUESTS"
)

// TODO: Include other error types.
var temporaryErrorCodes = map[ErrorCode]struct{}{
	BlobUploadInvalidErrorCode: struct{}{},
	TooManyRequestsErrorCode:   struct{}{},
}

// CheckError returns a structured error if the response status is not in codes.
func CheckError(resp *http.Response, codes ...int) error {
	for _, code := range codes {
		if resp.StatusCode == code {
			// This is one of the supported status codes.
			return nil
		}
	}
	b, err := ioutil.ReadAll(resp.Body)
	if err != nil {
		return err
	}

	// https://github.com/docker/distribution/blob/master/docs/spec/api.md#errors
	structuredError := &Error{}

	// This can fail if e.g. the response body is not valid JSON. That's fine,
	// we'll construct an appropriate error string from the body and status code.
	_ = json.Unmarshal(b, structuredError)

	structuredError.rawBody = string(b)
	structuredError.StatusCode = resp.StatusCode
	structuredError.request = resp.Request

	return structuredError
}
<commit_msg>Fix gofmt (#774)<commit_after>// Copyright 2018 Google LLC All Rights Reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//    http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package transport

import (
	"encoding/json"
	"fmt"
	"io/ioutil"
	"net/http"
	"net/url"
	"strings"
)

// The set of query string keys that we expect to send as part of the registry
// protocol. Anything else is potentially dangerous to leak, as it's probably
// from a redirect. These redirects often included tokens or signed URLs.
var paramWhitelist = map[string]struct{}{
	// Token exchange
	"scope":   {},
	"service": {},
	// Cross-repo mounting
	"mount": {},
	"from":  {},
	// Layer PUT
	"digest": {},
	// Listing tags and catalog
	"n":    {},
	"last": {},
}

// Error implements error to support the following error specification:
// https://github.com/docker/distribution/blob/master/docs/spec/api.md#errors
type Error struct {
	Errors []Diagnostic `json:"errors,omitempty"`
	// The http status code returned.
	StatusCode int
	// The raw body if we couldn't understand it.
	rawBody string
	// The request that failed.
	request *http.Request
}

// Check that Error implements error
var _ error = (*Error)(nil)

// Error implements error
func (e *Error) Error() string {
	prefix := ""
	if e.request != nil {
		prefix = fmt.Sprintf("%s %s: ", e.request.Method, redact(e.request.URL))
	}
	return prefix + e.responseErr()
}

func (e *Error) responseErr() string {
	switch len(e.Errors) {
	case 0:
		if len(e.rawBody) == 0 {
			return fmt.Sprintf("unsupported status code %d", e.StatusCode)
		}
		return fmt.Sprintf("unsupported status code %d; body: %s", e.StatusCode, e.rawBody)
	case 1:
		return e.Errors[0].String()
	default:
		var errors []string
		for _, d := range e.Errors {
			errors = append(errors, d.String())
		}
		return fmt.Sprintf("multiple errors returned: %s",
			strings.Join(errors, "; "))
	}
}

// Temporary returns whether the request that preceded the error is temporary.
func (e *Error) Temporary() bool {
	if len(e.Errors) == 0 {
		return false
	}
	for _, d := range e.Errors {
		if _, ok := temporaryErrorCodes[d.Code]; !ok {
			return false
		}
	}
	return true
}

func redact(original *url.URL) *url.URL {
	qs := original.Query()
	for k, v := range qs {
		for i := range v {
			if _, ok := paramWhitelist[k]; !ok {
				// key is not in the whitelist
				v[i] = "REDACTED"
			}
		}
	}
	redacted := *original
	redacted.RawQuery = qs.Encode()
	return &redacted
}

// Diagnostic represents a single error returned by a Docker registry interaction.
type Diagnostic struct {
	Code    ErrorCode   `json:"code"`
	Message string      `json:"message,omitempty"`
	Detail  interface{} `json:"detail,omitempty"`
}

// String stringifies the Diagnostic in the form: $Code: $Message[; $Detail]
func (d Diagnostic) String() string {
	msg := fmt.Sprintf("%s: %s", d.Code, d.Message)
	if d.Detail != nil {
		msg = fmt.Sprintf("%s; %v", msg, d.Detail)
	}
	return msg
}

// ErrorCode is an enumeration of supported error codes.
type ErrorCode string

// The set of error conditions a registry may return:
// https://github.com/docker/distribution/blob/master/docs/spec/api.md#errors-2
const (
	BlobUnknownErrorCode         ErrorCode = "BLOB_UNKNOWN"
	BlobUploadInvalidErrorCode   ErrorCode = "BLOB_UPLOAD_INVALID"
	BlobUploadUnknownErrorCode   ErrorCode = "BLOB_UPLOAD_UNKNOWN"
	DigestInvalidErrorCode       ErrorCode = "DIGEST_INVALID"
	ManifestBlobUnknownErrorCode ErrorCode = "MANIFEST_BLOB_UNKNOWN"
	ManifestInvalidErrorCode     ErrorCode = "MANIFEST_INVALID"
	ManifestUnknownErrorCode     ErrorCode = "MANIFEST_UNKNOWN"
	ManifestUnverifiedErrorCode  ErrorCode = "MANIFEST_UNVERIFIED"
	NameInvalidErrorCode         ErrorCode = "NAME_INVALID"
	NameUnknownErrorCode         ErrorCode = "NAME_UNKNOWN"
	SizeInvalidErrorCode         ErrorCode = "SIZE_INVALID"
	TagInvalidErrorCode          ErrorCode = "TAG_INVALID"
	UnauthorizedErrorCode        ErrorCode = "UNAUTHORIZED"
	DeniedErrorCode              ErrorCode = "DENIED"
	UnsupportedErrorCode         ErrorCode = "UNSUPPORTED"
	TooManyRequestsErrorCode     ErrorCode = "TOOMANYREQUESTS"
)

// TODO: Include other error types.
var temporaryErrorCodes = map[ErrorCode]struct{}{
	BlobUploadInvalidErrorCode: {},
	TooManyRequestsErrorCode:   {},
}

// CheckError returns a structured error if the response status is not in codes.
func CheckError(resp *http.Response, codes ...int) error {
	for _, code := range codes {
		if resp.StatusCode == code {
			// This is one of the supported status codes.
			return nil
		}
	}
	b, err := ioutil.ReadAll(resp.Body)
	if err != nil {
		return err
	}

	// https://github.com/docker/distribution/blob/master/docs/spec/api.md#errors
	structuredError := &Error{}

	// This can fail if e.g. the response body is not valid JSON. That's fine,
	// we'll construct an appropriate error string from the body and status code.
	_ = json.Unmarshal(b, structuredError)

	structuredError.rawBody = string(b)
	structuredError.StatusCode = resp.StatusCode
	structuredError.request = resp.Request

	return structuredError
}
<|endoftext|>


<commit_before>package extension

import (
	"github.com/kayex/sirius"
	"googlemaps.github.io/maps"
	"golang.org/x/net/context"
	"errors"
	"strings"
	"fmt"
)

type Geocode struct {
	APIKey string
}

func (gc *Geocode) Run(m sirius.Message, cfg sirius.ExtensionConfig) (error, sirius.MessageAction) {
	if !strings.HasPrefix(m.Text, "<address") {
		return nil, sirius.NoAction()
	}

	address := strings.TrimPrefix(m.Text, "<address ")

	c, err := maps.NewClient(maps.WithAPIKey(gc.APIKey))

	if err != nil {
		return errors.New(fmt.Sprintf("Maps client error: %v", err.Error())), nil
	}

	r := &maps.GeocodingRequest{
		Address: address,
	}

	res, err := c.Geocode(context.Background(), r)

	if err != nil {
		return errors.New(fmt.Sprintf("Geocoding error: %v", err.Error())), nil
	}

	pos := res[0]
	location := pos.Geometry.Location
	coords := fmt.Sprintf("*%v*\n`(%.6f, %.6f)`", pos.FormattedAddress, location.Lat, location.Lng)

	edit := m.EditText().ReplaceWith(coords)

	return nil, edit
}
<commit_msg>Rename variable<commit_after>package extension

import (
	"github.com/kayex/sirius"
	"googlemaps.github.io/maps"
	"golang.org/x/net/context"
	"errors"
	"strings"
	"fmt"
)

type Geocode struct {
	APIKey string
}

func (gc *Geocode) Run(m sirius.Message, cfg sirius.ExtensionConfig) (error, sirius.MessageAction) {
	if !strings.HasPrefix(m.Text, "<address") {
		return nil, sirius.NoAction()
	}

	address := strings.TrimPrefix(m.Text, "<address ")

	c, err := maps.NewClient(maps.WithAPIKey(gc.APIKey))

	if err != nil {
		return errors.New(fmt.Sprintf("Maps client error: %v", err.Error())), nil
	}

	r := &maps.GeocodingRequest{
		Address: address,
	}

	res, err := c.Geocode(context.Background(), r)

	if err != nil {
		return errors.New(fmt.Sprintf("Geocoding error: %v", err.Error())), nil
	}

	pos := res[0]
	location := pos.Geometry.Location
	formatted := fmt.Sprintf("*%v*\n`(%.6f, %.6f)`", pos.FormattedAddress, location.Lat, location.Lng)

	edit := m.EditText().ReplaceWith(formatted)

	return nil, edit
}
<|endoftext|>


<commit_before>package global

import (
	"code.cloudfoundry.org/cli/integration/helpers"

	. "github.com/onsi/ginkgo"
	. "github.com/onsi/gomega"
	. "github.com/onsi/gomega/gbytes"
	. "github.com/onsi/gomega/gexec"
)

var _ = Describe("rename buildpack command", func() {
	Describe("help", func() {
		Context("when --help flag is set", func() {
			It("Displays command usage to output", func() {
				session := helpers.CF("rename-buildpack", "--help")
				Eventually(session).Should(Say("NAME:"))
				Eventually(session).Should(Say("rename-buildpack - Rename a buildpack"))
				Eventually(session).Should(Say("USAGE:"))
				Eventually(session).Should(Say("cf rename-buildpack BUILDPACK_NAME NEW_BUILDPACK_NAME"))
				Eventually(session).Should(Say("SEE ALSO:"))
				Eventually(session).Should(Say("update-buildpack"))
				Eventually(session).Should(Exit(0))
			})
		})
	})

	Context("when the environment is not setup correctly", func() {
		It("fails with the appropriate errors", func() {
			helpers.CheckEnvironmentTargetedCorrectly(false, false, ReadOnlyOrg, "rename-buildpack", "fake-buildpack", "some-name")
		})
	})

	Context("when the user is logged in", func() {
		var (
			oldBuildpackName      string
			newBuildpackName      string
			existingBuildpackName string
		)

		BeforeEach(func() {
			helpers.LoginCF()
		})

		Context("when the buildpack exists", func() {
			BeforeEach(func() {
				oldBuildpackName = helpers.NewBuildpack()
				newBuildpackName = helpers.NewBuildpack()

				helpers.BuildpackWithStack(func(buildpackPath string) {
					session := helpers.CF("create-buildpack", oldBuildpackName, buildpackPath, "1")
					Eventually(session).Should(Exit(0))
				}, "")
			})

			Context("when renaming to a unique name", func() {
				It("successfully renames buildpack", func() {
					session := helpers.CF("rename-buildpack", oldBuildpackName, newBuildpackName)
					Eventually(session).Should(Say("Renaming buildpack %s to %s", oldBuildpackName, newBuildpackName))
					Eventually(session).Should(Say("OK"))
					Eventually(session).Should(Exit(0))
				})
			})

			Context("when renaming to the same name", func() {
				It("successfully renames buildpack", func() {
					session := helpers.CF("rename-buildpack", oldBuildpackName, oldBuildpackName)
					Eventually(session).Should(Say("Renaming buildpack %s to %s", oldBuildpackName, oldBuildpackName))
					Eventually(session).Should(Say("OK"))
					Eventually(session).Should(Exit(0))
				})
			})

			Context("when renaming to the same name as another buildpack", func() {
				BeforeEach(func() {
					existingBuildpackName = helpers.NewBuildpack()

					helpers.BuildpackWithStack(func(buildpackPath string) {
						session := helpers.CF("create-buildpack", existingBuildpackName, buildpackPath, "1")
						Eventually(session).Should(Exit(0))
					}, "")
				})

				It("returns the appropriate error", func() {
					session := helpers.CF("rename-buildpack", oldBuildpackName, existingBuildpackName)
					Eventually(session).Should(Say("Renaming buildpack %s to %s", oldBuildpackName, existingBuildpackName))
					Eventually(session).Should(Say("FAILED"))
					Eventually(session.Err).Should(Say("Buildpack %s already exists without a stack", existingBuildpackName))
					Eventually(session).Should(Exit(1))
				})
			})
		})

		Context("when there are multiple ambiguous buildpacks", func() {
			BeforeEach(func() {
				helpers.SkipIfOneStack()

				stacks := helpers.FetchStacks()

				oldBuildpackName = helpers.NewBuildpack()
				newBuildpackName = helpers.NewBuildpack()

				helpers.BuildpackWithStack(func(buildpackPath string) {
					session := helpers.CF("create-buildpack", oldBuildpackName, buildpackPath, "1")
					Eventually(session).Should(Exit(0))
				}, stacks[0])
				helpers.BuildpackWithStack(func(buildpackPath string) {
					session := helpers.CF("create-buildpack", oldBuildpackName, buildpackPath, "1")
					Eventually(session).Should(Exit(0))
				}, stacks[1])
			})

			It("returns multiple buildpacks found error", func() {
				session := helpers.CF("rename-buildpack", oldBuildpackName, newBuildpackName)
				Eventually(session).Should(Say("Renaming buildpack %s to %s", oldBuildpackName, newBuildpackName))
				Eventually(session).Should(Say("FAILED"))
				Eventually(session.Err).Should(Say("Multiple buildpacks named %s found.", oldBuildpackName))
				Eventually(session).Should(Exit(1))
			})
		})
	})
})
<commit_msg>Pend rename buildpack integration test, unclear history but refactor is in progress today.<commit_after>package global

import (
	"code.cloudfoundry.org/cli/integration/helpers"

	. "github.com/onsi/ginkgo"
	. "github.com/onsi/gomega"
	. "github.com/onsi/gomega/gbytes"
	. "github.com/onsi/gomega/gexec"
)

var _ = Describe("rename buildpack command", func() {
	BeforeEach(func() {
		Skip("until #158613755 is complete")
	})

	Describe("help", func() {
		Context("when --help flag is set", func() {
			It("Displays command usage to output", func() {
				session := helpers.CF("rename-buildpack", "--help")
				Eventually(session).Should(Say("NAME:"))
				Eventually(session).Should(Say("rename-buildpack - Rename a buildpack"))
				Eventually(session).Should(Say("USAGE:"))
				Eventually(session).Should(Say("cf rename-buildpack BUILDPACK_NAME NEW_BUILDPACK_NAME"))
				Eventually(session).Should(Say("SEE ALSO:"))
				Eventually(session).Should(Say("update-buildpack"))
				Eventually(session).Should(Exit(0))
			})
		})
	})

	Context("when the environment is not setup correctly", func() {
		It("fails with the appropriate errors", func() {
			helpers.CheckEnvironmentTargetedCorrectly(false, false, ReadOnlyOrg, "rename-buildpack", "fake-buildpack", "some-name")
		})
	})

	Context("when the user is logged in", func() {
		var (
			oldBuildpackName      string
			newBuildpackName      string
			existingBuildpackName string
		)

		BeforeEach(func() {
			helpers.LoginCF()
		})

		Context("when the buildpack exists", func() {
			BeforeEach(func() {
				oldBuildpackName = helpers.NewBuildpack()
				newBuildpackName = helpers.NewBuildpack()

				helpers.BuildpackWithStack(func(buildpackPath string) {
					session := helpers.CF("create-buildpack", oldBuildpackName, buildpackPath, "1")
					Eventually(session).Should(Exit(0))
				}, "")
			})

			Context("when renaming to a unique name", func() {
				It("successfully renames buildpack", func() {
					session := helpers.CF("rename-buildpack", oldBuildpackName, newBuildpackName)
					Eventually(session).Should(Say("Renaming buildpack %s to %s", oldBuildpackName, newBuildpackName))
					Eventually(session).Should(Say("OK"))
					Eventually(session).Should(Exit(0))
				})
			})

			Context("when renaming to the same name", func() {
				It("successfully renames buildpack", func() {
					session := helpers.CF("rename-buildpack", oldBuildpackName, oldBuildpackName)
					Eventually(session).Should(Say("Renaming buildpack %s to %s", oldBuildpackName, oldBuildpackName))
					Eventually(session).Should(Say("OK"))
					Eventually(session).Should(Exit(0))
				})
			})

			Context("when renaming to the same name as another buildpack", func() {
				BeforeEach(func() {
					existingBuildpackName = helpers.NewBuildpack()

					helpers.BuildpackWithStack(func(buildpackPath string) {
						session := helpers.CF("create-buildpack", existingBuildpackName, buildpackPath, "1")
						Eventually(session).Should(Exit(0))
					}, "")
				})

				It("returns the appropriate error", func() {
					session := helpers.CF("rename-buildpack", oldBuildpackName, existingBuildpackName)
					Eventually(session).Should(Say("Renaming buildpack %s to %s", oldBuildpackName, existingBuildpackName))
					Eventually(session).Should(Say("FAILED"))
					Eventually(session.Err).Should(Say("Buildpack %s already exists without a stack", existingBuildpackName))
					Eventually(session).Should(Exit(1))
				})
			})
		})

		Context("when there are multiple ambiguous buildpacks", func() {
			BeforeEach(func() {
				helpers.SkipIfOneStack()

				stacks := helpers.FetchStacks()

				oldBuildpackName = helpers.NewBuildpack()
				newBuildpackName = helpers.NewBuildpack()

				helpers.BuildpackWithStack(func(buildpackPath string) {
					session := helpers.CF("create-buildpack", oldBuildpackName, buildpackPath, "1")
					Eventually(session).Should(Exit(0))
				}, stacks[0])
				helpers.BuildpackWithStack(func(buildpackPath string) {
					session := helpers.CF("create-buildpack", oldBuildpackName, buildpackPath, "1")
					Eventually(session).Should(Exit(0))
				}, stacks[1])
			})

			It("returns multiple buildpacks found error", func() {
				session := helpers.CF("rename-buildpack", oldBuildpackName, newBuildpackName)
				Eventually(session).Should(Say("Renaming buildpack %s to %s", oldBuildpackName, newBuildpackName))
				Eventually(session).Should(Say("FAILED"))
				Eventually(session.Err).Should(Say("Multiple buildpacks named %s found.", oldBuildpackName))
				Eventually(session).Should(Exit(1))
			})
		})
	})
})
<|endoftext|>


<commit_before>// Copyright 2013 The lime Authors.
// Use of this source code is governed by a 2-clause
// BSD-style license that can be found in the LICENSE file.

package main

import (
	"code.google.com/p/log4go"
	"flag"
	"fmt"
	"github.com/limetext/gopy/lib"
	"github.com/limetext/lime/backend"
	_ "github.com/limetext/lime/backend/commands"
	"github.com/limetext/lime/backend/keys"
	"github.com/limetext/lime/backend/render"
	"github.com/limetext/lime/backend/sublime"
	"github.com/limetext/lime/backend/textmate"
	"github.com/limetext/lime/backend/util"
	. "github.com/limetext/text"
	"io"
	"net/http"
	"runtime/debug"
	"strconv"
	"sync"
	"time"
	"unicode"
)

var (
	scheme *textmate.Theme
	blink  bool
	port   = flag.Int("port", 8080, "Configures which port to host lime on")
)

const (
	console_height  = 20
	render_chan_len = 2
)

type layout struct {
	x, y          int
	width, height int
	visible       Region
	lastUpdate    int
}
type tbfe struct {
	layout         map[*backend.View]layout
	status_message string
	dorender       chan bool
	lock           sync.Mutex
	dirty          bool
}

func htmlcol(c render.Colour) string {
	return fmt.Sprintf("%02X%02X%02X", c.R, c.G, c.B)
}

func (t *tbfe) renderView(wr io.Writer, v *backend.View, lay layout) {
	p := util.Prof.Enter("render")
	defer p.Exit()

	vr := lay.visible
	runes := v.Buffer().Substr(vr)
	recipie := v.Transform(scheme, vr).Transcribe()
	highlight_line := false
	if b, ok := v.Settings().Get("highlight_line", highlight_line).(bool); ok {
		highlight_line = b
	}
	lastEnd := 0
	for _, reg := range recipie {
		if lastEnd != reg.Region.Begin() {
			io.WriteString(wr, runes[lastEnd:reg.Region.Begin()])
		}
		fmt.Fprintf(wr, "%s", htmlcol(reg.Flavour.Foreground), htmlcol(reg.Flavour.Background), runes[reg.Region.Begin():reg.Region.End()])
		lastEnd = reg.Region.End()
	}
	if lastEnd != vr.End() {
		io.WriteString(wr, v.Buffer().Substr(Region{lastEnd, vr.End()}))
	}
}

func (t *tbfe) clip(v *backend.View, s, e int) Region {
	p := util.Prof.Enter("clip")
	defer p.Exit()
	h := t.layout[v].height
	if e-s > h {
		e = s + h
	} else if e-s < h {
		s = e - h
	}
	if e2, _ := v.Buffer().RowCol(v.Buffer().TextPoint(e, 0)); e2 < e {
		e = e2
	}
	if s < 0 {
		s = 0
	}
	e = s + h
	r := Region{v.Buffer().TextPoint(s, 0), v.Buffer().TextPoint(e, 0)}
	return v.Buffer().LineR(r)
}

func (t *tbfe) Show(v *backend.View, r Region) {
	t.lock.Lock()
	l := t.layout[v]
	t.lock.Unlock()
	if l.visible.Covers(r) {
		return
	}
	p := util.Prof.Enter("show")
	defer p.Exit()

	lv := l.visible

	s1, _ := v.Buffer().RowCol(lv.Begin())
	e1, _ := v.Buffer().RowCol(lv.End())
	s2, _ := v.Buffer().RowCol(r.Begin())
	e2, _ := v.Buffer().RowCol(r.End())

	r1 := Region{s1, e1}
	r2 := Region{s2, e2}

	r3 := r1.Cover(r2)
	diff := 0
	if d1, d2 := Abs(r1.Begin()-r3.Begin()), Abs(r1.End()-r3.End()); d1 > d2 {
		diff = r3.Begin() - r1.Begin()
	} else {
		diff = r3.End() - r1.End()
	}
	r3.A = r1.Begin() + diff
	r3.B = r1.End() + diff

	r3 = t.clip(v, r3.A, r3.B)
	l.visible = r3
	t.lock.Lock()
	t.layout[v] = l
	t.lock.Unlock()
}

func (t *tbfe) VisibleRegion(v *backend.View) Region {
	t.lock.Lock()
	r, ok := t.layout[v]
	t.lock.Unlock()
	if !ok || r.lastUpdate != v.Buffer().ChangeCount() {
		t.Show(v, r.visible)
		t.lock.Lock()
		r = t.layout[v]
		t.lock.Unlock()
	}
	return r.visible
}

func (t *tbfe) StatusMessage(msg string) {
	t.lock.Lock()
	defer t.lock.Unlock()
	t.status_message = msg
}

func (t *tbfe) ErrorMessage(msg string) {
	log4go.Error(msg)
}

// TODO(q): Actually show a dialog
func (t *tbfe) MessageDialog(msg string) {
	log4go.Info(msg)
}

// TODO(q): Actually show a dialog
func (t *tbfe) OkCancelDialog(msg, ok string) bool {
	log4go.Info(msg, ok)
	return false
}

func (t *tbfe) scroll(b Buffer, pos, delta int) {
	t.Show(backend.GetEditor().Console(), Region{b.Size(), b.Size()})
}

var pc = 0

func (t *tbfe) render(w io.Writer) {
	defer func() {
		if r := recover(); r != nil {
			log4go.Error("Panic in renderthread: %v\n%s", r, string(debug.Stack()))
			if pc > 1 {
				panic(r)
			}
			pc++
		}
	}()
	vs := make([]*backend.View, 0, len(t.layout))
	l := make([]layout, 0, len(t.layout))
	for k, v := range t.layout {
		vs = append(vs, k)
		l = append(l, v)
	}
	for i, v := range vs {
		t.renderView(w, v, l[i])
	}
	//	runes := []rune(t.status_message)
}
func (t *tbfe) key(w http.ResponseWriter, req *http.Request) {
	log4go.Debug("key: %s", req)
	kc := req.FormValue("keyCode")
	var kp keys.KeyPress
	v, _ := strconv.ParseInt(kc, 10, 32)

	if req.FormValue("altKey") == "true" {
		kp.Alt = true
	}
	if req.FormValue("ctrlKey") == "true" {
		kp.Ctrl = true
	}
	if req.FormValue("metaKey") == "true" {
		kp.Super = true
	}
	if req.FormValue("shiftKey") == "true" {
		kp.Shift = true
	}
	if !kp.Shift {
		v = int64(unicode.ToLower(rune(v)))
	}
	kp.Key = keys.Key(v)
	backend.GetEditor().HandleInput(kp)
}

func (t *tbfe) view(w http.ResponseWriter, req *http.Request) {
	log4go.Debug("view: %s", req)
	if t.dirty {
		t.dirty = false
		t.render(w)
	} else {
		w.WriteHeader(404)
	}
}

func (t *tbfe) ServeHTTP(w http.ResponseWriter, req *http.Request) {
	s := time.Now()
	w.Header().Set("Content-Type", "text/html")
	log4go.Debug("Serving client: %s", req)

	c := scheme.Spice(&render.ViewRegions{})

	fmt.Fprintf(w, `

`, htmlcol(c.Foreground), htmlcol(c.Background))
	io.WriteString(w, "")
	log4go.Debug("Done serving client: %s", time.Since(s))
}

func (t *tbfe) loop() {
	backend.OnNew.Add(func(v *backend.View) {
		v.Settings().AddOnChange("lime.frontend.html.render", func(name string) { t.dirty = true })
	})
	backend.OnModified.Add(func(v *backend.View) {
		t.dirty = true
	})
	backend.OnSelectionModified.Add(func(v *backend.View) {
		t.dirty = true
	})

	ed := backend.GetEditor()
	ed.SetFrontend(t)
	ed.LogInput(false)
	ed.LogCommands(false)
	c := ed.Console()
	if sc, err := textmate.LoadTheme("../../3rdparty/bundles/TextMate-Themes/GlitterBomb.tmTheme"); err != nil {
		log4go.Error(err)
	} else {
		scheme = sc
	}

	defer func() {
		fmt.Println(util.Prof)
	}()

	w := ed.NewWindow()
	v := w.OpenFile("main.go", 0)
	v.Settings().Set("trace", true)
	v.Settings().Set("syntax", "../../3rdparty/bundles/go.tmbundle/Syntaxes/Go.tmLanguage")
	c.Buffer().AddCallback(t.scroll)

	sel := v.Sel()
	sel.Clear()
	//	end := v.Buffer().Size() - 2
	sel.Add(Region{0, 0})
	// sel.Add(Region{end - 22, end - 22})
	// sel.Add(Region{end - 16, end - 20})
	// sel.Add(Region{end - 13, end - 10})

	{
		w, h := 800, 600
		t.lock.Lock()
		t.layout[v] = layout{0, 0, w, h - console_height - 1, Region{}, 0}
		t.layout[c] = layout{0, h - console_height + 1, w, console_height - 5, Region{}, 0}
		t.lock.Unlock()
		t.Show(v, Region{1, 1})
	}
	t.Show(v, Region{100, 100})
	t.Show(v, Region{1, 1})

	go func() {
		ed.Init()
		sublime.Init()
	}()
	log4go.Debug("serving")
	http.HandleFunc("/key", t.key)
	http.HandleFunc("/", t.ServeHTTP)
	http.HandleFunc("/view", t.view)
	if err := http.ListenAndServe(fmt.Sprintf("localhost:%d", *port), nil); err != nil {
		log4go.Error("Error serving: %s", err)
	}
	log4go.Debug("Done")
}

func main() {
	flag.Parse()
	log4go.AddFilter("file", log4go.FINEST, log4go.NewConsoleLogWriter())
	defer func() {
		py.NewLock()
		py.Finalize()
	}()

	var t tbfe
	t.dorender = make(chan bool, render_chan_len)
	t.layout = make(map[*backend.View]layout)
	t.loop()
}
<commit_msg>Adds Websocket support to HTML frontend<commit_after>// Copyright 2013 The lime Authors.
// Use of this source code is governed by a 2-clause
// BSD-style license that can be found in the LICENSE file.

package main

import (
	"code.google.com/p/log4go"
	"code.google.com/p/go.net/websocket"
	"flag"
	"fmt"
	"github.com/limetext/gopy/lib"
	"github.com/limetext/lime/backend"
	_ "github.com/limetext/lime/backend/commands"
	"github.com/limetext/lime/backend/keys"
	"github.com/limetext/lime/backend/render"
	"github.com/limetext/lime/backend/sublime"
	"github.com/limetext/lime/backend/textmate"
	"github.com/limetext/lime/backend/util"
	. "github.com/limetext/text"
	"io"
	"bytes"
	"bufio"
	"net/http"
	"runtime/debug"
	"strconv"
	"sync"
	"time"
	"unicode"
)

var (
	scheme *textmate.Theme
	blink  bool
	port   = flag.Int("port", 8080, "Configures which port to host lime on")
)

const (
	console_height  = 20
	render_chan_len = 2
)

type layout struct {
	x, y          int
	width, height int
	visible       Region
	lastUpdate    int
}
type tbfe struct {
	layout         map[*backend.View]layout
	status_message string
	dorender       chan bool
	lock           sync.Mutex
	dirty          bool
}

func htmlcol(c render.Colour) string {
	return fmt.Sprintf("%02X%02X%02X", c.R, c.G, c.B)
}

func (t *tbfe) renderView(wr io.Writer, v *backend.View, lay layout) {
	p := util.Prof.Enter("render")
	defer p.Exit()

	vr := lay.visible
	runes := v.Buffer().Substr(vr)
	recipie := v.Transform(scheme, vr).Transcribe()
	highlight_line := false
	if b, ok := v.Settings().Get("highlight_line", highlight_line).(bool); ok {
		highlight_line = b
	}
	lastEnd := 0
	for _, reg := range recipie {
		if lastEnd != reg.Region.Begin() {
			io.WriteString(wr, runes[lastEnd:reg.Region.Begin()])
		}
		fmt.Fprintf(wr, "%s", htmlcol(reg.Flavour.Foreground), htmlcol(reg.Flavour.Background), runes[reg.Region.Begin():reg.Region.End()])
		lastEnd = reg.Region.End()
	}
	if lastEnd != vr.End() {
		io.WriteString(wr, v.Buffer().Substr(Region{lastEnd, vr.End()}))
	}
}

func (t *tbfe) clip(v *backend.View, s, e int) Region {
	p := util.Prof.Enter("clip")
	defer p.Exit()
	h := t.layout[v].height
	if e-s > h {
		e = s + h
	} else if e-s < h {
		s = e - h
	}
	if e2, _ := v.Buffer().RowCol(v.Buffer().TextPoint(e, 0)); e2 < e {
		e = e2
	}
	if s < 0 {
		s = 0
	}
	e = s + h
	r := Region{v.Buffer().TextPoint(s, 0), v.Buffer().TextPoint(e, 0)}
	return v.Buffer().LineR(r)
}

func (t *tbfe) Show(v *backend.View, r Region) {
	t.lock.Lock()
	l := t.layout[v]
	t.lock.Unlock()
	if l.visible.Covers(r) {
		return
	}
	p := util.Prof.Enter("show")
	defer p.Exit()

	lv := l.visible

	s1, _ := v.Buffer().RowCol(lv.Begin())
	e1, _ := v.Buffer().RowCol(lv.End())
	s2, _ := v.Buffer().RowCol(r.Begin())
	e2, _ := v.Buffer().RowCol(r.End())

	r1 := Region{s1, e1}
	r2 := Region{s2, e2}

	r3 := r1.Cover(r2)
	diff := 0
	if d1, d2 := Abs(r1.Begin()-r3.Begin()), Abs(r1.End()-r3.End()); d1 > d2 {
		diff = r3.Begin() - r1.Begin()
	} else {
		diff = r3.End() - r1.End()
	}
	r3.A = r1.Begin() + diff
	r3.B = r1.End() + diff

	r3 = t.clip(v, r3.A, r3.B)
	l.visible = r3
	t.lock.Lock()
	t.layout[v] = l
	t.lock.Unlock()
}

func (t *tbfe) VisibleRegion(v *backend.View) Region {
	t.lock.Lock()
	r, ok := t.layout[v]
	t.lock.Unlock()
	if !ok || r.lastUpdate != v.Buffer().ChangeCount() {
		t.Show(v, r.visible)
		t.lock.Lock()
		r = t.layout[v]
		t.lock.Unlock()
	}
	return r.visible
}

func (t *tbfe) StatusMessage(msg string) {
	t.lock.Lock()
	defer t.lock.Unlock()
	t.status_message = msg
}

func (t *tbfe) ErrorMessage(msg string) {
	log4go.Error(msg)
}

// TODO(q): Actually show a dialog
func (t *tbfe) MessageDialog(msg string) {
	log4go.Info(msg)
}

// TODO(q): Actually show a dialog
func (t *tbfe) OkCancelDialog(msg, ok string) bool {
	log4go.Info(msg, ok)
	return false
}

func (t *tbfe) scroll(b Buffer, pos, delta int) {
	t.Show(backend.GetEditor().Console(), Region{b.Size(), b.Size()})
}

var pc = 0

func (t *tbfe) render(w io.Writer) {
	defer func() {
		if r := recover(); r != nil {
			log4go.Error("Panic in renderthread: %v\n%s", r, string(debug.Stack()))
			if pc > 1 {
				panic(r)
			}
			pc++
		}
	}()
	vs := make([]*backend.View, 0, len(t.layout))
	l := make([]layout, 0, len(t.layout))
	for k, v := range t.layout {
		vs = append(vs, k)
		l = append(l, v)
	}
	for i, v := range vs {
		t.renderView(w, v, l[i])
	}
	//	runes := []rune(t.status_message)
}
func (t *tbfe) key(w http.ResponseWriter, req *http.Request) {
	log4go.Debug("key: %s", req)
	kc := req.FormValue("keyCode")
	var kp keys.KeyPress
	v, _ := strconv.ParseInt(kc, 10, 32)

	if req.FormValue("altKey") == "true" {
		kp.Alt = true
	}
	if req.FormValue("ctrlKey") == "true" {
		kp.Ctrl = true
	}
	if req.FormValue("metaKey") == "true" {
		kp.Super = true
	}
	if req.FormValue("shiftKey") == "true" {
		kp.Shift = true
	}
	if !kp.Shift {
		v = int64(unicode.ToLower(rune(v)))
	}
	kp.Key = keys.Key(v)
	backend.GetEditor().HandleInput(kp)
}

func (t *tbfe) view(w http.ResponseWriter, req *http.Request) {
	log4go.Debug("view: %s", req)
	if t.dirty {
		t.dirty = false
		t.render(w)
	} else {
		w.WriteHeader(404)
	}
}

func (t *tbfe) ServeHTTP(w http.ResponseWriter, req *http.Request) {
	s := time.Now()
	w.Header().Set("Content-Type", "text/html")
	log4go.Debug("Serving client: %s", req)

	c := scheme.Spice(&render.ViewRegions{})

	fmt.Fprintf(w, `<title>Lime</title>

`, htmlcol(c.Foreground), htmlcol(c.Background))
	io.WriteString(w, "")
	log4go.Debug("Done serving client: %s", time.Since(s))
}

var clients []*websocket.Conn

func (t *tbfe) WebsocketServer(ws *websocket.Conn) {
	clients = append(clients, ws)

	var buf bytes.Buffer
	t.render(bufio.NewWriter(&buf))
	websocket.Message.Send(ws, buf.Bytes())
	buf.Reset()

	var data map[string]interface{}
	var kp keys.KeyPress
	for {
		err := websocket.JSON.Receive(ws, &data)
		if err != nil {
			log4go.Error(err)
			continue
		}
		//log4go.Debug("Received: %s", data)

		kp.Alt = data["altKey"].(bool)
		kp.Ctrl = data["ctrlKey"].(bool)
		kp.Super = data["metaKey"].(bool)
		kp.Shift = data["shiftKey"].(bool)

		v := int64(data["keyCode"].(float64))
		if !kp.Shift {
			v = int64(unicode.ToLower(rune(v)))
		}
		kp.Key = keys.Key(v)

		backend.GetEditor().HandleInput(kp)
	}
}

func (t *tbfe) SetDirty() {
	t.dirty = true

	var buf bytes.Buffer
	t.render(bufio.NewWriter(&buf))
	for _, ws := range clients {
		websocket.Message.Send(ws, buf.Bytes())
	}
}

func (t *tbfe) loop() {
	backend.OnNew.Add(func(v *backend.View) {
		v.Settings().AddOnChange("lime.frontend.html.render", func(name string) {
			t.SetDirty()
		})
	})
	backend.OnModified.Add(func(v *backend.View) {
		t.SetDirty()
	})
	backend.OnSelectionModified.Add(func(v *backend.View) {
		t.SetDirty()
	})

	ed := backend.GetEditor()
	ed.SetFrontend(t)
	ed.LogInput(false)
	ed.LogCommands(false)
	c := ed.Console()
	if sc, err := textmate.LoadTheme("../../3rdparty/bundles/TextMate-Themes/GlitterBomb.tmTheme"); err != nil {
		log4go.Error(err)
	} else {
		scheme = sc
	}

	defer func() {
		fmt.Println(util.Prof)
	}()

	w := ed.NewWindow()
	v := w.OpenFile("main.go", 0)
	v.Settings().Set("trace", true)
	v.Settings().Set("syntax", "../../3rdparty/bundles/go.tmbundle/Syntaxes/Go.tmLanguage")
	c.Buffer().AddCallback(t.scroll)

	sel := v.Sel()
	sel.Clear()
	//	end := v.Buffer().Size() - 2
	sel.Add(Region{0, 0})
	// sel.Add(Region{end - 22, end - 22})
	// sel.Add(Region{end - 16, end - 20})
	// sel.Add(Region{end - 13, end - 10})

	{
		w, h := 800, 600
		t.lock.Lock()
		t.layout[v] = layout{0, 0, w, h - console_height - 1, Region{}, 0}
		t.layout[c] = layout{0, h - console_height + 1, w, console_height - 5, Region{}, 0}
		t.lock.Unlock()
		t.Show(v, Region{1, 1})
	}
	t.Show(v, Region{100, 100})
	t.Show(v, Region{1, 1})

	go func() {
		ed.Init()
		sublime.Init()
	}()
	log4go.Debug("serving")
	http.HandleFunc("/key", t.key)
	http.HandleFunc("/", t.ServeHTTP)
	http.HandleFunc("/view", t.view)
	http.Handle("/ws", websocket.Handler(t.WebsocketServer))
	if err := http.ListenAndServe(fmt.Sprintf("localhost:%d", *port), nil); err != nil {
		log4go.Error("Error serving: %s", err)
	}
	log4go.Debug("Done")
}

func main() {
	flag.Parse()
	log4go.AddFilter("file", log4go.FINEST, log4go.NewConsoleLogWriter())
	defer func() {
		py.NewLock()
		py.Finalize()
	}()

	var t tbfe
	t.dorender = make(chan bool, render_chan_len)
	t.layout = make(map[*backend.View]layout)
	t.loop()
}
<|endoftext|>


<commit_before>package main

import (
	"bufio"
	"cred-alert/archiveiterator"
	"cred-alert/mimetype"
	"cred-alert/scanners"
	"cred-alert/scanners/filescanner"
	"cred-alert/sniff"
	"fmt"
	"io"
	"log"
	"os"
	"path/filepath"
	"strings"

	"code.cloudfoundry.org/lager"

	"github.com/jessevdk/go-flags"
)

type Opts struct {
	Directory string `short:"d" long:"directory" description:"the directory to scan" value-name:"DIR"`
	File      string `short:"f" long:"file" description:"the file to scan" value-name:"FILE"`
}

func main() {
	var opts Opts

	_, err := flags.ParseArgs(&opts, os.Args)
	if err != nil {
		os.Exit(1)
	}

	logger := lager.NewLogger("cred-alert-cli")
	logger.RegisterSink(lager.NewWriterSink(os.Stderr, lager.DEBUG))

	sniffer := sniff.NewDefaultSniffer()

	if opts.Directory != "" {
		scanDirectory(logger, sniffer, opts.Directory)
		os.Exit(0)
	}

	var f *os.File
	if opts.File != "" {
		var err error
		f, err = os.Open(opts.File)
		if err != nil {
			log.Fatalf("Failed to open file: %s", err.Error())
		}
		defer f.Close()
		scanFile(logger, sniffer, f, f.Name(), []string{})
		os.Exit(0)
	}

	scanFile(logger, sniffer, os.Stdin, "STDIN", []string{})
}

func scanFile(logger lager.Logger, sniffer sniff.Sniffer, r io.Reader, name string, ancestors []string) {
	logger = logger.WithData(lager.Data{
		"filename":  name,
		"ancestors": ancestors,
	})

	br := bufio.NewReader(r)
	mime, ok := mimetype.IsArchive(br)
	ancestors = append(ancestors, fmt.Sprintf(`%s (%s)`, name, mime))

	if ok {
		iterator := archiveiterator.NewIterator(logger, br, mime, name)

		for {
			entry, name := iterator.Next()
			if entry == nil {
				break
			}
			scanFile(logger, sniffer, entry, name, ancestors)
			entry.Close()
		}

		iterator.Close()
	} else {
		if strings.Contains(mime, "text") {
			scanner := filescanner.New(br, name)
			sniffer.Sniff(logger, scanner, handleViolation)
		}
	}
}

func handleViolation(line scanners.Line) error {
	fmt.Printf("Line matches pattern! File: %s, Line Number: %d, Content: %s\n", line.Path, line.LineNumber, line.Content)

	return nil
}

func scanDirectory(logger lager.Logger, sniffer sniff.Sniffer, directoryPath string) {
	stat, err := os.Stat(directoryPath)
	if err != nil {
		log.Fatalf("Cannot read directory %s\n", directoryPath)
	}

	if !stat.IsDir() {
		log.Fatalf("%s is not a directory\n", directoryPath)
	}

	walkFunc := func(path string, info os.FileInfo, err error) error {
		if !info.IsDir() {
			fh, err := os.Open(path)
			if err != nil {
				return err
			}
			defer fh.Close()

			scanner := filescanner.New(fh, fh.Name())
			sniffer.Sniff(logger, scanner, handleViolation)
		}
		return nil
	}

	err = filepath.Walk(directoryPath, walkFunc)
	if err != nil {
		log.Fatalf("Error traversing directory: %v", err)
	}
}
<commit_msg>Defer closing of iterator<commit_after>package main

import (
	"bufio"
	"cred-alert/archiveiterator"
	"cred-alert/mimetype"
	"cred-alert/scanners"
	"cred-alert/scanners/filescanner"
	"cred-alert/sniff"
	"fmt"
	"io"
	"log"
	"os"
	"path/filepath"
	"strings"

	"code.cloudfoundry.org/lager"

	"github.com/jessevdk/go-flags"
)

type Opts struct {
	Directory string `short:"d" long:"directory" description:"the directory to scan" value-name:"DIR"`
	File      string `short:"f" long:"file" description:"the file to scan" value-name:"FILE"`
}

func main() {
	var opts Opts

	_, err := flags.ParseArgs(&opts, os.Args)
	if err != nil {
		os.Exit(1)
	}

	logger := lager.NewLogger("cred-alert-cli")
	logger.RegisterSink(lager.NewWriterSink(os.Stderr, lager.DEBUG))

	sniffer := sniff.NewDefaultSniffer()

	if opts.Directory != "" {
		scanDirectory(logger, sniffer, opts.Directory)
		os.Exit(0)
	}

	var f *os.File
	if opts.File != "" {
		var err error
		f, err = os.Open(opts.File)
		if err != nil {
			log.Fatalf("Failed to open file: %s", err.Error())
		}
		defer f.Close()
		scanFile(logger, sniffer, f, f.Name(), []string{})
		os.Exit(0)
	}

	scanFile(logger, sniffer, os.Stdin, "STDIN", []string{})
}

func scanFile(logger lager.Logger, sniffer sniff.Sniffer, r io.Reader, name string, ancestors []string) {
	logger = logger.WithData(lager.Data{
		"filename":  name,
		"ancestors": ancestors,
	})

	br := bufio.NewReader(r)
	mime, ok := mimetype.IsArchive(br)
	ancestors = append(ancestors, fmt.Sprintf(`%s (%s)`, name, mime))

	if ok {
		iterator := archiveiterator.NewIterator(logger, br, mime, name)
		defer iterator.Close()

		for {
			entry, name := iterator.Next()
			if entry == nil {
				break
			}
			scanFile(logger, sniffer, entry, name, ancestors)
			entry.Close()
		}
	} else {
		if strings.Contains(mime, "text") {
			scanner := filescanner.New(br, name)
			sniffer.Sniff(logger, scanner, handleViolation)
		}
	}
}

func handleViolation(line scanners.Line) error {
	fmt.Printf("Line matches pattern! File: %s, Line Number: %d, Content: %s\n", line.Path, line.LineNumber, line.Content)

	return nil
}

func scanDirectory(logger lager.Logger, sniffer sniff.Sniffer, directoryPath string) {
	stat, err := os.Stat(directoryPath)
	if err != nil {
		log.Fatalf("Cannot read directory %s\n", directoryPath)
	}

	if !stat.IsDir() {
		log.Fatalf("%s is not a directory\n", directoryPath)
	}

	walkFunc := func(path string, info os.FileInfo, err error) error {
		if !info.IsDir() {
			fh, err := os.Open(path)
			if err != nil {
				return err
			}
			defer fh.Close()

			scanner := filescanner.New(fh, fh.Name())
			sniffer.Sniff(logger, scanner, handleViolation)
		}
		return nil
	}

	err = filepath.Walk(directoryPath, walkFunc)
	if err != nil {
		log.Fatalf("Error traversing directory: %v", err)
	}
}
<|endoftext|>


<commit_before>package main

import (
	"cred-alert/scanners"
	"cred-alert/scanners/filescanner"
	"cred-alert/sniff"
	"fmt"
	"log"
	"os"
	"path/filepath"

	"github.com/jessevdk/go-flags"
	"github.com/pivotal-golang/lager"
)

type Opts struct {
	Directory string `short:"d" long:"directory" description:"the directory to scan" value-name:"DIR"`
}

func main() {
	var opts Opts

	_, err := flags.ParseArgs(&opts, os.Args)
	if err != nil {
		os.Exit(1)
	}

	logger := lager.NewLogger("cred-alert-cli")
	logger.RegisterSink(lager.NewWriterSink(os.Stderr, lager.DEBUG))

	sniffer := sniff.NewDefaultSniffer()

	if opts.Directory != "" {
		scanDirectory(logger, sniffer, opts.Directory)
	} else {
		scanner := filescanner.New(os.Stdin, "STDIN")
		sniffer.Sniff(logger, scanner, handleViolation)
	}
}

func handleViolation(line scanners.Line) error {
	fmt.Printf("Line matches pattern! File: %s, Line Number: %d, Content: %s\n", line.Path, line.LineNumber, line.Content)

	return nil
}

func scanDirectory(logger lager.Logger, sniffer sniff.Sniffer, directoryPath string) {
	stat, err := os.Stat(directoryPath)
	if err != nil {
		log.Fatalf("Cannot read directory %s\n", directoryPath)
	}

	if !stat.IsDir() {
		log.Fatalf("%s is not a directory\n", directoryPath)
	}

	walkFunc := func(path string, info os.FileInfo, err error) error {
		if !info.IsDir() {
			fh, err := os.Open(path)
			if err != nil {
				return err
			}
			defer fh.Close()

			scanner := filescanner.New(fh, fh.Name())
			sniffer.Sniff(logger, scanner, handleViolation)
		}
		return nil
	}

	err = filepath.Walk(directoryPath, walkFunc)
	if err != nil {
		log.Fatalf("Error traversing directory: %v", err)
	}
}
<commit_msg>cred-alert-cli takes --file/-f<commit_after>package main

import (
	"cred-alert/scanners"
	"cred-alert/scanners/filescanner"
	"cred-alert/sniff"
	"fmt"
	"log"
	"os"
	"path/filepath"

	"github.com/jessevdk/go-flags"
	"github.com/pivotal-golang/lager"
)

type Opts struct {
	Directory string `short:"d" long:"directory" description:"the directory to scan" value-name:"DIR"`
	File      string `short:"f" long:"file" description:"the file to scan" value-name:"FILE"`
}

func main() {
	var opts Opts

	_, err := flags.ParseArgs(&opts, os.Args)
	if err != nil {
		os.Exit(1)
	}

	logger := lager.NewLogger("cred-alert-cli")
	logger.RegisterSink(lager.NewWriterSink(os.Stderr, lager.DEBUG))

	sniffer := sniff.NewDefaultSniffer()

	if opts.Directory != "" {
		scanDirectory(logger, sniffer, opts.Directory)
		os.Exit(0)
	}

	var f *os.File
	if opts.File != "" {
		var err error
		f, err = os.Open(opts.File)
		if err != nil {
			log.Fatalf("Failed to open file: %s", err.Error())
		}
		defer f.Close()
		scanner := filescanner.New(f, f.Name())
		sniffer.Sniff(logger, scanner, handleViolation)
		os.Exit(0)
	}

	scanner := filescanner.New(os.Stdin, os.Stdin.Name())
	sniffer.Sniff(logger, scanner, handleViolation)
	os.Exit(0)
}

func handleViolation(line scanners.Line) error {
	fmt.Printf("Line matches pattern! File: %s, Line Number: %d, Content: %s\n", line.Path, line.LineNumber, line.Content)

	return nil
}

func scanDirectory(logger lager.Logger, sniffer sniff.Sniffer, directoryPath string) {
	stat, err := os.Stat(directoryPath)
	if err != nil {
		log.Fatalf("Cannot read directory %s\n", directoryPath)
	}

	if !stat.IsDir() {
		log.Fatalf("%s is not a directory\n", directoryPath)
	}

	walkFunc := func(path string, info os.FileInfo, err error) error {
		if !info.IsDir() {
			fh, err := os.Open(path)
			if err != nil {
				return err
			}
			defer fh.Close()

			scanner := filescanner.New(fh, fh.Name())
			sniffer.Sniff(logger, scanner, handleViolation)
		}
		return nil
	}

	err = filepath.Walk(directoryPath, walkFunc)
	if err != nil {
		log.Fatalf("Error traversing directory: %v", err)
	}
}
<|endoftext|>


<commit_before>// Copyright 2014, The Serviced Authors. All rights reserved.
// Use of this source code is governed by a
// license that can be found in the LICENSE file.

package servicestate

import (
	"bytes"

	"github.com/zenoss/glog"
	"github.com/zenoss/serviced/domain/service"

	"fmt"
	"regexp"
	"strconv"
	"strings"
	"text/template"
	"time"

	"github.com/zenoss/serviced/domain"
	"github.com/zenoss/serviced/utils"
)

// An instantiation of a Service.
type ServiceState struct {
	ID          string
	ServiceID   string
	HostID      string
	DockerID    string
	PrivateIP   string
	Scheduled   time.Time
	Terminated  time.Time
	Started     time.Time
	PortMapping map[string][]domain.HostIPAndPort // protocol -> container port (internal) -> host port (external)
	// remove list?  PortMapping:map[6379/tcp:[{HostIP:0.0.0.0 HostPort:49195}]]
	//  i.e. redis:  PortMapping:map[6379/tcp: {HostIP:0.0.0.0 HostPort:49195} ]
	Endpoints  []service.ServiceEndpoint
	HostIP     string
	InstanceID int
}

func plus(a, b int) int {
	return a + b
}

//A new service instance (ServiceState)
func BuildFromService(service *service.Service, hostId string) (serviceState *ServiceState, err error) {
	serviceState = &ServiceState{}
	serviceState.ID, err = utils.NewUUID36()
	if err == nil {
		serviceState.ServiceID = service.ID
		serviceState.HostID = hostId
		serviceState.Scheduled = time.Now()
		serviceState.Endpoints = service.Endpoints
		funcmap := template.FuncMap{
			"plus": plus,
		}
		for j, ep := range serviceState.Endpoints {
			if ep.PortTemplate != "" {
				t := template.Must(template.New("PortTemplate").Funcs(funcmap).Parse(ep.PortTemplate))
				b := bytes.Buffer{}
				err := t.Execute(&b, serviceState)
				if err == nil {
					i, err := strconv.Atoi(b.String())
					if err == nil {
						ep.PortNumber = uint16(i)
					}
				}
				serviceState.Endpoints[j] = ep
			}
		}
	}
	return serviceState, err
}

// Retrieve service container port info.
func (ss *ServiceState) GetHostEndpointInfo(applicationRegex *regexp.Regexp) (hostPort, containerPort uint16, protocol string, match bool) {
	funcmap := template.FuncMap{
		"plus": plus,
	}
	for _, ep := range ss.Endpoints {

		if ep.Purpose == "export" {
			if applicationRegex.MatchString(ep.Application) {
				if ep.PortTemplate != "" {
					t := template.Must(template.New("PortTemplate").Funcs(funcmap).Parse(ep.PortTemplate))
					b := bytes.Buffer{}
					err := t.Execute(&b, ss)
					if err == nil {
						i, err := strconv.Atoi(b.String())
						if err != nil {
							glog.Errorf("%+v", err)
						} else {
							ep.PortNumber = uint16(i)
						}
					}
				}
				portS := fmt.Sprintf("%d/%s", ep.PortNumber, strings.ToLower(ep.Protocol))

				external := ss.PortMapping[portS]
				if len(external) == 0 {
					glog.Warningf("Found match for %s:%s, but no portmapping is available", applicationRegex, portS)
					break
				}

				extPort, err := strconv.ParseUint(external[0].HostPort, 10, 16)
				if err != nil {
					glog.Errorf("Portmap parsing failed for %s:%s %v", applicationRegex, portS, err)
					break
				}
				return uint16(extPort), ep.PortNumber, ep.Protocol, true
			}
		}
	}

	return 0, 0, "", false
}
<commit_msg>Clean up funcmap<commit_after>// Copyright 2014, The Serviced Authors. All rights reserved.
// Use of this source code is governed by a
// license that can be found in the LICENSE file.

package servicestate

import (
	"bytes"

	"github.com/zenoss/glog"
	"github.com/zenoss/serviced/domain/service"

	"fmt"
	"regexp"
	"strconv"
	"strings"
	"text/template"
	"time"

	"github.com/zenoss/serviced/domain"
	"github.com/zenoss/serviced/utils"
)

// Function map for evaluating PortTemplate fields
var funcmap = template.FuncMap{
	"plus": func(a, b int) int {
		return a + b
	},
}

// An instantiation of a Service.
type ServiceState struct {
	ID          string
	ServiceID   string
	HostID      string
	DockerID    string
	PrivateIP   string
	Scheduled   time.Time
	Terminated  time.Time
	Started     time.Time
	PortMapping map[string][]domain.HostIPAndPort // protocol -> container port (internal) -> host port (external)
	// remove list?  PortMapping:map[6379/tcp:[{HostIP:0.0.0.0 HostPort:49195}]]
	//  i.e. redis:  PortMapping:map[6379/tcp: {HostIP:0.0.0.0 HostPort:49195} ]
	Endpoints  []service.ServiceEndpoint
	HostIP     string
	InstanceID int
}

//A new service instance (ServiceState)
func BuildFromService(service *service.Service, hostId string) (serviceState *ServiceState, err error) {
	serviceState = &ServiceState{}
	serviceState.ID, err = utils.NewUUID36()
	if err == nil {
		serviceState.ServiceID = service.ID
		serviceState.HostID = hostId
		serviceState.Scheduled = time.Now()
		serviceState.Endpoints = service.Endpoints
		for j, ep := range serviceState.Endpoints {
			if ep.PortTemplate != "" {
				t := template.Must(template.New("PortTemplate").Funcs(funcmap).Parse(ep.PortTemplate))
				b := bytes.Buffer{}
				err := t.Execute(&b, serviceState)
				if err == nil {
					i, err := strconv.Atoi(b.String())
					if err == nil {
						ep.PortNumber = uint16(i)
					}
				}
				serviceState.Endpoints[j] = ep
			}
		}
	}
	return serviceState, err
}

// Retrieve service container port info.
func (ss *ServiceState) GetHostEndpointInfo(applicationRegex *regexp.Regexp) (hostPort, containerPort uint16, protocol string, match bool) {
	for _, ep := range ss.Endpoints {

		if ep.Purpose == "export" {
			if applicationRegex.MatchString(ep.Application) {
				if ep.PortTemplate != "" {
					t := template.Must(template.New("PortTemplate").Funcs(funcmap).Parse(ep.PortTemplate))
					b := bytes.Buffer{}
					err := t.Execute(&b, ss)
					if err == nil {
						i, err := strconv.Atoi(b.String())
						if err != nil {
							glog.Errorf("%+v", err)
						} else {
							ep.PortNumber = uint16(i)
						}
					}
				}
				portS := fmt.Sprintf("%d/%s", ep.PortNumber, strings.ToLower(ep.Protocol))

				external := ss.PortMapping[portS]
				if len(external) == 0 {
					glog.Warningf("Found match for %s:%s, but no portmapping is available", applicationRegex, portS)
					break
				}

				extPort, err := strconv.ParseUint(external[0].HostPort, 10, 16)
				if err != nil {
					glog.Errorf("Portmap parsing failed for %s:%s %v", applicationRegex, portS, err)
					break
				}
				return uint16(extPort), ep.PortNumber, ep.Protocol, true
			}
		}
	}

	return 0, 0, "", false
}
<|endoftext|>


<commit_before>package memsearch

import (
	"testing"

	"github.com/manishrjain/gocrud/testx"
)

func initialize(t *testing.T) *MemSearch {
	ms := new(MemSearch)
	ms.Init()

	testx.AddDocs(ms, t)
	return ms
}

func TestNewAndFilter(t *testing.T) {
	ms := initialize(t)
	testx.RunAndFilter(ms, t)
}

var soln = [...]string{
	"m81",
	"ngc 3370",
	"galaxy ngc 1512",
	"ngc 123",
	"whirlpool galaxy",
	"sombrero galaxy",
}

func TestNewOrFilter(t *testing.T) {
	ms := initialize(t)
	testx.RunOrFilter(ms, t)
}
<commit_msg>Fix Memsearch test<commit_after>package memsearch

import (
	"testing"

	"github.com/manishrjain/gocrud/testx"
)

func initialize() *MemSearch {
	ms := new(MemSearch)
	ms.Init()

	testx.AddDocs(ms)
	return ms
}

func TestNewAndFilter(t *testing.T) {
	testx.RunAndFilter(ms, t)
}

var soln = [...]string{
	"m81",
	"ngc 3370",
	"galaxy ngc 1512",
	"ngc 123",
	"whirlpool galaxy",
	"sombrero galaxy",
}

func TestNewOrFilter(t *testing.T) {
	testx.RunOrFilter(ms, t)
}

var ms *MemSearch

func init() {
	ms = initialize()
}
<|endoftext|>