<commit_before>//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package convert

import (
	"encoding/json"
	"errors"
	"fmt"
	"io/ioutil"
	"os"
	"os/exec"
	"path/filepath"
	"strings"
	"time"

	"github.com/Sirupsen/logrus"
	"github.com/appc/spec/schema"
	"github.com/appc/spec/schema/types"
	"github.com/opencontainers/specs"
)

type IsolatorCapSet struct {
	Sets []string `json:"set"`
}

type ResourceMem struct {
	Limit string `json:"limit"`
}

type ResourceCPU struct {
	Limit string `json:"limit"`
}

var manifestName string

func Oci2aciManifest(ociPath string) (string, error) {
	if bValidate := validateOCIProc(ociPath); bValidate != true {
		err := errors.New("Invalid oci bundle.")
		return "", err
	}

	dirWork := createWorkDir()
	// convert layout
	aciManifestPath, err := convertLayout(ociPath, dirWork)
	if err != nil {
		return "", err
	}
	return aciManifestPath, err

}

func Oci2aciImage(ociPath string) (string, error) {
	if bValidate := validateOCIProc(ociPath); bValidate != true {
		err := errors.New("Invalid oci bundle.")
		return "", err
	}

	dirWork := createWorkDir()
	// First, convert layout
	_, err := convertLayout(ociPath, dirWork)
	if err != nil {
		return "", err
	}

	// Second, build image
	aciImgPath, err := buildACI(dirWork)

	return aciImgPath, err

}

// Entry point of oci2aci,
// First convert oci layout to aci layout, then build aci layout to image.
func RunOCI2ACI(args []string, flagDebug bool, flagName string) error {
	var srcPath, dstPath string

	srcPath = args[0]
	if len(args) == 1 {
		dstPath = ""
	} else {
		dstPath = args[1]
		ext := filepath.Ext(dstPath)
		if ext != schema.ACIExtension {
			errStr := fmt.Sprintf("Extension must be %s (given %s)", schema.ACIExtension, ext)
			err := errors.New(errStr)
			return err
		}
	}

	if flagDebug {
		logrus.SetLevel(logrus.DebugLevel)
	} else {
		logrus.SetLevel(logrus.InfoLevel)
	}

	manifestName = flagName
	_, err := types.NewACName(manifestName)
	if err != nil {
		return err
	}

	if bValidate := validateOCIProc(srcPath); bValidate != true {
		logrus.Infof("Conversion stop.")
		return nil
	}

	dirWork := createWorkDir()
	// First, convert layout
	manifestPath, err := convertLayout(srcPath, dirWork)
	if err != nil {
		logrus.Debugf("Conversion from oci to aci layout failed: %v", err)
	} else {
		if dstPath != "" {
			logrus.Debugf("Manifest file converted successfully.")
		} else {
			logrus.Debugf("Manifest:%v generated successfully.", manifestPath)
		}
	}
	// Second, build image
	imgPath, err := buildACI(dirWork)
	if err != nil {
		logrus.Debugf("Generate aci image failed:%v", err)
	} else {
		if dstPath != "" {
			logrus.Debugf("ACI image converted successfully.")
		} else {
			logrus.Debugf("Image:%v generated successfully.", imgPath)
		}
	}
	// Save aci image to the path user specified
	if dstPath != "" {
		if err = run(exec.Command("mv", imgPath, dstPath)); err != nil {
			logrus.Debugf("Store aci image failed:%v", err)
		} else {
			logrus.Debugf("Image:%v generated successfully", dstPath)
		}
		run(exec.Command("mv", manifestPath, "./"))
		run(exec.Command("rm", "-rf", dirWork))
	}

	return nil
}

// Create work directory for the conversion output
func createWorkDir() string {
	idir, err := ioutil.TempDir("", "oci2aci")
	if err != nil {
		return ""
	}
	rootfs := filepath.Join(idir, "rootfs")
	os.MkdirAll(rootfs, 0755)

	data := []byte{}
	if err := ioutil.WriteFile(filepath.Join(idir, "manifest"), data, 0644); err != nil {
		return ""
	}
	return idir
}

// The structure of appc manifest:
// 1.acKind
// 2. acVersion
// 3. name
// 4. labels
//	4.1 version
//	4.2 os
//	4.3 arch
// 5. app
//	5.1 exec
//	5.2 user
//	5.3 group
//	5.4 eventHandlers
//	5.5 workingDirectory
//	5.6 environment
//	5.7 mountPoints
//	5.8 ports
//      5.9 isolators
// 6. annotations
//	6.1 created
//	6.2 authors
//	6.3 homepage
//	6.4 documentation
// 7. dependencies
//	7.1 imageName
//	7.2 imageID
//	7.3 labels
//	7.4 size
// 8. pathWhitelist

func genManifest(path string) *schema.ImageManifest {
	// Get runtime.json and config.json
	runtimePath := path + "/runtime.json"
	configPath := path + "/config.json"

	runtime, err := ioutil.ReadFile(runtimePath)
	if err != nil {
		logrus.Debugf("Open file runtime.json failed: %v", err)
		return nil
	}

	config, err := ioutil.ReadFile(configPath)
	if err != nil {
		logrus.Debugf("Open file config.json failed: %v", err)
		return nil
	}

	var spec specs.LinuxSpec
	err = json.Unmarshal(config, &spec)
	if err != nil {
		logrus.Debugf("Unmarshal config.json failed: %v", err)
		return nil
	}

	var runSpec specs.LinuxRuntimeSpec
	err = json.Unmarshal(runtime, &runSpec)
	if err != nil {
		logrus.Debugf("Unmarshal runtime.json failed: %v", err)
		return nil
	}
	// Begin to convert runtime.json/config.json to manifest
	m := new(schema.ImageManifest)

	// 1. Assemble "acKind" field
	m.ACKind = schema.ImageManifestKind

	// 2. Assemble "acVersion" field
	m.ACVersion = schema.AppContainerVersion

	// 3. Assemble "name" field
	m.Name = types.ACIdentifier(manifestName)

	// 4. Assemble "labels" field
	// 4.1 "version"
	label := new(types.Label)
	label.Name = types.ACIdentifier("version")
	label.Value = spec.Version
	m.Labels = append(m.Labels, *label)
	// 4.2 "os"
	label = new(types.Label)
	label.Name = types.ACIdentifier("os")
	label.Value = spec.Platform.OS
	m.Labels = append(m.Labels, *label)
	// 4.3 "arch"
	label = new(types.Label)
	label.Name = types.ACIdentifier("arch")
	label.Value = spec.Platform.Arch
	m.Labels = append(m.Labels, *label)

	// 5. Assemble "app" field
	app := new(types.App)
	// 5.1 "exec"
	app.Exec = spec.Process.Args

	prefixDir := ""
	//var exeStr string
	if app.Exec == nil {
		app.Exec = append(app.Exec, "/bin/sh")
	} else {
		if !filepath.IsAbs(app.Exec[0]) {
			if spec.Process.Cwd == "" {
				prefixDir = "/"
			} else {
				prefixDir = spec.Process.Cwd
			}
		}
		app.Exec[0] = prefixDir + app.Exec[0]
	}

	// 5.2 "user"
	app.User = fmt.Sprintf("%d", spec.Process.User.UID)
	// 5.3 "group"
	app.Group = fmt.Sprintf("%d", spec.Process.User.GID)
	for index := range spec.Process.User.AdditionalGids {
		app.SupplementaryGIDs = append(app.SupplementaryGIDs, int(spec.Process.User.AdditionalGids[index]))
	}
	// 5.4 "eventHandlers"
	event := new(types.EventHandler)
	event.Name = "pre-start"
	for index := range runSpec.Hooks.Prestart {
		event.Exec = append(event.Exec, runSpec.Hooks.Prestart[index].Path)
		event.Exec = append(event.Exec, runSpec.Hooks.Prestart[index].Args...)
		event.Exec = append(event.Exec, runSpec.Hooks.Prestart[index].Env...)
	}
	if len(event.Exec) == 0 {
		event.Exec = append(event.Exec, "/bin/echo")
		event.Exec = append(event.Exec, "-n")
	}
	app.EventHandlers = append(app.EventHandlers, *event)
	event = new(types.EventHandler)
	event.Name = "post-stop"
	for index := range runSpec.Hooks.Poststop {
		event.Exec = append(event.Exec, runSpec.Hooks.Poststop[index].Path)
		event.Exec = append(event.Exec, runSpec.Hooks.Poststop[index].Args...)
		event.Exec = append(event.Exec, runSpec.Hooks.Poststop[index].Env...)
	}
	if len(event.Exec) == 0 {
		event.Exec = append(event.Exec, "/bin/echo")
		event.Exec = append(event.Exec, "-n")
	}
	app.EventHandlers = append(app.EventHandlers, *event)
	// 5.5 "workingDirectory"
	app.WorkingDirectory = spec.Process.Cwd
	// 5.6 "environment"
	env := new(types.EnvironmentVariable)
	for index := range spec.Process.Env {
		s := strings.Split(spec.Process.Env[index], "=")
		env.Name = s[0]
		env.Value = s[1]
		app.Environment = append(app.Environment, *env)
	}

	// 5.7 "mountPoints"
	for index := range spec.Mounts {
		mount := new(types.MountPoint)
		mount.Name = types.ACName(spec.Mounts[index].Name)
		mount.Path = spec.Mounts[index].Path
		mount.ReadOnly = false
		app.MountPoints = append(app.MountPoints, *mount)
	}

	// 5.8 "ports"

	// 5.9 "isolators"
	if runSpec.Linux.Resources != nil {
		if runSpec.Linux.Resources.CPU.Quota != 0 {
			cpuLimt := new(ResourceCPU)
			cpuLimt.Limit = fmt.Sprintf("%dm", runSpec.Linux.Resources.CPU.Quota)
			isolator := new(types.Isolator)
			isolator.Name = types.ACIdentifier("resource/cpu")
			bytes, _ := json.Marshal(cpuLimt)

			valueRaw := json.RawMessage(bytes)
			isolator.ValueRaw = &valueRaw

			app.Isolators = append(app.Isolators, *isolator)
		}
		if runSpec.Linux.Resources.Memory.Limit != 0 {
			memLimt := new(ResourceMem)
			memLimt.Limit = fmt.Sprintf("%dG", runSpec.Linux.Resources.Memory.Limit/(1024*1024*1024))
			isolator := new(types.Isolator)
			isolator.Name = types.ACIdentifier("resource/memory")
			bytes, _ := json.Marshal(memLimt)

			valueRaw := json.RawMessage(bytes)
			isolator.ValueRaw = &valueRaw

			app.Isolators = append(app.Isolators, *isolator)
		}
	}

	if len(spec.Linux.Capabilities) != 0 {
		isolatorCapSet := new(IsolatorCapSet)
		isolatorCapSet.Sets = append(isolatorCapSet.Sets, spec.Linux.Capabilities...)

		isolator := new(types.Isolator)
		isolator.Name = types.ACIdentifier(types.LinuxCapabilitiesRetainSetName)
		bytes, _ := json.Marshal(isolatorCapSet)

		valueRaw := json.RawMessage(bytes)
		isolator.ValueRaw = &valueRaw

		app.Isolators = append(app.Isolators, *isolator)
	}

	m.App = app

	// 6. "annotations"
	anno := new(types.Annotation)
	anno.Name = types.ACIdentifier("created")
	anno.Value = time.Now().Format(time.RFC3339)
	m.Annotations = append(m.Annotations, *anno)
	anno.Name = types.ACIdentifier("authors")
	anno.Value = "chengtiesheng@huawei.com"
	m.Annotations = append(m.Annotations, *anno)
	anno.Name = types.ACIdentifier("homepage")
	anno.Value = "https://github.com/huawei-openlab/oci2aci"
	m.Annotations = append(m.Annotations, *anno)
	// 7. "dependencies"

	// 8. "pathWhitelist"

	return m
}

// Convert OCI layout to ACI layout
func convertLayout(srcPath, dstPath string) (string, error) {
	src, _ := filepath.Abs(srcPath)
	src += "/rootfs"
	if err := run(exec.Command("cp", "-rf", src, dstPath)); err != nil {
		return "", err
	}

	m := genManifest(srcPath)

	bytes, err := json.MarshalIndent(m, "", "\t")
	if err != nil {
		return "", err
	}

	manifestPath := dstPath + "/manifest"

	ioutil.WriteFile(manifestPath, bytes, 0644)
	return manifestPath, nil
}
<commit_msg>Minior bug fixed<commit_after>//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package convert

import (
	"encoding/json"
	"errors"
	"fmt"
	"io/ioutil"
	"os"
	"os/exec"
	"path/filepath"
	"strings"
	"time"

	"github.com/Sirupsen/logrus"
	"github.com/appc/spec/schema"
	"github.com/appc/spec/schema/types"
	"github.com/opencontainers/specs"
)

type IsolatorCapSet struct {
	Sets []string `json:"set"`
}

type ResourceMem struct {
	Limit string `json:"limit"`
}

type ResourceCPU struct {
	Limit string `json:"limit"`
}

var manifestName string

func Oci2aciManifest(ociPath string) (string, error) {
	if bValidate := validateOCIProc(ociPath); bValidate != true {
		err := errors.New("Invalid oci bundle.")
		return "", err
	}

	dirWork := createWorkDir()
	// convert layout
	aciManifestPath, err := convertLayout(ociPath, dirWork)
	if err != nil {
		return "", err
	}
	return aciManifestPath, err

}

func Oci2aciImage(ociPath string) (string, error) {
	if bValidate := validateOCIProc(ociPath); bValidate != true {
		err := errors.New("Invalid oci bundle.")
		return "", err
	}

	dirWork := createWorkDir()
	// First, convert layout
	_, err := convertLayout(ociPath, dirWork)
	if err != nil {
		return "", err
	}

	// Second, build image
	aciImgPath, err := buildACI(dirWork)

	return aciImgPath, err

}

// Entry point of oci2aci,
// First convert oci layout to aci layout, then build aci layout to image.
func RunOCI2ACI(args []string, flagDebug bool, flagName string) error {
	var srcPath, dstPath string

	srcPath = args[0]
	if len(args) == 1 {
		dstPath = ""
	} else {
		dstPath = args[1]
		ext := filepath.Ext(dstPath)
		if ext != schema.ACIExtension {
			errStr := fmt.Sprintf("Extension must be %s (given %s)", schema.ACIExtension, ext)
			err := errors.New(errStr)
			return err
		}
	}

	if flagDebug {
		logrus.SetLevel(logrus.DebugLevel)
	} else {
		logrus.SetLevel(logrus.InfoLevel)
	}

	manifestName = flagName
	_, err := types.NewACName(manifestName)
	if err != nil {
		return err
	}

	if bValidate := validateOCIProc(srcPath); bValidate != true {
		logrus.Infof("Conversion stop.")
		return nil
	}

	dirWork := createWorkDir()
	// First, convert layout
	manifestPath, err := convertLayout(srcPath, dirWork)
	if err != nil {
		logrus.Debugf("Conversion from oci to aci layout failed: %v", err)
	} else {
		if dstPath != "" {
			logrus.Debugf("Manifest file converted successfully.")
		} else {
			logrus.Debugf("Manifest:%v generated successfully.", manifestPath)
		}
	}
	// Second, build image
	imgPath, err := buildACI(dirWork)
	if err != nil {
		logrus.Debugf("Generate aci image failed:%v", err)
	} else {
		if dstPath != "" {
			logrus.Debugf("ACI image converted successfully.")
		} else {
			logrus.Debugf("Image:%v generated successfully.", imgPath)
		}
	}
	// Save aci image to the path user specified
	if dstPath != "" {
		if err = run(exec.Command("mv", imgPath, dstPath)); err != nil {
			logrus.Debugf("Store aci image failed:%v", err)
		} else {
			logrus.Debugf("Image:%v generated successfully", dstPath)
		}
		run(exec.Command("mv", manifestPath, "./"))
		run(exec.Command("rm", "-rf", dirWork))
	}

	return nil
}

// Create work directory for the conversion output
func createWorkDir() string {
	idir, err := ioutil.TempDir("", "oci2aci")
	if err != nil {
		return ""
	}
	rootfs := filepath.Join(idir, "rootfs")
	os.MkdirAll(rootfs, 0755)

	data := []byte{}
	if err := ioutil.WriteFile(filepath.Join(idir, "manifest"), data, 0644); err != nil {
		return ""
	}
	return idir
}

// The structure of appc manifest:
// 1.acKind
// 2. acVersion
// 3. name
// 4. labels
//	4.1 version
//	4.2 os
//	4.3 arch
// 5. app
//	5.1 exec
//	5.2 user
//	5.3 group
//	5.4 eventHandlers
//	5.5 workingDirectory
//	5.6 environment
//	5.7 mountPoints
//	5.8 ports
//      5.9 isolators
// 6. annotations
//	6.1 created
//	6.2 authors
//	6.3 homepage
//	6.4 documentation
// 7. dependencies
//	7.1 imageName
//	7.2 imageID
//	7.3 labels
//	7.4 size
// 8. pathWhitelist

func genManifest(path string) *schema.ImageManifest {
	// Get runtime.json and config.json
	runtimePath := path + "/runtime.json"
	configPath := path + "/config.json"

	runtime, err := ioutil.ReadFile(runtimePath)
	if err != nil {
		logrus.Debugf("Open file runtime.json failed: %v", err)
		return nil
	}

	config, err := ioutil.ReadFile(configPath)
	if err != nil {
		logrus.Debugf("Open file config.json failed: %v", err)
		return nil
	}

	var spec specs.LinuxSpec
	err = json.Unmarshal(config, &spec)
	if err != nil {
		logrus.Debugf("Unmarshal config.json failed: %v", err)
		return nil
	}

	var runSpec specs.LinuxRuntimeSpec
	err = json.Unmarshal(runtime, &runSpec)
	if err != nil {
		logrus.Debugf("Unmarshal runtime.json failed: %v", err)
		return nil
	}
	// Begin to convert runtime.json/config.json to manifest
	m := new(schema.ImageManifest)

	// 1. Assemble "acKind" field
	m.ACKind = schema.ImageManifestKind

	// 2. Assemble "acVersion" field
	m.ACVersion = schema.AppContainerVersion

	// 3. Assemble "name" field
	m.Name = types.ACIdentifier(manifestName)

	// 4. Assemble "labels" field
	// 4.1 "version"
	label := new(types.Label)
	label.Name = types.ACIdentifier("version")
	label.Value = spec.Version
	m.Labels = append(m.Labels, *label)
	// 4.2 "os"
	label = new(types.Label)
	label.Name = types.ACIdentifier("os")
	label.Value = spec.Platform.OS
	m.Labels = append(m.Labels, *label)
	// 4.3 "arch"
	label = new(types.Label)
	label.Name = types.ACIdentifier("arch")
	label.Value = spec.Platform.Arch
	m.Labels = append(m.Labels, *label)

	// 5. Assemble "app" field
	app := new(types.App)
	// 5.1 "exec"
	app.Exec = spec.Process.Args

	prefixDir := ""
	//var exeStr string
	if app.Exec == nil {
		app.Exec = append(app.Exec, "/bin/sh")
	} else {
		if !filepath.IsAbs(app.Exec[0]) {
			if spec.Process.Cwd == "" {
				prefixDir = "/"
			} else {
				prefixDir = spec.Process.Cwd
			}
		}
		app.Exec[0] = prefixDir + app.Exec[0]
		// If exec is /xx, it must be wrong, /bin will be prefixed it.
		var res []string
		res = strings.SplitAfter(app.Exec[0], "/")
		if len(res) <= 2 {
			app.Exec[0] = "/bin" + app.Exec[0]
		}
	}

	// 5.2 "user"
	app.User = fmt.Sprintf("%d", spec.Process.User.UID)
	// 5.3 "group"
	app.Group = fmt.Sprintf("%d", spec.Process.User.GID)
	for index := range spec.Process.User.AdditionalGids {
		app.SupplementaryGIDs = append(app.SupplementaryGIDs, int(spec.Process.User.AdditionalGids[index]))
	}
	// 5.4 "eventHandlers"
	event := new(types.EventHandler)
	event.Name = "pre-start"
	for index := range runSpec.Hooks.Prestart {
		event.Exec = append(event.Exec, runSpec.Hooks.Prestart[index].Path)
		event.Exec = append(event.Exec, runSpec.Hooks.Prestart[index].Args...)
		event.Exec = append(event.Exec, runSpec.Hooks.Prestart[index].Env...)
	}
	if len(event.Exec) == 0 {
		event.Exec = append(event.Exec, "/bin/echo")
		event.Exec = append(event.Exec, "-n")
	}
	app.EventHandlers = append(app.EventHandlers, *event)
	event = new(types.EventHandler)
	event.Name = "post-stop"
	for index := range runSpec.Hooks.Poststop {
		event.Exec = append(event.Exec, runSpec.Hooks.Poststop[index].Path)
		event.Exec = append(event.Exec, runSpec.Hooks.Poststop[index].Args...)
		event.Exec = append(event.Exec, runSpec.Hooks.Poststop[index].Env...)
	}
	if len(event.Exec) == 0 {
		event.Exec = append(event.Exec, "/bin/echo")
		event.Exec = append(event.Exec, "-n")
	}
	app.EventHandlers = append(app.EventHandlers, *event)
	// 5.5 "workingDirectory"
	app.WorkingDirectory = spec.Process.Cwd
	// 5.6 "environment"
	env := new(types.EnvironmentVariable)
	for index := range spec.Process.Env {
		s := strings.Split(spec.Process.Env[index], "=")
		env.Name = s[0]
		env.Value = s[1]
		app.Environment = append(app.Environment, *env)
	}

	// 5.7 "mountPoints"
	for index := range spec.Mounts {
		mount := new(types.MountPoint)
		mount.Name = types.ACName(spec.Mounts[index].Name)
		mount.Path = spec.Mounts[index].Path
		mount.ReadOnly = false
		app.MountPoints = append(app.MountPoints, *mount)
	}

	// 5.8 "ports"

	// 5.9 "isolators"
	if runSpec.Linux.Resources != nil {
		if runSpec.Linux.Resources.CPU.Quota != 0 {
			cpuLimt := new(ResourceCPU)
			cpuLimt.Limit = fmt.Sprintf("%dm", runSpec.Linux.Resources.CPU.Quota)
			isolator := new(types.Isolator)
			isolator.Name = types.ACIdentifier("resource/cpu")
			bytes, _ := json.Marshal(cpuLimt)

			valueRaw := json.RawMessage(bytes)
			isolator.ValueRaw = &valueRaw

			app.Isolators = append(app.Isolators, *isolator)
		}
		if runSpec.Linux.Resources.Memory.Limit != 0 {
			memLimt := new(ResourceMem)
			memLimt.Limit = fmt.Sprintf("%dG", runSpec.Linux.Resources.Memory.Limit/(1024*1024*1024))
			isolator := new(types.Isolator)
			isolator.Name = types.ACIdentifier("resource/memory")
			bytes, _ := json.Marshal(memLimt)

			valueRaw := json.RawMessage(bytes)
			isolator.ValueRaw = &valueRaw

			app.Isolators = append(app.Isolators, *isolator)
		}
	}

	if len(spec.Linux.Capabilities) != 0 {
		isolatorCapSet := new(IsolatorCapSet)
		isolatorCapSet.Sets = append(isolatorCapSet.Sets, spec.Linux.Capabilities...)

		isolator := new(types.Isolator)
		isolator.Name = types.ACIdentifier(types.LinuxCapabilitiesRetainSetName)
		bytes, _ := json.Marshal(isolatorCapSet)

		valueRaw := json.RawMessage(bytes)
		isolator.ValueRaw = &valueRaw

		app.Isolators = append(app.Isolators, *isolator)
	}

	m.App = app

	// 6. "annotations"
	anno := new(types.Annotation)
	anno.Name = types.ACIdentifier("created")
	anno.Value = time.Now().Format(time.RFC3339)
	m.Annotations = append(m.Annotations, *anno)
	anno.Name = types.ACIdentifier("authors")
	anno.Value = "chengtiesheng@huawei.com"
	m.Annotations = append(m.Annotations, *anno)
	anno.Name = types.ACIdentifier("homepage")
	anno.Value = "https://github.com/huawei-openlab/oci2aci"
	m.Annotations = append(m.Annotations, *anno)
	// 7. "dependencies"

	// 8. "pathWhitelist"

	return m
}

// Convert OCI layout to ACI layout
func convertLayout(srcPath, dstPath string) (string, error) {
	src, _ := filepath.Abs(srcPath)
	src += "/rootfs"
	if err := run(exec.Command("cp", "-rf", src, dstPath)); err != nil {
		return "", err
	}

	m := genManifest(srcPath)

	bytes, err := json.MarshalIndent(m, "", "\t")
	if err != nil {
		return "", err
	}

	manifestPath := dstPath + "/manifest"

	ioutil.WriteFile(manifestPath, bytes, 0644)
	return manifestPath, nil
}
<|endoftext|>


<commit_before>package crawler

import "net/url"

type taskFromDB struct {
	Host string
	URL  string
}

func (db *DB) readNotLoadedURLs(baseHosts []string, cnt int, ch chan<- taskFromDB) error {
	cntPerHost := cnt / len(baseHosts)
	if cntPerHost < 1 {
		cntPerHost = 1
	}

	leftCnts := make(map[string]int)
	for _, host := range baseHosts {
		leftCnts[host] = cntPerHost
	}

	return db.View(func(tx *Tx) error {
		b := tx.Bucket(DbBucketURLs)
		c := b.Cursor()

		var urlVal DbURL
		for urlKey, urlData := c.First(); urlKey != nil; urlKey, urlData = c.Next() {
			parsedURL, err := url.Parse(string(urlKey[:]))
			leftCnt, found := leftCnts[parsedURL.Host]
			if !found || leftCnt <= 0 {
				continue
			}

			_, err = urlVal.UnmarshalMsg(urlData)
			if err != nil {
				return err
			}

			if urlVal.ErrorType == PageTypeNone {
				ch <- taskFromDB{Host: parsedURL.Host, URL: string(urlKey[:])}
				leftCnts[parsedURL.Host]--
			}

			totalCnt := 0
			for _, i := range leftCnts {
				totalCnt += i
			}
			if totalCnt <= 0 {
				break
			}
		}

		for host, cntLeft := range leftCnts {
			if cntLeft > 0 {
				baseURL := url.URL{Host: host, Scheme: "http"}
				exists, err := b.Get([]byte(baseURL.String()), &urlVal)
				if err != nil {
					return err
				}
				if !exists {
					ch <- taskFromDB{Host: host, URL: baseURL.String()}
				}
			}
		}

		return nil
	})
}
<commit_msg>операция показа статистики по базе<commit_after>package crawler

import (
	"fmt"
	"net/url"
)

type taskFromDB struct {
	Host string
	URL  string
}

func (db *DB) readNotLoadedURLs(baseHosts map[string]int, ch chan<- taskFromDB) error {
	leftCnts := make(map[string]int, len(baseHosts))
	for k, v := range baseHosts {
		leftCnts[k] = v
	}

	return db.View(func(tx *Tx) error {
		b := tx.Bucket(DbBucketURLs)
		c := b.Cursor()

		var urlVal DbURL
		for urlKey, urlData := c.First(); urlKey != nil; urlKey, urlData = c.Next() {
			parsedURL, err := url.Parse(string(urlKey[:]))
			leftCnt, found := leftCnts[parsedURL.Host]
			if !found || leftCnt <= 0 {
				continue
			}

			_, err = urlVal.UnmarshalMsg(urlData)
			if err != nil {
				return err
			}

			if urlVal.ErrorType == PageTypeNone {
				ch <- taskFromDB{Host: parsedURL.Host, URL: string(urlKey[:])}
				leftCnts[parsedURL.Host]--
			}

			totalCnt := 0
			for _, i := range leftCnts {
				totalCnt += i
			}
			if totalCnt <= 0 {
				break
			}
		}

		for host, cntLeft := range leftCnts {
			if cntLeft > 0 {
				u := URLFromHost(host)
				exists, err := b.Get([]byte(u), &urlVal)
				if err != nil {
					return err
				}
				if !exists {
					ch <- taskFromDB{Host: host, URL: u}
				}
			}
		}

		return nil
	})
}

func (db *DB) showStatistics() error {
	return db.View(func(tx *Tx) error {
		bContents := tx.Bucket(DbBucketContents)
		fmt.Printf("\nStatistics:\n")
		fmt.Printf("Contents len = %d\n", bContents.Stats().KeyN)

		bURLs := tx.Bucket(DbBucketURLs)
		fmt.Printf("bURLs len = %d\n", bURLs.Stats().KeyN)

		bWrongURLs := tx.Bucket(DbBucketWrongURLs)
		fmt.Printf("WrongURLs len = %d\n", bWrongURLs.Stats().KeyN)
		return nil
	})
}
<|endoftext|>


<commit_before>// package swarm implements a connection muxer with a pair of channels
// to synchronize all network communication.
package swarm

import (
	"errors"
	"fmt"
	"sync"

	conn "github.com/jbenet/go-ipfs/net/conn"
	msg "github.com/jbenet/go-ipfs/net/message"
	peer "github.com/jbenet/go-ipfs/peer"
	u "github.com/jbenet/go-ipfs/util"
	ctxc "github.com/jbenet/go-ipfs/util/ctxcloser"
	"github.com/jbenet/go-ipfs/util/eventlog"

	context "github.com/jbenet/go-ipfs/Godeps/_workspace/src/code.google.com/p/go.net/context"
	ma "github.com/jbenet/go-ipfs/Godeps/_workspace/src/github.com/jbenet/go-multiaddr"
)

var log = eventlog.Logger("swarm")

// ErrAlreadyOpen signals that a connection to a peer is already open.
var ErrAlreadyOpen = errors.New("Error: Connection to this peer already open.")

// ListenErr contains a set of errors mapping to each of the swarms addresses.
// Used to return multiple errors, as in listen.
type ListenErr struct {
	Errors []error
}

func (e *ListenErr) Error() string {
	if e == nil {
		return "<nil error>"
	}
	var out string
	for i, v := range e.Errors {
		if v != nil {
			out += fmt.Sprintf("%d: %s\n", i, v)
		}
	}
	return out
}

// Swarm is a connection muxer, allowing connections to other peers to
// be opened and closed, while still using the same Chan for all
// communication. The Chan sends/receives Messages, which note the
// destination or source Peer.
type Swarm struct {

	// local is the peer this swarm represents
	local peer.Peer

	// peers is a collection of peers for swarm to use
	peers peer.Peerstore

	// Swarm includes a Pipe object.
	*msg.Pipe

	// errChan is the channel of errors.
	errChan chan error

	// conns are the open connections the swarm is handling.
	// these are MultiConns, which multiplex multiple separate underlying Conns.
	conns     conn.MultiConnMap
	connsLock sync.RWMutex

	// listeners for each network address
	listeners []conn.Listener

	// ContextCloser
	ctxc.ContextCloser
}

// NewSwarm constructs a Swarm, with a Chan.
func NewSwarm(ctx context.Context, listenAddrs []ma.Multiaddr, local peer.Peer, ps peer.Peerstore) (*Swarm, error) {
	s := &Swarm{
		Pipe:    msg.NewPipe(10),
		conns:   conn.MultiConnMap{},
		local:   local,
		peers:   ps,
		errChan: make(chan error, 100),
	}

	// ContextCloser for proper child management.
	s.ContextCloser = ctxc.NewContextCloser(ctx, s.close)

	s.Children().Add(1)
	go s.fanOut()
	return s, s.listen(listenAddrs)
}

// close stops a swarm. It's the underlying function called by ContextCloser
func (s *Swarm) close() error {
	// close listeners
	for _, list := range s.listeners {
		list.Close()
	}
	// close connections
	conn.CloseConns(s.Connections()...)
	return nil
}

// Dial connects to a peer.
//
// The idea is that the client of Swarm does not need to know what network
// the connection will happen over. Swarm can use whichever it choses.
// This allows us to use various transport protocols, do NAT traversal/relay,
// etc. to achive connection.
//
// For now, Dial uses only TCP. This will be extended.
func (s *Swarm) Dial(peer peer.Peer) (conn.Conn, error) {
	if peer.ID().Equal(s.local.ID()) {
		return nil, errors.New("Attempted connection to self!")
	}

	// check if we already have an open connection first
	c := s.GetConnection(peer.ID())
	if c != nil {
		return c, nil
	}

	// check if we don't have the peer in Peerstore
	peer, err := s.peers.Add(peer)
	if err != nil {
		return nil, err
	}

	// open connection to peer
	d := &conn.Dialer{
		LocalPeer: s.local,
		Peerstore: s.peers,
	}

	// try to connect to one of the peer's known addresses.
	// for simplicity, we do this sequentially.
	// A future commit will do this asynchronously.
	for _, addr := range peer.Addresses() {
		c, err = d.DialAddr(s.Context(), addr, peer)
		if err == nil {
			break
		}
	}
	if err != nil {
		return nil, err
	}

	c, err = s.connSetup(c)
	if err != nil {
		c.Close()
		return nil, err
	}

	// TODO replace the TODO ctx with a context passed in from caller
	log.Event(context.TODO(), "dial", peer)
	return c, nil
}

// GetConnection returns the connection in the swarm to given peer.ID
func (s *Swarm) GetConnection(pid peer.ID) conn.Conn {
	s.connsLock.RLock()
	c, found := s.conns[u.Key(pid)]
	s.connsLock.RUnlock()

	if !found {
		return nil
	}
	return c
}

// Connections returns a slice of all connections.
func (s *Swarm) Connections() []conn.Conn {
	s.connsLock.RLock()

	conns := make([]conn.Conn, 0, len(s.conns))
	for _, c := range s.conns {
		conns = append(conns, c)
	}

	s.connsLock.RUnlock()
	return conns
}

// CloseConnection removes a given peer from swarm + closes the connection
func (s *Swarm) CloseConnection(p peer.Peer) error {
	c := s.GetConnection(p.ID())
	if c == nil {
		return u.ErrNotFound
	}

	s.connsLock.Lock()
	delete(s.conns, u.Key(p.ID()))
	s.connsLock.Unlock()

	return c.Close()
}

func (s *Swarm) Error(e error) {
	s.errChan <- e
}

// GetErrChan returns the errors chan.
func (s *Swarm) GetErrChan() chan error {
	return s.errChan
}

// GetPeerList returns a copy of the set of peers swarm is connected to.
func (s *Swarm) GetPeerList() []peer.Peer {
	var out []peer.Peer
	s.connsLock.RLock()
	for _, p := range s.conns {
		out = append(out, p.RemotePeer())
	}
	s.connsLock.RUnlock()
	return out
}
<commit_msg>fix(swarm) Dial panic<commit_after>// package swarm implements a connection muxer with a pair of channels
// to synchronize all network communication.
package swarm

import (
	"errors"
	"fmt"
	"sync"

	conn "github.com/jbenet/go-ipfs/net/conn"
	msg "github.com/jbenet/go-ipfs/net/message"
	peer "github.com/jbenet/go-ipfs/peer"
	u "github.com/jbenet/go-ipfs/util"
	ctxc "github.com/jbenet/go-ipfs/util/ctxcloser"
	"github.com/jbenet/go-ipfs/util/eventlog"

	context "github.com/jbenet/go-ipfs/Godeps/_workspace/src/code.google.com/p/go.net/context"
	ma "github.com/jbenet/go-ipfs/Godeps/_workspace/src/github.com/jbenet/go-multiaddr"
)

var log = eventlog.Logger("swarm")

// ErrAlreadyOpen signals that a connection to a peer is already open.
var ErrAlreadyOpen = errors.New("Error: Connection to this peer already open.")

// ListenErr contains a set of errors mapping to each of the swarms addresses.
// Used to return multiple errors, as in listen.
type ListenErr struct {
	Errors []error
}

func (e *ListenErr) Error() string {
	if e == nil {
		return "<nil error>"
	}
	var out string
	for i, v := range e.Errors {
		if v != nil {
			out += fmt.Sprintf("%d: %s\n", i, v)
		}
	}
	return out
}

// Swarm is a connection muxer, allowing connections to other peers to
// be opened and closed, while still using the same Chan for all
// communication. The Chan sends/receives Messages, which note the
// destination or source Peer.
type Swarm struct {

	// local is the peer this swarm represents
	local peer.Peer

	// peers is a collection of peers for swarm to use
	peers peer.Peerstore

	// Swarm includes a Pipe object.
	*msg.Pipe

	// errChan is the channel of errors.
	errChan chan error

	// conns are the open connections the swarm is handling.
	// these are MultiConns, which multiplex multiple separate underlying Conns.
	conns     conn.MultiConnMap
	connsLock sync.RWMutex

	// listeners for each network address
	listeners []conn.Listener

	// ContextCloser
	ctxc.ContextCloser
}

// NewSwarm constructs a Swarm, with a Chan.
func NewSwarm(ctx context.Context, listenAddrs []ma.Multiaddr, local peer.Peer, ps peer.Peerstore) (*Swarm, error) {
	s := &Swarm{
		Pipe:    msg.NewPipe(10),
		conns:   conn.MultiConnMap{},
		local:   local,
		peers:   ps,
		errChan: make(chan error, 100),
	}

	// ContextCloser for proper child management.
	s.ContextCloser = ctxc.NewContextCloser(ctx, s.close)

	s.Children().Add(1)
	go s.fanOut()
	return s, s.listen(listenAddrs)
}

// close stops a swarm. It's the underlying function called by ContextCloser
func (s *Swarm) close() error {
	// close listeners
	for _, list := range s.listeners {
		list.Close()
	}
	// close connections
	conn.CloseConns(s.Connections()...)
	return nil
}

// Dial connects to a peer.
//
// The idea is that the client of Swarm does not need to know what network
// the connection will happen over. Swarm can use whichever it choses.
// This allows us to use various transport protocols, do NAT traversal/relay,
// etc. to achive connection.
//
// For now, Dial uses only TCP. This will be extended.
func (s *Swarm) Dial(peer peer.Peer) (conn.Conn, error) {
	if peer.ID().Equal(s.local.ID()) {
		return nil, errors.New("Attempted connection to self!")
	}

	// check if we already have an open connection first
	c := s.GetConnection(peer.ID())
	if c != nil {
		return c, nil
	}

	// check if we don't have the peer in Peerstore
	peer, err := s.peers.Add(peer)
	if err != nil {
		return nil, err
	}

	// open connection to peer
	d := &conn.Dialer{
		LocalPeer: s.local,
		Peerstore: s.peers,
	}

	if len(peer.Addresses()) == 0 {
		return nil, errors.New("peer has no addresses")
	}
	// try to connect to one of the peer's known addresses.
	// for simplicity, we do this sequentially.
	// A future commit will do this asynchronously.
	for _, addr := range peer.Addresses() {
		c, err = d.DialAddr(s.Context(), addr, peer)
		if err == nil {
			break
		}
	}
	if err != nil {
		return nil, err
	}

	c2, err := s.connSetup(c)
	if err != nil {
		c.Close()
		return nil, err
	}

	// TODO replace the TODO ctx with a context passed in from caller
	log.Event(context.TODO(), "dial", peer)
	return c2, nil
}

// GetConnection returns the connection in the swarm to given peer.ID
func (s *Swarm) GetConnection(pid peer.ID) conn.Conn {
	s.connsLock.RLock()
	defer s.connsLock.RUnlock()
	c, found := s.conns[u.Key(pid)]

	if !found {
		return nil
	}
	return c
}

// Connections returns a slice of all connections.
func (s *Swarm) Connections() []conn.Conn {
	s.connsLock.RLock()

	conns := make([]conn.Conn, 0, len(s.conns))
	for _, c := range s.conns {
		conns = append(conns, c)
	}

	s.connsLock.RUnlock()
	return conns
}

// CloseConnection removes a given peer from swarm + closes the connection
func (s *Swarm) CloseConnection(p peer.Peer) error {
	c := s.GetConnection(p.ID())
	if c == nil {
		return u.ErrNotFound
	}

	s.connsLock.Lock()
	delete(s.conns, u.Key(p.ID()))
	s.connsLock.Unlock()

	return c.Close()
}

func (s *Swarm) Error(e error) {
	s.errChan <- e
}

// GetErrChan returns the errors chan.
func (s *Swarm) GetErrChan() chan error {
	return s.errChan
}

// GetPeerList returns a copy of the set of peers swarm is connected to.
func (s *Swarm) GetPeerList() []peer.Peer {
	var out []peer.Peer
	s.connsLock.RLock()
	for _, p := range s.conns {
		out = append(out, p.RemotePeer())
	}
	s.connsLock.RUnlock()
	return out
}
<|endoftext|>


<commit_before>/*
Copyright 2014 Google Inc. All rights reserved.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

// e2e.go runs the e2e test suite. No non-standard package dependencies; call with "go run".
package main

import (
	"bytes"
	"flag"
	"fmt"
	"log"
	"os"
	"os/exec"
	"os/signal"
	"path"
	"path/filepath"
	"strings"
)

var (
	isup             = flag.Bool("isup", false, "Check to see if the e2e cluster is up, then exit.")
	build            = flag.Bool("build", false, "If true, build a new release. Otherwise, use whatever is there.")
	up               = flag.Bool("up", false, "If true, start the the e2e cluster. If cluster is already up, recreate it.")
	push             = flag.Bool("push", false, "If true, push to e2e cluster. Has no effect if -up is true.")
	down             = flag.Bool("down", false, "If true, tear down the cluster before exiting.")
	test             = flag.Bool("test", false, "Run all tests in hack/e2e-suite.")
	tests            = flag.String("tests", "", "Run only tests in hack/e2e-suite matching this glob. Ignored if -test is set.")
	root             = flag.String("root", absOrDie(filepath.Clean(filepath.Join(path.Base(os.Args[0]), ".."))), "Root directory of kubernetes repository.")
	verbose          = flag.Bool("v", false, "If true, print all command output.")
	checkVersionSkew = flag.Bool("check_version_skew", true, ""+
		"By default, verify that client and server have exact version match. "+
		"You can explicitly set to false if you're, e.g., testing client changes "+
		"for which the server version doesn't make a difference.")

	cfgCmd = flag.String("cfg", "", "If nonempty, pass this as an argument, and call kubecfg. Implies -v.")
	ctlCmd = flag.String("ctl", "", "If nonempty, pass this as an argument, and call kubectl. Implies -v. (-test, -cfg, -ctl are mutually exclusive)")
)

var signals = make(chan os.Signal, 100)

func absOrDie(path string) string {
	out, err := filepath.Abs(path)
	if err != nil {
		panic(err)
	}
	return out
}

func main() {
	flag.Parse()
	signal.Notify(signals, os.Interrupt)

	if *test {
		*tests = "*"
	}

	if *isup {
		status := 1
		if runBash("get status", `$KUBECFG -server_version`) {
			status = 0
			log.Printf("Cluster is UP")
		} else {
			log.Printf("Cluster is DOWN")
		}
		os.Exit(status)
	}

	if *build {
		if !runBash("build-release", `test-build-release`) {
			log.Fatal("Error building. Aborting.")
		}
	}

	if *up {
		if !Up() {
			log.Fatal("Error starting e2e cluster. Aborting.")
		}
	} else if *push {
		if !runBash("push", path.Join(*root, "/cluster/kube-push.sh")) {
			log.Fatal("Error pushing e2e cluster. Aborting.")
		}
	}

	failure := false
	switch {
	case *cfgCmd != "":
		failure = !runBash("'kubecfg "+*cfgCmd+"'", "$KUBECFG "+*cfgCmd)
	case *ctlCmd != "":
		failure = !runBash("'kubectl "+*ctlCmd+"'", "$KUBECTL "+*ctlCmd)
	case *tests != "":
		failed, passed := Test()
		log.Printf("Passed tests: %v", passed)
		log.Printf("Failed tests: %v", failed)
		failure = len(failed) > 0
	}

	if *down {
		TearDown()
	}

	if failure {
		os.Exit(1)
	}
}

func TearDown() {
	runBash("teardown", "test-teardown")
}

func Up() bool {
	if !tryUp() {
		log.Printf("kube-up failed; will tear down and retry. (Possibly your cluster was in some partially created state?)")
		TearDown()
		return tryUp()
	}
	return true
}

func tryUp() bool {
	return runBash("up", path.Join(*root, "/cluster/kube-up.sh; test-setup;"))
}

func Test() (failed, passed []string) {
	defer runBashUntil("watchEvents", "$KUBECTL --watch-only get events")()
	// run tests!
	dir, err := os.Open(filepath.Join(*root, "hack", "e2e-suite"))
	if err != nil {
		log.Fatal("Couldn't open e2e-suite dir")
	}
	defer dir.Close()
	names, err := dir.Readdirnames(0)
	if err != nil {
		log.Fatal("Couldn't read names in e2e-suite dir")
	}

	for i := range names {
		name := names[i]
		if name == "." || name == ".." {
			continue
		}
		if match, err := path.Match(*tests, name); !match && err == nil {
			continue
		}
		absName := filepath.Join(*root, "hack", "e2e-suite", name)
		log.Printf("%v matches %v. Starting test.", name, *tests)
		if runBash(name, absName) {
			log.Printf("%v passed", name)
			passed = append(passed, name)
		} else {
			log.Printf("%v failed", name)
			failed = append(failed, name)
		}
	}

	return
}

// All nonsense below is temporary until we have go versions of these things.

func runBash(stepName, bashFragment string) bool {
	cmd := exec.Command("bash", "-s")
	cmd.Stdin = strings.NewReader(bashWrap(bashFragment))
	return finishRunning(stepName, cmd)
}

// call the returned anonymous function to stop.
func runBashUntil(stepName, bashFragment string) func() {
	cmd := exec.Command("bash", "-s")
	cmd.Stdin = strings.NewReader(bashWrap(bashFragment))
	log.Printf("Running in background: %v", stepName)
	stdout, stderr := bytes.NewBuffer(nil), bytes.NewBuffer(nil)
	cmd.Stdout, cmd.Stderr = stdout, stderr
	if err := cmd.Start(); err != nil {
		log.Printf("Unable to start '%v': '%v'", stepName, err)
		return func() {}
	}
	return func() {
		cmd.Process.Signal(os.Interrupt)
		fmt.Printf("%v stdout:\n------\n%v\n------\n", stepName, string(stdout.Bytes()))
		fmt.Printf("%v stderr:\n------\n%v\n------\n", stepName, string(stderr.Bytes()))
	}
}

func run(stepName, cmdPath string) bool {
	return finishRunning(stepName, exec.Command(filepath.Join(*root, cmdPath)))
}

func finishRunning(stepName string, cmd *exec.Cmd) bool {
	log.Printf("Running: %v", stepName)
	stdout, stderr := bytes.NewBuffer(nil), bytes.NewBuffer(nil)
	if *verbose {
		cmd.Stdout = os.Stdout
		cmd.Stderr = os.Stderr
	} else {
		cmd.Stdout = stdout
		cmd.Stderr = stderr
	}

	done := make(chan struct{})
	defer close(done)
	go func() {
		for {
			select {
			case <-done:
				return
			case s := <-signals:
				cmd.Process.Signal(s)
			}
		}
	}()

	if err := cmd.Run(); err != nil {
		log.Printf("Error running %v: %v", stepName, err)
		if !*verbose {
			fmt.Printf("stdout:\n------\n%v\n------\n", string(stdout.Bytes()))
			fmt.Printf("stderr:\n------\n%v\n------\n", string(stderr.Bytes()))
		}
		return false
	}
	return true
}

// returns either "", or a list of args intended for appending with the
// kubecfg or kubectl commands (begining with a space).
func kubecfgArgs() string {
	if *checkVersionSkew {
		return " -expect_version_match"
	}
	return ""
}

// returns either "", or a list of args intended for appending with the
// kubectl command (begining with a space).
func kubectlArgs() string {
	if *checkVersionSkew {
		return " --match-server-version"
	}
	return ""
}

func bashWrap(cmd string) string {
	return `
set -o errexit
set -o nounset
set -o pipefail

export KUBE_CONFIG_FILE="config-test.sh"

# TODO(jbeda): This will break on usage if there is a space in
# ${KUBE_ROOT}.  Covert to an array?  Or an exported function?
export KUBECFG="` + *root + `/cluster/kubecfg.sh` + kubecfgArgs() + `"
export KUBECTL="` + *root + `/cluster/kubectl.sh` + kubectlArgs() + `"

source "` + *root + `/cluster/kube-env.sh"
source "` + *root + `/cluster/${KUBERNETES_PROVIDER}/util.sh"

prepare-e2e

` + cmd + `
`
}
<commit_msg>trace-bash flag for e2e.go<commit_after>/*
Copyright 2014 Google Inc. All rights reserved.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

// e2e.go runs the e2e test suite. No non-standard package dependencies; call with "go run".
package main

import (
	"bytes"
	"flag"
	"fmt"
	"log"
	"os"
	"os/exec"
	"os/signal"
	"path"
	"path/filepath"
	"strings"
)

var (
	isup             = flag.Bool("isup", false, "Check to see if the e2e cluster is up, then exit.")
	build            = flag.Bool("build", false, "If true, build a new release. Otherwise, use whatever is there.")
	up               = flag.Bool("up", false, "If true, start the the e2e cluster. If cluster is already up, recreate it.")
	push             = flag.Bool("push", false, "If true, push to e2e cluster. Has no effect if -up is true.")
	down             = flag.Bool("down", false, "If true, tear down the cluster before exiting.")
	test             = flag.Bool("test", false, "Run all tests in hack/e2e-suite.")
	tests            = flag.String("tests", "", "Run only tests in hack/e2e-suite matching this glob. Ignored if -test is set.")
	root             = flag.String("root", absOrDie(filepath.Clean(filepath.Join(path.Base(os.Args[0]), ".."))), "Root directory of kubernetes repository.")
	verbose          = flag.Bool("v", false, "If true, print all command output.")
	trace_bash       = flag.Bool("trace-bash", false, "If true, pass -x to bash to trace all bash commands")
	checkVersionSkew = flag.Bool("check_version_skew", true, ""+
		"By default, verify that client and server have exact version match. "+
		"You can explicitly set to false if you're, e.g., testing client changes "+
		"for which the server version doesn't make a difference.")

	cfgCmd = flag.String("cfg", "", "If nonempty, pass this as an argument, and call kubecfg. Implies -v.")
	ctlCmd = flag.String("ctl", "", "If nonempty, pass this as an argument, and call kubectl. Implies -v. (-test, -cfg, -ctl are mutually exclusive)")
)

var signals = make(chan os.Signal, 100)

func absOrDie(path string) string {
	out, err := filepath.Abs(path)
	if err != nil {
		panic(err)
	}
	return out
}

func main() {
	flag.Parse()
	signal.Notify(signals, os.Interrupt)

	if *test {
		*tests = "*"
	}

	if *isup {
		status := 1
		if runBash("get status", `$KUBECFG -server_version`) {
			status = 0
			log.Printf("Cluster is UP")
		} else {
			log.Printf("Cluster is DOWN")
		}
		os.Exit(status)
	}

	if *build {
		if !runBash("build-release", `test-build-release`) {
			log.Fatal("Error building. Aborting.")
		}
	}

	if *up {
		if !Up() {
			log.Fatal("Error starting e2e cluster. Aborting.")
		}
	} else if *push {
		if !runBash("push", path.Join(*root, "/cluster/kube-push.sh")) {
			log.Fatal("Error pushing e2e cluster. Aborting.")
		}
	}

	failure := false
	switch {
	case *cfgCmd != "":
		failure = !runBash("'kubecfg "+*cfgCmd+"'", "$KUBECFG "+*cfgCmd)
	case *ctlCmd != "":
		failure = !runBash("'kubectl "+*ctlCmd+"'", "$KUBECTL "+*ctlCmd)
	case *tests != "":
		failed, passed := Test()
		log.Printf("Passed tests: %v", passed)
		log.Printf("Failed tests: %v", failed)
		failure = len(failed) > 0
	}

	if *down {
		TearDown()
	}

	if failure {
		os.Exit(1)
	}
}

func TearDown() {
	runBash("teardown", "test-teardown")
}

func Up() bool {
	if !tryUp() {
		log.Printf("kube-up failed; will tear down and retry. (Possibly your cluster was in some partially created state?)")
		TearDown()
		return tryUp()
	}
	return true
}

func tryUp() bool {
	return runBash("up", path.Join(*root, "/cluster/kube-up.sh; test-setup;"))
}

func Test() (failed, passed []string) {
	defer runBashUntil("watchEvents", "$KUBECTL --watch-only get events")()
	// run tests!
	dir, err := os.Open(filepath.Join(*root, "hack", "e2e-suite"))
	if err != nil {
		log.Fatal("Couldn't open e2e-suite dir")
	}
	defer dir.Close()
	names, err := dir.Readdirnames(0)
	if err != nil {
		log.Fatal("Couldn't read names in e2e-suite dir")
	}

	for i := range names {
		name := names[i]
		if name == "." || name == ".." {
			continue
		}
		if match, err := path.Match(*tests, name); !match && err == nil {
			continue
		}
		absName := filepath.Join(*root, "hack", "e2e-suite", name)
		log.Printf("%v matches %v. Starting test.", name, *tests)
		if runBash(name, absName) {
			log.Printf("%v passed", name)
			passed = append(passed, name)
		} else {
			log.Printf("%v failed", name)
			failed = append(failed, name)
		}
	}

	return
}

// All nonsense below is temporary until we have go versions of these things.

func runBash(stepName, bashFragment string) bool {
	cmd := exec.Command("bash", "-s")
	if *trace_bash {
		cmd.Args = append(cmd.Args, "-x")
	}
	cmd.Stdin = strings.NewReader(bashWrap(bashFragment))
	return finishRunning(stepName, cmd)
}

// call the returned anonymous function to stop.
func runBashUntil(stepName, bashFragment string) func() {
	cmd := exec.Command("bash", "-s")
	cmd.Stdin = strings.NewReader(bashWrap(bashFragment))
	log.Printf("Running in background: %v", stepName)
	stdout, stderr := bytes.NewBuffer(nil), bytes.NewBuffer(nil)
	cmd.Stdout, cmd.Stderr = stdout, stderr
	if err := cmd.Start(); err != nil {
		log.Printf("Unable to start '%v': '%v'", stepName, err)
		return func() {}
	}
	return func() {
		cmd.Process.Signal(os.Interrupt)
		fmt.Printf("%v stdout:\n------\n%v\n------\n", stepName, string(stdout.Bytes()))
		fmt.Printf("%v stderr:\n------\n%v\n------\n", stepName, string(stderr.Bytes()))
	}
}

func run(stepName, cmdPath string) bool {
	return finishRunning(stepName, exec.Command(filepath.Join(*root, cmdPath)))
}

func finishRunning(stepName string, cmd *exec.Cmd) bool {
	log.Printf("Running: %v", stepName)
	stdout, stderr := bytes.NewBuffer(nil), bytes.NewBuffer(nil)
	if *verbose {
		cmd.Stdout = os.Stdout
		cmd.Stderr = os.Stderr
	} else {
		cmd.Stdout = stdout
		cmd.Stderr = stderr
	}

	done := make(chan struct{})
	defer close(done)
	go func() {
		for {
			select {
			case <-done:
				return
			case s := <-signals:
				cmd.Process.Signal(s)
			}
		}
	}()

	if err := cmd.Run(); err != nil {
		log.Printf("Error running %v: %v", stepName, err)
		if !*verbose {
			fmt.Printf("stdout:\n------\n%v\n------\n", string(stdout.Bytes()))
			fmt.Printf("stderr:\n------\n%v\n------\n", string(stderr.Bytes()))
		}
		return false
	}
	return true
}

// returns either "", or a list of args intended for appending with the
// kubecfg or kubectl commands (begining with a space).
func kubecfgArgs() string {
	if *checkVersionSkew {
		return " -expect_version_match"
	}
	return ""
}

// returns either "", or a list of args intended for appending with the
// kubectl command (begining with a space).
func kubectlArgs() string {
	if *checkVersionSkew {
		return " --match-server-version"
	}
	return ""
}

func bashWrap(cmd string) string {
	return `
set -o errexit
set -o nounset
set -o pipefail

export KUBE_CONFIG_FILE="config-test.sh"

# TODO(jbeda): This will break on usage if there is a space in
# ${KUBE_ROOT}.  Covert to an array?  Or an exported function?
export KUBECFG="` + *root + `/cluster/kubecfg.sh` + kubecfgArgs() + `"
export KUBECTL="` + *root + `/cluster/kubectl.sh` + kubectlArgs() + `"

source "` + *root + `/cluster/kube-env.sh"
source "` + *root + `/cluster/${KUBERNETES_PROVIDER}/util.sh"

prepare-e2e

` + cmd + `
`
}
<|endoftext|>


<commit_before>package main

import (
	"bytes"
	"github.com/dimfeld/glog"
	"github.com/dimfeld/gocache"
	"net/http"
	"os"
	"path"
	"strings"
	"time"
)

func error404(w http.ResponseWriter, r *http.Request) {
	// TODO Real error page here.
	http.NotFound(w, r)
}

func error500(w http.ResponseWriter, r *http.Request) {
	w.WriteHeader(http.StatusInternalServerError)
}

func handleError(w http.ResponseWriter, r *http.Request, err error) {
	if os.IsNotExist(err) {
		glog.Warningln(r.URL.Path, ":", err)
		error404(w, r)
	} else {
		glog.Errorln(r.URL.Path, ":", err)
		error500(w, r)
	}
}

// determineCompression figures out if compression can be used, and adds a .gz extension so that
// we get the compressed version of the file instead.
func determineCompression(w http.ResponseWriter, r *http.Request, path string) (outPath string,
	compression bool) {

	if _, ok := r.Header["Range"]; ok {
		// No compression if the user passed a range request, since returning a slice of the
		// compressed version from the cache would then return invalid data.
		return path, false
	}

	encodings := r.Header["Accept-Encoding"]
	outPath = path
	for index := range encodings {
		if strings.Contains(encodings[index], "gzip") {
			// Use the gzipped version.
			return path + ".gz", true
		}
	}

	return path, false
}

func postHandler(globalData *GlobalData, w http.ResponseWriter,
	r *http.Request, urlParams map[string]string) {

	filePath := path.Join(urlParams["year"], urlParams["month"], urlParams["post"]) + ".md"
	filePath, compression := determineCompression(w, r, filePath)

	data, err := globalData.cache.Get(filePath,
		PageSpec{globalData: globalData, customPage: false,
			generator: generatePostPage, params: urlParams})
	if err != nil {
		handleError(w, r, err)
		return
	}

	sendData(w, r, urlParams["post"]+".html", compression, data)
}

func archiveHandler(globalData *GlobalData, w http.ResponseWriter,
	r *http.Request, urlParams map[string]string) {

	year := urlParams["year"]
	if len(year) == 2 {
		year = "20" + year
	}
	month := urlParams["month"]
	if len(month) == 1 {
		month = "0" + month
	}
	filename := year + "-" + month
	filePath := path.Join("archive", filename)
	filePath, compression := determineCompression(w, r, filePath)

	data, err := globalData.cache.Get(filePath,
		PageSpec{globalData: globalData, customPage: false,
			generator: generateArchivePage, params: urlParams})
	if err != nil {
		handleError(w, r, err)
		return
	}

	sendData(w, r, filename+".html", compression, data)
}

func tagHandler(globalData *GlobalData, w http.ResponseWriter,
	r *http.Request, urlParams map[string]string) {

	filePath := path.Join("tags", urlParams["tag"])
	filePath, compression := determineCompression(w, r, filePath)

	data, err := globalData.cache.Get(filePath,
		PageSpec{globalData: globalData, customPage: false,
			generator: generateTagsPage, params: urlParams})
	if err != nil {
		handleError(w, r, err)
		return
	}

	sendData(w, r, urlParams["tag"]+".html", compression, data)
}

func indexHandler(globalData *GlobalData, w http.ResponseWriter,
	r *http.Request, urlParams map[string]string) {

	filename := "index.html"
	filePath, compression := determineCompression(w, r, filename)

	data, err := globalData.cache.Get(filePath,
		PageSpec{globalData: globalData, customPage: false,
			generator: generateIndexPage, params: urlParams})
	if err != nil {
		handleError(w, r, err)
		return
	}

	sendData(w, r, filename, compression, data)
}

func pageHandler(globalData *GlobalData, w http.ResponseWriter,
	r *http.Request, urlParams map[string]string) {

	page := urlParams["page"]

	filePath, compression := determineCompression(w, r, page)
	object, err := globalData.cache.Get(filePath,
		PageSpec{globalData: globalData, customPage: true,
			generator: generateCustomPage, params: urlParams})
	if err != nil {
		handleError(w, r, err)
		return
	}

	sendData(w, r, urlParams["page"]+".html", compression, object)
}

func atomHandler(globalData *GlobalData, w http.ResponseWriter,
	r *http.Request, urlParams map[string]string) {

	filename := "atom.xml"
	filePath, compression := determineCompression(w, r, filename)

	object, err := globalData.cache.Get(filePath,
		PageSpec{globalData: globalData, customTemplate: "atom.tmpl.html",
			generator: generateIndexPage, params: urlParams})
	if err != nil {
		handleError(w, r, err)
		return
	}

	sendData(w, r, filename, compression, object)
}

func staticCompressHandler(globalData *GlobalData, w http.ResponseWriter,
	r *http.Request, urlParams map[string]string) {
	filePath := urlParams["file"]
	filePath, compression := determineCompression(w, r, filePath)

	if glog.V(1) {
		glog.Infoln("Getting path", filePath)
	}
	object, err := globalData.cache.Get(filePath,
		DirectCacheFiller{globalData, true})
	if err != nil {
		handleError(w, r, err)
		return
	}

	setStaticAssetHeaders(w)
	sendData(w, r, urlParams["file"], compression, object)
}

func staticNoCompressHandler(globalData *GlobalData, w http.ResponseWriter,
	r *http.Request, urlParams map[string]string) {
	filePath := urlParams["file"]

	// Only read from the memCache, not the disk cache, since we aren't generating
	// compressed versions.
	object, err := globalData.memCache.Get(filePath,
		DirectCacheFiller{globalData, false})
	if err != nil {
		handleError(w, r, err)
		return
	}

	setStaticAssetHeaders(w)
	sendData(w, r, filePath, false, object)
}

func setStaticAssetHeaders(w http.ResponseWriter) {
	w.Header().Set("Expires", time.Now().AddDate(1, 0, 0).String())
	// One year in seconds
	w.Header().Set("Cache-Control", "public, max-age=31536000")
}

// sendData returns a file to the user, handling relevant headers in the request and response.
func sendData(w http.ResponseWriter, r *http.Request, name string,
	compression bool, object gocache.Object) {

	header := w.Header()
	header.Add("Vary", "Accept-Encoding")
	// 5 minutes in seconds
	if _, ok := header["Cache-Control"]; !ok {
		header.Set("Cache-Control", "public, max-age=300")
	}
	if _, ok := header["Expires"]; !ok {
		header.Set("Expires", time.Now().Add(300*time.Second).String())
	}

	if compression {
		w.Header().Set("Content-Encoding", "gzip")
	}

	if glog.V(1) {
		glog.Infoln("Sending data for %s%s [%d]",
			name,
			func() string {
				if compression {
					return ".gz"
				} else {
					return ""
				}
			}(),
			len(object.Data),
		)
	}

	reader := bytes.NewReader(object.Data)
	http.ServeContent(w, r, name, object.ModTime, reader)
}

type DirectCacheFiller struct {
	globalData  *GlobalData
	canCompress bool
}

func (d DirectCacheFiller) Fill(cacheObj gocache.Cache, pathStr string) (gocache.Object, error) {
	compressed := false
	if d.canCompress && strings.HasSuffix(pathStr, ".gz") {
		// Get the path without .gz at the end since we start with the uncompresed version.
		pathStr = pathStr[0 : len(pathStr)-3]
		compressed = true
	}

	f, err := http.Dir(config.DataDir).Open(pathStr)
	if err != nil {
		return gocache.Object{}, err
	}
	defer f.Close()

	fstat, err := f.Stat()
	if err != nil {
		return gocache.Object{}, err
	}

	data := make([]byte, fstat.Size())
	_, err = f.Read(data)
	if err != nil {
		return gocache.Object{}, err
	}

	if d.canCompress {
		uncompressedObj, compressedObj, err := gocache.CompressAndSet(cacheObj, pathStr, data, fstat.ModTime())
		if glog.V(2) {
			glog.Infoln("%s: compressed %d, uncompressed %d",
				pathStr, len(compressedObj.Data), len(uncompressedObj.Data))
		}
		if compressed {
			return compressedObj, err
		} else {
			return uncompressedObj, err
		}
	} else {
		obj := gocache.Object{Data: data, ModTime: fstat.ModTime()}
		cacheObj.Set(pathStr, obj)
		return obj, nil
	}
}
<commit_msg>More logging of 404 error information<commit_after>package main

import (
	"bytes"
	"github.com/dimfeld/glog"
	"github.com/dimfeld/gocache"
	"net/http"
	"os"
	"path"
	"strings"
	"time"
)

func error404(w http.ResponseWriter, r *http.Request) {
	// TODO Real error page here.
	http.NotFound(w, r)
}

func error500(w http.ResponseWriter, r *http.Request) {
	w.WriteHeader(http.StatusInternalServerError)
}

func handleError(w http.ResponseWriter, r *http.Request, err error) {
	if os.IsNotExist(err) {
		glog.Warningf("%s from %s, referrer %s: err %s",
			r.URL.Path, r.RemoteAddr, r.Referer(), err)
		error404(w, r)
	} else {
		glog.Errorln(r.URL.Path, ":", err)
		error500(w, r)
	}
}

// determineCompression figures out if compression can be used, and adds a .gz extension so that
// we get the compressed version of the file instead.
func determineCompression(w http.ResponseWriter, r *http.Request, path string) (outPath string,
	compression bool) {

	if _, ok := r.Header["Range"]; ok {
		// No compression if the user passed a range request, since returning a slice of the
		// compressed version from the cache would then return invalid data.
		return path, false
	}

	encodings := r.Header["Accept-Encoding"]
	outPath = path
	for index := range encodings {
		if strings.Contains(encodings[index], "gzip") {
			// Use the gzipped version.
			return path + ".gz", true
		}
	}

	return path, false
}

func postHandler(globalData *GlobalData, w http.ResponseWriter,
	r *http.Request, urlParams map[string]string) {

	filePath := path.Join(urlParams["year"], urlParams["month"], urlParams["post"]) + ".md"
	filePath, compression := determineCompression(w, r, filePath)

	data, err := globalData.cache.Get(filePath,
		PageSpec{globalData: globalData, customPage: false,
			generator: generatePostPage, params: urlParams})
	if err != nil {
		handleError(w, r, err)
		return
	}

	sendData(w, r, urlParams["post"]+".html", compression, data)
}

func archiveHandler(globalData *GlobalData, w http.ResponseWriter,
	r *http.Request, urlParams map[string]string) {

	year := urlParams["year"]
	if len(year) == 2 {
		year = "20" + year
	}
	month := urlParams["month"]
	if len(month) == 1 {
		month = "0" + month
	}
	filename := year + "-" + month
	filePath := path.Join("archive", filename)
	filePath, compression := determineCompression(w, r, filePath)

	data, err := globalData.cache.Get(filePath,
		PageSpec{globalData: globalData, customPage: false,
			generator: generateArchivePage, params: urlParams})
	if err != nil {
		handleError(w, r, err)
		return
	}

	sendData(w, r, filename+".html", compression, data)
}

func tagHandler(globalData *GlobalData, w http.ResponseWriter,
	r *http.Request, urlParams map[string]string) {

	filePath := path.Join("tags", urlParams["tag"])
	filePath, compression := determineCompression(w, r, filePath)

	data, err := globalData.cache.Get(filePath,
		PageSpec{globalData: globalData, customPage: false,
			generator: generateTagsPage, params: urlParams})
	if err != nil {
		handleError(w, r, err)
		return
	}

	sendData(w, r, urlParams["tag"]+".html", compression, data)
}

func indexHandler(globalData *GlobalData, w http.ResponseWriter,
	r *http.Request, urlParams map[string]string) {

	filename := "index.html"
	filePath, compression := determineCompression(w, r, filename)

	data, err := globalData.cache.Get(filePath,
		PageSpec{globalData: globalData, customPage: false,
			generator: generateIndexPage, params: urlParams})
	if err != nil {
		handleError(w, r, err)
		return
	}

	sendData(w, r, filename, compression, data)
}

func pageHandler(globalData *GlobalData, w http.ResponseWriter,
	r *http.Request, urlParams map[string]string) {

	page := urlParams["page"]

	filePath, compression := determineCompression(w, r, page)
	object, err := globalData.cache.Get(filePath,
		PageSpec{globalData: globalData, customPage: true,
			generator: generateCustomPage, params: urlParams})
	if err != nil {
		handleError(w, r, err)
		return
	}

	sendData(w, r, urlParams["page"]+".html", compression, object)
}

func atomHandler(globalData *GlobalData, w http.ResponseWriter,
	r *http.Request, urlParams map[string]string) {

	filename := "atom.xml"
	filePath, compression := determineCompression(w, r, filename)

	object, err := globalData.cache.Get(filePath,
		PageSpec{globalData: globalData, customTemplate: "atom.tmpl.html",
			generator: generateIndexPage, params: urlParams})
	if err != nil {
		handleError(w, r, err)
		return
	}

	sendData(w, r, filename, compression, object)
}

func staticCompressHandler(globalData *GlobalData, w http.ResponseWriter,
	r *http.Request, urlParams map[string]string) {
	filePath := urlParams["file"]
	filePath, compression := determineCompression(w, r, filePath)

	if glog.V(1) {
		glog.Infoln("Getting path", filePath)
	}
	object, err := globalData.cache.Get(filePath,
		DirectCacheFiller{globalData, true})
	if err != nil {
		handleError(w, r, err)
		return
	}

	setStaticAssetHeaders(w)
	sendData(w, r, urlParams["file"], compression, object)
}

func staticNoCompressHandler(globalData *GlobalData, w http.ResponseWriter,
	r *http.Request, urlParams map[string]string) {
	filePath := urlParams["file"]

	// Only read from the memCache, not the disk cache, since we aren't generating
	// compressed versions.
	object, err := globalData.memCache.Get(filePath,
		DirectCacheFiller{globalData, false})
	if err != nil {
		handleError(w, r, err)
		return
	}

	setStaticAssetHeaders(w)
	sendData(w, r, filePath, false, object)
}

func setStaticAssetHeaders(w http.ResponseWriter) {
	w.Header().Set("Expires", time.Now().AddDate(1, 0, 0).String())
	// One year in seconds
	w.Header().Set("Cache-Control", "public, max-age=31536000")
}

// sendData returns a file to the user, handling relevant headers in the request and response.
func sendData(w http.ResponseWriter, r *http.Request, name string,
	compression bool, object gocache.Object) {

	header := w.Header()
	header.Add("Vary", "Accept-Encoding")
	// 5 minutes in seconds
	if _, ok := header["Cache-Control"]; !ok {
		header.Set("Cache-Control", "public, max-age=300")
	}
	if _, ok := header["Expires"]; !ok {
		header.Set("Expires", time.Now().Add(300*time.Second).String())
	}

	if compression {
		w.Header().Set("Content-Encoding", "gzip")
	}

	if glog.V(1) {
		glog.Infoln("Sending data for %s%s [%d]",
			name,
			func() string {
				if compression {
					return ".gz"
				} else {
					return ""
				}
			}(),
			len(object.Data),
		)
	}

	reader := bytes.NewReader(object.Data)
	http.ServeContent(w, r, name, object.ModTime, reader)
}

type DirectCacheFiller struct {
	globalData  *GlobalData
	canCompress bool
}

func (d DirectCacheFiller) Fill(cacheObj gocache.Cache, pathStr string) (gocache.Object, error) {
	compressed := false
	if d.canCompress && strings.HasSuffix(pathStr, ".gz") {
		// Get the path without .gz at the end since we start with the uncompresed version.
		pathStr = pathStr[0 : len(pathStr)-3]
		compressed = true
	}

	f, err := http.Dir(config.DataDir).Open(pathStr)
	if err != nil {
		return gocache.Object{}, err
	}
	defer f.Close()

	fstat, err := f.Stat()
	if err != nil {
		return gocache.Object{}, err
	}

	data := make([]byte, fstat.Size())
	_, err = f.Read(data)
	if err != nil {
		return gocache.Object{}, err
	}

	if d.canCompress {
		uncompressedObj, compressedObj, err := gocache.CompressAndSet(cacheObj, pathStr, data, fstat.ModTime())
		if glog.V(2) {
			glog.Infoln("%s: compressed %d, uncompressed %d",
				pathStr, len(compressedObj.Data), len(uncompressedObj.Data))
		}
		if compressed {
			return compressedObj, err
		} else {
			return uncompressedObj, err
		}
	} else {
		obj := gocache.Object{Data: data, ModTime: fstat.ModTime()}
		cacheObj.Set(pathStr, obj)
		return obj, nil
	}
}
<|endoftext|>


<commit_before>/*******************************************************************************
*
* todolist - tiny single-user todolist app
* Copyright 2016 Stefan Majewsky <majewsky@gmx.net>
*
* This program is free software: you can redistribute it and/or modify
* it under the terms of the GNU Affero General Public License as published
* by the Free Software Foundation, either version 3 of the License, or
* (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
* GNU Affero General Public License for more details.
*
* You should have received a copy of the GNU Affero General Public License
* along with this program.  If not, see <http://www.gnu.org/licenses/>.
*
*******************************************************************************/

package main

import (
	"fmt"
	"net/http"
	"strconv"

	"github.com/gorilla/mux"
)

func collectRoutes(router *mux.Router) {
	router.HandleFunc("/", indexHandler).Name("index")
	router.HandleFunc("/toggle/{milestone:[0-9]+}/{task:[0-9]+}", toggleHandler).Name("toggle")
	router.HandleFunc("/edit", editHandler).Methods("GET").Name("edit")
	router.HandleFunc("/edit", saveHandler).Methods("POST").Name("save")
	router.HandleFunc("/prune", pruneHandler).Name("prune")
	router.HandleFunc("/backup", pruneHandler).Name("backup")
}

func indexHandler(w http.ResponseWriter, r *http.Request) {
	//TODO: login
	data := ReadData()
	if data == nil {
		serveError(w, 500, "Cannot read data. Check the server log for details.")
	}

	html := `<section>`
	hasDone := false

	for mIdx, milestone := range data.Milestones {
		html += `<h2>` + milestone.Name + `</h2>`
		if len(milestone.Tasks) == 0 {
			html += `<p>No tasks in this group.</p>`
		} else {
			html += `<div class="group">`

			for tIdx, task := range milestone.Tasks {
				class := "open"
				if task.Done {
					class = "done"
					hasDone = true
				}
				path, _ := Router.Get("toggle").URLPath(
					"milestone", strconv.Itoa(mIdx),
					"task", strconv.Itoa(tIdx),
				)
				html += fmt.Sprintf(`<a href="%s" class="%s">%s</a>`,
					path, class,
					HTMLEscapeString(task.Text),
				)
			}
			html += `</div>`
		}
	}

	html += `<div class="table"><div class="row"><a href="/edit" class="action">Edit</a>`
	if hasDone {
		html += `<a href="/prune" class="action">Prune</a>`
	}
	html += `<a href="/backup" class="action">Backup</a></div></div></section>`

	serveHTML(w, "Tasks", html)
}

func toggleHandler(w http.ResponseWriter, r *http.Request) {
	data := ReadData()
	if data == nil {
		serveError(w, 500, "Cannot read data. Check the server log for details.")
	}

	//retrieve parameters
	vars := mux.Vars(r)
	mIdx, _ := strconv.Atoi(vars["milestone"])
	tIdx, _ := strconv.Atoi(vars["task"])

	if mIdx < 0 || mIdx >= len(data.Milestones) {
		serveError(w, 400, "Milestone index out of range.")
	}
	milestone := data.Milestones[mIdx]
	if tIdx < 0 || tIdx >= len(milestone.Tasks) {
		serveError(w, 400, "Task index out of range.")
	}
	task := milestone.Tasks[tIdx]
	task.Done = !task.Done
	if !data.WriteData() {
		serveError(w, 500, "Cannot write data. Check the server log for details.")
	}

	w.Header().Add("Location", "/")
	w.WriteHeader(302)
}

func editHandler(w http.ResponseWriter, r *http.Request) {
	serveError(w, 500, "Not implemented")
}

func saveHandler(w http.ResponseWriter, r *http.Request) {
	serveError(w, 500, "Not implemented")
}

func pruneHandler(w http.ResponseWriter, r *http.Request) {
	serveError(w, 500, "Not implemented")
}

func backupHandler(w http.ResponseWriter, r *http.Request) {
	serveError(w, 500, "Not implemented")
}
<commit_msg>implement prune action<commit_after>/*******************************************************************************
*
* todolist - tiny single-user todolist app
* Copyright 2016 Stefan Majewsky <majewsky@gmx.net>
*
* This program is free software: you can redistribute it and/or modify
* it under the terms of the GNU Affero General Public License as published
* by the Free Software Foundation, either version 3 of the License, or
* (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
* GNU Affero General Public License for more details.
*
* You should have received a copy of the GNU Affero General Public License
* along with this program.  If not, see <http://www.gnu.org/licenses/>.
*
*******************************************************************************/

package main

import (
	"fmt"
	"net/http"
	"strconv"

	"github.com/gorilla/mux"
)

func collectRoutes(router *mux.Router) {
	router.HandleFunc("/", indexHandler).Name("index")
	router.HandleFunc("/toggle/{milestone:[0-9]+}/{task:[0-9]+}", toggleHandler).Name("toggle")
	router.HandleFunc("/edit", editHandler).Methods("GET").Name("edit")
	router.HandleFunc("/edit", saveHandler).Methods("POST").Name("save")
	router.HandleFunc("/prune", pruneHandler).Name("prune")
	router.HandleFunc("/backup", pruneHandler).Name("backup")
}

func indexHandler(w http.ResponseWriter, r *http.Request) {
	//TODO: login
	data := ReadData()
	if data == nil {
		serveError(w, 500, "Cannot read data. Check the server log for details.")
	}

	html := `<section>`
	hasDone := false

	for mIdx, milestone := range data.Milestones {
		html += `<h2>` + milestone.Name + `</h2>`
		if len(milestone.Tasks) == 0 {
			html += `<p>No tasks in this group.</p>`
		} else {
			html += `<div class="group">`

			for tIdx, task := range milestone.Tasks {
				class := "open"
				if task.Done {
					class = "done"
					hasDone = true
				}
				path, _ := Router.Get("toggle").URLPath(
					"milestone", strconv.Itoa(mIdx),
					"task", strconv.Itoa(tIdx),
				)
				html += fmt.Sprintf(`<a href="%s" class="%s">%s</a>`,
					path, class,
					HTMLEscapeString(task.Text),
				)
			}
			html += `</div>`
		}
	}

	html += `<div class="table"><div class="row"><a href="/edit" class="action">Edit</a>`
	if hasDone {
		html += `<a href="/prune" class="action">Prune</a>`
	}
	html += `<a href="/backup" class="action">Backup</a></div></div></section>`

	serveHTML(w, "Tasks", html)
}

func toggleHandler(w http.ResponseWriter, r *http.Request) {
	data := ReadData()
	if data == nil {
		serveError(w, 500, "Cannot read data. Check the server log for details.")
	}

	//retrieve parameters
	vars := mux.Vars(r)
	mIdx, _ := strconv.Atoi(vars["milestone"])
	tIdx, _ := strconv.Atoi(vars["task"])

	if mIdx < 0 || mIdx >= len(data.Milestones) {
		serveError(w, 400, "Milestone index out of range.")
	}
	milestone := data.Milestones[mIdx]
	if tIdx < 0 || tIdx >= len(milestone.Tasks) {
		serveError(w, 400, "Task index out of range.")
	}
	task := milestone.Tasks[tIdx]
	task.Done = !task.Done
	if !data.WriteData() {
		serveError(w, 500, "Cannot write data. Check the server log for details.")
	}

	w.Header().Add("Location", "/")
	w.WriteHeader(302)
}

func editHandler(w http.ResponseWriter, r *http.Request) {
	serveError(w, 500, "Not implemented")
}

func saveHandler(w http.ResponseWriter, r *http.Request) {
	serveError(w, 500, "Not implemented")
}

func pruneHandler(w http.ResponseWriter, r *http.Request) {
	data := ReadData()
	if data == nil {
		serveError(w, 500, "Cannot read data. Check the server log for details.")
	}

	//filter tasks that are done
	var openMilestones []*Milestone
	hasDone := false

	for _, milestone := range data.Milestones {
		var openTasks []*Task
		for _, task := range milestone.Tasks {
			if task.Done {
				hasDone = true
			} else {
				openTasks = append(openTasks, task)
			}
		}

		milestone.Tasks = openTasks
		if len(openTasks) > 0 {
			openMilestones = append(openMilestones, milestone)
		}
	}
	data.Milestones = openMilestones

	//write data only if changed
	if hasDone {
		if !data.WriteData() {
			serveError(w, 500, "Cannot write data. Check the server log for details.")
		}
	}

	w.Header().Add("Location", "/")
	w.WriteHeader(302)
}

func backupHandler(w http.ResponseWriter, r *http.Request) {
	serveError(w, 500, "Not implemented")
}
<|endoftext|>


<commit_before>package rdb

import (
	"github.com/Cepave/open-falcon-backend/common/testing"
	ocheck "github.com/Cepave/open-falcon-backend/common/testing/check"
	dbTest "github.com/Cepave/open-falcon-backend/common/testing/db"
	"github.com/Cepave/open-falcon-backend/modules/nqm-mng/model"
	. "gopkg.in/check.v1"
)

type TestUpdateOrInsertSuite struct{}

var _ = Suite(&TestUpdateOrInsertSuite{})

func (suite *TestUpdateOrInsertSuite) TestUpdateOrInsertHost(c *C) {
	testCases := []struct {
		hosts     []string
		timestamp string
		expect    int64
	}{
		{ // Add new 3 new hosts
			[]string{"001", "002", "003"}, "2014-05-05T10:20:30+08:00", 3,
		},
		{ // Update existing hosts
			[]string{"001", "002", "003"}, "2014-05-05T12:20:30+08:00", 3,
		},
		{ // Simulate old heartbeat and a new one.
			[]string{"001", "002", "003", "004"}, "2014-05-04T10:20:30+08:00", 1,
		},
	}

	for idx, testCase := range testCases {
		comment := ocheck.TestCaseComment(idx)
		ocheck.LogTestCase(c, testCase)

		sampleTime := testing.ParseTime(c, testCase.timestamp)
		// var sampleNumber string := idx + 1
		// sampleIp, sampleAgentVersion, samplePluginVersion :=
		// 	"127.0.0." + sampleNumber, "0.0." + sampleNumber, "12345abcd" + sampleNumber

		sampleHosts := make([]*model.AgentHeartbeat, len(testCase.hosts))
		for idx, hostName := range testCase.hosts {
			sampleHosts[idx] = &model.AgentHeartbeat{
				Hostname:      "nqm-mng-tc1-" + hostName,
				UpdateTime:    sampleTime.Unix(),
				IP:            "127.0.0.1",
				AgentVersion:  "0.0.1",
				PluginVersion: "12155256cec3926186de22e282e67f4ce11cdbf7",
			}
		}

		result := updateOrInsertHost(sampleHosts)

		var dbResult int64
		sql := `
		SELECT COUNT(*)
		FROM host
		WHERE update_at = FROM_UNIXTIME(?)
			AND hostname LIKE 'nqm-mng-tc1-%'
			AND ip = '127.0.0.1'
			AND agent_version = '0.0.1'
			AND plugin_version = '12155256cec3926186de22e282e67f4ce11cdbf7'
		`
		DbFacade.SqlxDbCtrl.QueryRowxAndScan(sql, []interface{}{sampleTime.Unix()}, &dbResult)
		c.Assert(result.RowsAffected, Equals, testCase.expect, comment)
		c.Assert(dbResult, Equals, testCase.expect, comment)
	}
}

func (suite *TestUpdateOrInsertSuite) TearDownTest(c *C) {
	var inTx = DbFacade.SqlDbCtrl.ExecQueriesInTx

	switch c.TestName() {
	case "TestUpdateOrInsertSuite.TestUpdateOrInsertHost":
		inTx(
			`DELETE FROM host WHERE hostname LIKE 'nqm-mng-tc1-%'`,
		)
	}
}

func (suite *TestUpdateOrInsertSuite) SetUpSuite(c *C) {
	DbFacade = dbTest.InitDbFacade(c)
}

func (suite *TestUpdateOrInsertSuite) TearDownSuite(c *C) {
	dbTest.ReleaseDbFacade(c, DbFacade)
	DbFacade = nil
}
<commit_msg>[OWL-1645] Test more precisely in TestUpdateOrInsertHost. Use dynamic field values.<commit_after>package rdb

import (
	"strconv"

	"github.com/Cepave/open-falcon-backend/common/testing"
	ocheck "github.com/Cepave/open-falcon-backend/common/testing/check"
	dbTest "github.com/Cepave/open-falcon-backend/common/testing/db"
	"github.com/Cepave/open-falcon-backend/modules/nqm-mng/model"
	. "gopkg.in/check.v1"
)

type TestUpdateOrInsertSuite struct{}

var _ = Suite(&TestUpdateOrInsertSuite{})

func (suite *TestUpdateOrInsertSuite) TestUpdateOrInsertHost(c *C) {
	testCases := []struct {
		hosts     []string
		timestamp string
		expect    int64
	}{
		{ // Add new 3 new hosts
			[]string{"001", "002", "003"}, "2014-05-05T10:20:30+08:00", 3,
		},
		{ // Update existing hosts
			[]string{"001", "002", "003"}, "2014-05-05T12:20:30+08:00", 3,
		},
		{ // Simulate old heartbeat and a new one.
			[]string{"001", "002", "003", "004"}, "2014-05-04T10:20:30+08:00", 1,
		},
	}

	for idx, testCase := range testCases {
		comment := ocheck.TestCaseComment(idx)
		ocheck.LogTestCase(c, testCase)

		sampleTime := testing.ParseTime(c, testCase.timestamp)
		sampleNumber := strconv.Itoa(idx + 1)
		sampleIP, sampleAgentVersion, samplePluginVersion :=
			"127.0.0."+sampleNumber, "0.0."+sampleNumber, "12345abcd"+sampleNumber

		sampleHosts := make([]*model.AgentHeartbeat, len(testCase.hosts))
		for idx, hostName := range testCase.hosts {
			sampleHosts[idx] = &model.AgentHeartbeat{
				Hostname:      "nqm-mng-tc1-" + hostName,
				UpdateTime:    sampleTime.Unix(),
				IP:            sampleIP,
				AgentVersion:  sampleAgentVersion,
				PluginVersion: samplePluginVersion,
			}
		}

		result := updateOrInsertHost(sampleHosts)

		var dbResult int64
		sql := `
		SELECT COUNT(*)
		FROM host
		WHERE update_at = FROM_UNIXTIME(?)
			AND hostname LIKE 'nqm-mng-tc1-%'
			AND ip = ?
			AND agent_version = ?
			AND plugin_version = ?
		`
		DbFacade.SqlxDbCtrl.QueryRowxAndScan(sql, []interface{}{sampleTime.Unix(), sampleIP, sampleAgentVersion, samplePluginVersion}, &dbResult)
		c.Assert(result.RowsAffected, Equals, testCase.expect, comment)
		c.Assert(dbResult, Equals, testCase.expect, comment)
	}
}

func (suite *TestUpdateOrInsertSuite) TearDownTest(c *C) {
	var inTx = DbFacade.SqlDbCtrl.ExecQueriesInTx

	switch c.TestName() {
	case "TestUpdateOrInsertSuite.TestUpdateOrInsertHost":
		inTx(
			`DELETE FROM host WHERE hostname LIKE 'nqm-mng-tc1-%'`,
		)
	}
}

func (suite *TestUpdateOrInsertSuite) SetUpSuite(c *C) {
	DbFacade = dbTest.InitDbFacade(c)
}

func (suite *TestUpdateOrInsertSuite) TearDownSuite(c *C) {
	dbTest.ReleaseDbFacade(c, DbFacade)
	DbFacade = nil
}
<|endoftext|>


<commit_before>package sender

import (
	"github.com/open-falcon/transfer/g"
	"stathat.com/c/consistent"
)

func initNodeRings() {
	cfg := g.Config()

	JudgeNodeRing = newConsistentHashNodesRing(cfg.Judge.Replicas, KeysOfMap(cfg.Judge.Cluster))
	GraphNodeRing = newConsistentHashNodesRing(cfg.Graph.Replicas, KeysOfMap(cfg.Graph.Cluster))
}

// TODO 考虑放到公共组件库,或utils库
func KeysOfMap(m map[string]string) []string {
	keys := make([]string, len(m))
	i := 0
	for key, _ := range m {
		keys[i] = key
		i++
	}

	return keys
}

// 一致性哈希环,用于管理服务器节点.
type ConsistentHashNodeRing struct {
	ring *consistent.Consistent
}

func newConsistentHashNodesRing(numberOfReplicas int, nodes []string) *ConsistentHashNodeRing {
	ret := &ConsistentHashNodeRing{ring: consistent.New()}
	ret.SetNumberOfReplicas(numberOfReplicas)
	ret.SetNodes(nodes)
	return ret
}

// 根据pk,获取node节点. chash(pk) -> node
func (this *ConsistentHashNodeRing) GetNode(pk string) (string, error) {
	return this.ring.Get(pk)
}

func (this *ConsistentHashNodeRing) SetNodes(nodes []string) {
	for _, node := range nodes {
		this.ring.Add(node)
	}
}

func (this *ConsistentHashNodeRing) SetNumberOfReplicas(num int) {
	this.ring.NumberOfReplicas = num
}
<commit_msg>fix consistent package import path<commit_after>package sender

import (
	"github.com/open-falcon/transfer/g"
	"github.com/toolkits/consistent"
)

func initNodeRings() {
	cfg := g.Config()

	JudgeNodeRing = newConsistentHashNodesRing(cfg.Judge.Replicas, KeysOfMap(cfg.Judge.Cluster))
	GraphNodeRing = newConsistentHashNodesRing(cfg.Graph.Replicas, KeysOfMap(cfg.Graph.Cluster))
}

// TODO 考虑放到公共组件库,或utils库
func KeysOfMap(m map[string]string) []string {
	keys := make([]string, len(m))
	i := 0
	for key, _ := range m {
		keys[i] = key
		i++
	}

	return keys
}

// 一致性哈希环,用于管理服务器节点.
type ConsistentHashNodeRing struct {
	ring *consistent.Consistent
}

func newConsistentHashNodesRing(numberOfReplicas int, nodes []string) *ConsistentHashNodeRing {
	ret := &ConsistentHashNodeRing{ring: consistent.New()}
	ret.SetNumberOfReplicas(numberOfReplicas)
	ret.SetNodes(nodes)
	return ret
}

// 根据pk,获取node节点. chash(pk) -> node
func (this *ConsistentHashNodeRing) GetNode(pk string) (string, error) {
	return this.ring.Get(pk)
}

func (this *ConsistentHashNodeRing) SetNodes(nodes []string) {
	for _, node := range nodes {
		this.ring.Add(node)
	}
}

func (this *ConsistentHashNodeRing) SetNumberOfReplicas(num int) {
	this.ring.NumberOfReplicas = num
}
<|endoftext|>


<commit_before>package xmpp

import (
	"fmt"
	"net"
)

const (
	// Standard port for XMPP clients to connect to.
	XMPP_CLIENT_PORT = 5222
)

// Perform a DNS SRV lookup and return an ordered list of "host:port" TCP
// addresses for the JID's home server. If no SRV records are found then assume
// the JID's domain is also the home server.
func HomeServerAddrs(jid JID) (addr []string, err error) {

	// DNS lookup.
	_, addrs, _ := net.LookupSRV("xmpp-client", "tcp", jid.Domain)

	// If there's nothing in DNS then assume the JID's domain and the standard
	// port will work.
	if len(addrs) == 0 {
		addr = []string{fmt.Sprintf("%s:%d", jid.Domain, XMPP_CLIENT_PORT)}
		return
	}

	// Build list of "host:port" strings.
	for _, a := range addrs {
		target := parseTargetDomainName(a.Target)
		addr = append(addr, fmt.Sprintf("%s:%d", target, a.Port))
	}
	return
}

// Remove the last dot in the domain name if exist
func parseTargetDomainName(domainName string) (ret string) {
	if domainName[len(domainName)-1] == '.' {
		ret = parseTargetDomainName(domainName[:len(domainName)-1])
	} else {
		ret = domainName
	}
	return
}
<commit_msg>Remove useless function that already exist in strings pakage.<commit_after>package xmpp

import (
	"fmt"
	"net"
	"strings"
)

const (
	// Standard port for XMPP clients to connect to.
	XMPP_CLIENT_PORT = 5222
)

// Perform a DNS SRV lookup and return an ordered list of "host:port" TCP
// addresses for the JID's home server. If no SRV records are found then assume
// the JID's domain is also the home server.
func HomeServerAddrs(jid JID) (addr []string, err error) {

	// DNS lookup.
	_, addrs, _ := net.LookupSRV("xmpp-client", "tcp", jid.Domain)

	// If there's nothing in DNS then assume the JID's domain and the standard
	// port will work.
	if len(addrs) == 0 {
		addr = []string{fmt.Sprintf("%s:%d", jid.Domain, XMPP_CLIENT_PORT)}
		return
	}

	// Build list of "host:port" strings.
	for _, a := range addrs {
		target := strings.TrimRight(a.Target, ".")
		addr = append(addr, fmt.Sprintf("%s:%d", target, a.Port))
	}
	return
}
<|endoftext|>


<commit_before>// Copyright 2015-2016, Cyrill @ Schumacher.fm and the CoreStore contributors
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package responseproxy_test

import (
	"bytes"
	"io"
	"net/http/httptest"
	"testing"

	"github.com/corestoreio/csfw/net/responseproxy"
	"github.com/stretchr/testify/assert"
)

func TestWrapPipe(t *testing.T) {
	wOrg := httptest.NewRecorder()
	data := []byte(`Commander Data encrypts the computer with a fractal algorithm to protect it from the Borgs.`)

	buf := new(bytes.Buffer)
	pw := responseproxy.WrapPiped(buf, wOrg)

	n, err := pw.Write(data)
	assert.NoError(t, err)
	assert.Exactly(t, len(data), n)
	assert.Exactly(t, 0, wOrg.Body.Len())
	assert.Exactly(t, len(data), buf.Len())

	io.Copy(pw.Unwrap(), buf)
	assert.Exactly(t, len(data), wOrg.Body.Len())
}
<commit_msg>net/responseproxy: Use mutex buffer to avoid race<commit_after>// Copyright 2015-2016, Cyrill @ Schumacher.fm and the CoreStore contributors
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package responseproxy_test

import (
	"io"
	"net/http/httptest"
	"testing"

	"github.com/corestoreio/csfw/net/responseproxy"
	"github.com/corestoreio/log"
	"github.com/stretchr/testify/assert"
)

func TestWrapPipe(t *testing.T) {
	wOrg := httptest.NewRecorder()
	data := []byte(`Commander Data encrypts the computer with a fractal algorithm to protect it from the Borgs.`)

	buf := new(log.MutexBuffer)
	pw := responseproxy.WrapPiped(buf, wOrg)

	n, err := pw.Write(data)
	assert.NoError(t, err)
	assert.Exactly(t, len(data), n)
	assert.Exactly(t, 0, wOrg.Body.Len())
	assert.Exactly(t, len(data), buf.Len())

	io.Copy(pw.Unwrap(), buf)
	assert.Exactly(t, len(data), wOrg.Body.Len())
}
<|endoftext|>


<commit_before>package hostess

import (
	"io/ioutil"
	"os"
	"strings"
)

const defaultOSX = `
##
# Host Database
#
# localhost is used to configure the loopback interface
# when the system is booting.  Do not change this entry.
##

127.0.0.1       localhost
255.255.255.255 broadcasthost
::1             localhost
fe80::1%lo0     localhost
`

const defaultLinux = `
127.0.0.1   localhost
127.0.1.1   HOSTNAME

# The following lines are desirable for IPv6 capable hosts
::1     localhost ip6-localhost ip6-loopback
fe00::0 ip6-localnet
ff00::0 ip6-mcastprefix
ff02::1 ip6-allnodes
ff02::2 ip6-allrouters
ff02::3 ip6-allhosts
`

// Hostfile represents /etc/hosts (or a similar file, depending on OS), and
// includes a list of Hostnames. Hostfile includes
type Hostfile struct {
	Path  string
	Hosts Hostlist
	data  []byte
}

// NewHostfile creates a new Hostfile object from the specified file.
func NewHostfile() *Hostfile {
	return &Hostfile{GetHostsPath(), Hostlist{}, []byte{}}
}

// GetHostsPath returns the location of the hostfile; either env HOSTESS_PATH
// or /etc/hosts if HOSTESS_PATH is not set.
func GetHostsPath() string {
	path := os.Getenv("HOSTESS_PATH")
	if path == "" {
		path = "/etc/hosts"
	}
	return path
}

// TrimWS (Trim Whitespace) removes space, newline, and tabs from a string
// using strings.Trim()
func TrimWS(s string) string {
	return strings.Trim(s, " \n\t")
}

// ParseLine parses an individual line in a hostfile, which may contain one
// (un)commented ip and one or more hostnames. For example
//
//	127.0.0.1 localhost mysite1 mysite2
func ParseLine(line string) Hostlist {
	var hostnames Hostlist

	if len(line) == 0 {
		return hostnames
	}

	// Parse leading # for disabled lines
	enabled := true
	if line[0:1] == "#" {
		enabled = false
		line = TrimWS(line[1:])
	}

	// Parse other #s for actual comments
	line = strings.Split(line, "#")[0]

	// Replace tabs and multispaces with single spaces throughout
	line = strings.Replace(line, "\t", " ", -1)
	for strings.Contains(line, "  ") {
		line = strings.Replace(line, "  ", " ", -1)
	}

	// Break line into words
	words := strings.Split(line, " ")

	// Separate the first bit (the ip) from the other bits (the domains)
	ip := words[0]
	domains := words[1:]

	if LooksLikeIPv4(ip) || LooksLikeIPv6(ip) {
		for _, v := range domains {
			hostname := NewHostname(v, ip, enabled)
			hostnames = append(hostnames, hostname)
		}
	}

	return hostnames
}

// Parse reads
func (h *Hostfile) Parse() []error {
	var errs []error
	for _, v := range strings.Split(string(h.data), "\n") {
		for _, hostname := range ParseLine(v) {
			err := h.Hosts.Add(hostname)
			if err != nil {
				errs = append(errs, err)
			}
		}
	}
	return errs
}

// Read the contents of the hostfile from disk
func (h *Hostfile) Read() error {
	data, err := ioutil.ReadFile(h.Path)
	if err == nil {
		h.data = data
	}
	return err
}

// LoadHostfile creates a new Hostfile struct and tries to populate it from
// disk. Read and/or parse errors are returned as a slice.
func LoadHostfile() (hostfile *Hostfile, errs []error) {
	hostfile = NewHostfile()
	readErr := hostfile.Read()
	if readErr != nil {
		errs = []error{readErr}
		return
	}
	errs = hostfile.Parse()
	hostfile.Hosts.Sort()
	return
}

// GetData returns the internal snapshot of the hostfile we read when we loaded
// this hostfile from disk (if we ever did that). This is implemented for
// testing and you probably won't need to use it.
func (h *Hostfile) GetData() []byte {
	return h.data
}

// Format takes the current list of Hostnames in this Hostfile and turns it
// into a string suitable for use as an /etc/hosts file.
// Sorting uses the following logic:
// 1. List is sorted by IP address
// 2. Commented items are left in place
// 3. 127.* appears at the top of the list (so boot resolvers don't break)
// 4. When present, localhost will always appear first in the domain list
func (h *Hostfile) Format() []byte {
	return h.Hosts.Format()
}

// Save writes the Hostfile to disk to /etc/hosts or to the location specified
// by the HOSTESS_PATH environment variable (if set).
func (h *Hostfile) Save() error {
	file, err := os.OpenFile(h.Path, os.O_RDWR|os.O_APPEND|os.O_TRUNC, 0644)
	if err != nil {
		return err
	}

	defer file.Close()
	_, err = file.Write(h.Format())

	return err
}
<commit_msg>Added default path for windows<commit_after>package hostess

import (
	"io/ioutil"
	"os"
	"runtime"
	"strings"
)

const defaultOSX = `
##
# Host Database
#
# localhost is used to configure the loopback interface
# when the system is booting.  Do not change this entry.
##

127.0.0.1       localhost
255.255.255.255 broadcasthost
::1             localhost
fe80::1%lo0     localhost
`

const defaultLinux = `
127.0.0.1   localhost
127.0.1.1   HOSTNAME

# The following lines are desirable for IPv6 capable hosts
::1     localhost ip6-localhost ip6-loopback
fe00::0 ip6-localnet
ff00::0 ip6-mcastprefix
ff02::1 ip6-allnodes
ff02::2 ip6-allrouters
ff02::3 ip6-allhosts
`

// Hostfile represents /etc/hosts (or a similar file, depending on OS), and
// includes a list of Hostnames. Hostfile includes
type Hostfile struct {
	Path  string
	Hosts Hostlist
	data  []byte
}

// NewHostfile creates a new Hostfile object from the specified file.
func NewHostfile() *Hostfile {
	return &Hostfile{GetHostsPath(), Hostlist{}, []byte{}}
}

// GetHostsPath returns the location of the hostfile; either env HOSTESS_PATH
// or /etc/hosts if HOSTESS_PATH is not set.
func GetHostsPath() string {
	path := os.Getenv("HOSTESS_PATH")
	if path == "" {
		if runtime.GOOS == "windows" {
			path = "C:\\Windows\\System32\\drivers\\etc\\hosts"
		} else {
			path = "/etc/hosts"
		}
	}
	return path
}

// TrimWS (Trim Whitespace) removes space, newline, and tabs from a string
// using strings.Trim()
func TrimWS(s string) string {
	return strings.Trim(s, " \n\t")
}

// ParseLine parses an individual line in a hostfile, which may contain one
// (un)commented ip and one or more hostnames. For example
//
//	127.0.0.1 localhost mysite1 mysite2
func ParseLine(line string) Hostlist {
	var hostnames Hostlist

	if len(line) == 0 {
		return hostnames
	}

	// Parse leading # for disabled lines
	enabled := true
	if line[0:1] == "#" {
		enabled = false
		line = TrimWS(line[1:])
	}

	// Parse other #s for actual comments
	line = strings.Split(line, "#")[0]

	// Replace tabs and multispaces with single spaces throughout
	line = strings.Replace(line, "\t", " ", -1)
	for strings.Contains(line, "  ") {
		line = strings.Replace(line, "  ", " ", -1)
	}

	// Break line into words
	words := strings.Split(line, " ")

	// Separate the first bit (the ip) from the other bits (the domains)
	ip := words[0]
	domains := words[1:]

	if LooksLikeIPv4(ip) || LooksLikeIPv6(ip) {
		for _, v := range domains {
			hostname := NewHostname(v, ip, enabled)
			hostnames = append(hostnames, hostname)
		}
	}

	return hostnames
}

// Parse reads
func (h *Hostfile) Parse() []error {
	var errs []error
	for _, v := range strings.Split(string(h.data), "\n") {
		for _, hostname := range ParseLine(v) {
			err := h.Hosts.Add(hostname)
			if err != nil {
				errs = append(errs, err)
			}
		}
	}
	return errs
}

// Read the contents of the hostfile from disk
func (h *Hostfile) Read() error {
	data, err := ioutil.ReadFile(h.Path)
	if err == nil {
		h.data = data
	}
	return err
}

// LoadHostfile creates a new Hostfile struct and tries to populate it from
// disk. Read and/or parse errors are returned as a slice.
func LoadHostfile() (hostfile *Hostfile, errs []error) {
	hostfile = NewHostfile()
	readErr := hostfile.Read()
	if readErr != nil {
		errs = []error{readErr}
		return
	}
	errs = hostfile.Parse()
	hostfile.Hosts.Sort()
	return
}

// GetData returns the internal snapshot of the hostfile we read when we loaded
// this hostfile from disk (if we ever did that). This is implemented for
// testing and you probably won't need to use it.
func (h *Hostfile) GetData() []byte {
	return h.data
}

// Format takes the current list of Hostnames in this Hostfile and turns it
// into a string suitable for use as an /etc/hosts file.
// Sorting uses the following logic:
// 1. List is sorted by IP address
// 2. Commented items are left in place
// 3. 127.* appears at the top of the list (so boot resolvers don't break)
// 4. When present, localhost will always appear first in the domain list
func (h *Hostfile) Format() []byte {
	return h.Hosts.Format()
}

// Save writes the Hostfile to disk to /etc/hosts or to the location specified
// by the HOSTESS_PATH environment variable (if set).
func (h *Hostfile) Save() error {
	file, err := os.OpenFile(h.Path, os.O_RDWR|os.O_APPEND|os.O_TRUNC, 0644)
	if err != nil {
		return err
	}

	defer file.Close()
	_, err = file.Write(h.Format())

	return err
}
<|endoftext|>