<commit_before>package gb

import (
	"fmt"
	"path/filepath"
	"runtime"
	"strings"
	"time"
)

// Build builds each of pkgs in succession. If pkg is a command, then the results of build include
// linking the final binary into pkg.Context.Bindir().
func Build(pkgs ...*Package) error {
	build, err := BuildPackages(pkgs...)
	if err != nil {
		return err
	}
	return ExecuteConcurrent(build, runtime.NumCPU())
}

// BuildAction produces a tree of *Actions that can be executed to build
// a *Package.
// BuildAction walks the tree of *Packages and returns a corresponding
// tree of *Actions representing the steps required to build *Package
// and any of its dependencies
func BuildPackages(pkgs ...*Package) (*Action, error) {
	targets := make(map[string]*Action) // maps package importpath ot Action name

	names := func(pkgs []*Package) []string {
		var names []string
		for _, pkg := range pkgs {
			names = append(names, pkg.ImportPath)
		}
		return names
	}

	// create top level build action to unify all packages
	build := Action{
		Name: fmt.Sprintf("build: %s", strings.Join(names(pkgs), ",")),
		Task: TaskFn(func() error {
			fmt.Println("built")
			return nil
		}),
	}

	for _, pkg := range pkgs {
		a, err := BuildPackage(targets, pkg)
		if err != nil {
			return nil, err
		}
		if a == nil {
			// nothing to do
			continue
		}
		build.Deps = append(build.Deps, a)
	}
	return &build, nil
}

// BuildPackage returns an Action representing the steps required to
// build this package.
func BuildPackage(targets map[string]*Action, pkg *Package) (*Action, error) {

	// if this action is already present in the map, return it
	// rather than creating a new action.
	if a, ok := targets[pkg.ImportPath]; ok {
		return a, nil
	}

	// step 0. are we stale ?
	// if this package is not stale, then by definition none of its
	// dependencies are stale, so ignore this whole tree.
	if !pkg.Stale {
		return nil, nil
	}

	// step 1. build dependencies
	deps, err := BuildDependencies(targets, pkg)
	if err != nil {
		return nil, err
	}
	build, err := Compile(pkg, deps...)
	if err != nil {
		return nil, err
	}

	if build == nil {
		panic("build action was nil") // shouldn't happen
	}

	// record the final action as the action that represents
	// building this package.
	targets[pkg.ImportPath] = build
	return build, nil
}

// Compile returns an Action representing the steps required to build this package.
func Compile(pkg *Package, deps ...*Action) (*Action, error) {
	// step 0. are there any .s files to assemble.

	var assemble []*Action
	var ofiles []string // additional ofiles to pack
	for _, sfile := range pkg.SFiles {
		sfile := sfile
		ofile := filepath.Join(pkg.Workdir(), pkg.ImportPath, stripext(sfile)+".6")
		assemble = append(assemble, &Action{
			Name: fmt.Sprintf("asm: %s/%s", pkg.ImportPath, sfile),
			Task: TaskFn(func() error {
				t0 := time.Now()
				err := pkg.tc.Asm(pkg, pkg.Dir, ofile, filepath.Join(pkg.Dir, sfile))
				pkg.Record("asm", time.Since(t0))
				return err
			}),
		})
		ofiles = append(ofiles, ofile)
	}

	var gofiles []string
	gofiles = append(gofiles, pkg.GoFiles...)

	// step 1. are there any .c files that we have to run cgo on ?

	if len(pkg.CgoFiles) > 0 {
		cgoACTION, cgoOFILES, cgoGOFILES, err := cgo(pkg)
		if err != nil {
			return nil, err
		}

		gofiles = append(gofiles, cgoGOFILES...)
		ofiles = append(ofiles, cgoOFILES...)
		deps = append(deps, cgoACTION)
	}

	// step 2. compile all the go files for this package, including pkg.CgoFiles

	compile := Action{
		Name: fmt.Sprintf("compile: %s", pkg.ImportPath),
		Deps: deps,
		Task: TaskFn(func() error {
			return Gc(pkg, gofiles)
		}),
	}

	build := &compile

	// Do we need to pack ? Yes, replace build action with pack.
	if len(ofiles) > 0 {
		pack := Action{
			Name: fmt.Sprintf("pack: %s", pkg.ImportPath),
			Deps: []*Action{
				&compile,
			},
			Task: TaskFn(func() error {
				// collect .o files, ofiles always starts with the gc compiled object.
				// TODO(dfc) objfile(pkg) should already be at the top of this set
				ofiles = append(
					[]string{objfile(pkg)},
					ofiles...,
				)

				// pack
				t0 := time.Now()
				err := pkg.tc.Pack(pkg, ofiles...)
				pkg.Record("pack", time.Since(t0))
				return err
			}),
		}
		pack.Deps = append(pack.Deps, assemble...)
		build = &pack
	}

	// should this package be cached
	// TODO(dfc) pkg.SkipInstall should become Install
	if !pkg.SkipInstall && pkg.Scope != "test" {
		install := Action{
			Name: fmt.Sprintf("install: %s", pkg.ImportPath),
			Deps: []*Action{
				build,
			},
			Task: TaskFn(func() error {
				return copyfile(pkgfile(pkg), objfile(pkg))
			}),
		}
		build = &install
	}

	// if this is a main package, add a link stage
	if pkg.isMain() {
		link := Action{
			Name: fmt.Sprintf("link: %s", pkg.ImportPath),
			Deps: []*Action{build},
			Task: TaskFn(func() error {
				return Link(pkg)
			}),
		}
		build = &link
	}
	return build, nil
}

// BuildDependencies returns a slice of Actions representing the steps required
// to build all dependant packages of this package.
func BuildDependencies(targets map[string]*Action, pkg *Package) ([]*Action, error) {
	var deps []*Action
	for _, i := range pkg.Imports() {
		a, err := BuildPackage(targets, i)
		if err != nil {
			return nil, err
		}
		if a == nil {
			// no action required for this Package
			continue
		}
		deps = append(deps, a)
	}
	return deps, nil
}

func Gc(pkg *Package, gofiles []string) error {
	t0 := time.Now()
	if pkg.Scope != "test" {
		// only log compilation message if not in test scope
		Infof(pkg.ImportPath)
	}
	includes := pkg.IncludePaths()
	importpath := pkg.ImportPath
	if pkg.Scope == "test" && pkg.ExtraIncludes != "" {
		// TODO(dfc) gross
		includes = append([]string{pkg.ExtraIncludes}, includes...)
	}
	for i := range gofiles {
		if filepath.IsAbs(gofiles[i]) {
			// terrible hack for cgo files which come with an absolute path
			continue
		}
		fullpath := filepath.Join(pkg.Dir, gofiles[i])
		path, err := filepath.Rel(pkg.Projectdir(), fullpath)
		if err == nil {
			gofiles[i] = path
		} else {
			gofiles[i] = fullpath
		}
	}
	err := pkg.tc.Gc(pkg, includes, importpath, pkg.Projectdir(), objfile(pkg), gofiles, pkg.Complete())
	pkg.Record("compile", time.Since(t0))
	return err
}

func Link(pkg *Package) error {
	t0 := time.Now()
	target := pkg.Binfile()
	if err := mkdir(filepath.Dir(target)); err != nil {
		return err
	}

	includes := pkg.IncludePaths()
	if pkg.Scope == "test" && pkg.ExtraIncludes != "" {
		// TODO(dfc) gross
		includes = append([]string{pkg.ExtraIncludes}, includes...)
		target += ".test"
	}
	err := pkg.tc.Ld(pkg, includes, target, objfile(pkg))
	pkg.Record("link", time.Since(t0))
	return err
}

// objfile returns the name of the object file for this package
func objfile(pkg *Package) string {
	return filepath.Join(pkg.Objdir(), objname(pkg))
}

func objname(pkg *Package) string {
	switch pkg.Name {
	case "main":
		return filepath.Join(filepath.Base(filepath.FromSlash(pkg.ImportPath)), "main.a")
	default:
		return filepath.Base(filepath.FromSlash(pkg.ImportPath)) + ".a"
	}
}

func pkgname(pkg *Package) string {
	if pkg.isMain() {
		return filepath.Base(filepath.FromSlash(pkg.ImportPath))
	}
	return pkg.Name
}

// Binfile returns the destination of the compiled target of this command.
// TODO(dfc) this should be Target.
func (pkg *Package) Binfile() string {
	// TODO(dfc) should have a check for package main, or should be merged in to objfile.
	var target string
	switch pkg.Scope {
	case "test":
		target = filepath.Join(pkg.Workdir(), filepath.FromSlash(pkg.ImportPath), "_test", binname(pkg))
	default:
		target = filepath.Join(pkg.Bindir(), binname(pkg))
	}
	if pkg.GOOS == "windows" {
		target += ".exe"
	}
	return target
}

func binname(pkg *Package) string {
	switch {
	case pkg.Name == "main":
		return filepath.Base(filepath.FromSlash(pkg.ImportPath))
	case pkg.Scope == "test":
		return pkg.Name
	default:
		panic("binname called with non main package: " + pkg.ImportPath)
	}
}
<commit_msg>cleanup and unexport helper functions<commit_after>package gb

import (
	"fmt"
	"path/filepath"
	"runtime"
	"strings"
	"time"
)

// Build builds each of pkgs in succession. If pkg is a command, then the results of build include
// linking the final binary into pkg.Context.Bindir().
func Build(pkgs ...*Package) error {
	build, err := BuildPackages(pkgs...)
	if err != nil {
		return err
	}
	return ExecuteConcurrent(build, runtime.NumCPU())
}

// BuildAction produces a tree of *Actions that can be executed to build
// a *Package.
// BuildAction walks the tree of *Packages and returns a corresponding
// tree of *Actions representing the steps required to build *Package
// and any of its dependencies
func BuildPackages(pkgs ...*Package) (*Action, error) {
	targets := make(map[string]*Action) // maps package importpath ot Action name

	names := func(pkgs []*Package) []string {
		var names []string
		for _, pkg := range pkgs {
			names = append(names, pkg.ImportPath)
		}
		return names
	}

	// create top level build action to unify all packages
	build := Action{
		Name: fmt.Sprintf("build: %s", strings.Join(names(pkgs), ",")),
		Task: TaskFn(func() error {
			fmt.Println("built")
			return nil
		}),
	}

	for _, pkg := range pkgs {
		a, err := BuildPackage(targets, pkg)
		if err != nil {
			return nil, err
		}
		if a == nil {
			// nothing to do
			continue
		}
		build.Deps = append(build.Deps, a)
	}
	return &build, nil
}

// BuildPackage returns an Action representing the steps required to
// build this package.
func BuildPackage(targets map[string]*Action, pkg *Package) (*Action, error) {

	// if this action is already present in the map, return it
	// rather than creating a new action.
	if a, ok := targets[pkg.ImportPath]; ok {
		return a, nil
	}

	// step 0. are we stale ?
	// if this package is not stale, then by definition none of its
	// dependencies are stale, so ignore this whole tree.
	if !pkg.Stale {
		return nil, nil
	}

	// step 1. build dependencies
	deps, err := BuildDependencies(targets, pkg)
	if err != nil {
		return nil, err
	}
	build, err := Compile(pkg, deps...)
	if err != nil {
		return nil, err
	}

	if build == nil {
		panic("build action was nil") // shouldn't happen
	}

	// record the final action as the action that represents
	// building this package.
	targets[pkg.ImportPath] = build
	return build, nil
}

// Compile returns an Action representing the steps required to compile this package.
func Compile(pkg *Package, deps ...*Action) (*Action, error) {

	// step 0. are there any .s files to assemble.
	var assemble []*Action
	var ofiles []string // additional ofiles to pack
	for _, sfile := range pkg.SFiles {
		sfile := sfile
		ofile := filepath.Join(pkg.Workdir(), pkg.ImportPath, stripext(sfile)+".6")
		assemble = append(assemble, &Action{
			Name: fmt.Sprintf("asm: %s/%s", pkg.ImportPath, sfile),
			Task: TaskFn(func() error {
				t0 := time.Now()
				err := pkg.tc.Asm(pkg, pkg.Dir, ofile, filepath.Join(pkg.Dir, sfile))
				pkg.Record("asm", time.Since(t0))
				return err
			}),
		})
		ofiles = append(ofiles, ofile)
	}

	var gofiles []string
	gofiles = append(gofiles, pkg.GoFiles...)

	// step 1. are there any .c files that we have to run cgo on ?
	if len(pkg.CgoFiles) > 0 {
		cgoACTION, cgoOFILES, cgoGOFILES, err := cgo(pkg)
		if err != nil {
			return nil, err
		}

		gofiles = append(gofiles, cgoGOFILES...)
		ofiles = append(ofiles, cgoOFILES...)
		deps = append(deps, cgoACTION)
	}

	// step 2. compile all the go files for this package, including pkg.CgoFiles
	compile := Action{
		Name: fmt.Sprintf("compile: %s", pkg.ImportPath),
		Deps: deps,
		Task: TaskFn(func() error {
			return gc(pkg, gofiles)
		}),
	}

	build := &compile

	// Do we need to pack ? Yes, replace build action with pack.
	if len(ofiles) > 0 {
		pack := Action{
			Name: fmt.Sprintf("pack: %s", pkg.ImportPath),
			Deps: []*Action{
				&compile,
			},
			Task: TaskFn(func() error {
				// collect .o files, ofiles always starts with the gc compiled object.
				// TODO(dfc) objfile(pkg) should already be at the top of this set
				ofiles = append(
					[]string{objfile(pkg)},
					ofiles...,
				)

				// pack
				t0 := time.Now()
				err := pkg.tc.Pack(pkg, ofiles...)
				pkg.Record("pack", time.Since(t0))
				return err
			}),
		}
		pack.Deps = append(pack.Deps, assemble...)
		build = &pack
	}

	// should this package be cached
	// TODO(dfc) pkg.SkipInstall should become Install
	if !pkg.SkipInstall && pkg.Scope != "test" {
		install := Action{
			Name: fmt.Sprintf("install: %s", pkg.ImportPath),
			Deps: []*Action{
				build,
			},
			Task: TaskFn(func() error {
				return copyfile(pkgfile(pkg), objfile(pkg))
			}),
		}
		build = &install
	}

	// if this is a main package, add a link stage
	if pkg.isMain() {
		link := Action{
			Name: fmt.Sprintf("link: %s", pkg.ImportPath),
			Deps: []*Action{build},
			Task: TaskFn(func() error {
				return link(pkg)
			}),
		}
		build = &link
	}
	return build, nil
}

// BuildDependencies returns a slice of Actions representing the steps required
// to build all dependant packages of this package.
func BuildDependencies(targets map[string]*Action, pkg *Package) ([]*Action, error) {
	var deps []*Action
	for _, i := range pkg.Imports() {
		a, err := BuildPackage(targets, i)
		if err != nil {
			return nil, err
		}
		if a == nil {
			// no action required for this Package
			continue
		}
		deps = append(deps, a)
	}
	return deps, nil
}

func gc(pkg *Package, gofiles []string) error {
	t0 := time.Now()
	if pkg.Scope != "test" {
		// only log compilation message if not in test scope
		Infof(pkg.ImportPath)
	}
	includes := pkg.IncludePaths()
	importpath := pkg.ImportPath
	if pkg.Scope == "test" && pkg.ExtraIncludes != "" {
		// TODO(dfc) gross
		includes = append([]string{pkg.ExtraIncludes}, includes...)
	}
	for i := range gofiles {
		if filepath.IsAbs(gofiles[i]) {
			// terrible hack for cgo files which come with an absolute path
			continue
		}
		fullpath := filepath.Join(pkg.Dir, gofiles[i])
		path, err := filepath.Rel(pkg.Projectdir(), fullpath)
		if err == nil {
			gofiles[i] = path
		} else {
			gofiles[i] = fullpath
		}
	}
	err := pkg.tc.Gc(pkg, includes, importpath, pkg.Projectdir(), objfile(pkg), gofiles, pkg.Complete())
	pkg.Record("compile", time.Since(t0))
	return err
}

func link(pkg *Package) error {
	t0 := time.Now()
	target := pkg.Binfile()
	if err := mkdir(filepath.Dir(target)); err != nil {
		return err
	}

	includes := pkg.IncludePaths()
	if pkg.Scope == "test" && pkg.ExtraIncludes != "" {
		// TODO(dfc) gross
		includes = append([]string{pkg.ExtraIncludes}, includes...)
		target += ".test"
	}
	err := pkg.tc.Ld(pkg, includes, target, objfile(pkg))
	pkg.Record("link", time.Since(t0))
	return err
}

// objfile returns the name of the object file for this package
func objfile(pkg *Package) string {
	return filepath.Join(pkg.Objdir(), objname(pkg))
}

func objname(pkg *Package) string {
	switch pkg.Name {
	case "main":
		return filepath.Join(filepath.Base(filepath.FromSlash(pkg.ImportPath)), "main.a")
	default:
		return filepath.Base(filepath.FromSlash(pkg.ImportPath)) + ".a"
	}
}

func pkgname(pkg *Package) string {
	if pkg.isMain() {
		return filepath.Base(filepath.FromSlash(pkg.ImportPath))
	}
	return pkg.Name
}

// Binfile returns the destination of the compiled target of this command.
func (pkg *Package) Binfile() string {
	// TODO(dfc) should have a check for package main, or should be merged in to objfile.
	var target string
	switch pkg.Scope {
	case "test":
		target = filepath.Join(pkg.Workdir(), filepath.FromSlash(pkg.ImportPath), "_test", binname(pkg))
	default:
		target = filepath.Join(pkg.Bindir(), binname(pkg))
	}
	if pkg.GOOS == "windows" {
		target += ".exe"
	}
	return target
}

func binname(pkg *Package) string {
	switch {
	case pkg.Name == "main":
		return filepath.Base(filepath.FromSlash(pkg.ImportPath))
	case pkg.Scope == "test":
		return pkg.Name
	default:
		panic("binname called with non main package: " + pkg.ImportPath)
	}
}
<|endoftext|>


<commit_before>// +build ignore

package main

import (
	"bytes"
	"crypto/md5"
	"encoding/json"
	"flag"
	"fmt"
	"io"
	"io/ioutil"
	"log"
	"os"
	"os/exec"
	"path/filepath"
	"regexp"
	"runtime"
	"strconv"
	"strings"
	"time"
)

var (
	versionRe        = regexp.MustCompile(`-[0-9]{1,3}-g[0-9a-f]{5,10}`)
	goarch           string
	goos             string
	version          string = "v1"
	race             bool
	workingDir       string
	serverBinaryName string = "grafana-server"
)

const minGoVersion = 1.3

func main() {
	log.SetOutput(os.Stdout)
	log.SetFlags(0)

	ensureGoPath()
	readVersionFromPackageJson()

	log.Printf("Version: %s\n", version)

	flag.StringVar(&goarch, "goarch", runtime.GOARCH, "GOARCH")
	flag.StringVar(&goos, "goos", runtime.GOOS, "GOOS")
	flag.BoolVar(&race, "race", race, "Use race detector")
	flag.Parse()

	if flag.NArg() == 0 {
		log.Println("Usage: go run build.go build")
		return
	}

	workingDir, _ = os.Getwd()

	for _, cmd := range flag.Args() {
		switch cmd {
		case "setup":
			setup()

		case "build":
			pkg := "."
			clean()
			build(pkg, []string{})

		case "test":
			test("./pkg/...")
			grunt("test")

		case "package":
			//verifyGitRepoIsClean()
			//grunt("release", "--pkgVer="+version)
			createLinuxPackages()

		case "latest":
			makeLatestDistCopies()

		case "clean":
			clean()

		default:
			log.Fatalf("Unknown command %q", cmd)
		}
	}
}

func makeLatestDistCopies() {
	runError("cp", "dist/grafana_"+version+"_amd64.deb", "dist/grafana_latest_amd64.deb")
	runError("cp", "dist/grafana-"+strings.Replace(version, "-", "_", 5)+"-1.x86_64.rpm", "dist/grafana-latest-1.x86_64.rpm")
	runError("cp", "dist/grafana-"+version+".x86_64.tar.gz", "dist/grafana-latest.x86_64.tar.gz")
}

func readVersionFromPackageJson() {
	reader, err := os.Open("package.json")
	if err != nil {
		log.Fatal("Failed to open package.json")
		return
	}
	defer reader.Close()

	jsonObj := map[string]interface{}{}
	jsonParser := json.NewDecoder(reader)

	if err := jsonParser.Decode(&jsonObj); err != nil {
		log.Fatal("Failed to decode package.json")
	}

	version = jsonObj["version"].(string)
}

type linuxPackageOptions struct {
	packageType            string
	homeDir                string
	binPath                string
	configDir              string
	configFilePath         string
	etcDefaultPath         string
	etcDefaultFilePath     string
	initdScriptFilePath    string
	systemdServiceFilePath string

	postinstSrc    string
	initdScriptSrc string
	defaultFileSrc string
	systemdFileSrc string

	depends []string
}

func createLinuxPackages() {
	createPackage(linuxPackageOptions{
		packageType:            "deb",
		homeDir:                "/usr/share/grafana",
		binPath:                "/usr/sbin/grafana-server",
		configDir:              "/etc/grafana",
		configFilePath:         "/etc/grafana/grafana.ini",
		etcDefaultPath:         "/etc/default",
		etcDefaultFilePath:     "/etc/default/grafana-server",
		initdScriptFilePath:    "/etc/init.d/grafana-server",
		systemdServiceFilePath: "/usr/lib/systemd/system/grafana-server.service",

		postinstSrc:    "packaging/deb/control/postinst",
		initdScriptSrc: "packaging/deb/init.d/grafana-server",
		defaultFileSrc: "packaging/deb/default/grafana-server",
		systemdFileSrc: "packaging/deb/systemd/grafana-server.service",

		depends: []string{"adduser", "libfontconfig"},
	})

	createPackage(linuxPackageOptions{
		packageType:            "rpm",
		homeDir:                "/usr/share/grafana",
		binPath:                "/usr/sbin/grafana-server",
		configDir:              "/etc/grafana",
		configFilePath:         "/etc/grafana/grafana.ini",
		etcDefaultPath:         "/etc/sysconfig",
		etcDefaultFilePath:     "/etc/sysconfig/grafana-server",
		initdScriptFilePath:    "/etc/init.d/grafana-server",
		systemdServiceFilePath: "/usr/lib/systemd/system/grafana-server.service",

		postinstSrc:    "packaging/rpm/control/postinst",
		initdScriptSrc: "packaging/rpm/init.d/grafana-server",
		defaultFileSrc: "packaging/rpm/sysconfig/grafana-server",
		systemdFileSrc: "packaging/rpm/systemd/grafana-server.service",

		depends: []string{"initscripts", "fontconfig"},
	})
}

func createPackage(options linuxPackageOptions) {
	packageRoot, _ := ioutil.TempDir("", "grafana-linux-pack")

	// create directories
	runPrint("mkdir", "-p", filepath.Join(packageRoot, options.homeDir))
	runPrint("mkdir", "-p", filepath.Join(packageRoot, options.configDir))
	runPrint("mkdir", "-p", filepath.Join(packageRoot, "/etc/init.d"))
	runPrint("mkdir", "-p", filepath.Join(packageRoot, options.etcDefaultPath))
	runPrint("mkdir", "-p", filepath.Join(packageRoot, "/usr/lib/systemd/system"))
	runPrint("mkdir", "-p", filepath.Join(packageRoot, "/usr/sbin"))

	// copy binary
	runPrint("cp", "-p", filepath.Join(workingDir, "tmp/bin/"+serverBinaryName), filepath.Join(packageRoot, options.binPath))
	// copy init.d script
	runPrint("cp", "-p", options.initdScriptSrc, filepath.Join(packageRoot, options.initdScriptFilePath))
	// copy environment var file
	runPrint("cp", "-p", options.defaultFileSrc, filepath.Join(packageRoot, options.etcDefaultFilePath))
	// copy systemd file
	runPrint("cp", "-p", options.systemdFileSrc, filepath.Join(packageRoot, options.systemdServiceFilePath))
	// copy release files
	runPrint("cp", "-a", filepath.Join(workingDir, "tmp")+"/.", filepath.Join(packageRoot, options.homeDir))
	// copy sample ini file to /etc/opt/grafana
	runPrint("cp", "conf/sample.ini", filepath.Join(packageRoot, options.configFilePath))

	args := []string{
		"-s", "dir",
		"--description", "Grafana",
		"-C", packageRoot,
		"--vendor", "Grafana",
		"--url", "http://grafana.org",
		"--license", "Apache 2.0",
		"--maintainer", "contact@grafana.org",
		"--config-files", options.configFilePath,
		"--config-files", options.initdScriptFilePath,
		"--config-files", options.etcDefaultFilePath,
		"--config-files", options.systemdServiceFilePath,
		"--after-install", options.postinstSrc,
		"--name", "grafana",
		"--version", version,
		"-p", "./dist",
	}

	// add dependenciesj
	for _, dep := range options.depends {
		args = append(args, "--depends", dep)
	}

	args = append(args, ".")

	fmt.Println("Creating package: ", options.packageType)
	runPrint("fpm", append([]string{"-t", options.packageType}, args...)...)
}

func verifyGitRepoIsClean() {
	rs, err := runError("git", "ls-files", "--modified")
	if err != nil {
		log.Fatalf("Failed to check if git tree was clean, %v, %v\n", string(rs), err)
		return
	}
	count := len(string(rs))
	if count > 0 {
		log.Fatalf("Git repository has modified files, aborting")
	}

	log.Println("Git repository is clean")
}

func ensureGoPath() {
	if os.Getenv("GOPATH") == "" {
		cwd, err := os.Getwd()
		if err != nil {
			log.Fatal(err)
		}
		gopath := filepath.Clean(filepath.Join(cwd, "../../../../"))
		log.Println("GOPATH is", gopath)
		os.Setenv("GOPATH", gopath)
	}
}

func ChangeWorkingDir(dir string) {
	os.Chdir(dir)
}

func grunt(params ...string) {
	runPrint("./node_modules/grunt-cli/bin/grunt", params...)
}

func setup() {
	runPrint("go", "get", "-v", "github.com/tools/godep")
	runPrint("go", "get", "-v", "github.com/mattn/go-sqlite3")
	runPrint("go", "install", "-v", "github.com/mattn/go-sqlite3")
}

func test(pkg string) {
	setBuildEnv()
	runPrint("go", "test", "-short", "-timeout", "60s", pkg)
}

func build(pkg string, tags []string) {
	binary := "./bin/" + serverBinaryName
	if goos == "windows" {
		binary += ".exe"
	}

	rmr(binary, binary+".md5")
	args := []string{"build", "-ldflags", ldflags()}
	if len(tags) > 0 {
		args = append(args, "-tags", strings.Join(tags, ","))
	}
	if race {
		args = append(args, "-race")
	}

	args = append(args, "-o", binary)
	args = append(args, pkg)
	setBuildEnv()
	runPrint("go", args...)

	// Create an md5 checksum of the binary, to be included in the archive for
	// automatic upgrades.
	err := md5File(binary)
	if err != nil {
		log.Fatal(err)
	}
}

func ldflags() string {
	var b bytes.Buffer
	b.WriteString("-w")
	b.WriteString(fmt.Sprintf(" -X main.version '%s'", version))
	b.WriteString(fmt.Sprintf(" -X main.commit '%s'", getGitSha()))
	b.WriteString(fmt.Sprintf(" -X main.buildstamp %d", buildStamp()))
	return b.String()
}

func rmr(paths ...string) {
	for _, path := range paths {
		log.Println("rm -r", path)
		os.RemoveAll(path)
	}
}

func clean() {
	rmr("bin", "Godeps/_workspace/pkg", "Godeps/_workspace/bin")
	rmr("dist")
	rmr("tmp")
	rmr(filepath.Join(os.Getenv("GOPATH"), fmt.Sprintf("pkg/%s_%s/github.com/grafana", goos, goarch)))
}

func setBuildEnv() {
	os.Setenv("GOOS", goos)
	if strings.HasPrefix(goarch, "armv") {
		os.Setenv("GOARCH", "arm")
		os.Setenv("GOARM", goarch[4:])
	} else {
		os.Setenv("GOARCH", goarch)
	}
	if goarch == "386" {
		os.Setenv("GO386", "387")
	}
	wd, err := os.Getwd()
	if err != nil {
		log.Println("Warning: can't determine current dir:", err)
		log.Println("Build might not work as expected")
	}
	os.Setenv("GOPATH", fmt.Sprintf("%s%c%s", filepath.Join(wd, "Godeps", "_workspace"), os.PathListSeparator, os.Getenv("GOPATH")))
	log.Println("GOPATH=" + os.Getenv("GOPATH"))
}

func getGitSha() string {
	v, err := runError("git", "describe", "--always", "--dirty")
	if err != nil {
		return "unknown-dev"
	}
	v = versionRe.ReplaceAllFunc(v, func(s []byte) []byte {
		s[0] = '+'
		return s
	})
	return string(v)
}

func buildStamp() int64 {
	bs, err := runError("git", "show", "-s", "--format=%ct")
	if err != nil {
		return time.Now().Unix()
	}
	s, _ := strconv.ParseInt(string(bs), 10, 64)
	return s
}

func buildArch() string {
	os := goos
	if os == "darwin" {
		os = "macosx"
	}
	return fmt.Sprintf("%s-%s", os, goarch)
}

func run(cmd string, args ...string) []byte {
	bs, err := runError(cmd, args...)
	if err != nil {
		log.Println(cmd, strings.Join(args, " "))
		log.Println(string(bs))
		log.Fatal(err)
	}
	return bytes.TrimSpace(bs)
}

func runError(cmd string, args ...string) ([]byte, error) {
	ecmd := exec.Command(cmd, args...)
	bs, err := ecmd.CombinedOutput()
	if err != nil {
		return nil, err
	}

	return bytes.TrimSpace(bs), nil
}

func runPrint(cmd string, args ...string) {
	log.Println(cmd, strings.Join(args, " "))
	ecmd := exec.Command(cmd, args...)
	ecmd.Stdout = os.Stdout
	ecmd.Stderr = os.Stderr
	err := ecmd.Run()
	if err != nil {
		log.Fatal(err)
	}
}

func md5File(file string) error {
	fd, err := os.Open(file)
	if err != nil {
		return err
	}
	defer fd.Close()

	h := md5.New()
	_, err = io.Copy(h, fd)
	if err != nil {
		return err
	}

	out, err := os.Create(file + ".md5")
	if err != nil {
		return err
	}

	_, err = fmt.Fprintf(out, "%x\n", h.Sum(nil))
	if err != nil {
		return err
	}

	return out.Close()
}
<commit_msg>Updated build.go<commit_after>// +build ignore

package main

import (
	"bytes"
	"crypto/md5"
	"encoding/json"
	"flag"
	"fmt"
	"io"
	"io/ioutil"
	"log"
	"os"
	"os/exec"
	"path/filepath"
	"regexp"
	"runtime"
	"strconv"
	"strings"
	"time"
)

var (
	versionRe        = regexp.MustCompile(`-[0-9]{1,3}-g[0-9a-f]{5,10}`)
	goarch           string
	goos             string
	version          string = "v1"
	race             bool
	workingDir       string
	serverBinaryName string = "grafana-server"
)

const minGoVersion = 1.3

func main() {
	log.SetOutput(os.Stdout)
	log.SetFlags(0)

	ensureGoPath()
	readVersionFromPackageJson()

	log.Printf("Version: %s\n", version)

	flag.StringVar(&goarch, "goarch", runtime.GOARCH, "GOARCH")
	flag.StringVar(&goos, "goos", runtime.GOOS, "GOOS")
	flag.BoolVar(&race, "race", race, "Use race detector")
	flag.Parse()

	if flag.NArg() == 0 {
		log.Println("Usage: go run build.go build")
		return
	}

	workingDir, _ = os.Getwd()

	for _, cmd := range flag.Args() {
		switch cmd {
		case "setup":
			setup()

		case "build":
			pkg := "."
			clean()
			build(pkg, []string{})

		case "test":
			test("./pkg/...")
			grunt("test")

		case "package":
			//verifyGitRepoIsClean()
			grunt("release", "--pkgVer="+version)
			createLinuxPackages()

		case "latest":
			makeLatestDistCopies()

		case "clean":
			clean()

		default:
			log.Fatalf("Unknown command %q", cmd)
		}
	}
}

func makeLatestDistCopies() {
	runError("cp", "dist/grafana_"+version+"_amd64.deb", "dist/grafana_latest_amd64.deb")
	runError("cp", "dist/grafana-"+strings.Replace(version, "-", "_", 5)+"-1.x86_64.rpm", "dist/grafana-latest-1.x86_64.rpm")
	runError("cp", "dist/grafana-"+version+".x86_64.tar.gz", "dist/grafana-latest.x86_64.tar.gz")
}

func readVersionFromPackageJson() {
	reader, err := os.Open("package.json")
	if err != nil {
		log.Fatal("Failed to open package.json")
		return
	}
	defer reader.Close()

	jsonObj := map[string]interface{}{}
	jsonParser := json.NewDecoder(reader)

	if err := jsonParser.Decode(&jsonObj); err != nil {
		log.Fatal("Failed to decode package.json")
	}

	version = jsonObj["version"].(string)
}

type linuxPackageOptions struct {
	packageType            string
	homeDir                string
	binPath                string
	configDir              string
	configFilePath         string
	etcDefaultPath         string
	etcDefaultFilePath     string
	initdScriptFilePath    string
	systemdServiceFilePath string

	postinstSrc    string
	initdScriptSrc string
	defaultFileSrc string
	systemdFileSrc string

	depends []string
}

func createLinuxPackages() {
	createPackage(linuxPackageOptions{
		packageType:            "deb",
		homeDir:                "/usr/share/grafana",
		binPath:                "/usr/sbin/grafana-server",
		configDir:              "/etc/grafana",
		configFilePath:         "/etc/grafana/grafana.ini",
		etcDefaultPath:         "/etc/default",
		etcDefaultFilePath:     "/etc/default/grafana-server",
		initdScriptFilePath:    "/etc/init.d/grafana-server",
		systemdServiceFilePath: "/usr/lib/systemd/system/grafana-server.service",

		postinstSrc:    "packaging/deb/control/postinst",
		initdScriptSrc: "packaging/deb/init.d/grafana-server",
		defaultFileSrc: "packaging/deb/default/grafana-server",
		systemdFileSrc: "packaging/deb/systemd/grafana-server.service",

		depends: []string{"adduser", "libfontconfig"},
	})

	createPackage(linuxPackageOptions{
		packageType:            "rpm",
		homeDir:                "/usr/share/grafana",
		binPath:                "/usr/sbin/grafana-server",
		configDir:              "/etc/grafana",
		configFilePath:         "/etc/grafana/grafana.ini",
		etcDefaultPath:         "/etc/sysconfig",
		etcDefaultFilePath:     "/etc/sysconfig/grafana-server",
		initdScriptFilePath:    "/etc/init.d/grafana-server",
		systemdServiceFilePath: "/usr/lib/systemd/system/grafana-server.service",

		postinstSrc:    "packaging/rpm/control/postinst",
		initdScriptSrc: "packaging/rpm/init.d/grafana-server",
		defaultFileSrc: "packaging/rpm/sysconfig/grafana-server",
		systemdFileSrc: "packaging/rpm/systemd/grafana-server.service",

		depends: []string{"initscripts", "fontconfig"},
	})
}

func createPackage(options linuxPackageOptions) {
	packageRoot, _ := ioutil.TempDir("", "grafana-linux-pack")

	// create directories
	runPrint("mkdir", "-p", filepath.Join(packageRoot, options.homeDir))
	runPrint("mkdir", "-p", filepath.Join(packageRoot, options.configDir))
	runPrint("mkdir", "-p", filepath.Join(packageRoot, "/etc/init.d"))
	runPrint("mkdir", "-p", filepath.Join(packageRoot, options.etcDefaultPath))
	runPrint("mkdir", "-p", filepath.Join(packageRoot, "/usr/lib/systemd/system"))
	runPrint("mkdir", "-p", filepath.Join(packageRoot, "/usr/sbin"))

	// copy binary
	runPrint("cp", "-p", filepath.Join(workingDir, "tmp/bin/"+serverBinaryName), filepath.Join(packageRoot, options.binPath))
	// copy init.d script
	runPrint("cp", "-p", options.initdScriptSrc, filepath.Join(packageRoot, options.initdScriptFilePath))
	// copy environment var file
	runPrint("cp", "-p", options.defaultFileSrc, filepath.Join(packageRoot, options.etcDefaultFilePath))
	// copy systemd file
	runPrint("cp", "-p", options.systemdFileSrc, filepath.Join(packageRoot, options.systemdServiceFilePath))
	// copy release files
	runPrint("cp", "-a", filepath.Join(workingDir, "tmp")+"/.", filepath.Join(packageRoot, options.homeDir))
	// copy sample ini file to /etc/opt/grafana
	runPrint("cp", "conf/sample.ini", filepath.Join(packageRoot, options.configFilePath))

	args := []string{
		"-s", "dir",
		"--description", "Grafana",
		"-C", packageRoot,
		"--vendor", "Grafana",
		"--url", "http://grafana.org",
		"--license", "Apache 2.0",
		"--maintainer", "contact@grafana.org",
		"--config-files", options.configFilePath,
		"--config-files", options.initdScriptFilePath,
		"--config-files", options.etcDefaultFilePath,
		"--config-files", options.systemdServiceFilePath,
		"--after-install", options.postinstSrc,
		"--name", "grafana",
		"--version", version,
		"-p", "./dist",
	}

	// add dependenciesj
	for _, dep := range options.depends {
		args = append(args, "--depends", dep)
	}

	args = append(args, ".")

	fmt.Println("Creating package: ", options.packageType)
	runPrint("fpm", append([]string{"-t", options.packageType}, args...)...)
}

func verifyGitRepoIsClean() {
	rs, err := runError("git", "ls-files", "--modified")
	if err != nil {
		log.Fatalf("Failed to check if git tree was clean, %v, %v\n", string(rs), err)
		return
	}
	count := len(string(rs))
	if count > 0 {
		log.Fatalf("Git repository has modified files, aborting")
	}

	log.Println("Git repository is clean")
}

func ensureGoPath() {
	if os.Getenv("GOPATH") == "" {
		cwd, err := os.Getwd()
		if err != nil {
			log.Fatal(err)
		}
		gopath := filepath.Clean(filepath.Join(cwd, "../../../../"))
		log.Println("GOPATH is", gopath)
		os.Setenv("GOPATH", gopath)
	}
}

func ChangeWorkingDir(dir string) {
	os.Chdir(dir)
}

func grunt(params ...string) {
	runPrint("./node_modules/grunt-cli/bin/grunt", params...)
}

func setup() {
	runPrint("go", "get", "-v", "github.com/tools/godep")
	runPrint("go", "get", "-v", "github.com/mattn/go-sqlite3")
	runPrint("go", "install", "-v", "github.com/mattn/go-sqlite3")
}

func test(pkg string) {
	setBuildEnv()
	runPrint("go", "test", "-short", "-timeout", "60s", pkg)
}

func build(pkg string, tags []string) {
	binary := "./bin/" + serverBinaryName
	if goos == "windows" {
		binary += ".exe"
	}

	rmr(binary, binary+".md5")
	args := []string{"build", "-ldflags", ldflags()}
	if len(tags) > 0 {
		args = append(args, "-tags", strings.Join(tags, ","))
	}
	if race {
		args = append(args, "-race")
	}

	args = append(args, "-o", binary)
	args = append(args, pkg)
	setBuildEnv()
	runPrint("go", args...)

	// Create an md5 checksum of the binary, to be included in the archive for
	// automatic upgrades.
	err := md5File(binary)
	if err != nil {
		log.Fatal(err)
	}
}

func ldflags() string {
	var b bytes.Buffer
	b.WriteString("-w")
	b.WriteString(fmt.Sprintf(" -X main.version '%s'", version))
	b.WriteString(fmt.Sprintf(" -X main.commit '%s'", getGitSha()))
	b.WriteString(fmt.Sprintf(" -X main.buildstamp %d", buildStamp()))
	return b.String()
}

func rmr(paths ...string) {
	for _, path := range paths {
		log.Println("rm -r", path)
		os.RemoveAll(path)
	}
}

func clean() {
	rmr("bin", "Godeps/_workspace/pkg", "Godeps/_workspace/bin")
	rmr("dist")
	rmr("tmp")
	rmr(filepath.Join(os.Getenv("GOPATH"), fmt.Sprintf("pkg/%s_%s/github.com/grafana", goos, goarch)))
}

func setBuildEnv() {
	os.Setenv("GOOS", goos)
	if strings.HasPrefix(goarch, "armv") {
		os.Setenv("GOARCH", "arm")
		os.Setenv("GOARM", goarch[4:])
	} else {
		os.Setenv("GOARCH", goarch)
	}
	if goarch == "386" {
		os.Setenv("GO386", "387")
	}
	wd, err := os.Getwd()
	if err != nil {
		log.Println("Warning: can't determine current dir:", err)
		log.Println("Build might not work as expected")
	}
	os.Setenv("GOPATH", fmt.Sprintf("%s%c%s", filepath.Join(wd, "Godeps", "_workspace"), os.PathListSeparator, os.Getenv("GOPATH")))
	log.Println("GOPATH=" + os.Getenv("GOPATH"))
}

func getGitSha() string {
	v, err := runError("git", "describe", "--always", "--dirty")
	if err != nil {
		return "unknown-dev"
	}
	v = versionRe.ReplaceAllFunc(v, func(s []byte) []byte {
		s[0] = '+'
		return s
	})
	return string(v)
}

func buildStamp() int64 {
	bs, err := runError("git", "show", "-s", "--format=%ct")
	if err != nil {
		return time.Now().Unix()
	}
	s, _ := strconv.ParseInt(string(bs), 10, 64)
	return s
}

func buildArch() string {
	os := goos
	if os == "darwin" {
		os = "macosx"
	}
	return fmt.Sprintf("%s-%s", os, goarch)
}

func run(cmd string, args ...string) []byte {
	bs, err := runError(cmd, args...)
	if err != nil {
		log.Println(cmd, strings.Join(args, " "))
		log.Println(string(bs))
		log.Fatal(err)
	}
	return bytes.TrimSpace(bs)
}

func runError(cmd string, args ...string) ([]byte, error) {
	ecmd := exec.Command(cmd, args...)
	bs, err := ecmd.CombinedOutput()
	if err != nil {
		return nil, err
	}

	return bytes.TrimSpace(bs), nil
}

func runPrint(cmd string, args ...string) {
	log.Println(cmd, strings.Join(args, " "))
	ecmd := exec.Command(cmd, args...)
	ecmd.Stdout = os.Stdout
	ecmd.Stderr = os.Stderr
	err := ecmd.Run()
	if err != nil {
		log.Fatal(err)
	}
}

func md5File(file string) error {
	fd, err := os.Open(file)
	if err != nil {
		return err
	}
	defer fd.Close()

	h := md5.New()
	_, err = io.Copy(h, fd)
	if err != nil {
		return err
	}

	out, err := os.Create(file + ".md5")
	if err != nil {
		return err
	}

	_, err = fmt.Fprintf(out, "%x\n", h.Sum(nil))
	if err != nil {
		return err
	}

	return out.Close()
}
<|endoftext|>


<commit_before>package urknall

import (
	"bytes"
	"fmt"
	"path/filepath"
	"strings"

	"github.com/dynport/urknall/cmd"
	"github.com/dynport/urknall/pubsub"
)

// A shortcut creating a build from the given target and template. The
// resulting build is run immediately and the return value returned.
func Run(target Target, tpl Template) (e error) {
	return (&Build{Target: target, Template: tpl}).Run()
}

// A build is the glue between a target and template. It contains the basic
// parameters required for actually doing something. The example below contains
// most of the basic functionality urknall provides.
type Build struct {
	Target            // Where to run the build.
	Template          // What to actually build.
	DryRun   bool     // Should the commands actually be executed.
	Env      []string // Environment variables in the form `KEY=VALUE`.
}

// This will render the build's template into a package and run all its tasks.
func (b *Build) Run() error {
	pkg, e := renderTemplate(b.Template)
	if e != nil {
		return e
	}
	return pkg.Build(b)
}

func (build *Build) prepare() error {
	if build.User() == "" {
		return fmt.Errorf("User not set")
	}
	cmd, e := build.prepareCommand(fmt.Sprintf(`{ grep "^%s:" /etc/group | grep %s; } && [ -d /var/lib/urknall ]`, ukGROUP, build.User()))
	if e != nil {
		return e
	}
	if e := cmd.Run(); e != nil {
		// If user is missing the group, create group (if necessary), add user and restart ssh connection.
		cmds := []string{
			fmt.Sprintf(`{ grep -e '^%[1]s:' /etc/group > /dev/null || { groupadd %[1]s; }; }`, ukGROUP),
			fmt.Sprintf(`{ [ -d %[1]s ] || { mkdir -p -m 2775 %[1]s && chgrp %[2]s %[1]s; }; }`, ukCACHEDIR, ukGROUP),
			fmt.Sprintf("usermod -a -G %s %s", ukGROUP, build.User()),
		}

		cmd, e = build.prepareCommand(strings.Join(cmds, " && "))
		if e != nil {
			return e
		}
		out := &bytes.Buffer{}
		err := &bytes.Buffer{}
		cmd.SetStderr(err)
		cmd.SetStdout(out)
		if e := cmd.Run(); e != nil {
			return fmt.Errorf("failed to initiate user %q for provisioning: %s, out=%q err=%q", build.User(), e, out.String(), err.String())
		}
		build.Reset()
	}
	return nil
}

func (build *Build) buildTask(tsk *task, ct checksumTree) (e error) {
	commands, e := tsk.Commands()
	if e != nil {
		return e
	}
	cacheKey := tsk.name
	if cacheKey == "" {
		return fmt.Errorf("CacheKey must not be empty")
	}
	checksumDir := fmt.Sprintf(ukCACHEDIR+"/%s", tsk.name)

	var found bool
	var checksumHash map[string]struct{}
	if checksumHash, found = ct[cacheKey]; !found {
		ct[cacheKey] = map[string]struct{}{}
		checksumHash = ct[cacheKey]

		// Create checksum dir and set group bit (all new files will inherit the directory's group). This allows for
		// different users (being part of that group) to create, modify and delete the contained checksum and log files.
		createChecksumDirCmd := fmt.Sprintf("mkdir -m2775 -p %s", checksumDir)

		cmd, e := build.prepareCommand(createChecksumDirCmd)
		if e != nil {
			return e
		}
		err := &bytes.Buffer{}

		cmd.SetStderr(err)

		if e := cmd.Run(); e != nil {
			return fmt.Errorf(err.String() + ": " + e.Error())
		}
	}

	for _, cmd := range commands {
		logMsg := cmd.Logging()
		checksum := commandChecksum(cmd)
		m := &pubsub.Message{Key: pubsub.MessageRunlistsProvisionTask, TaskChecksum: checksum, Message: logMsg, Hostname: build.hostname(), RunlistName: cacheKey}
		if _, found := checksumHash[checksum]; found { // Task is cached.
			m.ExecStatus = pubsub.StatusCached
			m.Publish("finished")
			delete(checksumHash, checksum) // Delete checksums of cached tasks from hash.
			continue
		}

		if len(checksumHash) > 0 { // All remaining checksums are invalid, as something changed.
			if e = build.cleanUpRemainingCachedEntries(checksumDir, checksumHash); e != nil {
				return e
			}
			checksumHash = make(map[string]struct{})
		}
		m.ExecStatus = pubsub.StatusExecStart
		if build.DryRun {
			m.Publish("executed")
		} else {
			m.Publish("started")
			e = executeCommand(cmd, build, checksumDir)
			m.Error = e
			m.ExecStatus = pubsub.StatusExecFinished
			m.Publish("finished")
		}
		if e != nil {
			return e
		}
	}

	return nil
}

type checksumTree map[string]map[string]struct{}

func (build *Build) buildChecksumTree() (ct checksumTree, e error) {
	ct = checksumTree{}

	cmd, e := build.prepareCommand(fmt.Sprintf(`[ -d %[1]s ] && find %[1]s -type f -name \*.done`, ukCACHEDIR))
	if e != nil {
		return nil, e
	}
	out := &bytes.Buffer{}
	err := &bytes.Buffer{}

	cmd.SetStdout(out)
	cmd.SetStderr(err)

	if e := cmd.Run(); e != nil {
		return nil, fmt.Errorf("%s: out=%s err=%s", e.Error(), out.String(), err.String())
	}
	for _, line := range strings.Split(out.String(), "\n") {
		line = strings.TrimSpace(line)
		if line == "" {
			continue
		}

		pkgname := filepath.Dir(strings.TrimPrefix(line, ukCACHEDIR+"/"))
		checksum := strings.TrimSuffix(filepath.Base(line), ".done")
		if len(checksum) != 64 {
			return nil, fmt.Errorf("invalid checksum %q found for package %q", checksum, pkgname)
		}
		if _, found := ct[pkgname]; !found {
			ct[pkgname] = map[string]struct{}{}
		}
		ct[pkgname][checksum] = struct{}{}
	}

	return ct, nil
}

func (build *Build) cleanUpRemainingCachedEntries(checksumDir string, checksumHash map[string]struct{}) (e error) {
	invalidCacheEntries := make([]string, 0, len(checksumHash))
	for k, _ := range checksumHash {
		invalidCacheEntries = append(invalidCacheEntries, fmt.Sprintf("%s.done", k))
	}
	if build.DryRun {
		(&pubsub.Message{Key: pubsub.MessageCleanupCacheEntries, InvalidatedCacheEntries: invalidCacheEntries, Hostname: build.hostname()}).Publish(".dryrun")
	} else {
		cmd := fmt.Sprintf("cd %s && rm -f *.failed %s", checksumDir, strings.Join(invalidCacheEntries, " "))
		m := &pubsub.Message{Key: pubsub.MessageUrknallInternal, Hostname: build.hostname()}
		m.Publish("started")

		c, e := build.prepareCommand(cmd)
		if e != nil {
			return e
		}
		if e := c.Run(); e != nil {
			return e
		}
		//m.sshResult = result
		m.Publish("finished")
	}
	return nil
}

func (build *Build) prepareCommand(cmd string) (cmd.ExecCommand, error) {
	var sudo string
	if build.User() != "root" {
		sudo = "sudo "
	}
	cmd = fmt.Sprintf(sudo+"sh -x -e <<EOC\n%s\nEOC\n", cmd)
	return build.Command(cmd)
}

func (build *Build) hostname() string {
	if s, ok := build.Target.(fmt.Stringer); ok {
		return s.String()
	}
	return "MISSING"
}
<commit_msg>added dryrun short-cut function<commit_after>package urknall

import (
	"bytes"
	"fmt"
	"path/filepath"
	"strings"

	"github.com/dynport/urknall/cmd"
	"github.com/dynport/urknall/pubsub"
)

// A shortcut creating a build from the given target and template. The
// resulting build is run immediately and the return value returned.
func Run(target Target, tpl Template) (e error) {
	return (&Build{Target: target, Template: tpl}).Run()
}

// A shortcut creating a build from the given target and template with the
// DryRun flag set to true. The resulting build is run immediately and the
// return value returned. This is quite helpful to actually see which commands
// would be exeucted in the current setting, without actually doing anything.
func DryRun(target Target, tpl Template) (e error) {
	return (&Build{Target: target, Template: tpl, DryRun: true}).Run()
}

// A build is the glue between a target and template. It contains the basic
// parameters required for actually doing something. The example below contains
// most of the basic functionality urknall provides.
type Build struct {
	Target            // Where to run the build.
	Template          // What to actually build.
	DryRun   bool     // Should the commands actually be executed.
	Env      []string // Environment variables in the form `KEY=VALUE`.
}

// This will render the build's template into a package and run all its tasks.
func (b *Build) Run() error {
	pkg, e := renderTemplate(b.Template)
	if e != nil {
		return e
	}
	return pkg.Build(b)
}

func (build *Build) prepare() error {
	if build.User() == "" {
		return fmt.Errorf("User not set")
	}
	cmd, e := build.prepareCommand(fmt.Sprintf(`{ grep "^%s:" /etc/group | grep %s; } && [ -d /var/lib/urknall ]`, ukGROUP, build.User()))
	if e != nil {
		return e
	}
	if e := cmd.Run(); e != nil {
		// If user is missing the group, create group (if necessary), add user and restart ssh connection.
		cmds := []string{
			fmt.Sprintf(`{ grep -e '^%[1]s:' /etc/group > /dev/null || { groupadd %[1]s; }; }`, ukGROUP),
			fmt.Sprintf(`{ [ -d %[1]s ] || { mkdir -p -m 2775 %[1]s && chgrp %[2]s %[1]s; }; }`, ukCACHEDIR, ukGROUP),
			fmt.Sprintf("usermod -a -G %s %s", ukGROUP, build.User()),
		}

		cmd, e = build.prepareCommand(strings.Join(cmds, " && "))
		if e != nil {
			return e
		}
		out := &bytes.Buffer{}
		err := &bytes.Buffer{}
		cmd.SetStderr(err)
		cmd.SetStdout(out)
		if e := cmd.Run(); e != nil {
			return fmt.Errorf("failed to initiate user %q for provisioning: %s, out=%q err=%q", build.User(), e, out.String(), err.String())
		}
		build.Reset()
	}
	return nil
}

func (build *Build) buildTask(tsk *task, ct checksumTree) (e error) {
	commands, e := tsk.Commands()
	if e != nil {
		return e
	}
	cacheKey := tsk.name
	if cacheKey == "" {
		return fmt.Errorf("CacheKey must not be empty")
	}
	checksumDir := fmt.Sprintf(ukCACHEDIR+"/%s", tsk.name)

	var found bool
	var checksumHash map[string]struct{}
	if checksumHash, found = ct[cacheKey]; !found {
		ct[cacheKey] = map[string]struct{}{}
		checksumHash = ct[cacheKey]

		// Create checksum dir and set group bit (all new files will inherit the directory's group). This allows for
		// different users (being part of that group) to create, modify and delete the contained checksum and log files.
		createChecksumDirCmd := fmt.Sprintf("mkdir -m2775 -p %s", checksumDir)

		cmd, e := build.prepareCommand(createChecksumDirCmd)
		if e != nil {
			return e
		}
		err := &bytes.Buffer{}

		cmd.SetStderr(err)

		if e := cmd.Run(); e != nil {
			return fmt.Errorf(err.String() + ": " + e.Error())
		}
	}

	for _, cmd := range commands {
		logMsg := cmd.Logging()
		checksum := commandChecksum(cmd)
		m := &pubsub.Message{Key: pubsub.MessageRunlistsProvisionTask, TaskChecksum: checksum, Message: logMsg, Hostname: build.hostname(), RunlistName: cacheKey}
		if _, found := checksumHash[checksum]; found { // Task is cached.
			m.ExecStatus = pubsub.StatusCached
			m.Publish("finished")
			delete(checksumHash, checksum) // Delete checksums of cached tasks from hash.
			continue
		}

		if len(checksumHash) > 0 { // All remaining checksums are invalid, as something changed.
			if e = build.cleanUpRemainingCachedEntries(checksumDir, checksumHash); e != nil {
				return e
			}
			checksumHash = make(map[string]struct{})
		}
		m.ExecStatus = pubsub.StatusExecStart
		if build.DryRun {
			m.Publish("executed")
		} else {
			m.Publish("started")
			e = executeCommand(cmd, build, checksumDir)
			m.Error = e
			m.ExecStatus = pubsub.StatusExecFinished
			m.Publish("finished")
		}
		if e != nil {
			return e
		}
	}

	return nil
}

type checksumTree map[string]map[string]struct{}

func (build *Build) buildChecksumTree() (ct checksumTree, e error) {
	ct = checksumTree{}

	cmd, e := build.prepareCommand(fmt.Sprintf(`[ -d %[1]s ] && find %[1]s -type f -name \*.done`, ukCACHEDIR))
	if e != nil {
		return nil, e
	}
	out := &bytes.Buffer{}
	err := &bytes.Buffer{}

	cmd.SetStdout(out)
	cmd.SetStderr(err)

	if e := cmd.Run(); e != nil {
		return nil, fmt.Errorf("%s: out=%s err=%s", e.Error(), out.String(), err.String())
	}
	for _, line := range strings.Split(out.String(), "\n") {
		line = strings.TrimSpace(line)
		if line == "" {
			continue
		}

		pkgname := filepath.Dir(strings.TrimPrefix(line, ukCACHEDIR+"/"))
		checksum := strings.TrimSuffix(filepath.Base(line), ".done")
		if len(checksum) != 64 {
			return nil, fmt.Errorf("invalid checksum %q found for package %q", checksum, pkgname)
		}
		if _, found := ct[pkgname]; !found {
			ct[pkgname] = map[string]struct{}{}
		}
		ct[pkgname][checksum] = struct{}{}
	}

	return ct, nil
}

func (build *Build) cleanUpRemainingCachedEntries(checksumDir string, checksumHash map[string]struct{}) (e error) {
	invalidCacheEntries := make([]string, 0, len(checksumHash))
	for k, _ := range checksumHash {
		invalidCacheEntries = append(invalidCacheEntries, fmt.Sprintf("%s.done", k))
	}
	if build.DryRun {
		(&pubsub.Message{Key: pubsub.MessageCleanupCacheEntries, InvalidatedCacheEntries: invalidCacheEntries, Hostname: build.hostname()}).Publish(".dryrun")
	} else {
		cmd := fmt.Sprintf("cd %s && rm -f *.failed %s", checksumDir, strings.Join(invalidCacheEntries, " "))
		m := &pubsub.Message{Key: pubsub.MessageUrknallInternal, Hostname: build.hostname()}
		m.Publish("started")

		c, e := build.prepareCommand(cmd)
		if e != nil {
			return e
		}
		if e := c.Run(); e != nil {
			return e
		}
		//m.sshResult = result
		m.Publish("finished")
	}
	return nil
}

func (build *Build) prepareCommand(cmd string) (cmd.ExecCommand, error) {
	var sudo string
	if build.User() != "root" {
		sudo = "sudo "
	}
	cmd = fmt.Sprintf(sudo+"sh -x -e <<EOC\n%s\nEOC\n", cmd)
	return build.Command(cmd)
}

func (build *Build) hostname() string {
	if s, ok := build.Target.(fmt.Stringer); ok {
		return s.String()
	}
	return "MISSING"
}
<|endoftext|>


<commit_before>package dsc

import (
	"database/sql"
	"fmt"
	"reflect"
)

func asSQLDb(wrapped interface{}) (*sql.DB, error) {
	if result, ok := wrapped.(*sql.DB); ok {
		return result, nil
	}
	wrappedType := reflect.ValueOf(wrapped)
	return nil, fmt.Errorf(fmt.Sprintf("failed cast as sql.DB: was %v !", wrappedType.Type()))
}

func asSQLTx(wrapped interface{}) (*sql.Tx, error) {
	if wrapped == nil {
		return nil, nil
	}
	if result, ok := wrapped.(*sql.Tx); ok {
		return result, nil
	}
	wrappedType := reflect.ValueOf(wrapped)
	return nil, fmt.Errorf(fmt.Sprintf("failed cast as sql.Tx: was %v !", wrappedType.Type()))
}

func asScanner(wrapped interface{}) (Scanner, error) {
	if result, ok := reflect.ValueOf(wrapped).Interface().(Scanner); ok {
		return result, nil
	}
	wrappedType := reflect.ValueOf(wrapped)
	return nil, fmt.Errorf(fmt.Sprintf("failed cast as Scannable: was %v %v !", wrappedType.Type(), reflect.ValueOf(wrapped).Elem()))
}

type sqlExecutor interface {
	Exec(sql string, parameters ...interface{}) (sql.Result, error)
}

type sqlManager struct {
	*AbstractManager
}

func (m *sqlManager) initConnectionIfNeeded(connection Connection) error {
	if sqlConnection, ok := connection.(*sqlConnection); ok {
		if sqlConnection.init {
			return nil
		}
		sqlConnection.init = true
		dialect := GetDatastoreDialect(m.config.DriverName)
		return dialect.Init(m, connection)
	}
	return nil
}

func (m *sqlManager) ExecuteOnConnection(connection Connection, sql string, args []interface{}) (sql.Result, error) {
	db, err := asSQLDb(connection.Unwrap(sqlDbPointer))
	if err == nil {
		err = m.initConnectionIfNeeded(connection)
	}
	if err != nil {
		return nil, err
	}

	var executable sqlExecutor = db
	tx, err := asSQLTx(connection.Unwrap(sqlTxtPointer))
	if err != nil {
		return nil, err
	}
	if tx != nil {
		executable = tx
	}
	if args == nil {
		args = make([]interface{}, 0)
	}

	dialect := GetDatastoreDialect(m.config.DriverName)
	sql = dialect.NormalizeSQL(sql)
	if sqlConnection, ok := connection.(*sqlConnection); ok {
		if !sqlConnection.init {

		}
	}

	result, err := executable.Exec(sql, args...)
	if !dialect.CanHandleTransaction() {
		result = NewSQLResult(1, 0)
	}
	Logf("[%v]:%v %v", m.config.username, sql, args)
	if err != nil {
		return nil, fmt.Errorf(fmt.Sprintf("failed to execute %v %v on %v due to:\n%v", sql, args, m.Manager.Config().Parameters, err.Error()))
	}
	return result, err
}

func (m *sqlManager) ReadAllOnWithHandlerOnConnection(connection Connection, query string, args []interface{}, readingHandler func(scanner Scanner) (toContinue bool, err error)) error {
	db, err := asSQLDb(connection.Unwrap((*sql.DB)(nil)))
	if err == nil {
		err = m.initConnectionIfNeeded(connection)
	}
	if err != nil {
		return err
	}

	dialect := GetDatastoreDialect(m.config.DriverName)
	query = dialect.NormalizeSQL(query)

	Logf("[%v]:%v", m.config.username, query)
	sqlStatement, sqlError := db.Prepare(query)
	if sqlError != nil {
		return fmt.Errorf("failed to prepare sql: %v with %v due to:%v\n\t", query, args, sqlError.Error())
	}
	defer sqlStatement.Close()
	rows, queryError := m.executeQuery(sqlStatement, query, args)
	if queryError != nil {
		return fmt.Errorf(fmt.Sprintf("failed to execute sql: %v with %v due to:%v\n\t", query, args, queryError.Error()))
	}

	defer rows.Close()
	for rows.Next() {
		scanner, _ := asScanner(rows)
		toContinue, err := readingHandler(NewScanner(scanner))
		if err != nil {
			return err
		}
		if !toContinue {
			break
		}
	}
	return rows.Err()
}

func (m *sqlManager) executeQuery(sqlStatement *sql.Stmt, query string, args []interface{}) (rows *sql.Rows, err error) {
	if args == nil {
		args = make([]interface{}, 0)
	}
	rows, err = sqlStatement.Query(args...)
	if err != nil {
		return nil, err
	}
	return rows, nil
}
<commit_msg>removed unused code<commit_after>package dsc

import (
	"database/sql"
	"fmt"
	"reflect"
)

func asSQLDb(wrapped interface{}) (*sql.DB, error) {
	if result, ok := wrapped.(*sql.DB); ok {
		return result, nil
	}
	wrappedType := reflect.ValueOf(wrapped)
	return nil, fmt.Errorf(fmt.Sprintf("failed cast as sql.DB: was %v !", wrappedType.Type()))
}

func asSQLTx(wrapped interface{}) (*sql.Tx, error) {
	if wrapped == nil {
		return nil, nil
	}
	if result, ok := wrapped.(*sql.Tx); ok {
		return result, nil
	}
	wrappedType := reflect.ValueOf(wrapped)
	return nil, fmt.Errorf(fmt.Sprintf("failed cast as sql.Tx: was %v !", wrappedType.Type()))
}

func asScanner(wrapped interface{}) (Scanner, error) {
	if result, ok := reflect.ValueOf(wrapped).Interface().(Scanner); ok {
		return result, nil
	}
	wrappedType := reflect.ValueOf(wrapped)
	return nil, fmt.Errorf(fmt.Sprintf("failed cast as Scannable: was %v %v !", wrappedType.Type(), reflect.ValueOf(wrapped).Elem()))
}

type sqlExecutor interface {
	Exec(sql string, parameters ...interface{}) (sql.Result, error)
}

type sqlManager struct {
	*AbstractManager
}

func (m *sqlManager) initConnectionIfNeeded(connection Connection) error {
	if sqlConnection, ok := connection.(*sqlConnection); ok {
		if sqlConnection.init {
			return nil
		}
		sqlConnection.init = true
		dialect := GetDatastoreDialect(m.config.DriverName)
		return dialect.Init(m, connection)
	}
	return nil
}

func (m *sqlManager) ExecuteOnConnection(connection Connection, sql string, args []interface{}) (sql.Result, error) {
	db, err := asSQLDb(connection.Unwrap(sqlDbPointer))
	if err == nil {
		err = m.initConnectionIfNeeded(connection)
	}
	if err != nil {
		return nil, err
	}

	var executable sqlExecutor = db
	tx, err := asSQLTx(connection.Unwrap(sqlTxtPointer))
	if err != nil {
		return nil, err
	}
	if tx != nil {
		executable = tx
	}
	if args == nil {
		args = make([]interface{}, 0)
	}

	dialect := GetDatastoreDialect(m.config.DriverName)
	sql = dialect.NormalizeSQL(sql)
	result, err := executable.Exec(sql, args...)
	if !dialect.CanHandleTransaction() {
		result = NewSQLResult(1, 0)
	}
	Logf("[%v]:%v %v", m.config.username, sql, args)
	if err != nil {
		return nil, fmt.Errorf(fmt.Sprintf("failed to execute %v %v on %v due to:\n%v", sql, args, m.Manager.Config().Parameters, err.Error()))
	}
	return result, err
}

func (m *sqlManager) ReadAllOnWithHandlerOnConnection(connection Connection, query string, args []interface{}, readingHandler func(scanner Scanner) (toContinue bool, err error)) error {
	db, err := asSQLDb(connection.Unwrap((*sql.DB)(nil)))
	if err == nil {
		err = m.initConnectionIfNeeded(connection)
	}
	if err != nil {
		return err
	}

	dialect := GetDatastoreDialect(m.config.DriverName)
	query = dialect.NormalizeSQL(query)
	Logf("[%v]:%v", m.config.username, query)
	sqlStatement, sqlError := db.Prepare(query)
	if sqlError != nil {
		return fmt.Errorf("failed to prepare sql: %v with %v due to:%v\n\t", query, args, sqlError.Error())
	}
	defer sqlStatement.Close()
	rows, queryError := m.executeQuery(sqlStatement, query, args)
	if queryError != nil {
		return fmt.Errorf(fmt.Sprintf("failed to execute sql: %v with %v due to:%v\n\t", query, args, queryError.Error()))
	}

	defer rows.Close()
	for rows.Next() {
		scanner, _ := asScanner(rows)
		toContinue, err := readingHandler(NewScanner(scanner))
		if err != nil {
			return err
		}
		if !toContinue {
			break
		}
	}
	return rows.Err()
}

func (m *sqlManager) executeQuery(sqlStatement *sql.Stmt, query string, args []interface{}) (rows *sql.Rows, err error) {
	if args == nil {
		args = make([]interface{}, 0)
	}
	rows, err = sqlStatement.Query(args...)
	if err != nil {
		return nil, err
	}
	return rows, nil
}
<|endoftext|>


<commit_before>package levant

import (
	"bytes"
	"fmt"
	"io/ioutil"
	"path"
	"text/template"

	"github.com/jrasell/levant/helper"
	"github.com/jrasell/levant/logging"
	yaml "gopkg.in/yaml.v2"

	nomad "github.com/hashicorp/nomad/api"
	"github.com/hashicorp/nomad/jobspec"
	"github.com/hashicorp/terraform/config"
)

const (
	terraformVarExtension = ".tf"
	yamlVarExtension      = ".yaml"
	ymlVarExtension       = ".yml"
)

// RenderJob takes in a template and variables performing a render of the
// template followed by Nomad jobspec parse.
func RenderJob(templateFile, variableFile string, flagVars *map[string]string) (job *nomad.Job, err error) {
	var tpl *bytes.Buffer
	tpl, err = RenderTemplate(templateFile, variableFile, flagVars)
	if err != nil {
		return
	}

	job, err = jobspec.Parse(tpl)
	return
}

// RenderTemplate is the main entry point to render the template based on the
// passed variables file.
func RenderTemplate(templateFile, variableFile string, flagVars *map[string]string) (tpl *bytes.Buffer, err error) {
	if variableFile == "" {
		variableFile = helper.GetDefaultVarFile()
	}

	// Process the variable file extension and log DEBUG so the template can be
	// correctly rendered.
	ext := path.Ext(variableFile)
	if variableFile != "" {
		logging.Debug("levant/templater: variable file extension %s detected", ext)
	}

	src, err := ioutil.ReadFile(templateFile)
	if err != nil {
		return
	}

	// If no command line variables are passed; log this as DEBUG to provide much
	// greater feedback.
	if len(*flagVars) == 0 {
		logging.Debug("levant/templater: no command line variables passed")
	}

	switch ext {
	case terraformVarExtension:
		logging.Debug("levant/templater: detected .tf variable file extension")
		tpl, err = renderTFTemplte(string(src), variableFile, flagVars)

	case yamlVarExtension, ymlVarExtension:
		// Run the render using a YAML varaible file.
		tpl, err = renderYAMLVarsTemplate(string(src), variableFile, flagVars)

	case "":
		// No varibles file passed; render using any passed CLI variables.
		logging.Debug("levant/templater: variable file not passed")
		tpl, err = readJobFile(string(src), flagVars)

	default:
		err = fmt.Errorf("variables file extension %v not supported", ext)
	}

	return
}

func renderTFTemplte(src, variableFile string, flagVars *map[string]string) (tpl *bytes.Buffer, err error) {

	c := &config.Config{}

	// Setup our variables map and template
	variables := make(map[string]interface{})
	tpl = &bytes.Buffer{}

	// Load the variables file
	if c, err = config.LoadFile(variableFile); err != nil {
		return
	}

	// Populate our map with the parsed variables
	for _, variable := range c.Variables {
		variables[variable.Name] = variable.Default
	}

	// Merge variables passed on the CLI with those passed through a variables
	// file.
	mergedVars := helper.VariableMerge(&variables, flagVars)

	// Setup the template file for rendering
	t := newTemplate()

	if t, err = t.Parse(src); err != nil {
		return
	}

	if err = t.Execute(tpl, mergedVars); err != nil {
		return
	}
	return tpl, nil
}

func renderYAMLVarsTemplate(src, variableFile string, flagVars *map[string]string) (tpl *bytes.Buffer, err error) {

	// Setup our variables map and template
	variables := make(map[string]interface{})
	tpl = &bytes.Buffer{}

	yamlFile, err := ioutil.ReadFile(variableFile)
	if err != nil {
		return
	}

	if err = yaml.Unmarshal(yamlFile, &variables); err != nil {
		return
	}

	// Merge variables passed on the CLI with those passed through a variables
	// file.
	mergedVars := helper.VariableMerge(&variables, flagVars)

	// Setup the template file for rendering
	t := newTemplate()
	if t, err = t.Parse(string(src)); err != nil {
		return
	}

	if err = t.Execute(tpl, mergedVars); err != nil {
		return
	}

	return tpl, nil
}

func readJobFile(src string, flagVars *map[string]string) (tpl *bytes.Buffer, err error) {

	tpl = &bytes.Buffer{}

	// Setup the template file for rendering
	t := newTemplate()
	if t, err = t.Parse(src); err != nil {
		return
	}

	if err = t.Execute(tpl, flagVars); err != nil {
		return
	}

	return tpl, nil
}

// newTemplate returns an empty template with default options set
func newTemplate() *template.Template {
	return template.New("jobTemplate").Delims("[[", "]]").Option("missingkey=error")
}
<commit_msg>Rebase branch<commit_after>package levant

import (
	"bytes"
	"fmt"
	"io/ioutil"
	"path"
	"text/template"

	"github.com/jrasell/levant/helper"
	"github.com/jrasell/levant/logging"
	yaml "gopkg.in/yaml.v2"

	nomad "github.com/hashicorp/nomad/api"
	"github.com/hashicorp/nomad/jobspec"
	"github.com/hashicorp/terraform/config"
)

const (
	terraformVarExtension = ".tf"
	yamlVarExtension      = ".yaml"
	ymlVarExtension       = ".yml"
)

// RenderJob takes in a template and variables performing a render of the
// template followed by Nomad jobspec parse.
func RenderJob(templateFile, variableFile string, flagVars *map[string]string) (job *nomad.Job, err error) {
	var tpl *bytes.Buffer
	tpl, err = RenderTemplate(templateFile, variableFile, flagVars)
	if err != nil {
		return
	}

	job, err = jobspec.Parse(tpl)
	return
}

// RenderTemplate is the main entry point to render the template based on the
// passed variables file.
func RenderTemplate(templateFile, variableFile string, flagVars *map[string]string) (tpl *bytes.Buffer, err error) {
	if variableFile == "" {
		logging.Debug("levant/templater: no variable file passed, trying defaults")
		if variableFile = helper.GetDefaultVarFile(); variableFile != "" {
			logging.Debug("levant/templater: found default variable file, using %s", variableFile)
		}
	}

	// Process the variable file extension and log DEBUG so the template can be
	// correctly rendered.
	var ext string
	if ext = path.Ext(variableFile); ext != "" {
		logging.Debug("levant/templater: variable file extension %s detected", ext)
	}

	src, err := ioutil.ReadFile(templateFile)
	if err != nil {
		return
	}

	// If no command line variables are passed; log this as DEBUG to provide much
	// greater feedback.
	if len(*flagVars) == 0 {
		logging.Debug("levant/templater: no command line variables passed")
	}

	switch ext {
	case terraformVarExtension:
		tpl, err = renderTFTemplte(string(src), variableFile, flagVars)

	case yamlVarExtension, ymlVarExtension:
		// Run the render using a YAML variable file.
		tpl, err = renderYAMLVarsTemplate(string(src), variableFile, flagVars)

	case "":
		// No variables file passed; render using any passed CLI variables.
		logging.Debug("levant/templater: variable file not passed")
		tpl, err = readJobFile(string(src), flagVars)

	default:
		err = fmt.Errorf("variables file extension %v not supported", ext)
	}

	return
}

func renderTFTemplte(src, variableFile string, flagVars *map[string]string) (tpl *bytes.Buffer, err error) {

	c := &config.Config{}

	// Setup our variables map and template
	variables := make(map[string]interface{})
	tpl = &bytes.Buffer{}

	// Load the variables file
	if c, err = config.LoadFile(variableFile); err != nil {
		return
	}

	// Populate our map with the parsed variables
	for _, variable := range c.Variables {
		variables[variable.Name] = variable.Default
	}

	// Merge variables passed on the CLI with those passed through a variables
	// file.
	mergedVars := helper.VariableMerge(&variables, flagVars)

	// Setup the template file for rendering
	t := newTemplate()

	if t, err = t.Parse(src); err != nil {
		return
	}

	if err = t.Execute(tpl, mergedVars); err != nil {
		return
	}
	return tpl, nil
}

func renderYAMLVarsTemplate(src, variableFile string, flagVars *map[string]string) (tpl *bytes.Buffer, err error) {

	// Setup our variables map and template
	variables := make(map[string]interface{})
	tpl = &bytes.Buffer{}

	yamlFile, err := ioutil.ReadFile(variableFile)
	if err != nil {
		return
	}

	if err = yaml.Unmarshal(yamlFile, &variables); err != nil {
		return
	}

	// Merge variables passed on the CLI with those passed through a variables
	// file.
	mergedVars := helper.VariableMerge(&variables, flagVars)

	// Setup the template file for rendering
	t := newTemplate()
	if t, err = t.Parse(string(src)); err != nil {
		return
	}

	if err = t.Execute(tpl, mergedVars); err != nil {
		return
	}

	return tpl, nil
}

func readJobFile(src string, flagVars *map[string]string) (tpl *bytes.Buffer, err error) {

	tpl = &bytes.Buffer{}

	// Setup the template file for rendering
	t := newTemplate()
	if t, err = t.Parse(src); err != nil {
		return
	}

	if err = t.Execute(tpl, flagVars); err != nil {
		return
	}

	return tpl, nil
}

// newTemplate returns an empty template with default options set
func newTemplate() *template.Template {
	return template.New("jobTemplate").Delims("[[", "]]").Option("missingkey=error")
}
<|endoftext|>


<commit_before>package command

import (
	"net/http"
	"strconv"
	"strings"
	"time"

	"google.golang.org/grpc/reflection"

	"github.com/chrislusf/seaweedfs/weed/glog"
	"github.com/chrislusf/seaweedfs/weed/pb"
	"github.com/chrislusf/seaweedfs/weed/pb/filer_pb"
	"github.com/chrislusf/seaweedfs/weed/security"
	"github.com/chrislusf/seaweedfs/weed/server"
	"github.com/chrislusf/seaweedfs/weed/util"
)

var (
	f FilerOptions
)

type FilerOptions struct {
	masters                 *string
	ip                      *string
	bindIp                  *string
	port                    *int
	publicPort              *int
	collection              *string
	defaultReplicaPlacement *string
	disableDirListing       *bool
	maxMB                   *int
	dirListingLimit         *int
	dataCenter              *string
	enableNotification      *bool
	disableHttp             *bool
	cipher                  *bool
	peers                   *string

	// default leveldb directory, used in "weed server" mode
	defaultLevelDbDirectory *string
}

func init() {
	cmdFiler.Run = runFiler // break init cycle
	f.masters = cmdFiler.Flag.String("master", "localhost:9333", "comma-separated master servers")
	f.collection = cmdFiler.Flag.String("collection", "", "all data will be stored in this collection")
	f.ip = cmdFiler.Flag.String("ip", util.DetectedHostAddress(), "filer server http listen ip address")
	f.bindIp = cmdFiler.Flag.String("ip.bind", "0.0.0.0", "ip address to bind to")
	f.port = cmdFiler.Flag.Int("port", 8888, "filer server http listen port")
	f.publicPort = cmdFiler.Flag.Int("port.readonly", 0, "readonly port opened to public")
	f.defaultReplicaPlacement = cmdFiler.Flag.String("defaultReplicaPlacement", "000", "default replication type if not specified")
	f.disableDirListing = cmdFiler.Flag.Bool("disableDirListing", false, "turn off directory listing")
	f.maxMB = cmdFiler.Flag.Int("maxMB", 32, "split files larger than the limit")
	f.dirListingLimit = cmdFiler.Flag.Int("dirListLimit", 100000, "limit sub dir listing size")
	f.dataCenter = cmdFiler.Flag.String("dataCenter", "", "prefer to write to volumes in this data center")
	f.disableHttp = cmdFiler.Flag.Bool("disableHttp", false, "disable http request, only gRpc operations are allowed")
	f.cipher = cmdFiler.Flag.Bool("encryptVolumeData", false, "encrypt data on volume servers")
	f.peers = cmdFiler.Flag.String("peers", "", "all filers sharing the same filer store in comma separated ip:port list")
}

var cmdFiler = &Command{
	UsageLine: "filer -port=8888 -master=<ip:port>[,<ip:port>]*",
	Short:     "start a file server that points to a master server, or a list of master servers",
	Long: `start a file server which accepts REST operation for any files.

	//create or overwrite the file, the directories /path/to will be automatically created
	POST /path/to/file
	//get the file content
	GET /path/to/file
	//create or overwrite the file, the filename in the multipart request will be used
	POST /path/to/
	//return a json format subdirectory and files listing
	GET /path/to/

	The configuration file "filer.toml" is read from ".", "$HOME/.seaweedfs/", or "/etc/seaweedfs/", in that order.

	The example filer.toml configuration file can be generated by "weed scaffold -config=filer"

`,
}

func runFiler(cmd *Command, args []string) bool {

	util.LoadConfiguration("security", false)

	f.startFiler()

	return true
}

func (fo *FilerOptions) startFiler() {

	defaultMux := http.NewServeMux()
	publicVolumeMux := defaultMux

	if *fo.publicPort != 0 {
		publicVolumeMux = http.NewServeMux()
	}

	defaultLevelDbDirectory := "./filerldb2"
	if fo.defaultLevelDbDirectory != nil {
		defaultLevelDbDirectory = util.ResolvePath(*fo.defaultLevelDbDirectory + "/filerldb2")
	}

	var peers []string
	if *fo.peers != "" {
		peers = strings.Split(*fo.peers, ",")
	}

	fs, nfs_err := weed_server.NewFilerServer(defaultMux, publicVolumeMux, &weed_server.FilerOption{
		Masters:            strings.Split(*fo.masters, ","),
		Collection:         *fo.collection,
		DefaultReplication: *fo.defaultReplicaPlacement,
		DisableDirListing:  *fo.disableDirListing,
		MaxMB:              *fo.maxMB,
		DirListingLimit:    *fo.dirListingLimit,
		DataCenter:         *fo.dataCenter,
		DefaultLevelDbDir:  defaultLevelDbDirectory,
		DisableHttp:        *fo.disableHttp,
		Host:               *fo.ip,
		Port:               uint32(*fo.port),
		Cipher:             *fo.cipher,
		Filers:             peers,
	})
	if nfs_err != nil {
		glog.Fatalf("Filer startup error: %v", nfs_err)
	}

	if *fo.publicPort != 0 {
		publicListeningAddress := *fo.bindIp + ":" + strconv.Itoa(*fo.publicPort)
		glog.V(0).Infoln("Start Seaweed filer server", util.Version(), "public at", publicListeningAddress)
		publicListener, e := util.NewListener(publicListeningAddress, 0)
		if e != nil {
			glog.Fatalf("Filer server public listener error on port %d:%v", *fo.publicPort, e)
		}
		go func() {
			if e := http.Serve(publicListener, publicVolumeMux); e != nil {
				glog.Fatalf("Volume server fail to serve public: %v", e)
			}
		}()
	}

	glog.V(0).Infof("Start Seaweed Filer %s at %s:%d", util.Version(), *fo.ip, *fo.port)
	filerListener, e := util.NewListener(
		*fo.bindIp+":"+strconv.Itoa(*fo.port),
		time.Duration(10)*time.Second,
	)
	if e != nil {
		glog.Fatalf("Filer listener error: %v", e)
	}

	// starting grpc server
	grpcPort := *fo.port + 10000
	grpcL, err := util.NewListener(":"+strconv.Itoa(grpcPort), 0)
	if err != nil {
		glog.Fatalf("failed to listen on grpc port %d: %v", grpcPort, err)
	}
	grpcS := pb.NewGrpcServer(security.LoadServerTLS(util.GetViper(), "grpc.filer"))
	filer_pb.RegisterSeaweedFilerServer(grpcS, fs)
	reflection.Register(grpcS)
	go grpcS.Serve(grpcL)

	httpS := &http.Server{Handler: defaultMux}
	if err := httpS.Serve(filerListener); err != nil {
		glog.Fatalf("Filer Fail to serve: %v", e)
	}

}
<commit_msg>Filer: ip bind<commit_after>package command

import (
	"net/http"
	"strconv"
	"strings"
	"time"

	"google.golang.org/grpc/reflection"

	"github.com/chrislusf/seaweedfs/weed/glog"
	"github.com/chrislusf/seaweedfs/weed/pb"
	"github.com/chrislusf/seaweedfs/weed/pb/filer_pb"
	"github.com/chrislusf/seaweedfs/weed/security"
	"github.com/chrislusf/seaweedfs/weed/server"
	"github.com/chrislusf/seaweedfs/weed/util"
)

var (
	f FilerOptions
)

type FilerOptions struct {
	masters                 *string
	ip                      *string
	bindIp                  *string
	port                    *int
	publicPort              *int
	collection              *string
	defaultReplicaPlacement *string
	disableDirListing       *bool
	maxMB                   *int
	dirListingLimit         *int
	dataCenter              *string
	enableNotification      *bool
	disableHttp             *bool
	cipher                  *bool
	peers                   *string

	// default leveldb directory, used in "weed server" mode
	defaultLevelDbDirectory *string
}

func init() {
	cmdFiler.Run = runFiler // break init cycle
	f.masters = cmdFiler.Flag.String("master", "localhost:9333", "comma-separated master servers")
	f.collection = cmdFiler.Flag.String("collection", "", "all data will be stored in this collection")
	f.ip = cmdFiler.Flag.String("ip", util.DetectedHostAddress(), "filer server http listen ip address")
	f.bindIp = cmdFiler.Flag.String("ip.bind", "0.0.0.0", "ip address to bind to")
	f.port = cmdFiler.Flag.Int("port", 8888, "filer server http listen port")
	f.publicPort = cmdFiler.Flag.Int("port.readonly", 0, "readonly port opened to public")
	f.defaultReplicaPlacement = cmdFiler.Flag.String("defaultReplicaPlacement", "000", "default replication type if not specified")
	f.disableDirListing = cmdFiler.Flag.Bool("disableDirListing", false, "turn off directory listing")
	f.maxMB = cmdFiler.Flag.Int("maxMB", 32, "split files larger than the limit")
	f.dirListingLimit = cmdFiler.Flag.Int("dirListLimit", 100000, "limit sub dir listing size")
	f.dataCenter = cmdFiler.Flag.String("dataCenter", "", "prefer to write to volumes in this data center")
	f.disableHttp = cmdFiler.Flag.Bool("disableHttp", false, "disable http request, only gRpc operations are allowed")
	f.cipher = cmdFiler.Flag.Bool("encryptVolumeData", false, "encrypt data on volume servers")
	f.peers = cmdFiler.Flag.String("peers", "", "all filers sharing the same filer store in comma separated ip:port list")
}

var cmdFiler = &Command{
	UsageLine: "filer -port=8888 -master=<ip:port>[,<ip:port>]*",
	Short:     "start a file server that points to a master server, or a list of master servers",
	Long: `start a file server which accepts REST operation for any files.

	//create or overwrite the file, the directories /path/to will be automatically created
	POST /path/to/file
	//get the file content
	GET /path/to/file
	//create or overwrite the file, the filename in the multipart request will be used
	POST /path/to/
	//return a json format subdirectory and files listing
	GET /path/to/

	The configuration file "filer.toml" is read from ".", "$HOME/.seaweedfs/", or "/etc/seaweedfs/", in that order.

	The example filer.toml configuration file can be generated by "weed scaffold -config=filer"

`,
}

func runFiler(cmd *Command, args []string) bool {

	util.LoadConfiguration("security", false)

	f.startFiler()

	return true
}

func (fo *FilerOptions) startFiler() {

	defaultMux := http.NewServeMux()
	publicVolumeMux := defaultMux

	if *fo.publicPort != 0 {
		publicVolumeMux = http.NewServeMux()
	}

	defaultLevelDbDirectory := "./filerldb2"
	if fo.defaultLevelDbDirectory != nil {
		defaultLevelDbDirectory = util.ResolvePath(*fo.defaultLevelDbDirectory + "/filerldb2")
	}

	var peers []string
	if *fo.peers != "" {
		peers = strings.Split(*fo.peers, ",")
	}

	fs, nfs_err := weed_server.NewFilerServer(defaultMux, publicVolumeMux, &weed_server.FilerOption{
		Masters:            strings.Split(*fo.masters, ","),
		Collection:         *fo.collection,
		DefaultReplication: *fo.defaultReplicaPlacement,
		DisableDirListing:  *fo.disableDirListing,
		MaxMB:              *fo.maxMB,
		DirListingLimit:    *fo.dirListingLimit,
		DataCenter:         *fo.dataCenter,
		DefaultLevelDbDir:  defaultLevelDbDirectory,
		DisableHttp:        *fo.disableHttp,
		Host:               *fo.ip,
		Port:               uint32(*fo.port),
		Cipher:             *fo.cipher,
		Filers:             peers,
	})
	if nfs_err != nil {
		glog.Fatalf("Filer startup error: %v", nfs_err)
	}

	if *fo.publicPort != 0 {
		publicListeningAddress := *fo.bindIp + ":" + strconv.Itoa(*fo.publicPort)
		glog.V(0).Infoln("Start Seaweed filer server", util.Version(), "public at", publicListeningAddress)
		publicListener, e := util.NewListener(publicListeningAddress, 0)
		if e != nil {
			glog.Fatalf("Filer server public listener error on port %d:%v", *fo.publicPort, e)
		}
		go func() {
			if e := http.Serve(publicListener, publicVolumeMux); e != nil {
				glog.Fatalf("Volume server fail to serve public: %v", e)
			}
		}()
	}

	glog.V(0).Infof("Start Seaweed Filer %s at %s:%d", util.Version(), *fo.ip, *fo.port)
	filerListener, e := util.NewListener(
		*fo.bindIp+":"+strconv.Itoa(*fo.port),
		time.Duration(10)*time.Second,
	)
	if e != nil {
		glog.Fatalf("Filer listener error: %v", e)
	}

	// starting grpc server
	grpcPort := *fo.port + 10000
	grpcL, err := util.NewListener(*fo.bindIp+":"+strconv.Itoa(grpcPort), 0)
	if err != nil {
		glog.Fatalf("failed to listen on grpc port %d: %v", grpcPort, err)
	}
	grpcS := pb.NewGrpcServer(security.LoadServerTLS(util.GetViper(), "grpc.filer"))
	filer_pb.RegisterSeaweedFilerServer(grpcS, fs)
	reflection.Register(grpcS)
	go grpcS.Serve(grpcL)

	httpS := &http.Server{Handler: defaultMux}
	if err := httpS.Serve(filerListener); err != nil {
		glog.Fatalf("Filer Fail to serve: %v", e)
	}

}
<|endoftext|>


<commit_before>package connection

import (
	"bytes"
	"encoding/binary"
	"fmt"
	"log"
)

func (conn *connection) ToggleReader() bool {
	if conn.ReaderListening {
		go conn.StopReader()
		return false
	}
	go conn.StartReader()
	return true
}

func (conn *connection) StartReader() {
	conn.ReaderListening = true

	var messageBuffer bytes.Buffer
	var bytesToRead int

	for conn.ReaderListening {
		fmt.Println("Buffer Length:   ", messageBuffer.Len())
		fmt.Println("Buffer Capacity: ", messageBuffer.Cap())
		buf := make([]byte, 1400)

		dataSize, err := conn.Socket.Read(buf)
		if err != nil {
			break
		}
		data := buf[0:dataSize]
		messageBuffer.Write(data)

		if messageBuffer.Len() >= bytesToRead {
			for {
				if bytesToRead != 0 {
					message := make([]byte, bytesToRead)
					messageBytes, err := messageBuffer.Read(message)
					if err != nil {
						log.Fatalln(err)
					}
					if messageBytes != bytesToRead {
						log.Fatalln("Something went wrong, bytes to read != read bytes")
					}
					conn.IncommingMessages <- message
					bytesToRead = 0
				}

				if bytesToRead == 0 && messageBuffer.Len() > 2 {
					btrBuffer := make([]byte, 2)
					btrBytes, err := messageBuffer.Read(btrBuffer)
					if err != nil {
						log.Fatalln(err)
					}
					if btrBytes != 2 {
						log.Fatalln("Something went wrong, btrBytes != 2")
					}

					bytesToRead = int(binary.LittleEndian.Uint16(btrBuffer))
				}

				if bytesToRead == 0 || (messageBuffer.Len() < bytesToRead) {
					break
				}
			}
		}
	}
	conn.ReaderListening = false
}

func (conn *connection) StopReader() {

}
<commit_msg>Connection Reader Completed<commit_after>package connection

import (
	"bytes"
	"encoding/binary"
	"fmt"
	"log"
)

func (conn *connection) ToggleReader() bool {
	if conn.ReaderListening {
		go conn.StopReader()
		return false
	}
	go conn.StartReader()
	return true
}

func (conn *connection) StartReader() {
	conn.ReaderListening = true

	var messageBuffer bytes.Buffer
	var bytesToRead int

	for conn.ReaderListening {
		fmt.Println("Buffer Length:   ", messageBuffer.Len())
		fmt.Println("Buffer Capacity: ", messageBuffer.Cap())
		buf := make([]byte, 1400)

		dataSize, err := conn.Socket.Read(buf)
		if err != nil {
			break
		}
		data := buf[0:dataSize]
		messageBuffer.Write(data)

		if messageBuffer.Len() >= bytesToRead {
			for {
				if bytesToRead != 0 {
					message := make([]byte, bytesToRead)
					messageBytes, err := messageBuffer.Read(message)
					if err != nil {
						log.Fatalln(err)
					}
					if messageBytes != bytesToRead {
						log.Fatalln("Something went wrong, bytes to read != read bytes")
					}
					conn.IncommingMessages <- message
					bytesToRead = 0
				}

				if bytesToRead == 0 && messageBuffer.Len() > 2 {
					btrBuffer := make([]byte, 2)
					btrBytes, err := messageBuffer.Read(btrBuffer)
					if err != nil {
						log.Fatalln(err)
					}
					if btrBytes != 2 {
						log.Fatalln("Something went wrong, btrBytes != 2")
					}

					bytesToRead = int(binary.LittleEndian.Uint16(btrBuffer))
				}

				if bytesToRead == 0 || (messageBuffer.Len() < bytesToRead) {
					break
				}
			}
		}
	}
}

func (conn *connection) StopReader() {
	conn.ReaderListening = false
}
<|endoftext|>


<commit_before>// Copyright 2013 James Pirruccello <james@carbocation.com>
// All rights reserved.
// Use of this source code is governed by an MIT-style
// license that can be found in the LICENSE file.

// Package binarytree implements a binary tree
//
// To iterate over a tree (where tree is a *Tree),
// use the Walk() function or a similar method.
//
package binarytree

// Tree is an element in the tree.
type Tree struct {
	// Parent, left, and right pointers in the tree of elements.
	// The root of the tree has parent = nil
    // Each leaf of a tree has left = nil.
	parent, left, right *Tree

	// The contents of this tree element.
	Value interface{}
}

func New(value interface{}) *Tree {
    return &Tree{Value: value}
}

// Left returns the left tree element or nil.
func (e *Tree) Left() *Tree { return e.left }

// Right returns the right tree element or nil.
func (e *Tree) Right() *Tree { return e.right }

// Parent returns the parent tree element or nil.
func (e *Tree) Parent() *Tree { return e.parent }

// PushLeft inserts a left node's value under the specified Tree and returns a new Tree containing the value.
func (e *Tree) PushLeft(value interface{}) *Tree {
    return e.pushTree("left", value)
}

// PushRight inserts a right node's value under the specified Tree and returns a new Tree containing the value.
func (e *Tree) PushRight(value interface{}) *Tree {
    return e.pushTree("right", value)
}

// Wrapped
func (e *Tree) pushTree(direction string, value interface{}) *Tree {
    // Create the new element el, record that the parent is e
    el := &Tree{e, nil, nil, value}

    // SWITCH/CASE to append to left or right depending on correct output
    // Update the parent element e, record that its left value is el
    switch {
    case direction == "left":
        e.left = el

    case direction == "right":
        e.right = el
    }

    // Return the new element
    return el
}

// Traverse the tree and send output to a channel
// Note that this is simply one reference implementation. For some applications,
// e.g., threaded discussion, something like this would be implemented in the view.
func Walk(el *Tree, ch chan interface{}) {
    if el == nil {
        return
    }

    ch <- el.Value
    Walk(el.left, ch)
    Walk(el.right, ch)
}

// Wrapper for the Walk method that creates a string channel for output
func Walker(e *Tree) <-chan interface{} {
	ch := make(chan interface{})
	go func() {
		Walk(e, ch)
		close(ch)
	}()
	return ch
}
<commit_msg>Allowing the parent / left / right tree elements to be set.<commit_after>// Copyright 2013 James Pirruccello <james@carbocation.com>
// All rights reserved.
// Use of this source code is governed by an MIT-style
// license that can be found in the LICENSE file.

// Package binarytree implements a binary tree
//
// To iterate over a tree (where tree is a *Tree),
// use the Walk() function or a similar method.
//
package binarytree

// Tree is an element in the tree.
type Tree struct {
	// Parent, left, and right pointers in the tree of elements.
	// The root of the tree has parent = nil
    // Each leaf of a tree has left = nil.
	parent, left, right *Tree

	// The contents of this tree element.
	Value interface{}
}

func New(value interface{}) *Tree {
    return &Tree{Value: value}
}

// Left returns the left tree element or nil.
func (e *Tree) Left() *Tree { return e.left }

// Right returns the right tree element or nil.
func (e *Tree) Right() *Tree { return e.right }

// Parent returns the parent tree element or nil.
func (e *Tree) Parent() *Tree { return e.parent }

// Set the left tree element or nil
func (e *Tree) SetLeft(replacement *Tree) { e.left = replacement }

// Set the right tree element or nil
func (e *Tree) SetRight(replacement *Tree) { e.right = replacement }

// Set the parent tree element or nil
func (e *Tree) SetParent(replacement *Tree) { e.parent = replacement }

// PushLeft inserts a left node's value under the specified Tree and returns a new Tree containing the value.
func (e *Tree) PushLeft(value interface{}) *Tree {
    return e.pushTree("left", value)
}

// PushRight inserts a right node's value under the specified Tree and returns a new Tree containing the value.
func (e *Tree) PushRight(value interface{}) *Tree {
    return e.pushTree("right", value)
}

// Wrapped
func (e *Tree) pushTree(direction string, value interface{}) *Tree {
    // Create the new element el, record that the parent is e
    el := &Tree{e, nil, nil, value}

    // SWITCH/CASE to append to left or right depending on correct output
    // Update the parent element e, record that its left value is el
    switch {
    case direction == "left":
        e.left = el

    case direction == "right":
        e.right = el
    }

    // Return the new element
    return el
}

// Traverse the tree and send output to a channel
// Note that this is simply one reference implementation. For some applications,
// e.g., threaded discussion, something like this would be implemented in the view.
func Walk(el *Tree, ch chan interface{}) {
    if el == nil {
        return
    }

    ch <- el.Value
    Walk(el.left, ch)
    Walk(el.right, ch)
}

// Wrapper for the Walk method that creates a string channel for output
func Walker(e *Tree) <-chan interface{} {
	ch := make(chan interface{})
	go func() {
		Walk(e, ch)
		close(ch)
	}()
	return ch
}
<|endoftext|>


<commit_before>// Copyright 2015 lessOS.com, All rights reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package frontend

import (
	// "bytes"
	"crypto/md5"
	"fmt"
	"io"
	"math/rand"
	"net/http"
	"os"
	"path/filepath"
	"strings"
	"time"

	gd "github.com/eryx/go-gd"
	"github.com/lessos/lessgo/httpsrv"
	"github.com/lessos/lessgo/sync"

	"../../config"
	"../../store"
)

var (
	rezlocker = sync.NewPermitPool(-1)
)

type S2 struct {
	*httpsrv.Controller
}

func (c S2) IndexAction() {

	c.AutoRender = false

	var (
		ipn         = c.Params.Get("ipn")
		object_path = strings.Trim(filepath.Clean(c.Request.RequestPath), "/")[3:]
		abs_path    = config.Config.Prefix + "/var/storage/" + object_path
	)

	if ipn == "" {

		if objfp, err := os.Open(abs_path); err == nil {

			c.Response.Out.Header().Set("Cache-Control", "max-age=86400")
			http.ServeContent(c.Response.Out, c.Request.Request, object_path, time.Now(), objfp)
			objfp.Close()

		} else {
			c.RenderError(404, "Object Not Found")
		}

		return
	}

	ext := strings.ToLower(filepath.Ext(object_path))

	switch ext {

	case ".jpg", ".jpeg", ".png", ".gif":
		//

	default:
		c.RenderError(400, "Bad Request #01")
		return
	}

	h1 := md5.New()
	io.WriteString(h1, object_path+"."+ipn)
	hid := fmt.Sprintf("%x", h1.Sum(nil))[0:12]
	if rs := store.CacheGet(hid); rs.Status == "OK" {

		c.Response.Out.Header().Set("Cache-Control", "max-age=86400")

		switch ext {
		case ".jpg", ".jpeg":
			c.Response.Out.Header().Set("Content-type", "image/jpeg")
		case ".png":
			c.Response.Out.Header().Set("Content-type", "image/png")
		case ".git":
			c.Response.Out.Header().Set("Content-type", "image/gif")
		}

		c.Response.Out.Write(rs.Bytes())
		return
	}

	width, height := 2000, 2000

	switch ipn {
	case "i6040":
		width, height = 60, 40
	case "thumb":
		width, height = 150, 150
	case "medium":
		width, height = 300, 168
	case "large":
		width, height = 800, 450
	}

	pn := rezlocker.Pull()
	defer rezlocker.Push(pn)

	// buf := new(bytes.Buffer)
	// buf.ReadFrom(objfp)

	defer func() {
		if x := recover(); x != nil {
			c.RenderError(404, "Object Not Found")
		}
	}()

	var ims *gd.Image

	switch ext {

	case ".jpg", ".jpeg":

		// ims = gd.CreateFromJpegFp(objfp)
		ims = gd.CreateFromJpeg(abs_path)
		// ims = gd.CreateFromJpegPtr(buf.Bytes())

	case ".png":

		// ims = gd.CreateFromPngFp(objfp)
		ims = gd.CreateFromPng(abs_path)
		// ims = gd.CreateFromPngPtr(buf.Bytes())

	case ".gif":

		// ims = gd.CreateFromGifFp(objfp)
		ims = gd.CreateFromGif(abs_path)
		// ims = gd.CreateFromGifPtr(buf.Bytes())
	}

	if ims == nil {
		return
	}

	defer ims.Destroy()

	sx, sy, sw, sh, dstw, dsth, _ := _resize_dimensions(ims.Sx(), ims.Sy(), width, height)

	imd := gd.CreateTrueColor(dstw, dsth)
	defer imd.Destroy()

	if ext == ".png" {
		imd.AlphaBlending(false)
		imd.SaveAlpha(true)
	}

	ims.CopyResampled(imd, 0, 0, sx, sy, dstw, dsth, sw, sh)

	var out []byte

	switch ext {

	case ".jpg", ".jpeg":

		c.Response.Out.Header().Set("Content-type", "image/jpeg")
		out = gd.ImageToJpegBuffer(imd, 85)

	case ".png":

		c.Response.Out.Header().Set("Content-type", "image/jpeg")
		out = gd.ImageToPngBuffer(imd)

	case ".gif":

		c.Response.Out.Header().Set("Content-type", "image/gif")
		out = gd.ImageToGifBuffer(imd)

	}

	if len(out) > 10 {
		store.CacheSetBytes([]byte(hid), out, 36000+rand.Intn(36000))
	}

	// fmt.Println("rez size", len(out))

	c.Response.Out.Header().Set("Cache-Control", "max-age=86400")
	c.Response.Out.Write(out)
}

func _resize_dimensions(srcw, srch, dstw, dsth int) (sx, sy, sw, sh, dw, dh int, do bool) {

	dst_ratio := float32(dstw) / float32(dsth)

	sdw := int(_float_min(float32(srch)*dst_ratio, float32(srcw)))
	sdh := int(_float_min(float32(srcw)/dst_ratio, float32(srch)))

	if dstw > sdw {
		dstw = sdw
	}

	if dsth > sdh {
		dsth = sdh
	}

	sdx := int((srcw - sdw) / 2)
	sdy := int((srch - sdh) / 2)

	return sdx, sdy, sdw, sdh, dstw, dsth, true
}

func _float_min(a, b float32) float32 {

	if a < b {
		return a
	}

	return b
}
<commit_msg>crop in image resize<commit_after>// Copyright 2015 lessOS.com, All rights reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package frontend

import (
	// "bytes"
	"crypto/md5"
	"fmt"
	"io"
	// "math/rand"
	"net/http"
	"os"
	"path/filepath"
	"strings"
	"time"

	gd "github.com/eryx/go-gd"
	"github.com/lessos/lessgo/httpsrv"
	"github.com/lessos/lessgo/sync"

	"../../config"
	"../../store"
)

var (
	rezlocker = sync.NewPermitPool(-1)
)

type S2 struct {
	*httpsrv.Controller
}

func (c S2) IndexAction() {

	c.AutoRender = false

	var (
		ipn         = c.Params.Get("ipn")
		object_path = strings.Trim(filepath.Clean(c.Request.RequestPath), "/")[3:]
		abs_path    = config.Config.Prefix + "/var/storage/" + object_path
	)

	if ipn == "" {

		if objfp, err := os.Open(abs_path); err == nil {

			c.Response.Out.Header().Set("Cache-Control", "max-age=86400")
			http.ServeContent(c.Response.Out, c.Request.Request, object_path, time.Now(), objfp)
			objfp.Close()

		} else {
			c.RenderError(404, "Object Not Found")
		}

		return
	}

	ext := strings.ToLower(filepath.Ext(object_path))

	switch ext {

	case ".jpg", ".jpeg", ".png", ".gif":
		//

	default:
		c.RenderError(400, "Bad Request #01")
		return
	}

	h1 := md5.New()
	io.WriteString(h1, object_path+"."+ipn)
	hid := fmt.Sprintf("%x", h1.Sum(nil))[0:12]
	if rs := store.CacheGet(hid); rs.Status == "OK" {

		c.Response.Out.Header().Set("Cache-Control", "max-age=86400")

		switch ext {
		case ".jpg", ".jpeg":
			c.Response.Out.Header().Set("Content-type", "image/jpeg")
		case ".png":
			c.Response.Out.Header().Set("Content-type", "image/png")
		case ".git":
			c.Response.Out.Header().Set("Content-type", "image/gif")
		}

		c.Response.Out.Write(rs.Bytes())
		return
	}

	width, height, crop := 2000, 2000, true

	switch ipn {
	case "s800":
		width, height, crop = 800, 800, false
	case "thumb":
		width, height = 150, 150
	case "medium":
		width, height = 300, 168
	case "large":
		width, height = 800, 450
	}

	pn := rezlocker.Pull()
	defer rezlocker.Push(pn)

	// buf := new(bytes.Buffer)
	// buf.ReadFrom(objfp)

	defer func() {
		if x := recover(); x != nil {
			c.RenderError(404, "Object Not Found")
		}
	}()

	var ims *gd.Image

	switch ext {

	case ".jpg", ".jpeg":

		// ims = gd.CreateFromJpegFp(objfp)
		ims = gd.CreateFromJpeg(abs_path)
		// ims = gd.CreateFromJpegPtr(buf.Bytes())

	case ".png":

		// ims = gd.CreateFromPngFp(objfp)
		ims = gd.CreateFromPng(abs_path)
		// ims = gd.CreateFromPngPtr(buf.Bytes())

	case ".gif":

		// ims = gd.CreateFromGifFp(objfp)
		ims = gd.CreateFromGif(abs_path)
		// ims = gd.CreateFromGifPtr(buf.Bytes())
	}

	if ims == nil {
		return
	}

	defer ims.Destroy()

	sx, sy, sw, sh, dstw, dsth, _ := _resize_dimensions(ims.Sx(), ims.Sy(), width, height, crop)

	imd := gd.CreateTrueColor(dstw, dsth)
	defer imd.Destroy()

	if ext == ".png" {
		imd.AlphaBlending(false)
		imd.SaveAlpha(true)
	}

	ims.CopyResampled(imd, 0, 0, sx, sy, dstw, dsth, sw, sh)

	var out []byte

	switch ext {

	case ".jpg", ".jpeg":

		c.Response.Out.Header().Set("Content-type", "image/jpeg")
		out = gd.ImageToJpegBuffer(imd, 85)

	case ".png":

		c.Response.Out.Header().Set("Content-type", "image/jpeg")
		out = gd.ImageToPngBuffer(imd)

	case ".gif":

		c.Response.Out.Header().Set("Content-type", "image/gif")
		out = gd.ImageToGifBuffer(imd)

	}

	if len(out) > 10 {
		// store.CacheSetBytes([]byte(hid), out, 36000+rand.Intn(36000))
	}

	// fmt.Println("rez size", len(out))

	c.Response.Out.Header().Set("Cache-Control", "max-age=86400")
	c.Response.Out.Write(out)
}

func _resize_dimensions(srcw, srch, dstw, dsth int, crop bool) (sx, sy, sw, sh, dw, dh int, do bool) {

	if dstw > srcw && dsth > srch {
		return 0, 0, srcw, srch, srcw, srch, true
	}

	if !crop {

		cropw_ratio := float32(dstw) / float32(srcw)
		croph_ratio := float32(dsth) / float32(srch)

		if cropw_ratio < croph_ratio {
			dsth = int(cropw_ratio * float32(srch))
		} else {
			dstw = int(croph_ratio * float32(srcw))
		}
	}

	dst_ratio := float32(dstw) / float32(dsth)
	sdw := int(_float_min(float32(srch)*dst_ratio, float32(srcw)))
	sdh := int(_float_min(float32(srcw)/dst_ratio, float32(srch)))

	if dstw > sdw {
		dstw = sdw
	}

	if dsth > sdh {
		dsth = sdh
	}

	sdx := int((srcw - sdw) / 2)
	sdy := int((srch - sdh) / 2)

	return sdx, sdy, sdw, sdh, dstw, dsth, true
}

func _float_min(a, b float32) float32 {

	if a < b {
		return a
	}

	return b
}

func _float_max(a, b float32) float32 {

	if a > b {
		return a
	}

	return b
}
<|endoftext|>