<commit_before>// Package daemonigo provides a simple wrapper to daemonize applications.
package daemonigo

import (
	"flag"
	"fmt"
	"io"
	"os"
	"path/filepath"
	"strconv"
	"strings"
	"syscall"
	"time"
	"os/exec"
)

// Name of environment variable used to distinguish
// parent and daemonized processes.
var EnvVarName = "_DAEMONIGO"
// Value of environment variable used to distinguish
// parent and daemonized processes.
var EnvVarValue = "1"

// Path to daemon working directory.
// If not set, the current user directory will be used.
var WorkDir = ""
// Value of file mask for PID-file.
var PidFileMask os.FileMode = 0644
// Value of umask for daemonized process.
var Umask = 027

// Application name to daemonize.
// Used for printing in default daemon actions.
var AppName = "daemon"
// Path to application executable.
// Used only for default start/restart actions.
var AppPath = "./" + filepath.Base(os.Args[0])

// Absolute or relative path from working directory to PID file.
var PidFile = "daemon.pid"
// Pointer to PID file to keep file-lock alive.
var pidFile *os.File

// This function wraps application with daemonization.
// Returns isDaemon value to distinguish parent and daemonized processes.
func Daemonize() (isDaemon bool, err error) {
	const errLoc = "daemonigo.Daemonize()"
	isDaemon = os.Getenv(EnvVarName) == EnvVarValue
	if len(WorkDir) != 0 {
		if err = os.Chdir(WorkDir); err != nil {
			err = fmt.Errorf("%s: changing working directory failed, reason -> %s", errLoc, err.Error())
			return
		}
	}
	if isDaemon {
		syscall.Umask(int(Umask))
		if _, err = syscall.Setsid(); err != nil {
			err = fmt.Errorf("%s: setsid failed, reason -> %s", errLoc, err.Error())
			return
		}
		if pidFile, err = lockPidFile(); err != nil {
			err = fmt.Errorf("%s: locking PID file failed, reason -> %s", errLoc, err.Error())
		}
	} else {
		if !flag.Parsed() {
			flag.Parse()
		}
		action, exist := actions[flag.Arg(0)]
		if exist {
			action()
		} else {
			arr := make([]string, 0, len(actions))
			for k, _ := range actions {
				arr = append(arr, k)
			}
			fmt.Println("Usage: " + os.Args[0] + " {" + strings.Join(arr, "|") + "}")
		}
	}
	return
}

// Locks PID file with a file lock.
// Keeps PID file open until applications exits.
func lockPidFile() (pidFile *os.File, err error) {
	var file *os.File
	file, err = os.OpenFile(PidFile, os.O_WRONLY|os.O_CREATE|os.O_TRUNC, PidFileMask)
	if err != nil {
		return
	}
	defer func() {
		// file must be open whole runtime to keep lock on itself
		if err != nil {
			file.Close()
		}
	}()

	if err = syscall.Flock(int(file.Fd()), syscall.LOCK_EX); err != nil {
		return
	}
	var fileLen int
	fileLen, err = fmt.Fprint(file, os.Getpid())
	if err != nil {
		return
	}
	if err = file.Truncate(int64(fileLen)); err != nil {
		return
	}

	return file, err
}

// Unlocks PID file and closes it.
//
// This function can be useful for graceful restarts or other
// untrivial scenarios, but usually there is no need to use it.
func UnlockPidFile() {
	if pidFile != nil {
		syscall.Flock(int(pidFile.Fd()), syscall.LOCK_UN)
		pidFile.Close()
	}
}

// Checks status of daemonized process.
// Can be used in daemon actions to perate with daemonized process.
func Status() (isRunning bool, pr *os.Process, e error) {
	const errLoc = "daemonigo.Status()"
	var (
		err  error
		file *os.File
	)

	file, err = os.Open(PidFile)
	if err != nil {
		if !os.IsNotExist(err) {
			e = fmt.Errorf("%s: could not open PID file, reason -> %s", errLoc, err.Error())
		}
		return
	}
	defer file.Close()
	fd := int(file.Fd())
	if err = syscall.Flock(fd, syscall.LOCK_EX|syscall.LOCK_NB); err != syscall.EWOULDBLOCK {
		if err == nil {
			syscall.Flock(fd, syscall.LOCK_UN)
		} else {
			e = fmt.Errorf("%s: PID file locking attempt failed, reason -> %s", errLoc, err.Error())
		}
		return
	}

	isRunning = true
	var n, pid int
	content := make([]byte, 128)
	n, err = file.Read(content)
	if err != nil && err != io.EOF {
		e = fmt.Errorf("%s: could not read from PID file, reason -> %s", errLoc, err.Error())
		return
	}
	pid, err = strconv.Atoi(string(content[:n]))
	if n < 1 || err != nil {
		e = fmt.Errorf("%s: bad PID format, PID file is possibly corrupted", errLoc)
		return
	}
	pr, err = os.FindProcess(pid)
	if err != nil {
		e = fmt.Errorf("%s: cannot find process by PID, reason -> %s", errLoc, err.Error())
	}

	return
}

// Starts daemon process and waits timeout number of seconds.
// If daemonized process keeps running after timeout seconds passed
// then process seems to be successfully started.
//
// This function can also be used when writing your own daemon actions.
func Start(timeout uint8) (e error) {
	const errLoc = "daemonigo.Start()"
	path, err := filepath.Abs(AppPath)
	if err != nil {
		e = fmt.Errorf("%s: failed to resolve absolute path of %s, reason -> %s", errLoc, AppName, err.Error())
		return
	}
	cmd := exec.Command(path)
	cmd.Env = append(
		os.Environ(),
		fmt.Sprintf("%s=%s", EnvVarName, EnvVarValue),
	)
	if err = cmd.Start(); err != nil {
		e = fmt.Errorf("%s: failed to start %s, reason -> %s", errLoc, AppName, err.Error())
		return
	}
	select {
	case <-func() chan bool {
		ch := make(chan bool)
		go func() {
			if err := cmd.Wait(); err!= nil {
				e = fmt.Errorf("%s: %s running failed, reason -> %s", errLoc, AppName, err.Error())
			} else {
				e = fmt.Errorf("%s: %s stopped and not running", errLoc, AppName)
			}
			ch <- true
		}()
		return ch
	}():
	case <-time.After(time.Duration(timeout) * time.Second):
	}
	return
}

// Stops daemon process.
// Sends signal os.Interrupt to daemonized process.
//
// This function can also be used when writing your own daemon actions.
func Stop(process *os.Process) (e error){
	if err := process.Signal(os.Interrupt); err != nil {
		e = fmt.Errorf("daemonigo.Stop(): failed to send interrupt signal to %s, reason -> %s", AppName, err.Error())
		return
	}
	for {
		time.Sleep(200 * time.Millisecond)
		switch isRunning, _, err := Status(); {
		case err != nil:
			e = fmt.Errorf("daemonigo.Stop(): checking status of %s failed, reason -> %s", AppName, err.Error())
			return
		case !isRunning:
			return
		}
	}
}
<commit_msg>upd with gofmt<commit_after>// Package daemonigo provides a simple wrapper to daemonize applications.
package daemonigo

import (
	"flag"
	"fmt"
	"io"
	"os"
	"os/exec"
	"path/filepath"
	"strconv"
	"strings"
	"syscall"
	"time"
)

// Name of environment variable used to distinguish
// parent and daemonized processes.
var EnvVarName = "_DAEMONIGO"

// Value of environment variable used to distinguish
// parent and daemonized processes.
var EnvVarValue = "1"

// Path to daemon working directory.
// If not set, the current user directory will be used.
var WorkDir = ""

// Value of file mask for PID-file.
var PidFileMask os.FileMode = 0644

// Value of umask for daemonized process.
var Umask = 027

// Application name to daemonize.
// Used for printing in default daemon actions.
var AppName = "daemon"

// Path to application executable.
// Used only for default start/restart actions.
var AppPath = "./" + filepath.Base(os.Args[0])

// Absolute or relative path from working directory to PID file.
var PidFile = "daemon.pid"

// Pointer to PID file to keep file-lock alive.
var pidFile *os.File

// This function wraps application with daemonization.
// Returns isDaemon value to distinguish parent and daemonized processes.
func Daemonize() (isDaemon bool, err error) {
	const errLoc = "daemonigo.Daemonize()"
	isDaemon = os.Getenv(EnvVarName) == EnvVarValue
	if len(WorkDir) != 0 {
		if err = os.Chdir(WorkDir); err != nil {
			err = fmt.Errorf("%s: changing working directory failed, reason -> %s", errLoc, err.Error())
			return
		}
	}
	if isDaemon {
		syscall.Umask(int(Umask))
		if _, err = syscall.Setsid(); err != nil {
			err = fmt.Errorf("%s: setsid failed, reason -> %s", errLoc, err.Error())
			return
		}
		if pidFile, err = lockPidFile(); err != nil {
			err = fmt.Errorf("%s: locking PID file failed, reason -> %s", errLoc, err.Error())
		}
	} else {
		if !flag.Parsed() {
			flag.Parse()
		}
		action, exist := actions[flag.Arg(0)]
		if exist {
			action()
		} else {
			arr := make([]string, 0, len(actions))
			for k, _ := range actions {
				arr = append(arr, k)
			}
			fmt.Println("Usage: " + os.Args[0] + " {" + strings.Join(arr, "|") + "}")
		}
	}
	return
}

// Locks PID file with a file lock.
// Keeps PID file open until applications exits.
func lockPidFile() (pidFile *os.File, err error) {
	var file *os.File
	file, err = os.OpenFile(PidFile, os.O_WRONLY|os.O_CREATE|os.O_TRUNC, PidFileMask)
	if err != nil {
		return
	}
	defer func() {
		// file must be open whole runtime to keep lock on itself
		if err != nil {
			file.Close()
		}
	}()

	if err = syscall.Flock(int(file.Fd()), syscall.LOCK_EX); err != nil {
		return
	}
	var fileLen int
	fileLen, err = fmt.Fprint(file, os.Getpid())
	if err != nil {
		return
	}
	if err = file.Truncate(int64(fileLen)); err != nil {
		return
	}

	return file, err
}

// Unlocks PID file and closes it.
//
// This function can be useful for graceful restarts or other
// untrivial scenarios, but usually there is no need to use it.
func UnlockPidFile() {
	if pidFile != nil {
		syscall.Flock(int(pidFile.Fd()), syscall.LOCK_UN)
		pidFile.Close()
	}
}

// Checks status of daemonized process.
// Can be used in daemon actions to perate with daemonized process.
func Status() (isRunning bool, pr *os.Process, e error) {
	const errLoc = "daemonigo.Status()"
	var (
		err  error
		file *os.File
	)

	file, err = os.Open(PidFile)
	if err != nil {
		if !os.IsNotExist(err) {
			e = fmt.Errorf("%s: could not open PID file, reason -> %s", errLoc, err.Error())
		}
		return
	}
	defer file.Close()
	fd := int(file.Fd())
	if err = syscall.Flock(fd, syscall.LOCK_EX|syscall.LOCK_NB); err != syscall.EWOULDBLOCK {
		if err == nil {
			syscall.Flock(fd, syscall.LOCK_UN)
		} else {
			e = fmt.Errorf("%s: PID file locking attempt failed, reason -> %s", errLoc, err.Error())
		}
		return
	}

	isRunning = true
	var n, pid int
	content := make([]byte, 128)
	n, err = file.Read(content)
	if err != nil && err != io.EOF {
		e = fmt.Errorf("%s: could not read from PID file, reason -> %s", errLoc, err.Error())
		return
	}
	pid, err = strconv.Atoi(string(content[:n]))
	if n < 1 || err != nil {
		e = fmt.Errorf("%s: bad PID format, PID file is possibly corrupted", errLoc)
		return
	}
	pr, err = os.FindProcess(pid)
	if err != nil {
		e = fmt.Errorf("%s: cannot find process by PID, reason -> %s", errLoc, err.Error())
	}

	return
}

// Starts daemon process and waits timeout number of seconds.
// If daemonized process keeps running after timeout seconds passed
// then process seems to be successfully started.
//
// This function can also be used when writing your own daemon actions.
func Start(timeout uint8) (e error) {
	const errLoc = "daemonigo.Start()"
	path, err := filepath.Abs(AppPath)
	if err != nil {
		e = fmt.Errorf("%s: failed to resolve absolute path of %s, reason -> %s", errLoc, AppName, err.Error())
		return
	}
	cmd := exec.Command(path)
	cmd.Env = append(
		os.Environ(),
		fmt.Sprintf("%s=%s", EnvVarName, EnvVarValue),
	)
	if err = cmd.Start(); err != nil {
		e = fmt.Errorf("%s: failed to start %s, reason -> %s", errLoc, AppName, err.Error())
		return
	}
	select {
	case <-func() chan bool {
		ch := make(chan bool)
		go func() {
			if err := cmd.Wait(); err != nil {
				e = fmt.Errorf("%s: %s running failed, reason -> %s", errLoc, AppName, err.Error())
			} else {
				e = fmt.Errorf("%s: %s stopped and not running", errLoc, AppName)
			}
			ch <- true
		}()
		return ch
	}():
	case <-time.After(time.Duration(timeout) * time.Second):
	}
	return
}

// Stops daemon process.
// Sends signal os.Interrupt to daemonized process.
//
// This function can also be used when writing your own daemon actions.
func Stop(process *os.Process) (e error) {
	if err := process.Signal(os.Interrupt); err != nil {
		e = fmt.Errorf("daemonigo.Stop(): failed to send interrupt signal to %s, reason -> %s", AppName, err.Error())
		return
	}
	for {
		time.Sleep(200 * time.Millisecond)
		switch isRunning, _, err := Status(); {
		case err != nil:
			e = fmt.Errorf("daemonigo.Stop(): checking status of %s failed, reason -> %s", AppName, err.Error())
			return
		case !isRunning:
			return
		}
	}
}
<|endoftext|>


<commit_before>package python

import (
	"encoding/json"
	"fmt"
	"io/ioutil"
	"log"
	"net/url"
	"os"
	"os/exec"
	"path/filepath"

	"strings"

	"sourcegraph.com/sourcegraph/srclib/graph"
	"sourcegraph.com/sourcegraph/srclib/toolchain"
	"sourcegraph.com/sourcegraph/srclib/unit"
)

type GraphContext struct {
	Unit *unit.SourceUnit
	Reqs []*requirement
}

func NewGraphContext(unit *unit.SourceUnit) *GraphContext {
	var g GraphContext
	g.Unit = unit
	for _, dep := range unit.Dependencies {
		if req, err := asRequirement(dep); err == nil {
			g.Reqs = append(g.Reqs, req)
		}
	}
	return &g
}

// Graphs the Python source unit. If run outside of a Docker container, this assumes that the source unit has already
// been installed (via pip or `python setup.py install`).
func (c *GraphContext) Graph() (*graph.Output, error) {
	programMode := os.Getenv("IN_DOCKER_CONTAINER") == ""
	tc, err := toolchain.Lookup("sourcegraph.com/sourcegraph/srclib-python")
	if err != nil {
		return nil, err
	}
	pipBin := "pip"
	pythonBin := "python"

	if programMode {
		tempPath, err := getTempPath()
		if err != nil {
			return nil, err
		}
		envName := fmt.Sprintf("%s-%s-env", getHash(c.Unit.Dir), url.QueryEscape(c.Unit.Name))
		envDir := filepath.Join(tempPath, envName)

		// Use binaries from our virutal env.
		pipBin = filepath.Join(envDir, "bin", "pip")
		pythonBin = filepath.Join(envDir, "bin", "python")

		if _, err := os.Stat(filepath.Join(envDir)); os.IsNotExist(err) {
			// We don't have virtual env for this SourceUnit, create one.
			tcVENVBinPath := filepath.Join(tc.Dir, ".env", "bin")
			cmd := exec.Command(filepath.Join(tcVENVBinPath, "virtualenv"), envDir)
			if err := runCmdStderr(cmd); err != nil {
				return nil, err
			}
			// Install our dependencies.
			// Todo(MaikuMori): Use symlinks from toolchains virtualenv to project virtual env.
			// NOTE: If SourceUnit requirements overwrite our requirements, things will fail.
			// 			 We could install them last, but then we would have to do this before each
			//			 graphing which noticably increases graphing time (since our deps are always
			//       downloaded by pip due to dependency on git commit not actual package version).
			requirementFile := filepath.Join(tc.Dir, "requirements.txt")
			if err := runCmdStderr(exec.Command(pipBin, "install", "-r", requirementFile)); err != nil {
				return nil, err
			}
			if err := runCmdStderr(exec.Command(pipBin, "install", "-e", tc.Dir)); err != nil {
				return nil, err
			}
		}
	}

	// NOTE: this may cause an error when graphing any source unit that depends
	// on jedi (or any other dependency of the graph code)
	requirementFiles, err := filepath.Glob(filepath.Join(c.Unit.Dir, "*requirements*.txt"))
	if err != nil {
		return nil, err
	}
	if _, err := os.Stat(filepath.Join(c.Unit.Dir, "setup.py")); !os.IsNotExist(err) {
		runCmdLogError(exec.Command(pipBin, "install", "-I", c.Unit.Dir))
	}
	installPipRequirements(pipBin, requirementFiles)

	cmd := exec.Command(pythonBin, "-m", "grapher.graph", "--verbose", "--dir", c.Unit.Dir, "--files")
	cmd.Args = append(cmd.Args, c.Unit.Files...)
	cmd.Stderr = os.Stderr
	log.Printf("Running %v", cmd.Args)
	b, err := cmd.Output()
	if err != nil {
		return nil, err
	}

	var raw RawOutput
	if err := json.Unmarshal(b, &raw); err != nil {
		return nil, err
	}

	out := c.transform(&raw, c.Unit)
	return out, nil
}

func (c *GraphContext) transform(raw *RawOutput, unit *unit.SourceUnit) *graph.Output {
	var out graph.Output

	for _, def := range raw.Defs {
		out.Defs = append(out.Defs, c.transformDef(def))
		if doc := c.transformDefDoc(def); doc != nil {
			out.Docs = append(out.Docs, doc)
		}
	}
	for _, ref := range raw.Refs {
		if outRef, err := c.transformRef(ref); err == nil {
			out.Refs = append(out.Refs, outRef)
		} else {
			log.Printf("Could not transform ref %v: %s", ref, err)
		}
	}

	return &out
}

var jediKindToDefKind = map[string]string{
	"statement":        "var",
	"statementelement": "var",
	"param":            "var",
	"module":           "module",
	"submodule":        "module",
	"class":            "type",
	"function":         "func",
	"lambda":           "func",
	"import":           "var",
}

func (c *GraphContext) transformDef(rawDef *RawDef) *graph.Def {
	return &graph.Def{
		DefKey: graph.DefKey{
			Repo:     c.Unit.Repo,
			Unit:     c.Unit.Name,
			UnitType: c.Unit.Type,
			Path:     string(rawDef.Path),
		},
		TreePath: string(rawDef.Path), // TODO: make this consistent w/ old way
		Kind:     jediKindToDefKind[rawDef.Kind],
		Name:     rawDef.Name,
		File:     rawDef.File,
		DefStart: rawDef.DefStart,
		DefEnd:   rawDef.DefEnd,
		Exported: rawDef.Exported,
		Data:     nil, // TODO
	}
}

func (c *GraphContext) transformRef(rawRef *RawRef) (*graph.Ref, error) {
	defUnit, err := c.inferSourceUnit(rawRef, c.Reqs)
	if err != nil {
		return nil, err
	}

	defPath := string(rawRef.DefPath)
	if defPath == "" {
		defPath = "."
	}

	return &graph.Ref{
		DefRepo:     defUnit.Repo,
		DefUnitType: defUnit.Type,
		DefUnit:     defUnit.Name,
		DefPath:     defPath,

		Repo:     c.Unit.Repo,
		Unit:     c.Unit.Name,
		UnitType: c.Unit.Type,

		File:  rawRef.File,
		Start: rawRef.Start,
		End:   rawRef.End,
		Def:   rawRef.Def,
	}, nil
}

func (c *GraphContext) transformDefDoc(rawDef *RawDef) *graph.Doc {
	return &graph.Doc{
		DefKey: graph.DefKey{
			Repo:     c.Unit.Repo,
			Unit:     c.Unit.Name,
			UnitType: c.Unit.Type,
			Path:     string(rawDef.Path),
		},
		Data:  rawDef.Docstring,
	}
}

func (c *GraphContext) inferSourceUnit(rawRef *RawRef, reqs []*requirement) (*unit.SourceUnit, error) {
	if rawRef.ToBuiltin {
		return stdLibPkg.SourceUnit(), nil
	}
	return c.inferSourceUnitFromFile(rawRef.DefFile, reqs)
}

// Note: file is expected to be an absolute path
func (c *GraphContext) inferSourceUnitFromFile(file string, reqs []*requirement) (*unit.SourceUnit, error) {
	// Case: in current source unit (u)
	pwd, _ := os.Getwd()
	if isSubPath(pwd, file) {
		return c.Unit, nil
	}

	// Case: in dependent source unit(depUnits)
	fileCmps := strings.Split(file, string(filepath.Separator))
	pkgsDirIdx := -1
	for i, cmp := range fileCmps {
		if cmp == "site-packages" || cmp == "dist-packages" {
			pkgsDirIdx = i
			break
		}
	}
	if pkgsDirIdx != -1 {
		fileSubCmps := fileCmps[pkgsDirIdx+1:]
		fileSubPath := filepath.Join(fileSubCmps...)

		var foundReq *requirement
	FindReq:
		for _, req := range reqs {
			for _, pkg := range req.Packages {
				if isSubPath(moduleToFilepath(pkg, true), fileSubPath) {
					foundReq = req
					break FindReq
				}
			}
			for _, mod := range req.Modules {
				if moduleToFilepath(mod, false) == fileSubPath {
					foundReq = req
					break FindReq
				}
			}
		}

		if foundReq == nil {
			var candidatesStr string
			if len(reqs) <= 7 {
				candidatesStr = fmt.Sprintf("%v", reqs)
			} else {
				candidatesStr = fmt.Sprintf("%v...", reqs[:7])
			}
			// FIXME: This doesn't work, note the pointer in `[]*requirement`. As error you get
			//        string representation of array of pointers.
			return nil, fmt.Errorf("Could not find requirement that contains file %s. Candidates were: %s",
				file, candidatesStr)
		}

		return foundReq.SourceUnit(), nil
	}

	// Case 3: in std lib
	pythonDirIdx := -1
	for i, cmp := range fileCmps {
		if strings.HasPrefix(cmp, "python") {
			pythonDirIdx = i
			break
		}
	}
	if pythonDirIdx != -1 {
		return stdLibPkg.SourceUnit(), nil
	}

	return nil, fmt.Errorf("Cannot infer source unit for file %s", file)
}

func isSubPath(parent, child string) bool {
	relpath, err := filepath.Rel(parent, child)
	return err == nil && !strings.HasPrefix(relpath, "..")
}

func moduleToFilepath(moduleName string, isPackage bool) string {
	moduleName = strings.Replace(moduleName, ".", "/", -1)
	if !isPackage {
		moduleName += ".py"
	}
	return moduleName
}

type RawOutput struct {
	Defs []*RawDef
	Refs []*RawRef
}

type RawDef struct {
	Path      string
	Kind      string
	Name      string
	File      string // relative path (to source unit directory)
	DefStart  uint32
	DefEnd    uint32
	Exported  bool
	Docstring string
	Data      interface{}
}

type RawRef struct {
	DefPath   string
	Def       bool
	DefFile   string // absolute path
	File      string // relative path (to source unit directory)
	Start     uint32
	End       uint32
	ToBuiltin bool
}

func installPipRequirements(pipBin string, requirementFiles []string) {
	for _, requirementFile := range requirementFiles {
		err := runCmdStderr(exec.Command(pipBin, "install", "-r", requirementFile))
		if err != nil {
			log.Printf("Error installing dependencies in %s. Trying piecemeal install", requirementFile)
			if b, err := ioutil.ReadFile(requirementFile); err == nil {
				for _, req := range strings.Split(string(b), "\n") {
					runCmdLogError(exec.Command(pipBin, "install", req))
				}
			} else {
				log.Printf("Could not read %s: %s", requirementFile, err)
			}
		}
	}
}
<commit_msg>Grapher: fix error output.<commit_after>package python

import (
	"encoding/json"
	"fmt"
	"io/ioutil"
	"log"
	"net/url"
	"os"
	"os/exec"
	"path/filepath"

	"strings"

	"sourcegraph.com/sourcegraph/srclib/graph"
	"sourcegraph.com/sourcegraph/srclib/toolchain"
	"sourcegraph.com/sourcegraph/srclib/unit"
)

type GraphContext struct {
	Unit *unit.SourceUnit
	Reqs []*requirement
}

func NewGraphContext(unit *unit.SourceUnit) *GraphContext {
	var g GraphContext
	g.Unit = unit
	for _, dep := range unit.Dependencies {
		if req, err := asRequirement(dep); err == nil {
			g.Reqs = append(g.Reqs, req)
		}
	}
	return &g
}

// Graphs the Python source unit. If run outside of a Docker container, this assumes that the source unit has already
// been installed (via pip or `python setup.py install`).
func (c *GraphContext) Graph() (*graph.Output, error) {
	programMode := os.Getenv("IN_DOCKER_CONTAINER") == ""
	tc, err := toolchain.Lookup("sourcegraph.com/sourcegraph/srclib-python")
	if err != nil {
		return nil, err
	}
	pipBin := "pip"
	pythonBin := "python"

	if programMode {
		tempPath, err := getTempPath()
		if err != nil {
			return nil, err
		}
		envName := fmt.Sprintf("%s-%s-env", getHash(c.Unit.Dir), url.QueryEscape(c.Unit.Name))
		envDir := filepath.Join(tempPath, envName)

		// Use binaries from our virutal env.
		pipBin = filepath.Join(envDir, "bin", "pip")
		pythonBin = filepath.Join(envDir, "bin", "python")

		if _, err := os.Stat(filepath.Join(envDir)); os.IsNotExist(err) {
			// We don't have virtual env for this SourceUnit, create one.
			tcVENVBinPath := filepath.Join(tc.Dir, ".env", "bin")
			cmd := exec.Command(filepath.Join(tcVENVBinPath, "virtualenv"), envDir)
			if err := runCmdStderr(cmd); err != nil {
				return nil, err
			}
			// Install our dependencies.
			// Todo(MaikuMori): Use symlinks from toolchains virtualenv to project virtual env.
			// NOTE: If SourceUnit requirements overwrite our requirements, things will fail.
			// 			 We could install them last, but then we would have to do this before each
			//			 graphing which noticably increases graphing time (since our deps are always
			//       downloaded by pip due to dependency on git commit not actual package version).
			requirementFile := filepath.Join(tc.Dir, "requirements.txt")
			if err := runCmdStderr(exec.Command(pipBin, "install", "-r", requirementFile)); err != nil {
				return nil, err
			}
			if err := runCmdStderr(exec.Command(pipBin, "install", "-e", tc.Dir)); err != nil {
				return nil, err
			}
		}
	}

	// NOTE: this may cause an error when graphing any source unit that depends
	// on jedi (or any other dependency of the graph code)
	requirementFiles, err := filepath.Glob(filepath.Join(c.Unit.Dir, "*requirements*.txt"))
	if err != nil {
		return nil, err
	}
	if _, err := os.Stat(filepath.Join(c.Unit.Dir, "setup.py")); !os.IsNotExist(err) {
		runCmdLogError(exec.Command(pipBin, "install", "-I", c.Unit.Dir))
	}
	installPipRequirements(pipBin, requirementFiles)

	cmd := exec.Command(pythonBin, "-m", "grapher.graph", "--verbose", "--dir", c.Unit.Dir, "--files")
	cmd.Args = append(cmd.Args, c.Unit.Files...)
	cmd.Stderr = os.Stderr
	log.Printf("Running %v", cmd.Args)
	b, err := cmd.Output()
	if err != nil {
		return nil, err
	}

	var raw RawOutput
	if err := json.Unmarshal(b, &raw); err != nil {
		return nil, err
	}

	out := c.transform(&raw, c.Unit)
	return out, nil
}

func (c *GraphContext) transform(raw *RawOutput, unit *unit.SourceUnit) *graph.Output {
	var out graph.Output

	for _, def := range raw.Defs {
		out.Defs = append(out.Defs, c.transformDef(def))
		if doc := c.transformDefDoc(def); doc != nil {
			out.Docs = append(out.Docs, doc)
		}
	}
	for _, ref := range raw.Refs {
		if outRef, err := c.transformRef(ref); err == nil {
			out.Refs = append(out.Refs, outRef)
		} else {
			log.Printf("Could not transform ref %v: %s", ref, err)
		}
	}

	return &out
}

var jediKindToDefKind = map[string]string{
	"statement":        "var",
	"statementelement": "var",
	"param":            "var",
	"module":           "module",
	"submodule":        "module",
	"class":            "type",
	"function":         "func",
	"lambda":           "func",
	"import":           "var",
}

func (c *GraphContext) transformDef(rawDef *RawDef) *graph.Def {
	return &graph.Def{
		DefKey: graph.DefKey{
			Repo:     c.Unit.Repo,
			Unit:     c.Unit.Name,
			UnitType: c.Unit.Type,
			Path:     string(rawDef.Path),
		},
		TreePath: string(rawDef.Path), // TODO: make this consistent w/ old way
		Kind:     jediKindToDefKind[rawDef.Kind],
		Name:     rawDef.Name,
		File:     rawDef.File,
		DefStart: rawDef.DefStart,
		DefEnd:   rawDef.DefEnd,
		Exported: rawDef.Exported,
		Data:     nil, // TODO
	}
}

func (c *GraphContext) transformRef(rawRef *RawRef) (*graph.Ref, error) {
	defUnit, err := c.inferSourceUnit(rawRef, c.Reqs)
	if err != nil {
		return nil, err
	}

	defPath := string(rawRef.DefPath)
	if defPath == "" {
		defPath = "."
	}

	return &graph.Ref{
		DefRepo:     defUnit.Repo,
		DefUnitType: defUnit.Type,
		DefUnit:     defUnit.Name,
		DefPath:     defPath,

		Repo:     c.Unit.Repo,
		Unit:     c.Unit.Name,
		UnitType: c.Unit.Type,

		File:  rawRef.File,
		Start: rawRef.Start,
		End:   rawRef.End,
		Def:   rawRef.Def,
	}, nil
}

func (c *GraphContext) transformDefDoc(rawDef *RawDef) *graph.Doc {
	return &graph.Doc{
		DefKey: graph.DefKey{
			Repo:     c.Unit.Repo,
			Unit:     c.Unit.Name,
			UnitType: c.Unit.Type,
			Path:     string(rawDef.Path),
		},
		Data: rawDef.Docstring,
	}
}

func (c *GraphContext) inferSourceUnit(rawRef *RawRef, reqs []*requirement) (*unit.SourceUnit, error) {
	if rawRef.ToBuiltin {
		return stdLibPkg.SourceUnit(), nil
	}
	return c.inferSourceUnitFromFile(rawRef.DefFile, reqs)
}

// Note: file is expected to be an absolute path
func (c *GraphContext) inferSourceUnitFromFile(file string, reqs []*requirement) (*unit.SourceUnit, error) {
	// Case: in current source unit (u)
	pwd, _ := os.Getwd()
	if isSubPath(pwd, file) {
		return c.Unit, nil
	}

	// Case: in dependent source unit(depUnits)
	fileCmps := strings.Split(file, string(filepath.Separator))
	pkgsDirIdx := -1
	for i, cmp := range fileCmps {
		if cmp == "site-packages" || cmp == "dist-packages" {
			pkgsDirIdx = i
			break
		}
	}
	if pkgsDirIdx != -1 {
		fileSubCmps := fileCmps[pkgsDirIdx+1:]
		fileSubPath := filepath.Join(fileSubCmps...)

		var foundReq *requirement
	FindReq:
		for _, req := range reqs {
			for _, pkg := range req.Packages {
				if isSubPath(moduleToFilepath(pkg, true), fileSubPath) {
					foundReq = req
					break FindReq
				}
			}
			for _, mod := range req.Modules {
				if moduleToFilepath(mod, false) == fileSubPath {
					foundReq = req
					break FindReq
				}
			}
		}

		if foundReq == nil {
			var formattedCanditates []string
			end := ""
			candiates := reqs

			if len(reqs) > 7 {
				candiates = reqs[:7]
				end = ", ..."
			}

			for _, candidate := range candiates {
				formattedCanditates = append(formattedCanditates, fmt.Sprintf("%v", *candidate))
			}

			return nil, fmt.Errorf("Could not find requirement that contains file %s. Candidates were: %s",
				file, strings.Join(formattedCanditates, ", ")+end)
		}

		return foundReq.SourceUnit(), nil
	}

	// Case 3: in std lib
	pythonDirIdx := -1
	for i, cmp := range fileCmps {
		if strings.HasPrefix(cmp, "python") {
			pythonDirIdx = i
			break
		}
	}
	if pythonDirIdx != -1 {
		return stdLibPkg.SourceUnit(), nil
	}

	return nil, fmt.Errorf("Cannot infer source unit for file %s", file)
}

func isSubPath(parent, child string) bool {
	relpath, err := filepath.Rel(parent, child)
	return err == nil && !strings.HasPrefix(relpath, "..")
}

func moduleToFilepath(moduleName string, isPackage bool) string {
	moduleName = strings.Replace(moduleName, ".", "/", -1)
	if !isPackage {
		moduleName += ".py"
	}
	return moduleName
}

type RawOutput struct {
	Defs []*RawDef
	Refs []*RawRef
}

type RawDef struct {
	Path      string
	Kind      string
	Name      string
	File      string // relative path (to source unit directory)
	DefStart  uint32
	DefEnd    uint32
	Exported  bool
	Docstring string
	Data      interface{}
}

type RawRef struct {
	DefPath   string
	Def       bool
	DefFile   string // absolute path
	File      string // relative path (to source unit directory)
	Start     uint32
	End       uint32
	ToBuiltin bool
}

func installPipRequirements(pipBin string, requirementFiles []string) {
	for _, requirementFile := range requirementFiles {
		err := runCmdStderr(exec.Command(pipBin, "install", "-r", requirementFile))
		if err != nil {
			log.Printf("Error installing dependencies in %s. Trying piecemeal install", requirementFile)
			if b, err := ioutil.ReadFile(requirementFile); err == nil {
				for _, req := range strings.Split(string(b), "\n") {
					runCmdLogError(exec.Command(pipBin, "install", req))
				}
			} else {
				log.Printf("Could not read %s: %s", requirementFile, err)
			}
		}
	}
}
<|endoftext|>


<commit_before>/**********************************************************\
|                                                          |
|                          hprose                          |
|                                                          |
| Official WebSite: http://www.hprose.com/                 |
|                   http://www.hprose.org/                 |
|                                                          |
\**********************************************************/
/**********************************************************\
 *                                                        *
 * promise/future.go                                      *
 *                                                        *
 * future promise implementation for Go.                  *
 *                                                        *
 * LastModified: Aug 13, 2015                             *
 * Author: Ma Bingyao <andot@hprose.com>                  *
 *                                                        *
\**********************************************************/

package promise

import (
	"sync/atomic"
	"time"
)

type subscriber struct {
	onFulfilled OnFulfilled
	onRejected  OnRejected
	next        Promise
}

type future struct {
	value       interface{}
	reason      error
	state       uint32
	subscribers []subscriber
}

// New creates a PENDING Promise object
func New() Promise {
	return new(future)
}

func (p *future) then(onFulfilled OnFulfilled, onRejected OnRejected) Promise {
	next := New()
	switch State(p.state) {
	case FULFILLED:
		resolve(next, onFulfilled, p.value)
	case REJECTED:
		reject(next, onRejected, p.reason)
	default:
		p.subscribers = append(p.subscribers,
			subscriber{onFulfilled, onRejected, next})
	}
	return next
}

func (p *future) Then(onFulfilled OnFulfilled, onRejected ...OnRejected) Promise {
	if len(onRejected) == 0 {
		return p.then(onFulfilled, nil)
	}
	return p.then(onFulfilled, onRejected[0])
}

func (p *future) catch(onRejected OnRejected, test TestFunc) Promise {
	if test == nil {
		return p.then(nil, onRejected)
	}
	return p.then(nil, func(e error) (interface{}, error) {
		if test(e) {
			return p.then(nil, onRejected), nil
		}
		return nil, e
	})
}

func (p *future) Catch(onRejected OnRejected, test ...TestFunc) Promise {
	if len(test) == 0 {
		return p.catch(onRejected, nil)
	}
	return p.catch(onRejected, test[0])
}

func (p *future) Complete(onCompleted OnCompleted) Promise {
	return p.then(OnFulfilled(onCompleted), func(e error) (interface{}, error) {
		return onCompleted(e)
	})
}

func (p *future) WhenComplete(action func()) Promise {
	return p.then(func(v interface{}) (interface{}, error) {
		action()
		return v, nil
	}, func(e error) (interface{}, error) {
		action()
		return nil, e
	})
}

func (p *future) Done(onFulfilled OnFulfilled, onRejected ...OnRejected) {
	p.
		Then(onFulfilled, onRejected...).
		Then(nil, func(e error) (interface{}, error) {
			go panic(e)
			return nil, nil
		})
}

func (p *future) Fail(onRejected OnRejected) {
	p.Done(nil, onRejected)
}

func (p *future) Always(onCompleted OnCompleted) {
	p.Done(OnFulfilled(onCompleted), func(e error) (interface{}, error) {
		return onCompleted(e)
	})
}

func (p *future) State() State {
	return State(p.state)
}

func (p *future) resolveThenable(thenable Thenable) {
	var done uint32
	defer func() {
		if e := recover(); e != nil && atomic.CompareAndSwapUint32(&done, 0, 1) {
			p.Reject(NewPanicError(e))
		}
	}()
	thenable.Then(func(y interface{}) (interface{}, error) {
		if atomic.CompareAndSwapUint32(&done, 0, 1) {
			p.Resolve(y)
		}
		return nil, nil
	}, func(e error) (interface{}, error) {
		if atomic.CompareAndSwapUint32(&done, 0, 1) {
			p.Reject(e)
		}
		return nil, nil
	})
}

func (p *future) reslove(value interface{}) {
	if atomic.CompareAndSwapUint32(&p.state, uint32(PENDING), uint32(FULFILLED)) {
		p.value = value
		subscribers := p.subscribers
		p.subscribers = nil
		for _, subscriber := range subscribers {
			resolve(subscriber.next, subscriber.onFulfilled, value)
		}
	}
}

func (p *future) Resolve(value interface{}) {
	if promise, ok := value.(*future); ok && promise == p {
		p.Reject(TypeError{"Self resolution"})
	} else if promise, ok := value.(Promise); ok {
		promise.Fill(p)
	} else if thenable, ok := value.(Thenable); ok {
		p.resolveThenable(thenable)
	} else {
		p.reslove(value)
	}
}

func (p *future) Reject(reason error) {
	if atomic.CompareAndSwapUint32(&p.state, uint32(PENDING), uint32(REJECTED)) {
		p.reason = reason
		subscribers := p.subscribers
		p.subscribers = nil
		for _, subscriber := range subscribers {
			reject(subscriber.next, subscriber.onRejected, reason)
		}
	}
}

func (p *future) Fill(promise Promise) {
	resolveFunc := func(v interface{}) (interface{}, error) {
		promise.Resolve(v)
		return nil, nil
	}
	rejectFunc := func(e error) (interface{}, error) {
		promise.Reject(e)
		return nil, nil
	}
	p.Then(resolveFunc, rejectFunc)
}

func (p *future) Timeout(duration time.Duration, reason ...error) Promise {
	return timeout(p, duration, reason...)
}

func (p *future) Delay(duration time.Duration) Promise {
	next := New()
	p.then(func(v interface{}) (interface{}, error) {
		go func() {
			time.Sleep(duration)
			next.Resolve(v)
		}()
		return nil, nil
	}, func(e error) (interface{}, error) {
		next.Reject(e)
		return nil, nil
	})
	return next
}

func (p *future) Tap(onfulfilledSideEffect OnfulfilledSideEffect) Promise {
	return tap(p, onfulfilledSideEffect)
}

func (p *future) Get() (interface{}, error) {
	c := make(chan interface{})
	p.then(func(v interface{}) (interface{}, error) {
		c <- v
		return nil, nil
	}, func(e error) (interface{}, error) {
		c <- e
		return nil, nil
	})
	v := <-c
	if e, ok := v.(error); ok {
		return nil, e
	}
	return v, nil
}
<commit_msg>Improved Then method.<commit_after>/**********************************************************\
|                                                          |
|                          hprose                          |
|                                                          |
| Official WebSite: http://www.hprose.com/                 |
|                   http://www.hprose.org/                 |
|                                                          |
\**********************************************************/
/**********************************************************\
 *                                                        *
 * promise/future.go                                      *
 *                                                        *
 * future promise implementation for Go.                  *
 *                                                        *
 * LastModified: Aug 13, 2015                             *
 * Author: Ma Bingyao <andot@hprose.com>                  *
 *                                                        *
\**********************************************************/

package promise

import (
	"sync/atomic"
	"time"
)

type subscriber struct {
	onFulfilled OnFulfilled
	onRejected  OnRejected
	next        Promise
}

type future struct {
	value       interface{}
	reason      error
	state       uint32
	subscribers []subscriber
}

// New creates a PENDING Promise object
func New() Promise {
	return new(future)
}

func (p *future) then(onFulfilled OnFulfilled, onRejected OnRejected) Promise {
	next := New()
	switch State(p.state) {
	case FULFILLED:
		if onFulfilled == nil {
			return fulfilled{p.value}
		}
		resolve(next, onFulfilled, p.value)
	case REJECTED:
		if onRejected == nil {
			return rejected{p.reason}
		}
		reject(next, onRejected, p.reason)
	default:
		p.subscribers = append(p.subscribers,
			subscriber{onFulfilled, onRejected, next})
	}
	return next
}

func (p *future) Then(onFulfilled OnFulfilled, onRejected ...OnRejected) Promise {
	if len(onRejected) == 0 {
		return p.then(onFulfilled, nil)
	}
	return p.then(onFulfilled, onRejected[0])
}

func (p *future) catch(onRejected OnRejected, test TestFunc) Promise {
	if test == nil {
		return p.then(nil, onRejected)
	}
	return p.then(nil, func(e error) (interface{}, error) {
		if test(e) {
			return p.then(nil, onRejected), nil
		}
		return nil, e
	})
}

func (p *future) Catch(onRejected OnRejected, test ...TestFunc) Promise {
	if len(test) == 0 {
		return p.catch(onRejected, nil)
	}
	return p.catch(onRejected, test[0])
}

func (p *future) Complete(onCompleted OnCompleted) Promise {
	return p.then(OnFulfilled(onCompleted), func(e error) (interface{}, error) {
		return onCompleted(e)
	})
}

func (p *future) WhenComplete(action func()) Promise {
	return p.then(func(v interface{}) (interface{}, error) {
		action()
		return v, nil
	}, func(e error) (interface{}, error) {
		action()
		return nil, e
	})
}

func (p *future) Done(onFulfilled OnFulfilled, onRejected ...OnRejected) {
	p.
		Then(onFulfilled, onRejected...).
		Then(nil, func(e error) (interface{}, error) {
			go panic(e)
			return nil, nil
		})
}

func (p *future) Fail(onRejected OnRejected) {
	p.Done(nil, onRejected)
}

func (p *future) Always(onCompleted OnCompleted) {
	p.Done(OnFulfilled(onCompleted), func(e error) (interface{}, error) {
		return onCompleted(e)
	})
}

func (p *future) State() State {
	return State(p.state)
}

func (p *future) resolveThenable(thenable Thenable) {
	var done uint32
	defer func() {
		if e := recover(); e != nil && atomic.CompareAndSwapUint32(&done, 0, 1) {
			p.Reject(NewPanicError(e))
		}
	}()
	thenable.Then(func(y interface{}) (interface{}, error) {
		if atomic.CompareAndSwapUint32(&done, 0, 1) {
			p.Resolve(y)
		}
		return nil, nil
	}, func(e error) (interface{}, error) {
		if atomic.CompareAndSwapUint32(&done, 0, 1) {
			p.Reject(e)
		}
		return nil, nil
	})
}

func (p *future) reslove(value interface{}) {
	if atomic.CompareAndSwapUint32(&p.state, uint32(PENDING), uint32(FULFILLED)) {
		p.value = value
		subscribers := p.subscribers
		p.subscribers = nil
		for _, subscriber := range subscribers {
			resolve(subscriber.next, subscriber.onFulfilled, value)
		}
	}
}

func (p *future) Resolve(value interface{}) {
	if promise, ok := value.(*future); ok && promise == p {
		p.Reject(TypeError{"Self resolution"})
	} else if promise, ok := value.(Promise); ok {
		promise.Fill(p)
	} else if thenable, ok := value.(Thenable); ok {
		p.resolveThenable(thenable)
	} else {
		p.reslove(value)
	}
}

func (p *future) Reject(reason error) {
	if atomic.CompareAndSwapUint32(&p.state, uint32(PENDING), uint32(REJECTED)) {
		p.reason = reason
		subscribers := p.subscribers
		p.subscribers = nil
		for _, subscriber := range subscribers {
			reject(subscriber.next, subscriber.onRejected, reason)
		}
	}
}

func (p *future) Fill(promise Promise) {
	resolveFunc := func(v interface{}) (interface{}, error) {
		promise.Resolve(v)
		return nil, nil
	}
	rejectFunc := func(e error) (interface{}, error) {
		promise.Reject(e)
		return nil, nil
	}
	p.Then(resolveFunc, rejectFunc)
}

func (p *future) Timeout(duration time.Duration, reason ...error) Promise {
	return timeout(p, duration, reason...)
}

func (p *future) Delay(duration time.Duration) Promise {
	next := New()
	p.then(func(v interface{}) (interface{}, error) {
		go func() {
			time.Sleep(duration)
			next.Resolve(v)
		}()
		return nil, nil
	}, func(e error) (interface{}, error) {
		next.Reject(e)
		return nil, nil
	})
	return next
}

func (p *future) Tap(onfulfilledSideEffect OnfulfilledSideEffect) Promise {
	return tap(p, onfulfilledSideEffect)
}

func (p *future) Get() (interface{}, error) {
	c := make(chan interface{})
	p.then(func(v interface{}) (interface{}, error) {
		c <- v
		return nil, nil
	}, func(e error) (interface{}, error) {
		c <- e
		return nil, nil
	})
	v := <-c
	if e, ok := v.(error); ok {
		return nil, e
	}
	return v, nil
}
<|endoftext|>


<commit_before>package dashing

import (
	"encoding/json"
	"fmt"
	"log"
	"net/http"
	"os"
	"path/filepath"
	"time"

	"github.com/go-martini/martini"
	"github.com/karlseguin/gerb"
	"github.com/martini-contrib/encoder"
)

type dashingServer struct {
	martini *martini.Martini
	broker  *Broker
}

func NewServer() *dashingServer {
	s := dashingServer{
		martini: martini.New(),
		broker:  NewBroker(),
	}

	s.initDashing()

	return &s
}

func (s *dashingServer) Start() {
	// Start the event broker
	s.broker.Start()

	// Start the jobs
	for _, j := range registry {
		go j.Work(s.broker.events)
	}

	// Start Martini
	s.martini.Run()
}

func (s *dashingServer) initDashing() {
	// Setup middleware
	s.martini.Use(martini.Recovery())
	s.martini.Use(martini.Logger())
	s.martini.Use(martini.Static("public"))

	// Setup encoder
	s.martini.Use(func(c martini.Context, w http.ResponseWriter) {
		c.MapTo(encoder.JsonEncoder{}, (*encoder.Encoder)(nil))
		w.Header().Set("Content-Type", "application/json; charset=utf-8")
	})

	// Setup and inject event broker
	s.martini.Map(s.broker)

	// Setup routes
	r := martini.NewRouter()

	r.Get("/", func(w http.ResponseWriter, r *http.Request) {
		files, _ := filepath.Glob("dashboards/*.gerb")

		for _, file := range files {
			dashboard := file[11 : len(file)-5]
			if dashboard != "layout" {
				http.Redirect(w, r, "/"+dashboard, http.StatusTemporaryRedirect)
				return
			}
		}

		http.NotFound(w, r)
		return
	})

	r.Get("/events", func(w http.ResponseWriter, r *http.Request, e encoder.Encoder, b *Broker) {
		f, ok := w.(http.Flusher)
		if !ok {
			http.Error(w, "Streaming unsupported!", http.StatusInternalServerError)
			return
		}

		c, ok := w.(http.CloseNotifier)
		if !ok {
			http.Error(w, "Close notification unsupported!", http.StatusInternalServerError)
			return
		}

		// Create a new channel, over which the broker can
		// send this client events.
		events := make(chan *Event)

		// Add this client to the map of those that should
		// receive updates
		b.newClients <- events

		// Remove this client from the map of attached clients
		// when the handler exits.
		defer func() {
			b.defunctClients <- events
		}()

		w.Header().Set("Content-Type", "text/event-stream")
		w.Header().Set("Cache-Control", "no-cache")
		w.Header().Set("Connection", "keep-alive")
		w.Header().Set("X-Accel-Buffering", "no")
		closer := c.CloseNotify()

		for {
			select {
			case event := <-events:
				data := event.Body
				data["id"] = event.ID
				data["updatedAt"] = int32(time.Now().Unix())
				if event.Target != "" {
					fmt.Fprintf(w, "event: %s\n", event.Target)
				}
				fmt.Fprintf(w, "data: %s\n\n", encoder.Must(e.Encode(data)))
				f.Flush()
			case <-closer:
				log.Println("Closing connection")
				return
			}
		}
	})

	r.Get("/:dashboard", func(r *http.Request, w http.ResponseWriter, params martini.Params) {
		template, err := gerb.ParseFile(true, "dashboards/"+params["dashboard"]+".gerb", "dashboards/layout.gerb")

		if err != nil {
			http.NotFound(w, r)
			return
		}

		w.Header().Set("Content-Type", "text/html; charset=UTF-8")

		template.Render(w, map[string]interface{}{
			"dashboard":   params["dashboard"],
			"development": os.Getenv("DEV") != "",
			"request":     r,
		})
		return
	})

	r.Post("/dashboards/:id", func(r *http.Request, params martini.Params, b *Broker) (int, string) {
		if r.Body != nil {
			defer r.Body.Close()
		}

		var data map[string]interface{}

		if err := json.NewDecoder(r.Body).Decode(&data); err != nil {
			return 400, ""
		}

		b.events <- &Event{params["id"], data, "dashboards"}
		return 204, ""
	})

	r.Post("/widgets/:id", func(r *http.Request, params martini.Params, b *Broker) (int, string) {
		if r.Body != nil {
			defer r.Body.Close()
		}

		var data map[string]interface{}

		if err := json.NewDecoder(r.Body).Decode(&data); err != nil {
			return 400, ""
		}

		b.events <- &Event{params["id"], data, ""}
		return 204, ""
	})

	r.Get("/views/:widget.html", func(w http.ResponseWriter, r *http.Request, params martini.Params) {
		template, err := gerb.ParseFile(true, "widgets/"+params["widget"]+"/"+params["widget"]+".html")

		if err != nil {
			http.NotFound(w, r)
			return
		}

		w.Header().Set("Content-Type", "text/html; charset=UTF-8")

		template.Render(w, nil)
		return
	})

	// Add the router action
	s.martini.Action(r.Handle)
}

// Start all jobs and listen to requests.
func Start() {
	server := NewServer()

	server.Start()
}
<commit_msg>Add staticDirectory parameter to specify static assets directory<commit_after>package dashing

import (
	"encoding/json"
	"fmt"
	"log"
	"net/http"
	"os"
	"path/filepath"
	"time"

	"github.com/go-martini/martini"
	"github.com/karlseguin/gerb"
	"github.com/martini-contrib/encoder"
)

type dashingServer struct {
	martini *martini.Martini
	broker  *Broker

	staticDirectory string
}

func NewServer(staticDirectory string) *dashingServer {
	s := dashingServer{
		martini:         martini.New(),
		broker:          NewBroker(),
		staticDirectory: staticDirectory,
	}

	s.initDashing()

	return &s
}

func (s *dashingServer) Start() {
	// Start the event broker
	s.broker.Start()

	// Start the jobs
	for _, j := range registry {
		go j.Work(s.broker.events)
	}

	// Start Martini
	s.martini.Run()
}

func (s *dashingServer) initDashing() {
	// Setup middleware
	s.martini.Use(martini.Recovery())
	s.martini.Use(martini.Logger())
	s.martini.Use(martini.Static(s.staticDirectory))

	// Setup encoder
	s.martini.Use(func(c martini.Context, w http.ResponseWriter) {
		c.MapTo(encoder.JsonEncoder{}, (*encoder.Encoder)(nil))
		w.Header().Set("Content-Type", "application/json; charset=utf-8")
	})

	// Setup and inject event broker
	s.martini.Map(s.broker)

	// Setup routes
	r := martini.NewRouter()

	r.Get("/", func(w http.ResponseWriter, r *http.Request) {
		files, _ := filepath.Glob("dashboards/*.gerb")

		for _, file := range files {
			dashboard := file[11 : len(file)-5]
			if dashboard != "layout" {
				http.Redirect(w, r, "/"+dashboard, http.StatusTemporaryRedirect)
				return
			}
		}

		http.NotFound(w, r)
		return
	})

	r.Get("/events", func(w http.ResponseWriter, r *http.Request, e encoder.Encoder, b *Broker) {
		f, ok := w.(http.Flusher)
		if !ok {
			http.Error(w, "Streaming unsupported!", http.StatusInternalServerError)
			return
		}

		c, ok := w.(http.CloseNotifier)
		if !ok {
			http.Error(w, "Close notification unsupported!", http.StatusInternalServerError)
			return
		}

		// Create a new channel, over which the broker can
		// send this client events.
		events := make(chan *Event)

		// Add this client to the map of those that should
		// receive updates
		b.newClients <- events

		// Remove this client from the map of attached clients
		// when the handler exits.
		defer func() {
			b.defunctClients <- events
		}()

		w.Header().Set("Content-Type", "text/event-stream")
		w.Header().Set("Cache-Control", "no-cache")
		w.Header().Set("Connection", "keep-alive")
		w.Header().Set("X-Accel-Buffering", "no")
		closer := c.CloseNotify()

		for {
			select {
			case event := <-events:
				data := event.Body
				data["id"] = event.ID
				data["updatedAt"] = int32(time.Now().Unix())
				if event.Target != "" {
					fmt.Fprintf(w, "event: %s\n", event.Target)
				}
				fmt.Fprintf(w, "data: %s\n\n", encoder.Must(e.Encode(data)))
				f.Flush()
			case <-closer:
				log.Println("Closing connection")
				return
			}
		}
	})

	r.Get("/:dashboard", func(r *http.Request, w http.ResponseWriter, params martini.Params) {
		template, err := gerb.ParseFile(true, "dashboards/"+params["dashboard"]+".gerb", "dashboards/layout.gerb")

		if err != nil {
			http.NotFound(w, r)
			return
		}

		w.Header().Set("Content-Type", "text/html; charset=UTF-8")

		template.Render(w, map[string]interface{}{
			"dashboard":   params["dashboard"],
			"development": os.Getenv("DEV") != "",
			"request":     r,
		})
		return
	})

	r.Post("/dashboards/:id", func(r *http.Request, params martini.Params, b *Broker) (int, string) {
		if r.Body != nil {
			defer r.Body.Close()
		}

		var data map[string]interface{}

		if err := json.NewDecoder(r.Body).Decode(&data); err != nil {
			return 400, ""
		}

		b.events <- &Event{params["id"], data, "dashboards"}
		return 204, ""
	})

	r.Post("/widgets/:id", func(r *http.Request, params martini.Params, b *Broker) (int, string) {
		if r.Body != nil {
			defer r.Body.Close()
		}

		var data map[string]interface{}

		if err := json.NewDecoder(r.Body).Decode(&data); err != nil {
			return 400, ""
		}

		b.events <- &Event{params["id"], data, ""}
		return 204, ""
	})

	r.Get("/views/:widget.html", func(w http.ResponseWriter, r *http.Request, params martini.Params) {
		template, err := gerb.ParseFile(true, "widgets/"+params["widget"]+"/"+params["widget"]+".html")

		if err != nil {
			http.NotFound(w, r)
			return
		}

		w.Header().Set("Content-Type", "text/html; charset=UTF-8")

		template.Render(w, nil)
		return
	})

	// Add the router action
	s.martini.Action(r.Handle)
}

// Start all jobs and listen to requests.
func Start() {
	StartWithStaticDirectory("public")
}

func StartWithStaticDirectory(staticDirectory string) {
	server := NewServer(staticDirectory)

	server.Start()
}
<|endoftext|>


<commit_before>// Package prosper is a set of APIs that wrap the raw HTTP Prosper REST APIs.

package prosper

import (
	"github.com/mtlynch/gofn-prosper/prosper/auth"
	"github.com/mtlynch/gofn-prosper/prosper/thin"
	"github.com/mtlynch/gofn-prosper/types"
)

// Client is a Prosper client that communicates with the Prosper HTTP endpoints.
type Client struct {
	rawClient     thin.RawApiHandler
	ap            accountsParser
	nrp           notesResponseParser
	listingParser listingParser
	orderParser   orderParser
}

// NewClient creates a new Client with the given Prosper credentials.
func NewClient(creds types.ClientCredentials) Client {
	tokenMgr := auth.NewTokenManager(auth.NewAuthenticator(creds))
	return Client{
		rawClient:     thin.NewClient(tokenMgr),
		ap:            defaultAccountsParser{},
		nrp:           NewNotesResponseParser(),
		listingParser: defaultListingParser{},
		orderParser:   defaultOrderParser{},
	}
}
<commit_msg>Fixing blank line that breaks prosper package documentation (#35)<commit_after>// Package prosper is a set of APIs that wrap the raw HTTP Prosper REST APIs.
package prosper

import (
	"github.com/mtlynch/gofn-prosper/prosper/auth"
	"github.com/mtlynch/gofn-prosper/prosper/thin"
	"github.com/mtlynch/gofn-prosper/types"
)

// Client is a Prosper client that communicates with the Prosper HTTP endpoints.
type Client struct {
	rawClient     thin.RawApiHandler
	ap            accountsParser
	nrp           notesResponseParser
	listingParser listingParser
	orderParser   orderParser
}

// NewClient creates a new Client with the given Prosper credentials.
func NewClient(creds types.ClientCredentials) Client {
	tokenMgr := auth.NewTokenManager(auth.NewAuthenticator(creds))
	return Client{
		rawClient:     thin.NewClient(tokenMgr),
		ap:            defaultAccountsParser{},
		nrp:           NewNotesResponseParser(),
		listingParser: defaultListingParser{},
		orderParser:   defaultOrderParser{},
	}
}
<|endoftext|>


<commit_before>package protocol

import (
	log "github.com/sirupsen/logrus"

	"github.com/bytom/errors"
	"github.com/bytom/protocol/bc"
	"github.com/bytom/protocol/bc/legacy"
	"github.com/bytom/protocol/state"
	"github.com/bytom/protocol/validation"
)

var (
	// ErrBadBlock is returned when a block is invalid.
	ErrBadBlock = errors.New("invalid block")

	// ErrStaleState is returned when the Chain does not have a current
	// blockchain state.
	ErrStaleState = errors.New("stale blockchain state")

	// ErrBadStateRoot is returned when the computed assets merkle root
	// disagrees with the one declared in a block header.
	ErrBadStateRoot = errors.New("invalid state merkle root")
)

func (c *Chain) BlockExist(hash *bc.Hash) bool {
	return c.orphanManage.BlockExist(hash) || c.store.BlockExist(hash)
}

func (c *Chain) GetBlockByHash(hash *bc.Hash) (*legacy.Block, error) {
	return c.store.GetBlock(hash)
}

func (c *Chain) GetBlockByHeight(height uint64) (*legacy.Block, error) {
	c.state.cond.L.Lock()
	hash, ok := c.state.mainChain[height]
	c.state.cond.L.Unlock()
	if !ok {
		return nil, errors.New("can't find block in given hight")
	}
	return c.GetBlockByHash(hash)
}

// ValidateBlock validates an incoming block in advance of applying it
// to a snapshot (with ApplyValidBlock) and committing it to the
// blockchain (with CommitAppliedBlock).
func (c *Chain) ValidateBlock(block, prev *legacy.Block) error {
	blockEnts := legacy.MapBlock(block)
	prevEnts := legacy.MapBlock(prev)
	if err := validation.ValidateBlock(blockEnts, prevEnts); err != nil {
		return errors.Sub(ErrBadBlock, err)
	}
	return nil
}

// ApplyValidBlock creates an updated snapshot without validating the
// block.
func (c *Chain) ConnectBlock(block *legacy.Block) error {
	newSnapshot := state.Copy(c.state.snapshot)
	if err := newSnapshot.ApplyBlock(legacy.MapBlock(block)); err != nil {
		return err
	}

	blockHash := block.Hash()
	if err := c.setState(block, newSnapshot, map[uint64]*bc.Hash{block.Height: &blockHash}); err != nil {
		return err
	}

	for _, tx := range block.Transactions {
		c.txPool.RemoveTransaction(&tx.Tx.ID)
	}
	return nil
}

func (c *Chain) getReorganizeBlocks(block *legacy.Block) ([]*legacy.Block, []*legacy.Block) {
	attachBlocks := []*legacy.Block{}
	detachBlocks := []*legacy.Block{}
	ancestor := block

	for !c.InMainchain(ancestor) {
		attachBlocks = append([]*legacy.Block{ancestor}, attachBlocks...)
		ancestor, _ = c.GetBlockByHash(&ancestor.PreviousBlockHash)
	}

	for d := c.state.block; d.Hash() != ancestor.Hash(); d, _ = c.GetBlockByHash(&d.PreviousBlockHash) {
		detachBlocks = append(detachBlocks, d)
	}

	return attachBlocks, detachBlocks
}

func (c *Chain) reorganizeChain(block *legacy.Block) error {
	attachBlocks, detachBlocks := c.getReorganizeBlocks(block)
	newSnapshot := state.Copy(c.state.snapshot)
	chainChanges := map[uint64]*bc.Hash{}

	for _, d := range detachBlocks {
		if err := newSnapshot.DetachBlock(legacy.MapBlock(d)); err != nil {
			return err
		}
	}

	for _, a := range attachBlocks {
		if err := newSnapshot.ApplyBlock(legacy.MapBlock(a)); err != nil {
			return err
		}
		aHash := a.Hash()
		chainChanges[a.Height] = &aHash
	}

	return c.setState(block, newSnapshot, chainChanges)
}

func (c *Chain) SaveBlock(block *legacy.Block) error {
	preBlock, _ := c.GetBlockByHash(&block.PreviousBlockHash)
	if err := c.ValidateBlock(block, preBlock); err != nil {
		return err
	}
	if err := c.store.SaveBlock(block); err != nil {
		return err
	}

	preorphans, ok := c.orphanManage.preOrphans[block.Hash()]
	if !ok {
		return nil
	}
	for _, preorphan := range preorphans {
		orphanBlock, ok := c.orphanManage.Get(preorphan)
		if !ok {
			continue
		}
		c.SaveBlock(orphanBlock)
		c.orphanManage.Delete(preorphan)
	}
	return nil
}

func (c *Chain) ProcessBlock(block *legacy.Block) (bool, error) {
	blockHash := block.Hash()
	if c.BlockExist(&blockHash) {
		log.WithFields(log.Fields{"hash": blockHash.String()}).Info("Skip process due to block already been handled")
		return false, nil
	}
	if !c.BlockExist(&block.PreviousBlockHash) {
		log.WithFields(log.Fields{"hash": blockHash.String()}).Info("Add to orphan block setg")
		c.orphanManage.Add(block)
		return true, nil
	}
	if err := c.SaveBlock(block); err != nil {
		return false, err
	}

	c.state.cond.L.Lock()
	if c.state.block.Hash() == block.PreviousBlockHash {
		defer c.state.cond.L.Unlock()
		return false, c.ConnectBlock(block)
	}

	if block.Height > c.state.height && block.Bits >= c.state.block.Bits {
		defer c.state.cond.L.Unlock()
		return false, c.reorganizeChain(block)
	}
	c.state.cond.L.Unlock()
	return false, nil
}
<commit_msg>update log format<commit_after>package protocol

import (
	log "github.com/sirupsen/logrus"

	"github.com/bytom/errors"
	"github.com/bytom/protocol/bc"
	"github.com/bytom/protocol/bc/legacy"
	"github.com/bytom/protocol/state"
	"github.com/bytom/protocol/validation"
)

var (
	// ErrBadBlock is returned when a block is invalid.
	ErrBadBlock = errors.New("invalid block")

	// ErrStaleState is returned when the Chain does not have a current
	// blockchain state.
	ErrStaleState = errors.New("stale blockchain state")

	// ErrBadStateRoot is returned when the computed assets merkle root
	// disagrees with the one declared in a block header.
	ErrBadStateRoot = errors.New("invalid state merkle root")
)

func (c *Chain) BlockExist(hash *bc.Hash) bool {
	return c.orphanManage.BlockExist(hash) || c.store.BlockExist(hash)
}

func (c *Chain) GetBlockByHash(hash *bc.Hash) (*legacy.Block, error) {
	return c.store.GetBlock(hash)
}

func (c *Chain) GetBlockByHeight(height uint64) (*legacy.Block, error) {
	c.state.cond.L.Lock()
	hash, ok := c.state.mainChain[height]
	c.state.cond.L.Unlock()
	if !ok {
		return nil, errors.New("can't find block in given hight")
	}
	return c.GetBlockByHash(hash)
}

// ValidateBlock validates an incoming block in advance of applying it
// to a snapshot (with ApplyValidBlock) and committing it to the
// blockchain (with CommitAppliedBlock).
func (c *Chain) ValidateBlock(block, prev *legacy.Block) error {
	blockEnts := legacy.MapBlock(block)
	prevEnts := legacy.MapBlock(prev)
	if err := validation.ValidateBlock(blockEnts, prevEnts); err != nil {
		return errors.Sub(ErrBadBlock, err)
	}
	return nil
}

// ApplyValidBlock creates an updated snapshot without validating the
// block.
func (c *Chain) ConnectBlock(block *legacy.Block) error {
	newSnapshot := state.Copy(c.state.snapshot)
	if err := newSnapshot.ApplyBlock(legacy.MapBlock(block)); err != nil {
		return err
	}

	blockHash := block.Hash()
	if err := c.setState(block, newSnapshot, map[uint64]*bc.Hash{block.Height: &blockHash}); err != nil {
		return err
	}

	for _, tx := range block.Transactions {
		c.txPool.RemoveTransaction(&tx.Tx.ID)
	}
	return nil
}

func (c *Chain) getReorganizeBlocks(block *legacy.Block) ([]*legacy.Block, []*legacy.Block) {
	attachBlocks := []*legacy.Block{}
	detachBlocks := []*legacy.Block{}
	ancestor := block

	for !c.InMainchain(ancestor) {
		attachBlocks = append([]*legacy.Block{ancestor}, attachBlocks...)
		ancestor, _ = c.GetBlockByHash(&ancestor.PreviousBlockHash)
	}

	for d := c.state.block; d.Hash() != ancestor.Hash(); d, _ = c.GetBlockByHash(&d.PreviousBlockHash) {
		detachBlocks = append(detachBlocks, d)
	}

	return attachBlocks, detachBlocks
}

func (c *Chain) reorganizeChain(block *legacy.Block) error {
	attachBlocks, detachBlocks := c.getReorganizeBlocks(block)
	newSnapshot := state.Copy(c.state.snapshot)
	chainChanges := map[uint64]*bc.Hash{}

	for _, d := range detachBlocks {
		if err := newSnapshot.DetachBlock(legacy.MapBlock(d)); err != nil {
			return err
		}
	}

	for _, a := range attachBlocks {
		if err := newSnapshot.ApplyBlock(legacy.MapBlock(a)); err != nil {
			return err
		}
		aHash := a.Hash()
		chainChanges[a.Height] = &aHash
	}

	return c.setState(block, newSnapshot, chainChanges)
}

func (c *Chain) SaveBlock(block *legacy.Block) error {
	preBlock, _ := c.GetBlockByHash(&block.PreviousBlockHash)
	if err := c.ValidateBlock(block, preBlock); err != nil {
		return err
	}
	if err := c.store.SaveBlock(block); err != nil {
		return err
	}

	preorphans, ok := c.orphanManage.preOrphans[block.Hash()]
	if !ok {
		return nil
	}
	for _, preorphan := range preorphans {
		orphanBlock, ok := c.orphanManage.Get(preorphan)
		if !ok {
			continue
		}
		c.SaveBlock(orphanBlock)
		c.orphanManage.Delete(preorphan)
	}
	return nil
}

func (c *Chain) ProcessBlock(block *legacy.Block) (bool, error) {
	blockHash := block.Hash()
	if c.BlockExist(&blockHash) {
		log.WithField("hash", blockHash.String()).Info("Skip process due to block already been handled")
		return false, nil
	}
	if !c.BlockExist(&block.PreviousBlockHash) {
		log.WithField("hash", blockHash.String()).Info("Add to orphan block setg")
		c.orphanManage.Add(block)
		return true, nil
	}
	if err := c.SaveBlock(block); err != nil {
		return false, err
	}

	c.state.cond.L.Lock()
	if c.state.block.Hash() == block.PreviousBlockHash {
		defer c.state.cond.L.Unlock()
		return false, c.ConnectBlock(block)
	}

	if block.Height > c.state.height && block.Bits >= c.state.block.Bits {
		defer c.state.cond.L.Unlock()
		return false, c.reorganizeChain(block)
	}
	c.state.cond.L.Unlock()
	return false, nil
}
<|endoftext|>


<commit_before>/*
 *  Copyright 2016 Adobe Systems Incorporated. All rights reserved.
 *  This file is licensed to you under the Apache License, Version 2.0 (the "License");
 *  you may not use this file except in compliance with the License. You may obtain a copy
 *  of the License at http://www.apache.org/licenses/LICENSE-2.0
 *
 *  Unless required by applicable law or agreed to in writing, software distributed under
 *  the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR REPRESENTATIONS
 *  OF ANY KIND, either express or implied. See the License for the specific language
 *  governing permissions and limitations under the License.
 */
package provision

import (
	"crypto/md5"
	"encoding/hex"
	"fmt"
	yaml "gopkg.in/yaml.v2"
	"io/ioutil"
	"os"
	"os/exec"
	"path"
	"strings"
	"time"

	"github.com/adobe-platform/porter/conf"
	"github.com/adobe-platform/porter/constants"
	"github.com/inconshreveable/log15"
)

// Package creates the service payload to deliver to S3
func Package(log log15.Logger, config *conf.Config) (success bool) {

	// clean up old artifacts before building
	exec.Command("rm", "-rf", constants.PayloadWorkingDir).Run()

	// clean up artifacts after building
	defer exec.Command("rm", "-rf", constants.PayloadWorkingDir).Run()

	exec.Command("mkdir", "-p", constants.PayloadWorkingDir).Run()

	revParseOutput, err := exec.Command("git", "rev-parse", "--short", "HEAD").Output()
	if err != nil {
		log.Error("git rev-parse", "Error", err)
		return
	}

	now := time.Now().Unix()
	config.ServiceVersion = strings.TrimSpace(string(revParseOutput))

	dockerRegistry := os.Getenv(constants.EnvDockerRegistry)
	dockerRepository := os.Getenv(constants.EnvDockerRepository)
	dockerPushUsername := os.Getenv(constants.EnvDockerPushUsername)
	dockerPushPassword := os.Getenv(constants.EnvDockerPushPassword)

	if dockerRegistry != "" && dockerPushUsername != "" && dockerPushPassword != "" {

		log.Info("docker login")
		loginCmd := exec.Command("docker", "login",
			"-u", dockerPushUsername,
			"-p", dockerPushPassword,
			dockerRegistry)
		loginCmd.Stdout = os.Stdout
		loginCmd.Stderr = os.Stderr
		err := loginCmd.Run()
		if err != nil {
			log.Error("docker login", "Error", err)
			return
		}
	}

	uniqueContainers := make(map[string]*conf.Container)

	// This is in a loop but assumes we're building a single container
	// TODO support multiple containers
	for _, environment := range config.Environments {

		for _, region := range environment.Regions {

			for _, container := range region.Containers {

				container.OriginalName = container.Name

				// Alter the name in the config so we know which image names are part
				// of the service payload. This is important for hotswap to know which
				// of the available images on the host are the ones to be swapped in.
				if dockerRegistry == "" && dockerRepository == "" {

					container.Name = fmt.Sprintf("s3/s3:porter-%s-%d-%s",
						config.ServiceVersion, now, container.Name)
				} else {

					container.Name = fmt.Sprintf("%s/%s:porter-%s-%d-%s",
						dockerRegistry, dockerRepository,
						config.ServiceVersion, now, container.Name)
				}

				// a unique container is the combination of its name and
				// Dockerfiles used to build it
				uid := container.Name + container.Dockerfile + container.DockerfileBuild

				if _, exists := uniqueContainers[uid]; !exists {

					uniqueContainers[uid] = container
				}
			}
		}
	}

	successChan := make(chan bool)

	for _, container := range uniqueContainers {

		go func(container *conf.Container) {

			successChan <- buildContainer(log, container.Name,
				container.Dockerfile, container.DockerfileBuild)

		}(container)
	}

	for i := 0; i < len(uniqueContainers); i++ {
		success = <-successChan
		if !success {
			return
		}
	}

	if !copyPathBasedFiles(log, config) {
		return
	}

	configBytes, err := yaml.Marshal(config)
	if err != nil {
		return
	}

	// for later build stages
	err = ioutil.WriteFile(constants.AlteredConfigPath, configBytes, 0644)
	if err != nil {
		log.Error("WriteFile", "Path", constants.AlteredConfigPath)
		return
	}

	// for the service payload about to be created
	err = ioutil.WriteFile(constants.PackPayloadConfigPath, configBytes, 0644)
	if err != nil {
		log.Error("WriteFile", "Path", constants.PackPayloadConfigPath)
		return
	}

	log.Info(fmt.Sprintf("creating service payload at %s", constants.PayloadPath))

	tarCmd := exec.Command("tar", "-C", constants.PayloadWorkingDir, "-czf", constants.PayloadPath, ".")
	tarCmd.Stdout = os.Stdout
	tarCmd.Stderr = os.Stderr
	err = tarCmd.Run()
	if err != nil {
		log.Error("tar", "Error", err)
		return
	}

	success = true
	return
}

func buildContainer(log log15.Logger, containerName, dockerfile, dockerfileBuild string) (success bool) {

	log = log.New("ImageTag", containerName)

	imagePath := fmt.Sprintf("%s/%s.docker", constants.PayloadWorkingDir, containerName)

	_, err := os.Stat(dockerfile)
	if err != nil {
		log.Error("Dockerfile stat", "Error", err)
		return
	}

	haveBuilder := true
	_, err = os.Stat(dockerfileBuild)
	if err != nil {
		haveBuilder = false
	}

	if haveBuilder {
		var err error

		buildBuilderCmd := exec.Command("docker", "build", "-t", containerName+"-builder", "-f", dockerfileBuild, ".")
		buildBuilderCmd.Stdout = os.Stdout
		buildBuilderCmd.Stderr = os.Stderr
		err = buildBuilderCmd.Run()
		if err != nil {
			log.Error("build Dockerfile.build", "Error", err)
			return
		}

		runCmd := exec.Command("docker", "run", "--rm", containerName+"-builder")

		runCmdStdoutPipe, err := runCmd.StdoutPipe()
		if err != nil {
			log.Error("couldn't create StdoutPipe", "Error", err)
			return
		}

		buildCmd := exec.Command("docker", "build",
			"-t", containerName,
			"-f", dockerfile,
			"-")
		buildCmd.Stdin = runCmdStdoutPipe
		buildCmd.Stdout = os.Stdout
		buildCmd.Stderr = os.Stderr

		err = runCmd.Start()
		if err != nil {
			log.Error("docker run", "Error", err)
			return
		}

		err = buildCmd.Start()
		if err != nil {
			log.Error("build Dockerfile", "Error", err)
			return
		}

		runCmd.Wait()
		buildCmd.Wait()
	} else {
		buildCmd := exec.Command("docker", "build",
			"-t", containerName,
			"-f", dockerfile,
			".")
		buildCmd.Stdout = os.Stdout
		buildCmd.Stderr = os.Stderr
		err := buildCmd.Run()
		if err != nil {
			log.Error("build Dockerfile", "Error", err)
			return
		}
	}

	dockerRegistry := os.Getenv(constants.EnvDockerRegistry)

	if dockerRegistry == "" {
		log.Info("saving docker image to " + imagePath)

		exec.Command("mkdir", "-p", path.Dir(imagePath)).Run()

		saveCmd := exec.Command("docker", "save", "-o", imagePath, containerName)
		saveCmd.Stdout = os.Stdout
		saveCmd.Stderr = os.Stderr
		err = saveCmd.Run()
		if err != nil {
			log.Error("docker save", "Error", err)
			return
		}

	} else {

		log.Info("docker push")

		pushCmd := exec.Command("docker", "push", containerName)
		pushCmd.Stdout = os.Stdout
		pushCmd.Stderr = os.Stderr
		err := pushCmd.Run()
		if err != nil {
			log.Error("docker push", "Error", err)
			return
		}
	}

	success = true
	return
}

// Ensure the files that are specified with paths in the config are part of the
// temp directory which is passed between the pack and provision stages in GoCD.
// If we fetched materials in every stage then the referenced files would always
// be there, and this function wouldn't be strictly necessary.
func copyPathBasedFiles(log log15.Logger, config *conf.Config) bool {
	for _, environment := range config.Environments {
		if digest, success := digestAndCopy(log, environment.StackDefinitionPath); success {
			environment.StackDefinitionPath = digest
		} else {
			return false
		}

		for _, region := range environment.Regions {
			if digest, success := digestAndCopy(log, region.StackDefinitionPath); success {
				region.StackDefinitionPath = digest
			} else {
				return false
			}
		}
	}

	return true
}

func digestAndCopy(log log15.Logger, filePath string) (string, bool) {
	if filePath == "" {
		return "", true
	}

	log = log.New("Filepath", filePath)

	fileBytes, err := ioutil.ReadFile(filePath)
	if err != nil {
		log.Error("ioutil.ReadFile", "Error", err)
		return "", false
	}
	digestArray := md5.Sum(fileBytes)
	digest := hex.EncodeToString(digestArray[:])

	newFilePath := constants.TempDir + "/" + digest
	err = ioutil.WriteFile(newFilePath, fileBytes, 0644)
	if err != nil {
		log.Error("ioutil.ReadFile", "Error", err)
		return "", false
	}

	return newFilePath, true
}
<commit_msg>serial docker save<commit_after>/*
 *  Copyright 2016 Adobe Systems Incorporated. All rights reserved.
 *  This file is licensed to you under the Apache License, Version 2.0 (the "License");
 *  you may not use this file except in compliance with the License. You may obtain a copy
 *  of the License at http://www.apache.org/licenses/LICENSE-2.0
 *
 *  Unless required by applicable law or agreed to in writing, software distributed under
 *  the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR REPRESENTATIONS
 *  OF ANY KIND, either express or implied. See the License for the specific language
 *  governing permissions and limitations under the License.
 */
package provision

import (
	"crypto/md5"
	"encoding/hex"
	"fmt"
	yaml "gopkg.in/yaml.v2"
	"io/ioutil"
	"os"
	"os/exec"
	"path"
	"strings"
	"sync"
	"time"

	"github.com/adobe-platform/porter/conf"
	"github.com/adobe-platform/porter/constants"
	"github.com/inconshreveable/log15"
)

var dockerSaveLock sync.Mutex

// Package creates the service payload to deliver to S3
func Package(log log15.Logger, config *conf.Config) (success bool) {

	// clean up old artifacts before building
	exec.Command("rm", "-rf", constants.PayloadWorkingDir).Run()

	// clean up artifacts after building
	defer exec.Command("rm", "-rf", constants.PayloadWorkingDir).Run()

	exec.Command("mkdir", "-p", constants.PayloadWorkingDir).Run()

	revParseOutput, err := exec.Command("git", "rev-parse", "--short", "HEAD").Output()
	if err != nil {
		log.Error("git rev-parse", "Error", err)
		return
	}

	now := time.Now().Unix()
	config.ServiceVersion = strings.TrimSpace(string(revParseOutput))

	dockerRegistry := os.Getenv(constants.EnvDockerRegistry)
	dockerRepository := os.Getenv(constants.EnvDockerRepository)
	dockerPushUsername := os.Getenv(constants.EnvDockerPushUsername)
	dockerPushPassword := os.Getenv(constants.EnvDockerPushPassword)

	if dockerRegistry != "" && dockerPushUsername != "" && dockerPushPassword != "" {

		log.Info("docker login")
		loginCmd := exec.Command("docker", "login",
			"-u", dockerPushUsername,
			"-p", dockerPushPassword,
			dockerRegistry)
		loginCmd.Stdout = os.Stdout
		loginCmd.Stderr = os.Stderr
		err := loginCmd.Run()
		if err != nil {
			log.Error("docker login", "Error", err)
			return
		}
	}

	uniqueContainers := make(map[string]*conf.Container)

	// This is in a loop but assumes we're building a single container
	// TODO support multiple containers
	for _, environment := range config.Environments {

		for _, region := range environment.Regions {

			for _, container := range region.Containers {

				container.OriginalName = container.Name

				// Alter the name in the config so we know which image names are part
				// of the service payload. This is important for hotswap to know which
				// of the available images on the host are the ones to be swapped in.
				if dockerRegistry == "" && dockerRepository == "" {

					container.Name = fmt.Sprintf("s3/s3:porter-%s-%d-%s",
						config.ServiceVersion, now, container.Name)
				} else {

					container.Name = fmt.Sprintf("%s/%s:porter-%s-%d-%s",
						dockerRegistry, dockerRepository,
						config.ServiceVersion, now, container.Name)
				}

				// a unique container is the combination of its name and
				// Dockerfiles used to build it
				uid := container.Name + container.Dockerfile + container.DockerfileBuild

				if _, exists := uniqueContainers[uid]; !exists {

					uniqueContainers[uid] = container
				}
			}
		}
	}

	successChan := make(chan bool)

	for _, container := range uniqueContainers {

		go func(container *conf.Container) {

			successChan <- buildContainer(log, container.Name,
				container.Dockerfile, container.DockerfileBuild)

		}(container)
	}

	for i := 0; i < len(uniqueContainers); i++ {
		success = <-successChan
		if !success {
			return
		}
	}

	if !copyPathBasedFiles(log, config) {
		return
	}

	configBytes, err := yaml.Marshal(config)
	if err != nil {
		return
	}

	// for later build stages
	err = ioutil.WriteFile(constants.AlteredConfigPath, configBytes, 0644)
	if err != nil {
		log.Error("WriteFile", "Path", constants.AlteredConfigPath)
		return
	}

	// for the service payload about to be created
	err = ioutil.WriteFile(constants.PackPayloadConfigPath, configBytes, 0644)
	if err != nil {
		log.Error("WriteFile", "Path", constants.PackPayloadConfigPath)
		return
	}

	log.Info(fmt.Sprintf("creating service payload at %s", constants.PayloadPath))

	tarCmd := exec.Command("tar", "-C", constants.PayloadWorkingDir, "-czf", constants.PayloadPath, ".")
	tarCmd.Stdout = os.Stdout
	tarCmd.Stderr = os.Stderr
	err = tarCmd.Run()
	if err != nil {
		log.Error("tar", "Error", err)
		return
	}

	success = true
	return
}

func buildContainer(log log15.Logger, containerName, dockerfile, dockerfileBuild string) (success bool) {

	log = log.New("ImageTag", containerName)

	imagePath := fmt.Sprintf("%s/%s.docker", constants.PayloadWorkingDir, containerName)

	_, err := os.Stat(dockerfile)
	if err != nil {
		log.Error("Dockerfile stat", "Error", err)
		return
	}

	haveBuilder := true
	_, err = os.Stat(dockerfileBuild)
	if err != nil {
		haveBuilder = false
	}

	if haveBuilder {
		var err error

		buildBuilderCmd := exec.Command("docker", "build", "-t", containerName+"-builder", "-f", dockerfileBuild, ".")
		buildBuilderCmd.Stdout = os.Stdout
		buildBuilderCmd.Stderr = os.Stderr
		err = buildBuilderCmd.Run()
		if err != nil {
			log.Error("build Dockerfile.build", "Error", err)
			return
		}

		runCmd := exec.Command("docker", "run", "--rm", containerName+"-builder")

		runCmdStdoutPipe, err := runCmd.StdoutPipe()
		if err != nil {
			log.Error("couldn't create StdoutPipe", "Error", err)
			return
		}

		buildCmd := exec.Command("docker", "build",
			"-t", containerName,
			"-f", dockerfile,
			"-")
		buildCmd.Stdin = runCmdStdoutPipe
		buildCmd.Stdout = os.Stdout
		buildCmd.Stderr = os.Stderr

		err = runCmd.Start()
		if err != nil {
			log.Error("docker run", "Error", err)
			return
		}

		err = buildCmd.Start()
		if err != nil {
			log.Error("build Dockerfile", "Error", err)
			return
		}

		runCmd.Wait()
		buildCmd.Wait()
	} else {
		buildCmd := exec.Command("docker", "build",
			"-t", containerName,
			"-f", dockerfile,
			".")
		buildCmd.Stdout = os.Stdout
		buildCmd.Stderr = os.Stderr
		err := buildCmd.Run()
		if err != nil {
			log.Error("build Dockerfile", "Error", err)
			return
		}
	}

	dockerRegistry := os.Getenv(constants.EnvDockerRegistry)

	if dockerRegistry == "" {
		log.Info("saving docker image to " + imagePath)

		exec.Command("mkdir", "-p", path.Dir(imagePath)).Run()

		// concurrent docker saves give this
		// Error response from daemon: open /var/lib/docker/devicemapper/mnt/0faf0a543943f7c709a018aacb339edbd85e307fd59d2a0f873af93ef25bf243/rootfs/etc/ssl/certs/ca-certificates.crt: no such file or directory
		dockerSaveLock.Lock()
		defer dockerSaveLock.Unlock()

		saveCmd := exec.Command("docker", "save", "-o", imagePath, containerName)
		saveCmd.Stdout = os.Stdout
		saveCmd.Stderr = os.Stderr
		err = saveCmd.Run()
		if err != nil {
			log.Error("docker save", "Error", err)
			return
		}

	} else {

		log.Info("docker push")

		pushCmd := exec.Command("docker", "push", containerName)
		pushCmd.Stdout = os.Stdout
		pushCmd.Stderr = os.Stderr
		err := pushCmd.Run()
		if err != nil {
			log.Error("docker push", "Error", err)
			return
		}
	}

	success = true
	return
}

// Ensure the files that are specified with paths in the config are part of the
// temp directory which is passed between the pack and provision stages in GoCD.
// If we fetched materials in every stage then the referenced files would always
// be there, and this function wouldn't be strictly necessary.
func copyPathBasedFiles(log log15.Logger, config *conf.Config) bool {
	for _, environment := range config.Environments {
		if digest, success := digestAndCopy(log, environment.StackDefinitionPath); success {
			environment.StackDefinitionPath = digest
		} else {
			return false
		}

		for _, region := range environment.Regions {
			if digest, success := digestAndCopy(log, region.StackDefinitionPath); success {
				region.StackDefinitionPath = digest
			} else {
				return false
			}
		}
	}

	return true
}

func digestAndCopy(log log15.Logger, filePath string) (string, bool) {
	if filePath == "" {
		return "", true
	}

	log = log.New("Filepath", filePath)

	fileBytes, err := ioutil.ReadFile(filePath)
	if err != nil {
		log.Error("ioutil.ReadFile", "Error", err)
		return "", false
	}
	digestArray := md5.Sum(fileBytes)
	digest := hex.EncodeToString(digestArray[:])

	newFilePath := constants.TempDir + "/" + digest
	err = ioutil.WriteFile(newFilePath, fileBytes, 0644)
	if err != nil {
		log.Error("ioutil.ReadFile", "Error", err)
		return "", false
	}

	return newFilePath, true
}
<|endoftext|>


<commit_before>package httptools

import (
	"fmt"
	"net/http"
	"regexp"
	"sort"
)

type regexpRule struct {
	re *regexp.Regexp
	h  http.Handler
}

type regexpSwitch []regexpRule

func (rs regexpSwitch) Len() int {
	return len(rs)
}

func (rs regexpSwitch) Swap(i, j int) {
	rs[i], rs[j] = rs[j], rs[i]
}

func (rs regexpSwitch) Less(i, j int) bool {
	return len(rs[i].re.String()) < len(rs[j].re.String())
}

func (rs regexpSwitch) ServeHTTP(w http.ResponseWriter, r *http.Request) {
	orw, ok := w.(*ourResponseWriter)
	if !ok {
		orw = newOurResponseWriter(w)
	}

	for _, rule := range rs {
		if m := rule.re.FindStringSubmatch(r.URL.Path); m != nil {
			for i := 1; i < len(m); i++ {
				orw.Vars()[fmt.Sprintf("%d", i)] = m[i]
			}
			rule.h.ServeHTTP(orw, r)
			return
		}
	}
	http.Error(w, "Not found", http.StatusNotFound)
}

// A regexp switch takes a map of regexp strings and handlers.
// If a request path matches a regexp, the corresponding handler is
// executed. Submatches will be put inside a VarsResponseWriter with the
// keys "1", "2", ...
// Longer patterns take precedence over shorter ones.
func NewRegexpSwitch(routes map[string]http.Handler) http.Handler {
	rs := regexpSwitch{}
	for re, h := range routes {
		rs = append(rs, regexpRule{
			re: mustRegexp("^" + re + "$"),
			h:  h,
		})
	}
	sort.Sort(sort.Reverse(rs))
	return rs
}

func mustRegexp(re string) *regexp.Regexp {
	r, err := regexp.CompilePOSIX(re)
	if err != nil {
		panic(err)
	}
	return r
}
<commit_msg>Expose RegexpSwitch array for dynamic generation<commit_after>package httptools

import (
	"fmt"
	"net/http"
	"regexp"
	"sort"
)

type regexpRule struct {
	*regexp.Regexp
	http.Handler
}

// RegexpRule represents a single rule in a RegexpSwitch.
type RegexpRule interface {
	// Same method provided by regexp.Regexp.
	// The returned array will be saved to the VarsResponseWriter.
	FindStringSubmatch(s string) []string
	http.Handler
}

// RegexpSwitch is a slice of RegexpRules. They will be checked
// in the order they have been provided. If a rule matches
// (i.e. Regexp.Rule.FindStringSubmatch return value is non-nil), the
// Handler will be called and the slice traversal is stopped.
type RegexpSwitch []RegexpRule

func (rs RegexpSwitch) ServeHTTP(w http.ResponseWriter, r *http.Request) {
	orw, ok := w.(*ourResponseWriter)
	if !ok {
		orw = newOurResponseWriter(w)
	}

	for _, rule := range rs {
		if m := rule.FindStringSubmatch(r.URL.Path); m != nil {
			for i := 1; i < len(m); i++ {
				orw.Vars()[fmt.Sprintf("%d", i)] = m[i]
			}
			rule.ServeHTTP(orw, r)
			return
		}
	}
	http.Error(w, "Not found", http.StatusNotFound)
}

type regexpSwitch []regexpRule

func (rs regexpSwitch) Len() int {
	return len(rs)
}

func (rs regexpSwitch) Swap(i, j int) {
	rs[i], rs[j] = rs[j], rs[i]
}

func (rs regexpSwitch) Less(i, j int) bool {
	return len(rs[i].String()) < len(rs[j].String())
}

// A regexp switch takes a map of regexp strings and handlers.
// A regexp is considered a match if it matches the whole string.
// Longer patterns take precedence over shorter ones.
func NewRegexpSwitch(routes map[string]http.Handler) RegexpSwitch {
	rs := make(regexpSwitch, 0, len(routes))
	for re, h := range routes {
		rs = append(rs, regexpRule{
			Regexp:  regexp.MustCompilePOSIX("^" + re + "$"),
			Handler: h,
		})
	}
	sort.Sort(sort.Reverse(rs))

	nrs := make(RegexpSwitch, 0, len(routes))
	for _, rr := range rs {
		nrs = append(nrs, RegexpRule(rr))
	}
	return nrs
}
<|endoftext|>