<commit_before>// Copyright 2012 The Go Authors.  All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

package main

import (
	"bytes"
	"encoding/json"
	"fmt"
	"os"
	"os/exec"
	"regexp"
	"strings"
)

// A vcsCmd describes how to use a version control system
// like Mercurial, Git, or Subversion.
type vcsCmd struct {
	name string
	cmd  string // name of binary to invoke command

	createCmd   string // command to download a fresh copy of a repository
	downloadCmd string // command to download updates into an existing repository

	tagCmd     []tagCmd // commands to list tags
	tagDefault string   // default tag to use
	tagSyncCmd string   // command to sync to specific tag
}

// A tagCmd describes a command to list available tags
// that can be passed to tagSyncCmd.
type tagCmd struct {
	cmd     string // command to list tags
	pattern string // regexp to extract tags from list
}

// vcsList lists the known version control systems
var vcsList = []*vcsCmd{
	vcsHg,
	vcsGit,
	vcsSvn,
	vcsBzr,
}

// vcsByCmd returns the version control system for the given
// command name (hg, git, svn, bzr).
func vcsByCmd(cmd string) *vcsCmd {
	for _, vcs := range vcsList {
		if vcs.cmd == cmd {
			return vcs
		}
	}
	return nil
}

// vcsHg describes how to use Mercurial.
var vcsHg = &vcsCmd{
	name: "Mercurial",
	cmd:  "hg",

	createCmd:   "clone -U {repo} {dir}",
	downloadCmd: "pull",

	// We allow both tag and branch names as 'tags'
	// for selecting a version.  This lets people have
	// a go.release.r60 branch and a go.1 branch
	// and make changes in both, without constantly
	// editing .hgtags.
	tagCmd: []tagCmd{
		{"tags", `^(\S+)`},
		{"branches", `^(\S+)`},
	},
	tagDefault: "default",
	tagSyncCmd: "update -r {tag}",
}

// vcsGit describes how to use Git.
var vcsGit = &vcsCmd{
	name: "Git",
	cmd:  "git",

	createCmd:   "clone {repo} {dir}",
	downloadCmd: "fetch",

	tagCmd:     []tagCmd{{"tag", `^(\S+)$`}},
	tagDefault: "master",
	tagSyncCmd: "checkout {tag}",
}

// vcsBzr describes how to use Bazaar.
var vcsBzr = &vcsCmd{
	name: "Bazaar",
	cmd:  "bzr",

	createCmd:   "branch {repo} {dir}",
	downloadCmd: "pull --overwrite", // TODO: REALLY?

	tagCmd:     []tagCmd{{"tags", `^(\S+)`}},
	tagDefault: "revno:-1",
	tagSyncCmd: "update -r {tag}",
}

// vcsSvn describes how to use Subversion.
var vcsSvn = &vcsCmd{
	name: "Subversion",
	cmd:  "svn",

	createCmd:   "checkout {repo} {dir}",
	downloadCmd: "update",

	// There is no tag command in subversion.
	// The branch information is all in the path names.
}

func (v *vcsCmd) String() string {
	return v.name
}

// run runs the command line cmd in the given directory.
// keyval is a list of key, value pairs.  run expands
// instances of {key} in cmd into value, but only after
// splitting cmd into individual arguments.
// If an error occurs, run prints the command line and the
// command's combined stdout+stderr to standard error.
// Otherwise run discards the command's output.
func (v *vcsCmd) run(dir string, cmd string, keyval ...string) error {
	_, err := v.run1(dir, false, cmd, keyval)
	return err
}

// runOutput is like run but returns the output of the command.
func (v *vcsCmd) runOutput(dir string, cmd string, keyval ...string) ([]byte, error) {
	return v.run1(dir, true, cmd, keyval)
}

// run1 is the generalized implementation of run and runOutput.
func (v *vcsCmd) run1(dir string, output bool, cmdline string, keyval []string) ([]byte, error) {
	m := make(map[string]string)
	for i := 0; i < len(keyval); i += 2 {
		m[keyval[i]] = keyval[i+1]
	}
	args := strings.Fields(cmdline)
	for i, arg := range args {
		args[i] = expand(m, arg)
	}

	cmd := exec.Command(v.cmd, args...)
	cmd.Dir = dir
	if buildX {
		fmt.Printf("cd %s\n", dir)
		fmt.Printf("%s %s\n", v.cmd, strings.Join(args, " "))
	}
	var buf bytes.Buffer
	cmd.Stdout = &buf
	cmd.Stderr = &buf
	out := buf.Bytes()
	err := cmd.Run()
	if err != nil {
		fmt.Fprintf(os.Stderr, "# cd %s; %s %s\n", dir, v.cmd, strings.Join(args, " "))
		os.Stderr.Write(out)
		return nil, err
	}
	return out, nil
}

// create creates a new copy of repo in dir.
// The parent of dir must exist; dir must not.
func (v *vcsCmd) create(dir, repo string) error {
	return v.run(".", v.createCmd, "dir", dir, "repo", repo)
}

// download downloads any new changes for the repo in dir.
func (v *vcsCmd) download(dir string) error {
	return v.run(dir, v.downloadCmd)
}

// tags returns the list of available tags for the repo in dir.
func (v *vcsCmd) tags(dir string) ([]string, error) {
	var tags []string
	for _, tc := range v.tagCmd {
		out, err := v.runOutput(dir, tc.cmd)
		if err != nil {
			return nil, err
		}
		re := regexp.MustCompile(`(?m-s)` + tc.pattern)
		tags = append(tags, re.FindAllString(string(out), -1)...)
	}
	return tags, nil
}

// tagSync syncs the repo in dir to the named tag,
// which either is a tag returned by tags or is v.tagDefault.
func (v *vcsCmd) tagSync(dir, tag string) error {
	if v.tagSyncCmd == "" {
		return nil
	}
	return v.run(dir, v.tagSyncCmd, "tag", tag)
}

// A vcsPath is describes how to convert an import path into a
// version control system and repository name.
type vcsPath struct {
	prefix string                              // prefix this description applies to
	re     string                              // pattern for import path
	repo   string                              // repository to use (expand with match of re)
	vcs    string                              // version control system to use (expand with match of re)
	check  func(match map[string]string) error // additional checks

	regexp *regexp.Regexp // cached compiled form of re
}

// vcsForImportPath analyzes importPath to determine the
// version control system, and code repository to use.
// On return, repo is the repository URL and root is the
// import path corresponding to the root of the repository
// (thus root is a prefix of importPath).
func vcsForImportPath(importPath string) (vcs *vcsCmd, repo, root string, err error) {
	for _, srv := range vcsPaths {
		if !strings.HasPrefix(importPath, srv.prefix) {
			continue
		}
		m := srv.regexp.FindStringSubmatch(importPath)
		if m == nil {
			if srv.prefix != "" {
				return nil, "", "", fmt.Errorf("invalid %s import path %q", srv.prefix, importPath)
			}
			continue
		}

		// Build map of named subexpression matches for expand.
		match := map[string]string{
			"prefix": srv.prefix,
			"import": importPath,
		}
		for i, name := range srv.regexp.SubexpNames() {
			if name != "" && match[name] == "" {
				match[name] = m[i]
			}
		}
		if srv.vcs != "" {
			match["vcs"] = expand(match, srv.vcs)
		}
		if srv.repo != "" {
			match["repo"] = expand(match, srv.repo)
		}
		if srv.check != nil {
			if err := srv.check(match); err != nil {
				return nil, "", "", err
			}
		}
		vcs := vcsByCmd(match["vcs"])
		if vcs == nil {
			return nil, "", "", fmt.Errorf("unknown version control system %q", match["vcs"])
		}
		return vcs, match["repo"], match["root"], nil
	}
	return nil, "", "", fmt.Errorf("unrecognized import path %q", importPath)
}

// expand rewrites s to replace {k} with match[k] for each key k in match.
func expand(match map[string]string, s string) string {
	for k, v := range match {
		s = strings.Replace(s, "{"+k+"}", v, -1)
	}
	return s
}

// vcsPaths lists the known vcs paths.
var vcsPaths = []*vcsPath{
	// Google Code - new syntax
	{
		prefix: "code.google.com/",
		re:     `^(?P<root>code\.google\.com/p/(?P<project>[a-z0-9\-]+)(\.(?P<subrepo>[a-z0-9\-]+))?)(/[A-Za-z0-9_.\-]+)*$`,
		repo:   "https://{root}",
		check:  googleCodeVCS,
	},

	// Google Code - old syntax
	{
		re:    `^(?P<project>[a-z0-9_\-.]+)\.googlecode\.com/(git|hg|svn)(?P<path>/.*)?$`,
		check: oldGoogleCode,
	},

	// Github
	{
		prefix: "github.com/",
		re:     `^(?P<root>github\.com/[A-Za-z0-9_.\-]+/[A-Za-z0-9_.\-]+)(/[A-Za-z0-9_.\-]+)*$`,
		vcs:    "git",
		repo:   "https://{root}",
		check:  noVCSSuffix,
	},

	// Bitbucket
	{
		prefix: "bitbucket.org/",
		re:     `^(?P<root>bitbucket\.org/(?P<bitname>[A-Za-z0-9_.\-]+/[A-Za-z0-9_.\-]+))(/[A-Za-z0-9_.\-]+)*$`,
		repo:   "https://{root}",
		check:  bitbucketVCS,
	},

	// Launchpad
	{
		prefix: "launchpad.net/",
		re:     `^(?P<root>launchpad\.net/([A-Za-z0-9_.\-]+(/[A-Za-z0-9_.\-]+)?|~[A-Za-z0-9_.\-]+/(\+junk|[A-Za-z0-9_.\-]+)/[A-Za-z0-9_.\-]+))(/[A-Za-z0-9_.\-]+)*$`,
		vcs:    "bzr",
		repo:   "https://{root}",
	},

	// General syntax for any server.
	{
		re: `^(?P<root>(?P<repo>([a-z0-9.\-]+\.)+[a-z0-9.\-]+(:[0-9]+)?/[A-Za-z0-9_.\-/]*?)\.(?P<vcs>bzr|git|hg|svn))(/[A-Za-z0-9_.\-]+)*$`,
	},
}

func init() {
	// fill in cached regexps.
	// Doing this eagerly discovers invalid regexp syntax
	// without having to run a command that needs that regexp.
	for _, srv := range vcsPaths {
		srv.regexp = regexp.MustCompile(srv.re)
	}
}

// noVCSSuffix checks that the repository name does not
// end in .foo for any version control system foo.
// The usual culprit is ".git".
func noVCSSuffix(match map[string]string) error {
	repo := match["repo"]
	for _, vcs := range vcsList {
		if strings.HasSuffix(repo, "."+vcs.cmd) {
			return fmt.Errorf("invalid version control suffix in %s path", match["prefix"])
		}
	}
	return nil
}

var googleCheckout = regexp.MustCompile(`id="checkoutcmd">(hg|git|svn)`)

// googleCodeVCS determines the version control system for
// a code.google.com repository, by scraping the project's
// /source/checkout page.
func googleCodeVCS(match map[string]string) error {
	if err := noVCSSuffix(match); err != nil {
		return err
	}
	data, err := httpGET(expand(match, "https://code.google.com/p/{project}/source/checkout?repo={subrepo}"))
	if err != nil {
		return err
	}

	if m := googleCheckout.FindSubmatch(data); m != nil {
		if vcs := vcsByCmd(string(m[1])); vcs != nil {
			// Subversion requires the old URLs.
			// TODO: Test.
			if vcs == vcsSvn {
				if match["subrepo"] != "" {
					return fmt.Errorf("sub-repositories not supported in Google Code Subversion projects")
				}
				match["repo"] = expand(match, "https://{project}.googlecode.com/svn")
			}
			match["vcs"] = vcs.cmd
			return nil
		}
	}

	return fmt.Errorf("unable to detect version control system for code.google.com/ path")
}

// oldGoogleCode is invoked for old-style foo.googlecode.com paths.
// It prints an error giving the equivalent new path.
func oldGoogleCode(match map[string]string) error {
	return fmt.Errorf("invalid Google Code import path: use %s instead",
		expand(match, "code.google.com/p/{project}{path}"))
}

// bitbucketVCS determines the version control system for a
// BitBucket repository, by using the BitBucket API.
func bitbucketVCS(match map[string]string) error {
	if err := noVCSSuffix(match); err != nil {
		return err
	}

	var resp struct {
		SCM string `json:"scm"`
	}
	url := expand(match, "https://api.bitbucket.org/1.0/repositories/{bitname}")
	data, err := httpGET(url)
	if err != nil {
		return err
	}
	if err := json.Unmarshal(data, &resp); err != nil {
		return fmt.Errorf("decoding %s: %v", url, err)
	}

	if vcsByCmd(resp.SCM) != nil {
		match["vcs"] = resp.SCM
		if resp.SCM == "git" {
			match["repo"] += ".git"
		}
		return nil
	}

	return fmt.Errorf("unable to detect version control system for bitbucket.org/ path")
}
<commit_msg>cmd/go: solve ambiguity of get lp.net/project/foo<commit_after>// Copyright 2012 The Go Authors.  All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

package main

import (
	"bytes"
	"encoding/json"
	"fmt"
	"os"
	"os/exec"
	"regexp"
	"strings"
)

// A vcsCmd describes how to use a version control system
// like Mercurial, Git, or Subversion.
type vcsCmd struct {
	name string
	cmd  string // name of binary to invoke command

	createCmd   string // command to download a fresh copy of a repository
	downloadCmd string // command to download updates into an existing repository

	tagCmd     []tagCmd // commands to list tags
	tagDefault string   // default tag to use
	tagSyncCmd string   // command to sync to specific tag
}

// A tagCmd describes a command to list available tags
// that can be passed to tagSyncCmd.
type tagCmd struct {
	cmd     string // command to list tags
	pattern string // regexp to extract tags from list
}

// vcsList lists the known version control systems
var vcsList = []*vcsCmd{
	vcsHg,
	vcsGit,
	vcsSvn,
	vcsBzr,
}

// vcsByCmd returns the version control system for the given
// command name (hg, git, svn, bzr).
func vcsByCmd(cmd string) *vcsCmd {
	for _, vcs := range vcsList {
		if vcs.cmd == cmd {
			return vcs
		}
	}
	return nil
}

// vcsHg describes how to use Mercurial.
var vcsHg = &vcsCmd{
	name: "Mercurial",
	cmd:  "hg",

	createCmd:   "clone -U {repo} {dir}",
	downloadCmd: "pull",

	// We allow both tag and branch names as 'tags'
	// for selecting a version.  This lets people have
	// a go.release.r60 branch and a go.1 branch
	// and make changes in both, without constantly
	// editing .hgtags.
	tagCmd: []tagCmd{
		{"tags", `^(\S+)`},
		{"branches", `^(\S+)`},
	},
	tagDefault: "default",
	tagSyncCmd: "update -r {tag}",
}

// vcsGit describes how to use Git.
var vcsGit = &vcsCmd{
	name: "Git",
	cmd:  "git",

	createCmd:   "clone {repo} {dir}",
	downloadCmd: "fetch",

	tagCmd:     []tagCmd{{"tag", `^(\S+)$`}},
	tagDefault: "master",
	tagSyncCmd: "checkout {tag}",
}

// vcsBzr describes how to use Bazaar.
var vcsBzr = &vcsCmd{
	name: "Bazaar",
	cmd:  "bzr",

	createCmd: "branch {repo} {dir}",

	// Without --overwrite bzr will not pull tags that changed.
	// Replace by --overwrite-tags after http://pad.lv/681792 goes in.
	downloadCmd: "pull --overwrite",

	tagCmd:     []tagCmd{{"tags", `^(\S+)`}},
	tagDefault: "revno:-1",
	tagSyncCmd: "update -r {tag}",
}

// vcsSvn describes how to use Subversion.
var vcsSvn = &vcsCmd{
	name: "Subversion",
	cmd:  "svn",

	createCmd:   "checkout {repo} {dir}",
	downloadCmd: "update",

	// There is no tag command in subversion.
	// The branch information is all in the path names.
}

func (v *vcsCmd) String() string {
	return v.name
}

// run runs the command line cmd in the given directory.
// keyval is a list of key, value pairs.  run expands
// instances of {key} in cmd into value, but only after
// splitting cmd into individual arguments.
// If an error occurs, run prints the command line and the
// command's combined stdout+stderr to standard error.
// Otherwise run discards the command's output.
func (v *vcsCmd) run(dir string, cmd string, keyval ...string) error {
	_, err := v.run1(dir, false, cmd, keyval)
	return err
}

// runOutput is like run but returns the output of the command.
func (v *vcsCmd) runOutput(dir string, cmd string, keyval ...string) ([]byte, error) {
	return v.run1(dir, true, cmd, keyval)
}

// run1 is the generalized implementation of run and runOutput.
func (v *vcsCmd) run1(dir string, output bool, cmdline string, keyval []string) ([]byte, error) {
	m := make(map[string]string)
	for i := 0; i < len(keyval); i += 2 {
		m[keyval[i]] = keyval[i+1]
	}
	args := strings.Fields(cmdline)
	for i, arg := range args {
		args[i] = expand(m, arg)
	}

	cmd := exec.Command(v.cmd, args...)
	cmd.Dir = dir
	if buildX {
		fmt.Printf("cd %s\n", dir)
		fmt.Printf("%s %s\n", v.cmd, strings.Join(args, " "))
	}
	var buf bytes.Buffer
	cmd.Stdout = &buf
	cmd.Stderr = &buf
	out := buf.Bytes()
	err := cmd.Run()
	if err != nil {
		fmt.Fprintf(os.Stderr, "# cd %s; %s %s\n", dir, v.cmd, strings.Join(args, " "))
		os.Stderr.Write(out)
		return nil, err
	}
	return out, nil
}

// create creates a new copy of repo in dir.
// The parent of dir must exist; dir must not.
func (v *vcsCmd) create(dir, repo string) error {
	return v.run(".", v.createCmd, "dir", dir, "repo", repo)
}

// download downloads any new changes for the repo in dir.
func (v *vcsCmd) download(dir string) error {
	return v.run(dir, v.downloadCmd)
}

// tags returns the list of available tags for the repo in dir.
func (v *vcsCmd) tags(dir string) ([]string, error) {
	var tags []string
	for _, tc := range v.tagCmd {
		out, err := v.runOutput(dir, tc.cmd)
		if err != nil {
			return nil, err
		}
		re := regexp.MustCompile(`(?m-s)` + tc.pattern)
		tags = append(tags, re.FindAllString(string(out), -1)...)
	}
	return tags, nil
}

// tagSync syncs the repo in dir to the named tag,
// which either is a tag returned by tags or is v.tagDefault.
func (v *vcsCmd) tagSync(dir, tag string) error {
	if v.tagSyncCmd == "" {
		return nil
	}
	return v.run(dir, v.tagSyncCmd, "tag", tag)
}

// A vcsPath describes how to convert an import path into a
// version control system and repository name.
type vcsPath struct {
	prefix string                              // prefix this description applies to
	re     string                              // pattern for import path
	repo   string                              // repository to use (expand with match of re)
	vcs    string                              // version control system to use (expand with match of re)
	check  func(match map[string]string) error // additional checks

	regexp *regexp.Regexp // cached compiled form of re
}

// vcsForImportPath analyzes importPath to determine the
// version control system, and code repository to use.
// On return, repo is the repository URL and root is the
// import path corresponding to the root of the repository
// (thus root is a prefix of importPath).
func vcsForImportPath(importPath string) (vcs *vcsCmd, repo, root string, err error) {
	for _, srv := range vcsPaths {
		if !strings.HasPrefix(importPath, srv.prefix) {
			continue
		}
		m := srv.regexp.FindStringSubmatch(importPath)
		if m == nil {
			if srv.prefix != "" {
				return nil, "", "", fmt.Errorf("invalid %s import path %q", srv.prefix, importPath)
			}
			continue
		}

		// Build map of named subexpression matches for expand.
		match := map[string]string{
			"prefix": srv.prefix,
			"import": importPath,
		}
		for i, name := range srv.regexp.SubexpNames() {
			if name != "" && match[name] == "" {
				match[name] = m[i]
			}
		}
		if srv.vcs != "" {
			match["vcs"] = expand(match, srv.vcs)
		}
		if srv.repo != "" {
			match["repo"] = expand(match, srv.repo)
		}
		if srv.check != nil {
			if err := srv.check(match); err != nil {
				return nil, "", "", err
			}
		}
		vcs := vcsByCmd(match["vcs"])
		if vcs == nil {
			return nil, "", "", fmt.Errorf("unknown version control system %q", match["vcs"])
		}
		return vcs, match["repo"], match["root"], nil
	}
	return nil, "", "", fmt.Errorf("unrecognized import path %q", importPath)
}

// expand rewrites s to replace {k} with match[k] for each key k in match.
func expand(match map[string]string, s string) string {
	for k, v := range match {
		s = strings.Replace(s, "{"+k+"}", v, -1)
	}
	return s
}

// vcsPaths lists the known vcs paths.
var vcsPaths = []*vcsPath{
	// Google Code - new syntax
	{
		prefix: "code.google.com/",
		re:     `^(?P<root>code\.google\.com/p/(?P<project>[a-z0-9\-]+)(\.(?P<subrepo>[a-z0-9\-]+))?)(/[A-Za-z0-9_.\-]+)*$`,
		repo:   "https://{root}",
		check:  googleCodeVCS,
	},

	// Google Code - old syntax
	{
		re:    `^(?P<project>[a-z0-9_\-.]+)\.googlecode\.com/(git|hg|svn)(?P<path>/.*)?$`,
		check: oldGoogleCode,
	},

	// Github
	{
		prefix: "github.com/",
		re:     `^(?P<root>github\.com/[A-Za-z0-9_.\-]+/[A-Za-z0-9_.\-]+)(/[A-Za-z0-9_.\-]+)*$`,
		vcs:    "git",
		repo:   "https://{root}",
		check:  noVCSSuffix,
	},

	// Bitbucket
	{
		prefix: "bitbucket.org/",
		re:     `^(?P<root>bitbucket\.org/(?P<bitname>[A-Za-z0-9_.\-]+/[A-Za-z0-9_.\-]+))(/[A-Za-z0-9_.\-]+)*$`,
		repo:   "https://{root}",
		check:  bitbucketVCS,
	},

	// Launchpad
	{
		prefix: "launchpad.net/",
		re:     `^(?P<root>launchpad\.net/((?P<project>[A-Za-z0-9_.\-]+)(?P<series>/[A-Za-z0-9_.\-]+)?|~[A-Za-z0-9_.\-]+/(\+junk|[A-Za-z0-9_.\-]+)/[A-Za-z0-9_.\-]+))(/[A-Za-z0-9_.\-]+)*$`,
		vcs:    "bzr",
		repo:   "https://{root}",
		check:  launchpadVCS,
	},

	// General syntax for any server.
	{
		re: `^(?P<root>(?P<repo>([a-z0-9.\-]+\.)+[a-z0-9.\-]+(:[0-9]+)?/[A-Za-z0-9_.\-/]*?)\.(?P<vcs>bzr|git|hg|svn))(/[A-Za-z0-9_.\-]+)*$`,
	},
}

func init() {
	// fill in cached regexps.
	// Doing this eagerly discovers invalid regexp syntax
	// without having to run a command that needs that regexp.
	for _, srv := range vcsPaths {
		srv.regexp = regexp.MustCompile(srv.re)
	}
}

// noVCSSuffix checks that the repository name does not
// end in .foo for any version control system foo.
// The usual culprit is ".git".
func noVCSSuffix(match map[string]string) error {
	repo := match["repo"]
	for _, vcs := range vcsList {
		if strings.HasSuffix(repo, "."+vcs.cmd) {
			return fmt.Errorf("invalid version control suffix in %s path", match["prefix"])
		}
	}
	return nil
}

var googleCheckout = regexp.MustCompile(`id="checkoutcmd">(hg|git|svn)`)

// googleCodeVCS determines the version control system for
// a code.google.com repository, by scraping the project's
// /source/checkout page.
func googleCodeVCS(match map[string]string) error {
	if err := noVCSSuffix(match); err != nil {
		return err
	}
	data, err := httpGET(expand(match, "https://code.google.com/p/{project}/source/checkout?repo={subrepo}"))
	if err != nil {
		return err
	}

	if m := googleCheckout.FindSubmatch(data); m != nil {
		if vcs := vcsByCmd(string(m[1])); vcs != nil {
			// Subversion requires the old URLs.
			// TODO: Test.
			if vcs == vcsSvn {
				if match["subrepo"] != "" {
					return fmt.Errorf("sub-repositories not supported in Google Code Subversion projects")
				}
				match["repo"] = expand(match, "https://{project}.googlecode.com/svn")
			}
			match["vcs"] = vcs.cmd
			return nil
		}
	}

	return fmt.Errorf("unable to detect version control system for code.google.com/ path")
}

// oldGoogleCode is invoked for old-style foo.googlecode.com paths.
// It prints an error giving the equivalent new path.
func oldGoogleCode(match map[string]string) error {
	return fmt.Errorf("invalid Google Code import path: use %s instead",
		expand(match, "code.google.com/p/{project}{path}"))
}

// bitbucketVCS determines the version control system for a
// BitBucket repository, by using the BitBucket API.
func bitbucketVCS(match map[string]string) error {
	if err := noVCSSuffix(match); err != nil {
		return err
	}

	var resp struct {
		SCM string `json:"scm"`
	}
	url := expand(match, "https://api.bitbucket.org/1.0/repositories/{bitname}")
	data, err := httpGET(url)
	if err != nil {
		return err
	}
	if err := json.Unmarshal(data, &resp); err != nil {
		return fmt.Errorf("decoding %s: %v", url, err)
	}

	if vcsByCmd(resp.SCM) != nil {
		match["vcs"] = resp.SCM
		if resp.SCM == "git" {
			match["repo"] += ".git"
		}
		return nil
	}

	return fmt.Errorf("unable to detect version control system for bitbucket.org/ path")
}

// launchpadVCS solves the ambiguity for "lp.net/project/foo". In this case,
// "foo" could be a series name registered in Launchpad with its own branch,
// and it could also be the name of a directory within the main project
// branch one level up.
func launchpadVCS(match map[string]string) error {
	if match["project"] == "" || match["series"] == "" {
		return nil
	}
	_, err := httpGET(expand(match, "https://code.launchpad.net/{project}{series}/.bzr/branch-format"))
	if err != nil {
		match["root"] = expand(match, "launchpad.net/{project}")
		match["repo"] = expand(match, "https://{root}")
	}
	return nil
}
<|endoftext|>


<commit_before>package main

import (
	"os/exec"
	"strings"
	"encoding/gob"
	"fmt"
	"bufio"
	"io"
	"log"
	"net"
	"os"

	"github.com/flynn/go-discover/discover"
	"github.com/flynn/lorne/types"
	"github.com/titanous/go-dockerclient"
	"github.com/flynn/sampi/types"
	sc "github.com/flynn/sampi/client"
	lc "github.com/flynn/lorne/client"
)

// WARNING: assumes one host at the moment (firstHost will always be the same)

var sd *discover.Client
var sched *sc.Client
var host *lc.Client
var hostid string

func init() {
	var err error
	sd, err = discover.NewClient()
	if err != nil {
		log.Fatal(err)
	}

	sched, err = sc.New()
	if err != nil {
		log.Fatal(err)
	}

	hostid = findHost()
	host, err = lc.New(hostid)
	if err != nil {
		log.Fatal(err)
	}
}

func main() {
	root := "/var/lib/demo/apps"
	hostname := shell("curl -s icanhazip.com")

	set, _ := sd.Services("shelf")
	addrs := set.OnlineAddrs()
	if len(addrs) < 1 {
		panic("Shelf is not discoverable")
	}
	shelfHost := addrs[0]

	app := os.Args[2]
	os.MkdirAll(root+"/"+app, 0755)

	fmt.Printf("-----> Building %s on %s ...\n", app, hostname)

	scheduleAndAttach(app + ".build", docker.Config{
		Image:        "flynn/slugbuilder",
		Cmd:          []string{"http://" + shelfHost + "/" + app + ".tgz"},
		Tty:          false,
		AttachStdin:  true,
		AttachStdout: true,
		AttachStderr: true,
		OpenStdin:    true,
		StdinOnce:    true,
	})

	fmt.Printf("-----> Deploying %s ...\n", app)

	jobid := app + ".web"

	stopIfExists(jobid)
	scheduleWithTcpPort(jobid, docker.Config{
		Image:        "flynn/slugrunner",
		Cmd:          []string{"start", "web"},
		Tty:          false,
		AttachStdin:  false,
		AttachStdout: false,
		AttachStderr: false,
		OpenStdin:    false,
		StdinOnce:    false,
		Env: []string{
			"SLUG_URL=http://" + shelfHost + "/" + app + ".tgz",
		},
	})

	fmt.Printf("=====> Application deployed:\n")
	fmt.Printf("       http://%s:%s\n", hostname, getPort(jobid))
	fmt.Println("")

}

func shell(cmdline string) string {
        out, err := exec.Command("bash", "-c", cmdline).Output()
        if err != nil {
                panic(err)
        }
        return strings.Trim(string(out), " \n")
}

func stopIfExists(jobid string) {
	_, err := host.GetJob(jobid)
	if err != nil {
		return
	}
	if err := host.StopJob(jobid); err != nil {
		return
	}
}

func scheduleWithTcpPort(jobid string, config docker.Config) {

	schedReq := &sampi.ScheduleReq{
		Incremental: true,
		HostJobs:    map[string][]*sampi.Job{hostid: {{ID: jobid, Config: &config, TCPPorts: 1}}},
	}
	if _, err := sched.Schedule(schedReq); err != nil {
		log.Fatal(err)
	}
}

func getPort(jobid string) string {
	job, err := host.GetJob(jobid)
	if err != nil {
		log.Fatal(err)
	}
	return job.Job.Config.PortSpecs[0]
}

func findHost() string {
	state, err := sched.State()
	if err != nil {
		log.Fatal(err)
	}

	var firstHost string
	for k := range state {
		firstHost = k
		break
	}
	if firstHost == "" {
		log.Fatal("no hosts")
	}
	return firstHost
}

func scheduleAndAttach(jobid string, config docker.Config) {

	services, err := sd.Services("flynn-lorne-attach." + hostid)
	if err != nil {
		log.Fatal(err)
	}
	conn, err := net.Dial("tcp", services.OnlineAddrs()[0])
	if err != nil {
		log.Fatal(err)
	}
	err = gob.NewEncoder(conn).Encode(&lorne.AttachReq{
		JobID: jobid,
		Flags: lorne.AttachFlagStdout | lorne.AttachFlagStderr | lorne.AttachFlagStdin | lorne.AttachFlagStream,
	})
	if err != nil {
		log.Fatal(err)
	}
	attachState := make([]byte, 1)
	if _, err := conn.Read(attachState); err != nil {
		log.Fatal(err)
	}
	switch attachState[0] {
	case lorne.AttachError:
		log.Fatal("attach error")
	}

	schedReq := &sampi.ScheduleReq{
		Incremental: true,
		HostJobs:    map[string][]*sampi.Job{hostid: {{ID: jobid, Config: &config}}},
	}
	if _, err := sched.Schedule(schedReq); err != nil {
		log.Fatal(err)
	}

	if _, err := conn.Read(attachState); err != nil {
		log.Fatal(err)
	}

	go func() {
		io.Copy(conn, os.Stdin)
		conn.(*net.TCPConn).CloseWrite()
	}()
	scanner := bufio.NewScanner(conn)
	for scanner.Scan() {
		fmt.Fprintln(os.Stdout, scanner.Text()[8:])
	}
	conn.Close()
}
<commit_msg>pulling port from epxosed ports<commit_after>package main

import (
	"os/exec"
	"strings"
	"encoding/gob"
	"fmt"
	"bufio"
	"io"
	"log"
	"net"
	"time"
	"os"

	"github.com/flynn/go-discover/discover"
	"github.com/flynn/lorne/types"
	"github.com/titanous/go-dockerclient"
	"github.com/flynn/sampi/types"
	sc "github.com/flynn/sampi/client"
	lc "github.com/flynn/lorne/client"
)

// WARNING: assumes one host at the moment (firstHost will always be the same)

var sd *discover.Client
var sched *sc.Client
var host *lc.Client
var hostid string

func init() {
	var err error
	sd, err = discover.NewClient()
	if err != nil {
		log.Fatal(err)
	}

	sched, err = sc.New()
	if err != nil {
		log.Fatal(err)
	}

	hostid = findHost()
	host, err = lc.New(hostid)
	if err != nil {
		log.Fatal(err)
	}
}

func main() {
	root := "/var/lib/demo/apps"
	hostname := shell("curl -s icanhazip.com")

	set, _ := sd.Services("shelf")
	addrs := set.OnlineAddrs()
	if len(addrs) < 1 {
		panic("Shelf is not discoverable")
	}
	shelfHost := addrs[0]

	app := os.Args[2]
	os.MkdirAll(root+"/"+app, 0755)

	fmt.Printf("-----> Building %s on %s ...\n", app, hostname)

	scheduleAndAttach(app + ".build", docker.Config{
		Image:        "flynn/slugbuilder",
		Cmd:          []string{"http://" + shelfHost + "/" + app + ".tgz"},
		Tty:          false,
		AttachStdin:  true,
		AttachStdout: true,
		AttachStderr: true,
		OpenStdin:    true,
		StdinOnce:    true,
	})

	fmt.Printf("-----> Deploying %s ...\n", app)

	jobid := app + ".web"

	stopIfExists(jobid)
	scheduleWithTcpPort(jobid, docker.Config{
		Image:        "flynn/slugrunner",
		Cmd:          []string{"start", "web"},
		Tty:          false,
		AttachStdin:  false,
		AttachStdout: false,
		AttachStderr: false,
		OpenStdin:    false,
		StdinOnce:    false,
		Env: []string{
			"SLUG_URL=http://" + shelfHost + "/" + app + ".tgz",
		},
	})

	time.Sleep(1 * time.Second)
	fmt.Printf("=====> Application deployed:\n")
	fmt.Printf("       http://%s:%s\n", hostname, getPort(jobid))
	fmt.Println("")

}

func shell(cmdline string) string {
        out, err := exec.Command("bash", "-c", cmdline).Output()
        if err != nil {
                panic(err)
        }
        return strings.Trim(string(out), " \n")
}

func stopIfExists(jobid string) {
	_, err := host.GetJob(jobid)
	if err != nil {
		return
	}
	if err := host.StopJob(jobid); err != nil {
		return
	}
}

func scheduleWithTcpPort(jobid string, config docker.Config) {

	schedReq := &sampi.ScheduleReq{
		Incremental: true,
		HostJobs:    map[string][]*sampi.Job{hostid: {{ID: jobid, Config: &config, TCPPorts: 1}}},
	}
	if _, err := sched.Schedule(schedReq); err != nil {
		log.Fatal(err)
	}
}

func getPort(jobid string) string {
	job, err := host.GetJob(jobid)
	if err != nil {
		log.Fatal(err)
	}
	for portspec := range job.Job.Config.ExposedPorts {
		return strings.Split(portspec, "/", 1)[0]
	}
}

func findHost() string {
	state, err := sched.State()
	if err != nil {
		log.Fatal(err)
	}

	var firstHost string
	for k := range state {
		firstHost = k
		break
	}
	if firstHost == "" {
		log.Fatal("no hosts")
	}
	return firstHost
}

func scheduleAndAttach(jobid string, config docker.Config) {

	services, err := sd.Services("flynn-lorne-attach." + hostid)
	if err != nil {
		log.Fatal(err)
	}
	conn, err := net.Dial("tcp", services.OnlineAddrs()[0])
	if err != nil {
		log.Fatal(err)
	}
	err = gob.NewEncoder(conn).Encode(&lorne.AttachReq{
		JobID: jobid,
		Flags: lorne.AttachFlagStdout | lorne.AttachFlagStderr | lorne.AttachFlagStdin | lorne.AttachFlagStream,
	})
	if err != nil {
		log.Fatal(err)
	}
	attachState := make([]byte, 1)
	if _, err := conn.Read(attachState); err != nil {
		log.Fatal(err)
	}
	switch attachState[0] {
	case lorne.AttachError:
		log.Fatal("attach error")
	}

	schedReq := &sampi.ScheduleReq{
		Incremental: true,
		HostJobs:    map[string][]*sampi.Job{hostid: {{ID: jobid, Config: &config}}},
	}
	if _, err := sched.Schedule(schedReq); err != nil {
		log.Fatal(err)
	}

	if _, err := conn.Read(attachState); err != nil {
		log.Fatal(err)
	}

	go func() {
		io.Copy(conn, os.Stdin)
		conn.(*net.TCPConn).CloseWrite()
	}()
	scanner := bufio.NewScanner(conn)
	for scanner.Scan() {
		fmt.Fprintln(os.Stdout, scanner.Text()[8:])
	}
	conn.Close()
}
<|endoftext|>


<commit_before>package main

import (
	"bufio"
	"fmt"
	"io"
	"log"
	"os"
	"os/exec"
	"strings"
	"time"

	"github.com/flynn/flynn-host/types"
	"github.com/flynn/go-discoverd"
	"github.com/flynn/go-dockerclient"
	"github.com/flynn/go-flynn/cluster"
)

var clusterc *cluster.Client

func init() {
	var err error
	clusterc, err = cluster.NewClient()
	if err != nil {
		log.Fatal(err)
	}
}

func main() {
	root := "/var/lib/demo/apps"

	services, _ := discoverd.Services("shelf", discoverd.DefaultTimeout)
	if len(services) < 1 {
		panic("Shelf is not discoverable")
	}
	shelfHost := services[0].Addr

	app := os.Args[2]
	os.MkdirAll(root+"/"+app, 0755)

	fmt.Printf("-----> Building %s...\n", app)

	scheduleAndAttach(cluster.RandomJobID(app+"-build."), docker.Config{
		Image:        "flynn/slugbuilder",
		Cmd:          []string{"http://" + shelfHost + "/" + app + ".tgz"},
		Tty:          false,
		AttachStdin:  true,
		AttachStdout: true,
		AttachStderr: true,
		OpenStdin:    true,
		StdinOnce:    true,
	})

	fmt.Printf("-----> Deploying %s ...\n", app)

	jobid := cluster.RandomJobID(app + "-web.")

	hostid := scheduleWithTcpPort(jobid, docker.Config{
		Image:        "flynn/slugrunner",
		Cmd:          []string{"start", "web"},
		Tty:          false,
		AttachStdin:  false,
		AttachStdout: false,
		AttachStderr: false,
		OpenStdin:    false,
		StdinOnce:    false,
		Env: []string{
			"SLUG_URL=http://" + shelfHost + "/" + app + ".tgz",
			"SD_NAME=" + app,
		},
	})

	time.Sleep(1 * time.Second)
	fmt.Printf("=====> Application deployed:\n")
	fmt.Printf("       http://10.0.2.15:%s\n", getPort(hostid, jobid))
	fmt.Println("")

}

func shell(cmdline string) string {
	out, err := exec.Command("bash", "-c", cmdline).Output()
	if err != nil {
		panic(err)
	}
	return strings.Trim(string(out), " \n")
}

func scheduleWithTcpPort(jobid string, config docker.Config) (hostid string) {
	hostid = randomHost()
	addReq := &host.AddJobsReq{
		Incremental: true,
		HostJobs:    map[string][]*host.Job{hostid: {{ID: jobid, Config: &config, TCPPorts: 1}}},
	}
	if _, err := clusterc.AddJobs(addReq); err != nil {
		log.Fatal(err)
	}
	return
}

func getPort(hostid string, jobid string) string {
	client, err := clusterc.ConnectHost(hostid)
	if err != nil {
		log.Fatal(err)
	}
	job, err := client.GetJob(jobid)
	if err != nil {
		log.Fatal(err)
	}
	for portspec := range job.Job.Config.ExposedPorts {
		return strings.Split(portspec, "/")[0]
	}
	return ""
}

func randomHost() (hostid string) {
	hosts, err := clusterc.ListHosts()
	if err != nil {
		log.Fatal(err)
	}

	for hostid = range hosts {
		break
	}
	if hostid == "" {
		log.Fatal("no hosts found")
	}
	return
}

func scheduleAndAttach(jobid string, config docker.Config) {
	hostid := randomHost()

	client, err := clusterc.ConnectHost(hostid)
	if err != nil {
		log.Fatal(err)
	}
	conn, attachWait, err := client.Attach(&host.AttachReq{
		JobID: jobid,
		Flags: host.AttachFlagStdout | host.AttachFlagStderr | host.AttachFlagStdin | host.AttachFlagStream,
	}, true)
	if err != nil {
		log.Fatal(err)
	}

	addReq := &host.AddJobsReq{
		Incremental: true,
		HostJobs:    map[string][]*host.Job{hostid: {{ID: jobid, Config: &config}}},
	}
	if _, err := clusterc.AddJobs(addReq); err != nil {
		log.Fatal(err)
	}

	if err := attachWait(); err != nil {
		log.Fatal(err)
	}

	go func() {
		io.Copy(conn, os.Stdin)
		conn.CloseWrite()
	}()
	scanner := bufio.NewScanner(conn)
	for scanner.Scan() {
		fmt.Fprintln(os.Stdout, scanner.Text()[8:])
	}
	conn.Close()
}
<commit_msg>Update flynn-host type usage<commit_after>package main

import (
	"bufio"
	"fmt"
	"io"
	"log"
	"os"
	"os/exec"
	"strings"
	"time"

	"github.com/flynn/flynn-host/types"
	"github.com/flynn/go-discoverd"
	"github.com/flynn/go-dockerclient"
	"github.com/flynn/go-flynn/cluster"
)

var clusterc *cluster.Client

func init() {
	var err error
	clusterc, err = cluster.NewClient()
	if err != nil {
		log.Fatal(err)
	}
}

func main() {
	root := "/var/lib/demo/apps"

	services, _ := discoverd.Services("shelf", discoverd.DefaultTimeout)
	if len(services) < 1 {
		panic("Shelf is not discoverable")
	}
	shelfHost := services[0].Addr

	app := os.Args[2]
	os.MkdirAll(root+"/"+app, 0755)

	fmt.Printf("-----> Building %s...\n", app)

	scheduleAndAttach(cluster.RandomJobID(app+"-build."), docker.Config{
		Image:        "flynn/slugbuilder",
		Cmd:          []string{"http://" + shelfHost + "/" + app + ".tgz"},
		Tty:          false,
		AttachStdin:  true,
		AttachStdout: true,
		AttachStderr: true,
		OpenStdin:    true,
		StdinOnce:    true,
	})

	fmt.Printf("-----> Deploying %s ...\n", app)

	jobid := cluster.RandomJobID(app + "-web.")

	hostid := scheduleWithTcpPort(jobid, docker.Config{
		Image:        "flynn/slugrunner",
		Cmd:          []string{"start", "web"},
		Tty:          false,
		AttachStdin:  false,
		AttachStdout: false,
		AttachStderr: false,
		OpenStdin:    false,
		StdinOnce:    false,
		Env: []string{
			"SLUG_URL=http://" + shelfHost + "/" + app + ".tgz",
			"SD_NAME=" + app,
		},
	})

	time.Sleep(1 * time.Second)
	fmt.Printf("=====> Application deployed:\n")
	fmt.Printf("       http://10.0.2.15:%s\n", getPort(hostid, jobid))
	fmt.Println("")

}

func shell(cmdline string) string {
	out, err := exec.Command("bash", "-c", cmdline).Output()
	if err != nil {
		panic(err)
	}
	return strings.Trim(string(out), " \n")
}

func scheduleWithTcpPort(jobid string, config docker.Config) (hostid string) {
	hostid = randomHost()
	addReq := &host.AddJobsReq{
		HostJobs: map[string][]*host.Job{hostid: {{ID: jobid, Config: &config, TCPPorts: 1}}},
	}
	if _, err := clusterc.AddJobs(addReq); err != nil {
		log.Fatal(err)
	}
	return
}

func getPort(hostid string, jobid string) string {
	client, err := clusterc.ConnectHost(hostid)
	if err != nil {
		log.Fatal(err)
	}
	job, err := client.GetJob(jobid)
	if err != nil {
		log.Fatal(err)
	}
	for portspec := range job.Job.Config.ExposedPorts {
		return strings.Split(portspec, "/")[0]
	}
	return ""
}

func randomHost() (hostid string) {
	hosts, err := clusterc.ListHosts()
	if err != nil {
		log.Fatal(err)
	}

	for hostid = range hosts {
		break
	}
	if hostid == "" {
		log.Fatal("no hosts found")
	}
	return
}

func scheduleAndAttach(jobid string, config docker.Config) {
	hostid := randomHost()

	client, err := clusterc.ConnectHost(hostid)
	if err != nil {
		log.Fatal(err)
	}
	conn, attachWait, err := client.Attach(&host.AttachReq{
		JobID: jobid,
		Flags: host.AttachFlagStdout | host.AttachFlagStderr | host.AttachFlagStdin | host.AttachFlagStream,
	}, true)
	if err != nil {
		log.Fatal(err)
	}

	addReq := &host.AddJobsReq{
		HostJobs: map[string][]*host.Job{hostid: {{ID: jobid, Config: &config}}},
	}
	if _, err := clusterc.AddJobs(addReq); err != nil {
		log.Fatal(err)
	}

	if err := attachWait(); err != nil {
		log.Fatal(err)
	}

	go func() {
		io.Copy(conn, os.Stdin)
		conn.CloseWrite()
	}()
	scanner := bufio.NewScanner(conn)
	for scanner.Scan() {
		fmt.Fprintln(os.Stdout, scanner.Text()[8:])
	}
	conn.Close()
}
<|endoftext|>


<commit_before>package main

import (
	"encoding/json"
	"fmt"
	"io"
	"io/ioutil"
	"net/http"

	"golang.org/x/oauth2"

	log "github.com/Sirupsen/logrus"
	"github.com/google/go-github/github"
)

func init() {
	http.HandleFunc("/gist", gistHandler)
}

func gistHandler(w http.ResponseWriter, r *http.Request) {
	log.Debug("gistHandler()")

	// get access token
	c, err := r.Cookie("access_token")
	if err != nil {
		log.Error("get cookie error: ", err)
		return
	}

	ts := oauth2.StaticTokenSource(
		&oauth2.Token{AccessToken: c.Value},
	)
	tc := oauth2.NewClient(oauth2.NoContext, ts)

	// github api
	client := github.NewClient(tc)

	gist, err := makeGist(r.Body)
	if err != nil {
		log.Errorf("make gist error: %v", err)
	}

	retGist, _, err := client.Gists.Create(&gist)
	if err != nil {
		log.Error("create gist error: ", err)
		return
	}

	log.Debug(*retGist.HTMLURL)
}

func makeGist(httpBody io.Reader) (github.Gist, error) {
	// get body
	byteBody, err := ioutil.ReadAll(httpBody)
	if err != nil {
		return github.Gist{}, fmt.Errorf("get body error: %v", err)
	}

	// unmarshal
	postGist := struct {
		FileName    string `json:"file_name"`
		Description string `json:"description"`
		Public      bool   `json:"public"`
		Code        string `json:"code"`
	}{}
	err = json.Unmarshal(byteBody, &postGist)
	if err != nil {
		return github.Gist{}, fmt.Errorf("json unmarshal error: %v", err)
	}

	// create gist
	gist := github.Gist{
		Description: &postGist.Description,
		Public:      &postGist.Public,
		Files: map[github.GistFilename]github.GistFile{
			github.GistFilename(postGist.FileName + ".go"): github.GistFile{
				Content: &postGist.Code,
			},
		},
	}

	return gist, nil
}
<commit_msg>add TrimSpace<commit_after>package main

import (
	"encoding/json"
	"fmt"
	"io"
	"io/ioutil"
	"net/http"
	"strings"

	"golang.org/x/oauth2"

	log "github.com/Sirupsen/logrus"
	"github.com/google/go-github/github"
)

func init() {
	http.HandleFunc("/gist", gistHandler)
}

func gistHandler(w http.ResponseWriter, r *http.Request) {
	log.Debug("gistHandler()")

	// get access token
	c, err := r.Cookie("access_token")
	if err != nil {
		log.Error("get cookie error: ", err)
		return
	}

	ts := oauth2.StaticTokenSource(
		&oauth2.Token{AccessToken: c.Value},
	)
	tc := oauth2.NewClient(oauth2.NoContext, ts)

	// github api
	client := github.NewClient(tc)

	gist, err := makeGist(r.Body)
	if err != nil {
		log.Errorf("make gist error: %v", err)
	}

	retGist, _, err := client.Gists.Create(&gist)
	if err != nil {
		log.Error("create gist error: ", err)
		return
	}

	log.Debug(*retGist)
}

func makeGist(httpBody io.Reader) (github.Gist, error) {
	// get body
	byteBody, err := ioutil.ReadAll(httpBody)
	if err != nil {
		return github.Gist{}, fmt.Errorf("get body error: %v", err)
	}

	// unmarshal
	postGist := struct {
		FileName    string `json:"file_name"`
		Description string `json:"description"`
		Public      bool   `json:"public"`
		Code        string `json:"code"`
	}{}
	err = json.Unmarshal(byteBody, &postGist)
	if err != nil {
		return github.Gist{}, fmt.Errorf("json unmarshal error: %v", err)
	}

	// create gist
	gist := github.Gist{
		Description: github.String(strings.TrimSpace(postGist.Description)),
		Public:      &postGist.Public,
		Files: map[github.GistFilename]github.GistFile{
			github.GistFilename(strings.TrimSpace(postGist.FileName) + ".go"): github.GistFile{
				Content: &postGist.Code,
			},
		},
	}

	return gist, nil
}
<|endoftext|>


<commit_before>package collectors

import (
	"strconv"
	"strings"

	"github.com/StackExchange/scollector/metadata"
	"github.com/StackExchange/scollector/opentsdb"
)

func init() {
	collectors = append(collectors, &IntervalCollector{F: c_conntrack_linux})
}

func c_conntrack_linux() (opentsdb.MultiDataPoint, error) {
	var md opentsdb.MultiDataPoint
	var max, count float64
	if err := readLine("/proc/sys/net/netfilter/nf_conntrack_count", func(s string) {
		values := strings.Fields(s)
		if len(values) > 0 {
			var err error
			count, err = strconv.ParseFloat(values[0], 64)
			if err != nil {
				return
			}
			Add(&md, "linux.net.conntrack.count", count, nil, metadata.Unknown, metadata.None, "")
		}
	}); err != nil {
		return nil, err
	}
	if err := readLine("/proc/sys/net/netfilter/nf_conntrack_max", func(s string) {
		values := strings.Fields(s)
		if len(values) > 0 {
			var err error
			max, err = strconv.ParseFloat(values[0], 64)
			if err != nil {
				return
			}
			Add(&md, "linux.net.conntrack.max", max, nil, metadata.Unknown, metadata.None, "")
		}
	}); err != nil {
		return nil, err
	}
	if max != 0 {
		Add(&md, "linux.net.conntrack.percent_used", count/max*100, nil, metadata.Unknown, metadata.None, "")
	}
	return md, nil
}
<commit_msg>cmd/scollector: Enable detection for conntrack<commit_after>package collectors

import (
	"os"
	"strconv"
	"strings"

	"github.com/StackExchange/scollector/metadata"
	"github.com/StackExchange/scollector/opentsdb"
)

func init() {
	collectors = append(collectors, &IntervalCollector{F: c_conntrack_linux, Enable: conntrackEnable})
}

const (
	conntrackCount = "/proc/sys/net/netfilter/nf_conntrack_count"
	conntrackMax   = "/proc/sys/net/netfilter/nf_conntrack_max"
)

func conntrackEnable() bool {
	f, err := os.Open(conntrackCount)
	defer f.Close()
	return err == nil
}

func c_conntrack_linux() (opentsdb.MultiDataPoint, error) {
	var md opentsdb.MultiDataPoint
	var max, count float64
	if err := readLine(conntrackCount, func(s string) {
		values := strings.Fields(s)
		if len(values) > 0 {
			var err error
			count, err = strconv.ParseFloat(values[0], 64)
			if err != nil {
				return
			}
			Add(&md, "linux.net.conntrack.count", count, nil, metadata.Unknown, metadata.None, "")
		}
	}); err != nil {
		return nil, err
	}
	if err := readLine(conntrackMax, func(s string) {
		values := strings.Fields(s)
		if len(values) > 0 {
			var err error
			max, err = strconv.ParseFloat(values[0], 64)
			if err != nil {
				return
			}
			Add(&md, "linux.net.conntrack.max", max, nil, metadata.Unknown, metadata.None, "")
		}
	}); err != nil {
		return nil, err
	}
	if max != 0 {
		Add(&md, "linux.net.conntrack.percent_used", count/max*100, nil, metadata.Unknown, metadata.None, "")
	}
	return md, nil
}
<|endoftext|>


<commit_before>package app

import (
	"flag"
	"fmt"
	"os"
	"sort"
	"strings"

	"github.com/fsouza/go-dockerclient"
	"github.com/involucro/involucro/ilog"
	runtime "github.com/involucro/involucro/internal"
)

// Main represents the usual main method of the
// whole program. It is moved to its own package
// to testing using go utils.
func Main(args []string) error {
	initializeFlagSet()
	flags.Init("involucro "+version, flag.ContinueOnError)
	if err := flags.Parse(args[1:]); err != nil {
		return err
	}

	if showVersion {
		fmt.Printf("involucro %s\n", versionNotice)
		return nil
	}

	if relativeWorkDir == "." {
		if val := os.Getenv("INVOLUCRO_WORKDIR"); val != "" {
			relativeWorkDir = val
		}
	}

	if remoteWrapTask != "" {
		ilog.StdLog.SetMinPrintLevel(-2)
		step := runtime.DecodeWrapStep(remoteWrapTask)
		client, err := docker.NewClient("unix:///sock")
		if err != nil {
			return err
		}

		ctx := runtime.New(make(map[string]string), client, "/")
		if err := step.Take(&ctx); err != nil {
			return err
		}
		return nil
	}

	ilog.StdLog.SetMinPrintLevel(-verbosity)

	var client *docker.Client
	var err error
	if dockerURL != defaultDockerURL {
		client, err = docker.NewClient(dockerURL)
	} else {
		client, err = docker.NewClientFromEnv()
	}
	if err != nil {
		return fmt.Errorf("Unable to create Docker client: %s", err)
	}

	if err := client.Ping(); err != nil {
		return fmt.Errorf("Docker not reachable: %s", err)
	}

	ctx := runtime.New(variables, client, relativeWorkDir)

	if controlScript != "" && isControlFileOverriden() {
		return fmt.Errorf("Specified both -e and -f")
	}

	if controlScript != "" {
		if err := ctx.RunString(controlScript); err != nil {
			return err
		}
	} else {
		filename := controlFile

		if _, err := os.Stat(filename); os.IsNotExist(err) {
			if _, err := os.Stat(filename + ".md"); err == nil {
				filename += ".md"
			}
		}

		if strings.HasSuffix(filename, ".md") {
			if err := ctx.RunLiterateFile(filename); err != nil {
				return err
			}
		} else {
			if err := ctx.RunFile(filename); err != nil {
				return err
			}
		}
	}

	if showTasks {
		tasks := ctx.TaskIDList()
		sort.Strings(tasks)
		for _, id := range tasks {
			fmt.Println(id)
		}
		return nil
	}

	for _, element := range flags.Args() {
		if err := ctx.RunTask(element); err != nil {
			return err
		}
	}
	return nil
}
<commit_msg>Slightly decreasing cyclomatic complexity<commit_after>package app

import (
	"flag"
	"fmt"
	"os"
	"sort"
	"strings"

	"github.com/fsouza/go-dockerclient"
	"github.com/involucro/involucro/ilog"
	runtime "github.com/involucro/involucro/internal"
)

// Main represents the usual main method of the
// whole program. It is moved to its own package
// to testing using go utils.
func Main(args []string) error {
	initializeFlagSet()
	flags.Init("involucro "+version, flag.ContinueOnError)
	if err := flags.Parse(args[1:]); err != nil {
		return err
	}

	if showVersion {
		fmt.Printf("involucro %s\n", versionNotice)
		return nil
	}

	if relativeWorkDir == "." {
		if val := os.Getenv("INVOLUCRO_WORKDIR"); val != "" {
			relativeWorkDir = val
		}
	}

	if remoteWrapTask != "" {
		ilog.StdLog.SetMinPrintLevel(-2)
		step := runtime.DecodeWrapStep(remoteWrapTask)
		client, err := docker.NewClient("unix:///sock")
		if err != nil {
			return err
		}

		ctx := runtime.New(make(map[string]string), client, "/")
		if err := step.Take(&ctx); err != nil {
			return err
		}
		return nil
	}

	ilog.StdLog.SetMinPrintLevel(-verbosity)

	var client *docker.Client
	var err error
	if dockerURL != defaultDockerURL {
		client, err = docker.NewClient(dockerURL)
	} else {
		client, err = docker.NewClientFromEnv()
	}
	if err != nil {
		return fmt.Errorf("Unable to create Docker client: %s", err)
	}

	if err := client.Ping(); err != nil {
		return fmt.Errorf("Docker not reachable: %s", err)
	}

	ctx := runtime.New(variables, client, relativeWorkDir)

	if controlScript != "" && isControlFileOverriden() {
		return fmt.Errorf("Specified both -e and -f")
	}

	runControlScriptOn(&ctx)

	if showTasks {
		tasks := ctx.TaskIDList()
		sort.Strings(tasks)
		for _, id := range tasks {
			fmt.Println(id)
		}
		return nil
	}

	for _, element := range flags.Args() {
		if err := ctx.RunTask(element); err != nil {
			return err
		}
	}
	return nil
}

func runControlScriptOn(ctx *runtime.Runtime) error {
	if controlScript != "" {
		return ctx.RunString(controlScript)
	}

	filename := controlFile
	if _, err := os.Stat(filename); os.IsNotExist(err) {
		filename += ".md"
	}
	if _, err := os.Stat(filename); os.IsNotExist(err) {
		return fmt.Errorf("Control file %v (and %v.md) not found", filename, filename)
	}

	if strings.HasSuffix(filename, ".md") {
		return ctx.RunLiterateFile(filename)
	}

	return ctx.RunFile(filename)
}
<|endoftext|>


<commit_before>package collectors

import (
	"regexp"
	"strconv"
	"strings"

	"github.com/StackExchange/scollector/opentsdb"
	"github.com/StackExchange/slog"
)

func init() {
	collectors = append(collectors, &IntervalCollector{F: c_procstats_linux})
}

var uptimeRE = regexp.MustCompile(`(\S+)\s+(\S+)`)
var meminfoRE = regexp.MustCompile(`(\w+):\s+(\d+)\s+(\w+)`)
var vmstatRE = regexp.MustCompile(`(\w+)\s+(\d+)`)
var statRE = regexp.MustCompile(`(\w+)\s+(.*)`)
var statCpuRE = regexp.MustCompile(`cpu(\d+)`)
var loadavgRE = regexp.MustCompile(`(\S+)\s+(\S+)\s+(\S+)\s+(\d+)/(\d+)\s+`)

var CPU_FIELDS = []string{
	"user",
	"nice",
	"system",
	"idle",
	"iowait",
	"irq",
	"softirq",
	"guest",
	"guest_nice",
}

func c_procstats_linux() opentsdb.MultiDataPoint {
	var md opentsdb.MultiDataPoint
	readLine("/proc/uptime", func(s string) {
		m := uptimeRE.FindStringSubmatch(s)
		if m == nil {
			return
		}
		Add(&md, "linux.uptime_total", m[1], nil)
		Add(&md, "linux.uptime_now", m[2], nil)
	})
	readLine("/proc/meminfo", func(s string) {
		m := meminfoRE.FindStringSubmatch(s)
		if m == nil {
			return
		}
		Add(&md, "linux.mem."+strings.ToLower(m[1]), m[2], nil)
	})
	readLine("/proc/vmstat", func(s string) {
		m := vmstatRE.FindStringSubmatch(s)
		if m == nil {
			return
		}
		switch m[1] {
		case "pgpgin", "pgpgout", "pswpin", "pswpout", "pgfault", "pgmajfault":
			Add(&md, "linux.mem."+m[1], m[2], nil)
		}
	})
	num_cores := 0
	readLine("/proc/stat", func(s string) {
		m := statRE.FindStringSubmatch(s)
		if m == nil {
			return
		}
		if strings.HasPrefix(m[1], "cpu") {
			metric_percpu := ""
			tag_cpu := ""
			cpu_m := statCpuRE.FindStringSubmatch(m[1])
			if cpu_m != nil {
				num_cores += 1
				metric_percpu = ".percpu"
				tag_cpu = cpu_m[1]
			}
			fields := strings.Fields(m[2])
			for i, value := range fields {
				if i >= len(CPU_FIELDS) {
					break
				}
				tags := opentsdb.TagSet{
					"type": CPU_FIELDS[i],
				}
				if tag_cpu != "" {
					tags["cpu"] = tag_cpu
				}
				Add(&md, "linux.cpu"+metric_percpu, value, tags)
			}
		} else if m[1] == "intr" {
			Add(&md, "linux.intr", strings.Fields(m[2])[0], nil)
		} else if m[1] == "ctxt" {
			Add(&md, "linux.ctxt", m[2], nil)
		} else if m[1] == "processes" {
			Add(&md, "linux.processes", m[2], nil)
		} else if m[1] == "procs_blocked" {
			Add(&md, "linux.procs_blocked", m[2], nil)
		}
	})
	readLine("/proc/loadavg", func(s string) {
		m := loadavgRE.FindStringSubmatch(s)
		if m == nil {
			return
		}
		Add(&md, "linux.loadavg_1_min", m[1], nil)
		Add(&md, "linux.loadavg_5_min", m[2], nil)
		Add(&md, "linux.loadavg_15_min", m[3], nil)
		Add(&md, "linux.loadavg_runnable", m[4], nil)
		Add(&md, "linux.loadavg_total_threads", m[5], nil)
	})
	readLine("/proc/sys/kernel/random/entropy_avail", func(s string) {
		Add(&md, "linux.entropy_avail", strings.TrimSpace(s), nil)
	})
	num_cpus := 0
	readLine("/proc/interrupts", func(s string) {
		cols := strings.Fields(s)
		if num_cpus == 0 {
			num_cpus = len(cols)
			return
		} else if len(cols) < 2 {
			return
		}
		irq_type := strings.TrimRight(cols[0], ":")
		if !IsAlNum(irq_type) {
			return
		}
		if IsDigit(irq_type) {
			if cols[len(cols)-2] == "PCI-MSI-edge" && strings.Contains(cols[len(cols)-1], "eth") {
				irq_type = cols[len(cols)-1]
			} else {
				// Interrupt type is just a number, ignore.
				return
			}
		}
		for i, val := range cols[1:] {
			if i >= num_cpus {
				// All values read, remaining cols contain textual description.
				break
			}
			if !IsDigit(val) {
				// Something is weird, there should only be digit values.
				slog.Infoln("interrupts: unexpected value", val)
				break
			}
			Add(&md, "linux.interrupts", val, opentsdb.TagSet{"type": irq_type, "cpu": strconv.Itoa(i)})
		}
	})
	readLine("/proc/uptime", func(s string) {
		cols := strings.Fields(s)
		if len(cols) < 2 {
			return
		}
		total_time, err := strconv.ParseFloat(cols[0], 64)
		idle_time, err := strconv.ParseFloat(cols[1], 64)
		if err != nil {
			return
		}
		if num_cores != 0 {
			Add(&md, "os.cpu", (total_time-(idle_time/float64(num_cores)))*100, nil)
		}
	})
	return md
}
<commit_msg>cmd/scollector: Add os namespace memory information for linux<commit_after>package collectors

import (
	"regexp"
	"strconv"
	"strings"

	"github.com/StackExchange/scollector/opentsdb"
	"github.com/StackExchange/slog"
)

func init() {
	collectors = append(collectors, &IntervalCollector{F: c_procstats_linux})
}

var uptimeRE = regexp.MustCompile(`(\S+)\s+(\S+)`)
var meminfoRE = regexp.MustCompile(`(\w+):\s+(\d+)\s+(\w+)`)
var vmstatRE = regexp.MustCompile(`(\w+)\s+(\d+)`)
var statRE = regexp.MustCompile(`(\w+)\s+(.*)`)
var statCpuRE = regexp.MustCompile(`cpu(\d+)`)
var loadavgRE = regexp.MustCompile(`(\S+)\s+(\S+)\s+(\S+)\s+(\d+)/(\d+)\s+`)

var CPU_FIELDS = []string{
	"user",
	"nice",
	"system",
	"idle",
	"iowait",
	"irq",
	"softirq",
	"guest",
	"guest_nice",
}

func c_procstats_linux() opentsdb.MultiDataPoint {
	var md opentsdb.MultiDataPoint
	readLine("/proc/uptime", func(s string) {
		m := uptimeRE.FindStringSubmatch(s)
		if m == nil {
			return
		}
		Add(&md, "linux.uptime_total", m[1], nil)
		Add(&md, "linux.uptime_now", m[2], nil)
	})
	mem := make(map[string]float64)
	readLine("/proc/meminfo", func(s string) {
		m := meminfoRE.FindStringSubmatch(s)
		if m == nil {
			return
		}
		i, err := strconv.ParseFloat(m[2], 64)
		if err != nil {
			slog.Errorln(err)
		}
		mem[m[1]] = i
		Add(&md, "linux.mem."+strings.ToLower(m[1]), m[2], nil)
	})
	Add(&md, "os.mem.total", int(mem["MemTotal"])*1024, nil)
	Add(&md, "os.mem.free", int(mem["MemFree"])*1024, nil)
	if mem["MemTotal"] != 0 {
		Add(&md, "os.mem.percent_free", (mem["MemFree"]+mem["Buffers"]+mem["Cached"])/mem["MemTotal"], nil)
	}

	readLine("/proc/vmstat", func(s string) {
		m := vmstatRE.FindStringSubmatch(s)
		if m == nil {
			return
		}

		switch m[1] {
		case "pgpgin", "pgpgout", "pswpin", "pswpout", "pgfault", "pgmajfault":
			Add(&md, "linux.mem."+m[1], m[2], nil)
		}
	})
	num_cores := 0
	readLine("/proc/stat", func(s string) {
		m := statRE.FindStringSubmatch(s)
		if m == nil {
			return
		}
		if strings.HasPrefix(m[1], "cpu") {
			metric_percpu := ""
			tag_cpu := ""
			cpu_m := statCpuRE.FindStringSubmatch(m[1])
			if cpu_m != nil {
				num_cores += 1
				metric_percpu = ".percpu"
				tag_cpu = cpu_m[1]
			}
			fields := strings.Fields(m[2])
			for i, value := range fields {
				if i >= len(CPU_FIELDS) {
					break
				}
				tags := opentsdb.TagSet{
					"type": CPU_FIELDS[i],
				}
				if tag_cpu != "" {
					tags["cpu"] = tag_cpu
				}
				Add(&md, "linux.cpu"+metric_percpu, value, tags)
			}
		} else if m[1] == "intr" {
			Add(&md, "linux.intr", strings.Fields(m[2])[0], nil)
		} else if m[1] == "ctxt" {
			Add(&md, "linux.ctxt", m[2], nil)
		} else if m[1] == "processes" {
			Add(&md, "linux.processes", m[2], nil)
		} else if m[1] == "procs_blocked" {
			Add(&md, "linux.procs_blocked", m[2], nil)
		}
	})
	readLine("/proc/loadavg", func(s string) {
		m := loadavgRE.FindStringSubmatch(s)
		if m == nil {
			return
		}
		Add(&md, "linux.loadavg_1_min", m[1], nil)
		Add(&md, "linux.loadavg_5_min", m[2], nil)
		Add(&md, "linux.loadavg_15_min", m[3], nil)
		Add(&md, "linux.loadavg_runnable", m[4], nil)
		Add(&md, "linux.loadavg_total_threads", m[5], nil)
	})
	readLine("/proc/sys/kernel/random/entropy_avail", func(s string) {
		Add(&md, "linux.entropy_avail", strings.TrimSpace(s), nil)
	})
	num_cpus := 0
	readLine("/proc/interrupts", func(s string) {
		cols := strings.Fields(s)
		if num_cpus == 0 {
			num_cpus = len(cols)
			return
		} else if len(cols) < 2 {
			return
		}
		irq_type := strings.TrimRight(cols[0], ":")
		if !IsAlNum(irq_type) {
			return
		}
		if IsDigit(irq_type) {
			if cols[len(cols)-2] == "PCI-MSI-edge" && strings.Contains(cols[len(cols)-1], "eth") {
				irq_type = cols[len(cols)-1]
			} else {
				// Interrupt type is just a number, ignore.
				return
			}
		}
		for i, val := range cols[1:] {
			if i >= num_cpus {
				// All values read, remaining cols contain textual description.
				break
			}
			if !IsDigit(val) {
				// Something is weird, there should only be digit values.
				slog.Infoln("interrupts: unexpected value", val)
				break
			}
			Add(&md, "linux.interrupts", val, opentsdb.TagSet{"type": irq_type, "cpu": strconv.Itoa(i)})
		}
	})
	readLine("/proc/uptime", func(s string) {
		cols := strings.Fields(s)
		if len(cols) < 2 {
			return
		}
		total_time, err := strconv.ParseFloat(cols[0], 64)
		idle_time, err := strconv.ParseFloat(cols[1], 64)
		if err != nil {
			return
		}
		if num_cores != 0 {
			Add(&md, "os.cpu", (total_time-(idle_time/float64(num_cores)))*100, nil)
		}
	})
	return md
}
<|endoftext|>


<commit_before>package main

import (
	"fmt"
	"io"
	"os"
	"os/exec"
	"path/filepath"

	functions "github.com/iron-io/functions_go"
	"github.com/urfave/cli"
)

func publish() cli.Command {
	cmd := publishcmd{
		commoncmd: &commoncmd{},
		RoutesApi: functions.NewRoutesApi(),
	}
	var flags []cli.Flag
	flags = append(flags, cmd.flags()...)
	flags = append(flags, cmd.commoncmd.flags()...)
	flags = append(flags, confFlags(&cmd.Configuration)...)
	return cli.Command{
		Name:   "publish",
		Usage:  "scan local directory for functions, build and publish them.",
		Flags:  flags,
		Action: cmd.scan,
	}
}

type publishcmd struct {
	*commoncmd
	*functions.RoutesApi

	dry      bool
	skippush bool
}

func (p *publishcmd) flags() []cli.Flag {
	return []cli.Flag{
		cli.BoolFlag{
			Name:        "skip-push",
			Usage:       "does not push Docker built images onto Docker Hub - useful for local development.",
			Destination: &p.skippush,
		},
	}
}

func (p *publishcmd) scan(c *cli.Context) error {
	p.commoncmd.scan(p.walker)
	return nil
}

func (p *publishcmd) walker(path string, info os.FileInfo, err error, w io.Writer) error {
	walker(path, info, err, w, p.publish)
	return nil
}

// publish will take the found function and check for the presence of a
// Dockerfile, and run a three step process: parse functions file, build and
// push the container, and finally it will update function's route. Optionally,
// the route can be overriden inside the functions file.
func (p *publishcmd) publish(path string) error {
	fmt.Fprintln(p.verbwriter, "publishing", path)

	funcfile, err := p.buildfunc(path)
	if err != nil {
		return err
	}

	if p.skippush {
		return nil
	}

	if err := p.dockerpush(funcfile.Image); err != nil {
		return err
	}

	if err := p.route(path, funcfile); err != nil {
		return err
	}

	return nil
}

func (p publishcmd) dockerpush(image string) error {
	cmd := exec.Command("docker", "push", image)
	cmd.Stderr = p.verbwriter
	cmd.Stdout = p.verbwriter
	if err := cmd.Run(); err != nil {
		return fmt.Errorf("error running docker push: %v", err)
	}
	return nil
}

func (p *publishcmd) route(path string, ff *funcfile) error {
	resetBasePath(&p.Configuration)

	an, r := extractAppNameRoute(path)
	if ff.App == nil {
		ff.App = &an
	}
	if ff.Route == nil {
		ff.Route = &r
	}

	body := functions.RouteWrapper{
		Route: functions.Route{
			Path:  *ff.Route,
			Image: ff.Image,
		},
	}

	fmt.Fprintf(p.verbwriter, "updating API with appName: %s route: %s image: %s \n", *ff.App, *ff.Route, ff.Image)

	_, _, err := p.AppsAppRoutesPost(*ff.App, body)
	if err != nil {
		return fmt.Errorf("error getting routes: %v", err)
	}

	return nil
}

func extractAppNameRoute(path string) (appName, route string) {

	// The idea here is to extract the root-most directory name
	// as application name, it turns out that stdlib tools are great to
	// extract the deepest one. Thus, we revert the string and use the
	// stdlib as it is - and revert back to its normal content. Not fastest
	// ever, but it is simple.

	rpath := reverse(path)
	rroute, rappName := filepath.Split(rpath)
	route = filepath.Dir(reverse(rroute))
	return reverse(rappName), route
}

func reverse(s string) string {
	r := []rune(s)
	for i, j := 0, len(r)-1; i < len(r)/2; i, j = i+1, j-1 {
		r[i], r[j] = r[j], r[i]
	}
	return string(r)
}
<commit_msg>fnctl: remove unused field in datastructure<commit_after>package main

import (
	"fmt"
	"io"
	"os"
	"os/exec"
	"path/filepath"

	functions "github.com/iron-io/functions_go"
	"github.com/urfave/cli"
)

func publish() cli.Command {
	cmd := publishcmd{
		commoncmd: &commoncmd{},
		RoutesApi: functions.NewRoutesApi(),
	}
	var flags []cli.Flag
	flags = append(flags, cmd.flags()...)
	flags = append(flags, cmd.commoncmd.flags()...)
	flags = append(flags, confFlags(&cmd.Configuration)...)
	return cli.Command{
		Name:   "publish",
		Usage:  "scan local directory for functions, build and publish them.",
		Flags:  flags,
		Action: cmd.scan,
	}
}

type publishcmd struct {
	*commoncmd
	*functions.RoutesApi

	skippush bool
}

func (p *publishcmd) flags() []cli.Flag {
	return []cli.Flag{
		cli.BoolFlag{
			Name:        "skip-push",
			Usage:       "does not push Docker built images onto Docker Hub - useful for local development.",
			Destination: &p.skippush,
		},
	}
}

func (p *publishcmd) scan(c *cli.Context) error {
	p.commoncmd.scan(p.walker)
	return nil
}

func (p *publishcmd) walker(path string, info os.FileInfo, err error, w io.Writer) error {
	walker(path, info, err, w, p.publish)
	return nil
}

// publish will take the found function and check for the presence of a
// Dockerfile, and run a three step process: parse functions file, build and
// push the container, and finally it will update function's route. Optionally,
// the route can be overriden inside the functions file.
func (p *publishcmd) publish(path string) error {
	fmt.Fprintln(p.verbwriter, "publishing", path)

	funcfile, err := p.buildfunc(path)
	if err != nil {
		return err
	}

	if p.skippush {
		return nil
	}

	if err := p.dockerpush(funcfile.Image); err != nil {
		return err
	}

	if err := p.route(path, funcfile); err != nil {
		return err
	}

	return nil
}

func (p publishcmd) dockerpush(image string) error {
	cmd := exec.Command("docker", "push", image)
	cmd.Stderr = p.verbwriter
	cmd.Stdout = p.verbwriter
	if err := cmd.Run(); err != nil {
		return fmt.Errorf("error running docker push: %v", err)
	}
	return nil
}

func (p *publishcmd) route(path string, ff *funcfile) error {
	resetBasePath(&p.Configuration)

	an, r := extractAppNameRoute(path)
	if ff.App == nil {
		ff.App = &an
	}
	if ff.Route == nil {
		ff.Route = &r
	}

	body := functions.RouteWrapper{
		Route: functions.Route{
			Path:  *ff.Route,
			Image: ff.Image,
		},
	}

	fmt.Fprintf(p.verbwriter, "updating API with appName: %s route: %s image: %s \n", *ff.App, *ff.Route, ff.Image)

	_, _, err := p.AppsAppRoutesPost(*ff.App, body)
	if err != nil {
		return fmt.Errorf("error getting routes: %v", err)
	}

	return nil
}

func extractAppNameRoute(path string) (appName, route string) {

	// The idea here is to extract the root-most directory name
	// as application name, it turns out that stdlib tools are great to
	// extract the deepest one. Thus, we revert the string and use the
	// stdlib as it is - and revert back to its normal content. Not fastest
	// ever, but it is simple.

	rpath := reverse(path)
	rroute, rappName := filepath.Split(rpath)
	route = filepath.Dir(reverse(rroute))
	return reverse(rappName), route
}

func reverse(s string) string {
	r := []rune(s)
	for i, j := 0, len(r)-1; i < len(r)/2; i, j = i+1, j-1 {
		r[i], r[j] = r[j], r[i]
	}
	return string(r)
}
<|endoftext|>


<commit_before>/*
Copyright 2017 The Kubernetes Authors.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package main

import (
	"flag"
	"time"

	"github.com/golang/glog"
	kube_flag "k8s.io/apiserver/pkg/util/flag"
	"k8s.io/autoscaler/vertical-pod-autoscaler/common"
	vpa_clientset "k8s.io/autoscaler/vertical-pod-autoscaler/pkg/client/clientset/versioned"
	updater "k8s.io/autoscaler/vertical-pod-autoscaler/pkg/updater/logic"
	"k8s.io/autoscaler/vertical-pod-autoscaler/pkg/utils/metrics"
	metrics_updater "k8s.io/autoscaler/vertical-pod-autoscaler/pkg/utils/metrics/updater"
	vpa_api_util "k8s.io/autoscaler/vertical-pod-autoscaler/pkg/utils/vpa"
	kube_client "k8s.io/client-go/kubernetes"
	kube_restclient "k8s.io/client-go/rest"
)

var (
	updaterInterval = flag.Duration("updater-interval", 1*time.Minute,
		`How often updater should run`)

	minReplicas = flag.Int("min-replicas", 2,
		`Minimum number of replicas to perform update`)

	evictionToleranceFraction = flag.Float64("eviction-tolerance", 0.5,
		`Fraction of replica count that can be evicted for update, if more than one pod can be evicted.`)

	address = flag.String("address", ":8943", "The address to expose Prometheus metrics.")
)

func main() {
	kube_flag.InitFlags()
	glog.V(1).Infof("Vertical Pod Autoscaler %s Updater", common.VerticalPodAutoscalerVersion)

	healthCheck := metrics.NewHealthCheck(*updaterInterval*5, true)
	metrics.Initialize(*address, healthCheck)
	metrics_updater.Register()

	kubeClient, vpaClient := createKubeClients()
	updater, err := updater.NewUpdater(kubeClient, vpaClient, *minReplicas, *evictionToleranceFraction, vpa_api_util.NewCappingRecommendationProcessor(), nil)
	if err != nil {
		glog.Fatalf("Failed to create updater: %v")
	}
	ticker := time.Tick(*updaterInterval)
	for range ticker {
		updater.RunOnce()
		healthCheck.UpdateLastActivity()
	}
}

func createKubeClients() (kube_client.Interface, *vpa_clientset.Clientset) {
	config, err := kube_restclient.InClusterConfig()
	if err != nil {
		glog.Fatalf("Failed to build Kubernetes client : fail to create config: %v", err)
	}
	return kube_client.NewForConfigOrDie(config), vpa_clientset.NewForConfigOrDie(config)
}
<commit_msg>Fix Fatalf format string<commit_after>/*
Copyright 2017 The Kubernetes Authors.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package main

import (
	"flag"
	"time"

	"github.com/golang/glog"
	kube_flag "k8s.io/apiserver/pkg/util/flag"
	"k8s.io/autoscaler/vertical-pod-autoscaler/common"
	vpa_clientset "k8s.io/autoscaler/vertical-pod-autoscaler/pkg/client/clientset/versioned"
	updater "k8s.io/autoscaler/vertical-pod-autoscaler/pkg/updater/logic"
	"k8s.io/autoscaler/vertical-pod-autoscaler/pkg/utils/metrics"
	metrics_updater "k8s.io/autoscaler/vertical-pod-autoscaler/pkg/utils/metrics/updater"
	vpa_api_util "k8s.io/autoscaler/vertical-pod-autoscaler/pkg/utils/vpa"
	kube_client "k8s.io/client-go/kubernetes"
	kube_restclient "k8s.io/client-go/rest"
)

var (
	updaterInterval = flag.Duration("updater-interval", 1*time.Minute,
		`How often updater should run`)

	minReplicas = flag.Int("min-replicas", 2,
		`Minimum number of replicas to perform update`)

	evictionToleranceFraction = flag.Float64("eviction-tolerance", 0.5,
		`Fraction of replica count that can be evicted for update, if more than one pod can be evicted.`)

	address = flag.String("address", ":8943", "The address to expose Prometheus metrics.")
)

func main() {
	kube_flag.InitFlags()
	glog.V(1).Infof("Vertical Pod Autoscaler %s Updater", common.VerticalPodAutoscalerVersion)

	healthCheck := metrics.NewHealthCheck(*updaterInterval*5, true)
	metrics.Initialize(*address, healthCheck)
	metrics_updater.Register()

	kubeClient, vpaClient := createKubeClients()
	updater, err := updater.NewUpdater(kubeClient, vpaClient, *minReplicas, *evictionToleranceFraction, vpa_api_util.NewCappingRecommendationProcessor(), nil)
	if err != nil {
		glog.Fatalf("Failed to create updater: %v", err)
	}
	ticker := time.Tick(*updaterInterval)
	for range ticker {
		updater.RunOnce()
		healthCheck.UpdateLastActivity()
	}
}

func createKubeClients() (kube_client.Interface, *vpa_clientset.Clientset) {
	config, err := kube_restclient.InClusterConfig()
	if err != nil {
		glog.Fatalf("Failed to build Kubernetes client : fail to create config: %v", err)
	}
	return kube_client.NewForConfigOrDie(config), vpa_clientset.NewForConfigOrDie(config)
}
<|endoftext|>


<commit_before>package kubernetes

import (
	"fmt"
	"log"

	"github.com/hashicorp/terraform/helper/schema"
	"k8s.io/apimachinery/pkg/api/errors"
	meta_v1 "k8s.io/apimachinery/pkg/apis/meta/v1"
	pkgApi "k8s.io/apimachinery/pkg/types"
	api "k8s.io/kubernetes/pkg/apis/autoscaling/v1"
	kubernetes "k8s.io/kubernetes/pkg/client/clientset_generated/clientset"
)

func resourceKubernetesHorizontalPodAutoscaler() *schema.Resource {
	return &schema.Resource{
		Create: resourceKubernetesHorizontalPodAutoscalerCreate,
		Read:   resourceKubernetesHorizontalPodAutoscalerRead,
		Exists: resourceKubernetesHorizontalPodAutoscalerExists,
		Update: resourceKubernetesHorizontalPodAutoscalerUpdate,
		Delete: resourceKubernetesHorizontalPodAutoscalerDelete,
		Importer: &schema.ResourceImporter{
			State: schema.ImportStatePassthrough,
		},

		Schema: map[string]*schema.Schema{
			"metadata": namespacedMetadataSchema("horizontal pod autoscaler", true),
			"spec": {
				Type:        schema.TypeList,
				Description: "Behaviour of the autoscaler. More info: http://releases.k8s.io/HEAD/docs/devel/api-conventions.md#spec-and-status.",
				Required:    true,
				MaxItems:    1,
				Elem: &schema.Resource{
					Schema: map[string]*schema.Schema{
						"max_replicas": {
							Type:        schema.TypeInt,
							Description: "Upper limit for the number of pods that can be set by the autoscaler.",
							Required:    true,
						},
						"min_replicas": {
							Type:        schema.TypeInt,
							Description: "Lower limit for the number of pods that can be set by the autoscaler, defaults to `1`.",
							Optional:    true,
							Default:     1,
						},
						"scale_target_ref": {
							Type:        schema.TypeList,
							Description: "Reference to scaled resource. e.g. Replication Controller",
							Required:    true,
							MaxItems:    1,
							Elem: &schema.Resource{
								Schema: map[string]*schema.Schema{
									"api_version": {
										Type:        schema.TypeString,
										Description: "API version of the referent",
										Optional:    true,
									},
									"kind": {
										Type:        schema.TypeString,
										Description: "Kind of the referent. e.g. `ReplicationController`. More info: http://releases.k8s.io/HEAD/docs/devel/api-conventions.md#types-kinds",
										Required:    true,
									},
									"name": {
										Type:        schema.TypeString,
										Description: "Name of the referent. More info: http://kubernetes.io/docs/user-guide/identifiers#names",
										Required:    true,
									},
								},
							},
						},
						"target_cpu_utilization_percentage": {
							Type:        schema.TypeInt,
							Description: "Target average CPU utilization (represented as a percentage of requested CPU) over all the pods. If not specified the default autoscaling policy will be used.",
							Optional:    true,
						},
					},
				},
			},
		},
	}
}

func resourceKubernetesHorizontalPodAutoscalerCreate(d *schema.ResourceData, meta interface{}) error {
	conn := meta.(*kubernetes.Clientset)

	metadata := expandMetadata(d.Get("metadata").([]interface{}))
	svc := api.HorizontalPodAutoscaler{
		ObjectMeta: metadata,
		Spec:       expandHorizontalPodAutoscalerSpec(d.Get("spec").([]interface{})),
	}
	log.Printf("[INFO] Creating new horizontal pod autoscaler: %#v", svc)
	out, err := conn.AutoscalingV1().HorizontalPodAutoscalers(metadata.Namespace).Create(&svc)
	if err != nil {
		return err
	}

	log.Printf("[INFO] Submitted new horizontal pod autoscaler: %#v", out)
	d.SetId(buildId(out.ObjectMeta))

	return resourceKubernetesHorizontalPodAutoscalerRead(d, meta)
}

func resourceKubernetesHorizontalPodAutoscalerRead(d *schema.ResourceData, meta interface{}) error {
	conn := meta.(*kubernetes.Clientset)

	namespace, name := idParts(d.Id())
	log.Printf("[INFO] Reading horizontal pod autoscaler %s", name)
	svc, err := conn.AutoscalingV1().HorizontalPodAutoscalers(namespace).Get(name, meta_v1.GetOptions{})
	if err != nil {
		log.Printf("[DEBUG] Received error: %#v", err)
		return err
	}
	log.Printf("[INFO] Received horizontal pod autoscaler: %#v", svc)
	err = d.Set("metadata", flattenMetadata(svc.ObjectMeta))
	if err != nil {
		return err
	}

	flattened := flattenHorizontalPodAutoscalerSpec(svc.Spec)
	log.Printf("[DEBUG] Flattened horizontal pod autoscaler spec: %#v", flattened)
	err = d.Set("spec", flattened)
	if err != nil {
		return err
	}

	return nil
}

func resourceKubernetesHorizontalPodAutoscalerUpdate(d *schema.ResourceData, meta interface{}) error {
	conn := meta.(*kubernetes.Clientset)

	namespace, name := idParts(d.Id())

	ops := patchMetadata("metadata.0.", "/metadata/", d)
	if d.HasChange("spec") {
		diffOps := patchHorizontalPodAutoscalerSpec("spec.0.", "/spec", d)
		ops = append(ops, diffOps...)
	}
	data, err := ops.MarshalJSON()
	if err != nil {
		return fmt.Errorf("Failed to marshal update operations: %s", err)
	}
	log.Printf("[INFO] Updating horizontal pod autoscaler %q: %v", name, string(data))
	out, err := conn.AutoscalingV1().HorizontalPodAutoscalers(namespace).Patch(name, pkgApi.JSONPatchType, data)
	if err != nil {
		return fmt.Errorf("Failed to update horizontal pod autoscaler: %s", err)
	}
	log.Printf("[INFO] Submitted updated horizontal pod autoscaler: %#v", out)
	d.SetId(buildId(out.ObjectMeta))

	return resourceKubernetesHorizontalPodAutoscalerRead(d, meta)
}

func resourceKubernetesHorizontalPodAutoscalerDelete(d *schema.ResourceData, meta interface{}) error {
	conn := meta.(*kubernetes.Clientset)

	namespace, name := idParts(d.Id())
	log.Printf("[INFO] Deleting horizontal pod autoscaler: %#v", name)
	err := conn.AutoscalingV1().HorizontalPodAutoscalers(namespace).Delete(name, &meta_v1.DeleteOptions{})
	if err != nil {
		return err
	}

	log.Printf("[INFO] Horizontal Pod Autoscaler %s deleted", name)

	d.SetId("")
	return nil
}

func resourceKubernetesHorizontalPodAutoscalerExists(d *schema.ResourceData, meta interface{}) (bool, error) {
	conn := meta.(*kubernetes.Clientset)

	namespace, name := idParts(d.Id())
	log.Printf("[INFO] Checking horizontal pod autoscaler %s", name)
	_, err := conn.AutoscalingV1().HorizontalPodAutoscalers(namespace).Get(name, meta_v1.GetOptions{})
	if err != nil {
		if statusErr, ok := err.(*errors.StatusError); ok && statusErr.ErrStatus.Code == 404 {
			return false, nil
		}
		log.Printf("[DEBUG] Received error: %#v", err)
	}
	return true, err
}
<commit_msg>provider/kubernetes: Mark HPA's target_cpu_utilization_percentage as computed<commit_after>package kubernetes

import (
	"fmt"
	"log"

	"github.com/hashicorp/terraform/helper/schema"
	"k8s.io/apimachinery/pkg/api/errors"
	meta_v1 "k8s.io/apimachinery/pkg/apis/meta/v1"
	pkgApi "k8s.io/apimachinery/pkg/types"
	api "k8s.io/kubernetes/pkg/apis/autoscaling/v1"
	kubernetes "k8s.io/kubernetes/pkg/client/clientset_generated/clientset"
)

func resourceKubernetesHorizontalPodAutoscaler() *schema.Resource {
	return &schema.Resource{
		Create: resourceKubernetesHorizontalPodAutoscalerCreate,
		Read:   resourceKubernetesHorizontalPodAutoscalerRead,
		Exists: resourceKubernetesHorizontalPodAutoscalerExists,
		Update: resourceKubernetesHorizontalPodAutoscalerUpdate,
		Delete: resourceKubernetesHorizontalPodAutoscalerDelete,
		Importer: &schema.ResourceImporter{
			State: schema.ImportStatePassthrough,
		},

		Schema: map[string]*schema.Schema{
			"metadata": namespacedMetadataSchema("horizontal pod autoscaler", true),
			"spec": {
				Type:        schema.TypeList,
				Description: "Behaviour of the autoscaler. More info: http://releases.k8s.io/HEAD/docs/devel/api-conventions.md#spec-and-status.",
				Required:    true,
				MaxItems:    1,
				Elem: &schema.Resource{
					Schema: map[string]*schema.Schema{
						"max_replicas": {
							Type:        schema.TypeInt,
							Description: "Upper limit for the number of pods that can be set by the autoscaler.",
							Required:    true,
						},
						"min_replicas": {
							Type:        schema.TypeInt,
							Description: "Lower limit for the number of pods that can be set by the autoscaler, defaults to `1`.",
							Optional:    true,
							Default:     1,
						},
						"scale_target_ref": {
							Type:        schema.TypeList,
							Description: "Reference to scaled resource. e.g. Replication Controller",
							Required:    true,
							MaxItems:    1,
							Elem: &schema.Resource{
								Schema: map[string]*schema.Schema{
									"api_version": {
										Type:        schema.TypeString,
										Description: "API version of the referent",
										Optional:    true,
									},
									"kind": {
										Type:        schema.TypeString,
										Description: "Kind of the referent. e.g. `ReplicationController`. More info: http://releases.k8s.io/HEAD/docs/devel/api-conventions.md#types-kinds",
										Required:    true,
									},
									"name": {
										Type:        schema.TypeString,
										Description: "Name of the referent. More info: http://kubernetes.io/docs/user-guide/identifiers#names",
										Required:    true,
									},
								},
							},
						},
						"target_cpu_utilization_percentage": {
							Type:        schema.TypeInt,
							Description: "Target average CPU utilization (represented as a percentage of requested CPU) over all the pods. If not specified the default autoscaling policy will be used.",
							Optional:    true,
							Computed:    true,
						},
					},
				},
			},
		},
	}
}

func resourceKubernetesHorizontalPodAutoscalerCreate(d *schema.ResourceData, meta interface{}) error {
	conn := meta.(*kubernetes.Clientset)

	metadata := expandMetadata(d.Get("metadata").([]interface{}))
	svc := api.HorizontalPodAutoscaler{
		ObjectMeta: metadata,
		Spec:       expandHorizontalPodAutoscalerSpec(d.Get("spec").([]interface{})),
	}
	log.Printf("[INFO] Creating new horizontal pod autoscaler: %#v", svc)
	out, err := conn.AutoscalingV1().HorizontalPodAutoscalers(metadata.Namespace).Create(&svc)
	if err != nil {
		return err
	}

	log.Printf("[INFO] Submitted new horizontal pod autoscaler: %#v", out)
	d.SetId(buildId(out.ObjectMeta))

	return resourceKubernetesHorizontalPodAutoscalerRead(d, meta)
}

func resourceKubernetesHorizontalPodAutoscalerRead(d *schema.ResourceData, meta interface{}) error {
	conn := meta.(*kubernetes.Clientset)

	namespace, name := idParts(d.Id())
	log.Printf("[INFO] Reading horizontal pod autoscaler %s", name)
	svc, err := conn.AutoscalingV1().HorizontalPodAutoscalers(namespace).Get(name, meta_v1.GetOptions{})
	if err != nil {
		log.Printf("[DEBUG] Received error: %#v", err)
		return err
	}
	log.Printf("[INFO] Received horizontal pod autoscaler: %#v", svc)
	err = d.Set("metadata", flattenMetadata(svc.ObjectMeta))
	if err != nil {
		return err
	}

	flattened := flattenHorizontalPodAutoscalerSpec(svc.Spec)
	log.Printf("[DEBUG] Flattened horizontal pod autoscaler spec: %#v", flattened)
	err = d.Set("spec", flattened)
	if err != nil {
		return err
	}

	return nil
}

func resourceKubernetesHorizontalPodAutoscalerUpdate(d *schema.ResourceData, meta interface{}) error {
	conn := meta.(*kubernetes.Clientset)

	namespace, name := idParts(d.Id())

	ops := patchMetadata("metadata.0.", "/metadata/", d)
	if d.HasChange("spec") {
		diffOps := patchHorizontalPodAutoscalerSpec("spec.0.", "/spec", d)
		ops = append(ops, diffOps...)
	}
	data, err := ops.MarshalJSON()
	if err != nil {
		return fmt.Errorf("Failed to marshal update operations: %s", err)
	}
	log.Printf("[INFO] Updating horizontal pod autoscaler %q: %v", name, string(data))
	out, err := conn.AutoscalingV1().HorizontalPodAutoscalers(namespace).Patch(name, pkgApi.JSONPatchType, data)
	if err != nil {
		return fmt.Errorf("Failed to update horizontal pod autoscaler: %s", err)
	}
	log.Printf("[INFO] Submitted updated horizontal pod autoscaler: %#v", out)
	d.SetId(buildId(out.ObjectMeta))

	return resourceKubernetesHorizontalPodAutoscalerRead(d, meta)
}

func resourceKubernetesHorizontalPodAutoscalerDelete(d *schema.ResourceData, meta interface{}) error {
	conn := meta.(*kubernetes.Clientset)

	namespace, name := idParts(d.Id())
	log.Printf("[INFO] Deleting horizontal pod autoscaler: %#v", name)
	err := conn.AutoscalingV1().HorizontalPodAutoscalers(namespace).Delete(name, &meta_v1.DeleteOptions{})
	if err != nil {
		return err
	}

	log.Printf("[INFO] Horizontal Pod Autoscaler %s deleted", name)

	d.SetId("")
	return nil
}

func resourceKubernetesHorizontalPodAutoscalerExists(d *schema.ResourceData, meta interface{}) (bool, error) {
	conn := meta.(*kubernetes.Clientset)

	namespace, name := idParts(d.Id())
	log.Printf("[INFO] Checking horizontal pod autoscaler %s", name)
	_, err := conn.AutoscalingV1().HorizontalPodAutoscalers(namespace).Get(name, meta_v1.GetOptions{})
	if err != nil {
		if statusErr, ok := err.(*errors.StatusError); ok && statusErr.ErrStatus.Code == 404 {
			return false, nil
		}
		log.Printf("[DEBUG] Received error: %#v", err)
	}
	return true, err
}
<|endoftext|>