<commit_before>package config

import (
	"fmt"
	"os"
	"strings"

	"github.com/caarlos0/env"
	iom "github.com/grokify/gotilla/io/ioutilmore"
	"github.com/joho/godotenv"
)

// EnvFileToJSONFile Converts an .env file to a JSON file using the definition
// provided in data.
func EnvFileToJSONFile(data interface{}, filepathENV, filepathJSON string, perm os.FileMode, pretty bool) error {
	err := godotenv.Load(filepathENV)
	if err != nil {
		return err
	}

	err = env.Parse(data)
	if err != nil {
		return err
	}

	return iom.WriteFileJSON(filepathJSON, data, perm, pretty)
}

// Return a merged environment var which is split into multiple
// vars. This is useful when the system has a size limit on
// environment variables.
func JoinEnvNumbered(prefix, delimiter string, startInt uint8, includeBase bool) string {
	vals := []string{}
	if includeBase {
		val := os.Getenv(prefix)
		if len(val) > 0 {
			vals = append(vals, val)
		}
	}
	i := startInt
	for {
		val := os.Getenv(fmt.Sprintf("%s_%d", prefix, i))
		if len(val) > 0 {
			vals = append(vals, val)
		} else {
			break
		}
		i++
	}
	return strings.Join(vals, delimiter)
}
<commit_msg>update config/env/EnvFileToJSONFile<commit_after>package config

import (
	"fmt"
	"os"
	"strings"

	"github.com/caarlos0/env"
	iom "github.com/grokify/gotilla/io/ioutilmore"
	"github.com/joho/godotenv"
)

// EnvFileToJSONFile Converts an .env file to a JSON file using the definition
// provided in data.
func EnvFileToJSONFile(data interface{}, filepathENV, filepathJSON string, perm os.FileMode, prefix, indent string) error {
	err := godotenv.Load(filepathENV)
	if err != nil {
		return err
	}

	err = env.Parse(data)
	if err != nil {
		return err
	}

	return iom.WriteFileJSON(filepathJSON, data, perm, prefix, indent)
}

// Return a merged environment var which is split into multiple
// vars. This is useful when the system has a size limit on
// environment variables.
func JoinEnvNumbered(prefix, delimiter string, startInt uint8, includeBase bool) string {
	vals := []string{}
	if includeBase {
		val := os.Getenv(prefix)
		if len(val) > 0 {
			vals = append(vals, val)
		}
	}
	i := startInt
	for {
		val := os.Getenv(fmt.Sprintf("%s_%d", prefix, i))
		if len(val) > 0 {
			vals = append(vals, val)
		} else {
			break
		}
		i++
	}
	return strings.Join(vals, delimiter)
}
<|endoftext|>


<commit_before>package apps

import (
	"archive/tar"
	"bytes"
	"context"
	"errors"
	"fmt"
	"io"
	"io/ioutil"
	"net/http"
	"net/url"
	"os"
	"os/exec"
	"path/filepath"
	"regexp"
	"strings"
	"time"

	git "github.com/cozy/go-git"
	gitPlumbing "github.com/cozy/go-git/plumbing"
	gitObject "github.com/cozy/go-git/plumbing/object"
	gitStorage "github.com/cozy/go-git/storage/filesystem"
	"github.com/sirupsen/logrus"
	"github.com/spf13/afero"
	gitOsFS "gopkg.in/src-d/go-billy.v2/osfs"
)

var errCloneTimeout = errors.New("git: repository cloning timed out")
var cloneTimeout = 20 * time.Second

const (
	ghRawManifestURL = "https://raw.githubusercontent.com/%s/%s/%s/%s"
	glRawManifestURL = "https://%s/%s/%s/raw/%s/%s"
)

var (
	// ghURLRegex is used to identify github
	ghURLRegex = regexp.MustCompile(`/([^/]+)/([^/]+).git`)
	// glURLRegex is used to identify gitlab
	glURLRegex = regexp.MustCompile(`/([^/]+)/([^/]+).git`)
)

type gitFetcher struct {
	manFilename string
	log         *logrus.Entry
}

func newGitFetcher(appType AppType, log *logrus.Entry) *gitFetcher {
	var manFilename string
	switch appType {
	case Webapp:
		manFilename = WebappManifestName
	case Konnector:
		manFilename = KonnectorManifestName
	}
	return &gitFetcher{
		manFilename: manFilename,
		log:         log,
	}
}

var manifestClient = &http.Client{
	Timeout: 60 * time.Second,
}

func isGithub(src *url.URL) bool {
	return src.Host == "github.com"
}

func isGitlab(src *url.URL) bool {
	return src.Host == "framagit.org" || strings.Contains(src.Host, "gitlab")
}

func (g *gitFetcher) FetchManifest(src *url.URL) (r io.ReadCloser, err error) {
	defer func() {
		if err != nil {
			g.log.Errorf("[git] Error while fetching app manifest %s: %s",
				src.String(), err.Error())
		}
	}()

	if src.Scheme == "git+ssh" || src.Scheme == "ssh+git" {
		return g.fetchManifestFromGitArchive(src)
	}

	var u string
	if isGithub(src) {
		u, err = resolveGithubURL(src, g.manFilename)
	} else if isGitlab(src) {
		u, err = resolveGitlabURL(src, g.manFilename)
	} else {
		u, err = resolveManifestURL(src, g.manFilename)
	}
	if err != nil {
		return nil, err
	}

	g.log.Infof("[git] Fetching manifest on %s", u)
	res, err := manifestClient.Get(u)
	if err != nil || res.StatusCode != 200 {
		g.log.Errorf("[git] Error while fetching manifest on %s", u)
		return nil, ErrManifestNotReachable
	}

	return res.Body, nil
}

// Use the git archive method to download a manifest from the git repository.
func (g *gitFetcher) fetchManifestFromGitArchive(src *url.URL) (io.ReadCloser, error) {
	var branch string
	src, branch = getRemoteURL(src)
	ctx, cancel := context.WithTimeout(context.Background(), cloneTimeout)
	defer cancel()
	cmd := exec.CommandContext(ctx, "git",
		"archive",
		"--remote", src.String(),
		fmt.Sprintf("refs/heads/%s", branch),
		g.manFilename) // #nosec
	stdout, err := cmd.CombinedOutput()
	if err != nil {
		if err == exec.ErrNotFound {
			return nil, ErrNotSupportedSource
		}
		return nil, ErrManifestNotReachable
	}
	buf := new(bytes.Buffer)
	r := tar.NewReader(bytes.NewReader(stdout))
	for {
		h, err := r.Next()
		if err == io.EOF {
			break
		}
		if err != nil {
			return nil, ErrManifestNotReachable
		}
		if h.Name != g.manFilename {
			continue
		}
		if _, err = io.Copy(buf, r); err != nil {
			return nil, ErrManifestNotReachable
		}
	}
	return ioutil.NopCloser(buf), nil
}

func (g *gitFetcher) Fetch(src *url.URL, fs Copier, man Manifest) (err error) {
	defer func() {
		if err != nil {
			g.log.Errorf("[git] Error while fetching or copying repository %s: %s",
				src.String(), err.Error())
		}
	}()

	osFs := afero.NewOsFs()
	gitDir, err := afero.TempDir(osFs, "", "cozy-app-"+man.Slug())
	if err != nil {
		return err
	}
	defer osFs.RemoveAll(gitDir)

	gitFs := afero.NewBasePathFs(osFs, gitDir)
	// XXX Gitlab doesn't support the git protocol
	if src.Scheme == "git" && isGitlab(src) {
		src.Scheme = "https"
	}

	// If the scheme uses ssh, we have to use the git command.
	if src.Scheme == "git+ssh" || src.Scheme == "ssh+git" {
		err = g.fetchWithGit(gitFs, gitDir, src, fs, man)
		if err == exec.ErrNotFound {
			return ErrNotSupportedSource
		}
		return err
	}

	err = g.fetchWithGit(gitFs, gitDir, src, fs, man)
	if err != exec.ErrNotFound {
		return err
	}

	return g.fetchWithGoGit(gitDir, src, fs, man)
}

func (g *gitFetcher) fetchWithGit(gitFs afero.Fs, gitDir string, src *url.URL, fs Copier, man Manifest) (err error) {
	var branch string
	src, branch = getRemoteURL(src)
	srcStr := src.String()

	ctx, cancel := context.WithTimeout(context.Background(), cloneTimeout)
	defer cancel()

	// The first command we execute is a ls-remote to check the last commit from
	// the remote branch and see if we already have a checked-out version of this
	// tree.
	cmd := exec.CommandContext(ctx, "git",
		"ls-remote", "--quiet",
		srcStr, fmt.Sprintf("refs/heads/%s", branch)) // #nosec
	lsRemote, err := cmd.CombinedOutput()
	if err != nil {
		if err != exec.ErrNotFound {
			g.log.Errorf("[git] ls-remote error of %s %s: %s", srcStr, err.Error(),
				lsRemote)
		}
		return err
	}

	lsRemoteFields := bytes.Fields(lsRemote)
	if len(lsRemoteFields) == 0 {
		return fmt.Errorf("git: unexpected ls-remote output")
	}

	slug := man.Slug()
	version := man.Version() + "-" + string(lsRemoteFields[0])

	// The git fetcher needs to update the actual version of the application to
	// reflect the git version of the repository.
	man.SetVersion(version)

	// If the application folder already exists, we can bail early.
	exists, err := fs.Start(slug, version)
	if err != nil {
		return err
	}
	defer func() {
		if errc := fs.Close(); errc != nil {
			err = errc
		}
	}()
	if exists {
		return nil
	}

	cmd = exec.CommandContext(ctx, "git",
		"clone",
		"--quiet",
		"--depth", "1",
		"--single-branch",
		"--branch", branch,
		"--", srcStr, gitDir) // #nosec

	g.log.Infof("[git] Clone with git %s %s in %s: %s", srcStr, branch, gitDir,
		strings.Join(cmd.Args, " "))
	stdoutStderr, err := cmd.CombinedOutput()
	if err != nil {
		if err != exec.ErrNotFound {
			g.log.Errorf("[git] Clone error of %s %s: %s", srcStr, stdoutStderr,
				err.Error())
		}
		return err
	}

	return afero.Walk(gitFs, "/", func(path string, info os.FileInfo, err error) error {
		if err != nil {
			return err
		}
		if info.IsDir() {
			if info.Name() == ".git" {
				return filepath.SkipDir
			}
			return nil
		}
		src, err := gitFs.Open(path)
		if err != nil {
			return err
		}
		return fs.Copy(&fileInfo{
			name: path,
			size: info.Size(),
			mode: info.Mode(),
		}, src)
	})
}

func (g *gitFetcher) fetchWithGoGit(gitDir string, src *url.URL, fs Copier, man Manifest) (err error) {
	var branch string
	src, branch = getRemoteURL(src)

	storage, err := gitStorage.NewStorage(gitOsFS.New(gitDir))
	if err != nil {
		return err
	}

	errch := make(chan error)
	repch := make(chan *git.Repository)

	srcStr := src.String()
	g.log.Infof("[git] Clone with go-git %s %s in %s", srcStr, branch, gitDir)
	go func() {
		repc, errc := git.Clone(storage, nil, &git.CloneOptions{
			URL:           srcStr,
			Depth:         1,
			SingleBranch:  true,
			ReferenceName: gitPlumbing.ReferenceName(branch),
		})
		if errc != nil {
			errch <- errc
		} else {
			repch <- repc
		}
	}()

	var rep *git.Repository
	select {
	case rep = <-repch:
	case err = <-errch:
		g.log.Errorf("[git] Clone error of %s: %s", srcStr, err.Error())
		return err
	case <-time.After(cloneTimeout):
		g.log.Errorf("[git] Clone timeout of %s", srcStr)
		return errCloneTimeout
	}

	ref, err := rep.Head()
	if err != nil {
		return err
	}

	slug := man.Slug()
	version := man.Version() + "-" + ref.Hash().String()

	// The git fetcher needs to update the actual version of the application to
	// reflect the git version of the repository.
	man.SetVersion(version)

	// If the application folder already exists, we can bail early.
	exists, err := fs.Start(slug, version)
	if err != nil {
		return err
	}
	defer func() {
		if errc := fs.Close(); errc != nil {
			err = errc
		}
	}()
	if exists {
		return nil
	}

	commit, err := rep.CommitObject(ref.Hash())
	if err != nil {
		return err
	}

	files, err := commit.Files()
	if err != nil {
		return err
	}

	return files.ForEach(func(f *gitObject.File) error {
		var r io.ReadCloser
		r, err = f.Reader()
		if err != nil {
			return err
		}
		defer r.Close()
		return fs.Copy(&fileInfo{
			name: f.Name,
			size: f.Size,
			mode: os.FileMode(f.Mode),
		}, r)
	})
}

func getWebBranch(src *url.URL) string {
	if src.Fragment != "" {
		return src.Fragment
	}
	return "HEAD"
}

func getRemoteURL(src *url.URL) (*url.URL, string) {
	branch := src.Fragment
	if branch == "" {
		branch = "master"
	}
	clonedSrc := *src
	clonedSrc.Fragment = ""
	return &clonedSrc, branch
}

func resolveGithubURL(src *url.URL, filename string) (string, error) {
	match := ghURLRegex.FindStringSubmatch(src.Path)
	if len(match) != 3 {
		return "", &url.Error{
			Op:  "parsepath",
			URL: src.String(),
			Err: errors.New("Could not parse url git path"),
		}
	}

	user, project := match[1], match[2]
	branch := getWebBranch(src)

	u := fmt.Sprintf(ghRawManifestURL, user, project, branch, filename)
	return u, nil
}

func resolveGitlabURL(src *url.URL, filename string) (string, error) {
	match := glURLRegex.FindStringSubmatch(src.Path)
	if len(match) != 3 {
		return "", &url.Error{
			Op:  "parsepath",
			URL: src.String(),
			Err: errors.New("Could not parse url git path"),
		}
	}

	user, project := match[1], match[2]
	branch := getWebBranch(src)

	u := fmt.Sprintf(glRawManifestURL, src.Host, user, project, branch, filename)
	return u, nil
}

func resolveManifestURL(src *url.URL, filename string) (string, error) {
	// TODO check that it works with a branch
	srccopy, _ := url.Parse(src.String())
	srccopy.Scheme = "http"
	if srccopy.Path == "" || srccopy.Path[len(srccopy.Path)-1] != '/' {
		srccopy.Path += "/"
	}
	srccopy.Path = srccopy.Path + filename
	return srccopy.String(), nil
}

var (
	_ Fetcher = &gitFetcher{}
)
<commit_msg>Fix gometalinter<commit_after>package apps

import (
	"archive/tar"
	"bytes"
	"context"
	"errors"
	"fmt"
	"io"
	"io/ioutil"
	"net/http"
	"net/url"
	"os"
	"os/exec"
	"path/filepath"
	"regexp"
	"strings"
	"time"

	git "github.com/cozy/go-git"
	gitPlumbing "github.com/cozy/go-git/plumbing"
	gitObject "github.com/cozy/go-git/plumbing/object"
	gitStorage "github.com/cozy/go-git/storage/filesystem"
	"github.com/sirupsen/logrus"
	"github.com/spf13/afero"
	gitOsFS "gopkg.in/src-d/go-billy.v2/osfs"
)

var errCloneTimeout = errors.New("git: repository cloning timed out")
var cloneTimeout = 20 * time.Second

const (
	ghRawManifestURL = "https://raw.githubusercontent.com/%s/%s/%s/%s"
	glRawManifestURL = "https://%s/%s/%s/raw/%s/%s"
)

var (
	// ghURLRegex is used to identify github
	ghURLRegex = regexp.MustCompile(`/([^/]+)/([^/]+).git`)
	// glURLRegex is used to identify gitlab
	glURLRegex = regexp.MustCompile(`/([^/]+)/([^/]+).git`)
)

type gitFetcher struct {
	manFilename string
	log         *logrus.Entry
}

func newGitFetcher(appType AppType, log *logrus.Entry) *gitFetcher {
	var manFilename string
	switch appType {
	case Webapp:
		manFilename = WebappManifestName
	case Konnector:
		manFilename = KonnectorManifestName
	}
	return &gitFetcher{
		manFilename: manFilename,
		log:         log,
	}
}

var manifestClient = &http.Client{
	Timeout: 60 * time.Second,
}

func isGithub(src *url.URL) bool {
	return src.Host == "github.com"
}

func isGitlab(src *url.URL) bool {
	return src.Host == "framagit.org" || strings.Contains(src.Host, "gitlab")
}

func (g *gitFetcher) FetchManifest(src *url.URL) (r io.ReadCloser, err error) {
	defer func() {
		if err != nil {
			g.log.Errorf("[git] Error while fetching app manifest %s: %s",
				src.String(), err.Error())
		}
	}()

	if isGitSSHScheme(src.Scheme) {
		return g.fetchManifestFromGitArchive(src)
	}

	var u string
	if isGithub(src) {
		u, err = resolveGithubURL(src, g.manFilename)
	} else if isGitlab(src) {
		u, err = resolveGitlabURL(src, g.manFilename)
	} else {
		u, err = resolveManifestURL(src, g.manFilename)
	}
	if err != nil {
		return nil, err
	}

	g.log.Infof("[git] Fetching manifest on %s", u)
	res, err := manifestClient.Get(u)
	if err != nil || res.StatusCode != 200 {
		g.log.Errorf("[git] Error while fetching manifest on %s", u)
		return nil, ErrManifestNotReachable
	}

	return res.Body, nil
}

// Use the git archive method to download a manifest from the git repository.
func (g *gitFetcher) fetchManifestFromGitArchive(src *url.URL) (io.ReadCloser, error) {
	var branch string
	src, branch = getRemoteURL(src)
	ctx, cancel := context.WithTimeout(context.Background(), cloneTimeout)
	defer cancel()
	cmd := exec.CommandContext(ctx, "git",
		"archive",
		"--remote", src.String(),
		fmt.Sprintf("refs/heads/%s", branch),
		g.manFilename) // #nosec
	stdout, err := cmd.CombinedOutput()
	if err != nil {
		if err == exec.ErrNotFound {
			return nil, ErrNotSupportedSource
		}
		return nil, ErrManifestNotReachable
	}
	buf := new(bytes.Buffer)
	r := tar.NewReader(bytes.NewReader(stdout))
	for {
		h, err := r.Next()
		if err == io.EOF {
			break
		}
		if err != nil {
			return nil, ErrManifestNotReachable
		}
		if h.Name != g.manFilename {
			continue
		}
		if _, err = io.Copy(buf, r); err != nil {
			return nil, ErrManifestNotReachable
		}
	}
	return ioutil.NopCloser(buf), nil
}

func (g *gitFetcher) Fetch(src *url.URL, fs Copier, man Manifest) (err error) {
	defer func() {
		if err != nil {
			g.log.Errorf("[git] Error while fetching or copying repository %s: %s",
				src.String(), err.Error())
		}
	}()

	osFs := afero.NewOsFs()
	gitDir, err := afero.TempDir(osFs, "", "cozy-app-"+man.Slug())
	if err != nil {
		return err
	}
	defer osFs.RemoveAll(gitDir)

	gitFs := afero.NewBasePathFs(osFs, gitDir)
	// XXX Gitlab doesn't support the git protocol
	if src.Scheme == "git" && isGitlab(src) {
		src.Scheme = "https"
	}

	// If the scheme uses ssh, we have to use the git command.
	if isGitSSHScheme(src.Scheme) {
		err = g.fetchWithGit(gitFs, gitDir, src, fs, man)
		if err == exec.ErrNotFound {
			return ErrNotSupportedSource
		}
		return err
	}

	err = g.fetchWithGit(gitFs, gitDir, src, fs, man)
	if err != exec.ErrNotFound {
		return err
	}

	return g.fetchWithGoGit(gitDir, src, fs, man)
}

func (g *gitFetcher) fetchWithGit(gitFs afero.Fs, gitDir string, src *url.URL, fs Copier, man Manifest) (err error) {
	var branch string
	src, branch = getRemoteURL(src)
	srcStr := src.String()

	ctx, cancel := context.WithTimeout(context.Background(), cloneTimeout)
	defer cancel()

	// The first command we execute is a ls-remote to check the last commit from
	// the remote branch and see if we already have a checked-out version of this
	// tree.
	cmd := exec.CommandContext(ctx, "git",
		"ls-remote", "--quiet",
		srcStr, fmt.Sprintf("refs/heads/%s", branch)) // #nosec
	lsRemote, err := cmd.CombinedOutput()
	if err != nil {
		if err != exec.ErrNotFound {
			g.log.Errorf("[git] ls-remote error of %s %s: %s", srcStr, err.Error(),
				lsRemote)
		}
		return err
	}

	lsRemoteFields := bytes.Fields(lsRemote)
	if len(lsRemoteFields) == 0 {
		return fmt.Errorf("git: unexpected ls-remote output")
	}

	slug := man.Slug()
	version := man.Version() + "-" + string(lsRemoteFields[0])

	// The git fetcher needs to update the actual version of the application to
	// reflect the git version of the repository.
	man.SetVersion(version)

	// If the application folder already exists, we can bail early.
	exists, err := fs.Start(slug, version)
	if err != nil {
		return err
	}
	defer func() {
		if errc := fs.Close(); errc != nil {
			err = errc
		}
	}()
	if exists {
		return nil
	}

	cmd = exec.CommandContext(ctx, "git",
		"clone",
		"--quiet",
		"--depth", "1",
		"--single-branch",
		"--branch", branch,
		"--", srcStr, gitDir) // #nosec

	g.log.Infof("[git] Clone with git %s %s in %s: %s", srcStr, branch, gitDir,
		strings.Join(cmd.Args, " "))
	stdoutStderr, err := cmd.CombinedOutput()
	if err != nil {
		if err != exec.ErrNotFound {
			g.log.Errorf("[git] Clone error of %s %s: %s", srcStr, stdoutStderr,
				err.Error())
		}
		return err
	}

	return afero.Walk(gitFs, "/", func(path string, info os.FileInfo, err error) error {
		if err != nil {
			return err
		}
		if info.IsDir() {
			if info.Name() == ".git" {
				return filepath.SkipDir
			}
			return nil
		}
		src, err := gitFs.Open(path)
		if err != nil {
			return err
		}
		return fs.Copy(&fileInfo{
			name: path,
			size: info.Size(),
			mode: info.Mode(),
		}, src)
	})
}

func (g *gitFetcher) fetchWithGoGit(gitDir string, src *url.URL, fs Copier, man Manifest) (err error) {
	var branch string
	src, branch = getRemoteURL(src)

	storage, err := gitStorage.NewStorage(gitOsFS.New(gitDir))
	if err != nil {
		return err
	}

	errch := make(chan error)
	repch := make(chan *git.Repository)

	srcStr := src.String()
	g.log.Infof("[git] Clone with go-git %s %s in %s", srcStr, branch, gitDir)
	go func() {
		repc, errc := git.Clone(storage, nil, &git.CloneOptions{
			URL:           srcStr,
			Depth:         1,
			SingleBranch:  true,
			ReferenceName: gitPlumbing.ReferenceName(branch),
		})
		if errc != nil {
			errch <- errc
		} else {
			repch <- repc
		}
	}()

	var rep *git.Repository
	select {
	case rep = <-repch:
	case err = <-errch:
		g.log.Errorf("[git] Clone error of %s: %s", srcStr, err.Error())
		return err
	case <-time.After(cloneTimeout):
		g.log.Errorf("[git] Clone timeout of %s", srcStr)
		return errCloneTimeout
	}

	ref, err := rep.Head()
	if err != nil {
		return err
	}

	slug := man.Slug()
	version := man.Version() + "-" + ref.Hash().String()

	// The git fetcher needs to update the actual version of the application to
	// reflect the git version of the repository.
	man.SetVersion(version)

	// If the application folder already exists, we can bail early.
	exists, err := fs.Start(slug, version)
	if err != nil {
		return err
	}
	defer func() {
		if errc := fs.Close(); errc != nil {
			err = errc
		}
	}()
	if exists {
		return nil
	}

	commit, err := rep.CommitObject(ref.Hash())
	if err != nil {
		return err
	}

	files, err := commit.Files()
	if err != nil {
		return err
	}

	return files.ForEach(func(f *gitObject.File) error {
		var r io.ReadCloser
		r, err = f.Reader()
		if err != nil {
			return err
		}
		defer r.Close()
		return fs.Copy(&fileInfo{
			name: f.Name,
			size: f.Size,
			mode: os.FileMode(f.Mode),
		}, r)
	})
}

func getWebBranch(src *url.URL) string {
	if src.Fragment != "" {
		return src.Fragment
	}
	return "HEAD"
}

func getRemoteURL(src *url.URL) (*url.URL, string) {
	branch := src.Fragment
	if branch == "" {
		branch = "master"
	}
	clonedSrc := *src
	clonedSrc.Fragment = ""
	return &clonedSrc, branch
}

func resolveGithubURL(src *url.URL, filename string) (string, error) {
	match := ghURLRegex.FindStringSubmatch(src.Path)
	if len(match) != 3 {
		return "", &url.Error{
			Op:  "parsepath",
			URL: src.String(),
			Err: errors.New("Could not parse url git path"),
		}
	}

	user, project := match[1], match[2]
	branch := getWebBranch(src)

	u := fmt.Sprintf(ghRawManifestURL, user, project, branch, filename)
	return u, nil
}

func resolveGitlabURL(src *url.URL, filename string) (string, error) {
	match := glURLRegex.FindStringSubmatch(src.Path)
	if len(match) != 3 {
		return "", &url.Error{
			Op:  "parsepath",
			URL: src.String(),
			Err: errors.New("Could not parse url git path"),
		}
	}

	user, project := match[1], match[2]
	branch := getWebBranch(src)

	u := fmt.Sprintf(glRawManifestURL, src.Host, user, project, branch, filename)
	return u, nil
}

func resolveManifestURL(src *url.URL, filename string) (string, error) {
	// TODO check that it works with a branch
	srccopy, _ := url.Parse(src.String())
	srccopy.Scheme = "http"
	if srccopy.Path == "" || srccopy.Path[len(srccopy.Path)-1] != '/' {
		srccopy.Path += "/"
	}
	srccopy.Path = srccopy.Path + filename
	return srccopy.String(), nil
}

func isGitSSHScheme(scheme string) bool {
	return scheme == "git+ssh" || scheme == "ssh+git"
}

var (
	_ Fetcher = &gitFetcher{}
)
<|endoftext|>


<commit_before>package log

import (
	"fmt"
	"runtime"
	"strings"

	"github.com/sirupsen/logrus"
)

var logrusLogger = logrus.New()

// Fields wraps logrus.Fields, which is a map[string]interface{}
type Fields logrus.Fields

func SetLogLevel(level logrus.Level) {
	logrusLogger.Level = level
}

func SetLogFormatter(formatter logrus.Formatter) {
	logrusLogger.Formatter = formatter
}

// Debug logs a message at level Debug on the standard logrusLogger.
func Debug(args ...interface{}) {
	if logrusLogger.Level >= logrus.DebugLevel {
		entry := logrusLogger.WithFields(logrus.Fields{})
		entry.Data["file"] = fileInfo(2)
		entry.Debug(args...)
	}
}

// Debug logs a message at level Debug on the standard logrusLogger.
func Debugf(args ...interface{}) {
	if logrusLogger.Level >= logrus.DebugLevel {
		entry := logrusLogger.WithFields(logrus.Fields{})
		entry.Data["file"] = fileInfo(2)
		entry.Debug(args...)
	}
}

// Info logs a message at level Info on the standard logrusLogger.
func Info(args ...interface{}) {
	if logrusLogger.Level >= logrus.InfoLevel {
		entry := logrusLogger.WithFields(logrus.Fields{})
		entry.Data["file"] = fileInfo(2)
		entry.Info(args...)
	}
}

// Info logs a message at level Info on the standard logrusLogger.
func Infof(format string, args ...interface{}) {
	if logrusLogger.Level >= logrus.InfoLevel {
		entry := logrusLogger.WithFields(logrus.Fields{})
		entry.Data["file"] = fileInfo(2)
		entry.Infof(format, args...)
	}
}

// Warn logs a message at level Warn on the standard logrusLogger.
func Warn(args ...interface{}) {
	if logrusLogger.Level >= logrus.WarnLevel {
		entry := logrusLogger.WithFields(logrus.Fields{})
		entry.Data["file"] = fileInfo(2)
		entry.Warn(args...)
	}
}

// Warn logs a message at level Warn on the standard logrusLogger.
func Warnf(args ...interface{}) {
	if logrusLogger.Level >= logrus.WarnLevel {
		entry := logrusLogger.WithFields(logrus.Fields{})
		entry.Data["file"] = fileInfo(2)
		entry.Warn(args...)
	}
}

// Error logs a message at level Error on the standard logrusLogger.
func Error(args ...interface{}) {
	if logrusLogger.Level >= logrus.ErrorLevel {
		entry := logrusLogger.WithFields(logrus.Fields{})
		entry.Data["file"] = fileInfo(2)
		entry.Error(args...)
	}
}

// Error logs a message at level Error on the standard logrusLogger.
func Errorf(args ...interface{}) {
	if logrusLogger.Level >= logrus.ErrorLevel {
		entry := logrusLogger.WithFields(logrus.Fields{})
		entry.Data["file"] = fileInfo(2)
		entry.Error(args...)
	}
}

// Fatal logs a message at level Fatal on the standard logrusLogger.
func Fatal(args ...interface{}) {
	if logrusLogger.Level >= logrus.FatalLevel {
		entry := logrusLogger.WithFields(logrus.Fields{})
		entry.Data["file"] = fileInfo(2)
		entry.Fatal(args...)
	}
}

// Fatal logs a message at level Fatal on the standard logrusLogger.
func Fatalf(args ...interface{}) {
	if logrusLogger.Level >= logrus.FatalLevel {
		entry := logrusLogger.WithFields(logrus.Fields{})
		entry.Data["file"] = fileInfo(2)
		entry.Fatal(args...)
	}
}

// Panic logs a message at level Panic on the standard logrusLogger.
func Panic(args ...interface{}) {
	if logrusLogger.Level >= logrus.PanicLevel {
		entry := logrusLogger.WithFields(logrus.Fields{})
		entry.Data["file"] = fileInfo(2)
		entry.Panic(args...)
	}
}

// Panic logs a message at level Panic on the standard logrusLogger.
func Panicf(args ...interface{}) {
	if logrusLogger.Level >= logrus.PanicLevel {
		entry := logrusLogger.WithFields(logrus.Fields{})
		entry.Data["file"] = fileInfo(2)
		entry.Panic(args...)
	}
}

func fileInfo(skip int) string {
	_, file, line, ok := runtime.Caller(skip)
	if !ok {
		file = "<???>"
		line = 1
	} else {
		slash := strings.LastIndex(file, "/")
		if slash >= 0 {
			file = file[slash+1:]
		}
	}
	return fmt.Sprintf("%s:%d", file, line)
}
<commit_msg>fix logger<commit_after>package log

import (
	"fmt"
	"runtime"
	"strings"

	"github.com/sirupsen/logrus"
)

var logrusLogger = logrus.New()

// Fields wraps logrus.Fields, which is a map[string]interface{}
type Fields logrus.Fields

func SetLogLevel(level logrus.Level) {
	logrusLogger.Level = level
}

func SetLogFormatter(formatter logrus.Formatter) {
	logrusLogger.Formatter = formatter
}

// Debug logs a message at level Debug on the standard logrusLogger.
func Debug(args ...interface{}) {
	if logrusLogger.Level >= logrus.DebugLevel {
		entry := logrusLogger.WithFields(logrus.Fields{})
		entry.Data["file"] = fileInfo(2)
		entry.Debug(args...)
	}
}

// Debug logs a message at level Debug on the standard logrusLogger.
func Debugf(format string, args ...interface{}) {
	if logrusLogger.Level >= logrus.DebugLevel {
		entry := logrusLogger.WithFields(logrus.Fields{})
		entry.Data["file"] = fileInfo(2)
		entry.Debugf(format, args...)
	}
}

// Info logs a message at level Info on the standard logrusLogger.
func Info(args ...interface{}) {
	if logrusLogger.Level >= logrus.InfoLevel {
		entry := logrusLogger.WithFields(logrus.Fields{})
		entry.Data["file"] = fileInfo(2)
		entry.Info(args...)
	}
}

// Info logs a message at level Info on the standard logrusLogger.
func Infof(format string, args ...interface{}) {
	if logrusLogger.Level >= logrus.InfoLevel {
		entry := logrusLogger.WithFields(logrus.Fields{})
		entry.Data["file"] = fileInfo(2)
		entry.Infof(format, args...)
	}
}

// Warn logs a message at level Warn on the standard logrusLogger.
func Warn(args ...interface{}) {
	if logrusLogger.Level >= logrus.WarnLevel {
		entry := logrusLogger.WithFields(logrus.Fields{})
		entry.Data["file"] = fileInfo(2)
		entry.Warn(args...)
	}
}

// Warn logs a message at level Warn on the standard logrusLogger.
func Warnf(format string, args ...interface{}) {
	if logrusLogger.Level >= logrus.WarnLevel {
		entry := logrusLogger.WithFields(logrus.Fields{})
		entry.Data["file"] = fileInfo(2)
		entry.Warnf(format, args...)
	}
}

// Error logs a message at level Error on the standard logrusLogger.
func Error(args ...interface{}) {
	if logrusLogger.Level >= logrus.ErrorLevel {
		entry := logrusLogger.WithFields(logrus.Fields{})
		entry.Data["file"] = fileInfo(2)
		entry.Error(args...)
	}
}

// Error logs a message at level Error on the standard logrusLogger.
func Errorf(format string, args ...interface{}) {
	if logrusLogger.Level >= logrus.ErrorLevel {
		entry := logrusLogger.WithFields(logrus.Fields{})
		entry.Data["file"] = fileInfo(2)
		entry.Errorf(format, args...)
	}
}

// Fatal logs a message at level Fatal on the standard logrusLogger.
func Fatal(args ...interface{}) {
	if logrusLogger.Level >= logrus.FatalLevel {
		entry := logrusLogger.WithFields(logrus.Fields{})
		entry.Data["file"] = fileInfo(2)
		entry.Fatal(args...)
	}
}

// Fatal logs a message at level Fatal on the standard logrusLogger.
func Fatalf(format string, args ...interface{}) {
	if logrusLogger.Level >= logrus.FatalLevel {
		entry := logrusLogger.WithFields(logrus.Fields{})
		entry.Data["file"] = fileInfo(2)
		entry.Fatalf(format, args...)
	}
}

// Panic logs a message at level Panic on the standard logrusLogger.
func Panic(args ...interface{}) {
	if logrusLogger.Level >= logrus.PanicLevel {
		entry := logrusLogger.WithFields(logrus.Fields{})
		entry.Data["file"] = fileInfo(2)
		entry.Panic(args...)
	}
}

// Panic logs a message at level Panic on the standard logrusLogger.
func Panicf(format string, args ...interface{}) {
	if logrusLogger.Level >= logrus.PanicLevel {
		entry := logrusLogger.WithFields(logrus.Fields{})
		entry.Data["file"] = fileInfo(2)
		entry.Panicf(format, args...)
	}
}

func fileInfo(skip int) string {
	_, file, line, ok := runtime.Caller(skip)
	if !ok {
		file = "<???>"
		line = 1
	} else {
		slash := strings.LastIndex(file, "/")
		if slash >= 0 {
			file = file[slash+1:]
		}
	}
	return fmt.Sprintf("%s:%d", file, line)
}
<|endoftext|>


<commit_before>package main

import (
	"fmt"
	"os"
	"os/exec"
	"strings"
)

func startup() error {
	fmt.Println("Detected Windows platform...")
	fmt.Println("Looking for existing task users...")
	out, err := exec.Command("C:\\Windows\\System32\\wbem\\WMIC.exe", "useraccount", "get", "name").Output()
	if err != nil {
		fmt.Printf("%v\n", err)
		return err
	}
	for _, line := range strings.Split(string(out), "\r\n") {
		if strings.HasPrefix(line, "Task_") {
			removeOSUser(line)
		}
	}
	return nil
}

func removeOSUser(user string) {
	fmt.Println("Attempting to remove OS user " + user + "...")
	out, err := exec.Command("net", "user", user, "/delete").Output()
	if err != nil {
		fmt.Printf("%v\n", err)
	}
	fmt.Println(string(out))
	out, err = exec.Command("wmic", "path", "win32_userprofile", "where", "name=\""+user+"\"", "delete").Output()
	if err != nil {
		fmt.Printf("%v\n", err)
	}
	fmt.Println(string(out))
}

func (task *TaskRun) generateCommand() *exec.Cmd {
	// TODO: below is the *nix implementation, Windows needs to generate a .bat file
	// and run it using PsExec instead...
	cmd := exec.Command(task.Payload.Command[0], task.Payload.Command[1:]...)
	cmd.Stdout = os.Stdout
	cmd.Stderr = os.Stderr
	task.prepEnvVars(cmd)
	return cmd
}
<commit_msg>Work in progress...<commit_after>package main

import (
	"fmt"
	"os"
	"os/exec"
	"strings"
)

func startup() error {
	fmt.Println("Detected Windows platform...")
	fmt.Println("Looking for existing task users...")
	out, err := exec.Command("C:\\Windows\\System32\\wbem\\WMIC.exe", "useraccount", "get", "name").Output()
	if err != nil {
		fmt.Printf("%v\n", err)
		return err
	}
	for _, line := range strings.Split(string(out), "\r\n") {
		trimmedLine := strings.Trim(line, "\r\n")
		if strings.HasPrefix(trimmedLine, "Task_") {
			removeOSUser(trimmedLine)
		}
	}
	return nil
}

func removeOSUser(user string) {
	fmt.Println("Attempting to remove OS user " + user + "...")
	out, err := exec.Command("net", "user", user, "/delete").Output()
	if err != nil {
		fmt.Printf("%v\n", err)
	}
	fmt.Println(string(out))
	out, err = exec.Command("wmic", "path", "win32_userprofile", "where", "name=\""+user+"\"", "delete").Output()
	if err != nil {
		fmt.Printf("%v\n", err)
	}
	fmt.Println(string(out))
}

func (task *TaskRun) generateCommand() *exec.Cmd {
	// TODO: below is the *nix implementation, Windows needs to generate a .bat file
	// and run it using PsExec instead...
	cmd := exec.Command(task.Payload.Command[0], task.Payload.Command[1:]...)
	cmd.Stdout = os.Stdout
	cmd.Stderr = os.Stderr
	task.prepEnvVars(cmd)
	return cmd
}
<|endoftext|>


<commit_before>package tabular

import (
	"log"
	"math"
	"regexp"
)

// This is a lot of helper functions. To keep things straight, here's a diagram
// of where they're used.
// mean -> sqrDiff -> variance -> stdDev -> normal -> chauvenet -> ProbabalisticSplit
// [fork]          -> chauvenet -> ProbabalisticSplit

// meanInt uses sum to find the mean of a list of ints.
func meanInt(data []int) float64 {
	// sumInt finds the sum of a list of ints. Exciting, I know.
	sumInt := func(data []int) (sum int) {
		for _, i := range data {
			sum += i
		}
		return sum
	}
	return float64(sumInt(data)) / float64(len(data))
}

// meanFloat uses sum to find the mean of a list of float.
func meanFloat(data []float64) float64 {
	// sumFloat finds the sum of a list of floats. Exciting, I know.
	sumFloat := func(data []float64) (sum float64) {
		for _, i := range data {
			sum += i
		}
		return sum
	}
	return sumFloat(data) / float64(len(data))
}

// getSquaredDifferences returns a list of the squared differences between each
// point and the mean. Used in variance and chauvenet->highestVarianceIndex
func getSquaredDifferences(data []int) (sqrDiffs []float64) {
	xbar := meanInt(data)
	for _, xn := range data {
		sqrDiff := math.Pow((xbar - float64(xn)), 2)
		sqrDiffs = append(sqrDiffs, sqrDiff)
	}
	return sqrDiffs
}

// variance is the mean of the squared differences between each observed
// point and the mean (xbar)
func variance(data []int) float64 {
	return meanFloat(getSquaredDifferences(data))
}

// stdDev returns the standard deviation of a data set of integers
func stdDev(data []int) float64 {
	return math.Sqrt(variance(data))
}

// normalDistribution returns the probability that the data point x
// lands where it does, based on the mean (mu) and standard deviation (sigma)
func normalDistribution(x float64, mu float64, sigma float64) float64 {
	coefficient := 1 / (sigma * math.Sqrt(2*math.Pi))
	exponent := -(math.Pow(x-mu, 2) / (2 * math.Pow(sigma, 2)))
	return coefficient * math.Pow(math.E, exponent)
}

type compare func(x float64, y float64) bool

var maxFunc compare = func(x float64, y float64) bool { return x > y }
var minFunc compare = func(x float64, y float64) bool { return x < y }

// extremaIndex finds the index of a value in a list that when compared with
// the any of other data with comparisonFunc will return true. It is most easily
// applicable in finding maxes and mins
func extremaIndex(comparisonFunc compare, data []float64) (index int) {
	if len(data) < 1 {
		return 0
	}
	extrema := data[0]
	extremaIndex := 0
	for i, datum := range data {
		if comparisonFunc(datum, extrema) {
			extrema = datum
			extremaIndex = i
		}
	}
	return extremaIndex
}

// chauvenet simply takes a slice of integers, applies Chauvenet's Criterion,
// and potentially discards a single outlier. Not necessarily, though!
// https://en.wikipedia.org/wiki/Chauvenet%27s_criterion
func chauvenet(data []int) (result []int) {
	// isOutlier applies chauvenet's criterion to determine whether or not
	// x is an outlier
	isOutlier := func(x float64, data []int) bool {
		xbar := meanInt(data)
		sigma := stdDev(data)
		probability := normalDistribution(x, xbar, sigma)
		if probability < float64(1/(2*len(data))) {
			return true
		}
		return false
	}
	// if its an outlier, cut it out. If not, leave it in.
	// find the index with the highest variance
	index := extremaIndex(maxFunc, getSquaredDifferences(data))
	potentialOutlier := float64(data[index])
	// test if that datum is an outlier
	if isOutlier(potentialOutlier, data) {
		return append(data[:index], data[index+1:]...)
	}
	return data
}

// ProbabalisticSplit splits a string based on the regexp that gives the most
// consistent line length (potentially discarding one outlier)
func ProbabalisticSplit(str string) (output Table) {
	// allEqual checks to see if all of the given list of integers are the same
	allEqual := func(ints []int) bool {
		if len(ints) < 1 {
			return true
		}
		sentinel := ints[0]
		for _, i := range ints {
			if i != sentinel {
				return false
			}
		}
		return true
	}
	// notAllOne ensures that the given list of integers are not all == 1
	notAllOne := func(ints []int) bool {
		for _, i := range ints {
			if i != 1 {
				return true
			}
		}
		return false
	}
	// getRowLengths returns row length counts for each table
	getRowLengths := func(tables []Table) (rowLengths [][]int) {
		for _, table := range tables {
			var lengths []int
			for _, row := range table {
				lengths = append(lengths, len(row))
			}
			rowLengths = append(rowLengths, lengths)
		}
		return rowLengths
	}

	// getColumnRegex is the core of the logic. It determines which regex most
	// accurately splits the data into columns by testing the deviation in the
	// row lengths using different regexps.
	// TODO still sucks when the regex doesn't split anything
	// TODO fix: only use that regexp if you find a match at all
	getColumnRegex := func(str string, rowSep *regexp.Regexp) *regexp.Regexp {
		// different column separators to try out
		colSeps := []*regexp.Regexp{
			regexp.MustCompile("\\s+"),    // any whitespace
			regexp.MustCompile("\\s{2,}"), // two+ whitespace (spaces in cols)
			regexp.MustCompile("\\s{4}"),  // exactly four whitespaces
			//regexp.MustCompile("\\t+"),    // tabs
		}
		// separate the data based on the above column regexps
		var tables []Table
		for _, colSep := range colSeps {
			table := SeparateString(rowSep, colSep, str)
			tables = append(tables, table)
		}
		// see if any of them had utterly consistent row lengths
		rowLengths := getRowLengths(tables)
		if len(rowLengths) != len(tables) {
			log.Fatal("Internal error: len(rowLengths) != len(tables)")
		}
		for i, lengths := range rowLengths {
			if allEqual(lengths) && notAllOne(lengths) {
				return colSeps[i]
			}
		}
		// if not, cast out outliers and try again
		for i, lengths := range rowLengths {
			rowLengths[i] = chauvenet(lengths)
		}
		for i, lengths := range rowLengths {
			if allEqual(lengths) && notAllOne(lengths) {
				return colSeps[i]
			}
		}
		// if still not done, just pick the one with the lowest variance
		var variances []float64
		for _, lengths := range rowLengths {
			variances = append(variances, variance(lengths))
		}
		// ensure that index can be found in tables
		minVarianceIndex := extremaIndex(minFunc, variances)
		if len(tables) <= minVarianceIndex {
			log.Fatal("Internal error: minVarianceIndex couldn't be found in tables")
		}
		return colSeps[minVarianceIndex]
	}
	rowSep := regexp.MustCompile("\n+")
	colSep := getColumnRegex(str, rowSep)
	//fmt.Println("THE CHOSEN REGEX: " + colSep.String())
	return SeparateString(rowSep, colSep, str)
}
<commit_msg>Added error catching to probabalistic split<commit_after>package tabular

import (
	"fmt"
	"log"
	"math"
	"regexp"
)

// This is a lot of helper functions. To keep things straight, here's a diagram
// of where they're used.
// mean -> sqrDiff -> variance -> stdDev -> normal -> chauvenet -> ProbabalisticSplit
// [fork]          -> chauvenet -> ProbabalisticSplit

// meanInt uses sum to find the mean of a list of ints.
func meanInt(data []int) float64 {
	// sumInt finds the sum of a list of ints. Exciting, I know.
	sumInt := func(data []int) (sum int) {
		for _, i := range data {
			sum += i
		}
		return sum
	}
	return float64(sumInt(data)) / float64(len(data))
}

// meanFloat uses sum to find the mean of a list of float.
func meanFloat(data []float64) float64 {
	// sumFloat finds the sum of a list of floats. Exciting, I know.
	sumFloat := func(data []float64) (sum float64) {
		for _, i := range data {
			sum += i
		}
		return sum
	}
	return sumFloat(data) / float64(len(data))
}

// getSquaredDifferences returns a list of the squared differences between each
// point and the mean. Used in variance and chauvenet->highestVarianceIndex
func getSquaredDifferences(data []int) (sqrDiffs []float64) {
	xbar := meanInt(data)
	for _, xn := range data {
		sqrDiff := math.Pow((xbar - float64(xn)), 2)
		sqrDiffs = append(sqrDiffs, sqrDiff)
	}
	return sqrDiffs
}

// variance is the mean of the squared differences between each observed
// point and the mean (xbar)
func variance(data []int) float64 {
	return meanFloat(getSquaredDifferences(data))
}

// stdDev returns the standard deviation of a data set of integers
func stdDev(data []int) float64 {
	return math.Sqrt(variance(data))
}

// normalDistribution returns the probability that the data point x
// lands where it does, based on the mean (mu) and standard deviation (sigma)
func normalDistribution(x float64, mu float64, sigma float64) float64 {
	coefficient := 1 / (sigma * math.Sqrt(2*math.Pi))
	exponent := -(math.Pow(x-mu, 2) / (2 * math.Pow(sigma, 2)))
	return coefficient * math.Pow(math.E, exponent)
}

type compare func(x float64, y float64) bool

var maxFunc compare = func(x float64, y float64) bool { return x > y }
var minFunc compare = func(x float64, y float64) bool { return x < y }

// extremaIndex finds the index of a value in a list that when compared with
// the any of other data with comparisonFunc will return true. It is most easily
// applicable in finding maxes and mins
func extremaIndex(comparisonFunc compare, data []float64) (index int) {
	if len(data) < 1 {
		return 0
	}
	extrema := data[0]
	extremaIndex := 0
	for i, datum := range data {
		if comparisonFunc(datum, extrema) {
			extrema = datum
			extremaIndex = i
		}
	}
	return extremaIndex
}

// chauvenet simply takes a slice of integers, applies Chauvenet's Criterion,
// and potentially discards a single outlier. Not necessarily, though!
// https://en.wikipedia.org/wiki/Chauvenet%27s_criterion
func chauvenet(data []int) (result []int) {
	// isOutlier applies chauvenet's criterion to determine whether or not
	// x is an outlier
	isOutlier := func(x float64, data []int) bool {
		xbar := meanInt(data)
		sigma := stdDev(data)
		probability := normalDistribution(x, xbar, sigma)
		if probability < float64(1/(2*len(data))) {
			return true
		}
		return false
	}
	// if its an outlier, cut it out. If not, leave it in.
	// find the index with the highest variance
	index := extremaIndex(maxFunc, getSquaredDifferences(data))
	potentialOutlier := float64(data[index])
	// test if that datum is an outlier
	if isOutlier(potentialOutlier, data) {
		return append(data[:index], data[index+1:]...)
	}
	return data
}

// ProbabalisticSplit splits a string based on the regexp that gives the most
// consistent line length (potentially discarding one outlier)
func ProbabalisticSplit(str string) (output Table) {
	// allEqual checks to see if all of the given list of integers are the same
	allEqual := func(ints []int) bool {
		if len(ints) < 1 {
			return true
		}
		sentinel := ints[0]
		for _, i := range ints {
			if i != sentinel {
				return false
			}
		}
		return true
	}
	// matchesMost is used to ensure that our regexp actually is splitting the
	// lines of a table, instead of just returning them whole.
	matchesMost := func(re *regexp.Regexp, rows []string) bool {
		count := 0
		for _, row := range rows {
			if re.MatchString(row) {
				count++
			}
		}
		return count > (len(rows) / 2)
	}
	// getRowLengths returns row length counts for each table
	getRowLengths := func(tables []Table) (rowLengths [][]int) {
		for _, table := range tables {
			var lengths []int
			for _, row := range table {
				lengths = append(lengths, len(row))
			}
			rowLengths = append(rowLengths, lengths)
		}
		return rowLengths
	}

	// getColumnRegex is the core of the logic. It determines which regex most
	// accurately splits the data into columns by testing the deviation in the
	// row lengths using different regexps.
	getColumnRegex := func(str string, rowSep *regexp.Regexp) *regexp.Regexp {
		// different column separators to try out
		initialColSeps := []*regexp.Regexp{
			regexp.MustCompile("\\s+"),    // any whitespace
			regexp.MustCompile("\\s{2,}"), // two+ whitespace (spaces in cols)
			regexp.MustCompile("\\s{4}"),  // exactly four whitespaces
			regexp.MustCompile("\\t+"),    // tabs
		}
		// filter regexps that have no matches at all - they will always return
		// rows of even length (length 1).
		colSeps := []*regexp.Regexp{}
		rows := rowSep.Split(str, -1)
		for _, re := range initialColSeps {
			if matchesMost(re, rows) {
				colSeps = append(colSeps, re)
			}
		}
		if len(colSeps) < 1 {
			msg := "ProbabalisticSplit couldn't divide the table."
			msg += "\n\tAttempted regexps: " + fmt.Sprint(initialColSeps)
			log.Fatal(msg)
		}
		// separate the data based on the above column regexps
		var tables []Table
		for _, colSep := range colSeps {
			table := SeparateString(rowSep, colSep, str)
			tables = append(tables, table)
		}
		// see if any of them had utterly consistent row lengths
		rowLengths := getRowLengths(tables)
		if len(rowLengths) != len(tables) {
			log.Fatal("Internal error: len(rowLengths) != len(tables)")
		}
		for i, lengths := range rowLengths {
			if allEqual(lengths) {
				return colSeps[i]
			}
		}
		// if not, cast out outliers and try again
		for i, lengths := range rowLengths {
			rowLengths[i] = chauvenet(lengths)
		}
		for i, lengths := range rowLengths {
			if allEqual(lengths) {
				return colSeps[i]
			}
		}
		// if still not done, just pick the one with the lowest variance
		var variances []float64
		for _, lengths := range rowLengths {
			variances = append(variances, variance(lengths))
		}
		// ensure that index can be found in tables
		minVarianceIndex := extremaIndex(minFunc, variances)
		if len(tables) <= minVarianceIndex {
			log.Fatal("Internal error: minVarianceIndex couldn't be found in tables")
		}
		return colSeps[minVarianceIndex]
	}
	rowSep := regexp.MustCompile("\n+")
	colSep := getColumnRegex(str, rowSep)
	//fmt.Println("THE CHOSEN REGEX: " + colSep.String())
	return SeparateString(rowSep, colSep, str)
}
<|endoftext|>


<commit_before>package main

import (
	"flag"
	"fmt"
	"log"
	"os"
	"sync"
)

// Commands lists the available commands and help topics.
// The order here is the order in which they are printed by 'aonui help'.
var commands = []*Command{
	cmdInv,
	cmdReorder,

	helpTawhiri,
}

func main() {
	flag.Usage = usage
	flag.Parse()
	log.SetFlags(0)

	args := flag.Args()
	if len(args) < 1 {
		usage()
	}

	if args[0] == "help" {
		help(args[1:])
		return
	}

	for _, cmd := range commands {
		if cmd.Name() == args[0] && cmd.Run != nil {
			cmd.Flag.Usage = func() { cmd.Usage() }
			if cmd.CustomFlags {
				args = args[1:]
			} else {
				cmd.Flag.Parse(args[1:])
				args = cmd.Flag.Args()
			}
			cmd.Run(cmd, args)
			exit()
			return
		}
	}

	fmt.Fprintf(os.Stderr, "aonui: unknown subcommand %q\nRun 'aonui help' for usage.\n", args[0])
	setExitStatus(2)
	exit()
}

var exitStatus = 0
var exitMu sync.Mutex

func setExitStatus(n int) {
	exitMu.Lock()
	if exitStatus < n {
		exitStatus = n
	}
	exitMu.Unlock()
}

var atexitFuncs []func()

func atexit(f func()) {
	atexitFuncs = append(atexitFuncs, f)
}

func exit() {
	for _, f := range atexitFuncs {
		f()
	}
	os.Exit(exitStatus)
}
<commit_msg>cmd/aonui: call atexit functions on keyboard interrupt<commit_after>package main

import (
	"flag"
	"fmt"
	"log"
	"os"
	"os/signal"
	"sync"
)

// Commands lists the available commands and help topics.
// The order here is the order in which they are printed by 'aonui help'.
var commands = []*Command{
	cmdInv,
	cmdReorder,

	helpTawhiri,
}

func main() {
	flag.Usage = usage
	flag.Parse()
	log.SetFlags(0)

	args := flag.Args()
	if len(args) < 1 {
		usage()
	}

	if args[0] == "help" {
		help(args[1:])
		return
	}

	// Set signal handler so that "atexit" functions are called on keyboard
	// interrupt.
	c := make(chan os.Signal, 1)
	signal.Notify(c, os.Interrupt)
	go func() {
		for s := range c {
			log.Printf("captured %v, cleaning up", s)
			exit()
		}
	}()

	for _, cmd := range commands {
		if cmd.Name() == args[0] && cmd.Run != nil {
			cmd.Flag.Usage = func() { cmd.Usage() }
			if cmd.CustomFlags {
				args = args[1:]
			} else {
				cmd.Flag.Parse(args[1:])
				args = cmd.Flag.Args()
			}
			cmd.Run(cmd, args)
			exit()
			return
		}
	}

	fmt.Fprintf(os.Stderr, "aonui: unknown subcommand %q\nRun 'aonui help' for usage.\n", args[0])
	setExitStatus(2)
	exit()
}

var exitStatus = 0
var exitMu sync.Mutex

func setExitStatus(n int) {
	exitMu.Lock()
	if exitStatus < n {
		exitStatus = n
	}
	exitMu.Unlock()
}

var atexitFuncs []func()

func atexit(f func()) {
	atexitFuncs = append(atexitFuncs, f)
}

func exit() {
	for _, f := range atexitFuncs {
		f()
	}
	os.Exit(exitStatus)
}
<|endoftext|>


<commit_before>package bkeybase

import (
	"strconv"

	"github.com/42wim/matterbridge/bridge"
	"github.com/42wim/matterbridge/bridge/config"
	"github.com/keybase/go-keybase-chat-bot/kbchat"
)

// Bkeybase bridge structure
type Bkeybase struct {
	kbc     *kbchat.API
	user    string
	channel string
	team    string
	*bridge.Config
}

// New initializes Bkeybase object and sets team
func New(cfg *bridge.Config) bridge.Bridger {
	b := &Bkeybase{Config: cfg}
	b.team = b.Config.GetString("Team")
	return b
}

// Connect starts keybase API and listener loop
func (b *Bkeybase) Connect() error {
	var err error
	b.Log.Infof("Connecting %s", b.GetString("Team"))

	// use default keybase location (`keybase`)
	b.kbc, err = kbchat.Start(kbchat.RunOptions{})
	if err != nil {
		return err
	}
	b.user = b.kbc.GetUsername()
	b.Log.Info("Connection succeeded")
	go b.handleKeybase()
	return nil
}

// Disconnect doesn't do anything for now
func (b *Bkeybase) Disconnect() error {
	return nil
}

// JoinChannel sets channel name in struct
func (b *Bkeybase) JoinChannel(channel config.ChannelInfo) error {
	if _, err := b.kbc.JoinChannel(b.team, channel.Name); err != nil {
		return err
	}
	b.channel = channel.Name
	return nil
}

// Send receives bridge messages and sends them to Keybase chat room
func (b *Bkeybase) Send(msg config.Message) (string, error) {
	b.Log.Debugf("=> Receiving %#v", msg)

	// Handle /me events
	if msg.Event == config.EventUserAction {
		msg.Text = "_" + msg.Text + "_"
	}

	// Delete message if we have an ID
	// Delete message not supported by keybase go library yet

	// Upload a file if it exists
	// kbchat lib does not support attachments yet

	// Edit message if we have an ID
	// kbchat lib does not support message editing yet

	// Send regular message
	resp, err := b.kbc.SendMessageByTeamName(b.team, msg.Username+msg.Text, &b.channel)
	if err != nil {
		return "", err
	}

	return strconv.Itoa(resp.Result.MsgID), err
}
<commit_msg>Add support for receiving attachments (keybase) (#923)<commit_after>package bkeybase

import (
	"io/ioutil"
	"os"
	"path/filepath"
	"strconv"

	"github.com/42wim/matterbridge/bridge"
	"github.com/42wim/matterbridge/bridge/config"
	"github.com/keybase/go-keybase-chat-bot/kbchat"
)

// Bkeybase bridge structure
type Bkeybase struct {
	kbc     *kbchat.API
	user    string
	channel string
	team    string
	*bridge.Config
}

// New initializes Bkeybase object and sets team
func New(cfg *bridge.Config) bridge.Bridger {
	b := &Bkeybase{Config: cfg}
	b.team = b.Config.GetString("Team")
	return b
}

// Connect starts keybase API and listener loop
func (b *Bkeybase) Connect() error {
	var err error
	b.Log.Infof("Connecting %s", b.GetString("Team"))

	// use default keybase location (`keybase`)
	b.kbc, err = kbchat.Start(kbchat.RunOptions{})
	if err != nil {
		return err
	}
	b.user = b.kbc.GetUsername()
	b.Log.Info("Connection succeeded")
	go b.handleKeybase()
	return nil
}

// Disconnect doesn't do anything for now
func (b *Bkeybase) Disconnect() error {
	return nil
}

// JoinChannel sets channel name in struct
func (b *Bkeybase) JoinChannel(channel config.ChannelInfo) error {
	if _, err := b.kbc.JoinChannel(b.team, channel.Name); err != nil {
		return err
	}
	b.channel = channel.Name
	return nil
}

// Send receives bridge messages and sends them to Keybase chat room
func (b *Bkeybase) Send(msg config.Message) (string, error) {
	b.Log.Debugf("=> Receiving %#v", msg)

	// Handle /me events
	if msg.Event == config.EventUserAction {
		msg.Text = "_" + msg.Text + "_"
	}

	// Delete message if we have an ID
	// Delete message not supported by keybase go library yet

	// Edit message if we have an ID
	// kbchat lib does not support message editing yet

	if len(msg.Extra["file"]) > 0 {
		// Upload a file
		dir, err := ioutil.TempDir("", "matterbridge")
		if err != nil {
			return "", err
		}
		defer os.RemoveAll(dir)

		for _, f := range msg.Extra["file"] {
			fname := f.(config.FileInfo).Name
			fdata := *f.(config.FileInfo).Data
			fcaption := f.(config.FileInfo).Comment
			fpath := filepath.Join(dir, fname)

			if err = ioutil.WriteFile(fpath, fdata, 0600); err != nil {
				return "", err
			}

			_, _ = b.kbc.SendAttachmentByTeam(b.team, fpath, fcaption, &b.channel)
		}

		return "", nil
	}

	// Send regular message
	resp, err := b.kbc.SendMessageByTeamName(b.team, msg.Username+msg.Text, &b.channel)
	if err != nil {
		return "", err
	}
	return strconv.Itoa(resp.Result.MsgID), err
}
<|endoftext|>


<commit_before>// +build linux darwin freebsd

package cmd

import (
	"fmt"
	"github.com/spf13/cobra"
)

var configDelCmd = &cobra.Command{
	Use:     "del",
	Aliases: []string{"rm"},
	Short:   "Delete configuration for Docker container from Consul.",
	PreRun: func(cmd *cobra.Command, args []string) {
		checkConfigDelFlags()
		LoadConfig()
	},
	Long: `Delete configuration for Docker container from Consul.`,
	Run:  startConfigDel,
}

func startConfigDel(cmd *cobra.Command, args []string) {
	fmt.Println("Del")
}

func checkConfigDelFlags() {
	Log("Checking flags", "info")
}

func init() {
	configCmd.AddCommand(configDelCmd)
}
<commit_msg>Added ConfigEnv Del().<commit_after>// +build linux darwin freebsd

package cmd

import (
	"fmt"
	"github.com/spf13/cobra"
	"os"
)

var configDelCmd = &cobra.Command{
	Use:     "del",
	Aliases: []string{"rm"},
	Short:   "Delete configuration for Docker container from Consul.",
	PreRun: func(cmd *cobra.Command, args []string) {
		checkConfigDelFlags()
		LoadConfig()
	},
	Long: `Delete configuration for Docker container from Consul.`,
	Run:  startConfigDel,
}

func startConfigDel(cmd *cobra.Command, args []string) {
	config := ConfigEnv{Container: Container, Key: ConfigKey}
	value := config.Del()
	if value {
		Log(fmt.Sprintf("Deleted %s", config.Path()), "info")
	}
}

func checkConfigDelFlags() {
	Log("Checking cli flags.", "debug")
	if Container == "" {
		fmt.Println("A container is required: -c")
		os.Exit(1)
	}
	if ConfigKey == "" {
		fmt.Println("A key is required: --key")
		os.Exit(1)
	}
	Log("Required cli flags are present.", "debug")
}

func init() {
	configCmd.AddCommand(configDelCmd)
}

// Del returns the value of the key passed.
func (c *ConfigEnv) Del() bool {
	consul, err := ConsulSetup()
	if err != nil {
		Log("Fatal Consul setup problem.", "info")
	}
	value := ConsulDel(consul, c.Path())
	return value
}
<|endoftext|>


<commit_before>// Copyright © 2017 Mike Hudgins <mchudgins@gmail.com>
//
// Permission is hereby granted, free of charge, to any person obtaining a copy
// of this software and associated documentation files (the "Software"), to deal
// in the Software without restriction, including without limitation the rights
// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
// copies of the Software, and to permit persons to whom the Software is
// furnished to do so, subject to the following conditions:
//
// The above copyright notice and this permission notice shall be included in
// all copies or substantial portions of the Software.
//
// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
// THE SOFTWARE.

package cmd

import (
	"log"

	echo "github.com/dstcorp/rpc-golang/service"
	"github.com/spf13/cobra"
	"golang.org/x/net/context"
	"google.golang.org/grpc"
	"google.golang.org/grpc/credentials"
)

// echoClientCmd represents the echoClient command
var echoClientCmd = &cobra.Command{
	Use:   "echoClient",
	Short: "A brief description of your command",
	Long: `A longer description that spans multiple lines and likely contains examples
and usage of using your command. For example:

Cobra is a CLI library for Go that empowers applications.
This application is a tool to generate the needed files
to quickly create a Cobra application.`,
	Run: func(cmd *cobra.Command, args []string) {
		creds, err := credentials.NewClientTLSFromFile("/usr/local/share/ca-certificates/dst-root.crt",
			"")
		if err != nil {
			panic(err)
		}
		conn, err := grpc.Dial("echo.local.dstcorp.io:50050",
			grpc.WithTransportCredentials(creds),
			grpc.WithCompressor(grpc.NewGZIPCompressor()),
			grpc.WithDecompressor(grpc.NewGZIPDecompressor()))
		if err != nil {
			panic(err)
		}
		defer conn.Close()

		client := echo.NewEchoServiceClient(conn)

		request := &echo.EchoRequest{
			Message: args[0],
		}
		response, err := client.Echo(context.Background(), request)
		if err != nil {
			panic(err)
		}
		log.Printf("response: %s\n", response.Message)
	},
}

func init() {
	RootCmd.AddCommand(echoClientCmd)

	// Here you will define your flags and configuration settings.

	// Cobra supports Persistent Flags which will work for this command
	// and all subcommands, e.g.:
	// echoClientCmd.PersistentFlags().String("foo", "", "A help for foo")

	// Cobra supports local flags which will only run when this command
	// is called directly, e.g.:
	// echoClientCmd.Flags().BoolP("toggle", "t", false, "Help message for toggle")

}
<commit_msg>wip for echoClient metrics<commit_after>// Copyright © 2017 Mike Hudgins <mchudgins@gmail.com>
//
// Permission is hereby granted, free of charge, to any person obtaining a copy
// of this software and associated documentation files (the "Software"), to deal
// in the Software without restriction, including without limitation the rights
// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
// copies of the Software, and to permit persons to whom the Software is
// furnished to do so, subject to the following conditions:
//
// The above copyright notice and this permission notice shall be included in
// all copies or substantial portions of the Software.
//
// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
// THE SOFTWARE.

package cmd

import (
	"log"

	echo "github.com/dstcorp/rpc-golang/service"
	"github.com/grpc-ecosystem/go-grpc-prometheus"
	"github.com/spf13/cobra"
	"golang.org/x/net/context"
	"google.golang.org/grpc"
	"google.golang.org/grpc/credentials"
)

// echoClientCmd represents the echoClient command
var echoClientCmd = &cobra.Command{
	Use:   "echoClient",
	Short: "A brief description of your command",
	Long: `A longer description that spans multiple lines and likely contains examples
and usage of using your command. For example:

Cobra is a CLI library for Go that empowers applications.
This application is a tool to generate the needed files
to quickly create a Cobra application.`,
	Run: func(cmd *cobra.Command, args []string) {
		creds, err := credentials.NewClientTLSFromFile("/usr/local/share/ca-certificates/dst-root.crt",
			"")
		if err != nil {
			panic(err)
		}
		conn, err := grpc.Dial("echo.local.dstcorp.io:50050",
			grpc.WithTransportCredentials(creds),
			grpc.WithCompressor(grpc.NewGZIPCompressor()),
			grpc.WithDecompressor(grpc.NewGZIPDecompressor()),
			grpc.UnaryClientInterceptor(grpc_prometheus.UnaryClientInterceptor))
		if err != nil {
			panic(err)
		}
		defer conn.Close()

		client := echo.NewEchoServiceClient(conn)

		request := &echo.EchoRequest{
			Message: args[0],
		}
		response, err := client.Echo(context.Background(), request)
		if err != nil {
			panic(err)
		}
		log.Printf("response: %s\n", response.Message)
	},
}

func init() {
	RootCmd.AddCommand(echoClientCmd)

	// Here you will define your flags and configuration settings.

	// Cobra supports Persistent Flags which will work for this command
	// and all subcommands, e.g.:
	// echoClientCmd.PersistentFlags().String("foo", "", "A help for foo")

	// Cobra supports local flags which will only run when this command
	// is called directly, e.g.:
	// echoClientCmd.Flags().BoolP("toggle", "t", false, "Help message for toggle")

}
<|endoftext|>


<commit_before>package main

import (
	"encoding/json"
	"fmt"
	"io"
	"net"
	"net/http"
	"os"
	"time"

	gclient "github.com/cloudfoundry-incubator/garden/client"
	gconn "github.com/cloudfoundry-incubator/garden/client/connection"
	"github.com/concourse/atc"
	"github.com/concourse/tsa"
	"github.com/pivotal-golang/lager"
	"github.com/tedsuo/ifrit"
	"github.com/tedsuo/rata"
	"golang.org/x/crypto/ssh"
)

type registrarSSHServer struct {
	logger            lager.Logger
	atcEndpoint       *rata.RequestGenerator
	heartbeatInterval time.Duration
	forwardHost       string
	config            *ssh.ServerConfig
	httpClient        *http.Client
}

func (server *registrarSSHServer) Serve(listener net.Listener) {
	for {
		c, err := listener.Accept()
		if err != nil {
			server.logger.Error("failed-to-accept", err)
			return
		}

		logger := server.logger.Session("connection")

		conn, chans, reqs, err := ssh.NewServerConn(c, server.config)
		if err != nil {
			logger.Info("handshake-failed", lager.Data{"error": err.Error()})
			continue
		}

		go server.handleConn(logger, conn, chans, reqs)
	}
}

type forwardedTCPIP struct {
	process   ifrit.Process
	boundPort uint32
}

func (server *registrarSSHServer) handleConn(logger lager.Logger, conn *ssh.ServerConn, chans <-chan ssh.NewChannel, reqs <-chan *ssh.Request) {
	defer conn.Close()

	forwardedTCPIPs := make(chan forwardedTCPIP, 1)
	go server.handleForwardRequests(logger, conn, reqs, forwardedTCPIPs)

	var processes []ifrit.Process

	// ensure processes get cleaned up
	defer func() {
		cleanupLog := logger.Session("cleanup")

		for _, p := range processes {
			cleanupLog.Debug("interrupting")

			p.Signal(os.Interrupt)
		}

		for _, p := range processes {
			err := <-p.Wait()
			if err != nil {
				cleanupLog.Error("process-exited-with-failure", err)
			} else {
				cleanupLog.Debug("process-exited-successfully")
			}
		}
	}()

	for newChannel := range chans {
		if newChannel.ChannelType() != "session" {
			logger.Info("rejecting-unknown-channel-type", lager.Data{
				"type": newChannel.ChannelType(),
			})

			newChannel.Reject(ssh.UnknownChannelType, "unknown channel type")
			continue
		}

		channel, requests, err := newChannel.Accept()
		if err != nil {
			logger.Error("failed-to-accept-channel", err)
			return
		}

		defer channel.Close()

		for req := range requests {
			logger.Info("channel-request", lager.Data{
				"type": req.Type,
			})

			if req.Type != "exec" {
				logger.Info("rejecting")

				req.Reply(false, nil)
				continue
			}

			var request execRequest
			err = ssh.Unmarshal(req.Payload, &request)
			if err != nil {
				logger.Error("malformed-exec-request", err)

				req.Reply(false, nil)

				return
			}

			switch request.Command {
			case "register-worker":
				logger := logger.Session("register-worker")

				req.Reply(true, nil)

				process, err := server.continuouslyRegisterWorkerDirectly(logger, channel)
				if err != nil {
					logger.Error("failed-to-register", err)
					return
				}

				processes = append(processes, process)

				err = conn.Wait()
				logger.Error("connection-closed", err)

			case "forward-worker":
				logger := logger.Session("forward-worker")

				var forwarded forwardedTCPIP

				select {
				case forwarded = <-forwardedTCPIPs:
					logger.Info("forwarded-tcpip", lager.Data{
						"bound-port": forwarded.boundPort,
					})

					processes = append(processes, forwarded.process)

					process, err := server.continuouslyRegisterForwardedWorker(logger, channel, forwarded.boundPort)
					if err != nil {
						logger.Error("failed-to-register", err)
						return
					}

					processes = append(processes, process)

					err = conn.Wait()
					logger.Error("connection-closed", err)

				case <-time.After(10 * time.Second): // todo better?
					logger.Info("never-forwarded-tcpip")
				}

			default:
				logger.Info("invalid-command", lager.Data{
					"command": request.Command,
				})

				req.Reply(false, nil)
			}
		}
	}
}

func (server *registrarSSHServer) continuouslyRegisterWorkerDirectly(
	logger lager.Logger,
	channel ssh.Channel,
) (ifrit.Process, error) {
	logger.Session("start")
	defer logger.Session("done")

	var worker atc.Worker
	err := json.NewDecoder(channel).Decode(&worker)
	if err != nil {
		return nil, err
	}

	return server.heartbeatWorker(logger, worker), nil
}

func (server *registrarSSHServer) continuouslyRegisterForwardedWorker(
	logger lager.Logger,
	channel ssh.Channel,
	boundPort uint32,
) (ifrit.Process, error) {
	logger.Session("start")
	defer logger.Session("done")

	var worker atc.Worker
	err := json.NewDecoder(channel).Decode(&worker)
	if err != nil {
		return nil, err
	}

	worker.Addr = fmt.Sprintf("%s:%d", server.forwardHost, boundPort)

	return server.heartbeatWorker(logger, worker), nil
}

func (server *registrarSSHServer) heartbeatWorker(logger lager.Logger, worker atc.Worker) ifrit.Process {
	return ifrit.Background(tsa.NewHeartbeater(
		logger,
		server.heartbeatInterval,
		gclient.New(gconn.New("tcp", worker.Addr)),
		server.atcEndpoint,
		worker,
	))
}

func (server *registrarSSHServer) handleForwardRequests(
	logger lager.Logger,
	conn *ssh.ServerConn,
	reqs <-chan *ssh.Request,
	forwardedTCPIPs chan<- forwardedTCPIP,
) {
	var forwardedThings int

	for r := range reqs {
		switch r.Type {
		case "tcpip-forward":
			logger := logger.Session("tcpip-forward")

			forwardedThings++

			if forwardedThings > 1 {
				logger.Info("rejecting-extra-forward-request")
				r.Reply(false, nil)
				continue
			}

			var req tcpipForwardRequest
			err := ssh.Unmarshal(r.Payload, &req)
			if err != nil {
				logger.Error("malformed-tcpip-request", err)
				r.Reply(false, nil)
				continue
			}

			bindAddr := net.JoinHostPort(req.BindIP, fmt.Sprintf("%d", req.BindPort))

			logger.Info("forwarding-tcpip", lager.Data{
				"bind-addr": bindAddr,
			})

			listener, err := net.Listen("tcp", bindAddr)
			if err != nil {
				r.Reply(false, nil)
				continue
			}

			defer listener.Close()

			_, port, err := net.SplitHostPort(listener.Addr().String())
			if err != nil {
				r.Reply(false, nil)
				continue
			}

			var res tcpipForwardResponse
			_, err = fmt.Sscanf(port, "%d", &res.BoundPort)
			if err != nil {
				r.Reply(false, nil)
				continue
			}

			process := server.forwardTCPIP(logger, conn, listener, req.BindIP, res.BoundPort)

			forwardedTCPIPs <- forwardedTCPIP{
				boundPort: res.BoundPort,
				process:   process,
			}

			r.Reply(true, ssh.Marshal(res))

		default:
			logger.Info("ignoring-request", lager.Data{"type": r.Type})
			r.Reply(false, nil)
		}
	}
}

func (server *registrarSSHServer) forwardTCPIP(
	logger lager.Logger,
	conn *ssh.ServerConn,
	listener net.Listener,
	forwardIP string,
	forwardPort uint32,
) ifrit.Process {
	return ifrit.Background(ifrit.RunFunc(func(signals <-chan os.Signal, ready chan<- struct{}) error {
		go func() {
			<-signals

			listener.Close()
		}()

		close(ready)

		for {
			localConn, err := listener.Accept()
			if err != nil {
				logger.Error("failed-to-accept", err)
				break
			}

			go forwardLocalConn(logger, localConn, conn, forwardIP, forwardPort)
		}

		return nil
	}))
}

func forwardLocalConn(logger lager.Logger, localConn net.Conn, conn *ssh.ServerConn, forwardIP string, forwardPort uint32) {
	defer localConn.Close()

	var req forwardTCPIPChannelRequest
	req.ForwardIP = forwardIP
	req.ForwardPort = forwardPort

	host, port, err := net.SplitHostPort(localConn.RemoteAddr().String())
	if err != nil {
		logger.Error("failed-to-split-host-port", err)
		return
	}

	req.OriginIP = host
	_, err = fmt.Sscanf(port, "%d", &req.OriginPort)
	if err != nil {
		logger.Error("failed-to-parse-port", err)
		return
	}

	channel, reqs, err := conn.OpenChannel("forwarded-tcpip", ssh.Marshal(req))
	if err != nil {
		logger.Error("failed-to-open-channel", err)
		return
	}

	defer channel.Close()

	go func() {
		for r := range reqs {
			logger.Info("ignoring-request", lager.Data{
				"type": r.Type,
			})

			r.Reply(false, nil)
		}
	}()

	go io.Copy(localConn, channel)
	io.Copy(channel, localConn)
}
<commit_msg>fix connection closing once more<commit_after>package main

import (
	"encoding/json"
	"fmt"
	"io"
	"net"
	"net/http"
	"os"
	"sync"
	"time"

	gclient "github.com/cloudfoundry-incubator/garden/client"
	gconn "github.com/cloudfoundry-incubator/garden/client/connection"
	"github.com/concourse/atc"
	"github.com/concourse/tsa"
	"github.com/pivotal-golang/lager"
	"github.com/tedsuo/ifrit"
	"github.com/tedsuo/rata"
	"golang.org/x/crypto/ssh"
)

type registrarSSHServer struct {
	logger            lager.Logger
	atcEndpoint       *rata.RequestGenerator
	heartbeatInterval time.Duration
	forwardHost       string
	config            *ssh.ServerConfig
	httpClient        *http.Client
}

func (server *registrarSSHServer) Serve(listener net.Listener) {
	for {
		c, err := listener.Accept()
		if err != nil {
			server.logger.Error("failed-to-accept", err)
			return
		}

		logger := server.logger.Session("connection")

		conn, chans, reqs, err := ssh.NewServerConn(c, server.config)
		if err != nil {
			logger.Info("handshake-failed", lager.Data{"error": err.Error()})
			continue
		}

		go server.handleConn(logger, conn, chans, reqs)
	}
}

type forwardedTCPIP struct {
	process   ifrit.Process
	boundPort uint32
}

func (server *registrarSSHServer) handleConn(logger lager.Logger, conn *ssh.ServerConn, chans <-chan ssh.NewChannel, reqs <-chan *ssh.Request) {
	defer conn.Close()

	forwardedTCPIPs := make(chan forwardedTCPIP, 1)
	go server.handleForwardRequests(logger, conn, reqs, forwardedTCPIPs)

	var processes []ifrit.Process

	// ensure processes get cleaned up
	defer func() {
		cleanupLog := logger.Session("cleanup")

		for _, p := range processes {
			cleanupLog.Debug("interrupting")

			p.Signal(os.Interrupt)
		}

		for _, p := range processes {
			err := <-p.Wait()
			if err != nil {
				cleanupLog.Error("process-exited-with-failure", err)
			} else {
				cleanupLog.Debug("process-exited-successfully")
			}
		}
	}()

	for newChannel := range chans {
		if newChannel.ChannelType() != "session" {
			logger.Info("rejecting-unknown-channel-type", lager.Data{
				"type": newChannel.ChannelType(),
			})

			newChannel.Reject(ssh.UnknownChannelType, "unknown channel type")
			continue
		}

		channel, requests, err := newChannel.Accept()
		if err != nil {
			logger.Error("failed-to-accept-channel", err)
			return
		}

		defer channel.Close()

		for req := range requests {
			logger.Info("channel-request", lager.Data{
				"type": req.Type,
			})

			if req.Type != "exec" {
				logger.Info("rejecting")

				req.Reply(false, nil)
				continue
			}

			var request execRequest
			err = ssh.Unmarshal(req.Payload, &request)
			if err != nil {
				logger.Error("malformed-exec-request", err)

				req.Reply(false, nil)

				return
			}

			switch request.Command {
			case "register-worker":
				logger := logger.Session("register-worker")

				req.Reply(true, nil)

				process, err := server.continuouslyRegisterWorkerDirectly(logger, channel)
				if err != nil {
					logger.Error("failed-to-register", err)
					return
				}

				processes = append(processes, process)

				err = conn.Wait()
				logger.Error("connection-closed", err)

			case "forward-worker":
				logger := logger.Session("forward-worker")

				var forwarded forwardedTCPIP

				select {
				case forwarded = <-forwardedTCPIPs:
					logger.Info("forwarded-tcpip", lager.Data{
						"bound-port": forwarded.boundPort,
					})

					processes = append(processes, forwarded.process)

					process, err := server.continuouslyRegisterForwardedWorker(logger, channel, forwarded.boundPort)
					if err != nil {
						logger.Error("failed-to-register", err)
						return
					}

					processes = append(processes, process)

					err = conn.Wait()
					logger.Error("connection-closed", err)

				case <-time.After(10 * time.Second): // todo better?
					logger.Info("never-forwarded-tcpip")
				}

			default:
				logger.Info("invalid-command", lager.Data{
					"command": request.Command,
				})

				req.Reply(false, nil)
			}
		}
	}
}

func (server *registrarSSHServer) continuouslyRegisterWorkerDirectly(
	logger lager.Logger,
	channel ssh.Channel,
) (ifrit.Process, error) {
	logger.Session("start")
	defer logger.Session("done")

	var worker atc.Worker
	err := json.NewDecoder(channel).Decode(&worker)
	if err != nil {
		return nil, err
	}

	return server.heartbeatWorker(logger, worker), nil
}

func (server *registrarSSHServer) continuouslyRegisterForwardedWorker(
	logger lager.Logger,
	channel ssh.Channel,
	boundPort uint32,
) (ifrit.Process, error) {
	logger.Session("start")
	defer logger.Session("done")

	var worker atc.Worker
	err := json.NewDecoder(channel).Decode(&worker)
	if err != nil {
		return nil, err
	}

	worker.Addr = fmt.Sprintf("%s:%d", server.forwardHost, boundPort)

	return server.heartbeatWorker(logger, worker), nil
}

func (server *registrarSSHServer) heartbeatWorker(logger lager.Logger, worker atc.Worker) ifrit.Process {
	return ifrit.Background(tsa.NewHeartbeater(
		logger,
		server.heartbeatInterval,
		gclient.New(gconn.New("tcp", worker.Addr)),
		server.atcEndpoint,
		worker,
	))
}

func (server *registrarSSHServer) handleForwardRequests(
	logger lager.Logger,
	conn *ssh.ServerConn,
	reqs <-chan *ssh.Request,
	forwardedTCPIPs chan<- forwardedTCPIP,
) {
	var forwardedThings int

	for r := range reqs {
		switch r.Type {
		case "tcpip-forward":
			logger := logger.Session("tcpip-forward")

			forwardedThings++

			if forwardedThings > 1 {
				logger.Info("rejecting-extra-forward-request")
				r.Reply(false, nil)
				continue
			}

			var req tcpipForwardRequest
			err := ssh.Unmarshal(r.Payload, &req)
			if err != nil {
				logger.Error("malformed-tcpip-request", err)
				r.Reply(false, nil)
				continue
			}

			bindAddr := net.JoinHostPort(req.BindIP, fmt.Sprintf("%d", req.BindPort))

			logger.Info("forwarding-tcpip", lager.Data{
				"bind-addr": bindAddr,
			})

			listener, err := net.Listen("tcp", bindAddr)
			if err != nil {
				r.Reply(false, nil)
				continue
			}

			defer listener.Close()

			_, port, err := net.SplitHostPort(listener.Addr().String())
			if err != nil {
				r.Reply(false, nil)
				continue
			}

			var res tcpipForwardResponse
			_, err = fmt.Sscanf(port, "%d", &res.BoundPort)
			if err != nil {
				r.Reply(false, nil)
				continue
			}

			process := server.forwardTCPIP(logger, conn, listener, req.BindIP, res.BoundPort)

			forwardedTCPIPs <- forwardedTCPIP{
				boundPort: res.BoundPort,
				process:   process,
			}

			r.Reply(true, ssh.Marshal(res))

		default:
			logger.Info("ignoring-request", lager.Data{"type": r.Type})
			r.Reply(false, nil)
		}
	}
}

func (server *registrarSSHServer) forwardTCPIP(
	logger lager.Logger,
	conn *ssh.ServerConn,
	listener net.Listener,
	forwardIP string,
	forwardPort uint32,
) ifrit.Process {
	return ifrit.Background(ifrit.RunFunc(func(signals <-chan os.Signal, ready chan<- struct{}) error {
		go func() {
			<-signals

			listener.Close()
		}()

		close(ready)

		for {
			localConn, err := listener.Accept()
			if err != nil {
				logger.Error("failed-to-accept", err)
				break
			}

			go forwardLocalConn(logger, localConn, conn, forwardIP, forwardPort)
		}

		return nil
	}))
}

func forwardLocalConn(logger lager.Logger, localConn net.Conn, conn *ssh.ServerConn, forwardIP string, forwardPort uint32) {
	defer localConn.Close()

	var req forwardTCPIPChannelRequest
	req.ForwardIP = forwardIP
	req.ForwardPort = forwardPort

	host, port, err := net.SplitHostPort(localConn.RemoteAddr().String())
	if err != nil {
		logger.Error("failed-to-split-host-port", err)
		return
	}

	req.OriginIP = host
	_, err = fmt.Sscanf(port, "%d", &req.OriginPort)
	if err != nil {
		logger.Error("failed-to-parse-port", err)
		return
	}

	channel, reqs, err := conn.OpenChannel("forwarded-tcpip", ssh.Marshal(req))
	if err != nil {
		logger.Error("failed-to-open-channel", err)
		return
	}

	defer channel.Close()

	go func() {
		for r := range reqs {
			logger.Info("ignoring-request", lager.Data{
				"type": r.Type,
			})

			r.Reply(false, nil)
		}
	}()

	wg := new(sync.WaitGroup)

	pipe := func(to io.WriteCloser, from io.ReadCloser) {
		// if either end breaks, close both ends to ensure they're both unblocked,
		// otherwise io.Copy can block forever if e.g. reading after write end has
		// gone away
		defer to.Close()
		defer from.Close()
		defer wg.Done()

		io.Copy(to, from)
	}

	wg.Add(1)
	go pipe(localConn, channel)

	wg.Add(1)
	go pipe(channel, localConn)

	wg.Wait()
}
<|endoftext|>


<commit_before>/* Copyright (c) 2015, Daniel Martí <mvdan@mvdan.cc> */
/* See LICENSE for licensing information */

package main

import (
	"bufio"
	"flag"
	"fmt"
	"io"
	"os"
	"regexp"

	"github.com/mvdan/xurls"
)

var (
	matching = flag.String("m", "", "")
	relaxed  = flag.Bool("r", false, "")
)

func init() {
	flag.Usage = func() {
		fmt.Fprintln(os.Stderr, `Usage: xurls [-h]`)
		fmt.Fprintln(os.Stderr)
		fmt.Fprintln(os.Stderr, `   -m <regexp>   only match urls whose scheme matches a regexp`)
		fmt.Fprintln(os.Stderr, `                    example: "https?://|mailto:"`)
		fmt.Fprintln(os.Stderr, `   -r            also match urls without a scheme (relaxed)`)
	}
}

func scan(re *regexp.Regexp, r io.Reader) {
	scanner := bufio.NewScanner(r)
	scanner.Split(bufio.ScanWords)
	for scanner.Scan() {
		word := scanner.Text()
		for _, match := range re.FindAllString(word, -1) {
			fmt.Println(match)
		}
	}
}

func main() {
	flag.Parse()
	if *relaxed && *matching != "" {
		fmt.Fprintf(os.Stderr, "-r and -m at the same time don't make much sense.\n")
		os.Exit(1)
	}
	re := xurls.Strict
	if *relaxed {
		re = xurls.Relaxed
	} else if *matching != "" {
		var err error
		if re, err = xurls.StrictMatching(*matching); err != nil {
			fmt.Fprintf(os.Stderr, "invalid regular expression '%s': %v\n", *matching, err)
			os.Exit(2)
		}
	}
	args := flag.Args()
	if len(args) == 0 {
		scan(re, os.Stdin)
	} else {
		for _, path := range args {
			if path == "-" {
				scan(re, os.Stdin)
				continue
			}
			file, err := os.Open(path)
			if err != nil {
				fmt.Fprintf(os.Stdout, "could not open file '%s': %v\n", path, err)
			}
			scan(re, file)
			file.Close()
		}
	}
}
<commit_msg>cmd/xurls: Ranging over a nil/empty slice is fine<commit_after>/* Copyright (c) 2015, Daniel Martí <mvdan@mvdan.cc> */
/* See LICENSE for licensing information */

package main

import (
	"bufio"
	"flag"
	"fmt"
	"io"
	"os"
	"regexp"

	"github.com/mvdan/xurls"
)

var (
	matching = flag.String("m", "", "")
	relaxed  = flag.Bool("r", false, "")
)

func init() {
	flag.Usage = func() {
		fmt.Fprintln(os.Stderr, `Usage: xurls [-h]`)
		fmt.Fprintln(os.Stderr)
		fmt.Fprintln(os.Stderr, `   -m <regexp>   only match urls whose scheme matches a regexp`)
		fmt.Fprintln(os.Stderr, `                    example: "https?://|mailto:"`)
		fmt.Fprintln(os.Stderr, `   -r            also match urls without a scheme (relaxed)`)
	}
}

func scan(re *regexp.Regexp, r io.Reader) {
	scanner := bufio.NewScanner(r)
	scanner.Split(bufio.ScanWords)
	for scanner.Scan() {
		word := scanner.Text()
		for _, match := range re.FindAllString(word, -1) {
			fmt.Println(match)
		}
	}
}

func main() {
	flag.Parse()
	if *relaxed && *matching != "" {
		fmt.Fprintf(os.Stderr, "-r and -m at the same time don't make much sense.\n")
		os.Exit(1)
	}
	re := xurls.Strict
	if *relaxed {
		re = xurls.Relaxed
	} else if *matching != "" {
		var err error
		if re, err = xurls.StrictMatching(*matching); err != nil {
			fmt.Fprintf(os.Stderr, "invalid regular expression '%s': %v\n", *matching, err)
			os.Exit(2)
		}
	}
	args := flag.Args()
	if len(args) == 0 {
		scan(re, os.Stdin)
	}
	for _, path := range args {
		if path == "-" {
			scan(re, os.Stdin)
			continue
		}
		file, err := os.Open(path)
		if err != nil {
			fmt.Fprintf(os.Stdout, "could not open file '%s': %v\n", path, err)
		}
		scan(re, file)
		file.Close()
	}
}
<|endoftext|>


<commit_before>/*
 * ZLint Copyright 2017 Regents of the University of Michigan
 *
 * Licensed under the Apache License, Version 2.0 (the "License"); you may not
 * use this file except in compliance with the License. You may obtain a copy
 * of the License at http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
 * implied. See the License for the specific language governing
 * permissions and limitations under the License.
 */

package main

import (
	"bytes"
	"encoding/base64"
	"encoding/json"
	"encoding/pem"
	"flag"
	"fmt"
	"io/ioutil"
	"os"
	"sort"
	"strings"

	log "github.com/Sirupsen/logrus"
	"github.com/zmap/zcrypto/x509"
	"github.com/zmap/zlint"
	"github.com/zmap/zlint/lints"
)

var ( // flags
	listLintsJSON   bool
	listLintsSchema bool
	prettyprint     bool
	format          string
)

func init() {
	flag.BoolVar(&listLintsJSON, "list-lints-json", false, "Print supported lints in JSON format, one per line")
	flag.BoolVar(&listLintsSchema, "list-lints-schema", false, "Print supported lints as a ZSchema")
	flag.StringVar(&format, "format", "pem", "One of {pem, der, base64}")
	flag.BoolVar(&prettyprint, "pretty", false, "Pretty-print output")
	flag.Usage = func() {
		fmt.Fprintf(os.Stderr, "Usage: %s [flags] file...\n", os.Args[0])
		flag.PrintDefaults()
	}
	flag.Parse()
	log.SetLevel(log.InfoLevel)
}

func main() {

	if listLintsJSON {
		zlint.EncodeLintDescriptionsToJSON(os.Stdout)
		return
	}

	if listLintsSchema {
		names := make([]string, 0, len(lints.Lints))
		for lintName := range lints.Lints {
			names = append(names, lintName)
		}
		sort.Strings(names)
		fmt.Printf("Lints = SubRecord({\n")
		for _, lintName := range names {
			fmt.Printf("    \"%s\":LintBool(),\n", lintName)
		}
		fmt.Printf("})\n")
		return
	}

	var inform = strings.ToLower(format)
	if flag.NArg() < 1 || flag.Arg(0) == "-" {
		lint(os.Stdin, inform)
	} else {
		for _, filePath := range flag.Args() {
			var inputFile *os.File
			var err error
			inputFile, err = os.Open(filePath)
			if err != nil {
				log.Fatalf("unable to open file %s: %s", filePath, err)
			}
			var fmt = inform
			switch {
			case strings.HasSuffix(filePath, ".der"):
				fmt = "der"
			case strings.HasSuffix(filePath, ".pem"):
				fmt = "pem"
			}
			lint(inputFile, fmt)
			inputFile.Close()
		}
	}
}

func lint(inputFile *os.File, inform string) {
	fileBytes, err := ioutil.ReadAll(inputFile)
	if err != nil {
		log.Fatalf("unable to read file %s: %s", inputFile.Name(), err)
	}

	var asn1Data []byte
	switch inform {
	case "pem":
		p, _ := pem.Decode(fileBytes)
		if p == nil || p.Type != "CERTIFICATE" {
			log.Fatal("unable to parse PEM")
		}
		asn1Data = p.Bytes
	case "der":
		asn1Data = fileBytes
	case "base64":
		asn1Data, err = base64.StdEncoding.DecodeString(string(fileBytes))
		if err != nil {
			log.Fatalf("unable to parse base64: %s", err)
		}
	default:
		log.Fatalf("unknown input format %s", format)
	}

	c, err := x509.ParseCertificate(asn1Data)
	if err != nil {
		log.Fatalf("unable to parse certificate: %s", err)
	}

	zlintResult := zlint.LintCertificate(c)
	jsonBytes, err := json.Marshal(zlintResult.Results)
	if err != nil {
		log.Fatalf("unable to encode lints JSON: %s", err)
	}
	if prettyprint {
		var out bytes.Buffer
		if err := json.Indent(&out, jsonBytes, "", " "); err != nil {
			log.Fatalf("can't format output: %s", err)
		}
		os.Stdout.Write(out.Bytes())
	} else {
		os.Stdout.Write(jsonBytes)
	}
	os.Stdout.Write([]byte{'\n'})
	os.Stdout.Sync()
}
<commit_msg>Add lints include/exclude flag to executable (#247)<commit_after>/*
 * ZLint Copyright 2017 Regents of the University of Michigan
 *
 * Licensed under the Apache License, Version 2.0 (the "License"); you may not
 * use this file except in compliance with the License. You may obtain a copy
 * of the License at http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
 * implied. See the License for the specific language governing
 * permissions and limitations under the License.
 */

package main

import (
	"bytes"
	"encoding/base64"
	"encoding/json"
	"encoding/pem"
	"flag"
	"fmt"
	"io/ioutil"
	"os"
	"sort"
	"strings"

	log "github.com/Sirupsen/logrus"
	"github.com/zmap/zcrypto/x509"
	"github.com/zmap/zlint"
	"github.com/zmap/zlint/lints"
)

var ( // flags
	listLintsJSON   bool
	listLintsSchema bool
	prettyprint     bool
	format          string
	include         string
	exclude         string
)

func init() {
	flag.BoolVar(&listLintsJSON, "list-lints-json", false, "Print supported lints in JSON format, one per line")
	flag.BoolVar(&listLintsSchema, "list-lints-schema", false, "Print supported lints as a ZSchema")
	flag.StringVar(&format, "format", "pem", "One of {pem, der, base64}")
	flag.StringVar(&include, "include", "", "Comma-separated list of lints to include by name")
	flag.StringVar(&exclude, "exclude", "", "Comma-separated list of lints to exclude by name")

	flag.BoolVar(&prettyprint, "pretty", false, "Pretty-print output")
	flag.Usage = func() {
		fmt.Fprintf(os.Stderr, "Usage: %s [flags] file...\n", os.Args[0])
		flag.PrintDefaults()
	}
	flag.Parse()
	log.SetLevel(log.InfoLevel)
}

func main() {

	if listLintsJSON {
		zlint.EncodeLintDescriptionsToJSON(os.Stdout)
		return
	}

	if listLintsSchema {
		names := make([]string, 0, len(lints.Lints))
		for lintName := range lints.Lints {
			names = append(names, lintName)
		}
		sort.Strings(names)
		fmt.Printf("Lints = SubRecord({\n")
		for _, lintName := range names {
			fmt.Printf("    \"%s\":LintBool(),\n", lintName)
		}
		fmt.Printf("})\n")
		return
	}

	// include/exclude lints based on flags
	setLints()

	var inform = strings.ToLower(format)
	if flag.NArg() < 1 || flag.Arg(0) == "-" {
		lint(os.Stdin, inform)
	} else {
		for _, filePath := range flag.Args() {
			var inputFile *os.File
			var err error
			inputFile, err = os.Open(filePath)
			if err != nil {
				log.Fatalf("unable to open file %s: %s", filePath, err)
			}
			var fileInform = inform
			switch {
			case strings.HasSuffix(filePath, ".der"):
				fileInform = "der"
			case strings.HasSuffix(filePath, ".pem"):
				fileInform = "pem"
			}

			lint(inputFile, fileInform)
			inputFile.Close()
		}
	}
}

func lint(inputFile *os.File, inform string) {
	fileBytes, err := ioutil.ReadAll(inputFile)
	if err != nil {
		log.Fatalf("unable to read file %s: %s", inputFile.Name(), err)
	}

	var asn1Data []byte
	switch inform {
	case "pem":
		p, _ := pem.Decode(fileBytes)
		if p == nil || p.Type != "CERTIFICATE" {
			log.Fatal("unable to parse PEM")
		}
		asn1Data = p.Bytes
	case "der":
		asn1Data = fileBytes
	case "base64":
		asn1Data, err = base64.StdEncoding.DecodeString(string(fileBytes))
		if err != nil {
			log.Fatalf("unable to parse base64: %s", err)
		}
	default:
		log.Fatalf("unknown input format %s", format)
	}

	c, err := x509.ParseCertificate(asn1Data)
	if err != nil {
		log.Fatalf("unable to parse certificate: %s", err)
	}

	zlintResult := zlint.LintCertificate(c)
	jsonBytes, err := json.Marshal(zlintResult.Results)
	if err != nil {
		log.Fatalf("unable to encode lints JSON: %s", err)
	}
	if prettyprint {
		var out bytes.Buffer
		if err := json.Indent(&out, jsonBytes, "", " "); err != nil {
			log.Fatalf("can't format output: %s", err)
		}
		os.Stdout.Write(out.Bytes())
	} else {
		os.Stdout.Write(jsonBytes)
	}
	os.Stdout.Write([]byte{'\n'})
	os.Stdout.Sync()
}

func setLints() {
	if include != "" && exclude != "" {
		log.Fatal("unable to use include and exclude flag at the same time")
	}

	includeLints()
	excludeLints()
}

func includeLints() {
	if include == "" {
		return
	}

	// parse includes to map for easier matching
	var includes = strings.Split(include, ",")
	var includesMap = make(map[string]struct{}, len(includes))
	for _, includeName := range includes {
		includeName = strings.TrimSpace(includeName)
		if _, ok := lints.Lints[includeName]; !ok {
			log.Fatalf("unknown lint %q in include list", includeName)
		}

		includesMap[includeName] = struct{}{}
	}

	// clear all initialised lints except for includes
	for lintName := range lints.Lints {
		if _, ok := includesMap[lintName]; !ok {
			delete(lints.Lints, lintName)
		}
	}
}

func excludeLints() {
	if exclude == "" {
		return
	}

	// parse excludes to map to get rid of duplicates
	var excludes = strings.Split(exclude, ",")
	var excludesMap = make(map[string]struct{}, len(excludes))
	for _, excludeName := range excludes {
		excludesMap[strings.TrimSpace(excludeName)] = struct{}{}
	}

	// exclude lints
	for excludeName := range excludesMap {
		if _, ok := lints.Lints[excludeName]; !ok {
			log.Fatalf("unknown lint %q in exclude list", excludeName)
		}

		delete(lints.Lints, excludeName)
	}
}
<|endoftext|>