<commit_before>// Command ghr is a tool to create a Github Release and upload your
// artifacts in parallel.
package main

import (
	"context"
	"flag"
	"fmt"
	"io"
	"log"
	"os"
	"regexp"
	"runtime"
	"time"

	"github.com/google/go-github/github"
	"github.com/mitchellh/colorstring"
	"github.com/tcnksm/go-gitconfig"
)

const (
	// EnvGitHubToken is an environment var containing the GitHub API token
	EnvGitHubToken = "GITHUB_TOKEN"

	// EnvGitHubAPI is an environment var containing the GitHub API base endpoint.
	// This is used mainly by GitHub Enterprise users.
	EnvGitHubAPI = "GITHUB_API"

	// EnvDebug is an environment var to handle debug mode
	EnvDebug = "GHR_DEBUG"
)

// Exit codes are set to a value that represent an exit code for a particular error.
const (
	ExitCodeOK int = 0

	// Errors start at 10
	ExitCodeError = 10 + iota
	ExitCodeParseFlagsError
	ExitCodeBadArgs
	ExitCodeInvalidURL
	ExitCodeTokenNotFound
	ExitCodeOwnerNotFound
	ExitCodeRepoNotFound
	ExitCodeReleaseError
)

const (
	defaultCheckTimeout = 2 * time.Second
	defaultBaseURL      = "https://api.github.com/"
	defaultParallel     = -1
)

// Debugf prints debug output when EnvDebug is set
func Debugf(format string, args ...interface{}) {
	if env := os.Getenv(EnvDebug); len(env) != 0 {
		log.Printf("[DEBUG] "+format+"\n", args...)
	}
}

// PrintRedf prints red error message to console.
func PrintRedf(w io.Writer, format string, args ...interface{}) {
	format = fmt.Sprintf("[red]%s[reset]", format)
	fmt.Fprint(w,
		colorstring.Color(fmt.Sprintf(format, args...)))
}

// CLI is the main command line object
type CLI struct {
	// outStream and errStream correspond to stdout and stderr, respectively,
	// to take messages from the CLI.
	outStream, errStream io.Writer
}

// Run invokes the CLI with the given arguments.
func (cli *CLI) Run(args []string) int {

	var (
		owner string
		repo  string
		token string

		commitish  string
		name       string
		body       string
		draft      bool
		prerelease bool

		parallel int

		recreate bool
		replace  bool

		stat    bool
		version bool
		debug   bool
	)

	flags := flag.NewFlagSet(Name, flag.ContinueOnError)
	flags.SetOutput(cli.errStream)
	flags.Usage = func() {
		fmt.Fprint(cli.errStream, helpText)
	}

	flags.StringVar(&owner, "username", "", "")
	flags.StringVar(&owner, "owner", "", "")
	flags.StringVar(&owner, "u", "", "")

	flags.StringVar(&repo, "repository", "", "")
	flags.StringVar(&repo, "r", "", "")

	flags.StringVar(&token, "token", os.Getenv(EnvGitHubToken), "")
	flags.StringVar(&token, "t", os.Getenv(EnvGitHubToken), "")

	flags.StringVar(&commitish, "commitish", "", "")
	flags.StringVar(&commitish, "c", "", "")

	flags.StringVar(&name, "name", token, "")
	flags.StringVar(&name, "n", token, "")

	flags.StringVar(&body, "body", "", "")
	flags.StringVar(&body, "b", "", "")

	flags.BoolVar(&draft, "draft", false, "")
	flags.BoolVar(&prerelease, "prerelease", false, "")

	flags.IntVar(&parallel, "parallel", defaultParallel, "")
	flags.IntVar(&parallel, "p", defaultParallel, "")

	flags.BoolVar(&recreate, "delete", false, "")
	flags.BoolVar(&recreate, "recreate", false, "")

	flags.BoolVar(&replace, "replace", false, "")

	flags.BoolVar(&version, "version", false, "")
	flags.BoolVar(&version, "v", false, "")

	flags.BoolVar(&debug, "debug", false, "")

	// Deprecated
	flags.BoolVar(&stat, "stat", false, "")

	// Parse flags
	if err := flags.Parse(args[1:]); err != nil {
		return ExitCodeParseFlagsError
	}

	if debug {
		os.Setenv(EnvDebug, "1")
		Debugf("Run as DEBUG mode")
	}

	// Show version and check latest version release
	if version {
		fmt.Fprintf(cli.outStream, OutputVersion())
		return ExitCodeOK
	}

	parsedArgs := flags.Args()
	if len(parsedArgs) != 2 {
		PrintRedf(cli.errStream,
			"Invalid argument: you must set TAG and PATH name.\n")
		return ExitCodeBadArgs
	}
	tag, path := parsedArgs[0], parsedArgs[1]

	// Extract github repository owner username.
	// If it's not provided via command line flag, read it from .gitconfig
	// (github user or git user).
	if len(owner) == 0 {
		origin, err := gitconfig.OriginURL()
		if err == nil {
			owner = retrieveOwnerName(origin)
		}
		if len(owner) == 0 {
			owner, err = gitconfig.GithubUser()
			if err != nil {
				owner, err = gitconfig.Username()
			}

			if err != nil {
				PrintRedf(cli.errStream,
					"Failed to set up ghr: repository owner name not found\n")
				fmt.Fprintf(cli.errStream,
					"Please set it via `-u` option.\n\n"+
						"You can set default owner name in `github.username` or `user.name`\n"+
						"in `~/.gitconfig` file\n")
				return ExitCodeOwnerNotFound
			}
		}
	}
	Debugf("Owner: %s", owner)

	// Extract repository name from files.
	// If not provided, read it from .git/config file.
	if len(repo) == 0 {
		var err error
		repo, err = gitconfig.Repository()
		if err != nil {
			PrintRedf(cli.errStream,
				"Failed to set up ghr: repository name not found\n")
			fmt.Fprintf(cli.errStream,
				"ghr reads it from `.git/config` file. Change directory to \n"+
					"repository root directory or setup git repository.\n"+
					"Or set it via `-r` option.\n")
			return ExitCodeRepoNotFound
		}
	}
	Debugf("Repository: %s", repo)

	// If GitHub API token is not provided via command line flag
	// or env var then read it from .gitconfig file.
	if len(token) == 0 {
		var err error
		token, err = gitconfig.GithubToken()
		if err != nil {
			PrintRedf(cli.errStream, "Failed to set up ghr: token not found\n")
			fmt.Fprintf(cli.errStream,
				"To use ghr, you need a GitHub API token.\n"+
					"Please set it via `%s` env var or `-t` option.\n\n"+
					"If you don't have one, visit official doc (goo.gl/jSnoI)\n"+
					"and get it first.\n",
				EnvGitHubToken)
			return ExitCodeTokenNotFound
		}
	}
	Debugf("Github API Token: %s", maskString(token))

	// Set Base GitHub API URL. Base URL can also be provided via env var for use with GHE.
	baseURLStr := defaultBaseURL
	if urlStr := os.Getenv(EnvGitHubAPI); len(urlStr) != 0 {
		baseURLStr = urlStr
	}
	Debugf("Base GitHub API URL: %s", baseURLStr)

	if parallel <= 0 {
		parallel = runtime.NumCPU()
	}
	Debugf("Parallel factor: %d", parallel)

	localAssets, err := LocalAssets(path)
	if err != nil {
		PrintRedf(cli.errStream,
			"Failed to find assets from %s: %s\n", path, err)
		return ExitCodeError
	}
	Debugf("Number of file to upload: %d", len(localAssets))

	// Create a GitHub client
	gitHubClient, err := NewGitHubClient(owner, repo, token, baseURLStr)
	if err != nil {
		PrintRedf(cli.errStream, "Failed to construct GitHub client: %s\n", err)
		return ExitCodeError
	}

	ghr := GHR{
		GitHub:    gitHubClient,
		outStream: cli.outStream,
	}

	Debugf("Name: %s", name)

	// Prepare create release request
	req := &github.RepositoryRelease{
		Name:            github.String(name),
		TagName:         github.String(tag),
		Prerelease:      github.Bool(prerelease),
		Draft:           github.Bool(draft),
		TargetCommitish: github.String(commitish),
		Body:            github.String(body),
	}

	ctx := context.TODO()
	release, err := ghr.CreateRelease(ctx, req, recreate)
	if err != nil {
		PrintRedf(cli.errStream, "Failed to create GitHub release page: %s\n", err)
		return ExitCodeError
	}

	if replace {
		err := ghr.DeleteAssets(ctx, *release.ID, localAssets, parallel)
		if err != nil {
			PrintRedf(cli.errStream, "Failed to delete existing assets: %s\n", err)
			return ExitCodeError
		}
	}

	// FIXME(tcnksm): More ideal way to change this
	// This is for Github enterprise
	if err := ghr.GitHub.SetUploadURL(*release.UploadURL); err != nil {
		fmt.Fprintf(cli.errStream, "Failed to set upload URL %s: %s\n", *release.UploadURL, err)
		return ExitCodeError
	}

	err = ghr.UploadAssets(ctx, *release.ID, localAssets, parallel)
	if err != nil {
		PrintRedf(cli.errStream, "Failed to upload one of assets: %s\n", err)
		return ExitCodeError
	}

	if !draft {
		_, err := ghr.GitHub.EditRelease(ctx, *release.ID, &github.RepositoryRelease{
			Draft: github.Bool(false),
		})
		if err != nil {
			PrintRedf(cli.errStream, "Failed to publish release: %s\n", err)
			return ExitCodeError
		}
	}

	return ExitCodeOK
}

var ownerNameReg = regexp.MustCompile(`([-a-zA-Z0-9]+)/[^/]+$`)

func retrieveOwnerName(repoURL string) string {
	matched := ownerNameReg.FindStringSubmatch(repoURL)
	if len(matched) < 2 {
		return ""
	}
	return matched[1]
}

// maskString is used to mask a string which should not be displayed
// directly, like the auth token
func maskString(s string) string {
	if len(s) < 5 {
		return "**** (masked)"
	}

	return s[:5] + "**** (masked)"
}

var helpText = `Usage: ghr [options...] TAG PATH

ghr is a tool to create Release on Github and upload your
artifacts to it. ghr parallelizes upload of multiple artifacts.

You must specify tag (e.g., v1.0.0) and PATH to local artifacts.
If PATH is directory, ghr globs all files in the directory and
upload it. If PATH is a file then, upload only it.

And you also must provide GitHub API token which has enough permission
(For a private repository you need the 'repo' scope and for a public
repository need 'public_repo' scope). You can get token from GitHub's
account setting page.

You can use ghr on GitHub Enterprise. Set base URL via GITHUB_API
environment variable.

Options:

  -username, -u      Github repository owner name. By default, ghr
                     extracts it from global gitconfig value.

  -repository, -r    GitHub repository name. By default, ghr extracts
                     repository name from current directory's .git/config.

  -name, -n          GitHub release title. By default the tag is used.

  -token, -t         GitHub API Token. By default, ghr reads it from
                     'GITHUB_TOKEN' env var.

  -parallel=-1       Parallelization factor. This option limits amount
                     of parallelism of uploading. By default, ghr uses
                     number of logic CPU.

  -recreate          Recreate release if it already exists. If want to
                     upload to same release and replace use '-replace'.

  -replace           Replace artifacts if it is already uploaded. ghr
                     thinks it's same when local artifact base name
                     and uploaded file name are same.

`
<commit_msg>set empty string by default<commit_after>// Command ghr is a tool to create a Github Release and upload your
// artifacts in parallel.
package main

import (
	"context"
	"flag"
	"fmt"
	"io"
	"log"
	"os"
	"regexp"
	"runtime"
	"time"

	"github.com/google/go-github/github"
	"github.com/mitchellh/colorstring"
	"github.com/tcnksm/go-gitconfig"
)

const (
	// EnvGitHubToken is an environment var containing the GitHub API token
	EnvGitHubToken = "GITHUB_TOKEN"

	// EnvGitHubAPI is an environment var containing the GitHub API base endpoint.
	// This is used mainly by GitHub Enterprise users.
	EnvGitHubAPI = "GITHUB_API"

	// EnvDebug is an environment var to handle debug mode
	EnvDebug = "GHR_DEBUG"
)

// Exit codes are set to a value that represent an exit code for a particular error.
const (
	ExitCodeOK int = 0

	// Errors start at 10
	ExitCodeError = 10 + iota
	ExitCodeParseFlagsError
	ExitCodeBadArgs
	ExitCodeInvalidURL
	ExitCodeTokenNotFound
	ExitCodeOwnerNotFound
	ExitCodeRepoNotFound
	ExitCodeReleaseError
)

const (
	defaultCheckTimeout = 2 * time.Second
	defaultBaseURL      = "https://api.github.com/"
	defaultParallel     = -1
)

// Debugf prints debug output when EnvDebug is set
func Debugf(format string, args ...interface{}) {
	if env := os.Getenv(EnvDebug); len(env) != 0 {
		log.Printf("[DEBUG] "+format+"\n", args...)
	}
}

// PrintRedf prints red error message to console.
func PrintRedf(w io.Writer, format string, args ...interface{}) {
	format = fmt.Sprintf("[red]%s[reset]", format)
	fmt.Fprint(w,
		colorstring.Color(fmt.Sprintf(format, args...)))
}

// CLI is the main command line object
type CLI struct {
	// outStream and errStream correspond to stdout and stderr, respectively,
	// to take messages from the CLI.
	outStream, errStream io.Writer
}

// Run invokes the CLI with the given arguments.
func (cli *CLI) Run(args []string) int {

	var (
		owner string
		repo  string
		token string

		commitish  string
		name       string
		body       string
		draft      bool
		prerelease bool

		parallel int

		recreate bool
		replace  bool

		stat    bool
		version bool
		debug   bool
	)

	flags := flag.NewFlagSet(Name, flag.ContinueOnError)
	flags.SetOutput(cli.errStream)
	flags.Usage = func() {
		fmt.Fprint(cli.errStream, helpText)
	}

	flags.StringVar(&owner, "username", "", "")
	flags.StringVar(&owner, "owner", "", "")
	flags.StringVar(&owner, "u", "", "")

	flags.StringVar(&repo, "repository", "", "")
	flags.StringVar(&repo, "r", "", "")

	flags.StringVar(&token, "token", os.Getenv(EnvGitHubToken), "")
	flags.StringVar(&token, "t", os.Getenv(EnvGitHubToken), "")

	flags.StringVar(&commitish, "commitish", "", "")
	flags.StringVar(&commitish, "c", "", "")

	flags.StringVar(&name, "name", "", "")
	flags.StringVar(&name, "n", "", "")

	flags.StringVar(&body, "body", "", "")
	flags.StringVar(&body, "b", "", "")

	flags.BoolVar(&draft, "draft", false, "")
	flags.BoolVar(&prerelease, "prerelease", false, "")

	flags.IntVar(&parallel, "parallel", defaultParallel, "")
	flags.IntVar(&parallel, "p", defaultParallel, "")

	flags.BoolVar(&recreate, "delete", false, "")
	flags.BoolVar(&recreate, "recreate", false, "")

	flags.BoolVar(&replace, "replace", false, "")

	flags.BoolVar(&version, "version", false, "")
	flags.BoolVar(&version, "v", false, "")

	flags.BoolVar(&debug, "debug", false, "")

	// Deprecated
	flags.BoolVar(&stat, "stat", false, "")

	// Parse flags
	if err := flags.Parse(args[1:]); err != nil {
		return ExitCodeParseFlagsError
	}

	if debug {
		os.Setenv(EnvDebug, "1")
		Debugf("Run as DEBUG mode")
	}

	// Show version and check latest version release
	if version {
		fmt.Fprintf(cli.outStream, OutputVersion())
		return ExitCodeOK
	}

	parsedArgs := flags.Args()
	if len(parsedArgs) != 2 {
		PrintRedf(cli.errStream,
			"Invalid argument: you must set TAG and PATH name.\n")
		return ExitCodeBadArgs
	}
	tag, path := parsedArgs[0], parsedArgs[1]

	// Extract github repository owner username.
	// If it's not provided via command line flag, read it from .gitconfig
	// (github user or git user).
	if len(owner) == 0 {
		origin, err := gitconfig.OriginURL()
		if err == nil {
			owner = retrieveOwnerName(origin)
		}
		if len(owner) == 0 {
			owner, err = gitconfig.GithubUser()
			if err != nil {
				owner, err = gitconfig.Username()
			}

			if err != nil {
				PrintRedf(cli.errStream,
					"Failed to set up ghr: repository owner name not found\n")
				fmt.Fprintf(cli.errStream,
					"Please set it via `-u` option.\n\n"+
						"You can set default owner name in `github.username` or `user.name`\n"+
						"in `~/.gitconfig` file\n")
				return ExitCodeOwnerNotFound
			}
		}
	}
	Debugf("Owner: %s", owner)

	// Extract repository name from files.
	// If not provided, read it from .git/config file.
	if len(repo) == 0 {
		var err error
		repo, err = gitconfig.Repository()
		if err != nil {
			PrintRedf(cli.errStream,
				"Failed to set up ghr: repository name not found\n")
			fmt.Fprintf(cli.errStream,
				"ghr reads it from `.git/config` file. Change directory to \n"+
					"repository root directory or setup git repository.\n"+
					"Or set it via `-r` option.\n")
			return ExitCodeRepoNotFound
		}
	}
	Debugf("Repository: %s", repo)

	// If GitHub API token is not provided via command line flag
	// or env var then read it from .gitconfig file.
	if len(token) == 0 {
		var err error
		token, err = gitconfig.GithubToken()
		if err != nil {
			PrintRedf(cli.errStream, "Failed to set up ghr: token not found\n")
			fmt.Fprintf(cli.errStream,
				"To use ghr, you need a GitHub API token.\n"+
					"Please set it via `%s` env var or `-t` option.\n\n"+
					"If you don't have one, visit official doc (goo.gl/jSnoI)\n"+
					"and get it first.\n",
				EnvGitHubToken)
			return ExitCodeTokenNotFound
		}
	}
	Debugf("Github API Token: %s", maskString(token))

	// Set Base GitHub API URL. Base URL can also be provided via env var for use with GHE.
	baseURLStr := defaultBaseURL
	if urlStr := os.Getenv(EnvGitHubAPI); len(urlStr) != 0 {
		baseURLStr = urlStr
	}
	Debugf("Base GitHub API URL: %s", baseURLStr)

	if parallel <= 0 {
		parallel = runtime.NumCPU()
	}
	Debugf("Parallel factor: %d", parallel)

	localAssets, err := LocalAssets(path)
	if err != nil {
		PrintRedf(cli.errStream,
			"Failed to find assets from %s: %s\n", path, err)
		return ExitCodeError
	}
	Debugf("Number of file to upload: %d", len(localAssets))

	// Create a GitHub client
	gitHubClient, err := NewGitHubClient(owner, repo, token, baseURLStr)
	if err != nil {
		PrintRedf(cli.errStream, "Failed to construct GitHub client: %s\n", err)
		return ExitCodeError
	}

	ghr := GHR{
		GitHub:    gitHubClient,
		outStream: cli.outStream,
	}

	Debugf("Name: %s", name)

	// Prepare create release request
	req := &github.RepositoryRelease{
		Name:            github.String(name),
		TagName:         github.String(tag),
		Prerelease:      github.Bool(prerelease),
		Draft:           github.Bool(draft),
		TargetCommitish: github.String(commitish),
		Body:            github.String(body),
	}

	ctx := context.TODO()
	release, err := ghr.CreateRelease(ctx, req, recreate)
	if err != nil {
		PrintRedf(cli.errStream, "Failed to create GitHub release page: %s\n", err)
		return ExitCodeError
	}

	if replace {
		err := ghr.DeleteAssets(ctx, *release.ID, localAssets, parallel)
		if err != nil {
			PrintRedf(cli.errStream, "Failed to delete existing assets: %s\n", err)
			return ExitCodeError
		}
	}

	// FIXME(tcnksm): More ideal way to change this
	// This is for Github enterprise
	if err := ghr.GitHub.SetUploadURL(*release.UploadURL); err != nil {
		fmt.Fprintf(cli.errStream, "Failed to set upload URL %s: %s\n", *release.UploadURL, err)
		return ExitCodeError
	}

	err = ghr.UploadAssets(ctx, *release.ID, localAssets, parallel)
	if err != nil {
		PrintRedf(cli.errStream, "Failed to upload one of assets: %s\n", err)
		return ExitCodeError
	}

	if !draft {
		_, err := ghr.GitHub.EditRelease(ctx, *release.ID, &github.RepositoryRelease{
			Draft: github.Bool(false),
		})
		if err != nil {
			PrintRedf(cli.errStream, "Failed to publish release: %s\n", err)
			return ExitCodeError
		}
	}

	return ExitCodeOK
}

var ownerNameReg = regexp.MustCompile(`([-a-zA-Z0-9]+)/[^/]+$`)

func retrieveOwnerName(repoURL string) string {
	matched := ownerNameReg.FindStringSubmatch(repoURL)
	if len(matched) < 2 {
		return ""
	}
	return matched[1]
}

// maskString is used to mask a string which should not be displayed
// directly, like the auth token
func maskString(s string) string {
	if len(s) < 5 {
		return "**** (masked)"
	}

	return s[:5] + "**** (masked)"
}

var helpText = `Usage: ghr [options...] TAG PATH

ghr is a tool to create Release on Github and upload your
artifacts to it. ghr parallelizes upload of multiple artifacts.

You must specify tag (e.g., v1.0.0) and PATH to local artifacts.
If PATH is directory, ghr globs all files in the directory and
upload it. If PATH is a file then, upload only it.

And you also must provide GitHub API token which has enough permission
(For a private repository you need the 'repo' scope and for a public
repository need 'public_repo' scope). You can get token from GitHub's
account setting page.

You can use ghr on GitHub Enterprise. Set base URL via GITHUB_API
environment variable.

Options:

  -username, -u      Github repository owner name. By default, ghr
                     extracts it from global gitconfig value.

  -repository, -r    GitHub repository name. By default, ghr extracts
                     repository name from current directory's .git/config.

  -name, -n          GitHub release title. By default the tag is used.

  -token, -t         GitHub API Token. By default, ghr reads it from
                     'GITHUB_TOKEN' env var.

  -parallel=-1       Parallelization factor. This option limits amount
                     of parallelism of uploading. By default, ghr uses
                     number of logic CPU.

  -recreate          Recreate release if it already exists. If want to
                     upload to same release and replace use '-replace'.

  -replace           Replace artifacts if it is already uploaded. ghr
                     thinks it's same when local artifact base name
                     and uploaded file name are same.

`
<|endoftext|>


<commit_before>package main

import (
	"bytes"
	"encoding/json"
	"fmt"
	"io"
	"io/ioutil"
	"net/http"
	"net/url"
	"os"
	"strconv"
)

func infocmd(opt Options) error {
	user := nvls(opt.Info.User, EnvUser)
	repo := nvls(opt.Info.Repo, EnvRepo)
	token := nvls(opt.Info.Token, EnvToken)
	tag := opt.Info.Tag

	if user == "" || repo == "" {
		return fmt.Errorf("user and repo need to be passed as arguments")
	}

	/* find regular git tags */
	allTags, err := Tags(user, repo, token)
	if err != nil {
		return fmt.Errorf("could not fetch tags, %v", err)
	}
	if len(allTags) == 0 {
		return fmt.Errorf("no tags available for %v/%v", user, repo)
	}

	/* list all tags */
	tags := make([]Tag, 0, len(allTags))
	for _, t := range allTags {
		/* if the user only requested see one tag, skip the ones that
		 * don't match */
		if tag != "" && t.Name != tag {
			continue
		}
		tags = append(tags, t)
	}

	/* if no tags conformed to the users' request, exit */
	if len(tags) == 0 {
		return fmt.Errorf("no tag '%v' was found for %v/%v", tag, user, repo)
	}

	fmt.Println("git tags:")
	for _, t := range tags {
		fmt.Println("-", t.String())
	}

	/* list releases + assets */
	var releases []Release
	if tag == "" {
		/* get all releases */
		vprintf("%v/%v: getting information for all releases\n", user, repo)
		releases, err = Releases(user, repo, token)
		if err != nil {
			return err
		}
	} else {
		/* get only one release */
		vprintf("%v/%v/%v: getting information for the release\n", user, repo, tag)
		release, err := ReleaseOfTag(user, repo, tag, token)
		if err != nil {
			return err
		}
		releases = []Release{*release}
	}

	/* if no tags conformed to the users' request, exit */
	if len(releases) == 0 {
		return fmt.Errorf("no release(s) were found for %v/%v (%v)", user, repo, tag)
	}

	fmt.Println("releases:")
	for _, release := range releases {
		fmt.Println("-", release.String())
	}

	return nil
}

func uploadcmd(opt Options) error {
	user := nvls(opt.Upload.User, EnvUser)
	repo := nvls(opt.Upload.Repo, EnvRepo)
	token := nvls(opt.Upload.Token, EnvToken)
	tag := opt.Upload.Tag
	name := opt.Upload.Name
	file := opt.Upload.File

	vprintln("uploading...")

	if file == nil {
		return fmt.Errorf("provided file was not valid")
	}

	if err := ValidateCredentials(user, repo, token, tag); err != nil {
		return err
	}

	/* find the release corresponding to the entered tag, if any */
	rel, err := ReleaseOfTag(user, repo, tag, token)
	if err != nil {
		return err
	}

	v := url.Values{}
	v.Set("name", name)

	url := rel.CleanUploadUrl() + "?" + v.Encode()

	resp, err := DoAuthRequest("POST", url, "application/octet-stream",
		token, nil, file)
	if err != nil {
		return fmt.Errorf("can't create upload request to %v, %v", url, err)
	}
	defer resp.Body.Close()

	vprintln("RESPONSE:", resp)
	if resp.StatusCode != http.StatusCreated {
		if msg, err := ToMessage(resp.Body); err == nil {
			return fmt.Errorf("could not upload, status code (%v), %v",
				resp.Status, msg)
		} else {
			return fmt.Errorf("could not upload, status code (%v)", resp.Status)
		}
	}

	if VERBOSITY != 0 {
		body, err := ioutil.ReadAll(resp.Body)
		if err != nil {
			return fmt.Errorf("error while reading response, %v", err)
		}
		vprintln("BODY:", string(body))
	}

	return nil
}

func downloadcmd(opt Options) error {
	user := nvls(opt.Download.User, EnvUser)
	repo := nvls(opt.Download.Repo, EnvRepo)
	token := nvls(opt.Download.Token, EnvToken)
	tag := opt.Download.Tag
	name := opt.Download.Name

	vprintln("downloading...")

	if err := ValidateTarget(user, repo, tag); err != nil {
		return err
	}

	/* find the release corresponding to the entered tag, if any */
	rel, err := ReleaseOfTag(user, repo, tag, token)
	if err != nil {
		return err
	}

	assetId := 0
	for _, asset := range rel.Assets {
		if asset.Name == name {
			assetId = asset.Id
		}
	}

	if assetId == 0 {
		return fmt.Errorf("coud not find asset named %s", name)
	}

	var resp *http.Response
	var url string
	if token == "" {
		url = GH_URL + fmt.Sprintf("/%s/%s/releases/download/%s/%s", user, repo, tag, name)
		resp, err = http.Get(url)
	} else {
		url = ApiURL() + fmt.Sprintf(ASSET_DOWNLOAD_URI, user, repo, assetId)
		resp, err = DoAuthRequest("GET", url, "", token, map[string]string{
			"Accept": "application/octet-stream",
		}, nil)
	}
	if err != nil {
		return fmt.Errorf("could not fetch releases, %v", err)
	}

	defer resp.Body.Close()
	vprintln("GET", url, "->", resp)

	contentLength, err := strconv.ParseInt(resp.Header.Get("Content-Length"), 10, 64)
	if err != nil {
		return err
	}

	if resp.StatusCode != http.StatusOK {
		return fmt.Errorf("github did not respond with 200 OK but with %v", resp.Status)
	}

	out, err := os.Create(name)
	if err != nil {
		return fmt.Errorf("could not create file %s", name)
	}
	defer out.Close()

	n, err := io.Copy(out, resp.Body)
	if err != nil {
		return err
	}
	if n != contentLength {
		return fmt.Errorf("downloaded data did not match content length %s != %s", contentLength, n)
	}

	return nil
}

func ValidateTarget(user, repo, tag string) error {
	if user == "" {
		return fmt.Errorf("empty user")
	}
	if repo == "" {
		return fmt.Errorf("empty repo")
	}
	if tag == "" {
		return fmt.Errorf("empty tag")
	}
	return nil
}

func ValidateCredentials(user, repo, token, tag string) error {
	if err := ValidateTarget(user, repo, tag); err != nil {
		return err
	}
	if token == "" {
		return fmt.Errorf("empty token")
	}
	return nil
}

func releasecmd(opt Options) error {
	cmdopt := opt.Release
	user := nvls(cmdopt.User, EnvUser)
	repo := nvls(cmdopt.Repo, EnvRepo)
	token := nvls(cmdopt.Token, EnvToken)
	tag := cmdopt.Tag
	name := nvls(cmdopt.Name, tag)
	desc := nvls(cmdopt.Desc, tag)
	draft := cmdopt.Draft
	prerelease := cmdopt.Prerelease

	vprintln("releasing...")

	if err := ValidateCredentials(user, repo, token, tag); err != nil {
		return err
	}

	params := ReleaseCreate{
		TagName:    tag,
		Name:       name,
		Body:       desc,
		Draft:      draft,
		Prerelease: prerelease,
	}

	/* encode params as json */
	payload, err := json.Marshal(params)
	if err != nil {
		return fmt.Errorf("can't encode release creation params, %v", err)
	}
	reader := bytes.NewReader(payload)

	uri := fmt.Sprintf("/repos/%s/%s/releases", user, repo)
	resp, err := DoAuthRequest("POST", ApiURL()+uri, "application/json",
		token, nil, reader)
	if err != nil {
		return fmt.Errorf("while submitting %v, %v", string(payload), err)
	}
	defer resp.Body.Close()

	vprintln("RESPONSE:", resp)
	if resp.StatusCode != http.StatusCreated {
		if resp.StatusCode == 422 {
			return fmt.Errorf("github returned %v (this is probably because the release already exists)",
				resp.Status)
		}
		return fmt.Errorf("github returned %v", resp.Status)
	}

	if VERBOSITY != 0 {
		body, err := ioutil.ReadAll(resp.Body)
		if err != nil {
			return fmt.Errorf("error while reading response, %v", err)
		}
		vprintln("BODY:", string(body))
	}

	return nil
}

func editcmd(opt Options) error {
	cmdopt := opt.Edit
	user := nvls(cmdopt.User, EnvUser)
	repo := nvls(cmdopt.Repo, EnvRepo)
	token := nvls(cmdopt.Token, EnvToken)
	tag := cmdopt.Tag
	name := nvls(cmdopt.Name, tag)
	desc := nvls(cmdopt.Desc, tag)
	draft := cmdopt.Draft
	prerelease := cmdopt.Prerelease

	vprintln("editing...")

	if err := ValidateCredentials(user, repo, token, tag); err != nil {
		return err
	}

	id, err := IdOfTag(user, repo, tag, token)
	if err != nil {
		return err
	}

	vprintf("release %v has id %v\n", tag, id)

	/* the release create struct works for editing releases as well */
	params := ReleaseCreate{
		TagName:    tag,
		Name:       name,
		Body:       desc,
		Draft:      draft,
		Prerelease: prerelease,
	}

	/* encode the parameters as JSON, as required by the github API */
	payload, err := json.Marshal(params)
	if err != nil {
		return fmt.Errorf("can't encode release creation params, %v", err)
	}

	uri := fmt.Sprintf("/repos/%s/%s/releases/%d", user, repo, id)
	resp, err := DoAuthRequest("PATCH", ApiURL()+uri, "application/json",
		token, nil, bytes.NewReader(payload))
	if err != nil {
		return fmt.Errorf("while submitting %v, %v", string(payload), err)
	}
	defer resp.Body.Close()

	vprintln("RESPONSE:", resp)
	if resp.StatusCode != http.StatusOK {
		if resp.StatusCode == 422 {
			return fmt.Errorf("github returned %v (this is probably because the release already exists)",
				resp.Status)
		}
		return fmt.Errorf("github returned unexpected status code %v", resp.Status)
	}

	if VERBOSITY != 0 {
		body, err := ioutil.ReadAll(resp.Body)
		if err != nil {
			return fmt.Errorf("error while reading response, %v", err)
		}
		vprintln("BODY:", string(body))
	}

	return nil
}

func deletecmd(opt Options) error {
	user, repo, token, tag := nvls(opt.Delete.User, EnvUser),
		nvls(opt.Delete.Repo, EnvRepo),
		nvls(opt.Delete.Token, EnvToken),
		opt.Delete.Tag
	vprintln("deleting...")

	id, err := IdOfTag(user, repo, tag, token)
	if err != nil {
		return err
	}

	vprintf("release %v has id %v\n", tag, id)

	resp, err := httpDelete(ApiURL()+fmt.Sprintf("/repos/%s/%s/releases/%d",
		user, repo, id), token)
	if err != nil {
		return fmt.Errorf("release deletion unsuccesful, %v", err)
	}
	defer resp.Body.Close()

	if resp.StatusCode != http.StatusNoContent {
		return fmt.Errorf("could not delete the release corresponding to tag %s on repo %s/%s",
			tag, user, repo)
	}

	return nil
}

func httpDelete(url, token string) (*http.Response, error) {
	resp, err := DoAuthRequest("DELETE", url, "application/json", token, nil, nil)
	if err != nil {
		return nil, err
	}

	return resp, nil
}
<commit_msg>cmd: fix vet warning (real error)<commit_after>package main

import (
	"bytes"
	"encoding/json"
	"fmt"
	"io"
	"io/ioutil"
	"net/http"
	"net/url"
	"os"
	"strconv"
)

func infocmd(opt Options) error {
	user := nvls(opt.Info.User, EnvUser)
	repo := nvls(opt.Info.Repo, EnvRepo)
	token := nvls(opt.Info.Token, EnvToken)
	tag := opt.Info.Tag

	if user == "" || repo == "" {
		return fmt.Errorf("user and repo need to be passed as arguments")
	}

	/* find regular git tags */
	allTags, err := Tags(user, repo, token)
	if err != nil {
		return fmt.Errorf("could not fetch tags, %v", err)
	}
	if len(allTags) == 0 {
		return fmt.Errorf("no tags available for %v/%v", user, repo)
	}

	/* list all tags */
	tags := make([]Tag, 0, len(allTags))
	for _, t := range allTags {
		/* if the user only requested see one tag, skip the ones that
		 * don't match */
		if tag != "" && t.Name != tag {
			continue
		}
		tags = append(tags, t)
	}

	/* if no tags conformed to the users' request, exit */
	if len(tags) == 0 {
		return fmt.Errorf("no tag '%v' was found for %v/%v", tag, user, repo)
	}

	fmt.Println("git tags:")
	for _, t := range tags {
		fmt.Println("-", t.String())
	}

	/* list releases + assets */
	var releases []Release
	if tag == "" {
		/* get all releases */
		vprintf("%v/%v: getting information for all releases\n", user, repo)
		releases, err = Releases(user, repo, token)
		if err != nil {
			return err
		}
	} else {
		/* get only one release */
		vprintf("%v/%v/%v: getting information for the release\n", user, repo, tag)
		release, err := ReleaseOfTag(user, repo, tag, token)
		if err != nil {
			return err
		}
		releases = []Release{*release}
	}

	/* if no tags conformed to the users' request, exit */
	if len(releases) == 0 {
		return fmt.Errorf("no release(s) were found for %v/%v (%v)", user, repo, tag)
	}

	fmt.Println("releases:")
	for _, release := range releases {
		fmt.Println("-", release.String())
	}

	return nil
}

func uploadcmd(opt Options) error {
	user := nvls(opt.Upload.User, EnvUser)
	repo := nvls(opt.Upload.Repo, EnvRepo)
	token := nvls(opt.Upload.Token, EnvToken)
	tag := opt.Upload.Tag
	name := opt.Upload.Name
	file := opt.Upload.File

	vprintln("uploading...")

	if file == nil {
		return fmt.Errorf("provided file was not valid")
	}

	if err := ValidateCredentials(user, repo, token, tag); err != nil {
		return err
	}

	/* find the release corresponding to the entered tag, if any */
	rel, err := ReleaseOfTag(user, repo, tag, token)
	if err != nil {
		return err
	}

	v := url.Values{}
	v.Set("name", name)

	url := rel.CleanUploadUrl() + "?" + v.Encode()

	resp, err := DoAuthRequest("POST", url, "application/octet-stream",
		token, nil, file)
	if err != nil {
		return fmt.Errorf("can't create upload request to %v, %v", url, err)
	}
	defer resp.Body.Close()

	vprintln("RESPONSE:", resp)
	if resp.StatusCode != http.StatusCreated {
		if msg, err := ToMessage(resp.Body); err == nil {
			return fmt.Errorf("could not upload, status code (%v), %v",
				resp.Status, msg)
		} else {
			return fmt.Errorf("could not upload, status code (%v)", resp.Status)
		}
	}

	if VERBOSITY != 0 {
		body, err := ioutil.ReadAll(resp.Body)
		if err != nil {
			return fmt.Errorf("error while reading response, %v", err)
		}
		vprintln("BODY:", string(body))
	}

	return nil
}

func downloadcmd(opt Options) error {
	user := nvls(opt.Download.User, EnvUser)
	repo := nvls(opt.Download.Repo, EnvRepo)
	token := nvls(opt.Download.Token, EnvToken)
	tag := opt.Download.Tag
	name := opt.Download.Name

	vprintln("downloading...")

	if err := ValidateTarget(user, repo, tag); err != nil {
		return err
	}

	/* find the release corresponding to the entered tag, if any */
	rel, err := ReleaseOfTag(user, repo, tag, token)
	if err != nil {
		return err
	}

	assetId := 0
	for _, asset := range rel.Assets {
		if asset.Name == name {
			assetId = asset.Id
		}
	}

	if assetId == 0 {
		return fmt.Errorf("coud not find asset named %s", name)
	}

	var resp *http.Response
	var url string
	if token == "" {
		url = GH_URL + fmt.Sprintf("/%s/%s/releases/download/%s/%s", user, repo, tag, name)
		resp, err = http.Get(url)
	} else {
		url = ApiURL() + fmt.Sprintf(ASSET_DOWNLOAD_URI, user, repo, assetId)
		resp, err = DoAuthRequest("GET", url, "", token, map[string]string{
			"Accept": "application/octet-stream",
		}, nil)
	}
	if err != nil {
		return fmt.Errorf("could not fetch releases, %v", err)
	}

	defer resp.Body.Close()
	vprintln("GET", url, "->", resp)

	contentLength, err := strconv.ParseInt(resp.Header.Get("Content-Length"), 10, 64)
	if err != nil {
		return err
	}

	if resp.StatusCode != http.StatusOK {
		return fmt.Errorf("github did not respond with 200 OK but with %v", resp.Status)
	}

	out, err := os.Create(name)
	if err != nil {
		return fmt.Errorf("could not create file %s", name)
	}
	defer out.Close()

	n, err := io.Copy(out, resp.Body)
	if err != nil {
		return err
	}
	if n != contentLength {
		return fmt.Errorf("downloaded data did not match content length %d != %d", contentLength, n)
	}

	return nil
}

func ValidateTarget(user, repo, tag string) error {
	if user == "" {
		return fmt.Errorf("empty user")
	}
	if repo == "" {
		return fmt.Errorf("empty repo")
	}
	if tag == "" {
		return fmt.Errorf("empty tag")
	}
	return nil
}

func ValidateCredentials(user, repo, token, tag string) error {
	if err := ValidateTarget(user, repo, tag); err != nil {
		return err
	}
	if token == "" {
		return fmt.Errorf("empty token")
	}
	return nil
}

func releasecmd(opt Options) error {
	cmdopt := opt.Release
	user := nvls(cmdopt.User, EnvUser)
	repo := nvls(cmdopt.Repo, EnvRepo)
	token := nvls(cmdopt.Token, EnvToken)
	tag := cmdopt.Tag
	name := nvls(cmdopt.Name, tag)
	desc := nvls(cmdopt.Desc, tag)
	draft := cmdopt.Draft
	prerelease := cmdopt.Prerelease

	vprintln("releasing...")

	if err := ValidateCredentials(user, repo, token, tag); err != nil {
		return err
	}

	params := ReleaseCreate{
		TagName:    tag,
		Name:       name,
		Body:       desc,
		Draft:      draft,
		Prerelease: prerelease,
	}

	/* encode params as json */
	payload, err := json.Marshal(params)
	if err != nil {
		return fmt.Errorf("can't encode release creation params, %v", err)
	}
	reader := bytes.NewReader(payload)

	uri := fmt.Sprintf("/repos/%s/%s/releases", user, repo)
	resp, err := DoAuthRequest("POST", ApiURL()+uri, "application/json",
		token, nil, reader)
	if err != nil {
		return fmt.Errorf("while submitting %v, %v", string(payload), err)
	}
	defer resp.Body.Close()

	vprintln("RESPONSE:", resp)
	if resp.StatusCode != http.StatusCreated {
		if resp.StatusCode == 422 {
			return fmt.Errorf("github returned %v (this is probably because the release already exists)",
				resp.Status)
		}
		return fmt.Errorf("github returned %v", resp.Status)
	}

	if VERBOSITY != 0 {
		body, err := ioutil.ReadAll(resp.Body)
		if err != nil {
			return fmt.Errorf("error while reading response, %v", err)
		}
		vprintln("BODY:", string(body))
	}

	return nil
}

func editcmd(opt Options) error {
	cmdopt := opt.Edit
	user := nvls(cmdopt.User, EnvUser)
	repo := nvls(cmdopt.Repo, EnvRepo)
	token := nvls(cmdopt.Token, EnvToken)
	tag := cmdopt.Tag
	name := nvls(cmdopt.Name, tag)
	desc := nvls(cmdopt.Desc, tag)
	draft := cmdopt.Draft
	prerelease := cmdopt.Prerelease

	vprintln("editing...")

	if err := ValidateCredentials(user, repo, token, tag); err != nil {
		return err
	}

	id, err := IdOfTag(user, repo, tag, token)
	if err != nil {
		return err
	}

	vprintf("release %v has id %v\n", tag, id)

	/* the release create struct works for editing releases as well */
	params := ReleaseCreate{
		TagName:    tag,
		Name:       name,
		Body:       desc,
		Draft:      draft,
		Prerelease: prerelease,
	}

	/* encode the parameters as JSON, as required by the github API */
	payload, err := json.Marshal(params)
	if err != nil {
		return fmt.Errorf("can't encode release creation params, %v", err)
	}

	uri := fmt.Sprintf("/repos/%s/%s/releases/%d", user, repo, id)
	resp, err := DoAuthRequest("PATCH", ApiURL()+uri, "application/json",
		token, nil, bytes.NewReader(payload))
	if err != nil {
		return fmt.Errorf("while submitting %v, %v", string(payload), err)
	}
	defer resp.Body.Close()

	vprintln("RESPONSE:", resp)
	if resp.StatusCode != http.StatusOK {
		if resp.StatusCode == 422 {
			return fmt.Errorf("github returned %v (this is probably because the release already exists)",
				resp.Status)
		}
		return fmt.Errorf("github returned unexpected status code %v", resp.Status)
	}

	if VERBOSITY != 0 {
		body, err := ioutil.ReadAll(resp.Body)
		if err != nil {
			return fmt.Errorf("error while reading response, %v", err)
		}
		vprintln("BODY:", string(body))
	}

	return nil
}

func deletecmd(opt Options) error {
	user, repo, token, tag := nvls(opt.Delete.User, EnvUser),
		nvls(opt.Delete.Repo, EnvRepo),
		nvls(opt.Delete.Token, EnvToken),
		opt.Delete.Tag
	vprintln("deleting...")

	id, err := IdOfTag(user, repo, tag, token)
	if err != nil {
		return err
	}

	vprintf("release %v has id %v\n", tag, id)

	resp, err := httpDelete(ApiURL()+fmt.Sprintf("/repos/%s/%s/releases/%d",
		user, repo, id), token)
	if err != nil {
		return fmt.Errorf("release deletion unsuccesful, %v", err)
	}
	defer resp.Body.Close()

	if resp.StatusCode != http.StatusNoContent {
		return fmt.Errorf("could not delete the release corresponding to tag %s on repo %s/%s",
			tag, user, repo)
	}

	return nil
}

func httpDelete(url, token string) (*http.Response, error) {
	resp, err := DoAuthRequest("DELETE", url, "application/json", token, nil, nil)
	if err != nil {
		return nil, err
	}

	return resp, nil
}
<|endoftext|>


<commit_before>package df2014

import (
	"bufio"
	"bytes"
	"encoding/binary"
	"fmt"
	"io"
	"time"
)

type CMVStream struct {
	Header CMVHeader
	Sounds *CMVSounds
	Frames <-chan CMVFrame
}

type CMVHeader struct {
	Version uint32

	Columns   uint32
	Rows      uint32
	FrameTime time.Duration
}

type CMVSounds struct {
	Files  []string
	Timing [200][16]uint32
}

type CMVFrame struct {
	Characters [][]CMVCharacter
	Attributes [][]CMVAttribute
}

type CMVCharacter uint8

func (c CMVCharacter) Byte() byte {
	return byte(c)
}

func (c CMVCharacter) Rune() rune {
	return cp437[c]
}

func (c CMVCharacter) String() string {
	return string(c.Rune())
}

type CMVAttribute uint8

func (c CMVAttribute) Fg() CMVColor {
	return CMVColor((c<<1)&0xE | (c >> 6))
}

func (c CMVAttribute) Bg() CMVColor {
	return CMVColor((c >> 2) & 0xE)
}

type CMVColor uint8

const (
	ColorBlack CMVColor = iota
	ColorDGray
	ColorBlue
	ColorLBlue
	ColorGreen
	ColorLGreen
	ColorCyan
	ColorLCyan
	ColorRed
	ColorLRed
	ColorMagenta
	ColorLMagenta
	ColorBrown
	ColorYellow
	ColorLGray
	ColorWhite
)

func RawCMV(r io.Reader) (header CMVHeader, sounds *CMVSounds, frames io.Reader, err error) {
	in := &Reader{bufio.NewReader(r)}

	err = binary.Read(in, binary.LittleEndian, &header.Version)
	if err != nil {
		return
	}

	if header.Version < 10000 || header.Version > 10001 {
		err = fmt.Errorf("df2014: unhandled version %d", header.Version)
		return
	}

	err = binary.Read(in, binary.LittleEndian, &header.Columns)
	if err != nil {
		return
	}

	err = binary.Read(in, binary.LittleEndian, &header.Rows)
	if err != nil {
		return
	}

	var frameTimeRaw uint32
	err = binary.Read(in, binary.LittleEndian, &frameTimeRaw)
	if err != nil {
		return
	}
	if frameTimeRaw == 0 {
		frameTimeRaw = 2
	}
	header.FrameTime = time.Duration(frameTimeRaw) * time.Second / 100

	if header.Version >= 10001 {
		sounds = new(CMVSounds)

		var soundsRaw [][50]byte
		err = in.DecodeSimple(&soundsRaw)
		if err != nil {
			return
		}

		for _, sound := range soundsRaw {
			sounds.Files = append(sounds.Files, string(sound[:bytes.IndexByte(sound[:], 0)]))
		}

		err = in.DecodeSimple(&sounds.Timing)
		if err != nil {
			return
		}
	}

	in.Reader = &compression1Reader{r: in.Reader}
	frames = in
	return
}

func StreamCMV(in io.ReadCloser, buffer int) (cmv CMVStream, err error) {
	header, sounds, r, err := RawCMV(in)
	if err != nil {
		in.Close()
		return
	}

	frames := make(chan CMVFrame, buffer)
	cmv.Header, cmv.Sounds, cmv.Frames = header, sounds, frames

	size := cmv.Header.Columns * cmv.Header.Rows

	go func() {
		defer close(frames)
		defer in.Close()

		for {
			buf := make([]byte, size)
			_, err := io.ReadFull(r, buf)
			if err != nil {
				if err == io.EOF {
					return
				}
				// TODO: some sort of error handling
				return
			}

			var frame CMVFrame

			characters := make([]CMVCharacter, size)
			frame.Characters = make([][]CMVCharacter, cmv.Header.Columns)
			for i := range frame.Characters {
				frame.Characters[i] = characters[i*int(cmv.Header.Rows) : (i+1)*int(cmv.Header.Rows)]
			}

			for i, b := range buf {
				characters[i] = CMVCharacter(b)
			}

			_, err = io.ReadFull(r, buf)
			if err != nil {
				// TODO: some sort of error handling
				return
			}

			attributes := make([]CMVAttribute, size)
			frame.Attributes = make([][]CMVAttribute, cmv.Header.Columns)
			for i := range frame.Attributes {
				frame.Attributes[i] = attributes[i*int(cmv.Header.Rows) : (i+1)*int(cmv.Header.Rows)]
			}

			for i, b := range buf {
				attributes[i] = CMVAttribute(b)
			}

			frames <- frame
		}
	}()
	return
}
<commit_msg>remove old CMV code<commit_after><|endoftext|>


<commit_before>package cwg

import (
	"sync/atomic"
	"time"
)

type CWG struct {
	channel chan interface{}
	counter int64
}

func New() *CWG {
	cwg := new(CWG)
	cwg.channel = make(chan interface{})
	return cwg
}

func (cwg *CWG) Add(a int64) int64 {
	return atomic.AddInt64(&cwg.counter, a)
}

func (cwg *CWG) Counter() int64 {
	return atomic.LoadInt64(&cwg.counter)
}

func (cwg *CWG) Done(v interface{}) {
	cwg.channel <- v
}

func (cwg *CWG) Wait() (msgs []interface{}) {
	msgs = []interface{}{}
	stop := false
	for !stop {
		select {
		case msg := <- cwg.channel:
			msgs = append(msgs, msg)
			counter := cwg.Add(-1)
			if counter == 0 {
				stop = true
				break
			}
		}
	}
	return
}

func (cwg *CWG) WaitWithTimeout(timeout time.Duration) (ok bool, msgs []interface{}) {
	msgs = []interface{}{}
	for {
		select {
		case msg := <- cwg.channel:
			msgs = append(msgs, msg)
			counter := cwg.Add(-1)
			if counter == 0 {
				ok = true
				break
			}
		case <- time.After(timeout):
			break
		}
	}
	return
}<commit_msg>add WaitWithoutResults<commit_after>package cwg

import (
	"sync/atomic"
	"time"
)

type CWG struct {
	channel chan interface{}
	counter int64
}

func New() *CWG {
	cwg := new(CWG)
	cwg.channel = make(chan interface{})
	return cwg
}

func (cwg *CWG) Add(a int64) int64 {
	return atomic.AddInt64(&cwg.counter, a)
}

func (cwg *CWG) Counter() int64 {
	return atomic.LoadInt64(&cwg.counter)
}

func (cwg *CWG) Done(v interface{}) {
	cwg.channel <- v
}

func (cwg *CWG) WaitWithoutResults() {
	stop := false
	for !stop {
		select {
		case <- cwg.channel:
			counter := cwg.Add(-1)
			if counter == 0 {
				stop = true
				break
			}
		}
	}
}

func (cwg *CWG) Wait() (msgs []interface{}) {
	msgs = []interface{}{}
	stop := false
	for !stop {
		select {
		case msg := <- cwg.channel:
			msgs = append(msgs, msg)
			counter := cwg.Add(-1)
			if counter == 0 {
				stop = true
				break
			}
		}
	}
	return
}

func (cwg *CWG) WaitWithTimeout(timeout time.Duration) (ok bool, msgs []interface{}) {
	msgs = []interface{}{}
	for {
		select {
		case msg := <- cwg.channel:
			msgs = append(msgs, msg)
			counter := cwg.Add(-1)
			if counter == 0 {
				ok = true
				break
			}
		case <- time.After(timeout):
			break
		}
	}
	return
}<|endoftext|>


<commit_before>package cluster

import (
	"context"
	"fmt"
	"io/ioutil"
	"os"
	"reflect"
	"sync/atomic"
	"testing"
	"time"

	"github.com/coreos/etcd/clientv3"
	"github.com/coreos/etcd/pkg/transport"
)

var testTLS = transport.TLSInfo{
	TrustedCAFile:  "../test-certs/trusted-ca.pem",
	CertFile:       "../test-certs/test-cert.pem",
	KeyFile:        "../test-certs/test-cert-key.pem",
	ClientCertAuth: true,
}

type keyValue struct {
	key string
	val string
}

var basePort uint32 = 1300

/*
func TestCluster_Start_no_TLS(t *testing.T) {
	testCluster(t, Config{EmbeddedClient: true, Size: 3}, false, false)
}

func TestCluster_Start_no_TLS(t *testing.T) {
	testCluster(t, Config{EmbeddedClient: true, Size: 3}, false, false)
}

func TestCluster_Start_peer_manual_TLS(t *testing.T) {
	testCluster(t, Config{EmbeddedClient: true, Size: 3, PeerTLSInfo: testTLS}, false, false)
}

func TestCluster_Start_peer_auto_TLS(t *testing.T) {
	testCluster(t, Config{EmbeddedClient: true, Size: 3, PeerAutoTLS: true}, false, false)
}

func TestCluster_Start_client_manual_TLS_no_scheme(t *testing.T) {
	testCluster(t, Config{Size: 3, ClientTLSInfo: testTLS}, false, false)
}

func TestCluster_Start_client_manual_TLS_scheme(t *testing.T) {
	testCluster(t, Config{EmbeddedClient: true, Size: 3, ClientTLSInfo: testTLS}, true, false)
}

func TestCluster_Start_client_auto_TLS_no_scheme(t *testing.T) {
	testCluster(t, Config{EmbeddedClient: true, Size: 3, ClientAutoTLS: true}, false, false)
}

func TestCluster_Start_client_auto_TLS_scheme(t *testing.T) {
	testCluster(t, Config{EmbeddedClient: true, Size: 3, ClientAutoTLS: true}, true, false)
}

func TestCluster_Recover_no_TLS(t *testing.T) {
	testCluster(t, Config{EmbeddedClient: true, Size: 3}, false, true)
}

func TestCluster_Recover_peer_manual_TLS(t *testing.T) {
	testCluster(t, Config{EmbeddedClient: true, Size: 3, PeerTLSInfo: testTLS}, false, true)
}

func TestCluster_Recover_peer_auto_TLS(t *testing.T) {
	testCluster(t, Config{EmbeddedClient: true, Size: 3, PeerAutoTLS: true}, false, true)
}

func TestCluster_Recover_client_manual_TLS_no_scheme(t *testing.T) {
	testCluster(t, Config{EmbeddedClient: true, Size: 3, ClientTLSInfo: testTLS}, false, true)
}

func TestCluster_Recover_client_manual_TLS_scheme(t *testing.T) {
	testCluster(t, Config{EmbeddedClient: true, Size: 3, ClientTLSInfo: testTLS}, true, true)
}

func TestCluster_Recover_client_auto_TLS_no_scheme(t *testing.T) {
	testCluster(t, Config{EmbeddedClient: true, Size: 3, ClientAutoTLS: true}, false, true)
}

func TestCluster_Recover_client_auto_TLS_scheme(t *testing.T) {
	testCluster(t, Config{EmbeddedClient: true, Size: 3, ClientAutoTLS: true}, true, true)
}
*/

func TestCluster_Recover_peer_client_manual_TLS_scheme(t *testing.T) {
	testCluster(t, Config{EmbeddedClient: true, Size: 3, PeerTLSInfo: testTLS, ClientTLSInfo: testTLS}, true, true)
}

func testCluster(t *testing.T, cfg Config, scheme, stopRecover bool) {
	dir, err := ioutil.TempDir(os.TempDir(), "cluster-test")
	if err != nil {
		t.Fatal(err)
	}
	cfg.RootDir = dir
	cfg.RootPort = int(atomic.LoadUint32(&basePort))

	if cfg.RootCtx == nil || cfg.RootCancel == nil {
		rootCtx, rootCancel := context.WithCancel(context.Background())
		defer rootCancel()
		cfg.RootCtx = rootCtx
		cfg.RootCancel = rootCancel
	}

	atomic.AddUint32(&basePort, 10)

	println()
	println()
	println()
	fmt.Println("starting cluster")
	c, err := Start(cfg)
	if err != nil {
		t.Fatal(err)
	}
	defer func() {
		println()
		println()
		println()
		fmt.Println("shutting down the cluster")
		c.Shutdown()
	}()

	ks := []keyValue{
		{key: "foo1", val: "bar1"},
		{key: "foo2", val: "bar2"},
		{key: "foo3", val: "bar3"},
	}
	func() {
		println()
		println()
		println()
		fmt.Println("making write requests")
		cli, _, err := c.Client(c.AllEndpoints(scheme)...)
		if err != nil {
			t.Fatal(err)
		}
		defer cli.Close()

		for i, kv := range ks {
			ctx, cancel := context.WithTimeout(context.Background(), 3*time.Second)
			_, err = cli.Put(ctx, kv.key, kv.val)
			cancel()
			if err != nil {
				t.Fatalf("%d: PUT failed %v", i, err)
			}
			time.Sleep(5 * time.Second)
		}
	}()

	println()
	println()
	println()
	fmt.Println("calling UpdateMemberStatus")
	c.UpdateMemberStatus()
	hashes1 := make([]int, len(c.Members))
	for i := range c.Members {
		hashes1[i] = c.Members[i].status.Hash
	}

	if stopRecover {
		println()
		println()
		println()
		fmt.Println("stopping leader")
		leadidx := c.LeadIdx
		c.Stop(leadidx)
		time.Sleep(5 * time.Second)

		if err := c.WaitForLeader(); err != nil {
			t.Fatal(err)
		}

		println()
		println()
		println()
		fmt.Println("recovering old leader")
		if err = c.Restart(leadidx); err != nil {
			t.Fatal(err)
		}
		time.Sleep(5 * time.Second)

		if err := c.WaitForLeader(); err != nil {
			t.Fatal(err)
		}
	}

	println()
	println()
	println()
	fmt.Println("calling UpdateMemberStatus")
	c.UpdateMemberStatus()
	hashes2 := make([]int, len(c.Members))
	for i := range c.Members {
		hashes2[i] = c.Members[i].status.Hash
	}
	if !reflect.DeepEqual(hashes1, hashes2) {
		t.Fatalf("hashes1 %v != hashes2 %v", hashes1, hashes2)
	}

	func() {
		println()
		println()
		println()
		fmt.Println("making read requests")
		cli, _, err := c.Client(c.AllEndpoints(scheme)...)
		if err != nil {
			t.Fatal(err)
		}
		defer cli.Close()

		for i, kv := range ks {
			ctx, cancel := context.WithTimeout(context.Background(), 3*time.Second)
			var resp *clientv3.GetResponse
			resp, err = cli.Get(ctx, kv.key)
			cancel()
			if err != nil {
				t.Fatalf("%d: GET failed %v", i, err)
			}
			if string(resp.Kvs[0].Key) != kv.key {
				t.Fatalf("#%d: key expected %q, got %q", i, kv.key, string(resp.Kvs[0].Key))
			}
			if string(resp.Kvs[0].Value) != kv.val {
				t.Fatalf("#%d: value expected %q, got %q", i, kv.val, string(resp.Kvs[0].Value))
			}
			time.Sleep(5 * time.Second)
		}
	}()

	println()
	println()
	println()
	fmt.Println("calling UpdateMemberStatus")
	c.UpdateMemberStatus()
	hashes3 := make([]int, len(c.Members))
	for i := range c.Members {
		hashes3[i] = c.Members[i].status.Hash
	}
	if !reflect.DeepEqual(hashes1, hashes3) {
		t.Fatalf("hashes1 %v != hashes3 %v", hashes1, hashes3)
	}

	func() {
		println()
		println()
		println()
		fmt.Println("adding a new member")
		if err := c.Add(); err != nil {
			t.Fatal(err)
		}
		if err := c.WaitForLeader(); err != nil {
			t.Fatal(err)
		}
	}()

	println()
	println()
	println()
	for i, st := range c.AllMemberStatus() {
		fmt.Printf("Member Status: %q, %+v\n", c.Members[i].cfg.Name, st)
	}

	func() {
		time.Sleep(10 * time.Second)
		if err := c.WaitForLeader(); err != nil {
			t.Fatal(err)
		}
		println()
		println()
		println()
		fmt.Println("removing the member")
		leadidx := c.LeadIdx
		if err := c.Remove(leadidx); err != nil {
			t.Fatal(err)
		}
		if err := c.WaitForLeader(); err != nil {
			t.Fatal(err)
		}
	}()

	println()
	println()
	println()
	for i, st := range c.AllMemberStatus() {
		fmt.Printf("Member Status: %q, %+v\n", c.Members[i].cfg.Name, st)
	}
}
<commit_msg>cluster: use uint32 for Hash tests<commit_after>package cluster

import (
	"context"
	"fmt"
	"io/ioutil"
	"os"
	"reflect"
	"sync/atomic"
	"testing"
	"time"

	"github.com/coreos/etcd/clientv3"
	"github.com/coreos/etcd/pkg/transport"
)

var testTLS = transport.TLSInfo{
	TrustedCAFile:  "../test-certs/trusted-ca.pem",
	CertFile:       "../test-certs/test-cert.pem",
	KeyFile:        "../test-certs/test-cert-key.pem",
	ClientCertAuth: true,
}

type keyValue struct {
	key string
	val string
}

var basePort uint32 = 1300

/*
func TestCluster_Start_no_TLS(t *testing.T) {
	testCluster(t, Config{EmbeddedClient: true, Size: 3}, false, false)
}

func TestCluster_Start_no_TLS(t *testing.T) {
	testCluster(t, Config{EmbeddedClient: true, Size: 3}, false, false)
}

func TestCluster_Start_peer_manual_TLS(t *testing.T) {
	testCluster(t, Config{EmbeddedClient: true, Size: 3, PeerTLSInfo: testTLS}, false, false)
}

func TestCluster_Start_peer_auto_TLS(t *testing.T) {
	testCluster(t, Config{EmbeddedClient: true, Size: 3, PeerAutoTLS: true}, false, false)
}

func TestCluster_Start_client_manual_TLS_no_scheme(t *testing.T) {
	testCluster(t, Config{Size: 3, ClientTLSInfo: testTLS}, false, false)
}

func TestCluster_Start_client_manual_TLS_scheme(t *testing.T) {
	testCluster(t, Config{EmbeddedClient: true, Size: 3, ClientTLSInfo: testTLS}, true, false)
}

func TestCluster_Start_client_auto_TLS_no_scheme(t *testing.T) {
	testCluster(t, Config{EmbeddedClient: true, Size: 3, ClientAutoTLS: true}, false, false)
}

func TestCluster_Start_client_auto_TLS_scheme(t *testing.T) {
	testCluster(t, Config{EmbeddedClient: true, Size: 3, ClientAutoTLS: true}, true, false)
}

func TestCluster_Recover_no_TLS(t *testing.T) {
	testCluster(t, Config{EmbeddedClient: true, Size: 3}, false, true)
}

func TestCluster_Recover_peer_manual_TLS(t *testing.T) {
	testCluster(t, Config{EmbeddedClient: true, Size: 3, PeerTLSInfo: testTLS}, false, true)
}

func TestCluster_Recover_peer_auto_TLS(t *testing.T) {
	testCluster(t, Config{EmbeddedClient: true, Size: 3, PeerAutoTLS: true}, false, true)
}

func TestCluster_Recover_client_manual_TLS_no_scheme(t *testing.T) {
	testCluster(t, Config{EmbeddedClient: true, Size: 3, ClientTLSInfo: testTLS}, false, true)
}

func TestCluster_Recover_client_manual_TLS_scheme(t *testing.T) {
	testCluster(t, Config{EmbeddedClient: true, Size: 3, ClientTLSInfo: testTLS}, true, true)
}

func TestCluster_Recover_client_auto_TLS_no_scheme(t *testing.T) {
	testCluster(t, Config{EmbeddedClient: true, Size: 3, ClientAutoTLS: true}, false, true)
}

func TestCluster_Recover_client_auto_TLS_scheme(t *testing.T) {
	testCluster(t, Config{EmbeddedClient: true, Size: 3, ClientAutoTLS: true}, true, true)
}
*/

func TestCluster_Recover_peer_client_manual_TLS_scheme(t *testing.T) {
	testCluster(t, Config{EmbeddedClient: true, Size: 3, PeerTLSInfo: testTLS, ClientTLSInfo: testTLS}, true, true)
}

func testCluster(t *testing.T, cfg Config, scheme, stopRecover bool) {
	dir, err := ioutil.TempDir(os.TempDir(), "cluster-test")
	if err != nil {
		t.Fatal(err)
	}
	cfg.RootDir = dir
	cfg.RootPort = int(atomic.LoadUint32(&basePort))

	if cfg.RootCtx == nil || cfg.RootCancel == nil {
		rootCtx, rootCancel := context.WithCancel(context.Background())
		defer rootCancel()
		cfg.RootCtx = rootCtx
		cfg.RootCancel = rootCancel
	}

	atomic.AddUint32(&basePort, 10)

	println()
	println()
	println()
	fmt.Println("starting cluster")
	c, err := Start(cfg)
	if err != nil {
		t.Fatal(err)
	}
	defer func() {
		println()
		println()
		println()
		fmt.Println("shutting down the cluster")
		c.Shutdown()
	}()

	ks := []keyValue{
		{key: "foo1", val: "bar1"},
		{key: "foo2", val: "bar2"},
		{key: "foo3", val: "bar3"},
	}
	func() {
		println()
		println()
		println()
		fmt.Println("making write requests")
		cli, _, err := c.Client(c.AllEndpoints(scheme)...)
		if err != nil {
			t.Fatal(err)
		}
		defer cli.Close()

		for i, kv := range ks {
			ctx, cancel := context.WithTimeout(context.Background(), 3*time.Second)
			_, err = cli.Put(ctx, kv.key, kv.val)
			cancel()
			if err != nil {
				t.Fatalf("%d: PUT failed %v", i, err)
			}
			time.Sleep(5 * time.Second)
		}
	}()

	println()
	println()
	println()
	fmt.Println("calling UpdateMemberStatus")
	c.UpdateMemberStatus()
	hashes1 := make([]uint32, len(c.Members))
	for i := range c.Members {
		hashes1[i] = c.Members[i].status.Hash
	}

	if stopRecover {
		println()
		println()
		println()
		fmt.Println("stopping leader")
		leadidx := c.LeadIdx
		c.Stop(leadidx)
		time.Sleep(5 * time.Second)

		if err := c.WaitForLeader(); err != nil {
			t.Fatal(err)
		}

		println()
		println()
		println()
		fmt.Println("recovering old leader")
		if err = c.Restart(leadidx); err != nil {
			t.Fatal(err)
		}
		time.Sleep(5 * time.Second)

		if err := c.WaitForLeader(); err != nil {
			t.Fatal(err)
		}
	}

	println()
	println()
	println()
	fmt.Println("calling UpdateMemberStatus")
	c.UpdateMemberStatus()
	hashes2 := make([]uint32, len(c.Members))
	for i := range c.Members {
		hashes2[i] = c.Members[i].status.Hash
	}
	if !reflect.DeepEqual(hashes1, hashes2) {
		t.Fatalf("hashes1 %v != hashes2 %v", hashes1, hashes2)
	}

	func() {
		println()
		println()
		println()
		fmt.Println("making read requests")
		cli, _, err := c.Client(c.AllEndpoints(scheme)...)
		if err != nil {
			t.Fatal(err)
		}
		defer cli.Close()

		for i, kv := range ks {
			ctx, cancel := context.WithTimeout(context.Background(), 3*time.Second)
			var resp *clientv3.GetResponse
			resp, err = cli.Get(ctx, kv.key)
			cancel()
			if err != nil {
				t.Fatalf("%d: GET failed %v", i, err)
			}
			if string(resp.Kvs[0].Key) != kv.key {
				t.Fatalf("#%d: key expected %q, got %q", i, kv.key, string(resp.Kvs[0].Key))
			}
			if string(resp.Kvs[0].Value) != kv.val {
				t.Fatalf("#%d: value expected %q, got %q", i, kv.val, string(resp.Kvs[0].Value))
			}
			time.Sleep(5 * time.Second)
		}
	}()

	println()
	println()
	println()
	fmt.Println("calling UpdateMemberStatus")
	c.UpdateMemberStatus()
	hashes3 := make([]uint32, len(c.Members))
	for i := range c.Members {
		hashes3[i] = c.Members[i].status.Hash
	}
	if !reflect.DeepEqual(hashes1, hashes3) {
		t.Fatalf("hashes1 %v != hashes3 %v", hashes1, hashes3)
	}

	func() {
		println()
		println()
		println()
		fmt.Println("adding a new member")
		if err := c.Add(); err != nil {
			t.Fatal(err)
		}
		if err := c.WaitForLeader(); err != nil {
			t.Fatal(err)
		}
	}()

	println()
	println()
	println()
	for i, st := range c.AllMemberStatus() {
		fmt.Printf("Member Status: %q, %+v\n", c.Members[i].cfg.Name, st)
	}

	func() {
		time.Sleep(10 * time.Second)
		if err := c.WaitForLeader(); err != nil {
			t.Fatal(err)
		}
		println()
		println()
		println()
		fmt.Println("removing the member")
		leadidx := c.LeadIdx
		if err := c.Remove(leadidx); err != nil {
			t.Fatal(err)
		}
		if err := c.WaitForLeader(); err != nil {
			t.Fatal(err)
		}
	}()

	println()
	println()
	println()
	for i, st := range c.AllMemberStatus() {
		fmt.Printf("Member Status: %q, %+v\n", c.Members[i].cfg.Name, st)
	}
}
<|endoftext|>


<commit_before>package main

import (
	"bytes"
	"flag"
	"fmt"
	"go/build"
	"go/format"
	"io/ioutil"
	"os"
	"strings"
	"text/template"

	"golang.org/x/tools/go/loader"
	"golang.org/x/tools/go/types"
	"gopkg.in/errgo.v1"
)

// TODO:
// - generate exported types if the parameter/response types aren't exported?
// - deal with literal interface and struct types.
// - copy doc comments from server methods.

var currentDir string

func main() {
	flag.Usage = func() {
		fmt.Fprintf(os.Stderr, "usage: httprequest-generate server-package server-type client-type\n")
		os.Exit(2)
	}
	flag.Parse()
	if flag.NArg() != 3 {
		flag.Usage()
	}

	serverPkg, serverType, clientType := flag.Arg(0), flag.Arg(1), flag.Arg(2)

	if err := generate(serverPkg, serverType, clientType); err != nil {
		fmt.Fprintf(os.Stderr, "%v\n", err)
		os.Exit(1)
	}
}

type templateArg struct {
	CommandLine string
	PkgName     string
	Imports     []string
	Methods     []method
	ClientType  string
}

var code = template.Must(template.New("").Parse(`
// The code in this file was automatically generated by running
// 	httprequest-generate-client {{.CommandLine}}
// DO NOT EDIT

package {{.PkgName}}
import (
	{{range .Imports}}{{printf "%q" .}}
	{{end}}
)

type {{.ClientType}} struct {
	Client httprequest.Client
}

{{range .Methods}}
{{if .RespType}}
	func (c *{{$.ClientType}}) {{.Name}}(p *{{.ParamType}}) ({{.RespType}}, error) {
		var r {{.RespType}}
		err := c.Client.Call(p, &r)
		return r, err
	}
{{else}}
	func (c *{{$.ClientType}}) {{.Name}}(p *{{.ParamType}}) (error) {
		return c.Client.Call(p, nil)
	}
{{end}}
{{end}}
`))

func generate(serverPkg, serverType, clientType string) error {
	cwd, err := os.Getwd()
	if err != nil {
		return err
	}
	currentDir = cwd

	methods, imports, err := serverMethods(serverPkg, serverType)
	if err != nil {
		return errgo.Mask(err)
	}
	localPkg, err := build.Import(".", currentDir, 0)
	if err != nil {
		return errgo.Notef(err, "cannot open package in current directory")
	}
	arg := templateArg{
		CommandLine: strings.Join(flag.Args(), " "),
		Imports:     imports,
		Methods:     methods,
		PkgName:     localPkg.Name,
		ClientType:  clientType,
	}
	var buf bytes.Buffer
	if err := code.Execute(&buf, arg); err != nil {
		return errgo.Mask(err)
	}
	data, err := format.Source(buf.Bytes())
	if err != nil {
		return errgo.Notef(err, "cannot format source")
	}
	if err := writeOutput(data, clientType); err != nil {
		return errgo.Mask(err)
	}
	return nil
}

func writeOutput(data []byte, clientType string) error {
	filename := strings.ToLower(clientType) + "_generated.go"
	if err := ioutil.WriteFile(filename, data, 0644); err != nil {
		return errgo.Mask(err)
	}
	return nil
}

type method struct {
	Name      string
	ParamType string
	RespType  string
}

func serverMethods(serverPkg, serverType string) ([]method, []string, error) {
	cfg := loader.Config{
		TypeCheckFuncBodies: func(string) bool {
			return false
		},
		ImportPkgs: map[string]bool{
			serverPkg: false, // false means don't load tests.
		},
	}
	prog, err := cfg.Load()
	if err != nil {
		return nil, nil, errgo.Notef(err, "cannot load %q", serverPkg)
	}
	pkgInfo := prog.Imported[serverPkg]
	if pkgInfo == nil {
		return nil, nil, errgo.Newf("cannot find %q in imported code", serverPkg)
	}
	pkg := pkgInfo.Pkg
	obj := pkg.Scope().Lookup(serverType)
	if obj == nil {
		return nil, nil, errgo.Newf("type %s not found in %s", serverType, serverPkg)
	}
	objTypeName, ok := obj.(*types.TypeName)
	if !ok {
		return nil, nil, errgo.Newf("%s is not a type", serverType)
	}
	// Use the pointer type to get as many methods as possible.
	ptrObjType := types.NewPointer(objTypeName.Type())

	imports := map[string]string{
		"github.com/juju/httprequest": "httprequest",
	}
	var methods []method
	mset := types.NewMethodSet(ptrObjType)
	for i := 0; i < mset.Len(); i++ {
		sel := mset.At(i)
		if !sel.Obj().Exported() {
			continue
		}
		name := sel.Obj().Name()
		if name == "Close" {
			continue
		}
		ptype, rtype, err := parseMethodType(sel.Type().(*types.Signature))
		if err != nil {
			fmt.Fprintf(os.Stderr, "ignoring method %s: %v\n", name, err)
			continue
		}
		methods = append(methods, method{
			Name:      name,
			ParamType: typeStr(ptype, imports),
			RespType:  typeStr(rtype, imports),
		})
	}
	var allImports []string
	for path := range imports {
		allImports = append(allImports, path)
	}
	return methods, allImports, nil
}

// typeStr returns the type string to be used when using the
// given type. It adds any needed import paths to the given
// imports map (map from package path to package id).
func typeStr(t types.Type, imports map[string]string) string {
	if t == nil {
		return ""
	}
	qualify := func(pkg *types.Package) string {
		if name := imports[pkg.Path()]; name != "" {
			return name
		}
		name := pkg.Name()
		// Make sure we're not duplicating the name.
		// TODO if we are, make a new non-duplicated version.
		for oldPkg, oldName := range imports {
			if oldName == name {
				panic(errgo.Newf("duplicate package name %s vs %s", pkg.Path(), oldPkg))
			}
		}
		imports[pkg.Path()] = name
		return name
	}
	return types.TypeString(t, qualify)
}

func parseMethodType(t *types.Signature) (ptype, rtype types.Type, err error) {
	mp := t.Params()
	if mp.Len() != 1 && mp.Len() != 2 {
		return nil, nil, errgo.New("wrong argument count")
	}
	ptype0 := mp.At(mp.Len() - 1).Type()
	ptype1, ok := ptype0.(*types.Pointer)
	if !ok {
		return nil, nil, errgo.New("parameter is not a pointer")
	}
	ptype = ptype1.Elem()
	if _, ok := ptype.Underlying().(*types.Struct); !ok {
		return nil, nil, errgo.Newf("parameter is %s, not a pointer to struct", ptype1.Elem())
	}
	rp := t.Results()
	if rp.Len() > 2 {
		return nil, nil, errgo.New("wrong result count")
	}
	if rp.Len() == 2 {
		rtype = rp.At(0).Type()
	}
	return ptype, rtype, nil
}
<commit_msg>cmd/httprequest-generate-client: generate method comments too<commit_after>package main

import (
	"bytes"
	"flag"
	"fmt"
	"go/ast"
	"go/build"
	"go/format"
	"go/parser"
	"io/ioutil"
	"os"
	"strings"
	"text/template"

	"golang.org/x/tools/go/loader"
	"golang.org/x/tools/go/types"
	"gopkg.in/errgo.v1"
)

// TODO:
// - generate exported types if the parameter/response types aren't exported?
// - deal with literal interface and struct types.
// - copy doc comments from server methods.

var currentDir string

func main() {
	flag.Usage = func() {
		fmt.Fprintf(os.Stderr, "usage: httprequest-generate server-package server-type client-type\n")
		os.Exit(2)
	}
	flag.Parse()
	if flag.NArg() != 3 {
		flag.Usage()
	}

	serverPkg, serverType, clientType := flag.Arg(0), flag.Arg(1), flag.Arg(2)

	if err := generate(serverPkg, serverType, clientType); err != nil {
		fmt.Fprintf(os.Stderr, "%v\n", err)
		os.Exit(1)
	}
}

type templateArg struct {
	CommandLine string
	PkgName     string
	Imports     []string
	Methods     []method
	ClientType  string
}

var code = template.Must(template.New("").Parse(`
// The code in this file was automatically generated by running
// 	httprequest-generate-client {{.CommandLine}}
// DO NOT EDIT

package {{.PkgName}}
import (
	{{range .Imports}}{{printf "%q" .}}
	{{end}}
)

type {{.ClientType}} struct {
	Client httprequest.Client
}

{{range .Methods}}
{{if .RespType}}
	{{.Doc}}
	func (c *{{$.ClientType}}) {{.Name}}(p *{{.ParamType}}) ({{.RespType}}, error) {
		var r {{.RespType}}
		err := c.Client.Call(p, &r)
		return r, err
	}
{{else}}
	{{.Doc}}
	func (c *{{$.ClientType}}) {{.Name}}(p *{{.ParamType}}) (error) {
		return c.Client.Call(p, nil)
	}
{{end}}
{{end}}
`))

func generate(serverPkg, serverType, clientType string) error {
	cwd, err := os.Getwd()
	if err != nil {
		return err
	}
	currentDir = cwd

	methods, imports, err := serverMethods(serverPkg, serverType)
	if err != nil {
		return errgo.Mask(err)
	}
	localPkg, err := build.Import(".", currentDir, 0)
	if err != nil {
		return errgo.Notef(err, "cannot open package in current directory")
	}
	arg := templateArg{
		CommandLine: strings.Join(flag.Args(), " "),
		Imports:     imports,
		Methods:     methods,
		PkgName:     localPkg.Name,
		ClientType:  clientType,
	}
	var buf bytes.Buffer
	if err := code.Execute(&buf, arg); err != nil {
		return errgo.Mask(err)
	}
	data, err := format.Source(buf.Bytes())
	if err != nil {
		return errgo.Notef(err, "cannot format source")
	}
	if err := writeOutput(data, clientType); err != nil {
		return errgo.Mask(err)
	}
	return nil
}

func writeOutput(data []byte, clientType string) error {
	filename := strings.ToLower(clientType) + "_generated.go"
	if err := ioutil.WriteFile(filename, data, 0644); err != nil {
		return errgo.Mask(err)
	}
	return nil
}

type method struct {
	Name      string
	Doc       string
	ParamType string
	RespType  string
}

func serverMethods(serverPkg, serverType string) ([]method, []string, error) {
	cfg := loader.Config{
		TypeCheckFuncBodies: func(string) bool {
			return false
		},
		ImportPkgs: map[string]bool{
			serverPkg: false, // false means don't load tests.
		},
		ParserMode: parser.ParseComments,
	}
	prog, err := cfg.Load()
	if err != nil {
		return nil, nil, errgo.Notef(err, "cannot load %q", serverPkg)
	}
	pkgInfo := prog.Imported[serverPkg]
	if pkgInfo == nil {
		return nil, nil, errgo.Newf("cannot find %q in imported code", serverPkg)
	}
	pkg := pkgInfo.Pkg
	obj := pkg.Scope().Lookup(serverType)
	if obj == nil {
		return nil, nil, errgo.Newf("type %s not found in %s", serverType, serverPkg)
	}
	objTypeName, ok := obj.(*types.TypeName)
	if !ok {
		return nil, nil, errgo.Newf("%s is not a type", serverType)
	}
	// Use the pointer type to get as many methods as possible.
	ptrObjType := types.NewPointer(objTypeName.Type())

	imports := map[string]string{
		"github.com/juju/httprequest": "httprequest",
	}
	var methods []method
	mset := types.NewMethodSet(ptrObjType)
	for i := 0; i < mset.Len(); i++ {
		sel := mset.At(i)
		if !sel.Obj().Exported() {
			continue
		}
		name := sel.Obj().Name()
		if name == "Close" {
			continue
		}
		ptype, rtype, err := parseMethodType(sel.Type().(*types.Signature))
		if err != nil {
			fmt.Fprintf(os.Stderr, "ignoring method %s: %v\n", name, err)
			continue
		}
		comment := docComment(prog, sel)
		methods = append(methods, method{
			Name:      name,
			Doc:       comment,
			ParamType: typeStr(ptype, imports),
			RespType:  typeStr(rtype, imports),
		})
	}
	var allImports []string
	for path := range imports {
		allImports = append(allImports, path)
	}
	return methods, allImports, nil
}

// docComment returns the doc comment for the method referred to
// by the given selection.
func docComment(prog *loader.Program, sel *types.Selection) string {
	obj := sel.Obj()
	tokFile := prog.Fset.File(obj.Pos())
	if tokFile == nil {
		panic("no file found for method")
	}
	filename := tokFile.Name()
	for _, pkgInfo := range prog.AllPackages {
		for _, f := range pkgInfo.Files {
			if tokFile := prog.Fset.File(f.Pos()); tokFile == nil || tokFile.Name() != filename {
				continue
			}
			// We've found the file we're looking for. Now traverse all
			// top level declarations looking for the right function declaration.
			for _, decl := range f.Decls {
				fdecl, ok := decl.(*ast.FuncDecl)
				if ok && fdecl.Name.Pos() == obj.Pos() {
					// Found it!
					return commentStr(fdecl.Doc)
				}
			}
		}
	}
	panic("method declaration not found")
}

func commentStr(c *ast.CommentGroup) string {
	if c == nil {
		return ""
	}
	var b []byte
	for i, cc := range c.List {
		if i > 0 {
			b = append(b, '\n')
		}
		b = append(b, cc.Text...)
	}
	return string(b)
}

// typeStr returns the type string to be used when using the
// given type. It adds any needed import paths to the given
// imports map (map from package path to package id).
func typeStr(t types.Type, imports map[string]string) string {
	if t == nil {
		return ""
	}
	qualify := func(pkg *types.Package) string {
		if name := imports[pkg.Path()]; name != "" {
			return name
		}
		name := pkg.Name()
		// Make sure we're not duplicating the name.
		// TODO if we are, make a new non-duplicated version.
		for oldPkg, oldName := range imports {
			if oldName == name {
				panic(errgo.Newf("duplicate package name %s vs %s", pkg.Path(), oldPkg))
			}
		}
		imports[pkg.Path()] = name
		return name
	}
	return types.TypeString(t, qualify)
}

func parseMethodType(t *types.Signature) (ptype, rtype types.Type, err error) {
	mp := t.Params()
	if mp.Len() != 1 && mp.Len() != 2 {
		return nil, nil, errgo.New("wrong argument count")
	}
	ptype0 := mp.At(mp.Len() - 1).Type()
	ptype1, ok := ptype0.(*types.Pointer)
	if !ok {
		return nil, nil, errgo.New("parameter is not a pointer")
	}
	ptype = ptype1.Elem()
	if _, ok := ptype.Underlying().(*types.Struct); !ok {
		return nil, nil, errgo.Newf("parameter is %s, not a pointer to struct", ptype1.Elem())
	}
	rp := t.Results()
	if rp.Len() > 2 {
		return nil, nil, errgo.New("wrong result count")
	}
	if rp.Len() == 2 {
		rtype = rp.At(0).Type()
	}
	return ptype, rtype, nil
}
<|endoftext|>


<commit_before>/*
Copyright 2017 The Kubernetes Authors.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package phases

import (
	"errors"
	"fmt"

	"github.com/spf13/cobra"
	"k8s.io/apimachinery/pkg/util/sets"
	kubeadmapi "k8s.io/kubernetes/cmd/kubeadm/app/apis/kubeadm"
	kubeadmscheme "k8s.io/kubernetes/cmd/kubeadm/app/apis/kubeadm/scheme"
	kubeadmapiv1beta1 "k8s.io/kubernetes/cmd/kubeadm/app/apis/kubeadm/v1beta1"
	"k8s.io/kubernetes/cmd/kubeadm/app/apis/kubeadm/validation"
	"k8s.io/kubernetes/cmd/kubeadm/app/cmd/options"
	"k8s.io/kubernetes/cmd/kubeadm/app/cmd/phases/workflow"
	cmdutil "k8s.io/kubernetes/cmd/kubeadm/app/cmd/util"
	"k8s.io/kubernetes/cmd/kubeadm/app/preflight"
	kubeadmutil "k8s.io/kubernetes/cmd/kubeadm/app/util"
	configutil "k8s.io/kubernetes/cmd/kubeadm/app/util/config"
	"k8s.io/kubernetes/pkg/util/normalizer"
	utilsexec "k8s.io/utils/exec"
)

var (
	masterPreflightExample = normalizer.Examples(`
		# Run master pre-flight checks using a config file.
		kubeadm init phase preflight --config kubeadm-config.yml
		`)

	nodePreflightLongDesc = normalizer.LongDesc(`
		Run node pre-flight checks, functionally equivalent to what implemented by kubeadm join.
		` + cmdutil.AlphaDisclaimer)

	nodePreflightExample = normalizer.Examples(`
		# Run node pre-flight checks.
		kubeadm alpha phase preflight node
	`)

	errorMissingConfigFlag = errors.New("the --config flag is mandatory")
)

// preflightMasterData defines the behavior that a runtime data struct passed to the PreflightMaster master phase
// should have. Please note that we are using an interface in order to make this phase reusable in different workflows
// (and thus with different runtime data struct, all of them requested to be compliant to this interface)
type preflightMasterData interface {
	Cfg() *kubeadmapi.InitConfiguration
	DryRun() bool
	IgnorePreflightErrors() sets.String
}

// NewPreflightMasterPhase creates a kubeadm workflow phase that implements preflight checks for a new master node.
func NewPreflightMasterPhase() workflow.Phase {
	return workflow.Phase{
		Name:    "preflight",
		Short:   "Run master pre-flight checks",
		Long:    "Run master pre-flight checks, functionally equivalent to what implemented by kubeadm init.",
		Example: masterPreflightExample,
		Run:     runPreflightMaster,
	}
}

// runPreflightMaster executes preflight checks logic.
func runPreflightMaster(c workflow.RunData) error {
	data, ok := c.(preflightMasterData)
	if !ok {
		return fmt.Errorf("preflight phase invoked with an invalid data struct")
	}

	fmt.Println("[preflight] running pre-flight checks")
	if err := preflight.RunInitMasterChecks(utilsexec.New(), data.Cfg(), data.IgnorePreflightErrors()); err != nil {
		return nil
	}

	if !data.DryRun() {
		fmt.Println("[preflight] Pulling images required for setting up a Kubernetes cluster")
		fmt.Println("[preflight] This might take a minute or two, depending on the speed of your internet connection")
		fmt.Println("[preflight] You can also perform this action in beforehand using 'kubeadm config images pull'")
		if err := preflight.RunPullImagesCheck(utilsexec.New(), data.Cfg(), data.IgnorePreflightErrors()); err != nil {
			return err
		}
	} else {
		fmt.Println("[preflight] Would pull the required images (like 'kubeadm config images pull')")
	}

	return nil
}

// NewCmdPreFlight calls cobra.Command for preflight checks
func NewCmdPreFlight() *cobra.Command {
	var cfgPath string
	var ignorePreflightErrors []string

	cmd := &cobra.Command{
		Use:   "preflight",
		Short: "Run pre-flight checks",
		Long:  cmdutil.MacroCommandLongDescription,
	}

	options.AddConfigFlag(cmd.PersistentFlags(), &cfgPath)
	options.AddIgnorePreflightErrorsFlag(cmd.PersistentFlags(), &ignorePreflightErrors)

	cmd.AddCommand(NewCmdPreFlightNode(&cfgPath, &ignorePreflightErrors))

	return cmd
}

// NewCmdPreFlightNode calls cobra.Command for node preflight checks
func NewCmdPreFlightNode(cfgPath *string, ignorePreflightErrors *[]string) *cobra.Command {
	cmd := &cobra.Command{
		Use:     "node",
		Short:   "Run node pre-flight checks",
		Long:    nodePreflightLongDesc,
		Example: nodePreflightExample,
		Run: func(cmd *cobra.Command, args []string) {
			if len(*cfgPath) == 0 {
				kubeadmutil.CheckErr(errorMissingConfigFlag)
			}
			ignorePreflightErrorsSet, err := validation.ValidateIgnorePreflightErrors(*ignorePreflightErrors)
			kubeadmutil.CheckErr(err)

			cfg := &kubeadmapiv1beta1.JoinConfiguration{}
			kubeadmscheme.Scheme.Default(cfg)

			internalcfg, err := configutil.JoinConfigFileAndDefaultsToInternalConfig(*cfgPath, cfg)
			kubeadmutil.CheckErr(err)
			err = configutil.VerifyAPIServerBindAddress(internalcfg.APIEndpoint.AdvertiseAddress)
			kubeadmutil.CheckErr(err)

			fmt.Println("[preflight] running pre-flight checks")

			err = preflight.RunJoinNodeChecks(utilsexec.New(), internalcfg, ignorePreflightErrorsSet)
			kubeadmutil.CheckErr(err)

			fmt.Println("[preflight] pre-flight checks passed")
		},
	}

	return cmd
}
<commit_msg>fix incorrect return value of kubeadm pre-flight checks<commit_after>/*
Copyright 2017 The Kubernetes Authors.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package phases

import (
	"errors"
	"fmt"

	"github.com/spf13/cobra"
	"k8s.io/apimachinery/pkg/util/sets"
	kubeadmapi "k8s.io/kubernetes/cmd/kubeadm/app/apis/kubeadm"
	kubeadmscheme "k8s.io/kubernetes/cmd/kubeadm/app/apis/kubeadm/scheme"
	kubeadmapiv1beta1 "k8s.io/kubernetes/cmd/kubeadm/app/apis/kubeadm/v1beta1"
	"k8s.io/kubernetes/cmd/kubeadm/app/apis/kubeadm/validation"
	"k8s.io/kubernetes/cmd/kubeadm/app/cmd/options"
	"k8s.io/kubernetes/cmd/kubeadm/app/cmd/phases/workflow"
	cmdutil "k8s.io/kubernetes/cmd/kubeadm/app/cmd/util"
	"k8s.io/kubernetes/cmd/kubeadm/app/preflight"
	kubeadmutil "k8s.io/kubernetes/cmd/kubeadm/app/util"
	configutil "k8s.io/kubernetes/cmd/kubeadm/app/util/config"
	"k8s.io/kubernetes/pkg/util/normalizer"
	utilsexec "k8s.io/utils/exec"
)

var (
	masterPreflightExample = normalizer.Examples(`
		# Run master pre-flight checks using a config file.
		kubeadm init phase preflight --config kubeadm-config.yml
		`)

	nodePreflightLongDesc = normalizer.LongDesc(`
		Run node pre-flight checks, functionally equivalent to what implemented by kubeadm join.
		` + cmdutil.AlphaDisclaimer)

	nodePreflightExample = normalizer.Examples(`
		# Run node pre-flight checks.
		kubeadm alpha phase preflight node
	`)

	errorMissingConfigFlag = errors.New("the --config flag is mandatory")
)

// preflightMasterData defines the behavior that a runtime data struct passed to the PreflightMaster master phase
// should have. Please note that we are using an interface in order to make this phase reusable in different workflows
// (and thus with different runtime data struct, all of them requested to be compliant to this interface)
type preflightMasterData interface {
	Cfg() *kubeadmapi.InitConfiguration
	DryRun() bool
	IgnorePreflightErrors() sets.String
}

// NewPreflightMasterPhase creates a kubeadm workflow phase that implements preflight checks for a new master node.
func NewPreflightMasterPhase() workflow.Phase {
	return workflow.Phase{
		Name:    "preflight",
		Short:   "Run master pre-flight checks",
		Long:    "Run master pre-flight checks, functionally equivalent to what implemented by kubeadm init.",
		Example: masterPreflightExample,
		Run:     runPreflightMaster,
	}
}

// runPreflightMaster executes preflight checks logic.
func runPreflightMaster(c workflow.RunData) error {
	data, ok := c.(preflightMasterData)
	if !ok {
		return fmt.Errorf("preflight phase invoked with an invalid data struct")
	}

	fmt.Println("[preflight] running pre-flight checks")
	if err := preflight.RunInitMasterChecks(utilsexec.New(), data.Cfg(), data.IgnorePreflightErrors()); err != nil {
		return err
	}

	if !data.DryRun() {
		fmt.Println("[preflight] Pulling images required for setting up a Kubernetes cluster")
		fmt.Println("[preflight] This might take a minute or two, depending on the speed of your internet connection")
		fmt.Println("[preflight] You can also perform this action in beforehand using 'kubeadm config images pull'")
		if err := preflight.RunPullImagesCheck(utilsexec.New(), data.Cfg(), data.IgnorePreflightErrors()); err != nil {
			return err
		}
	} else {
		fmt.Println("[preflight] Would pull the required images (like 'kubeadm config images pull')")
	}

	return nil
}

// NewCmdPreFlight calls cobra.Command for preflight checks
func NewCmdPreFlight() *cobra.Command {
	var cfgPath string
	var ignorePreflightErrors []string

	cmd := &cobra.Command{
		Use:   "preflight",
		Short: "Run pre-flight checks",
		Long:  cmdutil.MacroCommandLongDescription,
	}

	options.AddConfigFlag(cmd.PersistentFlags(), &cfgPath)
	options.AddIgnorePreflightErrorsFlag(cmd.PersistentFlags(), &ignorePreflightErrors)

	cmd.AddCommand(NewCmdPreFlightNode(&cfgPath, &ignorePreflightErrors))

	return cmd
}

// NewCmdPreFlightNode calls cobra.Command for node preflight checks
func NewCmdPreFlightNode(cfgPath *string, ignorePreflightErrors *[]string) *cobra.Command {
	cmd := &cobra.Command{
		Use:     "node",
		Short:   "Run node pre-flight checks",
		Long:    nodePreflightLongDesc,
		Example: nodePreflightExample,
		Run: func(cmd *cobra.Command, args []string) {
			if len(*cfgPath) == 0 {
				kubeadmutil.CheckErr(errorMissingConfigFlag)
			}
			ignorePreflightErrorsSet, err := validation.ValidateIgnorePreflightErrors(*ignorePreflightErrors)
			kubeadmutil.CheckErr(err)

			cfg := &kubeadmapiv1beta1.JoinConfiguration{}
			kubeadmscheme.Scheme.Default(cfg)

			internalcfg, err := configutil.JoinConfigFileAndDefaultsToInternalConfig(*cfgPath, cfg)
			kubeadmutil.CheckErr(err)
			err = configutil.VerifyAPIServerBindAddress(internalcfg.APIEndpoint.AdvertiseAddress)
			kubeadmutil.CheckErr(err)

			fmt.Println("[preflight] running pre-flight checks")

			err = preflight.RunJoinNodeChecks(utilsexec.New(), internalcfg, ignorePreflightErrorsSet)
			kubeadmutil.CheckErr(err)

			fmt.Println("[preflight] pre-flight checks passed")
		},
	}

	return cmd
}
<|endoftext|>


<commit_before>package replica

type dependencies []InstanceIdType

// union unions the dep into the receiver
func (d dependencies) union(toUnion dependencies) bool {
	same := true
	for i := range d {
		if len(toUnion) < i {
			break
		}

		if d[i] != toUnion[i] {
			same = false
			if d[i] < toUnion[i] {
				d[i] = toUnion[i]
			}
		}
	}
	return same
}
<commit_msg>refactor deps.go<commit_after>package replica

type dependencies []InstanceIdType

// union unions the deps into the receiver
func (d dependencies) union(other dependencies) bool {
	if len(d) != len(other) {
		panic("union: size different!")
	}

	same := true
	for i := range d {
		if d[i] != other[i] {
			same = false
			if d[i] < other[i] {
				d[i] = other[i]
			}
		}
	}
	return same
}
<|endoftext|>


<commit_before>package fetcher

import (
	"bytes"
	"compress/bzip2"
	"fmt"
	"io"
	"net/http"
	"net/url"
	"sync"
	"time"

	"github.com/htcat/htcat"
	"github.com/inconshreveable/log15"
	"github.com/spf13/viper"
	"github.com/vulsio/goval-dictionary/util"
	"golang.org/x/xerrors"
)

type fetchRequest struct {
	target       string
	url          string
	bzip2        bool
	concurrently bool
}

//FetchResult has url and OVAL definitions
type FetchResult struct {
	Target string
	URL    string
	Body   []byte
}

func fetchFeedFiles(reqs []fetchRequest) (results []FetchResult, err error) {
	reqChan := make(chan fetchRequest, len(reqs))
	resChan := make(chan FetchResult, len(reqs))
	errChan := make(chan error, len(reqs))
	defer close(reqChan)
	defer close(resChan)
	defer close(errChan)

	for _, r := range reqs {
		log15.Info("Fetching... ", "URL", r.url)
	}

	go func() {
		for _, r := range reqs {
			reqChan <- r
		}
	}()

	concurrency := len(reqs)
	tasks := util.GenWorkers(concurrency)
	wg := new(sync.WaitGroup)
	for range reqs {
		wg.Add(1)
		tasks <- func() {
			select {
			case req := <-reqChan:
				var body []byte
				if req.concurrently {
					body, err = fetchFileConcurrently(req, 20/len(reqs))
				} else {
					body, err = fetchFileWithUA(req)
				}
				wg.Done()
				if err != nil {
					errChan <- err
					return
				}
				resChan <- FetchResult{
					Target: req.target,
					URL:    req.url,
					Body:   body,
				}
			}
			return
		}
	}
	wg.Wait()

	errs := []error{}
	timeout := time.After(10 * 60 * time.Second)
	for range reqs {
		select {
		case res := <-resChan:
			results = append(results, res)
			log15.Info("Fetched... ", "URL", res.URL)
		case err := <-errChan:
			errs = append(errs, err)
		case <-timeout:
			return results, fmt.Errorf("Timeout Fetching")
		}
	}
	log15.Info("Finished fetching OVAL definitions")
	if 0 < len(errs) {
		return results, fmt.Errorf("%s", errs)
	}
	return results, nil
}

func fetchFileConcurrently(req fetchRequest, concurrency int) (body []byte, err error) {
	var proxyURL *url.URL
	httpClient := &http.Client{}
	httpProxy := viper.GetString("http-proxy")
	if httpProxy != "" {
		if proxyURL, err = url.Parse(httpProxy); err != nil {
			return nil, xerrors.Errorf("Failed to parse proxy url. err: %w", err)
		}
		httpClient = &http.Client{Transport: &http.Transport{Proxy: http.ProxyURL(proxyURL)}}
	}

	u, err := url.Parse(req.url)
	if err != nil {
		return nil, xerrors.Errorf("aborting: could not parse given URL: %v", err)
	}

	buf := bytes.Buffer{}
	htc := htcat.New(httpClient, u, concurrency)
	if _, err := htc.WriteTo(&buf); err != nil {
		return nil, xerrors.Errorf("aborting: could not write to output stream: %v",
			err)
	}

	var bytesBody []byte
	if req.bzip2 {
		var b bytes.Buffer
		if _, err := b.ReadFrom(bzip2.NewReader(bytes.NewReader(buf.Bytes()))); err != nil {
			return body, err
		}
		bytesBody = b.Bytes()
	} else {
		bytesBody = buf.Bytes()
	}

	return bytesBody, nil
}

func fetchFileWithUA(req fetchRequest) (body []byte, err error) {
	var errs []error
	var proxyURL *url.URL
	var resp *http.Response

	httpClient := &http.Client{}
	httpProxy := viper.GetString("http-proxy")
	if httpProxy != "" {
		if proxyURL, err = url.Parse(httpProxy); err != nil {
			return nil, xerrors.Errorf("Failed to parse proxy url. err: %w", err)
		}
		httpClient = &http.Client{Transport: &http.Transport{Proxy: http.ProxyURL(proxyURL)}}
	}

	httpreq, err := http.NewRequest("GET", req.url, nil)
	if err != nil {
		return nil, xerrors.Errorf("Failed to download. err: %w", err)
	}

	httpreq.Header.Set("User-Agent", "curl/7.37.0")
	resp, err = httpClient.Do(httpreq)
	if err != nil {
		return nil, xerrors.Errorf("Failed to download. err: %w", err)
	}
	defer resp.Body.Close()

	buf := bytes.NewBuffer(nil)
	if _, err := io.Copy(buf, resp.Body); err != nil {
		return nil, err
	}
	if len(errs) > 0 || resp == nil || resp.StatusCode != 200 {
		return nil, fmt.Errorf(
			"HTTP error. errs: %v, url: %s", errs, req.url)
	}

	var bytesBody []byte
	if req.bzip2 {
		bz := bzip2.NewReader(buf)
		var b bytes.Buffer
		if _, err := b.ReadFrom(bz); err != nil {
			return nil, err
		}
		bytesBody = b.Bytes()
	} else {
		bytesBody = buf.Bytes()
	}

	return bytesBody, nil
}
<commit_msg>chore: fix error handling (#163)<commit_after>package fetcher

import (
	"bytes"
	"compress/bzip2"
	"fmt"
	"io"
	"net/http"
	"net/url"
	"sync"
	"time"

	"github.com/htcat/htcat"
	"github.com/inconshreveable/log15"
	"github.com/spf13/viper"
	"github.com/vulsio/goval-dictionary/util"
	"golang.org/x/xerrors"
)

type fetchRequest struct {
	target       string
	url          string
	bzip2        bool
	concurrently bool
}

//FetchResult has url and OVAL definitions
type FetchResult struct {
	Target string
	URL    string
	Body   []byte
}

func fetchFeedFiles(reqs []fetchRequest) (results []FetchResult, err error) {
	reqChan := make(chan fetchRequest, len(reqs))
	resChan := make(chan FetchResult, len(reqs))
	errChan := make(chan error, len(reqs))
	defer close(reqChan)
	defer close(resChan)
	defer close(errChan)

	for _, r := range reqs {
		log15.Info("Fetching... ", "URL", r.url)
	}

	go func() {
		for _, r := range reqs {
			reqChan <- r
		}
	}()

	concurrency := len(reqs)
	tasks := util.GenWorkers(concurrency)
	wg := new(sync.WaitGroup)
	for range reqs {
		wg.Add(1)
		tasks <- func() {
			select {
			case req := <-reqChan:
				var body []byte
				if req.concurrently {
					body, err = fetchFileConcurrently(req, 20/len(reqs))
				} else {
					body, err = fetchFileWithUA(req)
				}
				wg.Done()
				if err != nil {
					errChan <- err
					return
				}
				resChan <- FetchResult{
					Target: req.target,
					URL:    req.url,
					Body:   body,
				}
			}
			return
		}
	}
	wg.Wait()

	errs := []error{}
	timeout := time.After(10 * 60 * time.Second)
	for range reqs {
		select {
		case res := <-resChan:
			results = append(results, res)
			log15.Info("Fetched... ", "URL", res.URL)
		case err := <-errChan:
			errs = append(errs, err)
		case <-timeout:
			return results, fmt.Errorf("Timeout Fetching")
		}
	}
	log15.Info("Finished fetching OVAL definitions")
	if 0 < len(errs) {
		return results, fmt.Errorf("%s", errs)
	}
	return results, nil
}

func fetchFileConcurrently(req fetchRequest, concurrency int) (body []byte, err error) {
	var proxyURL *url.URL
	httpClient := &http.Client{}
	httpProxy := viper.GetString("http-proxy")
	if httpProxy != "" {
		if proxyURL, err = url.Parse(httpProxy); err != nil {
			return nil, xerrors.Errorf("Failed to parse proxy url. err: %w", err)
		}
		httpClient = &http.Client{Transport: &http.Transport{Proxy: http.ProxyURL(proxyURL)}}
	}

	u, err := url.Parse(req.url)
	if err != nil {
		return nil, xerrors.Errorf("Failed to parse given URL: %w", err)
	}

	buf := bytes.Buffer{}
	htc := htcat.New(httpClient, u, concurrency)
	if _, err := htc.WriteTo(&buf); err != nil {
		return nil, xerrors.Errorf("Failed to write to output stream: %w", err)
	}

	var bytesBody []byte
	if req.bzip2 {
		var b bytes.Buffer
		if _, err := b.ReadFrom(bzip2.NewReader(bytes.NewReader(buf.Bytes()))); err != nil {
			return body, err
		}
		bytesBody = b.Bytes()
	} else {
		bytesBody = buf.Bytes()
	}

	return bytesBody, nil
}

func fetchFileWithUA(req fetchRequest) (body []byte, err error) {
	var proxyURL *url.URL
	var resp *http.Response

	httpClient := &http.Client{}
	httpProxy := viper.GetString("http-proxy")
	if httpProxy != "" {
		if proxyURL, err = url.Parse(httpProxy); err != nil {
			return nil, xerrors.Errorf("Failed to parse proxy url. err: %w", err)
		}
		httpClient = &http.Client{Transport: &http.Transport{Proxy: http.ProxyURL(proxyURL)}}
	}

	httpreq, err := http.NewRequest("GET", req.url, nil)
	if err != nil {
		return nil, xerrors.Errorf("Failed to download. err: %w", err)
	}

	httpreq.Header.Set("User-Agent", "curl/7.37.0")
	resp, err = httpClient.Do(httpreq)
	if err != nil {
		return nil, xerrors.Errorf("Failed to download. err: %w", err)
	}
	defer func() {
		if resp != nil {
			resp.Body.Close()
		}
	}()
	if resp.StatusCode != 200 {
		return nil, fmt.Errorf("Failed to HTTP GET. url: %s, response: %+v", req.url, resp)
	}

	buf := bytes.NewBuffer(nil)
	if _, err := io.Copy(buf, resp.Body); err != nil {
		return nil, err
	}

	var bytesBody []byte
	if req.bzip2 {
		bz := bzip2.NewReader(buf)
		var b bytes.Buffer
		if _, err := b.ReadFrom(bz); err != nil {
			return nil, err
		}
		bytesBody = b.Bytes()
	} else {
		bytesBody = buf.Bytes()
	}

	return bytesBody, nil
}
<|endoftext|>


<commit_before>package device

import (
	"sync"

	"github.com/tarm/serial"
)

// Manager manages devices that are plugged into the system. It supports auto
// detection of devices.
//
// Serial ports are opened each for a device, and a clean API for communicating
// is provided via Read, Write and Flush methods.
//
// The devices are monitored via udev, and any changes that requires reloading
// of the  ports are handled by reloading the ports to the devices.
//
// This is safe to use concurrently in multiple goroutines
type Manager struct {
	devices map[string]serial.Config
	conn    []*Conn
	mu      sync.RWMutex
}

func (m *Manager) AddDevice(name string) error {
	cfg := serial.Config{Name: name}
	m.mu.Lock()
	m.devices[name] = cfg
	m.mu.Unlock()
	return nil
}

func (m *Manager) RemoveDevice(name string) error {
	m.mu.RLock()
	delete(m.devices, name)
	m.mu.RUnlock()
	return nil
}

type Conn struct {
	device serial.Config
	port   *serial.Port
	isOpen bool
}

func (c *Conn) Open() error {
	p, err := serial.OpenPort(&c.device)
	if err != nil {
		return nil
	}
	c.port = p
	c.isOpen = true
	return nil
}

// Close closes the port helt by *Conn.
func (c *Conn) Close() error {
	if c.isOpen {
		return c.port.Close()
	}
	return nil
}
<commit_msg>Add docummentation<commit_after>package device

import (
	"sync"

	"github.com/tarm/serial"
)

// Manager manages devices that are plugged into the system. It supports auto
// detection of devices.
//
// Serial ports are opened each for a device, and a clean API for communicating
// is provided via Read, Write and Flush methods.
//
// The devices are monitored via udev, and any changes that requires reloading
// of the  ports are handled by reloading the ports to the devices.
//
// This is safe to use concurrently in multiple goroutines
type Manager struct {
	devices map[string]serial.Config
	conn    []*Conn
	mu      sync.RWMutex
}

// AddDevice adds device name to the manager
func (m *Manager) AddDevice(name string) error {
	cfg := serial.Config{Name: name}
	m.mu.Lock()
	m.devices[name] = cfg
	m.mu.Unlock()
	return nil
}

// RemoveDevice removes device name from the manager
func (m *Manager) RemoveDevice(name string) error {
	m.mu.RLock()
	delete(m.devices, name)
	m.mu.RUnlock()
	return nil
}

type Conn struct {
	device serial.Config
	port   *serial.Port
	isOpen bool
}

func (c *Conn) Open() error {
	p, err := serial.OpenPort(&c.device)
	if err != nil {
		return nil
	}
	c.port = p
	c.isOpen = true
	return nil
}

// Close closes the port helt by *Conn.
func (c *Conn) Close() error {
	if c.isOpen {
		return c.port.Close()
	}
	return nil
}
<|endoftext|>