<commit_before>package client_builder

import (
	"crypto/tls"
	"net"
	"net/http"
	"time"

	"net/url"

	"errors"

	"github.com/golang/glog"
)

const (
	DEFAULT_TIMEOUT     = 30 * time.Second
	KEEPALIVE           = 30 * time.Second
	TLSHANDSHAKETIMEOUT = 10 * time.Second
)

type DialFunc func(network, address string) (net.Conn, error)

type HttpClientBuilder interface {
	Build() *http.Client
	BuildRoundTripper() http.RoundTripper
	WithProxy() HttpClientBuilder
	WithoutProxy() HttpClientBuilder
	WithRedirects() HttpClientBuilder
	WithoutRedirects() HttpClientBuilder
	WithTimeout(timeout time.Duration) HttpClientBuilder
	WithDialFunc(dialFunc DialFunc) HttpClientBuilder
}

type httpClientBuilder struct {
	proxy         Proxy
	checkRedirect CheckRedirect
	timeout       time.Duration
	dialFunc      DialFunc
}

type Proxy func(req *http.Request) (*url.URL, error)

type CheckRedirect func(req *http.Request, via []*http.Request) error

func New() *httpClientBuilder {
	b := new(httpClientBuilder)
	b.WithoutProxy()
	b.WithRedirects()
	b.timeout = DEFAULT_TIMEOUT
	return b
}

func (h *httpClientBuilder) WithTimeout(timeout time.Duration) HttpClientBuilder {
	h.timeout = timeout
	return h
}

func (h *httpClientBuilder) WithDialFunc(dialFunc DialFunc) HttpClientBuilder {
	h.dialFunc = dialFunc
	return h
}

func (b *httpClientBuilder) BuildDialFunc() DialFunc {
	if b.dialFunc != nil {
		return b.dialFunc
	}
	return (&net.Dialer{
		Timeout: b.timeout,
		//		KeepAlive: KEEPALIVE,
	}).Dial
}

func (b *httpClientBuilder) BuildRoundTripper() http.RoundTripper {
	if glog.V(4) {
		glog.Infof("build http transport")
	}
	return &http.Transport{
		Proxy:           b.proxy,
		Dial:            b.BuildDialFunc(),
		TLSClientConfig: &tls.Config{InsecureSkipVerify: true},
		//		TLSHandshakeTimeout: TLSHANDSHAKETIMEOUT,
	}
}

func (b *httpClientBuilder) Build() *http.Client {
	if glog.V(4) {
		glog.Infof("build http client")
	}
	return &http.Client{
		Transport:     b.BuildRoundTripper(),
		CheckRedirect: b.checkRedirect,
	}
}

func (b *httpClientBuilder) WithProxy() HttpClientBuilder {
	b.proxy = http.ProxyFromEnvironment
	return b
}

func (b *httpClientBuilder) WithoutProxy() HttpClientBuilder {
	b.proxy = nil
	return b
}

func (b *httpClientBuilder) WithRedirects() HttpClientBuilder {
	b.checkRedirect = func(req *http.Request, via []*http.Request) error {
		if len(via) >= 10 {
			return errors.New("stopped after 10 redirects")
		}
		return nil
	}
	return b
}

func (b *httpClientBuilder) WithoutRedirects() HttpClientBuilder {
	b.checkRedirect = func(req *http.Request, via []*http.Request) error {
		if len(via) >= 1 {
			return errors.New("redirects")
		}
		return nil
	}
	return b
}
<commit_msg>decrease loglevel<commit_after>package client_builder

import (
	"crypto/tls"
	"net"
	"net/http"
	"time"

	"net/url"

	"errors"

	"github.com/golang/glog"
)

const (
	DEFAULT_TIMEOUT     = 30 * time.Second
	KEEPALIVE           = 30 * time.Second
	TLSHANDSHAKETIMEOUT = 10 * time.Second
)

type DialFunc func(network, address string) (net.Conn, error)

type HttpClientBuilder interface {
	Build() *http.Client
	BuildRoundTripper() http.RoundTripper
	WithProxy() HttpClientBuilder
	WithoutProxy() HttpClientBuilder
	WithRedirects() HttpClientBuilder
	WithoutRedirects() HttpClientBuilder
	WithTimeout(timeout time.Duration) HttpClientBuilder
	WithDialFunc(dialFunc DialFunc) HttpClientBuilder
}

type httpClientBuilder struct {
	proxy         Proxy
	checkRedirect CheckRedirect
	timeout       time.Duration
	dialFunc      DialFunc
}

type Proxy func(req *http.Request) (*url.URL, error)

type CheckRedirect func(req *http.Request, via []*http.Request) error

func New() *httpClientBuilder {
	b := new(httpClientBuilder)
	b.WithoutProxy()
	b.WithRedirects()
	b.timeout = DEFAULT_TIMEOUT
	return b
}

func (h *httpClientBuilder) WithTimeout(timeout time.Duration) HttpClientBuilder {
	h.timeout = timeout
	return h
}

func (h *httpClientBuilder) WithDialFunc(dialFunc DialFunc) HttpClientBuilder {
	h.dialFunc = dialFunc
	return h
}

func (b *httpClientBuilder) BuildDialFunc() DialFunc {
	if b.dialFunc != nil {
		return b.dialFunc
	}
	return (&net.Dialer{
		Timeout: b.timeout,
		//		KeepAlive: KEEPALIVE,
	}).Dial
}

func (b *httpClientBuilder) BuildRoundTripper() http.RoundTripper {
	if glog.V(5) {
		glog.Infof("build http transport")
	}
	return &http.Transport{
		Proxy:           b.proxy,
		Dial:            b.BuildDialFunc(),
		TLSClientConfig: &tls.Config{InsecureSkipVerify: true},
		//		TLSHandshakeTimeout: TLSHANDSHAKETIMEOUT,
	}
}

func (b *httpClientBuilder) Build() *http.Client {
	if glog.V(5) {
		glog.Infof("build http client")
	}
	return &http.Client{
		Transport:     b.BuildRoundTripper(),
		CheckRedirect: b.checkRedirect,
	}
}

func (b *httpClientBuilder) WithProxy() HttpClientBuilder {
	b.proxy = http.ProxyFromEnvironment
	return b
}

func (b *httpClientBuilder) WithoutProxy() HttpClientBuilder {
	b.proxy = nil
	return b
}

func (b *httpClientBuilder) WithRedirects() HttpClientBuilder {
	b.checkRedirect = func(req *http.Request, via []*http.Request) error {
		if len(via) >= 10 {
			return errors.New("stopped after 10 redirects")
		}
		return nil
	}
	return b
}

func (b *httpClientBuilder) WithoutRedirects() HttpClientBuilder {
	b.checkRedirect = func(req *http.Request, via []*http.Request) error {
		if len(via) >= 1 {
			return errors.New("redirects")
		}
		return nil
	}
	return b
}
<|endoftext|>


<commit_before>// Copyright 2014-2015 The DevMine authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

// Package repotool is able to fetch information from a source code repository.
// Typically, it can get all commits, their authors and commiters and so on
// and return this information in a JSON object. Alternatively, it is able
// to populate the information into a PostgreSQL database.
// Currently, on the Git VCS is supported.
package main

import (
	"bytes"
	"database/sql"
	"encoding/hex"
	"encoding/json"
	"errors"
	"flag"
	"fmt"
	"io"
	"os"
	"path/filepath"
	"strconv"
	"strings"
	"time"

	_ "github.com/lib/pq"
	mmh3 "github.com/spaolacci/murmur3"

	"github.com/DevMine/repotool/config"
	"github.com/DevMine/repotool/model"
	"github.com/DevMine/repotool/repo"
)

const version = "0.1.0"

func main() {
	flag.Usage = func() {
		fmt.Printf("usage: %s [OPTION(S)] [REPOSITORY PATH]\n", os.Args[0])
		flag.PrintDefaults()
		os.Exit(0)
	}

	configPath := flag.String("c", "", "configuration file")
	vflag := flag.Bool("v", false, "print version.")
	jsonflag := flag.Bool("json", true, "json output")
	dbflag := flag.Bool("db", false, "import data into the database")
	flag.Parse()

	if *vflag {
		fmt.Printf("%s - %s\n", filepath.Base(os.Args[0]), version)
		os.Exit(0)
	}

	if len(flag.Args()) != 1 {
		fmt.Fprintln(os.Stderr, "invalid # of arguments")
		flag.Usage()
	}

	if *dbflag && len(*configPath) == 0 {
		fatal(errors.New("a configuration file must be specified when using db option"))
	}

	cfg, err := config.ReadConfig(*configPath)
	if err != nil {
		fatal(err)
	}

	repoPath := flag.Arg(0)
	repository, err := repo.New(cfg.Data, repoPath)
	if err != nil {
		fatal(err)
	}

	fmt.Fprintln(os.Stderr, "fetching repository commits...")
	tic := time.Now()
	err = repository.FetchCommits()
	if err != nil {
		fatal(err)
	}
	toc := time.Now()
	fmt.Fprintln(os.Stderr, "done in ", toc.Sub(tic))

	if *jsonflag {
		bs, err := json.Marshal(repository)
		if err != nil {
			fatal(err)
		}

		fmt.Println(string(bs))
	}

	if *dbflag {
		db, err := OpenDBSession(cfg.Database)
		if err != nil {
			fatal(err)
		}
		defer db.Close()

		fmt.Fprintf(os.Stderr, "inserting %d commits into the database...\n",
			len(repository.GetCommits()))
		tic := time.Now()
		insertRepoData(db, repository)
		toc := time.Now()
		fmt.Fprintln(os.Stderr, "done in ", toc.Sub(tic))
	}
}

// fatal prints an error on standard error stream and exits.
func fatal(a ...interface{}) {
	fmt.Fprintln(os.Stderr, a...)
	os.Exit(1)
}

// OpenDBSession creates a session to the database.
func OpenDBSession(cfg config.DatabaseConfig) (*sql.DB, error) {
	dbURL := fmt.Sprintf(
		"user='%s' password='%s' host='%s' port=%d dbname='%s' sslmode='%s'",
		cfg.UserName, cfg.Password, cfg.HostName, cfg.Port, cfg.DBName, cfg.SSLMode)

	return sql.Open("postgres", dbURL)
}

// insertRepoData inserts repository data into the database, or updates it
// if it is already there.
func insertRepoData(db *sql.DB, r repo.Repo) {
	if db == nil {
		fatal(errors.New("nil database given"))
	}

	repoID := getRepoID(db, r)
	if repoID == 0 {
		fatal(errors.New("no corresponding repository found in the database: impossible to insert data"))
	}

	for _, c := range r.GetCommits() {
		insertCommit(db, repoID, c)
	}
}

// insertCommit inserts a commit into the database
func insertCommit(db *sql.DB, repoID int, c model.Commit) {
	commitFields := []string{
		"repository_id",
		"author_id",
		"committer_id",
		"hash",
		"vcs_id",
		"message",
		"author_date",
		"commit_date",
		"file_changed_count",
		"insertions_count",
		"deletions_count"}

	authorID := getUserID(db, c.Author.Email)
	committerID := getUserID(db, c.Committer.Email)
	hash := genCommitHash(c)

	query := genInsQuery("commits", commitFields...)
	_, err := db.Exec(query,
		repoID, authorID, committerID, hash,
		c.VCSID, c.Message, c.AuthorDate, c.CommitDate,
		c.FileChangedCount, c.InsertionsCount, c.DeletionsCount)
	if err != nil {
		fatal(err)
	}
}

// genCommitHash generates a hash (mmh3) from commit fields.
// This hash can then be used to uniquely identify a commit.
func genCommitHash(c model.Commit) string {
	h := mmh3.New128()

	io.WriteString(h, c.VCSID)
	io.WriteString(h, c.Message)
	io.WriteString(h, c.Author.Name)
	io.WriteString(h, c.Author.Email)
	io.WriteString(h, c.Committer.Name)
	io.WriteString(h, c.Committer.Email)
	io.WriteString(h, c.AuthorDate.String())
	io.WriteString(h, c.CommitDate.String())
	io.WriteString(h, strconv.FormatInt(int64(c.FileChangedCount), 10))
	io.WriteString(h, strconv.FormatInt(int64(c.InsertionsCount), 10))
	io.WriteString(h, strconv.FormatInt(int64(c.DeletionsCount), 10))

	return hex.EncodeToString(h.Sum(nil))
}

// getRepoID returns the repository id of a repo in repositories table.
// If repo is not in the table, then 0 is returned.
func getRepoID(db *sql.DB, r repo.Repo) int {
	if db == nil {
		fatal(errors.New("nil database given"))
	}

	var id int
	// Clone URL is unique
	err := db.QueryRow("SELECT id FROM repositories WHERE clone_url=$1", r.GetCloneURL()).Scan(&id)
	switch {
	case err == sql.ErrNoRows:
		return 0
	case err != nil:
		fatal(err)
	}
	return id
}

// getUserID attempts to find a user ID given its email address.
// Email addresses are unique, however they may not be provided.
// If no user ID is found, nil is returned, otherwhise the user ID
// is returned.
func getUserID(db *sql.DB, email string) *int {
	if db == nil {
		fatal(errors.New("nil database given"))
	}

	if len(email) == 0 {
		return nil
	}

	var id *int
	err := db.QueryRow("SELECT id FROM users WHERE email=$1", email).Scan(&id)
	switch {
	case err == sql.ErrNoRows:
		return nil
	case err != nil:
		fatal(err)
	}
	return id
}

// genInsQuery generates a query string for an insertion in the database.
func genInsQuery(tableName string, fields ...string) string {
	var buf bytes.Buffer

	buf.WriteString(fmt.Sprintf("INSERT INTO %s(%s)\n",
		tableName, strings.Join(fields, ",")))
	buf.WriteString("VALUES(")

	for ind := range fields {
		if ind > 0 {
			buf.WriteString(",")
		}

		buf.WriteString(fmt.Sprintf("$%d", ind+1))
	}

	buf.WriteString(")\n")

	return buf.String()
}
<commit_msg>repotool: Do not export OpenDBSession function.<commit_after>// Copyright 2014-2015 The DevMine authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

// Package repotool is able to fetch information from a source code repository.
// Typically, it can get all commits, their authors and commiters and so on
// and return this information in a JSON object. Alternatively, it is able
// to populate the information into a PostgreSQL database.
// Currently, on the Git VCS is supported.
package main

import (
	"bytes"
	"database/sql"
	"encoding/hex"
	"encoding/json"
	"errors"
	"flag"
	"fmt"
	"io"
	"os"
	"path/filepath"
	"strconv"
	"strings"
	"time"

	_ "github.com/lib/pq"
	mmh3 "github.com/spaolacci/murmur3"

	"github.com/DevMine/repotool/config"
	"github.com/DevMine/repotool/model"
	"github.com/DevMine/repotool/repo"
)

const version = "0.1.0"

func main() {
	flag.Usage = func() {
		fmt.Printf("usage: %s [OPTION(S)] [REPOSITORY PATH]\n", os.Args[0])
		flag.PrintDefaults()
		os.Exit(0)
	}

	configPath := flag.String("c", "", "configuration file")
	vflag := flag.Bool("v", false, "print version.")
	jsonflag := flag.Bool("json", true, "json output")
	dbflag := flag.Bool("db", false, "import data into the database")
	flag.Parse()

	if *vflag {
		fmt.Printf("%s - %s\n", filepath.Base(os.Args[0]), version)
		os.Exit(0)
	}

	if len(flag.Args()) != 1 {
		fmt.Fprintln(os.Stderr, "invalid # of arguments")
		flag.Usage()
	}

	if *dbflag && len(*configPath) == 0 {
		fatal(errors.New("a configuration file must be specified when using db option"))
	}

	cfg, err := config.ReadConfig(*configPath)
	if err != nil {
		fatal(err)
	}

	repoPath := flag.Arg(0)
	repository, err := repo.New(cfg.Data, repoPath)
	if err != nil {
		fatal(err)
	}

	fmt.Fprintln(os.Stderr, "fetching repository commits...")
	tic := time.Now()
	err = repository.FetchCommits()
	if err != nil {
		fatal(err)
	}
	toc := time.Now()
	fmt.Fprintln(os.Stderr, "done in ", toc.Sub(tic))

	if *jsonflag {
		bs, err := json.Marshal(repository)
		if err != nil {
			fatal(err)
		}

		fmt.Println(string(bs))
	}

	if *dbflag {
		db, err := openDBSession(cfg.Database)
		if err != nil {
			fatal(err)
		}
		defer db.Close()

		fmt.Fprintf(os.Stderr, "inserting %d commits into the database...\n",
			len(repository.GetCommits()))
		tic := time.Now()
		insertRepoData(db, repository)
		toc := time.Now()
		fmt.Fprintln(os.Stderr, "done in ", toc.Sub(tic))
	}
}

// fatal prints an error on standard error stream and exits.
func fatal(a ...interface{}) {
	fmt.Fprintln(os.Stderr, a...)
	os.Exit(1)
}

// openDBSession creates a session to the database.
func openDBSession(cfg config.DatabaseConfig) (*sql.DB, error) {
	dbURL := fmt.Sprintf(
		"user='%s' password='%s' host='%s' port=%d dbname='%s' sslmode='%s'",
		cfg.UserName, cfg.Password, cfg.HostName, cfg.Port, cfg.DBName, cfg.SSLMode)

	return sql.Open("postgres", dbURL)
}

// insertRepoData inserts repository data into the database, or updates it
// if it is already there.
func insertRepoData(db *sql.DB, r repo.Repo) {
	if db == nil {
		fatal(errors.New("nil database given"))
	}

	repoID := getRepoID(db, r)
	if repoID == 0 {
		fatal(errors.New("no corresponding repository found in the database: impossible to insert data"))
	}

	for _, c := range r.GetCommits() {
		insertCommit(db, repoID, c)
	}
}

// insertCommit inserts a commit into the database
func insertCommit(db *sql.DB, repoID int, c model.Commit) {
	commitFields := []string{
		"repository_id",
		"author_id",
		"committer_id",
		"hash",
		"vcs_id",
		"message",
		"author_date",
		"commit_date",
		"file_changed_count",
		"insertions_count",
		"deletions_count"}

	authorID := getUserID(db, c.Author.Email)
	committerID := getUserID(db, c.Committer.Email)
	hash := genCommitHash(c)

	query := genInsQuery("commits", commitFields...)
	_, err := db.Exec(query,
		repoID, authorID, committerID, hash,
		c.VCSID, c.Message, c.AuthorDate, c.CommitDate,
		c.FileChangedCount, c.InsertionsCount, c.DeletionsCount)
	if err != nil {
		fatal(err)
	}
}

// genCommitHash generates a hash (mmh3) from commit fields.
// This hash can then be used to uniquely identify a commit.
func genCommitHash(c model.Commit) string {
	h := mmh3.New128()

	io.WriteString(h, c.VCSID)
	io.WriteString(h, c.Message)
	io.WriteString(h, c.Author.Name)
	io.WriteString(h, c.Author.Email)
	io.WriteString(h, c.Committer.Name)
	io.WriteString(h, c.Committer.Email)
	io.WriteString(h, c.AuthorDate.String())
	io.WriteString(h, c.CommitDate.String())
	io.WriteString(h, strconv.FormatInt(int64(c.FileChangedCount), 10))
	io.WriteString(h, strconv.FormatInt(int64(c.InsertionsCount), 10))
	io.WriteString(h, strconv.FormatInt(int64(c.DeletionsCount), 10))

	return hex.EncodeToString(h.Sum(nil))
}

// getRepoID returns the repository id of a repo in repositories table.
// If repo is not in the table, then 0 is returned.
func getRepoID(db *sql.DB, r repo.Repo) int {
	if db == nil {
		fatal(errors.New("nil database given"))
	}

	var id int
	// Clone URL is unique
	err := db.QueryRow("SELECT id FROM repositories WHERE clone_url=$1", r.GetCloneURL()).Scan(&id)
	switch {
	case err == sql.ErrNoRows:
		return 0
	case err != nil:
		fatal(err)
	}
	return id
}

// getUserID attempts to find a user ID given its email address.
// Email addresses are unique, however they may not be provided.
// If no user ID is found, nil is returned, otherwhise the user ID
// is returned.
func getUserID(db *sql.DB, email string) *int {
	if db == nil {
		fatal(errors.New("nil database given"))
	}

	if len(email) == 0 {
		return nil
	}

	var id *int
	err := db.QueryRow("SELECT id FROM users WHERE email=$1", email).Scan(&id)
	switch {
	case err == sql.ErrNoRows:
		return nil
	case err != nil:
		fatal(err)
	}
	return id
}

// genInsQuery generates a query string for an insertion in the database.
func genInsQuery(tableName string, fields ...string) string {
	var buf bytes.Buffer

	buf.WriteString(fmt.Sprintf("INSERT INTO %s(%s)\n",
		tableName, strings.Join(fields, ",")))
	buf.WriteString("VALUES(")

	for ind := range fields {
		if ind > 0 {
			buf.WriteString(",")
		}

		buf.WriteString(fmt.Sprintf("$%d", ind+1))
	}

	buf.WriteString(")\n")

	return buf.String()
}
<|endoftext|>


<commit_before>// Package cmdline implements the parser for the command line arguments.
package cmdline

import (
	"errors"
	"github.com/rmohid/h2c/cli/rpc"
)

func Parse(args []string) (*rpc.Command, error) {
	if len(args) == 1 && (args[0] == HELP_OPTION.short || args[0] == HELP_OPTION.long) {
		// h2c --help
		return nil, errors.New(globalUsage())
	}
	cmd, err := findCommand(args)
	if err != nil {
		return nil, err
	}
	remainingArgs, options, err := parseOptions(args, cmd)
	if err != nil {
		return nil, err
	}
	if HELP_OPTION.IsSet(options) {
		return nil, errors.New(help(cmd))
	}
	if len(remainingArgs) < cmd.minArgs+1 || len(remainingArgs) > cmd.maxArgs+1 {
		return nil, errors.New(usage(cmd))
	}
	cmdArgs := make([]string, 0)
	if len(remainingArgs) > 1 {
		cmdArgs = remainingArgs[1:]
		if cmd.areArgsValid != nil && !cmd.areArgsValid(cmdArgs) {
			return nil, errors.New(usage(cmd))
		}
	}
	return rpc.NewCommand(cmd.name, cmdArgs, options)
}

func parseOptions(args []string, cmd *command) ([]string, map[string]string, error) {
	foundOptions := make(map[string]string)
	for _, opt := range options {
		if opt.supportsCommand(cmd) {
			i, found := opt.findIndex(args)
			if found {
				if opt.hasParam {
					if len(args) <= i+1 {
						return nil, nil, errors.New(globalUsage())
					}
					if !opt.isParamValid(args[i+1]) {
						return nil, nil, errors.New(globalUsage())
					}
					opt.Set(args[i+1], foundOptions)
					args = append(args[:i], args[i+2:]...)
				} else {
					opt.Set("", foundOptions)
					args = append(args[:i], args[i+1:]...)
				}
			}
		}
	}
	return args, foundOptions, nil
}

func globalUsage() string {
	result := "Usage: h2c ["
	first := true
	for _, cmd := range commands {
		if !first {
			result += "|"
		}
		result += cmd.name
		first = false
	}
	result += "] <args>\nRun 'h2c [cmd] " + HELP_OPTION.long + "' to learn more about a command."
	return result
}

func usage(cmd *command) string {
	return "Usage: " + cmd.usage + "\nRun 'h2c " + cmd.name + " " + HELP_OPTION.long + "' for help."
}

func help(cmd *command) string {
	result := cmd.description
	result += "\nUsage: " + cmd.usage
	availableOptions := make([]*option, 0)
	for _, opt := range options {
		if opt.supportsCommand(cmd) {
			availableOptions = append(availableOptions, opt)
		}
	}
	if len(availableOptions) > 0 {
		result += "\nAvailable options are:"
		for _, opt := range availableOptions {
			result += "\n    " + opt.short + " " + opt.long + ": " + opt.description
		}
	}
	return result
}

func findCommand(args []string) (*command, error) {
	if len(args) < 1 {
		return nil, errors.New(globalUsage())
	}
	// make '--version' be like 'version'
	if args[0] == "--"+VERSION_COMMAND.name {
		return VERSION_COMMAND, nil
	}
	for _, cmd := range commands {
		if args[0] == cmd.name {
			return cmd, nil
		}
	}
	for _, opt := range options {
		if args[0] == opt.short || args[0] == opt.long {
			if opt.hasParam {
				if len(args) < 2 {
					return nil, errors.New(globalUsage())
				} else {
					return findCommand(args[2:])
				}
			}
			return findCommand(args[1:])
		}
	}
	return nil, errors.New(args[0] + ": Unknown command. Run 'h2c " + HELP_OPTION.long + "' for help.")
}

func (opt *option) findIndex(argv []string) (int, bool) {
	for i, arg := range argv {
		if arg == opt.short || arg == opt.long {
			return i, true
		}
	}
	return -1, false
}

func (opt *option) supportsCommand(cmd *command) bool {
	for _, c := range opt.commands {
		if c.name == cmd.name {
			return true
		}
	}
	return false
}
<commit_msg>version reset<commit_after>// Package cmdline implements the parser for the command line arguments.
package cmdline

import (
	"errors"
	"github.com/rmohid/h2c/cli/rpc"
)

func Parse(args []string) (*rpc.Command, error) {
	if len(args) == 1 && (args[0] == HELP_OPTION.short || args[0] == HELP_OPTION.long) {
		// h2c --help
		return nil, errors.New(globalUsage())
	}
	cmd, err := findCommand(args)
	if err != nil {
		return nil, err
	}
	remainingArgs, options, err := parseOptions(args, cmd)
	if err != nil {
		return nil, err
	}
	if HELP_OPTION.IsSet(options) {
		return nil, errors.New(help(cmd))
	}
	if len(remainingArgs) < cmd.minArgs+1 || len(remainingArgs) > cmd.maxArgs+1 {
		return nil, errors.New(usage(cmd))
	}
	cmdArgs := make([]string, 0)
	if len(remainingArgs) > 1 {
		cmdArgs = remainingArgs[1:]
		if cmd.areArgsValid != nil && !cmd.areArgsValid(cmdArgs) {
			return nil, errors.New(usage(cmd))
		}
	}
	return rpc.NewCommand(cmd.name, cmdArgs, options)
}

func parseOptions(args []string, cmd *command) ([]string, map[string]string, error) {
	foundOptions := make(map[string]string)
	for _, opt := range options {
		if opt.supportsCommand(cmd) {
			i, found := opt.findIndex(args)
			if found {
				if opt.hasParam {
					if len(args) <= i+1 {
						return nil, nil, errors.New(globalUsage())
					}
					if !opt.isParamValid(args[i+1]) {
						return nil, nil, errors.New(globalUsage())
					}
					opt.Set(args[i+1], foundOptions)
					args = append(args[:i], args[i+2:]...)
				} else {
					opt.Set("", foundOptions)
					args = append(args[:i], args[i+1:]...)
				}
			}
		}
	}
	return args, foundOptions, nil
}

func globalUsage() string {
	result := "Usage: h2c (rmohid) ["
	first := true
	for _, cmd := range commands {
		if !first {
			result += "|"
		}
		result += cmd.name
		first = false
	}
	result += "] <args>\nRun 'h2c [cmd] " + HELP_OPTION.long + "' to learn more about a command."
	return result
}

func usage(cmd *command) string {
	return "Usage: " + cmd.usage + "\nRun 'h2c " + cmd.name + " " + HELP_OPTION.long + "' for help."
}

func help(cmd *command) string {
	result := cmd.description
	result += "\nUsage: " + cmd.usage
	availableOptions := make([]*option, 0)
	for _, opt := range options {
		if opt.supportsCommand(cmd) {
			availableOptions = append(availableOptions, opt)
		}
	}
	if len(availableOptions) > 0 {
		result += "\nAvailable options are:"
		for _, opt := range availableOptions {
			result += "\n    " + opt.short + " " + opt.long + ": " + opt.description
		}
	}
	return result
}

func findCommand(args []string) (*command, error) {
	if len(args) < 1 {
		return nil, errors.New(globalUsage())
	}
	// make '--version' be like 'version'
	if args[0] == "--"+VERSION_COMMAND.name {
		return VERSION_COMMAND, nil
	}
	for _, cmd := range commands {
		if args[0] == cmd.name {
			return cmd, nil
		}
	}
	for _, opt := range options {
		if args[0] == opt.short || args[0] == opt.long {
			if opt.hasParam {
				if len(args) < 2 {
					return nil, errors.New(globalUsage())
				} else {
					return findCommand(args[2:])
				}
			}
			return findCommand(args[1:])
		}
	}
	return nil, errors.New(args[0] + ": Unknown command. Run 'h2c " + HELP_OPTION.long + "' for help.")
}

func (opt *option) findIndex(argv []string) (int, bool) {
	for i, arg := range argv {
		if arg == opt.short || arg == opt.long {
			return i, true
		}
	}
	return -1, false
}

func (opt *option) supportsCommand(cmd *command) bool {
	for _, c := range opt.commands {
		if c.name == cmd.name {
			return true
		}
	}
	return false
}
<|endoftext|>


<commit_before>package main

import (
	"fmt"
	"log"
	"path"
	"strings"

	"pault.ag/go/minion/minion"
	"pault.ag/go/reprepro"
	"pault.ag/go/service"
)

var archive *string
var fqdn *string

var repreproCommand = Command{
	Name:  "reprepro",
	Run:   repreproRun,
	Usage: ``,
}

func init() {
	archive = repreproCommand.Flag.String("archive", "", "archive we're hacking on")
	fqdn = repreproCommand.Flag.String("fqdn", "", "root fqdn")
}

type Incoming struct {
	Type      string
	Suite     string
	Flavor    string
	Component string
	Arch      string
	Package   string
	Version   string
	Files     []string
}

func (i *Incoming) Parse(args []string) error {
	if len(args) < 8 {
		return fmt.Errorf("Malformed request: %s", args)
	}
	i.Type = args[0]
	i.Suite = args[1]
	i.Flavor = args[2]
	i.Component = args[3]
	i.Arch = args[4]
	i.Package = args[5]
	i.Version = args[6]

	for _, pkg := range args[8:] {
		i.Files = append(i.Files, pkg)
	}

	return nil
}

func (i *Incoming) GetDSC() (string, error) {
	if i.Flavor != "dsc" {
		return "", fmt.Errorf("Flavor is '%s', not dsc", i.Flavor)
	}
	for _, pkg := range i.Files {
		if strings.HasSuffix(pkg, ".dsc") {
			return pkg, nil
		}
	}
	return "", fmt.Errorf("No such file D:")
}

func repreproRun(config minion.MinionConfig, cmd *Command, args []string) {
	incoming := Incoming{}
	incoming.Parse(args)
	if incoming.Type != "add" {
		return
	}

	dscPath, err := incoming.GetDSC()
	if err != nil {
		log.Fatalf("Error! %s\n", err)
	}
	log.Printf("%s\n", dscPath)

	conn, err := service.DialFromKeys(
		fmt.Sprintf("%s:%d", config.Host, config.Port),
		config.Cert, config.Key, config.CaCert,
	)
	if err != nil {
		log.Fatalf("Error! %s\n", err)
	}
	proxy := minion.CoordinatorProxy{service.Client(conn)}
	log.Printf("Queueing build\n")

	archiveRoot := fmt.Sprintf("http://%s", path.Join(*fqdn, *archive))

	repo := reprepro.GetWorkingRepo()
	buildNeeding, err := repo.BuildNeeding(incoming.Suite, "any", &incoming.Package)
	if err != nil {
		log.Fatalf("Error! %s\n", err)
	}

	for _, build := range buildNeeding {
		proxy.QueueBuild(minion.Build{
			Archives: []minion.Archive{
				minion.Archive{
					Root:     archiveRoot,
					Key:      fmt.Sprintf("%s.asc", archiveRoot),
					Suite:    incoming.Suite,
					Sections: []string{incoming.Component},
				},
			},
			Chroot: minion.Chroot{
				Chroot: incoming.Suite,
				Target: incoming.Suite,
			},
			Arch: build.Arch,
			DSC:  fmt.Sprintf("%s/%s", archiveRoot, dscPath),
			Upload: minion.Upload{
				Host:    *fqdn,
				Port:    1984,
				Archive: *archive,
			},
		})
	}
	log.Printf("Queued\n")
}
<commit_msg>Refactor queue code.<commit_after>package main

import (
	"fmt"
	"log"
	"path"
	"strings"

	"pault.ag/go/minion/minion"
	"pault.ag/go/reprepro"
	"pault.ag/go/service"
)

var archive *string
var fqdn *string

var repreproCommand = Command{
	Name:  "reprepro",
	Run:   repreproRun,
	Usage: ``,
}

func init() {
	archive = repreproCommand.Flag.String("archive", "", "archive we're hacking on")
	fqdn = repreproCommand.Flag.String("fqdn", "", "root fqdn")
}

type Incoming struct {
	Type      string
	Suite     string
	Flavor    string
	Component string
	Arch      string
	Package   string
	Version   string
	Files     []string
}

func (i *Incoming) Parse(args []string) error {
	if len(args) < 8 {
		return fmt.Errorf("Malformed request: %s", args)
	}
	i.Type = args[0]
	i.Suite = args[1]
	i.Flavor = args[2]
	i.Component = args[3]
	i.Arch = args[4]
	i.Package = args[5]
	i.Version = args[6]

	for _, pkg := range args[8:] {
		i.Files = append(i.Files, pkg)
	}

	return nil
}

func (i *Incoming) GetDSC() (string, error) {
	if i.Flavor != "dsc" {
		return "", fmt.Errorf("Flavor is '%s', not dsc", i.Flavor)
	}
	for _, pkg := range i.Files {
		if strings.HasSuffix(pkg, ".dsc") {
			return pkg, nil
		}
	}
	return "", fmt.Errorf("No such file D:")
}

func repreproRun(config minion.MinionConfig, cmd *Command, args []string) {
	incoming := Incoming{}
	incoming.Parse(args)
	if incoming.Type != "add" {
		return
	}

	dscPath, err := incoming.GetDSC()
	if err != nil {
		log.Fatalf("Error! %s\n", err)
	}
	log.Printf("%s\n", dscPath)

	conn, err := service.DialFromKeys(
		fmt.Sprintf("%s:%d", config.Host, config.Port),
		config.Cert, config.Key, config.CaCert,
	)
	if err != nil {
		log.Fatalf("Error! %s\n", err)
	}
	proxy := minion.CoordinatorProxy{service.Client(conn)}
	log.Printf("Queueing build\n")

	archiveRoot := fmt.Sprintf("http://%s", path.Join(*fqdn, *archive))

	repo := reprepro.GetWorkingRepo()
	buildNeeding, err := repo.BuildNeeding(incoming.Suite, "any", &incoming.Package)
	if err != nil {
		log.Fatalf("Error! %s\n", err)
	}

	for _, build := range buildNeeding {
		QueueBuildNeeding(proxy, archiveRoot, build, incoming,
			dscPath, *fqdn, *archive)
	}
	log.Printf("Queued\n")
}

func QueueBuildNeeding(
	proxy minion.CoordinatorProxy,
	archiveRoot string,
	build reprepro.BuildNeedingPackage,
	incoming Incoming,
	dscPath string,
	uploadHost string,
	uploadArchive string,
) {
	proxy.QueueBuild(minion.Build{
		Archives: []minion.Archive{
			minion.Archive{
				Root:     archiveRoot,
				Key:      fmt.Sprintf("%s.asc", archiveRoot),
				Suite:    incoming.Suite,
				Sections: []string{incoming.Component},
			},
		},
		Chroot: minion.Chroot{
			Chroot: incoming.Suite,
			Target: incoming.Suite,
		},
		Arch: build.Arch,
		DSC:  fmt.Sprintf("%s/%s", archiveRoot, dscPath),
		Upload: minion.Upload{
			Host:    uploadHost,
			Port:    1984,
			Archive: uploadArchive,
		},
	})
}
<|endoftext|>


<commit_before>// Copyright 2015 by caixw, All rights reserved.
// Use of this source code is governed by a MIT
// license that can be found in the LICENSE file.

package web

import (
	"errors"
	"fmt"
	"net/http"
	"sync"

	"github.com/issue9/mux"
)

var (
	serveMux = mux.NewServeMux()

	modules   = map[string]*Module{} // 所有模块的列表。
	modulesMu sync.Mutex
)

var ErrModuleExists = errors.New("该名称的模块已经存在")

// Module 模块化管理路由项。相对于 mux.Group，添加了模块依赖管理。
type Module struct {
	Name         string   // 名称
	Dependencies []string // 依赖项
	group        *mux.Group
}

// Modules 所有模块列表。
func Modules() []*Module {
	ret := make([]*Module, 0, len(modules))
	for _, m := range modules {
		ret = append(ret, m)
	}
	return ret
}

// GetModule 获取指定名称的模块，若不存在，则返回 nil
func GetModule(name string) *Module {
	modulesMu.Lock()
	defer modulesMu.Unlock()

	return modules[name]
}

// NewModule 声明一个新的模块，若该名称已经存在，则返回错误信息。
// name 模块名称
// dependencies 该模块的依赖模块列表。
func NewModule(name string, dependencies ...string) (*Module, error) {
	modulesMu.Lock()
	defer modulesMu.Unlock()

	// 确保没有同名存在。
	if _, found := modules[name]; found {
		return nil, ErrModuleExists
	}

	// 检测依赖模块是否都已经存在
	for _, m := range dependencies {
		if _, found := modules[m]; !found {
			return nil, fmt.Errorf("依赖项[%v]不存在", m)
		}
	}

	m := &Module{
		Name:         name,
		Dependencies: dependencies,
		group:        serveMux.Group(),
	}
	modules[name] = m

	return m, nil
}

// IsRunning 当前模块的路由是否处于运行状态
func (m *Module) IsRunning() bool {
	return m.group.IsRunning()
}

// Start 将当前模块改为运行状态
func (m *Module) Start() {
	m.group.Start()
}

// Stop 将当前模块改为暂停状态。
func (m *Module) Stop() {
	m.group.Stop()
}

// Add 添加一个路由项。
// 具体参数说明，可参考 github.com/issue9/mux.ServeMux.Add() 方法。
func (m *Module) Add(pattern string, h http.Handler, methods ...string) *Module {
	m.group.Add(pattern, h, methods...)
	return m
}

// Get 相当于 Module.Add(pattern, h, "GET")
func (m *Module) Get(pattern string, h http.Handler) *Module {
	return m.Add(pattern, h, "GET")
}

// Post 相当于 Module.Add(pattern, h, "POST")
func (m *Module) Post(pattern string, h http.Handler) *Module {
	return m.Add(pattern, h, "POST")
}

// Delete 相当于 Module.Add(pattern, h, "DELETE")
func (m *Module) Delete(pattern string, h http.Handler) *Module {
	return m.Add(pattern, h, "DELETE")
}

// Put 相当于 Module.Add(pattern, h, "PUT")
func (m *Module) Put(pattern string, h http.Handler) *Module {
	return m.Add(pattern, h, "PUT")
}

// Patch 相当于 Module.Add(pattern, h, "PATCH")
func (m *Module) Patch(pattern string, h http.Handler) *Module {
	return m.Add(pattern, h, "PATCH")
}

// Any 相当于 Module.Add(pattern, h)
func (m *Module) Any(pattern string, h http.Handler) *Module {
	m.group.Any(pattern, h)
	return m
}

// AddFunc 相当于 Module.Add(pattern, http.HandlerFunc(fun), methods...)
func (m *Module) AddFunc(pattern string, fun func(http.ResponseWriter, *http.Request), methods ...string) *Module {
	m.group.AddFunc(pattern, fun, methods...)
	return m
}

// GetFunc 相当于 Module.AddFunc(pattern, fun, "GET")
func (m *Module) GetFunc(pattern string, fun func(http.ResponseWriter, *http.Request)) *Module {
	return m.AddFunc(pattern, fun, "GET")
}

// PutFunc 相当于 Module.AddFunc(pattern, fun, "PUT")
func (m *Module) PutFunc(pattern string, fun func(http.ResponseWriter, *http.Request)) *Module {
	return m.AddFunc(pattern, fun, "PUT")
}

// PostFunc 相当于 Module.AddFunc(pattern, fun, "POST")
func (m *Module) PostFunc(pattern string, fun func(http.ResponseWriter, *http.Request)) *Module {
	return m.AddFunc(pattern, fun, "POST")
}

// DeleteFunc 相当于 Module.Addunc(pattern, fun, "DELETE")
func (m *Module) DeleteFunc(pattern string, fun func(http.ResponseWriter, *http.Request)) *Module {
	return m.AddFunc(pattern, fun, "DELETE")
}

// PatchFunc 相当于 Module.AddFunc(pattern, fun, "PATCH")
func (m *Module) PatchFunc(pattern string, fun func(http.ResponseWriter, *http.Request)) *Module {
	return m.AddFunc(pattern, fun, "PATCH")
}

// AnyFunc 相当于 Module.AddFunc(pattern, fn)
func (m *Module) AnyFunc(pattern string, fun func(http.ResponseWriter, *http.Request)) *Module {
	m.group.AnyFunc(pattern, fun)
	return m
}

// Clean 清除当前模块的所有路由项
func (m *Module) Clean() *Module {
	m.group.Clean()
	return m
}

// Prefix 创建一个 mux.Prefix 对象，具体可参考该实例说明。
func (m *Module) Prefix(prefix string) *mux.Prefix {
	return m.group.Prefix(prefix)
}
<commit_msg>优化锁机制<commit_after>// Copyright 2015 by caixw, All rights reserved.
// Use of this source code is governed by a MIT
// license that can be found in the LICENSE file.

package web

import (
	"errors"
	"fmt"
	"net/http"
	"sync"

	"github.com/issue9/mux"
)

var (
	serveMux = mux.NewServeMux()

	modules   = map[string]*Module{} // 所有模块的列表。
	modulesMu sync.RWMutex
)

var ErrModuleExists = errors.New("该名称的模块已经存在")

// Module 模块化管理路由项。相对于 mux.Group，添加了模块依赖管理。
type Module struct {
	Name         string   // 名称
	Dependencies []string // 依赖项
	Author       string   // 模块作者
	Version      string   // 模块版本
	group        *mux.Group
}

// Modules 所有模块列表。
func Modules() []*Module {
	ret := make([]*Module, 0, len(modules))

	modulesMu.RLock()
	for _, m := range modules {
		ret = append(ret, m)
	}
	modulesMu.RUnlock()

	return ret
}

// GetModule 获取指定名称的模块，若不存在，则返回 nil
func GetModule(name string) *Module {
	modulesMu.RLock()
	ret, found := modules[name]
	modulesMu.RUnlock()

	if found {
		return ret
	}
	return nil
}

// NewModule 声明一个新的模块，若该名称已经存在，则返回错误信息。
// name 模块名称
// dependencies 该模块的依赖模块列表。
func NewModule(name string, dependencies ...string) (*Module, error) {
	modulesMu.Lock()
	defer modulesMu.Unlock()

	// 确保没有同名存在。
	if _, found := modules[name]; found {
		return nil, ErrModuleExists
	}

	// 检测依赖模块是否都已经存在
	for _, m := range dependencies {
		if _, found := modules[m]; !found {
			return nil, fmt.Errorf("依赖项[%v]不存在", m)
		}
	}

	m := &Module{
		Name:         name,
		Dependencies: dependencies,
		group:        serveMux.Group(),
	}
	modules[name] = m

	return m, nil
}

// IsRunning 当前模块的路由是否处于运行状态
func (m *Module) IsRunning() bool {
	return m.group.IsRunning()
}

// Start 将当前模块改为运行状态
func (m *Module) Start() {
	m.group.Start()
}

// Stop 将当前模块改为暂停状态。
func (m *Module) Stop() {
	m.group.Stop()
}

// Add 添加一个路由项。
// 具体参数说明，可参考 github.com/issue9/mux.ServeMux.Add() 方法。
func (m *Module) Add(pattern string, h http.Handler, methods ...string) *Module {
	m.group.Add(pattern, h, methods...)
	return m
}

// Get 相当于 Module.Add(pattern, h, "GET")
func (m *Module) Get(pattern string, h http.Handler) *Module {
	return m.Add(pattern, h, "GET")
}

// Post 相当于 Module.Add(pattern, h, "POST")
func (m *Module) Post(pattern string, h http.Handler) *Module {
	return m.Add(pattern, h, "POST")
}

// Delete 相当于 Module.Add(pattern, h, "DELETE")
func (m *Module) Delete(pattern string, h http.Handler) *Module {
	return m.Add(pattern, h, "DELETE")
}

// Put 相当于 Module.Add(pattern, h, "PUT")
func (m *Module) Put(pattern string, h http.Handler) *Module {
	return m.Add(pattern, h, "PUT")
}

// Patch 相当于 Module.Add(pattern, h, "PATCH")
func (m *Module) Patch(pattern string, h http.Handler) *Module {
	return m.Add(pattern, h, "PATCH")
}

// Any 相当于 Module.Add(pattern, h)
func (m *Module) Any(pattern string, h http.Handler) *Module {
	m.group.Any(pattern, h)
	return m
}

// AddFunc 相当于 Module.Add(pattern, http.HandlerFunc(fun), methods...)
func (m *Module) AddFunc(pattern string, fun func(http.ResponseWriter, *http.Request), methods ...string) *Module {
	m.group.AddFunc(pattern, fun, methods...)
	return m
}

// GetFunc 相当于 Module.AddFunc(pattern, fun, "GET")
func (m *Module) GetFunc(pattern string, fun func(http.ResponseWriter, *http.Request)) *Module {
	return m.AddFunc(pattern, fun, "GET")
}

// PutFunc 相当于 Module.AddFunc(pattern, fun, "PUT")
func (m *Module) PutFunc(pattern string, fun func(http.ResponseWriter, *http.Request)) *Module {
	return m.AddFunc(pattern, fun, "PUT")
}

// PostFunc 相当于 Module.AddFunc(pattern, fun, "POST")
func (m *Module) PostFunc(pattern string, fun func(http.ResponseWriter, *http.Request)) *Module {
	return m.AddFunc(pattern, fun, "POST")
}

// DeleteFunc 相当于 Module.Addunc(pattern, fun, "DELETE")
func (m *Module) DeleteFunc(pattern string, fun func(http.ResponseWriter, *http.Request)) *Module {
	return m.AddFunc(pattern, fun, "DELETE")
}

// PatchFunc 相当于 Module.AddFunc(pattern, fun, "PATCH")
func (m *Module) PatchFunc(pattern string, fun func(http.ResponseWriter, *http.Request)) *Module {
	return m.AddFunc(pattern, fun, "PATCH")
}

// AnyFunc 相当于 Module.AddFunc(pattern, fn)
func (m *Module) AnyFunc(pattern string, fun func(http.ResponseWriter, *http.Request)) *Module {
	m.group.AnyFunc(pattern, fun)
	return m
}

// Clean 清除当前模块的所有路由项
func (m *Module) Clean() *Module {
	m.group.Clean()
	return m
}

// Prefix 创建一个 mux.Prefix 对象，具体可参考该实例说明。
func (m *Module) Prefix(prefix string) *mux.Prefix {
	return m.group.Prefix(prefix)
}
<|endoftext|>


<commit_before>package testing

import (
	"testing"
)

var infoParams = [][]string {
	//{}, // FIXME: Doesn't work
	//{"*"}, // FIXME: Doesn't work
	{"host", "vm"}, // TODO: choose something more meaningful
}

func TestDatastore(t *testing.T) {
	initDriverAcceptanceTest(t)

	d := NewTestDriver(t)
	ds, err := d.FindDatastore(DefaultDatastore)
	if err != nil {
		t.Fatalf("`FindDatatore` can't find default datastore '%v'. Error: %v",
			DefaultDatastore, err)
	}
	for _, params := range infoParams {
		_, err := ds.Info(params...)
		if err != nil {
			t.Errorf("Cannot read datastore properties with parameters %v: %v",
				params, err)
		}
	}
	// TODO: add more checks
}
<commit_msg>Simplify datastore test<commit_after>package testing

import (
	"testing"
	"github.com/vmware/govmomi/vim25/mo"
	"github.com/jetbrains-infra/packer-builder-vsphere/driver"
)

func TestDatastoreAcc(t *testing.T) {
	initDriverAcceptanceTest(t)

	d := NewTestDriver(t)
	ds, err := d.FindDatastore(DefaultDatastore)
	if err != nil {
		t.Fatalf("Cannot find default datastore '%v': %v", DefaultDatastore, err)
	}
	CheckDatastoreName(t, ds, DefaultDatastore)
}
<|endoftext|>


<commit_before>package main

import (
	"context"
	"io"

	hcsschema "github.com/Microsoft/hcsshim/internal/schema2"
	"github.com/Microsoft/hcsshim/internal/shimdiag"
	"github.com/Microsoft/hcsshim/internal/uvm"
)

type taskDiagnostics interface {
	ExecInHost(ctx context.Context, req *shimdiag.ExecProcessRequest) (int, error)
}

func execInUvm(ctx context.Context, vm *uvm.UtilityVM, req *shimdiag.ExecProcessRequest) (int, error) {
	np, err := newNpipeIO(ctx, "", "", req.Stdin, req.Stdout, req.Stderr, req.Terminal)
	if err != nil {
		return 0, err
	}
	defer np.Close()
	wd := req.Workdir
	if wd == "" {
		if vm.OS() == "windows" {
			wd = `c:\`
		} else {
			wd = "/"
		}
	}
	proc, err := vm.ComputeSystem().CreateProcess(&hcsschema.ProcessParameters{
		CommandArgs:      req.Args,
		CreateStdInPipe:  req.Stdin != "",
		CreateStdOutPipe: req.Stdout != "",
		CreateStdErrPipe: req.Stderr != "",
		EmulateConsole:   req.Terminal,
		WorkingDirectory: wd,
	})
	if err != nil {
		return 0, err
	}
	pin, pout, perr, err := proc.Stdio()
	if err != nil {
		return 0, err
	}
	if pin != nil {
		go func() {
			io.Copy(pin, np.Stdin())
			pin.Close()
			proc.CloseStdin()
		}()
	}
	if pout != nil {
		go func() {
			io.Copy(np.Stdout(), pout)
			pout.Close()
		}()
	}
	if perr != nil {
		go func() {
			io.Copy(np.Stderr(), perr)
			perr.Close()
		}()
	}
	err = proc.Wait()
	if err != nil {
		proc.Kill()
		proc.Wait()
	}
	ec, err := proc.ExitCode()
	proc.Close()
	return ec, err
}
<commit_msg>shim: Fix build break<commit_after>package main

import (
	"context"
	"io"

	hcsschema "github.com/Microsoft/hcsshim/internal/schema2"
	"github.com/Microsoft/hcsshim/internal/shimdiag"
	"github.com/Microsoft/hcsshim/internal/uvm"
)

type taskDiagnostics interface {
	ExecInHost(ctx context.Context, req *shimdiag.ExecProcessRequest) (int, error)
}

func execInUvm(ctx context.Context, vm *uvm.UtilityVM, req *shimdiag.ExecProcessRequest) (int, error) {
	np, err := newNpipeIO(ctx, "", "", req.Stdin, req.Stdout, req.Stderr, req.Terminal)
	if err != nil {
		return 0, err
	}
	defer np.Close()
	wd := req.Workdir
	if wd == "" {
		if vm.OS() == "windows" {
			wd = `c:\`
		} else {
			wd = "/"
		}
	}
	proc, err := vm.ComputeSystem().CreateProcess(&hcsschema.ProcessParameters{
		CommandArgs:      req.Args,
		CreateStdInPipe:  req.Stdin != "",
		CreateStdOutPipe: req.Stdout != "",
		CreateStdErrPipe: req.Stderr != "",
		EmulateConsole:   req.Terminal,
		WorkingDirectory: wd,
	})
	if err != nil {
		return 0, err
	}
	pin, pout, perr := proc.Stdio()
	if err != nil {
		return 0, err
	}
	if pin != nil {
		go func() {
			io.Copy(pin, np.Stdin())
			proc.CloseStdin()
		}()
	}
	if pout != nil {
		go func() {
			io.Copy(np.Stdout(), pout)
		}()
	}
	if perr != nil {
		go func() {
			io.Copy(np.Stderr(), perr)
		}()
	}
	err = proc.Wait()
	if err != nil {
		proc.Kill()
		proc.Wait()
	}
	ec, err := proc.ExitCode()
	proc.Close()
	return ec, err
}
<|endoftext|>


<commit_before>package main

import (
	"bytes"
	"encoding/json"
	"fmt"
	"io/ioutil"
	"net/http"
	"net/http/httputil"

	"github.com/zan-xhipe/rest/jsondata"
)

type Response struct {
	Raw     []byte
	display []byte

	resp *http.Response

	Filter       string
	Pretty       bool
	PrettyIndent string

	verbose int
}

func (r *Response) Load(resp *http.Response, s Settings) error {
	r.resp = resp

	r.Pretty = s.Pretty.Bool
	r.PrettyIndent = s.PrettyIndent.String

	switch r.verbose {
	case 1:
		fmt.Println(r.resp.Status)
	case 2, 3:
		// at level 3 display the raw response
		extra := false
		if r.verbose >= 3 {
			extra = true
		}

		dump, err := httputil.DumpResponse(r.resp, extra)
		if err != nil {
			// this is only the verbose logging, so carry on in case of error
			fmt.Println(err)
			break
		}
		fmt.Println(string(dump))
	}

	defer resp.Body.Close()
	body, err := ioutil.ReadAll(resp.Body)
	if err != nil {
		return err
	}
	r.Raw = body
	if err := r.Prepare(); err != nil {
		return err
	}

	return nil
}

func (r Response) String() string {
	return fmt.Sprint(string(r.display))
}

func (r *Response) Prepare() error {
	switch {
	case r.Filter != "":
		if err := r.filter(); err != nil {
			return err
		}
	case r.Pretty:
		var msg json.RawMessage
		err := json.Unmarshal(r.Raw, &msg)
		if err != nil {
			return err
		}

		r.display, err = json.MarshalIndent(msg, "", r.PrettyIndent)
		if err != nil {
			return err
		}
	default:
		r.display = r.Raw
	}

	return nil
}

func (r *Response) filter() error {
	data, err := jsondata.New(string(r.Raw))
	if err != nil {
		return err
	}

	out, err := data.Filter(r.Filter)
	if err != nil {
		return err
	}

	if r.Pretty {
		r.display, err = json.MarshalIndent(out, "", r.PrettyIndent)
		if err != nil {
			return err
		}
		return nil
	}

	r.display, err = json.Marshal(out)
	if err != nil {
		return err
	}

	// the pretty flag removes quotes from results, this was added for
	// filtered results to make them easier to work with, so you can
	// directly put them into a parameter or header without doing your
	// own trimming. This should be changed if a better UI for this
	// behaviour is figured out.
	if r.Pretty {
		r.display = bytes.TrimPrefix(r.display, []byte{'"'})
		r.display = bytes.TrimSuffix(r.display, []byte{'"'})
	}

	return nil
}

func (r Response) ExitCode() int {
	status := r.resp.StatusCode
	// exit non zero if not a 200 response
	if status < 200 || status > 300 {
		// if the exit value gets too high it gets mangled
		// so only keep the hundreds
		return status / 100
	}

	return 0
}
<commit_msg>Fix pretty not removing quotes from filtered output<commit_after>package main

import (
	"bytes"
	"encoding/json"
	"fmt"
	"io/ioutil"
	"net/http"
	"net/http/httputil"

	"github.com/zan-xhipe/rest/jsondata"
)

type Response struct {
	Raw     []byte
	display []byte

	resp *http.Response

	Filter       string
	Pretty       bool
	PrettyIndent string

	verbose int
}

func (r *Response) Load(resp *http.Response, s Settings) error {
	r.resp = resp

	r.Pretty = s.Pretty.Bool
	r.PrettyIndent = s.PrettyIndent.String

	switch r.verbose {
	case 1:
		fmt.Println(r.resp.Status)
	case 2, 3:
		// at level 3 display the raw response
		extra := false
		if r.verbose >= 3 {
			extra = true
		}

		dump, err := httputil.DumpResponse(r.resp, extra)
		if err != nil {
			// this is only the verbose logging, so carry on in case of error
			fmt.Println(err)
			break
		}
		fmt.Println(string(dump))
	}

	defer resp.Body.Close()
	body, err := ioutil.ReadAll(resp.Body)
	if err != nil {
		return err
	}
	r.Raw = body
	if err := r.Prepare(); err != nil {
		return err
	}

	return nil
}

func (r Response) String() string {
	return fmt.Sprint(string(r.display))
}

func (r *Response) Prepare() error {
	switch {
	case r.Filter != "":
		if err := r.filter(); err != nil {
			return err
		}
	case r.Pretty:
		var msg json.RawMessage
		err := json.Unmarshal(r.Raw, &msg)
		if err != nil {
			return err
		}

		r.display, err = json.MarshalIndent(msg, "", r.PrettyIndent)
		if err != nil {
			return err
		}
	default:
		r.display = r.Raw
	}

	return nil
}

func (r *Response) filter() error {
	data, err := jsondata.New(string(r.Raw))
	if err != nil {
		return err
	}

	out, err := data.Filter(r.Filter)
	if err != nil {
		return err
	}

	if r.Pretty {
		r.display, err = json.MarshalIndent(out, "", r.PrettyIndent)
		if err != nil {
			return err
		}

		// the pretty flag removes quotes from results, this was added for
		// filtered results to make them easier to work with, so you can
		// directly put them into a parameter or header without doing your
		// own trimming. This should be changed if a better UI for this
		// behaviour is figured out.
		r.display = bytes.TrimPrefix(r.display, []byte{'"'})
		r.display = bytes.TrimSuffix(r.display, []byte{'"'})

		return nil
	}

	r.display, err = json.Marshal(out)
	if err != nil {
		return err
	}

	return nil
}

func (r Response) ExitCode() int {
	status := r.resp.StatusCode
	// exit non zero if not a 200 response
	if status < 200 || status > 300 {
		// if the exit value gets too high it gets mangled
		// so only keep the hundreds
		return status / 100
	}

	return 0
}
<|endoftext|>


<commit_before>package sdk

/*
   Copyright 2016 Alexander I.Grafov <grafov@gmail.com>
   Copyright 2016-2019 The Grafana SDK authors

   Licensed under the Apache License, Version 2.0 (the "License");
   you may not use this file except in compliance with the License.
   You may obtain a copy of the License at

	   http://www.apache.org/licenses/LICENSE-2.0

   Unless required by applicable law or agreed to in writing, software
   distributed under the License is distributed on an "AS IS" BASIS,
   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
   See the License for the specific language governing permissions and
   limitations under the License.

   ॐ तारे तुत्तारे तुरे स्व
*/

import (
	"bytes"
	"encoding/json"
	"fmt"
	"net/http"
)

// CreateOrg creates a new organization.
// It reflects POST /api/orgs API call.
func (r *Client) CreateOrg(org Org) (StatusMessage, error) {
	var (
		raw  []byte
		resp StatusMessage
		err  error
	)
	if raw, err = json.Marshal(org); err != nil {
		return StatusMessage{}, err
	}
	if raw, _, err = r.post("api/orgs", nil, raw); err != nil {
		return StatusMessage{}, err
	}
	if err = json.Unmarshal(raw, &resp); err != nil {
		return StatusMessage{}, err
	}
	return resp, nil
}

// GetAllOrgs returns all organizations.
// It reflects GET /api/orgs API call.
func (r *Client) GetAllOrgs() ([]Org, error) {
	var (
		raw  []byte
		orgs []Org
		code int
		err  error
	)
	if raw, code, err = r.get("api/orgs", nil); err != nil {
		return orgs, err
	}

	if code != http.StatusOK {
		return orgs, fmt.Errorf("HTTP error %d: returns %s", code, raw)
	}
	dec := json.NewDecoder(bytes.NewReader(raw))
	dec.UseNumber()
	if err := dec.Decode(&orgs); err != nil {
		return orgs, fmt.Errorf("unmarshal orgs: %s\n%s", err, raw)
	}
	return orgs, err
}

// GetActualOrg gets current organization.
// It reflects GET /api/org API call.
func (r *Client) GetActualOrg() (Org, error) {
	var (
		raw  []byte
		org  Org
		code int
		err  error
	)
	if raw, code, err = r.get("api/org", nil); err != nil {
		return org, err
	}
	if code != http.StatusOK {
		return org, fmt.Errorf("HTTP error %d: returns %s", code, raw)
	}
	dec := json.NewDecoder(bytes.NewReader(raw))
	dec.UseNumber()
	if err := dec.Decode(&org); err != nil {
		return org, fmt.Errorf("unmarshal org: %s\n%s", err, raw)
	}
	return org, err
}

// GetOrgById gets organization by organization Id.
// It reflects GET /api/orgs/:orgId API call.
func (r *Client) GetOrgById(oid uint) (Org, error) {
	var (
		raw  []byte
		org  Org
		code int
		err  error
	)
	if raw, code, err = r.get(fmt.Sprintf("api/orgs/%d", oid), nil); err != nil {
		return org, err
	}

	if code != http.StatusOK {
		return org, fmt.Errorf("HTTP error %d: returns %s", code, raw)
	}
	dec := json.NewDecoder(bytes.NewReader(raw))
	dec.UseNumber()
	if err := dec.Decode(&org); err != nil {
		return org, fmt.Errorf("unmarshal org: %s\n%s", err, raw)
	}
	return org, err
}

// GetOrgByOrgName gets organization by organization name.
// It reflects GET /api/orgs/name/:orgName API call.
func (r *Client) GetOrgByOrgName(name string) (Org, error) {
	var (
		raw  []byte
		org  Org
		code int
		err  error
	)
	if raw, code, err = r.get(fmt.Sprintf("api/orgs/name/%s", name), nil); err != nil {
		return org, err
	}

	if code != http.StatusOK {
		return org, fmt.Errorf("HTTP error %d: returns %s", code, raw)
	}
	dec := json.NewDecoder(bytes.NewReader(raw))
	dec.UseNumber()
	if err := dec.Decode(&org); err != nil {
		return org, fmt.Errorf("unmarshal org: %s\n%s", err, raw)
	}
	return org, err
}

// UpdateActualOrg updates current organization.
// It reflects PUT /api/org API call.
func (r *Client) UpdateActualOrg(org Org) (StatusMessage, error) {
	var (
		raw  []byte
		resp StatusMessage
		err  error
	)
	if raw, err = json.Marshal(org); err != nil {
		return StatusMessage{}, err
	}
	if raw, _, err = r.put("api/org", nil, raw); err != nil {
		return StatusMessage{}, err
	}
	if err = json.Unmarshal(raw, &resp); err != nil {
		return StatusMessage{}, err
	}
	return resp, nil
}

// UpdateOrg updates the organization identified by oid.
// It reflects PUT /api/orgs/:orgId API call.
func (r *Client) UpdateOrg(org Org, oid uint) (StatusMessage, error) {
	var (
		raw  []byte
		resp StatusMessage
		err  error
	)
	if raw, err = json.Marshal(org); err != nil {
		return StatusMessage{}, err
	}
	if raw, _, err = r.put(fmt.Sprintf("api/orgs/%d", oid), nil, raw); err != nil {
		return StatusMessage{}, err
	}
	if err = json.Unmarshal(raw, &resp); err != nil {
		return StatusMessage{}, err
	}
	return resp, nil
}

// DeleteOrg deletes the organization identified by the oid.
// Reflects DELETE /api/orgs/:orgId API call.
func (r *Client) DeleteOrg(oid uint) (StatusMessage, error) {
	var (
		raw  []byte
		resp StatusMessage
		err  error
	)
	if raw, _, err = r.delete(fmt.Sprintf("api/orgs/%d", oid)); err != nil {
		return StatusMessage{}, err
	}
	if err = json.Unmarshal(raw, &resp); err != nil {
		return StatusMessage{}, err
	}
	return resp, nil
}

// GetActualOrgUsers get all users within the actual organisation.
// Reflects GET /api/org/users API call.
func (r *Client) GetActualOrgUsers() ([]OrgUser, error) {
	var (
		raw   []byte
		users []OrgUser
		code  int
		err   error
	)
	if raw, code, err = r.get("api/org/users", nil); err != nil {
		return nil, err
	}
	if code != http.StatusOK {
		return nil, fmt.Errorf("HTTP error %d: returns %s", code, raw)
	}
	dec := json.NewDecoder(bytes.NewReader(raw))
	dec.UseNumber()
	if err := dec.Decode(&users); err != nil {
		return nil, fmt.Errorf("unmarshal org: %s\n%s", err, raw)
	}
	return users, err
}

// GetOrgUsers gets the users for the organization specified by oid.
// Reflects GET /api/orgs/:orgId/users API call.
func (r *Client) GetOrgUsers(oid uint) ([]OrgUser, error) {
	var (
		raw   []byte
		users []OrgUser
		code  int
		err   error
	)
	if raw, code, err = r.get(fmt.Sprintf("api/orgs/%d/users", oid), nil); err != nil {
		return nil, err
	}
	if code != http.StatusOK {
		return nil, fmt.Errorf("HTTP error %d: returns %s", code, raw)
	}
	dec := json.NewDecoder(bytes.NewReader(raw))
	dec.UseNumber()
	if err := dec.Decode(&users); err != nil {
		return nil, fmt.Errorf("unmarshal org: %s\n%s", err, raw)
	}
	return users, err
}

// AddActualOrgUser adds a global user to the current organization.
// Reflects POST /api/org/users API call.
func (r *Client) AddActualOrgUser(userRole UserRole) (StatusMessage, error) {
	var (
		raw  []byte
		resp StatusMessage
		err  error
	)
	if raw, err = json.Marshal(userRole); err != nil {
		return StatusMessage{}, err
	}
	if raw, _, err = r.post("api/org/users", nil, raw); err != nil {
		return StatusMessage{}, err
	}
	if err = json.Unmarshal(raw, &resp); err != nil {
		return StatusMessage{}, err
	}
	return resp, nil
}

// UpdateUser updates the existing user.
// Reflects POST /api/org/users/:userId API call.
func (r *Client) UpdateActualOrgUser(user UserRole, uid uint) (StatusMessage, error) {
	var (
		raw  []byte
		resp StatusMessage
		err  error
	)
	if raw, err = json.Marshal(user); err != nil {
		return StatusMessage{}, err
	}
	if raw, _, err = r.post(fmt.Sprintf("api/org/users/%d", uid), nil, raw); err != nil {
		return StatusMessage{}, err
	}
	if err = json.Unmarshal(raw, &resp); err != nil {
		return StatusMessage{}, err
	}
	return resp, nil
}

// DeleteActualOrgUser delete user in actual organization.
// Reflects DELETE /api/org/users/:userId API call.
func (r *Client) DeleteActualOrgUser(uid uint) (StatusMessage, error) {
	var (
		raw   []byte
		reply StatusMessage
		err   error
	)
	if raw, _, err = r.delete(fmt.Sprintf("api/org/users/%d", uid)); err != nil {
		return StatusMessage{}, err
	}
	err = json.Unmarshal(raw, &reply)
	return reply, err
}

// AddUserToOrg add user to organization with oid.
// Reflects POST /api/orgs/:orgId/users API call.
func (r *Client) AddOrgUser(user UserRole, oid uint) (StatusMessage, error) {
	var (
		raw   []byte
		reply StatusMessage
		err   error
	)
	if raw, err = json.Marshal(user); err != nil {
		return StatusMessage{}, err
	}
	if raw, _, err = r.post(fmt.Sprintf("api/orgs/%d/users", oid), nil, raw); err != nil {
		return StatusMessage{}, err
	}
	err = json.Unmarshal(raw, &reply)
	return reply, err
}

// UpdateOrgUser updates the user specified by uid within the organization specified by oid.
// Reflects PATCH /api/orgs/:orgId/users/:userId API call.
func (r *Client) UpdateOrgUser(user UserRole, oid, uid uint) (StatusMessage, error) {
	var (
		raw   []byte
		reply StatusMessage
		err   error
	)
	if raw, err = json.Marshal(user); err != nil {
		return StatusMessage{}, err
	}
	if raw, _, err = r.patch(fmt.Sprintf("api/orgs/%d/users/%d", oid, uid), nil, raw); err != nil {
		return StatusMessage{}, err
	}
	err = json.Unmarshal(raw, &reply)
	return reply, err
}

// DeleteOrgUser deletes the user specified by uid within the organization specified by oid.
// Reflects DELETE /api/orgs/:orgId/users/:userId API call.
func (r *Client) DeleteOrgUser(oid, uid uint) (StatusMessage, error) {
	var (
		raw   []byte
		reply StatusMessage
		err   error
	)
	if raw, _, err = r.delete(fmt.Sprintf("api/orgs/%d/users/%d", oid, uid)); err != nil {
		return StatusMessage{}, err
	}
	err = json.Unmarshal(raw, &reply)
	return reply, err
}

// UpdateActualOrgPreferences updates preferences of the actual organization.
// Reflects PUT /api/org/preferences API call.
func (r *Client) UpdateActualOrgPreferences(prefs Preferences) (StatusMessage, error) {
	var (
		raw  []byte
		resp StatusMessage
		err  error
	)
	if raw, err = json.Marshal(prefs); err != nil {
		return StatusMessage{}, err
	}
	if raw, _, err = r.put("api/org/preferences/", nil, raw); err != nil {
		return StatusMessage{}, err
	}
	if err = json.Unmarshal(raw, &resp); err != nil {
		return StatusMessage{}, err
	}
	return resp, nil
}

// GetActualOrgPreferences gets preferences of the actual organization.
// It reflects GET /api/org/preferences API call.
func (r *Client) GetActualOrgPreferences() (Preferences, error) {
	var (
		raw  []byte
		pref Preferences
		code int
		err  error
	)
	if raw, code, err = r.get("/api/org/preferences", nil); err != nil {
		return pref, err
	}

	if code != http.StatusOK {
		return pref, fmt.Errorf("HTTP error %d: returns %s", code, raw)
	}
	dec := json.NewDecoder(bytes.NewReader(raw))
	dec.UseNumber()
	if err := dec.Decode(&pref); err != nil {
		return pref, fmt.Errorf("unmarshal prefs: %s\n%s", err, raw)
	}
	return pref, err
}
<commit_msg>Fix gofmt error<commit_after>package sdk

/*
   Copyright 2016 Alexander I.Grafov <grafov@gmail.com>
   Copyright 2016-2019 The Grafana SDK authors

   Licensed under the Apache License, Version 2.0 (the "License");
   you may not use this file except in compliance with the License.
   You may obtain a copy of the License at

	   http://www.apache.org/licenses/LICENSE-2.0

   Unless required by applicable law or agreed to in writing, software
   distributed under the License is distributed on an "AS IS" BASIS,
   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
   See the License for the specific language governing permissions and
   limitations under the License.

   ॐ तारे तुत्तारे तुरे स्व
*/

import (
	"bytes"
	"encoding/json"
	"fmt"
	"net/http"
)

// CreateOrg creates a new organization.
// It reflects POST /api/orgs API call.
func (r *Client) CreateOrg(org Org) (StatusMessage, error) {
	var (
		raw  []byte
		resp StatusMessage
		err  error
	)
	if raw, err = json.Marshal(org); err != nil {
		return StatusMessage{}, err
	}
	if raw, _, err = r.post("api/orgs", nil, raw); err != nil {
		return StatusMessage{}, err
	}
	if err = json.Unmarshal(raw, &resp); err != nil {
		return StatusMessage{}, err
	}
	return resp, nil
}

// GetAllOrgs returns all organizations.
// It reflects GET /api/orgs API call.
func (r *Client) GetAllOrgs() ([]Org, error) {
	var (
		raw  []byte
		orgs []Org
		code int
		err  error
	)
	if raw, code, err = r.get("api/orgs", nil); err != nil {
		return orgs, err
	}

	if code != http.StatusOK {
		return orgs, fmt.Errorf("HTTP error %d: returns %s", code, raw)
	}
	dec := json.NewDecoder(bytes.NewReader(raw))
	dec.UseNumber()
	if err := dec.Decode(&orgs); err != nil {
		return orgs, fmt.Errorf("unmarshal orgs: %s\n%s", err, raw)
	}
	return orgs, err
}

// GetActualOrg gets current organization.
// It reflects GET /api/org API call.
func (r *Client) GetActualOrg() (Org, error) {
	var (
		raw  []byte
		org  Org
		code int
		err  error
	)
	if raw, code, err = r.get("api/org", nil); err != nil {
		return org, err
	}
	if code != http.StatusOK {
		return org, fmt.Errorf("HTTP error %d: returns %s", code, raw)
	}
	dec := json.NewDecoder(bytes.NewReader(raw))
	dec.UseNumber()
	if err := dec.Decode(&org); err != nil {
		return org, fmt.Errorf("unmarshal org: %s\n%s", err, raw)
	}
	return org, err
}

// GetOrgById gets organization by organization Id.
// It reflects GET /api/orgs/:orgId API call.
func (r *Client) GetOrgById(oid uint) (Org, error) {
	var (
		raw  []byte
		org  Org
		code int
		err  error
	)
	if raw, code, err = r.get(fmt.Sprintf("api/orgs/%d", oid), nil); err != nil {
		return org, err
	}

	if code != http.StatusOK {
		return org, fmt.Errorf("HTTP error %d: returns %s", code, raw)
	}
	dec := json.NewDecoder(bytes.NewReader(raw))
	dec.UseNumber()
	if err := dec.Decode(&org); err != nil {
		return org, fmt.Errorf("unmarshal org: %s\n%s", err, raw)
	}
	return org, err
}

// GetOrgByOrgName gets organization by organization name.
// It reflects GET /api/orgs/name/:orgName API call.
func (r *Client) GetOrgByOrgName(name string) (Org, error) {
	var (
		raw  []byte
		org  Org
		code int
		err  error
	)
	if raw, code, err = r.get(fmt.Sprintf("api/orgs/name/%s", name), nil); err != nil {
		return org, err
	}

	if code != http.StatusOK {
		return org, fmt.Errorf("HTTP error %d: returns %s", code, raw)
	}
	dec := json.NewDecoder(bytes.NewReader(raw))
	dec.UseNumber()
	if err := dec.Decode(&org); err != nil {
		return org, fmt.Errorf("unmarshal org: %s\n%s", err, raw)
	}
	return org, err
}

// UpdateActualOrg updates current organization.
// It reflects PUT /api/org API call.
func (r *Client) UpdateActualOrg(org Org) (StatusMessage, error) {
	var (
		raw  []byte
		resp StatusMessage
		err  error
	)
	if raw, err = json.Marshal(org); err != nil {
		return StatusMessage{}, err
	}
	if raw, _, err = r.put("api/org", nil, raw); err != nil {
		return StatusMessage{}, err
	}
	if err = json.Unmarshal(raw, &resp); err != nil {
		return StatusMessage{}, err
	}
	return resp, nil
}

// UpdateOrg updates the organization identified by oid.
// It reflects PUT /api/orgs/:orgId API call.
func (r *Client) UpdateOrg(org Org, oid uint) (StatusMessage, error) {
	var (
		raw  []byte
		resp StatusMessage
		err  error
	)
	if raw, err = json.Marshal(org); err != nil {
		return StatusMessage{}, err
	}
	if raw, _, err = r.put(fmt.Sprintf("api/orgs/%d", oid), nil, raw); err != nil {
		return StatusMessage{}, err
	}
	if err = json.Unmarshal(raw, &resp); err != nil {
		return StatusMessage{}, err
	}
	return resp, nil
}

// DeleteOrg deletes the organization identified by the oid.
// Reflects DELETE /api/orgs/:orgId API call.
func (r *Client) DeleteOrg(oid uint) (StatusMessage, error) {
	var (
		raw  []byte
		resp StatusMessage
		err  error
	)
	if raw, _, err = r.delete(fmt.Sprintf("api/orgs/%d", oid)); err != nil {
		return StatusMessage{}, err
	}
	if err = json.Unmarshal(raw, &resp); err != nil {
		return StatusMessage{}, err
	}
	return resp, nil
}

// GetActualOrgUsers get all users within the actual organisation.
// Reflects GET /api/org/users API call.
func (r *Client) GetActualOrgUsers() ([]OrgUser, error) {
	var (
		raw   []byte
		users []OrgUser
		code  int
		err   error
	)
	if raw, code, err = r.get("api/org/users", nil); err != nil {
		return nil, err
	}
	if code != http.StatusOK {
		return nil, fmt.Errorf("HTTP error %d: returns %s", code, raw)
	}
	dec := json.NewDecoder(bytes.NewReader(raw))
	dec.UseNumber()
	if err := dec.Decode(&users); err != nil {
		return nil, fmt.Errorf("unmarshal org: %s\n%s", err, raw)
	}
	return users, err
}

// GetOrgUsers gets the users for the organization specified by oid.
// Reflects GET /api/orgs/:orgId/users API call.
func (r *Client) GetOrgUsers(oid uint) ([]OrgUser, error) {
	var (
		raw   []byte
		users []OrgUser
		code  int
		err   error
	)
	if raw, code, err = r.get(fmt.Sprintf("api/orgs/%d/users", oid), nil); err != nil {
		return nil, err
	}
	if code != http.StatusOK {
		return nil, fmt.Errorf("HTTP error %d: returns %s", code, raw)
	}
	dec := json.NewDecoder(bytes.NewReader(raw))
	dec.UseNumber()
	if err := dec.Decode(&users); err != nil {
		return nil, fmt.Errorf("unmarshal org: %s\n%s", err, raw)
	}
	return users, err
}

// AddActualOrgUser adds a global user to the current organization.
// Reflects POST /api/org/users API call.
func (r *Client) AddActualOrgUser(userRole UserRole) (StatusMessage, error) {
	var (
		raw  []byte
		resp StatusMessage
		err  error
	)
	if raw, err = json.Marshal(userRole); err != nil {
		return StatusMessage{}, err
	}
	if raw, _, err = r.post("api/org/users", nil, raw); err != nil {
		return StatusMessage{}, err
	}
	if err = json.Unmarshal(raw, &resp); err != nil {
		return StatusMessage{}, err
	}
	return resp, nil
}

// UpdateUser updates the existing user.
// Reflects POST /api/org/users/:userId API call.
func (r *Client) UpdateActualOrgUser(user UserRole, uid uint) (StatusMessage, error) {
	var (
		raw  []byte
		resp StatusMessage
		err  error
	)
	if raw, err = json.Marshal(user); err != nil {
		return StatusMessage{}, err
	}
	if raw, _, err = r.post(fmt.Sprintf("api/org/users/%d", uid), nil, raw); err != nil {
		return StatusMessage{}, err
	}
	if err = json.Unmarshal(raw, &resp); err != nil {
		return StatusMessage{}, err
	}
	return resp, nil
}

// DeleteActualOrgUser delete user in actual organization.
// Reflects DELETE /api/org/users/:userId API call.
func (r *Client) DeleteActualOrgUser(uid uint) (StatusMessage, error) {
	var (
		raw   []byte
		reply StatusMessage
		err   error
	)
	if raw, _, err = r.delete(fmt.Sprintf("api/org/users/%d", uid)); err != nil {
		return StatusMessage{}, err
	}
	err = json.Unmarshal(raw, &reply)
	return reply, err
}

// AddUserToOrg add user to organization with oid.
// Reflects POST /api/orgs/:orgId/users API call.
func (r *Client) AddOrgUser(user UserRole, oid uint) (StatusMessage, error) {
	var (
		raw   []byte
		reply StatusMessage
		err   error
	)
	if raw, err = json.Marshal(user); err != nil {
		return StatusMessage{}, err
	}
	if raw, _, err = r.post(fmt.Sprintf("api/orgs/%d/users", oid), nil, raw); err != nil {
		return StatusMessage{}, err
	}
	err = json.Unmarshal(raw, &reply)
	return reply, err
}

// UpdateOrgUser updates the user specified by uid within the organization specified by oid.
// Reflects PATCH /api/orgs/:orgId/users/:userId API call.
func (r *Client) UpdateOrgUser(user UserRole, oid, uid uint) (StatusMessage, error) {
	var (
		raw   []byte
		reply StatusMessage
		err   error
	)
	if raw, err = json.Marshal(user); err != nil {
		return StatusMessage{}, err
	}
	if raw, _, err = r.patch(fmt.Sprintf("api/orgs/%d/users/%d", oid, uid), nil, raw); err != nil {
		return StatusMessage{}, err
	}
	err = json.Unmarshal(raw, &reply)
	return reply, err
}

// DeleteOrgUser deletes the user specified by uid within the organization specified by oid.
// Reflects DELETE /api/orgs/:orgId/users/:userId API call.
func (r *Client) DeleteOrgUser(oid, uid uint) (StatusMessage, error) {
	var (
		raw   []byte
		reply StatusMessage
		err   error
	)
	if raw, _, err = r.delete(fmt.Sprintf("api/orgs/%d/users/%d", oid, uid)); err != nil {
		return StatusMessage{}, err
	}
	err = json.Unmarshal(raw, &reply)
	return reply, err
}

// UpdateActualOrgPreferences updates preferences of the actual organization.
// Reflects PUT /api/org/preferences API call.
func (r *Client) UpdateActualOrgPreferences(prefs Preferences) (StatusMessage, error) {
	var (
		raw  []byte
		resp StatusMessage
		err  error
	)
	if raw, err = json.Marshal(prefs); err != nil {
		return StatusMessage{}, err
	}
	if raw, _, err = r.put("api/org/preferences/", nil, raw); err != nil {
		return StatusMessage{}, err
	}
	if err = json.Unmarshal(raw, &resp); err != nil {
		return StatusMessage{}, err
	}
	return resp, nil
}

// GetActualOrgPreferences gets preferences of the actual organization.
// It reflects GET /api/org/preferences API call.
func (r *Client) GetActualOrgPreferences() (Preferences, error) {
	var (
		raw  []byte
		pref Preferences
		code int
		err  error
	)
	if raw, code, err = r.get("/api/org/preferences", nil); err != nil {
		return pref, err
	}

	if code != http.StatusOK {
		return pref, fmt.Errorf("HTTP error %d: returns %s", code, raw)
	}
	dec := json.NewDecoder(bytes.NewReader(raw))
	dec.UseNumber()
	if err := dec.Decode(&pref); err != nil {
		return pref, fmt.Errorf("unmarshal prefs: %s\n%s", err, raw)
	}
	return pref, err
}
<|endoftext|>


<commit_before>package main

// merges multiple frequency count files into one
// this is could use a merge sort and be smart

import (
	"bufio"
	"compress/gzip"
	"flag"
	"fmt"
	"log"
	"os"
	"sort"
	"strconv"
	"strings"
)

// freqCount is mapping of string->count
type freqCount map[string]int

// make a new counter with some minor preallocation
//  each month has about 2.2M uniques
func newFreqCount() freqCount {
	return make(freqCount, 3000000)
}

// LoadCSV loads in a CSV in form of WORD,COUNT
func loadCSV(counts freqCount, fname string) error {
	fi, err := os.Open(fname)
	if err != nil {
		return err
	}
	defer fi.Close()

	fizip, err := gzip.NewReader(fi)
	if err != nil {
		return err
	}
	defer fizip.Close()

	scanner := bufio.NewScanner(fizip)
	for scanner.Scan() {
		line := scanner.Text()
		parts := strings.SplitN(line, ",", 2)
		if len(parts) != 2 {
			return fmt.Errorf("Got extra junk in line: %q", line)
		}
		c, err := strconv.Atoi(parts[1])
		if err != nil {
			return fmt.Errorf("Number conversion failed: %q", line)
		}

		counts[parts[0]] += c
	}
	return scanner.Err()
}

func main() {
	outfile := flag.String("o", "", "output file name")
	flag.Parse()
	if *outfile == "" {
		log.Fatalf("Must specificy outfile")
	}
	args := flag.Args()
	counts := newFreqCount()
	for _, arg := range args {
		log.Printf("Loading %s", arg)
		err := loadCSV(counts, arg)
		if err != nil {
			log.Fatalf("%s: %s", arg, err)
		}
	}
	fo, err := os.Create(*outfile)
	if err != nil {
		log.Fatalf("OH NO, unable to write: %s", err)
	}
	fout := gzip.NewWriter(fo)

	keys := make([]string, 0, len(counts))
	total := 0
	for k, v := range counts {
		keys = append(keys, k)
		total += v
	}
	sort.Strings(keys)
	for _, k := range keys {
		fout.Write([]byte(fmt.Sprintf("%s,%d\n", k, counts[k])))
	}

	fout.Close()
	fo.Close()
	log.Printf("DONE: wrote %s got %d unique words from %d", *outfile, len(counts), total)
}
<commit_msg>add filtering by min frequency<commit_after>package main

// merges multiple frequency count files into one
// this is could use a merge sort and be smart

import (
	"bufio"
	"compress/gzip"
	"flag"
	"fmt"
	"log"
	"os"
	"sort"
	"strconv"
	"strings"
)

// freqCount is mapping of string->count
type freqCount map[string]int

// make a new counter with some minor preallocation
//  each month has about 2.2M uniques
func newFreqCount() freqCount {
	return make(freqCount, 3000000)
}

// LoadCSV loads in a CSV in form of WORD,COUNT
func loadCSV(counts freqCount, fname string) error {
	fi, err := os.Open(fname)
	if err != nil {
		return err
	}
	defer fi.Close()

	fizip, err := gzip.NewReader(fi)
	if err != nil {
		return err
	}
	defer fizip.Close()

	scanner := bufio.NewScanner(fizip)
	for scanner.Scan() {
		line := scanner.Text()
		parts := strings.SplitN(line, ",", 2)
		if len(parts) != 2 {
			return fmt.Errorf("Got extra junk in line: %q", line)
		}
		c, err := strconv.Atoi(parts[1])
		if err != nil {
			return fmt.Errorf("Number conversion failed: %q", line)
		}

		counts[parts[0]] += c
	}
	return scanner.Err()
}

func main() {
	outfile := flag.String("o", "", "output file name")
	mincount := flag.Int("mincount", 0, "only output if freqcount greater than this, 0 = all")
	flag.Parse()
	if *outfile == "" {
		log.Fatalf("Must specificy outfile")
	}
	args := flag.Args()
	counts := newFreqCount()
	for _, arg := range args {
		log.Printf("Loading %s", arg)
		err := loadCSV(counts, arg)
		if err != nil {
			log.Fatalf("%s: %s", arg, err)
		}
	}
	fo, err := os.Create(*outfile)
	if err != nil {
		log.Fatalf("OH NO, unable to write: %s", err)
	}
	fout := gzip.NewWriter(fo)

	keys := make([]string, 0, len(counts))
	total := 0
	for k, v := range counts {
		keys = append(keys, k)
		total += v
	}
	sort.Strings(keys)
	for _, k := range keys {
		if counts[k] > *mincount {
			fout.Write([]byte(fmt.Sprintf("%s,%d\n", k, counts[k])))
		}
	}

	fout.Close()
	fo.Close()
	log.Printf("DONE: wrote %s got %d unique words from %d", *outfile, len(counts), total)
}
<|endoftext|>


<commit_before>// Copyright ©2015 The Gonum Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

package distuv

import (
	"math"
	"testing"

	"gonum.org/v1/gonum/floats"
	"gonum.org/v1/gonum/floats/scalar"
	"gonum.org/v1/gonum/integrate/quad"
	"gonum.org/v1/gonum/stat"
)

type meaner interface {
	Mean() float64
}

type quantiler interface {
	Quantile(float64) float64
}

type medianer interface {
	quantiler
	Median() float64
}

type varStder interface {
	StdDev() float64
	Variance() float64
}

type entropyer interface {
	LogProber
	Entropy() float64
}

type exKurtosiser interface {
	ExKurtosis() float64
	Mean() float64
}

type skewnesser interface {
	StdDev() float64
	Mean() float64
	Skewness() float64
}

type cumulanter interface {
	Quantiler
	CDF(x float64) float64
	Survival(x float64) float64
}

func generateSamples(x []float64, r Rander) {
	for i := range x {
		x[i] = r.Rand()
	}
}

type probLogprober interface {
	Prob(x float64) float64
	LogProb(x float64) float64
}

type cumulantProber interface {
	cumulanter
	probLogprober
}

func checkMean(t *testing.T, i int, x []float64, m meaner, tol float64) {
	mean := stat.Mean(x, nil)
	if !scalar.EqualWithinAbsOrRel(mean, m.Mean(), tol, tol) {
		t.Errorf("Mean mismatch case %v: want: %v, got: %v", i, mean, m.Mean())
	}
}

func checkMedian(t *testing.T, i int, x []float64, m medianer, tol float64) {
	median := stat.Quantile(0.5, stat.Empirical, x, nil)
	if !scalar.EqualWithinAbsOrRel(median, m.Median(), tol, tol) {
		t.Errorf("Median mismatch case %v: want: %v, got: %v", i, median, m.Median())
	}
}

func checkVarAndStd(t *testing.T, i int, x []float64, v varStder, tol float64) {
	variance := stat.Variance(x, nil)
	if !scalar.EqualWithinAbsOrRel(variance, v.Variance(), tol, tol) {
		t.Errorf("Variance mismatch case %v: want: %v, got: %v", i, variance, v.Variance())
	}
	std := math.Sqrt(variance)
	if !scalar.EqualWithinAbsOrRel(std, v.StdDev(), tol, tol) {
		t.Errorf("StdDev mismatch case %v: want: %v, got: %v", i, std, v.StdDev())
	}
}

func checkEntropy(t *testing.T, i int, x []float64, e entropyer, tol float64) {
	tmp := make([]float64, len(x))
	for i, v := range x {
		tmp[i] = -e.LogProb(v)
	}
	entropy := stat.Mean(tmp, nil)
	if !scalar.EqualWithinAbsOrRel(entropy, e.Entropy(), tol, tol) {
		t.Errorf("Entropy mismatch case %v: want: %v, got: %v", i, entropy, e.Entropy())
	}
}

func checkExKurtosis(t *testing.T, i int, x []float64, e exKurtosiser, tol float64) {
	mean := e.Mean()
	tmp := make([]float64, len(x))
	for i, x := range x {
		tmp[i] = math.Pow(x-mean, 4)
	}
	variance := stat.Variance(x, nil)
	mu4 := stat.Mean(tmp, nil)
	kurtosis := mu4/(variance*variance) - 3
	if !scalar.EqualWithinAbsOrRel(kurtosis, e.ExKurtosis(), tol, tol) {
		t.Errorf("ExKurtosis mismatch case %v: want: %v, got: %v", i, kurtosis, e.ExKurtosis())
	}
}

func checkSkewness(t *testing.T, i int, x []float64, s skewnesser, tol float64) {
	mean := s.Mean()
	std := s.StdDev()
	tmp := make([]float64, len(x))
	for i, v := range x {
		tmp[i] = math.Pow(v-mean, 3)
	}
	mu3 := stat.Mean(tmp, nil)
	skewness := mu3 / math.Pow(std, 3)
	if !scalar.EqualWithinAbsOrRel(skewness, s.Skewness(), tol, tol) {
		t.Errorf("Skewness mismatch case %v: want: %v, got: %v", i, skewness, s.Skewness())
	}
}

func checkQuantileCDFSurvival(t *testing.T, i int, xs []float64, c cumulanter, tol float64) {
	// Quantile, CDF, and survival check.
	for i, p := range []float64{0.1, 0.25, 0.5, 0.75, 0.9} {
		x := c.Quantile(p)
		cdf := c.CDF(x)
		estCDF := stat.CDF(x, stat.Empirical, xs, nil)
		if !scalar.EqualWithinAbsOrRel(cdf, estCDF, tol, tol) {
			t.Errorf("CDF mismatch case %v: want: %v, got: %v", i, estCDF, cdf)
		}
		if !scalar.EqualWithinAbsOrRel(cdf, p, tol, tol) {
			t.Errorf("Quantile/CDF mismatch case %v: want: %v, got: %v", i, p, cdf)
		}
		if math.Abs(1-cdf-c.Survival(x)) > 1e-14 {
			t.Errorf("Survival/CDF mismatch case %v: want: %v, got: %v", i, 1-cdf, c.Survival(x))
		}
	}
	if !panics(func() { c.Quantile(-0.0001) }) {
		t.Errorf("Expected panic with negative argument to Quantile")
	}
	if !panics(func() { c.Quantile(1.0001) }) {
		t.Errorf("Expected panic with Quantile argument above 1")
	}
}

// checkProbContinuous checks that the PDF is consistent with LogPDF
// and integrates to 1 from the lower to upper bound.
func checkProbContinuous(t *testing.T, i int, x []float64, lower float64, upper float64, p probLogprober, tol float64) {
	q := quad.Fixed(p.Prob, lower, upper, 1000000, nil, 0)
	if math.Abs(q-1) > tol {
		t.Errorf("Probability distribution doesn't integrate to 1. Case %v: Got %v", i, q)
	}

	// Check that PDF and LogPDF are consistent.
	for i, v := range x {
		if math.Abs(math.Log(p.Prob(v))-p.LogProb(v)) > 1e-14 {
			t.Errorf("Prob and LogProb mismatch case %v at %v: want %v, got %v", i, v, math.Log(v), p.LogProb(v))
			break
		}
	}
}

// checkProbQuantContinuous checks that the Prob, Rand, and Quantile are all consistent.
// checkProbContinuous only checks that Prob is a valid distribution (integrates
// to 1 and greater than 0). However, this is also true if the PDF of a different
// distribution is used. This checks that PDF is also consistent with the
// CDF implementation and the random samples.
func checkProbQuantContinuous(t *testing.T, i int, xs []float64, c cumulantProber, tol float64) {
	ps := make([]float64, 101)
	floats.Span(ps, 0, 1)

	var xp, x float64
	for i, p := range ps {
		x = c.Quantile(p)
		if p == 0 {
			xp = x
			if floats.Min(xs) < x {
				t.Errorf("Sample of x less than Quantile(0). Case %v.", i)
				break
			}
			continue
		}
		if p == 1 {
			if floats.Max(xs) > x {
				t.Errorf("Sample of x greater than Quantile(1). Case %v.", i)
				break
			}
		}

		// The integral of the PDF between xp and x should be the difference in
		// the quantiles.
		q := quad.Fixed(c.Prob, xp, x, 1000, nil, 0)
		if math.Abs(q-(p-ps[i-1])) > 1e-5 {
			t.Errorf("Integral of PDF doesn't match quantile. Case %v. Want %v, got %v.", i, p-ps[i-1], q)
			break
		}

		pEst := stat.CDF(x, stat.Empirical, xs, nil)
		if math.Abs(pEst-p) > tol {
			t.Errorf("Empirical CDF doesn't match quantile. Case %v.", i)
		}
		xp = x
	}
}

type moder interface {
	Mode() float64
}

func checkMode(t *testing.T, i int, xs []float64, m moder, dx float64, tol float64) {
	rXs := make([]float64, len(xs))
	for j, x := range xs {
		rXs[j] = math.RoundToEven(x/dx) * dx
	}
	want, _ := stat.Mode(rXs, nil)
	got := m.Mode()
	if !scalar.EqualWithinAbs(want, got, tol) {
		t.Errorf("Mode mismatch case %d: want %g, got %v", i, want, got)
	}
}

// checkProbDiscrete confirms that PDF and Rand are consistent for discrete distributions.
func checkProbDiscrete(t *testing.T, i int, xs []float64, p probLogprober, tol float64) {
	// Make a map of all of the unique samples.
	m := make(map[float64]int)
	for _, v := range xs {
		m[v]++
	}
	for x, count := range m {
		prob := float64(count) / float64(len(xs))
		if math.Abs(prob-p.Prob(x)) > tol {
			t.Errorf("PDF mismatch case %v at %v: want %v, got %v", i, x, prob, p.Prob(x))
		}
		if math.Abs(math.Log(p.Prob(x))-p.LogProb(x)) > 1e-14 {
			t.Errorf("Prob and LogProb mismatch case %v at %v: want %v, got %v", i, x, math.Log(x), p.LogProb(x))
		}
	}
}

// testRandLogProb tests that LogProb and Rand give consistent results. This
// can be used when the distribution does not implement CDF.
func testRandLogProbContinuous(t *testing.T, i int, min float64, x []float64, f LogProber, tol float64, bins int) {
	for cdf := 1 / float64(bins); cdf <= 1-1/float64(bins); cdf += 1 / float64(bins) {
		// Get the estimated CDF from the samples
		pt := stat.Quantile(cdf, stat.Empirical, x, nil)

		prob := func(x float64) float64 {
			return math.Exp(f.LogProb(x))
		}
		// Integrate the PDF to find the CDF
		estCDF := quad.Fixed(prob, min, pt, 10000, nil, 0)
		if !scalar.EqualWithinAbsOrRel(cdf, estCDF, tol, tol) {
			t.Errorf("Mismatch between integral of PDF and empirical CDF. Case %v. Want %v, got %v", i, cdf, estCDF)
		}
	}
}

func panics(fun func()) (b bool) {
	defer func() {
		err := recover()
		if err != nil {
			b = true
		}
	}()
	fun()
	return
}
<commit_msg>stat/disuv: make helpers and disambiguate case indexes<commit_after>// Copyright ©2015 The Gonum Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

package distuv

import (
	"math"
	"testing"

	"gonum.org/v1/gonum/floats"
	"gonum.org/v1/gonum/floats/scalar"
	"gonum.org/v1/gonum/integrate/quad"
	"gonum.org/v1/gonum/stat"
)

type meaner interface {
	Mean() float64
}

type quantiler interface {
	Quantile(float64) float64
}

type medianer interface {
	quantiler
	Median() float64
}

type varStder interface {
	StdDev() float64
	Variance() float64
}

type entropyer interface {
	LogProber
	Entropy() float64
}

type exKurtosiser interface {
	ExKurtosis() float64
	Mean() float64
}

type skewnesser interface {
	StdDev() float64
	Mean() float64
	Skewness() float64
}

type cumulanter interface {
	Quantiler
	CDF(x float64) float64
	Survival(x float64) float64
}

func generateSamples(x []float64, r Rander) {
	for i := range x {
		x[i] = r.Rand()
	}
}

type probLogprober interface {
	Prob(x float64) float64
	LogProb(x float64) float64
}

type cumulantProber interface {
	cumulanter
	probLogprober
}

func checkMean(t *testing.T, cas int, x []float64, m meaner, tol float64) {
	t.Helper()
	mean := stat.Mean(x, nil)
	if !scalar.EqualWithinAbsOrRel(mean, m.Mean(), tol, tol) {
		t.Errorf("Mean mismatch case %v: want: %v, got: %v", cas, mean, m.Mean())
	}
}

func checkMedian(t *testing.T, cas int, x []float64, m medianer, tol float64) {
	t.Helper()
	median := stat.Quantile(0.5, stat.Empirical, x, nil)
	if !scalar.EqualWithinAbsOrRel(median, m.Median(), tol, tol) {
		t.Errorf("Median mismatch case %v: want: %v, got: %v", cas, median, m.Median())
	}
}

func checkVarAndStd(t *testing.T, cas int, x []float64, v varStder, tol float64) {
	t.Helper()
	variance := stat.Variance(x, nil)
	if !scalar.EqualWithinAbsOrRel(variance, v.Variance(), tol, tol) {
		t.Errorf("Variance mismatch case %v: want: %v, got: %v", cas, variance, v.Variance())
	}
	std := math.Sqrt(variance)
	if !scalar.EqualWithinAbsOrRel(std, v.StdDev(), tol, tol) {
		t.Errorf("StdDev mismatch case %v: want: %v, got: %v", cas, std, v.StdDev())
	}
}

func checkEntropy(t *testing.T, cas int, x []float64, e entropyer, tol float64) {
	t.Helper()
	tmp := make([]float64, len(x))
	for i, v := range x {
		tmp[i] = -e.LogProb(v)
	}
	entropy := stat.Mean(tmp, nil)
	if !scalar.EqualWithinAbsOrRel(entropy, e.Entropy(), tol, tol) {
		t.Errorf("Entropy mismatch case %v: want: %v, got: %v", cas, entropy, e.Entropy())
	}
}

func checkExKurtosis(t *testing.T, cas int, x []float64, e exKurtosiser, tol float64) {
	t.Helper()
	mean := e.Mean()
	tmp := make([]float64, len(x))
	for i, x := range x {
		tmp[i] = math.Pow(x-mean, 4)
	}
	variance := stat.Variance(x, nil)
	mu4 := stat.Mean(tmp, nil)
	kurtosis := mu4/(variance*variance) - 3
	if !scalar.EqualWithinAbsOrRel(kurtosis, e.ExKurtosis(), tol, tol) {
		t.Errorf("ExKurtosis mismatch case %v: want: %v, got: %v", cas, kurtosis, e.ExKurtosis())
	}
}

func checkSkewness(t *testing.T, cas int, x []float64, s skewnesser, tol float64) {
	t.Helper()
	mean := s.Mean()
	std := s.StdDev()
	tmp := make([]float64, len(x))
	for i, v := range x {
		tmp[i] = math.Pow(v-mean, 3)
	}
	mu3 := stat.Mean(tmp, nil)
	skewness := mu3 / math.Pow(std, 3)
	if !scalar.EqualWithinAbsOrRel(skewness, s.Skewness(), tol, tol) {
		t.Errorf("Skewness mismatch case %v: want: %v, got: %v", cas, skewness, s.Skewness())
	}
}

func checkQuantileCDFSurvival(t *testing.T, cas int, xs []float64, c cumulanter, tol float64) {
	t.Helper()
	// Quantile, CDF, and survival check.
	for _, p := range []float64{0.1, 0.25, 0.5, 0.75, 0.9} {
		x := c.Quantile(p)
		cdf := c.CDF(x)
		estCDF := stat.CDF(x, stat.Empirical, xs, nil)
		if !scalar.EqualWithinAbsOrRel(cdf, estCDF, tol, tol) {
			t.Errorf("CDF mismatch case %v: want: %v, got: %v", cas, estCDF, cdf)
		}
		if !scalar.EqualWithinAbsOrRel(cdf, p, tol, tol) {
			t.Errorf("Quantile/CDF mismatch case %v: want: %v, got: %v", cas, p, cdf)
		}
		if math.Abs(1-cdf-c.Survival(x)) > 1e-14 {
			t.Errorf("Survival/CDF mismatch case %v: want: %v, got: %v", cas, 1-cdf, c.Survival(x))
		}
	}
	if !panics(func() { c.Quantile(-0.0001) }) {
		t.Errorf("Expected panic with negative argument to Quantile")
	}
	if !panics(func() { c.Quantile(1.0001) }) {
		t.Errorf("Expected panic with Quantile argument above 1")
	}
}

// checkProbContinuous checks that the PDF is consistent with LogPDF
// and integrates to 1 from the lower to upper bound.
func checkProbContinuous(t *testing.T, cas int, x []float64, lower float64, upper float64, p probLogprober, tol float64) {
	t.Helper()
	q := quad.Fixed(p.Prob, lower, upper, 1000000, nil, 0)
	if math.Abs(q-1) > tol {
		t.Errorf("Probability distribution doesn't integrate to 1. Case %v: Got %v", cas, q)
	}

	// Check that PDF and LogPDF are consistent.
	for _, v := range x {
		if math.Abs(math.Log(p.Prob(v))-p.LogProb(v)) > 1e-14 {
			t.Errorf("Prob and LogProb mismatch case %v at %v: want %v, got %v", cas, v, math.Log(v), p.LogProb(v))
			break
		}
	}
}

// checkProbQuantContinuous checks that the Prob, Rand, and Quantile are all consistent.
// checkProbContinuous only checks that Prob is a valid distribution (integrates
// to 1 and greater than 0). However, this is also true if the PDF of a different
// distribution is used. This checks that PDF is also consistent with the
// CDF implementation and the random samples.
func checkProbQuantContinuous(t *testing.T, cas int, xs []float64, c cumulantProber, tol float64) {
	t.Helper()
	ps := make([]float64, 101)
	floats.Span(ps, 0, 1)

	var xp, x float64
	for i, p := range ps {
		x = c.Quantile(p)
		if p == 0 {
			xp = x
			if floats.Min(xs) < x {
				t.Errorf("Sample of x less than Quantile(0). Case %v.", cas)
				break
			}
			continue
		}
		if p == 1 {
			if floats.Max(xs) > x {
				t.Errorf("Sample of x greater than Quantile(1). Case %v.", cas)
				break
			}
		}

		// The integral of the PDF between xp and x should be the difference in
		// the quantiles.
		q := quad.Fixed(c.Prob, xp, x, 1000, nil, 0)
		if math.Abs(q-(p-ps[i-1])) > 1e-5 {
			t.Errorf("Integral of PDF doesn't match quantile. Case %v. Want %v, got %v.", cas, p-ps[i-1], q)
			break
		}

		pEst := stat.CDF(x, stat.Empirical, xs, nil)
		if math.Abs(pEst-p) > tol {
			t.Errorf("Empirical CDF doesn't match quantile. Case %v.", cas)
		}
		xp = x
	}
}

type moder interface {
	Mode() float64
}

func checkMode(t *testing.T, cas int, xs []float64, m moder, dx float64, tol float64) {
	t.Helper()
	rXs := make([]float64, len(xs))
	for j, x := range xs {
		rXs[j] = math.RoundToEven(x/dx) * dx
	}
	want, _ := stat.Mode(rXs, nil)
	got := m.Mode()
	if !scalar.EqualWithinAbs(want, got, tol) {
		t.Errorf("Mode mismatch case %d: want %g, got %v", cas, want, got)
	}
}

// checkProbDiscrete confirms that PDF and Rand are consistent for discrete distributions.
func checkProbDiscrete(t *testing.T, cas int, xs []float64, p probLogprober, tol float64) {
	t.Helper()
	// Make a map of all of the unique samples.
	m := make(map[float64]int)
	for _, v := range xs {
		m[v]++
	}
	for x, count := range m {
		prob := float64(count) / float64(len(xs))
		if math.Abs(prob-p.Prob(x)) > tol {
			t.Errorf("PDF mismatch case %v at %v: want %v, got %v", cas, x, prob, p.Prob(x))
		}
		if math.Abs(math.Log(p.Prob(x))-p.LogProb(x)) > 1e-14 {
			t.Errorf("Prob and LogProb mismatch case %v at %v: want %v, got %v", cas, x, math.Log(x), p.LogProb(x))
		}
	}
}

// testRandLogProb tests that LogProb and Rand give consistent results. This
// can be used when the distribution does not implement CDF.
func testRandLogProbContinuous(t *testing.T, cas int, min float64, x []float64, f LogProber, tol float64, bins int) {
	t.Helper()
	for cdf := 1 / float64(bins); cdf <= 1-1/float64(bins); cdf += 1 / float64(bins) {
		// Get the estimated CDF from the samples
		pt := stat.Quantile(cdf, stat.Empirical, x, nil)

		prob := func(x float64) float64 {
			return math.Exp(f.LogProb(x))
		}
		// Integrate the PDF to find the CDF
		estCDF := quad.Fixed(prob, min, pt, 10000, nil, 0)
		if !scalar.EqualWithinAbsOrRel(cdf, estCDF, tol, tol) {
			t.Errorf("Mismatch between integral of PDF and empirical CDF. Case %v. Want %v, got %v", cas, cdf, estCDF)
		}
	}
}

func panics(fun func()) (b bool) {
	defer func() {
		err := recover()
		if err != nil {
			b = true
		}
	}()
	fun()
	return
}
<|endoftext|>


<commit_before>package containerd

import (
	"context"
	"testing"

	"github.com/containerd/containerd/content"
	"github.com/containerd/containerd/content/testsuite"
	"github.com/pkg/errors"
)

func newContentStore(ctx context.Context, root string) (content.Store, func() error, error) {
	client, err := New(address)
	if err != nil {
		return nil, nil, err
	}

	cs := client.ContentStore()

	return cs, func() error {
		statuses, err := cs.ListStatuses(ctx)
		if err != nil {
			return err
		}
		for _, st := range statuses {
			if err := cs.Abort(ctx, st.Ref); err != nil {
				return errors.Wrapf(err, "failed to abort %s", st.Ref)
			}
		}
		return cs.Walk(ctx, func(info content.Info) error {
			return cs.Delete(ctx, info.Digest)
		})

	}, nil
}

func TestContentClient(t *testing.T) {
	if testing.Short() {
		t.Skip()
	}
	testsuite.ContentSuite(t, "ContentClient", newContentStore)
}
<commit_msg>integration: ignore not found error on cleanup<commit_after>package containerd

import (
	"context"
	"testing"

	"github.com/containerd/containerd/content"
	"github.com/containerd/containerd/content/testsuite"
	"github.com/containerd/containerd/errdefs"
	"github.com/pkg/errors"
)

func newContentStore(ctx context.Context, root string) (content.Store, func() error, error) {
	client, err := New(address)
	if err != nil {
		return nil, nil, err
	}

	cs := client.ContentStore()

	return cs, func() error {
		statuses, err := cs.ListStatuses(ctx)
		if err != nil {
			return err
		}
		for _, st := range statuses {
			if err := cs.Abort(ctx, st.Ref); err != nil {
				return errors.Wrapf(err, "failed to abort %s", st.Ref)
			}
		}
		return cs.Walk(ctx, func(info content.Info) error {
			if err := cs.Delete(ctx, info.Digest); err != nil {
				if errdefs.IsNotFound(err) {
					return nil
				}

				return err
			}
			return nil
		})

	}, nil
}

func TestContentClient(t *testing.T) {
	if testing.Short() {
		t.Skip()
	}
	testsuite.ContentSuite(t, "ContentClient", newContentStore)
}
<|endoftext|>