<commit_before>/*
Copyright 2014 Google Inc. All rights reserved.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package e2e

import (
	"fmt"
	"os"
	"os/signal"
	"path"
	"regexp"
	"strings"
	"syscall"

	"github.com/GoogleCloudPlatform/kubernetes/pkg/cloudprovider"
	"github.com/GoogleCloudPlatform/kubernetes/pkg/util"
	"github.com/golang/glog"
	"github.com/onsi/ginkgo"
	"github.com/onsi/ginkgo/config"
	"github.com/onsi/ginkgo/reporters"
	"github.com/onsi/gomega"
)

type testResult bool

type CloudConfig struct {
	ProjectID  string
	Zone       string
	MasterName string

	Provider cloudprovider.Interface
}

func init() {
	// Turn on verbose by default to get spec names
	config.DefaultReporterConfig.Verbose = true

	// Turn on EmitSpecProgress to get spec progress (especially on interrupt)
	config.GinkgoConfig.EmitSpecProgress = true

	// Randomize specs as well as suites
	config.GinkgoConfig.RandomizeAllSpecs = true
}

func (t *testResult) Fail() { *t = false }

// Convert any received TERM signals into INT signals.
// The Ginkgo runner only handles SIGINT, so if Jenkins aborts a run,
// we lose all reporting unless we also handle SIGTERM.
// This function never returns.
func convertSigTermIntoInterrupt() {
	p, err := os.FindProcess(os.Getpid())
	if err != nil {
		glog.Fatalf("Failed looking up own process: %s", err.Error())
	}
	c := make(chan os.Signal, 1)
	signal.Notify(c, syscall.SIGTERM)
	for {
		<-c
		p.Signal(os.Interrupt)
	}
}

// Run each Go end-to-end-test. This function assumes the
// creation of a test cluster.
func RunE2ETests(context *TestContextType, orderseed int64, times int, reportDir string, testList []string) {
	testContext = *context
	util.ReallyCrash = true
	util.InitLogs()
	defer util.FlushLogs()
	go convertSigTermIntoInterrupt()

	if len(testList) != 0 {
		if config.GinkgoConfig.FocusString != "" || config.GinkgoConfig.SkipString != "" {
			glog.Fatal("Either specify --test/-t or --ginkgo.focus/--ginkgo.skip but not both.")
		}
		var testRegexps []string
		for _, t := range testList {
			testRegexps = append(testRegexps, regexp.QuoteMeta(t))
		}
		config.GinkgoConfig.FocusString = `\b(` + strings.Join(testRegexps, "|") + `)\b`
	}

	// Disable density test unless it's explicitly requested.
	if config.GinkgoConfig.FocusString == "" && config.GinkgoConfig.SkipString == "" {
		config.GinkgoConfig.SkipString = "Skipped"
	}

	// TODO: Make orderseed work again.
	var passed testResult = true
	gomega.RegisterFailHandler(ginkgo.Fail)
	// Run the existing tests with output to console + JUnit for Jenkins
	for i := 0; i < times && passed; i++ {
		var r []ginkgo.Reporter
		if reportDir != "" {
			r = append(r, reporters.NewJUnitReporter(path.Join(reportDir, fmt.Sprintf("junit_%d.xml", i+1))))
		}
		ginkgo.RunSpecsWithDefaultAndCustomReporters(&passed, fmt.Sprintf("Kubernetes e2e Suite run %d of %d", i+1, times), r)
	}

	if !passed {
		glog.Fatalf("At least one test failed")
	} else {
		glog.Infof("All tests pass")
	}
}
<commit_msg>Revert hack to handle SIGTERM in e2e tests.<commit_after>/*
Copyright 2014 Google Inc. All rights reserved.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package e2e

import (
	"fmt"
	"path"
	"regexp"
	"strings"

	"github.com/GoogleCloudPlatform/kubernetes/pkg/cloudprovider"
	"github.com/GoogleCloudPlatform/kubernetes/pkg/util"
	"github.com/golang/glog"
	"github.com/onsi/ginkgo"
	"github.com/onsi/ginkgo/config"
	"github.com/onsi/ginkgo/reporters"
	"github.com/onsi/gomega"
)

type testResult bool

type CloudConfig struct {
	ProjectID  string
	Zone       string
	MasterName string

	Provider cloudprovider.Interface
}

func init() {
	// Turn on verbose by default to get spec names
	config.DefaultReporterConfig.Verbose = true

	// Turn on EmitSpecProgress to get spec progress (especially on interrupt)
	config.GinkgoConfig.EmitSpecProgress = true

	// Randomize specs as well as suites
	config.GinkgoConfig.RandomizeAllSpecs = true
}

func (t *testResult) Fail() { *t = false }

// Run each Go end-to-end-test. This function assumes the
// creation of a test cluster.
func RunE2ETests(context *TestContextType, orderseed int64, times int, reportDir string, testList []string) {
	testContext = *context
	util.ReallyCrash = true
	util.InitLogs()
	defer util.FlushLogs()

	if len(testList) != 0 {
		if config.GinkgoConfig.FocusString != "" || config.GinkgoConfig.SkipString != "" {
			glog.Fatal("Either specify --test/-t or --ginkgo.focus/--ginkgo.skip but not both.")
		}
		var testRegexps []string
		for _, t := range testList {
			testRegexps = append(testRegexps, regexp.QuoteMeta(t))
		}
		config.GinkgoConfig.FocusString = `\b(` + strings.Join(testRegexps, "|") + `)\b`
	}

	// Disable density test unless it's explicitly requested.
	if config.GinkgoConfig.FocusString == "" && config.GinkgoConfig.SkipString == "" {
		config.GinkgoConfig.SkipString = "Skipped"
	}

	// TODO: Make orderseed work again.
	var passed testResult = true
	gomega.RegisterFailHandler(ginkgo.Fail)
	// Run the existing tests with output to console + JUnit for Jenkins
	for i := 0; i < times && passed; i++ {
		var r []ginkgo.Reporter
		if reportDir != "" {
			r = append(r, reporters.NewJUnitReporter(path.Join(reportDir, fmt.Sprintf("junit_%d.xml", i+1))))
		}
		ginkgo.RunSpecsWithDefaultAndCustomReporters(&passed, fmt.Sprintf("Kubernetes e2e Suite run %d of %d", i+1, times), r)
	}

	if !passed {
		glog.Fatalf("At least one test failed")
	} else {
		glog.Infof("All tests pass")
	}
}
<|endoftext|>


<commit_before>/*
Copyright 2017, 2018 Ankyra

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package state

import (
	"time"
)

type StatusCode string

const (
	// Initial state
	Empty StatusCode = "empty"

	// Configured, but not running
	Pending = "pending"

	// Build and deployment phase
	RunningPreStep  = "running_pre_step"
	RunningMainStep = "running_main_step"
	RunningPostStep = "running_post_step"

	// Build and deployment failure
	Failure = "failure"

	// Build, deployment, test and smoke success
	OK = "ok"

	// Test and smoke phase
	TestPending     = "test_pending"
	RunningTestStep = "running_test_step"

	// Test and smoke failure
	TestFailure = "test_failure"

	// Destroy phase
	DestroyPending          = "destroy_pending"
	DestroyAndDeletePending = "destroy_and_delete_pending"
	RunningPreDestroyStep   = "running_pre_destroy_step"
	RunningMainDestroyStep  = "running_main_destroy_step"
	RunningPostDestroyStep  = "running_post_destroy_step"

	// Destroy failure
	DestroyFailure = "destroy_failure"
)

// Can you go from one state to another?
var StatusTransitions = map[StatusCode][]StatusCode{
	Empty:          []StatusCode{RunningPreStep, Pending},
	Pending:        []StatusCode{RunningPreStep},
	OK:             []StatusCode{RunningPreStep, RunningTestStep, DestroyPending, DestroyAndDeletePending, TestPending, Pending, RunningPreDestroyStep},
	Failure:        []StatusCode{RunningPreStep, Pending, DestroyPending, DestroyAndDeletePending, RunningPreDestroyStep},
	TestFailure:    []StatusCode{RunningTestStep, RunningPreStep, DestroyPending, DestroyAndDeletePending, TestPending, Pending, RunningPreDestroyStep},
	DestroyFailure: []StatusCode{RunningPreDestroyStep, DestroyPending, DestroyAndDeletePending, Pending, RunningPreStep},

	RunningPreStep:  []StatusCode{RunningMainStep, Failure},
	RunningMainStep: []StatusCode{RunningPostStep, Failure},
	RunningPostStep: []StatusCode{RunningTestStep, OK, Failure},

	RunningTestStep: []StatusCode{OK, TestFailure},

	DestroyPending:          []StatusCode{RunningPreDestroyStep},
	DestroyAndDeletePending: []StatusCode{RunningPreDestroyStep},
	RunningPreDestroyStep:   []StatusCode{RunningMainDestroyStep, DestroyFailure},
	RunningMainDestroyStep:  []StatusCode{RunningPostDestroyStep, DestroyFailure},
	RunningPostDestroyStep:  []StatusCode{Empty, DestroyFailure},
}

// Can you go from s1 -> s2?
func StatusTransitionAllowed(s1, s2 StatusCode) bool {
	transitions := StatusTransitions[s1]
	for _, allowed := range transitions {
		if allowed == s2 {
			return true
		}
	}
	return false
}

var ErrorStatuses = map[StatusCode]bool{
	Failure:        true,
	TestFailure:    true,
	DestroyFailure: true,
}
var OKStatuses = map[StatusCode]bool{
	Empty:   true,
	Pending: true,
	OK:      true,
}
var RunningStatus = map[StatusCode]bool{}

type Status struct {
	Code       StatusCode `json:"status"`
	UpdatedAt  time.Time  `json:"updated_at,omitempty"`
	UpdatedBy  string     `json:"updated_by,omitempty"`
	Data       string     `json:"data,omitempty"`
	TryAgainAt *time.Time `json:"try_again_at,omitempty"`
	Tried      int        `json:"tried,omitempty"`
}

func NewStatus(code StatusCode) *Status {
	return &Status{
		Code:      code,
		UpdatedAt: time.Now(),
	}
}

func (s *Status) IsError() bool {
	_, found := ErrorStatuses[s.Code]
	return found
}

func (s *Status) IsOK() bool {
	_, found := OKStatuses[s.Code]
	return found
}

func (s *Status) IsRunning() bool {
	return !(s.IsError() || s.IsOK())
}

func (s *Status) IsOneOf(codes ...StatusCode) bool {
	for _, c := range codes {
		if s.Code == c {
			return true
		}
	}
	return false
}
<commit_msg>Don't allow to transition from 'empty' status directly to 'pre_running_step', go through 'pending' first.<commit_after>/*
Copyright 2017, 2018 Ankyra

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package state

import (
	"time"
)

type StatusCode string

const (
	// Initial state
	Empty StatusCode = "empty"

	// Configured, but not running
	Pending = "pending"

	// Build and deployment phase
	RunningPreStep  = "running_pre_step"
	RunningMainStep = "running_main_step"
	RunningPostStep = "running_post_step"

	// Build and deployment failure
	Failure = "failure"

	// Build, deployment, test and smoke success
	OK = "ok"

	// Test and smoke phase
	TestPending     = "test_pending"
	RunningTestStep = "running_test_step"

	// Test and smoke failure
	TestFailure = "test_failure"

	// Destroy phase
	DestroyPending          = "destroy_pending"
	DestroyAndDeletePending = "destroy_and_delete_pending"
	RunningPreDestroyStep   = "running_pre_destroy_step"
	RunningMainDestroyStep  = "running_main_destroy_step"
	RunningPostDestroyStep  = "running_post_destroy_step"

	// Destroy failure
	DestroyFailure = "destroy_failure"
)

// Can you go from one state to another?
var StatusTransitions = map[StatusCode][]StatusCode{
	Empty:          []StatusCode{Pending},
	Pending:        []StatusCode{RunningPreStep},
	OK:             []StatusCode{RunningPreStep, RunningTestStep, DestroyPending, DestroyAndDeletePending, TestPending, Pending, RunningPreDestroyStep},
	Failure:        []StatusCode{RunningPreStep, Pending, DestroyPending, DestroyAndDeletePending, RunningPreDestroyStep},
	TestFailure:    []StatusCode{RunningTestStep, RunningPreStep, DestroyPending, DestroyAndDeletePending, TestPending, Pending, RunningPreDestroyStep},
	DestroyFailure: []StatusCode{RunningPreDestroyStep, DestroyPending, DestroyAndDeletePending, Pending, RunningPreStep},

	RunningPreStep:  []StatusCode{RunningMainStep, Failure},
	RunningMainStep: []StatusCode{RunningPostStep, Failure},
	RunningPostStep: []StatusCode{RunningTestStep, OK, Failure},

	RunningTestStep: []StatusCode{OK, TestFailure},

	DestroyPending:          []StatusCode{RunningPreDestroyStep},
	DestroyAndDeletePending: []StatusCode{RunningPreDestroyStep},
	RunningPreDestroyStep:   []StatusCode{RunningMainDestroyStep, DestroyFailure},
	RunningMainDestroyStep:  []StatusCode{RunningPostDestroyStep, DestroyFailure},
	RunningPostDestroyStep:  []StatusCode{Empty, DestroyFailure},
}

// Can you go from s1 -> s2?
func StatusTransitionAllowed(s1, s2 StatusCode) bool {
	transitions := StatusTransitions[s1]
	for _, allowed := range transitions {
		if allowed == s2 {
			return true
		}
	}
	return false
}

var ErrorStatuses = map[StatusCode]bool{
	Failure:        true,
	TestFailure:    true,
	DestroyFailure: true,
}
var OKStatuses = map[StatusCode]bool{
	Empty:   true,
	Pending: true,
	OK:      true,
}
var RunningStatus = map[StatusCode]bool{}

type Status struct {
	Code       StatusCode `json:"status"`
	UpdatedAt  time.Time  `json:"updated_at,omitempty"`
	UpdatedBy  string     `json:"updated_by,omitempty"`
	Data       string     `json:"data,omitempty"`
	TryAgainAt *time.Time `json:"try_again_at,omitempty"`
	Tried      int        `json:"tried,omitempty"`
}

func NewStatus(code StatusCode) *Status {
	return &Status{
		Code:      code,
		UpdatedAt: time.Now(),
	}
}

func (s *Status) IsError() bool {
	_, found := ErrorStatuses[s.Code]
	return found
}

func (s *Status) IsOK() bool {
	_, found := OKStatuses[s.Code]
	return found
}

func (s *Status) IsRunning() bool {
	return !(s.IsError() || s.IsOK())
}

func (s *Status) IsOneOf(codes ...StatusCode) bool {
	for _, c := range codes {
		if s.Code == c {
			return true
		}
	}
	return false
}
<|endoftext|>


<commit_before>// Copyright 2016-2019 Google LLC. All Rights Reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package fetch

import (
	"encoding/json"
	"fmt"
	"hash/crc64"
	"io"
	"io/ioutil"
	"math"
	"math/rand"
	"net/http"
	"net/url"
	"os"
	"path/filepath"
	"strings"
	"time"

	"github.com/googlecodelabs/tools/claat/fetch/drive/auth"
	"github.com/googlecodelabs/tools/claat/parser"
	"github.com/googlecodelabs/tools/claat/types"
	"github.com/googlecodelabs/tools/claat/util"
)

const (
	// supported codelab source types must be registered parsers
	// TODO: define these in claat/parser/..., e.g. in parser/gdoc
	// alternate TODO: make this an iota-based enum?
	SrcInvalid   srcType = ""
	SrcGoogleDoc srcType = "gdoc" // Google Docs doc
	SrcMarkdown  srcType = "md"   // Markdown text

	// driveAPI is a base URL for Drive API
	driveAPI = "https://www.googleapis.com/drive/v3"
)

// TODO: create an enum for use with "nometa" for readability's sake

// srcType is codelab source type
type srcType string

// resource is a codelab resource, loaded from local file
// or fetched from remote location.
type resource struct {
	typ  srcType       // source type
	body io.ReadCloser // resource body
	mod  time.Time     // last update of content
}

// codelab wraps types.Codelab, while adding source type
// and modified timestamp fields.
type codelab struct {
	*types.Codelab
	Typ srcType   //  source type
	Mod time.Time // last modified timestamp
}

type Fetcher struct {
	authHelper   *auth.Helper
	authToken    string
	crcTable     *crc64.Table
	passMetadata map[string]bool
}

func NewFetcher(at string, pm map[string]bool, rt http.RoundTripper) (*Fetcher, error) {
	h, err := auth.NewHelper(at, auth.ProviderGoogle, rt)
	if err != nil {
		return nil, err
	}

	return &Fetcher{
		authHelper:   h,
		authToken:    at,
		crcTable:     crc64.MakeTable(crc64.ECMA),
		passMetadata: pm,
	}, nil
}

// SlurpCodelab retrieves and parses codelab source.
// It takes the source, plus an auth token and a set of extra metadata to pass along.
// It returns parsed codelab and its source type.
//
// The function will also fetch and parse fragments included
// with types.ImportNode.
func (f *Fetcher) SlurpCodelab(src string) (*codelab, error) {
	res, err := f.fetch(src)
	if err != nil {
		return nil, err
	}
	defer res.body.Close()

	opts := *parser.NewOptions()
	opts.PassMetadata = f.passMetadata

	clab, err := parser.Parse(string(res.typ), res.body, opts)
	if err != nil {
		return nil, err
	}

	// fetch imports and parse them as fragments
	var imports []*types.ImportNode
	for _, st := range clab.Steps {
		imports = append(imports, types.ImportNodes(st.Content.Nodes)...)
	}
	ch := make(chan error, len(imports))
	defer close(ch)
	for _, imp := range imports {
		go func(n *types.ImportNode) {
			frag, err := f.slurpFragment(n.URL)
			if err != nil {
				ch <- fmt.Errorf("%s: %v", n.URL, err)
				return
			}
			n.Content.Nodes = frag
			ch <- nil
		}(imp)
	}
	for range imports {
		if err := <-ch; err != nil {
			return nil, err
		}
	}

	v := &codelab{
		Codelab: clab,
		Typ:     res.typ,
		Mod:     res.mod,
	}
	return v, nil
}

func (f *Fetcher) SlurpImages(src, dir string, steps []*types.Step) (map[string]string, error) {
	// make sure img dir exists
	if err := os.MkdirAll(dir, 0755); err != nil {
		return nil, err
	}

	type res struct {
		url, file string
		err       error
	}

	ch := make(chan *res, 100)
	defer close(ch)
	var count int
	for _, st := range steps {
		nodes := types.ImageNodes(st.Content.Nodes)
		count += len(nodes)
		for _, n := range nodes {
			go func(n *types.ImageNode) {
				url := n.Src
				file, err := f.slurpBytes(src, dir, url)
				if err == nil {
					n.Src = filepath.Join(util.ImgDirname, file)
				}
				ch <- &res{url, file, err}
			}(n)
		}
	}

	imap := make(map[string]string, count)
	var err error
	for i := 0; i < count; i++ {
		r := <-ch
		imap[r.file] = r.url
		if r.err != nil && err == nil {
			// record first error
			err = fmt.Errorf("%s => %s: %v", r.url, r.file, r.err)
		}
	}

	return imap, err
}

func (f *Fetcher) slurpBytes(codelabSrc, dir, imgURL string) (string, error) {
	// images can be local in Markdown cases or remote.
	// Only proceed a simple copy on local reference.
	var b []byte
	var ext string
	u, err := url.Parse(imgURL)
	if err != nil {
		return "", err
	}

	// If the codelab source is being downloaded from the network, then we should interpret
	// the image URL in the same way.
	srcUrl, err := url.Parse(codelabSrc)
	if err == nil && srcUrl.Host != "" {
		u = srcUrl.ResolveReference(u)
	}

	if u.Host == "" {
		if imgURL, err = restrictPathToParent(imgURL, filepath.Dir(codelabSrc)); err != nil {
			return "", err
		}
		b, err = ioutil.ReadFile(imgURL)
		ext = filepath.Ext(imgURL)
	} else {
		b, err = f.slurpRemoteBytes(u.String(), 5)
		if string(b[6:10]) == "JFIF" {
			ext = ".jpeg"
		} else if string(b[0:3]) == "GIF" {
			ext = ".gif"
		} else {
			ext = ".png"
		}
	}
	if err != nil {
		return "", err
	}

	crc := crc64.Checksum(b, f.crcTable)
	file := fmt.Sprintf("%x%s", crc, ext)
	dst := filepath.Join(dir, file)
	return file, ioutil.WriteFile(dst, b, 0644)
}

func (f *Fetcher) slurpFragment(url string) ([]types.Node, error) {
	res, err := f.fetchRemote(url, true)
	if err != nil {
		return nil, err
	}
	defer res.body.Close()
	return parser.ParseFragment(string(res.typ), res.body)
}

// fetch retrieves codelab doc either from local disk
// or a remote location.
// The caller is responsible for closing returned stream.
func (f *Fetcher) fetch(name string) (*resource, error) {
	fi, err := os.Stat(name)
	if os.IsNotExist(err) {
		return f.fetchRemote(name, false)
	}
	r, err := os.Open(name)
	if err != nil {
		return nil, err
	}
	return &resource{
		body: r,
		typ:  SrcMarkdown,
		mod:  fi.ModTime(),
	}, nil
}

// fetchRemote retrieves resource r from the network.
//
// If urlStr is not a URL, i.e. does not have the host part, it is considered to be
// a Google Doc ID and fetched accordingly. Otherwise, a simple GET request
// is used to retrieve the contents.
//
// The caller is responsible for closing returned stream.
// If nometa is true, resource.mod may have zero value.
func (f *Fetcher) fetchRemote(urlStr string, nometa bool) (*resource, error) {
	u, err := url.Parse(urlStr)
	if err != nil {
		return nil, err
	}
	if u.Host == "" || u.Host == "docs.google.com" {
		return f.fetchDriveFile(urlStr, nometa)
	}
	return f.fetchRemoteFile(urlStr)
}

// fetchRemoteFile retrieves codelab resource from url.
// It is a special case of fetchRemote function.
func (f *Fetcher) fetchRemoteFile(url string) (*resource, error) {
	res, err := retryGet(f.authHelper.DriveClient(), url, 3)
	if err != nil {
		return nil, err
	}
	t, err := http.ParseTime(res.Header.Get("last-modified"))
	if err != nil {
		t = time.Now()
	}
	return &resource{
		body: res.Body,
		mod:  t,
		typ:  SrcMarkdown,
	}, nil
}

// fetchDriveFile uses Drive API to retrieve HTML representation of a Google Doc.
// See https://developers.google.com/drive/web/manage-downloads#downloading_google_documents
// for more details.
//
// If nometa is true, resource.mod will have zero value.
func (f *Fetcher) fetchDriveFile(id string, nometa bool) (*resource, error) {
	id = gdocID(id)
	exportURL := gdocExportURL(id)

	if nometa {
		res, err := retryGet(f.authHelper.DriveClient(), exportURL, 7)
		if err != nil {
			return nil, err
		}
		return &resource{body: res.Body, typ: SrcGoogleDoc}, nil
	}

	q := url.Values{
		"fields":             {"id,mimeType,modifiedTime"},
		"supportsTeamDrives": {"true"},
	}
	u := fmt.Sprintf("%s/files/%s?%s", driveAPI, id, q.Encode())
	res, err := retryGet(f.authHelper.DriveClient(), u, 7)
	if err != nil {
		return nil, err
	}
	defer res.Body.Close()
	meta := &struct {
		ID       string    `json:"id"`
		MimeType string    `json:"mimeType"`
		Modified time.Time `json:"modifiedTime"`
	}{}
	if err := json.NewDecoder(res.Body).Decode(meta); err != nil {
		return nil, err
	}
	if meta.MimeType != "application/vnd.google-apps.document" {
		return nil, fmt.Errorf("%s: invalid mime type: %s", id, meta.MimeType)
	}

	if res, err = retryGet(f.authHelper.DriveClient(), exportURL, 7); err != nil {
		return nil, err
	}
	return &resource{
		body: res.Body,
		mod:  meta.Modified,
		typ:  SrcGoogleDoc,
	}, nil
}

func (f *Fetcher) slurpRemoteBytes(url string, n int) ([]byte, error) {
	res, err := retryGet(f.authHelper.DriveClient(), url, n)
	if err != nil {
		return nil, err
	}
	defer res.Body.Close()
	return ioutil.ReadAll(res.Body)
}

// retryGet tries to GET specified url up to n times.
// Attempts are spaced out with exponential backoff.
// Default client will be used if not provided.
func retryGet(client *http.Client, url string, n int) (*http.Response, error) {
	if client == nil {
		client = http.DefaultClient
	}
	for i := 0; i <= n; i++ {
		if i > 0 {
			t := time.Duration((math.Pow(2, float64(i)) + rand.Float64()) * float64(time.Second))
			time.Sleep(t)
		}
		res, err := client.Get(url)
		// return early with a good response
		// the rest is error handling
		if err == nil && res.StatusCode == http.StatusOK {
			return res, nil
		}

		// sometimes Drive API wouldn't even start a response,
		// we get net/http: TLS handshake timeout instead:
		// consider this a temporary failure and retry again
		if err != nil {
			continue
		}
		// otherwise, decode error response and check for "rate limit"
		defer res.Body.Close()
		var erres struct {
			Error struct {
				Errors []struct{ Reason string }
			}
		}
		b, _ := ioutil.ReadAll(res.Body)
		json.Unmarshal(b, &erres)
		var rateLimit bool
		for _, e := range erres.Error.Errors {
			if e.Reason == "rateLimitExceeded" || e.Reason == "userRateLimitExceeded" {
				rateLimit = true
				break
			}
		}
		// this is neither a rate limit error, nor a server error:
		// retrying is useless
		if !rateLimit && res.StatusCode < http.StatusInternalServerError {
			return nil, fmt.Errorf("fetch %s: %s; %s", url, res.Status, b)
		}
	}
	return nil, fmt.Errorf("%s: failed after %d retries", url, n)
}

func gdocID(url string) string {
	const s = "/document/d/"
	if i := strings.Index(url, s); i >= 0 {
		url = url[i+len(s):]
	}
	if i := strings.IndexRune(url, '/'); i > 0 {
		url = url[:i]
	}
	return url
}

func gdocExportURL(id string) string {
	return fmt.Sprintf("%s/files/%s/export?mimeType=text/html", driveAPI, id)
}

// restrictPathToParent will ensure that assetPath is in parent.
// It will thus return an absolute path to the asset.
func restrictPathToParent(assetPath, parent string) (string, error) {
	parent, err := filepath.Abs(parent)
	if err != nil {
		return "", err
	}
	if !filepath.IsAbs(assetPath) {
		assetPath = filepath.Join(parent, assetPath)
	}
	if !strings.HasPrefix(assetPath, parent) {
		return "", fmt.Errorf("%s isn't a subdirectory of %s", assetPath, parent)
	}
	return assetPath, nil
}
<commit_msg>Improve error reporting in SlurpImages.<commit_after>// Copyright 2016-2019 Google LLC. All Rights Reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package fetch

import (
	"encoding/json"
	"errors"
	"fmt"
	"hash/crc64"
	"io"
	"io/ioutil"
	"math"
	"math/rand"
	"net/http"
	"net/url"
	"os"
	"path/filepath"
	"strings"
	"time"

	"github.com/googlecodelabs/tools/claat/fetch/drive/auth"
	"github.com/googlecodelabs/tools/claat/parser"
	"github.com/googlecodelabs/tools/claat/types"
	"github.com/googlecodelabs/tools/claat/util"
)

const (
	// supported codelab source types must be registered parsers
	// TODO: define these in claat/parser/..., e.g. in parser/gdoc
	// alternate TODO: make this an iota-based enum?
	SrcInvalid   srcType = ""
	SrcGoogleDoc srcType = "gdoc" // Google Docs doc
	SrcMarkdown  srcType = "md"   // Markdown text

	// driveAPI is a base URL for Drive API
	driveAPI = "https://www.googleapis.com/drive/v3"
)

// TODO: create an enum for use with "nometa" for readability's sake

// srcType is codelab source type
type srcType string

// resource is a codelab resource, loaded from local file
// or fetched from remote location.
type resource struct {
	typ  srcType       // source type
	body io.ReadCloser // resource body
	mod  time.Time     // last update of content
}

// codelab wraps types.Codelab, while adding source type
// and modified timestamp fields.
type codelab struct {
	*types.Codelab
	Typ srcType   //  source type
	Mod time.Time // last modified timestamp
}

type Fetcher struct {
	authHelper   *auth.Helper
	authToken    string
	crcTable     *crc64.Table
	passMetadata map[string]bool
}

func NewFetcher(at string, pm map[string]bool, rt http.RoundTripper) (*Fetcher, error) {
	h, err := auth.NewHelper(at, auth.ProviderGoogle, rt)
	if err != nil {
		return nil, err
	}

	return &Fetcher{
		authHelper:   h,
		authToken:    at,
		crcTable:     crc64.MakeTable(crc64.ECMA),
		passMetadata: pm,
	}, nil
}

// SlurpCodelab retrieves and parses codelab source.
// It takes the source, plus an auth token and a set of extra metadata to pass along.
// It returns parsed codelab and its source type.
//
// The function will also fetch and parse fragments included
// with types.ImportNode.
func (f *Fetcher) SlurpCodelab(src string) (*codelab, error) {
	res, err := f.fetch(src)
	if err != nil {
		return nil, err
	}
	defer res.body.Close()

	opts := *parser.NewOptions()
	opts.PassMetadata = f.passMetadata

	clab, err := parser.Parse(string(res.typ), res.body, opts)
	if err != nil {
		return nil, err
	}

	// fetch imports and parse them as fragments
	var imports []*types.ImportNode
	for _, st := range clab.Steps {
		imports = append(imports, types.ImportNodes(st.Content.Nodes)...)
	}
	ch := make(chan error, len(imports))
	defer close(ch)
	for _, imp := range imports {
		go func(n *types.ImportNode) {
			frag, err := f.slurpFragment(n.URL)
			if err != nil {
				ch <- fmt.Errorf("%s: %v", n.URL, err)
				return
			}
			n.Content.Nodes = frag
			ch <- nil
		}(imp)
	}
	for range imports {
		if err := <-ch; err != nil {
			return nil, err
		}
	}

	v := &codelab{
		Codelab: clab,
		Typ:     res.typ,
		Mod:     res.mod,
	}
	return v, nil
}

func (f *Fetcher) SlurpImages(src, dir string, steps []*types.Step) (map[string]string, error) {
	// make sure img dir exists
	if err := os.MkdirAll(dir, 0755); err != nil {
		return nil, err
	}

	type res struct {
		url, file string
		err       error
	}

	ch := make(chan *res, 100)
	defer close(ch)
	var count int
	for _, st := range steps {
		nodes := types.ImageNodes(st.Content.Nodes)
		count += len(nodes)
		for _, n := range nodes {
			go func(n *types.ImageNode) {
				url := n.Src
				file, err := f.slurpBytes(src, dir, url)
				if err == nil {
					n.Src = filepath.Join(util.ImgDirname, file)
				}
				ch <- &res{url, file, err}
			}(n)
		}
	}

	imap := make(map[string]string, count)
	errBuilder := strings.Builder{}
	for i := 0; i < count; i++ {
		r := <-ch
		imap[r.file] = r.url
		if r.err != nil {
			errBuilder.WriteString(fmt.Sprintf("%s => %s: %v", r.url, r.file, r.err))
		}
	}
	if errBuilder.Len() > 0 {
		return nil, errors.New(errBuilder.String())
	}

	return imap, nil
}

func (f *Fetcher) slurpBytes(codelabSrc, dir, imgURL string) (string, error) {
	// images can be local in Markdown cases or remote.
	// Only proceed a simple copy on local reference.
	var b []byte
	var ext string
	u, err := url.Parse(imgURL)
	if err != nil {
		return "", err
	}

	// If the codelab source is being downloaded from the network, then we should interpret
	// the image URL in the same way.
	srcUrl, err := url.Parse(codelabSrc)
	if err == nil && srcUrl.Host != "" {
		u = srcUrl.ResolveReference(u)
	}

	if u.Host == "" {
		if imgURL, err = restrictPathToParent(imgURL, filepath.Dir(codelabSrc)); err != nil {
			return "", err
		}
		b, err = ioutil.ReadFile(imgURL)
		ext = filepath.Ext(imgURL)
	} else {
		b, err = f.slurpRemoteBytes(u.String(), 5)
		if string(b[6:10]) == "JFIF" {
			ext = ".jpeg"
		} else if string(b[0:3]) == "GIF" {
			ext = ".gif"
		} else {
			ext = ".png"
		}
	}
	if err != nil {
		return "", err
	}

	crc := crc64.Checksum(b, f.crcTable)
	file := fmt.Sprintf("%x%s", crc, ext)
	dst := filepath.Join(dir, file)
	return file, ioutil.WriteFile(dst, b, 0644)
}

func (f *Fetcher) slurpFragment(url string) ([]types.Node, error) {
	res, err := f.fetchRemote(url, true)
	if err != nil {
		return nil, err
	}
	defer res.body.Close()
	return parser.ParseFragment(string(res.typ), res.body)
}

// fetch retrieves codelab doc either from local disk
// or a remote location.
// The caller is responsible for closing returned stream.
func (f *Fetcher) fetch(name string) (*resource, error) {
	fi, err := os.Stat(name)
	if os.IsNotExist(err) {
		return f.fetchRemote(name, false)
	}
	r, err := os.Open(name)
	if err != nil {
		return nil, err
	}
	return &resource{
		body: r,
		typ:  SrcMarkdown,
		mod:  fi.ModTime(),
	}, nil
}

// fetchRemote retrieves resource r from the network.
//
// If urlStr is not a URL, i.e. does not have the host part, it is considered to be
// a Google Doc ID and fetched accordingly. Otherwise, a simple GET request
// is used to retrieve the contents.
//
// The caller is responsible for closing returned stream.
// If nometa is true, resource.mod may have zero value.
func (f *Fetcher) fetchRemote(urlStr string, nometa bool) (*resource, error) {
	u, err := url.Parse(urlStr)
	if err != nil {
		return nil, err
	}
	if u.Host == "" || u.Host == "docs.google.com" {
		return f.fetchDriveFile(urlStr, nometa)
	}
	return f.fetchRemoteFile(urlStr)
}

// fetchRemoteFile retrieves codelab resource from url.
// It is a special case of fetchRemote function.
func (f *Fetcher) fetchRemoteFile(url string) (*resource, error) {
	res, err := retryGet(f.authHelper.DriveClient(), url, 3)
	if err != nil {
		return nil, err
	}
	t, err := http.ParseTime(res.Header.Get("last-modified"))
	if err != nil {
		t = time.Now()
	}
	return &resource{
		body: res.Body,
		mod:  t,
		typ:  SrcMarkdown,
	}, nil
}

// fetchDriveFile uses Drive API to retrieve HTML representation of a Google Doc.
// See https://developers.google.com/drive/web/manage-downloads#downloading_google_documents
// for more details.
//
// If nometa is true, resource.mod will have zero value.
func (f *Fetcher) fetchDriveFile(id string, nometa bool) (*resource, error) {
	id = gdocID(id)
	exportURL := gdocExportURL(id)

	if nometa {
		res, err := retryGet(f.authHelper.DriveClient(), exportURL, 7)
		if err != nil {
			return nil, err
		}
		return &resource{body: res.Body, typ: SrcGoogleDoc}, nil
	}

	q := url.Values{
		"fields":             {"id,mimeType,modifiedTime"},
		"supportsTeamDrives": {"true"},
	}
	u := fmt.Sprintf("%s/files/%s?%s", driveAPI, id, q.Encode())
	res, err := retryGet(f.authHelper.DriveClient(), u, 7)
	if err != nil {
		return nil, err
	}
	defer res.Body.Close()
	meta := &struct {
		ID       string    `json:"id"`
		MimeType string    `json:"mimeType"`
		Modified time.Time `json:"modifiedTime"`
	}{}
	if err := json.NewDecoder(res.Body).Decode(meta); err != nil {
		return nil, err
	}
	if meta.MimeType != "application/vnd.google-apps.document" {
		return nil, fmt.Errorf("%s: invalid mime type: %s", id, meta.MimeType)
	}

	if res, err = retryGet(f.authHelper.DriveClient(), exportURL, 7); err != nil {
		return nil, err
	}
	return &resource{
		body: res.Body,
		mod:  meta.Modified,
		typ:  SrcGoogleDoc,
	}, nil
}

func (f *Fetcher) slurpRemoteBytes(url string, n int) ([]byte, error) {
	res, err := retryGet(f.authHelper.DriveClient(), url, n)
	if err != nil {
		return nil, err
	}
	defer res.Body.Close()
	return ioutil.ReadAll(res.Body)
}

// retryGet tries to GET specified url up to n times.
// Attempts are spaced out with exponential backoff.
// Default client will be used if not provided.
func retryGet(client *http.Client, url string, n int) (*http.Response, error) {
	if client == nil {
		client = http.DefaultClient
	}
	for i := 0; i <= n; i++ {
		if i > 0 {
			t := time.Duration((math.Pow(2, float64(i)) + rand.Float64()) * float64(time.Second))
			time.Sleep(t)
		}
		res, err := client.Get(url)
		// return early with a good response
		// the rest is error handling
		if err == nil && res.StatusCode == http.StatusOK {
			return res, nil
		}

		// sometimes Drive API wouldn't even start a response,
		// we get net/http: TLS handshake timeout instead:
		// consider this a temporary failure and retry again
		if err != nil {
			continue
		}
		// otherwise, decode error response and check for "rate limit"
		defer res.Body.Close()
		var erres struct {
			Error struct {
				Errors []struct{ Reason string }
			}
		}
		b, _ := ioutil.ReadAll(res.Body)
		json.Unmarshal(b, &erres)
		var rateLimit bool
		for _, e := range erres.Error.Errors {
			if e.Reason == "rateLimitExceeded" || e.Reason == "userRateLimitExceeded" {
				rateLimit = true
				break
			}
		}
		// this is neither a rate limit error, nor a server error:
		// retrying is useless
		if !rateLimit && res.StatusCode < http.StatusInternalServerError {
			return nil, fmt.Errorf("fetch %s: %s; %s", url, res.Status, b)
		}
	}
	return nil, fmt.Errorf("%s: failed after %d retries", url, n)
}

func gdocID(url string) string {
	const s = "/document/d/"
	if i := strings.Index(url, s); i >= 0 {
		url = url[i+len(s):]
	}
	if i := strings.IndexRune(url, '/'); i > 0 {
		url = url[:i]
	}
	return url
}

func gdocExportURL(id string) string {
	return fmt.Sprintf("%s/files/%s/export?mimeType=text/html", driveAPI, id)
}

// restrictPathToParent will ensure that assetPath is in parent.
// It will thus return an absolute path to the asset.
func restrictPathToParent(assetPath, parent string) (string, error) {
	parent, err := filepath.Abs(parent)
	if err != nil {
		return "", err
	}
	if !filepath.IsAbs(assetPath) {
		assetPath = filepath.Join(parent, assetPath)
	}
	if !strings.HasPrefix(assetPath, parent) {
		return "", fmt.Errorf("%s isn't a subdirectory of %s", assetPath, parent)
	}
	return assetPath, nil
}
<|endoftext|>


<commit_before>package mesh

import (
	"encoding/gob"
	"fmt"
	"net"
	"sync"
	"time"
)

// localPeer is the only "active" peer in the mesh. It extends Peer with
// additional behaviors, mostly to retrieve and manage connection state.
type localPeer struct {
	sync.RWMutex
	*Peer
	router              *Router
	actionChan          chan<- localPeerAction
	topologyUpdatesChan chan<- []*Peer
}

// The actor closure used by localPeer.
type localPeerAction func()

// newLocalPeer returns a usable LocalPeer.
func newLocalPeer(name PeerName, nickName string, router *Router) *localPeer {
	actionChan := make(chan localPeerAction, ChannelSize)
	topologyUpdatesChan := make(chan []*Peer, ChannelSize)
	peer := &localPeer{
		Peer:                newPeer(name, nickName, randomPeerUID(), 0, randomPeerShortID()),
		router:              router,
		actionChan:          actionChan,
		topologyUpdatesChan: topologyUpdatesChan,
	}
	go peer.actorLoop(actionChan)
	go peer.brodcastTopologyUpdates(topologyUpdatesChan)
	return peer
}

// Connections returns all the connections that the local peer is aware of.
func (peer *localPeer) getConnections() connectionSet {
	connections := make(connectionSet)
	peer.RLock()
	defer peer.RUnlock()
	for _, conn := range peer.connections {
		connections[conn] = struct{}{}
	}
	return connections
}

// ConnectionTo returns the connection to the named peer, if any.
//
// TODO(pb): Weave Net invokes router.Ourself.ConnectionTo;
// it may be better to provide that on Router directly.
func (peer *localPeer) ConnectionTo(name PeerName) (Connection, bool) {
	peer.RLock()
	defer peer.RUnlock()
	conn, found := peer.connections[name]
	return conn, found // yes, you really can't inline that. FFS.
}

// ConnectionsTo returns all known connections to the named peers.
//
// TODO(pb): Weave Net invokes router.Ourself.ConnectionsTo;
// it may be better to provide that on Router directly.
func (peer *localPeer) ConnectionsTo(names []PeerName) []Connection {
	if len(names) == 0 {
		return nil
	}
	conns := make([]Connection, 0, len(names))
	peer.RLock()
	defer peer.RUnlock()
	for _, name := range names {
		conn, found := peer.connections[name]
		// Again, !found could just be due to a race.
		if found {
			conns = append(conns, conn)
		}
	}
	return conns
}

// createConnection creates a new connection, originating from
// localAddr, to peerAddr. If acceptNewPeer is false, peerAddr must
// already be a member of the mesh.
func (peer *localPeer) createConnection(localAddr string, peerAddr string, acceptNewPeer bool, logger Logger) error {
	if err := peer.checkConnectionLimit(); err != nil {
		return err
	}
	localTCPAddr, err := net.ResolveTCPAddr("tcp", localAddr)
	if err != nil {
		return err
	}
	remoteTCPAddr, err := net.ResolveTCPAddr("tcp", peerAddr)
	if err != nil {
		return err
	}
	tcpConn, err := net.DialTCP("tcp", localTCPAddr, remoteTCPAddr)
	if err != nil {
		return err
	}
	connRemote := newRemoteConnection(peer.Peer, nil, peerAddr, true, false)
	startLocalConnection(connRemote, tcpConn, peer.router, acceptNewPeer, logger)
	return nil
}

// ACTOR client API

// Synchronous.
func (peer *localPeer) doAddConnection(conn ourConnection, isRestartedPeer bool) error {
	resultChan := make(chan error)
	peer.actionChan <- func() {
		resultChan <- peer.handleAddConnection(conn, isRestartedPeer)
	}
	return <-resultChan
}

// Asynchronous.
func (peer *localPeer) doConnectionEstablished(conn ourConnection) {
	peer.actionChan <- func() {
		peer.handleConnectionEstablished(conn)
	}
}

// Synchronous.
func (peer *localPeer) doDeleteConnection(conn ourConnection) {
	resultChan := make(chan interface{})
	peer.actionChan <- func() {
		peer.handleDeleteConnection(conn)
		resultChan <- nil
	}
	<-resultChan
}

func (peer *localPeer) encode(enc *gob.Encoder) {
	peer.RLock()
	defer peer.RUnlock()
	peer.Peer.encode(enc)
}

// ACTOR server

func (peer *localPeer) actorLoop(actionChan <-chan localPeerAction) {
	gossipInterval := defaultGossipInterval
	if peer.router != nil {
		gossipInterval = peer.router.gossipInterval()
	}
	gossipTimer := time.Tick(gossipInterval)
	for {
		select {
		case action := <-actionChan:
			action()
		case <-gossipTimer:
			peer.router.sendAllGossip()
		}
	}
}

func (peer *localPeer) brodcastTopologyUpdates(topologyUpdatesChan <-chan []*Peer) {
	for {
		select {
		case update := <-topologyUpdatesChan:
			peer.router.Routes.recalculate()
			updates := [][]*Peer{update}
			for i := 0; i < len(topologyUpdatesChan); i++ {
				u := <-topologyUpdatesChan
				updates = append(updates, u)
			}
			peer.router.broadcastTopologyUpdate(updates...)
		}
	}
}

func (peer *localPeer) handleAddConnection(conn ourConnection, isRestartedPeer bool) error {
	if peer.Peer != conn.getLocal() {
		panic("Attempt made to add connection to peer where peer is not the source of connection")
	}
	if conn.Remote() == nil {
		panic("Attempt made to add connection to peer with unknown remote peer")
	}
	toName := conn.Remote().Name
	dupErr := fmt.Errorf("Multiple connections to %s added to %s", conn.Remote(), peer.String())
	// deliberately non symmetrical
	if dupConn, found := peer.connections[toName]; found {
		if dupConn == conn {
			return nil
		}
		dupOurConn := dupConn.(ourConnection)
		switch conn.breakTie(dupOurConn) {
		case tieBreakWon:
			dupOurConn.shutdown(dupErr)
			peer.handleDeleteConnection(dupOurConn)
		case tieBreakLost:
			return dupErr
		case tieBreakTied:
			// oh good grief. Sod it, just kill both of them.
			dupOurConn.shutdown(dupErr)
			peer.handleDeleteConnection(dupOurConn)
			return dupErr
		}
	}
	if err := peer.checkConnectionLimit(); err != nil {
		return err
	}
	_, isConnectedPeer := peer.router.Routes.Unicast(toName)
	peer.addConnection(conn)
	switch {
	case isRestartedPeer:
		conn.logf("connection added (restarted peer)")
		peer.router.sendAllGossipDown(conn)
	case isConnectedPeer:
		conn.logf("connection added")
	default:
		conn.logf("connection added (new peer)")
		peer.router.sendAllGossipDown(conn)
	}
	peer.broadcastPeerUpdate(conn.Remote())

	return nil
}

func (peer *localPeer) handleConnectionEstablished(conn ourConnection) {
	if peer.Peer != conn.getLocal() {
		panic("Peer informed of active connection where peer is not the source of connection")
	}
	if dupConn, found := peer.connections[conn.Remote().Name]; !found || conn != dupConn {
		conn.shutdown(fmt.Errorf("Cannot set unknown connection active"))
		return
	}
	peer.connectionEstablished(conn)
	conn.logf("connection fully established")
	peer.broadcastPeerUpdate()
}

func (peer *localPeer) handleDeleteConnection(conn ourConnection) {
	if peer.Peer != conn.getLocal() {
		panic("Attempt made to delete connection from peer where peer is not the source of connection")
	}
	if conn.Remote() == nil {
		panic("Attempt made to delete connection to peer with unknown remote peer")
	}
	toName := conn.Remote().Name
	if connFound, found := peer.connections[toName]; !found || connFound != conn {
		return
	}
	peer.deleteConnection(conn)
	conn.logf("connection deleted")
	// Must do garbage collection first to ensure we don't send out an
	// update with unreachable peers (can cause looping)
	peer.router.Peers.GarbageCollect()
	peer.broadcastPeerUpdate()
}

// helpers

func (peer *localPeer) broadcastPeerUpdate(peers ...*Peer) {
	// Some tests run without a router.  This should be fixed so
	// that the relevant part of Router can be easily run in the
	// context of a test, but that will involve significant
	// reworking of tests.
	if peer.router != nil {
		peer.topologyUpdatesChan <- append(peers, peer.Peer)
	}
}

func (peer *localPeer) checkConnectionLimit() error {
	limit := peer.router.ConnLimit
	if 0 != limit && peer.connectionCount() >= limit {
		return fmt.Errorf("Connection limit reached (%v)", limit)
	}
	return nil
}

func (peer *localPeer) addConnection(conn Connection) {
	peer.Lock()
	defer peer.Unlock()
	peer.connections[conn.Remote().Name] = conn
	peer.Version++
}

func (peer *localPeer) deleteConnection(conn Connection) {
	peer.Lock()
	defer peer.Unlock()
	delete(peer.connections, conn.Remote().Name)
	peer.Version++
}

func (peer *localPeer) connectionEstablished(conn Connection) {
	peer.Lock()
	defer peer.Unlock()
	peer.Version++
}

func (peer *localPeer) connectionCount() int {
	peer.RLock()
	defer peer.RUnlock()
	return len(peer.connections)
}

func (peer *localPeer) setShortID(shortID PeerShortID) {
	peer.Lock()
	defer peer.Unlock()
	peer.ShortID = shortID
	peer.Version++
}

func (peer *localPeer) setVersionBeyond(version uint64) bool {
	peer.Lock()
	defer peer.Unlock()
	if version >= peer.Version {
		peer.Version = version + 1
		return true
	}
	return false
}
<commit_msg>use timer to trigger the broadcast topology updates and coalesce updates<commit_after>package mesh

import (
	"encoding/gob"
	"fmt"
	"net"
	"sync"
	"time"
)

const (
	topologyUpdateInterval = 1 * time.Second
)

// localPeer is the only "active" peer in the mesh. It extends Peer with
// additional behaviors, mostly to retrieve and manage connection state.
type localPeer struct {
	sync.RWMutex
	*Peer
	router                *Router
	actionChan            chan<- localPeerAction
	topologyUpdatesChan   chan []*Peer
	timer                 *time.Timer
	pendingTopologyUpdate bool
}

// The actor closure used by localPeer.
type localPeerAction func()

// newLocalPeer returns a usable LocalPeer.
func newLocalPeer(name PeerName, nickName string, router *Router) *localPeer {
	actionChan := make(chan localPeerAction, ChannelSize)
	topologyUpdatesChan := make(chan []*Peer, ChannelSize)
	peer := &localPeer{
		Peer:                newPeer(name, nickName, randomPeerUID(), 0, randomPeerShortID()),
		router:              router,
		actionChan:          actionChan,
		topologyUpdatesChan: topologyUpdatesChan,
		timer:               time.NewTimer(topologyUpdateInterval),
	}
	peer.timer.Stop()
	go peer.actorLoop(actionChan)
	return peer
}

// Connections returns all the connections that the local peer is aware of.
func (peer *localPeer) getConnections() connectionSet {
	connections := make(connectionSet)
	peer.RLock()
	defer peer.RUnlock()
	for _, conn := range peer.connections {
		connections[conn] = struct{}{}
	}
	return connections
}

// ConnectionTo returns the connection to the named peer, if any.
//
// TODO(pb): Weave Net invokes router.Ourself.ConnectionTo;
// it may be better to provide that on Router directly.
func (peer *localPeer) ConnectionTo(name PeerName) (Connection, bool) {
	peer.RLock()
	defer peer.RUnlock()
	conn, found := peer.connections[name]
	return conn, found // yes, you really can't inline that. FFS.
}

// ConnectionsTo returns all known connections to the named peers.
//
// TODO(pb): Weave Net invokes router.Ourself.ConnectionsTo;
// it may be better to provide that on Router directly.
func (peer *localPeer) ConnectionsTo(names []PeerName) []Connection {
	if len(names) == 0 {
		return nil
	}
	conns := make([]Connection, 0, len(names))
	peer.RLock()
	defer peer.RUnlock()
	for _, name := range names {
		conn, found := peer.connections[name]
		// Again, !found could just be due to a race.
		if found {
			conns = append(conns, conn)
		}
	}
	return conns
}

// createConnection creates a new connection, originating from
// localAddr, to peerAddr. If acceptNewPeer is false, peerAddr must
// already be a member of the mesh.
func (peer *localPeer) createConnection(localAddr string, peerAddr string, acceptNewPeer bool, logger Logger) error {
	if err := peer.checkConnectionLimit(); err != nil {
		return err
	}
	localTCPAddr, err := net.ResolveTCPAddr("tcp", localAddr)
	if err != nil {
		return err
	}
	remoteTCPAddr, err := net.ResolveTCPAddr("tcp", peerAddr)
	if err != nil {
		return err
	}
	tcpConn, err := net.DialTCP("tcp", localTCPAddr, remoteTCPAddr)
	if err != nil {
		return err
	}
	connRemote := newRemoteConnection(peer.Peer, nil, peerAddr, true, false)
	startLocalConnection(connRemote, tcpConn, peer.router, acceptNewPeer, logger)
	return nil
}

// ACTOR client API

// Synchronous.
func (peer *localPeer) doAddConnection(conn ourConnection, isRestartedPeer bool) error {
	resultChan := make(chan error)
	peer.actionChan <- func() {
		resultChan <- peer.handleAddConnection(conn, isRestartedPeer)
	}
	return <-resultChan
}

// Asynchronous.
func (peer *localPeer) doConnectionEstablished(conn ourConnection) {
	peer.actionChan <- func() {
		peer.handleConnectionEstablished(conn)
	}
}

// Synchronous.
func (peer *localPeer) doDeleteConnection(conn ourConnection) {
	resultChan := make(chan interface{})
	peer.actionChan <- func() {
		peer.handleDeleteConnection(conn)
		resultChan <- nil
	}
	<-resultChan
}

func (peer *localPeer) encode(enc *gob.Encoder) {
	peer.RLock()
	defer peer.RUnlock()
	peer.Peer.encode(enc)
}

// ACTOR server

func (peer *localPeer) actorLoop(actionChan <-chan localPeerAction) {
	gossipInterval := defaultGossipInterval
	if peer.router != nil {
		gossipInterval = peer.router.gossipInterval()
	}
	gossipTimer := time.Tick(gossipInterval)
	for {
		select {
		case action := <-actionChan:
			action()
		case <-gossipTimer:
			peer.router.sendAllGossip()
		case <-peer.timer.C:
			peer.brodcastTopologyUpdates()
			if !peer.timer.Stop() {
				select {
				case <-peer.timer.C:
				default:
				}
			}
			peer.pendingTopologyUpdate = false
		}
	}
}

func (peer *localPeer) brodcastTopologyUpdates() {
	peer.router.Routes.recalculate()
	if len(peer.topologyUpdatesChan) == 0 {
		return
	}
	updates := [][]*Peer{}
	for i := 0; i < len(peer.topologyUpdatesChan); i++ {
		u := <-peer.topologyUpdatesChan
		updates = append(updates, u)
	}
	peer.router.broadcastTopologyUpdate(updates...)
}

func (peer *localPeer) handleAddConnection(conn ourConnection, isRestartedPeer bool) error {
	if peer.Peer != conn.getLocal() {
		panic("Attempt made to add connection to peer where peer is not the source of connection")
	}
	if conn.Remote() == nil {
		panic("Attempt made to add connection to peer with unknown remote peer")
	}
	toName := conn.Remote().Name
	dupErr := fmt.Errorf("Multiple connections to %s added to %s", conn.Remote(), peer.String())
	// deliberately non symmetrical
	if dupConn, found := peer.connections[toName]; found {
		if dupConn == conn {
			return nil
		}
		dupOurConn := dupConn.(ourConnection)
		switch conn.breakTie(dupOurConn) {
		case tieBreakWon:
			dupOurConn.shutdown(dupErr)
			peer.handleDeleteConnection(dupOurConn)
		case tieBreakLost:
			return dupErr
		case tieBreakTied:
			// oh good grief. Sod it, just kill both of them.
			dupOurConn.shutdown(dupErr)
			peer.handleDeleteConnection(dupOurConn)
			return dupErr
		}
	}
	if err := peer.checkConnectionLimit(); err != nil {
		return err
	}
	_, isConnectedPeer := peer.router.Routes.Unicast(toName)
	peer.addConnection(conn)
	switch {
	case isRestartedPeer:
		conn.logf("connection added (restarted peer)")
		peer.router.sendAllGossipDown(conn)
	case isConnectedPeer:
		conn.logf("connection added")
	default:
		conn.logf("connection added (new peer)")
		peer.router.sendAllGossipDown(conn)
	}
	peer.broadcastPeerUpdate(conn.Remote())

	return nil
}

func (peer *localPeer) handleConnectionEstablished(conn ourConnection) {
	if peer.Peer != conn.getLocal() {
		panic("Peer informed of active connection where peer is not the source of connection")
	}
	if dupConn, found := peer.connections[conn.Remote().Name]; !found || conn != dupConn {
		conn.shutdown(fmt.Errorf("Cannot set unknown connection active"))
		return
	}
	peer.connectionEstablished(conn)
	conn.logf("connection fully established")
	peer.broadcastPeerUpdate()
}

func (peer *localPeer) handleDeleteConnection(conn ourConnection) {
	if peer.Peer != conn.getLocal() {
		panic("Attempt made to delete connection from peer where peer is not the source of connection")
	}
	if conn.Remote() == nil {
		panic("Attempt made to delete connection to peer with unknown remote peer")
	}
	toName := conn.Remote().Name
	if connFound, found := peer.connections[toName]; !found || connFound != conn {
		return
	}
	peer.deleteConnection(conn)
	conn.logf("connection deleted")
	// Must do garbage collection first to ensure we don't send out an
	// update with unreachable peers (can cause looping)
	peer.router.Peers.GarbageCollect()
	peer.broadcastPeerUpdate()
}

// helpers

func (peer *localPeer) broadcastPeerUpdate(peers ...*Peer) {
	// Some tests run without a router.  This should be fixed so
	// that the relevant part of Router can be easily run in the
	// context of a test, but that will involve significant
	// reworking of tests.
	if peer.router != nil {
		peer.Lock()
		defer peer.Unlock()
		if !peer.pendingTopologyUpdate {
			peer.timer.Reset(topologyUpdateInterval)
			peer.pendingTopologyUpdate = true
		}
		peer.topologyUpdatesChan <- append(peers, peer.Peer)
	}
}

func (peer *localPeer) checkConnectionLimit() error {
	limit := peer.router.ConnLimit
	if 0 != limit && peer.connectionCount() >= limit {
		return fmt.Errorf("Connection limit reached (%v)", limit)
	}
	return nil
}

func (peer *localPeer) addConnection(conn Connection) {
	peer.Lock()
	defer peer.Unlock()
	peer.connections[conn.Remote().Name] = conn
	peer.Version++
}

func (peer *localPeer) deleteConnection(conn Connection) {
	peer.Lock()
	defer peer.Unlock()
	delete(peer.connections, conn.Remote().Name)
	peer.Version++
}

func (peer *localPeer) connectionEstablished(conn Connection) {
	peer.Lock()
	defer peer.Unlock()
	peer.Version++
}

func (peer *localPeer) connectionCount() int {
	peer.RLock()
	defer peer.RUnlock()
	return len(peer.connections)
}

func (peer *localPeer) setShortID(shortID PeerShortID) {
	peer.Lock()
	defer peer.Unlock()
	peer.ShortID = shortID
	peer.Version++
}

func (peer *localPeer) setVersionBeyond(version uint64) bool {
	peer.Lock()
	defer peer.Unlock()
	if version >= peer.Version {
		peer.Version = version + 1
		return true
	}
	return false
}
<|endoftext|>


<commit_before>package clients

// Copyright 2019 Google LLC
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// 	https://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

import (
	"context"
	"fmt"

	container "google.golang.org/api/container/v1"
	"google.golang.org/api/option"
)

// Container client.
type Container struct {
	container *container.Service
}

// NewContainer returns and initializes a Container client.
func NewContainer(ctx context.Context, authFile string) (*Container, error) {
	cc, err := container.NewService(ctx, option.WithCredentialsFile(authFile))
	if err != nil {
		return nil, fmt.Errorf("Failed to init container service: %q", err)
	}

	return &Container{
		container: cc,
	}, nil
}

// DisableKubernetesDashboard disables the kubernetes dashboard for a given cluster.
func (c *Container) DisableKubernetesDashboard(ctx context.Context, projectID, zone, clusterID string) (*container.Operation, error) {
	disableKubernetesDashboard := true
	configRequest := initSetAddonsConfigRequest(disableKubernetesDashboard)
	return c.container.Projects.Zones.Clusters.Addons(projectID, zone, clusterID, configRequest).Context(ctx).Do()
}

// newSetAddonsConfigRequest returns a new SetAddonsConfigRequest with a given AddonsConfig.
func newSetAddonsConfigRequest(addonsConfig *container.AddonsConfig) *container.SetAddonsConfigRequest {
	var setAddonsConfigRequest container.SetAddonsConfigRequest
	setAddonsConfigRequest.AddonsConfig = addonsConfig
	return &setAddonsConfigRequest
}

// newAddonsConfig returns a new AddonsConfig with a given KubernetesDashboard.
func newAddonsConfig(kubernetesDashboard *container.KubernetesDashboard) *container.AddonsConfig {
	var addonsConfig container.AddonsConfig
	addonsConfig.KubernetesDashboard = kubernetesDashboard
	return &addonsConfig
}

// newKubernetesDashboard returns a new KubernetesDashboard.
func newKubernetesDashboard(disabled bool) *container.KubernetesDashboard {
	var kubernetesDashboard container.KubernetesDashboard
	kubernetesDashboard.Disabled = disabled
	return &kubernetesDashboard
}

// initSetAddonsConfigRequest initializes SetAddonsConfigRequest used to disable the Kubernetes Dashboard.
func initSetAddonsConfigRequest(disabled bool) *container.SetAddonsConfigRequest {
	kubernetesDashboard := newKubernetesDashboard(disabled)
	addonsConfig := newAddonsConfig(kubernetesDashboard)
	return newSetAddonsConfigRequest(addonsConfig)
}
<commit_msg>Simplifies the configRequest initialization<commit_after>package clients

// Copyright 2019 Google LLC
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// 	https://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

import (
	"context"
	"fmt"

	container "google.golang.org/api/container/v1"
	"google.golang.org/api/option"
)

// Container client.
type Container struct {
	container *container.Service
}

// NewContainer returns and initializes a Container client.
func NewContainer(ctx context.Context, authFile string) (*Container, error) {
	cc, err := container.NewService(ctx, option.WithCredentialsFile(authFile))
	if err != nil {
		return nil, fmt.Errorf("Failed to init container service: %q", err)
	}

	return &Container{
		container: cc,
	}, nil
}

// DisableKubernetesDashboard disables the kubernetes dashboard for a given cluster.
func (c *Container) DisableKubernetesDashboard(ctx context.Context, projectID, zone, clusterID string) (*container.Operation, error) {
	configRequest := &container.SetAddonsConfigRequest{
		AddonsConfig: &container.AddonsConfig{
			KubernetesDashboard: &container.KubernetesDashboard{
				Disabled: true,
			},
		},
	}
	return c.container.Projects.Zones.Clusters.Addons(projectID, zone, clusterID, configRequest).Context(ctx).Do()
}
<|endoftext|>


<commit_before>package types

import (
	"go/ast"
	"go/parser"
	"go/token"
	"strings"

	"github.com/nelsam/hel/packages"
)

type Types struct {
	dir     string
	testPkg string
	types   []*ast.TypeSpec
}

func (t Types) Dir() string {
	return t.dir
}

func (t Types) Len() int {
	return len(t.types)
}

func (t Types) TestPackage() string {
	return t.testPkg
}

func (t Types) ExportedTypes() []*ast.TypeSpec {
	return t.types
}

func Load(packages ...packages.Package) []Types {
	types := make([]Types, 0, len(packages))
	for _, pkg := range packages {
		t := Types{
			dir: pkg.Path,
		}
		pkgs, err := parser.ParseDir(token.NewFileSet(), pkg.Path, nil, 0)
		if err != nil {
			panic(err)
		}
		for name, pkg := range pkgs {
			if strings.HasSuffix(name, "_test") {
				t.testPkg = name
				continue
			}
			newTypes, testsFound := loadPkgTypeSpecs(pkg)
			t.types = append(t.types, newTypes...)
			if testsFound && t.testPkg == "" {
				t.testPkg = name
			}
		}
		types = append(types, t)
	}
	return types
}

func loadPkgTypeSpecs(pkg *ast.Package) (specs []*ast.TypeSpec, hasTests bool) {
	for name, f := range pkg.Files {
		if strings.HasSuffix(name, "_test.go") {
			hasTests = true
			continue
		}
		specs = append(specs, loadFileTypeSpecs(f)...)
	}
	return specs, hasTests
}

func loadFileTypeSpecs(f *ast.File) (specs []*ast.TypeSpec) {
	for _, obj := range f.Scope.Objects {
		spec, ok := obj.Decl.(*ast.TypeSpec)
		if !ok {
			continue
		}
		if _, ok = spec.Type.(*ast.InterfaceType); !ok {
			continue
		}
		specs = append(specs, spec)
	}
	return specs
}
<commit_msg>Update types to use an interface type for finding ast packages; fixed minor bug when generating mocks in directories that have no test files<commit_after>package types

import (
	"go/ast"
	"strings"
)

type GoDir interface {
	Path() string
	Packages() map[string]*ast.Package
}

type Types struct {
	dir     string
	testPkg string
	types   []*ast.TypeSpec
}

func (t Types) Dir() string {
	return t.dir
}

func (t Types) Len() int {
	return len(t.types)
}

func (t Types) TestPackage() string {
	return t.testPkg
}

func (t Types) ExportedTypes() []*ast.TypeSpec {
	return t.types
}

func Load(dirs ...GoDir) []Types {
	types := make([]Types, 0, len(dirs))
	for _, dir := range dirs {
		t := Types{
			dir: dir.Path(),
		}
		for name, pkg := range dir.Packages() {
			if t.testPkg == "" {
				// This will get overridden if we later find pre-existing test
				// files in one of the packages.  As such, don't worry about
				// test packages getting an extra "_test", since test packages
				// will be made up of only test files.
				t.testPkg = name + "_test"
			}
			newTypes, testsFound := loadPkgTypeSpecs(pkg)
			if testsFound {
				t.testPkg = name
			}
			t.types = append(t.types, newTypes...)
		}
		types = append(types, t)
	}
	return types
}

func loadPkgTypeSpecs(pkg *ast.Package) (specs []*ast.TypeSpec, hasTests bool) {
	for name, f := range pkg.Files {
		if strings.HasSuffix(name, "_test.go") {
			hasTests = true
			continue
		}
		specs = append(specs, loadFileTypeSpecs(f)...)
	}
	return specs, hasTests
}

func loadFileTypeSpecs(f *ast.File) (specs []*ast.TypeSpec) {
	for _, obj := range f.Scope.Objects {
		spec, ok := obj.Decl.(*ast.TypeSpec)
		if !ok {
			continue
		}
		if _, ok = spec.Type.(*ast.InterfaceType); !ok {
			continue
		}
		specs = append(specs, spec)
	}
	return specs
}
<|endoftext|>


<commit_before>/*
Package "types" provides functions to descriminate the type of given matrix.
*/
package types

import (
	"github.com/mitsuse/matrix-go"
	"github.com/mitsuse/matrix-go/elements"
)

// Check whether "m" is zero matrix or not.
func IsZeros(m matrix.Matrix) bool {
	return !m.NonZeros().HasNext()
}

// Check whether "m" is square matrix or not.
func IsSquare(m matrix.Matrix) bool {
	return m.Rows() == m.Columns()
}

// Check whether "m" is special diagonal matrix satisfying arbitary condition.
func IsSpecialDiagonal(m matrix.Matrix, match elements.Match) bool {
	if !IsSquare(m) {
		return false
	}

	elements := m.All()

	for elements.HasNext() {
		element, row, column := elements.Get()
		if !match(element, row, column) {
			return false
		}
	}

	return true
}

// Check whether "m" is diagonal matrix or not.
func IsDiagonal(m matrix.Matrix) bool {
	match := func(element float64, row, column int) bool {
		return row == column || element == 0
	}

	return IsSpecialDiagonal(m, match)
}

// Check whether "m" is identity matrix or not.
func IsIdentity(m matrix.Matrix) bool {
	match := func(element float64, row, column int) bool {
		if row == column {
			return element == 1
		} else {
			return element == 0
		}
	}

	return IsSpecialDiagonal(m, match)
}

// Check whether "m" is scalar matrix or not.
func IsScalar(m matrix.Matrix) bool {
	if rows, columns := m.Shape(); rows == 0 && columns == 0 {
		return true
	}

	scalar := m.Get(0, 0)

	match := func(element float64, row, column int) bool {
		if row == column {
			return element == scalar
		} else {
			return element == 0
		}
	}

	return IsSpecialDiagonal(m, match)
}
<commit_msg>The rows and columns should be positive numbers.<commit_after>/*
Package "types" provides functions to descriminate the type of given matrix.
*/
package types

import (
	"github.com/mitsuse/matrix-go"
	"github.com/mitsuse/matrix-go/elements"
)

// Check whether "m" is zero matrix or not.
func IsZeros(m matrix.Matrix) bool {
	return !m.NonZeros().HasNext()
}

// Check whether "m" is square matrix or not.
func IsSquare(m matrix.Matrix) bool {
	return m.Rows() == m.Columns()
}

// Check whether "m" is special diagonal matrix satisfying arbitary condition.
func IsSpecialDiagonal(m matrix.Matrix, match elements.Match) bool {
	if !IsSquare(m) {
		return false
	}

	elements := m.All()

	for elements.HasNext() {
		element, row, column := elements.Get()
		if !match(element, row, column) {
			return false
		}
	}

	return true
}

// Check whether "m" is diagonal matrix or not.
func IsDiagonal(m matrix.Matrix) bool {
	match := func(element float64, row, column int) bool {
		return row == column || element == 0
	}

	return IsSpecialDiagonal(m, match)
}

// Check whether "m" is identity matrix or not.
func IsIdentity(m matrix.Matrix) bool {
	match := func(element float64, row, column int) bool {
		if row == column {
			return element == 1
		} else {
			return element == 0
		}
	}

	return IsSpecialDiagonal(m, match)
}

// Check whether "m" is scalar matrix or not.
func IsScalar(m matrix.Matrix) bool {
	scalar := m.Get(0, 0)

	match := func(element float64, row, column int) bool {
		if row == column {
			return element == scalar
		} else {
			return element == 0
		}
	}

	return IsSpecialDiagonal(m, match)
}
<|endoftext|>


<commit_before>package main

import (
	"fmt"
	"gongs/argparser"
	"os"
)

const statName = "stat"
const statDesc = "stat fastq file"
const statVersion = "2015.08.19.1"

var statArger = argparser.New(mainName, statName)

func init() {
	statArger.Add("prefix", "-p", "--prefix", "output file prefix name", "stat")
	statArger.Add("thread", "-t", "--thread", "runtime thread Number default (all available cpu)", 0)
}

func statUsage() {
	statArger.Usage()
}

func statRunner(args ...string) {
	if len(args) == 0 {
		statUsage()
		os.Exit(1)
	}
	if err := statRun(args...); err != nil {
		fmt.Fprintln(os.Stderr, err)
		os.Exit(1)
	}
}

func statRun(args ...string) error {
	if err := statArger.Parse(args...); err != nil {
		return err
	}

	prefix := statArger.Get("prefix").(string)

	// setting multi-threads
	setThread(statArger.Get("thread").(int))

}
<commit_msg>update stat.go<commit_after>package main

import (
	"fmt"
	"gongs/argparser"
	"os"
)

const statName = "stat"
const statDesc = "stat fastq file"
const statVersion = "2015.08.19.1"

var statArger = argparser.New(mainName, statName)

func init() {
	statArger.Add("prefix", "-p", "--prefix", "output file prefix name", "stat")
	statArger.Add("thread", "-t", "--thread", "runtime thread Number default (all available cpu)", 0)
}

func statUsage() {
	statArger.Usage()
}

func statRunner(args ...string) {
	if len(args) == 0 {
		statUsage()
		os.Exit(1)
	}
	if err := statRun(args...); err != nil {
		fmt.Fprintln(os.Stderr, err)
		os.Exit(1)
	}
}

func statRun(args ...string) error {
	if err := statArger.Parse(args...); err != nil {
		return err
	}

	// prefix := statArger.Get("prefix").(string)

	// setting multi-threads
	setThread(statArger.Get("thread").(int))
	return nil
}
<|endoftext|>


<commit_before>package v1payload

import "k8s.io/apimachinery/pkg/api/resource"

// CreateBillingPackageInput is the input for assigning a billing package to a project.
// This results in the creation of a quota in the right namespace.
type CreateBillingPackageInput struct {
	BillingPackageID string `json:"billing_package_id"`
	RequestsCPU      string `json:"requests_cpu"`
	RequestsMemory   string `json:"requests_memory"`
}

// CreateBillingPackageOutput is the output from assigning a billing package to a project.
type CreateBillingPackageOutput struct {
	ProjectID        string             `json:"project_id" valid:"required"`
	BillingPackageID string             `json:"billing_package_id" valid:"required"`
	RequestsCPU      *resource.Quantity `json:"requests_cpu"`
	RequestsMemory   *resource.Quantity `json:"requests_memory"`
}

// UpdateBillingPackageInput is the input for updating the billing package capacity
type UpdateBillingPackageInput struct {
	RequestsCPU    string `json:"requests_cpu"`
	RequestsMemory string `json:"requests_memory"`
}

// UpdateBillingPackageOutput is the output for updating the billing package capacity
type UpdateBillingPackageOutput struct {
	ProjectID        string             `json:"project_id" valid:"required"`
	BillingPackageID string             `json:"billing_package_id" valid:"required"`
	RequestsCPU      *resource.Quantity `json:"requests_cpu"`
	RequestsMemory   *resource.Quantity `json:"requests_memory"`
}

// DescribeBillingPackageInput is the input for describing a billing package.
// Parameters are fetched through the url
type DescribeBillingPackageInput struct {
}

// DescribeBillingPackageOutput is the output from describing a billingPackage.
// Same as the DescribeBillingPackageInput, a billing package is not always
// assigned to a project so the projectID can be empty.
type DescribeBillingPackageOutput struct {
	ProjectID        string             `json:"project_id"`
	BillingPackageID string             `json:"billing_package_id" valid:"required"`
	RequestsCPU      *resource.Quantity `json:"requests_cpu"`
	RequestsMemory   *resource.Quantity `json:"requests_memory"`
	UsedCPU          *resource.Quantity `json:"used_cpu"`
	UsedMemory       *resource.Quantity `json:"used_memory"`
}

// RemoveBillingPackageInput is the input for removing a billing package from a project
type RemoveBillingPackageInput struct {
}

// RemoveBillingPackageOutput is the output from removing a billing package from a project
type RemoveBillingPackageOutput struct {
}

// DeleteBillingPackageInput is the input for deleting a billing package
type DeleteBillingPackageInput struct {
}

// DeleteBillingPackageOutput is the output from deleting a billing package
type DeleteBillingPackageOutput struct {
}

//BillingPackageSummary is summary of a billing package
type BillingPackageSummary struct {
	ProjectID        string `json:"project_id" valid:"required"`
	BillingPackageID string `json:"billing_package_id" valid:"required"`
}

// ListBillingPackagesInput is the input for listing billing packages.
type ListBillingPackagesInput struct {
}

// ListBillingPackagesOutput is the output from listing billing packages of a project
type ListBillingPackagesOutput struct {
	ProjectID       string                   `json:"project_id" valid:"required"`
	BillingPackages []*BillingPackageSummary `json:"billing_packages" valid:"required"`
}
<commit_msg>back to cpu/memory types as strings<commit_after>package v1payload

// CreateBillingPackageInput is the input for assigning a billing package to a project.
// This results in the creation of a quota in the right namespace.
type CreateBillingPackageInput struct {
	BillingPackageID string `json:"billing_package_id"`
	RequestsCPU      string `json:"requests_cpu"`
	RequestsMemory   string `json:"requests_memory"`
}

// CreateBillingPackageOutput is the output from assigning a billing package to a project.
type CreateBillingPackageOutput struct {
	ProjectID        string `json:"project_id" valid:"required"`
	BillingPackageID string `json:"billing_package_id" valid:"required"`
	RequestsCPU      string `json:"requests_cpu"`
	RequestsMemory   string `json:"requests_memory"`
}

// UpdateBillingPackageInput is the input for updating the billing package capacity
type UpdateBillingPackageInput struct {
	OnDemand       bool   `json:"on_demand"`
	RequestsCPU    string `json:"requests_cpu"`
	RequestsMemory string `json:"requests_memory"`
}

// UpdateBillingPackageOutput is the output for updating the billing package capacity
type UpdateBillingPackageOutput struct {
	ProjectID        string `json:"project_id" valid:"required"`
	BillingPackageID string `json:"billing_package_id" valid:"required"`
	RequestsCPU      string `json:"requests_cpu"`
	RequestsMemory   string `json:"requests_memory"`
}

// DescribeBillingPackageInput is the input for describing a billing package.
// Parameters are fetched through the url
type DescribeBillingPackageInput struct {
}

// DescribeBillingPackageOutput is the output from describing a billingPackage.
// Same as the DescribeBillingPackageInput, a billing package is not always
// assigned to a project so the projectID can be empty.
type DescribeBillingPackageOutput struct {
	ProjectID        string `json:"project_id"`
	BillingPackageID string `json:"billing_package_id" valid:"required"`
	RequestsCPU      string `json:"requests_cpu"`
	RequestsMemory   string `json:"requests_memory"`
	UsedCPU          string `json:"used_cpu"`
	UsedMemory       string `json:"used_memory"`
}

// RemoveBillingPackageInput is the input for removing a billing package from a project
type RemoveBillingPackageInput struct {
}

// RemoveBillingPackageOutput is the output from removing a billing package from a project
type RemoveBillingPackageOutput struct {
}

// DeleteBillingPackageInput is the input for deleting a billing package
type DeleteBillingPackageInput struct {
}

// DeleteBillingPackageOutput is the output from deleting a billing package
type DeleteBillingPackageOutput struct {
}

//BillingPackageSummary is summary of a billing package
type BillingPackageSummary struct {
	ProjectID        string `json:"project_id" valid:"required"`
	BillingPackageID string `json:"billing_package_id" valid:"required"`
}

// ListBillingPackagesInput is the input for listing billing packages.
type ListBillingPackagesInput struct {
}

// ListBillingPackagesOutput is the output from listing billing packages of a project
type ListBillingPackagesOutput struct {
	ProjectID       string                   `json:"project_id" valid:"required"`
	BillingPackages []*BillingPackageSummary `json:"billing_packages" valid:"required"`
}
<|endoftext|>


<commit_before>package evaluations

import (
	"encoding/json"
	"os"
	"os/signal"
	"strconv"
	"time"

	consul "github.com/hashicorp/consul/api"
	nomad "github.com/hashicorp/nomad/api"
	"github.com/seatgeek/nomad-firehose/helper"
	"github.com/seatgeek/nomad-firehose/sink"
	log "github.com/sirupsen/logrus"
)

const (
	consulLockKey   = "nomad-firehose/evaluations.lock"
	consulLockValue = "nomad-firehose/evaluations.value"
)

// Firehose ...
type Firehose struct {
	nomadClient     *nomad.Client
	consulClient    *consul.Client
	consulSessionID string
	consulLock      *consul.Lock
	stopCh          chan struct{}
	lastChangeIndex uint64
	sink            sink.Sink
}

// NewFirehose ...
func NewFirehose() (*Firehose, error) {
	lock, sessionID, err := helper.WaitForLock(consulLockKey)
	if err != nil {
		return nil, err
	}
	defer lock.Unlock()

	nomadClient, err := nomad.NewClient(nomad.DefaultConfig())
	if err != nil {
		return nil, err
	}

	consulClient, err := consul.NewClient(consul.DefaultConfig())
	if err != nil {
		return nil, err
	}

	sink, err := sink.GetSink()
	if err != nil {
		log.Fatal(err)
		os.Exit(1)
	}

	return &Firehose{
		nomadClient:     nomadClient,
		consulClient:    consulClient,
		consulSessionID: sessionID,
		consulLock:      lock,
		sink:            sink,
	}, nil
}

// Start the firehose
func (f *Firehose) Start() error {
	// Restore the last change time from Consul
	err := f.restoreLastChangeTime()
	if err != nil {
		return err
	}

	go f.sink.Start()

	// Stop chan for all tasks to depend on
	f.stopCh = make(chan struct{})

	// setup signal handler for graceful shutdown
	go f.signalHandler()

	// watch for allocation changes
	go f.watch()

	// Save the last event time every 10s
	go f.persistLastChangeTime(10)

	// wait forever for a stop signal to happen
	for {
		select {
		case <-f.stopCh:
			return nil
		}
	}
}

// Stop the firehose
func (f *Firehose) Stop() {
	close(f.stopCh)
	f.sink.Stop()
	f.writeLastChangeTime()
}

// Read the Last Change Time from Consul KV, so we don't re-process tasks over and over on restart
func (f *Firehose) restoreLastChangeTime() error {
	kv, _, err := f.consulClient.KV().Get(consulLockValue, &consul.QueryOptions{})
	if err != nil {
		return err
	}

	// Ensure we got
	if kv != nil && kv.Value != nil {
		sv := string(kv.Value)
		v, err := strconv.ParseInt(sv, 10, 64)
		if err != nil {
			return err
		}

		f.lastChangeIndex = uint64(v)
		log.Infof("Restoring Last Change Time to %s", sv)
	} else {
		log.Info("No Last Change Time restore point, starting from scratch")
	}

	return nil
}

// Write the Last Change Time to Consul so if the process restarts,
// it will try to resume from where it left off, not emitting tons of double events for
// old events
func (f *Firehose) persistLastChangeTime(interval time.Duration) {
	ticker := time.NewTicker(interval * time.Second)

	for {
		select {
		case <-f.stopCh:
			break
		case <-ticker.C:
			f.writeLastChangeTime()
		}
	}
}

func (f *Firehose) writeLastChangeTime() {
	v := strconv.FormatUint(f.lastChangeIndex, 10)

	log.Infof("Writing lastChangedTime to KV: %s", v)
	kv := &consul.KVPair{
		Key:     consulLockValue,
		Value:   []byte(v),
		Session: f.consulSessionID,
	}
	_, err := f.consulClient.KV().Put(kv, &consul.WriteOptions{})
	if err != nil {
		log.Error(err)
	}
}

// Publish an update from the firehose
func (f *Firehose) Publish(update *nomad.Evaluation) {
	b, err := json.Marshal(update)
	if err != nil {
		log.Error(err)
	}

	f.sink.Put(b)
}

// Continously watch for changes to the allocation list and publish it as updates
func (f *Firehose) watch() {
	q := &nomad.QueryOptions{
		WaitIndex:  f.lastChangeIndex,
		WaitTime:   5 * time.Minute,
		AllowStale: true,
	}

	for {
		log.Infof("Fetching evaluations from Nomad: %+v", q)

		evaluations, meta, err := f.nomadClient.Evaluations().List(q)
		if err != nil {
			log.Errorf("Unable to fetch evaluations: %s", err)
			time.Sleep(10 * time.Second)
			continue
		}

		// Only work if the WaitIndex have changed
		if meta.LastIndex <= f.lastChangeIndex {
			log.Infof("Evaluations index is unchanged (%d <= %d)", meta.LastIndex, f.lastChangeIndex)
			continue
		}

		log.Infof("Evaluations index is changed (%d <> %d)", meta.LastIndex, f.lastChangeIndex)

		// Iterate clients and find events that have changed since last run
		for _, evaluation := range evaluations {
			if evaluation.ModifyIndex <= f.lastChangeIndex {
				continue
			}

			f.Publish(evaluation)
		}

		// Update WaitIndex and Last Change Time for next iteration
		f.lastChangeIndex = meta.LastIndex
		q.WaitIndex = meta.LastIndex
	}
}

// Close the stopCh if we get a signal, so we can gracefully shut down
func (f *Firehose) signalHandler() {
	c := make(chan os.Signal, 1)
	signal.Notify(c, os.Interrupt)

	select {
	case <-c:
		log.Info("Caught signal, releasing lock and stopping...")
		f.Stop()
	case <-f.stopCh:
		break
	}
}
<commit_msg>manually clear properties<commit_after>package evaluations

import (
	"encoding/json"
	"os"
	"os/signal"
	"strconv"
	"time"

	consul "github.com/hashicorp/consul/api"
	nomad "github.com/hashicorp/nomad/api"
	"github.com/seatgeek/nomad-firehose/helper"
	"github.com/seatgeek/nomad-firehose/sink"
	log "github.com/sirupsen/logrus"
)

const (
	consulLockKey   = "nomad-firehose/evaluations.lock"
	consulLockValue = "nomad-firehose/evaluations.value"
)

// Firehose ...
type Firehose struct {
	nomadClient     *nomad.Client
	consulClient    *consul.Client
	consulSessionID string
	consulLock      *consul.Lock
	stopCh          chan struct{}
	lastChangeIndex uint64
	sink            sink.Sink
}

// NewFirehose ...
func NewFirehose() (*Firehose, error) {
	lock, sessionID, err := helper.WaitForLock(consulLockKey)
	if err != nil {
		return nil, err
	}
	defer lock.Unlock()

	nomadClient, err := nomad.NewClient(nomad.DefaultConfig())
	if err != nil {
		return nil, err
	}

	consulClient, err := consul.NewClient(consul.DefaultConfig())
	if err != nil {
		return nil, err
	}

	sink, err := sink.GetSink()
	if err != nil {
		log.Fatal(err)
		os.Exit(1)
	}

	return &Firehose{
		nomadClient:     nomadClient,
		consulClient:    consulClient,
		consulSessionID: sessionID,
		consulLock:      lock,
		sink:            sink,
	}, nil
}

// Start the firehose
func (f *Firehose) Start() error {
	// Restore the last change time from Consul
	err := f.restoreLastChangeTime()
	if err != nil {
		return err
	}

	go f.sink.Start()

	// Stop chan for all tasks to depend on
	f.stopCh = make(chan struct{})

	// setup signal handler for graceful shutdown
	go f.signalHandler()

	// watch for allocation changes
	go f.watch()

	// Save the last event time every 10s
	go f.persistLastChangeTime(10)

	// wait forever for a stop signal to happen
	for {
		select {
		case <-f.stopCh:
			return nil
		}
	}
}

// Stop the firehose
func (f *Firehose) Stop() {
	close(f.stopCh)
	f.sink.Stop()
	f.writeLastChangeTime()
}

// Read the Last Change Time from Consul KV, so we don't re-process tasks over and over on restart
func (f *Firehose) restoreLastChangeTime() error {
	kv, _, err := f.consulClient.KV().Get(consulLockValue, &consul.QueryOptions{})
	if err != nil {
		return err
	}

	// Ensure we got
	if kv != nil && kv.Value != nil {
		sv := string(kv.Value)
		v, err := strconv.ParseInt(sv, 10, 64)
		if err != nil {
			return err
		}

		f.lastChangeIndex = uint64(v)
		log.Infof("Restoring Last Change Time to %s", sv)
	} else {
		log.Info("No Last Change Time restore point, starting from scratch")
	}

	return nil
}

// Write the Last Change Time to Consul so if the process restarts,
// it will try to resume from where it left off, not emitting tons of double events for
// old events
func (f *Firehose) persistLastChangeTime(interval time.Duration) {
	ticker := time.NewTicker(interval * time.Second)

	for {
		select {
		case <-f.stopCh:
			break
		case <-ticker.C:
			f.writeLastChangeTime()
		}
	}
}

func (f *Firehose) writeLastChangeTime() {
	v := strconv.FormatUint(f.lastChangeIndex, 10)

	log.Infof("Writing lastChangedTime to KV: %s", v)
	kv := &consul.KVPair{
		Key:     consulLockValue,
		Value:   []byte(v),
		Session: f.consulSessionID,
	}
	_, err := f.consulClient.KV().Put(kv, &consul.WriteOptions{})
	if err != nil {
		log.Error(err)
	}
}

// Publish an update from the firehose
func (f *Firehose) Publish(update *nomad.Evaluation) {
	b, err := json.Marshal(update)
	if err != nil {
		log.Error(err)
	}

	f.sink.Put(b)
}

// Continously watch for changes to the allocation list and publish it as updates
func (f *Firehose) watch() {
	q := &nomad.QueryOptions{
		WaitIndex:  f.lastChangeIndex,
		WaitTime:   5 * time.Minute,
		AllowStale: true,
	}

	for {
		log.Infof("Fetching evaluations from Nomad: %+v", q)

		evaluations, meta, err := f.nomadClient.Evaluations().List(q)
		if err != nil {
			log.Errorf("Unable to fetch evaluations: %s", err)
			time.Sleep(10 * time.Second)
			continue
		}

		// Only work if the WaitIndex have changed
		if meta.LastIndex <= f.lastChangeIndex {
			log.Infof("Evaluations index is unchanged (%d <= %d)", meta.LastIndex, f.lastChangeIndex)
			continue
		}

		log.Infof("Evaluations index is changed (%d <> %d)", meta.LastIndex, f.lastChangeIndex)

		// Iterate clients and find events that have changed since last run
		for _, evaluation := range evaluations {
			if evaluation.ModifyIndex <= f.lastChangeIndex {
				continue
			}

			f.Publish(evaluation)
			evaluation = nil
		}

		evaluations = nil

		// Update WaitIndex and Last Change Time for next iteration
		f.lastChangeIndex = meta.LastIndex
		q.WaitIndex = meta.LastIndex
	}
}

// Close the stopCh if we get a signal, so we can gracefully shut down
func (f *Firehose) signalHandler() {
	c := make(chan os.Signal, 1)
	signal.Notify(c, os.Interrupt)

	select {
	case <-c:
		log.Info("Caught signal, releasing lock and stopping...")
		f.Stop()
	case <-f.stopCh:
		break
	}
}
<|endoftext|>


<commit_before>package cli

import (
	"strings"
	"testing"
	"io"
	"io/ioutil"
	"os"

	"github.com/ipfs/go-ipfs/commands"
)

type kvs map[string]interface{}
type words []string

func sameWords(a words, b words) bool {
	if len(a) != len(b) {
		return false
	}
	for i, w := range a {
		if w != b[i] {
			return false
		}
	}
	return true
}

func sameKVs(a kvs, b kvs) bool {
	if len(a) != len(b) {
		return false
	}
	for k, v := range a {
		if v != b[k] {
			return false
		}
	}
	return true
}

func TestSameWords(t *testing.T) {
	a := []string{"v1", "v2"}
	b := []string{"v1", "v2", "v3"}
	c := []string{"v2", "v3"}
	d := []string{"v2"}
	e := []string{"v2", "v3"}
	f := []string{"v2", "v1"}

	test := func(a words, b words, v bool) {
		if sameWords(a, b) != v {
			t.Errorf("sameWords('%v', '%v') != %v", a, b, v)
		}
	}

	test(a, b, false)
	test(a, a, true)
	test(a, c, false)
	test(b, c, false)
	test(c, d, false)
	test(c, e, true)
	test(b, e, false)
	test(a, b, false)
	test(a, f, false)
	test(e, f, false)
	test(f, f, true)
}

func TestOptionParsing(t *testing.T) {
	subCmd := &commands.Command{}
	cmd := &commands.Command{
		Options: []commands.Option{
			commands.StringOption("string", "s", "a string"),
			commands.BoolOption("bool", "b", "a bool"),
		},
		Subcommands: map[string]*commands.Command{
			"test": subCmd,
		},
	}

	testHelper := func(args string, expectedOpts kvs, expectedWords words, expectErr bool) {
		_, opts, input, _, err := parseOpts(strings.Split(args, " "), cmd)
		if expectErr {
			if err == nil {
				t.Errorf("Command line '%v' parsing should have failed", args)
			}
		} else if err != nil {
			t.Errorf("Command line '%v' failed to parse: %v", args, err)
		} else if !sameWords(input, expectedWords) || !sameKVs(opts, expectedOpts) {
			t.Errorf("Command line '%v':\n  parsed as  %v %v\n  instead of %v %v",
				args, opts, input, expectedOpts, expectedWords)
		}
	}

	testFail := func(args string) {
		testHelper(args, kvs{}, words{}, true)
	}

	test := func(args string, expectedOpts kvs, expectedWords words) {
		testHelper(args, expectedOpts, expectedWords, false)
	}

	test("-", kvs{}, words{"-"})
	testFail("-b -b")
	test("beep boop", kvs{}, words{"beep", "boop"})
	test("test beep boop", kvs{}, words{"beep", "boop"})
	testFail("-s")
	test("-s foo", kvs{"s": "foo"}, words{})
	test("-sfoo", kvs{"s": "foo"}, words{})
	test("-s=foo", kvs{"s": "foo"}, words{})
	test("-b", kvs{"b": ""}, words{})
	test("-bs foo", kvs{"b": "", "s": "foo"}, words{})
	test("-sb", kvs{"s": "b"}, words{})
	test("-b foo", kvs{"b": ""}, words{"foo"})
	test("--bool foo", kvs{"bool": ""}, words{"foo"})
	testFail("--bool=foo")
	testFail("--string")
	test("--string foo", kvs{"string": "foo"}, words{})
	test("--string=foo", kvs{"string": "foo"}, words{})
	test("-- -b", kvs{}, words{"-b"})
	test("foo -b", kvs{"b": ""}, words{"foo"})
}

func TestArgumentParsing(t *testing.T) {
	rootCmd := &commands.Command{
		Subcommands: map[string]*commands.Command{
			"noarg": &commands.Command{},
			"onearg": &commands.Command{
				Arguments: []commands.Argument{
					commands.StringArg("a", true, false, "some arg"),
				},
			},
			"twoargs": &commands.Command{
				Arguments: []commands.Argument{
					commands.StringArg("a", true, false, "some arg"),
					commands.StringArg("b", true, false, "another arg"),
				},
			},
			"variadic": &commands.Command{
				Arguments: []commands.Argument{
					commands.StringArg("a", true, true, "some arg"),
				},
			},
			"optional": &commands.Command{
				Arguments: []commands.Argument{
					commands.StringArg("b", false, true, "another arg"),
				},
			},
			"reversedoptional": &commands.Command{
				Arguments: []commands.Argument{
					commands.StringArg("a", false, false, "some arg"),
					commands.StringArg("b", true, false, "another arg"),
				},
			},
			"stdinenabled": &commands.Command{
				Arguments: []commands.Argument{
					commands.StringArg("a", true, true, "some arg").EnableStdin(),
				},
			},
		},
	}

	test := func(cmd words, f *os.File, res words) {
		if f != nil {
			if _, err := f.Seek(0, os.SEEK_SET); err != nil {
				t.Fatal(err)
			}
		}
		req, _, _, err := Parse(cmd, f, rootCmd)
		if err != nil {
			t.Errorf("Command '%v' should have passed parsing", cmd)
		}
		if !sameWords(req.Arguments(), res) {
			t.Errorf("Arguments parsed from '%v' are not '%v'", cmd, res)
		}
	}

	testFail := func(cmd words, msg string) {
		_, _, _, err := Parse(cmd, nil, rootCmd)
		if err == nil {
			t.Errorf("Should have failed: %v", msg)
		}
	}

	test([]string{"noarg"}, nil, []string{})
	testFail([]string{"noarg", "value!"}, "provided an arg, but command didn't define any")

	test([]string{"onearg", "value!"}, nil, []string{"value!"})
	testFail([]string{"onearg"}, "didn't provide any args, arg is required")

	test([]string{"twoargs", "value1", "value2"}, nil, []string{"value1", "value2"})
	testFail([]string{"twoargs", "value!"}, "only provided 1 arg, needs 2")
	testFail([]string{"twoargs"}, "didn't provide any args, 2 required")

	test([]string{"variadic", "value!"}, nil, []string{"value!"})
	test([]string{"variadic", "value1", "value2", "value3"}, nil, []string{"value1", "value2", "value3"})
	testFail([]string{"variadic"}, "didn't provide any args, 1 required")

	test([]string{"optional", "value!"}, nil, []string{"value!"})
	test([]string{"optional"}, nil, []string{})

	test([]string{"reversedoptional", "value1", "value2"}, nil, []string{"value1", "value2"})
	test([]string{"reversedoptional", "value!"}, nil, []string{"value!"})

	testFail([]string{"reversedoptional"}, "didn't provide any args, 1 required")
	testFail([]string{"reversedoptional", "value1", "value2", "value3"}, "provided too many args, only takes 1")

	// Use a temp file to simulate stdin
	fileToSimulateStdin := func(t *testing.T, content string) (*os.File) {
		fstdin, err := ioutil.TempFile("", "")
		if err != nil {
			t.Fatal(err)
		}
		defer os.Remove(fstdin.Name())

		if _, err := io.WriteString(fstdin, content); err != nil {
			t.Fatal(err)
		}
		return fstdin
	}

	test([]string{"stdinenabled", "value1", "value2"}, nil, []string{"value1", "value2"})

	fstdin := fileToSimulateStdin(t, "stdin1")

	test([]string{"stdinenabled"}, fstdin, []string{"stdin1"})
	test([]string{"stdinenabled", "value1"}, fstdin, []string{"stdin1", "value1"})
	test([]string{"stdinenabled", "value1", "value2"}, fstdin, []string{"stdin1", "value1", "value2"})
}
<commit_msg>parse_test: test multiline stdin<commit_after>package cli

import (
	"strings"
	"testing"
	"io"
	"io/ioutil"
	"os"

	"github.com/ipfs/go-ipfs/commands"
)

type kvs map[string]interface{}
type words []string

func sameWords(a words, b words) bool {
	if len(a) != len(b) {
		return false
	}
	for i, w := range a {
		if w != b[i] {
			return false
		}
	}
	return true
}

func sameKVs(a kvs, b kvs) bool {
	if len(a) != len(b) {
		return false
	}
	for k, v := range a {
		if v != b[k] {
			return false
		}
	}
	return true
}

func TestSameWords(t *testing.T) {
	a := []string{"v1", "v2"}
	b := []string{"v1", "v2", "v3"}
	c := []string{"v2", "v3"}
	d := []string{"v2"}
	e := []string{"v2", "v3"}
	f := []string{"v2", "v1"}

	test := func(a words, b words, v bool) {
		if sameWords(a, b) != v {
			t.Errorf("sameWords('%v', '%v') != %v", a, b, v)
		}
	}

	test(a, b, false)
	test(a, a, true)
	test(a, c, false)
	test(b, c, false)
	test(c, d, false)
	test(c, e, true)
	test(b, e, false)
	test(a, b, false)
	test(a, f, false)
	test(e, f, false)
	test(f, f, true)
}

func TestOptionParsing(t *testing.T) {
	subCmd := &commands.Command{}
	cmd := &commands.Command{
		Options: []commands.Option{
			commands.StringOption("string", "s", "a string"),
			commands.BoolOption("bool", "b", "a bool"),
		},
		Subcommands: map[string]*commands.Command{
			"test": subCmd,
		},
	}

	testHelper := func(args string, expectedOpts kvs, expectedWords words, expectErr bool) {
		_, opts, input, _, err := parseOpts(strings.Split(args, " "), cmd)
		if expectErr {
			if err == nil {
				t.Errorf("Command line '%v' parsing should have failed", args)
			}
		} else if err != nil {
			t.Errorf("Command line '%v' failed to parse: %v", args, err)
		} else if !sameWords(input, expectedWords) || !sameKVs(opts, expectedOpts) {
			t.Errorf("Command line '%v':\n  parsed as  %v %v\n  instead of %v %v",
				args, opts, input, expectedOpts, expectedWords)
		}
	}

	testFail := func(args string) {
		testHelper(args, kvs{}, words{}, true)
	}

	test := func(args string, expectedOpts kvs, expectedWords words) {
		testHelper(args, expectedOpts, expectedWords, false)
	}

	test("-", kvs{}, words{"-"})
	testFail("-b -b")
	test("beep boop", kvs{}, words{"beep", "boop"})
	test("test beep boop", kvs{}, words{"beep", "boop"})
	testFail("-s")
	test("-s foo", kvs{"s": "foo"}, words{})
	test("-sfoo", kvs{"s": "foo"}, words{})
	test("-s=foo", kvs{"s": "foo"}, words{})
	test("-b", kvs{"b": ""}, words{})
	test("-bs foo", kvs{"b": "", "s": "foo"}, words{})
	test("-sb", kvs{"s": "b"}, words{})
	test("-b foo", kvs{"b": ""}, words{"foo"})
	test("--bool foo", kvs{"bool": ""}, words{"foo"})
	testFail("--bool=foo")
	testFail("--string")
	test("--string foo", kvs{"string": "foo"}, words{})
	test("--string=foo", kvs{"string": "foo"}, words{})
	test("-- -b", kvs{}, words{"-b"})
	test("foo -b", kvs{"b": ""}, words{"foo"})
}

func TestArgumentParsing(t *testing.T) {
	rootCmd := &commands.Command{
		Subcommands: map[string]*commands.Command{
			"noarg": &commands.Command{},
			"onearg": &commands.Command{
				Arguments: []commands.Argument{
					commands.StringArg("a", true, false, "some arg"),
				},
			},
			"twoargs": &commands.Command{
				Arguments: []commands.Argument{
					commands.StringArg("a", true, false, "some arg"),
					commands.StringArg("b", true, false, "another arg"),
				},
			},
			"variadic": &commands.Command{
				Arguments: []commands.Argument{
					commands.StringArg("a", true, true, "some arg"),
				},
			},
			"optional": &commands.Command{
				Arguments: []commands.Argument{
					commands.StringArg("b", false, true, "another arg"),
				},
			},
			"reversedoptional": &commands.Command{
				Arguments: []commands.Argument{
					commands.StringArg("a", false, false, "some arg"),
					commands.StringArg("b", true, false, "another arg"),
				},
			},
			"stdinenabled": &commands.Command{
				Arguments: []commands.Argument{
					commands.StringArg("a", true, true, "some arg").EnableStdin(),
				},
			},
		},
	}

	test := func(cmd words, f *os.File, res words) {
		if f != nil {
			if _, err := f.Seek(0, os.SEEK_SET); err != nil {
				t.Fatal(err)
			}
		}
		req, _, _, err := Parse(cmd, f, rootCmd)
		if err != nil {
			t.Errorf("Command '%v' should have passed parsing", cmd)
		}
		if !sameWords(req.Arguments(), res) {
			t.Errorf("Arguments parsed from '%v' are not '%v'", cmd, res)
		}
	}

	testFail := func(cmd words, msg string) {
		_, _, _, err := Parse(cmd, nil, rootCmd)
		if err == nil {
			t.Errorf("Should have failed: %v", msg)
		}
	}

	test([]string{"noarg"}, nil, []string{})
	testFail([]string{"noarg", "value!"}, "provided an arg, but command didn't define any")

	test([]string{"onearg", "value!"}, nil, []string{"value!"})
	testFail([]string{"onearg"}, "didn't provide any args, arg is required")

	test([]string{"twoargs", "value1", "value2"}, nil, []string{"value1", "value2"})
	testFail([]string{"twoargs", "value!"}, "only provided 1 arg, needs 2")
	testFail([]string{"twoargs"}, "didn't provide any args, 2 required")

	test([]string{"variadic", "value!"}, nil, []string{"value!"})
	test([]string{"variadic", "value1", "value2", "value3"}, nil, []string{"value1", "value2", "value3"})
	testFail([]string{"variadic"}, "didn't provide any args, 1 required")

	test([]string{"optional", "value!"}, nil, []string{"value!"})
	test([]string{"optional"}, nil, []string{})

	test([]string{"reversedoptional", "value1", "value2"}, nil, []string{"value1", "value2"})
	test([]string{"reversedoptional", "value!"}, nil, []string{"value!"})

	testFail([]string{"reversedoptional"}, "didn't provide any args, 1 required")
	testFail([]string{"reversedoptional", "value1", "value2", "value3"}, "provided too many args, only takes 1")

	// Use a temp file to simulate stdin
	fileToSimulateStdin := func(t *testing.T, content string) (*os.File) {
		fstdin, err := ioutil.TempFile("", "")
		if err != nil {
			t.Fatal(err)
		}
		defer os.Remove(fstdin.Name())

		if _, err := io.WriteString(fstdin, content); err != nil {
			t.Fatal(err)
		}
		return fstdin
	}

	test([]string{"stdinenabled", "value1", "value2"}, nil, []string{"value1", "value2"})

	fstdin := fileToSimulateStdin(t, "stdin1")

	test([]string{"stdinenabled"}, fstdin, []string{"stdin1"})
	test([]string{"stdinenabled", "value1"}, fstdin, []string{"stdin1", "value1"})
	test([]string{"stdinenabled", "value1", "value2"}, fstdin, []string{"stdin1", "value1", "value2"})

	fstdin = fileToSimulateStdin(t, "stdin1\nstdin2")
	test([]string{"stdinenabled"}, fstdin, []string{"stdin1", "stdin2"})

	fstdin = fileToSimulateStdin(t, "stdin1\nstdin2\nstdin3")
	test([]string{"stdinenabled"}, fstdin, []string{"stdin1", "stdin2", "stdin3"})
}
<|endoftext|>