<commit_before>// Copyright 2016-2019 Google LLC. All Rights Reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package fetch

import (
	"encoding/json"
	"errors"
	"fmt"
	"hash/crc64"
	"io"
	"io/ioutil"
	"math"
	"math/rand"
	"net/http"
	"net/url"
	"os"
	"path/filepath"
	"strings"
	"time"

	"github.com/googlecodelabs/tools/claat/fetch/drive/auth"
	"github.com/googlecodelabs/tools/claat/parser"
	"github.com/googlecodelabs/tools/claat/types"
	"github.com/googlecodelabs/tools/claat/util"
)

const (
	// supported codelab source types must be registered parsers
	// TODO: define these in claat/parser/..., e.g. in parser/gdoc
	// alternate TODO: make this an iota-based enum?
	SrcInvalid   srcType = ""
	SrcGoogleDoc srcType = "gdoc" // Google Docs doc
	SrcMarkdown  srcType = "md"   // Markdown text

	// driveAPI is a base URL for Drive API
	driveAPI = "https://www.googleapis.com/drive/v3"
)

// TODO: create an enum for use with "nometa" for readability's sake

// srcType is codelab source type
type srcType string

// resource is a codelab resource, loaded from local file
// or fetched from remote location.
type resource struct {
	typ  srcType       // source type
	body io.ReadCloser // resource body
	mod  time.Time     // last update of content
}

// codelab wraps types.Codelab, while adding source type
// and modified timestamp fields.
type codelab struct {
	*types.Codelab
	Typ srcType   //  source type
	Mod time.Time // last modified timestamp
}

type Fetcher struct {
	authHelper   *auth.Helper
	authToken    string
	crcTable     *crc64.Table
	passMetadata map[string]bool
}

func NewFetcher(at string, pm map[string]bool, rt http.RoundTripper) (*Fetcher, error) {
	h, err := auth.NewHelper(at, auth.ProviderGoogle, rt)
	if err != nil {
		return nil, err
	}

	return &Fetcher{
		authHelper:   h,
		authToken:    at,
		crcTable:     crc64.MakeTable(crc64.ECMA),
		passMetadata: pm,
	}, nil
}

// SlurpCodelab retrieves and parses codelab source.
// It takes the source, plus an auth token and a set of extra metadata to pass along.
// It returns parsed codelab and its source type.
//
// The function will also fetch and parse fragments included
// with types.ImportNode.
func (f *Fetcher) SlurpCodelab(src string) (*codelab, error) {
	res, err := f.fetch(src)
	if err != nil {
		return nil, err
	}
	defer res.body.Close()

	opts := *parser.NewOptions()
	opts.PassMetadata = f.passMetadata

	clab, err := parser.Parse(string(res.typ), res.body, opts)
	if err != nil {
		return nil, err
	}

	// fetch imports and parse them as fragments
	var imports []*types.ImportNode
	for _, st := range clab.Steps {
		imports = append(imports, types.ImportNodes(st.Content.Nodes)...)
	}
	ch := make(chan error, len(imports))
	defer close(ch)
	for _, imp := range imports {
		go func(n *types.ImportNode) {
			frag, err := f.slurpFragment(n.URL)
			if err != nil {
				ch <- fmt.Errorf("%s: %v", n.URL, err)
				return
			}
			n.Content.Nodes = frag
			ch <- nil
		}(imp)
	}
	for range imports {
		if err := <-ch; err != nil {
			return nil, err
		}
	}

	v := &codelab{
		Codelab: clab,
		Typ:     res.typ,
		Mod:     res.mod,
	}
	return v, nil
}

func (f *Fetcher) SlurpImages(src, dir string, steps []*types.Step) (map[string]string, error) {
	// make sure img dir exists
	if err := os.MkdirAll(dir, 0755); err != nil {
		return nil, err
	}

	type res struct {
		url, file string
		err       error
	}

	ch := make(chan *res, 100)
	defer close(ch)
	var count int
	for _, st := range steps {
		nodes := types.ImageNodes(st.Content.Nodes)
		count += len(nodes)
		for _, n := range nodes {
			go func(n *types.ImageNode) {
				url := n.Src
				file, err := f.slurpBytes(src, dir, url)
				if err == nil {
					n.Src = filepath.Join(util.ImgDirname, file)
				}
				ch <- &res{url, file, err}
			}(n)
		}
	}

	imap := make(map[string]string, count)
	errBuilder := strings.Builder{}
	for i := 0; i < count; i++ {
		r := <-ch
		imap[r.file] = r.url
		if r.err != nil {
			errBuilder.WriteString(fmt.Sprintf("%s => %s: %v\n", r.url, r.file, r.err))
		}
	}
	if errBuilder.Len() > 0 {
		return nil, errors.New(errBuilder.String())
	}

	return imap, nil
}

func (f *Fetcher) slurpBytes(codelabSrc, dir, imgURL string) (string, error) {
	// images can be local in Markdown cases or remote.
	// Only proceed a simple copy on local reference.
	var b []byte
	var ext string
	u, err := url.Parse(imgURL)
	if err != nil {
		return "", err
	}

	// If the codelab source is being downloaded from the network, then we should interpret
	// the image URL in the same way.
	srcUrl, err := url.Parse(codelabSrc)
	if err == nil && srcUrl.Host != "" {
		u = srcUrl.ResolveReference(u)
	}

	if u.Host == "" {
		if imgURL, err = restrictPathToParent(imgURL, filepath.Dir(codelabSrc)); err != nil {
			return "", err
		}
		b, err = ioutil.ReadFile(imgURL)
		ext = filepath.Ext(imgURL)
	} else {
		b, err = f.slurpRemoteBytes(u.String(), 5)
		if string(b[6:10]) == "JFIF" {
			ext = ".jpeg"
		} else if string(b[0:3]) == "GIF" {
			ext = ".gif"
		} else {
			ext = ".png"
		}
	}
	if err != nil {
		return "", err
	}

	crc := crc64.Checksum(b, f.crcTable)
	file := fmt.Sprintf("%x%s", crc, ext)
	dst := filepath.Join(dir, file)
	return file, ioutil.WriteFile(dst, b, 0644)
}

func (f *Fetcher) slurpFragment(url string) ([]types.Node, error) {
	res, err := f.fetchRemote(url, true)
	if err != nil {
		return nil, err
	}
	defer res.body.Close()
	return parser.ParseFragment(string(res.typ), res.body)
}

// fetch retrieves codelab doc either from local disk
// or a remote location.
// The caller is responsible for closing returned stream.
func (f *Fetcher) fetch(name string) (*resource, error) {
	fi, err := os.Stat(name)
	if os.IsNotExist(err) {
		return f.fetchRemote(name, false)
	}
	r, err := os.Open(name)
	if err != nil {
		return nil, err
	}
	return &resource{
		body: r,
		typ:  SrcMarkdown,
		mod:  fi.ModTime(),
	}, nil
}

// fetchRemote retrieves resource r from the network.
//
// If urlStr is not a URL, i.e. does not have the host part, it is considered to be
// a Google Doc ID and fetched accordingly. Otherwise, a simple GET request
// is used to retrieve the contents.
//
// The caller is responsible for closing returned stream.
// If nometa is true, resource.mod may have zero value.
func (f *Fetcher) fetchRemote(urlStr string, nometa bool) (*resource, error) {
	u, err := url.Parse(urlStr)
	if err != nil {
		return nil, err
	}
	if u.Host == "" || u.Host == "docs.google.com" {
		return f.fetchDriveFile(urlStr, nometa)
	}
	return f.fetchRemoteFile(urlStr)
}

// fetchRemoteFile retrieves codelab resource from url.
// It is a special case of fetchRemote function.
func (f *Fetcher) fetchRemoteFile(url string) (*resource, error) {
	res, err := retryGet(f.authHelper.DriveClient(), url, 3)
	if err != nil {
		return nil, err
	}
	t, err := http.ParseTime(res.Header.Get("last-modified"))
	if err != nil {
		t = time.Now()
	}
	return &resource{
		body: res.Body,
		mod:  t,
		typ:  SrcMarkdown,
	}, nil
}

// fetchDriveFile uses Drive API to retrieve HTML representation of a Google Doc.
// See https://developers.google.com/drive/web/manage-downloads#downloading_google_documents
// for more details.
//
// If nometa is true, resource.mod will have zero value.
func (f *Fetcher) fetchDriveFile(id string, nometa bool) (*resource, error) {
	id = gdocID(id)
	exportURL := gdocExportURL(id)

	if nometa {
		res, err := retryGet(f.authHelper.DriveClient(), exportURL, 7)
		if err != nil {
			return nil, err
		}
		return &resource{body: res.Body, typ: SrcGoogleDoc}, nil
	}

	q := url.Values{
		"fields":             {"id,mimeType,modifiedTime"},
		"supportsTeamDrives": {"true"},
	}
	u := fmt.Sprintf("%s/files/%s?%s", driveAPI, id, q.Encode())
	res, err := retryGet(f.authHelper.DriveClient(), u, 7)
	if err != nil {
		return nil, err
	}
	defer res.Body.Close()
	meta := &struct {
		ID       string    `json:"id"`
		MimeType string    `json:"mimeType"`
		Modified time.Time `json:"modifiedTime"`
	}{}
	if err := json.NewDecoder(res.Body).Decode(meta); err != nil {
		return nil, err
	}
	if meta.MimeType != "application/vnd.google-apps.document" {
		return nil, fmt.Errorf("%s: invalid mime type: %s", id, meta.MimeType)
	}

	if res, err = retryGet(f.authHelper.DriveClient(), exportURL, 7); err != nil {
		return nil, err
	}
	return &resource{
		body: res.Body,
		mod:  meta.Modified,
		typ:  SrcGoogleDoc,
	}, nil
}

func (f *Fetcher) slurpRemoteBytes(url string, n int) ([]byte, error) {
	res, err := retryGet(f.authHelper.DriveClient(), url, n)
	if err != nil {
		return nil, err
	}
	defer res.Body.Close()
	return ioutil.ReadAll(res.Body)
}

// retryGet tries to GET specified url up to n times.
// Attempts are spaced out with exponential backoff.
// Default client will be used if not provided.
func retryGet(client *http.Client, url string, n int) (*http.Response, error) {
	if client == nil {
		client = http.DefaultClient
	}
	for i := 0; i <= n; i++ {
		if i > 0 {
			t := time.Duration((math.Pow(2, float64(i)) + rand.Float64()) * float64(time.Second))
			time.Sleep(t)
		}
		res, err := client.Get(url)
		// return early with a good response
		// the rest is error handling
		if err == nil && res.StatusCode == http.StatusOK {
			return res, nil
		}

		// sometimes Drive API wouldn't even start a response,
		// we get net/http: TLS handshake timeout instead:
		// consider this a temporary failure and retry again
		if err != nil {
			continue
		}
		// otherwise, decode error response and check for "rate limit"
		defer res.Body.Close()
		var erres struct {
			Error struct {
				Errors []struct{ Reason string }
			}
		}
		b, _ := ioutil.ReadAll(res.Body)
		json.Unmarshal(b, &erres)
		var rateLimit bool
		for _, e := range erres.Error.Errors {
			if e.Reason == "rateLimitExceeded" || e.Reason == "userRateLimitExceeded" {
				rateLimit = true
				break
			}
		}
		// this is neither a rate limit error, nor a server error:
		// retrying is useless
		if !rateLimit && res.StatusCode < http.StatusInternalServerError {
			return nil, fmt.Errorf("fetch %s: %s; %s", url, res.Status, b)
		}
	}
	return nil, fmt.Errorf("%s: failed after %d retries", url, n)
}

func gdocID(url string) string {
	const s = "/document/d/"
	if i := strings.Index(url, s); i >= 0 {
		url = url[i+len(s):]
	}
	if i := strings.IndexRune(url, '/'); i > 0 {
		url = url[:i]
	}
	return url
}

func gdocExportURL(id string) string {
	return fmt.Sprintf("%s/files/%s/export?mimeType=text/html", driveAPI, id)
}

// restrictPathToParent will ensure that assetPath is in parent.
// It will thus return an absolute path to the asset.
func restrictPathToParent(assetPath, parent string) (string, error) {
	parent, err := filepath.Abs(parent)
	if err != nil {
		return "", err
	}
	if !filepath.IsAbs(assetPath) {
		assetPath = filepath.Join(parent, assetPath)
	}
	if !strings.HasPrefix(assetPath, parent) {
		return "", fmt.Errorf("%s isn't a subdirectory of %s", assetPath, parent)
	}
	return assetPath, nil
}
<commit_msg>Remove strings.Builder usage from fetch.go.<commit_after>// Copyright 2016-2019 Google LLC. All Rights Reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package fetch

import (
	"encoding/json"
	"errors"
	"fmt"
	"hash/crc64"
	"io"
	"io/ioutil"
	"math"
	"math/rand"
	"net/http"
	"net/url"
	"os"
	"path/filepath"
	"strings"
	"time"

	"github.com/googlecodelabs/tools/claat/fetch/drive/auth"
	"github.com/googlecodelabs/tools/claat/parser"
	"github.com/googlecodelabs/tools/claat/types"
	"github.com/googlecodelabs/tools/claat/util"
)

const (
	// supported codelab source types must be registered parsers
	// TODO: define these in claat/parser/..., e.g. in parser/gdoc
	// alternate TODO: make this an iota-based enum?
	SrcInvalid   srcType = ""
	SrcGoogleDoc srcType = "gdoc" // Google Docs doc
	SrcMarkdown  srcType = "md"   // Markdown text

	// driveAPI is a base URL for Drive API
	driveAPI = "https://www.googleapis.com/drive/v3"
)

// TODO: create an enum for use with "nometa" for readability's sake

// srcType is codelab source type
type srcType string

// resource is a codelab resource, loaded from local file
// or fetched from remote location.
type resource struct {
	typ  srcType       // source type
	body io.ReadCloser // resource body
	mod  time.Time     // last update of content
}

// codelab wraps types.Codelab, while adding source type
// and modified timestamp fields.
type codelab struct {
	*types.Codelab
	Typ srcType   //  source type
	Mod time.Time // last modified timestamp
}

type Fetcher struct {
	authHelper   *auth.Helper
	authToken    string
	crcTable     *crc64.Table
	passMetadata map[string]bool
}

func NewFetcher(at string, pm map[string]bool, rt http.RoundTripper) (*Fetcher, error) {
	h, err := auth.NewHelper(at, auth.ProviderGoogle, rt)
	if err != nil {
		return nil, err
	}

	return &Fetcher{
		authHelper:   h,
		authToken:    at,
		crcTable:     crc64.MakeTable(crc64.ECMA),
		passMetadata: pm,
	}, nil
}

// SlurpCodelab retrieves and parses codelab source.
// It takes the source, plus an auth token and a set of extra metadata to pass along.
// It returns parsed codelab and its source type.
//
// The function will also fetch and parse fragments included
// with types.ImportNode.
func (f *Fetcher) SlurpCodelab(src string) (*codelab, error) {
	res, err := f.fetch(src)
	if err != nil {
		return nil, err
	}
	defer res.body.Close()

	opts := *parser.NewOptions()
	opts.PassMetadata = f.passMetadata

	clab, err := parser.Parse(string(res.typ), res.body, opts)
	if err != nil {
		return nil, err
	}

	// fetch imports and parse them as fragments
	var imports []*types.ImportNode
	for _, st := range clab.Steps {
		imports = append(imports, types.ImportNodes(st.Content.Nodes)...)
	}
	ch := make(chan error, len(imports))
	defer close(ch)
	for _, imp := range imports {
		go func(n *types.ImportNode) {
			frag, err := f.slurpFragment(n.URL)
			if err != nil {
				ch <- fmt.Errorf("%s: %v", n.URL, err)
				return
			}
			n.Content.Nodes = frag
			ch <- nil
		}(imp)
	}
	for range imports {
		if err := <-ch; err != nil {
			return nil, err
		}
	}

	v := &codelab{
		Codelab: clab,
		Typ:     res.typ,
		Mod:     res.mod,
	}
	return v, nil
}

func (f *Fetcher) SlurpImages(src, dir string, steps []*types.Step) (map[string]string, error) {
	// make sure img dir exists
	if err := os.MkdirAll(dir, 0755); err != nil {
		return nil, err
	}

	type res struct {
		url, file string
		err       error
	}

	ch := make(chan *res, 100)
	defer close(ch)
	var count int
	for _, st := range steps {
		nodes := types.ImageNodes(st.Content.Nodes)
		count += len(nodes)
		for _, n := range nodes {
			go func(n *types.ImageNode) {
				url := n.Src
				file, err := f.slurpBytes(src, dir, url)
				if err == nil {
					n.Src = filepath.Join(util.ImgDirname, file)
				}
				ch <- &res{url, file, err}
			}(n)
		}
	}

	imap := make(map[string]string, count)
	var errStr string
	for i := 0; i < count; i++ {
		r := <-ch
		imap[r.file] = r.url
		if r.err != nil {
			errStr += fmt.Sprintf("%s => %s: %v\n", r.url, r.file, r.err)
		}
	}
	if len(errStr) > 0 {
		return nil, errors.New(errStr)
	}

	return imap, nil
}

func (f *Fetcher) slurpBytes(codelabSrc, dir, imgURL string) (string, error) {
	// images can be local in Markdown cases or remote.
	// Only proceed a simple copy on local reference.
	var b []byte
	var ext string
	u, err := url.Parse(imgURL)
	if err != nil {
		return "", err
	}

	// If the codelab source is being downloaded from the network, then we should interpret
	// the image URL in the same way.
	srcUrl, err := url.Parse(codelabSrc)
	if err == nil && srcUrl.Host != "" {
		u = srcUrl.ResolveReference(u)
	}

	if u.Host == "" {
		if imgURL, err = restrictPathToParent(imgURL, filepath.Dir(codelabSrc)); err != nil {
			return "", err
		}
		b, err = ioutil.ReadFile(imgURL)
		ext = filepath.Ext(imgURL)
	} else {
		b, err = f.slurpRemoteBytes(u.String(), 5)
		if string(b[6:10]) == "JFIF" {
			ext = ".jpeg"
		} else if string(b[0:3]) == "GIF" {
			ext = ".gif"
		} else {
			ext = ".png"
		}
	}
	if err != nil {
		return "", err
	}

	crc := crc64.Checksum(b, f.crcTable)
	file := fmt.Sprintf("%x%s", crc, ext)
	dst := filepath.Join(dir, file)
	return file, ioutil.WriteFile(dst, b, 0644)
}

func (f *Fetcher) slurpFragment(url string) ([]types.Node, error) {
	res, err := f.fetchRemote(url, true)
	if err != nil {
		return nil, err
	}
	defer res.body.Close()
	return parser.ParseFragment(string(res.typ), res.body)
}

// fetch retrieves codelab doc either from local disk
// or a remote location.
// The caller is responsible for closing returned stream.
func (f *Fetcher) fetch(name string) (*resource, error) {
	fi, err := os.Stat(name)
	if os.IsNotExist(err) {
		return f.fetchRemote(name, false)
	}
	r, err := os.Open(name)
	if err != nil {
		return nil, err
	}
	return &resource{
		body: r,
		typ:  SrcMarkdown,
		mod:  fi.ModTime(),
	}, nil
}

// fetchRemote retrieves resource r from the network.
//
// If urlStr is not a URL, i.e. does not have the host part, it is considered to be
// a Google Doc ID and fetched accordingly. Otherwise, a simple GET request
// is used to retrieve the contents.
//
// The caller is responsible for closing returned stream.
// If nometa is true, resource.mod may have zero value.
func (f *Fetcher) fetchRemote(urlStr string, nometa bool) (*resource, error) {
	u, err := url.Parse(urlStr)
	if err != nil {
		return nil, err
	}
	if u.Host == "" || u.Host == "docs.google.com" {
		return f.fetchDriveFile(urlStr, nometa)
	}
	return f.fetchRemoteFile(urlStr)
}

// fetchRemoteFile retrieves codelab resource from url.
// It is a special case of fetchRemote function.
func (f *Fetcher) fetchRemoteFile(url string) (*resource, error) {
	res, err := retryGet(f.authHelper.DriveClient(), url, 3)
	if err != nil {
		return nil, err
	}
	t, err := http.ParseTime(res.Header.Get("last-modified"))
	if err != nil {
		t = time.Now()
	}
	return &resource{
		body: res.Body,
		mod:  t,
		typ:  SrcMarkdown,
	}, nil
}

// fetchDriveFile uses Drive API to retrieve HTML representation of a Google Doc.
// See https://developers.google.com/drive/web/manage-downloads#downloading_google_documents
// for more details.
//
// If nometa is true, resource.mod will have zero value.
func (f *Fetcher) fetchDriveFile(id string, nometa bool) (*resource, error) {
	id = gdocID(id)
	exportURL := gdocExportURL(id)

	if nometa {
		res, err := retryGet(f.authHelper.DriveClient(), exportURL, 7)
		if err != nil {
			return nil, err
		}
		return &resource{body: res.Body, typ: SrcGoogleDoc}, nil
	}

	q := url.Values{
		"fields":             {"id,mimeType,modifiedTime"},
		"supportsTeamDrives": {"true"},
	}
	u := fmt.Sprintf("%s/files/%s?%s", driveAPI, id, q.Encode())
	res, err := retryGet(f.authHelper.DriveClient(), u, 7)
	if err != nil {
		return nil, err
	}
	defer res.Body.Close()
	meta := &struct {
		ID       string    `json:"id"`
		MimeType string    `json:"mimeType"`
		Modified time.Time `json:"modifiedTime"`
	}{}
	if err := json.NewDecoder(res.Body).Decode(meta); err != nil {
		return nil, err
	}
	if meta.MimeType != "application/vnd.google-apps.document" {
		return nil, fmt.Errorf("%s: invalid mime type: %s", id, meta.MimeType)
	}

	if res, err = retryGet(f.authHelper.DriveClient(), exportURL, 7); err != nil {
		return nil, err
	}
	return &resource{
		body: res.Body,
		mod:  meta.Modified,
		typ:  SrcGoogleDoc,
	}, nil
}

func (f *Fetcher) slurpRemoteBytes(url string, n int) ([]byte, error) {
	res, err := retryGet(f.authHelper.DriveClient(), url, n)
	if err != nil {
		return nil, err
	}
	defer res.Body.Close()
	return ioutil.ReadAll(res.Body)
}

// retryGet tries to GET specified url up to n times.
// Attempts are spaced out with exponential backoff.
// Default client will be used if not provided.
func retryGet(client *http.Client, url string, n int) (*http.Response, error) {
	if client == nil {
		client = http.DefaultClient
	}
	for i := 0; i <= n; i++ {
		if i > 0 {
			t := time.Duration((math.Pow(2, float64(i)) + rand.Float64()) * float64(time.Second))
			time.Sleep(t)
		}
		res, err := client.Get(url)
		// return early with a good response
		// the rest is error handling
		if err == nil && res.StatusCode == http.StatusOK {
			return res, nil
		}

		// sometimes Drive API wouldn't even start a response,
		// we get net/http: TLS handshake timeout instead:
		// consider this a temporary failure and retry again
		if err != nil {
			continue
		}
		// otherwise, decode error response and check for "rate limit"
		defer res.Body.Close()
		var erres struct {
			Error struct {
				Errors []struct{ Reason string }
			}
		}
		b, _ := ioutil.ReadAll(res.Body)
		json.Unmarshal(b, &erres)
		var rateLimit bool
		for _, e := range erres.Error.Errors {
			if e.Reason == "rateLimitExceeded" || e.Reason == "userRateLimitExceeded" {
				rateLimit = true
				break
			}
		}
		// this is neither a rate limit error, nor a server error:
		// retrying is useless
		if !rateLimit && res.StatusCode < http.StatusInternalServerError {
			return nil, fmt.Errorf("fetch %s: %s; %s", url, res.Status, b)
		}
	}
	return nil, fmt.Errorf("%s: failed after %d retries", url, n)
}

func gdocID(url string) string {
	const s = "/document/d/"
	if i := strings.Index(url, s); i >= 0 {
		url = url[i+len(s):]
	}
	if i := strings.IndexRune(url, '/'); i > 0 {
		url = url[:i]
	}
	return url
}

func gdocExportURL(id string) string {
	return fmt.Sprintf("%s/files/%s/export?mimeType=text/html", driveAPI, id)
}

// restrictPathToParent will ensure that assetPath is in parent.
// It will thus return an absolute path to the asset.
func restrictPathToParent(assetPath, parent string) (string, error) {
	parent, err := filepath.Abs(parent)
	if err != nil {
		return "", err
	}
	if !filepath.IsAbs(assetPath) {
		assetPath = filepath.Join(parent, assetPath)
	}
	if !strings.HasPrefix(assetPath, parent) {
		return "", fmt.Errorf("%s isn't a subdirectory of %s", assetPath, parent)
	}
	return assetPath, nil
}
<|endoftext|>


<commit_before>package cache

import (
	"encoding/json"
	"errors"
	"flag"
	"fmt"
	"io/ioutil"
	"log"
	"path"
	"strings"
	"sync"
	"time"

	"github.com/asjoyner/shade"
	"github.com/asjoyner/shade/drive"
)

var (
	defaultCacheDir = path.Join(shade.ConfigDir(), "cache")
	cacheDir        = flag.String("cache", defaultCacheDir, "Where to store the drive data cache")
	cacheDebug      = flag.Bool("cacheDebug", false, "Print cache debugging traces")
)

// A very compact representation of a file
type Node struct {
	Filename     string
	Filesize     uint64
	ModifiedTime time.Time
	Sha256sum    []byte // the sha of the associated File
	// TODO(asjoyner): use a struct{} here for efficiency?
	Children map[string]bool
}

func (n *Node) Synthetic() bool {
	if n.Sha256sum == nil {
		return true
	}
	return false
}

// Reader is a wrapper around a slice of cloud storage backends.  It presents an
// interface to query for the union of the set of known files by an integer ID,
// which will be stable across single processes invoking this cache, a node
// representing that file, or a single chunk of that file.  It can also cache a
// configurable quantity of chunks to disk.
//
// TODO(asjoyner): implement disk caching of data blocks.
type Reader struct {
	clients []drive.Client
	nodes   map[string]Node // full path to node
	sync.RWMutex
}

func New(clients []drive.Client, t *time.Ticker) (*Reader, error) {
	c := &Reader{
		clients: clients,
		nodes: map[string]Node{
			"/": {
				Filename: "/",
				Children: make(map[string]bool),
			}},
	}
	if err := c.refresh(); err != nil {
		return nil, err
	}
	go c.periodicRefresh(t)
	return c, nil
}

// NodeByPath returns the current file object for a given path.
func (c *Reader) NodeByPath(p string) (Node, error) {
	c.RLock()
	defer c.RUnlock()
	if n, ok := c.nodes[p]; ok {
		return n, nil
	}
	fmt.Printf("%+v\n", c.nodes)
	return Node{}, fmt.Errorf("no such node: %q", p)
}

// FileByNode returns the full shade.File object for a given node.
func (c *Reader) FileByNode(n Node) (*shade.File, error) {
	if n.Sha256sum == nil {
		return nil, errors.New("no shade.File defined")
	}
	f, err := retrieveChunk(n.Sha256sum)
	if err != nil {
		return nil, fmt.Errorf("retrieveChunk(%x)", n.Sha256sum, err)
	}
	file := &shade.File{}
	if err := json.Unmarshal(f, file); err != nil {
		return nil, fmt.Errorf("Failed to unmarshal sha256sum %x: %s", n.Sha256sum, err)
	}
	return file, nil
}

func (c *Reader) HasChild(parent, child string) bool {
	c.RLock()
	defer c.RUnlock()
	return c.nodes[parent].Children[child]
}

// NumNodes returns the number of nodes (files + synthetic directories) in the
// system.
func (c *Reader) NumNodes() int {
	c.RLock()
	defer c.RUnlock()
	return len(c.nodes)
}

// refresh updates the cache
func (c *Reader) refresh() error {
	debug("Begining cache refresh cycle.")
	knownNodes := make(map[string]bool)
	for _, client := range c.clients {
		lfm, err := client.ListFiles()
		if err != nil {
			return err
		}
		debug(fmt.Sprintf("Found %d file(s) via %s", len(lfm), client.GetConfig().Provider))
		// fetch all those files into the local disk cache
		for id, sha256sum := range lfm {
			// check if we have already processed this Node
			if knownNodes[string(sha256sum)] {
				continue // we've already processed this file
			}

			// check if the File is already in the disk cache
			f, err := retrieveChunk(sha256sum)
			if err != nil {
				// we have to fetch the file Chunk
				f, err = client.GetFile(id)
				if err != nil {
					// TODO(asjoyner): retry
					log.Printf("Failed to fetch %s with fileId %s: %s", sha256sum, id, err)
					continue // the client did not have the file?
				}
				// store it in the disk cache
				storeChunk(sha256sum, f)
				if err != nil {
					log.Printf("Failed to store checksum %s: %s", sha256sum, err)
				}
			}

			// unmarshal and populate c.nodes as the shade.files go by
			file := &shade.File{}
			if err := json.Unmarshal(f, file); err != nil {
				log.Printf("Failed to unmarshal fileID %s: %s", id, err)
				continue
			}
			node := Node{
				Filename:     file.Filename,
				Filesize:     uint64(file.Filesize),
				ModifiedTime: file.ModifiedTime,
				Sha256sum:    sha256sum,
				Children:     nil,
			}
			c.Lock()
			existing, ok := c.nodes[node.Filename]
			// TODO(asjoyner): handle file + directory collisions
			if ok && existing.ModifiedTime.After(node.ModifiedTime) {
				c.Unlock()
				continue
			}
			c.nodes[node.Filename] = node
			c.addParents(node.Filename)
			c.Unlock()
			knownNodes[string(sha256sum)] = true
		}
	}
	debug(fmt.Sprintf("Refresh complete with %d file(s).", len(knownNodes)))
	return nil
}

// recursive function to update parent dirs
func (c *Reader) addParents(filepath string) {
	dir, f := path.Split(filepath)
	debug(fmt.Sprintf("adding %s as child of %s", f, dir))
	// TODO(asjoyner): handle file + directory collisions
	parent, ok := c.nodes[dir]
	if !ok {
		// if the parent node doesn't yet exist, initialize it
		parent.Children = make(map[string]bool)
	}
	parent.Children[f] = true
	if dir == "" {
		c.nodes["/"].Children[f] = true
	} else {
		c.addParents(strings.TrimSuffix(dir, "/"))
	}
}

func (c *Reader) periodicRefresh(t *time.Ticker) {
	for {
		<-t.C
		c.refresh()
	}
}

func storeChunk(sha256sum []byte, data []byte) error {
	filename := path.Join(*cacheDir, string(sha256sum))
	if err := ioutil.WriteFile(filename, data, 0600); err != nil {
		return err
	}
	return nil
}

func retrieveChunk(sha256sum []byte) ([]byte, error) {
	filename := path.Join(*cacheDir, string(sha256sum))
	f, err := ioutil.ReadFile(filename)
	if err != nil {
		return nil, err
	}
	return f, nil
}

func debug(args interface{}) {
	if !*cacheDebug {
		return
	}
	log.Printf("CACHE: %s\n", args)
}
<commit_msg>Abandon local caching in cache.Reader<commit_after>package cache

import (
	"encoding/json"
	"errors"
	"flag"
	"fmt"
	"log"
	"path"
	"strings"
	"sync"
	"time"

	"github.com/asjoyner/shade"
	"github.com/asjoyner/shade/drive"
)

var (
	defaultCacheDir = path.Join(shade.ConfigDir(), "cache")
	cacheDir        = flag.String("cache", defaultCacheDir, "Where to store the drive data cache")
	cacheDebug      = flag.Bool("cacheDebug", false, "Print cache debugging traces")
)

// A very compact representation of a file
type Node struct {
	Filename     string
	Filesize     uint64
	ModifiedTime time.Time
	Sha256sum    []byte // the sha of the associated File
	// TODO(asjoyner): use a struct{} here for efficiency?
	Children map[string]bool
}

func (n *Node) Synthetic() bool {
	if n.Sha256sum == nil {
		return true
	}
	return false
}

// Reader is a wrapper around a slice of cloud storage backends.  It presents an
// interface to query for the union of the set of known files by an integer ID,
// which will be stable across single processes invoking this cache, a node
// representing that file, or a single chunk of that file.  It can also cache a
// configurable quantity of chunks to disk.
//
// TODO(asjoyner): implement disk caching of data blocks.
type Reader struct {
	clients []drive.Client
	nodes   map[string]Node // full path to node
	sync.RWMutex
}

func New(clients []drive.Client, t *time.Ticker) (*Reader, error) {
	c := &Reader{
		clients: clients,
		nodes: map[string]Node{
			"/": {
				Filename: "/",
				Children: make(map[string]bool),
			}},
	}
	if err := c.refresh(); err != nil {
		return nil, err
	}
	go c.periodicRefresh(t)
	return c, nil
}

// NodeByPath returns the current file object for a given path.
func (c *Reader) NodeByPath(p string) (Node, error) {
	c.RLock()
	defer c.RUnlock()
	if n, ok := c.nodes[p]; ok {
		return n, nil
	}
	fmt.Printf("%+v\n", c.nodes)
	return Node{}, fmt.Errorf("no such node: %q", p)
}

// FileByNode returns the full shade.File object for a given node.
func (c *Reader) FileByNode(n Node) (*shade.File, error) {
	if n.Sha256sum == nil {
		return nil, errors.New("no shade.File defined")
	}
	var fj []byte
	var err error
	for _, client := range c.clients {
		fj, err = client.GetFile(n.Sha256sum)
		if err != nil {
			log.Printf("Failed to fetch %s: %s", n.Sha256sum, err)
			continue
		}
	}
	if fj == nil || len(fj) == 0 {
		return nil, fmt.Errorf("Could not find JSON for node: %q", n.Filename)
	}
	file := &shade.File{}
	if err := json.Unmarshal(fj, file); err != nil {
		return nil, fmt.Errorf("Failed to unmarshal sha256sum %x: %s", n.Sha256sum, err)
	}
	return file, nil
}

func (c *Reader) HasChild(parent, child string) bool {
	c.RLock()
	defer c.RUnlock()
	return c.nodes[parent].Children[child]
}

// NumNodes returns the number of nodes (files + synthetic directories) in the
// system.
func (c *Reader) NumNodes() int {
	c.RLock()
	defer c.RUnlock()
	return len(c.nodes)
}

func (c *Reader) GetChunk(sha256sum []byte) {
}

// refresh updates the cache
func (c *Reader) refresh() error {
	debug("Begining cache refresh cycle.")
	knownNodes := make(map[string]bool)
	for _, client := range c.clients {
		lfm, err := client.ListFiles()
		if err != nil {
			return err
		}
		debug(fmt.Sprintf("Found %d file(s) via %s", len(lfm), client.GetConfig().Provider))
		// fetch all those files into the local disk cache
		for id, sha256sum := range lfm {
			// check if we have already processed this Node
			if knownNodes[string(sha256sum)] {
				continue // we've already processed this file
			}

			// fetch the file Chunk
			f, err := client.GetFile(id)
			if err != nil {
				// TODO(asjoyner): if !client.Local()... retry?
				log.Printf("Failed to fetch %s with fileId %s: %s", sha256sum, id, err)
				continue
			}
			// ensure this file is known to all the writable clients
			for _, lc := range c.clients {
				if lc.GetConfig().Write {
					if err := lc.PutFile(sha256sum, f); err != nil {
						log.Printf("Failed to store checksum %s in %s: %s", sha256sum, client.GetConfig().Provider, err)
					}
				}
			}

			// unmarshal and populate c.nodes as the shade.files go by
			file := &shade.File{}
			if err := json.Unmarshal(f, file); err != nil {
				log.Printf("Failed to unmarshal fileID %s: %s", id, err)
				continue
			}
			node := Node{
				Filename:     file.Filename,
				Filesize:     uint64(file.Filesize),
				ModifiedTime: file.ModifiedTime,
				Sha256sum:    sha256sum,
				Children:     nil,
			}
			c.Lock()
			existing, ok := c.nodes[node.Filename]
			// TODO(asjoyner): handle file + directory collisions
			if ok && existing.ModifiedTime.After(node.ModifiedTime) {
				c.Unlock()
				continue
			}
			c.nodes[node.Filename] = node
			c.addParents(node.Filename)
			c.Unlock()
			knownNodes[string(sha256sum)] = true
		}
	}
	debug(fmt.Sprintf("Refresh complete with %d file(s).", len(knownNodes)))
	return nil
}

// recursive function to update parent dirs
func (c *Reader) addParents(filepath string) {
	dir, f := path.Split(filepath)
	debug(fmt.Sprintf("adding %s as child of %s", f, dir))
	// TODO(asjoyner): handle file + directory collisions
	parent, ok := c.nodes[dir]
	if !ok {
		// if the parent node doesn't yet exist, initialize it
		parent.Children = make(map[string]bool)
	}
	parent.Children[f] = true
	if dir == "" {
		c.nodes["/"].Children[f] = true
	} else {
		c.addParents(strings.TrimSuffix(dir, "/"))
	}
}

func (c *Reader) periodicRefresh(t *time.Ticker) {
	for {
		<-t.C
		c.refresh()
	}
}

func debug(args interface{}) {
	if !*cacheDebug {
		return
	}
	log.Printf("CACHE: %s\n", args)
}
<|endoftext|>


<commit_before>// Copyright (c) 2012-2016 The Revel Framework Authors, All rights reserved.
// Revel Framework source code and usage is governed by a MIT style
// license that can be found in the LICENSE file.

package cache

import (
	"time"

	"github.com/garyburd/redigo/redis"
	"github.com/revel/revel"
)

// RedisCache wraps the Redis client to meet the Cache interface.
type RedisCache struct {
	pool              *redis.Pool
	defaultExpiration time.Duration
}

// NewRedisCache returns a new RedisCache with given parameters
// until redigo supports sharding/clustering, only one host will be in hostList
func NewRedisCache(host string, password string, defaultExpiration time.Duration) RedisCache {
	var pool = &redis.Pool{
		MaxIdle:     revel.Config.IntDefault("cache.redis.maxidle", 5),
		MaxActive:   revel.Config.IntDefault("cache.redis.maxactive", 0),
		IdleTimeout: time.Duration(revel.Config.IntDefault("cache.redis.idletimeout", 240)) * time.Second,
		Dial: func() (redis.Conn, error) {
			protocol := revel.Config.StringDefault("cache.redis.protocol", "tcp")
			toc := time.Millisecond * time.Duration(revel.Config.IntDefault("cache.redis.timeout.connect", 10000))
			tor := time.Millisecond * time.Duration(revel.Config.IntDefault("cache.redis.timeout.read", 5000))
			tow := time.Millisecond * time.Duration(revel.Config.IntDefault("cache.redis.timeout.write", 5000))
			c, err := redis.Dial(protocol, host,
				redis.DialConnectTimeout(toc),
				redis.DialReadTimeout(tor),
				redis.DialWriteTimeout(tow))
			if err != nil {
				return nil, err
			}
			if len(password) > 0 {
				if _, err = c.Do("AUTH", password); err != nil {
					_ = c.Close()
					return nil, err
				}
			} else {
				// check with PING
				if _, err = c.Do("PING"); err != nil {
					_ = c.Close()
					return nil, err
				}
			}
			return c, err
		},
		// custom connection test method
		TestOnBorrow: func(c redis.Conn, t time.Time) error {
			_, err := c.Do("PING")
			return err
		},
	}
	return RedisCache{pool, defaultExpiration}
}

func (c RedisCache) Set(key string, value interface{}, expires time.Duration) error {
	conn := c.pool.Get()
	defer func() {
		_ = conn.Close()
	}()
	return c.invoke(conn.Do, key, value, expires)
}

func (c RedisCache) Add(key string, value interface{}, expires time.Duration) error {
	conn := c.pool.Get()
	defer func() {
		_ = conn.Close()
	}()

	existed, err := exists(conn, key)
	if err != nil {
		return err
	} else if existed {
		return ErrNotStored
	}
	return c.invoke(conn.Do, key, value, expires)
}

func (c RedisCache) Replace(key string, value interface{}, expires time.Duration) error {
	conn := c.pool.Get()
	defer func() {
		_ = conn.Close()
	}()

	existed, err := exists(conn, key)
	if err != nil {
		return err
	} else if !existed {
		return ErrNotStored
	}

	err = c.invoke(conn.Do, key, value, expires)
	if value == nil {
		return ErrNotStored
	}
	return err
}

func (c RedisCache) Get(key string, ptrValue interface{}) error {
	conn := c.pool.Get()
	defer func() {
		_ = conn.Close()
	}()
	raw, err := conn.Do("GET", key)
	if err != nil {
		return err
	} else if raw == nil {
		return ErrCacheMiss
	}
	item, err := redis.Bytes(raw, err)
	if err != nil {
		return err
	}
	return Deserialize(item, ptrValue)
}

func generalizeStringSlice(strs []string) []interface{} {
	ret := make([]interface{}, len(strs))
	for i, str := range strs {
		ret[i] = str
	}
	return ret
}

func (c RedisCache) GetMulti(keys ...string) (Getter, error) {
	conn := c.pool.Get()
	defer func() {
		_ = conn.Close()
	}()

	items, err := redis.Values(conn.Do("MGET", generalizeStringSlice(keys)...))
	if err != nil {
		return nil, err
	} else if items == nil {
		return nil, ErrCacheMiss
	}

	m := make(map[string][]byte)
	for i, key := range keys {
		m[key] = nil
		if i < len(items) && items[i] != nil {
			s, ok := items[i].([]byte)
			if ok {
				m[key] = s
			}
		}
	}
	return RedisItemMapGetter(m), nil
}

func exists(conn redis.Conn, key string) (bool, error) {
	return redis.Bool(conn.Do("EXISTS", key))
}

func (c RedisCache) Delete(key string) error {
	conn := c.pool.Get()
	defer func() {
		_ = conn.Close()
	}()
	existed, err := redis.Bool(conn.Do("DEL", key))
	if err == nil && !existed {
		err = ErrCacheMiss
	}
	return err
}

func (c RedisCache) Increment(key string, delta uint64) (uint64, error) {
	conn := c.pool.Get()
	defer func() {
		_ = conn.Close()
	}()
	// Check for existence *before* increment as per the cache contract.
	// redis will auto create the key, and we don't want that. Since we need to do increment
	// ourselves instead of natively via INCRBY (redis doesn't support wrapping), we get the value
	// and do the exists check this way to minimize calls to Redis
	val, err := conn.Do("GET", key)
	if err != nil {
		return 0, err
	} else if val == nil {
		return 0, ErrCacheMiss
	}
	currentVal, err := redis.Int64(val, nil)
	if err != nil {
		return 0, err
	}
	sum := currentVal + int64(delta)
	_, err = conn.Do("SET", key, sum)
	if err != nil {
		return 0, err
	}
	return uint64(sum), nil
}

func (c RedisCache) Decrement(key string, delta uint64) (newValue uint64, err error) {
	conn := c.pool.Get()
	defer func() {
		_ = conn.Close()
	}()
	// Check for existence *before* increment as per the cache contract.
	// redis will auto create the key, and we don't want that, hence the exists call
	existed, err := exists(conn, key)
	if err != nil {
		return 0, err
	} else if !existed {
		return 0, ErrCacheMiss
	}
	// Decrement contract says you can only go to 0
	// so we go fetch the value and if the delta is greater than the amount,
	// 0 out the value
	currentVal, err := redis.Int64(conn.Do("GET", key))
	if err != nil {
		return 0, err
	}
	if delta > uint64(currentVal) {
		var tempint int64
		tempint, err = redis.Int64(conn.Do("DECRBY", key, currentVal))
		return uint64(tempint), err
	}
	tempint, err := redis.Int64(conn.Do("DECRBY", key, delta))
	return uint64(tempint), err
}

func (c RedisCache) Flush() error {
	conn := c.pool.Get()
	defer func() {
		_ = conn.Close()
	}()
	_, err := conn.Do("FLUSHALL")
	return err
}

func (c RedisCache) invoke(f func(string, ...interface{}) (interface{}, error),
	key string, value interface{}, expires time.Duration) error {

	switch expires {
	case DefaultExpiryTime:
		expires = c.defaultExpiration
	case ForEverNeverExpiry:
		expires = time.Duration(0)
	}

	b, err := Serialize(value)
	if err != nil {
		return err
	}
	conn := c.pool.Get()
	defer func() {
		_ = conn.Close()
	}()
	if expires > 0 {
		_, err = f("SETEX", key, int32(expires/time.Second), b)
		return err
	}
	_, err = f("SET", key, b)
	return err
}

// RedisItemMapGetter implements a Getter on top of the returned item map.
type RedisItemMapGetter map[string][]byte

func (g RedisItemMapGetter) Get(key string, ptrValue interface{}) error {
	item, ok := g[key]
	if !ok {
		return ErrCacheMiss
	}
	return Deserialize(item, ptrValue)
}
<commit_msg>Correction redis import<commit_after>// Copyright (c) 2012-2016 The Revel Framework Authors, All rights reserved.
// Revel Framework source code and usage is governed by a MIT style
// license that can be found in the LICENSE file.

package cache

import (
	"time"

	"github.com/gomodule/redigo/redis"
	"github.com/revel/revel"
)

// RedisCache wraps the Redis client to meet the Cache interface.
type RedisCache struct {
	pool              *redis.Pool
	defaultExpiration time.Duration
}

// NewRedisCache returns a new RedisCache with given parameters
// until redigo supports sharding/clustering, only one host will be in hostList
func NewRedisCache(host string, password string, defaultExpiration time.Duration) RedisCache {
	var pool = &redis.Pool{
		MaxIdle:     revel.Config.IntDefault("cache.redis.maxidle", 5),
		MaxActive:   revel.Config.IntDefault("cache.redis.maxactive", 0),
		IdleTimeout: time.Duration(revel.Config.IntDefault("cache.redis.idletimeout", 240)) * time.Second,
		Dial: func() (redis.Conn, error) {
			protocol := revel.Config.StringDefault("cache.redis.protocol", "tcp")
			toc := time.Millisecond * time.Duration(revel.Config.IntDefault("cache.redis.timeout.connect", 10000))
			tor := time.Millisecond * time.Duration(revel.Config.IntDefault("cache.redis.timeout.read", 5000))
			tow := time.Millisecond * time.Duration(revel.Config.IntDefault("cache.redis.timeout.write", 5000))
			c, err := redis.Dial(protocol, host,
				redis.DialConnectTimeout(toc),
				redis.DialReadTimeout(tor),
				redis.DialWriteTimeout(tow))
			if err != nil {
				return nil, err
			}
			if len(password) > 0 {
				if _, err = c.Do("AUTH", password); err != nil {
					_ = c.Close()
					return nil, err
				}
			} else {
				// check with PING
				if _, err = c.Do("PING"); err != nil {
					_ = c.Close()
					return nil, err
				}
			}
			return c, err
		},
		// custom connection test method
		TestOnBorrow: func(c redis.Conn, t time.Time) error {
			_, err := c.Do("PING")
			return err
		},
	}
	return RedisCache{pool, defaultExpiration}
}

func (c RedisCache) Set(key string, value interface{}, expires time.Duration) error {
	conn := c.pool.Get()
	defer func() {
		_ = conn.Close()
	}()
	return c.invoke(conn.Do, key, value, expires)
}

func (c RedisCache) Add(key string, value interface{}, expires time.Duration) error {
	conn := c.pool.Get()
	defer func() {
		_ = conn.Close()
	}()

	existed, err := exists(conn, key)
	if err != nil {
		return err
	} else if existed {
		return ErrNotStored
	}
	return c.invoke(conn.Do, key, value, expires)
}

func (c RedisCache) Replace(key string, value interface{}, expires time.Duration) error {
	conn := c.pool.Get()
	defer func() {
		_ = conn.Close()
	}()

	existed, err := exists(conn, key)
	if err != nil {
		return err
	} else if !existed {
		return ErrNotStored
	}

	err = c.invoke(conn.Do, key, value, expires)
	if value == nil {
		return ErrNotStored
	}
	return err
}

func (c RedisCache) Get(key string, ptrValue interface{}) error {
	conn := c.pool.Get()
	defer func() {
		_ = conn.Close()
	}()
	raw, err := conn.Do("GET", key)
	if err != nil {
		return err
	} else if raw == nil {
		return ErrCacheMiss
	}
	item, err := redis.Bytes(raw, err)
	if err != nil {
		return err
	}
	return Deserialize(item, ptrValue)
}

func generalizeStringSlice(strs []string) []interface{} {
	ret := make([]interface{}, len(strs))
	for i, str := range strs {
		ret[i] = str
	}
	return ret
}

func (c RedisCache) GetMulti(keys ...string) (Getter, error) {
	conn := c.pool.Get()
	defer func() {
		_ = conn.Close()
	}()

	items, err := redis.Values(conn.Do("MGET", generalizeStringSlice(keys)...))
	if err != nil {
		return nil, err
	} else if items == nil {
		return nil, ErrCacheMiss
	}

	m := make(map[string][]byte)
	for i, key := range keys {
		m[key] = nil
		if i < len(items) && items[i] != nil {
			s, ok := items[i].([]byte)
			if ok {
				m[key] = s
			}
		}
	}
	return RedisItemMapGetter(m), nil
}

func exists(conn redis.Conn, key string) (bool, error) {
	return redis.Bool(conn.Do("EXISTS", key))
}

func (c RedisCache) Delete(key string) error {
	conn := c.pool.Get()
	defer func() {
		_ = conn.Close()
	}()
	existed, err := redis.Bool(conn.Do("DEL", key))
	if err == nil && !existed {
		err = ErrCacheMiss
	}
	return err
}

func (c RedisCache) Increment(key string, delta uint64) (uint64, error) {
	conn := c.pool.Get()
	defer func() {
		_ = conn.Close()
	}()
	// Check for existence *before* increment as per the cache contract.
	// redis will auto create the key, and we don't want that. Since we need to do increment
	// ourselves instead of natively via INCRBY (redis doesn't support wrapping), we get the value
	// and do the exists check this way to minimize calls to Redis
	val, err := conn.Do("GET", key)
	if err != nil {
		return 0, err
	} else if val == nil {
		return 0, ErrCacheMiss
	}
	currentVal, err := redis.Int64(val, nil)
	if err != nil {
		return 0, err
	}
	sum := currentVal + int64(delta)
	_, err = conn.Do("SET", key, sum)
	if err != nil {
		return 0, err
	}
	return uint64(sum), nil
}

func (c RedisCache) Decrement(key string, delta uint64) (newValue uint64, err error) {
	conn := c.pool.Get()
	defer func() {
		_ = conn.Close()
	}()
	// Check for existence *before* increment as per the cache contract.
	// redis will auto create the key, and we don't want that, hence the exists call
	existed, err := exists(conn, key)
	if err != nil {
		return 0, err
	} else if !existed {
		return 0, ErrCacheMiss
	}
	// Decrement contract says you can only go to 0
	// so we go fetch the value and if the delta is greater than the amount,
	// 0 out the value
	currentVal, err := redis.Int64(conn.Do("GET", key))
	if err != nil {
		return 0, err
	}
	if delta > uint64(currentVal) {
		var tempint int64
		tempint, err = redis.Int64(conn.Do("DECRBY", key, currentVal))
		return uint64(tempint), err
	}
	tempint, err := redis.Int64(conn.Do("DECRBY", key, delta))
	return uint64(tempint), err
}

func (c RedisCache) Flush() error {
	conn := c.pool.Get()
	defer func() {
		_ = conn.Close()
	}()
	_, err := conn.Do("FLUSHALL")
	return err
}

func (c RedisCache) invoke(f func(string, ...interface{}) (interface{}, error),
	key string, value interface{}, expires time.Duration) error {

	switch expires {
	case DefaultExpiryTime:
		expires = c.defaultExpiration
	case ForEverNeverExpiry:
		expires = time.Duration(0)
	}

	b, err := Serialize(value)
	if err != nil {
		return err
	}
	conn := c.pool.Get()
	defer func() {
		_ = conn.Close()
	}()
	if expires > 0 {
		_, err = f("SETEX", key, int32(expires/time.Second), b)
		return err
	}
	_, err = f("SET", key, b)
	return err
}

// RedisItemMapGetter implements a Getter on top of the returned item map.
type RedisItemMapGetter map[string][]byte

func (g RedisItemMapGetter) Get(key string, ptrValue interface{}) error {
	item, ok := g[key]
	if !ok {
		return ErrCacheMiss
	}
	return Deserialize(item, ptrValue)
}
<|endoftext|>


<commit_before>package core

import (
	"testing"

	config "github.com/jbenet/go-ipfs/config"
)

func TestDatastores(t *testing.T) {

	good := []*config.Config{
		&config.Config{Datastore: &config.Datastore{Type: "memory"}},
		&config.Config{Datastore: &config.Datastore{Type: "leveldb", Path: ".testdb"}},
	}

	bad := []*config.Config{
		&config.Config{Datastore: &config.Datastore{}},
		&config.Config{Datastore: &config.Datastore{Type: "badtype"}},
		&config.Config{},
		nil,
	}

	for i, c := range good {
		n, err := NewIpfsNode(c, false)
		if n == nil || err != nil {
			t.Error("Should have constructed.", i, err)
		}
	}

	for i, c := range bad {
		n, err := NewIpfsNode(c, false)
		if n != nil || err == nil {
			t.Error("Should have failed to construct.", i)
		}
	}
}
<commit_msg>core: Updated basic initialization tests<commit_after>package core

import (
	"testing"

	config "github.com/jbenet/go-ipfs/config"
)

func TestInitialization(t *testing.T) {
	id :=  &config.Identity {
		PeerID: "QmNgdzLieYi8tgfo2WfTUzNVH5hQK9oAYGVf6dxN12NrHt",
		Address: "/ip4/127.0.0.1/tcp/8000",
		PrivKey: "CAASrRIwggkpAgEAAoICAQCwt67GTUQ8nlJhks6CgbLKOx7F5tl1r9zF4m3TUrG3Pe8h64vi+ILDRFd7QJxaJ/n8ux9RUDoxLjzftL4uTdtv5UXl2vaufCc/C0bhCRvDhuWPhVsD75/DZPbwLsepxocwVWTyq7/ZHsCfuWdoh/KNczfy+Gn33gVQbHCnip/uhTVxT7ARTiv8Qa3d7qmmxsR+1zdL/IRO0mic/iojcb3Oc/PRnYBTiAZFbZdUEit/99tnfSjMDg02wRayZaT5ikxa6gBTMZ16Yvienq7RwSELzMQq2jFA4i/TdiGhS9uKywltiN2LrNDBcQJSN02pK12DKoiIy+wuOCRgs2NTQEhU2sXCk091v7giTTOpFX2ij9ghmiRfoSiBFPJA5RGwiH6ansCHtWKY1K8BS5UORM0o3dYk87mTnKbCsdz4bYnGtOWafujYwzueGx8r+IWiys80IPQKDeehnLW6RgoyjszKgL/2XTyP54xMLSW+Qb3BPgDcPaPO0hmop1hW9upStxKsefW2A2d46Ds4HEpJEry7PkS5M4gKL/zCKHuxuXVk14+fZQ1rstMuvKjrekpAC2aVIKMI9VRA3awtnje8HImQMdj+r+bPmv0N8rTTr3eS4J8Yl7k12i95LLfK+fWnmUh22oTNzkRlaiERQrUDyE4XNCtJc0xs1oe1yXGqazCIAQIDAQABAoICAQCk1N/ftahlRmOfAXk//8wNl7FvdJD3le6+YSKBj0uWmN1ZbUSQk64chr12iGCOM2WY180xYjy1LOS44PTXaeW5bEiTSnb3b3SH+HPHaWCNM2EiSogHltYVQjKW+3tfH39vlOdQ9uQ+l9Gh6iTLOqsCRyszpYPqIBwi1NMLY2Ej8PpVU7ftnFWouHZ9YKS7nAEiMoowhTu/7cCIVwZlAy3AySTuKxPMVj9LORqC32PVvBHZaMPJ+X1Xyijqg6aq39WyoztkXg3+Xxx5j5eOrK6vO/Lp6ZUxaQilHDXoJkKEJjgIBDZpluss08UPfOgiWAGkW+L4fgUxY0qDLDAEMhyEBAn6KOKVL1JhGTX6GjhWziI94bddSpHKYOEIDzUy4H8BXnKhtnyQV6ELS65C2hj9D0IMBTj7edCF1poJy0QfdK0cuXgMvxHLeUO5uc2YWfbNosvKxqygB9rToy4b22YvNwsZUXsTY6Jt+p9V2OgXSKfB5VPeRbjTJL6xqvvUJpQytmII/C9JmSDUtCbYceHj6X9jgigLk20VV6nWHqCTj3utXD6NPAjoycVpLKDlnWEgfVELDIk0gobxUqqSm3jTPEKRPJgxkgPxbwxYumtw++1UY2y35w3WRDc2xYPaWKBCQeZy+mL6ByXp9bWlNvxS3Knb6oZp36/ovGnf2pGvdQKCAQEAyKpipz2lIUySDyE0avVWAmQb2tWGKXALPohzj7AwkcfEg2GuwoC6GyVE2sTJD1HRazIjOKn3yQORg2uOPeG7sx7EKHxSxCKDrbPawkvLCq8JYSy9TLvhqKUVVGYPqMBzu2POSLEA81QXas+aYjKOFWA2Zrjq26zV9ey3+6Lc6WULePgRQybU8+RHJc6fdjUCCfUxgOrUO2IQOuTJ+FsDpVnrMUGlokmWn23OjL4qTL9wGDnWGUs2pjSzNbj3qA0d8iqaiMUyHX/D/VS0wpeT1osNBSm8suvSibYBn+7wbIApbwXUxZaxMv2OHGz3empae4ckvNZs7r8wsI9UwFt8mwKCAQEA4XK6gZkv9t+3YCcSPw2ensLvL/xU7i2bkC9tfTGdjnQfzZXIf5KNdVuj/SerOl2S1s45NMs3ysJbADwRb4ahElD/V71nGzV8fpFTitC20ro9fuX4J0+twmBolHqeH9pmeGTjAeL1rvt6vxs4FkeG/yNft7GdXpXTtEGaObn8Mt0tPY+aB3UnKrnCQoQAlPyGHFrVRX0UEcp6wyyNGhJCNKeNOvqCHTFObhbhO+KWpWSN0MkVHnqaIBnIn1Te8FtvP/iTwXGnKc0YXJUG6+LM6LmOguW6tg8ZqiQeYyyR+e9eCFH4csLzkrTl1GxCxwEsoSLIMm7UDcjttW6tYEghkwKCAQEAmeCO5lCPYImnN5Lu71ZTLmI2OgmjaANTnBBnDbi+hgv61gUCToUIMejSdDCTPfwv61P3TmyIZs0luPGxkiKYHTNqmOE9Vspgz8Mr7fLRMNApESuNvloVIY32XVImj/GEzh4rAfM6F15U1sN8T/EUo6+0B/Glp+9R49QzAfRSE2g48/rGwgf1JVHYfVWFUtAzUA+GdqWdOixo5cCsYJbqpNHfWVZN/bUQnBFIYwUwysnC29D+LUdQEQQ4qOm+gFAOtrWU62zMkXJ4iLt8Ify6kbrvsRXgbhQIzzGS7WH9XDarj0eZciuslr15TLMC1Azadf+cXHLR9gMHA13mT9vYIQKCAQA/DjGv8cKCkAvf7s2hqROGYAs6Jp8yhrsN1tYOwAPLRhtnCs+rLrg17M2vDptLlcRuI/vIElamdTmylRpjUQpX7yObzLO73nfVhpwRJVMdGU394iBIDncQ+JoHfUwgqJskbUM40dvZdyjbrqc/Q/4z+hbZb+oN/GXb8sVKBATPzSDMKQ/xqgisYIw+wmDPStnPsHAaIWOtni47zIgilJzD0WEk78/YjmPbUrboYvWziK5JiRRJFA1rkQqV1c0M+OXixIm+/yS8AksgCeaHr0WUieGcJtjT9uE8vyFop5ykhRiNxy9wGaq6i7IEecsrkd6DqxDHWkwhFuO1bSE83q/VAoIBAEA+RX1i/SUi08p71ggUi9WFMqXmzELp1L3hiEjOc2AklHk2rPxsaTh9+G95BvjhP7fRa/Yga+yDtYuyjO99nedStdNNSg03aPXILl9gs3r2dPiQKUEXZJ3FrH6tkils/8BlpOIRfbkszrdZIKTO9GCdLWQ30dQITDACs8zV/1GFGrHFrqnnMe/NpIFHWNZJ0/WZMi8wgWO6Ik8jHEpQtVXRiXLqy7U6hk170pa4GHOzvftfPElOZZjy9qn7KjdAQqy6spIrAE94OEL+fBgbHQZGLpuTlj6w6YGbMtPU8uo7sXKoc6WOCb68JWft3tejGLDa1946HAWqVM9B/UcneNc=",
	}

	good := []*config.Config{
		&config.Config {
			Identity: id,
			Datastore: &config.Datastore{
				Type: "memory",
			},
		},

		&config.Config {
			Identity: id,
			Datastore: &config.Datastore {
				Type: "leveldb",
				Path: ".testdb",
			},
		},
	}

	bad := []*config.Config{
		&config.Config{Identity: id, Datastore: &config.Datastore{}},
		&config.Config{Identity: id, Datastore: &config.Datastore{Type: "badtype"}},
		&config.Config{},
		&config.Config{Datastore: &config.Datastore{Type: "memory"}},
		nil,
	}

	for i, c := range good {
		n, err := NewIpfsNode(c, false)
		if n == nil || err != nil {
			t.Error("Should have constructed.", i, err)
		}
	}

	for i, c := range bad {
		n, err := NewIpfsNode(c, false)
		if n != nil || err == nil {
			t.Error("Should have failed to construct.", i)
		}
	}
}
<|endoftext|>


<commit_before>// Copyright (C) 2017 Damon Revoe. All rights reserved.
// Use of this source code is governed by the MIT
// license, which can be found in the LICENSE file.

package main

import (
	"errors"
	"log"

	"github.com/spf13/cobra"
)

func generatePackageSources(packages []string) error {
	packageIndex, err := buildPackageIndex()
	if err != nil {
		return err
	}

	for _, pkg := range packages {
		if _, ok := packageIndex.packageByName[pkg]; !ok {
			return errors.New("no such package: " + pkg)
		}
	}

	for _, pkg := range packages {
		pd := packageIndex.packageByName[pkg]

		err = generateBuildFilesFromEmbeddedTemplate(&appTemplate,
			"output-"+pd.packageName, pd)
		if err != nil {
			return err
		}
	}

	return nil
}

// SelectCmd represents the select command
var selectCmd = &cobra.Command{
	Use:   "select [package_range...]",
	Short: "Choose one or more packages to work on",
	Run: func(_ *cobra.Command, args []string) {
		if err := generatePackageSources(args); err != nil {
			log.Fatal(err)
		}
	},
}

func init() {
	RootCmd.AddCommand(selectCmd)

	selectCmd.Flags().SortFlags = false
	addQuietFlag(selectCmd)
	addWorkspaceDirFlag(selectCmd)
	addPkgPathFlag(selectCmd)
}
<commit_msg>Require at least one argument in select command<commit_after>// Copyright (C) 2017 Damon Revoe. All rights reserved.
// Use of this source code is governed by the MIT
// license, which can be found in the LICENSE file.

package main

import (
	"errors"
	"log"

	"github.com/spf13/cobra"
)

func generatePackageSources(packages []string) error {
	packageIndex, err := buildPackageIndex()
	if err != nil {
		return err
	}

	for _, pkg := range packages {
		if _, ok := packageIndex.packageByName[pkg]; !ok {
			return errors.New("no such package: " + pkg)
		}
	}

	for _, pkg := range packages {
		pd := packageIndex.packageByName[pkg]

		err = generateBuildFilesFromEmbeddedTemplate(&appTemplate,
			"output-"+pd.packageName, pd)
		if err != nil {
			return err
		}
	}

	return nil
}

// SelectCmd represents the select command
var selectCmd = &cobra.Command{
	Use:   "select package_range...",
	Short: "Choose one or more packages to work on",
	Args:  cobra.MinimumNArgs(1),
	Run: func(_ *cobra.Command, args []string) {
		if err := generatePackageSources(args); err != nil {
			log.Fatal(err)
		}
	},
}

func init() {
	RootCmd.AddCommand(selectCmd)

	selectCmd.Flags().SortFlags = false
	addQuietFlag(selectCmd)
	addWorkspaceDirFlag(selectCmd)
	addPkgPathFlag(selectCmd)
}
<|endoftext|>


<commit_before>package core

import (
	"fmt"
	"testing"
	"time"
)

var parsetests = []struct {
	in, name string
	octave   int
	dura     float32
	acc      int
	dot      bool
	vel      int
}{
	{"C", "C", 4, 0.25, 0, false, Normal},
	{"C3", "C", 3, 0.25, 0, false, Normal},
	{"C3", "C", 3, 0.25, 0, false, Normal},
	{"4C3", "C", 3, 0.25, 0, false, Normal},
	{"C#3", "C", 3, 0.25, 1, false, Normal},
	{"4.C#3", "C", 3, 0.25, 1, true, Normal},
	{"16C", "C", 4, 0.0625, 0, false, Normal},
	{"4C#", "C", 4, 0.25, 1, false, Normal},
	{"C#", "C", 4, 0.25, 1, false, Normal},
	{"B_", "B", 4, 0.25, -1, false, Normal}, //8
	{".F#9", "F", 9, 0.25, 1, true, Normal},
	{"1C", "C", 4, 1, 0, false, Normal},
	{"=", "=", 4, 0.25, 0, false, Normal},
	{"D++", "D", 4, 0.25, 0, false, VelocityF},
	{"D+", "D", 4, 0.25, 0, false, VelocityMF},
	{"D+++", "D", 4, 0.25, 0, false, VelocityFF},
	{"E-", "E", 4, 0.25, 0, false, VelocityMP},
	{"E--", "E", 4, 0.25, 0, false, VelocityP},
	{"E---", "E", 4, 0.25, 0, false, VelocityPP},
	{"Bo", "B", 4, 0.25, 0, false, Normal},
	{"<", "<", 0, 0, 0, false, 0},
	{"^", "^", 0, 0, 0, false, 0},
}

func TestParseNote(t *testing.T) {
	for i, each := range parsetests {
		n, err := ParseNote(each.in)
		if err != nil {
			t.Errorf("got [%v] for %s", err, each.in)
		}
		if n.Name != each.name {
			t.Fatal("name: line,exp,act", i, each.name, n.Name)
		}
		if n.Octave != each.octave {
			t.Fatal("oct: line,exp,act", i, each.octave, n.Octave)
		}
		if n.fraction != each.dura {
			t.Fatal("dur: line,exp,act", i, each.dura, n.fraction)
		}
		if n.Accidental != each.acc {
			t.Fatal("acc: line,exp,act", i, each.acc, n.Accidental)
		}
		if n.Dotted != each.dot {
			t.Fatal("dot: line,exp,act", i, each.dot, n.Dotted)
		}
		if n.Velocity != each.vel {
			t.Fatal("vel: line,exp,act", each.in, i, each.vel, n.Velocity)
		}
	}
}

var midi = []struct {
	note string
	nr   int
}{
	{"C", 60},
	{"C#", 61},
	{"C0", 12},
	{"C9", 120},
	{"G9", 127},
}

func TestMIDI(t *testing.T) {
	for _, each := range midi {
		n, _ := ParseNote(each.note)
		if n.MIDI() != each.nr {
			t.Error("line,exp,act", each.note, each.nr, n.MIDI())
		}
	}
}

func TestMIDIAll(t *testing.T) {
	for i := 12; i < 127; i++ {
		n, err := MIDItoNote(0.25, i, 1.0)
		if err != nil {
			t.Error(err)
		}
		m := n.MIDI()
		if m != i {
			t.Error("exp,act,note", i, m, n)
		}
	}
}

var pitchers = []struct {
	before string
	by     int
	after  string
}{
	{"C", 2, "D"},
	{"B", 1, "C5"},
	{"D", -2, "C"},
	{"C", 12, "C5"},
	{"C5", 2, "D5"},
	{"C#3", 0, "C#3"},
	{"C~2C", 2, "D~2D"},
}

func TestModifiedPitch(t *testing.T) {
	for i, each := range pitchers {
		n, _ := ParseNote(each.before)
		n = n.Pitched(each.by)
		if got, want := n.String(), each.after; got != want {
			t.Errorf("%d: got %v want %v", i, got, want)
		}
	}
}

func ExampleParseNote() {
	n1, _ := ParseNote("2C#3")
	n2, _ := ParseNote("2.E_2")
	n3, _ := ParseNote(".F_2++")
	e1, _ := ParseNote("2C#")
	e2, _ := ParseNote("8B_")
	e3, _ := ParseNote("4.D")
	e4, _ := ParseNote("E_")
	t1, _ := ParseNote("2c~4c")
	fmt.Println(e1)
	fmt.Println(e2)
	fmt.Println(e3)
	fmt.Println(e4)
	fmt.Println(n1)
	fmt.Println(n2)
	fmt.Println(n3)
	fmt.Println(t1)
	// Output:
	// 2C#
	// 8B_
	// .D
	// E_
	// 2C#3
	// 2.E_2
	// .F_2++
	// 2C~C
}

// Failures

func TestNote_Storex(t *testing.T) {
	n, _ := NewNote("A", 4, 0.25, 1, false, 1)
	if got, want := n.Storex(), `note('A#-----')`; got != want {
		t.Errorf("got [%v] want [%v]", got, want)
	}
}

func TestNoteDurationFactor(t *testing.T) {
	for _, each := range []struct {
		note   string
		length float32
	}{
		{"=", 0.25},
		{".=", 0.375},
		{"c", 0.25},
		{".c", 0.375},
		{"2c", 0.5},
		{"2.c", 0.75},
		{"1c", 1.0},
		{"1.c", 1.5},
		{"4c", 0.25},
		{"4.c", 0.375},
		{"8c", 0.125},
		{"16c", 0.0625},
		{">", 0},
		{"^", 0},
		{"<", 0},
		{"2c~8c", 0.5 + 0.125},
	} {
		n := MustParseNote(each.note)
		if got, want := n.DurationFactor(), each.length; got != want {
			t.Errorf("got [%s] [%v:%T] want [%v:%T]", each.note, got, got, want, want)
		}
	}
}

func TestNoteWithDynamic(t *testing.T) {
	for _, each := range []struct {
		in      string
		dynamic string
		out     string
	}{
		{"c", "-", "note('C-')"},
		{"2.c#2", "--", "note('2.C#2--')"},
		{"e~2e", "+", "note('E+~2E+')"},
	} {
		nin := MustParseNote(each.in)
		before := nin.Storex()
		nout := nin.WithDynamic(each.dynamic)
		after := nin.Storex()
		if got, want := after, before; got != want {
			t.Errorf("got [%v:%T] want unchanged [%v:%T]", got, got, want, want)
		}
		if got, want := nout.Storex(), each.out; got != want {
			t.Errorf("got [%v:%T] want changed [%v:%T]", got, got, want, want)
		}
	}
}

func TestQuantizeNote(t *testing.T) {
	bpm := 120.0
	w := WholeNoteDuration(bpm)
	w16 := w / 16
	if got, want := w16, time.Duration(125)*time.Millisecond; got != want {
		t.Errorf("got [%v:%T] want [%v:%T]", got, got, want, want)
	}
}

func TestQuantizedFractions(t *testing.T) {
	s := MustParseSequence("16c 16.c 8c 8.c 4c 4.c 2c 2.c 1c 1.c 16c~2c")
	for _, each := range s.Notes {
		t.Log(each[0].String(), each[0].DurationFactor())
	}
}

func TestQuantizeFraction(t *testing.T) {
	type args struct {
		length time.Duration
		whole  time.Duration
	}
	tests := []struct {
		name         string
		df           float32
		wantFraction float32
		wantDotted   bool
		wantOk       bool
	}{
		{
			"1/32",
			1.0 / 32.0,
			0.0,
			false,
			false,
		},
		{
			"1/16",
			1.0 / 16.0,
			0.0625,
			false,
			true,
		},
		{
			"3/32",
			3.0 / 32.0,
			0.09375,
			true,
			true,
		},
		{
			"1/8",
			1.0 / 8.0,
			0.125,
			false,
			true,
		},
		{
			"1.75",
			7.0 / 4.0,
			1.5,
			true,
			true,
		},
	}
	for _, tt := range tests {
		t.Run(tt.name, func(t *testing.T) {
			gotFraction, gotDotted, gotOk := QuantizeFraction(tt.df)
			if gotFraction != tt.wantFraction {
				t.Errorf("QuantizeFraction() gotFraction = %v, want %v", gotFraction, tt.wantFraction)
			}
			if gotDotted != tt.wantDotted {
				t.Errorf("QuantizeFraction() gotDotted = %v, want %v", gotDotted, tt.wantDotted)
			}
			if gotOk != tt.wantOk {
				t.Errorf("QuantizeFraction() gotOk = %v, want %v", gotOk, tt.wantOk)
			}
		})
	}
}
<commit_msg>link check<commit_after>package core

import (
	"fmt"
	"testing"
	"time"
)

var parsetests = []struct {
	in, name string
	octave   int
	dura     float32
	acc      int
	dot      bool
	vel      int
}{
	{"C", "C", 4, 0.25, 0, false, Normal},
	{"C3", "C", 3, 0.25, 0, false, Normal},
	{"C3", "C", 3, 0.25, 0, false, Normal},
	{"4C3", "C", 3, 0.25, 0, false, Normal},
	{"C#3", "C", 3, 0.25, 1, false, Normal},
	{"4.C#3", "C", 3, 0.25, 1, true, Normal},
	{"16C", "C", 4, 0.0625, 0, false, Normal},
	{"4C#", "C", 4, 0.25, 1, false, Normal},
	{"C#", "C", 4, 0.25, 1, false, Normal},
	{"B_", "B", 4, 0.25, -1, false, Normal}, //8
	{".F#9", "F", 9, 0.25, 1, true, Normal},
	{"1C", "C", 4, 1, 0, false, Normal},
	{"=", "=", 4, 0.25, 0, false, Normal},
	{"D++", "D", 4, 0.25, 0, false, VelocityF},
	{"D+", "D", 4, 0.25, 0, false, VelocityMF},
	{"D+++", "D", 4, 0.25, 0, false, VelocityFF},
	{"E-", "E", 4, 0.25, 0, false, VelocityMP},
	{"E--", "E", 4, 0.25, 0, false, VelocityP},
	{"E---", "E", 4, 0.25, 0, false, VelocityPP},
	{"Bo", "B", 4, 0.25, 0, false, Normal},
	{"<", "<", 0, 0, 0, false, 0},
	{"^", "^", 0, 0, 0, false, 0},
}

func TestParseNote(t *testing.T) {
	for i, each := range parsetests {
		n, err := ParseNote(each.in)
		if err != nil {
			t.Errorf("got [%v] for %s", err, each.in)
		}
		if n.Name != each.name {
			t.Fatal("name: line,exp,act", i, each.name, n.Name)
		}
		if n.Octave != each.octave {
			t.Fatal("oct: line,exp,act", i, each.octave, n.Octave)
		}
		if n.fraction != each.dura {
			t.Fatal("dur: line,exp,act", i, each.dura, n.fraction)
		}
		if n.Accidental != each.acc {
			t.Fatal("acc: line,exp,act", i, each.acc, n.Accidental)
		}
		if n.Dotted != each.dot {
			t.Fatal("dot: line,exp,act", i, each.dot, n.Dotted)
		}
		if n.Velocity != each.vel {
			t.Fatal("vel: line,exp,act", each.in, i, each.vel, n.Velocity)
		}
	}
}

var midi = []struct {
	note string
	nr   int
}{
	{"C", 60},
	{"C#", 61},
	{"C0", 12},
	{"C9", 120},
	{"G9", 127},
}

func TestMIDI(t *testing.T) {
	for _, each := range midi {
		n, _ := ParseNote(each.note)
		if n.MIDI() != each.nr {
			t.Error("line,exp,act", each.note, each.nr, n.MIDI())
		}
	}
}

func TestMIDIAll(t *testing.T) {
	for i := 12; i < 127; i++ {
		n, err := MIDItoNote(0.25, i, 1.0)
		if err != nil {
			t.Error(err)
		}
		m := n.MIDI()
		if m != i {
			t.Error("exp,act,note", i, m, n)
		}
	}
}

var pitchers = []struct {
	before string
	by     int
	after  string
}{
	{"C", 2, "D"},
	{"B", 1, "C5"},
	{"D", -2, "C"},
	{"C", 12, "C5"},
	{"C5", 2, "D5"},
	{"C#3", 0, "C#3"},
	{"C~2C", 2, "D~2D"},
}

func TestModifiedPitch(t *testing.T) {
	for i, each := range pitchers {
		n, _ := ParseNote(each.before)
		n = n.Pitched(each.by)
		if got, want := n.String(), each.after; got != want {
			t.Errorf("%d: got %v want %v", i, got, want)
		}
	}
}

func ExampleParseNote() {
	n1, _ := ParseNote("2C#3")
	n2, _ := ParseNote("2.E_2")
	n3, _ := ParseNote(".F_2++")
	e1, _ := ParseNote("2C#")
	e2, _ := ParseNote("8B_")
	e3, _ := ParseNote("4.D")
	e4, _ := ParseNote("E_")
	t1, _ := ParseNote("2c~4c")
	fmt.Println(e1)
	fmt.Println(e2)
	fmt.Println(e3)
	fmt.Println(e4)
	fmt.Println(n1)
	fmt.Println(n2)
	fmt.Println(n3)
	fmt.Println(t1)
	// Output:
	// 2C#
	// 8B_
	// .D
	// E_
	// 2C#3
	// 2.E_2
	// .F_2++
	// 2C~C
}

// Failures

func TestNote_Storex(t *testing.T) {
	n, _ := NewNote("A", 4, 0.25, 1, false, 1)
	if got, want := n.Storex(), `note('A#-----')`; got != want {
		t.Errorf("got [%v] want [%v]", got, want)
	}
}

func TestNoteDurationFactor(t *testing.T) {
	for _, each := range []struct {
		note   string
		length float32
	}{
		{"=", 0.25},
		{".=", 0.375},
		{"c", 0.25},
		{".c", 0.375},
		{"2c", 0.5},
		{"2.c", 0.75},
		{"1c", 1.0},
		{"1.c", 1.5},
		{"4c", 0.25},
		{"4.c", 0.375},
		{"8c", 0.125},
		{"16c", 0.0625},
		{">", 0},
		{"^", 0},
		{"<", 0},
		{"2c~8c", 0.5 + 0.125},
	} {
		n := MustParseNote(each.note)
		if got, want := n.DurationFactor(), each.length; got != want {
			t.Errorf("got [%s] [%v:%T] want [%v:%T]", each.note, got, got, want, want)
		}
	}
}

func TestNoteWithDynamic(t *testing.T) {
	for _, each := range []struct {
		in      string
		dynamic string
		out     string
	}{
		{"c", "-", "note('C-')"},
		{"2.c#2", "--", "note('2.C#2--')"},
		{"e~2e", "+", "note('E+~2E+')"},
	} {
		nin := MustParseNote(each.in)
		before := nin.Storex()
		nout := nin.WithDynamic(each.dynamic)
		after := nin.Storex()
		if got, want := after, before; got != want {
			t.Errorf("got [%v:%T] want unchanged [%v:%T]", got, got, want, want)
		}
		if got, want := nout.Storex(), each.out; got != want {
			t.Errorf("got [%v:%T] want changed [%v:%T]", got, got, want, want)
		}
	}
}

func TestQuantizeNote(t *testing.T) {
	bpm := 120.0
	w := WholeNoteDuration(bpm)
	w16 := w / 16
	if got, want := w16, time.Duration(125)*time.Millisecond; got != want {
		t.Errorf("got [%v:%T] want [%v:%T]", got, got, want, want)
	}
}

func TestQuantizedFractions(t *testing.T) {
	s := MustParseSequence("16c 16.c 8c 8.c 4c 4.c 2c 2.c 1c 1.c 16c~2c")
	for _, each := range s.Notes {
		t.Log(each[0].String(), each[0].DurationFactor())
	}
}

func TestQuantizeFraction(t *testing.T) {
	tests := []struct {
		name         string
		df           float32
		wantFraction float32
		wantDotted   bool
		wantOk       bool
	}{
		{
			"1/32",
			1.0 / 32.0,
			0.0,
			false,
			false,
		},
		{
			"1/16",
			1.0 / 16.0,
			0.0625,
			false,
			true,
		},
		{
			"3/32",
			3.0 / 32.0,
			0.09375,
			true,
			true,
		},
		{
			"1/8",
			1.0 / 8.0,
			0.125,
			false,
			true,
		},
		{
			"1.75",
			7.0 / 4.0,
			1.5,
			true,
			true,
		},
	}
	for _, tt := range tests {
		t.Run(tt.name, func(t *testing.T) {
			gotFraction, gotDotted, gotOk := QuantizeFraction(tt.df)
			if gotFraction != tt.wantFraction {
				t.Errorf("QuantizeFraction() gotFraction = %v, want %v", gotFraction, tt.wantFraction)
			}
			if gotDotted != tt.wantDotted {
				t.Errorf("QuantizeFraction() gotDotted = %v, want %v", gotDotted, tt.wantDotted)
			}
			if gotOk != tt.wantOk {
				t.Errorf("QuantizeFraction() gotOk = %v, want %v", gotOk, tt.wantOk)
			}
		})
	}
}
<|endoftext|>


<commit_before>package dns

import "github.com/jen20/riviera/azure"

type GetDNSZoneResponse struct {
	ID                    *string             `mapstructure:"id"`
	Name                  *string             `mapstructure:"name"`
	Location              *string             `mapstructure:"location"`
	Tags                  *map[string]*string `mapstructure:"tags"`
	NumberOfRecordSets    *string             `mapstructure:"numberOfRecordSets"`
	MaxNumberOfRecordSets *string             `mapstructure:"maxNumberOfRecordSets"`
}

type GetDNSZone struct {
	Name              string `json:"-"`
	ResourceGroupName string `json:"-"`
}

func (s GetDNSZone) APIInfo() azure.APIInfo {
	return azure.APIInfo{
		APIVersion:  apiVersion,
		Method:      "GET",
		URLPathFunc: dnsZoneDefaultURLPathFunc(s.ResourceGroupName, s.Name),
		ResponseTypeFunc: func() interface{} {
			return &GetDNSZoneResponse{}
		},
	}
}
<commit_msg>Add the newly created NameServers to the DNS Get Response<commit_after>package dns

import "github.com/jen20/riviera/azure"

type GetDNSZoneResponse struct {
	ID                    *string             `mapstructure:"id"`
	Name                  *string             `mapstructure:"name"`
	Location              *string             `mapstructure:"location"`
	Tags                  *map[string]*string `mapstructure:"tags"`
	NumberOfRecordSets    *string             `mapstructure:"numberOfRecordSets"`
	MaxNumberOfRecordSets *string             `mapstructure:"maxNumberOfRecordSets"`
	NameServers           *[]string           `mapstructure:"nameServers"`
}

type GetDNSZone struct {
	Name              string `json:"-"`
	ResourceGroupName string `json:"-"`
}

func (s GetDNSZone) APIInfo() azure.APIInfo {
	return azure.APIInfo{
		APIVersion:  apiVersion,
		Method:      "GET",
		URLPathFunc: dnsZoneDefaultURLPathFunc(s.ResourceGroupName, s.Name),
		ResponseTypeFunc: func() interface{} {
			return &GetDNSZoneResponse{}
		},
	}
}
<|endoftext|>


<commit_before>/*
Copyright 2016 The Kubernetes Authors.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package cla

import (
	"fmt"
	"net/url"

	"github.com/Sirupsen/logrus"

	"k8s.io/test-infra/prow/github"
	"k8s.io/test-infra/prow/plugins"
)

const (
	pluginName             = "cla"
	claContextName         = "cla/linuxfoundation"
	claYesLabel            = "cncf-cla: yes"
	claNoLabel             = "cncf-cla: no"
	cncfclaNotFoundMessage = `Thanks for your pull request. Before we can look at your pull request, you'll need to sign a Contributor License Agreement (CLA).

:memo: **Please follow instructions at <https://github.com/kubernetes/kubernetes/wiki/CLA-FAQ> to sign the CLA.**

Once you've signed, please reply here (e.g. "I signed it!") and we'll verify.  Thanks.

---

- If you've already signed a CLA, it's possible we don't have your GitHub username or you're using a different email address.  Check your existing CLA data and verify that your [email is set on your git commits](https://help.github.com/articles/setting-your-email-in-git/).
- If you signed the CLA as a corporation, please sign in with your organization's credentials at <https://identity.linuxfoundation.org/projects/cncf> to be authorized.

<!-- need_sender_cla -->

<details>
%s
</details>
	`
)

func init() {
	plugins.RegisterStatusEventHandler(pluginName, handleStatusEvent)
}

type gitHubClient interface {
	CreateComment(owner, repo string, number int, comment string) error
	AddLabel(owner, repo string, number int, label string) error
	RemoveLabel(owner, repo string, number int, label string) error
	GetPullRequest(owner, repo string, number int) (*github.PullRequest, error)
	FindIssues(query string) ([]github.Issue, error)
}

func handleStatusEvent(pc plugins.PluginClient, se github.StatusEvent) error {
	return handle(pc.GitHubClient, pc.Logger, se)
}

// 1. Check that the status event received from the webhook is for the CNCF-CLA.
// 2. Use the github search API to search for the PRs which match the commit hash corresponding to the status event.
// 3. For each issue that matches, check that the PR's HEAD commit hash against the commit hash for which the status
//    was received. This is because we only care about the status associated with the last (latest) commit in a PR.
// 4. Set the corresponding CLA label if needed.
func handle(gc gitHubClient, log *logrus.Entry, se github.StatusEvent) error {
	if se.State == "" || se.Context == "" {
		return fmt.Errorf("invalid status event delivered with empty state/context")
	}

	if se.Context != claContextName {
		// Not the CNCF CLA context, do not process this.
		return nil
	}

	if se.State == github.StatusPending {
		// do nothing and wait for state to be updated.
		return nil
	}

	org := se.Repo.Owner.Login
	repo := se.Repo.Name
	log.Info("Searching for PRs matching the commit.")
	issues, err := gc.FindIssues(url.QueryEscape(fmt.Sprintf("%s repo:%s/%s type:pr state:open", se.SHA, org, repo)))
	if err != nil {
		return fmt.Errorf("error searching for issues matching commit: %v", err)
	}
	log.Infof("Found %d PRs matching commit.", len(issues))

	for _, issue := range issues {
		l := log.WithField("pr", issue.Number)
		hasCncfYes := issue.HasLabel(claYesLabel)
		hasCncfNo := issue.HasLabel(claNoLabel)
		if hasCncfYes && se.State == github.StatusSuccess {
			// Nothing to update.
			l.Infof("PR has up-to-date %s label.", claYesLabel)
			continue
		}

		if hasCncfNo && (se.State == github.StatusFailure || se.State == github.StatusError) {
			// Nothing to update.
			l.Infof("PR has up-to-date %s label.", claNoLabel)
			continue
		}

		l.Info("PR labels may be out of date. Getting pull request info.")
		pr, err := gc.GetPullRequest(org, repo, issue.Number)
		if err != nil {
			l.WithError(err).Warningf("Unable to fetch PR-%d from %s/%s.", issue.Number, org, repo)
			continue
		}

		// Check if this is the latest commit in the PR.
		if pr.Head.SHA != se.SHA {
			l.Info("Event is not for PR HEAD, skipping.")
			continue
		}

		number := pr.Number
		if se.State == github.StatusSuccess {
			if hasCncfNo {
				if err := gc.RemoveLabel(org, repo, number, claNoLabel); err != nil {
					l.WithError(err).Warningf("Could not remove %s label.", claNoLabel)
				}
			}
			if err := gc.AddLabel(org, repo, number, claYesLabel); err != nil {
				l.WithError(err).Warningf("Could not add %s label.", claYesLabel)
			}
			continue
		}

		// If we end up here, the status is a failure/error.
		if hasCncfYes {
			if err := gc.RemoveLabel(org, repo, number, claYesLabel); err != nil {
				l.WithError(err).Warningf("Could not remove %s label.", claYesLabel)
			}
		}
		if err := gc.CreateComment(org, repo, number, fmt.Sprintf(cncfclaNotFoundMessage, plugins.AboutThisBot)); err != nil {
			l.WithError(err).Warning("Could not create CLA not found comment.")
		}
		if err := gc.AddLabel(org, repo, number, claNoLabel); err != nil {
			l.WithError(err).Warningf("Could not add %s label.", claNoLabel)
		}
	}
	return nil
}
<commit_msg>Add retry loop to finding commits by sha.<commit_after>/*
Copyright 2016 The Kubernetes Authors.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package cla

import (
	"fmt"
	"net/url"
	"time"

	"github.com/Sirupsen/logrus"

	"k8s.io/test-infra/prow/github"
	"k8s.io/test-infra/prow/plugins"
)

const (
	pluginName             = "cla"
	claContextName         = "cla/linuxfoundation"
	claYesLabel            = "cncf-cla: yes"
	claNoLabel             = "cncf-cla: no"
	cncfclaNotFoundMessage = `Thanks for your pull request. Before we can look at your pull request, you'll need to sign a Contributor License Agreement (CLA).

:memo: **Please follow instructions at <https://github.com/kubernetes/kubernetes/wiki/CLA-FAQ> to sign the CLA.**

Once you've signed, please reply here (e.g. "I signed it!") and we'll verify.  Thanks.

---

- If you've already signed a CLA, it's possible we don't have your GitHub username or you're using a different email address.  Check your existing CLA data and verify that your [email is set on your git commits](https://help.github.com/articles/setting-your-email-in-git/).
- If you signed the CLA as a corporation, please sign in with your organization's credentials at <https://identity.linuxfoundation.org/projects/cncf> to be authorized.

<!-- need_sender_cla -->

<details>
%s
</details>
	`
	maxRetries = 5
)

func init() {
	plugins.RegisterStatusEventHandler(pluginName, handleStatusEvent)
}

type gitHubClient interface {
	CreateComment(owner, repo string, number int, comment string) error
	AddLabel(owner, repo string, number int, label string) error
	RemoveLabel(owner, repo string, number int, label string) error
	GetPullRequest(owner, repo string, number int) (*github.PullRequest, error)
	FindIssues(query string) ([]github.Issue, error)
}

func handleStatusEvent(pc plugins.PluginClient, se github.StatusEvent) error {
	return handle(pc.GitHubClient, pc.Logger, se)
}

// 1. Check that the status event received from the webhook is for the CNCF-CLA.
// 2. Use the github search API to search for the PRs which match the commit hash corresponding to the status event.
// 3. For each issue that matches, check that the PR's HEAD commit hash against the commit hash for which the status
//    was received. This is because we only care about the status associated with the last (latest) commit in a PR.
// 4. Set the corresponding CLA label if needed.
func handle(gc gitHubClient, log *logrus.Entry, se github.StatusEvent) error {
	if se.State == "" || se.Context == "" {
		return fmt.Errorf("invalid status event delivered with empty state/context")
	}

	if se.Context != claContextName {
		// Not the CNCF CLA context, do not process this.
		return nil
	}

	if se.State == github.StatusPending {
		// do nothing and wait for state to be updated.
		return nil
	}

	org := se.Repo.Owner.Login
	repo := se.Repo.Name
	log.Info("Searching for PRs matching the commit.")

	var issues []github.Issue
	var err error
	for i := 0; i < maxRetries; i++ {
		issues, err = gc.FindIssues(url.QueryEscape(fmt.Sprintf("%s repo:%s/%s type:pr state:open", se.SHA, org, repo)))
		if err != nil {
			return fmt.Errorf("error searching for issues matching commit: %v", err)
		}
		if len(issues) > 0 {
			break
		}
		time.Sleep(10 * time.Second)
	}
	log.Infof("Found %d PRs matching commit.", len(issues))

	for _, issue := range issues {
		l := log.WithField("pr", issue.Number)
		hasCncfYes := issue.HasLabel(claYesLabel)
		hasCncfNo := issue.HasLabel(claNoLabel)
		if hasCncfYes && se.State == github.StatusSuccess {
			// Nothing to update.
			l.Infof("PR has up-to-date %s label.", claYesLabel)
			continue
		}

		if hasCncfNo && (se.State == github.StatusFailure || se.State == github.StatusError) {
			// Nothing to update.
			l.Infof("PR has up-to-date %s label.", claNoLabel)
			continue
		}

		l.Info("PR labels may be out of date. Getting pull request info.")
		pr, err := gc.GetPullRequest(org, repo, issue.Number)
		if err != nil {
			l.WithError(err).Warningf("Unable to fetch PR-%d from %s/%s.", issue.Number, org, repo)
			continue
		}

		// Check if this is the latest commit in the PR.
		if pr.Head.SHA != se.SHA {
			l.Info("Event is not for PR HEAD, skipping.")
			continue
		}

		number := pr.Number
		if se.State == github.StatusSuccess {
			if hasCncfNo {
				if err := gc.RemoveLabel(org, repo, number, claNoLabel); err != nil {
					l.WithError(err).Warningf("Could not remove %s label.", claNoLabel)
				}
			}
			if err := gc.AddLabel(org, repo, number, claYesLabel); err != nil {
				l.WithError(err).Warningf("Could not add %s label.", claYesLabel)
			}
			continue
		}

		// If we end up here, the status is a failure/error.
		if hasCncfYes {
			if err := gc.RemoveLabel(org, repo, number, claYesLabel); err != nil {
				l.WithError(err).Warningf("Could not remove %s label.", claYesLabel)
			}
		}
		if err := gc.CreateComment(org, repo, number, fmt.Sprintf(cncfclaNotFoundMessage, plugins.AboutThisBot)); err != nil {
			l.WithError(err).Warning("Could not create CLA not found comment.")
		}
		if err := gc.AddLabel(org, repo, number, claNoLabel); err != nil {
			l.WithError(err).Warningf("Could not add %s label.", claNoLabel)
		}
	}
	return nil
}
<|endoftext|>


<commit_before>/*
Copyright 2015 The Kubernetes Authors All rights reserved.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package e2e

import (
	"fmt"
	"math"

	. "github.com/onsi/ginkgo"
	. "github.com/onsi/gomega"
	"k8s.io/kubernetes/pkg/api"
	"k8s.io/kubernetes/pkg/api/unversioned"
	client "k8s.io/kubernetes/pkg/client/unversioned"
	"k8s.io/kubernetes/pkg/labels"
	"k8s.io/kubernetes/pkg/util"
	"k8s.io/kubernetes/pkg/util/intstr"
	"k8s.io/kubernetes/pkg/util/sets"
	"k8s.io/kubernetes/test/e2e/framework"
)

var _ = framework.KubeDescribe("Ubernetes Lite", func() {
	f := framework.NewDefaultFramework("ubernetes-lite")
	var zoneCount int
	var err error
	image := "gcr.io/google_containers/serve_hostname:v1.4"
	BeforeEach(func() {
		if zoneCount <= 0 {
			zoneCount, err = getZoneCount(f.Client)
			Expect(err).NotTo(HaveOccurred())
		}
		By(fmt.Sprintf("Checking for multi-zone cluster.  Zone count = %d", zoneCount))
		framework.SkipUnlessAtLeast(zoneCount, 2, "Zone count is %d, only run for multi-zone clusters, skipping test")
		framework.SkipUnlessProviderIs("gce", "gke", "aws")
		// TODO: SkipUnlessDefaultScheduler() // Non-default schedulers might not spread
	})
	It("should spread the pods of a service across zones", func() {
		SpreadServiceOrFail(f, (2*zoneCount)+1, image)
	})

	It("should spread the pods of a replication controller across zones", func() {
		SpreadRCOrFail(f, int32((2*zoneCount)+1), image)
	})
})

// Check that the pods comprising a service get spread evenly across available zones
func SpreadServiceOrFail(f *framework.Framework, replicaCount int, image string) {
	// First create the service
	serviceName := "test-service"
	serviceSpec := &api.Service{
		ObjectMeta: api.ObjectMeta{
			Name:      serviceName,
			Namespace: f.Namespace.Name,
		},
		Spec: api.ServiceSpec{
			Selector: map[string]string{
				"service": serviceName,
			},
			Ports: []api.ServicePort{{
				Port:       80,
				TargetPort: intstr.FromInt(80),
			}},
		},
	}
	_, err := f.Client.Services(f.Namespace.Name).Create(serviceSpec)
	Expect(err).NotTo(HaveOccurred())

	// Now create some pods behind the service
	podSpec := &api.Pod{
		ObjectMeta: api.ObjectMeta{
			Name:   serviceName,
			Labels: map[string]string{"service": serviceName},
		},
		Spec: api.PodSpec{
			Containers: []api.Container{
				{
					Name:  "test",
					Image: framework.GetPauseImageName(f.Client),
				},
			},
		},
	}
	framework.StartPods(f.Client, replicaCount, f.Namespace.Name, serviceName, *podSpec, false)

	// Wait for all of them to be scheduled
	selector := labels.SelectorFromSet(labels.Set(map[string]string{"service": serviceName}))
	pods, err := framework.WaitForPodsWithLabelScheduled(f.Client, f.Namespace.Name, selector)
	Expect(err).NotTo(HaveOccurred())

	// Now make sure they're spread across zones
	zoneNames, err := getZoneNames(f.Client)
	Expect(err).NotTo(HaveOccurred())
	Expect(checkZoneSpreading(f.Client, pods, zoneNames)).To(Equal(true))
}

// Find the name of the zone in which a Node is running
func getZoneNameForNode(node api.Node) (string, error) {
	for key, value := range node.Labels {
		if key == unversioned.LabelZoneFailureDomain {
			return value, nil
		}
	}
	return "", fmt.Errorf("Zone name for node %s not found. No label with key %s",
		node.Name, unversioned.LabelZoneFailureDomain)
}

// Find the names of all zones in which we have nodes in this cluster.
func getZoneNames(c *client.Client) ([]string, error) {
	zoneNames := sets.NewString()
	nodes, err := c.Nodes().List(api.ListOptions{})
	if err != nil {
		return nil, err
	}
	for _, node := range nodes.Items {
		zoneName, err := getZoneNameForNode(node)
		Expect(err).NotTo(HaveOccurred())
		zoneNames.Insert(zoneName)
	}
	return zoneNames.List(), nil
}

// Return the number of zones in which we have nodes in this cluster.
func getZoneCount(c *client.Client) (int, error) {
	zoneNames, err := getZoneNames(c)
	if err != nil {
		return -1, err
	}
	return len(zoneNames), nil
}

// Find the name of the zone in which the pod is scheduled
func getZoneNameForPod(c *client.Client, pod api.Pod) (string, error) {
	By(fmt.Sprintf("Getting zone name for pod %s, on node %s", pod.Name, pod.Spec.NodeName))
	node, err := c.Nodes().Get(pod.Spec.NodeName)
	Expect(err).NotTo(HaveOccurred())
	return getZoneNameForNode(*node)
}

// Determine whether a set of pods are approximately evenly spread
// across a given set of zones
func checkZoneSpreading(c *client.Client, pods *api.PodList, zoneNames []string) (bool, error) {
	podsPerZone := make(map[string]int)
	for _, zoneName := range zoneNames {
		podsPerZone[zoneName] = 0
	}
	for _, pod := range pods.Items {
		if pod.DeletionTimestamp != nil {
			continue
		}
		zoneName, err := getZoneNameForPod(c, pod)
		Expect(err).NotTo(HaveOccurred())
		podsPerZone[zoneName] = podsPerZone[zoneName] + 1
	}
	minPodsPerZone := math.MaxInt32
	maxPodsPerZone := 0
	for _, podCount := range podsPerZone {
		if podCount < minPodsPerZone {
			minPodsPerZone = podCount
		}
		if podCount > maxPodsPerZone {
			maxPodsPerZone = podCount
		}
	}
	Expect(minPodsPerZone).To(BeNumerically("~", maxPodsPerZone, 1),
		"Pods were not evenly spread across zones.  %d in one zone and %d in another zone",
		minPodsPerZone, maxPodsPerZone)
	return true, nil
}

// Check that the pods comprising a replication controller get spread evenly across available zones
func SpreadRCOrFail(f *framework.Framework, replicaCount int32, image string) {
	name := "ubelite-spread-rc-" + string(util.NewUUID())
	By(fmt.Sprintf("Creating replication controller %s", name))
	controller, err := f.Client.ReplicationControllers(f.Namespace.Name).Create(&api.ReplicationController{
		ObjectMeta: api.ObjectMeta{
			Namespace: f.Namespace.Name,
			Name:      name,
		},
		Spec: api.ReplicationControllerSpec{
			Replicas: replicaCount,
			Selector: map[string]string{
				"name": name,
			},
			Template: &api.PodTemplateSpec{
				ObjectMeta: api.ObjectMeta{
					Labels: map[string]string{"name": name},
				},
				Spec: api.PodSpec{
					Containers: []api.Container{
						{
							Name:  name,
							Image: image,
							Ports: []api.ContainerPort{{ContainerPort: 9376}},
						},
					},
				},
			},
		},
	})
	Expect(err).NotTo(HaveOccurred())
	// Cleanup the replication controller when we are done.
	defer func() {
		// Resize the replication controller to zero to get rid of pods.
		if err := framework.DeleteRC(f.Client, f.Namespace.Name, controller.Name); err != nil {
			framework.Logf("Failed to cleanup replication controller %v: %v.", controller.Name, err)
		}
	}()
	// List the pods, making sure we observe all the replicas.
	selector := labels.SelectorFromSet(labels.Set(map[string]string{"name": name}))
	pods, err := framework.PodsCreated(f.Client, f.Namespace.Name, name, replicaCount)
	Expect(err).NotTo(HaveOccurred())

	// Wait for all of them to be scheduled
	By(fmt.Sprintf("Waiting for %d replicas of %s to be scheduled.  Selector: %v", replicaCount, name, selector))
	pods, err = framework.WaitForPodsWithLabelScheduled(f.Client, f.Namespace.Name, selector)
	Expect(err).NotTo(HaveOccurred())

	// Now make sure they're spread across zones
	zoneNames, err := getZoneNames(f.Client)
	Expect(err).NotTo(HaveOccurred())
	Expect(checkZoneSpreading(f.Client, pods, zoneNames)).To(Equal(true))
}
<commit_msg>Check provider first<commit_after>/*
Copyright 2015 The Kubernetes Authors All rights reserved.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package e2e

import (
	"fmt"
	"math"

	. "github.com/onsi/ginkgo"
	. "github.com/onsi/gomega"
	"k8s.io/kubernetes/pkg/api"
	"k8s.io/kubernetes/pkg/api/unversioned"
	client "k8s.io/kubernetes/pkg/client/unversioned"
	"k8s.io/kubernetes/pkg/labels"
	"k8s.io/kubernetes/pkg/util"
	"k8s.io/kubernetes/pkg/util/intstr"
	"k8s.io/kubernetes/pkg/util/sets"
	"k8s.io/kubernetes/test/e2e/framework"
)

var _ = framework.KubeDescribe("Ubernetes Lite", func() {
	f := framework.NewDefaultFramework("ubernetes-lite")
	var zoneCount int
	var err error
	image := "gcr.io/google_containers/serve_hostname:v1.4"
	BeforeEach(func() {
		framework.SkipUnlessProviderIs("gce", "gke", "aws")
		if zoneCount <= 0 {
			zoneCount, err = getZoneCount(f.Client)
			Expect(err).NotTo(HaveOccurred())
		}
		By(fmt.Sprintf("Checking for multi-zone cluster.  Zone count = %d", zoneCount))
		framework.SkipUnlessAtLeast(zoneCount, 2, "Zone count is %d, only run for multi-zone clusters, skipping test")
		// TODO: SkipUnlessDefaultScheduler() // Non-default schedulers might not spread
	})
	It("should spread the pods of a service across zones", func() {
		SpreadServiceOrFail(f, (2*zoneCount)+1, image)
	})

	It("should spread the pods of a replication controller across zones", func() {
		SpreadRCOrFail(f, int32((2*zoneCount)+1), image)
	})
})

// Check that the pods comprising a service get spread evenly across available zones
func SpreadServiceOrFail(f *framework.Framework, replicaCount int, image string) {
	// First create the service
	serviceName := "test-service"
	serviceSpec := &api.Service{
		ObjectMeta: api.ObjectMeta{
			Name:      serviceName,
			Namespace: f.Namespace.Name,
		},
		Spec: api.ServiceSpec{
			Selector: map[string]string{
				"service": serviceName,
			},
			Ports: []api.ServicePort{{
				Port:       80,
				TargetPort: intstr.FromInt(80),
			}},
		},
	}
	_, err := f.Client.Services(f.Namespace.Name).Create(serviceSpec)
	Expect(err).NotTo(HaveOccurred())

	// Now create some pods behind the service
	podSpec := &api.Pod{
		ObjectMeta: api.ObjectMeta{
			Name:   serviceName,
			Labels: map[string]string{"service": serviceName},
		},
		Spec: api.PodSpec{
			Containers: []api.Container{
				{
					Name:  "test",
					Image: framework.GetPauseImageName(f.Client),
				},
			},
		},
	}
	framework.StartPods(f.Client, replicaCount, f.Namespace.Name, serviceName, *podSpec, false)

	// Wait for all of them to be scheduled
	selector := labels.SelectorFromSet(labels.Set(map[string]string{"service": serviceName}))
	pods, err := framework.WaitForPodsWithLabelScheduled(f.Client, f.Namespace.Name, selector)
	Expect(err).NotTo(HaveOccurred())

	// Now make sure they're spread across zones
	zoneNames, err := getZoneNames(f.Client)
	Expect(err).NotTo(HaveOccurred())
	Expect(checkZoneSpreading(f.Client, pods, zoneNames)).To(Equal(true))
}

// Find the name of the zone in which a Node is running
func getZoneNameForNode(node api.Node) (string, error) {
	for key, value := range node.Labels {
		if key == unversioned.LabelZoneFailureDomain {
			return value, nil
		}
	}
	return "", fmt.Errorf("Zone name for node %s not found. No label with key %s",
		node.Name, unversioned.LabelZoneFailureDomain)
}

// Find the names of all zones in which we have nodes in this cluster.
func getZoneNames(c *client.Client) ([]string, error) {
	zoneNames := sets.NewString()
	nodes, err := c.Nodes().List(api.ListOptions{})
	if err != nil {
		return nil, err
	}
	for _, node := range nodes.Items {
		zoneName, err := getZoneNameForNode(node)
		Expect(err).NotTo(HaveOccurred())
		zoneNames.Insert(zoneName)
	}
	return zoneNames.List(), nil
}

// Return the number of zones in which we have nodes in this cluster.
func getZoneCount(c *client.Client) (int, error) {
	zoneNames, err := getZoneNames(c)
	if err != nil {
		return -1, err
	}
	return len(zoneNames), nil
}

// Find the name of the zone in which the pod is scheduled
func getZoneNameForPod(c *client.Client, pod api.Pod) (string, error) {
	By(fmt.Sprintf("Getting zone name for pod %s, on node %s", pod.Name, pod.Spec.NodeName))
	node, err := c.Nodes().Get(pod.Spec.NodeName)
	Expect(err).NotTo(HaveOccurred())
	return getZoneNameForNode(*node)
}

// Determine whether a set of pods are approximately evenly spread
// across a given set of zones
func checkZoneSpreading(c *client.Client, pods *api.PodList, zoneNames []string) (bool, error) {
	podsPerZone := make(map[string]int)
	for _, zoneName := range zoneNames {
		podsPerZone[zoneName] = 0
	}
	for _, pod := range pods.Items {
		if pod.DeletionTimestamp != nil {
			continue
		}
		zoneName, err := getZoneNameForPod(c, pod)
		Expect(err).NotTo(HaveOccurred())
		podsPerZone[zoneName] = podsPerZone[zoneName] + 1
	}
	minPodsPerZone := math.MaxInt32
	maxPodsPerZone := 0
	for _, podCount := range podsPerZone {
		if podCount < minPodsPerZone {
			minPodsPerZone = podCount
		}
		if podCount > maxPodsPerZone {
			maxPodsPerZone = podCount
		}
	}
	Expect(minPodsPerZone).To(BeNumerically("~", maxPodsPerZone, 1),
		"Pods were not evenly spread across zones.  %d in one zone and %d in another zone",
		minPodsPerZone, maxPodsPerZone)
	return true, nil
}

// Check that the pods comprising a replication controller get spread evenly across available zones
func SpreadRCOrFail(f *framework.Framework, replicaCount int32, image string) {
	name := "ubelite-spread-rc-" + string(util.NewUUID())
	By(fmt.Sprintf("Creating replication controller %s", name))
	controller, err := f.Client.ReplicationControllers(f.Namespace.Name).Create(&api.ReplicationController{
		ObjectMeta: api.ObjectMeta{
			Namespace: f.Namespace.Name,
			Name:      name,
		},
		Spec: api.ReplicationControllerSpec{
			Replicas: replicaCount,
			Selector: map[string]string{
				"name": name,
			},
			Template: &api.PodTemplateSpec{
				ObjectMeta: api.ObjectMeta{
					Labels: map[string]string{"name": name},
				},
				Spec: api.PodSpec{
					Containers: []api.Container{
						{
							Name:  name,
							Image: image,
							Ports: []api.ContainerPort{{ContainerPort: 9376}},
						},
					},
				},
			},
		},
	})
	Expect(err).NotTo(HaveOccurred())
	// Cleanup the replication controller when we are done.
	defer func() {
		// Resize the replication controller to zero to get rid of pods.
		if err := framework.DeleteRC(f.Client, f.Namespace.Name, controller.Name); err != nil {
			framework.Logf("Failed to cleanup replication controller %v: %v.", controller.Name, err)
		}
	}()
	// List the pods, making sure we observe all the replicas.
	selector := labels.SelectorFromSet(labels.Set(map[string]string{"name": name}))
	pods, err := framework.PodsCreated(f.Client, f.Namespace.Name, name, replicaCount)
	Expect(err).NotTo(HaveOccurred())

	// Wait for all of them to be scheduled
	By(fmt.Sprintf("Waiting for %d replicas of %s to be scheduled.  Selector: %v", replicaCount, name, selector))
	pods, err = framework.WaitForPodsWithLabelScheduled(f.Client, f.Namespace.Name, selector)
	Expect(err).NotTo(HaveOccurred())

	// Now make sure they're spread across zones
	zoneNames, err := getZoneNames(f.Client)
	Expect(err).NotTo(HaveOccurred())
	Expect(checkZoneSpreading(f.Client, pods, zoneNames)).To(Equal(true))
}
<|endoftext|>