<commit_before>/*
 * Copyright 2018 Google Inc. All rights reserved.
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 *   http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */

// Binary repotester tests the high level validity of repo extraction. It tries
// to run similar logic to sibling binary extractrepo on a specified target
// repositories, and reports results.
//
// For simple extraction, the results are merely based on the fraction of java
// files in the repo that end up in the kindex files.
//
// An extraction config can be optionally read from a specified file.  The
// format follows kythe.proto.ExtractionConfiguration.
//
// Usage:
//   repotester -repos <comma_delimited,repo_urls> [-config <config_file_path>]
//   repotester -repo_list_file <file> [-config <config_file_path>]
package main

import (
	"bufio"
	"context"
	"flag"
	"fmt"
	"io/ioutil"
	"log"
	"os"
	"path"
	"path/filepath"
	"regexp"
	"strings"

	"github.com/google/go-github/github"
	// TODO(danielmoy): auth!
	// "golang.org/x/oauth2"
	"kythe.io/kythe/go/extractors/config"
	"kythe.io/kythe/go/platform/kindex"
)

var (
	repos     = flag.String("repos", "", "A comma delimited list of repos to test.")
	reposFile = flag.String("repo_list_file", "", "A file that contains a newline delimited list of repos to test.")
	// TODO(danielmoy): auth!
	// githubToken = flag.String("github_token", "", "An oauth2 token to contact github with. https://help.github.com/articles/creating-a-personal-access-token-for-the-command-line/ to generate.")
	configPath = flag.String("config", "", "An optional config to specify for every repo.")
)

func init() {
	flag.Usage = func() {
		fmt.Fprintf(os.Stderr, `Usage: %s -repos <comma_delimited,repo_urls>
%s -repo_list_file <file_containing_line_delimited_repo_urls>

This tool tests repo extraction. If specifying file list format, you can also
specify a config file comma separated after the repo:

https://repo.url, /file/path/to/config

Any config specified in this way overwrites the default top-level -config passed
as a binary flag.

This binary requires both Git and Docker to be on the $PATH during execution.

Options:
`, filepath.Base(os.Args[0]))
		flag.PrintDefaults()
	}
}

func main() {
	flag.Parse()
	verifyFlags()

	// Print some header
	fmt.Printf("|%9s |%9s |%9s | %s\n", "download", "extract", "coverage", "repo")
	fmt.Printf("|%9s |%9s |%9s |%s\n", " ----", " ----", " ----", " ----")

	repos, err := getRepos()
	if err != nil {
		log.Fatalf("Failed to get repos to read: %v", err)
	}
	for _, repo := range repos {
		res, err := testRepo(repo)
		if err != nil {
			log.Printf("Failed to test repo: %s", err)
		} else {
			fmt.Printf("|%9t |%9t |      %2.0f%% | %s\n", res.downloaded, res.extracted, 100*res.fileCoverage, repo)
		}
	}
}

func verifyFlags() {
	if flag.NArg() > 0 {
		log.Fatalf("Unknown arguments: %v", flag.Args())
	}
	if (*repos == "" && *reposFile == "") || (*repos != "" && *reposFile != "") {
		log.Fatalf("Must specify one of -repos or -repo_list_file, but no both.")
	}
	// TODO(danielmoy): auth!
	// if *githubToken == "" {
	// 	log.Fatalf("Must specify -github_token.")
	// }
}

func getRepos() ([]string, error) {
	switch {
	case *repos != "":
		return strings.Split(*repos, ","), nil
	case *reposFile != "":
		return getReposFromFile()
	default:
		return nil, fmt.Errorf("Invalid state - need a source of repos")
	}
}

func getReposFromFile() ([]string, error) {
	file, err := os.Open(*reposFile)
	if err != nil {
		return nil, fmt.Errorf("cannot read repo file: %v", err)
	}
	defer file.Close()

	ret := []string{}
	scanner := bufio.NewScanner(file)
	for scanner.Scan() {
		ret = append(ret, scanner.Text())
	}
	if err := scanner.Err(); err != nil {
		return nil, fmt.Errorf("error reading repo file: %v", err)
	}
	return ret, nil
}

// result is a simple container for the results of a single repo test.  It may
// contain useful information about whether or not the repo was accessible,
// extracted at all, or the extent to which we got good file coverage from the
// extraction.
type result struct {
	// Whether the repo was successfully downloaded or extracted.
	downloaded, extracted bool
	// Should be in range [0.0, 1.0]
	fileCoverage float32
}

func testRepo(repo string) (result, error) {
	fromExtraction, err := filenamesFromExtraction(repo)
	if err != nil {
		log.Printf("Failed to extract repo: %v", err)
		// TODO(danielmoy): consider handling errors independently and
		// returning separate false results if either err != nil.
		return result{false, false, 0.0}, nil
	}
	fromRepo, err := filenamesFromRepo(repo)
	if err != nil {
		log.Printf("Failed to read repo from remote: %v", err)
		return result{false, true, 0.0}, nil
	}

	var coverageTotal int32
	var coverageCount int32
	// TODO(danielmoy): the repos won't necessarily line up properly. This
	// needs to be fixed to be more extensible. Potentially with a suffix
	// trie on successive path elements (basename and then directory
	// backwards).
	for k := range fromRepo {
		coverageTotal = coverageTotal + 1
		if _, ok := fromExtraction[k]; ok {
			coverageCount = coverageCount + 1
		}
	}

	return result{
		downloaded:   true,
		extracted:    true,
		fileCoverage: float32(coverageCount) / float32(coverageTotal),
	}, nil
}

// gitpath is a container for storing actual paths, since what github API calls
// "path" is actually just a basename, and we need the full path.
type gitpath struct {
	sha, path string
}

func filenamesFromRepo(repoURL string) (map[string]bool, error) {
	// TODO(danielmoy): auth!
	// src := oauth2.StaticTokenSource(&oauth2.Token{AccessToken: *githubToken})
	// httpClient := oauth.NewClient(context.Background(), src)

	owner, repoName, err := getNames(repoURL)
	if err != nil {
		return nil, err
	}

	client := github.NewClient(nil)
	rootTree, err := getRootTree(client, owner, repoName)
	if err != nil {
		return nil, err
	}
	if rootTree == nil {
		return nil, fmt.Errorf("Failed to get commit tree for repo %s/%s", owner, repoName)
	}

	// The tree will recursively contain stuff, so build up a queue-like thingy
	// and go to work.
	if rootTree.SHA == nil {
		return nil, fmt.Errorf("Failed to get any tree data for repo %s/%s", owner, repoName)
	}
	trees := []gitpath{gitpath{*rootTree.SHA, ""}}

	ret := map[string]bool{}
	// TODO(danielmoy): consider parallelism in here, within reasonable
	// bounds given rate limiting.
	for len(trees) > 0 {
		tree := trees[0]
		trees = trees[1:]
		contents, err := readTree(client, owner, repoName, tree.sha)
		if err != nil {
			return nil, err
			continue
		}
		if contents == nil {
			return nil, fmt.Errorf("failed to read repo %s/%s tree %s", owner, repoName, tree.sha)
		}
		for _, entry := range contents.Entries {
			if entry.SHA == nil || entry.Path == nil {
				return nil, fmt.Errorf("failed to read repo %s/%s tree %s", owner, repoName, tree.sha)
			}
			newpath := path.Join(tree.path, *entry.Path)
			switch *entry.Type {
			case "blob":
				appendFile(ret, newpath)
			case "tree":
				if entry.SHA != nil {
					trees = append(trees, gitpath{*entry.SHA, newpath})
				}
			default:
				log.Printf("Unknown tree entry %s", entry.Type)
			}
		}
	}

	return ret, nil
}

func getRootTree(client *github.Client, owner, repo string) (*github.Tree, error) {
	opt := &github.CommitsListOptions{ListOptions: github.ListOptions{PerPage: 1}}
	repos, _, err := client.Repositories.ListCommits(context.Background(), owner, repo, opt)

	if err != nil {
		return nil, err
	}
	if len(repos) == 0 {
		return nil, fmt.Errorf("failed to find latest commit for repo %s/%s", owner, repo)
	}
	if len(repos) > 1 {
		log.Fatalf("Somehow got more than one commit for repo %s/%s", owner, repo)
	}
	sha := repos[0].SHA
	if sha == nil {
		return nil, fmt.Errorf("failed to get commit hash for repo %s/%s commit %v", owner, repo, repos[0])
	}
	commit, _, err := client.Git.GetCommit(context.Background(), owner, repo, *sha)
	if commit == nil || err != nil {
		return nil, fmt.Errorf("failed to get commit data for repo %s/%s commit %v", owner, repo, repos[0])
	}
	return commit.Tree, nil
}

// getNames tries to extract the owner/repo from a github repo url.  If the
// passed string is not supported, returns an error to that effect.
func getNames(repo string) (string, string, error) {
	re := regexp.MustCompile(`https://github.com/(?P<Owner>\w+)/(?P<Repo>[\w-_]+)`)
	n := re.FindStringSubmatch(repo)
	// Recall that regex libraries like returning the whole matched thing as
	// the first bit, because... reasons.  Anyways just ignore it.
	if n == nil || len(n) != 3 {
		return "", "", fmt.Errorf("failed to parse repo %s", repo)
	}
	return n[1], n[2], nil
}

func readTree(client *github.Client, owner, repo, treeSHA string) (*github.Tree, error) {
	// Note we don't read recursively (, false), because it only supports
	// max of 200 files.
	tree, _, err := client.Git.GetTree(context.Background(), owner, repo, treeSHA, false)
	if tree == nil || err != nil {
		return nil, err
	}
	return tree, nil
}

// appendFile sees if this is a supported file type, and then wappends it to the
// list of known files.
func appendFile(ret map[string]bool, path string) {
	if strings.HasSuffix(path, ".java") {
		ret[path] = true
	}
	return
}

func filenamesFromExtraction(repo string) (map[string]bool, error) {
	_, repoName, err := getNames(repo)
	tmpOutDir, err := ioutil.TempDir("", repoName)
	if err != nil {
		return nil, fmt.Errorf("failed to create temp dir for repo %s: %v", repo, err)
	}
	defer os.RemoveAll(tmpOutDir)

	err = config.ExtractRepo(repo, tmpOutDir, *configPath)
	ret := map[string]bool{}
	if err != nil {
		return ret, err
	}
	walkFunc := func(path string, info os.FileInfo, err error) error {
		if err != nil {
			return err
		}
		if strings.HasSuffix(path, ".kindex") {
			cu, err := kindex.Open(context.Background(), path)
			if err != nil {
				return err
			}
			if cu.Proto != nil {
				for _, v := range cu.Proto.SourceFile {
					if strings.HasSuffix(v, ".java") {
						ret[v] = true
					}
				}
			}
		}
		return nil
	}

	err = filepath.Walk(tmpOutDir, walkFunc)
	return ret, err
}
<commit_msg>Fix lint error arising from refactored flag usage string.<commit_after>/*
 * Copyright 2018 Google Inc. All rights reserved.
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 *   http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */

// Binary repotester tests the high level validity of repo extraction. It tries
// to run similar logic to sibling binary extractrepo on a specified target
// repositories, and reports results.
//
// For simple extraction, the results are merely based on the fraction of java
// files in the repo that end up in the kindex files.
//
// An extraction config can be optionally read from a specified file.  The
// format follows kythe.proto.ExtractionConfiguration.
//
// Usage:
//   repotester -repos <comma_delimited,repo_urls> [-config <config_file_path>]
//   repotester -repo_list_file <file> [-config <config_file_path>]
package main

import (
	"bufio"
	"context"
	"flag"
	"fmt"
	"io/ioutil"
	"log"
	"os"
	"path"
	"path/filepath"
	"regexp"
	"strings"

	"github.com/google/go-github/github"
	// TODO(danielmoy): auth!
	// "golang.org/x/oauth2"
	"kythe.io/kythe/go/extractors/config"
	"kythe.io/kythe/go/platform/kindex"
)

var (
	repos     = flag.String("repos", "", "A comma delimited list of repos to test.")
	reposFile = flag.String("repo_list_file", "", "A file that contains a newline delimited list of repos to test.")
	// TODO(danielmoy): auth!
	// githubToken = flag.String("github_token", "", "An oauth2 token to contact github with. https://help.github.com/articles/creating-a-personal-access-token-for-the-command-line/ to generate.")
	configPath = flag.String("config", "", "An optional config to specify for every repo.")
)

func init() {
	flag.Usage = func() {
		binary := filepath.Base(os.Args[0])
		fmt.Fprintf(os.Stderr, `Usage: %s -repos <comma_delimited,repo_urls>
%s -repo_list_file <file_containing_line_delimited_repo_urls>

This tool tests repo extraction. If specifying file list format, you can also
specify a config file comma separated after the repo:

https://repo.url, /file/path/to/config

Any config specified in this way overwrites the default top-level -config passed
as a binary flag.

This binary requires both Git and Docker to be on the $PATH during execution.

Options:
`, binary, binary)
		flag.PrintDefaults()
	}
}

func main() {
	flag.Parse()
	verifyFlags()

	// Print some header
	fmt.Printf("|%9s |%9s |%9s | %s\n", "download", "extract", "coverage", "repo")
	fmt.Printf("|%9s |%9s |%9s |%s\n", " ----", " ----", " ----", " ----")

	repos, err := getRepos()
	if err != nil {
		log.Fatalf("Failed to get repos to read: %v", err)
	}
	for _, repo := range repos {
		res, err := testRepo(repo)
		if err != nil {
			log.Printf("Failed to test repo: %s", err)
		} else {
			fmt.Printf("|%9t |%9t |      %2.0f%% | %s\n", res.downloaded, res.extracted, 100*res.fileCoverage, repo)
		}
	}
}

func verifyFlags() {
	if flag.NArg() > 0 {
		log.Fatalf("Unknown arguments: %v", flag.Args())
	}
	if (*repos == "" && *reposFile == "") || (*repos != "" && *reposFile != "") {
		log.Fatalf("Must specify one of -repos or -repo_list_file, but no both.")
	}
	// TODO(danielmoy): auth!
	// if *githubToken == "" {
	// 	log.Fatalf("Must specify -github_token.")
	// }
}

func getRepos() ([]string, error) {
	switch {
	case *repos != "":
		return strings.Split(*repos, ","), nil
	case *reposFile != "":
		return getReposFromFile()
	default:
		return nil, fmt.Errorf("Invalid state - need a source of repos")
	}
}

func getReposFromFile() ([]string, error) {
	file, err := os.Open(*reposFile)
	if err != nil {
		return nil, fmt.Errorf("cannot read repo file: %v", err)
	}
	defer file.Close()

	ret := []string{}
	scanner := bufio.NewScanner(file)
	for scanner.Scan() {
		ret = append(ret, scanner.Text())
	}
	if err := scanner.Err(); err != nil {
		return nil, fmt.Errorf("error reading repo file: %v", err)
	}
	return ret, nil
}

// result is a simple container for the results of a single repo test.  It may
// contain useful information about whether or not the repo was accessible,
// extracted at all, or the extent to which we got good file coverage from the
// extraction.
type result struct {
	// Whether the repo was successfully downloaded or extracted.
	downloaded, extracted bool
	// Should be in range [0.0, 1.0]
	fileCoverage float32
}

func testRepo(repo string) (result, error) {
	fromExtraction, err := filenamesFromExtraction(repo)
	if err != nil {
		log.Printf("Failed to extract repo: %v", err)
		// TODO(danielmoy): consider handling errors independently and
		// returning separate false results if either err != nil.
		return result{false, false, 0.0}, nil
	}
	fromRepo, err := filenamesFromRepo(repo)
	if err != nil {
		log.Printf("Failed to read repo from remote: %v", err)
		return result{false, true, 0.0}, nil
	}

	var coverageTotal int32
	var coverageCount int32
	// TODO(danielmoy): the repos won't necessarily line up properly. This
	// needs to be fixed to be more extensible. Potentially with a suffix
	// trie on successive path elements (basename and then directory
	// backwards).
	for k := range fromRepo {
		coverageTotal = coverageTotal + 1
		if _, ok := fromExtraction[k]; ok {
			coverageCount = coverageCount + 1
		}
	}

	return result{
		downloaded:   true,
		extracted:    true,
		fileCoverage: float32(coverageCount) / float32(coverageTotal),
	}, nil
}

// gitpath is a container for storing actual paths, since what github API calls
// "path" is actually just a basename, and we need the full path.
type gitpath struct {
	sha, path string
}

func filenamesFromRepo(repoURL string) (map[string]bool, error) {
	// TODO(danielmoy): auth!
	// src := oauth2.StaticTokenSource(&oauth2.Token{AccessToken: *githubToken})
	// httpClient := oauth.NewClient(context.Background(), src)

	owner, repoName, err := getNames(repoURL)
	if err != nil {
		return nil, err
	}

	client := github.NewClient(nil)
	rootTree, err := getRootTree(client, owner, repoName)
	if err != nil {
		return nil, err
	}
	if rootTree == nil {
		return nil, fmt.Errorf("Failed to get commit tree for repo %s/%s", owner, repoName)
	}

	// The tree will recursively contain stuff, so build up a queue-like thingy
	// and go to work.
	if rootTree.SHA == nil {
		return nil, fmt.Errorf("Failed to get any tree data for repo %s/%s", owner, repoName)
	}
	trees := []gitpath{gitpath{*rootTree.SHA, ""}}

	ret := map[string]bool{}
	// TODO(danielmoy): consider parallelism in here, within reasonable
	// bounds given rate limiting.
	for len(trees) > 0 {
		tree := trees[0]
		trees = trees[1:]
		contents, err := readTree(client, owner, repoName, tree.sha)
		if err != nil {
			return nil, err
			continue
		}
		if contents == nil {
			return nil, fmt.Errorf("failed to read repo %s/%s tree %s", owner, repoName, tree.sha)
		}
		for _, entry := range contents.Entries {
			if entry.SHA == nil || entry.Path == nil {
				return nil, fmt.Errorf("failed to read repo %s/%s tree %s", owner, repoName, tree.sha)
			}
			newpath := path.Join(tree.path, *entry.Path)
			switch *entry.Type {
			case "blob":
				appendFile(ret, newpath)
			case "tree":
				if entry.SHA != nil {
					trees = append(trees, gitpath{*entry.SHA, newpath})
				}
			default:
				log.Printf("Unknown tree entry %s", entry.Type)
			}
		}
	}

	return ret, nil
}

func getRootTree(client *github.Client, owner, repo string) (*github.Tree, error) {
	opt := &github.CommitsListOptions{ListOptions: github.ListOptions{PerPage: 1}}
	repos, _, err := client.Repositories.ListCommits(context.Background(), owner, repo, opt)

	if err != nil {
		return nil, err
	}
	if len(repos) == 0 {
		return nil, fmt.Errorf("failed to find latest commit for repo %s/%s", owner, repo)
	}
	if len(repos) > 1 {
		log.Fatalf("Somehow got more than one commit for repo %s/%s", owner, repo)
	}
	sha := repos[0].SHA
	if sha == nil {
		return nil, fmt.Errorf("failed to get commit hash for repo %s/%s commit %v", owner, repo, repos[0])
	}
	commit, _, err := client.Git.GetCommit(context.Background(), owner, repo, *sha)
	if commit == nil || err != nil {
		return nil, fmt.Errorf("failed to get commit data for repo %s/%s commit %v", owner, repo, repos[0])
	}
	return commit.Tree, nil
}

// getNames tries to extract the owner/repo from a github repo url.  If the
// passed string is not supported, returns an error to that effect.
func getNames(repo string) (string, string, error) {
	re := regexp.MustCompile(`https://github.com/(?P<Owner>\w+)/(?P<Repo>[\w-_]+)`)
	n := re.FindStringSubmatch(repo)
	// Recall that regex libraries like returning the whole matched thing as
	// the first bit, because... reasons.  Anyways just ignore it.
	if n == nil || len(n) != 3 {
		return "", "", fmt.Errorf("failed to parse repo %s", repo)
	}
	return n[1], n[2], nil
}

func readTree(client *github.Client, owner, repo, treeSHA string) (*github.Tree, error) {
	// Note we don't read recursively (, false), because it only supports
	// max of 200 files.
	tree, _, err := client.Git.GetTree(context.Background(), owner, repo, treeSHA, false)
	if tree == nil || err != nil {
		return nil, err
	}
	return tree, nil
}

// appendFile sees if this is a supported file type, and then wappends it to the
// list of known files.
func appendFile(ret map[string]bool, path string) {
	if strings.HasSuffix(path, ".java") {
		ret[path] = true
	}
	return
}

func filenamesFromExtraction(repo string) (map[string]bool, error) {
	_, repoName, err := getNames(repo)
	tmpOutDir, err := ioutil.TempDir("", repoName)
	if err != nil {
		return nil, fmt.Errorf("failed to create temp dir for repo %s: %v", repo, err)
	}
	defer os.RemoveAll(tmpOutDir)

	err = config.ExtractRepo(repo, tmpOutDir, *configPath)
	ret := map[string]bool{}
	if err != nil {
		return ret, err
	}
	walkFunc := func(path string, info os.FileInfo, err error) error {
		if err != nil {
			return err
		}
		if strings.HasSuffix(path, ".kindex") {
			cu, err := kindex.Open(context.Background(), path)
			if err != nil {
				return err
			}
			if cu.Proto != nil {
				for _, v := range cu.Proto.SourceFile {
					if strings.HasSuffix(v, ".java") {
						ret[v] = true
					}
				}
			}
		}
		return nil
	}

	err = filepath.Walk(tmpOutDir, walkFunc)
	return ret, err
}
<|endoftext|>


<commit_before>package uploader_test

import (
	"crypto/md5"
	"crypto/tls"
	"crypto/x509"
	"encoding/base64"
	"fmt"
	"io/ioutil"
	"net/http"
	"net/http/httptest"
	"net/url"
	"os"
	"strconv"
	"sync"
	"time"

	"code.cloudfoundry.org/cfhttp"
	"code.cloudfoundry.org/executor/depot/uploader"
	"code.cloudfoundry.org/lager/lagertest"
	"code.cloudfoundry.org/tlsconfig"

	. "github.com/onsi/ginkgo"
	. "github.com/onsi/gomega"
	"github.com/onsi/gomega/gbytes"
)

var _ = Describe("Uploader", func() {
	var (
		upldr             uploader.Uploader
		testServer        *httptest.Server
		serverRequests    []*http.Request
		serverRequestBody []string

		logger        *lagertest.TestLogger
		url           *url.URL
		file          *os.File
		expectedBytes int
		expectedMD5   string
	)

	BeforeEach(func() {
		testServer = nil
		serverRequestBody = []string{}
		serverRequests = []*http.Request{}
		logger = lagertest.NewTestLogger("test")

		file, _ = ioutil.TempFile("", "foo")
		contentString := "content that we can check later"
		expectedBytes, _ = file.WriteString(contentString)
		rawMD5 := md5.Sum([]byte(contentString))
		expectedMD5 = base64.StdEncoding.EncodeToString(rawMD5[:])
		file.Close()
	})

	AfterEach(func() {
		file.Close()
		if testServer != nil {
			testServer.Close()
		}
		os.Remove(file.Name())
	})

	Describe("Insecure Upload", func() {
		BeforeEach(func() {
			upldr = uploader.New(logger, 100*time.Millisecond, nil)
		})

		Context("when the upload is successful", func() {
			BeforeEach(func() {
				testServer = httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
					serverRequests = append(serverRequests, r)

					data, err := ioutil.ReadAll(r.Body)
					Expect(err).NotTo(HaveOccurred())
					serverRequestBody = append(serverRequestBody, string(data))

					fmt.Fprintln(w, "Hello, client")
				}))

				serverUrl := testServer.URL + "/somepath"
				url, _ = url.Parse(serverUrl)
			})

			var err error
			var numBytes int64
			JustBeforeEach(func() {
				numBytes, err = upldr.Upload(file.Name(), url, nil)
			})

			It("uploads the file to the url", func() {
				Expect(len(serverRequests)).To(Equal(1))

				request := serverRequests[0]
				data := serverRequestBody[0]

				Expect(request.URL.Path).To(Equal("/somepath"))
				Expect(request.Header.Get("Content-Type")).To(Equal("application/octet-stream"))
				Expect(request.Header.Get("Content-MD5")).To(Equal(expectedMD5))
				Expect(strconv.Atoi(request.Header.Get("Content-Length"))).To(BeNumerically("==", 31))
				Expect(string(data)).To(Equal("content that we can check later"))
			})

			It("returns the number of bytes written", func() {
				Expect(numBytes).To(Equal(int64(expectedBytes)))
			})

			It("does not return an error", func() {
				Expect(err).NotTo(HaveOccurred())
			})
		})

		Context("when the upload is canceled", func() {
			var flushRequests chan struct{}
			var requestsInFlight *sync.WaitGroup

			BeforeEach(func() {
				flushRequests = make(chan struct{})

				requestsInFlight = new(sync.WaitGroup)
				testServer = httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
					defer requestsInFlight.Done()
					<-flushRequests
				}))

				serverUrl := testServer.URL + "/somepath"
				url, _ = url.Parse(serverUrl)
			})

			AfterEach(func() {
				close(flushRequests)
				requestsInFlight.Wait()
			})

			It("interrupts the client and returns an error", func() {
				upldrWithoutTimeout := uploader.New(logger, 0, nil)

				cancel := make(chan struct{})
				errs := make(chan error)

				requestsInFlight.Add(1)

				go func() {
					_, err := upldrWithoutTimeout.Upload(file.Name(), url, cancel)
					errs <- err
				}()

				Consistently(errs).ShouldNot(Receive())

				close(cancel)

				Eventually(errs).Should(Receive(Equal(uploader.ErrUploadCancelled)))
			})
		})

		Context("when the upload times out", func() {
			var requestInitiated chan struct{}

			BeforeEach(func() {
				requestInitiated = make(chan struct{})

				testServer = httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
					requestInitiated <- struct{}{}

					time.Sleep(300 * time.Millisecond)
					fmt.Fprintln(w, "Hello, client")
				}))

				serverUrl := testServer.URL + "/somepath"
				url, _ = url.Parse(serverUrl)
			})

			It("should retry and log 3 times and return an error", func() {
				errs := make(chan error)

				go func() {
					_, err := upldr.Upload(file.Name(), url, nil)
					errs <- err
				}()

				Eventually(logger.TestSink.Buffer).Should(gbytes.Say("attempt"))
				Eventually(requestInitiated).Should(Receive())

				Eventually(logger.TestSink.Buffer).Should(gbytes.Say("attempt"))
				Eventually(requestInitiated).Should(Receive())

				Eventually(logger.TestSink.Buffer).Should(gbytes.Say("attempt"))
				Eventually(requestInitiated).Should(Receive())

				Eventually(logger.TestSink.Buffer).Should(gbytes.Say("failed-upload"))

				Expect(<-errs).To(HaveOccurred())
			})
		})

		Context("when the upload fails with a protocol error", func() {
			BeforeEach(func() {
				// No server to handle things!

				serverUrl := "http://127.0.0.1:54321/somepath"
				url, _ = url.Parse(serverUrl)
			})

			It("should return the error", func() {
				_, err := upldr.Upload(file.Name(), url, nil)
				Expect(err).NotTo(BeNil())
			})
		})

		Context("when the upload fails with a status code error", func() {
			BeforeEach(func() {
				testServer = httptest.NewServer(http.NotFoundHandler())

				serverUrl := testServer.URL + "/somepath"
				url, _ = url.Parse(serverUrl)
			})

			It("should return the error", func() {
				_, err := upldr.Upload(file.Name(), url, nil)
				Expect(err).NotTo(BeNil())
			})
		})
	})

	Describe("Secure Upload", func() {
		Context("when the server supports tls", func() {
			var (
				err                 error
				numBytes            int64
				fileserverTLSConfig *tls.Config
				tlsConfig           *tls.Config
			)

			BeforeEach(func() {
				testServer = httptest.NewUnstartedServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
					serverRequests = append(serverRequests, r)

					data, err := ioutil.ReadAll(r.Body)
					Expect(err).NotTo(HaveOccurred())
					serverRequestBody = append(serverRequestBody, string(data))

					fmt.Fprintln(w, "Hello, client")
				}))

				fileserverTLSConfig, err = tlsconfig.Build(
					tlsconfig.WithInternalServiceDefaults(),
					tlsconfig.WithIdentityFromFile("fixtures/correct/server.crt", "fixtures/correct/server.key"),
				).Server(
					tlsconfig.WithClientAuthenticationFromFile("fixtures/correct/server-ca.crt"),
				)
				Expect(err).NotTo(HaveOccurred())
			})

			JustBeforeEach(func() {
				testServer.TLS = fileserverTLSConfig
				testServer.StartTLS()
				serverUrl := testServer.URL + "/somepath"
				url, err = url.Parse(serverUrl)
				Expect(err).NotTo(HaveOccurred())
			})

			Context("when the client has the correct credentials", func() {
				BeforeEach(func() {
					tlsConfig, err = tlsconfig.Build(
						tlsconfig.WithInternalServiceDefaults(),
						tlsconfig.WithIdentityFromFile("fixtures/correct/client.crt", "fixtures/correct/client.key"),
					).Client(
						tlsconfig.WithAuthorityFromFile("fixtures/correct/server-ca.crt"),
					)
					Expect(err).NotTo(HaveOccurred())
				})

				It("uploads the file to the url", func() {
					upldr = uploader.New(logger, 100*time.Millisecond, tlsConfig)
					numBytes, err = upldr.Upload(file.Name(), url, nil)
					Expect(err).NotTo(HaveOccurred())

					Expect(len(serverRequests)).To(Equal(1))

					request := serverRequests[0]
					data := serverRequestBody[0]

					Expect(request.URL.Path).To(Equal("/somepath"))
					Expect(request.Header.Get("Content-Type")).To(Equal("application/octet-stream"))
					Expect(request.Header.Get("Content-MD5")).To(Equal(expectedMD5))
					Expect(strconv.Atoi(request.Header.Get("Content-Length"))).To(BeNumerically("==", 31))
					Expect(string(data)).To(Equal("content that we can check later"))
				})

				It("returns the number of bytes written", func() {
					upldr = uploader.New(logger, 100*time.Millisecond, tlsConfig)
					numBytes, err = upldr.Upload(file.Name(), url, nil)
					Expect(err).NotTo(HaveOccurred())

					Expect(numBytes).To(Equal(int64(expectedBytes)))
				})
			})

			Context("when the client has a CA, but no keypair", func() {
				BeforeEach(func() {
					fileserverTLSConfig.ClientAuth = tls.NoClientCert

					tlsConfig = &tls.Config{
						Certificates:       []tls.Certificate{},
						InsecureSkipVerify: false,
						CipherSuites:       cfhttp.SUPPORTED_CIPHER_SUITES,
						MinVersion:         tls.VersionTLS12,
					}

					certBytes, err := ioutil.ReadFile("fixtures/correct/server-ca.crt")
					Expect(err).NotTo(HaveOccurred())

					caCertPool := x509.NewCertPool()
					ok := caCertPool.AppendCertsFromPEM(certBytes)
					Expect(ok).To(BeTrue())

					tlsConfig.RootCAs = caCertPool
				})

				It("can communicate with the fileserver via one-sided TLS", func() {
					upldr = uploader.New(logger, 100*time.Millisecond, tlsConfig)
					numBytes, err = upldr.Upload(file.Name(), url, nil)
					Expect(err).NotTo(HaveOccurred())
				})
			})

			Context("when the client has incorrect certs", func() {
				It("fails when no certs are provided", func() {
					upldr = uploader.New(logger, 100*time.Millisecond, nil)
					numBytes, err = upldr.Upload(file.Name(), url, nil)
					Expect(err).To(HaveOccurred())
				})

				It("fails when wrong cert/keypair is provided", func() {
					tlsConfig, err = tlsconfig.Build(
						tlsconfig.WithInternalServiceDefaults(),
						tlsconfig.WithIdentityFromFile("fixtures/incorrect/client.crt", "fixtures/incorrect/client.key"),
					).Client(
						tlsconfig.WithAuthorityFromFile("fixtures/correct/server-ca.crt"),
					)
					Expect(err).NotTo(HaveOccurred())
					upldr = uploader.New(logger, 100*time.Millisecond, tlsConfig)
					numBytes, err = upldr.Upload(file.Name(), url, nil)
					Expect(err).To(HaveOccurred())
				})

				It("fails when ca cert is wrong", func() {
					tlsConfig, err = tlsconfig.Build(
						tlsconfig.WithInternalServiceDefaults(),
						tlsconfig.WithIdentityFromFile("fixtures/incorrect/client.crt", "fixtures/incorrect/client.key"),
					).Client(
						tlsconfig.WithAuthorityFromFile("fixtures/incorrect/server-ca.crt"),
					)
					Expect(err).NotTo(HaveOccurred())
					upldr = uploader.New(logger, 100*time.Millisecond, tlsConfig)
					numBytes, err = upldr.Upload(file.Name(), url, nil)
					Expect(err).To(HaveOccurred())
				})
			})
		})
	})
})
<commit_msg>Executor test should use tlsconfig for defaults, not cfhttp<commit_after>package uploader_test

import (
	"crypto/md5"
	"crypto/tls"
	"encoding/base64"
	"fmt"
	"io/ioutil"
	"net/http"
	"net/http/httptest"
	"net/url"
	"os"
	"strconv"
	"sync"
	"time"

	"code.cloudfoundry.org/executor/depot/uploader"
	"code.cloudfoundry.org/lager/lagertest"
	"code.cloudfoundry.org/tlsconfig"

	. "github.com/onsi/ginkgo"
	. "github.com/onsi/gomega"
	"github.com/onsi/gomega/gbytes"
)

var _ = Describe("Uploader", func() {
	var (
		upldr             uploader.Uploader
		testServer        *httptest.Server
		serverRequests    []*http.Request
		serverRequestBody []string

		logger        *lagertest.TestLogger
		url           *url.URL
		file          *os.File
		expectedBytes int
		expectedMD5   string
	)

	BeforeEach(func() {
		testServer = nil
		serverRequestBody = []string{}
		serverRequests = []*http.Request{}
		logger = lagertest.NewTestLogger("test")

		file, _ = ioutil.TempFile("", "foo")
		contentString := "content that we can check later"
		expectedBytes, _ = file.WriteString(contentString)
		rawMD5 := md5.Sum([]byte(contentString))
		expectedMD5 = base64.StdEncoding.EncodeToString(rawMD5[:])
		file.Close()
	})

	AfterEach(func() {
		file.Close()
		if testServer != nil {
			testServer.Close()
		}
		os.Remove(file.Name())
	})

	Describe("Insecure Upload", func() {
		BeforeEach(func() {
			upldr = uploader.New(logger, 100*time.Millisecond, nil)
		})

		Context("when the upload is successful", func() {
			BeforeEach(func() {
				testServer = httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
					serverRequests = append(serverRequests, r)

					data, err := ioutil.ReadAll(r.Body)
					Expect(err).NotTo(HaveOccurred())
					serverRequestBody = append(serverRequestBody, string(data))

					fmt.Fprintln(w, "Hello, client")
				}))

				serverUrl := testServer.URL + "/somepath"
				url, _ = url.Parse(serverUrl)
			})

			var err error
			var numBytes int64
			JustBeforeEach(func() {
				numBytes, err = upldr.Upload(file.Name(), url, nil)
			})

			It("uploads the file to the url", func() {
				Expect(len(serverRequests)).To(Equal(1))

				request := serverRequests[0]
				data := serverRequestBody[0]

				Expect(request.URL.Path).To(Equal("/somepath"))
				Expect(request.Header.Get("Content-Type")).To(Equal("application/octet-stream"))
				Expect(request.Header.Get("Content-MD5")).To(Equal(expectedMD5))
				Expect(strconv.Atoi(request.Header.Get("Content-Length"))).To(BeNumerically("==", 31))
				Expect(string(data)).To(Equal("content that we can check later"))
			})

			It("returns the number of bytes written", func() {
				Expect(numBytes).To(Equal(int64(expectedBytes)))
			})

			It("does not return an error", func() {
				Expect(err).NotTo(HaveOccurred())
			})
		})

		Context("when the upload is canceled", func() {
			var flushRequests chan struct{}
			var requestsInFlight *sync.WaitGroup

			BeforeEach(func() {
				flushRequests = make(chan struct{})

				requestsInFlight = new(sync.WaitGroup)
				testServer = httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
					defer requestsInFlight.Done()
					<-flushRequests
				}))

				serverUrl := testServer.URL + "/somepath"
				url, _ = url.Parse(serverUrl)
			})

			AfterEach(func() {
				close(flushRequests)
				requestsInFlight.Wait()
			})

			It("interrupts the client and returns an error", func() {
				upldrWithoutTimeout := uploader.New(logger, 0, nil)

				cancel := make(chan struct{})
				errs := make(chan error)

				requestsInFlight.Add(1)

				go func() {
					_, err := upldrWithoutTimeout.Upload(file.Name(), url, cancel)
					errs <- err
				}()

				Consistently(errs).ShouldNot(Receive())

				close(cancel)

				Eventually(errs).Should(Receive(Equal(uploader.ErrUploadCancelled)))
			})
		})

		Context("when the upload times out", func() {
			var requestInitiated chan struct{}

			BeforeEach(func() {
				requestInitiated = make(chan struct{})

				testServer = httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
					requestInitiated <- struct{}{}

					time.Sleep(300 * time.Millisecond)
					fmt.Fprintln(w, "Hello, client")
				}))

				serverUrl := testServer.URL + "/somepath"
				url, _ = url.Parse(serverUrl)
			})

			It("should retry and log 3 times and return an error", func() {
				errs := make(chan error)

				go func() {
					_, err := upldr.Upload(file.Name(), url, nil)
					errs <- err
				}()

				Eventually(logger.TestSink.Buffer).Should(gbytes.Say("attempt"))
				Eventually(requestInitiated).Should(Receive())

				Eventually(logger.TestSink.Buffer).Should(gbytes.Say("attempt"))
				Eventually(requestInitiated).Should(Receive())

				Eventually(logger.TestSink.Buffer).Should(gbytes.Say("attempt"))
				Eventually(requestInitiated).Should(Receive())

				Eventually(logger.TestSink.Buffer).Should(gbytes.Say("failed-upload"))

				Expect(<-errs).To(HaveOccurred())
			})
		})

		Context("when the upload fails with a protocol error", func() {
			BeforeEach(func() {
				// No server to handle things!

				serverUrl := "http://127.0.0.1:54321/somepath"
				url, _ = url.Parse(serverUrl)
			})

			It("should return the error", func() {
				_, err := upldr.Upload(file.Name(), url, nil)
				Expect(err).NotTo(BeNil())
			})
		})

		Context("when the upload fails with a status code error", func() {
			BeforeEach(func() {
				testServer = httptest.NewServer(http.NotFoundHandler())

				serverUrl := testServer.URL + "/somepath"
				url, _ = url.Parse(serverUrl)
			})

			It("should return the error", func() {
				_, err := upldr.Upload(file.Name(), url, nil)
				Expect(err).NotTo(BeNil())
			})
		})
	})

	Describe("Secure Upload", func() {
		Context("when the server supports tls", func() {
			var (
				err                 error
				numBytes            int64
				fileserverTLSConfig *tls.Config
				tlsConfig           *tls.Config
			)

			BeforeEach(func() {
				testServer = httptest.NewUnstartedServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
					serverRequests = append(serverRequests, r)

					data, err := ioutil.ReadAll(r.Body)
					Expect(err).NotTo(HaveOccurred())
					serverRequestBody = append(serverRequestBody, string(data))

					fmt.Fprintln(w, "Hello, client")
				}))

				fileserverTLSConfig, err = tlsconfig.Build(
					tlsconfig.WithInternalServiceDefaults(),
					tlsconfig.WithIdentityFromFile("fixtures/correct/server.crt", "fixtures/correct/server.key"),
				).Server(
					tlsconfig.WithClientAuthenticationFromFile("fixtures/correct/server-ca.crt"),
				)
				Expect(err).NotTo(HaveOccurred())
			})

			JustBeforeEach(func() {
				testServer.TLS = fileserverTLSConfig
				testServer.StartTLS()
				serverUrl := testServer.URL + "/somepath"
				url, err = url.Parse(serverUrl)
				Expect(err).NotTo(HaveOccurred())
			})

			Context("when the client has the correct credentials", func() {
				BeforeEach(func() {
					tlsConfig, err = tlsconfig.Build(
						tlsconfig.WithInternalServiceDefaults(),
						tlsconfig.WithIdentityFromFile("fixtures/correct/client.crt", "fixtures/correct/client.key"),
					).Client(
						tlsconfig.WithAuthorityFromFile("fixtures/correct/server-ca.crt"),
					)
					Expect(err).NotTo(HaveOccurred())
				})

				It("uploads the file to the url", func() {
					upldr = uploader.New(logger, 100*time.Millisecond, tlsConfig)
					numBytes, err = upldr.Upload(file.Name(), url, nil)
					Expect(err).NotTo(HaveOccurred())

					Expect(len(serverRequests)).To(Equal(1))

					request := serverRequests[0]
					data := serverRequestBody[0]

					Expect(request.URL.Path).To(Equal("/somepath"))
					Expect(request.Header.Get("Content-Type")).To(Equal("application/octet-stream"))
					Expect(request.Header.Get("Content-MD5")).To(Equal(expectedMD5))
					Expect(strconv.Atoi(request.Header.Get("Content-Length"))).To(BeNumerically("==", 31))
					Expect(string(data)).To(Equal("content that we can check later"))
				})

				It("returns the number of bytes written", func() {
					upldr = uploader.New(logger, 100*time.Millisecond, tlsConfig)
					numBytes, err = upldr.Upload(file.Name(), url, nil)
					Expect(err).NotTo(HaveOccurred())

					Expect(numBytes).To(Equal(int64(expectedBytes)))
				})
			})

			Context("when the client has a CA, but no keypair", func() {
				BeforeEach(func() {
					fileserverTLSConfig.ClientAuth = tls.NoClientCert

					tlsConfig, err = tlsconfig.Build(
						tlsconfig.WithInternalServiceDefaults(),
					).Client(
						tlsconfig.WithAuthorityFromFile("fixtures/correct/server-ca.crt"),
					)
					Expect(err).NotTo(HaveOccurred())
				})

				It("can communicate with the fileserver via one-sided TLS", func() {
					upldr = uploader.New(logger, 100*time.Millisecond, tlsConfig)
					numBytes, err = upldr.Upload(file.Name(), url, nil)
					Expect(err).NotTo(HaveOccurred())
				})
			})

			Context("when the client has incorrect certs", func() {
				It("fails when no certs are provided", func() {
					upldr = uploader.New(logger, 100*time.Millisecond, nil)
					numBytes, err = upldr.Upload(file.Name(), url, nil)
					Expect(err).To(HaveOccurred())
				})

				It("fails when wrong cert/keypair is provided", func() {
					tlsConfig, err = tlsconfig.Build(
						tlsconfig.WithInternalServiceDefaults(),
						tlsconfig.WithIdentityFromFile("fixtures/incorrect/client.crt", "fixtures/incorrect/client.key"),
					).Client(
						tlsconfig.WithAuthorityFromFile("fixtures/correct/server-ca.crt"),
					)
					Expect(err).NotTo(HaveOccurred())
					upldr = uploader.New(logger, 100*time.Millisecond, tlsConfig)
					numBytes, err = upldr.Upload(file.Name(), url, nil)
					Expect(err).To(HaveOccurred())
				})

				It("fails when ca cert is wrong", func() {
					tlsConfig, err = tlsconfig.Build(
						tlsconfig.WithInternalServiceDefaults(),
						tlsconfig.WithIdentityFromFile("fixtures/incorrect/client.crt", "fixtures/incorrect/client.key"),
					).Client(
						tlsconfig.WithAuthorityFromFile("fixtures/incorrect/server-ca.crt"),
					)
					Expect(err).NotTo(HaveOccurred())
					upldr = uploader.New(logger, 100*time.Millisecond, tlsConfig)
					numBytes, err = upldr.Upload(file.Name(), url, nil)
					Expect(err).To(HaveOccurred())
				})
			})
		})
	})
})
<|endoftext|>


<commit_before>// plivo
/*
	func main() {
		p := Plivo{
			Host:     "https://api.plivo.com/v1/Account",
			User:     "",
			Password: "",
		}

		data := map[string]string{
			"dst":  "",
			"src":  "",
			"text": "",
		}

		fmt.Println(p)
		p.Send(data)

		fmt.Println(p.RenderPath("/Message/"))
		fmt.Println(p.RenderPath("/Message"))
	}
*/
package plivo

import (
	"bytes"
	"encoding/json"
	"fmt"
	"io/ioutil"
	"net"
	"net/http"
	"net/url"
	"path"
	"strings"
	"time"
)

// Fixed http too many open files.
var httpClient = &http.Client{Transport: &http.Transport{
	Proxy: http.ProxyFromEnvironment,
	Dial: (&net.Dialer{
		Timeout:   0,
		KeepAlive: 0,
	}).Dial,
	TLSHandshakeTimeout: 10 * time.Second,
},
}

// Plivo struct
type Plivo struct {
	Host     string
	User     string
	Password string
}

func (p Plivo) Send(data map[string]string) {
	jsonData, _ := json.Marshal(data)
	a, _ := http.NewRequest("POST", p.renderPath("/Message/"), bytes.NewReader(jsonData))
	a.URL.User = url.UserPassword(p.User, p.Password)

	header := http.Header{}
	header.Add("Content-Type", "application/json")
	a.Header = header

	if false {
		resp, err := httpClient.Do(a)
		if err != nil {
			fmt.Printf("Error >>> %s \n", err)
		} else {
			body, _ := ioutil.ReadAll(resp.Body)
			fmt.Printf("Resp >>> \n%s\n", body)
		}
	} else {
		fmt.Println(a, p)
	}
}

func (p Plivo) renderPath(urlpath string) string {
	URLPath, _ := url.ParseRequestURI(p.Host)
	URLPath.Path = path.Join(URLPath.Path, p.User, urlpath)
	if strings.LastIndex(urlpath, "/") >= 0 {
		return fmt.Sprintf("%s/", URLPath.String())
	}
	return URLPath.String()
}
<commit_msg>Add `Message` struct.<commit_after>// plivo
/*
	func main() {
		p := Plivo{
			Host:     "https://api.plivo.com/v1/Account",
			User:     "",
			Password: "",
		}

		data := map[string]string{
			"dst":  "",
			"src":  "",
			"text": "",
		}

		fmt.Println(p)
		p.Send(data)

		fmt.Println(p.RenderPath("/Message/"))
		fmt.Println(p.RenderPath("/Message"))
	}
*/
package plivo

import (
	"bytes"
	"encoding/json"
	"fmt"
	"io/ioutil"
	"net"
	"net/http"
	"net/url"
	"path"
	"strings"
	"time"
)

// Fixed http too many open files.
var httpClient = &http.Client{Transport: &http.Transport{
	Proxy: http.ProxyFromEnvironment,
	Dial: (&net.Dialer{
		Timeout:   0,
		KeepAlive: 0,
	}).Dial,
	TLSHandshakeTimeout: 10 * time.Second,
},
}

// Message struct
type Message struct {
	dst  string
	src  string
	text string
}

func NewMessage(dst, src, text string) *Message {
	return &Message{dst: dst, src: src, text: text}
}

func (m Message) ToMap() map[string]string {
	var result = make(map[string]string)
	result["dst"] = m.dst
	result["src"] = m.src
	result["text"] = m.text
	return result
}

// Plivo struct
type Plivo struct {
	Host     string
	User     string
	Password string
}

func (p Plivo) Send(data map[string]string) {
	jsonData, _ := json.Marshal(data)
	a, _ := http.NewRequest("POST", p.renderPath("/Message/"), bytes.NewReader(jsonData))
	a.URL.User = url.UserPassword(p.User, p.Password)

	header := http.Header{}
	header.Add("Content-Type", "application/json")
	a.Header = header

	if false {
		resp, err := httpClient.Do(a)
		if err != nil {
			fmt.Printf("Error >>> %s \n", err)
		} else {
			body, _ := ioutil.ReadAll(resp.Body)
			fmt.Printf("Resp >>> \n%s\n", body)
		}
	} else {
		fmt.Println(a, p)
	}
}

func (p Plivo) renderPath(urlpath string) string {
	URLPath, _ := url.ParseRequestURI(p.Host)
	URLPath.Path = path.Join(URLPath.Path, p.User, urlpath)
	if strings.LastIndex(urlpath, "/") >= 0 {
		return fmt.Sprintf("%s/", URLPath.String())
	}
	return URLPath.String()
}
<|endoftext|>


<commit_before>package main

import (
	"flag"
	"fmt"
	"github.com/anchor/bletchley/dataframe"
	"os"
)

const (
	Version           = "0.1.0"
	DefaultFrameCount = 100
)

// Given the value of the split-files argument and the number of bursts
// we've written so far, return a file pointer to the current correct
// output stream.
func getCurrentOutputStream(splitFiles string, burstIndex int) (*os.File, error) {
	var err error
	fo := os.Stdout
	if splitFiles != "" {
		fName := fmt.Sprintf("%v.%02d", splitFiles, burstIndex)
		fo, err = os.Create(fName)
		if err != nil {
			return nil, err
		}
	}
	return fo, nil
}

func main() {
	frameCount := flag.Int("count", 100, "Number of frames to generate (if -burst is false, this is forced to 1).")
	burstPack := flag.Bool("burst", true, "Generate DataBursts rather than plain DataFrames.")
	burstLen := flag.Int("burst-len", 100, "Number of DataFrames per DataBurst (only used with -burst).")
	splitFiles := flag.String("split-files", "", "Write generated DataBursts to (<count>/<burst-len>) files, named numerically using the value of this argument as the prefix.")

	flag.Usage = func() {
		helpMessage := "framegen will generate random DataFrames for testing purposes. By default, it will write them to stdout.\n\n" +
			fmt.Sprintf("Usage: %s [options]\n\n", os.Args[0]) +
			"Options:\n\n"
		fmt.Fprintf(os.Stderr, helpMessage)
		flag.PrintDefaults()
	}
	flag.Parse()

	frameBatch := make([]*dataframe.DataFrame, *burstLen)
	burstCount := 0

	nFrames := *frameCount
	if !*burstPack {
		nFrames = 1
	}

	fileCount := 0
	for i := 0; i < nFrames; i++ {
		frame := dataframe.GenTestDataFrame()
		if !*burstPack {
			bytes, err := dataframe.MarshalDataFrame(frame)
			if err != nil {
				fmt.Printf("Error marshalling frame %v: %v\n", frame, err)
			} else {
				os.Stdout.Write(bytes)
			}
		} else {
			frameBatch[burstCount] = frame
			burstCount += 1
			if burstCount == *burstLen {
				fo, err := getCurrentOutputStream(*splitFiles, fileCount)
				if err != nil {
					fmt.Println(err)
					os.Exit(1)
				}
				burst := dataframe.BuildDataBurst(frameBatch)
				bytes, err := dataframe.MarshalDataBurst(burst)
				if err != nil {
					fmt.Printf("Error marshalling burst: %v\n", err)
				} else {
					fo.Write(bytes)
				}
				frameBatch = make([]*dataframe.DataFrame, *burstLen)
				burstCount = 0
				fileCount += 1
			}
		}
	}
}
<commit_msg>Restructure to make it easier to add new datatypes<commit_after>package main

import (
	"flag"
	"fmt"
	"github.com/anchor/bletchley/dataframe"
	_ "github.com/anchor/chevalier"
	"os"
)

const (
	Version           = "0.1.0"
	DefaultFrameCount = 100
)

// Given the value of the split-files argument and the number of bursts
// we've written so far, return a file pointer to the current correct
// output stream.
func getCurrentOutputStream(splitFiles string, burstIndex int) (*os.File, error) {
	var err error
	fo := os.Stdout
	if splitFiles != "" {
		fName := fmt.Sprintf("%v.%02d", splitFiles, burstIndex)
		fo, err = os.Create(fName)
		if err != nil {
			return nil, err
		}
	}
	return fo, nil
}

func genFrames(nFrames int, splitFiles string, burstLen int, burstPack bool) {
	fileCount := 0
	frameBatch := make([]*dataframe.DataFrame, burstLen)
	burstCount := 0
	for i := 0; i < nFrames; i++ {
		frame := dataframe.GenTestDataFrame()
		if !burstPack {
			bytes, err := dataframe.MarshalDataFrame(frame)
			if err != nil {
				fmt.Printf("Error marshalling frame %v: %v\n", frame, err)
			} else {
				os.Stdout.Write(bytes)
			}
		} else {
			frameBatch[burstCount] = frame
			burstCount += 1
			if burstCount == burstLen {
				fo, err := getCurrentOutputStream(splitFiles, fileCount)
				if err != nil {
					fmt.Println(err)
					os.Exit(1)
				}
				burst := dataframe.BuildDataBurst(frameBatch)
				bytes, err := dataframe.MarshalDataBurst(burst)
				if err != nil {
					fmt.Printf("Error marshalling burst: %v\n", err)
				} else {
					fo.Write(bytes)
				}
				frameBatch = make([]*dataframe.DataFrame, burstLen)
				burstCount = 0
				fileCount += 1
			}
		}
	}
}

func main() {
	frameCount := flag.Int("count", 100, "Number of frames to generate (if -burst is false, this is forced to 1).")
	burstPack := flag.Bool("burst", true, "Generate DataBursts rather than plain DataFrames.")
	sourceReq := flag.Bool("source-req", false, "Generate SourceRequests rather than DataFrames")
	burstLen := flag.Int("burst-len", 100, "Number of DataFrames per DataBurst (only used with -burst).")
	splitFiles := flag.String("split-files", "", "Write generated DataBursts to (<count>/<burst-len>) files, named numerically using the value of this argument as the prefix.")

	flag.Usage = func() {
		helpMessage := "framegen will generate random DataFrames for testing purposes. By default, it will write them to stdout.\n\n" +
			fmt.Sprintf("Usage: %s [options]\n\n", os.Args[0]) +
			"Options:\n\n"
		fmt.Fprintf(os.Stderr, helpMessage)
		flag.PrintDefaults()
	}
	flag.Parse()

	nFrames := *frameCount
	if !*burstPack {
		nFrames = 1
	}

	if !(*sourceReq) {
		genFrames(nFrames, *splitFiles, *burstLen, *burstPack)
	}
}
<|endoftext|>


<commit_before>// Copyright © 2016 Zenly <hello@zen.ly>.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//   http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package bank

import (
	"context"

	"github.com/gogo/protobuf/proto"
	"github.com/pkg/errors"
	"github.com/znly/protein"
	tuyau "github.com/znly/tuyauDB"
)

// -----------------------------------------------------------------------------

// Tuyau implements a Bank that integrates with znly/tuyauDB in order to keep
// its local in-memory cache in sync with a TuyauDB store.
type Tuyau struct {
	c *tuyau.Client

	schemas map[string]*protein.ProtobufSchema
	// reverse-mapping of fully-qualified names to UIDs
	revmap map[string][]string
}

// NewTuyau returns a new Tuyau that uses `c` as its underlying client for
// accessing a TuyauDB store.
//
// It is the caller's responsibility to close the client once he's done with it.
func NewTuyau(c *tuyau.Client) *Tuyau {
	return &Tuyau{
		c:       c,
		schemas: map[string]*protein.ProtobufSchema{},
		revmap:  map[string][]string{},
	}
}

// -----------------------------------------------------------------------------

// Get retrieves the ProtobufSchema associated with the specified identifier,
// plus all of its direct & indirect dependencies.
//
// The retrieval process is done in two steps:
// - First, the root schema, as identified by `uid`, is fetched from the local
//   in-memory cache; if it cannot be found in there, it'll be retrieved from
//   the backing TuyauDB store.
//   If it cannot be found in the TuyauDB store, then a "schema not found"
//   error is returned.
// - Second, the same process is applied for every direct & indirect dependency
//   of the root schema.
//   The only difference is that all the dependencies missing from the local
//   cache will be bulk-fetched from the TuyauDB store to avoid unnecessary
//   round-trips.
//   A "schemas not found" error is returned if one or more dependencies couldn't
//   be found.
func (t *Tuyau) Get(ctx context.Context, uid string) (map[string]*protein.ProtobufSchema, error) {
	schemas := map[string]*protein.ProtobufSchema{}

	// get root schema
	if s, ok := schemas[uid]; ok { // try the in-memory cache first..
		schemas[uid] = s
	} else { // ..then fallback on the remote tuyauDB store
		b, err := t.c.Get(ctx, uid)
		if err != nil {
			return nil, errors.Wrapf(err, "`%s`: schema not found", uid)
		}
		var root protein.ProtobufSchema
		if err := proto.Unmarshal(b.Data, &root); err != nil {
			return nil, errors.Wrapf(err, "`%s`: invalid schema", uid)
		}
		schemas[uid] = &root
	}

	// get dependency schemas
	deps := schemas[uid].GetDeps()

	// try the in-memory cache first..
	psNotFound := make(map[string]struct{}, len(deps))
	for depUID := range deps {
		if s, ok := schemas[depUID]; ok {
			schemas[depUID] = s
			continue
		}
		psNotFound[depUID] = struct{}{}
	}
	if len(psNotFound) <= 0 { // found everything needed in local cache!
		return schemas, nil
	}

	// ..then fallback on the remote tuyauDB store
	psToFetch := make([]string, 0, len(psNotFound))
	for depUID := range psNotFound {
		psToFetch = append(psToFetch, depUID)
	}
	blobs, err := t.c.GetMulti(ctx, psToFetch)
	if err != nil {
		return nil, errors.WithStack(err)
	}
	for _, b := range blobs {
		delete(psNotFound, b.Key) // it's been found!
	}
	if len(psNotFound) > 0 {
		err := errors.Errorf("one or more dependencies couldn't be found")
		for depUID := range psNotFound {
			err = errors.Wrapf(err, "`%s`: dependency not found", depUID)
		}
		return nil, err
	}
	for _, b := range blobs {
		var ps protein.ProtobufSchema
		if err := proto.Unmarshal(b.Data, &ps); err != nil {
			return nil, errors.Wrapf(err, "`%s`: invalid schema (dependency)", b.Key)
		}
		schemas[b.Key] = &ps
	}

	return schemas, nil
}

// FQNameToUID returns the UID associated with the given fully-qualified name.
//
// It is possible that multiple versions of a schema identified by a FQ name
// are currently available in the bank; in which case all of the associated UIDs
// will be returned to the caller, *in random order*.
//
// The reverse-mapping is pre-computed; don't hesitate to call this method, it'll
// be real fast.
//
// It returns nil if `fqName` doesn't match any schema in the bank.
func (t *Tuyau) FQNameToUID(fqName string) []string { return t.revmap[fqName] }

// Put synchronously adds the specified ProtobufSchemas to the local in-memory
// cache; then pushes them to the underlying tuyau client's pipe.
// Whether this push is synchronous or not depends on the implementation
// of the tuyau.Client used.
//
// Put doesn't care about pre-existing keys: if a schema with the same key
// already exist, it will be overwritten; both in the local cache as well in the
// TuyauDB store.
//
// TODO(cmc): note about CAS that doesn't matter here
func (t *Tuyau) Put(ctx context.Context, pss ...*protein.ProtobufSchema) error {
	blobs := make([]*tuyau.Blob, 0, len(pss))
	var b []byte
	var err error

	// in-memory push
	for _, ps := range pss {
		b, err = proto.Marshal(ps)
		if err != nil {
			return errors.WithStack(err)
		}
		uid := ps.GetUID()
		blobs = append(blobs, &tuyau.Blob{
			Key: uid, Data: b, TTL: 0, Flags: 0,
		})
		t.schemas[uid] = ps
		t.revmap[ps.GetFQName()] = append(t.revmap[ps.GetFQName()], uid)
	}

	// asynchronous push
	if err := t.c.Push(ctx, nil, blobs...); err != nil {
		return errors.WithStack(err)
	}

	// synchronous push
	if err := t.c.SetMulti(ctx, blobs); err != nil {
		// NOTE: if SetMulti fails for any reason, this fallbacks to multiple
		//       Set operations. This doesn't only check for KVErrOpNotSupported
		//       errors because some redis implementations such as Netflix's
		//       Dynomite simply crash the connection if they don't support it.
		for _, b := range blobs {
			if err := t.c.Set(ctx, b); err != nil {
				return errors.WithStack(err)
			}
		}
	}
	return nil
}
<commit_msg>bank:tuyau > On Get, the local cache was never used nor populated<commit_after>// Copyright © 2016 Zenly <hello@zen.ly>.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//   http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package bank

import (
	"context"

	"github.com/gogo/protobuf/proto"
	"github.com/pkg/errors"
	"github.com/znly/protein"
	tuyau "github.com/znly/tuyauDB"
)

// -----------------------------------------------------------------------------

// Tuyau implements a Bank that integrates with znly/tuyauDB in order to keep
// its local in-memory cache in sync with a TuyauDB store.
type Tuyau struct {
	c *tuyau.Client

	schemas map[string]*protein.ProtobufSchema
	// reverse-mapping of fully-qualified names to UIDs
	revmap map[string][]string
}

// NewTuyau returns a new Tuyau that uses `c` as its underlying client for
// accessing a TuyauDB store.
//
// It is the caller's responsibility to close the client once he's done with it.
func NewTuyau(c *tuyau.Client) *Tuyau {
	return &Tuyau{
		c:       c,
		schemas: map[string]*protein.ProtobufSchema{},
		revmap:  map[string][]string{},
	}
}

// -----------------------------------------------------------------------------

// Get retrieves the ProtobufSchema associated with the specified identifier,
// plus all of its direct & indirect dependencies.
//
// The retrieval process is done in two steps:
// - First, the root schema, as identified by `uid`, is fetched from the local
//   in-memory cache; if it cannot be found in there, it'll be retrieved from
//   the backing TuyauDB store.
//   If it cannot be found in the TuyauDB store, then a "schema not found"
//   error is returned.
// - Second, the same process is applied for every direct & indirect dependency
//   of the root schema.
//   The only difference is that all the dependencies missing from the local
//   cache will be bulk-fetched from the TuyauDB store to avoid unnecessary
//   round-trips.
//   A "schemas not found" error is returned if one or more dependencies couldn't
//   be found.
func (t *Tuyau) Get(ctx context.Context, uid string) (map[string]*protein.ProtobufSchema, error) {
	schemas := map[string]*protein.ProtobufSchema{}

	// get root schema
	if s, ok := t.schemas[uid]; ok { // try the in-memory cache first..
		schemas[uid] = s
	} else { // ..then fallback on the remote tuyauDB store
		b, err := t.c.Get(ctx, uid)
		if err != nil {
			return nil, errors.Wrapf(err, "`%s`: schema not found", uid)
		}
		var root protein.ProtobufSchema
		if err := proto.Unmarshal(b.Data, &root); err != nil {
			return nil, errors.Wrapf(err, "`%s`: invalid schema", uid)
		}
		schemas[uid] = &root
		t.schemas[uid] = &root
	}

	// get dependency schemas
	deps := schemas[uid].GetDeps()

	// try the in-memory cache first..
	psNotFound := make(map[string]struct{}, len(deps))
	for depUID := range deps {
		if s, ok := t.schemas[depUID]; ok {
			schemas[depUID] = s
			continue
		}
		psNotFound[depUID] = struct{}{}
	}
	if len(psNotFound) <= 0 { // found everything needed in local cache!
		return schemas, nil
	}

	// ..then fallback on the remote tuyauDB store
	psToFetch := make([]string, 0, len(psNotFound))
	for depUID := range psNotFound {
		psToFetch = append(psToFetch, depUID)
	}
	blobs, err := t.c.GetMulti(ctx, psToFetch)
	if err != nil {
		return nil, errors.WithStack(err)
	}
	for _, b := range blobs {
		delete(psNotFound, b.Key) // it's been found!
	}
	if len(psNotFound) > 0 {
		err := errors.Errorf("one or more dependencies couldn't be found")
		for depUID := range psNotFound {
			err = errors.Wrapf(err, "`%s`: dependency not found", depUID)
		}
		return nil, err
	}
	for _, b := range blobs {
		var ps protein.ProtobufSchema
		if err := proto.Unmarshal(b.Data, &ps); err != nil {
			return nil, errors.Wrapf(err, "`%s`: invalid schema (dependency)", b.Key)
		}
		schemas[b.Key] = &ps
		t.schemas[b.Key] = &ps
	}

	return schemas, nil
}

// FQNameToUID returns the UID associated with the given fully-qualified name.
//
// It is possible that multiple versions of a schema identified by a FQ name
// are currently available in the bank; in which case all of the associated UIDs
// will be returned to the caller, *in random order*.
//
// The reverse-mapping is pre-computed; don't hesitate to call this method, it'll
// be real fast.
//
// It returns nil if `fqName` doesn't match any schema in the bank.
func (t *Tuyau) FQNameToUID(fqName string) []string { return t.revmap[fqName] }

// Put synchronously adds the specified ProtobufSchemas to the local in-memory
// cache; then pushes them to the underlying tuyau client's pipe.
// Whether this push is synchronous or not depends on the implementation
// of the tuyau.Client used.
//
// Put doesn't care about pre-existing keys: if a schema with the same key
// already exist, it will be overwritten; both in the local cache as well in the
// TuyauDB store.
//
// TODO(cmc): note about CAS that doesn't matter here
func (t *Tuyau) Put(ctx context.Context, pss ...*protein.ProtobufSchema) error {
	blobs := make([]*tuyau.Blob, 0, len(pss))
	var b []byte
	var err error

	// in-memory push
	for _, ps := range pss {
		b, err = proto.Marshal(ps)
		if err != nil {
			return errors.WithStack(err)
		}
		uid := ps.GetUID()
		blobs = append(blobs, &tuyau.Blob{
			Key: uid, Data: b, TTL: 0, Flags: 0,
		})
		t.schemas[uid] = ps
		t.revmap[ps.GetFQName()] = append(t.revmap[ps.GetFQName()], uid)
	}

	// asynchronous push
	if err := t.c.Push(ctx, nil, blobs...); err != nil {
		return errors.WithStack(err)
	}

	// synchronous push
	if err := t.c.SetMulti(ctx, blobs); err != nil {
		// NOTE: if SetMulti fails for any reason, this fallbacks to multiple
		//       Set operations. This doesn't only check for KVErrOpNotSupported
		//       errors because some redis implementations such as Netflix's
		//       Dynomite simply crash the connection if they don't support it.
		for _, b := range blobs {
			if err := t.c.Set(ctx, b); err != nil {
				return errors.WithStack(err)
			}
		}
	}
	return nil
}
<|endoftext|>


<commit_before>// +build windows !darwin !freebsd !linux !openbsd !netbsd

package main

import (
	"math"
)

func filesystemFree() (uint64, error) {
	return uint64(math.MaxFloat64), noFSFree
}
<commit_msg>Fix windows build.<commit_after>// +build windows !darwin !freebsd !linux !openbsd !netbsd

package main

import (
	"math"
)

func filesystemFree() (uint64, error) {
	return uint64(math.MaxInt64), noFSFree
}
<|endoftext|>


<commit_before>package kit

import (
	"crypto/md5"
	"encoding/hex"
	"encoding/json"
	"fmt"
	"io/ioutil"
	"net/http"
	"net/http/httptest"
	"os"
	"runtime"
	"testing"

	"github.com/hashicorp/go-version"
	"github.com/stretchr/testify/assert"
	"github.com/stretchr/testify/suite"
)

var (
	updatePath      = clean("../fixtures/updateme")
	oldFile         = []byte{0xDE, 0xAD, 0xBE, 0xEF}
	newFile         = []byte{0x01, 0x02, 0x03, 0x04, 0x05, 0x06}
	newFileChecksum = md5.Sum(newFile)
)

func TestLibraryInfo(t *testing.T) {
	messageSeparator := "\n----------------------------------------------------------------\n"
	info := fmt.Sprintf("\t%s %s", "ThemeKit - Shopify Theme Utilities", ThemeKitVersion.String())
	assert.Equal(t, fmt.Sprintf("%s%s%s", messageSeparator, info, messageSeparator), LibraryInfo())
}

type VersionTestSuite struct {
	suite.Suite
}

func (suite *VersionTestSuite) SetupSuite() {
	ThemeKitVersion, _ = version.NewVersion("0.5.0")
}

func (suite *VersionTestSuite) SetupTest() {
	os.Create(updatePath)
}

func (suite *VersionTestSuite) TearDownTest() {
	os.Remove(updatePath)
}

func (suite *VersionTestSuite) TestIsNewUpdateAvailable() {
	server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
		fmt.Fprintf(w, jsonFixture("responses/all_releases"))
	}))
	releasesURL = server.URL
	defer server.Close()
	ThemeKitVersion, _ = version.NewVersion("20.0.0")
	assert.Equal(suite.T(), false, IsNewUpdateAvailable())

	ThemeKitVersion, _ = version.NewVersion("0.0.0")
	assert.Equal(suite.T(), true, IsNewUpdateAvailable())
}

func (suite *VersionTestSuite) TestInstallThemeKitVersion() {
	requests := 0
	var server *httptest.Server
	server = httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
		if requests <= 1 {
			fmt.Fprintf(w, jsonFixture("responses/all_releases"))
		} else if requests == 2 {
			out, _ := json.Marshal([]release{
				{
					Version: "20.0.0",
					Platforms: []platform{
						{
							Name:       runtime.GOOS + "-" + runtime.GOARCH,
							URL:        server.URL,
							Digest:     hex.EncodeToString(newFileChecksum[:]),
							TargetPath: updatePath,
						},
					},
				},
			})

			fmt.Fprintf(w, string(out))
		} else {
			fmt.Fprintf(w, string(newFile))
		}
		requests++
	}))
	defer server.Close()
	releasesURL = server.URL

	ThemeKitVersion, _ = version.NewVersion("0.4.7")
	err := InstallThemeKitVersion("latest")
	assert.Equal(suite.T(), "No applicable update available.", err.Error())

	ThemeKitVersion, _ = version.NewVersion("0.4.4")
	err = InstallThemeKitVersion("0.0.0")
	assert.Equal(suite.T(), "Version 0.0.0 not found.", err.Error())

	ThemeKitVersion, _ = version.NewVersion("0.4.4")
	err = InstallThemeKitVersion("latest")
	assert.Nil(suite.T(), err)
}

func (suite *VersionTestSuite) TestFetchReleases() {
	server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
		fmt.Fprintf(w, jsonFixture("responses/all_releases"))
	}))
	releasesURL = server.URL

	releases, err := fetchReleases()
	assert.Nil(suite.T(), err)
	assert.Equal(suite.T(), 4, len(releases))
	server.Close()

	server = httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
		fmt.Fprintf(w, "this is not json")
	}))
	releasesURL = server.URL
	releases, err = fetchReleases()
	assert.NotNil(suite.T(), err)
	server.Close()

	server = httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
		w.WriteHeader(404)
		fmt.Fprintf(w, "404")
	}))
	releasesURL = server.URL
	releases, err = fetchReleases()
	assert.NotNil(suite.T(), err)
	server.Close()
}

func (suite *VersionTestSuite) TestApplyUpdate() {
	server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
		fmt.Fprintf(w, string(newFile))
	}))

	err := applyUpdate(platform{
		URL:        server.URL,
		Digest:     hex.EncodeToString(newFileChecksum[:]),
		TargetPath: updatePath,
	})
	assert.Nil(suite.T(), err)

	buf, err := ioutil.ReadFile(updatePath)
	assert.Nil(suite.T(), err)
	assert.Equal(suite.T(), newFile, buf)
	server.Close()

	err = applyUpdate(platform{})
	assert.NotNil(suite.T(), err)

	server = httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
		w.WriteHeader(404)
		fmt.Fprintf(w, "404")
	}))
	err = applyUpdate(platform{URL: server.URL})
	assert.NotNil(suite.T(), err)
	server.Close()
}

func TestVersionTestSuite(t *testing.T) {
	suite.Run(t, new(VersionTestSuite))
}
<commit_msg>cleaing up file handles for testing<commit_after>package kit

import (
	"crypto/md5"
	"encoding/hex"
	"encoding/json"
	"fmt"
	"io/ioutil"
	"net/http"
	"net/http/httptest"
	"os"
	"runtime"
	"testing"

	"github.com/hashicorp/go-version"
	"github.com/stretchr/testify/assert"
	"github.com/stretchr/testify/suite"
)

var (
	updatePath      = clean("../fixtures/updateme")
	oldFile         = []byte{0xDE, 0xAD, 0xBE, 0xEF}
	newFile         = []byte{0x01, 0x02, 0x03, 0x04, 0x05, 0x06}
	newFileChecksum = md5.Sum(newFile)
)

func TestLibraryInfo(t *testing.T) {
	messageSeparator := "\n----------------------------------------------------------------\n"
	info := fmt.Sprintf("\t%s %s", "ThemeKit - Shopify Theme Utilities", ThemeKitVersion.String())
	assert.Equal(t, fmt.Sprintf("%s%s%s", messageSeparator, info, messageSeparator), LibraryInfo())
}

type VersionTestSuite struct {
	suite.Suite
}

func (suite *VersionTestSuite) SetupSuite() {
	ThemeKitVersion, _ = version.NewVersion("0.5.0")
}

func (suite *VersionTestSuite) SetupTest() {
	file, err := os.Create(updatePath)
	if err == nil {
		file.Close()
	}
}

func (suite *VersionTestSuite) TearDownTest() {
	os.Remove(updatePath)
}

func (suite *VersionTestSuite) TestIsNewUpdateAvailable() {
	server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
		fmt.Fprintf(w, jsonFixture("responses/all_releases"))
	}))
	releasesURL = server.URL
	defer server.Close()
	ThemeKitVersion, _ = version.NewVersion("20.0.0")
	assert.Equal(suite.T(), false, IsNewUpdateAvailable())

	ThemeKitVersion, _ = version.NewVersion("0.0.0")
	assert.Equal(suite.T(), true, IsNewUpdateAvailable())
}

func (suite *VersionTestSuite) TestInstallThemeKitVersion() {
	requests := 0
	var server *httptest.Server
	server = httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
		if requests <= 1 {
			fmt.Fprintf(w, jsonFixture("responses/all_releases"))
		} else if requests == 2 {
			out, _ := json.Marshal([]release{
				{
					Version: "20.0.0",
					Platforms: []platform{
						{
							Name:       runtime.GOOS + "-" + runtime.GOARCH,
							URL:        server.URL,
							Digest:     hex.EncodeToString(newFileChecksum[:]),
							TargetPath: updatePath,
						},
					},
				},
			})

			fmt.Fprintf(w, string(out))
		} else {
			fmt.Fprintf(w, string(newFile))
		}
		requests++
	}))
	defer server.Close()
	releasesURL = server.URL

	ThemeKitVersion, _ = version.NewVersion("0.4.7")
	err := InstallThemeKitVersion("latest")
	assert.Equal(suite.T(), "No applicable update available.", err.Error())

	ThemeKitVersion, _ = version.NewVersion("0.4.4")
	err = InstallThemeKitVersion("0.0.0")
	assert.Equal(suite.T(), "Version 0.0.0 not found.", err.Error())

	ThemeKitVersion, _ = version.NewVersion("0.4.4")
	err = InstallThemeKitVersion("latest")
	assert.Nil(suite.T(), err)
}

func (suite *VersionTestSuite) TestFetchReleases() {
	server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
		fmt.Fprintf(w, jsonFixture("responses/all_releases"))
	}))
	releasesURL = server.URL

	releases, err := fetchReleases()
	assert.Nil(suite.T(), err)
	assert.Equal(suite.T(), 4, len(releases))
	server.Close()

	server = httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
		fmt.Fprintf(w, "this is not json")
	}))
	releasesURL = server.URL
	releases, err = fetchReleases()
	assert.NotNil(suite.T(), err)
	server.Close()

	server = httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
		w.WriteHeader(404)
		fmt.Fprintf(w, "404")
	}))
	releasesURL = server.URL
	releases, err = fetchReleases()
	assert.NotNil(suite.T(), err)
	server.Close()
}

func (suite *VersionTestSuite) TestApplyUpdate() {
	server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
		fmt.Fprintf(w, string(newFile))
	}))

	err := applyUpdate(platform{
		URL:        server.URL,
		Digest:     hex.EncodeToString(newFileChecksum[:]),
		TargetPath: updatePath,
	})
	assert.Nil(suite.T(), err)

	buf, err := ioutil.ReadFile(updatePath)
	assert.Nil(suite.T(), err)
	assert.Equal(suite.T(), newFile, buf)
	server.Close()

	err = applyUpdate(platform{})
	assert.NotNil(suite.T(), err)

	server = httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
		w.WriteHeader(404)
		fmt.Fprintf(w, "404")
	}))
	err = applyUpdate(platform{URL: server.URL})
	assert.NotNil(suite.T(), err)
	server.Close()
}

func TestVersionTestSuite(t *testing.T) {
	suite.Run(t, new(VersionTestSuite))
}
<|endoftext|>


<commit_before>package setcd

import (
	"context"
	"encoding/json"
	"fmt"
	"github.com/coreos/etcd/client"
	"github.com/shawnfeng/sutil/slog/slog"
	"github.com/shawnfeng/sutil/stime"
	"time"
)

type EtcdInstance struct {
	Api client.KeysAPI
}

func NewEtcdInstanceWichApi(api client.KeysAPI) *EtcdInstance {
	return &EtcdInstance{
		Api: api,
	}
}

func NewEtcdInstance(cluster []string) (*EtcdInstance, error) {
	cfg := client.Config{
		Endpoints: cluster,
		Transport: client.DefaultTransport,
	}
	c, err := client.New(cfg)
	if err != nil {
		return nil, fmt.Errorf("create etchd client cfg error")
	}
	api := client.NewKeysAPI(c)
	if api == nil {
		return nil, fmt.Errorf("create etchd api error")
	}
	return NewEtcdInstanceWichApi(api), nil
}
func (m *EtcdInstance) Get(ctx context.Context, path string) (string, error) {
	r, err := m.Api.Get(ctx, path, &client.GetOptions{
		Recursive: false,
		Sort:      false,
	})
	if err != nil {
		return "", err
	}

	if r.Node == nil {
		return "", fmt.Errorf("etcdIns node value err location:%s", path)
	}

	return r.Node.Value, nil
}
func (m *EtcdInstance) GetNode(ctx context.Context, path string) (*client.Node, error) {
	r, err := m.Api.Get(ctx, path, &client.GetOptions{
		Recursive: true,
		Sort:      false,
	})
	if err != nil {
		return nil, err
	}

	if r.Node == nil {
		return nil, fmt.Errorf("etcdIns node value err location:%s", path)
	}

	return r.Node, nil
}
func (m *EtcdInstance) Set(ctx context.Context, path, val string) error {
	r, err := m.Api.Set(ctx, path, val, &client.SetOptions{})
	if err != nil {
		return err
	}

	if r.Node == nil {
		return fmt.Errorf("etcdIns node value err location:%s", path)
	}

	return nil
}
func (m *EtcdInstance) CreateDir(ctx context.Context, path string) error {
	_, err := m.Api.Set(ctx, path, "", &client.SetOptions{
		Dir:       true,
		PrevExist: client.PrevNoExist,
	})
	if err != nil {
		return err
	}
	return nil
}
func (m *EtcdInstance) SetTtl(ctx context.Context, path, val string, ttl time.Duration) error {
	_, err := m.Api.Set(ctx, path, val, &client.SetOptions{
		TTL: ttl,
	})
	if err != nil {
		return err
	}

	return nil
}
func (m *EtcdInstance) RefreshTtl(ctx context.Context, path string, ttl time.Duration) error {

	_, err := m.Api.Set(ctx, path, "", &client.SetOptions{
		PrevExist: client.PrevExist,
		Refresh:   true,
		TTL:       ttl,
	})
	if err != nil {
		return err
	}

	return nil
}
func (m *EtcdInstance) SetNx(ctx context.Context, path, val string) error {
	_, err := m.Api.Set(ctx, path, val, &client.SetOptions{
		PrevExist: client.PrevNoExist,
	})
	if err != nil {
		return err
	}

	return nil
}
func (m *EtcdInstance) Regist(ctx context.Context, path, val string, heatbeat time.Duration, ttl time.Duration) error {
	fun := "EtcdInstance.Regist -->"
	var isset = true
	go func() {
		for i := 0; ; i++ {
			var err error
			if isset {
				slog.Warnf(ctx, "%s create idx:%d val:%s", fun, i, val)
				_, err = m.Api.Set(ctx, path, val, &client.SetOptions{
					TTL: ttl,
				})
				if err == nil {
					isset = false
				}
			} else {
				slog.Infof(ctx, "%s refresh ttl idx:%d val:%s", fun, i, val)
				_, err = m.Api.Set(ctx, path, "", &client.SetOptions{
					PrevExist: client.PrevExist,
					TTL:       ttl,
					Refresh:   true,
				})
			}
			if err != nil {
				slog.Errorf(ctx, "%s reg idx:%d err:%s", fun, i, err)

			}

			time.Sleep(heatbeat)
		}
	}()

	return nil
}
func (m *EtcdInstance) Watch(ctx context.Context, path string, hander func(*client.Response)) {
	fun := "EtcdInstance.Watch -->"
	backoff := stime.NewBackOffCtrl(time.Millisecond*10, time.Second*5)
	var chg chan *client.Response
	go func() {
		slog.Infof(ctx, "%s start watch:%s", fun, path)
		for {
			if chg == nil {
				slog.Infof(ctx, "%s loop watch new receiver:%s", fun, path)
				chg = make(chan *client.Response)
				go m.startWatch(ctx, chg, path)
			}

			r, ok := <-chg
			if !ok {
				slog.Errorf(ctx, "%s chg info nil:%s", fun, path)
				chg = nil
				backoff.BackOff()
			} else {
				slog.Infof(ctx, "%s update path:%s", fun, r.Node.Key)
				hander(r)
				backoff.Reset()
			}
		}
	}()
}

func (m *EtcdInstance) startWatch(ctx context.Context, chg chan *client.Response, path string) {
	fun := "EtcdInstance.startWatch -->"

	for i := 0; ; i++ {
		r, err := m.Api.Get(ctx, path, &client.GetOptions{Recursive: true, Sort: false})
		if err != nil {
			slog.Warnf(ctx, "%s get path:%s err:%s", fun, path, err)
		} else {
			chg <- r
		}
		index := uint64(0)
		if r != nil {
			index = r.Index
			slog.Infof(ctx, "%s init get action:%s nodes:%d index:%d path:%s", fun, r.Action, len(r.Node.Nodes), r.Index, path)
		}

		wop := &client.WatcherOptions{
			Recursive:  true,
			AfterIndex: index,
		}
		watcher := m.Api.Watcher(path, wop)
		if watcher == nil {
			slog.Errorf(ctx, "%s new watcher path:%s", fun, path)
			return
		}

		resp, err := watcher.Next(context.Background())
		// etcdIns 关闭时候会返回
		if err != nil {
			slog.Errorf(ctx, "%s watch path:%s err:%s", fun, path, err)
			close(chg)
			return
		} else {
			slog.Infof(ctx, "%s next get idx:%d action:%s nodes:%d index:%d after:%d path:%s", fun, i, resp.Action, len(resp.Node.Nodes), resp.Index, wop.AfterIndex, path)
			// 测试发现next获取到的返回，index，重新获取总有问题，触发两次，不确定，为什么？为什么？
			// 所以这里每次next前使用的afterindex都重新get了
		}
	}

}
<commit_msg>remove useless code<commit_after>package setcd

import (
	"context"
	"fmt"
	"time"

	"github.com/coreos/etcd/client"
	"github.com/shawnfeng/sutil/slog/slog"
	"github.com/shawnfeng/sutil/stime"
)

type EtcdInstance struct {
	Api client.KeysAPI
}

func NewEtcdInstanceWichApi(api client.KeysAPI) *EtcdInstance {
	return &EtcdInstance{
		Api: api,
	}
}

func NewEtcdInstance(cluster []string) (*EtcdInstance, error) {
	cfg := client.Config{
		Endpoints: cluster,
		Transport: client.DefaultTransport,
	}
	c, err := client.New(cfg)
	if err != nil {
		return nil, fmt.Errorf("create etchd client cfg error")
	}
	api := client.NewKeysAPI(c)
	if api == nil {
		return nil, fmt.Errorf("create etchd api error")
	}
	return NewEtcdInstanceWichApi(api), nil
}
func (m *EtcdInstance) Get(ctx context.Context, path string) (string, error) {
	r, err := m.Api.Get(ctx, path, &client.GetOptions{
		Recursive: false,
		Sort:      false,
	})
	if err != nil {
		return "", err
	}

	if r.Node == nil {
		return "", fmt.Errorf("etcdIns node value err location:%s", path)
	}

	return r.Node.Value, nil
}
func (m *EtcdInstance) GetNode(ctx context.Context, path string) (*client.Node, error) {
	r, err := m.Api.Get(ctx, path, &client.GetOptions{
		Recursive: true,
		Sort:      false,
	})
	if err != nil {
		return nil, err
	}

	if r.Node == nil {
		return nil, fmt.Errorf("etcdIns node value err location:%s", path)
	}

	return r.Node, nil
}
func (m *EtcdInstance) Set(ctx context.Context, path, val string) error {
	r, err := m.Api.Set(ctx, path, val, &client.SetOptions{})
	if err != nil {
		return err
	}

	if r.Node == nil {
		return fmt.Errorf("etcdIns node value err location:%s", path)
	}

	return nil
}
func (m *EtcdInstance) CreateDir(ctx context.Context, path string) error {
	_, err := m.Api.Set(ctx, path, "", &client.SetOptions{
		Dir:       true,
		PrevExist: client.PrevNoExist,
	})
	if err != nil {
		return err
	}
	return nil
}
func (m *EtcdInstance) SetTtl(ctx context.Context, path, val string, ttl time.Duration) error {
	_, err := m.Api.Set(ctx, path, val, &client.SetOptions{
		TTL: ttl,
	})
	if err != nil {
		return err
	}

	return nil
}
func (m *EtcdInstance) RefreshTtl(ctx context.Context, path string, ttl time.Duration) error {

	_, err := m.Api.Set(ctx, path, "", &client.SetOptions{
		PrevExist: client.PrevExist,
		Refresh:   true,
		TTL:       ttl,
	})
	if err != nil {
		return err
	}

	return nil
}
func (m *EtcdInstance) SetNx(ctx context.Context, path, val string) error {
	_, err := m.Api.Set(ctx, path, val, &client.SetOptions{
		PrevExist: client.PrevNoExist,
	})
	if err != nil {
		return err
	}

	return nil
}
func (m *EtcdInstance) Regist(ctx context.Context, path, val string, heatbeat time.Duration, ttl time.Duration) error {
	fun := "EtcdInstance.Regist -->"
	var isset = true
	go func() {
		for i := 0; ; i++ {
			var err error
			if isset {
				slog.Warnf(ctx, "%s create idx:%d val:%s", fun, i, val)
				_, err = m.Api.Set(ctx, path, val, &client.SetOptions{
					TTL: ttl,
				})
				if err == nil {
					isset = false
				}
			} else {
				slog.Infof(ctx, "%s refresh ttl idx:%d val:%s", fun, i, val)
				_, err = m.Api.Set(ctx, path, "", &client.SetOptions{
					PrevExist: client.PrevExist,
					TTL:       ttl,
					Refresh:   true,
				})
			}
			if err != nil {
				slog.Errorf(ctx, "%s reg idx:%d err:%s", fun, i, err)

			}

			time.Sleep(heatbeat)
		}
	}()

	return nil
}
func (m *EtcdInstance) Watch(ctx context.Context, path string, hander func(*client.Response)) {
	fun := "EtcdInstance.Watch -->"
	backoff := stime.NewBackOffCtrl(time.Millisecond*10, time.Second*5)
	var chg chan *client.Response
	go func() {
		slog.Infof(ctx, "%s start watch:%s", fun, path)
		for {
			if chg == nil {
				slog.Infof(ctx, "%s loop watch new receiver:%s", fun, path)
				chg = make(chan *client.Response)
				go m.startWatch(ctx, chg, path)
			}

			r, ok := <-chg
			if !ok {
				slog.Errorf(ctx, "%s chg info nil:%s", fun, path)
				chg = nil
				backoff.BackOff()
			} else {
				slog.Infof(ctx, "%s update path:%s", fun, r.Node.Key)
				hander(r)
				backoff.Reset()
			}
		}
	}()
}

func (m *EtcdInstance) startWatch(ctx context.Context, chg chan *client.Response, path string) {
	fun := "EtcdInstance.startWatch -->"

	for i := 0; ; i++ {
		r, err := m.Api.Get(ctx, path, &client.GetOptions{Recursive: true, Sort: false})
		if err != nil {
			slog.Warnf(ctx, "%s get path:%s err:%s", fun, path, err)
		} else {
			chg <- r
		}
		index := uint64(0)
		if r != nil {
			index = r.Index
			slog.Infof(ctx, "%s init get action:%s nodes:%d index:%d path:%s", fun, r.Action, len(r.Node.Nodes), r.Index, path)
		}

		wop := &client.WatcherOptions{
			Recursive:  true,
			AfterIndex: index,
		}
		watcher := m.Api.Watcher(path, wop)
		if watcher == nil {
			slog.Errorf(ctx, "%s new watcher path:%s", fun, path)
			return
		}

		resp, err := watcher.Next(context.Background())
		// etcdIns 关闭时候会返回
		if err != nil {
			slog.Errorf(ctx, "%s watch path:%s err:%s", fun, path, err)
			close(chg)
			return
		} else {
			slog.Infof(ctx, "%s next get idx:%d action:%s nodes:%d index:%d after:%d path:%s", fun, i, resp.Action, len(resp.Node.Nodes), resp.Index, wop.AfterIndex, path)
			// 测试发现next获取到的返回，index，重新获取总有问题，触发两次，不确定，为什么？为什么？
			// 所以这里每次next前使用的afterindex都重新get了
		}
	}

}
<|endoftext|>