<commit_before>// Copyright 2021 Google LLC
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     https://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
//
// Usage: generate_catalog SRC_REPO_DIR/ DEST_MD_DIR/
//
// The command will create a README.md file under DEST_MD_DIR/ containing a table
// of collected functions. Source files for the catalog will also appear in
// DEST_MD_DIR/
package main

import (
	"bytes"
	"encoding/json"
	"fmt"
	"io/ioutil"
	"os"
	"os/exec"
	"path/filepath"
	"regexp"
	"sort"
	"strings"

	"golang.org/x/mod/semver"
	"gopkg.in/yaml.v2"
)

func main() {
	if len(os.Args) < 3 {
		fmt.Fprintf(os.Stderr, "Usage: generate-catalog SRC_REPO_DIR/ DEST_MD_DIR/\n")
		os.Exit(1)
	}
	source, err := filepath.Abs(os.Args[1])
	if err != nil {
		fmt.Fprintf(os.Stderr, "%v\n", err)
		os.Exit(1)
	}
	dest, err := filepath.Abs(os.Args[2])
	if err != nil {
		fmt.Fprintf(os.Stderr, "%v\n", err)
		os.Exit(1)
	}

	branches, err := getBranches()
	if err != nil {
		fmt.Fprintf(os.Stderr, "%v\n", err)
		os.Exit(1)
	}

	functions := getFunctions(branches, source, dest)
	err = writeFunctionIndex(functions, source, dest)
	if err != nil {
		fmt.Fprintf(os.Stderr, "%v\n", err)
		os.Exit(1)
	}

	err = writeExampleIndex(functions, source, dest)
	if err != nil {
		fmt.Fprintf(os.Stderr, "%v\n", err)
		os.Exit(1)
	}
}

type function struct {
	FunctionName      string
	ImagePath         string
	VersionToExamples map[string]map[string]example
	LatestVersion     string
	Path              string
	Description       string
	Tags              string
	Gcp               bool
}

type example struct {
	LocalExamplePath  string
	RemoteExamplePath string
	RemoteSourcePath  string
}

type metadata struct {
	Image              string
	Description        string
	Tags               []string
	SourceUrl          string   `yaml:"sourceURL"`
	ExamplePackageUrls []string `yaml:"examplePackageURLs"`
	Hidden             bool
}

var (
	// Match start of a version such as v1.9.1
	branchSemverPrefix = regexp.MustCompile(`[-\w]*\/(v\d*\.\d*)`)
	functionDirPrefix  = regexp.MustCompile(`.+/functions/`)
)

func getBranches() ([]string, error) {
	verBranches := make([]string, 0)

	var buf bytes.Buffer
	cmd := exec.Command("git", "branch", "-a")
	cmd.Stdout = &buf
	err := cmd.Run()
	if err != nil {
		return verBranches, err
	}

	for _, branch := range strings.Split(buf.String(), "\n") {
		if branchSemverPrefix.MatchString(branch) {
			verBranches = append(verBranches, strings.TrimSpace(branch))
		}
	}
	return verBranches, err
}

func getFunctions(branches []string, source string, dest string) []function {
	functions := make(map[string]function)
	for _, b := range branches {
		segments := strings.Split(b, "/")
		funcName := segments[len(segments)-2]
		minorVersion := segments[len(segments)-1]
		funcDest := filepath.Join(dest, funcName)
		versionDest := filepath.Join(funcDest, minorVersion)
		relativeFuncPath, err := getRelativeFunctionPath(source, funcName)
		if err != nil {
			fmt.Fprintf(os.Stderr, "%v\n", err)
			continue
		}

		// Skip contributed functions.
		if strings.Contains(relativeFuncPath, "contrib/") {
			continue
		}

		// Functions with the hidden field enabled should not be processed.
		metadataPath := strings.TrimSpace(fmt.Sprintf("%v:%v", b, filepath.Join(relativeFuncPath, "metadata.yaml")))
		md, err := getMetadata(metadataPath)
		if err != nil {
			fmt.Fprintf(os.Stderr, "Error getting metadata for %q from %q: %v\n", funcName, b, err)
			os.Exit(1)
		}
		if md.Hidden {
			continue
		}
		err = copyExamples(b, md.ExamplePackageUrls, versionDest, minorVersion)
		if err != nil {
			fmt.Fprintf(os.Stderr, "Error getting examples for %q from %q: %v\n", funcName, b, err)
			os.Exit(1)
		}

		err = copyReadme(b, funcName, relativeFuncPath, versionDest)
		if err != nil {
			fmt.Fprintf(os.Stderr, "Error getting README for %q from %q: %v\n", funcName, b, err)
			os.Exit(1)
		}

		f := functions[funcName]
		f.FunctionName = funcName
		if f.VersionToExamples == nil {
			f.VersionToExamples = make(map[string]map[string]example)
		}

		functions[funcName] = parseMetadata(f, md, minorVersion, versionDest)
	}

	flattenedFunctions := make([]function, 0)
	for _, f := range functions {
		flattenedFunctions = append(flattenedFunctions, f)
	}
	sort.Slice(flattenedFunctions, func(i int, j int) bool {
		return flattenedFunctions[i].FunctionName < flattenedFunctions[j].FunctionName
	})
	return flattenedFunctions
}

func copyExamples(b string, exampleSources []string, versionDest, minorVersion string) error {
	// Copy examples for the function's version to a temporary directory.
	tempDir, err := ioutil.TempDir("", "examples")
	if err != nil {
		return err
	}

	// Prepare destination for versioned examples.
	err = os.MkdirAll(versionDest, 0744)
	if err != nil {
		return err
	}

	for _, exampleSource := range exampleSources {
		splitedPaths := strings.SplitN(exampleSource, minorVersion+string(filepath.Separator), 2)
		if len(splitedPaths) != 2 {
			return fmt.Errorf("expect 2 substring after spliting %q by %q", exampleSource, minorVersion+string(filepath.Separator))
		}
		relativePath := splitedPaths[1]
		// Fetch example into temporary directory.
		cmd := exec.Command("git", fmt.Sprintf("--work-tree=%v", tempDir), "checkout", b, "--", relativePath)
		err = cmd.Run()
		if err != nil {
			return fmt.Errorf("Error running %v: %v", cmd, err)
		}

		exampleName := filepath.Base(relativePath)
		src := filepath.Join(tempDir, relativePath)
		dest := filepath.Join(versionDest, exampleName)

		// Move example content to the site's example directory.
		err = os.Rename(src, dest)
		if err != nil {
			return err
		}

	}

	return nil
}

func copyReadme(b string, funcName string, relativeFuncPath string, versionDest string) error {
	// Copy README for the function's version to the example directory.
	tempDir, err := ioutil.TempDir("", "functions")
	if err != nil {
		return err
	}
	cmd := exec.Command("git", fmt.Sprintf("--work-tree=%v", tempDir), "checkout", b, "--", filepath.Join(relativeFuncPath, "README.md"))
	err = cmd.Run()
	if err != nil {
		return fmt.Errorf("Error running %v: %v", cmd, err)
	}

	// Find the README in the example directory.
	m, err := filepath.Glob(filepath.Join(tempDir, "functions", "*", funcName, "README.md"))
	if err != nil {
		return err
	}

	// Move the README to the destination directory.
	err = os.Rename(m[0], filepath.Join(versionDest, "README.md"))
	if err != nil {
		return err
	}

	return nil
}

func getMetadata(metadataPath string) (metadata, error) {
	var buf bytes.Buffer
	var md metadata
	// Get the content of metadata.yaml from the appropriate release branch.
	cmd := exec.Command("git", "cat-file", "blob", metadataPath)
	cmd.Stdout = &buf
	err := cmd.Run()
	if err != nil {
		return md, err
	}

	yaml.Unmarshal(buf.Bytes(), &md)
	return md, nil
}

func parseMetadata(f function, md metadata, version string, versionDest string) function {

	// Add examples to version map.
	if f.VersionToExamples[version] == nil {
		f.VersionToExamples[version] = make(map[string]example)
	}
	e := f.VersionToExamples[version]
	for _, exUrl := range md.ExamplePackageUrls {
		exUrlSegments := strings.Split(exUrl, "/")
		exName := exUrlSegments[len(exUrlSegments)-1]
		ex := example{
			RemoteSourcePath:  md.SourceUrl,
			RemoteExamplePath: exUrl,
			LocalExamplePath:  filepath.Join(versionDest, exName),
		}
		e[exName] = ex
	}
	f.VersionToExamples[version] = e
	if semver.Compare(f.LatestVersion, version) == 1 {
		return f
	}

	// If this is the latest version,
	// update latest version, default path, description and rags.
	f.LatestVersion = version
	f.Path = versionDest
	f.Description = md.Description
	functionTags := make([]string, 0)
	for _, tag := range md.Tags {
		normalizedTag := strings.ToLower(tag)
		if normalizedTag == "gcp" {
			f.Gcp = true
		} else {
			functionTags = append(functionTags, normalizedTag)
		}
	}
	sort.Strings(functionTags)
	f.Tags = strings.Join(functionTags, ", ")
	f.ImagePath = md.Image

	return f
}

func getRelativeFunctionPath(source string, funcName string) (string, error) {
	// Find the directory for the function's source.
	sourcePattern := filepath.Join(source, "functions", "*", funcName)
	m, err := filepath.Glob(sourcePattern)
	if err != nil {
		return "", err
	}
	if m == nil {
		return "", fmt.Errorf("Could not find a function with the following pattern: %v", sourcePattern)
	}

	return functionDirPrefix.ReplaceAllString(m[0], "functions/"), nil
}

func writeFunctionIndex(functions []function, source string, dest string) error {
	out := []string{"# Functions Catalog", ""}
	genericFunctions := make([]function, 0)
	gcp := make([]function, 0)
	for _, f := range functions {
		if f.Gcp {
			gcp = append(gcp, f)
		} else {
			genericFunctions = append(genericFunctions, f)
		}
	}

	out = append(out, getFunctionTable(genericFunctions, source)...)

	if len(gcp) > 0 {
		out = append(out, "", "## GCP Functions", "")
		out = append(out, getFunctionTable(gcp, source)...)
	}

	o := strings.Join(out, "\n")
	err := ioutil.WriteFile(filepath.Join(dest, "README.md"), []byte(o), 0744)
	return err
}

func getFunctionTable(functions []function, source string) []string {
	out := []string{"| Name | Description | Tags |", "| ---- | ----------- | ---- |"}
	for _, f := range functions {
		functionEntry := fmt.Sprintf("| [%v](%v/) | %v | %v |", f.FunctionName, strings.Replace(f.Path, filepath.Join(source, "site"), "", 1), f.Description, f.Tags)
		out = append(out, functionEntry)
	}
	return out
}

func writeExampleIndex(functions []function, source string, dest string) error {
	// Key a function's version's examples by the function's name -> version
	functionVersionMap := make(map[string]map[string]map[string]example)
	for _, f := range functions {
		vToE := make(map[string]map[string]example)
		for v, examples := range f.VersionToExamples {
			exampleToPaths := make(map[string]example)
			for exName, ex := range examples {
				e := ex
				e.LocalExamplePath = strings.Replace(ex.LocalExamplePath, filepath.Join(source, "site"), "", 1)
				exampleToPaths[exName] = e
			}
			vToE[v] = exampleToPaths
		}
		functionVersionMap[f.FunctionName] = vToE
	}

	funcJson, err := json.Marshal(functionVersionMap)
	if err != nil {
		return err
	}

	err = ioutil.WriteFile(filepath.Join(dest, "catalog.json"), funcJson, 0600)
	return err
}
<commit_msg>Add globbing logic for contributed functions. (#534)<commit_after>// Copyright 2021 Google LLC
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     https://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
//
// Usage: generate_catalog SRC_REPO_DIR/ DEST_MD_DIR/
//
// The command will create a README.md file under DEST_MD_DIR/ containing a table
// of collected functions. Source files for the catalog will also appear in
// DEST_MD_DIR/
package main

import (
	"bytes"
	"encoding/json"
	"fmt"
	"io/ioutil"
	"os"
	"os/exec"
	"path/filepath"
	"regexp"
	"sort"
	"strings"

	"golang.org/x/mod/semver"
	"gopkg.in/yaml.v2"
)

func main() {
	if len(os.Args) < 3 {
		fmt.Fprintf(os.Stderr, "Usage: generate-catalog SRC_REPO_DIR/ DEST_MD_DIR/\n")
		os.Exit(1)
	}
	source, err := filepath.Abs(os.Args[1])
	if err != nil {
		fmt.Fprintf(os.Stderr, "%v\n", err)
		os.Exit(1)
	}
	dest, err := filepath.Abs(os.Args[2])
	if err != nil {
		fmt.Fprintf(os.Stderr, "%v\n", err)
		os.Exit(1)
	}

	branches, err := getBranches()
	if err != nil {
		fmt.Fprintf(os.Stderr, "%v\n", err)
		os.Exit(1)
	}

	functions := getFunctions(branches, source, dest)
	err = writeFunctionIndex(functions, source, dest)
	if err != nil {
		fmt.Fprintf(os.Stderr, "%v\n", err)
		os.Exit(1)
	}

	err = writeExampleIndex(functions, source, dest)
	if err != nil {
		fmt.Fprintf(os.Stderr, "%v\n", err)
		os.Exit(1)
	}
}

type function struct {
	FunctionName      string
	ImagePath         string
	VersionToExamples map[string]map[string]example
	LatestVersion     string
	Path              string
	Description       string
	Tags              string
	Gcp               bool
}

type example struct {
	LocalExamplePath  string
	RemoteExamplePath string
	RemoteSourcePath  string
}

type metadata struct {
	Image              string
	Description        string
	Tags               []string
	SourceUrl          string   `yaml:"sourceURL"`
	ExamplePackageUrls []string `yaml:"examplePackageURLs"`
	Hidden             bool
}

var (
	// Match start of a version such as v1.9.1
	branchSemverPrefix = regexp.MustCompile(`[-\w]*\/(v\d*\.\d*)`)
	functionDirPrefix  = regexp.MustCompile(`.+/functions/`)
)

func getBranches() ([]string, error) {
	verBranches := make([]string, 0)

	var buf bytes.Buffer
	cmd := exec.Command("git", "branch", "-a")
	cmd.Stdout = &buf
	err := cmd.Run()
	if err != nil {
		return verBranches, err
	}

	for _, branch := range strings.Split(buf.String(), "\n") {
		if branchSemverPrefix.MatchString(branch) {
			verBranches = append(verBranches, strings.TrimSpace(branch))
		}
	}
	return verBranches, err
}

func getFunctions(branches []string, source string, dest string) []function {
	functions := make(map[string]function)
	for _, b := range branches {
		segments := strings.Split(b, "/")
		funcName := segments[len(segments)-2]
		minorVersion := segments[len(segments)-1]
		funcDest := filepath.Join(dest, funcName)
		versionDest := filepath.Join(funcDest, minorVersion)
		relativeFuncPath, err := getRelativeFunctionPath(source, funcName)
		if err != nil {
			fmt.Fprintf(os.Stderr, "%v\n", err)
			continue
		}

		// Skip contributed functions.
		if strings.Contains(relativeFuncPath, "contrib/") {
			continue
		}

		// Functions with the hidden field enabled should not be processed.
		metadataPath := strings.TrimSpace(fmt.Sprintf("%v:%v", b, filepath.Join(relativeFuncPath, "metadata.yaml")))
		md, err := getMetadata(metadataPath)
		if err != nil {
			fmt.Fprintf(os.Stderr, "Error getting metadata for %q from %q: %v\n", funcName, b, err)
			os.Exit(1)
		}
		if md.Hidden {
			continue
		}
		err = copyExamples(b, md.ExamplePackageUrls, versionDest, minorVersion)
		if err != nil {
			fmt.Fprintf(os.Stderr, "Error getting examples for %q from %q: %v\n", funcName, b, err)
			os.Exit(1)
		}

		err = copyReadme(b, funcName, relativeFuncPath, versionDest)
		if err != nil {
			fmt.Fprintf(os.Stderr, "Error getting README for %q from %q: %v\n", funcName, b, err)
			os.Exit(1)
		}

		f := functions[funcName]
		f.FunctionName = funcName
		if f.VersionToExamples == nil {
			f.VersionToExamples = make(map[string]map[string]example)
		}

		functions[funcName] = parseMetadata(f, md, minorVersion, versionDest)
	}

	flattenedFunctions := make([]function, 0)
	for _, f := range functions {
		flattenedFunctions = append(flattenedFunctions, f)
	}
	sort.Slice(flattenedFunctions, func(i int, j int) bool {
		return flattenedFunctions[i].FunctionName < flattenedFunctions[j].FunctionName
	})
	return flattenedFunctions
}

func copyExamples(b string, exampleSources []string, versionDest, minorVersion string) error {
	// Copy examples for the function's version to a temporary directory.
	tempDir, err := ioutil.TempDir("", "examples")
	if err != nil {
		return err
	}

	// Prepare destination for versioned examples.
	err = os.MkdirAll(versionDest, 0744)
	if err != nil {
		return err
	}

	for _, exampleSource := range exampleSources {
		splitedPaths := strings.SplitN(exampleSource, minorVersion+string(filepath.Separator), 2)
		if len(splitedPaths) != 2 {
			return fmt.Errorf("expect 2 substring after spliting %q by %q", exampleSource, minorVersion+string(filepath.Separator))
		}
		relativePath := splitedPaths[1]
		// Fetch example into temporary directory.
		cmd := exec.Command("git", fmt.Sprintf("--work-tree=%v", tempDir), "checkout", b, "--", relativePath)
		err = cmd.Run()
		if err != nil {
			return fmt.Errorf("Error running %v: %v", cmd, err)
		}

		exampleName := filepath.Base(relativePath)
		src := filepath.Join(tempDir, relativePath)
		dest := filepath.Join(versionDest, exampleName)

		// Move example content to the site's example directory.
		err = os.Rename(src, dest)
		if err != nil {
			return err
		}

	}

	return nil
}

func copyReadme(b string, funcName string, relativeFuncPath string, versionDest string) error {
	// Copy README for the function's version to the example directory.
	tempDir, err := ioutil.TempDir("", "functions")
	if err != nil {
		return err
	}
	cmd := exec.Command("git", fmt.Sprintf("--work-tree=%v", tempDir), "checkout", b, "--", filepath.Join(relativeFuncPath, "README.md"))
	err = cmd.Run()
	if err != nil {
		return fmt.Errorf("Error running %v: %v", cmd, err)
	}

	// Find the README in the example directory.
	m, err := filepath.Glob(filepath.Join(tempDir, "functions", "*", funcName, "README.md"))
	if err != nil {
		return err
	}

	// Move the README to the destination directory.
	err = os.Rename(m[0], filepath.Join(versionDest, "README.md"))
	if err != nil {
		return err
	}

	return nil
}

func getMetadata(metadataPath string) (metadata, error) {
	var buf bytes.Buffer
	var md metadata
	// Get the content of metadata.yaml from the appropriate release branch.
	cmd := exec.Command("git", "cat-file", "blob", metadataPath)
	cmd.Stdout = &buf
	err := cmd.Run()
	if err != nil {
		return md, err
	}

	yaml.Unmarshal(buf.Bytes(), &md)
	return md, nil
}

func parseMetadata(f function, md metadata, version string, versionDest string) function {

	// Add examples to version map.
	if f.VersionToExamples[version] == nil {
		f.VersionToExamples[version] = make(map[string]example)
	}
	e := f.VersionToExamples[version]
	for _, exUrl := range md.ExamplePackageUrls {
		exUrlSegments := strings.Split(exUrl, "/")
		exName := exUrlSegments[len(exUrlSegments)-1]
		ex := example{
			RemoteSourcePath:  md.SourceUrl,
			RemoteExamplePath: exUrl,
			LocalExamplePath:  filepath.Join(versionDest, exName),
		}
		e[exName] = ex
	}
	f.VersionToExamples[version] = e
	if semver.Compare(f.LatestVersion, version) == 1 {
		return f
	}

	// If this is the latest version,
	// update latest version, default path, description and rags.
	f.LatestVersion = version
	f.Path = versionDest
	f.Description = md.Description
	functionTags := make([]string, 0)
	for _, tag := range md.Tags {
		normalizedTag := strings.ToLower(tag)
		if normalizedTag == "gcp" {
			f.Gcp = true
		} else {
			functionTags = append(functionTags, normalizedTag)
		}
	}
	sort.Strings(functionTags)
	f.Tags = strings.Join(functionTags, ", ")
	f.ImagePath = md.Image

	return f
}

func getRelativeFunctionPath(source string, funcName string) (string, error) {
	// Find the directory for the function's source.
	sourcePattern := filepath.Join(source, "functions", "*", funcName)
	m, err := filepath.Glob(sourcePattern)
	if err != nil {
		return "", err
	}
	if m == nil {
		contribPattern := filepath.Join(source, "functions", "contrib", "*", funcName)
		m, err = filepath.Glob(contribPattern)
		if err != nil {
			return "", err
		}
		if m == nil {
			return "", fmt.Errorf("Could not find a function with the following name: %v", funcName)
		}
	}

	return functionDirPrefix.ReplaceAllString(m[0], "functions/"), nil
}

func writeFunctionIndex(functions []function, source string, dest string) error {
	out := []string{"# Functions Catalog", ""}
	genericFunctions := make([]function, 0)
	gcp := make([]function, 0)
	for _, f := range functions {
		if f.Gcp {
			gcp = append(gcp, f)
		} else {
			genericFunctions = append(genericFunctions, f)
		}
	}

	out = append(out, getFunctionTable(genericFunctions, source)...)

	if len(gcp) > 0 {
		out = append(out, "", "## GCP Functions", "")
		out = append(out, getFunctionTable(gcp, source)...)
	}

	o := strings.Join(out, "\n")
	err := ioutil.WriteFile(filepath.Join(dest, "README.md"), []byte(o), 0744)
	return err
}

func getFunctionTable(functions []function, source string) []string {
	out := []string{"| Name | Description | Tags |", "| ---- | ----------- | ---- |"}
	for _, f := range functions {
		functionEntry := fmt.Sprintf("| [%v](%v/) | %v | %v |", f.FunctionName, strings.Replace(f.Path, filepath.Join(source, "site"), "", 1), f.Description, f.Tags)
		out = append(out, functionEntry)
	}
	return out
}

func writeExampleIndex(functions []function, source string, dest string) error {
	// Key a function's version's examples by the function's name -> version
	functionVersionMap := make(map[string]map[string]map[string]example)
	for _, f := range functions {
		vToE := make(map[string]map[string]example)
		for v, examples := range f.VersionToExamples {
			exampleToPaths := make(map[string]example)
			for exName, ex := range examples {
				e := ex
				e.LocalExamplePath = strings.Replace(ex.LocalExamplePath, filepath.Join(source, "site"), "", 1)
				exampleToPaths[exName] = e
			}
			vToE[v] = exampleToPaths
		}
		functionVersionMap[f.FunctionName] = vToE
	}

	funcJson, err := json.Marshal(functionVersionMap)
	if err != nil {
		return err
	}

	err = ioutil.WriteFile(filepath.Join(dest, "catalog.json"), funcJson, 0600)
	return err
}
<|endoftext|>


<commit_before>// Copyright 2009 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

// HTTP Response reading and parsing.

package http

import (
	"bufio"
	"encoding/json"
	"errors"
	"io"
	"net/textproto"
	"net/url"
	"strconv"
	"strings"
)

var respExcludeHeader = map[string]bool{
	"Trailer": true,
}

var knownHeaders = map[string]bool{
	"access_control_allow_origin": true,
	"accept_patch":                true,
	"accept_ranges":               true,
	"age":                         true,
	"allow":                       true,
	"cache_control":               true,
	"connection":                  true,
	"content_disposition":         true,
	"content_encoding":            true,
	"content_language":            true,
	"content_length":              true,
	"content_location":            true,
	"content_md5":                 true,
	"content_range":               true,
	"content_type":                true,
	"expires":                     true,
	"last_modified":               true,
	"link":                        true,
	"location":                    true,
	"p3p":                         true,
	"pragma":                      true,
	"proxy_agent":                 true,
	"proxy_authenticate":          true,
	"public_key_pins":             true,
	"refresh":                     true,
	"retry_after":                 true,
	"server":                      true,
	"set_cookie":                  true,
	"status":                      true,
	"strict_transport_security":   true,
	"trailer":                     true,
	"transfer_encoding":           true,
	"upgrade":                     true,
	"vary":                        true,
	"via":                         true,
	"warning":                     true,
	"www_authenticate":            true,
	"x_frame_options":             true,
	"x_xss_protection":            true,
	"content_security_policy":     true,
	"x_content_security_policy":   true,
	"x_webkit_csp":                true,
	"x_content_type_options":      true,
	"x_powered_by":                true,
	"x_ua_compatible":             true,
	"x_content_duration":          true,
	"x_real_ip":                   true,
	"x_forwarded_for":             true,
}

// Response represents the response from an HTTP request.
//
type Response struct {
	Status     string   `json:"status_line,omitempty"` // e.g. "200 OK"
	StatusCode int      `json:"status_code,omitempty"` // e.g. 200
	Protocol   Protocol `json:"protocol, omitempty"`

	// Header maps header keys to values.  If the response had multiple
	// headers with the same key, they will be concatenated, with comma
	// delimiters.  (Section 4.2 of RFC 2616 requires that multiple headers
	// be semantically equivalent to a comma-delimited sequence.) Values
	// duplicated by other fields in this struct (e.g., ContentLength) are
	// omitted from Header.
	//
	// Keys in the map are canonicalized (see CanonicalHeaderKey).
	Headers Header `json:"headers,omitempty"`

	// Body represents the response body.
	//
	// The http Client and Transport guarantee that Body is always
	// non-nil, even on responses without a body or responses with
	// a zero-lengthed body.
	Body       io.ReadCloser `json:"-"`
	BodyText   string        `json:"body,omitempty"`
	BodySHA256 []byte        `json:"body_sha256,omitempty"`

	// ContentLength records the length of the associated content.  The
	// value -1 indicates that the length is unknown.  Unless RequestMethod
	// is "HEAD", values >= 0 indicate that the given number of bytes may
	// be read from Body.
	ContentLength int64 `json:"content_length,omitempty"`

	// Contains transfer encodings from outer-most to inner-most. Value is
	// nil, means that "identity" encoding is used.
	TransferEncoding []string `json:"transfer_encoding,omitempty"`

	// Close records whether the header directed that the connection be
	// closed after reading Body.  The value is advice for clients: neither
	// ReadResponse nor Response.Write ever closes a connection.
	Close bool `json:"-"`

	// Trailer maps trailer keys to values, in the same
	// format as the header.
	Trailers Header `json:"trailers,omitempty"`

	// The Request that was sent to obtain this Response.
	// Request's Body is nil (having already been consumed).
	// This is only populated for Client requests.
	Request *Request `json:"request,omitempty"`
}

// Cookies parses and returns the cookies set in the Set-Cookie headers.
func (r *Response) Cookies() []*Cookie {
	return readSetCookies(r.Headers)
}

var ErrNoLocation = errors.New("http: no Location header in response")

// Location returns the URL of the response's "Location" header,
// if present.  Relative redirects are resolved relative to
// the Response's Request.  ErrNoLocation is returned if no
// Location header is present.
func (r *Response) Location() (*url.URL, error) {
	lv := r.Headers.Get("Location")
	if lv == "" {
		return nil, ErrNoLocation
	}
	if r.Request != nil && r.Request.URL != nil {
		return r.Request.URL.Parse(lv)
	}
	return url.Parse(lv)
}

// ReadResponse reads and returns an HTTP response from r.  The
// req parameter specifies the Request that corresponds to
// this Response.  Clients must call resp.Body.Close when finished
// reading resp.Body.  After that call, clients can inspect
// resp.Trailer to find key/value pairs included in the response
// trailer.
func ReadResponse(r *bufio.Reader, req *Request) (resp *Response, err error) {

	tp := textproto.NewReader(r)
	resp = new(Response)

	resp.Request = req
	resp.Request.Method = strings.ToUpper(resp.Request.Method)

	// Parse the first line of the response.
	line, err := tp.ReadLine()
	if err != nil {
		if err == io.EOF {
			err = io.ErrUnexpectedEOF
		}
		return nil, err
	}
	f := strings.SplitN(line, " ", 3)
	if len(f) < 2 {
		return nil, &badStringError{"malformed HTTP response", line}
	}
	reasonPhrase := ""
	if len(f) > 2 {
		reasonPhrase = f[2]
	}
	resp.Status = f[1] + " " + reasonPhrase
	resp.StatusCode, err = strconv.Atoi(f[1])
	if err != nil {
		return nil, &badStringError{"malformed HTTP status code", f[1]}
	}

	resp.Protocol = *(new(Protocol))
	resp.Protocol.Name = f[0]
	var ok bool
	if resp.Protocol.Major, resp.Protocol.Minor, ok = ParseHTTPVersion(resp.Protocol.Name); !ok {
		return nil, &badStringError{"malformed HTTP version", resp.Protocol.Name}
	}

	// Parse the response headers.
	mimeHeader, err := tp.ReadMIMEHeader()
	if err != nil {
		return nil, err
	}

	resp.Headers = Header(mimeHeader)

	//remove unknown headers
	filterHeaders(resp.Headers)

	fixPragmaCacheControl(resp.Headers)

	err = readTransfer(resp, r)
	if err != nil {
		return nil, err
	}

	return resp, nil
}

// RFC2616: Should treat
//	Pragma: no-cache
// like
//	Cache-Control: no-cache
func fixPragmaCacheControl(header Header) {
	if hp, ok := header["Pragma"]; ok && len(hp) > 0 && hp[0] == "no-cache" {
		if _, presentcc := header["Cache-Control"]; !presentcc {
			header["Cache-Control"] = []string{"no-cache"}
		}
	}
}

func filterHeaders(h Header) {
	var unknownHeaders []UnknownHeader
	for header, values := range h {
		if _, ok := knownHeaders[FormatHeaderName(header)]; !ok {
			unk := UnknownHeader{
				Key:    FormatHeaderName(header),
				Values: values,
			}
			unknownHeaders = append(unknownHeaders, unk)
			h.Del(header)
		}
	}
	if len(unknownHeaders) > 0 {
		if unknownHeaderStr, err := json.Marshal(unknownHeaders); err == nil {
			h["Unknown"] = []string{string(unknownHeaderStr)}
		}
	}
}

// ProtoAtLeast returns whether the HTTP protocol used
// in the response is at least major.minor.
func (r *Response) ProtoAtLeast(major, minor int) bool {
	return r.Protocol.Major > major ||
		r.Protocol.Major == major && r.Protocol.Minor >= minor
}

// Writes the response (header, body and trailer) in wire format. This method
// consults the following fields of the response:
//
//  StatusCode
//  ProtoMajor
//  ProtoMinor
//  RequestMethod
//  TransferEncoding
//  Trailer
//  Body
//  ContentLength
//  Header, values for non-canonical keys will have unpredictable behavior
//
func (r *Response) Write(w io.Writer) error {

	// RequestMethod should be upper-case
	if r.Request != nil {
		r.Request.Method = strings.ToUpper(r.Request.Method)
	}

	// Status line
	text := r.Status
	if text == "" {
		var ok bool
		text, ok = statusText[r.StatusCode]
		if !ok {
			text = "status code " + strconv.Itoa(r.StatusCode)
		}
	}
	io.WriteString(w, "HTTP/"+strconv.Itoa(r.Protocol.Major)+".")
	io.WriteString(w, strconv.Itoa(r.Protocol.Minor)+" ")
	io.WriteString(w, strconv.Itoa(r.StatusCode)+" "+text+"\r\n")

	// Process Body,ContentLength,Close,Trailer
	tw, err := newTransferWriter(r)
	if err != nil {
		return err
	}
	err = tw.WriteHeader(w)
	if err != nil {
		return err
	}

	// Rest of header
	err = r.Headers.WriteSubset(w, respExcludeHeader)
	if err != nil {
		return err
	}

	// End-of-header
	io.WriteString(w, "\r\n")

	// Write body and trailer
	err = tw.WriteBody(w)
	if err != nil {
		return err
	}

	// Success
	return nil
}
<commit_msg>add alt_svc and alternate_protocol as known headers<commit_after>// Copyright 2009 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

// HTTP Response reading and parsing.

package http

import (
	"bufio"
	"encoding/json"
	"errors"
	"io"
	"net/textproto"
	"net/url"
	"strconv"
	"strings"
)

var respExcludeHeader = map[string]bool{
	"Trailer": true,
}

var knownHeaders = map[string]bool{
	"access_control_allow_origin": true,
	"accept_patch":                true,
	"accept_ranges":               true,
	"age":                         true,
	"allow":                       true,
	"alt_svc":                     true,
	"alternate_protocol":          true,
	"cache_control":               true,
	"connection":                  true,
	"content_disposition":         true,
	"content_encoding":            true,
	"content_language":            true,
	"content_length":              true,
	"content_location":            true,
	"content_md5":                 true,
	"content_range":               true,
	"content_type":                true,
	"expires":                     true,
	"last_modified":               true,
	"link":                        true,
	"location":                    true,
	"p3p":                         true,
	"pragma":                      true,
	"proxy_agent":                 true,
	"proxy_authenticate":          true,
	"public_key_pins":             true,
	"refresh":                     true,
	"retry_after":                 true,
	"server":                      true,
	"set_cookie":                  true,
	"status":                      true,
	"strict_transport_security":   true,
	"trailer":                     true,
	"transfer_encoding":           true,
	"upgrade":                     true,
	"vary":                        true,
	"via":                         true,
	"warning":                     true,
	"www_authenticate":            true,
	"x_frame_options":             true,
	"x_xss_protection":            true,
	"content_security_policy":     true,
	"x_content_security_policy":   true,
	"x_webkit_csp":                true,
	"x_content_type_options":      true,
	"x_powered_by":                true,
	"x_ua_compatible":             true,
	"x_content_duration":          true,
	"x_real_ip":                   true,
	"x_forwarded_for":             true,
}

// Response represents the response from an HTTP request.
//
type Response struct {
	Status     string   `json:"status_line,omitempty"` // e.g. "200 OK"
	StatusCode int      `json:"status_code,omitempty"` // e.g. 200
	Protocol   Protocol `json:"protocol, omitempty"`

	// Header maps header keys to values.  If the response had multiple
	// headers with the same key, they will be concatenated, with comma
	// delimiters.  (Section 4.2 of RFC 2616 requires that multiple headers
	// be semantically equivalent to a comma-delimited sequence.) Values
	// duplicated by other fields in this struct (e.g., ContentLength) are
	// omitted from Header.
	//
	// Keys in the map are canonicalized (see CanonicalHeaderKey).
	Headers Header `json:"headers,omitempty"`

	// Body represents the response body.
	//
	// The http Client and Transport guarantee that Body is always
	// non-nil, even on responses without a body or responses with
	// a zero-lengthed body.
	Body       io.ReadCloser `json:"-"`
	BodyText   string        `json:"body,omitempty"`
	BodySHA256 []byte        `json:"body_sha256,omitempty"`

	// ContentLength records the length of the associated content.  The
	// value -1 indicates that the length is unknown.  Unless RequestMethod
	// is "HEAD", values >= 0 indicate that the given number of bytes may
	// be read from Body.
	ContentLength int64 `json:"content_length,omitempty"`

	// Contains transfer encodings from outer-most to inner-most. Value is
	// nil, means that "identity" encoding is used.
	TransferEncoding []string `json:"transfer_encoding,omitempty"`

	// Close records whether the header directed that the connection be
	// closed after reading Body.  The value is advice for clients: neither
	// ReadResponse nor Response.Write ever closes a connection.
	Close bool `json:"-"`

	// Trailer maps trailer keys to values, in the same
	// format as the header.
	Trailers Header `json:"trailers,omitempty"`

	// The Request that was sent to obtain this Response.
	// Request's Body is nil (having already been consumed).
	// This is only populated for Client requests.
	Request *Request `json:"request,omitempty"`
}

// Cookies parses and returns the cookies set in the Set-Cookie headers.
func (r *Response) Cookies() []*Cookie {
	return readSetCookies(r.Headers)
}

var ErrNoLocation = errors.New("http: no Location header in response")

// Location returns the URL of the response's "Location" header,
// if present.  Relative redirects are resolved relative to
// the Response's Request.  ErrNoLocation is returned if no
// Location header is present.
func (r *Response) Location() (*url.URL, error) {
	lv := r.Headers.Get("Location")
	if lv == "" {
		return nil, ErrNoLocation
	}
	if r.Request != nil && r.Request.URL != nil {
		return r.Request.URL.Parse(lv)
	}
	return url.Parse(lv)
}

// ReadResponse reads and returns an HTTP response from r.  The
// req parameter specifies the Request that corresponds to
// this Response.  Clients must call resp.Body.Close when finished
// reading resp.Body.  After that call, clients can inspect
// resp.Trailer to find key/value pairs included in the response
// trailer.
func ReadResponse(r *bufio.Reader, req *Request) (resp *Response, err error) {

	tp := textproto.NewReader(r)
	resp = new(Response)

	resp.Request = req
	resp.Request.Method = strings.ToUpper(resp.Request.Method)

	// Parse the first line of the response.
	line, err := tp.ReadLine()
	if err != nil {
		if err == io.EOF {
			err = io.ErrUnexpectedEOF
		}
		return nil, err
	}
	f := strings.SplitN(line, " ", 3)
	if len(f) < 2 {
		return nil, &badStringError{"malformed HTTP response", line}
	}
	reasonPhrase := ""
	if len(f) > 2 {
		reasonPhrase = f[2]
	}
	resp.Status = f[1] + " " + reasonPhrase
	resp.StatusCode, err = strconv.Atoi(f[1])
	if err != nil {
		return nil, &badStringError{"malformed HTTP status code", f[1]}
	}

	resp.Protocol = *(new(Protocol))
	resp.Protocol.Name = f[0]
	var ok bool
	if resp.Protocol.Major, resp.Protocol.Minor, ok = ParseHTTPVersion(resp.Protocol.Name); !ok {
		return nil, &badStringError{"malformed HTTP version", resp.Protocol.Name}
	}

	// Parse the response headers.
	mimeHeader, err := tp.ReadMIMEHeader()
	if err != nil {
		return nil, err
	}

	resp.Headers = Header(mimeHeader)

	//remove unknown headers
	filterHeaders(resp.Headers)

	fixPragmaCacheControl(resp.Headers)

	err = readTransfer(resp, r)
	if err != nil {
		return nil, err
	}

	return resp, nil
}

// RFC2616: Should treat
//	Pragma: no-cache
// like
//	Cache-Control: no-cache
func fixPragmaCacheControl(header Header) {
	if hp, ok := header["Pragma"]; ok && len(hp) > 0 && hp[0] == "no-cache" {
		if _, presentcc := header["Cache-Control"]; !presentcc {
			header["Cache-Control"] = []string{"no-cache"}
		}
	}
}

func filterHeaders(h Header) {
	var unknownHeaders []UnknownHeader
	for header, values := range h {
		if _, ok := knownHeaders[FormatHeaderName(header)]; !ok {
			unk := UnknownHeader{
				Key:    FormatHeaderName(header),
				Values: values,
			}
			unknownHeaders = append(unknownHeaders, unk)
			h.Del(header)
		}
	}
	if len(unknownHeaders) > 0 {
		if unknownHeaderStr, err := json.Marshal(unknownHeaders); err == nil {
			h["Unknown"] = []string{string(unknownHeaderStr)}
		}
	}
}

// ProtoAtLeast returns whether the HTTP protocol used
// in the response is at least major.minor.
func (r *Response) ProtoAtLeast(major, minor int) bool {
	return r.Protocol.Major > major ||
		r.Protocol.Major == major && r.Protocol.Minor >= minor
}

// Writes the response (header, body and trailer) in wire format. This method
// consults the following fields of the response:
//
//  StatusCode
//  ProtoMajor
//  ProtoMinor
//  RequestMethod
//  TransferEncoding
//  Trailer
//  Body
//  ContentLength
//  Header, values for non-canonical keys will have unpredictable behavior
//
func (r *Response) Write(w io.Writer) error {

	// RequestMethod should be upper-case
	if r.Request != nil {
		r.Request.Method = strings.ToUpper(r.Request.Method)
	}

	// Status line
	text := r.Status
	if text == "" {
		var ok bool
		text, ok = statusText[r.StatusCode]
		if !ok {
			text = "status code " + strconv.Itoa(r.StatusCode)
		}
	}
	io.WriteString(w, "HTTP/"+strconv.Itoa(r.Protocol.Major)+".")
	io.WriteString(w, strconv.Itoa(r.Protocol.Minor)+" ")
	io.WriteString(w, strconv.Itoa(r.StatusCode)+" "+text+"\r\n")

	// Process Body,ContentLength,Close,Trailer
	tw, err := newTransferWriter(r)
	if err != nil {
		return err
	}
	err = tw.WriteHeader(w)
	if err != nil {
		return err
	}

	// Rest of header
	err = r.Headers.WriteSubset(w, respExcludeHeader)
	if err != nil {
		return err
	}

	// End-of-header
	io.WriteString(w, "\r\n")

	// Write body and trailer
	err = tw.WriteBody(w)
	if err != nil {
		return err
	}

	// Success
	return nil
}
<|endoftext|>


<commit_before>package aws

import (
	"bytes"
	"container/list"
	"fmt"
	"io"
	"io/ioutil"
	"net/http"
	"net/http/httputil"
	"net/url"
	"reflect"
	"regexp"
	"time"
)

const SDKName = "aws-sdk-go"
const SDKVersion = "0.5.0"

func UserAgentHandler(r *Request) {
	r.HTTPRequest.Header.Set("User-Agent", SDKName+"/"+SDKVersion)
}

func SendHandler(r *Request) {
	r.HTTPResponse, r.Error = r.Service.Config.HTTPClient.Do(r.HTTPRequest)
}

type Operation struct {
	*OperationBindings
	Name       string
	HTTPMethod string
	HTTPPath   string
}

type OperationBindings struct {
	InPayload   string
	OutPayload  string
	Required    []string
	QueryParams []string
	URIParams   []string
	InHeaders   []string
	OutHeaders  []string
}

type Service struct {
	Config       *Config
	Handlers     Handlers
	ManualSend   bool
	ServiceName  string
	APIVersion   string
	Endpoint     string
	JSONVersion  string
	TargetPrefix string
}

var schemeRE = regexp.MustCompile("^([^:]+)://")

func (s *Service) Initialize() {
	if s.Config.HTTPClient == nil {
		s.Config.HTTPClient = http.DefaultClient
	}

	s.Handlers.Build.PushBack(UserAgentHandler)
	s.Handlers.Sign.PushBack(BuildContentLength)
	s.Handlers.Send.PushBack(SendHandler)
	s.AddDebugHandlers()
	s.buildEndpoint()
}

type Request struct {
	*Service
	Handlers     Handlers
	Time         time.Time
	ExpireTime   time.Duration
	Operation    *Operation
	HTTPRequest  *http.Request
	HTTPResponse *http.Response
	Body         io.ReadSeeker
	Params       interface{}
	Error        error
	Data         interface{}
	RequestID    string

	built bool
}

func NewRequest(service *Service, operation *Operation, params interface{}, data interface{}) *Request {
	method := operation.HTTPMethod
	if method == "" {
		method = "POST"
	}
	httpReq, _ := http.NewRequest(method, "", nil)
	httpReq.URL, _ = url.Parse(service.Endpoint + operation.HTTPPath)

	r := &Request{
		Service:     service,
		Handlers:    service.Handlers.copy(),
		Time:        time.Now(),
		ExpireTime:  300,
		Operation:   operation,
		HTTPRequest: httpReq,
		Body:        nil,
		Params:      params,
		Error:       nil,
		Data:        data,
	}

	return r
}

func (s *Service) buildEndpoint() {
	if s.Config.Endpoint != "" {
		s.Endpoint = s.Config.Endpoint
	} else {
		s.Endpoint = s.endpointForRegion()
	}

	if !schemeRE.MatchString(s.Endpoint) {
		scheme := "https"
		if s.Config.DisableSSL {
			scheme = "http"
		}
		s.Endpoint = scheme + "://" + s.Endpoint
	}
}

func (s *Service) AddDebugHandlers() {
	if s.Config.LogLevel == 0 {
		return
	}

	s.Handlers.Sign.PushBack(func(r *Request) {
		dumpedBody, _ := httputil.DumpRequest(r.HTTPRequest, true)

		fmt.Printf("=> [%s] %s.%s(%+v)\n", r.Time,
			r.Service.ServiceName, r.Operation.Name, r.Params)
		fmt.Printf("---[ REQUEST  ]--------------------------------------\n")
		fmt.Printf("%s\n", string(dumpedBody))
		fmt.Printf("-----------------------------------------------------\n\n")
	})
	s.Handlers.Send.PushBack(func(r *Request) {
		defer r.HTTPResponse.Body.Close()
		dumpedBody, _ := httputil.DumpResponse(r.HTTPResponse, true)

		fmt.Printf("---[ RESPONSE ]--------------------------------------\n")
		fmt.Printf("%s\n", string(dumpedBody))
		fmt.Printf("-----------------------------------------------------\n\n")
	})
}

func (r Request) ParamsFilled() bool {
	return reflect.ValueOf(r.Params).Elem().IsValid()
}

func (r Request) DataFilled() bool {
	return reflect.ValueOf(r.Data).Elem().IsValid()
}

func (r *Request) SetBufferBody(buf []byte) {
	r.SetReaderBody(bytes.NewReader(buf))
}

func (r *Request) SetReaderBody(reader io.ReadSeeker) {
	r.HTTPRequest.Body = ioutil.NopCloser(reader)
	r.Body = reader
}

func (r *Request) Presign(expireTime time.Duration) (string, error) {
	r.ExpireTime = expireTime
	r.Sign()
	if r.Error != nil {
		return "", r.Error
	} else {
		return r.HTTPRequest.URL.String(), nil
	}
}

func (r *Request) Build() error {
	if !r.built {
		r.Error = nil
		r.Handlers.Build.Run(r)
		r.built = true
	}

	return r.Error
}

func (r *Request) Sign() error {
	r.Build()
	if r.Error != nil {
		return r.Error
	}

	r.Handlers.Sign.Run(r)
	return r.Error
}

func (r *Request) Send() error {
	r.Sign()
	if r.Error != nil {
		return r.Error
	}

	r.Handlers.Send.Run(r)
	if r.Error != nil {
		return r.Error
	}

	r.Handlers.ValidateResponse.Run(r)
	if r.Error != nil {
		return r.Error
	}

	r.Handlers.Unmarshal.Run(r)
	if r.Error != nil {
		return r.Error
	}

	return nil
}

type Handlers struct {
	Build            HandlerList
	Sign             HandlerList
	Send             HandlerList
	ValidateResponse HandlerList
	Unmarshal        HandlerList
}

func (h Handlers) copy() Handlers {
	return Handlers{
		Build:            h.Build.copy(),
		Sign:             h.Sign.copy(),
		Send:             h.Send.copy(),
		ValidateResponse: h.ValidateResponse.copy(),
		Unmarshal:        h.Unmarshal.copy(),
	}
}

type HandlerList struct {
	list.List
}

func (l HandlerList) copy() HandlerList {
	var n HandlerList
	for e := l.Front(); e != nil; e = e.Next() {
		h := e.Value.(func(*Request))
		n.PushBack(h)
	}
	return n
}

func (l *HandlerList) Run(r *Request) {
	for e := l.Front(); e != nil; e = e.Next() {
		h := e.Value.(func(*Request))
		h(r)
	}
}
<commit_msg>Add Handlers.Clear function<commit_after>package aws

import (
	"bytes"
	"container/list"
	"fmt"
	"io"
	"io/ioutil"
	"net/http"
	"net/http/httputil"
	"net/url"
	"reflect"
	"regexp"
	"time"
)

const SDKName = "aws-sdk-go"
const SDKVersion = "0.5.0"

func UserAgentHandler(r *Request) {
	r.HTTPRequest.Header.Set("User-Agent", SDKName+"/"+SDKVersion)
}

func SendHandler(r *Request) {
	r.HTTPResponse, r.Error = r.Service.Config.HTTPClient.Do(r.HTTPRequest)
}

type Operation struct {
	*OperationBindings
	Name       string
	HTTPMethod string
	HTTPPath   string
}

type OperationBindings struct {
	InPayload   string
	OutPayload  string
	Required    []string
	QueryParams []string
	URIParams   []string
	InHeaders   []string
	OutHeaders  []string
}

type Service struct {
	Config       *Config
	Handlers     Handlers
	ManualSend   bool
	ServiceName  string
	APIVersion   string
	Endpoint     string
	JSONVersion  string
	TargetPrefix string
}

var schemeRE = regexp.MustCompile("^([^:]+)://")

func (s *Service) Initialize() {
	if s.Config.HTTPClient == nil {
		s.Config.HTTPClient = http.DefaultClient
	}

	s.Handlers.Build.PushBack(UserAgentHandler)
	s.Handlers.Sign.PushBack(BuildContentLength)
	s.Handlers.Send.PushBack(SendHandler)
	s.AddDebugHandlers()
	s.buildEndpoint()
}

type Request struct {
	*Service
	Handlers     Handlers
	Time         time.Time
	ExpireTime   time.Duration
	Operation    *Operation
	HTTPRequest  *http.Request
	HTTPResponse *http.Response
	Body         io.ReadSeeker
	Params       interface{}
	Error        error
	Data         interface{}
	RequestID    string

	built bool
}

func NewRequest(service *Service, operation *Operation, params interface{}, data interface{}) *Request {
	method := operation.HTTPMethod
	if method == "" {
		method = "POST"
	}
	httpReq, _ := http.NewRequest(method, "", nil)
	httpReq.URL, _ = url.Parse(service.Endpoint + operation.HTTPPath)

	r := &Request{
		Service:     service,
		Handlers:    service.Handlers.copy(),
		Time:        time.Now(),
		ExpireTime:  300,
		Operation:   operation,
		HTTPRequest: httpReq,
		Body:        nil,
		Params:      params,
		Error:       nil,
		Data:        data,
	}

	return r
}

func (s *Service) buildEndpoint() {
	if s.Config.Endpoint != "" {
		s.Endpoint = s.Config.Endpoint
	} else {
		s.Endpoint = s.endpointForRegion()
	}

	if !schemeRE.MatchString(s.Endpoint) {
		scheme := "https"
		if s.Config.DisableSSL {
			scheme = "http"
		}
		s.Endpoint = scheme + "://" + s.Endpoint
	}
}

func (s *Service) AddDebugHandlers() {
	if s.Config.LogLevel == 0 {
		return
	}

	s.Handlers.Sign.PushBack(func(r *Request) {
		dumpedBody, _ := httputil.DumpRequest(r.HTTPRequest, true)

		fmt.Printf("=> [%s] %s.%s(%+v)\n", r.Time,
			r.Service.ServiceName, r.Operation.Name, r.Params)
		fmt.Printf("---[ REQUEST  ]--------------------------------------\n")
		fmt.Printf("%s\n", string(dumpedBody))
		fmt.Printf("-----------------------------------------------------\n\n")
	})
	s.Handlers.Send.PushBack(func(r *Request) {
		defer r.HTTPResponse.Body.Close()
		dumpedBody, _ := httputil.DumpResponse(r.HTTPResponse, true)

		fmt.Printf("---[ RESPONSE ]--------------------------------------\n")
		fmt.Printf("%s\n", string(dumpedBody))
		fmt.Printf("-----------------------------------------------------\n\n")
	})
}

func (r Request) ParamsFilled() bool {
	return reflect.ValueOf(r.Params).Elem().IsValid()
}

func (r Request) DataFilled() bool {
	return reflect.ValueOf(r.Data).Elem().IsValid()
}

func (r *Request) SetBufferBody(buf []byte) {
	r.SetReaderBody(bytes.NewReader(buf))
}

func (r *Request) SetReaderBody(reader io.ReadSeeker) {
	r.HTTPRequest.Body = ioutil.NopCloser(reader)
	r.Body = reader
}

func (r *Request) Presign(expireTime time.Duration) (string, error) {
	r.ExpireTime = expireTime
	r.Sign()
	if r.Error != nil {
		return "", r.Error
	} else {
		return r.HTTPRequest.URL.String(), nil
	}
}

func (r *Request) Build() error {
	if !r.built {
		r.Error = nil
		r.Handlers.Build.Run(r)
		r.built = true
	}

	return r.Error
}

func (r *Request) Sign() error {
	r.Build()
	if r.Error != nil {
		return r.Error
	}

	r.Handlers.Sign.Run(r)
	return r.Error
}

func (r *Request) Send() error {
	r.Sign()
	if r.Error != nil {
		return r.Error
	}

	r.Handlers.Send.Run(r)
	if r.Error != nil {
		return r.Error
	}

	r.Handlers.ValidateResponse.Run(r)
	if r.Error != nil {
		return r.Error
	}

	r.Handlers.Unmarshal.Run(r)
	if r.Error != nil {
		return r.Error
	}

	return nil
}

type Handlers struct {
	Build            HandlerList
	Sign             HandlerList
	Send             HandlerList
	ValidateResponse HandlerList
	Unmarshal        HandlerList
}

func (h Handlers) copy() Handlers {
	return Handlers{
		Build:            h.Build.copy(),
		Sign:             h.Sign.copy(),
		Send:             h.Send.copy(),
		ValidateResponse: h.ValidateResponse.copy(),
		Unmarshal:        h.Unmarshal.copy(),
	}
}

// Clear removes callback functions for all handlers
func (h *Handlers) Clear() {
	h.Build.Init()
	h.Send.Init()
	h.Sign.Init()
	h.Unmarshal.Init()
	h.ValidateResponse.Init()
}

type HandlerList struct {
	list.List
}

func (l HandlerList) copy() HandlerList {
	var n HandlerList
	for e := l.Front(); e != nil; e = e.Next() {
		h := e.Value.(func(*Request))
		n.PushBack(h)
	}
	return n
}

func (l *HandlerList) Run(r *Request) {
	for e := l.Front(); e != nil; e = e.Next() {
		h := e.Value.(func(*Request))
		h(r)
	}
}
<|endoftext|>


<commit_before>package pooly

import (
	"errors"
	"time"
)

// Pooly default constants.
const (
	DefaultMaxConns   = 10
	DefaultMaxRetries = 3
	DefaultRetryDelay = 10 * time.Millisecond

	DefaultPrespawnConns        = 1
	DefaultMaxAttempts          = 3
	DefaultCloseDeadline        = 30 * time.Second
	DefaultDecayDuration        = 1 * time.Minute
	DefaultMemoizeScoreDuration = 100 * time.Millisecond
)

// Pooly global errors.
var (
	ErrInvalidArg      = errors.New("pooly: invalid argument")
	ErrPoolClosed      = errors.New("pooly: pool is closed")
	ErrOpTimeout       = errors.New("pooly: operation timed out")
	ErrNoHostAvailable = errors.New("pooly: no host available")
)
<commit_msg>Bump MaxConns to 30 by default<commit_after>package pooly

import (
	"errors"
	"time"
)

// Pooly default constants.
const (
	DefaultMaxConns   = 30
	DefaultMaxRetries = 3
	DefaultRetryDelay = 10 * time.Millisecond

	DefaultPrespawnConns        = 1
	DefaultMaxAttempts          = 3
	DefaultCloseDeadline        = 30 * time.Second
	DefaultDecayDuration        = 1 * time.Minute
	DefaultMemoizeScoreDuration = 100 * time.Millisecond
)

// Pooly global errors.
var (
	ErrInvalidArg      = errors.New("pooly: invalid argument")
	ErrPoolClosed      = errors.New("pooly: pool is closed")
	ErrOpTimeout       = errors.New("pooly: operation timed out")
	ErrNoHostAvailable = errors.New("pooly: no host available")
)
<|endoftext|>


<commit_before>package main

import (
	"flag"
	"fmt"
	"log"
	"net/http"
	_ "net/http/pprof"
	"os"
	"path/filepath"
)

var (
	g_is_server = flag.Bool("s", false, "run a server instead of a client")
	g_format    = flag.String("f", "nice", "output format (vim | emacs | nice | csv | json)")
	g_input     = flag.String("in", "", "use this file instead of stdin input")
	g_sock      = create_sock_flag("sock", "socket type (unix | tcp)")
	g_addr      = flag.String("addr", "127.0.0.1:37373", "address for tcp socket")
	g_debug     = flag.Bool("debug", false, "enable server-side debug mode")
	g_profile   = flag.Int("profile", 0, "port on which to expose profiling information for pprof; 0 to disable profiling")
)

func get_socket_filename() string {
	user := os.Getenv("USER")
	if user == "" {
		user = "all"
	}
	return filepath.Join(os.TempDir(), fmt.Sprintf("gocode-daemon.%s", user))
}

func show_usage() {
	fmt.Fprintf(os.Stderr,
		"Usage: %s [-s] [-f=<format>] [-in=<path>] [-sock=<type>] [-addr=<addr>]\n"+
			"       <command> [<args>]\n\n",
		os.Args[0])
	fmt.Fprintf(os.Stderr,
		"Flags:\n")
	flag.PrintDefaults()
	fmt.Fprintf(os.Stderr,
		"\nCommands:\n"+
			"  autocomplete [<path>] <offset>     main autocompletion command\n"+
			"  close                              close the gocode daemon\n"+
			"  status                             gocode daemon status report\n"+
			"  drop-cache                         drop gocode daemon's cache\n"+
			"  set [<name> [<value>]]             list or set config options\n")
}

func main() {
	flag.Usage = show_usage
	flag.Parse()

	var retval int
	if *g_is_server {
		go func() {
			if *g_profile <= 0 {
				return
			}
			addr := fmt.Sprintf("localhost:%d", *g_profile)
			// Use the following commands to profile the binary:
			// go tool pprof http://localhost:6060/debug/pprof/profile   # 30-second CPU profile
			// go tool pprof http://localhost:6060/debug/pprof/heap      # heap profile
			// go tool pprof http://localhost:6060/debug/pprof/block     # goroutine blocking profile
			// See http://blog.golang.org/profiling-go-programs for more info.
			log.Printf("enabling  profiler on %s", addr)
			log.Print(http.ListenAndServe(addr, nil))
		}()
		retval = do_server()
	} else {
		retval = do_client()
	}
	os.Exit(retval)
}
<commit_msg>Add "csv-with-package" format to command line help output.<commit_after>package main

import (
	"flag"
	"fmt"
	"log"
	"net/http"
	_ "net/http/pprof"
	"os"
	"path/filepath"
)

var (
	g_is_server = flag.Bool("s", false, "run a server instead of a client")
	g_format    = flag.String("f", "nice", "output format (vim | emacs | nice | csv | csv-with-package | json)")
	g_input     = flag.String("in", "", "use this file instead of stdin input")
	g_sock      = create_sock_flag("sock", "socket type (unix | tcp)")
	g_addr      = flag.String("addr", "127.0.0.1:37373", "address for tcp socket")
	g_debug     = flag.Bool("debug", false, "enable server-side debug mode")
	g_profile   = flag.Int("profile", 0, "port on which to expose profiling information for pprof; 0 to disable profiling")
)

func get_socket_filename() string {
	user := os.Getenv("USER")
	if user == "" {
		user = "all"
	}
	return filepath.Join(os.TempDir(), fmt.Sprintf("gocode-daemon.%s", user))
}

func show_usage() {
	fmt.Fprintf(os.Stderr,
		"Usage: %s [-s] [-f=<format>] [-in=<path>] [-sock=<type>] [-addr=<addr>]\n"+
			"       <command> [<args>]\n\n",
		os.Args[0])
	fmt.Fprintf(os.Stderr,
		"Flags:\n")
	flag.PrintDefaults()
	fmt.Fprintf(os.Stderr,
		"\nCommands:\n"+
			"  autocomplete [<path>] <offset>     main autocompletion command\n"+
			"  close                              close the gocode daemon\n"+
			"  status                             gocode daemon status report\n"+
			"  drop-cache                         drop gocode daemon's cache\n"+
			"  set [<name> [<value>]]             list or set config options\n")
}

func main() {
	flag.Usage = show_usage
	flag.Parse()

	var retval int
	if *g_is_server {
		go func() {
			if *g_profile <= 0 {
				return
			}
			addr := fmt.Sprintf("localhost:%d", *g_profile)
			// Use the following commands to profile the binary:
			// go tool pprof http://localhost:6060/debug/pprof/profile   # 30-second CPU profile
			// go tool pprof http://localhost:6060/debug/pprof/heap      # heap profile
			// go tool pprof http://localhost:6060/debug/pprof/block     # goroutine blocking profile
			// See http://blog.golang.org/profiling-go-programs for more info.
			log.Printf("enabling  profiler on %s", addr)
			log.Print(http.ListenAndServe(addr, nil))
		}()
		retval = do_server()
	} else {
		retval = do_client()
	}
	os.Exit(retval)
}
<|endoftext|>


<commit_before>package main

import (
	"fmt"
	"go/parser"
	"strings"
	"io/ioutil"
	"os"
)

// TODO: probably change hand-written string literals processing to a
// "scanner"-based one

func skipSpaces(i int, s string) int {
	for i < len(s) && (s[i] == ' ' || s[i] == '\t') {
		i++
	}
	return i
}

func skipToSpace(i int, s string) int {
	for i < len(s) && s[i] != ' ' && s[i] != '\t' {
		i++
	}
	return i
}

// convert package name to a nice ident, e.g.: "go/ast" -> "ast"
func identifyPackage(s string) string {
	i := len(s)-1

	// 'i > 0' is correct here, because we should never see '/' at the
	// beginning of the name anyway
	for ; i > 0; i-- {
		if s[i] == '/' {
			break
		}
	}
	if s[i] == '/' {
		return s[i+1:]
	}
	return s
}

func extractPackage(i int, s string) (string, string) {
	pkg := ""

	b := i // first '"'
	i++

	for i < len(s) && s[i] != '"' {
		i++
	}

	if i == len(s) {
		return s, pkg
	}

	e := i // second '"'
	if b+1 != e {
		// wow, we actually have something here
		pkg = s[b+1:e]
	}

	i += 2 // skip to a first symbol after dot
	s = s[0:b] + s[i:] // strip package clause completely

	return s, pkg
}

// returns modified 's' with package stripped from the method and the package name
func extractPackageFromMethod(i int, s string) (string, string) {
	pkg := ""
	for {
		for i < len(s) && s[i] != ')' && s[i] != '"' {
			i++
		}

		if s[i] == ')' || i == len(s) {
			return s, pkg
		}

		b := i // first '"'
		i++

		for i < len(s) && s[i] != ')' && s[i] != '"' {
			i++
		}

		if s[i] == ')' || i == len(s) {
			return s, pkg
		}

		e := i // second '"'
		if b+1 != e {
			// wow, we actually have something here
			pkg = s[b+1:e]
		}

		i += 2 // skip to a first symbol after dot
		s = s[0:b] + s[i:] // strip package clause completely

		i = b
	}
	panic("unreachable")
	return "", ""
}

func expandPackages(s string) string {
	i := 0
	for {
		pkg := ""
		for i < len(s) && s[i] != '"' && s[i] != '=' {
			i++
		}

		if i == len(s) || s[i] == '=' {
			return s
		}

		b := i // first '"'
		i++

		for i < len(s) && !(s[i] == '"' && s[i-1] != '\\') && s[i] != '=' {
			i++
		}

		if i == len(s) || s[i] == '=' {
			return s
		}

		e := i // second '"'
		if s[b-1] == ':' {
			// special case, struct attribute literal, just remove ':'
			s = s[0:b-1] + s[b:]
			i = e
		} else if b+1 != e {
			// wow, we actually have something here
			pkg = identifyPackage(s[b+1:e])
			i++ // skip to a first symbol after second '"'
			s = s[0:b] + pkg + s[i:] // strip package clause completely
			i = b
		} else {
			i += 2
			s = s[0:b] + s[i:]
			i = b
		}

	}
	panic("unreachable")
	return ""
}

func preprocessConstDecl(s string) string {
	i := strings.Index(s, "=")
	if i == -1 {
		return s
	}

	for i < len(s) && !(s[i] >= '0' && s[i] <= '9') && s[i] != '"' && s[i] != '\'' {
		i++
	}

	if i == len(s) || s[i] == '"' || s[i] == '\'' {
		return s
	}

	// ok, we have a digit!
	b := i
	for i < len(s) && ((s[i] >= '0' && s[i] <= '9') || s[i] == 'p' || s[i] == '-' || s[i] == '+') {
		i++
	}
	e := i

	return s[0:b-1] + "0" + s[e:]
}

// feed one definition line from .a file here
// returns:
// 1. a go/parser parsable string representing one Go declaration
// 2. and a package name this declaration belongs to
func processExport(s string) (string, string) {
	i := 0
	pkg := ""

	// skip to a decl type: (type | func | const | var | import)
	i = skipSpaces(i, s)
	if i == len(s) {
		return "", pkg
	}
	b := i
	i = skipToSpace(i, s)
	if i == len(s) {
		return "", pkg
	}
	e := i

	// skip import and const(TODO) decls, we don't need them
	if s[b:e] == "import" {
		return "", pkg
	} else if s[b:e] == "const" {
		s = preprocessConstDecl(s)
	}
	i++ // skip space after a decl type

	// extract a package this decl belongs to
	switch s[i] {
	case '(':
		s, pkg = extractPackageFromMethod(i, s)
	case '"':
		s, pkg = extractPackage(i, s)
	}

	// make everything parser friendly
	s = strings.Replace(s, "?", "", -1)
	s = expandPackages(s)

	// skip system functions (Init, etc.)
	i = strings.Index(s, "·")
	if i != -1 {
		return "", ""
	}

	return s, pkg
}

func processPackage(filename string) {
	data, err := ioutil.ReadFile(filename)
	if err != nil {
		panic("Failed to open archive file")
	}
	s := string(data)

	i := strings.Index(s, "import\n$$\n")
	if i == -1 {
		panic("Can't find the import section in the archive file")
	}
	s = s[i+len("import\n$$\n"):]
	i = strings.Index(s, "$$\n")
	if i == -1 {
		panic("Can't find the end of the import section in the archive file")
	}
	s = s[0:i] // leave only import section

	i = strings.Index(s, "\n")
	if i == -1 {
		panic("Wrong file")
	}
	pkgname := s[len("package "):i-1]
	fmt.Printf("parsing package '%s'\n", pkgname)
	s = s[i+1:]
	for {
		// for each line
		i := strings.Index(s, "\n")
		if i == -1 {
			break
		}
		decl := strings.TrimSpace(s[0:i])
		if len(decl) == 0 {
			s = s[i+1:]
			continue
		}
		decl2, pkg := processExport(decl)
		if len(decl2) == 0 {
			s = s[i+1:]
			continue
		}

		_, err := parser.ParseDeclList("", decl2 + "\n", nil)
		if err != nil {
			fmt.Printf("!!!!!!!!!!!!!!!! FAILURE !!!!!!!!!!!!!!!!\n%s\n", decl2)
			panic(fmt.Sprintf("%s\n", err.String()))
		} else {
			fmt.Printf("OK")
		}
		min := 80
		if len(decl2) < min {
			min = len(decl2)
		}
		fmt.Printf("\t%s\t%s\n", pkg, decl2[0:min])
		s = s[i+1:]
	}
}

func main() {
	for _, arg := range os.Args[1:] {
		processPackage(arg)
	}
}
<commit_msg>Jiggle code a little bit.<commit_after>package main

import (
	"fmt"
	"go/parser"
	"strings"
	"io/ioutil"
	"os"
)

// TODO: probably change hand-written string literals processing to a
// "scanner"-based one

func skipSpaces(i int, s string) int {
	for i < len(s) && (s[i] == ' ' || s[i] == '\t') {
		i++
	}
	return i
}

func skipToSpace(i int, s string) int {
	for i < len(s) && s[i] != ' ' && s[i] != '\t' {
		i++
	}
	return i
}

// convert package name to a nice ident, e.g.: "go/ast" -> "ast"
func identifyPackage(s string) string {
	i := len(s)-1

	// 'i > 0' is correct here, because we should never see '/' at the
	// beginning of the name anyway
	for ; i > 0; i-- {
		if s[i] == '/' {
			break
		}
	}
	if s[i] == '/' {
		return s[i+1:]
	}
	return s
}

func extractPackage(i int, s string) (string, string) {
	pkg := ""

	b := i // first '"'
	i++

	for i < len(s) && s[i] != '"' {
		i++
	}

	if i == len(s) {
		return s, pkg
	}

	e := i // second '"'
	if b+1 != e {
		// wow, we actually have something here
		pkg = s[b+1:e]
	}

	i += 2 // skip to a first symbol after dot
	s = s[0:b] + s[i:] // strip package clause completely

	return s, pkg
}

// returns modified 's' with package stripped from the method and the package name
func extractPackageFromMethod(i int, s string) (string, string) {
	pkg := ""
	for {
		for i < len(s) && s[i] != ')' && s[i] != '"' {
			i++
		}

		if s[i] == ')' || i == len(s) {
			return s, pkg
		}

		b := i // first '"'
		i++

		for i < len(s) && s[i] != ')' && s[i] != '"' {
			i++
		}

		if s[i] == ')' || i == len(s) {
			return s, pkg
		}

		e := i // second '"'
		if b+1 != e {
			// wow, we actually have something here
			pkg = s[b+1:e]
		}

		i += 2 // skip to a first symbol after dot
		s = s[0:b] + s[i:] // strip package clause completely

		i = b
	}
	panic("unreachable")
	return "", ""
}

func expandPackages(s string) string {
	i := 0
	for {
		pkg := ""
		for i < len(s) && s[i] != '"' && s[i] != '=' {
			i++
		}

		if i == len(s) || s[i] == '=' {
			return s
		}

		b := i // first '"'
		i++

		for i < len(s) && !(s[i] == '"' && s[i-1] != '\\') && s[i] != '=' {
			i++
		}

		if i == len(s) || s[i] == '=' {
			return s
		}

		e := i // second '"'
		if s[b-1] == ':' {
			// special case, struct attribute literal, just remove ':'
			s = s[0:b-1] + s[b:]
			i = e
		} else if b+1 != e {
			// wow, we actually have something here
			pkg = identifyPackage(s[b+1:e])
			i++ // skip to a first symbol after second '"'
			s = s[0:b] + pkg + s[i:] // strip package clause completely
			i = b
		} else {
			i += 2
			s = s[0:b] + s[i:]
			i = b
		}

	}
	panic("unreachable")
	return ""
}

func preprocessConstDecl(s string) string {
	i := strings.Index(s, "=")
	if i == -1 {
		return s
	}

	for i < len(s) && !(s[i] >= '0' && s[i] <= '9') && s[i] != '"' && s[i] != '\'' {
		i++
	}

	if i == len(s) || s[i] == '"' || s[i] == '\'' {
		return s
	}

	// ok, we have a digit!
	b := i
	for i < len(s) && ((s[i] >= '0' && s[i] <= '9') || s[i] == 'p' || s[i] == '-' || s[i] == '+') {
		i++
	}
	e := i

	return s[0:b-1] + "0" + s[e:]
}

// feed one definition line from .a file here
// returns:
// 1. a go/parser parsable string representing one Go declaration
// 2. and a package name this declaration belongs to
func processExport(s string) (string, string) {
	i := 0
	pkg := ""

	// skip to a decl type: (type | func | const | var | import)
	i = skipSpaces(i, s)
	if i == len(s) {
		return "", pkg
	}
	b := i
	i = skipToSpace(i, s)
	if i == len(s) {
		return "", pkg
	}
	e := i

	// skip import and const(TODO) decls, we don't need them
	if s[b:e] == "import" {
		return "", pkg
	} else if s[b:e] == "const" {
		s = preprocessConstDecl(s)
	}
	i++ // skip space after a decl type

	// extract a package this decl belongs to
	switch s[i] {
	case '(':
		s, pkg = extractPackageFromMethod(i, s)
	case '"':
		s, pkg = extractPackage(i, s)
	}

	// make everything parser friendly
	s = strings.Replace(s, "?", "", -1)
	s = expandPackages(s)

	// skip system functions (Init, etc.)
	i = strings.Index(s, "·")
	if i != -1 {
		return "", ""
	}

	return s, pkg
}

func processPackage(filename string) {
	data, err := ioutil.ReadFile(filename)
	if err != nil {
		panic("Failed to open archive file")
	}
	s := string(data)

	i := strings.Index(s, "import\n$$\n")
	if i == -1 {
		panic("Can't find the import section in the archive file")
	}
	s = s[i+len("import\n$$\n"):]
	i = strings.Index(s, "$$\n")
	if i == -1 {
		panic("Can't find the end of the import section in the archive file")
	}
	s = s[0:i] // leave only import section

	i = strings.Index(s, "\n")
	if i == -1 {
		panic("Wrong file")
	}
	pkgname := s[len("package "):i-1]
	fmt.Printf("parsing package '%s'\n", pkgname)
	s = s[i+1:]

	internalPackages := make(map[string]string, 50)
	for {
		// for each line
		i := strings.Index(s, "\n")
		if i == -1 {
			break
		}
		decl := strings.TrimSpace(s[0:i])
		if len(decl) == 0 {
			s = s[i+1:]
			continue
		}
		decl2, pkg := processExport(decl)
		if len(decl2) == 0 {
			s = s[i+1:]
			continue
		}

		if pkg == "" {
			pkg = "__this__"
		}

		if _, ok := internalPackages[pkg]; ok {
			internalPackages[pkg] = internalPackages[pkg] + decl2 + "\n"
		} else {
			internalPackages[pkg] = decl2 + "\n"
		}
		/*decls, err := parser.ParseDeclList("", decl2 + "\n", nil)
		if err != nil {
			fmt.Printf("!!!!!!!!!!!!!!!! FAILURE !!!!!!!!!!!!!!!!\n%s\n", decl2)
			panic(fmt.Sprintf("%s\n", err.String()))
		} else {
			fmt.Printf("OK")
		}
		*/
		s = s[i+1:]

		//fmt.Printf("\t%s\t%s\n", pkg, decl2)
	}

	for key, value := range internalPackages {
		decls, err := parser.ParseDeclList("", value, nil)
		if err != nil {
			fmt.Printf("!!!!!!!!!!!!!!!! FAILURE !!!!!!!!!!!!!!!!\n%s\n", value)
			panic(fmt.Sprintf("%s\n", err.String()))
		} else {
			fmt.Printf("\t%s: \033[32mOK\033[0m (ndecls: %d)\n", key, len(decls))
		}
	}
}

func main() {
	for _, arg := range os.Args[1:] {
		processPackage(arg)
	}
	fmt.Printf("Total number of packages: %d\n", len(os.Args)-1)
}
<|endoftext|>


<commit_before>// Copyright 2013 Frederik Zipp. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

// Gofind searches for Go packages via pkg.go.dev.
//
// Usage:
//      gofind [<flag> ...] <query> ...
//
// Flags
//      -raw   don't apply any formatting if set
package main

import (
	"flag"
	"fmt"
	"go/doc"
	"io"
	"net/http"
	"net/url"
	"os"
	"strconv"
	"strings"

	"github.com/PuerkitoBio/goquery"
)

func usage() {
	_, _ = fmt.Fprintf(os.Stderr, usageDoc)
	os.Exit(2)
}

const usageDoc = `Find Go modules via pkg.go.dev.

Usage:
    gofind [-a] [-raw] query ...

Flags:
    -a     load all search results if set, not just the first 10 results
    -raw   don't apply any formatting if set

Examples:
    gofind logging
    gofind -a logging
    gofind go cloud        # Search for multiple terms
    gofind "go cloud"      # Search for an exact match
    gofind yaml OR json    # Combine searches
`

func main() {
	allFlag := flag.Bool("a", false, "load all search results, not just the first 10 results")
	rawFlag := flag.Bool("raw", false, "don't apply any formatting")

	flag.Usage = usage
	flag.Parse()

	if flag.NArg() == 0 {
		usage()
	}
	args := flag.Args()
	for i, arg := range args {
		if strings.Contains(arg, " ") {
			args[i] = `"` + arg + `"`
		}
	}
	query := strings.Join(args, " ")

	run(query, *allFlag, *rawFlag)
}

func run(query string, all, raw bool) {
	modules, err := search(query, all)
	check(err)
	for _, mod := range modules {
		if raw {
			fmt.Println(mod.moduleName + "\t" + mod.synopsis + "\t" + mod.info)
			continue
		}
		check(mod.writeTo(os.Stdout))
	}
}

func search(query string, all bool) (sr []searchResult, err error) {
	lastPage := 1
	for page := 1; page <= lastPage; page++ {
		var psr []searchResult
		psr, lastPage, err = searchPage(query, page)
		if err != nil {
			return sr, err
		}
		if !all {
			lastPage = 1
		}
		sr = append(sr, psr...)
	}
	return sr, err
}

func searchPage(query string, page int) (sr []searchResult, lastPage int, err error) {
	v := url.Values{}
	v.Set("q", query)
	v.Set("page", strconv.Itoa(page))
	req, err := http.NewRequest("GET", "https://pkg.go.dev/search?"+v.Encode(), nil)
	if err != nil {
		return nil, 0, err
	}
	req.Header.Add("Accept", "text/html")

	client := &http.Client{}
	res, err := client.Do(req)
	if err != nil {
		return nil, 0, err
	}
	defer res.Body.Close()
	if res.StatusCode != 200 {
		return nil, 0, fmt.Errorf("HTTP status code error: %d %sr", res.StatusCode, res.Status)
	}

	htmlDoc, err := goquery.NewDocumentFromReader(res.Body)
	if err != nil {
		return nil, 0, err
	}

	return scrapeSearchResults(htmlDoc)
}

func scrapeSearchResults(htmlDoc *goquery.Document) (sr []searchResult, lastPage int, err error) {
	htmlDoc.Find(".SearchSnippet").Each(func(i int, s *goquery.Selection) {
		moduleName := strings.TrimSpace(s.Find(".SearchSnippet-header").Text())
		synopsis := strings.TrimSpace(s.Find(".SearchSnippet-synopsis").Text())
		info := formatInfo(s.Find(".SearchSnippet-infoLabel").Text())
		sr = append(sr, searchResult{
			moduleName: moduleName,
			synopsis:   synopsis,
			info:       info,
		})
	})
	lastPageStr := strings.TrimSpace(htmlDoc.Find(".Pagination-number").Last().Text())
	if lastPageStr != "" {
		lastPage, err = strconv.Atoi(lastPageStr)
	}
	return sr, lastPage, err
}

func formatInfo(info string) string {
	var parts []string
	for _, p := range strings.Split(info, "|") {
		s := strings.SplitN(p, ":", 2)
		if len(s) > 1 {
			label := strings.TrimSpace(s[0])
			value := strings.TrimSpace(s[1])
			parts = append(parts, label+": "+value)
		}
	}
	return strings.Join(parts, " | ")
}

type searchResult struct {
	moduleName string
	synopsis   string
	info       string
}

const (
	punchCardWidth = 80
	indent         = "    "
)

func (s searchResult) writeTo(w io.Writer) error {
	_, err := fmt.Fprintln(w, s.moduleName)
	if err != nil {
		return err
	}
	if s.synopsis != "" {
		doc.ToText(w, s.synopsis, indent, "", punchCardWidth-2*len(indent))
	}
	_, err = fmt.Fprintln(w)
	_, err = fmt.Fprintln(w, indent+s.info)
	_, err = fmt.Fprintln(w)
	return err
}

func check(err error) {
	if err != nil {
		fail(err)
	}
}

func fail(msg ...interface{}) {
	_, _ = fmt.Fprintln(os.Stderr, msg...)
	os.Exit(1)
}
<commit_msg>remove unneccessary labels to shorten output<commit_after>// Copyright 2013 Frederik Zipp. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

// Gofind searches for Go packages via pkg.go.dev.
//
// Usage:
//      gofind [<flag> ...] <query> ...
//
// Flags
//      -raw   don't apply any formatting if set
package main

import (
	"flag"
	"fmt"
	"go/doc"
	"io"
	"net/http"
	"net/url"
	"os"
	"strconv"
	"strings"

	"github.com/PuerkitoBio/goquery"
)

func usage() {
	_, _ = fmt.Fprintf(os.Stderr, usageDoc)
	os.Exit(2)
}

const usageDoc = `Find Go modules via pkg.go.dev.

Usage:
    gofind [-a] [-raw] query ...

Flags:
    -a     load all search results if set, not just the first 10 results
    -raw   don't apply any formatting if set

Examples:
    gofind logging
    gofind -a logging
    gofind go cloud        # Search for multiple terms
    gofind "go cloud"      # Search for an exact match
    gofind yaml OR json    # Combine searches
`

func main() {
	allFlag := flag.Bool("a", false, "load all search results, not just the first 10 results")
	rawFlag := flag.Bool("raw", false, "don't apply any formatting")

	flag.Usage = usage
	flag.Parse()

	if flag.NArg() == 0 {
		usage()
	}
	args := flag.Args()
	for i, arg := range args {
		if strings.Contains(arg, " ") {
			args[i] = `"` + arg + `"`
		}
	}
	query := strings.Join(args, " ")

	run(query, *allFlag, *rawFlag)
}

func run(query string, all, raw bool) {
	modules, err := search(query, all)
	check(err)
	for _, mod := range modules {
		if raw {
			fmt.Println(mod.moduleName + "\t" + mod.synopsis + "\t" + mod.info)
			continue
		}
		check(mod.writeTo(os.Stdout))
	}
}

func search(query string, all bool) (sr []searchResult, err error) {
	lastPage := 1
	for page := 1; page <= lastPage; page++ {
		var psr []searchResult
		psr, lastPage, err = searchPage(query, page)
		if err != nil {
			return sr, err
		}
		if !all {
			lastPage = 1
		}
		sr = append(sr, psr...)
	}
	return sr, err
}

func searchPage(query string, page int) (sr []searchResult, lastPage int, err error) {
	v := url.Values{}
	v.Set("q", query)
	v.Set("page", strconv.Itoa(page))
	req, err := http.NewRequest("GET", "https://pkg.go.dev/search?"+v.Encode(), nil)
	if err != nil {
		return nil, 0, err
	}
	req.Header.Add("Accept", "text/html")

	client := &http.Client{}
	res, err := client.Do(req)
	if err != nil {
		return nil, 0, err
	}
	defer res.Body.Close()
	if res.StatusCode != 200 {
		return nil, 0, fmt.Errorf("HTTP status code error: %d %sr", res.StatusCode, res.Status)
	}

	htmlDoc, err := goquery.NewDocumentFromReader(res.Body)
	if err != nil {
		return nil, 0, err
	}

	return scrapeSearchResults(htmlDoc)
}

func scrapeSearchResults(htmlDoc *goquery.Document) (sr []searchResult, lastPage int, err error) {
	htmlDoc.Find(".SearchSnippet").Each(func(i int, s *goquery.Selection) {
		moduleName := strings.TrimSpace(s.Find(".SearchSnippet-header").Text())
		synopsis := strings.TrimSpace(s.Find(".SearchSnippet-synopsis").Text())
		info := formatInfo(s.Find(".SearchSnippet-infoLabel").Text())
		sr = append(sr, searchResult{
			moduleName: moduleName,
			synopsis:   synopsis,
			info:       info,
		})
	})
	lastPageStr := strings.TrimSpace(htmlDoc.Find(".Pagination-number").Last().Text())
	if lastPageStr != "" {
		lastPage, err = strconv.Atoi(lastPageStr)
	}
	return sr, lastPage, err
}

func formatInfo(info string) string {
	var parts []string
	for _, p := range strings.Split(info, "|") {
		s := strings.SplitN(p, ":", 2)
		if len(s) > 1 {
			label := strings.TrimSpace(s[0])
			value := strings.TrimSpace(s[1])
			parts = append(parts, label+": "+value)
		}
	}
	return strings.Join(parts, " | ")
}

type searchResult struct {
	moduleName string
	synopsis   string
	info       string
}

const (
	punchCardWidth = 80
	indent         = "    "
)

func (s searchResult) writeTo(w io.Writer) error {
	_, err := fmt.Fprintln(w, s.moduleName)
	if err != nil {
		return err
	}
	if s.synopsis != "" {
		doc.ToText(w, s.synopsis, indent, "", punchCardWidth-2*len(indent))
	}
	s.info = strings.ReplaceAll(s.info, "Version: ", "")
	s.info = strings.ReplaceAll(s.info, "Published: ", "")
	s.info = strings.ReplaceAll(s.info, "License: ", "")
	s.info = strings.ReplaceAll(s.info, "Licenses: ", "")
	_, err = fmt.Fprintf(w, "\n%s%s\n\n", indent, s.info)
	return err
}

func check(err error) {
	if err != nil {
		fail(err)
	}
}

func fail(msg ...interface{}) {
	_, _ = fmt.Fprintln(os.Stderr, msg...)
	os.Exit(1)
}
<|endoftext|>


<commit_before>/*
gogive is a tool to serve vanity import paths for Go programs.

Usage:

	gogive [-a addr] file

gogive will bind to the address specified by addr, or :9625 if
none is specified, and serve HTTP requests issued by the go
tool as described in https://golang.org/cmd/go/#hdr-Remote_import_paths

File must be a utf-8 encoded plain text file containing path
redirect specifications, one per line, of the form

	/path vcs vcsrepo

HTTP GET requests of the form /path?go-get=1 will receive an
html page with the following meta tag:

	<meta name="go-import" content="example.com/path vcs vcsrepo">

Note that gogive will use the value of the Host: header in the HTTP request
in the output HTML. HTTP requests that do not contain the "go-get"
query parameter will be redirected to godoc.org.
*/
package main

import (
	"bufio"
	"flag"
	"fmt"
	"html/template"
	"log"
	"net/http"
	"os"
	"os/signal"
	"strings"
	"syscall"
)

var (
	addr = flag.String("a", ":9625", "Address to listen on")
)

var pageTmpl = template.Must(template.New("HTML").Parse(
	`

		<meta name="go-import" content="{{.Host}}{{.Path}} {{.Vcs}} {{.Url}}">

`))

type Source struct {
	Vcs, Url string
}

type Router map[string]Source

type Server struct {
	config string
	Routes chan Router
}

func (s *Server) ServeHTTP(w http.ResponseWriter, r *http.Request) {
	var match struct {
		Source
		Host string
		Path string
	}
	if r.Method != "GET" {
		http.Error(w, "Method Not Allowed", 405)
		return
	}
	routes := <-s.Routes

	if src, root, ok := routes.findPath(r.URL.Path); !ok {
		http.Error(w, "Not Found", 404)
		return
	} else {
		match.Source = src
		match.Host = r.Host
		match.Path = root
	}
	if r.FormValue("go-get") != "1" {
		// if this request is not coming from the go tool, redirect
		// to godoc.org
		http.Redirect(w, r, "http://godoc.org/"+r.Host+r.URL.Path, http.StatusSeeOther)
		return
	}
	if err := pageTmpl.Execute(w, match); err != nil {
		log.Print(err)
	}
}

func (r Router) findPath(path string) (Source, string, bool) {
	nodes := strings.Split(path, "/")
	for len(nodes) > 0 {
		path := strings.Join(nodes, "/")
		if src, ok := r[path]; ok {
			return src, path, true
		}
		nodes = nodes[:len(nodes)-1]
	}
	return Source{}, "", false
}

func main() {
	log.SetFlags(0)
	log.SetPrefix("")
	flag.Usage = func() {
		log.Printf("Usage: %s [-a addr] config\n", os.Args[0])
		os.Exit(2)
	}
	flag.Parse()

	if len(flag.Args()) != 1 {
		flag.Usage()
	}

	s := NewServer(flag.Arg(0))
	srv := &http.Server{
		Handler: s,
		Addr:    *addr,
	}
	go srv.ListenAndServe()
	log.Print("Listening on ", *addr)
	if err := s.loadConfig(); err != nil {
		log.Fatal(err)
	}
}

func NewServer(config string) *Server {
	return &Server{
		config: config,
		Routes: make(chan Router),
	}
}

// runs in its own goroutine.
func (srv *Server) loadConfig() error {
	r, err := NewRouter(srv.config)
	if err != nil {
		return err
	}
	sig := make(chan os.Signal)
	signal.Notify(sig, syscall.SIGHUP)

	for {
		select {
		case srv.Routes <- r:
		case <-sig:
			if nr, err := NewRouter(srv.config); err != nil {
				log.Print(err)
			} else {
				r = nr
			}
		}
	}
}

func NewRouter(filename string) (Router, error) {
	r := make(Router)

	file, err := os.Open(filename)
	if err != nil {
		return nil, err
	}
	n := 0
	scanner := bufio.NewScanner(file)
	for scanner.Scan() {
		n++
		if strings.HasPrefix(scanner.Text(), "#") {
			continue
		}
		fields := strings.Fields(scanner.Text())
		if len(fields) == 0 {
			continue
		}
		if len(fields) != 3 {
			continue
			log.Printf("%s:%d: (%d) fields (expected 3) in %q",
				filename, n, len(fields), scanner.Text())
		}
		if _, ok := r[fields[0]]; ok {
			return nil, fmt.Errorf("%s:%d: duplicate entry %s", filename, n, fields[0])
		}
		r[fields[0]] = Source{fields[1], fields[2]}
	}
	return r, scanner.Err()
}
<commit_msg>Remove unreached log statement<commit_after>/*
gogive is a tool to serve vanity import paths for Go programs.

Usage:

	gogive [-a addr] file

gogive will bind to the address specified by addr, or :9625 if
none is specified, and serve HTTP requests issued by the go
tool as described in https://golang.org/cmd/go/#hdr-Remote_import_paths

File must be a utf-8 encoded plain text file containing path
redirect specifications, one per line, of the form

	/path vcs vcsrepo

HTTP GET requests of the form /path?go-get=1 will receive an
html page with the following meta tag:

	<meta name="go-import" content="example.com/path vcs vcsrepo">

Note that gogive will use the value of the Host: header in the HTTP request
in the output HTML. HTTP requests that do not contain the "go-get"
query parameter will be redirected to godoc.org.
*/
package main

import (
	"bufio"
	"flag"
	"fmt"
	"html/template"
	"log"
	"net/http"
	"os"
	"os/signal"
	"strings"
	"syscall"
)

var (
	addr = flag.String("a", ":9625", "Address to listen on")
)

var pageTmpl = template.Must(template.New("HTML").Parse(
	`

		<meta name="go-import" content="{{.Host}}{{.Path}} {{.Vcs}} {{.Url}}">

`))

type Source struct {
	Vcs, Url string
}

type Router map[string]Source

type Server struct {
	config string
	Routes chan Router
}

func (s *Server) ServeHTTP(w http.ResponseWriter, r *http.Request) {
	var match struct {
		Source
		Host string
		Path string
	}
	if r.Method != "GET" {
		http.Error(w, "Method Not Allowed", 405)
		return
	}
	routes := <-s.Routes

	if src, root, ok := routes.findPath(r.URL.Path); !ok {
		http.Error(w, "Not Found", 404)
		return
	} else {
		match.Source = src
		match.Host = r.Host
		match.Path = root
	}
	if r.FormValue("go-get") != "1" {
		// if this request is not coming from the go tool, redirect
		// to godoc.org
		http.Redirect(w, r, "http://godoc.org/"+r.Host+r.URL.Path, http.StatusSeeOther)
		return
	}
	if err := pageTmpl.Execute(w, match); err != nil {
		log.Print(err)
	}
}

func (r Router) findPath(path string) (Source, string, bool) {
	nodes := strings.Split(path, "/")
	for len(nodes) > 0 {
		path := strings.Join(nodes, "/")
		if src, ok := r[path]; ok {
			return src, path, true
		}
		nodes = nodes[:len(nodes)-1]
	}
	return Source{}, "", false
}

func main() {
	log.SetFlags(0)
	log.SetPrefix("")
	flag.Usage = func() {
		log.Printf("Usage: %s [-a addr] config\n", os.Args[0])
		os.Exit(2)
	}
	flag.Parse()

	if len(flag.Args()) != 1 {
		flag.Usage()
	}

	s := NewServer(flag.Arg(0))
	srv := &http.Server{
		Handler: s,
		Addr:    *addr,
	}
	go srv.ListenAndServe()
	log.Print("Listening on ", *addr)
	if err := s.loadConfig(); err != nil {
		log.Fatal(err)
	}
}

func NewServer(config string) *Server {
	return &Server{
		config: config,
		Routes: make(chan Router),
	}
}

// runs in its own goroutine.
func (srv *Server) loadConfig() error {
	r, err := NewRouter(srv.config)
	if err != nil {
		return err
	}
	sig := make(chan os.Signal)
	signal.Notify(sig, syscall.SIGHUP)

	for {
		select {
		case srv.Routes <- r:
		case <-sig:
			if nr, err := NewRouter(srv.config); err != nil {
				log.Print(err)
			} else {
				r = nr
			}
		}
	}
}

func NewRouter(filename string) (Router, error) {
	r := make(Router)

	file, err := os.Open(filename)
	if err != nil {
		return nil, err
	}
	n := 0
	scanner := bufio.NewScanner(file)
	for scanner.Scan() {
		n++
		if strings.HasPrefix(scanner.Text(), "#") {
			continue
		}
		fields := strings.Fields(scanner.Text())
		if len(fields) == 0 {
			continue
		}
		if len(fields) != 3 {
			continue
		}
		if _, ok := r[fields[0]]; ok {
			return nil, fmt.Errorf("%s:%d: duplicate entry %s", filename, n, fields[0])
		}
		r[fields[0]] = Source{fields[1], fields[2]}
	}
	return r, scanner.Err()
}
<|endoftext|>