<commit_before>// srclib-docco is a docco-like static documentation generator.
// TODO: write this in a literal style.
package srclib_docco

import (
	"bytes"
	"encoding/json"
	"fmt"
	"io"
	"io/ioutil"
	"log"
	"os"
	"os/exec"
	"path/filepath"
	"sort"
	"strings"

	"github.com/jessevdk/go-flags"
	"github.com/sourcegraph/annotate"
	"github.com/sourcegraph/syntaxhighlight"
)

var CLI = flags.NewNamedParser("src-docco", flags.Default)

var GlobalOpt struct {
	Verbose bool `short:"v" description:"show verbose output"`
}

var vLogger = log.New(os.Stderr, "", 0)

func vLogf(format string, v ...interface{}) {
	if !GlobalOpt.Verbose {
		return
	}
	vLogger.Printf(format, v...)
}

func vLog(v ...interface{}) {
	if !GlobalOpt.Verbose {
		return
	}
	vLogger.Println(v...)
}

func init() {
	CLI.LongDescription = "TODO"
	CLI.AddGroup("Global options", "", &GlobalOpt)

	_, err := CLI.AddCommand("gen",
		"generate documentation",
		"Generate docco-like documentation for a thing",
		&genCmd,
	)
	if err != nil {
		log.Fatal(err)
	}
}

// TODO: add limitations help output.

type GenCmd struct {
	Dir string `long:"dir" description:"The root directory for the project"`
	// TODO: must be relative to Dir.
	SiteDirName string `long:"site-dir" description:"The directory name for the output files" default:"site"`
}

var genCmd GenCmd

// Source units have lots of information associated with them, but we
// only care about the files.
type unit struct {
	Files []string
}

type units []unit

func (us units) collateFiles() []string {
	var fs []string
	for _, u := range us {
		for _, f := range u.Files {
			fs = append(fs, f)
		}
	}
	return fs
}

type failedCmd struct {
	cmd interface{}
	err interface{}
}

// TODO: Is this correct?
func (f failedCmd) Error() string {
	return fmt.Sprintf("command %v failed: %s", f.cmd, f.err)
}

func ensureSrclibExists() error {
	cmd := exec.Command("src", "version")
	stdout, stderr := &bytes.Buffer{}, &bytes.Buffer{}
	cmd.Stdout, cmd.Stderr = stdout, stderr
	if err := cmd.Run(); err != nil {
		return fmt.Errorf(
			"error with srclib: %v, %s, %s",
			err,
			stdout.String(),
			stderr.String(),
		)
	}
	return nil
}

func command(argv []string) (cmd *exec.Cmd, stdout *bytes.Buffer, stderr *bytes.Buffer) {
	if len(argv) == 0 {
		panic("command: argv must have at least one item")
	}
	cmd = exec.Command(argv[0], argv[1:]...)
	stdout, stderr = &bytes.Buffer{}, &bytes.Buffer{}
	cmd.Stdout, cmd.Stderr = stdout, stderr
	return cmd, stdout, stderr
}

func (c *GenCmd) Execute(args []string) error {
	if err := ensureSrclibExists(); err != nil {
		log.Fatal(err)
	}

	// First, we need to get a list of all of the files that we
	// want to generate.
	//
	// We could import sourcegraph.com/sourcegraph/srclib, but I
	// want to demonstrate how to use its command line interface.
	var dir string
	if c.Dir == "" {
		d, err := os.Getwd()
		if err != nil {
			log.Fatal(err)
		}
		dir = d
	} else {
		d, err := filepath.Abs(c.Dir)
		if err != nil {
			log.Fatal(err)
		}
		dir = d
	}
	argv := []string{"src", "api", "units", dir}
	cmd, stdout, stderr := command(argv)
	vLogf("Running %v", argv)
	// TODO: remove failedCmd
	if err := cmd.Run(); err != nil {
		log.Fatal(failedCmd{argv, []interface{}{err, stdout.String(), stderr.String()}})
	}
	if stdout.Len() == 0 {
		log.Fatal(failedCmd{argv, "no output"})
	}
	var us units
	if err := json.Unmarshal(stdout.Bytes(), &us); err != nil {
		log.Fatal(err)
	}
	return genSite(dir, c.SiteDirName, us.collateFiles())
}

func genSite(root, siteName string, files []string) error {
	vLog("Generating Site")
	sitePath := filepath.Join(root, siteName)
	if err := os.MkdirAll(sitePath, 0755); err != nil {
		log.Fatal(err)
	}
	for _, f := range files {
		vLog("Processing", f)
		src, err := ioutil.ReadFile(filepath.Join(root, f))
		if err != nil {
			return err
		}
		htmlFile := "/" + f + ".html"
		as, err := ann(src, root, f, htmlFile)
		if err != nil {
			return err
		}
		vLog("Never get here")
		if err := os.MkdirAll(filepath.Dir(filepath.Join(sitePath, htmlFile)), 0755); err != nil {
			log.Fatal(err)
		}
		w, err := os.Create(filepath.Join(sitePath, htmlFile))
		if err != nil {
			return err
		}
		if err := writeAnns(w, src, as); err != nil {
			return err
		}
	}
	return nil
}

type annotation struct {
	annotate.Annotation
	Comment bool
}

type htmlAnnotator syntaxhighlight.HTMLConfig

func (c htmlAnnotator) class(kind syntaxhighlight.Kind) string {
	switch kind {
	case syntaxhighlight.String:
		return c.String
	case syntaxhighlight.Keyword:
		return c.Keyword
	case syntaxhighlight.Comment:
		return c.Comment
	case syntaxhighlight.Type:
		return c.Type
	case syntaxhighlight.Literal:
		return c.Literal
	case syntaxhighlight.Punctuation:
		return c.Punctuation
	case syntaxhighlight.Plaintext:
		return c.Plaintext
	case syntaxhighlight.Tag:
		return c.Tag
	case syntaxhighlight.HTMLTag:
		return c.HTMLTag
	case syntaxhighlight.HTMLAttrName:
		return c.HTMLAttrName
	case syntaxhighlight.HTMLAttrValue:
		return c.HTMLAttrValue
	case syntaxhighlight.Decimal:
		return c.Decimal
	}
	return ""
}

func (a htmlAnnotator) Annotate(start int, kind syntaxhighlight.Kind, tokText string) (*annotate.Annotation, error) {
	class := a.class(kind)
	if class == "" {
		return nil, nil
	}
	return &annotate.Annotation{
		Start: start,
		End:   start + len(tokText),
		Left:  []byte(class),
		Right: nil,
	}, nil
}

func ann(src []byte, root, file, htmlFile string) ([]annotation, error) {
	vLog("Annotating", file)
	annotations, err := syntaxhighlight.Annotate(src, htmlAnnotator(syntaxhighlight.DefaultHTMLConfig))
	if err != nil {
		return nil, err
	}
	sort.Sort(annotations)
	as := make([]annotation, 0, len(annotations))
	argv := []string{"src", "api", "list", "--file", filepath.Join(root, file)}
	cmd, stdout, stderr := command(argv)
	vLog("Running", argv)
	if err := cmd.Run(); err != nil {
		return nil, failedCmd{argv, []interface{}{err, stdout.String(), stderr.String()}}
	}
	type ref struct {
		DefUnit string
		DefPath string
		Start   uint32
	}
	refs := []ref{}
	if err := json.Unmarshal(stdout.Bytes(), &refs); err != nil {
		log.Fatal(err)
	}
	var refAtIndex int
	refAt := func(start uint32) (r ref, found bool) {
		for refAtIndex < len(refs) {
			if refs[refAtIndex].Start == start {
				refAtIndex++
				return r, true
			} else if refs[refAtIndex].Start < start {
				refAtIndex++
			} else { // refs[refAtIndex].Start > start
				return ref{}, false
			}
		}
		return ref{}, false
	}
	for _, a := range annotations {
		if string(a.Left) == "com" {
			as = append(as, annotation{*a, true})
			continue
		}
		r, found := refAt(uint32(a.Start))
		if !found {
			a.Left = []byte(fmt.Sprintf(`<span class="%s">`, string(a.Left)))
			a.Right = []byte(`</span>`)
			as = append(as, annotation{*a, false})
			continue
		}
		a.Left = []byte(fmt.Sprintf(
			`<span class="%s" id="%s"><a href="%s">`,
			string(a.Left),
			filepath.Join(r.DefUnit, r.DefPath),
			htmlFile,
		))
		a.Right = []byte(`</span></a>`)
		as = append(as, annotation{*a, false})
	}
	return as, nil
}

func writeAnns(w io.Writer, src []byte, as []annotation) error {
	vLog("Writing annotations")
	line := 0
	if _, err := w.Write([]byte(fmt.Sprintf(`<pre><div line=%d>`, line))); err != nil {
		return err
	}
	addDivs := func(src []byte) []byte {
		buf := &bytes.Buffer{}
		buf.Grow(len(src))
		for _, b := range src {
			if b == '\n' {
				line++
				buf.Write([]byte(fmt.Sprintf("
\n
", line)))
				continue
			}
			buf.WriteByte(b)
		}
		return buf.Bytes()
	}
	defer func() {
		w.Write([]byte("
\n
"))
	}()
	for i := 0; i < len(src); {
		log.Println(i)
		if len(as) == 0 {
			_, err := w.Write(addDivs(src[i:]))
			return err
		}
		a := as[0]
		if i > a.Start {
			log.Fatal("writeAnns: illegal state: i > a.Start")
		}
		if i < a.Start {
			if _, err := w.Write(addDivs(src[i : i+1])); err != nil {
				return err
			}
			i++
			continue
		}
		// i == a.Start
		for _, b := range src[a.Start:a.End] {
			if b == '\n' {
				log.Fatal(`writeAnns: illegal state: \n in annotation`)
			}
		}
		if _, err := w.Write([]byte(strings.Join([]string{
			string(a.Left),
			string(src[a.Start:a.End]),
			string(a.Right),
		}, ""))); err != nil {
			return err
		}
		i = a.End
		as = as[1:]
	}
	return nil
}

func Main() error {
	log.SetFlags(log.Lshortfile)
	_, err := CLI.Parse()
	return err
}
<commit_msg>remove extra log<commit_after>// srclib-docco is a docco-like static documentation generator.
// TODO: write this in a literal style.
package srclib_docco

import (
	"bytes"
	"encoding/json"
	"fmt"
	"io"
	"io/ioutil"
	"log"
	"os"
	"os/exec"
	"path/filepath"
	"sort"
	"strings"

	"github.com/jessevdk/go-flags"
	"github.com/sourcegraph/annotate"
	"github.com/sourcegraph/syntaxhighlight"
)

var CLI = flags.NewNamedParser("src-docco", flags.Default)

var GlobalOpt struct {
	Verbose bool `short:"v" description:"show verbose output"`
}

var vLogger = log.New(os.Stderr, "", 0)

func vLogf(format string, v ...interface{}) {
	if !GlobalOpt.Verbose {
		return
	}
	vLogger.Printf(format, v...)
}

func vLog(v ...interface{}) {
	if !GlobalOpt.Verbose {
		return
	}
	vLogger.Println(v...)
}

func init() {
	CLI.LongDescription = "TODO"
	CLI.AddGroup("Global options", "", &GlobalOpt)

	_, err := CLI.AddCommand("gen",
		"generate documentation",
		"Generate docco-like documentation for a thing",
		&genCmd,
	)
	if err != nil {
		log.Fatal(err)
	}
}

// TODO: add limitations help output.

type GenCmd struct {
	Dir string `long:"dir" description:"The root directory for the project"`
	// TODO: must be relative to Dir.
	SiteDirName string `long:"site-dir" description:"The directory name for the output files" default:"site"`
}

var genCmd GenCmd

// Source units have lots of information associated with them, but we
// only care about the files.
type unit struct {
	Files []string
}

type units []unit

func (us units) collateFiles() []string {
	var fs []string
	for _, u := range us {
		for _, f := range u.Files {
			fs = append(fs, f)
		}
	}
	return fs
}

type failedCmd struct {
	cmd interface{}
	err interface{}
}

// TODO: Is this correct?
func (f failedCmd) Error() string {
	return fmt.Sprintf("command %v failed: %s", f.cmd, f.err)
}

func ensureSrclibExists() error {
	cmd := exec.Command("src", "version")
	stdout, stderr := &bytes.Buffer{}, &bytes.Buffer{}
	cmd.Stdout, cmd.Stderr = stdout, stderr
	if err := cmd.Run(); err != nil {
		return fmt.Errorf(
			"error with srclib: %v, %s, %s",
			err,
			stdout.String(),
			stderr.String(),
		)
	}
	return nil
}

func command(argv []string) (cmd *exec.Cmd, stdout *bytes.Buffer, stderr *bytes.Buffer) {
	if len(argv) == 0 {
		panic("command: argv must have at least one item")
	}
	cmd = exec.Command(argv[0], argv[1:]...)
	stdout, stderr = &bytes.Buffer{}, &bytes.Buffer{}
	cmd.Stdout, cmd.Stderr = stdout, stderr
	return cmd, stdout, stderr
}

func (c *GenCmd) Execute(args []string) error {
	if err := ensureSrclibExists(); err != nil {
		log.Fatal(err)
	}

	// First, we need to get a list of all of the files that we
	// want to generate.
	//
	// We could import sourcegraph.com/sourcegraph/srclib, but I
	// want to demonstrate how to use its command line interface.
	var dir string
	if c.Dir == "" {
		d, err := os.Getwd()
		if err != nil {
			log.Fatal(err)
		}
		dir = d
	} else {
		d, err := filepath.Abs(c.Dir)
		if err != nil {
			log.Fatal(err)
		}
		dir = d
	}
	argv := []string{"src", "api", "units", dir}
	cmd, stdout, stderr := command(argv)
	vLogf("Running %v", argv)
	// TODO: remove failedCmd
	if err := cmd.Run(); err != nil {
		log.Fatal(failedCmd{argv, []interface{}{err, stdout.String(), stderr.String()}})
	}
	if stdout.Len() == 0 {
		log.Fatal(failedCmd{argv, "no output"})
	}
	var us units
	if err := json.Unmarshal(stdout.Bytes(), &us); err != nil {
		log.Fatal(err)
	}
	return genSite(dir, c.SiteDirName, us.collateFiles())
}

func genSite(root, siteName string, files []string) error {
	vLog("Generating Site")
	sitePath := filepath.Join(root, siteName)
	if err := os.MkdirAll(sitePath, 0755); err != nil {
		log.Fatal(err)
	}
	for _, f := range files {
		vLog("Processing", f)
		src, err := ioutil.ReadFile(filepath.Join(root, f))
		if err != nil {
			return err
		}
		htmlFile := "/" + f + ".html"
		as, err := ann(src, root, f, htmlFile)
		if err != nil {
			return err
		}
		vLog("Never get here")
		if err := os.MkdirAll(filepath.Dir(filepath.Join(sitePath, htmlFile)), 0755); err != nil {
			log.Fatal(err)
		}
		w, err := os.Create(filepath.Join(sitePath, htmlFile))
		if err != nil {
			return err
		}
		if err := writeAnns(w, src, as); err != nil {
			return err
		}
	}
	return nil
}

type annotation struct {
	annotate.Annotation
	Comment bool
}

type htmlAnnotator syntaxhighlight.HTMLConfig

func (c htmlAnnotator) class(kind syntaxhighlight.Kind) string {
	switch kind {
	case syntaxhighlight.String:
		return c.String
	case syntaxhighlight.Keyword:
		return c.Keyword
	case syntaxhighlight.Comment:
		return c.Comment
	case syntaxhighlight.Type:
		return c.Type
	case syntaxhighlight.Literal:
		return c.Literal
	case syntaxhighlight.Punctuation:
		return c.Punctuation
	case syntaxhighlight.Plaintext:
		return c.Plaintext
	case syntaxhighlight.Tag:
		return c.Tag
	case syntaxhighlight.HTMLTag:
		return c.HTMLTag
	case syntaxhighlight.HTMLAttrName:
		return c.HTMLAttrName
	case syntaxhighlight.HTMLAttrValue:
		return c.HTMLAttrValue
	case syntaxhighlight.Decimal:
		return c.Decimal
	}
	return ""
}

func (a htmlAnnotator) Annotate(start int, kind syntaxhighlight.Kind, tokText string) (*annotate.Annotation, error) {
	class := a.class(kind)
	if class == "" {
		return nil, nil
	}
	return &annotate.Annotation{
		Start: start,
		End:   start + len(tokText),
		Left:  []byte(class),
		Right: nil,
	}, nil
}

func ann(src []byte, root, file, htmlFile string) ([]annotation, error) {
	vLog("Annotating", file)
	annotations, err := syntaxhighlight.Annotate(src, htmlAnnotator(syntaxhighlight.DefaultHTMLConfig))
	if err != nil {
		return nil, err
	}
	sort.Sort(annotations)
	as := make([]annotation, 0, len(annotations))
	argv := []string{"src", "api", "list", "--file", filepath.Join(root, file)}
	cmd, stdout, stderr := command(argv)
	vLog("Running", argv)
	if err := cmd.Run(); err != nil {
		return nil, failedCmd{argv, []interface{}{err, stdout.String(), stderr.String()}}
	}
	type ref struct {
		DefUnit string
		DefPath string
		Start   uint32
	}
	refs := []ref{}
	if err := json.Unmarshal(stdout.Bytes(), &refs); err != nil {
		log.Fatal(err)
	}
	var refAtIndex int
	refAt := func(start uint32) (r ref, found bool) {
		for refAtIndex < len(refs) {
			if refs[refAtIndex].Start == start {
				refAtIndex++
				return r, true
			} else if refs[refAtIndex].Start < start {
				refAtIndex++
			} else { // refs[refAtIndex].Start > start
				return ref{}, false
			}
		}
		return ref{}, false
	}
	for _, a := range annotations {
		if string(a.Left) == "com" {
			as = append(as, annotation{*a, true})
			continue
		}
		r, found := refAt(uint32(a.Start))
		if !found {
			a.Left = []byte(fmt.Sprintf(`<span class="%s">`, string(a.Left)))
			a.Right = []byte(`</span>`)
			as = append(as, annotation{*a, false})
			continue
		}
		a.Left = []byte(fmt.Sprintf(
			`<span class="%s" id="%s"><a href="%s">`,
			string(a.Left),
			filepath.Join(r.DefUnit, r.DefPath),
			htmlFile,
		))
		a.Right = []byte(`</span></a>`)
		as = append(as, annotation{*a, false})
	}
	return as, nil
}

func writeAnns(w io.Writer, src []byte, as []annotation) error {
	vLog("Writing annotations")
	line := 0
	if _, err := w.Write([]byte(fmt.Sprintf(`<pre><div line=%d>`, line))); err != nil {
		return err
	}
	addDivs := func(src []byte) []byte {
		buf := &bytes.Buffer{}
		buf.Grow(len(src))
		for _, b := range src {
			if b == '\n' {
				line++
				buf.Write([]byte(fmt.Sprintf("
\n
", line)))
				continue
			}
			buf.WriteByte(b)
		}
		return buf.Bytes()
	}
	defer func() {
		w.Write([]byte("
\n
"))
	}()
	for i := 0; i < len(src); {
		if len(as) == 0 {
			_, err := w.Write(addDivs(src[i:]))
			return err
		}
		a := as[0]
		if i > a.Start {
			log.Fatal("writeAnns: illegal state: i > a.Start")
		}
		if i < a.Start {
			if _, err := w.Write(addDivs(src[i : i+1])); err != nil {
				return err
			}
			i++
			continue
		}
		// i == a.Start
		for _, b := range src[a.Start:a.End] {
			if b == '\n' {
				log.Fatal(`writeAnns: illegal state: \n in annotation`)
			}
		}
		if _, err := w.Write([]byte(strings.Join([]string{
			string(a.Left),
			string(src[a.Start:a.End]),
			string(a.Right),
		}, ""))); err != nil {
			return err
		}
		i = a.End
		as = as[1:]
	}
	return nil
}

func Main() error {
	log.SetFlags(log.Lshortfile)
	_, err := CLI.Parse()
	return err
}
<|endoftext|>


<commit_before>package isolated

import (
	"io/ioutil"
	"os"
	"path/filepath"
	"runtime"
	"strings"

	"code.cloudfoundry.org/cli/integration/helpers"
	"code.cloudfoundry.org/cli/util/configv3"
	. "github.com/onsi/ginkgo"
	. "github.com/onsi/ginkgo/extensions/table"
	. "github.com/onsi/gomega"
	. "github.com/onsi/gomega/gbytes"
	. "github.com/onsi/gomega/gexec"
)

var _ = Describe("Verbose", func() {
	BeforeEach(func() {
		helpers.SkipIfClientCredentialsTestMode()
	})

	DescribeTable("displays verbose output to terminal",
		func(env string, configTrace string, flag bool) {
			tmpDir, err := ioutil.TempDir("", "")
			defer os.RemoveAll(tmpDir)
			Expect(err).NotTo(HaveOccurred())

			helpers.SetupCF(ReadOnlyOrg, ReadOnlySpace)

			// Invalidate the access token to cause a token refresh in order to
			// test the call to the UAA.
			helpers.SetConfig(func(config *configv3.Config) {
				config.ConfigFile.AccessToken = helpers.ExpiredAccessToken()
			})

			var envMap map[string]string
			if env != "" {
				if string(env[0]) == "/" {
					env = filepath.Join(tmpDir, env)
				}
				envMap = map[string]string{"CF_TRACE": env}
			}

			command := []string{"run-task", "app", "echo"}

			if flag {
				command = append(command, "-v")
			}

			if configTrace != "" {
				if string(configTrace[0]) == "/" {
					configTrace = filepath.Join(tmpDir, configTrace)
				}
				session := helpers.CF("config", "--trace", configTrace)
				Eventually(session).Should(Exit(0))
			}

			session := helpers.CFWithEnv(envMap, command...)

			Eventually(session).Should(Say("REQUEST:"))
			Eventually(session).Should(Say("POST /oauth/token"))
			Eventually(session).Should(Say(`User-Agent: cf/[\w.+-]+ \(go\d+\.\d+(\.\d+)?; %s %s\)`, runtime.GOARCH, runtime.GOOS))
			Eventually(session).Should(Say(`\[PRIVATE DATA HIDDEN\]`)) //This is required to test the previous line. If it fails, the previous matcher went too far.
			Eventually(session).Should(Say("RESPONSE:"))
			Eventually(session).Should(Say("REQUEST:"))
			Eventually(session).Should(Say("GET /v3/apps"))
			Eventually(session).Should(Say(`User-Agent: cf/[\w.+-]+ \(go\d+\.\d+(\.\d+)?; %s %s\)`, runtime.GOARCH, runtime.GOOS))
			Eventually(session).Should(Say("RESPONSE:"))
			Eventually(session).Should(Exit(1))
		},

		Entry("CF_TRACE true: enables verbose", "true", "", false),
		Entry("CF_Trace true, config trace false: enables verbose", "true", "false", false),
		Entry("CF_Trace true, config trace file path: enables verbose AND logging to file", "true", "/foo", false),

		Entry("CF_TRACE false, '-v': enables verbose", "false", "", true),
		Entry("CF_TRACE false, config trace file path, '-v': enables verbose AND logging to file", "false", "/foo", true),

		Entry("CF_TRACE empty:, '-v': enables verbose", "", "", true),
		Entry("CF_TRACE empty, config trace true: enables verbose", "", "true", false),
		Entry("CF_TRACE empty, config trace file path, '-v': enables verbose AND logging to file", "", "/foo", true),

		Entry("CF_TRACE filepath, '-v': enables logging to file", "/foo", "", true),
		Entry("CF_TRACE filepath, config trace true: enables verbose AND logging to file", "/foo", "true", false),
		Entry("CF_TRACE filepath, config trace filepath, '-v': enables verbose AND logging to file for BOTH paths", "/foo", "/bar", true),
	)

	DescribeTable("displays verbose output to multiple files",
		func(env string, configTrace string, flag bool, location []string) {
			tmpDir, err := ioutil.TempDir("", "")
			defer os.RemoveAll(tmpDir)
			Expect(err).NotTo(HaveOccurred())

			helpers.SetupCF(ReadOnlyOrg, ReadOnlySpace)

			// Invalidate the access token to cause a token refresh in order to
			// test the call to the UAA.
			helpers.SetConfig(func(config *configv3.Config) {
				config.ConfigFile.AccessToken = helpers.ExpiredAccessToken()
			})

			var envMap map[string]string
			if env != "" {
				if string(env[0]) == "/" {
					env = filepath.Join(tmpDir, env)
				}
				envMap = map[string]string{"CF_TRACE": env}
			}

			command := []string{"run-task", "app", "echo"}

			if flag {
				command = append(command, "-v")
			}

			if configTrace != "" {
				if string(configTrace[0]) == "/" {
					configTrace = filepath.Join(tmpDir, configTrace)
				}
				session := helpers.CF("config", "--trace", configTrace)
				Eventually(session).Should(Exit(0))
			}

			session := helpers.CFWithEnv(envMap, command...)
			Eventually(session).Should(Exit(1))

			for _, filePath := range location {
				contents, err := ioutil.ReadFile(tmpDir + filePath)
				Expect(err).ToNot(HaveOccurred())

				Expect(string(contents)).To(MatchRegexp("REQUEST:"))
				Expect(string(contents)).To(MatchRegexp("GET /v3/apps"))
				Expect(string(contents)).To(MatchRegexp("RESPONSE:"))
				Expect(string(contents)).To(MatchRegexp("REQUEST:"))
				Expect(string(contents)).To(MatchRegexp("POST /oauth/token"))
				Expect(string(contents)).To(MatchRegexp("RESPONSE:"))

				stat, err := os.Stat(tmpDir + filePath)
				Expect(err).ToNot(HaveOccurred())

				if runtime.GOOS == "windows" {
					Expect(stat.Mode().String()).To(Equal(os.FileMode(0666).String()))
				} else {
					Expect(stat.Mode().String()).To(Equal(os.FileMode(0600).String()))
				}
			}
		},

		Entry("CF_Trace true, config trace file path: enables verbose AND logging to file", "true", "/foo", false, []string{"/foo"}),

		Entry("CF_TRACE false, config trace file path: enables logging to file", "false", "/foo", false, []string{"/foo"}),
		Entry("CF_TRACE false, config trace file path, '-v': enables verbose AND logging to file", "false", "/foo", true, []string{"/foo"}),

		Entry("CF_TRACE empty, config trace file path: enables logging to file", "", "/foo", false, []string{"/foo"}),
		Entry("CF_TRACE empty, config trace file path, '-v': enables verbose AND logging to file", "", "/foo", true, []string{"/foo"}),

		Entry("CF_TRACE filepath: enables logging to file", "/foo", "", false, []string{"/foo"}),
		Entry("CF_TRACE filepath, '-v': enables logging to file", "/foo", "", true, []string{"/foo"}),
		Entry("CF_TRACE filepath, config trace true: enables verbose AND logging to file", "/foo", "true", false, []string{"/foo"}),
		Entry("CF_TRACE filepath, config trace filepath: enables logging to file for BOTH paths", "/foo", "/bar", false, []string{"/foo", "/bar"}),
		Entry("CF_TRACE filepath, config trace filepath, '-v': enables verbose AND logging to file for BOTH paths", "/foo", "/bar", true, []string{"/foo", "/bar"}),
	)

	Describe("NOAA", func() {
		var orgName string

		BeforeEach(func() {
			orgName = helpers.NewOrgName()
			spaceName := helpers.NewSpaceName()

			helpers.SetupCF(orgName, spaceName)
		})

		AfterEach(func() {
			Eventually(helpers.CF("config", "--trace", "false")).Should(Exit(0))
			helpers.QuickDeleteOrg(orgName)
		})

		DescribeTable("displays verbose output to terminal",
			func(env string, configTrace string, flag bool) {
				tmpDir, err := ioutil.TempDir("", "")
				defer os.RemoveAll(tmpDir)
				Expect(err).NotTo(HaveOccurred())

				appName := helpers.PrefixedRandomName("app")

				helpers.WithHelloWorldApp(func(appDir string) {
					Eventually(helpers.CF("push", appName, "--no-start", "-p", appDir, "-b", "staticfile_buildpack", "--no-route")).Should(Exit(0))
				})

				var envMap map[string]string
				if env != "" {
					if string(env[0]) == "/" {
						env = filepath.Join(tmpDir, env)
					}
					envMap = map[string]string{"CF_TRACE": env}
				}

				command := []string{"logs", appName}

				if flag {
					command = append(command, "-v")
				}

				if configTrace != "" {
					if string(configTrace[0]) == "/" {
						configTrace = filepath.Join(tmpDir, configTrace)
					}
					session := helpers.CF("config", "--trace", configTrace)
					Eventually(session).Should(Exit(0))
				}

				session := helpers.CFWithEnv(envMap, command...)

				Eventually(session).Should(Say("REQUEST:"))
				Eventually(session).Should(Say("GET /v2/info"))
				Eventually(session).Should(Say("WEBSOCKET REQUEST:"))
				Eventually(session).Should(Say(`Authorization: \[PRIVATE DATA HIDDEN\]`))
				Eventually(session.Kill()).Should(Exit())
			},

			Entry("CF_TRACE true: enables verbose", "true", "", false),
			Entry("CF_Trace true, config trace false: enables verbose", "true", "false", false),
			Entry("CF_Trace true, config trace file path: enables verbose AND logging to file", "true", "/foo", false),

			Entry("CF_TRACE false, '-v': enables verbose", "false", "", true),
			Entry("CF_TRACE false, config trace file path, '-v': enables verbose AND logging to file", "false", "/foo", true),

			Entry("CF_TRACE empty:, '-v': enables verbose", "", "", true),
			Entry("CF_TRACE empty, config trace true: enables verbose", "", "true", false),
			Entry("CF_TRACE empty, config trace file path, '-v': enables verbose AND logging to file", "", "/foo", true),

			Entry("CF_TRACE filepath, '-v': enables logging to file", "/foo", "", true),
			Entry("CF_TRACE filepath, config trace true: enables verbose AND logging to file", "/foo", "true", false),
			Entry("CF_TRACE filepath, config trace filepath, '-v': enables verbose AND logging to file for BOTH paths", "/foo", "/bar", true),
		)

		DescribeTable("displays verbose output to multiple files",
			func(env string, configTrace string, location []string) {
				tmpDir, err := ioutil.TempDir("", "")
				defer os.RemoveAll(tmpDir)
				Expect(err).NotTo(HaveOccurred())

				appName := helpers.PrefixedRandomName("app")

				helpers.WithHelloWorldApp(func(appDir string) {
					Eventually(helpers.CF("push", appName, "--no-start", "-p", appDir, "-b", "staticfile_buildpack", "--no-route")).Should(Exit(0))
				})

				var envMap map[string]string
				if env != "" {
					if string(env[0]) == "/" {
						env = filepath.Join(tmpDir, env)
					}
					envMap = map[string]string{"CF_TRACE": env}
				}

				if configTrace != "" {
					if strings.HasPrefix(configTrace, "/") {
						configTrace = filepath.Join(tmpDir, configTrace)
					}
					session := helpers.CF("config", "--trace", configTrace)
					Eventually(session).Should(Exit(0))
				}

				session := helpers.CFWithEnv(envMap, "logs", "-v", appName)

				Eventually(session).Should(Say("WEBSOCKET RESPONSE"))
				Eventually(session.Kill()).Should(Exit())

				for _, filePath := range location {
					contents, err := ioutil.ReadFile(tmpDir + filePath)
					Expect(err).ToNot(HaveOccurred())

					Expect(string(contents)).To(MatchRegexp("REQUEST:"))
					Expect(string(contents)).To(MatchRegexp("GET /v2/info"))
					Expect(string(contents)).To(MatchRegexp("WEBSOCKET REQUEST:"))
					Expect(string(contents)).To(MatchRegexp(`Authorization: \[PRIVATE DATA HIDDEN\]`))

					stat, err := os.Stat(tmpDir + filePath)
					Expect(err).ToNot(HaveOccurred())

					if runtime.GOOS == "windows" {
						Expect(stat.Mode().String()).To(Equal(os.FileMode(0666).String()))
					} else {
						Expect(stat.Mode().String()).To(Equal(os.FileMode(0600).String()))
					}
				}
			},

			Entry("CF_Trace true, config trace file path: enables verbose AND logging to file", "true", "/foo", []string{"/foo"}),

			Entry("CF_TRACE false, config trace file path: enables logging to file", "false", "/foo", []string{"/foo"}),
			Entry("CF_TRACE false, config trace file path, '-v': enables verbose AND logging to file", "false", "/foo", []string{"/foo"}),

			Entry("CF_TRACE empty, config trace file path: enables logging to file", "", "/foo", []string{"/foo"}),
			Entry("CF_TRACE empty, config trace file path, '-v': enables verbose AND logging to file", "", "/foo", []string{"/foo"}),

			Entry("CF_TRACE filepath: enables logging to file", "/foo", "", []string{"/foo"}),
			Entry("CF_TRACE filepath, '-v': enables logging to file", "/foo", "", []string{"/foo"}),
			Entry("CF_TRACE filepath, config trace true: enables verbose AND logging to file", "/foo", "true", []string{"/foo"}),
			Entry("CF_TRACE filepath, config trace filepath: enables logging to file for BOTH paths", "/foo", "/bar", []string{"/foo", "/bar"}),
			Entry("CF_TRACE filepath, config trace filepath, '-v': enables verbose AND logging to file for BOTH paths", "/foo", "/bar", []string{"/foo", "/bar"}),
		)
	})
})
<commit_msg>Yet another v7 test that needs an update to `run-task`<commit_after>package isolated

import (
	"io/ioutil"
	"os"
	"path/filepath"
	"runtime"
	"strings"

	"code.cloudfoundry.org/cli/integration/helpers"
	"code.cloudfoundry.org/cli/util/configv3"
	. "github.com/onsi/ginkgo"
	. "github.com/onsi/ginkgo/extensions/table"
	. "github.com/onsi/gomega"
	. "github.com/onsi/gomega/gbytes"
	. "github.com/onsi/gomega/gexec"
)

var _ = Describe("Verbose", func() {
	BeforeEach(func() {
		helpers.SkipIfClientCredentialsTestMode()
	})

	DescribeTable("displays verbose output to terminal",
		func(env string, configTrace string, flag bool) {
			tmpDir, err := ioutil.TempDir("", "")
			defer os.RemoveAll(tmpDir)
			Expect(err).NotTo(HaveOccurred())

			helpers.SetupCF(ReadOnlyOrg, ReadOnlySpace)

			// Invalidate the access token to cause a token refresh in order to
			// test the call to the UAA.
			helpers.SetConfig(func(config *configv3.Config) {
				config.ConfigFile.AccessToken = helpers.ExpiredAccessToken()
			})

			var envMap map[string]string
			if env != "" {
				if string(env[0]) == "/" {
					env = filepath.Join(tmpDir, env)
				}
				envMap = map[string]string{"CF_TRACE": env}
			}

			command := []string{"run-task", "app", "--command", "echo"}

			if flag {
				command = append(command, "-v")
			}

			if configTrace != "" {
				if string(configTrace[0]) == "/" {
					configTrace = filepath.Join(tmpDir, configTrace)
				}
				session := helpers.CF("config", "--trace", configTrace)
				Eventually(session).Should(Exit(0))
			}

			session := helpers.CFWithEnv(envMap, command...)

			Eventually(session).Should(Say("REQUEST:"))
			Eventually(session).Should(Say("POST /oauth/token"))
			Eventually(session).Should(Say(`User-Agent: cf/[\w.+-]+ \(go\d+\.\d+(\.\d+)?; %s %s\)`, runtime.GOARCH, runtime.GOOS))
			Eventually(session).Should(Say(`\[PRIVATE DATA HIDDEN\]`)) //This is required to test the previous line. If it fails, the previous matcher went too far.
			Eventually(session).Should(Say("RESPONSE:"))
			Eventually(session).Should(Say("REQUEST:"))
			Eventually(session).Should(Say("GET /v3/apps"))
			Eventually(session).Should(Say(`User-Agent: cf/[\w.+-]+ \(go\d+\.\d+(\.\d+)?; %s %s\)`, runtime.GOARCH, runtime.GOOS))
			Eventually(session).Should(Say("RESPONSE:"))
			Eventually(session).Should(Exit(1))
		},

		Entry("CF_TRACE true: enables verbose", "true", "", false),
		Entry("CF_Trace true, config trace false: enables verbose", "true", "false", false),
		Entry("CF_Trace true, config trace file path: enables verbose AND logging to file", "true", "/foo", false),

		Entry("CF_TRACE false, '-v': enables verbose", "false", "", true),
		Entry("CF_TRACE false, config trace file path, '-v': enables verbose AND logging to file", "false", "/foo", true),

		Entry("CF_TRACE empty:, '-v': enables verbose", "", "", true),
		Entry("CF_TRACE empty, config trace true: enables verbose", "", "true", false),
		Entry("CF_TRACE empty, config trace file path, '-v': enables verbose AND logging to file", "", "/foo", true),

		Entry("CF_TRACE filepath, '-v': enables logging to file", "/foo", "", true),
		Entry("CF_TRACE filepath, config trace true: enables verbose AND logging to file", "/foo", "true", false),
		Entry("CF_TRACE filepath, config trace filepath, '-v': enables verbose AND logging to file for BOTH paths", "/foo", "/bar", true),
	)

	DescribeTable("displays verbose output to multiple files",
		func(env string, configTrace string, flag bool, location []string) {
			tmpDir, err := ioutil.TempDir("", "")
			defer os.RemoveAll(tmpDir)
			Expect(err).NotTo(HaveOccurred())

			helpers.SetupCF(ReadOnlyOrg, ReadOnlySpace)

			// Invalidate the access token to cause a token refresh in order to
			// test the call to the UAA.
			helpers.SetConfig(func(config *configv3.Config) {
				config.ConfigFile.AccessToken = helpers.ExpiredAccessToken()
			})

			var envMap map[string]string
			if env != "" {
				if string(env[0]) == "/" {
					env = filepath.Join(tmpDir, env)
				}
				envMap = map[string]string{"CF_TRACE": env}
			}

			command := []string{"run-task", "app", "--command", "echo"}

			if flag {
				command = append(command, "-v")
			}

			if configTrace != "" {
				if string(configTrace[0]) == "/" {
					configTrace = filepath.Join(tmpDir, configTrace)
				}
				session := helpers.CF("config", "--trace", configTrace)
				Eventually(session).Should(Exit(0))
			}

			session := helpers.CFWithEnv(envMap, command...)
			Eventually(session).Should(Exit(1))

			for _, filePath := range location {
				contents, err := ioutil.ReadFile(tmpDir + filePath)
				Expect(err).ToNot(HaveOccurred())

				Expect(string(contents)).To(MatchRegexp("REQUEST:"))
				Expect(string(contents)).To(MatchRegexp("GET /v3/apps"))
				Expect(string(contents)).To(MatchRegexp("RESPONSE:"))
				Expect(string(contents)).To(MatchRegexp("REQUEST:"))
				Expect(string(contents)).To(MatchRegexp("POST /oauth/token"))
				Expect(string(contents)).To(MatchRegexp("RESPONSE:"))

				stat, err := os.Stat(tmpDir + filePath)
				Expect(err).ToNot(HaveOccurred())

				if runtime.GOOS == "windows" {
					Expect(stat.Mode().String()).To(Equal(os.FileMode(0666).String()))
				} else {
					Expect(stat.Mode().String()).To(Equal(os.FileMode(0600).String()))
				}
			}
		},

		Entry("CF_Trace true, config trace file path: enables verbose AND logging to file", "true", "/foo", false, []string{"/foo"}),

		Entry("CF_TRACE false, config trace file path: enables logging to file", "false", "/foo", false, []string{"/foo"}),
		Entry("CF_TRACE false, config trace file path, '-v': enables verbose AND logging to file", "false", "/foo", true, []string{"/foo"}),

		Entry("CF_TRACE empty, config trace file path: enables logging to file", "", "/foo", false, []string{"/foo"}),
		Entry("CF_TRACE empty, config trace file path, '-v': enables verbose AND logging to file", "", "/foo", true, []string{"/foo"}),

		Entry("CF_TRACE filepath: enables logging to file", "/foo", "", false, []string{"/foo"}),
		Entry("CF_TRACE filepath, '-v': enables logging to file", "/foo", "", true, []string{"/foo"}),
		Entry("CF_TRACE filepath, config trace true: enables verbose AND logging to file", "/foo", "true", false, []string{"/foo"}),
		Entry("CF_TRACE filepath, config trace filepath: enables logging to file for BOTH paths", "/foo", "/bar", false, []string{"/foo", "/bar"}),
		Entry("CF_TRACE filepath, config trace filepath, '-v': enables verbose AND logging to file for BOTH paths", "/foo", "/bar", true, []string{"/foo", "/bar"}),
	)

	Describe("NOAA", func() {
		var orgName string

		BeforeEach(func() {
			orgName = helpers.NewOrgName()
			spaceName := helpers.NewSpaceName()

			helpers.SetupCF(orgName, spaceName)
		})

		AfterEach(func() {
			Eventually(helpers.CF("config", "--trace", "false")).Should(Exit(0))
			helpers.QuickDeleteOrg(orgName)
		})

		DescribeTable("displays verbose output to terminal",
			func(env string, configTrace string, flag bool) {
				tmpDir, err := ioutil.TempDir("", "")
				defer os.RemoveAll(tmpDir)
				Expect(err).NotTo(HaveOccurred())

				appName := helpers.PrefixedRandomName("app")

				helpers.WithHelloWorldApp(func(appDir string) {
					Eventually(helpers.CF("push", appName, "--no-start", "-p", appDir, "-b", "staticfile_buildpack", "--no-route")).Should(Exit(0))
				})

				var envMap map[string]string
				if env != "" {
					if string(env[0]) == "/" {
						env = filepath.Join(tmpDir, env)
					}
					envMap = map[string]string{"CF_TRACE": env}
				}

				command := []string{"logs", appName}

				if flag {
					command = append(command, "-v")
				}

				if configTrace != "" {
					if string(configTrace[0]) == "/" {
						configTrace = filepath.Join(tmpDir, configTrace)
					}
					session := helpers.CF("config", "--trace", configTrace)
					Eventually(session).Should(Exit(0))
				}

				session := helpers.CFWithEnv(envMap, command...)

				Eventually(session).Should(Say("REQUEST:"))
				Eventually(session).Should(Say("GET /v2/info"))
				Eventually(session).Should(Say("WEBSOCKET REQUEST:"))
				Eventually(session).Should(Say(`Authorization: \[PRIVATE DATA HIDDEN\]`))
				Eventually(session.Kill()).Should(Exit())
			},

			Entry("CF_TRACE true: enables verbose", "true", "", false),
			Entry("CF_Trace true, config trace false: enables verbose", "true", "false", false),
			Entry("CF_Trace true, config trace file path: enables verbose AND logging to file", "true", "/foo", false),

			Entry("CF_TRACE false, '-v': enables verbose", "false", "", true),
			Entry("CF_TRACE false, config trace file path, '-v': enables verbose AND logging to file", "false", "/foo", true),

			Entry("CF_TRACE empty:, '-v': enables verbose", "", "", true),
			Entry("CF_TRACE empty, config trace true: enables verbose", "", "true", false),
			Entry("CF_TRACE empty, config trace file path, '-v': enables verbose AND logging to file", "", "/foo", true),

			Entry("CF_TRACE filepath, '-v': enables logging to file", "/foo", "", true),
			Entry("CF_TRACE filepath, config trace true: enables verbose AND logging to file", "/foo", "true", false),
			Entry("CF_TRACE filepath, config trace filepath, '-v': enables verbose AND logging to file for BOTH paths", "/foo", "/bar", true),
		)

		DescribeTable("displays verbose output to multiple files",
			func(env string, configTrace string, location []string) {
				tmpDir, err := ioutil.TempDir("", "")
				defer os.RemoveAll(tmpDir)
				Expect(err).NotTo(HaveOccurred())

				appName := helpers.PrefixedRandomName("app")

				helpers.WithHelloWorldApp(func(appDir string) {
					Eventually(helpers.CF("push", appName, "--no-start", "-p", appDir, "-b", "staticfile_buildpack", "--no-route")).Should(Exit(0))
				})

				var envMap map[string]string
				if env != "" {
					if string(env[0]) == "/" {
						env = filepath.Join(tmpDir, env)
					}
					envMap = map[string]string{"CF_TRACE": env}
				}

				if configTrace != "" {
					if strings.HasPrefix(configTrace, "/") {
						configTrace = filepath.Join(tmpDir, configTrace)
					}
					session := helpers.CF("config", "--trace", configTrace)
					Eventually(session).Should(Exit(0))
				}

				session := helpers.CFWithEnv(envMap, "logs", "-v", appName)

				Eventually(session).Should(Say("WEBSOCKET RESPONSE"))
				Eventually(session.Kill()).Should(Exit())

				for _, filePath := range location {
					contents, err := ioutil.ReadFile(tmpDir + filePath)
					Expect(err).ToNot(HaveOccurred())

					Expect(string(contents)).To(MatchRegexp("REQUEST:"))
					Expect(string(contents)).To(MatchRegexp("GET /v2/info"))
					Expect(string(contents)).To(MatchRegexp("WEBSOCKET REQUEST:"))
					Expect(string(contents)).To(MatchRegexp(`Authorization: \[PRIVATE DATA HIDDEN\]`))

					stat, err := os.Stat(tmpDir + filePath)
					Expect(err).ToNot(HaveOccurred())

					if runtime.GOOS == "windows" {
						Expect(stat.Mode().String()).To(Equal(os.FileMode(0666).String()))
					} else {
						Expect(stat.Mode().String()).To(Equal(os.FileMode(0600).String()))
					}
				}
			},

			Entry("CF_Trace true, config trace file path: enables verbose AND logging to file", "true", "/foo", []string{"/foo"}),

			Entry("CF_TRACE false, config trace file path: enables logging to file", "false", "/foo", []string{"/foo"}),
			Entry("CF_TRACE false, config trace file path, '-v': enables verbose AND logging to file", "false", "/foo", []string{"/foo"}),

			Entry("CF_TRACE empty, config trace file path: enables logging to file", "", "/foo", []string{"/foo"}),
			Entry("CF_TRACE empty, config trace file path, '-v': enables verbose AND logging to file", "", "/foo", []string{"/foo"}),

			Entry("CF_TRACE filepath: enables logging to file", "/foo", "", []string{"/foo"}),
			Entry("CF_TRACE filepath, '-v': enables logging to file", "/foo", "", []string{"/foo"}),
			Entry("CF_TRACE filepath, config trace true: enables verbose AND logging to file", "/foo", "true", []string{"/foo"}),
			Entry("CF_TRACE filepath, config trace filepath: enables logging to file for BOTH paths", "/foo", "/bar", []string{"/foo", "/bar"}),
			Entry("CF_TRACE filepath, config trace filepath, '-v': enables verbose AND logging to file for BOTH paths", "/foo", "/bar", []string{"/foo", "/bar"}),
		)
	})
})
<|endoftext|>


<commit_before>// srclib-docco is a docco-like static documentation generator.
// TODO: write this in a literal style.
package srclib_docco

import (
	"bytes"
	"encoding/json"
	"fmt"
	"io/ioutil"
	"log"
	"os"
	"os/exec"
	"path/filepath"
	"sort"
	"text/template"

	"github.com/jessevdk/go-flags"
	"github.com/sourcegraph/annotate"
	"github.com/sourcegraph/syntaxhighlight"
)

var CLI = flags.NewNamedParser("src-docco", flags.Default)

var GlobalOpt struct {
	Verbose bool `short:"v" description:"show verbose output"`
}

var vLogger = log.New(os.Stderr, "", 0)

func vLogf(format string, v ...interface{}) {
	if !GlobalOpt.Verbose {
		return
	}
	vLogger.Printf(format, v...)
}

func vLog(v ...interface{}) {
	if !GlobalOpt.Verbose {
		return
	}
	vLogger.Println(v...)
}

func init() {
	CLI.LongDescription = "TODO"
	CLI.AddGroup("Global options", "", &GlobalOpt)

	_, err := CLI.AddCommand("gen",
		"generate documentation",
		"Generate docco-like documentation for a thing",
		&genCmd,
	)
	if err != nil {
		log.Fatal(err)
	}
}

// TODO: add limitations help output.

type GenCmd struct {
	Dir string `long:"dir" description:"The root directory for the project"`
	// TODO: must be relative to Dir.
	SiteDirName string `long:"site-dir" description:"The directory name for the output files" default:"site"`
}

var genCmd GenCmd

// Source units have lots of information associated with them, but we
// only care about the files.
type unit struct {
	Files []string
}

type units []unit

func (us units) collateFiles() []string {
	var fs []string
	for _, u := range us {
		for _, f := range u.Files {
			fs = append(fs, f)
		}
	}
	return fs
}

type failedCmd struct {
	cmd interface{}
	err interface{}
}

// TODO: Is this correct?
func (f failedCmd) Error() string {
	return fmt.Sprintf("command %v failed: %s", f.cmd, f.err)
}

func ensureSrclibExists() error {
	cmd := exec.Command("src", "version")
	stdout, stderr := &bytes.Buffer{}, &bytes.Buffer{}
	cmd.Stdout, cmd.Stderr = stdout, stderr
	if err := cmd.Run(); err != nil {
		return fmt.Errorf(
			"error with srclib: %v, %s, %s",
			err,
			stdout.String(),
			stderr.String(),
		)
	}
	return nil
}

func command(argv []string) (cmd *exec.Cmd, stdout *bytes.Buffer, stderr *bytes.Buffer) {
	if len(argv) == 0 {
		panic("command: argv must have at least one item")
	}
	cmd = exec.Command(argv[0], argv[1:]...)
	stdout, stderr = &bytes.Buffer{}, &bytes.Buffer{}
	cmd.Stdout, cmd.Stderr = stdout, stderr
	return cmd, stdout, stderr
}

func (c *GenCmd) Execute(args []string) error {
	if err := ensureSrclibExists(); err != nil {
		log.Fatal(err)
	}

	// First, we need to get a list of all of the files that we
	// want to generate.
	//
	// We could import sourcegraph.com/sourcegraph/srclib, but I
	// want to demonstrate how to use its command line interface.
	var dir string
	if c.Dir == "" {
		d, err := os.Getwd()
		if err != nil {
			log.Fatal(err)
		}
		dir = d
	} else {
		d, err := filepath.Abs(c.Dir)
		if err != nil {
			log.Fatal(err)
		}
		dir = d
	}
	argv := []string{"src", "api", "units", dir}
	cmd, stdout, stderr := command(argv)
	vLogf("Running %v", argv)
	if err := cmd.Run(); err != nil {
		log.Fatal(failedCmd{argv, []interface{}{err, stdout.String(), stderr.String()}})
	}
	if stdout.Len() == 0 {
		log.Fatal(failedCmd{argv, "no output"})
	}
	var us units
	if err := json.Unmarshal(stdout.Bytes(), &us); err != nil {
		log.Fatal(err)
	}
	return genSite(dir, c.SiteDirName, us.collateFiles())
}

type doc struct {
	Format string
	Data   string
	Start  uint32
	End    uint32
}

type docs []doc

func (d docs) Len() int      { return len(d) }
func (d docs) Swap(i, j int) { d[i], d[j] = d[j], d[i] }
func (d docs) Less(i, j int) bool {
	return d[i].Start < d[j].Start || (d[i].Start == d[j].Start && d[i].End < d[j].End)
}

var _ sort.Interface = docs{}

type ref struct {
	DefUnit string
	DefPath string
	File    string
	Start   uint32
}

type refs []ref

func (r refs) Len() int           { return len(r) }
func (r refs) Swap(i, j int)      { r[i], r[j] = r[j], r[i] }
func (r refs) Less(i, j int) bool { return r[i].Start < r[j].Start }

var _ sort.Interface = refs{}

func genSite(root, siteName string, files []string) error {
	vLog("Generating Site")
	sitePath := filepath.Join(root, siteName)
	if err := os.MkdirAll(sitePath, 0755); err != nil {
		log.Fatal(err)
	}
	filesMap := map[string]struct{}{}
	for _, f := range files {
		filesMap[f] = struct{}{}
	}
	for _, f := range files {
		vLog("Processing", f)
		src, err := ioutil.ReadFile(filepath.Join(root, f))
		if err != nil {
			return err
		}
		argv := []string{"src", "api", "list", "--file", filepath.Join(root, f), "--no-defs"}
		cmd, stdout, stderr := command(argv)
		vLog("Running", argv)
		if err := cmd.Run(); err != nil {
			return failedCmd{argv, []interface{}{err, stdout.String(), stderr.String()}}
		}
		out := struct {
			Refs []ref
			Docs []doc
		}{}
		if err := json.Unmarshal(stdout.Bytes(), &out); err != nil {
			log.Fatal(err)
		}

		seenHTMLDoc := make(map[struct{ start, end uint32 }]struct{})
		var htmlDocs []doc
		for _, d := range out.Docs {
			if d.Format == "text/html" {
				if _, seen := seenHTMLDoc[struct{ start, end uint32 }{d.Start, d.End}]; !seen {
					htmlDocs = append(htmlDocs, d)
					seenHTMLDoc[struct{ start, end uint32 }{d.Start, d.End}] = struct{}{}
				}
			}
		}
		sort.Sort(docs(htmlDocs))
		sort.Sort(refs(out.Refs))
		anns, err := ann(src, out.Refs, f, filesMap)
		if err != nil {
			return err
		}
		htmlFile := htmlFilename(f)
		vLogf("Creating dir %s", filepath.Dir(filepath.Join(sitePath, htmlFile)))
		if err := os.MkdirAll(filepath.Dir(filepath.Join(sitePath, htmlFile)), 0755); err != nil {
			log.Fatal(err)
		}
		s, err := createSegments(src, anns, htmlDocs)
		if err != nil {
			return err
		}
		vLogf("Creating file %s", filepath.Join(sitePath, htmlFile))
		w, err := os.Create(filepath.Join(sitePath, htmlFile))
		if err != nil {
			return err
		}
		if err := codeTemplate.Execute(w, HTMLOutput{f, s}); err != nil {
			return err
		}
	}
	return nil
}

type HTMLOutput struct {
	Title    string
	Segments []segment
}

var codeTemplate = template.Must(template.New("code").Parse(codeText))

var codeText = `
<!DOCTYPE html>

    <title>{{.Title}}</title>
    <link rel="stylesheet" href="https://maxcdn.bootstrapcdn.com/bootstrap/3.3.2/css/bootstrap.min.css">

      {{ range .Segments}}

{{.DocHTML}}

{{.CodeHTML}}

      {{ end }}

`

type annotation struct {
	annotate.Annotation
	Comment bool
}

type htmlAnnotator syntaxhighlight.HTMLConfig

func (c htmlAnnotator) class(kind syntaxhighlight.Kind) string {
	switch kind {
	case syntaxhighlight.String:
		return c.String
	case syntaxhighlight.Keyword:
		return c.Keyword
	case syntaxhighlight.Comment:
		return c.Comment
	case syntaxhighlight.Type:
		return c.Type
	case syntaxhighlight.Literal:
		return c.Literal
	case syntaxhighlight.Punctuation:
		return c.Punctuation
	case syntaxhighlight.Plaintext:
		return c.Plaintext
	case syntaxhighlight.Tag:
		return c.Tag
	case syntaxhighlight.HTMLTag:
		return c.HTMLTag
	case syntaxhighlight.HTMLAttrName:
		return c.HTMLAttrName
	case syntaxhighlight.HTMLAttrValue:
		return c.HTMLAttrValue
	case syntaxhighlight.Decimal:
		return c.Decimal
	}
	return ""
}

func (a htmlAnnotator) Annotate(start int, kind syntaxhighlight.Kind, tokText string) (*annotate.Annotation, error) {
	class := a.class(kind)
	if class == "" {
		return nil, nil
	}
	return &annotate.Annotation{
		Start: start,
		End:   start + len(tokText),
		Left:  []byte(class),
		Right: nil,
	}, nil
}

func htmlFilename(filename string) string {
	return filepath.Join("/", filename+".html")
}

func ann(src []byte, refs []ref, filename string, filesMap map[string]struct{}) ([]annotation, error) {
	vLog("Annotating", filename)
	annotations, err := syntaxhighlight.Annotate(src, htmlAnnotator(syntaxhighlight.DefaultHTMLConfig))
	if err != nil {
		return nil, err
	}
	sort.Sort(annotations)

	var refAtIndex int
	refAt := func(start uint32) (r ref, found bool) {
		for refAtIndex < len(refs) {
			if refs[refAtIndex].Start == start {
				defer func() { refAtIndex++ }()
				return refs[refAtIndex], true
			} else if refs[refAtIndex].Start < start {
				refAtIndex++
			} else { // refs[refAtIndex].Start > start
				return ref{}, false
			}
		}
		return ref{}, false
	}

	anns := make([]annotation, 0, len(annotations))
	for _, a := range annotations {
		r, found := refAt(uint32(a.Start))
		if !found {
			a.Left = []byte(fmt.Sprintf(`<span class="%s">`, string(a.Left)))
			a.Right = []byte(`</span>`)
			anns = append(anns, annotation{*a, false})
			continue
		}
		id := filepath.Join(r.DefUnit, r.DefPath)
		var href string
		if _, in := filesMap[r.File]; in {
			href = htmlFilename(r.File) + "#" + id
			a.Left = []byte(fmt.Sprintf(
				`<span class="%s" id="%s"><a href="%s">`,
				string(a.Left),
				id,
				href,
			))
			a.Right = []byte(`</span></a>`)
		} else {
			a.Left = []byte(fmt.Sprintf(
				`<span class="%s" id="%s">`,
				string(a.Left),
				id,
			))
			a.Right = []byte(`</span>`)
		}
		anns = append(anns, annotation{*a, false})
	}

	return anns, nil
}

type segment struct {
	DocHTML  string
	CodeHTML string
}

// anns and docs must be sorted.
func createSegments(src []byte, anns []annotation, docs []doc) ([]segment, error) {
	vLog("Creating segments")
	var segments []segment
	var s segment
	for i := 0; i < len(src); {
		for len(docs) != 0 && docs[0].Start == uint32(i) {
			// Add doc
			s.DocHTML = docs[0].Data
			i = int(docs[0].End)
			docs = docs[1:]
		}
		var runTo int
		if len(docs) != 0 {
			runTo = int(docs[0].Start)
		} else {
			runTo = len(src)
		}
		for len(anns) != 0 && i > anns[0].Start {
			anns = anns[1:]
		}
		for src[i] == '\n' {
			i++
		}
		for i < runTo {
			if len(anns) == 0 {
				s.CodeHTML += template.HTMLEscapeString(string(src[i:runTo]))
				i = runTo
				break
			}
			a := anns[0]
			if i  runTo {
				log.Fatal("createSegment: illegal state: a.End > runTo")
			}
			s.CodeHTML += string(a.Left) +
				template.HTMLEscapeString(string(src[a.Start:a.End])) +
				string(a.Right)
			i = a.End
			anns = anns[1:]
		}
		segments = append(segments, s)
		s = segment{}
	}
	return segments, nil
}

func Main() error {
	log.SetFlags(log.Lshortfile)
	_, err := CLI.Parse()
	return err
}
<commit_msg>skip ids that have been seen<commit_after>// srclib-docco is a docco-like static documentation generator.
// TODO: write this in a literal style.
package srclib_docco

import (
	"bytes"
	"encoding/json"
	"fmt"
	"io/ioutil"
	"log"
	"os"
	"os/exec"
	"path/filepath"
	"sort"
	"text/template"

	"github.com/jessevdk/go-flags"
	"github.com/sourcegraph/annotate"
	"github.com/sourcegraph/syntaxhighlight"
)

var CLI = flags.NewNamedParser("src-docco", flags.Default)

var GlobalOpt struct {
	Verbose bool `short:"v" description:"show verbose output"`
}

var vLogger = log.New(os.Stderr, "", 0)

func vLogf(format string, v ...interface{}) {
	if !GlobalOpt.Verbose {
		return
	}
	vLogger.Printf(format, v...)
}

func vLog(v ...interface{}) {
	if !GlobalOpt.Verbose {
		return
	}
	vLogger.Println(v...)
}

func init() {
	CLI.LongDescription = "TODO"
	CLI.AddGroup("Global options", "", &GlobalOpt)

	_, err := CLI.AddCommand("gen",
		"generate documentation",
		"Generate docco-like documentation for a thing",
		&genCmd,
	)
	if err != nil {
		log.Fatal(err)
	}
}

// TODO: add limitations help output.

type GenCmd struct {
	Dir string `long:"dir" description:"The root directory for the project"`
	// TODO: must be relative to Dir.
	SiteDirName string `long:"site-dir" description:"The directory name for the output files" default:"site"`
}

var genCmd GenCmd

// Source units have lots of information associated with them, but we
// only care about the files.
type unit struct {
	Files []string
}

type units []unit

func (us units) collateFiles() []string {
	var fs []string
	for _, u := range us {
		for _, f := range u.Files {
			fs = append(fs, f)
		}
	}
	return fs
}

type failedCmd struct {
	cmd interface{}
	err interface{}
}

// TODO: Is this correct?
func (f failedCmd) Error() string {
	return fmt.Sprintf("command %v failed: %s", f.cmd, f.err)
}

func ensureSrclibExists() error {
	cmd := exec.Command("src", "version")
	stdout, stderr := &bytes.Buffer{}, &bytes.Buffer{}
	cmd.Stdout, cmd.Stderr = stdout, stderr
	if err := cmd.Run(); err != nil {
		return fmt.Errorf(
			"error with srclib: %v, %s, %s",
			err,
			stdout.String(),
			stderr.String(),
		)
	}
	return nil
}

func command(argv []string) (cmd *exec.Cmd, stdout *bytes.Buffer, stderr *bytes.Buffer) {
	if len(argv) == 0 {
		panic("command: argv must have at least one item")
	}
	cmd = exec.Command(argv[0], argv[1:]...)
	stdout, stderr = &bytes.Buffer{}, &bytes.Buffer{}
	cmd.Stdout, cmd.Stderr = stdout, stderr
	return cmd, stdout, stderr
}

func (c *GenCmd) Execute(args []string) error {
	if err := ensureSrclibExists(); err != nil {
		log.Fatal(err)
	}

	// First, we need to get a list of all of the files that we
	// want to generate.
	//
	// We could import sourcegraph.com/sourcegraph/srclib, but I
	// want to demonstrate how to use its command line interface.
	var dir string
	if c.Dir == "" {
		d, err := os.Getwd()
		if err != nil {
			log.Fatal(err)
		}
		dir = d
	} else {
		d, err := filepath.Abs(c.Dir)
		if err != nil {
			log.Fatal(err)
		}
		dir = d
	}
	argv := []string{"src", "api", "units", dir}
	cmd, stdout, stderr := command(argv)
	vLogf("Running %v", argv)
	if err := cmd.Run(); err != nil {
		log.Fatal(failedCmd{argv, []interface{}{err, stdout.String(), stderr.String()}})
	}
	if stdout.Len() == 0 {
		log.Fatal(failedCmd{argv, "no output"})
	}
	var us units
	if err := json.Unmarshal(stdout.Bytes(), &us); err != nil {
		log.Fatal(err)
	}
	return genSite(dir, c.SiteDirName, us.collateFiles())
}

type doc struct {
	Format string
	Data   string
	Start  uint32
	End    uint32
}

type docs []doc

func (d docs) Len() int      { return len(d) }
func (d docs) Swap(i, j int) { d[i], d[j] = d[j], d[i] }
func (d docs) Less(i, j int) bool {
	return d[i].Start < d[j].Start || (d[i].Start == d[j].Start && d[i].End < d[j].End)
}

var _ sort.Interface = docs{}

type ref struct {
	DefUnit string
	DefPath string
	File    string
	Start   uint32
}

type refs []ref

func (r refs) Len() int           { return len(r) }
func (r refs) Swap(i, j int)      { r[i], r[j] = r[j], r[i] }
func (r refs) Less(i, j int) bool { return r[i].Start < r[j].Start }

var _ sort.Interface = refs{}

func genSite(root, siteName string, files []string) error {
	vLog("Generating Site")
	sitePath := filepath.Join(root, siteName)
	if err := os.MkdirAll(sitePath, 0755); err != nil {
		log.Fatal(err)
	}
	filesMap := map[string]struct{}{}
	for _, f := range files {
		filesMap[f] = struct{}{}
	}
	for _, f := range files {
		vLog("Processing", f)
		src, err := ioutil.ReadFile(filepath.Join(root, f))
		if err != nil {
			return err
		}
		argv := []string{"src", "api", "list", "--file", filepath.Join(root, f), "--no-defs"}
		cmd, stdout, stderr := command(argv)
		vLog("Running", argv)
		if err := cmd.Run(); err != nil {
			return failedCmd{argv, []interface{}{err, stdout.String(), stderr.String()}}
		}
		out := struct {
			Refs []ref
			Docs []doc
		}{}
		if err := json.Unmarshal(stdout.Bytes(), &out); err != nil {
			log.Fatal(err)
		}

		seenHTMLDoc := make(map[struct{ start, end uint32 }]struct{})
		var htmlDocs []doc
		for _, d := range out.Docs {
			if d.Format == "text/html" {
				if _, seen := seenHTMLDoc[struct{ start, end uint32 }{d.Start, d.End}]; !seen {
					htmlDocs = append(htmlDocs, d)
					seenHTMLDoc[struct{ start, end uint32 }{d.Start, d.End}] = struct{}{}
				}
			}
		}
		sort.Sort(docs(htmlDocs))
		sort.Sort(refs(out.Refs))
		anns, err := ann(src, out.Refs, f, filesMap)
		if err != nil {
			return err
		}
		htmlFile := htmlFilename(f)
		vLogf("Creating dir %s", filepath.Dir(filepath.Join(sitePath, htmlFile)))
		if err := os.MkdirAll(filepath.Dir(filepath.Join(sitePath, htmlFile)), 0755); err != nil {
			log.Fatal(err)
		}
		s, err := createSegments(src, anns, htmlDocs)
		if err != nil {
			return err
		}
		vLogf("Creating file %s", filepath.Join(sitePath, htmlFile))
		w, err := os.Create(filepath.Join(sitePath, htmlFile))
		if err != nil {
			return err
		}
		if err := codeTemplate.Execute(w, HTMLOutput{f, s}); err != nil {
			return err
		}
	}
	return nil
}

type HTMLOutput struct {
	Title    string
	Segments []segment
}

var codeTemplate = template.Must(template.New("code").Parse(codeText))

var codeText = `
<!DOCTYPE html>

    <title>{{.Title}}</title>
    <link rel="stylesheet" href="https://maxcdn.bootstrapcdn.com/bootstrap/3.3.2/css/bootstrap.min.css">

      {{ range .Segments}}

{{.DocHTML}}

{{.CodeHTML}}

      {{ end }}

`

type annotation struct {
	annotate.Annotation
	Comment bool
}

type htmlAnnotator syntaxhighlight.HTMLConfig

func (c htmlAnnotator) class(kind syntaxhighlight.Kind) string {
	switch kind {
	case syntaxhighlight.String:
		return c.String
	case syntaxhighlight.Keyword:
		return c.Keyword
	case syntaxhighlight.Comment:
		return c.Comment
	case syntaxhighlight.Type:
		return c.Type
	case syntaxhighlight.Literal:
		return c.Literal
	case syntaxhighlight.Punctuation:
		return c.Punctuation
	case syntaxhighlight.Plaintext:
		return c.Plaintext
	case syntaxhighlight.Tag:
		return c.Tag
	case syntaxhighlight.HTMLTag:
		return c.HTMLTag
	case syntaxhighlight.HTMLAttrName:
		return c.HTMLAttrName
	case syntaxhighlight.HTMLAttrValue:
		return c.HTMLAttrValue
	case syntaxhighlight.Decimal:
		return c.Decimal
	}
	return ""
}

func (a htmlAnnotator) Annotate(start int, kind syntaxhighlight.Kind, tokText string) (*annotate.Annotation, error) {
	class := a.class(kind)
	if class == "" {
		return nil, nil
	}
	return &annotate.Annotation{
		Start: start,
		End:   start + len(tokText),
		Left:  []byte(class),
		Right: nil,
	}, nil
}

func htmlFilename(filename string) string {
	return filepath.Join("/", filename+".html")
}

func ann(src []byte, refs []ref, filename string, filesMap map[string]struct{}) ([]annotation, error) {
	vLog("Annotating", filename)
	annotations, err := syntaxhighlight.Annotate(src, htmlAnnotator(syntaxhighlight.DefaultHTMLConfig))
	if err != nil {
		return nil, err
	}
	sort.Sort(annotations)

	var refAtIndex int
	refAt := func(start uint32) (r ref, found bool) {
		for refAtIndex < len(refs) {
			if refs[refAtIndex].Start == start {
				defer func() { refAtIndex++ }()
				return refs[refAtIndex], true
			} else if refs[refAtIndex].Start < start {
				refAtIndex++
			} else { // refs[refAtIndex].Start > start
				return ref{}, false
			}
		}
		return ref{}, false
	}

	idSeen := map[string]bool{}
	anns := make([]annotation, 0, len(annotations))
	for _, a := range annotations {
		r, found := refAt(uint32(a.Start))
		if !found {
			a.Left = []byte(fmt.Sprintf(`<span class="%s">`, string(a.Left)))
			a.Right = []byte(`</span>`)
			anns = append(anns, annotation{*a, false})
			continue
		}
		if _, in := filesMap[r.File]; in {
			id := filepath.Join(r.DefUnit, r.DefPath)
			href := htmlFilename(r.File) + "#" + id
			if idSeen[id] {
				a.Left = []byte(fmt.Sprintf(
					`<span class="%s"><a href="%s">`,
					string(a.Left),
					href,
				))
			} else {
				a.Left = []byte(fmt.Sprintf(
					`<span class="%s" id="%s"><a href="%s">`,
					string(a.Left),
					id,
					href,
				))
			}
			a.Right = []byte(`</span></a>`)
		} else {
			a.Left = []byte(fmt.Sprintf(`<span class="%s">`, string(a.Left)))
			a.Right = []byte(`</span>`)
		}
		anns = append(anns, annotation{*a, false})
	}

	return anns, nil
}

type segment struct {
	DocHTML  string
	CodeHTML string
}

// anns and docs must be sorted.
func createSegments(src []byte, anns []annotation, docs []doc) ([]segment, error) {
	vLog("Creating segments")
	var segments []segment
	var s segment
	for i := 0; i < len(src); {
		for len(docs) != 0 && docs[0].Start == uint32(i) {
			// Add doc
			s.DocHTML = docs[0].Data
			i = int(docs[0].End)
			docs = docs[1:]
		}
		var runTo int
		if len(docs) != 0 {
			runTo = int(docs[0].Start)
		} else {
			runTo = len(src)
		}
		for len(anns) != 0 && i > anns[0].Start {
			anns = anns[1:]
		}
		for src[i] == '\n' {
			i++
		}
		for i < runTo {
			if len(anns) == 0 {
				s.CodeHTML += template.HTMLEscapeString(string(src[i:runTo]))
				i = runTo
				break
			}
			a := anns[0]
			if i  runTo {
				log.Fatal("createSegment: illegal state: a.End > runTo")
			}
			s.CodeHTML += string(a.Left) +
				template.HTMLEscapeString(string(src[a.Start:a.End])) +
				string(a.Right)
			i = a.End
			anns = anns[1:]
		}
		segments = append(segments, s)
		s = segment{}
	}
	return segments, nil
}

func Main() error {
	log.SetFlags(log.Lshortfile)
	_, err := CLI.Parse()
	return err
}
<|endoftext|>


<commit_before>/*
Copyright 2015 The Kubernetes Authors All rights reserved.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package e2e

import (
	"fmt"
	"os"
	"sort"
	"strconv"
	"sync"
	"time"

	"k8s.io/kubernetes/pkg/api"
	unversioned "k8s.io/kubernetes/pkg/api/unversioned"
	"k8s.io/kubernetes/pkg/client/cache"
	client "k8s.io/kubernetes/pkg/client/unversioned"
	"k8s.io/kubernetes/pkg/controller/framework"
	"k8s.io/kubernetes/pkg/fields"
	"k8s.io/kubernetes/pkg/labels"
	"k8s.io/kubernetes/pkg/runtime"
	"k8s.io/kubernetes/pkg/util"
	"k8s.io/kubernetes/pkg/watch"

	. "github.com/onsi/ginkgo"
	. "github.com/onsi/gomega"
)

var _ = Describe("Latency [Skipped]", func() {
	var c *client.Client
	var nodeCount int
	var additionalPodsPrefix string
	var ns string
	var uuid string

	AfterEach(func() {
		By("Removing additional pods if any")
		for i := 1; i <= nodeCount; i++ {
			name := additionalPodsPrefix + "-" + strconv.Itoa(i)
			c.Pods(ns).Delete(name, nil)
		}

		By(fmt.Sprintf("Destroying namespace for this suite %v", ns))
		if err := c.Namespaces().Delete(ns); err != nil {
			Failf("Couldn't delete ns %s", err)
		}

		expectNoError(writePerfData(c, fmt.Sprintf(testContext.OutputDir+"/%s", uuid), "after"))

		// Verify latency metrics
		highLatencyRequests, err := HighLatencyRequests(c)
		expectNoError(err)
		Expect(highLatencyRequests).NotTo(BeNumerically(">", 0), "There should be no high-latency requests")
	})

	framework := NewFramework("latency")
	framework.NamespaceDeletionTimeout = time.Hour

	BeforeEach(func() {
		c = framework.Client
		ns = framework.Namespace.Name
		var err error

		nodes, err := c.Nodes().List(labels.Everything(), fields.Everything())
		expectNoError(err)
		nodeCount = len(nodes.Items)
		Expect(nodeCount).NotTo(BeZero())

		// Terminating a namespace (deleting the remaining objects from it - which
		// generally means events) can affect the current run. Thus we wait for all
		// terminating namespace to be finally deleted before starting this test.
		expectNoError(checkTestingNSDeletedExcept(c, ns))

		uuid = string(util.NewUUID())

		expectNoError(resetMetrics(c))
		expectNoError(os.Mkdir(fmt.Sprintf(testContext.OutputDir+"/%s", uuid), 0777))
		expectNoError(writePerfData(c, fmt.Sprintf(testContext.OutputDir+"/%s", uuid), "before"))

		Logf("Listing nodes for easy debugging:\n")
		for _, node := range nodes.Items {
			for _, address := range node.Status.Addresses {
				if address.Type == api.NodeInternalIP {
					Logf("Name: %v IP: %v", node.ObjectMeta.Name, address.Address)
				}
			}
		}
	})

	It("pod start latency should be acceptable", func() {
		runLatencyTest(nodeCount, c, ns)
	})
})

func runLatencyTest(nodeCount int, c *client.Client, ns string) {
	var (
		nodes              = make(map[string]string, 0)           // pod name -> node name
		createTimestamps   = make(map[string]unversioned.Time, 0) // pod name -> create time
		scheduleTimestamps = make(map[string]unversioned.Time, 0) // pod name -> schedule time
		startTimestamps    = make(map[string]unversioned.Time, 0) // pod name -> time to run
		watchTimestamps    = make(map[string]unversioned.Time, 0) // pod name -> time to read from informer

		additionalPodsPrefix = "latency-pod-" + string(util.NewUUID())
	)

	var mutex sync.Mutex
	readPodInfo := func(p *api.Pod) {
		mutex.Lock()
		defer mutex.Unlock()
		defer GinkgoRecover()

		if p.Status.Phase == api.PodRunning {
			if _, found := watchTimestamps[p.Name]; !found {
				watchTimestamps[p.Name] = unversioned.Now()
				createTimestamps[p.Name] = p.CreationTimestamp
				nodes[p.Name] = p.Spec.NodeName
				var startTimestamp unversioned.Time
				for _, cs := range p.Status.ContainerStatuses {
					if cs.State.Running != nil {
						if startTimestamp.Before(cs.State.Running.StartedAt) {
							startTimestamp = cs.State.Running.StartedAt
						}
					}
				}
				if startTimestamp != unversioned.NewTime(time.Time{}) {
					startTimestamps[p.Name] = startTimestamp
				} else {
					Failf("Pod %v is reported to be running, but none of its containers are", p.Name)
				}
			}
		}
	}

	// Create a informer to read timestamps for each pod
	stopCh := make(chan struct{})
	_, informer := framework.NewInformer(
		&cache.ListWatch{
			ListFunc: func() (runtime.Object, error) {
				return c.Pods(ns).List(labels.SelectorFromSet(labels.Set{"name": additionalPodsPrefix}), fields.Everything())
			},
			WatchFunc: func(options api.ListOptions) (watch.Interface, error) {
				return c.Pods(ns).Watch(labels.SelectorFromSet(labels.Set{"name": additionalPodsPrefix}), fields.Everything(), options)
			},
		},
		&api.Pod{},
		0,
		framework.ResourceEventHandlerFuncs{
			AddFunc: func(obj interface{}) {
				p, ok := obj.(*api.Pod)
				Expect(ok).To(Equal(true))
				go readPodInfo(p)
			},
			UpdateFunc: func(oldObj, newObj interface{}) {
				p, ok := newObj.(*api.Pod)
				Expect(ok).To(Equal(true))
				go readPodInfo(p)
			},
		},
	)
	go informer.Run(stopCh)

	// Create  additional pods with throughput ~5 pods/sec.
	var wg sync.WaitGroup
	wg.Add(nodeCount)
	podLabels := map[string]string{
		"name": additionalPodsPrefix,
	}
	for i := 1; i <= nodeCount; i++ {
		name := additionalPodsPrefix + "-" + strconv.Itoa(i)
		go createRunningPod(&wg, c, name, ns, "gcr.io/google_containers/pause:go", podLabels)
		time.Sleep(200 * time.Millisecond)
	}
	wg.Wait()

	Logf("Waiting for all Pods begin observed by the watch...")
	for start := time.Now(); len(watchTimestamps) < nodeCount; time.Sleep(10 * time.Second) {
		if time.Since(start) < timeout {
			Failf("Timeout reached waiting for all Pods being observed by the watch.")
		}
	}
	close(stopCh)

	// Read the schedule timestamp by checking the scheduler event for each pod
	schedEvents, err := c.Events(ns).List(
		labels.Everything(),
		fields.Set{
			"involvedObject.kind":      "Pod",
			"involvedObject.namespace": ns,
			"source":                   "scheduler",
		}.AsSelector())
	expectNoError(err)
	for k := range createTimestamps {
		for _, event := range schedEvents.Items {
			if event.InvolvedObject.Name == k {
				scheduleTimestamps[k] = event.FirstTimestamp
				break
			}
		}
	}

	var (
		scheduleLatencies        = make([]podLatencyData, 0)
		startLatencies           = make([]podLatencyData, 0)
		watchLatencies           = make([]podLatencyData, 0)
		scheduleToWatchLatencies = make([]podLatencyData, 0)
		e2eLatencies             = make([]podLatencyData, 0)
	)

	for name, podNode := range nodes {
		createTs, ok := createTimestamps[name]
		Expect(ok).To(Equal(true))
		scheduleTs, ok := scheduleTimestamps[name]
		Expect(ok).To(Equal(true))
		runTs, ok := startTimestamps[name]
		Expect(ok).To(Equal(true))
		watchTs, ok := watchTimestamps[name]
		Expect(ok).To(Equal(true))

		var (
			scheduleLatency        = podLatencyData{name, podNode, scheduleTs.Time.Sub(createTs.Time)}
			startLatency           = podLatencyData{name, podNode, runTs.Time.Sub(scheduleTs.Time)}
			watchLatency           = podLatencyData{name, podNode, watchTs.Time.Sub(runTs.Time)}
			scheduleToWatchLatency = podLatencyData{name, podNode, watchTs.Time.Sub(scheduleTs.Time)}
			e2eLatency             = podLatencyData{name, podNode, watchTs.Time.Sub(createTs.Time)}
		)

		scheduleLatencies = append(scheduleLatencies, scheduleLatency)
		startLatencies = append(startLatencies, startLatency)
		watchLatencies = append(watchLatencies, watchLatency)
		scheduleToWatchLatencies = append(scheduleToWatchLatencies, scheduleToWatchLatency)
		e2eLatencies = append(e2eLatencies, e2eLatency)
	}

	sort.Sort(latencySlice(scheduleLatencies))
	sort.Sort(latencySlice(startLatencies))
	sort.Sort(latencySlice(watchLatencies))
	sort.Sort(latencySlice(scheduleToWatchLatencies))
	sort.Sort(latencySlice(e2eLatencies))

	printLatencies(scheduleLatencies, "worst schedule latencies")
	printLatencies(startLatencies, "worst run-after-schedule latencies")
	printLatencies(watchLatencies, "worst watch latencies")
	printLatencies(scheduleToWatchLatencies, "worst scheduled-to-end total latencies")
	printLatencies(e2eLatencies, "worst e2e total latencies")

	// Ensure all scheduleLatencies are under expected ceilings.
	// These numbers were guessed based on numerous Jenkins e2e runs.
	testMaximumLatencyValue(scheduleLatencies, 1*time.Second, "scheduleLatencies")
	testMaximumLatencyValue(startLatencies, 15*time.Second, "startLatencies")
	testMaximumLatencyValue(watchLatencies, 8*time.Second, "watchLatencies")
	testMaximumLatencyValue(scheduleToWatchLatencies, 5*time.Second, "scheduleToWatchLatencies")
	testMaximumLatencyValue(e2eLatencies, 5*time.Second, "e2eLatencies")

	// Test whether e2e pod startup time is acceptable.
	podStartupLatency := PodStartupLatency{Latency: extractLatencyMetrics(e2eLatencies)}
	expectNoError(VerifyPodStartupLatency(podStartupLatency))

	// Log suspicious latency metrics/docker errors from all nodes that had slow startup times
	logSuspiciousLatency(startLatencies, nil, nodeCount, c)
}
<commit_msg>Fix latency.go e2e test<commit_after>/*
Copyright 2015 The Kubernetes Authors All rights reserved.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package e2e

import (
	"fmt"
	"os"
	"sort"
	"strconv"
	"sync"
	"time"

	"k8s.io/kubernetes/pkg/api"
	unversioned "k8s.io/kubernetes/pkg/api/unversioned"
	"k8s.io/kubernetes/pkg/client/cache"
	client "k8s.io/kubernetes/pkg/client/unversioned"
	"k8s.io/kubernetes/pkg/controller/framework"
	"k8s.io/kubernetes/pkg/fields"
	"k8s.io/kubernetes/pkg/labels"
	"k8s.io/kubernetes/pkg/runtime"
	"k8s.io/kubernetes/pkg/util"
	"k8s.io/kubernetes/pkg/watch"

	. "github.com/onsi/ginkgo"
	. "github.com/onsi/gomega"
)

var _ = Describe("Latency [Skipped]", func() {
	var c *client.Client
	var nodeCount int
	var additionalPodsPrefix string
	var ns string
	var uuid string

	AfterEach(func() {
		By("Removing additional pods if any")
		for i := 1; i <= nodeCount; i++ {
			name := additionalPodsPrefix + "-" + strconv.Itoa(i)
			c.Pods(ns).Delete(name, nil)
		}

		expectNoError(writePerfData(c, fmt.Sprintf(testContext.OutputDir+"/%s", uuid), "after"))

		// Verify latency metrics
		highLatencyRequests, err := HighLatencyRequests(c)
		expectNoError(err)
		Expect(highLatencyRequests).NotTo(BeNumerically(">", 0), "There should be no high-latency requests")
	})

	framework := NewFramework("latency")
	framework.NamespaceDeletionTimeout = time.Hour

	BeforeEach(func() {
		c = framework.Client
		ns = framework.Namespace.Name
		var err error

		nodes, err := c.Nodes().List(labels.Everything(), fields.Everything())
		expectNoError(err)
		nodeCount = len(nodes.Items)
		Expect(nodeCount).NotTo(BeZero())

		// Terminating a namespace (deleting the remaining objects from it - which
		// generally means events) can affect the current run. Thus we wait for all
		// terminating namespace to be finally deleted before starting this test.
		expectNoError(checkTestingNSDeletedExcept(c, ns))

		uuid = string(util.NewUUID())

		expectNoError(resetMetrics(c))
		expectNoError(os.Mkdir(fmt.Sprintf(testContext.OutputDir+"/%s", uuid), 0777))
		expectNoError(writePerfData(c, fmt.Sprintf(testContext.OutputDir+"/%s", uuid), "before"))

		Logf("Listing nodes for easy debugging:\n")
		for _, node := range nodes.Items {
			for _, address := range node.Status.Addresses {
				if address.Type == api.NodeInternalIP {
					Logf("Name: %v IP: %v", node.ObjectMeta.Name, address.Address)
				}
			}
		}
	})

	It("pod start latency should be acceptable", func() {
		runLatencyTest(nodeCount, c, ns)
	})
})

func runLatencyTest(nodeCount int, c *client.Client, ns string) {
	var (
		nodes              = make(map[string]string, 0)           // pod name -> node name
		createTimestamps   = make(map[string]unversioned.Time, 0) // pod name -> create time
		scheduleTimestamps = make(map[string]unversioned.Time, 0) // pod name -> schedule time
		startTimestamps    = make(map[string]unversioned.Time, 0) // pod name -> time to run
		watchTimestamps    = make(map[string]unversioned.Time, 0) // pod name -> time to read from informer

		additionalPodsPrefix = "latency-pod-" + string(util.NewUUID())
	)

	var mutex sync.Mutex
	readPodInfo := func(p *api.Pod) {
		mutex.Lock()
		defer mutex.Unlock()
		defer GinkgoRecover()

		if p.Status.Phase == api.PodRunning {
			if _, found := watchTimestamps[p.Name]; !found {
				watchTimestamps[p.Name] = unversioned.Now()
				createTimestamps[p.Name] = p.CreationTimestamp
				nodes[p.Name] = p.Spec.NodeName
				var startTimestamp unversioned.Time
				for _, cs := range p.Status.ContainerStatuses {
					if cs.State.Running != nil {
						if startTimestamp.Before(cs.State.Running.StartedAt) {
							startTimestamp = cs.State.Running.StartedAt
						}
					}
				}
				if startTimestamp != unversioned.NewTime(time.Time{}) {
					startTimestamps[p.Name] = startTimestamp
				} else {
					Failf("Pod %v is reported to be running, but none of its containers are", p.Name)
				}
			}
		}
	}

	// Create a informer to read timestamps for each pod
	stopCh := make(chan struct{})
	_, informer := framework.NewInformer(
		&cache.ListWatch{
			ListFunc: func() (runtime.Object, error) {
				return c.Pods(ns).List(labels.SelectorFromSet(labels.Set{"name": additionalPodsPrefix}), fields.Everything())
			},
			WatchFunc: func(options api.ListOptions) (watch.Interface, error) {
				return c.Pods(ns).Watch(labels.SelectorFromSet(labels.Set{"name": additionalPodsPrefix}), fields.Everything(), options)
			},
		},
		&api.Pod{},
		0,
		framework.ResourceEventHandlerFuncs{
			AddFunc: func(obj interface{}) {
				p, ok := obj.(*api.Pod)
				Expect(ok).To(Equal(true))
				go readPodInfo(p)
			},
			UpdateFunc: func(oldObj, newObj interface{}) {
				p, ok := newObj.(*api.Pod)
				Expect(ok).To(Equal(true))
				go readPodInfo(p)
			},
		},
	)
	go informer.Run(stopCh)

	// Create  additional pods with throughput ~5 pods/sec.
	var wg sync.WaitGroup
	wg.Add(nodeCount)
	podLabels := map[string]string{
		"name": additionalPodsPrefix,
	}
	for i := 1; i <= nodeCount; i++ {
		name := additionalPodsPrefix + "-" + strconv.Itoa(i)
		go createRunningPod(&wg, c, name, ns, "gcr.io/google_containers/pause:go", podLabels)
		time.Sleep(200 * time.Millisecond)
	}
	wg.Wait()

	Logf("Waiting for all Pods begin observed by the watch...")
	for start := time.Now(); len(watchTimestamps) < nodeCount; time.Sleep(10 * time.Second) {
		if time.Since(start) < timeout {
			Failf("Timeout reached waiting for all Pods being observed by the watch.")
		}
	}
	close(stopCh)

	// Read the schedule timestamp by checking the scheduler event for each pod
	schedEvents, err := c.Events(ns).List(
		labels.Everything(),
		fields.Set{
			"involvedObject.kind":      "Pod",
			"involvedObject.namespace": ns,
			"source":                   "scheduler",
		}.AsSelector())
	expectNoError(err)
	for k := range createTimestamps {
		for _, event := range schedEvents.Items {
			if event.InvolvedObject.Name == k {
				scheduleTimestamps[k] = event.FirstTimestamp
				break
			}
		}
	}

	var (
		scheduleLatencies        = make([]podLatencyData, 0)
		startLatencies           = make([]podLatencyData, 0)
		watchLatencies           = make([]podLatencyData, 0)
		scheduleToWatchLatencies = make([]podLatencyData, 0)
		e2eLatencies             = make([]podLatencyData, 0)
	)

	for name, podNode := range nodes {
		createTs, ok := createTimestamps[name]
		Expect(ok).To(Equal(true))
		scheduleTs, ok := scheduleTimestamps[name]
		Expect(ok).To(Equal(true))
		runTs, ok := startTimestamps[name]
		Expect(ok).To(Equal(true))
		watchTs, ok := watchTimestamps[name]
		Expect(ok).To(Equal(true))

		var (
			scheduleLatency        = podLatencyData{name, podNode, scheduleTs.Time.Sub(createTs.Time)}
			startLatency           = podLatencyData{name, podNode, runTs.Time.Sub(scheduleTs.Time)}
			watchLatency           = podLatencyData{name, podNode, watchTs.Time.Sub(runTs.Time)}
			scheduleToWatchLatency = podLatencyData{name, podNode, watchTs.Time.Sub(scheduleTs.Time)}
			e2eLatency             = podLatencyData{name, podNode, watchTs.Time.Sub(createTs.Time)}
		)

		scheduleLatencies = append(scheduleLatencies, scheduleLatency)
		startLatencies = append(startLatencies, startLatency)
		watchLatencies = append(watchLatencies, watchLatency)
		scheduleToWatchLatencies = append(scheduleToWatchLatencies, scheduleToWatchLatency)
		e2eLatencies = append(e2eLatencies, e2eLatency)
	}

	sort.Sort(latencySlice(scheduleLatencies))
	sort.Sort(latencySlice(startLatencies))
	sort.Sort(latencySlice(watchLatencies))
	sort.Sort(latencySlice(scheduleToWatchLatencies))
	sort.Sort(latencySlice(e2eLatencies))

	printLatencies(scheduleLatencies, "worst schedule latencies")
	printLatencies(startLatencies, "worst run-after-schedule latencies")
	printLatencies(watchLatencies, "worst watch latencies")
	printLatencies(scheduleToWatchLatencies, "worst scheduled-to-end total latencies")
	printLatencies(e2eLatencies, "worst e2e total latencies")

	// Ensure all scheduleLatencies are under expected ceilings.
	// These numbers were guessed based on numerous Jenkins e2e runs.
	testMaximumLatencyValue(scheduleLatencies, 1*time.Second, "scheduleLatencies")
	testMaximumLatencyValue(startLatencies, 15*time.Second, "startLatencies")
	testMaximumLatencyValue(watchLatencies, 8*time.Second, "watchLatencies")
	testMaximumLatencyValue(scheduleToWatchLatencies, 5*time.Second, "scheduleToWatchLatencies")
	testMaximumLatencyValue(e2eLatencies, 5*time.Second, "e2eLatencies")

	// Test whether e2e pod startup time is acceptable.
	podStartupLatency := PodStartupLatency{Latency: extractLatencyMetrics(e2eLatencies)}
	expectNoError(VerifyPodStartupLatency(podStartupLatency))

	// Log suspicious latency metrics/docker errors from all nodes that had slow startup times
	logSuspiciousLatency(startLatencies, nil, nodeCount, c)
}
<|endoftext|>


<commit_before>// Copyright 2018 ETH Zurich
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//   http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package network

import (
	"net"

	"github.com/scionproto/scion/go/godispatcher/internal/metrics"
	"github.com/scionproto/scion/go/godispatcher/internal/respool"
	"github.com/scionproto/scion/go/lib/addr"
	"github.com/scionproto/scion/go/lib/common"
	"github.com/scionproto/scion/go/lib/hpkt"
	"github.com/scionproto/scion/go/lib/l4"
	"github.com/scionproto/scion/go/lib/log"
	"github.com/scionproto/scion/go/lib/ringbuf"
	"github.com/scionproto/scion/go/lib/scmp"
	"github.com/scionproto/scion/go/lib/spkt"
)

const (
	ErrUnsupportedL4              = "unsupported SCION L4 protocol"
	ErrUnsupportedDestination     = "unsupported destination address type"
	ErrUnsupportedSCMPDestination = "unsupported SCMP destination address type"
	ErrUnsupportedQuotedL4Type    = "unsupported quoted L4 protocol type"
	ErrMalformedL4Quote           = "malformed L4 quote"
)

// NetToRingDataplane reads SCION packets from the overlay socket, routes them
// to determine the destination process, and then enqueues the packets on the
// application's ingress ring.
//
// The rings are used to provide non-blocking IO for the overlay receiver.
type NetToRingDataplane struct {
	OverlayConn  net.PacketConn
	RoutingTable *IATable
}

func (dp *NetToRingDataplane) Run() error {
	for {
		pkt := respool.GetPacket()
		// XXX(scrye): we don't release the reference on error conditions, and
		// let the GC take care of this situation as they should be fairly
		// rare.

		if err := pkt.DecodeFromConn(dp.OverlayConn); err != nil {
			log.Warn("error receiving next packet from overlay conn", "err", err)
			continue
		}

		dst, err := ComputeDestination(&pkt.Info)
		if err != nil {
			log.Warn("unable to route packet", "err", err)
			metrics.IncomingPackets.WithLabelValues(metrics.PacketOutcomeRouteNotFound).Inc()
			continue
		}
		metrics.IncomingPackets.WithLabelValues(metrics.PacketOutcomeOk).Inc()
		dst.Send(dp, pkt)
	}
}

func ComputeDestination(packet *spkt.ScnPkt) (Destination, error) {
	switch header := packet.L4.(type) {
	case *l4.UDP:
		return ComputeUDPDestination(packet, header)
	case *scmp.Hdr:
		return ComputeSCMPDestination(packet, header)
	default:
		return nil, common.NewBasicError(ErrUnsupportedL4, nil, "type", header.L4Type())
	}
}

func ComputeUDPDestination(packet *spkt.ScnPkt, header *l4.UDP) (Destination, error) {
	switch packet.DstHost.Type() {
	case addr.HostTypeIPv4, addr.HostTypeIPv6:
		return &UDPDestination{IP: packet.DstHost.IP(), Port: int(header.DstPort)}, nil
	case addr.HostTypeSVC:
		return SVCDestination(packet.DstHost.(addr.HostSVC)), nil
	default:
		return nil, common.NewBasicError(ErrUnsupportedDestination, nil,
			"type", packet.DstHost.Type())
	}
}

func ComputeSCMPDestination(packet *spkt.ScnPkt, header *scmp.Hdr) (Destination, error) {
	if packet.DstHost.Type() != addr.HostTypeIPv4 && packet.DstHost.Type() != addr.HostTypeIPv6 {
		return nil, common.NewBasicError(ErrUnsupportedSCMPDestination, nil,
			"type", packet.DstHost.Type())
	}
	if header.Class == scmp.C_General {
		return ComputeSCMPGeneralDestination(packet, header)
	} else {
		return ComputeSCMPErrorDestination(packet, header)
	}
}

func ComputeSCMPGeneralDestination(s *spkt.ScnPkt, header *scmp.Hdr) (Destination, error) {
	id := getSCMPGeneralID(s)
	if id == 0 {
		return nil, common.NewBasicError("Invalid SCMP ID", nil, "id", id)
	}
	switch {
	case isSCMPGeneralRequest(header):
		invertSCMPGeneralType(header)
		return SCMPHandlerDestination{}, nil
	case isSCMPGeneralReply(header):
		return &SCMPAppDestination{ID: id}, nil
	default:
		return nil, common.NewBasicError("Unsupported SCMP General type", nil, "type", header.Type)
	}
}

func ComputeSCMPErrorDestination(packet *spkt.ScnPkt, header *scmp.Hdr) (Destination, error) {
	scmpPayload := packet.Pld.(*scmp.Payload)
	switch scmpPayload.Meta.L4Proto {
	case common.L4UDP:
		quotedUDPHeader, err := l4.UDPFromRaw(scmpPayload.L4Hdr)
		if err != nil {
			return nil, common.NewBasicError(ErrMalformedL4Quote, nil, "err", err)
		}
		return &UDPDestination{IP: packet.DstHost.IP(), Port: int(quotedUDPHeader.SrcPort)}, nil
	case common.L4SCMP:

		id, err := getQuotedSCMPGeneralID(scmpPayload)
		if id == 0 {
			return nil, common.NewBasicError(ErrMalformedL4Quote, err)
		}
		return &SCMPAppDestination{ID: id}, nil
	default:
		return nil, common.NewBasicError(ErrUnsupportedQuotedL4Type, nil,
			"type", scmpPayload.Meta.L4Proto)
	}
}

type Destination interface {
	// Send takes ownership of pkt, and then sends it to the location described
	// by this destination.
	Send(dp *NetToRingDataplane, pkt *respool.Packet)
}

var _ Destination = (*UDPDestination)(nil)

type UDPDestination net.UDPAddr

func (d *UDPDestination) Send(dp *NetToRingDataplane, pkt *respool.Packet) {
	routingEntry, ok := dp.RoutingTable.LookupPublic(pkt.Info.DstIA, (*net.UDPAddr)(d))
	if !ok {
		log.Warn("destination address not found", "ia", pkt.Info.DstIA,
			"udpAddr", (*net.UDPAddr)(d))
		return
	}
	sendPacket(routingEntry, pkt)
}

var _ Destination = SVCDestination(addr.SvcNone)

type SVCDestination addr.HostSVC

func (d SVCDestination) Send(dp *NetToRingDataplane, pkt *respool.Packet) {
	// FIXME(scrye): This should deliver to the correct IP address, based on
	// information found in the overlay IP header.
	routingEntries := dp.RoutingTable.LookupService(pkt.Info.DstIA, addr.HostSVC(d), nil)
	if len(routingEntries) == 0 {
		log.Warn("destination address not found", "ia", pkt.Info.DstIA, "svc", d)
		return
	}
	// Increase reference count for all extra copies
	for i := 0; i < len(routingEntries)-1; i++ {
		pkt.Dup()
	}
	for _, routingEntry := range routingEntries {
		sendPacket(routingEntry, pkt)
	}
}

var _ Destination = (*SCMPAppDestination)(nil)

type SCMPAppDestination struct {
	ID uint64
}

func (d *SCMPAppDestination) Send(dp *NetToRingDataplane, pkt *respool.Packet) {
	routingEntry, ok := dp.RoutingTable.LookupID(pkt.Info.DstIA, d.ID)
	if !ok {
		log.Warn("destination address not found", "SCMP", d.ID)
		return
	}
	sendPacket(routingEntry, pkt)
}

// sendPacket puts pkt on the routing entry's ring buffer, and releases the
// reference to pkt.
func sendPacket(routingEntry *TableEntry, pkt *respool.Packet) {
	// Move packet reference to other goroutine.
	count, _ := routingEntry.appIngressRing.Write(ringbuf.EntryList{pkt}, false)
	if count <= 0 {
		// Release buffer if we couldn't transmit it to the other goroutine.
		pkt.Free()
	}
}

var _ Destination = (*SCMPHandlerDestination)(nil)

type SCMPHandlerDestination struct{}

func (h SCMPHandlerDestination) Send(dp *NetToRingDataplane, pkt *respool.Packet) {
	if err := pkt.Info.Reverse(); err != nil {
		log.Warn("Unable to reverse SCMP packet.", "err", err)
		return
	}

	b := respool.GetBuffer()
	pkt.Info.HBHExt = removeSCMPHBH(pkt.Info.HBHExt)
	n, err := hpkt.WriteScnPkt(&pkt.Info, b)
	if err != nil {
		log.Warn("Unable to create reply SCMP packet", "err", err)
		return
	}

	_, err = dp.OverlayConn.WriteTo(b[:n], pkt.OverlayRemote)
	if err != nil {
		log.Warn("Unable to write to overlay socket.", "err", err)
		return
	}
	respool.PutBuffer(b)
	pkt.Free()
}
<commit_msg>Print SVC address as string in dispatcher log (#2609)<commit_after>// Copyright 2018 ETH Zurich
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//   http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package network

import (
	"net"

	"github.com/scionproto/scion/go/godispatcher/internal/metrics"
	"github.com/scionproto/scion/go/godispatcher/internal/respool"
	"github.com/scionproto/scion/go/lib/addr"
	"github.com/scionproto/scion/go/lib/common"
	"github.com/scionproto/scion/go/lib/hpkt"
	"github.com/scionproto/scion/go/lib/l4"
	"github.com/scionproto/scion/go/lib/log"
	"github.com/scionproto/scion/go/lib/ringbuf"
	"github.com/scionproto/scion/go/lib/scmp"
	"github.com/scionproto/scion/go/lib/spkt"
)

const (
	ErrUnsupportedL4              = "unsupported SCION L4 protocol"
	ErrUnsupportedDestination     = "unsupported destination address type"
	ErrUnsupportedSCMPDestination = "unsupported SCMP destination address type"
	ErrUnsupportedQuotedL4Type    = "unsupported quoted L4 protocol type"
	ErrMalformedL4Quote           = "malformed L4 quote"
)

// NetToRingDataplane reads SCION packets from the overlay socket, routes them
// to determine the destination process, and then enqueues the packets on the
// application's ingress ring.
//
// The rings are used to provide non-blocking IO for the overlay receiver.
type NetToRingDataplane struct {
	OverlayConn  net.PacketConn
	RoutingTable *IATable
}

func (dp *NetToRingDataplane) Run() error {
	for {
		pkt := respool.GetPacket()
		// XXX(scrye): we don't release the reference on error conditions, and
		// let the GC take care of this situation as they should be fairly
		// rare.

		if err := pkt.DecodeFromConn(dp.OverlayConn); err != nil {
			log.Warn("error receiving next packet from overlay conn", "err", err)
			continue
		}

		dst, err := ComputeDestination(&pkt.Info)
		if err != nil {
			log.Warn("unable to route packet", "err", err)
			metrics.IncomingPackets.WithLabelValues(metrics.PacketOutcomeRouteNotFound).Inc()
			continue
		}
		metrics.IncomingPackets.WithLabelValues(metrics.PacketOutcomeOk).Inc()
		dst.Send(dp, pkt)
	}
}

func ComputeDestination(packet *spkt.ScnPkt) (Destination, error) {
	switch header := packet.L4.(type) {
	case *l4.UDP:
		return ComputeUDPDestination(packet, header)
	case *scmp.Hdr:
		return ComputeSCMPDestination(packet, header)
	default:
		return nil, common.NewBasicError(ErrUnsupportedL4, nil, "type", header.L4Type())
	}
}

func ComputeUDPDestination(packet *spkt.ScnPkt, header *l4.UDP) (Destination, error) {
	switch packet.DstHost.Type() {
	case addr.HostTypeIPv4, addr.HostTypeIPv6:
		return &UDPDestination{IP: packet.DstHost.IP(), Port: int(header.DstPort)}, nil
	case addr.HostTypeSVC:
		return SVCDestination(packet.DstHost.(addr.HostSVC)), nil
	default:
		return nil, common.NewBasicError(ErrUnsupportedDestination, nil,
			"type", packet.DstHost.Type())
	}
}

func ComputeSCMPDestination(packet *spkt.ScnPkt, header *scmp.Hdr) (Destination, error) {
	if packet.DstHost.Type() != addr.HostTypeIPv4 && packet.DstHost.Type() != addr.HostTypeIPv6 {
		return nil, common.NewBasicError(ErrUnsupportedSCMPDestination, nil,
			"type", packet.DstHost.Type())
	}
	if header.Class == scmp.C_General {
		return ComputeSCMPGeneralDestination(packet, header)
	} else {
		return ComputeSCMPErrorDestination(packet, header)
	}
}

func ComputeSCMPGeneralDestination(s *spkt.ScnPkt, header *scmp.Hdr) (Destination, error) {
	id := getSCMPGeneralID(s)
	if id == 0 {
		return nil, common.NewBasicError("Invalid SCMP ID", nil, "id", id)
	}
	switch {
	case isSCMPGeneralRequest(header):
		invertSCMPGeneralType(header)
		return SCMPHandlerDestination{}, nil
	case isSCMPGeneralReply(header):
		return &SCMPAppDestination{ID: id}, nil
	default:
		return nil, common.NewBasicError("Unsupported SCMP General type", nil, "type", header.Type)
	}
}

func ComputeSCMPErrorDestination(packet *spkt.ScnPkt, header *scmp.Hdr) (Destination, error) {
	scmpPayload := packet.Pld.(*scmp.Payload)
	switch scmpPayload.Meta.L4Proto {
	case common.L4UDP:
		quotedUDPHeader, err := l4.UDPFromRaw(scmpPayload.L4Hdr)
		if err != nil {
			return nil, common.NewBasicError(ErrMalformedL4Quote, nil, "err", err)
		}
		return &UDPDestination{IP: packet.DstHost.IP(), Port: int(quotedUDPHeader.SrcPort)}, nil
	case common.L4SCMP:

		id, err := getQuotedSCMPGeneralID(scmpPayload)
		if id == 0 {
			return nil, common.NewBasicError(ErrMalformedL4Quote, err)
		}
		return &SCMPAppDestination{ID: id}, nil
	default:
		return nil, common.NewBasicError(ErrUnsupportedQuotedL4Type, nil,
			"type", scmpPayload.Meta.L4Proto)
	}
}

type Destination interface {
	// Send takes ownership of pkt, and then sends it to the location described
	// by this destination.
	Send(dp *NetToRingDataplane, pkt *respool.Packet)
}

var _ Destination = (*UDPDestination)(nil)

type UDPDestination net.UDPAddr

func (d *UDPDestination) Send(dp *NetToRingDataplane, pkt *respool.Packet) {
	routingEntry, ok := dp.RoutingTable.LookupPublic(pkt.Info.DstIA, (*net.UDPAddr)(d))
	if !ok {
		log.Warn("destination address not found", "ia", pkt.Info.DstIA,
			"udpAddr", (*net.UDPAddr)(d))
		return
	}
	sendPacket(routingEntry, pkt)
}

var _ Destination = SVCDestination(addr.SvcNone)

type SVCDestination addr.HostSVC

func (d SVCDestination) Send(dp *NetToRingDataplane, pkt *respool.Packet) {
	// FIXME(scrye): This should deliver to the correct IP address, based on
	// information found in the overlay IP header.
	routingEntries := dp.RoutingTable.LookupService(pkt.Info.DstIA, addr.HostSVC(d), nil)
	if len(routingEntries) == 0 {
		log.Warn("destination address not found", "ia", pkt.Info.DstIA, "svc", addr.HostSVC(d))
		return
	}
	// Increase reference count for all extra copies
	for i := 0; i < len(routingEntries)-1; i++ {
		pkt.Dup()
	}
	for _, routingEntry := range routingEntries {
		sendPacket(routingEntry, pkt)
	}
}

var _ Destination = (*SCMPAppDestination)(nil)

type SCMPAppDestination struct {
	ID uint64
}

func (d *SCMPAppDestination) Send(dp *NetToRingDataplane, pkt *respool.Packet) {
	routingEntry, ok := dp.RoutingTable.LookupID(pkt.Info.DstIA, d.ID)
	if !ok {
		log.Warn("destination address not found", "SCMP", d.ID)
		return
	}
	sendPacket(routingEntry, pkt)
}

// sendPacket puts pkt on the routing entry's ring buffer, and releases the
// reference to pkt.
func sendPacket(routingEntry *TableEntry, pkt *respool.Packet) {
	// Move packet reference to other goroutine.
	count, _ := routingEntry.appIngressRing.Write(ringbuf.EntryList{pkt}, false)
	if count <= 0 {
		// Release buffer if we couldn't transmit it to the other goroutine.
		pkt.Free()
	}
}

var _ Destination = (*SCMPHandlerDestination)(nil)

type SCMPHandlerDestination struct{}

func (h SCMPHandlerDestination) Send(dp *NetToRingDataplane, pkt *respool.Packet) {
	if err := pkt.Info.Reverse(); err != nil {
		log.Warn("Unable to reverse SCMP packet.", "err", err)
		return
	}

	b := respool.GetBuffer()
	pkt.Info.HBHExt = removeSCMPHBH(pkt.Info.HBHExt)
	n, err := hpkt.WriteScnPkt(&pkt.Info, b)
	if err != nil {
		log.Warn("Unable to create reply SCMP packet", "err", err)
		return
	}

	_, err = dp.OverlayConn.WriteTo(b[:n], pkt.OverlayRemote)
	if err != nil {
		log.Warn("Unable to write to overlay socket.", "err", err)
		return
	}
	respool.PutBuffer(b)
	pkt.Free()
}
<|endoftext|>