<commit_before>package main

import (
	"fmt"
	"regexp"
	"strconv"
	"strings"
)

var (
	sequenses = regexp.MustCompile(`(?:[^/\\]|\\.)*`)
	branches  = regexp.MustCompile(`(?:[^,\\]|\\.)*`)
)

func parseExpr(expr string) ([][]string, error) {
	expr = strings.Replace(expr, `\,`, `\\,`, -1)
	expr = strings.Replace(expr, `\/`, `\\/`, -1)
	expr, err := strconv.Unquote(`"` + expr + `"`)
	if err != nil {
		return nil, err
	}

	sls := sequenses.FindAllString(expr, -1)
	tree := make([][]string, len(sls))
	for si := 0; si < len(sls); si++ {
		bls := branches.FindAllString(sls[si], -1)
		for bi := 0; bi < len(bls); bi++ {
			bls[bi] = strings.Replace(bls[bi], `\,`, `,`, -1)
			bls[bi] = strings.Replace(bls[bi], `\/`, `/`, -1)
			bls[bi] = regexp.QuoteMeta(bls[bi])
		}
		tree[si] = bls
	}
	return tree, nil
}

func newMatcher(expr string) (m *regexp.Regexp, err error) {
	tree, err := parseExpr(expr)
	if err != nil {
		return nil, err
	}

	sls := make([]string, len(tree))
	for si, bls := range tree {
		sls[si] = "(" + strings.Join(bls, "|") + ")"
	}
	return regexp.Compile(strings.Join(sls, ""))
}

func newReplacement(exprFrom, exprTo string) ([]map[string]string, error) {
	from, err := parseExpr(exprFrom)
	if err != nil {
		return nil, err
	}
	to, err := parseExpr(exprTo)
	if err != nil {
		return nil, err
	}

	if len(from) != len(to) {
		return nil, fmt.Errorf("mismatch the number of sequense")
	}

	r := make([]map[string]string, len(from))
	for si := 0; si < len(from); si++ {
		if len(from[si]) != len(to[si]) {
			return nil, fmt.Errorf("mismatch the number of branch[%q]", si)
		}

		r[si] = make(map[string]string)
		for bi := 0; bi < len(from[si]); bi++ {
			src, dst := from[si][bi], to[si][bi]
			if _, exist := r[si][src]; exist {
				return nil, fmt.Errorf("branch[%q] has duplicate item", si)
			}
			r[si][src] = dst
		}
	}
	return r, nil
}
<commit_msg>Define Replacer<commit_after>package main

import (
	"fmt"
	"regexp"
	"strconv"
	"strings"
)

var (
	sequenses = regexp.MustCompile(`(?:[^/\\]|\\.)*`)
	branches  = regexp.MustCompile(`(?:[^,\\]|\\.)*`)
)

func parseExpr(expr string) ([][]string, error) {
	expr = strings.Replace(expr, `\,`, `\\,`, -1)
	expr = strings.Replace(expr, `\/`, `\\/`, -1)
	expr, err := strconv.Unquote(`"` + expr + `"`)
	if err != nil {
		return nil, err
	}

	sls := sequenses.FindAllString(expr, -1)
	tree := make([][]string, len(sls))
	for si := 0; si < len(sls); si++ {
		bls := branches.FindAllString(sls[si], -1)
		for bi := 0; bi < len(bls); bi++ {
			bls[bi] = strings.Replace(bls[bi], `\,`, `,`, -1)
			bls[bi] = strings.Replace(bls[bi], `\/`, `/`, -1)
			bls[bi] = regexp.QuoteMeta(bls[bi])
		}
		tree[si] = bls
	}
	return tree, nil
}

func newMatcher(expr string) (m *regexp.Regexp, err error) {
	tree, err := parseExpr(expr)
	if err != nil {
		return nil, err
	}

	sls := make([]string, len(tree))
	for si, bls := range tree {
		sls[si] = "(" + strings.Join(bls, "|") + ")"
	}
	return regexp.Compile(strings.Join(sls, ""))
}

func newReplacement(exprFrom, exprTo string) ([]map[string]string, error) {
	from, err := parseExpr(exprFrom)
	if err != nil {
		return nil, err
	}
	to, err := parseExpr(exprTo)
	if err != nil {
		return nil, err
	}

	if len(from) != len(to) {
		return nil, fmt.Errorf("mismatch the number of sequense")
	}

	r := make([]map[string]string, len(from))
	for si := 0; si < len(from); si++ {
		if len(from[si]) != len(to[si]) {
			return nil, fmt.Errorf("mismatch the number of branch[%q]", si)
		}

		r[si] = make(map[string]string)
		for bi := 0; bi < len(from[si]); bi++ {
			src, dst := from[si][bi], to[si][bi]
			if _, exist := r[si][src]; exist {
				return nil, fmt.Errorf("branch[%q] has duplicate item", si)
			}
			r[si][src] = dst
		}
	}
	return r, nil
}

type Replacer struct {
	matcher     *regexp.Regexp
	replacement []map[string]string
}

func NewReplacer(from, to string) (r *Replacer, err error) {
	r = &Replacer{}

	r.matcher, err = newMatcher(from)
	if err != nil {
		return nil, err
	}
	r.replacement, err = newReplacement(from, to)
	if err != nil {
		return nil, err
	}

	return r, nil
}
<|endoftext|>


<commit_before>package tdigest

import (
	"fmt"
	"math"
	"sort"
)

type centroid struct {
	mean  float64
	count uint32
}

func (c centroid) isValid() bool {
	return !math.IsNaN(c.mean) && c.count > 0
}

func (c *centroid) Update(x float64, weight uint32) {
	c.count += weight
	c.mean += float64(weight) * (x - c.mean) / float64(c.count)
}

var invalidCentroid = centroid{mean: math.NaN(), count: 0}

type summary struct {
	keys   []float64
	counts []uint32
}

func newSummary(initialCapacity uint) *summary {
	return &summary{
		keys:   make([]float64, 0, initialCapacity),
		counts: make([]uint32, 0, initialCapacity),
	}
}

func (s summary) Len() int {
	return len(s.keys)
}

func (s summary) String() string {
	return fmt.Sprintf("SortedSlice(size=%d, keys=%v)", len(s.keys), s.keys)
}

func (s *summary) Add(key float64, value uint32) error {

	if math.IsNaN(key) {
		return fmt.Errorf("Key must not be NaN")
	}

	if value == 0 {
		return fmt.Errorf("Count must be >0")
	}

	idx := s.FindIndex(key)

	if idx < len(s.keys) && s.keys[idx] == key {
		return fmt.Errorf("Duplicate key %f", key)
	}

	s.keys = append(s.keys, math.NaN())
	s.counts = append(s.counts, 0)

	copy(s.keys[idx+1:], s.keys[idx:])
	copy(s.counts[idx+1:], s.counts[idx:])

	s.keys[idx] = key
	s.counts[idx] = value

	return nil
}

func (s summary) Find(x float64) centroid {
	idx := s.FindIndex(x)

	if idx < s.Len() && s.keys[idx] == x {
		return centroid{x, s.counts[idx]}
	}

	return invalidCentroid
}

func (s summary) FindIndex(x float64) int {
	// FIXME When is linear scan better than binsearch()?
	//       should I even bother?
	if len(s.keys) < 30 {
		for i, item := range s.keys {
			if item >= x {
				return i
			}
		}
		return len(s.keys)
	}

	return sort.Search(len(s.keys), func(i int) bool {
		return s.keys[i] >= x
	})
}

func (s *summary) Remove(x float64) centroid {
	idx := s.FindIndex(x)

	if idx >= s.Len() || s.keys[idx] != x {
		return invalidCentroid
	}

	removed := s.counts[idx]

	s.keys = append(s.keys[:idx], s.keys[idx+1:]...)
	s.counts = append(s.counts[:idx], s.counts[idx+1:]...)

	return centroid{x, removed}
}

func (s summary) At(index int) centroid {
	if s.Len()-1 < index || index < 0 {
		return invalidCentroid
	}

	return centroid{s.keys[index], s.counts[index]}
}

func (s summary) Iterate(f func(c centroid) bool) {
	for i := 0; i < s.Len(); i++ {
		if !f(centroid{s.keys[i], s.counts[i]}) {
			break
		}
	}
}

func (s summary) Min() centroid {
	return s.At(0)
}

func (s summary) Max() centroid {
	return s.At(s.Len() - 1)
}

func (s summary) Data() []centroid {
	data := make([]centroid, 0, s.Len())
	s.Iterate(func(c centroid) bool {
		data = append(data, c)
		return true
	})
	return data
}

func (s summary) successorAndPredecessorItems(mean float64) (centroid, centroid) {
	idx := s.FindIndex(mean)
	return s.At(idx + 1), s.At(idx - 1)
}

func (s summary) ceilingAndFloorItems(mean float64) (centroid, centroid) {
	idx := s.FindIndex(mean)

	// Case 1: item is greater than all items in the summary
	if idx == s.Len() {
		return invalidCentroid, s.Max()
	}

	item := s.At(idx)

	// Case 2: item exists in the summary
	if item.isValid() && mean == item.mean {
		return item, item
	}

	// Case 3: item is smaller than all items in the summary
	if idx == 0 {
		return s.Min(), invalidCentroid
	}

	return item, s.At(idx - 1)
}

func (s summary) sumUntilMean(mean float64) uint32 {
	var cumSum uint32
	for i := 0; i < len(s.keys); i++ {
		if s.keys[i] < mean {
			cumSum += s.counts[i]
		} else {
			break
		}
	}
	return cumSum
}
<commit_msg>Get rid of summary.String()<commit_after>package tdigest

import (
	"fmt"
	"math"
	"sort"
)

type centroid struct {
	mean  float64
	count uint32
}

func (c centroid) isValid() bool {
	return !math.IsNaN(c.mean) && c.count > 0
}

func (c *centroid) Update(x float64, weight uint32) {
	c.count += weight
	c.mean += float64(weight) * (x - c.mean) / float64(c.count)
}

var invalidCentroid = centroid{mean: math.NaN(), count: 0}

type summary struct {
	keys   []float64
	counts []uint32
}

func newSummary(initialCapacity uint) *summary {
	return &summary{
		keys:   make([]float64, 0, initialCapacity),
		counts: make([]uint32, 0, initialCapacity),
	}
}

func (s summary) Len() int {
	return len(s.keys)
}

func (s *summary) Add(key float64, value uint32) error {

	if math.IsNaN(key) {
		return fmt.Errorf("Key must not be NaN")
	}

	if value == 0 {
		return fmt.Errorf("Count must be >0")
	}

	idx := s.FindIndex(key)

	if idx < len(s.keys) && s.keys[idx] == key {
		return fmt.Errorf("Duplicate key %f", key)
	}

	s.keys = append(s.keys, math.NaN())
	s.counts = append(s.counts, 0)

	copy(s.keys[idx+1:], s.keys[idx:])
	copy(s.counts[idx+1:], s.counts[idx:])

	s.keys[idx] = key
	s.counts[idx] = value

	return nil
}

func (s summary) Find(x float64) centroid {
	idx := s.FindIndex(x)

	if idx < s.Len() && s.keys[idx] == x {
		return centroid{x, s.counts[idx]}
	}

	return invalidCentroid
}

func (s summary) FindIndex(x float64) int {
	// FIXME When is linear scan better than binsearch()?
	//       should I even bother?
	if len(s.keys) < 30 {
		for i, item := range s.keys {
			if item >= x {
				return i
			}
		}
		return len(s.keys)
	}

	return sort.Search(len(s.keys), func(i int) bool {
		return s.keys[i] >= x
	})
}

func (s *summary) Remove(x float64) centroid {
	idx := s.FindIndex(x)

	if idx >= s.Len() || s.keys[idx] != x {
		return invalidCentroid
	}

	removed := s.counts[idx]

	s.keys = append(s.keys[:idx], s.keys[idx+1:]...)
	s.counts = append(s.counts[:idx], s.counts[idx+1:]...)

	return centroid{x, removed}
}

func (s summary) At(index int) centroid {
	if s.Len()-1 < index || index < 0 {
		return invalidCentroid
	}

	return centroid{s.keys[index], s.counts[index]}
}

func (s summary) Iterate(f func(c centroid) bool) {
	for i := 0; i < s.Len(); i++ {
		if !f(centroid{s.keys[i], s.counts[i]}) {
			break
		}
	}
}

func (s summary) Min() centroid {
	return s.At(0)
}

func (s summary) Max() centroid {
	return s.At(s.Len() - 1)
}

func (s summary) Data() []centroid {
	data := make([]centroid, 0, s.Len())
	s.Iterate(func(c centroid) bool {
		data = append(data, c)
		return true
	})
	return data
}

func (s summary) successorAndPredecessorItems(mean float64) (centroid, centroid) {
	idx := s.FindIndex(mean)
	return s.At(idx + 1), s.At(idx - 1)
}

func (s summary) ceilingAndFloorItems(mean float64) (centroid, centroid) {
	idx := s.FindIndex(mean)

	// Case 1: item is greater than all items in the summary
	if idx == s.Len() {
		return invalidCentroid, s.Max()
	}

	item := s.At(idx)

	// Case 2: item exists in the summary
	if item.isValid() && mean == item.mean {
		return item, item
	}

	// Case 3: item is smaller than all items in the summary
	if idx == 0 {
		return s.Min(), invalidCentroid
	}

	return item, s.At(idx - 1)
}

func (s summary) sumUntilMean(mean float64) uint32 {
	var cumSum uint32
	for i := 0; i < len(s.keys); i++ {
		if s.keys[i] < mean {
			cumSum += s.counts[i]
		} else {
			break
		}
	}
	return cumSum
}
<|endoftext|>


<commit_before>// SPDX-License-Identifier: Apache-2.0
// Copyright 2017-2018 Authors of Cilium

package cmd

import (
	"fmt"
	"os"
	"reflect"
	"sort"
	"strconv"
	"strings"

	"github.com/spf13/cobra"

	"github.com/cilium/cilium/api/v1/models"
	"github.com/cilium/cilium/pkg/command"
	"github.com/cilium/cilium/pkg/option"
)

var (
	numPages                   int
	listReadOnlyConfigurations bool
	listAllConfigurations      bool
)

// configCmd represents the config command
var configCmd = &cobra.Command{
	Use:   "config [<option>=(enable|disable) ...]",
	Short: "Cilium configuration options",
	Run: func(cmd *cobra.Command, args []string) {
		if listOptions {
			for k, s := range option.DaemonMutableOptionLibrary {
				fmt.Printf("%-24s %s\n", k, s.Description)
			}
			return
		}

		configDaemon(cmd, args)
	},
}

func init() {
	rootCmd.AddCommand(configCmd)
	configCmd.Flags().BoolVarP(&listOptions, "list-options", "", false, "List available options")
	configCmd.Flags().BoolVarP(&listReadOnlyConfigurations, "read-only", "r", false, "Display read only configurations")
	configCmd.Flags().BoolVarP(&listAllConfigurations, "all", "a", false, "Display all cilium configurations")
	configCmd.Flags().IntVarP(&numPages, "num-pages", "n", 0, "Number of pages for perf ring buffer. New values have to be > 0")
	command.AddJSONOutput(configCmd)
}

func configDaemon(cmd *cobra.Command, opts []string) {
	dOpts := make(models.ConfigurationMap, len(opts))

	resp, err := client.ConfigGet()
	if err != nil {
		Fatalf("Error while retrieving configuration: %s", err)
	}
	if resp.Status == nil {
		Fatalf("Empty configuration status returned")
	}

	cfgStatus := resp.Status
	if numPages > 0 {
		if cfgStatus.NodeMonitor != nil && numPages != int(cfgStatus.NodeMonitor.Npages) {
			dOpts["MonitorNumPages"] = strconv.Itoa(numPages)
		}
	} else if len(opts) == 0 {
		printConfigurations(cfgStatus)
		return
	}

	var cfg models.DaemonConfigurationSpec

	for k := range opts {
		// TODO FIXME - this is a hack, and is not clean
		optionSplit := strings.SplitN(opts[k], "=", 2)
		if len(optionSplit) < 2 {
			Fatalf("Improper configuration format provided")
		}
		arg := optionSplit[0]
		if arg == "PolicyEnforcement" {
			cfg.PolicyEnforcement = optionSplit[1]
			continue
		}

		name, value, err := option.ParseDaemonOption(opts[k])
		if err != nil {
			fmt.Printf("%s\n", err)
			os.Exit(1)
		}

		if opt, ok := option.DaemonMutableOptionLibrary[name]; !ok || opt.Parse == nil {
			if value == option.OptionDisabled {
				dOpts[name] = "Disabled"
			} else {
				dOpts[name] = "Enabled"
			}
		} else {
			dOpts[name] = optionSplit[1]
		}
	}

	cfg.Options = dOpts
	if err := client.ConfigPatch(cfg); err != nil {
		Fatalf("Unable to change agent configuration: %s\n", err)
	}
}

func printConfigurations(cfgStatus *models.DaemonConfigurationStatus) {
	if command.OutputJSON() {
		if listReadOnlyConfigurations {
			if err := command.PrintOutput(cfgStatus.DaemonConfigurationMap); err != nil {
				Fatalf("Cannot show configuratons: %v", err)
			}
		} else if listAllConfigurations {
			if err := command.PrintOutputWithPatch(cfgStatus.DaemonConfigurationMap, cfgStatus.Realized); err != nil {
				Fatalf("Cannot show configuratons: %v", err)
			}
		} else {
			if err := command.PrintOutput(cfgStatus.Realized.Options); err != nil {
				Fatalf("Cannot show configuratons: %v", err)
			}
		}
		return
	}
	if listReadOnlyConfigurations {
		dumpReadOnlyConfigs(cfgStatus)
	} else if listAllConfigurations {
		dumpReadOnlyConfigs(cfgStatus)
		dumpReadWriteConfigs(cfgStatus)
	} else {
		dumpReadWriteConfigs(cfgStatus)
	}
}

func dumpReadOnlyConfigs(cfgStatus *models.DaemonConfigurationStatus) {
	fmt.Println("#### Read-only configurations ####")
	keys := make([]string, 0, len(cfgStatus.DaemonConfigurationMap))
	for k := range cfgStatus.DaemonConfigurationMap {
		keys = append(keys, k)
	}
	sort.Strings(keys)
	for _, k := range keys {
		v := cfgStatus.DaemonConfigurationMap[k]
		if reflect.ValueOf(v).Kind() == reflect.Map {
			mapString := make(map[string]string)
			m, ok := v.(map[string]interface{})
			if ok {
				fmt.Println(k)
				for key, value := range m {
					mapString[key] = fmt.Sprintf("%v", value)
				}
				dumpConfig(mapString, true)
				continue
			} else {
				fmt.Fprintf(os.Stderr, "Error: cannot cast daemon config map to map[string]interface{}\n")
			}
		}
		fmt.Printf("%-34s: %v\n", k, v)
	}
	fmt.Printf("%-34s: %s\n", "k8s-configuration", cfgStatus.K8sConfiguration)
	fmt.Printf("%-34s: %s\n", "k8s-endpoint", cfgStatus.K8sEndpoint)
	dumpConfig(cfgStatus.Immutable, false)
}

func dumpReadWriteConfigs(cfgStatus *models.DaemonConfigurationStatus) {
	fmt.Println("##### Read-write configurations #####")
	dumpConfig(cfgStatus.Realized.Options, false)
	if cfgStatus.NodeMonitor != nil {
		fmt.Printf("%-34s: %d\n", "MonitorNumPages", cfgStatus.NodeMonitor.Npages)
	}
	fmt.Printf("%-34s: %s\n", "PolicyEnforcement", cfgStatus.Realized.PolicyEnforcement)
}
<commit_msg>Fix the typo in Fatalf message of printConfigurations<commit_after>// SPDX-License-Identifier: Apache-2.0
// Copyright 2017-2018 Authors of Cilium

package cmd

import (
	"fmt"
	"os"
	"reflect"
	"sort"
	"strconv"
	"strings"

	"github.com/spf13/cobra"

	"github.com/cilium/cilium/api/v1/models"
	"github.com/cilium/cilium/pkg/command"
	"github.com/cilium/cilium/pkg/option"
)

var (
	numPages                   int
	listReadOnlyConfigurations bool
	listAllConfigurations      bool
)

// configCmd represents the config command
var configCmd = &cobra.Command{
	Use:   "config [<option>=(enable|disable) ...]",
	Short: "Cilium configuration options",
	Run: func(cmd *cobra.Command, args []string) {
		if listOptions {
			for k, s := range option.DaemonMutableOptionLibrary {
				fmt.Printf("%-24s %s\n", k, s.Description)
			}
			return
		}

		configDaemon(cmd, args)
	},
}

func init() {
	rootCmd.AddCommand(configCmd)
	configCmd.Flags().BoolVarP(&listOptions, "list-options", "", false, "List available options")
	configCmd.Flags().BoolVarP(&listReadOnlyConfigurations, "read-only", "r", false, "Display read only configurations")
	configCmd.Flags().BoolVarP(&listAllConfigurations, "all", "a", false, "Display all cilium configurations")
	configCmd.Flags().IntVarP(&numPages, "num-pages", "n", 0, "Number of pages for perf ring buffer. New values have to be > 0")
	command.AddJSONOutput(configCmd)
}

func configDaemon(cmd *cobra.Command, opts []string) {
	dOpts := make(models.ConfigurationMap, len(opts))

	resp, err := client.ConfigGet()
	if err != nil {
		Fatalf("Error while retrieving configuration: %s", err)
	}
	if resp.Status == nil {
		Fatalf("Empty configuration status returned")
	}

	cfgStatus := resp.Status
	if numPages > 0 {
		if cfgStatus.NodeMonitor != nil && numPages != int(cfgStatus.NodeMonitor.Npages) {
			dOpts["MonitorNumPages"] = strconv.Itoa(numPages)
		}
	} else if len(opts) == 0 {
		printConfigurations(cfgStatus)
		return
	}

	var cfg models.DaemonConfigurationSpec

	for k := range opts {
		// TODO FIXME - this is a hack, and is not clean
		optionSplit := strings.SplitN(opts[k], "=", 2)
		if len(optionSplit) < 2 {
			Fatalf("Improper configuration format provided")
		}
		arg := optionSplit[0]
		if arg == "PolicyEnforcement" {
			cfg.PolicyEnforcement = optionSplit[1]
			continue
		}

		name, value, err := option.ParseDaemonOption(opts[k])
		if err != nil {
			fmt.Printf("%s\n", err)
			os.Exit(1)
		}

		if opt, ok := option.DaemonMutableOptionLibrary[name]; !ok || opt.Parse == nil {
			if value == option.OptionDisabled {
				dOpts[name] = "Disabled"
			} else {
				dOpts[name] = "Enabled"
			}
		} else {
			dOpts[name] = optionSplit[1]
		}
	}

	cfg.Options = dOpts
	if err := client.ConfigPatch(cfg); err != nil {
		Fatalf("Unable to change agent configuration: %s\n", err)
	}
}

func printConfigurations(cfgStatus *models.DaemonConfigurationStatus) {
	if command.OutputJSON() {
		if listReadOnlyConfigurations {
			if err := command.PrintOutput(cfgStatus.DaemonConfigurationMap); err != nil {
				Fatalf("Cannot show configurations: %v", err)
			}
		} else if listAllConfigurations {
			if err := command.PrintOutputWithPatch(cfgStatus.DaemonConfigurationMap, cfgStatus.Realized); err != nil {
				Fatalf("Cannot show configurations: %v", err)
			}
		} else {
			if err := command.PrintOutput(cfgStatus.Realized.Options); err != nil {
				Fatalf("Cannot show configurations: %v", err)
			}
		}
		return
	}
	if listReadOnlyConfigurations {
		dumpReadOnlyConfigs(cfgStatus)
	} else if listAllConfigurations {
		dumpReadOnlyConfigs(cfgStatus)
		dumpReadWriteConfigs(cfgStatus)
	} else {
		dumpReadWriteConfigs(cfgStatus)
	}
}

func dumpReadOnlyConfigs(cfgStatus *models.DaemonConfigurationStatus) {
	fmt.Println("#### Read-only configurations ####")
	keys := make([]string, 0, len(cfgStatus.DaemonConfigurationMap))
	for k := range cfgStatus.DaemonConfigurationMap {
		keys = append(keys, k)
	}
	sort.Strings(keys)
	for _, k := range keys {
		v := cfgStatus.DaemonConfigurationMap[k]
		if reflect.ValueOf(v).Kind() == reflect.Map {
			mapString := make(map[string]string)
			m, ok := v.(map[string]interface{})
			if ok {
				fmt.Println(k)
				for key, value := range m {
					mapString[key] = fmt.Sprintf("%v", value)
				}
				dumpConfig(mapString, true)
				continue
			} else {
				fmt.Fprintf(os.Stderr, "Error: cannot cast daemon config map to map[string]interface{}\n")
			}
		}
		fmt.Printf("%-34s: %v\n", k, v)
	}
	fmt.Printf("%-34s: %s\n", "k8s-configuration", cfgStatus.K8sConfiguration)
	fmt.Printf("%-34s: %s\n", "k8s-endpoint", cfgStatus.K8sEndpoint)
	dumpConfig(cfgStatus.Immutable, false)
}

func dumpReadWriteConfigs(cfgStatus *models.DaemonConfigurationStatus) {
	fmt.Println("##### Read-write configurations #####")
	dumpConfig(cfgStatus.Realized.Options, false)
	if cfgStatus.NodeMonitor != nil {
		fmt.Printf("%-34s: %d\n", "MonitorNumPages", cfgStatus.NodeMonitor.Npages)
	}
	fmt.Printf("%-34s: %s\n", "PolicyEnforcement", cfgStatus.Realized.PolicyEnforcement)
}
<|endoftext|>


<commit_before>package jellyfish

// Metaphone calculates the metaphone code for a string.
//
// The Metaphone algorithm was designed as an improvement on Soundex.
// It transforms a word into a string consisting of '0BFHJKLMNPRSTWXY' where '0' is pronounced 'th' and 'X' is a '[sc]h' sound.
//
// For example:
//    Metaphone("Klumpz") == Metaphone("Clumps")    // KLMPS
//
// See the Metaphone article at Wikipedia (http://en.wikipedia.org/wiki/Metaphone) for more details.
func Metaphone(s string) string {
	r := normalize(s)
	rlen := len(r)

	// skip first character sometimes

	if rlen > 1 {
		switch {
		case r[0] == 'K' && r[1] == 'N',
			r[0] == 'G' && r[1] == 'N',
			r[0] == 'P' && r[1] == 'N',
			r[0] == 'A' && r[1] == 'C',
			r[0] == 'W' && r[1] == 'R',
			r[0] == 'A' && r[1] == 'E':
			r = r[1:]
			rlen -= 1
		}
	}

	var next rune
	var nextnext rune
	var result []rune

	for i := 0; i < rlen; i++ {
		// get current and next 2 runes
		c := r[i]
		if i+1 < rlen {
			next = r[i+1]
			if i+2 < rlen {
				nextnext = r[i+2]
			} else {
				nextnext = 0
			}
		} else {
			next = 0
		}

		// skip doubles except cc
		if c == next && c != 'C' {
			continue
		}

		// main rules
		switch c {
		case 'A', 'E', 'I', 'O', 'U':
			if i == 0 || r[i-1] == ' ' {
				result = append(result, c)
			}
		case 'B':
			if i == 0 || r[i-1] != 'M' || next != 0 {
				result = append(result, 'B')
			}
		case 'C':
			if next == 'I' && nextnext == 'A' || next == 'H' {
				result = append(result, 'X')
				i++
			} else if next == 'I' || next == 'E' || next == 'Y' {
				result = append(result, 'S')
				i++
			} else {
				result = append(result, 'K')
			}
		case 'D':
			if next == 'G' && (nextnext == 'I' || nextnext == 'E' || nextnext == 'Y') {
				result = append(result, 'J')
				i++
			} else {
				result = append(result, 'T')
			}
		case 'F', 'J', 'L', 'M', 'N', 'R':
			result = append(result, c)
		case 'G':
			if next == 'I' || next == 'E' || next == 'Y' {
				result = append(result, 'J')
			} else if next != 'H' && next != 'N' {
				result = append(result, 'K')
			} else if next == 'H' && !isVowel(nextnext) {
				i++
			}
		case 'H':
			if i == 0 || isVowel(next) || isVowel(r[i-1]) {
				result = append(result, 'H')
			}
		case 'K':
			if i == 0 || r[i-1] != 'C' {
				result = append(result, 'K')
			}
		case 'P':
			if next == 'H' {
				result = append(result, 'F')
				i++
			} else {
				result = append(result, 'P')
			}
		case 'Q':
			result = append(result, 'K')
		case 'S':
			if next == 'H' {
				result = append(result, 'X')
				i++
			} else if next == 'I' && (nextnext == 'O' || nextnext == 'A') {
				result = append(result, 'X')
				i++
			} else {
				result = append(result, 'S')
			}
		case 'T':
			if next == 'I' && (nextnext == 'O' || nextnext == 'A') {
				result = append(result, 'X')
			} else if next == 'H' {
				result = append(result, '0')
				i++
			} else if next != 'C' || nextnext != 'H' {
				result = append(result, 'T')
			}
		case 'V':
			result = append(result, 'F')
		case 'W':
			if i == 0 && next == 'H' {
				i++
			}
			if isVowel(nextnext) {
				result = append(result, 'W')
			}
		case 'X':
			if i == 0 {
				if next == 'H' || (next == 'I' && (nextnext == 'O' || nextnext == 'A')) {
					result = append(result, 'X')
				} else {
					result = append(result, 'S')
				}
			} else {
				result = append(result, 'K')
				result = append(result, 'S')
			}
		case 'Y':
			if isVowel(next) {
				result = append(result, 'Y')
			}
		case 'Z':
			result = append(result, 'S')
		case ' ':
			if len(result) > 0 && result[len(result)-1] != ' ' {
				result = append(result, ' ')
			}
		}
	}

	return string(result)
}
<commit_msg>update Metaphone rules to handle UTAH/WH test cases<commit_after>package jellyfish

// Metaphone calculates the metaphone code for a string.
//
// The Metaphone algorithm was designed as an improvement on Soundex.
// It transforms a word into a string consisting of '0BFHJKLMNPRSTWXY' where '0' is pronounced 'th' and 'X' is a '[sc]h' sound.
//
// For example:
//    Metaphone("Klumpz") == Metaphone("Clumps")    // KLMPS
//
// See the Metaphone article at Wikipedia (http://en.wikipedia.org/wiki/Metaphone) for more details.
func Metaphone(s string) string {
	r := normalize(s)
	rlen := len(r)

	// skip first character sometimes

	if rlen > 1 {
		switch {
		case r[0] == 'K' && r[1] == 'N',
			r[0] == 'G' && r[1] == 'N',
			r[0] == 'P' && r[1] == 'N',
			r[0] == 'A' && r[1] == 'C',
			r[0] == 'W' && r[1] == 'R',
			r[0] == 'A' && r[1] == 'E':
			r = r[1:]
			rlen -= 1
		}
	}

	var next rune
	var nextnext rune
	var result []rune

	for i := 0; i < rlen; i++ {
		// get current and next 2 runes
		c := r[i]
		if i+1 < rlen {
			next = r[i+1]
			if i+2 < rlen {
				nextnext = r[i+2]
			} else {
				nextnext = 0
			}
		} else {
			next = 0
		}

		// skip doubles except cc
		if c == next && c != 'C' {
			continue
		}

		// main rules
		switch c {
		case 'A', 'E', 'I', 'O', 'U':
			if i == 0 || r[i-1] == ' ' {
				result = append(result, c)
			}
		case 'B':
			if i == 0 || r[i-1] != 'M' || next != 0 {
				result = append(result, 'B')
			}
		case 'C':
			if next == 'I' && nextnext == 'A' || next == 'H' {
				result = append(result, 'X')
				i++
			} else if next == 'I' || next == 'E' || next == 'Y' {
				result = append(result, 'S')
				i++
			} else {
				result = append(result, 'K')
			}
		case 'D':
			if next == 'G' && (nextnext == 'I' || nextnext == 'E' || nextnext == 'Y') {
				result = append(result, 'J')
				i++
			} else {
				result = append(result, 'T')
			}
		case 'F', 'J', 'L', 'M', 'N', 'R':
			result = append(result, c)
		case 'G':
			if next == 'I' || next == 'E' || next == 'Y' {
				result = append(result, 'J')
			} else if next != 'H' && next != 'N' {
				result = append(result, 'K')
			} else if next == 'H' && !isVowel(nextnext) {
				i++
			}
		case 'H':
			if i == 0 || isVowel(next) || !isVowel(r[i-1]) {
				result = append(result, 'H')
			}
		case 'K':
			if i == 0 || r[i-1] != 'C' {
				result = append(result, 'K')
			}
		case 'P':
			if next == 'H' {
				result = append(result, 'F')
				i++
			} else {
				result = append(result, 'P')
			}
		case 'Q':
			result = append(result, 'K')
		case 'S':
			if next == 'H' {
				result = append(result, 'X')
				i++
			} else if next == 'I' && (nextnext == 'O' || nextnext == 'A') {
				result = append(result, 'X')
				i++
			} else {
				result = append(result, 'S')
			}
		case 'T':
			if next == 'I' && (nextnext == 'O' || nextnext == 'A') {
				result = append(result, 'X')
			} else if next == 'H' {
				result = append(result, '0')
				i++
			} else if next != 'C' || nextnext != 'H' {
				result = append(result, 'T')
			}
		case 'V':
			result = append(result, 'F')
		case 'W':
			if i == 0 && next == 'H' {
				i++
			}
			if isVowel(nextnext) || nextnext == 0 {
				result = append(result, 'W')
			}
		case 'X':
			if i == 0 {
				if next == 'H' || (next == 'I' && (nextnext == 'O' || nextnext == 'A')) {
					result = append(result, 'X')
				} else {
					result = append(result, 'S')
				}
			} else {
				result = append(result, 'K')
				result = append(result, 'S')
			}
		case 'Y':
			if isVowel(next) {
				result = append(result, 'Y')
			}
		case 'Z':
			result = append(result, 'S')
		case ' ':
			if len(result) > 0 && result[len(result)-1] != ' ' {
				result = append(result, ' ')
			}
		}
	}

	return string(result)
}
<|endoftext|>


<commit_before>// Copyright 2013 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

package imports

import (
	"fmt"
	"go/ast"
	"go/build"
	"go/parser"
	"go/token"
	"os"
	"path"
	"path/filepath"
	"strings"
	"sync"

	"code.google.com/p/go.tools/astutil"
)

// importToGroup is a list of functions which map from an import path to
// a group number.
var importToGroup = []func(importPath string) (num int, ok bool){
	func(importPath string) (num int, ok bool) {
		if strings.HasPrefix(importPath, "appengine") {
			return 2, true
		}
		return
	},
	func(importPath string) (num int, ok bool) {
		if strings.Contains(importPath, ".") {
			return 1, true
		}
		return
	},
}

func importGroup(importPath string) int {
	for _, fn := range importToGroup {
		if n, ok := fn(importPath); ok {
			return n
		}
	}
	return 0
}

func fixImports(fset *token.FileSet, f *ast.File) (added []string, err error) {
	// refs are a set of possible package references currently unsatisfied by imports.
	// first key: either base package (e.g. "fmt") or renamed package
	// second key: referenced package symbol (e.g. "Println")
	refs := make(map[string]map[string]bool)

	// decls are the current package imports. key is base package or renamed package.
	decls := make(map[string]*ast.ImportSpec)

	// collect potential uses of packages.
	var visitor visitFn
	visitor = visitFn(func(node ast.Node) ast.Visitor {
		if node == nil {
			return visitor
		}
		switch v := node.(type) {
		case *ast.ImportSpec:
			if v.Name != nil {
				decls[v.Name.Name] = v
			} else {
				local := importPathToName(strings.Trim(v.Path.Value, `\"`))
				decls[local] = v
			}
		case *ast.SelectorExpr:
			xident, ok := v.X.(*ast.Ident)
			if !ok {
				break
			}
			if xident.Obj != nil {
				// if the parser can resolve it, it's not a package ref
				break
			}
			pkgName := xident.Name
			if refs[pkgName] == nil {
				refs[pkgName] = make(map[string]bool)
			}
			if decls[pkgName] == nil {
				refs[pkgName][v.Sel.Name] = true
			}
		}
		return visitor
	})
	ast.Walk(visitor, f)

	// Search for imports matching potential package references.
	searches := 0
	type result struct {
		ipath string
		name  string
		err   error
	}
	results := make(chan result)
	for pkgName, symbols := range refs {
		if len(symbols) == 0 {
			continue // skip over packages already imported
		}
		go func(pkgName string, symbols map[string]bool) {
			ipath, rename, err := findImport(pkgName, symbols)
			r := result{ipath: ipath, err: err}
			if rename {
				r.name = pkgName
			}
			results <- r
		}(pkgName, symbols)
		searches++
	}
	for i := 0; i < searches; i++ {
		result := <-results
		if result.err != nil {
			return nil, result.err
		}
		if result.ipath != "" {
			if result.name != "" {
				astutil.AddNamedImport(fset, f, result.name, result.ipath)
			} else {
				astutil.AddImport(fset, f, result.ipath)
			}
			added = append(added, result.ipath)
		}
	}

	// Nil out any unused ImportSpecs, to be removed in following passes
	unusedImport := map[string]bool{}
	for pkg, is := range decls {
		if refs[pkg] == nil && pkg != "_" && pkg != "." {
			unusedImport[strings.Trim(is.Path.Value, `"`)] = true
		}
	}
	for ipath := range unusedImport {
		if ipath == "C" {
			// Don't remove cgo stuff.
			continue
		}
		astutil.DeleteImport(fset, f, ipath)
	}

	return added, nil
}

// importPathToName returns the package name for the given import path.
var importPathToName = importPathToNameGoPath

// importPathToNameBasic assumes the package name is the base of import path.
func importPathToNameBasic(importPath string) (packageName string) {
	return path.Base(importPath)
}

// importPathToNameGoPath finds out the actual package name, as declared in its .go files.
// If there's a problem, it falls back to using importPathToNameBasic.
func importPathToNameGoPath(importPath string) (packageName string) {
	if buildPkg, err := build.Import(importPath, "", 0); err == nil {
		return buildPkg.Name
	} else {
		return importPathToNameBasic(importPath)
	}
}

type pkg struct {
	importpath string // full pkg import path, e.g. "net/http"
	dir        string // absolute file path to pkg directory e.g. "/usr/lib/go/src/fmt"
}

var pkgIndexOnce sync.Once

var pkgIndex struct {
	sync.Mutex
	m map[string][]pkg // shortname => []pkg, e.g "http" => "net/http"
}

// gate is a semaphore for limiting concurrency.
type gate chan struct{}

func (g gate) enter() { g <- struct{}{} }
func (g gate) leave() { <-g }

// fsgate protects the OS & filesystem from too much concurrency.
// Too much disk I/O -> too many threads -> swapping and bad scheduling.
var fsgate = make(gate, 8)

func loadPkgIndex() {
	pkgIndex.Lock()
	pkgIndex.m = make(map[string][]pkg)
	pkgIndex.Unlock()

	var wg sync.WaitGroup
	for _, path := range build.Default.SrcDirs() {
		fsgate.enter()
		f, err := os.Open(path)
		if err != nil {
			fsgate.leave()
			fmt.Fprint(os.Stderr, err)
			continue
		}
		children, err := f.Readdir(-1)
		f.Close()
		fsgate.leave()
		if err != nil {
			fmt.Fprint(os.Stderr, err)
			continue
		}
		for _, child := range children {
			if child.IsDir() {
				wg.Add(1)
				go func(path, name string) {
					defer wg.Done()
					loadPkg(&wg, path, name)
				}(path, child.Name())
			}
		}
	}
	wg.Wait()
}

func loadPkg(wg *sync.WaitGroup, root, pkgrelpath string) {
	importpath := filepath.ToSlash(pkgrelpath)
	dir := filepath.Join(root, importpath)

	fsgate.enter()
	defer fsgate.leave()
	pkgDir, err := os.Open(dir)
	if err != nil {
		return
	}
	children, err := pkgDir.Readdir(-1)
	pkgDir.Close()
	if err != nil {
		return
	}
	// hasGo tracks whether a directory actually appears to be a
	// Go source code directory. If $GOPATH == $HOME, and
	// $HOME/src has lots of other large non-Go projects in it,
	// then the calls to importPathToName below can be expensive.
	hasGo := false
	for _, child := range children {
		name := child.Name()
		if name == "" {
			continue
		}
		if c := name[0]; c == '.' || ('0' <= c && c <= '9') {
			continue
		}
		if strings.HasSuffix(name, ".go") {
			hasGo = true
		}
		if child.IsDir() {
			wg.Add(1)
			go func(root, name string) {
				defer wg.Done()
				loadPkg(wg, root, name)
			}(root, filepath.Join(importpath, name))
		}
	}
	if hasGo {
		shortName := importPathToName(importpath)
		pkgIndex.Lock()
		pkgIndex.m[shortName] = append(pkgIndex.m[shortName], pkg{
			importpath: importpath,
			dir:        dir,
		})
		pkgIndex.Unlock()
	}

}

// loadExports returns a list exports for a package.
var loadExports = loadExportsGoPath

func loadExportsGoPath(dir string) map[string]bool {
	exports := make(map[string]bool)
	buildPkg, err := build.ImportDir(dir, 0)
	if err != nil {
		if strings.Contains(err.Error(), "no buildable Go source files in") {
			return nil
		}
		fmt.Fprintf(os.Stderr, "could not import %q: %v", dir, err)
		return nil
	}
	fset := token.NewFileSet()
	for _, file := range buildPkg.GoFiles {
		f, err := parser.ParseFile(fset, filepath.Join(dir, file), nil, 0)
		if err != nil {
			fmt.Fprintf(os.Stderr, "could not parse %q: %v", file, err)
			continue
		}
		for name := range f.Scope.Objects {
			if ast.IsExported(name) {
				exports[name] = true
			}
		}
	}
	return exports
}

// findImport searches for a package with the given symbols.
// If no package is found, findImport returns "".
// Declared as a variable rather than a function so goimports can be easily
// extended by adding a file with an init function.
var findImport = findImportGoPath

func findImportGoPath(pkgName string, symbols map[string]bool) (string, bool, error) {
	// Fast path for the standard library.
	// In the common case we hopefully never have to scan the GOPATH, which can
	// be slow with moving disks.
	if pkg, rename, ok := findImportStdlib(pkgName, symbols); ok {
		return pkg, rename, nil
	}

	// TODO(sameer): look at the import lines for other Go files in the
	// local directory, since the user is likely to import the same packages
	// in the current Go file.  Return rename=true when the other Go files
	// use a renamed package that's also used in the current file.

	pkgIndexOnce.Do(loadPkgIndex)

	// Collect exports for packages with matching names.
	var wg sync.WaitGroup
	var pkgsMu sync.Mutex // guards pkgs
	// full importpath => exported symbol => True
	// e.g. "net/http" => "Client" => True
	pkgs := make(map[string]map[string]bool)
	pkgIndex.Lock()
	for _, pkg := range pkgIndex.m[pkgName] {
		wg.Add(1)
		go func(importpath, dir string) {
			defer wg.Done()
			exports := loadExports(dir)
			if exports != nil {
				pkgsMu.Lock()
				pkgs[importpath] = exports
				pkgsMu.Unlock()
			}
		}(pkg.importpath, pkg.dir)
	}
	pkgIndex.Unlock()
	wg.Wait()

	// Filter out packages missing required exported symbols.
	for symbol := range symbols {
		for importpath, exports := range pkgs {
			if !exports[symbol] {
				delete(pkgs, importpath)
			}
		}
	}
	if len(pkgs) == 0 {
		return "", false, nil
	}

	// If there are multiple candidate packages, the shortest one wins.
	// This is a heuristic to prefer the standard library (e.g. "bytes")
	// over e.g. "github.com/foo/bar/bytes".
	shortest := ""
	for importPath := range pkgs {
		if shortest == "" || len(importPath) < len(shortest) {
			shortest = importPath
		}
	}
	return shortest, false, nil
}

type visitFn func(node ast.Node) ast.Visitor

func (fn visitFn) Visit(node ast.Node) ast.Visitor {
	return fn(node)
}

func findImportStdlib(shortPkg string, symbols map[string]bool) (importPath string, rename, ok bool) {
	for symbol := range symbols {
		path := stdlib[shortPkg+"."+symbol]
		if path == "" {
			return "", false, false
		}
		if importPath != "" && importPath != path {
			// Ambiguous. Symbols pointed to different things.
			return "", false, false
		}
		importPath = path
	}
	return importPath, false, importPath != ""
}
<commit_msg>goimports: print \n at end of errors<commit_after>// Copyright 2013 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

package imports

import (
	"fmt"
	"go/ast"
	"go/build"
	"go/parser"
	"go/token"
	"os"
	"path"
	"path/filepath"
	"strings"
	"sync"

	"code.google.com/p/go.tools/astutil"
)

// importToGroup is a list of functions which map from an import path to
// a group number.
var importToGroup = []func(importPath string) (num int, ok bool){
	func(importPath string) (num int, ok bool) {
		if strings.HasPrefix(importPath, "appengine") {
			return 2, true
		}
		return
	},
	func(importPath string) (num int, ok bool) {
		if strings.Contains(importPath, ".") {
			return 1, true
		}
		return
	},
}

func importGroup(importPath string) int {
	for _, fn := range importToGroup {
		if n, ok := fn(importPath); ok {
			return n
		}
	}
	return 0
}

func fixImports(fset *token.FileSet, f *ast.File) (added []string, err error) {
	// refs are a set of possible package references currently unsatisfied by imports.
	// first key: either base package (e.g. "fmt") or renamed package
	// second key: referenced package symbol (e.g. "Println")
	refs := make(map[string]map[string]bool)

	// decls are the current package imports. key is base package or renamed package.
	decls := make(map[string]*ast.ImportSpec)

	// collect potential uses of packages.
	var visitor visitFn
	visitor = visitFn(func(node ast.Node) ast.Visitor {
		if node == nil {
			return visitor
		}
		switch v := node.(type) {
		case *ast.ImportSpec:
			if v.Name != nil {
				decls[v.Name.Name] = v
			} else {
				local := importPathToName(strings.Trim(v.Path.Value, `\"`))
				decls[local] = v
			}
		case *ast.SelectorExpr:
			xident, ok := v.X.(*ast.Ident)
			if !ok {
				break
			}
			if xident.Obj != nil {
				// if the parser can resolve it, it's not a package ref
				break
			}
			pkgName := xident.Name
			if refs[pkgName] == nil {
				refs[pkgName] = make(map[string]bool)
			}
			if decls[pkgName] == nil {
				refs[pkgName][v.Sel.Name] = true
			}
		}
		return visitor
	})
	ast.Walk(visitor, f)

	// Search for imports matching potential package references.
	searches := 0
	type result struct {
		ipath string
		name  string
		err   error
	}
	results := make(chan result)
	for pkgName, symbols := range refs {
		if len(symbols) == 0 {
			continue // skip over packages already imported
		}
		go func(pkgName string, symbols map[string]bool) {
			ipath, rename, err := findImport(pkgName, symbols)
			r := result{ipath: ipath, err: err}
			if rename {
				r.name = pkgName
			}
			results <- r
		}(pkgName, symbols)
		searches++
	}
	for i := 0; i < searches; i++ {
		result := <-results
		if result.err != nil {
			return nil, result.err
		}
		if result.ipath != "" {
			if result.name != "" {
				astutil.AddNamedImport(fset, f, result.name, result.ipath)
			} else {
				astutil.AddImport(fset, f, result.ipath)
			}
			added = append(added, result.ipath)
		}
	}

	// Nil out any unused ImportSpecs, to be removed in following passes
	unusedImport := map[string]bool{}
	for pkg, is := range decls {
		if refs[pkg] == nil && pkg != "_" && pkg != "." {
			unusedImport[strings.Trim(is.Path.Value, `"`)] = true
		}
	}
	for ipath := range unusedImport {
		if ipath == "C" {
			// Don't remove cgo stuff.
			continue
		}
		astutil.DeleteImport(fset, f, ipath)
	}

	return added, nil
}

// importPathToName returns the package name for the given import path.
var importPathToName = importPathToNameGoPath

// importPathToNameBasic assumes the package name is the base of import path.
func importPathToNameBasic(importPath string) (packageName string) {
	return path.Base(importPath)
}

// importPathToNameGoPath finds out the actual package name, as declared in its .go files.
// If there's a problem, it falls back to using importPathToNameBasic.
func importPathToNameGoPath(importPath string) (packageName string) {
	if buildPkg, err := build.Import(importPath, "", 0); err == nil {
		return buildPkg.Name
	} else {
		return importPathToNameBasic(importPath)
	}
}

type pkg struct {
	importpath string // full pkg import path, e.g. "net/http"
	dir        string // absolute file path to pkg directory e.g. "/usr/lib/go/src/fmt"
}

var pkgIndexOnce sync.Once

var pkgIndex struct {
	sync.Mutex
	m map[string][]pkg // shortname => []pkg, e.g "http" => "net/http"
}

// gate is a semaphore for limiting concurrency.
type gate chan struct{}

func (g gate) enter() { g <- struct{}{} }
func (g gate) leave() { <-g }

// fsgate protects the OS & filesystem from too much concurrency.
// Too much disk I/O -> too many threads -> swapping and bad scheduling.
var fsgate = make(gate, 8)

func loadPkgIndex() {
	pkgIndex.Lock()
	pkgIndex.m = make(map[string][]pkg)
	pkgIndex.Unlock()

	var wg sync.WaitGroup
	for _, path := range build.Default.SrcDirs() {
		fsgate.enter()
		f, err := os.Open(path)
		if err != nil {
			fsgate.leave()
			fmt.Fprint(os.Stderr, err)
			continue
		}
		children, err := f.Readdir(-1)
		f.Close()
		fsgate.leave()
		if err != nil {
			fmt.Fprint(os.Stderr, err)
			continue
		}
		for _, child := range children {
			if child.IsDir() {
				wg.Add(1)
				go func(path, name string) {
					defer wg.Done()
					loadPkg(&wg, path, name)
				}(path, child.Name())
			}
		}
	}
	wg.Wait()
}

func loadPkg(wg *sync.WaitGroup, root, pkgrelpath string) {
	importpath := filepath.ToSlash(pkgrelpath)
	dir := filepath.Join(root, importpath)

	fsgate.enter()
	defer fsgate.leave()
	pkgDir, err := os.Open(dir)
	if err != nil {
		return
	}
	children, err := pkgDir.Readdir(-1)
	pkgDir.Close()
	if err != nil {
		return
	}
	// hasGo tracks whether a directory actually appears to be a
	// Go source code directory. If $GOPATH == $HOME, and
	// $HOME/src has lots of other large non-Go projects in it,
	// then the calls to importPathToName below can be expensive.
	hasGo := false
	for _, child := range children {
		name := child.Name()
		if name == "" {
			continue
		}
		if c := name[0]; c == '.' || ('0' <= c && c <= '9') {
			continue
		}
		if strings.HasSuffix(name, ".go") {
			hasGo = true
		}
		if child.IsDir() {
			wg.Add(1)
			go func(root, name string) {
				defer wg.Done()
				loadPkg(wg, root, name)
			}(root, filepath.Join(importpath, name))
		}
	}
	if hasGo {
		shortName := importPathToName(importpath)
		pkgIndex.Lock()
		pkgIndex.m[shortName] = append(pkgIndex.m[shortName], pkg{
			importpath: importpath,
			dir:        dir,
		})
		pkgIndex.Unlock()
	}

}

// loadExports returns a list exports for a package.
var loadExports = loadExportsGoPath

func loadExportsGoPath(dir string) map[string]bool {
	exports := make(map[string]bool)
	buildPkg, err := build.ImportDir(dir, 0)
	if err != nil {
		if strings.Contains(err.Error(), "no buildable Go source files in") {
			return nil
		}
		fmt.Fprintf(os.Stderr, "could not import %q: %v\n", dir, err)
		return nil
	}
	fset := token.NewFileSet()
	for _, file := range buildPkg.GoFiles {
		f, err := parser.ParseFile(fset, filepath.Join(dir, file), nil, 0)
		if err != nil {
			fmt.Fprintf(os.Stderr, "could not parse %q: %v\n", file, err)
			continue
		}
		for name := range f.Scope.Objects {
			if ast.IsExported(name) {
				exports[name] = true
			}
		}
	}
	return exports
}

// findImport searches for a package with the given symbols.
// If no package is found, findImport returns "".
// Declared as a variable rather than a function so goimports can be easily
// extended by adding a file with an init function.
var findImport = findImportGoPath

func findImportGoPath(pkgName string, symbols map[string]bool) (string, bool, error) {
	// Fast path for the standard library.
	// In the common case we hopefully never have to scan the GOPATH, which can
	// be slow with moving disks.
	if pkg, rename, ok := findImportStdlib(pkgName, symbols); ok {
		return pkg, rename, nil
	}

	// TODO(sameer): look at the import lines for other Go files in the
	// local directory, since the user is likely to import the same packages
	// in the current Go file.  Return rename=true when the other Go files
	// use a renamed package that's also used in the current file.

	pkgIndexOnce.Do(loadPkgIndex)

	// Collect exports for packages with matching names.
	var wg sync.WaitGroup
	var pkgsMu sync.Mutex // guards pkgs
	// full importpath => exported symbol => True
	// e.g. "net/http" => "Client" => True
	pkgs := make(map[string]map[string]bool)
	pkgIndex.Lock()
	for _, pkg := range pkgIndex.m[pkgName] {
		wg.Add(1)
		go func(importpath, dir string) {
			defer wg.Done()
			exports := loadExports(dir)
			if exports != nil {
				pkgsMu.Lock()
				pkgs[importpath] = exports
				pkgsMu.Unlock()
			}
		}(pkg.importpath, pkg.dir)
	}
	pkgIndex.Unlock()
	wg.Wait()

	// Filter out packages missing required exported symbols.
	for symbol := range symbols {
		for importpath, exports := range pkgs {
			if !exports[symbol] {
				delete(pkgs, importpath)
			}
		}
	}
	if len(pkgs) == 0 {
		return "", false, nil
	}

	// If there are multiple candidate packages, the shortest one wins.
	// This is a heuristic to prefer the standard library (e.g. "bytes")
	// over e.g. "github.com/foo/bar/bytes".
	shortest := ""
	for importPath := range pkgs {
		if shortest == "" || len(importPath) < len(shortest) {
			shortest = importPath
		}
	}
	return shortest, false, nil
}

type visitFn func(node ast.Node) ast.Visitor

func (fn visitFn) Visit(node ast.Node) ast.Visitor {
	return fn(node)
}

func findImportStdlib(shortPkg string, symbols map[string]bool) (importPath string, rename, ok bool) {
	for symbol := range symbols {
		path := stdlib[shortPkg+"."+symbol]
		if path == "" {
			return "", false, false
		}
		if importPath != "" && importPath != path {
			// Ambiguous. Symbols pointed to different things.
			return "", false, false
		}
		importPath = path
	}
	return importPath, false, importPath != ""
}
<|endoftext|>


<commit_before>// Copyright 2012 Aaron Jacobs. All Rights Reserved.
// Author: aaronjjacobs@gmail.com (Aaron Jacobs)
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package http_test

import (
	"github.com/jacobsa/aws/s3/http"
	. "github.com/jacobsa/oglematchers"
	. "github.com/jacobsa/ogletest"
	"testing"
)

func TestConn(t *testing.T) { RunTests(t) }

////////////////////////////////////////////////////////////////////////
// Helpers
////////////////////////////////////////////////////////////////////////

type ConnTest struct {
}

func init() { RegisterTestSuite(&ConnTest{}) }

////////////////////////////////////////////////////////////////////////
// Tests
////////////////////////////////////////////////////////////////////////

func (t *ConnTest) InvalidScheme() {
	// Connection
	_, err := http.NewConn("localhost", "taco")

	ExpectThat(err, Error(HasSubstr("scheme")))
	ExpectThat(err, Error(HasSubstr("taco")))
}

func (t *ConnTest) UnknownHost() {
	// Connection
	conn, err := http.NewConn("localhost", "taco")
	AssertEq(nil, err)

	// Request
	req := &http.Request{
		Verb: "GET",
		Path: "/foo",
		Headers: map[string]string{},
	}

	// Call
	_, err = conn.SendRequest(req)

	ExpectThat(err, Error(HasSubstr("TODO")))
}

func (t *ConnTest) InvalidVerb() {
	ExpectEq("TODO", "")
}

func (t *ConnTest) ServerReturns200() {
	ExpectEq("TODO", "")
}

func (t *ConnTest) ServerReturns404() {
	ExpectEq("TODO", "")
}

func (t *ConnTest) ServerReturns500() {
	ExpectEq("TODO", "")
}

func (t *ConnTest) ServerReturnsEmptyBody() {
	ExpectEq("TODO", "")
}

func (t *ConnTest) HttpsWorksProperly() {
	ExpectEq("TODO", "")
}
<commit_msg>Added a test name.<commit_after>// Copyright 2012 Aaron Jacobs. All Rights Reserved.
// Author: aaronjjacobs@gmail.com (Aaron Jacobs)
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package http_test

import (
	"github.com/jacobsa/aws/s3/http"
	. "github.com/jacobsa/oglematchers"
	. "github.com/jacobsa/ogletest"
	"testing"
)

func TestConn(t *testing.T) { RunTests(t) }

////////////////////////////////////////////////////////////////////////
// Helpers
////////////////////////////////////////////////////////////////////////

type ConnTest struct {
}

func init() { RegisterTestSuite(&ConnTest{}) }

////////////////////////////////////////////////////////////////////////
// Tests
////////////////////////////////////////////////////////////////////////

func (t *ConnTest) InvalidScheme() {
	// Connection
	_, err := http.NewConn("localhost", "taco")

	ExpectThat(err, Error(HasSubstr("scheme")))
	ExpectThat(err, Error(HasSubstr("taco")))
}

func (t *ConnTest) UnknownHost() {
	// Connection
	conn, err := http.NewConn("localhost", "taco")
	AssertEq(nil, err)

	// Request
	req := &http.Request{
		Verb: "GET",
		Path: "/foo",
		Headers: map[string]string{},
	}

	// Call
	_, err = conn.SendRequest(req)

	ExpectThat(err, Error(HasSubstr("TODO")))
}

func (t *ConnTest) InvalidVerb() {
	ExpectEq("TODO", "")
}

func (t *ConnTest) PassesOnRequestInfo() {
	ExpectEq("TODO", "")
}

func (t *ConnTest) ServerReturns200() {
	ExpectEq("TODO", "")
}

func (t *ConnTest) ServerReturns404() {
	ExpectEq("TODO", "")
}

func (t *ConnTest) ServerReturns500() {
	ExpectEq("TODO", "")
}

func (t *ConnTest) ServerReturnsEmptyBody() {
	ExpectEq("TODO", "")
}

func (t *ConnTest) HttpsWorksProperly() {
	ExpectEq("TODO", "")
}
<|endoftext|>


<commit_before>package main

import (
	"encoding/json"
	"flag"
	"fmt"
	"io/ioutil"
	"net/http"
	"net/url"
	"os"
	"os/exec"
	"path/filepath"
	"strings"
	"sync"

	log "github.com/Sirupsen/logrus"
	"github.com/datasektionen/taitan/fuzz"
	"github.com/datasektionen/taitan/pages"
)

var (
	debug     bool   // Show debug level messages.
	info      bool   // Show info level messages.
	responses Atomic // Our parsed responses.
)

func usage() {
	fmt.Fprintf(os.Stderr, "Usage: %s [OPTIONS] ROOT\n", os.Args[0])
	flag.PrintDefaults()
	os.Exit(1)
}

func init() {
	flag.BoolVar(&debug, "vv", false, "Print debug messages.")
	flag.BoolVar(&info, "v", false, "Print info messages.")
	flag.Usage = usage
	flag.Parse()
}

func getEnv(env string) string {
	e := os.Getenv(env)
	if e == "" {
		log.Fatalf("$%s environmental variable is not set.\n", env)
	}
	return e
}

func getRoot() string {
	content := getEnv("CONTENT_URL")
	u, err := url.Parse(content)
	if err != nil {
		log.Fatalln("getContent: ", err)
	}

	// https://<token>@github.com/username/repo.git
	u.User = url.User(getEnv("TOKEN"))

	base := filepath.Base(u.Path)
	return strings.TrimSuffix(base, filepath.Ext(base))
}

func getContent() {
	content := getEnv("CONTENT_URL")
	u, err := url.Parse(content)
	if err != nil {
		log.Fatalln("getContent: ", err)
	}

	// https://<token>@github.com/username/repo.git
	u.User = url.User(getEnv("TOKEN"))

	root := getRoot()
	if _, err = os.Stat(root); os.IsNotExist(err) {
		runGit("clone", []string{"clone", u.String()})
	} else {
		runGit("pull", []string{"-C", root, "pull"})
	}
}

func runGit(action string, args []string) {
	log.Infof("Found root directory - %sing updates!", action)
	log.Debugf("Commands %#v!", args)
	cmd := exec.Command("git", args...)
	err := cmd.Start()
	if err != nil {
		log.Fatal(err)
	}
	log.Infof("Waiting for git %s to finish...", action)
	err = cmd.Wait()
	if err != nil {
		log.Warnf("%sed with error: %v\n", action, err)
	}
	log.Infof("Git %s finished!", action)
}

// setVerbosity sets the amount of messages printed.
func setVerbosity() {
	switch {
	case debug:
		log.SetLevel(log.DebugLevel)
	case info:
		log.SetLevel(log.InfoLevel)
	default:
		log.SetLevel(log.WarnLevel)
	}
}

// Atomic responses.
type Atomic struct {
	sync.Mutex
	Resps map[string]*pages.Resp
}

func validRoot(root string) {
	fi, err := os.Stat(root)
	if err != nil {
		if os.IsNotExist(err) {
			log.Fatalf("Directory doesn't exist: %q", root)
		}
		log.Fatalln(err)
	}
	if !fi.IsDir() {
		log.Fatalf("Supplied path is not a directory: %q", root)
	}
}

var jumpfile map[string]interface{}

func main() {
	setVerbosity()

	// Get port or die.
	port := getEnv("PORT")

	// Get content or die.
	getContent()

	root := getRoot()
	log.WithField("Root", root).Info("Our root directory")

	// We'll parse and store the responses ahead of time.
	resps, err := pages.Load(root)
	if err != nil {
		log.Fatalf("pages.Load: unexpected error: %s", err)
	}
	log.WithField("Resps", resps).Debug("The parsed responses")
	responses = Atomic{Resps: resps}

	log.Info("Starting server.")
	log.Info("Listening on port: ", port)

	buf, err := ioutil.ReadFile("jumpfile.json")
	if err != nil {
		log.Fatalf("jumpfile readfile: unexpected error: %s", err)
	}
	err = json.Unmarshal(buf, &jumpfile)
	if err != nil {
		log.Fatalf("jumpfile unmarshal: unexpected error: %s", err)
	}
	log.Debugln(jumpfile)

	// Our request handler.
	http.HandleFunc("/", handler)

	// Listen on port and serve with our handler.
	err = http.ListenAndServe(":"+port, nil)
	if err != nil {
		panic(err)
	}
}

type Subs struct {
	Children []string `json:"children"`
}

// handler parses and serves responses to our file queries.
func handler(res http.ResponseWriter, req *http.Request) {
	if v, ok := jumpfile[filepath.Clean(req.URL.Path)]; ok {
		req.URL.Path = v.(string)
		http.Redirect(res, req, req.URL.String(), http.StatusSeeOther)
		log.Infoln("Redirect: " + req.URL.String())
		return
	}
	if req.URL.Path == "/fuzzyfile" {
		log.Info("Fuzzyfile")
		buf, err := json.Marshal(fuzz.NewFile(responses.Resps))
		if err != nil {
			log.Warnf("handler: unexpected error: %#v\n", err)
			res.WriteHeader(http.StatusInternalServerError)
			return
		}
		log.Debugf("Response: %#v\n", string(buf))
		res.Header().Set("Content-Type", "application/json; charset=utf-8")
		res.Write(buf)
		return
	}
	if req.Header.Get("X-Github-Event") == "push" {
		var err error
		log.Infoln("Push hook")
		getContent()
		responses.Lock()
		responses.Resps, err = pages.Load(getRoot())
		if err != nil {
			log.Error(err)
		}
		responses.Unlock()
		return
	}
	// Requested URL. We extract the path.
	query := req.URL.Path
	log.WithField("query", query).Info("Recieved query")

	clean := filepath.Clean(query)
	log.WithField("clean", clean).Info("Sanitized path")

	responses.Lock()
	r, ok := responses.Resps[clean]
	responses.Unlock()
	if !ok {
		log.WithField("page", clean).Warn("Page doesn't exist")
		res.WriteHeader(http.StatusNotFound)
		return
	}
	log.Info("Marshaling the response.")

	ps := make([]pages.Child, 0)
	if _, ok := req.URL.Query()["subpages"]; ok {
		for p := range responses.Resps {
			if strings.HasPrefix(p, req.URL.Path) && p != req.URL.Path {
				ps = append(ps, pages.Child{Slug: p, Title: responses.Resps[p].Title})
			}
		}
		log.Warnln(ps)
	}

	r.Children = ps
	buf, err := json.Marshal(r)
	if err != nil {
		log.Warnf("handler: unexpected error: %#v\n", err)
		res.WriteHeader(http.StatusInternalServerError)
		return
	}
	log.Info("Serve the response.")
	log.Debugf("Response: %#v\n", string(buf))
	res.Header().Set("Content-Type", "application/json; charset=utf-8")
	res.Write(buf)
}
<commit_msg>always children<commit_after>package main

import (
	"encoding/json"
	"flag"
	"fmt"
	"io/ioutil"
	"net/http"
	"net/url"
	"os"
	"os/exec"
	"path/filepath"
	"strings"
	"sync"

	log "github.com/Sirupsen/logrus"
	"github.com/datasektionen/taitan/fuzz"
	"github.com/datasektionen/taitan/pages"
)

var (
	debug     bool   // Show debug level messages.
	info      bool   // Show info level messages.
	responses Atomic // Our parsed responses.
)

func usage() {
	fmt.Fprintf(os.Stderr, "Usage: %s [OPTIONS] ROOT\n", os.Args[0])
	flag.PrintDefaults()
	os.Exit(1)
}

func init() {
	flag.BoolVar(&debug, "vv", false, "Print debug messages.")
	flag.BoolVar(&info, "v", false, "Print info messages.")
	flag.Usage = usage
	flag.Parse()
}

func getEnv(env string) string {
	e := os.Getenv(env)
	if e == "" {
		log.Fatalf("$%s environmental variable is not set.\n", env)
	}
	return e
}

func getRoot() string {
	content := getEnv("CONTENT_URL")
	u, err := url.Parse(content)
	if err != nil {
		log.Fatalln("getContent: ", err)
	}

	// https://<token>@github.com/username/repo.git
	u.User = url.User(getEnv("TOKEN"))

	base := filepath.Base(u.Path)
	return strings.TrimSuffix(base, filepath.Ext(base))
}

func getContent() {
	content := getEnv("CONTENT_URL")
	u, err := url.Parse(content)
	if err != nil {
		log.Fatalln("getContent: ", err)
	}

	// https://<token>@github.com/username/repo.git
	u.User = url.User(getEnv("TOKEN"))

	root := getRoot()
	if _, err = os.Stat(root); os.IsNotExist(err) {
		runGit("clone", []string{"clone", u.String()})
	} else {
		runGit("pull", []string{"-C", root, "pull"})
	}
}

func runGit(action string, args []string) {
	log.Infof("Found root directory - %sing updates!", action)
	log.Debugf("Commands %#v!", args)
	cmd := exec.Command("git", args...)
	err := cmd.Start()
	if err != nil {
		log.Fatal(err)
	}
	log.Infof("Waiting for git %s to finish...", action)
	err = cmd.Wait()
	if err != nil {
		log.Warnf("%sed with error: %v\n", action, err)
	}
	log.Infof("Git %s finished!", action)
}

// setVerbosity sets the amount of messages printed.
func setVerbosity() {
	switch {
	case debug:
		log.SetLevel(log.DebugLevel)
	case info:
		log.SetLevel(log.InfoLevel)
	default:
		log.SetLevel(log.WarnLevel)
	}
}

// Atomic responses.
type Atomic struct {
	sync.Mutex
	Resps map[string]*pages.Resp
}

func validRoot(root string) {
	fi, err := os.Stat(root)
	if err != nil {
		if os.IsNotExist(err) {
			log.Fatalf("Directory doesn't exist: %q", root)
		}
		log.Fatalln(err)
	}
	if !fi.IsDir() {
		log.Fatalf("Supplied path is not a directory: %q", root)
	}
}

var jumpfile map[string]interface{}

func main() {
	setVerbosity()

	// Get port or die.
	port := getEnv("PORT")

	// Get content or die.
	getContent()

	root := getRoot()
	log.WithField("Root", root).Info("Our root directory")

	// We'll parse and store the responses ahead of time.
	resps, err := pages.Load(root)
	if err != nil {
		log.Fatalf("pages.Load: unexpected error: %s", err)
	}
	log.WithField("Resps", resps).Debug("The parsed responses")
	responses = Atomic{Resps: resps}

	log.Info("Starting server.")
	log.Info("Listening on port: ", port)

	buf, err := ioutil.ReadFile("jumpfile.json")
	if err != nil {
		log.Fatalf("jumpfile readfile: unexpected error: %s", err)
	}
	err = json.Unmarshal(buf, &jumpfile)
	if err != nil {
		log.Fatalf("jumpfile unmarshal: unexpected error: %s", err)
	}
	log.Debugln(jumpfile)

	// Our request handler.
	http.HandleFunc("/", handler)

	// Listen on port and serve with our handler.
	err = http.ListenAndServe(":"+port, nil)
	if err != nil {
		panic(err)
	}
}

type Subs struct {
	Children []string `json:"children"`
}

// handler parses and serves responses to our file queries.
func handler(res http.ResponseWriter, req *http.Request) {
	if v, ok := jumpfile[filepath.Clean(req.URL.Path)]; ok {
		req.URL.Path = v.(string)
		http.Redirect(res, req, req.URL.String(), http.StatusSeeOther)
		log.Infoln("Redirect: " + req.URL.String())
		return
	}
	if req.URL.Path == "/fuzzyfile" {
		log.Info("Fuzzyfile")
		buf, err := json.Marshal(fuzz.NewFile(responses.Resps))
		if err != nil {
			log.Warnf("handler: unexpected error: %#v\n", err)
			res.WriteHeader(http.StatusInternalServerError)
			return
		}
		log.Debugf("Response: %#v\n", string(buf))
		res.Header().Set("Content-Type", "application/json; charset=utf-8")
		res.Write(buf)
		return
	}
	if req.Header.Get("X-Github-Event") == "push" {
		var err error
		log.Infoln("Push hook")
		getContent()
		responses.Lock()
		responses.Resps, err = pages.Load(getRoot())
		if err != nil {
			log.Error(err)
		}
		responses.Unlock()
		return
	}
	// Requested URL. We extract the path.
	query := req.URL.Path
	log.WithField("query", query).Info("Recieved query")

	clean := filepath.Clean(query)
	log.WithField("clean", clean).Info("Sanitized path")

	responses.Lock()
	r, ok := responses.Resps[clean]
	responses.Unlock()
	if !ok {
		log.WithField("page", clean).Warn("Page doesn't exist")
		res.WriteHeader(http.StatusNotFound)
		return
	}
	log.Info("Marshaling the response.")

	ps := make([]pages.Child, 0)
	for p := range responses.Resps {
		if strings.HasPrefix(p, req.URL.Path) && p != req.URL.Path {
			ps = append(ps, pages.Child{Slug: p, Title: responses.Resps[p].Title})
		}
	}
	log.Warnln(ps)

	r.Children = ps
	buf, err := json.Marshal(r)
	if err != nil {
		log.Warnf("handler: unexpected error: %#v\n", err)
		res.WriteHeader(http.StatusInternalServerError)
		return
	}
	log.Info("Serve the response.")
	log.Debugf("Response: %#v\n", string(buf))
	res.Header().Set("Content-Type", "application/json; charset=utf-8")
	res.Write(buf)
}
<|endoftext|>


<commit_before>package chromedp

import (
	"context"
	"encoding/json"
	"errors"
	"sync"
	"sync/atomic"

	"github.com/mailru/easyjson"

	"github.com/chromedp/cdproto"
	"github.com/chromedp/cdproto/cdp"
	"github.com/chromedp/cdproto/dom"
	"github.com/chromedp/cdproto/page"
	"github.com/chromedp/cdproto/runtime"
	"github.com/chromedp/cdproto/target"
)

// Target manages a Chrome DevTools Protocol target.
type Target struct {
	browser   *Browser
	SessionID target.SessionID
	TargetID  target.ID

	listenersMu sync.Mutex
	listeners   []cancelableListener

	messageQueue chan *cdproto.Message

	// frameMu protects both frames and cur.
	frameMu sync.RWMutex
	// frames is the set of encountered frames.
	frames       map[cdp.FrameID]*cdp.Frame
	execContexts map[cdp.FrameID]runtime.ExecutionContextID
	// cur is the current top level frame.
	cur cdp.FrameID

	// logging funcs
	logf, errf func(string, ...interface{})

	// Indicates if the target is a worker target.
	isWorker bool
}

func (t *Target) enclosingFrame(node *cdp.Node) cdp.FrameID {
	t.frameMu.RLock()
	top := t.frames[t.cur]
	t.frameMu.RUnlock()
	top.RLock()
	defer top.RUnlock()
	for {
		if node == nil {
			// Avoid crashing. This can happen if we're using an old
			// node that has been replaced, for example.
			return ""
		}
		if node.FrameID != "" {
			break
		}
		node = top.Nodes[node.ParentID]
	}
	return node.FrameID
}

// ensureFrame ensures the top frame of this target is loaded and returns the top frame,
// the root node and the ExecutionContextID of this top frame; otherwise, it will return
// false as its last return value.
func (t *Target) ensureFrame() (*cdp.Frame, *cdp.Node, runtime.ExecutionContextID, bool) {
	t.frameMu.RLock()
	frame := t.frames[t.cur]
	execCtx := t.execContexts[t.cur]
	t.frameMu.RUnlock()

	// the frame hasn't loaded yet.
	if frame == nil || execCtx == 0 {
		return nil, nil, 0, false
	}

	frame.RLock()
	root := frame.Root
	frame.RUnlock()

	if root == nil {
		// not root node yet?
		return nil, nil, 0, false
	}
	return frame, root, execCtx, true
}

func (t *Target) run(ctx context.Context) {
	type eventValue struct {
		method cdproto.MethodType
		value  interface{}
	}
	// syncEventQueue is used to handle events synchronously within Target.
	// TODO: If this queue gets full, the goroutine below could get stuck on
	// a send, and response callbacks would never run, resulting in a
	// deadlock. Can we fix this without potentially using lots of memory?
	syncEventQueue := make(chan eventValue, 4096)

	// This goroutine receives events from the browser, calls listeners, and
	// then passes the events onto the main goroutine for the target handler
	// to update itself.
	go func() {
		for {
			select {
			case <-ctx.Done():
				return

			case msg := <-t.messageQueue:
				if msg.ID != 0 {
					t.listenersMu.Lock()
					t.listeners = runListeners(t.listeners, msg)
					t.listenersMu.Unlock()
					continue
				}
				ev, err := cdproto.UnmarshalMessage(msg)
				if err != nil {
					if _, ok := err.(cdp.ErrUnknownCommandOrEvent); ok {
						// This is most likely an event received from an older
						// Chrome which a newer cdproto doesn't have, as it is
						// deprecated. Ignore that error.
						continue
					}
					t.errf("could not unmarshal event: %v", err)
					continue
				}
				t.listenersMu.Lock()
				t.listeners = runListeners(t.listeners, ev)
				t.listenersMu.Unlock()

				switch msg.Method.Domain() {
				case "Runtime", "Page", "DOM":
					select {
					case <-ctx.Done():
						return
					case syncEventQueue <- eventValue{msg.Method, ev}:
					}
				}
			}
		}
	}()

	for {
		select {
		case <-ctx.Done():
			return
		case ev := <-syncEventQueue:
			switch ev.method.Domain() {
			case "Runtime":
				t.runtimeEvent(ev.value)
			case "Page":
				t.pageEvent(ev.value)
			case "DOM":
				t.domEvent(ctx, ev.value)
			}
		}
	}
}

func (t *Target) Execute(ctx context.Context, method string, params easyjson.Marshaler, res easyjson.Unmarshaler) error {
	if method == target.CommandCloseTarget {
		return errors.New("to close the target, cancel its context or use chromedp.Cancel")
	}

	id := atomic.AddInt64(&t.browser.next, 1)
	lctx, cancel := context.WithCancel(ctx)
	ch := make(chan *cdproto.Message, 1)
	fn := func(ev interface{}) {
		if msg, ok := ev.(*cdproto.Message); ok && msg.ID == id {
			select {
			case <-ctx.Done():
			case ch <- msg:
			}
			cancel()
		}
	}
	t.listenersMu.Lock()
	t.listeners = append(t.listeners, cancelableListener{lctx, fn})
	t.listenersMu.Unlock()

	// send command
	var buf []byte
	if params != nil {
		var err error
		buf, err = easyjson.Marshal(params)
		if err != nil {
			return err
		}
	}
	cmd := &cdproto.Message{
		ID:        id,
		SessionID: t.SessionID,
		Method:    cdproto.MethodType(method),
		Params:    buf,
	}
	select {
	case <-ctx.Done():
		return ctx.Err()
	case t.browser.cmdQueue <- cmd:
	}

	// wait for result
	select {
	case <-ctx.Done():
		return ctx.Err()
	case msg := <-ch:
		switch {
		case msg == nil:
			return ErrChannelClosed
		case msg.Error != nil:
			return msg.Error
		case res != nil:
			return easyjson.Unmarshal(msg.Result, res)
		}
	}
	return nil
}

// runtimeEvent handles incoming runtime events.
func (t *Target) runtimeEvent(ev interface{}) {
	switch ev := ev.(type) {
	case *runtime.EventExecutionContextCreated:
		var aux struct {
			FrameID cdp.FrameID
		}
		if len(ev.Context.AuxData) == 0 {
			break
		}
		if err := json.Unmarshal(ev.Context.AuxData, &aux); err != nil {
			t.errf("could not decode executionContextCreated auxData %q: %v", ev.Context.AuxData, err)
			break
		}
		if aux.FrameID != "" {
			t.frameMu.Lock()
			t.execContexts[aux.FrameID] = ev.Context.ID
			t.frameMu.Unlock()
		}
	case *runtime.EventExecutionContextDestroyed:
		t.frameMu.Lock()
		for frameID, ctxID := range t.execContexts {
			if ctxID == ev.ExecutionContextID {
				delete(t.execContexts, frameID)
			}
		}
		t.frameMu.Unlock()
	case *runtime.EventExecutionContextsCleared:
		t.frameMu.Lock()
		for frameID := range t.execContexts {
			delete(t.execContexts, frameID)
		}
		t.frameMu.Unlock()
	}
}

// documentUpdated handles the document updated event, retrieving the document
// root for the root frame.
func (t *Target) documentUpdated(ctx context.Context) {
	t.frameMu.RLock()
	f := t.frames[t.cur]
	t.frameMu.RUnlock()
	if f == nil {
		// TODO: This seems to happen on CI, when running the tests
		// under the headless-shell Docker image. Figure out why.
		t.errf("received DOM.documentUpdated when there's no top-level frame")
		return
	}
	f.Lock()
	defer f.Unlock()

	// invalidate nodes
	if f.Root != nil {
		close(f.Root.Invalidated)
	}

	f.Nodes = make(map[cdp.NodeID]*cdp.Node)
	var err error
	f.Root, err = dom.GetDocument().Do(cdp.WithExecutor(ctx, t))
	if err == context.Canceled {
		return // TODO: perhaps not necessary, but useful to keep the tests less noisy
	}
	if err != nil {
		t.errf("could not retrieve document root for %s: %v", f.ID, err)
		return
	}
	f.Root.Invalidated = make(chan struct{})
	walk(f.Nodes, f.Root)
}

// pageEvent handles incoming page events.
func (t *Target) pageEvent(ev interface{}) {
	var id cdp.FrameID
	var op frameOp

	switch e := ev.(type) {
	case *page.EventFrameNavigated:
		t.frameMu.Lock()
		t.frames[e.Frame.ID] = e.Frame
		if e.Frame.ParentID == "" {
			// This frame is only the new top-level frame if it has
			// no parent.
			t.cur = e.Frame.ID
		}
		t.frameMu.Unlock()
		return

	case *page.EventFrameAttached:
		id, op = e.FrameID, frameAttached(e.ParentFrameID)

	case *page.EventFrameDetached:
		id, op = e.FrameID, frameDetached

	case *page.EventFrameStartedLoading:
		id, op = e.FrameID, frameStartedLoading

	case *page.EventFrameStoppedLoading:
		id, op = e.FrameID, frameStoppedLoading

		// ignored events
	case *page.EventFrameRequestedNavigation:
		return
	case *page.EventDomContentEventFired:
		return
	case *page.EventLoadEventFired:
		return
	case *page.EventFrameResized:
		return
	case *page.EventLifecycleEvent:
		return
	case *page.EventNavigatedWithinDocument:
		return
	case *page.EventJavascriptDialogOpening:
		return
	case *page.EventJavascriptDialogClosed:
		return
	case *page.EventWindowOpen:
		return
	case *page.EventDownloadWillBegin:
		return
	case *page.EventDownloadProgress:
		return
	case *page.EventDocumentOpened:
		return

	default:
		t.errf("unhandled page event %T", ev)
		return
	}

	t.frameMu.Lock()
	f := t.frames[id]
	if f == nil {
		// This can happen if a frame is attached or starts loading
		// before it's ever navigated to. We won't have all the frame
		// details just yet, but that's okay.
		f = &cdp.Frame{ID: id}
		t.frames[id] = f
	}
	t.frameMu.Unlock()

	f.Lock()
	op(f)
	f.Unlock()
}

// domEvent handles incoming DOM events.
func (t *Target) domEvent(ctx context.Context, ev interface{}) {
	f := t.frames[t.cur]
	var id cdp.NodeID
	var op nodeOp

	switch e := ev.(type) {
	case *dom.EventDocumentUpdated:
		t.documentUpdated(ctx)
		return

	case *dom.EventSetChildNodes:
		id, op = e.ParentID, setChildNodes(f.Nodes, e.Nodes)

	case *dom.EventAttributeModified:
		id, op = e.NodeID, attributeModified(e.Name, e.Value)

	case *dom.EventAttributeRemoved:
		id, op = e.NodeID, attributeRemoved(e.Name)

	case *dom.EventInlineStyleInvalidated:
		if len(e.NodeIds) == 0 {
			return
		}

		id, op = e.NodeIds[0], inlineStyleInvalidated(e.NodeIds[1:])

	case *dom.EventCharacterDataModified:
		id, op = e.NodeID, characterDataModified(e.CharacterData)

	case *dom.EventChildNodeCountUpdated:
		id, op = e.NodeID, childNodeCountUpdated(e.ChildNodeCount)

	case *dom.EventChildNodeInserted:
		id, op = e.ParentNodeID, childNodeInserted(f.Nodes, e.PreviousNodeID, e.Node)

	case *dom.EventChildNodeRemoved:
		id, op = e.ParentNodeID, childNodeRemoved(f.Nodes, e.NodeID)

	case *dom.EventShadowRootPushed:
		id, op = e.HostID, shadowRootPushed(f.Nodes, e.Root)

	case *dom.EventShadowRootPopped:
		id, op = e.HostID, shadowRootPopped(f.Nodes, e.RootID)

	case *dom.EventPseudoElementAdded:
		id, op = e.ParentID, pseudoElementAdded(f.Nodes, e.PseudoElement)

	case *dom.EventPseudoElementRemoved:
		id, op = e.ParentID, pseudoElementRemoved(f.Nodes, e.PseudoElementID)

	case *dom.EventDistributedNodesUpdated:
		id, op = e.InsertionPointID, distributedNodesUpdated(e.DistributedNodes)

	default:
		t.errf("unhandled node event %T", ev)
		return
	}

	n, ok := f.Nodes[id]
	if !ok {
		// Node ID has been invalidated. Nothing to do.
		return
	}

	f.Lock()
	op(n)
	f.Unlock()
}
<commit_msg>adding case for unhandled screencast frame and visibility changed page events<commit_after>package chromedp

import (
	"context"
	"encoding/json"
	"errors"
	"sync"
	"sync/atomic"

	"github.com/mailru/easyjson"

	"github.com/chromedp/cdproto"
	"github.com/chromedp/cdproto/cdp"
	"github.com/chromedp/cdproto/dom"
	"github.com/chromedp/cdproto/page"
	"github.com/chromedp/cdproto/runtime"
	"github.com/chromedp/cdproto/target"
)

// Target manages a Chrome DevTools Protocol target.
type Target struct {
	browser   *Browser
	SessionID target.SessionID
	TargetID  target.ID

	listenersMu sync.Mutex
	listeners   []cancelableListener

	messageQueue chan *cdproto.Message

	// frameMu protects both frames and cur.
	frameMu sync.RWMutex
	// frames is the set of encountered frames.
	frames       map[cdp.FrameID]*cdp.Frame
	execContexts map[cdp.FrameID]runtime.ExecutionContextID
	// cur is the current top level frame.
	cur cdp.FrameID

	// logging funcs
	logf, errf func(string, ...interface{})

	// Indicates if the target is a worker target.
	isWorker bool
}

func (t *Target) enclosingFrame(node *cdp.Node) cdp.FrameID {
	t.frameMu.RLock()
	top := t.frames[t.cur]
	t.frameMu.RUnlock()
	top.RLock()
	defer top.RUnlock()
	for {
		if node == nil {
			// Avoid crashing. This can happen if we're using an old
			// node that has been replaced, for example.
			return ""
		}
		if node.FrameID != "" {
			break
		}
		node = top.Nodes[node.ParentID]
	}
	return node.FrameID
}

// ensureFrame ensures the top frame of this target is loaded and returns the top frame,
// the root node and the ExecutionContextID of this top frame; otherwise, it will return
// false as its last return value.
func (t *Target) ensureFrame() (*cdp.Frame, *cdp.Node, runtime.ExecutionContextID, bool) {
	t.frameMu.RLock()
	frame := t.frames[t.cur]
	execCtx := t.execContexts[t.cur]
	t.frameMu.RUnlock()

	// the frame hasn't loaded yet.
	if frame == nil || execCtx == 0 {
		return nil, nil, 0, false
	}

	frame.RLock()
	root := frame.Root
	frame.RUnlock()

	if root == nil {
		// not root node yet?
		return nil, nil, 0, false
	}
	return frame, root, execCtx, true
}

func (t *Target) run(ctx context.Context) {
	type eventValue struct {
		method cdproto.MethodType
		value  interface{}
	}
	// syncEventQueue is used to handle events synchronously within Target.
	// TODO: If this queue gets full, the goroutine below could get stuck on
	// a send, and response callbacks would never run, resulting in a
	// deadlock. Can we fix this without potentially using lots of memory?
	syncEventQueue := make(chan eventValue, 4096)

	// This goroutine receives events from the browser, calls listeners, and
	// then passes the events onto the main goroutine for the target handler
	// to update itself.
	go func() {
		for {
			select {
			case <-ctx.Done():
				return

			case msg := <-t.messageQueue:
				if msg.ID != 0 {
					t.listenersMu.Lock()
					t.listeners = runListeners(t.listeners, msg)
					t.listenersMu.Unlock()
					continue
				}
				ev, err := cdproto.UnmarshalMessage(msg)
				if err != nil {
					if _, ok := err.(cdp.ErrUnknownCommandOrEvent); ok {
						// This is most likely an event received from an older
						// Chrome which a newer cdproto doesn't have, as it is
						// deprecated. Ignore that error.
						continue
					}
					t.errf("could not unmarshal event: %v", err)
					continue
				}
				t.listenersMu.Lock()
				t.listeners = runListeners(t.listeners, ev)
				t.listenersMu.Unlock()

				switch msg.Method.Domain() {
				case "Runtime", "Page", "DOM":
					select {
					case <-ctx.Done():
						return
					case syncEventQueue <- eventValue{msg.Method, ev}:
					}
				}
			}
		}
	}()

	for {
		select {
		case <-ctx.Done():
			return
		case ev := <-syncEventQueue:
			switch ev.method.Domain() {
			case "Runtime":
				t.runtimeEvent(ev.value)
			case "Page":
				t.pageEvent(ev.value)
			case "DOM":
				t.domEvent(ctx, ev.value)
			}
		}
	}
}

func (t *Target) Execute(ctx context.Context, method string, params easyjson.Marshaler, res easyjson.Unmarshaler) error {
	if method == target.CommandCloseTarget {
		return errors.New("to close the target, cancel its context or use chromedp.Cancel")
	}

	id := atomic.AddInt64(&t.browser.next, 1)
	lctx, cancel := context.WithCancel(ctx)
	ch := make(chan *cdproto.Message, 1)
	fn := func(ev interface{}) {
		if msg, ok := ev.(*cdproto.Message); ok && msg.ID == id {
			select {
			case <-ctx.Done():
			case ch <- msg:
			}
			cancel()
		}
	}
	t.listenersMu.Lock()
	t.listeners = append(t.listeners, cancelableListener{lctx, fn})
	t.listenersMu.Unlock()

	// send command
	var buf []byte
	if params != nil {
		var err error
		buf, err = easyjson.Marshal(params)
		if err != nil {
			return err
		}
	}
	cmd := &cdproto.Message{
		ID:        id,
		SessionID: t.SessionID,
		Method:    cdproto.MethodType(method),
		Params:    buf,
	}
	select {
	case <-ctx.Done():
		return ctx.Err()
	case t.browser.cmdQueue <- cmd:
	}

	// wait for result
	select {
	case <-ctx.Done():
		return ctx.Err()
	case msg := <-ch:
		switch {
		case msg == nil:
			return ErrChannelClosed
		case msg.Error != nil:
			return msg.Error
		case res != nil:
			return easyjson.Unmarshal(msg.Result, res)
		}
	}
	return nil
}

// runtimeEvent handles incoming runtime events.
func (t *Target) runtimeEvent(ev interface{}) {
	switch ev := ev.(type) {
	case *runtime.EventExecutionContextCreated:
		var aux struct {
			FrameID cdp.FrameID
		}
		if len(ev.Context.AuxData) == 0 {
			break
		}
		if err := json.Unmarshal(ev.Context.AuxData, &aux); err != nil {
			t.errf("could not decode executionContextCreated auxData %q: %v", ev.Context.AuxData, err)
			break
		}
		if aux.FrameID != "" {
			t.frameMu.Lock()
			t.execContexts[aux.FrameID] = ev.Context.ID
			t.frameMu.Unlock()
		}
	case *runtime.EventExecutionContextDestroyed:
		t.frameMu.Lock()
		for frameID, ctxID := range t.execContexts {
			if ctxID == ev.ExecutionContextID {
				delete(t.execContexts, frameID)
			}
		}
		t.frameMu.Unlock()
	case *runtime.EventExecutionContextsCleared:
		t.frameMu.Lock()
		for frameID := range t.execContexts {
			delete(t.execContexts, frameID)
		}
		t.frameMu.Unlock()
	}
}

// documentUpdated handles the document updated event, retrieving the document
// root for the root frame.
func (t *Target) documentUpdated(ctx context.Context) {
	t.frameMu.RLock()
	f := t.frames[t.cur]
	t.frameMu.RUnlock()
	if f == nil {
		// TODO: This seems to happen on CI, when running the tests
		// under the headless-shell Docker image. Figure out why.
		t.errf("received DOM.documentUpdated when there's no top-level frame")
		return
	}
	f.Lock()
	defer f.Unlock()

	// invalidate nodes
	if f.Root != nil {
		close(f.Root.Invalidated)
	}

	f.Nodes = make(map[cdp.NodeID]*cdp.Node)
	var err error
	f.Root, err = dom.GetDocument().Do(cdp.WithExecutor(ctx, t))
	if err == context.Canceled {
		return // TODO: perhaps not necessary, but useful to keep the tests less noisy
	}
	if err != nil {
		t.errf("could not retrieve document root for %s: %v", f.ID, err)
		return
	}
	f.Root.Invalidated = make(chan struct{})
	walk(f.Nodes, f.Root)
}

// pageEvent handles incoming page events.
func (t *Target) pageEvent(ev interface{}) {
	var id cdp.FrameID
	var op frameOp

	switch e := ev.(type) {
	case *page.EventFrameNavigated:
		t.frameMu.Lock()
		t.frames[e.Frame.ID] = e.Frame
		if e.Frame.ParentID == "" {
			// This frame is only the new top-level frame if it has
			// no parent.
			t.cur = e.Frame.ID
		}
		t.frameMu.Unlock()
		return

	case *page.EventFrameAttached:
		id, op = e.FrameID, frameAttached(e.ParentFrameID)

	case *page.EventFrameDetached:
		id, op = e.FrameID, frameDetached

	case *page.EventFrameStartedLoading:
		id, op = e.FrameID, frameStartedLoading

	case *page.EventFrameStoppedLoading:
		id, op = e.FrameID, frameStoppedLoading

		// ignored events
	case *page.EventFrameRequestedNavigation:
		return
	case *page.EventDomContentEventFired:
		return
	case *page.EventLoadEventFired:
		return
	case *page.EventFrameResized:
		return
	case *page.EventLifecycleEvent:
		return
	case *page.EventNavigatedWithinDocument:
		return
	case *page.EventJavascriptDialogOpening:
		return
	case *page.EventJavascriptDialogClosed:
		return
	case *page.EventWindowOpen:
		return
	case *page.EventDownloadWillBegin:
		return
	case *page.EventDownloadProgress:
		return
	case *page.EventDocumentOpened:
		return
	case *page.EventScreencastVisibilityChanged:
		return
	case *page.EventScreencastFrame:
		return

	default:
		t.errf("unhandled page event %T", ev)
		return
	}

	t.frameMu.Lock()
	f := t.frames[id]
	if f == nil {
		// This can happen if a frame is attached or starts loading
		// before it's ever navigated to. We won't have all the frame
		// details just yet, but that's okay.
		f = &cdp.Frame{ID: id}
		t.frames[id] = f
	}
	t.frameMu.Unlock()

	f.Lock()
	op(f)
	f.Unlock()
}

// domEvent handles incoming DOM events.
func (t *Target) domEvent(ctx context.Context, ev interface{}) {
	f := t.frames[t.cur]
	var id cdp.NodeID
	var op nodeOp

	switch e := ev.(type) {
	case *dom.EventDocumentUpdated:
		t.documentUpdated(ctx)
		return

	case *dom.EventSetChildNodes:
		id, op = e.ParentID, setChildNodes(f.Nodes, e.Nodes)

	case *dom.EventAttributeModified:
		id, op = e.NodeID, attributeModified(e.Name, e.Value)

	case *dom.EventAttributeRemoved:
		id, op = e.NodeID, attributeRemoved(e.Name)

	case *dom.EventInlineStyleInvalidated:
		if len(e.NodeIds) == 0 {
			return
		}

		id, op = e.NodeIds[0], inlineStyleInvalidated(e.NodeIds[1:])

	case *dom.EventCharacterDataModified:
		id, op = e.NodeID, characterDataModified(e.CharacterData)

	case *dom.EventChildNodeCountUpdated:
		id, op = e.NodeID, childNodeCountUpdated(e.ChildNodeCount)

	case *dom.EventChildNodeInserted:
		id, op = e.ParentNodeID, childNodeInserted(f.Nodes, e.PreviousNodeID, e.Node)

	case *dom.EventChildNodeRemoved:
		id, op = e.ParentNodeID, childNodeRemoved(f.Nodes, e.NodeID)

	case *dom.EventShadowRootPushed:
		id, op = e.HostID, shadowRootPushed(f.Nodes, e.Root)

	case *dom.EventShadowRootPopped:
		id, op = e.HostID, shadowRootPopped(f.Nodes, e.RootID)

	case *dom.EventPseudoElementAdded:
		id, op = e.ParentID, pseudoElementAdded(f.Nodes, e.PseudoElement)

	case *dom.EventPseudoElementRemoved:
		id, op = e.ParentID, pseudoElementRemoved(f.Nodes, e.PseudoElementID)

	case *dom.EventDistributedNodesUpdated:
		id, op = e.InsertionPointID, distributedNodesUpdated(e.DistributedNodes)

	default:
		t.errf("unhandled node event %T", ev)
		return
	}

	n, ok := f.Nodes[id]
	if !ok {
		// Node ID has been invalidated. Nothing to do.
		return
	}

	f.Lock()
	op(n)
	f.Unlock()
}
<|endoftext|>