<commit_before>package main

import (
	"bufio"
	"bytes"
	"encoding/json"
	"flag"
	"fmt"
	"io/ioutil"
	"os"
	"os/exec"
	"path/filepath"
	"regexp"
	"sort"
	"strings"
	"text/tabwriter"

	"github.com/pmezard/licenses/assets"
)

type Template struct {
	Title    string
	Nickname string
	Words    map[string]bool
}

func parseTemplate(content string) (*Template, error) {
	t := Template{}
	text := []byte{}
	state := 0
	scanner := bufio.NewScanner(strings.NewReader(content))
	for scanner.Scan() {
		line := strings.TrimSpace(scanner.Text())
		if state == 0 {
			if line == "---" {
				state = 1
			}
		} else if state == 1 {
			if line == "---" {
				state = 2
			} else {
				if strings.HasPrefix(line, "title:") {
					t.Title = strings.TrimSpace(line[len("title:"):])
				} else if strings.HasPrefix(line, "nickname:") {
					t.Nickname = strings.TrimSpace(line[len("nickname:"):])
				}
			}
		} else if state == 2 {
			text = append(text, scanner.Bytes()...)
			text = append(text, []byte("\n")...)
		}
	}
	t.Words = makeWordSet(text)
	return &t, scanner.Err()
}

func loadTemplates() ([]*Template, error) {
	templates := []*Template{}
	for _, a := range assets.Assets {
		templ, err := parseTemplate(a.Content)
		if err != nil {
			return nil, err
		}
		templates = append(templates, templ)
	}
	return templates, nil
}

var (
	reWords     = regexp.MustCompile(`[\w']+`)
	reCopyright = regexp.MustCompile(
		`\s*Copyright (?:©|\(c\)|\xC2\xA9)? ?(?:\d{4}|\[year\]).*?\s*`)
)

func makeWordSet(data []byte) map[string]bool {
	words := map[string]bool{}
	data = bytes.ToLower(data)
	data = reCopyright.ReplaceAll(data, nil)
	matches := reWords.FindAll(data, -1)
	for _, m := range matches {
		words[string(m)] = true
	}
	return words
}

// matchTemplates returns the best license template matching supplied data, and
// its score between 0 and 1.
func matchTemplates(license []byte, templates []*Template) (*Template, float64) {
	bestScore := float64(-1)
	var bestTemplate *Template
	words := makeWordSet(license)
	for _, t := range templates {
		common := 0
		for w := range words {
			if _, ok := t.Words[w]; ok {
				common++
			}
		}
		score := 2 * float64(common) / (float64(len(words)) + float64(len(t.Words)))
		if score > bestScore {
			bestScore = score
			bestTemplate = t
		}
	}
	return bestTemplate, bestScore
}

// fixEnv returns a copy of the process environment where GOPATH is adjusted to
// supplied value. It returns nil if gopath is empty.
func fixEnv(gopath string) []string {
	if gopath == "" {
		return nil
	}
	kept := []string{
		"GOPATH=" + gopath,
	}
	for _, env := range os.Environ() {
		if !strings.HasPrefix(env, "GOPATH=") {
			kept = append(kept, env)
		}
	}
	return kept
}

type MissingError struct {
	Err string
}

func (err *MissingError) Error() string {
	return err.Err
}

func listDependencies(gopath, pkg string) ([]string, error) {
	templ := "{{range .Deps}}{{.}}|{{end}}"
	cmd := exec.Command("go", "list", "-f", templ, pkg)
	cmd.Env = fixEnv(gopath)
	out, err := cmd.CombinedOutput()
	if err != nil {
		if strings.Contains(string(out), "cannot find package") {
			return nil, &MissingError{Err: string(out)}
		}
		return nil, fmt.Errorf("'go list -f %s %s' failed with:\n%s",
			templ, pkg, string(out))
	}
	deps := []string{}
	for _, s := range strings.Split(string(out), "|") {
		s = strings.TrimSpace(s)
		if s != "" {
			deps = append(deps, s)
		}
	}
	sort.Strings(deps)
	return deps, nil
}

func listStandardPackages(gopath string) ([]string, error) {
	cmd := exec.Command("go", "list", "std")
	cmd.Env = fixEnv(gopath)
	out, err := cmd.CombinedOutput()
	if err != nil {
		return nil, fmt.Errorf("go list std failed with:\n%s", string(out))
	}
	names := []string{}
	for _, s := range strings.Split(string(out), "\n") {
		s = strings.TrimSpace(s)
		if s != "" {
			names = append(names, s)
		}
	}
	return names, nil
}

type PkgInfo struct {
	Name       string
	Dir        string
	Root       string
	ImportPath string
}

func getPackagesInfo(gopath string, pkgs []string) ([]*PkgInfo, error) {
	args := []string{"list", "-json"}
	// TODO: split the list for platforms which do not support massive argument
	// lists.
	args = append(args, pkgs...)
	cmd := exec.Command("go", args...)
	cmd.Env = fixEnv(gopath)
	out, err := cmd.CombinedOutput()
	if err != nil {
		return nil, fmt.Errorf("go %s failed with:\n%s",
			strings.Join(args, " "), string(out))
	}
	infos := make([]*PkgInfo, 0, len(pkgs))
	decoder := json.NewDecoder(bytes.NewBuffer(out))
	for _, pkg := range pkgs {
		info := &PkgInfo{}
		err := decoder.Decode(info)
		if err != nil {
			return nil, fmt.Errorf("could not retrieve package information for %s", pkg)
		}
		if pkg != info.ImportPath {
			return nil, fmt.Errorf("package information mismatch: asked for %s, got %s",
				pkg, info.ImportPath)
		}
		infos = append(infos, info)
	}
	return infos, err
}

var (
	licenseFiles = []string{
		"LICENSE",
		"LICENSE.txt",
		"LICENSE.md",
		"COPYING",
		"COPYRIGHT",
	}
	reLicense = regexp.MustCompile(`(?i)^(?:` +
		`((?:un)?licen[sc]e)|` +
		`((?:un)?licen[sc]e\.(?:md|markdown|txt))|` +
		`(copy(?:ing|right)(?:\.[^.]+)?)|` +
		`(licen[sc]e\.[^.]+)` +
		`)$`)
)

// scoreLicenseName returns a factor between 0 and 1 weighting how likely
// supplied filename is a license file.
func scoreLicenseName(name string) float64 {
	m := reLicense.FindStringSubmatch(name)
	switch {
	case m == nil:
		break
	case m[1] != "":
		return 1.0
	case m[2] != "":
		return 0.9
	case m[3] != "":
		return 0.8
	case m[4] != "":
		return 0.7
	}
	return 0.
}

// findLicense looks for license files in package import path, and down to
// parent directories until a file is found or $GOPATH/src is reached. It
// returns the path and score of the best entry, an empty string if none was
// found.
func findLicense(info *PkgInfo) (string, error) {
	path := info.ImportPath
	for ; path != "."; path = filepath.Dir(path) {
		fis, err := ioutil.ReadDir(filepath.Join(info.Root, "src", path))
		if err != nil {
			return "", err
		}
		bestScore := float64(0)
		bestName := ""
		for _, fi := range fis {
			if !fi.Mode().IsRegular() {
				continue
			}
			score := scoreLicenseName(fi.Name())
			if score > bestScore {
				bestScore = score
				bestName = fi.Name()
			}
		}
		if bestName != "" {
			return filepath.Join(path, bestName), nil
		}
	}
	return "", nil
}

type License struct {
	Package  string
	Score    float64
	Template *Template
	Path     string
}

func listLicenses(gopath, pkg string) ([]License, error) {
	templates, err := loadTemplates()
	if err != nil {
		return nil, err
	}
	deps, err := listDependencies(gopath, pkg)
	if err != nil {
		if _, ok := err.(*MissingError); ok {
			return nil, err
		}
		return nil, fmt.Errorf("could not list %s dependencies: %s", pkg, err)
	}
	deps = append(deps, pkg)
	std, err := listStandardPackages(gopath)
	if err != nil {
		return nil, fmt.Errorf("could not list standard packages: %s", err)
	}
	stdSet := map[string]bool{}
	for _, n := range std {
		stdSet[n] = true
	}
	infos, err := getPackagesInfo(gopath, deps)
	if err != nil {
		return nil, err
	}
	licenses := []License{}
	for _, info := range infos {
		if stdSet[info.ImportPath] {
			continue
		}
		path, err := findLicense(info)
		if err != nil {
			return nil, err
		}
		license := License{
			Package: info.ImportPath,
			Path:    path,
		}
		if path != "" {
			data, err := ioutil.ReadFile(filepath.Join(info.Root, "src", path))
			if err != nil {
				return nil, err
			}
			t, score := matchTemplates(data, templates)
			license.Score = score
			license.Template = t
		}
		licenses = append(licenses, license)
	}
	return licenses, nil
}

// longestCommonPrefix returns the longest common prefix over import path
// components of supplied licenses.
func longestCommonPrefix(licenses []License) string {
	type Node struct {
		Name     string
		Children map[string]*Node
	}
	// Build a prefix tree. Not super efficient, but easy to do.
	root := &Node{
		Children: map[string]*Node{},
	}
	for _, l := range licenses {
		n := root
		for _, part := range strings.Split(l.Package, "/") {
			c := n.Children[part]
			if c == nil {
				c = &Node{
					Name:     part,
					Children: map[string]*Node{},
				}
				n.Children[part] = c
			}
			n = c
		}
	}
	n := root
	prefix := []string{}
	for {
		if len(n.Children) != 1 {
			break
		}
		for _, c := range n.Children {
			prefix = append(prefix, c.Name)
			n = c
			break
		}
	}
	return strings.Join(prefix, "/")
}

// groupLicenses returns the input licenses after grouping them by license path
// and find their longest import path common prefix. Entries with empty paths
// are left unchanged.
func groupLicenses(licenses []License) ([]License, error) {
	paths := map[string][]License{}
	for _, l := range licenses {
		if l.Path == "" {
			continue
		}
		paths[l.Path] = append(paths[l.Path], l)
	}
	for k, v := range paths {
		if len(v) <= 1 {
			continue
		}
		prefix := longestCommonPrefix(v)
		if prefix == "" {
			return nil, fmt.Errorf(
				"packages share the same license but not common prefix: %v", v)
		}
		l := v[0]
		l.Package = prefix
		paths[k] = []License{l}
	}
	kept := []License{}
	for _, l := range licenses {
		if l.Path == "" {
			kept = append(kept, l)
			continue
		}
		if v, ok := paths[l.Path]; ok {
			kept = append(kept, v[0])
			delete(paths, l.Path)
		}
	}
	return kept, nil
}

func printLicenses() error {
	flag.Usage = func() {
		fmt.Println(`Usage: licenses IMPORTPATH

licenses lists all dependencies of specified package or command, excluding
standard library packages, and prints their licenses. Licenses are detected by
looking for files named like LICENSE, COPYING, COPYRIGHT and other variants in
the package directory, and its parent directories until one is found. Files
content is matched against a set of well-known licenses and the best match is
displayed along with its score.

With -a, all individual packages are displayed instead of grouping them by
license files.
`)
		os.Exit(1)
	}
	all := flag.Bool("a", false, "display all individual packages")
	flag.Parse()
	if flag.NArg() != 1 {
		return fmt.Errorf("expect a single package argument, got %d", flag.NArg())
	}
	pkg := flag.Arg(0)

	confidence := 0.9
	licenses, err := listLicenses("", pkg)
	if err != nil {
		return err
	}
	if !*all {
		licenses, err = groupLicenses(licenses)
		if err != nil {
			return err
		}
	}
	w := tabwriter.NewWriter(os.Stdout, 1, 4, 2, ' ', 0)
	for _, l := range licenses {
		license := "?"
		if l.Template != nil {
			if l.Score >= confidence {
				license = fmt.Sprintf("%s (%2d%%)", l.Template.Title, int(100*l.Score))
			} else {
				license = fmt.Sprintf("? (%s, %2d%%)", l.Template.Title, int(100*l.Score))
			}
		}
		_, err = w.Write([]byte(l.Package + "\t" + license + "\n"))
		if err != nil {
			return err
		}
	}
	return w.Flush()
}

func main() {
	err := printLicenses()
	if err != nil {
		fmt.Fprintf(os.Stderr, "error: %s\n", err)
		os.Exit(1)
	}
}
<commit_msg>licenses: cache matched licenses by path<commit_after>package main

import (
	"bufio"
	"bytes"
	"encoding/json"
	"flag"
	"fmt"
	"io/ioutil"
	"os"
	"os/exec"
	"path/filepath"
	"regexp"
	"sort"
	"strings"
	"text/tabwriter"

	"github.com/pmezard/licenses/assets"
)

type Template struct {
	Title    string
	Nickname string
	Words    map[string]bool
}

func parseTemplate(content string) (*Template, error) {
	t := Template{}
	text := []byte{}
	state := 0
	scanner := bufio.NewScanner(strings.NewReader(content))
	for scanner.Scan() {
		line := strings.TrimSpace(scanner.Text())
		if state == 0 {
			if line == "---" {
				state = 1
			}
		} else if state == 1 {
			if line == "---" {
				state = 2
			} else {
				if strings.HasPrefix(line, "title:") {
					t.Title = strings.TrimSpace(line[len("title:"):])
				} else if strings.HasPrefix(line, "nickname:") {
					t.Nickname = strings.TrimSpace(line[len("nickname:"):])
				}
			}
		} else if state == 2 {
			text = append(text, scanner.Bytes()...)
			text = append(text, []byte("\n")...)
		}
	}
	t.Words = makeWordSet(text)
	return &t, scanner.Err()
}

func loadTemplates() ([]*Template, error) {
	templates := []*Template{}
	for _, a := range assets.Assets {
		templ, err := parseTemplate(a.Content)
		if err != nil {
			return nil, err
		}
		templates = append(templates, templ)
	}
	return templates, nil
}

var (
	reWords     = regexp.MustCompile(`[\w']+`)
	reCopyright = regexp.MustCompile(
		`\s*Copyright (?:©|\(c\)|\xC2\xA9)? ?(?:\d{4}|\[year\]).*?\s*`)
)

func makeWordSet(data []byte) map[string]bool {
	words := map[string]bool{}
	data = bytes.ToLower(data)
	data = reCopyright.ReplaceAll(data, nil)
	matches := reWords.FindAll(data, -1)
	for _, m := range matches {
		words[string(m)] = true
	}
	return words
}

// matchTemplates returns the best license template matching supplied data, and
// its score between 0 and 1.
func matchTemplates(license []byte, templates []*Template) (*Template, float64) {
	bestScore := float64(-1)
	var bestTemplate *Template
	words := makeWordSet(license)
	for _, t := range templates {
		common := 0
		for w := range words {
			if _, ok := t.Words[w]; ok {
				common++
			}
		}
		score := 2 * float64(common) / (float64(len(words)) + float64(len(t.Words)))
		if score > bestScore {
			bestScore = score
			bestTemplate = t
		}
	}
	return bestTemplate, bestScore
}

// fixEnv returns a copy of the process environment where GOPATH is adjusted to
// supplied value. It returns nil if gopath is empty.
func fixEnv(gopath string) []string {
	if gopath == "" {
		return nil
	}
	kept := []string{
		"GOPATH=" + gopath,
	}
	for _, env := range os.Environ() {
		if !strings.HasPrefix(env, "GOPATH=") {
			kept = append(kept, env)
		}
	}
	return kept
}

type MissingError struct {
	Err string
}

func (err *MissingError) Error() string {
	return err.Err
}

func listDependencies(gopath, pkg string) ([]string, error) {
	templ := "{{range .Deps}}{{.}}|{{end}}"
	cmd := exec.Command("go", "list", "-f", templ, pkg)
	cmd.Env = fixEnv(gopath)
	out, err := cmd.CombinedOutput()
	if err != nil {
		if strings.Contains(string(out), "cannot find package") {
			return nil, &MissingError{Err: string(out)}
		}
		return nil, fmt.Errorf("'go list -f %s %s' failed with:\n%s",
			templ, pkg, string(out))
	}
	deps := []string{}
	for _, s := range strings.Split(string(out), "|") {
		s = strings.TrimSpace(s)
		if s != "" {
			deps = append(deps, s)
		}
	}
	sort.Strings(deps)
	return deps, nil
}

func listStandardPackages(gopath string) ([]string, error) {
	cmd := exec.Command("go", "list", "std")
	cmd.Env = fixEnv(gopath)
	out, err := cmd.CombinedOutput()
	if err != nil {
		return nil, fmt.Errorf("go list std failed with:\n%s", string(out))
	}
	names := []string{}
	for _, s := range strings.Split(string(out), "\n") {
		s = strings.TrimSpace(s)
		if s != "" {
			names = append(names, s)
		}
	}
	return names, nil
}

type PkgInfo struct {
	Name       string
	Dir        string
	Root       string
	ImportPath string
}

func getPackagesInfo(gopath string, pkgs []string) ([]*PkgInfo, error) {
	args := []string{"list", "-json"}
	// TODO: split the list for platforms which do not support massive argument
	// lists.
	args = append(args, pkgs...)
	cmd := exec.Command("go", args...)
	cmd.Env = fixEnv(gopath)
	out, err := cmd.CombinedOutput()
	if err != nil {
		return nil, fmt.Errorf("go %s failed with:\n%s",
			strings.Join(args, " "), string(out))
	}
	infos := make([]*PkgInfo, 0, len(pkgs))
	decoder := json.NewDecoder(bytes.NewBuffer(out))
	for _, pkg := range pkgs {
		info := &PkgInfo{}
		err := decoder.Decode(info)
		if err != nil {
			return nil, fmt.Errorf("could not retrieve package information for %s", pkg)
		}
		if pkg != info.ImportPath {
			return nil, fmt.Errorf("package information mismatch: asked for %s, got %s",
				pkg, info.ImportPath)
		}
		infos = append(infos, info)
	}
	return infos, err
}

var (
	licenseFiles = []string{
		"LICENSE",
		"LICENSE.txt",
		"LICENSE.md",
		"COPYING",
		"COPYRIGHT",
	}
	reLicense = regexp.MustCompile(`(?i)^(?:` +
		`((?:un)?licen[sc]e)|` +
		`((?:un)?licen[sc]e\.(?:md|markdown|txt))|` +
		`(copy(?:ing|right)(?:\.[^.]+)?)|` +
		`(licen[sc]e\.[^.]+)` +
		`)$`)
)

// scoreLicenseName returns a factor between 0 and 1 weighting how likely
// supplied filename is a license file.
func scoreLicenseName(name string) float64 {
	m := reLicense.FindStringSubmatch(name)
	switch {
	case m == nil:
		break
	case m[1] != "":
		return 1.0
	case m[2] != "":
		return 0.9
	case m[3] != "":
		return 0.8
	case m[4] != "":
		return 0.7
	}
	return 0.
}

// findLicense looks for license files in package import path, and down to
// parent directories until a file is found or $GOPATH/src is reached. It
// returns the path and score of the best entry, an empty string if none was
// found.
func findLicense(info *PkgInfo) (string, error) {
	path := info.ImportPath
	for ; path != "."; path = filepath.Dir(path) {
		fis, err := ioutil.ReadDir(filepath.Join(info.Root, "src", path))
		if err != nil {
			return "", err
		}
		bestScore := float64(0)
		bestName := ""
		for _, fi := range fis {
			if !fi.Mode().IsRegular() {
				continue
			}
			score := scoreLicenseName(fi.Name())
			if score > bestScore {
				bestScore = score
				bestName = fi.Name()
			}
		}
		if bestName != "" {
			return filepath.Join(path, bestName), nil
		}
	}
	return "", nil
}

type License struct {
	Package  string
	Score    float64
	Template *Template
	Path     string
}

func listLicenses(gopath, pkg string) ([]License, error) {
	templates, err := loadTemplates()
	if err != nil {
		return nil, err
	}
	deps, err := listDependencies(gopath, pkg)
	if err != nil {
		if _, ok := err.(*MissingError); ok {
			return nil, err
		}
		return nil, fmt.Errorf("could not list %s dependencies: %s", pkg, err)
	}
	deps = append(deps, pkg)
	std, err := listStandardPackages(gopath)
	if err != nil {
		return nil, fmt.Errorf("could not list standard packages: %s", err)
	}
	stdSet := map[string]bool{}
	for _, n := range std {
		stdSet[n] = true
	}
	infos, err := getPackagesInfo(gopath, deps)
	if err != nil {
		return nil, err
	}

	// Cache matched licenses by path. Useful for package with a lot of
	// subpackages like bleve.
	type Match struct {
		t     *Template
		score float64
	}
	matched := map[string]Match{}

	licenses := []License{}
	for _, info := range infos {
		if stdSet[info.ImportPath] {
			continue
		}
		path, err := findLicense(info)
		if err != nil {
			return nil, err
		}
		license := License{
			Package: info.ImportPath,
			Path:    path,
		}
		if path != "" {
			fpath := filepath.Join(info.Root, "src", path)
			m, ok := matched[fpath]
			if !ok {
				data, err := ioutil.ReadFile(fpath)
				if err != nil {
					return nil, err
				}
				t, score := matchTemplates(data, templates)
				m = Match{
					t:     t,
					score: score,
				}
				matched[fpath] = m
			}
			license.Score = m.score
			license.Template = m.t
		}
		licenses = append(licenses, license)
	}
	return licenses, nil
}

// longestCommonPrefix returns the longest common prefix over import path
// components of supplied licenses.
func longestCommonPrefix(licenses []License) string {
	type Node struct {
		Name     string
		Children map[string]*Node
	}
	// Build a prefix tree. Not super efficient, but easy to do.
	root := &Node{
		Children: map[string]*Node{},
	}
	for _, l := range licenses {
		n := root
		for _, part := range strings.Split(l.Package, "/") {
			c := n.Children[part]
			if c == nil {
				c = &Node{
					Name:     part,
					Children: map[string]*Node{},
				}
				n.Children[part] = c
			}
			n = c
		}
	}
	n := root
	prefix := []string{}
	for {
		if len(n.Children) != 1 {
			break
		}
		for _, c := range n.Children {
			prefix = append(prefix, c.Name)
			n = c
			break
		}
	}
	return strings.Join(prefix, "/")
}

// groupLicenses returns the input licenses after grouping them by license path
// and find their longest import path common prefix. Entries with empty paths
// are left unchanged.
func groupLicenses(licenses []License) ([]License, error) {
	paths := map[string][]License{}
	for _, l := range licenses {
		if l.Path == "" {
			continue
		}
		paths[l.Path] = append(paths[l.Path], l)
	}
	for k, v := range paths {
		if len(v) <= 1 {
			continue
		}
		prefix := longestCommonPrefix(v)
		if prefix == "" {
			return nil, fmt.Errorf(
				"packages share the same license but not common prefix: %v", v)
		}
		l := v[0]
		l.Package = prefix
		paths[k] = []License{l}
	}
	kept := []License{}
	for _, l := range licenses {
		if l.Path == "" {
			kept = append(kept, l)
			continue
		}
		if v, ok := paths[l.Path]; ok {
			kept = append(kept, v[0])
			delete(paths, l.Path)
		}
	}
	return kept, nil
}

func printLicenses() error {
	flag.Usage = func() {
		fmt.Println(`Usage: licenses IMPORTPATH

licenses lists all dependencies of specified package or command, excluding
standard library packages, and prints their licenses. Licenses are detected by
looking for files named like LICENSE, COPYING, COPYRIGHT and other variants in
the package directory, and its parent directories until one is found. Files
content is matched against a set of well-known licenses and the best match is
displayed along with its score.

With -a, all individual packages are displayed instead of grouping them by
license files.
`)
		os.Exit(1)
	}
	all := flag.Bool("a", false, "display all individual packages")
	flag.Parse()
	if flag.NArg() != 1 {
		return fmt.Errorf("expect a single package argument, got %d", flag.NArg())
	}
	pkg := flag.Arg(0)

	confidence := 0.9
	licenses, err := listLicenses("", pkg)
	if err != nil {
		return err
	}
	if !*all {
		licenses, err = groupLicenses(licenses)
		if err != nil {
			return err
		}
	}
	w := tabwriter.NewWriter(os.Stdout, 1, 4, 2, ' ', 0)
	for _, l := range licenses {
		license := "?"
		if l.Template != nil {
			if l.Score >= confidence {
				license = fmt.Sprintf("%s (%2d%%)", l.Template.Title, int(100*l.Score))
			} else {
				license = fmt.Sprintf("? (%s, %2d%%)", l.Template.Title, int(100*l.Score))
			}
		}
		_, err = w.Write([]byte(l.Package + "\t" + license + "\n"))
		if err != nil {
			return err
		}
	}
	return w.Flush()
}

func main() {
	err := printLicenses()
	if err != nil {
		fmt.Fprintf(os.Stderr, "error: %s\n", err)
		os.Exit(1)
	}
}
<|endoftext|>


<commit_before>package srcgraph

import (
	"flag"
	"fmt"
	"io/ioutil"
	"log"
	"os"
	"os/exec"

	"strings"
	"github.com/sourcegraph/go-vcs"
	"github.com/sourcegraph/makex"
	"sourcegraph.com/sourcegraph/srcgraph/build"
	"sourcegraph.com/sourcegraph/srcgraph/buildstore"
	"sourcegraph.com/sourcegraph/srcgraph/task2"
)

func makefile(args []string) {
	make_(append(args, "-mf"))
}

func make_(args []string) {
	params := mustParseMakeParams(args)
	if err := params.verify(); err != nil {
		log.Fatal(err)
	}
	repoConfig := params.RepositoryConfig.GetRepositoryConfig(task2.DefaultContext)

	repoStore, err := buildstore.NewRepositoryStore(params.Repository.RootDir)
	if err != nil {
		log.Fatal(err)
	}

	// Get build directory (${REPO}/.sourcegraph-data/...)
	var buildDir string
	if params.Test {
		var err error
		buildDir, err = ioutil.TempDir("", fmt.Sprintf("sourcegraph-data.%s.%s-", strings.Replace(string(repoConfig.URI), "/", "-", -1),
			params.Repository.CommitID))
		if err != nil {
			log.Fatal(err)
		}
		if params.TestKeep {
			defer log.Printf("Test build directory: %s", buildDir)
		} else {
			defer os.RemoveAll(buildDir)
		}
	} else {
		buildDir, err = buildstore.BuildDir(repoStore, params.Repository.CommitID)
		if err != nil {
			log.Fatal(err)
		}
	}

	// Create Makefile
	mf, err := build.CreateMakefile(buildDir, repoConfig)
	if err != nil {
		log.Fatalf("error creating Makefile: %s", err)
	}

	if *Verbose || params.ShowOnly {
		// Show Makefile
		data, err := makex.Marshal(mf)
		if err != nil {
			log.Fatal(err)
		}
		fmt.Println(string(data))
		if params.ShowOnly {
			return
		}
	}

	// Run Makefile
	err = params.runMakefile(mf)
	if err != nil {
		log.Fatal(err)
	}

	if params.Test {
		// Compare expected with actual
		expectedBuildDir, err := buildstore.BuildDir(repoStore, params.Repository.CommitID)
		if err != nil {
			log.Fatal(err)
		}

		diffOut, err := exec.Command("diff", "-ur", "--exclude=config.json", expectedBuildDir, buildDir).CombinedOutput()
		log.Print("\n\n\n")
		log.Print("###########################")
		log.Print("##      TEST RESULTS     ##")
		log.Print("###########################")
		if len(diffOut) > 0 {
			diffStr := string(diffOut)
			diffStr = strings.Replace(diffStr, buildDir, "<test-build>", -1)
			log.Printf(diffStr)
			log.Printf("** FAIL **")
		} else if err != nil {
			log.Fatal(err)
			log.Printf("** ERROR **")
		} else if err == nil {
			log.Printf("** PASS **")
		}
	}
}

type makeParams struct {
	Repository       *repository
	RepositoryConfig *repositoryConfigurator
	Goals            []string

	ShowOnly bool
	Test     bool
	TestKeep bool
	Makex    *makex.Config
}

func mustParseMakeParams(args []string) *makeParams {
	fs := flag.NewFlagSet("make", flag.ExitOnError)
	r := AddRepositoryFlags(fs)
	rc := AddRepositoryConfigFlags(fs, r)
	showOnly := fs.Bool("mf", false, "print generated Makefile and exit")
	test := fs.Bool("test", false, "test build output against expected output in .sourcegraph-data/")
	testKeep := fs.Bool("test-keep", false, "do NOT delete test build directory after test, used in conjunction with -test")

	conf := &makex.Default
	makex.Flags(fs, conf, "")

	fs.Usage = func() {
		fmt.Fprintln(os.Stderr, `usage: `+Name+` make [options] [target...]

	   Generates and executes a Makefile that processes a repository, creating graph of
	   definitions, references, and dependencies in a repository's code at a specific
	   revision.

	   Run "`+Name+` makefile" to print the generated Makefile and exit.

	   This command uses makex to execute the Makefile, but the Makefile is also
	   compatible with GNU make. You can use the "`+Name+` makefile" command to
	   generate a Makefile to use with GNU make, if you'd like.

	   The options are:
	   `)
		fs.PrintDefaults()
		fmt.Fprintln(os.Stderr)
		os.Exit(1)
	}
	fs.Parse(args)

	return &makeParams{
		Repository:       r,
		RepositoryConfig: rc,
		Goals:            fs.Args(),
		ShowOnly:         *showOnly,
		Test:             *test,
		TestKeep:         *testKeep,
		Makex:            conf,
	}
}

func (p *makeParams) verify() error {
	vcsType := vcs.VCSByName[p.Repository.vcsTypeName]
	if vcsType == nil {
		return fmt.Errorf("%s: unknown VCS type %q", Name, p.Repository.vcsTypeName)
	}
	return nil
}

func (p *makeParams) runMakefile(mf *makex.Makefile) error {
	goals := p.Goals
	if len(goals) == 0 {
		if defaultRule := mf.DefaultRule(); defaultRule != nil {
			goals = []string{defaultRule.Target()}
		} else {
			// No rules in Makefile
			return nil
		}
	}

	mk := p.Makex.NewMaker(mf, goals...)

	if p.Makex.DryRun {
		err := mk.DryRun(os.Stdout)
		if err != nil {
			return err
		}
		return nil
	}

	err := os.Chdir(p.Repository.RootDir)
	if err != nil {
		return err
	}

	err = mk.Run()
	if err != nil {
		return err
	}

	return nil
}
<commit_msg>colorize srcgraph make -test output<commit_after>package srcgraph

import (
	"flag"
	"fmt"
	"io/ioutil"
	"log"
	"os"
	"os/exec"

	"strings"
	"github.com/aybabtme/color/brush"
	"github.com/sourcegraph/go-vcs"
	"github.com/sourcegraph/makex"
	"sourcegraph.com/sourcegraph/srcgraph/build"
	"sourcegraph.com/sourcegraph/srcgraph/buildstore"
	"sourcegraph.com/sourcegraph/srcgraph/task2"
)

func makefile(args []string) {
	make_(append(args, "-mf"))
}

func make_(args []string) {
	params := mustParseMakeParams(args)
	if err := params.verify(); err != nil {
		log.Fatal(err)
	}
	repoConfig := params.RepositoryConfig.GetRepositoryConfig(task2.DefaultContext)

	repoStore, err := buildstore.NewRepositoryStore(params.Repository.RootDir)
	if err != nil {
		log.Fatal(err)
	}

	// Get build directory (${REPO}/.sourcegraph-data/...)
	var buildDir string
	if params.Test {
		var err error
		buildDir, err = ioutil.TempDir("", fmt.Sprintf("sourcegraph-data.%s.%s-", strings.Replace(string(repoConfig.URI), "/", "-", -1),
			params.Repository.CommitID))
		if err != nil {
			log.Fatal(err)
		}
		if params.TestKeep {
			defer log.Printf("Test build directory: %s", buildDir)
		} else {
			defer os.RemoveAll(buildDir)
		}
	} else {
		buildDir, err = buildstore.BuildDir(repoStore, params.Repository.CommitID)
		if err != nil {
			log.Fatal(err)
		}
	}

	// Create Makefile
	mf, err := build.CreateMakefile(buildDir, repoConfig)
	if err != nil {
		log.Fatalf("error creating Makefile: %s", err)
	}

	if *Verbose || params.ShowOnly {
		// Show Makefile
		data, err := makex.Marshal(mf)
		if err != nil {
			log.Fatal(err)
		}
		fmt.Println(string(data))
		if params.ShowOnly {
			return
		}
	}

	// Run Makefile
	err = params.runMakefile(mf)
	if err != nil {
		log.Fatal(err)
	}

	if params.Test {
		// Compare expected with actual
		expectedBuildDir, err := buildstore.BuildDir(repoStore, params.Repository.CommitID)
		if err != nil {
			log.Fatal(err)
		}

		diffOut, err := exec.Command("diff", "-ur", "--exclude=config.json", expectedBuildDir, buildDir).CombinedOutput()
		log.Print("\n\n\n")
		log.Print("###########################")
		log.Print("##      TEST RESULTS     ##")
		log.Print("###########################")
		if len(diffOut) > 0 {
			diffStr := string(diffOut)
			diffStr = strings.Replace(diffStr, buildDir, "<test-build>", -1)
			log.Printf(diffStr)
			log.Printf(brush.Red("** FAIL **").String())
		} else if err != nil {
			log.Fatal(err)
			log.Printf(brush.Red("** ERROR **").String())
		} else if err == nil {
			log.Printf(brush.Green("** PASS **").String())
		}
	}
}

type makeParams struct {
	Repository       *repository
	RepositoryConfig *repositoryConfigurator
	Goals            []string

	ShowOnly bool
	Test     bool
	TestKeep bool
	Makex    *makex.Config
}

func mustParseMakeParams(args []string) *makeParams {
	fs := flag.NewFlagSet("make", flag.ExitOnError)
	r := AddRepositoryFlags(fs)
	rc := AddRepositoryConfigFlags(fs, r)
	showOnly := fs.Bool("mf", false, "print generated Makefile and exit")
	test := fs.Bool("test", false, "test build output against expected output in .sourcegraph-data/")
	testKeep := fs.Bool("test-keep", false, "do NOT delete test build directory after test, used in conjunction with -test")

	conf := &makex.Default
	makex.Flags(fs, conf, "")

	fs.Usage = func() {
		fmt.Fprintln(os.Stderr, `usage: `+Name+` make [options] [target...]

	   Generates and executes a Makefile that processes a repository, creating graph of
	   definitions, references, and dependencies in a repository's code at a specific
	   revision.

	   Run "`+Name+` makefile" to print the generated Makefile and exit.

	   This command uses makex to execute the Makefile, but the Makefile is also
	   compatible with GNU make. You can use the "`+Name+` makefile" command to
	   generate a Makefile to use with GNU make, if you'd like.

	   The options are:
	   `)
		fs.PrintDefaults()
		fmt.Fprintln(os.Stderr)
		os.Exit(1)
	}
	fs.Parse(args)

	return &makeParams{
		Repository:       r,
		RepositoryConfig: rc,
		Goals:            fs.Args(),
		ShowOnly:         *showOnly,
		Test:             *test,
		TestKeep:         *testKeep,
		Makex:            conf,
	}
}

func (p *makeParams) verify() error {
	vcsType := vcs.VCSByName[p.Repository.vcsTypeName]
	if vcsType == nil {
		return fmt.Errorf("%s: unknown VCS type %q", Name, p.Repository.vcsTypeName)
	}
	return nil
}

func (p *makeParams) runMakefile(mf *makex.Makefile) error {
	goals := p.Goals
	if len(goals) == 0 {
		if defaultRule := mf.DefaultRule(); defaultRule != nil {
			goals = []string{defaultRule.Target()}
		} else {
			// No rules in Makefile
			return nil
		}
	}

	mk := p.Makex.NewMaker(mf, goals...)

	if p.Makex.DryRun {
		err := mk.DryRun(os.Stdout)
		if err != nil {
			return err
		}
		return nil
	}

	err := os.Chdir(p.Repository.RootDir)
	if err != nil {
		return err
	}

	err = mk.Run()
	if err != nil {
		return err
	}

	return nil
}
<|endoftext|>


<commit_before>package main

import (
	"log"
	"net"

	baps3 "github.com/UniversityRadioYork/baps3-go"
)

// Maintains communications with the connector and connected clients.
// Also does any processing needed with the commands.
type hub struct {
	// All current connections, and their outbound channel.
	clients map[net.Conn]chan<- baps3.Message

	// For communication with the connector.
	cReqCh chan<- baps3.Message
	cResCh <-chan baps3.Message

	// Where new requests from clients come through.
	reqCh chan baps3.Message

	// Handlers for adding/removing connections.
	addCh chan *Client
	rmCh  chan *Client
	Quit  chan bool
}

var h = hub{
	clients: make(map[net.Conn]chan<- baps3.Message),

	reqCh: make(chan baps3.Message),

	addCh: make(chan *Client),
	rmCh:  make(chan *Client),
	Quit:  make(chan bool),
}

// Handles a new client connection.
// conn is the new connection object.
func (h *hub) handleNewConnection(conn net.Conn) {
	defer conn.Close()
	client := &Client{
		conn:  conn,
		resCh: make(chan baps3.Message),
		tok:   baps3.NewTokeniser(),
	}

	// Register user
	h.addCh <- client

	go client.Read(h.reqCh, h.rmCh)
	client.Write(client.resCh, h.rmCh)
}

func makeWelcomeMsg() *baps3.Message {
	return baps3.NewMessage(baps3.RsOhai).AddArg("listd-" + LD_VERSION)
}

func makeFeaturesMsg() *baps3.Message {
	return baps3.NewMessage(baps3.RsFeatures).AddArg("lol")
}

// Handles a request from a client.
// Falls through to the connector cReqCh if command is "not understood".
func (h *hub) processRequest(req baps3.Message) {
	// TODO: Do something else
	log.Println("New request:", req.String())
	h.cReqCh <- req
}

// Broadcasts a response (res) to all connected clients.
func (h *hub) processResponse(res baps3.Message) {
	// TODO: Do something else
	log.Println("New response:", res.String())
	for _, ch := range h.clients {
		ch <- res
	}
}

// Listens for new connections on addr:port and spins up the relevant goroutines.
func (h *hub) runListener(addr string, port string) {
	netListener, err := net.Listen("tcp", addr+":"+port)
	if err != nil {
		log.Println("Listening error:", err.Error())
		return
	}

	// Get new connections
	go func() {
		for {
			conn, err := netListener.Accept()
			if err != nil {
				log.Println("Error accepting connection:", err.Error())
				continue
			}

			go h.handleNewConnection(conn)
		}
	}()

	for {
		select {
		case msg := <-h.cResCh:
			h.processResponse(msg)
		case msg := <-h.reqCh:
			h.processRequest(msg)
		case client := <-h.addCh:
			h.clients[client.conn] = client.resCh
			client.resCh <- *makeWelcomeMsg()
			client.resCh <- *makeFeaturesMsg()
			log.Println("New connection from", client.conn.RemoteAddr())
		case client := <-h.rmCh:
			close(client.resCh)
			delete(h.clients, client.conn)
			log.Println("Closed connection from", client.conn.RemoteAddr())
		case <-h.Quit:
			log.Println("Closing all connections")
			for conn, ch := range h.clients {
				close(ch)
				delete(h.clients, conn)
			}
			//			h.Quit <- true
		}
	}
}

// Sets up the connector channels for the hub object.
func (h *hub) setConnector(cReqCh chan<- baps3.Message, cResCh <-chan baps3.Message) {
	h.cReqCh = cReqCh
	h.cResCh = cResCh
}
<commit_msg>Store clients, instead of their connections<commit_after>package main

import (
	"log"
	"net"

	baps3 "github.com/UniversityRadioYork/baps3-go"
)

// Maintains communications with the connector and connected clients.
// Also does any processing needed with the commands.
type hub struct {
	// All current clients.
	clients map[*Client]bool

	// For communication with the connector.
	cReqCh chan<- baps3.Message
	cResCh <-chan baps3.Message

	// Where new requests from clients come through.
	reqCh chan baps3.Message

	// Handlers for adding/removing connections.
	addCh chan *Client
	rmCh  chan *Client
	Quit  chan bool
}

var h = hub{
	clients: make(map[*Client]bool),

	reqCh: make(chan baps3.Message),

	addCh: make(chan *Client),
	rmCh:  make(chan *Client),
	Quit:  make(chan bool),
}

// Handles a new client connection.
// conn is the new connection object.
func (h *hub) handleNewConnection(conn net.Conn) {
	defer conn.Close()
	client := &Client{
		conn:  conn,
		resCh: make(chan baps3.Message),
		tok:   baps3.NewTokeniser(),
	}

	// Register user
	h.addCh <- client

	go client.Read(h.reqCh, h.rmCh)
	client.Write(client.resCh, h.rmCh)
}

func makeWelcomeMsg() *baps3.Message {
	return baps3.NewMessage(baps3.RsOhai).AddArg("listd-" + LD_VERSION)
}

func makeFeaturesMsg() *baps3.Message {
	return baps3.NewMessage(baps3.RsFeatures).AddArg("lol")
}

// Handles a request from a client.
// Falls through to the connector cReqCh if command is "not understood".
func (h *hub) processRequest(req baps3.Message) {
	// TODO: Do something else
	log.Println("New request:", req.String())
	h.cReqCh <- req
}

// Broadcasts a response (res) to all connected clients.
func (h *hub) processResponse(res baps3.Message) {
	// TODO: Do something else
	log.Println("New response:", res.String())
	for c, _ := range h.clients {
		c.resCh <- res
	}
}

// Listens for new connections on addr:port and spins up the relevant goroutines.
func (h *hub) runListener(addr string, port string) {
	netListener, err := net.Listen("tcp", addr+":"+port)
	if err != nil {
		log.Println("Listening error:", err.Error())
		return
	}

	// Get new connections
	go func() {
		for {
			conn, err := netListener.Accept()
			if err != nil {
				log.Println("Error accepting connection:", err.Error())
				continue
			}

			go h.handleNewConnection(conn)
		}
	}()

	for {
		select {
		case msg := <-h.cResCh:
			h.processResponse(msg)
		case msg := <-h.reqCh:
			h.processRequest(msg)
		case client := <-h.addCh:
			h.clients[client] = true
			client.resCh <- *makeWelcomeMsg()
			client.resCh <- *makeFeaturesMsg()
			log.Println("New connection from", client.conn.RemoteAddr())
		case client := <-h.rmCh:
			close(client.resCh)
			delete(h.clients, client)
			log.Println("Closed connection from", client.conn.RemoteAddr())
		case <-h.Quit:
			log.Println("Closing all connections")
			for c, _ := range h.clients {
				close(c.resCh)
				delete(h.clients, c)
			}
			//			h.Quit <- true
		}
	}
}

// Sets up the connector channels for the hub object.
func (h *hub) setConnector(cReqCh chan<- baps3.Message, cResCh <-chan baps3.Message) {
	h.cReqCh = cReqCh
	h.cResCh = cResCh
}
<|endoftext|>


<commit_before>package peerstream

import (
	"errors"
	"net"
	"time"
)

type Listener struct {
	netList net.Listener
	groups  groupSet
	swarm   *Swarm

	acceptErr chan error
}

func newListener(nl net.Listener, s *Swarm) *Listener {
	return &Listener{
		netList:   nl,
		swarm:     s,
		acceptErr: make(chan error, 10),
	}
}

// NetListener is the underlying net.Listener
func (l *Listener) NetListener() net.Listener {
	return l.netList
}

// Groups returns the groups this Listener belongs to
func (l *Listener) Groups() []Group {
	return l.groups.Groups()
}

// InGroup returns whether this Listener belongs to a Group
func (l *Listener) InGroup(g Group) bool {
	return l.groups.Has(g)
}

// AddGroup assigns given Group to Listener
func (l *Listener) AddGroup(g Group) {
	l.groups.Add(g)
}

// ListenersWithGroup narrows down a set of listeners to those in given group.
func ListenersWithGroup(g Group, ls []*Listener) []*Listener {
	var out []*Listener
	for _, l := range ls {
		if l.InGroup(g) {
			out = append(out, l)
		}
	}
	return out
}

// accept continously accepts incoming connections and
// adds them to the listener's Swarm. is is meant to be
// run in a goroutine.
// TODO: add rate limiting
func (l *Listener) accept() {
	defer l.teardown()

	// catching the error here is odd. doing what net/http does:
	// http://golang.org/src/net/http/server.go?s=51504:51550#L1728
	var tempDelay time.Duration // how long to sleep on accept failure

	isTemporaryErr := func(e error) bool {
		if ne, ok := e.(net.Error); ok && ne.Temporary() {
			if tempDelay == 0 {
				tempDelay = 5 * time.Millisecond
			} else {
				tempDelay *= 2
			}
			if max := 1 * time.Second; tempDelay > max {
				tempDelay = max
			}

			time.Sleep(tempDelay)
			return true
		}
		return false
	}

	// loop forever accepting connections
	for {
		conn, err := l.netList.Accept()
		if err != nil {
			l.acceptErr <- err
			if isTemporaryErr(err) {
				continue
			}
			return // ok, problems. bail.
		}
		tempDelay = 0

		// add conn to swarm and listen for incoming streams
		// log.Printf("accepted conn %s\n", conn.RemoteAddr())
		conn2, err := l.swarm.addConn(conn, true)
		if err != nil {
			l.acceptErr <- err
			continue
		}
		conn2.groups.AddSet(&l.groups) // add out groups
	}
}

// AcceptError returns the error that we **might** on listener close
func (l *Listener) AcceptErrors() <-chan error {
	return l.acceptErr
}

func (l *Listener) teardown() {
	// in case we exit from network errors (accept fails) but
	// (a) client doesn't call Close, and (b) listener remains open)
	l.netList.Close()

	close(l.acceptErr)

	// remove self from swarm
	l.swarm.listenerLock.Lock()
	delete(l.swarm.listeners, l)
	l.swarm.listenerLock.Unlock()
}

func (l *Listener) Close() error {
	return l.netList.Close()
}

// addListener is the internal version of AddListener.
func (s *Swarm) addListener(nl net.Listener) (*Listener, error) {
	if nl == nil {
		return nil, errors.New("nil listener")
	}

	s.listenerLock.Lock()
	defer s.listenerLock.Unlock()

	// first, check if we already have it...
	for l := range s.listeners {
		if l.netList == nl {
			return l, nil
		}
	}

	l := newListener(nl, s)
	s.listeners[l] = struct{}{}
	go l.accept()
	return l, nil
}
<commit_msg>listener: do not block on temporary errs<commit_after>package peerstream

import (
	"errors"
	"net"
	"time"
)

type Listener struct {
	netList net.Listener
	groups  groupSet
	swarm   *Swarm

	acceptErr chan error
}

func newListener(nl net.Listener, s *Swarm) *Listener {
	return &Listener{
		netList:   nl,
		swarm:     s,
		acceptErr: make(chan error, 10),
	}
}

// NetListener is the underlying net.Listener
func (l *Listener) NetListener() net.Listener {
	return l.netList
}

// Groups returns the groups this Listener belongs to
func (l *Listener) Groups() []Group {
	return l.groups.Groups()
}

// InGroup returns whether this Listener belongs to a Group
func (l *Listener) InGroup(g Group) bool {
	return l.groups.Has(g)
}

// AddGroup assigns given Group to Listener
func (l *Listener) AddGroup(g Group) {
	l.groups.Add(g)
}

// ListenersWithGroup narrows down a set of listeners to those in given group.
func ListenersWithGroup(g Group, ls []*Listener) []*Listener {
	var out []*Listener
	for _, l := range ls {
		if l.InGroup(g) {
			out = append(out, l)
		}
	}
	return out
}

// accept continously accepts incoming connections and
// adds them to the listener's Swarm. is is meant to be
// run in a goroutine.
// TODO: add rate limiting
func (l *Listener) accept() {
	defer l.teardown()

	// catching the error here is odd. doing what net/http does:
	// http://golang.org/src/net/http/server.go?s=51504:51550#L1728
	var tempDelay time.Duration // how long to sleep on accept failure

	isTemporaryErr := func(e error) bool {
		if ne, ok := e.(net.Error); ok && ne.Temporary() {
			if tempDelay == 0 {
				tempDelay = 5 * time.Millisecond
			} else {
				tempDelay *= 2
			}
			if max := 1 * time.Second; tempDelay > max {
				tempDelay = max
			}

			time.Sleep(tempDelay)
			return true
		}
		return false
	}

	// loop forever accepting connections
	for {
		conn, err := l.netList.Accept()
		if err != nil {
			if isTemporaryErr(err) {
				continue
			}
			l.acceptErr <- err
			return // ok, problems. bail.
		}
		tempDelay = 0

		// add conn to swarm and listen for incoming streams
		// log.Printf("accepted conn %s\n", conn.RemoteAddr())
		conn2, err := l.swarm.addConn(conn, true)
		if err != nil {
			l.acceptErr <- err
			continue
		}
		conn2.groups.AddSet(&l.groups) // add out groups
	}
}

// AcceptError returns the error that we **might** on listener close
func (l *Listener) AcceptErrors() <-chan error {
	return l.acceptErr
}

func (l *Listener) teardown() {
	// in case we exit from network errors (accept fails) but
	// (a) client doesn't call Close, and (b) listener remains open)
	l.netList.Close()

	close(l.acceptErr)

	// remove self from swarm
	l.swarm.listenerLock.Lock()
	delete(l.swarm.listeners, l)
	l.swarm.listenerLock.Unlock()
}

func (l *Listener) Close() error {
	return l.netList.Close()
}

// addListener is the internal version of AddListener.
func (s *Swarm) addListener(nl net.Listener) (*Listener, error) {
	if nl == nil {
		return nil, errors.New("nil listener")
	}

	s.listenerLock.Lock()
	defer s.listenerLock.Unlock()

	// first, check if we already have it...
	for l := range s.listeners {
		if l.netList == nl {
			return l, nil
		}
	}

	l := newListener(nl, s)
	s.listeners[l] = struct{}{}
	go l.accept()
	return l, nil
}
<|endoftext|>


<commit_before>// Copyright (c) 2016 Pani Networks
// All Rights Reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License"); you may
// not use this file except in compliance with the License. You may obtain
// a copy of the License at
//
//   http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
// WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
// License for the specific language governing permissions and limitations
// under the License.

// Listens to Kubernetes for policy updates
package kubernetes

import (
	"bytes"
	"errors"
	"fmt"
	"github.com/romana/core/common"
	"github.com/romana/core/tenant"
	"io"
	"log"
	"net/http"
	"strings"
	//	"encoding/json"
	//	"time"
	//	"net/http"
	//	"runtime"
)

// readChunk reads the next chunk from the provided reader.
func readChunk(reader io.Reader) ([]byte, error) {
	result := make([]byte, 0)
	for {
		buf := make([]byte, bytes.MinRead)
		n, err := reader.Read(buf)
		if n < bytes.MinRead {
			result = append(result, buf[0:n]...)
			if err != nil {
				return result, err
			} else {
				break
			}
		}
		result = append(result, buf...)
	}
	return result, nil
}

// listen connects to kubernetesURL and sends each
// chunk received to the ch channel. It exits on an
// error.
//func listen(ch chan []byte, kubernetesURL string) {
//	log.Printf("Listening to %s, currently %d goroutines are running", kubernetesURL, runtime.NumGoroutine())
//	resp, err := http.Get(kubernetesURL)
//	if err != nil {
//		log.Printf("Error connecting: %#v", err)
//		log.Println("Closing channel")
//		close(ch)
//		log.Println("Exiting goroutine")
//		return
//	}
//	//	chunkedReader := resp.Body
//	for {
//		chunk, err := readChunk(resp.Body)
//		if err != nil {
//			if err != io.EOF {
//				log.Printf("Error reading chunk: %#v", err)
//			}
//			if len(chunk) != 0 {
//				ch <- chunk
//			}
//			log.Println("Closing channel")
//			close(ch)
//			log.Println("Exiting goroutine")
//			return
//		}
//		if len(chunk) != 0 {
//			time.Sleep(1 * time.Millisecond)
//		} else {
//			ch <- chunk
//		}
//	}
//}

// processChunk processes a chunk received from Kubernetes. A chunk
// is a new update.
//func processChunk(chunk []byte) error {
//	m := make(map[string]interface{})
//	err := json.Unmarshal(chunk, &m)
//	if err != nil {
//		return err
//	}
//	method := m["method"]
//	if method == nil {
//		return errors.New("Expected 'method' field")
//	}
//	methodStr = strings.ToLower(method.(string))
//	networkPolicyIfc := m["policy_definition"]
//	if networkPolicyIfc == nil {
//		return errors.New("Expected 'policy_definition' field")
//	}
//	kubeNetworkPolicy := networkPolicyIfc.(map[string]interface{})
//	romanaNetworkPolicy, err := translateNetworkPolicy(kubeNetworkPolicy)
//	if err != nil {
//		return err
//	}
//
//	if methodStr == "added" {
//		applyNetworkPolicy(networkPolicyActionAdd, romanaNetworkPolicy)
//	} else if methodStr == "deleted" {
//		applyNetworkPolicy(networkPolicyActioDelete, romanaNetworkPolicy)
//	} else if methodStr == "modified" {
//		applyNetworkPolicy(networkPolicyActioModify, romanaNetworkPolicy)
//	} else {
//		return common.NewError("Unexpected method '%s'", methodStr)
//	}
//}

// Run implements the main loop, reconnecting as needed.
//func RunListener0(rootURL string, kubeURL string) {
//	l := kubeListener{kubeURL: kubeURL, restClient: common.NewRestClient(common.GetDefaultRestClientConfig(rootURL))}
//	l.Run()
//}
//
//func (l kubeListener) Run0() {
//	for {
//		ch := make(chan []byte)
//		go listen(ch, l.kubernetesURL)
//		for {
//			chunk, ok := <-ch
//			log.Println(ok)
//			if chunk != "" {
//				log.Printf("Read chunk %#v\n%s\n------------", ok, string(chunk))
//				err = processChunk(chunk)
//				if err != nil {
//					// TODO is there any other way to handle this?
//					log.Printf("Error processing chunk %s: %#v", string(chunk), err)
//				}
//			}
//			if !ok {
//				break
//			}
//		}
//	}
//}

/////////////////////////////////////////////////////////////

type networkPolicyAction int

const (
	networkPolicyActionDelete networkPolicyAction = iota
	networkPolicyActionAdd
	networkPolicyActionModify
)

type kubeListener struct {
	config                    common.ServiceConfig
	restClient                *common.RestClient
	kubeUrl                   string
	namespaceNotificationPath string
	policyNotificationPathPrefix string
	policyNotificationPathPostfix string
	segmentLabelName          string
}

// Routes returns various routes used in the service.
func (l *kubeListener) Routes() common.Routes {
	routes := common.Routes{}
	return routes
}

// Name implements method of Service interface.
func (l *kubeListener) Name() string {
	return "kubernetesListener"
}

// SetConfig implements SetConfig function of the Service interface.
func (l *kubeListener) SetConfig(config common.ServiceConfig) error {
	m := config.ServiceSpecific
	if m["kubernetes_url"] == "" {
		return errors.New("kubernetes_url required.")
	}
	l.kubeUrl = m["kubernetes_url"].(string)

	if m["namespace_notification_path"] == "" {
		return errors.New("namespace_notification_path required.")
	}
	l.namespaceNotificationPath = m["namespace_notification_path"].(string)

	if m["policy_notification_path_prefix"] == "" {
		return errors.New("policy_notification_path_prefix required.")
	}
	l.policyNotificationPathPrefix = m["policy_notification_path_prefix"].(string)

	if m["policy_notification_path_postfix"] == "" {
		return errors.New("policy_notification_path_postfix required.")
	}
	l.policyNotificationPathPostfix = m["policy_notification_path_postfix"].(string)

	if m["segment_label_name"] == "" {
		return errors.New("segment_label_name required.")
	}
	l.segmentLabelName = m["segment_label_name"].(string)

	return nil
}

// Run configures and runs listener service.
func Run(rootServiceURL string, cred *common.Credential) (*common.RestServiceInfo, error) {
	clientConfig := common.GetDefaultRestClientConfig(rootServiceURL)
	clientConfig.Credential = cred
	client, err := common.NewRestClient(clientConfig)
	if err != nil {
		return nil, err
	}
	kubeListener := &kubeListener{}
	kubeListener.restClient = client
	config, err := client.GetServiceConfig(kubeListener.Name())
	if err != nil {
		return nil, err
	}
	return common.InitializeService(kubeListener, *config)
}

// getOrAddSegment finds a segment (based on segment selector)
//
func (l *kubeListener) getOrAddSegment(tenantServiceURL string, namespace string, kubeSegmentID string) (*tenant.Segment, error) {
	segment := &tenant.Segment{}
	segmentsURL := fmt.Sprintf("%s/tenants/%s/segments", tenantServiceURL, namespace)
	err := l.restClient.Get(fmt.Sprintf("%s/%s", segmentsURL, kubeSegmentID), segment)
	if err == nil {
		return segment, nil
	}
	switch err := err.(type) {
	case common.HttpError:
		if err.StatusCode == http.StatusNotFound {
			// Not found, so let's create a segment.
			segreq := tenant.Segment{Name: kubeSegmentID, ExternalID: kubeSegmentID}
			err2 := l.restClient.Post(segmentsURL, segreq, segment)
			if err2 == nil {
				// Successful creation.
				return segment, nil
			}
			// Creation of non-existing segment gave an error.
			switch err2 := err2.(type) {
			case common.HttpError:
				// Maybe someone else just created a segment between the original
				// lookup and now?
				if err2.StatusCode == http.StatusConflict {
					switch details := err2.Details.(type) {
					case tenant.Segment:
						// We expect the existing segment to be returned in the details field.
						return &details, nil
					default:
						// This is unexpected...
						return nil, err
					}
				}
				// Any other HTTP error other than a Conflict here - return it.
				return nil, err2
			default:
				// Any other error - return it
				return nil, err2
			}
		}
		// Any other HTTP error other than a Not found here - return it
		return nil, err
	default:
		// Any other error - return it
		return nil, err
	}
}

// translateNetworkPolicy translates a Kubernetes policy into
// Romana policy (see common.Policy) with the following rules:
// 1. Kubernetes Namespace corresponds to Romana Tenant
// 2. If Romana Tenant does not exist it is an error (a tenant should
//    automatically have been created when the namespace was added)
func (l *kubeListener) translateNetworkPolicy(kubePolicy *KubeObject) (common.Policy, error) {
	log.Printf("translateNetworkPolicy(): Received %#v", kubePolicy)
	romanaPolicy := &common.Policy{Direction: common.PolicyDirectionIngress}
	ns := kubePolicy.Metadata.Namespace
	tenantURL, err := l.restClient.GetServiceUrl("tenant")
	if err != nil {
		return *romanaPolicy, err
	}
	t := &tenant.Tenant{}
	err = l.restClient.Get(fmt.Sprintf("%s/tenants/%s", tenantURL, ns), t)
	log.Printf("translateNetworkPolicy(): For namespace %s got %#v / %#v", ns, t, err)
	if err != nil {
		return *romanaPolicy, err
	}
	tenantId := t.ID
	tenantExternalId := t.ExternalID

	kubeSegmentID := kubePolicy.Spec.PodSelector[l.segmentLabelName]
	if kubeSegmentID == "" {
		return *romanaPolicy, common.NewError("Expected segment to be specified in podSelector part as '%s'", l.segmentLabelName)
	}

	segment, err := l.getOrAddSegment(tenantURL, ns, kubeSegmentID)
	if err != nil {
		return *romanaPolicy, err
	}
	segmentId := segment.ID
	appliedTo := common.Endpoint{TenantID: tenantId, SegmentID: segmentId}
	romanaPolicy.AppliedTo = make([]common.Endpoint, 1)
	romanaPolicy.AppliedTo[0] = appliedTo
	romanaPolicy.Peers = []common.Endpoint{}
	from := kubePolicy.Spec.AllowIncoming.From
	// This is subject to change once the network specification in Kubernetes is finalized.
	// Right now it is a work in progress.
	if from != nil {
		for _, entry := range from {
			pods := entry.Pods
			fromKubeSegmentID := pods[l.segmentLabelName]
			if fromKubeSegmentID == "" {
				return *romanaPolicy, common.NewError("Expected segment to be specified in podSelector part as '%s'", l.segmentLabelName)
			}
			fromSegment, err := l.getOrAddSegment(tenantURL, ns, fromKubeSegmentID)
			if err != nil {
				return *romanaPolicy, err
			}
			peer := common.Endpoint{TenantID: tenantId, TenantExternalID: tenantExternalId, SegmentID: fromSegment.ID, SegmentExternalID: fromSegment.ExternalID}
			romanaPolicy.Peers = append(romanaPolicy.Peers, peer)
		}
		toPorts := kubePolicy.Spec.AllowIncoming.ToPorts
		romanaPolicy.Rules = common.Rules{}
		for _, toPort := range toPorts {
			proto := strings.ToLower(toPort.Protocol)
			ports := []uint{toPort.Port}
			rule := common.Rule{Protocol: proto, Ports: ports}
			romanaPolicy.Rules = append(romanaPolicy.Rules, rule)
		}
	}
	err = romanaPolicy.Validate()
	if err != nil {
		return *romanaPolicy, err
	}
	return *romanaPolicy, nil
}

func (l *kubeListener) applyNetworkPolicy(action networkPolicyAction, romanaNetworkPolicy common.Policy) error {
	policyUrl, err := l.restClient.GetServiceUrl("policy")
	if err != nil {
		return err
	}
	policyUrl = fmt.Sprintf("%s/policies", policyUrl)
	switch action {
	case networkPolicyActionAdd:
		err := l.restClient.Post(policyUrl, romanaNetworkPolicy, &romanaNetworkPolicy)
		if err != nil {
			return err
		}
		log.Printf("Applied policy %#v", romanaNetworkPolicy)
	case networkPolicyActionDelete:
		policyUrl = fmt.Sprintf("%s/%d", policyUrl, romanaNetworkPolicy.ID)
		err := l.restClient.Delete(policyUrl, nil, &romanaNetworkPolicy)
		if err != nil {
			return err
		}
	default:
		return errors.New("Unsupported operation")
	}
	return nil
}

func (l *kubeListener) Initialize() error {
	log.Printf("%s: Starting server", l.Name())
	nsUrl, err := common.CleanURL(fmt.Sprintf("%s%s", l.kubeUrl, l.namespaceNotificationPath))
	if err != nil {
		return err
	}
	log.Printf("Starting to listen on %s", nsUrl)
	done := make(chan Done)
	nsEvents, err := l.nsWatch(done, nsUrl)
	if err != nil {
		log.Fatal("Namespace watcher failed to start", err)
	}

	events := l.conductor(nsEvents, done)
	l.process(events, done)
	log.Println("All routines started")
	return nil
}

func CreateSchema(rootServiceURL string, overwrite bool) error {
	return nil
}
<commit_msg>Added doc<commit_after>// Copyright (c) 2016 Pani Networks
// All Rights Reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License"); you may
// not use this file except in compliance with the License. You may obtain
// a copy of the License at
//
//   http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
// WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
// License for the specific language governing permissions and limitations
// under the License.

// Listens to Kubernetes for policy updates
package kubernetes

import (
	"bytes"
	"errors"
	"fmt"
	"github.com/romana/core/common"
	"github.com/romana/core/tenant"
	"io"
	"log"
	"net/http"
	"strings"
	//	"encoding/json"
	//	"time"
	//	"net/http"
	//	"runtime"
)

// readChunk reads the next chunk from the provided reader.
func readChunk(reader io.Reader) ([]byte, error) {
	result := make([]byte, 0)
	for {
		buf := make([]byte, bytes.MinRead)
		n, err := reader.Read(buf)
		if n < bytes.MinRead {
			result = append(result, buf[0:n]...)
			if err != nil {
				return result, err
			} else {
				break
			}
		}
		result = append(result, buf...)
	}
	return result, nil
}

// listen connects to kubernetesURL and sends each
// chunk received to the ch channel. It exits on an
// error.
//func listen(ch chan []byte, kubernetesURL string) {
//	log.Printf("Listening to %s, currently %d goroutines are running", kubernetesURL, runtime.NumGoroutine())
//	resp, err := http.Get(kubernetesURL)
//	if err != nil {
//		log.Printf("Error connecting: %#v", err)
//		log.Println("Closing channel")
//		close(ch)
//		log.Println("Exiting goroutine")
//		return
//	}
//	//	chunkedReader := resp.Body
//	for {
//		chunk, err := readChunk(resp.Body)
//		if err != nil {
//			if err != io.EOF {
//				log.Printf("Error reading chunk: %#v", err)
//			}
//			if len(chunk) != 0 {
//				ch <- chunk
//			}
//			log.Println("Closing channel")
//			close(ch)
//			log.Println("Exiting goroutine")
//			return
//		}
//		if len(chunk) != 0 {
//			time.Sleep(1 * time.Millisecond)
//		} else {
//			ch <- chunk
//		}
//	}
//}

// processChunk processes a chunk received from Kubernetes. A chunk
// is a new update.
//func processChunk(chunk []byte) error {
//	m := make(map[string]interface{})
//	err := json.Unmarshal(chunk, &m)
//	if err != nil {
//		return err
//	}
//	method := m["method"]
//	if method == nil {
//		return errors.New("Expected 'method' field")
//	}
//	methodStr = strings.ToLower(method.(string))
//	networkPolicyIfc := m["policy_definition"]
//	if networkPolicyIfc == nil {
//		return errors.New("Expected 'policy_definition' field")
//	}
//	kubeNetworkPolicy := networkPolicyIfc.(map[string]interface{})
//	romanaNetworkPolicy, err := translateNetworkPolicy(kubeNetworkPolicy)
//	if err != nil {
//		return err
//	}
//
//	if methodStr == "added" {
//		applyNetworkPolicy(networkPolicyActionAdd, romanaNetworkPolicy)
//	} else if methodStr == "deleted" {
//		applyNetworkPolicy(networkPolicyActioDelete, romanaNetworkPolicy)
//	} else if methodStr == "modified" {
//		applyNetworkPolicy(networkPolicyActioModify, romanaNetworkPolicy)
//	} else {
//		return common.NewError("Unexpected method '%s'", methodStr)
//	}
//}

// Run implements the main loop, reconnecting as needed.
//func RunListener0(rootURL string, kubeURL string) {
//	l := kubeListener{kubeURL: kubeURL, restClient: common.NewRestClient(common.GetDefaultRestClientConfig(rootURL))}
//	l.Run()
//}
//
//func (l kubeListener) Run0() {
//	for {
//		ch := make(chan []byte)
//		go listen(ch, l.kubernetesURL)
//		for {
//			chunk, ok := <-ch
//			log.Println(ok)
//			if chunk != "" {
//				log.Printf("Read chunk %#v\n%s\n------------", ok, string(chunk))
//				err = processChunk(chunk)
//				if err != nil {
//					// TODO is there any other way to handle this?
//					log.Printf("Error processing chunk %s: %#v", string(chunk), err)
//				}
//			}
//			if !ok {
//				break
//			}
//		}
//	}
//}

/////////////////////////////////////////////////////////////

type networkPolicyAction int

const (
	networkPolicyActionDelete networkPolicyAction = iota
	networkPolicyActionAdd
	networkPolicyActionModify
)

// kubeListener is a Service that listens to updates
// from Kubernetes by connecting to the endpoints specified
// and consuming chunked JSON documents. The endpoints are
// constructed from kubeUrl and the following paths:
// 1. namespaceNotificationPath for namespace additions/deletions
// 2. policyNotificationPathPrefix + <namespace name> + policyNotificationPathPostfix
//    for policy additions/deletions.
type kubeListener struct {
	config                    common.ServiceConfig
	restClient                *common.RestClient
	kubeUrl                   string
	namespaceNotificationPath string
	policyNotificationPathPrefix string
	policyNotificationPathPostfix string
	segmentLabelName          string
}

// Routes returns various routes used in the service.
func (l *kubeListener) Routes() common.Routes {
	routes := common.Routes{}
	return routes
}

// Name implements method of Service interface.
func (l *kubeListener) Name() string {
	return "kubernetesListener"
}

// SetConfig implements SetConfig function of the Service interface.
func (l *kubeListener) SetConfig(config common.ServiceConfig) error {
	m := config.ServiceSpecific
	if m["kubernetes_url"] == "" {
		return errors.New("kubernetes_url required.")
	}
	l.kubeUrl = m["kubernetes_url"].(string)

	if m["namespace_notification_path"] == "" {
		return errors.New("namespace_notification_path required.")
	}
	l.namespaceNotificationPath = m["namespace_notification_path"].(string)

	if m["policy_notification_path_prefix"] == "" {
		return errors.New("policy_notification_path_prefix required.")
	}
	l.policyNotificationPathPrefix = m["policy_notification_path_prefix"].(string)

	if m["policy_notification_path_postfix"] == "" {
		return errors.New("policy_notification_path_postfix required.")
	}
	l.policyNotificationPathPostfix = m["policy_notification_path_postfix"].(string)

	if m["segment_label_name"] == "" {
		return errors.New("segment_label_name required.")
	}
	l.segmentLabelName = m["segment_label_name"].(string)

	return nil
}

// Run configures and runs listener service.
func Run(rootServiceURL string, cred *common.Credential) (*common.RestServiceInfo, error) {
	clientConfig := common.GetDefaultRestClientConfig(rootServiceURL)
	clientConfig.Credential = cred
	client, err := common.NewRestClient(clientConfig)
	if err != nil {
		return nil, err
	}
	kubeListener := &kubeListener{}
	kubeListener.restClient = client
	config, err := client.GetServiceConfig(kubeListener.Name())
	if err != nil {
		return nil, err
	}
	return common.InitializeService(kubeListener, *config)
}

// getOrAddSegment finds a segment (based on segment selector)
//
func (l *kubeListener) getOrAddSegment(tenantServiceURL string, namespace string, kubeSegmentID string) (*tenant.Segment, error) {
	segment := &tenant.Segment{}
	segmentsURL := fmt.Sprintf("%s/tenants/%s/segments", tenantServiceURL, namespace)
	err := l.restClient.Get(fmt.Sprintf("%s/%s", segmentsURL, kubeSegmentID), segment)
	if err == nil {
		return segment, nil
	}
	switch err := err.(type) {
	case common.HttpError:
		if err.StatusCode == http.StatusNotFound {
			// Not found, so let's create a segment.
			segreq := tenant.Segment{Name: kubeSegmentID, ExternalID: kubeSegmentID}
			err2 := l.restClient.Post(segmentsURL, segreq, segment)
			if err2 == nil {
				// Successful creation.
				return segment, nil
			}
			// Creation of non-existing segment gave an error.
			switch err2 := err2.(type) {
			case common.HttpError:
				// Maybe someone else just created a segment between the original
				// lookup and now?
				if err2.StatusCode == http.StatusConflict {
					switch details := err2.Details.(type) {
					case tenant.Segment:
						// We expect the existing segment to be returned in the details field.
						return &details, nil
					default:
						// This is unexpected...
						return nil, err
					}
				}
				// Any other HTTP error other than a Conflict here - return it.
				return nil, err2
			default:
				// Any other error - return it
				return nil, err2
			}
		}
		// Any other HTTP error other than a Not found here - return it
		return nil, err
	default:
		// Any other error - return it
		return nil, err
	}
}

// translateNetworkPolicy translates a Kubernetes policy into
// Romana policy (see common.Policy) with the following rules:
// 1. Kubernetes Namespace corresponds to Romana Tenant
// 2. If Romana Tenant does not exist it is an error (a tenant should
//    automatically have been created when the namespace was added)
func (l *kubeListener) translateNetworkPolicy(kubePolicy *KubeObject) (common.Policy, error) {
	log.Printf("translateNetworkPolicy(): Received %#v", kubePolicy)
	romanaPolicy := &common.Policy{Direction: common.PolicyDirectionIngress}
	ns := kubePolicy.Metadata.Namespace
	tenantURL, err := l.restClient.GetServiceUrl("tenant")
	if err != nil {
		return *romanaPolicy, err
	}
	t := &tenant.Tenant{}
	err = l.restClient.Get(fmt.Sprintf("%s/tenants/%s", tenantURL, ns), t)
	log.Printf("translateNetworkPolicy(): For namespace %s got %#v / %#v", ns, t, err)
	if err != nil {
		return *romanaPolicy, err
	}
	tenantId := t.ID
	tenantExternalId := t.ExternalID

	kubeSegmentID := kubePolicy.Spec.PodSelector[l.segmentLabelName]
	if kubeSegmentID == "" {
		return *romanaPolicy, common.NewError("Expected segment to be specified in podSelector part as '%s'", l.segmentLabelName)
	}

	segment, err := l.getOrAddSegment(tenantURL, ns, kubeSegmentID)
	if err != nil {
		return *romanaPolicy, err
	}
	segmentId := segment.ID
	appliedTo := common.Endpoint{TenantID: tenantId, SegmentID: segmentId}
	romanaPolicy.AppliedTo = make([]common.Endpoint, 1)
	romanaPolicy.AppliedTo[0] = appliedTo
	romanaPolicy.Peers = []common.Endpoint{}
	from := kubePolicy.Spec.AllowIncoming.From
	// This is subject to change once the network specification in Kubernetes is finalized.
	// Right now it is a work in progress.
	if from != nil {
		for _, entry := range from {
			pods := entry.Pods
			fromKubeSegmentID := pods[l.segmentLabelName]
			if fromKubeSegmentID == "" {
				return *romanaPolicy, common.NewError("Expected segment to be specified in podSelector part as '%s'", l.segmentLabelName)
			}
			fromSegment, err := l.getOrAddSegment(tenantURL, ns, fromKubeSegmentID)
			if err != nil {
				return *romanaPolicy, err
			}
			peer := common.Endpoint{TenantID: tenantId, TenantExternalID: tenantExternalId, SegmentID: fromSegment.ID, SegmentExternalID: fromSegment.ExternalID}
			romanaPolicy.Peers = append(romanaPolicy.Peers, peer)
		}
		toPorts := kubePolicy.Spec.AllowIncoming.ToPorts
		romanaPolicy.Rules = common.Rules{}
		for _, toPort := range toPorts {
			proto := strings.ToLower(toPort.Protocol)
			ports := []uint{toPort.Port}
			rule := common.Rule{Protocol: proto, Ports: ports}
			romanaPolicy.Rules = append(romanaPolicy.Rules, rule)
		}
	}
	err = romanaPolicy.Validate()
	if err != nil {
		return *romanaPolicy, err
	}
	return *romanaPolicy, nil
}

func (l *kubeListener) applyNetworkPolicy(action networkPolicyAction, romanaNetworkPolicy common.Policy) error {
	policyUrl, err := l.restClient.GetServiceUrl("policy")
	if err != nil {
		return err
	}
	policyUrl = fmt.Sprintf("%s/policies", policyUrl)
	switch action {
	case networkPolicyActionAdd:
		err := l.restClient.Post(policyUrl, romanaNetworkPolicy, &romanaNetworkPolicy)
		if err != nil {
			return err
		}
		log.Printf("Applied policy %#v", romanaNetworkPolicy)
	case networkPolicyActionDelete:
		policyUrl = fmt.Sprintf("%s/%d", policyUrl, romanaNetworkPolicy.ID)
		err := l.restClient.Delete(policyUrl, nil, &romanaNetworkPolicy)
		if err != nil {
			return err
		}
	default:
		return errors.New("Unsupported operation")
	}
	return nil
}

func (l *kubeListener) Initialize() error {
	log.Printf("%s: Starting server", l.Name())
	nsUrl, err := common.CleanURL(fmt.Sprintf("%s%s", l.kubeUrl, l.namespaceNotificationPath))
	if err != nil {
		return err
	}
	log.Printf("Starting to listen on %s", nsUrl)
	done := make(chan Done)
	nsEvents, err := l.nsWatch(done, nsUrl)
	if err != nil {
		log.Fatal("Namespace watcher failed to start", err)
	}

	events := l.conductor(nsEvents, done)
	l.process(events, done)
	log.Println("All routines started")
	return nil
}

func CreateSchema(rootServiceURL string, overwrite bool) error {
	return nil
}
<|endoftext|>


<commit_before>// Code generated by go generate. DO NOT EDIT.
// Source: vendor/istio.io/api/mixer/v1/global_dictionary.yaml

package attribute

func GlobalList() ([]string) {
    tmp := make([]string, len(globalList))
    copy(tmp, globalList)
    return tmp
}

var (
    globalList = []string{
		"source.ip",
		"source.port",
		"source.name",
		"source.uid",
		"source.namespace",
		"source.labels",
		"source.user",
		"target.ip",
		"target.port",
		"target.service",
		"target.name",
		"target.uid",
		"target.namespace",
		"target.labels",
		"target.user",
		"request.headers",
		"request.id",
		"request.path",
		"request.host",
		"request.method",
		"request.reason",
		"request.referer",
		"request.scheme",
		"request.size",
		"request.time",
		"request.useragent",
		"response.headers",
		"response.size",
		"response.time",
		"response.duration",
		"response.code",
		":authority",
		":method",
		":path",
		":scheme",
		":status",
		"access-control-allow-origin",
		"access-control-allow-methods",
		"access-control-allow-headers",
		"access-control-max-age",
		"access-control-request-method",
		"access-control-request-headers",
		"accept-charset",
		"accept-encoding",
		"accept-language",
		"accept-ranges",
		"accept",
		"access-control-allow",
		"age",
		"allow",
		"authorization",
		"cache-control",
		"content-disposition",
		"content-encoding",
		"content-language",
		"content-length",
		"content-location",
		"content-range",
		"content-type",
		"cookie",
		"date",
		"etag",
		"expect",
		"expires",
		"from",
		"host",
		"if-match",
		"if-modified-since",
		"if-none-match",
		"if-range",
		"if-unmodified-since",
		"keep-alive",
		"last-modified",
		"link",
		"location",
		"max-forwards",
		"proxy-authenticate",
		"proxy-authorization",
		"range",
		"referer",
		"refresh",
		"retry-after",
		"server",
		"set-cookie",
		"strict-transport-sec",
		"transfer-encoding",
		"user-agent",
		"vary",
		"via",
		"www-authenticate",
		"GET",
		"POST",
		"http",
		"envoy",
		"'200'",
		"Keep-Alive",
		"chunked",
		"x-envoy-service-time",
		"x-forwarded-for",
		"x-forwarded-host",
		"x-forwarded-proto",
		"x-http-method-override",
		"x-request-id",
		"x-requested-with",
		"application/json",
		"application/xml",
		"gzip",
		"text/html",
		"text/html; charset=utf-8",
		"text/plain",
		"text/plain; charset=utf-8",
		"'0'",
		"'1'",
		"true",
		"false",
		"gzip, deflate",
		"max-age=0",
		"x-envoy-upstream-service-time",
		"x-envoy-internal",
		"x-envoy-expected-rq-timeout-ms",
		"x-ot-span-context",
		"x-b3-traceid",
		"x-b3-sampled",
		"x-b3-spanid",
		"tcp",
		"connection.id",
		"connection.received.bytes",
		"connection.received.bytes_total",
		"connection.sent.bytes",
		"connection.sent.bytes_total",
		"connection.duration",
		"context.protocol",
		"context.timestamp",
		"context.time",
		"0",
		"1",
		"200",
		"302",
		"400",
		"401",
		"403",
		"404",
		"409",
		"429",
		"499",
		"500",
		"501",
		"502",
		"503",
		"504",
		"destination.ip",
		"destination.port",
		"destination.service",
		"destination.name",
		"destination.uid",
		"destination.namespace",
		"destination.labels",
		"destination.user",
		"source.service",
		"api.service",
		"api.version",
		"api.operation",
		"api.protocol",
		"request.auth.principal",
		"request.auth.audiences",
		"request.auth.presenter",
		"request.api_key",
		"check.error_code",
		"check.error_message",
    }
)
<commit_msg>Generate docs when compiling protos (#3136)<commit_after>// Code generated by go generate. DO NOT EDIT.
// Source: vendor/istio.io/api/mixer/v1/global_dictionary.yaml

package attribute

func GlobalList() ([]string) {
    tmp := make([]string, len(globalList))
    copy(tmp, globalList)
    return tmp
}

var (
    globalList = []string{
		"source.ip",
		"source.port",
		"source.name",
		"source.uid",
		"source.namespace",
		"source.labels",
		"source.user",
		"target.ip",
		"target.port",
		"target.service",
		"target.name",
		"target.uid",
		"target.namespace",
		"target.labels",
		"target.user",
		"request.headers",
		"request.id",
		"request.path",
		"request.host",
		"request.method",
		"request.reason",
		"request.referer",
		"request.scheme",
		"request.size",
		"request.time",
		"request.useragent",
		"response.headers",
		"response.size",
		"response.time",
		"response.duration",
		"response.code",
		":authority",
		":method",
		":path",
		":scheme",
		":status",
		"access-control-allow-origin",
		"access-control-allow-methods",
		"access-control-allow-headers",
		"access-control-max-age",
		"access-control-request-method",
		"access-control-request-headers",
		"accept-charset",
		"accept-encoding",
		"accept-language",
		"accept-ranges",
		"accept",
		"access-control-allow",
		"age",
		"allow",
		"authorization",
		"cache-control",
		"content-disposition",
		"content-encoding",
		"content-language",
		"content-length",
		"content-location",
		"content-range",
		"content-type",
		"cookie",
		"date",
		"etag",
		"expect",
		"expires",
		"from",
		"host",
		"if-match",
		"if-modified-since",
		"if-none-match",
		"if-range",
		"if-unmodified-since",
		"keep-alive",
		"last-modified",
		"link",
		"location",
		"max-forwards",
		"proxy-authenticate",
		"proxy-authorization",
		"range",
		"referer",
		"refresh",
		"retry-after",
		"server",
		"set-cookie",
		"strict-transport-sec",
		"transfer-encoding",
		"user-agent",
		"vary",
		"via",
		"www-authenticate",
		"GET",
		"POST",
		"http",
		"envoy",
		"'200'",
		"Keep-Alive",
		"chunked",
		"x-envoy-service-time",
		"x-forwarded-for",
		"x-forwarded-host",
		"x-forwarded-proto",
		"x-http-method-override",
		"x-request-id",
		"x-requested-with",
		"application/json",
		"application/xml",
		"gzip",
		"text/html",
		"text/html; charset=utf-8",
		"text/plain",
		"text/plain; charset=utf-8",
		"'0'",
		"'1'",
		"true",
		"false",
		"gzip, deflate",
		"max-age=0",
		"x-envoy-upstream-service-time",
		"x-envoy-internal",
		"x-envoy-expected-rq-timeout-ms",
		"x-ot-span-context",
		"x-b3-traceid",
		"x-b3-sampled",
		"x-b3-spanid",
		"tcp",
		"connection.id",
		"connection.received.bytes",
		"connection.received.bytes_total",
		"connection.sent.bytes",
		"connection.sent.bytes_total",
		"connection.duration",
		"context.protocol",
		"context.timestamp",
		"context.time",
		"0",
		"1",
		"200",
		"302",
		"400",
		"401",
		"403",
		"404",
		"409",
		"429",
		"499",
		"500",
		"501",
		"502",
		"503",
		"504",
		"destination.ip",
		"destination.port",
		"destination.service",
		"destination.name",
		"destination.uid",
		"destination.namespace",
		"destination.labels",
		"destination.user",
		"source.service",
		"api.service",
		"api.version",
		"api.operation",
		"api.protocol",
		"request.auth.principal",
		"request.auth.audiences",
		"request.auth.presenter",
		"request.api_key",
		"check.error_code",
		"check.error_message",
    }
)
<|endoftext|>


<commit_before>package main

import (
	"encoding/json"
	"fmt"
	"github.com/nutrun/lentil"
	"io/ioutil"
	"log"
	"net/smtp"
	"os"
	"os/exec"
	"os/signal"
	"strings"
	"syscall"
	"time"
)

type Listener struct {
	q        *lentil.Beanstalkd
	verbose  bool
	jobqueue *JobQueue
	sig      os.Signal
	proc     *os.Process
}

func NewListener(verbose, inclusive bool, filter []string) (*Listener, error) {
	this := new(Listener)
	q, err := lentil.Dial(Config.QueueAddr)

	if err != nil {
		return nil, err
	}

	this.q = q
	this.verbose = verbose
	this.jobqueue = NewJobQueue(this.q, inclusive, filter)
	return this, nil
}

func (this *Listener) execute(msg map[string]string) {
	workdir := msg["workdir"]
	e := os.Chdir(workdir)
	if e != nil {
		this.catch(msg, e)
	}
	messagetokens := strings.Split(msg["cmd"], " ")
	command := messagetokens[0]
	args := messagetokens[1:len(messagetokens)]
	cmd := exec.Command(command, args...)
	f, e := os.Create(msg["out"])
	if e != nil {
		this.catch(msg, e)
	}
	cmd.Stderr = f
	cmd.Stdout = f
	e = cmd.Start()
	if e != nil {
		this.catch(msg, e)
	}
	this.proc = cmd.Process
	e = cmd.Wait()
	if e != nil {
		this.catch(msg, e)
	}
	this.proc = nil
	f.Close()
}

func (this *Listener) run() {
	go this.trap()

listenerloop:
	for {
		if this.sig != nil {
			os.Exit(0)
		}
		Config.Load()
		job, e := this.jobqueue.Next()
		if e != nil {
			if strings.Contains(e.Error(), "TIMED_OUT") {
				time.Sleep(time.Second)
				goto listenerloop
			}
			log.Fatal(e)
		}
		if this.verbose {
			log.Printf("RUNNING: %s", job.Body)
		}
		msg := make(map[string]string)
		json.Unmarshal([]byte(job.Body), &msg)
		this.execute(msg)
		e = this.jobqueue.Delete(job.Id)
		if e != nil {
			this.catch(msg, e)
		}
	}
}

// Log and email errors
func (this *Listener) catch(msg map[string]string, e error) {
	log.Printf("ERROR: %s\n", e.Error())
	this.mail(msg, e)
	this.publishError(msg, e)

}

func (this *Listener) publishError(msg map[string]string, e error) {
	err := this.q.Use(Config.errorQueue)
	if err != nil {
		log.Printf("ERROR: %s\n", err)
		return
	}
	msg["error"] = e.Error()
	msg["log"] = this.readLog(msg)
	payload, err := json.Marshal(msg)
	if err != nil {
		log.Printf("ERROR: %s\n", err)
		return
	}
	_, err = this.q.Put(0, 0, 60*60, payload)
	if err != nil {
		log.Printf("ERROR: %s\n", err)
	}
}

func (this *Listener) readLog(msg map[string]string) string {
	content := make([]byte, 0)
	info, err := os.Stat(msg["out"])
	if err != nil {
		content = []byte(fmt.Sprintf("Could not read job log from [%s]. %s", msg["out"], err.Error()))
	}
	if info.Size() > 104857 {
		content = []byte(fmt.Sprintf("Could send job log [%s]. File too big", msg["out"]))

	} else {
		content, err = ioutil.ReadFile(msg["out"])
		if err != nil {
			content = []byte(fmt.Sprintf("Could not read job log from [%s]", msg["out"]))
		}
	}
	return string(content)

}

func (this *Listener) mail(msg map[string]string, e error) {
	if Config.SmtpServerAddr == "" {
		return
	}
	if len(msg["mailto"]) < 1 { //no email addresses
		return
	}
	to := strings.Split(msg["mailto"], ",")
	subject := fmt.Sprintf("Subject: FAILED: %s\r\n\r\n", msg["cmd"])
	hostname, _ := os.Hostname()
	content := []byte(this.readLog(msg))
	mail := fmt.Sprintf("%s%s", subject, fmt.Sprintf("Ran on [%s]\n%s\n%s\n%s", hostname, subject, e, content))
	e = smtp.SendMail(Config.SmtpServerAddr, nil, Config.MailFrom, to, []byte(mail))
	if e != nil {
		log.Printf("ERROR: %s\n", e)
	}
}

// Wait for currently running job to finish before exiting on SIGTERM and SIGINT
func (this *Listener) trap() {
	receivedSignal := make(chan os.Signal)
	signal.Notify(receivedSignal, syscall.SIGTERM, syscall.SIGINT)
	this.sig = <-receivedSignal
	if this.sig.String() == syscall.SIGTERM.String() {
		log.Printf("Got signal %d. Killing current job.", this.sig)
		if this.proc != nil {
			this.proc.Kill()
		}
		os.Exit(1)
	}
	log.Printf("Got signal %d. Waiting for current job to complete. sig term is [%v]", this.sig, syscall.SIGTERM)
}
<commit_msg>fixed output bugs<commit_after>package main

import (
	"encoding/json"
	"fmt"
	"github.com/nutrun/lentil"
	"io/ioutil"
	"log"
	"net/smtp"
	"os"
	"os/exec"
	"os/signal"
	"strings"
	"syscall"
	"time"
)

type Listener struct {
	q        *lentil.Beanstalkd
	verbose  bool
	jobqueue *JobQueue
	sig      os.Signal
	proc     *os.Process
}

func NewListener(verbose, inclusive bool, filter []string) (*Listener, error) {
	this := new(Listener)
	q, err := lentil.Dial(Config.QueueAddr)

	if err != nil {
		return nil, err
	}

	this.q = q
	this.verbose = verbose
	this.jobqueue = NewJobQueue(this.q, inclusive, filter)
	return this, nil
}

func (this *Listener) execute(msg map[string]string) {
	workdir := msg["workdir"]
	e := os.Chdir(workdir)
	if e != nil {
		this.catch(msg, e)
	}
	messagetokens := strings.Split(msg["cmd"], " ")
	command := messagetokens[0]
	args := messagetokens[1:len(messagetokens)]
	cmd := exec.Command(command, args...)
	f, e := os.Create(msg["out"])
	if e != nil {
		this.catch(msg, e)
		return
	}
	cmd.Stderr = f
	cmd.Stdout = f
	e = cmd.Start()
	if e != nil {
		this.catch(msg, e)
	}
	this.proc = cmd.Process
	e = cmd.Wait()
	if e != nil {
		this.catch(msg, e)
	}
	this.proc = nil
	f.Close()
}

func (this *Listener) run() {
	go this.trap()

listenerloop:
	for {
		if this.sig != nil {
			os.Exit(0)
		}
		Config.Load()
		job, e := this.jobqueue.Next()
		if e != nil {
			if strings.Contains(e.Error(), "TIMED_OUT") {
				time.Sleep(time.Second)
				goto listenerloop
			}
			log.Fatal(e)
		}
		if this.verbose {
			log.Printf("RUNNING: %s", job.Body)
		}
		msg := make(map[string]string)
		json.Unmarshal([]byte(job.Body), &msg)
		this.execute(msg)
		e = this.jobqueue.Delete(job.Id)
		if e != nil {
			this.catch(msg, e)
		}
	}
}

// Log and email errors
func (this *Listener) catch(msg map[string]string, e error) {
	log.Printf("ERROR: %s\n", e.Error())
	this.mail(msg, e)
	this.publishError(msg, e)

}

func (this *Listener) publishError(msg map[string]string, e error) {
	err := this.q.Use(Config.errorQueue)
	if err != nil {
		log.Printf("ERROR: %s\n", err)
		return
	}
	msg["error"] = e.Error()
	msg["log"] = this.readLog(msg)
	payload, err := json.Marshal(msg)
	if err != nil {
		log.Printf("ERROR: %s\n", err)
		return
	}
	_, err = this.q.Put(0, 0, 60*60, payload)
	if err != nil {
		log.Printf("ERROR: %s\n", err)
	}
}

func (this *Listener) readLog(msg map[string]string) string {
	content := make([]byte, 0)
	info, err := os.Stat(msg["out"])
	if err != nil {
		content = []byte(fmt.Sprintf("Could not read job log from [%s]. %s", msg["out"], err.Error()))
		return string(content)
	}
	if info.Size() > 104857 {
		content = []byte(fmt.Sprintf("Could send job log [%s]. File too big", msg["out"]))
	} else {
		content, err = ioutil.ReadFile(msg["out"])
		if err != nil {
			content = []byte(fmt.Sprintf("Could not read job log from [%s]", msg["out"]))
		}
	}
	return string(content)

}

func (this *Listener) mail(msg map[string]string, e error) {
	if Config.SmtpServerAddr == "" {
		return
	}
	if len(msg["mailto"]) < 1 { //no email addresses
		return
	}
	to := strings.Split(msg["mailto"], ",")
	subject := fmt.Sprintf("Subject: FAILED: %s\r\n\r\n", msg["cmd"])
	hostname, _ := os.Hostname()
	content := []byte(this.readLog(msg))
	mail := fmt.Sprintf("%s%s", subject, fmt.Sprintf("Ran on [%s]\n%s\n%s\n%s", hostname, subject, e, content))
	e = smtp.SendMail(Config.SmtpServerAddr, nil, Config.MailFrom, to, []byte(mail))
	if e != nil {
		log.Printf("ERROR: %s\n", e)
	}
}

// Wait for currently running job to finish before exiting on SIGTERM and SIGINT
func (this *Listener) trap() {
	receivedSignal := make(chan os.Signal)
	signal.Notify(receivedSignal, syscall.SIGTERM, syscall.SIGINT)
	this.sig = <-receivedSignal
	if this.sig.String() == syscall.SIGTERM.String() {
		log.Printf("Got signal %d. Killing current job.", this.sig)
		if this.proc != nil {
			this.proc.Kill()
		}
		os.Exit(1)
	}
	log.Printf("Got signal %d. Waiting for current job to complete. sig term is [%v]", this.sig, syscall.SIGTERM)
}
<|endoftext|>


<commit_before>package lmdb

/*
#include <stdlib.h>
#include <stdio.h>
#include "lmdb.h"
*/
import "C"

import "unsafe"

const valMaxSize = 1<<32 - 1

// Multi is a wrapper for a contiguous page of sorted, fixed-length values
// passed to Cursor.PutMulti or retrieved using Cursor.Get with the
// GetMultiple/NextMultiple flag.
//
// Multi values are only useful in databases opened with DupSort|DupFixed.
type Multi struct {
	page   []byte
	stride int
}

// WrapMulti converts a page of contiguous values with stride size into a
// Multi.  WrapMulti panics if len(page) is not a multiple of stride.
//
//		_, val, _ := cursor.Get(nil, nil, lmdb.FirstDup)
//		_, page, _ := cursor.Get(nil, nil, lmdb.GetMultiple)
//		multi := lmdb.WrapMulti(page, len(val))
//
// See mdb_cursor_get and MDB_GET_MULTIPLE.
func WrapMulti(page []byte, stride int) *Multi {
	if len(page)%stride != 0 {
		panic("incongruent arguments")
	}
	return &Multi{page: page, stride: stride}
}

// Vals returns a slice containing the values in m.  The returned slice has
// length m.Len() and each item has length m.Stride().
func (m *Multi) Vals() [][]byte {
	n := m.Len()
	ps := make([][]byte, n)
	for i := 0; i < n; i++ {
		ps[i] = m.Val(i)
	}
	return ps
}

// Val returns the value at index i.  Val panics if i is out of range.
func (m *Multi) Val(i int) []byte {
	off := i * m.stride
	return m.page[off : off+m.stride]
}

// Len returns the number of values in the Multi.
func (m *Multi) Len() int {
	return len(m.page) / m.stride
}

// Stride returns the length of an individual value in the m.
func (m *Multi) Stride() int {
	return m.stride
}

// Size returns the total size of the Multi data and is equal to
//
//		m.Len()*m.Stride()
//
func (m *Multi) Size() int {
	return len(m.page)
}

// Page returns the Multi page data as a raw slice of bytes with length
// m.Size().
func (m *Multi) Page() []byte {
	return m.page[:len(m.page):len(m.page)]
}

var eb = []byte{0}

func valBytes(b []byte) ([]byte, int) {
	if len(b) == 0 {
		return eb, 0
	}
	return b, len(b)
}

func wrapVal(b []byte) *C.MDB_val {
	p, n := valBytes(b)
	return &C.MDB_val{
		mv_data: unsafe.Pointer(&p[0]),
		mv_size: C.size_t(n),
	}
}

func getBytes(val *C.MDB_val) []byte {
	return (*[valMaxSize]byte)(unsafe.Pointer(val.mv_data))[:val.mv_size:val.mv_size]
}

func getBytesCopy(val *C.MDB_val) []byte {
	return C.GoBytes(val.mv_data, C.int(val.mv_size))
}
<commit_msg>lmdb: comment about the exact value of valMaxSize<commit_after>package lmdb

/*
#include <stdlib.h>
#include <stdio.h>
#include "lmdb.h"
*/
import "C"

import "unsafe"

// valMaxSize is the largest data sized allowed by lmdb-go.  Luckily, it
// coincides with the maximum data value for LMDB (MAXDATASIZE).
const valMaxSize = 1<<32 - 1

// Multi is a wrapper for a contiguous page of sorted, fixed-length values
// passed to Cursor.PutMulti or retrieved using Cursor.Get with the
// GetMultiple/NextMultiple flag.
//
// Multi values are only useful in databases opened with DupSort|DupFixed.
type Multi struct {
	page   []byte
	stride int
}

// WrapMulti converts a page of contiguous values with stride size into a
// Multi.  WrapMulti panics if len(page) is not a multiple of stride.
//
//		_, val, _ := cursor.Get(nil, nil, lmdb.FirstDup)
//		_, page, _ := cursor.Get(nil, nil, lmdb.GetMultiple)
//		multi := lmdb.WrapMulti(page, len(val))
//
// See mdb_cursor_get and MDB_GET_MULTIPLE.
func WrapMulti(page []byte, stride int) *Multi {
	if len(page)%stride != 0 {
		panic("incongruent arguments")
	}
	return &Multi{page: page, stride: stride}
}

// Vals returns a slice containing the values in m.  The returned slice has
// length m.Len() and each item has length m.Stride().
func (m *Multi) Vals() [][]byte {
	n := m.Len()
	ps := make([][]byte, n)
	for i := 0; i < n; i++ {
		ps[i] = m.Val(i)
	}
	return ps
}

// Val returns the value at index i.  Val panics if i is out of range.
func (m *Multi) Val(i int) []byte {
	off := i * m.stride
	return m.page[off : off+m.stride]
}

// Len returns the number of values in the Multi.
func (m *Multi) Len() int {
	return len(m.page) / m.stride
}

// Stride returns the length of an individual value in the m.
func (m *Multi) Stride() int {
	return m.stride
}

// Size returns the total size of the Multi data and is equal to
//
//		m.Len()*m.Stride()
//
func (m *Multi) Size() int {
	return len(m.page)
}

// Page returns the Multi page data as a raw slice of bytes with length
// m.Size().
func (m *Multi) Page() []byte {
	return m.page[:len(m.page):len(m.page)]
}

var eb = []byte{0}

func valBytes(b []byte) ([]byte, int) {
	if len(b) == 0 {
		return eb, 0
	}
	return b, len(b)
}

func wrapVal(b []byte) *C.MDB_val {
	p, n := valBytes(b)
	return &C.MDB_val{
		mv_data: unsafe.Pointer(&p[0]),
		mv_size: C.size_t(n),
	}
}

func getBytes(val *C.MDB_val) []byte {
	return (*[valMaxSize]byte)(unsafe.Pointer(val.mv_data))[:val.mv_size:val.mv_size]
}

func getBytesCopy(val *C.MDB_val) []byte {
	return C.GoBytes(val.mv_data, C.int(val.mv_size))
}
<|endoftext|>


<commit_before>// Copyright 2017 The Gitea Authors. All rights reserved.
// Use of this source code is governed by a MIT-style
// license that can be found in the LICENSE file.

package log

import (
	"fmt"

	"github.com/go-xorm/core"
)

// XORMLogBridge a logger bridge from Logger to xorm
type XORMLogBridge struct {
	loggers []*Logger
	showSQL bool
	level   core.LogLevel
}

var (
	// XORMLogger the logger for xorm
	XORMLogger *XORMLogBridge
)

// DiscardXORMLogger inits a blank logger for xorm
func DiscardXORMLogger() {
	XORMLogger = &XORMLogBridge{
		showSQL: false,
	}
}

// NewXORMLogger generate logger for xorm FIXME: configable
func NewXORMLogger(bufferlen int64, mode, config string) {
	logger := newLogger(bufferlen)
	logger.SetLogger(mode, config)
	if XORMLogger == nil {
		XORMLogger = &XORMLogBridge{
			showSQL: true,
		}
	}
	XORMLogger.loggers = append(XORMLogger.loggers, logger)
}

func (l *XORMLogBridge) writerMsg(skip, level int, msg string) error {
	for _, logger := range l.loggers {
		if err := logger.writerMsg(skip, level, msg); err != nil {
			return err
		}
	}
	return nil
}

// Debug show debug log
func (l *XORMLogBridge) Debug(v ...interface{}) {
	if l.level <= core.LOG_DEBUG {
		msg := fmt.Sprint(v...)
		l.writerMsg(0, DEBUG, "[D]"+msg)
	}
}

// Debugf show debug log
func (l *XORMLogBridge) Debugf(format string, v ...interface{}) {
	if l.level <= core.LOG_DEBUG {
		for _, logger := range l.loggers {
			logger.Debug(format, v...)
		}
	}
}

// Error show error log
func (l *XORMLogBridge) Error(v ...interface{}) {
	if l.level <= core.LOG_ERR {
		msg := fmt.Sprint(v...)
		l.writerMsg(0, ERROR, "[E]"+msg)
	}
}

// Errorf show error log
func (l *XORMLogBridge) Errorf(format string, v ...interface{}) {
	if l.level <= core.LOG_ERR {
		for _, logger := range l.loggers {
			logger.Error(0, format, v...)
		}
	}
}

// Info show information level log
func (l *XORMLogBridge) Info(v ...interface{}) {
	if l.level <= core.LOG_INFO {
		msg := fmt.Sprint(v...)
		l.writerMsg(0, INFO, "[I]"+msg)
	}
}

// Infof show information level log
func (l *XORMLogBridge) Infof(format string, v ...interface{}) {
	if l.level <= core.LOG_INFO {
		for _, logger := range l.loggers {
			logger.Info(format, v...)
		}
	}
}

// Warn show warning log
func (l *XORMLogBridge) Warn(v ...interface{}) {
	if l.level <= core.LOG_WARNING {
		msg := fmt.Sprint(v...)
		l.writerMsg(0, WARN, "[W] "+msg)
	}
}

// Warnf show warnning log
func (l *XORMLogBridge) Warnf(format string, v ...interface{}) {
	if l.level <= core.LOG_WARNING {
		for _, logger := range l.loggers {
			logger.Warn(format, v...)
		}
	}
}

// Level get logger level
func (l *XORMLogBridge) Level() core.LogLevel {
	return l.level
}

// SetLevel set logger level
func (l *XORMLogBridge) SetLevel(level core.LogLevel) {
	l.level = level
}

// ShowSQL set if record SQL
func (l *XORMLogBridge) ShowSQL(show ...bool) {
	if len(show) > 0 {
		l.showSQL = show[0]
	} else {
		l.showSQL = true
	}
}

// IsShowSQL if record SQL
func (l *XORMLogBridge) IsShowSQL() bool {
	return l.showSQL
}
<commit_msg>Delete xorm.go<commit_after><|endoftext|>


<commit_before>package margelet

import (
	"github.com/zhulik/telegram-bot-api"
)

type Margelet struct {
	bot        *tgbotapi.BotAPI
	responders []Responder
}

func NewMargelet(token string) (*Margelet, error) {
	bot, err := tgbotapi.NewBotAPI(token)
	if err != nil {
		return nil, err
	}

	bot.Debug = false

	var ucfg tgbotapi.UpdateConfig = tgbotapi.NewUpdate(0)
	ucfg.Timeout = 60
	err = bot.UpdatesChan(ucfg)

	if err != nil {
		return nil, err
	}

	return &Margelet{bot, []Responder{}}, nil
}

func (this *Margelet) addResponder(responder Responder) {
	this.responders = append(this.responders, responder)
}

func (this *Margelet) Send(c tgbotapi.Chattable) {
	this.bot.Send(c)
}

func (this *Margelet) Run() {
	for {
		select {
		case update := <-this.bot.Updates:
			for _, responder := range this.responders {
				msg, err := responder.Response(update.Message)

				if err != nil {
					msg = tgbotapi.NewMessage(update.Message.Chat.ID, "Error occured: "+err.Error())
				}

				this.Send(msg)
			}
		}
	}
}
<commit_msg>Typo fix<commit_after>package margelet

import (
	"github.com/zhulik/telegram-bot-api"
)

type Margelet struct {
	bot        *tgbotapi.BotAPI
	responders []Responder
}

func NewMargelet(token string) (*Margelet, error) {
	bot, err := tgbotapi.NewBotAPI(token)
	if err != nil {
		return nil, err
	}

	bot.Debug = false

	var ucfg tgbotapi.UpdateConfig = tgbotapi.NewUpdate(0)
	ucfg.Timeout = 60
	err = bot.UpdatesChan(ucfg)

	if err != nil {
		return nil, err
	}

	return &Margelet{bot, []Responder{}}, nil
}

func (this *Margelet) AddResponder(responder Responder) {
	this.responders = append(this.responders, responder)
}

func (this *Margelet) Send(c tgbotapi.Chattable) {
	this.bot.Send(c)
}

func (this *Margelet) Run() {
	for {
		select {
		case update := <-this.bot.Updates:
			for _, responder := range this.responders {
				msg, err := responder.Response(update.Message)

				if err != nil {
					msg = tgbotapi.NewMessage(update.Message.Chat.ID, "Error occured: "+err.Error())
				}

				this.Send(msg)
			}
		}
	}
}
<|endoftext|>