<commit_before>package torznab

import (
	"errors"
	"fmt"
	"log"
	"net/url"
	"strconv"
	"strings"
)

// Query represents a torznab query
type Query struct {
	Type          string
	Q, Ep, Season string
	Limit, Offset int
	Extended      bool
	Categories    []int
	APIKey        string
}

// Episode returns either the season + episode in the format S00E00 or just the season as S00 if
// no episode has been specified.
func (query Query) Episode() (s string) {
	if query.Season != "" {
		s += fmt.Sprintf("S%s", padLeft(query.Season, "0", 2))
	}
	if query.Ep != "" {
		s += fmt.Sprintf("E%s", padLeft(query.Ep, "0", 2))
	}
	return s
}

// Keywords returns the query formatted as search keywords
func (query Query) Keywords() string {
	tokens := []string{}

	if query.Q != "" {
		tokens = append(tokens, query.Q)
	}

	if query.Season != "" || query.Ep != "" {
		tokens = append(tokens, query.Episode())
	}

	return strings.Join(tokens, " ")
}

// Encode returns the query as a url query string
func (query Query) Encode() string {
	v := url.Values{}

	if query.Type != "" {
		v.Set("t", query.Type)
	} else {
		v.Set("t", "search")
	}

	if query.Q != "" {
		v.Set("q", query.Q)
	}

	if query.Ep != "" {
		v.Set("ep", query.Ep)
	}

	if query.Season != "" {
		v.Set("season", query.Season)
	}

	if query.Offset != 0 {
		v.Set("offset", strconv.Itoa(query.Offset))
	}

	if query.Limit != 0 {
		v.Set("limit", strconv.Itoa(query.Limit))
	}

	if query.Extended {
		v.Set("extended", "1")
	}

	if query.APIKey != "" {
		v.Set("apikey", query.APIKey)
	}

	if len(query.Categories) > 0 {
		cats := []string{}

		for _, cat := range query.Categories {
			cats = append(cats, strconv.Itoa(cat))
		}

		v.Set("cat", strings.Join(cats, ","))
	}

	return v.Encode()
}

func (query Query) String() string {
	return query.Encode()
}

// ParseQuery takes the query string parameters for a torznab query and parses them
func ParseQuery(v url.Values) (Query, error) {
	query := Query{}

	for k, vals := range v {
		switch k {
		case "t":
			if len(vals) > 1 {
				return query, errors.New("Multiple t parameters not allowed")
			}
			query.Type = vals[0]

		case "q":
			query.Q = strings.Join(vals, " ")

		case "ep":
			if len(vals) > 1 {
				return query, errors.New("Multiple ep parameters not allowed")
			}
			query.Ep = vals[0]

		case "season":
			if len(vals) > 1 {
				return query, errors.New("Multiple season parameters not allowed")
			}
			query.Season = vals[0]

		case "apikey":
			if len(vals) > 1 {
				return query, errors.New("Multiple apikey parameters not allowed")
			}
			query.APIKey = vals[0]

		case "limit":
			if len(vals) > 1 {
				return query, errors.New("Multiple limit parameters not allowed")
			}
			limit, err := strconv.Atoi(vals[0])
			if err != nil {
				return query, err
			}
			query.Limit = limit

		case "offset":
			if len(vals) > 1 {
				return query, errors.New("Multiple offset parameters not allowed")
			}
			offset, err := strconv.Atoi(vals[0])
			if err != nil {
				return query, err
			}
			query.Offset = offset

		case "extended":
			if len(vals) > 1 {
				return query, errors.New("Multiple extended parameters not allowed")
			}
			extended, err := strconv.ParseBool(vals[0])
			if err != nil {
				return query, err
			}
			query.Extended = extended

		case "cat":
			query.Categories = []int{}
			for _, val := range vals {
				ints, err := splitInts(val, ",")
				if err != nil {
					return Query{}, fmt.Errorf("Unable to parse cats %q", vals[0])
				}
				query.Categories = append(query.Categories, ints...)
			}

		default:
			log.Printf("Unknown torznab request key %q", k)
		}
	}

	return query, nil
}

func splitInts(s, delim string) (i []int, err error) {
	for _, v := range strings.Split(s, delim) {
		vInt, err := strconv.Atoi(v)
		if err != nil {
			return i, err
		}
		i = append(i, vInt)
	}
	return i, err
}

func padLeft(str, pad string, length int) string {
	for len(str) < length {
		str = pad + str
	}
	return str
}
<commit_msg>Fix logging in torznab package<commit_after>package torznab

import (
	"errors"
	"fmt"
	"net/url"
	"strconv"
	"strings"

	"github.com/cardigann/cardigann/logger"
)

// Query represents a torznab query
type Query struct {
	Type          string
	Q, Ep, Season string
	Limit, Offset int
	Extended      bool
	Categories    []int
	APIKey        string
}

// Episode returns either the season + episode in the format S00E00 or just the season as S00 if
// no episode has been specified.
func (query Query) Episode() (s string) {
	if query.Season != "" {
		s += fmt.Sprintf("S%s", padLeft(query.Season, "0", 2))
	}
	if query.Ep != "" {
		s += fmt.Sprintf("E%s", padLeft(query.Ep, "0", 2))
	}
	return s
}

// Keywords returns the query formatted as search keywords
func (query Query) Keywords() string {
	tokens := []string{}

	if query.Q != "" {
		tokens = append(tokens, query.Q)
	}

	if query.Season != "" || query.Ep != "" {
		tokens = append(tokens, query.Episode())
	}

	return strings.Join(tokens, " ")
}

// Encode returns the query as a url query string
func (query Query) Encode() string {
	v := url.Values{}

	if query.Type != "" {
		v.Set("t", query.Type)
	} else {
		v.Set("t", "search")
	}

	if query.Q != "" {
		v.Set("q", query.Q)
	}

	if query.Ep != "" {
		v.Set("ep", query.Ep)
	}

	if query.Season != "" {
		v.Set("season", query.Season)
	}

	if query.Offset != 0 {
		v.Set("offset", strconv.Itoa(query.Offset))
	}

	if query.Limit != 0 {
		v.Set("limit", strconv.Itoa(query.Limit))
	}

	if query.Extended {
		v.Set("extended", "1")
	}

	if query.APIKey != "" {
		v.Set("apikey", query.APIKey)
	}

	if len(query.Categories) > 0 {
		cats := []string{}

		for _, cat := range query.Categories {
			cats = append(cats, strconv.Itoa(cat))
		}

		v.Set("cat", strings.Join(cats, ","))
	}

	return v.Encode()
}

func (query Query) String() string {
	return query.Encode()
}

// ParseQuery takes the query string parameters for a torznab query and parses them
func ParseQuery(v url.Values) (Query, error) {
	query := Query{}

	for k, vals := range v {
		switch k {
		case "t":
			if len(vals) > 1 {
				return query, errors.New("Multiple t parameters not allowed")
			}
			query.Type = vals[0]

		case "q":
			query.Q = strings.Join(vals, " ")

		case "ep":
			if len(vals) > 1 {
				return query, errors.New("Multiple ep parameters not allowed")
			}
			query.Ep = vals[0]

		case "season":
			if len(vals) > 1 {
				return query, errors.New("Multiple season parameters not allowed")
			}
			query.Season = vals[0]

		case "apikey":
			if len(vals) > 1 {
				return query, errors.New("Multiple apikey parameters not allowed")
			}
			query.APIKey = vals[0]

		case "limit":
			if len(vals) > 1 {
				return query, errors.New("Multiple limit parameters not allowed")
			}
			limit, err := strconv.Atoi(vals[0])
			if err != nil {
				return query, err
			}
			query.Limit = limit

		case "offset":
			if len(vals) > 1 {
				return query, errors.New("Multiple offset parameters not allowed")
			}
			offset, err := strconv.Atoi(vals[0])
			if err != nil {
				return query, err
			}
			query.Offset = offset

		case "extended":
			if len(vals) > 1 {
				return query, errors.New("Multiple extended parameters not allowed")
			}
			extended, err := strconv.ParseBool(vals[0])
			if err != nil {
				return query, err
			}
			query.Extended = extended

		case "cat":
			query.Categories = []int{}
			for _, val := range vals {
				ints, err := splitInts(val, ",")
				if err != nil {
					return Query{}, fmt.Errorf("Unable to parse cats %q", vals[0])
				}
				query.Categories = append(query.Categories, ints...)
			}

		case "format":

		default:
			logger.Logger.Warnf("Unknown torznab request key %q", k)
		}
	}

	return query, nil
}

func splitInts(s, delim string) (i []int, err error) {
	for _, v := range strings.Split(s, delim) {
		vInt, err := strconv.Atoi(v)
		if err != nil {
			return i, err
		}
		i = append(i, vInt)
	}
	return i, err
}

func padLeft(str, pad string, length int) string {
	for len(str) < length {
		str = pad + str
	}
	return str
}
<|endoftext|>


<commit_before>/*
Copyright 2019 The Kubernetes Authors.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package bumper

import (
	"fmt"
	"os"
	"os/exec"
	"path/filepath"
	"regexp"
	"sort"
	"strings"

	"github.com/sirupsen/logrus"

	"k8s.io/test-infra/experiment/image-bumper/bumper"
	"k8s.io/test-infra/prow/github"
	"k8s.io/test-infra/robots/pr-creator/updater"
)

const (
	prowPrefix      = "gcr.io/k8s-prow/"
	testImagePrefix = "gcr.io/k8s-testimages/"
	prowRepo        = "https://github.com/kubernetes/test-infra"
	testImageRepo   = prowRepo
)

func Call(cmd string, args ...string) error {
	c := exec.Command(cmd, args...)
	c.Stdout = os.Stdout
	c.Stderr = os.Stderr
	return c.Run()
}

// UpdatePR updates with github client "gc" the PR of github repo org/repo
// with "matchTitle" from "source" to "branch"
// "images" contains the tag replacements that have been made which is returned from "UpdateReferences([]string{"."}, extraFiles)"
// "images" and "extraLineInPRBody" are used to generate commit summary and body of the PR
func UpdatePR(gc github.Client, org, repo string, images map[string]string, extraLineInPRBody string, matchTitle, source, branch string) error {
	return updatePR(gc, org, repo, makeCommitSummary(images), generatePRBody(images, extraLineInPRBody), matchTitle, source, branch)
}

func updatePR(gc github.Client, org, repo, title, body, matchTitle, source, branch string) error {
	logrus.Info("Creating PR...")
	n, err := updater.UpdatePR(org, repo, title, body, matchTitle, gc)
	if err != nil {
		return fmt.Errorf("failed to update %d: %v", n, err)
	}
	if n == nil {
		pr, err := gc.CreatePullRequest(org, repo, title, body, source, branch, true)
		if err != nil {
			return fmt.Errorf("failed to create PR: %v", err)
		}
		n = &pr
	}

	logrus.Infof("PR %s/%s#%d will merge %s into %s: %s", org, repo, *n, source, branch, title)
	return nil
}

// UpdateReferences update the references of prow-images and testimages
// in the files in any of "subfolders" of the current dir
// if the file is a yaml file (*.yaml) or extraFiles[file]=true
func UpdateReferences(subfolders []string, extraFiles map[string]bool) (map[string]string, error) {
	logrus.Info("Bumping image references...")
	filter := regexp.MustCompile(prowPrefix + "|" + testImagePrefix)

	for _, dir := range subfolders {
		err := filepath.Walk(dir, func(path string, info os.FileInfo, err error) error {
			if strings.HasSuffix(path, ".yaml") || extraFiles[path] {
				if err := bumper.UpdateFile(path, filter); err != nil {
					logrus.WithError(err).Errorf("Failed to update path %s.", path)
				}
			}
			return nil
		})
		if err != nil {
			return nil, err
		}
	}

	return bumper.GetReplacements(), nil
}

func getNewProwVersion(images map[string]string) string {
	for k, v := range images {
		if strings.HasPrefix(k, prowPrefix) {
			return v
		}
	}
	return ""
}

func makeCommitSummary(images map[string]string) string {
	return fmt.Sprintf("Update prow to %s, and other images as necessary.", getNewProwVersion(images))
}

// MakeGitCommit runs a sequence of git commands to
// commit and push the changes the "remote" on "remoteBranch"
// "name" and "email" are used for git-commit command
// "images" contains the tag replacements that have been made which is returned from "UpdateReferences([]string{"."}, extraFiles)"
// "images" is used to generate commit message
func MakeGitCommit(remote, remoteBranch, name, email string, images map[string]string) error {
	logrus.Info("Making git commit...")
	if err := Call("git", "add", "-A"); err != nil {
		return fmt.Errorf("failed to git add: %v", err)
	}
	message := makeCommitSummary(images)
	commitArgs := []string{"commit", "-m", message}
	if name != "" && email != "" {
		commitArgs = append(commitArgs, "--author", fmt.Sprintf("%s <%s>", name, email))
	}
	if err := Call("git", commitArgs...); err != nil {
		return fmt.Errorf("failed to git commit: %v", err)
	}
	logrus.Info("Pushing to remote...")
	if err := Call("git", "push", "-f", remote, fmt.Sprintf("HEAD:%s", remoteBranch)); err != nil {
		return fmt.Errorf("failed to git push: %v", err)
	}
	return nil
}

func tagFromName(name string) string {
	parts := strings.Split(name, ":")
	if len(parts) < 2 {
		return ""
	}
	return parts[1]
}

func componentFromName(name string) string {
	s := strings.Split(strings.Split(name, ":")[0], "/")
	return s[len(s)-1]
}

func formatTagDate(d string) string {
	if len(d) != 8 {
		return d
	}
	// ‑ = U+2011 NON-BREAKING HYPHEN, to prevent line wraps.
	return fmt.Sprintf("%s‑%s‑%s", d[0:4], d[4:6], d[6:8])
}

func generateSummary(name, repo, prefix string, summarise bool, images map[string]string) string {
	type delta struct {
		oldCommit string
		newCommit string
		oldDate   string
		newDate   string
		variant   string
		component string
	}
	versions := map[string][]delta{}
	for image, newTag := range images {
		if !strings.HasPrefix(image, prefix) {
			continue
		}
		if strings.HasSuffix(image, ":"+newTag) {
			continue
		}
		oldDate, oldCommit, oldVariant := bumper.DeconstructTag(tagFromName(image))
		newDate, newCommit, _ := bumper.DeconstructTag(newTag)
		k := oldCommit + ":" + newCommit
		d := delta{
			oldCommit: oldCommit,
			newCommit: newCommit,
			oldDate:   oldDate,
			newDate:   newDate,
			variant:   oldVariant,
			component: componentFromName(image),
		}
		versions[k] = append(versions[k], d)
	}

	switch {
	case len(versions) == 0:
		return fmt.Sprintf("No %s changes.", name)
	case len(versions) == 1 && summarise:
		for k, v := range versions {
			s := strings.Split(k, ":")
			return fmt.Sprintf("%s changes: %s/compare/%s...%s (%s → %s)", name, repo, s[0], s[1], formatTagDate(v[0].oldDate), formatTagDate(v[0].newDate))
		}
	default:
		changes := make([]string, 0, len(versions))
		for k, v := range versions {
			s := strings.Split(k, ":")
			names := make([]string, 0, len(v))
			for _, d := range v {
				names = append(names, d.component+d.variant)
			}
			sort.Strings(names)
			changes = append(changes, fmt.Sprintf("%s/compare/%s...%s | %s → %s | %s",
				repo, s[0], s[1], formatTagDate(v[0].oldDate), formatTagDate(v[0].newDate), strings.Join(names, ", ")))
		}
		sort.Slice(changes, func(i, j int) bool { return strings.Split(changes[i], "|")[1] < strings.Split(changes[j], "|")[1] })
		return fmt.Sprintf("Multiple distinct %s changes:\n\nCommits | Dates | Images\n--- | --- | ---\n%s\n", name, strings.Join(changes, "\n"))
	}
	panic("unreachable!")
}

func generatePRBody(images map[string]string, assignment string) string {
	prowSummary := generateSummary("Prow", prowRepo, prowPrefix, true, images)
	testImagesSummary := generateSummary("test-image", testImageRepo, testImagePrefix, false, images)
	return prowSummary + "\n\n" + testImagesSummary + "\n\n" + assignment + "\n"
}
<commit_msg>Make 2 more functions in autobumper public<commit_after>/*
Copyright 2019 The Kubernetes Authors.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package bumper

import (
	"fmt"
	"os"
	"os/exec"
	"path/filepath"
	"regexp"
	"sort"
	"strings"

	"github.com/sirupsen/logrus"

	"k8s.io/test-infra/experiment/image-bumper/bumper"
	"k8s.io/test-infra/prow/github"
	"k8s.io/test-infra/robots/pr-creator/updater"
)

const (
	prowPrefix      = "gcr.io/k8s-prow/"
	testImagePrefix = "gcr.io/k8s-testimages/"
	prowRepo        = "https://github.com/kubernetes/test-infra"
	testImageRepo   = prowRepo
)

func Call(cmd string, args ...string) error {
	c := exec.Command(cmd, args...)
	c.Stdout = os.Stdout
	c.Stderr = os.Stderr
	return c.Run()
}

// UpdatePR updates with github client "gc" the PR of github repo org/repo
// with "matchTitle" from "source" to "branch"
// "images" contains the tag replacements that have been made which is returned from "UpdateReferences([]string{"."}, extraFiles)"
// "images" and "extraLineInPRBody" are used to generate commit summary and body of the PR
func UpdatePR(gc github.Client, org, repo string, images map[string]string, extraLineInPRBody string, matchTitle, source, branch string) error {
	return UpdatePullRequest(gc, org, repo, makeCommitSummary(images), generatePRBody(images, extraLineInPRBody), matchTitle, source, branch)
}

// UpdatePullRequest updates with github client "gc" the PR of github repo org/repo
// with "title" and "body" of PR matching "matchTitle" from "source" to "branch"
func UpdatePullRequest(gc github.Client, org, repo, title, body, matchTitle, source, branch string) error {
	logrus.Info("Creating PR...")
	n, err := updater.UpdatePR(org, repo, title, body, matchTitle, gc)
	if err != nil {
		return fmt.Errorf("failed to update %d: %v", n, err)
	}
	if n == nil {
		pr, err := gc.CreatePullRequest(org, repo, title, body, source, branch, true)
		if err != nil {
			return fmt.Errorf("failed to create PR: %v", err)
		}
		n = &pr
	}

	logrus.Infof("PR %s/%s#%d will merge %s into %s: %s", org, repo, *n, source, branch, title)
	return nil
}

// UpdateReferences update the references of prow-images and testimages
// in the files in any of "subfolders" of the current dir
// if the file is a yaml file (*.yaml) or extraFiles[file]=true
func UpdateReferences(subfolders []string, extraFiles map[string]bool) (map[string]string, error) {
	logrus.Info("Bumping image references...")
	filter := regexp.MustCompile(prowPrefix + "|" + testImagePrefix)

	for _, dir := range subfolders {
		err := filepath.Walk(dir, func(path string, info os.FileInfo, err error) error {
			if strings.HasSuffix(path, ".yaml") || extraFiles[path] {
				if err := bumper.UpdateFile(path, filter); err != nil {
					logrus.WithError(err).Errorf("Failed to update path %s.", path)
				}
			}
			return nil
		})
		if err != nil {
			return nil, err
		}
	}

	return bumper.GetReplacements(), nil
}

func getNewProwVersion(images map[string]string) string {
	for k, v := range images {
		if strings.HasPrefix(k, prowPrefix) {
			return v
		}
	}
	return ""
}

func makeCommitSummary(images map[string]string) string {
	return fmt.Sprintf("Update prow to %s, and other images as necessary.", getNewProwVersion(images))
}

// MakeGitCommit runs a sequence of git commands to
// commit and push the changes the "remote" on "remoteBranch"
// "name" and "email" are used for git-commit command
// "images" contains the tag replacements that have been made which is returned from "UpdateReferences([]string{"."}, extraFiles)"
// "images" is used to generate commit message
func MakeGitCommit(remote, remoteBranch, name, email string, images map[string]string) error {
	return GitCommitAndPush(remote, remoteBranch, name, email, makeCommitSummary(images))
}

// GitCommitAndPush runs a sequence of git commands to
// commit and push the changes the "remote" on "remoteBranch"
// "name", "email", and "message" are used for git-commit command
func GitCommitAndPush(remote, remoteBranch, name, email, message string) error {
	logrus.Info("Making git commit...")
	if err := Call("git", "add", "-A"); err != nil {
		return fmt.Errorf("failed to git add: %v", err)
	}
	commitArgs := []string{"commit", "-m", message}
	if name != "" && email != "" {
		commitArgs = append(commitArgs, "--author", fmt.Sprintf("%s <%s>", name, email))
	}
	if err := Call("git", commitArgs...); err != nil {
		return fmt.Errorf("failed to git commit: %v", err)
	}
	logrus.Info("Pushing to remote...")
	if err := Call("git", "push", "-f", remote, fmt.Sprintf("HEAD:%s", remoteBranch)); err != nil {
		return fmt.Errorf("failed to git push: %v", err)
	}
	return nil
}

func tagFromName(name string) string {
	parts := strings.Split(name, ":")
	if len(parts) < 2 {
		return ""
	}
	return parts[1]
}

func componentFromName(name string) string {
	s := strings.Split(strings.Split(name, ":")[0], "/")
	return s[len(s)-1]
}

func formatTagDate(d string) string {
	if len(d) != 8 {
		return d
	}
	// ‑ = U+2011 NON-BREAKING HYPHEN, to prevent line wraps.
	return fmt.Sprintf("%s‑%s‑%s", d[0:4], d[4:6], d[6:8])
}

func generateSummary(name, repo, prefix string, summarise bool, images map[string]string) string {
	type delta struct {
		oldCommit string
		newCommit string
		oldDate   string
		newDate   string
		variant   string
		component string
	}
	versions := map[string][]delta{}
	for image, newTag := range images {
		if !strings.HasPrefix(image, prefix) {
			continue
		}
		if strings.HasSuffix(image, ":"+newTag) {
			continue
		}
		oldDate, oldCommit, oldVariant := bumper.DeconstructTag(tagFromName(image))
		newDate, newCommit, _ := bumper.DeconstructTag(newTag)
		k := oldCommit + ":" + newCommit
		d := delta{
			oldCommit: oldCommit,
			newCommit: newCommit,
			oldDate:   oldDate,
			newDate:   newDate,
			variant:   oldVariant,
			component: componentFromName(image),
		}
		versions[k] = append(versions[k], d)
	}

	switch {
	case len(versions) == 0:
		return fmt.Sprintf("No %s changes.", name)
	case len(versions) == 1 && summarise:
		for k, v := range versions {
			s := strings.Split(k, ":")
			return fmt.Sprintf("%s changes: %s/compare/%s...%s (%s → %s)", name, repo, s[0], s[1], formatTagDate(v[0].oldDate), formatTagDate(v[0].newDate))
		}
	default:
		changes := make([]string, 0, len(versions))
		for k, v := range versions {
			s := strings.Split(k, ":")
			names := make([]string, 0, len(v))
			for _, d := range v {
				names = append(names, d.component+d.variant)
			}
			sort.Strings(names)
			changes = append(changes, fmt.Sprintf("%s/compare/%s...%s | %s → %s | %s",
				repo, s[0], s[1], formatTagDate(v[0].oldDate), formatTagDate(v[0].newDate), strings.Join(names, ", ")))
		}
		sort.Slice(changes, func(i, j int) bool { return strings.Split(changes[i], "|")[1] < strings.Split(changes[j], "|")[1] })
		return fmt.Sprintf("Multiple distinct %s changes:\n\nCommits | Dates | Images\n--- | --- | ---\n%s\n", name, strings.Join(changes, "\n"))
	}
	panic("unreachable!")
}

func generatePRBody(images map[string]string, assignment string) string {
	prowSummary := generateSummary("Prow", prowRepo, prowPrefix, true, images)
	testImagesSummary := generateSummary("test-image", testImageRepo, testImagePrefix, false, images)
	return prowSummary + "\n\n" + testImagesSummary + "\n\n" + assignment + "\n"
}
<|endoftext|>


<commit_before>package checkerlution

import (
	"fmt"
	"github.com/couchbaselabs/logg"
	cbot "github.com/tleyden/checkers-bot"
	ng "github.com/tleyden/neurgo"
	nv "github.com/tleyden/neurvolve"
	"math"
	"time"
)

type CheckerlutionTrainer struct{}

func (trainer *CheckerlutionTrainer) RunPopulationTrainer(checkerlutionFlags CheckerlutionFlags) {

	checkersBotFlags := checkerlutionFlags.CheckersBotFlags

	// setup the scape
	scape := &CheckerlutionScape{}
	scape.SetSyncGatewayUrl(checkersBotFlags.SyncGatewayUrl)
	scape.SetFeedType(checkersBotFlags.FeedType)
	scape.SetTeam(checkersBotFlags.Team)
	scape.SetRandomDelayBeforeMove(checkersBotFlags.RandomDelayBeforeMove)

	// create population trainer ...
	pt := &nv.PopulationTrainer{
		FitnessThreshold: 1000, // set very high, will never hit it ..
		MaxGenerations:   25,
		CortexMutator:    nv.MutateAllWeightsBellCurve,
		NumOpponents:     5,
	}

	nv.RegisterHandlers(pt)

	recorder := NewRecorder(
		checkersBotFlags.SyncGatewayUrl,
		checkerlutionFlags.PopulationName,
	)

	cortexes := []*ng.Cortex{}
	if len(recorder.GetLatestGenerationCortexes()) > 0 {
		cortexes = recorder.GetLatestGenerationCortexes()
		if len(cortexes) == 0 {
			logg.LogPanic("No cortexes found in latest generation")
		}
		logg.LogTo("CHECKERLUTION", "Starting from existing generation")

	} else {
		cortexes = getInitialGeneration()
		logg.LogTo("CHECKERLUTION", "Starting from new")
	}

	for _, cortex := range cortexes {
		logg.LogTo("CHECKERLUTION", "Cortex: %v", cortex.NodeId.UUID)
	}

	fitGeneration, succeeded := pt.Train(cortexes, scape, recorder)

	if succeeded {
		logg.LogTo("MAIN", "Training finished (exceeded threshold)")
	} else {
		logg.LogTo("MAIN", "Training finished (did not exceed threshold)")
	}

	for i, evaldCortex := range fitGeneration {

		logg.LogTo("MAIN", "Cortex %d fitness: %v", i, evaldCortex.Fitness)

	}

}

func (trainer *CheckerlutionTrainer) RunTopologyMutatingTrainer(checkersBotFlags cbot.CheckersBotFlags) {

	// setup the scape
	scape := &CheckerlutionScape{}
	scape.SetSyncGatewayUrl(checkersBotFlags.SyncGatewayUrl)
	scape.SetFeedType(checkersBotFlags.FeedType)
	scape.SetTeam(checkersBotFlags.Team)
	scape.SetRandomDelayBeforeMove(checkersBotFlags.RandomDelayBeforeMove)

	// create a stochastic hill climber (required by topology mutation trainer)
	shc := &nv.StochasticHillClimber{
		FitnessThreshold:           150,
		MaxIterationsBeforeRestart: 10,
		MaxAttempts:                500,
		WeightSaturationRange:      []float64{-2 * math.Pi, 2 * math.Pi},
	}

	// this thing will train the network by randomly mutating and calculating fitness
	tmt := &nv.TopologyMutatingTrainer{
		MaxAttempts:                100,
		MaxIterationsBeforeRestart: 5,
		StochasticHillClimber:      shc,
	}
	initialCortex := getInitialCortex()
	cortexTrained, succeeded := tmt.Train(initialCortex, scape)
	if succeeded {
		logg.LogTo("MAIN", "Training succeeded")
	} else {
		logg.LogTo("MAIN", "Training Failed")
	}

	filename := fmt.Sprintf("/tmp/checkerlution-%v.json", time.Now().Unix())
	logg.LogTo("MAIN", "Saving Cortex to %v", filename)
	cortexTrained.MarshalJSONToFile(filename)

}

func getInitialGeneration() (generation []*ng.Cortex) {
	generation = make([]*ng.Cortex, 0)
	for i := 0; i < 10; i++ {

		thinker := &Checkerlution{}

		thinker.CreateNeurgoCortex()

		generation = append(generation, thinker.Cortex())
	}
	return
}

func getInitialCortex() (cortex *ng.Cortex) {
	thinker := &Checkerlution{}
	thinker.CreateNeurgoCortex()
	return thinker.Cortex()
}
<commit_msg>Use 30 generations to better match Blondie24<commit_after>package checkerlution

import (
	"fmt"
	"github.com/couchbaselabs/logg"
	cbot "github.com/tleyden/checkers-bot"
	ng "github.com/tleyden/neurgo"
	nv "github.com/tleyden/neurvolve"
	"math"
	"time"
)

type CheckerlutionTrainer struct{}

func (trainer *CheckerlutionTrainer) RunPopulationTrainer(checkerlutionFlags CheckerlutionFlags) {

	checkersBotFlags := checkerlutionFlags.CheckersBotFlags

	// setup the scape
	scape := &CheckerlutionScape{}
	scape.SetSyncGatewayUrl(checkersBotFlags.SyncGatewayUrl)
	scape.SetFeedType(checkersBotFlags.FeedType)
	scape.SetTeam(checkersBotFlags.Team)
	scape.SetRandomDelayBeforeMove(checkersBotFlags.RandomDelayBeforeMove)

	// create population trainer ...
	pt := &nv.PopulationTrainer{
		FitnessThreshold: 1000, // set very high, will never hit it ..
		MaxGenerations:   25,
		CortexMutator:    nv.MutateAllWeightsBellCurve,
		NumOpponents:     5,
	}

	nv.RegisterHandlers(pt)

	recorder := NewRecorder(
		checkersBotFlags.SyncGatewayUrl,
		checkerlutionFlags.PopulationName,
	)

	cortexes := []*ng.Cortex{}
	if len(recorder.GetLatestGenerationCortexes()) > 0 {
		cortexes = recorder.GetLatestGenerationCortexes()
		if len(cortexes) == 0 {
			logg.LogPanic("No cortexes found in latest generation")
		}
		logg.LogTo("CHECKERLUTION", "Starting from existing generation")

	} else {
		cortexes = getInitialGeneration()
		logg.LogTo("CHECKERLUTION", "Starting from new")
	}

	for _, cortex := range cortexes {
		logg.LogTo("CHECKERLUTION", "Cortex: %v", cortex.NodeId.UUID)
	}

	fitGeneration, succeeded := pt.Train(cortexes, scape, recorder)

	if succeeded {
		logg.LogTo("MAIN", "Training finished (exceeded threshold)")
	} else {
		logg.LogTo("MAIN", "Training finished (did not exceed threshold)")
	}

	for i, evaldCortex := range fitGeneration {

		logg.LogTo("MAIN", "Cortex %d fitness: %v", i, evaldCortex.Fitness)

	}

}

func (trainer *CheckerlutionTrainer) RunTopologyMutatingTrainer(checkersBotFlags cbot.CheckersBotFlags) {

	// setup the scape
	scape := &CheckerlutionScape{}
	scape.SetSyncGatewayUrl(checkersBotFlags.SyncGatewayUrl)
	scape.SetFeedType(checkersBotFlags.FeedType)
	scape.SetTeam(checkersBotFlags.Team)
	scape.SetRandomDelayBeforeMove(checkersBotFlags.RandomDelayBeforeMove)

	// create a stochastic hill climber (required by topology mutation trainer)
	shc := &nv.StochasticHillClimber{
		FitnessThreshold:           150,
		MaxIterationsBeforeRestart: 10,
		MaxAttempts:                500,
		WeightSaturationRange:      []float64{-2 * math.Pi, 2 * math.Pi},
	}

	// this thing will train the network by randomly mutating and calculating fitness
	tmt := &nv.TopologyMutatingTrainer{
		MaxAttempts:                100,
		MaxIterationsBeforeRestart: 5,
		StochasticHillClimber:      shc,
	}
	initialCortex := getInitialCortex()
	cortexTrained, succeeded := tmt.Train(initialCortex, scape)
	if succeeded {
		logg.LogTo("MAIN", "Training succeeded")
	} else {
		logg.LogTo("MAIN", "Training Failed")
	}

	filename := fmt.Sprintf("/tmp/checkerlution-%v.json", time.Now().Unix())
	logg.LogTo("MAIN", "Saving Cortex to %v", filename)
	cortexTrained.MarshalJSONToFile(filename)

}

func getInitialGeneration() (generation []*ng.Cortex) {
	generation = make([]*ng.Cortex, 0)
	for i := 0; i < 30; i++ {

		thinker := &Checkerlution{}

		thinker.CreateNeurgoCortex()

		generation = append(generation, thinker.Cortex())
	}
	return
}

func getInitialCortex() (cortex *ng.Cortex) {
	thinker := &Checkerlution{}
	thinker.CreateNeurgoCortex()
	return thinker.Cortex()
}
<|endoftext|>


<commit_before>package commands

import (
	"errors"

	cmds "github.com/jbenet/go-ipfs/commands"
	u "github.com/jbenet/go-ipfs/util"
)

var resolveCmd = &cmds.Command{
	Helptext: cmds.HelpText{
		Tagline: "Gets the value currently published at an IPNS name",
		ShortDescription: `
IPNS is a PKI namespace, where names are the hashes of public keys, and
the private key enables publishing new (signed) values. In resolve, the
default value of <name> is your own identity public key.
`,
		LongDescription: `
IPNS is a PKI namespace, where names are the hashes of public keys, and
the private key enables publishing new (signed) values. In resolve, the
default value of <name> is your own identity public key.

Examples:

Resolve the value of your identity:

  > ipfs name resolve
  QmatmE9msSfkKxoffpHwNLNKgwZG8eT9Bud6YoPab52vpy

Resolve te value of another name:

  > ipfs name resolve QmbCMUZw6JFeZ7Wp9jkzbye3Fzp2GGcPgC3nmeUjfVF87n
  QmatmE9msSfkKxoffpHwNLNKgwZG8eT9Bud6YoPab52vpy

`,
	},

	Arguments: []cmds.Argument{
		cmds.StringArg("name", false, false, "The IPNS name to resolve. Defaults to your node's peerID."),
	},
	Run: func(req cmds.Request) (interface{}, error) {

		n, err := req.Context().GetNode()
		if err != nil {
			return nil, err
		}

		var name string

		if n.Network == nil {
			return nil, errNotOnline
		}

		if len(req.Arguments()) == 0 {
			if n.Identity == nil {
				return nil, errors.New("Identity not loaded!")
			}
			name = n.Identity.ID().String()

		} else {
			var ok bool
			name, ok = req.Arguments()[0].(string)
			if !ok {
				return nil, u.ErrCast()
			}
		}

		output, err := n.Namesys.Resolve(name)
		if err != nil {
			return nil, err
		}

		return output, nil
	},
	Marshalers: cmds.MarshalerMap{
		cmds.Text: func(res cmds.Response) ([]byte, error) {
			output := res.Output().(string)
			return []byte(output), nil
		},
	},
}
<commit_msg>resolve: Added a TODO<commit_after>package commands

import (
	"errors"

	cmds "github.com/jbenet/go-ipfs/commands"
	u "github.com/jbenet/go-ipfs/util"
)

var resolveCmd = &cmds.Command{
	Helptext: cmds.HelpText{
		Tagline: "Gets the value currently published at an IPNS name",
		ShortDescription: `
IPNS is a PKI namespace, where names are the hashes of public keys, and
the private key enables publishing new (signed) values. In resolve, the
default value of <name> is your own identity public key.
`,
		LongDescription: `
IPNS is a PKI namespace, where names are the hashes of public keys, and
the private key enables publishing new (signed) values. In resolve, the
default value of <name> is your own identity public key.

Examples:

Resolve the value of your identity:

  > ipfs name resolve
  QmatmE9msSfkKxoffpHwNLNKgwZG8eT9Bud6YoPab52vpy

Resolve te value of another name:

  > ipfs name resolve QmbCMUZw6JFeZ7Wp9jkzbye3Fzp2GGcPgC3nmeUjfVF87n
  QmatmE9msSfkKxoffpHwNLNKgwZG8eT9Bud6YoPab52vpy

`,
	},

	Arguments: []cmds.Argument{
		cmds.StringArg("name", false, false, "The IPNS name to resolve. Defaults to your node's peerID."),
	},
	Run: func(req cmds.Request) (interface{}, error) {

		n, err := req.Context().GetNode()
		if err != nil {
			return nil, err
		}

		var name string

		if n.Network == nil {
			return nil, errNotOnline
		}

		if len(req.Arguments()) == 0 {
			if n.Identity == nil {
				return nil, errors.New("Identity not loaded!")
			}
			name = n.Identity.ID().String()

		} else {
			var ok bool
			name, ok = req.Arguments()[0].(string)
			if !ok {
				return nil, u.ErrCast()
			}
		}

		output, err := n.Namesys.Resolve(name)
		if err != nil {
			return nil, err
		}

		// TODO: better errors (in the case of not finding the name, we get "failed to find any peer in table")

		return output, nil
	},
	Marshalers: cmds.MarshalerMap{
		cmds.Text: func(res cmds.Response) ([]byte, error) {
			output := res.Output().(string)
			return []byte(output), nil
		},
	},
}
<|endoftext|>


<commit_before>// Copyright 2016 Google Inc. All Rights Reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//      http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package bigquery

import (
	"io"

	bq "google.golang.org/api/bigquery/v2"
)

// A ReaderSource is a source for a load operation that gets
// data from an io.Reader.
type ReaderSource struct {
	r io.Reader
	FileConfig
}

// NewReaderSource creates a ReaderSource from an io.Reader. You may
// optionally configure properties on the ReaderSource that describe the
// data being read, before passing it to Table.LoaderFrom.
func NewReaderSource(r io.Reader) *ReaderSource {
	return &ReaderSource{r: r}
}

func (r *ReaderSource) populateInsertJobConfForLoad(conf *insertJobConf) {
	conf.media = r.r
	r.FileConfig.populateLoadConfig(conf.job.Configuration.Load)
}

// FileConfig contains configuration options that pertain to files, typically
// text files that require interpretation to be used as a BigQuery table. A
// file may live in Google Cloud Storage (see GCSReference), or it may be
// loaded into a table via the Table.LoaderFromReader.
type FileConfig struct {
	// SourceFormat is the format of the GCS data to be read.
	// Allowed values are: CSV, Avro, JSON, DatastoreBackup.  The default is CSV.
	SourceFormat DataFormat

	// FieldDelimiter is the separator for fields in a CSV file, used when
	// reading or exporting data. The default is ",".
	FieldDelimiter string

	// The number of rows at the top of a CSV file that BigQuery will skip when
	// reading data.
	SkipLeadingRows int64

	// AllowJaggedRows causes missing trailing optional columns to be tolerated
	// when reading CSV data. Missing values are treated as nulls.
	AllowJaggedRows bool

	// AllowQuotedNewlines sets whether quoted data sections containing
	// newlines are allowed when reading CSV data.
	AllowQuotedNewlines bool

	// Encoding is the character encoding of data to be read.
	Encoding Encoding

	// MaxBadRecords is the maximum number of bad records that will be ignored
	// when reading data.
	MaxBadRecords int64

	// IgnoreUnknownValues causes values not matching the schema to be
	// tolerated. Unknown values are ignored. For CSV this ignores extra values
	// at the end of a line. For JSON this ignores named values that do not
	// match any column name. If this field is not set, records containing
	// unknown values are treated as bad records. The MaxBadRecords field can
	// be used to customize how bad records are handled.
	IgnoreUnknownValues bool

	// Schema describes the data. It is required when reading CSV or JSON data,
	// unless the data is being loaded into a table that already exists.
	Schema Schema

	// Quote is the value used to quote data sections in a CSV file. The
	// default quotation character is the double quote ("), which is used if
	// both Quote and ForceZeroQuote are unset.
	// To specify that no character should be interpreted as a quotation
	// character, set ForceZeroQuote to true.
	// Only used when reading data.
	Quote          string
	ForceZeroQuote bool
}

// quote returns the CSV quote character, or nil if unset.
func (fc *FileConfig) quote() *string {
	if fc.ForceZeroQuote {
		quote := ""
		return &quote
	}
	if fc.Quote == "" {
		return nil
	}
	return &fc.Quote
}

func (fc *FileConfig) populateLoadConfig(conf *bq.JobConfigurationLoad) {
	conf.SkipLeadingRows = fc.SkipLeadingRows
	conf.SourceFormat = string(fc.SourceFormat)
	conf.AllowJaggedRows = fc.AllowJaggedRows
	conf.AllowQuotedNewlines = fc.AllowQuotedNewlines
	conf.Encoding = string(fc.Encoding)
	conf.FieldDelimiter = fc.FieldDelimiter
	conf.IgnoreUnknownValues = fc.IgnoreUnknownValues
	conf.MaxBadRecords = fc.MaxBadRecords
	if fc.Schema != nil {
		conf.Schema = fc.Schema.asTableSchema()
	}
	conf.Quote = fc.quote()
}

func (fc *FileConfig) populateExternalDataConfig(conf *bq.ExternalDataConfiguration) {
	format := fc.SourceFormat
	if format == "" {
		// Format must be explicitly set for external data sources.
		format = CSV
	}
	// TODO(jba): support AutoDetect.
	conf.IgnoreUnknownValues = fc.IgnoreUnknownValues
	conf.MaxBadRecords = fc.MaxBadRecords
	conf.SourceFormat = string(format)
	if fc.Schema != nil {
		conf.Schema = fc.Schema.asTableSchema()
	}
	if format == CSV {
		conf.CsvOptions = &bq.CsvOptions{
			AllowJaggedRows:     fc.AllowJaggedRows,
			AllowQuotedNewlines: fc.AllowQuotedNewlines,
			Encoding:            string(fc.Encoding),
			FieldDelimiter:      fc.FieldDelimiter,
			SkipLeadingRows:     fc.SkipLeadingRows,
			Quote:               fc.quote(),
		}
	}
}

// DataFormat describes the format of BigQuery table data.
type DataFormat string

// Constants describing the format of BigQuery table data.
const (
	CSV             DataFormat = "CSV"
	Avro            DataFormat = "AVRO"
	JSON            DataFormat = "NEWLINE_DELIMITED_JSON"
	DatastoreBackup DataFormat = "DATASTORE_BACKUP"
)

// Encoding specifies the character encoding of data to be loaded into BigQuery.
// See https://cloud.google.com/bigquery/docs/reference/v2/jobs#configuration.load.encoding
// for more details about how this is used.
type Encoding string

const (
	UTF_8      Encoding = "UTF-8"
	ISO_8859_1 Encoding = "ISO-8859-1"
)
<commit_msg>bigquery: add autodetect support to load job<commit_after>// Copyright 2016 Google Inc. All Rights Reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//      http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package bigquery

import (
	"io"

	bq "google.golang.org/api/bigquery/v2"
)

// A ReaderSource is a source for a load operation that gets
// data from an io.Reader.
type ReaderSource struct {
	r io.Reader
	FileConfig
}

// NewReaderSource creates a ReaderSource from an io.Reader. You may
// optionally configure properties on the ReaderSource that describe the
// data being read, before passing it to Table.LoaderFrom.
func NewReaderSource(r io.Reader) *ReaderSource {
	return &ReaderSource{r: r}
}

func (r *ReaderSource) populateInsertJobConfForLoad(conf *insertJobConf) {
	conf.media = r.r
	r.FileConfig.populateLoadConfig(conf.job.Configuration.Load)
}

// FileConfig contains configuration options that pertain to files, typically
// text files that require interpretation to be used as a BigQuery table. A
// file may live in Google Cloud Storage (see GCSReference), or it may be
// loaded into a table via the Table.LoaderFromReader.
type FileConfig struct {
	// SourceFormat is the format of the GCS data to be read.
	// Allowed values are: CSV, Avro, JSON, DatastoreBackup.  The default is CSV.
	SourceFormat DataFormat

	// FieldDelimiter is the separator for fields in a CSV file, used when
	// reading or exporting data. The default is ",".
	FieldDelimiter string

	// The number of rows at the top of a CSV file that BigQuery will skip when
	// reading data.
	SkipLeadingRows int64

	// AllowJaggedRows causes missing trailing optional columns to be tolerated
	// when reading CSV data. Missing values are treated as nulls.
	AllowJaggedRows bool

	// AllowQuotedNewlines sets whether quoted data sections containing
	// newlines are allowed when reading CSV data.
	AllowQuotedNewlines bool

	// Indicates if we should automatically infer the options and
	// schema for CSV and JSON sources.
	AutoDetect bool

	// Encoding is the character encoding of data to be read.
	Encoding Encoding

	// MaxBadRecords is the maximum number of bad records that will be ignored
	// when reading data.
	MaxBadRecords int64

	// IgnoreUnknownValues causes values not matching the schema to be
	// tolerated. Unknown values are ignored. For CSV this ignores extra values
	// at the end of a line. For JSON this ignores named values that do not
	// match any column name. If this field is not set, records containing
	// unknown values are treated as bad records. The MaxBadRecords field can
	// be used to customize how bad records are handled.
	IgnoreUnknownValues bool

	// Schema describes the data. It is required when reading CSV or JSON data,
	// unless the data is being loaded into a table that already exists.
	Schema Schema

	// Quote is the value used to quote data sections in a CSV file. The
	// default quotation character is the double quote ("), which is used if
	// both Quote and ForceZeroQuote are unset.
	// To specify that no character should be interpreted as a quotation
	// character, set ForceZeroQuote to true.
	// Only used when reading data.
	Quote          string
	ForceZeroQuote bool
}

// quote returns the CSV quote character, or nil if unset.
func (fc *FileConfig) quote() *string {
	if fc.ForceZeroQuote {
		quote := ""
		return &quote
	}
	if fc.Quote == "" {
		return nil
	}
	return &fc.Quote
}

func (fc *FileConfig) populateLoadConfig(conf *bq.JobConfigurationLoad) {
	conf.SkipLeadingRows = fc.SkipLeadingRows
	conf.SourceFormat = string(fc.SourceFormat)
	conf.Autodetect = fc.AutoDetect
	conf.AllowJaggedRows = fc.AllowJaggedRows
	conf.AllowQuotedNewlines = fc.AllowQuotedNewlines
	conf.Encoding = string(fc.Encoding)
	conf.FieldDelimiter = fc.FieldDelimiter
	conf.IgnoreUnknownValues = fc.IgnoreUnknownValues
	conf.MaxBadRecords = fc.MaxBadRecords
	if fc.Schema != nil {
		conf.Schema = fc.Schema.asTableSchema()
	}
	conf.Quote = fc.quote()
}

func (fc *FileConfig) populateExternalDataConfig(conf *bq.ExternalDataConfiguration) {
	format := fc.SourceFormat
	if format == "" {
		// Format must be explicitly set for external data sources.
		format = CSV
	}
	// TODO(jba): support AutoDetect.
	conf.IgnoreUnknownValues = fc.IgnoreUnknownValues
	conf.MaxBadRecords = fc.MaxBadRecords
	conf.SourceFormat = string(format)
	if fc.Schema != nil {
		conf.Schema = fc.Schema.asTableSchema()
	}
	if format == CSV {
		conf.CsvOptions = &bq.CsvOptions{
			AllowJaggedRows:     fc.AllowJaggedRows,
			AllowQuotedNewlines: fc.AllowQuotedNewlines,
			Encoding:            string(fc.Encoding),
			FieldDelimiter:      fc.FieldDelimiter,
			SkipLeadingRows:     fc.SkipLeadingRows,
			Quote:               fc.quote(),
		}
	}
}

// DataFormat describes the format of BigQuery table data.
type DataFormat string

// Constants describing the format of BigQuery table data.
const (
	CSV             DataFormat = "CSV"
	Avro            DataFormat = "AVRO"
	JSON            DataFormat = "NEWLINE_DELIMITED_JSON"
	DatastoreBackup DataFormat = "DATASTORE_BACKUP"
)

// Encoding specifies the character encoding of data to be loaded into BigQuery.
// See https://cloud.google.com/bigquery/docs/reference/v2/jobs#configuration.load.encoding
// for more details about how this is used.
type Encoding string

const (
	UTF_8      Encoding = "UTF-8"
	ISO_8859_1 Encoding = "ISO-8859-1"
)
<|endoftext|>


<commit_before>package models

import (
	. "github.com/onsi/gomega"
	"testing"
	"encoding/json"
)

func TestConvertJsonStringToResponseDelayConfig(t *testing.T) {
	RegisterTestingT(t)

	jsonConf := `
	{
		"data": [{
				"urlPattern": ".",
				"delay": 1
			}]
	}`
	var responseDelayJson ResponseDelayJson
	json.Unmarshal([]byte(jsonConf), &responseDelayJson)
	err := ValidateResponseDelayJson(responseDelayJson)
	Expect(err).To(BeNil())
}

func TestErrorIfHostPatternNotSet(t *testing.T) {
	RegisterTestingT(t)

	jsonConf := `
	{
		"data": [{
				"delay": 2
			}]
	}`
	var responseDelayJson ResponseDelayJson
	json.Unmarshal([]byte(jsonConf), &responseDelayJson)
	err := ValidateResponseDelayJson(responseDelayJson)
	Expect(err).To(Not(BeNil()))
}

func TestErrprIfDelayNotSet(t *testing.T) {
	RegisterTestingT(t)

	jsonConf := `
	{
		"data": [{
				"urlPattern": "."
			}]
	}`
	var responseDelayJson ResponseDelayJson
	json.Unmarshal([]byte(jsonConf), &responseDelayJson)
	err := ValidateResponseDelayJson(responseDelayJson)
	Expect(err).To(Not(BeNil()))
}

func TestHostPatternMustBeAValidRegexPattern(t *testing.T) {
	RegisterTestingT(t)

	jsonConf := `
	{
		"data": [{
				"urlPattern": "*",
				"delay": 1
			}]
	}`
	var responseDelayJson ResponseDelayJson
	json.Unmarshal([]byte(jsonConf), &responseDelayJson)
	err := ValidateResponseDelayJson(responseDelayJson)
	Expect(err).To(Not(BeNil()))
}

func TestErrorIfHostPatternUsed(t *testing.T) {
	RegisterTestingT(t)

	jsonConf := `
	{
		"data": [{
				"hostPattern": ".",
				"delay": 1
			}]
	}`
	var responseDelayJson ResponseDelayJson
	json.Unmarshal([]byte(jsonConf), &responseDelayJson)
	err := ValidateResponseDelayJson(responseDelayJson)
	Expect(err).To(Not(BeNil()))
}

func TestGetDelayWithRegexMatch(t *testing.T) {
	RegisterTestingT(t)

	delay := ResponseDelay{
		UrlPattern: "example(.+)",
		Delay:       100,
	}
	delays := ResponseDelayList{delay}

	delayMatch := delays.GetDelay("delayexample.com", "method-dummy")
	Expect(*delayMatch).To(Equal(delay))

	var nilDelay *ResponseDelay
	delayMatch = delays.GetDelay("nodelay.com", "method-dummy")
	Expect(delayMatch).To(Equal(nilDelay))
}

func TestMultipleMatchingDelaysReturnsTheFirst(t *testing.T) {
	RegisterTestingT(t)

	delayOne := ResponseDelay{
		UrlPattern: "example.com",
		Delay:       100,
	}
	delayTwo := ResponseDelay{
		UrlPattern: "example",
		Delay:       100,
	}
	delays := ResponseDelayList{delayOne, delayTwo}

	delayMatch := delays.GetDelay("delayexample.com", "method-dummy")
	Expect(*delayMatch).To(Equal(delayOne))
}

func TestNoMatchIfMethodsDontMatch(t *testing.T) {
	RegisterTestingT(t)

	delay := ResponseDelay{
		UrlPattern: "example.com",
		Delay:       100,
		HttpMethod: "PURPLE",
	}
	delays := ResponseDelayList{delay}

	var nilDelay *ResponseDelay
	delayMatch := delays.GetDelay("delayexample.com", "GET")
	Expect(delayMatch).To(Equal(nilDelay))
	Expect(delayMatch).To(BeNil())
}

func TestReturnMatchIfMethodsMatch(t *testing.T) {
	RegisterTestingT(t)

	delay := ResponseDelay{
		UrlPattern: "example.com",
		Delay:       100,
		HttpMethod: "GET",
	}
	delays := ResponseDelayList{delay}

	delayMatch := delays.GetDelay("delayexample.com", "GET")
	Expect(*delayMatch).To(Equal(delay))
}

func TestIfDelayMethodBlankThenMatchesAnyMethod(t *testing.T) {
	RegisterTestingT(t)

	delay := ResponseDelay{
		UrlPattern: "example(.+)",
		Delay:       100,
	}
	delays := ResponseDelayList{delay}

	delayMatch := delays.GetDelay("delayexample.com", "method-dummy")
	Expect(*delayMatch).To(Equal(delay))
}<commit_msg>Removed some nil structs from delay_test<commit_after>package models

import (
	. "github.com/onsi/gomega"
	"testing"
	"encoding/json"
)

func TestConvertJsonStringToResponseDelayConfig(t *testing.T) {
	RegisterTestingT(t)

	jsonConf := `
	{
		"data": [{
				"urlPattern": ".",
				"delay": 1
			}]
	}`
	var responseDelayJson ResponseDelayJson
	json.Unmarshal([]byte(jsonConf), &responseDelayJson)
	err := ValidateResponseDelayJson(responseDelayJson)
	Expect(err).To(BeNil())
}

func TestErrorIfHostPatternNotSet(t *testing.T) {
	RegisterTestingT(t)

	jsonConf := `
	{
		"data": [{
				"delay": 2
			}]
	}`
	var responseDelayJson ResponseDelayJson
	json.Unmarshal([]byte(jsonConf), &responseDelayJson)
	err := ValidateResponseDelayJson(responseDelayJson)
	Expect(err).To(Not(BeNil()))
}

func TestErrprIfDelayNotSet(t *testing.T) {
	RegisterTestingT(t)

	jsonConf := `
	{
		"data": [{
				"urlPattern": "."
			}]
	}`
	var responseDelayJson ResponseDelayJson
	json.Unmarshal([]byte(jsonConf), &responseDelayJson)
	err := ValidateResponseDelayJson(responseDelayJson)
	Expect(err).To(Not(BeNil()))
}

func TestHostPatternMustBeAValidRegexPattern(t *testing.T) {
	RegisterTestingT(t)

	jsonConf := `
	{
		"data": [{
				"urlPattern": "*",
				"delay": 1
			}]
	}`
	var responseDelayJson ResponseDelayJson
	json.Unmarshal([]byte(jsonConf), &responseDelayJson)
	err := ValidateResponseDelayJson(responseDelayJson)
	Expect(err).To(Not(BeNil()))
}

func TestErrorIfHostPatternUsed(t *testing.T) {
	RegisterTestingT(t)

	jsonConf := `
	{
		"data": [{
				"hostPattern": ".",
				"delay": 1
			}]
	}`
	var responseDelayJson ResponseDelayJson
	json.Unmarshal([]byte(jsonConf), &responseDelayJson)
	err := ValidateResponseDelayJson(responseDelayJson)
	Expect(err).To(Not(BeNil()))
}

func TestGetDelayWithRegexMatch(t *testing.T) {
	RegisterTestingT(t)

	delay := ResponseDelay{
		UrlPattern: "example(.+)",
		Delay:       100,
	}
	delays := ResponseDelayList{delay}

	delayMatch := delays.GetDelay("delayexample.com", "method-dummy")
	Expect(*delayMatch).To(Equal(delay))

	delayMatch = delays.GetDelay("nodelay.com", "method-dummy")
	Expect(delayMatch).To(BeNil())
}

func TestMultipleMatchingDelaysReturnsTheFirst(t *testing.T) {
	RegisterTestingT(t)

	delayOne := ResponseDelay{
		UrlPattern: "example.com",
		Delay:       100,
	}
	delayTwo := ResponseDelay{
		UrlPattern: "example",
		Delay:       100,
	}
	delays := ResponseDelayList{delayOne, delayTwo}

	delayMatch := delays.GetDelay("delayexample.com", "method-dummy")
	Expect(*delayMatch).To(Equal(delayOne))
}

func TestNoMatchIfMethodsDontMatch(t *testing.T) {
	RegisterTestingT(t)

	delay := ResponseDelay{
		UrlPattern: "example.com",
		Delay:       100,
		HttpMethod: "PURPLE",
	}
	delays := ResponseDelayList{delay}

	delayMatch := delays.GetDelay("delayexample.com", "GET")
	Expect(delayMatch).To(BeNil())
}

func TestReturnMatchIfMethodsMatch(t *testing.T) {
	RegisterTestingT(t)

	delay := ResponseDelay{
		UrlPattern: "example.com",
		Delay:       100,
		HttpMethod: "GET",
	}
	delays := ResponseDelayList{delay}

	delayMatch := delays.GetDelay("delayexample.com", "GET")
	Expect(*delayMatch).To(Equal(delay))
}

func TestIfDelayMethodBlankThenMatchesAnyMethod(t *testing.T) {
	RegisterTestingT(t)

	delay := ResponseDelay{
		UrlPattern: "example(.+)",
		Delay:       100,
	}
	delays := ResponseDelayList{delay}

	delayMatch := delays.GetDelay("delayexample.com", "method-dummy")
	Expect(*delayMatch).To(Equal(delay))
}<|endoftext|>


<commit_before>package trello

import (
	"github.com/joonasmyhrberg/go-trello"
)

type TrelloConfig struct {
	Key              string
	Token            string
	User             string
	Board            string
	StartingListName string
	FinishedListName string
}

var trelloClient *trello.Client
var trelloConfig TrelloConfig

func Authenticate(config TrelloConfig) error {

	trelloConfig = config

	trello, err := trello.NewAuthClient(config.Key, &config.Token)
	trelloClient = trello

	return err
}
<commit_msg>Add NotifyChannel to TrelloConfig<commit_after>package trello

import (
	"github.com/joonasmyhrberg/go-trello"
)

type TrelloConfig struct {
	Key              string
	Token            string
	User             string
	Board            string
	StartingListName string
	FinishedListName string
	NotifyChannel    string
}

var trelloClient *trello.Client
var trelloConfig TrelloConfig

func Authenticate(config TrelloConfig) error {

	trelloConfig = config

	trello, err := trello.NewAuthClient(config.Key, &config.Token)
	trelloClient = trello

	return err
}
<|endoftext|>


<commit_before>package trello

import (
	"github.com/joonasmyhrberg/go-trello"
)

type TrelloConfig struct {
	Key   string
	Token string
	User  string
}

var trelloClient *trello.Client
var trelloConfig TrelloConfig

func Authenticate(config TrelloConfig) error {

	trelloConfig = config

	trello, err := trello.NewAuthClient(config.Key, &config.Token)
	trelloClient = trello

	return err
}
<commit_msg>Add ActionHandler to trello config<commit_after>package trello

import (
	"github.com/joonasmyhrberg/go-trello"
)

type TrelloConfig struct {
	Key           string
	Token         string
	User          string
	ActionHandler func(action trello.Action)
}

var trelloClient *trello.Client
var trelloConfig TrelloConfig

func Authenticate(config TrelloConfig) error {

	trelloConfig = config

	trello, err := trello.NewAuthClient(config.Key, &config.Token)
	trelloClient = trello

	return err
}
<|endoftext|>


<commit_before>package libhoney

import (
	"fmt"
	"net/http"
	"path/filepath"
	"reflect"
	"runtime"
	"strings"
	"testing"
	"time"
)

func testOK(t testing.TB, err error) {
	if err != nil {
		_, file, line, _ := runtime.Caller(1)
		t.Fatalf("%s:%d: unexpected error: %s", filepath.Base(file), line, err.Error())
	}
}
func testErr(t testing.TB, err error) {
	if err == nil {
		_, file, line, _ := runtime.Caller(1)
		t.Fatalf("%s:%d: error expected!", filepath.Base(file), line)
	}
}

func testEquals(t testing.TB, actual, expected interface{}, msg ...string) {
	if !reflect.DeepEqual(actual, expected) {
		testCommonErr(t, actual, expected, msg)
	}
}

func testNotEquals(t testing.TB, actual, expected interface{}, msg ...string) {
	if reflect.DeepEqual(actual, expected) {
		testCommonErr(t, actual, expected, msg)
	}
}

func testCommonErr(t testing.TB, actual, expected interface{}, msg []string) {
	message := strings.Join(msg, ", ")
	_, file, line, _ := runtime.Caller(2)

	t.Errorf(
		"%s:%d: %s -- actual(%T): %v, expected(%T): %v",
		filepath.Base(file),
		line,
		message,
		testDeref(actual),
		testDeref(actual),
		testDeref(expected),
		testDeref(expected),
	)
}

func testGetResponse(t testing.TB, ch chan Response) Response {
	_, file, line, _ := runtime.Caller(2)
	var resp Response
	select {
	case resp = <-ch:
	case <-time.After(5 * time.Millisecond): // block on read but prevent deadlocking tests
		t.Errorf("%s:%d: expected response on channel and timed out waiting for it!", filepath.Base(file), line)
	}
	return resp
}

func testIsPlaceholderResponse(t testing.TB, actual Response, msg ...string) {
	if actual.StatusCode != http.StatusTeapot {
		message := strings.Join(msg, ", ")
		_, file, line, _ := runtime.Caller(1)
		t.Errorf(
			"%s:%d placeholder expected -- %s",
			filepath.Base(file),
			line,
			message,
		)
	}
}

func testDeref(v interface{}) interface{} {
	switch t := v.(type) {
	case *string:
		return fmt.Sprintf("*(%v)", *t)
	case *int64:
		return fmt.Sprintf("*(%v)", *t)
	case *float64:
		return fmt.Sprintf("*(%v)", *t)
	case *bool:
		return fmt.Sprintf("*(%v)", *t)
	default:
		return v
	}
}

// for easy time manipulation during tests
type fakeNower struct {
	iter int
}

// Now() supports changing/increasing the returned Now() based on the number of
// times it's called in succession
func (f *fakeNower) Now() time.Time {
	now := time.Unix(1277132645, 0).Add(time.Second * 10 * time.Duration(f.iter))
	f.iter++
	return now
}
<commit_msg>Bump testGetResponse timeout for tests<commit_after>package libhoney

import (
	"fmt"
	"net/http"
	"path/filepath"
	"reflect"
	"runtime"
	"strings"
	"testing"
	"time"
)

func testOK(t testing.TB, err error) {
	if err != nil {
		_, file, line, _ := runtime.Caller(1)
		t.Fatalf("%s:%d: unexpected error: %s", filepath.Base(file), line, err.Error())
	}
}
func testErr(t testing.TB, err error) {
	if err == nil {
		_, file, line, _ := runtime.Caller(1)
		t.Fatalf("%s:%d: error expected!", filepath.Base(file), line)
	}
}

func testEquals(t testing.TB, actual, expected interface{}, msg ...string) {
	if !reflect.DeepEqual(actual, expected) {
		testCommonErr(t, actual, expected, msg)
	}
}

func testNotEquals(t testing.TB, actual, expected interface{}, msg ...string) {
	if reflect.DeepEqual(actual, expected) {
		testCommonErr(t, actual, expected, msg)
	}
}

func testCommonErr(t testing.TB, actual, expected interface{}, msg []string) {
	message := strings.Join(msg, ", ")
	_, file, line, _ := runtime.Caller(2)

	t.Errorf(
		"%s:%d: %s -- actual(%T): %v, expected(%T): %v",
		filepath.Base(file),
		line,
		message,
		testDeref(actual),
		testDeref(actual),
		testDeref(expected),
		testDeref(expected),
	)
}

func testGetResponse(t testing.TB, ch chan Response) Response {
	_, file, line, _ := runtime.Caller(2)
	var resp Response
	select {
	case resp = <-ch:
	case <-time.After(50 * time.Millisecond): // block on read but prevent deadlocking tests
		t.Errorf("%s:%d: expected response on channel and timed out waiting for it!", filepath.Base(file), line)
	}
	return resp
}

func testIsPlaceholderResponse(t testing.TB, actual Response, msg ...string) {
	if actual.StatusCode != http.StatusTeapot {
		message := strings.Join(msg, ", ")
		_, file, line, _ := runtime.Caller(1)
		t.Errorf(
			"%s:%d placeholder expected -- %s",
			filepath.Base(file),
			line,
			message,
		)
	}
}

func testDeref(v interface{}) interface{} {
	switch t := v.(type) {
	case *string:
		return fmt.Sprintf("*(%v)", *t)
	case *int64:
		return fmt.Sprintf("*(%v)", *t)
	case *float64:
		return fmt.Sprintf("*(%v)", *t)
	case *bool:
		return fmt.Sprintf("*(%v)", *t)
	default:
		return v
	}
}

// for easy time manipulation during tests
type fakeNower struct {
	iter int
}

// Now() supports changing/increasing the returned Now() based on the number of
// times it's called in succession
func (f *fakeNower) Now() time.Time {
	now := time.Unix(1277132645, 0).Add(time.Second * 10 * time.Duration(f.iter))
	f.iter++
	return now
}
<|endoftext|>


<commit_before>package ttml

import (
	"encoding/xml"
	"fmt"
	"io/ioutil"
	"os"
)

//LoadTtml loads the TTML file from a given filepath
func LoadTtml(v *Tt, filepath string) {
	f, err := os.Open(filepath)
	if err != nil {
		fmt.Println("Cannot read file", filepath)
		os.Exit(1)
	}
	bytef, berr := ioutil.ReadAll(f)
	if berr != nil {
		fmt.Println("error decoding")
	}
	xml.Unmarshal(bytef, &v)
}

func ParseTtml(filename string) *Tt {
	v := &Tt{}
	LoadTtml(v, filename)
	return v
}
<commit_msg>Some more documentation<commit_after>package ttml

import (
	"encoding/xml"
	"fmt"
	"io/ioutil"
	"os"
)

//LoadTtml loads the TTML file from a given filepath
func LoadTtml(v *Tt, filepath string) {
	f, err := os.Open(filepath)
	if err != nil {
		fmt.Println("Cannot read file", filepath)
		os.Exit(1)
	}
	bytef, berr := ioutil.ReadAll(f)
	if berr != nil {
		fmt.Println("error decoding")
	}
	xml.Unmarshal(bytef, &v)
}

//Generic loader for TTML files
func ParseTtml(filename string) *Tt {
	v := &Tt{}
	LoadTtml(v, filename)
	return v
}
<|endoftext|>


<commit_before>package collector

import (
	"encoding/json"
	"strings"
	"time"

	"github.com/openconfig/gnmi/proto/gnmi"
)

type NotificationRspMsg struct {
	Meta             map[string]interface{} `json:"meta,omitempty"`
	Source           string                 `json:"source,omitempty"`
	SystemName       string                 `json:"system-name,omitempty"`
	SubscriptionName string                 `json:"subscription-name,omitempty"`
	Timestamp        int64                  `json:"timestamp,omitempty"`
	Time             *time.Time             `json:"time,omitempty"`
	Prefix           string                 `json:"prefix,omitempty"`
	Target           string                 `json:"target,omitempty"`
	Updates          []update               `json:"updates,omitempty"`
	Deletes          []string               `json:"deletes,omitempty"`
}
type update struct {
	Path   string
	Values map[string]interface{} `json:"values,omitempty"`
}
type capRequest struct {
	Extentions []string `json:"extentions,omitempty"`
}
type capResponse struct {
	Version         string   `json:"version,omitempty"`
	SupportedModels []model  `json:"supported-models,omitempty"`
	Encodings       []string `json:"encodings,omitempty"`
}
type model struct {
	Name         string `json:"name,omitempty"`
	Organization string `json:"organization,omitempty"`
	Version      string `json:"version,omitempty"`
}

type getRqMsg struct {
	Prefix   string   `json:"prefix,omitempty"`
	Paths    []string `json:"paths,omitempty"`
	Encoding string   `json:"encoding,omitempty"`
	DataType string   `json:"data-type,omitempty"`
	Models   []model  `json:"models,omitempty"`
}

type setRspMsg struct {
	Source    string            `json:"source,omitempty"`
	Timestamp int64             `json:"timestamp,omitempty"`
	Time      time.Time         `json:"time,omitempty"`
	Prefix    string            `json:"prefix,omitempty"`
	Results   []updateResultMsg `json:"results,omitempty"`
}

type updateResultMsg struct {
	Operation string `json:"operation,omitempty"`
	Path      string `json:"path,omitempty"`
}

type setReqMsg struct {
	Prefix  string      `json:"prefix,omitempty"`
	Delete  []string    `json:"delete,omitempty"`
	Replace []updateMsg `json:"replace,omitempty"`
	Update  []updateMsg `json:"update,omitempty"`
	// extension is not implemented
}

type updateMsg struct {
	Path string `json:"path,omitempty"`
	Val  string `json:"val,omitempty"`
}

type subscribeReq struct {
	Subscribe subscribe         `json:"subscribe,omitempty"`
	Poll      *poll             `json:"poll,omitempty"`
	Aliases   map[string]string `json:"aliases,omitempty"`
}
type poll struct{}
type subscribe struct {
	Target           string         `json:"target,omitempty"`
	Prefix           string         `json:"prefix,omitempty"`
	Subscriptions    []subscription `json:"subscriptions,omitempty"`
	UseAliases       bool           `json:"use-aliases,omitempty"`
	Qos              uint32         `json:"qos,omitempty"`
	Mode             string         `json:"mode,omitempty"`
	AllowAggregation bool           `json:"allow-aggregation,omitempty"`
	UseModels        []model        `json:"use-models,omitempty"`
	Encoding         string         `json:"encoding,omitempty"`
	UpdatesOnly      bool           `json:"updates-only,omitempty"`
}
type subscription struct {
	Path              string `json:"path,omitempty"`
	Mode              string `json:"mode,omitempty"`
	SampleInterval    uint64 `json:"sample-interval,omitempty"`
	SuppressRedundant bool   `json:"suppress-redundant,omitempty"`
	HeartbeatInterval uint64 `json:"heartbeat-interval,omitempty"`
}

func gnmiPathToXPath(p *gnmi.Path) string {
	if p == nil {
		return ""
	}
	sb := strings.Builder{}
	if p.Origin != "" {
		sb.WriteString(p.Origin)
		sb.WriteString(":")
	}
	elems := p.GetElem()
	numElems := len(elems)
	if numElems > 0 {
		sb.WriteString("/")
	}
	for i, pe := range elems {
		sb.WriteString(pe.GetName())
		for k, v := range pe.GetKey() {
			sb.WriteString("[")
			sb.WriteString(k)
			sb.WriteString("=")
			sb.WriteString(v)
			sb.WriteString("]")
		}
		if i+1 != numElems {
			sb.WriteString("/")
		}
	}
	return sb.String()
}

func getValue(updValue *gnmi.TypedValue) (interface{}, error) {
	if updValue == nil {
		return nil, nil
	}
	var value interface{}
	var jsondata []byte
	switch updValue.Value.(type) {
	case *gnmi.TypedValue_AsciiVal:
		value = updValue.GetAsciiVal()
	case *gnmi.TypedValue_BoolVal:
		value = updValue.GetBoolVal()
	case *gnmi.TypedValue_BytesVal:
		value = updValue.GetBytesVal()
	case *gnmi.TypedValue_DecimalVal:
		value = updValue.GetDecimalVal()
	case *gnmi.TypedValue_FloatVal:
		value = updValue.GetFloatVal()
	case *gnmi.TypedValue_IntVal:
		value = updValue.GetIntVal()
	case *gnmi.TypedValue_StringVal:
		value = updValue.GetStringVal()
	case *gnmi.TypedValue_UintVal:
		value = updValue.GetUintVal()
	case *gnmi.TypedValue_JsonIetfVal:
		jsondata = updValue.GetJsonIetfVal()
	case *gnmi.TypedValue_JsonVal:
		jsondata = updValue.GetJsonVal()
	case *gnmi.TypedValue_LeaflistVal:
		value = updValue.GetLeaflistVal()
	case *gnmi.TypedValue_ProtoBytes:
		value = updValue.GetProtoBytes()
	case *gnmi.TypedValue_AnyVal:
		value = updValue.GetAnyVal()
	}
	if value == nil {
		err := json.Unmarshal(jsondata, &value)
		if err != nil {
			return nil, err
		}
	}
	return value, nil
}
<commit_msg>remove '/' as prefix of xpaths<commit_after>package collector

import (
	"encoding/json"
	"strings"
	"time"

	"github.com/openconfig/gnmi/proto/gnmi"
)

type NotificationRspMsg struct {
	Meta             map[string]interface{} `json:"meta,omitempty"`
	Source           string                 `json:"source,omitempty"`
	SystemName       string                 `json:"system-name,omitempty"`
	SubscriptionName string                 `json:"subscription-name,omitempty"`
	Timestamp        int64                  `json:"timestamp,omitempty"`
	Time             *time.Time             `json:"time,omitempty"`
	Prefix           string                 `json:"prefix,omitempty"`
	Target           string                 `json:"target,omitempty"`
	Updates          []update               `json:"updates,omitempty"`
	Deletes          []string               `json:"deletes,omitempty"`
}
type update struct {
	Path   string
	Values map[string]interface{} `json:"values,omitempty"`
}
type capRequest struct {
	Extentions []string `json:"extentions,omitempty"`
}
type capResponse struct {
	Version         string   `json:"version,omitempty"`
	SupportedModels []model  `json:"supported-models,omitempty"`
	Encodings       []string `json:"encodings,omitempty"`
}
type model struct {
	Name         string `json:"name,omitempty"`
	Organization string `json:"organization,omitempty"`
	Version      string `json:"version,omitempty"`
}

type getRqMsg struct {
	Prefix   string   `json:"prefix,omitempty"`
	Paths    []string `json:"paths,omitempty"`
	Encoding string   `json:"encoding,omitempty"`
	DataType string   `json:"data-type,omitempty"`
	Models   []model  `json:"models,omitempty"`
}

type setRspMsg struct {
	Source    string            `json:"source,omitempty"`
	Timestamp int64             `json:"timestamp,omitempty"`
	Time      time.Time         `json:"time,omitempty"`
	Prefix    string            `json:"prefix,omitempty"`
	Results   []updateResultMsg `json:"results,omitempty"`
}

type updateResultMsg struct {
	Operation string `json:"operation,omitempty"`
	Path      string `json:"path,omitempty"`
}

type setReqMsg struct {
	Prefix  string      `json:"prefix,omitempty"`
	Delete  []string    `json:"delete,omitempty"`
	Replace []updateMsg `json:"replace,omitempty"`
	Update  []updateMsg `json:"update,omitempty"`
	// extension is not implemented
}

type updateMsg struct {
	Path string `json:"path,omitempty"`
	Val  string `json:"val,omitempty"`
}

type subscribeReq struct {
	Subscribe subscribe         `json:"subscribe,omitempty"`
	Poll      *poll             `json:"poll,omitempty"`
	Aliases   map[string]string `json:"aliases,omitempty"`
}
type poll struct{}
type subscribe struct {
	Target           string         `json:"target,omitempty"`
	Prefix           string         `json:"prefix,omitempty"`
	Subscriptions    []subscription `json:"subscriptions,omitempty"`
	UseAliases       bool           `json:"use-aliases,omitempty"`
	Qos              uint32         `json:"qos,omitempty"`
	Mode             string         `json:"mode,omitempty"`
	AllowAggregation bool           `json:"allow-aggregation,omitempty"`
	UseModels        []model        `json:"use-models,omitempty"`
	Encoding         string         `json:"encoding,omitempty"`
	UpdatesOnly      bool           `json:"updates-only,omitempty"`
}
type subscription struct {
	Path              string `json:"path,omitempty"`
	Mode              string `json:"mode,omitempty"`
	SampleInterval    uint64 `json:"sample-interval,omitempty"`
	SuppressRedundant bool   `json:"suppress-redundant,omitempty"`
	HeartbeatInterval uint64 `json:"heartbeat-interval,omitempty"`
}

func gnmiPathToXPath(p *gnmi.Path) string {
	if p == nil {
		return ""
	}
	sb := strings.Builder{}
	if p.Origin != "" {
		sb.WriteString(p.Origin)
		sb.WriteString(":")
	}
	elems := p.GetElem()
	numElems := len(elems)
	for i, pe := range elems {
		sb.WriteString(pe.GetName())
		for k, v := range pe.GetKey() {
			sb.WriteString("[")
			sb.WriteString(k)
			sb.WriteString("=")
			sb.WriteString(v)
			sb.WriteString("]")
		}
		if i+1 != numElems {
			sb.WriteString("/")
		}
	}
	return sb.String()
}

func getValue(updValue *gnmi.TypedValue) (interface{}, error) {
	if updValue == nil {
		return nil, nil
	}
	var value interface{}
	var jsondata []byte
	switch updValue.Value.(type) {
	case *gnmi.TypedValue_AsciiVal:
		value = updValue.GetAsciiVal()
	case *gnmi.TypedValue_BoolVal:
		value = updValue.GetBoolVal()
	case *gnmi.TypedValue_BytesVal:
		value = updValue.GetBytesVal()
	case *gnmi.TypedValue_DecimalVal:
		value = updValue.GetDecimalVal()
	case *gnmi.TypedValue_FloatVal:
		value = updValue.GetFloatVal()
	case *gnmi.TypedValue_IntVal:
		value = updValue.GetIntVal()
	case *gnmi.TypedValue_StringVal:
		value = updValue.GetStringVal()
	case *gnmi.TypedValue_UintVal:
		value = updValue.GetUintVal()
	case *gnmi.TypedValue_JsonIetfVal:
		jsondata = updValue.GetJsonIetfVal()
	case *gnmi.TypedValue_JsonVal:
		jsondata = updValue.GetJsonVal()
	case *gnmi.TypedValue_LeaflistVal:
		value = updValue.GetLeaflistVal()
	case *gnmi.TypedValue_ProtoBytes:
		value = updValue.GetProtoBytes()
	case *gnmi.TypedValue_AnyVal:
		value = updValue.GetAnyVal()
	}
	if value == nil {
		err := json.Unmarshal(jsondata, &value)
		if err != nil {
			return nil, err
		}
	}
	return value, nil
}
<|endoftext|>


<commit_before>package procfs

import (
	"io/ioutil"
	"os"
	"path"
	"strconv"
	"strings"
)

const procfsdir = "/proc"

type Filler interface {
	Fill()
}

type Lister interface {
	List(string)
}

type Getter interface {
	Get(string)
}

type ProcFS struct {
	Processes map[string]*Process
	Self      string

	Uptime    int
	Idletime  int
}

const (
	PROCFS_PROCESSES = "Processes"
	PROCFS_SELF = "Self"
	PROCFS_UPTIME = "Uptime"
	PROCFS_IDLETIME = "Idletime"
)

func (pfs *ProcFS) Fill() {
	pfs.List(PROCFS_PROCESSES)
	for _, p := range pfs.Processes {
		p.Fill()
	}
	pfs.Get(PROCFS_SELF)

	pfs.Get(PROCFS_UPTIME)
	pfs.Get(PROCFS_IDLETIME)
}

func (pfs *ProcFS) List(k string) {
	switch k {
	case PROCFS_PROCESSES:
		if !exists(procfsdir) {
			return
		}
		pfs.Processes = make(map[string]*Process)
		ds, err := ioutil.ReadDir(procfsdir)
		if err != nil {
			return
		}
		// get all numeric entries
		for _, d := range ds {
			n := d.Name
			id, err := strconv.Atoi(n)
			if isNumeric(n) && err == nil {
				proc := Process{PID: id}
				pfs.Processes[n] = &proc
			}
		}
	}
}

func (pfs *ProcFS) Get(k string) {
	var uf = path.Join(procfsdir, "uptime")
	switch k {
	case PROCFS_SELF:
		var selfdir = path.Join(procfsdir, "self")
		if !exists(selfdir) {
			return
		}
		fi, _ := os.Readlink(selfdir)
		pfs.Self = fi
	case PROCFS_UPTIME:
		str, err := ioutil.ReadFile(uf)
		if err == nil {
			ss := strings.Fields(string(str))
			if len(ss) >= 2 {
				it, _ := strconv.Atof64(ss[0])
				pfs.Uptime = int(it)
			}
		}
	case PROCFS_IDLETIME:
		str, err := ioutil.ReadFile(uf)
		if err == nil {
			ss := strings.Fields(string(str))
			if len(ss) >= 2 {
				it, _ := strconv.Atof64(ss[1])
				pfs.Idletime = int(it)
			}
		}
	}
}

type Process struct {
	PID     int
	Auxv    []byte
	Cmdline []string
	Cwd     string
	Environ map[string]string
	Exe     string
	Fds     map[string]*Fd
	Root    string
	Status  map[string]string
	Threads map[string]*Thread
}
// TODO limits, maps, mem, mountinfo, mounts, mountstats, ns, smaps, stat

const (
	PROCFS_PROC_AUXV = "Process.Auxv"
	PROCFS_PROC_CMDLINE = "Process.Cmdline"
	PROCFS_PROC_CWD = "Process.Cwd"
	PROCFS_PROC_ENVIRON = "Process.Environ"
	PROCFS_PROC_EXE = "Process.Exe"
	PROCFS_PROC_ROOT = "Process.Root"
	PROCFS_PROC_STATUS = "Process.Status"

	PROCFS_PROC_FDS = "Process.Fds"
	PROCFS_PROC_THREADS = "Process.Threads"
)

func (p *Process) Fill() {
	p.Get(PROCFS_PROC_AUXV)
	p.Get(PROCFS_PROC_CMDLINE)
	p.Get(PROCFS_PROC_CWD)
	p.Get(PROCFS_PROC_ENVIRON)
	p.Get(PROCFS_PROC_EXE)
	p.Get(PROCFS_PROC_ROOT)
	p.Get(PROCFS_PROC_STATUS)

	// Fds
	p.List(PROCFS_PROC_FDS)
	for _, f := range p.Fds {
		f.Fill()
	}

	// Threads
	p.List(PROCFS_PROC_THREADS)
	for _, t := range p.Threads {
		t.Fill()
	}
}

func (p *Process) List(k string) {

}

func (p *Process) Get(k string) {
	pdir := path.Join(procfsdir, strconv.Itoa(p.PID))
	switch k {
	case PROCFS_PROC_AUXV:
		p.Auxv, _ = ioutil.ReadFile(path.Join(pdir, "auxv"))
	case PROCFS_PROC_CMDLINE:
		cl, err := ioutil.ReadFile(path.Join(pdir, "cmdline"))
		if err == nil {
			p.Cmdline = splitNull(cl)
		}
	case PROCFS_PROC_CWD:
		p.Cwd, _ = os.Readlink(path.Join(pdir, "cwd"))
	case PROCFS_PROC_ENVIRON:
		envB, err := ioutil.ReadFile(path.Join(pdir, "environ"))
		if err == nil {
			p.Environ = make(map[string]string)
			envS := splitNull(envB)
			for _, s := range envS {
				// split on =
				ss := strings.SplitN(s, "=", 2)
				if len(ss) == 2 {
					p.Environ[ss[0]] = ss[1]
				}
			}
		}
	case PROCFS_PROC_EXE:
		p.Exe, _ = os.Readlink(path.Join(pdir, "exe"))
	case PROCFS_PROC_ROOT:
		p.Root, _ = os.Readlink(path.Join(pdir, "root"))
	case PROCFS_PROC_STATUS:
		statLines, err := ioutil.ReadFile(path.Join(pdir, "status"))
		if err == nil {
			p.Status = make(map[string]string)
			statS := strings.Split(string(statLines), "\n")
			for _, s := range statS {
				ss := strings.SplitN(s, ":", 2)
				if len(ss) == 2 {
					p.Status[ss[0]] = strings.TrimSpace(ss[1])
				}
			}
		}
	}
}

type Fd struct {
	Path  string
	Pos   int
	Flags int
}

const (
	PROCFS_PROC_FD_PATH = "Process.Fd.Path"
	PROCFS_PROC_FD_POS = "Process.Fd.Pos"
	PROCFS_PROC_FD_FLAGS = "Process.Fd.Flags"
)

func (f *Fd) Fill() {
	f.Get(PROCFS_PROC_FD_PATH)
	f.Get(PROCFS_PROC_FD_POS)
	f.Get(PROCFS_PROC_FD_FLAGS)
}

func (f *Fd) Get(k string) {
	switch k {

	}
}

type Thread struct {
	// TODO
}

func (t *Thread) Fill() {

}

func (t *Thread) Get(k string) {

}
<commit_msg>Attempting to read mounts file<commit_after>package procfs

import (
	"io/ioutil"
	"os"
	"path"
	"strconv"
	"strings"
)

const procfsdir = "/proc"

type Filler interface {
	Fill()
}

type Lister interface {
	List(string)
}

type Getter interface {
	Get(string)
}

type ProcFS struct {
	Processes map[string]*Process
	Self      string
	Mounts    []Mount

	Uptime    int
	Idletime  int
}

type Mount struct {
	Device     string
	MountPoint string
	Type       string
	Options    []string
}

func getMounts(f string) []Mount {
	var ms = []Mount{}
	cont, err := ioutil.ReadFile(f)
	if err == nil {
		lines := strings.Split(string(cont), "\n")
		for _, l := range lines {
			fs := strings.Fields(l)
			optStr := fs[3]
			opts := (strings.Split(optStr, ","))
			m := Mount{
				Device: fs[0],
				MountPoint: fs[1],
				Type: fs[2],
				Options: opts,
			}
			ms = append(ms, m)
		}
	}
	return ms
}

const (
	PROCFS_PROCESSES = "Processes"
	PROCFS_SELF = "Self"
	PROCFS_UPTIME = "Uptime"
	PROCFS_IDLETIME = "Idletime"
	PROCFS_MOUNTS = "Mounts"
)

func (pfs *ProcFS) Fill() {
	pfs.List(PROCFS_PROCESSES)
	for _, p := range pfs.Processes {
		p.Fill()
	}
	pfs.Get(PROCFS_SELF)

	pfs.Get(PROCFS_UPTIME)
	pfs.Get(PROCFS_IDLETIME)
	pfs.Get(PROCFS_MOUNTS)
}

func (pfs *ProcFS) List(k string) {
	switch k {
	case PROCFS_PROCESSES:
		if !exists(procfsdir) {
			return
		}
		pfs.Processes = make(map[string]*Process)
		ds, err := ioutil.ReadDir(procfsdir)
		if err != nil {
			return
		}
		// get all numeric entries
		for _, d := range ds {
			n := d.Name
			id, err := strconv.Atoi(n)
			if isNumeric(n) && err == nil {
				proc := Process{PID: id}
				pfs.Processes[n] = &proc
			}
		}
	}
}

func (pfs *ProcFS) Get(k string) {
	var uf = path.Join(procfsdir, "uptime")
	switch k {
	case PROCFS_SELF:
		var selfdir = path.Join(procfsdir, "self")
		if !exists(selfdir) {
			return
		}
		fi, _ := os.Readlink(selfdir)
		pfs.Self = fi
	case PROCFS_UPTIME:
		str, err := ioutil.ReadFile(uf)
		if err == nil {
			ss := strings.Fields(string(str))
			if len(ss) >= 2 {
				it, _ := strconv.Atof64(ss[0])
				pfs.Uptime = int(it)
			}
		}
	case PROCFS_IDLETIME:
		str, err := ioutil.ReadFile(uf)
		if err == nil {
			ss := strings.Fields(string(str))
			if len(ss) >= 2 {
				it, _ := strconv.Atof64(ss[1])
				pfs.Idletime = int(it)
			}
		}
	case PROCFS_MOUNTS:
		pfs.Mounts = getMounts(path.Join(procfsdir, "mounts"))
	}
}

type Process struct {
	PID     int
	Auxv    []byte
	Cmdline []string
	Cwd     string
	Environ map[string]string
	Exe     string
	Fds     map[string]*Fd
	Root    string
	Status  map[string]string
	Threads map[string]*Thread
	Mounts  []Mount
}
// TODO limits, maps, mem, mountinfo, mounts, mountstats, ns, smaps, stat

const (
	PROCFS_PROC_AUXV = "Process.Auxv"
	PROCFS_PROC_CMDLINE = "Process.Cmdline"
	PROCFS_PROC_CWD = "Process.Cwd"
	PROCFS_PROC_ENVIRON = "Process.Environ"
	PROCFS_PROC_EXE = "Process.Exe"
	PROCFS_PROC_ROOT = "Process.Root"
	PROCFS_PROC_STATUS = "Process.Status"
	PROCFS_PROC_MOUNTS = "Process.Mounts"

	PROCFS_PROC_FDS = "Process.Fds"
	PROCFS_PROC_THREADS = "Process.Threads"
)

func (p *Process) Fill() {
	p.Get(PROCFS_PROC_AUXV)
	p.Get(PROCFS_PROC_CMDLINE)
	p.Get(PROCFS_PROC_CWD)
	p.Get(PROCFS_PROC_ENVIRON)
	p.Get(PROCFS_PROC_EXE)
	p.Get(PROCFS_PROC_ROOT)
	p.Get(PROCFS_PROC_STATUS)
	p.Get(PROCFS_PROC_MOUNTS)

	// Fds
	p.List(PROCFS_PROC_FDS)
	for _, f := range p.Fds {
		f.Fill()
	}

	// Threads
	p.List(PROCFS_PROC_THREADS)
	for _, t := range p.Threads {
		t.Fill()
	}
}

func (p *Process) List(k string) {

}

func (p *Process) Get(k string) {
	pdir := path.Join(procfsdir, strconv.Itoa(p.PID))
	switch k {
	case PROCFS_PROC_AUXV:
		p.Auxv, _ = ioutil.ReadFile(path.Join(pdir, "auxv"))
	case PROCFS_PROC_CMDLINE:
		cl, err := ioutil.ReadFile(path.Join(pdir, "cmdline"))
		if err == nil {
			p.Cmdline = splitNull(cl)
		}
	case PROCFS_PROC_CWD:
		p.Cwd, _ = os.Readlink(path.Join(pdir, "cwd"))
	case PROCFS_PROC_ENVIRON:
		envB, err := ioutil.ReadFile(path.Join(pdir, "environ"))
		if err == nil {
			p.Environ = make(map[string]string)
			envS := splitNull(envB)
			for _, s := range envS {
				// split on =
				ss := strings.SplitN(s, "=", 2)
				if len(ss) == 2 {
					p.Environ[ss[0]] = ss[1]
				}
			}
		}
	case PROCFS_PROC_EXE:
		p.Exe, _ = os.Readlink(path.Join(pdir, "exe"))
	case PROCFS_PROC_ROOT:
		p.Root, _ = os.Readlink(path.Join(pdir, "root"))
	case PROCFS_PROC_STATUS:
		statLines, err := ioutil.ReadFile(path.Join(pdir, "status"))
		if err == nil {
			p.Status = make(map[string]string)
			statS := strings.Split(string(statLines), "\n")
			for _, s := range statS {
				ss := strings.SplitN(s, ":", 2)
				if len(ss) == 2 {
					p.Status[ss[0]] = strings.TrimSpace(ss[1])
				}
			}
		}
	case PROCFS_PROC_MOUNTS:
		p.Mounts = getMounts(path.Join(pdir, "mounts"))
	}
}

type Fd struct {
	Path  string
	Pos   int
	Flags int
}

const (
	PROCFS_PROC_FD_PATH = "Process.Fd.Path"
	PROCFS_PROC_FD_POS = "Process.Fd.Pos"
	PROCFS_PROC_FD_FLAGS = "Process.Fd.Flags"
)

func (f *Fd) Fill() {
	f.Get(PROCFS_PROC_FD_PATH)
	f.Get(PROCFS_PROC_FD_POS)
	f.Get(PROCFS_PROC_FD_FLAGS)
}

func (f *Fd) Get(k string) {
	switch k {

	}
}

type Thread struct {
	// TODO
}

func (t *Thread) Fill() {

}

func (t *Thread) Get(k string) {

}
<|endoftext|>


<commit_before>package json

import (
	"strings"
	"testing"
)

func TestDecoder(t *testing.T) {
	d := NewDecoder(strings.NewReader("\"Hello\" {} [{\"Pi\": 3.12159E1, \"Meaning Of Life\": 42}, {\"Good?\": true, \"Bad\": false, \"Ugly?\"	:null}] "))
	lengths := []int{7, 3, 89}
	for n, l := range lengths {
		d.readValue()
		if len(d.p.read) != l {
			t.Errorf("test %d: expecting read length of %d, got %d", n+1, l, len(d.p.read))
		}
		d.p.read = d.p.read[:0]
	}
}
<commit_msg>Renamed test<commit_after>package json

import (
	"strings"
	"testing"
)

func TestJSONRead(t *testing.T) {
	d := NewDecoder(strings.NewReader("\"Hello\" {} [{\"Pi\": 3.12159E1, \"Meaning Of Life\": 42}, {\"Good?\": true, \"Bad\": false, \"Ugly?\"	:null}] "))
	lengths := []int{7, 3, 89}
	for n, l := range lengths {
		d.readValue()
		if len(d.p.read) != l {
			t.Errorf("test %d: expecting read length of %d, got %d", n+1, l, len(d.p.read))
		}
		d.p.read = d.p.read[:0]
	}
}
<|endoftext|>