<commit_before>// Copyright 2015 clair authors
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package ubuntu

import (
	"bufio"
	"bytes"
	"fmt"
	"io"
	"io/ioutil"
	"os"
	"regexp"
	"strconv"
	"strings"

	"github.com/coreos/clair/database"
	"github.com/coreos/clair/updater"
	"github.com/coreos/clair/utils"
	cerrors "github.com/coreos/clair/utils/errors"
	"github.com/coreos/clair/utils/types"
	"github.com/coreos/pkg/capnslog"
)

const (
	ubuntuTrackerURI  = "https://launchpad.net/ubuntu-cve-tracker"
	ubuntuTracker     = "lp:ubuntu-cve-tracker"
	ubuntuUpdaterFlag = "ubuntuUpdater"
)

var (
	repositoryLocalPath string

	ubuntuIgnoredReleases = map[string]struct{}{
		"upstream": struct{}{},
		"devel":    struct{}{},

		"dapper":   struct{}{},
		"edgy":     struct{}{},
		"feisty":   struct{}{},
		"gutsy":    struct{}{},
		"hardy":    struct{}{},
		"intrepid": struct{}{},
		"jaunty":   struct{}{},
		"karmic":   struct{}{},
		"lucid":    struct{}{},
		"maverick": struct{}{},
		"natty":    struct{}{},
		"oneiric":  struct{}{},
		"saucy":    struct{}{},

		"vivid/stable-phone-overlay": struct{}{},

		// Syntax error
		"Patches": struct{}{},
		// Product
		"product": struct{}{},
	}

	affectsCaptureRegexp      = regexp.MustCompile(`(?P<release>.*)_(?P<package>.*): (?P<status>[^\s]*)( \(+(?P<note>[^()]*)\)+)?`)
	affectsCaptureRegexpNames = affectsCaptureRegexp.SubexpNames()

	log = capnslog.NewPackageLogger("github.com/coreos/clair", "updater/fetchers/ubuntu")
)

// UbuntuFetcher implements updater.Fetcher and get vulnerability updates from
// the Ubuntu CVE Tracker.
type UbuntuFetcher struct{}

func init() {
	//updater.RegisterFetcher("Ubuntu", &UbuntuFetcher{})
}

// FetchUpdate gets vulnerability updates from the Ubuntu CVE Tracker.
func (fetcher *UbuntuFetcher) FetchUpdate(datastore database.Datastore) (resp updater.FetcherResponse, err error) {
	log.Info("fetching Ubuntu vulnerabilities")

	// Check to see if the repository does not already exist.
	if _, pathExists := os.Stat(repositoryLocalPath); repositoryLocalPath == "" || os.IsNotExist(pathExists) {
		// Create a temporary folder and download the repository.
		p, err := ioutil.TempDir(os.TempDir(), "ubuntu-cve-tracker")
		if err != nil {
			return resp, ErrFilesystem
		}

		// bzr wants an empty target directory.
		repositoryLocalPath = p + "/repository"

		// Create the new repository.
		err = createRepository(repositoryLocalPath)
		if err != nil {
			return resp, err
		}
	} else {
		// Update the repository that's already on disk.
		err = updateRepository(repositoryLocalPath)
		if err != nil {
			return resp, err
		}
	}

	// Get revision number.
	revisionNumber, err := getRevisionNumber(repositoryLocalPath)
	if err != nil {
		return resp, err
	}

	// Get the latest revision number we successfully applied in the database.
	dbRevisionNumber, err := datastore.GetKeyValue("ubuntuUpdater")
	if err != nil {
		return resp, err
	}

	// Get the list of vulnerabilities that we have to update.
	modifiedCVE, err := collectModifiedVulnerabilities(revisionNumber, dbRevisionNumber, repositoryLocalPath)
	if err != nil {
		return resp, err
	}

	// Parse and add the vulnerabilities.
	for cvePath := range modifiedCVE {
		file, err := os.Open(repositoryLocalPath + "/" + cvePath)
		if err != nil {
			// This can happen when a file is modified and then moved in another
			// commit.
			continue
		}

		v, pkgs, unknownReleases, err := parseUbuntuCVE(file)
		if err != nil {
			return resp, err
		}

		if len(v.FixedInNodes) > 0 {
			resp.Vulnerabilities = append(resp.Vulnerabilities, v)
			resp.Packages = append(resp.Packages, pkgs...)
		}

		// Log any unknown releases.
		for k := range unknownReleases {
			note := fmt.Sprintf("Ubuntu %s is not mapped to any version number (eg. trusty->14.04). Please update me.", k)
			resp.Notes = append(resp.Notes, note)
			log.Warning(note)

			// If we encountered unknown Ubuntu release, we don't want the revision
			// number to be considered as managed.
			dbRevisionNumberInt, _ := strconv.Atoi(dbRevisionNumber)
			revisionNumber = dbRevisionNumberInt
		}

		// Close the file manually.
		//
		// We do that instead of using defer because defer works on a function-level scope.
		// We would open many files and close them all at once at the end of the function,
		// which could lead to exceed fs.file-max.
		file.Close()
	}

	// Add flag information
	resp.FlagName = ubuntuUpdaterFlag
	resp.FlagValue = strconv.Itoa(revisionNumber)

	return
}

func collectModifiedVulnerabilities(revision int, dbRevision, repositoryLocalPath string) (map[string]struct{}, error) {
	modifiedCVE := make(map[string]struct{})

	// Handle a brand new database.
	if dbRevision == "" {
		for _, folder := range []string{"active", "retired"} {
			d, err := os.Open(repositoryLocalPath + "/" + folder)
			if err != nil {
				log.Errorf("could not open Ubuntu vulnerabilities repository's folder: %s", err)
				return nil, ErrFilesystem
			}

			// Get the FileInfo of all the files in the directory.
			names, err := d.Readdirnames(-1)
			if err != nil {
				log.Errorf("could not read Ubuntu vulnerabilities repository's folder:: %s.", err)
				return nil, ErrFilesystem
			}

			// Add the vulnerabilities to the list.
			for _, name := range names {
				if strings.HasPrefix(name, "CVE-") {
					modifiedCVE[folder+"/"+name] = struct{}{}
				}
			}

			// Close the file manually.
			//
			// We do that instead of using defer because defer works on a function-level scope.
			// We would open many files and close them all at once at the end of the function,
			// which could lead to exceed fs.file-max.
			d.Close()
		}

		return modifiedCVE, nil
	}

	// Handle an up to date database.
	dbRevisionInt, _ := strconv.Atoi(dbRevision)
	if revision == dbRevisionInt {
		log.Debug("no Ubuntu update")
		return modifiedCVE, nil
	}

	// Handle a database that needs upgrading.
	out, err := utils.Exec(repositoryLocalPath, "bzr", "log", "--verbose", "-r"+strconv.Itoa(dbRevisionInt+1)+"..", "-n0")
	if err != nil {
		log.Errorf("could not get Ubuntu vulnerabilities repository logs: %s. output: %s", err, out)
		return nil, cerrors.ErrCouldNotDownload
	}

	scanner := bufio.NewScanner(bytes.NewReader(out))
	for scanner.Scan() {
		text := strings.TrimSpace(scanner.Text())
		if strings.Contains(text, "CVE-") && (strings.HasPrefix(text, "active/") || strings.HasPrefix(text, "retired/")) {
			if strings.Contains(text, " => ") {
				text = text[strings.Index(text, " => ")+4:]
			}
			modifiedCVE[text] = struct{}{}
		}
	}

	return modifiedCVE, nil
}

func createRepository(pathToRepo string) error {
	// Branch repository
	out, err := utils.Exec("/tmp/", "bzr", "branch", ubuntuTracker, pathToRepo)
	if err != nil {
		log.Errorf("could not branch Ubuntu repository: %s. output: %s", err, out)
		return cerrors.ErrCouldNotDownload
	}
	return nil
}

func updateRepository(pathToRepo string) error {
	// Pull repository
	out, err := utils.Exec(pathToRepo, "bzr", "pull", "--overwrite")
	if err != nil {
		log.Errorf("could not pull Ubuntu repository: %s. output: %s", err, out)
		return cerrors.ErrCouldNotDownload
	}
	return nil
}

func getRevisionNumber(pathToRepo string) (int, error) {
	out, err := utils.Exec(pathToRepo, "bzr", "revno")
	if err != nil {
		log.Errorf("could not get Ubuntu repository's revision number: %s. output: %s", err, out)
		return 0, cerrors.ErrCouldNotDownload
	}
	revno, err := strconv.Atoi(strings.TrimSpace(string(out)))
	if err != nil {
		log.Errorf("could not parse Ubuntu repository's revision number: %s. output: %s", err, out)
		return 0, cerrors.ErrCouldNotDownload
	}
	return revno, nil
}

func parseUbuntuCVE(fileContent io.Reader) (vulnerability *database.Vulnerability, packages []*database.Package, unknownReleases map[string]struct{}, err error) {
	vulnerability = &database.Vulnerability{}
	unknownReleases = make(map[string]struct{})
	readingDescription := false
	scanner := bufio.NewScanner(fileContent)

	for scanner.Scan() {
		line := strings.TrimSpace(scanner.Text())

		// Skip any comments.
		if strings.HasPrefix(line, "#") {
			continue
		}

		// Parse the name.
		if strings.HasPrefix(line, "Candidate:") {
			vulnerability.ID = strings.TrimSpace(strings.TrimPrefix(line, "Candidate:"))
			continue
		}

		// Parse the link.
		if vulnerability.Link == "" && strings.HasPrefix(line, "http") {
			vulnerability.Link = strings.TrimSpace(line)
			continue
		}

		// Parse the priority.
		if strings.HasPrefix(line, "Priority:") {
			priority := strings.TrimSpace(strings.TrimPrefix(line, "Priority:"))

			// Handle syntax error: Priority: medium (heap-protector)
			if strings.Contains(priority, " ") {
				priority = priority[:strings.Index(priority, " ")]
			}

			vulnerability.Severity = ubuntuPriorityToPriority(priority)
			continue
		}

		// Parse the description.
		if strings.HasPrefix(line, "Description:") {
			readingDescription = true
			vulnerability.Description = strings.TrimSpace(strings.TrimPrefix(line, "Description:")) // In case there is a formatting error and the description starts on the same line
			continue
		}
		if readingDescription {
			if strings.HasPrefix(line, "Ubuntu-Description:") || strings.HasPrefix(line, "Notes:") || strings.HasPrefix(line, "Bugs:") || strings.HasPrefix(line, "Priority:") || strings.HasPrefix(line, "Discovered-by:") || strings.HasPrefix(line, "Assigned-to:") {
				readingDescription = false
			} else {
				vulnerability.Description = vulnerability.Description + " " + line
				continue
			}
		}

		// Try to parse the package that the vulnerability affects.
		affectsCaptureArr := affectsCaptureRegexp.FindAllStringSubmatch(line, -1)
		if len(affectsCaptureArr) > 0 {
			affectsCapture := affectsCaptureArr[0]

			md := map[string]string{}
			for i, n := range affectsCapture {
				md[affectsCaptureRegexpNames[i]] = strings.TrimSpace(n)
			}

			// Ignore Linux kernels.
			if strings.HasPrefix(md["package"], "linux") {
				continue
			}

			// Only consider the package if its status is needed, active, deferred
			// or released. Ignore DNE, needs-triage, not-affected, ignored, pending.
			if md["status"] == "needed" || md["status"] == "active" || md["status"] == "deferred" || md["status"] == "released" {
				if _, isReleaseIgnored := ubuntuIgnoredReleases[md["release"]]; isReleaseIgnored {
					continue
				}
				if _, isReleaseKnown := database.UbuntuReleasesMapping[md["release"]]; !isReleaseKnown {
					unknownReleases[md["release"]] = struct{}{}
					continue
				}

				var version types.Version
				if md["status"] == "released" {
					if md["note"] != "" {
						var err error
						version, err = types.NewVersion(md["note"])
						if err != nil {
							log.Warningf("could not parse package version '%s': %s. skipping", md["note"], err)
						}
					}
				} else {
					version = types.MaxVersion
				}
				if version.String() == "" {
					continue
				}

				// Create and add the new package.
				pkg := &database.Package{
					OS:      "ubuntu:" + database.UbuntuReleasesMapping[md["release"]],
					Name:    md["package"],
					Version: version,
				}
				packages = append(packages, pkg)
				vulnerability.FixedInNodes = append(vulnerability.FixedInNodes, pkg.GetNode())
			}
		}
	}

	// Trim extra spaces in the description
	vulnerability.Description = strings.TrimSpace(vulnerability.Description)

	// If no link has been provided (CVE-2006-NNN0 for instance), add the link to the tracker
	if vulnerability.Link == "" {
		vulnerability.Link = ubuntuTrackerURI
	}

	// If no priority has been provided (CVE-2007-0667 for instance), set the priority to Unknown
	if vulnerability.Severity == "" {
		vulnerability.Severity = types.Unknown
	}

	return
}

func ubuntuPriorityToPriority(priority string) types.Severity {
	switch priority {
	case "untriaged":
		return types.Unknown
	case "negligible":
		return types.Negligible
	case "low":
		return types.Low
	case "medium":
		return types.Medium
	case "high":
		return types.High
	case "critical":
		return types.Critical
	}

	log.Warning("Could not determine a vulnerability priority from: %s", priority)
	return types.Unknown
}
<commit_msg>updater: update Ubuntu fetcher and add not-affected capability<commit_after>// Copyright 2015 clair authors
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package ubuntu

import (
	"bufio"
	"bytes"
	"errors"
	"fmt"
	"io"
	"io/ioutil"
	"os"
	"regexp"
	"strconv"
	"strings"

	"github.com/coreos/clair/database"
	"github.com/coreos/clair/updater"
	"github.com/coreos/clair/utils"
	cerrors "github.com/coreos/clair/utils/errors"
	"github.com/coreos/clair/utils/types"
	"github.com/coreos/pkg/capnslog"
)

const (
	ubuntuTrackerURI  = "https://launchpad.net/ubuntu-cve-tracker"
	ubuntuTracker     = "lp:ubuntu-cve-tracker"
	ubuntuUpdaterFlag = "ubuntuUpdater"
)

var (
	repositoryLocalPath string

	ubuntuIgnoredReleases = map[string]struct{}{
		"upstream": struct{}{},
		"devel":    struct{}{},

		"dapper":   struct{}{},
		"edgy":     struct{}{},
		"feisty":   struct{}{},
		"gutsy":    struct{}{},
		"hardy":    struct{}{},
		"intrepid": struct{}{},
		"jaunty":   struct{}{},
		"karmic":   struct{}{},
		"lucid":    struct{}{},
		"maverick": struct{}{},
		"natty":    struct{}{},
		"oneiric":  struct{}{},
		"saucy":    struct{}{},

		"vivid/stable-phone-overlay": struct{}{},

		// Syntax error
		"Patches": struct{}{},
		// Product
		"product": struct{}{},
	}

	affectsCaptureRegexp      = regexp.MustCompile(`(?P<release>.*)_(?P<package>.*): (?P<status>[^\s]*)( \(+(?P<note>[^()]*)\)+)?`)
	affectsCaptureRegexpNames = affectsCaptureRegexp.SubexpNames()

	log = capnslog.NewPackageLogger("github.com/coreos/clair", "updater/fetchers/ubuntu")

	// ErrFilesystem is returned when a fetcher fails to interact with the local filesystem.
	ErrFilesystem = errors.New("updater/fetchers: something went wrong when interacting with the fs")
)

// UbuntuFetcher implements updater.Fetcher and get vulnerability updates from
// the Ubuntu CVE Tracker.
type UbuntuFetcher struct{}

func init() {
	updater.RegisterFetcher("Ubuntu", &UbuntuFetcher{})
}

// FetchUpdate gets vulnerability updates from the Ubuntu CVE Tracker.
func (fetcher *UbuntuFetcher) FetchUpdate(datastore database.Datastore) (resp updater.FetcherResponse, err error) {
	log.Info("fetching Ubuntu vulnerabilities")

	// Check to see if the repository does not already exist.
	if _, pathExists := os.Stat(repositoryLocalPath); repositoryLocalPath == "" || os.IsNotExist(pathExists) {
		// Create a temporary folder and download the repository.
		p, err := ioutil.TempDir(os.TempDir(), "ubuntu-cve-tracker")
		if err != nil {
			return resp, ErrFilesystem
		}

		// bzr wants an empty target directory.
		repositoryLocalPath = p + "/repository"

		// Create the new repository.
		err = createRepository(repositoryLocalPath)
		if err != nil {
			return resp, err
		}
	} else {
		// Update the repository that's already on disk.
		err = updateRepository(repositoryLocalPath)
		if err != nil {
			return resp, err
		}
	}

	// Get revision number.
	revisionNumber, err := getRevisionNumber(repositoryLocalPath)
	if err != nil {
		return resp, err
	}

	// Get the latest revision number we successfully applied in the database.
	dbRevisionNumber, err := datastore.GetKeyValue("ubuntuUpdater")
	if err != nil {
		return resp, err
	}

	// Get the list of vulnerabilities that we have to update.
	modifiedCVE, err := collectModifiedVulnerabilities(revisionNumber, dbRevisionNumber, repositoryLocalPath)
	if err != nil {
		return resp, err
	}

	// Parse and add the vulnerabilities.
	for cvePath := range modifiedCVE {
		file, err := os.Open(repositoryLocalPath + "/" + cvePath)
		if err != nil {
			// This can happen when a file is modified and then moved in another
			// commit.
			continue
		}

		v, unknownReleases, err := parseUbuntuCVE(file)
		if err != nil {
			return resp, err
		}

		if len(v.FixedIn) > 0 {
			resp.Vulnerabilities = append(resp.Vulnerabilities, v)
		}

		// Log any unknown releases.
		for k := range unknownReleases {
			note := fmt.Sprintf("Ubuntu %s is not mapped to any version number (eg. trusty->14.04). Please update me.", k)
			resp.Notes = append(resp.Notes, note)
			log.Warning(note)

			// If we encountered unknown Ubuntu release, we don't want the revision
			// number to be considered as managed.
			dbRevisionNumberInt, _ := strconv.Atoi(dbRevisionNumber)
			revisionNumber = dbRevisionNumberInt
		}

		// Close the file manually.
		//
		// We do that instead of using defer because defer works on a function-level scope.
		// We would open many files and close them all at once at the end of the function,
		// which could lead to exceed fs.file-max.
		file.Close()
	}

	// Add flag information
	resp.FlagName = ubuntuUpdaterFlag
	resp.FlagValue = strconv.Itoa(revisionNumber)

	return
}

func collectModifiedVulnerabilities(revision int, dbRevision, repositoryLocalPath string) (map[string]struct{}, error) {
	modifiedCVE := make(map[string]struct{})

	// Handle a brand new database.
	if dbRevision == "" {
		for _, folder := range []string{"active", "retired"} {
			d, err := os.Open(repositoryLocalPath + "/" + folder)
			if err != nil {
				log.Errorf("could not open Ubuntu vulnerabilities repository's folder: %s", err)
				return nil, ErrFilesystem
			}

			// Get the FileInfo of all the files in the directory.
			names, err := d.Readdirnames(-1)
			if err != nil {
				log.Errorf("could not read Ubuntu vulnerabilities repository's folder:: %s.", err)
				return nil, ErrFilesystem
			}

			// Add the vulnerabilities to the list.
			for _, name := range names {
				if strings.HasPrefix(name, "CVE-") {
					modifiedCVE[folder+"/"+name] = struct{}{}
				}
			}

			// Close the file manually.
			//
			// We do that instead of using defer because defer works on a function-level scope.
			// We would open many files and close them all at once at the end of the function,
			// which could lead to exceed fs.file-max.
			d.Close()
		}

		return modifiedCVE, nil
	}

	// Handle an up to date database.
	dbRevisionInt, _ := strconv.Atoi(dbRevision)
	if revision == dbRevisionInt {
		log.Debug("no Ubuntu update")
		return modifiedCVE, nil
	}

	// Handle a database that needs upgrading.
	out, err := utils.Exec(repositoryLocalPath, "bzr", "log", "--verbose", "-r"+strconv.Itoa(dbRevisionInt+1)+"..", "-n0")
	if err != nil {
		log.Errorf("could not get Ubuntu vulnerabilities repository logs: %s. output: %s", err, out)
		return nil, cerrors.ErrCouldNotDownload
	}

	scanner := bufio.NewScanner(bytes.NewReader(out))
	for scanner.Scan() {
		text := strings.TrimSpace(scanner.Text())
		if strings.Contains(text, "CVE-") && (strings.HasPrefix(text, "active/") || strings.HasPrefix(text, "retired/")) {
			if strings.Contains(text, " => ") {
				text = text[strings.Index(text, " => ")+4:]
			}
			modifiedCVE[text] = struct{}{}
		}
	}

	return modifiedCVE, nil
}

func createRepository(pathToRepo string) error {
	// Branch repository
	out, err := utils.Exec("/tmp/", "bzr", "branch", ubuntuTracker, pathToRepo)
	if err != nil {
		log.Errorf("could not branch Ubuntu repository: %s. output: %s", err, out)
		return cerrors.ErrCouldNotDownload
	}
	return nil
}

func updateRepository(pathToRepo string) error {
	// Pull repository
	out, err := utils.Exec(pathToRepo, "bzr", "pull", "--overwrite")
	if err != nil {
		log.Errorf("could not pull Ubuntu repository: %s. output: %s", err, out)
		return cerrors.ErrCouldNotDownload
	}
	return nil
}

func getRevisionNumber(pathToRepo string) (int, error) {
	out, err := utils.Exec(pathToRepo, "bzr", "revno")
	if err != nil {
		log.Errorf("could not get Ubuntu repository's revision number: %s. output: %s", err, out)
		return 0, cerrors.ErrCouldNotDownload
	}
	revno, err := strconv.Atoi(strings.TrimSpace(string(out)))
	if err != nil {
		log.Errorf("could not parse Ubuntu repository's revision number: %s. output: %s", err, out)
		return 0, cerrors.ErrCouldNotDownload
	}
	return revno, nil
}

func parseUbuntuCVE(fileContent io.Reader) (vulnerability database.Vulnerability, unknownReleases map[string]struct{}, err error) {
	unknownReleases = make(map[string]struct{})
	readingDescription := false
	scanner := bufio.NewScanner(fileContent)

	for scanner.Scan() {
		line := strings.TrimSpace(scanner.Text())

		// Skip any comments.
		if strings.HasPrefix(line, "#") {
			continue
		}

		// Parse the name.
		if strings.HasPrefix(line, "Candidate:") {
			vulnerability.Name = strings.TrimSpace(strings.TrimPrefix(line, "Candidate:"))
			continue
		}

		// Parse the link.
		if vulnerability.Link == "" && strings.HasPrefix(line, "http") {
			vulnerability.Link = strings.TrimSpace(line)
			continue
		}

		// Parse the priority.
		if strings.HasPrefix(line, "Priority:") {
			priority := strings.TrimSpace(strings.TrimPrefix(line, "Priority:"))

			// Handle syntax error: Priority: medium (heap-protector)
			if strings.Contains(priority, " ") {
				priority = priority[:strings.Index(priority, " ")]
			}

			vulnerability.Severity = ubuntuPriorityToSeverity(priority)
			continue
		}

		// Parse the description.
		if strings.HasPrefix(line, "Description:") {
			readingDescription = true
			vulnerability.Description = strings.TrimSpace(strings.TrimPrefix(line, "Description:")) // In case there is a formatting error and the description starts on the same line
			continue
		}
		if readingDescription {
			if strings.HasPrefix(line, "Ubuntu-Description:") || strings.HasPrefix(line, "Notes:") || strings.HasPrefix(line, "Bugs:") || strings.HasPrefix(line, "Priority:") || strings.HasPrefix(line, "Discovered-by:") || strings.HasPrefix(line, "Assigned-to:") {
				readingDescription = false
			} else {
				vulnerability.Description = vulnerability.Description + " " + line
				continue
			}
		}

		// Try to parse the package that the vulnerability affects.
		affectsCaptureArr := affectsCaptureRegexp.FindAllStringSubmatch(line, -1)
		if len(affectsCaptureArr) > 0 {
			affectsCapture := affectsCaptureArr[0]

			md := map[string]string{}
			for i, n := range affectsCapture {
				md[affectsCaptureRegexpNames[i]] = strings.TrimSpace(n)
			}

			// Ignore Linux kernels.
			if strings.HasPrefix(md["package"], "linux") {
				continue
			}

			// Only consider the package if its status is needed, active, deferred, not-affected or
			// released. Ignore DNE (package does not exist), needs-triage, ignored, pending.
			if md["status"] == "needed" || md["status"] == "active" || md["status"] == "deferred" || md["status"] == "released" || md["status"] == "not-affected" {
				if _, isReleaseIgnored := ubuntuIgnoredReleases[md["release"]]; isReleaseIgnored {
					continue
				}
				if _, isReleaseKnown := database.UbuntuReleasesMapping[md["release"]]; !isReleaseKnown {
					unknownReleases[md["release"]] = struct{}{}
					continue
				}

				var version types.Version
				if md["status"] == "released" {
					if md["note"] != "" {
						var err error
						version, err = types.NewVersion(md["note"])
						if err != nil {
							log.Warningf("could not parse package version '%s': %s. skipping", md["note"], err)
						}
					}
				} else if md["status"] == "not-affected" {
					version = types.MinVersion
				} else {
					version = types.MaxVersion
				}
				if version.String() == "" {
					continue
				}

				// Create and add the new package.
				featureVersion := database.FeatureVersion{
					Feature: database.Feature{
						Namespace: database.Namespace{Name: "ubuntu:" + database.UbuntuReleasesMapping[md["release"]]},
						Name:      md["package"],
					},
					Version: version,
				}
				vulnerability.FixedIn = append(vulnerability.FixedIn, featureVersion)
			}
		}
	}

	// Trim extra spaces in the description
	vulnerability.Description = strings.TrimSpace(vulnerability.Description)

	// If no link has been provided (CVE-2006-NNN0 for instance), add the link to the tracker
	if vulnerability.Link == "" {
		vulnerability.Link = ubuntuTrackerURI
	}

	// If no priority has been provided (CVE-2007-0667 for instance), set the priority to Unknown
	if vulnerability.Severity == "" {
		vulnerability.Severity = types.Unknown
	}

	return
}

func ubuntuPriorityToSeverity(priority string) types.Priority {
	switch priority {
	case "untriaged":
		return types.Unknown
	case "negligible":
		return types.Negligible
	case "low":
		return types.Low
	case "medium":
		return types.Medium
	case "high":
		return types.High
	case "critical":
		return types.Critical
	}

	log.Warning("Could not determine a vulnerability priority from: %s", priority)
	return types.Unknown
}
<|endoftext|>


<commit_before>// Copyright © 2016 The Things Network
// Use of this source code is governed by the MIT license that can be found in the LICENSE file.

package cmd

import (
	"fmt"

	"github.com/TheThingsNetwork/ttn/core"
	"github.com/TheThingsNetwork/ttn/core/components/handler"
	"github.com/TheThingsNetwork/ttn/ttnctl/util"
	"github.com/gosuri/uitable"
	"github.com/spf13/cobra"
	"github.com/spf13/viper"
	"golang.org/x/net/context"
)

const emptyCell = "-"

func getHandlerManager() core.AuthHandlerClient {
	cli, err := handler.NewClient(viper.GetString("ttn-handler"))
	if err != nil {
		ctx.Fatalf("Could not connect: %v", err)
	}
	return cli
}

// devicesCmd represents the `devices` command
var devicesCmd = &cobra.Command{
	Use:   "devices",
	Short: "Manage devices on the Handler",
	Long:  `ttnctl devices retrieves a list of devices that your application registered on the Handler.`,
	Run: func(cmd *cobra.Command, args []string) {
		appEUI, err := util.Parse64(viper.GetString("app-eui"))
		if err != nil {
			ctx.Fatalf("Invalid AppEUI: %s", err)
		}

		manager := getHandlerManager()

		res, err := manager.ListDevices(context.Background(), &core.ListDevicesHandlerReq{
			Token:  viper.GetString("app-token"),
			AppEUI: appEUI,
		})
		if err != nil {
			ctx.WithError(err).Fatal("Could not get device list")
		}

		ctx.Infof("Found %d personalized devices (ABP)", len(res.ABP))
		table := uitable.New()
		table.MaxColWidth = 70
		table.AddRow("DevAddr", "NwkSKey", "AppSKey", "FCntUp", "FCntDown")
		for _, device := range res.ABP {
			devAddr := fmt.Sprintf("%X", device.DevAddr)
			nwkSKey := fmt.Sprintf("%X", device.NwkSKey)
			appSKey := fmt.Sprintf("%X", device.AppSKey)
			table.AddRow(devAddr, nwkSKey, appSKey, device.FCntUp, device.FCntDown)
		}
		fmt.Println(table)

		ctx.Infof("Found %d dynamic devices (OTAA)", len(res.OTAA))
		table = uitable.New()
		table.MaxColWidth = 40
		table.AddRow("DevEUI", "DevAddr", "NwkSKey", "AppSKey", "AppKey", "FCntUp", "FCntDown")
		for _, device := range res.OTAA {
			devEUI := fmt.Sprintf("%X", device.DevEUI)
			devAddr := fmt.Sprintf("%X", device.DevAddr)
			nwkSKey := fmt.Sprintf("%X", device.NwkSKey)
			appSKey := fmt.Sprintf("%X", device.AppSKey)
			appKey := fmt.Sprintf("%X", device.AppKey)
			table.AddRow(devEUI, devAddr, nwkSKey, appSKey, appKey, device.FCntUp, device.FCntDown)
		}
		fmt.Println(table)
	},
}

// devicesRegisterCmd represents the `device register` command
var devicesRegisterCmd = &cobra.Command{
	Use:   "register [DevEUI] [AppKey]",
	Short: "Create or Update registrations on the Handler",
	Long:  `ttnctl device register creates or updates an OTAA registration on the Handler`,
	Run: func(cmd *cobra.Command, args []string) {
		if len(args) < 2 {
			ctx.Fatal("Insufficient arguments")
		}

		appEUI, err := util.Parse64(viper.GetString("app-eui"))
		if err != nil {
			ctx.Fatalf("Invalid AppEUI: %s", err)
		}

		devEUI, err := util.Parse64(args[0])
		if err != nil {
			ctx.Fatalf("Invalid DevEUI: %s", err)
		}

		appKey, err := util.Parse128(args[1])
		if err != nil {
			ctx.Fatalf("Invalid AppKey: %s", err)
		}

		manager := getHandlerManager()
		res, err := manager.UpsertOTAA(context.Background(), &core.UpsertOTAAHandlerReq{
			Token:  viper.GetString("app-token"),
			AppEUI: appEUI,
			DevEUI: devEUI,
			AppKey: appKey,
		})
		if err != nil || res == nil {
			ctx.WithError(err).Fatal("Could not register device")
		}
		ctx.Info("Ok")
	},
}

// devicesRegisterPersonalizedCmd represents the `device register personalized` command
var devicesRegisterPersonalizedCmd = &cobra.Command{
	Use:   "personalized [DevAddr] [NwkSKey] [AppSKey]",
	Short: "Create or Update ABP registrations on the Handler",
	Long:  `ttnctl device register creates or updates an ABP registration on the Handler`,
	Run: func(cmd *cobra.Command, args []string) {
		if len(args) < 3 {
			ctx.Fatal("Insufficient arguments")
		}

		appEUI, err := util.Parse64(viper.GetString("app-eui"))
		if err != nil {
			ctx.Fatalf("Invalid AppEUI: %s", err)
		}

		devAddr, err := util.Parse32(args[0])
		if err != nil {
			ctx.Fatalf("Invalid DevAddr: %s", err)
		}

		nwkSKey, err := util.Parse128(args[1])
		if err != nil {
			ctx.Fatalf("Invalid NwkSKey: %s", err)
		}

		appSKey, err := util.Parse128(args[2])
		if err != nil {
			ctx.Fatalf("Invalid AppSKey: %s", err)
		}

		manager := getHandlerManager()
		res, err := manager.UpsertABP(context.Background(), &core.UpsertABPHandlerReq{
			Token:   viper.GetString("app-token"),
			AppEUI:  appEUI,
			DevAddr: devAddr,
			AppSKey: appSKey,
			NwkSKey: nwkSKey,
		})
		if err != nil || res == nil {
			ctx.WithError(err).Fatal("Could not register device")
		}
		ctx.Info("Ok")
	},
}

func init() {
	RootCmd.AddCommand(devicesCmd)
	devicesCmd.AddCommand(devicesRegisterCmd)
	devicesRegisterCmd.AddCommand(devicesRegisterPersonalizedCmd)
}
<commit_msg>Aligned instruction with command<commit_after>// Copyright © 2016 The Things Network
// Use of this source code is governed by the MIT license that can be found in the LICENSE file.

package cmd

import (
	"fmt"

	"github.com/TheThingsNetwork/ttn/core"
	"github.com/TheThingsNetwork/ttn/core/components/handler"
	"github.com/TheThingsNetwork/ttn/ttnctl/util"
	"github.com/gosuri/uitable"
	"github.com/spf13/cobra"
	"github.com/spf13/viper"
	"golang.org/x/net/context"
)

const emptyCell = "-"

func getHandlerManager() core.AuthHandlerClient {
	cli, err := handler.NewClient(viper.GetString("ttn-handler"))
	if err != nil {
		ctx.Fatalf("Could not connect: %v", err)
	}
	return cli
}

// devicesCmd represents the `devices` command
var devicesCmd = &cobra.Command{
	Use:   "devices",
	Short: "Manage devices on the Handler",
	Long:  `ttnctl devices retrieves a list of devices that your application registered on the Handler.`,
	Run: func(cmd *cobra.Command, args []string) {
		appEUI, err := util.Parse64(viper.GetString("app-eui"))
		if err != nil {
			ctx.Fatalf("Invalid AppEUI: %s", err)
		}

		manager := getHandlerManager()

		res, err := manager.ListDevices(context.Background(), &core.ListDevicesHandlerReq{
			Token:  viper.GetString("app-token"),
			AppEUI: appEUI,
		})
		if err != nil {
			ctx.WithError(err).Fatal("Could not get device list")
		}

		ctx.Infof("Found %d personalized devices (ABP)", len(res.ABP))
		table := uitable.New()
		table.MaxColWidth = 70
		table.AddRow("DevAddr", "NwkSKey", "AppSKey", "FCntUp", "FCntDown")
		for _, device := range res.ABP {
			devAddr := fmt.Sprintf("%X", device.DevAddr)
			nwkSKey := fmt.Sprintf("%X", device.NwkSKey)
			appSKey := fmt.Sprintf("%X", device.AppSKey)
			table.AddRow(devAddr, nwkSKey, appSKey, device.FCntUp, device.FCntDown)
		}
		fmt.Println(table)

		ctx.Infof("Found %d dynamic devices (OTAA)", len(res.OTAA))
		table = uitable.New()
		table.MaxColWidth = 40
		table.AddRow("DevEUI", "DevAddr", "NwkSKey", "AppSKey", "AppKey", "FCntUp", "FCntDown")
		for _, device := range res.OTAA {
			devEUI := fmt.Sprintf("%X", device.DevEUI)
			devAddr := fmt.Sprintf("%X", device.DevAddr)
			nwkSKey := fmt.Sprintf("%X", device.NwkSKey)
			appSKey := fmt.Sprintf("%X", device.AppSKey)
			appKey := fmt.Sprintf("%X", device.AppKey)
			table.AddRow(devEUI, devAddr, nwkSKey, appSKey, appKey, device.FCntUp, device.FCntDown)
		}
		fmt.Println(table)
	},
}

// devicesRegisterCmd represents the `device register` command
var devicesRegisterCmd = &cobra.Command{
	Use:   "register [DevEUI] [AppKey]",
	Short: "Create or Update registrations on the Handler",
	Long:  `ttnctl devices register creates or updates an OTAA registration on the Handler`,
	Run: func(cmd *cobra.Command, args []string) {
		if len(args) < 2 {
			ctx.Fatal("Insufficient arguments")
		}

		appEUI, err := util.Parse64(viper.GetString("app-eui"))
		if err != nil {
			ctx.Fatalf("Invalid AppEUI: %s", err)
		}

		devEUI, err := util.Parse64(args[0])
		if err != nil {
			ctx.Fatalf("Invalid DevEUI: %s", err)
		}

		appKey, err := util.Parse128(args[1])
		if err != nil {
			ctx.Fatalf("Invalid AppKey: %s", err)
		}

		manager := getHandlerManager()
		res, err := manager.UpsertOTAA(context.Background(), &core.UpsertOTAAHandlerReq{
			Token:  viper.GetString("app-token"),
			AppEUI: appEUI,
			DevEUI: devEUI,
			AppKey: appKey,
		})
		if err != nil || res == nil {
			ctx.WithError(err).Fatal("Could not register device")
		}
		ctx.Info("Ok")
	},
}

// devicesRegisterPersonalizedCmd represents the `device register personalized` command
var devicesRegisterPersonalizedCmd = &cobra.Command{
	Use:   "personalized [DevAddr] [NwkSKey] [AppSKey]",
	Short: "Create or Update ABP registrations on the Handler",
	Long:  `ttnctl devices register creates or updates an ABP registration on the Handler`,
	Run: func(cmd *cobra.Command, args []string) {
		if len(args) < 3 {
			ctx.Fatal("Insufficient arguments")
		}

		appEUI, err := util.Parse64(viper.GetString("app-eui"))
		if err != nil {
			ctx.Fatalf("Invalid AppEUI: %s", err)
		}

		devAddr, err := util.Parse32(args[0])
		if err != nil {
			ctx.Fatalf("Invalid DevAddr: %s", err)
		}

		nwkSKey, err := util.Parse128(args[1])
		if err != nil {
			ctx.Fatalf("Invalid NwkSKey: %s", err)
		}

		appSKey, err := util.Parse128(args[2])
		if err != nil {
			ctx.Fatalf("Invalid AppSKey: %s", err)
		}

		manager := getHandlerManager()
		res, err := manager.UpsertABP(context.Background(), &core.UpsertABPHandlerReq{
			Token:   viper.GetString("app-token"),
			AppEUI:  appEUI,
			DevAddr: devAddr,
			AppSKey: appSKey,
			NwkSKey: nwkSKey,
		})
		if err != nil || res == nil {
			ctx.WithError(err).Fatal("Could not register device")
		}
		ctx.Info("Ok")
	},
}

func init() {
	RootCmd.AddCommand(devicesCmd)
	devicesCmd.AddCommand(devicesRegisterCmd)
	devicesRegisterCmd.AddCommand(devicesRegisterPersonalizedCmd)
}
<|endoftext|>


<commit_before>package randomword_test

import (
	"time"

	. "code.cloudfoundry.org/cli/util/randomword"

	. "github.com/onsi/ginkgo"
	. "github.com/onsi/gomega"
)

var _ = Describe("Generator", func() {
	var gen Generator

	Describe("RandomAdjective", func() {
		It("generates a random adjective each time it is called", func() {
			setOne := []string{}
			setTwo := []string{}

			for i := 0; i < 3; i++ {
				setOne = append(setOne, gen.RandomAdjective())
				// We wait for 3 millisecond because the seed we use to generate the
				// randomness has a unit of 1 nanosecond plus random test flakiness
				time.Sleep(3 * time.Nanosecond)
				setTwo = append(setTwo, gen.RandomAdjective())
			}
			Expect(setOne).ToNot(ConsistOf(setTwo))
		})
	})

	Describe("RandomNoun", func() {
		It("generates a random noun each time it is called", func() {
			setOne := []string{}
			setTwo := []string{}

			for i := 0; i < 3; i++ {
				setOne = append(setOne, gen.RandomNoun())
				// We wait for 3 millisecond because the seed we use to generate the
				// randomness has a unit of 1 nanosecond plus random test flakiness
				time.Sleep(3 * time.Nanosecond)
				setTwo = append(setTwo, gen.RandomNoun())
			}
			Expect(setOne).ToNot(ConsistOf(setTwo))
		})
	})

	Describe("Babble", func() {
		It("generates a random adjective noun pair each time it is called", func() {
			wordPair := gen.Babble()
			Expect(wordPair).To(MatchRegexp(`^\w+-\w+$`))
		})
	})
})
<commit_msg>fix comment about sleep time in randomword tests<commit_after>package randomword_test

import (
	"time"

	. "code.cloudfoundry.org/cli/util/randomword"

	. "github.com/onsi/ginkgo"
	. "github.com/onsi/gomega"
)

var _ = Describe("Generator", func() {
	var gen Generator

	Describe("RandomAdjective", func() {
		It("generates a random adjective each time it is called", func() {
			setOne := []string{}
			setTwo := []string{}

			for i := 0; i < 3; i++ {
				setOne = append(setOne, gen.RandomAdjective())
				// We wait for 3 nanoseconds because the seed we use to generate the
				// randomness has a unit of 1 nanosecond plus random test flakiness
				time.Sleep(3 * time.Nanosecond)
				setTwo = append(setTwo, gen.RandomAdjective())
			}
			Expect(setOne).ToNot(ConsistOf(setTwo))
		})
	})

	Describe("RandomNoun", func() {
		It("generates a random noun each time it is called", func() {
			setOne := []string{}
			setTwo := []string{}

			for i := 0; i < 3; i++ {
				setOne = append(setOne, gen.RandomNoun())
				// We wait for 3 nanoseconds because the seed we use to generate the
				// randomness has a unit of 1 nanosecond plus random test flakiness
				time.Sleep(3 * time.Nanosecond)
				setTwo = append(setTwo, gen.RandomNoun())
			}
			Expect(setOne).ToNot(ConsistOf(setTwo))
		})
	})

	Describe("Babble", func() {
		It("generates a random adjective noun pair each time it is called", func() {
			wordPair := gen.Babble()
			Expect(wordPair).To(MatchRegexp(`^\w+-\w+$`))
		})
	})
})
<|endoftext|>


<commit_before>package randomword_test

import (
	"time"

	. "code.cloudfoundry.org/cli/util/randomword"

	. "github.com/onsi/ginkgo"
	. "github.com/onsi/gomega"
)

var _ = Describe("Generator", func() {
	var gen Generator

	BeforeEach(func() {
		gen = Generator{}
	})

	Describe("RandomAdjective", func() {
		It("generates a random adjective each time it is called", func() {
			adj := gen.RandomAdjective()
			// We wait for 1 millisecond because the seed we use to generate the randomness has a unit of 1 nanosecond
			time.Sleep(1)
			Expect(adj).ToNot(Equal(gen.RandomAdjective()))
		})
	})

	Describe("RandomNoun", func() {
		It("generates a random noun each time it is called", func() {
			noun := gen.RandomNoun()
			// We wait for 1 millisecond because the seed we use to generate the randomness has a unit of 1 nanosecond
			time.Sleep(1)
			Expect(noun).ToNot(Equal(gen.RandomNoun()))
		})
	})

	Describe("Babble", func() {
		It("generates a random adjective noun pair each time it is called", func() {
			wordPair := gen.Babble()
			Expect(wordPair).To(MatchRegexp("^\\w+-\\w+$"))
		})
	})
})
<commit_msg>Fix flakiness from random seed generator<commit_after>package randomword_test

import (
	"time"

	. "code.cloudfoundry.org/cli/util/randomword"

	. "github.com/onsi/ginkgo"
	. "github.com/onsi/gomega"
)

var _ = Describe("Generator", func() {
	var gen Generator

	BeforeEach(func() {
		gen = Generator{}
	})

	Describe("RandomAdjective", func() {
		It("generates a random adjective each time it is called", func() {
			adj := gen.RandomAdjective()
			// We wait for 3 millisecond because the seed we use to generate the
			// randomness has a unit of 1 nanosecond plus random test flakiness
			time.Sleep(3)
			Expect(adj).ToNot(Equal(gen.RandomAdjective()))
		})
	})

	Describe("RandomNoun", func() {
		It("generates a random noun each time it is called", func() {
			noun := gen.RandomNoun()
			// We wait for 3 millisecond because the seed we use to generate the
			// randomness has a unit of 1 nanosecond plus random test flakiness
			time.Sleep(3)
			Expect(noun).ToNot(Equal(gen.RandomNoun()))
		})
	})

	Describe("Babble", func() {
		It("generates a random adjective noun pair each time it is called", func() {
			wordPair := gen.Babble()
			Expect(wordPair).To(MatchRegexp("^\\w+-\\w+$"))
		})
	})
})
<|endoftext|>


<commit_before>// Karatsuba Big Int multiplication
package karatsuba

import (
	"fmt"
	"math/big"
	"testing"
)

func TestMultipy(t *testing.T) {
	a := big.NewInt(1234)
	b := big.NewInt(1234)

	if k_multiply(a, b).Cmp(mul(a, b)) != 0 {
		fmt.Println(k_multiply(a, b), mul(a, b))
		t.Fatal("Wrong result")
	}
}
<commit_msg>benchmark karatsuba<commit_after>// Karatsuba Big Int multiplication
package karatsuba

import (
	"fmt"
	"math/big"
	"testing"
)

func TestMultipy(t *testing.T) {
	a := big.NewInt(1234)

	if k_multiply(a, a).Cmp(mul(a, a)) != 0 {
		fmt.Println(k_multiply(a, a), mul(a, a))
		t.Fatal("Wrong result")
	}
}

func BenchmarkMultiply(b *testing.B) {
	for i := 0; i < b.N; i++ {
		a := big.NewInt(1234567890123456789)
		mul(a, a)
	}
}

func BenchmarkKaratsubaMultiply(b *testing.B) {
	for i := 0; i < b.N; i++ {
		a := big.NewInt(1234567890123456789)
		k_multiply(a, a)
	}
}
<|endoftext|>


<commit_before>package convert

import (
	"fmt"
	"io"
	"time"

	dto "github.com/prometheus/client_model/go"
	"github.com/prometheus/common/expfmt"
	"github.com/digitalocean/vulcan/bus"
)

// PromTextToSG reads prometheus metrics in text exposition format and
// produces a SampleGroup
func PromTextToSG(in io.Reader) (bus.SampleGroup, error) {
	fam := []*dto.MetricFamily{}
	dec := expfmt.NewDecoder(in, expfmt.FmtText)
	for {
		var f dto.MetricFamily
		err := dec.Decode(&f)
		if err != nil {
			if err == io.EOF {
				break
			}
			return bus.SampleGroup{}, err
		}
		fam = append(fam, &f)
	}
	sg := familyToSampleGroup(fam)
	return sg, nil
}

func familyToSampleGroup(metricFamilies []*dto.MetricFamily) bus.SampleGroup {
	samples := bus.SampleGroup{}
	for _, mf := range metricFamilies {
		for _, prom := range mf.GetMetric() {
			m := bus.Metric{
				Name:   mf.GetName(),
				Labels: map[string]string{},
			}
			for _, label := range prom.GetLabel() {
				m.Labels[label.GetName()] = label.GetValue()
			}
			switch mf.GetType() {
			case dto.MetricType_COUNTER:
				s := bus.Sample{
					Metric: m,
					Datapoint: bus.Datapoint{
						Value:     prom.Counter.GetValue(),
						Timestamp: promTimestamp(prom),
					},
				}
				s.Metric.Labels["__type__"] = "counter"
				samples = append(samples, &s)
			case dto.MetricType_GAUGE:
				s := bus.Sample{
					Metric: m,
					Datapoint: bus.Datapoint{
						Value:     prom.Gauge.GetValue(),
						Timestamp: promTimestamp(prom),
					},
				}
				s.Metric.Labels["__type__"] = "gauge"
				samples = append(samples, &s)
			case dto.MetricType_HISTOGRAM:
				// add bucket values, sum, and count for this prometheus metric
				m.Labels["__type__"] = "histogram"
				bucketName := m.Name + "_bucket"
				for _, b := range prom.Histogram.Bucket {
					myMetric := cloneMetric(m)
					myMetric.Name = bucketName
					myMetric.Labels["le"] = fmt.Sprint(b.GetUpperBound())
					s := bus.Sample{
						Metric: myMetric,
						Datapoint: bus.Datapoint{
							Value:     float64(b.GetCumulativeCount()),
							Timestamp: promTimestamp(prom),
						},
					}
					samples = append(samples, &s)
				}
				// do sum
				sum := cloneMetric(m)
				sum.Name = sum.Name + "_sum"
				s := bus.Sample{
					Metric: sum,
					Datapoint: bus.Datapoint{
						Value:     float64(*prom.Histogram.SampleSum),
						Timestamp: promTimestamp(prom),
					},
				}
				samples = append(samples, &s)
				// do count
				count := cloneMetric(m)
				count.Name = count.Name + "_count"
				s = bus.Sample{
					Metric: count,
					Datapoint: bus.Datapoint{
						Value:     float64(*prom.Histogram.SampleCount),
						Timestamp: promTimestamp(prom),
					},
				}
				samples = append(samples, &s)
			case dto.MetricType_SUMMARY:
				m.Labels["__type__"] = "summary"
				for _, q := range prom.Summary.Quantile {
					myMetric := cloneMetric(m)
					myMetric.Labels["quantile"] = fmt.Sprint(q.GetQuantile())
					s := bus.Sample{
						Metric: myMetric,
						Datapoint: bus.Datapoint{
							Value:     float64(q.GetValue()),
							Timestamp: promTimestamp(prom),
						},
					}
					samples = append(samples, &s)
				}
				// do sum
				sum := cloneMetric(m)
				sum.Name = sum.Name + "_sum"
				s := bus.Sample{
					Metric: sum,
					Datapoint: bus.Datapoint{
						Value:     float64(*prom.Summary.SampleSum),
						Timestamp: promTimestamp(prom),
					},
				}
				samples = append(samples, &s)
				// do count
				count := cloneMetric(m)
				count.Name = count.Name + "_count"
				s = bus.Sample{
					Metric: count,
					Datapoint: bus.Datapoint{
						Value:     float64(*prom.Summary.SampleCount),
						Timestamp: promTimestamp(prom),
					},
				}
				samples = append(samples, &s)
			}
		}
	}
	return samples
}

func promTimestamp(m *dto.Metric) bus.Timestamp {
	if m.TimestampMs != nil {
		return bus.Timestamp(*m.TimestampMs)
	}
	return bus.Timestamp(time.Now().UnixNano() / 1e6)
}

func cloneMetric(m bus.Metric) bus.Metric {
	result := bus.Metric{
		Name:   m.Name,
		Labels: map[string]string{},
	}
	for k, v := range m.Labels {
		result.Labels[k] = v
	}
	return result
}
<commit_msg>convert: minor change to import order of promtesttosg.go (#24)<commit_after>package convert

import (
	"fmt"
	"io"
	"time"

	"github.com/digitalocean/vulcan/bus"

	dto "github.com/prometheus/client_model/go"
	"github.com/prometheus/common/expfmt"
)

// PromTextToSG reads prometheus metrics in text exposition format and
// produces a SampleGroup
func PromTextToSG(in io.Reader) (bus.SampleGroup, error) {
	fam := []*dto.MetricFamily{}
	dec := expfmt.NewDecoder(in, expfmt.FmtText)
	for {
		var f dto.MetricFamily
		err := dec.Decode(&f)
		if err != nil {
			if err == io.EOF {
				break
			}
			return bus.SampleGroup{}, err
		}
		fam = append(fam, &f)
	}
	sg := familyToSampleGroup(fam)
	return sg, nil
}

func familyToSampleGroup(metricFamilies []*dto.MetricFamily) bus.SampleGroup {
	samples := bus.SampleGroup{}
	for _, mf := range metricFamilies {
		for _, prom := range mf.GetMetric() {
			m := bus.Metric{
				Name:   mf.GetName(),
				Labels: map[string]string{},
			}
			for _, label := range prom.GetLabel() {
				m.Labels[label.GetName()] = label.GetValue()
			}
			switch mf.GetType() {
			case dto.MetricType_COUNTER:
				s := bus.Sample{
					Metric: m,
					Datapoint: bus.Datapoint{
						Value:     prom.Counter.GetValue(),
						Timestamp: promTimestamp(prom),
					},
				}
				s.Metric.Labels["__type__"] = "counter"
				samples = append(samples, &s)
			case dto.MetricType_GAUGE:
				s := bus.Sample{
					Metric: m,
					Datapoint: bus.Datapoint{
						Value:     prom.Gauge.GetValue(),
						Timestamp: promTimestamp(prom),
					},
				}
				s.Metric.Labels["__type__"] = "gauge"
				samples = append(samples, &s)
			case dto.MetricType_HISTOGRAM:
				// add bucket values, sum, and count for this prometheus metric
				m.Labels["__type__"] = "histogram"
				bucketName := m.Name + "_bucket"
				for _, b := range prom.Histogram.Bucket {
					myMetric := cloneMetric(m)
					myMetric.Name = bucketName
					myMetric.Labels["le"] = fmt.Sprint(b.GetUpperBound())
					s := bus.Sample{
						Metric: myMetric,
						Datapoint: bus.Datapoint{
							Value:     float64(b.GetCumulativeCount()),
							Timestamp: promTimestamp(prom),
						},
					}
					samples = append(samples, &s)
				}
				// do sum
				sum := cloneMetric(m)
				sum.Name = sum.Name + "_sum"
				s := bus.Sample{
					Metric: sum,
					Datapoint: bus.Datapoint{
						Value:     float64(*prom.Histogram.SampleSum),
						Timestamp: promTimestamp(prom),
					},
				}
				samples = append(samples, &s)
				// do count
				count := cloneMetric(m)
				count.Name = count.Name + "_count"
				s = bus.Sample{
					Metric: count,
					Datapoint: bus.Datapoint{
						Value:     float64(*prom.Histogram.SampleCount),
						Timestamp: promTimestamp(prom),
					},
				}
				samples = append(samples, &s)
			case dto.MetricType_SUMMARY:
				m.Labels["__type__"] = "summary"
				for _, q := range prom.Summary.Quantile {
					myMetric := cloneMetric(m)
					myMetric.Labels["quantile"] = fmt.Sprint(q.GetQuantile())
					s := bus.Sample{
						Metric: myMetric,
						Datapoint: bus.Datapoint{
							Value:     float64(q.GetValue()),
							Timestamp: promTimestamp(prom),
						},
					}
					samples = append(samples, &s)
				}
				// do sum
				sum := cloneMetric(m)
				sum.Name = sum.Name + "_sum"
				s := bus.Sample{
					Metric: sum,
					Datapoint: bus.Datapoint{
						Value:     float64(*prom.Summary.SampleSum),
						Timestamp: promTimestamp(prom),
					},
				}
				samples = append(samples, &s)
				// do count
				count := cloneMetric(m)
				count.Name = count.Name + "_count"
				s = bus.Sample{
					Metric: count,
					Datapoint: bus.Datapoint{
						Value:     float64(*prom.Summary.SampleCount),
						Timestamp: promTimestamp(prom),
					},
				}
				samples = append(samples, &s)
			}
		}
	}
	return samples
}

func promTimestamp(m *dto.Metric) bus.Timestamp {
	if m.TimestampMs != nil {
		return bus.Timestamp(*m.TimestampMs)
	}
	return bus.Timestamp(time.Now().UnixNano() / 1e6)
}

func cloneMetric(m bus.Metric) bus.Metric {
	result := bus.Metric{
		Name:   m.Name,
		Labels: map[string]string{},
	}
	for k, v := range m.Labels {
		result.Labels[k] = v
	}
	return result
}
<|endoftext|>


<commit_before>package ignorechat

import (
	"github.com/graffic/wanon/bot"
	"github.com/graffic/wanon/telegram"
	"github.com/op/go-logging"
)

var log = logging.MustGetLogger("wanon.messages.ignorechat")

type ignoreHandler struct {
	allowed map[int]bool
}

// IgnoreConfiguration configuration for the ignore message
type IgnoreConfiguration struct {
	Allow []int
}

func (handler *ignoreHandler) Check(message *telegram.Message, context *bot.Context) int {
	if handler.allowed[message.Chat.ID] {
		log.Debug("Accepted")
		return bot.RouteNothing
	}
	log.Debug("Ignored")
	return bot.RouteStop
}

func (handler *ignoreHandler) Handle(message *telegram.Message, context *bot.Context) {}

// Create creates the ignore handler
func Create(conf *bot.ConfService) bot.Handler {
	myConf := new(IgnoreConfiguration)
	conf.Get(myConf)
	log.Notice("Allowing only from: %d", myConf.Allow)

	allowed := make(map[int]bool)
	for _, chatID := range myConf.Allow {
		allowed[chatID] = true
	}

	return &ignoreHandler{allowed}
}
<commit_msg>Log the id of the message chat ignored<commit_after>package ignorechat

import (
	"fmt"

	"github.com/graffic/wanon/bot"
	"github.com/graffic/wanon/telegram"
	"github.com/op/go-logging"
)

var log = logging.MustGetLogger("wanon.messages.ignorechat")

type ignoreHandler struct {
	allowed map[int]bool
}

// IgnoreConfiguration configuration for the ignore message
type configuration struct {
	Allow []int
}

func (handler *ignoreHandler) Check(message *telegram.Message, context *bot.Context) int {
	if handler.allowed[message.Chat.ID] {
		log.Debug("Accepted")
		return bot.RouteNothing
	}
	log.Debug(fmt.Sprintf("Ignored from %d", message.Chat.ID))
	return bot.RouteStop
}

func (handler *ignoreHandler) Handle(message *telegram.Message, context *bot.Context) {}

// Create creates the ignore handler
func Create(conf *bot.ConfService) bot.Handler {
	myConf := new(configuration)
	conf.Get(myConf)
	log.Notice("Allowing only from: %d", myConf.Allow)

	allowed := make(map[int]bool)
	for _, chatID := range myConf.Allow {
		allowed[chatID] = true
	}

	return &ignoreHandler{allowed}
}
<|endoftext|>


<commit_before>package main
<commit_msg>Remove stub file<commit_after><|endoftext|>


<commit_before>// Copyright 2015 Canonical Ltd.
// Licensed under the AGPLv3, see LICENCE file for details.

package rackspace_test

import (
	gc "gopkg.in/check.v1"

	"github.com/juju/errors"
	"github.com/juju/juju/cloud"
	"github.com/juju/juju/environs"
	"github.com/juju/juju/environs/config"
	"github.com/juju/juju/provider/rackspace"
	coretesting "github.com/juju/juju/testing"
	"github.com/juju/testing"
)

type providerSuite struct {
	provider      environs.EnvironProvider
	innerProvider *fakeProvider
}

var _ = gc.Suite(&providerSuite{})

func (s *providerSuite) SetUpTest(c *gc.C) {
	s.innerProvider = new(fakeProvider)
	s.provider = rackspace.NewProvider(s.innerProvider)
}

func (s *providerSuite) TestValidate(c *gc.C) {
	cfg, err := config.New(config.UseDefaults, map[string]interface{}{
		"name":            "some-name",
		"type":            "some-type",
		"uuid":            coretesting.ModelTag.Id(),
		"controller-uuid": coretesting.ModelTag.Id(),
		"authorized-keys": "key",
	})
	c.Check(err, gc.IsNil)
	_, err = s.provider.Validate(cfg, nil)
	c.Check(err, gc.IsNil)
	s.innerProvider.CheckCallNames(c, "Validate")
}

func (s *providerSuite) TestBootstrapConfig(c *gc.C) {
	args := environs.BootstrapConfigParams{CloudRegion: "dfw"}
	s.provider.BootstrapConfig(args)

	expect := args
	expect.CloudRegion = "DFW"
	s.innerProvider.CheckCalls(c, []testing.StubCall{
		{"BootstrapConfig", []interface{}{expect}},
	})
}

type fakeProvider struct {
	testing.Stub
}

func (p *fakeProvider) Open(cfg *config.Config) (environs.Environ, error) {
	p.MethodCall(p, "Open", cfg)
	return nil, nil
}

func (p *fakeProvider) RestrictedConfigAttributes() []string {
	p.MethodCall(p, "RestrictedConfigAttributes")
	return nil
}

func (p *fakeProvider) PrepareForCreateEnvironment(cfg *config.Config) (*config.Config, error) {
	p.MethodCall(p, "PrepareForCreateEnvironment", cfg)
	return nil, nil
}

func (p *fakeProvider) BootstrapConfig(args environs.BootstrapConfigParams) (*config.Config, error) {
	p.MethodCall(p, "BootstrapConfig", args)
	return nil, nil
}

func (p *fakeProvider) PrepareForBootstrap(ctx environs.BootstrapContext, cfg *config.Config) (environs.Environ, error) {
	p.MethodCall(p, "PrepareForBootstrap", ctx, cfg)
	return nil, nil
}

func (p *fakeProvider) Validate(cfg, old *config.Config) (valid *config.Config, err error) {
	p.MethodCall(p, "Validate", cfg, old)
	return cfg, nil
}

func (p *fakeProvider) SecretAttrs(cfg *config.Config) (map[string]string, error) {
	p.MethodCall(p, "SecretAttrs", cfg)
	return nil, nil
}

func (p *fakeProvider) CredentialSchemas() map[cloud.AuthType]cloud.CredentialSchema {
	p.MethodCall(p, "CredentialSchemas")
	return nil
}

func (p *fakeProvider) DetectCredentials() (*cloud.CloudCredential, error) {
	p.MethodCall(p, "DetectCredentials")
	return nil, errors.NotFoundf("credentials")
}
<commit_msg>Address review comment<commit_after>// Copyright 2015 Canonical Ltd.
// Licensed under the AGPLv3, see LICENCE file for details.

package rackspace_test

import (
	"github.com/juju/testing"
	gc "gopkg.in/check.v1"

	"github.com/juju/errors"
	"github.com/juju/juju/cloud"
	"github.com/juju/juju/environs"
	"github.com/juju/juju/environs/config"
	"github.com/juju/juju/provider/rackspace"
	coretesting "github.com/juju/juju/testing"
)

type providerSuite struct {
	provider      environs.EnvironProvider
	innerProvider *fakeProvider
}

var _ = gc.Suite(&providerSuite{})

func (s *providerSuite) SetUpTest(c *gc.C) {
	s.innerProvider = new(fakeProvider)
	s.provider = rackspace.NewProvider(s.innerProvider)
}

func (s *providerSuite) TestValidate(c *gc.C) {
	cfg, err := config.New(config.UseDefaults, map[string]interface{}{
		"name":            "some-name",
		"type":            "some-type",
		"uuid":            coretesting.ModelTag.Id(),
		"controller-uuid": coretesting.ModelTag.Id(),
		"authorized-keys": "key",
	})
	c.Check(err, gc.IsNil)
	_, err = s.provider.Validate(cfg, nil)
	c.Check(err, gc.IsNil)
	s.innerProvider.CheckCallNames(c, "Validate")
}

func (s *providerSuite) TestBootstrapConfig(c *gc.C) {
	args := environs.BootstrapConfigParams{CloudRegion: "dfw"}
	s.provider.BootstrapConfig(args)

	expect := args
	expect.CloudRegion = "DFW"
	s.innerProvider.CheckCalls(c, []testing.StubCall{
		{"BootstrapConfig", []interface{}{expect}},
	})
}

type fakeProvider struct {
	testing.Stub
}

func (p *fakeProvider) Open(cfg *config.Config) (environs.Environ, error) {
	p.MethodCall(p, "Open", cfg)
	return nil, nil
}

func (p *fakeProvider) RestrictedConfigAttributes() []string {
	p.MethodCall(p, "RestrictedConfigAttributes")
	return nil
}

func (p *fakeProvider) PrepareForCreateEnvironment(cfg *config.Config) (*config.Config, error) {
	p.MethodCall(p, "PrepareForCreateEnvironment", cfg)
	return nil, nil
}

func (p *fakeProvider) BootstrapConfig(args environs.BootstrapConfigParams) (*config.Config, error) {
	p.MethodCall(p, "BootstrapConfig", args)
	return nil, nil
}

func (p *fakeProvider) PrepareForBootstrap(ctx environs.BootstrapContext, cfg *config.Config) (environs.Environ, error) {
	p.MethodCall(p, "PrepareForBootstrap", ctx, cfg)
	return nil, nil
}

func (p *fakeProvider) Validate(cfg, old *config.Config) (valid *config.Config, err error) {
	p.MethodCall(p, "Validate", cfg, old)
	return cfg, nil
}

func (p *fakeProvider) SecretAttrs(cfg *config.Config) (map[string]string, error) {
	p.MethodCall(p, "SecretAttrs", cfg)
	return nil, nil
}

func (p *fakeProvider) CredentialSchemas() map[cloud.AuthType]cloud.CredentialSchema {
	p.MethodCall(p, "CredentialSchemas")
	return nil
}

func (p *fakeProvider) DetectCredentials() (*cloud.CloudCredential, error) {
	p.MethodCall(p, "DetectCredentials")
	return nil, errors.NotFoundf("credentials")
}
<|endoftext|>


<commit_before>/*
Copyright 2020 The Kubernetes Authors.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package cloudup

import (
	"fmt"
	"net/url"

	"github.com/blang/semver/v4"

	"k8s.io/kops/pkg/apis/kops"
	"k8s.io/kops/pkg/assets"
	"k8s.io/kops/upup/pkg/fi"
	"k8s.io/kops/util/pkg/architectures"
	"k8s.io/kops/util/pkg/hashing"
)

const (
	// containerd packages URLs for v1.4.x+
	containerdVersionUrlAmd64 = "https://github.com/containerd/containerd/releases/download/v%s/cri-containerd-cni-%s-linux-amd64.tar.gz"
	// containerd packages URLs for v1.6.x (ARM64)+
	containerdVersionUrlArm64 = "https://github.com/containerd/containerd/releases/download/v%s/cri-containerd-cni-%s-linux-arm64.tar.gz"
	// containerd legacy packages URLs for v1.2.x and v1.3.x
	containerdLegacyUrlAmd64 = "https://storage.googleapis.com/cri-containerd-release/cri-containerd-%s.linux-amd64.tar.gz"
	// containerd version that is available for both AMD64 and ARM64, used in case the selected version is not available for ARM64
	containerdFallbackVersion = "1.4.11"
)

func findContainerdAsset(c *kops.Cluster, assetBuilder *assets.AssetBuilder, arch architectures.Architecture) (*url.URL, *hashing.Hash, error) {
	if c.Spec.Containerd == nil {
		return nil, nil, fmt.Errorf("unable to find containerd config")
	}
	containerd := c.Spec.Containerd

	if containerd.Packages != nil {
		if arch == architectures.ArchitectureAmd64 && containerd.Packages.UrlAmd64 != nil && containerd.Packages.HashAmd64 != nil {
			assetUrl := fi.StringValue(containerd.Packages.UrlAmd64)
			assetHash := fi.StringValue(containerd.Packages.HashAmd64)
			return findAssetsUrlHash(assetBuilder, assetUrl, assetHash)
		}
		if arch == architectures.ArchitectureArm64 && containerd.Packages.UrlArm64 != nil && containerd.Packages.HashArm64 != nil {
			assetUrl := fi.StringValue(containerd.Packages.UrlArm64)
			assetHash := fi.StringValue(containerd.Packages.HashArm64)
			return findAssetsUrlHash(assetBuilder, assetUrl, assetHash)
		}
	}

	version := fi.StringValue(containerd.Version)
	if version == "" {
		return nil, nil, fmt.Errorf("unable to find containerd version")
	}
	assetUrl, assetHash, err := findContainerdVersionUrlHash(arch, version)
	if err != nil {
		return nil, nil, err
	}

	return findAssetsUrlHash(assetBuilder, assetUrl, assetHash)
}

func findContainerdVersionUrlHash(arch architectures.Architecture, version string) (u string, h string, e error) {
	var containerdAssetUrl, containerdAssetHash string

	if findAllContainerdHashesAmd64()[version] != "" {
		var err error
		containerdAssetUrl, err = findContainerdVersionUrl(arch, version)
		if err != nil {
			return "", "", err
		}
		containerdAssetHash, err = findContainerdVersionHash(arch, version)
		if err != nil {
			return "", "", err
		}
	} else {
		// Fall back to Docker packages
		dv := findAllContainerdDockerMappings()[version]
		if dv != "" {
			var err error
			containerdAssetUrl, err = findDockerVersionUrl(arch, dv)
			if err != nil {
				return "", "", err
			}
			containerdAssetHash, err = findDockerVersionHash(arch, dv)
			if err != nil {
				return "", "", err
			}
			println(dv)
		} else {
			return "", "", fmt.Errorf("unknown url and hash for containerd version: %s - %s", arch, version)
		}
	}

	return containerdAssetUrl, containerdAssetHash, nil
}

func findContainerdVersionUrl(arch architectures.Architecture, version string) (string, error) {
	sv, err := semver.ParseTolerant(version)
	if err != nil {
		return "", fmt.Errorf("unable to parse version string: %q", version)
	}
	if sv.LT(semver.MustParse("1.3.4")) {
		return "", fmt.Errorf("unsupported legacy containerd version: %q", version)
	}

	var u string
	switch arch {
	case architectures.ArchitectureAmd64:
		if sv.GTE(semver.MustParse("1.3.8")) {
			u = fmt.Sprintf(containerdVersionUrlAmd64, version, version)
		} else {
			u = fmt.Sprintf(containerdLegacyUrlAmd64, version)
		}
	case architectures.ArchitectureArm64:
		if sv.GTE(semver.MustParse("1.6.0-beta.2")) {
			u = fmt.Sprintf(containerdVersionUrlArm64, version, version)
		} else if findAllContainerdHashesAmd64()[version] != "" {
			if findAllContainerdDockerMappings()[version] != "" {
				u = fmt.Sprintf(dockerVersionUrlArm64, findAllContainerdDockerMappings()[version])
			} else {
				u = fmt.Sprintf(dockerVersionUrlArm64, findAllContainerdDockerMappings()[containerdFallbackVersion])
			}
		}
	default:
		return "", fmt.Errorf("unknown arch: %q", arch)
	}

	if u == "" {
		return "", fmt.Errorf("unknown url for containerd version: %s - %s", arch, version)
	}

	return u, nil
}

func findContainerdVersionHash(arch architectures.Architecture, version string) (string, error) {
	sv, err := semver.ParseTolerant(version)
	if err != nil {
		return "", fmt.Errorf("unable to parse version string: %q", version)
	}
	if sv.LT(semver.MustParse("1.3.4")) {
		return "", fmt.Errorf("unsupported legacy containerd version: %q", version)
	}

	var h string
	switch arch {
	case architectures.ArchitectureAmd64:
		h = findAllContainerdHashesAmd64()[version]
	case architectures.ArchitectureArm64:
		if sv.GTE(semver.MustParse("1.6.0-beta.2")) {
			h = findAllContainerdHashesArm64()[version]
		} else if findAllContainerdHashesAmd64()[version] != "" {
			if findAllContainerdDockerMappings()[version] != "" {
				h = findAllDockerHashesArm64()[findAllContainerdDockerMappings()[version]]
			} else {
				h = findAllDockerHashesArm64()[findAllContainerdDockerMappings()[containerdFallbackVersion]]
			}
		}
	default:
		return "", fmt.Errorf("unknown arch: %q", arch)
	}

	if h == "" {
		return "", fmt.Errorf("unknown hash for containerd version: %s - %s", arch, version)
	}

	return h, nil
}

func findAllContainerdHashesAmd64() map[string]string {
	hashes := map[string]string{
		"1.3.4":        "4616971c3ad21c24f2f2320fa1c085577a91032a068dd56a41c7c4b71a458087",
		"1.3.9":        "96663699e0f888fbf232ae6629a367aa7421f6b95044e7ee5d4d4e02841fac75",
		"1.3.10":       "69e23e49cdf1232d475a77bf7ecd7145ff4a80295154e190125c4d8a20e241da",
		"1.4.0":        "b379f29417efd583f77e095173d4d0bd6bb001f0081b2a63d152ee7aef653ce1",
		"1.4.1":        "757efb93a4f3161efc447a943317503d8a7ded5cb4cc0cba3f3318d7ce1542ed",
		"1.4.2":        "9d0fd5f4d2bc58b345728432b7daac75fc99c1da91afa4f41e6103f618e74012",
		"1.4.3":        "2697a342e3477c211ab48313e259fd7e32ad1f5ded19320e6a559f50a82bff3d",
		"1.4.4":        "96641849cb78a0a119223a427dfdc1ade88412ef791a14193212c8c8e29d447b",
		"1.4.5":        "f8155278fd256526ca9804219e1ee46f5db11c6ddf455086b04c0887c868822a",
		"1.4.6":        "6ae4763598c9583f8b50605f19d6c7e9ef93c216706465e73dfc84ee6b63a238",
		"1.4.7":        "daa14638344fe0772f645e190e4d8eb9549b52743364cb8000521490f9e410b8",
		"1.4.8":        "96e815c9ab664a02dd5be35e31d15890ea6bef04dfaa39f99f14676c3d6561e8",
		"1.4.9":        "9911479f86012d6eab7e0f532da8f807a8b0f555ee09ef89367d8c31243073bb",
		"1.4.10":       "5b256b372a02fd37c84939c87a6a81cae06058a7881a60d682525c11f6dea7d1",
		"1.4.11":       "a4a4af4776316833cad5996c66d59f8b4a2af4da716b7902b7a2d5f5ac362dcc",
		"1.4.12":       "f6120552408175ca332fd3b5d31c5edd115d8426d6731664e4ea3951c5eee3b4",
		"1.5.0":        "aee7b553ab88842fdafe43955757abe746b8e9995b2be55c603f0a236186ff9b",
		"1.5.1":        "2fd97916b24396c13849cfcd89805170e1ef0265a2f7fce8e74ae044a6a6a169",
		"1.5.2":        "e7adbb6c6f6e67639460579a8aa991e9ce4de2062ed36d3261e6e4865574d947",
		"1.5.3":        "32a9bf1b7ab2adbd9d2a16b17bf1aa6e61592938655adfb5114c40d527aa9be7",
		"1.5.4":        "591e4e087ea2f5007e6c64deb382df58d419b7b6922eab45a1923d843d57615f",
		"1.5.5":        "45f02cfc65db47cf088c95555906e1dcba7baf5a3fbad3d947dd6b9af476a144",
		"1.5.6":        "afc51718ebe46cb9b985edac816e63fe86c07e37d28cdd21b2c0302dec6fa7ae",
		"1.5.7":        "7fce75bab43a39d6f9efb3c370de2da49723f0e1dbaa9732d68fa7f620d720c8",
		"1.6.0-beta.2": "617044417781ea9a700187955834f712f141b37fe94f2ab173b5f7a5ed2c26a7",
	}

	return hashes
}

func findAllContainerdHashesArm64() map[string]string {
	hashes := map[string]string{
		"1.6.0-beta.2": "8c55169cce0a198265ab9b8cad1691174c5d055f6b2468d159ac8b911fa3c0ec",
	}

	return hashes
}

func findAllContainerdDockerMappings() map[string]string {
	versions := map[string]string{
		"1.3.7":  "19.03.13",
		"1.3.9":  "19.03.14",
		"1.4.3":  "20.10.0",
		"1.4.4":  "20.10.6",
		"1.4.6":  "20.10.7",
		"1.4.9":  "20.10.8",
		"1.4.11": "20.10.9",
	}

	return versions
}
<commit_msg>Add hash for containerd v1.5.8<commit_after>/*
Copyright 2020 The Kubernetes Authors.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package cloudup

import (
	"fmt"
	"net/url"

	"github.com/blang/semver/v4"

	"k8s.io/kops/pkg/apis/kops"
	"k8s.io/kops/pkg/assets"
	"k8s.io/kops/upup/pkg/fi"
	"k8s.io/kops/util/pkg/architectures"
	"k8s.io/kops/util/pkg/hashing"
)

const (
	// containerd packages URLs for v1.4.x+
	containerdVersionUrlAmd64 = "https://github.com/containerd/containerd/releases/download/v%s/cri-containerd-cni-%s-linux-amd64.tar.gz"
	// containerd packages URLs for v1.6.x (ARM64)+
	containerdVersionUrlArm64 = "https://github.com/containerd/containerd/releases/download/v%s/cri-containerd-cni-%s-linux-arm64.tar.gz"
	// containerd legacy packages URLs for v1.2.x and v1.3.x
	containerdLegacyUrlAmd64 = "https://storage.googleapis.com/cri-containerd-release/cri-containerd-%s.linux-amd64.tar.gz"
	// containerd version that is available for both AMD64 and ARM64, used in case the selected version is not available for ARM64
	containerdFallbackVersion = "1.4.11"
)

func findContainerdAsset(c *kops.Cluster, assetBuilder *assets.AssetBuilder, arch architectures.Architecture) (*url.URL, *hashing.Hash, error) {
	if c.Spec.Containerd == nil {
		return nil, nil, fmt.Errorf("unable to find containerd config")
	}
	containerd := c.Spec.Containerd

	if containerd.Packages != nil {
		if arch == architectures.ArchitectureAmd64 && containerd.Packages.UrlAmd64 != nil && containerd.Packages.HashAmd64 != nil {
			assetUrl := fi.StringValue(containerd.Packages.UrlAmd64)
			assetHash := fi.StringValue(containerd.Packages.HashAmd64)
			return findAssetsUrlHash(assetBuilder, assetUrl, assetHash)
		}
		if arch == architectures.ArchitectureArm64 && containerd.Packages.UrlArm64 != nil && containerd.Packages.HashArm64 != nil {
			assetUrl := fi.StringValue(containerd.Packages.UrlArm64)
			assetHash := fi.StringValue(containerd.Packages.HashArm64)
			return findAssetsUrlHash(assetBuilder, assetUrl, assetHash)
		}
	}

	version := fi.StringValue(containerd.Version)
	if version == "" {
		return nil, nil, fmt.Errorf("unable to find containerd version")
	}
	assetUrl, assetHash, err := findContainerdVersionUrlHash(arch, version)
	if err != nil {
		return nil, nil, err
	}

	return findAssetsUrlHash(assetBuilder, assetUrl, assetHash)
}

func findContainerdVersionUrlHash(arch architectures.Architecture, version string) (u string, h string, e error) {
	var containerdAssetUrl, containerdAssetHash string

	if findAllContainerdHashesAmd64()[version] != "" {
		var err error
		containerdAssetUrl, err = findContainerdVersionUrl(arch, version)
		if err != nil {
			return "", "", err
		}
		containerdAssetHash, err = findContainerdVersionHash(arch, version)
		if err != nil {
			return "", "", err
		}
	} else {
		// Fall back to Docker packages
		dv := findAllContainerdDockerMappings()[version]
		if dv != "" {
			var err error
			containerdAssetUrl, err = findDockerVersionUrl(arch, dv)
			if err != nil {
				return "", "", err
			}
			containerdAssetHash, err = findDockerVersionHash(arch, dv)
			if err != nil {
				return "", "", err
			}
			println(dv)
		} else {
			return "", "", fmt.Errorf("unknown url and hash for containerd version: %s - %s", arch, version)
		}
	}

	return containerdAssetUrl, containerdAssetHash, nil
}

func findContainerdVersionUrl(arch architectures.Architecture, version string) (string, error) {
	sv, err := semver.ParseTolerant(version)
	if err != nil {
		return "", fmt.Errorf("unable to parse version string: %q", version)
	}
	if sv.LT(semver.MustParse("1.3.4")) {
		return "", fmt.Errorf("unsupported legacy containerd version: %q", version)
	}

	var u string
	switch arch {
	case architectures.ArchitectureAmd64:
		if sv.GTE(semver.MustParse("1.3.8")) {
			u = fmt.Sprintf(containerdVersionUrlAmd64, version, version)
		} else {
			u = fmt.Sprintf(containerdLegacyUrlAmd64, version)
		}
	case architectures.ArchitectureArm64:
		if sv.GTE(semver.MustParse("1.6.0-beta.2")) {
			u = fmt.Sprintf(containerdVersionUrlArm64, version, version)
		} else if findAllContainerdHashesAmd64()[version] != "" {
			if findAllContainerdDockerMappings()[version] != "" {
				u = fmt.Sprintf(dockerVersionUrlArm64, findAllContainerdDockerMappings()[version])
			} else {
				u = fmt.Sprintf(dockerVersionUrlArm64, findAllContainerdDockerMappings()[containerdFallbackVersion])
			}
		}
	default:
		return "", fmt.Errorf("unknown arch: %q", arch)
	}

	if u == "" {
		return "", fmt.Errorf("unknown url for containerd version: %s - %s", arch, version)
	}

	return u, nil
}

func findContainerdVersionHash(arch architectures.Architecture, version string) (string, error) {
	sv, err := semver.ParseTolerant(version)
	if err != nil {
		return "", fmt.Errorf("unable to parse version string: %q", version)
	}
	if sv.LT(semver.MustParse("1.3.4")) {
		return "", fmt.Errorf("unsupported legacy containerd version: %q", version)
	}

	var h string
	switch arch {
	case architectures.ArchitectureAmd64:
		h = findAllContainerdHashesAmd64()[version]
	case architectures.ArchitectureArm64:
		if sv.GTE(semver.MustParse("1.6.0-beta.2")) {
			h = findAllContainerdHashesArm64()[version]
		} else if findAllContainerdHashesAmd64()[version] != "" {
			if findAllContainerdDockerMappings()[version] != "" {
				h = findAllDockerHashesArm64()[findAllContainerdDockerMappings()[version]]
			} else {
				h = findAllDockerHashesArm64()[findAllContainerdDockerMappings()[containerdFallbackVersion]]
			}
		}
	default:
		return "", fmt.Errorf("unknown arch: %q", arch)
	}

	if h == "" {
		return "", fmt.Errorf("unknown hash for containerd version: %s - %s", arch, version)
	}

	return h, nil
}

func findAllContainerdHashesAmd64() map[string]string {
	hashes := map[string]string{
		"1.3.4":        "4616971c3ad21c24f2f2320fa1c085577a91032a068dd56a41c7c4b71a458087",
		"1.3.9":        "96663699e0f888fbf232ae6629a367aa7421f6b95044e7ee5d4d4e02841fac75",
		"1.3.10":       "69e23e49cdf1232d475a77bf7ecd7145ff4a80295154e190125c4d8a20e241da",
		"1.4.0":        "b379f29417efd583f77e095173d4d0bd6bb001f0081b2a63d152ee7aef653ce1",
		"1.4.1":        "757efb93a4f3161efc447a943317503d8a7ded5cb4cc0cba3f3318d7ce1542ed",
		"1.4.2":        "9d0fd5f4d2bc58b345728432b7daac75fc99c1da91afa4f41e6103f618e74012",
		"1.4.3":        "2697a342e3477c211ab48313e259fd7e32ad1f5ded19320e6a559f50a82bff3d",
		"1.4.4":        "96641849cb78a0a119223a427dfdc1ade88412ef791a14193212c8c8e29d447b",
		"1.4.5":        "f8155278fd256526ca9804219e1ee46f5db11c6ddf455086b04c0887c868822a",
		"1.4.6":        "6ae4763598c9583f8b50605f19d6c7e9ef93c216706465e73dfc84ee6b63a238",
		"1.4.7":        "daa14638344fe0772f645e190e4d8eb9549b52743364cb8000521490f9e410b8",
		"1.4.8":        "96e815c9ab664a02dd5be35e31d15890ea6bef04dfaa39f99f14676c3d6561e8",
		"1.4.9":        "9911479f86012d6eab7e0f532da8f807a8b0f555ee09ef89367d8c31243073bb",
		"1.4.10":       "5b256b372a02fd37c84939c87a6a81cae06058a7881a60d682525c11f6dea7d1",
		"1.4.11":       "a4a4af4776316833cad5996c66d59f8b4a2af4da716b7902b7a2d5f5ac362dcc",
		"1.4.12":       "f6120552408175ca332fd3b5d31c5edd115d8426d6731664e4ea3951c5eee3b4",
		"1.5.0":        "aee7b553ab88842fdafe43955757abe746b8e9995b2be55c603f0a236186ff9b",
		"1.5.1":        "2fd97916b24396c13849cfcd89805170e1ef0265a2f7fce8e74ae044a6a6a169",
		"1.5.2":        "e7adbb6c6f6e67639460579a8aa991e9ce4de2062ed36d3261e6e4865574d947",
		"1.5.3":        "32a9bf1b7ab2adbd9d2a16b17bf1aa6e61592938655adfb5114c40d527aa9be7",
		"1.5.4":        "591e4e087ea2f5007e6c64deb382df58d419b7b6922eab45a1923d843d57615f",
		"1.5.5":        "45f02cfc65db47cf088c95555906e1dcba7baf5a3fbad3d947dd6b9af476a144",
		"1.5.6":        "afc51718ebe46cb9b985edac816e63fe86c07e37d28cdd21b2c0302dec6fa7ae",
		"1.5.7":        "7fce75bab43a39d6f9efb3c370de2da49723f0e1dbaa9732d68fa7f620d720c8",
		"1.5.8":        "5dbb7f43c0ac1fda79999ff63e648926e3464d7d1034402ee2117e6a93868431",
		"1.6.0-beta.2": "617044417781ea9a700187955834f712f141b37fe94f2ab173b5f7a5ed2c26a7",
	}

	return hashes
}

func findAllContainerdHashesArm64() map[string]string {
	hashes := map[string]string{
		"1.6.0-beta.2": "8c55169cce0a198265ab9b8cad1691174c5d055f6b2468d159ac8b911fa3c0ec",
	}

	return hashes
}

func findAllContainerdDockerMappings() map[string]string {
	versions := map[string]string{
		"1.3.7":  "19.03.13",
		"1.3.9":  "19.03.14",
		"1.4.3":  "20.10.0",
		"1.4.4":  "20.10.6",
		"1.4.6":  "20.10.7",
		"1.4.9":  "20.10.8",
		"1.4.11": "20.10.9",
	}

	return versions
}
<|endoftext|>


<commit_before>// Copyright 2016 Google Inc. All Rights Reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//    http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

//go:generate ./COMPILE-PROTOS.sh

package main

import (
	"bytes"
	"encoding/json"
	"fmt"
	"io"
	"io/ioutil"
	"os"
	"os/exec"
	"regexp"

	"github.com/golang/protobuf/proto"
	"github.com/googleapis/openapi-compiler/OpenAPIv2"
	"github.com/googleapis/openapi-compiler/compiler"
	plugins "github.com/googleapis/openapi-compiler/plugins"
)

type PluginCall struct {
	Name   string
	Output string
}

func writeFile(name string, bytes []byte) {
	var writer io.Writer
	if name == "-" {
		writer = os.Stdout
	} else {
		file, _ := os.Create(name)
		defer file.Close()
		writer = file
	}
	writer.Write(bytes)
}

func main() {
	pb_regex, err := regexp.Compile("--pb_out=(.+)")
	json_regex, err := regexp.Compile("--json_out=(.+)")
	text_regex, err := regexp.Compile("--text_out=(.+)")
	errors_regex, err := regexp.Compile("--errors_out=(.+)")
	plugin_regex, err := regexp.Compile("--(.+)_out=(.+)")

	sourceName := ""
	binaryProtoFileName := ""
	jsonProtoFileName := ""
	textProtoFileName := ""
	errorFileName := ""
	keepReferences := false

	var pluginCalls []*PluginCall

	for i, arg := range os.Args {
		if i == 0 {
			continue
		}
		argbytes := []byte(arg)
		var m [][]byte
		if m = pb_regex.FindSubmatch(argbytes); m != nil {
			binaryProtoFileName = string(m[1])
		} else if m = json_regex.FindSubmatch(argbytes); m != nil {
			jsonProtoFileName = string(m[1])
		} else if m = text_regex.FindSubmatch(argbytes); m != nil {
			textProtoFileName = string(m[1])
		} else if m = errors_regex.FindSubmatch(argbytes); m != nil {
			errorFileName = string(m[1])
		} else if m = plugin_regex.FindSubmatch(argbytes); m != nil {
			pluginCall := &PluginCall{Name: string(m[1]), Output: string(m[2])}
			pluginCalls = append(pluginCalls, pluginCall)
		} else if arg == "--keep_refs" {
			keepReferences = true
		} else {
			sourceName = arg
		}
	}

	usage := `
Usage: openapic OPENAPI_SOURCE [OPTIONS]
  OPENAPI_SOURCE is the filename or URL of the OpenAPI description to read.
Options:
  --pb_out=FILENAME       Write a binary proto to a file with the specified name.
  --text_out=FILENAME     Write a text proto to a file with the specified name.
  --json_out=FILENAME     Write a json proto to a file with the specified name.
  --errors_out=FILENAME   Write compilation errors to a file with the specified name.
  --PLUGIN_out=FILENAME   Run the specified plugin and write results to a file with the specified name.
  --keep_refs             Disable resolution of $ref references.
`

	if textProtoFileName == "" &&
		jsonProtoFileName == "" &&
		binaryProtoFileName == "" &&
		errorFileName == "" &&
		len(pluginCalls) == 0 {
		fmt.Printf("Missing output directives.\n%s\n", usage)
		os.Exit(-1)
	}

	if sourceName == "" {
		fmt.Printf("No input specified.\n%s\n", usage)
		os.Exit(-1)
	}

	if errorFileName == "" {
		errorFileName = "-"
	}

	raw, err := compiler.ReadFile(sourceName)
	if err != nil {
		fmt.Printf("Error: %+v\n", err)
		os.Exit(-1)
	}

	document, err := openapi_v2.NewDocument(raw, compiler.NewContext("$root", nil))
	if err != nil {
		writeFile(errorFileName, []byte(err.Error()))
		os.Exit(-1)
	}

	if !keepReferences {
		_, err = document.ResolveReferences(sourceName)
		if err != nil {
			fmt.Printf("%+v\n", err)
			if errorFileName != "" {
				ioutil.WriteFile(errorFileName, []byte(err.Error()), 0644)
			}
			os.Exit(-1)
		}
	}

	if textProtoFileName != "" {
		bytes := []byte(proto.MarshalTextString(document))
		writeFile(textProtoFileName, bytes)
	}

	if jsonProtoFileName != "" {
		jsonBytes, _ := json.Marshal(document)
		writeFile(jsonProtoFileName, jsonBytes)
	}

	if binaryProtoFileName != "" {
		protoBytes, _ := proto.Marshal(document)
		writeFile(binaryProtoFileName, protoBytes)
	}

	for _, pluginCall := range pluginCalls {
		if pluginCall.Name != "" {
			request := &plugins.PluginRequest{}
			request.Parameter = ""

			version := &plugins.Version{}
			version.Major = 0
			version.Minor = 1
			version.Patch = 0
			request.CompilerVersion = version

			wrapper := &plugins.Wrapper{}
			wrapper.Name = sourceName
			wrapper.Version = "v2"
			protoBytes, _ := proto.Marshal(document)
			wrapper.Value = protoBytes
			request.Wrapper = []*plugins.Wrapper{wrapper}
			requestBytes, _ := proto.Marshal(request)

			cmd := exec.Command("openapi_" + pluginCall.Name)
			cmd.Stdin = bytes.NewReader(requestBytes)
			output, err := cmd.Output()
			if err != nil {
				fmt.Printf("Error: %+v\n", err)
			}
			response := &plugins.PluginResponse{}
			err = proto.Unmarshal(output, response)

			var writer io.Writer
			if pluginCall.Output == "-" {
				writer = os.Stdout
			} else {
				file, _ := os.Create(pluginCall.Output)
				defer file.Close()
				writer = file
			}
			for _, text := range response.Text {
				writer.Write([]byte(text))
			}
		}
	}
}
<commit_msg>Minor improvements to the way errors are printed.<commit_after>// Copyright 2016 Google Inc. All Rights Reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//    http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

//go:generate ./COMPILE-PROTOS.sh

package main

import (
	"bytes"
	"encoding/json"
	"fmt"
	"io"
	"os"
	"os/exec"
	"regexp"

	"github.com/golang/protobuf/proto"
	"github.com/googleapis/openapi-compiler/OpenAPIv2"
	"github.com/googleapis/openapi-compiler/compiler"
	plugins "github.com/googleapis/openapi-compiler/plugins"
)

type PluginCall struct {
	Name   string
	Output string
}

func writeFile(name string, bytes []byte) {
	var writer io.Writer
	if name == "-" {
		writer = os.Stdout
	} else {
		file, _ := os.Create(name)
		defer file.Close()
		writer = file
	}
	writer.Write(bytes)
	if name == "-" {
		writer.Write([]byte("\n"))
	}
}

func main() {
	pb_regex, err := regexp.Compile("--pb_out=(.+)")
	json_regex, err := regexp.Compile("--json_out=(.+)")
	text_regex, err := regexp.Compile("--text_out=(.+)")
	errors_regex, err := regexp.Compile("--errors_out=(.+)")
	plugin_regex, err := regexp.Compile("--(.+)_out=(.+)")

	sourceName := ""
	binaryProtoFileName := ""
	jsonProtoFileName := ""
	textProtoFileName := ""
	errorFileName := ""
	keepReferences := false

	var pluginCalls []*PluginCall

	for i, arg := range os.Args {
		if i == 0 {
			continue
		}
		argbytes := []byte(arg)
		var m [][]byte
		if m = pb_regex.FindSubmatch(argbytes); m != nil {
			binaryProtoFileName = string(m[1])
		} else if m = json_regex.FindSubmatch(argbytes); m != nil {
			jsonProtoFileName = string(m[1])
		} else if m = text_regex.FindSubmatch(argbytes); m != nil {
			textProtoFileName = string(m[1])
		} else if m = errors_regex.FindSubmatch(argbytes); m != nil {
			errorFileName = string(m[1])
		} else if m = plugin_regex.FindSubmatch(argbytes); m != nil {
			pluginCall := &PluginCall{Name: string(m[1]), Output: string(m[2])}
			pluginCalls = append(pluginCalls, pluginCall)
		} else if arg == "--keep_refs" {
			keepReferences = true
		} else {
			sourceName = arg
		}
	}

	usage := `
Usage: openapic OPENAPI_SOURCE [OPTIONS]
  OPENAPI_SOURCE is the filename or URL of the OpenAPI description to read.
Options:
  --pb_out=FILENAME       Write a binary proto to a file with the specified name.
  --text_out=FILENAME     Write a text proto to a file with the specified name.
  --json_out=FILENAME     Write a json proto to a file with the specified name.
  --errors_out=FILENAME   Write compilation errors to a file with the specified name.
  --PLUGIN_out=FILENAME   Run the specified plugin and write results to a file with the specified name.
  --keep_refs             Disable resolution of $ref references.
`

	if textProtoFileName == "" &&
		jsonProtoFileName == "" &&
		binaryProtoFileName == "" &&
		errorFileName == "" &&
		len(pluginCalls) == 0 {
		fmt.Printf("Missing output directives.\n%s\n", usage)
		os.Exit(-1)
	}

	if sourceName == "" {
		fmt.Printf("No input specified.\n%s\n", usage)
		os.Exit(-1)
	}

	if errorFileName == "" {
		errorFileName = "-"
	}

	raw, err := compiler.ReadFile(sourceName)
	if err != nil {
		fmt.Printf("Error: %+v\n", err)
		os.Exit(-1)
	}

	document, err := openapi_v2.NewDocument(raw, compiler.NewContext("$root", nil))
	if err != nil {
		writeFile(errorFileName, []byte(err.Error()))
		os.Exit(-1)
	}

	if !keepReferences {
		_, err = document.ResolveReferences(sourceName)
		if err != nil {
			writeFile(errorFileName, []byte(err.Error()))
			os.Exit(-1)
		}
	}

	if textProtoFileName != "" {
		bytes := []byte(proto.MarshalTextString(document))
		writeFile(textProtoFileName, bytes)
	}

	if jsonProtoFileName != "" {
		jsonBytes, _ := json.Marshal(document)
		writeFile(jsonProtoFileName, jsonBytes)
	}

	if binaryProtoFileName != "" {
		protoBytes, _ := proto.Marshal(document)
		writeFile(binaryProtoFileName, protoBytes)
	}

	for _, pluginCall := range pluginCalls {
		if pluginCall.Name != "" {
			request := &plugins.PluginRequest{}
			request.Parameter = ""

			version := &plugins.Version{}
			version.Major = 0
			version.Minor = 1
			version.Patch = 0
			request.CompilerVersion = version

			wrapper := &plugins.Wrapper{}
			wrapper.Name = sourceName
			wrapper.Version = "v2"
			protoBytes, _ := proto.Marshal(document)
			wrapper.Value = protoBytes
			request.Wrapper = []*plugins.Wrapper{wrapper}
			requestBytes, _ := proto.Marshal(request)

			cmd := exec.Command("openapi_" + pluginCall.Name)
			cmd.Stdin = bytes.NewReader(requestBytes)
			output, err := cmd.Output()
			if err != nil {
				fmt.Printf("Error: %+v\n", err)
			}
			response := &plugins.PluginResponse{}
			err = proto.Unmarshal(output, response)

			var writer io.Writer
			if pluginCall.Output == "-" {
				writer = os.Stdout
			} else {
				file, _ := os.Create(pluginCall.Output)
				defer file.Close()
				writer = file
			}
			for _, text := range response.Text {
				writer.Write([]byte(text))
			}
		}
	}
}
<|endoftext|>


<commit_before>// Tabletesting package provides helper structures and interfaces for dealing
// with table based tests.
package tabletesting

import (
    "reflect"
    "testing"
)

type TableTester interface {
    RunAgainst(interface{}, *testing.T) bool
}

type StringTableTester struct {
    Field string
    Value string
}

func (stt StringTableTester) RunAgainst(ti interface{}, t *testing.T) bool {
    retval := true
    actual := reflect.ValueOf(ti).FieldByName(stt.Field).Interface()
    expected := stt.Value
    if actual != expected {
        t.Logf("Fail on %s, expected %s, actual %s\n", stt.Field, expected, actual)
        retval = false
    }
    return retval
}
<commit_msg>adding proper error log for missing field name in test item<commit_after>// Tabletesting package provides helper structures and interfaces for dealing
// with table based tests.
package tabletesting

import (
    "reflect"
    "testing"
)

type TableTester interface {
    RunAgainst(interface{}, *testing.T) bool
}

type StringTableTester struct {
    Field string
    Value string
}

func (stt StringTableTester) RunAgainst(ti interface{}, t *testing.T) bool {
    field := reflect.ValueOf(ti).FieldByName(stt.Field)
    if field.IsValid() == false {
        t.Logf("Failed to find field %s\n", stt.Field)
        return false
    }
    actual := field.Interface()
    expected := stt.Value
    if actual != expected {
        t.Logf("Fail on %s, expected %s, actual %s\n", stt.Field, expected, actual)
        return false
    }
    return true
}
<|endoftext|>


<commit_before>package tag_test

import (
	"strings"
	"testing"

	"github.com/CotaPreco/Horus/tag"
)

var tags = map[string]bool{
	"#invalid": false,
	"#":        false,
	"":         false,
	strings.Repeat("invalid", 38): false,
	"valid":     true,
	":valid":    true,
	"**valid**": true,
}

func TestTag(t *testing.T) {
	for tstr, valid := range tags {
		tag, err := tag.NewTag(tstr)

		if (err != nil && valid) || (err == nil && !valid) {
			t.Error()
		}

		if err == nil {
			if tag.String() != tstr {
				t.Error()
			}
		}
	}
}
<commit_msg>Improving `Tag` tests..<commit_after>package tag_test

import (
	"strings"
	"testing"

	"github.com/CotaPreco/Horus/tag"
	"github.com/stretchr/testify/assert"
)

var tags = map[string]bool{
	"#invalid": false,
	"#":        false,
	"":         false,
	strings.Repeat("invalid", 38): false,
	"valid":     true,
	":valid":    true,
	"**valid**": true,
}

func TestTagCastsToString(t *testing.T) {
	tag, err := tag.NewTag(":user")

	assert.Nil(t, err)
	assert.Equal(t, ":user", tag.String())
}

func TestTag(t *testing.T) {
	for tagAsString, valid := range tags {
		_, err := tag.NewTag(tagAsString)

		if valid {
			assert.Nil(t, err)
		}

		if !valid {
			assert.NotNil(t, err)
		}
	}
}
<|endoftext|>


<commit_before>package cpu

import (
	"fmt"
	"os/exec"
	"regexp"
	"strconv"
	"strings"

	"github.com/shirou/gopsutil/internal/common"
)

// sys/resource.h
const (
	CPUser    = 0
	CPNice    = 1
	CPSys     = 2
	CPIntr    = 3
	CPIdle    = 4
	CPUStates = 5
)

var ClocksPerSec = float64(128)
var cpuMatch = regexp.MustCompile(`^CPU:`)
var originMatch = regexp.MustCompile(`Origin\s*=\s*"(.+)"\s+Id\s*=\s*(.+)\s+Family\s*=\s*(.+)\s+Model\s*=\s*(.+)\s+Stepping\s*=\s*(.+)`)
var featuresMatch = regexp.MustCompile(`Features=.+<(.+)>`)
var featuresMatch2 = regexp.MustCompile(`Features2=[a-f\dx]+<(.+)>`)
var cpuEnd = regexp.MustCompile(`^Trying to mount root`)
var cpuCores = regexp.MustCompile(`FreeBSD/SMP: (\d*) package\(s\) x (\d*) core\(s\)`)

func init() {
	getconf, err := exec.LookPath("/usr/bin/getconf")
	if err != nil {
		return
	}
	out, err := invoke.Command(getconf, "CLK_TCK")
	// ignore errors
	if err == nil {
		i, err := strconv.ParseFloat(strings.TrimSpace(string(out)), 64)
		if err == nil {
			ClocksPerSec = float64(i)
		}
	}
}

func Times(percpu bool) ([]TimesStat, error) {
	var ret []TimesStat

	var sysctlCall string
	var ncpu int
	if percpu {
		sysctlCall = "kern.cp_times"
		ncpu, _ = Counts(true)
	} else {
		sysctlCall = "kern.cp_time"
		ncpu = 1
	}

	cpuTimes, err := common.DoSysctrl(sysctlCall)
	if err != nil {
		return ret, err
	}

	for i := 0; i < ncpu; i++ {
		offset := CPUStates * i
		user, err := strconv.ParseFloat(cpuTimes[CPUser+offset], 64)
		if err != nil {
			return ret, err
		}
		nice, err := strconv.ParseFloat(cpuTimes[CPNice+offset], 64)
		if err != nil {
			return ret, err
		}
		sys, err := strconv.ParseFloat(cpuTimes[CPSys+offset], 64)
		if err != nil {
			return ret, err
		}
		idle, err := strconv.ParseFloat(cpuTimes[CPIdle+offset], 64)
		if err != nil {
			return ret, err
		}
		intr, err := strconv.ParseFloat(cpuTimes[CPIntr+offset], 64)
		if err != nil {
			return ret, err
		}

		c := TimesStat{
			User:   float64(user / ClocksPerSec),
			Nice:   float64(nice / ClocksPerSec),
			System: float64(sys / ClocksPerSec),
			Idle:   float64(idle / ClocksPerSec),
			Irq:    float64(intr / ClocksPerSec),
		}
		if !percpu {
			c.CPU = "cpu-total"
		} else {
			c.CPU = fmt.Sprintf("cpu%d", i)
		}

		ret = append(ret, c)
	}

	return ret, nil
}

// Returns only one InfoStat on FreeBSD.  The information regarding core
// count, however is accurate and it is assumed that all InfoStat attributes
// are the same across CPUs.
func Info() ([]InfoStat, error) {
	const dmesgBoot = "/var/run/dmesg.boot"

	c, num, err := parseDmesgBoot(dmesgBoot)
	if err != nil {
		return nil, err
	}
	var vals []string
	if vals, err = common.DoSysctrl("hw.clockrate"); err != nil {
		return nil, err
	}
	if c.Mhz, err = strconv.ParseFloat(vals[0], 64); err != nil {
		return nil, fmt.Errorf("unable to parse FreeBSD CPU clock rate: %v", err)
	}

	if vals, err = common.DoSysctrl("hw.ncpu"); err != nil {
		return nil, err
	}
	var i64 int64
	if i64, err = strconv.ParseInt(vals[0], 10, 32); err != nil {
		return nil, fmt.Errorf("unable to parse FreeBSD cores: %v", err)
	}
	c.Cores = int32(i64)

	if vals, err = common.DoSysctrl("hw.model"); err != nil {
		return nil, err
	}
	c.ModelName = strings.Join(vals, " ")

	ret := make([]InfoStat, num)
	for i := 0; i < num; i++ {
		ret[i] = c
	}

	return ret, nil
}

func parseDmesgBoot(fileName string) (InfoStat, int, error) {
	c := InfoStat{}
	lines, _ := common.ReadLines(fileName)
	var cpuNum int
	for _, line := range lines {
		if matches := cpuEnd.FindStringSubmatch(line); matches != nil {
			break
		} else if matches := originMatch.FindStringSubmatch(line); matches != nil {
			c.VendorID = matches[1]
			c.Family = matches[3]
			c.Model = matches[4]
			t, err := strconv.ParseInt(matches[5], 10, 32)
			if err != nil {
				return c, 0, fmt.Errorf("unable to parse FreeBSD CPU stepping information from %q: %v", line, err)
			}
			c.Stepping = int32(t)
		} else if matches := featuresMatch.FindStringSubmatch(line); matches != nil {
			for _, v := range strings.Split(matches[1], ",") {
				c.Flags = append(c.Flags, strings.ToLower(v))
			}
		} else if matches := featuresMatch2.FindStringSubmatch(line); matches != nil {
			for _, v := range strings.Split(matches[1], ",") {
				c.Flags = append(c.Flags, strings.ToLower(v))
			}
		} else if matches := cpuCores.FindStringSubmatch(line); matches != nil {
			t, err := strconv.ParseInt(matches[1], 10, 32)
			if err != nil {
				return c, 0, fmt.Errorf("unable to parse FreeBSD CPU Nums from %q: %v", line, err)
			}
			cpuNum = int(t)
			t2, err := strconv.ParseInt(matches[2], 10, 32)
			if err != nil {
				return c, 0, fmt.Errorf("unable to parse FreeBSD CPU cores from %q: %v", line, err)
			}
			c.Cores = int32(t2)
		}
	}

	return c, cpuNum, nil
}
<commit_msg>[freebsd] cpu: fix CPUInfo on 10.3R<commit_after>package cpu

import (
	"fmt"
	"os/exec"
	"regexp"
	"strconv"
	"strings"

	"github.com/shirou/gopsutil/internal/common"
)

// sys/resource.h
const (
	CPUser    = 0
	CPNice    = 1
	CPSys     = 2
	CPIntr    = 3
	CPIdle    = 4
	CPUStates = 5
)

var ClocksPerSec = float64(128)
var cpuMatch = regexp.MustCompile(`^CPU:`)
var originMatch = regexp.MustCompile(`Origin\s*=\s*"(.+)"\s+Id\s*=\s*(.+)\s+Family\s*=\s*(.+)\s+Model\s*=\s*(.+)\s+Stepping\s*=\s*(.+)`)
var featuresMatch = regexp.MustCompile(`Features=.+<(.+)>`)
var featuresMatch2 = regexp.MustCompile(`Features2=[a-f\dx]+<(.+)>`)
var cpuEnd = regexp.MustCompile(`^Trying to mount root`)
var cpuCores = regexp.MustCompile(`FreeBSD/SMP: (\d*) package\(s\) x (\d*) core\(s\)`)

func init() {
	getconf, err := exec.LookPath("/usr/bin/getconf")
	if err != nil {
		return
	}
	out, err := invoke.Command(getconf, "CLK_TCK")
	// ignore errors
	if err == nil {
		i, err := strconv.ParseFloat(strings.TrimSpace(string(out)), 64)
		if err == nil {
			ClocksPerSec = float64(i)
		}
	}
}

func Times(percpu bool) ([]TimesStat, error) {
	var ret []TimesStat

	var sysctlCall string
	var ncpu int
	if percpu {
		sysctlCall = "kern.cp_times"
		ncpu, _ = Counts(true)
	} else {
		sysctlCall = "kern.cp_time"
		ncpu = 1
	}

	cpuTimes, err := common.DoSysctrl(sysctlCall)
	if err != nil {
		return ret, err
	}

	for i := 0; i < ncpu; i++ {
		offset := CPUStates * i
		user, err := strconv.ParseFloat(cpuTimes[CPUser+offset], 64)
		if err != nil {
			return ret, err
		}
		nice, err := strconv.ParseFloat(cpuTimes[CPNice+offset], 64)
		if err != nil {
			return ret, err
		}
		sys, err := strconv.ParseFloat(cpuTimes[CPSys+offset], 64)
		if err != nil {
			return ret, err
		}
		idle, err := strconv.ParseFloat(cpuTimes[CPIdle+offset], 64)
		if err != nil {
			return ret, err
		}
		intr, err := strconv.ParseFloat(cpuTimes[CPIntr+offset], 64)
		if err != nil {
			return ret, err
		}

		c := TimesStat{
			User:   float64(user / ClocksPerSec),
			Nice:   float64(nice / ClocksPerSec),
			System: float64(sys / ClocksPerSec),
			Idle:   float64(idle / ClocksPerSec),
			Irq:    float64(intr / ClocksPerSec),
		}
		if !percpu {
			c.CPU = "cpu-total"
		} else {
			c.CPU = fmt.Sprintf("cpu%d", i)
		}

		ret = append(ret, c)
	}

	return ret, nil
}

// Returns only one InfoStat on FreeBSD.  The information regarding core
// count, however is accurate and it is assumed that all InfoStat attributes
// are the same across CPUs.
func Info() ([]InfoStat, error) {
	const dmesgBoot = "/var/run/dmesg.boot"

	c, num, err := parseDmesgBoot(dmesgBoot)
	if err != nil {
		return nil, err
	}
	var vals []string
	if vals, err = common.DoSysctrl("hw.clockrate"); err != nil {
		return nil, err
	}
	if c.Mhz, err = strconv.ParseFloat(vals[0], 64); err != nil {
		return nil, fmt.Errorf("unable to parse FreeBSD CPU clock rate: %v", err)
	}

	if vals, err = common.DoSysctrl("hw.ncpu"); err != nil {
		return nil, err
	}
	var i64 int64
	if i64, err = strconv.ParseInt(vals[0], 10, 32); err != nil {
		return nil, fmt.Errorf("unable to parse FreeBSD cores: %v", err)
	}
	c.Cores = int32(i64)

	if vals, err = common.DoSysctrl("hw.model"); err != nil {
		return nil, err
	}
	c.ModelName = strings.Join(vals, " ")

	ret := make([]InfoStat, num)
	for i := 0; i < num; i++ {
		ret[i] = c
	}

	return ret, nil
}

func parseDmesgBoot(fileName string) (InfoStat, int, error) {
	c := InfoStat{}
	lines, _ := common.ReadLines(fileName)
	cpuNum := 1 // default cpu num is 1
	for _, line := range lines {
		if matches := cpuEnd.FindStringSubmatch(line); matches != nil {
			break
		} else if matches := originMatch.FindStringSubmatch(line); matches != nil {
			c.VendorID = matches[1]
			c.Family = matches[3]
			c.Model = matches[4]
			t, err := strconv.ParseInt(matches[5], 10, 32)
			if err != nil {
				return c, 0, fmt.Errorf("unable to parse FreeBSD CPU stepping information from %q: %v", line, err)
			}
			c.Stepping = int32(t)
		} else if matches := featuresMatch.FindStringSubmatch(line); matches != nil {
			for _, v := range strings.Split(matches[1], ",") {
				c.Flags = append(c.Flags, strings.ToLower(v))
			}
		} else if matches := featuresMatch2.FindStringSubmatch(line); matches != nil {
			for _, v := range strings.Split(matches[1], ",") {
				c.Flags = append(c.Flags, strings.ToLower(v))
			}
		} else if matches := cpuCores.FindStringSubmatch(line); matches != nil {
			t, err := strconv.ParseInt(matches[1], 10, 32)
			if err != nil {
				return c, 0, fmt.Errorf("unable to parse FreeBSD CPU Nums from %q: %v", line, err)
			}
			cpuNum = int(t)
			t2, err := strconv.ParseInt(matches[2], 10, 32)
			if err != nil {
				return c, 0, fmt.Errorf("unable to parse FreeBSD CPU cores from %q: %v", line, err)
			}
			c.Cores = int32(t2)
		}
	}

	return c, cpuNum, nil
}
<|endoftext|>