<commit_before>// Package vultr implements a DNS provider for solving the DNS-01 challenge using the Vultr DNS.
// See https://www.vultr.com/api/#dns
package vultr

import (
	"context"
	"crypto/tls"
	"errors"
	"fmt"
	"net/http"
	"strconv"
	"strings"
	"time"

	"github.com/go-acme/lego/challenge/dns01"
	"github.com/go-acme/lego/platform/config/env"
	"github.com/vultr/govultr"
)

// Config is used to configure the creation of the DNSProvider
type Config struct {
	APIKey             string
	PropagationTimeout time.Duration
	PollingInterval    time.Duration
	TTL                int
	HTTPClient         *http.Client
}

// NewDefaultConfig returns a default configuration for the DNSProvider
func NewDefaultConfig() *Config {
	return &Config{
		TTL:                env.GetOrDefaultInt("VULTR_TTL", dns01.DefaultTTL),
		PropagationTimeout: env.GetOrDefaultSecond("VULTR_PROPAGATION_TIMEOUT", dns01.DefaultPropagationTimeout),
		PollingInterval:    env.GetOrDefaultSecond("VULTR_POLLING_INTERVAL", dns01.DefaultPollingInterval),
		HTTPClient: &http.Client{
			Timeout: env.GetOrDefaultSecond("VULTR_HTTP_TIMEOUT", 0),
			// from Vultr Client
			Transport: &http.Transport{
				TLSNextProto: make(map[string]func(string, *tls.Conn) http.RoundTripper),
			},
		},
	}
}

// DNSProvider is an implementation of the acme.ChallengeProvider interface.
type DNSProvider struct {
	config *Config
	client *govultr.Client
}

// NewDNSProvider returns a DNSProvider instance with a configured Vultr client.
// Authentication uses the VULTR_API_KEY environment variable.
func NewDNSProvider() (*DNSProvider, error) {
	values, err := env.Get("VULTR_API_KEY")
	if err != nil {
		return nil, fmt.Errorf("vultr: %v", err)
	}

	config := NewDefaultConfig()
	config.APIKey = values["VULTR_API_KEY"]

	return NewDNSProviderConfig(config)
}

// NewDNSProviderConfig return a DNSProvider instance configured for Vultr.
func NewDNSProviderConfig(config *Config) (*DNSProvider, error) {
	if config == nil {
		return nil, errors.New("vultr: the configuration of the DNS provider is nil")
	}

	if config.APIKey == "" {
		return nil, fmt.Errorf("vultr: credentials missing")
	}

	client := govultr.NewClient(config.HTTPClient, config.APIKey)

	return &DNSProvider{client: client, config: config}, nil
}

// Present creates a TXT record to fulfill the DNS-01 challenge.
func (d *DNSProvider) Present(domain, token, keyAuth string) error {
	ctx := context.Background()

	fqdn, value := dns01.GetRecord(domain, keyAuth)

	zoneDomain, err := d.getHostedZone(ctx, domain)
	if err != nil {
		return fmt.Errorf("vultr: %v", err)
	}

	name := d.extractRecordName(fqdn, zoneDomain)

	err = d.client.DNSRecord.Create(ctx, zoneDomain, "TXT", name, value, d.config.TTL, 0)
	if err != nil {
		return fmt.Errorf("vultr: API call failed: %v", err)
	}

	return nil
}

// CleanUp removes the TXT record matching the specified parameters.
func (d *DNSProvider) CleanUp(domain, token, keyAuth string) error {
	ctx := context.Background()

	fqdn, _ := dns01.GetRecord(domain, keyAuth)

	zoneDomain, records, err := d.findTxtRecords(ctx, domain, fqdn)
	if err != nil {
		return fmt.Errorf("vultr: %v", err)
	}

	var allErr []string
	for _, rec := range records {
		err := d.client.DNSRecord.Delete(ctx, zoneDomain, strconv.Itoa(rec.RecordID))
		if err != nil {
			allErr = append(allErr, err.Error())
		}
	}

	if len(allErr) > 0 {
		return errors.New(strings.Join(allErr, ": "))
	}

	return nil
}

// Timeout returns the timeout and interval to use when checking for DNS propagation.
// Adjusting here to cope with spikes in propagation times.
func (d *DNSProvider) Timeout() (timeout, interval time.Duration) {
	return d.config.PropagationTimeout, d.config.PollingInterval
}

func (d *DNSProvider) getHostedZone(ctx context.Context, domain string) (string, error) {
	domains, err := d.client.DNSDomain.List(ctx)
	if err != nil {
		return "", fmt.Errorf("API call failed: %v", err)
	}

	var hostedDomain govultr.DNSDomain
	for _, dom := range domains {
		if strings.HasSuffix(domain, dom.Domain) {
			if len(dom.Domain) > len(hostedDomain.Domain) {
				hostedDomain = dom
			}
		}
	}
	if hostedDomain.Domain == "" {
		return "", fmt.Errorf("no matching Vultr domain found for domain %s", domain)
	}

	return hostedDomain.Domain, nil
}

func (d *DNSProvider) findTxtRecords(ctx context.Context, domain, fqdn string) (string, []govultr.DNSRecord, error) {
	zoneDomain, err := d.getHostedZone(ctx, domain)
	if err != nil {
		return "", nil, err
	}

	var records []govultr.DNSRecord
	result, err := d.client.DNSRecord.List(ctx, zoneDomain)
	if err != nil {
		return "", records, fmt.Errorf("API call has failed: %v", err)
	}

	recordName := d.extractRecordName(fqdn, zoneDomain)
	for _, record := range result {
		if record.Type == "TXT" && record.Name == recordName {
			records = append(records, record)
		}
	}

	return zoneDomain, records, nil
}

func (d *DNSProvider) extractRecordName(fqdn, domain string) string {
	name := dns01.UnFqdn(fqdn)
	if idx := strings.Index(name, "."+domain); idx != -1 {
		return name[:idx]
	}
	return name
}
<commit_msg>vultr: quote TXT record (#940)<commit_after>// Package vultr implements a DNS provider for solving the DNS-01 challenge using the Vultr DNS.
// See https://www.vultr.com/api/#dns
package vultr

import (
	"context"
	"crypto/tls"
	"errors"
	"fmt"
	"net/http"
	"strconv"
	"strings"
	"time"

	"github.com/go-acme/lego/challenge/dns01"
	"github.com/go-acme/lego/platform/config/env"
	"github.com/vultr/govultr"
)

// Config is used to configure the creation of the DNSProvider
type Config struct {
	APIKey             string
	PropagationTimeout time.Duration
	PollingInterval    time.Duration
	TTL                int
	HTTPClient         *http.Client
}

// NewDefaultConfig returns a default configuration for the DNSProvider
func NewDefaultConfig() *Config {
	return &Config{
		TTL:                env.GetOrDefaultInt("VULTR_TTL", dns01.DefaultTTL),
		PropagationTimeout: env.GetOrDefaultSecond("VULTR_PROPAGATION_TIMEOUT", dns01.DefaultPropagationTimeout),
		PollingInterval:    env.GetOrDefaultSecond("VULTR_POLLING_INTERVAL", dns01.DefaultPollingInterval),
		HTTPClient: &http.Client{
			Timeout: env.GetOrDefaultSecond("VULTR_HTTP_TIMEOUT", 0),
			// from Vultr Client
			Transport: &http.Transport{
				TLSNextProto: make(map[string]func(string, *tls.Conn) http.RoundTripper),
			},
		},
	}
}

// DNSProvider is an implementation of the acme.ChallengeProvider interface.
type DNSProvider struct {
	config *Config
	client *govultr.Client
}

// NewDNSProvider returns a DNSProvider instance with a configured Vultr client.
// Authentication uses the VULTR_API_KEY environment variable.
func NewDNSProvider() (*DNSProvider, error) {
	values, err := env.Get("VULTR_API_KEY")
	if err != nil {
		return nil, fmt.Errorf("vultr: %v", err)
	}

	config := NewDefaultConfig()
	config.APIKey = values["VULTR_API_KEY"]

	return NewDNSProviderConfig(config)
}

// NewDNSProviderConfig return a DNSProvider instance configured for Vultr.
func NewDNSProviderConfig(config *Config) (*DNSProvider, error) {
	if config == nil {
		return nil, errors.New("vultr: the configuration of the DNS provider is nil")
	}

	if config.APIKey == "" {
		return nil, fmt.Errorf("vultr: credentials missing")
	}

	client := govultr.NewClient(config.HTTPClient, config.APIKey)

	return &DNSProvider{client: client, config: config}, nil
}

// Present creates a TXT record to fulfill the DNS-01 challenge.
func (d *DNSProvider) Present(domain, token, keyAuth string) error {
	ctx := context.Background()

	fqdn, value := dns01.GetRecord(domain, keyAuth)

	zoneDomain, err := d.getHostedZone(ctx, domain)
	if err != nil {
		return fmt.Errorf("vultr: %v", err)
	}

	name := d.extractRecordName(fqdn, zoneDomain)

	err = d.client.DNSRecord.Create(ctx, zoneDomain, "TXT", name, `"`+value+`"`, d.config.TTL, 0)
	if err != nil {
		return fmt.Errorf("vultr: API call failed: %v", err)
	}

	return nil
}

// CleanUp removes the TXT record matching the specified parameters.
func (d *DNSProvider) CleanUp(domain, token, keyAuth string) error {
	ctx := context.Background()

	fqdn, _ := dns01.GetRecord(domain, keyAuth)

	zoneDomain, records, err := d.findTxtRecords(ctx, domain, fqdn)
	if err != nil {
		return fmt.Errorf("vultr: %v", err)
	}

	var allErr []string
	for _, rec := range records {
		err := d.client.DNSRecord.Delete(ctx, zoneDomain, strconv.Itoa(rec.RecordID))
		if err != nil {
			allErr = append(allErr, err.Error())
		}
	}

	if len(allErr) > 0 {
		return errors.New(strings.Join(allErr, ": "))
	}

	return nil
}

// Timeout returns the timeout and interval to use when checking for DNS propagation.
// Adjusting here to cope with spikes in propagation times.
func (d *DNSProvider) Timeout() (timeout, interval time.Duration) {
	return d.config.PropagationTimeout, d.config.PollingInterval
}

func (d *DNSProvider) getHostedZone(ctx context.Context, domain string) (string, error) {
	domains, err := d.client.DNSDomain.List(ctx)
	if err != nil {
		return "", fmt.Errorf("API call failed: %v", err)
	}

	var hostedDomain govultr.DNSDomain
	for _, dom := range domains {
		if strings.HasSuffix(domain, dom.Domain) {
			if len(dom.Domain) > len(hostedDomain.Domain) {
				hostedDomain = dom
			}
		}
	}
	if hostedDomain.Domain == "" {
		return "", fmt.Errorf("no matching Vultr domain found for domain %s", domain)
	}

	return hostedDomain.Domain, nil
}

func (d *DNSProvider) findTxtRecords(ctx context.Context, domain, fqdn string) (string, []govultr.DNSRecord, error) {
	zoneDomain, err := d.getHostedZone(ctx, domain)
	if err != nil {
		return "", nil, err
	}

	var records []govultr.DNSRecord
	result, err := d.client.DNSRecord.List(ctx, zoneDomain)
	if err != nil {
		return "", records, fmt.Errorf("API call has failed: %v", err)
	}

	recordName := d.extractRecordName(fqdn, zoneDomain)
	for _, record := range result {
		if record.Type == "TXT" && record.Name == recordName {
			records = append(records, record)
		}
	}

	return zoneDomain, records, nil
}

func (d *DNSProvider) extractRecordName(fqdn, domain string) string {
	name := dns01.UnFqdn(fqdn)
	if idx := strings.Index(name, "."+domain); idx != -1 {
		return name[:idx]
	}
	return name
}
<|endoftext|>


<commit_before>package utils

import (
	"bytes"
	"fmt"
	"io/ioutil"
	"math/rand"
	"net/http"
	"os"
	"os/exec"
	"strings"
	"testing"
	"text/template"
	"time"

	"github.com/ThomasRooney/gexpect"
)

// Deis points to the CLI used to run tests.
var Deis = "deis "

// DeisTestConfig allows tests to be repeated against different
// targets, with different example apps, using specific credentials, and so on.
type DeisTestConfig struct {
	AuthKey     string
	Hosts       string
	Domain      string
	SSHKey      string
	ClusterName string
	UserName    string
	Password    string
	Email       string
	ExampleApp  string
	AppName     string
	ProcessNum  string
	ImageID     string
	Version     string
	AppUser     string
}

// randomApp is used for the test run if DEIS_TEST_APP isn't set
var randomApp = GetRandomApp()

// GetGlobalConfig returns a test configuration object.
func GetGlobalConfig() *DeisTestConfig {
	authKey := os.Getenv("DEIS_TEST_AUTH_KEY")
	if authKey == "" {
		authKey = "deis"
	}
	hosts := os.Getenv("DEIS_TEST_HOSTS")
	if hosts == "" {
		hosts = "172.17.8.100"
	}
	domain := os.Getenv("DEIS_TEST_DOMAIN")
	if domain == "" {
		domain = "local.deisapp.com"
	}
	sshKey := os.Getenv("DEIS_TEST_SSH_KEY")
	if sshKey == "" {
		sshKey = "~/.vagrant.d/insecure_private_key"
	}
	exampleApp := os.Getenv("DEIS_TEST_APP")
	if exampleApp == "" {
		exampleApp = randomApp
	}
	var envCfg = DeisTestConfig{
		AuthKey:     authKey,
		Hosts:       hosts,
		Domain:      domain,
		SSHKey:      sshKey,
		ClusterName: "dev",
		UserName:    "test",
		Password:    "asdf1234",
		Email:       "test@test.co.nz",
		ExampleApp:  exampleApp,
		AppName:     "sample",
		ProcessNum:  "2",
		ImageID:     "buildtest",
		Version:     "2",
		AppUser:     "test1",
	}
	return &envCfg
}

// Curl connects to a Deis endpoint to see if the example app is running.
func Curl(t *testing.T, params *DeisTestConfig) {
	url := "http://" + params.AppName + "." + params.Domain
	// FIXME: make an initial request to nginx to remove stale worker
	_, err := http.Get(url)
	if err != nil {
		t.Fatalf("not reachable:\n%v", err)
	}
	// FIXME: sleep a bit before curling
	time.Sleep(5000 * time.Millisecond)
	response, err := http.Get(url)
	if err != nil {
		t.Fatalf("not reachable:\n%v", err)
	}
	body, err := ioutil.ReadAll(response.Body)
	fmt.Println(string(body))
	if !strings.Contains(string(body), "Powered by Deis") {
		t.Fatalf("App not started")
	}
}

// AuthCancel tests whether `deis auth:cancel` destroys a user's account.
func AuthCancel(t *testing.T, params *DeisTestConfig) {
	fmt.Println("deis auth:cancel")
	child, err := gexpect.Spawn(Deis + " auth:cancel")
	if err != nil {
		t.Fatalf("command not started\n%v", err)
	}
	fmt.Println("username:")
	err = child.Expect("username:")
	if err != nil {
		t.Fatalf("expect username failed\n%v", err)
	}
	child.SendLine(params.UserName)
	fmt.Print("password:")
	err = child.Expect("password:")
	if err != nil {
		t.Fatalf("expect password failed\n%v", err)
	}
	child.SendLine(params.Password)
	err = child.ExpectRegex("(y/n)")
	if err != nil {
		t.Fatalf("expect cancel \n%v", err)
	}
	child.SendLine("y")
	err = child.Expect("Account cancelled")
	if err != nil {
		t.Fatalf("command executiuon failed\n%v", err)
	}
	child.Close()

}

// CheckList executes a command and optionally tests whether its output does
// or does not contain a given string.
func CheckList(
	t *testing.T, cmd string, params interface{}, contain string, notflag bool) {
	var cmdBuf bytes.Buffer
	tmpl := template.Must(template.New("cmd").Parse(cmd))
	if err := tmpl.Execute(&cmdBuf, params); err != nil {
		t.Fatal(err)
	}
	cmdString := cmdBuf.String()
	fmt.Println(cmdString)
	var cmdl *exec.Cmd
	if strings.Contains(cmd, "cat") {
		cmdl = exec.Command("sh", "-c", cmdString)
	} else {
		cmdl = exec.Command("sh", "-c", Deis+cmdString)
	}
	stdout, _, err := RunCommandWithStdoutStderr(cmdl)
	if err != nil {
		t.Fatal(err)
	}
	if strings.Contains(stdout.String(), contain) == notflag {
		if notflag {
			t.Fatalf(
				"Didn't expect '%s' in command output:\n%v", contain, stdout)
		}
		t.Fatalf("Expected '%s' in command output:\n%v", contain, stdout)
	}
}

// Execute takes command string and parameters required to execute the command,
// a failflag to check whether the command is expected to fail, and an expect
// string to check whether the command has failed according to failflag.
//
// If failflag is true and the command failed, check the stdout and stderr for
// the expect string.
func Execute(t *testing.T, cmd string, params interface{}, failFlag bool, expect string) {
	var cmdBuf bytes.Buffer
	tmpl := template.Must(template.New("cmd").Parse(cmd))
	if err := tmpl.Execute(&cmdBuf, params); err != nil {
		t.Fatal(err)
	}
	cmdString := cmdBuf.String()
	fmt.Println(cmdString)
	var cmdl *exec.Cmd
	if strings.Contains(cmd, "git") {
		cmdl = exec.Command("sh", "-c", cmdString)
	} else {
		cmdl = exec.Command("sh", "-c", Deis+cmdString)
	}

	switch failFlag {
	case true:
		if stdout, stderr, err := RunCommandWithStdoutStderr(cmdl); err != nil {
			if strings.Contains(stdout.String(), expect) || strings.Contains(stderr.String(), expect) {
				fmt.Println("(Error expected...ok)")
			} else {
				t.Fatal(err)
			}
		} else {
			if strings.Contains(stdout.String(), expect) || strings.Contains(stderr.String(), expect) {
				fmt.Println("(Error expected...ok)" + expect)
			} else {
				t.Fatal(err)
			}
		}
	case false:
		if _, _, err := RunCommandWithStdoutStderr(cmdl); err != nil {
			t.Fatal(err)
		} else {
			fmt.Println("ok")
		}
	}
}

// AppsDestroyTest destroys a Deis app and checks that it was successful.
func AppsDestroyTest(t *testing.T, params *DeisTestConfig) {
	cmd := "apps:destroy --app={{.AppName}} --confirm={{.AppName}}"
	if err := Chdir(params.ExampleApp); err != nil {
		t.Fatal(err)
	}
	Execute(t, cmd, params, false, "")
	if err := Chdir(".."); err != nil {
		t.Fatal(err)
	}
	if err := Rmdir(params.ExampleApp); err != nil {
		t.Fatal(err)
	}
}

// GetRandomApp returns a known working example app at random for testing.
func GetRandomApp() string {
	rand.Seed(int64(time.Now().Unix()))
	apps := []string{
		"example-clojure-ring",
		// "example-dart",
		"example-dockerfile-python",
		"example-go",
		"example-java-jetty",
		"example-nodejs-express",
		// "example-php",
		"example-play",
		"example-python-django",
		"example-python-flask",
		"example-ruby-sinatra",
		"example-scala",
	}
	return apps[rand.Intn(len(apps))]
}
<commit_msg>fix(tests): make curl loop a few times<commit_after>package utils

import (
	"bytes"
	"fmt"
	"io/ioutil"
	"math/rand"
	"net/http"
	"os"
	"os/exec"
	"strings"
	"testing"
	"text/template"
	"time"

	"github.com/ThomasRooney/gexpect"
)

// Deis points to the CLI used to run tests.
var Deis = "deis "

// DeisTestConfig allows tests to be repeated against different
// targets, with different example apps, using specific credentials, and so on.
type DeisTestConfig struct {
	AuthKey     string
	Hosts       string
	Domain      string
	SSHKey      string
	ClusterName string
	UserName    string
	Password    string
	Email       string
	ExampleApp  string
	AppName     string
	ProcessNum  string
	ImageID     string
	Version     string
	AppUser     string
}

// randomApp is used for the test run if DEIS_TEST_APP isn't set
var randomApp = GetRandomApp()

// GetGlobalConfig returns a test configuration object.
func GetGlobalConfig() *DeisTestConfig {
	authKey := os.Getenv("DEIS_TEST_AUTH_KEY")
	if authKey == "" {
		authKey = "deis"
	}
	hosts := os.Getenv("DEIS_TEST_HOSTS")
	if hosts == "" {
		hosts = "172.17.8.100"
	}
	domain := os.Getenv("DEIS_TEST_DOMAIN")
	if domain == "" {
		domain = "local.deisapp.com"
	}
	sshKey := os.Getenv("DEIS_TEST_SSH_KEY")
	if sshKey == "" {
		sshKey = "~/.vagrant.d/insecure_private_key"
	}
	exampleApp := os.Getenv("DEIS_TEST_APP")
	if exampleApp == "" {
		exampleApp = randomApp
	}
	var envCfg = DeisTestConfig{
		AuthKey:     authKey,
		Hosts:       hosts,
		Domain:      domain,
		SSHKey:      sshKey,
		ClusterName: "dev",
		UserName:    "test",
		Password:    "asdf1234",
		Email:       "test@test.co.nz",
		ExampleApp:  exampleApp,
		AppName:     "sample",
		ProcessNum:  "2",
		ImageID:     "buildtest",
		Version:     "2",
		AppUser:     "test1",
	}
	return &envCfg
}

func doCurl(url string) ([]byte, error) {
	response, err := http.Get(url)
	if err != nil {
		return nil, err
	}

	body, err := ioutil.ReadAll(response.Body)

	if !strings.Contains(string(body), "Powered by Deis") {
		return nil, fmt.Errorf("App not started")
	}

	return body, nil
}

// Curl connects to a Deis endpoint to see if the example app is running.
func Curl(t *testing.T, params *DeisTestConfig) {
	url := "http://" + params.AppName + "." + params.Domain

	// FIXME: try the curl a few times
	for i := 0; i < 20; i++ {
		body, err := doCurl(url)
		if err == nil {
			fmt.Println(string(body))
			return
		}
		time.Sleep(1 * time.Second)
	}

	// once more to fail with an error
	body, err := doCurl(url)
	if err != nil {
		t.Fatal(err)
	}
	fmt.Println(string(body))

}

// AuthCancel tests whether `deis auth:cancel` destroys a user's account.
func AuthCancel(t *testing.T, params *DeisTestConfig) {
	fmt.Println("deis auth:cancel")
	child, err := gexpect.Spawn(Deis + " auth:cancel")
	if err != nil {
		t.Fatalf("command not started\n%v", err)
	}
	fmt.Println("username:")
	err = child.Expect("username:")
	if err != nil {
		t.Fatalf("expect username failed\n%v", err)
	}
	child.SendLine(params.UserName)
	fmt.Print("password:")
	err = child.Expect("password:")
	if err != nil {
		t.Fatalf("expect password failed\n%v", err)
	}
	child.SendLine(params.Password)
	err = child.ExpectRegex("(y/n)")
	if err != nil {
		t.Fatalf("expect cancel \n%v", err)
	}
	child.SendLine("y")
	err = child.Expect("Account cancelled")
	if err != nil {
		t.Fatalf("command executiuon failed\n%v", err)
	}
	child.Close()

}

// CheckList executes a command and optionally tests whether its output does
// or does not contain a given string.
func CheckList(
	t *testing.T, cmd string, params interface{}, contain string, notflag bool) {
	var cmdBuf bytes.Buffer
	tmpl := template.Must(template.New("cmd").Parse(cmd))
	if err := tmpl.Execute(&cmdBuf, params); err != nil {
		t.Fatal(err)
	}
	cmdString := cmdBuf.String()
	fmt.Println(cmdString)
	var cmdl *exec.Cmd
	if strings.Contains(cmd, "cat") {
		cmdl = exec.Command("sh", "-c", cmdString)
	} else {
		cmdl = exec.Command("sh", "-c", Deis+cmdString)
	}
	stdout, _, err := RunCommandWithStdoutStderr(cmdl)
	if err != nil {
		t.Fatal(err)
	}
	if strings.Contains(stdout.String(), contain) == notflag {
		if notflag {
			t.Fatalf(
				"Didn't expect '%s' in command output:\n%v", contain, stdout)
		}
		t.Fatalf("Expected '%s' in command output:\n%v", contain, stdout)
	}
}

// Execute takes command string and parameters required to execute the command,
// a failflag to check whether the command is expected to fail, and an expect
// string to check whether the command has failed according to failflag.
//
// If failflag is true and the command failed, check the stdout and stderr for
// the expect string.
func Execute(t *testing.T, cmd string, params interface{}, failFlag bool, expect string) {
	var cmdBuf bytes.Buffer
	tmpl := template.Must(template.New("cmd").Parse(cmd))
	if err := tmpl.Execute(&cmdBuf, params); err != nil {
		t.Fatal(err)
	}
	cmdString := cmdBuf.String()
	fmt.Println(cmdString)
	var cmdl *exec.Cmd
	if strings.Contains(cmd, "git") {
		cmdl = exec.Command("sh", "-c", cmdString)
	} else {
		cmdl = exec.Command("sh", "-c", Deis+cmdString)
	}

	switch failFlag {
	case true:
		if stdout, stderr, err := RunCommandWithStdoutStderr(cmdl); err != nil {
			if strings.Contains(stdout.String(), expect) || strings.Contains(stderr.String(), expect) {
				fmt.Println("(Error expected...ok)")
			} else {
				t.Fatal(err)
			}
		} else {
			if strings.Contains(stdout.String(), expect) || strings.Contains(stderr.String(), expect) {
				fmt.Println("(Error expected...ok)" + expect)
			} else {
				t.Fatal(err)
			}
		}
	case false:
		if _, _, err := RunCommandWithStdoutStderr(cmdl); err != nil {
			t.Fatal(err)
		} else {
			fmt.Println("ok")
		}
	}
}

// AppsDestroyTest destroys a Deis app and checks that it was successful.
func AppsDestroyTest(t *testing.T, params *DeisTestConfig) {
	cmd := "apps:destroy --app={{.AppName}} --confirm={{.AppName}}"
	if err := Chdir(params.ExampleApp); err != nil {
		t.Fatal(err)
	}
	Execute(t, cmd, params, false, "")
	if err := Chdir(".."); err != nil {
		t.Fatal(err)
	}
	if err := Rmdir(params.ExampleApp); err != nil {
		t.Fatal(err)
	}
}

// GetRandomApp returns a known working example app at random for testing.
func GetRandomApp() string {
	rand.Seed(int64(time.Now().Unix()))
	apps := []string{
		"example-clojure-ring",
		// "example-dart",
		"example-dockerfile-python",
		"example-go",
		"example-java-jetty",
		"example-nodejs-express",
		// "example-php",
		"example-play",
		"example-python-django",
		"example-python-flask",
		"example-ruby-sinatra",
		"example-scala",
	}
	return apps[rand.Intn(len(apps))]
}
<|endoftext|>


<commit_before>/*
Copyright 2018 The Kubernetes Authors.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

// Package client implements a client that can handle multiple gerrit instances
// derived from https://github.com/andygrunwald/go-gerrit
package client

import (
	"fmt"
	"io/ioutil"
	"net/url"
	"strings"
	"time"

	"github.com/andygrunwald/go-gerrit"
	"github.com/sirupsen/logrus"
)

const (
	// LGTM means all presubmits passed, but need someone else to approve before merge.
	LGTM = "+1"
	// LBTM means some presubmits failed, perfer not merge.
	LBTM = "-1"
	// CodeReview is the default gerrit code review label
	CodeReview = "Code-Review"

	// GerritID identifies a gerrit change
	GerritID = "prow.k8s.io/gerrit-id"
	// GerritInstance is the gerrit host url
	GerritInstance = "prow.k8s.io/gerrit-instance"
	// GerritRevision is the SHA of current patchset from a gerrit change
	GerritRevision = "prow.k8s.io/gerrit-revision"
	// GerritReportLabel is the gerrit label prow will cast vote on, fallback to CodeReview label if unset
	GerritReportLabel = "prow.k8s.io/gerrit-report-label"

	// Merged status indicates a Gerrit change has been merged
	Merged = "MERGED"
	// New status indicates a Gerrit change is new (ie pending)
	New = "NEW"
)

// ProjectsFlag is the flag type for gerrit projects when initializing a gerrit client
type ProjectsFlag map[string][]string

func (p ProjectsFlag) String() string {
	var hosts []string
	for host, repos := range p {
		hosts = append(hosts, host+"="+strings.Join(repos, ","))
	}
	return strings.Join(hosts, " ")
}

// Set populates ProjectsFlag upon flag.Parse()
func (p ProjectsFlag) Set(value string) error {
	parts := strings.SplitN(value, "=", 2)
	if len(parts) != 2 {
		return fmt.Errorf("%s not in the form of host=repo-a,repo-b,etc", value)
	}
	host := parts[0]
	if _, ok := p[host]; ok {
		return fmt.Errorf("duplicate host: %s", host)
	}
	repos := strings.Split(parts[1], ",")
	p[host] = repos
	return nil
}

type gerritAuthentication interface {
	SetCookieAuth(name, value string)
}

type gerritAccount interface {
	GetAccount(name string) (*gerrit.AccountInfo, *gerrit.Response, error)
	SetUsername(accountID string, input *gerrit.UsernameInput) (*string, *gerrit.Response, error)
}

type gerritChange interface {
	QueryChanges(opt *gerrit.QueryChangeOptions) (*[]gerrit.ChangeInfo, *gerrit.Response, error)
	SetReview(changeID, revisionID string, input *gerrit.ReviewInput) (*gerrit.ReviewResult, *gerrit.Response, error)
}

type gerritProjects interface {
	GetBranch(projectName, branchID string) (*gerrit.BranchInfo, *gerrit.Response, error)
}

// gerritInstanceHandler holds all actual gerrit handlers
type gerritInstanceHandler struct {
	instance string
	projects []string

	authService    gerritAuthentication
	accountService gerritAccount
	changeService  gerritChange
	projectService gerritProjects
}

// Client holds a instance:handler map
type Client struct {
	handlers map[string]*gerritInstanceHandler
}

// ChangeInfo is a gerrit.ChangeInfo
type ChangeInfo = gerrit.ChangeInfo

// RevisionInfo is a gerrit.RevisionInfo
type RevisionInfo = gerrit.RevisionInfo

// FileInfo is a gerrit.FileInfo
type FileInfo = gerrit.FileInfo

// NewClient returns a new gerrit client
func NewClient(instances map[string][]string) (*Client, error) {
	c := &Client{
		handlers: map[string]*gerritInstanceHandler{},
	}
	for instance := range instances {
		gc, err := gerrit.NewClient(instance, nil)
		if err != nil {
			return nil, err
		}

		c.handlers[instance] = &gerritInstanceHandler{
			instance:       instance,
			projects:       instances[instance],
			authService:    gc.Authentication,
			accountService: gc.Accounts,
			changeService:  gc.Changes,
			projectService: gc.Projects,
		}
	}

	return c, nil
}

func auth(c *Client, cookiefilePath string) {
	logrus.Info("Starting auth loop...")
	var previousToken string
	wait := 10 * time.Minute
	for {
		raw, err := ioutil.ReadFile(cookiefilePath)
		if err != nil {
			logrus.WithError(err).Error("Failed to read auth cookie")
		}
		fields := strings.Fields(string(raw))
		token := fields[len(fields)-1]

		if token == previousToken {
			time.Sleep(wait)
			continue
		}

		logrus.Info("New token, updating handlers...")

		// update auth token for each instance
		for _, handler := range c.handlers {
			handler.authService.SetCookieAuth("o", token)

			self, _, err := handler.accountService.GetAccount("self")
			if err != nil {
				logrus.WithError(err).Error("Failed to auth with token")
				continue
			}

			logrus.Infof("Authentication to %s successful, Username: %s", handler.instance, self.Name)
		}
		previousToken = token
		time.Sleep(wait)
	}
}

// Start will authenticate the client with gerrit periodically
// Start must be called before user calls any client functions.
func (c *Client) Start(cookiefilePath string) {
	if cookiefilePath != "" {
		go auth(c, cookiefilePath)
	}
}

// QueryChanges queries for all changes from all projects after lastUpdate time
// returns an instance:changes map
func (c *Client) QueryChanges(lastUpdate time.Time, rateLimit int) map[string][]ChangeInfo {
	result := map[string][]ChangeInfo{}
	for _, h := range c.handlers {
		changes := h.queryAllChanges(lastUpdate, rateLimit)
		if len(changes) > 0 {
			result[h.instance] = []ChangeInfo{}
			for _, change := range changes {
				result[h.instance] = append(result[h.instance], change)
			}
		}
	}
	return result
}

// SetReview writes a review comment base on the change id + revision
func (c *Client) SetReview(instance, id, revision, message string, labels map[string]string) error {
	h, ok := c.handlers[instance]
	if !ok {
		return fmt.Errorf("not activated gerrit instance: %s", instance)
	}

	if _, _, err := h.changeService.SetReview(id, revision, &gerrit.ReviewInput{
		Message: message,
		Labels:  labels,
	}); err != nil {
		return fmt.Errorf("cannot comment to gerrit: %v", err)
	}

	return nil
}

// GetBranchRevision returns SHA of HEAD of a branch
func (c *Client) GetBranchRevision(instance, project, branch string) (string, error) {
	h, ok := c.handlers[instance]
	if !ok {
		return "", fmt.Errorf("not activated gerrit instance: %s", instance)
	}

	res, _, err := h.projectService.GetBranch(project, url.QueryEscape(branch))
	if err != nil {
		return "", err
	}

	return res.Revision, nil
}

// private handler implementation details

func (h *gerritInstanceHandler) queryAllChanges(lastUpdate time.Time, rateLimit int) []gerrit.ChangeInfo {
	result := []gerrit.ChangeInfo{}
	for _, project := range h.projects {
		changes, err := h.queryChangesForProject(project, lastUpdate, rateLimit)
		if err != nil {
			// don't halt on error from one project, log & continue
			logrus.WithError(err).Errorf("fail to query changes for project %s", project)
			continue
		}
		result = append(result, changes...)
	}

	return result
}

func parseStamp(value gerrit.Timestamp) time.Time {
	return value.Time
}

func (h *gerritInstanceHandler) queryChangesForProject(project string, lastUpdate time.Time, rateLimit int) ([]gerrit.ChangeInfo, error) {
	pending := []gerrit.ChangeInfo{}

	opt := &gerrit.QueryChangeOptions{}
	opt.Query = append(opt.Query, "project:"+project)
	opt.AdditionalFields = []string{"CURRENT_REVISION", "CURRENT_COMMIT", "CURRENT_FILES", "MESSAGES"}

	start := 0

	for {
		opt.Limit = rateLimit
		opt.Start = start

		// The change output is sorted by the last update time, most recently updated to oldest updated.
		// Gerrit API docs: https://gerrit-review.googlesource.com/Documentation/rest-api-changes.html#list-changes
		changes, _, err := h.changeService.QueryChanges(opt)
		if err != nil {
			// should not happen? Let next sync loop catch up
			return nil, fmt.Errorf("failed to query gerrit changes: %v", err)
		}

		if changes == nil || len(*changes) == 0 {
			logrus.Infof("no more changes from query, returning...")
			return pending, nil
		}

		logrus.Infof("Find %d changes from query %v", len(*changes), opt.Query)

		start += len(*changes)

		for _, change := range *changes {
			// if we already processed this change, then we stop the current sync loop
			updated := parseStamp(change.Updated)

			logrus.Infof("Change %d, last updated %s, status %s", change.Number, change.Updated, change.Status)

			// process if updated later than last updated
			// stop if update was stale
			if updated.After(lastUpdate) {
				switch change.Status {
				case Merged:
					submitted := parseStamp(*change.Submitted)
					if submitted.Before(lastUpdate) {
						logrus.Infof("Change %d, submitted %s before lastUpdate %s, skipping this patchset", change.Number, submitted, lastUpdate)
						continue
					}
					pending = append(pending, change)
				case New:
					// we need to make sure the change update is from a fresh commit change
					rev, ok := change.Revisions[change.CurrentRevision]
					if !ok {
						logrus.WithError(err).Errorf("(should not happen?)cannot find current revision for change %v", change.ID)
						continue
					}

					created := parseStamp(rev.Created)
					changeMessages := change.Messages
					newMessages := false

					for _, message := range changeMessages {
						if message.RevisionNumber == rev.Number {
							messageTime := parseStamp(message.Date)
							if messageTime.After(lastUpdate) {
								newMessages = true
								break
							}
						}
					}

					if !newMessages && created.Before(lastUpdate) {
						// stale commit
						logrus.Infof("Change %d, latest revision updated %s before lastUpdate %s, skipping this patchset", change.Number, created, lastUpdate)
						continue
					}

					pending = append(pending, change)
				default:
					// change has been abandoned, do nothing
				}
			} else {
				logrus.Infof("Change %d, updated %s before lastUpdate %s, return", change.Number, change.Updated, lastUpdate)
				return pending, nil
			}
		}
	}
}
<commit_msg>Add some logging for new messages on changes.<commit_after>/*
Copyright 2018 The Kubernetes Authors.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

// Package client implements a client that can handle multiple gerrit instances
// derived from https://github.com/andygrunwald/go-gerrit
package client

import (
	"fmt"
	"io/ioutil"
	"net/url"
	"strings"
	"time"

	"github.com/andygrunwald/go-gerrit"
	"github.com/sirupsen/logrus"
)

const (
	// LGTM means all presubmits passed, but need someone else to approve before merge.
	LGTM = "+1"
	// LBTM means some presubmits failed, perfer not merge.
	LBTM = "-1"
	// CodeReview is the default gerrit code review label
	CodeReview = "Code-Review"

	// GerritID identifies a gerrit change
	GerritID = "prow.k8s.io/gerrit-id"
	// GerritInstance is the gerrit host url
	GerritInstance = "prow.k8s.io/gerrit-instance"
	// GerritRevision is the SHA of current patchset from a gerrit change
	GerritRevision = "prow.k8s.io/gerrit-revision"
	// GerritReportLabel is the gerrit label prow will cast vote on, fallback to CodeReview label if unset
	GerritReportLabel = "prow.k8s.io/gerrit-report-label"

	// Merged status indicates a Gerrit change has been merged
	Merged = "MERGED"
	// New status indicates a Gerrit change is new (ie pending)
	New = "NEW"
)

// ProjectsFlag is the flag type for gerrit projects when initializing a gerrit client
type ProjectsFlag map[string][]string

func (p ProjectsFlag) String() string {
	var hosts []string
	for host, repos := range p {
		hosts = append(hosts, host+"="+strings.Join(repos, ","))
	}
	return strings.Join(hosts, " ")
}

// Set populates ProjectsFlag upon flag.Parse()
func (p ProjectsFlag) Set(value string) error {
	parts := strings.SplitN(value, "=", 2)
	if len(parts) != 2 {
		return fmt.Errorf("%s not in the form of host=repo-a,repo-b,etc", value)
	}
	host := parts[0]
	if _, ok := p[host]; ok {
		return fmt.Errorf("duplicate host: %s", host)
	}
	repos := strings.Split(parts[1], ",")
	p[host] = repos
	return nil
}

type gerritAuthentication interface {
	SetCookieAuth(name, value string)
}

type gerritAccount interface {
	GetAccount(name string) (*gerrit.AccountInfo, *gerrit.Response, error)
	SetUsername(accountID string, input *gerrit.UsernameInput) (*string, *gerrit.Response, error)
}

type gerritChange interface {
	QueryChanges(opt *gerrit.QueryChangeOptions) (*[]gerrit.ChangeInfo, *gerrit.Response, error)
	SetReview(changeID, revisionID string, input *gerrit.ReviewInput) (*gerrit.ReviewResult, *gerrit.Response, error)
}

type gerritProjects interface {
	GetBranch(projectName, branchID string) (*gerrit.BranchInfo, *gerrit.Response, error)
}

// gerritInstanceHandler holds all actual gerrit handlers
type gerritInstanceHandler struct {
	instance string
	projects []string

	authService    gerritAuthentication
	accountService gerritAccount
	changeService  gerritChange
	projectService gerritProjects
}

// Client holds a instance:handler map
type Client struct {
	handlers map[string]*gerritInstanceHandler
}

// ChangeInfo is a gerrit.ChangeInfo
type ChangeInfo = gerrit.ChangeInfo

// RevisionInfo is a gerrit.RevisionInfo
type RevisionInfo = gerrit.RevisionInfo

// FileInfo is a gerrit.FileInfo
type FileInfo = gerrit.FileInfo

// NewClient returns a new gerrit client
func NewClient(instances map[string][]string) (*Client, error) {
	c := &Client{
		handlers: map[string]*gerritInstanceHandler{},
	}
	for instance := range instances {
		gc, err := gerrit.NewClient(instance, nil)
		if err != nil {
			return nil, err
		}

		c.handlers[instance] = &gerritInstanceHandler{
			instance:       instance,
			projects:       instances[instance],
			authService:    gc.Authentication,
			accountService: gc.Accounts,
			changeService:  gc.Changes,
			projectService: gc.Projects,
		}
	}

	return c, nil
}

func auth(c *Client, cookiefilePath string) {
	logrus.Info("Starting auth loop...")
	var previousToken string
	wait := 10 * time.Minute
	for {
		raw, err := ioutil.ReadFile(cookiefilePath)
		if err != nil {
			logrus.WithError(err).Error("Failed to read auth cookie")
		}
		fields := strings.Fields(string(raw))
		token := fields[len(fields)-1]

		if token == previousToken {
			time.Sleep(wait)
			continue
		}

		logrus.Info("New token, updating handlers...")

		// update auth token for each instance
		for _, handler := range c.handlers {
			handler.authService.SetCookieAuth("o", token)

			self, _, err := handler.accountService.GetAccount("self")
			if err != nil {
				logrus.WithError(err).Error("Failed to auth with token")
				continue
			}

			logrus.Infof("Authentication to %s successful, Username: %s", handler.instance, self.Name)
		}
		previousToken = token
		time.Sleep(wait)
	}
}

// Start will authenticate the client with gerrit periodically
// Start must be called before user calls any client functions.
func (c *Client) Start(cookiefilePath string) {
	if cookiefilePath != "" {
		go auth(c, cookiefilePath)
	}
}

// QueryChanges queries for all changes from all projects after lastUpdate time
// returns an instance:changes map
func (c *Client) QueryChanges(lastUpdate time.Time, rateLimit int) map[string][]ChangeInfo {
	result := map[string][]ChangeInfo{}
	for _, h := range c.handlers {
		changes := h.queryAllChanges(lastUpdate, rateLimit)
		if len(changes) > 0 {
			result[h.instance] = []ChangeInfo{}
			for _, change := range changes {
				result[h.instance] = append(result[h.instance], change)
			}
		}
	}
	return result
}

// SetReview writes a review comment base on the change id + revision
func (c *Client) SetReview(instance, id, revision, message string, labels map[string]string) error {
	h, ok := c.handlers[instance]
	if !ok {
		return fmt.Errorf("not activated gerrit instance: %s", instance)
	}

	if _, _, err := h.changeService.SetReview(id, revision, &gerrit.ReviewInput{
		Message: message,
		Labels:  labels,
	}); err != nil {
		return fmt.Errorf("cannot comment to gerrit: %v", err)
	}

	return nil
}

// GetBranchRevision returns SHA of HEAD of a branch
func (c *Client) GetBranchRevision(instance, project, branch string) (string, error) {
	h, ok := c.handlers[instance]
	if !ok {
		return "", fmt.Errorf("not activated gerrit instance: %s", instance)
	}

	res, _, err := h.projectService.GetBranch(project, url.QueryEscape(branch))
	if err != nil {
		return "", err
	}

	return res.Revision, nil
}

// private handler implementation details

func (h *gerritInstanceHandler) queryAllChanges(lastUpdate time.Time, rateLimit int) []gerrit.ChangeInfo {
	result := []gerrit.ChangeInfo{}
	for _, project := range h.projects {
		changes, err := h.queryChangesForProject(project, lastUpdate, rateLimit)
		if err != nil {
			// don't halt on error from one project, log & continue
			logrus.WithError(err).Errorf("fail to query changes for project %s", project)
			continue
		}
		result = append(result, changes...)
	}

	return result
}

func parseStamp(value gerrit.Timestamp) time.Time {
	return value.Time
}

func (h *gerritInstanceHandler) queryChangesForProject(project string, lastUpdate time.Time, rateLimit int) ([]gerrit.ChangeInfo, error) {
	pending := []gerrit.ChangeInfo{}

	opt := &gerrit.QueryChangeOptions{}
	opt.Query = append(opt.Query, "project:"+project)
	opt.AdditionalFields = []string{"CURRENT_REVISION", "CURRENT_COMMIT", "CURRENT_FILES", "MESSAGES"}

	start := 0

	for {
		opt.Limit = rateLimit
		opt.Start = start

		// The change output is sorted by the last update time, most recently updated to oldest updated.
		// Gerrit API docs: https://gerrit-review.googlesource.com/Documentation/rest-api-changes.html#list-changes
		changes, _, err := h.changeService.QueryChanges(opt)
		if err != nil {
			// should not happen? Let next sync loop catch up
			return nil, fmt.Errorf("failed to query gerrit changes: %v", err)
		}

		if changes == nil || len(*changes) == 0 {
			logrus.Infof("no more changes from query, returning...")
			return pending, nil
		}

		logrus.Infof("Find %d changes from query %v", len(*changes), opt.Query)

		start += len(*changes)

		for _, change := range *changes {
			// if we already processed this change, then we stop the current sync loop
			updated := parseStamp(change.Updated)

			logrus.Infof("Change %d, last updated %s, status %s", change.Number, change.Updated, change.Status)

			// process if updated later than last updated
			// stop if update was stale
			if updated.After(lastUpdate) {
				switch change.Status {
				case Merged:
					submitted := parseStamp(*change.Submitted)
					if submitted.Before(lastUpdate) {
						logrus.Infof("Change %d, submitted %s before lastUpdate %s, skipping this patchset", change.Number, submitted, lastUpdate)
						continue
					}
					pending = append(pending, change)
				case New:
					// we need to make sure the change update is from a fresh commit change
					rev, ok := change.Revisions[change.CurrentRevision]
					if !ok {
						logrus.WithError(err).Errorf("(should not happen?)cannot find current revision for change %v", change.ID)
						continue
					}

					created := parseStamp(rev.Created)
					changeMessages := change.Messages
					newMessages := false

					for _, message := range changeMessages {
						if message.RevisionNumber == rev.Number {
							messageTime := parseStamp(message.Date)
							if messageTime.After(lastUpdate) {
								logrus.Infof("Change %d: Found a new message %s at time %v after lastSync at %v", change.Number, message.Message, messageTime, lastUpdate)
								newMessages = true
								break
							}
						}
					}

					if !newMessages && created.Before(lastUpdate) {
						// stale commit
						logrus.Infof("Change %d, latest revision updated %s before lastUpdate %s, skipping this patchset", change.Number, created, lastUpdate)
						continue
					}

					pending = append(pending, change)
				default:
					// change has been abandoned, do nothing
				}
			} else {
				logrus.Infof("Change %d, updated %s before lastUpdate %s, return", change.Number, change.Updated, lastUpdate)
				return pending, nil
			}
		}
	}
}
<|endoftext|>


<commit_before>// Copyright 2015 Google Inc. All Rights Reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package manager

import (
	"fmt"
	"sync"
	"time"

	"github.com/golang/glog"

	"k8s.io/heapster/model"
	"k8s.io/heapster/sinks"
	sink_api "k8s.io/heapster/sinks/api"
	"k8s.io/heapster/sinks/cache"
	source_api "k8s.io/heapster/sources/api"
	"k8s.io/heapster/store"
)

// Manager provides an interface to control the core of heapster.
// Implementations are not required to be thread safe.
type Manager interface {
	// Housekeep collects data from all the configured sources and
	// stores the data to all the configured sinks.
	Housekeep()

	// HousekeepModel performs housekeeping for the Model entity
	HousekeepModel()

	// Export the latest data point of all metrics.
	ExportMetrics() ([]*sink_api.Point, error)

	// Set the sinks to use
	SetSinkUris(Uris) error

	// Get the sinks currently in use
	SinkUris() Uris

	// Get a reference to the cluster entity of the model, if it exists.
	GetCluster() model.Cluster
}

type realManager struct {
	sources      []source_api.Source
	cache        cache.Cache
	model        model.Cluster
	sinkManager  sinks.ExternalSinkManager
	sinkUris     Uris
	lastSync     time.Time
	resolution   time.Duration
	align        bool
	decoder      sink_api.Decoder
	sinkStopChan chan<- struct{}
}

type syncData struct {
	data  source_api.AggregateData
	mutex sync.Mutex
}

func NewManager(sources []source_api.Source, sinkManager sinks.ExternalSinkManager, res, bufferDuration time.Duration, c cache.Cache, useModel bool, modelRes time.Duration, align bool) (Manager, error) {
	// TimeStore constructor passed to the cluster implementation.
	tsConstructor := func() store.TimeStore {
		// TODO(afein): determine default analogy of cache duration to Timestore durations.
		return store.NewGCStore(store.NewCMAStore(), 5*bufferDuration)
	}
	var newCluster model.Cluster = nil
	if useModel {
		newCluster = model.NewCluster(tsConstructor, modelRes)
	}
	firstSync := time.Now()
	if align {
		firstSync = firstSync.Truncate(res).Add(res)
	}
	return &realManager{
		sources:      sources,
		sinkManager:  sinkManager,
		cache:        c,
		model:        newCluster,
		lastSync:     firstSync,
		resolution:   res,
		align:        align,
		decoder:      sink_api.NewDecoder(),
		sinkStopChan: sinkManager.Sync(),
	}, nil
}

func (rm *realManager) GetCluster() model.Cluster {
	return rm.model
}

func (rm *realManager) scrapeSource(s source_api.Source, start, end time.Time, sd *syncData, errChan chan<- error) {
	glog.V(2).Infof("attempting to get data from source %q", s.Name())
	data, err := s.GetInfo(start, end, rm.resolution, rm.align)
	if err != nil {
		errChan <- fmt.Errorf("failed to get information from source %q - %v", s.Name(), err)
		return
	}
	sd.mutex.Lock()
	defer sd.mutex.Unlock()
	sd.data.Merge(&data)
	errChan <- nil
}

// HousekeepModel periodically populates the manager model from the manager cache.
func (rm *realManager) HousekeepModel() {
	if rm.model != nil {
		if err := rm.model.Update(rm.cache); err != nil {
			glog.V(1).Infof("Model housekeeping returned error: %s", err.Error())
		}
	}
}

func (rm *realManager) Housekeep() {
	errChan := make(chan error, len(rm.sources))
	var sd syncData
	start := rm.lastSync
	end := time.Now()
	if rm.align {
		end = end.Truncate(rm.resolution)
		if start.After(end) {
			return
		}
	}
	rm.lastSync = end
	glog.V(2).Infof("starting to scrape data from sources start:%v end:%v", start, end)
	for idx := range rm.sources {
		s := rm.sources[idx]
		go rm.scrapeSource(s, start, end, &sd, errChan)
	}
	var errors []string
	for i := 0; i < len(rm.sources); i++ {
		if err := <-errChan; err != nil {
			errors = append(errors, err.Error())
		}
	}
	glog.V(2).Infof("completed scraping data from sources. Errors: %v", errors)
	if err := rm.cache.StorePods(sd.data.Pods); err != nil {
		errors = append(errors, err.Error())
	}
	if err := rm.cache.StoreContainers(sd.data.Machine); err != nil {
		errors = append(errors, err.Error())
	}
	if err := rm.cache.StoreContainers(sd.data.Containers); err != nil {
		errors = append(errors, err.Error())
	}
	if len(errors) > 0 {
		glog.V(1).Infof("housekeeping resulted in following errors: %v", errors)
	}
}

func (rm *realManager) ExportMetrics() ([]*sink_api.Point, error) {
	var zero time.Time

	// Get all pods as points.
	pods := trimStatsForPods(rm.cache.GetPods(zero, zero))
	timeseries, err := rm.decoder.TimeseriesFromPods(pods)
	if err != nil {
		return []*sink_api.Point{}, err
	}
	points := make([]*sink_api.Point, 0, len(timeseries))
	points = appendPoints(points, timeseries)

	// Get all nodes as points.
	containers := trimStatsForContainers(rm.cache.GetNodes(zero, zero))
	timeseries, err = rm.decoder.TimeseriesFromContainers(containers)
	if err != nil {
		return []*sink_api.Point{}, err
	}
	points = appendPoints(points, timeseries)

	// Get all free containers as points.
	containers = trimStatsForContainers(rm.cache.GetFreeContainers(zero, zero))
	timeseries, err = rm.decoder.TimeseriesFromContainers(containers)
	if err != nil {
		return []*sink_api.Point{}, err
	}
	points = appendPoints(points, timeseries)

	return points, nil
}

// Extract the points from the specified timeseries and append them to output.
func appendPoints(output []*sink_api.Point, toExtract []sink_api.Timeseries) []*sink_api.Point {
	for i := range toExtract {
		output = append(output, toExtract[i].Point)
	}
	return output
}

// Only keep latest stats for the specified pods
func trimStatsForPods(pods []*cache.PodElement) []*cache.PodElement {
	for _, pod := range pods {
		trimStatsForContainers(pod.Containers)
	}
	return pods
}

// Only keep latest stats for the specified containers
func trimStatsForContainers(containers []*cache.ContainerElement) []*cache.ContainerElement {
	for _, cont := range containers {
		onlyKeepLatestStat(cont)
	}
	return containers
}

// Only keep the latest stats data point.
func onlyKeepLatestStat(cont *cache.ContainerElement) {
	if len(cont.Metrics) > 1 {
		cont.Metrics = cont.Metrics[0:1]
	}
}

func (rm *realManager) SetSinkUris(sinkUris Uris) error {
	sinks, err := newSinks(sinkUris)
	if err != nil {
		return err
	}
	if err := rm.sinkManager.SetSinks(sinks); err != nil {
		return err
	}
	rm.sinkUris = sinkUris
	return nil
}

func (rm *realManager) SinkUris() Uris {
	return rm.sinkUris
}
<commit_msg>Revert "Merge pull request #423 from piosz/align"<commit_after>// Copyright 2015 Google Inc. All Rights Reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package manager

import (
	"fmt"
	"sync"
	"time"

	"github.com/golang/glog"

	"k8s.io/heapster/model"
	"k8s.io/heapster/sinks"
	sink_api "k8s.io/heapster/sinks/api"
	"k8s.io/heapster/sinks/cache"
	source_api "k8s.io/heapster/sources/api"
	"k8s.io/heapster/store"
)

// Manager provides an interface to control the core of heapster.
// Implementations are not required to be thread safe.
type Manager interface {
	// Housekeep collects data from all the configured sources and
	// stores the data to all the configured sinks.
	Housekeep()

	// HousekeepModel performs housekeeping for the Model entity
	HousekeepModel()

	// Export the latest data point of all metrics.
	ExportMetrics() ([]*sink_api.Point, error)

	// Set the sinks to use
	SetSinkUris(Uris) error

	// Get the sinks currently in use
	SinkUris() Uris

	// Get a reference to the cluster entity of the model, if it exists.
	GetCluster() model.Cluster
}

type realManager struct {
	sources      []source_api.Source
	cache        cache.Cache
	model        model.Cluster
	sinkManager  sinks.ExternalSinkManager
	sinkUris     Uris
	lastSync     time.Time
	resolution   time.Duration
	align        bool
	decoder      sink_api.Decoder
	sinkStopChan chan<- struct{}
}

type syncData struct {
	data  source_api.AggregateData
	mutex sync.Mutex
}

func NewManager(sources []source_api.Source, sinkManager sinks.ExternalSinkManager, res, bufferDuration time.Duration, c cache.Cache, useModel bool, modelRes time.Duration, align bool) (Manager, error) {
	// TimeStore constructor passed to the cluster implementation.
	tsConstructor := func() store.TimeStore {
		// TODO(afein): determine default analogy of cache duration to Timestore durations.
		return store.NewGCStore(store.NewCMAStore(), 5*bufferDuration)
	}
	var newCluster model.Cluster = nil
	if useModel {
		newCluster = model.NewCluster(tsConstructor, modelRes)
	}
	return &realManager{
		sources:      sources,
		sinkManager:  sinkManager,
		cache:        c,
		model:        newCluster,
		lastSync:     time.Now(),
		resolution:   res,
		align:        align,
		decoder:      sink_api.NewDecoder(),
		sinkStopChan: sinkManager.Sync(),
	}, nil
}

func (rm *realManager) GetCluster() model.Cluster {
	return rm.model
}

func (rm *realManager) scrapeSource(s source_api.Source, start, end time.Time, sd *syncData, errChan chan<- error) {
	glog.V(2).Infof("attempting to get data from source %q", s.Name())
	data, err := s.GetInfo(start, end, rm.resolution, rm.align)
	if err != nil {
		errChan <- fmt.Errorf("failed to get information from source %q - %v", s.Name(), err)
		return
	}
	sd.mutex.Lock()
	defer sd.mutex.Unlock()
	sd.data.Merge(&data)
	errChan <- nil
}

// HousekeepModel periodically populates the manager model from the manager cache.
func (rm *realManager) HousekeepModel() {
	if rm.model != nil {
		if err := rm.model.Update(rm.cache); err != nil {
			glog.V(1).Infof("Model housekeeping returned error: %s", err.Error())
		}
	}
}

func (rm *realManager) Housekeep() {
	errChan := make(chan error, len(rm.sources))
	var sd syncData
	start := rm.lastSync
	end := time.Now()
	rm.lastSync = end
	glog.V(2).Infof("starting to scrape data from sources")
	for idx := range rm.sources {
		s := rm.sources[idx]
		go rm.scrapeSource(s, start, end, &sd, errChan)
	}
	var errors []string
	for i := 0; i < len(rm.sources); i++ {
		if err := <-errChan; err != nil {
			errors = append(errors, err.Error())
		}
	}
	glog.V(2).Infof("completed scraping data from sources. Errors: %v", errors)
	if err := rm.cache.StorePods(sd.data.Pods); err != nil {
		errors = append(errors, err.Error())
	}
	if err := rm.cache.StoreContainers(sd.data.Machine); err != nil {
		errors = append(errors, err.Error())
	}
	if err := rm.cache.StoreContainers(sd.data.Containers); err != nil {
		errors = append(errors, err.Error())
	}
	if len(errors) > 0 {
		glog.V(1).Infof("housekeeping resulted in following errors: %v", errors)
	}
}

func (rm *realManager) ExportMetrics() ([]*sink_api.Point, error) {
	var zero time.Time

	// Get all pods as points.
	pods := trimStatsForPods(rm.cache.GetPods(zero, zero))
	timeseries, err := rm.decoder.TimeseriesFromPods(pods)
	if err != nil {
		return []*sink_api.Point{}, err
	}
	points := make([]*sink_api.Point, 0, len(timeseries))
	points = appendPoints(points, timeseries)

	// Get all nodes as points.
	containers := trimStatsForContainers(rm.cache.GetNodes(zero, zero))
	timeseries, err = rm.decoder.TimeseriesFromContainers(containers)
	if err != nil {
		return []*sink_api.Point{}, err
	}
	points = appendPoints(points, timeseries)

	// Get all free containers as points.
	containers = trimStatsForContainers(rm.cache.GetFreeContainers(zero, zero))
	timeseries, err = rm.decoder.TimeseriesFromContainers(containers)
	if err != nil {
		return []*sink_api.Point{}, err
	}
	points = appendPoints(points, timeseries)

	return points, nil
}

// Extract the points from the specified timeseries and append them to output.
func appendPoints(output []*sink_api.Point, toExtract []sink_api.Timeseries) []*sink_api.Point {
	for i := range toExtract {
		output = append(output, toExtract[i].Point)
	}
	return output
}

// Only keep latest stats for the specified pods
func trimStatsForPods(pods []*cache.PodElement) []*cache.PodElement {
	for _, pod := range pods {
		trimStatsForContainers(pod.Containers)
	}
	return pods
}

// Only keep latest stats for the specified containers
func trimStatsForContainers(containers []*cache.ContainerElement) []*cache.ContainerElement {
	for _, cont := range containers {
		onlyKeepLatestStat(cont)
	}
	return containers
}

// Only keep the latest stats data point.
func onlyKeepLatestStat(cont *cache.ContainerElement) {
	if len(cont.Metrics) > 1 {
		cont.Metrics = cont.Metrics[0:1]
	}
}

func (rm *realManager) SetSinkUris(sinkUris Uris) error {
	sinks, err := newSinks(sinkUris)
	if err != nil {
		return err
	}
	if err := rm.sinkManager.SetSinks(sinks); err != nil {
		return err
	}
	rm.sinkUris = sinkUris
	return nil
}

func (rm *realManager) SinkUris() Uris {
	return rm.sinkUris
}
<|endoftext|>


<commit_before>/*
Copyright IBM Corp. 2016 All Rights Reserved.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

		 http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package ca

import (
	"crypto/ecdsa"
	"crypto/hmac"
	"crypto/rand"
	"crypto/x509"
	"crypto/x509/pkix"
	"encoding/asn1"
	"errors"
	"fmt"
	"math/big"
	"strconv"
	"time"

	"github.com/golang/protobuf/proto"
	"github.com/hyperledger/fabric/core/crypto/attributes"
	"github.com/hyperledger/fabric/core/crypto/primitives"
	"github.com/hyperledger/fabric/core/util"
	pb "github.com/hyperledger/fabric/membersrvc/protos"
	"github.com/spf13/viper"
	"golang.org/x/net/context"

	"google/protobuf"
)

// TCAP serves the public GRPC interface of the TCA.
type TCAP struct {
	tca *TCA
}

// ReadCACertificate reads the certificate of the TCA.
func (tcap *TCAP) ReadCACertificate(ctx context.Context, in *pb.Empty) (*pb.Cert, error) {
	Trace.Println("grpc TCAP:ReadCACertificate")

	return &pb.Cert{Cert: tcap.tca.raw}, nil
}

func (tcap *TCAP) selectValidAttributes(certRaw []byte) ([]*pb.ACAAttribute, error) {
	cert, err := x509.ParseCertificate(certRaw)
	if err != nil {
		return nil, err
	}

	var ans []*pb.ACAAttribute

	if cert.Extensions == nil {
		return ans, nil
	}
	currentTime := time.Now()
	for _, extension := range cert.Extensions {
		acaAtt := &pb.ACAAttribute{AttributeName: "", AttributeValue: nil, ValidFrom: &google_protobuf.Timestamp{Seconds: 0, Nanos: 0}, ValidTo: &google_protobuf.Timestamp{Seconds: 0, Nanos: 0}}

		if IsAttributeOID(extension.Id) {
			if err := proto.Unmarshal(extension.Value, acaAtt); err != nil {
				continue
			}

			if acaAtt.AttributeName == "" {
				continue
			}
			var from, to time.Time
			if acaAtt.ValidFrom != nil {
				from = time.Unix(acaAtt.ValidFrom.Seconds, int64(acaAtt.ValidFrom.Nanos))
			}
			if acaAtt.ValidTo != nil {
				to = time.Unix(acaAtt.ValidTo.Seconds, int64(acaAtt.ValidTo.Nanos))
			}

			//Check if the attribute still being valid.
			if (from.Before(currentTime) || from.Equal(currentTime)) && (to.IsZero() || to.After(currentTime)) {
				ans = append(ans, acaAtt)
			}
		}
	}
	return ans, nil
}

func (tcap *TCAP) requestAttributes(id string, ecert []byte, attrs []*pb.TCertAttribute) ([]*pb.ACAAttribute, error) {
	//TODO we are creation a new client connection per each ecer request. We should be implement a connections pool.
	sock, acaP, err := GetACAClient()
	if err != nil {
		return nil, err
	}
	defer sock.Close()
	var attrNames []*pb.TCertAttribute

	for _, att := range attrs {
		attrName := pb.TCertAttribute{AttributeName: att.AttributeName}
		attrNames = append(attrNames, &attrName)
	}

	req := &pb.ACAAttrReq{
		Ts:         &google_protobuf.Timestamp{Seconds: time.Now().Unix(), Nanos: 0},
		Id:         &pb.Identity{Id: id},
		ECert:      &pb.Cert{Cert: ecert},
		Attributes: attrNames,
		Signature:  nil}

	var rawReq []byte
	rawReq, err = proto.Marshal(req)
	if err != nil {
		return nil, err
	}

	var r, s *big.Int

	r, s, err = primitives.ECDSASignDirect(tcap.tca.priv, rawReq)

	if err != nil {
		return nil, err
	}

	R, _ := r.MarshalText()
	S, _ := s.MarshalText()

	req.Signature = &pb.Signature{Type: pb.CryptoType_ECDSA, R: R, S: S}

	resp, err := acaP.RequestAttributes(context.Background(), req)
	if err != nil {
		return nil, err
	}

	if resp.Status >= pb.ACAAttrResp_FAILURE_MINVAL && resp.Status <= pb.ACAAttrResp_FAILURE_MAXVAL {
		return nil, errors.New(fmt.Sprint("Error fetching attributes = ", resp.Status))
	}

	return tcap.selectValidAttributes(resp.Cert.Cert)
}

// CreateCertificateSet requests the creation of a new transaction certificate set by the TCA.
func (tcap *TCAP) CreateCertificateSet(ctx context.Context, in *pb.TCertCreateSetReq) (*pb.TCertCreateSetResp, error) {
	Trace.Println("grpc TCAP:CreateCertificateSet")

	id := in.Id.Id
	raw, err := tcap.tca.eca.readCertificateByKeyUsage(id, x509.KeyUsageDigitalSignature)
	if err != nil {
		return nil, err
	}

	return tcap.createCertificateSet(ctx, raw, in)
}

func (tcap *TCAP) createCertificateSet(ctx context.Context, raw []byte, in *pb.TCertCreateSetReq) (*pb.TCertCreateSetResp, error) {
	var attrs = []*pb.ACAAttribute{}
	var err error
	var id = in.Id.Id
	var timestamp = in.Ts.Seconds
	const TCERT_SUBJECT_COMMON_NAME_VALUE string = "Transaction Certificate"

	if in.Attributes != nil && viper.GetBool("aca.enabled") {
		attrs, err = tcap.requestAttributes(id, raw, in.Attributes)
		if err != nil {
			return nil, err
		}
	}

	cert, err := x509.ParseCertificate(raw)
	if err != nil {
		return nil, err
	}

	pub := cert.PublicKey.(*ecdsa.PublicKey)

	r, s := big.NewInt(0), big.NewInt(0)
	r.UnmarshalText(in.Sig.R)
	s.UnmarshalText(in.Sig.S)

	//sig := in.Sig
	in.Sig = nil

	hash := primitives.NewHash()
	raw, _ = proto.Marshal(in)
	hash.Write(raw)
	if ecdsa.Verify(pub, hash.Sum(nil), r, s) == false {
		return nil, errors.New("signature does not verify")
	}

	// Generate nonce for TCertIndex
	nonce := make([]byte, 16) // 8 bytes rand, 8 bytes timestamp
	rand.Reader.Read(nonce[:8])

	mac := hmac.New(primitives.GetDefaultHash(), tcap.tca.hmacKey)
	raw, _ = x509.MarshalPKIXPublicKey(pub)
	mac.Write(raw)
	kdfKey := mac.Sum(nil)

	num := int(in.Num)
	if num == 0 {
		num = 1
	}

	// the batch of TCerts
	var set []*pb.TCert

	for i := 0; i < num; i++ {
		tcertid := util.GenerateIntUUID()

		// Compute TCertIndex
		tidx := []byte(strconv.Itoa(2*i + 1))
		tidx = append(tidx[:], nonce[:]...)
		tidx = append(tidx[:], Padding...)

		mac := hmac.New(primitives.GetDefaultHash(), kdfKey)
		mac.Write([]byte{1})
		extKey := mac.Sum(nil)[:32]

		mac = hmac.New(primitives.GetDefaultHash(), kdfKey)
		mac.Write([]byte{2})
		mac = hmac.New(primitives.GetDefaultHash(), mac.Sum(nil))
		mac.Write(tidx)

		one := new(big.Int).SetInt64(1)
		k := new(big.Int).SetBytes(mac.Sum(nil))
		k.Mod(k, new(big.Int).Sub(pub.Curve.Params().N, one))
		k.Add(k, one)

		tmpX, tmpY := pub.ScalarBaseMult(k.Bytes())
		txX, txY := pub.Curve.Add(pub.X, pub.Y, tmpX, tmpY)
		txPub := ecdsa.PublicKey{Curve: pub.Curve, X: txX, Y: txY}

		// Compute encrypted TCertIndex
		encryptedTidx, err := primitives.CBCPKCS7Encrypt(extKey, tidx)
		if err != nil {
			return nil, err
		}

		extensions, preK0, err := tcap.generateExtensions(tcertid, encryptedTidx, cert, attrs)

		if err != nil {
			return nil, err
		}

		spec := NewDefaultPeriodCertificateSpecWithCommonName(id, TCERT_SUBJECT_COMMON_NAME_VALUE, tcertid, &txPub, x509.KeyUsageDigitalSignature, extensions...)
		if raw, err = tcap.tca.createCertificateFromSpec(spec, timestamp, kdfKey, false); err != nil {
			Error.Println(err)
			return nil, err
		}

		set = append(set, &pb.TCert{Cert: raw, Prek0: preK0})
	}

	tcap.tca.persistCertificateSet(id, timestamp, nonce, kdfKey)

	return &pb.TCertCreateSetResp{Certs: &pb.CertSet{Ts: in.Ts, Id: in.Id, Key: kdfKey, Certs: set}}, nil
}

// Generate encrypted extensions to be included into the TCert (TCertIndex, EnrollmentID and attributes).
func (tcap *TCAP) generateExtensions(tcertid *big.Int, tidx []byte, enrollmentCert *x509.Certificate, attrs []*pb.ACAAttribute) ([]pkix.Extension, []byte, error) {
	// For each TCert we need to store and retrieve to the user the list of Ks used to encrypt the EnrollmentID and the attributes.
	extensions := make([]pkix.Extension, len(attrs))

	// Compute preK_1 to encrypt attributes and enrollment ID
	preK1, err := tcap.tca.getPreKFrom(enrollmentCert)
	if err != nil {
		return nil, nil, err
	}

	mac := hmac.New(primitives.GetDefaultHash(), preK1)
	mac.Write(tcertid.Bytes())
	preK0 := mac.Sum(nil)

	// Compute encrypted EnrollmentID
	mac = hmac.New(primitives.GetDefaultHash(), preK0)
	mac.Write([]byte("enrollmentID"))
	enrollmentIDKey := mac.Sum(nil)[:32]

	enrollmentID := []byte(enrollmentCert.Subject.CommonName)
	enrollmentID = append(enrollmentID, Padding...)

	encEnrollmentID, err := primitives.CBCPKCS7Encrypt(enrollmentIDKey, enrollmentID)
	if err != nil {
		return nil, nil, err
	}

	attributeIdentifierIndex := 9
	count := 0
	attrsHeader := make(map[string]int)
	// Encrypt and append attrs to the extensions slice
	for _, a := range attrs {
		count++

		value := []byte(a.AttributeValue)

		//Save the position of the attribute extension on the header.
		attrsHeader[a.AttributeName] = count

		if isEnabledAttributesEncryption() {
			value, err = attributes.EncryptAttributeValuePK0(preK0, a.AttributeName, value)
			if err != nil {
				return nil, nil, err
			}
		}

		// Generate an ObjectIdentifier for the extension holding the attribute
		TCertEncAttributes := asn1.ObjectIdentifier{1, 2, 3, 4, 5, 6, attributeIdentifierIndex + count}

		// Add the attribute extension to the extensions array
		extensions[count-1] = pkix.Extension{Id: TCertEncAttributes, Critical: false, Value: value}
	}

	// Append the TCertIndex to the extensions
	extensions = append(extensions, pkix.Extension{Id: TCertEncTCertIndex, Critical: true, Value: tidx})

	// Append the encrypted EnrollmentID to the extensions
	extensions = append(extensions, pkix.Extension{Id: TCertEncEnrollmentID, Critical: false, Value: encEnrollmentID})

	// Append the attributes header if there was attributes to include in the TCert
	if len(attrs) > 0 {
		headerValue, err := attributes.BuildAttributesHeader(attrsHeader)
		if err != nil {
			return nil, nil, err
		}
		if isEnabledAttributesEncryption() {
			headerValue, err = attributes.EncryptAttributeValuePK0(preK0, attributes.HeaderAttributeName, headerValue)
			if err != nil {
				return nil, nil, err
			}
		}
		extensions = append(extensions, pkix.Extension{Id: TCertAttributesHeaders, Critical: false, Value: headerValue})
	}

	return extensions, preK0, nil
}

// RevokeCertificate revokes a certificate from the TCA.  Not yet implemented.
func (tcap *TCAP) RevokeCertificate(context.Context, *pb.TCertRevokeReq) (*pb.CAStatus, error) {
	Trace.Println("grpc TCAP:RevokeCertificate")

	return nil, errors.New("not yet implemented")
}

// RevokeCertificateSet revokes a certificate set from the TCA.  Not yet implemented.
func (tcap *TCAP) RevokeCertificateSet(context.Context, *pb.TCertRevokeSetReq) (*pb.CAStatus, error) {
	Trace.Println("grpc TCAP:RevokeCertificateSet")

	return nil, errors.New("not yet implemented")
}

func isEnabledAttributesEncryption() bool {
	//TODO this code is commented because attributes encryption is not yet implemented.
	//return viper.GetBool("tca.attribute-encryption.enabled")
	return false
}
<commit_msg>Addition of the timestamp to the nonce calculation of the TcertIndex (#2099)<commit_after>/*
Copyright IBM Corp. 2016 All Rights Reserved.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

		 http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package ca

import (
	"crypto/ecdsa"
	"crypto/hmac"
	"crypto/rand"
	"crypto/x509"
	"crypto/x509/pkix"
	"encoding/asn1"
	"encoding/binary"
	"errors"
	"fmt"
	"math/big"
	"strconv"
	"time"

	"github.com/golang/protobuf/proto"
	"github.com/hyperledger/fabric/core/crypto/attributes"
	"github.com/hyperledger/fabric/core/crypto/primitives"
	"github.com/hyperledger/fabric/core/util"
	pb "github.com/hyperledger/fabric/membersrvc/protos"
	"github.com/spf13/viper"
	"golang.org/x/net/context"

	"google/protobuf"
)

// TCAP serves the public GRPC interface of the TCA.
type TCAP struct {
	tca *TCA
}

// ReadCACertificate reads the certificate of the TCA.
func (tcap *TCAP) ReadCACertificate(ctx context.Context, in *pb.Empty) (*pb.Cert, error) {
	Trace.Println("grpc TCAP:ReadCACertificate")

	return &pb.Cert{Cert: tcap.tca.raw}, nil
}

func (tcap *TCAP) selectValidAttributes(certRaw []byte) ([]*pb.ACAAttribute, error) {
	cert, err := x509.ParseCertificate(certRaw)
	if err != nil {
		return nil, err
	}

	var ans []*pb.ACAAttribute

	if cert.Extensions == nil {
		return ans, nil
	}
	currentTime := time.Now()
	for _, extension := range cert.Extensions {
		acaAtt := &pb.ACAAttribute{AttributeName: "", AttributeValue: nil, ValidFrom: &google_protobuf.Timestamp{Seconds: 0, Nanos: 0}, ValidTo: &google_protobuf.Timestamp{Seconds: 0, Nanos: 0}}

		if IsAttributeOID(extension.Id) {
			if err := proto.Unmarshal(extension.Value, acaAtt); err != nil {
				continue
			}

			if acaAtt.AttributeName == "" {
				continue
			}
			var from, to time.Time
			if acaAtt.ValidFrom != nil {
				from = time.Unix(acaAtt.ValidFrom.Seconds, int64(acaAtt.ValidFrom.Nanos))
			}
			if acaAtt.ValidTo != nil {
				to = time.Unix(acaAtt.ValidTo.Seconds, int64(acaAtt.ValidTo.Nanos))
			}

			//Check if the attribute still being valid.
			if (from.Before(currentTime) || from.Equal(currentTime)) && (to.IsZero() || to.After(currentTime)) {
				ans = append(ans, acaAtt)
			}
		}
	}
	return ans, nil
}

func (tcap *TCAP) requestAttributes(id string, ecert []byte, attrs []*pb.TCertAttribute) ([]*pb.ACAAttribute, error) {
	//TODO we are creation a new client connection per each ecer request. We should be implement a connections pool.
	sock, acaP, err := GetACAClient()
	if err != nil {
		return nil, err
	}
	defer sock.Close()
	var attrNames []*pb.TCertAttribute

	for _, att := range attrs {
		attrName := pb.TCertAttribute{AttributeName: att.AttributeName}
		attrNames = append(attrNames, &attrName)
	}

	req := &pb.ACAAttrReq{
		Ts:         &google_protobuf.Timestamp{Seconds: time.Now().Unix(), Nanos: 0},
		Id:         &pb.Identity{Id: id},
		ECert:      &pb.Cert{Cert: ecert},
		Attributes: attrNames,
		Signature:  nil}

	var rawReq []byte
	rawReq, err = proto.Marshal(req)
	if err != nil {
		return nil, err
	}

	var r, s *big.Int

	r, s, err = primitives.ECDSASignDirect(tcap.tca.priv, rawReq)

	if err != nil {
		return nil, err
	}

	R, _ := r.MarshalText()
	S, _ := s.MarshalText()

	req.Signature = &pb.Signature{Type: pb.CryptoType_ECDSA, R: R, S: S}

	resp, err := acaP.RequestAttributes(context.Background(), req)
	if err != nil {
		return nil, err
	}

	if resp.Status >= pb.ACAAttrResp_FAILURE_MINVAL && resp.Status <= pb.ACAAttrResp_FAILURE_MAXVAL {
		return nil, errors.New(fmt.Sprint("Error fetching attributes = ", resp.Status))
	}

	return tcap.selectValidAttributes(resp.Cert.Cert)
}

// CreateCertificateSet requests the creation of a new transaction certificate set by the TCA.
func (tcap *TCAP) CreateCertificateSet(ctx context.Context, in *pb.TCertCreateSetReq) (*pb.TCertCreateSetResp, error) {
	Trace.Println("grpc TCAP:CreateCertificateSet")

	id := in.Id.Id
	raw, err := tcap.tca.eca.readCertificateByKeyUsage(id, x509.KeyUsageDigitalSignature)
	if err != nil {
		return nil, err
	}

	return tcap.createCertificateSet(ctx, raw, in)
}

func (tcap *TCAP) createCertificateSet(ctx context.Context, raw []byte, in *pb.TCertCreateSetReq) (*pb.TCertCreateSetResp, error) {
	var attrs = []*pb.ACAAttribute{}
	var err error
	var id = in.Id.Id
	var timestamp = in.Ts.Seconds
	const TCERT_SUBJECT_COMMON_NAME_VALUE string = "Transaction Certificate"

	if in.Attributes != nil && viper.GetBool("aca.enabled") {
		attrs, err = tcap.requestAttributes(id, raw, in.Attributes)
		if err != nil {
			return nil, err
		}
	}

	cert, err := x509.ParseCertificate(raw)
	if err != nil {
		return nil, err
	}

	pub := cert.PublicKey.(*ecdsa.PublicKey)

	r, s := big.NewInt(0), big.NewInt(0)
	r.UnmarshalText(in.Sig.R)
	s.UnmarshalText(in.Sig.S)

	//sig := in.Sig
	in.Sig = nil

	hash := primitives.NewHash()
	raw, _ = proto.Marshal(in)
	hash.Write(raw)
	if ecdsa.Verify(pub, hash.Sum(nil), r, s) == false {
		return nil, errors.New("signature does not verify")
	}

	// Generate nonce for TCertIndex
	nonce := make([]byte, 16) // 8 bytes rand, 8 bytes timestamp
	rand.Reader.Read(nonce[:8])
	binary.LittleEndian.PutUint64(nonce[8:], uint64(in.Ts.Seconds))

	mac := hmac.New(primitives.GetDefaultHash(), tcap.tca.hmacKey)
	raw, _ = x509.MarshalPKIXPublicKey(pub)
	mac.Write(raw)
	kdfKey := mac.Sum(nil)

	num := int(in.Num)
	if num == 0 {
		num = 1
	}

	// the batch of TCerts
	var set []*pb.TCert

	for i := 0; i < num; i++ {
		tcertid := util.GenerateIntUUID()

		// Compute TCertIndex
		tidx := []byte(strconv.Itoa(2*i + 1))
		tidx = append(tidx[:], nonce[:]...)
		tidx = append(tidx[:], Padding...)

		mac := hmac.New(primitives.GetDefaultHash(), kdfKey)
		mac.Write([]byte{1})
		extKey := mac.Sum(nil)[:32]

		mac = hmac.New(primitives.GetDefaultHash(), kdfKey)
		mac.Write([]byte{2})
		mac = hmac.New(primitives.GetDefaultHash(), mac.Sum(nil))
		mac.Write(tidx)

		one := new(big.Int).SetInt64(1)
		k := new(big.Int).SetBytes(mac.Sum(nil))
		k.Mod(k, new(big.Int).Sub(pub.Curve.Params().N, one))
		k.Add(k, one)

		tmpX, tmpY := pub.ScalarBaseMult(k.Bytes())
		txX, txY := pub.Curve.Add(pub.X, pub.Y, tmpX, tmpY)
		txPub := ecdsa.PublicKey{Curve: pub.Curve, X: txX, Y: txY}

		// Compute encrypted TCertIndex
		encryptedTidx, err := primitives.CBCPKCS7Encrypt(extKey, tidx)
		if err != nil {
			return nil, err
		}

		extensions, preK0, err := tcap.generateExtensions(tcertid, encryptedTidx, cert, attrs)

		if err != nil {
			return nil, err
		}

		spec := NewDefaultPeriodCertificateSpecWithCommonName(id, TCERT_SUBJECT_COMMON_NAME_VALUE, tcertid, &txPub, x509.KeyUsageDigitalSignature, extensions...)
		if raw, err = tcap.tca.createCertificateFromSpec(spec, timestamp, kdfKey, false); err != nil {
			Error.Println(err)
			return nil, err
		}

		set = append(set, &pb.TCert{Cert: raw, Prek0: preK0})
	}

	tcap.tca.persistCertificateSet(id, timestamp, nonce, kdfKey)

	return &pb.TCertCreateSetResp{Certs: &pb.CertSet{Ts: in.Ts, Id: in.Id, Key: kdfKey, Certs: set}}, nil
}

// Generate encrypted extensions to be included into the TCert (TCertIndex, EnrollmentID and attributes).
func (tcap *TCAP) generateExtensions(tcertid *big.Int, tidx []byte, enrollmentCert *x509.Certificate, attrs []*pb.ACAAttribute) ([]pkix.Extension, []byte, error) {
	// For each TCert we need to store and retrieve to the user the list of Ks used to encrypt the EnrollmentID and the attributes.
	extensions := make([]pkix.Extension, len(attrs))

	// Compute preK_1 to encrypt attributes and enrollment ID
	preK1, err := tcap.tca.getPreKFrom(enrollmentCert)
	if err != nil {
		return nil, nil, err
	}

	mac := hmac.New(primitives.GetDefaultHash(), preK1)
	mac.Write(tcertid.Bytes())
	preK0 := mac.Sum(nil)

	// Compute encrypted EnrollmentID
	mac = hmac.New(primitives.GetDefaultHash(), preK0)
	mac.Write([]byte("enrollmentID"))
	enrollmentIDKey := mac.Sum(nil)[:32]

	enrollmentID := []byte(enrollmentCert.Subject.CommonName)
	enrollmentID = append(enrollmentID, Padding...)

	encEnrollmentID, err := primitives.CBCPKCS7Encrypt(enrollmentIDKey, enrollmentID)
	if err != nil {
		return nil, nil, err
	}

	attributeIdentifierIndex := 9
	count := 0
	attrsHeader := make(map[string]int)
	// Encrypt and append attrs to the extensions slice
	for _, a := range attrs {
		count++

		value := []byte(a.AttributeValue)

		//Save the position of the attribute extension on the header.
		attrsHeader[a.AttributeName] = count

		if isEnabledAttributesEncryption() {
			value, err = attributes.EncryptAttributeValuePK0(preK0, a.AttributeName, value)
			if err != nil {
				return nil, nil, err
			}
		}

		// Generate an ObjectIdentifier for the extension holding the attribute
		TCertEncAttributes := asn1.ObjectIdentifier{1, 2, 3, 4, 5, 6, attributeIdentifierIndex + count}

		// Add the attribute extension to the extensions array
		extensions[count-1] = pkix.Extension{Id: TCertEncAttributes, Critical: false, Value: value}
	}

	// Append the TCertIndex to the extensions
	extensions = append(extensions, pkix.Extension{Id: TCertEncTCertIndex, Critical: true, Value: tidx})

	// Append the encrypted EnrollmentID to the extensions
	extensions = append(extensions, pkix.Extension{Id: TCertEncEnrollmentID, Critical: false, Value: encEnrollmentID})

	// Append the attributes header if there was attributes to include in the TCert
	if len(attrs) > 0 {
		headerValue, err := attributes.BuildAttributesHeader(attrsHeader)
		if err != nil {
			return nil, nil, err
		}
		if isEnabledAttributesEncryption() {
			headerValue, err = attributes.EncryptAttributeValuePK0(preK0, attributes.HeaderAttributeName, headerValue)
			if err != nil {
				return nil, nil, err
			}
		}
		extensions = append(extensions, pkix.Extension{Id: TCertAttributesHeaders, Critical: false, Value: headerValue})
	}

	return extensions, preK0, nil
}

// RevokeCertificate revokes a certificate from the TCA.  Not yet implemented.
func (tcap *TCAP) RevokeCertificate(context.Context, *pb.TCertRevokeReq) (*pb.CAStatus, error) {
	Trace.Println("grpc TCAP:RevokeCertificate")

	return nil, errors.New("not yet implemented")
}

// RevokeCertificateSet revokes a certificate set from the TCA.  Not yet implemented.
func (tcap *TCAP) RevokeCertificateSet(context.Context, *pb.TCertRevokeSetReq) (*pb.CAStatus, error) {
	Trace.Println("grpc TCAP:RevokeCertificateSet")

	return nil, errors.New("not yet implemented")
}

func isEnabledAttributesEncryption() bool {
	//TODO this code is commented because attributes encryption is not yet implemented.
	//return viper.GetBool("tca.attribute-encryption.enabled")
	return false
}
<|endoftext|>


<commit_before>// Use and distribution licensed under the Apache license version 2.
//
// See the COPYING file in the root project directory for full text.
//

package ghw

import (
	"fmt"
	"io/ioutil"
	"os"
	"path/filepath"
	"sort"
	"strconv"
	"strings"
)

func cachesForNode(nodeID int) ([]*MemoryCache, error) {
	// The /sys/devices/node/nodeX directory contains a subdirectory called
	// 'cpuX' for each logical processor assigned to the node. Each of those
	// subdirectories containers a 'cache' subdirectory which contains a number
	// of subdirectories beginning with 'index' and ending in the cache's
	// internal 0-based identifier. Those subdirectories contain a number of
	// files, including 'shared_cpu_list', 'size', and 'type' which we use to
	// determine cache characteristics.
	path := filepath.Join(
		pathSysDevicesSystemNode(),
		fmt.Sprintf("node%d", nodeID),
	)
	caches := make(map[string]*MemoryCache, 0)

	files, err := ioutil.ReadDir(path)
	if err != nil {
		return nil, err
	}
	for _, file := range files {
		filename := file.Name()
		if !strings.HasPrefix(filename, "cpu") {
			continue
		}
		if filename == "cpumap" || filename == "cpulist" {
			// There are two files in the node directory that start with 'cpu'
			// but are not subdirectories ('cpulist' and 'cpumap'). Ignore
			// these files.
			continue
		}
		// Grab the logical processor ID by cutting the integer from the
		// /sys/devices/system/node/nodeX/cpuX filename
		cpuPath := filepath.Join(path, filename)
		lpID, _ := strconv.Atoi(filename[3:])

		// Inspect the caches for each logical processor. There will be a
		// /sys/devices/system/node/nodeX/cpuX/cache directory containing a
		// number of directories beginning with the prefix "index" followed by
		// a number. The number indicates the level of the cache, which
		// indicates the "distance" from the processor. Each of these
		// directories contains information about the size of that level of
		// cache and the processors mapped to it.
		cachePath := filepath.Join(cpuPath, "cache")
		cacheDirFiles, err := ioutil.ReadDir(cachePath)
		if err != nil {
			return nil, err
		}
		for _, cacheDirFile := range cacheDirFiles {
			cacheDirFileName := cacheDirFile.Name()
			if !strings.HasPrefix(cacheDirFileName, "index") {
				continue
			}

			typePath := filepath.Join(cachePath, cacheDirFileName, "type")
			cacheTypeContents, err := ioutil.ReadFile(typePath)
			if err != nil {
				continue
			}
			var cacheType MemoryCacheType
			switch string(cacheTypeContents[:len(cacheTypeContents)-1]) {
			case "Data":
				cacheType = DATA
			case "Instruction":
				cacheType = INSTRUCTION
			default:
				cacheType = UNIFIED
			}
			level := memoryCacheLevel(nodeID, lpID)
			size := memoryCacheSize(nodeID, lpID, level)

			scpuPath := filepath.Join(
				cachePath,
				cacheDirFileName,
				"shared_cpu_map",
			)
			sharedCpuMap, err := ioutil.ReadFile(scpuPath)
			if err != nil {
				continue
			}
			// The cache information is repeated for each node, so here, we
			// just ensure that we only have a one MemoryCache object for each
			// unique combination of level, type and processor map
			cacheKey := fmt.Sprintf("%d-%d-%s", level, cacheType, sharedCpuMap[:len(sharedCpuMap)-1])
			cache, exists := caches[cacheKey]
			if !exists {
				cache = &MemoryCache{
					Level:             uint8(level),
					Type:              cacheType,
					SizeBytes:         uint64(size) * uint64(KB),
					LogicalProcessors: make([]uint32, 0),
				}
				caches[cacheKey] = cache
			}
			cache.LogicalProcessors = append(
				cache.LogicalProcessors,
				uint32(lpID),
			)
		}
	}

	cacheVals := make([]*MemoryCache, len(caches))
	x := 0
	for _, c := range caches {
		// ensure the cache's processor set is sorted by logical process ID
		sort.Sort(SortByLogicalProcessorId(c.LogicalProcessors))
		cacheVals[x] = c
		x++
	}

	return cacheVals, nil
}

func pathNodeCPU(nodeID int, lpID int) string {
	return filepath.Join(
		pathSysDevicesSystemNode(),
		fmt.Sprintf("node%d", nodeID),
		fmt.Sprintf("cpu%d", lpID),
	)
}

func pathNodeCPUCache(nodeID int, lpID int) string {
	return filepath.Join(
		pathNodeCPU(nodeID, lpID),
		"cache",
	)
}

func pathNodeCPUCacheLevel(nodeID int, lpID int, cacheLevel int) string {
	return filepath.Join(
		pathNodeCPUCache(nodeID, lpID),
		fmt.Sprintf("index%d", cacheLevel),
	)
}

func memoryCacheSize(nodeID int, lpID int, cacheLevel int) int {
	sizePath := filepath.Join(
		pathNodeCPUCacheLevel(nodeID, lpID, cacheLevel),
		"size",
	)
	sizeContents, err := ioutil.ReadFile(sizePath)
	if err != nil {
		fmt.Fprintf(os.Stderr, "Unable to read %s: %s", sizePath, err)
		return -1
	}
	// size comes as XK\n, so we trim off the K and the newline.
	size, err := strconv.Atoi(string(sizeContents[:len(sizeContents)-2]))
	if err != nil {
		fmt.Fprintf(os.Stderr, "Unable to parse int from %s", sizeContents)
		return -1
	}
	return size
}

func memoryCacheLevel(nodeID int, lpID int) int {
	levelPath := filepath.Join(
		pathNodeCPUCache(nodeID, lpID),
		"level",
	)
	levelContents, err := ioutil.ReadFile(levelPath)
	if err != nil {
		fmt.Fprintf(os.Stderr, "Unable to read %s: %s", levelPath, err)
		return -1
	}
	// levelContents is now a []byte with the last byte being a newline
	// character. Trim that off and convert the contents to an integer.
	level, err := strconv.Atoi(string(levelContents[:len(levelContents)-1]))
	if err != nil {
		fmt.Fprintf(os.Stderr, "Unable to parse int from %s", levelContents)
		return -1
	}
	return level
}
<commit_msg>lint-fixes: pull cache type getter out<commit_after>// Use and distribution licensed under the Apache license version 2.
//
// See the COPYING file in the root project directory for full text.
//

package ghw

import (
	"fmt"
	"io/ioutil"
	"os"
	"path/filepath"
	"sort"
	"strconv"
	"strings"
)

func cachesForNode(nodeID int) ([]*MemoryCache, error) {
	// The /sys/devices/node/nodeX directory contains a subdirectory called
	// 'cpuX' for each logical processor assigned to the node. Each of those
	// subdirectories containers a 'cache' subdirectory which contains a number
	// of subdirectories beginning with 'index' and ending in the cache's
	// internal 0-based identifier. Those subdirectories contain a number of
	// files, including 'shared_cpu_list', 'size', and 'type' which we use to
	// determine cache characteristics.
	path := filepath.Join(
		pathSysDevicesSystemNode(),
		fmt.Sprintf("node%d", nodeID),
	)
	caches := make(map[string]*MemoryCache, 0)

	files, err := ioutil.ReadDir(path)
	if err != nil {
		return nil, err
	}
	for _, file := range files {
		filename := file.Name()
		if !strings.HasPrefix(filename, "cpu") {
			continue
		}
		if filename == "cpumap" || filename == "cpulist" {
			// There are two files in the node directory that start with 'cpu'
			// but are not subdirectories ('cpulist' and 'cpumap'). Ignore
			// these files.
			continue
		}
		// Grab the logical processor ID by cutting the integer from the
		// /sys/devices/system/node/nodeX/cpuX filename
		cpuPath := filepath.Join(path, filename)
		lpID, _ := strconv.Atoi(filename[3:])

		// Inspect the caches for each logical processor. There will be a
		// /sys/devices/system/node/nodeX/cpuX/cache directory containing a
		// number of directories beginning with the prefix "index" followed by
		// a number. The number indicates the level of the cache, which
		// indicates the "distance" from the processor. Each of these
		// directories contains information about the size of that level of
		// cache and the processors mapped to it.
		cachePath := filepath.Join(cpuPath, "cache")
		cacheDirFiles, err := ioutil.ReadDir(cachePath)
		if err != nil {
			return nil, err
		}
		for _, cacheDirFile := range cacheDirFiles {
			cacheDirFileName := cacheDirFile.Name()
			if !strings.HasPrefix(cacheDirFileName, "index") {
				continue
			}

			cacheType := memoryCacheType(nodeID, lpID)
			level := memoryCacheLevel(nodeID, lpID)
			size := memoryCacheSize(nodeID, lpID, level)

			scpuPath := filepath.Join(
				cachePath,
				cacheDirFileName,
				"shared_cpu_map",
			)
			sharedCpuMap, err := ioutil.ReadFile(scpuPath)
			if err != nil {
				continue
			}
			// The cache information is repeated for each node, so here, we
			// just ensure that we only have a one MemoryCache object for each
			// unique combination of level, type and processor map
			cacheKey := fmt.Sprintf("%d-%d-%s", level, cacheType, sharedCpuMap[:len(sharedCpuMap)-1])
			cache, exists := caches[cacheKey]
			if !exists {
				cache = &MemoryCache{
					Level:             uint8(level),
					Type:              cacheType,
					SizeBytes:         uint64(size) * uint64(KB),
					LogicalProcessors: make([]uint32, 0),
				}
				caches[cacheKey] = cache
			}
			cache.LogicalProcessors = append(
				cache.LogicalProcessors,
				uint32(lpID),
			)
		}
	}

	cacheVals := make([]*MemoryCache, len(caches))
	x := 0
	for _, c := range caches {
		// ensure the cache's processor set is sorted by logical process ID
		sort.Sort(SortByLogicalProcessorId(c.LogicalProcessors))
		cacheVals[x] = c
		x++
	}

	return cacheVals, nil
}

func pathNodeCPU(nodeID int, lpID int) string {
	return filepath.Join(
		pathSysDevicesSystemNode(),
		fmt.Sprintf("node%d", nodeID),
		fmt.Sprintf("cpu%d", lpID),
	)
}

func pathNodeCPUCache(nodeID int, lpID int) string {
	return filepath.Join(
		pathNodeCPU(nodeID, lpID),
		"cache",
	)
}

func pathNodeCPUCacheLevel(nodeID int, lpID int, cacheLevel int) string {
	return filepath.Join(
		pathNodeCPUCache(nodeID, lpID),
		fmt.Sprintf("index%d", cacheLevel),
	)
}

func memoryCacheSize(nodeID int, lpID int, cacheLevel int) int {
	sizePath := filepath.Join(
		pathNodeCPUCacheLevel(nodeID, lpID, cacheLevel),
		"size",
	)
	sizeContents, err := ioutil.ReadFile(sizePath)
	if err != nil {
		fmt.Fprintf(os.Stderr, "Unable to read %s: %s", sizePath, err)
		return -1
	}
	// size comes as XK\n, so we trim off the K and the newline.
	size, err := strconv.Atoi(string(sizeContents[:len(sizeContents)-2]))
	if err != nil {
		fmt.Fprintf(os.Stderr, "Unable to parse int from %s", sizeContents)
		return -1
	}
	return size
}

func memoryCacheLevel(nodeID int, lpID int) int {
	levelPath := filepath.Join(
		pathNodeCPUCache(nodeID, lpID),
		"level",
	)
	levelContents, err := ioutil.ReadFile(levelPath)
	if err != nil {
		fmt.Fprintf(os.Stderr, "Unable to read %s: %s", levelPath, err)
		return -1
	}
	// levelContents is now a []byte with the last byte being a newline
	// character. Trim that off and convert the contents to an integer.
	level, err := strconv.Atoi(string(levelContents[:len(levelContents)-1]))
	if err != nil {
		fmt.Fprintf(os.Stderr, "Unable to parse int from %s", levelContents)
		return -1
	}
	return level
}

func memoryCacheType(nodeID int, lpID int) MemoryCacheType {
	typePath := filepath.Join(
		pathNodeCPUCache(nodeID, lpID),
		"type",
	)
	cacheTypeContents, err := ioutil.ReadFile(typePath)
	if err != nil {
		fmt.Fprintf(os.Stderr, "Unable to read %s: %s", typePath, err)
		return UNIFIED
	}
	switch string(cacheTypeContents[:len(cacheTypeContents)-1]) {
	case "Data":
		return DATA
	case "Instruction":
		return INSTRUCTION
	default:
		return UNIFIED
	}
}
<|endoftext|>