<commit_before>package vespa

import (
	"bytes"
	"crypto/tls"
	"encoding/json"
	"fmt"
	"math"
	"net/http"
	"sort"
	"strconv"
	"strings"
	"time"

	"github.com/vespa-engine/vespa/client/go/auth/auth0"
	"github.com/vespa-engine/vespa/client/go/auth/zts"
	"github.com/vespa-engine/vespa/client/go/util"
	"github.com/vespa-engine/vespa/client/go/version"
)

// CloudOptions configures URL and authentication for a cloud target.
type APIOptions struct {
	System         System
	TLSOptions     TLSOptions
	APIKey         []byte
	AuthConfigPath string
}

// CloudDeploymentOptions configures the deployment to manage through a cloud target.
type CloudDeploymentOptions struct {
	Deployment  Deployment
	TLSOptions  TLSOptions
	ClusterURLs map[string]string // Endpoints keyed on cluster name
}

type cloudTarget struct {
	apiOptions        APIOptions
	deploymentOptions CloudDeploymentOptions
	logOptions        LogOptions
	httpClient        util.HTTPClient
	ztsClient         ztsClient
}

type deploymentEndpoint struct {
	Cluster string `json:"cluster"`
	URL     string `json:"url"`
	Scope   string `json:"scope"`
}

type deploymentResponse struct {
	Endpoints []deploymentEndpoint `json:"endpoints"`
}

type jobResponse struct {
	Active bool                    `json:"active"`
	Status string                  `json:"status"`
	Log    map[string][]logMessage `json:"log"`
	LastID int64                   `json:"lastId"`
}

type logMessage struct {
	At      int64  `json:"at"`
	Type    string `json:"type"`
	Message string `json:"message"`
}

type ztsClient interface {
	AccessToken(domain string, certficiate tls.Certificate) (string, error)
}

// CloudTarget creates a Target for the Vespa Cloud or hosted Vespa platform.
func CloudTarget(httpClient util.HTTPClient, apiOptions APIOptions, deploymentOptions CloudDeploymentOptions, logOptions LogOptions) (Target, error) {
	ztsClient, err := zts.NewClient(zts.DefaultURL, httpClient)
	if err != nil {
		return nil, err
	}
	return &cloudTarget{
		httpClient:        httpClient,
		apiOptions:        apiOptions,
		deploymentOptions: deploymentOptions,
		logOptions:        logOptions,
		ztsClient:         ztsClient,
	}, nil
}

func (t *cloudTarget) findClusterURL(cluster string) (string, error) {
	clusters := make([]string, 0, len(t.deploymentOptions.ClusterURLs))
	for c := range t.deploymentOptions.ClusterURLs {
		clusters = append(clusters, c)
	}
	if cluster == "" {
		for _, url := range t.deploymentOptions.ClusterURLs {
			if len(t.deploymentOptions.ClusterURLs) == 1 {
				return url, nil
			} else {
				return "", fmt.Errorf("no cluster specified: found multiple clusters '%s'", strings.Join(clusters, "', '"))
			}
		}
	} else {
		url, ok := t.deploymentOptions.ClusterURLs[cluster]
		if !ok {
			return "", fmt.Errorf("invalid cluster '%s': must be one of '%s'", cluster, strings.Join(clusters, "', '"))
		}
		return url, nil
	}
	return "", fmt.Errorf("no endpoints found")
}

func (t *cloudTarget) Type() string {
	switch t.apiOptions.System.Name {
	case MainSystem.Name, CDSystem.Name:
		return TargetHosted
	}
	return TargetCloud
}

func (t *cloudTarget) Deployment() Deployment { return t.deploymentOptions.Deployment }

func (t *cloudTarget) Service(name string, timeout time.Duration, runID int64, cluster string) (*Service, error) {
	switch name {
	case DeployService:
		service := &Service{
			Name:       name,
			BaseURL:    t.apiOptions.System.URL,
			TLSOptions: t.apiOptions.TLSOptions,
			ztsClient:  t.ztsClient,
			httpClient: t.httpClient,
		}
		if timeout > 0 {
			status, err := service.Wait(timeout)
			if err != nil {
				return nil, err
			}
			if !isOK(status) {
				return nil, fmt.Errorf("got status %d from deploy service at %s", status, service.BaseURL)
			}
		}
		return service, nil
	case QueryService, DocumentService:
		if t.deploymentOptions.ClusterURLs == nil {
			if err := t.waitForEndpoints(timeout, runID); err != nil {
				return nil, err
			}
		}
		url, err := t.findClusterURL(cluster)
		if err != nil {
			return nil, err
		}
		t.deploymentOptions.TLSOptions.AthenzDomain = t.apiOptions.System.AthenzDomain
		return &Service{
			Name:       name,
			BaseURL:    url,
			TLSOptions: t.deploymentOptions.TLSOptions,
			ztsClient:  t.ztsClient,
			httpClient: t.httpClient,
		}, nil
	}
	return nil, fmt.Errorf("unknown service: %s", name)
}

func (t *cloudTarget) SignRequest(req *http.Request, keyID string) error {
	if t.apiOptions.System.IsPublic() {
		if t.apiOptions.APIKey != nil {
			signer := NewRequestSigner(keyID, t.apiOptions.APIKey)
			return signer.SignRequest(req)
		} else {
			return t.addAuth0AccessToken(req)
		}
	} else {
		if t.apiOptions.TLSOptions.KeyPair.Certificate == nil {
			return fmt.Errorf("system %s requires a certificate for authentication", t.apiOptions.System.Name)
		}
		return nil
	}
}

func (t *cloudTarget) CheckVersion(clientVersion version.Version) error {
	if clientVersion.IsZero() { // development version is always fine
		return nil
	}
	req, err := http.NewRequest("GET", fmt.Sprintf("%s/cli/v1/", t.apiOptions.System.URL), nil)
	if err != nil {
		return err
	}
	response, err := t.httpClient.Do(req, 10*time.Second)
	if err != nil {
		return err
	}
	defer response.Body.Close()
	var cliResponse struct {
		MinVersion string `json:"minVersion"`
	}
	dec := json.NewDecoder(response.Body)
	if err := dec.Decode(&cliResponse); err != nil {
		return err
	}
	minVersion, err := version.Parse(cliResponse.MinVersion)
	if err != nil {
		return err
	}
	if clientVersion.Less(minVersion) {
		return fmt.Errorf("client version %s is less than the minimum supported version: %s", clientVersion, minVersion)
	}
	return nil
}

func (t *cloudTarget) addAuth0AccessToken(request *http.Request) error {
	client, err := auth0.New(t.apiOptions.AuthConfigPath, t.apiOptions.System.Name, t.apiOptions.System.URL)
	if err != nil {
		return err
	}
	accessToken, err := client.GetAccessToken()
	if err != nil {
		return err
	}
	if request.Header == nil {
		request.Header = make(http.Header)
	}
	request.Header.Set("Authorization", "Bearer "+accessToken)
	return nil
}

func (t *cloudTarget) logsURL() string {
	return fmt.Sprintf("%s/application/v4/tenant/%s/application/%s/instance/%s/environment/%s/region/%s/logs",
		t.apiOptions.System.URL,
		t.deploymentOptions.Deployment.Application.Tenant, t.deploymentOptions.Deployment.Application.Application, t.deploymentOptions.Deployment.Application.Instance,
		t.deploymentOptions.Deployment.Zone.Environment, t.deploymentOptions.Deployment.Zone.Region)
}

func (t *cloudTarget) PrintLog(options LogOptions) error {
	req, err := http.NewRequest("GET", t.logsURL(), nil)
	if err != nil {
		return err
	}
	lastFrom := options.From
	requestFunc := func() *http.Request {
		fromMillis := lastFrom.Unix() * 1000
		q := req.URL.Query()
		q.Set("from", strconv.FormatInt(fromMillis, 10))
		if !options.To.IsZero() {
			toMillis := options.To.Unix() * 1000
			q.Set("to", strconv.FormatInt(toMillis, 10))
		}
		req.URL.RawQuery = q.Encode()
		t.SignRequest(req, t.deploymentOptions.Deployment.Application.SerializedForm())
		return req
	}
	logFunc := func(status int, response []byte) (bool, error) {
		if ok, err := isCloudOK(status); !ok {
			return ok, err
		}
		logEntries, err := ReadLogEntries(bytes.NewReader(response))
		if err != nil {
			return true, err
		}
		for _, le := range logEntries {
			if !le.Time.After(lastFrom) {
				continue
			}
			if LogLevel(le.Level) > options.Level {
				continue
			}
			fmt.Fprintln(options.Writer, le.Format(options.Dequote))
		}
		if len(logEntries) > 0 {
			lastFrom = logEntries[len(logEntries)-1].Time
		}
		return false, nil
	}
	var timeout time.Duration
	if options.Follow {
		timeout = math.MaxInt64 // No timeout
	}
	_, err = wait(t.httpClient, logFunc, requestFunc, &t.apiOptions.TLSOptions.KeyPair, timeout)
	return err
}

func (t *cloudTarget) waitForEndpoints(timeout time.Duration, runID int64) error {
	if runID > 0 {
		if err := t.waitForRun(runID, timeout); err != nil {
			return err
		}
	}
	return t.discoverEndpoints(timeout)
}

func (t *cloudTarget) waitForRun(runID int64, timeout time.Duration) error {
	runURL := fmt.Sprintf("%s/application/v4/tenant/%s/application/%s/instance/%s/job/%s-%s/run/%d",
		t.apiOptions.System.URL,
		t.deploymentOptions.Deployment.Application.Tenant, t.deploymentOptions.Deployment.Application.Application, t.deploymentOptions.Deployment.Application.Instance,
		t.deploymentOptions.Deployment.Zone.Environment, t.deploymentOptions.Deployment.Zone.Region, runID)
	req, err := http.NewRequest("GET", runURL, nil)
	if err != nil {
		return err
	}
	lastID := int64(-1)
	requestFunc := func() *http.Request {
		q := req.URL.Query()
		q.Set("after", strconv.FormatInt(lastID, 10))
		req.URL.RawQuery = q.Encode()
		if err := t.SignRequest(req, t.deploymentOptions.Deployment.Application.SerializedForm()); err != nil {
			panic(err)
		}
		return req
	}
	jobSuccessFunc := func(status int, response []byte) (bool, error) {
		if ok, err := isCloudOK(status); !ok {
			return ok, err
		}
		var resp jobResponse
		if err := json.Unmarshal(response, &resp); err != nil {
			return false, nil
		}
		if t.logOptions.Writer != nil {
			lastID = t.printLog(resp, lastID)
		}
		if resp.Active {
			return false, nil
		}
		if resp.Status != "success" {
			return false, fmt.Errorf("run %d ended with unsuccessful status: %s", runID, resp.Status)
		}
		return true, nil
	}
	_, err = wait(t.httpClient, jobSuccessFunc, requestFunc, &t.apiOptions.TLSOptions.KeyPair, timeout)
	return err
}

func (t *cloudTarget) printLog(response jobResponse, last int64) int64 {
	if response.LastID == 0 {
		return last
	}
	var msgs []logMessage
	for step, stepMsgs := range response.Log {
		for _, msg := range stepMsgs {
			if step == "copyVespaLogs" && LogLevel(msg.Type) > t.logOptions.Level || LogLevel(msg.Type) == 3 {
				continue
			}
			msgs = append(msgs, msg)
		}
	}
	sort.Slice(msgs, func(i, j int) bool { return msgs[i].At < msgs[j].At })
	for _, msg := range msgs {
		tm := time.Unix(msg.At/1000, (msg.At%1000)*1000)
		fmtTime := tm.Format("15:04:05")
		fmt.Fprintf(t.logOptions.Writer, "[%s] %-7s %s\n", fmtTime, msg.Type, msg.Message)
	}
	return response.LastID
}

func (t *cloudTarget) discoverEndpoints(timeout time.Duration) error {
	deploymentURL := fmt.Sprintf("%s/application/v4/tenant/%s/application/%s/instance/%s/environment/%s/region/%s",
		t.apiOptions.System.URL,
		t.deploymentOptions.Deployment.Application.Tenant, t.deploymentOptions.Deployment.Application.Application, t.deploymentOptions.Deployment.Application.Instance,
		t.deploymentOptions.Deployment.Zone.Environment, t.deploymentOptions.Deployment.Zone.Region)
	req, err := http.NewRequest("GET", deploymentURL, nil)
	if err != nil {
		return err
	}
	if err := t.SignRequest(req, t.deploymentOptions.Deployment.Application.SerializedForm()); err != nil {
		return err
	}
	urlsByCluster := make(map[string]string)
	endpointFunc := func(status int, response []byte) (bool, error) {
		if ok, err := isCloudOK(status); !ok {
			return ok, err
		}
		var resp deploymentResponse
		if err := json.Unmarshal(response, &resp); err != nil {
			return false, nil
		}
		if len(resp.Endpoints) == 0 {
			return false, nil
		}
		for _, endpoint := range resp.Endpoints {
			if endpoint.Scope != "zone" {
				continue
			}
			urlsByCluster[endpoint.Cluster] = endpoint.URL
		}
		return true, nil
	}
	if _, err = wait(t.httpClient, endpointFunc, func() *http.Request { return req }, &t.apiOptions.TLSOptions.KeyPair, timeout); err != nil {
		return err
	}
	if len(urlsByCluster) == 0 {
		return fmt.Errorf("no endpoints discovered")
	}
	t.deploymentOptions.ClusterURLs = urlsByCluster
	return nil
}

func isCloudOK(status int) (bool, error) {
	if status == 401 {
		// when retrying we should give up immediately if we're not authorized
		return false, fmt.Errorf("status %d: invalid credentials", status)
	}
	return isOK(status), nil
}
<commit_msg>Include deployment in endpoint discovery error<commit_after>package vespa

import (
	"bytes"
	"crypto/tls"
	"encoding/json"
	"fmt"
	"math"
	"net/http"
	"sort"
	"strconv"
	"strings"
	"time"

	"github.com/vespa-engine/vespa/client/go/auth/auth0"
	"github.com/vespa-engine/vespa/client/go/auth/zts"
	"github.com/vespa-engine/vespa/client/go/util"
	"github.com/vespa-engine/vespa/client/go/version"
)

// CloudOptions configures URL and authentication for a cloud target.
type APIOptions struct {
	System         System
	TLSOptions     TLSOptions
	APIKey         []byte
	AuthConfigPath string
}

// CloudDeploymentOptions configures the deployment to manage through a cloud target.
type CloudDeploymentOptions struct {
	Deployment  Deployment
	TLSOptions  TLSOptions
	ClusterURLs map[string]string // Endpoints keyed on cluster name
}

type cloudTarget struct {
	apiOptions        APIOptions
	deploymentOptions CloudDeploymentOptions
	logOptions        LogOptions
	httpClient        util.HTTPClient
	ztsClient         ztsClient
}

type deploymentEndpoint struct {
	Cluster string `json:"cluster"`
	URL     string `json:"url"`
	Scope   string `json:"scope"`
}

type deploymentResponse struct {
	Endpoints []deploymentEndpoint `json:"endpoints"`
}

type jobResponse struct {
	Active bool                    `json:"active"`
	Status string                  `json:"status"`
	Log    map[string][]logMessage `json:"log"`
	LastID int64                   `json:"lastId"`
}

type logMessage struct {
	At      int64  `json:"at"`
	Type    string `json:"type"`
	Message string `json:"message"`
}

type ztsClient interface {
	AccessToken(domain string, certficiate tls.Certificate) (string, error)
}

// CloudTarget creates a Target for the Vespa Cloud or hosted Vespa platform.
func CloudTarget(httpClient util.HTTPClient, apiOptions APIOptions, deploymentOptions CloudDeploymentOptions, logOptions LogOptions) (Target, error) {
	ztsClient, err := zts.NewClient(zts.DefaultURL, httpClient)
	if err != nil {
		return nil, err
	}
	return &cloudTarget{
		httpClient:        httpClient,
		apiOptions:        apiOptions,
		deploymentOptions: deploymentOptions,
		logOptions:        logOptions,
		ztsClient:         ztsClient,
	}, nil
}

func (t *cloudTarget) findClusterURL(cluster string) (string, error) {
	clusters := make([]string, 0, len(t.deploymentOptions.ClusterURLs))
	for c := range t.deploymentOptions.ClusterURLs {
		clusters = append(clusters, c)
	}
	if cluster == "" {
		for _, url := range t.deploymentOptions.ClusterURLs {
			if len(t.deploymentOptions.ClusterURLs) == 1 {
				return url, nil
			} else {
				return "", fmt.Errorf("no cluster specified: found multiple clusters '%s'", strings.Join(clusters, "', '"))
			}
		}
	} else {
		url, ok := t.deploymentOptions.ClusterURLs[cluster]
		if !ok {
			return "", fmt.Errorf("invalid cluster '%s': must be one of '%s'", cluster, strings.Join(clusters, "', '"))
		}
		return url, nil
	}
	return "", fmt.Errorf("no endpoints found")
}

func (t *cloudTarget) Type() string {
	switch t.apiOptions.System.Name {
	case MainSystem.Name, CDSystem.Name:
		return TargetHosted
	}
	return TargetCloud
}

func (t *cloudTarget) Deployment() Deployment { return t.deploymentOptions.Deployment }

func (t *cloudTarget) Service(name string, timeout time.Duration, runID int64, cluster string) (*Service, error) {
	switch name {
	case DeployService:
		service := &Service{
			Name:       name,
			BaseURL:    t.apiOptions.System.URL,
			TLSOptions: t.apiOptions.TLSOptions,
			ztsClient:  t.ztsClient,
			httpClient: t.httpClient,
		}
		if timeout > 0 {
			status, err := service.Wait(timeout)
			if err != nil {
				return nil, err
			}
			if !isOK(status) {
				return nil, fmt.Errorf("got status %d from deploy service at %s", status, service.BaseURL)
			}
		}
		return service, nil
	case QueryService, DocumentService:
		if t.deploymentOptions.ClusterURLs == nil {
			if err := t.waitForEndpoints(timeout, runID); err != nil {
				return nil, err
			}
		}
		url, err := t.findClusterURL(cluster)
		if err != nil {
			return nil, err
		}
		t.deploymentOptions.TLSOptions.AthenzDomain = t.apiOptions.System.AthenzDomain
		return &Service{
			Name:       name,
			BaseURL:    url,
			TLSOptions: t.deploymentOptions.TLSOptions,
			ztsClient:  t.ztsClient,
			httpClient: t.httpClient,
		}, nil
	}
	return nil, fmt.Errorf("unknown service: %s", name)
}

func (t *cloudTarget) SignRequest(req *http.Request, keyID string) error {
	if t.apiOptions.System.IsPublic() {
		if t.apiOptions.APIKey != nil {
			signer := NewRequestSigner(keyID, t.apiOptions.APIKey)
			return signer.SignRequest(req)
		} else {
			return t.addAuth0AccessToken(req)
		}
	} else {
		if t.apiOptions.TLSOptions.KeyPair.Certificate == nil {
			return fmt.Errorf("system %s requires a certificate for authentication", t.apiOptions.System.Name)
		}
		return nil
	}
}

func (t *cloudTarget) CheckVersion(clientVersion version.Version) error {
	if clientVersion.IsZero() { // development version is always fine
		return nil
	}
	req, err := http.NewRequest("GET", fmt.Sprintf("%s/cli/v1/", t.apiOptions.System.URL), nil)
	if err != nil {
		return err
	}
	response, err := t.httpClient.Do(req, 10*time.Second)
	if err != nil {
		return err
	}
	defer response.Body.Close()
	var cliResponse struct {
		MinVersion string `json:"minVersion"`
	}
	dec := json.NewDecoder(response.Body)
	if err := dec.Decode(&cliResponse); err != nil {
		return err
	}
	minVersion, err := version.Parse(cliResponse.MinVersion)
	if err != nil {
		return err
	}
	if clientVersion.Less(minVersion) {
		return fmt.Errorf("client version %s is less than the minimum supported version: %s", clientVersion, minVersion)
	}
	return nil
}

func (t *cloudTarget) addAuth0AccessToken(request *http.Request) error {
	client, err := auth0.New(t.apiOptions.AuthConfigPath, t.apiOptions.System.Name, t.apiOptions.System.URL)
	if err != nil {
		return err
	}
	accessToken, err := client.GetAccessToken()
	if err != nil {
		return err
	}
	if request.Header == nil {
		request.Header = make(http.Header)
	}
	request.Header.Set("Authorization", "Bearer "+accessToken)
	return nil
}

func (t *cloudTarget) logsURL() string {
	return fmt.Sprintf("%s/application/v4/tenant/%s/application/%s/instance/%s/environment/%s/region/%s/logs",
		t.apiOptions.System.URL,
		t.deploymentOptions.Deployment.Application.Tenant, t.deploymentOptions.Deployment.Application.Application, t.deploymentOptions.Deployment.Application.Instance,
		t.deploymentOptions.Deployment.Zone.Environment, t.deploymentOptions.Deployment.Zone.Region)
}

func (t *cloudTarget) PrintLog(options LogOptions) error {
	req, err := http.NewRequest("GET", t.logsURL(), nil)
	if err != nil {
		return err
	}
	lastFrom := options.From
	requestFunc := func() *http.Request {
		fromMillis := lastFrom.Unix() * 1000
		q := req.URL.Query()
		q.Set("from", strconv.FormatInt(fromMillis, 10))
		if !options.To.IsZero() {
			toMillis := options.To.Unix() * 1000
			q.Set("to", strconv.FormatInt(toMillis, 10))
		}
		req.URL.RawQuery = q.Encode()
		t.SignRequest(req, t.deploymentOptions.Deployment.Application.SerializedForm())
		return req
	}
	logFunc := func(status int, response []byte) (bool, error) {
		if ok, err := isCloudOK(status); !ok {
			return ok, err
		}
		logEntries, err := ReadLogEntries(bytes.NewReader(response))
		if err != nil {
			return true, err
		}
		for _, le := range logEntries {
			if !le.Time.After(lastFrom) {
				continue
			}
			if LogLevel(le.Level) > options.Level {
				continue
			}
			fmt.Fprintln(options.Writer, le.Format(options.Dequote))
		}
		if len(logEntries) > 0 {
			lastFrom = logEntries[len(logEntries)-1].Time
		}
		return false, nil
	}
	var timeout time.Duration
	if options.Follow {
		timeout = math.MaxInt64 // No timeout
	}
	_, err = wait(t.httpClient, logFunc, requestFunc, &t.apiOptions.TLSOptions.KeyPair, timeout)
	return err
}

func (t *cloudTarget) waitForEndpoints(timeout time.Duration, runID int64) error {
	if runID > 0 {
		if err := t.waitForRun(runID, timeout); err != nil {
			return err
		}
	}
	return t.discoverEndpoints(timeout)
}

func (t *cloudTarget) waitForRun(runID int64, timeout time.Duration) error {
	runURL := fmt.Sprintf("%s/application/v4/tenant/%s/application/%s/instance/%s/job/%s-%s/run/%d",
		t.apiOptions.System.URL,
		t.deploymentOptions.Deployment.Application.Tenant, t.deploymentOptions.Deployment.Application.Application, t.deploymentOptions.Deployment.Application.Instance,
		t.deploymentOptions.Deployment.Zone.Environment, t.deploymentOptions.Deployment.Zone.Region, runID)
	req, err := http.NewRequest("GET", runURL, nil)
	if err != nil {
		return err
	}
	lastID := int64(-1)
	requestFunc := func() *http.Request {
		q := req.URL.Query()
		q.Set("after", strconv.FormatInt(lastID, 10))
		req.URL.RawQuery = q.Encode()
		if err := t.SignRequest(req, t.deploymentOptions.Deployment.Application.SerializedForm()); err != nil {
			panic(err)
		}
		return req
	}
	jobSuccessFunc := func(status int, response []byte) (bool, error) {
		if ok, err := isCloudOK(status); !ok {
			return ok, err
		}
		var resp jobResponse
		if err := json.Unmarshal(response, &resp); err != nil {
			return false, nil
		}
		if t.logOptions.Writer != nil {
			lastID = t.printLog(resp, lastID)
		}
		if resp.Active {
			return false, nil
		}
		if resp.Status != "success" {
			return false, fmt.Errorf("run %d ended with unsuccessful status: %s", runID, resp.Status)
		}
		return true, nil
	}
	_, err = wait(t.httpClient, jobSuccessFunc, requestFunc, &t.apiOptions.TLSOptions.KeyPair, timeout)
	return err
}

func (t *cloudTarget) printLog(response jobResponse, last int64) int64 {
	if response.LastID == 0 {
		return last
	}
	var msgs []logMessage
	for step, stepMsgs := range response.Log {
		for _, msg := range stepMsgs {
			if step == "copyVespaLogs" && LogLevel(msg.Type) > t.logOptions.Level || LogLevel(msg.Type) == 3 {
				continue
			}
			msgs = append(msgs, msg)
		}
	}
	sort.Slice(msgs, func(i, j int) bool { return msgs[i].At < msgs[j].At })
	for _, msg := range msgs {
		tm := time.Unix(msg.At/1000, (msg.At%1000)*1000)
		fmtTime := tm.Format("15:04:05")
		fmt.Fprintf(t.logOptions.Writer, "[%s] %-7s %s\n", fmtTime, msg.Type, msg.Message)
	}
	return response.LastID
}

func (t *cloudTarget) discoverEndpoints(timeout time.Duration) error {
	deploymentURL := fmt.Sprintf("%s/application/v4/tenant/%s/application/%s/instance/%s/environment/%s/region/%s",
		t.apiOptions.System.URL,
		t.deploymentOptions.Deployment.Application.Tenant, t.deploymentOptions.Deployment.Application.Application, t.deploymentOptions.Deployment.Application.Instance,
		t.deploymentOptions.Deployment.Zone.Environment, t.deploymentOptions.Deployment.Zone.Region)
	req, err := http.NewRequest("GET", deploymentURL, nil)
	if err != nil {
		return err
	}
	if err := t.SignRequest(req, t.deploymentOptions.Deployment.Application.SerializedForm()); err != nil {
		return err
	}
	urlsByCluster := make(map[string]string)
	endpointFunc := func(status int, response []byte) (bool, error) {
		if ok, err := isCloudOK(status); !ok {
			return ok, err
		}
		var resp deploymentResponse
		if err := json.Unmarshal(response, &resp); err != nil {
			return false, nil
		}
		if len(resp.Endpoints) == 0 {
			return false, nil
		}
		for _, endpoint := range resp.Endpoints {
			if endpoint.Scope != "zone" {
				continue
			}
			urlsByCluster[endpoint.Cluster] = endpoint.URL
		}
		return true, nil
	}
	if _, err = wait(t.httpClient, endpointFunc, func() *http.Request { return req }, &t.apiOptions.TLSOptions.KeyPair, timeout); err != nil {
		return err
	}
	if len(urlsByCluster) == 0 {
		return fmt.Errorf("no endpoints discovered for %s", t.deploymentOptions.Deployment)
	}
	t.deploymentOptions.ClusterURLs = urlsByCluster
	return nil
}

func isCloudOK(status int) (bool, error) {
	if status == 401 {
		// when retrying we should give up immediately if we're not authorized
		return false, fmt.Errorf("status %d: invalid credentials", status)
	}
	return isOK(status), nil
}
<|endoftext|>


<commit_before>// Copyright (c) Facebook, Inc. and its affiliates.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package cvefeed

import (
	"sort"
	"strings"
	"sync"
	"sync/atomic"
	"unsafe"

	"github.com/facebookincubator/flog"
	"github.com/facebookincubator/nvdtools/wfn"
)

const cacheEvictPercentage = 0.1 // every eviction cycle invalidates this part of cache size at once

// Index maps the CPEs to the entries in the NVD feed they mentioned in
type Index map[string][]Vuln

// NewIndex creates new Index from a slice of CVE entries
func NewIndex(d Dictionary) Index {
	idx := Index{}
	for _, entry := range d {
		set := map[string]bool{}
		for _, cpe := range entry.Config() {
			// Can happen, for instance, when the feed contains illegal binding of CPE name. Unfortunately, it happens to NVD,
			// e.g. embedded ? in cpe:2.3:a:disney:where\\'s_my_perry?_free:1.5.1:*:*:*:*:android:*:* of CVE-2014-5606
			if cpe == nil {
				continue
			}
			product := cpe.Product
			if wfn.HasWildcard(product) {
				product = wfn.Any
			}
			if !set[product] {
				set[product] = true
				idx[product] = append(idx[product], entry)
			}
		}
	}
	return idx
}

// MatchResult stores CVE and a slice of CPEs that matched it
type MatchResult struct {
	CVE  Vuln
	CPEs []*wfn.Attributes
}

// cachedCVEs stores cached CVEs, a channel to signal if the value is ready
type cachedCVEs struct {
	res           []MatchResult
	ready         chan struct{}
	size          int64
	evictionIndex int // position in eviction queue
}

// updateResSize calculates the size of cached MatchResult and assigns it to cves.size
func (cves *cachedCVEs) updateResSize(key string) {
	cves.size = int64(int(unsafe.Sizeof(key)) + len(key))
	if cves == nil {
		return
	}
	cves.size += int64(unsafe.Sizeof(cves.res))
	for i := range cves.res {
		cves.size += int64(unsafe.Sizeof(cves.res[i].CVE))
		for _, attr := range cves.res[i].CPEs {
			cves.size += int64(len(attr.Part)) + int64(unsafe.Sizeof(attr.Part))
			cves.size += int64(len(attr.Vendor)) + int64(unsafe.Sizeof(attr.Vendor))
			cves.size += int64(len(attr.Product)) + int64(unsafe.Sizeof(attr.Product))
			cves.size += int64(len(attr.Version)) + int64(unsafe.Sizeof(attr.Version))
			cves.size += int64(len(attr.Update)) + int64(unsafe.Sizeof(attr.Update))
			cves.size += int64(len(attr.Edition)) + int64(unsafe.Sizeof(attr.Edition))
			cves.size += int64(len(attr.SWEdition)) + int64(unsafe.Sizeof(attr.SWEdition))
			cves.size += int64(len(attr.TargetHW)) + int64(unsafe.Sizeof(attr.TargetHW))
			cves.size += int64(len(attr.Other)) + int64(unsafe.Sizeof(attr.Other))
			cves.size += int64(len(attr.Language)) + int64(unsafe.Sizeof(attr.Language))
		}
	}
}

// Cache caches CVEs for known CPEs
type Cache struct {
	data           map[string]*cachedCVEs
	evictionQ      *evictionQueue
	mu             sync.Mutex
	Dict           Dictionary
	Idx            Index
	RequireVersion bool  // ignore matching specifications that have Version == ANY
	MaxSize        int64 // maximum size of the cache, 0 -- unlimited, -1 -- no caching
	size           int64 // current size of the cache

	// Used to compute the hit ratio
	numLookups int64
	numHits    int64
}

// NewCache creates new Cache instance with dictionary dict.
func NewCache(dict Dictionary) *Cache {
	return &Cache{Dict: dict, evictionQ: new(evictionQueue)}
}

// SetRequireVersion sets if the instance of cache fails matching the dictionary
// records without Version attribute of CPE name.
// Returns a pointer to the instance of Cache, for easy chaining.
func (c *Cache) SetRequireVersion(requireVersion bool) *Cache {
	c.RequireVersion = requireVersion
	return c
}

// SetMaxSize sets maximum size of the cache to some pre-defined value,
// size of 0 disables eviction (makes the cache grow indefinitely),
// negative size disables caching.
// Returns a pointer to the instance of Cache, for easy chaining.
func (c *Cache) SetMaxSize(size int64) *Cache {
	c.MaxSize = size
	return c
}

// Get returns slice of CVEs for CPE names from cpes parameter;
// if CVEs aren't cached (and the feature is enabled) it finds them in cveDict and caches the results
func (c *Cache) Get(cpes []*wfn.Attributes) []MatchResult {
	atomic.AddInt64(&c.numLookups, 1)

	// negative max size of the cache disables caching
	if c.MaxSize < 0 {
		return c.match(cpes)
	}

	// otherwise, let's get to the business
	key := cacheKey(cpes)
	c.mu.Lock()
	if c.data == nil {
		c.data = make(map[string]*cachedCVEs)
	}
	cves := c.data[key]
	if cves != nil {
		atomic.AddInt64(&c.numHits, 1)

		// value is being computed, wait till ready
		c.mu.Unlock()
		<-cves.ready
		c.mu.Lock() // TODO: XXX: ugly, consider using atomic.Value instead
		cves.evictionIndex = c.evictionQ.touch(cves.evictionIndex)
		c.mu.Unlock()
		return cves.res
	}
	// first request; the goroutine that sent it computes the value
	cves = &cachedCVEs{ready: make(chan struct{})}
	c.data[key] = cves
	c.mu.Unlock()
	// now other requests for same key wait on the channel, and the requests for the different keys aren't blocked
	cves.res = c.match(cpes)
	cves.updateResSize(key)
	c.mu.Lock()
	c.size += cves.size
	if c.MaxSize != 0 && c.size > c.MaxSize {
		c.evict(int64(cacheEvictPercentage * float64(c.MaxSize)))
	}
	cves.evictionIndex = c.evictionQ.push(key)
	c.mu.Unlock()
	close(cves.ready)
	return cves.res
}

// match will return all match results based on the given cpes
func (c *Cache) match(cpes []*wfn.Attributes) []MatchResult {
	d := c.Dict
	if c.Idx != nil {
		d = c.dictFromIndex(cpes)
	}
	return c.matchDict(cpes, d)
}

// dictFromIndex creates CVE dictionary from entries indexed by CPE names
func (c *Cache) dictFromIndex(cpes []*wfn.Attributes) Dictionary {
	d := Dictionary{}
	if c.Idx == nil {
		return d
	}

	knownEntries := map[Vuln]bool{}
	addVulns := func(product string) {
		for _, vuln := range c.Idx[product] {
			if !knownEntries[vuln] {
				knownEntries[vuln] = true
				d[vuln.ID()] = vuln
			}
		}
	}

	for _, cpe := range cpes {
		if cpe == nil { // should never happen
			flog.Warning("nil CPE in list")
			continue
		}
		// any of the CPEs having product=ANY would mean we need to match against the entire dictionary
		if cpe.Product == wfn.Any {
			return c.Dict
		}
		addVulns(cpe.Product)
	}
	addVulns(wfn.Any)

	return d
}

// match matches the CPE names against internal vulnerability dictionary and returns a slice of matching resutls
func (c *Cache) matchDict(cpes []*wfn.Attributes, dict Dictionary) (results []MatchResult) {
	for _, v := range dict {
		if matches := v.Match(cpes, c.RequireVersion); len(matches) > 0 {
			results = append(results, MatchResult{v, matches})
		}
	}
	return results
}

// evict the least recently used records untile nbytes of capacity is achieved or no more records left.
// It is not concurrency-safe, c.mu should be locked before calling it.
func (c *Cache) evict(nbytes int64) {
	for c.size+nbytes > c.MaxSize {
		key := c.evictionQ.pop()
		cd, ok := c.data[key]
		if !ok { // should not happen
			panic("attempted to evict non-existent record")
		}
		c.size -= cd.size
		delete(c.data, key)
	}
}

func cacheKey(cpes []*wfn.Attributes) string {
	parts := make([]string, 0, len(cpes))
	for _, cpe := range cpes {
		if cpe == nil {
			continue
		}
		var out strings.Builder
		out.WriteString(cpe.Part)
		out.WriteByte('^')
		out.WriteString(cpe.Vendor)
		out.WriteByte('^')
		out.WriteString(cpe.Product)
		out.WriteByte('^')
		out.WriteString(cpe.Version)
		out.WriteByte('^')
		out.WriteString(cpe.Update)
		out.WriteByte('^')
		out.WriteString(cpe.Edition)
		out.WriteByte('^')
		out.WriteString(cpe.SWEdition)
		out.WriteByte('^')
		out.WriteString(cpe.TargetSW)
		out.WriteByte('^')
		out.WriteString(cpe.TargetHW)
		out.WriteByte('^')
		out.WriteString(cpe.Other)
		out.WriteByte('^')
		out.WriteString(cpe.Language)
		parts = append(parts, out.String())
	}
	sort.Strings(parts)
	return strings.Join(parts, "#")
}

// HitRatio returns the cache hit ratio, the number of cache hits to the number
// of lookups, as a percentage.
func (c *Cache) HitRatio() float64 {
	if c.numLookups == 0 {
		return 0
	}
	return float64(c.numHits) / float64(c.numLookups) * 100
}
<commit_msg>Fix #154.<commit_after>// Copyright (c) Facebook, Inc. and its affiliates.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package cvefeed

import (
	"sort"
	"strings"
	"sync"
	"sync/atomic"
	"unsafe"

	"github.com/facebookincubator/flog"
	"github.com/facebookincubator/nvdtools/wfn"
)

const cacheEvictPercentage = 0.1 // every eviction cycle invalidates this part of cache size at once

// Index maps the CPEs to the entries in the NVD feed they mentioned in
type Index map[string][]Vuln

// NewIndex creates new Index from a slice of CVE entries
func NewIndex(d Dictionary) Index {
	idx := Index{}
	for _, entry := range d {
		set := map[string]bool{}
		for _, cpe := range entry.Config() {
			// Can happen, for instance, when the feed contains illegal binding of CPE name. Unfortunately, it happens to NVD,
			// e.g. embedded ? in cpe:2.3:a:disney:where\\'s_my_perry?_free:1.5.1:*:*:*:*:android:*:* of CVE-2014-5606
			if cpe == nil {
				continue
			}
			product := cpe.Product
			if wfn.HasWildcard(product) {
				product = wfn.Any
			}
			if !set[product] {
				set[product] = true
				idx[product] = append(idx[product], entry)
			}
		}
	}
	return idx
}

// MatchResult stores CVE and a slice of CPEs that matched it
type MatchResult struct {
	CVE  Vuln
	CPEs []*wfn.Attributes
}

// cachedCVEs stores cached CVEs, a channel to signal if the value is ready
type cachedCVEs struct {
	res           []MatchResult
	ready         chan struct{}
	size          int64
	evictionIndex int // position in eviction queue
}

// updateResSize calculates the size of cached MatchResult and assigns it to cves.size
func (cves *cachedCVEs) updateResSize(key string) {
	cves.size = int64(int(unsafe.Sizeof(key)) + len(key))
	if cves == nil {
		return
	}
	cves.size += int64(unsafe.Sizeof(cves.res))
	for i := range cves.res {
		cves.size += int64(unsafe.Sizeof(cves.res[i].CVE))
		for _, attr := range cves.res[i].CPEs {
			cves.size += int64(len(attr.Part)) + int64(unsafe.Sizeof(attr.Part))
			cves.size += int64(len(attr.Vendor)) + int64(unsafe.Sizeof(attr.Vendor))
			cves.size += int64(len(attr.Product)) + int64(unsafe.Sizeof(attr.Product))
			cves.size += int64(len(attr.Version)) + int64(unsafe.Sizeof(attr.Version))
			cves.size += int64(len(attr.Update)) + int64(unsafe.Sizeof(attr.Update))
			cves.size += int64(len(attr.Edition)) + int64(unsafe.Sizeof(attr.Edition))
			cves.size += int64(len(attr.SWEdition)) + int64(unsafe.Sizeof(attr.SWEdition))
			cves.size += int64(len(attr.TargetHW)) + int64(unsafe.Sizeof(attr.TargetHW))
			cves.size += int64(len(attr.Other)) + int64(unsafe.Sizeof(attr.Other))
			cves.size += int64(len(attr.Language)) + int64(unsafe.Sizeof(attr.Language))
		}
	}
}

// Cache caches CVEs for known CPEs
type Cache struct {
	data           map[string]*cachedCVEs
	evictionQ      *evictionQueue
	mu             sync.Mutex
	Dict           Dictionary
	Idx            Index
	RequireVersion bool  // ignore matching specifications that have Version == ANY
	MaxSize        int64 // maximum size of the cache, 0 -- unlimited, -1 -- no caching
	size           int64 // current size of the cache

	// Used to compute the hit ratio
	numLookups int64
	numHits    int64
}

// NewCache creates new Cache instance with dictionary dict.
func NewCache(dict Dictionary) *Cache {
	return &Cache{Dict: dict, evictionQ: new(evictionQueue)}
}

// SetRequireVersion sets if the instance of cache fails matching the dictionary
// records without Version attribute of CPE name.
// Returns a pointer to the instance of Cache, for easy chaining.
func (c *Cache) SetRequireVersion(requireVersion bool) *Cache {
	c.RequireVersion = requireVersion
	return c
}

// SetMaxSize sets maximum size of the cache to some pre-defined value,
// size of 0 disables eviction (makes the cache grow indefinitely),
// negative size disables caching.
// Returns a pointer to the instance of Cache, for easy chaining.
func (c *Cache) SetMaxSize(size int64) *Cache {
	c.MaxSize = size
	return c
}

// Get returns slice of CVEs for CPE names from cpes parameter;
// if CVEs aren't cached (and the feature is enabled) it finds them in cveDict and caches the results
func (c *Cache) Get(cpes []*wfn.Attributes) []MatchResult {
	atomic.AddInt64(&c.numLookups, 1)

	// negative max size of the cache disables caching
	if c.MaxSize < 0 {
		return c.match(cpes)
	}

	// otherwise, let's get to the business
	key := cacheKey(cpes)
	c.mu.Lock()
	if c.data == nil {
		c.data = make(map[string]*cachedCVEs)
	}
	cves := c.data[key]
	if cves != nil {
		atomic.AddInt64(&c.numHits, 1)

		// value is being computed, wait till ready
		c.mu.Unlock()
		<-cves.ready
		c.mu.Lock() // TODO: XXX: ugly, consider using atomic.Value instead
		cves.evictionIndex = c.evictionQ.touch(cves.evictionIndex)
		c.mu.Unlock()
		return cves.res
	}
	// first request; the goroutine that sent it computes the value
	cves = &cachedCVEs{ready: make(chan struct{})}
	c.data[key] = cves
	c.mu.Unlock()
	// now other requests for same key wait on the channel, and the requests for the different keys aren't blocked
	cves.res = c.match(cpes)
	cves.updateResSize(key)
	c.mu.Lock()
	if c.MaxSize != 0 && c.size+cves.size > c.MaxSize {
		c.evict(int64(cacheEvictPercentage*float64(c.MaxSize)) + cves.size)
	}
	c.size += cves.size
	cves.evictionIndex = c.evictionQ.push(key)
	c.mu.Unlock()
	close(cves.ready)
	return cves.res
}

// match will return all match results based on the given cpes
func (c *Cache) match(cpes []*wfn.Attributes) []MatchResult {
	d := c.Dict
	if c.Idx != nil {
		d = c.dictFromIndex(cpes)
	}
	return c.matchDict(cpes, d)
}

// dictFromIndex creates CVE dictionary from entries indexed by CPE names
func (c *Cache) dictFromIndex(cpes []*wfn.Attributes) Dictionary {
	d := Dictionary{}
	if c.Idx == nil {
		return d
	}

	knownEntries := map[Vuln]bool{}
	addVulns := func(product string) {
		for _, vuln := range c.Idx[product] {
			if !knownEntries[vuln] {
				knownEntries[vuln] = true
				d[vuln.ID()] = vuln
			}
		}
	}

	for _, cpe := range cpes {
		if cpe == nil { // should never happen
			flog.Warning("nil CPE in list")
			continue
		}
		// any of the CPEs having product=ANY would mean we need to match against the entire dictionary
		if cpe.Product == wfn.Any {
			return c.Dict
		}
		addVulns(cpe.Product)
	}
	addVulns(wfn.Any)

	return d
}

// match matches the CPE names against internal vulnerability dictionary and returns a slice of matching resutls
func (c *Cache) matchDict(cpes []*wfn.Attributes, dict Dictionary) (results []MatchResult) {
	for _, v := range dict {
		if matches := v.Match(cpes, c.RequireVersion); len(matches) > 0 {
			results = append(results, MatchResult{v, matches})
		}
	}
	return results
}

// evict the least recently used records untile nbytes of capacity is achieved or no more records left.
// It is not concurrency-safe, c.mu should be locked before calling it.
func (c *Cache) evict(nbytes int64) {
	for c.size > 0 && c.size+nbytes > c.MaxSize {
		key := c.evictionQ.pop()
		cd, ok := c.data[key]
		if !ok { // should not happen
			panic("attempted to evict non-existent record")
		}
		c.size -= cd.size
		delete(c.data, key)
	}
}

func cacheKey(cpes []*wfn.Attributes) string {
	parts := make([]string, 0, len(cpes))
	for _, cpe := range cpes {
		if cpe == nil {
			continue
		}
		var out strings.Builder
		out.WriteString(cpe.Part)
		out.WriteByte('^')
		out.WriteString(cpe.Vendor)
		out.WriteByte('^')
		out.WriteString(cpe.Product)
		out.WriteByte('^')
		out.WriteString(cpe.Version)
		out.WriteByte('^')
		out.WriteString(cpe.Update)
		out.WriteByte('^')
		out.WriteString(cpe.Edition)
		out.WriteByte('^')
		out.WriteString(cpe.SWEdition)
		out.WriteByte('^')
		out.WriteString(cpe.TargetSW)
		out.WriteByte('^')
		out.WriteString(cpe.TargetHW)
		out.WriteByte('^')
		out.WriteString(cpe.Other)
		out.WriteByte('^')
		out.WriteString(cpe.Language)
		parts = append(parts, out.String())
	}
	sort.Strings(parts)
	return strings.Join(parts, "#")
}

// HitRatio returns the cache hit ratio, the number of cache hits to the number
// of lookups, as a percentage.
func (c *Cache) HitRatio() float64 {
	if c.numLookups == 0 {
		return 0
	}
	return float64(c.numHits) / float64(c.numLookups) * 100
}
<|endoftext|>


<commit_before>// Copyright Istio Authors
//
//   Licensed under the Apache License, Version 2.0 (the "License");
//   you may not use this file except in compliance with the License.
//   You may obtain a copy of the License at
//
//       http://www.apache.org/licenses/LICENSE-2.0
//
//   Unless required by applicable law or agreed to in writing, software
//   distributed under the License is distributed on an "AS IS" BASIS,
//   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
//   See the License for the specific language governing permissions and
//   limitations under the License.

package test

import (
	"bufio"
	"bytes"
	"io"
	"os"
	"testing"

	"github.com/google/go-cmp/cmp"
	apiextv1 "k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1"
	kubeyaml "k8s.io/apimachinery/pkg/util/yaml"
	"sigs.k8s.io/yaml"
)

const crdFilePath = "generated/kubernetes/customresourcedefinitions.gen.yaml"

func TestCRDwithStatus(t *testing.T) {
	crds := readCRDFile(t)

	expected := apiextv1.JSONSchemaProps{
		Description: "Status is the test status field.",
		Type:        "object",
		Properties: map[string]apiextv1.JSONSchemaProps{
			"analysis": {
				Type:        "string",
				Format:      "string",
				Description: "Analysis message.",
			},
			"condition": {
				Type:        "string",
				Format:      "string",
				Description: "Current state.",
			},
		},
	}

	for _, c := range crds {
		for _, v := range c.Spec.Versions {
			// status subresource must be enabled.
			if v.Subresources == nil && v.Subresources.Status == nil {
				t.Error("status subresource in spec must be enabled.")
			}

			got := v.Schema.OpenAPIV3Schema.Properties["status"]
			if e := cmp.Equal(expected, got); !e {
				t.Errorf("status specs are not equal, expected:\n%v\n, but got:\n%v", expected, got)
			}
		}
	}
}

func TestBazPreserveUnknownFields(t *testing.T) {
	crds := readCRDFile(t)

	for _, c := range crds {
		if c.Name == "bazs.foo.istio.io" {
			for _, v := range c.Spec.Versions {
				if v.Name == "v1alpha3" {
					if v.Schema.OpenAPIV3Schema.Properties["spec"].Properties["fieldB"].XPreserveUnknownFields == nil {
						t.Errorf("baz.v1alpha3.fieldB is expected to have x-preserve-unknown-fields set to true")
					}
					if v.Schema.OpenAPIV3Schema.Properties["spec"].Properties["fooBaz"].Properties["qux"].Items.Schema.Properties["value"].XPreserveUnknownFields == nil {
						t.Errorf("baz.v1alpha3.fooBaz.qux.[].value is expected to have x-preserve-unknown-fields set to true")
					}
				}
			}
		}
	}
}

func readCRDFile(t *testing.T) []*apiextv1.CustomResourceDefinition {
	r, err := os.Open(crdFilePath)
	if err != nil {
		t.Fatal(err)
	}

	f := bufio.NewReader(r)

	decoder := kubeyaml.NewYAMLReader(f)
	crds := make([]*apiextv1.CustomResourceDefinition, 0)
	for {
		doc, err := decoder.Read()
		if err == io.EOF {
			break
		}
		if err != nil {
			t.Fatalf("unable to read yaml document: %v", err)
			break
		}
		chunk := bytes.TrimSpace(doc)

		if len(chunk) == 0 {
			break
		}

		crd := apiextv1.CustomResourceDefinition{}
		if err = yaml.Unmarshal(chunk, &crd); err != nil {
			t.Fatal(err)
		}

		crds = append(crds, &crd)

	}
	return crds
}
<commit_msg>Fix lint error with golangci-lint 1.49.0 (#2116)<commit_after>// Copyright Istio Authors
//
//   Licensed under the Apache License, Version 2.0 (the "License");
//   you may not use this file except in compliance with the License.
//   You may obtain a copy of the License at
//
//       http://www.apache.org/licenses/LICENSE-2.0
//
//   Unless required by applicable law or agreed to in writing, software
//   distributed under the License is distributed on an "AS IS" BASIS,
//   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
//   See the License for the specific language governing permissions and
//   limitations under the License.

package test

import (
	"bufio"
	"bytes"
	"io"
	"os"
	"testing"

	"github.com/google/go-cmp/cmp"
	apiextv1 "k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1"
	kubeyaml "k8s.io/apimachinery/pkg/util/yaml"
	"sigs.k8s.io/yaml"
)

const crdFilePath = "generated/kubernetes/customresourcedefinitions.gen.yaml"

func TestCRDwithStatus(t *testing.T) {
	crds := readCRDFile(t)

	expected := apiextv1.JSONSchemaProps{
		Description: "Status is the test status field.",
		Type:        "object",
		Properties: map[string]apiextv1.JSONSchemaProps{
			"analysis": {
				Type:        "string",
				Format:      "string",
				Description: "Analysis message.",
			},
			"condition": {
				Type:        "string",
				Format:      "string",
				Description: "Current state.",
			},
		},
	}

	for _, c := range crds {
		for _, v := range c.Spec.Versions {
			// status subresource must be enabled.
			if v.Subresources == nil && v.Subresources.Status == nil {
				t.Error("status subresource in spec must be enabled.")
			}

			got := v.Schema.OpenAPIV3Schema.Properties["status"]
			if e := cmp.Equal(expected, got); !e {
				t.Errorf("status specs are not equal, expected:\n%v\n, but got:\n%v", expected, got)
			}
		}
	}
}

func TestBazPreserveUnknownFields(t *testing.T) {
	crds := readCRDFile(t)

	for _, c := range crds {
		if c.Name == "bazs.foo.istio.io" {
			for _, v := range c.Spec.Versions {
				if v.Name == "v1alpha3" {
					if v.Schema.OpenAPIV3Schema.Properties["spec"].Properties["fieldB"].XPreserveUnknownFields == nil {
						t.Errorf("baz.v1alpha3.fieldB is expected to have x-preserve-unknown-fields set to true")
					}
					if v.Schema.OpenAPIV3Schema.Properties["spec"].Properties["fooBaz"].Properties["qux"].Items.Schema.Properties["value"].XPreserveUnknownFields == nil {
						t.Errorf("baz.v1alpha3.fooBaz.qux.[].value is expected to have x-preserve-unknown-fields set to true")
					}
				}
			}
		}
	}
}

func readCRDFile(t *testing.T) []*apiextv1.CustomResourceDefinition {
	r, err := os.Open(crdFilePath)
	if err != nil {
		t.Fatal(err)
	}

	f := bufio.NewReader(r)

	decoder := kubeyaml.NewYAMLReader(f)
	crds := make([]*apiextv1.CustomResourceDefinition, 0)
	for {
		doc, err := decoder.Read()
		if err == io.EOF {
			break
		}
		if err != nil {
			t.Fatalf("unable to read yaml document: %v", err)
		}
		chunk := bytes.TrimSpace(doc)

		if len(chunk) == 0 {
			break
		}

		crd := apiextv1.CustomResourceDefinition{}
		if err = yaml.Unmarshal(chunk, &crd); err != nil {
			t.Fatal(err)
		}

		crds = append(crds, &crd)

	}
	return crds
}
<|endoftext|>


<commit_before>package main

import (
	"bufio"
	"regexp"
	"strings"

	"helm.sh/helm/v3/pkg/strvals"
)

//
// cluster information
//

const (
	clusterUnknown = iota
	clusterDockerDesktop
	clusterMinikube
	clusterKIND
	clusterK3D
	clusterGKE
	clusterAKS
	clusterEKS
	clusterEC2
)

// clusterInfoMessages represent custom messages for some environments
type clusterInfoMessages struct {
	// getServiceIP is the message printed for how to get the service IP
	getServiceIP string
}

// clusterInfo describes some properties about the cluster where the installation is performed
type clusterInfo struct {
	// a name for this kind of cluster (ie, gke)
	name string

	// True if this is a local environment (ie, minikube)
	isLocal bool

	// extra Chart values to set in this environment, as a list of assignments
	chartValues []string

	// customMessages are some custom messages for this environment
	customMessages clusterInfoMessages
}

func (c clusterInfo) CopyChartValuesTo(chartValues map[string]interface{}) {
	for _, assignment := range c.chartValues {
		err := strvals.ParseInto(assignment, chartValues)
		if err != nil {
			panic(err) // this should never happen: only if we have created a wrong `chartValues`
		}
	}
}

// clusterInfoDatabase is the database of information about
var clusterInfoDatabase = map[int]clusterInfo{
	clusterUnknown: {
		name: "unknown",
	},
	clusterDockerDesktop: {
		name:    "docker-desktop",
		isLocal: true,
	},
	clusterMinikube: {
		name:    "minikube",
		isLocal: true,
		customMessages: clusterInfoMessages{
			getServiceIP: "minikube service -n ambassador ambassador",
		},
	},
	clusterKIND: {
		name:    "KIND",
		isLocal: true,
		chartValues: []string{
			"service.type=NodePort",
		},
	},
	clusterK3D: {
		name:    "K3D",
		isLocal: true,
		chartValues: []string{
			"service.type=NodePort",
		},
	},
	clusterGKE: {
		name: "GKE",
	},
	clusterAKS: {
		name: "AKS",
	},
	clusterEKS: {
		name: "EKS",
	},
	clusterEC2: {
		name: "AEC2",
	},
}

func newClusterInfoFromNodeLabels(clusterNodeLabels string) clusterInfo {
	if strings.Contains(clusterNodeLabels, "docker-desktop") {
		return clusterInfoDatabase[clusterDockerDesktop]
	} else if strings.Contains(clusterNodeLabels, "minikube") {
		return clusterInfoDatabase[clusterMinikube]
	} else if strings.Contains(clusterNodeLabels, "kind") {
		return clusterInfoDatabase[clusterKIND]
	} else if strings.Contains(clusterNodeLabels, "k3d") {
		return clusterInfoDatabase[clusterK3D]
	} else if strings.Contains(clusterNodeLabels, "gke") {
		return clusterInfoDatabase[clusterGKE]
	} else if strings.Contains(clusterNodeLabels, "aks") {
		return clusterInfoDatabase[clusterAKS]
	} else if strings.Contains(clusterNodeLabels, "compute") {
		return clusterInfoDatabase[clusterEKS]
	} else if strings.Contains(clusterNodeLabels, "ec2") {
		return clusterInfoDatabase[clusterEC2]
	}
	return clusterInfoDatabase[clusterUnknown]
}

//
// installation methods
//

const (
	instNone = iota
	instOSS
	instAES
	instEdgectl
	instOperator
	instHelm
)

type installationMethodInfo struct {
	Method   int
	Label    string
	Name     string
	LongName string
	Image    *regexp.Regexp
}

// defInstallationMethodsInfo contains information
// about different installation methods. It can be used for detecting previous
// installation methods.
// NOTE: this is an ordered-list: higher-precision labels are first
var defInstallationMethodsInfo = []installationMethodInfo{
	{
		Method:   instEdgectl,
		Label:    "app.kubernetes.io/managed-by=edgectl",
		Name:     "edgectl",
		LongName: "edgectl",
		Image:    regexp.MustCompile("quay[.]io/datawire/aes:([[:^space:]]+)"),
	},
	{
		Method:   instOperator,
		Label:    "app.kubernetes.io/managed-by=amb-oper",
		Name:     "operator",
		LongName: "the Ambassador Operator",
		Image:    regexp.MustCompile("quay[.]io/datawire/aes:([[:^space:]]+)"),
	},
	{
		Method:   instHelm,
		Label:    "app.kubernetes.io/name=ambassador",
		Name:     "helm",
		LongName: "Helm",
		Image:    regexp.MustCompile("quay[.]io/datawire/aes:([[:^space:]]+)"),
	},
	{
		Method:   instAES,
		Label:    "product=aes",
		Name:     "aes",
		LongName: "AES manifests",
		Image:    regexp.MustCompile("quay[.]io/datawire/aes:([[:^space:]]+)"),
	},
	{
		Method:   instOSS,
		Label:    "service=ambassador",
		Name:     "oss",
		LongName: "OSS manifests",
		Image:    regexp.MustCompile("quay[.]io/datawire/ambassador:([[:^space:]]+)"),
	},
}

type deployGetter func(string) (string, error)

// getExistingInstallation tries to find an existing deployment by looking at a list of predefined labels,
// If such a deployment is found, it returns the image and the installation "family" (aes, oss, helm, etc).
// It returns an empty string if no installation could be found.
//
// TODO: Try to search all namespaces (which may fail due to RBAC) and capture a
//       correct namespace for an Ambassador installation (what if there is more than
//       one?), then proceed operating on that Ambassador in that namespace. Right now
//       we hard-code the "ambassador" namespace in a number of spots.
//
func getExistingInstallation(deploys deployGetter) (string, installationMethodInfo, error) {
	findFor := func(label string, imageRe *regexp.Regexp) (string, error) {
		deploys, err := deploys(label)
		if err != nil {
			return "", err
		}
		scanner := bufio.NewScanner(strings.NewReader(deploys))
		for scanner.Scan() {
			image := strings.TrimSpace(scanner.Text())
			if matches := imageRe.FindStringSubmatch(image); len(matches) == 2 {
				return matches[1], nil
			}
		}
		return "", scanner.Err()
	}

	for _, info := range defInstallationMethodsInfo {
		if info.Label == "" {
			continue
		}
		version, err := findFor(info.Label, info.Image)
		if err != nil {
			continue // ignore errors
		}
		if version != "" {
			return version, info, nil
		}
	}
	return "", installationMethodInfo{Method: instNone}, nil
}
<commit_msg>Recognize other Operator installations<commit_after>package main

import (
	"bufio"
	"regexp"
	"strings"

	"helm.sh/helm/v3/pkg/strvals"
)

//
// cluster information
//

const (
	clusterUnknown = iota
	clusterDockerDesktop
	clusterMinikube
	clusterKIND
	clusterK3D
	clusterGKE
	clusterAKS
	clusterEKS
	clusterEC2
)

// clusterInfoMessages represent custom messages for some environments
type clusterInfoMessages struct {
	// getServiceIP is the message printed for how to get the service IP
	getServiceIP string
}

// clusterInfo describes some properties about the cluster where the installation is performed
type clusterInfo struct {
	// a name for this kind of cluster (ie, gke)
	name string

	// True if this is a local environment (ie, minikube)
	isLocal bool

	// extra Chart values to set in this environment, as a list of assignments
	chartValues []string

	// customMessages are some custom messages for this environment
	customMessages clusterInfoMessages
}

func (c clusterInfo) CopyChartValuesTo(chartValues map[string]interface{}) {
	for _, assignment := range c.chartValues {
		err := strvals.ParseInto(assignment, chartValues)
		if err != nil {
			panic(err) // this should never happen: only if we have created a wrong `chartValues`
		}
	}
}

// clusterInfoDatabase is the database of information about
var clusterInfoDatabase = map[int]clusterInfo{
	clusterUnknown: {
		name: "unknown",
	},
	clusterDockerDesktop: {
		name:    "docker-desktop",
		isLocal: true,
	},
	clusterMinikube: {
		name:    "minikube",
		isLocal: true,
		customMessages: clusterInfoMessages{
			getServiceIP: "minikube service -n ambassador ambassador",
		},
	},
	clusterKIND: {
		name:    "KIND",
		isLocal: true,
		chartValues: []string{
			"service.type=NodePort",
		},
	},
	clusterK3D: {
		name:    "K3D",
		isLocal: true,
		chartValues: []string{
			"service.type=NodePort",
		},
	},
	clusterGKE: {
		name: "GKE",
	},
	clusterAKS: {
		name: "AKS",
	},
	clusterEKS: {
		name: "EKS",
	},
	clusterEC2: {
		name: "AEC2",
	},
}

func newClusterInfoFromNodeLabels(clusterNodeLabels string) clusterInfo {
	if strings.Contains(clusterNodeLabels, "docker-desktop") {
		return clusterInfoDatabase[clusterDockerDesktop]
	} else if strings.Contains(clusterNodeLabels, "minikube") {
		return clusterInfoDatabase[clusterMinikube]
	} else if strings.Contains(clusterNodeLabels, "kind") {
		return clusterInfoDatabase[clusterKIND]
	} else if strings.Contains(clusterNodeLabels, "k3d") {
		return clusterInfoDatabase[clusterK3D]
	} else if strings.Contains(clusterNodeLabels, "gke") {
		return clusterInfoDatabase[clusterGKE]
	} else if strings.Contains(clusterNodeLabels, "aks") {
		return clusterInfoDatabase[clusterAKS]
	} else if strings.Contains(clusterNodeLabels, "compute") {
		return clusterInfoDatabase[clusterEKS]
	} else if strings.Contains(clusterNodeLabels, "ec2") {
		return clusterInfoDatabase[clusterEC2]
	}
	return clusterInfoDatabase[clusterUnknown]
}

//
// installation methods
//

const (
	instNone = iota
	instOSS
	instAES
	instEdgectl
	instOperator
	instHelm
)

type installationMethodInfo struct {
	Method   int
	Label    string
	Name     string
	LongName string
	Image    *regexp.Regexp
}

// defInstallationMethodsInfo contains information
// about different installation methods. It can be used for detecting previous
// installation methods.
// NOTE: this is an ordered-list: higher-precision labels are first
var defInstallationMethodsInfo = []installationMethodInfo{
	{
		Method:   instEdgectl,
		Label:    "app.kubernetes.io/managed-by=edgectl",
		Name:     "edgectl",
		LongName: "edgectl",
		Image:    regexp.MustCompile("quay[.]io/datawire/aes:([[:^space:]]+)"),
	},
	{
		Method:   instOperator,
		Label:    "getambassador.io/installer=operator",
		Name:     "operator",
		LongName: "the Ambassador Operator",
		Image:    regexp.MustCompile("quay[.]io/datawire/aes:([[:^space:]]+)"),
	},
	{
		Method:   instHelm,
		Label:    "app.kubernetes.io/name=ambassador",
		Name:     "helm",
		LongName: "Helm",
		Image:    regexp.MustCompile("quay[.]io/datawire/aes:([[:^space:]]+)"),
	},
	{
		Method:   instAES,
		Label:    "product=aes",
		Name:     "aes",
		LongName: "AES manifests",
		Image:    regexp.MustCompile("quay[.]io/datawire/aes:([[:^space:]]+)"),
	},
	{
		Method:   instOSS,
		Label:    "service=ambassador",
		Name:     "oss",
		LongName: "OSS manifests",
		Image:    regexp.MustCompile("quay[.]io/datawire/ambassador:([[:^space:]]+)"),
	},
}

type deployGetter func(string) (string, error)

// getExistingInstallation tries to find an existing deployment by looking at a list of predefined labels,
// If such a deployment is found, it returns the image and the installation "family" (aes, oss, helm, etc).
// It returns an empty string if no installation could be found.
//
// TODO: Try to search all namespaces (which may fail due to RBAC) and capture a
//       correct namespace for an Ambassador installation (what if there is more than
//       one?), then proceed operating on that Ambassador in that namespace. Right now
//       we hard-code the "ambassador" namespace in a number of spots.
//
func getExistingInstallation(deploys deployGetter) (string, installationMethodInfo, error) {
	findFor := func(label string, imageRe *regexp.Regexp) (string, error) {
		deploys, err := deploys(label)
		if err != nil {
			return "", err
		}
		scanner := bufio.NewScanner(strings.NewReader(deploys))
		for scanner.Scan() {
			image := strings.TrimSpace(scanner.Text())
			if matches := imageRe.FindStringSubmatch(image); len(matches) == 2 {
				return matches[1], nil
			}
		}
		return "", scanner.Err()
	}

	for _, info := range defInstallationMethodsInfo {
		if info.Label == "" {
			continue
		}
		version, err := findFor(info.Label, info.Image)
		if err != nil {
			continue // ignore errors
		}
		if version != "" {
			return version, info, nil
		}
	}
	return "", installationMethodInfo{Method: instNone}, nil
}
<|endoftext|>


<commit_before>package mccli

import (
	"fmt"
	"os"

	"github.com/codegangsta/cli"
	"github.com/materials-commons/mcstore/cmd/pkg/mc"
)

var (
	projectCreateCommand = cli.Command{
		Name:    "create",
		Aliases: []string{"cr", "c"},
		Usage:   "Create a new project",
		Flags: []cli.Flag{
			cli.StringFlag{
				Name:  "directory, dir, d",
				Usage: "The base directory for the project",
			},
		},
		Action: projectCreateCLI,
	}
)

// projectCreateCLI implements the project create command.
func projectCreateCLI(c *cli.Context) {
	if len(c.Args()) != 1 {
		fmt.Println("You must specify a project name")
		os.Exit(1)
	}
	projectName := c.Args()[0]

	dirPath := c.String("directory")
	if !validateDirectoryPath(dirPath) {
		os.Exit(1)
	}

	client := mc.NewClientAPI()
	if err := client.CreateProject(projectName, dirPath); err != nil {
		fmt.Println("Unable to create project:", err)
		os.Exit(1)
	}

	fmt.Println("Project successfully created")
}

// validateDirectoryPath checks that the given directory path exists.
func validateDirectoryPath(path string) bool {
	if path == "" {
		fmt.Println("You must specify a local directory path where the project files are located.")
		return false
	}

	if _, err := os.Stat(path); err != nil {
		fmt.Println("Directory doesn't exist or you don't have access")
		return false
	}

	return true
}
<commit_msg>Add option to upload project after creating it.<commit_after>package mccli

import (
	"fmt"
	"os"

	"github.com/codegangsta/cli"
	"github.com/materials-commons/mcstore/cmd/pkg/mc"
)

var (
	projectCreateCommand = cli.Command{
		Name:    "create",
		Aliases: []string{"cr", "c"},
		Usage:   "Create a new project",
		Flags: []cli.Flag{
			cli.StringFlag{
				Name:  "directory, dir, d",
				Usage: "The base directory for the project",
			},
			cli.BoolFlag{
				Name:  "upload, up, u",
				Usage: "Upload project after creating it",
			},
			cli.IntFlag{
				Name:  "parallel, n",
				Value: 3,
				Usage: "Number of simultaneous uploads to perform, defaults to 3",
			},
		},
		Action: projectCreateCLI,
	}
)

// projectCreateCLI implements the project create command.
func projectCreateCLI(c *cli.Context) {
	if len(c.Args()) != 1 {
		fmt.Println("You must specify a project name")
		os.Exit(1)
	}
	projectName := c.Args()[0]

	dirPath := c.String("directory")
	if !validateDirectoryPath(dirPath) {
		os.Exit(1)
	}

	client := mc.NewClientAPI()
	if err := client.CreateProject(projectName, dirPath); err != nil {
		fmt.Println("Unable to create project:", err)
		os.Exit(1)
	}

	fmt.Println("Project successfully created.")

	if c.Bool("upload") {
		numThreads := getNumThreads(c)
		if err := client.UploadProject(projectName, numThreads); err != nil {
			fmt.Println("Project upload failed:", err)
			os.Exit(1)
		}
		fmt.Println("Project successfully uploaded.")
	}
}

// validateDirectoryPath checks that the given directory path exists.
func validateDirectoryPath(path string) bool {
	if path == "" {
		fmt.Println("You must specify a local directory path where the project files are located.")
		return false
	}

	if _, err := os.Stat(path); err != nil {
		fmt.Println("Directory doesn't exist or you don't have access")
		return false
	}

	return true
}
<|endoftext|>


<commit_before>package main

import (
	"errors"
	"os"
	"path/filepath"
	"strings"
	"testing"

	"github.com/richardlehane/siegfried/pkg/core/bytematcher"
	"github.com/richardlehane/siegfried/pkg/pronom"
)

var testSigs = filepath.Join("..", "r2d2", "data", "pronom.gob")

var b *bytematcher.Bytematcher

func init() {
	var err error
	droid, _, reports := pronom.ConfigPaths()
	puids, err = pronom.PuidsFromDroid(droid, reports)
	if err != nil {
		panic(err)
	}
	b, err = load(testSigs)
	if err != nil {
		panic(err)
	}
}

var root = filepath.Join(".", "testdata", "skeleton-suite")

func TestLoad(t *testing.T) {
	_, err := load(testSigs)
	if err != nil {
		t.Error(err)
	}
}

func check(i string, j []int) bool {
	for _, v := range j {
		if i == puids[v] {
			return true
		}
	}
	return false
}

func matchString(i []int) string {
	str := "[ "
	for _, v := range i {
		str += puids[v]
		str += " "
	}
	return str + "]"
}

func TestSuite(t *testing.T) {
	expect := make([]string, 0)
	names := make([]string, 0)
	wf := func(path string, info os.FileInfo, err error) error {
		if info.IsDir() {
			return nil
		}
		last := strings.Split(path, string(os.PathSeparator))
		path = last[len(last)-1]
		idx := strings.Index(path, "signature")
		if idx < 0 {
			idx = len(path)
		}
		strs := strings.Split(path[:idx-1], "-")
		if len(strs) == 2 {
			expect = append(expect, strings.Join(strs, "/"))
		} else if len(strs) == 3 {
			expect = append(expect, "x-fmt/"+strs[2])
		} else {
			return errors.New("long string encountered: " + path)
		}
		names = append(names, path)
		return nil
	}
	err := filepath.Walk(root, wf)
	if err != nil {
		t.Fatal(err)
	}
	matches, err := multiIdentify(b, root)
	if err != nil {
		t.Fatal(err)
	}
	if len(expect) != len(matches) {
		t.Error("Expect should equal matches")
	}
	var iter int
	for i, v := range expect {
		if !check(v, matches[i]) {
			t.Errorf("Failed to match signature %v; got %v; expected %v", names[i], matchString(matches[i]), v)

		} else {
			iter++
		}
	}
	if iter != len(expect) {
		t.Errorf("Matched %v out of %v signatures", iter, len(expect))
	}
}

func TestBrokenTravis(t *testing.T) {
	matches, err := identify(b, filepath.Join(root, "fmt", "fmt-288-signature-id-304.nul"))
	if err != nil {
		t.Fatal(err)
	}
	if len(matches) == 0 {
		t.Fatal("Expected fmt/288, got no results")
	}
	var match bool
	for _, v := range matches {
		if puids[v] == "fmt/288" {
			match = true
		}
	}
	if !match {
		t.Errorf("Expected fmt/288, got %v", matches)
	}
}

// Benchmarks
func BenchmarkNew(bench *testing.B) {
	for i := 0; i < bench.N; i++ {
		load(testSigs)
	}
}

func benchidentify(ext string) {
	file := filepath.Join(".", "testdata", "benchmark", "Benchmark")
	file += "." + ext
	identify(b, file)
}

func BenchmarkACCDB(bench *testing.B) {
	for i := 0; i < bench.N; i++ {
		benchidentify("accdb")
	}
}

func BenchmarkBMP(bench *testing.B) {
	for i := 0; i < bench.N; i++ {
		benchidentify("bmp")
	}
}

func BenchmarkDOCX(bench *testing.B) {
	for i := 0; i < bench.N; i++ {
		benchidentify("docx")
	}
}

func BenchmarkGIF(bench *testing.B) {
	for i := 0; i < bench.N; i++ {
		benchidentify("gif")
	}
}

func BenchmarkJPG(bench *testing.B) {
	for i := 0; i < bench.N; i++ {
		benchidentify("jpg")
	}
}

func BenchmarkMSG(bench *testing.B) {
	for i := 0; i < bench.N; i++ {
		benchidentify("msg")
	}
}

func BenchmarkODT(bench *testing.B) {
	for i := 0; i < bench.N; i++ {
		benchidentify("odt")
	}
}

func BenchmarkPDF(bench *testing.B) {
	for i := 0; i < bench.N; i++ {
		benchidentify("pdf")
	}
}

func BenchmarkPNG(bench *testing.B) {
	for i := 0; i < bench.N; i++ {
		benchidentify("png")
	}
}

func BenchmarkPPTX(bench *testing.B) {
	for i := 0; i < bench.N; i++ {
		benchidentify("pptx")
	}
}

func BenchmarkRTF(bench *testing.B) {
	for i := 0; i < bench.N; i++ {
		benchidentify("rtf")
	}
}

func BenchmarkTIF(bench *testing.B) {
	for i := 0; i < bench.N; i++ {
		benchidentify("tif")
	}
}

func BenchmarkXLSX(bench *testing.B) {
	for i := 0; i < bench.N; i++ {
		benchidentify("xlsx")
	}
}

func BenchmarkXML(bench *testing.B) {
	for i := 0; i < bench.N; i++ {
		benchidentify("xml")
	}
}
<commit_msg>removing Travis specific test now that it passes<commit_after>package main

import (
	"errors"
	"os"
	"path/filepath"
	"strings"
	"testing"

	"github.com/richardlehane/siegfried/pkg/core/bytematcher"
	"github.com/richardlehane/siegfried/pkg/pronom"
)

var testSigs = filepath.Join("..", "r2d2", "data", "pronom.gob")

var b *bytematcher.Bytematcher

func init() {
	var err error
	droid, _, reports := pronom.ConfigPaths()
	puids, err = pronom.PuidsFromDroid(droid, reports)
	if err != nil {
		panic(err)
	}
	b, err = load(testSigs)
	if err != nil {
		panic(err)
	}
}

var root = filepath.Join(".", "testdata", "skeleton-suite")

func TestLoad(t *testing.T) {
	_, err := load(testSigs)
	if err != nil {
		t.Error(err)
	}
}

func check(i string, j []int) bool {
	for _, v := range j {
		if i == puids[v] {
			return true
		}
	}
	return false
}

func matchString(i []int) string {
	str := "[ "
	for _, v := range i {
		str += puids[v]
		str += " "
	}
	return str + "]"
}

func TestSuite(t *testing.T) {
	expect := make([]string, 0)
	names := make([]string, 0)
	wf := func(path string, info os.FileInfo, err error) error {
		if info.IsDir() {
			return nil
		}
		last := strings.Split(path, string(os.PathSeparator))
		path = last[len(last)-1]
		idx := strings.Index(path, "signature")
		if idx < 0 {
			idx = len(path)
		}
		strs := strings.Split(path[:idx-1], "-")
		if len(strs) == 2 {
			expect = append(expect, strings.Join(strs, "/"))
		} else if len(strs) == 3 {
			expect = append(expect, "x-fmt/"+strs[2])
		} else {
			return errors.New("long string encountered: " + path)
		}
		names = append(names, path)
		return nil
	}
	err := filepath.Walk(root, wf)
	if err != nil {
		t.Fatal(err)
	}
	matches, err := multiIdentify(b, root)
	if err != nil {
		t.Fatal(err)
	}
	if len(expect) != len(matches) {
		t.Error("Expect should equal matches")
	}
	var iter int
	for i, v := range expect {
		if !check(v, matches[i]) {
			t.Errorf("Failed to match signature %v; got %v; expected %v", names[i], matchString(matches[i]), v)

		} else {
			iter++
		}
	}
	if iter != len(expect) {
		t.Errorf("Matched %v out of %v signatures", iter, len(expect))
	}
}

// Benchmarks
func BenchmarkNew(bench *testing.B) {
	for i := 0; i < bench.N; i++ {
		load(testSigs)
	}
}

func benchidentify(ext string) {
	file := filepath.Join(".", "testdata", "benchmark", "Benchmark")
	file += "." + ext
	identify(b, file)
}

func BenchmarkACCDB(bench *testing.B) {
	for i := 0; i < bench.N; i++ {
		benchidentify("accdb")
	}
}

func BenchmarkBMP(bench *testing.B) {
	for i := 0; i < bench.N; i++ {
		benchidentify("bmp")
	}
}

func BenchmarkDOCX(bench *testing.B) {
	for i := 0; i < bench.N; i++ {
		benchidentify("docx")
	}
}

func BenchmarkGIF(bench *testing.B) {
	for i := 0; i < bench.N; i++ {
		benchidentify("gif")
	}
}

func BenchmarkJPG(bench *testing.B) {
	for i := 0; i < bench.N; i++ {
		benchidentify("jpg")
	}
}

func BenchmarkMSG(bench *testing.B) {
	for i := 0; i < bench.N; i++ {
		benchidentify("msg")
	}
}

func BenchmarkODT(bench *testing.B) {
	for i := 0; i < bench.N; i++ {
		benchidentify("odt")
	}
}

func BenchmarkPDF(bench *testing.B) {
	for i := 0; i < bench.N; i++ {
		benchidentify("pdf")
	}
}

func BenchmarkPNG(bench *testing.B) {
	for i := 0; i < bench.N; i++ {
		benchidentify("png")
	}
}

func BenchmarkPPTX(bench *testing.B) {
	for i := 0; i < bench.N; i++ {
		benchidentify("pptx")
	}
}

func BenchmarkRTF(bench *testing.B) {
	for i := 0; i < bench.N; i++ {
		benchidentify("rtf")
	}
}

func BenchmarkTIF(bench *testing.B) {
	for i := 0; i < bench.N; i++ {
		benchidentify("tif")
	}
}

func BenchmarkXLSX(bench *testing.B) {
	for i := 0; i < bench.N; i++ {
		benchidentify("xlsx")
	}
}

func BenchmarkXML(bench *testing.B) {
	for i := 0; i < bench.N; i++ {
		benchidentify("xml")
	}
}
<|endoftext|>


<commit_before>package test

import (
	"encoding/json"
	"log"
	"net/http"

	x "github.com/javinc/puto"
	"github.com/javinc/puto/test/resource"
	"github.com/javinc/puto/test/service"
)

var (
	// Model for import use
	Model = resource.Model{}
)

// Handler catches /test endpoint
func Handler(w http.ResponseWriter, r *http.Request) {
	log.Println("controllers TestHandler")

	// extract field from get params
	o := new(resource.Options)
	err := x.Decoder.Decode(o, r.URL.Query())
	if err != nil {
		log.Panic(err)
	}

	log.Println("options", o)

	// extract field from post data
	err = r.ParseForm()
	if err != nil {
		log.Panic(err)
	}

	p := new(resource.Model)
	err = x.Decoder.Decode(p, r.PostForm)
	if err != nil {
		log.Panic(err)
	}

	log.Println("payload", p)

	// lets base on the request type
	// use service
	var b []byte
	switch r.Method {
	case "GET":
		b, err = json.Marshal(service.Find(o))

	case "POST":
		b, err = json.Marshal(service.Create(p))
	}

	// render some value
	if err != nil {
		log.Panic(err)
	}

	w.Write(b)
}
<commit_msg>fix empty payload<commit_after>package test

import (
	"encoding/json"
	"log"
	"net/http"

	x "github.com/javinc/puto"
	"github.com/javinc/puto/test/resource"
	"github.com/javinc/puto/test/service"
)

var (
	// Model for import use
	Model = resource.Model{}
)

// Handler catches /test endpoint
func Handler(w http.ResponseWriter, r *http.Request) {
	log.Println("test handler")

	// extract field from get params
	o := new(resource.Options)
	err := x.Decoder.Decode(o, r.URL.Query())
	if err != nil {
		log.Panic(err)
	}

	log.Println("options", o)

	// parse json post
	decoder := json.NewDecoder(r.Body)
	p := new(resource.Model)
	decoder.Decode(&p)

	log.Println("payload", p)

	// lets base on the request type
	// use service
	var b []byte
	switch r.Method {
	case "GET":
		b, err = json.Marshal(service.Find(o))

	case "POST":
		b, err = json.Marshal(service.Create(p))
	}

	// render some value
	if err != nil {
		log.Panic(err)
	}

	w.Write(b)
}

func parseJSON(rw http.ResponseWriter, req *http.Request) {

}
<|endoftext|>


<commit_before>// Copyright (c) Alex Ellis 2017. All rights reserved.
// Licensed under the MIT license. See LICENSE file in the project root for full license information.

package stack

// Provider for the FaaS set of functions.
type Provider struct {
	Name       string `yaml:"name"`
	GatewayURL string `yaml:"gateway"`
	Network    string `yaml:"network"`
}

// Function as deployed or built on FaaS
type Function struct {
	// Name of deployed function
	Name     string `yaml:"-"`
	Language string `yaml:"lang"`

	// Handler Local folder to use for function
	Handler string `yaml:"handler"`

	// Image Docker image name
	Image string `yaml:"image"`

	// Docker registry Authorization
	RegistryAuth string `yaml:"registry_auth,omitempty"`

	FProcess string `yaml:"fprocess"`

	Environment map[string]string `yaml:"environment"`

	// Secrets list of secrets to be made available to function
	Secrets []string `yaml:"secrets"`

	SkipBuild bool `yaml:"skip_build"`

	Constraints *[]string `yaml:"constraints"`

	// EnvironmentFile is a list of files to import and override environmental variables.
	// These are overriden in order.
	EnvironmentFile []string `yaml:"environment_file"`

	Labels *map[string]string `yaml:"labels"`

	// Limits for function
	Limits *FunctionResources `yaml:"limits"`

	// Requests of resources requested by function
	Requests *FunctionResources `yaml:"requests"`

	// BuildOptions to determine native packages
	BuildOptions []string `yaml:"build_options"`
}

// FunctionResources Memory and CPU
type FunctionResources struct {
	Memory string `yaml:"memory"`
	CPU    string `yaml:"cpu"`
}

// EnvironmentFile represents external file for environment data
type EnvironmentFile struct {
	Environment map[string]string `yaml:"environment"`
}

// Services root level YAML file to define FaaS function-set
type Services struct {
	Functions map[string]Function `yaml:"functions,omitempty"`
	Provider  Provider            `yaml:"provider,omitempty"`
}

// LanguageTemplate read from template.yml within root of a language template folder
type LanguageTemplate struct {
	Language     string        `yaml:"language"`
	FProcess     string        `yaml:"fprocess"`
	BuildOptions []BuildOption `yaml:"build_options"`
	// WelcomeMessage is printed to the user after generating a function
	WelcomeMessage string `yaml:"welcomeMessage"`
}

type BuildOption struct {
	Name     string   `yaml:"name"`
	Packages []string `yaml:"packages"`
}
<commit_msg>Updated welcome message field to welcome_message<commit_after>// Copyright (c) Alex Ellis 2017. All rights reserved.
// Licensed under the MIT license. See LICENSE file in the project root for full license information.

package stack

// Provider for the FaaS set of functions.
type Provider struct {
	Name       string `yaml:"name"`
	GatewayURL string `yaml:"gateway"`
	Network    string `yaml:"network"`
}

// Function as deployed or built on FaaS
type Function struct {
	// Name of deployed function
	Name     string `yaml:"-"`
	Language string `yaml:"lang"`

	// Handler Local folder to use for function
	Handler string `yaml:"handler"`

	// Image Docker image name
	Image string `yaml:"image"`

	// Docker registry Authorization
	RegistryAuth string `yaml:"registry_auth,omitempty"`

	FProcess string `yaml:"fprocess"`

	Environment map[string]string `yaml:"environment"`

	// Secrets list of secrets to be made available to function
	Secrets []string `yaml:"secrets"`

	SkipBuild bool `yaml:"skip_build"`

	Constraints *[]string `yaml:"constraints"`

	// EnvironmentFile is a list of files to import and override environmental variables.
	// These are overriden in order.
	EnvironmentFile []string `yaml:"environment_file"`

	Labels *map[string]string `yaml:"labels"`

	// Limits for function
	Limits *FunctionResources `yaml:"limits"`

	// Requests of resources requested by function
	Requests *FunctionResources `yaml:"requests"`

	// BuildOptions to determine native packages
	BuildOptions []string `yaml:"build_options"`
}

// FunctionResources Memory and CPU
type FunctionResources struct {
	Memory string `yaml:"memory"`
	CPU    string `yaml:"cpu"`
}

// EnvironmentFile represents external file for environment data
type EnvironmentFile struct {
	Environment map[string]string `yaml:"environment"`
}

// Services root level YAML file to define FaaS function-set
type Services struct {
	Functions map[string]Function `yaml:"functions,omitempty"`
	Provider  Provider            `yaml:"provider,omitempty"`
}

// LanguageTemplate read from template.yml within root of a language template folder
type LanguageTemplate struct {
	Language     string        `yaml:"language"`
	FProcess     string        `yaml:"fprocess"`
	BuildOptions []BuildOption `yaml:"build_options"`
	// WelcomeMessage is printed to the user after generating a function
	WelcomeMessage string `yaml:"welcome_message"`
}

type BuildOption struct {
	Name     string   `yaml:"name"`
	Packages []string `yaml:"packages"`
}
<|endoftext|>


<commit_before>// Copyright 2015 Marc-Antoine Ruel. All rights reserved.
// Use of this source code is governed under the Apache License, Version 2.0
// that can be found in the LICENSE file.

// This file contains the code to process sources, to be able to deduct the
// original types.

package stack

import (
	"bytes"
	"fmt"
	"go/ast"
	"go/parser"
	"go/token"
	"io/ioutil"
	"log"
	"math"
	"strings"
)

// cache is a cache of sources on the file system.
type cache struct {
	files  map[string][]byte
	parsed map[string]*parsedFile
}

// Augment processes source files to improve calls to be more descriptive.
//
// It modifies goroutines in place.
func Augment(goroutines []Goroutine) {
	c := &cache{}
	for i := range goroutines {
		c.augmentGoroutine(&goroutines[i])
	}
}

// augmentGoroutine processes source files to improve call to be more
// descriptive.
//
// It modifies the routine.
func (c *cache) augmentGoroutine(goroutine *Goroutine) {
	if c.files == nil {
		c.files = map[string][]byte{}
	}
	if c.parsed == nil {
		c.parsed = map[string]*parsedFile{}
	}
	// For each call site, look at the next call and populate it. Then we can
	// walk back and reformat things.
	for i := range goroutine.Stack.Calls {
		c.load(goroutine.Stack.Calls[i].SourcePath)
	}

	// Once all loaded, we can look at the next call when available.
	for i := 0; i < len(goroutine.Stack.Calls)-1; i++ {
		// Get the AST from the previous call and process the call line with it.
		if f := c.getFuncAST(&goroutine.Stack.Calls[i]); f != nil {
			processCall(&goroutine.Stack.Calls[i], f)
		}
	}
}

// Private stuff.

// load loads a source file and parses the AST tree. Failures are ignored.
func (c *cache) load(fileName string) {
	if _, ok := c.parsed[fileName]; ok {
		return
	}
	c.parsed[fileName] = nil
	if !strings.HasSuffix(fileName, ".go") {
		// Ignore C and assembly.
		c.files[fileName] = nil
		return
	}
	log.Printf("load(%s)", fileName)
	if _, ok := c.files[fileName]; !ok {
		var err error
		if c.files[fileName], err = ioutil.ReadFile(fileName); err != nil {
			log.Printf("Failed to read %s: %s", fileName, err)
			c.files[fileName] = nil
			return
		}
	}
	fset := token.NewFileSet()
	src := c.files[fileName]
	parsed, err := parser.ParseFile(fset, fileName, src, 0)
	if err != nil {
		log.Printf("Failed to parse %s: %s", fileName, err)
		return
	}
	// Convert the line number into raw file offset.
	offsets := []int{0, 0}
	start := 0
	for l := 1; start < len(src); l++ {
		start += bytes.IndexByte(src[start:], '\n') + 1
		offsets = append(offsets, start)
	}
	c.parsed[fileName] = &parsedFile{offsets, parsed}
}

func (c *cache) getFuncAST(call *Call) *ast.FuncDecl {
	if p := c.parsed[call.SourcePath]; p != nil {
		return p.getFuncAST(call.Func.Name(), call.Line)
	}
	return nil
}

type parsedFile struct {
	lineToByteOffset []int
	parsed           *ast.File
}

// getFuncAST gets the callee site function AST representation for the code
// inside the function f at line l.
func (p *parsedFile) getFuncAST(f string, l int) (d *ast.FuncDecl) {
	// Walk the AST to find the lineToByteOffset that fits the line number.
	var lastFunc *ast.FuncDecl
	var found ast.Node
	// Inspect() goes depth first. This means for example that a function like:
	// func a() {
	//   b := func() {}
	//   c()
	// }
	//
	// Were we are looking at the c() call can return confused values. It is
	// important to look at the actual ast.Node hierarchy.
	ast.Inspect(p.parsed, func(n ast.Node) bool {
		if d != nil {
			return false
		}
		if n == nil {
			return true
		}
		if found != nil {
			// We are walking up.
		}
		if int(n.Pos()) >= p.lineToByteOffset[l] {
			// We are expecting a ast.CallExpr node. It can be harder to figure out
			// when there are multiple calls on a single line, as the stack trace
			// doesn't have file byte offset information, only line based.
			// gofmt will always format to one function call per line but there can
			// be edge cases, like:
			//   a = A{Foo(), Bar()}
			d = lastFunc
			//p.processNode(call, n)
			return false
		} else if f, ok := n.(*ast.FuncDecl); ok {
			lastFunc = f
		}
		return true
	})
	return
}

func name(n ast.Node) string {
	switch t := n.(type) {
	case *ast.InterfaceType:
		return "interface{}"
	case *ast.Ident:
		return t.Name
	case *ast.SelectorExpr:
		return t.Sel.Name
	case *ast.StarExpr:
		return "*" + name(t.X)
	default:
		return "<unknown>"
	}
}

// fieldToType returns the type name and whether if it's an ellipsis.
func fieldToType(f *ast.Field) (string, bool) {
	switch arg := f.Type.(type) {
	case *ast.ArrayType:
		return "[]" + name(arg.Elt), false
	case *ast.Ellipsis:
		return name(arg.Elt), true
	case *ast.FuncType:
		// Do not print the function signature to not overload the trace.
		return "func", false
	case *ast.Ident:
		return arg.Name, false
	case *ast.InterfaceType:
		return "interface{}", false
	case *ast.SelectorExpr:
		return arg.Sel.Name, false
	case *ast.StarExpr:
		return "*" + name(arg.X), false
	case *ast.MapType:
		return fmt.Sprintf("map[%s]%s", name(arg.Key), name(arg.Value)), false
	case *ast.ChanType:
		return fmt.Sprintf("chan %s", name(arg.Value)), false
	default:
		// TODO(maruel): Implement anything missing.
		return "<unknown>", false
	}
}

// extractArgumentsType returns the name of the type of each input argument.
func extractArgumentsType(f *ast.FuncDecl) ([]string, bool) {
	var fields []*ast.Field
	if f.Recv != nil {
		if len(f.Recv.List) != 1 {
			panic("Expect only one receiver; please fix panicparse's code")
		}
		// If it is an object receiver (vs a pointer receiver), its address is not
		// printed in the stack trace so it needs to be ignored.
		if _, ok := f.Recv.List[0].Type.(*ast.StarExpr); ok {
			fields = append(fields, f.Recv.List[0])
		}
	}
	var types []string
	extra := false
	for _, arg := range append(fields, f.Type.Params.List...) {
		// Assert that extra is only set on the last item of fields?
		var t string
		t, extra = fieldToType(arg)
		mult := len(arg.Names)
		if mult == 0 {
			mult = 1
		}
		for i := 0; i < mult; i++ {
			types = append(types, t)
		}
	}
	return types, extra
}

// processCall walks the function and populate call accordingly.
func processCall(call *Call, f *ast.FuncDecl) {
	values := make([]uint64, len(call.Args.Values))
	for i := range call.Args.Values {
		values[i] = call.Args.Values[i].Value
	}
	index := 0
	pop := func() uint64 {
		if len(values) != 0 {
			x := values[0]
			values = values[1:]
			index++
			return x
		}
		return 0
	}
	popName := func() string {
		n := call.Args.Values[index].Name
		v := pop()
		if len(n) == 0 {
			return fmt.Sprintf("0x%x", v)
		}
		return n
	}

	types, extra := extractArgumentsType(f)
	for i := 0; len(values) != 0; i++ {
		var t string
		if i >= len(types) {
			if !extra {
				// These are unexpected value! Print them as hex.
				call.Args.Processed = append(call.Args.Processed, popName())
				continue
			}
			t = types[len(types)-1]
		} else {
			t = types[i]
		}
		switch t {
		case "float32":
			call.Args.Processed = append(call.Args.Processed, fmt.Sprintf("%g", math.Float32frombits(uint32(pop()))))
		case "float64":
			call.Args.Processed = append(call.Args.Processed, fmt.Sprintf("%g", math.Float64frombits(pop())))
		case "int", "int8", "int16", "int32", "int64", "uint", "uint8", "uint16", "uint32", "uint64":
			call.Args.Processed = append(call.Args.Processed, fmt.Sprintf("%d", pop()))
		case "string":
			call.Args.Processed = append(call.Args.Processed, fmt.Sprintf("%s(%s, len=%d)", t, popName(), pop()))
		default:
			if strings.HasPrefix(t, "*") {
				call.Args.Processed = append(call.Args.Processed, fmt.Sprintf("%s(%s)", t, popName()))
			} else if strings.HasPrefix(t, "[]") {
				call.Args.Processed = append(call.Args.Processed, fmt.Sprintf("%s(%s len=%d cap=%d)", t, popName(), pop(), pop()))
			} else {
				// Assumes it's an interface. For now, discard the object value, which
				// is probably not a good idea.
				call.Args.Processed = append(call.Args.Processed, fmt.Sprintf("%s(%s)", t, popName()))
				pop()
			}
		}
		if len(values) == 0 && call.Args.Elided {
			return
		}
	}
}
<commit_msg>stack: don't try to read past the end of a file.<commit_after>// Copyright 2015 Marc-Antoine Ruel. All rights reserved.
// Use of this source code is governed under the Apache License, Version 2.0
// that can be found in the LICENSE file.

// This file contains the code to process sources, to be able to deduct the
// original types.

package stack

import (
	"bytes"
	"fmt"
	"go/ast"
	"go/parser"
	"go/token"
	"io/ioutil"
	"log"
	"math"
	"strings"
)

// cache is a cache of sources on the file system.
type cache struct {
	files  map[string][]byte
	parsed map[string]*parsedFile
}

// Augment processes source files to improve calls to be more descriptive.
//
// It modifies goroutines in place.
func Augment(goroutines []Goroutine) {
	c := &cache{}
	for i := range goroutines {
		c.augmentGoroutine(&goroutines[i])
	}
}

// augmentGoroutine processes source files to improve call to be more
// descriptive.
//
// It modifies the routine.
func (c *cache) augmentGoroutine(goroutine *Goroutine) {
	if c.files == nil {
		c.files = map[string][]byte{}
	}
	if c.parsed == nil {
		c.parsed = map[string]*parsedFile{}
	}
	// For each call site, look at the next call and populate it. Then we can
	// walk back and reformat things.
	for i := range goroutine.Stack.Calls {
		c.load(goroutine.Stack.Calls[i].SourcePath)
	}

	// Once all loaded, we can look at the next call when available.
	for i := 0; i < len(goroutine.Stack.Calls)-1; i++ {
		// Get the AST from the previous call and process the call line with it.
		if f := c.getFuncAST(&goroutine.Stack.Calls[i]); f != nil {
			processCall(&goroutine.Stack.Calls[i], f)
		}
	}
}

// Private stuff.

// load loads a source file and parses the AST tree. Failures are ignored.
func (c *cache) load(fileName string) {
	if _, ok := c.parsed[fileName]; ok {
		return
	}
	c.parsed[fileName] = nil
	if !strings.HasSuffix(fileName, ".go") {
		// Ignore C and assembly.
		c.files[fileName] = nil
		return
	}
	log.Printf("load(%s)", fileName)
	if _, ok := c.files[fileName]; !ok {
		var err error
		if c.files[fileName], err = ioutil.ReadFile(fileName); err != nil {
			log.Printf("Failed to read %s: %s", fileName, err)
			c.files[fileName] = nil
			return
		}
	}
	fset := token.NewFileSet()
	src := c.files[fileName]
	parsed, err := parser.ParseFile(fset, fileName, src, 0)
	if err != nil {
		log.Printf("Failed to parse %s: %s", fileName, err)
		return
	}
	// Convert the line number into raw file offset.
	offsets := []int{0, 0}
	start := 0
	for l := 1; start < len(src); l++ {
		start += bytes.IndexByte(src[start:], '\n') + 1
		offsets = append(offsets, start)
	}
	c.parsed[fileName] = &parsedFile{offsets, parsed}
}

func (c *cache) getFuncAST(call *Call) *ast.FuncDecl {
	if p := c.parsed[call.SourcePath]; p != nil {
		return p.getFuncAST(call.Func.Name(), call.Line)
	}
	return nil
}

type parsedFile struct {
	lineToByteOffset []int
	parsed           *ast.File
}

// getFuncAST gets the callee site function AST representation for the code
// inside the function f at line l.
func (p *parsedFile) getFuncAST(f string, l int) (d *ast.FuncDecl) {
	if len(p.lineToByteOffset) <= l {
		// The line number in the stack trace line does not exist in the file. That
		// can only mean that the sources on disk do not match the sources used to
		// build the binary.
		// TODO(maruel): This should be surfaced, so that source parsing is
		// completely ignored.
		return
	}

	// Walk the AST to find the lineToByteOffset that fits the line number.
	var lastFunc *ast.FuncDecl
	var found ast.Node
	// Inspect() goes depth first. This means for example that a function like:
	// func a() {
	//   b := func() {}
	//   c()
	// }
	//
	// Were we are looking at the c() call can return confused values. It is
	// important to look at the actual ast.Node hierarchy.
	ast.Inspect(p.parsed, func(n ast.Node) bool {
		if d != nil {
			return false
		}
		if n == nil {
			return true
		}
		if found != nil {
			// We are walking up.
		}
		if int(n.Pos()) >= p.lineToByteOffset[l] {
			// We are expecting a ast.CallExpr node. It can be harder to figure out
			// when there are multiple calls on a single line, as the stack trace
			// doesn't have file byte offset information, only line based.
			// gofmt will always format to one function call per line but there can
			// be edge cases, like:
			//   a = A{Foo(), Bar()}
			d = lastFunc
			//p.processNode(call, n)
			return false
		} else if f, ok := n.(*ast.FuncDecl); ok {
			lastFunc = f
		}
		return true
	})
	return
}

func name(n ast.Node) string {
	switch t := n.(type) {
	case *ast.InterfaceType:
		return "interface{}"
	case *ast.Ident:
		return t.Name
	case *ast.SelectorExpr:
		return t.Sel.Name
	case *ast.StarExpr:
		return "*" + name(t.X)
	default:
		return "<unknown>"
	}
}

// fieldToType returns the type name and whether if it's an ellipsis.
func fieldToType(f *ast.Field) (string, bool) {
	switch arg := f.Type.(type) {
	case *ast.ArrayType:
		return "[]" + name(arg.Elt), false
	case *ast.Ellipsis:
		return name(arg.Elt), true
	case *ast.FuncType:
		// Do not print the function signature to not overload the trace.
		return "func", false
	case *ast.Ident:
		return arg.Name, false
	case *ast.InterfaceType:
		return "interface{}", false
	case *ast.SelectorExpr:
		return arg.Sel.Name, false
	case *ast.StarExpr:
		return "*" + name(arg.X), false
	case *ast.MapType:
		return fmt.Sprintf("map[%s]%s", name(arg.Key), name(arg.Value)), false
	case *ast.ChanType:
		return fmt.Sprintf("chan %s", name(arg.Value)), false
	default:
		// TODO(maruel): Implement anything missing.
		return "<unknown>", false
	}
}

// extractArgumentsType returns the name of the type of each input argument.
func extractArgumentsType(f *ast.FuncDecl) ([]string, bool) {
	var fields []*ast.Field
	if f.Recv != nil {
		if len(f.Recv.List) != 1 {
			panic("Expect only one receiver; please fix panicparse's code")
		}
		// If it is an object receiver (vs a pointer receiver), its address is not
		// printed in the stack trace so it needs to be ignored.
		if _, ok := f.Recv.List[0].Type.(*ast.StarExpr); ok {
			fields = append(fields, f.Recv.List[0])
		}
	}
	var types []string
	extra := false
	for _, arg := range append(fields, f.Type.Params.List...) {
		// Assert that extra is only set on the last item of fields?
		var t string
		t, extra = fieldToType(arg)
		mult := len(arg.Names)
		if mult == 0 {
			mult = 1
		}
		for i := 0; i < mult; i++ {
			types = append(types, t)
		}
	}
	return types, extra
}

// processCall walks the function and populate call accordingly.
func processCall(call *Call, f *ast.FuncDecl) {
	values := make([]uint64, len(call.Args.Values))
	for i := range call.Args.Values {
		values[i] = call.Args.Values[i].Value
	}
	index := 0
	pop := func() uint64 {
		if len(values) != 0 {
			x := values[0]
			values = values[1:]
			index++
			return x
		}
		return 0
	}
	popName := func() string {
		n := call.Args.Values[index].Name
		v := pop()
		if len(n) == 0 {
			return fmt.Sprintf("0x%x", v)
		}
		return n
	}

	types, extra := extractArgumentsType(f)
	for i := 0; len(values) != 0; i++ {
		var t string
		if i >= len(types) {
			if !extra {
				// These are unexpected value! Print them as hex.
				call.Args.Processed = append(call.Args.Processed, popName())
				continue
			}
			t = types[len(types)-1]
		} else {
			t = types[i]
		}
		switch t {
		case "float32":
			call.Args.Processed = append(call.Args.Processed, fmt.Sprintf("%g", math.Float32frombits(uint32(pop()))))
		case "float64":
			call.Args.Processed = append(call.Args.Processed, fmt.Sprintf("%g", math.Float64frombits(pop())))
		case "int", "int8", "int16", "int32", "int64", "uint", "uint8", "uint16", "uint32", "uint64":
			call.Args.Processed = append(call.Args.Processed, fmt.Sprintf("%d", pop()))
		case "string":
			call.Args.Processed = append(call.Args.Processed, fmt.Sprintf("%s(%s, len=%d)", t, popName(), pop()))
		default:
			if strings.HasPrefix(t, "*") {
				call.Args.Processed = append(call.Args.Processed, fmt.Sprintf("%s(%s)", t, popName()))
			} else if strings.HasPrefix(t, "[]") {
				call.Args.Processed = append(call.Args.Processed, fmt.Sprintf("%s(%s len=%d cap=%d)", t, popName(), pop(), pop()))
			} else {
				// Assumes it's an interface. For now, discard the object value, which
				// is probably not a good idea.
				call.Args.Processed = append(call.Args.Processed, fmt.Sprintf("%s(%s)", t, popName()))
				pop()
			}
		}
		if len(values) == 0 && call.Args.Elided {
			return
		}
	}
}
<|endoftext|>