<commit_before>/*
 *
 * Copyright 2019 gRPC authors.
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 *     http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 *
 */

// Package bootstrap provides the functionality to initialize certain aspects
// of an xDS client by reading a bootstrap file.
package bootstrap

import (
	"bytes"
	"encoding/json"
	"fmt"
	"io/ioutil"

	v2corepb "github.com/envoyproxy/go-control-plane/envoy/api/v2/core"
	v3corepb "github.com/envoyproxy/go-control-plane/envoy/config/core/v3"
	"github.com/golang/protobuf/jsonpb"
	"github.com/golang/protobuf/proto"
	"google.golang.org/grpc"
	"google.golang.org/grpc/credentials/google"
	"google.golang.org/grpc/credentials/insecure"
	"google.golang.org/grpc/credentials/tls/certprovider"
	"google.golang.org/grpc/internal"
	"google.golang.org/grpc/internal/pretty"
	"google.golang.org/grpc/internal/xds/env"
	"google.golang.org/grpc/xds/internal/version"
)

const (
	// The "server_features" field in the bootstrap file contains a list of
	// features supported by the server. A value of "xds_v3" indicates that the
	// server supports the v3 version of the xDS transport protocol.
	serverFeaturesV3 = "xds_v3"

	// Type name for Google default credentials.
	credsGoogleDefault              = "google_default"
	credsInsecure                   = "insecure"
	gRPCUserAgentName               = "gRPC Go"
	clientFeatureNoOverprovisioning = "envoy.lb.does_not_support_overprovisioning"
)

var gRPCVersion = fmt.Sprintf("%s %s", gRPCUserAgentName, grpc.Version)

// For overriding in unit tests.
var bootstrapFileReadFunc = ioutil.ReadFile

// Config provides the xDS client with several key bits of information that it
// requires in its interaction with the management server. The Config is
// initialized from the bootstrap file.
type Config struct {
	// BalancerName is the name of the management server to connect to.
	//
	// The bootstrap file contains a list of servers (with name+creds), but we
	// pick the first one.
	BalancerName string
	// Creds contains the credentials to be used while talking to the xDS
	// server, as a grpc.DialOption.
	Creds grpc.DialOption
	// TransportAPI indicates the API version of xDS transport protocol to use.
	// This describes the xDS gRPC endpoint and version of
	// DiscoveryRequest/Response used on the wire.
	TransportAPI version.TransportAPI
	// NodeProto contains the Node proto to be used in xDS requests. The actual
	// type depends on the transport protocol version used.
	NodeProto proto.Message
	// CertProviderConfigs contains a mapping from certificate provider plugin
	// instance names to parsed buildable configs.
	CertProviderConfigs map[string]*certprovider.BuildableConfig
	// ServerListenerResourceNameTemplate is a template for the name of the
	// Listener resource to subscribe to for a gRPC server. If the token `%s` is
	// present in the string, it will be replaced with the server's listening
	// "IP:port" (e.g., "0.0.0.0:8080", "[::]:8080"). For example, a value of
	// "example/resource/%s" could become "example/resource/0.0.0.0:8080".
	ServerListenerResourceNameTemplate string
}

type channelCreds struct {
	Type   string          `json:"type"`
	Config json.RawMessage `json:"config"`
}

type xdsServer struct {
	ServerURI      string         `json:"server_uri"`
	ChannelCreds   []channelCreds `json:"channel_creds"`
	ServerFeatures []string       `json:"server_features"`
}

func bootstrapConfigFromEnvVariable() ([]byte, error) {
	fName := env.BootstrapFileName
	fContent := env.BootstrapFileContent

	// Bootstrap file name has higher priority than bootstrap content.
	if fName != "" {
		// If file name is set
		// - If file not found (or other errors), fail
		// - Otherwise, use the content.
		//
		// Note that even if the content is invalid, we don't failover to the
		// file content env variable.
		logger.Debugf("xds: using bootstrap file with name %q", fName)
		return bootstrapFileReadFunc(fName)
	}

	if fContent != "" {
		return []byte(fContent), nil
	}

	return nil, fmt.Errorf("none of the bootstrap environment variables (%q or %q) defined", env.BootstrapFileNameEnv, env.BootstrapFileContentEnv)
}

// NewConfig returns a new instance of Config initialized by reading the
// bootstrap file found at ${GRPC_XDS_BOOTSTRAP}.
//
// The format of the bootstrap file will be as follows:
// {
//    "xds_server": {
//      "server_uri": <string containing URI of management server>,
//      "channel_creds": [
//        {
//          "type": <string containing channel cred type>,
//          "config": <JSON object containing config for the type>
//        }
//      ],
//      "server_features": [ ... ],
//    },
//    "node": <JSON form of Node proto>,
//    "certificate_providers" : {
//      "default": {
//        "plugin_name": "default-plugin-name",
//        "config": { default plugin config in JSON }
//       },
//      "foo": {
//        "plugin_name": "foo",
//        "config": { foo plugin config in JSON }
//      }
//    },
//    "server_listener_resource_name_template": "grpc/server?xds.resource.listening_address=%s"
// }
//
// Currently, we support exactly one type of credential, which is
// "google_default", where we use the host's default certs for transport
// credentials and a Google oauth token for call credentials.
//
// This function tries to process as much of the bootstrap file as possible (in
// the presence of the errors) and may return a Config object with certain
// fields left unspecified, in which case the caller should use some sane
// defaults.
func NewConfig() (*Config, error) {
	data, err := bootstrapConfigFromEnvVariable()
	if err != nil {
		return nil, fmt.Errorf("xds: Failed to read bootstrap config: %v", err)
	}
	logger.Debugf("Bootstrap content: %s", data)
	return NewConfigFromContents(data)
}

// NewConfigFromContents returns a new Config using the specified bootstrap
// file contents instead of reading the environment variable.  This is only
// suitable for testing purposes.
func NewConfigFromContents(data []byte) (*Config, error) {
	config := &Config{}

	var jsonData map[string]json.RawMessage
	if err := json.Unmarshal(data, &jsonData); err != nil {
		return nil, fmt.Errorf("xds: Failed to parse bootstrap config: %v", err)
	}

	serverSupportsV3 := false
	m := jsonpb.Unmarshaler{AllowUnknownFields: true}
	for k, v := range jsonData {
		switch k {
		case "node":
			// We unconditionally convert the JSON into a v3.Node proto. The v3
			// proto does not contain the deprecated field "build_version" from
			// the v2 proto. We do not expect the bootstrap file to contain the
			// "build_version" field. In any case, the unmarshal will succeed
			// because we have set the `AllowUnknownFields` option on the
			// unmarshaler.
			n := &v3corepb.Node{}
			if err := m.Unmarshal(bytes.NewReader(v), n); err != nil {
				return nil, fmt.Errorf("xds: jsonpb.Unmarshal(%v) for field %q failed during bootstrap: %v", string(v), k, err)
			}
			config.NodeProto = n
		case "xds_servers":
			var servers []*xdsServer
			if err := json.Unmarshal(v, &servers); err != nil {
				return nil, fmt.Errorf("xds: json.Unmarshal(%v) for field %q failed during bootstrap: %v", string(v), k, err)
			}
			if len(servers) < 1 {
				return nil, fmt.Errorf("xds: bootstrap file parsing failed during bootstrap: file doesn't contain any management server to connect to")
			}
			xs := servers[0]
			config.BalancerName = xs.ServerURI
			for _, cc := range xs.ChannelCreds {
				// We stop at the first credential type that we support.
				if cc.Type == credsGoogleDefault {
					config.Creds = grpc.WithCredentialsBundle(google.NewDefaultCredentials())
					break
				} else if cc.Type == credsInsecure {
					config.Creds = grpc.WithTransportCredentials(insecure.NewCredentials())
					break
				}
			}
			for _, f := range xs.ServerFeatures {
				switch f {
				case serverFeaturesV3:
					serverSupportsV3 = true
				}
			}
		case "certificate_providers":
			var providerInstances map[string]json.RawMessage
			if err := json.Unmarshal(v, &providerInstances); err != nil {
				return nil, fmt.Errorf("xds: json.Unmarshal(%v) for field %q failed during bootstrap: %v", string(v), k, err)
			}
			configs := make(map[string]*certprovider.BuildableConfig)
			getBuilder := internal.GetCertificateProviderBuilder.(func(string) certprovider.Builder)
			for instance, data := range providerInstances {
				var nameAndConfig struct {
					PluginName string          `json:"plugin_name"`
					Config     json.RawMessage `json:"config"`
				}
				if err := json.Unmarshal(data, &nameAndConfig); err != nil {
					return nil, fmt.Errorf("xds: json.Unmarshal(%v) for field %q failed during bootstrap: %v", string(v), instance, err)
				}

				name := nameAndConfig.PluginName
				parser := getBuilder(nameAndConfig.PluginName)
				if parser == nil {
					// We ignore plugins that we do not know about.
					continue
				}
				bc, err := parser.ParseConfig(nameAndConfig.Config)
				if err != nil {
					return nil, fmt.Errorf("xds: Config parsing for plugin %q failed: %v", name, err)
				}
				configs[instance] = bc
			}
			config.CertProviderConfigs = configs
		case "server_listener_resource_name_template":
			if err := json.Unmarshal(v, &config.ServerListenerResourceNameTemplate); err != nil {
				return nil, fmt.Errorf("xds: json.Unmarshal(%v) for field %q failed during bootstrap: %v", string(v), k, err)
			}
		}
		// Do not fail the xDS bootstrap when an unknown field is seen. This can
		// happen when an older version client reads a newer version bootstrap
		// file with new fields.
	}

	if config.BalancerName == "" {
		return nil, fmt.Errorf("xds: Required field %q not found in bootstrap %s", "xds_servers.server_uri", jsonData["xds_servers"])
	}
	if config.Creds == nil {
		return nil, fmt.Errorf("xds: Required field %q doesn't contain valid value in bootstrap %s", "xds_servers.channel_creds", jsonData["xds_servers"])
	}

	// We end up using v3 transport protocol version only if the server supports
	// v3, indicated by the presence of "xds_v3" in server_features. The default
	// value of the enum type "version.TransportAPI" is v2.
	if serverSupportsV3 {
		config.TransportAPI = version.TransportV3
	}

	if err := config.updateNodeProto(); err != nil {
		return nil, err
	}
	logger.Infof("Bootstrap config for creating xds-client: %v", pretty.ToJSON(config))
	return config, nil
}

// updateNodeProto updates the node proto read from the bootstrap file.
//
// Node proto in Config contains a v3.Node protobuf message corresponding to the
// JSON contents found in the bootstrap file. This method performs some post
// processing on it:
// 1. If we don't find a nodeProto in the bootstrap file, we create an empty one
// here. That way, callers of this function can always expect that the NodeProto
// field is non-nil.
// 2. If the transport protocol version to be used is not v3, we convert the
// current v3.Node proto in a v2.Node proto.
// 3. Some additional fields which are not expected to be set in the bootstrap
// file are populated here.
func (c *Config) updateNodeProto() error {
	if c.TransportAPI == version.TransportV3 {
		v3, _ := c.NodeProto.(*v3corepb.Node)
		if v3 == nil {
			v3 = &v3corepb.Node{}
		}
		v3.UserAgentName = gRPCUserAgentName
		v3.UserAgentVersionType = &v3corepb.Node_UserAgentVersion{UserAgentVersion: grpc.Version}
		v3.ClientFeatures = append(v3.ClientFeatures, clientFeatureNoOverprovisioning)
		c.NodeProto = v3
		return nil
	}

	v2 := &v2corepb.Node{}
	if c.NodeProto != nil {
		v3, err := proto.Marshal(c.NodeProto)
		if err != nil {
			return fmt.Errorf("xds: proto.Marshal(%v): %v", c.NodeProto, err)
		}
		if err := proto.Unmarshal(v3, v2); err != nil {
			return fmt.Errorf("xds: proto.Unmarshal(%v): %v", v3, err)
		}
	}
	c.NodeProto = v2

	// BuildVersion is deprecated, and is replaced by user_agent_name and
	// user_agent_version. But the management servers are still using the old
	// field, so we will keep both set.
	v2.BuildVersion = gRPCVersion
	v2.UserAgentName = gRPCUserAgentName
	v2.UserAgentVersionType = &v2corepb.Node_UserAgentVersion{UserAgentVersion: grpc.Version}
	v2.ClientFeatures = append(v2.ClientFeatures, clientFeatureNoOverprovisioning)
	return nil
}
<commit_msg>Fix bootstrap format in comment (#4586)<commit_after>/*
 *
 * Copyright 2019 gRPC authors.
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 *     http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 *
 */

// Package bootstrap provides the functionality to initialize certain aspects
// of an xDS client by reading a bootstrap file.
package bootstrap

import (
	"bytes"
	"encoding/json"
	"fmt"
	"io/ioutil"

	v2corepb "github.com/envoyproxy/go-control-plane/envoy/api/v2/core"
	v3corepb "github.com/envoyproxy/go-control-plane/envoy/config/core/v3"
	"github.com/golang/protobuf/jsonpb"
	"github.com/golang/protobuf/proto"
	"google.golang.org/grpc"
	"google.golang.org/grpc/credentials/google"
	"google.golang.org/grpc/credentials/insecure"
	"google.golang.org/grpc/credentials/tls/certprovider"
	"google.golang.org/grpc/internal"
	"google.golang.org/grpc/internal/pretty"
	"google.golang.org/grpc/internal/xds/env"
	"google.golang.org/grpc/xds/internal/version"
)

const (
	// The "server_features" field in the bootstrap file contains a list of
	// features supported by the server. A value of "xds_v3" indicates that the
	// server supports the v3 version of the xDS transport protocol.
	serverFeaturesV3 = "xds_v3"

	// Type name for Google default credentials.
	credsGoogleDefault              = "google_default"
	credsInsecure                   = "insecure"
	gRPCUserAgentName               = "gRPC Go"
	clientFeatureNoOverprovisioning = "envoy.lb.does_not_support_overprovisioning"
)

var gRPCVersion = fmt.Sprintf("%s %s", gRPCUserAgentName, grpc.Version)

// For overriding in unit tests.
var bootstrapFileReadFunc = ioutil.ReadFile

// Config provides the xDS client with several key bits of information that it
// requires in its interaction with the management server. The Config is
// initialized from the bootstrap file.
type Config struct {
	// BalancerName is the name of the management server to connect to.
	//
	// The bootstrap file contains a list of servers (with name+creds), but we
	// pick the first one.
	BalancerName string
	// Creds contains the credentials to be used while talking to the xDS
	// server, as a grpc.DialOption.
	Creds grpc.DialOption
	// TransportAPI indicates the API version of xDS transport protocol to use.
	// This describes the xDS gRPC endpoint and version of
	// DiscoveryRequest/Response used on the wire.
	TransportAPI version.TransportAPI
	// NodeProto contains the Node proto to be used in xDS requests. The actual
	// type depends on the transport protocol version used.
	NodeProto proto.Message
	// CertProviderConfigs contains a mapping from certificate provider plugin
	// instance names to parsed buildable configs.
	CertProviderConfigs map[string]*certprovider.BuildableConfig
	// ServerListenerResourceNameTemplate is a template for the name of the
	// Listener resource to subscribe to for a gRPC server. If the token `%s` is
	// present in the string, it will be replaced with the server's listening
	// "IP:port" (e.g., "0.0.0.0:8080", "[::]:8080"). For example, a value of
	// "example/resource/%s" could become "example/resource/0.0.0.0:8080".
	ServerListenerResourceNameTemplate string
}

type channelCreds struct {
	Type   string          `json:"type"`
	Config json.RawMessage `json:"config"`
}

type xdsServer struct {
	ServerURI      string         `json:"server_uri"`
	ChannelCreds   []channelCreds `json:"channel_creds"`
	ServerFeatures []string       `json:"server_features"`
}

func bootstrapConfigFromEnvVariable() ([]byte, error) {
	fName := env.BootstrapFileName
	fContent := env.BootstrapFileContent

	// Bootstrap file name has higher priority than bootstrap content.
	if fName != "" {
		// If file name is set
		// - If file not found (or other errors), fail
		// - Otherwise, use the content.
		//
		// Note that even if the content is invalid, we don't failover to the
		// file content env variable.
		logger.Debugf("xds: using bootstrap file with name %q", fName)
		return bootstrapFileReadFunc(fName)
	}

	if fContent != "" {
		return []byte(fContent), nil
	}

	return nil, fmt.Errorf("none of the bootstrap environment variables (%q or %q) defined", env.BootstrapFileNameEnv, env.BootstrapFileContentEnv)
}

// NewConfig returns a new instance of Config initialized by reading the
// bootstrap file found at ${GRPC_XDS_BOOTSTRAP}.
//
// The format of the bootstrap file will be as follows:
// {
//    "xds_servers": [
//      {
//        "server_uri": <string containing URI of management server>,
//        "channel_creds": [
//          {
//            "type": <string containing channel cred type>,
//            "config": <JSON object containing config for the type>
//          }
//        ],
//        "server_features": [ ... ],
//      }
//    ],
//    "node": <JSON form of Node proto>,
//    "certificate_providers" : {
//      "default": {
//        "plugin_name": "default-plugin-name",
//        "config": { default plugin config in JSON }
//       },
//      "foo": {
//        "plugin_name": "foo",
//        "config": { foo plugin config in JSON }
//      }
//    },
//    "server_listener_resource_name_template": "grpc/server?xds.resource.listening_address=%s"
// }
//
// Currently, we support exactly one type of credential, which is
// "google_default", where we use the host's default certs for transport
// credentials and a Google oauth token for call credentials.
//
// This function tries to process as much of the bootstrap file as possible (in
// the presence of the errors) and may return a Config object with certain
// fields left unspecified, in which case the caller should use some sane
// defaults.
func NewConfig() (*Config, error) {
	data, err := bootstrapConfigFromEnvVariable()
	if err != nil {
		return nil, fmt.Errorf("xds: Failed to read bootstrap config: %v", err)
	}
	logger.Debugf("Bootstrap content: %s", data)
	return NewConfigFromContents(data)
}

// NewConfigFromContents returns a new Config using the specified bootstrap
// file contents instead of reading the environment variable.  This is only
// suitable for testing purposes.
func NewConfigFromContents(data []byte) (*Config, error) {
	config := &Config{}

	var jsonData map[string]json.RawMessage
	if err := json.Unmarshal(data, &jsonData); err != nil {
		return nil, fmt.Errorf("xds: Failed to parse bootstrap config: %v", err)
	}

	serverSupportsV3 := false
	m := jsonpb.Unmarshaler{AllowUnknownFields: true}
	for k, v := range jsonData {
		switch k {
		case "node":
			// We unconditionally convert the JSON into a v3.Node proto. The v3
			// proto does not contain the deprecated field "build_version" from
			// the v2 proto. We do not expect the bootstrap file to contain the
			// "build_version" field. In any case, the unmarshal will succeed
			// because we have set the `AllowUnknownFields` option on the
			// unmarshaler.
			n := &v3corepb.Node{}
			if err := m.Unmarshal(bytes.NewReader(v), n); err != nil {
				return nil, fmt.Errorf("xds: jsonpb.Unmarshal(%v) for field %q failed during bootstrap: %v", string(v), k, err)
			}
			config.NodeProto = n
		case "xds_servers":
			var servers []*xdsServer
			if err := json.Unmarshal(v, &servers); err != nil {
				return nil, fmt.Errorf("xds: json.Unmarshal(%v) for field %q failed during bootstrap: %v", string(v), k, err)
			}
			if len(servers) < 1 {
				return nil, fmt.Errorf("xds: bootstrap file parsing failed during bootstrap: file doesn't contain any management server to connect to")
			}
			xs := servers[0]
			config.BalancerName = xs.ServerURI
			for _, cc := range xs.ChannelCreds {
				// We stop at the first credential type that we support.
				if cc.Type == credsGoogleDefault {
					config.Creds = grpc.WithCredentialsBundle(google.NewDefaultCredentials())
					break
				} else if cc.Type == credsInsecure {
					config.Creds = grpc.WithTransportCredentials(insecure.NewCredentials())
					break
				}
			}
			for _, f := range xs.ServerFeatures {
				switch f {
				case serverFeaturesV3:
					serverSupportsV3 = true
				}
			}
		case "certificate_providers":
			var providerInstances map[string]json.RawMessage
			if err := json.Unmarshal(v, &providerInstances); err != nil {
				return nil, fmt.Errorf("xds: json.Unmarshal(%v) for field %q failed during bootstrap: %v", string(v), k, err)
			}
			configs := make(map[string]*certprovider.BuildableConfig)
			getBuilder := internal.GetCertificateProviderBuilder.(func(string) certprovider.Builder)
			for instance, data := range providerInstances {
				var nameAndConfig struct {
					PluginName string          `json:"plugin_name"`
					Config     json.RawMessage `json:"config"`
				}
				if err := json.Unmarshal(data, &nameAndConfig); err != nil {
					return nil, fmt.Errorf("xds: json.Unmarshal(%v) for field %q failed during bootstrap: %v", string(v), instance, err)
				}

				name := nameAndConfig.PluginName
				parser := getBuilder(nameAndConfig.PluginName)
				if parser == nil {
					// We ignore plugins that we do not know about.
					continue
				}
				bc, err := parser.ParseConfig(nameAndConfig.Config)
				if err != nil {
					return nil, fmt.Errorf("xds: Config parsing for plugin %q failed: %v", name, err)
				}
				configs[instance] = bc
			}
			config.CertProviderConfigs = configs
		case "server_listener_resource_name_template":
			if err := json.Unmarshal(v, &config.ServerListenerResourceNameTemplate); err != nil {
				return nil, fmt.Errorf("xds: json.Unmarshal(%v) for field %q failed during bootstrap: %v", string(v), k, err)
			}
		}
		// Do not fail the xDS bootstrap when an unknown field is seen. This can
		// happen when an older version client reads a newer version bootstrap
		// file with new fields.
	}

	if config.BalancerName == "" {
		return nil, fmt.Errorf("xds: Required field %q not found in bootstrap %s", "xds_servers.server_uri", jsonData["xds_servers"])
	}
	if config.Creds == nil {
		return nil, fmt.Errorf("xds: Required field %q doesn't contain valid value in bootstrap %s", "xds_servers.channel_creds", jsonData["xds_servers"])
	}

	// We end up using v3 transport protocol version only if the server supports
	// v3, indicated by the presence of "xds_v3" in server_features. The default
	// value of the enum type "version.TransportAPI" is v2.
	if serverSupportsV3 {
		config.TransportAPI = version.TransportV3
	}

	if err := config.updateNodeProto(); err != nil {
		return nil, err
	}
	logger.Infof("Bootstrap config for creating xds-client: %v", pretty.ToJSON(config))
	return config, nil
}

// updateNodeProto updates the node proto read from the bootstrap file.
//
// Node proto in Config contains a v3.Node protobuf message corresponding to the
// JSON contents found in the bootstrap file. This method performs some post
// processing on it:
// 1. If we don't find a nodeProto in the bootstrap file, we create an empty one
// here. That way, callers of this function can always expect that the NodeProto
// field is non-nil.
// 2. If the transport protocol version to be used is not v3, we convert the
// current v3.Node proto in a v2.Node proto.
// 3. Some additional fields which are not expected to be set in the bootstrap
// file are populated here.
func (c *Config) updateNodeProto() error {
	if c.TransportAPI == version.TransportV3 {
		v3, _ := c.NodeProto.(*v3corepb.Node)
		if v3 == nil {
			v3 = &v3corepb.Node{}
		}
		v3.UserAgentName = gRPCUserAgentName
		v3.UserAgentVersionType = &v3corepb.Node_UserAgentVersion{UserAgentVersion: grpc.Version}
		v3.ClientFeatures = append(v3.ClientFeatures, clientFeatureNoOverprovisioning)
		c.NodeProto = v3
		return nil
	}

	v2 := &v2corepb.Node{}
	if c.NodeProto != nil {
		v3, err := proto.Marshal(c.NodeProto)
		if err != nil {
			return fmt.Errorf("xds: proto.Marshal(%v): %v", c.NodeProto, err)
		}
		if err := proto.Unmarshal(v3, v2); err != nil {
			return fmt.Errorf("xds: proto.Unmarshal(%v): %v", v3, err)
		}
	}
	c.NodeProto = v2

	// BuildVersion is deprecated, and is replaced by user_agent_name and
	// user_agent_version. But the management servers are still using the old
	// field, so we will keep both set.
	v2.BuildVersion = gRPCVersion
	v2.UserAgentName = gRPCUserAgentName
	v2.UserAgentVersionType = &v2corepb.Node_UserAgentVersion{UserAgentVersion: grpc.Version}
	v2.ClientFeatures = append(v2.ClientFeatures, clientFeatureNoOverprovisioning)
	return nil
}
<|endoftext|>


<commit_before>// Copyright © 2016 The Things Network
// Use of this source code is governed by the MIT license that can be found in the LICENSE file.

package collection

import (
	"testing"

	. "github.com/smartystreets/assertions"
)

func TestDecode(t *testing.T) {
	a := New(t)

	functions := &Functions{
		Decoder: `function(payload) {
      return {
        value: (payload[0] << 8) | payload[1]
      };
    }`,
	}
	payload := []byte{0x48, 0x65}

	m, err := functions.Decode(payload)
	a.So(err, ShouldBeNil)

	size, ok := m["value"]
	a.So(ok, ShouldBeTrue)
	a.So(size, ShouldEqual, 18533)
}

func TestConvert(t *testing.T) {
	a := New(t)

	withFunction := &Functions{
		Converter: `function(data) {
      return {
        celcius: data.temperature * 2
      };
    }`,
	}
	data, err := withFunction.Convert(map[string]interface{}{"temperature": 11})
	a.So(err, ShouldBeNil)
	a.So(data["celcius"], ShouldEqual, 22)

	withoutFunction := &Functions{}
	data, err = withoutFunction.Convert(map[string]interface{}{"temperature": 11})
	a.So(err, ShouldBeNil)
	a.So(data["temperature"], ShouldEqual, 11)
}

func TestValidate(t *testing.T) {
	a := New(t)

	withFunction := &Functions{
		Validator: `function(data) {
      return data.temperature < 20;
    }`,
	}
	valid, err := withFunction.Validate(map[string]interface{}{"temperature": 10})
	a.So(err, ShouldBeNil)
	a.So(valid, ShouldBeTrue)
	valid, err = withFunction.Validate(map[string]interface{}{"temperature": 30})
	a.So(err, ShouldBeNil)
	a.So(valid, ShouldBeFalse)

	withoutFunction := &Functions{}
	valid, err = withoutFunction.Validate(map[string]interface{}{"temperature": 10})
	a.So(err, ShouldBeNil)
	a.So(valid, ShouldBeTrue)
}

func TestProcess(t *testing.T) {
	a := New(t)

	functions := &Functions{
		Decoder: `function(payload) {
			return {
				temperature: payload[0],
				humidity: payload[1]
			}
		}`,
		Converter: `function(data) {
			data.temperature /= 2;
			return data;
		}`,
		Validator: `function(data) {
			return data.humidity >= 0 && data.humidity <= 100;
		}`,
	}

	data, valid, err := functions.Process([]byte{40, 110})
	a.So(err, ShouldBeNil)
	a.So(valid, ShouldBeFalse)
	a.So(data["temperature"], ShouldEqual, 20)
	a.So(data["humidity"], ShouldEqual, 110)
}
<commit_msg>Consistent indenting<commit_after>// Copyright © 2016 The Things Network
// Use of this source code is governed by the MIT license that can be found in the LICENSE file.

package collection

import (
	"testing"

	. "github.com/smartystreets/assertions"
)

func TestDecode(t *testing.T) {
	a := New(t)

	functions := &Functions{
		Decoder: `function(payload) {
  return {
    value: (payload[0] << 8) | payload[1]
  };
}`,
	}
	payload := []byte{0x48, 0x65}

	m, err := functions.Decode(payload)
	a.So(err, ShouldBeNil)

	size, ok := m["value"]
	a.So(ok, ShouldBeTrue)
	a.So(size, ShouldEqual, 18533)
}

func TestConvert(t *testing.T) {
	a := New(t)

	withFunction := &Functions{
		Converter: `function(data) {
  return {
    celcius: data.temperature * 2
  };
}`,
	}
	data, err := withFunction.Convert(map[string]interface{}{"temperature": 11})
	a.So(err, ShouldBeNil)
	a.So(data["celcius"], ShouldEqual, 22)

	withoutFunction := &Functions{}
	data, err = withoutFunction.Convert(map[string]interface{}{"temperature": 11})
	a.So(err, ShouldBeNil)
	a.So(data["temperature"], ShouldEqual, 11)
}

func TestValidate(t *testing.T) {
	a := New(t)

	withFunction := &Functions{
		Validator: `function(data) {
      return data.temperature < 20;
    }`,
	}
	valid, err := withFunction.Validate(map[string]interface{}{"temperature": 10})
	a.So(err, ShouldBeNil)
	a.So(valid, ShouldBeTrue)
	valid, err = withFunction.Validate(map[string]interface{}{"temperature": 30})
	a.So(err, ShouldBeNil)
	a.So(valid, ShouldBeFalse)

	withoutFunction := &Functions{}
	valid, err = withoutFunction.Validate(map[string]interface{}{"temperature": 10})
	a.So(err, ShouldBeNil)
	a.So(valid, ShouldBeTrue)
}

func TestProcess(t *testing.T) {
	a := New(t)

	functions := &Functions{
		Decoder: `function(payload) {
	return {
		temperature: payload[0],
		humidity: payload[1]
	}
}`,
		Converter: `function(data) {
	data.temperature /= 2;
	return data;
}`,
		Validator: `function(data) {
	return data.humidity >= 0 && data.humidity <= 100;
}`,
	}

	data, valid, err := functions.Process([]byte{40, 110})
	a.So(err, ShouldBeNil)
	a.So(valid, ShouldBeFalse)
	a.So(data["temperature"], ShouldEqual, 20)
	a.So(data["humidity"], ShouldEqual, 110)
}
<|endoftext|>


<commit_before>package rt

import (
	"emacs/lisp"
)

type Slice struct {
	data   lisp.Object
	offset int
	len    int
	cap    int
}

func SliceLen(slice *Slice) int { return slice.len }
func SliceCap(slice *Slice) int { return slice.cap }

// MakeSlice creates a new slice with cap=len.
// All values initialized to specified zero value.
func MakeSlice(length int, zv lisp.Object) *Slice {
	return &Slice{
		data: lisp.Call("make-vector", length, zv),
		len:  length,
		cap:  length,
	}
}

// MakeSliceCap creates a new slice.
// Each value within length bounds is initialized to specified zero value.
func MakeSliceCap(length, capacity int, zv lisp.Object) *Slice {
	if length == capacity {
		return MakeSlice(length, zv)
	}
	data := lisp.Call("make-vector", capacity, lisp.Intern("nil"))
	for i := 0; i < length; i++ {
		lisp.Aset(data, i, zv)
	}
	return &Slice{data: data, len: length, cap: capacity}
}

// ArrayToSlice constructs a new slice from given data vector.
// Vector is not copied.
func ArrayToSlice(data lisp.Object) *Slice {
	length := lisp.Length(data)
	return &Slice{data: data, len: length, cap: length}
}

// SliceGet extract slice value using specified index.
func SliceGet(slice *Slice, index int) lisp.Object {
	return aref(slice.data, slice.offset+index)
}

// SliceSet sets slice value at specified index.
func SliceSet(slice *Slice, index int, val lisp.Object) {
	lisp.Aset(slice.data, index+slice.offset, val)
}

// SlicePush = "append(slice, val)".
func SlicePush(slice *Slice, val lisp.Object) *Slice {
	pos := slice.len
	if pos == slice.cap {
		// Need to extend slice storage.
		// Create a new vector with 1st element set to "val"
		// then re-set slice data with "oldData+newData".
		newData := lisp.Call("make-vector", memExtendPush, lisp.Intern("nil"))
		lisp.Aset(newData, 0, val)
		// For slices with offset a sub-vector should
		// be taken to avoid memory leaks.
		if slice.offset == 0 {
			newData = vconcat2(slice.data, newData)
		} else {
			newData = vconcat2(
				substringFrom(slice.data, slice.offset),
				newData,
			)
		}
		return &Slice{
			data:   newData,
			len:    pos + 1,
			cap:    slice.cap + memExtendPush,
			offset: 0,
		}
	}
	// Insert new value directly.
	slice.len = pos + 1
	SliceSet(slice, pos, val)
	return slice
}

func sliceLenBound(slice *Slice, index int) {
	if index < 0 || index > slice.len {
		lisp.Error("slice bounds out of range")
	}
}

func sliceCapBound(slice *Slice, index int) {
	if index < 0 || index > slice.cap {
		lisp.Error("slice bounds out of range")
	}
}

// SliceCopyFast is SliceCopy specialization that is appliable if both
// `dst' and `src' have zero offset.
func SliceCopyFast(dst, src *Slice) {
	dstData := dst.data
	srcData := src.data
	count := lisp.MinInt(dst.len, src.len)
	for i := 0; i < count; i++ {
		lisp.Aset(dstData, i, aref(srcData, i))
	}
}

// SliceCopy copies one slice contents to another.
// Up to "min(len(dst), len(src))" elements are copied.
func SliceCopy(dst, src *Slice) {
	if dst.offset == 0 && src.offset == 0 {
		SliceCopyFast(dst, src)
		return
	}
	count := lisp.MinInt(dst.len, src.len)
	for i := 0; i < count; i++ {
		SliceSet(dst, i, SliceGet(src, i))
	}
}

// SliceSlice2 = "slice[low:high]".
func SliceSlice2(slice *Slice, low, high int) *Slice {
	sliceLenBound(slice, low)
	sliceCapBound(slice, high)
	return &Slice{
		data:   slice.data,
		offset: slice.offset + low,
		len:    high - low,
		cap:    slice.cap - low,
	}
}

// SliceSliceLow = "slice[low:]".
func SliceSliceLow(slice *Slice, low int) *Slice {
	sliceLenBound(slice, low)
	return &Slice{
		data:   slice.data,
		offset: slice.offset + low,
		len:    slice.len - low,
		cap:    slice.cap - low,
	}
}

// SliceSliceHigh = "slice[:high]".
func SliceSliceHigh(slice *Slice, high int) *Slice {
	sliceCapBound(slice, high)
	return &Slice{
		data:   slice.data,
		offset: slice.offset,
		len:    high,
		cap:    slice.cap,
	}
}

// ArraySlice2 slices an array: "arr[low:high]".
func ArraySlice2(arr lisp.Object, low, high int) *Slice {
	return &Slice{
		data:   arr,
		offset: low,
		len:    high - low,
		cap:    lisp.Length(arr) - low,
	}
}

// ArraySliceLow slices an array: "arr[low:]".
func ArraySliceLow(arr lisp.Object, low int) *Slice {
	length := lisp.Length(arr)
	return &Slice{
		data:   arr,
		offset: low,
		len:    length - low,
		cap:    length - low,
	}
}

// ArraySliceHigh slices an array: "arr[:high]".
func ArraySliceHigh(arr lisp.Object, high int) *Slice {
	return &Slice{
		data:   arr,
		offset: 0,
		len:    high,
		cap:    lisp.Length(arr),
	}
}
<commit_msg>add missing rt.Slice type comment to satisfy the linter<commit_after>package rt

import (
	"emacs/lisp"
)

// Slice - Go slice.
type Slice struct {
	data   lisp.Object
	offset int
	len    int
	cap    int
}

func SliceLen(slice *Slice) int { return slice.len }
func SliceCap(slice *Slice) int { return slice.cap }

// MakeSlice creates a new slice with cap=len.
// All values initialized to specified zero value.
func MakeSlice(length int, zv lisp.Object) *Slice {
	return &Slice{
		data: lisp.Call("make-vector", length, zv),
		len:  length,
		cap:  length,
	}
}

// MakeSliceCap creates a new slice.
// Each value within length bounds is initialized to specified zero value.
func MakeSliceCap(length, capacity int, zv lisp.Object) *Slice {
	if length == capacity {
		return MakeSlice(length, zv)
	}
	data := lisp.Call("make-vector", capacity, lisp.Intern("nil"))
	for i := 0; i < length; i++ {
		lisp.Aset(data, i, zv)
	}
	return &Slice{data: data, len: length, cap: capacity}
}

// ArrayToSlice constructs a new slice from given data vector.
// Vector is not copied.
func ArrayToSlice(data lisp.Object) *Slice {
	length := lisp.Length(data)
	return &Slice{data: data, len: length, cap: length}
}

// SliceGet extract slice value using specified index.
func SliceGet(slice *Slice, index int) lisp.Object {
	return aref(slice.data, slice.offset+index)
}

// SliceSet sets slice value at specified index.
func SliceSet(slice *Slice, index int, val lisp.Object) {
	lisp.Aset(slice.data, index+slice.offset, val)
}

// SlicePush = "append(slice, val)".
func SlicePush(slice *Slice, val lisp.Object) *Slice {
	pos := slice.len
	if pos == slice.cap {
		// Need to extend slice storage.
		// Create a new vector with 1st element set to "val"
		// then re-set slice data with "oldData+newData".
		newData := lisp.Call("make-vector", memExtendPush, lisp.Intern("nil"))
		lisp.Aset(newData, 0, val)
		// For slices with offset a sub-vector should
		// be taken to avoid memory leaks.
		if slice.offset == 0 {
			newData = vconcat2(slice.data, newData)
		} else {
			newData = vconcat2(
				substringFrom(slice.data, slice.offset),
				newData,
			)
		}
		return &Slice{
			data:   newData,
			len:    pos + 1,
			cap:    slice.cap + memExtendPush,
			offset: 0,
		}
	}
	// Insert new value directly.
	slice.len = pos + 1
	SliceSet(slice, pos, val)
	return slice
}

func sliceLenBound(slice *Slice, index int) {
	if index < 0 || index > slice.len {
		lisp.Error("slice bounds out of range")
	}
}

func sliceCapBound(slice *Slice, index int) {
	if index < 0 || index > slice.cap {
		lisp.Error("slice bounds out of range")
	}
}

// SliceCopyFast is SliceCopy specialization that is appliable if both
// `dst' and `src' have zero offset.
func SliceCopyFast(dst, src *Slice) {
	dstData := dst.data
	srcData := src.data
	count := lisp.MinInt(dst.len, src.len)
	for i := 0; i < count; i++ {
		lisp.Aset(dstData, i, aref(srcData, i))
	}
}

// SliceCopy copies one slice contents to another.
// Up to "min(len(dst), len(src))" elements are copied.
func SliceCopy(dst, src *Slice) {
	if dst.offset == 0 && src.offset == 0 {
		SliceCopyFast(dst, src)
		return
	}
	count := lisp.MinInt(dst.len, src.len)
	for i := 0; i < count; i++ {
		SliceSet(dst, i, SliceGet(src, i))
	}
}

// SliceSlice2 = "slice[low:high]".
func SliceSlice2(slice *Slice, low, high int) *Slice {
	sliceLenBound(slice, low)
	sliceCapBound(slice, high)
	return &Slice{
		data:   slice.data,
		offset: slice.offset + low,
		len:    high - low,
		cap:    slice.cap - low,
	}
}

// SliceSliceLow = "slice[low:]".
func SliceSliceLow(slice *Slice, low int) *Slice {
	sliceLenBound(slice, low)
	return &Slice{
		data:   slice.data,
		offset: slice.offset + low,
		len:    slice.len - low,
		cap:    slice.cap - low,
	}
}

// SliceSliceHigh = "slice[:high]".
func SliceSliceHigh(slice *Slice, high int) *Slice {
	sliceCapBound(slice, high)
	return &Slice{
		data:   slice.data,
		offset: slice.offset,
		len:    high,
		cap:    slice.cap,
	}
}

// ArraySlice2 slices an array: "arr[low:high]".
func ArraySlice2(arr lisp.Object, low, high int) *Slice {
	return &Slice{
		data:   arr,
		offset: low,
		len:    high - low,
		cap:    lisp.Length(arr) - low,
	}
}

// ArraySliceLow slices an array: "arr[low:]".
func ArraySliceLow(arr lisp.Object, low int) *Slice {
	length := lisp.Length(arr)
	return &Slice{
		data:   arr,
		offset: low,
		len:    length - low,
		cap:    length - low,
	}
}

// ArraySliceHigh slices an array: "arr[:high]".
func ArraySliceHigh(arr lisp.Object, high int) *Slice {
	return &Slice{
		data:   arr,
		offset: 0,
		len:    high,
		cap:    lisp.Length(arr),
	}
}
<|endoftext|>


<commit_before>package matching_test

import (
	"testing"

	"github.com/SpectoLabs/hoverfly/core/matching"
	. "github.com/onsi/gomega"
)

func Test_ExactMatch_MatchesTrueWithExactMatch(t *testing.T) {
	RegisterTestingT(t)

	Expect(matching.ExactMatch("yes", "yes")).To(BeTrue())
}

func Test_ExactMatch_MatchesFalseWithIncorrectExactMatch(t *testing.T) {
	RegisterTestingT(t)

	Expect(matching.ExactMatch("yes", "no")).To(BeFalse())
}

func Test_ExactMatch_MatchesTrueWithJSON(t *testing.T) {
	RegisterTestingT(t)

	Expect(matching.ExactMatch(`{"test":{"json":true,"minified":true}}`, `{"test":{"json":true,"minified":true}}`)).To(BeTrue())
}

func Test_ExactMatch_MatchesTrueWithUnminifiedJSON(t *testing.T) {
	RegisterTestingT(t)

	Expect(matching.ExactMatch(`{"test":{"json":true,"minified":true}}`, `{
		"test": {
			"json": true,
			"minified":
			true
		}
	}`)).To(BeFalse())
}
<commit_msg>Fixing formatting issue in test json<commit_after>package matching_test

import (
	"testing"

	"github.com/SpectoLabs/hoverfly/core/matching"
	. "github.com/onsi/gomega"
)

func Test_ExactMatch_MatchesTrueWithExactMatch(t *testing.T) {
	RegisterTestingT(t)

	Expect(matching.ExactMatch("yes", "yes")).To(BeTrue())
}

func Test_ExactMatch_MatchesFalseWithIncorrectExactMatch(t *testing.T) {
	RegisterTestingT(t)

	Expect(matching.ExactMatch("yes", "no")).To(BeFalse())
}

func Test_ExactMatch_MatchesTrueWithJSON(t *testing.T) {
	RegisterTestingT(t)

	Expect(matching.ExactMatch(`{"test":{"json":true,"minified":true}}`, `{"test":{"json":true,"minified":true}}`)).To(BeTrue())
}

func Test_ExactMatch_MatchesTrueWithUnminifiedJSON(t *testing.T) {
	RegisterTestingT(t)

	Expect(matching.ExactMatch(`{"test":{"json":true,"minified":true}}`, `{
		"test": {
			"json": true,
			"minified": true
		}
	}`)).To(BeFalse())
}
<|endoftext|>


<commit_before>package blob

import (
	"bytes"
	"compress/gzip"
	"io"
	"log"
	"net/http"
	"strings"
)

const (
	TemplateFiles = "templates"
	StaticFiles   = "static"
)

var mimeMap = map[string]string{
	"css": "text/css",
	"js":  "text/javascript",
}

func GetFile(bucket string, name string) ([]byte, error) {
	reader := bytes.NewReader(files[bucket][name])
	gz, err := gzip.NewReader(reader)
	if err != nil {
		return nil, err
	}

	var b bytes.Buffer
	io.Copy(&b, gz)
	gz.Close()

	return b.Bytes(), nil
}

type Handler struct{}

func (h Handler) ServeHTTP(w http.ResponseWriter, r *http.Request) {
	name := r.URL.String()
	if name == "" {
		name = "index.html"
	}

	file, err := GetFile(StaticFiles, name)
	if err != nil {
		if err != io.EOF {
			log.Printf("Could not get file: %s", err)
		}
		w.WriteHeader(http.StatusNotFound)
		return
	}
	contentType := http.DetectContentType(file)
	if strings.Contains(contentType, "text/plain") || strings.Contains(contentType, "application/octet-stream") {
		parts := strings.Split(name, ".")
		contentType = mimeMap[parts[len(parts)-1]]
	}
	w.Header().Set("Content-Type", contentType)
	w.Write(file)
}
<commit_msg>Take Path fragment from URL instead of whole URL.<commit_after>package blob

import (
	"bytes"
	"compress/gzip"
	"fmt"
	"io"
	"log"
	"net/http"
	"strings"
)

const (
	TemplateFiles = "templates"
	StaticFiles   = "static"
)

var mimeMap = map[string]string{
	"css": "text/css",
	"js":  "text/javascript",
}

func GetFile(bucket string, name string) ([]byte, error) {
	blob, ok := files[bucket][name]
	if !ok {
		return nil, fmt.Errorf("Could not find %s/%s. Missing/updated files.go?", bucket, name)
	}
	reader := bytes.NewReader(blob)
	gz, err := gzip.NewReader(reader)
	if err != nil {
		return nil, err
	}

	var b bytes.Buffer
	io.Copy(&b, gz)
	gz.Close()

	return b.Bytes(), nil
}

type Handler struct{}

func (h Handler) ServeHTTP(w http.ResponseWriter, r *http.Request) {
	name := r.URL.Path
	if name == "" {
		name = "index.html"
	}

	file, err := GetFile(StaticFiles, name)
	if err != nil {
		if err != io.EOF {
			log.Printf("Could not get file: %s", err)
		}
		w.WriteHeader(http.StatusNotFound)
		return
	}
	contentType := http.DetectContentType(file)
	if strings.Contains(contentType, "text/plain") || strings.Contains(contentType, "application/octet-stream") {
		parts := strings.Split(name, ".")
		contentType = mimeMap[parts[len(parts)-1]]
	}
	w.Header().Set("Content-Type", contentType)
	w.Write(file)
}
<|endoftext|>


<commit_before>// Copyright (c) 2016 PaddlePaddle Authors. All Rights Reserve.

// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at

// http://www.apache.org/licenses/LICENSE-2.0

// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package pserver

// #cgo CFLAGS: -I ../../
// #cgo LDFLAGS: ${SRCDIR}/client/c/libpaddle_go_optimizer.a -lstdc++ -lm
// #include "paddle/optimizer/optimizer.h"
// #include <stdlib.h>
// #include <string.h>
import "C"

import (
	"fmt"
	"unsafe"

	log "github.com/inconshreveable/log15"
)

type optimizer struct {
	opt         *C.struct_paddle_optimizer
	elementType ElementType
	contentLen  int
	config      []byte
}

func cArrayToSlice(p unsafe.Pointer, len int) []byte {
	if p == nil {
		return nil
	}

	// create a Go clice backed by a C array, reference:
	// https://github.com/golang/go/wiki/cgo#turning-c-arrays-into-go-slices
	//
	// Go garbage collector will not interact with this data, need
	// to be freed properly.
	return (*[1 << 30]byte)(p)[:len:len]
}

func newOptimizer(paramWithConfigs ParameterWithConfig, State []byte) *optimizer {
	o := &optimizer{}
	o.elementType = paramWithConfigs.Param.ElementType
	o.contentLen = len(paramWithConfigs.Param.Content)
	p := paramWithConfigs.Param
	c := paramWithConfigs.Config
	s := State
	paramBufferSize := C.size_t(len(p.Content))
	log.Info("New Optimizer Created with config", log.Ctx{
		"ElementType": p.ElementType,
		"ParamSize":   paramBufferSize,
		"ConfigSize":  len(c),
		"StateSize":   len(s),
	})
	var cbuffer unsafe.Pointer
	cbuffer = C.malloc(paramBufferSize)

	C.memcpy(cbuffer, unsafe.Pointer(&p.Content[0]), paramBufferSize)
	var cstate unsafe.Pointer
	if len(s) != 0 {
		cstate = unsafe.Pointer(&s[0])
	}

	var cptr (*C.uchar)
	if len(c) > 0 {
		cptr = (*C.uchar)(&c[0])
	}
	o.config = c
	o.opt = C.paddle_create_optimizer(
		cptr,
		C.int(len(c)),
		C.paddle_element_type(p.ElementType),
		cbuffer,
		C.int(paramBufferSize),
		(*C.char)(cstate),
		C.int(len(s)),
	)
	return o
}

func (o *optimizer) GetWeights() []byte {
	var buffer unsafe.Pointer
	// we do not own the buffer, no need to free later.
	bufferLen := C.paddle_optimizer_get_weights(o.opt, &buffer)
	return cArrayToSlice(buffer, int(bufferLen)*C.sizeof_float)
}

func (o *optimizer) GetStates() []byte {
	var cbuffer *C.char
	// we owns the state buffer, need to free later.
	cbufferLen := C.paddle_optimizer_get_state(o.opt, &cbuffer)
	buf := cArrayToSlice(unsafe.Pointer(cbuffer), int(cbufferLen))
	cpy := make([]byte, len(buf))
	copy(cpy, buf)
	C.free(unsafe.Pointer(cbuffer))
	return cpy
}

func (o *optimizer) UpdateParameter(g Gradient) error {
	if o.elementType != g.ElementType {
		return fmt.Errorf("Name: %s, parameter and gradient element type not match, parameter: %v, gradient: %v", g.Name, o.elementType, g.ElementType)
	}

	if o.contentLen != len(g.Content) {
		return fmt.Errorf("Name: %s, parameter and gradient does not have same content len, parameter: %d, gradient: %d", g.Name, o.contentLen, len(g.Content))
	}

	r := C.paddle_update_parameter(o.opt, C.paddle_element_type(g.ElementType), unsafe.Pointer(&g.Content[0]), C.int(len(g.Content)))
	if r != 0 {
		return fmt.Errorf("optimizer update returned error code: %d", r)
	}
	return nil
}

func (o *optimizer) Cleanup() {
	if unsafe.Pointer(o.opt) != nil {
		C.paddle_release_optimizer(o.opt)
		o.opt = (*C.struct_paddle_optimizer)(nil)
	}
}
<commit_msg>Fix according to comments<commit_after>// Copyright (c) 2016 PaddlePaddle Authors. All Rights Reserve.

// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at

// http://www.apache.org/licenses/LICENSE-2.0

// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package pserver

// #cgo CFLAGS: -I ../../
// #cgo LDFLAGS: ${SRCDIR}/client/c/libpaddle_go_optimizer.a -lstdc++ -lm
// #include "paddle/optimizer/optimizer.h"
// #include <stdlib.h>
// #include <string.h>
import "C"

import (
	"fmt"
	"unsafe"

	log "github.com/inconshreveable/log15"
)

type optimizer struct {
	opt         *C.struct_paddle_optimizer
	elementType ElementType
	contentLen  int
	config      []byte
}

func cArrayToSlice(p unsafe.Pointer, len int) []byte {
	if p == nil {
		return nil
	}

	// create a Go clice backed by a C array, reference:
	// https://github.com/golang/go/wiki/cgo#turning-c-arrays-into-go-slices
	//
	// Go garbage collector will not interact with this data, need
	// to be freed properly.
	return (*[1 << 30]byte)(p)[:len:len]
}

func newOptimizer(paramWithConfigs ParameterWithConfig, State []byte) *optimizer {
	o := &optimizer{}
	o.elementType = paramWithConfigs.Param.ElementType
	o.contentLen = len(paramWithConfigs.Param.Content)
	p := paramWithConfigs.Param
	c := paramWithConfigs.Config
	s := State
	paramBufferSize := C.size_t(len(p.Content))
	log.Info("New Optimizer Created with config", log.Ctx{
		"ElementType": p.ElementType,
		"ParamSize":   paramBufferSize,
		"ConfigSize":  len(c),
		"StateSize":   len(s),
	})
	var cbuffer unsafe.Pointer
	cbuffer = C.malloc(paramBufferSize)

	C.memcpy(cbuffer, unsafe.Pointer(&p.Content[0]), paramBufferSize)
	var cstate unsafe.Pointer
	if len(s) != 0 {
		cstate = unsafe.Pointer(&s[0])
	}

	var cptr (*C.uchar)
	if len(c) > 0 {
		cptr = (*C.uchar)(&c[0])
	} else {
		log.Error("empty config", "param name", paramWithConfigs.Param.Name)
	}
	o.config = c
	o.opt = C.paddle_create_optimizer(
		cptr,
		C.int(len(c)),
		C.paddle_element_type(p.ElementType),
		cbuffer,
		C.int(paramBufferSize),
		(*C.char)(cstate),
		C.int(len(s)),
	)
	return o
}

func (o *optimizer) GetWeights() []byte {
	var buffer unsafe.Pointer
	// we do not own the buffer, no need to free later.
	bufferLen := C.paddle_optimizer_get_weights(o.opt, &buffer)
	return cArrayToSlice(buffer, int(bufferLen)*C.sizeof_float)
}

func (o *optimizer) GetStates() []byte {
	var cbuffer *C.char
	// we owns the state buffer, need to free later.
	cbufferLen := C.paddle_optimizer_get_state(o.opt, &cbuffer)
	buf := cArrayToSlice(unsafe.Pointer(cbuffer), int(cbufferLen))
	cpy := make([]byte, len(buf))
	copy(cpy, buf)
	C.free(unsafe.Pointer(cbuffer))
	return cpy
}

func (o *optimizer) UpdateParameter(g Gradient) error {
	if o.elementType != g.ElementType {
		return fmt.Errorf("Name: %s, parameter and gradient element type not match, parameter: %v, gradient: %v", g.Name, o.elementType, g.ElementType)
	}

	if o.contentLen != len(g.Content) {
		return fmt.Errorf("Name: %s, parameter and gradient does not have same content len, parameter: %d, gradient: %d", g.Name, o.contentLen, len(g.Content))
	}

	r := C.paddle_update_parameter(o.opt, C.paddle_element_type(g.ElementType), unsafe.Pointer(&g.Content[0]), C.int(len(g.Content)))
	if r != 0 {
		return fmt.Errorf("optimizer update returned error code: %d", r)
	}
	return nil
}

func (o *optimizer) Cleanup() {
	if unsafe.Pointer(o.opt) != nil {
		C.paddle_release_optimizer(o.opt)
		o.opt = (*C.struct_paddle_optimizer)(nil)
	}
}
<|endoftext|>


<commit_before>package gqt_test

import (
	"fmt"
	"path/filepath"
	"strconv"
	"strings"

	"code.cloudfoundry.org/garden"
	"code.cloudfoundry.org/guardian/gqt/cgrouper"
	"code.cloudfoundry.org/guardian/gqt/runner"
	"code.cloudfoundry.org/guardian/rundmc/cgroups"
	"code.cloudfoundry.org/guardian/sysinfo"
	. "github.com/onsi/ginkgo"
	. "github.com/onsi/gomega"
)

var _ = Describe("CPU shares rebalancing", func() {
	var (
		goodCgroupPath string
		badCgroupPath  string
		client         *runner.RunningGarden
	)

	BeforeEach(func() {
		skipIfNotCPUThrottling()

		// We want an aggressive throttling check to speed moving containers across cgroups up
		// in order to reduce test run time
		config.CPUThrottlingCheckInterval = uint64ptr(1)
	})

	JustBeforeEach(func() {
		client = runner.Start(config)
		var err error
		goodCgroupPath, err = cgrouper.GetCGroupPath(client.CgroupsRootPath(), "cpu", strconv.Itoa(GinkgoParallelNode()), false, cpuThrottlingEnabled())
		Expect(err).NotTo(HaveOccurred())
		badCgroupPath = filepath.Join(goodCgroupPath, "..", "bad")
	})

	AfterEach(func() {
		Expect(client.DestroyAndStop()).To(Succeed())
	})

	It("starts with all shares allocated to the good cgroup", func() {
		Eventually(func() int64 { return readCgroupFile(goodCgroupPath, "cpu.shares") }).Should(BeNumerically(">", 1024))
		Eventually(func() int64 { return readCgroupFile(badCgroupPath, "cpu.shares") }).Should(Equal(int64(2)))
	})

	Describe("rebalancing", func() {
		var (
			container               garden.Container
			containerPort           uint32
			containerGoodCgroupPath string
			containerBadCgroupPath  string
			goodCgroupInitialShares int64
		)

		JustBeforeEach(func() {
			Eventually(func() int64 { return readCgroupFile(badCgroupPath, "cpu.shares") }).Should(Equal(int64(2)))
			goodCgroupInitialShares = readCgroupFile(goodCgroupPath, "cpu.shares")

			var err error
			container, err = client.Create(garden.ContainerSpec{
				Image: garden.ImageRef{URI: "docker:///cfgarden/throttled-or-not"},
				Limits: garden.Limits{
					CPU: garden.CPULimits{
						Weight: 1000,
					},
				},
			})
			Expect(err).NotTo(HaveOccurred())

			containerPort, _, err = container.NetIn(0, 8080)
			Expect(err).NotTo(HaveOccurred())

			_, err = container.Run(garden.ProcessSpec{Path: "/go/src/app/main"}, garden.ProcessIO{})
			Expect(err).NotTo(HaveOccurred())

			Eventually(func() (string, error) {
				return httpGet(fmt.Sprintf("http://%s:%d/ping", externalIP(container), containerPort))
			}).Should(Equal("pong"))

			containerGoodCgroupPath = ensureInCgroup(container, containerPort, cgroups.GoodCgroupName)
			containerBadCgroupPath = strings.Replace(containerGoodCgroupPath, cgroups.GoodCgroupName, cgroups.BadCgroupName, 1)
			fmt.Println(containerBadCgroupPath)
		})

		When("the application is punished to the bad cgroup", func() {
			JustBeforeEach(func() {
				Expect(spin(container, containerPort)).To(Succeed())
				ensureInCgroup(container, containerPort, cgroups.BadCgroupName)
			})

			It("redistributes the container shares to the bad cgroup", func() {
				Eventually(func() int64 { return readCgroupFile(goodCgroupPath, "cpu.shares") }).Should(Equal(int64(goodCgroupInitialShares - (1000 - 2))))
				Eventually(func() int64 { return readCgroupFile(badCgroupPath, "cpu.shares") }).Should(Equal(int64(1000)))
			})

			When("the application is released back to the good cgroup", func() {
				JustBeforeEach(func() {
					Expect(unspin(container, containerPort)).To(Succeed())
					ensureInCgroup(container, containerPort, cgroups.GoodCgroupName)
				})

				It("redistributes the container shares to the good cgroup", func() {
					Eventually(func() int64 { return readCgroupFile(goodCgroupPath, "cpu.shares") }).Should(Equal(goodCgroupInitialShares))
					Eventually(func() int64 { return readCgroupFile(badCgroupPath, "cpu.shares") }).Should(Equal(int64(2)))
				})
			})

			When("cpu-entitlement-per-share is explicitly set", func() {
				BeforeEach(func() {
					resourcesProvider := sysinfo.NewResourcesProvider(config.DepotDir)
					memoryInBytes, err := resourcesProvider.TotalMemory()
					Expect(err).NotTo(HaveOccurred())
					memoryInMbs := memoryInBytes / 1024 / 1024
					cpuCores, err := resourcesProvider.CPUCores()
					Expect(err).NotTo(HaveOccurred())

					defaultEntitlementPerShare := float64(100*cpuCores) / float64(memoryInMbs)
					config.CPUEntitlementPerShare = float64ptr(2 * defaultEntitlementPerShare)
				})

				It("sets the bad cgroup shares proportionally", func() {
					Expect(readCgroupFile(badCgroupPath, "cpu.shares")).To(BeNumerically("~", 2000, 1))
				})
			})
		})
	})
})

func ensureInCgroup(container garden.Container, containerPort uint32, cgroupType string) string {
	cgroupPath := ""
	EventuallyWithOffset(1, func() (string, error) {
		var err error
		cgroupPath, err = getCgroup(container, containerPort)
		return cgroupPath, err
	}, "2m", "100ms").Should(HaveSuffix(filepath.Join(cgroupType, container.Handle())))

	return getAbsoluteCPUCgroupPath(config.Tag, cgroupPath)
}
<commit_msg>Fix a flaky test<commit_after>package gqt_test

import (
	"fmt"
	"path/filepath"
	"strconv"
	"strings"

	"code.cloudfoundry.org/garden"
	"code.cloudfoundry.org/guardian/gqt/cgrouper"
	"code.cloudfoundry.org/guardian/gqt/runner"
	"code.cloudfoundry.org/guardian/rundmc/cgroups"
	"code.cloudfoundry.org/guardian/sysinfo"
	. "github.com/onsi/ginkgo"
	. "github.com/onsi/gomega"
)

var _ = Describe("CPU shares rebalancing", func() {
	var (
		goodCgroupPath string
		badCgroupPath  string
		client         *runner.RunningGarden
	)

	BeforeEach(func() {
		skipIfNotCPUThrottling()

		// We want an aggressive throttling check to speed moving containers across cgroups up
		// in order to reduce test run time
		config.CPUThrottlingCheckInterval = uint64ptr(1)
	})

	JustBeforeEach(func() {
		client = runner.Start(config)
		var err error
		goodCgroupPath, err = cgrouper.GetCGroupPath(client.CgroupsRootPath(), "cpu", strconv.Itoa(GinkgoParallelNode()), false, cpuThrottlingEnabled())
		Expect(err).NotTo(HaveOccurred())
		badCgroupPath = filepath.Join(goodCgroupPath, "..", "bad")
	})

	AfterEach(func() {
		Expect(client.DestroyAndStop()).To(Succeed())
	})

	It("starts with all shares allocated to the good cgroup", func() {
		Eventually(func() int64 { return readCgroupFile(goodCgroupPath, "cpu.shares") }).Should(BeNumerically(">", 1024))
		Eventually(func() int64 { return readCgroupFile(badCgroupPath, "cpu.shares") }).Should(Equal(int64(2)))
	})

	Describe("rebalancing", func() {
		var (
			container               garden.Container
			containerPort           uint32
			containerGoodCgroupPath string
			containerBadCgroupPath  string
			goodCgroupInitialShares int64
		)

		JustBeforeEach(func() {
			Eventually(func() int64 { return readCgroupFile(badCgroupPath, "cpu.shares") }).Should(Equal(int64(2)))
			goodCgroupInitialShares = readCgroupFile(goodCgroupPath, "cpu.shares")

			var err error
			container, err = client.Create(garden.ContainerSpec{
				Image: garden.ImageRef{URI: "docker:///cfgarden/throttled-or-not"},
				Limits: garden.Limits{
					CPU: garden.CPULimits{
						Weight: 1000,
					},
				},
			})
			Expect(err).NotTo(HaveOccurred())

			containerPort, _, err = container.NetIn(0, 8080)
			Expect(err).NotTo(HaveOccurred())

			_, err = container.Run(garden.ProcessSpec{Path: "/go/src/app/main"}, garden.ProcessIO{})
			Expect(err).NotTo(HaveOccurred())

			Eventually(func() (string, error) {
				return httpGet(fmt.Sprintf("http://%s:%d/ping", externalIP(container), containerPort))
			}).Should(Equal("pong"))

			containerGoodCgroupPath = ensureInCgroup(container, containerPort, cgroups.GoodCgroupName)
			containerBadCgroupPath = strings.Replace(containerGoodCgroupPath, cgroups.GoodCgroupName, cgroups.BadCgroupName, 1)
			fmt.Println(containerBadCgroupPath)
		})

		When("the application is punished to the bad cgroup", func() {
			JustBeforeEach(func() {
				Expect(spin(container, containerPort)).To(Succeed())
				ensureInCgroup(container, containerPort, cgroups.BadCgroupName)
			})

			It("redistributes the container shares to the bad cgroup", func() {
				Eventually(func() int64 { return readCgroupFile(goodCgroupPath, "cpu.shares") }).Should(Equal(int64(goodCgroupInitialShares - (1000 - 2))))
				Eventually(func() int64 { return readCgroupFile(badCgroupPath, "cpu.shares") }).Should(Equal(int64(1000)))
			})

			When("the application is released back to the good cgroup", func() {
				JustBeforeEach(func() {
					Expect(unspin(container, containerPort)).To(Succeed())
					ensureInCgroup(container, containerPort, cgroups.GoodCgroupName)
				})

				It("redistributes the container shares to the good cgroup", func() {
					Eventually(func() int64 { return readCgroupFile(goodCgroupPath, "cpu.shares") }).Should(Equal(goodCgroupInitialShares))
					Eventually(func() int64 { return readCgroupFile(badCgroupPath, "cpu.shares") }).Should(Equal(int64(2)))
				})
			})

			When("cpu-entitlement-per-share is explicitly set", func() {
				BeforeEach(func() {
					resourcesProvider := sysinfo.NewResourcesProvider(config.DepotDir)
					memoryInBytes, err := resourcesProvider.TotalMemory()
					Expect(err).NotTo(HaveOccurred())
					memoryInMbs := memoryInBytes / 1024 / 1024
					cpuCores, err := resourcesProvider.CPUCores()
					Expect(err).NotTo(HaveOccurred())

					defaultEntitlementPerShare := float64(100*cpuCores) / float64(memoryInMbs)
					config.CPUEntitlementPerShare = float64ptr(2 * defaultEntitlementPerShare)
				})

				It("sets the bad cgroup shares proportionally", func() {
					Eventually(readCgroupFile(badCgroupPath, "cpu.shares")).Should(BeNumerically("~", 2000, 1))
				})
			})
		})
	})
})

func ensureInCgroup(container garden.Container, containerPort uint32, cgroupType string) string {
	cgroupPath := ""
	EventuallyWithOffset(1, func() (string, error) {
		var err error
		cgroupPath, err = getCgroup(container, containerPort)
		return cgroupPath, err
	}, "2m", "100ms").Should(HaveSuffix(filepath.Join(cgroupType, container.Handle())))

	return getAbsoluteCPUCgroupPath(config.Tag, cgroupPath)
}
<|endoftext|>


<commit_before>/*
 * The Interpreter takes entities and breaks them down into component shapes
 * to be drawn by the lower-level graphics components.
 *
 */

package graphics

import (
	"image/color"
	"math"
)

import (
	"github.com/DiscoViking/goBrains/creature"
	"github.com/DiscoViking/goBrains/entity"
	"github.com/DiscoViking/goBrains/locationmanager"
)
import "github.com/DiscoViking/goBrains/food"

func Interpret(lm locationmanager.Location, in chan entity.Entity, out chan Primitive) {
	defer close(out)
	for e := range in {
		switch e := e.(type) {
		case *creature.Creature:
			breakCreature(lm, e, out)
		case *food.Food:
			breakFood(lm, e, out)
		default:
			breakEntity(lm, e, out)
		}
	}
}

func breakEntity(lm locationmanager.Location, e entity.Entity, out chan Primitive) {
	ok, comb := lm.GetLocation(e)
	if !ok {
		return
	}
	x, y := comb.X, comb.Y

	out <- Circle{int16(x), int16(y), uint16(10), 0, color.Black}
}

func breakCreature(lm locationmanager.Location, c *creature.Creature, out chan Primitive) {
	ok, comb := lm.GetLocation(c)
	if !ok {
		return
	}

	x, y, o := comb.X, comb.Y, comb.Orient
	cosO := math.Cos(o)
	sinO := math.Sin(o)

	// Body
	out <- Circle{int16(x), int16(y), uint16(8), 0, color.RGBA{200, 50, 50, 255}}
	dx := cosO * 6
	dy := sinO * 6
	out <- Circle{int16(x - dx), int16(y - dy), uint16(6), 0, color.RGBA{200, 50, 50, 255}}
	dx = cosO * 10
	dy = sinO * 10
	out <- Circle{int16(x - dx), int16(y - dy), uint16(4), 0, color.RGBA{200, 50, 50, 255}}

	// Mouth
	dx = cosO * 6
	dy = sinO * 6
	out <- Circle{int16(x + dx), int16(y + dy), uint16(2), 0, color.Black}

	// Antennae
	dx = math.Cos(o+math.Pi/6) * 40
	dy = math.Sin(o+math.Pi/6) * 40
	out <- Circle{int16(x + dx), int16(y + dy), uint16(2), 0, color.RGBA{200, 200, 50, 255}}
	dx = math.Cos(o-math.Pi/6) * 40
	dy = math.Sin(o-math.Pi/6) * 40
	out <- Circle{int16(x + dx), int16(y + dy), uint16(2), 0, color.RGBA{200, 200, 50, 255}}
}

func breakFood(lm locationmanager.Location, f *food.Food, out chan Primitive) {
	ok, comb := lm.GetLocation(f)
	if !ok {
		return
	}
	x, y := comb.X, comb.Y
	out <- Circle{int16(x), int16(y), uint16(f.GetRadius()), 0, color.RGBA{50, 200, 50, 255}}
}
<commit_msg>Made the interpeter draw lines for antenna.<commit_after>/*
 * The Interpreter takes entities and breaks them down into component shapes
 * to be drawn by the lower-level graphics components.
 *
 */

package graphics

import (
	"image/color"
	"math"
)

import (
	"github.com/DiscoViking/goBrains/creature"
	"github.com/DiscoViking/goBrains/entity"
	"github.com/DiscoViking/goBrains/locationmanager"
)
import "github.com/DiscoViking/goBrains/food"

func Interpret(lm locationmanager.Location, in chan entity.Entity, out chan Primitive) {
	defer close(out)
	for e := range in {
		switch e := e.(type) {
		case *creature.Creature:
			breakCreature(lm, e, out)
		case *food.Food:
			breakFood(lm, e, out)
		default:
			breakEntity(lm, e, out)
		}
	}
}

func breakEntity(lm locationmanager.Location, e entity.Entity, out chan Primitive) {
	ok, comb := lm.GetLocation(e)
	if !ok {
		return
	}
	x, y := comb.X, comb.Y

	out <- Circle{int16(x), int16(y), uint16(10), 0, color.Black}
}

func breakCreature(lm locationmanager.Location, c *creature.Creature, out chan Primitive) {
	ok, comb := lm.GetLocation(c)
	if !ok {
		return
	}
	var dx, dy float64

	x, y, o := comb.X, comb.Y, comb.Orient
	cosO := math.Cos(o)
	sinO := math.Sin(o)

	// Draw the antenna lines first, so that the circles cover them.
	dx = math.Cos(o+math.Pi/6) * 40
	dy = math.Sin(o+math.Pi/6) * 40
	out <- Line{int16(x), int16(y), int16(x + dx), int16(y + dy), color.RGBA{170, 170, 170, 255}}
	dx = math.Cos(o-math.Pi/6) * 40
	dy = math.Sin(o-math.Pi/6) * 40
	out <- Line{int16(x), int16(y), int16(x + dx), int16(y + dy), color.RGBA{170, 170, 170, 255}}

	// Body
	out <- Circle{int16(x), int16(y), uint16(8), 0, color.RGBA{200, 50, 50, 255}}
	dx = cosO * 6
	dy = sinO * 6
	out <- Circle{int16(x - dx), int16(y - dy), uint16(6), 0, color.RGBA{200, 50, 50, 255}}
	dx = cosO * 10
	dy = sinO * 10
	out <- Circle{int16(x - dx), int16(y - dy), uint16(4), 0, color.RGBA{200, 50, 50, 255}}

	// Mouth
	dx = cosO * 6
	dy = sinO * 6
	out <- Circle{int16(x + dx), int16(y + dy), uint16(2), 0, color.Black}

	// Antennae
	dx = math.Cos(o+math.Pi/6) * 40
	dy = math.Sin(o+math.Pi/6) * 40
	out <- Circle{int16(x + dx), int16(y + dy), uint16(2), 0, color.RGBA{200, 200, 50, 255}}
	dx = math.Cos(o-math.Pi/6) * 40
	dy = math.Sin(o-math.Pi/6) * 40
	out <- Circle{int16(x + dx), int16(y + dy), uint16(2), 0, color.RGBA{200, 200, 50, 255}}
}

func breakFood(lm locationmanager.Location, f *food.Food, out chan Primitive) {
	ok, comb := lm.GetLocation(f)
	if !ok {
		return
	}
	x, y := comb.X, comb.Y
	out <- Circle{int16(x), int16(y), uint16(f.GetRadius()), 0, color.RGBA{50, 200, 50, 255}}
}
<|endoftext|>


<commit_before>package byteutils

func Cut(a []byte, from, to int) []byte {
	copy(a[from:], a[to:])
	a = a[:len(a)-to+from]

	return a
}

func Insert(a []byte, i int, b []byte) []byte {
	a = append(a, make([]byte, len(b))...)
	copy(a[i+len(b):], a[i:])
	copy(a[i:i+len(b)], b)

	return a
}

// Unlike bytes.Replace it allows you to specify range
func Replace(a []byte, from, to int, new []byte) []byte {
	lenDiff := len(new) - (to - from)

	if lenDiff > 0 {
		// Extend if new segment bigger
		a = append(a, make([]byte, lenDiff)...)
		copy(a[to+lenDiff:], a[to:])
		copy(a[from:from+len(new)], new)

		return a
	} else if lenDiff < 0 {
		copy(a[from:], new)
		copy(a[from+len(new):], a[to:])
		return a[:len(a)+lenDiff]
	} else { // same size
		copy(a[from:], new)
		return a
	}
}
<commit_msg>Improve byteutils<commit_after>// Package byteutils probvides helpers for working with byte slices
package byteutils

// Cut elements from slice for a given range
func Cut(a []byte, from, to int) []byte {
	copy(a[from:], a[to:])
	a = a[:len(a)-to+from]

	return a
}

// Insert new slice at specified position
func Insert(a []byte, i int, b []byte) []byte {
	a = append(a, make([]byte, len(b))...)
	copy(a[i+len(b):], a[i:])
	copy(a[i:i+len(b)], b)

	return a
}

// Replace function unlike bytes.Replace allows you to specify range
func Replace(a []byte, from, to int, new []byte) []byte {
	lenDiff := len(new) - (to - from)

	if lenDiff > 0 {
		// Extend if new segment bigger
		a = append(a, make([]byte, lenDiff)...)
		copy(a[to+lenDiff:], a[to:])
		copy(a[from:from+len(new)], new)

		return a
	} else if lenDiff < 0 {
		copy(a[from:], new)
		copy(a[from+len(new):], a[to:])
		return a[:len(a)+lenDiff]
	} else { // same size
		copy(a[from:], new)
		return a
	}
}
<|endoftext|>


<commit_before>package webapi

import (
	"flag"
	"fmt"
	"log"
	"net/http"
	"strconv"

	"github.com/gophergala2016/goad/sqsadaptor"
	"github.com/gorilla/websocket"
)

var addr = flag.String("addr", ":8080", "http service address")
var upgrader = websocket.Upgrader{}

func serveResults(w http.ResponseWriter, r *http.Request) {
	if r.URL.Path != "/goad" {
		http.Error(w, "Not found", 404)
		return
	}
	if r.Method != "GET" {
		http.Error(w, "Method not allowed", 405)
		return
	}
	w.Header().Set("Content-Type", "application/json; charset=utf-8")

	url := r.URL.Query().Get("url")
	if len(url) == 0 {
		http.Error(w, "Missing URL", 400)
		return
	}

	concurrencyStr := r.URL.Query().Get("c")
	concurrency, cerr := strconv.Atoi(concurrencyStr)
	if cerr != nil {
		http.Error(w, "Invalid concurrency", 400)
		return
	}

	totStr := r.URL.Query().Get("tot")
	tot, toterr := strconv.Atoi(totStr)
	if toterr != nil {
		http.Error(w, "Invalid total", 400)
		return
	}

	timeoutStr := r.URL.Query().Get("timeout")
	timeout, timeouterr := strconv.Atoi(timeoutStr)
	if timeouterr != nil {
		http.Error(w, "Invalid timeout", 400)
		return
	}

	c, err := upgrader.Upgrade(w, r, nil)
	if err != nil {
		log.Print("Websocket upgrade:", err)
		return
	}
	defer c.Close()

	resultChan := make(chan sqsadaptor.RegionsAggData)

	// go startTest(url, concurrency, tot, timeout)

	sqsadaptor.Aggregate(resultChan)

	for {
		result, more := <-resultChan
		if !more {
			break
		}
		fmt.Println(result) // stuff the results over the websocket
		err = c.WriteMessage(websocket.TextMessage, []byte("{\"hello\" : \"goodbye\"}"))
		if err != nil {
			log.Println("write:", err)
			break
		}
	}
}

// Serve waits for connections and serves the results
func Serve() {
	http.HandleFunc("/goad", serveResults)
	err := http.ListenAndServe(*addr, nil)
	if err != nil {
		log.Fatal("ListenAndServe: ", err)
	}
}
<commit_msg>Fix websockets to send real aggregated data<commit_after>package webapi

import (
	"encoding/json"
	"flag"
	"log"
	"net/http"
	"strconv"

	"github.com/gophergala2016/goad/sqsadaptor"
	"github.com/gorilla/websocket"
)

var addr = flag.String("addr", ":8080", "http service address")
var upgrader = websocket.Upgrader{}

func jsonFromRegionsAggData(result sqsadaptor.RegionsAggData) (string, error) {
	data, jsonerr := json.Marshal(result)
	if jsonerr != nil {
		return "", jsonerr
	}
	return string(data), nil
}

func serveResults(w http.ResponseWriter, r *http.Request) {
	if r.URL.Path != "/goad" {
		http.Error(w, "Not found", 404)
		return
	}
	if r.Method != "GET" {
		http.Error(w, "Method not allowed", 405)
		return
	}
	w.Header().Set("Content-Type", "application/json; charset=utf-8")

	url := r.URL.Query().Get("url")
	if len(url) == 0 {
		http.Error(w, "Missing URL", 400)
		return
	}

	concurrencyStr := r.URL.Query().Get("c")
	concurrency, cerr := strconv.Atoi(concurrencyStr)
	if cerr != nil {
		http.Error(w, "Invalid concurrency", 400)
		return
	}

	totStr := r.URL.Query().Get("tot")
	tot, toterr := strconv.Atoi(totStr)
	if toterr != nil {
		http.Error(w, "Invalid total", 400)
		return
	}

	timeoutStr := r.URL.Query().Get("timeout")
	timeout, timeouterr := strconv.Atoi(timeoutStr)
	if timeouterr != nil {
		http.Error(w, "Invalid timeout", 400)
		return
	}

	c, err := upgrader.Upgrade(w, r, nil)
	if err != nil {
		log.Print("Websocket upgrade:", err)
		return
	}
	defer c.Close()

	resultChan := make(chan sqsadaptor.RegionsAggData)

	// go startTest(url, concurrency, tot, timeout)

	//	sqsadaptor.Aggregate(resultChan)

	for {
		result, more := <-resultChan
		if !more {
			break
		}

		message, jsonerr := jsonFromRegionsAggData(result)
		if jsonerr != nil {
			log.Println(jsonerr)
			break
		}
		err = c.WriteMessage(websocket.TextMessage, []byte(message))
		if err != nil {
			log.Println("write:", err)
			break
		}
	}
}

// Serve waits for connections and serves the results
func Serve() {
	http.HandleFunc("/goad", serveResults)
	err := http.ListenAndServe(*addr, nil)
	if err != nil {
		log.Fatal("ListenAndServe: ", err)
	}
}
<|endoftext|>


<commit_before>package caching

import (
	"errors"
	"io/ioutil"
	"log"
	"os"
	"os/exec"
	"strconv"
	"strings"
	"time"

	aw "github.com/deanishe/awgo"
	"github.com/rkoval/alfred-aws-console-services-workflow/awsconfig"
	"github.com/rkoval/alfred-aws-console-services-workflow/searchers/searchutil"
	"github.com/rkoval/alfred-aws-console-services-workflow/util"
)

var jobName = "fetch"

func handleExpiredCache(wf *aw.Workflow, cacheName string, lastFetchErrPath string, searchArgs searchutil.SearchArgs) error {
	maxCacheAgeSeconds := 180
	m := os.Getenv("ALFRED_AWS_CONSOLE_SERVICES_WORKFLOW_MAX_CACHE_AGE_SECONDS")
	if m != "" {
		converted, err := strconv.Atoi(m)
		if err != nil {
			panic(err)
		}
		if converted != 0 {
			log.Printf("using custom max cache age of %v seconds", converted)
			maxCacheAgeSeconds = converted
		}
	}

	maxCacheAge := time.Duration(maxCacheAgeSeconds) * time.Second
	if wf.Cache.Expired(cacheName, maxCacheAge) {
		log.Printf("cache with key `%s` was expired (older than %d seconds) in %s", cacheName, maxCacheAgeSeconds, wf.CacheDir())
		wf.Rerun(0.5)
		if !wf.IsRunning(jobName) {
			cmd := exec.Command(os.Args[0], "-query="+searchArgs.FullQuery+"", "-fetch")
			log.Printf("running `%s` in background as job `%s` ...", cmd, jobName)
			if err := wf.RunInBackground(jobName, cmd); err != nil {
				panic(err)
			}
		} else {
			log.Printf("background job `%s` already running", jobName)
		}

		return handleFetchErr(wf, lastFetchErrPath, searchArgs)
	}

	return nil
}

func handleFetchErr(wf *aw.Workflow, lastFetchErrPath string, searchArgs searchutil.SearchArgs) error {
	data, err := ioutil.ReadFile(lastFetchErrPath)
	if err != nil {
		if !os.IsNotExist(err) {
			// this file will often not exist, so don't spam logs if it doesn't
			log.Println(err)
		}
		return nil
	}

	// TODO need to fix "no results" display when there's really a fetch error

	userHomePath := os.Getenv("HOME")
	errString := string(data)
	wf.Configure(aw.SuppressUIDs(true))
	if strings.HasPrefix(errString, "NoCredentialProviders") {
		credentialsFilePath := strings.Replace(awsconfig.GetAwsCredentialsFilePath(), userHomePath, "~", 1)
		util.NewURLItem(wf, "AWS credentials not set in "+credentialsFilePath+" for profile \""+searchArgs.Profile+"\"").
			Subtitle("Press enter to open AWS docs on how to configure").
			Arg("https://aws.github.io/aws-sdk-go-v2/docs/configuring-sdk/#creating-the-credentials-file").
			Icon(aw.IconError).
			Valid(true)
	} else if strings.HasPrefix(errString, "MissingRegion") {
		configFilePath := strings.Replace(awsconfig.GetAwsProfileFilePath(), userHomePath, "~", 1)
		util.NewURLItem(wf, "AWS region not set in "+configFilePath+" for profile \""+searchArgs.Profile+"\"").
			Subtitle("Press enter to open AWS docs on how to configure").
			Arg("https://aws.github.io/aws-sdk-go-v2/docs/configuring-sdk/#creating-the-config-file").
			Icon(aw.IconError).
			Valid(true)
	} else {
		wf.NewItem(errString).
			Icon(aw.IconError)
	}

	return errors.New(errString)
}
<commit_msg>fixed error messages<commit_after>package caching

import (
	"errors"
	"io/ioutil"
	"log"
	"os"
	"os/exec"
	"strconv"
	"strings"
	"time"

	aw "github.com/deanishe/awgo"
	"github.com/rkoval/alfred-aws-console-services-workflow/awsconfig"
	"github.com/rkoval/alfred-aws-console-services-workflow/searchers/searchutil"
	"github.com/rkoval/alfred-aws-console-services-workflow/util"
)

var jobName = "fetch"

func handleExpiredCache(wf *aw.Workflow, cacheName string, lastFetchErrPath string, searchArgs searchutil.SearchArgs) error {
	maxCacheAgeSeconds := 180
	m := os.Getenv("ALFRED_AWS_CONSOLE_SERVICES_WORKFLOW_MAX_CACHE_AGE_SECONDS")
	if m != "" {
		converted, err := strconv.Atoi(m)
		if err != nil {
			panic(err)
		}
		if converted != 0 {
			log.Printf("using custom max cache age of %v seconds", converted)
			maxCacheAgeSeconds = converted
		}
	}

	maxCacheAge := time.Duration(maxCacheAgeSeconds) * time.Second
	if wf.Cache.Expired(cacheName, maxCacheAge) {
		log.Printf("cache with key `%s` was expired (older than %d seconds) in %s", cacheName, maxCacheAgeSeconds, wf.CacheDir())
		wf.Rerun(0.5)
		if !wf.IsRunning(jobName) {
			cmd := exec.Command(os.Args[0], "-query="+searchArgs.FullQuery+"", "-fetch")
			log.Printf("running `%s` in background as job `%s` ...", cmd, jobName)
			if err := wf.RunInBackground(jobName, cmd); err != nil {
				panic(err)
			}
		} else {
			log.Printf("background job `%s` already running", jobName)
		}

		return handleFetchErr(wf, lastFetchErrPath, searchArgs)
	}

	return nil
}

func handleFetchErr(wf *aw.Workflow, lastFetchErrPath string, searchArgs searchutil.SearchArgs) error {
	data, err := ioutil.ReadFile(lastFetchErrPath)
	if err != nil {
		if !os.IsNotExist(err) {
			// this file will often not exist, so don't spam logs if it doesn't
			log.Println(err)
		}
		return nil
	}

	// TODO need to fix "no results" display when there's really a fetch error

	userHomePath := os.Getenv("HOME")
	errString := string(data)
	wf.Configure(aw.SuppressUIDs(true))
	var profileDescription string
	if searchArgs.Profile == "" {
		profileDescription = "for default profile"
	} else {
		profileDescription = "for profile \"" + searchArgs.Profile + "\""
	}
	if strings.HasPrefix(errString, "NoCredentialProviders") {
		credentialsFilePath := strings.Replace(awsconfig.GetAwsCredentialsFilePath(), userHomePath, "~", 1)
		util.NewURLItem(wf, "AWS credentials not set in "+credentialsFilePath+" "+profileDescription).
			Subtitle("Press enter to open AWS docs on how to configure").
			Arg("https://aws.github.io/aws-sdk-go-v2/docs/configuring-sdk/#creating-the-credentials-file").
			Icon(aw.IconError).
			Valid(true)
	} else if strings.HasPrefix(errString, "MissingRegion") {
		configFilePath := strings.Replace(awsconfig.GetAwsProfileFilePath(), userHomePath, "~", 1)
		util.NewURLItem(wf, "AWS region not set in "+configFilePath+" "+profileDescription).
			Subtitle("Press enter to open AWS docs on how to configure").
			Arg("https://aws.github.io/aws-sdk-go-v2/docs/configuring-sdk/#creating-the-config-file").
			Icon(aw.IconError).
			Valid(true)
	} else {
		wf.NewItem(errString).
			Icon(aw.IconError)
	}

	return errors.New(errString)
}
<|endoftext|>


<commit_before>package config

const ColorIsOut = 5

const ForwardAcceleration = 10000 / 600
const ReverseAcceleration = 10000 / 1

//const MaxIrValue = 100

const MaxIrDistance = 80

const IgnoreBorderIrDistance = 40

// VisionIntensityMax is the maximum vision intensity
//const VisionIntensityMax = 100

// VisionAngleMax is the maximum vision angle (positive on the right)
//const VisionAngleMax = 100

const MaxSpeed = 10000

const StartTime = 500

const SeekMoveSpeed = 3500
const SeekMoveMillis = 860
const SeekTurnSpeed = 3700
const SeekTurnMillis = 1200

const BackTurn1SpeedOuter = MaxSpeed
const BackTurn1SpeedInner = MaxSpeed / 2
const BackTurn1Millis = 80
const BackTurn2Speed = 3700
const BackTurn2Millis = 500
const BackMoveSpeed = MaxSpeed
const BackMoveMillis = 5
const BackTurn3Speed = 3700
const BackTurn3Millis = 1600

const CircleFindBorderMillis = 150
const CircleFindBorderOuterSpeed = MaxSpeed * 80 / 100
const CircleFindBorderInnerSpeed = MaxSpeed * 40 / 100
const CircleFindBorderOuterSpeedSlowLeft = MaxSpeed * 28 / 100
const CircleFindBorderInnerSpeedSlowLeft = MaxSpeed * 18 / 100
const CircleFindBorderOuterSpeedSlowRight = MaxSpeed * 28 / 100
const CircleFindBorderInnerSpeedSlowRight = MaxSpeed * 18 / 100
const CircleMillis = 2500
const CircleOuterSpeed = MaxSpeed
const CircleInnerSpeedLeft = 3000
const CircleInnerSpeedRight = 3000
const CircleAdjustInnerMax = 500
const CircleSpiralMillis = 450
const CircleSpiralOuterSpeed = MaxSpeed
const CircleSpiralInnerSpeed = 1500

const GoForwardMillis = 700
const GoForwardSpeed = MaxSpeed

// const GoForwardTurnMillis = 600
const GoForwardTurnMillis = 0
const GoForwardTurnOuterSpeed = MaxSpeed
const GoForwardTurnInnerSpeed = 1000

const TurnBackPreMoveMillis = 400
const TurnBackPreMoveSpeed = MaxSpeed
const TurnBackMillis = 120
const TurnBackOuterSpeed = MaxSpeed
const TurnBackInnerSpeed = -MaxSpeed
const TurnBackMoveMillis = 800
const TurnBackMoveSpeed = MaxSpeed

const TrackOnly1SensorOuterSpeed = MaxSpeed
const TrackOnly1SensorInnerSpeed = 8000
const TrackSpeed = MaxSpeed
const TrackCenterZone = 10
const TrackDifferenceCoefficent = 50
<commit_msg>Reset xl4 start time to official one.<commit_after>package config

const ColorIsOut = 5

const ForwardAcceleration = 10000 / 600
const ReverseAcceleration = 10000 / 1

//const MaxIrValue = 100

const MaxIrDistance = 80

const IgnoreBorderIrDistance = 40

// VisionIntensityMax is the maximum vision intensity
//const VisionIntensityMax = 100

// VisionAngleMax is the maximum vision angle (positive on the right)
//const VisionAngleMax = 100

const MaxSpeed = 10000

const StartTime = 5000

const SeekMoveSpeed = 3500
const SeekMoveMillis = 860
const SeekTurnSpeed = 3700
const SeekTurnMillis = 1200

const BackTurn1SpeedOuter = MaxSpeed
const BackTurn1SpeedInner = MaxSpeed / 2
const BackTurn1Millis = 80
const BackTurn2Speed = 3700
const BackTurn2Millis = 500
const BackMoveSpeed = MaxSpeed
const BackMoveMillis = 5
const BackTurn3Speed = 3700
const BackTurn3Millis = 1600

const CircleFindBorderMillis = 150
const CircleFindBorderOuterSpeed = MaxSpeed * 80 / 100
const CircleFindBorderInnerSpeed = MaxSpeed * 40 / 100
const CircleFindBorderOuterSpeedSlowLeft = MaxSpeed * 28 / 100
const CircleFindBorderInnerSpeedSlowLeft = MaxSpeed * 18 / 100
const CircleFindBorderOuterSpeedSlowRight = MaxSpeed * 28 / 100
const CircleFindBorderInnerSpeedSlowRight = MaxSpeed * 18 / 100
const CircleMillis = 2500
const CircleOuterSpeed = MaxSpeed
const CircleInnerSpeedLeft = 3000
const CircleInnerSpeedRight = 3000
const CircleAdjustInnerMax = 500
const CircleSpiralMillis = 450
const CircleSpiralOuterSpeed = MaxSpeed
const CircleSpiralInnerSpeed = 1500

const GoForwardMillis = 700
const GoForwardSpeed = MaxSpeed

// const GoForwardTurnMillis = 600
const GoForwardTurnMillis = 0
const GoForwardTurnOuterSpeed = MaxSpeed
const GoForwardTurnInnerSpeed = 1000

const TurnBackPreMoveMillis = 400
const TurnBackPreMoveSpeed = MaxSpeed
const TurnBackMillis = 120
const TurnBackOuterSpeed = MaxSpeed
const TurnBackInnerSpeed = -MaxSpeed
const TurnBackMoveMillis = 800
const TurnBackMoveSpeed = MaxSpeed

const TrackOnly1SensorOuterSpeed = MaxSpeed
const TrackOnly1SensorInnerSpeed = 8000
const TrackSpeed = MaxSpeed
const TrackCenterZone = 10
const TrackDifferenceCoefficent = 50
<|endoftext|>


<commit_before>package preloadlist

import (
	"io/ioutil"
	"os"
	"reflect"
	"testing"
)

func TestIndexing(t *testing.T) {
	list := PreloadList{
		Entries: []Entry{
			{
				Name:              "garron.NET",
				Mode:              "force-https",
				IncludeSubDomains: true,
			},
			{
				Name:              "example.com",
				Mode:              "",
				IncludeSubDomains: false,
			},
			{
				Name:              "bar",
				Mode:              "force-https",
				IncludeSubDomains: true,
			},
		},
	}

	idx := list.Index()

	if len(idx.index) != 3 {
		t.Errorf("Map has the wrong number of entries.")
	}

	_, ok := idx.Get("example")
	if ok != EntryNotFound {
		t.Errorf("Entry should not be present.")
	}

	entry, ok := idx.Get("GARRON.net")
	if ok != ExactEntryFound {
		t.Errorf("Entry should be present.")
	}
	if entry.Mode != "force-https" {
		t.Errorf("Map has invalid entry.")
	}

	entry, ok = idx.Get("www.garron.net")
	if ok != AncestorEntryFound {
		t.Errorf("Ancestor entry should be present.")
	}
	if entry.Name != "garron.NET" {
		t.Errorf("Wrong ancestor entry found.")
	}
	if !entry.IncludeSubDomains {
		t.Errorf("Ancestor entry does not include subdomains.")
	}

	entry, ok = idx.Get("test.example.com")
	if ok == AncestorEntryFound {
		t.Errorf("Ancestor entry found, but it does not include subdomains.")
	}
	if entry.IncludeSubDomains {
		t.Errorf("Ancestory entry should not include subdomains.")
	}

	entry, ok = idx.Get("foo.bar")
	if ok != AncestorEntryFound {
		t.Errorf("Ancestor entry should be present.")
	}
	if entry.Name != "bar" || entry.Mode != "force-https" {
		t.Errorf("Wrong ancestor entry found.")
	}
	if !entry.IncludeSubDomains {
		t.Errorf("Ancestor entry does not include subdomains.")
	}

	entry, ok = idx.Get("bar")
	if ok != ExactEntryFound {
		t.Errorf("Entry should be present.")
	}
	if entry.Name != "bar" || entry.Mode != "ForceHTTPS" || !entry.IncludeSubDomains {
		t.Errorf("Wrong entry found.")
	}
}

func TestNewFromLatest(t *testing.T) {
	if testing.Short() {
		t.Skip("Skipping test to avoid preload list download.")
	}

	list, err := NewFromLatest()
	if err != nil {
		t.Errorf("Could not retrieve preload list.")
	}

	firstEntry := list.Entries[0]
	if firstEntry.Name != "pinningtest.appspot.com" {
		t.Errorf("First entry of preload list does not have the expected name.")
	}
}

func TestNewFromChromiumURL(t *testing.T) {
	if testing.Short() {
		t.Skip("Skipping test to avoid preload list download.")
	}

	list, err := NewFromChromiumURL("https://chromium.googlesource.com/chromium/src/+/4f587d7d4532287308715d824d19e7465c9f663e/net/http/transport_security_state_static.json?format=TEXT")
	if err != nil {
		t.Error(err)
	}
	if len(list.Entries) != 3558 {
		t.Errorf("Wrong number of entries: %d", len(list.Entries))
	}
}

var (
	testJSON = `{
  "entries": [
  	// This is a comment.
    {"name": "garron.net", "include_subdomains": true, "mode": "force-https"},
    {"name": "example.com", "include_subdomains": false, "mode": "force-https"},
    {"name": "gmail.com", "mode": "force-https"},

    // Line above intentionally left blank.
    {"name": "google.com"},
    {"name": "pinned.badssl.com", "pins": "pinnymcpinnedkey"}
  ]
}`
	testParsed = PreloadList{Entries: []Entry{
		{"garron.net", "force-https", true},
		{"example.com", "force-https", false},
		{"gmail.com", "force-https", false},
		{"google.com", "", false},
		{"pinned.badssl.com", "", false}},
	}
)

func TestNewFromFile(t *testing.T) {
	f, err := ioutil.TempFile("", "preloadlist-test")
	if err != nil {
		t.Fatal(err)
	}
	defer os.Remove(f.Name())

	if _, err := f.Write([]byte(testJSON)); err != nil {
		t.Fatal(err)
	}

	list, err := NewFromFile(f.Name())
	if err != nil {
		t.Fatalf("Could not read preload list. %s", err)
	}

	if !reflect.DeepEqual(list, testParsed) {
		t.Errorf("Parsed list does not match expected. %#v", list)
	}
}
<commit_msg>Fix tests.<commit_after>package preloadlist

import (
	"io/ioutil"
	"os"
	"reflect"
	"testing"
)

func TestIndexing(t *testing.T) {
	list := PreloadList{
		Entries: []Entry{
			{
				Name:              "garron.NET",
				Mode:              "force-https",
				IncludeSubDomains: true,
			},
			{
				Name:              "example.com",
				Mode:              "",
				IncludeSubDomains: false,
			},
			{
				Name:              "bar",
				Mode:              "force-https",
				IncludeSubDomains: true,
			},
		},
	}

	idx := list.Index()

	if len(idx.index) != 3 {
		t.Errorf("Map has the wrong number of entries.")
	}

	_, ok := idx.Get("example")
	if ok != EntryNotFound {
		t.Errorf("Entry should not be present.")
	}

	entry, ok := idx.Get("GARRON.net")
	if ok != ExactEntryFound {
		t.Errorf("Entry should be present.")
	}
	if entry.Mode != "force-https" {
		t.Errorf("Map has invalid entry.")
	}

	entry, ok = idx.Get("www.garron.net")
	if ok != AncestorEntryFound {
		t.Errorf("Ancestor entry should be present.")
	}
	if entry.Name != "garron.NET" {
		t.Errorf("Wrong ancestor entry found.")
	}
	if !entry.IncludeSubDomains {
		t.Errorf("Ancestor entry does not include subdomains.")
	}

	entry, ok = idx.Get("test.example.com")
	if ok == AncestorEntryFound {
		t.Errorf("Ancestor entry found, but it does not include subdomains.")
	}
	if entry.IncludeSubDomains {
		t.Errorf("Ancestory entry should not include subdomains.")
	}

	entry, ok = idx.Get("foo.bar")
	if ok != AncestorEntryFound {
		t.Errorf("Ancestor entry should be present.")
	}
	if entry.Name != "bar" || entry.Mode != "force-https" {
		t.Errorf("Wrong ancestor entry found.")
	}
	if !entry.IncludeSubDomains {
		t.Errorf("Ancestor entry does not include subdomains.")
	}

	entry, ok = idx.Get("bar")
	if ok != ExactEntryFound {
		t.Errorf("Entry should be present.")
	}
	if entry.Name != "bar" || entry.Mode != "force-https" || !entry.IncludeSubDomains {
		t.Errorf("Wrong entry found.")
	}
}

func TestNewFromLatest(t *testing.T) {
	if testing.Short() {
		t.Skip("Skipping test to avoid preload list download.")
	}

	list, err := NewFromLatest()
	if err != nil {
		t.Errorf("Could not retrieve preload list.")
	}

	firstEntry := list.Entries[0]
	if firstEntry.Name != "pinningtest.appspot.com" {
		t.Errorf("First entry of preload list does not have the expected name.")
	}
}

func TestNewFromChromiumURL(t *testing.T) {
	if testing.Short() {
		t.Skip("Skipping test to avoid preload list download.")
	}

	list, err := NewFromChromiumURL("https://chromium.googlesource.com/chromium/src/+/4f587d7d4532287308715d824d19e7465c9f663e/net/http/transport_security_state_static.json?format=TEXT")
	if err != nil {
		t.Error(err)
	}
	if len(list.Entries) != 3558 {
		t.Errorf("Wrong number of entries: %d", len(list.Entries))
	}
}

var (
	testJSON = `{
  "entries": [
  	// This is a comment.
    {"name": "garron.net", "include_subdomains": true, "mode": "force-https"},
    {"name": "example.com", "include_subdomains": false, "mode": "force-https"},
    {"name": "gmail.com", "mode": "force-https"},

    // Line above intentionally left blank.
    {"name": "google.com"},
    {"name": "pinned.badssl.com", "pins": "pinnymcpinnedkey"}
  ]
}`
	testParsed = PreloadList{Entries: []Entry{
		{"garron.net", "force-https", true},
		{"example.com", "force-https", false},
		{"gmail.com", "force-https", false},
		{"google.com", "", false},
		{"pinned.badssl.com", "", false}},
	}
)

func TestNewFromFile(t *testing.T) {
	f, err := ioutil.TempFile("", "preloadlist-test")
	if err != nil {
		t.Fatal(err)
	}
	defer os.Remove(f.Name())

	if _, err := f.Write([]byte(testJSON)); err != nil {
		t.Fatal(err)
	}

	list, err := NewFromFile(f.Name())
	if err != nil {
		t.Fatalf("Could not read preload list. %s", err)
	}

	if !reflect.DeepEqual(list, testParsed) {
		t.Errorf("Parsed list does not match expected. %#v", list)
	}
}
<|endoftext|>


<commit_before>package report

import (
	"os"

	"github.com/luistm/banksaurus/app"
	"github.com/luistm/banksaurus/infrastructure/csv"
	"github.com/luistm/banksaurus/infrastructure/sqlite"
	"github.com/luistm/banksaurus/lib/seller"
	"github.com/luistm/banksaurus/lib/transaction"
	"github.com/luistm/banksaurus/services"
	"github.com/luistm/banksaurus/services/report"
	"github.com/luistm/banksaurus/services/reportgrouped"
)

// Command handles reports
type Command struct{}

// Execute the report command
func (rc *Command) Execute(arguments map[string]interface{}) error {
	var grouped bool

	if arguments["--grouped"].(bool) {
		grouped = true
	}

	CSVStorage, err := csv.New(arguments["<file>"].(string))
	if err != nil {
		return err
	}
	defer CSVStorage.Close()

	dbName, dbPath := app.DatabasePath()
	SQLStorage, err := sqlite.New(dbPath, dbName, false)
	if err != nil {
		return err
	}
	defer SQLStorage.Close()

	transactionRepository := transaction.NewRepository(CSVStorage, SQLStorage)
	sellersRepository := seller.NewRepository(SQLStorage)
	presenter := NewPresenter(os.Stdout)

	var rfr services.Servicer
	if grouped {
		rfr, err = reportgrouped.New(transactionRepository, sellersRepository, presenter)
	} else {
		rfr, err = report.New(transactionRepository, sellersRepository, presenter)
	}
	if err != nil {
		return err
	}

	if err := rfr.Execute(); err != nil {
		return nil
	}

	return nil
}
<commit_msg>Changes command so it can use the new report version<commit_after>package report

import (
	"os"

	"github.com/luistm/banksaurus/app"
	"github.com/luistm/banksaurus/infrastructure/csv"
	"github.com/luistm/banksaurus/infrastructure/sqlite"
	"github.com/luistm/banksaurus/lib/seller"
	"github.com/luistm/banksaurus/lib/transaction"
	"github.com/luistm/banksaurus/services"
	"github.com/luistm/banksaurus/services/reportgrouped"
)

// Command handles reports
type Command struct{}

// Execute the report command
func (rc *Command) Execute(arguments map[string]interface{}) error {
	var grouped bool

	if arguments["--grouped"].(bool) {
		grouped = true
	}

	if grouped {
		CSVStorage, err := csv.New(arguments["<file>"].(string))
		if err != nil {
			return err
		}
		defer CSVStorage.Close()

		dbName, dbPath := app.DatabasePath()
		SQLStorage, err := sqlite.New(dbPath, dbName, false)
		if err != nil {
			return err
		}
		defer SQLStorage.Close()

		transactionRepository := transaction.NewRepository(CSVStorage, SQLStorage)
		sellersRepository := seller.NewRepository(SQLStorage)
		presenter := NewPresenter(os.Stdout)

		var rfr services.Servicer

		rfr, err = reportgrouped.New(transactionRepository, sellersRepository, presenter)
		if err != nil {
			return err
		}

		if err := rfr.Execute(); err != nil {
			return nil
		}

	} else {
		// rfr, err = report.New(transactionRepository, sellersRepository, presenter)
	}

	return nil
}
<|endoftext|>