<commit_before>package google

import (
	"encoding/json"
	"fmt"
	"strings"

	"github.com/hashicorp/terraform/helper/schema"
	"github.com/hashicorp/terraform/terraform"
	"google.golang.org/api/compute/v1"
	"google.golang.org/api/googleapi"
)

// Provider returns a terraform.ResourceProvider.
func Provider() terraform.ResourceProvider {
	return &schema.Provider{
		Schema: map[string]*schema.Schema{
			"account_file": &schema.Schema{
				Type:        schema.TypeString,
				Optional:    true,
				DefaultFunc: schema.EnvDefaultFunc("GOOGLE_ACCOUNT_FILE", nil),
				Removed:     "Use the credentials field instead",
			},

			"credentials": &schema.Schema{
				Type:     schema.TypeString,
				Optional: true,
				DefaultFunc: schema.MultiEnvDefaultFunc([]string{
					"GOOGLE_CREDENTIALS",
					"GOOGLE_CLOUD_KEYFILE_JSON",
					"GCLOUD_KEYFILE_JSON",
				}, nil),
				ValidateFunc: validateCredentials,
			},

			"project": &schema.Schema{
				Type:     schema.TypeString,
				Optional: true,
				DefaultFunc: schema.MultiEnvDefaultFunc([]string{
					"GOOGLE_PROJECT",
					"GCLOUD_PROJECT",
					"CLOUDSDK_CORE_PROJECT",
				}, nil),
			},

			"region": &schema.Schema{
				Type:     schema.TypeString,
				Required: true,
				DefaultFunc: schema.MultiEnvDefaultFunc([]string{
					"GOOGLE_REGION",
					"GCLOUD_REGION",
					"CLOUDSDK_COMPUTE_REGION",
				}, nil),
			},
		},

		DataSourcesMap: map[string]*schema.Resource{
			"google_iam_policy":    dataSourceGoogleIamPolicy(),
			"google_compute_zones": dataSourceGoogleComputeZones(),
		},

		ResourcesMap: map[string]*schema.Resource{
			"google_compute_autoscaler":             resourceComputeAutoscaler(),
			"google_compute_address":                resourceComputeAddress(),
			"google_compute_backend_service":        resourceComputeBackendService(),
			"google_compute_disk":                   resourceComputeDisk(),
			"google_compute_firewall":               resourceComputeFirewall(),
			"google_compute_forwarding_rule":        resourceComputeForwardingRule(),
			"google_compute_global_address":         resourceComputeGlobalAddress(),
			"google_compute_global_forwarding_rule": resourceComputeGlobalForwardingRule(),
			"google_compute_health_check":           resourceComputeHealthCheck(),
			"google_compute_http_health_check":      resourceComputeHttpHealthCheck(),
			"google_compute_https_health_check":     resourceComputeHttpsHealthCheck(),
			"google_compute_image":                  resourceComputeImage(),
			"google_compute_instance":               resourceComputeInstance(),
			"google_compute_instance_group":         resourceComputeInstanceGroup(),
			"google_compute_instance_group_manager": resourceComputeInstanceGroupManager(),
			"google_compute_instance_template":      resourceComputeInstanceTemplate(),
			"google_compute_network":                resourceComputeNetwork(),
			"google_compute_project_metadata":       resourceComputeProjectMetadata(),
			"google_compute_region_backend_service": resourceComputeRegionBackendService(),
			"google_compute_route":                  resourceComputeRoute(),
			"google_compute_ssl_certificate":        resourceComputeSslCertificate(),
			"google_compute_subnetwork":             resourceComputeSubnetwork(),
			"google_compute_target_http_proxy":      resourceComputeTargetHttpProxy(),
			"google_compute_target_https_proxy":     resourceComputeTargetHttpsProxy(),
			"google_compute_target_pool":            resourceComputeTargetPool(),
			"google_compute_url_map":                resourceComputeUrlMap(),
			"google_compute_vpn_gateway":            resourceComputeVpnGateway(),
			"google_compute_vpn_tunnel":             resourceComputeVpnTunnel(),
			"google_container_cluster":              resourceContainerCluster(),
			"google_container_node_pool":            resourceContainerNodePool(),
			"google_dns_managed_zone":               resourceDnsManagedZone(),
			"google_dns_record_set":                 resourceDnsRecordSet(),
			"google_sql_database":                   resourceSqlDatabase(),
			"google_sql_database_instance":          resourceSqlDatabaseInstance(),
			"google_sql_user":                       resourceSqlUser(),
			"google_project":                        resourceGoogleProject(),
			"google_project_iam_policy":             resourceGoogleProjectIamPolicy(),
			"google_project_services":               resourceGoogleProjectServices(),
			"google_pubsub_topic":                   resourcePubsubTopic(),
			"google_pubsub_subscription":            resourcePubsubSubscription(),
			"google_service_account":                resourceGoogleServiceAccount(),
			"google_storage_bucket":                 resourceStorageBucket(),
			"google_storage_bucket_acl":             resourceStorageBucketAcl(),
			"google_storage_bucket_object":          resourceStorageBucketObject(),
			"google_storage_object_acl":             resourceStorageObjectAcl(),
		},

		ConfigureFunc: providerConfigure,
	}
}

func providerConfigure(d *schema.ResourceData) (interface{}, error) {
	credentials := d.Get("credentials").(string)
	config := Config{
		Credentials: credentials,
		Project:     d.Get("project").(string),
		Region:      d.Get("region").(string),
	}

	if err := config.loadAndValidate(); err != nil {
		return nil, err
	}

	return &config, nil
}

func validateCredentials(v interface{}, k string) (warnings []string, errors []error) {
	if v == nil || v.(string) == "" {
		return
	}
	creds := v.(string)
	var account accountFile
	if err := json.Unmarshal([]byte(creds), &account); err != nil {
		errors = append(errors,
			fmt.Errorf("credentials are not valid JSON '%s': %s", creds, err))
	}

	return
}

// getRegionFromZone returns the region from a zone for Google cloud.
func getRegionFromZone(zone string) string {
	if zone != "" && len(zone) > 2 {
		region := zone[:len(zone)-2]
		return region
	}
	return ""
}

// getRegion reads the "region" field from the given resource data and falls
// back to the provider's value if not given. If the provider's value is not
// given, an error is returned.
func getRegion(d *schema.ResourceData, config *Config) (string, error) {
	res, ok := d.GetOk("region")
	if !ok {
		if config.Region != "" {
			return config.Region, nil
		}
		return "", fmt.Errorf("%q: required field is not set", "region")
	}
	return res.(string), nil
}

// getProject reads the "project" field from the given resource data and falls
// back to the provider's value if not given. If the provider's value is not
// given, an error is returned.
func getProject(d *schema.ResourceData, config *Config) (string, error) {
	res, ok := d.GetOk("project")
	if !ok {
		if config.Project != "" {
			return config.Project, nil
		}
		return "", fmt.Errorf("%q: required field is not set", "project")
	}
	return res.(string), nil
}

func getZonalResourceFromRegion(getResource func(string) (interface{}, error), region string, compute *compute.Service, project string) (interface{}, error) {
	zoneList, err := compute.Zones.List(project).Do()
	if err != nil {
		return nil, err
	}
	var resource interface{}
	for _, zone := range zoneList.Items {
		if strings.Contains(zone.Name, region) {
			resource, err = getResource(zone.Name)
			if err != nil {
				if gerr, ok := err.(*googleapi.Error); ok && gerr.Code == 404 {
					// Resource was not found in this zone
					continue
				}
				return nil, fmt.Errorf("Error reading Resource: %s", err)
			}
			// Resource was found
			return resource, nil
		}
	}
	// Resource does not exist in this region
	return nil, nil
}

// getNetworkLink reads the "network" field from the given resource data and if the value:
// - is a resource URL, returns the string unchanged
// - is the network name only, then looks up the resource URL using the google client
func getNetworkLink(d *schema.ResourceData, config *Config, field string) (string, error) {
	if v, ok := d.GetOk(field); ok {
		network := v.(string)

		project, err := getProject(d, config)
		if err != nil {
			return "", err
		}

		if !strings.HasPrefix(network, "https://www.googleapis.com/compute/") {
			// Network value provided is just the name, lookup the network SelfLink
			networkData, err := config.clientCompute.Networks.Get(
				project, network).Do()
			if err != nil {
				return "", fmt.Errorf("Error reading network: %s", err)
			}
			network = networkData.SelfLink
		}

		return network, nil

	} else {
		return "", nil
	}
}

// getNetworkName reads the "network" field from the given resource data and if the value:
// - is a resource URL, extracts the network name from the URL and returns it
// - is the network name only (i.e not prefixed with http://www.googleapis.com/compute/...), is returned unchanged
func getNetworkName(d *schema.ResourceData, field string) (string, error) {
	if v, ok := d.GetOk(field); ok {
		network := v.(string)
		return getNetworkNameFromSelfLink(network)
	}
	return "", nil
}

func getNetworkNameFromSelfLink(network string) (string, error) {
	if strings.HasPrefix(network, "https://www.googleapis.com/compute/") {
		// extract the network name from SelfLink URL
		networkName := network[strings.LastIndex(network, "/")+1:]
		if networkName == "" {
			return "", fmt.Errorf("network url not valid")
		}
		return networkName, nil
	}

	return network, nil
}
<commit_msg>provider/google: drop the account file.<commit_after>package google

import (
	"encoding/json"
	"fmt"
	"strings"

	"github.com/hashicorp/terraform/helper/schema"
	"github.com/hashicorp/terraform/terraform"
	"google.golang.org/api/compute/v1"
	"google.golang.org/api/googleapi"
)

// Provider returns a terraform.ResourceProvider.
func Provider() terraform.ResourceProvider {
	return &schema.Provider{
		Schema: map[string]*schema.Schema{
			"credentials": &schema.Schema{
				Type:     schema.TypeString,
				Optional: true,
				DefaultFunc: schema.MultiEnvDefaultFunc([]string{
					"GOOGLE_CREDENTIALS",
					"GOOGLE_CLOUD_KEYFILE_JSON",
					"GCLOUD_KEYFILE_JSON",
				}, nil),
				ValidateFunc: validateCredentials,
			},

			"project": &schema.Schema{
				Type:     schema.TypeString,
				Optional: true,
				DefaultFunc: schema.MultiEnvDefaultFunc([]string{
					"GOOGLE_PROJECT",
					"GCLOUD_PROJECT",
					"CLOUDSDK_CORE_PROJECT",
				}, nil),
			},

			"region": &schema.Schema{
				Type:     schema.TypeString,
				Required: true,
				DefaultFunc: schema.MultiEnvDefaultFunc([]string{
					"GOOGLE_REGION",
					"GCLOUD_REGION",
					"CLOUDSDK_COMPUTE_REGION",
				}, nil),
			},
		},

		DataSourcesMap: map[string]*schema.Resource{
			"google_iam_policy":    dataSourceGoogleIamPolicy(),
			"google_compute_zones": dataSourceGoogleComputeZones(),
		},

		ResourcesMap: map[string]*schema.Resource{
			"google_compute_autoscaler":             resourceComputeAutoscaler(),
			"google_compute_address":                resourceComputeAddress(),
			"google_compute_backend_service":        resourceComputeBackendService(),
			"google_compute_disk":                   resourceComputeDisk(),
			"google_compute_firewall":               resourceComputeFirewall(),
			"google_compute_forwarding_rule":        resourceComputeForwardingRule(),
			"google_compute_global_address":         resourceComputeGlobalAddress(),
			"google_compute_global_forwarding_rule": resourceComputeGlobalForwardingRule(),
			"google_compute_health_check":           resourceComputeHealthCheck(),
			"google_compute_http_health_check":      resourceComputeHttpHealthCheck(),
			"google_compute_https_health_check":     resourceComputeHttpsHealthCheck(),
			"google_compute_image":                  resourceComputeImage(),
			"google_compute_instance":               resourceComputeInstance(),
			"google_compute_instance_group":         resourceComputeInstanceGroup(),
			"google_compute_instance_group_manager": resourceComputeInstanceGroupManager(),
			"google_compute_instance_template":      resourceComputeInstanceTemplate(),
			"google_compute_network":                resourceComputeNetwork(),
			"google_compute_project_metadata":       resourceComputeProjectMetadata(),
			"google_compute_region_backend_service": resourceComputeRegionBackendService(),
			"google_compute_route":                  resourceComputeRoute(),
			"google_compute_ssl_certificate":        resourceComputeSslCertificate(),
			"google_compute_subnetwork":             resourceComputeSubnetwork(),
			"google_compute_target_http_proxy":      resourceComputeTargetHttpProxy(),
			"google_compute_target_https_proxy":     resourceComputeTargetHttpsProxy(),
			"google_compute_target_pool":            resourceComputeTargetPool(),
			"google_compute_url_map":                resourceComputeUrlMap(),
			"google_compute_vpn_gateway":            resourceComputeVpnGateway(),
			"google_compute_vpn_tunnel":             resourceComputeVpnTunnel(),
			"google_container_cluster":              resourceContainerCluster(),
			"google_container_node_pool":            resourceContainerNodePool(),
			"google_dns_managed_zone":               resourceDnsManagedZone(),
			"google_dns_record_set":                 resourceDnsRecordSet(),
			"google_sql_database":                   resourceSqlDatabase(),
			"google_sql_database_instance":          resourceSqlDatabaseInstance(),
			"google_sql_user":                       resourceSqlUser(),
			"google_project":                        resourceGoogleProject(),
			"google_project_iam_policy":             resourceGoogleProjectIamPolicy(),
			"google_project_services":               resourceGoogleProjectServices(),
			"google_pubsub_topic":                   resourcePubsubTopic(),
			"google_pubsub_subscription":            resourcePubsubSubscription(),
			"google_service_account":                resourceGoogleServiceAccount(),
			"google_storage_bucket":                 resourceStorageBucket(),
			"google_storage_bucket_acl":             resourceStorageBucketAcl(),
			"google_storage_bucket_object":          resourceStorageBucketObject(),
			"google_storage_object_acl":             resourceStorageObjectAcl(),
		},

		ConfigureFunc: providerConfigure,
	}
}

func providerConfigure(d *schema.ResourceData) (interface{}, error) {
	credentials := d.Get("credentials").(string)
	config := Config{
		Credentials: credentials,
		Project:     d.Get("project").(string),
		Region:      d.Get("region").(string),
	}

	if err := config.loadAndValidate(); err != nil {
		return nil, err
	}

	return &config, nil
}

func validateCredentials(v interface{}, k string) (warnings []string, errors []error) {
	if v == nil || v.(string) == "" {
		return
	}
	creds := v.(string)
	var account accountFile
	if err := json.Unmarshal([]byte(creds), &account); err != nil {
		errors = append(errors,
			fmt.Errorf("credentials are not valid JSON '%s': %s", creds, err))
	}

	return
}

// getRegionFromZone returns the region from a zone for Google cloud.
func getRegionFromZone(zone string) string {
	if zone != "" && len(zone) > 2 {
		region := zone[:len(zone)-2]
		return region
	}
	return ""
}

// getRegion reads the "region" field from the given resource data and falls
// back to the provider's value if not given. If the provider's value is not
// given, an error is returned.
func getRegion(d *schema.ResourceData, config *Config) (string, error) {
	res, ok := d.GetOk("region")
	if !ok {
		if config.Region != "" {
			return config.Region, nil
		}
		return "", fmt.Errorf("%q: required field is not set", "region")
	}
	return res.(string), nil
}

// getProject reads the "project" field from the given resource data and falls
// back to the provider's value if not given. If the provider's value is not
// given, an error is returned.
func getProject(d *schema.ResourceData, config *Config) (string, error) {
	res, ok := d.GetOk("project")
	if !ok {
		if config.Project != "" {
			return config.Project, nil
		}
		return "", fmt.Errorf("%q: required field is not set", "project")
	}
	return res.(string), nil
}

func getZonalResourceFromRegion(getResource func(string) (interface{}, error), region string, compute *compute.Service, project string) (interface{}, error) {
	zoneList, err := compute.Zones.List(project).Do()
	if err != nil {
		return nil, err
	}
	var resource interface{}
	for _, zone := range zoneList.Items {
		if strings.Contains(zone.Name, region) {
			resource, err = getResource(zone.Name)
			if err != nil {
				if gerr, ok := err.(*googleapi.Error); ok && gerr.Code == 404 {
					// Resource was not found in this zone
					continue
				}
				return nil, fmt.Errorf("Error reading Resource: %s", err)
			}
			// Resource was found
			return resource, nil
		}
	}
	// Resource does not exist in this region
	return nil, nil
}

// getNetworkLink reads the "network" field from the given resource data and if the value:
// - is a resource URL, returns the string unchanged
// - is the network name only, then looks up the resource URL using the google client
func getNetworkLink(d *schema.ResourceData, config *Config, field string) (string, error) {
	if v, ok := d.GetOk(field); ok {
		network := v.(string)

		project, err := getProject(d, config)
		if err != nil {
			return "", err
		}

		if !strings.HasPrefix(network, "https://www.googleapis.com/compute/") {
			// Network value provided is just the name, lookup the network SelfLink
			networkData, err := config.clientCompute.Networks.Get(
				project, network).Do()
			if err != nil {
				return "", fmt.Errorf("Error reading network: %s", err)
			}
			network = networkData.SelfLink
		}

		return network, nil

	} else {
		return "", nil
	}
}

// getNetworkName reads the "network" field from the given resource data and if the value:
// - is a resource URL, extracts the network name from the URL and returns it
// - is the network name only (i.e not prefixed with http://www.googleapis.com/compute/...), is returned unchanged
func getNetworkName(d *schema.ResourceData, field string) (string, error) {
	if v, ok := d.GetOk(field); ok {
		network := v.(string)
		return getNetworkNameFromSelfLink(network)
	}
	return "", nil
}

func getNetworkNameFromSelfLink(network string) (string, error) {
	if strings.HasPrefix(network, "https://www.googleapis.com/compute/") {
		// extract the network name from SelfLink URL
		networkName := network[strings.LastIndex(network, "/")+1:]
		if networkName == "" {
			return "", fmt.Errorf("network url not valid")
		}
		return networkName, nil
	}

	return network, nil
}
<|endoftext|>


<commit_before><commit_msg>type is deprecated in es7<commit_after><|endoftext|>


<commit_before><commit_msg>Add /archer location for statistics, count how many requests were processed.<commit_after><|endoftext|>


<commit_before>package sdees

import (
	"encoding/json"
	"errors"
	"fmt"
	"io/ioutil"
	"log"
	"os"
	"os/exec"
	"path"
	"strings"
	"time"
)

type Config struct {
	Remote, Editor, CurrentDocument string
}

func SetupConfig() {
	var configParameters Config

	var yesno string
	err := errors.New("Incorrect remote")
	for {
		fmt.Print("Enter remote (e.g.: git@github.com:USER/REPO.git): ")
		fmt.Scanln(&yesno)
		cwd, _ := os.Getwd()
		os.Chdir(CachePath)
		if !exists(EncodeBase58(yesno)) {
			fmt.Println("Cloning " + yesno + " ...")
			cmd := exec.Command("git", "clone", yesno, EncodeBase58(yesno))
			out2, _ := cmd.StderrPipe()
			cmd.Start()
			out2b, _ := ioutil.ReadAll(out2)
			cmd.Wait()
			os.Chdir(cwd)
			if strings.Contains(string(out2b), "fatal: ") {
				fmt.Println(strings.TrimSpace(string(out2b)))
				fmt.Println("Could not clone, please re-enter")
			} else {
				break
			}
		} else {
			fmt.Println("Already exists, doing nothing.")
			break
		}
	}
	configParameters.Remote = yesno

	// Loop until user chooses an available program
	for {
		fmt.Printf("Which editor do you want to use: micro (default), vim,  nano, or emacs? ")
		fmt.Scanln(&yesno)
		if strings.TrimSpace(strings.ToLower(yesno)) == "nano" {
			configParameters.Editor = "nano"
		} else if strings.TrimSpace(strings.ToLower(yesno)) == "emacs" {
			configParameters.Editor = "emacs"
		} else if strings.TrimSpace(strings.ToLower(yesno)) == "vim" {
			configParameters.Editor = "vim"
		} else {
			configParameters.Editor = "micro"
		}
		if Version != "dev" && configParameters.Editor == "micro" {
			break
		}
		// check if it actually exists
		cmd := exec.Command(path.Join(ProgramPath, configParameters.Editor+Extension), "--version")
		_, err := cmd.Output()
		if err == nil {
			break
		}
		cmd = exec.Command(path.Join(".", configParameters.Editor+Extension), "--version")
		_, err = cmd.Output()
		if err == nil {
			break
		}
		fmt.Printf("\n%s not found, are you sure its installed?\n\n", configParameters.Editor+Extension)
	}

	configParameters.CurrentDocument = ""

	b, err := json.Marshal(configParameters)
	if err != nil {
		log.Println(err)
	}
	ioutil.WriteFile(path.Join(ConfigPath, "config.json"), b, 0644)
}

func LoadConfiguration() {
	defer timeTrack(time.Now(), "Loaded and saved configuration")
	var c Config
	data, err := ioutil.ReadFile(path.Join(ConfigPath, "config.json"))
	if err != nil {
		logger.Error("Could not load config.json")
		return
	}
	json.Unmarshal(data, &c)
	if len(CurrentDocument) == 0 {
		CurrentDocument = c.CurrentDocument
	}
	Editor = c.Editor
	Remote = c.Remote
	RemoteFolder = path.Join(CachePath, HashString(Remote))
	if len(Remote) == 0 {
		SetupConfig()
	}
}

func SaveConfiguration(editor string, remote string, currentdoc string) {
	defer timeTrack(time.Now(), "Saved configuration")
	c := Config{Editor: editor, Remote: remote, CurrentDocument: currentdoc}
	b, err := json.Marshal(c)
	if err != nil {
		log.Println(err)
	}
	ioutil.WriteFile(path.Join(ConfigPath, "config.json"), b, 0644)
}
<commit_msg>Just use hashstring<commit_after>package sdees

import (
	"encoding/json"
	"errors"
	"fmt"
	"io/ioutil"
	"log"
	"os"
	"os/exec"
	"path"
	"strings"
	"time"
)

type Config struct {
	Remote, Editor, CurrentDocument string
}

func SetupConfig() {
	var configParameters Config

	var yesno string
	err := errors.New("Incorrect remote")
	for {
		fmt.Print("Enter remote (e.g.: git@github.com:USER/REPO.git): ")
		fmt.Scanln(&yesno)
		cwd, _ := os.Getwd()
		os.Chdir(CachePath)
		if !exists(HashString(yesno)) {
			fmt.Println("Cloning " + yesno + " ...")
			cmd := exec.Command("git", "clone", yesno, HashString(yesno))
			out2, _ := cmd.StderrPipe()
			cmd.Start()
			out2b, _ := ioutil.ReadAll(out2)
			cmd.Wait()
			os.Chdir(cwd)
			if strings.Contains(string(out2b), "fatal: ") {
				fmt.Println(strings.TrimSpace(string(out2b)))
				fmt.Println("Could not clone, please re-enter")
			} else {
				break
			}
		} else {
			fmt.Println("Already exists, doing nothing.")
			break
		}
	}
	configParameters.Remote = yesno

	// Loop until user chooses an available program
	for {
		fmt.Printf("Which editor do you want to use: micro (default), vim,  nano, or emacs? ")
		fmt.Scanln(&yesno)
		if strings.TrimSpace(strings.ToLower(yesno)) == "nano" {
			configParameters.Editor = "nano"
		} else if strings.TrimSpace(strings.ToLower(yesno)) == "emacs" {
			configParameters.Editor = "emacs"
		} else if strings.TrimSpace(strings.ToLower(yesno)) == "vim" {
			configParameters.Editor = "vim"
		} else {
			configParameters.Editor = "micro"
		}
		if Version != "dev" && configParameters.Editor == "micro" {
			break
		}
		// check if it actually exists
		cmd := exec.Command(path.Join(ProgramPath, configParameters.Editor+Extension), "--version")
		_, err := cmd.Output()
		if err == nil {
			break
		}
		cmd = exec.Command(path.Join(".", configParameters.Editor+Extension), "--version")
		_, err = cmd.Output()
		if err == nil {
			break
		}
		fmt.Printf("\n%s not found, are you sure its installed?\n\n", configParameters.Editor+Extension)
	}

	configParameters.CurrentDocument = ""

	b, err := json.Marshal(configParameters)
	if err != nil {
		log.Println(err)
	}
	ioutil.WriteFile(path.Join(ConfigPath, "config.json"), b, 0644)
}

func LoadConfiguration() {
	defer timeTrack(time.Now(), "Loaded and saved configuration")
	var c Config
	data, err := ioutil.ReadFile(path.Join(ConfigPath, "config.json"))
	if err != nil {
		logger.Error("Could not load config.json")
		return
	}
	json.Unmarshal(data, &c)
	if len(CurrentDocument) == 0 {
		CurrentDocument = c.CurrentDocument
	}
	Editor = c.Editor
	Remote = c.Remote
	RemoteFolder = path.Join(CachePath, HashString(Remote))
	if len(Remote) == 0 {
		SetupConfig()
	}
}

func SaveConfiguration(editor string, remote string, currentdoc string) {
	defer timeTrack(time.Now(), "Saved configuration")
	c := Config{Editor: editor, Remote: remote, CurrentDocument: currentdoc}
	b, err := json.Marshal(c)
	if err != nil {
		log.Println(err)
	}
	ioutil.WriteFile(path.Join(ConfigPath, "config.json"), b, 0644)
}
<|endoftext|>


<commit_before>package bird

// Birdwatcher Birdc Configuration

type StatusConfig struct {
	ReconfigTimestampSource string `toml:"reconfig_timestamp_source"`
	ReconfigTimestampMatch  string `toml:"reconfig_timestamp_match"`

	FilterFields []string `toml:"filter_fields"`
}

type BirdConfig struct {
	Listen         string
	ConfigFilename string `toml:"config"`
	BirdCmd        string `toml:"birdc"`
}

type RateLimitConfig struct {
	Reqs    int `toml:"requests_per_minute"`
	Max     int `toml:"requests_per_minute"`
	Enabled bool
}
<commit_msg>only use key once<commit_after>package bird

// Birdwatcher Birdc Configuration

type StatusConfig struct {
	ReconfigTimestampSource string `toml:"reconfig_timestamp_source"`
	ReconfigTimestampMatch  string `toml:"reconfig_timestamp_match"`

	FilterFields []string `toml:"filter_fields"`
}

type BirdConfig struct {
	Listen         string
	ConfigFilename string `toml:"config"`
	BirdCmd        string `toml:"birdc"`
}

type RateLimitConfig struct {
	Reqs    int
	Max     int `toml:"requests_per_minute"`
	Enabled bool
}
<|endoftext|>


<commit_before>// Copyright 2017 Mathew Robinson <mrobinson@praelatus.io>. All rights reserved.
// Use of this source code is governed by the AGPLv3 license that can be found in
// the LICENSE file.

// Package lexer contains the lexer for PQL
package lexer

import (
	"github.com/praelatus/praelatus/ql/token"
)

// Lexer maintains document position and lexing the input
type Lexer struct {
	input        string
	position     int
	readPosition int
	ch           byte
}

// New create a new lexer for the given input
func New(input string) *Lexer {
	l := &Lexer{input: input}
	l.readChar()
	return l
}

func (l *Lexer) readChar() {
	if l.readPosition >= len(l.input) {
		l.ch = 0
	} else {
		l.ch = l.input[l.readPosition]
	}

	l.position = l.readPosition
	l.readPosition++
}

func (l *Lexer) peekChar() byte {
	if l.readPosition >= len(l.input) {
		return 0
	}

	return l.input[l.readPosition]
}

type validator func(byte) bool

func (l *Lexer) read(valid validator) string {
	start := l.position

	for valid(l.ch) {
		l.readChar()
	}

	return l.input[start:l.position]
}

func (l *Lexer) skipWhitespace() {
	for isWhitespace(l.ch) {
		l.readChar()
	}
}

func (l *Lexer) NextToken() token.Token {
	var tok token.Token

	l.skipWhitespace()

	switch l.ch {
	case '=':
		tok = newToken(token.EQ, l.ch)
	case '<':
		if l.peekChar() == '=' {
			tok = token.Token{token.LTE, "<="}
			break
		}

		tok = newToken(token.LT, l.ch)
	case '>':
		if l.peekChar() == '=' {
			tok = token.Token{token.GTE, ">="}
			break
		}

		tok = newToken(token.GT, l.ch)
	case '!':
		if l.peekChar() == '=' {
			l.readChar()
			tok = token.Token{token.NE, "!="}
			break
		}

		tok = newToken(token.ILLEGAL, l.ch)
	case ',':
		tok = newToken(token.COMMA, l.ch)
	case '(':
		tok = newToken(token.LPAREN, l.ch)
	case ')':
		tok = newToken(token.RPAREN, l.ch)
	case '~':
		tok = newToken(token.LIKE, l.ch)
	case 0:
		tok = token.Token{token.EOF, ""}
	default:
		if isDigit(l.ch) {
			tok.Type = token.INT
			tok.Literal = l.read(isDigit)
			return tok
		} else if isLetter(l.ch) {
			tok.Literal = l.read(isLetter)
			tok.Type = token.LookupIdent(tok.Literal)
			return tok
		} else if l.ch == '"' {
			// Skip opening quote
			l.readChar()
			tok.Type = token.STRING

			// When inside double quotes numbers and spaces are allows so treat them as such
			tok.Literal = l.read(func(ch byte) bool {
				return isLetter(ch) || ch == ' ' || isDigit(ch)
			})

			// Skip closing quote
			l.readChar()
			return tok
		}

		tok = newToken(token.ILLEGAL, l.ch)
	}

	l.readChar()
	return tok
}

func isWhitespace(ch byte) bool {
	return ' ' == ch || '\n' == ch || '\t' == ch
}

func isLetter(ch byte) bool {
	return 'a' <= ch && ch <= 'z' || 'A' <= ch && ch <= 'z' || ch == '_' || ch == '-'
}

func isDigit(ch byte) bool {
	return '0' <= ch && ch <= '9'
}

func newToken(tt token.TokenType, ch byte) token.Token {
	return token.Token{Type: tt, Literal: string(ch)}
}
<commit_msg>Add , to isLetter<commit_after>// Copyright 2017 Mathew Robinson <mrobinson@praelatus.io>. All rights reserved.
// Use of this source code is governed by the AGPLv3 license that can be found in
// the LICENSE file.

// Package lexer contains the lexer for PQL
package lexer

import (
	"github.com/praelatus/praelatus/ql/token"
)

// Lexer maintains document position and lexing the input
type Lexer struct {
	input        string
	position     int
	readPosition int
	ch           byte
}

// New create a new lexer for the given input
func New(input string) *Lexer {
	l := &Lexer{input: input}
	l.readChar()
	return l
}

func (l *Lexer) readChar() {
	if l.readPosition >= len(l.input) {
		l.ch = 0
	} else {
		l.ch = l.input[l.readPosition]
	}

	l.position = l.readPosition
	l.readPosition++
}

func (l *Lexer) peekChar() byte {
	if l.readPosition >= len(l.input) {
		return 0
	}

	return l.input[l.readPosition]
}

type validator func(byte) bool

func (l *Lexer) read(valid validator) string {
	start := l.position

	for valid(l.ch) {
		l.readChar()
	}

	return l.input[start:l.position]
}

func (l *Lexer) skipWhitespace() {
	for isWhitespace(l.ch) {
		l.readChar()
	}
}

func (l *Lexer) NextToken() token.Token {
	var tok token.Token

	l.skipWhitespace()

	switch l.ch {
	case '=':
		tok = newToken(token.EQ, l.ch)
	case '<':
		if l.peekChar() == '=' {
			tok = token.Token{token.LTE, "<="}
			break
		}

		tok = newToken(token.LT, l.ch)
	case '>':
		if l.peekChar() == '=' {
			tok = token.Token{token.GTE, ">="}
			break
		}

		tok = newToken(token.GT, l.ch)
	case '!':
		if l.peekChar() == '=' {
			l.readChar()
			tok = token.Token{token.NE, "!="}
			break
		}

		tok = newToken(token.ILLEGAL, l.ch)
	case ',':
		tok = newToken(token.COMMA, l.ch)
	case '(':
		tok = newToken(token.LPAREN, l.ch)
	case ')':
		tok = newToken(token.RPAREN, l.ch)
	case '~':
		tok = newToken(token.LIKE, l.ch)
	case 0:
		tok = token.Token{token.EOF, ""}
	default:
		if isDigit(l.ch) {
			tok.Type = token.INT
			tok.Literal = l.read(isDigit)
			return tok
		} else if isLetter(l.ch) {
			tok.Literal = l.read(isLetter)
			tok.Type = token.LookupIdent(tok.Literal)
			return tok
		} else if l.ch == '"' {
			// Skip opening quote
			l.readChar()
			tok.Type = token.STRING

			// When inside double quotes numbers and spaces are allows so treat them as such
			tok.Literal = l.read(func(ch byte) bool {
				return isLetter(ch) || ch == ' ' || isDigit(ch)
			})

			// Skip closing quote
			l.readChar()
			return tok
		}

		tok = newToken(token.ILLEGAL, l.ch)
	}

	l.readChar()
	return tok
}

func isWhitespace(ch byte) bool {
	return ' ' == ch || '\n' == ch || '\t' == ch
}

func isLetter(ch byte) bool {
	return 'a' <= ch && ch <= 'z' || 'A' <= ch && ch <= 'z' || ch == '_' || ch == '-' || ch == ','
}

func isDigit(ch byte) bool {
	return '0' <= ch && ch <= '9'
}

func newToken(tt token.TokenType, ch byte) token.Token {
	return token.Token{Type: tt, Literal: string(ch)}
}
<|endoftext|>


<commit_before>package config_service

import (
	"strings"
	cfgsvc "github.com/Flipkart/config-service/client-go"
	"github.com/kelseyhightower/confd/log"
	"errors"
	"reflect"
	"github.com/pquerna/ffjson/ffjson"
	"strconv"
)

// Client provides a wrapper around the zookeeper client
type Client struct {
	client *cfgsvc.ConfigServiceClient
}

type BucketListener struct{
	watchResp chan *watchResponse
	currentIndex uint64
}

type watchResponse struct {
	waitIndex uint64
	err       error
}

func (this *BucketListener) Connected(bucketName string) {
	log.Info("Connected! " + bucketName)
}

func (this *BucketListener) Disconnected(bucketName string, err error) {
	log.Info("Disconnected! " + bucketName)
}

func (this *BucketListener) Deleted(bucketName string) {
	log.Info("deleted " + bucketName)
	this.watchResp <- &watchResponse{waitIndex: 0, err: errors.New(bucketName + " was deleted")}
}

func (this *BucketListener) Updated(oldBucket *cfgsvc.Bucket, newBucket *cfgsvc.Bucket) {
	this.watchResp <- &watchResponse{waitIndex:this.currentIndex+1, err: nil}
}

func NewConfigClient(machines []string) (*Client, error) {
	c, err := cfgsvc.NewConfigServiceClient(50) //*10)
	if err != nil {
		panic(err)
	}
	return &Client{c}, nil
}

func (c *Client) GetValues(keys []string) (map[string]string, error) {
	vars := make(map[string]string)
	for _, v := range keys {
		bucketsKey := strings.Split(strings.TrimPrefix(v, "/"), "/")
		buckets := strings.Split(bucketsKey[0], ",")
		var key string
		if len(bucketsKey) >= 2 {
			key = bucketsKey[1]
		}

		dynamicBuckets, err := c.getDynamicBuckets(buckets)
		if err != nil {
			return vars, err
		}

		for _, dynamicBucket := range dynamicBuckets {
			var requestedKeys []string

			if key == "*" {
				//when key is "*" get all keys in a bucket,
				bucketKeys := dynamicBucket.GetKeys()
				requestedKeys = []string{}
				for k, _ := range bucketKeys {
					requestedKeys = append(requestedKeys, k)
				}
			} else {
				requestedKeys = []string{key}
			}

			//For each key in bucket store value in variable named 'vars'
			for _, k := range requestedKeys {

				val := dynamicBucket.GetKeys()[k]
				if val == nil {
					continue
				}

				valType := reflect.TypeOf(val).Kind()
				if valType == reflect.Slice {
					data, err := ffjson.Marshal(val)
					if err != nil {
						log.Error("Failed decoding from JSON")
					} else {
						vars[k] = string(data[:])
					}
				} else {
					switch val.(type) {
					case int, int64:
						vars[k] = strconv.FormatInt(val.(int64), 64)
					case string:
						vars[k] = val.(string)
					case bool:
						vars[k] = strconv.FormatBool(val.(bool))
					case float32, float64:
						vars[k] = strconv.FormatFloat(val.(float64), 'f', -1, 64)
					}
				}
			}
		}

	}
	return vars, nil
}

func (c *Client) getDynamicBuckets(buckets []string) ([]*cfgsvc.DynamicBucket, error) {
	var dynamicBuckets []*cfgsvc.DynamicBucket
	for _, bucket := range buckets {
		bucketName := strings.TrimSpace(bucket)
		dynamicBucket, err := c.client.GetDynamicBucket(bucketName)
		if err != nil {
			return dynamicBuckets, err
		}
		dynamicBuckets = append(dynamicBuckets, dynamicBucket)
	}
	return dynamicBuckets, nil
}

func setupDynamicBucketListeners(dynamicBuckets []*cfgsvc.DynamicBucket, bucketListener *BucketListener) {
	for _, dynamicBucket := range dynamicBuckets {
		dynamicBucket.AddListeners(bucketListener)
	}
}

func removeDynamicBucketListeners(dynamicBuckets []*cfgsvc.DynamicBucket, bucketListener *BucketListener) {
	for _, dynamicBucket := range dynamicBuckets {
		dynamicBucket.RemoveListeners(bucketListener)
	}
}

func (c *Client) WatchPrefix(prefix string, waitIndex uint64, stopChan chan bool) (uint64, error) {
	prefix = strings.TrimPrefix(prefix, "/")
	prefixes := strings.Split(prefix, ",")
	dynamicBuckets, err := c.getDynamicBuckets(prefixes)
	if err != nil {
		return waitIndex, err
	}

	if waitIndex == 0 {
		return waitIndex+1, nil
	}  else {
		watchResp := make(chan *watchResponse, len(dynamicBuckets))
		bucketListener := &BucketListener{watchResp: watchResp, currentIndex: waitIndex}
		setupDynamicBucketListeners(dynamicBuckets, bucketListener)
		select {
			case watchResp := <- watchResp:
				removeDynamicBucketListeners(dynamicBuckets, bucketListener)
		 		return watchResp.waitIndex, watchResp.err
		    case <-stopChan:
				removeDynamicBucketListeners(dynamicBuckets, bucketListener)
				return 0, nil
		}
	}
}

<commit_msg>reverted back original code snippet<commit_after>package config_service

import (
	"strings"
	cfgsvc "github.com/Flipkart/config-service/client-go"
	"github.com/kelseyhightower/confd/log"
	"errors"
	"reflect"
	"github.com/pquerna/ffjson/ffjson"
	"strconv"
)

// Client provides a wrapper around the zookeeper client
type Client struct {
	client *cfgsvc.ConfigServiceClient
}

type BucketListener struct{
	watchResp chan *watchResponse
	currentIndex uint64
}

type watchResponse struct {
	waitIndex uint64
	err       error
}

func (this *BucketListener) Connected(bucketName string) {
	log.Info("Connected! " + bucketName)
}

func (this *BucketListener) Disconnected(bucketName string, err error) {
	log.Info("Disconnected! " + bucketName)
}

func (this *BucketListener) Deleted(bucketName string) {
	log.Info("deleted " + bucketName)
	this.watchResp <- &watchResponse{waitIndex: 0, err: errors.New(bucketName + " was deleted")}
}

func (this *BucketListener) Updated(oldBucket *cfgsvc.Bucket, newBucket *cfgsvc.Bucket) {
	this.watchResp <- &watchResponse{waitIndex:this.currentIndex+1, err: nil}
}

func NewConfigClient(machines []string) (*Client, error) {
	c, err := cfgsvc.NewConfigServiceClient(50) //*10)
	if err != nil {
		panic(err)
	}
	return &Client{c}, nil
}

func (c *Client) GetValues(keys []string) (map[string]string, error) {
	vars := make(map[string]string)
	for _, v := range keys {
		bucketsKey := strings.Split(strings.TrimPrefix(v, "/"), "/")
		buckets := strings.Split(bucketsKey[0], ",")
		key := bucketsKey[1]

		dynamicBuckets, err := c.getDynamicBuckets(buckets)
		if err != nil {
			return vars, err
		}

		for _, dynamicBucket := range dynamicBuckets {
			var requestedKeys []string

			if key == "*" {
				//when key is "*" get all keys in a bucket,
				bucketKeys := dynamicBucket.GetKeys()
				requestedKeys = []string{}
				for k, _ := range bucketKeys {
					requestedKeys = append(requestedKeys, k)
				}
			} else {
				requestedKeys = []string{key}
			}

			//For each key in bucket store value in variable named 'vars'
			for _, k := range requestedKeys {

				val := dynamicBucket.GetKeys()[k]
				if val == nil {
					continue
				}

				valType := reflect.TypeOf(val).Kind()
				if valType == reflect.Slice {
					data, err := ffjson.Marshal(val)
					if err != nil {
						log.Error("Failed decoding from JSON")
					} else {
						vars[k] = string(data[:])
					}
				} else {
					switch val.(type) {
					case int, int64:
						vars[k] = strconv.FormatInt(val.(int64), 64)
					case string:
						vars[k] = val.(string)
					case bool:
						vars[k] = strconv.FormatBool(val.(bool))
					case float32, float64:
						vars[k] = strconv.FormatFloat(val.(float64), 'f', -1, 64)
					}
				}
			}
		}

	}
	return vars, nil
}

func (c *Client) getDynamicBuckets(buckets []string) ([]*cfgsvc.DynamicBucket, error) {
	var dynamicBuckets []*cfgsvc.DynamicBucket
	for _, bucket := range buckets {
		bucketName := strings.TrimSpace(bucket)
		dynamicBucket, err := c.client.GetDynamicBucket(bucketName)
		if err != nil {
			return dynamicBuckets, err
		}
		dynamicBuckets = append(dynamicBuckets, dynamicBucket)
	}
	return dynamicBuckets, nil
}

func setupDynamicBucketListeners(dynamicBuckets []*cfgsvc.DynamicBucket, bucketListener *BucketListener) {
	for _, dynamicBucket := range dynamicBuckets {
		dynamicBucket.AddListeners(bucketListener)
	}
}

func removeDynamicBucketListeners(dynamicBuckets []*cfgsvc.DynamicBucket, bucketListener *BucketListener) {
	for _, dynamicBucket := range dynamicBuckets {
		dynamicBucket.RemoveListeners(bucketListener)
	}
}

func (c *Client) WatchPrefix(prefix string, waitIndex uint64, stopChan chan bool) (uint64, error) {
	prefix = strings.TrimPrefix(prefix, "/")
	prefixes := strings.Split(prefix, ",")
	dynamicBuckets, err := c.getDynamicBuckets(prefixes)
	if err != nil {
		return waitIndex, err
	}

	if waitIndex == 0 {
		return waitIndex+1, nil
	}  else {
		watchResp := make(chan *watchResponse, len(dynamicBuckets))
		bucketListener := &BucketListener{watchResp: watchResp, currentIndex: waitIndex}
		setupDynamicBucketListeners(dynamicBuckets, bucketListener)
		select {
			case watchResp := <- watchResp:
				removeDynamicBucketListeners(dynamicBuckets, bucketListener)
		 		return watchResp.waitIndex, watchResp.err
		    case <-stopChan:
				removeDynamicBucketListeners(dynamicBuckets, bucketListener)
				return 0, nil
		}
	}
}

<|endoftext|>


<commit_before>// decoding the quantized float is a bit to messy to do in one function, so
// all the decoder steps are located in this file

package manta

import (
	"math"
)

// Quantized float flags
const qff_rounddown uint32 = (1 << 0)
const qff_roundup uint32 = (1 << 1)
const qff_encode_zero uint32 = (1 << 2)
const qff_encode_integers uint32 = (1 << 3)

// Quantized-decoder struct containing the computed properties
type QuantizedFloatDecoder struct {
	Field      *dt_field // points to datatable field
	Low        float32   // Gets recomputed for round up / down
	High       float32
	HighLowMul float32
	DecMul     float32
	Offset     float32
	Bitcount   uint32 // Gets recomputed for qff_encode_int
	Flags      uint32
	NoScale    bool // Whether to decodes this as a noscale
}

// Validates / recomputes decoder flags
func (qfd *QuantizedFloatDecoder) ValidateFlags() {
	// Check that we have some flags set
	if qfd.Field.Flags != nil {
		qfd.Flags = uint32(*qfd.Field.Flags)
	} else {
		qfd.Flags = 0
		return
	}

	// Discard zero flag when encoding min / max set to 0
	if (qfd.Low == 0.0 && (qfd.Flags&qff_rounddown) != 0) || (qfd.High == 0.0 && (qfd.Flags&qff_roundup) != 0) {
		qfd.Flags &= ^qff_encode_zero
	}

	// If min / max is zero when encoding zero, switch to round up / round down instead
	if qfd.Low == 0.0 && (qfd.Flags&qff_encode_zero) != 0 {
		qfd.Flags |= qff_rounddown
		qfd.Flags &= ^qff_encode_zero
	}

	if qfd.High == 0.0 && (qfd.Flags&qff_encode_zero) != 0 {
		qfd.Flags |= qff_roundup
		qfd.Flags &= ^qff_encode_zero
	}

	// Check if the range spans zero
	if qfd.Low > 0.0 || qfd.High < 0.0 {
		qfd.Flags &= ^qff_encode_zero
	}

	// If we are left with encode zero, only leave integer flag
	if (qfd.Flags & qff_encode_zero) != 0 {
		qfd.Flags &= ^(qff_roundup | qff_rounddown | qff_encode_zero)
	}

	// Verify that we don;t have roundup / rounddown set
	if qfd.Flags&(qff_rounddown|qff_roundup) == (qff_rounddown | qff_roundup) {
		_panicf("Roundup / Rounddown are mutually exclusive")
	}
}

// Assign multipliers
func (qfd *QuantizedFloatDecoder) AssignMultipliers(steps uint32) {
	qfd.HighLowMul = 0.0
	Range := qfd.High - qfd.Low

	High := uint32(0)
	if qfd.Bitcount == 32 {
		High = 0xFFFFFFFE
	} else {
		High = (1 << qfd.Bitcount) - 1
	}

	HighMul := float32(0.0)
	if math.Abs(float64(Range)) <= 0.0 {
		HighMul = float32(High)
	} else {
		HighMul = float32(High) / Range
	}

	// Adjust precision
	if (HighMul*Range > float32(High)) || (float64(HighMul*Range) > float64(High)) {
		multipliers := []float32{0.9999, 0.99, 0.9, 0.8, 0.7}

		for _, mult := range multipliers {
			HighMul = float32(High) / Range * mult

			if (HighMul*Range > float32(High)) || (float64(HighMul*Range) > float64(High)) {
				continue
			}

			break
		}
	}

	qfd.HighLowMul = HighMul
	qfd.DecMul = 1.0 / float32(steps-1)

	if qfd.HighLowMul == 0.0 {
		_panicf("Error computing high / low multiplier")
	}
}

// Quantize a float
func (qfd *QuantizedFloatDecoder) Quantize(val float32) float32 {
	if val < qfd.Low {
		if (qfd.Flags & qff_roundup) == 0 {
			_panicf("Field tried to quantize an out of range value")
		}

		return qfd.Low
	} else if val > qfd.High {
		if (qfd.Flags & qff_rounddown) == 0 {
			_panicf("Field tried to quantize an out of range value")
		}

		return qfd.High
	}

	i := uint32((val - qfd.Low) * qfd.HighLowMul)
	return qfd.Low + (qfd.High-qfd.Low)*(float32(i)*qfd.DecMul)
}

// Actual float decoding
func (qfd *QuantizedFloatDecoder) Decode(r *Reader) float32 {
	if (qfd.Flags&qff_rounddown) != 0 && r.readBoolean() {
		return qfd.Low
	}

	if (qfd.Flags&qff_roundup) != 0 && r.readBoolean() {
		return qfd.High
	}

	if (qfd.Flags&qff_encode_integers) != 0 && r.readBoolean() {
		return 0.0
	}

	return qfd.Low + (qfd.High-qfd.Low)*float32(r.readBits(int(qfd.Bitcount)))*qfd.DecMul
}

// Creates a new quantized float decoder based on given field
func InitQFD(f *dt_field) *QuantizedFloatDecoder {
	qfd := &QuantizedFloatDecoder{}
	qfd.Field = f

	// Set common properties
	if *f.BitCount == 0 || *f.BitCount >= 32 {
		qfd.NoScale = true
		qfd.Bitcount = 32
		return qfd
	} else {
		qfd.NoScale = false
		qfd.Bitcount = uint32(*f.BitCount)
		qfd.Offset = 0.0

		if f.LowValue != nil {
			qfd.Low = *f.LowValue
		} else {
			qfd.Low = 0.0
		}

		if f.HighValue != nil {
			qfd.High = *f.HighValue
		} else {
			qfd.High = 1.0
		}
	}

	// Validate flags
	qfd.ValidateFlags()

	// Handle Round Up, Round Down
	steps := (1 << uint(qfd.Bitcount))

	Range := float32(0)
	if (qfd.Flags & qff_rounddown) != 0 {
		Range = qfd.High - qfd.Low
		qfd.Offset = (Range / float32(steps))
		qfd.High -= qfd.Offset
	} else if (qfd.Flags & qff_roundup) != 0 {
		Range = qfd.High - qfd.Low
		qfd.Offset = (Range / float32(steps))
		qfd.Low += qfd.Offset
	}

	// Handle integer encoding flag
	if (qfd.Flags & qff_encode_integers) != 0 {
		delta := qfd.High - qfd.Low

		if delta < 1 {
			delta = 1
		}

		deltaLog2 := math.Ceil(math.Log2(float64(delta)))
		Range2 := (1 << uint(deltaLog2))
		bc := qfd.Bitcount

		for 1 == 1 {
			if (1 << uint(bc)) > Range2 {
				break
			} else {
				bc++
			}
		}

		if bc > qfd.Bitcount {
			_debugf("Upping bitcount for qf_encode_integers field %v -> %v", qfd.Bitcount, bc)
			qfd.Bitcount = bc
			steps = (1 << uint(qfd.Bitcount))
		}

		qfd.Offset = float32(Range2) / float32(steps)
		qfd.High = qfd.Low + float32(Range2) - qfd.Offset
	}

	// Assign multipliers
	qfd.AssignMultipliers(uint32(steps))

	// Remove unessecary flags
	if (qfd.Flags & qff_rounddown) != 0 {
		if qfd.Quantize(qfd.Low) == qfd.Low {
			qfd.Flags &= ^qff_rounddown
		}
	}

	if (qfd.Flags & qff_roundup) != 0 {
		if qfd.Quantize(qfd.High) == qfd.High {
			qfd.Flags &= ^qff_roundup
		}
	}

	if (qfd.Flags & qff_encode_zero) != 0 {
		if qfd.Quantize(0.0) == 0.0 {
			qfd.Flags &= ^qff_encode_zero
		}
	}

	return qfd
}
<commit_msg>Fix quantize float being off by one<commit_after>// decoding the quantized float is a bit to messy to do in one function, so
// all the decoder steps are located in this file

package manta

import (
	"math"
)

// Quantized float flags
const qff_rounddown uint32 = (1 << 0)
const qff_roundup uint32 = (1 << 1)
const qff_encode_zero uint32 = (1 << 2)
const qff_encode_integers uint32 = (1 << 3)

// Quantized-decoder struct containing the computed properties
type QuantizedFloatDecoder struct {
	Field      *dt_field // points to datatable field
	Low        float32   // Gets recomputed for round up / down
	High       float32
	HighLowMul float32
	DecMul     float32
	Offset     float32
	Bitcount   uint32 // Gets recomputed for qff_encode_int
	Flags      uint32
	NoScale    bool // Whether to decodes this as a noscale
}

// Validates / recomputes decoder flags
func (qfd *QuantizedFloatDecoder) ValidateFlags() {
	// Check that we have some flags set
	if qfd.Field.Flags != nil {
		qfd.Flags = uint32(*qfd.Field.Flags)
	} else {
		qfd.Flags = 0
		return
	}

	// Discard zero flag when encoding min / max set to 0
	if (qfd.Low == 0.0 && (qfd.Flags&qff_rounddown) != 0) || (qfd.High == 0.0 && (qfd.Flags&qff_roundup) != 0) {
		qfd.Flags &= ^qff_encode_zero
	}

	// If min / max is zero when encoding zero, switch to round up / round down instead
	if qfd.Low == 0.0 && (qfd.Flags&qff_encode_zero) != 0 {
		qfd.Flags |= qff_rounddown
		qfd.Flags &= ^qff_encode_zero
	}

	if qfd.High == 0.0 && (qfd.Flags&qff_encode_zero) != 0 {
		qfd.Flags |= qff_roundup
		qfd.Flags &= ^qff_encode_zero
	}

	// Check if the range spans zero
	if qfd.Low > 0.0 || qfd.High < 0.0 {
		qfd.Flags &= ^qff_encode_zero
	}

	// If we are left with encode zero, only leave integer flag
	if (qfd.Flags & qff_encode_zero) != 0 {
		qfd.Flags &= ^(qff_roundup | qff_rounddown | qff_encode_zero)
	}

	// Verify that we don;t have roundup / rounddown set
	if qfd.Flags&(qff_rounddown|qff_roundup) == (qff_rounddown | qff_roundup) {
		_panicf("Roundup / Rounddown are mutually exclusive")
	}
}

// Assign multipliers
func (qfd *QuantizedFloatDecoder) AssignMultipliers(steps uint32) {
	qfd.HighLowMul = 0.0
	Range := qfd.High - qfd.Low

	High := uint32(0)
	if qfd.Bitcount == 32 {
		High = 0xFFFFFFFE
	} else {
		High = (1 << qfd.Bitcount) - 1
	}

	HighMul := float32(0.0)
	if math.Abs(float64(Range)) <= 0.0 {
		HighMul = float32(High)
	} else {
		HighMul = float32(High) / Range
	}

	// Adjust precision
	if (HighMul*Range > float32(High)) || (float64(HighMul*Range) > float64(High)) {
		multipliers := []float32{0.9999, 0.99, 0.9, 0.8, 0.7}

		for _, mult := range multipliers {
			HighMul = float32(High) / Range * mult

			if (HighMul*Range > float32(High)) || (float64(HighMul*Range) > float64(High)) {
				continue
			}

			break
		}
	}

	qfd.HighLowMul = HighMul
	qfd.DecMul = 1.0 / float32(steps-1)

	if qfd.HighLowMul == 0.0 {
		_panicf("Error computing high / low multiplier")
	}
}

// Quantize a float
func (qfd *QuantizedFloatDecoder) Quantize(val float32) float32 {
	if val < qfd.Low {
		if (uint32(*qfd.Field.Flags) & qff_roundup) == 0 {
			_panicf("Field tried to quantize an out of range value")
		}

		return qfd.Low
	} else if val > qfd.High {
		if (uint32(*qfd.Field.Flags) & qff_rounddown) == 0 {
			_panicf("Field tried to quantize an out of range value")
		}

		return qfd.High
	}

	i := uint32((val - qfd.Low) * qfd.HighLowMul)
	return qfd.Low + (qfd.High-qfd.Low)*(float32(i)*qfd.DecMul)
}

// Actual float decoding
func (qfd *QuantizedFloatDecoder) Decode(r *Reader) float32 {
	if (qfd.Flags&qff_rounddown) != 0 && r.readBoolean() {
		return qfd.Low
	}

	if (qfd.Flags&qff_roundup) != 0 && r.readBoolean() {
		return qfd.High
	}

	if (qfd.Flags&qff_encode_zero) != 0 && r.readBoolean() {
		return 0.0
	}

	return qfd.Low + (qfd.High-qfd.Low)*float32(r.readBits(int(qfd.Bitcount)))*qfd.DecMul
}

// Creates a new quantized float decoder based on given field
func InitQFD(f *dt_field) *QuantizedFloatDecoder {
	qfd := &QuantizedFloatDecoder{}
	qfd.Field = f

	// Set common properties
	if *f.BitCount == 0 || *f.BitCount >= 32 {
		qfd.NoScale = true
		qfd.Bitcount = 32
		return qfd
	} else {
		qfd.NoScale = false
		qfd.Bitcount = uint32(*f.BitCount)
		qfd.Offset = 0.0

		if f.LowValue != nil {
			qfd.Low = *f.LowValue
		} else {
			qfd.Low = 0.0
		}

		if f.HighValue != nil {
			qfd.High = *f.HighValue
		} else {
			qfd.High = 1.0
		}
	}

	// Validate flags
	qfd.ValidateFlags()

	// Handle Round Up, Round Down
	steps := (1 << uint(qfd.Bitcount))

	Range := float32(0)
	if (qfd.Flags & qff_rounddown) != 0 {
		Range = qfd.High - qfd.Low
		qfd.Offset = (Range / float32(steps))
		qfd.High -= qfd.Offset
	} else if (qfd.Flags & qff_roundup) != 0 {
		Range = qfd.High - qfd.Low
		qfd.Offset = (Range / float32(steps))
		qfd.Low += qfd.Offset
	}

	// Handle integer encoding flag
	if (qfd.Flags & qff_encode_integers) != 0 {
		delta := qfd.High - qfd.Low

		if delta < 1 {
			delta = 1
		}

		deltaLog2 := math.Ceil(math.Log2(float64(delta)))
		Range2 := (1 << uint(deltaLog2))
		bc := qfd.Bitcount

		for 1 == 1 {
			if (1 << uint(bc)) > Range2 {
				break
			} else {
				bc++
			}
		}

		if bc > qfd.Bitcount {
			_debugf("Upping bitcount for qf_encode_integers field %v -> %v", qfd.Bitcount, bc)
			qfd.Bitcount = bc
			steps = (1 << uint(qfd.Bitcount))
		}

		qfd.Offset = float32(Range2) / float32(steps)
		qfd.High = qfd.Low + float32(Range2) - qfd.Offset
	}

	// Assign multipliers
	qfd.AssignMultipliers(uint32(steps))

	// Remove unessecary flags
	if (qfd.Flags & qff_rounddown) != 0 {
		if qfd.Quantize(qfd.Low) == qfd.Low {
			qfd.Flags &= ^qff_rounddown
		}
	}

	if (qfd.Flags & qff_roundup) != 0 {
		if qfd.Quantize(qfd.High) == qfd.High {
			qfd.Flags &= ^qff_roundup
		}
	}

	if (qfd.Flags & qff_encode_zero) != 0 {
		if qfd.Quantize(0.0) == 0.0 {
			qfd.Flags &= ^qff_encode_zero
		}
	}

	return qfd
}
<|endoftext|>


<commit_before>package sumologic

import (
	"encoding/json"
	"fmt"
)

type Source struct {
	Id 				int `json:"id,omitempty"`
	Type				string `json:"sourceType"`
	Name				string `json:"name"`
	Description			string `json:"description,omitempty"`
	Category			string `json:"category,omitempty"`
	HostName			string `json:"hostName,omitempty"`
	TimeZone			string `json:"timeZone,omitempty"`
	AutomaticDateParsing		bool `json:"automaticDateParsing,omitempty"`
	MultilineProcessingEnabled	bool `json:"multilineProcessingEnabled,omitempty"`
	UseAutolineMatching		bool `json:"useAutolineMatching,omitempty"`
	ManualPrefixRegexp		string `json:"manualPrefixRegexp,omitempty"`
	ForceTimeZone			bool `json:"forceTimeZone,omitempty"`
	DefaultDateFormat		string `json:"defaultDateFormat,omitempty"`
	Filters				[]string `json:"filters,omitempty"`
	CutoffTimestamp			int64 `json:"cutoffTimestamp,omitempty"`
	CutoffRelativeTime		string `json:"cutoffTimestamp,omitempty"`
}

type HttpSource struct {
	Source
	MessagePerRequest bool `json:"messagePerRequest,omitempty"`
	Url string `json:"url,omitempty"`
}

type PollingSource struct {
	Source
	ContentType   string               `json:"contentType"`
	ScanInterval  int                  `json:"scanInterval"`
	Paused        bool                 `json:"paused"`
	ThirdPartyRef PollingThirdPartyRef `json:"thirdPartyRef,omitempty"`
}

type PollingThirdPartyRef struct {
	Resources []PollingResource `json:"resources"`
}

type PollingResource struct {
	ServiceType    string                `json:"serviceType"`
	Authentication PollingAuthentication `json:"authentication"`
	Path           PollingPath           `json:"path"`
}

type PollingAuthentication struct {
	Type   string `json:"type"`
	AwsId  string `json:"awsId"`
	AwsKey string `json:"awsKey"`
}

type PollingPath struct {
	Type           string `json:"type"`
	BucketName     string `json:"bucketName"`
	PathExpression string `json:"pathExpression"`
}

func (s *SumologicClient) CreateHttpSource(
		name, category string,
		messagePerRequest bool,
		collectorId int,
	) (int, error) {

	type HttpSourceMessage struct {
		Source HttpSource `json:"source"`
	}

	request := HttpSourceMessage{}

	request.Source.Type = "HTTP"
	request.Source.Name = name
	request.Source.MessagePerRequest = messagePerRequest
	request.Source.Category = category

	urlPath := fmt.Sprintf("collectors/%d/sources", collectorId)
	body, err := s.Post(urlPath, request)

	if err != nil {
		return -1, err
	}

	var response HttpSourceMessage
	err = json.Unmarshal(body, &response)

	if err != nil {
		return -1, err
	}

	source := response.Source

	return source.Id, nil
}

func (s *SumologicClient) DestroySource(sourceId int, collectorId int) (error) {

	_, err := s.Delete(fmt.Sprintf("collectors/%d/sources/%d", collectorId, sourceId))

	return err
}

func (s *SumologicClient) GetHttpSource(collectorId, sourceId int) (*HttpSource, error) {

	urlPath := fmt.Sprintf("collectors/%d/sources/%d", collectorId, sourceId)
	body, err := s.Get(urlPath)

	if err != nil {
		return nil, err
	}

	type Response struct {
		Source HttpSource `json:"source"`
	}

	var response Response
	err = json.Unmarshal(body, &response)

	var source = response.Source

	return &source, nil
}

func (s *SumologicClient) CreatePollingSource(name, content_type, category string, scan_interval int, paused bool, collectorId int, auth PollingAuthentication, path PollingPath) (int, error) {

	type PollingSourceMessage struct {
		Source PollingSource `json:"source"`
	}

	request := PollingSourceMessage{}

	request.Source.Type = "Polling"
	request.Source.Name = name
	request.Source.Category = category
	request.Source.ContentType = content_type
	request.Source.ScanInterval = scan_interval
	request.Source.Paused = false
	request.Source.ThirdPartyRef = PollingThirdPartyRef{
				Resources: []PollingResource{
					{ServiceType: content_type, Authentication: auth, Path: path},
				},
			}

	urlPath := fmt.Sprintf("collectors/%d/sources", collectorId)

	body, err := s.Post(urlPath, request)

	if err != nil {
		return -1, err
	}

	var response PollingSourceMessage
	err = json.Unmarshal(body, &response)

	if err != nil {
		return -1, err
	}

	return response.Source.Id, nil
}

func (s *SumologicClient) GetPollingSource(collectorId, sourceId int) (*PollingSource, error) {
	urlPath := fmt.Sprintf("collectors/%d/sources/%d", collectorId, sourceId)
	body, err := s.Get(urlPath)

	if err != nil {
		return nil, err
	}
	var response GetPollingSource
	err = json.Unmarshal(body, &response)

	return &response.Source, nil
}

type GetPollingSource struct {
	Source PollingSource `json:"source"`
}
<commit_msg>Reorder methods<commit_after>package sumologic

import (
	"encoding/json"
	"fmt"
)

// Common for all sources
type Source struct {
	Id 				int `json:"id,omitempty"`
	Type				string `json:"sourceType"`
	Name				string `json:"name"`
	Description			string `json:"description,omitempty"`
	Category			string `json:"category,omitempty"`
	HostName			string `json:"hostName,omitempty"`
	TimeZone			string `json:"timeZone,omitempty"`
	AutomaticDateParsing		bool `json:"automaticDateParsing,omitempty"`
	MultilineProcessingEnabled	bool `json:"multilineProcessingEnabled,omitempty"`
	UseAutolineMatching		bool `json:"useAutolineMatching,omitempty"`
	ManualPrefixRegexp		string `json:"manualPrefixRegexp,omitempty"`
	ForceTimeZone			bool `json:"forceTimeZone,omitempty"`
	DefaultDateFormat		string `json:"defaultDateFormat,omitempty"`
	Filters				[]string `json:"filters,omitempty"`
	CutoffTimestamp			int64 `json:"cutoffTimestamp,omitempty"`
	CutoffRelativeTime		string `json:"cutoffTimestamp,omitempty"`
}

func (s *SumologicClient) DestroySource(sourceId int, collectorId int) (error) {

	_, err := s.Delete(fmt.Sprintf("collectors/%d/sources/%d", collectorId, sourceId))

	return err
}

// HTTP source specific
type HttpSource struct {
	Source
	MessagePerRequest bool `json:"messagePerRequest,omitempty"`
	Url string `json:"url,omitempty"`
}

func (s *SumologicClient) CreateHttpSource(
		name, category string,
		messagePerRequest bool,
		collectorId int,
	) (int, error) {

	type HttpSourceMessage struct {
		Source HttpSource `json:"source"`
	}

	request := HttpSourceMessage{}

	request.Source.Type = "HTTP"
	request.Source.Name = name
	request.Source.MessagePerRequest = messagePerRequest
	request.Source.Category = category

	urlPath := fmt.Sprintf("collectors/%d/sources", collectorId)
	body, err := s.Post(urlPath, request)

	if err != nil {
		return -1, err
	}

	var response HttpSourceMessage
	err = json.Unmarshal(body, &response)

	if err != nil {
		return -1, err
	}

	source := response.Source

	return source.Id, nil
}

func (s *SumologicClient) GetHttpSource(collectorId, sourceId int) (*HttpSource, error) {

	urlPath := fmt.Sprintf("collectors/%d/sources/%d", collectorId, sourceId)
	body, err := s.Get(urlPath)

	if err != nil {
		return nil, err
	}

	type Response struct {
		Source HttpSource `json:"source"`
	}

	var response Response
	err = json.Unmarshal(body, &response)

	var source = response.Source

	return &source, nil
}

// Polling source specific
type PollingSource struct {
	Source
	ContentType   string               `json:"contentType"`
	ScanInterval  int                  `json:"scanInterval"`
	Paused        bool                 `json:"paused"`
	ThirdPartyRef PollingThirdPartyRef `json:"thirdPartyRef,omitempty"`
}

type PollingThirdPartyRef struct {
	Resources []PollingResource `json:"resources"`
}

type PollingResource struct {
	ServiceType    string                `json:"serviceType"`
	Authentication PollingAuthentication `json:"authentication"`
	Path           PollingPath           `json:"path"`
}

type PollingAuthentication struct {
	Type   string `json:"type"`
	AwsId  string `json:"awsId"`
	AwsKey string `json:"awsKey"`
}

type PollingPath struct {
	Type           string `json:"type"`
	BucketName     string `json:"bucketName"`
	PathExpression string `json:"pathExpression"`
}

func (s *SumologicClient) CreatePollingSource(name, content_type, category string, scan_interval int, paused bool, collectorId int, auth PollingAuthentication, path PollingPath) (int, error) {

	type PollingSourceMessage struct {
		Source PollingSource `json:"source"`
	}

	request := PollingSourceMessage{}

	request.Source.Type = "Polling"
	request.Source.Name = name
	request.Source.Category = category
	request.Source.ContentType = content_type
	request.Source.ScanInterval = scan_interval
	request.Source.Paused = false
	request.Source.ThirdPartyRef = PollingThirdPartyRef{
				Resources: []PollingResource{
					{ServiceType: content_type, Authentication: auth, Path: path},
				},
			}

	urlPath := fmt.Sprintf("collectors/%d/sources", collectorId)

	body, err := s.Post(urlPath, request)

	if err != nil {
		return -1, err
	}

	var response PollingSourceMessage
	err = json.Unmarshal(body, &response)

	if err != nil {
		return -1, err
	}

	return response.Source.Id, nil
}

func (s *SumologicClient) GetPollingSource(collectorId, sourceId int) (*PollingSource, error) {
	urlPath := fmt.Sprintf("collectors/%d/sources/%d", collectorId, sourceId)
	body, err := s.Get(urlPath)

	if err != nil {
		return nil, err
	}

	type PollingSourceResponse struct {
		Source PollingSource `json:"source"`
	}

	var response PollingSourceResponse
	err = json.Unmarshal(body, &response)

	return &response.Source, nil
}
<|endoftext|>


<commit_before>package main

import (
	"fmt"
	"log"
	"net"
	"net/http"
	_ "net/http/pprof" // register in DefaultServerMux
	"os"
	"time"

	"crypto/tls"

	"github.com/grpc-ecosystem/go-grpc-middleware"
	"github.com/grpc-ecosystem/go-grpc-middleware/logging/logrus"
	"github.com/grpc-ecosystem/go-grpc-prometheus"
	"github.com/improbable-eng/grpc-web/go/grpcweb"
	"github.com/mwitkow/go-conntrack"
	"github.com/mwitkow/grpc-proxy/proxy"
	"github.com/prometheus/client_golang/prometheus/promhttp"
	"github.com/sirupsen/logrus"
	"github.com/spf13/pflag"
	"golang.org/x/net/context"
	_ "golang.org/x/net/trace" // register in DefaultServerMux
	"google.golang.org/grpc"
	"google.golang.org/grpc/metadata"
)

var (
	flagBindAddr    = pflag.String("server_bind_address", "0.0.0.0", "address to bind the server to")
	flagHttpPort    = pflag.Int("server_http_debug_port", 8080, "TCP port to listen on for HTTP1.1 debug calls.")
	flagHttpTlsPort = pflag.Int("server_http_tls_port", 8443, "TCP port to listen on for HTTPS (gRPC, gRPC-Web).")

	runHttpServer = pflag.Bool("run_http_server", true, "whether to run HTTP server")
	runTlsServer  = pflag.Bool("run_tls_server", true, "whether to run TLS server")

	flagHttpMaxWriteTimeout = pflag.Duration("server_http_max_write_timeout", 10*time.Second, "HTTP server config, max write duration.")
	flagHttpMaxReadTimeout  = pflag.Duration("server_http_max_read_timeout", 10*time.Second, "HTTP server config, max read duration.")
)

func main() {
	pflag.Parse()

	logrus.SetOutput(os.Stdout)

	logEntry := logrus.NewEntry(logrus.StandardLogger())

	grpcServer := buildGrpcProxyServer(logEntry)
	errChan := make(chan error)

	// gRPC-Web compatibility layer with CORS configured to accept on every
	wrappedGrpc := grpcweb.WrapServer(grpcServer, grpcweb.WithCorsForRegisteredEndpointsOnly(false))

	if *runHttpServer {
		// Debug server.
		debugServer := buildServer(wrappedGrpc)
		http.Handle("/metrics", promhttp.Handler())
		debugListener := buildListenerOrFail("http", *flagHttpPort)
		serveServer(debugServer, debugListener, "http", errChan)
	}

	if *runTlsServer {
		// Debug server.
		servingServer := buildServer(wrappedGrpc)
		servingListener := buildListenerOrFail("http", *flagHttpTlsPort)
		servingListener = tls.NewListener(servingListener, buildServerTlsOrFail())
		serveServer(servingServer, servingListener, "http_tls", errChan)
	}

	<-errChan
	// TODO(mwitkow): Add graceful shutdown.
}

func buildServer(wrappedGrpc *grpcweb.WrappedGrpcServer) *http.Server {
	return &http.Server{
		WriteTimeout: *flagHttpMaxWriteTimeout,
		ReadTimeout:  *flagHttpMaxReadTimeout,
		Handler: http.HandlerFunc(func(resp http.ResponseWriter, req *http.Request) {
			wrappedGrpc.ServeHTTP(resp, req)
		}),
	}
}

func serveServer(server *http.Server, listener net.Listener, name string, errChan chan error) {
	go func() {
		logrus.Infof("listening for %s on: %v", name, listener.Addr().String())
		if err := server.Serve(listener); err != nil {
			errChan <- fmt.Errorf("%s server error: %v", name, err)
		}
	}()
}

func buildGrpcProxyServer(logger *logrus.Entry) *grpc.Server {
	// gRPC-wide changes.
	grpc.EnableTracing = true
	grpc_logrus.ReplaceGrpcLogger(logger)

	// gRPC proxy logic.
	backendConn := dialBackendOrFail()
	director := func(ctx context.Context, fullMethodName string) (context.Context, *grpc.ClientConn, error) {
		md, _ := metadata.FromIncomingContext(ctx)
		outCtx, _ := context.WithCancel(ctx)
		outCtx = metadata.NewOutgoingContext(outCtx, md.Copy())
		return outCtx, backendConn, nil
	}
	// Server with logging and monitoring enabled.
	return grpc.NewServer(
		grpc.CustomCodec(proxy.Codec()), // needed for proxy to function.
		grpc.UnknownServiceHandler(proxy.TransparentHandler(director)),
		grpc_middleware.WithUnaryServerChain(
			grpc_logrus.UnaryServerInterceptor(logger),
			grpc_prometheus.UnaryServerInterceptor,
		),
		grpc_middleware.WithStreamServerChain(
			grpc_logrus.StreamServerInterceptor(logger),
			grpc_prometheus.StreamServerInterceptor,
		),
	)
}

func buildListenerOrFail(name string, port int) net.Listener {
	addr := fmt.Sprintf("%s:%d", *flagBindAddr, port)
	listener, err := net.Listen("tcp", addr)
	if err != nil {
		log.Fatalf("failed listening for '%v' on %v: %v", name, port, err)
	}
	return conntrack.NewListener(listener,
		conntrack.TrackWithName(name),
		conntrack.TrackWithTcpKeepAlive(20*time.Second),
		conntrack.TrackWithTracing(),
	)
}
<commit_msg>Check at least one of http or tls servers are configured to be active (#130)<commit_after>package main

import (
	"fmt"
	"log"
	"net"
	"net/http"
	_ "net/http/pprof" // register in DefaultServerMux
	"os"
	"time"

	"crypto/tls"

	"github.com/grpc-ecosystem/go-grpc-middleware"
	"github.com/grpc-ecosystem/go-grpc-middleware/logging/logrus"
	"github.com/grpc-ecosystem/go-grpc-prometheus"
	"github.com/improbable-eng/grpc-web/go/grpcweb"
	"github.com/mwitkow/go-conntrack"
	"github.com/mwitkow/grpc-proxy/proxy"
	"github.com/prometheus/client_golang/prometheus/promhttp"
	"github.com/sirupsen/logrus"
	"github.com/spf13/pflag"
	"golang.org/x/net/context"
	_ "golang.org/x/net/trace" // register in DefaultServerMux
	"google.golang.org/grpc"
	"google.golang.org/grpc/metadata"
)

var (
	flagBindAddr    = pflag.String("server_bind_address", "0.0.0.0", "address to bind the server to")
	flagHttpPort    = pflag.Int("server_http_debug_port", 8080, "TCP port to listen on for HTTP1.1 debug calls.")
	flagHttpTlsPort = pflag.Int("server_http_tls_port", 8443, "TCP port to listen on for HTTPS (gRPC, gRPC-Web).")

	runHttpServer = pflag.Bool("run_http_server", true, "whether to run HTTP server")
	runTlsServer  = pflag.Bool("run_tls_server", true, "whether to run TLS server")

	flagHttpMaxWriteTimeout = pflag.Duration("server_http_max_write_timeout", 10*time.Second, "HTTP server config, max write duration.")
	flagHttpMaxReadTimeout  = pflag.Duration("server_http_max_read_timeout", 10*time.Second, "HTTP server config, max read duration.")
)

func main() {
	pflag.Parse()

	logrus.SetOutput(os.Stdout)

	logEntry := logrus.NewEntry(logrus.StandardLogger())

	grpcServer := buildGrpcProxyServer(logEntry)
	errChan := make(chan error)

	// gRPC-Web compatibility layer with CORS configured to accept on every
	wrappedGrpc := grpcweb.WrapServer(grpcServer, grpcweb.WithCorsForRegisteredEndpointsOnly(false))

	if !*runHttpServer && !*runTlsServer {
		logrus.Fatalf("Both run_http_server and run_tls_server are set to false. At least one must be enabled for grpcweb proxy to function correctly.")
	}

	if *runHttpServer {
		// Debug server.
		debugServer := buildServer(wrappedGrpc)
		http.Handle("/metrics", promhttp.Handler())
		debugListener := buildListenerOrFail("http", *flagHttpPort)
		serveServer(debugServer, debugListener, "http", errChan)
	}

	if *runTlsServer {
		// Debug server.
		servingServer := buildServer(wrappedGrpc)
		servingListener := buildListenerOrFail("http", *flagHttpTlsPort)
		servingListener = tls.NewListener(servingListener, buildServerTlsOrFail())
		serveServer(servingServer, servingListener, "http_tls", errChan)
	}

	<-errChan
	// TODO(mwitkow): Add graceful shutdown.
}

func buildServer(wrappedGrpc *grpcweb.WrappedGrpcServer) *http.Server {
	return &http.Server{
		WriteTimeout: *flagHttpMaxWriteTimeout,
		ReadTimeout:  *flagHttpMaxReadTimeout,
		Handler: http.HandlerFunc(func(resp http.ResponseWriter, req *http.Request) {
			wrappedGrpc.ServeHTTP(resp, req)
		}),
	}
}

func serveServer(server *http.Server, listener net.Listener, name string, errChan chan error) {
	go func() {
		logrus.Infof("listening for %s on: %v", name, listener.Addr().String())
		if err := server.Serve(listener); err != nil {
			errChan <- fmt.Errorf("%s server error: %v", name, err)
		}
	}()
}

func buildGrpcProxyServer(logger *logrus.Entry) *grpc.Server {
	// gRPC-wide changes.
	grpc.EnableTracing = true
	grpc_logrus.ReplaceGrpcLogger(logger)

	// gRPC proxy logic.
	backendConn := dialBackendOrFail()
	director := func(ctx context.Context, fullMethodName string) (context.Context, *grpc.ClientConn, error) {
		md, _ := metadata.FromIncomingContext(ctx)
		outCtx, _ := context.WithCancel(ctx)
		outCtx = metadata.NewOutgoingContext(outCtx, md.Copy())
		return outCtx, backendConn, nil
	}
	// Server with logging and monitoring enabled.
	return grpc.NewServer(
		grpc.CustomCodec(proxy.Codec()), // needed for proxy to function.
		grpc.UnknownServiceHandler(proxy.TransparentHandler(director)),
		grpc_middleware.WithUnaryServerChain(
			grpc_logrus.UnaryServerInterceptor(logger),
			grpc_prometheus.UnaryServerInterceptor,
		),
		grpc_middleware.WithStreamServerChain(
			grpc_logrus.StreamServerInterceptor(logger),
			grpc_prometheus.StreamServerInterceptor,
		),
	)
}

func buildListenerOrFail(name string, port int) net.Listener {
	addr := fmt.Sprintf("%s:%d", *flagBindAddr, port)
	listener, err := net.Listen("tcp", addr)
	if err != nil {
		log.Fatalf("failed listening for '%v' on %v: %v", name, port, err)
	}
	return conntrack.NewListener(listener,
		conntrack.TrackWithName(name),
		conntrack.TrackWithTcpKeepAlive(20*time.Second),
		conntrack.TrackWithTracing(),
	)
}
<|endoftext|>


<commit_before>// Copyright 2017 ETH Zurich
// Copyright 2018 ETH Zurich, Anapaya Systems
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//   http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

// Package sciond queries local SCIOND servers for information.
//
// To query SCIOND, initialize a Service object by passing in the path to the
// UNIX socket. It is then possible to establish connections to SCIOND by
// calling Connect or ConnectTimeout on the service. The connections implement
// interface Connector, whose methods can be used to talk to SCIOND.
//
// Connector method calls return the entire answer of SCIOND.
//
// Fields prefixed with Raw (e.g., RawErrorCode) contain data in the format
// received from SCIOND.  These are used internally, and the accessors without
// the prefix (e.g., ErrorCode()) should be used instead.
package sciond

import (
	"context"
	"fmt"
	"strconv"
	"sync"
	"sync/atomic"
	"time"

	"github.com/patrickmn/go-cache"

	"github.com/scionproto/scion/go/lib/addr"
	"github.com/scionproto/scion/go/lib/common"
	"github.com/scionproto/scion/go/lib/ctrl/path_mgmt"
	"github.com/scionproto/scion/go/lib/infra/disp"
	"github.com/scionproto/scion/go/lib/infra/transport"
	"github.com/scionproto/scion/go/lib/log"
	"github.com/scionproto/scion/go/lib/sock/reliable"
	"github.com/scionproto/scion/go/proto"
)

// Time to live for cache entries
const (
	ASInfoTTL  = time.Hour
	IFInfoTTL  = time.Hour
	SVCInfoTTL = 10 * time.Second
	// DefaultSCIONDPath contains the system default for a SCIOND socket.
	DefaultSCIONDPath = "/run/shm/sciond/default.sock"
)

// Service describes a SCIOND endpoint. New connections to SCIOND can be
// initialized via Connect and ConnectTimeout.
type Service interface {
	// Connect connects to the SCIOND server described by Service. Future
	// method calls on the returned Connector request information from SCIOND.
	// The information is not guaranteed to be fresh, as the returned connector
	// caches ASInfo replies for ASInfoTTL time, IFInfo replies for IFInfoTTL
	// time and SVCInfo for SVCInfoTTL time.
	Connect() (Connector, error)
	// ConnectTimeout acts like Connect but takes a timeout.
	//
	// A timeout of 0 means infinite timeout.
	//
	// To check for timeout errors, type assert the returned error to
	// *net.OpError and call method Timeout().
	ConnectTimeout(timeout time.Duration) (Connector, error)
}

type service struct {
	path       string
	reconnects bool
}

// NewService returns a SCIOND API connection factory.
//
// If reconnects is true, connections created from the factory will tolerate
// SCIOND restarts.
func NewService(name string, reconnects bool) Service {
	return &service{
		path:       name,
		reconnects: reconnects,
	}
}

func (s *service) Connect() (Connector, error) {
	if s.reconnects {
		return newReconnector(s.path, 0)
	}
	return connect(s.path)
}

func (s *service) ConnectTimeout(timeout time.Duration) (Connector, error) {
	if s.reconnects {
		return newReconnector(s.path, timeout)
	}
	return connectTimeout(s.path, timeout)
}

// A Connector is used to query SCIOND. The connector maintains an internal
// cache for interface, service and AS information. All connector methods block until either
// an error occurs, or the method successfully returns.
type Connector interface {
	// Paths requests from SCIOND a set of end to end paths between src and
	// dst. max specifies the maximum number of paths returned.
	Paths(ctx context.Context, dst, src addr.IA, max uint16, f PathReqFlags) (*PathReply, error)
	// ASInfo requests from SCIOND information about AS ia.
	ASInfo(ctx context.Context, ia addr.IA) (*ASInfoReply, error)
	// IFInfo requests from SCIOND addresses and ports of interfaces.  Slice
	// ifs contains interface IDs of BRs. If empty, a fresh (i.e., uncached)
	// answer containing all interfaces is returned.
	IFInfo(ctx context.Context, ifs []common.IFIDType) (*IFInfoReply, error)
	// SVCInfo requests from SCIOND information about addresses and ports of
	// infrastructure services.  Slice svcTypes contains a list of desired
	// service types. If unset, a fresh (i.e., uncached) answer containing all
	// service types is returned.
	SVCInfo(ctx context.Context, svcTypes []proto.ServiceType) (*ServiceInfoReply, error)
	// RevNotification sends a raw revocation to SCIOND, as contained in an
	// SCMP message.
	RevNotificationFromRaw(ctx context.Context, b []byte) (*RevReply, error)
	// RevNotification sends a RevocationInfo message to SCIOND.
	RevNotification(ctx context.Context, sRevInfo *path_mgmt.SignedRevInfo) (*RevReply, error)
	// Close shuts down the connection to a SCIOND server.
	Close(ctx context.Context) error
}

type connector struct {
	sync.Mutex
	requestID  uint64
	dispatcher *disp.Dispatcher

	// TODO(kormat): Move the caches to `service`, so they can be shared across connectors.
	asInfos  *cache.Cache
	ifInfos  *cache.Cache
	svcInfos *cache.Cache
}

func connect(socketName string) (*connector, error) {
	return connectTimeout(socketName, 0)
}

func connectTimeout(socketName string, timeout time.Duration) (*connector, error) {
	conn, err := reliable.DialTimeout(socketName, timeout)
	if err != nil {
		return nil, err
	}
	return &connector{
		dispatcher: disp.New(
			transport.NewPacketTransport(conn),
			&Adapter{},
			log.Root(),
		),
		asInfos:  cache.New(ASInfoTTL, time.Minute),
		ifInfos:  cache.New(IFInfoTTL, time.Minute),
		svcInfos: cache.New(SVCInfoTTL, time.Minute),
	}, nil
}

// Self incrementing atomic counter for request IDs
func (c *connector) nextID() uint64 {
	return atomic.AddUint64(&c.requestID, 1)
}

func (c *connector) Paths(ctx context.Context, dst, src addr.IA, max uint16,
	f PathReqFlags) (*PathReply, error) {

	c.Lock()
	defer c.Unlock()
	reply, err := c.dispatcher.Request(
		ctx,
		&Pld{
			Id:    c.nextID(),
			Which: proto.SCIONDMsg_Which_pathReq,
			PathReq: &PathReq{
				Dst:      dst.IAInt(),
				Src:      src.IAInt(),
				MaxPaths: max,
				Flags:    f,
			},
		},
		nil,
	)
	if err != nil {
		return nil, err
	}
	return reply.(*Pld).PathReply, nil
}

func (c *connector) ASInfo(ctx context.Context, ia addr.IA) (*ASInfoReply, error) {
	c.Lock()
	defer c.Unlock()
	// Check if information for this ISD-AS is cached
	key := ia.String()
	if value, found := c.asInfos.Get(key); found {
		return value.(*ASInfoReply), nil
	}
	// Value not in cache, so we ask SCIOND
	pld, err := c.dispatcher.Request(
		ctx,
		&Pld{
			Id:    c.nextID(),
			Which: proto.SCIONDMsg_Which_asInfoReq,
			AsInfoReq: &ASInfoReq{
				Isdas: ia.IAInt(),
			},
		},
		nil,
	)
	if err != nil {
		return nil, err
	}
	asInfoReply := pld.(*Pld).AsInfoReply
	c.asInfos.SetDefault(key, asInfoReply)
	return asInfoReply, nil
}

func (c *connector) IFInfo(ctx context.Context, ifs []common.IFIDType) (*IFInfoReply, error) {
	c.Lock()
	defer c.Unlock()

	foundEntries, remainingIfs := c.getIFEntriesFromCache(ifs)
	if len(remainingIfs) == 0 && len(ifs) != 0 {
		return &IFInfoReply{RawEntries: foundEntries}, nil
	}
	// Some values were not in the cache, so we ask SCIOND for them
	pld, err := c.dispatcher.Request(
		ctx,
		&Pld{
			Id:    c.nextID(),
			Which: proto.SCIONDMsg_Which_ifInfoRequest,
			IfInfoRequest: &IFInfoRequest{
				IfIDs: remainingIfs,
			},
		},
		nil,
	)
	if err != nil {
		return nil, err
	}
	ifInfoReply := pld.(*Pld).IfInfoReply
	// Add new information to cache
	// If SCIOND does not find HostInfo for a requested IFID, the
	// null answer is not added to the cache.
	for _, entry := range ifInfoReply.RawEntries {
		c.ifInfos.SetDefault(entry.IfID.String(), entry)
	}
	// Append old cached entries to our reply
	ifInfoReply.RawEntries = append(ifInfoReply.RawEntries, foundEntries...)
	return ifInfoReply, nil
}

func (c *connector) getIFEntriesFromCache(
	ifs []common.IFIDType) ([]IFInfoReplyEntry, []common.IFIDType) {

	var remainingIfs []common.IFIDType
	var foundEntries []IFInfoReplyEntry
	for _, iface := range ifs {
		if value, found := c.ifInfos.Get(iface.String()); found {
			foundEntries = append(foundEntries, value.(IFInfoReplyEntry))
		} else {
			remainingIfs = append(remainingIfs, iface)
		}
	}
	return foundEntries, remainingIfs
}

func (c *connector) SVCInfo(ctx context.Context,
	svcTypes []proto.ServiceType) (*ServiceInfoReply, error) {

	c.Lock()
	defer c.Unlock()
	foundEntries, remainingSVCs := c.getSVCEntriesFromCache(svcTypes)
	if len(remainingSVCs) == 0 && len(svcTypes) != 0 {
		return &ServiceInfoReply{Entries: foundEntries}, nil
	}
	// Some values were not in the cache, so we ask SCIOND for them
	pld, err := c.dispatcher.Request(
		ctx,
		&Pld{
			Id:    c.nextID(),
			Which: proto.SCIONDMsg_Which_serviceInfoRequest,
			ServiceInfoRequest: &ServiceInfoRequest{
				ServiceTypes: remainingSVCs,
			},
		},
		nil,
	)
	if err != nil {
		return nil, err
	}
	serviceInfoReply := pld.(*Pld).ServiceInfoReply
	// Add new information to cache
	for _, entry := range serviceInfoReply.Entries {
		key := strconv.FormatUint(uint64(entry.ServiceType), 10)
		c.svcInfos.SetDefault(key, entry)
	}
	serviceInfoReply.Entries = append(serviceInfoReply.Entries, foundEntries...)
	return serviceInfoReply, nil
}

func (c *connector) getSVCEntriesFromCache(
	svcTypes []proto.ServiceType) ([]ServiceInfoReplyEntry, []proto.ServiceType) {

	remainingSVCs := make([]proto.ServiceType, 0, len(svcTypes))
	foundEntries := make([]ServiceInfoReplyEntry, 0, len(svcTypes))
	for _, svcType := range svcTypes {
		key := strconv.FormatUint(uint64(svcType), 10)
		if value, found := c.svcInfos.Get(key); found {
			foundEntries = append(foundEntries, value.(ServiceInfoReplyEntry))
		} else {
			remainingSVCs = append(remainingSVCs, svcType)
		}
	}
	return foundEntries, remainingSVCs
}

func (c *connector) RevNotificationFromRaw(ctx context.Context, b []byte) (*RevReply, error) {
	// Extract information from notification
	sRevInfo, err := path_mgmt.NewSignedRevInfoFromRaw(b)
	if err != nil {
		return nil, err
	}
	return c.RevNotification(ctx, sRevInfo)
}

func (c *connector) RevNotification(ctx context.Context,
	sRevInfo *path_mgmt.SignedRevInfo) (*RevReply, error) {

	c.Lock()
	defer c.Unlock()
	// Encapsulate RevInfo item in RevNotification object
	reply, err := c.dispatcher.Request(
		ctx,
		&Pld{
			Id:    c.nextID(),
			Which: proto.SCIONDMsg_Which_revNotification,
			RevNotification: &RevNotification{
				SRevInfo: sRevInfo,
			},
		},
		nil,
	)
	if err != nil {
		return nil, err
	}
	return reply.(*Pld).RevReply, nil
}

func (c *connector) Close(ctx context.Context) error {
	return c.dispatcher.Close(ctx)
}

// GetDefaultSCIONDPath return default sciond path for a given IA
func GetDefaultSCIONDPath(ia *addr.IA) string {
	if ia == nil || ia.IsZero() {
		return DefaultSCIONDPath
	}
	return fmt.Sprintf("/run/shm/sciond/sd%s.sock", ia.FileFmt(false))
}
<commit_msg>sciond-API: Wrap errors (#2269)<commit_after>// Copyright 2017 ETH Zurich
// Copyright 2018 ETH Zurich, Anapaya Systems
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//   http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

// Package sciond queries local SCIOND servers for information.
//
// To query SCIOND, initialize a Service object by passing in the path to the
// UNIX socket. It is then possible to establish connections to SCIOND by
// calling Connect or ConnectTimeout on the service. The connections implement
// interface Connector, whose methods can be used to talk to SCIOND.
//
// Connector method calls return the entire answer of SCIOND.
//
// Fields prefixed with Raw (e.g., RawErrorCode) contain data in the format
// received from SCIOND.  These are used internally, and the accessors without
// the prefix (e.g., ErrorCode()) should be used instead.
package sciond

import (
	"context"
	"fmt"
	"strconv"
	"sync"
	"sync/atomic"
	"time"

	"github.com/patrickmn/go-cache"

	"github.com/scionproto/scion/go/lib/addr"
	"github.com/scionproto/scion/go/lib/common"
	"github.com/scionproto/scion/go/lib/ctrl/path_mgmt"
	"github.com/scionproto/scion/go/lib/infra/disp"
	"github.com/scionproto/scion/go/lib/infra/transport"
	"github.com/scionproto/scion/go/lib/log"
	"github.com/scionproto/scion/go/lib/sock/reliable"
	"github.com/scionproto/scion/go/proto"
)

// Time to live for cache entries
const (
	ASInfoTTL  = time.Hour
	IFInfoTTL  = time.Hour
	SVCInfoTTL = 10 * time.Second
	// DefaultSCIONDPath contains the system default for a SCIOND socket.
	DefaultSCIONDPath = "/run/shm/sciond/default.sock"
)

// Service describes a SCIOND endpoint. New connections to SCIOND can be
// initialized via Connect and ConnectTimeout.
type Service interface {
	// Connect connects to the SCIOND server described by Service. Future
	// method calls on the returned Connector request information from SCIOND.
	// The information is not guaranteed to be fresh, as the returned connector
	// caches ASInfo replies for ASInfoTTL time, IFInfo replies for IFInfoTTL
	// time and SVCInfo for SVCInfoTTL time.
	Connect() (Connector, error)
	// ConnectTimeout acts like Connect but takes a timeout.
	//
	// A timeout of 0 means infinite timeout.
	//
	// To check for timeout errors, type assert the returned error to
	// *net.OpError and call method Timeout().
	ConnectTimeout(timeout time.Duration) (Connector, error)
}

type service struct {
	path       string
	reconnects bool
}

// NewService returns a SCIOND API connection factory.
//
// If reconnects is true, connections created from the factory will tolerate
// SCIOND restarts.
func NewService(name string, reconnects bool) Service {
	return &service{
		path:       name,
		reconnects: reconnects,
	}
}

func (s *service) Connect() (Connector, error) {
	if s.reconnects {
		return newReconnector(s.path, 0)
	}
	return connect(s.path)
}

func (s *service) ConnectTimeout(timeout time.Duration) (Connector, error) {
	if s.reconnects {
		return newReconnector(s.path, timeout)
	}
	return connectTimeout(s.path, timeout)
}

// A Connector is used to query SCIOND. The connector maintains an internal
// cache for interface, service and AS information. All connector methods block until either
// an error occurs, or the method successfully returns.
type Connector interface {
	// Paths requests from SCIOND a set of end to end paths between src and
	// dst. max specifies the maximum number of paths returned.
	Paths(ctx context.Context, dst, src addr.IA, max uint16, f PathReqFlags) (*PathReply, error)
	// ASInfo requests from SCIOND information about AS ia.
	ASInfo(ctx context.Context, ia addr.IA) (*ASInfoReply, error)
	// IFInfo requests from SCIOND addresses and ports of interfaces.  Slice
	// ifs contains interface IDs of BRs. If empty, a fresh (i.e., uncached)
	// answer containing all interfaces is returned.
	IFInfo(ctx context.Context, ifs []common.IFIDType) (*IFInfoReply, error)
	// SVCInfo requests from SCIOND information about addresses and ports of
	// infrastructure services.  Slice svcTypes contains a list of desired
	// service types. If unset, a fresh (i.e., uncached) answer containing all
	// service types is returned.
	SVCInfo(ctx context.Context, svcTypes []proto.ServiceType) (*ServiceInfoReply, error)
	// RevNotification sends a raw revocation to SCIOND, as contained in an
	// SCMP message.
	RevNotificationFromRaw(ctx context.Context, b []byte) (*RevReply, error)
	// RevNotification sends a RevocationInfo message to SCIOND.
	RevNotification(ctx context.Context, sRevInfo *path_mgmt.SignedRevInfo) (*RevReply, error)
	// Close shuts down the connection to a SCIOND server.
	Close(ctx context.Context) error
}

type connector struct {
	sync.Mutex
	requestID  uint64
	dispatcher *disp.Dispatcher

	// TODO(kormat): Move the caches to `service`, so they can be shared across connectors.
	asInfos  *cache.Cache
	ifInfos  *cache.Cache
	svcInfos *cache.Cache
}

func connect(socketName string) (*connector, error) {
	return connectTimeout(socketName, 0)
}

func connectTimeout(socketName string, timeout time.Duration) (*connector, error) {
	conn, err := reliable.DialTimeout(socketName, timeout)
	if err != nil {
		return nil, err
	}
	return &connector{
		dispatcher: disp.New(
			transport.NewPacketTransport(conn),
			&Adapter{},
			log.Root(),
		),
		asInfos:  cache.New(ASInfoTTL, time.Minute),
		ifInfos:  cache.New(IFInfoTTL, time.Minute),
		svcInfos: cache.New(SVCInfoTTL, time.Minute),
	}, nil
}

// Self incrementing atomic counter for request IDs
func (c *connector) nextID() uint64 {
	return atomic.AddUint64(&c.requestID, 1)
}

func (c *connector) Paths(ctx context.Context, dst, src addr.IA, max uint16,
	f PathReqFlags) (*PathReply, error) {

	c.Lock()
	defer c.Unlock()
	reply, err := c.dispatcher.Request(
		ctx,
		&Pld{
			Id:    c.nextID(),
			Which: proto.SCIONDMsg_Which_pathReq,
			PathReq: &PathReq{
				Dst:      dst.IAInt(),
				Src:      src.IAInt(),
				MaxPaths: max,
				Flags:    f,
			},
		},
		nil,
	)
	if err != nil {
		return nil, common.NewBasicError("[sciond-API] Failed to get Paths", err)
	}
	return reply.(*Pld).PathReply, nil
}

func (c *connector) ASInfo(ctx context.Context, ia addr.IA) (*ASInfoReply, error) {
	c.Lock()
	defer c.Unlock()
	// Check if information for this ISD-AS is cached
	key := ia.String()
	if value, found := c.asInfos.Get(key); found {
		return value.(*ASInfoReply), nil
	}
	// Value not in cache, so we ask SCIOND
	pld, err := c.dispatcher.Request(
		ctx,
		&Pld{
			Id:    c.nextID(),
			Which: proto.SCIONDMsg_Which_asInfoReq,
			AsInfoReq: &ASInfoReq{
				Isdas: ia.IAInt(),
			},
		},
		nil,
	)
	if err != nil {
		return nil, common.NewBasicError("[sciond-API] Failed to get ASInfo", err)
	}
	asInfoReply := pld.(*Pld).AsInfoReply
	c.asInfos.SetDefault(key, asInfoReply)
	return asInfoReply, nil
}

func (c *connector) IFInfo(ctx context.Context, ifs []common.IFIDType) (*IFInfoReply, error) {
	c.Lock()
	defer c.Unlock()

	foundEntries, remainingIfs := c.getIFEntriesFromCache(ifs)
	if len(remainingIfs) == 0 && len(ifs) != 0 {
		return &IFInfoReply{RawEntries: foundEntries}, nil
	}
	// Some values were not in the cache, so we ask SCIOND for them
	pld, err := c.dispatcher.Request(
		ctx,
		&Pld{
			Id:    c.nextID(),
			Which: proto.SCIONDMsg_Which_ifInfoRequest,
			IfInfoRequest: &IFInfoRequest{
				IfIDs: remainingIfs,
			},
		},
		nil,
	)
	if err != nil {
		return nil, common.NewBasicError("[sciond-API] Failed to get IFInfo", err)
	}
	ifInfoReply := pld.(*Pld).IfInfoReply
	// Add new information to cache
	// If SCIOND does not find HostInfo for a requested IFID, the
	// null answer is not added to the cache.
	for _, entry := range ifInfoReply.RawEntries {
		c.ifInfos.SetDefault(entry.IfID.String(), entry)
	}
	// Append old cached entries to our reply
	ifInfoReply.RawEntries = append(ifInfoReply.RawEntries, foundEntries...)
	return ifInfoReply, nil
}

func (c *connector) getIFEntriesFromCache(
	ifs []common.IFIDType) ([]IFInfoReplyEntry, []common.IFIDType) {

	var remainingIfs []common.IFIDType
	var foundEntries []IFInfoReplyEntry
	for _, iface := range ifs {
		if value, found := c.ifInfos.Get(iface.String()); found {
			foundEntries = append(foundEntries, value.(IFInfoReplyEntry))
		} else {
			remainingIfs = append(remainingIfs, iface)
		}
	}
	return foundEntries, remainingIfs
}

func (c *connector) SVCInfo(ctx context.Context,
	svcTypes []proto.ServiceType) (*ServiceInfoReply, error) {

	c.Lock()
	defer c.Unlock()
	foundEntries, remainingSVCs := c.getSVCEntriesFromCache(svcTypes)
	if len(remainingSVCs) == 0 && len(svcTypes) != 0 {
		return &ServiceInfoReply{Entries: foundEntries}, nil
	}
	// Some values were not in the cache, so we ask SCIOND for them
	pld, err := c.dispatcher.Request(
		ctx,
		&Pld{
			Id:    c.nextID(),
			Which: proto.SCIONDMsg_Which_serviceInfoRequest,
			ServiceInfoRequest: &ServiceInfoRequest{
				ServiceTypes: remainingSVCs,
			},
		},
		nil,
	)
	if err != nil {
		return nil, common.NewBasicError("[sciond-API] Failed to get SVCInfo", err)
	}
	serviceInfoReply := pld.(*Pld).ServiceInfoReply
	// Add new information to cache
	for _, entry := range serviceInfoReply.Entries {
		key := strconv.FormatUint(uint64(entry.ServiceType), 10)
		c.svcInfos.SetDefault(key, entry)
	}
	serviceInfoReply.Entries = append(serviceInfoReply.Entries, foundEntries...)
	return serviceInfoReply, nil
}

func (c *connector) getSVCEntriesFromCache(
	svcTypes []proto.ServiceType) ([]ServiceInfoReplyEntry, []proto.ServiceType) {

	remainingSVCs := make([]proto.ServiceType, 0, len(svcTypes))
	foundEntries := make([]ServiceInfoReplyEntry, 0, len(svcTypes))
	for _, svcType := range svcTypes {
		key := strconv.FormatUint(uint64(svcType), 10)
		if value, found := c.svcInfos.Get(key); found {
			foundEntries = append(foundEntries, value.(ServiceInfoReplyEntry))
		} else {
			remainingSVCs = append(remainingSVCs, svcType)
		}
	}
	return foundEntries, remainingSVCs
}

func (c *connector) RevNotificationFromRaw(ctx context.Context, b []byte) (*RevReply, error) {
	// Extract information from notification
	sRevInfo, err := path_mgmt.NewSignedRevInfoFromRaw(b)
	if err != nil {
		return nil, err
	}
	return c.RevNotification(ctx, sRevInfo)
}

func (c *connector) RevNotification(ctx context.Context,
	sRevInfo *path_mgmt.SignedRevInfo) (*RevReply, error) {

	c.Lock()
	defer c.Unlock()
	// Encapsulate RevInfo item in RevNotification object
	reply, err := c.dispatcher.Request(
		ctx,
		&Pld{
			Id:    c.nextID(),
			Which: proto.SCIONDMsg_Which_revNotification,
			RevNotification: &RevNotification{
				SRevInfo: sRevInfo,
			},
		},
		nil,
	)
	if err != nil {
		return nil, common.NewBasicError("[sciond-API] Failed to send RevNotification", err)
	}
	return reply.(*Pld).RevReply, nil
}

func (c *connector) Close(ctx context.Context) error {
	return c.dispatcher.Close(ctx)
}

// GetDefaultSCIONDPath return default sciond path for a given IA
func GetDefaultSCIONDPath(ia *addr.IA) string {
	if ia == nil || ia.IsZero() {
		return DefaultSCIONDPath
	}
	return fmt.Sprintf("/run/shm/sciond/sd%s.sock", ia.FileFmt(false))
}
<|endoftext|>