<commit_before>/* This Source Code Form is subject to the terms of the Mozilla Public
 * License, v. 2.0. If a copy of the MPL was not distributed with this
 * file, You can obtain one at http://mozilla.org/MPL/2.0/. */

package mapper

import (
	"github.com/ernestio/aws-definition-mapper/definition"
	"github.com/ernestio/aws-definition-mapper/output"
)

// MapRoute53Zones : Maps the zones from a given input payload.
func MapRoute53Zones(d definition.Definition) []output.Route53Zone {
	var zones []output.Route53Zone

	for _, zone := range d.Route53Zones {
		name := d.GeneratedName() + zone.Name

		z := output.Route53Zone{
			Name:             zone.Name,
			Private:          zone.Private,
			Tags:             mapTags(name, d.Name),
			ProviderType:     "$(datacenters.items.0.type)",
			DatacenterName:   "$(datacenters.items.0.name)",
			SecretAccessKey:  "$(datacenters.items.0.aws_secret_access_key)",
			AccessKeyID:      "$(datacenters.items.0.aws_access_key_id)",
			DatacenterRegion: "$(datacenters.items.0.region)",
			VPCID:            "$(vpcs.items.0.vpc_id)",
		}

		for _, record := range zone.Records {
			r := output.Record{
				Entry:  record.Entry,
				Type:   record.Type,
				Values: record.Values,
				TTL:    record.TTL,
			}

			// append instance and loadbalancer values
			r.Values = append(r.Values, MapRecordInstanceValues(d, record.Instances, zone.Private)...)
			r.Values = append(r.Values, MapRecordLoadbalancerValues(d, record.Loadbalancers)...)
			r.Values = append(r.Values, MapRecordRDSInstanceValues(d, record.RDSInstances)...)
			r.Values = append(r.Values, MapRecordRDSClusterValues(d, record.RDSClusters)...)

			z.Records = append(z.Records, r)
		}

		zones = append(zones, z)
	}

	return zones
}

// MapRecordInstanceValues takes a definition defined value and returns the template variables used on the build
func MapRecordInstanceValues(d definition.Definition, instances []string, private bool) []string {
	var values []string

	attr := "public_ip"
	if private {
		attr = "ip"
	}

	for _, name := range instances {
		// May need to unify this field with elastic_ip on instances
		values = append(values, `$(instances.items.#[name="`+d.GeneratedName()+name+`"].`+attr+`)`)
	}

	return values
}

// MapRecordLoadbalancerValues takes a definition defined value and returns the template variables used on the build
func MapRecordLoadbalancerValues(d definition.Definition, loadbalancers []string) []string {
	var values []string

	for _, name := range loadbalancers {
		values = append(values, `$(elbs.items.#[name="`+d.GeneratedName()+name+`"].dns_name)`)
	}

	return values
}

// MapRecordRDSInstanceValues takes a definition defined value and returns the template variables used on the build
func MapRecordRDSInstanceValues(d definition.Definition, rdsinstances []string) []string {
	var values []string

	for _, name := range rdsinstances {
		values = append(values, `$(rds_instances.items.#[name="`+d.GeneratedName()+name+`"].endpoint)`)
	}

	return values
}

// MapRecordRDSClusterValues takes a definition defined value and returns the template variables used on the build
func MapRecordRDSClusterValues(d definition.Definition, rdsclusters []string) []string {
	var values []string

	for _, name := range rdsclusters {
		values = append(values, `$(rds_clusters.items.#[name="`+d.GeneratedName()+name+`"].endpoint)`)
	}

	return values
}

// MapDefinitionRoute53Zones : Maps zones from the internal format to the input definition format
func MapDefinitionRoute53Zones(m *output.FSMMessage) []definition.Route53Zone {
	var zones []definition.Route53Zone

	prefix := m.Datacenters.Items[0].Name + "-" + m.ServiceName + "-"

	for _, zone := range m.Route53s.Items {
		z := definition.Route53Zone{
			Name:    zone.Name,
			Private: zone.Private,
		}

		for _, record := range zone.Records {
			r := definition.Record{
				Entry: record.Entry,
				Type:  record.Type,
				TTL:   record.TTL,
			}

			for _, v := range record.Values {
				set := false

				for _, i := range m.Instances.Items {
					if i.PublicIP == v || i.ElasticIP == v {
						r.Instances = append(r.Instances, ShortName(i.Name, prefix))
						set = true
						break
					}
				}

				for _, elb := range m.ELBs.Items {
					if elb.DNSName == v {
						r.Loadbalancers = append(r.Loadbalancers, ShortName(elb.Name, prefix))
						set = true
						break
					}
				}

				for _, rds := range m.RDSInstances.Items {
					if rds.Endpoint == v {
						r.RDSInstances = append(r.RDSInstances, ShortName(rds.Name, prefix))
						set = true
						break
					}
				}

				for _, rds := range m.RDSClusters.Items {
					if rds.Endpoint == v {
						r.RDSClusters = append(r.RDSClusters, ShortName(rds.Name, prefix))
						set = true
						break
					}
				}

				if set != true {
					r.Values = append(r.Values, v)
				}
			}

			z.Records = append(z.Records, r)
		}

		zones = append(zones, z)
	}

	return zones
}

// UpdateRoute53Values corrects missing values after an import
func UpdateRoute53Values(m *output.FSMMessage) {
	for i := 0; i < len(m.Route53s.Items); i++ {
		m.Route53s.Items[i].ProviderType = "$(datacenters.items.0.type)"
		m.Route53s.Items[i].AccessKeyID = "$(datacenters.items.0.aws_access_key_id)"
		m.Route53s.Items[i].SecretAccessKey = "$(datacenters.items.0.aws_secret_access_key)"
		m.Route53s.Items[i].DatacenterRegion = "$(datacenters.items.0.region)"
		m.Route53s.Items[i].VPCID = "$(vpcs.items.0.vpc_id)"
	}
}
<commit_msg>templating imported route53 values<commit_after>/* This Source Code Form is subject to the terms of the Mozilla Public
 * License, v. 2.0. If a copy of the MPL was not distributed with this
 * file, You can obtain one at http://mozilla.org/MPL/2.0/. */

package mapper

import (
	"github.com/ernestio/aws-definition-mapper/definition"
	"github.com/ernestio/aws-definition-mapper/output"
)

// MapRoute53Zones : Maps the zones from a given input payload.
func MapRoute53Zones(d definition.Definition) []output.Route53Zone {
	var zones []output.Route53Zone

	for _, zone := range d.Route53Zones {
		z := output.Route53Zone{
			Name:             zone.Name,
			Private:          zone.Private,
			Tags:             mapTagsServiceOnly(d.Name),
			ProviderType:     "$(datacenters.items.0.type)",
			DatacenterName:   "$(datacenters.items.0.name)",
			SecretAccessKey:  "$(datacenters.items.0.aws_secret_access_key)",
			AccessKeyID:      "$(datacenters.items.0.aws_access_key_id)",
			DatacenterRegion: "$(datacenters.items.0.region)",
			VPCID:            "$(vpcs.items.0.vpc_id)",
		}

		for _, record := range zone.Records {
			r := output.Record{
				Entry:  record.Entry,
				Type:   record.Type,
				Values: record.Values,
				TTL:    record.TTL,
			}

			// append instance and loadbalancer values
			r.Values = append(r.Values, MapRecordInstanceValues(d, record.Instances, zone.Private)...)
			r.Values = append(r.Values, MapRecordLoadbalancerValues(d, record.Loadbalancers)...)
			r.Values = append(r.Values, MapRecordRDSInstanceValues(d, record.RDSInstances)...)
			r.Values = append(r.Values, MapRecordRDSClusterValues(d, record.RDSClusters)...)

			z.Records = append(z.Records, r)
		}

		zones = append(zones, z)
	}

	return zones
}

// MapRecordInstanceValues takes a definition defined value and returns the template variables used on the build
func MapRecordInstanceValues(d definition.Definition, instances []string, private bool) []string {
	var values []string

	attr := "public_ip"
	if private {
		attr = "ip"
	}

	for _, name := range instances {
		// May need to unify this field with elastic_ip on instances
		values = append(values, `$(instances.items.#[name="`+d.GeneratedName()+name+`"].`+attr+`)`)
	}

	return values
}

// MapRecordLoadbalancerValues takes a definition defined value and returns the template variables used on the build
func MapRecordLoadbalancerValues(d definition.Definition, loadbalancers []string) []string {
	var values []string

	for _, name := range loadbalancers {
		values = append(values, `$(elbs.items.#[name="`+d.GeneratedName()+name+`"].dns_name)`)
	}

	return values
}

// MapRecordRDSInstanceValues takes a definition defined value and returns the template variables used on the build
func MapRecordRDSInstanceValues(d definition.Definition, rdsinstances []string) []string {
	var values []string

	for _, name := range rdsinstances {
		values = append(values, `$(rds_instances.items.#[name="`+d.GeneratedName()+name+`"].endpoint)`)
	}

	return values
}

// MapRecordRDSClusterValues takes a definition defined value and returns the template variables used on the build
func MapRecordRDSClusterValues(d definition.Definition, rdsclusters []string) []string {
	var values []string

	for _, name := range rdsclusters {
		values = append(values, `$(rds_clusters.items.#[name="`+d.GeneratedName()+name+`"].endpoint)`)
	}

	return values
}

// MapDefinitionRoute53Zones : Maps zones from the internal format to the input definition format
func MapDefinitionRoute53Zones(m *output.FSMMessage) []definition.Route53Zone {
	var zones []definition.Route53Zone

	prefix := m.Datacenters.Items[0].Name + "-" + m.ServiceName + "-"

	for _, zone := range m.Route53s.Items {
		z := definition.Route53Zone{
			Name:    zone.Name,
			Private: zone.Private,
		}

		for _, record := range zone.Records {
			r := definition.Record{
				Entry: record.Entry,
				Type:  record.Type,
				TTL:   record.TTL,
			}

			for _, v := range record.Values {
				set := false

				for _, i := range m.Instances.Items {
					if i.PublicIP == v || i.ElasticIP == v {
						r.Instances = append(r.Instances, ShortName(i.Name, prefix))
						set = true
						break
					}
				}

				for _, elb := range m.ELBs.Items {
					if elb.DNSName == v {
						r.Loadbalancers = append(r.Loadbalancers, ShortName(elb.Name, prefix))
						set = true
						break
					}
				}

				for _, rds := range m.RDSInstances.Items {
					if rds.Endpoint == v {
						r.RDSInstances = append(r.RDSInstances, ShortName(rds.Name, prefix))
						set = true
						break
					}
				}

				for _, rds := range m.RDSClusters.Items {
					if rds.Endpoint == v {
						r.RDSClusters = append(r.RDSClusters, ShortName(rds.Name, prefix))
						set = true
						break
					}
				}

				if set != true {
					r.Values = append(r.Values, v)
				}
			}

			z.Records = append(z.Records, r)
		}

		zones = append(zones, z)
	}

	return zones
}

// UpdateRoute53Values corrects missing values after an import
func UpdateRoute53Values(m *output.FSMMessage) {
	for i := 0; i < len(m.Route53s.Items); i++ {
		m.Route53s.Items[i].ProviderType = "$(datacenters.items.0.type)"
		m.Route53s.Items[i].AccessKeyID = "$(datacenters.items.0.aws_access_key_id)"
		m.Route53s.Items[i].SecretAccessKey = "$(datacenters.items.0.aws_secret_access_key)"
		m.Route53s.Items[i].DatacenterRegion = "$(datacenters.items.0.region)"
		m.Route53s.Items[i].VPCID = "$(vpcs.items.0.vpc_id)"

		for x := 0; x < len(m.Route53s.Items[i].Records); x++ {
			for z := 0; z < len(m.Route53s.Items[i].Records[x].Values); z++ {
				v := m.Route53s.Items[i].Records[x].Values[z]

				for _, ins := range m.Instances.Items {
					if ins.PublicIP == v {
						m.Route53s.Items[i].Records[x].Values[z] = `$(instances.items.#[name="` + ins.Name + `"].public_ip)`
					} else if ins.ElasticIP == v {
						m.Route53s.Items[i].Records[x].Values[z] = `$(instances.items.#[name="` + ins.Name + `"].elastic_ip)`
					} else if ins.IP.String() == v {
						m.Route53s.Items[i].Records[x].Values[z] = `$(instances.items.#[name="` + ins.Name + `"].ip)`
					}
				}

				for _, elb := range m.ELBs.Items {
					if elb.DNSName == v {
						m.Route53s.Items[i].Records[x].Values[z] = `$(elbs.items.#[name="` + elb.Name + `"].dns_name)`
					}
				}

				for _, rds := range m.RDSInstances.Items {
					if rds.Endpoint == v {
						m.Route53s.Items[i].Records[x].Values[z] = `$(rds_instances.items.#[name="` + rds.Name + `"].endpoint)`
					}
				}

				for _, rds := range m.RDSClusters.Items {
					if rds.Endpoint == v {
						m.Route53s.Items[i].Records[x].Values[z] = `$(rds_clusters.items.#[name="` + rds.Name + `"].endpoint)`
					}
				}

			}
		}
	}
}
<|endoftext|>


<commit_before><commit_msg>improve debug info<commit_after><|endoftext|>


<commit_before>package manifest_test

import (
	"strings"
	"testing"

	"github.com/docker-library/go-dockerlibrary/manifest"
)

func TestParseError(t *testing.T) {
	invalidManifest := `this is just completely bogus and invalid no matter how you slice it`

	man, err := manifest.Parse(strings.NewReader(invalidManifest))
	if err == nil {
		t.Errorf("Expected error, got valid manifest instead:\n%s", man)
	}
	if !strings.HasPrefix(err.Error(), "cannot parse manifest in either format:") {
		t.Errorf("Unexpected error: %v", err)
	}
}

func TestArchFile(t *testing.T) {
	tests := []struct {
		file            string
		defaultArchFile string
	}{{
		file:            "",
		defaultArchFile: "Dockerfile",
	}, {
		file:            "Dockerfile",
		defaultArchFile: "Dockerfile",
	}, {
		file:            "Dockerfile-foo",
		defaultArchFile: "Dockerfile-foo",
	},
	}

	for _, test := range tests {
		manString := `Maintainers: Giuseppe Valente <gvalente@arista.com> (@7AC)
GitCommit: abcdef
`
		if test.file != "" {
			manString += "File: " + test.file
		}
		man, err := manifest.Parse2822(strings.NewReader(manString))
		if err != nil {
			t.Fatalf("Unexpected error: %v", err)
		}
		if man.Global.ArchFile(manifest.DefaultArchitecture) != test.defaultArchFile {
			t.Fatalf("Unexpected arch file: %s", man.Global.ArchFile(manifest.DefaultArchitecture))
		}
	}
}
<commit_msg>manifest: test non-default arch<commit_after>package manifest_test

import (
	"strings"
	"testing"

	"github.com/docker-library/go-dockerlibrary/manifest"
)

func TestParseError(t *testing.T) {
	invalidManifest := `this is just completely bogus and invalid no matter how you slice it`

	man, err := manifest.Parse(strings.NewReader(invalidManifest))
	if err == nil {
		t.Errorf("Expected error, got valid manifest instead:\n%s", man)
	}
	if !strings.HasPrefix(err.Error(), "cannot parse manifest in either format:") {
		t.Errorf("Unexpected error: %v", err)
	}
}

func TestArchFile(t *testing.T) {
	tests := []struct {
		file         string
		arch         string
		expectedFile string
	}{{
		file:         "",
		arch:         manifest.DefaultArchitecture,
		expectedFile: "Dockerfile",
	}, {
		file:         "Dockerfile",
		arch:         manifest.DefaultArchitecture,
		expectedFile: "Dockerfile",
	}, {
		file:         "Dockerfile-foo",
		arch:         manifest.DefaultArchitecture,
		expectedFile: "Dockerfile-foo",
	}, {
		file:         "Dockerfile-i386",
		arch:         "i386",
		expectedFile: "Dockerfile-i386",
	},
	}

	for _, test := range tests {
		manString := `Maintainers: Giuseppe Valente <gvalente@arista.com> (@7AC)
GitCommit: abcdef
`
		if test.arch != manifest.DefaultArchitecture {
			manString += test.arch + "-"
		}
		if test.file != "" {
			manString += "File: " + test.file
		}
		man, err := manifest.Parse2822(strings.NewReader(manString))
		if err != nil {
			t.Fatalf("Unexpected error: %v", err)
		}
		actualFile := man.Global.ArchFile(test.arch)
		if actualFile != test.expectedFile {
			t.Fatalf("Unexpected arch file: %s (expected %q)", actualFile, test.expectedFile)
		}
	}
}
<|endoftext|>


<commit_before>package api

import "fmt"

//AzureEnvironmentSpecConfig is the overall configuration differences in different cloud environments.
type AzureEnvironmentSpecConfig struct {
	CloudName            string
	DockerSpecConfig     DockerSpecConfig
	KubernetesSpecConfig KubernetesSpecConfig
	DCOSSpecConfig       DCOSSpecConfig
	EndpointConfig       AzureEndpointConfig
	OSImageConfig        map[Distro]AzureOSImageConfig
}

//DockerSpecConfig is the configurations of docker
type DockerSpecConfig struct {
	DockerEngineRepo         string
	DockerComposeDownloadURL string
}

//DCOSSpecConfig is the configurations of DCOS
type DCOSSpecConfig struct {
	DCOS188BootstrapDownloadURL     string
	DCOS190BootstrapDownloadURL     string
	DCOS198BootstrapDownloadURL     string
	DCOS110BootstrapDownloadURL     string
	DCOS111BootstrapDownloadURL     string
	DCOSWindowsBootstrapDownloadURL string
	DcosRepositoryURL               string // For custom install, for example CI, need these three addributes
	DcosClusterPackageListID        string // the id of the package list file
	DcosProviderPackageID           string // the id of the dcos-provider-xxx package
}

//KubernetesSpecConfig is the kubernetes container images used.
type KubernetesSpecConfig struct {
	KubernetesImageBase              string
	TillerImageBase                  string
	ACIConnectorImageBase            string
	NVIDIAImageBase                  string
	AzureCNIImageBase                string
	EtcdDownloadURLBase              string
	KubeBinariesSASURLBase           string
	WindowsPackageSASURLBase         string
	WindowsTelemetryGUID             string
	CNIPluginsDownloadURL            string
	VnetCNILinuxPluginsDownloadURL   string
	VnetCNIWindowsPluginsDownloadURL string
	ContainerdDownloadURLBase        string
}

//AzureEndpointConfig describes an Azure endpoint
type AzureEndpointConfig struct {
	ResourceManagerVMDNSSuffix string
}

//AzureOSImageConfig describes an Azure OS image
type AzureOSImageConfig struct {
	ImageOffer     string
	ImageSku       string
	ImagePublisher string
	ImageVersion   string
}

var (
	//DefaultKubernetesSpecConfig is the default Docker image source of Kubernetes
	DefaultKubernetesSpecConfig = KubernetesSpecConfig{
		KubernetesImageBase:              "k8s.gcr.io/",
		TillerImageBase:                  "gcr.io/kubernetes-helm/",
		ACIConnectorImageBase:            "microsoft/",
		NVIDIAImageBase:                  "nvidia/",
		AzureCNIImageBase:                "containernetworking/",
		EtcdDownloadURLBase:              "https://acs-mirror.azureedge.net/github-coreos",
		KubeBinariesSASURLBase:           "https://acs-mirror.azureedge.net/wink8s/",
		WindowsPackageSASURLBase:         "https://acs-mirror.azureedge.net/wink8s/",
		WindowsTelemetryGUID:             "fb801154-36b9-41bc-89c2-f4d4f05472b0",
		CNIPluginsDownloadURL:            "https://acs-mirror.azureedge.net/cni/cni-plugins-amd64-" + CNIPluginVer + ".tgz",
		VnetCNILinuxPluginsDownloadURL:   "https://acs-mirror.azureedge.net/cni/azure-vnet-cni-linux-amd64-" + AzureCniPluginVerLinux + ".tgz",
		VnetCNIWindowsPluginsDownloadURL: "https://acs-mirror.azureedge.net/cni/azure-vnet-cni-windows-amd64-" + AzureCniPluginVerWindows + ".zip",
		ContainerdDownloadURLBase:        "https://storage.googleapis.com/cri-containerd-release/",
	}

	//DefaultDCOSSpecConfig is the default DC/OS binary download URL.
	DefaultDCOSSpecConfig = DCOSSpecConfig{
		DCOS188BootstrapDownloadURL:     fmt.Sprintf(AzureEdgeDCOSBootstrapDownloadURL, "stable", "5df43052907c021eeb5de145419a3da1898c58a5"),
		DCOS190BootstrapDownloadURL:     fmt.Sprintf(AzureEdgeDCOSBootstrapDownloadURL, "stable", "58fd0833ce81b6244fc73bf65b5deb43217b0bd7"),
		DCOS198BootstrapDownloadURL:     fmt.Sprintf(AzureEdgeDCOSBootstrapDownloadURL, "stable/1.9.8", "f4ae0d20665fc68ee25282d6f78681b2773c6e10"),
		DCOS110BootstrapDownloadURL:     fmt.Sprintf(AzureEdgeDCOSBootstrapDownloadURL, "stable/1.10.0", "4d92536e7381176206e71ee15b5ffe454439920c"),
		DCOS111BootstrapDownloadURL:     fmt.Sprintf(AzureEdgeDCOSBootstrapDownloadURL, "stable/1.11.0", "a0654657903fb68dff60f6e522a7f241c1bfbf0f"),
		DCOSWindowsBootstrapDownloadURL: "http://dcos-win.westus.cloudapp.azure.com/dcos-windows/stable/",
		DcosRepositoryURL:               "https://dcosio.azureedge.net/dcos/stable/1.11.0",
		DcosClusterPackageListID:        "248a66388bba1adbcb14a52fd3b7b424ab06fa76",
	}

	//DefaultDockerSpecConfig is the default Docker engine repo.
	DefaultDockerSpecConfig = DockerSpecConfig{
		DockerEngineRepo:         "https://aptdocker.azureedge.net/repo",
		DockerComposeDownloadURL: "https://github.com/docker/compose/releases/download",
	}

	//DefaultUbuntuImageConfig is the default Linux distribution.
	DefaultUbuntuImageConfig = AzureOSImageConfig{
		ImageOffer:     "UbuntuServer",
		ImageSku:       "16.04-LTS",
		ImagePublisher: "Canonical",
		ImageVersion:   "latest",
	}

	//SovereignCloudsUbuntuImageConfig is the Linux distribution for Azure Sovereign Clouds.
	SovereignCloudsUbuntuImageConfig = AzureOSImageConfig{
		ImageOffer:     "UbuntuServer",
		ImageSku:       "16.04-LTS",
		ImagePublisher: "Canonical",
		ImageVersion:   "latest",
	}

	//GermanCloudUbuntuImageConfig is the Linux distribution for Azure Sovereign Clouds.
	GermanCloudUbuntuImageConfig = AzureOSImageConfig{
		ImageOffer:     "UbuntuServer",
		ImageSku:       "16.04-LTS",
		ImagePublisher: "Canonical",
		ImageVersion:   "16.04.201801050",
	}

	//DefaultRHELOSImageConfig is the RHEL Linux distribution.
	DefaultRHELOSImageConfig = AzureOSImageConfig{
		ImageOffer:     "RHEL",
		ImageSku:       "7.3",
		ImagePublisher: "RedHat",
		ImageVersion:   "latest",
	}

	//DefaultCoreOSImageConfig is the CoreOS Linux distribution.
	DefaultCoreOSImageConfig = AzureOSImageConfig{
		ImageOffer:     "CoreOS",
		ImageSku:       "Stable",
		ImagePublisher: "CoreOS",
		ImageVersion:   "latest",
	}

	// DefaultAKSOSImageConfig is the AKS image based on Ubuntu 16.04.
	DefaultAKSOSImageConfig = AzureOSImageConfig{
		ImageOffer:     "aks",
		ImageSku:       "aks-ubuntu-1604-201810",
		ImagePublisher: "microsoft-aks",
		ImageVersion:   "2018.10.25",
	}

	// DefaultAKSDockerEngineOSImageConfig is the AKS image based on Ubuntu 16.04.
	DefaultAKSDockerEngineOSImageConfig = AzureOSImageConfig{
		ImageOffer:     "aks",
		ImageSku:       "aks-ubuntu-1604-docker-engine",
		ImagePublisher: "microsoft-aks",
		ImageVersion:   "2018.10.25",
	}

	//DefaultOpenShift39RHELImageConfig is the OpenShift on RHEL distribution.
	DefaultOpenShift39RHELImageConfig = AzureOSImageConfig{
		ImageOffer:     "acsengine-preview",
		ImageSku:       "rhel74",
		ImagePublisher: "redhat",
		ImageVersion:   "latest",
	}

	//DefaultOpenShift39CentOSImageConfig is the OpenShift on CentOS distribution.
	DefaultOpenShift39CentOSImageConfig = AzureOSImageConfig{
		ImageOffer:     "origin-acsengine-preview",
		ImageSku:       "centos7",
		ImagePublisher: "redhat",
		ImageVersion:   "latest",
	}

	//AzureCloudSpec is the default configurations for global azure.
	AzureCloudSpec = AzureEnvironmentSpecConfig{
		CloudName: azurePublicCloud,
		//DockerSpecConfig specify the docker engine download repo
		DockerSpecConfig: DefaultDockerSpecConfig,
		//KubernetesSpecConfig is the default kubernetes container image url.
		KubernetesSpecConfig: DefaultKubernetesSpecConfig,
		DCOSSpecConfig:       DefaultDCOSSpecConfig,

		EndpointConfig: AzureEndpointConfig{
			ResourceManagerVMDNSSuffix: "cloudapp.azure.com",
		},

		OSImageConfig: map[Distro]AzureOSImageConfig{
			Ubuntu:          DefaultUbuntuImageConfig,
			RHEL:            DefaultRHELOSImageConfig,
			CoreOS:          DefaultCoreOSImageConfig,
			AKS:             DefaultAKSOSImageConfig,
			AKSDockerEngine: DefaultAKSDockerEngineOSImageConfig,
			// Image config supported for OpenShift
			OpenShift39RHEL: DefaultOpenShift39RHELImageConfig,
			OpenShiftCentOS: DefaultOpenShift39CentOSImageConfig,
		},
	}

	//AzureGermanCloudSpec is the German cloud config.
	AzureGermanCloudSpec = AzureEnvironmentSpecConfig{
		CloudName:            azureGermanCloud,
		DockerSpecConfig:     DefaultDockerSpecConfig,
		KubernetesSpecConfig: DefaultKubernetesSpecConfig,
		DCOSSpecConfig:       DefaultDCOSSpecConfig,
		EndpointConfig: AzureEndpointConfig{
			ResourceManagerVMDNSSuffix: "cloudapp.microsoftazure.de",
		},
		OSImageConfig: map[Distro]AzureOSImageConfig{
			Ubuntu: GermanCloudUbuntuImageConfig,
			RHEL:   DefaultRHELOSImageConfig,
			CoreOS: DefaultCoreOSImageConfig,
			AKS:    GermanCloudUbuntuImageConfig,
		},
	}

	//AzureUSGovernmentCloud is the US government config.
	AzureUSGovernmentCloud = AzureEnvironmentSpecConfig{
		CloudName:            azureUSGovernmentCloud,
		DockerSpecConfig:     DefaultDockerSpecConfig,
		KubernetesSpecConfig: DefaultKubernetesSpecConfig,
		DCOSSpecConfig:       DefaultDCOSSpecConfig,
		EndpointConfig: AzureEndpointConfig{
			ResourceManagerVMDNSSuffix: "cloudapp.usgovcloudapi.net",
		},
		OSImageConfig: map[Distro]AzureOSImageConfig{
			Ubuntu: SovereignCloudsUbuntuImageConfig,
			RHEL:   DefaultRHELOSImageConfig,
			CoreOS: DefaultCoreOSImageConfig,
			AKS:    SovereignCloudsUbuntuImageConfig,
		},
	}

	//AzureChinaCloudSpec is the configurations for Azure China (Mooncake)
	AzureChinaCloudSpec = AzureEnvironmentSpecConfig{
		CloudName: azureChinaCloud,
		//DockerSpecConfig specify the docker engine download repo
		DockerSpecConfig: DockerSpecConfig{
			DockerEngineRepo:         "https://mirror.azure.cn/docker-engine/apt/repo/",
			DockerComposeDownloadURL: "https://mirror.azure.cn/docker-toolbox/linux/compose",
		},
		//KubernetesSpecConfig - Due to Chinese firewall issue, the default containers from google is blocked, use the Chinese local mirror instead
		KubernetesSpecConfig: KubernetesSpecConfig{
			KubernetesImageBase:              "gcr.azk8s.cn/google_containers/",
			TillerImageBase:                  "gcr.azk8s.cn/kubernetes-helm/",
			ACIConnectorImageBase:            "dockerhub.azk8s.cn/microsoft/",
			NVIDIAImageBase:                  "dockerhub.azk8s.cn/nvidia/",
			AzureCNIImageBase:                "dockerhub.azk8s.cn/containernetworking/",
			EtcdDownloadURLBase:              DefaultKubernetesSpecConfig.EtcdDownloadURLBase,
			KubeBinariesSASURLBase:           DefaultKubernetesSpecConfig.KubeBinariesSASURLBase,
			WindowsPackageSASURLBase:         DefaultKubernetesSpecConfig.WindowsPackageSASURLBase,
			WindowsTelemetryGUID:             DefaultKubernetesSpecConfig.WindowsTelemetryGUID,
			CNIPluginsDownloadURL:            DefaultKubernetesSpecConfig.CNIPluginsDownloadURL,
			VnetCNILinuxPluginsDownloadURL:   DefaultKubernetesSpecConfig.VnetCNILinuxPluginsDownloadURL,
			VnetCNIWindowsPluginsDownloadURL: DefaultKubernetesSpecConfig.VnetCNIWindowsPluginsDownloadURL,
			ContainerdDownloadURLBase:        "https://mirror.azure.cn/kubernetes/containerd/",
		},
		DCOSSpecConfig: DCOSSpecConfig{
			DCOS188BootstrapDownloadURL:     fmt.Sprintf(AzureChinaCloudDCOSBootstrapDownloadURL, "5df43052907c021eeb5de145419a3da1898c58a5"),
			DCOSWindowsBootstrapDownloadURL: "https://dcosdevstorage.blob.core.windows.net/dcos-windows",
			DCOS190BootstrapDownloadURL:     fmt.Sprintf(AzureChinaCloudDCOSBootstrapDownloadURL, "58fd0833ce81b6244fc73bf65b5deb43217b0bd7"),
			DCOS198BootstrapDownloadURL:     fmt.Sprintf(AzureChinaCloudDCOSBootstrapDownloadURL, "f4ae0d20665fc68ee25282d6f78681b2773c6e10"),
		},

		EndpointConfig: AzureEndpointConfig{
			ResourceManagerVMDNSSuffix: "cloudapp.chinacloudapi.cn",
		},
		OSImageConfig: map[Distro]AzureOSImageConfig{
			Ubuntu: SovereignCloudsUbuntuImageConfig,
			RHEL:   DefaultRHELOSImageConfig,
			CoreOS: DefaultCoreOSImageConfig,
			AKS:    SovereignCloudsUbuntuImageConfig,
		},
	}

	// AzureCloudSpecEnvMap is the environment configuration map for all the Azure cloid environments.
	AzureCloudSpecEnvMap = map[string]AzureEnvironmentSpecConfig{
		azureChinaCloud:        AzureChinaCloudSpec,
		azureGermanCloud:       AzureGermanCloudSpec,
		azureUSGovernmentCloud: AzureUSGovernmentCloud,
		azurePublicCloud:       AzureCloudSpec,
	}
)
<commit_msg>add VHD images w/ k8s 1.11.4 and k8s 1.12.2 (#4146)<commit_after>package api

import "fmt"

//AzureEnvironmentSpecConfig is the overall configuration differences in different cloud environments.
type AzureEnvironmentSpecConfig struct {
	CloudName            string
	DockerSpecConfig     DockerSpecConfig
	KubernetesSpecConfig KubernetesSpecConfig
	DCOSSpecConfig       DCOSSpecConfig
	EndpointConfig       AzureEndpointConfig
	OSImageConfig        map[Distro]AzureOSImageConfig
}

//DockerSpecConfig is the configurations of docker
type DockerSpecConfig struct {
	DockerEngineRepo         string
	DockerComposeDownloadURL string
}

//DCOSSpecConfig is the configurations of DCOS
type DCOSSpecConfig struct {
	DCOS188BootstrapDownloadURL     string
	DCOS190BootstrapDownloadURL     string
	DCOS198BootstrapDownloadURL     string
	DCOS110BootstrapDownloadURL     string
	DCOS111BootstrapDownloadURL     string
	DCOSWindowsBootstrapDownloadURL string
	DcosRepositoryURL               string // For custom install, for example CI, need these three addributes
	DcosClusterPackageListID        string // the id of the package list file
	DcosProviderPackageID           string // the id of the dcos-provider-xxx package
}

//KubernetesSpecConfig is the kubernetes container images used.
type KubernetesSpecConfig struct {
	KubernetesImageBase              string
	TillerImageBase                  string
	ACIConnectorImageBase            string
	NVIDIAImageBase                  string
	AzureCNIImageBase                string
	EtcdDownloadURLBase              string
	KubeBinariesSASURLBase           string
	WindowsPackageSASURLBase         string
	WindowsTelemetryGUID             string
	CNIPluginsDownloadURL            string
	VnetCNILinuxPluginsDownloadURL   string
	VnetCNIWindowsPluginsDownloadURL string
	ContainerdDownloadURLBase        string
}

//AzureEndpointConfig describes an Azure endpoint
type AzureEndpointConfig struct {
	ResourceManagerVMDNSSuffix string
}

//AzureOSImageConfig describes an Azure OS image
type AzureOSImageConfig struct {
	ImageOffer     string
	ImageSku       string
	ImagePublisher string
	ImageVersion   string
}

var (
	//DefaultKubernetesSpecConfig is the default Docker image source of Kubernetes
	DefaultKubernetesSpecConfig = KubernetesSpecConfig{
		KubernetesImageBase:              "k8s.gcr.io/",
		TillerImageBase:                  "gcr.io/kubernetes-helm/",
		ACIConnectorImageBase:            "microsoft/",
		NVIDIAImageBase:                  "nvidia/",
		AzureCNIImageBase:                "containernetworking/",
		EtcdDownloadURLBase:              "https://acs-mirror.azureedge.net/github-coreos",
		KubeBinariesSASURLBase:           "https://acs-mirror.azureedge.net/wink8s/",
		WindowsPackageSASURLBase:         "https://acs-mirror.azureedge.net/wink8s/",
		WindowsTelemetryGUID:             "fb801154-36b9-41bc-89c2-f4d4f05472b0",
		CNIPluginsDownloadURL:            "https://acs-mirror.azureedge.net/cni/cni-plugins-amd64-" + CNIPluginVer + ".tgz",
		VnetCNILinuxPluginsDownloadURL:   "https://acs-mirror.azureedge.net/cni/azure-vnet-cni-linux-amd64-" + AzureCniPluginVerLinux + ".tgz",
		VnetCNIWindowsPluginsDownloadURL: "https://acs-mirror.azureedge.net/cni/azure-vnet-cni-windows-amd64-" + AzureCniPluginVerWindows + ".zip",
		ContainerdDownloadURLBase:        "https://storage.googleapis.com/cri-containerd-release/",
	}

	//DefaultDCOSSpecConfig is the default DC/OS binary download URL.
	DefaultDCOSSpecConfig = DCOSSpecConfig{
		DCOS188BootstrapDownloadURL:     fmt.Sprintf(AzureEdgeDCOSBootstrapDownloadURL, "stable", "5df43052907c021eeb5de145419a3da1898c58a5"),
		DCOS190BootstrapDownloadURL:     fmt.Sprintf(AzureEdgeDCOSBootstrapDownloadURL, "stable", "58fd0833ce81b6244fc73bf65b5deb43217b0bd7"),
		DCOS198BootstrapDownloadURL:     fmt.Sprintf(AzureEdgeDCOSBootstrapDownloadURL, "stable/1.9.8", "f4ae0d20665fc68ee25282d6f78681b2773c6e10"),
		DCOS110BootstrapDownloadURL:     fmt.Sprintf(AzureEdgeDCOSBootstrapDownloadURL, "stable/1.10.0", "4d92536e7381176206e71ee15b5ffe454439920c"),
		DCOS111BootstrapDownloadURL:     fmt.Sprintf(AzureEdgeDCOSBootstrapDownloadURL, "stable/1.11.0", "a0654657903fb68dff60f6e522a7f241c1bfbf0f"),
		DCOSWindowsBootstrapDownloadURL: "http://dcos-win.westus.cloudapp.azure.com/dcos-windows/stable/",
		DcosRepositoryURL:               "https://dcosio.azureedge.net/dcos/stable/1.11.0",
		DcosClusterPackageListID:        "248a66388bba1adbcb14a52fd3b7b424ab06fa76",
	}

	//DefaultDockerSpecConfig is the default Docker engine repo.
	DefaultDockerSpecConfig = DockerSpecConfig{
		DockerEngineRepo:         "https://aptdocker.azureedge.net/repo",
		DockerComposeDownloadURL: "https://github.com/docker/compose/releases/download",
	}

	//DefaultUbuntuImageConfig is the default Linux distribution.
	DefaultUbuntuImageConfig = AzureOSImageConfig{
		ImageOffer:     "UbuntuServer",
		ImageSku:       "16.04-LTS",
		ImagePublisher: "Canonical",
		ImageVersion:   "latest",
	}

	//SovereignCloudsUbuntuImageConfig is the Linux distribution for Azure Sovereign Clouds.
	SovereignCloudsUbuntuImageConfig = AzureOSImageConfig{
		ImageOffer:     "UbuntuServer",
		ImageSku:       "16.04-LTS",
		ImagePublisher: "Canonical",
		ImageVersion:   "latest",
	}

	//GermanCloudUbuntuImageConfig is the Linux distribution for Azure Sovereign Clouds.
	GermanCloudUbuntuImageConfig = AzureOSImageConfig{
		ImageOffer:     "UbuntuServer",
		ImageSku:       "16.04-LTS",
		ImagePublisher: "Canonical",
		ImageVersion:   "16.04.201801050",
	}

	//DefaultRHELOSImageConfig is the RHEL Linux distribution.
	DefaultRHELOSImageConfig = AzureOSImageConfig{
		ImageOffer:     "RHEL",
		ImageSku:       "7.3",
		ImagePublisher: "RedHat",
		ImageVersion:   "latest",
	}

	//DefaultCoreOSImageConfig is the CoreOS Linux distribution.
	DefaultCoreOSImageConfig = AzureOSImageConfig{
		ImageOffer:     "CoreOS",
		ImageSku:       "Stable",
		ImagePublisher: "CoreOS",
		ImageVersion:   "latest",
	}

	// DefaultAKSOSImageConfig is the AKS image based on Ubuntu 16.04.
	DefaultAKSOSImageConfig = AzureOSImageConfig{
		ImageOffer:     "aks",
		ImageSku:       "aks-ubuntu-1604-201810",
		ImagePublisher: "microsoft-aks",
		ImageVersion:   "2018.10.26",
	}

	// DefaultAKSDockerEngineOSImageConfig is the AKS image based on Ubuntu 16.04.
	DefaultAKSDockerEngineOSImageConfig = AzureOSImageConfig{
		ImageOffer:     "aks",
		ImageSku:       "aks-ubuntu-1604-docker-engine",
		ImagePublisher: "microsoft-aks",
		ImageVersion:   "2018.10.26",
	}

	//DefaultOpenShift39RHELImageConfig is the OpenShift on RHEL distribution.
	DefaultOpenShift39RHELImageConfig = AzureOSImageConfig{
		ImageOffer:     "acsengine-preview",
		ImageSku:       "rhel74",
		ImagePublisher: "redhat",
		ImageVersion:   "latest",
	}

	//DefaultOpenShift39CentOSImageConfig is the OpenShift on CentOS distribution.
	DefaultOpenShift39CentOSImageConfig = AzureOSImageConfig{
		ImageOffer:     "origin-acsengine-preview",
		ImageSku:       "centos7",
		ImagePublisher: "redhat",
		ImageVersion:   "latest",
	}

	//AzureCloudSpec is the default configurations for global azure.
	AzureCloudSpec = AzureEnvironmentSpecConfig{
		CloudName: azurePublicCloud,
		//DockerSpecConfig specify the docker engine download repo
		DockerSpecConfig: DefaultDockerSpecConfig,
		//KubernetesSpecConfig is the default kubernetes container image url.
		KubernetesSpecConfig: DefaultKubernetesSpecConfig,
		DCOSSpecConfig:       DefaultDCOSSpecConfig,

		EndpointConfig: AzureEndpointConfig{
			ResourceManagerVMDNSSuffix: "cloudapp.azure.com",
		},

		OSImageConfig: map[Distro]AzureOSImageConfig{
			Ubuntu:          DefaultUbuntuImageConfig,
			RHEL:            DefaultRHELOSImageConfig,
			CoreOS:          DefaultCoreOSImageConfig,
			AKS:             DefaultAKSOSImageConfig,
			AKSDockerEngine: DefaultAKSDockerEngineOSImageConfig,
			// Image config supported for OpenShift
			OpenShift39RHEL: DefaultOpenShift39RHELImageConfig,
			OpenShiftCentOS: DefaultOpenShift39CentOSImageConfig,
		},
	}

	//AzureGermanCloudSpec is the German cloud config.
	AzureGermanCloudSpec = AzureEnvironmentSpecConfig{
		CloudName:            azureGermanCloud,
		DockerSpecConfig:     DefaultDockerSpecConfig,
		KubernetesSpecConfig: DefaultKubernetesSpecConfig,
		DCOSSpecConfig:       DefaultDCOSSpecConfig,
		EndpointConfig: AzureEndpointConfig{
			ResourceManagerVMDNSSuffix: "cloudapp.microsoftazure.de",
		},
		OSImageConfig: map[Distro]AzureOSImageConfig{
			Ubuntu: GermanCloudUbuntuImageConfig,
			RHEL:   DefaultRHELOSImageConfig,
			CoreOS: DefaultCoreOSImageConfig,
			AKS:    GermanCloudUbuntuImageConfig,
		},
	}

	//AzureUSGovernmentCloud is the US government config.
	AzureUSGovernmentCloud = AzureEnvironmentSpecConfig{
		CloudName:            azureUSGovernmentCloud,
		DockerSpecConfig:     DefaultDockerSpecConfig,
		KubernetesSpecConfig: DefaultKubernetesSpecConfig,
		DCOSSpecConfig:       DefaultDCOSSpecConfig,
		EndpointConfig: AzureEndpointConfig{
			ResourceManagerVMDNSSuffix: "cloudapp.usgovcloudapi.net",
		},
		OSImageConfig: map[Distro]AzureOSImageConfig{
			Ubuntu: SovereignCloudsUbuntuImageConfig,
			RHEL:   DefaultRHELOSImageConfig,
			CoreOS: DefaultCoreOSImageConfig,
			AKS:    SovereignCloudsUbuntuImageConfig,
		},
	}

	//AzureChinaCloudSpec is the configurations for Azure China (Mooncake)
	AzureChinaCloudSpec = AzureEnvironmentSpecConfig{
		CloudName: azureChinaCloud,
		//DockerSpecConfig specify the docker engine download repo
		DockerSpecConfig: DockerSpecConfig{
			DockerEngineRepo:         "https://mirror.azure.cn/docker-engine/apt/repo/",
			DockerComposeDownloadURL: "https://mirror.azure.cn/docker-toolbox/linux/compose",
		},
		//KubernetesSpecConfig - Due to Chinese firewall issue, the default containers from google is blocked, use the Chinese local mirror instead
		KubernetesSpecConfig: KubernetesSpecConfig{
			KubernetesImageBase:              "gcr.azk8s.cn/google_containers/",
			TillerImageBase:                  "gcr.azk8s.cn/kubernetes-helm/",
			ACIConnectorImageBase:            "dockerhub.azk8s.cn/microsoft/",
			NVIDIAImageBase:                  "dockerhub.azk8s.cn/nvidia/",
			AzureCNIImageBase:                "dockerhub.azk8s.cn/containernetworking/",
			EtcdDownloadURLBase:              DefaultKubernetesSpecConfig.EtcdDownloadURLBase,
			KubeBinariesSASURLBase:           DefaultKubernetesSpecConfig.KubeBinariesSASURLBase,
			WindowsPackageSASURLBase:         DefaultKubernetesSpecConfig.WindowsPackageSASURLBase,
			WindowsTelemetryGUID:             DefaultKubernetesSpecConfig.WindowsTelemetryGUID,
			CNIPluginsDownloadURL:            DefaultKubernetesSpecConfig.CNIPluginsDownloadURL,
			VnetCNILinuxPluginsDownloadURL:   DefaultKubernetesSpecConfig.VnetCNILinuxPluginsDownloadURL,
			VnetCNIWindowsPluginsDownloadURL: DefaultKubernetesSpecConfig.VnetCNIWindowsPluginsDownloadURL,
			ContainerdDownloadURLBase:        "https://mirror.azure.cn/kubernetes/containerd/",
		},
		DCOSSpecConfig: DCOSSpecConfig{
			DCOS188BootstrapDownloadURL:     fmt.Sprintf(AzureChinaCloudDCOSBootstrapDownloadURL, "5df43052907c021eeb5de145419a3da1898c58a5"),
			DCOSWindowsBootstrapDownloadURL: "https://dcosdevstorage.blob.core.windows.net/dcos-windows",
			DCOS190BootstrapDownloadURL:     fmt.Sprintf(AzureChinaCloudDCOSBootstrapDownloadURL, "58fd0833ce81b6244fc73bf65b5deb43217b0bd7"),
			DCOS198BootstrapDownloadURL:     fmt.Sprintf(AzureChinaCloudDCOSBootstrapDownloadURL, "f4ae0d20665fc68ee25282d6f78681b2773c6e10"),
		},

		EndpointConfig: AzureEndpointConfig{
			ResourceManagerVMDNSSuffix: "cloudapp.chinacloudapi.cn",
		},
		OSImageConfig: map[Distro]AzureOSImageConfig{
			Ubuntu: SovereignCloudsUbuntuImageConfig,
			RHEL:   DefaultRHELOSImageConfig,
			CoreOS: DefaultCoreOSImageConfig,
			AKS:    SovereignCloudsUbuntuImageConfig,
		},
	}

	// AzureCloudSpecEnvMap is the environment configuration map for all the Azure cloid environments.
	AzureCloudSpecEnvMap = map[string]AzureEnvironmentSpecConfig{
		azureChinaCloud:        AzureChinaCloudSpec,
		azureGermanCloud:       AzureGermanCloudSpec,
		azureUSGovernmentCloud: AzureUSGovernmentCloud,
		azurePublicCloud:       AzureCloudSpec,
	}
)
<|endoftext|>


<commit_before>// package merkledag implements the ipfs Merkle DAG datastructures.
package merkledag

import (
	"fmt"
	"sync"
	"time"

	"github.com/jbenet/go-ipfs/Godeps/_workspace/src/code.google.com/p/go.net/context"

	mh "github.com/jbenet/go-ipfs/Godeps/_workspace/src/github.com/jbenet/go-multihash"
	blocks "github.com/jbenet/go-ipfs/blocks"
	bserv "github.com/jbenet/go-ipfs/blockservice"
	u "github.com/jbenet/go-ipfs/util"
)

var log = u.Logger("merkledag")
var ErrNotFound = fmt.Errorf("merkledag: not found")

// NodeMap maps u.Keys to Nodes.
// We cannot use []byte/Multihash for keys :(
// so have to convert Multihash bytes to string (u.Key)
type NodeMap map[u.Key]*Node

// DAGService is an IPFS Merkle DAG service.
type DAGService interface {
	Add(*Node) (u.Key, error)
	AddRecursive(*Node) error
	Get(u.Key) (*Node, error)
	Remove(*Node) error

	// GetDAG returns, in order, all the single leve child
	// nodes of the passed in node.
	GetDAG(context.Context, *Node) <-chan *Node
}

func NewDAGService(bs *bserv.BlockService) DAGService {
	return &dagService{bs}
}

// Node represents a node in the IPFS Merkle DAG.
// nodes have opaque data and a set of navigable links.
type Node struct {
	Links []*Link
	Data  []byte

	// cache encoded/marshaled value
	encoded []byte

	cached mh.Multihash
}

// Link represents an IPFS Merkle DAG Link between Nodes.
type Link struct {
	// utf string name. should be unique per object
	Name string // utf8

	// cumulative size of target object
	Size uint64

	// multihash of the target object
	Hash mh.Multihash

	// a ptr to the actual node for graph manipulation
	Node *Node
}

// MakeLink creates a link to the given node
func MakeLink(n *Node) (*Link, error) {
	s, err := n.Size()
	if err != nil {
		return nil, err
	}

	h, err := n.Multihash()
	if err != nil {
		return nil, err
	}
	return &Link{
		Size: s,
		Hash: h,
	}, nil
}

// GetNode returns the MDAG Node that this link points to
func (l *Link) GetNode(serv DAGService) (*Node, error) {
	if l.Node != nil {
		return l.Node, nil
	}

	return serv.Get(u.Key(l.Hash))
}

// AddNodeLink adds a link to another node.
func (n *Node) AddNodeLink(name string, that *Node) error {
	lnk, err := MakeLink(that)
	if err != nil {
		return err
	}
	lnk.Name = name
	lnk.Node = that

	n.Links = append(n.Links, lnk)
	return nil
}

// AddNodeLink adds a link to another node. without keeping a reference to
// the child node
func (n *Node) AddNodeLinkClean(name string, that *Node) error {
	lnk, err := MakeLink(that)
	if err != nil {
		return err
	}
	lnk.Name = name

	n.Links = append(n.Links, lnk)
	return nil
}

// Remove a link on this node by the given name
func (n *Node) RemoveNodeLink(name string) error {
	for i, l := range n.Links {
		if l.Name == name {
			n.Links = append(n.Links[:i], n.Links[i+1:]...)
			return nil
		}
	}
	return ErrNotFound
}

// Copy returns a copy of the node.
// NOTE: does not make copies of Node objects in the links.
func (n *Node) Copy() *Node {
	nnode := new(Node)
	nnode.Data = make([]byte, len(n.Data))
	copy(nnode.Data, n.Data)

	nnode.Links = make([]*Link, len(n.Links))
	copy(nnode.Links, n.Links)
	return nnode
}

// Size returns the total size of the data addressed by node,
// including the total sizes of references.
func (n *Node) Size() (uint64, error) {
	b, err := n.Encoded(false)
	if err != nil {
		return 0, err
	}

	s := uint64(len(b))
	for _, l := range n.Links {
		s += l.Size
	}
	return s, nil
}

// Multihash hashes the encoded data of this node.
func (n *Node) Multihash() (mh.Multihash, error) {
	// Note: Encoded generates the hash and puts it in n.cached.
	_, err := n.Encoded(false)
	if err != nil {
		return nil, err
	}

	return n.cached, nil
}

// Key returns the Multihash as a key, for maps.
func (n *Node) Key() (u.Key, error) {
	h, err := n.Multihash()
	return u.Key(h), err
}

// dagService is an IPFS Merkle DAG service.
// - the root is virtual (like a forest)
// - stores nodes' data in a BlockService
// TODO: should cache Nodes that are in memory, and be
//       able to free some of them when vm pressure is high
type dagService struct {
	Blocks *bserv.BlockService
}

// Add adds a node to the dagService, storing the block in the BlockService
func (n *dagService) Add(nd *Node) (u.Key, error) {
	k, _ := nd.Key()
	log.Debugf("DagService Add [%s]", k)
	if n == nil {
		return "", fmt.Errorf("dagService is nil")
	}

	d, err := nd.Encoded(false)
	if err != nil {
		return "", err
	}

	b := new(blocks.Block)
	b.Data = d
	b.Multihash, err = nd.Multihash()
	if err != nil {
		return "", err
	}

	return n.Blocks.AddBlock(b)
}

// AddRecursive adds the given node and all child nodes to the BlockService
func (n *dagService) AddRecursive(nd *Node) error {
	_, err := n.Add(nd)
	if err != nil {
		log.Info("AddRecursive Error: %s\n", err)
		return err
	}

	for _, link := range nd.Links {
		if link.Node != nil {
			err := n.AddRecursive(link.Node)
			if err != nil {
				return err
			}
		}
	}

	return nil
}

// Get retrieves a node from the dagService, fetching the block in the BlockService
func (n *dagService) Get(k u.Key) (*Node, error) {
	if n == nil {
		return nil, fmt.Errorf("dagService is nil")
	}

	ctx, _ := context.WithTimeout(context.TODO(), time.Second*5)
	b, err := n.Blocks.GetBlock(ctx, k)
	if err != nil {
		return nil, err
	}

	return Decoded(b.Data)
}

// Remove deletes the given node and all of its children from the BlockService
func (n *dagService) Remove(nd *Node) error {
	for _, l := range nd.Links {
		if l.Node != nil {
			n.Remove(l.Node)
		}
	}
	k, err := nd.Key()
	if err != nil {
		return err
	}
	return n.Blocks.DeleteBlock(k)
}

// FetchGraph asynchronously fetches all nodes that are children of the given
// node, and returns a channel that may be waited upon for the fetch to complete
func FetchGraph(ctx context.Context, root *Node, serv DAGService) chan struct{} {
	log.Warning("Untested.")
	var wg sync.WaitGroup
	done := make(chan struct{})

	for _, l := range root.Links {
		wg.Add(1)
		go func(lnk *Link) {

			// Signal child is done on way out
			defer wg.Done()
			select {
			case <-ctx.Done():
				return
			}

			nd, err := lnk.GetNode(serv)
			if err != nil {
				log.Error(err)
				return
			}

			// Wait for children to finish
			<-FetchGraph(ctx, nd, serv)
		}(l)
	}

	go func() {
		wg.Wait()
		done <- struct{}{}
	}()

	return done
}

// Searches this nodes links for one to the given key,
// returns the index of said link
func FindLink(n *Node, k u.Key, found []*Node) (int, error) {
	for i, lnk := range n.Links {
		if u.Key(lnk.Hash) == k && found[i] == nil {
			return i, nil
		}
	}
	return -1, u.ErrNotFound
}

// GetDAG will fill out all of the links of the given Node.
// It returns a channel of nodes, which the caller can receive
// all the child nodes of 'root' on, in proper order.
func (ds *dagService) GetDAG(ctx context.Context, root *Node) <-chan *Node {
	sig := make(chan *Node)
	go func() {
		var keys []u.Key
		for _, lnk := range root.Links {
			keys = append(keys, u.Key(lnk.Hash))
		}
		blkchan := ds.Blocks.GetBlocks(ctx, keys)

		nodes := make([]*Node, len(root.Links))
		next := 0
		for blk := range blkchan {
			i, err := FindLink(root, blk.Key(), nodes)
			if err != nil {
				// NB: can only occur as a result of programmer error
				panic("Received block that wasnt in this nodes links!")
			}

			nd, err := Decoded(blk.Data)
			if err != nil {
				// NB: can occur in normal situations, with improperly formatted
				//		input data
				log.Error("Got back bad block!")
				break
			}
			nodes[i] = nd
			for { //Check for duplicate links
				ni, err := FindLink(root, blk.Key(), nodes)
				if err != nil {
					break
				}
				nodes[ni] = nd
			}

			if next == i {
				sig <- nd
				next++
				for ; next < len(nodes) && nodes[next] != nil; next++ {
					sig <- nodes[next]
				}
			}
		}
		if next < len(nodes) {
			// TODO: bubble errors back up.
			log.Errorf("Did not receive correct number of nodes!")
		}
		close(sig)
	}()

	return sig
}
<commit_msg>fix(merkle) use defer<commit_after>// package merkledag implements the ipfs Merkle DAG datastructures.
package merkledag

import (
	"fmt"
	"sync"
	"time"

	"github.com/jbenet/go-ipfs/Godeps/_workspace/src/code.google.com/p/go.net/context"

	mh "github.com/jbenet/go-ipfs/Godeps/_workspace/src/github.com/jbenet/go-multihash"
	blocks "github.com/jbenet/go-ipfs/blocks"
	bserv "github.com/jbenet/go-ipfs/blockservice"
	u "github.com/jbenet/go-ipfs/util"
)

var log = u.Logger("merkledag")
var ErrNotFound = fmt.Errorf("merkledag: not found")

// NodeMap maps u.Keys to Nodes.
// We cannot use []byte/Multihash for keys :(
// so have to convert Multihash bytes to string (u.Key)
type NodeMap map[u.Key]*Node

// DAGService is an IPFS Merkle DAG service.
type DAGService interface {
	Add(*Node) (u.Key, error)
	AddRecursive(*Node) error
	Get(u.Key) (*Node, error)
	Remove(*Node) error

	// GetDAG returns, in order, all the single leve child
	// nodes of the passed in node.
	GetDAG(context.Context, *Node) <-chan *Node
}

func NewDAGService(bs *bserv.BlockService) DAGService {
	return &dagService{bs}
}

// Node represents a node in the IPFS Merkle DAG.
// nodes have opaque data and a set of navigable links.
type Node struct {
	Links []*Link
	Data  []byte

	// cache encoded/marshaled value
	encoded []byte

	cached mh.Multihash
}

// Link represents an IPFS Merkle DAG Link between Nodes.
type Link struct {
	// utf string name. should be unique per object
	Name string // utf8

	// cumulative size of target object
	Size uint64

	// multihash of the target object
	Hash mh.Multihash

	// a ptr to the actual node for graph manipulation
	Node *Node
}

// MakeLink creates a link to the given node
func MakeLink(n *Node) (*Link, error) {
	s, err := n.Size()
	if err != nil {
		return nil, err
	}

	h, err := n.Multihash()
	if err != nil {
		return nil, err
	}
	return &Link{
		Size: s,
		Hash: h,
	}, nil
}

// GetNode returns the MDAG Node that this link points to
func (l *Link) GetNode(serv DAGService) (*Node, error) {
	if l.Node != nil {
		return l.Node, nil
	}

	return serv.Get(u.Key(l.Hash))
}

// AddNodeLink adds a link to another node.
func (n *Node) AddNodeLink(name string, that *Node) error {
	lnk, err := MakeLink(that)
	if err != nil {
		return err
	}
	lnk.Name = name
	lnk.Node = that

	n.Links = append(n.Links, lnk)
	return nil
}

// AddNodeLink adds a link to another node. without keeping a reference to
// the child node
func (n *Node) AddNodeLinkClean(name string, that *Node) error {
	lnk, err := MakeLink(that)
	if err != nil {
		return err
	}
	lnk.Name = name

	n.Links = append(n.Links, lnk)
	return nil
}

// Remove a link on this node by the given name
func (n *Node) RemoveNodeLink(name string) error {
	for i, l := range n.Links {
		if l.Name == name {
			n.Links = append(n.Links[:i], n.Links[i+1:]...)
			return nil
		}
	}
	return ErrNotFound
}

// Copy returns a copy of the node.
// NOTE: does not make copies of Node objects in the links.
func (n *Node) Copy() *Node {
	nnode := new(Node)
	nnode.Data = make([]byte, len(n.Data))
	copy(nnode.Data, n.Data)

	nnode.Links = make([]*Link, len(n.Links))
	copy(nnode.Links, n.Links)
	return nnode
}

// Size returns the total size of the data addressed by node,
// including the total sizes of references.
func (n *Node) Size() (uint64, error) {
	b, err := n.Encoded(false)
	if err != nil {
		return 0, err
	}

	s := uint64(len(b))
	for _, l := range n.Links {
		s += l.Size
	}
	return s, nil
}

// Multihash hashes the encoded data of this node.
func (n *Node) Multihash() (mh.Multihash, error) {
	// Note: Encoded generates the hash and puts it in n.cached.
	_, err := n.Encoded(false)
	if err != nil {
		return nil, err
	}

	return n.cached, nil
}

// Key returns the Multihash as a key, for maps.
func (n *Node) Key() (u.Key, error) {
	h, err := n.Multihash()
	return u.Key(h), err
}

// dagService is an IPFS Merkle DAG service.
// - the root is virtual (like a forest)
// - stores nodes' data in a BlockService
// TODO: should cache Nodes that are in memory, and be
//       able to free some of them when vm pressure is high
type dagService struct {
	Blocks *bserv.BlockService
}

// Add adds a node to the dagService, storing the block in the BlockService
func (n *dagService) Add(nd *Node) (u.Key, error) {
	k, _ := nd.Key()
	log.Debugf("DagService Add [%s]", k)
	if n == nil {
		return "", fmt.Errorf("dagService is nil")
	}

	d, err := nd.Encoded(false)
	if err != nil {
		return "", err
	}

	b := new(blocks.Block)
	b.Data = d
	b.Multihash, err = nd.Multihash()
	if err != nil {
		return "", err
	}

	return n.Blocks.AddBlock(b)
}

// AddRecursive adds the given node and all child nodes to the BlockService
func (n *dagService) AddRecursive(nd *Node) error {
	_, err := n.Add(nd)
	if err != nil {
		log.Info("AddRecursive Error: %s\n", err)
		return err
	}

	for _, link := range nd.Links {
		if link.Node != nil {
			err := n.AddRecursive(link.Node)
			if err != nil {
				return err
			}
		}
	}

	return nil
}

// Get retrieves a node from the dagService, fetching the block in the BlockService
func (n *dagService) Get(k u.Key) (*Node, error) {
	if n == nil {
		return nil, fmt.Errorf("dagService is nil")
	}

	ctx, _ := context.WithTimeout(context.TODO(), time.Second*5)
	b, err := n.Blocks.GetBlock(ctx, k)
	if err != nil {
		return nil, err
	}

	return Decoded(b.Data)
}

// Remove deletes the given node and all of its children from the BlockService
func (n *dagService) Remove(nd *Node) error {
	for _, l := range nd.Links {
		if l.Node != nil {
			n.Remove(l.Node)
		}
	}
	k, err := nd.Key()
	if err != nil {
		return err
	}
	return n.Blocks.DeleteBlock(k)
}

// FetchGraph asynchronously fetches all nodes that are children of the given
// node, and returns a channel that may be waited upon for the fetch to complete
func FetchGraph(ctx context.Context, root *Node, serv DAGService) chan struct{} {
	log.Warning("Untested.")
	var wg sync.WaitGroup
	done := make(chan struct{})

	for _, l := range root.Links {
		wg.Add(1)
		go func(lnk *Link) {

			// Signal child is done on way out
			defer wg.Done()
			select {
			case <-ctx.Done():
				return
			}

			nd, err := lnk.GetNode(serv)
			if err != nil {
				log.Error(err)
				return
			}

			// Wait for children to finish
			<-FetchGraph(ctx, nd, serv)
		}(l)
	}

	go func() {
		wg.Wait()
		done <- struct{}{}
	}()

	return done
}

// Searches this nodes links for one to the given key,
// returns the index of said link
func FindLink(n *Node, k u.Key, found []*Node) (int, error) {
	for i, lnk := range n.Links {
		if u.Key(lnk.Hash) == k && found[i] == nil {
			return i, nil
		}
	}
	return -1, u.ErrNotFound
}

// GetDAG will fill out all of the links of the given Node.
// It returns a channel of nodes, which the caller can receive
// all the child nodes of 'root' on, in proper order.
func (ds *dagService) GetDAG(ctx context.Context, root *Node) <-chan *Node {
	sig := make(chan *Node)
	go func() {
		defer close(sig)

		var keys []u.Key
		for _, lnk := range root.Links {
			keys = append(keys, u.Key(lnk.Hash))
		}
		blkchan := ds.Blocks.GetBlocks(ctx, keys)

		nodes := make([]*Node, len(root.Links))
		next := 0
		for blk := range blkchan {
			i, err := FindLink(root, blk.Key(), nodes)
			if err != nil {
				// NB: can only occur as a result of programmer error
				panic("Received block that wasnt in this nodes links!")
			}

			nd, err := Decoded(blk.Data)
			if err != nil {
				// NB: can occur in normal situations, with improperly formatted
				//		input data
				log.Error("Got back bad block!")
				break
			}
			nodes[i] = nd
			for { //Check for duplicate links
				ni, err := FindLink(root, blk.Key(), nodes)
				if err != nil {
					break
				}
				nodes[ni] = nd
			}

			if next == i {
				sig <- nd
				next++
				for ; next < len(nodes) && nodes[next] != nil; next++ {
					sig <- nodes[next]
				}
			}
		}
		if next < len(nodes) {
			// TODO: bubble errors back up.
			log.Errorf("Did not receive correct number of nodes!")
		}
	}()

	return sig
}
<|endoftext|>


<commit_before>package model

import (
	"encoding/json"
	"fmt"
	"strings"
	"time"
	"unicode/utf8"

	sarama "gopkg.in/Shopify/sarama.v1"
)

type Priority int
type Facility int
type Severity int
type Version int

type AuditSubMessage struct {
	Type uint16 `json:"type"`
	Data string `json:"data"`
}

type AuditMessageGroup struct {
	Seq       int                `json:"sequence"`
	AuditTime string             `json:"timestamp"`
	Msgs      []*AuditSubMessage `json:"messages"`
	UidMap    map[string]string  `json:"uid_map"`
}

type SyslogMessage struct {
	Priority         Priority               `json:"priority,string"`
	Facility         Facility               `json:"facility,string"`
	Severity         Severity               `json:"severity,string"`
	Version          Version                `json:"version,string"`
	TimeReported     time.Time              `json:"timereported,omitempty"`
	TimeGenerated    time.Time              `json:"timegenerated,omitempty"`
	Hostname         string                 `json:"hostname"`
	Appname          string                 `json:"appname"`
	Procid           string                 `json:"procid"`
	Msgid            string                 `json:"msgid"`
	Structured       string                 `json:"structured"`
	Message          string                 `json:"message"`
	AuditSubMessages []*AuditSubMessage     `json:"audit,omitempty"`
	Properties       map[string]interface{} `json:"properties,omitempty"`
}

type RawMessage struct {
	Message        string
	Client         string
	LocalPort      int
	UnixSocketPath string
}

type ParsedMessage struct {
	Fields         *SyslogMessage `json:"fields"`
	Client         string         `json:"client"`
	LocalPort      int            `json:"local_port,string"`
	UnixSocketPath string         `json:"unix_socket_path"`
}

type TcpUdpParsedMessage struct {
	Parsed *ParsedMessage `json:"parsed"`
	Uid    string         `json:"uid"`
	ConfId string         `json:"conf_id"`
}

type RelpRawMessage struct {
	Raw  *RawMessage
	Txnr int
}

type RelpParsedMessage struct {
	Parsed *ParsedMessage `json:"parsed"`
	Txnr   int            `json:"txnr"`
}

func (m *ParsedMessage) ToKafkaMessage(partitionKey string, topic string) (km *sarama.ProducerMessage, err error) {
	value, err := json.Marshal(m)
	if err != nil {
		return nil, err
	}

	kafka_msg := sarama.ProducerMessage{
		Key:       sarama.StringEncoder(partitionKey),
		Value:     sarama.ByteEncoder(value),
		Topic:     topic,
		Timestamp: m.Fields.TimeReported,
	}
	return &kafka_msg, nil
}

var SyslogMessageFmt string = `Facility: %d
Severity: %d
Version: %d
TimeReported: %s
TimeGenerated: %s
Hostname: %s
Appname: %s
ProcID: %s
MsgID: %s
Structured: %s
Message: %s
AuditSubMessages: %s
Properties: %s`

func (m *SyslogMessage) String() string {
	props := ""
	b, err := json.Marshal(m.Properties)
	if err == nil {
		props = string(b)
	}
	subs := ""
	b, err = json.Marshal(m.AuditSubMessages)
	if err == nil {
		subs = string(b)
	}
	return fmt.Sprintf(
		SyslogMessageFmt,
		m.Facility,
		m.Severity,
		m.Version,
		m.TimeReported.Format(time.RFC3339),
		m.TimeGenerated.Format(time.RFC3339),
		m.Hostname,
		m.Appname,
		m.Procid,
		m.Msgid,
		m.Structured,
		m.Message,
		subs,
		props,
	)
}

type Parser struct {
	format string
}

func (p *Parser) Parse(m string, dont_parse_sd bool) (*SyslogMessage, error) {
	return Parse(m, p.format, dont_parse_sd)
}

func GetParser(format string) *Parser {
	if format == "rfc5424" || format == "rfc3164" || format == "json" || format == "auto" {
		return &Parser{format: format}
	}
	return nil
}

func Parse(m string, format string, dont_parse_sd bool) (sm *SyslogMessage, err error) {

	switch format {
	case "rfc5424":
		sm, err = ParseRfc5424Format(m, dont_parse_sd)
	case "rfc3164":
		sm, err = ParseRfc3164Format(m)
	case "json":
		sm, err = ParseJsonFormat(m)
	case "auto":
		if m[0] == byte('{') {
			sm, err = ParseJsonFormat(m)
		} else if m[0] != byte('<') {
			sm, err = ParseRfc3164Format(m)
		} else {
			i := strings.Index(m, ">")
			if i < 2 {
				sm, err = ParseRfc3164Format(m)
			} else if len(m) == (i + 1) {
				sm, err = ParseRfc3164Format(m)
			} else if m[i+1] == byte('1') {
				sm, err = ParseRfc5424Format(m, dont_parse_sd)
			} else {
				sm, err = ParseRfc3164Format(m)
			}
		}

	default:
		return nil, &UnknownFormatError{format}
	}
	if err != nil {
		return nil, err
	}
	// special handling of JSON messages produced by go-audit
	if sm.Appname == "go-audit" {
		var auditMsg AuditMessageGroup
		err = json.Unmarshal([]byte(sm.Message), &auditMsg)
		if err != nil {
			return sm, nil
		}
		sm.AuditSubMessages = auditMsg.Msgs
		if len(auditMsg.UidMap) > 0 {
			if sm.Properties == nil {
				sm.Properties = map[string]interface{}{}
			}
			props := map[string]map[string]string{}
			props["uid_map"] = auditMsg.UidMap
			sm.Properties["audit"] = props
		}
		sm.Message = ""
	}
	return sm, nil
}

func TopicNameIsValid(name string) bool {
	if len(name) == 0 {
		return false
	}
	if len(name) > 249 {
		return false
	}
	if !utf8.ValidString(name) {
		return false
	}
	for _, r := range name {
		if !validRune(r) {
			return false
		}
	}
	return true
}

func validRune(r rune) bool {
	if r >= 'a' && r <= 'z' {
		return true
	}
	if r >= 'A' && r <= 'Z' {
		return true
	}
	if r >= '0' && r <= '9' {
		return true
	}
	if r == '.' {
		return true
	}
	if r == '_' {
		return true
	}
	if r == '-' {
		return true
	}
	return false
}
<commit_msg>omitempty some fields<commit_after>package model

import (
	"encoding/json"
	"fmt"
	"strings"
	"time"
	"unicode/utf8"

	sarama "gopkg.in/Shopify/sarama.v1"
)

type Priority int
type Facility int
type Severity int
type Version int

type AuditSubMessage struct {
	Type uint16 `json:"type"`
	Data string `json:"data"`
}

type AuditMessageGroup struct {
	Seq       int                `json:"sequence"`
	AuditTime string             `json:"timestamp"`
	Msgs      []*AuditSubMessage `json:"messages"`
	UidMap    map[string]string  `json:"uid_map"`
}

type SyslogMessage struct {
	Priority         Priority               `json:"priority,string"`
	Facility         Facility               `json:"facility,string"`
	Severity         Severity               `json:"severity,string"`
	Version          Version                `json:"version,string"`
	TimeReported     time.Time              `json:"timereported,omitempty"`
	TimeGenerated    time.Time              `json:"timegenerated,omitempty"`
	Hostname         string                 `json:"hostname"`
	Appname          string                 `json:"appname"`
	Procid           string                 `json:"procid,omitempty"`
	Msgid            string                 `json:"msgid,omitempty"`
	Structured       string                 `json:"structured,omitempty"`
	Message          string                 `json:"message"`
	AuditSubMessages []*AuditSubMessage     `json:"audit,omitempty"`
	Properties       map[string]interface{} `json:"properties,omitempty"`
}

type RawMessage struct {
	Message        string
	Client         string
	LocalPort      int
	UnixSocketPath string
}

type ParsedMessage struct {
	Fields         *SyslogMessage `json:"fields"`
	Client         string         `json:"client,omitempty"`
	LocalPort      int            `json:"local_port,string"`
	UnixSocketPath string         `json:"unix_socket_path,omitempty"`
}

type TcpUdpParsedMessage struct {
	Parsed *ParsedMessage `json:"parsed"`
	Uid    string         `json:"uid"`
	ConfId string         `json:"conf_id"`
}

type RelpRawMessage struct {
	Raw  *RawMessage
	Txnr int
}

type RelpParsedMessage struct {
	Parsed *ParsedMessage `json:"parsed"`
	Txnr   int            `json:"txnr"`
}

func (m *ParsedMessage) ToKafkaMessage(partitionKey string, topic string) (km *sarama.ProducerMessage, err error) {
	value, err := json.Marshal(m)
	if err != nil {
		return nil, err
	}

	kafka_msg := sarama.ProducerMessage{
		Key:       sarama.StringEncoder(partitionKey),
		Value:     sarama.ByteEncoder(value),
		Topic:     topic,
		Timestamp: m.Fields.TimeReported,
	}
	return &kafka_msg, nil
}

var SyslogMessageFmt string = `Facility: %d
Severity: %d
Version: %d
TimeReported: %s
TimeGenerated: %s
Hostname: %s
Appname: %s
ProcID: %s
MsgID: %s
Structured: %s
Message: %s
AuditSubMessages: %s
Properties: %s`

func (m *SyslogMessage) String() string {
	props := ""
	b, err := json.Marshal(m.Properties)
	if err == nil {
		props = string(b)
	}
	subs := ""
	b, err = json.Marshal(m.AuditSubMessages)
	if err == nil {
		subs = string(b)
	}
	return fmt.Sprintf(
		SyslogMessageFmt,
		m.Facility,
		m.Severity,
		m.Version,
		m.TimeReported.Format(time.RFC3339),
		m.TimeGenerated.Format(time.RFC3339),
		m.Hostname,
		m.Appname,
		m.Procid,
		m.Msgid,
		m.Structured,
		m.Message,
		subs,
		props,
	)
}

type Parser struct {
	format string
}

func (p *Parser) Parse(m string, dont_parse_sd bool) (*SyslogMessage, error) {
	return Parse(m, p.format, dont_parse_sd)
}

func GetParser(format string) *Parser {
	if format == "rfc5424" || format == "rfc3164" || format == "json" || format == "auto" {
		return &Parser{format: format}
	}
	return nil
}

func Parse(m string, format string, dont_parse_sd bool) (sm *SyslogMessage, err error) {

	switch format {
	case "rfc5424":
		sm, err = ParseRfc5424Format(m, dont_parse_sd)
	case "rfc3164":
		sm, err = ParseRfc3164Format(m)
	case "json":
		sm, err = ParseJsonFormat(m)
	case "auto":
		if m[0] == byte('{') {
			sm, err = ParseJsonFormat(m)
		} else if m[0] != byte('<') {
			sm, err = ParseRfc3164Format(m)
		} else {
			i := strings.Index(m, ">")
			if i < 2 {
				sm, err = ParseRfc3164Format(m)
			} else if len(m) == (i + 1) {
				sm, err = ParseRfc3164Format(m)
			} else if m[i+1] == byte('1') {
				sm, err = ParseRfc5424Format(m, dont_parse_sd)
			} else {
				sm, err = ParseRfc3164Format(m)
			}
		}

	default:
		return nil, &UnknownFormatError{format}
	}
	if err != nil {
		return nil, err
	}
	// special handling of JSON messages produced by go-audit
	if sm.Appname == "go-audit" {
		var auditMsg AuditMessageGroup
		err = json.Unmarshal([]byte(sm.Message), &auditMsg)
		if err != nil {
			return sm, nil
		}
		sm.AuditSubMessages = auditMsg.Msgs
		if len(auditMsg.UidMap) > 0 {
			if sm.Properties == nil {
				sm.Properties = map[string]interface{}{}
			}
			props := map[string]map[string]string{}
			props["uid_map"] = auditMsg.UidMap
			sm.Properties["audit"] = props
		}
		sm.Message = ""
	}
	return sm, nil
}

func TopicNameIsValid(name string) bool {
	if len(name) == 0 {
		return false
	}
	if len(name) > 249 {
		return false
	}
	if !utf8.ValidString(name) {
		return false
	}
	for _, r := range name {
		if !validRune(r) {
			return false
		}
	}
	return true
}

func validRune(r rune) bool {
	if r >= 'a' && r <= 'z' {
		return true
	}
	if r >= 'A' && r <= 'Z' {
		return true
	}
	if r >= '0' && r <= '9' {
		return true
	}
	if r == '.' {
		return true
	}
	if r == '_' {
		return true
	}
	if r == '-' {
		return true
	}
	return false
}
<|endoftext|>


<commit_before>package main

import (
	"common"
	"common/crypto"
	"common/erasure"
	"fmt"
)

// uploadSector splits a Sector into erasure-coded segments and distributes them across a quorum.
func uploadSector(mr common.MessageRouter, sec *common.Sector, k int, quorum [common.QuorumSize]common.Address) (ring *common.Ring, err error) {
	// create erasure-coded segments
	ring, err = erasure.EncodeRing(sec, k)
	if err != nil {
		return
	}

	// for now we just send segment i to node i
	// this may need to be randomized for security
	for i := range quorum {
		m := &common.Message{
			quorum[i],
			"ServerHandler.UploadSegment",
			ring.Segs[i],
			nil,
		}
		err = mr.SendMessage(m)
		if err != nil {
			return
		}
	}

	return
}

// downloadSector retrieves the erasure-coded segments corresponding to a given Sector from a quorum.
// It reconstructs the original data from the segments and returns the complete Sector
func downloadSector(mr common.MessageRouter, hash crypto.Hash, ring *common.Ring, quorum [common.QuorumSize]common.Address) (sec *common.Sector, err error) {
	// send requests to each member of the quorum
	for i := range quorum {
		var seg common.Segment
		m := &common.Message{
			quorum[i],
			"ServerHandler.DownloadSegment",
			hash,
			&seg,
		}
		if mr.SendMessage(m) == nil {
			ring.AddSegment(&seg)
		}
	}

	// rebuild file
	sec, err = erasure.RebuildSector(ring)
	return
}

func main() {
	var input string
	for {
		print("Please enter a command:")
		fmt.Scanln(&input)

		switch input {
		default:
			println("unrecognized command")
		case "j":
			println("joining quorum")
		case "u":
			println("uploading file")
		case "q":
			return
		}
	}
}
<commit_msg>use global vars and new Ring in client<commit_after>package main

import (
	"common"
	"common/crypto"
	"common/erasure"
	"fmt"
	"network"
)

// global variables
// (with apologies to Haskell)
var (
	mr       common.MessageRouter
	SectorDB map[crypto.Hash]*common.Ring
)

// uploadSector splits a Sector into erasure-coded segments and distributes them across a quorum.
// It creates a Ring from its arguments and stores it in the SectorDB.
func uploadSector(sec *common.Sector, k int, q common.Quorum) (err error) {
	// create ring
	ring, segs, err := erasure.EncodeRing(sec, k)
	if err != nil {
		return
	}
	ring.Hosts = q

	// for now we just send segment i to node i
	// this may need to be randomized for security
	for i := range q {
		m := &common.Message{
			q[i],
			"Server.UploadSegment",
			segs[i],
			nil,
		}
		err = mr.SendMessage(m)
		if err != nil {
			return
		}
	}

	// add ring to SectorDB
	SectorDB[sec.Hash] = ring

	return
}

// downloadSector retrieves the erasure-coded segments corresponding to a given Sector from a quorum.
// It reconstructs the original data from the segments and returns the complete Sector
func downloadSector(hash crypto.Hash) (sec *common.Sector, err error) {
	// retrieve ring from SectorDB
	ring := SectorDB[hash]
	if ring == nil {
		err = fmt.Errorf("hash not present in database")
		return
	}

	// send requests to each member of the quorum
	var segs []common.Segment
	for i := range ring.Hosts {
		var seg common.Segment
		m := &common.Message{
			ring.Hosts[i],
			"Server.DownloadSegment",
			ring.SegHashes[i],
			&seg,
		}
		sendErr := mr.SendMessage(m)
		if sendErr == nil {
			segs = append(segs, seg)
		} else {
			fmt.Println(sendErr)
		}
	}

	// rebuild file
	sec, err = erasure.RebuildSector(ring, segs)
	return
}

func readQuorumAddresses() (q [common.QuorumSize]common.Address) {
	var input int
	for i := range q {
		fmt.Print("Please enter port number ", i, ": ")
		fmt.Scanln(&input)
		q[i] = common.Address{2, "localhost", input}
	}
	return
}

func main() {
	mr, _ = network.NewRPCServer(9989)
	defer mr.Close()
	SectorDB = make(map[crypto.Hash]*common.Ring)
	var (
		input string
		q     [common.QuorumSize]common.Address
		s, rs *common.Sector
		h, rh crypto.Hash
	)
	data, err := crypto.RandomByteSlice(70000)
	s, err = common.NewSector(data)
	h, err = crypto.CalculateHash(data)
	if err != nil {
		fmt.Println("error:", err)
	}
	for {
		fmt.Print("Please enter a command: ")
		fmt.Scanln(&input)

		switch input {
		default:
			fmt.Println("unrecognized command")
		case "j":
			fmt.Println("joining quorum")
			q = readQuorumAddresses()
			fmt.Println("connected to quorum")
		case "u":
			fmt.Println("uploading file")
			err = uploadSector(s, 2, q)
			if err != nil {
				fmt.Println("error:", err)
				fmt.Println("upload failed")
				break
			}
			fmt.Println("upload successful")
			fmt.Println("hash:", h[:])
		case "d":
			fmt.Println("downloading file")
			rs, err = downloadSector(h)
			if err != nil {
				fmt.Println("error:", err)
				fmt.Println("download failed")
				break
			}
			rh, err = crypto.CalculateHash(rs.Data)
			if err != nil {
				fmt.Println("error:", err)
				break
			}
			fmt.Println("download successful")
			fmt.Println("hash:", rh[:])
		case "q":
			return
		}
	}
}
<|endoftext|>


<commit_before>package ts3sqlib

import (
	"strconv"
	"strings"
)

//Client hold all information for a client from the clientlist command.
type Client struct {
	Cid              int    `json:"-"`
	Clid             int    `json:"clid"`
	ClientDatabaseId int    `json:"-"`
	ClientNickname   string `json:"client_nickname"`
	ClientType       int    `json:"-"`
}

//NewClient creates a Client datastructure from a map of strings
func NewClient(cmap map[string]string) Client {
	var newC Client

	newC.Cid, _ = strconv.Atoi(cmap["cid"])
	newC.Clid, _ = strconv.Atoi(cmap["clid"])
	newC.ClientDatabaseId, _ = strconv.Atoi(cmap["client_database_id"])
	newC.ClientNickname = cmap["client_nickname"]
	newC.ClientType, _ = strconv.Atoi(cmap["client_type"])

	return newC
}

//ClientmapsToClients converts an array of string maps to an array of Client's.
func ClientmapsToClients(clientmaps []map[string]string) (clients []Client, err error) {
	clients = make([]Client, len(clientmaps))

	for i, clientmap := range clientmaps {
		clients[i] = NewClient(clientmap)
	}

	return
}

//ClientlistToClients gets the clientlist from the ts3 server and returns it as
//a slice of Client's.
//The params are described in the TS3 ServerQuery Manual.
func (c *SqConn) ClientlistToClients(params string) (clients []Client, err error) {
	clientmaps, err := c.ClientlistToMaps(params)
	if err != nil {
		return
	}

	clients, err = ClientmapsToClients(clientmaps)
	return
}

//ClientlistToMaps gets the clientlist from the ts3 server and returns it as
//a slice of maps.
//The params are described in the TS3 ServerQuery Manual.
func (c *SqConn) ClientlistToMaps(params string) (clients []map[string]string, err error) {
	answer, err := c.Send("clientlist " + params + "\n")
	if err != nil {
		return
	}

	clients, err = MsgToMaps(answer)
	return
}

//MsgToMaps converts a given ts3 serverquery answer into a slice of maps,
//with key-value-pairs seperated by a '='.
func MsgToMaps(msg string) (parts []map[string]string, err error) {
	lines := strings.Split(msg, "|")
	parts = make([]map[string]string, len(lines))

	for i := range lines {
		parts[i] = make(map[string]string)

		lines[i] = strings.Replace(lines[i], "\n", "", -1)
		pairs := strings.Split(lines[i], " ")

		for j := range pairs {
			pair := strings.Split(pairs[j], "=")
			if len(pair) != 2 {
				continue
			}
			parts[i][pair[0]] = Unescape(pair[1])
		}
	}

	return
}
<commit_msg>changing struct field name<commit_after>package ts3sqlib

import (
	"strconv"
	"strings"
)

//Client hold all information for a client from the clientlist command.
type Client struct {
	Cid              int    `json:"-"`
	Clid             int    `json:"clid"`
	ClientDatabaseID int    `json:"-"`
	ClientNickname   string `json:"client_nickname"`
	ClientType       int    `json:"-"`
}

//NewClient creates a Client datastructure from a map of strings
func NewClient(cmap map[string]string) Client {
	var newC Client

	newC.Cid, _ = strconv.Atoi(cmap["cid"])
	newC.Clid, _ = strconv.Atoi(cmap["clid"])
	newC.ClientDatabaseID, _ = strconv.Atoi(cmap["client_database_id"])
	newC.ClientNickname = cmap["client_nickname"]
	newC.ClientType, _ = strconv.Atoi(cmap["client_type"])

	return newC
}

//ClientmapsToClients converts an array of string maps to an array of Client's.
func ClientmapsToClients(clientmaps []map[string]string) (clients []Client, err error) {
	clients = make([]Client, len(clientmaps))

	for i, clientmap := range clientmaps {
		clients[i] = NewClient(clientmap)
	}

	return
}

//ClientlistToClients gets the clientlist from the ts3 server and returns it as
//a slice of Client's.
//The params are described in the TS3 ServerQuery Manual.
func (c *SqConn) ClientlistToClients(params string) (clients []Client, err error) {
	clientmaps, err := c.ClientlistToMaps(params)
	if err != nil {
		return
	}

	clients, err = ClientmapsToClients(clientmaps)
	return
}

//ClientlistToMaps gets the clientlist from the ts3 server and returns it as
//a slice of maps.
//The params are described in the TS3 ServerQuery Manual.
func (c *SqConn) ClientlistToMaps(params string) (clients []map[string]string, err error) {
	answer, err := c.Send("clientlist " + params + "\n")
	if err != nil {
		return
	}

	clients, err = MsgToMaps(answer)
	return
}

//MsgToMaps converts a given ts3 serverquery answer into a slice of maps,
//with key-value-pairs seperated by a '='.
func MsgToMaps(msg string) (parts []map[string]string, err error) {
	lines := strings.Split(msg, "|")
	parts = make([]map[string]string, len(lines))

	for i := range lines {
		parts[i] = make(map[string]string)

		lines[i] = strings.Replace(lines[i], "\n", "", -1)
		pairs := strings.Split(lines[i], " ")

		for j := range pairs {
			pair := strings.Split(pairs[j], "=")
			if len(pair) != 2 {
				continue
			}
			parts[i][pair[0]] = Unescape(pair[1])
		}
	}

	return
}
<|endoftext|>


<commit_before>package maxcdn

import (
	"encoding/json"
	"errors"
	"fmt"
	"io/ioutil"
	"net/http"
	"net/url"
	"strings"
	"sync"

	"github.com/garyburd/go-oauth/oauth"
)

const (
	userAgent   = "Go MaxCDN API Client"
	contentType = "application/x-www-form-urlencoded"
	logsPath    = "/v3/reporting/logs.json"
)

// APIHost is the hostname, including protocol, to MaxCDN's API.
var APIHost = "https://rws.netdna.com"

// MaxCDN is the core struct for interacting with MaxCDN.
//
// HTTPClient can be overridden as needed, but will be set to
// http.DefaultClient by default.
type MaxCDN struct {

	// MaxCDN Consumer Alias
	Alias string

	// Display raw http Request and Response for each http Transport
	Verbose    bool
	client     oauth.Client
	HTTPClient *http.Client
}

// NewMaxCDN sets up a new MaxCDN instance.
func NewMaxCDN(alias, token, secret string) *MaxCDN {
	return &MaxCDN{
		HTTPClient: http.DefaultClient,
		Alias:      alias,
		client: oauth.Client{
			Credentials: oauth.Credentials{
				Token:  token,
				Secret: secret,
			},
			TemporaryCredentialRequestURI: APIHost + "oauth/request_token",
			TokenRequestURI:               APIHost + "oauth/access_token",
		},
	}
}

// Get does an OAuth signed http.Get
func (max *MaxCDN) Get(endpointType interface{}, endpoint string, form url.Values) (*Response, error) {
	return max.DoParse(endpointType, "GET", endpoint, form)
}

// GetLogs is a seperate getter for MaxCDN's logs.json endpoint, as it currently doesn't follow
// the json format of other endpoints.
func (max *MaxCDN) GetLogs(form url.Values) (Logs, error) {
	var logs Logs
	rsp, err := max.Request("GET", logsPath, form)
	defer rsp.Body.Close()
	if err != nil {
		return logs, err
	}

	var raw []byte
	raw, err = ioutil.ReadAll(rsp.Body)
	if err != nil {
		return logs, err
	}

	err = json.Unmarshal(raw, &logs)
	return logs, err
}

// Post does an OAuth signed http.Post
func (max *MaxCDN) Post(endpointType interface{}, endpoint string, form url.Values) (*Response, error) {
	return max.DoParse(endpointType, "POST", endpoint, form)
}

// Put does an OAuth signed http.Put
func (max *MaxCDN) Put(endpointType interface{}, endpoint string, form url.Values) (*Response, error) {
	return max.DoParse(endpointType, "PUT", endpoint, form)
}

// Delete does an OAuth signed http.Delete
//
// Delete does not take an endpointType because delete only returns a status code.
func (max *MaxCDN) Delete(endpoint string, form url.Values) (*Response, error) {
	return max.Do("DELETE", endpoint, form)
}

// PurgeZone purges a specified zones cache.
func (max *MaxCDN) PurgeZone(zone int) (*Response, error) {
	return max.Delete(fmt.Sprintf("/zones/pull.json/%d/cache", zone), nil)
}

// PurgeZoneString purges a specified zones cache.
func (max *MaxCDN) PurgeZoneString(zone string) (*Response, error) {
	return max.Delete(fmt.Sprintf("/zones/pull.json/%s/cache", zone), nil)
}

// PurgeZonesString purges multiple zones caches.
func (max *MaxCDN) PurgeZonesString(zones []string) (resps []*Response, last error) {
	var resChannel = make(chan *Response)
	var errChannel = make(chan error)

	mutex := sync.Mutex{}
	for _, zone := range zones {
		go func(zone string) {
			res, err := max.PurgeZoneString(zone)
			resChannel <- res
			errChannel <- err
		}(zone)
	}

	// Wait for all responses to come back.
	// TODO: Consider adding some method of timing out.
	for _ = range zones {
		res := <-resChannel
		err := <-errChannel

		// I think the mutex might be overkill here, but I'm being
		// safe.
		mutex.Lock()
		resps = append(resps, res)
		last = err
		mutex.Unlock()
	}
	return
}

// PurgeZones purges multiple zones caches.
func (max *MaxCDN) PurgeZones(zones []int) (resps []*Response, last error) {
	zoneStrings := make([]string, len(zones))

	for i, zone := range zones {
		zoneStrings[i] = fmt.Sprintf("%d", zone)
	}

	return max.PurgeZonesString(zoneStrings)
}

// PurgeFile purges a specified file by zone from cache.
func (max *MaxCDN) PurgeFile(zone int, file string) (*Response, error) {
	return max.PurgeFileString(fmt.Sprintf("%d", zone), file)
}

// PurgeFile purges a specified file by zone from cache.
func (max *MaxCDN) PurgeFileString(zone string, file string) (*Response, error) {
	form := url.Values{}
	form.Set("file", file)

	return max.Delete(fmt.Sprintf("/zones/pull.json/%s/cache", zone), form)
}

// PurgeFiles purges multiple files from a zone.
func (max *MaxCDN) PurgeFiles(zone int, files []string) (resps []*Response, last error) {
	var resChannel = make(chan *Response)
	var errChannel = make(chan error)

	mutex := sync.Mutex{}
	for _, file := range files {
		go func(file string) {
			res, err := max.PurgeFile(zone, file)

			resChannel <- res
			errChannel <- err
		}(file)
	}

	// Wait for all responses to come back.
	// TODO: Consider adding some method of timing out.
	for _ = range files {
		res := <-resChannel
		err := <-errChannel

		// I think the mutex might be overkill here, but I'm being
		// safe.
		mutex.Lock()
		resps = append(resps, res)
		last = err
		mutex.Unlock()
	}
	return
}

func (max *MaxCDN) DoParse(endpointType interface{}, method, endpoint string, form url.Values) (rsp *Response, err error) {
	rsp, err = max.Do(method, endpoint, form)
	if err != nil {
		return
	}
	err = json.Unmarshal(rsp.Data, &endpointType)
	return
}

// Do is a low level method to interact with MaxCDN's RESTful API via Request
// and return a parsed Response. It's used by all other methods.
//
// This method closes the raw http.Response body.
func (max *MaxCDN) Do(method, endpoint string, form url.Values) (rsp *Response, err error) {
	rsp = new(Response)
	res, err := max.Request(method, endpoint, form)
	defer res.Body.Close()

	if err != nil {
		return
	}

	headers := res.Header
	rsp.Headers = &headers

	var raw []byte
	raw, err = ioutil.ReadAll(res.Body)
	if err != nil {
		return
	}

	err = json.Unmarshal(raw, &rsp)
	if err != nil {
		return
	}

	if rsp.Code > 299 {
		return rsp, fmt.Errorf("%s: %s", rsp.Error.Type, rsp.Error.Message)
	}

	return
}

// Request is a low level method to interact with MaxCDN's RESTful API. It's
// used by all other methods.
//
// If using this method, you must manually close the res.Body or bad things
// may happen.
func (max *MaxCDN) Request(method, endpoint string, form url.Values) (res *http.Response, err error) {
	var req *http.Request

	req, err = http.NewRequest(method, max.url(endpoint), nil)
	if err != nil {
		return
	}

	if method == "GET" && req.URL.RawQuery != "" {
		return nil, errors.New("oauth: url must not contain a query string")
	}

	if form != nil {
		if method == "GET" {
			req.URL.RawQuery = form.Encode()
		} else {
			req.Body = ioutil.NopCloser(strings.NewReader(form.Encode()))
		}

		// Only post needs a signed form.
		if method != "POST" {
			form = nil
		}
	}

	req.Header.Set("Authorization", max.client.AuthorizationHeader(nil, method, req.URL, form))
	req.Header.Set("Content-Type", contentType)
	req.Header.Set("User-Agent", userAgent)

	if max.Verbose {
		if j, e := json.MarshalIndent(req, "", "  "); e == nil {
			fmt.Printf("Request: %s\n---\n\n", j)
		}
	}

	res, err = max.HTTPClient.Do(req)
	if max.Verbose {
		if j, e := json.MarshalIndent(res, "", "  "); e == nil {
			fmt.Printf("Response: %s\n---\n\n", j)
		}
	}
	return
}

func (max *MaxCDN) url(endpoint string) string {
	endpoint = strings.TrimPrefix(endpoint, "/")
	return fmt.Sprintf("%s/%s/%s", APIHost, max.Alias, endpoint)
}
<commit_msg>updating api domain, netdna to maxcdn<commit_after>package maxcdn

import (
	"encoding/json"
	"errors"
	"fmt"
	"io/ioutil"
	"net/http"
	"net/url"
	"strings"
	"sync"

	"github.com/garyburd/go-oauth/oauth"
)

const (
	userAgent   = "Go MaxCDN API Client"
	contentType = "application/x-www-form-urlencoded"
	logsPath    = "/v3/reporting/logs.json"
)

// APIHost is the hostname, including protocol, to MaxCDN's API.
var APIHost = "https://rws.maxcdn.com"

// MaxCDN is the core struct for interacting with MaxCDN.
//
// HTTPClient can be overridden as needed, but will be set to
// http.DefaultClient by default.
type MaxCDN struct {

	// MaxCDN Consumer Alias
	Alias string

	// Display raw http Request and Response for each http Transport
	Verbose    bool
	client     oauth.Client
	HTTPClient *http.Client
}

// NewMaxCDN sets up a new MaxCDN instance.
func NewMaxCDN(alias, token, secret string) *MaxCDN {
	return &MaxCDN{
		HTTPClient: http.DefaultClient,
		Alias:      alias,
		client: oauth.Client{
			Credentials: oauth.Credentials{
				Token:  token,
				Secret: secret,
			},
			TemporaryCredentialRequestURI: APIHost + "oauth/request_token",
			TokenRequestURI:               APIHost + "oauth/access_token",
		},
	}
}

// Get does an OAuth signed http.Get
func (max *MaxCDN) Get(endpointType interface{}, endpoint string, form url.Values) (*Response, error) {
	return max.DoParse(endpointType, "GET", endpoint, form)
}

// GetLogs is a seperate getter for MaxCDN's logs.json endpoint, as it currently doesn't follow
// the json format of other endpoints.
func (max *MaxCDN) GetLogs(form url.Values) (Logs, error) {
	var logs Logs
	rsp, err := max.Request("GET", logsPath, form)
	defer rsp.Body.Close()
	if err != nil {
		return logs, err
	}

	var raw []byte
	raw, err = ioutil.ReadAll(rsp.Body)
	if err != nil {
		return logs, err
	}

	err = json.Unmarshal(raw, &logs)
	return logs, err
}

// Post does an OAuth signed http.Post
func (max *MaxCDN) Post(endpointType interface{}, endpoint string, form url.Values) (*Response, error) {
	return max.DoParse(endpointType, "POST", endpoint, form)
}

// Put does an OAuth signed http.Put
func (max *MaxCDN) Put(endpointType interface{}, endpoint string, form url.Values) (*Response, error) {
	return max.DoParse(endpointType, "PUT", endpoint, form)
}

// Delete does an OAuth signed http.Delete
//
// Delete does not take an endpointType because delete only returns a status code.
func (max *MaxCDN) Delete(endpoint string, form url.Values) (*Response, error) {
	return max.Do("DELETE", endpoint, form)
}

// PurgeZone purges a specified zones cache.
func (max *MaxCDN) PurgeZone(zone int) (*Response, error) {
	return max.Delete(fmt.Sprintf("/zones/pull.json/%d/cache", zone), nil)
}

// PurgeZoneString purges a specified zones cache.
func (max *MaxCDN) PurgeZoneString(zone string) (*Response, error) {
	return max.Delete(fmt.Sprintf("/zones/pull.json/%s/cache", zone), nil)
}

// PurgeZonesString purges multiple zones caches.
func (max *MaxCDN) PurgeZonesString(zones []string) (resps []*Response, last error) {
	var resChannel = make(chan *Response)
	var errChannel = make(chan error)

	mutex := sync.Mutex{}
	for _, zone := range zones {
		go func(zone string) {
			res, err := max.PurgeZoneString(zone)
			resChannel <- res
			errChannel <- err
		}(zone)
	}

	// Wait for all responses to come back.
	// TODO: Consider adding some method of timing out.
	for _ = range zones {
		res := <-resChannel
		err := <-errChannel

		// I think the mutex might be overkill here, but I'm being
		// safe.
		mutex.Lock()
		resps = append(resps, res)
		last = err
		mutex.Unlock()
	}
	return
}

// PurgeZones purges multiple zones caches.
func (max *MaxCDN) PurgeZones(zones []int) (resps []*Response, last error) {
	zoneStrings := make([]string, len(zones))

	for i, zone := range zones {
		zoneStrings[i] = fmt.Sprintf("%d", zone)
	}

	return max.PurgeZonesString(zoneStrings)
}

// PurgeFile purges a specified file by zone from cache.
func (max *MaxCDN) PurgeFile(zone int, file string) (*Response, error) {
	return max.PurgeFileString(fmt.Sprintf("%d", zone), file)
}

// PurgeFile purges a specified file by zone from cache.
func (max *MaxCDN) PurgeFileString(zone string, file string) (*Response, error) {
	form := url.Values{}
	form.Set("file", file)

	return max.Delete(fmt.Sprintf("/zones/pull.json/%s/cache", zone), form)
}

// PurgeFiles purges multiple files from a zone.
func (max *MaxCDN) PurgeFiles(zone int, files []string) (resps []*Response, last error) {
	var resChannel = make(chan *Response)
	var errChannel = make(chan error)

	mutex := sync.Mutex{}
	for _, file := range files {
		go func(file string) {
			res, err := max.PurgeFile(zone, file)

			resChannel <- res
			errChannel <- err
		}(file)
	}

	// Wait for all responses to come back.
	// TODO: Consider adding some method of timing out.
	for _ = range files {
		res := <-resChannel
		err := <-errChannel

		// I think the mutex might be overkill here, but I'm being
		// safe.
		mutex.Lock()
		resps = append(resps, res)
		last = err
		mutex.Unlock()
	}
	return
}

func (max *MaxCDN) DoParse(endpointType interface{}, method, endpoint string, form url.Values) (rsp *Response, err error) {
	rsp, err = max.Do(method, endpoint, form)
	if err != nil {
		return
	}
	err = json.Unmarshal(rsp.Data, &endpointType)
	return
}

// Do is a low level method to interact with MaxCDN's RESTful API via Request
// and return a parsed Response. It's used by all other methods.
//
// This method closes the raw http.Response body.
func (max *MaxCDN) Do(method, endpoint string, form url.Values) (rsp *Response, err error) {
	rsp = new(Response)
	res, err := max.Request(method, endpoint, form)
	defer res.Body.Close()

	if err != nil {
		return
	}

	headers := res.Header
	rsp.Headers = &headers

	var raw []byte
	raw, err = ioutil.ReadAll(res.Body)
	if err != nil {
		return
	}

	err = json.Unmarshal(raw, &rsp)
	if err != nil {
		return
	}

	if rsp.Code > 299 {
		return rsp, fmt.Errorf("%s: %s", rsp.Error.Type, rsp.Error.Message)
	}

	return
}

// Request is a low level method to interact with MaxCDN's RESTful API. It's
// used by all other methods.
//
// If using this method, you must manually close the res.Body or bad things
// may happen.
func (max *MaxCDN) Request(method, endpoint string, form url.Values) (res *http.Response, err error) {
	var req *http.Request

	req, err = http.NewRequest(method, max.url(endpoint), nil)
	if err != nil {
		return
	}

	if method == "GET" && req.URL.RawQuery != "" {
		return nil, errors.New("oauth: url must not contain a query string")
	}

	if form != nil {
		if method == "GET" {
			req.URL.RawQuery = form.Encode()
		} else {
			req.Body = ioutil.NopCloser(strings.NewReader(form.Encode()))
		}

		// Only post needs a signed form.
		if method != "POST" {
			form = nil
		}
	}

	req.Header.Set("Authorization", max.client.AuthorizationHeader(nil, method, req.URL, form))
	req.Header.Set("Content-Type", contentType)
	req.Header.Set("User-Agent", userAgent)

	if max.Verbose {
		if j, e := json.MarshalIndent(req, "", "  "); e == nil {
			fmt.Printf("Request: %s\n---\n\n", j)
		}
	}

	res, err = max.HTTPClient.Do(req)
	if max.Verbose {
		if j, e := json.MarshalIndent(res, "", "  "); e == nil {
			fmt.Printf("Response: %s\n---\n\n", j)
		}
	}
	return
}

func (max *MaxCDN) url(endpoint string) string {
	endpoint = strings.TrimPrefix(endpoint, "/")
	return fmt.Sprintf("%s/%s/%s", APIHost, max.Alias, endpoint)
}
<|endoftext|>