<commit_before>//  Copyright 2019 Google Inc. All Rights Reserved.
//
//  Licensed under the Apache License, Version 2.0 (the "License");
//  you may not use this file except in compliance with the License.
//  You may obtain a copy of the License at
//
//      http://www.apache.org/licenses/LICENSE-2.0
//
//  Unless required by applicable law or agreed to in writing, software
//  distributed under the License is distributed on an "AS IS" BASIS,
//  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
//  See the License for the specific language governing permissions and
//  limitations under the License.

// GCE OVF import tool
package main

import (
	"flag"
	"fmt"
	"os"
	"path"
	"path/filepath"
	"strings"

	"github.com/GoogleCloudPlatform/compute-image-tools/cli_tools/common/utils/daisy"
	"github.com/GoogleCloudPlatform/compute-image-tools/cli_tools/common/utils/flags"
	"github.com/GoogleCloudPlatform/compute-image-tools/cli_tools/common/utils/logging/service"
	"github.com/GoogleCloudPlatform/compute-image-tools/cli_tools/gce_ovf_import/domain"
	ovfimporter "github.com/GoogleCloudPlatform/compute-image-tools/cli_tools/gce_ovf_import/ovf_importer"
)

var (
	instanceNames               = flag.String(ovfimporter.InstanceNameFlagKey, "", "VM Instance names to be created, separated by commas.")
	machineImageName            = flag.String(ovfimporter.MachineImageNameFlagKey, "", "Name of the machine image to create.")
	clientID                    = flag.String(ovfimporter.ClientIDFlagKey, "", "Identifies the client of the importer, e.g. `gcloud` or `pantheon`")
	clientVersion               = flag.String("client-version", "", "Identifies the version of the client of the importer")
	ovfOvaGcsPath               = flag.String(ovfimporter.OvfGcsPathFlagKey, "", " Google Cloud Storage URI of the OVF or OVA file to import. For example: gs://my-bucket/my-vm.ovf.")
	noGuestEnvironment          = flag.Bool("no-guest-environment", false, "Google Guest Environment will not be installed on the image.")
	canIPForward                = flag.Bool("can-ip-forward", false, "If provided, allows the instances to send and receive packets with non-matching destination or source IP addresses.")
	deletionProtection          = flag.Bool("deletion-protection", false, "Enables deletion protection for the instance.")
	description                 = flag.String("description", "", "Specifies a textual description of the instances.")
	labels                      = flag.String("labels", "", "List of label KEY=VALUE pairs to add. Keys must start with a lowercase character and contain only hyphens (-), underscores (_), lowercase characters, and numbers. Values must contain only hyphens (-), underscores (_), lowercase characters, and numbers.")
	machineType                 = flag.String("machine-type", "", "Specifies the machine type used for the instances. To get a list of available machine types, run 'gcloud compute machine-types list'. If unspecified, the default type is n1-standard-1.")
	network                     = flag.String("network", "", "Name of the network in your project to use for the image import. The network must have access to Google Cloud Storage. If not specified, the network named default is used. If -subnet is also specified subnet must be a subnetwork of network specified by -network.")
	networkTier                 = flag.String("network-tier", "", "Specifies the network tier that will be used to configure the instance. NETWORK_TIER must be one of: PREMIUM, STANDARD. The default value is PREMIUM.")
	subnet                      = flag.String("subnet", "", "Name of the subnetwork in your project to use for the image import. If	the network resource is in legacy mode, do not provide this property. If the network is in auto subnet mode, providing the subnetwork is optional. If the network is in custom subnet mode, then this field should be specified. Zone should be specified if this field is specified.")
	privateNetworkIP            = flag.String("private-network-ip", "", "Specifies the RFC1918 IP to assign to the instance. The IP should be in the subnet or legacy network IP range.")
	noExternalIP                = flag.Bool("no-external-ip", false, "Specifies that VPC into which instances is being imported doesn't allow external IPs.")
	noRestartOnFailure          = flag.Bool("no-restart-on-failure", false, "the instance will not be restarted if it’s terminated by Compute Engine. This does not affect terminations performed by the user.")
	osID                        = flag.String("os", "", "Specifies the OS of the image being imported. OS must be one of: "+strings.Join(daisy.GetSortedOSIDs(), ", ")+".")
	shieldedIntegrityMonitoring = flag.Bool("shielded-integrity-monitoring", false, "Enables monitoring and attestation of the boot integrity of the instance. The attestation is performed against the integrity policy baseline. This baseline is initially derived from the implicitly trusted boot image when the instance is created. This baseline can be updated by using --shielded-vm-learn-integrity-policy.")
	shieldedSecureBoot          = flag.Bool("shielded-secure-boot", false, "The instance will boot with secure boot enabled.")
	shieldedVtpm                = flag.Bool("shielded-vtpm", false, "The instance will boot with the TPM (Trusted Platform Module) enabled. A TPM is a hardware module that can be used for different security operations such as remote attestation, encryption and sealing of keys.")
	tags                        = flag.String("tags", "", "Specifies a list of tags to apply to the instance. These tags allow network firewall rules and routes to be applied to specified VM instances. See `gcloud compute firewall-rules create` for more details.")
	zoneFlag                    = flag.String("zone", "", "Zone of the image to import. The zone in which to do the work of importing the image. Overrides the default compute/zone property value for this command invocation")
	bootDiskKmskey              = flag.String("boot-disk-kms-key", "", "The Cloud KMS (Key Management Service) cryptokey that will be used to protect the disk. The arguments in this group can be used to specify the attributes of this resource. ID of the key or fully qualified identifier for the key. This flag must be specified if any of the other arguments in this group are specified.")
	bootDiskKmsKeyring          = flag.String("boot-disk-kms-keyring", "", "The KMS keyring of the key.")
	bootDiskKmsLocation         = flag.String("boot-disk-kms-location", "", "The Cloud location for the key.")
	bootDiskKmsProject          = flag.String("boot-disk-kms-project", "", "The Cloud project for the key.")
	timeout                     = flag.String("timeout", "", "Maximum time a build can last before it is failed as TIMEOUT. For example, specifying 2h will fail the process after 2 hours. See `gcloud topic datetimes` for information on duration formats")
	project                     = flag.String("project", "", "project to run in, overrides what is set in workflow")
	scratchBucketGcsPath        = flag.String("scratch-bucket-gcs-path", "", "GCS scratch bucket to use, overrides what is set in workflow")
	oauth                       = flag.String("oauth", "", "path to oauth json file, overrides what is set in workflow")
	ce                          = flag.String("compute-endpoint-override", "", "API endpoint to override default")
	gcsLogsDisabled             = flag.Bool("disable-gcs-logging", false, "do not stream logs to GCS")
	cloudLogsDisabled           = flag.Bool("disable-cloud-logging", false, "do not stream logs to Cloud Logging")
	stdoutLogsDisabled          = flag.Bool("disable-stdout-logging", false, "do not display individual workflow logs on stdout")
	releaseTrack                = flag.String("release-track", domain.GA, fmt.Sprintf("Release track of OVF import. One of: %s, %s or %s. Impacts which compute API release track is used by the import tool.", domain.Alpha, domain.Beta, domain.GA))
	uefiCompatible              = flag.Bool("uefi-compatible", false, "Enables UEFI booting, which is an alternative system boot method. Most public images use the GRUB bootloader as their primary boot method.")
	hostname                    = flag.String(ovfimporter.HostnameFlagKey, "", "Specify the hostname of the instance to be created. The specified hostname must be RFC1035 compliant.")
	machineImageStorageLocation = flag.String(ovfimporter.MachineImageStorageLocationFlagKey, "", "GCS bucket storage location of the machine image being imported (regional or multi-regional)")
	buildID                     = flag.String("build-id", "", "Cloud Build ID override. This flag should be used if auto-generated or build ID provided by Cloud Build is not appropriate. For example, if running multiple imports in parallel in a single Cloud Build run, sharing build ID could cause premature temporary resource clean-up resulting in import failures.")
	nodeAffinityLabelsFlag      flags.StringArrayFlag
	currentExecutablePath       string
)

func init() {
	currentExecutablePath = string(os.Args[0])
	flag.Var(&nodeAffinityLabelsFlag, "node-affinity-label", "Node affinity label used to determine sole tenant node to schedule this instance on. Label is of the format: <key>,<operator>,<value>,<value2>... where <operator> can be one of: IN, NOT. For example: workload,IN,prod,test is a label with key 'workload' and values 'prod' and 'test'. This flag can be specified multiple times for multiple labels.")
}

func buildOVFImportParams() *domain.OVFImportParams {
	// The tool expects the daisy_workflows directory to be in the same directory as the binary:
	//  parent/
	//    ├── gce_ovf_import
	//    └── daisy_workflows/
	workflowDir := path.Join(filepath.Dir(os.Args[0]), "daisy_workflows")

	flag.Parse()
	return &domain.OVFImportParams{InstanceNames: *instanceNames,
		MachineImageName: *machineImageName, ClientID: *clientID,
		OvfOvaGcsPath: *ovfOvaGcsPath, NoGuestEnvironment: *noGuestEnvironment,
		CanIPForward: *canIPForward, DeletionProtection: *deletionProtection, Description: *description,
		Labels: *labels, MachineType: *machineType, Network: *network, NetworkTier: *networkTier,
		Subnet: *subnet, PrivateNetworkIP: *privateNetworkIP, NoExternalIP: *noExternalIP,
		NoRestartOnFailure: *noRestartOnFailure, OsID: *osID,
		ShieldedIntegrityMonitoring: *shieldedIntegrityMonitoring, ShieldedSecureBoot: *shieldedSecureBoot,
		ShieldedVtpm: *shieldedVtpm, Tags: *tags, Zone: *zoneFlag, BootDiskKmskey: *bootDiskKmskey,
		BootDiskKmsKeyring: *bootDiskKmsKeyring, BootDiskKmsLocation: *bootDiskKmsLocation,
		BootDiskKmsProject: *bootDiskKmsProject, Timeout: *timeout, Project: project,
		ScratchBucketGcsPath: *scratchBucketGcsPath, Oauth: *oauth, Ce: *ce,
		GcsLogsDisabled: *gcsLogsDisabled, CloudLogsDisabled: *cloudLogsDisabled,
		StdoutLogsDisabled: *stdoutLogsDisabled, NodeAffinityLabelsFlag: nodeAffinityLabelsFlag,
		CurrentExecutablePath: currentExecutablePath, ReleaseTrack: *releaseTrack,
		UefiCompatible: *uefiCompatible, Hostname: *hostname,
		MachineImageStorageLocation: *machineImageStorageLocation, BuildID: *buildID,
		WorkflowDir: workflowDir,
	}
}

func runImport() (service.Loggable, error) {
	var ovfImporter *ovfimporter.OVFImporter
	var err error
	defer func() {
		if ovfImporter != nil {
			ovfImporter.CleanUp()
		}
	}()

	if ovfImporter, err = ovfimporter.NewOVFImporter(buildOVFImportParams()); err != nil {
		return nil, err
	}

	wf, err := ovfImporter.Import()
	return service.NewLoggableFromWorkflow(wf), err
}

func main() {
	flag.Parse()

	var paramLog service.InputParams
	var action string

	isInstanceImport := *instanceNames != ""
	if isInstanceImport {
		paramLog = createInstanceImportInputParams()
		action = service.InstanceImportAction
	} else {
		paramLog = createMachineImageImportInputParams()
		action = service.MachineImageImportAction
	}

	if err := service.RunWithServerLogging(action, paramLog, project, runImport); err != nil {
		os.Exit(1)
	}
}

func createInstanceImportInputParams() service.InputParams {
	return service.InputParams{
		InstanceImportParams: &service.InstanceImportParams{
			CommonParams: createCommonInputParams(),

			InstanceName:                *instanceNames,
			OvfGcsPath:                  *ovfOvaGcsPath,
			CanIPForward:                *canIPForward,
			DeletionProtection:          *deletionProtection,
			MachineType:                 *machineType,
			NetworkInterface:            *network,
			NetworkTier:                 *networkTier,
			PrivateNetworkIP:            *privateNetworkIP,
			NoExternalIP:                *noExternalIP,
			NoRestartOnFailure:          *noRestartOnFailure,
			OS:                          *osID,
			ShieldedIntegrityMonitoring: *shieldedIntegrityMonitoring,
			ShieldedSecureBoot:          *shieldedSecureBoot,
			ShieldedVtpm:                *shieldedVtpm,
			Tags:                        *tags,
			HasBootDiskKmsKey:           *bootDiskKmskey != "",
			HasBootDiskKmsKeyring:       *bootDiskKmsKeyring != "",
			HasBootDiskKmsLocation:      *bootDiskKmsLocation != "",
			HasBootDiskKmsProject:       *bootDiskKmsProject != "",
			NoGuestEnvironment:          *noGuestEnvironment,
			NodeAffinityLabel:           nodeAffinityLabelsFlag.String(),
		},
	}
}

func createMachineImageImportInputParams() service.InputParams {
	return service.InputParams{
		MachineImageImportParams: &service.MachineImageImportParams{
			CommonParams: createCommonInputParams(),

			MachineImageName:            *machineImageName,
			OvfGcsPath:                  *ovfOvaGcsPath,
			CanIPForward:                *canIPForward,
			DeletionProtection:          *deletionProtection,
			MachineType:                 *machineType,
			NetworkInterface:            *network,
			NetworkTier:                 *networkTier,
			PrivateNetworkIP:            *privateNetworkIP,
			NoExternalIP:                *noExternalIP,
			NoRestartOnFailure:          *noRestartOnFailure,
			OS:                          *osID,
			ShieldedIntegrityMonitoring: *shieldedIntegrityMonitoring,
			ShieldedSecureBoot:          *shieldedSecureBoot,
			ShieldedVtpm:                *shieldedVtpm,
			Tags:                        *tags,
			HasBootDiskKmsKey:           *bootDiskKmskey != "",
			HasBootDiskKmsKeyring:       *bootDiskKmsKeyring != "",
			HasBootDiskKmsLocation:      *bootDiskKmsLocation != "",
			HasBootDiskKmsProject:       *bootDiskKmsProject != "",
			NoGuestEnvironment:          *noGuestEnvironment,
			NodeAffinityLabel:           nodeAffinityLabelsFlag.String(),
			Hostname:                    *hostname,
			MachineImageStorageLocation: *machineImageStorageLocation,
		},
	}
}

func createCommonInputParams() *service.CommonParams {
	return &service.CommonParams{
		ClientID:                *clientID,
		ClientVersion:           *clientVersion,
		Network:                 *network,
		Subnet:                  *subnet,
		Zone:                    *zoneFlag,
		Timeout:                 *timeout,
		Project:                 *project,
		ObfuscatedProject:       service.Hash(*project),
		Labels:                  *labels,
		ScratchBucketGcsPath:    *scratchBucketGcsPath,
		Oauth:                   *oauth,
		ComputeEndpointOverride: *ce,
		DisableGcsLogging:       *gcsLogsDisabled,
		DisableCloudLogging:     *cloudLogsDisabled,
		DisableStdoutLogging:    *stdoutLogsDisabled,
	}
}
<commit_msg>OVF import: Specify default timeout (#1509)<commit_after>//  Copyright 2019 Google Inc. All Rights Reserved.
//
//  Licensed under the Apache License, Version 2.0 (the "License");
//  you may not use this file except in compliance with the License.
//  You may obtain a copy of the License at
//
//      http://www.apache.org/licenses/LICENSE-2.0
//
//  Unless required by applicable law or agreed to in writing, software
//  distributed under the License is distributed on an "AS IS" BASIS,
//  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
//  See the License for the specific language governing permissions and
//  limitations under the License.

// GCE OVF import tool
package main

import (
	"flag"
	"fmt"
	"os"
	"path"
	"path/filepath"
	"strings"

	"github.com/GoogleCloudPlatform/compute-image-tools/cli_tools/common/utils/daisy"
	"github.com/GoogleCloudPlatform/compute-image-tools/cli_tools/common/utils/flags"
	"github.com/GoogleCloudPlatform/compute-image-tools/cli_tools/common/utils/logging/service"
	"github.com/GoogleCloudPlatform/compute-image-tools/cli_tools/gce_ovf_import/domain"
	ovfimporter "github.com/GoogleCloudPlatform/compute-image-tools/cli_tools/gce_ovf_import/ovf_importer"
)

var (
	instanceNames               = flag.String(ovfimporter.InstanceNameFlagKey, "", "VM Instance names to be created, separated by commas.")
	machineImageName            = flag.String(ovfimporter.MachineImageNameFlagKey, "", "Name of the machine image to create.")
	clientID                    = flag.String(ovfimporter.ClientIDFlagKey, "", "Identifies the client of the importer, e.g. `gcloud` or `pantheon`")
	clientVersion               = flag.String("client-version", "", "Identifies the version of the client of the importer")
	ovfOvaGcsPath               = flag.String(ovfimporter.OvfGcsPathFlagKey, "", " Google Cloud Storage URI of the OVF or OVA file to import. For example: gs://my-bucket/my-vm.ovf.")
	noGuestEnvironment          = flag.Bool("no-guest-environment", false, "Google Guest Environment will not be installed on the image.")
	canIPForward                = flag.Bool("can-ip-forward", false, "If provided, allows the instances to send and receive packets with non-matching destination or source IP addresses.")
	deletionProtection          = flag.Bool("deletion-protection", false, "Enables deletion protection for the instance.")
	description                 = flag.String("description", "", "Specifies a textual description of the instances.")
	labels                      = flag.String("labels", "", "List of label KEY=VALUE pairs to add. Keys must start with a lowercase character and contain only hyphens (-), underscores (_), lowercase characters, and numbers. Values must contain only hyphens (-), underscores (_), lowercase characters, and numbers.")
	machineType                 = flag.String("machine-type", "", "Specifies the machine type used for the instances. To get a list of available machine types, run 'gcloud compute machine-types list'. If unspecified, the default type is n1-standard-1.")
	network                     = flag.String("network", "", "Name of the network in your project to use for the image import. The network must have access to Google Cloud Storage. If not specified, the network named default is used. If -subnet is also specified subnet must be a subnetwork of network specified by -network.")
	networkTier                 = flag.String("network-tier", "", "Specifies the network tier that will be used to configure the instance. NETWORK_TIER must be one of: PREMIUM, STANDARD. The default value is PREMIUM.")
	subnet                      = flag.String("subnet", "", "Name of the subnetwork in your project to use for the image import. If	the network resource is in legacy mode, do not provide this property. If the network is in auto subnet mode, providing the subnetwork is optional. If the network is in custom subnet mode, then this field should be specified. Zone should be specified if this field is specified.")
	privateNetworkIP            = flag.String("private-network-ip", "", "Specifies the RFC1918 IP to assign to the instance. The IP should be in the subnet or legacy network IP range.")
	noExternalIP                = flag.Bool("no-external-ip", false, "Specifies that VPC into which instances is being imported doesn't allow external IPs.")
	noRestartOnFailure          = flag.Bool("no-restart-on-failure", false, "the instance will not be restarted if it’s terminated by Compute Engine. This does not affect terminations performed by the user.")
	osID                        = flag.String("os", "", "Specifies the OS of the image being imported. OS must be one of: "+strings.Join(daisy.GetSortedOSIDs(), ", ")+".")
	shieldedIntegrityMonitoring = flag.Bool("shielded-integrity-monitoring", false, "Enables monitoring and attestation of the boot integrity of the instance. The attestation is performed against the integrity policy baseline. This baseline is initially derived from the implicitly trusted boot image when the instance is created. This baseline can be updated by using --shielded-vm-learn-integrity-policy.")
	shieldedSecureBoot          = flag.Bool("shielded-secure-boot", false, "The instance will boot with secure boot enabled.")
	shieldedVtpm                = flag.Bool("shielded-vtpm", false, "The instance will boot with the TPM (Trusted Platform Module) enabled. A TPM is a hardware module that can be used for different security operations such as remote attestation, encryption and sealing of keys.")
	tags                        = flag.String("tags", "", "Specifies a list of tags to apply to the instance. These tags allow network firewall rules and routes to be applied to specified VM instances. See `gcloud compute firewall-rules create` for more details.")
	zoneFlag                    = flag.String("zone", "", "Zone of the image to import. The zone in which to do the work of importing the image. Overrides the default compute/zone property value for this command invocation")
	bootDiskKmskey              = flag.String("boot-disk-kms-key", "", "The Cloud KMS (Key Management Service) cryptokey that will be used to protect the disk. The arguments in this group can be used to specify the attributes of this resource. ID of the key or fully qualified identifier for the key. This flag must be specified if any of the other arguments in this group are specified.")
	bootDiskKmsKeyring          = flag.String("boot-disk-kms-keyring", "", "The KMS keyring of the key.")
	bootDiskKmsLocation         = flag.String("boot-disk-kms-location", "", "The Cloud location for the key.")
	bootDiskKmsProject          = flag.String("boot-disk-kms-project", "", "The Cloud project for the key.")
	timeout                     = flag.String("timeout", "2h", "Maximum time a build can last before it is failed as TIMEOUT. For example, specifying 2h will fail the process after 2 hours. See `gcloud topic datetimes` for information on duration formats")
	project                     = flag.String("project", "", "project to run in, overrides what is set in workflow")
	scratchBucketGcsPath        = flag.String("scratch-bucket-gcs-path", "", "GCS scratch bucket to use, overrides what is set in workflow")
	oauth                       = flag.String("oauth", "", "path to oauth json file, overrides what is set in workflow")
	ce                          = flag.String("compute-endpoint-override", "", "API endpoint to override default")
	gcsLogsDisabled             = flag.Bool("disable-gcs-logging", false, "do not stream logs to GCS")
	cloudLogsDisabled           = flag.Bool("disable-cloud-logging", false, "do not stream logs to Cloud Logging")
	stdoutLogsDisabled          = flag.Bool("disable-stdout-logging", false, "do not display individual workflow logs on stdout")
	releaseTrack                = flag.String("release-track", domain.GA, fmt.Sprintf("Release track of OVF import. One of: %s, %s or %s. Impacts which compute API release track is used by the import tool.", domain.Alpha, domain.Beta, domain.GA))
	uefiCompatible              = flag.Bool("uefi-compatible", false, "Enables UEFI booting, which is an alternative system boot method. Most public images use the GRUB bootloader as their primary boot method.")
	hostname                    = flag.String(ovfimporter.HostnameFlagKey, "", "Specify the hostname of the instance to be created. The specified hostname must be RFC1035 compliant.")
	machineImageStorageLocation = flag.String(ovfimporter.MachineImageStorageLocationFlagKey, "", "GCS bucket storage location of the machine image being imported (regional or multi-regional)")
	buildID                     = flag.String("build-id", "", "Cloud Build ID override. This flag should be used if auto-generated or build ID provided by Cloud Build is not appropriate. For example, if running multiple imports in parallel in a single Cloud Build run, sharing build ID could cause premature temporary resource clean-up resulting in import failures.")
	nodeAffinityLabelsFlag      flags.StringArrayFlag
	currentExecutablePath       string
)

func init() {
	currentExecutablePath = string(os.Args[0])
	flag.Var(&nodeAffinityLabelsFlag, "node-affinity-label", "Node affinity label used to determine sole tenant node to schedule this instance on. Label is of the format: <key>,<operator>,<value>,<value2>... where <operator> can be one of: IN, NOT. For example: workload,IN,prod,test is a label with key 'workload' and values 'prod' and 'test'. This flag can be specified multiple times for multiple labels.")
}

func buildOVFImportParams() *domain.OVFImportParams {
	// The tool expects the daisy_workflows directory to be in the same directory as the binary:
	//  parent/
	//    ├── gce_ovf_import
	//    └── daisy_workflows/
	workflowDir := path.Join(filepath.Dir(os.Args[0]), "daisy_workflows")

	flag.Parse()
	return &domain.OVFImportParams{InstanceNames: *instanceNames,
		MachineImageName: *machineImageName, ClientID: *clientID,
		OvfOvaGcsPath: *ovfOvaGcsPath, NoGuestEnvironment: *noGuestEnvironment,
		CanIPForward: *canIPForward, DeletionProtection: *deletionProtection, Description: *description,
		Labels: *labels, MachineType: *machineType, Network: *network, NetworkTier: *networkTier,
		Subnet: *subnet, PrivateNetworkIP: *privateNetworkIP, NoExternalIP: *noExternalIP,
		NoRestartOnFailure: *noRestartOnFailure, OsID: *osID,
		ShieldedIntegrityMonitoring: *shieldedIntegrityMonitoring, ShieldedSecureBoot: *shieldedSecureBoot,
		ShieldedVtpm: *shieldedVtpm, Tags: *tags, Zone: *zoneFlag, BootDiskKmskey: *bootDiskKmskey,
		BootDiskKmsKeyring: *bootDiskKmsKeyring, BootDiskKmsLocation: *bootDiskKmsLocation,
		BootDiskKmsProject: *bootDiskKmsProject, Timeout: *timeout, Project: project,
		ScratchBucketGcsPath: *scratchBucketGcsPath, Oauth: *oauth, Ce: *ce,
		GcsLogsDisabled: *gcsLogsDisabled, CloudLogsDisabled: *cloudLogsDisabled,
		StdoutLogsDisabled: *stdoutLogsDisabled, NodeAffinityLabelsFlag: nodeAffinityLabelsFlag,
		CurrentExecutablePath: currentExecutablePath, ReleaseTrack: *releaseTrack,
		UefiCompatible: *uefiCompatible, Hostname: *hostname,
		MachineImageStorageLocation: *machineImageStorageLocation, BuildID: *buildID,
		WorkflowDir: workflowDir,
	}
}

func runImport() (service.Loggable, error) {
	var ovfImporter *ovfimporter.OVFImporter
	var err error
	defer func() {
		if ovfImporter != nil {
			ovfImporter.CleanUp()
		}
	}()

	if ovfImporter, err = ovfimporter.NewOVFImporter(buildOVFImportParams()); err != nil {
		return nil, err
	}

	wf, err := ovfImporter.Import()
	return service.NewLoggableFromWorkflow(wf), err
}

func main() {
	flag.Parse()

	var paramLog service.InputParams
	var action string

	isInstanceImport := *instanceNames != ""
	if isInstanceImport {
		paramLog = createInstanceImportInputParams()
		action = service.InstanceImportAction
	} else {
		paramLog = createMachineImageImportInputParams()
		action = service.MachineImageImportAction
	}

	if err := service.RunWithServerLogging(action, paramLog, project, runImport); err != nil {
		os.Exit(1)
	}
}

func createInstanceImportInputParams() service.InputParams {
	return service.InputParams{
		InstanceImportParams: &service.InstanceImportParams{
			CommonParams: createCommonInputParams(),

			InstanceName:                *instanceNames,
			OvfGcsPath:                  *ovfOvaGcsPath,
			CanIPForward:                *canIPForward,
			DeletionProtection:          *deletionProtection,
			MachineType:                 *machineType,
			NetworkInterface:            *network,
			NetworkTier:                 *networkTier,
			PrivateNetworkIP:            *privateNetworkIP,
			NoExternalIP:                *noExternalIP,
			NoRestartOnFailure:          *noRestartOnFailure,
			OS:                          *osID,
			ShieldedIntegrityMonitoring: *shieldedIntegrityMonitoring,
			ShieldedSecureBoot:          *shieldedSecureBoot,
			ShieldedVtpm:                *shieldedVtpm,
			Tags:                        *tags,
			HasBootDiskKmsKey:           *bootDiskKmskey != "",
			HasBootDiskKmsKeyring:       *bootDiskKmsKeyring != "",
			HasBootDiskKmsLocation:      *bootDiskKmsLocation != "",
			HasBootDiskKmsProject:       *bootDiskKmsProject != "",
			NoGuestEnvironment:          *noGuestEnvironment,
			NodeAffinityLabel:           nodeAffinityLabelsFlag.String(),
		},
	}
}

func createMachineImageImportInputParams() service.InputParams {
	return service.InputParams{
		MachineImageImportParams: &service.MachineImageImportParams{
			CommonParams: createCommonInputParams(),

			MachineImageName:            *machineImageName,
			OvfGcsPath:                  *ovfOvaGcsPath,
			CanIPForward:                *canIPForward,
			DeletionProtection:          *deletionProtection,
			MachineType:                 *machineType,
			NetworkInterface:            *network,
			NetworkTier:                 *networkTier,
			PrivateNetworkIP:            *privateNetworkIP,
			NoExternalIP:                *noExternalIP,
			NoRestartOnFailure:          *noRestartOnFailure,
			OS:                          *osID,
			ShieldedIntegrityMonitoring: *shieldedIntegrityMonitoring,
			ShieldedSecureBoot:          *shieldedSecureBoot,
			ShieldedVtpm:                *shieldedVtpm,
			Tags:                        *tags,
			HasBootDiskKmsKey:           *bootDiskKmskey != "",
			HasBootDiskKmsKeyring:       *bootDiskKmsKeyring != "",
			HasBootDiskKmsLocation:      *bootDiskKmsLocation != "",
			HasBootDiskKmsProject:       *bootDiskKmsProject != "",
			NoGuestEnvironment:          *noGuestEnvironment,
			NodeAffinityLabel:           nodeAffinityLabelsFlag.String(),
			Hostname:                    *hostname,
			MachineImageStorageLocation: *machineImageStorageLocation,
		},
	}
}

func createCommonInputParams() *service.CommonParams {
	return &service.CommonParams{
		ClientID:                *clientID,
		ClientVersion:           *clientVersion,
		Network:                 *network,
		Subnet:                  *subnet,
		Zone:                    *zoneFlag,
		Timeout:                 *timeout,
		Project:                 *project,
		ObfuscatedProject:       service.Hash(*project),
		Labels:                  *labels,
		ScratchBucketGcsPath:    *scratchBucketGcsPath,
		Oauth:                   *oauth,
		ComputeEndpointOverride: *ce,
		DisableGcsLogging:       *gcsLogsDisabled,
		DisableCloudLogging:     *cloudLogsDisabled,
		DisableStdoutLogging:    *stdoutLogsDisabled,
	}
}
<|endoftext|>


<commit_before>package main

import (
	"encoding/json"
	"fmt"
	"io"
	"io/ioutil"
	"log"
	"mime"
	"mime/multipart"
	"net/http"
	"strings"
)

type inbound_parse struct {
	Test1 string
	Test2 string
}

type Configuration struct {
	Endpoint string `json:"endpoint"`
	Port     string `json:"port"`
}

func LoadConfig(path string) Configuration {
	file, err := ioutil.ReadFile(path)
	if err != nil {
		log.Fatal("Config File Missing. ", err)
	}
	var conf Configuration
	err = json.Unmarshal(file, &conf)
	if err != nil {
		log.Fatal("Config Parse Error: ", err)
	}

	return conf
}

func indexHandler(w http.ResponseWriter, r *http.Request) {
	fmt.Fprintf(w, "%s", "Hello World")
}

func inboundHandler(w http.ResponseWriter, r *http.Request) {
	mediaType, params, err := mime.ParseMediaType(r.Header.Get("Content-Type"))
	if err != nil {
		log.Fatal(err)
	}
	if strings.HasPrefix(mediaType, "multipart/") {
		mr := multipart.NewReader(r.Body, params["boundary"])
		for {
			p, err := mr.NextPart()
			if err == io.EOF {
				fmt.Printf("%s\n", "End of File")
				w.WriteHeader(http.StatusOK)
				return
			}
			if err != nil {
				log.Fatal(err)
			}
			slurp, err := ioutil.ReadAll(p)
			if err != nil {
				log.Fatal(err)
			}
			header := p.Header.Get("Content-Disposition")
			header = header[17 : len(header)-1]
			fmt.Printf("%q: %q\n", header, slurp)
		}
	}
}

func main() {
	conf := LoadConfig("./conf.json")
	http.HandleFunc("/", indexHandler)
	http.HandleFunc(conf.Endpoint, inboundHandler)
	http.ListenAndServe(conf.Port, nil)
}
<commit_msg>Default Inbound Parse now handles attachments<commit_after>package main

import (
	"encoding/json"
	"fmt"
	"io"
	"io/ioutil"
	"log"
	"mime"
	"mime/multipart"
	"net/http"
	"strings"
)

type inbound_parse struct {
	Test1 string
	Test2 string
}

type Configuration struct {
	Endpoint string `json:"endpoint"`
	Port     string `json:"port"`
}

func LoadConfig(path string) Configuration {
	file, err := ioutil.ReadFile(path)
	if err != nil {
		log.Fatal("Config File Missing. ", err)
	}
	var conf Configuration
	err = json.Unmarshal(file, &conf)
	if err != nil {
		log.Fatal("Config Parse Error: ", err)
	}

	return conf
}

func indexHandler(w http.ResponseWriter, r *http.Request) {
	fmt.Fprintf(w, "%s", "Hello World")
}

func inboundHandler(w http.ResponseWriter, r *http.Request) {
	mediaType, params, err := mime.ParseMediaType(r.Header.Get("Content-Type"))
	if err != nil {
		log.Fatal(err)
	}
	if strings.HasPrefix(mediaType, "multipart/") {
		fmt.Println("================MESSAGE RECEIVED===============")
		mr := multipart.NewReader(r.Body, params["boundary"])
		parsedEmail := make(map[string]string)
		emailHeader := make(map[string]string)
		for {
			p, err := mr.NextPart()
			// We have found an attachment
			if err == nil && p.FileName() != "" {
				fmt.Println("FileName: ", p.FileName())
				contentType := p.Header.Get("Content-Type")
				fmt.Println("Content-Type: ", contentType)
				contents, err := ioutil.ReadAll(p)
				if err != nil {
					log.Fatal(err)
				}
				// Binary file contents
				fmt.Println("Contents: ", contents)
				// Only works with text files, this is just for testing
				fmt.Println("Contents Decoded: ", string(contents))
			}
			if err == io.EOF {
				// We have finished parsing
				for key, value := range parsedEmail {
					fmt.Println("Key:", key, " Value:", value)
				}
				for key, value := range emailHeader {
					fmt.Println("eKey:", key, " eValue:", value)
				}
				w.WriteHeader(http.StatusOK)
				return
			}
			if err != nil {
				log.Fatal(err)
			}
			value, err := ioutil.ReadAll(p)
			if err != nil {
				log.Fatal(err)
			}
			header := p.Header.Get("Content-Disposition")

			if strings.Contains(header, "filename") != true {
				header = header[17 : len(header)-1]
				parsedEmail[header] = string(value)
			} else {
				header = header[11 :]
				f := strings.Split(header, "=")
				parsedEmail[f[1][1:len(f[1])-11]] = f[2][1:len(f[2])-1]
			}
			if header == "headers" {
				s := strings.Split(string(value), "\n")
				var a []string
				for _, v := range s {
					t := strings.Split(string(v), ": ")
					a = append(a, t...)
				}
				for i := 0; i < len(a)-1; i += 2 {
					emailHeader[a[i]] = a[i+1]
				}
			}
			delete(parsedEmail, "headers")
		}
	}
}

func main() {
	conf := LoadConfig("./conf.json")
	http.HandleFunc("/", indexHandler)
	http.HandleFunc(conf.Endpoint, inboundHandler)
	http.ListenAndServe(conf.Port, nil)
}
<|endoftext|>


<commit_before>// Copyright 2018 Istio Authors
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package filewatcher

import (
	"bufio"
	"crypto/md5"
	"errors"
	"fmt"
	"io"
	"os"
	"path/filepath"
	"sync"

	"github.com/fsnotify/fsnotify"
	multierror "github.com/hashicorp/go-multierror"
)

// FileWatcher is an interface that watches a set of files,
// delivering events to related channel.
type FileWatcher interface {
	Add(path string) error
	Remove(path string) error
	Close() error
	Events(path string) chan fsnotify.Event
	Errors(path string) chan error
}

type fsNotifyWatcher struct {
	mu sync.RWMutex
	// The watcher maintain a map of workers,
	// keyed by watched dir (parent dir of watched files).
	workers map[string]*worker
}

type worker struct {
	// watcher is a fsnotify watcher that watches the parent
	// dir of watchedFiles.
	watcher *fsnotify.Watcher
	// The worker maintain a map of event channel (included in fileTracker),
	// keyed by watched file path.
	// The worker watches parent path of given path,
	// and filter out events of given path, then redirect
	// to the result channel.
	// Note that for symlink files, the content in received events
	// do not have to be related to the file itself.
	watchedFiles map[string]*fileTracker
}

type fileTracker struct {
	events chan fsnotify.Event
	errors chan error
	// md5 sum to indicate if a file has been updated.
	md5Sum string
}

// NewWatcher return with a FileWatcher instance that implemented with fsnotify.
func NewWatcher() FileWatcher {
	w := &fsNotifyWatcher{
		workers: map[string]*worker{},
	}

	return w
}

// Add is implementation of Add interface of FileWatcher.
func (w *fsNotifyWatcher) Add(path string) error {
	w.mu.Lock()
	defer w.mu.Unlock()

	cleanedPath, parentPath := formatPath(path)
	if worker, workerExist := w.workers[parentPath]; workerExist {
		if _, pathExist := worker.watchedFiles[cleanedPath]; pathExist {
			return fmt.Errorf("path %s already added", path)
		}

		// And the path into worker maps if not exist.
		md5Sum, err := getMd5Sum(cleanedPath)
		if err != nil {
			return fmt.Errorf("failed to get md5 sum for %s: %v", path, err)
		}
		worker.watchedFiles[cleanedPath] = &fileTracker{
			events: make(chan fsnotify.Event, 1),
			errors: make(chan error, 1),
			md5Sum: md5Sum,
		}

		return nil
	}

	// Build a new worker if not exist.
	watcher, err := fsnotify.NewWatcher()
	if err != nil {
		return err
	}
	if err = watcher.Add(parentPath); err != nil {
		watcher.Close()
		return err
	}

	var md5Sum string

	if _, err = os.Stat(cleanedPath); err == nil {
		md5Sum, err = getMd5Sum(cleanedPath)
		if err != nil {
			watcher.Close()
			return fmt.Errorf("failed to get md5 sum for %s: %v", path, err)
		}
	} else {
		if !os.IsNotExist(err) {
			watcher.Close()
			return fmt.Errorf("failed to stat file: %s: %v", path, err)
		}

		// if the file doesn't exist, ignore the md5 sum.
	}

	wk := &worker{
		watcher: watcher,
		watchedFiles: map[string]*fileTracker{
			cleanedPath: {
				events: make(chan fsnotify.Event, 1),
				errors: make(chan error, 1),
				md5Sum: md5Sum,
			},
		},
	}
	w.workers[parentPath] = wk

	go func() {
		for {
			select {
			case event, ok := <-watcher.Events:
				if !ok { // 'Events' channel is closed
					return
				}

				for path, tracker := range wk.watchedFiles {
					newSum, _ := getMd5Sum(path)
					if newSum != "" && newSum != tracker.md5Sum {
						tracker.md5Sum = newSum
						tracker.events <- event
						break
					}
				}
			case err, ok := <-watcher.Errors:
				if !ok {
					// 'Errors' channel is closed. Construct an error for it.
					// We'll only close worker errors channel in "Remove" for consistency.
					err = errors.New("channel closed")
				}
				for _, tracker := range wk.watchedFiles {
					tracker.errors <- err
				}
				return
			}
		}
	}()

	return nil
}

// Remove is implementation of Remove interface of FileWatcher.
func (w *fsNotifyWatcher) Remove(path string) error {
	w.mu.Lock()
	defer w.mu.Unlock()

	return w.remove(path)
}

func (w *fsNotifyWatcher) remove(path string) error {
	cleanedPath, parentPath := formatPath(path)

	worker, workerExist := w.workers[parentPath]
	if !workerExist {
		return fmt.Errorf("path %s already removed", path)
	}

	tracker, pathExist := worker.watchedFiles[cleanedPath]
	if !pathExist {
		return fmt.Errorf("path %s already removed", path)
	}

	delete(worker.watchedFiles, cleanedPath)
	close(tracker.events)
	close(tracker.errors)
	if len(worker.watchedFiles) == 0 {
		// Remove the watch if all of its paths have been removed.
		delete(w.workers, parentPath)
		return worker.watcher.Close()
	}

	return nil
}

// Close is implementation of Close interface of FileWatcher.
func (w *fsNotifyWatcher) Close() error {
	w.mu.Lock()
	defer w.mu.Unlock()

	var errors *multierror.Error
	for _, worker := range w.workers {
		for path := range worker.watchedFiles {
			errors = multierror.Append(errors, w.remove(path))
		}
	}

	return errors.ErrorOrNil()
}

// Events is implementation of Events interface of FileWatcher.
func (w *fsNotifyWatcher) Events(path string) chan fsnotify.Event {
	w.mu.RLock()
	defer w.mu.RUnlock()

	cleanedPath, parentPath := formatPath(path)
	worker, workerExist := w.workers[parentPath]
	if !workerExist {
		return nil
	}

	tracker, pathExist := worker.watchedFiles[cleanedPath]
	if !pathExist {
		return nil
	}

	return tracker.events
}

// Errors is implementation of Errors interface of FileWatcher.
func (w *fsNotifyWatcher) Errors(path string) chan error {
	w.mu.RLock()
	defer w.mu.RUnlock()

	cleanedPath, parentPath := formatPath(path)
	worker, workerExist := w.workers[parentPath]
	if !workerExist {
		return nil
	}

	tracker, pathExist := worker.watchedFiles[cleanedPath]
	if !pathExist {
		return nil
	}

	return tracker.errors
}

// getMd5Sum is a helper func to calculate md5 sum.
func getMd5Sum(file string) (string, error) {
	f, err := os.Open(file)
	if err != nil {
		return "", err
	}
	defer f.Close()
	r := bufio.NewReader(f)

	h := md5.New()

	_, err = io.Copy(h, r)
	if err != nil {
		return "", err
	}

	return fmt.Sprintf("%x", h.Sum(nil)), nil
}

// formatPath return with the path after Clean,
// as well as its parent path.
func formatPath(path string) (string, string) {
	cleanedPath := filepath.Clean(path)
	parentPath, _ := filepath.Split(cleanedPath)

	return cleanedPath, parentPath
}
<commit_msg>Enable more linters and fix warnings/errors (#12993)<commit_after>// Copyright 2018 Istio Authors
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package filewatcher

import (
	"bufio"
	"crypto/md5"
	"errors"
	"fmt"
	"io"
	"os"
	"path/filepath"
	"sync"

	"github.com/fsnotify/fsnotify"
	multierror "github.com/hashicorp/go-multierror"
)

// FileWatcher is an interface that watches a set of files,
// delivering events to related channel.
type FileWatcher interface {
	Add(path string) error
	Remove(path string) error
	Close() error
	Events(path string) chan fsnotify.Event
	Errors(path string) chan error
}

type fsNotifyWatcher struct {
	mu sync.RWMutex
	// The watcher maintain a map of workers,
	// keyed by watched dir (parent dir of watched files).
	workers map[string]*worker
}

type worker struct {
	// watcher is a fsnotify watcher that watches the parent
	// dir of watchedFiles.
	watcher *fsnotify.Watcher
	// The worker maintain a map of event channel (included in fileTracker),
	// keyed by watched file path.
	// The worker watches parent path of given path,
	// and filter out events of given path, then redirect
	// to the result channel.
	// Note that for symlink files, the content in received events
	// do not have to be related to the file itself.
	watchedFiles map[string]*fileTracker
}

type fileTracker struct {
	events chan fsnotify.Event
	errors chan error
	// md5 sum to indicate if a file has been updated.
	md5Sum string
}

// NewWatcher return with a FileWatcher instance that implemented with fsnotify.
func NewWatcher() FileWatcher {
	w := &fsNotifyWatcher{
		workers: map[string]*worker{},
	}

	return w
}

// Add is implementation of Add interface of FileWatcher.
func (w *fsNotifyWatcher) Add(path string) error {
	w.mu.Lock()
	defer w.mu.Unlock()

	cleanedPath, parentPath := formatPath(path)
	if worker, workerExist := w.workers[parentPath]; workerExist {
		if _, pathExist := worker.watchedFiles[cleanedPath]; pathExist {
			return fmt.Errorf("path %s already added", path)
		}

		// And the path into worker maps if not exist.
		md5Sum, err := getMd5Sum(cleanedPath)
		if err != nil {
			return fmt.Errorf("failed to get md5 sum for %s: %v", path, err)
		}
		worker.watchedFiles[cleanedPath] = &fileTracker{
			events: make(chan fsnotify.Event, 1),
			errors: make(chan error, 1),
			md5Sum: md5Sum,
		}

		return nil
	}

	// Build a new worker if not exist.
	watcher, err := fsnotify.NewWatcher()
	if err != nil {
		return err
	}
	if err = watcher.Add(parentPath); err != nil {
		watcher.Close()
		return err
	}

	var md5Sum string

	if _, err = os.Stat(cleanedPath); err == nil {
		md5Sum, err = getMd5Sum(cleanedPath)
		if err != nil {
			watcher.Close()
			return fmt.Errorf("failed to get md5 sum for %s: %v", path, err)
		}
	} else if !os.IsNotExist(err) {
		// if the file doesn't exist, ignore the md5 sum.
		watcher.Close()
		return fmt.Errorf("failed to stat file: %s: %v", path, err)
	}

	wk := &worker{
		watcher: watcher,
		watchedFiles: map[string]*fileTracker{
			cleanedPath: {
				events: make(chan fsnotify.Event, 1),
				errors: make(chan error, 1),
				md5Sum: md5Sum,
			},
		},
	}
	w.workers[parentPath] = wk

	go func() {
		for {
			select {
			case event, ok := <-watcher.Events:
				if !ok { // 'Events' channel is closed
					return
				}

				for path, tracker := range wk.watchedFiles {
					newSum, _ := getMd5Sum(path)
					if newSum != "" && newSum != tracker.md5Sum {
						tracker.md5Sum = newSum
						tracker.events <- event
						break
					}
				}
			case err, ok := <-watcher.Errors:
				if !ok {
					// 'Errors' channel is closed. Construct an error for it.
					// We'll only close worker errors channel in "Remove" for consistency.
					err = errors.New("channel closed")
				}
				for _, tracker := range wk.watchedFiles {
					tracker.errors <- err
				}
				return
			}
		}
	}()

	return nil
}

// Remove is implementation of Remove interface of FileWatcher.
func (w *fsNotifyWatcher) Remove(path string) error {
	w.mu.Lock()
	defer w.mu.Unlock()

	return w.remove(path)
}

func (w *fsNotifyWatcher) remove(path string) error {
	cleanedPath, parentPath := formatPath(path)

	worker, workerExist := w.workers[parentPath]
	if !workerExist {
		return fmt.Errorf("path %s already removed", path)
	}

	tracker, pathExist := worker.watchedFiles[cleanedPath]
	if !pathExist {
		return fmt.Errorf("path %s already removed", path)
	}

	delete(worker.watchedFiles, cleanedPath)
	close(tracker.events)
	close(tracker.errors)
	if len(worker.watchedFiles) == 0 {
		// Remove the watch if all of its paths have been removed.
		delete(w.workers, parentPath)
		return worker.watcher.Close()
	}

	return nil
}

// Close is implementation of Close interface of FileWatcher.
func (w *fsNotifyWatcher) Close() error {
	w.mu.Lock()
	defer w.mu.Unlock()

	var errors *multierror.Error
	for _, worker := range w.workers {
		for path := range worker.watchedFiles {
			errors = multierror.Append(errors, w.remove(path))
		}
	}

	return errors.ErrorOrNil()
}

// Events is implementation of Events interface of FileWatcher.
func (w *fsNotifyWatcher) Events(path string) chan fsnotify.Event {
	w.mu.RLock()
	defer w.mu.RUnlock()

	cleanedPath, parentPath := formatPath(path)
	worker, workerExist := w.workers[parentPath]
	if !workerExist {
		return nil
	}

	tracker, pathExist := worker.watchedFiles[cleanedPath]
	if !pathExist {
		return nil
	}

	return tracker.events
}

// Errors is implementation of Errors interface of FileWatcher.
func (w *fsNotifyWatcher) Errors(path string) chan error {
	w.mu.RLock()
	defer w.mu.RUnlock()

	cleanedPath, parentPath := formatPath(path)
	worker, workerExist := w.workers[parentPath]
	if !workerExist {
		return nil
	}

	tracker, pathExist := worker.watchedFiles[cleanedPath]
	if !pathExist {
		return nil
	}

	return tracker.errors
}

// getMd5Sum is a helper func to calculate md5 sum.
func getMd5Sum(file string) (string, error) {
	f, err := os.Open(file)
	if err != nil {
		return "", err
	}
	defer f.Close()
	r := bufio.NewReader(f)

	h := md5.New()

	_, err = io.Copy(h, r)
	if err != nil {
		return "", err
	}

	return fmt.Sprintf("%x", h.Sum(nil)), nil
}

// formatPath return with the path after Clean,
// as well as its parent path.
func formatPath(path string) (string, string) {
	cleanedPath := filepath.Clean(path)
	parentPath, _ := filepath.Split(cleanedPath)

	return cleanedPath, parentPath
}
<|endoftext|>


<commit_before>package saltpack

import (
	"crypto/rand"
	"encoding/binary"
)

// Nonce is a NaCl-style nonce, with 24 bytes of data, some of which can be
// counter values, and some of which can be random-ish values.
type Nonce [24]byte

func nonceForSenderKeySecretBox() *Nonce {
	var n Nonce
	copy(n[:], "saltpack_sender_secbox\x00\x00")
	return &n
}

func nonceForPayloadKeyBox() *Nonce {
	var n Nonce
	copy(n[:], "saltpack_payload_key_box")
	return &n
}

func nonceForMACKeyBox(headerHash []byte) *Nonce {
	var n Nonce
	copy(n[:], headerHash[:24])
	return &n
}

// Construct the nonce for the ith block of payload.
func nonceForChunkSecretBox(i encryptionBlockNumber) *Nonce {
	var n Nonce
	copy(n[0:16], "saltpack_ploadsb")
	binary.BigEndian.PutUint64(n[16:], uint64(i))
	return &n
}

// SigNonce is a nonce for signatures.
type SigNonce [16]byte

// NewSigNonce creates a SigNonce with random bytes.
func NewSigNonce() (SigNonce, error) {
	var n SigNonce
	if _, err := rand.Read(n[:]); err != nil {
		return SigNonce{}, err
	}
	return n, nil
}
<commit_msg>factor out the NonceBytes constant<commit_after>package saltpack

import (
	"crypto/rand"
	"encoding/binary"
)

const NonceBytes = 24

// Nonce is a NaCl-style nonce, with 24 bytes of data, some of which can be
// counter values, and some of which can be random-ish values.
type Nonce [NonceBytes]byte

func nonceForSenderKeySecretBox() *Nonce {
	var n Nonce
	copy(n[:], "saltpack_sender_secbox\x00\x00")
	return &n
}

func nonceForPayloadKeyBox() *Nonce {
	var n Nonce
	copy(n[:], "saltpack_payload_key_box")
	return &n
}

func nonceForMACKeyBox(headerHash []byte) *Nonce {
	var n Nonce
	copy(n[:], headerHash[:NonceBytes])
	return &n
}

// Construct the nonce for the ith block of payload.
func nonceForChunkSecretBox(i encryptionBlockNumber) *Nonce {
	var n Nonce
	copy(n[0:16], "saltpack_ploadsb")
	binary.BigEndian.PutUint64(n[16:], uint64(i))
	return &n
}

// SigNonce is a nonce for signatures.
type SigNonce [16]byte

// NewSigNonce creates a SigNonce with random bytes.
func NewSigNonce() (SigNonce, error) {
	var n SigNonce
	if _, err := rand.Read(n[:]); err != nil {
		return SigNonce{}, err
	}
	return n, nil
}
<|endoftext|>


<commit_before>package dns

import (
	"io"
	"hash"
	"strings"
	"crypto/sha1"
)

type saltWireFmt struct {
	Salt string "size-hex"
}

// Hash a string/label according to RFC5155
func HashNames(label string, ha int, iterations int, salt string) string {
	saltwire := new(saltWireFmt)
	saltwire.Salt = salt
	wire := make([]byte, DefaultMsgSize)
	n, ok := packStruct(saltwire, wire, 0)
	if !ok {
		return ""
	}
	wire = wire[:n]
	name := make([]byte, 255)
	off, ok1 := packDomainName(strings.ToLower(label), name, 0)
	if !ok1 {
		return ""
	}
	name = name[:off]
	var s hash.Hash
	switch ha {
	case HashSHA1:
		s = sha1.New()
        default:
                return ""
	}

	// k = 0
	name = append(name, wire...)
	io.WriteString(s, string(name))
	nsec3 := s.Sum()
        // k > 0
        for k := 0; k < iterations; k++ {
                s.Reset()
                nsec3 = append(nsec3, wire...)
                io.WriteString(s, string(nsec3))
                nsec3 = s.Sum()
        }
	return unpackBase32(nsec3)
}

// Hash the ownername and the next owner name in
// an NSEC3 record, use the parameters from the NSEC3 itself.
func (nsec3 *RR_NSEC3) HashNames() {
        nsec3.Header().Name = HashNames(nsec3.Header().Name, int(nsec3.Hash), int(nsec3.Iterations), nsec3.Salt)
        nsec3.NextDomain = HashNames(nsec3.NextDomain, int(nsec3.Hash), int(nsec3.Iterations), nsec3.Salt)
}
<commit_msg>nsec change<commit_after>package dns

import (
	"io"
	"hash"
	"strings"
	"crypto/sha1"
)

type saltWireFmt struct {
	Salt string "size-hex"
}

// Hash a string/label according to RFC5155
func HashName(label string, ha int, iterations int, salt string) string {
	saltwire := new(saltWireFmt)
	saltwire.Salt = salt
	wire := make([]byte, DefaultMsgSize)
	n, ok := packStruct(saltwire, wire, 0)
	if !ok {
		return ""
	}
	wire = wire[:n]
	name := make([]byte, 255)
	off, ok1 := packDomainName(strings.ToLower(label), name, 0)
	if !ok1 {
		return ""
	}
	name = name[:off]
	var s hash.Hash
	switch ha {
	case HashSHA1:
		s = sha1.New()
        default:
                return ""
	}

	// k = 0
	name = append(name, wire...)
	io.WriteString(s, string(name))
	nsec3 := s.Sum()
        // k > 0
        for k := 0; k < iterations; k++ {
                s.Reset()
                nsec3 = append(nsec3, wire...)
                io.WriteString(s, string(nsec3))
                nsec3 = s.Sum()
        }
	return unpackBase32(nsec3)
}

// Hash the ownername and the next owner name in
// an NSEC3 record, use the parameters from the NSEC3 itself.
func (nsec3 *RR_NSEC3) HashNames() {
        nsec3.Header().Name = HashName(nsec3.Header().Name, int(nsec3.Hash), int(nsec3.Iterations), nsec3.Salt)
        nsec3.NextDomain = HashName(nsec3.NextDomain, int(nsec3.Hash), int(nsec3.Iterations), nsec3.Salt)
}
<|endoftext|>


<commit_before>package eventlog

import (
	"encoding/json"
	"fmt"

	"github.com/jbenet/go-ipfs/Godeps/_workspace/src/github.com/maybebtc/logrus"
)

type PoliteJSONFormatter struct{}

func (f *PoliteJSONFormatter) Format(entry *logrus.Entry) ([]byte, error) {
	serialized, err := json.Marshal(entry.Data)
	if err != nil {
		return nil, fmt.Errorf("Failed to marshal fields to JSON, %v", err)
	}
	return append(serialized, '\n'), nil
}
<commit_msg>docs(eventlog/formatter)<commit_after>package eventlog

import (
	"encoding/json"
	"fmt"

	"github.com/jbenet/go-ipfs/Godeps/_workspace/src/github.com/maybebtc/logrus"
)

// PoliteJSONFormatter marshals entries into JSON encoded slices (without
// overwriting user-provided keys). How polite of it!
type PoliteJSONFormatter struct{}

func (f *PoliteJSONFormatter) Format(entry *logrus.Entry) ([]byte, error) {
	serialized, err := json.Marshal(entry.Data)
	if err != nil {
		return nil, fmt.Errorf("Failed to marshal fields to JSON, %v", err)
	}
	return append(serialized, '\n'), nil
}
<|endoftext|>


<commit_before>package bounded_tree

import (
	"github.com/chrislusf/seaweedfs/weed/glog"
	"github.com/chrislusf/seaweedfs/weed/util"
)

type Node struct {
	Parent   *Node
	Name     string
	Children map[string]*Node
}

type BoundedTree struct {
	root *Node
}

func NewBoundedTree() *BoundedTree {
	return &BoundedTree{
		root: &Node{
			Name: "/",
		},
	}
}

type VisitNodeFunc func(path util.FullPath) (childDirectories []string, err error)

// If the path is not visited, call the visitFn for each level of directory
// No action if the directory has been visited before or does not exist.
// A leaf node, which has no children, represents a directory not visited.
// A non-leaf node or a non-existing node represents a directory already visited, or does not need to visit.
func (t *BoundedTree) EnsureVisited(p util.FullPath, visitFn VisitNodeFunc) {
	// println()
	// println("EnsureVisited", p)

	if t.root == nil {
		return
	}
	components := p.Split()
	// fmt.Printf("components %v %d\n", components, len(components))
	if canDelete := t.ensureVisited(t.root, util.FullPath("/"), components, 0, visitFn); canDelete {
		t.root = nil
	}
}

func (t *BoundedTree) ensureVisited(n *Node, currentPath util.FullPath, components []string, i int, visitFn VisitNodeFunc) (canDeleteNode bool) {

	// println("ensureVisited", currentPath, i)

	if n == nil {
		// fmt.Printf("%s null\n", currentPath)
		return
	}

	if n.isVisited() {
		// fmt.Printf("%s visited %v\n", currentPath, n.Name)
	} else {
		// fmt.Printf("ensure %v\n", currentPath)

		children, err := visitFn(currentPath)
		if err != nil {
			glog.V(0).Infof("failed to visit %s: %v", currentPath, err)
			return
		}

		if len(children) == 0 {
			// fmt.Printf("  canDelete %v without children\n", currentPath)
			return true
		}

		n.Children = make(map[string]*Node)
		for _, child := range children {
			// fmt.Printf("  add child %v %v\n", currentPath, child)
			n.Children[child] = &Node{
				Name: child,
			}
		}
	}

	if i >= len(components) {
		return
	}

	// fmt.Printf("  check child %v %v\n", currentPath, components[i])

	toVisitNode, found := n.Children[components[i]]
	if !found {
		// fmt.Printf("  did not find child %v %v\n", currentPath, components[i])
		return
	}

	// fmt.Printf("  ensureVisited %v %v\n", currentPath, toVisitNode.Name)

	if canDelete := t.ensureVisited(toVisitNode, currentPath.Child(components[i]), components, i+1, visitFn); canDelete {

		// fmt.Printf("  delete %v %v\n", currentPath, components[i])
		delete(n.Children, components[i])

		if len(n.Children) == 0 {
			// fmt.Printf("  canDelete %v\n", currentPath)
			return true
		}
	}

	return false

}

func (n *Node) isVisited() bool {
	if n == nil {
		return true
	}
	if len(n.Children) > 0 {
		return true
	}
	return false
}

func (n *Node) getChild(childName string) *Node {
	if n == nil {
		return nil
	}
	if len(n.Children) > 0 {
		return n.Children[childName]
	}
	return nil
}

func (t *BoundedTree) HasVisited(p util.FullPath) bool {

	if t.root == nil {
		return true
	}

	components := p.Split()
	// fmt.Printf("components %v %d\n", components, len(components))
	return t.hasVisited(t.root, util.FullPath("/"), components, 0)
}

func (t *BoundedTree) hasVisited(n *Node, currentPath util.FullPath, components []string, i int) bool {

	if n == nil {
		return true
	}

	if !n.isVisited() {
		return false
	}

	// fmt.Printf("  hasVisited child %v %+v %d\n", currentPath, components, i)

	if i >= len(components) {
		return true
	}

	toVisitNode, found := n.Children[components[i]]
	if !found {
		return true
	}

	return t.hasVisited(toVisitNode, currentPath.Child(components[i]), components, i+1)

}
<commit_msg>FUSE mount: prevent concurrent modification<commit_after>package bounded_tree

import (
	"sync"

	"github.com/chrislusf/seaweedfs/weed/glog"
	"github.com/chrislusf/seaweedfs/weed/util"
)

type Node struct {
	Parent   *Node
	Name     string
	Children map[string]*Node
}

type BoundedTree struct {
	root *Node
	sync.Mutex
}

func NewBoundedTree() *BoundedTree {
	return &BoundedTree{
		root: &Node{
			Name: "/",
		},
	}
}

type VisitNodeFunc func(path util.FullPath) (childDirectories []string, err error)

// If the path is not visited, call the visitFn for each level of directory
// No action if the directory has been visited before or does not exist.
// A leaf node, which has no children, represents a directory not visited.
// A non-leaf node or a non-existing node represents a directory already visited, or does not need to visit.
func (t *BoundedTree) EnsureVisited(p util.FullPath, visitFn VisitNodeFunc) {
	t.Lock()
	defer t.Unlock()

	if t.root == nil {
		return
	}
	components := p.Split()
	// fmt.Printf("components %v %d\n", components, len(components))
	if canDelete := t.ensureVisited(t.root, util.FullPath("/"), components, 0, visitFn); canDelete {
		t.root = nil
	}
}

func (t *BoundedTree) ensureVisited(n *Node, currentPath util.FullPath, components []string, i int, visitFn VisitNodeFunc) (canDeleteNode bool) {

	// println("ensureVisited", currentPath, i)

	if n == nil {
		// fmt.Printf("%s null\n", currentPath)
		return
	}

	if n.isVisited() {
		// fmt.Printf("%s visited %v\n", currentPath, n.Name)
	} else {
		// fmt.Printf("ensure %v\n", currentPath)

		children, err := visitFn(currentPath)
		if err != nil {
			glog.V(0).Infof("failed to visit %s: %v", currentPath, err)
			return
		}

		if len(children) == 0 {
			// fmt.Printf("  canDelete %v without children\n", currentPath)
			return true
		}

		n.Children = make(map[string]*Node)
		for _, child := range children {
			// fmt.Printf("  add child %v %v\n", currentPath, child)
			n.Children[child] = &Node{
				Name: child,
			}
		}
	}

	if i >= len(components) {
		return
	}

	// fmt.Printf("  check child %v %v\n", currentPath, components[i])

	toVisitNode, found := n.Children[components[i]]
	if !found {
		// fmt.Printf("  did not find child %v %v\n", currentPath, components[i])
		return
	}

	// fmt.Printf("  ensureVisited %v %v\n", currentPath, toVisitNode.Name)

	if canDelete := t.ensureVisited(toVisitNode, currentPath.Child(components[i]), components, i+1, visitFn); canDelete {

		// fmt.Printf("  delete %v %v\n", currentPath, components[i])
		delete(n.Children, components[i])

		if len(n.Children) == 0 {
			// fmt.Printf("  canDelete %v\n", currentPath)
			return true
		}
	}

	return false

}

func (n *Node) isVisited() bool {
	if n == nil {
		return true
	}
	if len(n.Children) > 0 {
		return true
	}
	return false
}

func (n *Node) getChild(childName string) *Node {
	if n == nil {
		return nil
	}
	if len(n.Children) > 0 {
		return n.Children[childName]
	}
	return nil
}

func (t *BoundedTree) HasVisited(p util.FullPath) bool {

	if t.root == nil {
		return true
	}

	components := p.Split()
	// fmt.Printf("components %v %d\n", components, len(components))
	return t.hasVisited(t.root, util.FullPath("/"), components, 0)
}

func (t *BoundedTree) hasVisited(n *Node, currentPath util.FullPath, components []string, i int) bool {

	if n == nil {
		return true
	}

	if !n.isVisited() {
		return false
	}

	// fmt.Printf("  hasVisited child %v %+v %d\n", currentPath, components, i)

	if i >= len(components) {
		return true
	}

	toVisitNode, found := n.Children[components[i]]
	if !found {
		return true
	}

	return t.hasVisited(toVisitNode, currentPath.Child(components[i]), components, i+1)

}
<|endoftext|>


<commit_before>package info

import (
	"errors"
	"fmt"
	"io/ioutil"
	"net"
	"net/http"
	"regexp"
	"time"

	"github.com/koding/klient/info/publicip"
)

const (
	// The whois server WhoisQuery uses by default.
	whoisServer string = "whois.arin.net"

	// Default timeout for the whoisQuery
	whoisTimeout time.Duration = 5 * time.Second
)

// ProviderChecker funcs check the local machine to assert whether or
// not the current VM is is of that specific Provider.
type ProviderChecker func() (isProvider bool, err error)

// WhoisFunc returns the whois from the whois protocol.
type WhoisFunc func() (whois string, err error)

type ProviderName int

const (
	// UnknownProvider is the zero value of the ProviderName type.
	UnknownProvider ProviderName = iota

	AWS
	Azure
	DigitalOcean
	GoogleCloud
	HPCloud
	Joyent
	Rackspace
	SoftLayer
	Koding
)

func (pn ProviderName) String() string {
	switch pn {
	case AWS:
		return "AWS"
	case Azure:
		return "Azure"
	case DigitalOcean:
		return "DigitalOcean"
	case GoogleCloud:
		return "GoogleCloud"
	case HPCloud:
		return "HPCloud"
	case Joyent:
		return "Joyent"
	case Rackspace:
		return "Rackspace"
	case SoftLayer:
		return "SoftLayer"
	case Koding:
		return "Koding"
	default:
		return "UnknownProvider"
	}
}

// Checker returns the ProviderChecker for the given ProviderName
func (pn ProviderName) Checker() ProviderChecker {
	switch pn {
	case AWS:
		return CheckAWS
	case Azure:
		return CheckAzure
	case DigitalOcean:
		return CheckDigitalOcean
	case GoogleCloud:
		return CheckGoogleCloud
	case Joyent:
		return CheckJoyent
	case Rackspace:
		return CheckRackspace
	case SoftLayer:
		return CheckSoftLayer
	case Koding:
		return CheckKoding
	default:
		return nil
	}
}

// DefaultProviderCheckers is a slice of each ProviderName in the order
// that they will be checked.
var DefaultProvidersToCheck = []ProviderName{
	AWS,
	Azure,
	DigitalOcean,
	GoogleCloud,
	Joyent,
	Rackspace,
	SoftLayer,
	Koding,
}

// cachedProviderName is the ProviderName resulting from running through
// all of the DefaultProviderCheckers.
//
// This should not be written to by anyone but CheckProvider(), because
// an individual ProviderChecker may not be correct when compared to
// CheckProvider. For example, CheckAWS() will always return true, even
// if the ProviderName is actually Koding.
//
// ProviderCheckers are able to read from this value though, if desired.
var cachedProviderName ProviderName

// CheckProvider uses the current machine's IP and runs a whois on it, then
// feeds the whois to all DefaultProviderCheckers. Providers are free to use
// the whois query.
func CheckProvider() (ProviderName, error) {
	if cachedProviderName != UnknownProvider {
		return cachedProviderName, nil
	}

	providerName, err := checkProvider(DefaultProvidersToCheck)
	if err != nil {
		return UnknownProvider, err
	}

	cachedProviderName = providerName

	return providerName, nil
}

// checkProvider implements the testable functionality of CheckProvider.
// Ie, a pure func, aside from any impurities passed in via checkers.
func checkProvider(providers []ProviderName) (ProviderName, error) {
	for _, providerName := range providers {
		checker := providerName.Checker()
		if checker == nil {
			return UnknownProvider, errors.New(fmt.Sprintf(
				"checkProvider: No checker for ProviderName '%s'",
				providerName.String(),
			))
		}

		isProvider, err := checker()
		if err != nil {
			return UnknownProvider, err
		}

		if isProvider == true {
			return providerName, nil
		}
	}

	return UnknownProvider, nil
}

// generateWhoisChecker returns a ProviderChecker matching one or more whois
// regexp objects against the typical ProviderChecker whois.
func generateWhoisChecker(pn ProviderName,
	res ...*regexp.Regexp) ProviderChecker {

	return func() (bool, error) {
		if cachedProviderName == pn {
			return true, nil
		}

		whois, err := DefaultWhoisChecker()
		if err != nil {
			return false, err
		}

		for _, re := range res {
			if !re.MatchString(whois) {
				return false, nil
			}
		}

		return true, nil
	}
}

// CheckDigitalOcean is a ProviderChecker for DigitalOcean
func CheckDigitalOcean() (bool, error) {
	if cachedProviderName == DigitalOcean {
		return true, nil
	}

	return checkDigitalOcean("http://169.254.169.254/metadata/v1/hostname")
}

// checkDigitalOcean implements the testable functionality of
// CheckDigitalOcean by quering the given DigitalOcean API address
// and if it returns 404, the check fails.
func checkDigitalOcean(metadataApi string) (bool, error) {
	res, err := http.Get(metadataApi)
	if err != nil {
		return false, err
	}

	return res.StatusCode == http.StatusOK, nil
}

// CheckAWS is a generic whois checker for Amazon
var CheckAWS ProviderChecker = generateWhoisChecker(
	AWS,
	regexp.MustCompile(`(?i)amazon`),
)

var CheckAzure ProviderChecker = generateWhoisChecker(
	Azure,
	regexp.MustCompile(`(?i)azure`),
)

var CheckGoogleCloud ProviderChecker = generateWhoisChecker(
	GoogleCloud,
	regexp.MustCompile(`(?i)google\s*cloud`),
)

var CheckHPCloud ProviderChecker = generateWhoisChecker(
	HPCloud,
	regexp.MustCompile(`(?i)hp\s*cloud`),
)

var CheckJoyent ProviderChecker = generateWhoisChecker(
	Joyent,
	regexp.MustCompile(`(?i)joyent`),
)

var CheckRackspace ProviderChecker = generateWhoisChecker(
	Rackspace,
	regexp.MustCompile(`(?i)rackspace`),
)

var CheckSoftLayer ProviderChecker = generateWhoisChecker(
	SoftLayer,
	regexp.MustCompile(`(?i)softlayer`),
)

// WhoisQuery is a simple func to query a whois service with the (limited)
// whois protocol.
//
// It's worth noting that because the whois protocol is so basic, the
// response can be formatted in any way. Because of this, WhoisQuery has to
// simply return the entire response to the caller - and is unable to
// marshall/etc the response in any meaningful format.
func WhoisQuery(query, server string, timeout time.Duration) (string, error) {
	host := net.JoinHostPort(server, "43")
	conn, err := net.DialTimeout("tcp", host, timeout)
	if err != nil {
		return "", err
	}
	defer conn.Close()

	// Query the whois server with the ip or domain given to this func,
	// as per Whois spec.
	_, err = conn.Write([]byte(fmt.Sprintf("%s\r\n", query)))
	if err != nil {
		return "", err
	}

	// After the query, the server will respond with the unformatted data.
	// Read it all and return it.
	b, err := ioutil.ReadAll(conn)
	if err != nil {
		return "", err
	}

	return string(b), nil
}

// DefaultWhoisChecker is used for all whois based checkers
var DefaultWhoisChecker WhoisFunc = func() (string, error) {
	// Get the IP of this machine, to whois against
	ip, err := publicip.PublicIP()
	if err != nil {
		return "", err
	}

	// Get the whois of the current vm's IP
	return WhoisQuery(ip.String(), whoisServer, whoisTimeout)
}
<commit_msg>info: Check Koding before AWS, since all Koding VMs are AWS<commit_after>package info

import (
	"errors"
	"fmt"
	"io/ioutil"
	"net"
	"net/http"
	"regexp"
	"time"

	"github.com/koding/klient/info/publicip"
)

const (
	// The whois server WhoisQuery uses by default.
	whoisServer string = "whois.arin.net"

	// Default timeout for the whoisQuery
	whoisTimeout time.Duration = 5 * time.Second
)

// ProviderChecker funcs check the local machine to assert whether or
// not the current VM is is of that specific Provider.
type ProviderChecker func() (isProvider bool, err error)

// WhoisFunc returns the whois from the whois protocol.
type WhoisFunc func() (whois string, err error)

type ProviderName int

const (
	// UnknownProvider is the zero value of the ProviderName type.
	UnknownProvider ProviderName = iota

	AWS
	Azure
	DigitalOcean
	GoogleCloud
	HPCloud
	Joyent
	Rackspace
	SoftLayer
	Koding
)

func (pn ProviderName) String() string {
	switch pn {
	case AWS:
		return "AWS"
	case Azure:
		return "Azure"
	case DigitalOcean:
		return "DigitalOcean"
	case GoogleCloud:
		return "GoogleCloud"
	case HPCloud:
		return "HPCloud"
	case Joyent:
		return "Joyent"
	case Rackspace:
		return "Rackspace"
	case SoftLayer:
		return "SoftLayer"
	case Koding:
		return "Koding"
	default:
		return "UnknownProvider"
	}
}

// Checker returns the ProviderChecker for the given ProviderName
func (pn ProviderName) Checker() ProviderChecker {
	switch pn {
	case AWS:
		return CheckAWS
	case Azure:
		return CheckAzure
	case DigitalOcean:
		return CheckDigitalOcean
	case GoogleCloud:
		return CheckGoogleCloud
	case Joyent:
		return CheckJoyent
	case Rackspace:
		return CheckRackspace
	case SoftLayer:
		return CheckSoftLayer
	case Koding:
		return CheckKoding
	default:
		return nil
	}
}

// DefaultProviderCheckers is a slice of each ProviderName in the order
// that they will be checked.
var DefaultProvidersToCheck = []ProviderName{
	Azure,
	DigitalOcean,
	GoogleCloud,
	Joyent,
	Rackspace,
	SoftLayer,
	Koding,
	AWS,
}

// cachedProviderName is the ProviderName resulting from running through
// all of the DefaultProviderCheckers.
//
// This should not be written to by anyone but CheckProvider(), because
// an individual ProviderChecker may not be correct when compared to
// CheckProvider. For example, CheckAWS() will always return true, even
// if the ProviderName is actually Koding.
//
// ProviderCheckers are able to read from this value though, if desired.
var cachedProviderName ProviderName

// CheckProvider uses the current machine's IP and runs a whois on it, then
// feeds the whois to all DefaultProviderCheckers. Providers are free to use
// the whois query.
func CheckProvider() (ProviderName, error) {
	if cachedProviderName != UnknownProvider {
		return cachedProviderName, nil
	}

	providerName, err := checkProvider(DefaultProvidersToCheck)
	if err != nil {
		return UnknownProvider, err
	}

	cachedProviderName = providerName

	return providerName, nil
}

// checkProvider implements the testable functionality of CheckProvider.
// Ie, a pure func, aside from any impurities passed in via checkers.
func checkProvider(providers []ProviderName) (ProviderName, error) {
	for _, providerName := range providers {
		checker := providerName.Checker()
		if checker == nil {
			return UnknownProvider, errors.New(fmt.Sprintf(
				"checkProvider: No checker for ProviderName '%s'",
				providerName.String(),
			))
		}

		isProvider, err := checker()
		if err != nil {
			return UnknownProvider, err
		}

		if isProvider == true {
			return providerName, nil
		}
	}

	return UnknownProvider, nil
}

// generateWhoisChecker returns a ProviderChecker matching one or more whois
// regexp objects against the typical ProviderChecker whois.
func generateWhoisChecker(pn ProviderName,
	res ...*regexp.Regexp) ProviderChecker {

	return func() (bool, error) {
		if cachedProviderName == pn {
			return true, nil
		}

		whois, err := DefaultWhoisChecker()
		if err != nil {
			return false, err
		}

		for _, re := range res {
			if !re.MatchString(whois) {
				return false, nil
			}
		}

		return true, nil
	}
}

// CheckDigitalOcean is a ProviderChecker for DigitalOcean
func CheckDigitalOcean() (bool, error) {
	if cachedProviderName == DigitalOcean {
		return true, nil
	}

	return checkDigitalOcean("http://169.254.169.254/metadata/v1/hostname")
}

// checkDigitalOcean implements the testable functionality of
// CheckDigitalOcean by quering the given DigitalOcean API address
// and if it returns 404, the check fails.
func checkDigitalOcean(metadataApi string) (bool, error) {
	res, err := http.Get(metadataApi)
	if err != nil {
		return false, err
	}

	return res.StatusCode == http.StatusOK, nil
}

// CheckAWS is a generic whois checker for Amazon
var CheckAWS ProviderChecker = generateWhoisChecker(
	AWS,
	regexp.MustCompile(`(?i)amazon`),
)

var CheckAzure ProviderChecker = generateWhoisChecker(
	Azure,
	regexp.MustCompile(`(?i)azure`),
)

var CheckGoogleCloud ProviderChecker = generateWhoisChecker(
	GoogleCloud,
	regexp.MustCompile(`(?i)google\s*cloud`),
)

var CheckHPCloud ProviderChecker = generateWhoisChecker(
	HPCloud,
	regexp.MustCompile(`(?i)hp\s*cloud`),
)

var CheckJoyent ProviderChecker = generateWhoisChecker(
	Joyent,
	regexp.MustCompile(`(?i)joyent`),
)

var CheckRackspace ProviderChecker = generateWhoisChecker(
	Rackspace,
	regexp.MustCompile(`(?i)rackspace`),
)

var CheckSoftLayer ProviderChecker = generateWhoisChecker(
	SoftLayer,
	regexp.MustCompile(`(?i)softlayer`),
)

// WhoisQuery is a simple func to query a whois service with the (limited)
// whois protocol.
//
// It's worth noting that because the whois protocol is so basic, the
// response can be formatted in any way. Because of this, WhoisQuery has to
// simply return the entire response to the caller - and is unable to
// marshall/etc the response in any meaningful format.
func WhoisQuery(query, server string, timeout time.Duration) (string, error) {
	host := net.JoinHostPort(server, "43")
	conn, err := net.DialTimeout("tcp", host, timeout)
	if err != nil {
		return "", err
	}
	defer conn.Close()

	// Query the whois server with the ip or domain given to this func,
	// as per Whois spec.
	_, err = conn.Write([]byte(fmt.Sprintf("%s\r\n", query)))
	if err != nil {
		return "", err
	}

	// After the query, the server will respond with the unformatted data.
	// Read it all and return it.
	b, err := ioutil.ReadAll(conn)
	if err != nil {
		return "", err
	}

	return string(b), nil
}

// DefaultWhoisChecker is used for all whois based checkers
var DefaultWhoisChecker WhoisFunc = func() (string, error) {
	// Get the IP of this machine, to whois against
	ip, err := publicip.PublicIP()
	if err != nil {
		return "", err
	}

	// Get the whois of the current vm's IP
	return WhoisQuery(ip.String(), whoisServer, whoisTimeout)
}
<|endoftext|>


<commit_before>package main

import (
	"bytes"
	"database/sql"
	"fmt"
	"io"
	"log"
	"net/mail"
	"net/smtp"
	"os"
	"strconv"
	"strings"

	message "github.com/matsuev/go-message"
	charset "github.com/matsuev/go-message/charset"

	_ "github.com/go-sql-driver/mysql"
)

func main() {
	// Перенаправление логов в файл
	// создать файл лога, установить права доступа
	// l, err := os.OpenFile("./klshmail.log", os.O_APPEND|os.O_WRONLY|os.O_CREATE, 0644)
	// l, err := os.OpenFile("/var/log/klshmail.log", os.O_APPEND|os.O_WRONLY|os.O_CREATE, 0644)
	// logFatal(err)
	// defer l.Close()

	// log.SetOutput(l)

	// Заголовки исходного сообщения,
	// которые нужно оставить
	hh := []string{
		"MIME-Version",
		"Message-Id",
		"Content-Type",
		"Content-Transfer-Encoding",
		"In-Reply-To",
		"References",
		"Subject",
	}

	// Читаем сообщение из стандартного ввода
	msg, err := message.Read(os.Stdin)
	logFatal(err)
	log.Println("New message accepted...")

	// Разбор заголовков сообщения
	to, err := getMailHeader("To", msg.Header)
	logFatal(err)
	log.Printf("To: %s <%s>\n", to.Name, to.Address)

	from, err := getMailHeader("From", msg.Header)
	logFatal(err)
	log.Printf("From: %s <%s>\n", from.Name, from.Address)

	if to.Address == "" || from.Address == "" {
		log.Fatalln("Empty address. Reject message.")
	}

	// Сокдинение с сервером БД
	db, err := sql.Open("mysql", "klshmail:euXoe8uSha1xu4sh@/klshmail?charset=utf8")
	logFatal(err)

	// Проверка соединения с сервером БД
	err = db.Ping()
	logFatal(err)

	// Запрос данных о списке рассылки
	var lid uint64
	var lprefix string

	err = db.QueryRow(`
		SELECT list.id, list.prefix
		FROM list
		WHERE LCASE(list.email)=TRIM(LCASE(?))
		AND list.active
		`, to.Address).Scan(&lid, &lprefix)
	if err != nil {
		log.Println("No list with address:", to.Address)
		os.Exit(0)
	}

	// Запрос на проверку прав пользователя на отправку сообщений в список
	var uid uint64
	err = db.QueryRow(`
		SELECT user.id
		FROM user
		INNER JOIN user_list
		ON (user_list.lid=?
			AND user.id=user_list.uid
			AND user_list.canwrite
		)
		WHERE LCASE(user.email)=TRIM(LCASE(?))
		AND user.active
		`, lid, from.Address).Scan(&uid)
	if err != nil {
		log.Println("User", from.Address, "can't send messages to", to.Address)
		os.Exit(0)
	}

	// Формирование заголовков нового сообщения
	newHeader := make(message.Header)
	for _, hk := range hh {
		if hv := msg.Header.Get(hk); hv != "" {
			newHeader.Set(hk, hv)
		}
	}

	sender := new(mail.Address)
	sender.Name = from.Name
	sender.Address = from.Address

	from.Name = fmt.Sprintf("%s", lprefix)
	from.Address = to.Address

	newHeader.Set("From", from.String())
	newHeader.Set("Reply-To", to.Address)
	newHeader.Set("X-KLSH-Sender", strconv.FormatUint(uid, 10))

	var b bytes.Buffer
	w, err := message.CreateWriter(&b, newHeader)
	if err != nil {
		log.Fatal(err)
	}
	defer w.Close()

	if err := transform(w, msg, sender); err != nil {
		log.Fatal(err)
	}

	// Подключение к SMTP серверу
	c, err := smtp.Dial("127.0.0.1:25")
	if err != nil {
		log.Println("SMTP connection error")
		log.Fatal(err)
	}
	defer c.Close()

	// Заголовки для отправки
	c.Mail(to.Address)
	c.Rcpt(fmt.Sprintf("%v@klshmail", lid))

	wc, err := c.Data()
	logFatal(err)
	defer wc.Close()

	// Отправка сообщения
	br := bytes.NewReader(b.Bytes())
	if _, err = io.Copy(wc, br); err != nil {
		log.Println("SMTP send body error")
		log.Fatalln(err)
	}

	// Завершение работы
	log.Println("Message processing done.")

}

func logFatal(e error) {
	if e != nil {
		log.Fatalln(e)
	}
}

const senderHtml string = `<p><b>Сообщение от:</b> %s <<a href="mailto:%s">%s</a>><p>`
const senderPlain string = "| Сообщение от:  %s <%s>\n——\n\n"

func transform(w *message.Writer, e *message.Entity, sender *mail.Address) error {
	if mr := e.MultipartReader(); mr != nil {
		// This is a multipart entity, transform each of its parts
		for {
			p, err := mr.NextPart()
			if err == io.EOF {
				break
			} else if err != nil {
				return err
			}

			pw, err := w.CreatePart(p.Header)
			if err != nil {
				return err
			}

			if err := transform(pw, p, sender); err != nil {
				return err
			}

			pw.Close()
		}
		return nil
	} else {
		body := e.Body
		var newLine string
		if strings.HasPrefix(e.Header.Get("Content-Type"), "text/plain") {
			newLine = fmt.Sprintf(senderPlain, sender.Name, sender.Address)
		}
		if strings.HasPrefix(e.Header.Get("Content-Type"), "text/html") {
			newLine = fmt.Sprintf(senderHtml, sender.Name, sender.Address, sender.Address)
		}
		body = io.MultiReader(strings.NewReader(newLine), body)
		_, err := io.Copy(w, body)
		return err
	}
}

func getMailHeader(k string, h message.Header) (*mail.Address, error) {
	dh, err := charset.DecodeHeader(h.Get(k))
	if err != nil {
		return nil, err
	}

	rh, err := mail.ParseAddress(dh)
	return rh, err
}
<commit_msg>Update all<commit_after>package main

import (
	"bytes"
	"database/sql"
	"fmt"
	"io"
	"log"
	"mime"
	"net/mail"
	"net/smtp"
	"os"
	"strconv"
	"strings"

	message "github.com/matsuev/go-message"
	charset "github.com/matsuev/go-message/charset"

	_ "github.com/go-sql-driver/mysql"
)

func main() {
	// Перенаправление логов в файл
	// создать файл лога, установить права доступа
	// l, err := os.OpenFile("./klshmail.log", os.O_APPEND|os.O_WRONLY|os.O_CREATE, 0644)
	// l, err := os.OpenFile("/var/log/klshmail.log", os.O_APPEND|os.O_WRONLY|os.O_CREATE, 0644)
	// logFatal(err)
	// defer l.Close()

	// log.SetOutput(l)

	// Заголовки исходного сообщения,
	// которые нужно оставить
	hh := []string{
		"MIME-Version",
		"Message-Id",
		"Content-Type",
		"Content-Transfer-Encoding",
		"In-Reply-To",
		"References",
		// "Subject",
	}

	// Читаем сообщение из стандартного ввода
	msg, err := message.Read(os.Stdin)
	logFatal(err)
	log.Println("New message accepted...")

	// Разбор заголовков сообщения
	to, err := getMailHeader("To", msg.Header)
	logFatal(err)
	log.Printf("To: %s <%s>\n", to.Name, to.Address)

	from, err := getMailHeader("From", msg.Header)
	logFatal(err)
	log.Printf("From: %s <%s>\n", from.Name, from.Address)

	hSubject, err := charset.DecodeHeader(msg.Header.Get("Subject"))
	logFatal(err)

	subj := mime.BEncoding.Encode("utf-8", hSubject)

	if to.Address == "" || from.Address == "" {
		log.Fatalln("Empty address. Reject message.")
	}

	// Сокдинение с сервером БД
	db, err := sql.Open("mysql", "klshmail:euXoe8uSha1xu4sh@/klshmail?charset=utf8")
	logFatal(err)

	// Проверка соединения с сервером БД
	err = db.Ping()
	logFatal(err)

	// Запрос данных о списке рассылки
	var lid uint64
	var lprefix string

	err = db.QueryRow(`
		SELECT list.id, list.prefix
		FROM list
		WHERE LCASE(list.email)=TRIM(LCASE(?))
		AND list.active
		`, to.Address).Scan(&lid, &lprefix)
	if err != nil {
		log.Println("No list with address:", to.Address)
		os.Exit(0)
	}

	// Запрос на проверку прав пользователя на отправку сообщений в список
	var uid uint64
	err = db.QueryRow(`
		SELECT user.id
		FROM user
		INNER JOIN user_list
		ON (user_list.lid=?
			AND user.id=user_list.uid
			AND user_list.canwrite
		)
		WHERE LCASE(user.email)=TRIM(LCASE(?))
		AND user.active
		`, lid, from.Address).Scan(&uid)
	if err != nil {
		log.Println("User", from.Address, "can't send messages to", to.Address)
		os.Exit(0)
	}

	// Формирование заголовков нового сообщения
	newHeader := make(message.Header)
	for _, hk := range hh {
		if hv := msg.Header.Get(hk); hv != "" {
			newHeader.Set(hk, hv)
		}
	}

	sender := new(mail.Address)
	sender.Name = from.Name
	sender.Address = from.Address

	from.Name = fmt.Sprintf("%s", lprefix)
	from.Address = to.Address

	newHeader.Set("From", from.String())
	newHeader.Set("Reply-To", to.Address)
	newHeader.Set("Subject", subj)
	newHeader.Set("X-KLSH-Sender", strconv.FormatUint(uid, 10))

	var b bytes.Buffer
	w, err := message.CreateWriter(&b, newHeader)
	if err != nil {
		log.Fatal(err)
	}
	defer w.Close()

	if err := transform(w, msg, sender); err != nil {
		log.Fatal(err)
	}

	// Подключение к SMTP серверу
	c, err := smtp.Dial("127.0.0.1:25")
	if err != nil {
		log.Println("SMTP connection error")
		log.Fatal(err)
	}
	defer c.Close()

	// Заголовки для отправки
	c.Mail(to.Address)
	c.Rcpt(fmt.Sprintf("%v@klshmail", lid))

	wc, err := c.Data()
	logFatal(err)
	defer wc.Close()

	// Отправка сообщения
	br := bytes.NewReader(b.Bytes())
	if _, err = io.Copy(wc, br); err != nil {
		log.Println("SMTP send body error")
		log.Fatalln(err)
	}

	// Завершение работы
	log.Println("Message processing done.")

}

func logFatal(e error) {
	if e != nil {
		log.Fatalln(e)
	}
}

const senderHtml string = `<p><b>Сообщение от:</b> %s <<a href="mailto:%s">%s</a>><p>`
const senderPlain string = "| Сообщение от:  %s <%s>\n——\n\n"

func transform(w *message.Writer, e *message.Entity, sender *mail.Address) error {
	if mr := e.MultipartReader(); mr != nil {
		// This is a multipart entity, transform each of its parts
		for {
			p, err := mr.NextPart()
			if err == io.EOF {
				break
			} else if err != nil {
				return err
			}

			pw, err := w.CreatePart(p.Header)
			if err != nil {
				return err
			}

			if err := transform(pw, p, sender); err != nil {
				return err
			}

			pw.Close()
		}
		return nil
	} else {
		body := e.Body
		var newLine string
		if strings.HasPrefix(e.Header.Get("Content-Type"), "text/plain") {
			newLine = fmt.Sprintf(senderPlain, sender.Name, sender.Address)
		}
		if strings.HasPrefix(e.Header.Get("Content-Type"), "text/html") {
			newLine = fmt.Sprintf(senderHtml, sender.Name, sender.Address, sender.Address)
		}
		body = io.MultiReader(strings.NewReader(newLine), body)
		_, err := io.Copy(w, body)
		return err
	}
}

func getMailHeader(k string, h message.Header) (*mail.Address, error) {
	dh, err := charset.DecodeHeader(h.Get(k))
	if err != nil {
		return nil, err
	}

	rh, err := mail.ParseAddress(dh)
	return rh, err
}
<|endoftext|>


<commit_before>package main

import (
	"encoding/json"
	"flag"
	"github.com/client9/hjson/hjson"
	"log"
	"os"
)

func main() {
	compact := flag.Bool("c", false, "compact")
	tojson := flag.Bool("j", false, "to JSON")
	flag.Parse()
	args := flag.Args()
	if len(args) == 0 {
		log.Fatalf("Need input")
	}
	f, err := os.Open(args[0])
	if err != nil {
		log.Fatalf("error %s", err)
	}
	if *tojson {
		var v interface{}
		r := hjson.New(f)
		err = json.NewDecoder(r).Decode(&v)
		if err != nil {
			log.Fatalf("Unable to decode HJSON: %s", err)
		}
		rawout := []byte{}
		if *compact {
			rawout, err = json.Marshal(v)
		} else {
			rawout, err = json.MarshalIndent(v, "", "    ")
		}
		if err != nil {
			log.Fatalf("Unable to encode to JSON: %s", err)
		}
		os.Stdout.Write(rawout)
	}
}
<commit_msg>Changing hjson -> xson in import<commit_after>package main

import (
	"encoding/json"
	"flag"
	"github.com/client9/xson/hjson"
	"log"
	"os"
)

func main() {
	compact := flag.Bool("c", false, "compact")
	tojson := flag.Bool("j", false, "to JSON")
	flag.Parse()
	args := flag.Args()
	if len(args) == 0 {
		log.Fatalf("Need input")
	}
	f, err := os.Open(args[0])
	if err != nil {
		log.Fatalf("error %s", err)
	}
	if *tojson {
		var v interface{}
		r := hjson.New(f)
		err = json.NewDecoder(r).Decode(&v)
		if err != nil {
			log.Fatalf("Unable to decode HJSON: %s", err)
		}
		rawout := []byte{}
		if *compact {
			rawout, err = json.Marshal(v)
		} else {
			rawout, err = json.MarshalIndent(v, "", "    ")
		}
		if err != nil {
			log.Fatalf("Unable to encode to JSON: %s", err)
		}
		os.Stdout.Write(rawout)
	}
}
<|endoftext|>