<commit_before>// Copyright (c) 2017 Cisco and/or its affiliates.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at:
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package contiv

import (
	"crypto/sha1"
	"fmt"
	"strings"

	"github.com/ligato/cn-infra/logging"
)

// IPAM represents the basic Contiv IPAM module.
// TODO: this is just an initial implementation, requires a lot of work.
type IPAM struct {
	logging.Logger

	podNetworkIPPrefix  string // prefix with pod network IP, ID of the pod can be appended to this to get the real pod IP
	podNetworkPrefixLen uint32 // prefix length of pod network used on this host
	podSeqID            uint32 // pod sequence number used to allocate an uniquie IP address to each POD
}

// newIPAM return new IPAM module to be used on the host.
func newIPAM(logger logging.Logger, podSubnetCIDR string, podNetworkPrefixLen uint32, agentLabel string) *IPAM {
	ipam := &IPAM{
		Logger:   logger,
		podSeqID: 1, // .1 wil be the gateway address
	}

	// calculate POD subnetID ID based on agentLabel
	// TODO: implement proper allocation logic based on ETCD
	h := sha1.New()
	h.Write([]byte(agentLabel))
	sum := h.Sum(nil)
	subnetID := uint8(sum[0])
	logger.Infof("Will use %d as the POD subnetID ID", subnetID)

	// TODO: process podSubnetCIDR properly, for now this assumes pod subnet to be /16 and pod network /24
	cidrArr := strings.Split(podSubnetCIDR, ".")

	ipam.podNetworkIPPrefix = fmt.Sprintf("%s.%s.%d", cidrArr[0], cidrArr[1], subnetID)
	ipam.podNetworkPrefixLen = podNetworkPrefixLen

	logger.Infof("POD network for this node will be %sX/%d", ipam.podNetworkIPPrefix, ipam.podNetworkPrefixLen)

	return ipam
}

// getPodNetworkCIDR returns pod network CIDR ("network_address/prefix_length").
func (i *IPAM) getPodNetworkCIDR() string {
	return fmt.Sprintf("%s.%d/%d", i.podNetworkIPPrefix, 0, i.podNetworkPrefixLen)
}

// getPodGatewayIP returns gateway IP address for the pod subnet.
func (i *IPAM) getPodGatewayIP() string {
	return fmt.Sprintf("%s.%d", i.podNetworkIPPrefix, 1)
}

// getNextPodIP returns next available pod IP address.
func (i *IPAM) getNextPodIP() string {
	// TODO: implement proper pool logic instead of sequence numbers

	// assign next available IP
	i.podSeqID++
	ip := fmt.Sprintf("%s.%d", i.podNetworkIPPrefix, i.podSeqID)

	i.Logger.Infof("Assigned new pod IP %s", ip)

	return ip
}

// releasePodIP releases the pod IP address, so that it can be reused by the next pods.
func (i *IPAM) releasePodIP(ip string) error {
	// TODO: implement
	return fmt.Errorf("not yet implemented")
}
<commit_msg>fix typo<commit_after>// Copyright (c) 2017 Cisco and/or its affiliates.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at:
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package contiv

import (
	"crypto/sha1"
	"fmt"
	"strings"

	"github.com/ligato/cn-infra/logging"
)

// IPAM represents the basic Contiv IPAM module.
// TODO: this is just an initial implementation, requires a lot of work.
type IPAM struct {
	logging.Logger

	podNetworkIPPrefix  string // prefix with pod network IP, ID of the pod can be appended to this to get the real pod IP
	podNetworkPrefixLen uint32 // prefix length of pod network used on this host
	podSeqID            uint32 // pod sequence number used to allocate an uniquie IP address to each POD
}

// newIPAM returns new IPAM module to be used on the host.
func newIPAM(logger logging.Logger, podSubnetCIDR string, podNetworkPrefixLen uint32, agentLabel string) *IPAM {
	ipam := &IPAM{
		Logger:   logger,
		podSeqID: 1, // .1 wil be the gateway address
	}

	// calculate POD subnetID ID based on agentLabel
	// TODO: implement proper allocation logic based on ETCD
	h := sha1.New()
	h.Write([]byte(agentLabel))
	sum := h.Sum(nil)
	subnetID := uint8(sum[0])
	logger.Infof("Will use %d as the POD subnetID ID", subnetID)

	// TODO: process podSubnetCIDR properly, for now this assumes pod subnet to be /16 and pod network /24
	cidrArr := strings.Split(podSubnetCIDR, ".")

	ipam.podNetworkIPPrefix = fmt.Sprintf("%s.%s.%d", cidrArr[0], cidrArr[1], subnetID)
	ipam.podNetworkPrefixLen = podNetworkPrefixLen

	logger.Infof("POD network for this node will be %sX/%d", ipam.podNetworkIPPrefix, ipam.podNetworkPrefixLen)

	return ipam
}

// getPodNetworkCIDR returns pod network CIDR ("network_address/prefix_length").
func (i *IPAM) getPodNetworkCIDR() string {
	return fmt.Sprintf("%s.%d/%d", i.podNetworkIPPrefix, 0, i.podNetworkPrefixLen)
}

// getPodGatewayIP returns gateway IP address for the pod network.
func (i *IPAM) getPodGatewayIP() string {
	return fmt.Sprintf("%s.%d", i.podNetworkIPPrefix, 1)
}

// getNextPodIP returns next available pod IP address.
func (i *IPAM) getNextPodIP() string {
	// TODO: implement proper pool logic instead of sequence numbers

	// assign next available IP
	i.podSeqID++
	ip := fmt.Sprintf("%s.%d", i.podNetworkIPPrefix, i.podSeqID)

	i.Logger.Infof("Assigned new pod IP %s", ip)

	return ip
}

// releasePodIP releases the pod IP address, so that it can be reused by the next pods.
func (i *IPAM) releasePodIP(ip string) error {
	// TODO: implement
	return fmt.Errorf("not yet implemented")
}
<|endoftext|>


<commit_before>package errors

import (
	"fmt"
	"strconv"
	"strings"
)

type info struct {
	id   uint32
	name string
	doc  string
}

func (i info) String() string {
	return fmt.Sprintf("%d/0x%08x (%s): %s", i.id, i.id, i.name, i.doc)
}

func find(table []info, arg string) (info, error) {
	if r, err := findName(table, arg); err == nil {
		return r, nil
	}
	code, err := atoi(arg)
	if err != nil {
		return info{}, fmt.Errorf("%s: not found", arg)
	}
	r, err := findCode(table, code)
	if err != nil {
		return info{}, err
	}
	return r, nil
}

func findName(table []info, name string) (info, error) {
	for _, r := range table {
		if name == r.name {
			return r, nil
		}
	}
	return info{}, fmt.Errorf("%s: not found", name)
}

func findCode(table []info, code uint32) (info, error) {
	for _, r := range table {
		if code == r.id {
			return r, nil
		}
	}
	return info{}, fmt.Errorf("%s: not found", code)
}

func atoi(s string) (uint32, error) {
	base := 10
	if strings.HasPrefix(s, "0x") {
		s = s[2:]
		base = 16
	} else if strings.HasPrefix(s, "h") {
		s = s[2:]
		base = 16
	}
	r, err := strconv.ParseUint(s, base, 32)
	if err != nil {
		return 0, err
	}
	return uint32(r), nil
}
<commit_msg>plugins/errors: simple not found msg<commit_after>package errors

import (
	"errors"
	"fmt"
	"strconv"
	"strings"
)

type info struct {
	id   uint32
	name string
	doc  string
}

func (i info) String() string {
	return fmt.Sprintf("%d/0x%08x (%s): %s", i.id, i.id, i.name, i.doc)
}

func find(table []info, arg string) (info, error) {
	if r, err := findName(table, arg); err == nil {
		return r, nil
	}
	code, err := atoi(arg)
	if err != nil {
		return info{}, errors.New("not found")
	}
	r, err := findCode(table, code)
	if err != nil {
		return info{}, err
	}
	return r, nil
}

func findName(table []info, name string) (info, error) {
	for _, r := range table {
		if name == r.name {
			return r, nil
		}
	}
	return info{}, errors.New("not found")
}

func findCode(table []info, code uint32) (info, error) {
	for _, r := range table {
		if code == r.id {
			return r, nil
		}
	}
	return info{}, errors.New("not found")
}

func atoi(s string) (uint32, error) {
	base := 10
	if strings.HasPrefix(s, "0x") {
		s = s[2:]
		base = 16
	} else if strings.HasPrefix(s, "h") {
		s = s[2:]
		base = 16
	}
	r, err := strconv.ParseUint(s, base, 32)
	if err != nil {
		return 0, err
	}
	return uint32(r), nil
}
<|endoftext|>


<commit_before>// Copyright The OpenTelemetry Authors
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package conventions

// OpenTelemetry Semantic Convention values for Resource attribute names.
// See: https://github.com/open-telemetry/opentelemetry-specification/blob/master/specification/resource/semantic_conventions/README.md
const (
	AttributeServiceName           = "service.name"
	AttributeServiceNamespace      = "service.namespace"
	AttributeServiceInstance       = "service.instance.id"
	AttributeServiceVersion        = "service.version"
	AttributeTelemetrySDKName      = "telemetry.sdk.name"
	AttributeTelemetrySDKLanguage  = "telemetry.sdk.language"
	AttributeTelemetrySDKVersion   = "telemetry.sdk.version"
	AttributeContainerName         = "container.name"
	AttributeContainerImage        = "container.image.name"
	AttributeContainerTag          = "container.image.tag"
	AttributeFaasName              = "faas.name"
	AttributeFaasID                = "faas.id"
	AttributeFaasVersion           = "faas.version"
	AttributeFaasInstance          = "faas.instance"
	AttributeK8sCluster            = "k8s.cluster.name"
	AttributeK8sNamespace          = "k8s.namespace.name"
	AttributeK8sPod                = "k8s.pod.name"
	AttributeK8sDeployment         = "k8s.deployment.name"
	AttributeHostHostname          = "host.hostname"
	AttributeHostID                = "host.id"
	AttributeHostName              = "host.name"
	AttributeHostType              = "host.type"
	AttributeHostImageName         = "host.image.name"
	AttributeHostImageID           = "host.image.id"
	AttributeHostImageVersion      = "host.image.version"
	AttributeProcessID             = "process.id"
	AttributeProcessExecutableName = "process.executable.name"
	AttributeProcessExecutablePath = "process.executable.path"
	AttributeProcessCommand        = "process.command"
	AttributeProcessCommandLine    = "process.command_line"
	AttributeProcessUsername       = "process.username"
	AttributeCloudProvider         = "cloud.provider"
	AttributeCloudAccount          = "cloud.account.id"
	AttributeCloudRegion           = "cloud.region"
	AttributeCloudZone             = "cloud.zone"
)

// OpenTelemetry Semantic Convention values for general Span attribute names.
// See: https://github.com/open-telemetry/opentelemetry-specification/blob/master/specification/trace/semantic_conventions/span-general.md
const (
	AttributeComponent    = "component"
	AttributeNetTransport = "net.transport"
	AttributeNetPeerIP    = "net.peer.ip"
	AttributeNetPeerPort  = "net.peer.port"
	AttributeNetPeerName  = "net.peer.name"
	AttributeNetHostIP    = "net.host.ip"
	AttributeNetHostPort  = "net.host.port"
	AttributeNetHostName  = "net.host.name"
	AttributeEnduserID    = "enduser.id"
	AttributeEnduserRole  = "enduser.role"
	AttributeEnduserScope = "enduser.scope"
	AttributePeerService  = "peer.service"
)

// OpenTelemetry Semantic Convention values for component attribute values.
// Possibly being removed due to issue #336
const (
	ComponentTypeHTTP = "http"
	ComponentTypeGRPC = "grpc"
)

// OpenTelemetry Semantic Convention attribute names for HTTP related attributes
// See: https://github.com/open-telemetry/opentelemetry-specification/blob/master/specification/trace/semantic_conventions/http.md
const (
	AttributeHTTPMethod                            = "http.method"
	AttributeHTTPURL                               = "http.url"
	AttributeHTTPTarget                            = "http.target"
	AttributeHTTPHost                              = "http.host"
	AttributeHTTPScheme                            = "http.scheme"
	AttributeHTTPStatusCode                        = "http.status_code"
	AttributeHTTPStatusText                        = "http.status_text"
	AttributeHTTPFlavor                            = "http.flavor"
	AttributeHTTPServerName                        = "http.server_name"
	AttributeHTTPHostName                          = "host.name"
	AttributeHTTPHostPort                          = "host.port"
	AttributeHTTPRoute                             = "http.route"
	AttributeHTTPClientIP                          = "http.client_ip"
	AttributeHTTPUserAgent                         = "http.user_agent"
	AttributeHTTPRequestContentLength              = "http.request_content_length"
	AttributeHTTPRequestContentLengthUncompressed  = "http.request_content_length_uncompressed"
	AttributeHTTPResponseContentLength             = "http.response_content_length"
	AttributeHTTPResponseContentLengthUncompressed = "http.response_content_length_uncompressed"
)

// OpenTelemetry Semantic Convention attribute names for database related attributes
// See: https://github.com/open-telemetry/opentelemetry-specification/blob/master/specification/trace/semantic_conventions/database.md
const (
	AttributeDBSystem           = "db.system"
	AttributeDBConnectionString = "db.connection_string"
	AttributeDBUser             = "db.user"

	AttributeDBName      = "db.name"
	AttributeDBStatement = "db.statement"
	AttributeDBOperation = "db.operation"

	AttributeDBMsSQLInstanceName   = "db.mssql.instance_name"
	AttributeDBJDBCDriverClassname = "db.jdbc.driver_classname"
	AttributeDBCassandraKeyspace   = "db.cassandra.keyspace"
	AttributeDBHBaseNamespace      = "db.hbase.namespace"
	AttributeDBRedisDatabaseIndex  = "db.redis.database_index"
	AttributeDBMongoDBCollection   = "db.mongodb.collection"
)

// OpenTelemetry Semantic Convention attribute names for gRPC related attributes
// See: https://github.com/open-telemetry/opentelemetry-specification/blob/master/specification/trace/semantic_conventions/rpc.md
const (
	AttributeRPCSystem               = "rpc.system"
	AttributeRPCService              = "rpc.service"
	AttributeRPCMethod               = "rpc.method"
	EventTypeMessage                 = "message"
	AttributeMessageType             = "message.type"
	MessageTypeReceived              = "RECEIVED"
	MessageTypeSent                  = "SENT"
	AttributeMessageID               = "message.id"
	AttributeMessageCompressedSize   = "message.compressed_size"
	AttributeMessageUncompressedSize = "message.uncompressed_size"
)

// OpenTelemetry Semantic Convention attribute names for FaaS related attributes
// See: https://github.com/open-telemetry/opentelemetry-specification/blob/master/specification/trace/semantic_conventions/faas.md
const (
	AttributeFaaSTrigger            = "faas.trigger"
	AttributeFaaSExecution          = "faas.execution"
	AttributeFaaSDocumentCollection = "faas.document.collection"
	AttributeFaaSDocumentOperation  = "faas.document.operation"
	AttributeFaaSDocumentTime       = "faas.document.time"
	AttributeFaaSDocumentName       = "faas.document.name"
	AttributeFaaSTime               = "faas.time"
	AttributeFaaSCron               = "faas.cron"
	FaaSTriggerDataSource           = "datasource"
	FaaSTriggerHTTP                 = "http"
	FaaSTriggerPubSub               = "pubsub"
	FaaSTriggerTimer                = "timer"
	FaaSTriggerOther                = "other"
)

// OpenTelemetry Semantic Convention attribute names for messaging system related attributes
// See: https://github.com/open-telemetry/opentelemetry-specification/blob/master/specification/trace/semantic_conventions/messaging.md
const (
	AttributeMessagingSystem                = "messaging.system"
	AttributeMessagingDestination           = "messaging.destination"
	AttributeMessagingDestinationKind       = "messaging.destination_kind"
	AttributeMessagingTempDestination       = "messaging.temp_destination"
	AttributeMessagingProtocol              = "messaging.protocol"
	AttributeMessagingProtocolVersion       = "messaging.protocol_version"
	AttributeMessagingURL                   = "messaging.url"
	AttributeMessagingMessageID             = "messaging.message_id"
	AttributeMessagingConversationID        = "messaging.conversation_id"
	AttributeMessagingPayloadSize           = "messaging.message_payload_size_bytes"
	AttributeMessagingPayloadCompressedSize = "messaging.message_payload_compressed_size_bytes"
	AttributeMessagingOperation             = "messaging.operation"
)

// OpenTelemetry Semantic Convention attribute names for exceptions
// See: https://github.com/open-telemetry/opentelemetry-specification/blob/master/specification/trace/semantic_conventions/exceptions.md
const (
	AttributeExceptionEventName  = "exception"
	AttributeExceptionType       = "exception.type"
	AttributeExceptionMessage    = "exception.message"
	AttributeExceptionStacktrace = "exception.stacktrace"
)
<commit_msg>Add recently included conventions (#1519)<commit_after>// Copyright The OpenTelemetry Authors
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package conventions

// OpenTelemetry Semantic Convention values for Resource attribute names.
// See: https://github.com/open-telemetry/opentelemetry-specification/blob/master/specification/resource/semantic_conventions/README.md
const (
	AttributeServiceName           = "service.name"
	AttributeServiceNamespace      = "service.namespace"
	AttributeServiceInstance       = "service.instance.id"
	AttributeServiceVersion        = "service.version"
	AttributeTelemetrySDKName      = "telemetry.sdk.name"
	AttributeTelemetrySDKLanguage  = "telemetry.sdk.language"
	AttributeTelemetrySDKVersion   = "telemetry.sdk.version"
	AttributeContainerName         = "container.name"
	AttributeContainerID           = "container.id"
	AttributeContainerImage        = "container.image.name"
	AttributeContainerTag          = "container.image.tag"
	AttributeFaasName              = "faas.name"
	AttributeFaasID                = "faas.id"
	AttributeFaasVersion           = "faas.version"
	AttributeFaasInstance          = "faas.instance"
	AttributeK8sCluster            = "k8s.cluster.name"
	AttributeK8sNamespace          = "k8s.namespace.name"
	AttributeK8sContainer          = "k8s.container.name"
	AttributeK8sPod                = "k8s.pod.name"
	AttributeK8sPodUID             = "k8s.pod.uid"
	AttributeK8sCronJob            = "k8s.cronjob.name"
	AttributeK8sCronJobUID         = "k8s.cronjob.uid"
	AttributeK8sJob                = "k8s.job.name"
	AttributeK8sJobUID             = "k8s.job.uid"
	AttributeK8sDeployment         = "k8s.deployment.name"
	AttributeK8sDeploymentUID      = "k8s.deployment.uid"
	AttributeK8sDaemonSet          = "k8s.daemonset.name"
	AttributeK8sDaemonSetUID       = "k8s.daemonset.uid"
	AttributeK8sReplicaSet         = "k8s.replicaset.name"
	AttributeK8sReplicaSetUID      = "k8s.replicaset.uid"
	AttributeK8sStatefulSet        = "k8s.statefulset.name"
	AttributeK8sStatefulSetUID     = "k8s.statefulset.uid"
	AttributeHostHostname          = "host.hostname"
	AttributeHostID                = "host.id"
	AttributeHostName              = "host.name"
	AttributeHostType              = "host.type"
	AttributeHostImageName         = "host.image.name"
	AttributeHostImageID           = "host.image.id"
	AttributeHostImageVersion      = "host.image.version"
	AttributeProcessID             = "process.id"
	AttributeProcessExecutableName = "process.executable.name"
	AttributeProcessExecutablePath = "process.executable.path"
	AttributeProcessCommand        = "process.command"
	AttributeProcessCommandLine    = "process.command_line"
	AttributeProcessUsername       = "process.username"
	AttributeCloudProvider         = "cloud.provider"
	AttributeCloudAccount          = "cloud.account.id"
	AttributeCloudRegion           = "cloud.region"
	AttributeCloudZone             = "cloud.zone"
)

// OpenTelemetry Semantic Convention values for general Span attribute names.
// See: https://github.com/open-telemetry/opentelemetry-specification/blob/master/specification/trace/semantic_conventions/span-general.md
const (
	AttributeComponent    = "component"
	AttributeNetTransport = "net.transport"
	AttributeNetPeerIP    = "net.peer.ip"
	AttributeNetPeerPort  = "net.peer.port"
	AttributeNetPeerName  = "net.peer.name"
	AttributeNetHostIP    = "net.host.ip"
	AttributeNetHostPort  = "net.host.port"
	AttributeNetHostName  = "net.host.name"
	AttributeEnduserID    = "enduser.id"
	AttributeEnduserRole  = "enduser.role"
	AttributeEnduserScope = "enduser.scope"
	AttributePeerService  = "peer.service"
)

// OpenTelemetry Semantic Convention values for component attribute values.
// Possibly being removed due to issue #336
const (
	ComponentTypeHTTP = "http"
	ComponentTypeGRPC = "grpc"
)

// OpenTelemetry Semantic Convention attribute names for HTTP related attributes
// See: https://github.com/open-telemetry/opentelemetry-specification/blob/master/specification/trace/semantic_conventions/http.md
const (
	AttributeHTTPMethod                            = "http.method"
	AttributeHTTPURL                               = "http.url"
	AttributeHTTPTarget                            = "http.target"
	AttributeHTTPHost                              = "http.host"
	AttributeHTTPScheme                            = "http.scheme"
	AttributeHTTPStatusCode                        = "http.status_code"
	AttributeHTTPStatusText                        = "http.status_text"
	AttributeHTTPFlavor                            = "http.flavor"
	AttributeHTTPServerName                        = "http.server_name"
	AttributeHTTPHostName                          = "host.name"
	AttributeHTTPHostPort                          = "host.port"
	AttributeHTTPRoute                             = "http.route"
	AttributeHTTPClientIP                          = "http.client_ip"
	AttributeHTTPUserAgent                         = "http.user_agent"
	AttributeHTTPRequestContentLength              = "http.request_content_length"
	AttributeHTTPRequestContentLengthUncompressed  = "http.request_content_length_uncompressed"
	AttributeHTTPResponseContentLength             = "http.response_content_length"
	AttributeHTTPResponseContentLengthUncompressed = "http.response_content_length_uncompressed"
)

// OpenTelemetry Semantic Convention attribute names for database related attributes
// See: https://github.com/open-telemetry/opentelemetry-specification/blob/master/specification/trace/semantic_conventions/database.md
const (
	AttributeDBSystem           = "db.system"
	AttributeDBConnectionString = "db.connection_string"
	AttributeDBUser             = "db.user"

	AttributeDBName      = "db.name"
	AttributeDBStatement = "db.statement"
	AttributeDBOperation = "db.operation"

	AttributeDBMsSQLInstanceName   = "db.mssql.instance_name"
	AttributeDBJDBCDriverClassname = "db.jdbc.driver_classname"
	AttributeDBCassandraKeyspace   = "db.cassandra.keyspace"
	AttributeDBHBaseNamespace      = "db.hbase.namespace"
	AttributeDBRedisDatabaseIndex  = "db.redis.database_index"
	AttributeDBMongoDBCollection   = "db.mongodb.collection"
)

// OpenTelemetry Semantic Convention attribute names for gRPC related attributes
// See: https://github.com/open-telemetry/opentelemetry-specification/blob/master/specification/trace/semantic_conventions/rpc.md
const (
	AttributeRPCSystem               = "rpc.system"
	AttributeRPCService              = "rpc.service"
	AttributeRPCMethod               = "rpc.method"
	EventTypeMessage                 = "message"
	AttributeMessageType             = "message.type"
	MessageTypeReceived              = "RECEIVED"
	MessageTypeSent                  = "SENT"
	AttributeMessageID               = "message.id"
	AttributeMessageCompressedSize   = "message.compressed_size"
	AttributeMessageUncompressedSize = "message.uncompressed_size"
)

// OpenTelemetry Semantic Convention attribute names for FaaS related attributes
// See: https://github.com/open-telemetry/opentelemetry-specification/blob/master/specification/trace/semantic_conventions/faas.md
const (
	AttributeFaaSTrigger            = "faas.trigger"
	AttributeFaaSExecution          = "faas.execution"
	AttributeFaaSDocumentCollection = "faas.document.collection"
	AttributeFaaSDocumentOperation  = "faas.document.operation"
	AttributeFaaSDocumentTime       = "faas.document.time"
	AttributeFaaSDocumentName       = "faas.document.name"
	AttributeFaaSTime               = "faas.time"
	AttributeFaaSCron               = "faas.cron"
	FaaSTriggerDataSource           = "datasource"
	FaaSTriggerHTTP                 = "http"
	FaaSTriggerPubSub               = "pubsub"
	FaaSTriggerTimer                = "timer"
	FaaSTriggerOther                = "other"
)

// OpenTelemetry Semantic Convention attribute names for messaging system related attributes
// See: https://github.com/open-telemetry/opentelemetry-specification/blob/master/specification/trace/semantic_conventions/messaging.md
const (
	AttributeMessagingSystem                = "messaging.system"
	AttributeMessagingDestination           = "messaging.destination"
	AttributeMessagingDestinationKind       = "messaging.destination_kind"
	AttributeMessagingTempDestination       = "messaging.temp_destination"
	AttributeMessagingProtocol              = "messaging.protocol"
	AttributeMessagingProtocolVersion       = "messaging.protocol_version"
	AttributeMessagingURL                   = "messaging.url"
	AttributeMessagingMessageID             = "messaging.message_id"
	AttributeMessagingConversationID        = "messaging.conversation_id"
	AttributeMessagingPayloadSize           = "messaging.message_payload_size_bytes"
	AttributeMessagingPayloadCompressedSize = "messaging.message_payload_compressed_size_bytes"
	AttributeMessagingOperation             = "messaging.operation"
)

// OpenTelemetry Semantic Convention attribute names for exceptions
// See: https://github.com/open-telemetry/opentelemetry-specification/blob/master/specification/trace/semantic_conventions/exceptions.md
const (
	AttributeExceptionEventName  = "exception"
	AttributeExceptionType       = "exception.type"
	AttributeExceptionMessage    = "exception.message"
	AttributeExceptionStacktrace = "exception.stacktrace"
)
<|endoftext|>


<commit_before>package user

import (
	"bytes"
	"encoding/json"
	htmlTemplate "html/template"
	"net/http"
	"net/url"
	"xorkevin.dev/governor"
	"xorkevin.dev/governor/service/user/approval/model"
	"xorkevin.dev/governor/util/rank"
)

type (
	emailNewUser struct {
		Userid    string
		Key       string
		URL       string
		FirstName string
		LastName  string
		Username  string
	}

	queryEmailNewUser struct {
		Userid    string
		Key       string
		FirstName string
		LastName  string
		Username  string
	}
)

const (
	newUserTemplate = "newuser"
)

func (e *emailNewUser) Query() queryEmailNewUser {
	return queryEmailNewUser{
		Userid:    url.QueryEscape(e.Userid),
		Key:       url.QueryEscape(e.Key),
		FirstName: url.QueryEscape(e.FirstName),
		LastName:  url.QueryEscape(e.LastName),
		Username:  url.QueryEscape(e.Username),
	}
}

func (e *emailNewUser) computeURL(base string, tpl *htmlTemplate.Template) error {
	b := &bytes.Buffer{}
	if err := tpl.Execute(b, e.Query()); err != nil {
		return governor.NewError("Failed executing new user url template", http.StatusInternalServerError, err)
	}
	e.URL = base + b.String()
	return nil
}

type (
	resUserUpdate struct {
		Userid   string `json:"userid"`
		Username string `json:"username"`
	}
)

// CreateUser creates a new user and places it into approvals
func (s *service) CreateUser(ruser reqUserPost) (*resUserUpdate, error) {
	m2, err := s.users.GetByUsername(ruser.Username)
	if err != nil && governor.ErrorStatus(err) != http.StatusNotFound {
		return nil, err
	}
	if m2 != nil && m2.Username == ruser.Username {
		return nil, governor.NewErrorUser("Username is already taken", http.StatusBadRequest, nil)
	}

	m2, err = s.users.GetByEmail(ruser.Email)
	if err != nil && governor.ErrorStatus(err) != http.StatusNotFound {
		return nil, err
	}
	if m2 != nil && m2.Email == ruser.Email {
		return nil, governor.NewErrorUser("Email is already used by another account", http.StatusBadRequest, nil)
	}

	m, err := s.users.New(ruser.Username, ruser.Password, ruser.Email, ruser.FirstName, ruser.LastName)
	if err != nil {
		return nil, err
	}

	am := s.approvals.New(m)
	if s.userApproval {
		if err := s.approvals.Insert(am); err != nil {
			if governor.ErrorStatus(err) == http.StatusBadRequest {
				return nil, governor.NewErrorUser("", 0, err)
			}
			return nil, err
		}
	} else {
		code, err := s.approvals.RehashCode(am)
		if err != nil {
			return nil, err
		}
		if err := s.approvals.Insert(am); err != nil {
			if governor.ErrorStatus(err) == http.StatusBadRequest {
				return nil, governor.NewErrorUser("", 0, err)
			}
			return nil, err
		}
		if err := s.sendNewUserEmail(code, am); err != nil {
			return nil, err
		}
	}

	return &resUserUpdate{
		Userid:   m.Userid,
		Username: m.Username,
	}, nil
}

type (
	resApproval struct {
		Userid       string `json:"userid"`
		Username     string `json:"username"`
		Email        string `json:"email"`
		FirstName    string `json:"first_name"`
		LastName     string `json:"last_name"`
		CreationTime int64  `json:"creation_time"`
		Approved     bool   `json:"approved"`
		CodeTime     int64  `json:"code_time"`
	}

	resApprovals struct {
		Approvals []resApproval `json:"approvals"`
	}
)

func (s *service) GetUserApprovals(limit, offset int) (*resApprovals, error) {
	m, err := s.approvals.GetGroup(limit, offset)
	if err != nil {
		return nil, err
	}
	approvals := make([]resApproval, 0, len(m))
	for _, i := range m {
		approvals = append(approvals, resApproval{
			Userid:       i.Userid,
			Username:     i.Username,
			Email:        i.Email,
			FirstName:    i.FirstName,
			LastName:     i.LastName,
			CreationTime: i.CreationTime,
			Approved:     i.Approved,
			CodeTime:     i.CodeTime,
		})
	}
	return &resApprovals{
		Approvals: approvals,
	}, nil
}

func (s *service) ApproveUser(userid string) error {
	m, err := s.approvals.GetByID(userid)
	if err != nil {
		if governor.ErrorStatus(err) == http.StatusNotFound {
			return governor.NewErrorUser("", 0, err)
		}
		return err
	}
	code, err := s.approvals.RehashCode(m)
	if err != nil {
		return err
	}
	if err := s.approvals.Update(m); err != nil {
		return err
	}
	if err := s.sendNewUserEmail(code, m); err != nil {
		return err
	}
	return nil
}

func (s *service) DeleteUserApproval(userid string) error {
	m, err := s.approvals.GetByID(userid)
	if err != nil {
		if governor.ErrorStatus(err) == http.StatusNotFound {
			return governor.NewErrorUser("", 0, err)
		}
		return err
	}
	if err := s.approvals.Delete(m); err != nil {
		return err
	}
	return nil
}

func (s *service) sendNewUserEmail(code string, m *approvalmodel.Model) error {
	emdata := emailNewUser{
		Userid:    m.Userid,
		Key:       code,
		FirstName: m.FirstName,
		LastName:  m.LastName,
		Username:  m.Username,
	}
	if err := emdata.computeURL(s.emailurlbase, s.tplnewuser); err != nil {
		return err
	}
	if err := s.mailer.Send("", "", []string{m.Email}, newUserTemplate, emdata); err != nil {
		return governor.NewError("Failed to send account verification email", http.StatusInternalServerError, err)
	}
	return nil
}

// CommitUser takes a user from approvals and places it into the user db
func (s *service) CommitUser(userid string, key string) (*resUserUpdate, error) {
	am, err := s.approvals.GetByID(userid)
	if err != nil {
		if governor.ErrorStatus(err) == http.StatusNotFound {
			return nil, governor.NewErrorUser("", 0, err)
		}
		return nil, err
	}
	if !am.Approved {
		return nil, governor.NewErrorUser("Not approved", 0, nil)
	}
	if ok, err := s.approvals.ValidateCode(key, am); err != nil {
		return nil, governor.NewError("Failed to verify key", http.StatusInternalServerError, err)
	} else if !ok {
		return nil, governor.NewErrorUser("Invalid key", http.StatusForbidden, nil)
	}
	m := s.approvals.ToUserModel(am)

	userProps := NewUserProps{
		Userid:       m.Userid,
		Username:     m.Username,
		Email:        m.Email,
		FirstName:    m.FirstName,
		LastName:     m.LastName,
		CreationTime: m.CreationTime,
	}

	b := bytes.Buffer{}
	if err := json.NewEncoder(&b).Encode(userProps); err != nil {
		return nil, governor.NewError("Failed to encode user props to json", http.StatusInternalServerError, err)
	}

	if err := s.users.Insert(m); err != nil {
		if governor.ErrorStatus(err) == http.StatusBadRequest {
			return nil, governor.NewErrorUser("", 0, err)
		}
		return nil, err
	}
	if err := s.roles.InsertRoles(m.Userid, rank.BaseUser()); err != nil {
		return nil, err
	}

	if err := s.queue.Publish(NewUserQueueID, b.Bytes()); err != nil {
		s.logger.Error("Failed to publish new user", map[string]string{
			"error":      err.Error(),
			"actiontype": "publishnewuser",
		})
	}

	if err := s.approvals.Delete(am); err != nil {
		s.logger.Error("Failed to clean up user approval", map[string]string{
			"error":      err.Error(),
			"actiontype": "commitusercleanup",
		})
	}

	s.logger.Info("created user", map[string]string{
		"userid":     m.Userid,
		"username":   m.Username,
		"actiontype": "commituser",
	})

	return &resUserUpdate{
		Userid:   m.Userid,
		Username: m.Username,
	}, nil
}

func (s *service) DeleteUser(userid string, username string, password string) error {
	m, err := s.users.GetByID(userid)
	if err != nil {
		if governor.ErrorStatus(err) == http.StatusNotFound {
			return governor.NewErrorUser("", 0, err)
		}
		return err
	}

	if m.Username != username {
		return governor.NewErrorUser("Information does not match", http.StatusBadRequest, err)
	}
	if roles, err := s.roles.IntersectRoles(userid, rank.Rank{"admin": struct{}{}}); err != nil {
		return err
	} else if roles.Has("admin") {
		return governor.NewErrorUser("Not allowed to delete admin user", http.StatusForbidden, err)
	}
	if ok, err := s.users.ValidatePass(password, m); err != nil {
		return err
	} else if !ok {
		return governor.NewErrorUser("Incorrect password", http.StatusForbidden, nil)
	}

	userProps := DeleteUserProps{
		Userid: m.Userid,
	}
	b := bytes.Buffer{}
	if err := json.NewEncoder(&b).Encode(userProps); err != nil {
		return governor.NewError("Failed to encode user props to json", http.StatusInternalServerError, err)
	}

	if err := s.DeleteUserApikeys(userid); err != nil {
		return err
	}

	if err := s.KillAllSessions(userid); err != nil {
		return err
	}

	if err := s.roles.DeleteAllRoles(userid); err != nil {
		return err
	}

	if err := s.users.Delete(m); err != nil {
		return err
	}

	if err := s.queue.Publish(DeleteUserQueueID, b.Bytes()); err != nil {
		s.logger.Error("failed to publish delete user", map[string]string{
			"error":      err.Error(),
			"actiontype": "publishdeleteuser",
		})
	}
	return nil
}

// DecodeNewUserProps marshals json encoded new user props into a struct
func DecodeNewUserProps(msgdata []byte) (*NewUserProps, error) {
	m := &NewUserProps{}
	if err := json.NewDecoder(bytes.NewBuffer(msgdata)).Decode(m); err != nil {
		return nil, governor.NewError("Failed to decode new user props", http.StatusInternalServerError, err)
	}
	return m, nil
}

// DecodeDeleteUserProps marshals json encoded delete user props into a struct
func DecodeDeleteUserProps(msgdata []byte) (*DeleteUserProps, error) {
	m := &DeleteUserProps{}
	if err := json.NewDecoder(bytes.NewBuffer(msgdata)).Decode(m); err != nil {
		return nil, governor.NewError("Failed to decode delete user props", http.StatusInternalServerError, err)
	}
	return m, nil
}
<commit_msg>Fix new user code expiration<commit_after>package user

import (
	"bytes"
	"encoding/json"
	htmlTemplate "html/template"
	"net/http"
	"net/url"
	"time"
	"xorkevin.dev/governor"
	"xorkevin.dev/governor/service/user/approval/model"
	"xorkevin.dev/governor/util/rank"
)

type (
	emailNewUser struct {
		Userid    string
		Key       string
		URL       string
		FirstName string
		LastName  string
		Username  string
	}

	queryEmailNewUser struct {
		Userid    string
		Key       string
		FirstName string
		LastName  string
		Username  string
	}
)

const (
	newUserTemplate = "newuser"
)

func (e *emailNewUser) Query() queryEmailNewUser {
	return queryEmailNewUser{
		Userid:    url.QueryEscape(e.Userid),
		Key:       url.QueryEscape(e.Key),
		FirstName: url.QueryEscape(e.FirstName),
		LastName:  url.QueryEscape(e.LastName),
		Username:  url.QueryEscape(e.Username),
	}
}

func (e *emailNewUser) computeURL(base string, tpl *htmlTemplate.Template) error {
	b := &bytes.Buffer{}
	if err := tpl.Execute(b, e.Query()); err != nil {
		return governor.NewError("Failed executing new user url template", http.StatusInternalServerError, err)
	}
	e.URL = base + b.String()
	return nil
}

type (
	resUserUpdate struct {
		Userid   string `json:"userid"`
		Username string `json:"username"`
	}
)

// CreateUser creates a new user and places it into approvals
func (s *service) CreateUser(ruser reqUserPost) (*resUserUpdate, error) {
	m2, err := s.users.GetByUsername(ruser.Username)
	if err != nil && governor.ErrorStatus(err) != http.StatusNotFound {
		return nil, err
	}
	if m2 != nil && m2.Username == ruser.Username {
		return nil, governor.NewErrorUser("Username is already taken", http.StatusBadRequest, nil)
	}

	m2, err = s.users.GetByEmail(ruser.Email)
	if err != nil && governor.ErrorStatus(err) != http.StatusNotFound {
		return nil, err
	}
	if m2 != nil && m2.Email == ruser.Email {
		return nil, governor.NewErrorUser("Email is already used by another account", http.StatusBadRequest, nil)
	}

	m, err := s.users.New(ruser.Username, ruser.Password, ruser.Email, ruser.FirstName, ruser.LastName)
	if err != nil {
		return nil, err
	}

	am := s.approvals.New(m)
	if s.userApproval {
		if err := s.approvals.Insert(am); err != nil {
			if governor.ErrorStatus(err) == http.StatusBadRequest {
				return nil, governor.NewErrorUser("", 0, err)
			}
			return nil, err
		}
	} else {
		code, err := s.approvals.RehashCode(am)
		if err != nil {
			return nil, err
		}
		if err := s.approvals.Insert(am); err != nil {
			if governor.ErrorStatus(err) == http.StatusBadRequest {
				return nil, governor.NewErrorUser("", 0, err)
			}
			return nil, err
		}
		if err := s.sendNewUserEmail(code, am); err != nil {
			return nil, err
		}
	}

	return &resUserUpdate{
		Userid:   m.Userid,
		Username: m.Username,
	}, nil
}

type (
	resApproval struct {
		Userid       string `json:"userid"`
		Username     string `json:"username"`
		Email        string `json:"email"`
		FirstName    string `json:"first_name"`
		LastName     string `json:"last_name"`
		CreationTime int64  `json:"creation_time"`
		Approved     bool   `json:"approved"`
		CodeTime     int64  `json:"code_time"`
	}

	resApprovals struct {
		Approvals []resApproval `json:"approvals"`
	}
)

func (s *service) GetUserApprovals(limit, offset int) (*resApprovals, error) {
	m, err := s.approvals.GetGroup(limit, offset)
	if err != nil {
		return nil, err
	}
	approvals := make([]resApproval, 0, len(m))
	for _, i := range m {
		approvals = append(approvals, resApproval{
			Userid:       i.Userid,
			Username:     i.Username,
			Email:        i.Email,
			FirstName:    i.FirstName,
			LastName:     i.LastName,
			CreationTime: i.CreationTime,
			Approved:     i.Approved,
			CodeTime:     i.CodeTime,
		})
	}
	return &resApprovals{
		Approvals: approvals,
	}, nil
}

func (s *service) ApproveUser(userid string) error {
	m, err := s.approvals.GetByID(userid)
	if err != nil {
		if governor.ErrorStatus(err) == http.StatusNotFound {
			return governor.NewErrorUser("", 0, err)
		}
		return err
	}
	code, err := s.approvals.RehashCode(m)
	if err != nil {
		return err
	}
	if err := s.approvals.Update(m); err != nil {
		return err
	}
	if err := s.sendNewUserEmail(code, m); err != nil {
		return err
	}
	return nil
}

func (s *service) DeleteUserApproval(userid string) error {
	m, err := s.approvals.GetByID(userid)
	if err != nil {
		if governor.ErrorStatus(err) == http.StatusNotFound {
			return governor.NewErrorUser("", 0, err)
		}
		return err
	}
	if err := s.approvals.Delete(m); err != nil {
		return err
	}
	return nil
}

func (s *service) sendNewUserEmail(code string, m *approvalmodel.Model) error {
	emdata := emailNewUser{
		Userid:    m.Userid,
		Key:       code,
		FirstName: m.FirstName,
		LastName:  m.LastName,
		Username:  m.Username,
	}
	if err := emdata.computeURL(s.emailurlbase, s.tplnewuser); err != nil {
		return err
	}
	if err := s.mailer.Send("", "", []string{m.Email}, newUserTemplate, emdata); err != nil {
		return governor.NewError("Failed to send account verification email", http.StatusInternalServerError, err)
	}
	return nil
}

// CommitUser takes a user from approvals and places it into the user db
func (s *service) CommitUser(userid string, key string) (*resUserUpdate, error) {
	am, err := s.approvals.GetByID(userid)
	if err != nil {
		if governor.ErrorStatus(err) == http.StatusNotFound {
			return nil, governor.NewErrorUser("", 0, err)
		}
		return nil, err
	}
	if !am.Approved {
		return nil, governor.NewErrorUser("Not approved", http.StatusBadRequest, nil)
	}
	if time.Now().Round(0).Unix() > am.CodeTime+s.confirmTime {
		return nil, governor.NewErrorUser("Code expired", http.StatusBadRequest, nil)
	}
	if ok, err := s.approvals.ValidateCode(key, am); err != nil {
		return nil, governor.NewError("Failed to verify key", http.StatusInternalServerError, err)
	} else if !ok {
		return nil, governor.NewErrorUser("Invalid key", http.StatusForbidden, nil)
	}
	m := s.approvals.ToUserModel(am)

	userProps := NewUserProps{
		Userid:       m.Userid,
		Username:     m.Username,
		Email:        m.Email,
		FirstName:    m.FirstName,
		LastName:     m.LastName,
		CreationTime: m.CreationTime,
	}

	b := bytes.Buffer{}
	if err := json.NewEncoder(&b).Encode(userProps); err != nil {
		return nil, governor.NewError("Failed to encode user props to json", http.StatusInternalServerError, err)
	}

	if err := s.users.Insert(m); err != nil {
		if governor.ErrorStatus(err) == http.StatusBadRequest {
			return nil, governor.NewErrorUser("", 0, err)
		}
		return nil, err
	}
	if err := s.roles.InsertRoles(m.Userid, rank.BaseUser()); err != nil {
		return nil, err
	}

	if err := s.queue.Publish(NewUserQueueID, b.Bytes()); err != nil {
		s.logger.Error("Failed to publish new user", map[string]string{
			"error":      err.Error(),
			"actiontype": "publishnewuser",
		})
	}

	if err := s.approvals.Delete(am); err != nil {
		s.logger.Error("Failed to clean up user approval", map[string]string{
			"error":      err.Error(),
			"actiontype": "commitusercleanup",
		})
	}

	s.logger.Info("created user", map[string]string{
		"userid":     m.Userid,
		"username":   m.Username,
		"actiontype": "commituser",
	})

	return &resUserUpdate{
		Userid:   m.Userid,
		Username: m.Username,
	}, nil
}

func (s *service) DeleteUser(userid string, username string, password string) error {
	m, err := s.users.GetByID(userid)
	if err != nil {
		if governor.ErrorStatus(err) == http.StatusNotFound {
			return governor.NewErrorUser("", 0, err)
		}
		return err
	}

	if m.Username != username {
		return governor.NewErrorUser("Information does not match", http.StatusBadRequest, err)
	}
	if roles, err := s.roles.IntersectRoles(userid, rank.Rank{"admin": struct{}{}}); err != nil {
		return err
	} else if roles.Has("admin") {
		return governor.NewErrorUser("Not allowed to delete admin user", http.StatusForbidden, err)
	}
	if ok, err := s.users.ValidatePass(password, m); err != nil {
		return err
	} else if !ok {
		return governor.NewErrorUser("Incorrect password", http.StatusForbidden, nil)
	}

	userProps := DeleteUserProps{
		Userid: m.Userid,
	}
	b := bytes.Buffer{}
	if err := json.NewEncoder(&b).Encode(userProps); err != nil {
		return governor.NewError("Failed to encode user props to json", http.StatusInternalServerError, err)
	}

	if err := s.DeleteUserApikeys(userid); err != nil {
		return err
	}

	if err := s.KillAllSessions(userid); err != nil {
		return err
	}

	if err := s.roles.DeleteAllRoles(userid); err != nil {
		return err
	}

	if err := s.users.Delete(m); err != nil {
		return err
	}

	if err := s.queue.Publish(DeleteUserQueueID, b.Bytes()); err != nil {
		s.logger.Error("failed to publish delete user", map[string]string{
			"error":      err.Error(),
			"actiontype": "publishdeleteuser",
		})
	}
	return nil
}

// DecodeNewUserProps marshals json encoded new user props into a struct
func DecodeNewUserProps(msgdata []byte) (*NewUserProps, error) {
	m := &NewUserProps{}
	if err := json.NewDecoder(bytes.NewBuffer(msgdata)).Decode(m); err != nil {
		return nil, governor.NewError("Failed to decode new user props", http.StatusInternalServerError, err)
	}
	return m, nil
}

// DecodeDeleteUserProps marshals json encoded delete user props into a struct
func DecodeDeleteUserProps(msgdata []byte) (*DeleteUserProps, error) {
	m := &DeleteUserProps{}
	if err := json.NewDecoder(bytes.NewBuffer(msgdata)).Decode(m); err != nil {
		return nil, governor.NewError("Failed to decode delete user props", http.StatusInternalServerError, err)
	}
	return m, nil
}
<|endoftext|>


<commit_before>package drivers

import (
	"bytes"
	"fmt"
	"os"
	"os/exec"
	"path/filepath"
	"regexp"
	"strings"
	"sync"
	"time"

	log "github.com/Sirupsen/logrus"
	"github.com/docker/go-plugins-helpers/volume"
)

const (
	//MountTimeout timeout before killing a mount try in seconds
	MountTimeout = 30
)

type gvfsVolume struct {
	url         string
	driver      *gvfsVolumeDriver
	password    string
	mountpoint  string
	connections int
}

type gvfsVolumeDriver interface {
	id() DriverType
	isAvailable() bool
	mountpoint() (string, error)
}

//GVfsDriver the global driver responding to call
type GVfsDriver struct {
	sync.RWMutex
	root    string
	env     []string
	volumes map[string]*gvfsVolume
}

//Init start all needed deps and serve response to API call
func Init(root string, dbus string) *GVfsDriver {
	d := &GVfsDriver{
		root:    root,
		env:     make([]string, 1),
		volumes: make(map[string]*gvfsVolume),
	}
	if dbus == "" {
		// start needed dbus like (eval `dbus-launch --sh-syntax`) and get env variable
		result, err := exec.Command("dbus-launch", "--sh-syntax").CombinedOutput() //DBUS_SESSION_BUS_ADDRESS='unix:abstract=/tmp/dbus-JHGXLpeJ6A,guid=25ab632502ebccd43cd403bc58388fab';\n ...
		if err != nil {
			panic(err)
		}
		env := string(result)
		log.Debugf("dbus-launch --sh-syntax -> \n%s", env)
		reDBus := regexp.MustCompile("DBUS_SESSION_BUS_ADDRESS='(.*?)';")
		//rePID := regexp.MustCompile("DBUS_SESSION_BUS_PID=(.*?);")
		matchDBuse := reDBus.FindStringSubmatch(env)
		//matchPID := rePID.FindStringSubmatch(env)
		dbus = matchDBuse[1]
		//TODO plan to kill this add closing ?
	}
	d.env[0] = fmt.Sprintf("DBUS_SESSION_BUS_ADDRESS=%s", dbus)
	err := d.startFuseDeamon()
	if err != nil {
		panic(err) //Something went wrong
	}
	return d
}

func (d GVfsDriver) startFuseDeamon() error {
	//TODO check needed gvfsd + gvfsd-ftp Maybe allready on dbus ?
	// Normaly gvfsd-fuse block such so this like crash but global ?

	fi, err := os.Lstat(d.root)
	if os.IsNotExist(err) {
		if err := os.MkdirAll(d.root, 0755); err != nil {
			return err
		}
	} else if err != nil {
		return err
	}
	if fi != nil && !fi.IsDir() {
		return fmt.Errorf("%v already exist and it's not a directory", d.root)
	}

	err = d.startCmd("/usr/lib/gvfs/gvfsd --no-fuse") //Start global deamon
	if err != nil {
		return err
	}

	err = d.startCmd(fmt.Sprintf("/usr/lib/gvfs/gvfsd-fuse %s -f -o big_writes,use_ino,allow_other,auto_cache,umask=0022", d.root)) //Start ftp handler
	if err != nil {
		return err
	}
	return nil
}

// start deamon in context of this gvfs drive with custome env
func (d GVfsDriver) startCmd(cmd string) error {
	log.Debugf(cmd)
	return setEnv(cmd, d.env).Start()
}

// run deamon in context of this gvfs drive with custome env
func (d GVfsDriver) runCmd(cmd string) error {
	log.Debugf(cmd)
	return setEnv(cmd, d.env).Run()
}

//Create create and init the requested volume
func (d GVfsDriver) Create(r volume.Request) volume.Response {
	log.Debugf("Entering Create: name: %s, options %v", r.Name, r.Options)
	d.Lock()
	defer d.Unlock()

	if r.Options == nil || r.Options["url"] == "" {
		return volume.Response{Err: "url option required"}
	}

	dr, m, err := getDriver(r.Options["url"])
	if err != nil {
		return volume.Response{Err: err.Error()}
	}

	v := &gvfsVolume{
		url:         r.Options["url"],
		driver:      dr,
		password:    r.Options["password"],
		mountpoint:  filepath.Join(d.root, m),
		connections: 0,
	}

	d.volumes[r.Name] = v
	log.Debugf("Volume Created: %v", v)
	return volume.Response{}
}

//Remove remove the requested volume
func (d GVfsDriver) Remove(r volume.Request) volume.Response {
	log.Debugf("Entering Remove: name: %s, options %v", r.Name, r.Options)
	d.Lock()
	defer d.Unlock()
	v, ok := d.volumes[r.Name]

	if !ok {
		return volume.Response{Err: fmt.Sprintf("volume %s not found", r.Name)}
	}
	if v.connections == 0 {
		delete(d.volumes, r.Name)
		return volume.Response{}
	}
	return volume.Response{Err: fmt.Sprintf("volume %s is currently used by a container", r.Name)}
}

//List volumes handled by thos driver
func (d GVfsDriver) List(r volume.Request) volume.Response {
	log.Debugf("Entering List: name: %s, options %v", r.Name, r.Options)

	d.Lock()
	defer d.Unlock()

	var vols []*volume.Volume
	for name, v := range d.volumes {
		vols = append(vols, &volume.Volume{Name: name, Mountpoint: v.mountpoint})
		log.Debugf("Volume found: %s", v)
	}
	return volume.Response{Volumes: vols}
}

//Get get info on the requested volume
func (d GVfsDriver) Get(r volume.Request) volume.Response {
	log.Debugf("Entering Get: name: %s", r.Name)
	d.Lock()
	defer d.Unlock()

	v, ok := d.volumes[r.Name]
	if !ok {
		return volume.Response{Err: fmt.Sprintf("volume %s not found", r.Name)}
	}

	log.Debugf("Volume found: %s", v)
	return volume.Response{Volume: &volume.Volume{Name: r.Name, Mountpoint: v.mountpoint}}
}

//Path get path of the requested volume
func (d GVfsDriver) Path(r volume.Request) volume.Response {
	log.Debugf("Entering Path: name: %s, options %v", r.Name)

	d.RLock()
	defer d.RUnlock()
	v, ok := d.volumes[r.Name]
	if !ok {
		return volume.Response{Err: fmt.Sprintf("volume %s not found", r.Name)}
	}
	log.Debugf("Volume found: %s", v)
	return volume.Response{Mountpoint: v.mountpoint}
}

//Mount mount the requested volume
func (d GVfsDriver) Mount(r volume.MountRequest) volume.Response {
	log.Debugf("Entering Mount: %v", r)
	d.Lock()
	defer d.Unlock()

	v, ok := d.volumes[r.Name]
	if !ok {
		return volume.Response{Err: fmt.Sprintf("volume %s not found", r.Name)}
	}

	if v.connections > 0 {
		v.connections++
		return volume.Response{Mountpoint: v.mountpoint}
	}

	cmd := fmt.Sprintf("gvfs-mount %s", v.url)
	if v.password != "" {
		p := setEnv(cmd, d.env)
		inStd, err := p.StdinPipe()
		if err != nil { //Get a input buffer
			return volume.Response{Err: err.Error()}
		}
		var outStd bytes.Buffer
		p.Stdout = &outStd
		var errStd bytes.Buffer
		p.Stderr = &errStd

		if err := p.Start(); err != nil {
			return volume.Response{Err: err.Error()}
		}
		inStd.Write([]byte(v.password + "\n")) //Send password to process + Send return line

		// wait or timeout
		donec := make(chan error, 1)
		go func() {
			donec <- p.Wait() //Process finish
		}()
		select {
		case <-time.After(MountTimeout * time.Second):
			sOut := outStd.String()
			sErr := errStd.String()
			p.Process.Kill()
			log.Debugf("out : %s", sOut)
			log.Debugf("outErr : %s", sErr)
			return volume.Response{Err: fmt.Sprintf("The command %s timeout", cmd)}
		case <-donec:
			sOut := outStd.String()
			sErr := errStd.String()
			log.Debugf("Password send and command %s return", cmd)
			log.Debugf("out : %s", sOut)
			log.Debugf("outErr : %s", sErr)
			// handle erros like : "Error mounting location: Location is already mounted" or Error mounting location: Could not connect to 10.8.0.7: No route to host
			if strings.Contains(sErr, "Error mounting location") {
				return volume.Response{Err: fmt.Sprintf("Error mounting location : %s", sErr)}
			}
			v.connections++
			break
		}
	} else {
		if err := d.runCmd(cmd); err != nil {
			return volume.Response{Err: err.Error()}
		}
	}

	return volume.Response{Mountpoint: v.mountpoint}
}

//Unmount unmount the requested volume
//TODO Monitor for unmount to remount ?
func (d GVfsDriver) Unmount(r volume.UnmountRequest) volume.Response {
	//Execute gvfs-mount -u $params
	log.Debugf("Entering Unmount: %v", r)

	d.Lock()
	defer d.Unlock()
	v, ok := d.volumes[r.Name]
	if !ok {
		return volume.Response{Err: fmt.Sprintf("volume %s not found", r.Name)}
	}
	if v.connections <= 1 {
		cmd := fmt.Sprintf("gvfs-mount -u %s", v.url)
		if err := d.runCmd(cmd); err != nil {
			return volume.Response{Err: err.Error()}
		}
		v.connections = 0
	} else {
		v.connections--
	}

	return volume.Response{}
}

//Capabilities Send capabilities of the local driver
func (d GVfsDriver) Capabilities(r volume.Request) volume.Response {
	log.Debugf("Entering Capabilities: %v", r)
	return volume.Response{
		Capabilities: volume.Capability{
			Scope: "local",
		},
	}
}
<commit_msg>Fix misspell<commit_after>package drivers

import (
	"bytes"
	"fmt"
	"os"
	"os/exec"
	"path/filepath"
	"regexp"
	"strings"
	"sync"
	"time"

	log "github.com/Sirupsen/logrus"
	"github.com/docker/go-plugins-helpers/volume"
)

const (
	//MountTimeout timeout before killing a mount try in seconds
	MountTimeout = 30
)

type gvfsVolume struct {
	url         string
	driver      *gvfsVolumeDriver
	password    string
	mountpoint  string
	connections int
}

type gvfsVolumeDriver interface {
	id() DriverType
	isAvailable() bool
	mountpoint() (string, error)
}

//GVfsDriver the global driver responding to call
type GVfsDriver struct {
	sync.RWMutex
	root    string
	env     []string
	volumes map[string]*gvfsVolume
}

//Init start all needed deps and serve response to API call
func Init(root string, dbus string) *GVfsDriver {
	d := &GVfsDriver{
		root:    root,
		env:     make([]string, 1),
		volumes: make(map[string]*gvfsVolume),
	}
	if dbus == "" {
		// start needed dbus like (eval `dbus-launch --sh-syntax`) and get env variable
		result, err := exec.Command("dbus-launch", "--sh-syntax").CombinedOutput() //DBUS_SESSION_BUS_ADDRESS='unix:abstract=/tmp/dbus-JHGXLpeJ6A,guid=25ab632502ebccd43cd403bc58388fab';\n ...
		if err != nil {
			panic(err)
		}
		env := string(result)
		log.Debugf("dbus-launch --sh-syntax -> \n%s", env)
		reDBus := regexp.MustCompile("DBUS_SESSION_BUS_ADDRESS='(.*?)';")
		//rePID := regexp.MustCompile("DBUS_SESSION_BUS_PID=(.*?);")
		matchDBuse := reDBus.FindStringSubmatch(env)
		//matchPID := rePID.FindStringSubmatch(env)
		dbus = matchDBuse[1]
		//TODO plan to kill this add closing ?
	}
	d.env[0] = fmt.Sprintf("DBUS_SESSION_BUS_ADDRESS=%s", dbus)
	err := d.startFuseDeamon()
	if err != nil {
		panic(err) //Something went wrong
	}
	return d
}

func (d GVfsDriver) startFuseDeamon() error {
	//TODO check needed gvfsd + gvfsd-ftp Maybe already on dbus ?
	// Normaly gvfsd-fuse block such so this like crash but global ?

	fi, err := os.Lstat(d.root)
	if os.IsNotExist(err) {
		if err := os.MkdirAll(d.root, 0755); err != nil {
			return err
		}
	} else if err != nil {
		return err
	}
	if fi != nil && !fi.IsDir() {
		return fmt.Errorf("%v already exist and it's not a directory", d.root)
	}

	err = d.startCmd("/usr/lib/gvfs/gvfsd --no-fuse") //Start global deamon
	if err != nil {
		return err
	}

	err = d.startCmd(fmt.Sprintf("/usr/lib/gvfs/gvfsd-fuse %s -f -o big_writes,use_ino,allow_other,auto_cache,umask=0022", d.root)) //Start ftp handler
	if err != nil {
		return err
	}
	return nil
}

// start deamon in context of this gvfs drive with custome env
func (d GVfsDriver) startCmd(cmd string) error {
	log.Debugf(cmd)
	return setEnv(cmd, d.env).Start()
}

// run deamon in context of this gvfs drive with custome env
func (d GVfsDriver) runCmd(cmd string) error {
	log.Debugf(cmd)
	return setEnv(cmd, d.env).Run()
}

//Create create and init the requested volume
func (d GVfsDriver) Create(r volume.Request) volume.Response {
	log.Debugf("Entering Create: name: %s, options %v", r.Name, r.Options)
	d.Lock()
	defer d.Unlock()

	if r.Options == nil || r.Options["url"] == "" {
		return volume.Response{Err: "url option required"}
	}

	dr, m, err := getDriver(r.Options["url"])
	if err != nil {
		return volume.Response{Err: err.Error()}
	}

	v := &gvfsVolume{
		url:         r.Options["url"],
		driver:      dr,
		password:    r.Options["password"],
		mountpoint:  filepath.Join(d.root, m),
		connections: 0,
	}

	d.volumes[r.Name] = v
	log.Debugf("Volume Created: %v", v)
	return volume.Response{}
}

//Remove remove the requested volume
func (d GVfsDriver) Remove(r volume.Request) volume.Response {
	log.Debugf("Entering Remove: name: %s, options %v", r.Name, r.Options)
	d.Lock()
	defer d.Unlock()
	v, ok := d.volumes[r.Name]

	if !ok {
		return volume.Response{Err: fmt.Sprintf("volume %s not found", r.Name)}
	}
	if v.connections == 0 {
		delete(d.volumes, r.Name)
		return volume.Response{}
	}
	return volume.Response{Err: fmt.Sprintf("volume %s is currently used by a container", r.Name)}
}

//List volumes handled by thos driver
func (d GVfsDriver) List(r volume.Request) volume.Response {
	log.Debugf("Entering List: name: %s, options %v", r.Name, r.Options)

	d.Lock()
	defer d.Unlock()

	var vols []*volume.Volume
	for name, v := range d.volumes {
		vols = append(vols, &volume.Volume{Name: name, Mountpoint: v.mountpoint})
		log.Debugf("Volume found: %s", v)
	}
	return volume.Response{Volumes: vols}
}

//Get get info on the requested volume
func (d GVfsDriver) Get(r volume.Request) volume.Response {
	log.Debugf("Entering Get: name: %s", r.Name)
	d.Lock()
	defer d.Unlock()

	v, ok := d.volumes[r.Name]
	if !ok {
		return volume.Response{Err: fmt.Sprintf("volume %s not found", r.Name)}
	}

	log.Debugf("Volume found: %s", v)
	return volume.Response{Volume: &volume.Volume{Name: r.Name, Mountpoint: v.mountpoint}}
}

//Path get path of the requested volume
func (d GVfsDriver) Path(r volume.Request) volume.Response {
	log.Debugf("Entering Path: name: %s, options %v", r.Name)

	d.RLock()
	defer d.RUnlock()
	v, ok := d.volumes[r.Name]
	if !ok {
		return volume.Response{Err: fmt.Sprintf("volume %s not found", r.Name)}
	}
	log.Debugf("Volume found: %s", v)
	return volume.Response{Mountpoint: v.mountpoint}
}

//Mount mount the requested volume
func (d GVfsDriver) Mount(r volume.MountRequest) volume.Response {
	log.Debugf("Entering Mount: %v", r)
	d.Lock()
	defer d.Unlock()

	v, ok := d.volumes[r.Name]
	if !ok {
		return volume.Response{Err: fmt.Sprintf("volume %s not found", r.Name)}
	}

	if v.connections > 0 {
		v.connections++
		return volume.Response{Mountpoint: v.mountpoint}
	}

	cmd := fmt.Sprintf("gvfs-mount %s", v.url)
	if v.password != "" {
		p := setEnv(cmd, d.env)
		inStd, err := p.StdinPipe()
		if err != nil { //Get a input buffer
			return volume.Response{Err: err.Error()}
		}
		var outStd bytes.Buffer
		p.Stdout = &outStd
		var errStd bytes.Buffer
		p.Stderr = &errStd

		if err := p.Start(); err != nil {
			return volume.Response{Err: err.Error()}
		}
		inStd.Write([]byte(v.password + "\n")) //Send password to process + Send return line

		// wait or timeout
		donec := make(chan error, 1)
		go func() {
			donec <- p.Wait() //Process finish
		}()
		select {
		case <-time.After(MountTimeout * time.Second):
			sOut := outStd.String()
			sErr := errStd.String()
			p.Process.Kill()
			log.Debugf("out : %s", sOut)
			log.Debugf("outErr : %s", sErr)
			return volume.Response{Err: fmt.Sprintf("The command %s timeout", cmd)}
		case <-donec:
			sOut := outStd.String()
			sErr := errStd.String()
			log.Debugf("Password send and command %s return", cmd)
			log.Debugf("out : %s", sOut)
			log.Debugf("outErr : %s", sErr)
			// handle erros like : "Error mounting location: Location is already mounted" or Error mounting location: Could not connect to 10.8.0.7: No route to host
			if strings.Contains(sErr, "Error mounting location") {
				return volume.Response{Err: fmt.Sprintf("Error mounting location : %s", sErr)}
			}
			v.connections++
			break
		}
	} else {
		if err := d.runCmd(cmd); err != nil {
			return volume.Response{Err: err.Error()}
		}
	}

	return volume.Response{Mountpoint: v.mountpoint}
}

//Unmount unmount the requested volume
//TODO Monitor for unmount to remount ?
func (d GVfsDriver) Unmount(r volume.UnmountRequest) volume.Response {
	//Execute gvfs-mount -u $params
	log.Debugf("Entering Unmount: %v", r)

	d.Lock()
	defer d.Unlock()
	v, ok := d.volumes[r.Name]
	if !ok {
		return volume.Response{Err: fmt.Sprintf("volume %s not found", r.Name)}
	}
	if v.connections <= 1 {
		cmd := fmt.Sprintf("gvfs-mount -u %s", v.url)
		if err := d.runCmd(cmd); err != nil {
			return volume.Response{Err: err.Error()}
		}
		v.connections = 0
	} else {
		v.connections--
	}

	return volume.Response{}
}

//Capabilities Send capabilities of the local driver
func (d GVfsDriver) Capabilities(r volume.Request) volume.Response {
	log.Debugf("Entering Capabilities: %v", r)
	return volume.Response{
		Capabilities: volume.Capability{
			Scope: "local",
		},
	}
}
<|endoftext|>


<commit_before>package main

import (
	"bytes"
	"database/sql"
	"encoding/csv"
	"encoding/json"
	"fmt"
	"io"
	"io/ioutil"
	"log"
	"os"
	"strconv"
	"sync"

	"github.com/AdRoll/goamz/aws"
	"github.com/AdRoll/goamz/s3"
	"github.com/lib/pq"
)

type dbInfo struct {
	Host     string
	Database string
	Username string
	Password string
}

type userInfo struct {
	UserID int
	Email  string
}

func getDBSettings() *dbInfo {
	file, err := ioutil.ReadFile("./settings.json")
	if err != nil {
		log.Println("Error:", err)
		return nil
	}
	db := dbInfo{}
	err2 := json.Unmarshal(file, &db)
	if err2 != nil {
		log.Println("Error:", err2)
		return nil
	}
	return &db
}

type createdata func([]int) error

func createUserData() {
	ids := getUserIDs()
	return
	cdata := []createdata{createLang, createDevices, createGroups, createSubscriptions}
	for _, v := range cdata {
		if dataErr := v(ids); dataErr != nil {
			panic(fmt.Sprintf("%v - %v", v, dataErr))
		}
	}
}

func createLang(ids []int) error {
	return fmt.Errorf("Not implemented")
}

func createDevices(ids []int) error {
	return fmt.Errorf("Not implemented")

}

func createGroups(ids []int) error {
	return fmt.Errorf("Not implemented")

}

func createSubscriptions(ids []int) error {
	return fmt.Errorf("Not implemented")

}

func main() {
	//createUserData()
	//return
	for k, v := range os.Args {
		log.Println(k, v)
	}
	log.Println("-----")
	if len(os.Args) > 0 {
		data, err := downloadFromBucket("csv-stream-demo", "data.csv")
		if err != nil {
			log.Println("Error:", err)
			return
		}

		log.Println("DB Start")
		errDB := copyDataToDB(data)
		if errDB != nil {
			log.Println("Error:", errDB)
		}
		log.Println("DB END")
		return
	}
	log.Println("Error: os.Args was 1 length.")
}

func getUserIDs() []int {
	info := getDBSettings()
	db, errCon := sql.Open("postgres", fmt.Sprintf("host=%v user=%v password=%v dbname=%v sslmode=require", info.Host, info.Username, info.Password, info.Database))
	defer db.Close()
	if errCon != nil {
		log.Fatal(errCon)
	}
	log.Println("Connected...")
	return nil
}

func copyDataToDB(data []byte) error {
	info := getDBSettings()
	db, errCon := sql.Open("postgres", fmt.Sprintf("host=%v user=%v password=%v dbname=%v sslmode=require", info.Host, info.Username, info.Password, info.Database))
	defer db.Close()
	if errCon != nil {
		log.Fatal(errCon)
	}
	txn, errT := db.Begin()
	if errT != nil {
		log.Println(errT)
		return errT
	}
	stmt, errPrep := txn.Prepare(pq.CopyIn("userdata", "userid", "jobid"))
	if errPrep != nil {
		log.Fatal(errPrep)
	}
	r := bytes.NewReader(data)
	reader := csv.NewReader(r)
	reader.Comma = ','
	lineCount := 0
	log.Println("Start For...")
	var wg sync.WaitGroup
	for {

		record, err := reader.Read()
		if err == io.EOF {
			break
		} else if err != nil {
			fmt.Println("Error:", err)
			return err
		}

		email := record[0]
		_ = email
		jobid := "abc123"
		userID, _ := strconv.Atoi(record[1])
		_ = userID
		wg.Add(1)
		go func(id int, e string) {
			defer wg.Done()
			_, errA := stmt.Exec(id, e)
			if errA != nil {
				log.Fatal(errA)
			}
		}(userID, jobid)
		lineCount++
		if lineCount == 1000000 {
			break
		}
	}
	wg.Wait()
	log.Println("End For")
	log.Println("Start Exec")
	_, errEX := stmt.Exec()
	if errEX != nil {
		log.Fatal(errEX)
	}
	log.Println("End Exec")

	errClose := stmt.Close()
	if errClose != nil {
		log.Fatal(errClose)
	}
	log.Println("Start Commit")
	errCommit := txn.Commit()
	if errCommit != nil {
		log.Fatal(errCommit)
	}
	log.Println("End Commit")
	return nil
}

func getSettings() (string, string, error) {
	file, err := ioutil.ReadFile("./settings.json")
	if err != nil {
		return "", "", nil
	}
	settingsMap := make(map[string]string)
	json.Unmarshal(file, &settingsMap)
	return settingsMap["Access"], settingsMap["Secret"], nil
}

func downloadFromBucket(b string, f string) ([]byte, error) {
	p, s, setErr := getSettings()
	if setErr != nil {
		return nil, setErr
	}
	auth := aws.Auth{AccessKey: p, SecretKey: s}

	S3 := s3.New(auth, aws.USEast)
	bucket := S3.Bucket(b)
	log.Println("Starting Get...")
	data, err := bucket.Get(f)
	if err != nil {
		return nil, err
	}
	log.Println("Completed Get!", len(data))
	return data, nil
}
<commit_msg>update userID setting<commit_after>package main

import (
	"bytes"
	"database/sql"
	"encoding/csv"
	"encoding/json"
	"fmt"
	"io"
	"io/ioutil"
	"log"
	"os"
	"strconv"
	"sync"

	"github.com/AdRoll/goamz/aws"
	"github.com/AdRoll/goamz/s3"
	"github.com/lib/pq"
)

type dbInfo struct {
	Host     string
	Database string
	Username string
	Password string
}

type userInfo struct {
	UserID int
	Email  string
}

func getDBSettings() *dbInfo {
	file, err := ioutil.ReadFile("./settings.json")
	if err != nil {
		log.Println("Error:", err)
		return nil
	}
	db := dbInfo{}
	err2 := json.Unmarshal(file, &db)
	if err2 != nil {
		log.Println("Error:", err2)
		return nil
	}
	return &db
}

type createdata func([]int) error

func createUserData() {
	ids := getUserIDs()
	return
	cdata := []createdata{createLang, createDevices, createGroups, createSubscriptions}
	for _, v := range cdata {
		if dataErr := v(ids); dataErr != nil {
			panic(fmt.Sprintf("%v - %v", v, dataErr))
		}
	}
}

func createLang(ids []int) error {
	return fmt.Errorf("Not implemented")
}

func createDevices(ids []int) error {
	return fmt.Errorf("Not implemented")

}

func createGroups(ids []int) error {
	return fmt.Errorf("Not implemented")

}

func createSubscriptions(ids []int) error {
	return fmt.Errorf("Not implemented")

}

func main() {
	//createUserData()
	//return
	for k, v := range os.Args {
		log.Println(k, v)
	}
	log.Println("-----")
	if len(os.Args) > 0 {
		data, err := downloadFromBucket("csv-stream-demo", "data.csv")
		if err != nil {
			log.Println("Error:", err)
			return
		}

		log.Println("DB Start")
		errDB := copyDataToDB(data)
		if errDB != nil {
			log.Println("Error:", errDB)
		}
		log.Println("DB END")
		return
	}
	log.Println("Error: os.Args was 1 length.")
}

func getUserIDs() []int {
	info := getDBSettings()
	db, errCon := sql.Open("postgres", fmt.Sprintf("host=%v user=%v password=%v dbname=%v sslmode=require", info.Host, info.Username, info.Password, info.Database))
	defer db.Close()
	if errCon != nil {
		log.Fatal(errCon)
	}
	log.Println("Connected...")
	return nil
}

func copyDataToDB(data []byte) error {
	info := getDBSettings()
	db, errCon := sql.Open("postgres", fmt.Sprintf("host=%v user=%v password=%v dbname=%v sslmode=require", info.Host, info.Username, info.Password, info.Database))
	defer db.Close()
	if errCon != nil {
		log.Fatal(errCon)
	}
	txn, errT := db.Begin()
	if errT != nil {
		log.Println(errT)
		return errT
	}
	stmt, errPrep := txn.Prepare(pq.CopyIn("userdata", "userid", "jobid"))
	if errPrep != nil {
		log.Fatal(errPrep)
	}
	r := bytes.NewReader(data)
	reader := csv.NewReader(r)
	reader.Comma = ','
	lineCount := 0
	log.Println("Start For...")
	var wg sync.WaitGroup
	for {

		record, err := reader.Read()
		if err == io.EOF {
			break
		} else if err != nil {
			fmt.Println("Error:", err)
			return err
		}

		email := record[0]
		_ = email
		jobid := "abc123"
		userID, _ := strconv.Atoi(record[1])
		_ = userID
		wg.Add(1)
		go func(id int, e string) {
			defer wg.Done()
			_, errA := stmt.Exec(id, e)
			if errA != nil {
				log.Fatal(errA)
			}
		}(lineCount+1, jobid)
		lineCount++
		if lineCount == 1000000 {
			break
		}
	}
	wg.Wait()
	log.Println("End For")
	log.Println("Start Exec")
	_, errEX := stmt.Exec()
	if errEX != nil {
		log.Fatal(errEX)
	}
	log.Println("End Exec")

	errClose := stmt.Close()
	if errClose != nil {
		log.Fatal(errClose)
	}
	log.Println("Start Commit")
	errCommit := txn.Commit()
	if errCommit != nil {
		log.Fatal(errCommit)
	}
	log.Println("End Commit")
	return nil
}

func getSettings() (string, string, error) {
	file, err := ioutil.ReadFile("./settings.json")
	if err != nil {
		return "", "", nil
	}
	settingsMap := make(map[string]string)
	json.Unmarshal(file, &settingsMap)
	return settingsMap["Access"], settingsMap["Secret"], nil
}

func downloadFromBucket(b string, f string) ([]byte, error) {
	p, s, setErr := getSettings()
	if setErr != nil {
		return nil, setErr
	}
	auth := aws.Auth{AccessKey: p, SecretKey: s}

	S3 := s3.New(auth, aws.USEast)
	bucket := S3.Bucket(b)
	log.Println("Starting Get...")
	data, err := bucket.Get(f)
	if err != nil {
		return nil, err
	}
	log.Println("Completed Get!", len(data))
	return data, nil
}
<|endoftext|>


<commit_before>//  Copyright (c) 2014 Couchbase, Inc.
//  Licensed under the Apache License, Version 2.0 (the "License");
//  you may not use this file except in compliance with the
//  License. You may obtain a copy of the License at
//    http://www.apache.org/licenses/LICENSE-2.0
//  Unless required by applicable law or agreed to in writing,
//  software distributed under the License is distributed on an "AS
//  IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either
//  express or implied. See the License for the specific language
//  governing permissions and limitations under the License.

package main

import (
	"fmt"
	"sync"

	"github.com/couchbase/gomemcached"
	log "github.com/couchbaselabs/clog"

	"github.com/steveyen/cbdatasource"
)

// Implements both Feed and cbdatasource.Receiver interfaces.
type DCPFeed struct {
	name       string
	url        string
	poolName   string
	bucketName string
	bucketUUID string
	pf         StreamPartitionFunc
	streams    map[string]Stream
	bds        cbdatasource.BucketDataSource

	m       sync.Mutex
	closed  bool
	lastErr error

	seqs map[uint16]uint64 // To track max seq #'s we received per vbucketId.
	meta map[uint16][]byte // To track metadata blob's per vbucketId.

	numError         uint64
	numUpdate        uint64
	numDelete        uint64
	numSnapshotStart uint64
	numSetMetaData   uint64
	numGetMetaData   uint64
	numRollback      uint64
}

func NewDCPFeed(name, url, poolName, bucketName, bucketUUID string,
	pf StreamPartitionFunc, streams map[string]Stream) (*DCPFeed, error) {
	vbucketIds, err := ParsePartitionsToVBucketIds(streams)
	if err != nil {
		return nil, err
	}
	if len(vbucketIds) <= 0 {
		vbucketIds = nil
	}

	var authFunc cbdatasource.AuthFunc
	var options *cbdatasource.BucketDataSourceOptions

	feed := &DCPFeed{
		name:       name,
		url:        url,
		poolName:   poolName,
		bucketName: bucketName,
		bucketUUID: bucketUUID,
		pf:         pf,
		streams:    streams,
	}

	feed.bds, err = cbdatasource.NewBucketDataSource([]string{url},
		poolName, bucketName, bucketUUID,
		vbucketIds, authFunc, feed, options)
	if err != nil {
		return nil, err
	}

	return feed, nil
}

func (t *DCPFeed) Name() string {
	return t.name
}

func (t *DCPFeed) Start() error {
	log.Printf("DCPFeed.Start, name: %s", t.Name())
	return t.bds.Start()
}

func (t *DCPFeed) Close() error {
	t.m.Lock()
	if t.closed {
		t.m.Unlock()
		return fmt.Errorf("already closed")
	}
	t.closed = true
	t.m.Unlock()

	log.Printf("DCPFeed.Close, name: %s", t.Name())
	return t.bds.Close()
}

func (t *DCPFeed) Streams() map[string]Stream {
	return t.streams
}

// --------------------------------------------------------

func (r *DCPFeed) OnError(err error) {
	log.Printf("DCPFeed.OnError: %s: %v\n", r.name, err)

	r.m.Lock()
	defer r.m.Unlock()
	r.numError += 1

	r.lastErr = err
}

func (r *DCPFeed) DataUpdate(vbucketId uint16, key []byte, seq uint64,
	req *gomemcached.MCRequest) error {
	// log.Printf("DCPFeed.DataUpdate: %s: vbucketId: %d, key: %s, seq: %d, req: %v\n",
	// r.name, vbucketId, key, seq, req)

	partition, stream, err :=
		VBucketIdToPartitionStream(r.pf, r.streams, vbucketId, key)
	if err != nil {
		return err
	}

	r.m.Lock()
	r.numUpdate += 1
	r.updateSeqUnlocked(vbucketId, seq)
	r.m.Unlock()

	stream <- &StreamRequest{
		Op:        STREAM_OP_UPDATE,
		Partition: partition,
		SeqNo:     seq,
		Key:       req.Key,
		Val:       req.Body,
	}

	return nil
}

func (r *DCPFeed) DataDelete(vbucketId uint16, key []byte, seq uint64,
	req *gomemcached.MCRequest) error {
	// log.Printf("DCPFeed.DataDelete: %s: vbucketId: %d, key: %s, seq: %d, req: %#v",
	// r.name, vbucketId, key, seq, req)

	partition, stream, err :=
		VBucketIdToPartitionStream(r.pf, r.streams, vbucketId, key)
	if err != nil {
		return err
	}

	r.m.Lock()
	r.numDelete += 1
	r.updateSeqUnlocked(vbucketId, seq)
	r.m.Unlock()

	stream <- &StreamRequest{
		Op:        STREAM_OP_DELETE,
		Partition: partition,
		SeqNo:     seq,
		Key:       req.Key,
	}

	return nil
}

func (r *DCPFeed) updateSeqUnlocked(vbucketId uint16, seq uint64) {
	if r.seqs == nil {
		r.seqs = make(map[uint16]uint64)
	}
	if r.seqs[vbucketId] < seq {
		r.seqs[vbucketId] = seq // Remember the max seq for GetMetaData().
	}
}

func (r *DCPFeed) SnapshotStart(vbucketId uint16,
	snapStart, snapEnd uint64, snapType uint32) error {
	log.Printf("DCPFeed.SnapshotStart: %s: vbucketId: %d,"+
		" snapStart: %d, snapEnd: %d, snapType: %d",
		r.name, vbucketId, snapStart, snapEnd, snapType)

	r.m.Lock()
	defer r.m.Unlock()
	r.numSnapshotStart += 1

	return nil
}

func (r *DCPFeed) SetMetaData(vbucketId uint16, value []byte) error {
	log.Printf("DCPFeed.SetMetaData: %s: vbucketId: %d,"+
		" value: %s", r.name, vbucketId, value)

	r.m.Lock()
	defer r.m.Unlock()
	r.numSetMetaData += 1

	if r.meta == nil {
		r.meta = make(map[uint16][]byte)
	}
	r.meta[vbucketId] = value

	return nil
}

func (r *DCPFeed) GetMetaData(vbucketId uint16) (value []byte, lastSeq uint64, err error) {
	log.Printf("DCPFeed.GetMetaData: %s: vbucketId: %d", r.name, vbucketId)

	r.m.Lock()
	defer r.m.Unlock()
	r.numGetMetaData += 1

	rv := []byte(nil)
	if r.meta != nil {
		rv = r.meta[vbucketId]
	}

	if r.seqs != nil {
		lastSeq = r.seqs[vbucketId]
	}

	return rv, lastSeq, nil
}

func (r *DCPFeed) Rollback(vbucketId uint16, rollbackSeq uint64) error {
	log.Printf("DCPFeed.Rollback: %s: vbucketId: %d,"+
		" rollbackSeq: %d", r.name, vbucketId, rollbackSeq)

	r.m.Lock()
	defer r.m.Unlock()
	r.numRollback += 1

	return fmt.Errorf("bad-rollback")
}
<commit_msg>hooked up Get/SetMetaData for dcp feed<commit_after>//  Copyright (c) 2014 Couchbase, Inc.
//  Licensed under the Apache License, Version 2.0 (the "License");
//  you may not use this file except in compliance with the
//  License. You may obtain a copy of the License at
//    http://www.apache.org/licenses/LICENSE-2.0
//  Unless required by applicable law or agreed to in writing,
//  software distributed under the License is distributed on an "AS
//  IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either
//  express or implied. See the License for the specific language
//  governing permissions and limitations under the License.

package main

import (
	"fmt"
	"sync"

	"github.com/couchbase/gomemcached"
	log "github.com/couchbaselabs/clog"

	"github.com/steveyen/cbdatasource"
)

// Implements both Feed and cbdatasource.Receiver interfaces.
type DCPFeed struct {
	name       string
	url        string
	poolName   string
	bucketName string
	bucketUUID string
	pf         StreamPartitionFunc
	streams    map[string]Stream
	bds        cbdatasource.BucketDataSource

	m       sync.Mutex
	closed  bool
	lastErr error

	seqs map[uint16]uint64 // To track max seq #'s we received per vbucketId.

	numError         uint64
	numUpdate        uint64
	numDelete        uint64
	numSnapshotStart uint64
	numSetMetaData   uint64
	numGetMetaData   uint64
	numRollback      uint64
}

func NewDCPFeed(name, url, poolName, bucketName, bucketUUID string,
	pf StreamPartitionFunc, streams map[string]Stream) (*DCPFeed, error) {
	vbucketIds, err := ParsePartitionsToVBucketIds(streams)
	if err != nil {
		return nil, err
	}
	if len(vbucketIds) <= 0 {
		vbucketIds = nil
	}

	var authFunc cbdatasource.AuthFunc
	var options *cbdatasource.BucketDataSourceOptions

	feed := &DCPFeed{
		name:       name,
		url:        url,
		poolName:   poolName,
		bucketName: bucketName,
		bucketUUID: bucketUUID,
		pf:         pf,
		streams:    streams,
	}

	feed.bds, err = cbdatasource.NewBucketDataSource([]string{url},
		poolName, bucketName, bucketUUID,
		vbucketIds, authFunc, feed, options)
	if err != nil {
		return nil, err
	}

	return feed, nil
}

func (t *DCPFeed) Name() string {
	return t.name
}

func (t *DCPFeed) Start() error {
	log.Printf("DCPFeed.Start, name: %s", t.Name())
	return t.bds.Start()
}

func (t *DCPFeed) Close() error {
	t.m.Lock()
	if t.closed {
		t.m.Unlock()
		return fmt.Errorf("already closed")
	}
	t.closed = true
	t.m.Unlock()

	log.Printf("DCPFeed.Close, name: %s", t.Name())
	return t.bds.Close()
}

func (t *DCPFeed) Streams() map[string]Stream {
	return t.streams
}

// --------------------------------------------------------

func (r *DCPFeed) OnError(err error) {
	log.Printf("DCPFeed.OnError: %s: %v\n", r.name, err)

	r.m.Lock()
	defer r.m.Unlock()
	r.numError += 1

	r.lastErr = err
}

func (r *DCPFeed) DataUpdate(vbucketId uint16, key []byte, seq uint64,
	req *gomemcached.MCRequest) error {
	// log.Printf("DCPFeed.DataUpdate: %s: vbucketId: %d, key: %s, seq: %d, req: %v\n",
	// r.name, vbucketId, key, seq, req)

	partition, stream, err :=
		VBucketIdToPartitionStream(r.pf, r.streams, vbucketId, key)
	if err != nil {
		return err
	}

	r.m.Lock()
	r.numUpdate += 1
	r.updateSeqUnlocked(vbucketId, seq)
	r.m.Unlock()

	stream <- &StreamRequest{
		Op:        STREAM_OP_UPDATE,
		Partition: partition,
		SeqNo:     seq,
		Key:       req.Key,
		Val:       req.Body,
	}

	return nil
}

func (r *DCPFeed) DataDelete(vbucketId uint16, key []byte, seq uint64,
	req *gomemcached.MCRequest) error {
	// log.Printf("DCPFeed.DataDelete: %s: vbucketId: %d, key: %s, seq: %d, req: %#v",
	// r.name, vbucketId, key, seq, req)

	partition, stream, err :=
		VBucketIdToPartitionStream(r.pf, r.streams, vbucketId, key)
	if err != nil {
		return err
	}

	r.m.Lock()
	r.numDelete += 1
	r.updateSeqUnlocked(vbucketId, seq)
	r.m.Unlock()

	stream <- &StreamRequest{
		Op:        STREAM_OP_DELETE,
		Partition: partition,
		SeqNo:     seq,
		Key:       req.Key,
	}

	return nil
}

func (r *DCPFeed) updateSeqUnlocked(vbucketId uint16, seq uint64) {
	if r.seqs == nil {
		r.seqs = make(map[uint16]uint64)
	}
	if r.seqs[vbucketId] < seq {
		r.seqs[vbucketId] = seq // Remember the max seq for GetMetaData().
	}
}

func (r *DCPFeed) SnapshotStart(vbucketId uint16,
	snapStart, snapEnd uint64, snapType uint32) error {
	log.Printf("DCPFeed.SnapshotStart: %s: vbucketId: %d,"+
		" snapStart: %d, snapEnd: %d, snapType: %d",
		r.name, vbucketId, snapStart, snapEnd, snapType)

	r.m.Lock()
	defer r.m.Unlock()
	r.numSnapshotStart += 1

	return nil
}

func (r *DCPFeed) SetMetaData(vbucketId uint16, value []byte) error {
	log.Printf("DCPFeed.SetMetaData: %s: vbucketId: %d,"+
		" value: %s", r.name, vbucketId, value)

	partition, stream, err :=
		VBucketIdToPartitionStream(r.pf, r.streams, vbucketId, nil)
	if err != nil {
		return err
	}

	r.m.Lock()
	r.numSetMetaData += 1
	r.m.Unlock()

	stream <- &StreamRequest{
		Op:        STREAM_OP_SET_META,
		Partition: partition,
		Key:       []byte(partition),
		Val:       value,
	}

	return nil
}

func (r *DCPFeed) GetMetaData(vbucketId uint16) (value []byte, lastSeq uint64, err error) {
	log.Printf("DCPFeed.GetMetaData: %s: vbucketId: %d", r.name, vbucketId)

	partition, stream, err :=
		VBucketIdToPartitionStream(r.pf, r.streams, vbucketId, nil)
	if err != nil {
		return nil, 0, err
	}

	r.m.Lock()
	r.numGetMetaData += 1
	if r.seqs != nil {
		// TODO: Need to get this from stream instead of here.
		lastSeq = r.seqs[vbucketId]
	}
	r.m.Unlock()

	rvCh := make(chan []byte)
	stream <- &StreamRequest{
		Op:        STREAM_OP_GET_META,
		Partition: partition,
		Key:       []byte(partition),
		Misc:      rvCh,
	}
	rv, exists := <-rvCh
	if !exists {
		return nil, 0, nil
	}

	log.Printf("DCPFeed.GetMetaData: %s: vbucketId: %d, rv: %s", r.name, vbucketId, rv)

	return rv, lastSeq, nil
}

func (r *DCPFeed) Rollback(vbucketId uint16, rollbackSeq uint64) error {
	log.Printf("DCPFeed.Rollback: %s: vbucketId: %d,"+
		" rollbackSeq: %d", r.name, vbucketId, rollbackSeq)

	r.m.Lock()
	defer r.m.Unlock()
	r.numRollback += 1

	return fmt.Errorf("bad-rollback")
}
<|endoftext|>


<commit_before>package main

import (
	"flag"
	"fmt"
	"github.com/APTrust/exchange/context"
	"github.com/APTrust/exchange/dpn/network"
	"github.com/APTrust/exchange/models"
	"github.com/APTrust/exchange/util"
	"io/ioutil"
	"net/url"
	"os"
	"path/filepath"
	"strings"
)

func main() {
	pathToConfigFile := parseCommandLine()
	config, err := models.LoadConfigFile(pathToConfigFile)
	if err != nil {
		fmt.Fprintf(os.Stderr, err.Error())
		os.Exit(1)
	}
	_context := context.NewContext(config)
	dpnClient, err := network.NewDPNRestClient(
		_context.Config.DPN.RestClient.LocalServiceURL,
		_context.Config.DPN.DPNAPIVersion,
		_context.Config.DPN.RestClient.LocalAuthToken,
		_context.Config.DPN.LocalNode,
		_context.Config.DPN)
	if err != nil {
		fmt.Fprintf(os.Stderr, err.Error())
		os.Exit(1)
	}
	deleteReplicatedBags(_context, dpnClient)
}

// deleteReplicated bags deletes DPN bags (tar files) after they've
// been replicated to the required number of nodes. (That's 2 nodes
// as of February, 2017.) When we ingest bags and ask that other
// nodes replicate them, the bags go into a subdirectory of the DPN
// staging directory. That subdirectory has the domain name of the
// institution that owns the bag. For example, <dpn_staging>/ncsu.edu,
// <dpn_staging>/virginia.edu, etc.
func deleteReplicatedBags(_context *context.Context, dpnClient *network.DPNRestClient) {
	files, err := ioutil.ReadDir(_context.Config.DPN.StagingDirectory)
	if err != nil {
		_context.MessageLog.Error(err.Error())
		return
	}
	for _, f := range files {
		if f.IsDir() {
			pathToDir := filepath.Join(_context.Config.DPN.StagingDirectory, f.Name())
			cleanDirectory(_context, dpnClient, pathToDir)
		}
	}
}

func cleanDirectory(_context *context.Context, dpnClient *network.DPNRestClient, directory string) {
	_context.MessageLog.Info("Deleting replicated bags in %s", directory)
	files, err := ioutil.ReadDir(directory)
	if err != nil {
		_context.MessageLog.Error(err.Error())
		return
	}
	for _, finfo := range files {
		bagUUID := strings.Replace(finfo.Name(), ".tar", "", 1)
		if !util.LooksLikeUUID(bagUUID) {
			continue // Don't delete it if it's not a DPN tar file
		}
		params := url.Values{}
		params.Set("bag", bagUUID)
		params.Set("stored", "true")
		params.Set("from_node", _context.Config.DPN.LocalNode)
		resp := dpnClient.ReplicationTransferList(params)
		if resp.Error != nil {
			_context.MessageLog.Error("Error getting replication info for bag '%s': %v",
				bagUUID, resp.Error.Error())
			continue
		}
		tarfile := filepath.Join(_context.Config.DPN.StagingDirectory, finfo.Name())
		successfulReplications := resp.ReplicationTransfers()
		if len(successfulReplications) >= _context.Config.DPN.ReplicateToNumNodes {
			_context.MessageLog.Info("Deleting %s: %d successful replications",
				tarfile, len(successfulReplications))
			removeFile(_context, tarfile)
		} else {
			_context.MessageLog.Info("Leaving %s: only %d successful replications so far",
				tarfile, len(successfulReplications))
		}
	}
}

func removeFile(_context *context.Context, pathToFile string) {
	err := os.Remove(pathToFile)
	if err != nil {
		_context.MessageLog.Info("Error deleting %s: %v", pathToFile, err)
	} else {
		_context.MessageLog.Info("Deleted %s", pathToFile)
	}
}

func parseCommandLine() (configFile string) {
	flag.StringVar(&configFile, "config", "", "Path to APTrust config file")
	flag.Parse()
	if configFile == "" {
		printUsage()
		os.Exit(1)
	}
	return configFile
}

// Tell the user about the program.
func printUsage() {
	message := `
dpn_cleanup deletes tar files from the DPN staging directory and the
symlinks to those tar files from the /home/dpn.<user>/outbound directories
once the items have been replicated to the minimum number of required
nodes. The number of required nodes is defined in the config file,
under DPN.ReplicateToNumNodes.

Usage: dpn_cleanup -config=<path to config file>

Param -config is required. It can be an absolute path, or a path in the format
config/env.json, where env is dev, test, demo, integration or production.

`
	fmt.Println(message)
}
<commit_msg>Log replication URLs for debugging<commit_after>package main

import (
	"flag"
	"fmt"
	"github.com/APTrust/exchange/context"
	"github.com/APTrust/exchange/dpn/network"
	"github.com/APTrust/exchange/models"
	"github.com/APTrust/exchange/util"
	"io/ioutil"
	"net/url"
	"os"
	"path/filepath"
	"strings"
)

func main() {
	pathToConfigFile := parseCommandLine()
	config, err := models.LoadConfigFile(pathToConfigFile)
	if err != nil {
		fmt.Fprintf(os.Stderr, err.Error())
		os.Exit(1)
	}
	_context := context.NewContext(config)
	dpnClient, err := network.NewDPNRestClient(
		_context.Config.DPN.RestClient.LocalServiceURL,
		_context.Config.DPN.DPNAPIVersion,
		_context.Config.DPN.RestClient.LocalAuthToken,
		_context.Config.DPN.LocalNode,
		_context.Config.DPN)
	if err != nil {
		fmt.Fprintf(os.Stderr, err.Error())
		os.Exit(1)
	}
	deleteReplicatedBags(_context, dpnClient)
}

// deleteReplicated bags deletes DPN bags (tar files) after they've
// been replicated to the required number of nodes. (That's 2 nodes
// as of February, 2017.) When we ingest bags and ask that other
// nodes replicate them, the bags go into a subdirectory of the DPN
// staging directory. That subdirectory has the domain name of the
// institution that owns the bag. For example, <dpn_staging>/ncsu.edu,
// <dpn_staging>/virginia.edu, etc.
func deleteReplicatedBags(_context *context.Context, dpnClient *network.DPNRestClient) {
	files, err := ioutil.ReadDir(_context.Config.DPN.StagingDirectory)
	if err != nil {
		_context.MessageLog.Error(err.Error())
		return
	}
	for _, f := range files {
		if f.IsDir() {
			pathToDir := filepath.Join(_context.Config.DPN.StagingDirectory, f.Name())
			cleanDirectory(_context, dpnClient, pathToDir)
		}
	}
}

func cleanDirectory(_context *context.Context, dpnClient *network.DPNRestClient, directory string) {
	_context.MessageLog.Info("Deleting replicated bags in %s", directory)
	files, err := ioutil.ReadDir(directory)
	if err != nil {
		_context.MessageLog.Error(err.Error())
		return
	}
	for _, finfo := range files {
		bagUUID := strings.Replace(finfo.Name(), ".tar", "", 1)
		if !util.LooksLikeUUID(bagUUID) {
			continue // Don't delete it if it's not a DPN tar file
		}
		params := url.Values{}
		params.Set("bag", bagUUID)
		params.Set("stored", "true")
		params.Set("from_node", _context.Config.DPN.LocalNode)
		resp := dpnClient.ReplicationTransferList(params)
		_context.MessageLog.Info(resp.Request.URL.Opaque)
		if resp.Error != nil {
			_context.MessageLog.Error("Error getting replication info for bag '%s': %v",
				bagUUID, resp.Error.Error())
			continue
		}
		tarfile := filepath.Join(_context.Config.DPN.StagingDirectory, finfo.Name())
		successfulReplications := resp.ReplicationTransfers()
		if len(successfulReplications) >= _context.Config.DPN.ReplicateToNumNodes {
			_context.MessageLog.Info("Deleting %s: %d successful replications",
				tarfile, len(successfulReplications))
			removeFile(_context, tarfile)
		} else {
			_context.MessageLog.Info("Leaving %s: only %d successful replications so far",
				tarfile, len(successfulReplications))
		}
	}
}

func removeFile(_context *context.Context, pathToFile string) {
	err := os.Remove(pathToFile)
	if err != nil {
		_context.MessageLog.Info("Error deleting %s: %v", pathToFile, err)
	} else {
		_context.MessageLog.Info("Deleted %s", pathToFile)
	}
}

func parseCommandLine() (configFile string) {
	flag.StringVar(&configFile, "config", "", "Path to APTrust config file")
	flag.Parse()
	if configFile == "" {
		printUsage()
		os.Exit(1)
	}
	return configFile
}

// Tell the user about the program.
func printUsage() {
	message := `
dpn_cleanup deletes tar files from the DPN staging directory and the
symlinks to those tar files from the /home/dpn.<user>/outbound directories
once the items have been replicated to the minimum number of required
nodes. The number of required nodes is defined in the config file,
under DPN.ReplicateToNumNodes.

Usage: dpn_cleanup -config=<path to config file>

Param -config is required. It can be an absolute path, or a path in the format
config/env.json, where env is dev, test, demo, integration or production.

`
	fmt.Println(message)
}
<|endoftext|>


<commit_before>// +build windows darwin,cgo

package session

import (
	"context"
	"fmt"
	"path/filepath"
	"strings"
	"time"

	"github.com/pkg/errors"

	"github.com/rjeczalik/notify"
)

const (
	scanPollInterval      = 30 * time.Second
	watchEventsBufferSize = 10
	coalescingWindow      = 250 * time.Millisecond
)

func watch(context context.Context, root string, events chan struct{}) error {
	// Create a watch notifications channel.
	notifications := make(chan notify.EventInfo, watchEventsBufferSize)

	// Create a timer that we can use to coalesce events. It will be created
	// running, so make sure to stop it and consume its first event, if any.
	timer := time.NewTimer(coalescingWindow)
	if !timer.Stop() {
		<-timer.C
	}

	// Compute the parent directory of root. We watch this because (a) we may
	// have a file root and many systems don't support watching a file directly
	// (only a directory), (b) the root may not exist yet, and (c) deletion of
	// the watch root isn't seen on all platforms. Of course, if the parent of
	// root is deleted, we won't see that either here, but we'll eventually see
	// it in polling.
	parent := filepath.Dir(root)

	// Create a recursive watch. Ensure that it's cancelled when we're done.
	watchPath := fmt.Sprintf("%s/...", parent)
	if err := notify.Watch(watchPath, notifications, notify.All); err != nil {
		return errors.Wrap(err, "unable to create watcher")
	}
	defer notify.Stop(notifications)

	// Poll for the next notification, coalescing, or cancellation.
	for {
		select {
		case notification := <-notifications:
			// Only process notifications that match our target. This test might
			// be a bit fragile, but it should be okay since we normalize our
			// root path and notification paths are supposed to be absolute and
			// real. If we receive a match, reset the coalescing timer.
			if strings.HasPrefix(notification.Path(), root) {
				if !timer.Stop() {
					<-timer.C
				}
				timer.Reset(coalescingWindow)
			}
		case <-timer.C:
			// Forward a coalesced event in a non-blocking manner.
			select {
			case events <- struct{}{}:
			default:
			}
		case <-context.Done():
			return errors.New("cancelled")
		}
	}
}
<commit_msg>Removed watch event coallescing.<commit_after>// +build windows darwin,cgo

package session

import (
	"context"
	"fmt"
	"path/filepath"
	"strings"
	"time"

	"github.com/pkg/errors"

	"github.com/rjeczalik/notify"
)

const (
	scanPollInterval      = 30 * time.Second
	watchEventsBufferSize = 10
)

func watch(context context.Context, root string, events chan struct{}) error {
	// Create a watch notifications channel.
	notifications := make(chan notify.EventInfo, watchEventsBufferSize)

	// Compute the parent directory of root. We watch this because (a) we may
	// have a file root and many systems don't support watching a file directly
	// (only a directory), (b) the root may not exist yet, and (c) deletion of
	// the watch root isn't seen on all platforms. Of course, if the parent of
	// root is deleted, we won't see that either here, but we'll eventually see
	// it in polling.
	parent := filepath.Dir(root)

	// Create a recursive watch. Ensure that it's cancelled when we're done.
	watchPath := fmt.Sprintf("%s/...", parent)
	if err := notify.Watch(watchPath, notifications, notify.All); err != nil {
		return errors.Wrap(err, "unable to create watcher")
	}
	defer notify.Stop(notifications)

	// Poll for the next notification, coalescing, or cancellation.
	for {
		select {
		case notification := <-notifications:
			// Only process notifications that match our target. This test might
			// be a bit fragile, but it should be okay since we normalize our
			// root path and notification paths are supposed to be absolute and
			// real. If we receive a match, forward the event in a non-blocking
			// manner.
			if strings.HasPrefix(notification.Path(), root) {
				select {
				case events <- struct{}{}:
				default:
				}
			}
		case <-context.Done():
			return errors.New("watch cancelled")
		}
	}
}
<|endoftext|>