<commit_before>package bootstrappolicy

import (
	kapi "k8s.io/kubernetes/pkg/api"
	"k8s.io/kubernetes/pkg/serviceaccount"
)

const (
	// SecurityContextConstraintPrivileged is used as the name for the system default privileged scc.
	SecurityContextConstraintPrivileged     = "privileged"
	SecurityContextConstraintPrivilegedDesc = "privileged allows access to all privileged and host features and the ability to run as any user, any group, any fsGroup, and with any SELinux context.  WARNING: this is the most relaxed SCC and should be used only for cluster administration. Grant with caution."

	// SecurityContextConstraintRestricted is used as the name for the system default restricted scc.
	SecurityContextConstraintRestricted     = "restricted"
	SecurityContextConstraintRestrictedDesc = "restricted denies access to all host features and requires pods to be run with a UID, and SELinux context that are allocated to the namespace.  This is the most restrictive SCC."

	// SecurityContextConstraintNonRoot is used as the name for the system default non-root scc.
	SecurityContextConstraintNonRoot     = "nonroot"
	SecurityContextConstraintNonRootDesc = "nonroot provides all features of the restricted SCC but allows users to run with any non-root UID.  The user must specify the UID or it must be specified on the by the manifest of the container runtime."

	// SecurityContextConstraintHostMountAndAnyUID is used as the name for the system default host mount + any UID scc.
	SecurityContextConstraintHostMountAndAnyUID     = "hostmount-anyuid"
	SecurityContextConstraintHostMountAndAnyUIDDesc = "hostmount-anyuid provides all the features of the restricted SCC but allows host mounts and any UID by a pod.  This is primarily used by the persistent volume recycler. WARNING: this SCC allows host file system access as any UID, including UID 0.  Grant with caution."

	// SecurityContextConstraintHostNS is used as the name for the system default scc
	// that grants access to all host ns features.
	SecurityContextConstraintHostNS     = "hostaccess"
	SecurityContextConstraintHostNSDesc = "hostaccess allows access to all host namespaces but still requires pods to be run with a UID and SELinux context that are allocated to the namespace. WARNING: this SCC allows host access to namespaces, file systems, and PIDS.  It should only be used by trusted pods.  Grant with caution."

	// SecurityContextConstraintsAnyUID is used as the name for the system default scc that
	// grants access to run as any uid but is still restricted to specific SELinux contexts.
	SecurityContextConstraintsAnyUID     = "anyuid"
	SecurityContextConstraintsAnyUIDDesc = "anyuid provides all features of the restricted SCC but allows users to run with any UID and any GID.  This is the default SCC for authenticated users."

	// SecurityContextConstraintsHostNetwork is used as the name for the system default scc that
	// grants access to run with host networking and host ports but still allocates uid/gids/selinux from the
	// namespace.
	SecurityContextConstraintsHostNetwork     = "hostnetwork"
	SecurityContextConstraintsHostNetworkDesc = "hostnetwork allows using host networking and host ports but still requires pods to be run with a UID and SELinux context that are allocated to the namespace."

	// DescriptionAnnotation is the annotation used for attaching descriptions.
	DescriptionAnnotation = "kubernetes.io/description"
)

// GetBootstrapSecurityContextConstraints returns the slice of default SecurityContextConstraints
// for system bootstrapping.  This method takes additional users and groups that should be added
// to the strategies.  Use GetBoostrapSCCAccess to produce the default set of mappings.
func GetBootstrapSecurityContextConstraints(sccNameToAdditionalGroups map[string][]string, sccNameToAdditionalUsers map[string][]string) []kapi.SecurityContextConstraints {
	// define priorities here and reference them below so it is easy to see, at a glance
	// what we're setting
	var (
		// this is set to 10 to allow wiggle room for admins to set other priorities without
		// having to adjust anyUID.
		securityContextConstraintsAnyUIDPriority = 10
	)

	constraints := []kapi.SecurityContextConstraints{
		// SecurityContextConstraintPrivileged allows all access for every field
		{
			ObjectMeta: kapi.ObjectMeta{
				Name: SecurityContextConstraintPrivileged,
				Annotations: map[string]string{
					DescriptionAnnotation: SecurityContextConstraintPrivilegedDesc,
				},
			},
			AllowPrivilegedContainer: true,
			Volumes:                  []kapi.FSType{kapi.FSTypeAll},
			AllowHostNetwork:         true,
			AllowHostPorts:           true,
			AllowHostPID:             true,
			AllowHostIPC:             true,
			SELinuxContext: kapi.SELinuxContextStrategyOptions{
				Type: kapi.SELinuxStrategyRunAsAny,
			},
			RunAsUser: kapi.RunAsUserStrategyOptions{
				Type: kapi.RunAsUserStrategyRunAsAny,
			},
			FSGroup: kapi.FSGroupStrategyOptions{
				Type: kapi.FSGroupStrategyRunAsAny,
			},
			SupplementalGroups: kapi.SupplementalGroupsStrategyOptions{
				Type: kapi.SupplementalGroupsStrategyRunAsAny,
			},
		},
		// SecurityContextConstraintNonRoot does not allow host access, allocates SELinux labels
		// and allows the user to request a specific UID or provide the default in the dockerfile.
		{
			ObjectMeta: kapi.ObjectMeta{
				Name: SecurityContextConstraintNonRoot,
				Annotations: map[string]string{
					DescriptionAnnotation: SecurityContextConstraintNonRootDesc,
				},
			},
			Volumes: []kapi.FSType{kapi.FSTypeEmptyDir, kapi.FSTypeSecret, kapi.FSTypeDownwardAPI, kapi.FSTypeConfigMap, kapi.FSTypePersistentVolumeClaim},
			SELinuxContext: kapi.SELinuxContextStrategyOptions{
				// This strategy requires that annotations on the namespace which will be populated
				// by the admission controller.  If namespaces are not annotated creating the strategy
				// will fail.
				Type: kapi.SELinuxStrategyMustRunAs,
			},
			RunAsUser: kapi.RunAsUserStrategyOptions{
				// This strategy requires that the user request to run as a specific UID or that
				// the docker file contain a USER directive.
				Type: kapi.RunAsUserStrategyMustRunAsNonRoot,
			},
			FSGroup: kapi.FSGroupStrategyOptions{
				Type: kapi.FSGroupStrategyRunAsAny,
			},
			SupplementalGroups: kapi.SupplementalGroupsStrategyOptions{
				Type: kapi.SupplementalGroupsStrategyRunAsAny,
			},
		},
		// SecurityContextConstraintHostMountAndAnyUID is the same as the restricted scc but allows the use of the hostPath and NFS plugins, and running as any UID.
		// Used by the PV recycler.
		{
			ObjectMeta: kapi.ObjectMeta{
				Name: SecurityContextConstraintHostMountAndAnyUID,
				Annotations: map[string]string{
					DescriptionAnnotation: SecurityContextConstraintHostMountAndAnyUIDDesc,
				},
			},
			Volumes: []kapi.FSType{kapi.FSTypeHostPath, kapi.FSTypeEmptyDir, kapi.FSTypeSecret, kapi.FSTypeDownwardAPI, kapi.FSTypeConfigMap, kapi.FSTypePersistentVolumeClaim, kapi.FSTypeNFS},
			SELinuxContext: kapi.SELinuxContextStrategyOptions{
				// This strategy requires that annotations on the namespace which will be populated
				// by the admission controller.  If namespaces are not annotated creating the strategy
				// will fail.
				Type: kapi.SELinuxStrategyMustRunAs,
			},
			RunAsUser: kapi.RunAsUserStrategyOptions{
				// This strategy requires that annotations on the namespace which will be populated
				// by the admission controller.  If namespaces are not annotated creating the strategy
				// will fail.
				Type: kapi.RunAsUserStrategyRunAsAny,
			},
			FSGroup: kapi.FSGroupStrategyOptions{
				Type: kapi.FSGroupStrategyRunAsAny,
			},
			SupplementalGroups: kapi.SupplementalGroupsStrategyOptions{
				Type: kapi.SupplementalGroupsStrategyRunAsAny,
			},
		},
		// SecurityContextConstraintHostNS allows access to everything except privileged on the host
		// but still allocates UIDs and SELinux.
		{
			ObjectMeta: kapi.ObjectMeta{
				Name: SecurityContextConstraintHostNS,
				Annotations: map[string]string{
					DescriptionAnnotation: SecurityContextConstraintHostNSDesc,
				},
			},
			Volumes:          []kapi.FSType{kapi.FSTypeHostPath, kapi.FSTypeEmptyDir, kapi.FSTypeSecret, kapi.FSTypeDownwardAPI, kapi.FSTypeConfigMap, kapi.FSTypePersistentVolumeClaim},
			AllowHostNetwork: true,
			AllowHostPorts:   true,
			AllowHostPID:     true,
			AllowHostIPC:     true,
			SELinuxContext: kapi.SELinuxContextStrategyOptions{
				// This strategy requires that annotations on the namespace which will be populated
				// by the admission controller.  If namespaces are not annotated creating the strategy
				// will fail.
				Type: kapi.SELinuxStrategyMustRunAs,
			},
			RunAsUser: kapi.RunAsUserStrategyOptions{
				// This strategy requires that annotations on the namespace which will be populated
				// by the admission controller.  If namespaces are not annotated creating the strategy
				// will fail.
				Type: kapi.RunAsUserStrategyMustRunAsRange,
			},
			FSGroup: kapi.FSGroupStrategyOptions{
				Type: kapi.FSGroupStrategyMustRunAs,
			},
			SupplementalGroups: kapi.SupplementalGroupsStrategyOptions{
				Type: kapi.SupplementalGroupsStrategyRunAsAny,
			},
		},
		// SecurityContextConstraintRestricted allows no host access and allocates UIDs and SELinux.
		{
			ObjectMeta: kapi.ObjectMeta{
				Name: SecurityContextConstraintRestricted,
				Annotations: map[string]string{
					DescriptionAnnotation: SecurityContextConstraintRestrictedDesc,
				},
			},
			Volumes: []kapi.FSType{kapi.FSTypeEmptyDir, kapi.FSTypeSecret, kapi.FSTypeDownwardAPI, kapi.FSTypeConfigMap, kapi.FSTypePersistentVolumeClaim},
			SELinuxContext: kapi.SELinuxContextStrategyOptions{
				// This strategy requires that annotations on the namespace which will be populated
				// by the admission controller.  If namespaces are not annotated creating the strategy
				// will fail.
				Type: kapi.SELinuxStrategyMustRunAs,
			},
			RunAsUser: kapi.RunAsUserStrategyOptions{
				// This strategy requires that annotations on the namespace which will be populated
				// by the admission controller.  If namespaces are not annotated creating the strategy
				// will fail.
				Type: kapi.RunAsUserStrategyMustRunAsRange,
			},
			FSGroup: kapi.FSGroupStrategyOptions{
				Type: kapi.FSGroupStrategyMustRunAs,
			},
			SupplementalGroups: kapi.SupplementalGroupsStrategyOptions{
				Type: kapi.SupplementalGroupsStrategyRunAsAny,
			},
			// drops unsafe caps
			RequiredDropCapabilities: []kapi.Capability{"KILL", "MKNOD", "SYS_CHROOT", "SETUID", "SETGID"},
		},
		// SecurityContextConstraintsAnyUID allows no host access and allocates SELinux.
		{
			ObjectMeta: kapi.ObjectMeta{
				Name: SecurityContextConstraintsAnyUID,
				Annotations: map[string]string{
					DescriptionAnnotation: SecurityContextConstraintsAnyUIDDesc,
				},
			},
			Volumes: []kapi.FSType{kapi.FSTypeEmptyDir, kapi.FSTypeSecret, kapi.FSTypeDownwardAPI, kapi.FSTypeConfigMap, kapi.FSTypePersistentVolumeClaim},
			SELinuxContext: kapi.SELinuxContextStrategyOptions{
				// This strategy requires that annotations on the namespace which will be populated
				// by the admission controller.  If namespaces are not annotated creating the strategy
				// will fail.
				Type: kapi.SELinuxStrategyMustRunAs,
			},
			RunAsUser: kapi.RunAsUserStrategyOptions{
				Type: kapi.RunAsUserStrategyRunAsAny,
			},
			FSGroup: kapi.FSGroupStrategyOptions{
				Type: kapi.FSGroupStrategyRunAsAny,
			},
			SupplementalGroups: kapi.SupplementalGroupsStrategyOptions{
				Type: kapi.SupplementalGroupsStrategyRunAsAny,
			},
			// prefer the anyuid SCC over ones that force a uid
			Priority: &securityContextConstraintsAnyUIDPriority,
			// drops unsafe caps
			RequiredDropCapabilities: []kapi.Capability{"KILL", "MKNOD", "SYS_CHROOT"},
		},
		// SecurityContextConstraintsHostNetwork allows host network and host ports
		{
			ObjectMeta: kapi.ObjectMeta{
				Name: SecurityContextConstraintsHostNetwork,
				Annotations: map[string]string{
					DescriptionAnnotation: SecurityContextConstraintsHostNetworkDesc,
				},
			},
			AllowHostNetwork: true,
			AllowHostPorts:   true,
			Volumes:          []kapi.FSType{kapi.FSTypeEmptyDir, kapi.FSTypeSecret, kapi.FSTypeDownwardAPI, kapi.FSTypeConfigMap, kapi.FSTypePersistentVolumeClaim},
			SELinuxContext: kapi.SELinuxContextStrategyOptions{
				// This strategy requires that annotations on the namespace which will be populated
				// by the admission controller.  If namespaces are not annotated creating the strategy
				// will fail.
				Type: kapi.SELinuxStrategyMustRunAs,
			},
			RunAsUser: kapi.RunAsUserStrategyOptions{
				// This strategy requires that annotations on the namespace which will be populated
				// by the admission controller.  If namespaces are not annotated creating the strategy
				// will fail.
				Type: kapi.RunAsUserStrategyMustRunAsRange,
			},
			FSGroup: kapi.FSGroupStrategyOptions{
				Type: kapi.FSGroupStrategyMustRunAs,
			},
			SupplementalGroups: kapi.SupplementalGroupsStrategyOptions{
				Type: kapi.SupplementalGroupsStrategyMustRunAs,
			},
			// drops unsafe caps
			RequiredDropCapabilities: []kapi.Capability{"KILL", "MKNOD", "SYS_CHROOT", "SETUID", "SETGID"},
		},
	}

	// add default access
	for i, constraint := range constraints {
		if usersToAdd, ok := sccNameToAdditionalUsers[constraint.Name]; ok {
			constraints[i].Users = append(constraints[i].Users, usersToAdd...)
		}
		if groupsToAdd, ok := sccNameToAdditionalGroups[constraint.Name]; ok {
			constraints[i].Groups = append(constraints[i].Groups, groupsToAdd...)
		}
	}
	return constraints
}

// GetBoostrapSCCAccess provides the default set of access that should be passed to GetBootstrapSecurityContextConstraints.
func GetBoostrapSCCAccess(infraNamespace string) (map[string][]string, map[string][]string) {
	groups := map[string][]string{
		SecurityContextConstraintPrivileged: {ClusterAdminGroup, NodesGroup},
		SecurityContextConstraintsAnyUID:    {ClusterAdminGroup},
		SecurityContextConstraintRestricted: {AuthenticatedGroup},
	}

	buildControllerUsername := serviceaccount.MakeUsername(infraNamespace, InfraBuildControllerServiceAccountName)
	pvRecyclerControllerUsername := serviceaccount.MakeUsername(infraNamespace, InfraPersistentVolumeRecyclerControllerServiceAccountName)
	users := map[string][]string{
		SecurityContextConstraintPrivileged:         {buildControllerUsername},
		SecurityContextConstraintHostMountAndAnyUID: {pvRecyclerControllerUsername},
	}
	return groups, users
}
<commit_msg>do not force drop KILL cap on anyuid SCC<commit_after>package bootstrappolicy

import (
	kapi "k8s.io/kubernetes/pkg/api"
	"k8s.io/kubernetes/pkg/serviceaccount"
)

const (
	// SecurityContextConstraintPrivileged is used as the name for the system default privileged scc.
	SecurityContextConstraintPrivileged     = "privileged"
	SecurityContextConstraintPrivilegedDesc = "privileged allows access to all privileged and host features and the ability to run as any user, any group, any fsGroup, and with any SELinux context.  WARNING: this is the most relaxed SCC and should be used only for cluster administration. Grant with caution."

	// SecurityContextConstraintRestricted is used as the name for the system default restricted scc.
	SecurityContextConstraintRestricted     = "restricted"
	SecurityContextConstraintRestrictedDesc = "restricted denies access to all host features and requires pods to be run with a UID, and SELinux context that are allocated to the namespace.  This is the most restrictive SCC."

	// SecurityContextConstraintNonRoot is used as the name for the system default non-root scc.
	SecurityContextConstraintNonRoot     = "nonroot"
	SecurityContextConstraintNonRootDesc = "nonroot provides all features of the restricted SCC but allows users to run with any non-root UID.  The user must specify the UID or it must be specified on the by the manifest of the container runtime."

	// SecurityContextConstraintHostMountAndAnyUID is used as the name for the system default host mount + any UID scc.
	SecurityContextConstraintHostMountAndAnyUID     = "hostmount-anyuid"
	SecurityContextConstraintHostMountAndAnyUIDDesc = "hostmount-anyuid provides all the features of the restricted SCC but allows host mounts and any UID by a pod.  This is primarily used by the persistent volume recycler. WARNING: this SCC allows host file system access as any UID, including UID 0.  Grant with caution."

	// SecurityContextConstraintHostNS is used as the name for the system default scc
	// that grants access to all host ns features.
	SecurityContextConstraintHostNS     = "hostaccess"
	SecurityContextConstraintHostNSDesc = "hostaccess allows access to all host namespaces but still requires pods to be run with a UID and SELinux context that are allocated to the namespace. WARNING: this SCC allows host access to namespaces, file systems, and PIDS.  It should only be used by trusted pods.  Grant with caution."

	// SecurityContextConstraintsAnyUID is used as the name for the system default scc that
	// grants access to run as any uid but is still restricted to specific SELinux contexts.
	SecurityContextConstraintsAnyUID     = "anyuid"
	SecurityContextConstraintsAnyUIDDesc = "anyuid provides all features of the restricted SCC but allows users to run with any UID and any GID.  This is the default SCC for authenticated users."

	// SecurityContextConstraintsHostNetwork is used as the name for the system default scc that
	// grants access to run with host networking and host ports but still allocates uid/gids/selinux from the
	// namespace.
	SecurityContextConstraintsHostNetwork     = "hostnetwork"
	SecurityContextConstraintsHostNetworkDesc = "hostnetwork allows using host networking and host ports but still requires pods to be run with a UID and SELinux context that are allocated to the namespace."

	// DescriptionAnnotation is the annotation used for attaching descriptions.
	DescriptionAnnotation = "kubernetes.io/description"
)

// GetBootstrapSecurityContextConstraints returns the slice of default SecurityContextConstraints
// for system bootstrapping.  This method takes additional users and groups that should be added
// to the strategies.  Use GetBoostrapSCCAccess to produce the default set of mappings.
func GetBootstrapSecurityContextConstraints(sccNameToAdditionalGroups map[string][]string, sccNameToAdditionalUsers map[string][]string) []kapi.SecurityContextConstraints {
	// define priorities here and reference them below so it is easy to see, at a glance
	// what we're setting
	var (
		// this is set to 10 to allow wiggle room for admins to set other priorities without
		// having to adjust anyUID.
		securityContextConstraintsAnyUIDPriority = 10
	)

	constraints := []kapi.SecurityContextConstraints{
		// SecurityContextConstraintPrivileged allows all access for every field
		{
			ObjectMeta: kapi.ObjectMeta{
				Name: SecurityContextConstraintPrivileged,
				Annotations: map[string]string{
					DescriptionAnnotation: SecurityContextConstraintPrivilegedDesc,
				},
			},
			AllowPrivilegedContainer: true,
			Volumes:                  []kapi.FSType{kapi.FSTypeAll},
			AllowHostNetwork:         true,
			AllowHostPorts:           true,
			AllowHostPID:             true,
			AllowHostIPC:             true,
			SELinuxContext: kapi.SELinuxContextStrategyOptions{
				Type: kapi.SELinuxStrategyRunAsAny,
			},
			RunAsUser: kapi.RunAsUserStrategyOptions{
				Type: kapi.RunAsUserStrategyRunAsAny,
			},
			FSGroup: kapi.FSGroupStrategyOptions{
				Type: kapi.FSGroupStrategyRunAsAny,
			},
			SupplementalGroups: kapi.SupplementalGroupsStrategyOptions{
				Type: kapi.SupplementalGroupsStrategyRunAsAny,
			},
		},
		// SecurityContextConstraintNonRoot does not allow host access, allocates SELinux labels
		// and allows the user to request a specific UID or provide the default in the dockerfile.
		{
			ObjectMeta: kapi.ObjectMeta{
				Name: SecurityContextConstraintNonRoot,
				Annotations: map[string]string{
					DescriptionAnnotation: SecurityContextConstraintNonRootDesc,
				},
			},
			Volumes: []kapi.FSType{kapi.FSTypeEmptyDir, kapi.FSTypeSecret, kapi.FSTypeDownwardAPI, kapi.FSTypeConfigMap, kapi.FSTypePersistentVolumeClaim},
			SELinuxContext: kapi.SELinuxContextStrategyOptions{
				// This strategy requires that annotations on the namespace which will be populated
				// by the admission controller.  If namespaces are not annotated creating the strategy
				// will fail.
				Type: kapi.SELinuxStrategyMustRunAs,
			},
			RunAsUser: kapi.RunAsUserStrategyOptions{
				// This strategy requires that the user request to run as a specific UID or that
				// the docker file contain a USER directive.
				Type: kapi.RunAsUserStrategyMustRunAsNonRoot,
			},
			FSGroup: kapi.FSGroupStrategyOptions{
				Type: kapi.FSGroupStrategyRunAsAny,
			},
			SupplementalGroups: kapi.SupplementalGroupsStrategyOptions{
				Type: kapi.SupplementalGroupsStrategyRunAsAny,
			},
		},
		// SecurityContextConstraintHostMountAndAnyUID is the same as the restricted scc but allows the use of the hostPath and NFS plugins, and running as any UID.
		// Used by the PV recycler.
		{
			ObjectMeta: kapi.ObjectMeta{
				Name: SecurityContextConstraintHostMountAndAnyUID,
				Annotations: map[string]string{
					DescriptionAnnotation: SecurityContextConstraintHostMountAndAnyUIDDesc,
				},
			},
			Volumes: []kapi.FSType{kapi.FSTypeHostPath, kapi.FSTypeEmptyDir, kapi.FSTypeSecret, kapi.FSTypeDownwardAPI, kapi.FSTypeConfigMap, kapi.FSTypePersistentVolumeClaim, kapi.FSTypeNFS},
			SELinuxContext: kapi.SELinuxContextStrategyOptions{
				// This strategy requires that annotations on the namespace which will be populated
				// by the admission controller.  If namespaces are not annotated creating the strategy
				// will fail.
				Type: kapi.SELinuxStrategyMustRunAs,
			},
			RunAsUser: kapi.RunAsUserStrategyOptions{
				// This strategy requires that annotations on the namespace which will be populated
				// by the admission controller.  If namespaces are not annotated creating the strategy
				// will fail.
				Type: kapi.RunAsUserStrategyRunAsAny,
			},
			FSGroup: kapi.FSGroupStrategyOptions{
				Type: kapi.FSGroupStrategyRunAsAny,
			},
			SupplementalGroups: kapi.SupplementalGroupsStrategyOptions{
				Type: kapi.SupplementalGroupsStrategyRunAsAny,
			},
		},
		// SecurityContextConstraintHostNS allows access to everything except privileged on the host
		// but still allocates UIDs and SELinux.
		{
			ObjectMeta: kapi.ObjectMeta{
				Name: SecurityContextConstraintHostNS,
				Annotations: map[string]string{
					DescriptionAnnotation: SecurityContextConstraintHostNSDesc,
				},
			},
			Volumes:          []kapi.FSType{kapi.FSTypeHostPath, kapi.FSTypeEmptyDir, kapi.FSTypeSecret, kapi.FSTypeDownwardAPI, kapi.FSTypeConfigMap, kapi.FSTypePersistentVolumeClaim},
			AllowHostNetwork: true,
			AllowHostPorts:   true,
			AllowHostPID:     true,
			AllowHostIPC:     true,
			SELinuxContext: kapi.SELinuxContextStrategyOptions{
				// This strategy requires that annotations on the namespace which will be populated
				// by the admission controller.  If namespaces are not annotated creating the strategy
				// will fail.
				Type: kapi.SELinuxStrategyMustRunAs,
			},
			RunAsUser: kapi.RunAsUserStrategyOptions{
				// This strategy requires that annotations on the namespace which will be populated
				// by the admission controller.  If namespaces are not annotated creating the strategy
				// will fail.
				Type: kapi.RunAsUserStrategyMustRunAsRange,
			},
			FSGroup: kapi.FSGroupStrategyOptions{
				Type: kapi.FSGroupStrategyMustRunAs,
			},
			SupplementalGroups: kapi.SupplementalGroupsStrategyOptions{
				Type: kapi.SupplementalGroupsStrategyRunAsAny,
			},
		},
		// SecurityContextConstraintRestricted allows no host access and allocates UIDs and SELinux.
		{
			ObjectMeta: kapi.ObjectMeta{
				Name: SecurityContextConstraintRestricted,
				Annotations: map[string]string{
					DescriptionAnnotation: SecurityContextConstraintRestrictedDesc,
				},
			},
			Volumes: []kapi.FSType{kapi.FSTypeEmptyDir, kapi.FSTypeSecret, kapi.FSTypeDownwardAPI, kapi.FSTypeConfigMap, kapi.FSTypePersistentVolumeClaim},
			SELinuxContext: kapi.SELinuxContextStrategyOptions{
				// This strategy requires that annotations on the namespace which will be populated
				// by the admission controller.  If namespaces are not annotated creating the strategy
				// will fail.
				Type: kapi.SELinuxStrategyMustRunAs,
			},
			RunAsUser: kapi.RunAsUserStrategyOptions{
				// This strategy requires that annotations on the namespace which will be populated
				// by the admission controller.  If namespaces are not annotated creating the strategy
				// will fail.
				Type: kapi.RunAsUserStrategyMustRunAsRange,
			},
			FSGroup: kapi.FSGroupStrategyOptions{
				Type: kapi.FSGroupStrategyMustRunAs,
			},
			SupplementalGroups: kapi.SupplementalGroupsStrategyOptions{
				Type: kapi.SupplementalGroupsStrategyRunAsAny,
			},
			// drops unsafe caps
			RequiredDropCapabilities: []kapi.Capability{"KILL", "MKNOD", "SYS_CHROOT", "SETUID", "SETGID"},
		},
		// SecurityContextConstraintsAnyUID allows no host access and allocates SELinux.
		{
			ObjectMeta: kapi.ObjectMeta{
				Name: SecurityContextConstraintsAnyUID,
				Annotations: map[string]string{
					DescriptionAnnotation: SecurityContextConstraintsAnyUIDDesc,
				},
			},
			Volumes: []kapi.FSType{kapi.FSTypeEmptyDir, kapi.FSTypeSecret, kapi.FSTypeDownwardAPI, kapi.FSTypeConfigMap, kapi.FSTypePersistentVolumeClaim},
			SELinuxContext: kapi.SELinuxContextStrategyOptions{
				// This strategy requires that annotations on the namespace which will be populated
				// by the admission controller.  If namespaces are not annotated creating the strategy
				// will fail.
				Type: kapi.SELinuxStrategyMustRunAs,
			},
			RunAsUser: kapi.RunAsUserStrategyOptions{
				Type: kapi.RunAsUserStrategyRunAsAny,
			},
			FSGroup: kapi.FSGroupStrategyOptions{
				Type: kapi.FSGroupStrategyRunAsAny,
			},
			SupplementalGroups: kapi.SupplementalGroupsStrategyOptions{
				Type: kapi.SupplementalGroupsStrategyRunAsAny,
			},
			// prefer the anyuid SCC over ones that force a uid
			Priority: &securityContextConstraintsAnyUIDPriority,
			// drops unsafe caps
			RequiredDropCapabilities: []kapi.Capability{"MKNOD", "SYS_CHROOT"},
		},
		// SecurityContextConstraintsHostNetwork allows host network and host ports
		{
			ObjectMeta: kapi.ObjectMeta{
				Name: SecurityContextConstraintsHostNetwork,
				Annotations: map[string]string{
					DescriptionAnnotation: SecurityContextConstraintsHostNetworkDesc,
				},
			},
			AllowHostNetwork: true,
			AllowHostPorts:   true,
			Volumes:          []kapi.FSType{kapi.FSTypeEmptyDir, kapi.FSTypeSecret, kapi.FSTypeDownwardAPI, kapi.FSTypeConfigMap, kapi.FSTypePersistentVolumeClaim},
			SELinuxContext: kapi.SELinuxContextStrategyOptions{
				// This strategy requires that annotations on the namespace which will be populated
				// by the admission controller.  If namespaces are not annotated creating the strategy
				// will fail.
				Type: kapi.SELinuxStrategyMustRunAs,
			},
			RunAsUser: kapi.RunAsUserStrategyOptions{
				// This strategy requires that annotations on the namespace which will be populated
				// by the admission controller.  If namespaces are not annotated creating the strategy
				// will fail.
				Type: kapi.RunAsUserStrategyMustRunAsRange,
			},
			FSGroup: kapi.FSGroupStrategyOptions{
				Type: kapi.FSGroupStrategyMustRunAs,
			},
			SupplementalGroups: kapi.SupplementalGroupsStrategyOptions{
				Type: kapi.SupplementalGroupsStrategyMustRunAs,
			},
			// drops unsafe caps
			RequiredDropCapabilities: []kapi.Capability{"KILL", "MKNOD", "SYS_CHROOT", "SETUID", "SETGID"},
		},
	}

	// add default access
	for i, constraint := range constraints {
		if usersToAdd, ok := sccNameToAdditionalUsers[constraint.Name]; ok {
			constraints[i].Users = append(constraints[i].Users, usersToAdd...)
		}
		if groupsToAdd, ok := sccNameToAdditionalGroups[constraint.Name]; ok {
			constraints[i].Groups = append(constraints[i].Groups, groupsToAdd...)
		}
	}
	return constraints
}

// GetBoostrapSCCAccess provides the default set of access that should be passed to GetBootstrapSecurityContextConstraints.
func GetBoostrapSCCAccess(infraNamespace string) (map[string][]string, map[string][]string) {
	groups := map[string][]string{
		SecurityContextConstraintPrivileged: {ClusterAdminGroup, NodesGroup},
		SecurityContextConstraintsAnyUID:    {ClusterAdminGroup},
		SecurityContextConstraintRestricted: {AuthenticatedGroup},
	}

	buildControllerUsername := serviceaccount.MakeUsername(infraNamespace, InfraBuildControllerServiceAccountName)
	pvRecyclerControllerUsername := serviceaccount.MakeUsername(infraNamespace, InfraPersistentVolumeRecyclerControllerServiceAccountName)
	users := map[string][]string{
		SecurityContextConstraintPrivileged:         {buildControllerUsername},
		SecurityContextConstraintHostMountAndAnyUID: {pvRecyclerControllerUsername},
	}
	return groups, users
}
<|endoftext|>


<commit_before>package workflowtrigger

import (
	"fmt"
	"strings"
	"sync"

	"github.com/caicloud/cyclone/pkg/k8s/clientset"
	"github.com/pkg/errors"
	"github.com/robfig/cron"
	"github.com/satori/go.uuid"
	log "github.com/sirupsen/logrus"

	"github.com/caicloud/cyclone/pkg/apis/cyclone/v1alpha1"
)

const (
	KeyTemplate = "%s/%s"
)

type CronTrigger struct {
	Cron                *cron.Cron
	IsRunning           bool
	SuccCount           int
	FailCount           int
	Namespace           string
	WorkflowTriggerName string
	WorkflowRun         *v1alpha1.WorkflowRun
	Manage              *CronTriggerManager
}

type CronTriggerManager struct {
	Client         clientset.Interface
	CronTriggerMap map[string]*CronTrigger
	mutex          sync.Mutex
}

func NewTriggerManager(client clientset.Interface) *CronTriggerManager {
	return &CronTriggerManager{
		Client:         client,
		CronTriggerMap: make(map[string]*CronTrigger),
		mutex:          sync.Mutex{},
	}
}

func (trigger *CronTrigger) getKeyFromTrigger() string {
	return fmt.Sprintf(KeyTemplate, trigger.Namespace, trigger.WorkflowTriggerName)
}

func (m *CronTriggerManager) AddTrigger(trigger *CronTrigger) {
	wftKey := trigger.getKeyFromTrigger()
	m.mutex.Lock()
	defer m.mutex.Unlock()
	if wft, ok := m.CronTriggerMap[wftKey]; ok {
		// this situation may not happen for k8s will do same name check
		log.Warnf("failed to add cronTrigger, already exists: %+v\n", wft)
	} else {
		m.CronTriggerMap[wftKey] = trigger
	}
}

func (m *CronTriggerManager) DeleteTrigger(wftKey string) {
	m.mutex.Lock()
	defer m.mutex.Unlock()
	if wft, ok := m.CronTriggerMap[wftKey]; ok {
		wft.Cron.Stop()
		wft.IsRunning = false
		delete(m.CronTriggerMap, wftKey)
	} else {
		log.Warnf("failed to delete cronTrigger(%s), not exist\n", wftKey)
	}
}

func ToWorkflowTrigger(obj interface{}) (*v1alpha1.WorkflowTrigger, error) {

	wft, ok := obj.(*v1alpha1.WorkflowTrigger)
	if !ok {
		return nil, errors.New(fmt.Sprintf("I want type: WorkflowTrigger, but it is: %T", obj))
	} else {
		return wft, nil
	}
}

func (c *CronTrigger) Run() {

	c.WorkflowRun.Name = c.WorkflowTriggerName + "-" + strings.Replace(uuid.NewV1().String(), "-", "", -1)
	_, err := c.Manage.Client.CycloneV1alpha1().WorkflowRuns(c.Namespace).Create(c.WorkflowRun)
	if err != nil {
		c.FailCount++
		log.Warnf("can not create WorkflowRun: %s", err)
	} else {
		c.SuccCount++
	}
}

func getParaValue(items []v1alpha1.ParameterItem, key string) (string, bool) {
	for _, item := range items {
		if item.Name == key {
			return item.Value, true
		}
	}
	return "", false
}

func (m *CronTriggerManager) CreateCron(wft *v1alpha1.WorkflowTrigger) {

	schedule, has := getParaValue(wft.Spec.Parameters, "schedule")
	if !has {
		log.WithField("wft", wft.Name).Warn("Parameter 'schedule' not set in WorkflowTrigger spec")
		return
	}

	ct := &CronTrigger{
		Namespace:           wft.Namespace,
		WorkflowTriggerName: wft.Name,
	}

	wfr := &v1alpha1.WorkflowRun{
		Spec: wft.Spec.WorkflowRunSpec,
	}

	ct.WorkflowRun = wfr

	c := cron.New()
	err := c.AddJob(schedule, ct)
	if err != nil {
		log.Errorf("can not create Cron job: %s", err)
		return
	}

	ct.Cron = c
	ct.Manage = m
	m.AddTrigger(ct)

	if !wft.Spec.Disabled {
		ct.Cron.Start()
		ct.IsRunning = true
	}
}

func (m *CronTriggerManager) UpdateCron(wft *v1alpha1.WorkflowTrigger) {
	m.DeleteCron(wft)
	m.CreateCron(wft)
}

func (m *CronTriggerManager) DeleteCron(wft *v1alpha1.WorkflowTrigger) {
	wftKey := getKeyFromWorkflowTrigger(wft)
	m.DeleteTrigger(wftKey)
}

func getKeyFromWorkflowTrigger(wft *v1alpha1.WorkflowTrigger) string {
	return fmt.Sprintf(KeyTemplate, wft.Namespace, wft.Name)
}
<commit_msg>fix(trigger): support timezone for workflowTrigger<commit_after>package workflowtrigger

import (
	"fmt"
	"strconv"
	"strings"
	"sync"
	"time"

	"github.com/caicloud/cyclone/pkg/k8s/clientset"
	"github.com/pkg/errors"
	"github.com/robfig/cron"
	"github.com/satori/go.uuid"
	log "github.com/sirupsen/logrus"

	"github.com/caicloud/cyclone/pkg/apis/cyclone/v1alpha1"
)

const (
	KeyTemplate = "%s/%s"
)

type CronTrigger struct {
	Cron                *cron.Cron
	IsRunning           bool
	SuccCount           int
	FailCount           int
	Namespace           string
	WorkflowTriggerName string
	WorkflowRun         *v1alpha1.WorkflowRun
	Manage              *CronTriggerManager
}

type CronTriggerManager struct {
	Client         clientset.Interface
	CronTriggerMap map[string]*CronTrigger
	mutex          sync.Mutex
}

func NewTriggerManager(client clientset.Interface) *CronTriggerManager {
	return &CronTriggerManager{
		Client:         client,
		CronTriggerMap: make(map[string]*CronTrigger),
		mutex:          sync.Mutex{},
	}
}

func (trigger *CronTrigger) getKeyFromTrigger() string {
	return fmt.Sprintf(KeyTemplate, trigger.Namespace, trigger.WorkflowTriggerName)
}

func (m *CronTriggerManager) AddTrigger(trigger *CronTrigger) {
	wftKey := trigger.getKeyFromTrigger()
	m.mutex.Lock()
	defer m.mutex.Unlock()
	if wft, ok := m.CronTriggerMap[wftKey]; ok {
		// this situation may not happen for k8s will do same name check
		log.Warnf("failed to add cronTrigger, already exists: %+v\n", wft)
	} else {
		m.CronTriggerMap[wftKey] = trigger
	}
}

func (m *CronTriggerManager) DeleteTrigger(wftKey string) {
	m.mutex.Lock()
	defer m.mutex.Unlock()
	if wft, ok := m.CronTriggerMap[wftKey]; ok {
		wft.Cron.Stop()
		wft.IsRunning = false
		delete(m.CronTriggerMap, wftKey)
	} else {
		log.Warnf("failed to delete cronTrigger(%s), not exist\n", wftKey)
	}
}

func ToWorkflowTrigger(obj interface{}) (*v1alpha1.WorkflowTrigger, error) {

	wft, ok := obj.(*v1alpha1.WorkflowTrigger)
	if !ok {
		return nil, errors.New(fmt.Sprintf("I want type: WorkflowTrigger, but it is: %T", obj))
	} else {
		return wft, nil
	}
}

func (c *CronTrigger) Run() {

	c.WorkflowRun.Name = c.WorkflowTriggerName + "-" + strings.Replace(uuid.NewV1().String(), "-", "", -1)
	_, err := c.Manage.Client.CycloneV1alpha1().WorkflowRuns(c.Namespace).Create(c.WorkflowRun)
	if err != nil {
		c.FailCount++
		log.Warnf("can not create WorkflowRun: %s", err)
	} else {
		c.SuccCount++
	}
}

func getParaValue(items []v1alpha1.ParameterItem, key string) (string, bool) {
	for _, item := range items {
		if item.Name == key {
			return item.Value, true
		}
	}
	return "", false
}

func (m *CronTriggerManager) CreateCron(wft *v1alpha1.WorkflowTrigger) {

	schedule, has := getParaValue(wft.Spec.Parameters, "schedule")
	if !has {
		log.WithField("wft", wft.Name).Warn("Parameter 'schedule' not set in WorkflowTrigger spec")
		return
	}

	timezone, has := getParaValue(wft.Spec.Parameters, "timezone")
	if !has {
		timezone = "0"
	}

	minuteOffUTC, err := strconv.Atoi(timezone)
	if err != nil {
		log.Warnf("can not parse timezone(%s) to int", timezone)
		minuteOffUTC = 0
	}

	ct := &CronTrigger{
		Namespace:           wft.Namespace,
		WorkflowTriggerName: wft.Name,
	}

	wfr := &v1alpha1.WorkflowRun{
		Spec: wft.Spec.WorkflowRunSpec,
	}

	ct.WorkflowRun = wfr

	c := cron.NewWithLocation(time.FixedZone("userZone", -1*60*minuteOffUTC))
	err = c.AddJob(schedule, ct)
	if err != nil {
		log.Errorf("can not create Cron job: %s", err)
		return
	}

	ct.Cron = c
	ct.Manage = m
	m.AddTrigger(ct)

	if !wft.Spec.Disabled {
		ct.Cron.Start()
		ct.IsRunning = true
	}
}

func (m *CronTriggerManager) UpdateCron(wft *v1alpha1.WorkflowTrigger) {
	m.DeleteCron(wft)
	m.CreateCron(wft)
}

func (m *CronTriggerManager) DeleteCron(wft *v1alpha1.WorkflowTrigger) {
	wftKey := getKeyFromWorkflowTrigger(wft)
	m.DeleteTrigger(wftKey)
}

func getKeyFromWorkflowTrigger(wft *v1alpha1.WorkflowTrigger) string {
	return fmt.Sprintf(KeyTemplate, wft.Namespace, wft.Name)
}
<|endoftext|>


<commit_before>package geotypes

import "fmt"

type Config struct {
	Host             string
	Port             int
	SslMode          string
	DbUser           string
	DbName           string
	DbHost           string
	DbPort           int
	DbPass           string
	DataDir          string
	RedisAddr        string
	RedisPass        string
	AllowedCountries map[string]bool
}

type CHNode struct {
	Id int
	Coord
}

type Coord []float64

type Debugging bool

type Coordinate struct {
	Latitude  float64
	Longitude float64
	System    string
}

type Location struct {
	Address    Address
	Coordinate Coordinate
}

type Address struct {
	AdditionalInfo  string
	ApartmentLetter string
	ApartmentNumber int
	City            string
	Confidence      float64
	Resolution      int
	Country         string
	HouseNumber     int
	PostalCode      string
	Region          string
	Street          string
}

type Path struct {
	Length int   `json:"length"`
	Nodes  []int `json:"nodes"`
}

type Edge struct {
	Source Location
	Target Location
}

type NodeEdge struct {
	Source int `json:"source"`
	Target int `json:"target"`
}

type PathsInput struct {
	SpeedProfile int
	Edges        []Edge
}

type CoordinatePath struct {
	Distance int     `json:"distance"`
	Time     int     `json:"time"`
	Coords   []Coord `json:"coords"`
	SameRoad bool    `json:"-""`
}

type Matrix struct {
	Nodes []int `json:"sources"`
}

type Ratio struct {
	Id1, Id2, Dist40, Dist60, Dist80, Dist100, Dist120 int
	Ratio, Distance                                    float64
	Oneway                                             bool
	Coord                                              Coord
}

// GeoDB implements a database abstraction layer that hides all
// functionality related to databases.
type GeoDB interface {
	// Returns the point closest to the coordinates
	// in the road network, based on geoindexed data.
	QueryClosestPoint(point Coord, country string) (CHNode, error)

	// Transforms the nodes from the graph nodes,
	// which are indexed by integers, and returns the corresponding
	// coordinates.
	QueryCoordinates(nodes []int, country string) ([]Coord, error)

	// Gets an address using fuzzy search. Uses the street and city at
	// the moment.
	QueryFuzzyAddress(address Address, count int) ([]Location, error)

	// Calculates the distance between edges. If the array nodes is indexed
	// with the variable i, then edge pairs are formed like thus (i, i+1), (i+1, i+2)
	// and so on.
	QueryDistance(nodes []int, country string) (int, error)

	// Returns the server status, nil if everything works fine.
	QueryStatus() error
}

func (config *Config) String() string {
	s := fmt.Sprintf("user=%s dbname=%s host=%s port=%d password=%s sslmode=%s",
		config.DbUser, config.DbName, config.DbHost, config.DbPort, config.DbPass, config.SslMode)
	return s
}
<commit_msg>Add country code field.<commit_after>package geotypes

import "fmt"

type Config struct {
	Host             string
	Port             int
	SslMode          string
	DbUser           string
	DbName           string
	DbHost           string
	DbPort           int
	DbPass           string
	DataDir          string
	RedisAddr        string
	RedisPass        string
	AllowedCountries map[string]bool
}

type CHNode struct {
	Id int
	Coord
}

type Coord []float64

type Debugging bool

type Coordinate struct {
	Latitude  float64
	Longitude float64
	System    string
}

type Location struct {
	Address    Address
	Coordinate Coordinate
}

type Address struct {
	AdditionalInfo  string
	ApartmentLetter string
	ApartmentNumber int
	City            string
	Confidence      float64
	Resolution      int
	Country         string
	CountryCode     string
	HouseNumber     int
	PostalCode      string
	Region          string
	Street          string
}

type Path struct {
	Length int   `json:"length"`
	Nodes  []int `json:"nodes"`
}

type Edge struct {
	Source Location
	Target Location
}

type NodeEdge struct {
	Source int `json:"source"`
	Target int `json:"target"`
}

type PathsInput struct {
	SpeedProfile int
	Edges        []Edge
}

type CoordinatePath struct {
	Distance int     `json:"distance"`
	Time     int     `json:"time"`
	Coords   []Coord `json:"coords"`
	SameRoad bool    `json:"-""`
}

type Matrix struct {
	Nodes []int `json:"sources"`
}

type Ratio struct {
	Id1, Id2, Dist40, Dist60, Dist80, Dist100, Dist120 int
	Ratio, Distance                                    float64
	Oneway                                             bool
	Coord                                              Coord
}

// GeoDB implements a database abstraction layer that hides all
// functionality related to databases.
type GeoDB interface {
	// Returns the point closest to the coordinates
	// in the road network, based on geoindexed data.
	QueryClosestPoint(point Coord, country string) (CHNode, error)

	// Transforms the nodes from the graph nodes,
	// which are indexed by integers, and returns the corresponding
	// coordinates.
	QueryCoordinates(nodes []int, country string) ([]Coord, error)

	// Gets an address using fuzzy search. Uses the street and city at
	// the moment.
	QueryFuzzyAddress(address Address, count int) ([]Location, error)

	// Calculates the distance between edges. If the array nodes is indexed
	// with the variable i, then edge pairs are formed like thus (i, i+1), (i+1, i+2)
	// and so on.
	QueryDistance(nodes []int, country string) (int, error)

	// Returns the server status, nil if everything works fine.
	QueryStatus() error
}

func (config *Config) String() string {
	s := fmt.Sprintf("user=%s dbname=%s host=%s port=%d password=%s sslmode=%s",
		config.DbUser, config.DbName, config.DbHost, config.DbPort, config.DbPass, config.SslMode)
	return s
}
<|endoftext|>


<commit_before>package note

import (
	"fmt"
	"time"

	"github.com/cozy/cozy-stack/model/instance"
	"github.com/cozy/cozy-stack/model/vfs"
	"github.com/cozy/cozy-stack/pkg/consts"
	"github.com/cozy/cozy-stack/pkg/couchdb"
	"github.com/cozy/cozy-stack/pkg/jsonapi"
	"github.com/cozy/prosemirror-go/transform"
)

// Step is a patch to apply on a note.
type Step map[string]interface{}

// ID returns the step qualified identifier
func (s Step) ID() string {
	id, _ := s["_id"].(string)
	return id
}

// Rev returns the step revision
func (s Step) Rev() string {
	rev, _ := s["_rev"].(string)
	return rev
}

// DocType returns the document type
func (s Step) DocType() string { return consts.NotesSteps }

// Clone implements couchdb.Doc
func (s Step) Clone() couchdb.Doc {
	cloned := make(Step)
	for k, v := range s {
		cloned[k] = v
	}
	return cloned
}

// SetID changes the step qualified identifier
func (s Step) SetID(id string) {
	if id == "" {
		delete(s, "_id")
	} else {
		s["_id"] = id
	}
}

// SetRev changes the step revision
func (s Step) SetRev(rev string) {
	if rev == "" {
		delete(s, "_rev")
	} else {
		s["_rev"] = rev
	}
}

// Included is part of the jsonapi.Object interface
func (s Step) Included() []jsonapi.Object { return nil }

// Links is part of the jsonapi.Object interface
func (s Step) Links() *jsonapi.LinksList { return nil }

// Relationships is part of the jsonapi.Object interface
func (s Step) Relationships() jsonapi.RelationshipMap { return nil }

func (s Step) timestamp() int64 {
	switch t := s["timestamp"].(type) {
	case float64:
		return int64(t)
	case int64:
		return t
	}
	return 0
}

func stepID(noteID string, version int64) string {
	return fmt.Sprintf("%s/%08d", noteID, version)
}

func endkey(noteID string) string {
	return fmt.Sprintf("%s/%s", noteID, couchdb.MaxString)
}

// GetSteps returns the steps for the given note, starting from the version.
func GetSteps(inst *instance.Instance, fileID string, version int64) ([]Step, error) {
	lock := inst.NotesLock()
	if err := lock.Lock(); err != nil {
		return nil, err
	}
	defer lock.Unlock()

	return getSteps(inst, fileID, version)
}

// getSteps is the same as GetSteps, but with the notes lock already acquired
func getSteps(inst *instance.Instance, fileID string, version int64) ([]Step, error) {
	var steps []Step
	req := couchdb.AllDocsRequest{
		Limit:    1000,
		StartKey: stepID(fileID, version),
		EndKey:   endkey(fileID),
	}
	if err := couchdb.GetAllDocs(inst, consts.NotesSteps, &req, &steps); err != nil {
		return nil, err
	}

	// The first step plays the role of a sentinel: if it isn't here, the
	// version is too old. Same if we have too many steps.
	if len(steps) == 0 || len(steps) == req.Limit {
		return nil, ErrTooOld
	}

	steps = steps[1:] // Discard the sentinel
	return steps, nil
}

// ApplySteps takes a note and some steps, and tries to apply them. It is an
// all or nothing change: if there is one error, the note won't be changed.
func ApplySteps(inst *instance.Instance, file *vfs.FileDoc, lastVersion string, steps []Step) (*vfs.FileDoc, error) {
	lock := inst.NotesLock()
	if err := lock.Lock(); err != nil {
		return nil, err
	}
	defer lock.Unlock()

	if len(steps) == 0 {
		return nil, ErrNoSteps
	}

	doc, err := get(inst, file)
	if err != nil {
		return nil, err
	}
	if lastVersion != fmt.Sprintf("%d", doc.Version) {
		return nil, ErrCannotApply
	}

	if err := apply(inst, doc, steps); err != nil {
		return nil, err
	}
	if err := saveSteps(inst, steps); err != nil {
		return nil, err
	}
	publishSteps(inst, file.ID(), steps)

	if err := saveToCache(inst, doc); err != nil {
		return nil, err
	}
	return doc.asFile(file), nil
}

func apply(inst *instance.Instance, doc *Document, steps []Step) error {
	schema, err := doc.Schema()
	if err != nil {
		inst.Logger().WithField("nspace", "notes").
			Infof("Cannot instantiate the schema: %s", err)
		return ErrInvalidSchema
	}

	content, err := doc.Content()
	if err != nil {
		inst.Logger().WithField("nspace", "notes").
			Infof("Cannot instantiate the document: %s", err)
		return ErrInvalidFile
	}

	now := time.Now().Unix()
	for i, s := range steps {
		step, err := transform.StepFromJSON(schema, s)
		if err != nil {
			inst.Logger().WithField("nspace", "notes").
				Infof("Cannot instantiate a step: %s", err)
			return ErrInvalidSteps
		}
		result := step.Apply(content)
		if result.Failed != "" {
			inst.Logger().WithField("nspace", "notes").
				Infof("Cannot apply a step: %s (version=%d)", result.Failed, doc.Version)
			return ErrCannotApply
		}
		content = result.Doc
		doc.Version++
		steps[i].SetID(stepID(doc.ID(), doc.Version))
		steps[i]["version"] = doc.Version
		steps[i]["timestamp"] = now
	}
	doc.SetContent(content)
	return nil
}

func saveSteps(inst *instance.Instance, steps []Step) error {
	olds := make([]interface{}, len(steps))
	news := make([]interface{}, len(steps))
	for i, s := range steps {
		news[i] = s
	}
	if err := couchdb.BulkUpdateDocs(inst, consts.NotesSteps, news, olds); err != nil {
		if !couchdb.IsNoDatabaseError(err) {
			return err
		}
		if err := couchdb.EnsureDBExist(inst, consts.NotesSteps); err != nil {
			return err
		}
		if err := couchdb.BulkUpdateDocs(inst, consts.NotesSteps, news, olds); err != nil {
			return err
		}
	}
	return nil
}

func purgeOldSteps(inst *instance.Instance, fileID string) {
	var steps []Step
	req := couchdb.AllDocsRequest{
		Limit:    1000,
		StartKey: stepID(fileID, 0),
		EndKey:   endkey(fileID),
	}
	if err := couchdb.GetAllDocs(inst, consts.NotesSteps, &req, &steps); err != nil {
		if !couchdb.IsNoDatabaseError(err) {
			inst.Logger().WithField("nspace", "notes").
				Warnf("Cannot purge old steps for file %s: %s", fileID, err)
		}
		return
	}
	if len(steps) == 0 {
		return
	}

	limit := time.Now().Add(-cleanStepsAfter).Unix()
	docs := make([]couchdb.Doc, 0, len(steps))
	for i := range steps {
		if steps[i].timestamp() > limit {
			break
		}
		docs = append(docs, &steps[i])
	}
	if len(docs) == 0 {
		return
	}
	if err := couchdb.BulkDeleteDocs(inst, consts.NotesSteps, docs); err != nil {
		inst.Logger().WithField("nspace", "notes").
			Warnf("Cannot purge old steps for file %s: %s", fileID, err)
	}
}

var _ jsonapi.Object = &Step{}
<commit_msg>Fix issue with multi-bytes runes in notes<commit_after>package note

import (
	"fmt"
	"time"

	"github.com/cozy/cozy-stack/model/instance"
	"github.com/cozy/cozy-stack/model/vfs"
	"github.com/cozy/cozy-stack/pkg/consts"
	"github.com/cozy/cozy-stack/pkg/couchdb"
	"github.com/cozy/cozy-stack/pkg/jsonapi"
	"github.com/cozy/prosemirror-go/transform"
)

// Step is a patch to apply on a note.
type Step map[string]interface{}

// ID returns the step qualified identifier
func (s Step) ID() string {
	id, _ := s["_id"].(string)
	return id
}

// Rev returns the step revision
func (s Step) Rev() string {
	rev, _ := s["_rev"].(string)
	return rev
}

// DocType returns the document type
func (s Step) DocType() string { return consts.NotesSteps }

// Clone implements couchdb.Doc
func (s Step) Clone() couchdb.Doc {
	cloned := make(Step)
	for k, v := range s {
		cloned[k] = v
	}
	return cloned
}

// SetID changes the step qualified identifier
func (s Step) SetID(id string) {
	if id == "" {
		delete(s, "_id")
	} else {
		s["_id"] = id
	}
}

// SetRev changes the step revision
func (s Step) SetRev(rev string) {
	if rev == "" {
		delete(s, "_rev")
	} else {
		s["_rev"] = rev
	}
}

// Included is part of the jsonapi.Object interface
func (s Step) Included() []jsonapi.Object { return nil }

// Links is part of the jsonapi.Object interface
func (s Step) Links() *jsonapi.LinksList { return nil }

// Relationships is part of the jsonapi.Object interface
func (s Step) Relationships() jsonapi.RelationshipMap { return nil }

func (s Step) timestamp() int64 {
	switch t := s["timestamp"].(type) {
	case float64:
		return int64(t)
	case int64:
		return t
	}
	return 0
}

func stepID(noteID string, version int64) string {
	return fmt.Sprintf("%s/%08d", noteID, version)
}

func endkey(noteID string) string {
	return fmt.Sprintf("%s/%s", noteID, couchdb.MaxString)
}

// GetSteps returns the steps for the given note, starting from the version.
func GetSteps(inst *instance.Instance, fileID string, version int64) ([]Step, error) {
	lock := inst.NotesLock()
	if err := lock.Lock(); err != nil {
		return nil, err
	}
	defer lock.Unlock()

	return getSteps(inst, fileID, version)
}

// getSteps is the same as GetSteps, but with the notes lock already acquired
func getSteps(inst *instance.Instance, fileID string, version int64) ([]Step, error) {
	var steps []Step
	req := couchdb.AllDocsRequest{
		Limit:    1000,
		StartKey: stepID(fileID, version),
		EndKey:   endkey(fileID),
	}
	if err := couchdb.GetAllDocs(inst, consts.NotesSteps, &req, &steps); err != nil {
		return nil, err
	}

	// The first step plays the role of a sentinel: if it isn't here, the
	// version is too old. Same if we have too many steps.
	if len(steps) == 0 || len(steps) == req.Limit {
		return nil, ErrTooOld
	}

	if version > 0 {
		steps = steps[1:] // Discard the sentinel
	}
	return steps, nil
}

// ApplySteps takes a note and some steps, and tries to apply them. It is an
// all or nothing change: if there is one error, the note won't be changed.
func ApplySteps(inst *instance.Instance, file *vfs.FileDoc, lastVersion string, steps []Step) (*vfs.FileDoc, error) {
	lock := inst.NotesLock()
	if err := lock.Lock(); err != nil {
		return nil, err
	}
	defer lock.Unlock()

	if len(steps) == 0 {
		return nil, ErrNoSteps
	}

	doc, err := get(inst, file)
	if err != nil {
		return nil, err
	}
	if lastVersion != fmt.Sprintf("%d", doc.Version) {
		return nil, ErrCannotApply
	}

	if err := apply(inst, doc, steps); err != nil {
		return nil, err
	}
	if err := saveSteps(inst, steps); err != nil {
		return nil, err
	}
	publishSteps(inst, file.ID(), steps)

	if err := saveToCache(inst, doc); err != nil {
		return nil, err
	}
	return doc.asFile(file), nil
}

func apply(inst *instance.Instance, doc *Document, steps []Step) error {
	schema, err := doc.Schema()
	if err != nil {
		inst.Logger().WithField("nspace", "notes").
			Infof("Cannot instantiate the schema: %s", err)
		return ErrInvalidSchema
	}

	content, err := doc.Content()
	if err != nil {
		inst.Logger().WithField("nspace", "notes").
			Infof("Cannot instantiate the document: %s", err)
		return ErrInvalidFile
	}

	now := time.Now().Unix()
	for i, s := range steps {
		step, err := transform.StepFromJSON(schema, s)
		if err != nil {
			inst.Logger().WithField("nspace", "notes").
				Infof("Cannot instantiate a step: %s", err)
			return ErrInvalidSteps
		}
		result := step.Apply(content)
		if result.Failed != "" {
			inst.Logger().WithField("nspace", "notes").
				Infof("Cannot apply a step: %s (version=%d)", result.Failed, doc.Version)
			return ErrCannotApply
		}
		content = result.Doc
		doc.Version++
		steps[i].SetID(stepID(doc.ID(), doc.Version))
		steps[i]["version"] = doc.Version
		steps[i]["timestamp"] = now
	}
	doc.SetContent(content)
	return nil
}

func saveSteps(inst *instance.Instance, steps []Step) error {
	olds := make([]interface{}, len(steps))
	news := make([]interface{}, len(steps))
	for i, s := range steps {
		news[i] = s
	}
	if err := couchdb.BulkUpdateDocs(inst, consts.NotesSteps, news, olds); err != nil {
		if !couchdb.IsNoDatabaseError(err) {
			return err
		}
		if err := couchdb.EnsureDBExist(inst, consts.NotesSteps); err != nil {
			return err
		}
		if err := couchdb.BulkUpdateDocs(inst, consts.NotesSteps, news, olds); err != nil {
			return err
		}
	}
	return nil
}

func purgeOldSteps(inst *instance.Instance, fileID string) {
	var steps []Step
	req := couchdb.AllDocsRequest{
		Limit:    1000,
		StartKey: stepID(fileID, 0),
		EndKey:   endkey(fileID),
	}
	if err := couchdb.GetAllDocs(inst, consts.NotesSteps, &req, &steps); err != nil {
		if !couchdb.IsNoDatabaseError(err) {
			inst.Logger().WithField("nspace", "notes").
				Warnf("Cannot purge old steps for file %s: %s", fileID, err)
		}
		return
	}
	if len(steps) == 0 {
		return
	}

	limit := time.Now().Add(-cleanStepsAfter).Unix()
	docs := make([]couchdb.Doc, 0, len(steps))
	for i := range steps {
		if steps[i].timestamp() > limit {
			break
		}
		docs = append(docs, &steps[i])
	}
	if len(docs) == 0 {
		return
	}
	if err := couchdb.BulkDeleteDocs(inst, consts.NotesSteps, docs); err != nil {
		inst.Logger().WithField("nspace", "notes").
			Warnf("Cannot purge old steps for file %s: %s", fileID, err)
	}
}

var _ jsonapi.Object = &Step{}
<|endoftext|>


<commit_before>package bean

import (
	"fmt"
	"github.com/shaalx/sstruct/utils"
	"math"
	"sort"
	"strings"
)

type TopicMatix []TopicSlice

var filter []string = []string{
	",", "的", "在", "和", "了", "也", "上", "还", "是", "年", "有", "都", "而", "我", "这个", "这么", "将", "一个", "家", "最", "从", "能", "就", "不", "而是", "就是", "", "", "", "", "", "", "", "", "", "", "",
}

func IsFilterContains(str string) bool {
	filters := strings.Join(filter, ",")
	return strings.Contains(filters, ","+str+",")
}

// 统计结果
func (t *TopicMatix) Statistics() {
	// 统计词频，映射到map
	key_freq := make(map[string]int32, 1)
	for _, key_slice := range *t {
		for _, key := range key_slice {
			if IsFilterContains(key.Const) {
				continue
			}
			key_freq[key.Const]++
		}
	}
	// 按照词频排序
	cellSlice := make(CellSlice, len(key_freq))
	i := 0
	var sum_freq int32
	for key_word, freq := range key_freq {
		cell := Cell{key_word, freq}
		cellSlice[i] = &cell
		i++
		sum_freq += freq
	}
	sort.Sort(cellSlice)
	cellSlice.String()

	// 卡方检验
	// nq 期望值为 sum_freq/len(key_freq)
	nq := float64(sum_freq) / float64(len(key_freq))
	var score float64
	// 组合关键字
	sentences := make(Sens, len(*t))
	// 后续处理
	var minFreq int32
	for i, key_slice := range *t {
		score = 0.0
		minFreq = 1000
		for _, key_word := range key_slice {
			fr, ok := key_freq[key_word.Const]
			if minFreq > fr {
				minFreq = fr
			}
			if ok {
				score += math.Pow(float64(fr)-nq, 2.0) / nq
				// score += float32(float64(math.Pow(float64(fr-int32(len(it)*2)), 2.0)) / float64(len(it)*2))
				// score += float32(float64(math.Pow(float64(fr-int32(len(it))), 2.0)) / float64(len(it)))
			}
		}
		if minFreq < 2 {
			score *= 0.3
		}
		sen := Sen{Str: key_slice.WordStrings(), Sum: score, Avg: score}
		sentences[i] = &sen
	}
	// 排除重复
	sentences = *sentences.EjRepeat()
	// 按照卡方值排序
	sort.Sort(sentences)

	// 保存结果
	stringSaveChan := make(chan string, 5)
	go utils.SaveString(stringSaveChan, "stat.txt")
	fmt.Printf("\n%s\t %s\t %s\t\n", "index", "avg", "key-word")
	for _, it := range sentences {
		statStr := it.String()
		stringSaveChan <- statStr
		// fmt.Println(statStr)
	}
	fmt.Print(sentences.Top(20))
}

type Sen struct {
	Str string
	Sum float64
	Avg float64
}

func (s *Sen) String() string {
	// return fmt.Sprintf("%.3f\t %.3f\t %s\t", s.Avg, s.Sum, s.Str)
	return fmt.Sprintf("%.2f\t %s\t", s.Avg, s.Str)
}

type Sens []*Sen

func (c Sens) Len() int {
	return len(c)
}

func (c Sens) Less(i, j int) bool {
	return c[i].Avg > c[j].Avg
}

func (c Sens) Swap(i, j int) {
	c[i], c[j] = c[j], c[i]
}

func (s Sens) Top(n int) string {
	resStr := ""
	for i, it := range s {
		if i >= n {
			return resStr
		}
		resStr += fmt.Sprintf("%d\t %s\n", i+1, it)
	}
	return resStr
}

// 排除重复值
func (t *Sens) EjRepeat() *Sens {
	length := len(*t)
	ejMap := make(map[string]*Sen, length)
	for _, it := range *t {
		// _, ok := ejMap[it.Str]
		// if ok {
		// 	continue
		// } else {
		ejMap[it.Str] = it
		// }
	}
	i := 0
	result := make(Sens, len(ejMap))
	for _, v := range ejMap {
		result[i] = v
		i++
	}
	return &result
}
<commit_msg>result frequencd statistics<commit_after>package bean

import (
	"fmt"
	"github.com/shaalx/sstruct/utils"
	"math"
	"sort"
	"strings"
)

type TopicMatix []TopicSlice

var filter []string = []string{
	",", "的", "在", "和", "了", "也", "上", "还", "是", "年", "有", "都", "而", "我", "这个", "这么", "将", "一个", "家", "最", "从", "能", "就", "不", "而是", "就是", "该", "", "", "", "", "", "", "", "", "", "",
}

func IsFilterContains(str string) bool {
	filters := strings.Join(filter, ",")
	return strings.Contains(filters, ","+str+",")
}

// 统计结果
func (t *TopicMatix) Statistics() {
	// 统计词频，映射到map
	key_freq := make(map[string]int32, 1)
	for _, key_slice := range *t {
		for _, key := range key_slice {
			if IsFilterContains(key.Const) {
				continue
			}
			key_freq[key.Const]++
		}
	}
	// 按照词频排序
	cellSlice := make(CellSlice, len(key_freq))
	i := 0
	var sum_freq int32
	for key_word, freq := range key_freq {
		cell := Cell{key_word, freq}
		cellSlice[i] = &cell
		i++
		sum_freq += freq
	}
	sort.Sort(cellSlice)
	cellSlice.String()

	// 卡方检验
	// nq 期望值为 sum_freq/len(key_freq)
	nq := float64(sum_freq) / float64(len(key_freq))
	var score float64
	// 组合关键字
	sentences := make(Sens, len(*t))
	// 后续处理
	var minFreq int32
	for i, key_slice := range *t {
		score = 0.0
		minFreq = 1000
		for _, key_word := range key_slice {
			fr, ok := key_freq[key_word.Const]
			if minFreq > fr {
				minFreq = fr
			}
			if ok {
				score += math.Pow(float64(fr)-nq, 2.0) / nq
				// score += float32(float64(math.Pow(float64(fr-int32(len(it)*2)), 2.0)) / float64(len(it)*2))
				// score += float32(float64(math.Pow(float64(fr-int32(len(it))), 2.0)) / float64(len(it)))
			}
		}
		if minFreq < 2 {
			score *= 0.3
		}
		sen := Sen{Str: key_slice.WordStrings(), Sum: score, Avg: score}
		sentences[i] = &sen
	}
	// 排除重复
	sentences = *sentences.EjRepeat()
	// 按照卡方值排序
	sort.Sort(sentences)

	// 保存结果
	stringSaveChan := make(chan string, 5)
	go utils.SaveString(stringSaveChan, "stat.txt")
	fmt.Printf("\n%s\t %s\t %s\t\n", "index", "avg", "key-word")
	for _, it := range sentences {
		statStr := it.String()
		stringSaveChan <- statStr
		// fmt.Println(statStr)
	}
	fmt.Print(sentences.Top(50))
}

type Sen struct {
	Str string
	Sum float64
	Avg float64
	Fre int
}

func (s *Sen) String() string {
	// return fmt.Sprintf("%.3f\t %.3f\t %s\t", s.Avg, s.Sum, s.Str)
	return fmt.Sprintf("%.2f\t %s\t %d", s.Avg, s.Str, s.Fre)
}

type Sens []*Sen

func (c Sens) Len() int {
	return len(c)
}

func (c Sens) Less(i, j int) bool {
	return c[i].Avg > c[j].Avg
}

func (c Sens) Swap(i, j int) {
	c[i], c[j] = c[j], c[i]
}

func (s Sens) Top(n int) string {
	resStr := ""
	for i, it := range s {
		if i >= n {
			return resStr
		}
		resStr += fmt.Sprintf("%d\t %s\n", i+1, it)
	}
	return resStr
}

// 排除重复值
func (t *Sens) EjRepeat() *Sens {
	length := len(*t)
	ejMap := make(map[string]*Sen, length)
	ejMapCount := make(map[string]int, length)
	for _, it := range *t {
		// _, ok := ejMap[it.Str]
		// if ok {
		// 	continue
		// } else {
		ejMap[it.Str] = it
		ejMapCount[it.Str]++
		// }
	}
	i := 0
	result := make(Sens, len(ejMap))
	for _, v := range ejMap {
		v.Fre = ejMapCount[v.Str]
		result[i] = v
		i++
	}
	return &result
}
<|endoftext|>


<commit_before>package github

import (
	"fmt"
	"io/ioutil"
	"os"
	"path/filepath"
	"strconv"

	"github.com/BurntSushi/toml"
	"github.com/github/hub/utils"
	"github.com/howeyc/gopass"
)

var (
	defaultConfigsFile = filepath.Join(os.Getenv("HOME"), ".config", "hub")
)

type Credential struct {
	Host        string `toml:"host"`
	User        string `toml:"user"`
	AccessToken string `toml:"access_token"`
}

type Configs struct {
	Credentials []Credential `toml:"credentials"`
}

func (c *Configs) allCredentials() (creds []Credential) {
	for _, cred := range c.Credentials {
		creds = append(creds, cred)
	}

	return
}

func (c *Configs) PromptFor(host string) *Credential {
	cd := c.find(host)
	if cd == nil {
		user := c.PromptForUser()
		pass := c.PromptForPassword(host, user)

		// Create Client with a stub Credential
		client := Client{Credential: &Credential{Host: host}}
		token, err := client.FindOrCreateToken(user, pass, "")
		if err != nil {
			if ce, ok := err.(*ClientError); ok && ce.Is2FAError() {
				code := c.PromptForOTP()
				token, err = client.FindOrCreateToken(user, pass, code)
			}
		}
		utils.Check(err)

		client.Credential.AccessToken = token
		currentUser, err := client.CurrentUser()
		utils.Check(err)

		cd = &Credential{Host: host, User: currentUser.Login, AccessToken: token}
		c.Credentials = append(c.Credentials, *cd)
		err = saveTo(configsFile(), c)
		utils.Check(err)
	}

	return cd
}

func (c *Configs) PromptForUser() (user string) {
	user = os.Getenv("GITHUB_USER")
	if user != "" {
		return
	}

	fmt.Printf("%s username: ", GitHubHost)
	_, err := fmt.Scanln(&user)
	utils.Check(err)

	return
}

func (c *Configs) PromptForPassword(host, user string) (pass string) {
	pass = os.Getenv("GITHUB_PASSWORD")
	if pass != "" {
		return
	}

	fmt.Printf("%s password for %s (never stored): ", host, user)
	if isTerminal(os.Stdout.Fd()) {
		pass = string(gopass.GetPasswd())
	} else {
		_, err := fmt.Scanln(&pass)
		utils.Check(err)
	}

	return
}

func (c *Configs) PromptForOTP() string {
	var code string
	fmt.Print("two-factor authentication code: ")
	_, err := fmt.Scanln(&code)
	utils.Check(err)

	return code
}

func (c *Configs) find(host string) *Credential {
	for _, t := range c.Credentials {
		if t.Host == host {
			return &t
		}
	}

	return nil
}

func saveTo(filename string, v interface{}) error {
	err := os.MkdirAll(filepath.Dir(filename), 0771)
	if err != nil {
		return err
	}

	f, err := os.Create(filename)
	if err != nil {
		return err
	}
	defer f.Close()

	enc := toml.NewEncoder(f)
	return enc.Encode(v)
}

func loadFrom(filename string, c *Configs) (err error) {
	_, err = toml.DecodeFile(filename, c)
	return
}

func configsFile() string {
	configsFile := os.Getenv("GH_CONFIG")
	if configsFile == "" {
		configsFile = defaultConfigsFile
	}

	return configsFile
}

func CurrentConfigs() *Configs {
	c := &Configs{}

	configFile := configsFile()
	err := loadFrom(configFile, c)
	if err != nil {
		// load from YAML
	}

	return c
}

func (c *Configs) DefaultCredential() (credential *Credential) {
	if GitHubHostEnv != "" {
		credential = c.PromptFor(GitHubHostEnv)
	} else if len(c.Credentials) > 0 {
		credential = c.selectCredential()
	} else {
		credential = c.PromptFor(DefaultHost())
	}

	return
}

func (c *Configs) selectCredential() *Credential {
	creds := c.allCredentials()
	options := len(creds)

	if options == 1 {
		return &creds[0]
	}

	prompt := "Select host:\n"
	for idx, cred := range creds {
		prompt += fmt.Sprintf(" %d. %s\n", idx+1, cred.Host)
	}
	prompt += fmt.Sprint("> ")

	fmt.Printf(prompt)
	var index string
	_, err := fmt.Scanln(&index)
	utils.Check(err)

	i, err := strconv.Atoi(index)
	if err != nil || i < 1 || i > options {
		utils.Check(fmt.Errorf("Error: must enter a number [1-%d]", options))
	}

	return &creds[i-1]
}

func (c *Configs) Save() error {
	return saveTo(configsFile(), c)
}

// Public for testing purpose
func CreateTestConfigs(user, token string) *Configs {
	f, _ := ioutil.TempFile("", "test-config")
	defaultConfigsFile = f.Name()

	cred := Credential{
		User:        "jingweno",
		AccessToken: "123",
		Host:        GitHubHost,
	}

	c := &Configs{Credentials: []Credential{cred}}
	saveTo(f.Name(), c)

	return c
}
<commit_msg>Fix config permission according to hub's cuke<commit_after>package github

import (
	"fmt"
	"io/ioutil"
	"os"
	"path/filepath"
	"strconv"

	"github.com/BurntSushi/toml"
	"github.com/github/hub/utils"
	"github.com/howeyc/gopass"
)

var (
	defaultConfigsFile = filepath.Join(os.Getenv("HOME"), ".config", "hub")
)

type Credential struct {
	Host        string `toml:"host"`
	User        string `toml:"user"`
	AccessToken string `toml:"access_token"`
}

type Configs struct {
	Credentials []Credential `toml:"credentials"`
}

func (c *Configs) allCredentials() (creds []Credential) {
	for _, cred := range c.Credentials {
		creds = append(creds, cred)
	}

	return
}

func (c *Configs) PromptFor(host string) *Credential {
	cd := c.find(host)
	if cd == nil {
		user := c.PromptForUser()
		pass := c.PromptForPassword(host, user)

		// Create Client with a stub Credential
		client := Client{Credential: &Credential{Host: host}}
		token, err := client.FindOrCreateToken(user, pass, "")
		if err != nil {
			if ce, ok := err.(*ClientError); ok && ce.Is2FAError() {
				code := c.PromptForOTP()
				token, err = client.FindOrCreateToken(user, pass, code)
			}
		}
		utils.Check(err)

		client.Credential.AccessToken = token
		currentUser, err := client.CurrentUser()
		utils.Check(err)

		cd = &Credential{Host: host, User: currentUser.Login, AccessToken: token}
		c.Credentials = append(c.Credentials, *cd)
		err = saveTo(configsFile(), c)
		utils.Check(err)
	}

	return cd
}

func (c *Configs) PromptForUser() (user string) {
	user = os.Getenv("GITHUB_USER")
	if user != "" {
		return
	}

	fmt.Printf("%s username: ", GitHubHost)
	_, err := fmt.Scanln(&user)
	utils.Check(err)

	return
}

func (c *Configs) PromptForPassword(host, user string) (pass string) {
	pass = os.Getenv("GITHUB_PASSWORD")
	if pass != "" {
		return
	}

	fmt.Printf("%s password for %s (never stored): ", host, user)
	if isTerminal(os.Stdout.Fd()) {
		pass = string(gopass.GetPasswd())
	} else {
		_, err := fmt.Scanln(&pass)
		utils.Check(err)
	}

	return
}

func (c *Configs) PromptForOTP() string {
	var code string
	fmt.Print("two-factor authentication code: ")
	_, err := fmt.Scanln(&code)
	utils.Check(err)

	return code
}

func (c *Configs) find(host string) *Credential {
	for _, t := range c.Credentials {
		if t.Host == host {
			return &t
		}
	}

	return nil
}

func saveTo(filename string, v interface{}) error {
	err := os.MkdirAll(filepath.Dir(filename), 0771)
	if err != nil {
		return err
	}

	f, err := os.OpenFile(filename, os.O_RDWR|os.O_CREATE|os.O_TRUNC, 0600)
	if err != nil {
		return err
	}
	defer f.Close()

	enc := toml.NewEncoder(f)
	return enc.Encode(v)
}

func loadFrom(filename string, c *Configs) (err error) {
	_, err = toml.DecodeFile(filename, c)
	return
}

func configsFile() string {
	configsFile := os.Getenv("GH_CONFIG")
	if configsFile == "" {
		configsFile = defaultConfigsFile
	}

	return configsFile
}

func CurrentConfigs() *Configs {
	c := &Configs{}

	configFile := configsFile()
	err := loadFrom(configFile, c)
	if err != nil {
		// load from YAML
	}

	return c
}

func (c *Configs) DefaultCredential() (credential *Credential) {
	if GitHubHostEnv != "" {
		credential = c.PromptFor(GitHubHostEnv)
	} else if len(c.Credentials) > 0 {
		credential = c.selectCredential()
	} else {
		credential = c.PromptFor(DefaultHost())
	}

	return
}

func (c *Configs) selectCredential() *Credential {
	creds := c.allCredentials()
	options := len(creds)

	if options == 1 {
		return &creds[0]
	}

	prompt := "Select host:\n"
	for idx, cred := range creds {
		prompt += fmt.Sprintf(" %d. %s\n", idx+1, cred.Host)
	}
	prompt += fmt.Sprint("> ")

	fmt.Printf(prompt)
	var index string
	_, err := fmt.Scanln(&index)
	utils.Check(err)

	i, err := strconv.Atoi(index)
	if err != nil || i < 1 || i > options {
		utils.Check(fmt.Errorf("Error: must enter a number [1-%d]", options))
	}

	return &creds[i-1]
}

func (c *Configs) Save() error {
	return saveTo(configsFile(), c)
}

// Public for testing purpose
func CreateTestConfigs(user, token string) *Configs {
	f, _ := ioutil.TempFile("", "test-config")
	defaultConfigsFile = f.Name()

	cred := Credential{
		User:        "jingweno",
		AccessToken: "123",
		Host:        GitHubHost,
	}

	c := &Configs{Credentials: []Credential{cred}}
	saveTo(f.Name(), c)

	return c
}
<|endoftext|>


<commit_before>package build

import (
	"log"
	"os"
	"path/filepath"

	"github.com/davecheney/gogo"
)

// future implements a gogo.Future
type future struct {
	err chan error
}

func (f *future) Result() error {
	result := <-f.err
	f.err <- result
	return result
}

func Build(pkg *gogo.Package) gogo.Future {
	if pkg.Name() == "main" {
		return buildCommand(pkg)
	}
	return buildPackage(pkg)
}

func buildPackage(pkg *gogo.Package) gogo.Future {
	var deps []gogo.Future
	for _, dep := range pkg.Imports {
		deps = append(deps, buildPackage(dep))
	}
	if _, ok := pkg.Context.Targets[pkg]; !ok {
		compile := compile(pkg, deps, false)
		pkg.Context.Targets[pkg] = compile
	}
	return pkg.Context.Targets[pkg]
}

func buildCommand(pkg *gogo.Package) gogo.Future {
	var deps []gogo.Future
	for _, dep := range pkg.Imports {
		deps = append(deps, buildPackage(dep))
	}
	compile := compile(pkg, deps, false)
	ld := Ld(pkg, compile)
	return ld
}

// compile is a helper which combines all the steps required
// to build a go package
func compile(pkg *gogo.Package, deps []gogo.Future, includeTests bool) gogo.Future {
	var gofiles []string
	gofiles = append(gofiles, pkg.GoFiles...)
	var objs []objFuture
	if len(pkg.CgoFiles) > 0 {
		cgo, cgofiles := cgo(pkg, deps)
		deps = append(deps, cgo[0])
		objs = append(objs, cgo...)
		gofiles = append(gofiles, cgofiles...)
	}
	if includeTests {
		gofiles = append(gofiles, pkg.TestGoFiles...)
	}
	objs = append(objs, Gc(pkg, deps, gofiles))
	for _, sfile := range pkg.SFiles {
		objs = append(objs, Asm(pkg, sfile))
	}
	pack := Pack(pkg, objs)
	return pack
}

// objFuture represents a Future that produces an object file.
type objFuture interface {
	gogo.Future

	// objfile returns the name of the file that is
	// produced by the Future if successful.
	objfile() string
}

// pkgFuture represents a Future that produces a pkg (.a) file.
type pkgFuture interface {
	gogo.Future

	// pkgfile returns the name of the file that is
	// produced by the Future if successful.
	pkgfile() string
}

type packTarget struct {
	future
	deps     []objFuture
	objfiles []string
	*gogo.Package
}

// Pack returns a Future representing the result of packing a
// set of Context specific object files into an archive.
func Pack(pkg *gogo.Package, deps []objFuture) pkgFuture {
	t := &packTarget{
		future: future{
			err: make(chan error, 1),
		},
		deps:    deps,
		Package: pkg,
	}
	go t.execute()
	return t
}

func (t *packTarget) execute() {
	for _, dep := range t.deps {
		if err := dep.Result(); err != nil {
			t.future.err <- err
			return
		}
		// collect successful objfiles for packing
		t.objfiles = append(t.objfiles, dep.objfile())
	}
	log.Printf("pack %q: %s", t.Package.ImportPath(), t.objfiles)
	t.future.err <- t.build()
}

func (t *packTarget) pkgfile() string { return t.Package.ImportPath() + ".a" }

func (t *packTarget) build() error {
	ofile := t.pkgfile()
	pkgdir := filepath.Dir(filepath.Join(t.Pkgdir(), ofile))
	if err := os.MkdirAll(pkgdir, 0777); err != nil {
		return err
	}
	return t.Pack(ofile, t.Pkgdir(), t.objfiles...)
}

type gcTarget struct {
	future
	deps    []gogo.Future
	gofiles []string
	*gogo.Package
}

func (t *gcTarget) execute() {
	for _, dep := range t.deps {
		if err := dep.Result(); err != nil {
			t.future.err <- err
			return
		}
	}
	log.Printf("gc %q: %s", t.Package.ImportPath(), t.gofiles)
	t.future.err <- t.build()
}

// Gc returns a Future representing the result of compiling a
// set of gofiles with the Context specified gc compiler.
func Gc(pkg *gogo.Package, deps []gogo.Future, gofiles []string) objFuture {
	t := &gcTarget{
		future: future{
			err: make(chan error, 1),
		},
		deps:    deps,
		gofiles: gofiles,
		Package: pkg,
	}
	go t.execute()
	return t
}

func (t *gcTarget) objfile() string { return filepath.Join(t.Objdir(), "_go_.6") }

func (t *gcTarget) build() error {
	if err := os.MkdirAll(t.Objdir(), 0777); err != nil {
		return err
	}
	return t.Gc(t.ImportPath(), t.Srcdir(), t.objfile(), t.gofiles)
}

type asmTarget struct {
	future
	sfile string
	*gogo.Package
}

func (t *asmTarget) execute() {
	log.Printf("as %q: %s", t.Package.ImportPath(), t.sfile)
	t.future.err <- t.build()
}

// Asm returns a Future representing the result of assembling
// sfile with the Context specified asssembler.
func Asm(pkg *gogo.Package, sfile string) objFuture {
	t := &asmTarget{
		future: future{
			err: make(chan error, 1),
		},
		sfile:   sfile,
		Package: pkg,
	}
	go t.execute()
	return t
}

func (t *asmTarget) objfile() string {
	return filepath.Join(t.Objdir(), t.sfile[:len(t.sfile)-len(".s")]+".6")
}

func (t *asmTarget) build() error {
	if err := os.MkdirAll(t.Objdir(), 0777); err != nil {
		return err
	}
	return t.Asm(t.Srcdir(), t.objfile(), t.sfile)
}

type ldTarget struct {
	future
	deps []gogo.Future
	*gogo.Package
}

func (t *ldTarget) execute() {
	for _, dep := range t.deps {
		if err := dep.Result(); err != nil {
			t.future.err <- err
			return
		}
	}
	log.Printf("ld %q", t.Package.ImportPath())
	t.future.err <- t.build()
}

// Ld returns a Future representing the result of linking a
// Package into a command with the Context provided linker.
func Ld(pkg *gogo.Package, deps ...gogo.Future) gogo.Future {
	t := &ldTarget{
		future: future{
			err: make(chan error, 1),
		},
		deps:    deps,
		Package: pkg,
	}
	go t.execute()
	return &t.future
}

func (t *ldTarget) pkgfile() string { return filepath.Join(t.Workdir(), t.Package.ImportPath()+".a") }

func (t *ldTarget) build() error {
	bindir := t.Package.Context.Bindir()
	if err := os.MkdirAll(bindir, 0777); err != nil {
		return err
	}
	return t.Ld(filepath.Join(bindir, filepath.Base(t.Package.ImportPath())), t.pkgfile())
}
<commit_msg>added more documentation<commit_after>package build

import (
	"log"
	"os"
	"path/filepath"

	"github.com/davecheney/gogo"
)

// future implements a gogo.Future
type future struct {
	err chan error
}

func (f *future) Result() error {
	result := <-f.err
	f.err <- result
	return result
}

// Build returns a Future representing the result of compiling the
// package pkg, and its dependencies.
// If pkg is a a command, then the results of build include linking
// the final binary into pkg.Context.Bindir().
func Build(pkg *gogo.Package) gogo.Future {
	if pkg.Name() == "main" {
		return buildCommand(pkg)
	}
	return buildPackage(pkg)
}

// buildPackage returns a Future repesenting the results of compiling
// pkg and its dependencies.
func buildPackage(pkg *gogo.Package) gogo.Future {
	var deps []gogo.Future
	for _, dep := range pkg.Imports {
		deps = append(deps, buildPackage(dep))
	}
	if _, ok := pkg.Context.Targets[pkg]; !ok {
		compile := compile(pkg, deps, false)
		pkg.Context.Targets[pkg] = compile
	}
	return pkg.Context.Targets[pkg]
}

// buildCommand returns a Future repesenting the results of compiling
// pkg as a command and linking the result into pkg.Context.Bindir().
func buildCommand(pkg *gogo.Package) gogo.Future {
	var deps []gogo.Future
	for _, dep := range pkg.Imports {
		deps = append(deps, buildPackage(dep))
	}
	compile := compile(pkg, deps, false)
	ld := Ld(pkg, compile)
	return ld
}

// compile is a helper which combines all the steps required
// to build a go package
func compile(pkg *gogo.Package, deps []gogo.Future, includeTests bool) gogo.Future {
	var gofiles []string
	gofiles = append(gofiles, pkg.GoFiles...)
	var objs []objFuture
	if len(pkg.CgoFiles) > 0 {
		cgo, cgofiles := cgo(pkg, deps)
		deps = append(deps, cgo[0])
		objs = append(objs, cgo...)
		gofiles = append(gofiles, cgofiles...)
	}
	if includeTests {
		gofiles = append(gofiles, pkg.TestGoFiles...)
	}
	objs = append(objs, Gc(pkg, deps, gofiles))
	for _, sfile := range pkg.SFiles {
		objs = append(objs, Asm(pkg, sfile))
	}
	pack := Pack(pkg, objs)
	return pack
}

// objFuture represents a Future that produces an object file.
type objFuture interface {
	gogo.Future

	// objfile returns the name of the file that is
	// produced by the Future if successful.
	objfile() string
}

// pkgFuture represents a Future that produces a pkg (.a) file.
type pkgFuture interface {
	gogo.Future

	// pkgfile returns the name of the file that is
	// produced by the Future if successful.
	pkgfile() string
}

type packTarget struct {
	future
	deps     []objFuture
	objfiles []string
	*gogo.Package
}

// Pack returns a Future representing the result of packing a
// set of Context specific object files into an archive.
func Pack(pkg *gogo.Package, deps []objFuture) pkgFuture {
	t := &packTarget{
		future: future{
			err: make(chan error, 1),
		},
		deps:    deps,
		Package: pkg,
	}
	go t.execute()
	return t
}

func (t *packTarget) execute() {
	for _, dep := range t.deps {
		if err := dep.Result(); err != nil {
			t.future.err <- err
			return
		}
		// collect successful objfiles for packing
		t.objfiles = append(t.objfiles, dep.objfile())
	}
	log.Printf("pack %q: %s", t.Package.ImportPath(), t.objfiles)
	t.future.err <- t.build()
}

func (t *packTarget) pkgfile() string { return t.Package.ImportPath() + ".a" }

func (t *packTarget) build() error {
	ofile := t.pkgfile()
	pkgdir := filepath.Dir(filepath.Join(t.Pkgdir(), ofile))
	if err := os.MkdirAll(pkgdir, 0777); err != nil {
		return err
	}
	return t.Pack(ofile, t.Pkgdir(), t.objfiles...)
}

type gcTarget struct {
	future
	deps    []gogo.Future
	gofiles []string
	*gogo.Package
}

func (t *gcTarget) execute() {
	for _, dep := range t.deps {
		if err := dep.Result(); err != nil {
			t.future.err <- err
			return
		}
	}
	log.Printf("gc %q: %s", t.Package.ImportPath(), t.gofiles)
	t.future.err <- t.build()
}

// Gc returns a Future representing the result of compiling a
// set of gofiles with the Context specified gc compiler.
func Gc(pkg *gogo.Package, deps []gogo.Future, gofiles []string) objFuture {
	t := &gcTarget{
		future: future{
			err: make(chan error, 1),
		},
		deps:    deps,
		gofiles: gofiles,
		Package: pkg,
	}
	go t.execute()
	return t
}

func (t *gcTarget) objfile() string { return filepath.Join(t.Objdir(), "_go_.6") }

func (t *gcTarget) build() error {
	if err := os.MkdirAll(t.Objdir(), 0777); err != nil {
		return err
	}
	return t.Gc(t.ImportPath(), t.Srcdir(), t.objfile(), t.gofiles)
}

type asmTarget struct {
	future
	sfile string
	*gogo.Package
}

func (t *asmTarget) execute() {
	log.Printf("as %q: %s", t.Package.ImportPath(), t.sfile)
	t.future.err <- t.build()
}

// Asm returns a Future representing the result of assembling
// sfile with the Context specified asssembler.
func Asm(pkg *gogo.Package, sfile string) objFuture {
	t := &asmTarget{
		future: future{
			err: make(chan error, 1),
		},
		sfile:   sfile,
		Package: pkg,
	}
	go t.execute()
	return t
}

func (t *asmTarget) objfile() string {
	return filepath.Join(t.Objdir(), t.sfile[:len(t.sfile)-len(".s")]+".6")
}

func (t *asmTarget) build() error {
	if err := os.MkdirAll(t.Objdir(), 0777); err != nil {
		return err
	}
	return t.Asm(t.Srcdir(), t.objfile(), t.sfile)
}

type ldTarget struct {
	future
	deps []gogo.Future
	*gogo.Package
}

func (t *ldTarget) execute() {
	for _, dep := range t.deps {
		if err := dep.Result(); err != nil {
			t.future.err <- err
			return
		}
	}
	log.Printf("ld %q", t.Package.ImportPath())
	t.future.err <- t.build()
}

// Ld returns a Future representing the result of linking a
// Package into a command with the Context provided linker.
func Ld(pkg *gogo.Package, deps ...gogo.Future) gogo.Future {
	t := &ldTarget{
		future: future{
			err: make(chan error, 1),
		},
		deps:    deps,
		Package: pkg,
	}
	go t.execute()
	return &t.future
}

func (t *ldTarget) pkgfile() string { return filepath.Join(t.Workdir(), t.Package.ImportPath()+".a") }

func (t *ldTarget) build() error {
	bindir := t.Package.Context.Bindir()
	if err := os.MkdirAll(bindir, 0777); err != nil {
		return err
	}
	return t.Ld(filepath.Join(bindir, filepath.Base(t.Package.ImportPath())), t.pkgfile())
}
<|endoftext|>


<commit_before>package main

import (
	"fmt"
	"os"
	"time"
)

func main() {

	for i := 1; i < 10; i++ {
		if i%3 == 0 {
			fmt.Fprintf(os.Stderr, "STDERR i: %d\n", i)
		} else {
			fmt.Printf("STDOUT i: %d\n", i)
		}
		time.Sleep(1 * time.Second)
	}

}
<commit_msg>	modified:   main.go<commit_after>package main

import (
	"fmt"
	"os"
	"time"
)

func main() {

	for i := 1; i < 10; i++ {
		if i%3 == 0 {
			fmt.Fprintf(os.Stderr, "STDERR i: %d\n", i)
		} else {
			fmt.Printf("STDOUT i: %d\n", i)
		}
		time.Sleep(time.Second)
	}

}
<|endoftext|>


<commit_before>package main

import (
	"code.google.com/p/weed-fs/go/glog"
	"code.google.com/p/weed-fs/go/util"
	"code.google.com/p/weed-fs/go/weed/weed_server"
	"github.com/gorilla/mux"
	"net/http"
	"os"
	"runtime"
	"strconv"
	"strings"
	"sync"
	"time"
)

var (
	filer FilerOptions
)

func init() {
	cmdServer.Run = runServer // break init cycle
}

var cmdServer = &Command{
	UsageLine: "server -port=8080 -dir=/tmp -max=5 -ip=server_name",
	Short:     "start a server, including volume server, and automatically elect a master server",
	Long: `start both a volume server to provide storage spaces
  and a master server to provide volume=>location mapping service and sequence number of file ids

  This is provided as a convenient way to start both volume server and master server.
  The servers are exactly the same as starting them separately.

  So other volume servers can use this embedded master server also.

  Optionally, one filer server can be started. Logically, filer servers should not be in a cluster.
  They run with meta data on disk, not shared. So each filer server is different.

  `,
}

var (
	serverIp                      = cmdServer.Flag.String("ip", "", "ip or server name")
	serverPublicIp                = cmdServer.Flag.String("publicIp", "", "ip or server name")
	serverMaxCpu                  = cmdServer.Flag.Int("maxCpu", 0, "maximum number of CPUs. 0 means all available CPUs")
	serverTimeout                 = cmdServer.Flag.Int("idleTimeout", 10, "connection idle seconds")
	serverDataCenter              = cmdServer.Flag.String("dataCenter", "", "current volume server's data center name")
	serverRack                    = cmdServer.Flag.String("rack", "", "current volume server's rack name")
	serverWhiteListOption         = cmdServer.Flag.String("whiteList", "", "comma separated Ip addresses having write permission. No limit if empty.")
	serverPeers                   = cmdServer.Flag.String("master.peers", "", "other master nodes in comma separated ip:masterPort list")
	masterPort                    = cmdServer.Flag.Int("master.port", 9333, "master server http listen port")
	masterMetaFolder              = cmdServer.Flag.String("master.dir", "", "data directory to store meta data, default to same as -dir specified")
	masterVolumeSizeLimitMB       = cmdServer.Flag.Uint("master.volumeSizeLimitMB", 30*1000, "Master stops directing writes to oversized volumes.")
	masterConfFile                = cmdServer.Flag.String("master.conf", "/etc/weedfs/weedfs.conf", "xml configuration file")
	masterDefaultReplicaPlacement = cmdServer.Flag.String("master.defaultReplicaPlacement", "000", "Default replication type if not specified.")
	volumePort                    = cmdServer.Flag.Int("volume.port", 8080, "volume server http listen port")
	volumeDataFolders             = cmdServer.Flag.String("dir", os.TempDir(), "directories to store data files. dir[,dir]...")
	volumeMaxDataVolumeCounts     = cmdServer.Flag.String("volume.max", "7", "maximum numbers of volumes, count[,count]...")
	volumePulse                   = cmdServer.Flag.Int("pulseSeconds", 5, "number of seconds between heartbeats")
	isStartingFiler               = cmdServer.Flag.Bool("filer", false, "whether to start filer")

	serverWhiteList []string
)

func init() {
	filer.master = cmdServer.Flag.String("filer.master", "", "default to current master server")
	filer.collection = cmdServer.Flag.String("filer.collection", "", "all data will be stored in this collection")
	filer.port = cmdServer.Flag.Int("filer.port", 8888, "filer server http listen port")
	filer.dir = cmdServer.Flag.String("filer.dir", "", "directory to store meta data, default to a 'filer' sub directory of what -mdir is specified")
	filer.defaultReplicaPlacement = cmdServer.Flag.String("filer.defaultReplicaPlacement", "", "Default replication type if not specified during runtime.")
}

func runServer(cmd *Command, args []string) bool {

	if *serverPublicIp == "" {
		if *serverIp == "" {
			*serverPublicIp = "localhost"
		} else {
			*serverPublicIp = *serverIp
		}
	}

	*filer.master = *serverPublicIp + ":" + strconv.Itoa(*masterPort)

	if *filer.defaultReplicaPlacement == "" {
		*filer.defaultReplicaPlacement = *masterDefaultReplicaPlacement
	}

	if *serverMaxCpu < 1 {
		*serverMaxCpu = runtime.NumCPU()
	}
	runtime.GOMAXPROCS(*serverMaxCpu)

	folders := strings.Split(*volumeDataFolders, ",")
	maxCountStrings := strings.Split(*volumeMaxDataVolumeCounts, ",")
	maxCounts := make([]int, 0)
	for _, maxString := range maxCountStrings {
		if max, e := strconv.Atoi(maxString); e == nil {
			maxCounts = append(maxCounts, max)
		} else {
			glog.Fatalf("The max specified in -max not a valid number %s", maxString)
		}
	}
	if len(folders) != len(maxCounts) {
		glog.Fatalf("%d directories by -dir, but only %d max is set by -max", len(folders), len(maxCounts))
	}
	for _, folder := range folders {
		if err := util.TestFolderWritable(folder); err != nil {
			glog.Fatalf("Check Data Folder(-dir) Writable %s : %s", folder, err)
		}
	}

	if *masterMetaFolder == "" {
		*masterMetaFolder = folders[0]
	}
	if *filer.dir == "" {
		*filer.dir = *masterMetaFolder + "/filer"
		os.MkdirAll(*filer.dir, 0700)
	}
	if err := util.TestFolderWritable(*masterMetaFolder); err != nil {
		glog.Fatalf("Check Meta Folder (-mdir=\"%s\") Writable: %s", *masterMetaFolder, err)
	}
	if err := util.TestFolderWritable(*filer.dir); err != nil {
		glog.Fatalf("Check Mapping Meta Folder (-filer.dir=\"%s\") Writable: %s", *filer.dir, err)
	}

	if *serverWhiteListOption != "" {
		serverWhiteList = strings.Split(*serverWhiteListOption, ",")
	}

	if *isStartingFiler {
		go func() {
			r := http.NewServeMux()
			_, nfs_err := weed_server.NewFilerServer(r, *filer.port, *filer.master, *filer.dir, *filer.collection)
			if nfs_err != nil {
				glog.Fatalf(nfs_err.Error())
			}
			glog.V(0).Infoln("Start Weed Filer", util.VERSION, "at port", strconv.Itoa(*filer.port))
			filerListener, e := util.NewListener(
				":"+strconv.Itoa(*filer.port),
				time.Duration(10)*time.Second,
			)
			if e != nil {
				glog.Fatalf(e.Error())
			}
			if e := http.Serve(filerListener, r); e != nil {
				glog.Fatalf("Filer Fail to serve:%s", e.Error())
			}
		}()
	}

	var raftWaitForMaster sync.WaitGroup
	var volumeWait sync.WaitGroup

	raftWaitForMaster.Add(1)
	volumeWait.Add(1)

	go func() {
		r := mux.NewRouter()
		ms := weed_server.NewMasterServer(r, *masterPort, *masterMetaFolder,
			*masterVolumeSizeLimitMB, *volumePulse, *masterConfFile, *masterDefaultReplicaPlacement, *garbageThreshold, serverWhiteList,
		)

		glog.V(0).Infoln("Start Weed Master", util.VERSION, "at", *serverIp+":"+strconv.Itoa(*masterPort))
		masterListener, e := util.NewListener(*serverIp+":"+strconv.Itoa(*masterPort), time.Duration(*serverTimeout)*time.Second)
		if e != nil {
			glog.Fatalf(e.Error())
		}

		go func() {
			raftWaitForMaster.Wait()
			time.Sleep(100 * time.Millisecond)
			myAddress := *serverPublicIp + ":" + strconv.Itoa(*masterPort)
			var peers []string
			if *serverPeers != "" {
				peers = strings.Split(*serverPeers, ",")
			}
			raftServer := weed_server.NewRaftServer(r, peers, myAddress, *masterMetaFolder, ms.Topo, *volumePulse)
			ms.SetRaftServer(raftServer)
			volumeWait.Done()
		}()

		raftWaitForMaster.Done()
		if e := http.Serve(masterListener, r); e != nil {
			glog.Fatalf("Master Fail to serve:%s", e.Error())
		}
	}()

	volumeWait.Wait()
	time.Sleep(100 * time.Millisecond)
	r := http.NewServeMux()
	weed_server.NewVolumeServer(r, *serverIp, *volumePort, *serverPublicIp, folders, maxCounts,
		*serverIp+":"+strconv.Itoa(*masterPort), *volumePulse, *serverDataCenter, *serverRack, serverWhiteList,
	)

	glog.V(0).Infoln("Start Weed volume server", util.VERSION, "at", *serverIp+":"+strconv.Itoa(*volumePort))
	volumeListener, e := util.NewListener(
		*serverIp+":"+strconv.Itoa(*volumePort),
		time.Duration(*serverTimeout)*time.Second,
	)
	if e != nil {
		glog.Fatalf(e.Error())
	}
	if e := http.Serve(volumeListener, r); e != nil {
		glog.Fatalf("Fail to serve:%s", e.Error())
	}

	return true
}
<commit_msg>Add cpu profiling option.<commit_after>package main

import (
	"code.google.com/p/weed-fs/go/glog"
	"code.google.com/p/weed-fs/go/util"
	"code.google.com/p/weed-fs/go/weed/weed_server"
	"github.com/gorilla/mux"
	"net/http"
	"os"
	"os/signal"
	"runtime"
	"runtime/pprof"
	"strconv"
	"strings"
	"sync"
	"time"
)

type ServerOptions struct {
	cpuprofile *string
}

var (
	filer         FilerOptions
	serverOptions ServerOptions
)

func init() {
	cmdServer.Run = runServer // break init cycle
}

var cmdServer = &Command{
	UsageLine: "server -port=8080 -dir=/tmp -max=5 -ip=server_name",
	Short:     "start a server, including volume server, and automatically elect a master server",
	Long: `start both a volume server to provide storage spaces
  and a master server to provide volume=>location mapping service and sequence number of file ids

  This is provided as a convenient way to start both volume server and master server.
  The servers are exactly the same as starting them separately.

  So other volume servers can use this embedded master server also.

  Optionally, one filer server can be started. Logically, filer servers should not be in a cluster.
  They run with meta data on disk, not shared. So each filer server is different.

  `,
}

var (
	serverIp                      = cmdServer.Flag.String("ip", "", "ip or server name")
	serverPublicIp                = cmdServer.Flag.String("publicIp", "", "ip or server name")
	serverMaxCpu                  = cmdServer.Flag.Int("maxCpu", 0, "maximum number of CPUs. 0 means all available CPUs")
	serverTimeout                 = cmdServer.Flag.Int("idleTimeout", 10, "connection idle seconds")
	serverDataCenter              = cmdServer.Flag.String("dataCenter", "", "current volume server's data center name")
	serverRack                    = cmdServer.Flag.String("rack", "", "current volume server's rack name")
	serverWhiteListOption         = cmdServer.Flag.String("whiteList", "", "comma separated Ip addresses having write permission. No limit if empty.")
	serverPeers                   = cmdServer.Flag.String("master.peers", "", "other master nodes in comma separated ip:masterPort list")
	masterPort                    = cmdServer.Flag.Int("master.port", 9333, "master server http listen port")
	masterMetaFolder              = cmdServer.Flag.String("master.dir", "", "data directory to store meta data, default to same as -dir specified")
	masterVolumeSizeLimitMB       = cmdServer.Flag.Uint("master.volumeSizeLimitMB", 30*1000, "Master stops directing writes to oversized volumes.")
	masterConfFile                = cmdServer.Flag.String("master.conf", "/etc/weedfs/weedfs.conf", "xml configuration file")
	masterDefaultReplicaPlacement = cmdServer.Flag.String("master.defaultReplicaPlacement", "000", "Default replication type if not specified.")
	volumePort                    = cmdServer.Flag.Int("volume.port", 8080, "volume server http listen port")
	volumeDataFolders             = cmdServer.Flag.String("dir", os.TempDir(), "directories to store data files. dir[,dir]...")
	volumeMaxDataVolumeCounts     = cmdServer.Flag.String("volume.max", "7", "maximum numbers of volumes, count[,count]...")
	volumePulse                   = cmdServer.Flag.Int("pulseSeconds", 5, "number of seconds between heartbeats")
	isStartingFiler               = cmdServer.Flag.Bool("filer", false, "whether to start filer")

	serverWhiteList []string
)

func init() {
	filer.master = cmdServer.Flag.String("filer.master", "", "default to current master server")
	filer.collection = cmdServer.Flag.String("filer.collection", "", "all data will be stored in this collection")
	filer.port = cmdServer.Flag.Int("filer.port", 8888, "filer server http listen port")
	filer.dir = cmdServer.Flag.String("filer.dir", "", "directory to store meta data, default to a 'filer' sub directory of what -mdir is specified")
	filer.defaultReplicaPlacement = cmdServer.Flag.String("filer.defaultReplicaPlacement", "", "Default replication type if not specified during runtime.")
	serverOptions.cpuprofile = cmdServer.Flag.String("cpuprofile", "", "write cpu profile to file")
}

func runServer(cmd *Command, args []string) bool {
	if *serverOptions.cpuprofile != "" {
		f, err := os.Create(*serverOptions.cpuprofile)
		if err != nil {
			glog.Fatal(err)
		}
		pprof.StartCPUProfile(f)
		defer pprof.StopCPUProfile()

		signalChan := make(chan os.Signal, 1)
		signal.Notify(signalChan, os.Interrupt)
		go func() {
			for _ = range signalChan {
				// sig is a ^C, handle it
				pprof.StopCPUProfile()
				os.Exit(0)
			}
		}()

	}

	if *serverPublicIp == "" {
		if *serverIp == "" {
			*serverPublicIp = "localhost"
		} else {
			*serverPublicIp = *serverIp
		}
	}

	*filer.master = *serverPublicIp + ":" + strconv.Itoa(*masterPort)

	if *filer.defaultReplicaPlacement == "" {
		*filer.defaultReplicaPlacement = *masterDefaultReplicaPlacement
	}

	if *serverMaxCpu < 1 {
		*serverMaxCpu = runtime.NumCPU()
	}
	runtime.GOMAXPROCS(*serverMaxCpu)

	folders := strings.Split(*volumeDataFolders, ",")
	maxCountStrings := strings.Split(*volumeMaxDataVolumeCounts, ",")
	maxCounts := make([]int, 0)
	for _, maxString := range maxCountStrings {
		if max, e := strconv.Atoi(maxString); e == nil {
			maxCounts = append(maxCounts, max)
		} else {
			glog.Fatalf("The max specified in -max not a valid number %s", maxString)
		}
	}
	if len(folders) != len(maxCounts) {
		glog.Fatalf("%d directories by -dir, but only %d max is set by -max", len(folders), len(maxCounts))
	}
	for _, folder := range folders {
		if err := util.TestFolderWritable(folder); err != nil {
			glog.Fatalf("Check Data Folder(-dir) Writable %s : %s", folder, err)
		}
	}

	if *masterMetaFolder == "" {
		*masterMetaFolder = folders[0]
	}
	if *filer.dir == "" {
		*filer.dir = *masterMetaFolder + "/filer"
		os.MkdirAll(*filer.dir, 0700)
	}
	if err := util.TestFolderWritable(*masterMetaFolder); err != nil {
		glog.Fatalf("Check Meta Folder (-mdir=\"%s\") Writable: %s", *masterMetaFolder, err)
	}
	if err := util.TestFolderWritable(*filer.dir); err != nil {
		glog.Fatalf("Check Mapping Meta Folder (-filer.dir=\"%s\") Writable: %s", *filer.dir, err)
	}

	if *serverWhiteListOption != "" {
		serverWhiteList = strings.Split(*serverWhiteListOption, ",")
	}

	if *isStartingFiler {
		go func() {
			r := http.NewServeMux()
			_, nfs_err := weed_server.NewFilerServer(r, *filer.port, *filer.master, *filer.dir, *filer.collection)
			if nfs_err != nil {
				glog.Fatalf(nfs_err.Error())
			}
			glog.V(0).Infoln("Start Weed Filer", util.VERSION, "at port", strconv.Itoa(*filer.port))
			filerListener, e := util.NewListener(
				":"+strconv.Itoa(*filer.port),
				time.Duration(10)*time.Second,
			)
			if e != nil {
				glog.Fatalf(e.Error())
			}
			if e := http.Serve(filerListener, r); e != nil {
				glog.Fatalf("Filer Fail to serve:%s", e.Error())
			}
		}()
	}

	var raftWaitForMaster sync.WaitGroup
	var volumeWait sync.WaitGroup

	raftWaitForMaster.Add(1)
	volumeWait.Add(1)

	go func() {
		r := mux.NewRouter()
		ms := weed_server.NewMasterServer(r, *masterPort, *masterMetaFolder,
			*masterVolumeSizeLimitMB, *volumePulse, *masterConfFile, *masterDefaultReplicaPlacement, *garbageThreshold, serverWhiteList,
		)

		glog.V(0).Infoln("Start Weed Master", util.VERSION, "at", *serverIp+":"+strconv.Itoa(*masterPort))
		masterListener, e := util.NewListener(*serverIp+":"+strconv.Itoa(*masterPort), time.Duration(*serverTimeout)*time.Second)
		if e != nil {
			glog.Fatalf(e.Error())
		}

		go func() {
			raftWaitForMaster.Wait()
			time.Sleep(100 * time.Millisecond)
			myAddress := *serverPublicIp + ":" + strconv.Itoa(*masterPort)
			var peers []string
			if *serverPeers != "" {
				peers = strings.Split(*serverPeers, ",")
			}
			raftServer := weed_server.NewRaftServer(r, peers, myAddress, *masterMetaFolder, ms.Topo, *volumePulse)
			ms.SetRaftServer(raftServer)
			volumeWait.Done()
		}()

		raftWaitForMaster.Done()
		if e := http.Serve(masterListener, r); e != nil {
			glog.Fatalf("Master Fail to serve:%s", e.Error())
		}
	}()

	volumeWait.Wait()
	time.Sleep(100 * time.Millisecond)
	r := http.NewServeMux()
	weed_server.NewVolumeServer(r, *serverIp, *volumePort, *serverPublicIp, folders, maxCounts,
		*serverIp+":"+strconv.Itoa(*masterPort), *volumePulse, *serverDataCenter, *serverRack, serverWhiteList,
	)

	glog.V(0).Infoln("Start Weed volume server", util.VERSION, "at", *serverIp+":"+strconv.Itoa(*volumePort))
	volumeListener, e := util.NewListener(
		*serverIp+":"+strconv.Itoa(*volumePort),
		time.Duration(*serverTimeout)*time.Second,
	)
	if e != nil {
		glog.Fatalf(e.Error())
	}
	if e := http.Serve(volumeListener, r); e != nil {
		glog.Fatalf("Fail to serve:%s", e.Error())
	}

	return true
}
<|endoftext|>