<commit_before>/*
Copyright 2018 The OpenEBS Authors.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package volume

import (
	"bytes"
	"fmt"
	"strconv"
	"time"

	"github.com/golang/glog"
	apis "github.com/openebs/maya/pkg/apis/openebs.io/v1alpha1"
	"github.com/openebs/maya/pkg/util"
)

// VolumeOperator is the name of the tool that makes volume-related operations.
const (
	VolumeOperator   = "iscsi"
	IstgtConfPath    = "/usr/local/etc/istgt/istgt.conf"
	IstgtStatusCmd   = "STATUS"
	IstgtRefreshCmd  = "REFRESH"
	WaitTimeForIscsi = 3 * time.Second
)

//FileOperatorVar is used for doing File Operations
var FileOperatorVar util.FileOperator

//UnixSockVar is used for communication through Unix Socket
var UnixSockVar util.UnixSock

// CreateVolume creates a new cStor volume istgt config.
func CreateVolume(cStorVolume *apis.CStorVolume) error {
	// create conf file
	text := CreateIstgtConf(cStorVolume)
	err := FileOperatorVar.Write(IstgtConfPath, text, 0644)
	if err != nil {
		glog.Errorf("Failed to write istgt.conf")
	}
	glog.Info("Done writing istgt.conf")

	// send refresh command to istgt and read the response
	_, err = UnixSockVar.SendCommand(IstgtRefreshCmd)
	if err != nil {
		glog.Info("Failed to refresh iscsi service with new configuration.")
	}
	glog.Info("Creating Iscsi Volume Successful")
	return nil

}

// CreateIstgtConf creates istgt.conf file
func CreateIstgtConf(cStorVolume *apis.CStorVolume) []byte {

	var buffer bytes.Buffer
	buffer.WriteString(`# Global section
[Global]
`)
	buffer.WriteString("  NodeBase \"" + cStorVolume.Spec.NodeBase + "\"")
	buffer.WriteString(`
  PidFile "/var/run/istgt.pid"
  AuthFile "/usr/local/etc/istgt/auth.conf"
  LogFile "/usr/local/etc/istgt/logfile"
  Luworkers 6
  MediaDirectory "/mnt"
  Timeout 60
  NopInInterval 20
  MaxR2T 16
  DiscoveryAuthMethod None
  DiscoveryAuthGroup None
  MaxSessions 32
  MaxConnections 4
  FirstBurstLength 262144
  MaxBurstLength 1048576
  MaxRecvDataSegmentLength 262144
  MaxOutstandingR2T 16
  DefaultTime2Wait 2
  DefaultTime2Retain 20
  OperationalMode 0

# UnitControl section
[UnitControl]
  AuthMethod None
  AuthGroup None
`)
	buffer.WriteString("  Portal UC1 " + cStorVolume.Spec.TargetIP + ":3261\n")
	buffer.WriteString("  Netmask " + cStorVolume.Spec.TargetIP + "/8\n")
	buffer.WriteString(`
# PortalGroup section
[PortalGroup1]
`)
	buffer.WriteString("  Portal DA1 " + cStorVolume.Spec.TargetIP + ":3260\n")
	buffer.WriteString(`
# InitiatorGroup section
[InitiatorGroup1]
  InitiatorName "ALL"
  Netmask "ALL"

[InitiatorGroup2]
  InitiatorName "None"
  Netmask "None"

# LogicalUnit section
[LogicalUnit1]
`)
	buffer.WriteString("  TargetName " + cStorVolume.Name + "\n")
	buffer.WriteString("  TargetAlias nicknamefor-" + cStorVolume.Name)
	buffer.WriteString(`
  Mapping PortalGroup1 InitiatorGroup1
  AuthMethod None
  AuthGroup None
  UseDigest Auto
  ReadOnly No
`)
	buffer.WriteString("  ReplicationFactor " + strconv.Itoa(cStorVolume.Spec.ReplicationFactor) + "\n")
	buffer.WriteString("  ConsistencyFactor " + strconv.Itoa(cStorVolume.Spec.ConsistencyFactor))
	buffer.WriteString(`
  UnitType Disk
  UnitOnline Yes
  BlockLength 512
  QueueDepth 32
  Luworkers 1
`)
	buffer.WriteString("  UnitInquiry \"OpenEBS\" \"iscsi\" \"0\" \"" + string(cStorVolume.UID) + "\"")
	buffer.WriteString(`
  PhysRecordLength 4096
`)
	buffer.WriteString("  LUN0 Storage " + cStorVolume.Spec.Capacity + " 32k")
	buffer.WriteString(`
  LUN0 Option Unmap Disable
  LUN0 Option WZero Disable
  LUN0 Option ATS Disable
  LUN0 Option XCOPY Disable
`)

	return buffer.Bytes()
}

// CheckValidVolume checks for validity of CStorVolume resource.
func CheckValidVolume(cStorVolume *apis.CStorVolume) error {
	if len(string(cStorVolume.ObjectMeta.UID)) == 0 {
		return fmt.Errorf("Invalid volume resource")
	}
	if len(string(cStorVolume.Spec.TargetIP)) == 0 {
		return fmt.Errorf("targetIP cannot be empty")
	}
	if len(string(cStorVolume.Name)) == 0 {
		return fmt.Errorf("volumeName cannot be empty")
	}
	if len(string(cStorVolume.UID)) == 0 {
		return fmt.Errorf("volumeID cannot be empty")
	}
	if len(string(cStorVolume.Spec.Capacity)) == 0 {
		return fmt.Errorf("capacity cannot be empty")
	}
	if cStorVolume.Spec.ReplicationFactor == 0 {
		return fmt.Errorf("replicationFactor cannot be zero")
	}
	if cStorVolume.Spec.ConsistencyFactor == 0 {
		return fmt.Errorf("consistencyFactor cannot be zero")
	}
	if cStorVolume.Spec.ReplicationFactor < cStorVolume.Spec.ConsistencyFactor {
		return fmt.Errorf("replicationFactor cannot be less than consistencyFactor")
	}

	return nil
}

// CheckForIscsi is blocking call for checking status of istgt in cstor-istgt container.
func CheckForIscsi() {
	for {
		_, err := UnixSockVar.SendCommand(IstgtStatusCmd)
		if err != nil {
			time.Sleep(WaitTimeForIscsi)
			glog.Warningf("Waiting for istgt... err : %v", err)
			continue
		}
		break
	}
}
<commit_msg>modified luworkers count to 6 in lun section. this was missed in the last PR. Only global section luworkers were made to 6 in the earlier PR.<commit_after>/*
Copyright 2018 The OpenEBS Authors.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package volume

import (
	"bytes"
	"fmt"
	"strconv"
	"time"

	"github.com/golang/glog"
	apis "github.com/openebs/maya/pkg/apis/openebs.io/v1alpha1"
	"github.com/openebs/maya/pkg/util"
)

// VolumeOperator is the name of the tool that makes volume-related operations.
const (
	VolumeOperator   = "iscsi"
	IstgtConfPath    = "/usr/local/etc/istgt/istgt.conf"
	IstgtStatusCmd   = "STATUS"
	IstgtRefreshCmd  = "REFRESH"
	WaitTimeForIscsi = 3 * time.Second
)

//FileOperatorVar is used for doing File Operations
var FileOperatorVar util.FileOperator

//UnixSockVar is used for communication through Unix Socket
var UnixSockVar util.UnixSock

// CreateVolume creates a new cStor volume istgt config.
func CreateVolume(cStorVolume *apis.CStorVolume) error {
	// create conf file
	text := CreateIstgtConf(cStorVolume)
	err := FileOperatorVar.Write(IstgtConfPath, text, 0644)
	if err != nil {
		glog.Errorf("Failed to write istgt.conf")
	}
	glog.Info("Done writing istgt.conf")

	// send refresh command to istgt and read the response
	_, err = UnixSockVar.SendCommand(IstgtRefreshCmd)
	if err != nil {
		glog.Info("Failed to refresh iscsi service with new configuration.")
	}
	glog.Info("Creating Iscsi Volume Successful")
	return nil

}

// CreateIstgtConf creates istgt.conf file
func CreateIstgtConf(cStorVolume *apis.CStorVolume) []byte {

	var buffer bytes.Buffer
	buffer.WriteString(`# Global section
[Global]
`)
	buffer.WriteString("  NodeBase \"" + cStorVolume.Spec.NodeBase + "\"")
	buffer.WriteString(`
  PidFile "/var/run/istgt.pid"
  AuthFile "/usr/local/etc/istgt/auth.conf"
  LogFile "/usr/local/etc/istgt/logfile"
  Luworkers 6
  MediaDirectory "/mnt"
  Timeout 60
  NopInInterval 20
  MaxR2T 16
  DiscoveryAuthMethod None
  DiscoveryAuthGroup None
  MaxSessions 32
  MaxConnections 4
  FirstBurstLength 262144
  MaxBurstLength 1048576
  MaxRecvDataSegmentLength 262144
  MaxOutstandingR2T 16
  DefaultTime2Wait 2
  DefaultTime2Retain 20
  OperationalMode 0

# UnitControl section
[UnitControl]
  AuthMethod None
  AuthGroup None
`)
	buffer.WriteString("  Portal UC1 " + cStorVolume.Spec.TargetIP + ":3261\n")
	buffer.WriteString("  Netmask " + cStorVolume.Spec.TargetIP + "/8\n")
	buffer.WriteString(`
# PortalGroup section
[PortalGroup1]
`)
	buffer.WriteString("  Portal DA1 " + cStorVolume.Spec.TargetIP + ":3260\n")
	buffer.WriteString(`
# InitiatorGroup section
[InitiatorGroup1]
  InitiatorName "ALL"
  Netmask "ALL"

[InitiatorGroup2]
  InitiatorName "None"
  Netmask "None"

# LogicalUnit section
[LogicalUnit1]
`)
	buffer.WriteString("  TargetName " + cStorVolume.Name + "\n")
	buffer.WriteString("  TargetAlias nicknamefor-" + cStorVolume.Name)
	buffer.WriteString(`
  Mapping PortalGroup1 InitiatorGroup1
  AuthMethod None
  AuthGroup None
  UseDigest Auto
  ReadOnly No
`)
	buffer.WriteString("  ReplicationFactor " + strconv.Itoa(cStorVolume.Spec.ReplicationFactor) + "\n")
	buffer.WriteString("  ConsistencyFactor " + strconv.Itoa(cStorVolume.Spec.ConsistencyFactor))
	buffer.WriteString(`
  UnitType Disk
  UnitOnline Yes
  BlockLength 512
  QueueDepth 32
  Luworkers 6
`)
	buffer.WriteString("  UnitInquiry \"OpenEBS\" \"iscsi\" \"0\" \"" + string(cStorVolume.UID) + "\"")
	buffer.WriteString(`
  PhysRecordLength 4096
`)
	buffer.WriteString("  LUN0 Storage " + cStorVolume.Spec.Capacity + " 32k")
	buffer.WriteString(`
  LUN0 Option Unmap Disable
  LUN0 Option WZero Disable
  LUN0 Option ATS Disable
  LUN0 Option XCOPY Disable
`)

	return buffer.Bytes()
}

// CheckValidVolume checks for validity of CStorVolume resource.
func CheckValidVolume(cStorVolume *apis.CStorVolume) error {
	if len(string(cStorVolume.ObjectMeta.UID)) == 0 {
		return fmt.Errorf("Invalid volume resource")
	}
	if len(string(cStorVolume.Spec.TargetIP)) == 0 {
		return fmt.Errorf("targetIP cannot be empty")
	}
	if len(string(cStorVolume.Name)) == 0 {
		return fmt.Errorf("volumeName cannot be empty")
	}
	if len(string(cStorVolume.UID)) == 0 {
		return fmt.Errorf("volumeID cannot be empty")
	}
	if len(string(cStorVolume.Spec.Capacity)) == 0 {
		return fmt.Errorf("capacity cannot be empty")
	}
	if cStorVolume.Spec.ReplicationFactor == 0 {
		return fmt.Errorf("replicationFactor cannot be zero")
	}
	if cStorVolume.Spec.ConsistencyFactor == 0 {
		return fmt.Errorf("consistencyFactor cannot be zero")
	}
	if cStorVolume.Spec.ReplicationFactor < cStorVolume.Spec.ConsistencyFactor {
		return fmt.Errorf("replicationFactor cannot be less than consistencyFactor")
	}

	return nil
}

// CheckForIscsi is blocking call for checking status of istgt in cstor-istgt container.
func CheckForIscsi() {
	for {
		_, err := UnixSockVar.SendCommand(IstgtStatusCmd)
		if err != nil {
			time.Sleep(WaitTimeForIscsi)
			glog.Warningf("Waiting for istgt... err : %v", err)
			continue
		}
		break
	}
}
<|endoftext|>


<commit_before>/*
Copyright 2014 The Kubernetes Authors.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

// Package app does all of the work necessary to create a Kubernetes
// APIServer by binding together the API, master and APIServer infrastructure.
// It can be configured and called directly or via the hyperkube framework.
package app

import (
	"strings"
	"time"

	"github.com/golang/glog"
	"github.com/spf13/cobra"
	"github.com/spf13/pflag"

	"k8s.io/kubernetes/federation/cmd/federation-apiserver/app/options"
	"k8s.io/kubernetes/pkg/admission"
	"k8s.io/kubernetes/pkg/api"
	"k8s.io/kubernetes/pkg/api/unversioned"
	"k8s.io/kubernetes/pkg/apis/rbac"
	"k8s.io/kubernetes/pkg/apiserver/authenticator"
	"k8s.io/kubernetes/pkg/controller/framework/informers"
	"k8s.io/kubernetes/pkg/genericapiserver"
	"k8s.io/kubernetes/pkg/genericapiserver/authorizer"
	genericoptions "k8s.io/kubernetes/pkg/genericapiserver/options"
	genericvalidation "k8s.io/kubernetes/pkg/genericapiserver/validation"
	"k8s.io/kubernetes/pkg/registry/cachesize"
	"k8s.io/kubernetes/pkg/registry/clusterrole"
	clusterroleetcd "k8s.io/kubernetes/pkg/registry/clusterrole/etcd"
	"k8s.io/kubernetes/pkg/registry/clusterrolebinding"
	clusterrolebindingetcd "k8s.io/kubernetes/pkg/registry/clusterrolebinding/etcd"
	"k8s.io/kubernetes/pkg/registry/generic"
	"k8s.io/kubernetes/pkg/registry/role"
	roleetcd "k8s.io/kubernetes/pkg/registry/role/etcd"
	"k8s.io/kubernetes/pkg/registry/rolebinding"
	rolebindingetcd "k8s.io/kubernetes/pkg/registry/rolebinding/etcd"
	"k8s.io/kubernetes/pkg/util/wait"
)

// NewAPIServerCommand creates a *cobra.Command object with default parameters
func NewAPIServerCommand() *cobra.Command {
	s := options.NewServerRunOptions()
	s.AddFlags(pflag.CommandLine)
	cmd := &cobra.Command{
		Use: "federation-apiserver",
		Long: `The Kubernetes federation API server validates and configures data
for the api objects which include pods, services, replicationcontrollers, and
others. The API Server services REST operations and provides the frontend to the
cluster's shared state through which all other components interact.`,
		Run: func(cmd *cobra.Command, args []string) {
		},
	}
	return cmd
}

// Run runs the specified APIServer.  This should never exit.
func Run(s *options.ServerRunOptions) error {
	genericvalidation.VerifyEtcdServersList(s.ServerRunOptions)
	genericapiserver.DefaultAndValidateRunOptions(s.ServerRunOptions)

	// TODO: register cluster federation resources here.
	resourceConfig := genericapiserver.NewResourceConfig()

	storageGroupsToEncodingVersion, err := s.StorageGroupsToEncodingVersion()
	if err != nil {
		glog.Fatalf("error generating storage version map: %s", err)
	}
	storageFactory, err := genericapiserver.BuildDefaultStorageFactory(
		s.StorageConfig, s.DefaultStorageMediaType, api.Codecs,
		genericapiserver.NewDefaultResourceEncodingConfig(), storageGroupsToEncodingVersion,
		[]unversioned.GroupVersionResource{}, resourceConfig, s.RuntimeConfig)
	if err != nil {
		glog.Fatalf("error in initializing storage factory: %s", err)
	}

	for _, override := range s.EtcdServersOverrides {
		tokens := strings.Split(override, "#")
		if len(tokens) != 2 {
			glog.Errorf("invalid value of etcd server overrides: %s", override)
			continue
		}

		apiresource := strings.Split(tokens[0], "/")
		if len(apiresource) != 2 {
			glog.Errorf("invalid resource definition: %s", tokens[0])
			continue
		}
		group := apiresource[0]
		resource := apiresource[1]
		groupResource := unversioned.GroupResource{Group: group, Resource: resource}

		servers := strings.Split(tokens[1], ";")
		storageFactory.SetEtcdLocation(groupResource, servers)
	}

	authenticator, err := authenticator.New(authenticator.AuthenticatorConfig{
		BasicAuthFile:     s.BasicAuthFile,
		ClientCAFile:      s.ClientCAFile,
		TokenAuthFile:     s.TokenAuthFile,
		OIDCIssuerURL:     s.OIDCIssuerURL,
		OIDCClientID:      s.OIDCClientID,
		OIDCCAFile:        s.OIDCCAFile,
		OIDCUsernameClaim: s.OIDCUsernameClaim,
		OIDCGroupsClaim:   s.OIDCGroupsClaim,
		KeystoneURL:       s.KeystoneURL,
	})
	if err != nil {
		glog.Fatalf("Invalid Authentication Config: %v", err)
	}

	authorizationModeNames := strings.Split(s.AuthorizationMode, ",")

	modeEnabled := func(mode string) bool {
		for _, m := range authorizationModeNames {
			if m == mode {
				return true
			}
		}
		return false
	}

	authorizationConfig := authorizer.AuthorizationConfig{
		PolicyFile:                  s.AuthorizationPolicyFile,
		WebhookConfigFile:           s.AuthorizationWebhookConfigFile,
		WebhookCacheAuthorizedTTL:   s.AuthorizationWebhookCacheAuthorizedTTL,
		WebhookCacheUnauthorizedTTL: s.AuthorizationWebhookCacheUnauthorizedTTL,
		RBACSuperUser:               s.AuthorizationRBACSuperUser,
	}
	if modeEnabled(genericoptions.ModeRBAC) {
		mustGetRESTOptions := func(resource string) generic.RESTOptions {
			config, err := storageFactory.NewConfig(rbac.Resource(resource))
			if err != nil {
				glog.Fatalf("Unable to get %s storage: %v", resource, err)
			}
			return generic.RESTOptions{StorageConfig: config, Decorator: generic.UndecoratedStorage, ResourcePrefix: storageFactory.ResourcePrefix(rbac.Resource(resource))}
		}

		// For initial bootstrapping go directly to etcd to avoid privillege escalation check.
		authorizationConfig.RBACRoleRegistry = role.NewRegistry(roleetcd.NewREST(mustGetRESTOptions("roles")))
		authorizationConfig.RBACRoleBindingRegistry = rolebinding.NewRegistry(rolebindingetcd.NewREST(mustGetRESTOptions("rolebindings")))
		authorizationConfig.RBACClusterRoleRegistry = clusterrole.NewRegistry(clusterroleetcd.NewREST(mustGetRESTOptions("clusterroles")))
		authorizationConfig.RBACClusterRoleBindingRegistry = clusterrolebinding.NewRegistry(clusterrolebindingetcd.NewREST(mustGetRESTOptions("clusterrolebindings")))
	}

	authorizer, err := authorizer.NewAuthorizerFromAuthorizationConfig(authorizationModeNames, authorizationConfig)
	if err != nil {
		glog.Fatalf("Invalid Authorization Config: %v", err)
	}

	admissionControlPluginNames := strings.Split(s.AdmissionControl, ",")
	client, err := s.NewSelfClient()
	if err != nil {
		glog.Errorf("Failed to create clientset: %v", err)
	}
	sharedInformers := informers.NewSharedInformerFactory(client, 10*time.Minute)
	pluginInitializer := admission.NewPluginInitializer(sharedInformers)

	admissionController, err := admission.NewFromPlugins(client, admissionControlPluginNames, s.AdmissionControlConfigFile, pluginInitializer)
	if err != nil {
		glog.Fatalf("Failed to initialize plugins: %v", err)
	}
	genericConfig := genericapiserver.NewConfig(s.ServerRunOptions)
	// TODO: Move the following to generic api server as well.
	genericConfig.StorageFactory = storageFactory
	genericConfig.Authenticator = authenticator
	genericConfig.SupportsBasicAuth = len(s.BasicAuthFile) > 0
	genericConfig.Authorizer = authorizer
	genericConfig.AuthorizerRBACSuperUser = s.AuthorizationRBACSuperUser
	genericConfig.AdmissionControl = admissionController
	genericConfig.APIResourceConfigSource = storageFactory.APIResourceConfigSource
	genericConfig.MasterServiceNamespace = s.MasterServiceNamespace
	genericConfig.Serializer = api.Codecs

	// TODO: Move this to generic api server (Need to move the command line flag).
	if s.EnableWatchCache {
		cachesize.InitializeWatchCacheSizes(s.TargetRAMMB)
		cachesize.SetWatchCacheSizes(s.WatchCacheSizes)
	}

	m, err := genericConfig.New()
	if err != nil {
		return err
	}

	installFederationAPIs(s, m, storageFactory)
	installCoreAPIs(s, m, storageFactory)
	installExtensionsAPIs(s, m, storageFactory)

	sharedInformers.Start(wait.NeverStop)
	m.Run(s.ServerRunOptions)
	return nil
}

func createRESTOptionsOrDie(s *options.ServerRunOptions, g *genericapiserver.GenericAPIServer, f genericapiserver.StorageFactory, resource unversioned.GroupResource) generic.RESTOptions {
	config, err := f.NewConfig(resource)
	if err != nil {
		glog.Fatalf("Unable to find storage config for %v, due to %v", resource, err.Error())
	}
	return generic.RESTOptions{
		StorageConfig:           config,
		Decorator:               g.StorageDecorator(),
		DeleteCollectionWorkers: s.DeleteCollectionWorkers,
		ResourcePrefix:          f.ResourcePrefix(resource),
	}
}
<commit_msg>Cleanup non-rest apiserver handlers<commit_after>/*
Copyright 2014 The Kubernetes Authors.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

// Package app does all of the work necessary to create a Kubernetes
// APIServer by binding together the API, master and APIServer infrastructure.
// It can be configured and called directly or via the hyperkube framework.
package app

import (
	"strings"
	"time"

	"github.com/golang/glog"
	"github.com/spf13/cobra"
	"github.com/spf13/pflag"

	"k8s.io/kubernetes/federation/cmd/federation-apiserver/app/options"
	"k8s.io/kubernetes/pkg/admission"
	"k8s.io/kubernetes/pkg/api"
	"k8s.io/kubernetes/pkg/api/unversioned"
	"k8s.io/kubernetes/pkg/apis/rbac"
	"k8s.io/kubernetes/pkg/apiserver/authenticator"
	"k8s.io/kubernetes/pkg/controller/framework/informers"
	"k8s.io/kubernetes/pkg/genericapiserver"
	"k8s.io/kubernetes/pkg/genericapiserver/authorizer"
	genericoptions "k8s.io/kubernetes/pkg/genericapiserver/options"
	genericvalidation "k8s.io/kubernetes/pkg/genericapiserver/validation"
	"k8s.io/kubernetes/pkg/registry/cachesize"
	"k8s.io/kubernetes/pkg/registry/clusterrole"
	clusterroleetcd "k8s.io/kubernetes/pkg/registry/clusterrole/etcd"
	"k8s.io/kubernetes/pkg/registry/clusterrolebinding"
	clusterrolebindingetcd "k8s.io/kubernetes/pkg/registry/clusterrolebinding/etcd"
	"k8s.io/kubernetes/pkg/registry/generic"
	"k8s.io/kubernetes/pkg/registry/role"
	roleetcd "k8s.io/kubernetes/pkg/registry/role/etcd"
	"k8s.io/kubernetes/pkg/registry/rolebinding"
	rolebindingetcd "k8s.io/kubernetes/pkg/registry/rolebinding/etcd"
	"k8s.io/kubernetes/pkg/routes"
	"k8s.io/kubernetes/pkg/util/wait"
)

// NewAPIServerCommand creates a *cobra.Command object with default parameters
func NewAPIServerCommand() *cobra.Command {
	s := options.NewServerRunOptions()
	s.AddFlags(pflag.CommandLine)
	cmd := &cobra.Command{
		Use: "federation-apiserver",
		Long: `The Kubernetes federation API server validates and configures data
for the api objects which include pods, services, replicationcontrollers, and
others. The API Server services REST operations and provides the frontend to the
cluster's shared state through which all other components interact.`,
		Run: func(cmd *cobra.Command, args []string) {
		},
	}
	return cmd
}

// Run runs the specified APIServer.  This should never exit.
func Run(s *options.ServerRunOptions) error {
	genericvalidation.VerifyEtcdServersList(s.ServerRunOptions)
	genericapiserver.DefaultAndValidateRunOptions(s.ServerRunOptions)

	// TODO: register cluster federation resources here.
	resourceConfig := genericapiserver.NewResourceConfig()

	storageGroupsToEncodingVersion, err := s.StorageGroupsToEncodingVersion()
	if err != nil {
		glog.Fatalf("error generating storage version map: %s", err)
	}
	storageFactory, err := genericapiserver.BuildDefaultStorageFactory(
		s.StorageConfig, s.DefaultStorageMediaType, api.Codecs,
		genericapiserver.NewDefaultResourceEncodingConfig(), storageGroupsToEncodingVersion,
		[]unversioned.GroupVersionResource{}, resourceConfig, s.RuntimeConfig)
	if err != nil {
		glog.Fatalf("error in initializing storage factory: %s", err)
	}

	for _, override := range s.EtcdServersOverrides {
		tokens := strings.Split(override, "#")
		if len(tokens) != 2 {
			glog.Errorf("invalid value of etcd server overrides: %s", override)
			continue
		}

		apiresource := strings.Split(tokens[0], "/")
		if len(apiresource) != 2 {
			glog.Errorf("invalid resource definition: %s", tokens[0])
			continue
		}
		group := apiresource[0]
		resource := apiresource[1]
		groupResource := unversioned.GroupResource{Group: group, Resource: resource}

		servers := strings.Split(tokens[1], ";")
		storageFactory.SetEtcdLocation(groupResource, servers)
	}

	authenticator, err := authenticator.New(authenticator.AuthenticatorConfig{
		BasicAuthFile:     s.BasicAuthFile,
		ClientCAFile:      s.ClientCAFile,
		TokenAuthFile:     s.TokenAuthFile,
		OIDCIssuerURL:     s.OIDCIssuerURL,
		OIDCClientID:      s.OIDCClientID,
		OIDCCAFile:        s.OIDCCAFile,
		OIDCUsernameClaim: s.OIDCUsernameClaim,
		OIDCGroupsClaim:   s.OIDCGroupsClaim,
		KeystoneURL:       s.KeystoneURL,
	})
	if err != nil {
		glog.Fatalf("Invalid Authentication Config: %v", err)
	}

	authorizationModeNames := strings.Split(s.AuthorizationMode, ",")

	modeEnabled := func(mode string) bool {
		for _, m := range authorizationModeNames {
			if m == mode {
				return true
			}
		}
		return false
	}

	authorizationConfig := authorizer.AuthorizationConfig{
		PolicyFile:                  s.AuthorizationPolicyFile,
		WebhookConfigFile:           s.AuthorizationWebhookConfigFile,
		WebhookCacheAuthorizedTTL:   s.AuthorizationWebhookCacheAuthorizedTTL,
		WebhookCacheUnauthorizedTTL: s.AuthorizationWebhookCacheUnauthorizedTTL,
		RBACSuperUser:               s.AuthorizationRBACSuperUser,
	}
	if modeEnabled(genericoptions.ModeRBAC) {
		mustGetRESTOptions := func(resource string) generic.RESTOptions {
			config, err := storageFactory.NewConfig(rbac.Resource(resource))
			if err != nil {
				glog.Fatalf("Unable to get %s storage: %v", resource, err)
			}
			return generic.RESTOptions{StorageConfig: config, Decorator: generic.UndecoratedStorage, ResourcePrefix: storageFactory.ResourcePrefix(rbac.Resource(resource))}
		}

		// For initial bootstrapping go directly to etcd to avoid privillege escalation check.
		authorizationConfig.RBACRoleRegistry = role.NewRegistry(roleetcd.NewREST(mustGetRESTOptions("roles")))
		authorizationConfig.RBACRoleBindingRegistry = rolebinding.NewRegistry(rolebindingetcd.NewREST(mustGetRESTOptions("rolebindings")))
		authorizationConfig.RBACClusterRoleRegistry = clusterrole.NewRegistry(clusterroleetcd.NewREST(mustGetRESTOptions("clusterroles")))
		authorizationConfig.RBACClusterRoleBindingRegistry = clusterrolebinding.NewRegistry(clusterrolebindingetcd.NewREST(mustGetRESTOptions("clusterrolebindings")))
	}

	authorizer, err := authorizer.NewAuthorizerFromAuthorizationConfig(authorizationModeNames, authorizationConfig)
	if err != nil {
		glog.Fatalf("Invalid Authorization Config: %v", err)
	}

	admissionControlPluginNames := strings.Split(s.AdmissionControl, ",")
	client, err := s.NewSelfClient()
	if err != nil {
		glog.Errorf("Failed to create clientset: %v", err)
	}
	sharedInformers := informers.NewSharedInformerFactory(client, 10*time.Minute)
	pluginInitializer := admission.NewPluginInitializer(sharedInformers)

	admissionController, err := admission.NewFromPlugins(client, admissionControlPluginNames, s.AdmissionControlConfigFile, pluginInitializer)
	if err != nil {
		glog.Fatalf("Failed to initialize plugins: %v", err)
	}
	genericConfig := genericapiserver.NewConfig(s.ServerRunOptions)
	// TODO: Move the following to generic api server as well.
	genericConfig.StorageFactory = storageFactory
	genericConfig.Authenticator = authenticator
	genericConfig.SupportsBasicAuth = len(s.BasicAuthFile) > 0
	genericConfig.Authorizer = authorizer
	genericConfig.AuthorizerRBACSuperUser = s.AuthorizationRBACSuperUser
	genericConfig.AdmissionControl = admissionController
	genericConfig.APIResourceConfigSource = storageFactory.APIResourceConfigSource
	genericConfig.MasterServiceNamespace = s.MasterServiceNamespace
	genericConfig.Serializer = api.Codecs

	// TODO: Move this to generic api server (Need to move the command line flag).
	if s.EnableWatchCache {
		cachesize.InitializeWatchCacheSizes(s.TargetRAMMB)
		cachesize.SetWatchCacheSizes(s.WatchCacheSizes)
	}

	m, err := genericConfig.New()
	if err != nil {
		return err
	}

	routes.UIRedirect{}.Install(m.Mux, m.HandlerContainer)
	routes.Logs{}.Install(m.Mux, m.HandlerContainer)

	installFederationAPIs(s, m, storageFactory)
	installCoreAPIs(s, m, storageFactory)
	installExtensionsAPIs(s, m, storageFactory)

	sharedInformers.Start(wait.NeverStop)
	m.Run(s.ServerRunOptions)
	return nil
}

func createRESTOptionsOrDie(s *options.ServerRunOptions, g *genericapiserver.GenericAPIServer, f genericapiserver.StorageFactory, resource unversioned.GroupResource) generic.RESTOptions {
	config, err := f.NewConfig(resource)
	if err != nil {
		glog.Fatalf("Unable to find storage config for %v, due to %v", resource, err.Error())
	}
	return generic.RESTOptions{
		StorageConfig:           config,
		Decorator:               g.StorageDecorator(),
		DeleteCollectionWorkers: s.DeleteCollectionWorkers,
		ResourcePrefix:          f.ResourcePrefix(resource),
	}
}
<|endoftext|>


<commit_before>// Copyright (c) Ilia Kravets, 2015. All rights reserved. PROVIDED "AS IS"
// WITHOUT ANY WARRANTY, EXPRESS OR IMPLIED. See LICENSE file for details.

package rec

import (
	"fmt"
	"io"
	"time"

	"github.com/ikravets/errs"

	"my/ev/packet"
	"my/ev/packet/bats"
	"my/ev/packet/miax"
	"my/ev/packet/nasdaq"
	"my/ev/sim"
)

type EfhLoggerPrinter interface {
	PrintMessage(efhMessage) error
}

type EfhLoggerOutputMode byte

const (
	EfhLoggerOutputOrders EfhLoggerOutputMode = iota
	EfhLoggerOutputQuotes
)

type EfhLoggerConfig struct {
	Writer  io.Writer
	Printer EfhLoggerPrinter
	Mode    EfhLoggerOutputMode
}

type EfhLogger struct {
	tobLogger TobLogger
	printer   EfhLoggerPrinter
	mode      EfhLoggerOutputMode
	stream    Stream
}

var _ sim.Observer = &EfhLogger{}

func NewEfhLogger(c EfhLoggerConfig) *EfhLogger {
	l := &EfhLogger{
		tobLogger: *NewTobLogger(),
		printer:   c.Printer,
		mode:      c.Mode,
		stream:    *NewStream(),
	}
	if l.printer == nil {
		errs.Check(c.Writer != nil)
		l.printer = NewTestefhPrinter(c.Writer)
	}
	if l.mode == EfhLoggerOutputQuotes {
		l.tobLogger.SetAfterBookUpdateFlags(TobUpdateBothSides)
	}
	return l
}

func (l *EfhLogger) MessageArrived(idm *sim.SimMessage) {
	l.stream.MessageArrived(idm)
	l.tobLogger.MessageArrived(idm)
	switch m := l.stream.getExchangeMessage().(type) {
	case packet.TradeMessage:
		l.genUpdateTrades(m)
	case *nasdaq.IttoMessageOptionDirectory:
		l.genUpdateDefinitionsNom(m)
	case *bats.PitchMessageSymbolMapping:
		l.genUpdateDefinitionsBats(m)
	case *miax.TomMessageSeriesUpdate:
		l.genUpdateDefinitionsMiax(m)
	}
}
func (l *EfhLogger) OperationAppliedToOrders(operation sim.SimOperation) {
	l.tobLogger.OperationAppliedToOrders(operation)
}
func (l *EfhLogger) BeforeBookUpdate(book sim.Book, operation sim.SimOperation) {
	l.tobLogger.BeforeBookUpdate(book, operation)
}
func (l *EfhLogger) AfterBookUpdate(book sim.Book, operation sim.SimOperation) {
	if !l.tobLogger.AfterBookUpdate(book, operation) {
		return
	}
	if l.mode == EfhLoggerOutputOrders {
		l.genUpdateOrders(l.tobLogger.bid)
		l.genUpdateOrders(l.tobLogger.ask)
	} else {
		l.genUpdateQuotes(l.tobLogger.bid, l.tobLogger.ask)
	}
}

func (l *EfhLogger) genUpdateHeaderForOption(messageType uint8, oid packet.OptionId) efhm_header {
	return efhm_header{
		Type:           messageType,
		SecurityId:     oid.ToUint64(),
		SequenceNumber: l.stream.getSeqNum(),
		TimeStamp:      l.stream.getTimestamp(),
	}
}
func (l *EfhLogger) genUpdateHeader(messageType uint8) efhm_header {
	return l.genUpdateHeaderForOption(messageType, l.tobLogger.lastOptionId)
}
func (l *EfhLogger) genUpdateOrders(tob tob) {
	if !tob.updated() {
		return
	}
	m := efhm_order{
		efhm_header:     l.genUpdateHeader(EFHM_ORDER),
		Price:           uint32(tob.New.Price()),
		Size:            uint32(tob.New.Size(sim.SizeKindDefault)),
		AoNSize:         uint32(tob.New.Size(sim.SizeKindAON)),
		CustomerSize:    uint32(tob.New.Size(sim.SizeKindCustomer)),
		CustomerAoNSize: uint32(tob.New.Size(sim.SizeKindCustomerAON)),
		OrderType:       1,
	}
	switch tob.Side {
	case packet.MarketSideBid:
		m.OrderSide = EFH_ORDER_BID
	case packet.MarketSideAsk:
		m.OrderSide = EFH_ORDER_ASK
	}
	errs.CheckE(l.printer.PrintMessage(m))
}
func (l *EfhLogger) genUpdateQuotes(bid, ask tob) {
	m := efhm_quote{
		efhm_header:        l.genUpdateHeader(EFHM_QUOTE),
		BidPrice:           uint32(bid.New.Price()),
		BidSize:            uint32(bid.New.Size(sim.SizeKindDefault)),
		BidAoNSize:         uint32(bid.New.Size(sim.SizeKindAON)),
		BidCustomerSize:    uint32(bid.New.Size(sim.SizeKindCustomer)),
		BidCustomerAoNSize: uint32(bid.New.Size(sim.SizeKindCustomerAON)),
		AskPrice:           uint32(ask.New.Price()),
		AskSize:            uint32(ask.New.Size(sim.SizeKindDefault)),
		AskAoNSize:         uint32(ask.New.Size(sim.SizeKindAON)),
		AskCustomerSize:    uint32(ask.New.Size(sim.SizeKindCustomer)),
		AskCustomerAoNSize: uint32(ask.New.Size(sim.SizeKindCustomerAON)),
	}
	errs.CheckE(l.printer.PrintMessage(m))
}
func (l *EfhLogger) genUpdateTrades(msg packet.TradeMessage) {
	oid, price, size := msg.TradeInfo()
	m := efhm_trade{
		efhm_header: l.genUpdateHeaderForOption(EFHM_TRADE, oid),
		Price:       uint32(packet.PriceTo4Dec(price)),
		Size:        uint32(size),
	}
	errs.CheckE(l.printer.PrintMessage(m))
}
func (l *EfhLogger) genUpdateDefinitionsNom(msg *nasdaq.IttoMessageOptionDirectory) {
	m := efhm_definition_nom{
		efhm_header: l.genUpdateHeaderForOption(EFHM_DEFINITION_NOM, msg.OptionId()),
		StrikePrice: uint32(msg.StrikePrice),
	}
	year, month, day := msg.Expiration.Date()
	m.MaturityDate = uint64(day<<16 + int(month)<<8 + year%100)
	copy(m.Symbol[:], msg.Symbol)
	copy(m.UnderlyingSymbol[:], msg.UnderlyingSymbol)
	switch msg.OType {
	case 'C':
		m.PutOrCall = EFH_SECURITY_CALL
	case 'P':
		m.PutOrCall = EFH_SECURITY_PUT
	}
	errs.CheckE(l.printer.PrintMessage(m))
}

// FIXME boilerplate code left until there's guarantee that output for MIAX should have exactly the same fields as for NASDAQ
func (l *EfhLogger) genUpdateDefinitionsMiax(msg *miax.TomMessageSeriesUpdate) {
	m := efhm_definition_nom{
		efhm_header: l.genUpdateHeaderForOption(EFHM_DEFINITION_MIAX, msg.OptionId()),
		StrikePrice: uint32(msg.StrikePrice),
	}
	t, ok := time.Parse("20060102", msg.Expiration)
	errs.CheckE(ok)
	year, month, day := t.Date()
	m.MaturityDate = uint64(day<<16 + int(month)<<8 + year%100)
	copy(m.Symbol[:], msg.SecuritySymbol)
	copy(m.UnderlyingSymbol[:], msg.UnderlyingSymbol)
	switch msg.CallOrPut {
	case 'C':
		m.PutOrCall = EFH_SECURITY_CALL
	case 'P':
		m.PutOrCall = EFH_SECURITY_PUT
	}
	errs.CheckE(l.printer.PrintMessage(m))
}
func (l *EfhLogger) genUpdateDefinitionsBats(msg *bats.PitchMessageSymbolMapping) {
	m := efhm_definition_bats{
		efhm_header: l.genUpdateHeaderForOption(EFHM_DEFINITION_BATS, msg.OptionId()),
	}
	m.efhm_header.SequenceNumber = 0
	m.efhm_header.TimeStamp = 0
	copy(m.OsiSymbol[:], msg.OsiSymbol)
	errs.CheckE(l.printer.PrintMessage(m))
}

type testefhPrinter struct {
	w io.Writer
}

var _ EfhLoggerPrinter = &testefhPrinter{}

func NewTestefhPrinter(w io.Writer) EfhLoggerPrinter {
	return &testefhPrinter{w: w}
}
func (p *testefhPrinter) PrintMessage(m efhMessage) error {
	_, err := fmt.Fprintln(p.w, m)
	return err
}
<commit_msg>rec:EfhLogger: add EfhLoggerConfig.AssumeTobUpdate<commit_after>// Copyright (c) Ilia Kravets, 2015. All rights reserved. PROVIDED "AS IS"
// WITHOUT ANY WARRANTY, EXPRESS OR IMPLIED. See LICENSE file for details.

package rec

import (
	"fmt"
	"io"
	"time"

	"github.com/ikravets/errs"

	"my/ev/packet"
	"my/ev/packet/bats"
	"my/ev/packet/miax"
	"my/ev/packet/nasdaq"
	"my/ev/sim"
)

type EfhLoggerPrinter interface {
	PrintMessage(efhMessage) error
}

type EfhLoggerOutputMode byte

const (
	EfhLoggerOutputOrders EfhLoggerOutputMode = iota
	EfhLoggerOutputQuotes
)

type EfhLoggerConfig struct {
	Writer          io.Writer
	Printer         EfhLoggerPrinter
	Mode            EfhLoggerOutputMode
	AssumeTobUpdate bool
}

type EfhLogger struct {
	tobLogger TobLogger
	printer   EfhLoggerPrinter
	mode      EfhLoggerOutputMode
	stream    Stream
}

var _ sim.Observer = &EfhLogger{}

func NewEfhLogger(c EfhLoggerConfig) *EfhLogger {
	l := &EfhLogger{
		tobLogger: *NewTobLogger(),
		printer:   c.Printer,
		mode:      c.Mode,
		stream:    *NewStream(),
	}
	if l.printer == nil {
		errs.Check(c.Writer != nil)
		l.printer = NewTestefhPrinter(c.Writer)
	}
	var abuFlags TobUpdate
	if l.mode == EfhLoggerOutputQuotes {
		abuFlags |= TobUpdateBothSides
	}
	if c.AssumeTobUpdate {
		abuFlags |= TobUpdateAssumeUpdated
	}
	l.tobLogger.SetAfterBookUpdateFlags(abuFlags)
	return l
}

func (l *EfhLogger) MessageArrived(idm *sim.SimMessage) {
	l.stream.MessageArrived(idm)
	l.tobLogger.MessageArrived(idm)
	switch m := l.stream.getExchangeMessage().(type) {
	case packet.TradeMessage:
		l.genUpdateTrades(m)
	case *nasdaq.IttoMessageOptionDirectory:
		l.genUpdateDefinitionsNom(m)
	case *bats.PitchMessageSymbolMapping:
		l.genUpdateDefinitionsBats(m)
	case *miax.TomMessageSeriesUpdate:
		l.genUpdateDefinitionsMiax(m)
	}
}
func (l *EfhLogger) OperationAppliedToOrders(operation sim.SimOperation) {
	l.tobLogger.OperationAppliedToOrders(operation)
}
func (l *EfhLogger) BeforeBookUpdate(book sim.Book, operation sim.SimOperation) {
	l.tobLogger.BeforeBookUpdate(book, operation)
}
func (l *EfhLogger) AfterBookUpdate(book sim.Book, operation sim.SimOperation) {
	if !l.tobLogger.AfterBookUpdate(book, operation) {
		return
	}
	if l.mode == EfhLoggerOutputOrders {
		l.genUpdateOrders(l.tobLogger.bid)
		l.genUpdateOrders(l.tobLogger.ask)
	} else {
		l.genUpdateQuotes(l.tobLogger.bid, l.tobLogger.ask)
	}
}

func (l *EfhLogger) genUpdateHeaderForOption(messageType uint8, oid packet.OptionId) efhm_header {
	return efhm_header{
		Type:           messageType,
		SecurityId:     oid.ToUint64(),
		SequenceNumber: l.stream.getSeqNum(),
		TimeStamp:      l.stream.getTimestamp(),
	}
}
func (l *EfhLogger) genUpdateHeader(messageType uint8) efhm_header {
	return l.genUpdateHeaderForOption(messageType, l.tobLogger.lastOptionId)
}
func (l *EfhLogger) genUpdateOrders(tob tob) {
	if !tob.updated() {
		return
	}
	m := efhm_order{
		efhm_header:     l.genUpdateHeader(EFHM_ORDER),
		Price:           uint32(tob.New.Price()),
		Size:            uint32(tob.New.Size(sim.SizeKindDefault)),
		AoNSize:         uint32(tob.New.Size(sim.SizeKindAON)),
		CustomerSize:    uint32(tob.New.Size(sim.SizeKindCustomer)),
		CustomerAoNSize: uint32(tob.New.Size(sim.SizeKindCustomerAON)),
		OrderType:       1,
	}
	switch tob.Side {
	case packet.MarketSideBid:
		m.OrderSide = EFH_ORDER_BID
	case packet.MarketSideAsk:
		m.OrderSide = EFH_ORDER_ASK
	}
	errs.CheckE(l.printer.PrintMessage(m))
}
func (l *EfhLogger) genUpdateQuotes(bid, ask tob) {
	m := efhm_quote{
		efhm_header:        l.genUpdateHeader(EFHM_QUOTE),
		BidPrice:           uint32(bid.New.Price()),
		BidSize:            uint32(bid.New.Size(sim.SizeKindDefault)),
		BidAoNSize:         uint32(bid.New.Size(sim.SizeKindAON)),
		BidCustomerSize:    uint32(bid.New.Size(sim.SizeKindCustomer)),
		BidCustomerAoNSize: uint32(bid.New.Size(sim.SizeKindCustomerAON)),
		AskPrice:           uint32(ask.New.Price()),
		AskSize:            uint32(ask.New.Size(sim.SizeKindDefault)),
		AskAoNSize:         uint32(ask.New.Size(sim.SizeKindAON)),
		AskCustomerSize:    uint32(ask.New.Size(sim.SizeKindCustomer)),
		AskCustomerAoNSize: uint32(ask.New.Size(sim.SizeKindCustomerAON)),
	}
	errs.CheckE(l.printer.PrintMessage(m))
}
func (l *EfhLogger) genUpdateTrades(msg packet.TradeMessage) {
	oid, price, size := msg.TradeInfo()
	m := efhm_trade{
		efhm_header: l.genUpdateHeaderForOption(EFHM_TRADE, oid),
		Price:       uint32(packet.PriceTo4Dec(price)),
		Size:        uint32(size),
	}
	errs.CheckE(l.printer.PrintMessage(m))
}
func (l *EfhLogger) genUpdateDefinitionsNom(msg *nasdaq.IttoMessageOptionDirectory) {
	m := efhm_definition_nom{
		efhm_header: l.genUpdateHeaderForOption(EFHM_DEFINITION_NOM, msg.OptionId()),
		StrikePrice: uint32(msg.StrikePrice),
	}
	year, month, day := msg.Expiration.Date()
	m.MaturityDate = uint64(day<<16 + int(month)<<8 + year%100)
	copy(m.Symbol[:], msg.Symbol)
	copy(m.UnderlyingSymbol[:], msg.UnderlyingSymbol)
	switch msg.OType {
	case 'C':
		m.PutOrCall = EFH_SECURITY_CALL
	case 'P':
		m.PutOrCall = EFH_SECURITY_PUT
	}
	errs.CheckE(l.printer.PrintMessage(m))
}

// FIXME boilerplate code left until there's guarantee that output for MIAX should have exactly the same fields as for NASDAQ
func (l *EfhLogger) genUpdateDefinitionsMiax(msg *miax.TomMessageSeriesUpdate) {
	m := efhm_definition_nom{
		efhm_header: l.genUpdateHeaderForOption(EFHM_DEFINITION_MIAX, msg.OptionId()),
		StrikePrice: uint32(msg.StrikePrice),
	}
	t, ok := time.Parse("20060102", msg.Expiration)
	errs.CheckE(ok)
	year, month, day := t.Date()
	m.MaturityDate = uint64(day<<16 + int(month)<<8 + year%100)
	copy(m.Symbol[:], msg.SecuritySymbol)
	copy(m.UnderlyingSymbol[:], msg.UnderlyingSymbol)
	switch msg.CallOrPut {
	case 'C':
		m.PutOrCall = EFH_SECURITY_CALL
	case 'P':
		m.PutOrCall = EFH_SECURITY_PUT
	}
	errs.CheckE(l.printer.PrintMessage(m))
}
func (l *EfhLogger) genUpdateDefinitionsBats(msg *bats.PitchMessageSymbolMapping) {
	m := efhm_definition_bats{
		efhm_header: l.genUpdateHeaderForOption(EFHM_DEFINITION_BATS, msg.OptionId()),
	}
	m.efhm_header.SequenceNumber = 0
	m.efhm_header.TimeStamp = 0
	copy(m.OsiSymbol[:], msg.OsiSymbol)
	errs.CheckE(l.printer.PrintMessage(m))
}

type testefhPrinter struct {
	w io.Writer
}

var _ EfhLoggerPrinter = &testefhPrinter{}

func NewTestefhPrinter(w io.Writer) EfhLoggerPrinter {
	return &testefhPrinter{w: w}
}
func (p *testefhPrinter) PrintMessage(m efhMessage) error {
	_, err := fmt.Fprintln(p.w, m)
	return err
}
<|endoftext|>


<commit_before>package main

import (
	"errors"
	"flag"
	"fmt"
	"io"
	"net/http"
	"os"
	"path/filepath"
	"sort"
	"strconv"
	"strings"
	"sync"

	"github.com/kisielk/whisper-go/whisper"
	"github.com/raintank/dur"
	"github.com/raintank/metrictank/api"
	"github.com/raintank/metrictank/mdata"
	"github.com/raintank/metrictank/mdata/chunk"
	"github.com/raintank/metrictank/mdata/chunk/archive"
	"gopkg.in/raintank/schema.v1"
)

var (
	exitOnError = flag.Bool(
		"exit-on-error",
		true,
		"Exit with a message when there's an error",
	)
	verbose = flag.Bool(
		"verbose",
		false,
		"Write logs to terminal",
	)
	httpEndpoint = flag.String(
		"http-endpoint",
		"http://127.0.0.1:8080/chunks",
		"The http endpoint to send the data to",
	)
	chunkSpanStr = flag.String(
		"chunkspans",
		"10min",
		"List of chunk spans separated by ':'. The 1st whisper archive gets the 1st span, 2nd the 2nd, etc",
	)
	namePrefix = flag.String(
		"name-prefix",
		"",
		"Prefix to prepend before every metric name, should include the '.' if necessary",
	)
	threads = flag.Int(
		"threads",
		10,
		"Number of workers threads to process .wsp files",
	)
	skipUnfinishedChunks = flag.Bool(
		"skip-unfinished-chunks",
		true,
		"Chunks that have not completed their chunk span should be skipped",
	)
	orgId = flag.Int(
		"orgid",
		1,
		"Organization ID the data belongs to ",
	)
	whisperDirectory = flag.String(
		"whisper-directory",
		"/opt/graphite/storage/whisper",
		"The directory that contains the whisper file structure",
	)
	readArchivesStr = flag.String(
		"read-archives",
		"*",
		"Comma separated list of positive integers or '*' for all archives",
	)
	chunkSpans   []uint32
	readArchives map[int]struct{}
	printLock    sync.Mutex
)

func main() {
	flag.Parse()

	for _, chunkSpanStrSplit := range strings.Split(*chunkSpanStr, ",") {
		chunkSpan := dur.MustParseUNsec("chunkspan", chunkSpanStrSplit)

		if (mdata.Month_sec % chunkSpan) != 0 {
			panic("chunkSpan must fit without remainders into month_sec (28*24*60*60)")
		}

		_, ok := chunk.RevChunkSpans[chunkSpan]
		if !ok {
			panic(fmt.Sprintf("chunkSpan %d is not a valid value (https://github.com/raintank/metrictank/blob/master/docs/memory-server.md#valid-chunk-spans)", chunkSpan))
		}

		chunkSpans = append(chunkSpans, chunkSpan)
	}

	if *readArchivesStr != "*" {
		readArchives = make(map[int]struct{})
		for _, archiveIdStr := range strings.Split(*readArchivesStr, ",") {
			archiveId, err := strconv.Atoi(archiveIdStr)
			if err != nil {
				panic(fmt.Sprintf("Invalid archive id %q: %q", archiveIdStr, err))
			}
			readArchives[archiveId] = struct{}{}
		}
	}

	fileChan := make(chan string)

	wg := &sync.WaitGroup{}
	wg.Add(*threads)
	for i := 0; i < *threads; i++ {
		go processFromChan(fileChan, wg)
	}

	getFileListIntoChan(fileChan)
	wg.Wait()
}

func throwError(msg string) {
	msg = fmt.Sprintf("%s\n", msg)
	if *exitOnError {
		panic(msg)
	} else {
		printLock.Lock()
		fmt.Fprintln(os.Stderr, msg)
		printLock.Unlock()
	}
}

func log(msg string) {
	if *verbose {
		printLock.Lock()
		fmt.Println(msg)
		printLock.Unlock()
	}
}

func processFromChan(files chan string, wg *sync.WaitGroup) {
	client := &http.Client{}

	for file := range files {
		fd, err := os.Open(file)
		if err != nil {
			throwError(fmt.Sprintf("ERROR: Failed to open whisper file '%q': %q\n", file, err))
			continue
		}
		w, err := whisper.OpenWhisper(fd)
		if err != nil {
			throwError(fmt.Sprintf("ERROR: Failed to open whisper file '%q': %q\n", file, err))
			continue
		}

		log(fmt.Sprintf("Processing file %q", file))
		met, err := getMetric(w, file)
		if err != nil {
			throwError(fmt.Sprintf("Failed to get metric: %q", err))
			continue
		}

		b, err := met.MarshalCompressed()
		if err != nil {
			throwError(fmt.Sprintf("Failed to encode metric: %q", err))
			continue
		}

		req, err := http.NewRequest("POST", *httpEndpoint, io.Reader(b))
		if err != nil {
			panic(fmt.Sprintf("Cannot construct request to http endpoint %q: %q", *httpEndpoint, err))
		}
		req.Header.Set("Content-Type", "application/json")
		req.Header.Set("Content-Encoding", "gzip")

		_, err = client.Do(req)
		if err != nil {
			throwError(fmt.Sprintf("Error sending request to http endpoint %q: %q", *httpEndpoint, err))
			continue
		}
	}
	wg.Done()
}

// generate the metric name based on the file name and given prefix
func getMetricName(file string) string {
	// remove all leading '/' from file name
	for file[0] == '/' {
		file = file[1:]
	}

	return *namePrefix + strings.Replace(strings.TrimSuffix(file, ".wsp"), "/", ".", -1)
}

// pointSorter sorts points by timestamp
type pointSorter []whisper.Point

func (a pointSorter) Len() int           { return len(a) }
func (a pointSorter) Swap(i, j int)      { a[i], a[j] = a[j], a[i] }
func (a pointSorter) Less(i, j int) bool { return a[i].Timestamp < a[j].Timestamp }

// the whisper archives are organized like a ringbuffer. since we need to
// insert the points into the chunks in order we first need to sort them
func sortPoints(points pointSorter) pointSorter {
	sort.Sort(points)
	return points
}

func getMetric(w *whisper.Whisper, file string) (*archive.Metric, error) {
	if len(w.Header.Archives) == 0 {
		return nil, errors.New(fmt.Sprintf("ERROR: Whisper file contains no archives: %q", file))
	}

	archives := make([]archive.Archive, 0, len(w.Header.Archives))
	var chunkSpan uint32
	var rowKey string
	name := getMetricName(file)

	// md gets generated from the first archive in the whisper file
	md := getMetricData(name, int(w.Header.Archives[0].SecondsPerPoint))

	for archiveIdx, archiveInfo := range w.Header.Archives {
		if archiveIdx > 0 {
			rowKey = api.AggMetricKey(
				md.Id,
				whisper.AggregationMethod(w.Header.Metadata.AggregationMethod).String(),
				archiveInfo.SecondsPerPoint,
			)
		} else {
			rowKey = md.Id
		}

		// only read archive if archiveIdx is in readArchives
		if _, ok := readArchives[archiveIdx]; !ok && len(readArchives) > 0 {
			continue
		}

		encodedChunks := make([]chunk.IterGen, 0)

		if len(chunkSpans)-1 < archiveIdx {
			// if we have more archives than chunk spans are specified, we simply use the last one
			chunkSpan = chunkSpans[len(chunkSpans)-1]
		} else {
			chunkSpan = chunkSpans[archiveIdx]
		}

		points, err := w.DumpArchive(archiveIdx)
		if err != nil {
			return nil, errors.New(fmt.Sprintf("ERROR: Failed to read archive %d in %q, skipping: %q", archiveIdx, file, err))
		}

		var point whisper.Point
		var t0, prevT0 uint32
		var c *chunk.Chunk

		for _, point = range sortPoints(points) {
			if point.Timestamp == 0 {
				continue
			}

			t0 = point.Timestamp - (point.Timestamp % chunkSpan)
			if prevT0 == 0 {
				c = chunk.New(t0)
				prevT0 = t0
			} else if prevT0 != t0 {
				log(fmt.Sprintf("Mark chunk at t0 %d as finished", prevT0))
				c.Finish()

				encodedChunks = append(encodedChunks, *chunk.NewBareIterGen(c.Bytes(), c.T0, chunkSpan))

				log(fmt.Sprintf("Create new chunk at t0 %d with chunk span %d", t0, chunkSpan))
				c = chunk.New(t0)
				prevT0 = t0
			}

			err := c.Push(point.Timestamp, point.Value)
			if err != nil {
				return nil, errors.New(fmt.Sprintf("ERROR: Failed to push value into chunk at t0 %d: %q", t0, err))
			}
		}

		if int64(point.Timestamp) > md.Time {
			md.Time = int64(point.Timestamp)
		}

		// if the last written point was also the last one of the current chunk,
		// or if skipUnfinishedChunks is on, we close the chunk and
		if point.Timestamp == t0+chunkSpan-archiveInfo.SecondsPerPoint || !*skipUnfinishedChunks {
			log(fmt.Sprintf("Mark current (last) chunk at t0 %d as finished", t0))
			c.Finish()
			encodedChunks = append(encodedChunks, *chunk.NewBareIterGen(c.Bytes(), c.T0, chunkSpan))
		}

		log(fmt.Sprintf("Whisper file %q archive %d (%q) gets %d chunks", file, archiveIdx, name, len(encodedChunks)))
		archives = append(archives, archive.Archive{
			SecondsPerPoint: archiveInfo.SecondsPerPoint,
			Points:          archiveInfo.Points,
			Chunks:          encodedChunks,
			RowKey:          rowKey,
		})
	}

	return &archive.Metric{
		AggregationMethod: uint32(w.Header.Metadata.AggregationMethod),
		MetricData:        *md,
		Archives:          archives,
	}, nil
}

func getMetricData(name string, interval int) *schema.MetricData {
	md := &schema.MetricData{
		Name:     name,
		Metric:   name,
		Interval: interval,
		Value:    0,
		Unit:     "unknown",
		Time:     0,
		Mtype:    "gauge",
		Tags:     []string{},
		OrgId:    *orgId,
	}
	md.SetId()
	return md
}

// scan a directory and feed the list of whisper files relative to base into the given channel
func getFileListIntoChan(fileChan chan string) {
	filepath.Walk(
		*whisperDirectory,
		func(path string, info os.FileInfo, err error) error {
			if path[len(path)-4:] == ".wsp" {
				fileChan <- path
			}
			return nil
		},
	)

	close(fileChan)
}
<commit_msg>enforce proper stringification of agg method<commit_after>package main

import (
	"errors"
	"flag"
	"fmt"
	"io"
	"net/http"
	"os"
	"path/filepath"
	"sort"
	"strconv"
	"strings"
	"sync"

	"github.com/kisielk/whisper-go/whisper"
	"github.com/raintank/dur"
	"github.com/raintank/metrictank/api"
	"github.com/raintank/metrictank/mdata"
	"github.com/raintank/metrictank/mdata/chunk"
	"github.com/raintank/metrictank/mdata/chunk/archive"
	"gopkg.in/raintank/schema.v1"
)

var (
	exitOnError = flag.Bool(
		"exit-on-error",
		true,
		"Exit with a message when there's an error",
	)
	verbose = flag.Bool(
		"verbose",
		false,
		"Write logs to terminal",
	)
	httpEndpoint = flag.String(
		"http-endpoint",
		"http://127.0.0.1:8080/chunks",
		"The http endpoint to send the data to",
	)
	chunkSpanStr = flag.String(
		"chunkspans",
		"10min",
		"List of chunk spans separated by ':'. The 1st whisper archive gets the 1st span, 2nd the 2nd, etc",
	)
	namePrefix = flag.String(
		"name-prefix",
		"",
		"Prefix to prepend before every metric name, should include the '.' if necessary",
	)
	threads = flag.Int(
		"threads",
		10,
		"Number of workers threads to process .wsp files",
	)
	skipUnfinishedChunks = flag.Bool(
		"skip-unfinished-chunks",
		true,
		"Chunks that have not completed their chunk span should be skipped",
	)
	orgId = flag.Int(
		"orgid",
		1,
		"Organization ID the data belongs to ",
	)
	whisperDirectory = flag.String(
		"whisper-directory",
		"/opt/graphite/storage/whisper",
		"The directory that contains the whisper file structure",
	)
	readArchivesStr = flag.String(
		"read-archives",
		"*",
		"Comma separated list of positive integers or '*' for all archives",
	)
	chunkSpans   []uint32
	readArchives map[int]struct{}
	printLock    sync.Mutex
)

func main() {
	flag.Parse()

	for _, chunkSpanStrSplit := range strings.Split(*chunkSpanStr, ",") {
		chunkSpan := dur.MustParseUNsec("chunkspan", chunkSpanStrSplit)

		if (mdata.Month_sec % chunkSpan) != 0 {
			panic("chunkSpan must fit without remainders into month_sec (28*24*60*60)")
		}

		_, ok := chunk.RevChunkSpans[chunkSpan]
		if !ok {
			panic(fmt.Sprintf("chunkSpan %d is not a valid value (https://github.com/raintank/metrictank/blob/master/docs/memory-server.md#valid-chunk-spans)", chunkSpan))
		}

		chunkSpans = append(chunkSpans, chunkSpan)
	}

	if *readArchivesStr != "*" {
		readArchives = make(map[int]struct{})
		for _, archiveIdStr := range strings.Split(*readArchivesStr, ",") {
			archiveId, err := strconv.Atoi(archiveIdStr)
			if err != nil {
				panic(fmt.Sprintf("Invalid archive id %q: %q", archiveIdStr, err))
			}
			readArchives[archiveId] = struct{}{}
		}
	}

	fileChan := make(chan string)

	wg := &sync.WaitGroup{}
	wg.Add(*threads)
	for i := 0; i < *threads; i++ {
		go processFromChan(fileChan, wg)
	}

	getFileListIntoChan(fileChan)
	wg.Wait()
}

func throwError(msg string) {
	msg = fmt.Sprintf("%s\n", msg)
	if *exitOnError {
		panic(msg)
	} else {
		printLock.Lock()
		fmt.Fprintln(os.Stderr, msg)
		printLock.Unlock()
	}
}

func log(msg string) {
	if *verbose {
		printLock.Lock()
		fmt.Println(msg)
		printLock.Unlock()
	}
}

func processFromChan(files chan string, wg *sync.WaitGroup) {
	client := &http.Client{}

	for file := range files {
		fd, err := os.Open(file)
		if err != nil {
			throwError(fmt.Sprintf("ERROR: Failed to open whisper file %q: %q\n", file, err))
			continue
		}
		w, err := whisper.OpenWhisper(fd)
		if err != nil {
			throwError(fmt.Sprintf("ERROR: Failed to open whisper file %q: %q\n", file, err))
			continue
		}

		log(fmt.Sprintf("Processing file %q", file))
		met, err := getMetric(w, file)
		if err != nil {
			throwError(fmt.Sprintf("Failed to get metric: %q", err))
			continue
		}

		b, err := met.MarshalCompressed()
		if err != nil {
			throwError(fmt.Sprintf("Failed to encode metric: %q", err))
			continue
		}

		req, err := http.NewRequest("POST", *httpEndpoint, io.Reader(b))
		if err != nil {
			panic(fmt.Sprintf("Cannot construct request to http endpoint %q: %q", *httpEndpoint, err))
		}
		req.Header.Set("Content-Type", "application/json")
		req.Header.Set("Content-Encoding", "gzip")

		_, err = client.Do(req)
		if err != nil {
			throwError(fmt.Sprintf("Error sending request to http endpoint %q: %q", *httpEndpoint, err))
			continue
		}
	}
	wg.Done()
}

// generate the metric name based on the file name and given prefix
func getMetricName(file string) string {
	// remove all leading '/' from file name
	for file[0] == '/' {
		file = file[1:]
	}

	return *namePrefix + strings.Replace(strings.TrimSuffix(file, ".wsp"), "/", ".", -1)
}

// pointSorter sorts points by timestamp
type pointSorter []whisper.Point

func (a pointSorter) Len() int           { return len(a) }
func (a pointSorter) Swap(i, j int)      { a[i], a[j] = a[j], a[i] }
func (a pointSorter) Less(i, j int) bool { return a[i].Timestamp < a[j].Timestamp }

// the whisper archives are organized like a ringbuffer. since we need to
// insert the points into the chunks in order we first need to sort them
func sortPoints(points pointSorter) pointSorter {
	sort.Sort(points)
	return points
}

func shortAggMethodString(aggMethod whisper.AggregationMethod) string {
	switch aggMethod {
	case whisper.AggregationAverage:
		return "avg"
	case whisper.AggregationSum:
		return "sum"
	case whisper.AggregationMin:
		return "min"
	case whisper.AggregationMax:
		return "max"
	case whisper.AggregationLast:
		return "lst"
	default:
		return ""
	}
}

func getMetric(w *whisper.Whisper, file string) (*archive.Metric, error) {
	if len(w.Header.Archives) == 0 {
		return nil, errors.New(fmt.Sprintf("ERROR: Whisper file contains no archives: %q", file))
	}

	archives := make([]archive.Archive, 0, len(w.Header.Archives))
	var chunkSpan uint32
	var rowKey string
	var aggMethodStr string
	name := getMetricName(file)

	// md gets generated from the first archive in the whisper file
	md := getMetricData(name, int(w.Header.Archives[0].SecondsPerPoint))

	for archiveIdx, archiveInfo := range w.Header.Archives {
		if archiveIdx > 0 {

			// on the first aggregation we determin the aggregation method as string
			if aggMethodStr == "" {
				aggMethodStr = shortAggMethodString(w.Header.Metadata.AggregationMethod)

				if aggMethodStr == "" {
					return nil, errors.New(fmt.Sprintf(
						"ERROR: Aggregation method in file %s not allowed: %d(%s)\n",
						file,
						w.Header.Metadata.AggregationMethod,
						whisper.AggregationMethod(w.Header.Metadata.AggregationMethod).String(),
					))
				}
			}

			rowKey = api.AggMetricKey(
				md.Id,
				aggMethodStr,
				archiveInfo.SecondsPerPoint,
			)
		} else {
			rowKey = md.Id
		}

		// only read archive if archiveIdx is in readArchives
		if _, ok := readArchives[archiveIdx]; !ok && len(readArchives) > 0 {
			continue
		}

		encodedChunks := make([]chunk.IterGen, 0)

		if len(chunkSpans)-1 < archiveIdx {
			// if we have more archives than chunk spans are specified, we simply use the last one
			chunkSpan = chunkSpans[len(chunkSpans)-1]
		} else {
			chunkSpan = chunkSpans[archiveIdx]
		}

		points, err := w.DumpArchive(archiveIdx)
		if err != nil {
			return nil, errors.New(fmt.Sprintf("ERROR: Failed to read archive %d in %q, skipping: %q", archiveIdx, file, err))
		}

		var point whisper.Point
		var t0, prevT0 uint32
		var c *chunk.Chunk

		for _, point = range sortPoints(points) {
			if point.Timestamp == 0 {
				continue
			}

			t0 = point.Timestamp - (point.Timestamp % chunkSpan)
			if prevT0 == 0 {
				c = chunk.New(t0)
				prevT0 = t0
			} else if prevT0 != t0 {
				log(fmt.Sprintf("Mark chunk at t0 %d as finished", prevT0))
				c.Finish()

				encodedChunks = append(encodedChunks, *chunk.NewBareIterGen(c.Bytes(), c.T0, chunkSpan))

				log(fmt.Sprintf("Create new chunk at t0 %d with chunk span %d", t0, chunkSpan))
				c = chunk.New(t0)
				prevT0 = t0
			}

			err := c.Push(point.Timestamp, point.Value)
			if err != nil {
				return nil, errors.New(fmt.Sprintf("ERROR: Failed to push value into chunk at t0 %d: %q", t0, err))
			}
		}

		if int64(point.Timestamp) > md.Time {
			md.Time = int64(point.Timestamp)
		}

		// if the last written point was also the last one of the current chunk,
		// or if skipUnfinishedChunks is on, we close the chunk and
		if point.Timestamp == t0+chunkSpan-archiveInfo.SecondsPerPoint || !*skipUnfinishedChunks {
			log(fmt.Sprintf("Mark current (last) chunk at t0 %d as finished", t0))
			c.Finish()
			encodedChunks = append(encodedChunks, *chunk.NewBareIterGen(c.Bytes(), c.T0, chunkSpan))
		}

		log(fmt.Sprintf("Whisper file %q archive %d (%q) gets %d chunks", file, archiveIdx, name, len(encodedChunks)))
		archives = append(archives, archive.Archive{
			SecondsPerPoint: archiveInfo.SecondsPerPoint,
			Points:          archiveInfo.Points,
			Chunks:          encodedChunks,
			RowKey:          rowKey,
		})
	}

	return &archive.Metric{
		AggregationMethod: uint32(w.Header.Metadata.AggregationMethod),
		MetricData:        *md,
		Archives:          archives,
	}, nil
}

func getMetricData(name string, interval int) *schema.MetricData {
	md := &schema.MetricData{
		Name:     name,
		Metric:   name,
		Interval: interval,
		Value:    0,
		Unit:     "unknown",
		Time:     0,
		Mtype:    "gauge",
		Tags:     []string{},
		OrgId:    *orgId,
	}
	md.SetId()
	return md
}

// scan a directory and feed the list of whisper files relative to base into the given channel
func getFileListIntoChan(fileChan chan string) {
	filepath.Walk(
		*whisperDirectory,
		func(path string, info os.FileInfo, err error) error {
			if path[len(path)-4:] == ".wsp" {
				fileChan <- path
			}
			return nil
		},
	)

	close(fileChan)
}
<|endoftext|>


<commit_before>package config

import (
	"encoding/json"
	"fmt"
	"io/ioutil"
	"os"
	"path"
	"strings"

	"github.com/crazy-max/WindowsSpyBlocker/app/bindata"
	"github.com/crazy-max/WindowsSpyBlocker/app/utils/file"
	"github.com/crazy-max/WindowsSpyBlocker/app/utils/pathu"
	"github.com/crazy-max/WindowsSpyBlocker/app/utils/print"
)

// App
var (
	AppName    = "WindowsSpyBlocker"
	AppVersion = "1.0.0"
	AppPackage = "github.com/crazy-max/WindowsSpyBlocker"
	AppUrl     = "https://" + AppPackage
)

// Config
var (
	App      conf
	Settings settings
)

// Lib structure
type Lib struct {
	Url        string `json:"url"`
	Dest       string
	OutputPath string
	Checkfile  string `json:"checkfile"`
}

// NcsiProbe structure
type NcsiProbe struct {
	WebHostV4    string `json:"webHostV4"`
	WebPathV4    string `json:"webPathV4"`
	WebContentV4 string `json:"webContentV4"`
	WebHostV6    string `json:"webHostV6"`
	WebPathV6    string `json:"webPathV6"`
	WebContentV6 string `json:"webContentV6"`
	DnsHostV4    string `json:"dnsHostV4"`
	DnsContentV4 string `json:"dnsContentV4"`
	DnsHostV6    string `json:"dnsHostV6"`
	DnsContentV6 string `json:"dnsContentV6"`
}

type dataTpl struct {
	Head  string `json:"head"`
	Value string `json:"value"`
}

type conf struct {
	Version         string `json:"version"`
	Debug           bool   `json:"debug"`
	UseEmbeddedData bool   `json:"useEmbeddedData"`
	Proxifier       struct {
		LogPath string `json:"logPath"`
	} `json:"proxifier"`
	Sysmon struct {
		EvtxPath string `json:"evtxPath"`
	} `json:"sysmon"`
	Wireshark struct {
		PcapngPath string `json:"pcapngPath"`
		Capture    struct {
			Interface int    `json:"interface"`
			Filter    string `json:"filter"`
		} `json:"capture"`
	} `json:"wireshark"`
	Exclude struct {
		Ips   []string `json:"ips"`
		Hosts []string `json:"hosts"`
		Orgs  []string `json:"orgs"`
	} `json:"exclude"`
}

type settings struct {
	Uris struct {
		LatestVersion string `json:"latestVersion"`
		Threatcrowd   string `json:"threatcrowd"`
		Whatis        string `json:"whatis"`
		Dnsquery      string `json:"dnsquery"`
		Ipapi         string `json:"ipapi"`
		Ipinfo        string `json:"ipinfo"`
		Ipnf          string `json:"ipnf"`
	} `json:"uris"`
	Libs struct {
		Wireshark Lib `json:"wireshark"`
		Npcap     Lib `json:"npcap"`
		Sysmon    Lib `json:"sysmon"`
	} `json:"libs"`
	DataTpl struct {
		Dnscrypt dataTpl `json:"dnscrypt"`
		Openwrt  struct {
			Ip      dataTpl `json:"ip"`
			Domains dataTpl `json:"domains"`
		} `json:"openwrt"`
		P2p       dataTpl `json:"p2p"`
		Proxifier struct {
			Ip      dataTpl `json:"ip"`
			Domains dataTpl `json:"domains"`
		} `json:"proxifier"`
		Simplewall dataTpl `json:"simplewall"`
	} `json:"dataTpl"`
	Proxifier struct {
		UnvalidLines []string `json:"unvalidLines"`
	} `json:"proxifier"`
	Sysmon struct {
		EvtxPath string `json:"evtxPath"`
	} `json:"sysmon"`
	Ncsi struct {
		Reg struct {
			Key               string `json:"key"`
			WebProbeHost      string `json:"webProbeHost"`
			WebProbePath      string `json:"webProbePath"`
			WebProbeContent   string `json:"webProbeContent"`
			WebProbeHostV6    string `json:"webProbeHostV6"`
			WebProbePathV6    string `json:"webProbePathV6"`
			WebProbeContentV6 string `json:"webProbeContentV6"`
			DnsProbeHost      string `json:"dnsProbeHost"`
			DnsProbeContent   string `json:"dnsProbeContent"`
			DnsProbeHostV6    string `json:"dnsProbeHostV6"`
			DnsProbeContentV6 string `json:"dnsProbeContentV6"`
		} `json:"reg"`
		Probes struct {
			Microsoft NcsiProbe `json:"microsoft"`
			Wsb       NcsiProbe `json:"wsb"`
		} `json:"probes"`
	} `json:"ncsi"`
	WilcardSubdomains []string `json:"wilcardSubdomains"`
}

func init() {
	var err error
	var old conf

	cfgPath := path.Join(pathu.Current, "app.conf")

	// Load default config
	defaultConf, err := bindata.Asset("app.conf")
	if err != nil {
		err = fmt.Errorf("Cannot load asset app.conf: %s", err.Error())
		print.QuitFatal(err)
	}
	err = json.Unmarshal(defaultConf, &App)
	if err != nil {
		err = fmt.Errorf("Cannot unmarshall defaultConf: %s", err.Error())
		print.QuitFatal(err)
	}
	newVersion := App.Version

	// Create conf if not exists
	if _, err := os.Stat(cfgPath); err != nil {
		err = ioutil.WriteFile(cfgPath, defaultConf, 0644)
		if err != nil {
			err = fmt.Errorf("Cannot write file %s: %s", strings.TrimLeft(cfgPath, pathu.Current), err.Error())
			print.QuitFatal(err)
		}
	}

	// Load current config
	raw, err := ioutil.ReadFile(cfgPath)
	if err != nil {
		err = fmt.Errorf("Cannot read %s: %s", strings.TrimLeft(cfgPath, pathu.Current), err.Error())
		print.QuitFatal(err)
	}
	err = json.Unmarshal(raw, &old)
	if err != nil {
		err = fmt.Errorf("Cannot unmarshall %s: %s", strings.TrimLeft(cfgPath, pathu.Current), err.Error())
		print.QuitFatal(err)
	}

	// Perform upgrade if different version
	if newVersion != old.Version {
		if err := performUpgrade(); err != nil {
			print.QuitFatal(err)
		}
	}

	// Merge config
	err = json.Unmarshal(raw, &App)
	if err != nil {
		err = fmt.Errorf("Cannot unmarshall %s: %s", strings.TrimLeft(cfgPath, pathu.Current), err.Error())
		print.QuitFatal(err)
	}
	App.Version = newVersion

	// Write config
	cfgJson, _ := json.MarshalIndent(App, "", "  ")
	if err != nil {
		err = fmt.Errorf("Cannot marshal config: %s", err.Error())
		print.QuitFatal(err)
	}
	err = ioutil.WriteFile(cfgPath, cfgJson, 0644)
	if err != nil {
		err = fmt.Errorf("Cannot write file %s: %s", strings.TrimLeft(cfgPath, pathu.Current), err.Error())
		print.QuitFatal(err)
	}

	// Load settings
	rawSettings, err := bindata.Asset("app/settings.json")
	if err != nil {
		err = fmt.Errorf("Cannot load asset settings.json: %s", err.Error())
		print.QuitFatal(err)
	}
	err = json.Unmarshal(rawSettings, &Settings)
	if err != nil {
		err = fmt.Errorf("Cannot unmarshall settings: %s", err.Error())
		print.QuitFatal(err)
	}
}

func performUpgrade() error {
	// Remove content of libs folder
	if err := file.RemoveContents(pathu.Libs); err != nil {
		return err
	}

	return nil
}
<commit_msg>Add ability to perform DNS AAAA lookup test on hosts<commit_after>package config

import (
	"encoding/json"
	"fmt"
	"io/ioutil"
	"os"
	"path"
	"strings"

	"github.com/crazy-max/WindowsSpyBlocker/app/bindata"
	"github.com/crazy-max/WindowsSpyBlocker/app/utils/file"
	"github.com/crazy-max/WindowsSpyBlocker/app/utils/pathu"
	"github.com/crazy-max/WindowsSpyBlocker/app/utils/print"
)

// App
var (
	AppName    = "WindowsSpyBlocker"
	AppVersion = "1.0.0"
	AppPackage = "github.com/crazy-max/WindowsSpyBlocker"
	AppUrl     = "https://" + AppPackage
)

// Config
var (
	App      conf
	Settings settings
)

// Lib structure
type Lib struct {
	Url        string `json:"url"`
	Dest       string
	OutputPath string
	Checkfile  string `json:"checkfile"`
}

// NcsiProbe structure
type NcsiProbe struct {
	WebHostV4    string `json:"webHostV4"`
	WebPathV4    string `json:"webPathV4"`
	WebContentV4 string `json:"webContentV4"`
	WebHostV6    string `json:"webHostV6"`
	WebPathV6    string `json:"webPathV6"`
	WebContentV6 string `json:"webContentV6"`
	DnsHostV4    string `json:"dnsHostV4"`
	DnsContentV4 string `json:"dnsContentV4"`
	DnsHostV6    string `json:"dnsHostV6"`
	DnsContentV6 string `json:"dnsContentV6"`
}

type dataTpl struct {
	Head  string `json:"head"`
	Value string `json:"value"`
}

type conf struct {
	Version         string `json:"version"`
	Debug           bool   `json:"debug"`
	UseEmbeddedData bool   `json:"useEmbeddedData"`
	Proxifier       struct {
		LogPath string `json:"logPath"`
	} `json:"proxifier"`
	Sysmon struct {
		EvtxPath string `json:"evtxPath"`
	} `json:"sysmon"`
	Wireshark struct {
		PcapngPath string `json:"pcapngPath"`
		Capture    struct {
			Interface int    `json:"interface"`
			Filter    string `json:"filter"`
		} `json:"capture"`
	} `json:"wireshark"`
	Exclude struct {
		Ips   []string `json:"ips"`
		Hosts []string `json:"hosts"`
		Orgs  []string `json:"orgs"`
	} `json:"exclude"`
}

type settings struct {
	Uris struct {
		LatestVersion string `json:"latestVersion"`
		Threatcrowd   string `json:"threatcrowd"`
		Whatis        string `json:"whatis"`
		Dnsquery      string `json:"dnsquery"`
		Ipapi         string `json:"ipapi"`
		Ipinfo        string `json:"ipinfo"`
		Ipnf          string `json:"ipnf"`
		Ip6           string `json:"ip6"`
	} `json:"uris"`
	Libs struct {
		Wireshark Lib `json:"wireshark"`
		Npcap     Lib `json:"npcap"`
		Sysmon    Lib `json:"sysmon"`
	} `json:"libs"`
	DataTpl struct {
		Dnscrypt dataTpl `json:"dnscrypt"`
		Openwrt  struct {
			Ip      dataTpl `json:"ip"`
			Domains dataTpl `json:"domains"`
		} `json:"openwrt"`
		P2p       dataTpl `json:"p2p"`
		Proxifier struct {
			Ip      dataTpl `json:"ip"`
			Domains dataTpl `json:"domains"`
		} `json:"proxifier"`
		Simplewall dataTpl `json:"simplewall"`
	} `json:"dataTpl"`
	Proxifier struct {
		UnvalidLines []string `json:"unvalidLines"`
	} `json:"proxifier"`
	Sysmon struct {
		EvtxPath string `json:"evtxPath"`
	} `json:"sysmon"`
	Ncsi struct {
		Reg struct {
			Key               string `json:"key"`
			WebProbeHost      string `json:"webProbeHost"`
			WebProbePath      string `json:"webProbePath"`
			WebProbeContent   string `json:"webProbeContent"`
			WebProbeHostV6    string `json:"webProbeHostV6"`
			WebProbePathV6    string `json:"webProbePathV6"`
			WebProbeContentV6 string `json:"webProbeContentV6"`
			DnsProbeHost      string `json:"dnsProbeHost"`
			DnsProbeContent   string `json:"dnsProbeContent"`
			DnsProbeHostV6    string `json:"dnsProbeHostV6"`
			DnsProbeContentV6 string `json:"dnsProbeContentV6"`
		} `json:"reg"`
		Probes struct {
			Microsoft NcsiProbe `json:"microsoft"`
			Wsb       NcsiProbe `json:"wsb"`
		} `json:"probes"`
	} `json:"ncsi"`
	WilcardSubdomains []string `json:"wilcardSubdomains"`
}

func init() {
	var err error
	var old conf

	cfgPath := path.Join(pathu.Current, "app.conf")

	// Load default config
	defaultConf, err := bindata.Asset("app.conf")
	if err != nil {
		err = fmt.Errorf("Cannot load asset app.conf: %s", err.Error())
		print.QuitFatal(err)
	}
	err = json.Unmarshal(defaultConf, &App)
	if err != nil {
		err = fmt.Errorf("Cannot unmarshall defaultConf: %s", err.Error())
		print.QuitFatal(err)
	}
	newVersion := App.Version

	// Create conf if not exists
	if _, err := os.Stat(cfgPath); err != nil {
		err = ioutil.WriteFile(cfgPath, defaultConf, 0644)
		if err != nil {
			err = fmt.Errorf("Cannot write file %s: %s", strings.TrimLeft(cfgPath, pathu.Current), err.Error())
			print.QuitFatal(err)
		}
	}

	// Load current config
	raw, err := ioutil.ReadFile(cfgPath)
	if err != nil {
		err = fmt.Errorf("Cannot read %s: %s", strings.TrimLeft(cfgPath, pathu.Current), err.Error())
		print.QuitFatal(err)
	}
	err = json.Unmarshal(raw, &old)
	if err != nil {
		err = fmt.Errorf("Cannot unmarshall %s: %s", strings.TrimLeft(cfgPath, pathu.Current), err.Error())
		print.QuitFatal(err)
	}

	// Perform upgrade if different version
	if newVersion != old.Version {
		if err := performUpgrade(); err != nil {
			print.QuitFatal(err)
		}
	}

	// Merge config
	err = json.Unmarshal(raw, &App)
	if err != nil {
		err = fmt.Errorf("Cannot unmarshall %s: %s", strings.TrimLeft(cfgPath, pathu.Current), err.Error())
		print.QuitFatal(err)
	}
	App.Version = newVersion

	// Write config
	cfgJson, _ := json.MarshalIndent(App, "", "  ")
	if err != nil {
		err = fmt.Errorf("Cannot marshal config: %s", err.Error())
		print.QuitFatal(err)
	}
	err = ioutil.WriteFile(cfgPath, cfgJson, 0644)
	if err != nil {
		err = fmt.Errorf("Cannot write file %s: %s", strings.TrimLeft(cfgPath, pathu.Current), err.Error())
		print.QuitFatal(err)
	}

	// Load settings
	rawSettings, err := bindata.Asset("app/settings.json")
	if err != nil {
		err = fmt.Errorf("Cannot load asset settings.json: %s", err.Error())
		print.QuitFatal(err)
	}
	err = json.Unmarshal(rawSettings, &Settings)
	if err != nil {
		err = fmt.Errorf("Cannot unmarshall settings: %s", err.Error())
		print.QuitFatal(err)
	}
}

func performUpgrade() error {
	// Remove content of libs folder
	if err := file.RemoveContents(pathu.Libs); err != nil {
		return err
	}

	return nil
}
<|endoftext|>


<commit_before>package saml2aws

import (
	"fmt"
	"sort"

	"github.com/versent/saml2aws/v2/pkg/provider/netiq"

	"github.com/versent/saml2aws/v2/pkg/cfg"
	"github.com/versent/saml2aws/v2/pkg/creds"
	"github.com/versent/saml2aws/v2/pkg/provider/aad"
	"github.com/versent/saml2aws/v2/pkg/provider/adfs"
	"github.com/versent/saml2aws/v2/pkg/provider/adfs2"
	"github.com/versent/saml2aws/v2/pkg/provider/akamai"
	"github.com/versent/saml2aws/v2/pkg/provider/f5apm"
	"github.com/versent/saml2aws/v2/pkg/provider/googleapps"
	"github.com/versent/saml2aws/v2/pkg/provider/jumpcloud"
	"github.com/versent/saml2aws/v2/pkg/provider/keycloak"
	"github.com/versent/saml2aws/v2/pkg/provider/okta"
	"github.com/versent/saml2aws/v2/pkg/provider/onelogin"
	"github.com/versent/saml2aws/v2/pkg/provider/pingfed"
	"github.com/versent/saml2aws/v2/pkg/provider/pingone"
	"github.com/versent/saml2aws/v2/pkg/provider/shell"
	"github.com/versent/saml2aws/v2/pkg/provider/shibboleth"
	"github.com/versent/saml2aws/v2/pkg/provider/shibbolethecp"
)

// ProviderList list of providers with their MFAs
type ProviderList map[string][]string

// MFAsByProvider a list of providers with their respective supported MFAs
var MFAsByProvider = ProviderList{
	"AzureAD":       []string{"Auto", "PhoneAppOTP", "PhoneAppNotification", "OneWaySMS"},
	"ADFS":          []string{"Auto", "VIP", "Azure", "Defender"},
	"ADFS2":         []string{"Auto", "RSA"}, // nothing automatic about ADFS 2.x
	"Ping":          []string{"Auto"},        // automatically detects PingID
	"PingOne":       []string{"Auto"},        // automatically detects PingID
	"JumpCloud":     []string{"Auto", "TOTP", "WEBAUTHN"},
	"Okta":          []string{"Auto", "PUSH", "DUO", "SMS", "TOTP", "OKTA", "FIDO", "YUBICO TOKEN:HARDWARE"}, // automatically detects DUO, SMS, ToTP, and FIDO
	"OneLogin":      []string{"Auto", "OLP", "SMS", "TOTP", "YUBIKEY"},                                       // automatically detects OneLogin Protect, SMS and ToTP
	"KeyCloak":      []string{"Auto"},                                                                        // automatically detects ToTP
	"GoogleApps":    []string{"Auto"},                                                                        // automatically detects ToTP
	"Shibboleth":    []string{"Auto"},
	"F5APM":         []string{"Auto"},
	"Akamai":        []string{"Auto", "DUO", "SMS", "EMAIL", "TOTP"},
	"ShibbolethECP": []string{"auto", "phone", "push", "passcode"},
	"NetIQ":         []string{"Auto", "Privileged"},
}

// Names get a list of provider names
func (mfbp ProviderList) Names() []string {
	keys := []string{}
	for k := range mfbp {
		keys = append(keys, k)
	}

	sort.Strings(keys)

	return keys
}

// Mfas retrieve a sorted list of mfas from the provider list
func (mfbp ProviderList) Mfas(provider string) []string {
	mfas := mfbp[provider]

	sort.Strings(mfas)

	return mfas
}

func (mfbp ProviderList) stringInSlice(a string, list []string) bool {
	for _, b := range list {
		if b == a {
			return true
		}
	}
	return false
}

func invalidMFA(provider string, mfa string) bool {
	supportedMfas := MFAsByProvider.Mfas(provider)
	return !MFAsByProvider.stringInSlice(mfa, supportedMfas)
}

// SAMLClient client interface
type SAMLClient interface {
	Authenticate(loginDetails *creds.LoginDetails) (string, error)
	Validate(loginDetails *creds.LoginDetails) error
}

// NewSAMLClient create a new SAML client
func NewSAMLClient(idpAccount *cfg.IDPAccount) (SAMLClient, error) {
	switch idpAccount.Provider {
	case "AzureAD":
		if invalidMFA(idpAccount.Provider, idpAccount.MFA) {
			return nil, fmt.Errorf("Invalid MFA type: %v for %v provider", idpAccount.MFA, idpAccount.Provider)
		}
		return aad.New(idpAccount)
	case "ADFS":
		if invalidMFA(idpAccount.Provider, idpAccount.MFA) {
			return nil, fmt.Errorf("Invalid MFA type: %v for %v provider", idpAccount.MFA, idpAccount.Provider)
		}
		return adfs.New(idpAccount)
	case "ADFS2":
		if invalidMFA(idpAccount.Provider, idpAccount.MFA) {
			return nil, fmt.Errorf("Invalid MFA type: %v for %v provider", idpAccount.MFA, idpAccount.Provider)
		}
		return adfs2.New(idpAccount)
	case "Ping":
		if invalidMFA(idpAccount.Provider, idpAccount.MFA) {
			return nil, fmt.Errorf("Invalid MFA type: %v for %v provider", idpAccount.MFA, idpAccount.Provider)
		}
		return pingfed.New(idpAccount)
	case "PingOne":
		if invalidMFA(idpAccount.Provider, idpAccount.MFA) {
			return nil, fmt.Errorf("Invalid MFA type: %v for %v provider", idpAccount.MFA, idpAccount.Provider)
		}
		return pingone.New(idpAccount)
	case "JumpCloud":
		if invalidMFA(idpAccount.Provider, idpAccount.MFA) {
			return nil, fmt.Errorf("Invalid MFA type: %v for %v provider", idpAccount.MFA, idpAccount.Provider)
		}
		return jumpcloud.New(idpAccount)
	case "Okta":
		if invalidMFA(idpAccount.Provider, idpAccount.MFA) {
			return nil, fmt.Errorf("Invalid MFA type: %v for %v provider", idpAccount.MFA, idpAccount.Provider)
		}
		return okta.New(idpAccount)
	case "OneLogin":
		if invalidMFA(idpAccount.Provider, idpAccount.MFA) {
			return nil, fmt.Errorf("Invalid MFA type: %v for %v provider", idpAccount.MFA, idpAccount.Provider)
		}
		return onelogin.New(idpAccount)
	case "KeyCloak":
		if invalidMFA(idpAccount.Provider, idpAccount.MFA) {
			return nil, fmt.Errorf("Invalid MFA type: %v for %v provider", idpAccount.MFA, idpAccount.Provider)
		}
		return keycloak.New(idpAccount)
	case "GoogleApps":
		if invalidMFA(idpAccount.Provider, idpAccount.MFA) {
			return nil, fmt.Errorf("Invalid MFA type: %v for %v provider", idpAccount.MFA, idpAccount.Provider)
		}
		return googleapps.New(idpAccount)
	case "Shibboleth":
		if invalidMFA(idpAccount.Provider, idpAccount.MFA) {
			return nil, fmt.Errorf("Invalid MFA type: %v for %v provider", idpAccount.MFA, idpAccount.Provider)
		}
		return shibboleth.New(idpAccount)
	case "ShibbolethECP":
		if invalidMFA(idpAccount.Provider, idpAccount.MFA) {
			return nil, fmt.Errorf("Invalid MFA type: %v for %v provider", idpAccount.MFA, idpAccount.Provider)
		}
		return shibbolethecp.New(idpAccount)
	case "F5APM":
		if invalidMFA(idpAccount.Provider, idpAccount.MFA) {
			return nil, fmt.Errorf("Invalid MFA type: %v for %v provider", idpAccount.MFA, idpAccount.Provider)
		}
		return f5apm.New(idpAccount)
	case "Akamai":
		if invalidMFA(idpAccount.Provider, idpAccount.MFA) {
			return nil, fmt.Errorf("Invalid MFA type: %v for %v provider", idpAccount.MFA, idpAccount.Provider)
		}
		return akamai.New(idpAccount)
	case "Shell":
		return shell.New(idpAccount)
	case "NetIQ":
		if invalidMFA(idpAccount.Provider, idpAccount.MFA) {
			return nil, fmt.Errorf("Invalid MFA type: %v for %v provider", idpAccount.MFA, idpAccount.Provider)
		}
		return netiq.New(idpAccount, idpAccount.MFA)
	default:
		return nil, fmt.Errorf("Invalid provider: %v", idpAccount.Provider)
	}
}
<commit_msg>Add DUO to the allowed auth methods for JumpCloud<commit_after>package saml2aws

import (
	"fmt"
	"sort"

	"github.com/versent/saml2aws/v2/pkg/provider/netiq"

	"github.com/versent/saml2aws/v2/pkg/cfg"
	"github.com/versent/saml2aws/v2/pkg/creds"
	"github.com/versent/saml2aws/v2/pkg/provider/aad"
	"github.com/versent/saml2aws/v2/pkg/provider/adfs"
	"github.com/versent/saml2aws/v2/pkg/provider/adfs2"
	"github.com/versent/saml2aws/v2/pkg/provider/akamai"
	"github.com/versent/saml2aws/v2/pkg/provider/f5apm"
	"github.com/versent/saml2aws/v2/pkg/provider/googleapps"
	"github.com/versent/saml2aws/v2/pkg/provider/jumpcloud"
	"github.com/versent/saml2aws/v2/pkg/provider/keycloak"
	"github.com/versent/saml2aws/v2/pkg/provider/okta"
	"github.com/versent/saml2aws/v2/pkg/provider/onelogin"
	"github.com/versent/saml2aws/v2/pkg/provider/pingfed"
	"github.com/versent/saml2aws/v2/pkg/provider/pingone"
	"github.com/versent/saml2aws/v2/pkg/provider/shell"
	"github.com/versent/saml2aws/v2/pkg/provider/shibboleth"
	"github.com/versent/saml2aws/v2/pkg/provider/shibbolethecp"
)

// ProviderList list of providers with their MFAs
type ProviderList map[string][]string

// MFAsByProvider a list of providers with their respective supported MFAs
var MFAsByProvider = ProviderList{
	"AzureAD":       []string{"Auto", "PhoneAppOTP", "PhoneAppNotification", "OneWaySMS"},
	"ADFS":          []string{"Auto", "VIP", "Azure", "Defender"},
	"ADFS2":         []string{"Auto", "RSA"}, // nothing automatic about ADFS 2.x
	"Ping":          []string{"Auto"},        // automatically detects PingID
	"PingOne":       []string{"Auto"},        // automatically detects PingID
	"JumpCloud":     []string{"Auto", "TOTP", "WEBAUTHN", "DUO"},
	"Okta":          []string{"Auto", "PUSH", "DUO", "SMS", "TOTP", "OKTA", "FIDO", "YUBICO TOKEN:HARDWARE"}, // automatically detects DUO, SMS, ToTP, and FIDO
	"OneLogin":      []string{"Auto", "OLP", "SMS", "TOTP", "YUBIKEY"},                                       // automatically detects OneLogin Protect, SMS and ToTP
	"KeyCloak":      []string{"Auto"},                                                                        // automatically detects ToTP
	"GoogleApps":    []string{"Auto"},                                                                        // automatically detects ToTP
	"Shibboleth":    []string{"Auto"},
	"F5APM":         []string{"Auto"},
	"Akamai":        []string{"Auto", "DUO", "SMS", "EMAIL", "TOTP"},
	"ShibbolethECP": []string{"auto", "phone", "push", "passcode"},
	"NetIQ":         []string{"Auto", "Privileged"},
}

// Names get a list of provider names
func (mfbp ProviderList) Names() []string {
	keys := []string{}
	for k := range mfbp {
		keys = append(keys, k)
	}

	sort.Strings(keys)

	return keys
}

// Mfas retrieve a sorted list of mfas from the provider list
func (mfbp ProviderList) Mfas(provider string) []string {
	mfas := mfbp[provider]

	sort.Strings(mfas)

	return mfas
}

func (mfbp ProviderList) stringInSlice(a string, list []string) bool {
	for _, b := range list {
		if b == a {
			return true
		}
	}
	return false
}

func invalidMFA(provider string, mfa string) bool {
	supportedMfas := MFAsByProvider.Mfas(provider)
	return !MFAsByProvider.stringInSlice(mfa, supportedMfas)
}

// SAMLClient client interface
type SAMLClient interface {
	Authenticate(loginDetails *creds.LoginDetails) (string, error)
	Validate(loginDetails *creds.LoginDetails) error
}

// NewSAMLClient create a new SAML client
func NewSAMLClient(idpAccount *cfg.IDPAccount) (SAMLClient, error) {
	switch idpAccount.Provider {
	case "AzureAD":
		if invalidMFA(idpAccount.Provider, idpAccount.MFA) {
			return nil, fmt.Errorf("Invalid MFA type: %v for %v provider", idpAccount.MFA, idpAccount.Provider)
		}
		return aad.New(idpAccount)
	case "ADFS":
		if invalidMFA(idpAccount.Provider, idpAccount.MFA) {
			return nil, fmt.Errorf("Invalid MFA type: %v for %v provider", idpAccount.MFA, idpAccount.Provider)
		}
		return adfs.New(idpAccount)
	case "ADFS2":
		if invalidMFA(idpAccount.Provider, idpAccount.MFA) {
			return nil, fmt.Errorf("Invalid MFA type: %v for %v provider", idpAccount.MFA, idpAccount.Provider)
		}
		return adfs2.New(idpAccount)
	case "Ping":
		if invalidMFA(idpAccount.Provider, idpAccount.MFA) {
			return nil, fmt.Errorf("Invalid MFA type: %v for %v provider", idpAccount.MFA, idpAccount.Provider)
		}
		return pingfed.New(idpAccount)
	case "PingOne":
		if invalidMFA(idpAccount.Provider, idpAccount.MFA) {
			return nil, fmt.Errorf("Invalid MFA type: %v for %v provider", idpAccount.MFA, idpAccount.Provider)
		}
		return pingone.New(idpAccount)
	case "JumpCloud":
		if invalidMFA(idpAccount.Provider, idpAccount.MFA) {
			return nil, fmt.Errorf("Invalid MFA type: %v for %v provider", idpAccount.MFA, idpAccount.Provider)
		}
		return jumpcloud.New(idpAccount)
	case "Okta":
		if invalidMFA(idpAccount.Provider, idpAccount.MFA) {
			return nil, fmt.Errorf("Invalid MFA type: %v for %v provider", idpAccount.MFA, idpAccount.Provider)
		}
		return okta.New(idpAccount)
	case "OneLogin":
		if invalidMFA(idpAccount.Provider, idpAccount.MFA) {
			return nil, fmt.Errorf("Invalid MFA type: %v for %v provider", idpAccount.MFA, idpAccount.Provider)
		}
		return onelogin.New(idpAccount)
	case "KeyCloak":
		if invalidMFA(idpAccount.Provider, idpAccount.MFA) {
			return nil, fmt.Errorf("Invalid MFA type: %v for %v provider", idpAccount.MFA, idpAccount.Provider)
		}
		return keycloak.New(idpAccount)
	case "GoogleApps":
		if invalidMFA(idpAccount.Provider, idpAccount.MFA) {
			return nil, fmt.Errorf("Invalid MFA type: %v for %v provider", idpAccount.MFA, idpAccount.Provider)
		}
		return googleapps.New(idpAccount)
	case "Shibboleth":
		if invalidMFA(idpAccount.Provider, idpAccount.MFA) {
			return nil, fmt.Errorf("Invalid MFA type: %v for %v provider", idpAccount.MFA, idpAccount.Provider)
		}
		return shibboleth.New(idpAccount)
	case "ShibbolethECP":
		if invalidMFA(idpAccount.Provider, idpAccount.MFA) {
			return nil, fmt.Errorf("Invalid MFA type: %v for %v provider", idpAccount.MFA, idpAccount.Provider)
		}
		return shibbolethecp.New(idpAccount)
	case "F5APM":
		if invalidMFA(idpAccount.Provider, idpAccount.MFA) {
			return nil, fmt.Errorf("Invalid MFA type: %v for %v provider", idpAccount.MFA, idpAccount.Provider)
		}
		return f5apm.New(idpAccount)
	case "Akamai":
		if invalidMFA(idpAccount.Provider, idpAccount.MFA) {
			return nil, fmt.Errorf("Invalid MFA type: %v for %v provider", idpAccount.MFA, idpAccount.Provider)
		}
		return akamai.New(idpAccount)
	case "Shell":
		return shell.New(idpAccount)
	case "NetIQ":
		if invalidMFA(idpAccount.Provider, idpAccount.MFA) {
			return nil, fmt.Errorf("Invalid MFA type: %v for %v provider", idpAccount.MFA, idpAccount.Provider)
		}
		return netiq.New(idpAccount, idpAccount.MFA)
	default:
		return nil, fmt.Errorf("Invalid provider: %v", idpAccount.Provider)
	}
}
<|endoftext|>


<commit_before>package main

import (
	"flag"
	"fmt"
	"github.com/ericaro/mrepo"
	"io/ioutil"
	"os"
	"sync"
	"text/tabwriter"
)

const (
	Usage = `
USAGE %[1]s [-options] <command> <args...>

DESCRIPTION:

  %[1]s subrepositories manager for git dependencies.

  It manages two sets of subrepositories:

    - ".sbr": set made of subrepositories declarations in '.%[1]s' file
    - "disk": set made of actually subrepositories in current directory hierarchy

  <command> can be:

    - describe: print the "disk" dependency set
    - compare : diff ".sbr" and "disk" sets. In the form of operations to apply to ".sbr" set.
    - reflect : replace ".sbr" set by "disk" one.
    - apply   : apply ".sbr" dependencies to the current working dir (prune and clone)
    - merge   : edit two sets in meld.

OPTIONS:

`
	Example = `
EXAMPLES:

  - Print out all dependencies present in a workspace:
	  $ %[1]s describe

  - Init a .sbr to reflect the current working dir:
	  $ %[1]s reflect

  - Add a subrepository as usual, and update your .%[1]s:
      $ git clone git@github.com:ericaro/mrepo.git -b dev src/ericaro/mrepo
      $ %[1]s reflect

  - Apply .deps changes to the working dir:
      $ %[1]s apply

`
)

var (
	dotmrepo = flag.String("s", ".sbr", "override default dependency filename")
	// workingdir = flag.String("wd", ".", "path to be used as working dir")
	help = flag.Bool("h", false, "Print this help.")
)

func usage() {
	fmt.Printf(Usage, os.Args[0])
	flag.PrintDefaults()
	fmt.Printf(Example, os.Args[0])
}

func main() {
	flag.Parse()

	if flag.NArg() <= 0 || *help {
		usage()
		return
	}

	// use wd by default
	wd, err := os.Getwd()
	if err != nil {
		fmt.Printf("Error, cannot determine the current directory. %s\n", err.Error())
	}
	//creates a workspace to be able to read from/to sets
	workspace := mrepo.NewWorkspace(wd)
	cmd := flag.Arg(0)
	switch {

	case cmd == "describe": // not diff mode, hence, plain local mode
		// execute query on each subrepo
		current := workspace.WorkingDirSubrepositories()
		// and just print it out
		w := tabwriter.NewWriter(os.Stdout, 3, 8, 3, ' ', 0)
		for _, d := range current {
			fmt.Fprintf(w, "git\t%q\t%q\t%q\n", d.Rel(), d.Remote(), d.Branch())
		}
		w.Flush()
	case cmd == "merge":
		//generate a temp file
		current := workspace.WorkingDirSubrepositories()
		f, err := ioutil.TempFile("", "sbr")
		mrepo.WriteSubrepositoryTo(f, current)
		f.Close() //no defer to open it up just after.
		err = mrepo.Meld(workspace.Wd(), ".sbr set  |  disk set", workspace.Sbrfile(), f.Name())
		if err != nil {
			fmt.Printf("Meld returned with error: %s", err.Error())
			os.Exit(-1)
		}
		// shall I apply ?

	case cmd == "compare":

		del, ins := workspace.WorkingDirPatches()
		//WorkingDirPatches > (ins, del) are for the wd, here we are interested in the reverse
		// so we permute the assignmeent
		// therefore del are subrepo to be deleted from disk
		// the output will be fully tabbed

		w := tabwriter.NewWriter(os.Stdout, 3, 8, 3, ' ', 0)
		fmt.Fprintf(w, ".sbr\tpath\tremote\tbranch\n")
		for _, s := range del {
			fmt.Fprintf(w, "\033[00;32mDEL\033[00m\t%s\t%s\t%s\n", s.Rel(), s.Remote(), s.Branch())
		}
		for _, s := range ins {
			fmt.Fprintf(w, "\033[00;31mINS\033[00m\t%s\t%s\t%s\n", s.Rel(), s.Remote(), s.Branch())
		}
		w.Flush()

	case cmd == "reflect":
		//compute ins and del in the .sbr file
		del, ins := workspace.WorkingDirPatches()
		//WorkingDirPatches > (ins, del) are for the wd, here we are interested in the reverse
		// so we permute the assignmeent
		// therefore del are subrepo to be deleted from disk
		// the output will be fully tabbed

		//read ".sbr" content
		current := workspace.FileSubrepositories()

		current.RemoveAll(del)
		current.AddAll(ins)

		w := tabwriter.NewWriter(os.Stdout, 3, 8, 3, ' ', 0)
		fmt.Fprintf(w, ".sbr\tpath\tremote\tbranch\n")
		for _, s := range del {
			fmt.Fprintf(w, "\033[00;32mDEL\033[00m\t%s\t%s\t%s\n", s.Rel(), s.Remote(), s.Branch())
		}
		for _, s := range ins {
			fmt.Fprintf(w, "\033[00;31mINS\033[00m\t%s\t%s\t%s\n", s.Rel(), s.Remote(), s.Branch())
		}
		w.Flush()
		//always rewrite the file
		workspace.WriteSubrepositoryFile(current)
		fmt.Printf("Done (\033[00;32m%v\033[00m INS) (\033[00;32m%v\033[00m DEL)\n", len(ins), len(del))

	case cmd == "apply":
		ins, del := workspace.WorkingDirPatches()
		var waiter sync.WaitGroup // to wait for all commands to return
		var delCount, cloneCount int
		for _, sbr := range ins {
			waiter.Add(1)
			go func(d mrepo.Subrepository) {
				defer waiter.Done()
				_, err := d.Clone()
				if err != nil {
					fmt.Printf("\033[00;31mERR\033[00m  git clone %s -b %s %s:\n     %s\n", d.Rel(), d.Remote(), d.Branch(), d.Rel(), err.Error())
				} else {
					cloneCount++
					fmt.Printf("     Cloning into '%s'...\n", d.Rel())
				}
			}(sbr)
		}
		for _, sbr := range del {
			waiter.Add(1)
			go func(d mrepo.Subrepository) {
				defer waiter.Done()
				err = sbr.Prune()
				if err != nil {
					fmt.Printf("\033[00;31mERR\033[00m  rm -Rf %s :\n     %s\n", d.Rel(), d.Rel(), err.Error())
				} else {
					delCount++
					fmt.Printf("     Pruning '%s'...\n", d.Rel())
				}
			}(sbr)
		}
		waiter.Wait()
		fmt.Printf("Done (\033[00;32m%v\033[00m CLONE) (\033[00;32m%v\033[00m PRUNE)\n", cloneCount, delCount)

	default:
		usage()
		return

	}
}
<commit_msg>fixed typo while printing error<commit_after>package main

import (
	"flag"
	"fmt"
	"github.com/ericaro/mrepo"
	"io/ioutil"
	"os"
	"sync"
	"text/tabwriter"
)

const (
	Usage = `
USAGE %[1]s [-options] <command> <args...>

DESCRIPTION:

  %[1]s subrepositories manager for git dependencies.

  It manages two sets of subrepositories:

    - ".sbr": set made of subrepositories declarations in '.%[1]s' file
    - "disk": set made of actually subrepositories in current directory hierarchy

  <command> can be:

    - describe: print the "disk" dependency set
    - compare : diff ".sbr" and "disk" sets. In the form of operations to apply to ".sbr" set.
    - reflect : replace ".sbr" set by "disk" one.
    - apply   : apply ".sbr" dependencies to the current working dir (prune and clone)
    - merge   : edit two sets in meld.

OPTIONS:

`
	Example = `
EXAMPLES:

  - Print out all dependencies present in a workspace:
	  $ %[1]s describe

  - Init a .sbr to reflect the current working dir:
	  $ %[1]s reflect

  - Add a subrepository as usual, and update your .%[1]s:
      $ git clone git@github.com:ericaro/mrepo.git -b dev src/ericaro/mrepo
      $ %[1]s reflect

  - Apply .deps changes to the working dir:
      $ %[1]s apply

`
)

var (
	dotmrepo = flag.String("s", ".sbr", "override default dependency filename")
	// workingdir = flag.String("wd", ".", "path to be used as working dir")
	help = flag.Bool("h", false, "Print this help.")
)

func usage() {
	fmt.Printf(Usage, os.Args[0])
	flag.PrintDefaults()
	fmt.Printf(Example, os.Args[0])
}

func main() {
	flag.Parse()

	if flag.NArg() <= 0 || *help {
		usage()
		return
	}

	// use wd by default
	wd, err := os.Getwd()
	if err != nil {
		fmt.Printf("Error, cannot determine the current directory. %s\n", err.Error())
	}
	//creates a workspace to be able to read from/to sets
	workspace := mrepo.NewWorkspace(wd)
	cmd := flag.Arg(0)
	switch {

	case cmd == "describe": // not diff mode, hence, plain local mode
		// execute query on each subrepo
		current := workspace.WorkingDirSubrepositories()
		// and just print it out
		w := tabwriter.NewWriter(os.Stdout, 3, 8, 3, ' ', 0)
		for _, d := range current {
			fmt.Fprintf(w, "git\t%q\t%q\t%q\n", d.Rel(), d.Remote(), d.Branch())
		}
		w.Flush()
	case cmd == "merge":
		//generate a temp file
		current := workspace.WorkingDirSubrepositories()
		f, err := ioutil.TempFile("", "sbr")
		mrepo.WriteSubrepositoryTo(f, current)
		f.Close() //no defer to open it up just after.
		err = mrepo.Meld(workspace.Wd(), ".sbr set  |  disk set", workspace.Sbrfile(), f.Name())
		if err != nil {
			fmt.Printf("Meld returned with error: %s", err.Error())
			os.Exit(-1)
		}
		// shall I apply ?

	case cmd == "compare":

		del, ins := workspace.WorkingDirPatches()
		//WorkingDirPatches > (ins, del) are for the wd, here we are interested in the reverse
		// so we permute the assignmeent
		// therefore del are subrepo to be deleted from disk
		// the output will be fully tabbed

		w := tabwriter.NewWriter(os.Stdout, 3, 8, 3, ' ', 0)
		fmt.Fprintf(w, ".sbr\tpath\tremote\tbranch\n")
		for _, s := range del {
			fmt.Fprintf(w, "\033[00;32mDEL\033[00m\t%s\t%s\t%s\n", s.Rel(), s.Remote(), s.Branch())
		}
		for _, s := range ins {
			fmt.Fprintf(w, "\033[00;31mINS\033[00m\t%s\t%s\t%s\n", s.Rel(), s.Remote(), s.Branch())
		}
		w.Flush()

	case cmd == "reflect":
		//compute ins and del in the .sbr file
		del, ins := workspace.WorkingDirPatches()
		//WorkingDirPatches > (ins, del) are for the wd, here we are interested in the reverse
		// so we permute the assignmeent
		// therefore del are subrepo to be deleted from disk
		// the output will be fully tabbed

		//read ".sbr" content
		current := workspace.FileSubrepositories()

		current.RemoveAll(del)
		current.AddAll(ins)

		w := tabwriter.NewWriter(os.Stdout, 3, 8, 3, ' ', 0)
		fmt.Fprintf(w, ".sbr\tpath\tremote\tbranch\n")
		for _, s := range del {
			fmt.Fprintf(w, "\033[00;32mDEL\033[00m\t%s\t%s\t%s\n", s.Rel(), s.Remote(), s.Branch())
		}
		for _, s := range ins {
			fmt.Fprintf(w, "\033[00;31mINS\033[00m\t%s\t%s\t%s\n", s.Rel(), s.Remote(), s.Branch())
		}
		w.Flush()
		//always rewrite the file
		workspace.WriteSubrepositoryFile(current)
		fmt.Printf("Done (\033[00;32m%v\033[00m INS) (\033[00;32m%v\033[00m DEL)\n", len(ins), len(del))

	case cmd == "apply":
		ins, del := workspace.WorkingDirPatches()
		var waiter sync.WaitGroup // to wait for all commands to return
		var delCount, cloneCount int
		for _, sbr := range ins {
			waiter.Add(1)
			go func(d mrepo.Subrepository) {
				defer waiter.Done()
				_, err := d.Clone()
				if err != nil {
					fmt.Printf("\033[00;31mERR\033[00m  git clone %s -b %s %s:\n     %s\n", d.Rel(), d.Remote(), d.Branch(), err.Error())
				} else {
					cloneCount++
					fmt.Printf("     Cloning into '%s'...\n", d.Rel())
				}
			}(sbr)
		}
		for _, sbr := range del {
			waiter.Add(1)
			go func(d mrepo.Subrepository) {
				defer waiter.Done()
				err = sbr.Prune()
				if err != nil {
					fmt.Printf("\033[00;31mERR\033[00m  rm -Rf %s :\n     %s\n", d.Rel(), err.Error())
				} else {
					delCount++
					fmt.Printf("     Pruning '%s'...\n", d.Rel())
				}
			}(sbr)
		}
		waiter.Wait()
		fmt.Printf("Done (\033[00;32m%v\033[00m CLONE) (\033[00;32m%v\033[00m PRUNE)\n", cloneCount, delCount)

	default:
		usage()
		return

	}
}
<|endoftext|>