<commit_before>/*
Copyright 2021 The Kubernetes Authors.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package test

import (
	"context"
	"encoding/json"
	"fmt"
	"strings"
	"sync"
	"testing"

	appsv1 "k8s.io/api/apps/v1"
	corev1 "k8s.io/api/core/v1"
	apierrors "k8s.io/apimachinery/pkg/api/errors"
	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
	"k8s.io/client-go/kubernetes"
	"k8s.io/client-go/rest"
	"k8s.io/component-base/featuregate"
	"k8s.io/pod-security-admission/api"
	"k8s.io/pod-security-admission/policy"
)

// Options hold configuration for running integration tests against an existing server.
type Options struct {
	// ClientConfig is a client configuration with sufficient permission to create, update, and delete
	// namespaces, pods, and pod-template-containing objects.
	// Required.
	ClientConfig *rest.Config

	// Features optionally provides information about which feature gates are enabled.
	// This is used to skip failure cases for negative tests of data in alpha/beta fields.
	// If unset, all testcases are run.
	Features featuregate.FeatureGate

	// CreateNamespace is an optional stub for creating a namespace with the given name and labels.
	// Returning an error fails the test.
	// If nil, DefaultCreateNamespace is used.
	CreateNamespace func(client kubernetes.Interface, name string, labels map[string]string) (*corev1.Namespace, error)

	// These are the check ids/starting versions to exercise.
	// If unset, policy.DefaultChecks() are used.
	Checks []policy.Check

	// ExemptClient is an optional client interface to exercise behavior of an exempt client.
	ExemptClient kubernetes.Interface
	// ExemptNamespaces are optional namespaces not expected to have PodSecurity controls enforced.
	ExemptNamespaces []string
	// ExemptRuntimeClasses are optional runtimeclasses not expected to have PodSecurity controls enforced.
	ExemptRuntimeClasses []string
}

func toJSON(pod *corev1.Pod) string {
	data, _ := json.Marshal(pod)
	return string(data)
}

// checksForLevelAndVersion returns the set of check IDs that apply when evaluating the given level and version.
// checks are assumed to be well-formed and valid to pass to policy.NewEvaluator().
// level must be api.LevelRestricted or api.LevelBaseline
func checksForLevelAndVersion(checks []policy.Check, level api.Level, version api.Version) ([]string, error) {
	retval := []string{}
	for _, check := range checks {
		if !version.Older(check.Versions[0].MinimumVersion) && (level == check.Level || level == api.LevelRestricted) {
			retval = append(retval, check.ID)
		}
	}
	return retval, nil
}

// maxMinorVersionToTest returns the maximum minor version to exercise for a given set of checks.
// checks are assumed to be well-formed and valid to pass to policy.NewEvaluator().
func maxMinorVersionToTest(checks []policy.Check) (int, error) {
	// start with the release under development (1.22 at time of writing).
	// this can be incremented to the current version whenever is convenient.
	maxTestMinor := 22
	for _, check := range checks {
		lastCheckVersion := check.Versions[len(check.Versions)-1].MinimumVersion
		if lastCheckVersion.Major() != 1 {
			return 0, fmt.Errorf("expected major version 1, got %d", lastCheckVersion.Major())
		}
		if lastCheckVersion.Minor() > maxTestMinor {
			maxTestMinor = lastCheckVersion.Minor()
		}
	}
	return maxTestMinor, nil
}

type testWarningHandler struct {
	lock     sync.Mutex
	warnings []string
}

func (t *testWarningHandler) HandleWarningHeader(code int, agent string, warning string) {
	t.lock.Lock()
	defer t.lock.Unlock()
	t.warnings = append(t.warnings, warning)
}

func (t *testWarningHandler) FlushWarnings() []string {
	t.lock.Lock()
	defer t.lock.Unlock()
	warnings := t.warnings
	t.warnings = nil
	return warnings
}

// and ensures pod fixtures expected to pass and fail against that level/version work as expected.
func Run(t *testing.T, opts Options) {
	warningHandler := &testWarningHandler{}

	configCopy := rest.CopyConfig(opts.ClientConfig)
	configCopy.WarningHandler = warningHandler
	client, err := kubernetes.NewForConfig(configCopy)
	if err != nil {
		t.Fatalf("error creating client: %v", err)
	}

	if opts.CreateNamespace == nil {
		opts.CreateNamespace = DefaultCreateNamespace
	}
	if len(opts.Checks) == 0 {
		opts.Checks = policy.DefaultChecks()
	}
	_, err = policy.NewEvaluator(opts.Checks)
	if err != nil {
		t.Fatalf("invalid checks: %v", err)
	}
	maxMinor, err := maxMinorVersionToTest(opts.Checks)
	if err != nil {
		t.Fatalf("invalid checks: %v", err)
	}

	for _, level := range []api.Level{api.LevelBaseline, api.LevelRestricted} {
		for minor := 0; minor <= maxMinor; minor++ {
			version := api.MajorMinorVersion(1, minor)

			// create test name
			ns := fmt.Sprintf("podsecurity-%s-1-%d", level, minor)

			// create namespace
			_, err := opts.CreateNamespace(client, ns, map[string]string{
				api.EnforceLevelLabel:   string(level),
				api.EnforceVersionLabel: fmt.Sprintf("v1.%d", minor),
				api.WarnLevelLabel:      string(level),
				api.WarnVersionLabel:    fmt.Sprintf("v1.%d", minor),
			})
			if err != nil {
				t.Errorf("failed creating namespace %s: %v", ns, err)
				continue
			}
			t.Cleanup(func() {
				client.CoreV1().Namespaces().Delete(context.Background(), ns, metav1.DeleteOptions{})
			})

			// create service account (to allow pod to pass serviceaccount admission)
			sa, err := client.CoreV1().ServiceAccounts(ns).Create(
				context.Background(),
				&corev1.ServiceAccount{ObjectMeta: metav1.ObjectMeta{Name: "default"}},
				metav1.CreateOptions{},
			)
			if err != nil && !apierrors.IsAlreadyExists(err) {
				t.Errorf("failed creating serviceaccount %s: %v", ns, err)
				continue
			}
			t.Cleanup(func() {
				client.CoreV1().ServiceAccounts(ns).Delete(context.Background(), sa.Name, metav1.DeleteOptions{})
			})

			// create pod
			createPod := func(t *testing.T, i int, pod *corev1.Pod, expectSuccess bool, expectErrorSubstring string) {
				t.Helper()
				// avoid mutating original pod fixture
				pod = pod.DeepCopy()
				// assign pod name and serviceaccount
				pod.Name = "test"
				pod.Spec.ServiceAccountName = "default"
				// dry-run create
				_, err := client.CoreV1().Pods(ns).Create(context.Background(), pod, metav1.CreateOptions{DryRun: []string{metav1.DryRunAll}})
				if !expectSuccess {
					if err == nil {
						t.Errorf("%d: expected error creating %s, got none", i, toJSON(pod))
						return
					}
					if strings.Contains(err.Error(), policy.UnknownForbiddenReason) {
						t.Errorf("%d: unexpected unknown forbidden reason creating %s: %v", i, toJSON(pod), err)
						return
					}
					if !strings.Contains(err.Error(), expectErrorSubstring) {
						t.Errorf("%d: expected error with substring %q, got %v", i, expectErrorSubstring, err)
						return
					}
				}
				if expectSuccess && err != nil {
					t.Errorf("%d: unexpected error creating %s: %v", i, toJSON(pod), err)
				}
			}

			// create controller
			createController := func(t *testing.T, i int, pod *corev1.Pod, expectSuccess bool, expectErrorSubstring string) {
				t.Helper()
				// avoid mutating original pod fixture
				pod = pod.DeepCopy()
				if pod.Labels == nil {
					pod.Labels = map[string]string{}
				}
				pod.Labels["test"] = "true"

				warningHandler.FlushWarnings()
				// dry-run create
				deployment := &appsv1.Deployment{
					ObjectMeta: metav1.ObjectMeta{Name: "test"},
					Spec: appsv1.DeploymentSpec{
						Selector: &metav1.LabelSelector{MatchLabels: map[string]string{"test": "true"}},
						Template: corev1.PodTemplateSpec{
							ObjectMeta: pod.ObjectMeta,
							Spec:       pod.Spec,
						},
					},
				}
				_, err := client.AppsV1().Deployments(ns).Create(context.Background(), deployment, metav1.CreateOptions{DryRun: []string{metav1.DryRunAll}})
				if err != nil {
					t.Errorf("%d: unexpected error creating controller with %s: %v", i, toJSON(pod), err)
					return
				}
				warningText := strings.Join(warningHandler.FlushWarnings(), "; ")
				if !expectSuccess {
					if len(warningText) == 0 {
						t.Errorf("%d: expected warnings creating %s, got none", i, toJSON(pod))
						return
					}
					if strings.Contains(warningText, policy.UnknownForbiddenReason) {
						t.Errorf("%d: unexpected unknown forbidden reason creating %s: %v", i, toJSON(pod), warningText)
						return
					}
					if !strings.Contains(warningText, expectErrorSubstring) {
						t.Errorf("%d: expected warning with substring %q, got %v", i, expectErrorSubstring, warningText)
						return
					}
				}

				if expectSuccess && len(warningText) > 0 {
					if (len(expectErrorSubstring) > 0 && strings.Contains(warningText, expectErrorSubstring)) ||
						strings.Contains(warningText, policy.UnknownForbiddenReason) {
						t.Errorf("%d: unexpected warning creating %s: %v", i, toJSON(pod), warningText)
					}
				}
			}

			minimalValidPod, err := getMinimalValidPod(level, version)
			if err != nil {
				t.Fatal(err)
			}
			t.Run(ns+"_pass_base", func(t *testing.T) {
				createPod(t, 0, minimalValidPod.DeepCopy(), true, "")
				createController(t, 0, minimalValidPod.DeepCopy(), true, "")
			})

			checkIDs, err := checksForLevelAndVersion(opts.Checks, level, version)
			if err != nil {
				t.Fatal(err)
			}
			if len(checkIDs) == 0 {
				t.Fatal(fmt.Errorf("no checks registered for %s/1.%d", level, minor))
			}
			for _, checkID := range checkIDs {
				checkData, err := getFixtures(fixtureKey{level: level, version: version, check: checkID})
				if err != nil {
					t.Fatal(err)
				}

				t.Run(ns+"_pass_"+checkID, func(t *testing.T) {
					for i, pod := range checkData.pass {
						createPod(t, i, pod, true, "")
						createController(t, i, pod, true, "")
					}
				})

				// see if any features required for failure cases are disabled
				var disabledRequiredFeatures []featuregate.Feature
				for _, f := range checkData.failRequiresFeatures {
					if opts.Features != nil && !opts.Features.Enabled(f) {
						disabledRequiredFeatures = append(disabledRequiredFeatures, f)
					}
				}
				t.Run(ns+"_fail_"+checkID, func(t *testing.T) {
					if len(disabledRequiredFeatures) > 0 {
						t.Skipf("features required for failure cases are disabled: %v", disabledRequiredFeatures)
					}
					for i, pod := range checkData.fail {
						createPod(t, i, pod, false, checkData.expectErrorSubstring)
						createController(t, i, pod, false, checkData.expectErrorSubstring)
					}
				})
			}
		}
	}
}

func DefaultCreateNamespace(client kubernetes.Interface, name string, labels map[string]string) (*corev1.Namespace, error) {
	return client.CoreV1().Namespaces().Create(
		context.Background(),
		&corev1.Namespace{
			ObjectMeta: metav1.ObjectMeta{Name: name, Labels: labels},
		},
		metav1.CreateOptions{},
	)
}
<commit_msg>PodSecurity: make integration tests run sparsely<commit_after>/*
Copyright 2021 The Kubernetes Authors.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package test

import (
	"context"
	"encoding/json"
	"fmt"
	"sort"
	"strings"
	"sync"
	"testing"

	appsv1 "k8s.io/api/apps/v1"
	corev1 "k8s.io/api/core/v1"
	apierrors "k8s.io/apimachinery/pkg/api/errors"
	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
	"k8s.io/client-go/kubernetes"
	"k8s.io/client-go/rest"
	"k8s.io/component-base/featuregate"
	"k8s.io/pod-security-admission/api"
	"k8s.io/pod-security-admission/policy"
)

// Options hold configuration for running integration tests against an existing server.
type Options struct {
	// ClientConfig is a client configuration with sufficient permission to create, update, and delete
	// namespaces, pods, and pod-template-containing objects.
	// Required.
	ClientConfig *rest.Config

	// Features optionally provides information about which feature gates are enabled.
	// This is used to skip failure cases for negative tests of data in alpha/beta fields.
	// If unset, all testcases are run.
	Features featuregate.FeatureGate

	// CreateNamespace is an optional stub for creating a namespace with the given name and labels.
	// Returning an error fails the test.
	// If nil, DefaultCreateNamespace is used.
	CreateNamespace func(client kubernetes.Interface, name string, labels map[string]string) (*corev1.Namespace, error)

	// These are the check ids/starting versions to exercise.
	// If unset, policy.DefaultChecks() are used.
	Checks []policy.Check

	// ExemptClient is an optional client interface to exercise behavior of an exempt client.
	ExemptClient kubernetes.Interface
	// ExemptNamespaces are optional namespaces not expected to have PodSecurity controls enforced.
	ExemptNamespaces []string
	// ExemptRuntimeClasses are optional runtimeclasses not expected to have PodSecurity controls enforced.
	ExemptRuntimeClasses []string
}

func toJSON(pod *corev1.Pod) string {
	data, _ := json.Marshal(pod)
	return string(data)
}

// checksForLevelAndVersion returns the set of check IDs that apply when evaluating the given level and version.
// checks are assumed to be well-formed and valid to pass to policy.NewEvaluator().
// level must be api.LevelRestricted or api.LevelBaseline
func checksForLevelAndVersion(checks []policy.Check, level api.Level, version api.Version) ([]string, error) {
	retval := []string{}
	for _, check := range checks {
		if !version.Older(check.Versions[0].MinimumVersion) && (level == check.Level || level == api.LevelRestricted) {
			retval = append(retval, check.ID)
		}
	}
	return retval, nil
}

// computeVersionsToTest returns all the versions that have distinct checks defined,
// all the versions that have distinct minimal valid pod fixtures defined, and
// any hard-coded versions that should always be tested.
//
// This lets us sparsely test all versions with distinct fixture or policy changes
// without needing to exercise every intermediate version that had no changes.
func computeVersionsToTest(t *testing.T, checks []policy.Check) []api.Version {
	seenVersions := map[api.Version]bool{}

	// include all versions we have registered distinct checks for
	for _, check := range checks {
		for _, checkVersion := range check.Versions {
			if checkVersion.MinimumVersion.Major() != 1 {
				t.Fatalf("expected major version 1, got %d", checkVersion.MinimumVersion.Major())
			}
			seenVersions[checkVersion.MinimumVersion] = true
		}
	}
	if len(seenVersions) == 0 {
		t.Fatal("no versions defined for checks")
	}

	// include all versions we have registered distinct fixtures for
	for _, versionsForLevel := range minimalValidPods {
		for version := range versionsForLevel {
			if version.Major() != 1 {
				t.Fatalf("expected major version 1, got %d", version.Major())
			}
			seenVersions[version] = true
		}
	}

	alwaysIncludeVersions := []api.Version{
		// include the oldest version by default
		api.MajorMinorVersion(1, 0),
		// include the release under development (1.22 at time of writing).
		// this can be incremented to the current version whenever is convenient.
		// TODO: find a way to use api.LatestVersion() here
		api.MajorMinorVersion(1, 22),
	}
	for _, version := range alwaysIncludeVersions {
		seenVersions[version] = true
	}

	versions := []api.Version{}
	for version := range seenVersions {
		versions = append(versions, version)
	}
	sort.Slice(versions, func(i, j int) bool { return versions[i].Older(versions[j]) })

	// TODO: consider exposing an option to test all versions instead of sparse ones

	return versions
}

type testWarningHandler struct {
	lock     sync.Mutex
	warnings []string
}

func (t *testWarningHandler) HandleWarningHeader(code int, agent string, warning string) {
	t.lock.Lock()
	defer t.lock.Unlock()
	t.warnings = append(t.warnings, warning)
}

func (t *testWarningHandler) FlushWarnings() []string {
	t.lock.Lock()
	defer t.lock.Unlock()
	warnings := t.warnings
	t.warnings = nil
	return warnings
}

// and ensures pod fixtures expected to pass and fail against that level/version work as expected.
func Run(t *testing.T, opts Options) {
	warningHandler := &testWarningHandler{}

	configCopy := rest.CopyConfig(opts.ClientConfig)
	configCopy.WarningHandler = warningHandler
	client, err := kubernetes.NewForConfig(configCopy)
	if err != nil {
		t.Fatalf("error creating client: %v", err)
	}

	if opts.CreateNamespace == nil {
		opts.CreateNamespace = DefaultCreateNamespace
	}
	if len(opts.Checks) == 0 {
		opts.Checks = policy.DefaultChecks()
	}
	_, err = policy.NewEvaluator(opts.Checks)
	if err != nil {
		t.Fatalf("invalid checks: %v", err)
	}

	versionsToTest := computeVersionsToTest(t, opts.Checks)

	for _, level := range []api.Level{api.LevelBaseline, api.LevelRestricted} {
		for _, version := range versionsToTest {
			minor := version.Minor()
			// create test name
			ns := fmt.Sprintf("podsecurity-%s-1-%d", level, minor)

			// create namespace
			_, err := opts.CreateNamespace(client, ns, map[string]string{
				api.EnforceLevelLabel:   string(level),
				api.EnforceVersionLabel: fmt.Sprintf("v1.%d", minor),
				api.WarnLevelLabel:      string(level),
				api.WarnVersionLabel:    fmt.Sprintf("v1.%d", minor),
			})
			if err != nil {
				t.Errorf("failed creating namespace %s: %v", ns, err)
				continue
			}
			t.Cleanup(func() {
				client.CoreV1().Namespaces().Delete(context.Background(), ns, metav1.DeleteOptions{})
			})

			// create service account (to allow pod to pass serviceaccount admission)
			sa, err := client.CoreV1().ServiceAccounts(ns).Create(
				context.Background(),
				&corev1.ServiceAccount{ObjectMeta: metav1.ObjectMeta{Name: "default"}},
				metav1.CreateOptions{},
			)
			if err != nil && !apierrors.IsAlreadyExists(err) {
				t.Errorf("failed creating serviceaccount %s: %v", ns, err)
				continue
			}
			t.Cleanup(func() {
				client.CoreV1().ServiceAccounts(ns).Delete(context.Background(), sa.Name, metav1.DeleteOptions{})
			})

			// create pod
			createPod := func(t *testing.T, i int, pod *corev1.Pod, expectSuccess bool, expectErrorSubstring string) {
				t.Helper()
				// avoid mutating original pod fixture
				pod = pod.DeepCopy()
				// assign pod name and serviceaccount
				pod.Name = "test"
				pod.Spec.ServiceAccountName = "default"
				// dry-run create
				_, err := client.CoreV1().Pods(ns).Create(context.Background(), pod, metav1.CreateOptions{DryRun: []string{metav1.DryRunAll}})
				if !expectSuccess {
					if err == nil {
						t.Errorf("%d: expected error creating %s, got none", i, toJSON(pod))
						return
					}
					if strings.Contains(err.Error(), policy.UnknownForbiddenReason) {
						t.Errorf("%d: unexpected unknown forbidden reason creating %s: %v", i, toJSON(pod), err)
						return
					}
					if !strings.Contains(err.Error(), expectErrorSubstring) {
						t.Errorf("%d: expected error with substring %q, got %v", i, expectErrorSubstring, err)
						return
					}
				}
				if expectSuccess && err != nil {
					t.Errorf("%d: unexpected error creating %s: %v", i, toJSON(pod), err)
				}
			}

			// create controller
			createController := func(t *testing.T, i int, pod *corev1.Pod, expectSuccess bool, expectErrorSubstring string) {
				t.Helper()
				// avoid mutating original pod fixture
				pod = pod.DeepCopy()
				if pod.Labels == nil {
					pod.Labels = map[string]string{}
				}
				pod.Labels["test"] = "true"

				warningHandler.FlushWarnings()
				// dry-run create
				deployment := &appsv1.Deployment{
					ObjectMeta: metav1.ObjectMeta{Name: "test"},
					Spec: appsv1.DeploymentSpec{
						Selector: &metav1.LabelSelector{MatchLabels: map[string]string{"test": "true"}},
						Template: corev1.PodTemplateSpec{
							ObjectMeta: pod.ObjectMeta,
							Spec:       pod.Spec,
						},
					},
				}
				_, err := client.AppsV1().Deployments(ns).Create(context.Background(), deployment, metav1.CreateOptions{DryRun: []string{metav1.DryRunAll}})
				if err != nil {
					t.Errorf("%d: unexpected error creating controller with %s: %v", i, toJSON(pod), err)
					return
				}
				warningText := strings.Join(warningHandler.FlushWarnings(), "; ")
				if !expectSuccess {
					if len(warningText) == 0 {
						t.Errorf("%d: expected warnings creating %s, got none", i, toJSON(pod))
						return
					}
					if strings.Contains(warningText, policy.UnknownForbiddenReason) {
						t.Errorf("%d: unexpected unknown forbidden reason creating %s: %v", i, toJSON(pod), warningText)
						return
					}
					if !strings.Contains(warningText, expectErrorSubstring) {
						t.Errorf("%d: expected warning with substring %q, got %v", i, expectErrorSubstring, warningText)
						return
					}
				}

				if expectSuccess && len(warningText) > 0 {
					if (len(expectErrorSubstring) > 0 && strings.Contains(warningText, expectErrorSubstring)) ||
						strings.Contains(warningText, policy.UnknownForbiddenReason) {
						t.Errorf("%d: unexpected warning creating %s: %v", i, toJSON(pod), warningText)
					}
				}
			}

			minimalValidPod, err := getMinimalValidPod(level, version)
			if err != nil {
				t.Fatal(err)
			}
			t.Run(ns+"_pass_base", func(t *testing.T) {
				createPod(t, 0, minimalValidPod.DeepCopy(), true, "")
				createController(t, 0, minimalValidPod.DeepCopy(), true, "")
			})

			checkIDs, err := checksForLevelAndVersion(opts.Checks, level, version)
			if err != nil {
				t.Fatal(err)
			}
			if len(checkIDs) == 0 {
				t.Fatal(fmt.Errorf("no checks registered for %s/1.%d", level, minor))
			}
			for _, checkID := range checkIDs {
				checkData, err := getFixtures(fixtureKey{level: level, version: version, check: checkID})
				if err != nil {
					t.Fatal(err)
				}

				t.Run(ns+"_pass_"+checkID, func(t *testing.T) {
					for i, pod := range checkData.pass {
						createPod(t, i, pod, true, "")
						createController(t, i, pod, true, "")
					}
				})

				// see if any features required for failure cases are disabled
				var disabledRequiredFeatures []featuregate.Feature
				for _, f := range checkData.failRequiresFeatures {
					if opts.Features != nil && !opts.Features.Enabled(f) {
						disabledRequiredFeatures = append(disabledRequiredFeatures, f)
					}
				}
				t.Run(ns+"_fail_"+checkID, func(t *testing.T) {
					if len(disabledRequiredFeatures) > 0 {
						t.Skipf("features required for failure cases are disabled: %v", disabledRequiredFeatures)
					}
					for i, pod := range checkData.fail {
						createPod(t, i, pod, false, checkData.expectErrorSubstring)
						createController(t, i, pod, false, checkData.expectErrorSubstring)
					}
				})
			}
		}
	}
}

func DefaultCreateNamespace(client kubernetes.Interface, name string, labels map[string]string) (*corev1.Namespace, error) {
	return client.CoreV1().Namespaces().Create(
		context.Background(),
		&corev1.Namespace{
			ObjectMeta: metav1.ObjectMeta{Name: name, Labels: labels},
		},
		metav1.CreateOptions{},
	)
}
<|endoftext|>


<commit_before>package data

import (
	"fmt"
	"reflect"
	"time"
	"unicode"
	"unicode/utf8"
)

var timeType = reflect.TypeOf(time.Time{})

// New converts the given data into a Soy data value, using
// DefaultStructOptions for structs.
func New(value interface{}) Value {
	return NewWith(DefaultStructOptions, value)
}

// NewWith converts the given data value Soy data value, using the provided
// StructOptions for any structs encountered.
func NewWith(convert StructOptions, value interface{}) Value {
	// quick return if we're passed an existing data.Value
	if val, ok := value.(Value); ok {
		return val
	}

	if value == nil {
		return Null{}
	}

	// see if value implements MarshalValue
	if mar, ok := value.(Marshaler); ok {
		return mar.MarshalValue()
	}

	// drill through pointers and interfaces to the underlying type
	var v = reflect.ValueOf(value)
	for v.Kind() == reflect.Interface || v.Kind() == reflect.Ptr {
		v = v.Elem()
	}
	if !v.IsValid() {
		return Null{}
	}

	if v.Type() == timeType {
		return String(v.Interface().(time.Time).Format(convert.TimeFormat))
	}

	switch v.Kind() {
	case reflect.Int, reflect.Int8, reflect.Int16, reflect.Int32, reflect.Int64:
		return Int(v.Int())
	case reflect.Uint, reflect.Uint8, reflect.Uint16, reflect.Uint32, reflect.Uint64:
		return Int(v.Uint())
	case reflect.Float32, reflect.Float64:
		return Float(v.Float())
	case reflect.Bool:
		return Bool(v.Bool())
	case reflect.String:
		return String(v.String())
	case reflect.Slice:
		if v.IsNil() {
			return List(nil)
		}
		slice := []Value{}
		for i := 0; i < v.Len(); i++ {
			slice = append(slice, NewWith(convert, v.Index(i).Interface()))
		}
		return List(slice)
	case reflect.Map:
		var m = make(map[string]Value)
		for _, key := range v.MapKeys() {
			if key.Kind() != reflect.String {
				panic("map keys must be strings")
			}
			m[key.String()] = NewWith(convert, v.MapIndex(key).Interface())
		}
		return Map(m)
	case reflect.Struct:
		return convert.Data(v.Interface())
	default:
		panic(fmt.Errorf("unexpected data type: %T (%v)", value, value))
	}
}

var DefaultStructOptions = StructOptions{
	LowerCamel: true,
	TimeFormat: time.RFC3339,
}

// StructOptions provides flexibility in conversion of structs to soy's
// data.Map format.
type StructOptions struct {
	LowerCamel bool   // if true, convert field names to lowerCamel.
	TimeFormat string // format string for time.Time. (if empty, use ISO-8601)
}

func (c StructOptions) Data(obj interface{}) Map {
	var m = make(map[string]Value)
	var v = reflect.ValueOf(obj)
	var valType = v.Type()
	for i := 0; i < valType.NumField(); i++ {
		if !v.Field(i).CanInterface() {
			continue
		}
		var key = valType.Field(i).Name
		if c.LowerCamel {
			var firstRune, size = utf8.DecodeRuneInString(key)
			key = string(unicode.ToLower(firstRune)) + key[size:]
		}
		m[key] = NewWith(c, v.Field(i).Interface())
	}
	return Map(m)
}

// Marshaler is the interface implemented by entities that can marshal
// themselves into a data.Value.
type Marshaler interface {
	MarshalValue() Value
}
<commit_msg>data/convert: pre-size containers for converted data<commit_after>package data

import (
	"fmt"
	"reflect"
	"time"
	"unicode"
	"unicode/utf8"
)

var timeType = reflect.TypeOf(time.Time{})

// New converts the given data into a Soy data value, using
// DefaultStructOptions for structs.
func New(value interface{}) Value {
	return NewWith(DefaultStructOptions, value)
}

// NewWith converts the given data value Soy data value, using the provided
// StructOptions for any structs encountered.
func NewWith(convert StructOptions, value interface{}) Value {
	// quick return if we're passed an existing data.Value
	if val, ok := value.(Value); ok {
		return val
	}

	if value == nil {
		return Null{}
	}

	// see if value implements MarshalValue
	if mar, ok := value.(Marshaler); ok {
		return mar.MarshalValue()
	}

	// drill through pointers and interfaces to the underlying type
	var v = reflect.ValueOf(value)
	for v.Kind() == reflect.Interface || v.Kind() == reflect.Ptr {
		v = v.Elem()
	}
	if !v.IsValid() {
		return Null{}
	}

	if v.Type() == timeType {
		return String(v.Interface().(time.Time).Format(convert.TimeFormat))
	}

	switch v.Kind() {
	case reflect.Int, reflect.Int8, reflect.Int16, reflect.Int32, reflect.Int64:
		return Int(v.Int())
	case reflect.Uint, reflect.Uint8, reflect.Uint16, reflect.Uint32, reflect.Uint64:
		return Int(v.Uint())
	case reflect.Float32, reflect.Float64:
		return Float(v.Float())
	case reflect.Bool:
		return Bool(v.Bool())
	case reflect.String:
		return String(v.String())
	case reflect.Slice:
		if v.IsNil() {
			return List(nil)
		}
		slice := make(List, v.Len())
		for i := range slice {
			slice[i] = NewWith(convert, v.Index(i).Interface())
		}
		return slice
	case reflect.Map:
		var keys = v.MapKeys()
		var m = make(Map, len(keys))
		for _, key := range keys {
			if key.Kind() != reflect.String {
				panic("map keys must be strings")
			}
			m[key.String()] = NewWith(convert, v.MapIndex(key).Interface())
		}
		return m
	case reflect.Struct:
		return convert.Data(v.Interface())
	default:
		panic(fmt.Errorf("unexpected data type: %T (%v)", value, value))
	}
}

var DefaultStructOptions = StructOptions{
	LowerCamel: true,
	TimeFormat: time.RFC3339,
}

// StructOptions provides flexibility in conversion of structs to soy's
// data.Map format.
type StructOptions struct {
	LowerCamel bool   // if true, convert field names to lowerCamel.
	TimeFormat string // format string for time.Time. (if empty, use ISO-8601)
}

func (c StructOptions) Data(obj interface{}) Map {
	var v = reflect.ValueOf(obj)
	var valType = v.Type()
	var n = valType.NumField()
	var m = make(Map, n)
	for i := 0; i < n; i++ {
		if !v.Field(i).CanInterface() {
			continue
		}
		var key = valType.Field(i).Name
		if c.LowerCamel {
			var firstRune, size = utf8.DecodeRuneInString(key)
			key = string(unicode.ToLower(firstRune)) + key[size:]
		}
		m[key] = NewWith(c, v.Field(i).Interface())
	}
	return m
}

// Marshaler is the interface implemented by entities that can marshal
// themselves into a data.Value.
type Marshaler interface {
	MarshalValue() Value
}
<|endoftext|>


<commit_before>package lua

import (
	"fmt"
	"reflect"
	"syscall"
	"unsafe"
)

var lua_pushnil = luaDLL.NewProc("lua_pushnil")

func (this Lua) PushNil() {
	lua_pushnil.Call(this.State())
}

var lua_pushboolean = luaDLL.NewProc("lua_pushboolean")

func (this Lua) PushBool(value bool) {
	if value {
		lua_pushboolean.Call(this.State(), 1)
	} else {
		lua_pushboolean.Call(this.State(), 0)
	}
}

var lua_pushinteger = luaDLL.NewProc("lua_pushinteger")

func (this Lua) PushInteger(value Integer) {
	params := make([]uintptr, 0, 4)
	params = append(params, this.State())
	params = value.Expand(params)
	lua_pushinteger.Call(params...)
}

var lua_pushlstring = luaDLL.NewProc("lua_pushlstring")

func (this Lua) PushBytes(data []byte) {
	if data != nil && len(data) >= 1 {
		lua_pushlstring.Call(this.State(),
			uintptr(unsafe.Pointer(&data[0])),
			uintptr(len(data)))
	} else {
		zerobyte := []byte{'\000'}
		lua_pushlstring.Call(this.State(),
			uintptr(unsafe.Pointer(&zerobyte[0])),
			0)
	}
}

var lua_pushstring = luaDLL.NewProc("lua_pushstring")

func (this Lua) PushString(str string) {
	// BytePtrFromString can not use the string which contains NUL
	array := make([]byte, len(str)+1)
	copy(array, str)
	lua_pushlstring.Call(this.State(),
		uintptr(unsafe.Pointer(&array[0])),
		uintptr(len(str)))
}

var lua_pushlightuserdata = luaDLL.NewProc("lua_pushlightuserdata")

func (this Lua) PushLightUserData(p unsafe.Pointer) {
	lua_pushlightuserdata.Call(this.State(), uintptr(p))
}

var lua_pushvalue = luaDLL.NewProc("lua_pushvalue")

func (this Lua) PushValue(index int) {
	lua_pushvalue.Call(this.State(), uintptr(index))
}

var lua_pushcclosure = luaDLL.NewProc("lua_pushcclosure")

func (this Lua) PushGoClosure(fn func(Lua) int, n uintptr) {
	lua_pushcclosure.Call(this.State(), syscall.NewCallbackCDecl(fn), n)
}

func (this Lua) PushGoFunction(fn func(Lua) int) {
	this.PushGoClosure(fn, 0)
}

func UpValueIndex(i int) int {
	return LUA_REGISTRYINDEX - i
}

type TGoFunction func(Lua) int

func (this TGoFunction) Push(L Lua) int {
	L.PushGoFunction(this)
	return 1
}

func (this Lua) PushCFunction(fn uintptr) {
	lua_pushcclosure.Call(this.State(), fn, 0)
}

type Object interface {
	Push(Lua) int
}

func (this Lua) Push(values ...interface{}) int {
	for _, value := range values {
		if value == nil {
			this.PushNil()
			continue
		}
		switch t := value.(type) {
		case bool:
			this.PushBool(t)
		case int:
			this.PushInteger(Integer(t))
		case int64:
			this.PushInteger(Integer(t))
		case string:
			this.PushString(t)
		case func(L Lua) int:
			this.PushGoFunction(t)
		case []byte:
			this.PushBytes(t)
		case error:
			this.PushString(t.Error())
		case TTable:
			this.NewTable()
			for key, val := range t.Dict {
				this.PushString(key)
				this.Push(val)
				this.SetTable(-3)
			}
			for key, val := range t.Array {
				this.Push(key)
				this.Push(val)
				this.SetTable(-3)
			}
		case unsafe.Pointer:
			this.PushLightUserData(t)
		case Object:
			t.Push(this)
		default:
			if !this.PushReflect(value) {
				panic(fmt.Sprintf(
					"lua.Lua.Push(%T): value is not supported type", t))
			}
		}
	}
	return len(values)
}

func (this Lua) PushReflect(value interface{}) bool {
	return this.pushReflect(reflect.ValueOf(value))
}

func (this Lua) pushReflect(value reflect.Value) bool {
	switch value.Kind() {
	case reflect.Int, reflect.Int8, reflect.Int16, reflect.Int32, reflect.Int64:
		this.PushInteger(Integer(value.Int()))
	case reflect.Uint, reflect.Uint16, reflect.Uint32,
		reflect.Uint64, reflect.Uintptr:
		this.PushInteger(Integer(value.Uint()))
	case reflect.Bool:
		this.PushBool(value.Bool())
	case reflect.String:
		this.PushString(value.String())
	case reflect.Interface:
		this.Push(value.Interface())
	case reflect.Slice, reflect.Array:
		this.NewTable()
		for i, end := 0, value.Len(); i < end; i++ {
			val := value.Index(i)
			this.PushInteger(Integer(i + 1))
			this.pushReflect(val)
			this.SetTable(-3)
		}
	case reflect.Map:
		this.NewTable()
		for _, key := range value.MapKeys() {
			this.pushReflect(key)
			val := value.MapIndex(key)
			this.pushReflect(val)
			this.SetTable(-3)
		}
	default:
		return false
	}
	return true
}
<commit_msg>Fix: lua.Lua.PushReflect(nil) != lua.Lua.PushNil()<commit_after>package lua

import (
	"fmt"
	"reflect"
	"syscall"
	"unsafe"
)

var lua_pushnil = luaDLL.NewProc("lua_pushnil")

func (this Lua) PushNil() {
	lua_pushnil.Call(this.State())
}

var lua_pushboolean = luaDLL.NewProc("lua_pushboolean")

func (this Lua) PushBool(value bool) {
	if value {
		lua_pushboolean.Call(this.State(), 1)
	} else {
		lua_pushboolean.Call(this.State(), 0)
	}
}

var lua_pushinteger = luaDLL.NewProc("lua_pushinteger")

func (this Lua) PushInteger(value Integer) {
	params := make([]uintptr, 0, 4)
	params = append(params, this.State())
	params = value.Expand(params)
	lua_pushinteger.Call(params...)
}

var lua_pushlstring = luaDLL.NewProc("lua_pushlstring")

func (this Lua) PushBytes(data []byte) {
	if data != nil && len(data) >= 1 {
		lua_pushlstring.Call(this.State(),
			uintptr(unsafe.Pointer(&data[0])),
			uintptr(len(data)))
	} else {
		zerobyte := []byte{'\000'}
		lua_pushlstring.Call(this.State(),
			uintptr(unsafe.Pointer(&zerobyte[0])),
			0)
	}
}

var lua_pushstring = luaDLL.NewProc("lua_pushstring")

func (this Lua) PushString(str string) {
	// BytePtrFromString can not use the string which contains NUL
	array := make([]byte, len(str)+1)
	copy(array, str)
	lua_pushlstring.Call(this.State(),
		uintptr(unsafe.Pointer(&array[0])),
		uintptr(len(str)))
}

var lua_pushlightuserdata = luaDLL.NewProc("lua_pushlightuserdata")

func (this Lua) PushLightUserData(p unsafe.Pointer) {
	lua_pushlightuserdata.Call(this.State(), uintptr(p))
}

var lua_pushvalue = luaDLL.NewProc("lua_pushvalue")

func (this Lua) PushValue(index int) {
	lua_pushvalue.Call(this.State(), uintptr(index))
}

var lua_pushcclosure = luaDLL.NewProc("lua_pushcclosure")

func (this Lua) PushGoClosure(fn func(Lua) int, n uintptr) {
	lua_pushcclosure.Call(this.State(), syscall.NewCallbackCDecl(fn), n)
}

func (this Lua) PushGoFunction(fn func(Lua) int) {
	this.PushGoClosure(fn, 0)
}

func UpValueIndex(i int) int {
	return LUA_REGISTRYINDEX - i
}

type TGoFunction func(Lua) int

func (this TGoFunction) Push(L Lua) int {
	L.PushGoFunction(this)
	return 1
}

func (this Lua) PushCFunction(fn uintptr) {
	lua_pushcclosure.Call(this.State(), fn, 0)
}

type Object interface {
	Push(Lua) int
}

func (this Lua) Push(values ...interface{}) int {
	for _, value := range values {
		if value == nil {
			this.PushNil()
			continue
		}
		switch t := value.(type) {
		case bool:
			this.PushBool(t)
		case int:
			this.PushInteger(Integer(t))
		case int64:
			this.PushInteger(Integer(t))
		case string:
			this.PushString(t)
		case func(L Lua) int:
			this.PushGoFunction(t)
		case []byte:
			this.PushBytes(t)
		case error:
			this.PushString(t.Error())
		case TTable:
			this.NewTable()
			for key, val := range t.Dict {
				this.PushString(key)
				this.Push(val)
				this.SetTable(-3)
			}
			for key, val := range t.Array {
				this.Push(key)
				this.Push(val)
				this.SetTable(-3)
			}
		case unsafe.Pointer:
			this.PushLightUserData(t)
		case Object:
			t.Push(this)
		default:
			if !this.PushReflect(value) {
				panic(fmt.Sprintf(
					"lua.Lua.Push(%T): value is not supported type", t))
			}
		}
	}
	return len(values)
}

func (this Lua) PushReflect(value interface{}) bool {
	if value == nil {
		this.PushNil()
	}
	return this.pushReflect(reflect.ValueOf(value))
}

func (this Lua) pushReflect(value reflect.Value) bool {
	switch value.Kind() {
	case reflect.Int, reflect.Int8, reflect.Int16, reflect.Int32, reflect.Int64:
		this.PushInteger(Integer(value.Int()))
	case reflect.Uint, reflect.Uint16, reflect.Uint32,
		reflect.Uint64, reflect.Uintptr:
		this.PushInteger(Integer(value.Uint()))
	case reflect.Bool:
		this.PushBool(value.Bool())
	case reflect.String:
		this.PushString(value.String())
	case reflect.Interface:
		this.Push(value.Interface())
	case reflect.Slice, reflect.Array:
		this.NewTable()
		for i, end := 0, value.Len(); i < end; i++ {
			val := value.Index(i)
			this.PushInteger(Integer(i + 1))
			this.pushReflect(val)
			this.SetTable(-3)
		}
	case reflect.Map:
		this.NewTable()
		for _, key := range value.MapKeys() {
			this.pushReflect(key)
			val := value.MapIndex(key)
			this.pushReflect(val)
			this.SetTable(-3)
		}
	default:
		return false
	}
	return true
}
<|endoftext|>


<commit_before>package client

import (
	"bytes"
	"fmt"
	"net/http"
)

type Client struct {
	client http.Client
}

func NewClient(client http.Client) Client {
	return Client{
		client: client,
	}
}

func (c *Client) DownloadByMagnet(clientURL string, magnet string) bool {
	var jsonStr = []byte(fmt.Sprintf(`{"magnet":"%v"}`, magnet))
	resp, err := c.client.Post(clientURL, "application/json", bytes.NewBuffer(jsonStr))

	if err != nil {
		return false
	}

	return resp.StatusCode == 200
}
<commit_msg>Changed client constructor<commit_after>package client

import (
	"bytes"
	"fmt"
	"net/http"
)

type Client struct {
	client    http.Client
	clientURL string
}

func NewClient(client http.Client, clientURL string) Client {
	return Client{
		client:    client,
		clientURL: clientURL,
	}
}

func (c *Client) DownloadByMagnet(magnet string) bool {
	var jsonStr = []byte(fmt.Sprintf(`{"magnet":"%v"}`, magnet))
	resp, err := c.client.Post(c.clientURL, "application/json", bytes.NewBuffer(jsonStr))

	if err != nil {
		return false
	}

	return resp.StatusCode == 200
}
<|endoftext|>


<commit_before>package main

import (
	"encoding/json"
	"path"

	"github.com/gosexy/gettext"
	"github.com/lxc/lxd"
)

type moveCmd struct {
	httpAddr string
}

func (c *moveCmd) showByDefault() bool {
	return false
}

func (c *moveCmd) usage() string {
	return gettext.Gettext(
		"Move containers within or in between lxd instances.\n" +
			"\n" +
			"(currently only live migration is supported)\n" +
			"lxc move <source container> <destination container>\n")
}

func (c *moveCmd) flags() {}

func (c *moveCmd) run(config *lxd.Config, args []string) error {
	if len(args) != 2 {
		return errArgs
	}

	sourceRemote, sourceName := config.ParseRemoteAndContainer(args[0])
	destRemote, destName := config.ParseRemoteAndContainer(args[1])

	source, err := lxd.NewClient(config, sourceRemote)
	if err != nil {
		return err
	}

	dest, err := lxd.NewClient(config, destRemote)
	if err != nil {
		return err
	}

	to, err := source.MigrateTo(sourceName, dest)
	if err != nil {
		return err
	}

	secrets := map[string]string{}
	if err := json.Unmarshal(to.Metadata, &secrets); err != nil {
		return err
	}

	url := "wss://" + source.Remote.Addr + path.Join(to.Operation, "websocket")
	migration, err := dest.MigrateFrom(destName, url, secrets)
	if err != nil {
		return err
	}

	return dest.WaitForSuccess(migration.Operation)
}
<commit_msg>Warn users when trying to migrate on non-http remotes<commit_after>package main

import (
	"encoding/json"
	"fmt"
	"path"

	"github.com/gosexy/gettext"
	"github.com/lxc/lxd"
)

type moveCmd struct {
	httpAddr string
}

func (c *moveCmd) showByDefault() bool {
	return false
}

func (c *moveCmd) usage() string {
	return gettext.Gettext(
		"Move containers within or in between lxd instances.\n" +
			"\n" +
			"(currently only live migration is supported)\n" +
			"lxc move <source container> <destination container>\n")
}

func (c *moveCmd) flags() {}

func (c *moveCmd) run(config *lxd.Config, args []string) error {
	if len(args) != 2 {
		return errArgs
	}

	sourceRemote, sourceName := config.ParseRemoteAndContainer(args[0])
	destRemote, destName := config.ParseRemoteAndContainer(args[1])

	if sourceRemote == "" || destRemote == "" {
		return fmt.Errorf("non-http remotes are not supported for migration right now")
	}

	source, err := lxd.NewClient(config, sourceRemote)
	if err != nil {
		return err
	}

	dest, err := lxd.NewClient(config, destRemote)
	if err != nil {
		return err
	}

	to, err := source.MigrateTo(sourceName, dest)
	if err != nil {
		return err
	}

	secrets := map[string]string{}
	if err := json.Unmarshal(to.Metadata, &secrets); err != nil {
		return err
	}

	url := "wss://" + source.Remote.Addr + path.Join(to.Operation, "websocket")
	migration, err := dest.MigrateFrom(destName, url, secrets)
	if err != nil {
		return err
	}

	return dest.WaitForSuccess(migration.Operation)
}
<|endoftext|>


<commit_before>/* a-b-tester is a utility program that makes it easy to test and see if a
/* change to the core library is helping us build a better model. Normal usage
/* is to provide it a relativedifficulties.csv file and then it will output
/* r2, but you can also compare multiple configs and have it report the best
/* one. To do that, create different branches with each configuration set.
/* Then run a-b-tester with -b and a space delimited string of branch names to
/* try. a-b-tester will run each in turn, save out analysis and solves files
/* for each, and then report which one has the best r2.*/

package main

import (
	"errors"
	"flag"
	"fmt"
	"github.com/gosuri/uitable"
	"log"
	"os"
	"os/exec"
	"path"
	"strconv"
	"strings"
)

const pathToDokugenAnalysis = "../../"
const pathFromDokugenAnalysis = "internal/a-b-tester/"

const pathToWekaTrainer = "../weka-trainer/"
const pathFromWekaTrainer = "../a-b-tester/"

const rowSeparator = "****************"

const uncommittedChangesBranchName = "STASHED"
const committedChangesBranchName = "COMMITTED"

//TODO: amek this resilient to not being run in the package's directory

type appOptions struct {
	relativeDifficultiesFile       string
	solvesFile                     string
	analysisFile                   string
	stashMode                      bool
	startingWithUncommittedChanges bool
	branches                       string
	branchesList                   []string
	help                           bool
	flagSet                        *flag.FlagSet
}

func (a *appOptions) defineFlags() {
	if a.flagSet == nil {
		return
	}
	a.flagSet.BoolVar(&a.stashMode, "s", false, "If in stash mode, will do the a-b test between uncommitted and committed changes, automatically figuring out which state we're currently in. Cannot be combined with -b")
	a.flagSet.StringVar(&a.branches, "b", "", "Git branch to checkout. Can also be a space delimited list of multiple branches to checkout.")
	a.flagSet.StringVar(&a.relativeDifficultiesFile, "r", "relativedifficulties_SAMPLED.csv", "The file to use as relative difficulties input")
	a.flagSet.StringVar(&a.solvesFile, "o", "solves.csv", "The file to output solves to")
	a.flagSet.StringVar(&a.analysisFile, "a", "analysis.txt", "The file to output analysis to")
	a.flagSet.BoolVar(&a.help, "h", false, "If provided, will print help and exit.")
}

func (a *appOptions) fixUp() error {
	if a.branches != "" && a.stashMode {
		return errors.New("-b and -s cannot both be passed")
	}
	if a.stashMode {
		a.startingWithUncommittedChanges = gitUncommittedChanges()
		if a.startingWithUncommittedChanges {
			a.branchesList = []string{
				uncommittedChangesBranchName,
				committedChangesBranchName,
			}
		} else {
			a.branchesList = []string{
				committedChangesBranchName,
				uncommittedChangesBranchName,
			}
		}
	} else {
		a.branchesList = strings.Split(a.branches, " ")
	}
	a.solvesFile = strings.Replace(a.solvesFile, ".csv", "", -1)
	a.analysisFile = strings.Replace(a.analysisFile, ".txt", "", -1)
	return nil
}

func (a *appOptions) parse(args []string) error {
	a.flagSet.Parse(args)
	return a.fixUp()
}

func newAppOptions(flagSet *flag.FlagSet) *appOptions {
	a := &appOptions{
		flagSet: flagSet,
	}
	a.defineFlags()
	return a
}

func main() {
	a := newAppOptions(flag.CommandLine)
	if err := a.parse(os.Args[1:]); err != nil {
		log.Println("Invalid options provided:", err.Error())
		return
	}

	results := make(map[string]float64)

	startingBranch := gitCurrentBranch()

	for i, branch := range a.branchesList {

		if branch == "" {
			log.Println("Staying on the current branch.")
		} else {
			log.Println("Switching to branch", branch)
		}

		//a.analysisFile and a.solvesFile have had their extension removed, if they had one.
		effectiveSolvesFile := a.solvesFile + ".csv"
		effectiveAnalysisFile := a.analysisFile + ".txt"

		if branch != "" {

			effectiveSolvesFile = a.solvesFile + "_" + strings.ToUpper(branch) + ".csv"
			effectiveAnalysisFile = a.analysisFile + "_" + strings.ToUpper(branch) + ".txt"
		}

		if a.stashMode {
			// if i == 0
			switch i {
			case 0:
				//do nothing, we already ahve the right changes.
			case 1:
				//If we have uncommitted changes right now, stash them. Otherwise, stash pop.
				gitStash(a.startingWithUncommittedChanges)
			default:
				//This should never happen
				//Note: panicing here will mean we don't do any clean up.
				panic("Got more than 2 'branches' in stash mode")
			}
		} else {
			if !checkoutGitBranch(branch) {
				log.Println("Couldn't switch to branch", branch, " (perhaps you have uncommitted changes?). Quitting.")
				return
			}
		}

		runSolves(a.relativeDifficultiesFile, effectiveSolvesFile)

		branchKey := branch

		if branchKey == "" {
			branchKey = "<default>"
		}

		results[branchKey] = runWeka(effectiveSolvesFile, effectiveAnalysisFile)
	}

	if len(results) > 1 {
		//We only need to go to the trouble of painting the table if more than
		//one branch was run
		printR2Table(results)
	}

	//Put the repo back in the state it was when we found it.
	if a.stashMode {
		//Reverse the gitStash operation to put it back
		gitStash(!a.startingWithUncommittedChanges)
	} else {
		//If we aren't in the branch we started in, switch back to that branch
		if gitCurrentBranch() != startingBranch {
			checkoutGitBranch(startingBranch)
		}
	}

}

func printR2Table(results map[string]float64) {
	bestR2 := 0.0
	bestR2Branch := ""

	for key, val := range results {
		if val > bestR2 {
			bestR2 = val
			bestR2Branch = key
		}
	}

	fmt.Println(rowSeparator)
	fmt.Println("Results:")
	fmt.Println(rowSeparator)

	table := uitable.New()

	table.AddRow("Best?", "Branch", "R2")

	for key, val := range results {
		isBest := " "
		if key == bestR2Branch {
			isBest = "*"
		}
		table.AddRow(isBest, key, val)
	}

	fmt.Println(table.String())
	fmt.Println(rowSeparator)
}

func runSolves(difficultiesFile, solvesOutputFile string) {

	os.Chdir(pathToDokugenAnalysis)

	defer func() {
		os.Chdir(pathFromDokugenAnalysis)
	}()

	//Build the dokugen-analysis executable to make sure we get the freshest version of the sudoku pacakge.
	cmd := exec.Command("go", "build")
	err := cmd.Run()

	if err != nil {
		log.Println(err)
		return
	}

	outFile, err := os.Create(path.Join(pathFromDokugenAnalysis, solvesOutputFile))

	if err != nil {
		log.Println(err)
		return
	}

	analysisCmd := exec.Command("./dokugen-analysis", "-a", "-v", "-w", "-t", "-h", "-no-cache", path.Join(pathFromDokugenAnalysis, difficultiesFile))
	analysisCmd.Stdout = outFile
	analysisCmd.Stderr = os.Stderr
	err = analysisCmd.Run()

	if err != nil {
		log.Println(err)
	}
}

func runWeka(solvesFile string, analysisFile string) float64 {

	os.Chdir(pathToWekaTrainer)

	defer func() {
		os.Chdir(pathFromWekaTrainer)
	}()

	//Build the weka-trainer executable to make sure we get the freshest version of the sudoku pacakge.
	cmd := exec.Command("go", "build")
	err := cmd.Run()

	if err != nil {
		log.Println(err)
		return 0.0
	}

	trainCmd := exec.Command("./weka-trainer", "-i", path.Join(pathFromWekaTrainer, solvesFile), "-o", path.Join(pathFromWekaTrainer, analysisFile))
	trainCmd.Stderr = os.Stderr
	output, err := trainCmd.Output()

	if err != nil {
		log.Println(err)
		return 0.0
	}

	fmt.Printf("%s", string(output))

	return extractR2(string(output))

}

//extractR2 extracts R2 out of the string formatted like "R2 = <float>"
func extractR2(input string) float64 {

	input = strings.TrimPrefix(input, "R2 = ")
	input = strings.TrimSpace(input)

	result, _ := strconv.ParseFloat(input, 64)

	return result

}

//gitStash will use git stash if true, git stash pop if false.
func gitStash(stashChanges bool) bool {
	var stashCmd *exec.Cmd

	if stashChanges {
		stashCmd = exec.Command("git", "stash")
		if !gitUncommittedChanges() {
			//That's weird, there aren't any changes to stash
			return false
		}
	} else {
		stashCmd = exec.Command("git", "stash", "pop")
		if gitUncommittedChanges() {
			//That's weird, there are uncommitted changes that this would overwrite.
			return false
		}
	}

	err := stashCmd.Run()

	if err != nil {
		log.Println(err)
		return false
	}

	//Verify it worked
	if stashChanges {
		//Stashing apaprently didn't work
		if gitUncommittedChanges() {
			return false
		}
	} else {
		//Weird, stash popping didn't do anything.
		if !gitUncommittedChanges() {
			return false
		}
	}

	return true
}

//Returns true if there are currently uncommitted changes
func gitUncommittedChanges() bool {

	statusCmd := exec.Command("git", "status", "-s")

	output, err := statusCmd.Output()

	if err != nil {
		log.Println(err)
		return false
	}

	//In git status -s(hort), each line starts with two characters. ?? is hte
	//only prefix that we should ignore, since it means untracked files.

	for _, line := range strings.Split(string(output), "\n") {
		if !strings.HasPrefix(line, "??") {
			//Found a non-committed change
			return true
		}
	}

	return false

}

//gitCurrentBranch returns the current branch that the current repo is in.
func gitCurrentBranch() string {
	branchCmd := exec.Command("git", "branch")

	output, err := branchCmd.Output()

	if err != nil {
		log.Println(err)
		return ""
	}

	for _, line := range strings.Split(string(output), "\n") {
		if strings.Contains(line, "*") {
			//Found it!
			line = strings.Replace(line, "*", "", -1)
			line = strings.TrimSpace(line)
			return line
		}
	}

	return ""
}

func checkoutGitBranch(branch string) bool {

	if branch == "" {
		return true
	}

	checkoutCmd := exec.Command("git", "checkout", branch)
	checkoutCmd.Run()

	if gitCurrentBranch() != branch {
		return false
	}

	return true

}
<commit_msg>Unbelieveably, it mostly works, except for putting things back in the right state. Added a bit more error reporting for that case.<commit_after>/* a-b-tester is a utility program that makes it easy to test and see if a
/* change to the core library is helping us build a better model. Normal usage
/* is to provide it a relativedifficulties.csv file and then it will output
/* r2, but you can also compare multiple configs and have it report the best
/* one. To do that, create different branches with each configuration set.
/* Then run a-b-tester with -b and a space delimited string of branch names to
/* try. a-b-tester will run each in turn, save out analysis and solves files
/* for each, and then report which one has the best r2.*/

package main

import (
	"errors"
	"flag"
	"fmt"
	"github.com/gosuri/uitable"
	"log"
	"os"
	"os/exec"
	"path"
	"strconv"
	"strings"
)

const pathToDokugenAnalysis = "../../"
const pathFromDokugenAnalysis = "internal/a-b-tester/"

const pathToWekaTrainer = "../weka-trainer/"
const pathFromWekaTrainer = "../a-b-tester/"

const rowSeparator = "****************"

const uncommittedChangesBranchName = "STASHED"
const committedChangesBranchName = "COMMITTED"

//TODO: amek this resilient to not being run in the package's directory

type appOptions struct {
	relativeDifficultiesFile       string
	solvesFile                     string
	analysisFile                   string
	stashMode                      bool
	startingWithUncommittedChanges bool
	branches                       string
	branchesList                   []string
	help                           bool
	flagSet                        *flag.FlagSet
}

func (a *appOptions) defineFlags() {
	if a.flagSet == nil {
		return
	}
	a.flagSet.BoolVar(&a.stashMode, "s", false, "If in stash mode, will do the a-b test between uncommitted and committed changes, automatically figuring out which state we're currently in. Cannot be combined with -b")
	a.flagSet.StringVar(&a.branches, "b", "", "Git branch to checkout. Can also be a space delimited list of multiple branches to checkout.")
	a.flagSet.StringVar(&a.relativeDifficultiesFile, "r", "relativedifficulties_SAMPLED.csv", "The file to use as relative difficulties input")
	a.flagSet.StringVar(&a.solvesFile, "o", "solves.csv", "The file to output solves to")
	a.flagSet.StringVar(&a.analysisFile, "a", "analysis.txt", "The file to output analysis to")
	a.flagSet.BoolVar(&a.help, "h", false, "If provided, will print help and exit.")
}

func (a *appOptions) fixUp() error {
	if a.branches != "" && a.stashMode {
		return errors.New("-b and -s cannot both be passed")
	}
	if a.stashMode {
		a.startingWithUncommittedChanges = gitUncommittedChanges()
		if a.startingWithUncommittedChanges {
			a.branchesList = []string{
				uncommittedChangesBranchName,
				committedChangesBranchName,
			}
		} else {
			a.branchesList = []string{
				committedChangesBranchName,
				uncommittedChangesBranchName,
			}
		}
	} else {
		a.branchesList = strings.Split(a.branches, " ")
	}
	a.solvesFile = strings.Replace(a.solvesFile, ".csv", "", -1)
	a.analysisFile = strings.Replace(a.analysisFile, ".txt", "", -1)
	return nil
}

func (a *appOptions) parse(args []string) error {
	a.flagSet.Parse(args)
	return a.fixUp()
}

func newAppOptions(flagSet *flag.FlagSet) *appOptions {
	a := &appOptions{
		flagSet: flagSet,
	}
	a.defineFlags()
	return a
}

func main() {
	a := newAppOptions(flag.CommandLine)
	if err := a.parse(os.Args[1:]); err != nil {
		log.Println("Invalid options provided:", err.Error())
		return
	}

	results := make(map[string]float64)

	startingBranch := gitCurrentBranch()

	for i, branch := range a.branchesList {

		if branch == "" {
			log.Println("Staying on the current branch.")
		} else {
			log.Println("Switching to branch", branch)
		}

		//a.analysisFile and a.solvesFile have had their extension removed, if they had one.
		effectiveSolvesFile := a.solvesFile + ".csv"
		effectiveAnalysisFile := a.analysisFile + ".txt"

		if branch != "" {

			effectiveSolvesFile = a.solvesFile + "_" + strings.ToUpper(branch) + ".csv"
			effectiveAnalysisFile = a.analysisFile + "_" + strings.ToUpper(branch) + ".txt"
		}

		if a.stashMode {
			// if i == 0
			switch i {
			case 0:
				//do nothing, we already ahve the right changes.
			case 1:
				//If we have uncommitted changes right now, stash them. Otherwise, stash pop.
				if !gitStash(a.startingWithUncommittedChanges) {
					log.Println("We couldn't stash/stash-pop.")
					return
				}
			default:
				//This should never happen
				//Note: panicing here will mean we don't do any clean up.
				panic("Got more than 2 'branches' in stash mode")
			}
		} else {
			if !checkoutGitBranch(branch) {
				log.Println("Couldn't switch to branch", branch, " (perhaps you have uncommitted changes?). Quitting.")
				return
			}
		}

		runSolves(a.relativeDifficultiesFile, effectiveSolvesFile)

		branchKey := branch

		if branchKey == "" {
			branchKey = "<default>"
		}

		results[branchKey] = runWeka(effectiveSolvesFile, effectiveAnalysisFile)
	}

	if len(results) > 1 {
		//We only need to go to the trouble of painting the table if more than
		//one branch was run
		printR2Table(results)
	}

	//Put the repo back in the state it was when we found it.
	if a.stashMode {
		//Reverse the gitStash operation to put it back
		if !gitStash(!a.startingWithUncommittedChanges) {
			log.Println("We couldn't unstash/unpop to put the repo back in the same state.")
		}
	} else {
		//If we aren't in the branch we started in, switch back to that branch
		if gitCurrentBranch() != startingBranch {
			checkoutGitBranch(startingBranch)
		}
	}

}

func printR2Table(results map[string]float64) {
	bestR2 := 0.0
	bestR2Branch := ""

	for key, val := range results {
		if val > bestR2 {
			bestR2 = val
			bestR2Branch = key
		}
	}

	fmt.Println(rowSeparator)
	fmt.Println("Results:")
	fmt.Println(rowSeparator)

	table := uitable.New()

	table.AddRow("Best?", "Branch", "R2")

	for key, val := range results {
		isBest := " "
		if key == bestR2Branch {
			isBest = "*"
		}
		table.AddRow(isBest, key, val)
	}

	fmt.Println(table.String())
	fmt.Println(rowSeparator)
}

func runSolves(difficultiesFile, solvesOutputFile string) {

	os.Chdir(pathToDokugenAnalysis)

	defer func() {
		os.Chdir(pathFromDokugenAnalysis)
	}()

	//Build the dokugen-analysis executable to make sure we get the freshest version of the sudoku pacakge.
	cmd := exec.Command("go", "build")
	err := cmd.Run()

	if err != nil {
		log.Println(err)
		return
	}

	outFile, err := os.Create(path.Join(pathFromDokugenAnalysis, solvesOutputFile))

	if err != nil {
		log.Println(err)
		return
	}

	analysisCmd := exec.Command("./dokugen-analysis", "-a", "-v", "-w", "-t", "-h", "-no-cache", path.Join(pathFromDokugenAnalysis, difficultiesFile))
	analysisCmd.Stdout = outFile
	analysisCmd.Stderr = os.Stderr
	err = analysisCmd.Run()

	if err != nil {
		log.Println(err)
	}
}

func runWeka(solvesFile string, analysisFile string) float64 {

	os.Chdir(pathToWekaTrainer)

	defer func() {
		os.Chdir(pathFromWekaTrainer)
	}()

	//Build the weka-trainer executable to make sure we get the freshest version of the sudoku pacakge.
	cmd := exec.Command("go", "build")
	err := cmd.Run()

	if err != nil {
		log.Println(err)
		return 0.0
	}

	trainCmd := exec.Command("./weka-trainer", "-i", path.Join(pathFromWekaTrainer, solvesFile), "-o", path.Join(pathFromWekaTrainer, analysisFile))
	trainCmd.Stderr = os.Stderr
	output, err := trainCmd.Output()

	if err != nil {
		log.Println(err)
		return 0.0
	}

	fmt.Printf("%s", string(output))

	return extractR2(string(output))

}

//extractR2 extracts R2 out of the string formatted like "R2 = <float>"
func extractR2(input string) float64 {

	input = strings.TrimPrefix(input, "R2 = ")
	input = strings.TrimSpace(input)

	result, _ := strconv.ParseFloat(input, 64)

	return result

}

//gitStash will use git stash if true, git stash pop if false.
func gitStash(stashChanges bool) bool {
	var stashCmd *exec.Cmd

	if stashChanges {
		stashCmd = exec.Command("git", "stash")
		if !gitUncommittedChanges() {
			//That's weird, there aren't any changes to stash
			return false
		}
	} else {
		stashCmd = exec.Command("git", "stash", "pop")
		if gitUncommittedChanges() {
			//That's weird, there are uncommitted changes that this would overwrite.
			return false
		}
	}

	err := stashCmd.Run()

	if err != nil {
		log.Println(err)
		return false
	}

	//Verify it worked
	if stashChanges {
		//Stashing apaprently didn't work
		if gitUncommittedChanges() {
			return false
		}
	} else {
		//Weird, stash popping didn't do anything.
		if !gitUncommittedChanges() {
			return false
		}
	}

	return true
}

//Returns true if there are currently uncommitted changes
func gitUncommittedChanges() bool {

	statusCmd := exec.Command("git", "status", "-s")

	output, err := statusCmd.Output()

	if err != nil {
		log.Println(err)
		return false
	}

	//In git status -s(hort), each line starts with two characters. ?? is hte
	//only prefix that we should ignore, since it means untracked files.

	for _, line := range strings.Split(string(output), "\n") {
		if !strings.HasPrefix(line, "??") {
			//Found a non-committed change
			return true
		}
	}

	return false

}

//gitCurrentBranch returns the current branch that the current repo is in.
func gitCurrentBranch() string {
	branchCmd := exec.Command("git", "branch")

	output, err := branchCmd.Output()

	if err != nil {
		log.Println(err)
		return ""
	}

	for _, line := range strings.Split(string(output), "\n") {
		if strings.Contains(line, "*") {
			//Found it!
			line = strings.Replace(line, "*", "", -1)
			line = strings.TrimSpace(line)
			return line
		}
	}

	return ""
}

func checkoutGitBranch(branch string) bool {

	if branch == "" {
		return true
	}

	checkoutCmd := exec.Command("git", "checkout", branch)
	checkoutCmd.Run()

	if gitCurrentBranch() != branch {
		return false
	}

	return true

}
<|endoftext|>


<commit_before>/*
Copyright 2017 The Kubernetes Authors.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package apiconfig

import (
	"fmt"

	apierrors "k8s.io/apimachinery/pkg/api/errors"
	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
	clientset "k8s.io/client-go/kubernetes"
	"k8s.io/client-go/pkg/api/v1"
	rbac "k8s.io/client-go/pkg/apis/rbac/v1beta1"
	kubeadmconstants "k8s.io/kubernetes/cmd/kubeadm/app/constants"
	kubeadmutil "k8s.io/kubernetes/cmd/kubeadm/app/util"
	bootstrapapi "k8s.io/kubernetes/pkg/bootstrap/api"
	"k8s.io/kubernetes/pkg/util/version"
)

const (
	// KubeProxyClusterRoleName sets the name for the kube-proxy ClusterRole
	KubeProxyClusterRoleName = "system:node-proxier"
	// NodeBootstrapperClusterRoleName sets the name for the TLS Node Bootstrapper ClusterRole
	NodeBootstrapperClusterRoleName = "system:node-bootstrapper"
	// BootstrapSignerClusterRoleName sets the name for the ClusterRole that allows access to ConfigMaps in the kube-public ns
	BootstrapSignerClusterRoleName = "system:bootstrap-signer-clusterinfo"

	clusterRoleKind          = "ClusterRole"
	roleKind                 = "Role"
	serviceAccountKind       = "ServiceAccount"
	rbacAPIGroup             = "rbac.authorization.k8s.io"
	anonymousUser            = "system:anonymous"
	nodeAutoApproveBootstrap = "kubeadm:node-autoapprove-bootstrap"
)

// TODO: Are there any unit tests that could be made for this file other than duplicating all values and logic in a separate file?

// CreateServiceAccounts creates the necessary serviceaccounts that kubeadm uses/might use, if they don't already exist.
func CreateServiceAccounts(clientset clientset.Interface) error {
	serviceAccounts := []v1.ServiceAccount{
		{
			ObjectMeta: metav1.ObjectMeta{
				Name:      kubeadmconstants.KubeDNSServiceAccountName,
				Namespace: metav1.NamespaceSystem,
			},
		},
		{
			ObjectMeta: metav1.ObjectMeta{
				Name:      kubeadmconstants.KubeProxyServiceAccountName,
				Namespace: metav1.NamespaceSystem,
			},
		},
	}

	for _, sa := range serviceAccounts {
		if _, err := clientset.CoreV1().ServiceAccounts(metav1.NamespaceSystem).Create(&sa); err != nil {
			if !apierrors.IsAlreadyExists(err) {
				return err
			}
		}
	}
	return nil
}

// CreateRBACRules creates the essential RBAC rules for a minimally set-up cluster
func CreateRBACRules(clientset *clientset.Clientset, k8sVersion *version.Version) error {
	if err := createRoles(clientset); err != nil {
		return err
	}
	if err := createRoleBindings(clientset); err != nil {
		return err
	}
	if err := createClusterRoles(clientset); err != nil {
		return err
	}
	if err := createClusterRoleBindings(clientset); err != nil {
		return err
	}
	if err := deletePermissiveNodesBindingWhenUsingNodeAuthorization(clientset, k8sVersion); err != nil {
		return fmt.Errorf("failed to remove the permissive 'system:nodes' Group Subject in the 'system:node' ClusterRoleBinding: %v", err)
	}

	fmt.Println("[apiconfig] Created RBAC rules")
	return nil
}

func createRoles(clientset *clientset.Clientset) error {
	roles := []rbac.Role{
		{
			ObjectMeta: metav1.ObjectMeta{
				Name:      BootstrapSignerClusterRoleName,
				Namespace: metav1.NamespacePublic,
			},
			Rules: []rbac.PolicyRule{
				rbac.NewRule("get").Groups("").Resources("configmaps").RuleOrDie(),
			},
		},
	}
	for _, role := range roles {
		if _, err := clientset.RbacV1beta1().Roles(role.ObjectMeta.Namespace).Create(&role); err != nil {
			if !apierrors.IsAlreadyExists(err) {
				return fmt.Errorf("unable to create RBAC role: %v", err)
			}

			if _, err := clientset.RbacV1beta1().Roles(role.ObjectMeta.Namespace).Update(&role); err != nil {
				return fmt.Errorf("unable to update RBAC role: %v", err)
			}
		}
	}
	return nil
}

func createRoleBindings(clientset *clientset.Clientset) error {
	roleBindings := []rbac.RoleBinding{
		{
			ObjectMeta: metav1.ObjectMeta{
				Name:      "kubeadm:bootstrap-signer-clusterinfo",
				Namespace: metav1.NamespacePublic,
			},
			RoleRef: rbac.RoleRef{
				APIGroup: rbacAPIGroup,
				Kind:     roleKind,
				Name:     BootstrapSignerClusterRoleName,
			},
			Subjects: []rbac.Subject{
				{
					Kind: "User",
					Name: anonymousUser,
				},
			},
		},
	}

	for _, roleBinding := range roleBindings {
		if _, err := clientset.RbacV1beta1().RoleBindings(roleBinding.ObjectMeta.Namespace).Create(&roleBinding); err != nil {
			if !apierrors.IsAlreadyExists(err) {
				return fmt.Errorf("unable to create RBAC rolebinding: %v", err)
			}

			if _, err := clientset.RbacV1beta1().RoleBindings(roleBinding.ObjectMeta.Namespace).Update(&roleBinding); err != nil {
				return fmt.Errorf("unable to update RBAC rolebinding: %v", err)
			}
		}
	}
	return nil
}

func createClusterRoles(clientset *clientset.Clientset) error {
	clusterRoles := []rbac.ClusterRole{
		{
			ObjectMeta: metav1.ObjectMeta{
				Name: nodeAutoApproveBootstrap,
			},
			Rules: []rbac.PolicyRule{
				rbac.NewRule("create").Groups("certificates.k8s.io").Resources("certificatesigningrequests/nodeclient").RuleOrDie(),
			},
		},
	}

	for _, roleBinding := range clusterRoles {
		if _, err := clientset.RbacV1beta1().ClusterRoles().Create(&roleBinding); err != nil {
			if !apierrors.IsAlreadyExists(err) {
				return fmt.Errorf("unable to create RBAC clusterrole: %v", err)
			}

			if _, err := clientset.RbacV1beta1().ClusterRoles().Update(&roleBinding); err != nil {
				return fmt.Errorf("unable to update RBAC clusterrole: %v", err)
			}
		}
	}
	return nil
}

func createClusterRoleBindings(clientset *clientset.Clientset) error {
	clusterRoleBindings := []rbac.ClusterRoleBinding{
		{
			ObjectMeta: metav1.ObjectMeta{
				Name: "kubeadm:kubelet-bootstrap",
			},
			RoleRef: rbac.RoleRef{
				APIGroup: rbacAPIGroup,
				Kind:     clusterRoleKind,
				Name:     NodeBootstrapperClusterRoleName,
			},
			Subjects: []rbac.Subject{
				{
					Kind: "Group",
					Name: bootstrapapi.BootstrapGroup,
				},
			},
		},
		{
			ObjectMeta: metav1.ObjectMeta{
				Name: nodeAutoApproveBootstrap,
			},
			RoleRef: rbac.RoleRef{
				APIGroup: rbacAPIGroup,
				Kind:     clusterRoleKind,
				Name:     nodeAutoApproveBootstrap,
			},
			Subjects: []rbac.Subject{
				{
					Kind: "Group",
					Name: bootstrapapi.BootstrapGroup,
				},
			},
		},
		{
			ObjectMeta: metav1.ObjectMeta{
				Name: "kubeadm:node-proxier",
			},
			RoleRef: rbac.RoleRef{
				APIGroup: rbacAPIGroup,
				Kind:     clusterRoleKind,
				Name:     KubeProxyClusterRoleName,
			},
			Subjects: []rbac.Subject{
				{
					Kind:      serviceAccountKind,
					Name:      kubeadmconstants.KubeProxyServiceAccountName,
					Namespace: metav1.NamespaceSystem,
				},
			},
		},
	}

	for _, clusterRoleBinding := range clusterRoleBindings {
		if _, err := clientset.RbacV1beta1().ClusterRoleBindings().Create(&clusterRoleBinding); err != nil {
			if !apierrors.IsAlreadyExists(err) {
				return fmt.Errorf("unable to create RBAC clusterrolebinding: %v", err)
			}

			if _, err := clientset.RbacV1beta1().ClusterRoleBindings().Update(&clusterRoleBinding); err != nil {
				return fmt.Errorf("unable to update RBAC clusterrolebinding: %v", err)
			}
		}
	}
	return nil
}

func deletePermissiveNodesBindingWhenUsingNodeAuthorization(clientset *clientset.Clientset, k8sVersion *version.Version) error {

	// If the server version is higher than the Node Authorizer's minimum, try to delete the Group=system:nodes->ClusterRole=system:node binding
	// which is much more permissive than the Node Authorizer
	if kubeadmutil.IsNodeAuthorizerSupported(k8sVersion) {

		nodesRoleBinding, err := clientset.RbacV1beta1().ClusterRoleBindings().Get(kubeadmconstants.NodesClusterRoleBinding, metav1.GetOptions{})
		if err != nil {
			if apierrors.IsNotFound(err) {
				// Nothing to do; the RoleBinding doesn't exist
				return nil
			}
			return err
		}

		newSubjects := []rbac.Subject{}
		for _, subject := range nodesRoleBinding.Subjects {
			// Skip the subject that binds to the system:nodes group
			if subject.Name == kubeadmconstants.NodesGroup && subject.Kind == "Group" {
				continue
			}
			newSubjects = append(newSubjects, subject)
		}

		nodesRoleBinding.Subjects = newSubjects

		if _, err := clientset.RbacV1beta1().ClusterRoleBindings().Update(nodesRoleBinding); err != nil {
			return err
		}
	}

	return nil
}
<commit_msg>kubeadm: Expose only the cluster-info ConfigMap in the kube-public ns<commit_after>/*
Copyright 2017 The Kubernetes Authors.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package apiconfig

import (
	"fmt"

	apierrors "k8s.io/apimachinery/pkg/api/errors"
	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
	clientset "k8s.io/client-go/kubernetes"
	"k8s.io/client-go/pkg/api/v1"
	rbac "k8s.io/client-go/pkg/apis/rbac/v1beta1"
	kubeadmconstants "k8s.io/kubernetes/cmd/kubeadm/app/constants"
	kubeadmutil "k8s.io/kubernetes/cmd/kubeadm/app/util"
	bootstrapapi "k8s.io/kubernetes/pkg/bootstrap/api"
	"k8s.io/kubernetes/pkg/util/version"
)

const (
	// KubeProxyClusterRoleName sets the name for the kube-proxy ClusterRole
	KubeProxyClusterRoleName = "system:node-proxier"
	// NodeBootstrapperClusterRoleName sets the name for the TLS Node Bootstrapper ClusterRole
	NodeBootstrapperClusterRoleName = "system:node-bootstrapper"
	// BootstrapSignerClusterRoleName sets the name for the ClusterRole that allows access to ConfigMaps in the kube-public ns
	BootstrapSignerClusterRoleName = "system:bootstrap-signer-clusterinfo"

	clusterRoleKind          = "ClusterRole"
	roleKind                 = "Role"
	serviceAccountKind       = "ServiceAccount"
	rbacAPIGroup             = "rbac.authorization.k8s.io"
	anonymousUser            = "system:anonymous"
	nodeAutoApproveBootstrap = "kubeadm:node-autoapprove-bootstrap"
)

// TODO: Are there any unit tests that could be made for this file other than duplicating all values and logic in a separate file?

// CreateServiceAccounts creates the necessary serviceaccounts that kubeadm uses/might use, if they don't already exist.
func CreateServiceAccounts(clientset clientset.Interface) error {
	serviceAccounts := []v1.ServiceAccount{
		{
			ObjectMeta: metav1.ObjectMeta{
				Name:      kubeadmconstants.KubeDNSServiceAccountName,
				Namespace: metav1.NamespaceSystem,
			},
		},
		{
			ObjectMeta: metav1.ObjectMeta{
				Name:      kubeadmconstants.KubeProxyServiceAccountName,
				Namespace: metav1.NamespaceSystem,
			},
		},
	}

	for _, sa := range serviceAccounts {
		if _, err := clientset.CoreV1().ServiceAccounts(metav1.NamespaceSystem).Create(&sa); err != nil {
			if !apierrors.IsAlreadyExists(err) {
				return err
			}
		}
	}
	return nil
}

// CreateRBACRules creates the essential RBAC rules for a minimally set-up cluster
func CreateRBACRules(clientset *clientset.Clientset, k8sVersion *version.Version) error {
	if err := createRoles(clientset); err != nil {
		return err
	}
	if err := createRoleBindings(clientset); err != nil {
		return err
	}
	if err := createClusterRoles(clientset); err != nil {
		return err
	}
	if err := createClusterRoleBindings(clientset); err != nil {
		return err
	}
	if err := deletePermissiveNodesBindingWhenUsingNodeAuthorization(clientset, k8sVersion); err != nil {
		return fmt.Errorf("failed to remove the permissive 'system:nodes' Group Subject in the 'system:node' ClusterRoleBinding: %v", err)
	}

	fmt.Println("[apiconfig] Created RBAC rules")
	return nil
}

func createRoles(clientset *clientset.Clientset) error {
	roles := []rbac.Role{
		{
			ObjectMeta: metav1.ObjectMeta{
				Name:      BootstrapSignerClusterRoleName,
				Namespace: metav1.NamespacePublic,
			},
			Rules: []rbac.PolicyRule{
				rbac.NewRule("get").Groups("").Resources("configmaps").Names("cluster-info").RuleOrDie(),
			},
		},
	}
	for _, role := range roles {
		if _, err := clientset.RbacV1beta1().Roles(role.ObjectMeta.Namespace).Create(&role); err != nil {
			if !apierrors.IsAlreadyExists(err) {
				return fmt.Errorf("unable to create RBAC role: %v", err)
			}

			if _, err := clientset.RbacV1beta1().Roles(role.ObjectMeta.Namespace).Update(&role); err != nil {
				return fmt.Errorf("unable to update RBAC role: %v", err)
			}
		}
	}
	return nil
}

func createRoleBindings(clientset *clientset.Clientset) error {
	roleBindings := []rbac.RoleBinding{
		{
			ObjectMeta: metav1.ObjectMeta{
				Name:      "kubeadm:bootstrap-signer-clusterinfo",
				Namespace: metav1.NamespacePublic,
			},
			RoleRef: rbac.RoleRef{
				APIGroup: rbacAPIGroup,
				Kind:     roleKind,
				Name:     BootstrapSignerClusterRoleName,
			},
			Subjects: []rbac.Subject{
				{
					Kind: "User",
					Name: anonymousUser,
				},
			},
		},
	}

	for _, roleBinding := range roleBindings {
		if _, err := clientset.RbacV1beta1().RoleBindings(roleBinding.ObjectMeta.Namespace).Create(&roleBinding); err != nil {
			if !apierrors.IsAlreadyExists(err) {
				return fmt.Errorf("unable to create RBAC rolebinding: %v", err)
			}

			if _, err := clientset.RbacV1beta1().RoleBindings(roleBinding.ObjectMeta.Namespace).Update(&roleBinding); err != nil {
				return fmt.Errorf("unable to update RBAC rolebinding: %v", err)
			}
		}
	}
	return nil
}

func createClusterRoles(clientset *clientset.Clientset) error {
	clusterRoles := []rbac.ClusterRole{
		{
			ObjectMeta: metav1.ObjectMeta{
				Name: nodeAutoApproveBootstrap,
			},
			Rules: []rbac.PolicyRule{
				rbac.NewRule("create").Groups("certificates.k8s.io").Resources("certificatesigningrequests/nodeclient").RuleOrDie(),
			},
		},
	}

	for _, roleBinding := range clusterRoles {
		if _, err := clientset.RbacV1beta1().ClusterRoles().Create(&roleBinding); err != nil {
			if !apierrors.IsAlreadyExists(err) {
				return fmt.Errorf("unable to create RBAC clusterrole: %v", err)
			}

			if _, err := clientset.RbacV1beta1().ClusterRoles().Update(&roleBinding); err != nil {
				return fmt.Errorf("unable to update RBAC clusterrole: %v", err)
			}
		}
	}
	return nil
}

func createClusterRoleBindings(clientset *clientset.Clientset) error {
	clusterRoleBindings := []rbac.ClusterRoleBinding{
		{
			ObjectMeta: metav1.ObjectMeta{
				Name: "kubeadm:kubelet-bootstrap",
			},
			RoleRef: rbac.RoleRef{
				APIGroup: rbacAPIGroup,
				Kind:     clusterRoleKind,
				Name:     NodeBootstrapperClusterRoleName,
			},
			Subjects: []rbac.Subject{
				{
					Kind: "Group",
					Name: bootstrapapi.BootstrapGroup,
				},
			},
		},
		{
			ObjectMeta: metav1.ObjectMeta{
				Name: nodeAutoApproveBootstrap,
			},
			RoleRef: rbac.RoleRef{
				APIGroup: rbacAPIGroup,
				Kind:     clusterRoleKind,
				Name:     nodeAutoApproveBootstrap,
			},
			Subjects: []rbac.Subject{
				{
					Kind: "Group",
					Name: bootstrapapi.BootstrapGroup,
				},
			},
		},
		{
			ObjectMeta: metav1.ObjectMeta{
				Name: "kubeadm:node-proxier",
			},
			RoleRef: rbac.RoleRef{
				APIGroup: rbacAPIGroup,
				Kind:     clusterRoleKind,
				Name:     KubeProxyClusterRoleName,
			},
			Subjects: []rbac.Subject{
				{
					Kind:      serviceAccountKind,
					Name:      kubeadmconstants.KubeProxyServiceAccountName,
					Namespace: metav1.NamespaceSystem,
				},
			},
		},
	}

	for _, clusterRoleBinding := range clusterRoleBindings {
		if _, err := clientset.RbacV1beta1().ClusterRoleBindings().Create(&clusterRoleBinding); err != nil {
			if !apierrors.IsAlreadyExists(err) {
				return fmt.Errorf("unable to create RBAC clusterrolebinding: %v", err)
			}

			if _, err := clientset.RbacV1beta1().ClusterRoleBindings().Update(&clusterRoleBinding); err != nil {
				return fmt.Errorf("unable to update RBAC clusterrolebinding: %v", err)
			}
		}
	}
	return nil
}

func deletePermissiveNodesBindingWhenUsingNodeAuthorization(clientset *clientset.Clientset, k8sVersion *version.Version) error {

	// If the server version is higher than the Node Authorizer's minimum, try to delete the Group=system:nodes->ClusterRole=system:node binding
	// which is much more permissive than the Node Authorizer
	if kubeadmutil.IsNodeAuthorizerSupported(k8sVersion) {

		nodesRoleBinding, err := clientset.RbacV1beta1().ClusterRoleBindings().Get(kubeadmconstants.NodesClusterRoleBinding, metav1.GetOptions{})
		if err != nil {
			if apierrors.IsNotFound(err) {
				// Nothing to do; the RoleBinding doesn't exist
				return nil
			}
			return err
		}

		newSubjects := []rbac.Subject{}
		for _, subject := range nodesRoleBinding.Subjects {
			// Skip the subject that binds to the system:nodes group
			if subject.Name == kubeadmconstants.NodesGroup && subject.Kind == "Group" {
				continue
			}
			newSubjects = append(newSubjects, subject)
		}

		nodesRoleBinding.Subjects = newSubjects

		if _, err := clientset.RbacV1beta1().ClusterRoleBindings().Update(nodesRoleBinding); err != nil {
			return err
		}
	}

	return nil
}
<|endoftext|>


<commit_before>package main

import (
	"eaciit/gdrj/model"
	"eaciit/gdrj/modules"
	"os"

	"flag"
	"github.com/eaciit/dbox"
	"github.com/eaciit/toolkit"
	// "strings"
	"time"
)

var conn dbox.IConnection
var count int

var (
	t0     time.Time
	gtable string
)

var mastercostcenter = toolkit.M{}
var masterbranch = toolkit.M{}

// var masterbranchgroup = toolkit.M{}
var masterbranchgroup = toolkit.M{}
var masteraccountgroup = toolkit.M{}

func prepdatacostcenter() {
	toolkit.Println("--> Get Data cost center")

	// filter := dbox.Eq("key.date_fiscal", toolkit.Sprintf("%d-%d", fiscalyear-1, fiscalyear))
	csr, _ := conn.NewQuery().Select().From("costcenter").Cursor(nil)
	defer csr.Close()

	for {
		tkm := toolkit.M{}
		e := csr.Fetch(&tkm, 1, false)
		if e != nil {
			break
		}

		mastercostcenter.Set(tkm.GetString("_id"), tkm)
	}
}

func prepdatabranch() {
	toolkit.Println("--> Get branch center")

	// filter := dbox.Eq("key.date_fiscal", toolkit.Sprintf("%d-%d", fiscalyear-1, fiscalyear))
	csr, _ := conn.NewQuery().Select().From("masterbranch").Cursor(nil)
	defer csr.Close()

	for {
		tkm := toolkit.M{}
		e := csr.Fetch(&tkm, 1, false)
		if e != nil {
			break
		}

		masterbranch.Set(tkm.GetString("_id"), tkm)
	}
}

func prepdatabranchgroup() {
	toolkit.Println("--> Get branch group")

	// filter := dbox.Eq("key.date_fiscal", toolkit.Sprintf("%d-%d", fiscalyear-1, fiscalyear))
	csr, _ := conn.NewQuery().Select().From("masterbranchgroup").Cursor(nil)
	defer csr.Close()

	for {
		tkm := toolkit.M{}
		e := csr.Fetch(&tkm, 1, false)
		if e != nil {
			break
		}

		masterbranchgroup.Set(tkm.GetString("_id"), tkm)
	}
}

func prepdataaccountgroup() {
	toolkit.Println("--> Get account group")

	// filter := dbox.Eq("key.date_fiscal", toolkit.Sprintf("%d-%d", fiscalyear-1, fiscalyear))
	csr, _ := conn.NewQuery().Select().From("masteraccountgroup").Cursor(nil)
	defer csr.Close()

	for {
		tkm := toolkit.M{}
		e := csr.Fetch(&tkm, 1, false)
		if e != nil {
			break
		}

		masteraccountgroup.Set(tkm.GetString("accountdescription"), tkm.GetString("accountgroup"))
	}
}

func setinitialconnection() {
	var err error
	conn, err = modules.GetDboxIConnection("db_godrej")

	if err != nil {
		toolkit.Println("Initial connection found : ", err)
		os.Exit(1)
	}

	err = gdrj.SetDb(conn)
	if err != nil {
		toolkit.Println("Initial connection found : ", err)
		os.Exit(1)
	}
}

func getstep(count int) int {
	v := count / 100
	if v == 0 {
		return 1
	}
	return v
}

func main() {
	t0 = time.Now()
	flag.StringVar(&gtable, "table", "", "tablename")
	// flag.IntVar(&year, "year", 2014, "2014 year")
	flag.Parse()

	setinitialconnection()

	prepdatabranch()
	// prepdatacostcenter()
	prepdataaccountgroup()
	// prepdatabranchgroup()

	workerconn, _ := modules.GetDboxIConnection("db_godrej")
	defer workerconn.Close()

	toolkit.Println("Start data query...")
	// arrfilter := []*dbox.Filter{}
	// for i := 4; i < 10; i++ {
	// 	toolkit.Printfn("%d - 2014", i)
	// 	f := dbox.And(dbox.Eq("year", 2014), dbox.Eq("month", i))
	// 	arrfilter = append(arrfilter, f)
	// }

	// f := dbox.Or(arrfilter...)
	// f := dbox.Eq("key.date_fiscal", "2014-2015")
	csr, _ := workerconn.NewQuery().Select().From(gtable).Cursor(nil)
	defer csr.Close()

	scount := csr.Count()

	jobs := make(chan toolkit.M, scount)
	result := make(chan int, scount)
	for wi := 0; wi < 10; wi++ {
		go workersave(wi, jobs, result)
	}

	iscount := 0
	step := getstep(scount) * 5

	for {
		iscount++
		tkm := toolkit.M{}
		e := csr.Fetch(&tkm, 1, false)
		if e != nil {
			toolkit.Println("EOF")
			break
		}

		jobs <- tkm

		if iscount%step == 0 {
			toolkit.Printfn("Sending %d of %d (%d) in %s", iscount, scount, iscount*100/scount,
				time.Since(t0).String())
		}

	}

	close(jobs)

	for ri := 0; ri < scount; ri++ {
		<-result

		if ri%step == 0 {
			toolkit.Printfn("Saving %d of %d (%d pct) in %s",
				ri, scount, ri*100/scount, time.Since(t0).String())
		}
	}

	toolkit.Printfn("Processing done in %s",
		time.Since(t0).String())
}

func workersave(wi int, jobs <-chan toolkit.M, result chan<- int) {
	workerconn, _ := modules.GetDboxIConnection("db_godrej")
	defer workerconn.Close()

	qSave := workerconn.NewQuery().
		From(toolkit.Sprintf("%s-res", gtable)).
		SetConfig("multiexec", true).
		Save()

	trx := toolkit.M{}
	for trx = range jobs {
		key := trx.Get("_id", toolkit.M{}).(toolkit.M)
		trx.Set("key", key)

		id := toolkit.Sprintf("%d|%s|%s|%s|%s|%s|%s|%s", key.GetInt("year"), key.GetString("branchid"),
			key.GetString("branchname"), key.GetString("brancharea"), key.GetString("account"),
			key.GetString("accountdescription"), key.GetString("costgroup"), key.GetString("addinfo"))

		trx.Set("_id", id)

		// tdate := time.Date(trx.GetInt("year"), time.Month(trx.GetInt("period")), 1, 0, 0, 0, 0, time.UTC).
		// 	AddDate(0, 3, 0)
		// gdrjdate := gdrj.SetDate(tdate)

		// trx.Set("gdrjdate", gdrjdate)

		// cc := trx.GetString("ccid")
		// trx.Set("branchid", "CD00")
		// trx.Set("branchname", "OTHER")
		// trx.Set("brancharea", "OTHER")
		// trx.Set("costgroup", "OTHER")
		// trx.Set("accountgroup", "OTHER")
		// trx.Set("branchgroup", "OTHER")

		// key.Set("customer_branchgroup", "OTHER")

		// trx.Set("min_amountinidr", -trx.GetFloat64("amountinidr"))

		// if mastercostcenter.Has(cc) {
		// 	mcc := mastercostcenter[cc].(toolkit.M)
		// 	brid := mcc.GetString("branchid")

		// 	trx.Set("branchid", brid)
		// 	trx.Set("costgroup", mcc.GetString("costgroup01"))

		// 	if masterbranch.Has(brid) {
		// 		trx.Set("branchname", masterbranch[brid].(toolkit.M).GetString("name"))
		// 		trx.Set("brancharea", masterbranch[brid].(toolkit.M).GetString("area"))
		// 	}
		// }

		// branchid := trx.GetString("branchid")

		accdesc := trx.GetString("accountdescription")
		trx.Set("accountgroup", masteraccountgroup.GetString(accdesc))

		// if trx.GetString("costgroup") == "" {
		// 	trx.Set("costgroup", "OTHER")
		// }

		// if trx.GetString("branchname") == "" {
		// 	trx.Set("branchname", "OTHER")
		// }

		// if trx.GetString("brancharea") == "" {
		// 	trx.Set("brancharea", "OTHER")
		// }

		if trx.GetString("accountgroup") == "" {
			trx.Set("accountgroup", "OTHER")
		}

		// if trx.GetString("branchgroup") == "" {
		// 	trx.Set("branchgroup", "OTHER")
		// }

		// if trx.GetString("accountdescription") == "#N/A" {
		// 	trx.Set("accountdescription", "CONSUMABLE STORES & SPARES")
		// }

		// if trx.GetString("grouping") == "#N/A" {
		// 	trx.Set("grouping", "General and administrative expenses")
		// }

		//=== For data rawdata mode

		// branchid := trx.GetString("branchid")
		// if !masterbranch.Has(branchid) {
		// 	branchid = "CD00"
		// }

		// branchgroup := masterbranch.Get(branchid, toolkit.M{}).(toolkit.M)
		// trx.Set("branchgroup", branchgroup.GetString("branchgroup"))
		// trx.Set("branchlvl2", branchgroup.GetString("branchlvl2"))
		// trx.Set("idbranchlvl2", branchgroup.GetString("idbranchlvl2"))

		// if branchid == "HD11" && trx.GetString("addinfo") == "Jakarta" && trx.GetString("ccid") != "HD110313" {
		// 	trx.Set("branchgroup", "Jakarta")
		// }

		// if trx.GetString("branchgroup") == "" {
		// 	trx.Set("branchgroup", "OTHER")
		// }

		// if trx.GetString("branchlvl2") == "" {
		// 	trx.Set("branchlvl2", "OTHER")
		// }

		//idbranchlvl2

		//===========================================

		//=== For data salespls-summary mode consolidate

		// key := trx.Get("key", toolkit.M{}).(toolkit.M)
		// branchid := key.GetString("customer_branchid")

		// branchgroup := masterbranch.Get(branchid, toolkit.M{}).(toolkit.M)
		// key.Set("customer_branchgroup", branchgroup.GetString("branchgroup"))
		// key.Set("customer_branchlvl2", branchgroup.GetString("branchlvl2"))

		// if key.GetString("customer_branchgroup") == "" {
		// 	key.Set("customer_branchgroup", "OTHER")
		// }

		// if key.GetString("customer_branchlvl2") == "" {
		// 	key.Set("customer_branchlvl2", "OTHER")
		// }

		// /*other fix*/
		// if key.GetString("customer_reportsubchannel") == "R3" {
		// 	key.Set("customer_reportsubchannel", "R3 - Retailer Umum")
		// }

		// if key.GetString("customer_region") == "" || key.GetString("customer_region") == "Other" {
		// 	key.Set("customer_region", "OTHER")
		// }

		// if key.GetString("customer_zone") == "" || key.GetString("customer_zone") == "Other" {
		// 	key.Set("customer_zone", "OTHER")
		// }

		// trx.Set("key", key)
		//============================================

		// For cogs consolidate
		// arrstr := []string{"rm_perunit", "lc_perunit", "pf_perunit", "other_perunit", "fixed_perunit", "depre_perunit", "cogs_perunit"}
		// for _, v := range arrstr {
		// 	xval := trx.GetFloat64(v) * 6
		// 	trx.Set(v, xval)
		// }
		// ====================

		// trx.Set("addinfo", "")
		// if trx.GetString("branchid") == "HD11" && strings.Contains(trx.GetString("costcentername"), "Jkt") {
		// 	trx.Set("addinfo", "Jakarta")
		// }

		// i := 1
		// ratio := float64(3337787.02099609) / float64(207880004972.979)
		// key := trx.Get("key", toolkit.M{}).(toolkit.M)
		// if key.GetString("date_fiscal") == "2014-2015" {
		// 	for k, _ := range trx {
		// 		arrk := strings.Split(k, "_")
		// 		if (len(arrk) > 1 && arrk[1] == "Allocated") || k == "PL94A" {
		// 			val := trx.GetFloat64(k)
		// 			xval := val + (val * ratio)
		// 			trx.Set(k, xval)
		// 		}
		// 	}
		// }

		err := qSave.Exec(toolkit.M{}.Set("data", trx))
		if err != nil {
			toolkit.Println(err)
		}

		result <- 1
	}
}
<commit_msg>sga clean new<commit_after>package main

import (
	"eaciit/gdrj/model"
	"eaciit/gdrj/modules"
	"os"

	"flag"
	"github.com/eaciit/dbox"
	"github.com/eaciit/toolkit"
	// "strings"
	"time"
)

var conn dbox.IConnection
var count int

var (
	t0     time.Time
	gtable string
)

var mastercostcenter = toolkit.M{}
var masterbranch = toolkit.M{}

// var masterbranchgroup = toolkit.M{}
var masterbranchgroup = toolkit.M{}
var masteraccountgroup = toolkit.M{}

func prepdatacostcenter() {
	toolkit.Println("--> Get Data cost center")

	// filter := dbox.Eq("key.date_fiscal", toolkit.Sprintf("%d-%d", fiscalyear-1, fiscalyear))
	csr, _ := conn.NewQuery().Select().From("costcenter").Cursor(nil)
	defer csr.Close()

	for {
		tkm := toolkit.M{}
		e := csr.Fetch(&tkm, 1, false)
		if e != nil {
			break
		}

		mastercostcenter.Set(tkm.GetString("_id"), tkm)
	}
}

func prepdatabranch() {
	toolkit.Println("--> Get branch center")

	// filter := dbox.Eq("key.date_fiscal", toolkit.Sprintf("%d-%d", fiscalyear-1, fiscalyear))
	csr, _ := conn.NewQuery().Select().From("masterbranch").Cursor(nil)
	defer csr.Close()

	for {
		tkm := toolkit.M{}
		e := csr.Fetch(&tkm, 1, false)
		if e != nil {
			break
		}

		masterbranch.Set(tkm.GetString("_id"), tkm)
	}
}

func prepdatabranchgroup() {
	toolkit.Println("--> Get branch group")

	// filter := dbox.Eq("key.date_fiscal", toolkit.Sprintf("%d-%d", fiscalyear-1, fiscalyear))
	csr, _ := conn.NewQuery().Select().From("masterbranchgroup").Cursor(nil)
	defer csr.Close()

	for {
		tkm := toolkit.M{}
		e := csr.Fetch(&tkm, 1, false)
		if e != nil {
			break
		}

		masterbranchgroup.Set(tkm.GetString("_id"), tkm)
	}
}

func prepdataaccountgroup() {
	toolkit.Println("--> Get account group")

	// filter := dbox.Eq("key.date_fiscal", toolkit.Sprintf("%d-%d", fiscalyear-1, fiscalyear))
	csr, _ := conn.NewQuery().Select().From("masteraccountgroup").Cursor(nil)
	defer csr.Close()

	for {
		tkm := toolkit.M{}
		e := csr.Fetch(&tkm, 1, false)
		if e != nil {
			break
		}

		masteraccountgroup.Set(tkm.GetString("accountdescription"), tkm.GetString("accountgroup"))
	}
}

func setinitialconnection() {
	var err error
	conn, err = modules.GetDboxIConnection("db_godrej")

	if err != nil {
		toolkit.Println("Initial connection found : ", err)
		os.Exit(1)
	}

	err = gdrj.SetDb(conn)
	if err != nil {
		toolkit.Println("Initial connection found : ", err)
		os.Exit(1)
	}
}

func getstep(count int) int {
	v := count / 100
	if v == 0 {
		return 1
	}
	return v
}

func main() {
	t0 = time.Now()
	flag.StringVar(&gtable, "table", "", "tablename")
	// flag.IntVar(&year, "year", 2014, "2014 year")
	flag.Parse()

	setinitialconnection()

	prepdatabranch()
	prepdatacostcenter()
	prepdataaccountgroup()
	// prepdatabranchgroup()

	workerconn, _ := modules.GetDboxIConnection("db_godrej")
	defer workerconn.Close()

	toolkit.Println("Start data query...")
	// arrfilter := []*dbox.Filter{}
	// for i := 4; i < 10; i++ {
	// 	toolkit.Printfn("%d - 2014", i)
	// 	f := dbox.And(dbox.Eq("year", 2014), dbox.Eq("month", i))
	// 	arrfilter = append(arrfilter, f)
	// }

	// f := dbox.Or(arrfilter...)
	// f := dbox.Eq("key.date_fiscal", "2014-2015")
	csr, _ := workerconn.NewQuery().Select().From(gtable).Cursor(nil)
	defer csr.Close()

	scount := csr.Count()

	jobs := make(chan toolkit.M, scount)
	result := make(chan int, scount)
	for wi := 0; wi < 10; wi++ {
		go workersave(wi, jobs, result)
	}

	iscount := 0
	step := getstep(scount) * 10

	for {
		iscount++
		tkm := toolkit.M{}
		e := csr.Fetch(&tkm, 1, false)
		if e != nil {
			toolkit.Println("EOF")
			break
		}

		jobs <- tkm

		if iscount%step == 0 {
			toolkit.Printfn("Sending %d of %d (%d) in %s", iscount, scount, iscount*100/scount,
				time.Since(t0).String())
		}

	}

	close(jobs)

	for ri := 0; ri < scount; ri++ {
		<-result

		if ri%step == 0 {
			toolkit.Printfn("Saving %d of %d (%d pct) in %s",
				ri, scount, ri*100/scount, time.Since(t0).String())
		}
	}

	toolkit.Printfn("Processing done in %s",
		time.Since(t0).String())
}

func workersave(wi int, jobs <-chan toolkit.M, result chan<- int) {
	workerconn, _ := modules.GetDboxIConnection("db_godrej")
	defer workerconn.Close()

	qSave := workerconn.NewQuery().
		From(toolkit.Sprintf("%s-res", gtable)).
		SetConfig("multiexec", true).
		Save()

	trx := toolkit.M{}
	for trx = range jobs {
		key := trx.Get("_id", toolkit.M{}).(toolkit.M)
		trx.Set("key", key)

		id := toolkit.Sprintf("%d|%s|%s|%s|%s|%s|%s|%s", key.GetInt("year"), key.GetString("branchid"),
			key.GetString("branchname"), key.GetString("brancharea"), key.GetString("account"),
			key.GetString("accountdescription"), key.GetString("costgroup"), key.GetString("addinfo"))

		trx.Set("_id", id)

		// tdate := time.Date(trx.GetInt("year"), time.Month(trx.GetInt("period")), 1, 0, 0, 0, 0, time.UTC).
		// 	AddDate(0, 3, 0)
		// gdrjdate := gdrj.SetDate(tdate)

		// trx.Set("gdrjdate", gdrjdate)

		cc := trx.GetString("ccid")
		// trx.Set("branchid", "CD00")
		// trx.Set("branchname", "OTHER")
		// trx.Set("brancharea", "OTHER")
		// trx.Set("costgroup", "OTHER")
		// trx.Set("accountgroup", "OTHER")
		// trx.Set("branchgroup", "OTHER")

		// key.Set("customer_branchgroup", "OTHER")

		trx.Set("min_amountinidr", -trx.GetFloat64("amountinidr"))

		if mastercostcenter.Has(cc) {
			mcc := mastercostcenter[cc].(toolkit.M)
			brid := mcc.GetString("branchid")

			trx.Set("branchid", brid)
			trx.Set("costgroup", mcc.GetString("costgroup01"))

			if masterbranch.Has(brid) {
				trx.Set("branchname", masterbranch[brid].(toolkit.M).GetString("name"))
				trx.Set("brancharea", masterbranch[brid].(toolkit.M).GetString("area"))
			}
		}

		// branchid := trx.GetString("branchid")
		if trx.GetString("accountdescription") == "#N/A" {
			trx.Set("accountdescription", "CONSUMABLE STORES & SPARES")
		}

		if trx.GetString("grouping") == "#N/A" {
			trx.Set("grouping", "General and administrative expenses")
		}

		accdesc := trx.GetString("accountdescription")
		trx.Set("accountgroup", masteraccountgroup.GetString(accdesc))

		arrstrcheck := []string{"branchid", "branchname", "brancharea", "costgroup", "accountgroup", "branchgroup"}
		for _, v := range arrstrcheck {
			if trx.GetString(v) == "" {
				trx.Set(v, "OTHER")
				if v == "branchid" {
					trx.Set(v, "CD00")
				}
			}
		}

		//=== For data rawdata mode

		branchid := trx.GetString("branchid")
		if !masterbranch.Has(branchid) {
			branchid = "CD00"
		}

		branchgroup := masterbranch.Get(branchid, toolkit.M{}).(toolkit.M)
		trx.Set("branchgroup", branchgroup.GetString("branchgroup"))
		trx.Set("branchlvl2", branchgroup.GetString("branchlvl2"))
		trx.Set("idbranchlvl2", branchgroup.GetString("idbranchlvl2"))

		if branchid == "HD11" && trx.GetString("addinfo") == "Jakarta" && trx.GetString("ccid") != "HD110313" {
			trx.Set("branchgroup", "Jakarta")
		}

		if trx.GetString("branchgroup") == "" {
			trx.Set("branchgroup", "OTHER")
		}

		if trx.GetString("branchlvl2") == "" {
			trx.Set("branchlvl2", "OTHER")
		}

		//idbranchlvl2

		//===========================================

		//=== For data salespls-summary mode consolidate

		// key := trx.Get("key", toolkit.M{}).(toolkit.M)
		// branchid := key.GetString("customer_branchid")

		// branchgroup := masterbranch.Get(branchid, toolkit.M{}).(toolkit.M)
		// key.Set("customer_branchgroup", branchgroup.GetString("branchgroup"))
		// key.Set("customer_branchlvl2", branchgroup.GetString("branchlvl2"))

		// if key.GetString("customer_branchgroup") == "" {
		// 	key.Set("customer_branchgroup", "OTHER")
		// }

		// if key.GetString("customer_branchlvl2") == "" {
		// 	key.Set("customer_branchlvl2", "OTHER")
		// }

		// /*other fix*/
		// if key.GetString("customer_reportsubchannel") == "R3" {
		// 	key.Set("customer_reportsubchannel", "R3 - Retailer Umum")
		// }

		// if key.GetString("customer_region") == "" || key.GetString("customer_region") == "Other" {
		// 	key.Set("customer_region", "OTHER")
		// }

		// if key.GetString("customer_zone") == "" || key.GetString("customer_zone") == "Other" {
		// 	key.Set("customer_zone", "OTHER")
		// }

		// trx.Set("key", key)
		//============================================

		// For cogs consolidate
		// arrstr := []string{"rm_perunit", "lc_perunit", "pf_perunit", "other_perunit", "fixed_perunit", "depre_perunit", "cogs_perunit"}
		// for _, v := range arrstr {
		// 	xval := trx.GetFloat64(v) * 6
		// 	trx.Set(v, xval)
		// }
		// ====================

		// trx.Set("addinfo", "")
		// if trx.GetString("branchid") == "HD11" && strings.Contains(trx.GetString("costcentername"), "Jkt") {
		// 	trx.Set("addinfo", "Jakarta")
		// }

		// i := 1
		// ratio := float64(3337787.02099609) / float64(207880004972.979)
		// key := trx.Get("key", toolkit.M{}).(toolkit.M)
		// if key.GetString("date_fiscal") == "2014-2015" {
		// 	for k, _ := range trx {
		// 		arrk := strings.Split(k, "_")
		// 		if (len(arrk) > 1 && arrk[1] == "Allocated") || k == "PL94A" {
		// 			val := trx.GetFloat64(k)
		// 			xval := val + (val * ratio)
		// 			trx.Set(k, xval)
		// 		}
		// 	}
		// }

		err := qSave.Exec(toolkit.M{}.Set("data", trx))
		if err != nil {
			toolkit.Println(err)
		}

		result <- 1
	}
}
<|endoftext|>