<commit_before>/*
 *  Copyright 2016 Adobe Systems Incorporated. All rights reserved.
 *  This file is licensed to you under the Apache License, Version 2.0 (the "License");
 *  you may not use this file except in compliance with the License. You may obtain a copy
 *  of the License at http://www.apache.org/licenses/LICENSE-2.0
 *
 *  Unless required by applicable law or agreed to in writing, software distributed under
 *  the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR REPRESENTATIONS
 *  OF ANY KIND, either express or implied. See the License for the specific language
 *  governing permissions and limitations under the License.
 */
package build

import (
	"encoding/json"
	"flag"
	"io/ioutil"
	"os"

	"github.com/adobe-platform/porter/conf"
	"github.com/adobe-platform/porter/constants"
	"github.com/adobe-platform/porter/hook"
	"github.com/adobe-platform/porter/logger"
	"github.com/adobe-platform/porter/provision_state"
	"github.com/adobe-platform/porter/prune"
	"github.com/inconshreveable/log15"
	"github.com/phylake/go-cli"
)

type PruneCmd struct{}

func (recv *PruneCmd) Name() string {
	return "prune"
}

func (recv *PruneCmd) ShortHelp() string {
	return "Delete extra CloudFormation stacks"
}

func (recv *PruneCmd) LongHelp() string {
	return `NAME
    prune -- Delete extra CloudFormation stacks

SYNOPSIS
    prune [--keep <stacks to keep>]

DESCRIPTION
    Delete extra CloudFormation stacks. Instances attached to the configured
    ELB are not candidates for deletion.

OPTIONS
    --keep
        The number of stacks to keep with the status CREATE_COMPLETE,
        UPDATE_COMPLETE, or ROLLBACK_COMPLETE not including the stack with
        instances attached to the configured ELB.

        The default is 0 meaning only the stack with instances attached to the
        configured ELB will be kept.

        Eligible stacks will be sorted by creation time with the oldest being
        deleted first.`
}

func (recv *PruneCmd) SubCommands() []cli.Command {
	return nil
}

func (recv *PruneCmd) Execute(args []string) bool {

	if len(args) == 1 && args[0] == "--help" {
		return false
	}

	var elbTag string
	var keepCount int

	if len(args) > 0 {

		flagSet := flag.NewFlagSet("", flag.ContinueOnError)

		flagSet.IntVar(&keepCount, "keep", 0, "")
		flagSet.StringVar(&elbTag, "elb", "", "")
		flagSet.Parse(args)

		if keepCount < 0 {
			return false
		}
	}

	log := logger.CLI("cmd", "prune")

	provisionEnvBytes, err := ioutil.ReadFile(constants.ProvisionOutputPath)
	if err != nil {
		log.Error("ioutil.ReadFile", "Error", err)
		os.Exit(1)
	}

	stack := &provision_state.Stack{}
	err = json.Unmarshal(provisionEnvBytes, stack)
	if err != nil {
		log.Error("json.Unmarshal", "Error", err)
		os.Exit(1)
	}

	if stack.Hotswap {
		log.Info("No prune occurs during a hot swap")
		return true
	}

	if !doPrune(log, stack, keepCount, elbTag) {
		os.Exit(1)
	}

	log.Info("Prune complete")
	return true
}

func doPrune(log log15.Logger, stack *provision_state.Stack, keepCount int, elbTag string) (success bool) {

	defer func() {

		postHookSuccess := hook.Execute(log, constants.HookPostPrune,
			stack.Environment, stack.Regions, success)

		success = success && postHookSuccess
	}()

	config, success := conf.GetAlteredConfig(log)
	if !success {
		return
	}

	env, err := config.GetEnvironment(stack.Environment)
	if err != nil {
		log.Error("GetEnvironment", "Error", err)
		return
	}

	if !hook.Execute(log, constants.HookPrePrune, stack.Environment, stack.Regions, true) {
		return
	}

	success = prune.Do(log, config, env, keepCount, true, elbTag)
	return
}
<commit_msg>prune on hot swap<commit_after>/*
 *  Copyright 2016 Adobe Systems Incorporated. All rights reserved.
 *  This file is licensed to you under the Apache License, Version 2.0 (the "License");
 *  you may not use this file except in compliance with the License. You may obtain a copy
 *  of the License at http://www.apache.org/licenses/LICENSE-2.0
 *
 *  Unless required by applicable law or agreed to in writing, software distributed under
 *  the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR REPRESENTATIONS
 *  OF ANY KIND, either express or implied. See the License for the specific language
 *  governing permissions and limitations under the License.
 */
package build

import (
	"encoding/json"
	"flag"
	"io/ioutil"
	"os"

	"github.com/adobe-platform/porter/conf"
	"github.com/adobe-platform/porter/constants"
	"github.com/adobe-platform/porter/hook"
	"github.com/adobe-platform/porter/logger"
	"github.com/adobe-platform/porter/provision_state"
	"github.com/adobe-platform/porter/prune"
	"github.com/inconshreveable/log15"
	"github.com/phylake/go-cli"
)

type PruneCmd struct{}

func (recv *PruneCmd) Name() string {
	return "prune"
}

func (recv *PruneCmd) ShortHelp() string {
	return "Delete extra CloudFormation stacks"
}

func (recv *PruneCmd) LongHelp() string {
	return `NAME
    prune -- Delete extra CloudFormation stacks

SYNOPSIS
    prune [--keep <stacks to keep>]

DESCRIPTION
    Delete extra CloudFormation stacks. Instances attached to the configured
    ELB are not candidates for deletion.

OPTIONS
    --keep
        The number of stacks to keep with the status CREATE_COMPLETE,
        UPDATE_COMPLETE, or ROLLBACK_COMPLETE not including the stack with
        instances attached to the configured ELB.

        The default is 0 meaning only the stack with instances attached to the
        configured ELB will be kept.

        Eligible stacks will be sorted by creation time with the oldest being
        deleted first.`
}

func (recv *PruneCmd) SubCommands() []cli.Command {
	return nil
}

func (recv *PruneCmd) Execute(args []string) bool {

	if len(args) == 1 && args[0] == "--help" {
		return false
	}

	var elbTag string
	var keepCount int

	if len(args) > 0 {

		flagSet := flag.NewFlagSet("", flag.ContinueOnError)

		flagSet.IntVar(&keepCount, "keep", 0, "")
		flagSet.StringVar(&elbTag, "elb", "", "")
		flagSet.Parse(args)

		if keepCount < 0 {
			return false
		}
	}

	log := logger.CLI("cmd", "prune")

	provisionEnvBytes, err := ioutil.ReadFile(constants.ProvisionOutputPath)
	if err != nil {
		log.Error("ioutil.ReadFile", "Error", err)
		os.Exit(1)
	}

	stack := &provision_state.Stack{}
	err = json.Unmarshal(provisionEnvBytes, stack)
	if err != nil {
		log.Error("json.Unmarshal", "Error", err)
		os.Exit(1)
	}

	if !doPrune(log, stack, keepCount, elbTag) {
		os.Exit(1)
	}

	log.Info("Prune complete")
	return true
}

func doPrune(log log15.Logger, stack *provision_state.Stack, keepCount int, elbTag string) (success bool) {

	defer func() {

		postHookSuccess := hook.Execute(log, constants.HookPostPrune,
			stack.Environment, stack.Regions, success)

		success = success && postHookSuccess
	}()

	config, success := conf.GetAlteredConfig(log)
	if !success {
		return
	}

	env, err := config.GetEnvironment(stack.Environment)
	if err != nil {
		log.Error("GetEnvironment", "Error", err)
		return
	}

	if !hook.Execute(log, constants.HookPrePrune, stack.Environment, stack.Regions, true) {
		return
	}

	success = prune.Do(log, config, env, keepCount, true, elbTag)
	return
}
<|endoftext|>


<commit_before>/*
Copyright 2018 The OpenEBS Authors

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
package spc

import (
	"testing"
	apis "github.com/openebs/maya/pkg/apis/openebs.io/v1alpha1"
	api_meta "k8s.io/apimachinery/pkg/apis/meta/v1"
	"time"
)

func TestIsDeleteEvent(t *testing.T) {

	tests := map[string]struct {
		fakestoragepoolclaim *apis.StoragePoolClaim
		expectedResult       bool
	}{
		"DeletionTimestamp is nil": {
			fakestoragepoolclaim: &apis.StoragePoolClaim{
				ObjectMeta: api_meta.ObjectMeta{
					DeletionTimestamp: nil,
				},
			},
			expectedResult: false},
		"DeletionTimestamp is not nil": {
			fakestoragepoolclaim: &apis.StoragePoolClaim{
				ObjectMeta: api_meta.ObjectMeta{
					DeletionTimestamp: &api_meta.Time{time.Now(),
					},
				},
			},
			expectedResult: true},
	}
	for name, test := range tests {
		t.Run(name, func(t *testing.T) {
			result := IsDeleteEvent(test.fakestoragepoolclaim)
			if result != test.expectedResult {
				t.Errorf("Test case failed: expected '%v' but got '%v' ", test.expectedResult, result)
			}
		})
	}
}
<commit_msg>Add test for controller instance in spc controller (#418)<commit_after>/*
Copyright 2018 The OpenEBS Authors

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
package spc

import (
	"testing"
	apis "github.com/openebs/maya/pkg/apis/openebs.io/v1alpha1"
	api_meta "k8s.io/apimachinery/pkg/apis/meta/v1"
	"time"
	openebsFakeClientset "github.com/openebs/maya/pkg/client/clientset/versioned/fake"
	informers "github.com/openebs/maya/pkg/client/informers/externalversions"
	kubeinformers "k8s.io/client-go/informers"
	"k8s.io/client-go/kubernetes/fake"
)

// TestIsDeleteEvent function tests IsDeleteEvent function

// IsDeleteEvent function takes a kubernetes object as argument
// and returns a boolean value which tells whether the object is
// scheduled for deletion or not

func TestIsDeleteEvent(t *testing.T) {

	tests := map[string]struct {
		fakestoragepoolclaim *apis.StoragePoolClaim
		expectedResult       bool
	}{
		"DeletionTimestamp is nil": {
			fakestoragepoolclaim: &apis.StoragePoolClaim{
				ObjectMeta: api_meta.ObjectMeta{
					DeletionTimestamp: nil,
				},
			},
			expectedResult: false},
		"DeletionTimestamp is not nil": {
			fakestoragepoolclaim: &apis.StoragePoolClaim{
				ObjectMeta: api_meta.ObjectMeta{
					DeletionTimestamp: &api_meta.Time{time.Now(),
					},
				},
			},
			expectedResult: true},
	}
	for name, test := range tests {
		t.Run(name, func(t *testing.T) {
			result := IsDeleteEvent(test.fakestoragepoolclaim)
			if result != test.expectedResult {
				t.Errorf("Test case failed: expected '%v' but got '%v' ", test.expectedResult, result)
			}
		})
	}
}

// TestNewCStorPoolController function tests if the kubernetes, openebs clientsets
// and other required objects are present in controller instance.

func TestNewCStorPoolController(t *testing.T) {
	fakeKubeClient := fake.NewSimpleClientset()
	fakeOpenebsClient := openebsFakeClientset.NewSimpleClientset()

	kubeInformerFactory := kubeinformers.NewSharedInformerFactory(fakeKubeClient, time.Second*30)
	openebsInformerFactory := informers.NewSharedInformerFactory(fakeOpenebsClient, time.Second*30)
	// Instantiate the controller
	controller := NewController(fakeKubeClient, fakeOpenebsClient, kubeInformerFactory,
		openebsInformerFactory)

	if controller.spcSynced == nil{
		t.Errorf("No spc cache sync in controller object")
	}

	if controller.workqueue == nil{
		t.Errorf("No workqueue in controller object")
	}
	if controller.recorder == nil{
		t.Errorf("No recorder in controller object")
	}
	if controller.kubeclientset != fakeKubeClient {
		t.Errorf("SPC controller object's kubeclientset mismatch")
	}
	if controller.clientset != fakeOpenebsClient {
		t.Errorf("SPC controller object's openebsclientset mismatch")
	}
}
<|endoftext|>


<commit_before>package google

import (
	"fmt"
	"testing"

	"github.com/hashicorp/terraform/helper/acctest"
	"github.com/hashicorp/terraform/helper/resource"
)

func TestAccDataSourceRegionInstanceGroup(t *testing.T) {
	t.Parallel()
	name := "acctest-" + acctest.RandString(6)
	resource.Test(t, resource.TestCase{
		PreCheck:  func() { testAccPreCheck(t) },
		Providers: testAccProviders,
		Steps: []resource.TestStep{
			{
				Config: testAccDataSourceRegionInstanceGroup_basic(name),
				Check: resource.ComposeTestCheckFunc(
					resource.TestCheckResourceAttr("data.google_compute_region_instance_group.data_source", "name", name),
					resource.TestCheckResourceAttr("data.google_compute_region_instance_group.data_source", "project", getTestProjectFromEnv()),
					resource.TestCheckResourceAttr("data.google_compute_region_instance_group.data_source", "instances.#", "10")),
			},
		},
	})
}

func testAccDataSourceRegionInstanceGroup_basic(instanceManagerName string) string {
	return fmt.Sprintf(`
resource "google_compute_health_check" "autohealing" {
	name = "%s"
	check_interval_sec = 1
	timeout_sec = 1
	healthy_threshold = 2
	unhealthy_threshold = 10

	http_health_check {
		request_path = "/"
		port = "80"
	}
}

resource "google_compute_target_pool" "foo" {
	name = "%s"
}

data "google_compute_image" "debian" {
	project = "debian-cloud"
	name    = "debian-9-stretch-v20171129"
}

resource "google_compute_instance_template" "foo" {
	machine_type = "n1-standard-1"
	disk {
		source_image = "${data.google_compute_image.debian.self_link}"
	}
	network_interface {
		access_config {
		}
		network = "default"
	}
}

resource "google_compute_region_instance_group_manager" "foo" {
	name = "%s"
	base_instance_name = "foo"
	instance_template = "${google_compute_instance_template.foo.self_link}"
	region = "us-central1"
	target_pools = ["${google_compute_target_pool.foo.self_link}"]
	target_size = 1

	named_port {
		name = "web"
		port = 80
	}
	wait_for_instances = true

	auto_healing_policies {
		health_check = "${google_compute_health_check.autohealing.self_link}"
		initial_delay_sec = 10
	}
}

data "google_compute_region_instance_group" "data_source" {
	self_link = "${google_compute_region_instance_group_manager.foo.instance_group}"
}
`, acctest.RandomWithPrefix("test-rigm-"), acctest.RandomWithPrefix("test-rigm-"), instanceManagerName)
}
<commit_msg>Fix region instance group datasource test config to match new config format.<commit_after>package google

import (
	"fmt"
	"testing"

	"github.com/hashicorp/terraform/helper/acctest"
	"github.com/hashicorp/terraform/helper/resource"
)

func TestAccDataSourceRegionInstanceGroup(t *testing.T) {
	t.Parallel()
	name := "acctest-" + acctest.RandString(6)
	resource.Test(t, resource.TestCase{
		PreCheck:  func() { testAccPreCheck(t) },
		Providers: testAccProviders,
		Steps: []resource.TestStep{
			{
				Config: testAccDataSourceRegionInstanceGroup_basic(name),
				Check: resource.ComposeTestCheckFunc(
					resource.TestCheckResourceAttr("data.google_compute_region_instance_group.data_source", "name", name),
					resource.TestCheckResourceAttr("data.google_compute_region_instance_group.data_source", "project", getTestProjectFromEnv()),
					resource.TestCheckResourceAttr("data.google_compute_region_instance_group.data_source", "instances.#", "10")),
			},
		},
	})
}

func testAccDataSourceRegionInstanceGroup_basic(instanceManagerName string) string {
	return fmt.Sprintf(`
resource "google_compute_health_check" "autohealing" {
	name = "%s"
	check_interval_sec = 1
	timeout_sec = 1
	healthy_threshold = 2
	unhealthy_threshold = 10

	http_health_check {
		request_path = "/"
		port = "80"
	}
}

resource "google_compute_target_pool" "foo" {
	name = "%s"
}

data "google_compute_image" "debian" {
	project = "debian-cloud"
	name    = "debian-9-stretch-v20171129"
}

resource "google_compute_instance_template" "foo" {
	machine_type = "n1-standard-1"
	disk {
		source_image = "${data.google_compute_image.debian.self_link}"
	}
	network_interface {
		access_config {
		}
		network = "default"
	}
}

resource "google_compute_region_instance_group_manager" "foo" {
	name = "%s"
	base_instance_name = "foo"
	region = "us-central1"
	target_pools = ["${google_compute_target_pool.foo.self_link}"]
	target_size = 1

	version {
		name = "primary"
		instance_template = "${google_compute_instance_template.foo.self_link}"
	}

	named_port {
		name = "web"
		port = 80
	}
	wait_for_instances = true

	auto_healing_policies {
		health_check = "${google_compute_health_check.autohealing.self_link}"
		initial_delay_sec = 10
	}
	timeouts {
		create = "10m"
	}
}

data "google_compute_region_instance_group" "data_source" {
	self_link = "${google_compute_region_instance_group_manager.foo.instance_group}"
}
`, acctest.RandomWithPrefix("test-rigm-"), acctest.RandomWithPrefix("test-rigm-"), instanceManagerName)
}
<|endoftext|>


<commit_before>package allocrunner

import (
	"context"
	"sync"
	"testing"
	"time"

	consulapi "github.com/hashicorp/consul/api"
	"github.com/hashicorp/nomad/client/allocrunner/interfaces"
	"github.com/hashicorp/nomad/client/consul"
	cstructs "github.com/hashicorp/nomad/client/structs"
	agentconsul "github.com/hashicorp/nomad/command/agent/consul"
	"github.com/hashicorp/nomad/helper/testlog"
	"github.com/hashicorp/nomad/helper/uuid"
	"github.com/hashicorp/nomad/nomad/mock"
	"github.com/hashicorp/nomad/nomad/structs"
	"github.com/stretchr/testify/assert"
	"github.com/stretchr/testify/require"
)

// statically assert health hook implements the expected interfaces
var _ interfaces.RunnerPrerunHook = (*allocHealthWatcherHook)(nil)
var _ interfaces.RunnerUpdateHook = (*allocHealthWatcherHook)(nil)
var _ interfaces.RunnerPostrunHook = (*allocHealthWatcherHook)(nil)
var _ interfaces.ShutdownHook = (*allocHealthWatcherHook)(nil)

// allocHealth is emitted to a chan whenever SetHealth is called
type allocHealth struct {
	healthy    bool
	taskEvents map[string]*structs.TaskEvent
}

// mockHealthSetter implements healthSetter that stores health internally
type mockHealthSetter struct {
	setCalls   int
	clearCalls int
	healthy    *bool
	isDeploy   *bool
	taskEvents map[string]*structs.TaskEvent
	mu         sync.Mutex

	healthCh chan allocHealth
}

// newMockHealthSetter returns a mock HealthSetter that emits all SetHealth
// calls on a buffered chan. Callers who do need need notifications of health
// changes may just create the struct directly.
func newMockHealthSetter() *mockHealthSetter {
	return &mockHealthSetter{
		healthCh: make(chan allocHealth, 1),
	}
}

func (m *mockHealthSetter) SetHealth(healthy, isDeploy bool, taskEvents map[string]*structs.TaskEvent) {
	m.mu.Lock()
	defer m.mu.Unlock()

	m.setCalls++
	m.healthy = &healthy
	m.isDeploy = &isDeploy
	m.taskEvents = taskEvents

	if m.healthCh != nil {
		m.healthCh <- allocHealth{healthy, taskEvents}
	}
}

func (m *mockHealthSetter) ClearHealth() {
	m.mu.Lock()
	defer m.mu.Unlock()

	m.clearCalls++
	m.healthy = nil
	m.isDeploy = nil
	m.taskEvents = nil
}

// TestHealthHook_PrerunPostrun asserts a health hook does not error if it is
// run and postrunned.
func TestHealthHook_PrerunPostrun(t *testing.T) {
	t.Parallel()
	require := require.New(t)

	logger := testlog.HCLogger(t)

	b := cstructs.NewAllocBroadcaster(logger)
	defer b.Close()

	consul := consul.NewMockConsulServiceClient(t, logger)
	hs := &mockHealthSetter{}

	h := newAllocHealthWatcherHook(logger, mock.Alloc(), hs, b.Listen(), consul)

	// Assert we implemented the right interfaces
	prerunh, ok := h.(interfaces.RunnerPrerunHook)
	require.True(ok)
	_, ok = h.(interfaces.RunnerUpdateHook)
	require.True(ok)
	postrunh, ok := h.(interfaces.RunnerPostrunHook)
	require.True(ok)

	// Prerun
	require.NoError(prerunh.Prerun(context.Background()))

	// Assert isDeploy is false (other tests peek at isDeploy to determine
	// if an Update applied)
	ahw := h.(*allocHealthWatcherHook)
	ahw.hookLock.Lock()
	assert.False(t, ahw.isDeploy)
	ahw.hookLock.Unlock()

	// Postrun
	require.NoError(postrunh.Postrun())
}

// TestHealthHook_PrerunUpdatePostrun asserts Updates may be applied concurrently.
func TestHealthHook_PrerunUpdatePostrun(t *testing.T) {
	t.Parallel()
	require := require.New(t)

	alloc := mock.Alloc()

	logger := testlog.HCLogger(t)
	b := cstructs.NewAllocBroadcaster(logger)
	defer b.Close()

	consul := consul.NewMockConsulServiceClient(t, logger)
	hs := &mockHealthSetter{}

	h := newAllocHealthWatcherHook(logger, alloc.Copy(), hs, b.Listen(), consul).(*allocHealthWatcherHook)

	// Prerun
	require.NoError(h.Prerun(context.Background()))

	// Update multiple times in a goroutine to mimic Client behavior
	// (Updates are concurrent with alloc runner but are applied serially).
	errs := make(chan error, 2)
	go func() {
		defer close(errs)
		for i := 0; i < cap(errs); i++ {
			alloc.AllocModifyIndex++
			errs <- h.Update(&interfaces.RunnerUpdateRequest{Alloc: alloc.Copy()})
		}
	}()

	for err := range errs {
		assert.NoError(t, err)
	}

	// Postrun
	require.NoError(h.Postrun())
}

// TestHealthHook_UpdatePrerunPostrun asserts that a hook may have Update
// called before Prerun.
func TestHealthHook_UpdatePrerunPostrun(t *testing.T) {
	t.Parallel()
	require := require.New(t)

	alloc := mock.Alloc()

	logger := testlog.HCLogger(t)
	b := cstructs.NewAllocBroadcaster(logger)
	defer b.Close()

	consul := consul.NewMockConsulServiceClient(t, logger)
	hs := &mockHealthSetter{}

	h := newAllocHealthWatcherHook(logger, alloc.Copy(), hs, b.Listen(), consul).(*allocHealthWatcherHook)

	// Set a DeploymentID to cause ClearHealth to be called
	alloc.DeploymentID = uuid.Generate()

	// Update in a goroutine to mimic Client behavior (Updates are
	// concurrent with alloc runner).
	errs := make(chan error, 1)
	go func(alloc *structs.Allocation) {
		errs <- h.Update(&interfaces.RunnerUpdateRequest{Alloc: alloc})
		close(errs)
	}(alloc.Copy())

	for err := range errs {
		assert.NoError(t, err)
	}

	// Prerun should be a noop
	require.NoError(h.Prerun(context.Background()))

	// Assert that the Update took affect by isDeploy being true
	h.hookLock.Lock()
	assert.True(t, h.isDeploy)
	h.hookLock.Unlock()

	// Postrun
	require.NoError(h.Postrun())
}

// TestHealthHook_Postrun asserts that a hook may have only Postrun called.
func TestHealthHook_Postrun(t *testing.T) {
	t.Parallel()
	require := require.New(t)

	logger := testlog.HCLogger(t)
	b := cstructs.NewAllocBroadcaster(logger)
	defer b.Close()

	consul := consul.NewMockConsulServiceClient(t, logger)
	hs := &mockHealthSetter{}

	h := newAllocHealthWatcherHook(logger, mock.Alloc(), hs, b.Listen(), consul).(*allocHealthWatcherHook)

	// Postrun
	require.NoError(h.Postrun())
}

// TestHealthHook_SetHealth asserts SetHealth is called when health status is
// set. Uses task state and health checks.
func TestHealthHook_SetHealth(t *testing.T) {
	t.Parallel()
	require := require.New(t)

	alloc := mock.Alloc()
	alloc.Job.TaskGroups[0].Migrate.MinHealthyTime = 1 // let's speed things up
	task := alloc.Job.TaskGroups[0].Tasks[0]

	// Synthesize running alloc and tasks
	alloc.ClientStatus = structs.AllocClientStatusRunning
	alloc.TaskStates = map[string]*structs.TaskState{
		task.Name: {
			State:     structs.TaskStateRunning,
			StartedAt: time.Now(),
		},
	}

	// Make Consul response
	check := &consulapi.AgentCheck{
		Name:   task.Services[0].Checks[0].Name,
		Status: consulapi.HealthPassing,
	}
	taskRegs := map[string]*agentconsul.TaskRegistration{
		task.Name: {
			Services: map[string]*agentconsul.ServiceRegistration{
				task.Services[0].Name: {
					Service: &consulapi.AgentService{
						ID:      "foo",
						Service: task.Services[0].Name,
					},
					Checks: []*consulapi.AgentCheck{check},
				},
			},
		},
	}

	logger := testlog.HCLogger(t)
	b := cstructs.NewAllocBroadcaster(logger)
	defer b.Close()

	// Don't reply on the first call
	called := false
	consul := consul.NewMockConsulServiceClient(t, logger)
	consul.AllocRegistrationsFn = func(string) (*agentconsul.AllocRegistration, error) {
		if !called {
			called = true
			return nil, nil
		}

		reg := &agentconsul.AllocRegistration{
			Tasks: taskRegs,
		}

		return reg, nil
	}

	hs := newMockHealthSetter()

	h := newAllocHealthWatcherHook(logger, alloc.Copy(), hs, b.Listen(), consul).(*allocHealthWatcherHook)

	// Prerun
	require.NoError(h.Prerun(context.Background()))

	// Wait for health to be set (healthy)
	select {
	case <-time.After(5 * time.Second):
		t.Fatalf("timeout waiting for health to be set")
	case health := <-hs.healthCh:
		require.True(health.healthy)

		// Healthy allocs shouldn't emit task events
		ev := health.taskEvents[task.Name]
		require.Nilf(ev, "%#v", health.taskEvents)
	}

	// Postrun
	require.NoError(h.Postrun())
}

// TestHealthHook_SystemNoop asserts that system jobs return the noop tracker.
func TestHealthHook_SystemNoop(t *testing.T) {
	t.Parallel()

	h := newAllocHealthWatcherHook(testlog.HCLogger(t), mock.SystemAlloc(), nil, nil, nil)

	// Assert that it's the noop impl
	_, ok := h.(noopAllocHealthWatcherHook)
	require.True(t, ok)

	// Assert the noop impl does not implement any hooks
	_, ok = h.(interfaces.RunnerPrerunHook)
	require.False(t, ok)
	_, ok = h.(interfaces.RunnerUpdateHook)
	require.False(t, ok)
	_, ok = h.(interfaces.RunnerPostrunHook)
	require.False(t, ok)
	_, ok = h.(interfaces.ShutdownHook)
	require.False(t, ok)
}

// TestHealthHook_BatchNoop asserts that batch jobs return the noop tracker.
func TestHealthHook_BatchNoop(t *testing.T) {
	t.Parallel()

	h := newAllocHealthWatcherHook(testlog.HCLogger(t), mock.BatchAlloc(), nil, nil, nil)

	// Assert that it's the noop impl
	_, ok := h.(noopAllocHealthWatcherHook)
	require.True(t, ok)
}
<commit_msg>tests: implement HasHealth for mock health<commit_after>package allocrunner

import (
	"context"
	"sync"
	"testing"
	"time"

	consulapi "github.com/hashicorp/consul/api"
	"github.com/hashicorp/nomad/client/allocrunner/interfaces"
	"github.com/hashicorp/nomad/client/consul"
	cstructs "github.com/hashicorp/nomad/client/structs"
	agentconsul "github.com/hashicorp/nomad/command/agent/consul"
	"github.com/hashicorp/nomad/helper/testlog"
	"github.com/hashicorp/nomad/helper/uuid"
	"github.com/hashicorp/nomad/nomad/mock"
	"github.com/hashicorp/nomad/nomad/structs"
	"github.com/stretchr/testify/assert"
	"github.com/stretchr/testify/require"
)

// statically assert health hook implements the expected interfaces
var _ interfaces.RunnerPrerunHook = (*allocHealthWatcherHook)(nil)
var _ interfaces.RunnerUpdateHook = (*allocHealthWatcherHook)(nil)
var _ interfaces.RunnerPostrunHook = (*allocHealthWatcherHook)(nil)
var _ interfaces.ShutdownHook = (*allocHealthWatcherHook)(nil)

// allocHealth is emitted to a chan whenever SetHealth is called
type allocHealth struct {
	healthy    bool
	taskEvents map[string]*structs.TaskEvent
}

// mockHealthSetter implements healthSetter that stores health internally
type mockHealthSetter struct {
	setCalls   int
	clearCalls int
	healthy    *bool
	isDeploy   *bool
	taskEvents map[string]*structs.TaskEvent
	mu         sync.Mutex

	healthCh chan allocHealth
}

// newMockHealthSetter returns a mock HealthSetter that emits all SetHealth
// calls on a buffered chan. Callers who do need need notifications of health
// changes may just create the struct directly.
func newMockHealthSetter() *mockHealthSetter {
	return &mockHealthSetter{
		healthCh: make(chan allocHealth, 1),
	}
}

func (m *mockHealthSetter) SetHealth(healthy, isDeploy bool, taskEvents map[string]*structs.TaskEvent) {
	m.mu.Lock()
	defer m.mu.Unlock()

	m.setCalls++
	m.healthy = &healthy
	m.isDeploy = &isDeploy
	m.taskEvents = taskEvents

	if m.healthCh != nil {
		m.healthCh <- allocHealth{healthy, taskEvents}
	}
}

func (m *mockHealthSetter) ClearHealth() {
	m.mu.Lock()
	defer m.mu.Unlock()

	m.clearCalls++
	m.healthy = nil
	m.isDeploy = nil
	m.taskEvents = nil
}

func (m *mockHealthSetter) HasHealth() bool {
	m.mu.Lock()
	defer m.mu.Unlock()
	return m.healthy != nil
}

// TestHealthHook_PrerunPostrun asserts a health hook does not error if it is
// run and postrunned.
func TestHealthHook_PrerunPostrun(t *testing.T) {
	t.Parallel()
	require := require.New(t)

	logger := testlog.HCLogger(t)

	b := cstructs.NewAllocBroadcaster(logger)
	defer b.Close()

	consul := consul.NewMockConsulServiceClient(t, logger)
	hs := &mockHealthSetter{}

	h := newAllocHealthWatcherHook(logger, mock.Alloc(), hs, b.Listen(), consul)

	// Assert we implemented the right interfaces
	prerunh, ok := h.(interfaces.RunnerPrerunHook)
	require.True(ok)
	_, ok = h.(interfaces.RunnerUpdateHook)
	require.True(ok)
	postrunh, ok := h.(interfaces.RunnerPostrunHook)
	require.True(ok)

	// Prerun
	require.NoError(prerunh.Prerun(context.Background()))

	// Assert isDeploy is false (other tests peek at isDeploy to determine
	// if an Update applied)
	ahw := h.(*allocHealthWatcherHook)
	ahw.hookLock.Lock()
	assert.False(t, ahw.isDeploy)
	ahw.hookLock.Unlock()

	// Postrun
	require.NoError(postrunh.Postrun())
}

// TestHealthHook_PrerunUpdatePostrun asserts Updates may be applied concurrently.
func TestHealthHook_PrerunUpdatePostrun(t *testing.T) {
	t.Parallel()
	require := require.New(t)

	alloc := mock.Alloc()

	logger := testlog.HCLogger(t)
	b := cstructs.NewAllocBroadcaster(logger)
	defer b.Close()

	consul := consul.NewMockConsulServiceClient(t, logger)
	hs := &mockHealthSetter{}

	h := newAllocHealthWatcherHook(logger, alloc.Copy(), hs, b.Listen(), consul).(*allocHealthWatcherHook)

	// Prerun
	require.NoError(h.Prerun(context.Background()))

	// Update multiple times in a goroutine to mimic Client behavior
	// (Updates are concurrent with alloc runner but are applied serially).
	errs := make(chan error, 2)
	go func() {
		defer close(errs)
		for i := 0; i < cap(errs); i++ {
			alloc.AllocModifyIndex++
			errs <- h.Update(&interfaces.RunnerUpdateRequest{Alloc: alloc.Copy()})
		}
	}()

	for err := range errs {
		assert.NoError(t, err)
	}

	// Postrun
	require.NoError(h.Postrun())
}

// TestHealthHook_UpdatePrerunPostrun asserts that a hook may have Update
// called before Prerun.
func TestHealthHook_UpdatePrerunPostrun(t *testing.T) {
	t.Parallel()
	require := require.New(t)

	alloc := mock.Alloc()

	logger := testlog.HCLogger(t)
	b := cstructs.NewAllocBroadcaster(logger)
	defer b.Close()

	consul := consul.NewMockConsulServiceClient(t, logger)
	hs := &mockHealthSetter{}

	h := newAllocHealthWatcherHook(logger, alloc.Copy(), hs, b.Listen(), consul).(*allocHealthWatcherHook)

	// Set a DeploymentID to cause ClearHealth to be called
	alloc.DeploymentID = uuid.Generate()

	// Update in a goroutine to mimic Client behavior (Updates are
	// concurrent with alloc runner).
	errs := make(chan error, 1)
	go func(alloc *structs.Allocation) {
		errs <- h.Update(&interfaces.RunnerUpdateRequest{Alloc: alloc})
		close(errs)
	}(alloc.Copy())

	for err := range errs {
		assert.NoError(t, err)
	}

	// Prerun should be a noop
	require.NoError(h.Prerun(context.Background()))

	// Assert that the Update took affect by isDeploy being true
	h.hookLock.Lock()
	assert.True(t, h.isDeploy)
	h.hookLock.Unlock()

	// Postrun
	require.NoError(h.Postrun())
}

// TestHealthHook_Postrun asserts that a hook may have only Postrun called.
func TestHealthHook_Postrun(t *testing.T) {
	t.Parallel()
	require := require.New(t)

	logger := testlog.HCLogger(t)
	b := cstructs.NewAllocBroadcaster(logger)
	defer b.Close()

	consul := consul.NewMockConsulServiceClient(t, logger)
	hs := &mockHealthSetter{}

	h := newAllocHealthWatcherHook(logger, mock.Alloc(), hs, b.Listen(), consul).(*allocHealthWatcherHook)

	// Postrun
	require.NoError(h.Postrun())
}

// TestHealthHook_SetHealth asserts SetHealth is called when health status is
// set. Uses task state and health checks.
func TestHealthHook_SetHealth(t *testing.T) {
	t.Parallel()
	require := require.New(t)

	alloc := mock.Alloc()
	alloc.Job.TaskGroups[0].Migrate.MinHealthyTime = 1 // let's speed things up
	task := alloc.Job.TaskGroups[0].Tasks[0]

	// Synthesize running alloc and tasks
	alloc.ClientStatus = structs.AllocClientStatusRunning
	alloc.TaskStates = map[string]*structs.TaskState{
		task.Name: {
			State:     structs.TaskStateRunning,
			StartedAt: time.Now(),
		},
	}

	// Make Consul response
	check := &consulapi.AgentCheck{
		Name:   task.Services[0].Checks[0].Name,
		Status: consulapi.HealthPassing,
	}
	taskRegs := map[string]*agentconsul.TaskRegistration{
		task.Name: {
			Services: map[string]*agentconsul.ServiceRegistration{
				task.Services[0].Name: {
					Service: &consulapi.AgentService{
						ID:      "foo",
						Service: task.Services[0].Name,
					},
					Checks: []*consulapi.AgentCheck{check},
				},
			},
		},
	}

	logger := testlog.HCLogger(t)
	b := cstructs.NewAllocBroadcaster(logger)
	defer b.Close()

	// Don't reply on the first call
	called := false
	consul := consul.NewMockConsulServiceClient(t, logger)
	consul.AllocRegistrationsFn = func(string) (*agentconsul.AllocRegistration, error) {
		if !called {
			called = true
			return nil, nil
		}

		reg := &agentconsul.AllocRegistration{
			Tasks: taskRegs,
		}

		return reg, nil
	}

	hs := newMockHealthSetter()

	h := newAllocHealthWatcherHook(logger, alloc.Copy(), hs, b.Listen(), consul).(*allocHealthWatcherHook)

	// Prerun
	require.NoError(h.Prerun(context.Background()))

	// Wait for health to be set (healthy)
	select {
	case <-time.After(5 * time.Second):
		t.Fatalf("timeout waiting for health to be set")
	case health := <-hs.healthCh:
		require.True(health.healthy)

		// Healthy allocs shouldn't emit task events
		ev := health.taskEvents[task.Name]
		require.Nilf(ev, "%#v", health.taskEvents)
	}

	// Postrun
	require.NoError(h.Postrun())
}

// TestHealthHook_SystemNoop asserts that system jobs return the noop tracker.
func TestHealthHook_SystemNoop(t *testing.T) {
	t.Parallel()

	h := newAllocHealthWatcherHook(testlog.HCLogger(t), mock.SystemAlloc(), nil, nil, nil)

	// Assert that it's the noop impl
	_, ok := h.(noopAllocHealthWatcherHook)
	require.True(t, ok)

	// Assert the noop impl does not implement any hooks
	_, ok = h.(interfaces.RunnerPrerunHook)
	require.False(t, ok)
	_, ok = h.(interfaces.RunnerUpdateHook)
	require.False(t, ok)
	_, ok = h.(interfaces.RunnerPostrunHook)
	require.False(t, ok)
	_, ok = h.(interfaces.ShutdownHook)
	require.False(t, ok)
}

// TestHealthHook_BatchNoop asserts that batch jobs return the noop tracker.
func TestHealthHook_BatchNoop(t *testing.T) {
	t.Parallel()

	h := newAllocHealthWatcherHook(testlog.HCLogger(t), mock.BatchAlloc(), nil, nil, nil)

	// Assert that it's the noop impl
	_, ok := h.(noopAllocHealthWatcherHook)
	require.True(t, ok)
}
<|endoftext|>


<commit_before>/*
Copyright 2014 The Kubernetes Authors.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

// Package app does all of the work necessary to create a Kubernetes
// APIServer by binding together the API, master and APIServer infrastructure.
// It can be configured and called directly or via the hyperkube cache.
package app

import (
	"strings"
	"time"

	"github.com/golang/glog"
	"github.com/pborman/uuid"
	"github.com/spf13/cobra"
	"github.com/spf13/pflag"

	"k8s.io/kubernetes/federation/cmd/federation-apiserver/app/options"
	"k8s.io/kubernetes/pkg/admission"
	"k8s.io/kubernetes/pkg/api"
	"k8s.io/kubernetes/pkg/api/unversioned"
	"k8s.io/kubernetes/pkg/apis/rbac"
	"k8s.io/kubernetes/pkg/apiserver/authenticator"
	apiserveropenapi "k8s.io/kubernetes/pkg/apiserver/openapi"
	authorizerunion "k8s.io/kubernetes/pkg/auth/authorizer/union"
	"k8s.io/kubernetes/pkg/auth/user"
	"k8s.io/kubernetes/pkg/controller/informers"
	"k8s.io/kubernetes/pkg/generated/openapi"
	"k8s.io/kubernetes/pkg/genericapiserver"
	"k8s.io/kubernetes/pkg/genericapiserver/authorizer"
	genericoptions "k8s.io/kubernetes/pkg/genericapiserver/options"
	genericvalidation "k8s.io/kubernetes/pkg/genericapiserver/validation"
	"k8s.io/kubernetes/pkg/registry/cachesize"
	"k8s.io/kubernetes/pkg/registry/generic"
	"k8s.io/kubernetes/pkg/registry/generic/registry"
	"k8s.io/kubernetes/pkg/registry/rbac/clusterrole"
	clusterroleetcd "k8s.io/kubernetes/pkg/registry/rbac/clusterrole/etcd"
	"k8s.io/kubernetes/pkg/registry/rbac/clusterrolebinding"
	clusterrolebindingetcd "k8s.io/kubernetes/pkg/registry/rbac/clusterrolebinding/etcd"
	"k8s.io/kubernetes/pkg/registry/rbac/role"
	roleetcd "k8s.io/kubernetes/pkg/registry/rbac/role/etcd"
	"k8s.io/kubernetes/pkg/registry/rbac/rolebinding"
	rolebindingetcd "k8s.io/kubernetes/pkg/registry/rbac/rolebinding/etcd"
	"k8s.io/kubernetes/pkg/routes"
	"k8s.io/kubernetes/pkg/util/wait"
	authenticatorunion "k8s.io/kubernetes/plugin/pkg/auth/authenticator/request/union"
)

// NewAPIServerCommand creates a *cobra.Command object with default parameters
func NewAPIServerCommand() *cobra.Command {
	s := options.NewServerRunOptions()
	s.AddFlags(pflag.CommandLine)
	cmd := &cobra.Command{
		Use: "federation-apiserver",
		Long: `The Kubernetes federation API server validates and configures data
for the api objects which include pods, services, replicationcontrollers, and
others. The API Server services REST operations and provides the frontend to the
cluster's shared state through which all other components interact.`,
		Run: func(cmd *cobra.Command, args []string) {
		},
	}
	return cmd
}

// Run runs the specified APIServer.  This should never exit.
func Run(s *options.ServerRunOptions) error {
	genericvalidation.VerifyEtcdServersList(s.ServerRunOptions)
	genericapiserver.DefaultAndValidateRunOptions(s.ServerRunOptions)

	// TODO: register cluster federation resources here.
	resourceConfig := genericapiserver.NewResourceConfig()

	if s.StorageConfig.DeserializationCacheSize == 0 {
		// When size of cache is not explicitly set, set it to 50000
		s.StorageConfig.DeserializationCacheSize = 50000
	}
	storageGroupsToEncodingVersion, err := s.StorageGroupsToEncodingVersion()
	if err != nil {
		glog.Fatalf("error generating storage version map: %s", err)
	}
	storageFactory, err := genericapiserver.BuildDefaultStorageFactory(
		s.StorageConfig, s.DefaultStorageMediaType, api.Codecs,
		genericapiserver.NewDefaultResourceEncodingConfig(), storageGroupsToEncodingVersion,
		[]unversioned.GroupVersionResource{}, resourceConfig, s.RuntimeConfig)
	if err != nil {
		glog.Fatalf("error in initializing storage factory: %s", err)
	}

	for _, override := range s.EtcdServersOverrides {
		tokens := strings.Split(override, "#")
		if len(tokens) != 2 {
			glog.Errorf("invalid value of etcd server overrides: %s", override)
			continue
		}

		apiresource := strings.Split(tokens[0], "/")
		if len(apiresource) != 2 {
			glog.Errorf("invalid resource definition: %s", tokens[0])
			continue
		}
		group := apiresource[0]
		resource := apiresource[1]
		groupResource := unversioned.GroupResource{Group: group, Resource: resource}

		servers := strings.Split(tokens[1], ";")
		storageFactory.SetEtcdLocation(groupResource, servers)
	}

	apiAuthenticator, err := authenticator.New(authenticator.AuthenticatorConfig{
		Anonymous:         s.AnonymousAuth,
		AnyToken:          s.EnableAnyToken,
		BasicAuthFile:     s.BasicAuthFile,
		ClientCAFile:      s.ClientCAFile,
		TokenAuthFile:     s.TokenAuthFile,
		OIDCIssuerURL:     s.OIDCIssuerURL,
		OIDCClientID:      s.OIDCClientID,
		OIDCCAFile:        s.OIDCCAFile,
		OIDCUsernameClaim: s.OIDCUsernameClaim,
		OIDCGroupsClaim:   s.OIDCGroupsClaim,
		KeystoneURL:       s.KeystoneURL,
	})
	if err != nil {
		glog.Fatalf("Invalid Authentication Config: %v", err)
	}

	authorizationModeNames := strings.Split(s.AuthorizationMode, ",")

	modeEnabled := func(mode string) bool {
		for _, m := range authorizationModeNames {
			if m == mode {
				return true
			}
		}
		return false
	}

	authorizationConfig := authorizer.AuthorizationConfig{
		PolicyFile:                  s.AuthorizationPolicyFile,
		WebhookConfigFile:           s.AuthorizationWebhookConfigFile,
		WebhookCacheAuthorizedTTL:   s.AuthorizationWebhookCacheAuthorizedTTL,
		WebhookCacheUnauthorizedTTL: s.AuthorizationWebhookCacheUnauthorizedTTL,
		RBACSuperUser:               s.AuthorizationRBACSuperUser,
	}
	if modeEnabled(genericoptions.ModeRBAC) {
		mustGetRESTOptions := func(resource string) generic.RESTOptions {
			config, err := storageFactory.NewConfig(rbac.Resource(resource))
			if err != nil {
				glog.Fatalf("Unable to get %s storage: %v", resource, err)
			}
			return generic.RESTOptions{StorageConfig: config, Decorator: generic.UndecoratedStorage, ResourcePrefix: storageFactory.ResourcePrefix(rbac.Resource(resource))}
		}

		// For initial bootstrapping go directly to etcd to avoid privillege escalation check.
		authorizationConfig.RBACRoleRegistry = role.NewRegistry(roleetcd.NewREST(mustGetRESTOptions("roles")))
		authorizationConfig.RBACRoleBindingRegistry = rolebinding.NewRegistry(rolebindingetcd.NewREST(mustGetRESTOptions("rolebindings")))
		authorizationConfig.RBACClusterRoleRegistry = clusterrole.NewRegistry(clusterroleetcd.NewREST(mustGetRESTOptions("clusterroles")))
		authorizationConfig.RBACClusterRoleBindingRegistry = clusterrolebinding.NewRegistry(clusterrolebindingetcd.NewREST(mustGetRESTOptions("clusterrolebindings")))
	}

	apiAuthorizer, err := authorizer.NewAuthorizerFromAuthorizationConfig(authorizationModeNames, authorizationConfig)
	if err != nil {
		glog.Fatalf("Invalid Authorization Config: %v", err)
	}

	admissionControlPluginNames := strings.Split(s.AdmissionControl, ",")
	privilegedLoopbackToken := uuid.NewRandom().String()

	selfClientConfig, err := s.NewSelfClientConfig(privilegedLoopbackToken)
	if err != nil {
		glog.Fatalf("Failed to create clientset: %v", err)
	}
	client, err := s.NewSelfClient(privilegedLoopbackToken)
	if err != nil {
		glog.Errorf("Failed to create clientset: %v", err)
	}

	// TODO(dims): We probably need to add an option "EnableLoopbackToken"
	if apiAuthenticator != nil {
		var uid = uuid.NewRandom().String()
		tokens := make(map[string]*user.DefaultInfo)
		tokens[privilegedLoopbackToken] = &user.DefaultInfo{
			Name:   user.APIServerUser,
			UID:    uid,
			Groups: []string{user.SystemPrivilegedGroup},
		}

		tokenAuthenticator := authenticator.NewAuthenticatorFromTokens(tokens)
		apiAuthenticator = authenticatorunion.New(tokenAuthenticator, apiAuthenticator)

		tokenAuthorizer := authorizer.NewPrivilegedGroups(user.SystemPrivilegedGroup)
		apiAuthorizer = authorizerunion.New(tokenAuthorizer, apiAuthorizer)
	}

	sharedInformers := informers.NewSharedInformerFactory(client, 10*time.Minute)
	pluginInitializer := admission.NewPluginInitializer(sharedInformers, apiAuthorizer)

	admissionController, err := admission.NewFromPlugins(client, admissionControlPluginNames, s.AdmissionControlConfigFile, pluginInitializer)
	if err != nil {
		glog.Fatalf("Failed to initialize plugins: %v", err)
	}
	genericConfig := genericapiserver.NewConfig(s.ServerRunOptions)
	// TODO: Move the following to generic api server as well.
	genericConfig.LoopbackClientConfig = selfClientConfig
	genericConfig.Authenticator = apiAuthenticator
	genericConfig.SupportsBasicAuth = len(s.BasicAuthFile) > 0
	genericConfig.Authorizer = apiAuthorizer
	genericConfig.AuthorizerRBACSuperUser = s.AuthorizationRBACSuperUser
	genericConfig.AdmissionControl = admissionController
	genericConfig.APIResourceConfigSource = storageFactory.APIResourceConfigSource
	genericConfig.MasterServiceNamespace = s.MasterServiceNamespace
	genericConfig.Serializer = api.Codecs
	genericConfig.OpenAPIConfig.Definitions = openapi.OpenAPIDefinitions
	// Reusing api-server's GetOperationID function. if federation and api-server spec diverge and
	// this method does not provide good operation IDs for federation, we should create federation's own GetOperationID.
	genericConfig.OpenAPIConfig.GetOperationID = apiserveropenapi.GetOperationID
	genericConfig.EnableOpenAPISupport = true

	// TODO: Move this to generic api server (Need to move the command line flag).
	if s.EnableWatchCache {
		cachesize.InitializeWatchCacheSizes(s.TargetRAMMB)
		cachesize.SetWatchCacheSizes(s.WatchCacheSizes)
	}

	m, err := genericConfig.Complete().New()
	if err != nil {
		return err
	}

	routes.UIRedirect{}.Install(m.HandlerContainer)
	routes.Logs{}.Install(m.HandlerContainer)

	restOptionsFactory := restOptionsFactory{
		storageFactory:          storageFactory,
		deleteCollectionWorkers: s.DeleteCollectionWorkers,
	}
	if s.EnableWatchCache {
		restOptionsFactory.storageDecorator = registry.StorageWithCacher
	} else {
		restOptionsFactory.storageDecorator = generic.UndecoratedStorage
	}

	installFederationAPIs(m, restOptionsFactory)
	installCoreAPIs(s, m, restOptionsFactory)
	installExtensionsAPIs(m, restOptionsFactory)

	sharedInformers.Start(wait.NeverStop)
	m.Run()
	return nil
}

type restOptionsFactory struct {
	storageFactory          genericapiserver.StorageFactory
	storageDecorator        generic.StorageDecorator
	deleteCollectionWorkers int
}

func (f restOptionsFactory) NewFor(resource unversioned.GroupResource) generic.RESTOptions {
	config, err := f.storageFactory.NewConfig(resource)
	if err != nil {
		glog.Fatalf("Unable to find storage config for %v, due to %v", resource, err.Error())
	}
	return generic.RESTOptions{
		StorageConfig:           config,
		Decorator:               f.storageDecorator,
		DeleteCollectionWorkers: f.deleteCollectionWorkers,
		ResourcePrefix:          f.storageFactory.ResourcePrefix(resource),
	}
}
<commit_msg>run authorization from a cache<commit_after>/*
Copyright 2014 The Kubernetes Authors.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

// Package app does all of the work necessary to create a Kubernetes
// APIServer by binding together the API, master and APIServer infrastructure.
// It can be configured and called directly or via the hyperkube cache.
package app

import (
	"strings"
	"time"

	"github.com/golang/glog"
	"github.com/pborman/uuid"
	"github.com/spf13/cobra"
	"github.com/spf13/pflag"

	"k8s.io/kubernetes/federation/cmd/federation-apiserver/app/options"
	"k8s.io/kubernetes/pkg/admission"
	"k8s.io/kubernetes/pkg/api"
	"k8s.io/kubernetes/pkg/api/unversioned"
	"k8s.io/kubernetes/pkg/apiserver/authenticator"
	apiserveropenapi "k8s.io/kubernetes/pkg/apiserver/openapi"
	authorizerunion "k8s.io/kubernetes/pkg/auth/authorizer/union"
	"k8s.io/kubernetes/pkg/auth/user"
	"k8s.io/kubernetes/pkg/controller/informers"
	"k8s.io/kubernetes/pkg/generated/openapi"
	"k8s.io/kubernetes/pkg/genericapiserver"
	"k8s.io/kubernetes/pkg/genericapiserver/authorizer"
	genericvalidation "k8s.io/kubernetes/pkg/genericapiserver/validation"
	"k8s.io/kubernetes/pkg/registry/cachesize"
	"k8s.io/kubernetes/pkg/registry/generic"
	"k8s.io/kubernetes/pkg/registry/generic/registry"
	"k8s.io/kubernetes/pkg/routes"
	"k8s.io/kubernetes/pkg/util/wait"
	authenticatorunion "k8s.io/kubernetes/plugin/pkg/auth/authenticator/request/union"
)

// NewAPIServerCommand creates a *cobra.Command object with default parameters
func NewAPIServerCommand() *cobra.Command {
	s := options.NewServerRunOptions()
	s.AddFlags(pflag.CommandLine)
	cmd := &cobra.Command{
		Use: "federation-apiserver",
		Long: `The Kubernetes federation API server validates and configures data
for the api objects which include pods, services, replicationcontrollers, and
others. The API Server services REST operations and provides the frontend to the
cluster's shared state through which all other components interact.`,
		Run: func(cmd *cobra.Command, args []string) {
		},
	}
	return cmd
}

// Run runs the specified APIServer.  This should never exit.
func Run(s *options.ServerRunOptions) error {
	genericvalidation.VerifyEtcdServersList(s.ServerRunOptions)
	genericapiserver.DefaultAndValidateRunOptions(s.ServerRunOptions)

	// TODO: register cluster federation resources here.
	resourceConfig := genericapiserver.NewResourceConfig()

	if s.StorageConfig.DeserializationCacheSize == 0 {
		// When size of cache is not explicitly set, set it to 50000
		s.StorageConfig.DeserializationCacheSize = 50000
	}
	storageGroupsToEncodingVersion, err := s.StorageGroupsToEncodingVersion()
	if err != nil {
		glog.Fatalf("error generating storage version map: %s", err)
	}
	storageFactory, err := genericapiserver.BuildDefaultStorageFactory(
		s.StorageConfig, s.DefaultStorageMediaType, api.Codecs,
		genericapiserver.NewDefaultResourceEncodingConfig(), storageGroupsToEncodingVersion,
		[]unversioned.GroupVersionResource{}, resourceConfig, s.RuntimeConfig)
	if err != nil {
		glog.Fatalf("error in initializing storage factory: %s", err)
	}

	for _, override := range s.EtcdServersOverrides {
		tokens := strings.Split(override, "#")
		if len(tokens) != 2 {
			glog.Errorf("invalid value of etcd server overrides: %s", override)
			continue
		}

		apiresource := strings.Split(tokens[0], "/")
		if len(apiresource) != 2 {
			glog.Errorf("invalid resource definition: %s", tokens[0])
			continue
		}
		group := apiresource[0]
		resource := apiresource[1]
		groupResource := unversioned.GroupResource{Group: group, Resource: resource}

		servers := strings.Split(tokens[1], ";")
		storageFactory.SetEtcdLocation(groupResource, servers)
	}

	apiAuthenticator, err := authenticator.New(authenticator.AuthenticatorConfig{
		Anonymous:         s.AnonymousAuth,
		AnyToken:          s.EnableAnyToken,
		BasicAuthFile:     s.BasicAuthFile,
		ClientCAFile:      s.ClientCAFile,
		TokenAuthFile:     s.TokenAuthFile,
		OIDCIssuerURL:     s.OIDCIssuerURL,
		OIDCClientID:      s.OIDCClientID,
		OIDCCAFile:        s.OIDCCAFile,
		OIDCUsernameClaim: s.OIDCUsernameClaim,
		OIDCGroupsClaim:   s.OIDCGroupsClaim,
		KeystoneURL:       s.KeystoneURL,
	})
	if err != nil {
		glog.Fatalf("Invalid Authentication Config: %v", err)
	}

	privilegedLoopbackToken := uuid.NewRandom().String()
	selfClientConfig, err := s.NewSelfClientConfig(privilegedLoopbackToken)
	if err != nil {
		glog.Fatalf("Failed to create clientset: %v", err)
	}
	client, err := s.NewSelfClient(privilegedLoopbackToken)
	if err != nil {
		glog.Errorf("Failed to create clientset: %v", err)
	}
	sharedInformers := informers.NewSharedInformerFactory(client, 10*time.Minute)

	authorizationConfig := authorizer.AuthorizationConfig{
		PolicyFile:                  s.AuthorizationPolicyFile,
		WebhookConfigFile:           s.AuthorizationWebhookConfigFile,
		WebhookCacheAuthorizedTTL:   s.AuthorizationWebhookCacheAuthorizedTTL,
		WebhookCacheUnauthorizedTTL: s.AuthorizationWebhookCacheUnauthorizedTTL,
		RBACSuperUser:               s.AuthorizationRBACSuperUser,
		InformerFactory:             sharedInformers,
	}
	authorizationModeNames := strings.Split(s.AuthorizationMode, ",")
	apiAuthorizer, err := authorizer.NewAuthorizerFromAuthorizationConfig(authorizationModeNames, authorizationConfig)
	if err != nil {
		glog.Fatalf("Invalid Authorization Config: %v", err)
	}

	admissionControlPluginNames := strings.Split(s.AdmissionControl, ",")

	// TODO(dims): We probably need to add an option "EnableLoopbackToken"
	if apiAuthenticator != nil {
		var uid = uuid.NewRandom().String()
		tokens := make(map[string]*user.DefaultInfo)
		tokens[privilegedLoopbackToken] = &user.DefaultInfo{
			Name:   user.APIServerUser,
			UID:    uid,
			Groups: []string{user.SystemPrivilegedGroup},
		}

		tokenAuthenticator := authenticator.NewAuthenticatorFromTokens(tokens)
		apiAuthenticator = authenticatorunion.New(tokenAuthenticator, apiAuthenticator)

		tokenAuthorizer := authorizer.NewPrivilegedGroups(user.SystemPrivilegedGroup)
		apiAuthorizer = authorizerunion.New(tokenAuthorizer, apiAuthorizer)
	}

	pluginInitializer := admission.NewPluginInitializer(sharedInformers, apiAuthorizer)

	admissionController, err := admission.NewFromPlugins(client, admissionControlPluginNames, s.AdmissionControlConfigFile, pluginInitializer)
	if err != nil {
		glog.Fatalf("Failed to initialize plugins: %v", err)
	}
	genericConfig := genericapiserver.NewConfig(s.ServerRunOptions)
	// TODO: Move the following to generic api server as well.
	genericConfig.LoopbackClientConfig = selfClientConfig
	genericConfig.Authenticator = apiAuthenticator
	genericConfig.SupportsBasicAuth = len(s.BasicAuthFile) > 0
	genericConfig.Authorizer = apiAuthorizer
	genericConfig.AuthorizerRBACSuperUser = s.AuthorizationRBACSuperUser
	genericConfig.AdmissionControl = admissionController
	genericConfig.APIResourceConfigSource = storageFactory.APIResourceConfigSource
	genericConfig.MasterServiceNamespace = s.MasterServiceNamespace
	genericConfig.Serializer = api.Codecs
	genericConfig.OpenAPIConfig.Definitions = openapi.OpenAPIDefinitions
	// Reusing api-server's GetOperationID function. if federation and api-server spec diverge and
	// this method does not provide good operation IDs for federation, we should create federation's own GetOperationID.
	genericConfig.OpenAPIConfig.GetOperationID = apiserveropenapi.GetOperationID
	genericConfig.EnableOpenAPISupport = true

	// TODO: Move this to generic api server (Need to move the command line flag).
	if s.EnableWatchCache {
		cachesize.InitializeWatchCacheSizes(s.TargetRAMMB)
		cachesize.SetWatchCacheSizes(s.WatchCacheSizes)
	}

	m, err := genericConfig.Complete().New()
	if err != nil {
		return err
	}

	routes.UIRedirect{}.Install(m.HandlerContainer)
	routes.Logs{}.Install(m.HandlerContainer)

	restOptionsFactory := restOptionsFactory{
		storageFactory:          storageFactory,
		deleteCollectionWorkers: s.DeleteCollectionWorkers,
	}
	if s.EnableWatchCache {
		restOptionsFactory.storageDecorator = registry.StorageWithCacher
	} else {
		restOptionsFactory.storageDecorator = generic.UndecoratedStorage
	}

	installFederationAPIs(m, restOptionsFactory)
	installCoreAPIs(s, m, restOptionsFactory)
	installExtensionsAPIs(m, restOptionsFactory)

	sharedInformers.Start(wait.NeverStop)
	m.Run()
	return nil
}

type restOptionsFactory struct {
	storageFactory          genericapiserver.StorageFactory
	storageDecorator        generic.StorageDecorator
	deleteCollectionWorkers int
}

func (f restOptionsFactory) NewFor(resource unversioned.GroupResource) generic.RESTOptions {
	config, err := f.storageFactory.NewConfig(resource)
	if err != nil {
		glog.Fatalf("Unable to find storage config for %v, due to %v", resource, err.Error())
	}
	return generic.RESTOptions{
		StorageConfig:           config,
		Decorator:               f.storageDecorator,
		DeleteCollectionWorkers: f.deleteCollectionWorkers,
		ResourcePrefix:          f.storageFactory.ResourcePrefix(resource),
	}
}
<|endoftext|>


<commit_before>/*
Copyright 2017 The Kubernetes Authors.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package options

import (
	"fmt"

	"github.com/spf13/pflag"
)

// ServerOption is the main context object for the controller manager.
type ServerOption struct {
	Master               string
	Kubeconfig           string
	SchedulerName        string
	SchedulerConf        string
	NamespaceAsQueue     bool
	EnableLeaderElection bool
	LockObjectNamespace  string
}

// NewServerOption creates a new CMServer with a default config.
func NewServerOption() *ServerOption {
	s := ServerOption{}
	return &s
}

// AddFlags adds flags for a specific CMServer to the specified FlagSet
func (s *ServerOption) AddFlags(fs *pflag.FlagSet) {
	fs.StringVar(&s.Master, "master", s.Master, "The address of the Kubernetes API server (overrides any value in kubeconfig)")
	fs.StringVar(&s.Kubeconfig, "kubeconfig", s.Kubeconfig, "Path to kubeconfig file with authorization and master location information")
	// kube-arbitrator will ignore pods with scheduler names other than specified with the option
	fs.StringVar(&s.SchedulerName, "scheduler-name", "kube-batchd", "kube-batchd will handle pods with the scheduler-name")
	fs.StringVar(&s.SchedulerConf, "scheduler-conf", "", "The namespace and name of ConfigMap for scheduler configuration")
	fs.BoolVar(&s.EnableLeaderElection, "leader-elect", s.EnableLeaderElection, "Start a leader election client and gain leadership before "+
		"executing the main loop. Enable this when running replicated kar-scheduler for high availability")
	fs.BoolVar(&s.NamespaceAsQueue, "enable-namespace-as-queue", s.NamespaceAsQueue, "Make Namespace as Queue with weight one, "+
		"but kube-batchd will not handle Queue CRD anymore")
	fs.StringVar(&s.LockObjectNamespace, "lock-object-namespace", s.LockObjectNamespace, "Define the namespace of the lock object")
}

func (s *ServerOption) CheckOptionOrDie() error {
	if s.EnableLeaderElection && s.LockObjectNamespace == "" {
		return fmt.Errorf("lock-object-namespace must not be nil when LeaderElection is enabled")
	}
	return nil
}
<commit_msg>Enabled NamespaceAsQueue by default.<commit_after>/*
Copyright 2017 The Kubernetes Authors.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package options

import (
	"fmt"

	"github.com/spf13/pflag"
)

// ServerOption is the main context object for the controller manager.
type ServerOption struct {
	Master               string
	Kubeconfig           string
	SchedulerName        string
	SchedulerConf        string
	NamespaceAsQueue     bool
	EnableLeaderElection bool
	LockObjectNamespace  string
}

// NewServerOption creates a new CMServer with a default config.
func NewServerOption() *ServerOption {
	s := ServerOption{}
	return &s
}

// AddFlags adds flags for a specific CMServer to the specified FlagSet
func (s *ServerOption) AddFlags(fs *pflag.FlagSet) {
	fs.StringVar(&s.Master, "master", s.Master, "The address of the Kubernetes API server (overrides any value in kubeconfig)")
	fs.StringVar(&s.Kubeconfig, "kubeconfig", s.Kubeconfig, "Path to kubeconfig file with authorization and master location information")
	// kube-arbitrator will ignore pods with scheduler names other than specified with the option
	fs.StringVar(&s.SchedulerName, "scheduler-name", "kube-batchd", "kube-batchd will handle pods with the scheduler-name")
	fs.StringVar(&s.SchedulerConf, "scheduler-conf", "", "The namespace and name of ConfigMap for scheduler configuration")
	fs.BoolVar(&s.EnableLeaderElection, "leader-elect", s.EnableLeaderElection, "Start a leader election client and gain leadership before "+
		"executing the main loop. Enable this when running replicated kar-scheduler for high availability")
	fs.BoolVar(&s.NamespaceAsQueue, "enable-namespace-as-queue", true, "Make Namespace as Queue with weight one, "+
		"but kube-batchd will not handle Queue CRD anymore")
	fs.StringVar(&s.LockObjectNamespace, "lock-object-namespace", s.LockObjectNamespace, "Define the namespace of the lock object")
}

func (s *ServerOption) CheckOptionOrDie() error {
	if s.EnableLeaderElection && s.LockObjectNamespace == "" {
		return fmt.Errorf("lock-object-namespace must not be nil when LeaderElection is enabled")
	}
	return nil
}
<|endoftext|>


<commit_before>// The primary mechanism for managing state in Go is
// communication over channels. We saw this for example
// with [worker pools](worker-pools). There are a few other
// options for managing state though. Here we'll
// look at using the `sync/atomic` package for _atomic
// counters_ accessed by multiple goroutines.

package main

import "fmt"
import "time"
import "sync/atomic"
import "runtime"

func main() {

	//find the amount of physical cores and set go runtime
	//to use that many threads
	procs := runtime.NumCPU()
	runtime.GOMAXPROCS(procs)

	// We'll use an unsigned integer to represent our
	// (always-positive) counter.
	var ops uint64 = 0

	// To simulate concurrent updates, we'll start 50
	// goroutines that each increment the counter about
	// once a millisecond.
	for i := 0; i < 3; i++ {
		fmt.Println("go-routine", i)
		go func() {
			for {
				// To atomically increment the counter we
				// use `AddUint64`, giving it the memory
				// address of our `ops` counter with the
				// `&` syntax.
				atomic.AddUint64(&ops, 1)

				// Allow other goroutines to proceed.
				//runtime.Gosched()
			}
		}()
	}

	// Wait a second to allow some ops to accumulate.
	time.Sleep(time.Second)

	// In order to safely use the counter while it's still
	// being updated by other goroutines, we extract a
	// copy of the current value into `opsFinal` via
	// `LoadUint64`. As above we need to give this
	// function the memory address `&ops` from which to
	// fetch the value.
	opsFinal := atomic.LoadUint64(&ops)
	fmt.Println("ops:", opsFinal)
}
<commit_msg>comparison between go scheduler and thread matching go routines<commit_after>package main

import "fmt"
import "time"
import "sync/atomic"
import "runtime"

//A test program to compare the performance difference between
//using many goroutines managed by the go scheduler or matching
//the amount of goroutines to threads without using a scheduler.
//Comment the method you don't want to use out.
func main() {
	procNo := runtime.NumCPU()
	runtime.GOMAXPROCS(procNo)
	fmt.Println("Using", procNo, "processors for maximum thread count")

	//goroutinesWithScheduler()
	goroutinesCpuMatch()
}

func goroutinesWithScheduler() {
	//We'll use an unsigned integer as a counter.
	var ops uint64 = 0

	//To simulate concurrent updates, we'll start 10
	//goroutines that each increment the counter as often
	//as they can
	for i := 0; i < 10; i++ {
		go func() {
			for {
				//To atomically increment the counter we
				//use `AddUint64`, giving it the memory
				//address of our `ops` counter with the
				//`&` syntax.
				atomic.AddUint64(&ops, 1)

				//Allow other goroutines to proceed.
				runtime.Gosched()
			}
		}()
	}

	//Wait a second to allow some ops to accumulate.
	time.Sleep(time.Second)

	//In order to safely use the counter while it's still
	//being updated by other goroutines, we extract a
	//copy of the current value into `opsFinal` via
	//`LoadUint64`. As above we need to give this
	//function the memory address `&ops` from which to
	//fetch the value.
	opsFinal := atomic.LoadUint64(&ops)

	fmt.Println("Many goroutines using runtime.Gosched:", opsFinal, "ops per second")
}

func goroutinesCpuMatch() {
	//This benchmark is much the same as above so I have only
	//commented the differences
	var ops uint64 = 0

	//Only produce as many goroutines as there are free threads
	for i := 0; i < runtime.NumCPU()-1; i++ {
		go func() {
			for {
				//this time we just increment the atomic counter
				//as Gosched isn't needed
				atomic.AddUint64(&ops, 1)
			}
		}()
	}

	//Wait a second to allow some ops to accumulate.
	time.Sleep(time.Second)

	opsFinal := atomic.LoadUint64(&ops)

	fmt.Println("goroutines match CPU count:", opsFinal, "ops per second")
}
<|endoftext|>


<commit_before>package main

import (
	"fmt"
	"github.com/codegangsta/cli"
	"io/ioutil"
	"os"
	"os/exec"
	"path"
	"strconv"
	"strings"
	"text/template"
	"time"
)

var informationTemplate = `{{.Artist}}
{{.Date}}
{{.Album}}
{{.Tour}}

Lineage:

Notes:

This source is considered Source 1 for this date:
https://www.depechemode-live.com/wiki/{{wikiescape .Date}}_{{wikiescape .Album}}/Source_1

Track list:

{{range .Tracks}}{{printf "%02d" .Index}}. [{{.Duration}}] {{.Title}}{{if .HasAlternateLeadVocalist}} (*){{end}}
{{end}}Total time: {{.Duration}}

Torrent downloaded from https://www.depechemode-live.com
`

type AlbumData struct {
	Artist   string
	Date     string
	Album    string
	Tour     string
	Tracks   []TrackData
	Duration string
}

type TrackData struct {
	Index                    int
	Title                    string
	Duration                 string
	HasAlternateLeadVocalist bool
}

func generateInformation(c *cli.Context) {
	fileInfo, filepath := checkFilepathArgument(c)
	if fileInfo == nil {
		return
	}

	tourName := c.String("tour")
	if tourName == "" {
		cli.ShowSubcommandHelp(c)
		return
	}

	mode := "batch"
	if c.GlobalBool("single") {
		mode = "single"
	}

	tourfile := c.String("tour-file")
	if tourfile != "" {
		fileInfo, tourfile = getFileOfType(tourfile, false, "tour-file")
		if fileInfo == nil {
			return
		}

		fmt.Println("Processing tours from:", tourfile)
	}

	fmt.Println("The current tour is:", tourName)
	fmt.Printf("The following filepath (%s mode) will be processed: %s\n", mode, filepath)
	notifyDeleteMode(c)

	if !shouldContinue(c) {
		return
	}

	tour := new(Tour)
	tour.Name = tourName
	if tourfile != "" { // tourFile is only for reading "alternate vocalists" into tracks map
		if err := getTourFromTourFile(tourfile, tour); err != nil {
			fmt.Println("[Error]", err)
			if !shouldContinue(c) {
				return
			}
		}
	}

	// Stupid windows
	informationTemplate = strings.Replace(informationTemplate, "\n", "\r\n", -1)

	if mode == "single" {
		generateFile(filepath, fileInfo.Name(), *tour, c.GlobalBool("delete"))
		return
	}

	files, _ := ioutil.ReadDir(filepath)
	for _, file := range files {
		if file.IsDir() {
			name := file.Name()
			generateFile(path.Join(filepath, name), name, *tour, c.GlobalBool("delete"))
		}
	}
}

func generateFile(filepath string, name string, tour Tour, deleteMode bool) {
	var infoFile *os.File
	if filename := path.Join(filepath, name+".txt"); deleteMode {
		removeFile(filename)
		return
	} else {
		infoFile = createFile(filename)
		defer infoFile.Close()
	}

	album := new(AlbumData)
	album.Tour = tour.Name

	var duration int64 = 0 // duration incrementer for the album

	files, _ := ioutil.ReadDir(filepath)
	for _, file := range files {
		if name := file.Name(); (path.Ext(name) == ".flac") && !file.IsDir() {
			index := getTagsFromFile(path.Join(filepath, name), album, &duration)

			if tour.Tracks != nil {
				_, containsAlternateLeadVocalist := tour.Tracks[album.Tracks[index].Title]
				album.Tracks[index].HasAlternateLeadVocalist = containsAlternateLeadVocalist
			}
		}
	}

	format := "4:05" // minute:0second
	if duration >= 3600 {
		format = "15:04:05" // duration is longer than an hour
	}
	album.Duration = time.Unix(duration, 0).Format(format)

	funcMap := template.FuncMap{"wikiescape": wikiescape}
	t := template.Must(template.New("generate").Funcs(funcMap).Parse(informationTemplate))
	err := t.Execute(infoFile, album)
	if err != nil {
		panic(err)
	}
}

func getTagsFromFile(filepath string, album *AlbumData, albumDuration *int64) int {
	args := []string{
		"--show-total-samples",
		"--show-sample-rate",
	}

	nonTagArgs := len(args)
	tags := []string{"TITLE"}

	getAlbumData := album.Artist == ""
	if getAlbumData {
		tags = append(tags,
			"ARTIST",
			"DATE",
			"ALBUM",
		)
	}

	args = append(args, filepath)
	for _, tag := range tags {
		args = append(args, "--show-tag="+tag)
	}

	data, err := exec.Command(
		"metaflac",
		args[:]...,
	).Output()

	if err != nil {
		panic(err)
	}

	var track TrackData
	track.Index = len(album.Tracks) + 1

	lines := strings.Split(string(data), "\r\n")
	if len(lines) != len(args) {
		panic(fmt.Sprintf("[invalid metaflac output] Expected %d lines, got %d", len(args), len(lines)-1))
		// todo, return a bool to delete this file
		// and say that the current file is being skipped
		// perhaps an --ignore flag to enable this feature
		// false by default, to make it cancel the whole procedure?
	}

	var samples, sampleRate int64
	for i, line := range lines {
		line = strings.TrimSpace(line)

		switch {
		case i <= 1:
			value, err := strconv.Atoi(line)
			if err != nil {
				panic(err)
			}

			if i == 0 {
				samples = int64(value)
			} else {
				sampleRate = int64(value)
			}
		case i < len(args)-1:
			tagName := tags[i-nonTagArgs]
			prefix := tagName + "="
			tagValue := ifTrimPrefix(line, prefix)

			switch tagName {
			case "TITLE":
				track.Title = tagValue
			case "ARTIST":
				album.Artist = tagValue
			case "DATE":
				album.Date = tagValue
			case "ALBUM":
				album.Album = ifTrimPrefix(tagValue, album.Date+" ")
			}
		}
	}
	duration := samples / sampleRate
	*albumDuration += duration
	track.Duration = time.Unix(duration, 0).Format("4:05")

	// Finally, add the new track to the album
	album.Tracks = append(album.Tracks, track)
	return len(album.Tracks) - 1
}
<commit_msg>Add basic support for CDs in 'generate'<commit_after>package main

import (
	"fmt"
	"github.com/codegangsta/cli"
	"io/ioutil"
	"os/exec"
	"path"
	"strconv"
	"strings"
	"text/template"
	"time"
)

var informationTemplate = `{{.Artist}}
{{.Date}}
{{.Album}}
{{.Tour}}

Lineage:

Notes:

This source is considered Source 1 for this date:
https://www.depechemode-live.com/wiki/{{wikiescape .Date}}_{{wikiescape .Album}}/Source_1

Track list:

{{range .Tracks}}{{.Prefix}} [{{.Duration}}] {{.Title}}{{if .HasAlternateLeadVocalist}} (*){{end}}
{{end}}Total time: {{.Duration}}

Torrent downloaded from https://www.depechemode-live.com
`

type AlbumData struct {
	Artist   string
	Date     string
	Album    string
	Tour     string
	Tracks   []TrackData
	Duration string
}

type TrackData struct {
	Title                    string
	Duration                 string
	HasAlternateLeadVocalist bool
	Prefix                   string
}

func generateInformation(c *cli.Context) {
	fileInfo, filepath := checkFilepathArgument(c)
	if fileInfo == nil {
		return
	}

	tourName := c.String("tour")
	if tourName == "" {
		cli.ShowSubcommandHelp(c)
		return
	}

	mode := "batch"
	if c.GlobalBool("single") {
		mode = "single"
	}

	tourfile := c.String("tour-file")
	if tourfile != "" {
		fileInfo, tourfile = getFileOfType(tourfile, false, "tour-file")
		if fileInfo == nil {
			return
		}

		fmt.Println("Processing tours from:", tourfile)
	}

	fmt.Println("The current tour is:", tourName)
	fmt.Printf("The following filepath (%s mode) will be processed: %s\n", mode, filepath)
	notifyDeleteMode(c)

	if !shouldContinue(c) {
		return
	}

	tour := new(Tour)
	tour.Name = tourName
	if tourfile != "" { // tourFile is only for reading "alternate vocalists" into tracks map
		if err := getTourFromTourFile(tourfile, tour); err != nil {
			fmt.Println("[Error]", err)
			if !shouldContinue(c) {
				return
			}
		}
	}

	// Stupid windows
	informationTemplate = strings.Replace(informationTemplate, "\n", "\r\n", -1)

	if mode == "single" {
		generateFile(filepath, fileInfo.Name(), *tour, c.GlobalBool("delete"))
		return
	}

	files, _ := ioutil.ReadDir(filepath)
	for _, file := range files {
		if file.IsDir() {
			name := file.Name()
			generateFile(path.Join(filepath, name), name, *tour, c.GlobalBool("delete"))
		}
	}
}

func generateFile(filepath string, name string, tour Tour, deleteMode bool) {
	outputFilename := path.Join(filepath, name+".txt")
	if deleteMode {
		removeFile(outputFilename)
		return
	}

	album := new(AlbumData)
	album.Tour = tour.Name

	var duration int64 = 0 // duration incrementer for the album

	usesCDNames := 0
	folders := make([]string, 0)
	files := make([]string, 0)
	directoryContents, _ := ioutil.ReadDir(filepath)
	for _, fileinfo := range directoryContents {
		filename := fileinfo.Name()
		isDir := fileinfo.IsDir()
		if isDir {
			folders = append(folders, filename)
			if strings.HasPrefix(filename, "CD") {
				usesCDNames += 1
			}
		} else if (path.Ext(filename) == ".flac") && !isDir {
			files = append(files, filename)
		}
	}

	iterating := files
	if usesCDNames > 0 {

		if len(files) > 0 {
			// Contains extra files not in a specific CD
			// Do something!
		}

		// TODO: should we check subfolders inside
		// "CD1"?

		files := make([]string, 0)
		subfolders := make([]string, 0)
		for _, dirName := range folders {
			subdirectory, _ := ioutil.ReadDir(path.Join(filepath, dirName))
			for _, fileinfo := range subdirectory {
				subdirPath := path.Join(dirName, fileinfo.Name())
				if fileinfo.IsDir() {
					subfolders = append(subfolders, subdirPath)
				} else {
					files = append(files, subdirPath)
				}
			}
		}

		if len(subfolders) > 0 {
			fmt.Printf("Skipping! Filepath has depth=3 folders (%s)\n", filepath)
			return
		}

		iterating = files // set it to the new files

	}

	if len(folders) > usesCDNames {
		// Contains extra folders, do something!
		// There's probably a folder like "Bonus"
	}

	for _, file := range iterating {
		// if usesCDNames > 0 {
		// 	continue
		// }

		track := getTagsFromFile(path.Join(filepath, file), album, &duration)

		if tour.Tracks != nil {
			_, containsAlternateLeadVocalist := tour.Tracks[track.Title]
			track.HasAlternateLeadVocalist = containsAlternateLeadVocalist
		}

		prefix := fmt.Sprintf("%02d.", len(album.Tracks)+1)
		if usesCDNames > 0 {
			prefix = strings.TrimPrefix(path.Dir(file), "CD") + "." + prefix
		}
		track.Prefix = prefix

		// Finally, add the new track to the album
		album.Tracks = append(album.Tracks, track)
	}

	if len(album.Tracks) == 0 {
		fmt.Println("Could not create album - aborting creation of", outputFilename)
		return
	}

	format := "4:05" // minute:0second
	if duration >= 3600 {
		format = "15:04:05" // duration is longer than an hour
	}
	album.Duration = time.Unix(duration, 0).Format(format)

	funcMap := template.FuncMap{"wikiescape": wikiescape}
	t := template.Must(template.New("generate").Funcs(funcMap).Parse(informationTemplate))

	infoFile := createFile(outputFilename)
	defer infoFile.Close()
	err := t.Execute(infoFile, album)
	if err != nil {
		panic(err)
	}
}

func getTagsFromFile(filepath string, album *AlbumData, albumDuration *int64) TrackData {
	args := []string{
		"--show-total-samples",
		"--show-sample-rate",
	}

	nonTagArgs := len(args)
	tags := []string{"TITLE"}

	getAlbumData := album.Artist == ""
	if getAlbumData {
		tags = append(tags,
			"ARTIST",
			"DATE",
			"ALBUM",
		)
	}

	args = append(args, filepath)
	for _, tag := range tags {
		args = append(args, "--show-tag="+tag)
	}

	data, err := exec.Command(
		"metaflac",
		args[:]...,
	).Output()

	if err != nil {
		panic(err)
	}

	var track TrackData

	lines := strings.Split(string(data), "\r\n")
	if len(lines) != len(args) {
		panic(fmt.Sprintf("[invalid metaflac output] Expected %d lines, got %d", len(args), len(lines)-1))
		// todo, return a bool to delete this file
		// and say that the current file is being skipped
		// perhaps an --ignore flag to enable this feature
		// false by default, to make it cancel the whole procedure?
	}

	var samples, sampleRate int64
	for i, line := range lines {
		line = strings.TrimSpace(line)

		switch {
		case i <= 1:
			value, err := strconv.Atoi(line)
			if err != nil {
				panic(err)
			}

			if i == 0 {
				samples = int64(value)
			} else {
				sampleRate = int64(value)
			}
		case i < len(args)-1:
			tagName := tags[i-nonTagArgs]
			prefix := tagName + "="
			tagValue := ifTrimPrefix(line, prefix)

			switch tagName {
			case "TITLE":
				track.Title = tagValue
			case "ARTIST":
				album.Artist = tagValue
			case "DATE":
				album.Date = tagValue
			case "ALBUM":
				album.Album = ifTrimPrefix(tagValue, album.Date+" ")
			}
		}
	}
	duration := samples / sampleRate
	*albumDuration += duration
	track.Duration = time.Unix(duration, 0).Format("4:05")

	return track
}
<|endoftext|>