<commit_before>// Copyright 2014 Google Inc. All Rights Reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package api

import (
	"fmt"
	"os"
	"strconv"
	"testing"
	"time"

	info "github.com/google/cadvisor/info/v1"
	"github.com/google/cadvisor/integration/framework"
	"github.com/stretchr/testify/assert"
	"github.com/stretchr/testify/require"
)

// Sanity check the container by:
// - Checking that the specified alias is a valid one for this container.
// - Verifying that stats are not empty.
func sanityCheck(alias string, containerInfo info.ContainerInfo, t *testing.T) {
	assert.Contains(t, containerInfo.Aliases, alias, "Alias %q should be in list of aliases %v", alias, containerInfo.Aliases)
	assert.NotEmpty(t, containerInfo.Stats, "Expected container to have stats")
}

// Waits up to 5s for a container with the specified alias to appear.
func waitForContainer(alias string, fm framework.Framework) {
	err := framework.RetryForDuration(func() error {
		ret, err := fm.Cadvisor().Client().DockerContainer(alias, &info.ContainerInfoRequest{
			NumStats: 1,
		})
		if err != nil {
			return err
		}
		if len(ret.Stats) != 1 {
			return fmt.Errorf("no stats returned for container %q", alias)
		}

		return nil
	}, 5*time.Second)
	require.NoError(fm.T(), err, "Timed out waiting for container %q to be available in cAdvisor: %v", alias, err)
}

// A Docker container in /docker/<ID>
func TestDockerContainerById(t *testing.T) {
	fm := framework.New(t)
	defer fm.Cleanup()

	containerId := fm.Docker().RunPause()

	// Wait for the container to show up.
	waitForContainer(containerId, fm)

	request := &info.ContainerInfoRequest{
		NumStats: 1,
	}
	containerInfo, err := fm.Cadvisor().Client().DockerContainer(containerId, request)
	require.NoError(t, err)

	sanityCheck(containerId, containerInfo, t)
}

// A Docker container in /docker/<name>
func TestDockerContainerByName(t *testing.T) {
	fm := framework.New(t)
	defer fm.Cleanup()

	containerName := fmt.Sprintf("test-docker-container-by-name-%d", os.Getpid())
	fm.Docker().Run(framework.DockerRunArgs{
		Image: "kubernetes/pause",
		Args:  []string{"--name", containerName},
	})

	// Wait for the container to show up.
	waitForContainer(containerName, fm)

	request := &info.ContainerInfoRequest{
		NumStats: 1,
	}
	containerInfo, err := fm.Cadvisor().Client().DockerContainer(containerName, request)
	require.NoError(t, err)

	sanityCheck(containerName, containerInfo, t)
}

// Find the first container with the specified alias in containers.
func findContainer(alias string, containers []info.ContainerInfo, t *testing.T) info.ContainerInfo {
	for _, cont := range containers {
		for _, a := range cont.Aliases {
			if alias == a {
				return cont
			}
		}
	}
	t.Fatalf("Failed to find container %q in %+v", alias, containers)
	return info.ContainerInfo{}
}

// All Docker containers through /docker
func TestGetAllDockerContainers(t *testing.T) {
	fm := framework.New(t)
	defer fm.Cleanup()

	// Wait for the containers to show up.
	containerId1 := fm.Docker().RunPause()
	containerId2 := fm.Docker().RunPause()
	waitForContainer(containerId1, fm)
	waitForContainer(containerId2, fm)

	request := &info.ContainerInfoRequest{
		NumStats: 1,
	}
	containersInfo, err := fm.Cadvisor().Client().AllDockerContainers(request)
	require.NoError(t, err)

	if len(containersInfo) < 2 {
		t.Fatalf("At least 2 Docker containers should exist, received %d: %+v", len(containersInfo), containersInfo)
	}
	sanityCheck(containerId1, findContainer(containerId1, containersInfo, t), t)
	sanityCheck(containerId2, findContainer(containerId2, containersInfo, t), t)
}

// Check expected properties of a Docker container.
func TestBasicDockerContainer(t *testing.T) {
	fm := framework.New(t)
	defer fm.Cleanup()

	containerName := fmt.Sprintf("test-basic-docker-container-%d", os.Getpid())
	containerId := fm.Docker().Run(framework.DockerRunArgs{
		Image: "kubernetes/pause",
		Args: []string{
			"--name", containerName,
		},
	})

	// Wait for the container to show up.
	waitForContainer(containerId, fm)

	request := &info.ContainerInfoRequest{
		NumStats: 1,
	}
	containerInfo, err := fm.Cadvisor().Client().DockerContainer(containerId, request)
	require.NoError(t, err)

	// Check that the contianer is known by both its name and ID.
	sanityCheck(containerId, containerInfo, t)
	sanityCheck(containerName, containerInfo, t)

	assert.Empty(t, containerInfo.Subcontainers, "Should not have subcontainers")
	assert.Len(t, containerInfo.Stats, 1, "Should have exactly one stat")
}

// TODO(vmarmol): Handle if CPU or memory is not isolated on this system.
// Check the ContainerSpec.
func TestDockerContainerSpec(t *testing.T) {
	fm := framework.New(t)
	defer fm.Cleanup()

	cpuShares := uint64(2048)
	cpuMask := "0"
	memoryLimit := uint64(1 << 30) // 1GB
	containerId := fm.Docker().Run(framework.DockerRunArgs{
		Image: "kubernetes/pause",
		Args: []string{
			"--cpu-shares", strconv.FormatUint(cpuShares, 10),
			"--cpuset", cpuMask,
			"--memory", strconv.FormatUint(memoryLimit, 10),
		},
	})

	// Wait for the container to show up.
	waitForContainer(containerId, fm)

	request := &info.ContainerInfoRequest{
		NumStats: 1,
	}
	containerInfo, err := fm.Cadvisor().Client().DockerContainer(containerId, request)
	require.NoError(t, err)
	sanityCheck(containerId, containerInfo, t)

	assert := assert.New(t)

	assert.True(containerInfo.Spec.HasCpu, "CPU should be isolated")
	assert.Equal(containerInfo.Spec.Cpu.Limit, cpuShares, "Container should have %d shares, has %d", cpuShares, containerInfo.Spec.Cpu.Limit)
	assert.Equal(containerInfo.Spec.Cpu.Mask, cpuMask, "Cpu mask should be %q, but is %q", cpuMask, containerInfo.Spec.Cpu.Mask)
	assert.True(containerInfo.Spec.HasMemory, "Memory should be isolated")
	assert.Equal(containerInfo.Spec.Memory.Limit, memoryLimit, "Container should have memory limit of %d, has %d", memoryLimit, containerInfo.Spec.Memory.Limit)
	assert.True(containerInfo.Spec.HasNetwork, "Network should be isolated")
	assert.True(containerInfo.Spec.HasDiskIo, "Blkio should be isolated")
}

// Check the CPU ContainerStats.
func TestDockerContainerCpuStats(t *testing.T) {
	fm := framework.New(t)
	defer fm.Cleanup()

	// Wait for the container to show up.
	containerId := fm.Docker().RunBusybox("ping", "www.google.com")
	waitForContainer(containerId, fm)

	request := &info.ContainerInfoRequest{
		NumStats: 1,
	}
	containerInfo, err := fm.Cadvisor().Client().DockerContainer(containerId, request)
	if err != nil {
		t.Fatal(err)
	}
	sanityCheck(containerId, containerInfo, t)

	// Checks for CpuStats.
	checkCpuStats(t, containerInfo.Stats[0].Cpu)
}

// Check the memory ContainerStats.
func TestDockerContainerMemoryStats(t *testing.T) {
	fm := framework.New(t)
	defer fm.Cleanup()

	// Wait for the container to show up.
	containerId := fm.Docker().RunBusybox("ping", "www.google.com")
	waitForContainer(containerId, fm)

	request := &info.ContainerInfoRequest{
		NumStats: 1,
	}
	containerInfo, err := fm.Cadvisor().Client().DockerContainer(containerId, request)
	require.NoError(t, err)
	sanityCheck(containerId, containerInfo, t)

	// Checks for MemoryStats.
	checkMemoryStats(t, containerInfo.Stats[0].Memory)
}

// Check the network ContainerStats.
func TestDockerContainerNetworkStats(t *testing.T) {
	fm := framework.New(t)
	defer fm.Cleanup()

	// Wait for the container to show up.
	containerId := fm.Docker().RunBusybox("watch", "-n1", "wget", "https://www.google.com/")
	waitForContainer(containerId, fm)

	request := &info.ContainerInfoRequest{
		NumStats: 1,
	}
	containerInfo, err := fm.Cadvisor().Client().DockerContainer(containerId, request)
	require.NoError(t, err)
	sanityCheck(containerId, containerInfo, t)

	// Checks for NetworkStats.
	stat := containerInfo.Stats[0]
	assert := assert.New(t)
	assert.NotEqual(0, stat.Network.TxBytes, "Network tx bytes should not be zero")
	assert.NotEqual(0, stat.Network.TxPackets, "Network tx packets should not be zero")
	assert.NotEqual(0, stat.Network.RxBytes, "Network rx bytes should not be zero")
	assert.NotEqual(0, stat.Network.RxPackets, "Network rx packets should not be zero")
	assert.NotEqual(stat.Network.RxBytes, stat.Network.TxBytes, "Network tx and rx bytes should not be equal")
	assert.NotEqual(stat.Network.RxPackets, stat.Network.TxPackets, "Network tx and rx packets should not be equal")
}
<commit_msg>merging master Merge remote-tracking branch 'master/master'<commit_after>// Copyright 2014 Google Inc. All Rights Reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package api

/*
import (
	"fmt"
	"os"
	"strconv"
	"testing"
	"time"

	info "github.com/google/cadvisor/info/v1"
	"github.com/google/cadvisor/integration/framework"
	"github.com/stretchr/testify/assert"
	"github.com/stretchr/testify/require"
)

// Sanity check the container by:
// - Checking that the specified alias is a valid one for this container.
// - Verifying that stats are not empty.
func sanityCheck(alias string, containerInfo info.ContainerInfo, t *testing.T) {
	assert.Contains(t, containerInfo.Aliases, alias, "Alias %q should be in list of aliases %v", alias, containerInfo.Aliases)
	assert.NotEmpty(t, containerInfo.Stats, "Expected container to have stats")
}

// Waits up to 5s for a container with the specified alias to appear.
func waitForContainer(alias string, fm framework.Framework) {
	err := framework.RetryForDuration(func() error {
		ret, err := fm.Cadvisor().Client().DockerContainer(alias, &info.ContainerInfoRequest{
			NumStats: 1,
		})
		if err != nil {
			return err
		}
		if len(ret.Stats) != 1 {
			return fmt.Errorf("no stats returned for container %q", alias)
		}

		return nil
	}, 5*time.Second)
	require.NoError(fm.T(), err, "Timed out waiting for container %q to be available in cAdvisor: %v", alias, err)
}

// A Docker container in /docker/<ID>
func TestDockerContainerById(t *testing.T) {
	fm := framework.New(t)
	defer fm.Cleanup()

	containerId := fm.Docker().RunPause()

	// Wait for the container to show up.
	waitForContainer(containerId, fm)

	request := &info.ContainerInfoRequest{
		NumStats: 1,
	}
	containerInfo, err := fm.Cadvisor().Client().DockerContainer(containerId, request)
	require.NoError(t, err)

	sanityCheck(containerId, containerInfo, t)
}

// A Docker container in /docker/<name>
func TestDockerContainerByName(t *testing.T) {
	fm := framework.New(t)
	defer fm.Cleanup()

	containerName := fmt.Sprintf("test-docker-container-by-name-%d", os.Getpid())
	fm.Docker().Run(framework.DockerRunArgs{
		Image: "kubernetes/pause",
		Args:  []string{"--name", containerName},
	})

	// Wait for the container to show up.
	waitForContainer(containerName, fm)

	request := &info.ContainerInfoRequest{
		NumStats: 1,
	}
	containerInfo, err := fm.Cadvisor().Client().DockerContainer(containerName, request)
	require.NoError(t, err)

	sanityCheck(containerName, containerInfo, t)
}

// Find the first container with the specified alias in containers.
func findContainer(alias string, containers []info.ContainerInfo, t *testing.T) info.ContainerInfo {
	for _, cont := range containers {
		for _, a := range cont.Aliases {
			if alias == a {
				return cont
			}
		}
	}
	t.Fatalf("Failed to find container %q in %+v", alias, containers)
	return info.ContainerInfo{}
}

// All Docker containers through /docker
func TestGetAllDockerContainers(t *testing.T) {
	fm := framework.New(t)
	defer fm.Cleanup()

	// Wait for the containers to show up.
	containerId1 := fm.Docker().RunPause()
	containerId2 := fm.Docker().RunPause()
	waitForContainer(containerId1, fm)
	waitForContainer(containerId2, fm)

	request := &info.ContainerInfoRequest{
		NumStats: 1,
	}
	containersInfo, err := fm.Cadvisor().Client().AllDockerContainers(request)
	require.NoError(t, err)

	if len(containersInfo) < 2 {
		t.Fatalf("At least 2 Docker containers should exist, received %d: %+v", len(containersInfo), containersInfo)
	}
	sanityCheck(containerId1, findContainer(containerId1, containersInfo, t), t)
	sanityCheck(containerId2, findContainer(containerId2, containersInfo, t), t)
}

// Check expected properties of a Docker container.
func TestBasicDockerContainer(t *testing.T) {
	fm := framework.New(t)
	defer fm.Cleanup()

	containerName := fmt.Sprintf("test-basic-docker-container-%d", os.Getpid())
	containerId := fm.Docker().Run(framework.DockerRunArgs{
		Image: "kubernetes/pause",
		Args: []string{
			"--name", containerName,
		},
	})

	// Wait for the container to show up.
	waitForContainer(containerId, fm)

	request := &info.ContainerInfoRequest{
		NumStats: 1,
	}
	containerInfo, err := fm.Cadvisor().Client().DockerContainer(containerId, request)
	require.NoError(t, err)

	// Check that the contianer is known by both its name and ID.
	sanityCheck(containerId, containerInfo, t)
	sanityCheck(containerName, containerInfo, t)

	assert.Empty(t, containerInfo.Subcontainers, "Should not have subcontainers")
	assert.Len(t, containerInfo.Stats, 1, "Should have exactly one stat")
}

// TODO(vmarmol): Handle if CPU or memory is not isolated on this system.
// Check the ContainerSpec.
func TestDockerContainerSpec(t *testing.T) {
	fm := framework.New(t)
	defer fm.Cleanup()

	cpuShares := uint64(2048)
	cpuMask := "0"
	memoryLimit := uint64(1 << 30) // 1GB
	containerId := fm.Docker().Run(framework.DockerRunArgs{
		Image: "kubernetes/pause",
		Args: []string{
			"--cpu-shares", strconv.FormatUint(cpuShares, 10),
			"--cpuset", cpuMask,
			"--memory", strconv.FormatUint(memoryLimit, 10),
		},
	})

	// Wait for the container to show up.
	waitForContainer(containerId, fm)

	request := &info.ContainerInfoRequest{
		NumStats: 1,
	}
	containerInfo, err := fm.Cadvisor().Client().DockerContainer(containerId, request)
	require.NoError(t, err)
	sanityCheck(containerId, containerInfo, t)

	assert := assert.New(t)

	assert.True(containerInfo.Spec.HasCpu, "CPU should be isolated")
	assert.Equal(containerInfo.Spec.Cpu.Limit, cpuShares, "Container should have %d shares, has %d", cpuShares, containerInfo.Spec.Cpu.Limit)
	assert.Equal(containerInfo.Spec.Cpu.Mask, cpuMask, "Cpu mask should be %q, but is %q", cpuMask, containerInfo.Spec.Cpu.Mask)
	assert.True(containerInfo.Spec.HasMemory, "Memory should be isolated")
	assert.Equal(containerInfo.Spec.Memory.Limit, memoryLimit, "Container should have memory limit of %d, has %d", memoryLimit, containerInfo.Spec.Memory.Limit)
	assert.True(containerInfo.Spec.HasNetwork, "Network should be isolated")
	assert.True(containerInfo.Spec.HasDiskIo, "Blkio should be isolated")
}

// Check the CPU ContainerStats.
func TestDockerContainerCpuStats(t *testing.T) {
	fm := framework.New(t)
	defer fm.Cleanup()

	// Wait for the container to show up.
	containerId := fm.Docker().RunBusybox("ping", "www.google.com")
	waitForContainer(containerId, fm)

	request := &info.ContainerInfoRequest{
		NumStats: 1,
	}
	containerInfo, err := fm.Cadvisor().Client().DockerContainer(containerId, request)
	if err != nil {
		t.Fatal(err)
	}
	sanityCheck(containerId, containerInfo, t)

	// Checks for CpuStats.
	checkCpuStats(t, containerInfo.Stats[0].Cpu)
}

// Check the memory ContainerStats.
func TestDockerContainerMemoryStats(t *testing.T) {
	fm := framework.New(t)
	defer fm.Cleanup()

	// Wait for the container to show up.
	containerId := fm.Docker().RunBusybox("ping", "www.google.com")
	waitForContainer(containerId, fm)

	request := &info.ContainerInfoRequest{
		NumStats: 1,
	}
	containerInfo, err := fm.Cadvisor().Client().DockerContainer(containerId, request)
	require.NoError(t, err)
	sanityCheck(containerId, containerInfo, t)

	// Checks for MemoryStats.
	checkMemoryStats(t, containerInfo.Stats[0].Memory)
}

// Check the network ContainerStats.
func TestDockerContainerNetworkStats(t *testing.T) {
	fm := framework.New(t)
	defer fm.Cleanup()

	// Wait for the container to show up.
	containerId := fm.Docker().RunBusybox("watch", "-n1", "wget", "https://www.google.com/")
	waitForContainer(containerId, fm)

	request := &info.ContainerInfoRequest{
		NumStats: 1,
	}
	containerInfo, err := fm.Cadvisor().Client().DockerContainer(containerId, request)
	require.NoError(t, err)
	sanityCheck(containerId, containerInfo, t)

	// Checks for NetworkStats.
	stat := containerInfo.Stats[0]
	assert := assert.New(t)
	assert.NotEqual(0, stat.Network.TxBytes, "Network tx bytes should not be zero")
	assert.NotEqual(0, stat.Network.TxPackets, "Network tx packets should not be zero")
	assert.NotEqual(0, stat.Network.RxBytes, "Network rx bytes should not be zero")
	assert.NotEqual(0, stat.Network.RxPackets, "Network rx packets should not be zero")
	assert.NotEqual(stat.Network.RxBytes, stat.Network.TxBytes, "Network tx and rx bytes should not be equal")
	assert.NotEqual(stat.Network.RxPackets, stat.Network.TxPackets, "Network tx and rx packets should not be equal")
}
*/
<|endoftext|>


<commit_before>// Copyright 2020 PingCAP, Inc. Licensed under Apache-2.0.

package export

import (
	"context"
	"database/sql"

	"github.com/pingcap/errors"
	"github.com/pingcap/tidb/br/pkg/utils"
	"github.com/pingcap/tidb/br/pkg/version"
	tcontext "github.com/pingcap/tidb/dumpling/context"
)

const (
	// ConsistencyTypeAuto will use flush for MySQL/MariaDB and snapshot for TiDB.
	ConsistencyTypeAuto = "auto"
	// ConsistencyTypeFlush will use FLUSH TABLES WITH READ LOCK to temporarily interrupt the DML and DDL operations of the replica database,
	// to ensure the global consistency of the backup connection.
	ConsistencyTypeFlush = "flush"
	// ConsistencyTypeLock will add read locks on all tables to be exported.
	ConsistencyTypeLock = "lock"
	// ConsistencyTypeSnapshot gets a consistent snapshot of the specified timestamp and exports it.
	ConsistencyTypeSnapshot = "snapshot"
	// ConsistencyTypeNone doesn't guarantee for consistency.
	ConsistencyTypeNone = "none"
)

var errTiDBDisableTableLock = errors.New("try to apply lock consistency on TiDB but it doesn't enable table lock. please set enable-table-lock=true in tidb server config")

// NewConsistencyController returns a new consistency controller
func NewConsistencyController(ctx context.Context, conf *Config, session *sql.DB) (ConsistencyController, error) {
	conn, err := session.Conn(ctx)
	if err != nil {
		return nil, errors.Trace(err)
	}
	switch conf.Consistency {
	case ConsistencyTypeFlush:
		return &ConsistencyFlushTableWithReadLock{
			serverType: conf.ServerInfo.ServerType,
			conn:       conn,
		}, nil
	case ConsistencyTypeLock:
		return &ConsistencyLockDumpingTables{
			conn: conn,
			conf: conf,
		}, nil
	case ConsistencyTypeSnapshot:
		if conf.ServerInfo.ServerType != version.ServerTypeTiDB {
			return nil, errors.New("snapshot consistency is not supported for this server")
		}
		return &ConsistencyNone{}, nil
	case ConsistencyTypeNone:
		return &ConsistencyNone{}, nil
	default:
		return nil, errors.Errorf("invalid consistency option %s", conf.Consistency)
	}
}

// ConsistencyController is the interface that controls the consistency of exporting progress
type ConsistencyController interface {
	Setup(*tcontext.Context) error
	TearDown(context.Context) error
	PingContext(context.Context) error
}

// ConsistencyNone dumps without adding locks, which cannot guarantee consistency
type ConsistencyNone struct{}

// Setup implements ConsistencyController.Setup
func (*ConsistencyNone) Setup(_ *tcontext.Context) error {
	return nil
}

// TearDown implements ConsistencyController.TearDown
func (*ConsistencyNone) TearDown(_ context.Context) error {
	return nil
}

// PingContext implements ConsistencyController.PingContext
func (*ConsistencyNone) PingContext(_ context.Context) error {
	return nil
}

// ConsistencyFlushTableWithReadLock uses FlushTableWithReadLock before the dump
type ConsistencyFlushTableWithReadLock struct {
	serverType version.ServerType
	conn       *sql.Conn
}

// Setup implements ConsistencyController.Setup
func (c *ConsistencyFlushTableWithReadLock) Setup(tctx *tcontext.Context) error {
	if c.serverType == version.ServerTypeTiDB {
		return errors.New("'flush table with read lock' cannot be used to ensure the consistency in TiDB")
	}
	return FlushTableWithReadLock(tctx, c.conn)
}

// TearDown implements ConsistencyController.TearDown
func (c *ConsistencyFlushTableWithReadLock) TearDown(ctx context.Context) error {
	if c.conn == nil {
		return nil
	}
	defer func() {
		_ = c.conn.Close()
		c.conn = nil
	}()
	return UnlockTables(ctx, c.conn)
}

// PingContext implements ConsistencyController.PingContext
func (c *ConsistencyFlushTableWithReadLock) PingContext(ctx context.Context) error {
	if c.conn == nil {
		return errors.New("consistency connection has already been closed")
	}
	return c.conn.PingContext(ctx)
}

// ConsistencyLockDumpingTables execute lock tables read on all tables before dump
type ConsistencyLockDumpingTables struct {
	conn *sql.Conn
	conf *Config
}

// Setup implements ConsistencyController.Setup
func (c *ConsistencyLockDumpingTables) Setup(tctx *tcontext.Context) error {
	if c.conf.ServerInfo.ServerType == version.ServerTypeTiDB {
		if enableTableLock, err := CheckTiDBEnableTableLock(c.conn); err != nil || !enableTableLock {
			if err != nil {
				return err
			}
			return errTiDBDisableTableLock
		}
	}
	blockList := make(map[string]map[string]interface{})
	return utils.WithRetry(tctx, func() error {
		lockTablesSQL := buildLockTablesSQL(c.conf.Tables, blockList)
		_, err := c.conn.ExecContext(tctx, lockTablesSQL)
		if err == nil {
			if len(blockList) > 0 {
				filterTablesFunc(tctx, c.conf, func(db string, tbl string) bool {
					if blockTable, ok := blockList[db]; ok {
						if _, ok := blockTable[tbl]; ok {
							return false
						}
					}
					return true
				})
			}
		}
		return errors.Trace(err)
	}, newLockTablesBackoffer(tctx, blockList, c.conf))
}

// TearDown implements ConsistencyController.TearDown
func (c *ConsistencyLockDumpingTables) TearDown(ctx context.Context) error {
	if c.conn == nil {
		return nil
	}
	defer func() {
		_ = c.conn.Close()
		c.conn = nil
	}()
	return UnlockTables(ctx, c.conn)
}

// PingContext implements ConsistencyController.PingContext
func (c *ConsistencyLockDumpingTables) PingContext(ctx context.Context) error {
	if c.conn == nil {
		return errors.New("consistency connection has already been closed")
	}
	return c.conn.PingContext(ctx)
}

const snapshotFieldIndex = 1
<commit_msg>dumpling: display the SQL which causes error (#36604)<commit_after>// Copyright 2020 PingCAP, Inc. Licensed under Apache-2.0.

package export

import (
	"context"
	"database/sql"

	"github.com/pingcap/errors"
	"github.com/pingcap/tidb/br/pkg/utils"
	"github.com/pingcap/tidb/br/pkg/version"
	tcontext "github.com/pingcap/tidb/dumpling/context"
)

const (
	// ConsistencyTypeAuto will use flush for MySQL/MariaDB and snapshot for TiDB.
	ConsistencyTypeAuto = "auto"
	// ConsistencyTypeFlush will use FLUSH TABLES WITH READ LOCK to temporarily interrupt the DML and DDL operations of the replica database,
	// to ensure the global consistency of the backup connection.
	ConsistencyTypeFlush = "flush"
	// ConsistencyTypeLock will add read locks on all tables to be exported.
	ConsistencyTypeLock = "lock"
	// ConsistencyTypeSnapshot gets a consistent snapshot of the specified timestamp and exports it.
	ConsistencyTypeSnapshot = "snapshot"
	// ConsistencyTypeNone doesn't guarantee for consistency.
	ConsistencyTypeNone = "none"
)

var errTiDBDisableTableLock = errors.New("try to apply lock consistency on TiDB but it doesn't enable table lock. please set enable-table-lock=true in tidb server config")

// NewConsistencyController returns a new consistency controller
func NewConsistencyController(ctx context.Context, conf *Config, session *sql.DB) (ConsistencyController, error) {
	conn, err := session.Conn(ctx)
	if err != nil {
		return nil, errors.Trace(err)
	}
	switch conf.Consistency {
	case ConsistencyTypeFlush:
		return &ConsistencyFlushTableWithReadLock{
			serverType: conf.ServerInfo.ServerType,
			conn:       conn,
		}, nil
	case ConsistencyTypeLock:
		return &ConsistencyLockDumpingTables{
			conn: conn,
			conf: conf,
		}, nil
	case ConsistencyTypeSnapshot:
		if conf.ServerInfo.ServerType != version.ServerTypeTiDB {
			return nil, errors.New("snapshot consistency is not supported for this server")
		}
		return &ConsistencyNone{}, nil
	case ConsistencyTypeNone:
		return &ConsistencyNone{}, nil
	default:
		return nil, errors.Errorf("invalid consistency option %s", conf.Consistency)
	}
}

// ConsistencyController is the interface that controls the consistency of exporting progress
type ConsistencyController interface {
	Setup(*tcontext.Context) error
	TearDown(context.Context) error
	PingContext(context.Context) error
}

// ConsistencyNone dumps without adding locks, which cannot guarantee consistency
type ConsistencyNone struct{}

// Setup implements ConsistencyController.Setup
func (*ConsistencyNone) Setup(_ *tcontext.Context) error {
	return nil
}

// TearDown implements ConsistencyController.TearDown
func (*ConsistencyNone) TearDown(_ context.Context) error {
	return nil
}

// PingContext implements ConsistencyController.PingContext
func (*ConsistencyNone) PingContext(_ context.Context) error {
	return nil
}

// ConsistencyFlushTableWithReadLock uses FlushTableWithReadLock before the dump
type ConsistencyFlushTableWithReadLock struct {
	serverType version.ServerType
	conn       *sql.Conn
}

// Setup implements ConsistencyController.Setup
func (c *ConsistencyFlushTableWithReadLock) Setup(tctx *tcontext.Context) error {
	if c.serverType == version.ServerTypeTiDB {
		return errors.New("'flush table with read lock' cannot be used to ensure the consistency in TiDB")
	}
	return FlushTableWithReadLock(tctx, c.conn)
}

// TearDown implements ConsistencyController.TearDown
func (c *ConsistencyFlushTableWithReadLock) TearDown(ctx context.Context) error {
	if c.conn == nil {
		return nil
	}
	defer func() {
		_ = c.conn.Close()
		c.conn = nil
	}()
	return UnlockTables(ctx, c.conn)
}

// PingContext implements ConsistencyController.PingContext
func (c *ConsistencyFlushTableWithReadLock) PingContext(ctx context.Context) error {
	if c.conn == nil {
		return errors.New("consistency connection has already been closed")
	}
	return c.conn.PingContext(ctx)
}

// ConsistencyLockDumpingTables execute lock tables read on all tables before dump
type ConsistencyLockDumpingTables struct {
	conn *sql.Conn
	conf *Config
}

// Setup implements ConsistencyController.Setup
func (c *ConsistencyLockDumpingTables) Setup(tctx *tcontext.Context) error {
	if c.conf.ServerInfo.ServerType == version.ServerTypeTiDB {
		if enableTableLock, err := CheckTiDBEnableTableLock(c.conn); err != nil || !enableTableLock {
			if err != nil {
				return err
			}
			return errTiDBDisableTableLock
		}
	}
	blockList := make(map[string]map[string]interface{})
	return utils.WithRetry(tctx, func() error {
		lockTablesSQL := buildLockTablesSQL(c.conf.Tables, blockList)
		_, err := c.conn.ExecContext(tctx, lockTablesSQL)
		if err == nil {
			if len(blockList) > 0 {
				filterTablesFunc(tctx, c.conf, func(db string, tbl string) bool {
					if blockTable, ok := blockList[db]; ok {
						if _, ok := blockTable[tbl]; ok {
							return false
						}
					}
					return true
				})
			}
		}
		return errors.Annotatef(err, "sql: %s", lockTablesSQL)
	}, newLockTablesBackoffer(tctx, blockList, c.conf))
}

// TearDown implements ConsistencyController.TearDown
func (c *ConsistencyLockDumpingTables) TearDown(ctx context.Context) error {
	if c.conn == nil {
		return nil
	}
	defer func() {
		_ = c.conn.Close()
		c.conn = nil
	}()
	return UnlockTables(ctx, c.conn)
}

// PingContext implements ConsistencyController.PingContext
func (c *ConsistencyLockDumpingTables) PingContext(ctx context.Context) error {
	if c.conn == nil {
		return errors.New("consistency connection has already been closed")
	}
	return c.conn.PingContext(ctx)
}

const snapshotFieldIndex = 1
<|endoftext|>


<commit_before>/*
Copyright 2019 The Kubernetes Authors.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package controller

import (
	"context"
	"fmt"
	"os"
	"sync"
	"time"

	apiv1 "k8s.io/api/core/v1"
	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
	"k8s.io/apimachinery/pkg/util/sets"
	"k8s.io/apimachinery/pkg/util/uuid"
	"k8s.io/client-go/kubernetes"
	"k8s.io/client-go/kubernetes/scheme"
	typev1 "k8s.io/client-go/kubernetes/typed/core/v1"
	restclient "k8s.io/client-go/rest"
	"k8s.io/client-go/tools/clientcmd"
	"k8s.io/client-go/tools/leaderelection/resourcelock"
	"k8s.io/client-go/tools/record"
	log "k8s.io/klog"

	"k8s.io/cloud-provider-openstack/pkg/autohealing/cloudprovider"
	_ "k8s.io/cloud-provider-openstack/pkg/autohealing/cloudprovider/register"
	"k8s.io/cloud-provider-openstack/pkg/autohealing/config"
	"k8s.io/cloud-provider-openstack/pkg/autohealing/healthcheck"
)

// EventType type of event associated with an informer
type EventType string

const (
	// High enough QPS to fit all expected use cases. QPS=0 is not set here, because
	// client code is overriding it.
	defaultQPS = 1e6
	// High enough Burst to fit all expected use cases. Burst=0 is not set here, because
	// client code is overriding it.
	defaultBurst = 1e6

	// CreateEvent event associated with new objects in an informer
	CreateEvent EventType = "CREATE"
	// UpdateEvent event associated with an object update in an informer
	UpdateEvent EventType = "UPDATE"
	// DeleteEvent event associated when an object is removed from an informer
	DeleteEvent EventType = "DELETE"

	// LabelNodeRoleMaster specifies that a node is a master
	// Related discussion: https://github.com/kubernetes/kubernetes/pull/39112
	LabelNodeRoleMaster = "node-role.kubernetes.io/master"
)

// Event holds the context of an event
type Event struct {
	Type EventType
	Obj  interface{}
}

func createKubeClients(apiserverHost string, kubeConfig string) (*kubernetes.Clientset, *kubernetes.Clientset, error) {
	cfg, err := clientcmd.BuildConfigFromFlags(apiserverHost, kubeConfig)
	if err != nil {
		return nil, nil, err
	}

	cfg.QPS = defaultQPS
	cfg.Burst = defaultBurst
	cfg.ContentType = "application/vnd.kubernetes.protobuf"
	cfg.Timeout = 5 * time.Second

	log.V(4).Info("Creating kubernetes API clients")

	client, err := kubernetes.NewForConfig(cfg)
	if err != nil {
		return nil, nil, err
	}
	leaderElectionClient, err := kubernetes.NewForConfig(restclient.AddUserAgent(cfg, "leader-election"))
	if err != nil {
		return nil, nil, err
	}

	v, err := client.Discovery().ServerVersion()
	if err != nil {
		return nil, nil, err
	}
	log.V(4).Infof("Kubernetes API client created, server version: %s", fmt.Sprintf("v%v.%v", v.Major, v.Minor))

	return client, leaderElectionClient, nil
}

// NewController creates a new autohealer controller.
func NewController(conf config.Config) *Controller {
	// initialize cloud provider
	provider, err := cloudprovider.GetCloudProvider(conf.CloudProvider, conf)
	if err != nil {
		log.Fatalf("Failed to get the cloud provider %s: %v", conf.CloudProvider, err)
	}

	log.Infof("Using cloud provider: %s", provider.GetName())

	// initialize k8s clients
	kubeClient, leaderElectionClient, err := createKubeClients(conf.Kubernetes.ApiserverHost, conf.Kubernetes.KubeConfig)
	if err != nil {
		log.Fatalf("failed to initialize kubernetes client, error: %v", err)
	}

	// event
	eventBroadcaster := record.NewBroadcaster()
	eventBroadcaster.StartLogging(log.V(4).Infof)
	eventBroadcaster.StartRecordingToSink(&typev1.EventSinkImpl{
		Interface: kubeClient.CoreV1().Events(""),
	})
	recorder := eventBroadcaster.NewRecorder(scheme.Scheme, apiv1.EventSource{Component: "openstack-ingress-controller"})

	// Initialize the configured health checkers
	var workerCheckers []healthcheck.HealthCheck
	var masterCheckers []healthcheck.HealthCheck
	for _, item := range conf.HealthCheck.Worker {
		checker, err := healthcheck.GetHealthChecker(item.Type, item.Params)
		if err != nil {
			log.Fatalf("failed to get %s type health check for worker node, error: %v", item.Type, err)
		}
		if !checker.IsWorkerSupported() {
			log.Warningf("Plugin type %s does not support worker node health check, will skip", item.Type)
			continue
		}
		workerCheckers = append(workerCheckers, checker)
	}
	for _, item := range conf.HealthCheck.Master {
		checker, err := healthcheck.GetHealthChecker(item.Type, item.Params)
		if err != nil {
			log.Fatalf("failed to get %s type health check for master node, error: %v", item.Type, err)
		}
		if !checker.IsMasterSupported() {
			log.Warningf("Plugin type %s does not support master node health check, will skip", item.Type)
			continue
		}
		masterCheckers = append(masterCheckers, checker)
	}

	controller := &Controller{
		config:               conf,
		recorder:             recorder,
		provider:             provider,
		kubeClient:           kubeClient,
		leaderElectionClient: leaderElectionClient,
		masterCheckers:       masterCheckers,
		workerCheckers:       workerCheckers,
	}

	return controller
}

// Controller ...
type Controller struct {
	provider             cloudprovider.CloudProvider
	recorder             record.EventRecorder
	kubeClient           kubernetes.Interface
	leaderElectionClient kubernetes.Interface
	config               config.Config
	workerCheckers       []healthcheck.HealthCheck
	masterCheckers       []healthcheck.HealthCheck
}

// UpdateNodeAnnotation updates the specified node annotation, if value equals empty string, the annotation will be
// removed. This implements the interface healthcheck.NodeController
func (c *Controller) UpdateNodeAnnotation(node healthcheck.NodeInfo, annotation string, value string) error {
	n, err := c.kubeClient.CoreV1().Nodes().Get(node.KubeNode.Name, metav1.GetOptions{})
	if err != nil {
		return err
	}

	if value == "" {
		delete(n.Annotations, annotation)
	} else {
		n.Annotations[annotation] = value
	}

	if _, err := c.kubeClient.CoreV1().Nodes().Update(n); err != nil {
		return err
	}

	return nil
}

func (c *Controller) GetLeaderElectionLock() (resourcelock.Interface, error) {
	// Identity used to distinguish between multiple cloud controller manager instances
	id, err := os.Hostname()
	if err != nil {
		return nil, err
	}
	// add a uniquifier so that two processes on the same host don't accidentally both become active
	id = id + "_" + string(uuid.NewUUID())

	rl, err := resourcelock.New(
		"configmaps",
		"kube-system",
		"magnum-auto-healer",
		c.leaderElectionClient.CoreV1(),
		c.leaderElectionClient.CoordinationV1(),
		resourcelock.ResourceLockConfig{
			Identity:      id,
			EventRecorder: c.recorder,
		})
	if err != nil {
		return nil, err
	}

	return rl, nil
}

// getUnhealthyMasterNodes returns the master nodes that need to be repaired.
func (c *Controller) getUnhealthyMasterNodes() ([]healthcheck.NodeInfo, error) {
	var nodes []healthcheck.NodeInfo

	// If no checkers defined, skip
	if len(c.masterCheckers) == 0 {
		log.Info("No health check defined for master node, skip.")
		return nodes, nil
	}

	// Get all the master nodes need to check
	nodeList, err := c.kubeClient.CoreV1().Nodes().List(metav1.ListOptions{})
	if err != nil {
		return nil, err
	}
	for _, node := range nodeList.Items {
		if _, hasMasterRoleLabel := node.Labels[LabelNodeRoleMaster]; hasMasterRoleLabel {
			if time.Now().Before(node.ObjectMeta.GetCreationTimestamp().Add(c.config.CheckDelayAfterAdd)) {
				log.V(4).Infof("The node %s is created less than the configured check delay, skip", node.Name)
				continue
			}

			nodes = append(nodes, healthcheck.NodeInfo{KubeNode: node})
		}
	}

	// Do health check
	unhealthyNodes := healthcheck.CheckNodes(c.masterCheckers, nodes, c)

	return unhealthyNodes, nil
}

// getUnhealthyWorkerNodes returns the nodes that need to be repaired.
func (c *Controller) getUnhealthyWorkerNodes() ([]healthcheck.NodeInfo, error) {
	var nodes []healthcheck.NodeInfo

	// If no checkers defined, skip.
	if len(c.workerCheckers) == 0 {
		log.Info("No health check defined for worker node, skip.")
		return nodes, nil
	}

	// Get all the worker nodes.
	nodeList, err := c.kubeClient.CoreV1().Nodes().List(metav1.ListOptions{})
	if err != nil {
		return nil, err
	}
	for _, node := range nodeList.Items {
		if _, hasMasterRoleLabel := node.Labels[LabelNodeRoleMaster]; hasMasterRoleLabel {
			continue
		}
		if len(node.Status.Conditions) == 0 {
			continue
		}
		if time.Now().Before(node.ObjectMeta.GetCreationTimestamp().Add(c.config.CheckDelayAfterAdd)) {
			log.V(4).Infof("The node %s is created less than the configured check delay, skip", node.Name)
			continue
		}
		nodes = append(nodes, healthcheck.NodeInfo{KubeNode: node})
	}

	// Do health check
	unhealthyNodes := healthcheck.CheckNodes(c.workerCheckers, nodes, c)

	return unhealthyNodes, nil
}

func (c *Controller) repairNodes(unhealthyNodes []healthcheck.NodeInfo) {
	unhealthyNodeNames := sets.NewString()
	for _, n := range unhealthyNodes {
		unhealthyNodeNames.Insert(n.KubeNode.Name)
	}

	// Trigger unhealthy nodes repair.
	if len(unhealthyNodes) > 0 {
		if !c.provider.Enabled() {
			// The cloud provider doesn't allow to trigger node repair.
			log.Infof("Auto healing is ignored for nodes %s", unhealthyNodeNames.List())
		} else {
			log.Infof("Starting to repair nodes %s, dryrun: %t", unhealthyNodeNames.List(), c.config.DryRun)

			if !c.config.DryRun {
				// Cordon the nodes before repair.
				for _, node := range unhealthyNodes {
					nodeName := node.KubeNode.Name
					newNode := node.KubeNode.DeepCopy()
					newNode.Spec.Unschedulable = true

					if _, err := c.kubeClient.CoreV1().Nodes().Update(newNode); err != nil {
						log.Errorf("Failed to cordon node %s, error: %v", nodeName, err)
					} else {
						log.Infof("Node %s is cordoned", nodeName)
					}

					// TODO: Deal with the situation that the autohealer is running on a failed node
				}

				// Start to repair all the unhealthy nodes.
				if err := c.provider.Repair(unhealthyNodes); err != nil {
					log.Errorf("Failed to repair the nodes %s, error: %v", unhealthyNodeNames.List(), err)
				}
			}
		}
	}
}

// startMasterMonitor checks if there are failed master nodes and triggers the repair action. This function is supposed
// to be running in a goroutine.
func (c *Controller) startMasterMonitor(wg *sync.WaitGroup) {
	log.V(4).Info("Starting to check master nodes.")
	defer wg.Done()

	// Get all the unhealthy master nodes.
	unhealthyNodes, err := c.getUnhealthyMasterNodes()
	if err != nil {
		log.Errorf("Failed to get unhealthy master nodes, error: %v", err)
		return
	}

	c.repairNodes(unhealthyNodes)

	if len(unhealthyNodes) == 0 {
		log.Info("Master nodes are healthy")
	}
	log.V(4).Info("Finished checking master nodes.")
}

// startWorkerMonitor checks if there are failed worker nodes and triggers the repair action. This function is supposed
// to be running in a goroutine.
func (c *Controller) startWorkerMonitor(wg *sync.WaitGroup) {
	log.V(4).Info("Starting to check worker nodes.")
	defer wg.Done()

	// Get all the unhealthy worker nodes.
	unhealthyNodes, err := c.getUnhealthyWorkerNodes()
	if err != nil {
		log.Errorf("Failed to get unhealthy worker nodes, error: %v", err)
		return
	}

	c.repairNodes(unhealthyNodes)

	if len(unhealthyNodes) == 0 {
		log.Info("Worker nodes are healthy")
	}

	log.V(4).Info("Finished checking worker nodes.")
}

// Start starts the autohealing controller.
func (c *Controller) Start(ctx context.Context) {
	log.Info("Starting autohealing controller")

	ticker := time.NewTicker(c.config.MonitorInterval)
	defer ticker.Stop()

	var wg sync.WaitGroup
	for {
		select {
		case <-ticker.C:
			if c.config.MasterMonitorEnabled {
				wg.Add(1)
				go c.startMasterMonitor(&wg)
			}
			if c.config.WorkerMonitorEnabled {
				wg.Add(1)
				go c.startWorkerMonitor(&wg)
			}

			wg.Wait()
		}
	}
}
<commit_msg>Improve magnum-auto-healer log (#694)<commit_after>/*
Copyright 2019 The Kubernetes Authors.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package controller

import (
	"context"
	"fmt"
	"os"
	"sync"
	"time"

	apiv1 "k8s.io/api/core/v1"
	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
	"k8s.io/apimachinery/pkg/util/sets"
	"k8s.io/apimachinery/pkg/util/uuid"
	"k8s.io/client-go/kubernetes"
	"k8s.io/client-go/kubernetes/scheme"
	typev1 "k8s.io/client-go/kubernetes/typed/core/v1"
	restclient "k8s.io/client-go/rest"
	"k8s.io/client-go/tools/clientcmd"
	"k8s.io/client-go/tools/leaderelection/resourcelock"
	"k8s.io/client-go/tools/record"
	log "k8s.io/klog"

	"k8s.io/cloud-provider-openstack/pkg/autohealing/cloudprovider"
	_ "k8s.io/cloud-provider-openstack/pkg/autohealing/cloudprovider/register"
	"k8s.io/cloud-provider-openstack/pkg/autohealing/config"
	"k8s.io/cloud-provider-openstack/pkg/autohealing/healthcheck"
)

// EventType type of event associated with an informer
type EventType string

const (
	// High enough QPS to fit all expected use cases. QPS=0 is not set here, because
	// client code is overriding it.
	defaultQPS = 1e6
	// High enough Burst to fit all expected use cases. Burst=0 is not set here, because
	// client code is overriding it.
	defaultBurst = 1e6

	// CreateEvent event associated with new objects in an informer
	CreateEvent EventType = "CREATE"
	// UpdateEvent event associated with an object update in an informer
	UpdateEvent EventType = "UPDATE"
	// DeleteEvent event associated when an object is removed from an informer
	DeleteEvent EventType = "DELETE"

	// LabelNodeRoleMaster specifies that a node is a master
	// Related discussion: https://github.com/kubernetes/kubernetes/pull/39112
	LabelNodeRoleMaster = "node-role.kubernetes.io/master"
)

// Event holds the context of an event
type Event struct {
	Type EventType
	Obj  interface{}
}

func createKubeClients(apiserverHost string, kubeConfig string) (*kubernetes.Clientset, *kubernetes.Clientset, error) {
	cfg, err := clientcmd.BuildConfigFromFlags(apiserverHost, kubeConfig)
	if err != nil {
		return nil, nil, err
	}

	cfg.QPS = defaultQPS
	cfg.Burst = defaultBurst
	cfg.ContentType = "application/vnd.kubernetes.protobuf"
	cfg.Timeout = 5 * time.Second

	log.V(4).Info("Creating kubernetes API clients")

	client, err := kubernetes.NewForConfig(cfg)
	if err != nil {
		return nil, nil, err
	}
	leaderElectionClient, err := kubernetes.NewForConfig(restclient.AddUserAgent(cfg, "leader-election"))
	if err != nil {
		return nil, nil, err
	}

	v, err := client.Discovery().ServerVersion()
	if err != nil {
		return nil, nil, err
	}
	log.V(4).Infof("Kubernetes API client created, server version: %s", fmt.Sprintf("v%v.%v", v.Major, v.Minor))

	return client, leaderElectionClient, nil
}

// NewController creates a new autohealer controller.
func NewController(conf config.Config) *Controller {
	// initialize cloud provider
	provider, err := cloudprovider.GetCloudProvider(conf.CloudProvider, conf)
	if err != nil {
		log.Fatalf("Failed to get the cloud provider %s: %v", conf.CloudProvider, err)
	}

	log.Infof("Using cloud provider: %s", provider.GetName())

	// initialize k8s clients
	kubeClient, leaderElectionClient, err := createKubeClients(conf.Kubernetes.ApiserverHost, conf.Kubernetes.KubeConfig)
	if err != nil {
		log.Fatalf("failed to initialize kubernetes client, error: %v", err)
	}

	// event
	eventBroadcaster := record.NewBroadcaster()
	eventBroadcaster.StartLogging(log.V(4).Infof)
	eventBroadcaster.StartRecordingToSink(&typev1.EventSinkImpl{
		Interface: kubeClient.CoreV1().Events(""),
	})
	recorder := eventBroadcaster.NewRecorder(scheme.Scheme, apiv1.EventSource{Component: "openstack-ingress-controller"})

	// Initialize the configured health checkers
	var workerCheckers []healthcheck.HealthCheck
	var masterCheckers []healthcheck.HealthCheck
	for _, item := range conf.HealthCheck.Worker {
		checker, err := healthcheck.GetHealthChecker(item.Type, item.Params)
		if err != nil {
			log.Fatalf("failed to get %s type health check for worker node, error: %v", item.Type, err)
		}
		if !checker.IsWorkerSupported() {
			log.Warningf("Plugin type %s does not support worker node health check, will skip", item.Type)
			continue
		}
		workerCheckers = append(workerCheckers, checker)
	}
	for _, item := range conf.HealthCheck.Master {
		checker, err := healthcheck.GetHealthChecker(item.Type, item.Params)
		if err != nil {
			log.Fatalf("failed to get %s type health check for master node, error: %v", item.Type, err)
		}
		if !checker.IsMasterSupported() {
			log.Warningf("Plugin type %s does not support master node health check, will skip", item.Type)
			continue
		}
		masterCheckers = append(masterCheckers, checker)
	}

	controller := &Controller{
		config:               conf,
		recorder:             recorder,
		provider:             provider,
		kubeClient:           kubeClient,
		leaderElectionClient: leaderElectionClient,
		masterCheckers:       masterCheckers,
		workerCheckers:       workerCheckers,
	}

	return controller
}

// Controller ...
type Controller struct {
	provider             cloudprovider.CloudProvider
	recorder             record.EventRecorder
	kubeClient           kubernetes.Interface
	leaderElectionClient kubernetes.Interface
	config               config.Config
	workerCheckers       []healthcheck.HealthCheck
	masterCheckers       []healthcheck.HealthCheck
}

// UpdateNodeAnnotation updates the specified node annotation, if value equals empty string, the annotation will be
// removed. This implements the interface healthcheck.NodeController
func (c *Controller) UpdateNodeAnnotation(node healthcheck.NodeInfo, annotation string, value string) error {
	n, err := c.kubeClient.CoreV1().Nodes().Get(node.KubeNode.Name, metav1.GetOptions{})
	if err != nil {
		return err
	}

	if value == "" {
		delete(n.Annotations, annotation)
	} else {
		n.Annotations[annotation] = value
	}

	if _, err := c.kubeClient.CoreV1().Nodes().Update(n); err != nil {
		return err
	}

	return nil
}

func (c *Controller) GetLeaderElectionLock() (resourcelock.Interface, error) {
	// Identity used to distinguish between multiple cloud controller manager instances
	id, err := os.Hostname()
	if err != nil {
		return nil, err
	}
	// add a uniquifier so that two processes on the same host don't accidentally both become active
	id = id + "_" + string(uuid.NewUUID())

	rl, err := resourcelock.New(
		"configmaps",
		"kube-system",
		"magnum-auto-healer",
		c.leaderElectionClient.CoreV1(),
		c.leaderElectionClient.CoordinationV1(),
		resourcelock.ResourceLockConfig{
			Identity:      id,
			EventRecorder: c.recorder,
		})
	if err != nil {
		return nil, err
	}

	return rl, nil
}

// getUnhealthyMasterNodes returns the master nodes that need to be repaired.
func (c *Controller) getUnhealthyMasterNodes() ([]healthcheck.NodeInfo, error) {
	var nodes []healthcheck.NodeInfo

	// If no checkers defined, skip
	if len(c.masterCheckers) == 0 {
		log.V(3).Info("No health check defined for master node, skip.")
		return nodes, nil
	}

	// Get all the master nodes need to check
	nodeList, err := c.kubeClient.CoreV1().Nodes().List(metav1.ListOptions{})
	if err != nil {
		return nil, err
	}
	for _, node := range nodeList.Items {
		if _, hasMasterRoleLabel := node.Labels[LabelNodeRoleMaster]; hasMasterRoleLabel {
			if time.Now().Before(node.ObjectMeta.GetCreationTimestamp().Add(c.config.CheckDelayAfterAdd)) {
				log.V(4).Infof("The node %s is created less than the configured check delay, skip", node.Name)
				continue
			}

			nodes = append(nodes, healthcheck.NodeInfo{KubeNode: node})
		}
	}

	// Do health check
	unhealthyNodes := healthcheck.CheckNodes(c.masterCheckers, nodes, c)

	return unhealthyNodes, nil
}

// getUnhealthyWorkerNodes returns the nodes that need to be repaired.
func (c *Controller) getUnhealthyWorkerNodes() ([]healthcheck.NodeInfo, error) {
	var nodes []healthcheck.NodeInfo

	// If no checkers defined, skip.
	if len(c.workerCheckers) == 0 {
		log.V(3).Info("No health check defined for worker node, skip.")
		return nodes, nil
	}

	// Get all the worker nodes.
	nodeList, err := c.kubeClient.CoreV1().Nodes().List(metav1.ListOptions{})
	if err != nil {
		return nil, err
	}
	for _, node := range nodeList.Items {
		if _, hasMasterRoleLabel := node.Labels[LabelNodeRoleMaster]; hasMasterRoleLabel {
			continue
		}
		if len(node.Status.Conditions) == 0 {
			continue
		}
		if time.Now().Before(node.ObjectMeta.GetCreationTimestamp().Add(c.config.CheckDelayAfterAdd)) {
			log.V(4).Infof("The node %s is created less than the configured check delay, skip", node.Name)
			continue
		}
		nodes = append(nodes, healthcheck.NodeInfo{KubeNode: node})
	}

	// Do health check
	unhealthyNodes := healthcheck.CheckNodes(c.workerCheckers, nodes, c)

	return unhealthyNodes, nil
}

func (c *Controller) repairNodes(unhealthyNodes []healthcheck.NodeInfo) {
	unhealthyNodeNames := sets.NewString()
	for _, n := range unhealthyNodes {
		unhealthyNodeNames.Insert(n.KubeNode.Name)
	}

	// Trigger unhealthy nodes repair.
	if len(unhealthyNodes) > 0 {
		if !c.provider.Enabled() {
			// The cloud provider doesn't allow to trigger node repair.
			log.Infof("Auto healing is ignored for nodes %s", unhealthyNodeNames.List())
		} else {
			log.Infof("Starting to repair nodes %s, dryrun: %t", unhealthyNodeNames.List(), c.config.DryRun)

			if !c.config.DryRun {
				// Cordon the nodes before repair.
				for _, node := range unhealthyNodes {
					nodeName := node.KubeNode.Name
					newNode := node.KubeNode.DeepCopy()
					newNode.Spec.Unschedulable = true

					if _, err := c.kubeClient.CoreV1().Nodes().Update(newNode); err != nil {
						log.Errorf("Failed to cordon node %s, error: %v", nodeName, err)
					} else {
						log.Infof("Node %s is cordoned", nodeName)
					}

					// TODO: Deal with the situation that the autohealer is running on a failed node
				}

				// Start to repair all the unhealthy nodes.
				if err := c.provider.Repair(unhealthyNodes); err != nil {
					log.Errorf("Failed to repair the nodes %s, error: %v", unhealthyNodeNames.List(), err)
				}
			}
		}
	}
}

// startMasterMonitor checks if there are failed master nodes and triggers the repair action. This function is supposed
// to be running in a goroutine.
func (c *Controller) startMasterMonitor(wg *sync.WaitGroup) {
	log.V(3).Info("Starting to check master nodes.")
	defer wg.Done()

	// Get all the unhealthy master nodes.
	unhealthyNodes, err := c.getUnhealthyMasterNodes()
	if err != nil {
		log.Errorf("Failed to get unhealthy master nodes, error: %v", err)
		return
	}

	c.repairNodes(unhealthyNodes)

	if len(unhealthyNodes) == 0 {
		log.V(3).Info("Master nodes are healthy")
	}

	log.V(3).Info("Finished checking master nodes.")
}

// startWorkerMonitor checks if there are failed worker nodes and triggers the repair action. This function is supposed
// to be running in a goroutine.
func (c *Controller) startWorkerMonitor(wg *sync.WaitGroup) {
	log.V(3).Info("Starting to check worker nodes.")
	defer wg.Done()

	// Get all the unhealthy worker nodes.
	unhealthyNodes, err := c.getUnhealthyWorkerNodes()
	if err != nil {
		log.Errorf("Failed to get unhealthy worker nodes, error: %v", err)
		return
	}

	c.repairNodes(unhealthyNodes)

	if len(unhealthyNodes) == 0 {
		log.V(3).Info("Worker nodes are healthy")
	}

	log.V(3).Info("Finished checking worker nodes.")
}

// Start starts the autohealing controller.
func (c *Controller) Start(ctx context.Context) {
	log.Info("Starting autohealing controller")

	ticker := time.NewTicker(c.config.MonitorInterval)
	defer ticker.Stop()

	var wg sync.WaitGroup
	for {
		select {
		case <-ticker.C:
			if c.config.MasterMonitorEnabled {
				wg.Add(1)
				go c.startMasterMonitor(&wg)
			}
			if c.config.WorkerMonitorEnabled {
				wg.Add(1)
				go c.startWorkerMonitor(&wg)
			}

			wg.Wait()
		}
	}
}
<|endoftext|>


<commit_before>// Copyright 2015 Google Inc. All Rights Reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package gcsproxy

import (
	"errors"
	"io/ioutil"
	"strings"
	"testing"

	"github.com/jacobsa/gcloud/gcs"
	"github.com/jacobsa/gcloud/gcs/mock_gcs"
	. "github.com/jacobsa/oglematchers"
	. "github.com/jacobsa/oglemock"
	. "github.com/jacobsa/ogletest"
	"golang.org/x/net/context"
)

func TestAppendObjectCreator(t *testing.T) { RunTests(t) }

////////////////////////////////////////////////////////////////////////
// Boilerplate
////////////////////////////////////////////////////////////////////////

const prefix = ".gcsfuse_tmp/"

type AppendObjectCreatorTest struct {
	ctx     context.Context
	bucket  mock_gcs.MockBucket
	creator objectCreator

	srcObject   gcs.Object
	srcContents string
}

var _ SetUpInterface = &AppendObjectCreatorTest{}

func init() { RegisterTestSuite(&AppendObjectCreatorTest{}) }

func (t *AppendObjectCreatorTest) SetUp(ti *TestInfo) {
	t.ctx = ti.Ctx

	// Create the bucket.
	t.bucket = mock_gcs.NewMockBucket(ti.MockController, "bucket")

	// Create the creator.
	t.creator = newAppendObjectCreator(prefix, t.bucket)
}

func (t *AppendObjectCreatorTest) call() (o *gcs.Object, err error) {
	o, err = t.creator.Create(
		t.ctx,
		&t.srcObject,
		strings.NewReader(t.srcContents))

	return
}

////////////////////////////////////////////////////////////////////////
// Tests
////////////////////////////////////////////////////////////////////////

func (t *AppendObjectCreatorTest) CallsCreateObject() {
	t.srcContents = "taco"

	// CreateObject
	var req *gcs.CreateObjectRequest
	ExpectCall(t.bucket, "CreateObject")(Any(), Any()).
		WillOnce(DoAll(SaveArg(1, &req), Return(nil, errors.New(""))))

	// Call
	t.call()

	AssertNe(nil, req)
	ExpectTrue(strings.HasPrefix(req.Name, prefix), "Name: %s", req.Name)

	b, err := ioutil.ReadAll(req.Contents)
	AssertEq(nil, err)
	ExpectEq(t.srcContents, string(b))
}

func (t *AppendObjectCreatorTest) CreateObjectFails() {
	var err error

	// CreateObject
	ExpectCall(t.bucket, "CreateObject")(Any(), Any()).
		WillOnce(Return(nil, errors.New("taco")))

	// Call
	_, err = t.call()

	ExpectThat(err, Error(HasSubstr("CreateObject")))
	ExpectThat(err, Error(HasSubstr("taco")))
}

func (t *AppendObjectCreatorTest) CreateObjectReturnsPreconditionError() {
	var err error

	// CreateObject
	expected := &gcs.PreconditionError{}
	ExpectCall(t.bucket, "CreateObject")(Any(), Any()).
		WillOnce(Return(nil, expected))

	// Call
	_, err = t.call()

	ExpectEq(expected, err)
}

func (t *AppendObjectCreatorTest) CallsComposeObjects() {
	t.srcObject.Name = "foo"
	t.srcObject.Generation = 17

	// CreateObject
	tmpObject := &gcs.Object{
		Name:       "bar",
		Generation: 19,
	}

	ExpectCall(t.bucket, "CreateObject")(Any(), Any()).
		WillOnce(Return(&tmpObject, nil))

	// ComposeObjects
	var req *gcs.ComposeObjectsRequest
	ExpectCall(t.bucket, "ComposeObjects")(Any(), Any()).
		WillOnce(DoAll(SaveArg(1, &req), Return(nil, errors.New(""))))

	// DeleteObject
	ExpectCall(t.bucket, "DeleteObject")(Any(), tmpObject.Name).
		WillOnce(Return(nil))

	// Call
	t.call()

	AssertNe(nil, req)
	ExpectEq(t.srcObject.Name, req.DstName)
	ExpectThat(
		req.DstGenerationPrecondition,
		Pointee(Equals(t.srcObject.Generation)))

	AssertEq(2, len(req.Sources))
	var src gcs.ComposeSource

	src = req.Sources[0]
	ExpectEq(t.srcObject.Name, src.Name)
	ExpectEq(t.srcObject.Generation, src.Generation)

	src = req.Sources[1]
	ExpectEq(tmpObject.Name, src.Name)
	ExpectEq(tmpObject.Generation, src.Generation)
}

func (t *AppendObjectCreatorTest) ComposeObjectsFails() {
	// CreateObject
	tmpObject := &gcs.Object{
		Name: "bar",
	}

	ExpectCall(t.bucket, "CreateObject")(Any(), Any()).
		WillOnce(Return(&tmpObject, nil))

	// ComposeObjects
	ExpectCall(t.bucket, "ComposeObjects")(Any(), Any()).
		WillOnce(Return(nil, errors.New("taco")))

	// DeleteObject
	ExpectCall(t.bucket, "DeleteObject")(Any(), tmpObject.Name).
		WillOnce(Return(errors.New("")))

	// Call
	_, err := t.call()

	ExpectThat(err, Error(HasSubstr("ComposeObjects")))
	ExpectThat(err, Error(HasSubstr("taco")))
}

func (t *AppendObjectCreatorTest) ComposeObjectsReturnsPreconditionError() {
	// CreateObject
	tmpObject := &gcs.Object{
		Name: "bar",
	}

	ExpectCall(t.bucket, "CreateObject")(Any(), Any()).
		WillOnce(Return(&tmpObject, nil))

	// ComposeObjects
	expected := &gcs.PreconditionError{}
	ExpectCall(t.bucket, "ComposeObjects")(Any(), Any()).
		WillOnce(Return(nil, expected))

	// DeleteObject
	ExpectCall(t.bucket, "DeleteObject")(Any(), tmpObject.Name).
		WillOnce(Return(errors.New("")))

	// Call
	_, err := t.call()

	ExpectEq(expected, err)
}

func (t *AppendObjectCreatorTest) CallsDeleteObject() {
	AssertTrue(false, "TODO")
}

func (t *AppendObjectCreatorTest) DeleteObjectFails() {
	AssertTrue(false, "TODO")
}

func (t *AppendObjectCreatorTest) DeleteObjectSucceeds() {
	AssertTrue(false, "TODO")
}
<commit_msg>AppendObjectCreatorTest.DeleteObjectSucceeds<commit_after>// Copyright 2015 Google Inc. All Rights Reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package gcsproxy

import (
	"errors"
	"io/ioutil"
	"strings"
	"testing"

	"github.com/jacobsa/gcloud/gcs"
	"github.com/jacobsa/gcloud/gcs/mock_gcs"
	. "github.com/jacobsa/oglematchers"
	. "github.com/jacobsa/oglemock"
	. "github.com/jacobsa/ogletest"
	"golang.org/x/net/context"
)

func TestAppendObjectCreator(t *testing.T) { RunTests(t) }

////////////////////////////////////////////////////////////////////////
// Boilerplate
////////////////////////////////////////////////////////////////////////

const prefix = ".gcsfuse_tmp/"

type AppendObjectCreatorTest struct {
	ctx     context.Context
	bucket  mock_gcs.MockBucket
	creator objectCreator

	srcObject   gcs.Object
	srcContents string
}

var _ SetUpInterface = &AppendObjectCreatorTest{}

func init() { RegisterTestSuite(&AppendObjectCreatorTest{}) }

func (t *AppendObjectCreatorTest) SetUp(ti *TestInfo) {
	t.ctx = ti.Ctx

	// Create the bucket.
	t.bucket = mock_gcs.NewMockBucket(ti.MockController, "bucket")

	// Create the creator.
	t.creator = newAppendObjectCreator(prefix, t.bucket)
}

func (t *AppendObjectCreatorTest) call() (o *gcs.Object, err error) {
	o, err = t.creator.Create(
		t.ctx,
		&t.srcObject,
		strings.NewReader(t.srcContents))

	return
}

////////////////////////////////////////////////////////////////////////
// Tests
////////////////////////////////////////////////////////////////////////

func (t *AppendObjectCreatorTest) CallsCreateObject() {
	t.srcContents = "taco"

	// CreateObject
	var req *gcs.CreateObjectRequest
	ExpectCall(t.bucket, "CreateObject")(Any(), Any()).
		WillOnce(DoAll(SaveArg(1, &req), Return(nil, errors.New(""))))

	// Call
	t.call()

	AssertNe(nil, req)
	ExpectTrue(strings.HasPrefix(req.Name, prefix), "Name: %s", req.Name)

	b, err := ioutil.ReadAll(req.Contents)
	AssertEq(nil, err)
	ExpectEq(t.srcContents, string(b))
}

func (t *AppendObjectCreatorTest) CreateObjectFails() {
	var err error

	// CreateObject
	ExpectCall(t.bucket, "CreateObject")(Any(), Any()).
		WillOnce(Return(nil, errors.New("taco")))

	// Call
	_, err = t.call()

	ExpectThat(err, Error(HasSubstr("CreateObject")))
	ExpectThat(err, Error(HasSubstr("taco")))
}

func (t *AppendObjectCreatorTest) CreateObjectReturnsPreconditionError() {
	var err error

	// CreateObject
	expected := &gcs.PreconditionError{}
	ExpectCall(t.bucket, "CreateObject")(Any(), Any()).
		WillOnce(Return(nil, expected))

	// Call
	_, err = t.call()

	ExpectEq(expected, err)
}

func (t *AppendObjectCreatorTest) CallsComposeObjects() {
	t.srcObject.Name = "foo"
	t.srcObject.Generation = 17

	// CreateObject
	tmpObject := &gcs.Object{
		Name:       "bar",
		Generation: 19,
	}

	ExpectCall(t.bucket, "CreateObject")(Any(), Any()).
		WillOnce(Return(&tmpObject, nil))

	// ComposeObjects
	var req *gcs.ComposeObjectsRequest
	ExpectCall(t.bucket, "ComposeObjects")(Any(), Any()).
		WillOnce(DoAll(SaveArg(1, &req), Return(nil, errors.New(""))))

	// DeleteObject
	ExpectCall(t.bucket, "DeleteObject")(Any(), tmpObject.Name).
		WillOnce(Return(nil))

	// Call
	t.call()

	AssertNe(nil, req)
	ExpectEq(t.srcObject.Name, req.DstName)
	ExpectThat(
		req.DstGenerationPrecondition,
		Pointee(Equals(t.srcObject.Generation)))

	AssertEq(2, len(req.Sources))
	var src gcs.ComposeSource

	src = req.Sources[0]
	ExpectEq(t.srcObject.Name, src.Name)
	ExpectEq(t.srcObject.Generation, src.Generation)

	src = req.Sources[1]
	ExpectEq(tmpObject.Name, src.Name)
	ExpectEq(tmpObject.Generation, src.Generation)
}

func (t *AppendObjectCreatorTest) ComposeObjectsFails() {
	// CreateObject
	tmpObject := &gcs.Object{
		Name: "bar",
	}

	ExpectCall(t.bucket, "CreateObject")(Any(), Any()).
		WillOnce(Return(&tmpObject, nil))

	// ComposeObjects
	ExpectCall(t.bucket, "ComposeObjects")(Any(), Any()).
		WillOnce(Return(nil, errors.New("taco")))

	// DeleteObject
	ExpectCall(t.bucket, "DeleteObject")(Any(), tmpObject.Name).
		WillOnce(Return(errors.New("")))

	// Call
	_, err := t.call()

	ExpectThat(err, Error(HasSubstr("ComposeObjects")))
	ExpectThat(err, Error(HasSubstr("taco")))
}

func (t *AppendObjectCreatorTest) ComposeObjectsReturnsPreconditionError() {
	// CreateObject
	tmpObject := &gcs.Object{
		Name: "bar",
	}

	ExpectCall(t.bucket, "CreateObject")(Any(), Any()).
		WillOnce(Return(&tmpObject, nil))

	// ComposeObjects
	expected := &gcs.PreconditionError{}
	ExpectCall(t.bucket, "ComposeObjects")(Any(), Any()).
		WillOnce(Return(nil, expected))

	// DeleteObject
	ExpectCall(t.bucket, "DeleteObject")(Any(), tmpObject.Name).
		WillOnce(Return(errors.New("")))

	// Call
	_, err := t.call()

	ExpectEq(expected, err)
}

func (t *AppendObjectCreatorTest) CallsDeleteObject() {
	// CreateObject
	tmpObject := &gcs.Object{
		Name: "bar",
	}

	ExpectCall(t.bucket, "CreateObject")(Any(), Any()).
		WillOnce(Return(&tmpObject, nil))

	// ComposeObjects
	composed := &gcs.Object{}
	ExpectCall(t.bucket, "ComposeObjects")(Any(), Any()).
		WillOnce(Return(composed, nil))

	// DeleteObject
	ExpectCall(t.bucket, "DeleteObject")(Any(), tmpObject.Name).
		WillOnce(Return(errors.New("")))

	// Call
	t.call()
}

func (t *AppendObjectCreatorTest) DeleteObjectFails() {
	// CreateObject
	tmpObject := &gcs.Object{
		Name: "bar",
	}

	ExpectCall(t.bucket, "CreateObject")(Any(), Any()).
		WillOnce(Return(&tmpObject, nil))

	// ComposeObjects
	composed := &gcs.Object{}
	ExpectCall(t.bucket, "ComposeObjects")(Any(), Any()).
		WillOnce(Return(composed, nil))

	// DeleteObject
	ExpectCall(t.bucket, "DeleteObject")(Any(), Any()).
		WillOnce(Return(errors.New("taco")))

	// Call
	_, err := t.call()

	ExpectThat(err, Error(HasSubstr("DeleteObject")))
	ExpectThat(err, Error(HasSubstr("taco")))
}

func (t *AppendObjectCreatorTest) DeleteObjectSucceeds() {
	// CreateObject
	tmpObject := &gcs.Object{
		Name: "bar",
	}

	ExpectCall(t.bucket, "CreateObject")(Any(), Any()).
		WillOnce(Return(&tmpObject, nil))

	// ComposeObjects
	composed := &gcs.Object{}
	ExpectCall(t.bucket, "ComposeObjects")(Any(), Any()).
		WillOnce(Return(composed, nil))

	// DeleteObject
	ExpectCall(t.bucket, "DeleteObject")(Any(), Any()).
		WillOnce(Return(nil))

	// Call
	o, err := t.call()

	AssertEq(nil, err)
	ExpectEq(composed, o)
}
<|endoftext|>


<commit_before>package main

import (
	"archive/tar"
	"bufio"
	"compress/bzip2"
	"fmt"
	"io"
	"io/ioutil"
	"log"
	"net/http"
	"os"
	"os/exec"
	"path"
	"regexp"
	"runtime"
	"sort"
	"strconv"
	"strings"
	"time"

	"github.com/cheggaaa/pb"
	"github.com/nightlyone/lockfile"
)

type WebotsInstanceManager interface {
	Install(WebotsVersion) error
	Use(WebotsVersion) error
	IsUsed(WebotsVersion) bool
	Installed() []WebotsVersion
	ApplyAllTemplates() error
}

type SymlinkWebotsManager struct {
	basepath    string
	workpath    string
	installpath string
	usedpath    string
	lock        lockfile.Lockfile
	installed   WebotsVersionList
	inUse       *WebotsVersion
	archive     WebotsArchive
	templates   TemplateManager
	gid         int
}

func NewSymlinkManager(a WebotsArchive) (*SymlinkWebotsManager, error) {
	var err error
	res := &SymlinkWebotsManager{
		archive: a,
	}

	res.basepath, res.workpath, res.installpath, err = symlinkManagerPath()
	if err != nil {
		return nil, err
	}

	res.templates, err = NewHasHTemplateManager(path.Join(res.workpath, "templates"))
	if err != nil {
		return nil, err
	}

	res.usedpath = path.Join(res.workpath, "used")
	res.lock, err = lockfile.New(path.Join(res.workpath, "global.lock"))
	if err != nil {
		return nil, err
	}

	err = res.listInstalled()
	if err != nil {
		return nil, err
	}
	err = res.listUsed()
	if err != nil {
		return nil, err
	}

	//checks that we have the right gid
	res.gid, err = getGid("webots-manager")
	if err != nil {
		return nil, err
	}

	found := false
	userGroups, err := os.Getgroups()
	if err != nil {
		return nil, err
	}

	for _, g := range userGroups {
		if g == res.gid {
			found = true
			break
		}
	}

	if found == false {
		return nil, fmt.Errorf("Current use is not in 'webots-manager' group")
	}

	webotsHome := os.Getenv("WEBOTS_HOME")
	if len(webotsHome) == 0 {
		fmt.Printf("WEBOTS_HOME is not set, please consider exporting WEBOTS_HOME=%s", res.installpath)
	} else if webotsHome != res.installpath {
		return nil, fmt.Errorf("Invalid WEBOTS_HOME=%s, please use WEBOTS_HOME=%s", webotsHome, res.installpath)
	}

	return res, nil
}

func symlinkManagerPath() (string, string, string, error) {
	if runtime.GOOS != "linux" {
		return "", "", "", fmt.Errorf("Only linux is supported yet")
	}
	basepath := "/usr/local"
	workpath := path.Join(basepath, "webots-manager")
	installpath := path.Join(basepath, "webots")
	return basepath, workpath, installpath, nil

}

func (i *SymlinkWebotsManager) tryLock() error {
	if err := i.lock.TryLock(); err != nil {
		return fmt.Errorf("Could not lock %s: %s", i.lock, err)
	}
	return nil
}

func (i *SymlinkWebotsManager) unlock() {
	if err := i.lock.Unlock(); err != nil {
		panic(err)
	}
}

func (i *SymlinkWebotsManager) listInstalled() error {
	if err := i.tryLock(); err != nil {
		return err
	}
	defer i.unlock()

	files, err := ioutil.ReadDir(i.workpath)
	if err != nil {
		return err
	}

	i.installed = make(WebotsVersionList, 0, len(files))
	for _, fi := range files {
		if fi.IsDir() == false {
			continue
		}
		v, err := ParseWebotsVersion(fi.Name())
		if err != nil {
			continue
		}
		i.installed = append(i.installed, v)
	}

	sort.Sort(&i.installed)
	return nil
}

func (i *SymlinkWebotsManager) listUsed() error {
	fi, err := os.Stat(i.usedpath)
	if err != nil && os.IsNotExist(err) == false {
		return err
	}
	if os.IsNotExist(err) {
		i.inUse = nil
		return nil
	}

	if (fi.Mode() & os.ModeSymlink) == os.ModeSymlink {
		return fmt.Errorf("Set-up error, %s exist but it is not a symlink", i.usedpath)
	}

	dest, err := os.Readlink(i.usedpath)
	if err != nil {
		return err
	}

	v, err := ParseWebotsVersion(dest)
	if err != nil {
		return fmt.Errorf("Invalid %s target %s", i.installpath, dest)
	}
	found := false
	for _, vv := range i.installed {
		if v == vv {
			found = true
			break
		}
	}
	if found == false {
		return fmt.Errorf("Internal error, %s in use, but not listed as installed %v", v, i.installed)
	}
	i.inUse = &v
	return nil
}

type ProgressReader struct {
	reader   io.Reader
	progress chan int64
}

func (r *ProgressReader) Read(p []byte) (int, error) {
	n, err := r.reader.Read(p)
	if n > 0 {
		r.progress <- int64(n)
	}
	return n, err
}

func (m *SymlinkWebotsManager) extractFile(v WebotsVersion, h *tar.Header, r io.Reader) error {

	dest := strings.TrimPrefix(h.Name, "webots/")
	dest = path.Join(m.workpath, v.String(), dest)
	if dest == path.Join(m.workpath, v.String()) {
		return nil
	}

	switch h.Typeflag {
	case tar.TypeReg, tar.TypeRegA:
		destDir := path.Dir(dest)
		err := os.MkdirAll(destDir, 0775)
		if err != nil {
			return err
		}
		f, err := os.Create(dest)
		if err != nil {
			return err
		}
		_, err = io.Copy(f, r)
		if err != nil {
			return err
		}
	case tar.TypeDir:
		err := os.MkdirAll(dest, 0775)
		if err != nil {
			return err
		}
	case tar.TypeSymlink:
		destDir := path.Dir(dest)
		err := os.MkdirAll(destDir, 0775)
		if err != nil {
			return err
		}
		err = os.Symlink(h.Linkname, dest)
		if err != nil {
			return err
		}
		return nil
	default:
		return fmt.Errorf("Internal error, cannot handle file %s", h.Name)
	}

	err := os.Chtimes(dest, time.Now(), h.FileInfo().ModTime())
	if err != nil {
		return err
	}

	err = os.Chmod(dest, h.FileInfo().Mode())
	if err != nil {
		return err
	}

	return nil
}

func (m *SymlinkWebotsManager) uncompressFromHttp(v WebotsVersion, addr string) error {
	dest := path.Join(m.workpath, v.String())
	err := os.RemoveAll(dest)
	if err != nil {
		return err
	}
	err = os.MkdirAll(dest, 0775|os.ModeSetgid)
	if err != nil {
		return err
	}

	resp, err := http.Get(addr)
	if err != nil {
		return err
	}

	var netReader io.Reader = resp.Body
	if resp.ContentLength >= 0 {
		//adds a progress bar
		progress := make(chan int64)
		go func() {
			bar := pb.New64(resp.ContentLength)
			bar.Format("[=>_]")
			bar.Start()
			for n := range progress {
				bar.Add64(n)
			}
			bar.FinishPrint(fmt.Sprintf("Downloaded and extracted %s", addr))
		}()
		netReader = &ProgressReader{
			reader:   resp.Body,
			progress: progress,
		}
		defer close(progress)
	}

	tarReader := tar.NewReader(bzip2.NewReader(netReader))
	for {
		fileHeader, err := tarReader.Next()
		if err == io.EOF {
			break
		}
		if err != nil {
			return err
		}
		err = m.extractFile(v, fileHeader, tarReader)
		if err != nil {
			return fmt.Errorf("Cannot extract %s: %s", fileHeader.Name, err)
		}
	}
	return nil
}

func (m *SymlinkWebotsManager) install(v WebotsVersion) error {
	address, err := m.archive.GetUrl(v)
	if err != nil {
		return err
	}
	log.Printf("Downloading from %s", address)
	err = m.uncompressFromHttp(v, address)
	if err != nil {
		return err
	}

	found := false
	for _, vv := range m.installed {
		if vv == v {
			found = true
			break
		}
	}

	log.Printf("Installing templates for %s", v)
	err = m.templates.ApplyTemplates(path.Join(m.workpath, v.String()), v)
	if err != nil {
		return err
	}
	if found == false {
		log.Printf("Successfuly installed %s", v)
		m.installed = append(m.installed, v)
		sort.Sort(&m.installed)
	} else {
		log.Printf("Successfuly re-installed %s", v)
	}

	return nil
}

func (m *SymlinkWebotsManager) Install(v WebotsVersion) error {
	if err := m.tryLock(); err != nil {
		return err
	}
	defer m.unlock()
	return m.install(v)
}

func (m *SymlinkWebotsManager) Installed() []WebotsVersion {
	return []WebotsVersion(m.installed)
}

func (m *SymlinkWebotsManager) IsUsed(v WebotsVersion) bool {
	if m.inUse == nil {
		return false
	}
	return m.inUse.Major == v.Major && m.inUse.Minor == v.Minor && m.inUse.Patch == v.Patch
}

func getGid(g string) (int, error) {
	f, err := os.Open("/etc/group")
	if err != nil {
		return -1, err
	}
	r := bufio.NewReader(f)
	stopped := false
	groupRx := regexp.MustCompile(`^([a-z\-_A-Z0-9]+):[^:]*:([0-9]+):`)
	for stopped == false {
		l, err := r.ReadString('\n')
		if err == io.EOF {
			stopped = true
		} else if err != nil {
			return -1, err
		}
		m := groupRx.FindStringSubmatch(l)
		if m == nil {
			return -1, fmt.Errorf("Invalid line %s", l)
		}
		if m[1] == g {
			gid, err := strconv.ParseInt(m[2], 10, 0)
			if err != nil {
				return -1, err
			}
			return int(gid), nil
		}
	}
	return -1, nil
}

func SymlinkManagerSystemInit() error {
	if os.Getuid() != 0 || os.Geteuid() != 0 {
		return fmt.Errorf("need to be root")
	}

	gid, err := getGid("webots-manager")
	if err != nil {
		return fmt.Errorf("Could not check exitance of group webots-manager: %s", err)
	}
	if gid == -1 {
		cmd := exec.Command("addgroup", "webots-manager")
		err := cmd.Run()
		if err != nil {
			return err
		}
		fmt.Println("Added system group 'webots-manager'")
		gid, err = getGid("webots-manager")
		if err != nil {
			return err
		}
		if gid == -1 {
			return fmt.Errorf("Internal error, gid should not be -1")
		}
	}

	_, workpath, installpath, err := symlinkManagerPath()
	if err != nil {
		return err
	}

	err = os.MkdirAll(workpath, 0775|os.ModeSetgid)
	if err != nil {
		return err
	}

	err = os.Chown(workpath, 0, gid)
	if err != nil {
		return err
	}

	err = os.Chmod(workpath, 0775|os.ModeSetgid)
	if err != nil {
		return err
	}

	fi, err := os.Lstat(installpath)
	if err != nil && os.IsNotExist(err) == false {
		return err
	}
	usedpath := path.Join(workpath, "used")
	if os.IsNotExist(err) {
		err := os.Symlink(usedpath, installpath)
		if err != nil {
			return err
		}
	} else {
		if (fi.Mode() & os.ModeSymlink) != os.ModeSymlink {
			return fmt.Errorf("Webots seems already installed in %s, you should remove it.", installpath)
		}

		dest, err := os.Readlink(installpath)
		if err != nil {
			return err
		}
		if dest != path.Join(workpath, "used") {
			return fmt.Errorf("%s is a symling to incorrect target %s (require %s)", installpath, dest, usedpath)
		}
	}

	return nil
}

func (m *SymlinkWebotsManager) Use(v WebotsVersion) error {
	if err := m.tryLock(); err != nil {
		return err
	}
	defer m.unlock()

	found := false
	for _, vv := range m.installed {
		if vv == v {
			found = true
			break
		}
	}
	if found == false {
		log.Printf("Installing missing version %s", v)
		err := m.install(v)
		if err != nil {
			return err
		}
	}

	err := os.RemoveAll(m.usedpath)
	if err != nil {
		return err
	}
	err = os.Symlink(v.String(), m.usedpath)
	if err != nil {
		return err
	}

	m.inUse = &v

	return nil
}

func (m *SymlinkWebotsManager) ApplyAllTemplates() error {
	for _, v := range m.installed {
		err := m.templates.ApplyTemplates(path.Join(m.workpath, v.String()), v)
		if err != nil {
			return err
		}
	}
	return nil
}
<commit_msg>Accepts empty lines /etc/group<commit_after>package main

import (
	"archive/tar"
	"bufio"
	"compress/bzip2"
	"fmt"
	"io"
	"io/ioutil"
	"log"
	"net/http"
	"os"
	"os/exec"
	"path"
	"regexp"
	"runtime"
	"sort"
	"strconv"
	"strings"
	"time"

	"github.com/cheggaaa/pb"
	"github.com/nightlyone/lockfile"
)

type WebotsInstanceManager interface {
	Install(WebotsVersion) error
	Use(WebotsVersion) error
	IsUsed(WebotsVersion) bool
	Installed() []WebotsVersion
	ApplyAllTemplates() error
}

type SymlinkWebotsManager struct {
	basepath    string
	workpath    string
	installpath string
	usedpath    string
	lock        lockfile.Lockfile
	installed   WebotsVersionList
	inUse       *WebotsVersion
	archive     WebotsArchive
	templates   TemplateManager
	gid         int
}

func NewSymlinkManager(a WebotsArchive) (*SymlinkWebotsManager, error) {
	var err error
	res := &SymlinkWebotsManager{
		archive: a,
	}

	res.basepath, res.workpath, res.installpath, err = symlinkManagerPath()
	if err != nil {
		return nil, err
	}

	res.templates, err = NewHasHTemplateManager(path.Join(res.workpath, "templates"))
	if err != nil {
		return nil, err
	}

	res.usedpath = path.Join(res.workpath, "used")
	res.lock, err = lockfile.New(path.Join(res.workpath, "global.lock"))
	if err != nil {
		return nil, err
	}

	err = res.listInstalled()
	if err != nil {
		return nil, err
	}
	err = res.listUsed()
	if err != nil {
		return nil, err
	}

	//checks that we have the right gid
	res.gid, err = getGid("webots-manager")
	if err != nil {
		return nil, err
	}

	found := false
	userGroups, err := os.Getgroups()
	if err != nil {
		return nil, err
	}

	for _, g := range userGroups {
		if g == res.gid {
			found = true
			break
		}
	}

	if found == false {
		return nil, fmt.Errorf("Current use is not in 'webots-manager' group")
	}

	webotsHome := os.Getenv("WEBOTS_HOME")
	if len(webotsHome) == 0 {
		fmt.Printf("WEBOTS_HOME is not set, please consider exporting WEBOTS_HOME=%s", res.installpath)
	} else if webotsHome != res.installpath {
		return nil, fmt.Errorf("Invalid WEBOTS_HOME=%s, please use WEBOTS_HOME=%s", webotsHome, res.installpath)
	}

	return res, nil
}

func symlinkManagerPath() (string, string, string, error) {
	if runtime.GOOS != "linux" {
		return "", "", "", fmt.Errorf("Only linux is supported yet")
	}
	basepath := "/usr/local"
	workpath := path.Join(basepath, "webots-manager")
	installpath := path.Join(basepath, "webots")
	return basepath, workpath, installpath, nil

}

func (i *SymlinkWebotsManager) tryLock() error {
	if err := i.lock.TryLock(); err != nil {
		return fmt.Errorf("Could not lock %s: %s", i.lock, err)
	}
	return nil
}

func (i *SymlinkWebotsManager) unlock() {
	if err := i.lock.Unlock(); err != nil {
		panic(err)
	}
}

func (i *SymlinkWebotsManager) listInstalled() error {
	if err := i.tryLock(); err != nil {
		return err
	}
	defer i.unlock()

	files, err := ioutil.ReadDir(i.workpath)
	if err != nil {
		return err
	}

	i.installed = make(WebotsVersionList, 0, len(files))
	for _, fi := range files {
		if fi.IsDir() == false {
			continue
		}
		v, err := ParseWebotsVersion(fi.Name())
		if err != nil {
			continue
		}
		i.installed = append(i.installed, v)
	}

	sort.Sort(&i.installed)
	return nil
}

func (i *SymlinkWebotsManager) listUsed() error {
	fi, err := os.Stat(i.usedpath)
	if err != nil && os.IsNotExist(err) == false {
		return err
	}
	if os.IsNotExist(err) {
		i.inUse = nil
		return nil
	}

	if (fi.Mode() & os.ModeSymlink) == os.ModeSymlink {
		return fmt.Errorf("Set-up error, %s exist but it is not a symlink", i.usedpath)
	}

	dest, err := os.Readlink(i.usedpath)
	if err != nil {
		return err
	}

	v, err := ParseWebotsVersion(dest)
	if err != nil {
		return fmt.Errorf("Invalid %s target %s", i.installpath, dest)
	}
	found := false
	for _, vv := range i.installed {
		if v == vv {
			found = true
			break
		}
	}
	if found == false {
		return fmt.Errorf("Internal error, %s in use, but not listed as installed %v", v, i.installed)
	}
	i.inUse = &v
	return nil
}

type ProgressReader struct {
	reader   io.Reader
	progress chan int64
}

func (r *ProgressReader) Read(p []byte) (int, error) {
	n, err := r.reader.Read(p)
	if n > 0 {
		r.progress <- int64(n)
	}
	return n, err
}

func (m *SymlinkWebotsManager) extractFile(v WebotsVersion, h *tar.Header, r io.Reader) error {

	dest := strings.TrimPrefix(h.Name, "webots/")
	dest = path.Join(m.workpath, v.String(), dest)
	if dest == path.Join(m.workpath, v.String()) {
		return nil
	}

	switch h.Typeflag {
	case tar.TypeReg, tar.TypeRegA:
		destDir := path.Dir(dest)
		err := os.MkdirAll(destDir, 0775)
		if err != nil {
			return err
		}
		f, err := os.Create(dest)
		if err != nil {
			return err
		}
		_, err = io.Copy(f, r)
		if err != nil {
			return err
		}
	case tar.TypeDir:
		err := os.MkdirAll(dest, 0775)
		if err != nil {
			return err
		}
	case tar.TypeSymlink:
		destDir := path.Dir(dest)
		err := os.MkdirAll(destDir, 0775)
		if err != nil {
			return err
		}
		err = os.Symlink(h.Linkname, dest)
		if err != nil {
			return err
		}
		return nil
	default:
		return fmt.Errorf("Internal error, cannot handle file %s", h.Name)
	}

	err := os.Chtimes(dest, time.Now(), h.FileInfo().ModTime())
	if err != nil {
		return err
	}

	err = os.Chmod(dest, h.FileInfo().Mode())
	if err != nil {
		return err
	}

	return nil
}

func (m *SymlinkWebotsManager) uncompressFromHttp(v WebotsVersion, addr string) error {
	dest := path.Join(m.workpath, v.String())
	err := os.RemoveAll(dest)
	if err != nil {
		return err
	}
	err = os.MkdirAll(dest, 0775|os.ModeSetgid)
	if err != nil {
		return err
	}

	resp, err := http.Get(addr)
	if err != nil {
		return err
	}

	var netReader io.Reader = resp.Body
	if resp.ContentLength >= 0 {
		//adds a progress bar
		progress := make(chan int64)
		go func() {
			bar := pb.New64(resp.ContentLength)
			bar.Format("[=>_]")
			bar.Start()
			for n := range progress {
				bar.Add64(n)
			}
			bar.FinishPrint(fmt.Sprintf("Downloaded and extracted %s", addr))
		}()
		netReader = &ProgressReader{
			reader:   resp.Body,
			progress: progress,
		}
		defer close(progress)
	}

	tarReader := tar.NewReader(bzip2.NewReader(netReader))
	for {
		fileHeader, err := tarReader.Next()
		if err == io.EOF {
			break
		}
		if err != nil {
			return err
		}
		err = m.extractFile(v, fileHeader, tarReader)
		if err != nil {
			return fmt.Errorf("Cannot extract %s: %s", fileHeader.Name, err)
		}
	}
	return nil
}

func (m *SymlinkWebotsManager) install(v WebotsVersion) error {
	address, err := m.archive.GetUrl(v)
	if err != nil {
		return err
	}
	log.Printf("Downloading from %s", address)
	err = m.uncompressFromHttp(v, address)
	if err != nil {
		return err
	}

	found := false
	for _, vv := range m.installed {
		if vv == v {
			found = true
			break
		}
	}

	log.Printf("Installing templates for %s", v)
	err = m.templates.ApplyTemplates(path.Join(m.workpath, v.String()), v)
	if err != nil {
		return err
	}
	if found == false {
		log.Printf("Successfuly installed %s", v)
		m.installed = append(m.installed, v)
		sort.Sort(&m.installed)
	} else {
		log.Printf("Successfuly re-installed %s", v)
	}

	return nil
}

func (m *SymlinkWebotsManager) Install(v WebotsVersion) error {
	if err := m.tryLock(); err != nil {
		return err
	}
	defer m.unlock()
	return m.install(v)
}

func (m *SymlinkWebotsManager) Installed() []WebotsVersion {
	return []WebotsVersion(m.installed)
}

func (m *SymlinkWebotsManager) IsUsed(v WebotsVersion) bool {
	if m.inUse == nil {
		return false
	}
	return m.inUse.Major == v.Major && m.inUse.Minor == v.Minor && m.inUse.Patch == v.Patch
}

func getGid(g string) (int, error) {
	f, err := os.Open("/etc/group")
	if err != nil {
		return -1, err
	}
	r := bufio.NewReader(f)
	stopped := false
	groupRx := regexp.MustCompile(`^([a-z\-_A-Z0-9]+):[^:]*:([0-9]+):`)
	for stopped == false {
		l, err := r.ReadString('\n')
		if err == io.EOF {
			stopped = true
		} else if err != nil {
			return -1, err
		}
		if l == "\n" || len(l) == 0 {
			continue
		}

		m := groupRx.FindStringSubmatch(l)
		if m == nil {
			return -1, fmt.Errorf("Invalid line '%s'", l)
		}
		if m[1] == g {
			gid, err := strconv.ParseInt(m[2], 10, 0)
			if err != nil {
				return -1, err
			}
			return int(gid), nil
		}
	}
	return -1, nil
}

func SymlinkManagerSystemInit() error {
	if os.Getuid() != 0 || os.Geteuid() != 0 {
		return fmt.Errorf("need to be root")
	}

	gid, err := getGid("webots-manager")
	if err != nil {
		return fmt.Errorf("Could not check exitance of group webots-manager: %s", err)
	}
	if gid == -1 {
		cmd := exec.Command("addgroup", "webots-manager")
		err := cmd.Run()
		if err != nil {
			return err
		}
		fmt.Println("Added system group 'webots-manager'")
		gid, err = getGid("webots-manager")
		if err != nil {
			return err
		}
		if gid == -1 {
			return fmt.Errorf("Internal error, gid should not be -1")
		}
	}

	_, workpath, installpath, err := symlinkManagerPath()
	if err != nil {
		return err
	}

	err = os.MkdirAll(workpath, 0775|os.ModeSetgid)
	if err != nil {
		return err
	}

	err = os.Chown(workpath, 0, gid)
	if err != nil {
		return err
	}

	err = os.Chmod(workpath, 0775|os.ModeSetgid)
	if err != nil {
		return err
	}

	fi, err := os.Lstat(installpath)
	if err != nil && os.IsNotExist(err) == false {
		return err
	}
	usedpath := path.Join(workpath, "used")
	if os.IsNotExist(err) {
		err := os.Symlink(usedpath, installpath)
		if err != nil {
			return err
		}
	} else {
		if (fi.Mode() & os.ModeSymlink) != os.ModeSymlink {
			return fmt.Errorf("Webots seems already installed in %s, you should remove it.", installpath)
		}

		dest, err := os.Readlink(installpath)
		if err != nil {
			return err
		}
		if dest != path.Join(workpath, "used") {
			return fmt.Errorf("%s is a symling to incorrect target %s (require %s)", installpath, dest, usedpath)
		}
	}

	return nil
}

func (m *SymlinkWebotsManager) Use(v WebotsVersion) error {
	if err := m.tryLock(); err != nil {
		return err
	}
	defer m.unlock()

	found := false
	for _, vv := range m.installed {
		if vv == v {
			found = true
			break
		}
	}
	if found == false {
		log.Printf("Installing missing version %s", v)
		err := m.install(v)
		if err != nil {
			return err
		}
	}

	err := os.RemoveAll(m.usedpath)
	if err != nil {
		return err
	}
	err = os.Symlink(v.String(), m.usedpath)
	if err != nil {
		return err
	}

	m.inUse = &v

	return nil
}

func (m *SymlinkWebotsManager) ApplyAllTemplates() error {
	for _, v := range m.installed {
		err := m.templates.ApplyTemplates(path.Join(m.workpath, v.String()), v)
		if err != nil {
			return err
		}
	}
	return nil
}
<|endoftext|>


<commit_before>package softlayer

import (
	"errors"
	"fmt"
	"koding/db/mongodb"
	"koding/db/mongodb/modelhelper"
	"koding/kites/common"
	"koding/kites/kloud/api/sl"
	"koding/kites/kloud/contexthelper/request"
	"koding/kites/kloud/contexthelper/session"
	"koding/kites/kloud/dnsstorage"
	"koding/kites/kloud/eventer"
	"koding/kites/kloud/kloud"
	"koding/kites/kloud/pkg/dnsclient"
	"koding/kites/kloud/provider/helpers"
	"koding/kites/kloud/userdata"
	"time"

	"github.com/fatih/structs"
	"github.com/koding/kite"
	"github.com/koding/logging"
	"github.com/mitchellh/mapstructure"

	"golang.org/x/net/context"
	"gopkg.in/mgo.v2"
	"gopkg.in/mgo.v2/bson"
)

var (
	defaultKlientTimeout = 10 * time.Minute
	defaultStateTimeout  = 15 * time.Minute
)

type Provider struct {
	DB            *mongodb.MongoDB
	Log           logging.Logger
	Kite          *kite.Kite
	SLClient      *sl.Softlayer
	DNSClient     *dnsclient.Route53
	DNSStorage    *dnsstorage.MongodbStorage
	Userdata      *userdata.Userdata
	KlientTimeout time.Duration
	StateTimeout  time.Duration
}

type slCred struct {
	Username string `mapstructure:"username"`
	ApiKey   string `mapstructure:"api_key"`
}

func (p *Provider) Machine(ctx context.Context, id string) (interface{}, error) {
	if !bson.IsObjectIdHex(id) {
		return nil, fmt.Errorf("Invalid machine id: %q", id)
	}

	// let's first check if the id exists, because we are going to use
	// findAndModify() and it would be difficult to distinguish if the id
	// really doesn't exist or if there is an assignee which is a different
	// thing. (Because findAndModify() also returns "not found" for the case
	// where the id exist but someone else is the assignee).
	machine := &Machine{}
	if err := p.DB.Run("jMachines", func(c *mgo.Collection) error {
		return c.FindId(bson.ObjectIdHex(id)).One(&machine.Machine)
	}); err == mgo.ErrNotFound {
		return nil, kloud.NewError(kloud.ErrMachineNotFound)
	}

	req, ok := request.FromContext(ctx)
	if !ok {
		return nil, errors.New("request context is not available")
	}

	meta, err := machine.GetMeta()
	if err != nil {
		return nil, err
	}

	if meta.Datacenter == "" {
		// We choose DALLAS 01 because it has the largest capacity
		// http://www.softlayer.com/data-centers
		machine.Meta["datacenter"] = "sjc01"
		p.Log.Critical("[%s] datacenter is not set in. Fallback to sjc01", machine.ObjectId.Hex())
	}

	p.Log.Debug("Using datacenter=%q, image=%q", meta.Datacenter, meta.SourceImage)

	if err := p.AttachSession(ctx, machine); err != nil {
		return nil, err
	}

	// check for validation and permission
	if err := helpers.ValidateUser(machine.User, machine.Users, req); err != nil {
		return nil, err
	}

	machine.KlientTimeout = p.klientTimeout()
	machine.StateTimeout = p.stateTimeout()

	return machine, nil
}

func (p *Provider) AttachSession(ctx context.Context, machine *Machine) error {
	// get user model which contains user ssh keys or the list of users that
	// are allowed to use this machine
	if len(machine.Users) == 0 {
		return errors.New("permitted users list is empty")
	}

	// check if this is called via Kite call
	var requesterName string
	req, ok := request.FromContext(ctx)
	if ok {
		requesterName = req.Username
	}

	// get the user from the permitted list. If the list contains more than one
	// allowed person, fetch the one that is the same as requesterName, if
	// not pick up the first one.
	user, err := modelhelper.GetPermittedUser(requesterName, machine.Users)
	if err != nil {
		return err
	}

	// attach user specific log
	machine.Log = p.Log.New(machine.ObjectId.Hex())

	if traceID, ok := kloud.TraceFromContext(ctx); ok {
		machine.Log = common.NewLogger("kloud-softlayer", true).New(machine.ObjectId.Hex()).New(traceID)
	}

	sess := &session.Session{
		DB:         p.DB,
		Kite:       p.Kite,
		DNSClient:  p.DNSClient,
		DNSStorage: p.DNSStorage,
		Userdata:   p.Userdata,
		SLClient:   p.SLClient,
		Log:        machine.Log,
	}

	// we use session a lot of in Machine owned methods, so that's why we
	// assign it to a field for easy access
	machine.Session = sess

	// we pass it also to the context, so other packages, such as plans
	// checker can make use of it.
	ctx = session.NewContext(ctx, sess)

	machine.Username = user.Name
	machine.User = user

	ev, ok := eventer.FromContext(ctx)
	if ok {
		machine.Session.Eventer = ev
	}

	return nil
}

// getUserCredential fetches the credential for the given identifier. This is
// not used right now, but will be used once we decide to give custom softlayer
// instances
func (p *Provider) getUserCredential(identifier string) (*slCred, error) {
	creds, err := modelhelper.GetCredentialDatasFromIdentifiers(identifier)
	if err != nil {
		return nil, fmt.Errorf("could not fetch credential %q: %s", identifier, err.Error())
	}

	if len(creds) == 0 {
		return nil, fmt.Errorf("softlayer: no credential data available for credential: %s", identifier)
	}

	c := creds[0] // there is only one, pick up the first one

	var cred slCred
	if err := mapstructure.Decode(c.Meta, &cred); err != nil {
		return nil, err
	}

	if structs.HasZero(cred) {
		return nil, fmt.Errorf("softlayer data is incomplete: %v", c.Meta)
	}

	return &cred, nil
}

func (p *Provider) klientTimeout() time.Duration {
	if p.KlientTimeout != 0 {
		return p.KlientTimeout
	}
	return defaultKlientTimeout
}

func (p *Provider) stateTimeout() time.Duration {
	if p.StateTimeout != 0 {
		return p.StateTimeout
	}
	return defaultStateTimeout
}
<commit_msg>provider/softlayer: increase timeouts even more<commit_after>package softlayer

import (
	"errors"
	"fmt"
	"koding/db/mongodb"
	"koding/db/mongodb/modelhelper"
	"koding/kites/common"
	"koding/kites/kloud/api/sl"
	"koding/kites/kloud/contexthelper/request"
	"koding/kites/kloud/contexthelper/session"
	"koding/kites/kloud/dnsstorage"
	"koding/kites/kloud/eventer"
	"koding/kites/kloud/kloud"
	"koding/kites/kloud/pkg/dnsclient"
	"koding/kites/kloud/provider/helpers"
	"koding/kites/kloud/userdata"
	"time"

	"github.com/fatih/structs"
	"github.com/koding/kite"
	"github.com/koding/logging"
	"github.com/mitchellh/mapstructure"

	"golang.org/x/net/context"
	"gopkg.in/mgo.v2"
	"gopkg.in/mgo.v2/bson"
)

var (
	defaultKlientTimeout = 15 * time.Minute
	defaultStateTimeout  = 15 * time.Minute
)

type Provider struct {
	DB            *mongodb.MongoDB
	Log           logging.Logger
	Kite          *kite.Kite
	SLClient      *sl.Softlayer
	DNSClient     *dnsclient.Route53
	DNSStorage    *dnsstorage.MongodbStorage
	Userdata      *userdata.Userdata
	KlientTimeout time.Duration
	StateTimeout  time.Duration
}

type slCred struct {
	Username string `mapstructure:"username"`
	ApiKey   string `mapstructure:"api_key"`
}

func (p *Provider) Machine(ctx context.Context, id string) (interface{}, error) {
	if !bson.IsObjectIdHex(id) {
		return nil, fmt.Errorf("Invalid machine id: %q", id)
	}

	// let's first check if the id exists, because we are going to use
	// findAndModify() and it would be difficult to distinguish if the id
	// really doesn't exist or if there is an assignee which is a different
	// thing. (Because findAndModify() also returns "not found" for the case
	// where the id exist but someone else is the assignee).
	machine := &Machine{}
	if err := p.DB.Run("jMachines", func(c *mgo.Collection) error {
		return c.FindId(bson.ObjectIdHex(id)).One(&machine.Machine)
	}); err == mgo.ErrNotFound {
		return nil, kloud.NewError(kloud.ErrMachineNotFound)
	}

	req, ok := request.FromContext(ctx)
	if !ok {
		return nil, errors.New("request context is not available")
	}

	meta, err := machine.GetMeta()
	if err != nil {
		return nil, err
	}

	if meta.Datacenter == "" {
		// We choose DALLAS 01 because it has the largest capacity
		// http://www.softlayer.com/data-centers
		machine.Meta["datacenter"] = "sjc01"
		p.Log.Critical("[%s] datacenter is not set in. Fallback to sjc01", machine.ObjectId.Hex())
	}

	p.Log.Debug("Using datacenter=%q, image=%q", meta.Datacenter, meta.SourceImage)

	if err := p.AttachSession(ctx, machine); err != nil {
		return nil, err
	}

	// check for validation and permission
	if err := helpers.ValidateUser(machine.User, machine.Users, req); err != nil {
		return nil, err
	}

	machine.KlientTimeout = p.klientTimeout()
	machine.StateTimeout = p.stateTimeout()

	return machine, nil
}

func (p *Provider) AttachSession(ctx context.Context, machine *Machine) error {
	// get user model which contains user ssh keys or the list of users that
	// are allowed to use this machine
	if len(machine.Users) == 0 {
		return errors.New("permitted users list is empty")
	}

	// check if this is called via Kite call
	var requesterName string
	req, ok := request.FromContext(ctx)
	if ok {
		requesterName = req.Username
	}

	// get the user from the permitted list. If the list contains more than one
	// allowed person, fetch the one that is the same as requesterName, if
	// not pick up the first one.
	user, err := modelhelper.GetPermittedUser(requesterName, machine.Users)
	if err != nil {
		return err
	}

	// attach user specific log
	machine.Log = p.Log.New(machine.ObjectId.Hex())

	if traceID, ok := kloud.TraceFromContext(ctx); ok {
		machine.Log = common.NewLogger("kloud-softlayer", true).New(machine.ObjectId.Hex()).New(traceID)
	}

	sess := &session.Session{
		DB:         p.DB,
		Kite:       p.Kite,
		DNSClient:  p.DNSClient,
		DNSStorage: p.DNSStorage,
		Userdata:   p.Userdata,
		SLClient:   p.SLClient,
		Log:        machine.Log,
	}

	// we use session a lot of in Machine owned methods, so that's why we
	// assign it to a field for easy access
	machine.Session = sess

	// we pass it also to the context, so other packages, such as plans
	// checker can make use of it.
	ctx = session.NewContext(ctx, sess)

	machine.Username = user.Name
	machine.User = user

	ev, ok := eventer.FromContext(ctx)
	if ok {
		machine.Session.Eventer = ev
	}

	return nil
}

// getUserCredential fetches the credential for the given identifier. This is
// not used right now, but will be used once we decide to give custom softlayer
// instances
func (p *Provider) getUserCredential(identifier string) (*slCred, error) {
	creds, err := modelhelper.GetCredentialDatasFromIdentifiers(identifier)
	if err != nil {
		return nil, fmt.Errorf("could not fetch credential %q: %s", identifier, err.Error())
	}

	if len(creds) == 0 {
		return nil, fmt.Errorf("softlayer: no credential data available for credential: %s", identifier)
	}

	c := creds[0] // there is only one, pick up the first one

	var cred slCred
	if err := mapstructure.Decode(c.Meta, &cred); err != nil {
		return nil, err
	}

	if structs.HasZero(cred) {
		return nil, fmt.Errorf("softlayer data is incomplete: %v", c.Meta)
	}

	return &cred, nil
}

func (p *Provider) klientTimeout() time.Duration {
	if p.KlientTimeout != 0 {
		return p.KlientTimeout
	}
	return defaultKlientTimeout
}

func (p *Provider) stateTimeout() time.Duration {
	if p.StateTimeout != 0 {
		return p.StateTimeout
	}
	return defaultStateTimeout
}
<|endoftext|>