<commit_before>// Copyright 2016,2017 Yaacov Zamir <kobi.zamir@gmail.com>
// and other contributors.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//   http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

// Package alerts for alert rules
package alerts

import (
	"bytes"
	"encoding/json"
	"log"
	"net/http"
	"time"

	"github.com/MohawkTSDB/mohawk/storage"
)

type RangeIntervalType int

type Alert struct {
	ID              string            `mapstructure:"id"`
	Metric          string            `mapstructure:"metric"`
	Tenant          string            `mapstructure:"tenant"`
	From            float64           `mapstructure:"from"`
	To              float64           `mapstructure:"to"`
	Type            RangeIntervalType `mapstructure:"type"`
	State           bool
	TrigerValue     float64
	TrigerTimestamp int64
}

type Alerts struct {
	Backend        storage.Backend
	ServerURL      string
	Verbose        bool
	Alerts         []*Alert
	AlertsInterval int
	Heartbeat      int64
}

const (
	OUTSIDE RangeIntervalType = iota
	HIGHER_THAN
	LOWER_THAN
)

// Init fill in missing configuration data, and start the alert checking loop
func (a *Alerts) Init() {
	// if user omit the tenant field in the alerts config, fallback to default
	// tenant
	for _, alert := range a.Alerts {
		// fall back to _ops
		if alert.Tenant == "" {
			alert.Tenant = "_ops"
		}
	}

	// set default ServerURL
	if a.ServerURL == "" {
		a.ServerURL = "http://localhost:9099/append"
	}

	// check for alerts periodically.
	log.Printf("Start alerts, alert buffer url: %+v", a.ServerURL)
	log.Printf("Start alerts, alert interval: %+vs", a.AlertsInterval)
	go a.run()
}

// check for alert status chenge periodically
func (a *Alerts) run() {
	c := time.Tick(time.Second * time.Duration(a.AlertsInterval))

	for range c {
		a.checkAlerts()
	}
}

// updateAlertState update the alert status
func (alert *Alert) updateAlertState(value float64) {
	// valid metric values are:
	//    from < value >= to
	//    values outside this range will triger an alert
	switch alert.Type {
	case OUTSIDE:
		alert.State = value <= alert.From || value > alert.To
	case HIGHER_THAN:
		alert.State = value > alert.To
	case LOWER_THAN:
		alert.State = value <= alert.From
	}
}

// loop on all alerts and check for status change
func (a *Alerts) checkAlerts() {
	var end int64
	var start int64
	var tenant string
	var metric string
	var oldState bool

	// check only for current data
	end = int64(time.Now().UTC().Unix() * 1000)
	start = end - 60*60*1000

	for _, alert := range a.Alerts {
		// check out values for the alert metric
		tenant = alert.Tenant
		metric = alert.Metric
		rawData := a.Backend.GetRawData(tenant, metric, end, start, 1, "DESC")

		// if we have new data check for alert status change
		// [ if no new data found, leave alert status un changed ]
		if len(rawData) > 0 {
			oldState = alert.State

			// update alert state and report to user if changed.
			alert.updateAlertState(rawData[0].Value)
			if alert.State != oldState {
				// set triger values
				alert.TrigerValue = rawData[0].Value
				alert.TrigerTimestamp = rawData[0].Timestamp

				if b, err := json.Marshal(alert); err == nil {
					s := string(b)
					log.Println(s)
					a.post(s)
				}
			}
		}

		// update check compleat heart beat timestamp
		a.Heartbeat = end
	}
}

func (a *Alerts) post(s string) {
	client := http.Client{}
	req, err := http.NewRequest("POST", a.ServerURL, bytes.NewBufferString(s))
	if err == nil {
		client.Do(req)
	}
}
<commit_msg>Add alerts heartbeat fix<commit_after>// Copyright 2016,2017 Yaacov Zamir <kobi.zamir@gmail.com>
// and other contributors.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//   http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

// Package alerts for alert rules
package alerts

import (
	"bytes"
	"encoding/json"
	"log"
	"net/http"
	"time"

	"github.com/MohawkTSDB/mohawk/storage"
)

type RangeIntervalType int

type Alert struct {
	ID              string            `mapstructure:"id"`
	Metric          string            `mapstructure:"metric"`
	Tenant          string            `mapstructure:"tenant"`
	From            float64           `mapstructure:"from"`
	To              float64           `mapstructure:"to"`
	Type            RangeIntervalType `mapstructure:"type"`
	State           bool
	TrigerValue     float64
	TrigerTimestamp int64
}

type Alerts struct {
	Backend        storage.Backend
	ServerURL      string
	Verbose        bool
	Alerts         []*Alert
	AlertsInterval int
	Heartbeat      int64
}

const (
	OUTSIDE RangeIntervalType = iota
	HIGHER_THAN
	LOWER_THAN
)

// Init fill in missing configuration data, and start the alert checking loop
func (a *Alerts) Init() {
	// if user omit the tenant field in the alerts config, fallback to default
	// tenant
	for _, alert := range a.Alerts {
		// fall back to _ops
		if alert.Tenant == "" {
			alert.Tenant = "_ops"
		}
	}

	// set default ServerURL
	if a.ServerURL == "" {
		a.ServerURL = "http://localhost:9099/append"
	}

	// check for alerts periodically.
	log.Printf("Start alerts, alert buffer url: %+v", a.ServerURL)
	log.Printf("Start alerts, alert interval: %+vs", a.AlertsInterval)
	go a.run()
}

// check for alert status chenge periodically
func (a *Alerts) run() {
	c := time.Tick(time.Second * time.Duration(a.AlertsInterval))

	for range c {
		a.checkAlerts()
	}
}

// updateAlertState update the alert status
func (alert *Alert) updateAlertState(value float64) {
	// valid metric values are:
	//    from < value >= to
	//    values outside this range will triger an alert
	switch alert.Type {
	case OUTSIDE:
		alert.State = value <= alert.From || value > alert.To
	case HIGHER_THAN:
		alert.State = value > alert.To
	case LOWER_THAN:
		alert.State = value <= alert.From
	}
}

// loop on all alerts and check for status change
func (a *Alerts) checkAlerts() {
	var end int64
	var start int64
	var tenant string
	var metric string
	var oldState bool

	// check only for current data
	end = int64(time.Now().UTC().Unix() * 1000)
	start = end - 60*60*1000

	for _, alert := range a.Alerts {
		// check out values for the alert metric
		tenant = alert.Tenant
		metric = alert.Metric
		rawData := a.Backend.GetRawData(tenant, metric, end, start, 1, "DESC")

		// if we have new data check for alert status change
		// [ if no new data found, leave alert status un changed ]
		if len(rawData) > 0 {
			oldState = alert.State

			// update alert state and report to user if changed.
			alert.updateAlertState(rawData[0].Value)
			if alert.State != oldState {
				// set triger values
				alert.TrigerValue = rawData[0].Value
				alert.TrigerTimestamp = rawData[0].Timestamp

				if b, err := json.Marshal(alert); err == nil {
					s := string(b)
					log.Println(s)
					a.post(s)
				}
			}
		}
	}

	// update check compleat heart beat timestamp
	a.Heartbeat = end
}

func (a *Alerts) post(s string) {
	client := http.Client{}
	req, err := http.NewRequest("POST", a.ServerURL, bytes.NewBufferString(s))
	if err == nil {
		client.Do(req)
	}
}
<|endoftext|>


<commit_before>package alg

import (
	"errors"
	"math/rand"
)

// WeightedChoose will return toChoose indices from weights.
// the output can have duplicate indices, and zero-weights will
// cause this algorithm to malfunction.
// Deprecated: Use ChooseX or UniqueChooseX instead, as they'll
// perform more reliably and give better distributions.
func WeightedChoose(weights []float64, toChoose int) ([]int, error) {
	toChoosef := float64(toChoose)
	lengthWeights := len(weights)
	out := make([]int, toChoose)
	remainingWeights := make([]float64, lengthWeights)
	remainingWeights[lengthWeights-1] = weights[lengthWeights-1]
	for i := lengthWeights - 2; i >= 0; i-- {
		remainingWeights[i] = remainingWeights[i+1] + weights[i]
	}

	for i, v := range remainingWeights {
		if toChoose == 0 {
			return out, nil
		}
		if lengthWeights-i < toChoose {
			//ERROR Out
			return []int{}, errors.New("Tried to choose too many items from a slice")
		}
		if lengthWeights-i == toChoose {
			toChoose--
			toChoosef--
			out[toChoose] = i
		} else if rand.Float64() < toChoosef/v {
			toChoose--
			toChoosef--
			out[toChoose] = i
		}
	}

	return out, nil

}

// UniqueChooseX returns n indices from the input weights at a count
// relative to the weight of each index. This will never return duplicate indices.
// if n > len(weights), it will return -1 after depleting the n elements from
// weights.
func UniqueChooseX(weights []float64, n int) []int {
	out := make([]int, n)
	stwh := newSTWHeap(weights)
	for i := 0; i < n; i++ {
		out[i] = stwh.Pop()
	}
	return out
}

// ChooseX AKA Roulette search
//
// This returns n indices from the input weights at a count
// relative to the weight of each index. It can return the same index
// multiple times.
func ChooseX(weights []float64, n int) []int {
	lengthWeights := len(weights)
	out := make([]int, n)
	remainingWeights := make([]float64, lengthWeights)
	remainingWeights[lengthWeights-1] = weights[lengthWeights-1]
	for i := lengthWeights - 2; i >= 0; i-- {
		remainingWeights[i] = remainingWeights[i+1] + weights[i]
	}
	for i := 0; i < n; i++ {
		j := CumWeightedChooseOne(remainingWeights)
		// This might be deadcode, I can't get it to happen in tests
		// for weights[j] == 0 {
		// 	j = CumWeightedChooseOne(remainingWeights)
		// }
		out[i] = j
	}
	return out
}

// CumWeightedChooseOne returns a single index from the weights given
// at a rate relative to the magnitude of each weight
func CumWeightedChooseOne(remainingWeights []float64) int {
	totalWeight := remainingWeights[0]
	choice := rand.Float64() * totalWeight
	i := len(remainingWeights) / 2
	start := 0
	end := len(remainingWeights) - 1
	for {
		if remainingWeights[i] < choice {
			if remainingWeights[i-1] < choice {
				end = i
				i = (start + end) / 2
			} else {
				return i - 1
			}
		} else {
			if i != len(remainingWeights)-1 && remainingWeights[i+1] > choice {
				start = i

				i = (start + end) / 2
				if (start+end)%2 == 1 {
					i++
				}
			} else {
				return i
			}
		}
	}
}

// CumWeightedFromMap converts the input map into a set where keys are
// indices and values are weights for CumWeightedChooseOne, then returns
// the key for CumWeightedChooseOne of the weights.
func CumWeightedFromMap(weightMap map[int]float64) int {
	keys := make([]int, len(weightMap))
	values := make([]float64, len(weightMap))
	idx := 0
	for key, value := range weightMap {
		keys[idx] = key
		values[idx] = value
		idx++
	}

	return keys[CumWeightedChooseOne(values)]
}
<commit_msg>Add a helper to generate cumulative weights<commit_after>package alg

import (
	"errors"
	"math/rand"
)

// WeightedChoose will return toChoose indices from weights.
// the output can have duplicate indices, and zero-weights will
// cause this algorithm to malfunction.
// Deprecated: Use ChooseX or UniqueChooseX instead, as they'll
// perform more reliably and give better distributions.
func WeightedChoose(weights []float64, toChoose int) ([]int, error) {
	toChoosef := float64(toChoose)
	lengthWeights := len(weights)
	out := make([]int, toChoose)
	remainingWeights := make([]float64, lengthWeights)
	remainingWeights[lengthWeights-1] = weights[lengthWeights-1]
	for i := lengthWeights - 2; i >= 0; i-- {
		remainingWeights[i] = remainingWeights[i+1] + weights[i]
	}

	for i, v := range remainingWeights {
		if toChoose == 0 {
			return out, nil
		}
		if lengthWeights-i < toChoose {
			//ERROR Out
			return []int{}, errors.New("Tried to choose too many items from a slice")
		}
		if lengthWeights-i == toChoose {
			toChoose--
			toChoosef--
			out[toChoose] = i
		} else if rand.Float64() < toChoosef/v {
			toChoose--
			toChoosef--
			out[toChoose] = i
		}
	}

	return out, nil

}

// UniqueChooseX returns n indices from the input weights at a count
// relative to the weight of each index. This will never return duplicate indices.
// if n > len(weights), it will return -1 after depleting the n elements from
// weights.
func UniqueChooseX(weights []float64, n int) []int {
	out := make([]int, n)
	stwh := newSTWHeap(weights)
	for i := 0; i < n; i++ {
		out[i] = stwh.Pop()
	}
	return out
}

// ChooseX AKA Roulette search
//
// This returns n indices from the input weights at a count
// relative to the weight of each index. It can return the same index
// multiple times.
func ChooseX(weights []float64, n int) []int {
	lengthWeights := len(weights)
	out := make([]int, n)
	remainingWeights := make([]float64, lengthWeights)
	remainingWeights[lengthWeights-1] = weights[lengthWeights-1]
	for i := lengthWeights - 2; i >= 0; i-- {
		remainingWeights[i] = remainingWeights[i+1] + weights[i]
	}
	for i := 0; i < n; i++ {
		j := CumWeightedChooseOne(remainingWeights)
		// This might be deadcode, I can't get it to happen in tests
		// for weights[j] == 0 {
		// 	j = CumWeightedChooseOne(remainingWeights)
		// }
		out[i] = j
	}
	return out
}

// CumWeightedChooseOne returns a single index from the weights given
// at a rate relative to the magnitude of each weight
func CumWeightedChooseOne(remainingWeights []float64) int {
	totalWeight := remainingWeights[0]
	choice := rand.Float64() * totalWeight
	i := len(remainingWeights) / 2
	start := 0
	end := len(remainingWeights) - 1
	for {
		if remainingWeights[i] < choice {
			if remainingWeights[i-1] < choice {
				end = i
				i = (start + end) / 2
			} else {
				return i - 1
			}
		} else {
			if i != len(remainingWeights)-1 && remainingWeights[i+1] > choice {
				start = i

				i = (start + end) / 2
				if (start+end)%2 == 1 {
					i++
				}
			} else {
				return i
			}
		}
	}
}

// CumWeightedFromMap converts the input map into a set where keys are
// indices and values are weights for CumWeightedChooseOne, then returns
// the key for CumWeightedChooseOne of the weights.
func CumWeightedFromMap(weightMap map[int]float64) int {
	keys := make([]int, len(weightMap))
	values := make([]float64, len(weightMap))
	idx := 0
	for key, value := range weightMap {
		keys[idx] = key
		values[idx] = value
		idx++
	}

	return keys[CumWeightedChooseOne(values)]
}

// CumulativeWeights converts a slice of weights into
// a slice of cumulative weights, where each index
// is the sum of all weights up until that index in
// the original slice
func CumulativeWeights(weights []float64) []float64 {
	cum := make([]float64, len(weights))
	cum[0] = weights[0]
	for i := 1; i < len(weights); i++ {
		cum[i] = cum[i-1] + weights[i]
	}
	return cum
}
<|endoftext|>


<commit_before>package newrelic_collector_client

import (
	"testing"
)

const (
	TEST_APP_NAME    = "TestApp"
	NEWRELIC_LICENSE = "7bceac019c7dcafae1ef95be3e3a3ff8866de246"
)

func TestInit(t *testing.T) {
	if result := Init(NEWRELIC_LICENSE, TEST_APP_NAME); result != 0 {
		t.Errorf("Init test failed. Return: %d.", result)
	}
}

func TestDefaultWebTransactionHandler(t *testing.T) {
    DefaultWebTransactionHandler("test default transaction", 0.23)
}

var shutdownFlag = false
func TestRegisterShutdownCallback(t *testing.T) {
	RegisterShutdownCallback(func(){shutdownFlag = true})

    if result := RequestShutdown("shutdown reason 1"); result != 0 {
		t.Errorf("Shutdown test failed. Return: %d.", result)
	}
    if shutdownFlag == false {
        t.Errorf("Shutdown test failed. Callback was not called")
    }
}

func TestRequestShutdown(t *testing.T) {
	if result := RequestShutdown("shutdown reason2"); result != 0 {
		t.Errorf("Shutdown test failed. Return: %d.", result)
	}
}
<commit_msg>Changes in shutdown callback<commit_after>package newrelic_collector_client

import (
	"testing"
    "fmt"
)

const (
	TEST_APP_NAME    = "TestApp"
	NEWRELIC_LICENSE = "7bceac019c7dcafae1ef95be3e3a3ff8866de246"
)

func TestInit(t *testing.T) {
	if result := Init(NEWRELIC_LICENSE, TEST_APP_NAME); result != 0 {
		t.Errorf("Init test failed. Return: %d.", result)
	}
}

func TestDefaultWebTransactionHandler(t *testing.T) {
    DefaultWebTransactionHandler("test default transaction", 0.23)
}

var shutdownFlag = false
func TestRegisterShutdownCallback(t *testing.T) {
	RegisterShutdownCallback(func(){shutdownFlag = true; fmt.Printf("Shutdown callback called!\n")})

    if result := RequestShutdown("shutdown reason 1"); result != 0 {
		t.Errorf("Shutdown test failed. Return: %d.", result)
	}
    if shutdownFlag == false {
        t.Errorf("Shutdown test failed. Callback was not called")
    }
}

func TestRequestShutdown(t *testing.T) {
	if result := RequestShutdown("shutdown reason2"); result != 0 {
		t.Errorf("Shutdown test failed. Return: %d.", result)
	}
}
<|endoftext|>


<commit_before>package libcontainerd

import (
	"github.com/Microsoft/hcsshim"
	"github.com/opencontainers/runtime-spec/specs-go"
)

// Process contains information to start a specific application inside the container.
type Process specs.Process

// Summary contains a ProcessList item from HCS to support `top`
type Summary hcsshim.ProcessListItem

// StateInfo contains description about the new state container has entered.
type StateInfo struct {
	CommonStateInfo

	// Platform specific StateInfo

	UpdatePending bool // Indicates that there are some update operations pending that should be completed by a servicing container.
}

// Stats contains statics from HCS
type Stats hcsshim.Statistics

// Resources defines updatable container resource values.
type Resources struct{}

// ServicingOption is a CreateOption with a no-op application that signifies
// the container needs to be used for a Windows servicing operation.
type ServicingOption struct {
	IsServicing bool
}

// FlushOption is a CreateOption that signifies if the container should be
// started with flushes ignored until boot has completed. This is an optimisation
// for first boot of a container.
type FlushOption struct {
	IgnoreFlushesDuringBoot bool
}

// HyperVIsolationOption is a CreateOption that indicates whether the runtime
// should start the container as a Hyper-V container, and if so, the sandbox path.
type HyperVIsolationOption struct {
	IsHyperV    bool
	SandboxPath string `json:",omitempty"`
}

// LayerOption is a CreateOption that indicates to the runtime the layer folder
// and layer paths for a container.
type LayerOption struct {
	// LayerFolder is the path to the current layer folder. Empty for Hyper-V containers.
	LayerFolderPath string `json:",omitempty"`
	// Layer paths of the parent layers
	LayerPaths []string
}

// NetworkEndpointsOption is a CreateOption that provides the runtime list
// of network endpoints to which a container should be attached during its creation.
type NetworkEndpointsOption struct {
	Endpoints                []string
	AllowUnqualifiedDNSQuery bool
}

// CredentialsOption is a CreateOption that indicates the credentials from
// a credential spec to be used to the runtime
type CredentialsOption struct {
	Credentials string
}

// Checkpoint holds the details of a checkpoint (not supported in windows)
type Checkpoint struct {
	Name string
}

// Checkpoints contains the details of a checkpoint
type Checkpoints struct {
	Checkpoints []*Checkpoint
}
<commit_msg>fix some typos in libcontainer\types_windows.go<commit_after>package libcontainerd

import (
	"github.com/Microsoft/hcsshim"
	"github.com/opencontainers/runtime-spec/specs-go"
)

// Process contains information to start a specific application inside the container.
type Process specs.Process

// Summary contains a ProcessList item from HCS to support `top`
type Summary hcsshim.ProcessListItem

// StateInfo contains description about the new state container has entered.
type StateInfo struct {
	CommonStateInfo

	// Platform specific StateInfo
	UpdatePending bool // Indicates that there are some update operations pending that should be completed by a servicing container.
}

// Stats contains statistics from HCS
type Stats hcsshim.Statistics

// Resources defines updatable container resource values.
type Resources struct{}

// ServicingOption is a CreateOption with a no-op application that signifies
// the container needs to be used for a Windows servicing operation.
type ServicingOption struct {
	IsServicing bool
}

// FlushOption is a CreateOption that signifies if the container should be
// started with flushes ignored until boot has completed. This is an optimisation
// for first boot of a container.
type FlushOption struct {
	IgnoreFlushesDuringBoot bool
}

// HyperVIsolationOption is a CreateOption that indicates whether the runtime
// should start the container as a Hyper-V container, and if so, the sandbox path.
type HyperVIsolationOption struct {
	IsHyperV    bool
	SandboxPath string `json:",omitempty"`
}

// LayerOption is a CreateOption that indicates to the runtime the layer folder
// and layer paths for a container.
type LayerOption struct {
	// LayerFolderPath is the path to the current layer folder. Empty for Hyper-V containers.
	LayerFolderPath string `json:",omitempty"`
	// Layer paths of the parent layers
	LayerPaths []string
}

// NetworkEndpointsOption is a CreateOption that provides the runtime list
// of network endpoints to which a container should be attached during its creation.
type NetworkEndpointsOption struct {
	Endpoints                []string
	AllowUnqualifiedDNSQuery bool
}

// CredentialsOption is a CreateOption that indicates the credentials from
// a credential spec to be used to the runtime
type CredentialsOption struct {
	Credentials string
}

// Checkpoint holds the details of a checkpoint (not supported in windows)
type Checkpoint struct {
	Name string
}

// Checkpoints contains the details of a checkpoint
type Checkpoints struct {
	Checkpoints []*Checkpoint
}
<|endoftext|>


<commit_before>// Copyright 2015 The Cockroach Authors.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
// implied. See the License for the specific language governing
// permissions and limitations under the License. See the AUTHORS file
// for names of contributors.
//
// Author: Daniel Theophanes (kardianos@gmail.com)

package cli

import (
	"flag"

	"github.com/cockroachdb/cockroach/server"
)

// initFlags sets the server.Context values to flag values.
// Keep in sync with "server/context.go". Values in Context should be
// settable here.
func initFlags(ctx *server.Context) {
	// Server flags.
	flag.StringVar(&ctx.Addr, "addr", ctx.Addr, "when run as the server the host:port to bind for "+
		"HTTP/RPC traffic; when run as the client the address for connection to the cockroach cluster")

	flag.StringVar(&ctx.Certs, "certs", ctx.Certs, "directory containing RSA key and x509 certs")

	flag.StringVar(&ctx.Stores, "stores", ctx.Stores, "specify a comma-separated list of stores, "+
		"specified by a colon-separated list of device attributes followed by '=' and "+
		"either a filepath for a persistent store or an integer size in bytes for an "+
		"in-memory store. Device attributes typically include whether the store is "+
		"flash (ssd), spinny disk (hdd), fusion-io (fio), in-memory (mem); device "+
		"attributes might also include speeds and other specs (7200rpm, 200kiops, etc.). "+
		"For example, -store=hdd:7200rpm=/mnt/hda1,ssd=/mnt/ssd01,ssd=/mnt/ssd02,mem=1073741824")

	flag.StringVar(&ctx.Attrs, "attrs", ctx.Attrs, "specify a colon-separated list of node "+
		"attributes. Attributes are arbitrary strings specifying topography or "+
		"machine capabilities. Topography might include datacenter designation (e.g. "+
		"\"us-west-1a\", \"us-west-1b\", \"us-east-1c\"). Machine capabilities "+
		"might include specialized hardware or number of cores (e.g. \"gpu\", "+
		"\"x16c\"). For example: -attrs=us-west-1b,gpu")

	flag.DurationVar(&ctx.MaxOffset, "max_offset", ctx.MaxOffset, "specify "+
		"the maximum clock offset for the cluster. Clock offset is measured on all "+
		"node-to-node links and if any node notices it has clock offset in excess "+
		"of -max_offset, it will commit suicide. Setting this value too high may "+
		"decrease transaction performance in the presence of contention.")

	flag.BoolVar(&ctx.InitAndStart, "init_and_start", ctx.InitAndStart, "specify "+
		"to start the server after bootstrapping with the init command.")

	// Gossip flags.

	flag.StringVar(&ctx.GossipBootstrap, "gossip", ctx.GossipBootstrap,
		"addresses (comma-separated host:port pairs) of node addresses for gossip bootstrap")

	flag.DurationVar(&ctx.GossipInterval, "gossip_interval", ctx.GossipInterval,
		"approximate interval (time.Duration) for gossiping new information to peers")

	// KV flags.

	flag.BoolVar(&ctx.Linearizable, "linearizable", ctx.Linearizable, "enables linearizable behaviour "+
		"of operations on this node by making sure that no commit timestamp is reported "+
		"back to the client until all other node clocks have necessarily passed it.")

	// Engine flags.

	flag.Int64Var(&ctx.CacheSize, "cache_size", ctx.CacheSize, "total size in bytes for "+
		"caches, shared evenly if there are multiple storage devices")
}

func init() {
	initFlags(Context)
}
<commit_msg>update flag description for -attrs<commit_after>// Copyright 2015 The Cockroach Authors.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
// implied. See the License for the specific language governing
// permissions and limitations under the License. See the AUTHORS file
// for names of contributors.
//
// Author: Daniel Theophanes (kardianos@gmail.com)

package cli

import (
	"flag"

	"github.com/cockroachdb/cockroach/server"
)

// initFlags sets the server.Context values to flag values.
// Keep in sync with "server/context.go". Values in Context should be
// settable here.
func initFlags(ctx *server.Context) {
	// Server flags.
	flag.StringVar(&ctx.Addr, "addr", ctx.Addr, "when run as the server the host:port to bind for "+
		"HTTP/RPC traffic; when run as the client the address for connection to the cockroach cluster")

	flag.StringVar(&ctx.Certs, "certs", ctx.Certs, "directory containing RSA key and x509 certs")

	flag.StringVar(&ctx.Stores, "stores", ctx.Stores, "specify a comma-separated list of stores, "+
		"specified by a colon-separated list of device attributes followed by '=' and "+
		"either a filepath for a persistent store or an integer size in bytes for an "+
		"in-memory store. Device attributes typically include whether the store is "+
		"flash (ssd), spinny disk (hdd), fusion-io (fio), in-memory (mem); device "+
		"attributes might also include speeds and other specs (7200rpm, 200kiops, etc.). "+
		"For example, -store=hdd:7200rpm=/mnt/hda1,ssd=/mnt/ssd01,ssd=/mnt/ssd02,mem=1073741824")

	flag.StringVar(&ctx.Attrs, "attrs", ctx.Attrs, "specify an ordered, colon-separated list of node "+
		"attributes. Attributes are arbitrary strings specifying topography or "+
		"machine capabilities. Topography might include datacenter designation "+
		"(e.g. \"us-west-1a\", \"us-west-1b\", \"us-east-1c\"). Machine capabilities "+
		"might include specialized hardware or number of cores (e.g. \"gpu\", "+
		"\"x16c\"). "+
		"The relative geographic proximity of two nodes is inferred from the "+
		"common prefix of the attributes list, so topographic attributes should be "+
		"specified first and in the same order for all nodes. "+
		"For example: -attrs=us-west-1b,gpu.")

	flag.DurationVar(&ctx.MaxOffset, "max_offset", ctx.MaxOffset, "specify "+
		"the maximum clock offset for the cluster. Clock offset is measured on all "+
		"node-to-node links and if any node notices it has clock offset in excess "+
		"of -max_offset, it will commit suicide. Setting this value too high may "+
		"decrease transaction performance in the presence of contention.")

	flag.BoolVar(&ctx.InitAndStart, "init_and_start", ctx.InitAndStart, "specify "+
		"to start the server after bootstrapping with the init command.")

	// Gossip flags.

	flag.StringVar(&ctx.GossipBootstrap, "gossip", ctx.GossipBootstrap,
		"addresses (comma-separated host:port pairs) of node addresses for gossip bootstrap")

	flag.DurationVar(&ctx.GossipInterval, "gossip_interval", ctx.GossipInterval,
		"approximate interval (time.Duration) for gossiping new information to peers")

	// KV flags.

	flag.BoolVar(&ctx.Linearizable, "linearizable", ctx.Linearizable, "enables linearizable behaviour "+
		"of operations on this node by making sure that no commit timestamp is reported "+
		"back to the client until all other node clocks have necessarily passed it.")

	// Engine flags.

	flag.Int64Var(&ctx.CacheSize, "cache_size", ctx.CacheSize, "total size in bytes for "+
		"caches, shared evenly if there are multiple storage devices")
}

func init() {
	initFlags(Context)
}
<|endoftext|>


<commit_before>package bosh

import (
	"bytes"
	"errors"
	"fmt"
	"io"
	"os"
	"os/exec"
	"path/filepath"
	"regexp"
	"strings"
)

type Executor struct {
	command       command
	readFile      func(string) ([]byte, error)
	unmarshalJSON func([]byte, interface{}) error
	marshalJSON   func(interface{}) ([]byte, error)
	writeFile     func(string, []byte, os.FileMode) error
}

type InterpolateInput struct {
	DeploymentDir  string
	StateDir       string
	VarsDir        string
	IAAS           string
	DeploymentVars string
	BOSHState      map[string]interface{}
	Variables      string
	OpsFile        string
}

type CreateEnvInput struct {
	StateDir   string
	VarsDir    string
	Deployment string
}

type DeleteEnvInput CreateEnvInput

type command interface {
	GetBOSHPath() (string, error)
	Run(stdout io.Writer, workingDirectory string, args []string) error
}

type setupFile struct {
	path     string
	contents []byte
}

const VERSION_DEV_BUILD = "[DEV BUILD]"

func NewExecutor(cmd command, readFile func(string) ([]byte, error),
	unmarshalJSON func([]byte, interface{}) error,
	marshalJSON func(interface{}) ([]byte, error),
	writeFile func(string, []byte, os.FileMode) error) Executor {
	return Executor{
		command:       cmd,
		readFile:      readFile,
		unmarshalJSON: unmarshalJSON,
		marshalJSON:   marshalJSON,
		writeFile:     writeFile,
	}
}

func (e Executor) JumpboxCreateEnvArgs(input InterpolateInput) error {
	setupFiles := map[string]setupFile{
		"manifest": setupFile{
			path:     filepath.Join(input.DeploymentDir, "jumpbox.yml"),
			contents: MustAsset("vendor/github.com/cppforlife/jumpbox-deployment/jumpbox.yml"),
		},
		"vars-file": setupFile{
			path:     filepath.Join(input.VarsDir, "jumpbox-deployment-vars.yml"),
			contents: []byte(input.DeploymentVars),
		},
		"cpi": setupFile{
			path:     filepath.Join(input.DeploymentDir, "cpi.yml"),
			contents: MustAsset(filepath.Join("vendor/github.com/cppforlife/jumpbox-deployment", input.IAAS, "cpi.yml")),
		},
		"vars-store": setupFile{
			path:     filepath.Join(input.VarsDir, "jumpbox-variables.yml"),
			contents: []byte(input.Variables),
		},
	}

	for _, f := range setupFiles {
		err := e.writeFile(f.path, f.contents, os.ModePerm)
		if err != nil {
			return fmt.Errorf("Jumpbox write setup file: %s", err) //not tested
		}
	}

	sharedArgs := []string{
		"--vars-store", setupFiles["vars-store"].path,
		"--vars-file", setupFiles["vars-file"].path,
		"-o", setupFiles["cpi"].path,
	}

	jumpboxState := filepath.Join(input.VarsDir, "jumpbox-state.json")
	if input.BOSHState != nil {
		stateJSON, err := e.marshalJSON(input.BOSHState)
		if err != nil {
			return fmt.Errorf("Jumpbox marshal state json: %s", err) //not tested
		}

		err = e.writeFile(jumpboxState, stateJSON, os.ModePerm)
		if err != nil {
			return fmt.Errorf("Jumpbox write state json: %s", err) //not tested
		}
	}

	boshArgs := append([]string{
		setupFiles["manifest"].path,
		"--state", jumpboxState,
	}, sharedArgs...)

	boshPath, err := e.command.GetBOSHPath()
	if err != nil {
		return fmt.Errorf("Jumpbox get BOSH path: %s", err) //not tested
	}

	createEnvCmd := []byte(formatScript(boshPath, input.StateDir, "create-env", boshArgs))
	createJumpboxScript := filepath.Join(input.StateDir, "create-jumpbox.sh")
	err = e.writeFileUnlessExisting(createJumpboxScript, createEnvCmd, os.ModePerm, "Jumpbox write create-env script")
	if err != nil {
		return err
	}

	deleteEnvCmd := []byte(formatScript(boshPath, input.StateDir, "delete-env", boshArgs))
	deleteJumpboxScript := filepath.Join(input.StateDir, "delete-jumpbox.sh")
	err = e.writeFileUnlessExisting(deleteJumpboxScript, deleteEnvCmd, os.ModePerm, "Jumpbox write delete-env script")
	if err != nil {
		return err
	}

	return nil
}

func (e Executor) DirectorCreateEnvArgs(input InterpolateInput) error {
	setupFiles := map[string]setupFile{
		"manifest": setupFile{
			path:     filepath.Join(input.DeploymentDir, "bosh.yml"),
			contents: MustAsset("vendor/github.com/cloudfoundry/bosh-deployment/bosh.yml"),
		},
		"vars-file": setupFile{
			path:     filepath.Join(input.VarsDir, "director-deployment-vars.yml"),
			contents: []byte(input.DeploymentVars),
		},
		"vars-store": setupFile{
			path:     filepath.Join(input.VarsDir, "director-variables.yml"),
			contents: []byte(input.Variables),
		},
		"user-ops": setupFile{
			path:     filepath.Join(input.VarsDir, "user-ops-file.yml"),
			contents: []byte(input.OpsFile),
		},
	}

	opsFiles := []setupFile{
		setupFile{
			path:     filepath.Join(input.DeploymentDir, "cpi.yml"),
			contents: MustAsset(filepath.Join("vendor/github.com/cloudfoundry/bosh-deployment", input.IAAS, "cpi.yml")),
		},
		setupFile{
			path:     filepath.Join(input.DeploymentDir, "jumpbox-user.yml"),
			contents: MustAsset("vendor/github.com/cloudfoundry/bosh-deployment/jumpbox-user.yml"),
		},
		setupFile{
			path:     filepath.Join(input.DeploymentDir, "uaa.yml"),
			contents: MustAsset("vendor/github.com/cloudfoundry/bosh-deployment/uaa.yml"),
		},
		setupFile{
			path:     filepath.Join(input.DeploymentDir, "credhub.yml"),
			contents: MustAsset("vendor/github.com/cloudfoundry/bosh-deployment/credhub.yml"),
		},
	}

	switch input.IAAS {
	case "gcp":
		opsFiles = append(opsFiles, setupFile{
			path:     filepath.Join(input.DeploymentDir, "gcp-bosh-director-ephemeral-ip-ops.yml"),
			contents: []byte(GCPBoshDirectorEphemeralIPOps),
		})
	case "aws":
		opsFiles = append(opsFiles,
			setupFile{
				path:     filepath.Join(input.DeploymentDir, "aws-bosh-director-ephemeral-ip-ops.yml"),
				contents: []byte(AWSBoshDirectorEphemeralIPOps),
			},
			setupFile{
				path:     filepath.Join(input.DeploymentDir, "iam-instance-profile.yml"),
				contents: MustAsset("vendor/github.com/cloudfoundry/bosh-deployment/aws/iam-instance-profile.yml"),
			},
			setupFile{
				path:     filepath.Join(input.DeploymentDir, "aws-bosh-director-encrypt-disk-ops.yml"),
				contents: []byte(AWSEncryptDiskOps),
			})
	}

	for _, f := range setupFiles {
		err := e.writeFile(f.path, f.contents, os.ModePerm)
		if err != nil {
			return fmt.Errorf("write file: %s", err) //not tested
		}
	}

	for _, f := range opsFiles {
		err := e.writeFile(f.path, f.contents, os.ModePerm)
		if err != nil {
			return fmt.Errorf("write file: %s", err) //not tested
		}
	}

	sharedArgs := []string{
		"--vars-store", setupFiles["vars-store"].path,
		"--vars-file", setupFiles["vars-file"].path,
	}

	for _, f := range opsFiles {
		sharedArgs = append(sharedArgs, "-o", f.path)
	}

	if input.OpsFile != "" {
		sharedArgs = append(sharedArgs, "-o", filepath.Join(input.VarsDir, "user-ops-file.yml"))
	}

	boshState := filepath.Join(input.VarsDir, "bosh-state.json")
	if input.BOSHState != nil {
		stateJSON, err := e.marshalJSON(input.BOSHState)
		if err != nil {
			return fmt.Errorf("marshal JSON: %s", err) //not tested
		}

		err = e.writeFile(boshState, stateJSON, os.ModePerm)
		if err != nil {
			return fmt.Errorf("write file: %s", err) //not tested
		}
	}

	boshPath, err := e.command.GetBOSHPath()
	if err != nil {
		return fmt.Errorf("Director get BOSH path: %s", err) //not tested
	}

	boshArgs := append([]string{
		setupFiles["manifest"].path,
		"--state", boshState,
	}, sharedArgs...)

	createEnvCmd := []byte(formatScript(boshPath, input.StateDir, "create-env", boshArgs))
	err = e.writeFileUnlessExisting(filepath.Join(input.StateDir, "create-director.sh"), createEnvCmd, os.ModePerm, "Write create-env script for director")
	if err != nil {
		return err
	}

	deleteEnvCmd := []byte(formatScript(boshPath, input.StateDir, "delete-env", boshArgs))
	err = e.writeFileUnlessExisting(filepath.Join(input.StateDir, "delete-director.sh"), deleteEnvCmd, os.ModePerm, "Write delete-env script for director")
	if err != nil {
		return err
	}

	return nil
}

func formatScript(boshPath, stateDir, command string, args []string) string {
	script := fmt.Sprintf("#!/bin/sh\n%s %s \\\n", boshPath, command)
	for _, arg := range args {
		if arg[0] == '-' {
			script = fmt.Sprintf("%s  %s", script, arg)
		} else {
			script = fmt.Sprintf("%s  %s \\\n", script, arg)
		}
	}
	script = strings.Replace(script, stateDir, "${BBL_STATE_DIR}", -1)
	return fmt.Sprintf("%s\n", script[:len(script)-2])
}

func (e Executor) writeFileUnlessExisting(path string, contents []byte, mode os.FileMode, failureMessage string) error {
	_, err := os.Stat(path)
	if err != nil {
		err = e.writeFile(path, contents, mode)
		if err != nil {
			return fmt.Errorf("%s: %s", failureMessage, err) //not tested
		}
	}

	return nil
}

func (e Executor) CreateEnv(createEnvInput CreateEnvInput) (string, error) {
	os.Setenv("BBL_STATE_DIR", createEnvInput.StateDir)
	createEnvScript := filepath.Join(createEnvInput.StateDir, fmt.Sprintf("create-%s.sh", createEnvInput.Deployment))

	cmd := exec.Command(createEnvScript)

	err := cmd.Run()
	if err != nil {
		return "", fmt.Errorf("Run bosh create-env: %s", err)
	}

	varsStoreFileName := fmt.Sprintf("%s-variables.yml", createEnvInput.Deployment)
	varsStoreContents, err := e.readFile(filepath.Join(createEnvInput.VarsDir, varsStoreFileName))
	if err != nil {
		return "", fmt.Errorf("Reading vars file for %s deployment: %s", createEnvInput.Deployment, err) // not tested
	}

	return string(varsStoreContents), nil
}

func (e Executor) DeleteEnv(deleteEnvInput DeleteEnvInput) error {
	os.Setenv("BBL_STATE_DIR", deleteEnvInput.StateDir)
	deleteEnvScript := filepath.Join(deleteEnvInput.StateDir, fmt.Sprintf("delete-%s.sh", deleteEnvInput.Deployment))

	cmd := exec.Command(deleteEnvScript)

	err := cmd.Run()
	if err != nil {
		return fmt.Errorf("Run bosh delete-env: %s", err)
	}

	return nil
}

func (e Executor) Version() (string, error) {
	args := []string{"-v"}
	buffer := bytes.NewBuffer([]byte{})
	err := e.command.Run(buffer, "", args)
	if err != nil {
		return "", err
	}

	versionOutput := buffer.String()
	regex := regexp.MustCompile(`\d+.\d+.\d+`)

	version := regex.FindString(versionOutput)
	if version == "" {
		return "", NewBOSHVersionError(errors.New("BOSH version could not be parsed"))
	}

	return version, nil
}
<commit_msg>Print BOSH output to stdout/stderr<commit_after>package bosh

import (
	"bytes"
	"errors"
	"fmt"
	"io"
	"os"
	"os/exec"
	"path/filepath"
	"regexp"
	"strings"
)

type Executor struct {
	command       command
	readFile      func(string) ([]byte, error)
	unmarshalJSON func([]byte, interface{}) error
	marshalJSON   func(interface{}) ([]byte, error)
	writeFile     func(string, []byte, os.FileMode) error
}

type InterpolateInput struct {
	DeploymentDir  string
	StateDir       string
	VarsDir        string
	IAAS           string
	DeploymentVars string
	BOSHState      map[string]interface{}
	Variables      string
	OpsFile        string
}

type CreateEnvInput struct {
	StateDir   string
	VarsDir    string
	Deployment string
}

type DeleteEnvInput CreateEnvInput

type command interface {
	GetBOSHPath() (string, error)
	Run(stdout io.Writer, workingDirectory string, args []string) error
}

type setupFile struct {
	path     string
	contents []byte
}

const VERSION_DEV_BUILD = "[DEV BUILD]"

func NewExecutor(cmd command, readFile func(string) ([]byte, error),
	unmarshalJSON func([]byte, interface{}) error,
	marshalJSON func(interface{}) ([]byte, error),
	writeFile func(string, []byte, os.FileMode) error) Executor {
	return Executor{
		command:       cmd,
		readFile:      readFile,
		unmarshalJSON: unmarshalJSON,
		marshalJSON:   marshalJSON,
		writeFile:     writeFile,
	}
}

func (e Executor) JumpboxCreateEnvArgs(input InterpolateInput) error {
	setupFiles := map[string]setupFile{
		"manifest": setupFile{
			path:     filepath.Join(input.DeploymentDir, "jumpbox.yml"),
			contents: MustAsset("vendor/github.com/cppforlife/jumpbox-deployment/jumpbox.yml"),
		},
		"vars-file": setupFile{
			path:     filepath.Join(input.VarsDir, "jumpbox-deployment-vars.yml"),
			contents: []byte(input.DeploymentVars),
		},
		"cpi": setupFile{
			path:     filepath.Join(input.DeploymentDir, "cpi.yml"),
			contents: MustAsset(filepath.Join("vendor/github.com/cppforlife/jumpbox-deployment", input.IAAS, "cpi.yml")),
		},
		"vars-store": setupFile{
			path:     filepath.Join(input.VarsDir, "jumpbox-variables.yml"),
			contents: []byte(input.Variables),
		},
	}

	for _, f := range setupFiles {
		err := e.writeFile(f.path, f.contents, os.ModePerm)
		if err != nil {
			return fmt.Errorf("Jumpbox write setup file: %s", err) //not tested
		}
	}

	sharedArgs := []string{
		"--vars-store", setupFiles["vars-store"].path,
		"--vars-file", setupFiles["vars-file"].path,
		"-o", setupFiles["cpi"].path,
	}

	jumpboxState := filepath.Join(input.VarsDir, "jumpbox-state.json")
	if input.BOSHState != nil {
		stateJSON, err := e.marshalJSON(input.BOSHState)
		if err != nil {
			return fmt.Errorf("Jumpbox marshal state json: %s", err) //not tested
		}

		err = e.writeFile(jumpboxState, stateJSON, os.ModePerm)
		if err != nil {
			return fmt.Errorf("Jumpbox write state json: %s", err) //not tested
		}
	}

	boshArgs := append([]string{
		setupFiles["manifest"].path,
		"--state", jumpboxState,
	}, sharedArgs...)

	boshPath, err := e.command.GetBOSHPath()
	if err != nil {
		return fmt.Errorf("Jumpbox get BOSH path: %s", err) //not tested
	}

	createEnvCmd := []byte(formatScript(boshPath, input.StateDir, "create-env", boshArgs))
	createJumpboxScript := filepath.Join(input.StateDir, "create-jumpbox.sh")
	err = e.writeFileUnlessExisting(createJumpboxScript, createEnvCmd, os.ModePerm, "Jumpbox write create-env script")
	if err != nil {
		return err
	}

	deleteEnvCmd := []byte(formatScript(boshPath, input.StateDir, "delete-env", boshArgs))
	deleteJumpboxScript := filepath.Join(input.StateDir, "delete-jumpbox.sh")
	err = e.writeFileUnlessExisting(deleteJumpboxScript, deleteEnvCmd, os.ModePerm, "Jumpbox write delete-env script")
	if err != nil {
		return err
	}

	return nil
}

func (e Executor) DirectorCreateEnvArgs(input InterpolateInput) error {
	setupFiles := map[string]setupFile{
		"manifest": setupFile{
			path:     filepath.Join(input.DeploymentDir, "bosh.yml"),
			contents: MustAsset("vendor/github.com/cloudfoundry/bosh-deployment/bosh.yml"),
		},
		"vars-file": setupFile{
			path:     filepath.Join(input.VarsDir, "director-deployment-vars.yml"),
			contents: []byte(input.DeploymentVars),
		},
		"vars-store": setupFile{
			path:     filepath.Join(input.VarsDir, "director-variables.yml"),
			contents: []byte(input.Variables),
		},
		"user-ops": setupFile{
			path:     filepath.Join(input.VarsDir, "user-ops-file.yml"),
			contents: []byte(input.OpsFile),
		},
	}

	opsFiles := []setupFile{
		setupFile{
			path:     filepath.Join(input.DeploymentDir, "cpi.yml"),
			contents: MustAsset(filepath.Join("vendor/github.com/cloudfoundry/bosh-deployment", input.IAAS, "cpi.yml")),
		},
		setupFile{
			path:     filepath.Join(input.DeploymentDir, "jumpbox-user.yml"),
			contents: MustAsset("vendor/github.com/cloudfoundry/bosh-deployment/jumpbox-user.yml"),
		},
		setupFile{
			path:     filepath.Join(input.DeploymentDir, "uaa.yml"),
			contents: MustAsset("vendor/github.com/cloudfoundry/bosh-deployment/uaa.yml"),
		},
		setupFile{
			path:     filepath.Join(input.DeploymentDir, "credhub.yml"),
			contents: MustAsset("vendor/github.com/cloudfoundry/bosh-deployment/credhub.yml"),
		},
	}

	switch input.IAAS {
	case "gcp":
		opsFiles = append(opsFiles, setupFile{
			path:     filepath.Join(input.DeploymentDir, "gcp-bosh-director-ephemeral-ip-ops.yml"),
			contents: []byte(GCPBoshDirectorEphemeralIPOps),
		})
	case "aws":
		opsFiles = append(opsFiles,
			setupFile{
				path:     filepath.Join(input.DeploymentDir, "aws-bosh-director-ephemeral-ip-ops.yml"),
				contents: []byte(AWSBoshDirectorEphemeralIPOps),
			},
			setupFile{
				path:     filepath.Join(input.DeploymentDir, "iam-instance-profile.yml"),
				contents: MustAsset("vendor/github.com/cloudfoundry/bosh-deployment/aws/iam-instance-profile.yml"),
			},
			setupFile{
				path:     filepath.Join(input.DeploymentDir, "aws-bosh-director-encrypt-disk-ops.yml"),
				contents: []byte(AWSEncryptDiskOps),
			})
	}

	for _, f := range setupFiles {
		err := e.writeFile(f.path, f.contents, os.ModePerm)
		if err != nil {
			return fmt.Errorf("write file: %s", err) //not tested
		}
	}

	for _, f := range opsFiles {
		err := e.writeFile(f.path, f.contents, os.ModePerm)
		if err != nil {
			return fmt.Errorf("write file: %s", err) //not tested
		}
	}

	sharedArgs := []string{
		"--vars-store", setupFiles["vars-store"].path,
		"--vars-file", setupFiles["vars-file"].path,
	}

	for _, f := range opsFiles {
		sharedArgs = append(sharedArgs, "-o", f.path)
	}

	if input.OpsFile != "" {
		sharedArgs = append(sharedArgs, "-o", filepath.Join(input.VarsDir, "user-ops-file.yml"))
	}

	boshState := filepath.Join(input.VarsDir, "bosh-state.json")
	if input.BOSHState != nil {
		stateJSON, err := e.marshalJSON(input.BOSHState)
		if err != nil {
			return fmt.Errorf("marshal JSON: %s", err) //not tested
		}

		err = e.writeFile(boshState, stateJSON, os.ModePerm)
		if err != nil {
			return fmt.Errorf("write file: %s", err) //not tested
		}
	}

	boshPath, err := e.command.GetBOSHPath()
	if err != nil {
		return fmt.Errorf("Director get BOSH path: %s", err) //not tested
	}

	boshArgs := append([]string{
		setupFiles["manifest"].path,
		"--state", boshState,
	}, sharedArgs...)

	createEnvCmd := []byte(formatScript(boshPath, input.StateDir, "create-env", boshArgs))
	err = e.writeFileUnlessExisting(filepath.Join(input.StateDir, "create-director.sh"), createEnvCmd, os.ModePerm, "Write create-env script for director")
	if err != nil {
		return err
	}

	deleteEnvCmd := []byte(formatScript(boshPath, input.StateDir, "delete-env", boshArgs))
	err = e.writeFileUnlessExisting(filepath.Join(input.StateDir, "delete-director.sh"), deleteEnvCmd, os.ModePerm, "Write delete-env script for director")
	if err != nil {
		return err
	}

	return nil
}

func formatScript(boshPath, stateDir, command string, args []string) string {
	script := fmt.Sprintf("#!/bin/sh\n%s %s \\\n", boshPath, command)
	for _, arg := range args {
		if arg[0] == '-' {
			script = fmt.Sprintf("%s  %s", script, arg)
		} else {
			script = fmt.Sprintf("%s  %s \\\n", script, arg)
		}
	}
	script = strings.Replace(script, stateDir, "${BBL_STATE_DIR}", -1)
	return fmt.Sprintf("%s\n", script[:len(script)-2])
}

func (e Executor) writeFileUnlessExisting(path string, contents []byte, mode os.FileMode, failureMessage string) error {
	_, err := os.Stat(path)
	if err != nil {
		err = e.writeFile(path, contents, mode)
		if err != nil {
			return fmt.Errorf("%s: %s", failureMessage, err) //not tested
		}
	}

	return nil
}

func (e Executor) CreateEnv(createEnvInput CreateEnvInput) (string, error) {
	os.Setenv("BBL_STATE_DIR", createEnvInput.StateDir)
	createEnvScript := filepath.Join(createEnvInput.StateDir, fmt.Sprintf("create-%s.sh", createEnvInput.Deployment))

	cmd := exec.Command(createEnvScript)
	cmd.Stdout = os.Stdout
	cmd.Stderr = os.Stderr

	err := cmd.Run()
	if err != nil {
		return "", fmt.Errorf("Run bosh create-env: %s", err)
	}

	varsStoreFileName := fmt.Sprintf("%s-variables.yml", createEnvInput.Deployment)
	varsStoreContents, err := e.readFile(filepath.Join(createEnvInput.VarsDir, varsStoreFileName))
	if err != nil {
		return "", fmt.Errorf("Reading vars file for %s deployment: %s", createEnvInput.Deployment, err) // not tested
	}

	return string(varsStoreContents), nil
}

func (e Executor) DeleteEnv(deleteEnvInput DeleteEnvInput) error {
	os.Setenv("BBL_STATE_DIR", deleteEnvInput.StateDir)
	deleteEnvScript := filepath.Join(deleteEnvInput.StateDir, fmt.Sprintf("delete-%s.sh", deleteEnvInput.Deployment))

	cmd := exec.Command(deleteEnvScript)
	cmd.Stdout = os.Stdout
	cmd.Stderr = os.Stderr

	err := cmd.Run()
	if err != nil {
		return fmt.Errorf("Run bosh delete-env: %s", err)
	}

	return nil
}

func (e Executor) Version() (string, error) {
	args := []string{"-v"}
	buffer := bytes.NewBuffer([]byte{})
	err := e.command.Run(buffer, "", args)
	if err != nil {
		return "", err
	}

	versionOutput := buffer.String()
	regex := regexp.MustCompile(`\d+.\d+.\d+`)

	version := regex.FindString(versionOutput)
	if version == "" {
		return "", NewBOSHVersionError(errors.New("BOSH version could not be parsed"))
	}

	return version, nil
}
<|endoftext|>


<commit_before>package main

import (
	"log"
	"os"

	"github.com/MohamedBassem/getaredis"
	"github.com/robfig/cron"
)

func monitorHosts() {
	ctx, err := getaredis.Init("../../config.yml")
	errLogger := log.New(os.Stderr, "MonitorHosts", 0)
	outLogger := log.New(os.Stdout, "MonitorHosts", 0)
	outLogger.Println("Started")
	defer outLogger.Println("Finished")
	if err != nil {
		errLogger.Println("Error :" + err.Error())
		return
	}

	started, deleted, err := getaredis.MonitorHosts(ctx)
	if err != nil {
		errLogger.Println("Error :" + err.Error())
		return
	}

	if started {
		outLogger.Println("A new host has started!")
	}

	if deleted != nil && len(deleted) > 0 {
		outLogger.Printf("Hosts %v have been removed.", deleted)
	}
}

func cleanRedisInstances() {
	ctx, err := getaredis.Init("../../config.yml")
	errLogger := log.New(os.Stderr, "CleanRedisInstances", 0)
	outLogger := log.New(os.Stdout, "CleanRedisInstances", 0)
	outLogger.Println("Started")
	defer outLogger.Println("Finished")
	if err != nil {
		errLogger.Println("Error :" + err.Error())
		return
	}

	cleanedInstances := getaredis.CleanRedisInstances(ctx)
	if cleanedInstances != nil && len(cleanedInstances) > 0 {
		outLogger.Printf("Containers %v have been removed.", cleanedInstances)
	}
}

func main() {
	c := cron.New()
	c.AddFunc("@every 20m", cleanRedisInstances)
	c.AddFunc("@every 10m", monitorHosts)
	c.Start()
	<-make(chan struct{})
}
<commit_msg>Reading config files for the go jobs<commit_after>package main

import (
	"flag"
	"log"
	"os"

	"github.com/MohamedBassem/getaredis"
	"github.com/robfig/cron"
)

var configFileName *string

func monitorHosts() {
	ctx, err := getaredis.Init(*configFileName)
	errLogger := log.New(os.Stderr, "MonitorHosts", 0)
	outLogger := log.New(os.Stdout, "MonitorHosts", 0)
	outLogger.Println("Started")
	defer outLogger.Println("Finished")
	if err != nil {
		errLogger.Println("Error :" + err.Error())
		return
	}

	started, deleted, err := getaredis.MonitorHosts(ctx)
	if err != nil {
		errLogger.Println("Error :" + err.Error())
		return
	}

	if started {
		outLogger.Println("A new host has started!")
	}

	if deleted != nil && len(deleted) > 0 {
		outLogger.Printf("Hosts %v have been removed.", deleted)
	}
}

func cleanRedisInstances() {
	ctx, err := getaredis.Init(*configFileName)
	errLogger := log.New(os.Stderr, "CleanRedisInstances", 0)
	outLogger := log.New(os.Stdout, "CleanRedisInstances", 0)
	outLogger.Println("Started")
	defer outLogger.Println("Finished")
	if err != nil {
		errLogger.Println("Error :" + err.Error())
		return
	}

	cleanedInstances := getaredis.CleanRedisInstances(ctx)
	if cleanedInstances != nil && len(cleanedInstances) > 0 {
		outLogger.Printf("Containers %v have been removed.", cleanedInstances)
	}
}

func main() {
	configFileName = flag.String("config", "", "Configuration file path")
	flag.Parse()

	if *configFileName == "" {
		log.Fatal("A configuration file must be provided.")
	}
	c := cron.New()
	c.AddFunc("@every 20m", cleanRedisInstances)
	c.AddFunc("@every 10m", monitorHosts)
	c.Start()
	<-make(chan struct{})
}
<|endoftext|>


<commit_before>package restore

import (
	"archive/tar"
	"bytes"
	"errors"
	"flag"
	"fmt"
	"io"
	"net"
	"os"
	"path/filepath"
	"strconv"
	"sync"

	"github.com/influxdb/influxdb/cmd/influxd/backup"
	"github.com/influxdb/influxdb/services/meta"
	"github.com/influxdb/influxdb/snapshot"
)

// Command represents the program execution for "influxd restore".
type Command struct {
	Stdout io.Writer
	Stderr io.Writer

	backupFilesPath string
	metadir         string
	datadir         string
	database        string
	retention       string
	shard           string

	// TODO: when the new meta stuff is done this should not be exported or be gone
	MetaConfig *meta.Config
}

// NewCommand returns a new instance of Command with default settings.
func NewCommand() *Command {
	return &Command{
		Stdout:     os.Stdout,
		Stderr:     os.Stderr,
		MetaConfig: meta.NewConfig(),
	}
}

// Run executes the program.
func (cmd *Command) Run(args ...string) error {
	if err := cmd.parseFlags(args); err != nil {
		return err
	}

	if err := cmd.ensureStopped(); err != nil {
		fmt.Fprintln(cmd.Stderr, "influxd cannot be running during a restore.  Please stop any running instances and try again.")
		return err
	}

	if cmd.metadir != "" {
		if err := cmd.unpackMeta(); err != nil {
			return err
		}
	}

	if cmd.shard != "" {
		return cmd.unpackShard(cmd.shard)
	} else if cmd.retention != "" {
		return cmd.unpackRetention()
	} else if cmd.datadir != "" {
		return cmd.unpackDatabase()
	}
	return nil
}

// parseFlags parses and validates the command line arguments.
func (cmd *Command) parseFlags(args []string) error {
	fs := flag.NewFlagSet("", flag.ContinueOnError)
	fs.StringVar(&cmd.metadir, "metadir", "", "")
	fs.StringVar(&cmd.datadir, "datadir", "", "")
	fs.StringVar(&cmd.database, "database", "", "")
	fs.StringVar(&cmd.retention, "retention", "", "")
	fs.StringVar(&cmd.shard, "shard", "", "")
	fs.SetOutput(cmd.Stdout)
	fs.Usage = cmd.printUsage
	if err := fs.Parse(args); err != nil {
		return err
	}

	cmd.MetaConfig = meta.NewConfig()
	cmd.MetaConfig.Dir = cmd.metadir

	// Require output path.
	cmd.backupFilesPath = fs.Arg(0)
	if cmd.backupFilesPath == "" {
		return fmt.Errorf("path with backup files required")
	}

	// validate the arguments
	if cmd.metadir == "" && cmd.database == "" {
		return fmt.Errorf("-metadir or -database are required to restore")
	}

	if cmd.database != "" && cmd.datadir == "" {
		return fmt.Errorf("-datadir is required to restore")
	}

	if cmd.shard != "" {
		if cmd.database == "" {
			return fmt.Errorf("-database is required to restore shard")
		}
		if cmd.retention == "" {
			return fmt.Errorf("-retention is required to restore shard")
		}
	} else if cmd.retention != "" && cmd.database == "" {
		return fmt.Errorf("-database is required to restore retention policy")
	}

	return nil
}

func (cmd *Command) ensureStopped() error {
	ln, err := net.Listen("tcp", cmd.MetaConfig.BindAddress)
	if err != nil {
		return fmt.Errorf("influxd running on %s: aborting.", cmd.MetaConfig.BindAddress)
	}
	defer ln.Close()
	return nil
}

// unpackMeta reads the metadata from the backup directory and initializes a raft
// cluster and replaces the root metadata.
func (cmd *Command) unpackMeta(mr *snapshot.MultiReader, sf snapshot.File, config *Config) error {
	// find the meta file
	metaFiles, err := filepath.Glob(filepath.Join(cmd.backupFilesPath, backup.Metafile+".*"))
	if err != nil {
		return err
	}

	if len(metaFiles) == 0 {
		return fmt.Errorf("no metastore backups in %s", cmd.backupFilesPath)
	}

	latest := metaFiles[len(metaFiles)-1]

	fmt.Fprintf(cmd.Stdout, "Using metastore snapshot: %v\n", latest)
	// Read the metastore backup
	f, err := os.Open(latest)
	if err != nil {
		return err
	}

	var buf bytes.Buffer
	if _, err := io.Copy(&buf, f); err != nil {
		return fmt.Errorf("copy: %s", err)
	}

	// Unpack into metadata.
	var data meta.Data
	if err := data.UnmarshalBinary(buf.Bytes()); err != nil {
		return fmt.Errorf("unmarshal: %s", err)
	}

	// Copy meta config and remove peers so it starts in single mode.
	c := config.Meta
	c.JoinPeers = nil

	// Initialize meta store.
	store := meta.NewService(config.Meta)
	store.RaftListener = newNopListener()

	// Open the meta store.
	if err := store.Open(); err != nil {
		return fmt.Errorf("open store: %s", err)
	}
	defer store.Close()

	// Wait for the store to be ready or error.
	select {
	case err := <-store.Err():
		return err
	}

	// Force set the full metadata.
	if err := store.SetData(&data); err != nil {
		return fmt.Errorf("set data: %s", err)
	}

	fmt.Fprintln(cmd.Stdout, "Metastore restore successful")

	return nil
}

// unpackShard will look for all backup files in the path matching this shard ID
// and restore them to the data dir
func (cmd *Command) unpackShard(shardID string) error {
	// make sure the shard isn't already there so we don't clobber anything
	restorePath := filepath.Join(cmd.datadir, cmd.database, cmd.retention, shardID)
	if _, err := os.Stat(restorePath); err != nil && !os.IsNotExist(err) {
		return fmt.Errorf("shard already present: %s", restorePath)
	}

	id, err := strconv.ParseUint(shardID, 10, 64)
	if err != nil {
		return err
	}

	// find the shard backup files
	pat := filepath.Join(cmd.backupFilesPath, fmt.Sprintf(backup.BackupFilePattern, cmd.database, cmd.retention, id))
	return cmd.unpackFiles(pat + ".*")
}

// unpackDatabase will look for all backup files in the path matching this database
// and restore them to the data dir
func (cmd *Command) unpackDatabase() error {
	// make sure the shard isn't already there so we don't clobber anything
	restorePath := filepath.Join(cmd.datadir, cmd.database)
	if _, err := os.Stat(restorePath); err != nil && !os.IsNotExist(err) {
		return fmt.Errorf("database already present: %s", restorePath)
	}

	// find the database backup files
	pat := filepath.Join(cmd.backupFilesPath, cmd.database)
	return cmd.unpackFiles(pat + ".*")
}

// unpackRetention will look for all backup files in the path matching this retention
// and restore them to the data dir
func (cmd *Command) unpackRetention() error {
	// make sure the shard isn't already there so we don't clobber anything
	restorePath := filepath.Join(cmd.datadir, cmd.database, cmd.retention)
	if _, err := os.Stat(restorePath); err != nil && !os.IsNotExist(err) {
		return fmt.Errorf("retention already present: %s", restorePath)
	}

	// find the retention backup files
	pat := filepath.Join(cmd.backupFilesPath, cmd.database)
	return cmd.unpackFiles(fmt.Sprintf("%s.%s.*", pat, cmd.retention))
}

// unpackFiles will look for backup files matching the pattern and restore them to the data dir
func (cmd *Command) unpackFiles(pat string) error {
	fmt.Printf("Restoring from backup %s\n", pat)

	backupFiles, err := filepath.Glob(pat)
	if err != nil {
		return err
	}

	if len(backupFiles) == 0 {
		return fmt.Errorf("no backup files for %s in %s", pat, cmd.backupFilesPath)
	}

	for _, fn := range backupFiles {
		if err := cmd.unpackTar(fn); err != nil {
			return err
		}
	}

	return nil
}

// unpackTar will restore a single tar archive to the data dir
func (cmd *Command) unpackTar(tarFile string) error {
	f, err := os.Open(tarFile)
	if err != nil {
		return err
	}
	defer f.Close()

	tr := tar.NewReader(f)

	for {
		hdr, err := tr.Next()
		if err == io.EOF {
			return nil
		} else if err != nil {
			return err
		}

		if err := cmd.unpackFile(tr, hdr.Name); err != nil {
			return err
		}
	}
}

// unpackFile will copy the current file from the tar archive to the data dir
func (cmd *Command) unpackFile(tr *tar.Reader, fileName string) error {
	fn := filepath.Join(cmd.datadir, fileName)
	fmt.Printf("unpacking %s\n", fn)

	if err := os.MkdirAll(filepath.Dir(fn), 0777); err != nil {
		return fmt.Errorf("error making restore dir: %s", err.Error())
	}

	ff, err := os.Create(fn)
	if err != nil {
		return err
	}
	defer ff.Close()

	if _, err := io.Copy(ff, tr); err != nil {
		return err
	}

	return nil
}

// printUsage prints the usage message to STDERR.
func (cmd *Command) printUsage() {
	fmt.Fprintf(cmd.Stdout, `usage: influxd restore [flags] PATH

Restore uses backups from the PATH to restore the metastore, databases,
retention policies, or specific shards. The InfluxDB process must not be
running during restore.

Options:
  -metadir <path>
        Optional. If set the metastore will be recovered to the given path.
  -datadir <path>
        Optional. If set the restore process will recover the specified
        database, retention policy or shard to the given directory.
  -database <name>
        Optional. Required if no metadir given. Will restore the database
        TSM files.
  -retention <name>
        Optional. If given, database is required. Will restore the retention policy's
        TSM files.
  -shard <id>
    Optional. If given, database and retention are required. Will restore the shard's
    TSM files.

`)
}

type nopListener struct {
	mu      sync.Mutex
	closing chan struct{}
}

func newNopListener() *nopListener {
	return &nopListener{closing: make(chan struct{})}
}

func (ln *nopListener) Accept() (net.Conn, error) {
	ln.mu.Lock()
	defer ln.mu.Unlock()

	<-ln.closing
	return nil, errors.New("listener closing")
}

func (ln *nopListener) Close() error {
	if ln.closing != nil {
		close(ln.closing)
		ln.mu.Lock()
		defer ln.mu.Unlock()

		ln.closing = nil
	}
	return nil
}

func (ln *nopListener) Addr() net.Addr { return nil }
<commit_msg>fix build break<commit_after>package restore

import (
	"archive/tar"
	"bytes"
	"errors"
	"flag"
	"fmt"
	"io"
	"net"
	"os"
	"path/filepath"
	"strconv"
	"sync"

	"github.com/influxdb/influxdb/cmd/influxd/backup"
	"github.com/influxdb/influxdb/services/meta"
)

// Command represents the program execution for "influxd restore".
type Command struct {
	Stdout io.Writer
	Stderr io.Writer

	backupFilesPath string
	metadir         string
	datadir         string
	database        string
	retention       string
	shard           string

	// TODO: when the new meta stuff is done this should not be exported or be gone
	MetaConfig *meta.Config
}

// NewCommand returns a new instance of Command with default settings.
func NewCommand() *Command {
	return &Command{
		Stdout:     os.Stdout,
		Stderr:     os.Stderr,
		MetaConfig: meta.NewConfig(),
	}
}

// Run executes the program.
func (cmd *Command) Run(args ...string) error {
	if err := cmd.parseFlags(args); err != nil {
		return err
	}

	if err := cmd.ensureStopped(); err != nil {
		fmt.Fprintln(cmd.Stderr, "influxd cannot be running during a restore.  Please stop any running instances and try again.")
		return err
	}

	if cmd.metadir != "" {
		if err := cmd.unpackMeta(); err != nil {
			return err
		}
	}

	if cmd.shard != "" {
		return cmd.unpackShard(cmd.shard)
	} else if cmd.retention != "" {
		return cmd.unpackRetention()
	} else if cmd.datadir != "" {
		return cmd.unpackDatabase()
	}
	return nil
}

// parseFlags parses and validates the command line arguments.
func (cmd *Command) parseFlags(args []string) error {
	fs := flag.NewFlagSet("", flag.ContinueOnError)
	fs.StringVar(&cmd.metadir, "metadir", "", "")
	fs.StringVar(&cmd.datadir, "datadir", "", "")
	fs.StringVar(&cmd.database, "database", "", "")
	fs.StringVar(&cmd.retention, "retention", "", "")
	fs.StringVar(&cmd.shard, "shard", "", "")
	fs.SetOutput(cmd.Stdout)
	fs.Usage = cmd.printUsage
	if err := fs.Parse(args); err != nil {
		return err
	}

	cmd.MetaConfig = meta.NewConfig()
	cmd.MetaConfig.Dir = cmd.metadir

	// Require output path.
	cmd.backupFilesPath = fs.Arg(0)
	if cmd.backupFilesPath == "" {
		return fmt.Errorf("path with backup files required")
	}

	// validate the arguments
	if cmd.metadir == "" && cmd.database == "" {
		return fmt.Errorf("-metadir or -database are required to restore")
	}

	if cmd.database != "" && cmd.datadir == "" {
		return fmt.Errorf("-datadir is required to restore")
	}

	if cmd.shard != "" {
		if cmd.database == "" {
			return fmt.Errorf("-database is required to restore shard")
		}
		if cmd.retention == "" {
			return fmt.Errorf("-retention is required to restore shard")
		}
	} else if cmd.retention != "" && cmd.database == "" {
		return fmt.Errorf("-database is required to restore retention policy")
	}

	return nil
}

func (cmd *Command) ensureStopped() error {
	ln, err := net.Listen("tcp", cmd.MetaConfig.BindAddress)
	if err != nil {
		return fmt.Errorf("influxd running on %s: aborting.", cmd.MetaConfig.BindAddress)
	}
	defer ln.Close()
	return nil
}

// unpackMeta reads the metadata from the backup directory and initializes a raft
// cluster and replaces the root metadata.
func (cmd *Command) unpackMeta() error {
	// find the meta file
	metaFiles, err := filepath.Glob(filepath.Join(cmd.backupFilesPath, backup.Metafile+".*"))
	if err != nil {
		return err
	}

	if len(metaFiles) == 0 {
		return fmt.Errorf("no metastore backups in %s", cmd.backupFilesPath)
	}

	latest := metaFiles[len(metaFiles)-1]

	fmt.Fprintf(cmd.Stdout, "Using metastore snapshot: %v\n", latest)
	// Read the metastore backup
	f, err := os.Open(latest)
	if err != nil {
		return err
	}

	var buf bytes.Buffer
	if _, err := io.Copy(&buf, f); err != nil {
		return fmt.Errorf("copy: %s", err)
	}

	// Unpack into metadata.
	var data meta.Data
	if err := data.UnmarshalBinary(buf.Bytes()); err != nil {
		return fmt.Errorf("unmarshal: %s", err)
	}

	// Copy meta config and remove peers so it starts in single mode.
	c := cmd.MetaConfig
	c.JoinPeers = nil

	// Initialize meta store.
	store := meta.NewService(c)
	store.RaftListener = newNopListener()

	// Open the meta store.
	if err := store.Open(); err != nil {
		return fmt.Errorf("open store: %s", err)
	}
	defer store.Close()

	// Wait for the store to be ready or error.
	select {
	case err := <-store.Err():
		return err
	}

	// Force set the full metadata.
	if err := store.SetData(&data); err != nil {
		return fmt.Errorf("set data: %s", err)
	}

	fmt.Fprintln(cmd.Stdout, "Metastore restore successful")

	return nil
}

// unpackShard will look for all backup files in the path matching this shard ID
// and restore them to the data dir
func (cmd *Command) unpackShard(shardID string) error {
	// make sure the shard isn't already there so we don't clobber anything
	restorePath := filepath.Join(cmd.datadir, cmd.database, cmd.retention, shardID)
	if _, err := os.Stat(restorePath); err != nil && !os.IsNotExist(err) {
		return fmt.Errorf("shard already present: %s", restorePath)
	}

	id, err := strconv.ParseUint(shardID, 10, 64)
	if err != nil {
		return err
	}

	// find the shard backup files
	pat := filepath.Join(cmd.backupFilesPath, fmt.Sprintf(backup.BackupFilePattern, cmd.database, cmd.retention, id))
	return cmd.unpackFiles(pat + ".*")
}

// unpackDatabase will look for all backup files in the path matching this database
// and restore them to the data dir
func (cmd *Command) unpackDatabase() error {
	// make sure the shard isn't already there so we don't clobber anything
	restorePath := filepath.Join(cmd.datadir, cmd.database)
	if _, err := os.Stat(restorePath); err != nil && !os.IsNotExist(err) {
		return fmt.Errorf("database already present: %s", restorePath)
	}

	// find the database backup files
	pat := filepath.Join(cmd.backupFilesPath, cmd.database)
	return cmd.unpackFiles(pat + ".*")
}

// unpackRetention will look for all backup files in the path matching this retention
// and restore them to the data dir
func (cmd *Command) unpackRetention() error {
	// make sure the shard isn't already there so we don't clobber anything
	restorePath := filepath.Join(cmd.datadir, cmd.database, cmd.retention)
	if _, err := os.Stat(restorePath); err != nil && !os.IsNotExist(err) {
		return fmt.Errorf("retention already present: %s", restorePath)
	}

	// find the retention backup files
	pat := filepath.Join(cmd.backupFilesPath, cmd.database)
	return cmd.unpackFiles(fmt.Sprintf("%s.%s.*", pat, cmd.retention))
}

// unpackFiles will look for backup files matching the pattern and restore them to the data dir
func (cmd *Command) unpackFiles(pat string) error {
	fmt.Printf("Restoring from backup %s\n", pat)

	backupFiles, err := filepath.Glob(pat)
	if err != nil {
		return err
	}

	if len(backupFiles) == 0 {
		return fmt.Errorf("no backup files for %s in %s", pat, cmd.backupFilesPath)
	}

	for _, fn := range backupFiles {
		if err := cmd.unpackTar(fn); err != nil {
			return err
		}
	}

	return nil
}

// unpackTar will restore a single tar archive to the data dir
func (cmd *Command) unpackTar(tarFile string) error {
	f, err := os.Open(tarFile)
	if err != nil {
		return err
	}
	defer f.Close()

	tr := tar.NewReader(f)

	for {
		hdr, err := tr.Next()
		if err == io.EOF {
			return nil
		} else if err != nil {
			return err
		}

		if err := cmd.unpackFile(tr, hdr.Name); err != nil {
			return err
		}
	}
}

// unpackFile will copy the current file from the tar archive to the data dir
func (cmd *Command) unpackFile(tr *tar.Reader, fileName string) error {
	fn := filepath.Join(cmd.datadir, fileName)
	fmt.Printf("unpacking %s\n", fn)

	if err := os.MkdirAll(filepath.Dir(fn), 0777); err != nil {
		return fmt.Errorf("error making restore dir: %s", err.Error())
	}

	ff, err := os.Create(fn)
	if err != nil {
		return err
	}
	defer ff.Close()

	if _, err := io.Copy(ff, tr); err != nil {
		return err
	}

	return nil
}

// printUsage prints the usage message to STDERR.
func (cmd *Command) printUsage() {
	fmt.Fprintf(cmd.Stdout, `usage: influxd restore [flags] PATH

Restore uses backups from the PATH to restore the metastore, databases,
retention policies, or specific shards. The InfluxDB process must not be
running during restore.

Options:
  -metadir <path>
        Optional. If set the metastore will be recovered to the given path.
  -datadir <path>
        Optional. If set the restore process will recover the specified
        database, retention policy or shard to the given directory.
  -database <name>
        Optional. Required if no metadir given. Will restore the database
        TSM files.
  -retention <name>
        Optional. If given, database is required. Will restore the retention policy's
        TSM files.
  -shard <id>
    Optional. If given, database and retention are required. Will restore the shard's
    TSM files.

`)
}

type nopListener struct {
	mu      sync.Mutex
	closing chan struct{}
}

func newNopListener() *nopListener {
	return &nopListener{closing: make(chan struct{})}
}

func (ln *nopListener) Accept() (net.Conn, error) {
	ln.mu.Lock()
	defer ln.mu.Unlock()

	<-ln.closing
	return nil, errors.New("listener closing")
}

func (ln *nopListener) Close() error {
	if ln.closing != nil {
		close(ln.closing)
		ln.mu.Lock()
		defer ln.mu.Unlock()

		ln.closing = nil
	}
	return nil
}

func (ln *nopListener) Addr() net.Addr { return nil }
<|endoftext|>


<commit_before>// Copyright 2014 Canonical Ltd.
// Licensed under the AGPLv3, see LICENCE file for details.

package block_test

import (
	jc "github.com/juju/testing/checkers"
	gc "gopkg.in/check.v1"

	"github.com/juju/juju/cmd/envcmd"
	"github.com/juju/juju/cmd/juju/block"
	"github.com/juju/juju/testing"
	"strings"
)

type UnblockCommandSuite struct {
	ProtectionCommandSuite
	mockClient *block.MockBlockClient
}

func (s *UnblockCommandSuite) SetUpTest(c *gc.C) {
	s.FakeJujuHomeSuite.SetUpTest(c)
	s.mockClient = &block.MockBlockClient{}
	s.PatchValue(block.UnblockClient, func(p *block.UnblockCommand) (block.UnblockClientAPI, error) {
		return s.mockClient, nil
	})
}

var _ = gc.Suite(&UnblockCommandSuite{})

func runUnblockCommand(c *gc.C, args ...string) error {
	_, err := testing.RunCommand(c, envcmd.Wrap(&block.UnblockCommand{}), args...)
	return err
}

func (s *UnblockCommandSuite) assertRunUnblock(c *gc.C, operation string) {
	err := runUnblockCommand(c, operation)
	c.Assert(err, jc.ErrorIsNil)

	expectedOp := block.TypeFromOperation(strings.ToLower(operation))
	c.Assert(s.mockClient.BlockType, gc.DeepEquals, expectedOp)
}

func (s *UnblockCommandSuite) TestUnblockCmdNoOperation(c *gc.C) {
	s.assertErrorMatches(c, runUnblockCommand(c), `.*must specify one of.*`)
}

func (s *UnblockCommandSuite) TestUnblockCmdMoreThanOneOperation(c *gc.C) {
	s.assertErrorMatches(c, runUnblockCommand(c, "destroy-environment", "change"), `.*can only specify block type.*`)
}

func (s *UnblockCommandSuite) TestUnblockCmdOperationWithSeparator(c *gc.C) {
	s.assertErrorMatches(c, runUnblockCommand(c, "destroy-environment|"), `.*valid argument.*`)
}

func (s *UnblockCommandSuite) TestUnblockCmdUnknownJujuOperation(c *gc.C) {
	s.assertErrorMatches(c, runUnblockCommand(c, "add-machine"), `.*valid argument.*`)
}

func (s *UnblockCommandSuite) TestUnblockCmdUnknownOperation(c *gc.C) {
	s.assertErrorMatches(c, runUnblockCommand(c, "blah"), `.*valid argument.*`)
}

func (s *UnblockCommandSuite) TestUnblockCmdValidDestroyEnvOperationUpperCase(c *gc.C) {
	s.assertRunUnblock(c, "DESTROY-ENVIRONMENT")
}

func (s *UnblockCommandSuite) TestUnblockCmdValidDestroyEnvOperation(c *gc.C) {
	s.assertRunUnblock(c, "destroy-environment")
}
<commit_msg>Organised import.<commit_after>// Copyright 2014 Canonical Ltd.
// Licensed under the AGPLv3, see LICENCE file for details.

package block_test

import (
	"strings"

	jc "github.com/juju/testing/checkers"
	gc "gopkg.in/check.v1"

	"github.com/juju/juju/cmd/envcmd"
	"github.com/juju/juju/cmd/juju/block"
	"github.com/juju/juju/testing"
)

type UnblockCommandSuite struct {
	ProtectionCommandSuite
	mockClient *block.MockBlockClient
}

func (s *UnblockCommandSuite) SetUpTest(c *gc.C) {
	s.FakeJujuHomeSuite.SetUpTest(c)
	s.mockClient = &block.MockBlockClient{}
	s.PatchValue(block.UnblockClient, func(p *block.UnblockCommand) (block.UnblockClientAPI, error) {
		return s.mockClient, nil
	})
}

var _ = gc.Suite(&UnblockCommandSuite{})

func runUnblockCommand(c *gc.C, args ...string) error {
	_, err := testing.RunCommand(c, envcmd.Wrap(&block.UnblockCommand{}), args...)
	return err
}

func (s *UnblockCommandSuite) assertRunUnblock(c *gc.C, operation string) {
	err := runUnblockCommand(c, operation)
	c.Assert(err, jc.ErrorIsNil)

	expectedOp := block.TypeFromOperation(strings.ToLower(operation))
	c.Assert(s.mockClient.BlockType, gc.DeepEquals, expectedOp)
}

func (s *UnblockCommandSuite) TestUnblockCmdNoOperation(c *gc.C) {
	s.assertErrorMatches(c, runUnblockCommand(c), `.*must specify one of.*`)
}

func (s *UnblockCommandSuite) TestUnblockCmdMoreThanOneOperation(c *gc.C) {
	s.assertErrorMatches(c, runUnblockCommand(c, "destroy-environment", "change"), `.*can only specify block type.*`)
}

func (s *UnblockCommandSuite) TestUnblockCmdOperationWithSeparator(c *gc.C) {
	s.assertErrorMatches(c, runUnblockCommand(c, "destroy-environment|"), `.*valid argument.*`)
}

func (s *UnblockCommandSuite) TestUnblockCmdUnknownJujuOperation(c *gc.C) {
	s.assertErrorMatches(c, runUnblockCommand(c, "add-machine"), `.*valid argument.*`)
}

func (s *UnblockCommandSuite) TestUnblockCmdUnknownOperation(c *gc.C) {
	s.assertErrorMatches(c, runUnblockCommand(c, "blah"), `.*valid argument.*`)
}

func (s *UnblockCommandSuite) TestUnblockCmdValidDestroyEnvOperationUpperCase(c *gc.C) {
	s.assertRunUnblock(c, "DESTROY-ENVIRONMENT")
}

func (s *UnblockCommandSuite) TestUnblockCmdValidDestroyEnvOperation(c *gc.C) {
	s.assertRunUnblock(c, "destroy-environment")
}
<|endoftext|>