<commit_before>package indicators

import (
	"errors"
	"github.com/thetruetrade/gotrade"
)

// A Minus Directional Movement Indicator (MinusDm), no storage, for use in other indicators
type MinusDmWithoutStorage struct {
	*baseIndicator
	*baseFloatBounds

	// private variables
	valueAvailableAction ValueAvailableActionFloat
	periodCounter        int
	previousHigh         float64
	previousLow          float64
	previousMinusDm      float64
	timePeriod           int
}

// NewMinusDmWithoutStorage creates a Minus Directional Movement Indicator (MinusDm) without storage
func NewMinusDmWithoutStorage(timePeriod int, valueAvailableAction ValueAvailableActionFloat) (indicator *MinusDmWithoutStorage, err error) {

	// an indicator without storage MUST have a value available action
	if valueAvailableAction == nil {
		return nil, ErrValueAvailableActionIsNil
	}

	// the minimum timeperiod for this indicator is 1
	if timePeriod < 1 {
		return nil, errors.New("timePeriod is less than the minimum (1)")
	}

	// check the maximum timeperiod
	if timePeriod > MaximumLookbackPeriod {
		return nil, errors.New("timePeriod is greater than the maximum (100000)")
	}

	lookback := 1
	if timePeriod > 1 {
		lookback = timePeriod - 1
	}
	ind := MinusDmWithoutStorage{
		baseIndicator:        newBaseIndicator(lookback),
		baseFloatBounds:      newBaseFloatBounds(),
		periodCounter:        -1,
		previousMinusDm:      0.0,
		valueAvailableAction: valueAvailableAction,
		timePeriod:           timePeriod,
	}

	return &ind, nil
}

// A Minus Directional Movement Indicator (MinusDm)
type MinusDm struct {
	*MinusDmWithoutStorage

	// public variables
	Data []float64
}

// NewMinusDm creates a Minus Directional Movement Indicator (MinusDm) for online usage
func NewMinusDm(timePeriod int) (indicator *MinusDm, err error) {
	ind := MinusDm{}
	ind.MinusDmWithoutStorage, err = NewMinusDmWithoutStorage(timePeriod, func(dataItem float64, streamBarIndex int) {
		ind.Data = append(ind.Data, dataItem)
	})

	return &ind, err
}

// NewDefaultMinusDm creates a Minus Directional Movement Indicator (MinusDm) for online usage with default parameters
//	- timePeriod: 14
func NewDefaultMinusDm() (indicator *MinusDm, err error) {
	timePeriod := 14
	return NewMinusDm(timePeriod)
}

// NewMinusDmWithSrcLen creates a Minus Directional Movement Indicator (MinusDm) for offline usage
func NewMinusDmWithSrcLen(sourceLength uint, timePeriod int) (indicator *MinusDm, err error) {
	ind, err := NewMinusDm(timePeriod)

	// only initialise the storage if there is enough source data to require it
	if sourceLength-uint(ind.GetLookbackPeriod()) > 1 {
		ind.Data = make([]float64, 0, sourceLength-uint(ind.GetLookbackPeriod()))
	}

	return ind, err
}

// NewDefaultMinusDmWithSrcLen creates a Minus Directional Movement Indicator (MinusDm) for offline usage with default parameters
func NewDefaultMinusDmWithSrcLen(sourceLength uint) (indicator *MinusDm, err error) {
	ind, err := NewDefaultMinusDm()

	// only initialise the storage if there is enough source data to require it
	if sourceLength-uint(ind.GetLookbackPeriod()) > 1 {
		ind.Data = make([]float64, 0, sourceLength-uint(ind.GetLookbackPeriod()))
	}

	return ind, err
}

// NewMinusDmForStream creates a Minus Directional Movement Indicator (MinusDm) for online usage with a source data stream
func NewMinusDmForStream(priceStream gotrade.DOHLCVStreamSubscriber, timePeriod int) (indicator *MinusDm, err error) {
	ind, err := NewMinusDm(timePeriod)
	priceStream.AddTickSubscription(ind)
	return ind, err
}

// NewDefaultMinusDmForStream creates a Minus Directional Movement Indicator (MinusDm) for online usage with a source data stream
func NewDefaultMinusDmForStream(priceStream gotrade.DOHLCVStreamSubscriber) (indicator *MinusDm, err error) {
	ind, err := NewDefaultMinusDm()
	priceStream.AddTickSubscription(ind)
	return ind, err
}

// NewMinusDmForStreamWithSrcLen creates a Minus Directional Movement Indicator (MinusDm) for offline usage with a source data stream
func NewMinusDmForStreamWithSrcLen(sourceLength uint, priceStream gotrade.DOHLCVStreamSubscriber, timePeriod int) (indicator *MinusDm, err error) {
	ind, err := NewMinusDmWithSrcLen(sourceLength, timePeriod)
	priceStream.AddTickSubscription(ind)
	return ind, err
}

// NewDefaultMinusDmForStreamWithSrcLen creates a Minus Directional Movement Indicator (MinusDm) for offline usage with a source data stream
func NewDefaultMinusDmForStreamWithSrcLen(sourceLength uint, priceStream gotrade.DOHLCVStreamSubscriber) (indicator *MinusDm, err error) {
	ind, err := NewDefaultMinusDmWithSrcLen(sourceLength)
	priceStream.AddTickSubscription(ind)
	return ind, err
}

// ReceiveDOHLCVTick consumes a source data DOHLCV price tick
func (ind *MinusDmWithoutStorage) ReceiveDOHLCVTick(tickData gotrade.DOHLCV, streamBarIndex int) {
	ind.periodCounter += 1
	high := tickData.H()
	low := tickData.L()
	diffP := high - ind.previousHigh
	diffM := ind.previousLow - low

	if ind.lookbackPeriod == 1 {
		if ind.periodCounter > 0 {

			var result float64
			if (diffM > 0) && (diffP < diffM) {
				result = diffM
			} else {
				result = 0
			}

			// increment the number of results this indicator can be expected to return
			ind.dataLength += 1

			if ind.validFromBar == -1 {
				// set the streamBarIndex from which this indicator returns valid results
				ind.validFromBar = streamBarIndex
			}

			// update the maximum result value
			if result > ind.maxValue {
				ind.maxValue = result
			}

			// update the minimum result value
			if result < ind.minValue {
				ind.minValue = result
			}

			// notify of a new result value though the value available action
			ind.valueAvailableAction(result, streamBarIndex)
		}
	} else {
		if ind.periodCounter > 0 {
			if ind.periodCounter < ind.timePeriod {
				if (diffM > 0) && (diffP < diffM) {
					ind.previousMinusDm += diffM
				}

				if ind.periodCounter == ind.timePeriod-1 {

					result := ind.previousMinusDm

					// increment the number of results this indicator can be expected to return
					ind.dataLength += 1

					if ind.validFromBar == -1 {
						// set the streamBarIndex from which this indicator returns valid results
						ind.validFromBar = streamBarIndex
					}

					// update the maximum result value
					if result > ind.maxValue {
						ind.maxValue = result
					}

					// update the minimum result value
					if result < ind.minValue {
						ind.minValue = result
					}

					// notify of a new result value though the value available action
					ind.valueAvailableAction(result, streamBarIndex)

				}
			} else {
				var result float64
				if (diffM > 0) && (diffP < diffM) {
					result = ind.previousMinusDm - (ind.previousMinusDm / float64(ind.timePeriod)) + diffM
				} else {
					result = ind.previousMinusDm - (ind.previousMinusDm / float64(ind.timePeriod))
				}

				// increment the number of results this indicator can be expected to return
				ind.dataLength += 1

				if ind.validFromBar == -1 {
					// set the streamBarIndex from which this indicator returns valid results
					ind.validFromBar = streamBarIndex
				}

				// update the maximum result value
				if result > ind.maxValue {
					ind.maxValue = result
				}

				// update the minimum result value
				if result < ind.minValue {
					ind.minValue = result
				}

				// notify of a new result value though the value available action
				ind.valueAvailableAction(result, streamBarIndex)

				ind.previousMinusDm = result
			}
		}
	}

	ind.previousHigh = high
	ind.previousLow = low
}
<commit_msg>#76 Remove duplication - minusdm<commit_after>package indicators

import (
	"errors"
	"github.com/thetruetrade/gotrade"
)

// A Minus Directional Movement Indicator (MinusDm), no storage, for use in other indicators
type MinusDmWithoutStorage struct {
	*baseIndicatorWithFloatBounds

	// private variables
	periodCounter   int
	previousHigh    float64
	previousLow     float64
	previousMinusDm float64
	timePeriod      int
}

// NewMinusDmWithoutStorage creates a Minus Directional Movement Indicator (MinusDm) without storage
func NewMinusDmWithoutStorage(timePeriod int, valueAvailableAction ValueAvailableActionFloat) (indicator *MinusDmWithoutStorage, err error) {

	// an indicator without storage MUST have a value available action
	if valueAvailableAction == nil {
		return nil, ErrValueAvailableActionIsNil
	}

	// the minimum timeperiod for this indicator is 1
	if timePeriod < 1 {
		return nil, errors.New("timePeriod is less than the minimum (1)")
	}

	// check the maximum timeperiod
	if timePeriod > MaximumLookbackPeriod {
		return nil, errors.New("timePeriod is greater than the maximum (100000)")
	}

	lookback := 1
	if timePeriod > 1 {
		lookback = timePeriod - 1
	}
	ind := MinusDmWithoutStorage{
		baseIndicatorWithFloatBounds: newBaseIndicatorWithFloatBounds(lookback, valueAvailableAction),
		periodCounter:                -1,
		previousMinusDm:              0.0,
		timePeriod:                   timePeriod,
	}

	return &ind, nil
}

// A Minus Directional Movement Indicator (MinusDm)
type MinusDm struct {
	*MinusDmWithoutStorage

	// public variables
	Data []float64
}

// NewMinusDm creates a Minus Directional Movement Indicator (MinusDm) for online usage
func NewMinusDm(timePeriod int) (indicator *MinusDm, err error) {
	ind := MinusDm{}
	ind.MinusDmWithoutStorage, err = NewMinusDmWithoutStorage(timePeriod, func(dataItem float64, streamBarIndex int) {
		ind.Data = append(ind.Data, dataItem)
	})

	return &ind, err
}

// NewDefaultMinusDm creates a Minus Directional Movement Indicator (MinusDm) for online usage with default parameters
//	- timePeriod: 14
func NewDefaultMinusDm() (indicator *MinusDm, err error) {
	timePeriod := 14
	return NewMinusDm(timePeriod)
}

// NewMinusDmWithSrcLen creates a Minus Directional Movement Indicator (MinusDm) for offline usage
func NewMinusDmWithSrcLen(sourceLength uint, timePeriod int) (indicator *MinusDm, err error) {
	ind, err := NewMinusDm(timePeriod)

	// only initialise the storage if there is enough source data to require it
	if sourceLength-uint(ind.GetLookbackPeriod()) > 1 {
		ind.Data = make([]float64, 0, sourceLength-uint(ind.GetLookbackPeriod()))
	}

	return ind, err
}

// NewDefaultMinusDmWithSrcLen creates a Minus Directional Movement Indicator (MinusDm) for offline usage with default parameters
func NewDefaultMinusDmWithSrcLen(sourceLength uint) (indicator *MinusDm, err error) {
	ind, err := NewDefaultMinusDm()

	// only initialise the storage if there is enough source data to require it
	if sourceLength-uint(ind.GetLookbackPeriod()) > 1 {
		ind.Data = make([]float64, 0, sourceLength-uint(ind.GetLookbackPeriod()))
	}

	return ind, err
}

// NewMinusDmForStream creates a Minus Directional Movement Indicator (MinusDm) for online usage with a source data stream
func NewMinusDmForStream(priceStream gotrade.DOHLCVStreamSubscriber, timePeriod int) (indicator *MinusDm, err error) {
	ind, err := NewMinusDm(timePeriod)
	priceStream.AddTickSubscription(ind)
	return ind, err
}

// NewDefaultMinusDmForStream creates a Minus Directional Movement Indicator (MinusDm) for online usage with a source data stream
func NewDefaultMinusDmForStream(priceStream gotrade.DOHLCVStreamSubscriber) (indicator *MinusDm, err error) {
	ind, err := NewDefaultMinusDm()
	priceStream.AddTickSubscription(ind)
	return ind, err
}

// NewMinusDmForStreamWithSrcLen creates a Minus Directional Movement Indicator (MinusDm) for offline usage with a source data stream
func NewMinusDmForStreamWithSrcLen(sourceLength uint, priceStream gotrade.DOHLCVStreamSubscriber, timePeriod int) (indicator *MinusDm, err error) {
	ind, err := NewMinusDmWithSrcLen(sourceLength, timePeriod)
	priceStream.AddTickSubscription(ind)
	return ind, err
}

// NewDefaultMinusDmForStreamWithSrcLen creates a Minus Directional Movement Indicator (MinusDm) for offline usage with a source data stream
func NewDefaultMinusDmForStreamWithSrcLen(sourceLength uint, priceStream gotrade.DOHLCVStreamSubscriber) (indicator *MinusDm, err error) {
	ind, err := NewDefaultMinusDmWithSrcLen(sourceLength)
	priceStream.AddTickSubscription(ind)
	return ind, err
}

// ReceiveDOHLCVTick consumes a source data DOHLCV price tick
func (ind *MinusDmWithoutStorage) ReceiveDOHLCVTick(tickData gotrade.DOHLCV, streamBarIndex int) {
	ind.periodCounter += 1
	high := tickData.H()
	low := tickData.L()
	diffP := high - ind.previousHigh
	diffM := ind.previousLow - low

	if ind.lookbackPeriod == 1 {
		if ind.periodCounter > 0 {

			var result float64
			if (diffM > 0) && (diffP < diffM) {
				result = diffM
			} else {
				result = 0
			}

			ind.UpdateIndicatorWithNewValue(result, streamBarIndex)
		}
	} else {
		if ind.periodCounter > 0 {
			if ind.periodCounter < ind.timePeriod {
				if (diffM > 0) && (diffP < diffM) {
					ind.previousMinusDm += diffM
				}

				if ind.periodCounter == ind.timePeriod-1 {

					result := ind.previousMinusDm

					ind.UpdateIndicatorWithNewValue(result, streamBarIndex)

				}
			} else {
				var result float64
				if (diffM > 0) && (diffP < diffM) {
					result = ind.previousMinusDm - (ind.previousMinusDm / float64(ind.timePeriod)) + diffM
				} else {
					result = ind.previousMinusDm - (ind.previousMinusDm / float64(ind.timePeriod))
				}

				ind.UpdateIndicatorWithNewValue(result, streamBarIndex)

				ind.previousMinusDm = result
			}
		}
	}

	ind.previousHigh = high
	ind.previousLow = low
}
<|endoftext|>


<commit_before>package libnetwork

import (
	"github.com/docker/libnetwork/iptables"
	"github.com/docker/libnetwork/netlabel"
	"github.com/sirupsen/logrus"
)

const userChain = "DOCKER-USER"

func (c *controller) arrangeUserFilterRule() {
	c.Lock()

	if c.hasIPTablesEnabled() {
		arrangeUserFilterRule()
	}

	c.Unlock()

	iptables.OnReloaded(func() {
		c.Lock()

		if c.hasIPTablesEnabled() {
			arrangeUserFilterRule()
		}

		c.Unlock()
	})
}

func (c *controller) hasIPTablesEnabled() bool {
	// Locking c should be handled in the calling method.
	if c.cfg == nil || c.cfg.Daemon.DriverCfg[netlabel.GenericData] == nil {
		return false
	}

	genericData, ok := c.cfg.Daemon.DriverCfg[netlabel.GenericData]
	if !ok {
		return false
	}

	optMap := genericData.(map[string]interface{})
	enabled, ok := optMap["EnableIPTables"].(bool)
	if !ok {
		return false
	}

	return enabled
}

// This chain allow users to configure firewall policies in a way that persists
// docker operations/restarts. Docker will not delete or modify any pre-existing
// rules from the DOCKER-USER filter chain.
func arrangeUserFilterRule() {
	_, err := iptables.NewChain(userChain, iptables.Filter, false)
	if err != nil {
		logrus.Warnf("Failed to create %s chain: %v", userChain, err)
		return
	}

	if err = iptables.AddReturnRule(userChain); err != nil {
		logrus.Warnf("Failed to add the RETURN rule for %s: %v", userChain, err)
		return
	}

	err = iptables.EnsureJumpRule("FORWARD", userChain)
	if err != nil {
		logrus.Warnf("Failed to ensure the jump rule for %s: %v", userChain, err)
	}
}
<commit_msg>Revert "Merge pull request #2339 from phyber/iptables-check"<commit_after>package libnetwork

import (
	"github.com/docker/libnetwork/iptables"
	"github.com/sirupsen/logrus"
)

const userChain = "DOCKER-USER"

func (c *controller) arrangeUserFilterRule() {
	c.Lock()
	arrangeUserFilterRule()
	c.Unlock()
	iptables.OnReloaded(func() {
		c.Lock()
		arrangeUserFilterRule()
		c.Unlock()
	})
}

// This chain allow users to configure firewall policies in a way that persists
// docker operations/restarts. Docker will not delete or modify any pre-existing
// rules from the DOCKER-USER filter chain.
func arrangeUserFilterRule() {
	_, err := iptables.NewChain(userChain, iptables.Filter, false)
	if err != nil {
		logrus.Warnf("Failed to create %s chain: %v", userChain, err)
		return
	}

	if err = iptables.AddReturnRule(userChain); err != nil {
		logrus.Warnf("Failed to add the RETURN rule for %s: %v", userChain, err)
		return
	}

	err = iptables.EnsureJumpRule("FORWARD", userChain)
	if err != nil {
		logrus.Warnf("Failed to ensure the jump rule for %s: %v", userChain, err)
	}
}
<|endoftext|>


<commit_before>package MySQLProtocol
<commit_msg>Test empty statement exec<commit_after>package MySQLProtocol

import "testing"
import "github.com/stretchr/testify/assert"

var COM_STMT_EXECUTE_test_packets = []struct {
	packet  Proto
	context Context
}{

	{packet: Proto{data: StringToPacket(`
0a 00 00 00 00 00 00 00 00 00 00 00
00 00
`)}, context: Context{}},
}

func Test_Packet_COM_STMT_EXECUTE(t *testing.T) {
	var pkt Packet_COM_STMT_EXECUTE
	for _, value := range COM_STMT_EXECUTE_test_packets {
		pkt = Packet_COM_STMT_EXECUTE{}
		pkt.FromPacket(value.context, value.packet)
		assert.Equal(t, pkt.ToPacket(value.context), value.packet.data, "")
	}
}

func Benchmark_Packet_COM_STMT_EXECUTE_FromPacket(b *testing.B) {
	context := COM_STMT_EXECUTE_test_packets[0].context
	packet := COM_STMT_EXECUTE_test_packets[0].packet
	pkt := Packet_COM_STMT_EXECUTE{}
	b.ResetTimer()
	for i := 0; i < b.N; i++ {
		packet.offset = 0
		pkt.FromPacket(context, packet)
	}
}

func Benchmark_Packet_COM_STMT_EXECUTE_GetPacketSize(b *testing.B) {
	context := COM_STMT_EXECUTE_test_packets[0].context
	pkt := Packet_COM_STMT_EXECUTE{}
	pkt.FromPacket(context, COM_STMT_EXECUTE_test_packets[0].packet)
	b.ResetTimer()
	for i := 0; i < b.N; i++ {
		pkt.GetPacketSize(context)
	}
}

func Benchmark_Packet_COM_STMT_EXECUTE_ToPacket(b *testing.B) {
	context := COM_STMT_EXECUTE_test_packets[0].context
	pkt := Packet_COM_STMT_EXECUTE{}
	pkt.FromPacket(context, COM_STMT_EXECUTE_test_packets[0].packet)
	b.ResetTimer()
	for i := 0; i < b.N; i++ {
		pkt.ToPacket(context)
	}
}
<|endoftext|>


<commit_before>package workflow

import (
	"database/sql"
	"fmt"
	"time"

	"github.com/go-gorp/gorp"

	"github.com/ovh/cds/engine/api/application"
	"github.com/ovh/cds/engine/api/cache"
	"github.com/ovh/cds/engine/api/environment"
	"github.com/ovh/cds/engine/api/event"
	"github.com/ovh/cds/engine/api/project"
	"github.com/ovh/cds/engine/api/secret"
	"github.com/ovh/cds/sdk"
	"github.com/ovh/cds/sdk/log"
)

// UpdateNodeJobRunStatus Update status of an workflow_node_run_job
func UpdateNodeJobRunStatus(db gorp.SqlExecutor, store cache.Store, job *sdk.WorkflowNodeJobRun, status sdk.Status) error {
	log.Debug("UpdateNodeJobRunStatus> job.ID=%d status=%s", job.ID, status.String())

	var query string
	query = `SELECT status FROM workflow_node_run_job WHERE id = $1 FOR UPDATE`
	var currentStatus string
	if err := db.QueryRow(query, job.ID).Scan(&currentStatus); err != nil {
		return sdk.WrapError(err, "workflow.UpdateNodeJobRunStatus> Cannot lock node job run %d: %s", job.ID, err)
	}

	switch status {
	case sdk.StatusBuilding:
		if currentStatus != sdk.StatusWaiting.String() {
			return fmt.Errorf("workflow.UpdateNodeJobRunStatus> Cannot update status of WorkflowNodeJobRun %d to %s, expected current status %s, got %s",
				job.ID, status, sdk.StatusWaiting, currentStatus)
		}
		job.Start = time.Now()
		job.Status = status.String()

	case sdk.StatusFail, sdk.StatusSuccess, sdk.StatusDisabled, sdk.StatusSkipped:
		if currentStatus != string(sdk.StatusWaiting) && currentStatus != string(sdk.StatusBuilding) && status != sdk.StatusDisabled && status != sdk.StatusSkipped {
			log.Debug("workflow.UpdateNodeJobRunStatus> Status is %s, cannot update %d to %s", currentStatus, job.ID, status)
			// too late, Nate
			return nil
		}
		job.Done = time.Now()
		job.Status = status.String()
	default:
		return fmt.Errorf("workflow.UpdateNodeJobRunStatus> Cannot update WorkflowNodeJobRun %d to status %v", job.ID, status.String())
	}

	node, errLoad := LoadNodeRunByID(db, job.WorkflowNodeRunID)
	if errLoad != nil {
		return errLoad
	}

	//If the job has been set to building, set the stage to building
	var stageUpdated bool
	if job.Status == sdk.StatusBuilding.String() {
		log.Debug("UpdateNodeJobRunStatus> job:%d", job.ID)
		for i := range node.Stages {
			s := &node.Stages[i]
			var found bool
			//Find the right stage
			for _, j := range s.Jobs {
				if j.Action.ID == job.Job.Job.Action.ID {
					found = true
					break
				}
			}
			if found && s.Status == sdk.StatusWaiting {
				log.Debug("UpdateNodeJobRunStatus> stage:%s status from %s to %s", s.Name, s.Status, sdk.StatusBuilding)
				s.Status = sdk.StatusBuilding
				stageUpdated = true
				break
			}
		}
	}

	if stageUpdated {
		log.Debug("UpdateNodeJobRunStatus> stageUpdated, set status node from %s to %s", node.Status, sdk.StatusBuilding.String())
		node.Status = sdk.StatusBuilding.String()
		if err := UpdateNodeRun(db, node); err != nil {
			return sdk.WrapError(err, "workflow.UpdateNodeJobRunStatus> Unable to update workflow node run %d", node.ID)
		}
	} else {
		log.Debug("UpdateNodeJobRunStatus> call execute node")
		if errE := execute(db, store, node); errE != nil {
			return sdk.WrapError(errE, "workflow.UpdateNodeJobRunStatus> Cannot execute sync node")
		}
	}

	if err := UpdateNodeJobRun(db, store, job); err != nil {
		return sdk.WrapError(err, "workflow.UpdateNodeJobRunStatus> Cannot update WorkflowNodeJobRun %d", job.ID)
	}

	event.PublishJobRun(node, job)

	return nil
}

// AddSpawnInfosNodeJobRun saves spawn info before starting worker
func AddSpawnInfosNodeJobRun(db gorp.SqlExecutor, store cache.Store, id int64, infos []sdk.SpawnInfo) (*sdk.WorkflowNodeJobRun, error) {
	j, err := LoadAndLockNodeJobRun(db, store, id)
	if err != nil {
		return nil, sdk.WrapError(err, "AddSpawnInfosNodeJobRun> Cannot load node job run")
	}
	if err := prepareSpawnInfos(j, infos); err != nil {
		return nil, sdk.WrapError(err, "AddSpawnInfosNodeJobRun> Cannot prepare spawn infos")
	}

	if err := UpdateNodeJobRun(db, store, j); err != nil {
		return nil, sdk.WrapError(err, "AddSpawnInfosNodeJobRun> Cannot update node job run")
	}
	return j, nil
}

func prepareSpawnInfos(j *sdk.WorkflowNodeJobRun, infos []sdk.SpawnInfo) error {
	now := time.Now()
	for _, info := range infos {
		j.SpawnInfos = append(j.SpawnInfos, sdk.SpawnInfo{
			APITime:    now,
			RemoteTime: info.RemoteTime,
			Message:    info.Message,
		})
	}
	return nil
}

// TakeNodeJobRun Take an a job run for update
func TakeNodeJobRun(db gorp.SqlExecutor, store cache.Store, id int64, workerModel string, workerName string, workerID string, infos []sdk.SpawnInfo) (*sdk.WorkflowNodeJobRun, error) {
	job, err := LoadAndLockNodeJobRun(db, store, id)
	if err != nil {
		return nil, sdk.WrapError(err, "TakeNodeJobRun> Cannot load node job run")
	}
	if job.Status != sdk.StatusWaiting.String() {
		k := keyBookJob(id)
		h := sdk.Hatchery{}
		if store.Get(k, &h) {
			return nil, sdk.WrapError(sdk.ErrAlreadyTaken, "TakeNodeJobRun> job %d is not waiting status and was booked by hatchery %d. Current status:%s", id, h.ID, job.Status)
		}
		return nil, sdk.WrapError(sdk.ErrAlreadyTaken, "TakeNodeJobRun> job %d is not waiting status. Current status:%s", id, job.Status)
	}

	job.Model = workerModel
	job.Job.WorkerName = workerName
	job.Job.WorkerID = workerID
	job.Start = time.Now()

	if err := prepareSpawnInfos(job, infos); err != nil {
		return nil, sdk.WrapError(err, "TakeNodeJobRun> Cannot prepare spawn infos")
	}

	if err := UpdateNodeJobRunStatus(db, store, job, sdk.StatusBuilding); err != nil {
		log.Debug("TakeNodeJobRun> call UpdateNodeJobRunStatus on job %d set status from %s to %s", job.ID, job.Status, sdk.StatusBuilding)
		return nil, sdk.WrapError(err, "TakeNodeJobRun>Cannot update node job run")
	}

	return job, nil
}

// LoadNodeJobRunKeys loads all keys for a job run
func LoadNodeJobRunKeys(db gorp.SqlExecutor, store cache.Store, job *sdk.WorkflowNodeJobRun, nodeRun *sdk.WorkflowNodeRun, w *sdk.WorkflowRun) ([]sdk.Parameter, []sdk.Variable, error) {
	params := []sdk.Parameter{}
	secrets := []sdk.Variable{}

	p, errP := project.LoadByID(db, store, w.Workflow.ProjectID, nil, project.LoadOptions.WithKeys)
	if errP != nil {
		return nil, nil, sdk.WrapError(errP, "LoadNodeJobRunKeys> Cannot load project keys")
	}
	for _, k := range p.Keys {
		params = append(params, sdk.Parameter{
			Name:  "cds.proj." + k.Name + ".pub",
			Type:  "string",
			Value: k.Public,
		})
		params = append(params, sdk.Parameter{
			Name:  "cds.proj." + k.Name + ".id",
			Type:  "string",
			Value: k.KeyID,
		})
		secrets = append(secrets, sdk.Variable{
			Name:  "cds.proj." + k.Name + ".priv",
			Type:  "string",
			Value: k.Private,
		})
	}

	//Load node definition
	n := w.Workflow.GetNode(nodeRun.WorkflowNodeID)
	if n == nil {
		return nil, nil, sdk.WrapError(fmt.Errorf("Unable to find node %d in workflow", nodeRun.WorkflowNodeID), "LoadNodeJobRunSecrets>")
	}
	if n.Context != nil && n.Context.Application != nil {
		a, errA := application.LoadByID(db, store, n.Context.Application.ID, nil, application.LoadOptions.WithKeys)
		if errA != nil {
			return nil, nil, sdk.WrapError(errA, "loadActionBuildKeys> Cannot load application keys")
		}
		for _, k := range a.Keys {
			params = append(params, sdk.Parameter{
				Name:  "cds.app." + k.Name + ".pub",
				Type:  "string",
				Value: k.Public,
			})
			params = append(params, sdk.Parameter{
				Name:  "cds.app." + k.Name + ".id",
				Type:  "string",
				Value: k.KeyID,
			})
			secrets = append(secrets, sdk.Variable{
				Name:  "cds.app." + k.Name + ".priv",
				Type:  "string",
				Value: k.Private,
			})
		}
	}

	if n.Context != nil && n.Context.Environment != nil && n.Context.Environment.ID != sdk.DefaultEnv.ID {
		e, errE := environment.LoadEnvironmentByID(db, n.Context.Environment.ID)
		if errE != nil {
			return nil, nil, sdk.WrapError(errE, "loadActionBuildKeys> Cannot load environment keys")
		}
		for _, k := range e.Keys {
			params = append(params, sdk.Parameter{
				Name:  "cds.env." + k.Name + ".pub",
				Type:  "string",
				Value: k.Public,
			})
			params = append(params, sdk.Parameter{
				Name:  "cds.env." + k.Name + ".id",
				Type:  "string",
				Value: k.KeyID,
			})
			secrets = append(secrets, sdk.Variable{
				Name:  "cds.env." + k.Name + ".priv",
				Type:  "string",
				Value: k.Private,
			})
		}

	}
	return params, secrets, nil
}

// LoadNodeJobRunSecrets loads all secrets for a job run
func LoadNodeJobRunSecrets(db gorp.SqlExecutor, job *sdk.WorkflowNodeJobRun, nodeRun *sdk.WorkflowNodeRun, w *sdk.WorkflowRun) ([]sdk.Variable, error) {
	var secrets []sdk.Variable

	// Load project secrets
	pv, err := project.GetAllVariableInProject(db, w.Workflow.ProjectID, project.WithClearPassword())
	if err != nil {
		return nil, err
	}
	pv = sdk.VariablesFilter(pv, sdk.SecretVariable, sdk.KeyVariable)
	pv = sdk.VariablesPrefix(pv, "cds.proj")
	secrets = append(secrets, pv...)

	//Load node definition
	n := w.Workflow.GetNode(nodeRun.WorkflowNodeID)
	if n == nil {
		return nil, sdk.WrapError(fmt.Errorf("Unable to find node %d in workflow", nodeRun.WorkflowNodeID), "LoadNodeJobRunSecrets>")
	}

	//Application variables
	av := []sdk.Variable{}
	if n.Context != nil && n.Context.Application != nil {
		av = sdk.VariablesFilter(n.Context.Application.Variable, sdk.SecretVariable, sdk.KeyVariable)
		av = sdk.VariablesPrefix(av, "cds.app")
	}
	secrets = append(secrets, av...)

	//Environment variables
	ev := []sdk.Variable{}
	if n.Context != nil && n.Context.Environment != nil {
		ev = sdk.VariablesFilter(n.Context.Environment.Variable, sdk.SecretVariable, sdk.KeyVariable)
		ev = sdk.VariablesPrefix(ev, "cds.env")
	}
	secrets = append(secrets, ev...)

	//Decrypt secrets
	for i := range secrets {
		s := &secrets[i]
		if err := secret.DecryptVariable(s); err != nil {
			return nil, sdk.WrapError(err, "LoadNodeJobRunSecrets> Unable to decrypt variables")
		}
	}

	return secrets, nil
}

//BookNodeJobRun  Book a job for a hatchery
func BookNodeJobRun(store cache.Store, id int64, hatchery *sdk.Hatchery) (*sdk.Hatchery, error) {
	k := keyBookJob(id)
	h := sdk.Hatchery{}
	if !store.Get(k, &h) {
		// job not already booked, book it for 2 min
		store.SetWithTTL(k, hatchery, 120)
		return nil, nil
	}
	return &h, sdk.WrapError(sdk.ErrJobAlreadyBooked, "BookNodeJobRun> job %d already booked by %s (%d)", id, h.Name, h.ID)
}

//AddLog adds a build log
func AddLog(db gorp.SqlExecutor, job *sdk.WorkflowNodeJobRun, logs *sdk.Log) error {
	if job != nil {
		logs.PipelineBuildJobID = job.ID
		logs.PipelineBuildID = job.WorkflowNodeRunID
	}

	existingLogs, errLog := LoadStepLogs(db, logs.PipelineBuildJobID, logs.StepOrder)
	if errLog != nil && errLog != sql.ErrNoRows {
		return sdk.WrapError(errLog, "AddLog> Cannot load existing logs")
	}

	if existingLogs == nil {
		if err := insertLog(db, logs); err != nil {
			return sdk.WrapError(err, "AddLog> Cannot insert log")
		}
	} else {
		existingLogs.Val += logs.Val
		existingLogs.LastModified = logs.LastModified
		existingLogs.Done = logs.Done
		if err := updateLog(db, existingLogs); err != nil {
			return sdk.WrapError(err, "AddLog> Cannot update log")
		}
	}
	return nil
}
<commit_msg>fix (api): workflow var prefix (#1093)<commit_after>package workflow

import (
	"database/sql"
	"fmt"
	"time"

	"github.com/go-gorp/gorp"

	"github.com/ovh/cds/engine/api/application"
	"github.com/ovh/cds/engine/api/cache"
	"github.com/ovh/cds/engine/api/environment"
	"github.com/ovh/cds/engine/api/event"
	"github.com/ovh/cds/engine/api/project"
	"github.com/ovh/cds/engine/api/secret"
	"github.com/ovh/cds/sdk"
	"github.com/ovh/cds/sdk/log"
)

// UpdateNodeJobRunStatus Update status of an workflow_node_run_job
func UpdateNodeJobRunStatus(db gorp.SqlExecutor, store cache.Store, job *sdk.WorkflowNodeJobRun, status sdk.Status) error {
	log.Debug("UpdateNodeJobRunStatus> job.ID=%d status=%s", job.ID, status.String())

	var query string
	query = `SELECT status FROM workflow_node_run_job WHERE id = $1 FOR UPDATE`
	var currentStatus string
	if err := db.QueryRow(query, job.ID).Scan(&currentStatus); err != nil {
		return sdk.WrapError(err, "workflow.UpdateNodeJobRunStatus> Cannot lock node job run %d: %s", job.ID, err)
	}

	switch status {
	case sdk.StatusBuilding:
		if currentStatus != sdk.StatusWaiting.String() {
			return fmt.Errorf("workflow.UpdateNodeJobRunStatus> Cannot update status of WorkflowNodeJobRun %d to %s, expected current status %s, got %s",
				job.ID, status, sdk.StatusWaiting, currentStatus)
		}
		job.Start = time.Now()
		job.Status = status.String()

	case sdk.StatusFail, sdk.StatusSuccess, sdk.StatusDisabled, sdk.StatusSkipped:
		if currentStatus != string(sdk.StatusWaiting) && currentStatus != string(sdk.StatusBuilding) && status != sdk.StatusDisabled && status != sdk.StatusSkipped {
			log.Debug("workflow.UpdateNodeJobRunStatus> Status is %s, cannot update %d to %s", currentStatus, job.ID, status)
			// too late, Nate
			return nil
		}
		job.Done = time.Now()
		job.Status = status.String()
	default:
		return fmt.Errorf("workflow.UpdateNodeJobRunStatus> Cannot update WorkflowNodeJobRun %d to status %v", job.ID, status.String())
	}

	node, errLoad := LoadNodeRunByID(db, job.WorkflowNodeRunID)
	if errLoad != nil {
		return errLoad
	}

	//If the job has been set to building, set the stage to building
	var stageUpdated bool
	if job.Status == sdk.StatusBuilding.String() {
		log.Debug("UpdateNodeJobRunStatus> job:%d", job.ID)
		for i := range node.Stages {
			s := &node.Stages[i]
			var found bool
			//Find the right stage
			for _, j := range s.Jobs {
				if j.Action.ID == job.Job.Job.Action.ID {
					found = true
					break
				}
			}
			if found && s.Status == sdk.StatusWaiting {
				log.Debug("UpdateNodeJobRunStatus> stage:%s status from %s to %s", s.Name, s.Status, sdk.StatusBuilding)
				s.Status = sdk.StatusBuilding
				stageUpdated = true
				break
			}
		}
	}

	if stageUpdated {
		log.Debug("UpdateNodeJobRunStatus> stageUpdated, set status node from %s to %s", node.Status, sdk.StatusBuilding.String())
		node.Status = sdk.StatusBuilding.String()
		if err := UpdateNodeRun(db, node); err != nil {
			return sdk.WrapError(err, "workflow.UpdateNodeJobRunStatus> Unable to update workflow node run %d", node.ID)
		}
	} else {
		log.Debug("UpdateNodeJobRunStatus> call execute node")
		if errE := execute(db, store, node); errE != nil {
			return sdk.WrapError(errE, "workflow.UpdateNodeJobRunStatus> Cannot execute sync node")
		}
	}

	if err := UpdateNodeJobRun(db, store, job); err != nil {
		return sdk.WrapError(err, "workflow.UpdateNodeJobRunStatus> Cannot update WorkflowNodeJobRun %d", job.ID)
	}

	event.PublishJobRun(node, job)

	return nil
}

// AddSpawnInfosNodeJobRun saves spawn info before starting worker
func AddSpawnInfosNodeJobRun(db gorp.SqlExecutor, store cache.Store, id int64, infos []sdk.SpawnInfo) (*sdk.WorkflowNodeJobRun, error) {
	j, err := LoadAndLockNodeJobRun(db, store, id)
	if err != nil {
		return nil, sdk.WrapError(err, "AddSpawnInfosNodeJobRun> Cannot load node job run")
	}
	if err := prepareSpawnInfos(j, infos); err != nil {
		return nil, sdk.WrapError(err, "AddSpawnInfosNodeJobRun> Cannot prepare spawn infos")
	}

	if err := UpdateNodeJobRun(db, store, j); err != nil {
		return nil, sdk.WrapError(err, "AddSpawnInfosNodeJobRun> Cannot update node job run")
	}
	return j, nil
}

func prepareSpawnInfos(j *sdk.WorkflowNodeJobRun, infos []sdk.SpawnInfo) error {
	now := time.Now()
	for _, info := range infos {
		j.SpawnInfos = append(j.SpawnInfos, sdk.SpawnInfo{
			APITime:    now,
			RemoteTime: info.RemoteTime,
			Message:    info.Message,
		})
	}
	return nil
}

// TakeNodeJobRun Take an a job run for update
func TakeNodeJobRun(db gorp.SqlExecutor, store cache.Store, id int64, workerModel string, workerName string, workerID string, infos []sdk.SpawnInfo) (*sdk.WorkflowNodeJobRun, error) {
	job, err := LoadAndLockNodeJobRun(db, store, id)
	if err != nil {
		return nil, sdk.WrapError(err, "TakeNodeJobRun> Cannot load node job run")
	}
	if job.Status != sdk.StatusWaiting.String() {
		k := keyBookJob(id)
		h := sdk.Hatchery{}
		if store.Get(k, &h) {
			return nil, sdk.WrapError(sdk.ErrAlreadyTaken, "TakeNodeJobRun> job %d is not waiting status and was booked by hatchery %d. Current status:%s", id, h.ID, job.Status)
		}
		return nil, sdk.WrapError(sdk.ErrAlreadyTaken, "TakeNodeJobRun> job %d is not waiting status. Current status:%s", id, job.Status)
	}

	job.Model = workerModel
	job.Job.WorkerName = workerName
	job.Job.WorkerID = workerID
	job.Start = time.Now()

	if err := prepareSpawnInfos(job, infos); err != nil {
		return nil, sdk.WrapError(err, "TakeNodeJobRun> Cannot prepare spawn infos")
	}

	if err := UpdateNodeJobRunStatus(db, store, job, sdk.StatusBuilding); err != nil {
		log.Debug("TakeNodeJobRun> call UpdateNodeJobRunStatus on job %d set status from %s to %s", job.ID, job.Status, sdk.StatusBuilding)
		return nil, sdk.WrapError(err, "TakeNodeJobRun>Cannot update node job run")
	}

	return job, nil
}

// LoadNodeJobRunKeys loads all keys for a job run
func LoadNodeJobRunKeys(db gorp.SqlExecutor, store cache.Store, job *sdk.WorkflowNodeJobRun, nodeRun *sdk.WorkflowNodeRun, w *sdk.WorkflowRun) ([]sdk.Parameter, []sdk.Variable, error) {
	params := []sdk.Parameter{}
	secrets := []sdk.Variable{}

	p, errP := project.LoadByID(db, store, w.Workflow.ProjectID, nil, project.LoadOptions.WithKeys)
	if errP != nil {
		return nil, nil, sdk.WrapError(errP, "LoadNodeJobRunKeys> Cannot load project keys")
	}
	for _, k := range p.Keys {
		params = append(params, sdk.Parameter{
			Name:  "cds.proj." + k.Name + ".pub",
			Type:  "string",
			Value: k.Public,
		})
		params = append(params, sdk.Parameter{
			Name:  "cds.proj." + k.Name + ".id",
			Type:  "string",
			Value: k.KeyID,
		})
		secrets = append(secrets, sdk.Variable{
			Name:  "cds.proj." + k.Name + ".priv",
			Type:  "string",
			Value: k.Private,
		})
	}

	//Load node definition
	n := w.Workflow.GetNode(nodeRun.WorkflowNodeID)
	if n == nil {
		return nil, nil, sdk.WrapError(fmt.Errorf("Unable to find node %d in workflow", nodeRun.WorkflowNodeID), "LoadNodeJobRunSecrets>")
	}
	if n.Context != nil && n.Context.Application != nil {
		a, errA := application.LoadByID(db, store, n.Context.Application.ID, nil, application.LoadOptions.WithKeys)
		if errA != nil {
			return nil, nil, sdk.WrapError(errA, "loadActionBuildKeys> Cannot load application keys")
		}
		for _, k := range a.Keys {
			params = append(params, sdk.Parameter{
				Name:  "cds.app." + k.Name + ".pub",
				Type:  "string",
				Value: k.Public,
			})
			params = append(params, sdk.Parameter{
				Name:  "cds.app." + k.Name + ".id",
				Type:  "string",
				Value: k.KeyID,
			})
			secrets = append(secrets, sdk.Variable{
				Name:  "cds.app." + k.Name + ".priv",
				Type:  "string",
				Value: k.Private,
			})
		}
	}

	if n.Context != nil && n.Context.Environment != nil && n.Context.Environment.ID != sdk.DefaultEnv.ID {
		e, errE := environment.LoadEnvironmentByID(db, n.Context.Environment.ID)
		if errE != nil {
			return nil, nil, sdk.WrapError(errE, "loadActionBuildKeys> Cannot load environment keys")
		}
		for _, k := range e.Keys {
			params = append(params, sdk.Parameter{
				Name:  "cds.env." + k.Name + ".pub",
				Type:  "string",
				Value: k.Public,
			})
			params = append(params, sdk.Parameter{
				Name:  "cds.env." + k.Name + ".id",
				Type:  "string",
				Value: k.KeyID,
			})
			secrets = append(secrets, sdk.Variable{
				Name:  "cds.env." + k.Name + ".priv",
				Type:  "string",
				Value: k.Private,
			})
		}

	}
	return params, secrets, nil
}

// LoadNodeJobRunSecrets loads all secrets for a job run
func LoadNodeJobRunSecrets(db gorp.SqlExecutor, job *sdk.WorkflowNodeJobRun, nodeRun *sdk.WorkflowNodeRun, w *sdk.WorkflowRun) ([]sdk.Variable, error) {
	var secrets []sdk.Variable

	// Load project secrets
	pv, err := project.GetAllVariableInProject(db, w.Workflow.ProjectID, project.WithClearPassword())
	if err != nil {
		return nil, err
	}
	pv = sdk.VariablesFilter(pv, sdk.SecretVariable, sdk.KeyVariable)
	pv = sdk.VariablesPrefix(pv, "cds.proj.")
	secrets = append(secrets, pv...)

	//Load node definition
	n := w.Workflow.GetNode(nodeRun.WorkflowNodeID)
	if n == nil {
		return nil, sdk.WrapError(fmt.Errorf("Unable to find node %d in workflow", nodeRun.WorkflowNodeID), "LoadNodeJobRunSecrets>")
	}

	//Application variables
	av := []sdk.Variable{}
	if n.Context != nil && n.Context.Application != nil {
		av = sdk.VariablesFilter(n.Context.Application.Variable, sdk.SecretVariable, sdk.KeyVariable)
		av = sdk.VariablesPrefix(av, "cds.app.")
	}
	secrets = append(secrets, av...)

	//Environment variables
	ev := []sdk.Variable{}
	if n.Context != nil && n.Context.Environment != nil {
		ev = sdk.VariablesFilter(n.Context.Environment.Variable, sdk.SecretVariable, sdk.KeyVariable)
		ev = sdk.VariablesPrefix(ev, "cds.env.")
	}
	secrets = append(secrets, ev...)

	//Decrypt secrets
	for i := range secrets {
		s := &secrets[i]
		if err := secret.DecryptVariable(s); err != nil {
			return nil, sdk.WrapError(err, "LoadNodeJobRunSecrets> Unable to decrypt variables")
		}
	}
	return secrets, nil
}

//BookNodeJobRun  Book a job for a hatchery
func BookNodeJobRun(store cache.Store, id int64, hatchery *sdk.Hatchery) (*sdk.Hatchery, error) {
	k := keyBookJob(id)
	h := sdk.Hatchery{}
	if !store.Get(k, &h) {
		// job not already booked, book it for 2 min
		store.SetWithTTL(k, hatchery, 120)
		return nil, nil
	}
	return &h, sdk.WrapError(sdk.ErrJobAlreadyBooked, "BookNodeJobRun> job %d already booked by %s (%d)", id, h.Name, h.ID)
}

//AddLog adds a build log
func AddLog(db gorp.SqlExecutor, job *sdk.WorkflowNodeJobRun, logs *sdk.Log) error {
	if job != nil {
		logs.PipelineBuildJobID = job.ID
		logs.PipelineBuildID = job.WorkflowNodeRunID
	}

	existingLogs, errLog := LoadStepLogs(db, logs.PipelineBuildJobID, logs.StepOrder)
	if errLog != nil && errLog != sql.ErrNoRows {
		return sdk.WrapError(errLog, "AddLog> Cannot load existing logs")
	}

	if existingLogs == nil {
		if err := insertLog(db, logs); err != nil {
			return sdk.WrapError(err, "AddLog> Cannot insert log")
		}
	} else {
		existingLogs.Val += logs.Val
		existingLogs.LastModified = logs.LastModified
		existingLogs.Done = logs.Done
		if err := updateLog(db, existingLogs); err != nil {
			return sdk.WrapError(err, "AddLog> Cannot update log")
		}
	}
	return nil
}
<|endoftext|>


<commit_before>// Copyright (c) Microsoft Open Technologies, Inc.
// All Rights Reserved.
// Licensed under the Apache License, Version 2.0.
// See License.txt in the project root for license information.

package request

import (
	"bytes"
	"fmt"
)

func (m *Manager) CreateVirtualMachineDeploymentLin(isOSImage bool, serviceName, vmName, vmSize, certThumbprint, userName, osImageName, mediaLoc string) *Data {

	uri := fmt.Sprintf("https://management.core.windows.net/%s/services/hostedservices/%s/deployments", m.SubscrId, serviceName)

	var buff bytes.Buffer
	buff.WriteString("<Deployment xmlns:i='http://www.w3.org/2001/XMLSchema-instance' xmlns='http://schemas.microsoft.com/windowsazure'>")
	buff.WriteString("<Name>" + vmName + "</Name>")
	buff.WriteString("<DeploymentSlot>Production</DeploymentSlot>")
	buff.WriteString("" + vmName + "")
	buff.WriteString("<RoleList>")
	buff.WriteString("<Role i:type='PersistentVMRole'>")
	buff.WriteString("<RoleName>" + vmName + "</RoleName>")
	buff.WriteString("<RoleType>PersistentVMRole</RoleType>")
	buff.WriteString("<ConfigurationSets>")
	buff.WriteString("<ConfigurationSet i:type='LinuxProvisioningConfigurationSet'>")
	buff.WriteString("<ConfigurationSetType>LinuxProvisioningConfiguration</ConfigurationSetType>")
	buff.WriteString("<HostName>" + vmName + "</HostName>")
	buff.WriteString("<UserName>" + userName + "</UserName>")
	buff.WriteString("<DisableSshPasswordAuthentication>true</DisableSshPasswordAuthentication>")
	buff.WriteString("<SSH>")
	buff.WriteString("<PublicKeys>")
	buff.WriteString("<PublicKey>")
	buff.WriteString("<Fingerprint>" + certThumbprint + "</Fingerprint>")
	buff.WriteString("<Path>/home/" + userName + "/.ssh/authorized_keys</Path>")
	buff.WriteString("</PublicKey>")
	buff.WriteString("</PublicKeys>")
	buff.WriteString("</SSH>")
	buff.WriteString("</ConfigurationSet>")
	buff.WriteString("<ConfigurationSet i:type='NetworkConfigurationSet'>")
	buff.WriteString("<ConfigurationSetType>NetworkConfiguration</ConfigurationSetType>")
	buff.WriteString("<InputEndpoints>")
	buff.WriteString("<InputEndpoint>")
	buff.WriteString("<LocalPort>22</LocalPort>")
	buff.WriteString("<Name>SSH</Name>")
	buff.WriteString("<Port>22</Port>")
	buff.WriteString("<Protocol>tcp</Protocol>")
	buff.WriteString("</InputEndpoint>")
	buff.WriteString("</InputEndpoints>")
	buff.WriteString("</ConfigurationSet>")
	buff.WriteString("</ConfigurationSets>")
	if !isOSImage {
		buff.WriteString("<VMImageName>" + osImageName + "</VMImageName>")
	} else {
		buff.WriteString("<OSVirtualHardDisk>")
		buff.WriteString("<MediaLink>" + mediaLoc + "</MediaLink>")
		buff.WriteString("<SourceImageName>" + osImageName + "</SourceImageName>")
		buff.WriteString("</OSVirtualHardDisk>")
	}
	buff.WriteString("<RoleSize>" + vmSize + "</RoleSize>")
	buff.WriteString("<ProvisionGuestAgent>true</ProvisionGuestAgent>")
	buff.WriteString("</Role>")
	buff.WriteString("</RoleList>")
	buff.WriteString("</Deployment>")

	data := &Data{
		Verb: "POST",
		Uri:  uri,
		Body: buff.Bytes(),
	}

	return data
}

func (m *Manager) CreateVirtualMachineDeploymentWin(isOSImage bool, serviceName, vmName, vmSize, userName, userPassword, osImageName, mediaLoc string) *Data {

	uri := fmt.Sprintf("https://management.core.windows.net/%s/services/hostedservices/%s/deployments", m.SubscrId, serviceName)

	var buff bytes.Buffer
	buff.WriteString("<Deployment xmlns:i='http://www.w3.org/2001/XMLSchema-instance' xmlns='http://schemas.microsoft.com/windowsazure'>")
	buff.WriteString("<Name>" + vmName + "</Name>")
	buff.WriteString("<DeploymentSlot>Production</DeploymentSlot>")
	buff.WriteString("" + vmName + "")
	buff.WriteString("<RoleList>")
	buff.WriteString("<Role i:type='PersistentVMRole'>")
	buff.WriteString("<RoleName>" + vmName + "</RoleName>")
	buff.WriteString("<RoleType>PersistentVMRole</RoleType>")
	buff.WriteString("<ConfigurationSets>")
	buff.WriteString("<ConfigurationSet i:type='WindowsProvisioningConfigurationSet'>")
	buff.WriteString("<ConfigurationSetType>WindowsProvisioningConfiguration</ConfigurationSetType>")
	buff.WriteString("<ComputerName>" + vmName + "</ComputerName>")
	buff.WriteString("<AdminPassword>" + userPassword + "</AdminPassword>")
	buff.WriteString("<EnableAutomaticUpdates>true</EnableAutomaticUpdates>")
	buff.WriteString("<AdminUsername>" + userName + "</AdminUsername>")
	buff.WriteString("</ConfigurationSet>")
	buff.WriteString("<ConfigurationSet i:type='NetworkConfigurationSet'>")
	buff.WriteString("<ConfigurationSetType>NetworkConfiguration</ConfigurationSetType>")
	buff.WriteString("<InputEndpoints>")
	buff.WriteString("<InputEndpoint>")
	buff.WriteString("<LocalPort>5986</LocalPort>")
	buff.WriteString("<Name>PowerShell</Name>")
	buff.WriteString("<Port>5986</Port>")
	buff.WriteString("<Protocol>tcp</Protocol>")
	buff.WriteString("</InputEndpoint>")
	buff.WriteString("<InputEndpoint>")
	buff.WriteString("<LocalPort>3389</LocalPort>")
	buff.WriteString("<Name>RDP</Name>")
	buff.WriteString("<Port>3389</Port>")
	buff.WriteString("<Protocol>tcp</Protocol>")
	buff.WriteString("</InputEndpoint>")
	buff.WriteString("</InputEndpoints>")
	buff.WriteString("</ConfigurationSet>")
	buff.WriteString("</ConfigurationSets>")
	if !isOSImage {
		buff.WriteString("<VMImageName>" + osImageName + "</VMImageName>")
	} else {
		buff.WriteString("<OSVirtualHardDisk>")
		buff.WriteString("<MediaLink>" + mediaLoc + "</MediaLink>")
		buff.WriteString("<SourceImageName>" + osImageName + "</SourceImageName>")
		buff.WriteString("</OSVirtualHardDisk>")
	}
	buff.WriteString("" + vmName + "")
	buff.WriteString("<RoleSize>" + vmSize + "</RoleSize>")
	buff.WriteString("<ProvisionGuestAgent>true</ProvisionGuestAgent>")
	buff.WriteString("</Role>")
	buff.WriteString("</RoleList>")
	buff.WriteString("</Deployment>")

	data := &Data{
		Verb: "POST",
		Uri:  uri,
		Body: buff.Bytes(),
	}

	return data
}
<commit_msg>Fix password authentication<commit_after>// Copyright (c) Microsoft Open Technologies, Inc.
// All Rights Reserved.
// Licensed under the Apache License, Version 2.0.
// See License.txt in the project root for license information.

package request

import (
	"bytes"
	"fmt"
)

func (m *Manager) CreateVirtualMachineDeploymentLin(isOSImage bool, serviceName, vmName, vmSize, certThumbprint, userName, osImageName, mediaLoc string) *Data {

	uri := fmt.Sprintf("https://management.core.windows.net/%s/services/hostedservices/%s/deployments", m.SubscrId, serviceName)

	var buff bytes.Buffer
	buff.WriteString("<Deployment xmlns:i='http://www.w3.org/2001/XMLSchema-instance' xmlns='http://schemas.microsoft.com/windowsazure'>")
	buff.WriteString("<Name>" + vmName + "</Name>")
	buff.WriteString("<DeploymentSlot>Production</DeploymentSlot>")
	buff.WriteString("" + vmName + "")
	buff.WriteString("<RoleList>")
	buff.WriteString("<Role i:type='PersistentVMRole'>")
	buff.WriteString("<RoleName>" + vmName + "</RoleName>")
	buff.WriteString("<RoleType>PersistentVMRole</RoleType>")
	buff.WriteString("<ConfigurationSets>")
	buff.WriteString("<ConfigurationSet i:type='LinuxProvisioningConfigurationSet'>")
	buff.WriteString("<ConfigurationSetType>LinuxProvisioningConfiguration</ConfigurationSetType>")
	buff.WriteString("<HostName>" + vmName + "</HostName>")
	buff.WriteString("<UserName>" + userName + "</UserName>")
	buff.WriteString("<DisableSshPasswordAuthentication>false</DisableSshPasswordAuthentication>")
	buff.WriteString("<SSH>")
	buff.WriteString("<PublicKeys>")
	buff.WriteString("<PublicKey>")
	buff.WriteString("<Fingerprint>" + certThumbprint + "</Fingerprint>")
	buff.WriteString("<Path>/home/" + userName + "/.ssh/authorized_keys</Path>")
	buff.WriteString("</PublicKey>")
	buff.WriteString("</PublicKeys>")
	buff.WriteString("</SSH>")
	buff.WriteString("</ConfigurationSet>")
	buff.WriteString("<ConfigurationSet i:type='NetworkConfigurationSet'>")
	buff.WriteString("<ConfigurationSetType>NetworkConfiguration</ConfigurationSetType>")
	buff.WriteString("<InputEndpoints>")
	buff.WriteString("<InputEndpoint>")
	buff.WriteString("<LocalPort>22</LocalPort>")
	buff.WriteString("<Name>SSH</Name>")
	buff.WriteString("<Port>22</Port>")
	buff.WriteString("<Protocol>tcp</Protocol>")
	buff.WriteString("</InputEndpoint>")
	buff.WriteString("</InputEndpoints>")
	buff.WriteString("</ConfigurationSet>")
	buff.WriteString("</ConfigurationSets>")
	if !isOSImage {
		buff.WriteString("<VMImageName>" + osImageName + "</VMImageName>")
	} else {
		buff.WriteString("<OSVirtualHardDisk>")
		buff.WriteString("<MediaLink>" + mediaLoc + "</MediaLink>")
		buff.WriteString("<SourceImageName>" + osImageName + "</SourceImageName>")
		buff.WriteString("</OSVirtualHardDisk>")
	}
	buff.WriteString("<RoleSize>" + vmSize + "</RoleSize>")
	buff.WriteString("<ProvisionGuestAgent>true</ProvisionGuestAgent>")
	buff.WriteString("</Role>")
	buff.WriteString("</RoleList>")
	buff.WriteString("</Deployment>")

	data := &Data{
		Verb: "POST",
		Uri:  uri,
		Body: buff.Bytes(),
	}

	return data
}

func (m *Manager) CreateVirtualMachineDeploymentWin(isOSImage bool, serviceName, vmName, vmSize, userName, userPassword, osImageName, mediaLoc string) *Data {

	uri := fmt.Sprintf("https://management.core.windows.net/%s/services/hostedservices/%s/deployments", m.SubscrId, serviceName)

	var buff bytes.Buffer
	buff.WriteString("<Deployment xmlns:i='http://www.w3.org/2001/XMLSchema-instance' xmlns='http://schemas.microsoft.com/windowsazure'>")
	buff.WriteString("<Name>" + vmName + "</Name>")
	buff.WriteString("<DeploymentSlot>Production</DeploymentSlot>")
	buff.WriteString("" + vmName + "")
	buff.WriteString("<RoleList>")
	buff.WriteString("<Role i:type='PersistentVMRole'>")
	buff.WriteString("<RoleName>" + vmName + "</RoleName>")
	buff.WriteString("<RoleType>PersistentVMRole</RoleType>")
	buff.WriteString("<ConfigurationSets>")
	buff.WriteString("<ConfigurationSet i:type='WindowsProvisioningConfigurationSet'>")
	buff.WriteString("<ConfigurationSetType>WindowsProvisioningConfiguration</ConfigurationSetType>")
	buff.WriteString("<ComputerName>" + vmName + "</ComputerName>")
	buff.WriteString("<AdminPassword>" + userPassword + "</AdminPassword>")
	buff.WriteString("<EnableAutomaticUpdates>true</EnableAutomaticUpdates>")
	buff.WriteString("<AdminUsername>" + userName + "</AdminUsername>")
	buff.WriteString("</ConfigurationSet>")
	buff.WriteString("<ConfigurationSet i:type='NetworkConfigurationSet'>")
	buff.WriteString("<ConfigurationSetType>NetworkConfiguration</ConfigurationSetType>")
	buff.WriteString("<InputEndpoints>")
	buff.WriteString("<InputEndpoint>")
	buff.WriteString("<LocalPort>5986</LocalPort>")
	buff.WriteString("<Name>PowerShell</Name>")
	buff.WriteString("<Port>5986</Port>")
	buff.WriteString("<Protocol>tcp</Protocol>")
	buff.WriteString("</InputEndpoint>")
	buff.WriteString("<InputEndpoint>")
	buff.WriteString("<LocalPort>3389</LocalPort>")
	buff.WriteString("<Name>RDP</Name>")
	buff.WriteString("<Port>3389</Port>")
	buff.WriteString("<Protocol>tcp</Protocol>")
	buff.WriteString("</InputEndpoint>")
	buff.WriteString("</InputEndpoints>")
	buff.WriteString("</ConfigurationSet>")
	buff.WriteString("</ConfigurationSets>")
	if !isOSImage {
		buff.WriteString("<VMImageName>" + osImageName + "</VMImageName>")
	} else {
		buff.WriteString("<OSVirtualHardDisk>")
		buff.WriteString("<MediaLink>" + mediaLoc + "</MediaLink>")
		buff.WriteString("<SourceImageName>" + osImageName + "</SourceImageName>")
		buff.WriteString("</OSVirtualHardDisk>")
	}
	buff.WriteString("" + vmName + "")
	buff.WriteString("<RoleSize>" + vmSize + "</RoleSize>")
	buff.WriteString("<ProvisionGuestAgent>true</ProvisionGuestAgent>")
	buff.WriteString("</Role>")
	buff.WriteString("</RoleList>")
	buff.WriteString("</Deployment>")

	data := &Data{
		Verb: "POST",
		Uri:  uri,
		Body: buff.Bytes(),
	}

	return data
}
<|endoftext|>


<commit_before><commit_msg>Add TTL function to cached TURN credentials data<commit_after><|endoftext|>


<commit_before>// Copyright 2018 The Prometheus Authors
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package collector

import (
	"bytes"
	"database/sql"
	"regexp"
	"strconv"
	"strings"

	"github.com/prometheus/client_golang/prometheus"
)

const (
	// Exporter namespace.
	namespace = "mysql"
	// Math constant for picoseconds to seconds.
	picoSeconds = 1e12
	// Query to check whether user/table/client stats are enabled.
	userstatCheckQuery = `SHOW GLOBAL VARIABLES WHERE Variable_Name='userstat'
		OR Variable_Name='userstat_running'`
)

var logRE = regexp.MustCompile(`.+\.(\d+)$`)

func newDesc(subsystem, name, help string) *prometheus.Desc {
	return prometheus.NewDesc(
		prometheus.BuildFQName(namespace, subsystem, name),
		help, nil, nil,
	)
}

func parseStatus(data sql.RawBytes) (float64, bool) {
	dataString := strings.ToLower(string(data))
	switch dataString {
	case "yes", "on":
		return 1, true
	case "no", "off", "disabled":
		return 0, true
	// SHOW SLAVE STATUS Slave_IO_Running can return "Connecting" which is a non-running state.
	case "connecting":
		return 0, true
	// SHOW GLOBAL STATUS like 'wsrep_cluster_status' can return "Primary" or "non-Primary"/"Disconnected"
	case "primary":
		return 1, true
	case "non-primary", "disconnected":
		return 0, true
	}
	if logNum := logRE.Find(data); logNum != nil {
		value, err := strconv.ParseFloat(string(logNum), 64)
		return value, err == nil
	}
	value, err := strconv.ParseFloat(string(data), 64)
	return value, err == nil
}

func parsePrivilege(data sql.RawBytes) (float64, bool) {
	if bytes.Equal(data, []byte("Y")) {
		return 1, true
	}
	if bytes.Equal(data, []byte("N")) {
		return 0, true
	}
	return -1, false
}
<commit_msg>allow exporting dates as timestamps grom GLOBAL STATUS<commit_after>// Copyright 2018 The Prometheus Authors
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package collector

import (
	"bytes"
	"database/sql"
	"regexp"
	"strconv"
	"strings"
	"time"

	"github.com/prometheus/client_golang/prometheus"
)

const (
	// Exporter namespace.
	namespace = "mysql"
	// Math constant for picoseconds to seconds.
	picoSeconds = 1e12
	// Query to check whether user/table/client stats are enabled.
	userstatCheckQuery = `SHOW GLOBAL VARIABLES WHERE Variable_Name='userstat'
		OR Variable_Name='userstat_running'`
	// date layout
	dateLayout = "Jan 02 15:04:05 2006 GMT"
)

var logRE = regexp.MustCompile(`.+\.(\d+)$`)

func newDesc(subsystem, name, help string) *prometheus.Desc {
	return prometheus.NewDesc(
		prometheus.BuildFQName(namespace, subsystem, name),
		help, nil, nil,
	)
}

func parseStatus(data sql.RawBytes) (float64, bool) {
	dataString := strings.ToLower(string(data))
	switch dataString {
	case "yes", "on":
		return 1, true
	case "no", "off", "disabled":
		return 0, true
	// SHOW SLAVE STATUS Slave_IO_Running can return "Connecting" which is a non-running state.
	case "connecting":
		return 0, true
	// SHOW GLOBAL STATUS like 'wsrep_cluster_status' can return "Primary" or "non-Primary"/"Disconnected"
	case "primary":
		return 1, true
	case "non-primary", "disconnected":
		return 0, true
	}
	if ts, err := time.Parse(dateLayout, string(data)); err == nil {
		unixStamp := float64(ts.Unix())
		return unixStamp, err == nil
	}
	if logNum := logRE.Find(data); logNum != nil {
		value, err := strconv.ParseFloat(string(logNum), 64)
		return value, err == nil
	}
	value, err := strconv.ParseFloat(string(data), 64)
	return value, err == nil
}

func parsePrivilege(data sql.RawBytes) (float64, bool) {
	if bytes.Equal(data, []byte("Y")) {
		return 1, true
	}
	if bytes.Equal(data, []byte("N")) {
		return 0, true
	}
	return -1, false
}
<|endoftext|>


<commit_before>// +build !race,!hsm

// NOTE: we can't use this with HSM. We can't set testing mode on and it's not
// safe to use env vars since that provides an attack vector in the real world.
//
// The server tests have a go-metrics/exp manager race condition :(.

package command

import (
	"crypto/tls"
	"crypto/x509"
	"fmt"
	"io/ioutil"
	"net"
	"os"
	"strings"
	"sync"
	"testing"
	"time"

	"github.com/hashicorp/vault/sdk/physical"
	physInmem "github.com/hashicorp/vault/sdk/physical/inmem"
	"github.com/mitchellh/cli"
)

func testRandomPort(tb testing.TB) int {
	tb.Helper()

	addr, err := net.ResolveTCPAddr("tcp", "127.0.0.1:0")
	if err != nil {
		tb.Fatal(err)
	}

	l, err := net.ListenTCP("tcp", addr)
	if err != nil {
		tb.Fatal(err)
	}
	defer l.Close()

	return l.Addr().(*net.TCPAddr).Port
}

func testBaseHCL(tb testing.TB, listenerExtras string) string {
	tb.Helper()

	return strings.TrimSpace(fmt.Sprintf(`
		disable_mlock = true
		listener "tcp" {
			address     = "127.0.0.1:%d"
			tls_disable = "true"
			%s
		}
	`, testRandomPort(tb), listenerExtras))
}

const (
	goodListenerTimeouts = `http_read_header_timeout = 12
			http_read_timeout = "34s"
			http_write_timeout = "56m"
			http_idle_timeout = "78h"`

	badListenerReadHeaderTimeout = `http_read_header_timeout = "12km"`
	badListenerReadTimeout       = `http_read_timeout = "34日"`
	badListenerWriteTimeout      = `http_write_timeout = "56lbs"`
	badListenerIdleTimeout       = `http_idle_timeout = "78gophers"`

	inmemHCL = `
backend "inmem_ha" {
  advertise_addr       = "http://127.0.0.1:8200"
}
`
	haInmemHCL = `
ha_backend "inmem_ha" {
  redirect_addr        = "http://127.0.0.1:8200"
}
`

	badHAInmemHCL = `
ha_backend "inmem" {}
`

	reloadHCL = `
backend "inmem" {}
disable_mlock = true
listener "tcp" {
  address       = "127.0.0.1:8203"
  tls_cert_file = "TMPDIR/reload_cert.pem"
  tls_key_file  = "TMPDIR/reload_key.pem"
}
`
)

func testServerCommand(tb testing.TB) (*cli.MockUi, *ServerCommand) {
	tb.Helper()

	ui := cli.NewMockUi()
	return ui, &ServerCommand{
		BaseCommand: &BaseCommand{
			UI: ui,
		},
		ShutdownCh: MakeShutdownCh(),
		SighupCh:   MakeSighupCh(),
		SigUSR2Ch:  MakeSigUSR2Ch(),
		PhysicalBackends: map[string]physical.Factory{
			"inmem":    physInmem.NewInmem,
			"inmem_ha": physInmem.NewInmemHA,
		},

		// These prevent us from random sleep guessing...
		startedCh:  make(chan struct{}, 5),
		reloadedCh: make(chan struct{}, 5),
	}
}

func TestServer_ReloadListener(t *testing.T) {
	t.Parallel()

	wd, _ := os.Getwd()
	wd += "/server/test-fixtures/reload/"

	td, err := ioutil.TempDir("", "vault-test-")
	if err != nil {
		t.Fatal(err)
	}
	defer os.RemoveAll(td)

	wg := &sync.WaitGroup{}

	// Setup initial certs
	inBytes, _ := ioutil.ReadFile(wd + "reload_foo.pem")
	ioutil.WriteFile(td+"/reload_cert.pem", inBytes, 0777)
	inBytes, _ = ioutil.ReadFile(wd + "reload_foo.key")
	ioutil.WriteFile(td+"/reload_key.pem", inBytes, 0777)

	relhcl := strings.Replace(reloadHCL, "TMPDIR", td, -1)
	ioutil.WriteFile(td+"/reload.hcl", []byte(relhcl), 0777)

	inBytes, _ = ioutil.ReadFile(wd + "reload_ca.pem")
	certPool := x509.NewCertPool()
	ok := certPool.AppendCertsFromPEM(inBytes)
	if !ok {
		t.Fatal("not ok when appending CA cert")
	}

	ui, cmd := testServerCommand(t)
	_ = ui

	wg.Add(1)
	args := []string{"-config", td + "/reload.hcl"}
	go func() {
		if code := cmd.Run(args); code != 0 {
			output := ui.ErrorWriter.String() + ui.OutputWriter.String()
			t.Errorf("got a non-zero exit status: %s", output)
		}
		wg.Done()
	}()

	testCertificateName := func(cn string) error {
		conn, err := tls.Dial("tcp", "127.0.0.1:8203", &tls.Config{
			RootCAs: certPool,
		})
		if err != nil {
			return err
		}
		defer conn.Close()
		if err = conn.Handshake(); err != nil {
			return err
		}
		servName := conn.ConnectionState().PeerCertificates[0].Subject.CommonName
		if servName != cn {
			return fmt.Errorf("expected %s, got %s", cn, servName)
		}
		return nil
	}

	select {
	case <-cmd.startedCh:
	case <-time.After(5 * time.Second):
		t.Fatalf("timeout")
	}

	if err := testCertificateName("foo.example.com"); err != nil {
		t.Fatalf("certificate name didn't check out: %s", err)
	}

	relhcl = strings.Replace(reloadHCL, "TMPDIR", td, -1)
	inBytes, _ = ioutil.ReadFile(wd + "reload_bar.pem")
	ioutil.WriteFile(td+"/reload_cert.pem", inBytes, 0777)
	inBytes, _ = ioutil.ReadFile(wd + "reload_bar.key")
	ioutil.WriteFile(td+"/reload_key.pem", inBytes, 0777)
	ioutil.WriteFile(td+"/reload.hcl", []byte(relhcl), 0777)

	cmd.SighupCh <- struct{}{}
	select {
	case <-cmd.reloadedCh:
	case <-time.After(5 * time.Second):
		t.Fatalf("timeout")
	}

	if err := testCertificateName("bar.example.com"); err != nil {
		t.Fatalf("certificate name didn't check out: %s", err)
	}

	cmd.ShutdownCh <- struct{}{}

	wg.Wait()
}

func TestServer(t *testing.T) {
	t.Parallel()

	cases := []struct {
		name     string
		contents string
		exp      string
		code     int
		flag     string
	}{
		{
			"common_ha",
			testBaseHCL(t, "") + inmemHCL,
			"(HA available)",
			0,
			"-test-verify-only",
		},
		{
			"separate_ha",
			testBaseHCL(t, "") + inmemHCL + haInmemHCL,
			"HA Storage:",
			0,
			"-test-verify-only",
		},
		{
			"bad_separate_ha",
			testBaseHCL(t, "") + inmemHCL + badHAInmemHCL,
			"Specified HA storage does not support HA",
			1,
			"-test-verify-only",
		},
		{
			"good_listener_timeout_config",
			testBaseHCL(t, goodListenerTimeouts) + inmemHCL,
			"",
			0,
			"-test-server-config",
		},
		{
			"bad_listener_read_header_timeout_config",
			testBaseHCL(t, badListenerReadHeaderTimeout) + inmemHCL,
			"Could not parse a time value for http_read_header_timeout",
			1,
			"-test-server-config",
		},
		{
			"bad_listener_read_header_timeout_config",
			testBaseHCL(t, badListenerReadHeaderTimeout) + inmemHCL,
			"Could not parse a time value for http_read_header_timeout",
			1,
			"-test-server-config",
		},
		{
			"bad_listener_read_timeout_config",
			testBaseHCL(t, badListenerReadTimeout) + inmemHCL,
			"Could not parse a time value for http_read_timeout",
			1,
			"-test-server-config",
		},
		{
			"bad_listener_write_timeout_config",
			testBaseHCL(t, badListenerWriteTimeout) + inmemHCL,
			"Could not parse a time value for http_write_timeout",
			1,
			"-test-server-config",
		},
		{
			"bad_listener_idle_timeout_config",
			testBaseHCL(t, badListenerIdleTimeout) + inmemHCL,
			"Could not parse a time value for http_idle_timeout",
			1,
			"-test-server-config",
		},
	}

	for _, tc := range cases {
		tc := tc

		t.Run(tc.name, func(t *testing.T) {
			t.Parallel()

			ui, cmd := testServerCommand(t)
			f, err := ioutil.TempFile("", "")
			if err != nil {
				t.Fatalf("error creating temp dir: %v", err)
			}
			f.WriteString(tc.contents)
			f.Close()
			defer os.Remove(f.Name())

			code := cmd.Run([]string{
				"-config", f.Name(),
				tc.flag,
			})
			output := ui.ErrorWriter.String() + ui.OutputWriter.String()
			if code != tc.code {
				t.Errorf("expected %d to be %d: %s", code, tc.code, output)
			}

			if !strings.Contains(output, tc.exp) {
				t.Fatalf("expected %q to contain %q", output, tc.exp)
			}
		})
	}
}
<commit_msg>removes a duplicated test case in server_test (#6740)<commit_after>// +build !race,!hsm

// NOTE: we can't use this with HSM. We can't set testing mode on and it's not
// safe to use env vars since that provides an attack vector in the real world.
//
// The server tests have a go-metrics/exp manager race condition :(.

package command

import (
	"crypto/tls"
	"crypto/x509"
	"fmt"
	"io/ioutil"
	"net"
	"os"
	"strings"
	"sync"
	"testing"
	"time"

	"github.com/hashicorp/vault/sdk/physical"
	physInmem "github.com/hashicorp/vault/sdk/physical/inmem"
	"github.com/mitchellh/cli"
)

func testRandomPort(tb testing.TB) int {
	tb.Helper()

	addr, err := net.ResolveTCPAddr("tcp", "127.0.0.1:0")
	if err != nil {
		tb.Fatal(err)
	}

	l, err := net.ListenTCP("tcp", addr)
	if err != nil {
		tb.Fatal(err)
	}
	defer l.Close()

	return l.Addr().(*net.TCPAddr).Port
}

func testBaseHCL(tb testing.TB, listenerExtras string) string {
	tb.Helper()

	return strings.TrimSpace(fmt.Sprintf(`
		disable_mlock = true
		listener "tcp" {
			address     = "127.0.0.1:%d"
			tls_disable = "true"
			%s
		}
	`, testRandomPort(tb), listenerExtras))
}

const (
	goodListenerTimeouts = `http_read_header_timeout = 12
			http_read_timeout = "34s"
			http_write_timeout = "56m"
			http_idle_timeout = "78h"`

	badListenerReadHeaderTimeout = `http_read_header_timeout = "12km"`
	badListenerReadTimeout       = `http_read_timeout = "34日"`
	badListenerWriteTimeout      = `http_write_timeout = "56lbs"`
	badListenerIdleTimeout       = `http_idle_timeout = "78gophers"`

	inmemHCL = `
backend "inmem_ha" {
  advertise_addr       = "http://127.0.0.1:8200"
}
`
	haInmemHCL = `
ha_backend "inmem_ha" {
  redirect_addr        = "http://127.0.0.1:8200"
}
`

	badHAInmemHCL = `
ha_backend "inmem" {}
`

	reloadHCL = `
backend "inmem" {}
disable_mlock = true
listener "tcp" {
  address       = "127.0.0.1:8203"
  tls_cert_file = "TMPDIR/reload_cert.pem"
  tls_key_file  = "TMPDIR/reload_key.pem"
}
`
)

func testServerCommand(tb testing.TB) (*cli.MockUi, *ServerCommand) {
	tb.Helper()

	ui := cli.NewMockUi()
	return ui, &ServerCommand{
		BaseCommand: &BaseCommand{
			UI: ui,
		},
		ShutdownCh: MakeShutdownCh(),
		SighupCh:   MakeSighupCh(),
		SigUSR2Ch:  MakeSigUSR2Ch(),
		PhysicalBackends: map[string]physical.Factory{
			"inmem":    physInmem.NewInmem,
			"inmem_ha": physInmem.NewInmemHA,
		},

		// These prevent us from random sleep guessing...
		startedCh:  make(chan struct{}, 5),
		reloadedCh: make(chan struct{}, 5),
	}
}

func TestServer_ReloadListener(t *testing.T) {
	t.Parallel()

	wd, _ := os.Getwd()
	wd += "/server/test-fixtures/reload/"

	td, err := ioutil.TempDir("", "vault-test-")
	if err != nil {
		t.Fatal(err)
	}
	defer os.RemoveAll(td)

	wg := &sync.WaitGroup{}

	// Setup initial certs
	inBytes, _ := ioutil.ReadFile(wd + "reload_foo.pem")
	ioutil.WriteFile(td+"/reload_cert.pem", inBytes, 0777)
	inBytes, _ = ioutil.ReadFile(wd + "reload_foo.key")
	ioutil.WriteFile(td+"/reload_key.pem", inBytes, 0777)

	relhcl := strings.Replace(reloadHCL, "TMPDIR", td, -1)
	ioutil.WriteFile(td+"/reload.hcl", []byte(relhcl), 0777)

	inBytes, _ = ioutil.ReadFile(wd + "reload_ca.pem")
	certPool := x509.NewCertPool()
	ok := certPool.AppendCertsFromPEM(inBytes)
	if !ok {
		t.Fatal("not ok when appending CA cert")
	}

	ui, cmd := testServerCommand(t)
	_ = ui

	wg.Add(1)
	args := []string{"-config", td + "/reload.hcl"}
	go func() {
		if code := cmd.Run(args); code != 0 {
			output := ui.ErrorWriter.String() + ui.OutputWriter.String()
			t.Errorf("got a non-zero exit status: %s", output)
		}
		wg.Done()
	}()

	testCertificateName := func(cn string) error {
		conn, err := tls.Dial("tcp", "127.0.0.1:8203", &tls.Config{
			RootCAs: certPool,
		})
		if err != nil {
			return err
		}
		defer conn.Close()
		if err = conn.Handshake(); err != nil {
			return err
		}
		servName := conn.ConnectionState().PeerCertificates[0].Subject.CommonName
		if servName != cn {
			return fmt.Errorf("expected %s, got %s", cn, servName)
		}
		return nil
	}

	select {
	case <-cmd.startedCh:
	case <-time.After(5 * time.Second):
		t.Fatalf("timeout")
	}

	if err := testCertificateName("foo.example.com"); err != nil {
		t.Fatalf("certificate name didn't check out: %s", err)
	}

	relhcl = strings.Replace(reloadHCL, "TMPDIR", td, -1)
	inBytes, _ = ioutil.ReadFile(wd + "reload_bar.pem")
	ioutil.WriteFile(td+"/reload_cert.pem", inBytes, 0777)
	inBytes, _ = ioutil.ReadFile(wd + "reload_bar.key")
	ioutil.WriteFile(td+"/reload_key.pem", inBytes, 0777)
	ioutil.WriteFile(td+"/reload.hcl", []byte(relhcl), 0777)

	cmd.SighupCh <- struct{}{}
	select {
	case <-cmd.reloadedCh:
	case <-time.After(5 * time.Second):
		t.Fatalf("timeout")
	}

	if err := testCertificateName("bar.example.com"); err != nil {
		t.Fatalf("certificate name didn't check out: %s", err)
	}

	cmd.ShutdownCh <- struct{}{}

	wg.Wait()
}

func TestServer(t *testing.T) {
	t.Parallel()

	cases := []struct {
		name     string
		contents string
		exp      string
		code     int
		flag     string
	}{
		{
			"common_ha",
			testBaseHCL(t, "") + inmemHCL,
			"(HA available)",
			0,
			"-test-verify-only",
		},
		{
			"separate_ha",
			testBaseHCL(t, "") + inmemHCL + haInmemHCL,
			"HA Storage:",
			0,
			"-test-verify-only",
		},
		{
			"bad_separate_ha",
			testBaseHCL(t, "") + inmemHCL + badHAInmemHCL,
			"Specified HA storage does not support HA",
			1,
			"-test-verify-only",
		},
		{
			"good_listener_timeout_config",
			testBaseHCL(t, goodListenerTimeouts) + inmemHCL,
			"",
			0,
			"-test-server-config",
		},
		{
			"bad_listener_read_header_timeout_config",
			testBaseHCL(t, badListenerReadHeaderTimeout) + inmemHCL,
			"Could not parse a time value for http_read_header_timeout",
			1,
			"-test-server-config",
		},
		{
			"bad_listener_read_timeout_config",
			testBaseHCL(t, badListenerReadTimeout) + inmemHCL,
			"Could not parse a time value for http_read_timeout",
			1,
			"-test-server-config",
		},
		{
			"bad_listener_write_timeout_config",
			testBaseHCL(t, badListenerWriteTimeout) + inmemHCL,
			"Could not parse a time value for http_write_timeout",
			1,
			"-test-server-config",
		},
		{
			"bad_listener_idle_timeout_config",
			testBaseHCL(t, badListenerIdleTimeout) + inmemHCL,
			"Could not parse a time value for http_idle_timeout",
			1,
			"-test-server-config",
		},
	}

	for _, tc := range cases {
		tc := tc

		t.Run(tc.name, func(t *testing.T) {
			t.Parallel()

			ui, cmd := testServerCommand(t)
			f, err := ioutil.TempFile("", "")
			if err != nil {
				t.Fatalf("error creating temp dir: %v", err)
			}
			f.WriteString(tc.contents)
			f.Close()
			defer os.Remove(f.Name())

			code := cmd.Run([]string{
				"-config", f.Name(),
				tc.flag,
			})
			output := ui.ErrorWriter.String() + ui.OutputWriter.String()
			if code != tc.code {
				t.Errorf("expected %d to be %d: %s", code, tc.code, output)
			}

			if !strings.Contains(output, tc.exp) {
				t.Fatalf("expected %q to contain %q", output, tc.exp)
			}
		})
	}
}
<|endoftext|>


<commit_before>// Copyright 2018 The Hugo Authors. All rights reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package commands

import (
	"bytes"
	"errors"

	"io/ioutil"

	"github.com/gohugoio/hugo/common/herrors"
	"github.com/gohugoio/hugo/common/hugo"

	jww "github.com/spf13/jwalterweatherman"

	"os"
	"path/filepath"
	"regexp"
	"strings"
	"sync"
	"time"

	"github.com/gohugoio/hugo/common/loggers"
	"github.com/gohugoio/hugo/config"

	"github.com/spf13/cobra"

	"github.com/gohugoio/hugo/hugolib"
	"github.com/spf13/afero"

	"github.com/bep/debounce"
	"github.com/gohugoio/hugo/common/types"
	"github.com/gohugoio/hugo/deps"
	"github.com/gohugoio/hugo/helpers"
	"github.com/gohugoio/hugo/hugofs"
	"github.com/gohugoio/hugo/langs"
)

type commandeerHugoState struct {
	*deps.DepsCfg
	hugo     *hugolib.HugoSites
	fsCreate sync.Once
}

type commandeer struct {
	*commandeerHugoState

	logger *loggers.Logger

	// Currently only set when in "fast render mode". But it seems to
	// be fast enough that we could maybe just add it for all server modes.
	changeDetector *fileChangeDetector

	// We need to reuse this on server rebuilds.
	destinationFs afero.Fs

	h    *hugoBuilderCommon
	ftch flagsToConfigHandler

	visitedURLs *types.EvictingStringQueue

	doWithCommandeer func(c *commandeer) error

	// We watch these for changes.
	configFiles []string

	// Used in cases where we get flooded with events in server mode.
	debounce func(f func())

	serverPorts         []int
	languagesConfigured bool
	languages           langs.Languages
	doLiveReload        bool
	fastRenderMode      bool
	showErrorInBrowser  bool

	configured bool
	paused     bool

	// Any error from the last build.
	buildErr error
}

func (c *commandeer) errCount() int {
	return int(c.logger.ErrorCounter.Count())
}

func (c *commandeer) getErrorWithContext() interface{} {
	errCount := c.errCount()

	if errCount == 0 {
		return nil
	}

	m := make(map[string]interface{})

	m["Error"] = errors.New(removeErrorPrefixFromLog(c.logger.Errors()))
	m["Version"] = hugo.BuildVersionString()

	fe := herrors.UnwrapErrorWithFileContext(c.buildErr)
	if fe != nil {
		m["File"] = fe
	}

	if c.h.verbose {
		var b bytes.Buffer
		herrors.FprintStackTrace(&b, c.buildErr)
		m["StackTrace"] = b.String()
	}

	return m
}

func (c *commandeer) Set(key string, value interface{}) {
	if c.configured {
		panic("commandeer cannot be changed")
	}
	c.Cfg.Set(key, value)
}

func (c *commandeer) initFs(fs *hugofs.Fs) error {
	c.destinationFs = fs.Destination
	c.DepsCfg.Fs = fs

	return nil
}

func newCommandeer(mustHaveConfigFile, running bool, h *hugoBuilderCommon, f flagsToConfigHandler, doWithCommandeer func(c *commandeer) error, subCmdVs ...*cobra.Command) (*commandeer, error) {

	var rebuildDebouncer func(f func())
	if running {
		// The time value used is tested with mass content replacements in a fairly big Hugo site.
		// It is better to wait for some seconds in those cases rather than get flooded
		// with rebuilds.
		rebuildDebouncer = debounce.New(4 * time.Second)
	}

	c := &commandeer{
		h:                   h,
		ftch:                f,
		commandeerHugoState: &commandeerHugoState{},
		doWithCommandeer:    doWithCommandeer,
		visitedURLs:         types.NewEvictingStringQueue(10),
		debounce:            rebuildDebouncer,
		// This will be replaced later, but we need something to log to before the configuration is read.
		logger: loggers.NewLogger(jww.LevelError, jww.LevelError, os.Stdout, ioutil.Discard, running),
	}

	return c, c.loadConfig(mustHaveConfigFile, running)
}

type fileChangeDetector struct {
	sync.Mutex
	current map[string]string
	prev    map[string]string

	irrelevantRe *regexp.Regexp
}

func (f *fileChangeDetector) OnFileClose(name, md5sum string) {
	f.Lock()
	defer f.Unlock()
	f.current[name] = md5sum
}

func (f *fileChangeDetector) changed() []string {
	if f == nil {
		return nil
	}
	f.Lock()
	defer f.Unlock()
	var c []string
	for k, v := range f.current {
		vv, found := f.prev[k]
		if !found || v != vv {
			c = append(c, k)
		}
	}

	return f.filterIrrelevant(c)
}

func (f *fileChangeDetector) filterIrrelevant(in []string) []string {
	var filtered []string
	for _, v := range in {
		if !f.irrelevantRe.MatchString(v) {
			filtered = append(filtered, v)
		}
	}
	return filtered
}

func (f *fileChangeDetector) PrepareNew() {
	if f == nil {
		return
	}

	f.Lock()
	defer f.Unlock()

	if f.current == nil {
		f.current = make(map[string]string)
		f.prev = make(map[string]string)
		return
	}

	f.prev = make(map[string]string)
	for k, v := range f.current {
		f.prev[k] = v
	}
	f.current = make(map[string]string)
}

func (c *commandeer) loadConfig(mustHaveConfigFile, running bool) error {

	if c.DepsCfg == nil {
		c.DepsCfg = &deps.DepsCfg{}
	}

	if c.logger != nil {
		// Truncate the error log if this is a reload.
		c.logger.Reset()
	}

	cfg := c.DepsCfg
	c.configured = false
	cfg.Running = running

	var dir string
	if c.h.source != "" {
		dir, _ = filepath.Abs(c.h.source)
	} else {
		dir, _ = os.Getwd()
	}

	var sourceFs afero.Fs = hugofs.Os
	if c.DepsCfg.Fs != nil {
		sourceFs = c.DepsCfg.Fs.Source
	}

	environment := c.h.getEnvironment(running)

	doWithConfig := func(cfg config.Provider) error {

		if c.ftch != nil {
			c.ftch.flagsToConfig(cfg)
		}

		cfg.Set("workingDir", dir)
		cfg.Set("environment", environment)
		return nil
	}

	doWithCommandeer := func(cfg config.Provider) error {
		c.Cfg = cfg
		if c.doWithCommandeer == nil {
			return nil
		}
		err := c.doWithCommandeer(c)
		return err
	}

	configPath := c.h.source
	if configPath == "" {
		configPath = dir
	}
	config, configFiles, err := hugolib.LoadConfig(
		hugolib.ConfigSourceDescriptor{
			Fs:           sourceFs,
			Path:         configPath,
			WorkingDir:   dir,
			Filename:     c.h.cfgFile,
			AbsConfigDir: c.h.getConfigDir(dir),
			Environment:  environment},
		doWithCommandeer,
		doWithConfig)

	if err != nil {
		if mustHaveConfigFile {
			return err
		}
		if err != hugolib.ErrNoConfigFile {
			return err
		}

	}

	c.configFiles = configFiles

	if l, ok := c.Cfg.Get("languagesSorted").(langs.Languages); ok {
		c.languagesConfigured = true
		c.languages = l
	}

	// Set some commonly used flags
	c.doLiveReload = !c.h.buildWatch && !c.Cfg.GetBool("disableLiveReload")
	c.fastRenderMode = c.doLiveReload && !c.Cfg.GetBool("disableFastRender")
	c.showErrorInBrowser = c.doLiveReload && !c.Cfg.GetBool("disableBrowserError")

	// This is potentially double work, but we need to do this one more time now
	// that all the languages have been configured.
	if c.doWithCommandeer != nil {
		if err := c.doWithCommandeer(c); err != nil {
			return err
		}
	}

	logger, err := c.createLogger(config, running)
	if err != nil {
		return err
	}

	cfg.Logger = logger
	c.logger = logger

	createMemFs := config.GetBool("renderToMemory")

	if createMemFs {
		// Rendering to memoryFS, publish to Root regardless of publishDir.
		config.Set("publishDir", "/")
	}

	c.fsCreate.Do(func() {
		fs := hugofs.NewFrom(sourceFs, config)

		if c.destinationFs != nil {
			// Need to reuse the destination on server rebuilds.
			fs.Destination = c.destinationFs
		} else if createMemFs {
			// Hugo writes the output to memory instead of the disk.
			fs.Destination = new(afero.MemMapFs)
		}

		if c.fastRenderMode {
			// For now, fast render mode only. It should, however, be fast enough
			// for the full variant, too.
			changeDetector := &fileChangeDetector{
				// We use this detector to decide to do a Hot reload of a single path or not.
				// We need to filter out source maps and possibly some other to be able
				// to make that decision.
				irrelevantRe: regexp.MustCompile(`\.map$`),
			}
			changeDetector.PrepareNew()
			fs.Destination = hugofs.NewHashingFs(fs.Destination, changeDetector)
			c.changeDetector = changeDetector
		}

		err = c.initFs(fs)
		if err != nil {
			return
		}

		var h *hugolib.HugoSites

		h, err = hugolib.NewHugoSites(*c.DepsCfg)
		c.hugo = h

	})

	if err != nil {
		return err
	}

	cacheDir, err := helpers.GetCacheDir(sourceFs, config)
	if err != nil {
		return err
	}
	config.Set("cacheDir", cacheDir)

	cfg.Logger.INFO.Println("Using config file:", config.ConfigFileUsed())

	themeDir := c.hugo.PathSpec.GetFirstThemeDir()
	if themeDir != "" {
		if _, err := sourceFs.Stat(themeDir); os.IsNotExist(err) {
			return newSystemError("Unable to find theme Directory:", themeDir)
		}
	}

	dir, themeVersionMismatch, minVersion := c.isThemeVsHugoVersionMismatch(sourceFs)

	if themeVersionMismatch {
		name := filepath.Base(dir)
		cfg.Logger.ERROR.Printf("%s theme does not support Hugo version %s. Minimum version required is %s\n",
			strings.ToUpper(name), hugo.CurrentVersion.ReleaseVersion(), minVersion)
	}

	return nil

}
<commit_msg>commands: Fix doLiveReload logic<commit_after>// Copyright 2018 The Hugo Authors. All rights reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package commands

import (
	"bytes"
	"errors"

	"io/ioutil"

	"github.com/gohugoio/hugo/common/herrors"
	"github.com/gohugoio/hugo/common/hugo"

	jww "github.com/spf13/jwalterweatherman"

	"os"
	"path/filepath"
	"regexp"
	"strings"
	"sync"
	"time"

	"github.com/gohugoio/hugo/common/loggers"
	"github.com/gohugoio/hugo/config"

	"github.com/spf13/cobra"

	"github.com/gohugoio/hugo/hugolib"
	"github.com/spf13/afero"

	"github.com/bep/debounce"
	"github.com/gohugoio/hugo/common/types"
	"github.com/gohugoio/hugo/deps"
	"github.com/gohugoio/hugo/helpers"
	"github.com/gohugoio/hugo/hugofs"
	"github.com/gohugoio/hugo/langs"
)

type commandeerHugoState struct {
	*deps.DepsCfg
	hugo     *hugolib.HugoSites
	fsCreate sync.Once
}

type commandeer struct {
	*commandeerHugoState

	logger *loggers.Logger

	// Currently only set when in "fast render mode". But it seems to
	// be fast enough that we could maybe just add it for all server modes.
	changeDetector *fileChangeDetector

	// We need to reuse this on server rebuilds.
	destinationFs afero.Fs

	h    *hugoBuilderCommon
	ftch flagsToConfigHandler

	visitedURLs *types.EvictingStringQueue

	doWithCommandeer func(c *commandeer) error

	// We watch these for changes.
	configFiles []string

	// Used in cases where we get flooded with events in server mode.
	debounce func(f func())

	serverPorts         []int
	languagesConfigured bool
	languages           langs.Languages
	doLiveReload        bool
	fastRenderMode      bool
	showErrorInBrowser  bool

	configured bool
	paused     bool

	// Any error from the last build.
	buildErr error
}

func (c *commandeer) errCount() int {
	return int(c.logger.ErrorCounter.Count())
}

func (c *commandeer) getErrorWithContext() interface{} {
	errCount := c.errCount()

	if errCount == 0 {
		return nil
	}

	m := make(map[string]interface{})

	m["Error"] = errors.New(removeErrorPrefixFromLog(c.logger.Errors()))
	m["Version"] = hugo.BuildVersionString()

	fe := herrors.UnwrapErrorWithFileContext(c.buildErr)
	if fe != nil {
		m["File"] = fe
	}

	if c.h.verbose {
		var b bytes.Buffer
		herrors.FprintStackTrace(&b, c.buildErr)
		m["StackTrace"] = b.String()
	}

	return m
}

func (c *commandeer) Set(key string, value interface{}) {
	if c.configured {
		panic("commandeer cannot be changed")
	}
	c.Cfg.Set(key, value)
}

func (c *commandeer) initFs(fs *hugofs.Fs) error {
	c.destinationFs = fs.Destination
	c.DepsCfg.Fs = fs

	return nil
}

func newCommandeer(mustHaveConfigFile, running bool, h *hugoBuilderCommon, f flagsToConfigHandler, doWithCommandeer func(c *commandeer) error, subCmdVs ...*cobra.Command) (*commandeer, error) {

	var rebuildDebouncer func(f func())
	if running {
		// The time value used is tested with mass content replacements in a fairly big Hugo site.
		// It is better to wait for some seconds in those cases rather than get flooded
		// with rebuilds.
		rebuildDebouncer = debounce.New(4 * time.Second)
	}

	c := &commandeer{
		h:                   h,
		ftch:                f,
		commandeerHugoState: &commandeerHugoState{},
		doWithCommandeer:    doWithCommandeer,
		visitedURLs:         types.NewEvictingStringQueue(10),
		debounce:            rebuildDebouncer,
		// This will be replaced later, but we need something to log to before the configuration is read.
		logger: loggers.NewLogger(jww.LevelError, jww.LevelError, os.Stdout, ioutil.Discard, running),
	}

	return c, c.loadConfig(mustHaveConfigFile, running)
}

type fileChangeDetector struct {
	sync.Mutex
	current map[string]string
	prev    map[string]string

	irrelevantRe *regexp.Regexp
}

func (f *fileChangeDetector) OnFileClose(name, md5sum string) {
	f.Lock()
	defer f.Unlock()
	f.current[name] = md5sum
}

func (f *fileChangeDetector) changed() []string {
	if f == nil {
		return nil
	}
	f.Lock()
	defer f.Unlock()
	var c []string
	for k, v := range f.current {
		vv, found := f.prev[k]
		if !found || v != vv {
			c = append(c, k)
		}
	}

	return f.filterIrrelevant(c)
}

func (f *fileChangeDetector) filterIrrelevant(in []string) []string {
	var filtered []string
	for _, v := range in {
		if !f.irrelevantRe.MatchString(v) {
			filtered = append(filtered, v)
		}
	}
	return filtered
}

func (f *fileChangeDetector) PrepareNew() {
	if f == nil {
		return
	}

	f.Lock()
	defer f.Unlock()

	if f.current == nil {
		f.current = make(map[string]string)
		f.prev = make(map[string]string)
		return
	}

	f.prev = make(map[string]string)
	for k, v := range f.current {
		f.prev[k] = v
	}
	f.current = make(map[string]string)
}

func (c *commandeer) loadConfig(mustHaveConfigFile, running bool) error {

	if c.DepsCfg == nil {
		c.DepsCfg = &deps.DepsCfg{}
	}

	if c.logger != nil {
		// Truncate the error log if this is a reload.
		c.logger.Reset()
	}

	cfg := c.DepsCfg
	c.configured = false
	cfg.Running = running

	var dir string
	if c.h.source != "" {
		dir, _ = filepath.Abs(c.h.source)
	} else {
		dir, _ = os.Getwd()
	}

	var sourceFs afero.Fs = hugofs.Os
	if c.DepsCfg.Fs != nil {
		sourceFs = c.DepsCfg.Fs.Source
	}

	environment := c.h.getEnvironment(running)

	doWithConfig := func(cfg config.Provider) error {

		if c.ftch != nil {
			c.ftch.flagsToConfig(cfg)
		}

		cfg.Set("workingDir", dir)
		cfg.Set("environment", environment)
		return nil
	}

	doWithCommandeer := func(cfg config.Provider) error {
		c.Cfg = cfg
		if c.doWithCommandeer == nil {
			return nil
		}
		err := c.doWithCommandeer(c)
		return err
	}

	configPath := c.h.source
	if configPath == "" {
		configPath = dir
	}
	config, configFiles, err := hugolib.LoadConfig(
		hugolib.ConfigSourceDescriptor{
			Fs:           sourceFs,
			Path:         configPath,
			WorkingDir:   dir,
			Filename:     c.h.cfgFile,
			AbsConfigDir: c.h.getConfigDir(dir),
			Environment:  environment},
		doWithCommandeer,
		doWithConfig)

	if err != nil {
		if mustHaveConfigFile {
			return err
		}
		if err != hugolib.ErrNoConfigFile {
			return err
		}

	}

	c.configFiles = configFiles

	if l, ok := c.Cfg.Get("languagesSorted").(langs.Languages); ok {
		c.languagesConfigured = true
		c.languages = l
	}

	// Set some commonly used flags
	c.doLiveReload = running && !c.Cfg.GetBool("disableLiveReload")
	c.fastRenderMode = c.doLiveReload && !c.Cfg.GetBool("disableFastRender")
	c.showErrorInBrowser = c.doLiveReload && !c.Cfg.GetBool("disableBrowserError")

	// This is potentially double work, but we need to do this one more time now
	// that all the languages have been configured.
	if c.doWithCommandeer != nil {
		if err := c.doWithCommandeer(c); err != nil {
			return err
		}
	}

	logger, err := c.createLogger(config, running)
	if err != nil {
		return err
	}

	cfg.Logger = logger
	c.logger = logger

	createMemFs := config.GetBool("renderToMemory")

	if createMemFs {
		// Rendering to memoryFS, publish to Root regardless of publishDir.
		config.Set("publishDir", "/")
	}

	c.fsCreate.Do(func() {
		fs := hugofs.NewFrom(sourceFs, config)

		if c.destinationFs != nil {
			// Need to reuse the destination on server rebuilds.
			fs.Destination = c.destinationFs
		} else if createMemFs {
			// Hugo writes the output to memory instead of the disk.
			fs.Destination = new(afero.MemMapFs)
		}

		if c.fastRenderMode {
			// For now, fast render mode only. It should, however, be fast enough
			// for the full variant, too.
			changeDetector := &fileChangeDetector{
				// We use this detector to decide to do a Hot reload of a single path or not.
				// We need to filter out source maps and possibly some other to be able
				// to make that decision.
				irrelevantRe: regexp.MustCompile(`\.map$`),
			}

			changeDetector.PrepareNew()
			fs.Destination = hugofs.NewHashingFs(fs.Destination, changeDetector)
			c.changeDetector = changeDetector
		}

		err = c.initFs(fs)
		if err != nil {
			return
		}

		var h *hugolib.HugoSites

		h, err = hugolib.NewHugoSites(*c.DepsCfg)
		c.hugo = h

	})

	if err != nil {
		return err
	}

	cacheDir, err := helpers.GetCacheDir(sourceFs, config)
	if err != nil {
		return err
	}
	config.Set("cacheDir", cacheDir)

	cfg.Logger.INFO.Println("Using config file:", config.ConfigFileUsed())

	themeDir := c.hugo.PathSpec.GetFirstThemeDir()
	if themeDir != "" {
		if _, err := sourceFs.Stat(themeDir); os.IsNotExist(err) {
			return newSystemError("Unable to find theme Directory:", themeDir)
		}
	}

	dir, themeVersionMismatch, minVersion := c.isThemeVsHugoVersionMismatch(sourceFs)

	if themeVersionMismatch {
		name := filepath.Base(dir)
		cfg.Logger.ERROR.Printf("%s theme does not support Hugo version %s. Minimum version required is %s\n",
			strings.ToUpper(name), hugo.CurrentVersion.ReleaseVersion(), minVersion)
	}

	return nil

}
<|endoftext|>


<commit_before>package rpc

import (
	"github.com/mitchellh/packer/packer"
	"net/rpc"
)

// An implementation of packer.Builder where the builder is actually executed
// over an RPC connection.
type builder struct {
	client *rpc.Client
}

// BuilderServer wraps a packer.Builder implementation and makes it exportable
// as part of a Golang RPC server.
type BuilderServer struct {
	builder packer.Builder
}

type BuilderPrepareArgs struct {
	Config interface{}
}

type BuilderRunArgs struct {
	RPCAddress string
}

func Builder(client *rpc.Client) *builder {
	return &builder{client}
}

func (b *builder) Prepare(config interface{}) (err error) {
	cerr := b.client.Call("Builder.Prepare", &BuilderPrepareArgs{config}, &err)
	if cerr != nil {
		err = cerr
	}

	return
}

func (b *builder) Run(ui packer.Ui, hook packer.Hook) packer.Artifact {
	// Create and start the server for the Build and UI
	// TODO: Error handling
	server := rpc.NewServer()
	RegisterUi(server, ui)
	RegisterHook(server, hook)

	args := &BuilderRunArgs{serveSingleConn(server)}

	var reply string
	if err := b.client.Call("Builder.Run", args, &reply); err != nil {
		panic(err)
	}

	client, err := rpc.Dial("tcp", reply)
	if err != nil {
		panic(err)
	}

	return Artifact(client)
}

func (b *builder) Cancel() {

}

func (b *BuilderServer) Prepare(args *BuilderPrepareArgs, reply *error) error {
	err := b.builder.Prepare(args.Config)
	if err != nil {
		*reply = NewBasicError(err)
	}

	return nil
}

func (b *BuilderServer) Run(args *BuilderRunArgs, reply *string) error {
	client, err := rpc.Dial("tcp", args.RPCAddress)
	if err != nil {
		return err
	}

	hook := Hook(client)
	ui := &Ui{client}
	artifact := b.builder.Run(ui, hook)

	// Wrap the artifact
	server := rpc.NewServer()
	RegisterArtifact(server, artifact)

	*reply = serveSingleConn(server)
	return nil
}
<commit_msg>packer/rpc: Builder.Run() no longer blocks RPC connection<commit_after>package rpc

import (
	"encoding/gob"
	"github.com/mitchellh/packer/packer"
	"log"
	"net"
	"net/rpc"
)

// An implementation of packer.Builder where the builder is actually executed
// over an RPC connection.
type builder struct {
	client *rpc.Client
}

// BuilderServer wraps a packer.Builder implementation and makes it exportable
// as part of a Golang RPC server.
type BuilderServer struct {
	builder packer.Builder
}

type BuilderPrepareArgs struct {
	Config interface{}
}

type BuilderRunArgs struct {
	RPCAddress      string
	ResponseAddress string
}

type BuilderRunResponse struct {
	RPCAddress string
}

func Builder(client *rpc.Client) *builder {
	return &builder{client}
}

func (b *builder) Prepare(config interface{}) (err error) {
	cerr := b.client.Call("Builder.Prepare", &BuilderPrepareArgs{config}, &err)
	if cerr != nil {
		err = cerr
	}

	return
}

func (b *builder) Run(ui packer.Ui, hook packer.Hook) packer.Artifact {
	// Create and start the server for the Build and UI
	// TODO: Error handling
	server := rpc.NewServer()
	RegisterUi(server, ui)
	RegisterHook(server, hook)

	// Create a server for the response
	responseL := netListenerInRange(portRangeMin, portRangeMax)
	artifactAddress := make(chan string)
	go func() {
		defer responseL.Close()

		conn, err := responseL.Accept()
		if err != nil {
			log.Panic(err)
		}
		defer conn.Close()

		decoder := gob.NewDecoder(conn)

		var response BuilderRunResponse
		if err := decoder.Decode(&response); err != nil {
			log.Panic(err)
		}

		artifactAddress <- response.RPCAddress
	}()

	args := &BuilderRunArgs{
		serveSingleConn(server),
		responseL.Addr().String(),
	}

	if err := b.client.Call("Builder.Run", args, new(interface{})); err != nil {
		panic(err)
	}

	client, err := rpc.Dial("tcp", <-artifactAddress)
	if err != nil {
		panic(err)
	}

	return Artifact(client)
}

func (b *builder) Cancel() {

}

func (b *BuilderServer) Prepare(args *BuilderPrepareArgs, reply *error) error {
	err := b.builder.Prepare(args.Config)
	if err != nil {
		*reply = NewBasicError(err)
	}

	return nil
}

func (b *BuilderServer) Run(args *BuilderRunArgs, reply *interface{}) error {
	client, err := rpc.Dial("tcp", args.RPCAddress)
	if err != nil {
		return err
	}

	responseC, err := net.Dial("tcp", args.ResponseAddress)
	if err != nil {
		return err
	}

	responseWriter := gob.NewEncoder(responseC)

	// Run the build in a goroutine so we don't block the RPC connection
	go func () {
		defer responseC.Close()

		hook := Hook(client)
		ui := &Ui{client}
		artifact := b.builder.Run(ui, hook)

		// Wrap the artifact
		server := rpc.NewServer()
		RegisterArtifact(server, artifact)

		responseWriter.Encode(&BuilderRunResponse{serveSingleConn(server)})
	}()

	return nil
}
<|endoftext|>