<commit_before>package containerstore

import (
	"fmt"
	"io"
	"sync"
	"time"

	"github.com/cloudfoundry-incubator/executor"
	"github.com/cloudfoundry-incubator/executor/depot/event"
	"github.com/cloudfoundry-incubator/executor/depot/steps"
	"github.com/cloudfoundry-incubator/executor/depot/transformer"
	"github.com/cloudfoundry-incubator/garden"
	"github.com/cloudfoundry-incubator/garden/server"
	"github.com/cloudfoundry-incubator/runtime-schema/metric"
	"github.com/pivotal-golang/lager"
)

const ContainerInitializationFailedMessage = "failed to initialize container"
const ContainerExpirationMessage = "expired container"
const ContainerMissingMessage = "missing garden container"

const GardenContainerCreationDuration = metric.Duration("GardenContainerCreationDuration")

type storeNode struct {
	modifiedIndex uint

	info     executor.Container
	infoLock *sync.Mutex

	opLock          *sync.Mutex
	gardenClient    garden.Client
	gardenContainer garden.Container
	eventEmitter    event.Hub
	transformer     transformer.Transformer
	process         *runningProcess
	config          *ContainerConfig
}

type runningProcess struct {
	action            steps.Step
	done              chan struct{}
	healthCheckPassed <-chan struct{}
}

func newRunningProcess(action steps.Step, healthCheckPassed <-chan struct{}) *runningProcess {
	return &runningProcess{action: action, done: make(chan struct{}), healthCheckPassed: healthCheckPassed}
}

func newStoreNode(
	config *ContainerConfig,
	container executor.Container,
	gardenClient garden.Client,
	eventEmitter event.Hub,
	transformer transformer.Transformer,
) *storeNode {
	return &storeNode{
		config:        config,
		info:          container,
		infoLock:      &sync.Mutex{},
		opLock:        &sync.Mutex{},
		gardenClient:  gardenClient,
		eventEmitter:  eventEmitter,
		transformer:   transformer,
		modifiedIndex: 0,
	}
}

func (n *storeNode) acquireOpLock(logger lager.Logger) {
	startTime := time.Now()
	n.opLock.Lock()
	logger.Info("ops-lock-aquired", lager.Data{"lock-wait-time": time.Now().Sub(startTime)})
}

func (n *storeNode) releaseOpLock(logger lager.Logger) {
	n.opLock.Unlock()
	logger.Info("ops-lock-released")
}

func (n *storeNode) Info() executor.Container {
	n.infoLock.Lock()
	defer n.infoLock.Unlock()

	return n.info.Copy()
}

func (n *storeNode) GetFiles(logger lager.Logger, sourcePath string) (io.ReadCloser, error) {
	if n.gardenContainer == nil {
		return nil, executor.ErrContainerNotFound
	}

	return n.gardenContainer.StreamOut(garden.StreamOutSpec{Path: sourcePath, User: "root"})
}

func (n *storeNode) Initialize(logger lager.Logger, req *executor.RunRequest) error {
	n.infoLock.Lock()
	defer n.infoLock.Unlock()

	err := n.info.TransistionToInitialize(req)

	if err != nil {
		logger.Error("failed-to-initialize", err)
		return err
	}
	return nil
}

func (n *storeNode) Create(logger lager.Logger) error {
	n.acquireOpLock(logger)
	defer n.releaseOpLock(logger)

	var initialized bool
	n.infoLock.Lock()
	initialized = n.info.State == executor.StateInitializing
	n.infoLock.Unlock()
	if !initialized {
		logger.Error("failed-to-create", executor.ErrInvalidTransition)
		return executor.ErrInvalidTransition
	}

	logStreamer := logStreamerFromContainer(n.info)
	fmt.Fprintf(logStreamer.Stdout(), "Creating container\n")
	err := n.createInGarden(logger)
	if err != nil {
		logger.Error("failed-to-create-container", err)
		fmt.Fprintf(logStreamer.Stderr(), "Failed to create container\n")
		n.complete(logger, true, ContainerInitializationFailedMessage)
		return err
	}
	fmt.Fprintf(logStreamer.Stdout(), "Successfully created container\n")

	n.infoLock.Lock()
	err = n.info.TransistionToCreate()
	n.infoLock.Unlock()
	if err != nil {
		logger.Error("failed-to-transition-to-created", err)
		n.complete(logger, true, ContainerInitializationFailedMessage)
		return err
	}

	return nil
}

func (n *storeNode) createInGarden(logger lager.Logger) error {
	info := n.info.Copy()

	diskScope := garden.DiskLimitScopeExclusive
	if info.DiskScope == executor.TotalDiskLimit {
		diskScope = garden.DiskLimitScopeTotal
	}

	containerSpec := garden.ContainerSpec{
		Handle:     info.Guid,
		Privileged: info.Privileged,
		RootFSPath: info.RootFSPath,
		Limits: garden.Limits{
			Memory: garden.MemoryLimits{
				LimitInBytes: uint64(info.MemoryMB * 1024 * 1024),
			},
			Disk: garden.DiskLimits{
				ByteHard:  uint64(info.DiskMB * 1024 * 1024),
				InodeHard: n.config.INodeLimit,
				Scope:     diskScope,
			},
			CPU: garden.CPULimits{
				LimitInShares: uint64(float64(n.config.MaxCPUShares) * float64(info.CPUWeight) / 100.0),
			},
		},
		Properties: garden.Properties{
			ContainerOwnerProperty: n.config.OwnerName,
		},
	}

	for _, envVar := range info.Env {
		containerSpec.Env = append(containerSpec.Env, envVar.Name+"="+envVar.Value)
	}

	netOutRules := []garden.NetOutRule{}
	for _, rule := range info.EgressRules {
		if err := rule.Validate(); err != nil {
			logger.Error("invalid-egress-rule", err)
			return err
		}

		netOutRule, err := securityGroupRuleToNetOutRule(rule)
		if err != nil {
			logger.Error("failed-to-convert-to-net-out-rule", err)
			return err
		}

		netOutRules = append(netOutRules, netOutRule)
	}

	logger.Info("creating-container-in-garden")
	startTime := time.Now()
	gardenContainer, err := n.gardenClient.Create(containerSpec)
	if err != nil {
		logger.Error("failed-to-creating-container-in-garden", err)
		return err
	}
	GardenContainerCreationDuration.Send(time.Now().Sub(startTime))
	logger.Info("created-container-in-garden")

	for _, rule := range netOutRules {
		logger.Debug("net-out")
		err = gardenContainer.NetOut(rule)
		if err != nil {
			destroyErr := n.gardenClient.Destroy(n.info.Guid)
			if destroyErr != nil {
				logger.Error("failed-destroy-container", err)
			}
			logger.Error("net-out-failed", err)
			return err
		}
		logger.Debug("net-out-complete")
	}

	if info.Ports != nil {
		actualPortMappings := make([]executor.PortMapping, len(info.Ports))
		for i, portMapping := range info.Ports {
			logger.Debug("net-in")
			actualHost, actualContainerPort, err := gardenContainer.NetIn(uint32(portMapping.HostPort), uint32(portMapping.ContainerPort))
			if err != nil {
				logger.Error("net-in-failed", err)

				destroyErr := n.gardenClient.Destroy(info.Guid)
				if destroyErr != nil {
					logger.Error("failed-destroy-container", destroyErr)
				}

				return err
			}
			logger.Debug("net-in-complete")
			actualPortMappings[i].ContainerPort = uint16(actualContainerPort)
			actualPortMappings[i].HostPort = uint16(actualHost)
		}

		info.Ports = actualPortMappings
	}

	logger.Debug("container-info")
	gardenInfo, err := gardenContainer.Info()
	if err != nil {
		logger.Error("failed-container-info", err)

		destroyErr := n.gardenClient.Destroy(info.Guid)
		if destroyErr != nil {
			logger.Error("failed-destroy-container", destroyErr)
		}

		return err
	}
	logger.Debug("container-info-complete")

	info.ExternalIP = gardenInfo.ExternalIP
	n.gardenContainer = gardenContainer

	n.infoLock.Lock()
	n.info = info
	n.infoLock.Unlock()

	return nil
}

func (n *storeNode) Run(logger lager.Logger) error {
	n.acquireOpLock(logger)
	defer n.releaseOpLock(logger)

	if n.info.State != executor.StateCreated {
		logger.Error("failed-to-run", executor.ErrInvalidTransition)
		return executor.ErrInvalidTransition
	}

	logStreamer := logStreamerFromContainer(n.info)

	action, healthCheckPassed, err := n.transformer.StepsForContainer(logger, n.info, n.gardenContainer, logStreamer)
	if err != nil {
		logger.Error("failed-to-build-steps", err)
		return err
	}

	process := newRunningProcess(action, healthCheckPassed)
	n.process = process
	go n.run(logger)
	return nil
}

func (n *storeNode) run(logger lager.Logger) {
	resultCh := make(chan error)
	go func() {
		resultCh <- n.process.action.Perform()
	}()

	for {
		select {
		case err := <-resultCh:
			defer close(n.process.done)
			var failed bool
			var failureReason string

			if err != nil {
				failed = true
				failureReason = err.Error()
			}

			n.complete(logger, failed, failureReason)
			return

		case <-n.process.healthCheckPassed:
			n.infoLock.Lock()
			n.info.State = executor.StateRunning
			info := n.info
			n.infoLock.Unlock()
			go n.eventEmitter.Emit(executor.NewContainerRunningEvent(info))
		}
	}
}

func (n *storeNode) Stop(logger lager.Logger) error {
	n.acquireOpLock(logger)
	defer n.releaseOpLock(logger)

	return n.stop(logger)
}

func (n *storeNode) stop(logger lager.Logger) error {
	n.infoLock.Lock()
	n.info.RunResult.Stopped = true
	n.infoLock.Unlock()

	if n.process != nil {
		n.process.action.Cancel()
		<-n.process.done
	} else {
		n.complete(logger, true, "stopped-before-running")
	}
	return nil
}

func (n *storeNode) Destroy(logger lager.Logger) error {
	n.acquireOpLock(logger)
	defer n.releaseOpLock(logger)

	err := n.stop(logger)
	if err != nil {
		return err
	}

	logger.Debug("destroying-garden-container")
	err = n.gardenClient.Destroy(n.info.Guid)
	if err != nil {
		if _, ok := err.(garden.ContainerNotFoundError); ok {
			logger.Error("container-not-found-in-garden", err)
		} else if err.Error() == server.ErrConcurrentDestroy.Error() {
			logger.Error("container-destroy-in-progress", err)
		} else {
			logger.Error("failed-to-delete-garden-container", err)
			return err
		}
	}

	logger.Debug("destroyed-garden-container")
	return nil
}

func (n *storeNode) Expire(logger lager.Logger, now time.Time) bool {
	n.infoLock.Lock()
	defer n.infoLock.Unlock()

	if n.info.State != executor.StateReserved {
		return false
	}

	lifespan := now.Sub(time.Unix(0, n.info.AllocatedAt))
	if lifespan >= n.config.ReservedExpirationTime {
		n.info.TransitionToComplete(true, ContainerExpirationMessage)
		go n.eventEmitter.Emit(executor.NewContainerCompleteEvent(n.info))
		return true
	}

	return false
}

func (n *storeNode) Reap(logger lager.Logger) bool {
	n.infoLock.Lock()
	defer n.infoLock.Unlock()

	if n.info.IsCreated() {
		n.info.TransitionToComplete(true, ContainerMissingMessage)
		go n.eventEmitter.Emit(executor.NewContainerCompleteEvent(n.info))
		return true
	}

	return false
}

func (n *storeNode) complete(logger lager.Logger, failed bool, failureReason string) {
	n.infoLock.Lock()
	defer n.infoLock.Unlock()
	n.info.TransitionToComplete(failed, failureReason)

	go n.eventEmitter.Emit(executor.NewContainerCompleteEvent(n.info))
}
<commit_msg>pretty-print lock wait times in logs<commit_after>package containerstore

import (
	"fmt"
	"io"
	"sync"
	"time"

	"github.com/cloudfoundry-incubator/executor"
	"github.com/cloudfoundry-incubator/executor/depot/event"
	"github.com/cloudfoundry-incubator/executor/depot/steps"
	"github.com/cloudfoundry-incubator/executor/depot/transformer"
	"github.com/cloudfoundry-incubator/garden"
	"github.com/cloudfoundry-incubator/garden/server"
	"github.com/cloudfoundry-incubator/runtime-schema/metric"
	"github.com/pivotal-golang/lager"
)

const ContainerInitializationFailedMessage = "failed to initialize container"
const ContainerExpirationMessage = "expired container"
const ContainerMissingMessage = "missing garden container"

const GardenContainerCreationDuration = metric.Duration("GardenContainerCreationDuration")

type storeNode struct {
	modifiedIndex uint

	info     executor.Container
	infoLock *sync.Mutex

	opLock          *sync.Mutex
	gardenClient    garden.Client
	gardenContainer garden.Container
	eventEmitter    event.Hub
	transformer     transformer.Transformer
	process         *runningProcess
	config          *ContainerConfig
}

type runningProcess struct {
	action            steps.Step
	done              chan struct{}
	healthCheckPassed <-chan struct{}
}

func newRunningProcess(action steps.Step, healthCheckPassed <-chan struct{}) *runningProcess {
	return &runningProcess{action: action, done: make(chan struct{}), healthCheckPassed: healthCheckPassed}
}

func newStoreNode(
	config *ContainerConfig,
	container executor.Container,
	gardenClient garden.Client,
	eventEmitter event.Hub,
	transformer transformer.Transformer,
) *storeNode {
	return &storeNode{
		config:        config,
		info:          container,
		infoLock:      &sync.Mutex{},
		opLock:        &sync.Mutex{},
		gardenClient:  gardenClient,
		eventEmitter:  eventEmitter,
		transformer:   transformer,
		modifiedIndex: 0,
	}
}

func (n *storeNode) acquireOpLock(logger lager.Logger) {
	startTime := time.Now()
	n.opLock.Lock()
	logger.Info("ops-lock-aquired", lager.Data{"lock-wait-time": time.Now().Sub(startTime).String()})
}

func (n *storeNode) releaseOpLock(logger lager.Logger) {
	n.opLock.Unlock()
	logger.Info("ops-lock-released")
}

func (n *storeNode) Info() executor.Container {
	n.infoLock.Lock()
	defer n.infoLock.Unlock()

	return n.info.Copy()
}

func (n *storeNode) GetFiles(logger lager.Logger, sourcePath string) (io.ReadCloser, error) {
	if n.gardenContainer == nil {
		return nil, executor.ErrContainerNotFound
	}

	return n.gardenContainer.StreamOut(garden.StreamOutSpec{Path: sourcePath, User: "root"})
}

func (n *storeNode) Initialize(logger lager.Logger, req *executor.RunRequest) error {
	n.infoLock.Lock()
	defer n.infoLock.Unlock()

	err := n.info.TransistionToInitialize(req)

	if err != nil {
		logger.Error("failed-to-initialize", err)
		return err
	}
	return nil
}

func (n *storeNode) Create(logger lager.Logger) error {
	n.acquireOpLock(logger)
	defer n.releaseOpLock(logger)

	var initialized bool
	n.infoLock.Lock()
	initialized = n.info.State == executor.StateInitializing
	n.infoLock.Unlock()
	if !initialized {
		logger.Error("failed-to-create", executor.ErrInvalidTransition)
		return executor.ErrInvalidTransition
	}

	logStreamer := logStreamerFromContainer(n.info)
	fmt.Fprintf(logStreamer.Stdout(), "Creating container\n")
	err := n.createInGarden(logger)
	if err != nil {
		logger.Error("failed-to-create-container", err)
		fmt.Fprintf(logStreamer.Stderr(), "Failed to create container\n")
		n.complete(logger, true, ContainerInitializationFailedMessage)
		return err
	}
	fmt.Fprintf(logStreamer.Stdout(), "Successfully created container\n")

	n.infoLock.Lock()
	err = n.info.TransistionToCreate()
	n.infoLock.Unlock()
	if err != nil {
		logger.Error("failed-to-transition-to-created", err)
		n.complete(logger, true, ContainerInitializationFailedMessage)
		return err
	}

	return nil
}

func (n *storeNode) createInGarden(logger lager.Logger) error {
	info := n.info.Copy()

	diskScope := garden.DiskLimitScopeExclusive
	if info.DiskScope == executor.TotalDiskLimit {
		diskScope = garden.DiskLimitScopeTotal
	}

	containerSpec := garden.ContainerSpec{
		Handle:     info.Guid,
		Privileged: info.Privileged,
		RootFSPath: info.RootFSPath,
		Limits: garden.Limits{
			Memory: garden.MemoryLimits{
				LimitInBytes: uint64(info.MemoryMB * 1024 * 1024),
			},
			Disk: garden.DiskLimits{
				ByteHard:  uint64(info.DiskMB * 1024 * 1024),
				InodeHard: n.config.INodeLimit,
				Scope:     diskScope,
			},
			CPU: garden.CPULimits{
				LimitInShares: uint64(float64(n.config.MaxCPUShares) * float64(info.CPUWeight) / 100.0),
			},
		},
		Properties: garden.Properties{
			ContainerOwnerProperty: n.config.OwnerName,
		},
	}

	for _, envVar := range info.Env {
		containerSpec.Env = append(containerSpec.Env, envVar.Name+"="+envVar.Value)
	}

	netOutRules := []garden.NetOutRule{}
	for _, rule := range info.EgressRules {
		if err := rule.Validate(); err != nil {
			logger.Error("invalid-egress-rule", err)
			return err
		}

		netOutRule, err := securityGroupRuleToNetOutRule(rule)
		if err != nil {
			logger.Error("failed-to-convert-to-net-out-rule", err)
			return err
		}

		netOutRules = append(netOutRules, netOutRule)
	}

	logger.Info("creating-container-in-garden")
	startTime := time.Now()
	gardenContainer, err := n.gardenClient.Create(containerSpec)
	if err != nil {
		logger.Error("failed-to-creating-container-in-garden", err)
		return err
	}
	GardenContainerCreationDuration.Send(time.Now().Sub(startTime))
	logger.Info("created-container-in-garden")

	for _, rule := range netOutRules {
		logger.Debug("net-out")
		err = gardenContainer.NetOut(rule)
		if err != nil {
			destroyErr := n.gardenClient.Destroy(n.info.Guid)
			if destroyErr != nil {
				logger.Error("failed-destroy-container", err)
			}
			logger.Error("net-out-failed", err)
			return err
		}
		logger.Debug("net-out-complete")
	}

	if info.Ports != nil {
		actualPortMappings := make([]executor.PortMapping, len(info.Ports))
		for i, portMapping := range info.Ports {
			logger.Debug("net-in")
			actualHost, actualContainerPort, err := gardenContainer.NetIn(uint32(portMapping.HostPort), uint32(portMapping.ContainerPort))
			if err != nil {
				logger.Error("net-in-failed", err)

				destroyErr := n.gardenClient.Destroy(info.Guid)
				if destroyErr != nil {
					logger.Error("failed-destroy-container", destroyErr)
				}

				return err
			}
			logger.Debug("net-in-complete")
			actualPortMappings[i].ContainerPort = uint16(actualContainerPort)
			actualPortMappings[i].HostPort = uint16(actualHost)
		}

		info.Ports = actualPortMappings
	}

	logger.Debug("container-info")
	gardenInfo, err := gardenContainer.Info()
	if err != nil {
		logger.Error("failed-container-info", err)

		destroyErr := n.gardenClient.Destroy(info.Guid)
		if destroyErr != nil {
			logger.Error("failed-destroy-container", destroyErr)
		}

		return err
	}
	logger.Debug("container-info-complete")

	info.ExternalIP = gardenInfo.ExternalIP
	n.gardenContainer = gardenContainer

	n.infoLock.Lock()
	n.info = info
	n.infoLock.Unlock()

	return nil
}

func (n *storeNode) Run(logger lager.Logger) error {
	n.acquireOpLock(logger)
	defer n.releaseOpLock(logger)

	if n.info.State != executor.StateCreated {
		logger.Error("failed-to-run", executor.ErrInvalidTransition)
		return executor.ErrInvalidTransition
	}

	logStreamer := logStreamerFromContainer(n.info)

	action, healthCheckPassed, err := n.transformer.StepsForContainer(logger, n.info, n.gardenContainer, logStreamer)
	if err != nil {
		logger.Error("failed-to-build-steps", err)
		return err
	}

	process := newRunningProcess(action, healthCheckPassed)
	n.process = process
	go n.run(logger)
	return nil
}

func (n *storeNode) run(logger lager.Logger) {
	resultCh := make(chan error)
	go func() {
		resultCh <- n.process.action.Perform()
	}()

	for {
		select {
		case err := <-resultCh:
			defer close(n.process.done)
			var failed bool
			var failureReason string

			if err != nil {
				failed = true
				failureReason = err.Error()
			}

			n.complete(logger, failed, failureReason)
			return

		case <-n.process.healthCheckPassed:
			n.infoLock.Lock()
			n.info.State = executor.StateRunning
			info := n.info
			n.infoLock.Unlock()
			go n.eventEmitter.Emit(executor.NewContainerRunningEvent(info))
		}
	}
}

func (n *storeNode) Stop(logger lager.Logger) error {
	n.acquireOpLock(logger)
	defer n.releaseOpLock(logger)

	return n.stop(logger)
}

func (n *storeNode) stop(logger lager.Logger) error {
	n.infoLock.Lock()
	n.info.RunResult.Stopped = true
	n.infoLock.Unlock()

	if n.process != nil {
		n.process.action.Cancel()
		<-n.process.done
	} else {
		n.complete(logger, true, "stopped-before-running")
	}
	return nil
}

func (n *storeNode) Destroy(logger lager.Logger) error {
	n.acquireOpLock(logger)
	defer n.releaseOpLock(logger)

	err := n.stop(logger)
	if err != nil {
		return err
	}

	logger.Debug("destroying-garden-container")
	err = n.gardenClient.Destroy(n.info.Guid)
	if err != nil {
		if _, ok := err.(garden.ContainerNotFoundError); ok {
			logger.Error("container-not-found-in-garden", err)
		} else if err.Error() == server.ErrConcurrentDestroy.Error() {
			logger.Error("container-destroy-in-progress", err)
		} else {
			logger.Error("failed-to-delete-garden-container", err)
			return err
		}
	}

	logger.Debug("destroyed-garden-container")
	return nil
}

func (n *storeNode) Expire(logger lager.Logger, now time.Time) bool {
	n.infoLock.Lock()
	defer n.infoLock.Unlock()

	if n.info.State != executor.StateReserved {
		return false
	}

	lifespan := now.Sub(time.Unix(0, n.info.AllocatedAt))
	if lifespan >= n.config.ReservedExpirationTime {
		n.info.TransitionToComplete(true, ContainerExpirationMessage)
		go n.eventEmitter.Emit(executor.NewContainerCompleteEvent(n.info))
		return true
	}

	return false
}

func (n *storeNode) Reap(logger lager.Logger) bool {
	n.infoLock.Lock()
	defer n.infoLock.Unlock()

	if n.info.IsCreated() {
		n.info.TransitionToComplete(true, ContainerMissingMessage)
		go n.eventEmitter.Emit(executor.NewContainerCompleteEvent(n.info))
		return true
	}

	return false
}

func (n *storeNode) complete(logger lager.Logger, failed bool, failureReason string) {
	n.infoLock.Lock()
	defer n.infoLock.Unlock()
	n.info.TransitionToComplete(failed, failureReason)

	go n.eventEmitter.Emit(executor.NewContainerCompleteEvent(n.info))
}
<|endoftext|>


<commit_before>package itest

import (
	"context"
	"fmt"

	"github.com/btcsuite/btcd/wire"
	"github.com/lightningnetwork/lnd"
	"github.com/lightningnetwork/lnd/lncfg"
	"github.com/lightningnetwork/lnd/lnrpc"
	"github.com/lightningnetwork/lnd/lnrpc/invoicesrpc"
	"github.com/lightningnetwork/lnd/lnrpc/routerrpc"
	"github.com/lightningnetwork/lnd/lntest"
	"github.com/lightningnetwork/lnd/lntest/wait"
	"github.com/lightningnetwork/lnd/lntypes"
	"github.com/stretchr/testify/require"
)

// testMultiHopHtlcLocalChainClaim tests that in a multi-hop HTLC scenario, if
// we force close a channel with an incoming HTLC, and later find out the
// preimage via the witness beacon, we properly settle the HTLC on-chain using
// the HTLC success transaction in order to ensure we don't lose any funds.
func testMultiHopHtlcLocalChainClaim(net *lntest.NetworkHarness, t *harnessTest,
	alice, bob *lntest.HarnessNode, c commitType) {

	ctxb := context.Background()

	// First, we'll create a three hop network: Alice -> Bob -> Carol, with
	// Carol refusing to actually settle or directly cancel any HTLC's
	// self.
	aliceChanPoint, bobChanPoint, carol := createThreeHopNetwork(
		t, net, alice, bob, false, c,
	)

	// Clean up carol's node when the test finishes.
	defer shutdownAndAssert(net, t, carol)

	// With the network active, we'll now add a new hodl invoice at Carol's
	// end. Make sure the cltv expiry delta is large enough, otherwise Bob
	// won't send out the outgoing htlc.

	const invoiceAmt = 100000
	preimage := lntypes.Preimage{1, 2, 3}
	payHash := preimage.Hash()
	invoiceReq := &invoicesrpc.AddHoldInvoiceRequest{
		Value:      invoiceAmt,
		CltvExpiry: 40,
		Hash:       payHash[:],
	}
	ctxt, cancel := context.WithTimeout(ctxb, defaultTimeout)
	defer cancel()
	carolInvoice, err := carol.AddHoldInvoice(ctxt, invoiceReq)
	require.NoError(t.t, err)

	// Now that we've created the invoice, we'll send a single payment from
	// Alice to Carol. We won't wait for the response however, as Carol
	// will not immediately settle the payment.
	ctx, cancel := context.WithCancel(ctxb)
	defer cancel()

	_, err = alice.RouterClient.SendPaymentV2(
		ctx, &routerrpc.SendPaymentRequest{
			PaymentRequest: carolInvoice.PaymentRequest,
			TimeoutSeconds: 60,
			FeeLimitMsat:   noFeeLimitMsat,
		},
	)
	require.NoError(t.t, err)

	// At this point, all 3 nodes should now have an active channel with
	// the created HTLC pending on all of them.
	nodes := []*lntest.HarnessNode{alice, bob, carol}
	err = wait.NoError(func() error {
		return assertActiveHtlcs(nodes, payHash[:])
	}, defaultTimeout)
	require.NoError(t.t, err)

	// Wait for carol to mark invoice as accepted. There is a small gap to
	// bridge between adding the htlc to the channel and executing the exit
	// hop logic.
	waitForInvoiceAccepted(t, carol, payHash)

	// Increase the fee estimate so that the following force close tx will
	// be cpfp'ed.
	net.SetFeeEstimate(30000)

	// At this point, Bob decides that he wants to exit the channel
	// immediately, so he force closes his commitment transaction.
	ctxt, _ = context.WithTimeout(ctxb, channelCloseTimeout)
	bobForceClose := closeChannelAndAssertType(
		ctxt, t, net, bob, aliceChanPoint, c == commitTypeAnchors, true,
	)

	// Alice will sweep her commitment output immediately. If there are
	// anchors, Alice will also sweep hers.
	expectedTxes := 1
	if c == commitTypeAnchors {
		expectedTxes = 2
	}
	_, err = waitForNTxsInMempool(
		net.Miner.Node, expectedTxes, minerMempoolTimeout,
	)
	require.NoError(t.t, err)

	// Suspend Bob to force Carol to go to chain.
	restartBob, err := net.SuspendNode(bob)
	require.NoError(t.t, err)

	// Settle invoice. This will just mark the invoice as settled, as there
	// is no link anymore to remove the htlc from the commitment tx. For
	// this test, it is important to actually settle and not leave the
	// invoice in the accepted state, because without a known preimage, the
	// channel arbitrator won't go to chain.
	ctx, cancel = context.WithTimeout(ctxb, defaultTimeout)
	defer cancel()
	_, err = carol.SettleInvoice(ctx, &invoicesrpc.SettleInvoiceMsg{
		Preimage: preimage[:],
	})
	require.NoError(t.t, err)

	// We'll now mine enough blocks so Carol decides that she needs to go
	// on-chain to claim the HTLC as Bob has been inactive.
	numBlocks := padCLTV(uint32(invoiceReq.CltvExpiry -
		lncfg.DefaultIncomingBroadcastDelta))

	_, err = net.Miner.Node.Generate(numBlocks)
	require.NoError(t.t, err)

	// Carol's commitment transaction should now be in the mempool. If there
	// is an anchor, Carol will sweep that too.
	_, err = waitForNTxsInMempool(
		net.Miner.Node, expectedTxes, minerMempoolTimeout,
	)
	require.NoError(t.t, err)
	bobFundingTxid, err := lnd.GetChanPointFundingTxid(bobChanPoint)
	require.NoError(t.t, err)
	carolFundingPoint := wire.OutPoint{
		Hash:  *bobFundingTxid,
		Index: bobChanPoint.OutputIndex,
	}

	// Look up the closing transaction. It should be spending from the
	// funding transaction,
	closingTx := getSpendingTxInMempool(
		t, net.Miner.Node, minerMempoolTimeout, carolFundingPoint,
	)
	closingTxid := closingTx.TxHash()

	// Mine a block that should confirm the commit tx, the anchor if present
	// and the coinbase.
	block := mineBlocks(t, net, 1, expectedTxes)[0]
	require.Len(t.t, block.Transactions, expectedTxes+1)
	assertTxInBlock(t, block, &closingTxid)

	// Restart bob again.
	err = restartBob()
	require.NoError(t.t, err)

	// After the force close transacion is mined, Carol should broadcast her
	// second level HTLC transacion. Bob will broadcast a sweep tx to sweep
	// his output in the channel with Carol. He can do this immediately, as
	// the output is not timelocked since Carol was the one force closing.
	// If there are anchors on the commitment, Bob will also sweep his
	// anchor.
	expectedTxes = 2
	if c == commitTypeAnchors {
		expectedTxes = 3
	}
	txes, err := getNTxsFromMempool(
		net.Miner.Node, expectedTxes, minerMempoolTimeout,
	)
	require.NoError(t.t, err)

	// Both Carol's second level transaction and Bob's sweep should be
	// spending from the commitment transaction.
	assertAllTxesSpendFrom(t, txes, closingTxid)

	// At this point we suspend Alice to make sure she'll handle the
	// on-chain settle after a restart.
	restartAlice, err := net.SuspendNode(alice)
	require.NoError(t.t, err)

	// Mine a block to confirm the two transactions (+ the coinbase).
	block = mineBlocks(t, net, 1, expectedTxes)[0]
	require.Len(t.t, block.Transactions, expectedTxes+1)

	// Keep track of the second level tx maturity.
	carolSecondLevelCSV := uint32(defaultCSV)

	// When Bob notices Carol's second level transaction in the block, he
	// will extract the preimage and broadcast a second level tx to claim
	// the HTLC in his (already closed) channel with Alice.
	bobSecondLvlTx, err := waitForTxInMempool(
		net.Miner.Node, minerMempoolTimeout,
	)
	require.NoError(t.t, err)

	// It should spend from the commitment in the channel with Alice.
	tx, err := net.Miner.Node.GetRawTransaction(bobSecondLvlTx)
	require.NoError(t.t, err)

	require.Equal(
		t.t, *bobForceClose, tx.MsgTx().TxIn[0].PreviousOutPoint.Hash,
	)

	// At this point, Bob should have broadcast his second layer success
	// transaction, and should have sent it to the nursery for incubation.
	ctxt, _ = context.WithTimeout(ctxb, defaultTimeout)
	err = waitForNumChannelPendingForceClose(
		ctxt, bob, 1, func(c *lnrpcForceCloseChannel) error {
			if c.Channel.LocalBalance != 0 {
				return nil
			}

			if len(c.PendingHtlcs) != 1 {
				return fmt.Errorf("bob should have pending " +
					"htlc but doesn't")
			}

			if c.PendingHtlcs[0].Stage != 1 {
				return fmt.Errorf("bob's htlc should have "+
					"advanced to the first stage but was "+
					"stage: %v", c.PendingHtlcs[0].Stage)
			}

			return nil
		},
	)
	require.NoError(t.t, err)

	// We'll now mine a block which should confirm Bob's second layer
	// transaction.
	block = mineBlocks(t, net, 1, 1)[0]
	require.Len(t.t, block.Transactions, 2)
	assertTxInBlock(t, block, bobSecondLvlTx)

	// Keep track of Bob's second level maturity, and decrement our track
	// of Carol's.
	bobSecondLevelCSV := uint32(defaultCSV)
	carolSecondLevelCSV--

	// Now that the preimage from Bob has hit the chain, restart Alice to
	// ensure she'll pick it up.
	err = restartAlice()
	require.NoError(t.t, err)

	// If we then mine 3 additional blocks, Carol's second level tx should
	// mature, and she can pull the funds from it with a sweep tx.
	_, err = net.Miner.Node.Generate(carolSecondLevelCSV)
	require.NoError(t.t, err)
	bobSecondLevelCSV -= carolSecondLevelCSV

	carolSweep, err := waitForTxInMempool(net.Miner.Node, minerMempoolTimeout)
	require.NoError(t.t, err)

	// Mining one additional block, Bob's second level tx is mature, and he
	// can sweep the output.
	block = mineBlocks(t, net, bobSecondLevelCSV, 1)[0]
	assertTxInBlock(t, block, carolSweep)

	bobSweep, err := waitForTxInMempool(net.Miner.Node, minerMempoolTimeout)
	require.NoError(t.t, err)

	// Make sure it spends from the second level tx.
	tx, err = net.Miner.Node.GetRawTransaction(bobSweep)
	require.NoError(t.t, err)
	require.Equal(
		t.t, *bobSecondLvlTx, tx.MsgTx().TxIn[0].PreviousOutPoint.Hash,
	)

	// When we mine one additional block, that will confirm Bob's sweep.
	// Now Bob should have no pending channels anymore, as this just
	// resolved it by the confirmation of the sweep transaction.
	block = mineBlocks(t, net, 1, 1)[0]
	assertTxInBlock(t, block, bobSweep)

	ctxt, _ = context.WithTimeout(ctxb, defaultTimeout)
	err = waitForNumChannelPendingForceClose(ctxt, bob, 0, nil)
	require.NoError(t.t, err)
	assertNodeNumChannels(t, bob, 0)

	// Also Carol should have no channels left (open nor pending).
	err = waitForNumChannelPendingForceClose(ctxt, carol, 0, nil)
	require.NoError(t.t, err)
	assertNodeNumChannels(t, carol, 0)

	// Finally, check that the Alice's payment is correctly marked
	// succeeded.
	ctxt, _ = context.WithTimeout(ctxt, defaultTimeout)
	err = checkPaymentStatus(
		ctxt, alice, preimage, lnrpc.Payment_SUCCEEDED,
	)
	require.NoError(t.t, err)
}
<commit_msg>itest/local_chain_claim test: mine one less blocks for anchor sweeps<commit_after>package itest

import (
	"context"
	"fmt"

	"github.com/btcsuite/btcd/wire"
	"github.com/lightningnetwork/lnd"
	"github.com/lightningnetwork/lnd/lncfg"
	"github.com/lightningnetwork/lnd/lnrpc"
	"github.com/lightningnetwork/lnd/lnrpc/invoicesrpc"
	"github.com/lightningnetwork/lnd/lnrpc/routerrpc"
	"github.com/lightningnetwork/lnd/lntest"
	"github.com/lightningnetwork/lnd/lntest/wait"
	"github.com/lightningnetwork/lnd/lntypes"
	"github.com/stretchr/testify/require"
)

// testMultiHopHtlcLocalChainClaim tests that in a multi-hop HTLC scenario, if
// we force close a channel with an incoming HTLC, and later find out the
// preimage via the witness beacon, we properly settle the HTLC on-chain using
// the HTLC success transaction in order to ensure we don't lose any funds.
func testMultiHopHtlcLocalChainClaim(net *lntest.NetworkHarness, t *harnessTest,
	alice, bob *lntest.HarnessNode, c commitType) {

	ctxb := context.Background()

	// First, we'll create a three hop network: Alice -> Bob -> Carol, with
	// Carol refusing to actually settle or directly cancel any HTLC's
	// self.
	aliceChanPoint, bobChanPoint, carol := createThreeHopNetwork(
		t, net, alice, bob, false, c,
	)

	// Clean up carol's node when the test finishes.
	defer shutdownAndAssert(net, t, carol)

	// With the network active, we'll now add a new hodl invoice at Carol's
	// end. Make sure the cltv expiry delta is large enough, otherwise Bob
	// won't send out the outgoing htlc.

	const invoiceAmt = 100000
	preimage := lntypes.Preimage{1, 2, 3}
	payHash := preimage.Hash()
	invoiceReq := &invoicesrpc.AddHoldInvoiceRequest{
		Value:      invoiceAmt,
		CltvExpiry: 40,
		Hash:       payHash[:],
	}
	ctxt, cancel := context.WithTimeout(ctxb, defaultTimeout)
	defer cancel()
	carolInvoice, err := carol.AddHoldInvoice(ctxt, invoiceReq)
	require.NoError(t.t, err)

	// Now that we've created the invoice, we'll send a single payment from
	// Alice to Carol. We won't wait for the response however, as Carol
	// will not immediately settle the payment.
	ctx, cancel := context.WithCancel(ctxb)
	defer cancel()

	_, err = alice.RouterClient.SendPaymentV2(
		ctx, &routerrpc.SendPaymentRequest{
			PaymentRequest: carolInvoice.PaymentRequest,
			TimeoutSeconds: 60,
			FeeLimitMsat:   noFeeLimitMsat,
		},
	)
	require.NoError(t.t, err)

	// At this point, all 3 nodes should now have an active channel with
	// the created HTLC pending on all of them.
	nodes := []*lntest.HarnessNode{alice, bob, carol}
	err = wait.NoError(func() error {
		return assertActiveHtlcs(nodes, payHash[:])
	}, defaultTimeout)
	require.NoError(t.t, err)

	// Wait for carol to mark invoice as accepted. There is a small gap to
	// bridge between adding the htlc to the channel and executing the exit
	// hop logic.
	waitForInvoiceAccepted(t, carol, payHash)

	// Increase the fee estimate so that the following force close tx will
	// be cpfp'ed.
	net.SetFeeEstimate(30000)

	// At this point, Bob decides that he wants to exit the channel
	// immediately, so he force closes his commitment transaction.
	ctxt, _ = context.WithTimeout(ctxb, channelCloseTimeout)
	bobForceClose := closeChannelAndAssertType(
		ctxt, t, net, bob, aliceChanPoint, c == commitTypeAnchors, true,
	)

	// Alice will sweep her commitment output immediately. If there are
	// anchors, Alice will also sweep hers.
	expectedTxes := 1
	if c == commitTypeAnchors {
		expectedTxes = 2
	}
	_, err = waitForNTxsInMempool(
		net.Miner.Node, expectedTxes, minerMempoolTimeout,
	)
	require.NoError(t.t, err)

	// Suspend Bob to force Carol to go to chain.
	restartBob, err := net.SuspendNode(bob)
	require.NoError(t.t, err)

	// Settle invoice. This will just mark the invoice as settled, as there
	// is no link anymore to remove the htlc from the commitment tx. For
	// this test, it is important to actually settle and not leave the
	// invoice in the accepted state, because without a known preimage, the
	// channel arbitrator won't go to chain.
	ctx, cancel = context.WithTimeout(ctxb, defaultTimeout)
	defer cancel()
	_, err = carol.SettleInvoice(ctx, &invoicesrpc.SettleInvoiceMsg{
		Preimage: preimage[:],
	})
	require.NoError(t.t, err)

	// We'll now mine enough blocks so Carol decides that she needs to go
	// on-chain to claim the HTLC as Bob has been inactive.
	numBlocks := padCLTV(uint32(invoiceReq.CltvExpiry -
		lncfg.DefaultIncomingBroadcastDelta))

	_, err = net.Miner.Node.Generate(numBlocks)
	require.NoError(t.t, err)

	// Carol's commitment transaction should now be in the mempool. If there
	// is an anchor, Carol will sweep that too.
	_, err = waitForNTxsInMempool(
		net.Miner.Node, expectedTxes, minerMempoolTimeout,
	)
	require.NoError(t.t, err)
	bobFundingTxid, err := lnd.GetChanPointFundingTxid(bobChanPoint)
	require.NoError(t.t, err)
	carolFundingPoint := wire.OutPoint{
		Hash:  *bobFundingTxid,
		Index: bobChanPoint.OutputIndex,
	}

	// Look up the closing transaction. It should be spending from the
	// funding transaction,
	closingTx := getSpendingTxInMempool(
		t, net.Miner.Node, minerMempoolTimeout, carolFundingPoint,
	)
	closingTxid := closingTx.TxHash()

	// Mine a block that should confirm the commit tx, the anchor if present
	// and the coinbase.
	block := mineBlocks(t, net, 1, expectedTxes)[0]
	require.Len(t.t, block.Transactions, expectedTxes+1)
	assertTxInBlock(t, block, &closingTxid)

	// Restart bob again.
	err = restartBob()
	require.NoError(t.t, err)

	// After the force close transacion is mined, Carol should broadcast her
	// second level HTLC transacion. Bob will broadcast a sweep tx to sweep
	// his output in the channel with Carol. He can do this immediately, as
	// the output is not timelocked since Carol was the one force closing.
	// If there are anchors on the commitment, Bob will also sweep his
	// anchor.
	expectedTxes = 2
	if c == commitTypeAnchors {
		expectedTxes = 3
	}
	txes, err := getNTxsFromMempool(
		net.Miner.Node, expectedTxes, minerMempoolTimeout,
	)
	require.NoError(t.t, err)

	// Both Carol's second level transaction and Bob's sweep should be
	// spending from the commitment transaction.
	assertAllTxesSpendFrom(t, txes, closingTxid)

	// At this point we suspend Alice to make sure she'll handle the
	// on-chain settle after a restart.
	restartAlice, err := net.SuspendNode(alice)
	require.NoError(t.t, err)

	// Mine a block to confirm the two transactions (+ the coinbase).
	block = mineBlocks(t, net, 1, expectedTxes)[0]
	require.Len(t.t, block.Transactions, expectedTxes+1)

	var secondLevelMaturity uint32
	switch c {

	// If this is a channel of the anchor type, we will subtract one block
	// from the default CSV, as the Sweeper will handle the input, and the Sweeper
	// sweeps the input as soon as the lock expires.
	case commitTypeAnchors:
		secondLevelMaturity = defaultCSV - 1

	// For non-anchor channel types, the nursery will handle sweeping the
	// second level output, and it will wait one extra block before
	// sweeping it.
	default:
		secondLevelMaturity = defaultCSV
	}

	// Keep track of the second level tx maturity.
	carolSecondLevelCSV := secondLevelMaturity

	// When Bob notices Carol's second level transaction in the block, he
	// will extract the preimage and broadcast a second level tx to claim
	// the HTLC in his (already closed) channel with Alice.
	bobSecondLvlTx, err := waitForTxInMempool(
		net.Miner.Node, minerMempoolTimeout,
	)
	require.NoError(t.t, err)

	// It should spend from the commitment in the channel with Alice.
	tx, err := net.Miner.Node.GetRawTransaction(bobSecondLvlTx)
	require.NoError(t.t, err)

	require.Equal(
		t.t, *bobForceClose, tx.MsgTx().TxIn[0].PreviousOutPoint.Hash,
	)

	// At this point, Bob should have broadcast his second layer success
	// transaction, and should have sent it to the nursery for incubation.
	ctxt, _ = context.WithTimeout(ctxb, defaultTimeout)
	err = waitForNumChannelPendingForceClose(
		ctxt, bob, 1, func(c *lnrpcForceCloseChannel) error {
			if c.Channel.LocalBalance != 0 {
				return nil
			}

			if len(c.PendingHtlcs) != 1 {
				return fmt.Errorf("bob should have pending " +
					"htlc but doesn't")
			}

			if c.PendingHtlcs[0].Stage != 1 {
				return fmt.Errorf("bob's htlc should have "+
					"advanced to the first stage but was "+
					"stage: %v", c.PendingHtlcs[0].Stage)
			}

			return nil
		},
	)
	require.NoError(t.t, err)

	// We'll now mine a block which should confirm Bob's second layer
	// transaction.
	block = mineBlocks(t, net, 1, 1)[0]
	require.Len(t.t, block.Transactions, 2)
	assertTxInBlock(t, block, bobSecondLvlTx)

	// Keep track of Bob's second level maturity, and decrement our track
	// of Carol's.
	bobSecondLevelCSV := secondLevelMaturity
	carolSecondLevelCSV--

	// Now that the preimage from Bob has hit the chain, restart Alice to
	// ensure she'll pick it up.
	err = restartAlice()
	require.NoError(t.t, err)

	// If we then mine 3 additional blocks, Carol's second level tx should
	// mature, and she can pull the funds from it with a sweep tx.
	_, err = net.Miner.Node.Generate(carolSecondLevelCSV)
	require.NoError(t.t, err)
	bobSecondLevelCSV -= carolSecondLevelCSV

	carolSweep, err := waitForTxInMempool(net.Miner.Node, minerMempoolTimeout)
	require.NoError(t.t, err)

	// Mining one additional block, Bob's second level tx is mature, and he
	// can sweep the output.
	block = mineBlocks(t, net, bobSecondLevelCSV, 1)[0]
	assertTxInBlock(t, block, carolSweep)

	bobSweep, err := waitForTxInMempool(net.Miner.Node, minerMempoolTimeout)
	require.NoError(t.t, err)

	// Make sure it spends from the second level tx.
	tx, err = net.Miner.Node.GetRawTransaction(bobSweep)
	require.NoError(t.t, err)
	require.Equal(
		t.t, *bobSecondLvlTx, tx.MsgTx().TxIn[0].PreviousOutPoint.Hash,
	)

	// When we mine one additional block, that will confirm Bob's sweep.
	// Now Bob should have no pending channels anymore, as this just
	// resolved it by the confirmation of the sweep transaction.
	block = mineBlocks(t, net, 1, 1)[0]
	assertTxInBlock(t, block, bobSweep)

	ctxt, _ = context.WithTimeout(ctxb, defaultTimeout)
	err = waitForNumChannelPendingForceClose(ctxt, bob, 0, nil)
	require.NoError(t.t, err)
	assertNodeNumChannels(t, bob, 0)

	// Also Carol should have no channels left (open nor pending).
	err = waitForNumChannelPendingForceClose(ctxt, carol, 0, nil)
	require.NoError(t.t, err)
	assertNodeNumChannels(t, carol, 0)

	// Finally, check that the Alice's payment is correctly marked
	// succeeded.
	ctxt, _ = context.WithTimeout(ctxt, defaultTimeout)
	err = checkPaymentStatus(
		ctxt, alice, preimage, lnrpc.Payment_SUCCEEDED,
	)
	require.NoError(t.t, err)
}
<|endoftext|>


<commit_before>package gitsync

import (
	"bytes"
	"encoding/gob"
	log "github.com/ngmoco/timber"
	"net"
)

var (
	Port             = 9999 // mDNS/Bonjour uses 5353
	IP4MulticastAddr = &net.UDPAddr{
		IP:   net.ParseIP("224.0.0.251"),
		Port: Port,
	}

	IP6MulticastAddr = &net.UDPAddr{
		IP:   net.ParseIP("ff02::fb"),
		Port: Port,
	}
)

func init() {
	gob.Register(GitChange{})
}

func establishConnPair(addr *net.UDPAddr) (recvConn, sendConn *net.UDPConn, err error) {
	if recvConn, err = net.ListenMulticastUDP("udp", nil, addr); err != nil {
		return
	}

	if sendConn, err = net.DialUDP("udp", nil, addr); err != nil {
		return
	}

	return
}

// NetIO shares GitChanges on toNet with the network via a multicast group. It
// will pass on GitChanges from the network via fromNet. It uniques the daemon
// instance by changing the .Name member to be name@<host IP>/<original .Name)
func NetIO(l log.Logger, username string, addr *net.UDPAddr, fromNet, toNet chan GitChange) {
	var (
		err                error
		recvConn, sendConn *net.UDPConn // UDP connections to allow us to send and	receive change updates
	)

	l.Info("Joining %v multicast(%t) group", addr, addr.IP.IsMulticast())
	if recvConn, sendConn, err = establishConnPair(addr); err != nil {
		l.Critical("Error joining listening: %s\n", addr, err)
		return
	}

	l.Info("Successfully joined %v multicast(%t) group", addr, addr.IP.IsMulticast())
	defer recvConn.Close()
	defer sendConn.Close()
	hostIp := sendConn.LocalAddr().(*net.UDPAddr).IP.String()

	term := false
	defer func() { term = true }()
	rawFromNet := make(chan []byte, 128)
	go func() {
		for !term {
			b := make([]byte, 1024)

			if n, err := recvConn.Read(b); err != nil {
				l.Critical("Cannot read socket: %s", err)
				continue
			} else {
				rawFromNet <- b[:n]
			}
		}
	}()

	for {
		select {
		case req, ok := <-toNet:
			if !ok {
				return
			}

			req.User = username
			req.HostIp = hostIp

			l.Info("Sending %+v", req)
			buf := &bytes.Buffer{}
			enc := gob.NewEncoder(buf)

			if err := enc.Encode(req); err != nil {
				l.Critical("%s", err)
				continue
			}

			l.Fine("Sending %+v", buf.Bytes())
			if _, err := sendConn.Write(buf.Bytes()); err != nil {
				l.Critical("%s", err)
				continue
			}

		case resp := <-rawFromNet:
			var msg GitChange
			dec := gob.NewDecoder(bytes.NewReader(resp))

			if err := dec.Decode(&msg); err != nil {
				l.Critical("%s", err)
				continue
			} else {
				l.Debug("received %+v", msg)
			}

			// a host should ignore changes made by its
			// own user
			if msg.User != username {
				fromNet <- msg
			}
		}
	}
}
<commit_msg>No more restriction on user seeing his own changes<commit_after>package gitsync

import (
	"bytes"
	"encoding/gob"
	log "github.com/ngmoco/timber"
	"net"
)

var (
	Port             = 9999 // mDNS/Bonjour uses 5353
	IP4MulticastAddr = &net.UDPAddr{
		IP:   net.ParseIP("224.0.0.251"),
		Port: Port,
	}

	IP6MulticastAddr = &net.UDPAddr{
		IP:   net.ParseIP("ff02::fb"),
		Port: Port,
	}
)

func init() {
	gob.Register(GitChange{})
}

func establishConnPair(addr *net.UDPAddr) (recvConn, sendConn *net.UDPConn, err error) {
	if recvConn, err = net.ListenMulticastUDP("udp", nil, addr); err != nil {
		return
	}

	if sendConn, err = net.DialUDP("udp", nil, addr); err != nil {
		return
	}

	return
}

// NetIO shares GitChanges on toNet with the network via a multicast group. It
// will pass on GitChanges from the network via fromNet. It uniques the daemon
// instance by changing the .Name member to be name@<host IP>/<original .Name)
func NetIO(l log.Logger, username string, addr *net.UDPAddr, fromNet, toNet chan GitChange) {
	var (
		err                error
		recvConn, sendConn *net.UDPConn // UDP connections to allow us to send and	receive change updates
	)

	l.Info("Joining %v multicast(%t) group", addr, addr.IP.IsMulticast())
	if recvConn, sendConn, err = establishConnPair(addr); err != nil {
		l.Critical("Error joining listening: %s\n", addr, err)
		return
	}

	l.Info("Successfully joined %v multicast(%t) group", addr, addr.IP.IsMulticast())
	defer recvConn.Close()
	defer sendConn.Close()
	hostIp := sendConn.LocalAddr().(*net.UDPAddr).IP.String()

	term := false
	defer func() { term = true }()
	rawFromNet := make(chan []byte, 128)
	go func() {
		for !term {
			b := make([]byte, 1024)

			if n, err := recvConn.Read(b); err != nil {
				l.Critical("Cannot read socket: %s", err)
				continue
			} else {
				rawFromNet <- b[:n]
			}
		}
	}()

	for {
		select {
		case req, ok := <-toNet:
			if !ok {
				return
			}

			req.User = username
			req.HostIp = hostIp

			l.Info("Sending %+v", req)
			buf := &bytes.Buffer{}
			enc := gob.NewEncoder(buf)

			if err := enc.Encode(req); err != nil {
				l.Critical("%s", err)
				continue
			}

			l.Fine("Sending %+v", buf.Bytes())
			if _, err := sendConn.Write(buf.Bytes()); err != nil {
				l.Critical("%s", err)
				continue
			}

		case resp := <-rawFromNet:
			var msg GitChange
			dec := gob.NewDecoder(bytes.NewReader(resp))

			if err := dec.Decode(&msg); err != nil {
				l.Critical("%s", err)
				continue
			} else {
				l.Debug("received %+v", msg)
			}

			fromNet <- msg
		}
	}
}
<|endoftext|>


<commit_before>package xmetrics

import (
	"errors"
	"fmt"

	"github.com/go-kit/kit/metrics"
	gokitprometheus "github.com/go-kit/kit/metrics/prometheus"
	"github.com/go-kit/kit/metrics/provider"
	"github.com/prometheus/client_golang/prometheus"
)

const (
	CounterType   = "counter"
	GaugeType     = "gauge"
	HistogramType = "histogram"
	SummaryType   = "summary"
)

// Registry is the core abstraction for this package.  It is a Prometheus registry and a go-kit metrics.Provider all in one.
//
// The Provider implementation works slightly differently than the go-kit implementation.  For any metric that is already defined
// the provider returns a new go-kit wrapper for that metric.  Additionally, new metrics (including ad hoc metrics) are cached
// and returned by subsequent calles to the Provider methods.
type Registry interface {
	provider.Provider
	prometheus.Gatherer
	prometheus.Registerer
}

// registry is the internal Registry implementation
type registry struct {
	*prometheus.Registry

	namespace string
	subsystem string
	cache     map[string]prometheus.Collector
}

func (r *registry) NewCounter(name string) metrics.Counter {
	var counterVec *prometheus.CounterVec

	if existing, ok := r.cache[name]; ok {
		if counterVec, ok = existing.(*prometheus.CounterVec); !ok {
			panic(fmt.Errorf("The metric %s is not a counter", name))
		}
	} else {
		counterVec = prometheus.NewCounterVec(prometheus.CounterOpts{
			Namespace: r.namespace,
			Subsystem: r.subsystem,
			Name:      name,
			Help:      name,
		}, []string{})

		if err := r.Registry.Register(counterVec); err != nil {
			if already, ok := err.(prometheus.AlreadyRegisteredError); ok {
				counterVec = already.ExistingCollector.(*prometheus.CounterVec)
			} else {
				panic(err)
			}
		}

		r.cache[name] = counterVec
	}

	return gokitprometheus.NewCounter(counterVec)
}

func (r *registry) NewGauge(name string) metrics.Gauge {
	var gaugeVec *prometheus.GaugeVec

	if existing, ok := r.cache[name]; ok {
		if gaugeVec, ok = existing.(*prometheus.GaugeVec); !ok {
			panic(fmt.Errorf("The metric %s is not a gauge", name))
		}
	} else {
		gaugeVec = prometheus.NewGaugeVec(prometheus.GaugeOpts{
			Namespace: r.namespace,
			Subsystem: r.subsystem,
			Name:      name,
			Help:      name,
		}, []string{})

		if err := r.Registry.Register(gaugeVec); err != nil {
			if already, ok := err.(prometheus.AlreadyRegisteredError); ok {
				gaugeVec = already.ExistingCollector.(*prometheus.GaugeVec)
			} else {
				panic(err)
			}
		}

		r.cache[name] = gaugeVec
	}

	return gokitprometheus.NewGauge(gaugeVec)
}

// NewHistogram will return a Histogram for either a Summary or Histogram.  This is different
// behavior from metrics.Provider.
func (r *registry) NewHistogram(name string, buckets int) metrics.Histogram {
	if existing, ok := r.cache[name]; ok {
		switch vec := existing.(type) {
		case *prometheus.HistogramVec:
			return gokitprometheus.NewHistogram(vec)
		case *prometheus.SummaryVec:
			return gokitprometheus.NewSummary(vec)
		default:
			panic(fmt.Errorf("The metric %s is not a histogram or summary", name))
		}
	}

	histogramVec := prometheus.NewHistogramVec(prometheus.HistogramOpts{
		Namespace: r.namespace,
		Subsystem: r.subsystem,
		Name:      name,
		Help:      name,
	}, []string{})

	if err := r.Registry.Register(histogramVec); err != nil {
		if already, ok := err.(prometheus.AlreadyRegisteredError); ok {
			histogramVec = already.ExistingCollector.(*prometheus.HistogramVec)
		} else {
			panic(err)
		}
	}

	r.cache[name] = histogramVec
	return gokitprometheus.NewHistogram(histogramVec)
}

func (r *registry) Stop() {
}

func NewRegistry(o *Options) (Registry, error) {
	var (
		defaultNamespace = o.namespace()
		defaultSubsystem = o.subsystem()
		pr               *prometheus.Registry
	)

	if o.pedantic() {
		pr = prometheus.NewPedanticRegistry()
	} else {
		pr = prometheus.NewRegistry()
	}

	r := &registry{
		Registry:  pr,
		namespace: defaultNamespace,
		subsystem: defaultSubsystem,
		cache:     make(map[string]prometheus.Collector),
	}

	for name, m := range o.metrics() {
		if len(name) == 0 {
			return nil, errors.New("Metric names cannot be empty")
		}

		var (
			namespace = m.Namespace
			subsystem = m.Subsystem
			help      = m.Help
		)

		if len(namespace) == 0 {
			namespace = defaultNamespace
		}

		if len(subsystem) == 0 {
			subsystem = defaultSubsystem
		}

		if len(help) == 0 {
			help = name
		}

		switch m.Type {
		case CounterType:
			counterVec := prometheus.NewCounterVec(prometheus.CounterOpts{
				Namespace:   namespace,
				Subsystem:   subsystem,
				Name:        name,
				Help:        help,
				ConstLabels: prometheus.Labels(m.Labels),
			}, []string{})

			if err := r.Registry.Register(counterVec); err != nil {
				return nil, fmt.Errorf("Error while preregistering metric %s: %s", name, err)
			}

			r.cache[name] = counterVec

		case GaugeType:
			gaugeVec := prometheus.NewGaugeVec(prometheus.GaugeOpts{
				Namespace:   namespace,
				Subsystem:   subsystem,
				Name:        name,
				Help:        help,
				ConstLabels: prometheus.Labels(m.Labels),
			}, []string{})

			if err := r.Registry.Register(gaugeVec); err != nil {
				return nil, fmt.Errorf("Error while preregistering metric %s: %s", name, err)
			}

			r.cache[name] = gaugeVec

		case HistogramType:
			histogramVec := prometheus.NewHistogramVec(prometheus.HistogramOpts{
				Namespace:   namespace,
				Subsystem:   subsystem,
				Name:        name,
				Help:        help,
				Buckets:     m.Buckets,
				ConstLabels: prometheus.Labels(m.Labels),
			}, []string{})

			if err := r.Registry.Register(histogramVec); err != nil {
				return nil, fmt.Errorf("Error while preregistering metric %s: %s", name, err)
			}

			r.cache[name] = histogramVec

		case SummaryType:
			summaryVec := prometheus.NewSummaryVec(prometheus.SummaryOpts{
				Namespace:   namespace,
				Subsystem:   subsystem,
				Name:        name,
				Help:        help,
				Objectives:  m.Objectives,
				MaxAge:      m.MaxAge,
				AgeBuckets:  m.AgeBuckets,
				BufCap:      m.BufCap,
				ConstLabels: prometheus.Labels(m.Labels),
			}, []string{})

			if err := r.Registry.Register(summaryVec); err != nil {
				return nil, fmt.Errorf("Error while preregistering metric %s: %s", name, err)
			}

			r.cache[name] = summaryVec

		default:
			return nil, fmt.Errorf("Unsupported metric type: %s", m.Type)
		}
	}

	return r, nil
}
<commit_msg>Added a prometheus-specific interface for direct interaction with Prometheus code<commit_after>package xmetrics

import (
	"errors"
	"fmt"

	"github.com/go-kit/kit/metrics"
	gokitprometheus "github.com/go-kit/kit/metrics/prometheus"
	"github.com/go-kit/kit/metrics/provider"
	"github.com/prometheus/client_golang/prometheus"
)

const (
	CounterType   = "counter"
	GaugeType     = "gauge"
	HistogramType = "histogram"
	SummaryType   = "summary"
)

// PrometheusProvider is a Prometheus-specific version of go-kit's metrics.Provider.  Use this interface
// when interacting directly with Prometheus.
type PrometheusProvider interface {
	NewCounterVec(string) *prometheus.CounterVec
	NewGaugeVec(string) *prometheus.GaugeVec
	NewHistogramVec(string) *prometheus.HistogramVec
	NewSummaryVec(string) *prometheus.SummaryVec
}

// Registry is the core abstraction for this package.  It is a Prometheus registry and a go-kit metrics.Provider all in one.
//
// The Provider implementation works slightly differently than the go-kit implementation.  For any metric that is already defined
// the provider returns a new go-kit wrapper for that metric.  Additionally, new metrics (including ad hoc metrics) are cached
// and returned by subsequent calles to the Provider methods.
type Registry interface {
	PrometheusProvider
	provider.Provider
	prometheus.Gatherer
	prometheus.Registerer
}

// registry is the internal Registry implementation
type registry struct {
	*prometheus.Registry

	namespace string
	subsystem string
	cache     map[string]prometheus.Collector
}

func (r *registry) NewCounterVec(name string) *prometheus.CounterVec {
	var counterVec *prometheus.CounterVec

	if existing, ok := r.cache[name]; ok {
		if counterVec, ok = existing.(*prometheus.CounterVec); !ok {
			panic(fmt.Errorf("The metric %s is not a counter", name))
		}
	} else {
		counterVec = prometheus.NewCounterVec(prometheus.CounterOpts{
			Namespace: r.namespace,
			Subsystem: r.subsystem,
			Name:      name,
			Help:      name,
		}, []string{})

		if err := r.Registry.Register(counterVec); err != nil {
			if already, ok := err.(prometheus.AlreadyRegisteredError); ok {
				counterVec = already.ExistingCollector.(*prometheus.CounterVec)
			} else {
				panic(err)
			}
		}

		r.cache[name] = counterVec
	}

	return counterVec
}

func (r *registry) NewCounter(name string) metrics.Counter {
	return gokitprometheus.NewCounter(r.NewCounterVec(name))
}

func (r *registry) NewGaugeVec(name string) *prometheus.GaugeVec {
	var gaugeVec *prometheus.GaugeVec

	if existing, ok := r.cache[name]; ok {
		if gaugeVec, ok = existing.(*prometheus.GaugeVec); !ok {
			panic(fmt.Errorf("The metric %s is not a gauge", name))
		}
	} else {
		gaugeVec = prometheus.NewGaugeVec(prometheus.GaugeOpts{
			Namespace: r.namespace,
			Subsystem: r.subsystem,
			Name:      name,
			Help:      name,
		}, []string{})

		if err := r.Registry.Register(gaugeVec); err != nil {
			if already, ok := err.(prometheus.AlreadyRegisteredError); ok {
				gaugeVec = already.ExistingCollector.(*prometheus.GaugeVec)
			} else {
				panic(err)
			}
		}

		r.cache[name] = gaugeVec
	}

	return gaugeVec
}

func (r *registry) NewGauge(name string) metrics.Gauge {
	return gokitprometheus.NewGauge(r.NewGaugeVec(name))
}

func (r *registry) NewHistogramVec(name string) *prometheus.HistogramVec {
	var histogramVec *prometheus.HistogramVec

	if existing, ok := r.cache[name]; ok {
		if histogramVec, ok = existing.(*prometheus.HistogramVec); !ok {
			panic(fmt.Errorf("The metric %s is not a histogram", name))
		}
	} else {
		histogramVec := prometheus.NewHistogramVec(prometheus.HistogramOpts{
			Namespace: r.namespace,
			Subsystem: r.subsystem,
			Name:      name,
			Help:      name,
		}, []string{})

		if err := r.Registry.Register(histogramVec); err != nil {
			if already, ok := err.(prometheus.AlreadyRegisteredError); ok {
				histogramVec = already.ExistingCollector.(*prometheus.HistogramVec)
			} else {
				panic(err)
			}
		}

		r.cache[name] = histogramVec
	}

	return histogramVec
}

func (r *registry) NewSummaryVec(name string) *prometheus.SummaryVec {
	var summaryVec *prometheus.SummaryVec

	if existing, ok := r.cache[name]; ok {
		if summaryVec, ok = existing.(*prometheus.SummaryVec); !ok {
			panic(fmt.Errorf("The metric %s is not a histogram", name))
		}
	} else {
		summaryVec := prometheus.NewSummaryVec(prometheus.SummaryOpts{
			Namespace: r.namespace,
			Subsystem: r.subsystem,
			Name:      name,
			Help:      name,
		}, []string{})

		if err := r.Registry.Register(summaryVec); err != nil {
			if already, ok := err.(prometheus.AlreadyRegisteredError); ok {
				summaryVec = already.ExistingCollector.(*prometheus.SummaryVec)
			} else {
				panic(err)
			}
		}

		r.cache[name] = summaryVec
	}

	return summaryVec
}

// NewHistogram will return a Histogram for either a Summary or Histogram.  This is different
// behavior from metrics.Provider.
func (r *registry) NewHistogram(name string, _ int) metrics.Histogram {
	// we allow either a summary or a histogram to be wrapped as a go-kit Histogram
	if existing, ok := r.cache[name]; ok {
		switch vec := existing.(type) {
		case *prometheus.HistogramVec:
			return gokitprometheus.NewHistogram(vec)
		case *prometheus.SummaryVec:
			return gokitprometheus.NewSummary(vec)
		default:
			panic(fmt.Errorf("The metric %s is not a histogram or summary", name))
		}
	}

	return gokitprometheus.NewHistogram(r.NewHistogramVec(name))
}

func (r *registry) Stop() {
}

func NewRegistry(o *Options) (Registry, error) {
	var (
		defaultNamespace = o.namespace()
		defaultSubsystem = o.subsystem()
		pr               *prometheus.Registry
	)

	if o.pedantic() {
		pr = prometheus.NewPedanticRegistry()
	} else {
		pr = prometheus.NewRegistry()
	}

	r := &registry{
		Registry:  pr,
		namespace: defaultNamespace,
		subsystem: defaultSubsystem,
		cache:     make(map[string]prometheus.Collector),
	}

	for name, m := range o.metrics() {
		if len(name) == 0 {
			return nil, errors.New("Metric names cannot be empty")
		}

		var (
			namespace = m.Namespace
			subsystem = m.Subsystem
			help      = m.Help
		)

		if len(namespace) == 0 {
			namespace = defaultNamespace
		}

		if len(subsystem) == 0 {
			subsystem = defaultSubsystem
		}

		if len(help) == 0 {
			help = name
		}

		switch m.Type {
		case CounterType:
			counterVec := prometheus.NewCounterVec(prometheus.CounterOpts{
				Namespace:   namespace,
				Subsystem:   subsystem,
				Name:        name,
				Help:        help,
				ConstLabels: prometheus.Labels(m.Labels),
			}, []string{})

			if err := r.Registry.Register(counterVec); err != nil {
				return nil, fmt.Errorf("Error while preregistering metric %s: %s", name, err)
			}

			r.cache[name] = counterVec

		case GaugeType:
			gaugeVec := prometheus.NewGaugeVec(prometheus.GaugeOpts{
				Namespace:   namespace,
				Subsystem:   subsystem,
				Name:        name,
				Help:        help,
				ConstLabels: prometheus.Labels(m.Labels),
			}, []string{})

			if err := r.Registry.Register(gaugeVec); err != nil {
				return nil, fmt.Errorf("Error while preregistering metric %s: %s", name, err)
			}

			r.cache[name] = gaugeVec

		case HistogramType:
			histogramVec := prometheus.NewHistogramVec(prometheus.HistogramOpts{
				Namespace:   namespace,
				Subsystem:   subsystem,
				Name:        name,
				Help:        help,
				Buckets:     m.Buckets,
				ConstLabels: prometheus.Labels(m.Labels),
			}, []string{})

			if err := r.Registry.Register(histogramVec); err != nil {
				return nil, fmt.Errorf("Error while preregistering metric %s: %s", name, err)
			}

			r.cache[name] = histogramVec

		case SummaryType:
			summaryVec := prometheus.NewSummaryVec(prometheus.SummaryOpts{
				Namespace:   namespace,
				Subsystem:   subsystem,
				Name:        name,
				Help:        help,
				Objectives:  m.Objectives,
				MaxAge:      m.MaxAge,
				AgeBuckets:  m.AgeBuckets,
				BufCap:      m.BufCap,
				ConstLabels: prometheus.Labels(m.Labels),
			}, []string{})

			if err := r.Registry.Register(summaryVec); err != nil {
				return nil, fmt.Errorf("Error while preregistering metric %s: %s", name, err)
			}

			r.cache[name] = summaryVec

		default:
			return nil, fmt.Errorf("Unsupported metric type: %s", m.Type)
		}
	}

	return r, nil
}
<|endoftext|>


<commit_before>// Copyright 2016 Koichi Shiraishi. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

package quickfix

import (
	"bytes"
	"path/filepath"
	"regexp"
	"strconv"
	"strings"

	"nvim-go/context"

	"github.com/garyburd/neovim-go/vim"
)

// ErrorlistData represents an item in a quickfix and locationlist.
type ErrorlistData struct {
	// Buffer number
	Bufnr int `msgpack:"bufnr,omitempty"`

	// Name of a file; only used when bufnr is not present or it is invalid.
	FileName string `msgpack:"filename,omitempty"`

	// Line number in the file.
	LNum int `msgpack:"lnum,omitempty"`

	// Column number (first column is 1).
	Col int `msgpack:"col,omitempty"`

	// When Vcol is != 0,  Col is visual column.
	VCol int `msgpack:"vcol,omitempty"`

	// Error number.
	Nr int `msgpack:"nr,omitempty"`

	// Search pattern used to locate the error.
	Pattern string `msgpack:"pattern,omitempty"`

	// Description of the error.
	Text string `msgpack:"text,omitempty"`

	// Single-character error type, 'E', 'W', etc.
	Type string `msgpack:"type,omitempty"`

	// Valid is non-zero if this is a recognized error message.
	Valid int `msgpack:"valid,omitempty"`
}

// SetLoclist set the error results data to current buffer's locationlist.
func SetLoclist(v *vim.Vim, loclist []*ErrorlistData) error {
	// setloclist({nr}, {list} [, {action}])
	// v.Call(fname string, result interface{}, args ...interface{})
	if len(loclist) > 0 {
		v.Call("setloclist", nil, 0, loclist)
	} else {
		v.Command("lexpr ''")
	}

	return nil
}

// OpenLoclist open or close the current buffer's locationlist window.
func OpenLoclist(v *vim.Vim, w vim.Window, loclist []*ErrorlistData, keep bool) error {
	if len(loclist) == 0 {
		return v.Command("lclose")
	}

	v.Command("lopen")
	if keep {
		return v.SetCurrentWindow(w)
	}
	return nil
}

// CloseLoclist close the current buffer's locationlist window.
func CloseLoclist(v *vim.Vim) error {
	return v.Command("lclose")
}

// SetQuickfix set the error results data to quickfix list.
func SetQuickfix(p *vim.Pipeline, qflist []*ErrorlistData) error {
	p.Call("setqflist", nil, qflist)

	return nil
}

// OpenOuickfix open the quickfix list window.
func OpenOuickfix(p *vim.Pipeline, w vim.Window, keep bool) error {
	p.Command("copen")
	if keep {
		p.SetCurrentWindow(w)
	}
	if err := p.Wait(); err != nil {
		return err
	}

	return nil
}

// CloseQuickfix close the quickfix list window.
func CloseQuickfix(v *vim.Vim) error {
	return v.Command("cclose")
}

// SplitPos parses a string of form 'token.Pos', and return the relative
// filepath from the current working directory path.
func SplitPos(pos string, cwd string) (string, int, int) {
	slc := strings.Split(pos, ":")
	line, err := strconv.ParseInt(slc[1], 10, 64)
	if err != nil {
		line = 0
	}
	col, err := strconv.ParseInt(slc[2], 10, 64)
	if err != nil {
		col = 0
	}

	fname := slc[0]
	frel := strings.TrimPrefix(fname, cwd+string(filepath.Separator))
	if fname == frel {
		return fname, int(line), int(col)
	}

	return frel, int(line), int(col)
}

// ParseError parses a typical error message of Go compile tools.
// Error sample:
//  # nvim-go/nvim
//  echo.go:79: syntax error: non-declaration statement outside function body
//  # nvim-go/nvim/quickfix
//  locationlist.go:152: syntax error: unexpected case, expecting }
//  locationlist.go:160: syntax error: non-declaration statement outside function body
// TODO(zchee): More better regexp pattern and single for loop if possible.
func ParseError(errors []byte, cwd string, ctxt *context.Build) ([]*ErrorlistData, error) {
	var (
		errlist      []*ErrorlistData
		reErrPattern = regexp.MustCompile(`(?m)^#\s([-_./\w]+)\n([.,-_'":\s\w]+)`)
		reFile       = regexp.MustCompile(`([.\w]+):(\d+)(?::(\d+))?:\s(.*)`)
	)

	for _, m := range reErrPattern.FindAllSubmatch(errors, -1) {
		parent := string(m[1])
		errFile := m[2]

		for _, mm := range reFile.FindAllSubmatch(errFile, -1) {
			var fname string
			fpath := filepath.Join(parent, string(mm[1]))

			switch ctxt.Tool {
			case "go":
				sep := filepath.Join(ctxt.GOPATH, "src")
				c := strings.TrimPrefix(cwd, sep)
				fname = strings.TrimPrefix(filepath.Clean(fpath), c+string(filepath.Separator))

			case "gb":
				if !filepath.IsAbs(fpath) {
					fpath = filepath.Join(ctxt.ProjectDir, "src", fpath)
				}
				fname, _ = filepath.Rel(cwd, fpath)
			}

			line, _ := strconv.Atoi(string(mm[2]))
			col, _ := strconv.Atoi(string(mm[3]))

			errlist = append(errlist, &ErrorlistData{
				FileName: fname,
				LNum:     line,
				Col:      col,
				Text:     string(bytes.TrimSpace(mm[4])),
			})
		}
	}

	return errlist, nil
}
<commit_msg>nvim/quickfix: Add Separator to TrimPrefix base path for go compiler<commit_after>// Copyright 2016 Koichi Shiraishi. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

package quickfix

import (
	"bytes"
	"path/filepath"
	"regexp"
	"strconv"
	"strings"

	"nvim-go/context"

	"github.com/garyburd/neovim-go/vim"
)

// ErrorlistData represents an item in a quickfix and locationlist.
type ErrorlistData struct {
	// Buffer number
	Bufnr int `msgpack:"bufnr,omitempty"`

	// Name of a file; only used when bufnr is not present or it is invalid.
	FileName string `msgpack:"filename,omitempty"`

	// Line number in the file.
	LNum int `msgpack:"lnum,omitempty"`

	// Column number (first column is 1).
	Col int `msgpack:"col,omitempty"`

	// When Vcol is != 0,  Col is visual column.
	VCol int `msgpack:"vcol,omitempty"`

	// Error number.
	Nr int `msgpack:"nr,omitempty"`

	// Search pattern used to locate the error.
	Pattern string `msgpack:"pattern,omitempty"`

	// Description of the error.
	Text string `msgpack:"text,omitempty"`

	// Single-character error type, 'E', 'W', etc.
	Type string `msgpack:"type,omitempty"`

	// Valid is non-zero if this is a recognized error message.
	Valid int `msgpack:"valid,omitempty"`
}

// SetLoclist set the error results data to current buffer's locationlist.
func SetLoclist(v *vim.Vim, loclist []*ErrorlistData) error {
	// setloclist({nr}, {list} [, {action}])
	// v.Call(fname string, result interface{}, args ...interface{})
	if len(loclist) > 0 {
		v.Call("setloclist", nil, 0, loclist)
	} else {
		v.Command("lexpr ''")
	}

	return nil
}

// OpenLoclist open or close the current buffer's locationlist window.
func OpenLoclist(v *vim.Vim, w vim.Window, loclist []*ErrorlistData, keep bool) error {
	if len(loclist) == 0 {
		return v.Command("lclose")
	}

	v.Command("lopen")
	if keep {
		return v.SetCurrentWindow(w)
	}
	return nil
}

// CloseLoclist close the current buffer's locationlist window.
func CloseLoclist(v *vim.Vim) error {
	return v.Command("lclose")
}

// SetQuickfix set the error results data to quickfix list.
func SetQuickfix(p *vim.Pipeline, qflist []*ErrorlistData) error {
	p.Call("setqflist", nil, qflist)

	return nil
}

// OpenOuickfix open the quickfix list window.
func OpenOuickfix(p *vim.Pipeline, w vim.Window, keep bool) error {
	p.Command("copen")
	if keep {
		p.SetCurrentWindow(w)
	}
	if err := p.Wait(); err != nil {
		return err
	}

	return nil
}

// CloseQuickfix close the quickfix list window.
func CloseQuickfix(v *vim.Vim) error {
	return v.Command("cclose")
}

// SplitPos parses a string of form 'token.Pos', and return the relative
// filepath from the current working directory path.
func SplitPos(pos string, cwd string) (string, int, int) {
	slc := strings.Split(pos, ":")
	line, err := strconv.ParseInt(slc[1], 10, 64)
	if err != nil {
		line = 0
	}
	col, err := strconv.ParseInt(slc[2], 10, 64)
	if err != nil {
		col = 0
	}

	fname := slc[0]
	frel := strings.TrimPrefix(fname, cwd+string(filepath.Separator))
	if fname == frel {
		return fname, int(line), int(col)
	}

	return frel, int(line), int(col)
}

// ParseError parses a typical error message of Go compile tools.
// Error sample:
//  # nvim-go/nvim
//  echo.go:79: syntax error: non-declaration statement outside function body
//  # nvim-go/nvim/quickfix
//  locationlist.go:152: syntax error: unexpected case, expecting }
//  locationlist.go:160: syntax error: non-declaration statement outside function body
// TODO(zchee): More better regexp pattern and single for loop if possible.
func ParseError(errors []byte, cwd string, ctxt *context.Build) ([]*ErrorlistData, error) {
	var (
		errlist      []*ErrorlistData
		reErrPattern = regexp.MustCompile(`(?m)^#\s([-_./\w]+)\n([.,-_'":\s\w]+)`)
		reFile       = regexp.MustCompile(`([.\w]+):(\d+)(?::(\d+))?:\s(.*)`)
	)

	for _, m := range reErrPattern.FindAllSubmatch(errors, -1) {
		parent := string(m[1])
		errFile := m[2]

		for _, mm := range reFile.FindAllSubmatch(errFile, -1) {
			var fname string
			fpath := filepath.Join(parent, string(mm[1]))

			switch ctxt.Tool {
			case "go":
				sep := filepath.Join(ctxt.GOPATH, "src")
				c := strings.TrimPrefix(cwd, sep+string(filepath.Separator))
				fname = strings.TrimPrefix(filepath.Clean(fpath), c+string(filepath.Separator))

			case "gb":
				if !filepath.IsAbs(fpath) {
					fpath = filepath.Join(ctxt.ProjectDir, "src", fpath)
				}
				fname, _ = filepath.Rel(cwd, fpath)
			}

			line, _ := strconv.Atoi(string(mm[2]))
			col, _ := strconv.Atoi(string(mm[3]))

			errlist = append(errlist, &ErrorlistData{
				FileName: fname,
				LNum:     line,
				Col:      col,
				Text:     string(bytes.TrimSpace(mm[4])),
			})
		}
	}

	return errlist, nil
}
<|endoftext|>


<commit_before>// Copyright 2021 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

package types2

import "cmd/compile/internal/syntax"

// ----------------------------------------------------------------------------
// API

// A Union represents a union of terms embedded in an interface.
type Union struct {
	terms []*Term   // list of syntactical terms (not a canonicalized termlist)
	tset  *_TypeSet // type set described by this union, computed lazily
}

// NewUnion returns a new Union type with the given terms.
// It is an error to create an empty union; they are syntactically not possible.
func NewUnion(terms []*Term) *Union {
	if len(terms) == 0 {
		panic("empty union")
	}
	return &Union{terms, nil}
}

func (u *Union) Len() int         { return len(u.terms) }
func (u *Union) Term(i int) *Term { return u.terms[i] }

func (u *Union) Underlying() Type { return u }
func (u *Union) String() string   { return TypeString(u, nil) }

// A Term represents a term in a Union.
type Term term

// NewTerm returns a new union term.
func NewTerm(tilde bool, typ Type) *Term { return &Term{tilde, typ} }

func (t *Term) Tilde() bool    { return t.tilde }
func (t *Term) Type() Type     { return t.typ }
func (t *Term) String() string { return (*term)(t).String() }

// ----------------------------------------------------------------------------
// Implementation

// Avoid excessive type-checking times due to quadratic termlist operations.
const maxTermCount = 100

// parseUnion parses the given list of type expressions tlist as a union of
// those expressions. The result is a Union type, or Typ[Invalid] for some
// errors.
func parseUnion(check *Checker, tlist []syntax.Expr) Type {
	var terms []*Term
	for _, x := range tlist {
		tilde, typ := parseTilde(check, x)
		if len(tlist) == 1 && !tilde {
			// Single type. Ok to return early because all relevant
			// checks have been performed in parseTilde (no need to
			// run through term validity check below).
			return typ
		}
		if len(terms) >= maxTermCount {
			check.errorf(x, "cannot handle more than %d union terms (implementation limitation)", maxTermCount)
			return Typ[Invalid]
		}
		terms = append(terms, NewTerm(tilde, typ))
	}

	// Check validity of terms.
	// Do this check later because it requires types to be set up.
	// Note: This is a quadratic algorithm, but unions tend to be short.
	check.later(func() {
		for i, t := range terms {
			if t.typ == Typ[Invalid] {
				continue
			}

			x := tlist[i]
			pos := syntax.StartPos(x)
			// We may not know the position of x if it was a typechecker-
			// introduced ~T term for a type list entry T. Use the position
			// of T instead.
			// TODO(gri) remove this test once we don't support type lists anymore
			if !pos.IsKnown() {
				if op, _ := x.(*syntax.Operation); op != nil {
					pos = syntax.StartPos(op.X)
				}
			}

			u := under(t.typ)
			f, _ := u.(*Interface)
			if t.tilde {
				if f != nil {
					check.errorf(x, "invalid use of ~ (%s is an interface)", t.typ)
					continue // don't report another error for t
				}

				if !Identical(u, t.typ) {
					check.errorf(x, "invalid use of ~ (underlying type of %s is %s)", t.typ, u)
					continue // don't report another error for t
				}
			}

			// Stand-alone embedded interfaces are ok and are handled by the single-type case
			// in the beginning. Embedded interfaces with tilde are excluded above. If we reach
			// here, we must have at least two terms in the union.
			if f != nil && !f.typeSet().IsTypeSet() {
				check.errorf(pos, "cannot use %s in union (interface contains methods)", t)
				continue // don't report another error for t
			}

			// Report overlapping (non-disjoint) terms such as
			// a|a, a|~a, ~a|~a, and ~a|A (where under(A) == a).
			if j := overlappingTerm(terms[:i], t); j >= 0 {
				check.softErrorf(pos, "overlapping terms %s and %s", t, terms[j])
			}
		}
	})

	return &Union{terms, nil}
}

func parseTilde(check *Checker, x syntax.Expr) (tilde bool, typ Type) {
	if op, _ := x.(*syntax.Operation); op != nil && op.Op == syntax.Tilde {
		x = op.X
		tilde = true
	}
	typ = check.typ(x)
	// Embedding stand-alone type parameters is not permitted (issue #47127).
	// Do this check later because it requires computation of the underlying type (see also issue #46461).
	// Note: If an underlying type cannot be a type parameter, the call to
	//       under() will not be needed and then we don't need to delay this
	//       check to later and could return Typ[Invalid] instead.
	check.later(func() {
		if _, ok := under(typ).(*TypeParam); ok {
			check.error(x, "cannot embed a type parameter")
		}
	})
	return
}

// overlappingTerm reports the index of the term x in terms which is
// overlapping (not disjoint) from y. The result is < 0 if there is no
// such term.
func overlappingTerm(terms []*Term, y *Term) int {
	for i, x := range terms {
		// disjoint requires non-nil, non-top arguments
		if debug {
			if x == nil || x.typ == nil || y == nil || y.typ == nil {
				panic("empty or top union term")
			}
		}
		if !(*term)(x).disjoint((*term)(y)) {
			return i
		}
	}
	return -1
}
<commit_msg>cmd/compile/internal/types2: remove unused position computation (cleanup)<commit_after>// Copyright 2021 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

package types2

import "cmd/compile/internal/syntax"

// ----------------------------------------------------------------------------
// API

// A Union represents a union of terms embedded in an interface.
type Union struct {
	terms []*Term   // list of syntactical terms (not a canonicalized termlist)
	tset  *_TypeSet // type set described by this union, computed lazily
}

// NewUnion returns a new Union type with the given terms.
// It is an error to create an empty union; they are syntactically not possible.
func NewUnion(terms []*Term) *Union {
	if len(terms) == 0 {
		panic("empty union")
	}
	return &Union{terms, nil}
}

func (u *Union) Len() int         { return len(u.terms) }
func (u *Union) Term(i int) *Term { return u.terms[i] }

func (u *Union) Underlying() Type { return u }
func (u *Union) String() string   { return TypeString(u, nil) }

// A Term represents a term in a Union.
type Term term

// NewTerm returns a new union term.
func NewTerm(tilde bool, typ Type) *Term { return &Term{tilde, typ} }

func (t *Term) Tilde() bool    { return t.tilde }
func (t *Term) Type() Type     { return t.typ }
func (t *Term) String() string { return (*term)(t).String() }

// ----------------------------------------------------------------------------
// Implementation

// Avoid excessive type-checking times due to quadratic termlist operations.
const maxTermCount = 100

// parseUnion parses the given list of type expressions tlist as a union of
// those expressions. The result is a Union type, or Typ[Invalid] for some
// errors.
func parseUnion(check *Checker, tlist []syntax.Expr) Type {
	var terms []*Term
	for _, x := range tlist {
		tilde, typ := parseTilde(check, x)
		if len(tlist) == 1 && !tilde {
			// Single type. Ok to return early because all relevant
			// checks have been performed in parseTilde (no need to
			// run through term validity check below).
			return typ
		}
		if len(terms) >= maxTermCount {
			check.errorf(x, "cannot handle more than %d union terms (implementation limitation)", maxTermCount)
			return Typ[Invalid]
		}
		terms = append(terms, NewTerm(tilde, typ))
	}

	// Check validity of terms.
	// Do this check later because it requires types to be set up.
	// Note: This is a quadratic algorithm, but unions tend to be short.
	check.later(func() {
		for i, t := range terms {
			if t.typ == Typ[Invalid] {
				continue
			}

			u := under(t.typ)
			f, _ := u.(*Interface)
			if t.tilde {
				if f != nil {
					check.errorf(tlist[i], "invalid use of ~ (%s is an interface)", t.typ)
					continue // don't report another error for t
				}

				if !Identical(u, t.typ) {
					check.errorf(tlist[i], "invalid use of ~ (underlying type of %s is %s)", t.typ, u)
					continue // don't report another error for t
				}
			}

			// Stand-alone embedded interfaces are ok and are handled by the single-type case
			// in the beginning. Embedded interfaces with tilde are excluded above. If we reach
			// here, we must have at least two terms in the union.
			if f != nil && !f.typeSet().IsTypeSet() {
				check.errorf(tlist[i], "cannot use %s in union (interface contains methods)", t)
				continue // don't report another error for t
			}

			// Report overlapping (non-disjoint) terms such as
			// a|a, a|~a, ~a|~a, and ~a|A (where under(A) == a).
			if j := overlappingTerm(terms[:i], t); j >= 0 {
				check.softErrorf(tlist[i], "overlapping terms %s and %s", t, terms[j])
			}
		}
	})

	return &Union{terms, nil}
}

func parseTilde(check *Checker, x syntax.Expr) (tilde bool, typ Type) {
	if op, _ := x.(*syntax.Operation); op != nil && op.Op == syntax.Tilde {
		x = op.X
		tilde = true
	}
	typ = check.typ(x)
	// Embedding stand-alone type parameters is not permitted (issue #47127).
	// Do this check later because it requires computation of the underlying type (see also issue #46461).
	// Note: If an underlying type cannot be a type parameter, the call to
	//       under() will not be needed and then we don't need to delay this
	//       check to later and could return Typ[Invalid] instead.
	check.later(func() {
		if _, ok := under(typ).(*TypeParam); ok {
			check.error(x, "cannot embed a type parameter")
		}
	})
	return
}

// overlappingTerm reports the index of the term x in terms which is
// overlapping (not disjoint) from y. The result is < 0 if there is no
// such term.
func overlappingTerm(terms []*Term, y *Term) int {
	for i, x := range terms {
		// disjoint requires non-nil, non-top arguments
		if debug {
			if x == nil || x.typ == nil || y == nil || y.typ == nil {
				panic("empty or top union term")
			}
		}
		if !(*term)(x).disjoint((*term)(y)) {
			return i
		}
	}
	return -1
}
<|endoftext|>


<commit_before>package sessions

import (
	"strings"

	"github.com/tchap/steemwatch/server/users"

	"github.com/labstack/echo"
	"github.com/pkg/errors"
	"github.com/tchap/securecookie"
)

const SessionCookieName = "SID"

type SessionManager struct {
	cookie *securecookie.SecureCookie
	store  users.Store
	secure bool
}

func NewSessionManager(hashKey, blockKey []byte, store users.Store) (*SessionManager, error) {
	// Make sure the keys are of correct length.
	switch {
	case len(hashKey) != 64:
		return nil, errors.New("the hash key must be 64 bytes long")
	case len(blockKey) != 32:
		return nil, errors.New("the block key must be 32 bytes long")
	}

	// Create a SecureCookie.
	cookie := securecookie.New(hashKey, blockKey)

	// Return a new SessionManager.
	return &SessionManager{
		cookie: cookie,
		store:  store,
	}, nil
}

func (manager *SessionManager) SecureCookie(secure bool) {
	manager.secure = secure
}

func (manager *SessionManager) GetProfile(ctx echo.Context) (*users.User, error) {
	// Get the session cookie value.
	cookie, err := ctx.Cookie(SessionCookieName)
	if err != nil {
		if err == echo.ErrCookieNotFound {
			return nil, nil
		} else {
			return nil, err
		}
	}
	// Empty value is the same as no value at all.
	cookieValue := cookie.Value()
	if cookieValue == "" {
		return nil, nil
	}
	// Replace '0' with '='.
	cookieValue = strings.Replace(cookieValue, "0", "=", -1)

	// Decode the cookie value.
	var session string
	if err := manager.cookie.Decode(SessionCookieName, cookieValue, &session); err != nil {
		if err == securecookie.ErrMacInvalid {
			manager.ClearProfile(ctx)
			return nil, nil
		} else {
			return nil, err
		}
	}

	// Load the user profile.
	return manager.store.LoadUser(session)
}

func (manager *SessionManager) SetProfile(ctx echo.Context, profile *users.User) error {
	// Store the profile.
	session, err := manager.store.StoreUser(profile)
	if err != nil {
		return err
	}

	// Encode the profile to get the cookie value.
	cookieValue, err := manager.cookie.Encode(SessionCookieName, session)
	if err != nil {
		return err
	}
	// Replace '=' with '0'.
	cookieValue = strings.Replace(cookieValue, "=", "0", -1)

	// Assemble the cookie object.
	cookie := &echo.Cookie{}
	cookie.SetName(SessionCookieName)
	cookie.SetValue(cookieValue)
	cookie.SetHTTPOnly(true)
	cookie.SetSecure(manager.secure)

	// And finally, set the cookie.
	ctx.SetCookie(cookie)
	return nil
}

func (manager *SessionManager) ClearProfile(ctx echo.Context) error {
	// Assemble the cookie object so that the value is empty and expires is in the past.
	cookie := &echo.Cookie{}
	cookie.SetName(SessionCookieName)
	cookie.SetValue("")

	// Set the cookie.
	ctx.SetCookie(cookie)
	return nil
}
<commit_msg>server/sessions: Handle expired cookie properly<commit_after>package sessions

import (
	"strings"

	"github.com/tchap/steemwatch/server/users"

	"github.com/labstack/echo"
	"github.com/pkg/errors"
	"github.com/tchap/securecookie"
)

const SessionCookieName = "SID"

type SessionManager struct {
	cookie *securecookie.SecureCookie
	store  users.Store
	secure bool
}

func NewSessionManager(hashKey, blockKey []byte, store users.Store) (*SessionManager, error) {
	// Make sure the keys are of correct length.
	switch {
	case len(hashKey) != 64:
		return nil, errors.New("the hash key must be 64 bytes long")
	case len(blockKey) != 32:
		return nil, errors.New("the block key must be 32 bytes long")
	}

	// Create a SecureCookie.
	cookie := securecookie.New(hashKey, blockKey)

	// Return a new SessionManager.
	return &SessionManager{
		cookie: cookie,
		store:  store,
	}, nil
}

func (manager *SessionManager) SecureCookie(secure bool) {
	manager.secure = secure
}

func (manager *SessionManager) GetProfile(ctx echo.Context) (*users.User, error) {
	// Get the session cookie value.
	cookie, err := ctx.Cookie(SessionCookieName)
	if err != nil {
		if err == echo.ErrCookieNotFound {
			return nil, nil
		} else {
			return nil, err
		}
	}
	// Empty value is the same as no value at all.
	cookieValue := cookie.Value()
	if cookieValue == "" {
		return nil, nil
	}
	// Replace '0' with '='.
	cookieValue = strings.Replace(cookieValue, "0", "=", -1)

	// Decode the cookie value.
	var session string
	if err := manager.cookie.Decode(SessionCookieName, cookieValue, &session); err != nil {
		if ex, ok := err.(securecookie.Error); ok && ex.IsDecode() {
			manager.ClearProfile(ctx)
			return nil, nil
		} else {
			return nil, err
		}
	}

	// Load the user profile.
	return manager.store.LoadUser(session)
}

func (manager *SessionManager) SetProfile(ctx echo.Context, profile *users.User) error {
	// Store the profile.
	session, err := manager.store.StoreUser(profile)
	if err != nil {
		return err
	}

	// Encode the profile to get the cookie value.
	cookieValue, err := manager.cookie.Encode(SessionCookieName, session)
	if err != nil {
		return err
	}
	// Replace '=' with '0'.
	cookieValue = strings.Replace(cookieValue, "=", "0", -1)

	// Assemble the cookie object.
	cookie := &echo.Cookie{}
	cookie.SetName(SessionCookieName)
	cookie.SetValue(cookieValue)
	cookie.SetHTTPOnly(true)
	cookie.SetSecure(manager.secure)

	// And finally, set the cookie.
	ctx.SetCookie(cookie)
	return nil
}

func (manager *SessionManager) ClearProfile(ctx echo.Context) error {
	// Assemble the cookie object so that the value is empty and expires is in the past.
	cookie := &echo.Cookie{}
	cookie.SetName(SessionCookieName)
	cookie.SetValue("")

	// Set the cookie.
	ctx.SetCookie(cookie)
	return nil
}
<|endoftext|>


<commit_before>package download_step

import (
	"archive/tar"
	"io"
	"io/ioutil"
	"net/url"
	"os"
	"path/filepath"
	"time"

	"github.com/cloudfoundry-incubator/executor/steps/emittable_error"
	"github.com/cloudfoundry-incubator/garden/warden"
	"github.com/cloudfoundry-incubator/runtime-schema/models"
	steno "github.com/cloudfoundry/gosteno"
	"github.com/pivotal-golang/archiver/compressor"
	"github.com/pivotal-golang/archiver/extractor"
	"github.com/pivotal-golang/cacheddownloader"
)

type DownloadStep struct {
	container        warden.Container
	model            models.DownloadAction
	cachedDownloader cacheddownloader.CachedDownloader
	extractor        extractor.Extractor
	tempDir          string
	logger           *steno.Logger
}

func New(
	container warden.Container,
	model models.DownloadAction,
	cachedDownloader cacheddownloader.CachedDownloader,
	extractor extractor.Extractor,
	tempDir string,
	logger *steno.Logger,
) *DownloadStep {
	return &DownloadStep{
		container:        container,
		model:            model,
		cachedDownloader: cachedDownloader,
		extractor:        extractor,
		tempDir:          tempDir,
		logger:           logger,
	}
}

func (step *DownloadStep) Perform() error {
	step.logger.Infod(
		map[string]interface{}{
			"handle": step.container.Handle(),
		},
		"task.handle.download-action",
	)

	//Stream this to the extractor + container when we have streaming support!
	downloadedPath, err := step.download()
	if err != nil {
		step.logger.Errord(
			map[string]interface{}{
				"handle": step.container.Handle(),
				"from":   step.model.From,
				"to":     step.model.To,
				"error":  err,
			},
			"task.handle.download-failed",
		)

		return err
	}

	defer os.Remove(downloadedPath)

	if step.model.Extract {
		extractionDir, err := step.extract(downloadedPath)
		if err != nil {
			return emittable_error.New(err, "Extraction failed")
		}

		defer os.RemoveAll(extractionDir)

		err = step.copyExtractedFiles(extractionDir, step.model.To)
		if err != nil {
			return emittable_error.New(err, "Copying into the container failed")
		}

		return err
	} else {
		reader, writer := io.Pipe()
		defer reader.Close()

		go writeTarTo(filepath.Base(step.model.To), downloadedPath, writer)

		err = step.container.StreamIn(filepath.Dir(step.model.To), reader)
		if err != nil {
			return emittable_error.New(err, "Copying into the container failed")
		}

		return nil
	}
}

func (step *DownloadStep) download() (string, error) {
	url, err := url.ParseRequestURI(step.model.From)
	if err != nil {
		return "", err
	}

	tempFile, err := ioutil.TempFile(step.tempDir, "downloaded")
	if err != nil {
		return "", err
	}
	defer tempFile.Close()

	downloadedFile, err := step.cachedDownloader.Fetch(url, step.model.CacheKey)
	if err != nil {
		return "", err
	}
	defer downloadedFile.Close()

	_, err = io.Copy(tempFile, downloadedFile)
	if err != nil {
		return "", err
	}

	return tempFile.Name(), nil
}

func (step *DownloadStep) extract(downloadedPath string) (string, error) {
	extractionDir, err := ioutil.TempDir(step.tempDir, "extracted")
	if err != nil {
		return "", err
	}

	err = step.extractor.Extract(downloadedPath, extractionDir)
	if err != nil {
		step.logger.Warnd(
			map[string]interface{}{
				"error": err.Error(),
				"url":   step.model.From,
			},
			"downloader.extract-failed",
		)

		return "", err
	}

	return extractionDir, nil
}

func (step *DownloadStep) Cancel() {}

func (step *DownloadStep) Cleanup() {}

func (step *DownloadStep) copyExtractedFiles(source string, destination string) error {
	reader, writer := io.Pipe()

	go func() {
		compressor.WriteTar(source+string(filepath.Separator), writer)
		writer.Close()
	}()
	defer reader.Close()

	return step.container.StreamIn(destination, reader)
}

func writeTarTo(name string, sourcePath string, destination io.WriteCloser) error {
	source, err := os.Open(sourcePath)
	if err != nil {
		return err
	}
	defer source.Close()

	tarWriter := tar.NewWriter(destination)

	fileInfo, err := source.Stat()
	if err != nil {
		return err
	}

	err = tarWriter.WriteHeader(&tar.Header{
		Name:       name,
		Size:       fileInfo.Size(),
		Mode:       0644,
		AccessTime: time.Now(),
		ChangeTime: time.Now(),
	})
	if err != nil {
		return err
	}

	_, err = io.Copy(tarWriter, source)
	if err != nil {
		return err
	}

	if err := tarWriter.Flush(); err != nil {
		return err
	}

	if err := destination.Close(); err != nil {
		return err
	}

	return nil
}
<commit_msg>download step propagates error down the pipe<commit_after>package download_step

import (
	"archive/tar"
	"io"
	"io/ioutil"
	"net/url"
	"os"
	"path/filepath"
	"time"

	"github.com/cloudfoundry-incubator/executor/steps/emittable_error"
	"github.com/cloudfoundry-incubator/garden/warden"
	"github.com/cloudfoundry-incubator/runtime-schema/models"
	steno "github.com/cloudfoundry/gosteno"
	"github.com/pivotal-golang/archiver/compressor"
	"github.com/pivotal-golang/archiver/extractor"
	"github.com/pivotal-golang/cacheddownloader"
)

type DownloadStep struct {
	container        warden.Container
	model            models.DownloadAction
	cachedDownloader cacheddownloader.CachedDownloader
	extractor        extractor.Extractor
	tempDir          string
	logger           *steno.Logger
}

func New(
	container warden.Container,
	model models.DownloadAction,
	cachedDownloader cacheddownloader.CachedDownloader,
	extractor extractor.Extractor,
	tempDir string,
	logger *steno.Logger,
) *DownloadStep {
	return &DownloadStep{
		container:        container,
		model:            model,
		cachedDownloader: cachedDownloader,
		extractor:        extractor,
		tempDir:          tempDir,
		logger:           logger,
	}
}

func (step *DownloadStep) Perform() error {
	step.logger.Infod(
		map[string]interface{}{
			"handle": step.container.Handle(),
		},
		"task.handle.download-action",
	)

	//Stream this to the extractor + container when we have streaming support!
	downloadedPath, err := step.download()
	if err != nil {
		step.logger.Errord(
			map[string]interface{}{
				"handle": step.container.Handle(),
				"from":   step.model.From,
				"to":     step.model.To,
				"error":  err,
			},
			"task.handle.download-failed",
		)

		return err
	}

	defer os.Remove(downloadedPath)

	if step.model.Extract {
		extractionDir, err := step.extract(downloadedPath)
		if err != nil {
			return emittable_error.New(err, "Extraction failed")
		}

		defer os.RemoveAll(extractionDir)

		err = step.copyExtractedFiles(extractionDir, step.model.To)
		if err != nil {
			return emittable_error.New(err, "Copying into the container failed")
		}

		return err
	} else {
		reader, writer := io.Pipe()

		go writeTarTo(filepath.Base(step.model.To), downloadedPath, writer)

		err = step.container.StreamIn(filepath.Dir(step.model.To), reader)
		if err != nil {
			return emittable_error.New(err, "Copying into the container failed")
		}

		return nil
	}
}

func (step *DownloadStep) download() (string, error) {
	url, err := url.ParseRequestURI(step.model.From)
	if err != nil {
		return "", err
	}

	tempFile, err := ioutil.TempFile(step.tempDir, "downloaded")
	if err != nil {
		return "", err
	}
	defer tempFile.Close()

	downloadedFile, err := step.cachedDownloader.Fetch(url, step.model.CacheKey)
	if err != nil {
		return "", err
	}
	defer downloadedFile.Close()

	_, err = io.Copy(tempFile, downloadedFile)
	if err != nil {
		return "", err
	}

	return tempFile.Name(), nil
}

func (step *DownloadStep) extract(downloadedPath string) (string, error) {
	extractionDir, err := ioutil.TempDir(step.tempDir, "extracted")
	if err != nil {
		return "", err
	}

	err = step.extractor.Extract(downloadedPath, extractionDir)
	if err != nil {
		step.logger.Warnd(
			map[string]interface{}{
				"error": err.Error(),
				"url":   step.model.From,
			},
			"downloader.extract-failed",
		)

		return "", err
	}

	return extractionDir, nil
}

func (step *DownloadStep) Cancel() {}

func (step *DownloadStep) Cleanup() {}

func (step *DownloadStep) copyExtractedFiles(source string, destination string) error {
	reader, writer := io.Pipe()

	go func() {
		err := compressor.WriteTar(source+string(filepath.Separator), writer)
		if err == nil {
			writer.Close()
		} else {
			writer.CloseWithError(err)
		}
	}()

	return step.container.StreamIn(destination, reader)
}

func writeTarTo(name string, sourcePath string, destination *io.PipeWriter) {
	source, err := os.Open(sourcePath)
	if err != nil {
		destination.CloseWithError(err)
		return
	}
	defer source.Close()

	tarWriter := tar.NewWriter(destination)

	fileInfo, err := source.Stat()
	if err != nil {
		destination.CloseWithError(err)
		return
	}

	err = tarWriter.WriteHeader(&tar.Header{
		Name:       name,
		Size:       fileInfo.Size(),
		Mode:       0644,
		AccessTime: time.Now(),
		ChangeTime: time.Now(),
	})
	if err != nil {
		destination.CloseWithError(err)
		return
	}

	_, err = io.Copy(tarWriter, source)
	if err != nil {
		destination.CloseWithError(err)
		return
	}

	if err := tarWriter.Flush(); err != nil {
		destination.CloseWithError(err)
		return
	}

	destination.Close()
}
<|endoftext|>