<commit_before>package render

import (
	"regexp"

	"github.com/weaveworks/scope/probe/docker"
	"github.com/weaveworks/scope/report"
)

// Constants are used in the tests.
const (
	UncontainedID    = "uncontained"
	UncontainedMajor = "Uncontained"

	// Topology for IPs so we can differentiate them at the end
	IP = "IP"
)

// UncontainedIDPrefix is the prefix of uncontained pseudo nodes
var UncontainedIDPrefix = MakePseudoNodeID(UncontainedID, "")

// ContainerRenderer is a Renderer which produces a renderable container
// graph by merging the process graph and the container topology.
// NB We only want processes in container _or_ processes with network connections
// but we need to be careful to ensure we only include each edge once, by only
// including the ProcessRenderer once.
var ContainerRenderer = Memoise(MakeFilter(
	func(n report.Node) bool {
		// Drop deleted containers
		state, ok := n.Latest.Lookup(docker.ContainerState)
		return !ok || state != docker.StateDeleted
	},
	MakeReduce(
		MakeMap(
			MapProcess2Container,
			ProcessRenderer,
		),
		ConnectionJoin(MapContainer2IP, SelectContainer),
	),
))

const originalNodeID = "original_node_id"

// ConnectionJoin joins the given renderer with connections from the
// endpoints topology, using the toIPs function to extract IPs from
// the nodes.
func ConnectionJoin(toIPs func(report.Node) []string, r Renderer) Renderer {
	return connectionJoin{toIPs: toIPs, r: r}
}

type connectionJoin struct {
	toIPs func(report.Node) []string
	r     Renderer
}

func (c connectionJoin) Render(rpt report.Report) Nodes {
	local := LocalNetworks(rpt)
	inputNodes := c.r.Render(rpt)
	endpoints := SelectEndpoint.Render(rpt)

	// Collect all the IPs we are trying to map to, and which ID they map from
	var ipNodes = map[string]string{}
	for _, n := range inputNodes.Nodes {
		for _, ip := range c.toIPs(n) {
			if _, exists := ipNodes[ip]; exists {
				// If an IP is shared between multiple nodes, we can't reliably
				// attribute an connection based on its IP
				ipNodes[ip] = "" // blank out the mapping so we don't use it
			} else {
				ipNodes[ip] = n.ID
			}
		}
	}
	ret := newJoinResults(inputNodes.Nodes)

	// Now look at all the endpoints and see which map to IP nodes
	for _, m := range endpoints.Nodes {
		scope, addr, port, ok := report.ParseEndpointNodeID(m.ID)
		if !ok {
			continue
		}
		// Nodes without a hostid may be pseudo nodes - if so, pass through to result
		if _, ok := m.Latest.Lookup(report.HostNodeID); !ok {
			if id, ok := externalNodeID(m, addr, local); ok {
				ret.addChild(m, id, newPseudoNode)
				continue
			}
		}
		id, found := ipNodes[report.MakeScopedEndpointNodeID(scope, addr, "")]
		// We also allow for joining on ip:port pairs.  This is useful for
		// connections to the host IPs which have been port mapped to a
		// container can only be unambiguously identified with the port.
		if !found {
			id, found = ipNodes[report.MakeScopedEndpointNodeID(scope, addr, port)]
		}
		if found && id != "" { // not one we blanked out earlier
			// We are guaranteed to find the id, so no need to pass a node constructor.
			ret.addChild(m, id, nil)
		}
	}
	return ret.result(endpoints)
}

// FilterEmpty is a Renderer which filters out nodes which have no children
// from the specified topology.
func FilterEmpty(topology string, r Renderer) Renderer {
	return MakeFilter(HasChildren(topology), r)
}

// HasChildren returns true if the node has no children from the specified
// topology.
func HasChildren(topology string) FilterFunc {
	return func(n report.Node) bool {
		count := 0
		n.Children.ForEach(func(child report.Node) {
			if child.Topology == topology {
				count++
			}
		})
		return count > 0
	}
}

type containerWithImageNameRenderer struct {
	Renderer
}

// Render produces a container graph where the the latest metadata contains the
// container image name, if found.
func (r containerWithImageNameRenderer) Render(rpt report.Report) Nodes {
	containers := r.Renderer.Render(rpt)
	images := SelectContainerImage.Render(rpt)

	outputs := make(report.Nodes, len(containers.Nodes))
	for id, c := range containers.Nodes {
		outputs[id] = c
		imageID, ok := c.Latest.Lookup(docker.ImageID)
		if !ok {
			continue
		}
		image, ok := images.Nodes[report.MakeContainerImageNodeID(imageID)]
		if !ok {
			continue
		}
		imageName, ok := image.Latest.Lookup(docker.ImageName)
		if !ok {
			continue
		}
		imageNameWithoutVersion := docker.ImageNameWithoutVersion(imageName)
		imageNodeID := report.MakeContainerImageNodeID(imageNameWithoutVersion)

		c = propagateLatest(docker.ImageName, image, c)
		c = propagateLatest(docker.ImageSize, image, c)
		c = propagateLatest(docker.ImageVirtualSize, image, c)
		c = propagateLatest(docker.ImageLabelPrefix+"works.weave.role", image, c)
		c.Parents = c.Parents.
			Delete(report.ContainerImage).
			Add(report.ContainerImage, report.MakeStringSet(imageNodeID))
		outputs[id] = c
	}
	return Nodes{Nodes: outputs, Filtered: containers.Filtered}
}

// ContainerWithImageNameRenderer is a Renderer which produces a container
// graph where the ranks are the image names, not their IDs
var ContainerWithImageNameRenderer = Memoise(containerWithImageNameRenderer{ContainerRenderer})

// ContainerImageRenderer is a Renderer which produces a renderable container
// image graph by merging the container graph and the container image topology.
var ContainerImageRenderer = Memoise(FilterEmpty(report.Container,
	MakeMap(
		MapContainerImage2Name,
		MakeReduce(
			MakeMap(
				MapContainer2ContainerImage,
				ContainerWithImageNameRenderer,
			),
			SelectContainerImage,
		),
	),
))

// ContainerHostnameRenderer is a Renderer which produces a renderable container
// by hostname graph..
//
// not memoised
var ContainerHostnameRenderer = FilterEmpty(report.Container,
	MakeReduce(
		MakeMap(
			MapContainer2Hostname,
			ContainerWithImageNameRenderer,
		),
		// Grab *all* the hostnames, so we can count the number which were empty
		// for accurate stats.
		MakeMap(
			MapToEmpty,
			MakeMap(
				MapContainer2Hostname,
				ContainerRenderer,
			),
		),
	),
)

var portMappingMatch = regexp.MustCompile(`([0-9]{1,3}\.[0-9]{1,3}\.[0-9]{1,3}\.[0-9]{1,3}):([0-9]+)->([0-9]+)/tcp`)

// MapContainer2IP maps container nodes to their IP addresses (outputs
// multiple nodes).  This allows container to be joined directly with
// the endpoint topology.
func MapContainer2IP(m report.Node) []string {
	// if this container doesn't make connections, we can ignore it
	_, doesntMakeConnections := m.Latest.Lookup(report.DoesNotMakeConnections)
	// if this container belongs to the host's networking namespace
	// we cannot use its IP to attribute connections
	// (they could come from any other process on the host or DNAT-ed IPs)
	_, isInHostNetwork := m.Latest.Lookup(docker.IsInHostNetwork)
	if doesntMakeConnections || isInHostNetwork {
		return nil
	}

	result := []string{}
	if addrs, ok := m.Sets.Lookup(docker.ContainerIPsWithScopes); ok {
		for _, addr := range addrs {
			scope, addr, ok := report.ParseAddressNodeID(addr)
			if !ok {
				continue
			}
			// loopback addresses are shared among all namespaces
			// so we can't use them to attribute connections to a container
			if report.IsLoopback(addr) {
				continue
			}
			id := report.MakeScopedEndpointNodeID(scope, addr, "")
			result = append(result, id)
		}
	}

	// Also output all the host:port port mappings (see above comment).
	// In this case we assume this doesn't need a scope, as they are for host IPs.
	ports, _ := m.Sets.Lookup(docker.ContainerPorts)
	for _, portMapping := range ports {
		if mapping := portMappingMatch.FindStringSubmatch(portMapping); mapping != nil {
			ip, port := mapping[1], mapping[2]
			id := report.MakeScopedEndpointNodeID("", ip, port)
			result = append(result, id)
		}
	}

	return result
}

// MapProcess2Container maps process Nodes to container
// Nodes.
//
// Pseudo nodes are passed straight through.
//
// If this function is given a node without a docker_container_id, it
// will produce an "Uncontained" pseudo node.
//
// Otherwise, this function will produce a node with the correct ID
// format for a container, but without any Major or Minor labels.
// It does not have enough info to do that, and the resulting graph
// must be merged with a container graph to get that info.
func MapProcess2Container(n report.Node) report.Nodes {
	// Propagate pseudo nodes
	if n.Topology == Pseudo {
		return report.Nodes{n.ID: n}
	}

	// Otherwise, if the process is not in a container, group it into
	// an per-host "Uncontained" node.  If for whatever reason this
	// node doesn't have a host id in their node metadata, it'll all
	// get grouped into a single uncontained node.
	var (
		id   string
		node report.Node
	)
	if containerID, ok := n.Latest.Lookup(docker.ContainerID); ok {
		id = report.MakeContainerNodeID(containerID)
		node = NewDerivedNode(id, n).WithTopology(report.Container)
	} else {
		id = MakePseudoNodeID(UncontainedID, report.ExtractHostID(n))
		node = NewDerivedPseudoNode(id, n)
		node = propagateLatest(report.HostNodeID, n, node)
		node = propagateLatest(IsConnectedMark, n, node)
	}
	return report.Nodes{id: node}
}

// MapContainer2ContainerImage maps container Nodes to container
// image Nodes.
//
// Pseudo nodes are passed straight through.
//
// If this function is given a node without a docker_image_id
// it will drop that node.
//
// Otherwise, this function will produce a node with the correct ID
// format for a container image, but without any Major or Minor
// labels.  It does not have enough info to do that, and the resulting
// graph must be merged with a container image graph to get that info.
func MapContainer2ContainerImage(n report.Node) report.Nodes {
	// Propagate all pseudo nodes
	if n.Topology == Pseudo {
		return report.Nodes{n.ID: n}
	}

	// Otherwise, if some some reason the container doesn't have a image_id
	// (maybe slightly out of sync reports), just drop it
	imageID, timestamp, ok := n.Latest.LookupEntry(docker.ImageID)
	if !ok {
		return report.Nodes{}
	}

	// Add container id key to the counters, which will later be
	// counted to produce the minor label
	id := report.MakeContainerImageNodeID(imageID)
	result := NewDerivedNode(id, n).WithTopology(report.ContainerImage)
	result.Latest = result.Latest.Set(docker.ImageID, timestamp, imageID)
	result.Counters = result.Counters.Add(n.Topology, 1)
	return report.Nodes{id: result}
}

// MapContainerImage2Name ignores image versions
func MapContainerImage2Name(n report.Node) report.Nodes {
	// Propagate all pseudo nodes
	if n.Topology == Pseudo {
		return report.Nodes{n.ID: n}
	}

	imageName, ok := n.Latest.Lookup(docker.ImageName)
	if !ok {
		return report.Nodes{}
	}

	imageNameWithoutVersion := docker.ImageNameWithoutVersion(imageName)
	n.ID = report.MakeContainerImageNodeID(imageNameWithoutVersion)

	if imageID, ok := report.ParseContainerImageNodeID(n.ID); ok {
		n.Sets = n.Sets.Add(docker.ImageID, report.MakeStringSet(imageID))
	}

	return report.Nodes{n.ID: n}
}

// MapContainer2Hostname maps container Nodes to 'hostname' renderabled nodes..
func MapContainer2Hostname(n report.Node) report.Nodes {
	// Propagate all pseudo nodes
	if n.Topology == Pseudo {
		return report.Nodes{n.ID: n}
	}

	// Otherwise, if some some reason the container doesn't have a hostname
	// (maybe slightly out of sync reports), just drop it
	id, timestamp, ok := n.Latest.LookupEntry(docker.ContainerHostname)
	if !ok {
		return report.Nodes{}
	}

	node := NewDerivedNode(id, n).WithTopology(MakeGroupNodeTopology(n.Topology, docker.ContainerHostname))
	node.Latest = node.Latest.Set(docker.ContainerHostname, timestamp, id)
	node.Counters = node.Counters.Add(n.Topology, 1)
	return report.Nodes{id: node}
}

// MapToEmpty removes all the attributes, children, etc, of a node. Useful when
// we just want to count the presence of nodes.
func MapToEmpty(n report.Node) report.Nodes {
	return report.Nodes{n.ID: report.MakeNode(n.ID).WithTopology(n.Topology)}
}
<commit_msg>don't propagate ContainerHostname in MapContainer2Hostname<commit_after>package render

import (
	"regexp"

	"github.com/weaveworks/scope/probe/docker"
	"github.com/weaveworks/scope/report"
)

// Constants are used in the tests.
const (
	UncontainedID    = "uncontained"
	UncontainedMajor = "Uncontained"

	// Topology for IPs so we can differentiate them at the end
	IP = "IP"
)

// UncontainedIDPrefix is the prefix of uncontained pseudo nodes
var UncontainedIDPrefix = MakePseudoNodeID(UncontainedID, "")

// ContainerRenderer is a Renderer which produces a renderable container
// graph by merging the process graph and the container topology.
// NB We only want processes in container _or_ processes with network connections
// but we need to be careful to ensure we only include each edge once, by only
// including the ProcessRenderer once.
var ContainerRenderer = Memoise(MakeFilter(
	func(n report.Node) bool {
		// Drop deleted containers
		state, ok := n.Latest.Lookup(docker.ContainerState)
		return !ok || state != docker.StateDeleted
	},
	MakeReduce(
		MakeMap(
			MapProcess2Container,
			ProcessRenderer,
		),
		ConnectionJoin(MapContainer2IP, SelectContainer),
	),
))

const originalNodeID = "original_node_id"

// ConnectionJoin joins the given renderer with connections from the
// endpoints topology, using the toIPs function to extract IPs from
// the nodes.
func ConnectionJoin(toIPs func(report.Node) []string, r Renderer) Renderer {
	return connectionJoin{toIPs: toIPs, r: r}
}

type connectionJoin struct {
	toIPs func(report.Node) []string
	r     Renderer
}

func (c connectionJoin) Render(rpt report.Report) Nodes {
	local := LocalNetworks(rpt)
	inputNodes := c.r.Render(rpt)
	endpoints := SelectEndpoint.Render(rpt)

	// Collect all the IPs we are trying to map to, and which ID they map from
	var ipNodes = map[string]string{}
	for _, n := range inputNodes.Nodes {
		for _, ip := range c.toIPs(n) {
			if _, exists := ipNodes[ip]; exists {
				// If an IP is shared between multiple nodes, we can't reliably
				// attribute an connection based on its IP
				ipNodes[ip] = "" // blank out the mapping so we don't use it
			} else {
				ipNodes[ip] = n.ID
			}
		}
	}
	ret := newJoinResults(inputNodes.Nodes)

	// Now look at all the endpoints and see which map to IP nodes
	for _, m := range endpoints.Nodes {
		scope, addr, port, ok := report.ParseEndpointNodeID(m.ID)
		if !ok {
			continue
		}
		// Nodes without a hostid may be pseudo nodes - if so, pass through to result
		if _, ok := m.Latest.Lookup(report.HostNodeID); !ok {
			if id, ok := externalNodeID(m, addr, local); ok {
				ret.addChild(m, id, newPseudoNode)
				continue
			}
		}
		id, found := ipNodes[report.MakeScopedEndpointNodeID(scope, addr, "")]
		// We also allow for joining on ip:port pairs.  This is useful for
		// connections to the host IPs which have been port mapped to a
		// container can only be unambiguously identified with the port.
		if !found {
			id, found = ipNodes[report.MakeScopedEndpointNodeID(scope, addr, port)]
		}
		if found && id != "" { // not one we blanked out earlier
			// We are guaranteed to find the id, so no need to pass a node constructor.
			ret.addChild(m, id, nil)
		}
	}
	return ret.result(endpoints)
}

// FilterEmpty is a Renderer which filters out nodes which have no children
// from the specified topology.
func FilterEmpty(topology string, r Renderer) Renderer {
	return MakeFilter(HasChildren(topology), r)
}

// HasChildren returns true if the node has no children from the specified
// topology.
func HasChildren(topology string) FilterFunc {
	return func(n report.Node) bool {
		count := 0
		n.Children.ForEach(func(child report.Node) {
			if child.Topology == topology {
				count++
			}
		})
		return count > 0
	}
}

type containerWithImageNameRenderer struct {
	Renderer
}

// Render produces a container graph where the the latest metadata contains the
// container image name, if found.
func (r containerWithImageNameRenderer) Render(rpt report.Report) Nodes {
	containers := r.Renderer.Render(rpt)
	images := SelectContainerImage.Render(rpt)

	outputs := make(report.Nodes, len(containers.Nodes))
	for id, c := range containers.Nodes {
		outputs[id] = c
		imageID, ok := c.Latest.Lookup(docker.ImageID)
		if !ok {
			continue
		}
		image, ok := images.Nodes[report.MakeContainerImageNodeID(imageID)]
		if !ok {
			continue
		}
		imageName, ok := image.Latest.Lookup(docker.ImageName)
		if !ok {
			continue
		}
		imageNameWithoutVersion := docker.ImageNameWithoutVersion(imageName)
		imageNodeID := report.MakeContainerImageNodeID(imageNameWithoutVersion)

		c = propagateLatest(docker.ImageName, image, c)
		c = propagateLatest(docker.ImageSize, image, c)
		c = propagateLatest(docker.ImageVirtualSize, image, c)
		c = propagateLatest(docker.ImageLabelPrefix+"works.weave.role", image, c)
		c.Parents = c.Parents.
			Delete(report.ContainerImage).
			Add(report.ContainerImage, report.MakeStringSet(imageNodeID))
		outputs[id] = c
	}
	return Nodes{Nodes: outputs, Filtered: containers.Filtered}
}

// ContainerWithImageNameRenderer is a Renderer which produces a container
// graph where the ranks are the image names, not their IDs
var ContainerWithImageNameRenderer = Memoise(containerWithImageNameRenderer{ContainerRenderer})

// ContainerImageRenderer is a Renderer which produces a renderable container
// image graph by merging the container graph and the container image topology.
var ContainerImageRenderer = Memoise(FilterEmpty(report.Container,
	MakeMap(
		MapContainerImage2Name,
		MakeReduce(
			MakeMap(
				MapContainer2ContainerImage,
				ContainerWithImageNameRenderer,
			),
			SelectContainerImage,
		),
	),
))

// ContainerHostnameRenderer is a Renderer which produces a renderable container
// by hostname graph..
//
// not memoised
var ContainerHostnameRenderer = FilterEmpty(report.Container,
	MakeReduce(
		MakeMap(
			MapContainer2Hostname,
			ContainerWithImageNameRenderer,
		),
		// Grab *all* the hostnames, so we can count the number which were empty
		// for accurate stats.
		MakeMap(
			MapToEmpty,
			MakeMap(
				MapContainer2Hostname,
				ContainerRenderer,
			),
		),
	),
)

var portMappingMatch = regexp.MustCompile(`([0-9]{1,3}\.[0-9]{1,3}\.[0-9]{1,3}\.[0-9]{1,3}):([0-9]+)->([0-9]+)/tcp`)

// MapContainer2IP maps container nodes to their IP addresses (outputs
// multiple nodes).  This allows container to be joined directly with
// the endpoint topology.
func MapContainer2IP(m report.Node) []string {
	// if this container doesn't make connections, we can ignore it
	_, doesntMakeConnections := m.Latest.Lookup(report.DoesNotMakeConnections)
	// if this container belongs to the host's networking namespace
	// we cannot use its IP to attribute connections
	// (they could come from any other process on the host or DNAT-ed IPs)
	_, isInHostNetwork := m.Latest.Lookup(docker.IsInHostNetwork)
	if doesntMakeConnections || isInHostNetwork {
		return nil
	}

	result := []string{}
	if addrs, ok := m.Sets.Lookup(docker.ContainerIPsWithScopes); ok {
		for _, addr := range addrs {
			scope, addr, ok := report.ParseAddressNodeID(addr)
			if !ok {
				continue
			}
			// loopback addresses are shared among all namespaces
			// so we can't use them to attribute connections to a container
			if report.IsLoopback(addr) {
				continue
			}
			id := report.MakeScopedEndpointNodeID(scope, addr, "")
			result = append(result, id)
		}
	}

	// Also output all the host:port port mappings (see above comment).
	// In this case we assume this doesn't need a scope, as they are for host IPs.
	ports, _ := m.Sets.Lookup(docker.ContainerPorts)
	for _, portMapping := range ports {
		if mapping := portMappingMatch.FindStringSubmatch(portMapping); mapping != nil {
			ip, port := mapping[1], mapping[2]
			id := report.MakeScopedEndpointNodeID("", ip, port)
			result = append(result, id)
		}
	}

	return result
}

// MapProcess2Container maps process Nodes to container
// Nodes.
//
// Pseudo nodes are passed straight through.
//
// If this function is given a node without a docker_container_id, it
// will produce an "Uncontained" pseudo node.
//
// Otherwise, this function will produce a node with the correct ID
// format for a container, but without any Major or Minor labels.
// It does not have enough info to do that, and the resulting graph
// must be merged with a container graph to get that info.
func MapProcess2Container(n report.Node) report.Nodes {
	// Propagate pseudo nodes
	if n.Topology == Pseudo {
		return report.Nodes{n.ID: n}
	}

	// Otherwise, if the process is not in a container, group it into
	// an per-host "Uncontained" node.  If for whatever reason this
	// node doesn't have a host id in their node metadata, it'll all
	// get grouped into a single uncontained node.
	var (
		id   string
		node report.Node
	)
	if containerID, ok := n.Latest.Lookup(docker.ContainerID); ok {
		id = report.MakeContainerNodeID(containerID)
		node = NewDerivedNode(id, n).WithTopology(report.Container)
	} else {
		id = MakePseudoNodeID(UncontainedID, report.ExtractHostID(n))
		node = NewDerivedPseudoNode(id, n)
		node = propagateLatest(report.HostNodeID, n, node)
		node = propagateLatest(IsConnectedMark, n, node)
	}
	return report.Nodes{id: node}
}

// MapContainer2ContainerImage maps container Nodes to container
// image Nodes.
//
// Pseudo nodes are passed straight through.
//
// If this function is given a node without a docker_image_id
// it will drop that node.
//
// Otherwise, this function will produce a node with the correct ID
// format for a container image, but without any Major or Minor
// labels.  It does not have enough info to do that, and the resulting
// graph must be merged with a container image graph to get that info.
func MapContainer2ContainerImage(n report.Node) report.Nodes {
	// Propagate all pseudo nodes
	if n.Topology == Pseudo {
		return report.Nodes{n.ID: n}
	}

	// Otherwise, if some some reason the container doesn't have a image_id
	// (maybe slightly out of sync reports), just drop it
	imageID, timestamp, ok := n.Latest.LookupEntry(docker.ImageID)
	if !ok {
		return report.Nodes{}
	}

	// Add container id key to the counters, which will later be
	// counted to produce the minor label
	id := report.MakeContainerImageNodeID(imageID)
	result := NewDerivedNode(id, n).WithTopology(report.ContainerImage)
	result.Latest = result.Latest.Set(docker.ImageID, timestamp, imageID)
	result.Counters = result.Counters.Add(n.Topology, 1)
	return report.Nodes{id: result}
}

// MapContainerImage2Name ignores image versions
func MapContainerImage2Name(n report.Node) report.Nodes {
	// Propagate all pseudo nodes
	if n.Topology == Pseudo {
		return report.Nodes{n.ID: n}
	}

	imageName, ok := n.Latest.Lookup(docker.ImageName)
	if !ok {
		return report.Nodes{}
	}

	imageNameWithoutVersion := docker.ImageNameWithoutVersion(imageName)
	n.ID = report.MakeContainerImageNodeID(imageNameWithoutVersion)

	if imageID, ok := report.ParseContainerImageNodeID(n.ID); ok {
		n.Sets = n.Sets.Add(docker.ImageID, report.MakeStringSet(imageID))
	}

	return report.Nodes{n.ID: n}
}

// MapContainer2Hostname maps container Nodes to 'hostname' renderabled nodes..
func MapContainer2Hostname(n report.Node) report.Nodes {
	// Propagate all pseudo nodes
	if n.Topology == Pseudo {
		return report.Nodes{n.ID: n}
	}

	// Otherwise, if some some reason the container doesn't have a hostname
	// (maybe slightly out of sync reports), just drop it
	id, ok := n.Latest.Lookup(docker.ContainerHostname)
	if !ok {
		return report.Nodes{}
	}

	node := NewDerivedNode(id, n).WithTopology(MakeGroupNodeTopology(n.Topology, docker.ContainerHostname))
	node.Counters = node.Counters.Add(n.Topology, 1)
	return report.Nodes{id: node}
}

// MapToEmpty removes all the attributes, children, etc, of a node. Useful when
// we just want to count the presence of nodes.
func MapToEmpty(n report.Node) report.Nodes {
	return report.Nodes{n.ID: report.MakeNode(n.ID).WithTopology(n.Topology)}
}
<|endoftext|>


<commit_before>//
// DISCLAIMER
//
// Copyright 2018-2021 ArangoDB GmbH, Cologne, Germany
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
//
// Copyright holder is ArangoDB GmbH, Cologne, Germany
//
// Author Ewout Prangsma
// Author Tomasz Mielech
//

package driver

import (
	"context"
	"errors"
	"path"
	"strconv"
	"sync/atomic"
	"time"
)

// Content of the create batch resp
type batchMetadata struct {
	// Id of the batch
	ID string `json:"id"`
	// Last Tick reported by the server
	LastTickInt Tick `json:"lastTick,omitempty"`

	cl       *client
	serverID int64
	database string
	closed   int32
}

// ErrBatchClosed occurs when there is an attempt closing or prolonging closed batch
var ErrBatchClosed = errors.New("Batch already closed")

// CreateBatch creates a "batch" to prevent WAL file removal and to take a snapshot
func (c *client) CreateBatch(ctx context.Context, db Database, serverID int64, ttl time.Duration) (Batch, error) {
	req, err := c.conn.NewRequest("POST", path.Join("_db", db.Name(), "_api/replication/batch"))
	if err != nil {
		return nil, WithStack(err)
	}
	req = req.SetQuery("serverId", strconv.FormatInt(serverID, 10))
	params := struct {
		TTL float64 `json:"ttl"`
	}{TTL: ttl.Seconds()} // just use a default ttl value
	req, err = req.SetBody(params)
	if err != nil {
		return nil, WithStack(err)
	}
	resp, err := c.conn.Do(ctx, req)
	if err != nil {
		return nil, WithStack(err)
	}
	if err := resp.CheckStatus(200); err != nil {
		return nil, WithStack(err)
	}
	var batch batchMetadata
	if err := resp.ParseBody("", &batch); err != nil {
		return nil, WithStack(err)
	}
	batch.cl = c
	batch.serverID = serverID
	batch.database = db.Name()
	return &batch, nil
}

// Get the inventory of a server containing all collections (with entire details) of a database.
func (c *client) DatabaseInventory(ctx context.Context, db Database) (DatabaseInventory, error) {
	req, err := c.conn.NewRequest("GET", path.Join("_db", db.Name(), "_api/replication/inventory"))
	if err != nil {
		return DatabaseInventory{}, WithStack(err)
	}
	applyContextSettings(ctx, req)
	resp, err := c.conn.Do(ctx, req)
	if err != nil {
		return DatabaseInventory{}, WithStack(err)
	}
	if err := resp.CheckStatus(200); err != nil {
		return DatabaseInventory{}, WithStack(err)
	}
	var result DatabaseInventory
	if err := resp.ParseBody("", &result); err != nil {
		return DatabaseInventory{}, WithStack(err)
	}
	return result, nil
}

// BatchID reported by the server
func (b batchMetadata) BatchID() string {
	return b.ID
}

// LastTick reported by the server for this batch
func (b batchMetadata) LastTick() Tick {
	return b.LastTickInt
}

// Extend the lifetime of an existing batch on the server
func (b *batchMetadata) Extend(ctx context.Context, ttl time.Duration) error {
	if !atomic.CompareAndSwapInt32(&b.closed, 0, 0) {
		return WithStack(ErrBatchClosed)
	}

	req, err := b.cl.conn.NewRequest("PUT", path.Join("_db", b.database, "_api/replication/batch", b.ID))
	if err != nil {
		return WithStack(err)
	}
	req = req.SetQuery("serverId", strconv.FormatInt(b.serverID, 10))
	input := struct {
		TTL int64 `json:"ttl"`
	}{
		TTL: int64(ttl.Seconds()),
	}
	req, err = req.SetBody(input)
	if err != nil {
		return WithStack(err)
	}
	resp, err := b.cl.conn.Do(ctx, req)
	if err != nil {
		return WithStack(err)
	}
	if err := resp.CheckStatus(204); err != nil {
		return WithStack(err)
	}
	return nil
}

// Delete an existing dump batch
func (b *batchMetadata) Delete(ctx context.Context) error {
	if !atomic.CompareAndSwapInt32(&b.closed, 0, 1) {
		return WithStack(ErrBatchClosed)
	}

	req, err := b.cl.conn.NewRequest("DELETE", path.Join("_db", b.database, "_api/replication/batch", b.ID))
	if err != nil {
		return WithStack(err)
	}
	resp, err := b.cl.conn.Do(ctx, req)
	if err != nil {
		return WithStack(err)
	}
	if err := resp.CheckStatus(204); err != nil {
		return WithStack(err)
	}
	return nil
}
<commit_msg>[Bugfix] Race condittion when reading from batch (#314)<commit_after>//
// DISCLAIMER
//
// Copyright 2018-2021 ArangoDB GmbH, Cologne, Germany
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
//
// Copyright holder is ArangoDB GmbH, Cologne, Germany
//
// Author Ewout Prangsma
// Author Tomasz Mielech
//

package driver

import (
	"context"
	"errors"
	"path"
	"strconv"
	"sync/atomic"
	"time"
)

// Content of the create batch resp
type batchMetadata struct {
	// Id of the batch
	ID string `json:"id"`
	// Last Tick reported by the server
	LastTickInt Tick `json:"lastTick,omitempty"`

	cl       *client
	serverID int64
	database string
	closed   int32
}

// ErrBatchClosed occurs when there is an attempt closing or prolonging closed batch
var ErrBatchClosed = errors.New("Batch already closed")

// CreateBatch creates a "batch" to prevent WAL file removal and to take a snapshot
func (c *client) CreateBatch(ctx context.Context, db Database, serverID int64, ttl time.Duration) (Batch, error) {
	req, err := c.conn.NewRequest("POST", path.Join("_db", db.Name(), "_api/replication/batch"))
	if err != nil {
		return nil, WithStack(err)
	}
	req = req.SetQuery("serverId", strconv.FormatInt(serverID, 10))
	params := struct {
		TTL float64 `json:"ttl"`
	}{TTL: ttl.Seconds()} // just use a default ttl value
	req, err = req.SetBody(params)
	if err != nil {
		return nil, WithStack(err)
	}
	resp, err := c.conn.Do(ctx, req)
	if err != nil {
		return nil, WithStack(err)
	}
	if err := resp.CheckStatus(200); err != nil {
		return nil, WithStack(err)
	}
	var batch batchMetadata
	if err := resp.ParseBody("", &batch); err != nil {
		return nil, WithStack(err)
	}
	batch.cl = c
	batch.serverID = serverID
	batch.database = db.Name()
	return &batch, nil
}

// Get the inventory of a server containing all collections (with entire details) of a database.
func (c *client) DatabaseInventory(ctx context.Context, db Database) (DatabaseInventory, error) {
	req, err := c.conn.NewRequest("GET", path.Join("_db", db.Name(), "_api/replication/inventory"))
	if err != nil {
		return DatabaseInventory{}, WithStack(err)
	}
	applyContextSettings(ctx, req)
	resp, err := c.conn.Do(ctx, req)
	if err != nil {
		return DatabaseInventory{}, WithStack(err)
	}
	if err := resp.CheckStatus(200); err != nil {
		return DatabaseInventory{}, WithStack(err)
	}
	var result DatabaseInventory
	if err := resp.ParseBody("", &result); err != nil {
		return DatabaseInventory{}, WithStack(err)
	}
	return result, nil
}

// BatchID reported by the server
// The receiver is pointer because this struct contains the field `closed` and it can not be copied
// because race detector will complain.
func (b *batchMetadata) BatchID() string {
	return b.ID
}

// LastTick reported by the server for this batch
// The receiver is pointer because this struct contains the field `closed` and it can not be copied
// because race detector will complain.
func (b *batchMetadata) LastTick() Tick {
	return b.LastTickInt
}

// Extend the lifetime of an existing batch on the server
func (b *batchMetadata) Extend(ctx context.Context, ttl time.Duration) error {
	if !atomic.CompareAndSwapInt32(&b.closed, 0, 0) {
		return WithStack(ErrBatchClosed)
	}

	req, err := b.cl.conn.NewRequest("PUT", path.Join("_db", b.database, "_api/replication/batch", b.ID))
	if err != nil {
		return WithStack(err)
	}
	req = req.SetQuery("serverId", strconv.FormatInt(b.serverID, 10))
	input := struct {
		TTL int64 `json:"ttl"`
	}{
		TTL: int64(ttl.Seconds()),
	}
	req, err = req.SetBody(input)
	if err != nil {
		return WithStack(err)
	}
	resp, err := b.cl.conn.Do(ctx, req)
	if err != nil {
		return WithStack(err)
	}
	if err := resp.CheckStatus(204); err != nil {
		return WithStack(err)
	}
	return nil
}

// Delete an existing dump batch
func (b *batchMetadata) Delete(ctx context.Context) error {
	if !atomic.CompareAndSwapInt32(&b.closed, 0, 1) {
		return WithStack(ErrBatchClosed)
	}

	req, err := b.cl.conn.NewRequest("DELETE", path.Join("_db", b.database, "_api/replication/batch", b.ID))
	if err != nil {
		return WithStack(err)
	}
	resp, err := b.cl.conn.Do(ctx, req)
	if err != nil {
		return WithStack(err)
	}
	if err := resp.CheckStatus(204); err != nil {
		return WithStack(err)
	}
	return nil
}
<|endoftext|>


<commit_before>//go:generate esc -o ../../player/mp3/mp3.go -pkg mp3 ../../player/mp3

/*
	this is implementation azan in the form of cli
*/

package main

import (
	"encoding/json"
	"fmt"
	"io/ioutil"
	"os"
	"os/signal"
	"syscall"
	"time"

	azan "github.com/trihatmaja/Azan-Schedule"

	"github.com/trihatmaja/Azan-Schedule/calculation"
	"github.com/trihatmaja/Azan-Schedule/database"
	"github.com/trihatmaja/Azan-Schedule/player"
	"github.com/trihatmaja/Azan-Schedule/player/mp3"

	"github.com/jasonlvhit/gocron"
	"github.com/urfave/cli"
)

// apps var
var (
	latitude  float64
	longitude float64
	timezone  float64
	city      string
	outputdir string
	filename  string
)

// main var
var (
	Version string
	Build   string
)

func checkSchedule() {
	t := time.Now()
	tgl := t.Format("2006-January-2")
	jam := t.Format("15:04")

	f, err := ioutil.ReadFile("schedule.json")
	if err != nil {
		fmt.Println(err)
	}
	k := azan.CalcResult{}

	err = json.Unmarshal(f, &k)
	if err != nil {
		fmt.Println(err)
	}

	for _, v := range k.Schedule {
		if tgl == v.Date {
			switch {
			case jam == v.Fajr:
				player.Play(mp3.FSMustByte(false, "/player/mp3/fajr.mp3"))
			case jam == v.Zuhr:
				player.Play(mp3.FSMustByte(false, "/player/mp3/azan.mp3"))
			case jam == v.Asr:
				player.Play(mp3.FSMustByte(false, "/player/mp3/azan.mp3"))
			case jam == v.Maghrib:
				player.Play(mp3.FSMustByte(false, "/player/mp3/azan.mp3"))
			case jam == v.Isya:
				player.Play(mp3.FSMustByte(false, "/player/mp3/azan.mp3"))
			default:
				break
			}
		}
	}
}

func main() {
	app := cli.NewApp()
	app.HideVersion = true
	app.Name = "azan schedule"
	app.Usage = "generate files for azan schedule each day in a year"
	app.Commands = []cli.Command{
		cli.Command{
			Name:  "generate",
			Usage: "generate azan files",
			Action: func(c *cli.Context) error {
				opt := database.OptionFiles{
					OutputDir: outputdir,
					FileName:  filename,
				}

				db := database.NewFiles(opt)

				calc := calculation.NewTDjamaluddin()

				// in cli apps, no need cache mechanism
				// the cache mechanism is nil
				az := azan.New(db, nil, calc)
				az.Generate(latitude, longitude, timezone, city)
				return nil
			},
			Flags: []cli.Flag{
				cli.Float64Flag{
					Name:        "latitude",
					Destination: &latitude,
					Value:       -6.18,
				},
				cli.Float64Flag{
					Name:        "longitude",
					Destination: &longitude,
					Value:       106.83,
				},
				cli.Float64Flag{
					Name:        "timezone",
					Destination: &timezone,
					Value:       +7,
				},
				cli.StringFlag{
					Name:        "city",
					Destination: &city,
					Value:       "Jakarta",
				},
				cli.StringFlag{
					Name:        "outputdir",
					Destination: &outputdir,
					Value:       ".",
				},
				cli.StringFlag{
					Name:        "filename",
					Destination: &filename,
					Value:       "schedule.json",
				},
			},
		},
		cli.Command{
			Name:  "version",
			Usage: "azan schedule version",
			Action: func(c *cli.Context) error {
				fmt.Printf("{Version: %s, Build: %s}", Version, Build)
				return nil
			},
		},
		cli.Command{
			Name:  "play",
			Usage: "play azan",
			Action: func(c *cli.Context) error {
				player.Play(mp3.FSMustByte(false, "/player/mp3/azan.mp3"))
				return nil
			},
		},
		cli.Command{
			Name:  "start",
			Usage: "starting praying schedule",
			Action: func(c *cli.Context) error {
				fmt.Println("Starting schedule...")

				var gracefulStop = make(chan os.Signal)
				signal.Notify(gracefulStop, syscall.SIGTERM)
				signal.Notify(gracefulStop, syscall.SIGINT)
				go func() {
					sig := <-gracefulStop
					fmt.Println("\nCaught sig:", sig)
					fmt.Println("Wait for apps to gracefully stop")
					gocron.Remove(checkSchedule)
					gocron.Clear()
					os.Exit(0)
				}()

				fmt.Println("Check file schedule.json...")

				if _, err := os.Stat("schedule.json"); os.IsNotExist(err) {
					fmt.Println("Cannot find file schedule.json, please run generate first!")
					return nil
				}

				fmt.Println("Ok!")
				fmt.Println("Running job..")

				gocron.Every(1).Minute().Do(checkSchedule)
				<-gocron.Start()
				return nil
			},
		},
	}

	app.Run(os.Args)
}
<commit_msg>  fix not read file every minute..<commit_after>//go:generate esc -o ../../player/mp3/mp3.go -pkg mp3 ../../player/mp3

/*
	this is implementation azan in the form of cli
*/

package main

import (
	"encoding/json"
	"fmt"
	"io/ioutil"
	"os"
	"os/signal"
	"syscall"
	"time"

	azan "github.com/trihatmaja/Azan-Schedule"

	"github.com/trihatmaja/Azan-Schedule/calculation"
	"github.com/trihatmaja/Azan-Schedule/database"
	"github.com/trihatmaja/Azan-Schedule/player"
	"github.com/trihatmaja/Azan-Schedule/player/mp3"

	"github.com/jasonlvhit/gocron"
	"github.com/urfave/cli"
)

// apps var
var (
	latitude  float64
	longitude float64
	timezone  float64
	city      string
	outputdir string
	filename  string
)

// main var
var (
	Version string
	Build   string
)

func checkSchedule(k *azan.CalcResult) {
	t := time.Now()
	tgl := t.Format("2006-January-2")
	jam := t.Format("15:04")

	for _, v := range k.Schedule {
		if tgl == v.Date {
			switch {
			case jam == v.Fajr:
				fmt.Println("Fajr Pray Time")
				player.Play(mp3.FSMustByte(false, "/player/mp3/fajr.mp3"))
			case jam == v.Zuhr:
				fmt.Println("Zuhr Pray Time")
				player.Play(mp3.FSMustByte(false, "/player/mp3/azan.mp3"))
			case jam == v.Asr:
				fmt.Println("Asr Pray Time")
				player.Play(mp3.FSMustByte(false, "/player/mp3/azan.mp3"))
			case jam == v.Maghrib:
				fmt.Println("Maghrib Pray Time")
				player.Play(mp3.FSMustByte(false, "/player/mp3/azan.mp3"))
			case jam == v.Isya:
				fmt.Println("Isya' Pray Time")
				player.Play(mp3.FSMustByte(false, "/player/mp3/azan.mp3"))
			default:
				break
			}
		}
	}
}

func main() {
	app := cli.NewApp()
	app.HideVersion = true
	app.Name = "azan schedule"
	app.Usage = "generate files for azan schedule each day in a year"
	app.Commands = []cli.Command{
		cli.Command{
			Name:  "generate",
			Usage: "generate azan files",
			Action: func(c *cli.Context) error {
				opt := database.OptionFiles{
					OutputDir: outputdir,
					FileName:  filename,
				}

				db := database.NewFiles(opt)

				calc := calculation.NewTDjamaluddin()

				// in cli apps, no need cache mechanism
				// the cache mechanism is nil
				az := azan.New(db, nil, calc)
				az.Generate(latitude, longitude, timezone, city)
				return nil
			},
			Flags: []cli.Flag{
				cli.Float64Flag{
					Name:        "latitude",
					Destination: &latitude,
					Value:       -6.18,
				},
				cli.Float64Flag{
					Name:        "longitude",
					Destination: &longitude,
					Value:       106.83,
				},
				cli.Float64Flag{
					Name:        "timezone",
					Destination: &timezone,
					Value:       +7,
				},
				cli.StringFlag{
					Name:        "city",
					Destination: &city,
					Value:       "Jakarta",
				},
				cli.StringFlag{
					Name:        "outputdir",
					Destination: &outputdir,
					Value:       ".",
				},
				cli.StringFlag{
					Name:        "filename",
					Destination: &filename,
					Value:       "schedule.json",
				},
			},
		},
		cli.Command{
			Name:  "version",
			Usage: "azan schedule version",
			Action: func(c *cli.Context) error {
				fmt.Printf("{Version: %s, Build: %s}", Version, Build)
				return nil
			},
		},
		cli.Command{
			Name:  "play",
			Usage: "play azan",
			Action: func(c *cli.Context) error {
				player.Play(mp3.FSMustByte(false, "/player/mp3/azan.mp3"))
				return nil
			},
		},
		cli.Command{
			Name:  "start",
			Usage: "starting praying schedule",
			Action: func(c *cli.Context) error {
				fmt.Println("Starting schedule...")

				//gocron.Every(1).Minute().Do(checkSchedule, &k)
				s := gocron.NewScheduler()

				var gracefulStop = make(chan os.Signal)
				signal.Notify(gracefulStop, syscall.SIGTERM)
				signal.Notify(gracefulStop, syscall.SIGINT)
				go func(s *gocron.Scheduler) {
					sig := <-gracefulStop
					fmt.Println("\nCaught sig:", sig)
					fmt.Println("Wait for apps to gracefully stop")
					s.Remove(checkSchedule)
					s.Clear()
					os.Exit(0)
				}(s)

				fmt.Println("Check file schedule.json...")

				if _, err := os.Stat("schedule.json"); os.IsNotExist(err) {
					fmt.Println("Cannot find file schedule.json, please run generate first!")
					return nil
				}

				fmt.Println("Ok!")

				f, err := ioutil.ReadFile("schedule.json")
				if err != nil {
					fmt.Println(err)
				}
				k := azan.CalcResult{}

				err = json.Unmarshal(f, &k)
				if err != nil {
					fmt.Println(err)
				}

				fmt.Println("Running job..")
				s.Every(1).Minute().Do(checkSchedule, &k)
				<-s.Start()
				return nil
			},
		},
	}

	app.Run(os.Args)
}
<|endoftext|>


<commit_before>package restic_test

import (
	"bytes"
	"crypto/sha256"
	"io"
	"testing"

	"github.com/restic/chunker"
	"github.com/restic/restic"
	"github.com/restic/restic/backend"
	"github.com/restic/restic/crypto"
	"github.com/restic/restic/pack"
	. "github.com/restic/restic/test"
)

var testPol = chunker.Pol(0x3DA3358B4DC173)

type Rdr interface {
	io.ReadSeeker
	io.ReaderAt
}

func benchmarkChunkEncrypt(b testing.TB, buf, buf2 []byte, rd Rdr, key *crypto.Key) {
	rd.Seek(0, 0)
	ch := chunker.New(rd, testPol, sha256.New())

	for {
		chunk, err := ch.Next()

		if err == io.EOF {
			break
		}

		OK(b, err)

		// reduce length of buf
		buf = buf[:chunk.Length]
		n, err := io.ReadFull(chunk.Reader(rd), buf)
		OK(b, err)
		Assert(b, uint(n) == chunk.Length, "invalid length: got %d, expected %d", n, chunk.Length)

		_, err = crypto.Encrypt(key, buf2, buf)
		OK(b, err)
	}
}

func BenchmarkChunkEncrypt(b *testing.B) {
	repo := SetupRepo()
	defer TeardownRepo(repo)

	data := Random(23, 10<<20) // 10MiB
	rd := bytes.NewReader(data)

	buf := make([]byte, chunker.MaxSize)
	buf2 := make([]byte, chunker.MaxSize)

	b.ResetTimer()
	b.SetBytes(int64(len(data)))

	for i := 0; i < b.N; i++ {
		benchmarkChunkEncrypt(b, buf, buf2, rd, repo.Key())
	}
}

func benchmarkChunkEncryptP(b *testing.PB, buf []byte, rd Rdr, key *crypto.Key) {
	ch := chunker.New(rd, testPol, sha256.New())

	for {
		chunk, err := ch.Next()
		if err == io.EOF {
			break
		}

		// reduce length of chunkBuf
		buf = buf[:chunk.Length]
		io.ReadFull(chunk.Reader(rd), buf)
		crypto.Encrypt(key, buf, buf)
	}
}

func BenchmarkChunkEncryptParallel(b *testing.B) {
	repo := SetupRepo()
	defer TeardownRepo(repo)

	data := Random(23, 10<<20) // 10MiB

	buf := make([]byte, chunker.MaxSize)

	b.ResetTimer()
	b.SetBytes(int64(len(data)))

	b.RunParallel(func(pb *testing.PB) {
		for pb.Next() {
			rd := bytes.NewReader(data)
			benchmarkChunkEncryptP(pb, buf, rd, repo.Key())
		}
	})
}

func archiveDirectory(b testing.TB) {
	repo := SetupRepo()
	defer TeardownRepo(repo)

	arch := restic.NewArchiver(repo)

	_, id, err := arch.Snapshot(nil, []string{BenchArchiveDirectory}, nil)
	OK(b, err)

	b.Logf("snapshot archived as %v", id)
}

func TestArchiveDirectory(t *testing.T) {
	if BenchArchiveDirectory == "" {
		t.Skip("benchdir not set, skipping TestArchiveDirectory")
	}

	archiveDirectory(t)
}

func BenchmarkArchiveDirectory(b *testing.B) {
	if BenchArchiveDirectory == "" {
		b.Skip("benchdir not set, skipping BenchmarkArchiveDirectory")
	}

	for i := 0; i < b.N; i++ {
		archiveDirectory(b)
	}
}

func archiveWithDedup(t testing.TB) {
	repo := SetupRepo()
	defer TeardownRepo(repo)

	if BenchArchiveDirectory == "" {
		t.Skip("benchdir not set, skipping TestArchiverDedup")
	}

	var cnt struct {
		before, after, after2 struct {
			packs, dataBlobs, treeBlobs uint
		}
	}

	// archive a few files
	sn := SnapshotDir(t, repo, BenchArchiveDirectory, nil)
	t.Logf("archived snapshot %v", sn.ID().Str())

	// get archive stats
	cnt.before.packs = repo.Count(backend.Data)
	cnt.before.dataBlobs = repo.Index().Count(pack.Data)
	cnt.before.treeBlobs = repo.Index().Count(pack.Tree)
	t.Logf("packs %v, data blobs %v, tree blobs %v",
		cnt.before.packs, cnt.before.dataBlobs, cnt.before.treeBlobs)

	// archive the same files again, without parent snapshot
	sn2 := SnapshotDir(t, repo, BenchArchiveDirectory, nil)
	t.Logf("archived snapshot %v", sn2.ID().Str())

	// get archive stats again
	cnt.after.packs = repo.Count(backend.Data)
	cnt.after.dataBlobs = repo.Index().Count(pack.Data)
	cnt.after.treeBlobs = repo.Index().Count(pack.Tree)
	t.Logf("packs %v, data blobs %v, tree blobs %v",
		cnt.after.packs, cnt.after.dataBlobs, cnt.after.treeBlobs)

	// if there are more data blobs, something is wrong
	if cnt.after.dataBlobs > cnt.before.dataBlobs {
		t.Fatalf("TestArchiverDedup: too many data blobs in repository: before %d, after %d",
			cnt.before.dataBlobs, cnt.after.dataBlobs)
	}

	// archive the same files again, with a parent snapshot
	sn3 := SnapshotDir(t, repo, BenchArchiveDirectory, sn2.ID())
	t.Logf("archived snapshot %v, parent %v", sn3.ID().Str(), sn2.ID().Str())

	// get archive stats again
	cnt.after2.packs = repo.Count(backend.Data)
	cnt.after2.dataBlobs = repo.Index().Count(pack.Data)
	cnt.after2.treeBlobs = repo.Index().Count(pack.Tree)
	t.Logf("packs %v, data blobs %v, tree blobs %v",
		cnt.after2.packs, cnt.after2.dataBlobs, cnt.after2.treeBlobs)

	// if there are more data blobs, something is wrong
	if cnt.after2.dataBlobs > cnt.before.dataBlobs {
		t.Fatalf("TestArchiverDedup: too many data blobs in repository: before %d, after %d",
			cnt.before.dataBlobs, cnt.after2.dataBlobs)
	}
}

func TestArchiveDedup(t *testing.T) {
	archiveWithDedup(t)
}

func BenchmarkLoadTree(t *testing.B) {
	repo := SetupRepo()
	defer TeardownRepo(repo)

	if BenchArchiveDirectory == "" {
		t.Skip("benchdir not set, skipping TestArchiverDedup")
	}

	// archive a few files
	arch := restic.NewArchiver(repo)
	sn, _, err := arch.Snapshot(nil, []string{BenchArchiveDirectory}, nil)
	OK(t, err)
	t.Logf("archived snapshot %v", sn.ID())

	list := make([]backend.ID, 0, 10)
	done := make(chan struct{})

	for _, idx := range repo.Index().All() {
		for blob := range idx.Each(done) {
			if blob.Type != pack.Tree {
				continue
			}

			list = append(list, blob.ID)
			if len(list) == cap(list) {
				close(done)
				break
			}
		}
	}

	// start benchmark
	t.ResetTimer()

	for i := 0; i < t.N; i++ {
		for _, id := range list {
			_, err := restic.LoadTree(repo, id)
			OK(t, err)
		}
	}
}
<commit_msg>Add a test concurrently saving duplicated chunks<commit_after>package restic_test

import (
	"bytes"
	"crypto/sha256"
	"io"
	"math"
	"testing"

	"github.com/restic/chunker"
	"github.com/restic/restic"
	"github.com/restic/restic/backend"
	"github.com/restic/restic/checker"
	"github.com/restic/restic/crypto"
	"github.com/restic/restic/pack"
	"github.com/restic/restic/repository"
	. "github.com/restic/restic/test"
)

var testPol = chunker.Pol(0x3DA3358B4DC173)

type Rdr interface {
	io.ReadSeeker
	io.ReaderAt
}

type chunkedData struct {
	buf    []byte
	chunks []*chunker.Chunk
}

func benchmarkChunkEncrypt(b testing.TB, buf, buf2 []byte, rd Rdr, key *crypto.Key) {
	rd.Seek(0, 0)
	ch := chunker.New(rd, testPol, sha256.New())

	for {
		chunk, err := ch.Next()

		if err == io.EOF {
			break
		}

		OK(b, err)

		// reduce length of buf
		buf = buf[:chunk.Length]
		n, err := io.ReadFull(chunk.Reader(rd), buf)
		OK(b, err)
		Assert(b, uint(n) == chunk.Length, "invalid length: got %d, expected %d", n, chunk.Length)

		_, err = crypto.Encrypt(key, buf2, buf)
		OK(b, err)
	}
}

func BenchmarkChunkEncrypt(b *testing.B) {
	repo := SetupRepo()
	defer TeardownRepo(repo)

	data := Random(23, 10<<20) // 10MiB
	rd := bytes.NewReader(data)

	buf := make([]byte, chunker.MaxSize)
	buf2 := make([]byte, chunker.MaxSize)

	b.ResetTimer()
	b.SetBytes(int64(len(data)))

	for i := 0; i < b.N; i++ {
		benchmarkChunkEncrypt(b, buf, buf2, rd, repo.Key())
	}
}

func benchmarkChunkEncryptP(b *testing.PB, buf []byte, rd Rdr, key *crypto.Key) {
	ch := chunker.New(rd, testPol, sha256.New())

	for {
		chunk, err := ch.Next()
		if err == io.EOF {
			break
		}

		// reduce length of chunkBuf
		buf = buf[:chunk.Length]
		io.ReadFull(chunk.Reader(rd), buf)
		crypto.Encrypt(key, buf, buf)
	}
}

func BenchmarkChunkEncryptParallel(b *testing.B) {
	repo := SetupRepo()
	defer TeardownRepo(repo)

	data := Random(23, 10<<20) // 10MiB

	buf := make([]byte, chunker.MaxSize)

	b.ResetTimer()
	b.SetBytes(int64(len(data)))

	b.RunParallel(func(pb *testing.PB) {
		for pb.Next() {
			rd := bytes.NewReader(data)
			benchmarkChunkEncryptP(pb, buf, rd, repo.Key())
		}
	})
}

func archiveDirectory(b testing.TB) {
	repo := SetupRepo()
	defer TeardownRepo(repo)

	arch := restic.NewArchiver(repo)

	_, id, err := arch.Snapshot(nil, []string{BenchArchiveDirectory}, nil)
	OK(b, err)

	b.Logf("snapshot archived as %v", id)
}

func TestArchiveDirectory(t *testing.T) {
	if BenchArchiveDirectory == "" {
		t.Skip("benchdir not set, skipping TestArchiveDirectory")
	}

	archiveDirectory(t)
}

func BenchmarkArchiveDirectory(b *testing.B) {
	if BenchArchiveDirectory == "" {
		b.Skip("benchdir not set, skipping BenchmarkArchiveDirectory")
	}

	for i := 0; i < b.N; i++ {
		archiveDirectory(b)
	}
}

func archiveWithDedup(t testing.TB) {
	repo := SetupRepo()
	defer TeardownRepo(repo)

	if BenchArchiveDirectory == "" {
		t.Skip("benchdir not set, skipping TestArchiverDedup")
	}

	var cnt struct {
		before, after, after2 struct {
			packs, dataBlobs, treeBlobs uint
		}
	}

	// archive a few files
	sn := SnapshotDir(t, repo, BenchArchiveDirectory, nil)
	t.Logf("archived snapshot %v", sn.ID().Str())

	// get archive stats
	cnt.before.packs = repo.Count(backend.Data)
	cnt.before.dataBlobs = repo.Index().Count(pack.Data)
	cnt.before.treeBlobs = repo.Index().Count(pack.Tree)
	t.Logf("packs %v, data blobs %v, tree blobs %v",
		cnt.before.packs, cnt.before.dataBlobs, cnt.before.treeBlobs)

	// archive the same files again, without parent snapshot
	sn2 := SnapshotDir(t, repo, BenchArchiveDirectory, nil)
	t.Logf("archived snapshot %v", sn2.ID().Str())

	// get archive stats again
	cnt.after.packs = repo.Count(backend.Data)
	cnt.after.dataBlobs = repo.Index().Count(pack.Data)
	cnt.after.treeBlobs = repo.Index().Count(pack.Tree)
	t.Logf("packs %v, data blobs %v, tree blobs %v",
		cnt.after.packs, cnt.after.dataBlobs, cnt.after.treeBlobs)

	// if there are more data blobs, something is wrong
	if cnt.after.dataBlobs > cnt.before.dataBlobs {
		t.Fatalf("TestArchiverDedup: too many data blobs in repository: before %d, after %d",
			cnt.before.dataBlobs, cnt.after.dataBlobs)
	}

	// archive the same files again, with a parent snapshot
	sn3 := SnapshotDir(t, repo, BenchArchiveDirectory, sn2.ID())
	t.Logf("archived snapshot %v, parent %v", sn3.ID().Str(), sn2.ID().Str())

	// get archive stats again
	cnt.after2.packs = repo.Count(backend.Data)
	cnt.after2.dataBlobs = repo.Index().Count(pack.Data)
	cnt.after2.treeBlobs = repo.Index().Count(pack.Tree)
	t.Logf("packs %v, data blobs %v, tree blobs %v",
		cnt.after2.packs, cnt.after2.dataBlobs, cnt.after2.treeBlobs)

	// if there are more data blobs, something is wrong
	if cnt.after2.dataBlobs > cnt.before.dataBlobs {
		t.Fatalf("TestArchiverDedup: too many data blobs in repository: before %d, after %d",
			cnt.before.dataBlobs, cnt.after2.dataBlobs)
	}
}

func TestArchiveDedup(t *testing.T) {
	archiveWithDedup(t)
}

func BenchmarkLoadTree(t *testing.B) {
	repo := SetupRepo()
	defer TeardownRepo(repo)

	if BenchArchiveDirectory == "" {
		t.Skip("benchdir not set, skipping TestArchiverDedup")
	}

	// archive a few files
	arch := restic.NewArchiver(repo)
	sn, _, err := arch.Snapshot(nil, []string{BenchArchiveDirectory}, nil)
	OK(t, err)
	t.Logf("archived snapshot %v", sn.ID())

	list := make([]backend.ID, 0, 10)
	done := make(chan struct{})

	for _, idx := range repo.Index().All() {
		for blob := range idx.Each(done) {
			if blob.Type != pack.Tree {
				continue
			}

			list = append(list, blob.ID)
			if len(list) == cap(list) {
				close(done)
				break
			}
		}
	}

	// start benchmark
	t.ResetTimer()

	for i := 0; i < t.N; i++ {
		for _, id := range list {
			_, err := restic.LoadTree(repo, id)
			OK(t, err)
		}
	}
}

// Saves several identical chunks concurrently and later check that there are no
// unreferenced packs in the repository. See also #292 and #358.
// The combination of high duplication and high concurrency should provoke any
// issues leading to unreferenced packs.
func TestParallelSaveWithHighDuplication(t *testing.T) {
	repo := SetupRepo()
	defer TeardownRepo(repo)

	// For every seed a pseudo-random 32Mb blob is generated and split into
	// chunks. During the test all chunks of all blobs are processed in parallel
	// goroutines. To increase duplication, each chunk is processed
	// <duplication> times. Concurrency can be limited by changing <maxParallel>.
	// Note: seeds 5, 3, 66, 4, 12 produce the most chunks (descending)
	seeds := []int{5, 3, 66, 4, 12}
	maxParallel := math.MaxInt32
	duplication := 15

	arch := restic.NewArchiver(repo)
	data := getRandomData(seeds)

	barrier := make(chan struct{}, maxParallel)
	errChannels := [](<-chan error){}

	for _, d := range data {
		for _, c := range d.chunks {
			for dupIdx := 0; dupIdx < duplication; dupIdx++ {
				errChan := make(chan error)
				errChannels = append(errChannels, errChan)

				go func(buf *[]byte, c *chunker.Chunk, errChan chan<- error) {
					barrier <- struct{}{}

					hash := c.Digest
					id := backend.ID{}
					copy(id[:], hash)

					err := arch.Save(pack.Data, id, c.Length, c.Reader(bytes.NewReader(*buf)))
					<-barrier
					errChan <- err
				}(&d.buf, c, errChan)
			}
		}
	}

	for _, errChan := range errChannels {
		OK(t, <-errChan)
	}

	OK(t, repo.Flush())
	OK(t, repo.SaveIndex())

	chkr := createAndInitChecker(t, repo)
	assertNoUnreferencedPacks(t, chkr)
}

func getRandomData(seeds []int) []*chunkedData {
	chunks := []*chunkedData{}
	sem := make(chan struct{}, len(seeds))

	for seed := range seeds {
		c := &chunkedData{}
		chunks = append(chunks, c)

		go func(seed int, data *chunkedData) {
			data.buf = Random(seed, 32*1024*1024)
			chunker := chunker.New(bytes.NewReader(data.buf), testPol, sha256.New())

			for {
				c, err := chunker.Next()
				if err == io.EOF {
					break
				}
				data.chunks = append(data.chunks, c)
			}

			sem <- struct{}{}
		}(seed, c)
	}

	for i := 0; i < len(seeds); i++ {
		<-sem
	}
	return chunks
}

func createAndInitChecker(t *testing.T, repo *repository.Repository) *checker.Checker {
	chkr := checker.New(repo)

	hints, errs := chkr.LoadIndex()
	if len(errs) > 0 {
		t.Fatalf("expected no errors, got %v: %v", len(errs), errs)
	}

	if len(hints) > 0 {
		t.Errorf("expected no hints, got %v: %v", len(hints), hints)
	}

	return chkr
}

func assertNoUnreferencedPacks(t *testing.T, chkr *checker.Checker) {
	done := make(chan struct{})
	defer close(done)

	errChan := make(chan error)
	go chkr.Packs(errChan, done)

	for err := range errChan {
		OK(t, err)
	}
}
<|endoftext|>


<commit_before>// Copyright 2017 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

// Package argon2 implements the key derivation function Argon2.
// Argon2 was selected as the winner of the Password Hashing Competition and can
// be used to derive cryptographic keys from passwords.
// Argon2 is specfifed at https://github.com/P-H-C/phc-winner-argon2/blob/master/argon2-specs.pdf
package argon2

import (
	"encoding/binary"
	"sync"

	"golang.org/x/crypto/blake2b"
)

// The Argon2 version implemented by this package.
const Version = 0x13

const (
	argon2d = iota
	argon2i
	argon2id
)

// Key derives a key from the password, salt, and cost parameters using Argon2i
// returning a byte slice of length keyLen that can be used as cryptographic key.
// The CPU cost and parallism degree must be greater than zero.
//
// For example, you can get a derived key for e.g. AES-256 (which needs a 32-byte key) by doing:
// `key := argon2.Key([]byte("some password"), salt, 4, 32*1024, 4, 32)`
//
// The recommended parameters for interactive logins as of 2017 are time=4, memory=32*1024.
// The number of threads can be adjusted to the numbers of available CPUs.
// The time parameter specifies the number of passes over the memory and the memory
// parameter specifies the size of the memory in KiB. For example memory=32*1024 sets the
// memory cost to ~32 MB.
// The cost parameters should be increased as memory latency and CPU parallelism increases.
// Remember to get a good random salt.
func Key(password, salt []byte, time, memory uint32, threads uint8, keyLen uint32) []byte {
	return deriveKey(argon2i, password, salt, nil, nil, time, memory, threads, keyLen)
}

func deriveKey(mode int, password, salt, secret, data []byte, time, memory uint32, threads uint8, keyLen uint32) []byte {
	if time < 1 {
		panic("argon2: number of rounds too small")
	}
	if threads < 1 {
		panic("argon2: parallelism degree too low")
	}
	h0 := initHash(password, salt, secret, data, time, memory, uint32(threads), keyLen, mode)

	memory = memory / (syncPoints * uint32(threads)) * (syncPoints * uint32(threads))
	if memory < 2*syncPoints*uint32(threads) {
		memory = 2 * syncPoints * uint32(threads)
	}
	B := initBlocks(&h0, memory, uint32(threads))
	processBlocks(B, time, memory, uint32(threads), mode)
	return extractKey(B, memory, uint32(threads), keyLen)
}

const (
	blockLength = 128
	syncPoints  = 4
)

type block [blockLength]uint64

func initHash(password, salt, key, data []byte, time, memory, threads, keyLen uint32, mode int) [blake2b.Size + 8]byte {
	var (
		h0     [blake2b.Size + 8]byte
		params [24]byte
		tmp    [4]byte
	)

	b2, _ := blake2b.New512(nil)
	binary.LittleEndian.PutUint32(params[0:4], threads)
	binary.LittleEndian.PutUint32(params[4:8], keyLen)
	binary.LittleEndian.PutUint32(params[8:12], memory)
	binary.LittleEndian.PutUint32(params[12:16], time)
	binary.LittleEndian.PutUint32(params[16:20], uint32(Version))
	binary.LittleEndian.PutUint32(params[20:24], uint32(mode))
	b2.Write(params[:])
	binary.LittleEndian.PutUint32(tmp[:], uint32(len(password)))
	b2.Write(tmp[:])
	b2.Write(password)
	binary.LittleEndian.PutUint32(tmp[:], uint32(len(salt)))
	b2.Write(tmp[:])
	b2.Write(salt)
	binary.LittleEndian.PutUint32(tmp[:], uint32(len(key)))
	b2.Write(tmp[:])
	b2.Write(key)
	binary.LittleEndian.PutUint32(tmp[:], uint32(len(data)))
	b2.Write(tmp[:])
	b2.Write(data)
	b2.Sum(h0[:0])
	return h0
}

func initBlocks(h0 *[blake2b.Size + 8]byte, memory, threads uint32) []block {
	var block0 [1024]byte
	B := make([]block, memory)
	for lane := uint32(0); lane < threads; lane++ {
		j := lane * (memory / threads)
		binary.LittleEndian.PutUint32(h0[blake2b.Size+4:], lane)

		binary.LittleEndian.PutUint32(h0[blake2b.Size:], 0)
		blake2bHash(block0[:], h0[:])
		for i := range B[j+0] {
			B[j+0][i] = binary.LittleEndian.Uint64(block0[i*8:])
		}

		binary.LittleEndian.PutUint32(h0[blake2b.Size:], 1)
		blake2bHash(block0[:], h0[:])
		for i := range B[j+1] {
			B[j+1][i] = binary.LittleEndian.Uint64(block0[i*8:])
		}
	}
	return B
}

func processBlocks(B []block, time, memory, threads uint32, mode int) {
	lanes := memory / threads
	segments := lanes / syncPoints

	processSegment := func(n, slice, lane uint32, wg *sync.WaitGroup) {
		var addresses, in, zero block
		if mode == argon2i || (mode == argon2id && n == 0 && slice < syncPoints/2) {
			in[0] = uint64(n)
			in[1] = uint64(lane)
			in[2] = uint64(slice)
			in[3] = uint64(memory)
			in[4] = uint64(time)
			in[5] = uint64(mode)
		}

		index := uint32(0)
		if n == 0 && slice == 0 {
			index = 2 // we have already generated the first two blocks
			if mode == argon2i || mode == argon2id {
				in[6]++
				processBlock(&addresses, &in, &zero)
				processBlock(&addresses, &addresses, &zero)
			}
		}

		offset := lane*lanes + slice*segments + index
		var random uint64
		for index < segments {
			prev := offset - 1
			if index == 0 && slice == 0 {
				prev += lanes // last block in lane
			}
			if mode == argon2i || (mode == argon2id && n == 0 && slice < syncPoints/2) {
				if index%blockLength == 0 {
					in[6]++
					processBlock(&addresses, &in, &zero)
					processBlock(&addresses, &addresses, &zero)
				}
				random = addresses[index%blockLength]
			} else {
				random = B[prev][0]
			}
			newOffset := indexAlpha(random, lanes, segments, threads, n, slice, lane, index)
			processBlockXOR(&B[offset], &B[prev], &B[newOffset])
			index, offset = index+1, offset+1
		}
		wg.Done()
	}

	for n := uint32(0); n < time; n++ {
		for slice := uint32(0); slice < syncPoints; slice++ {
			var wg sync.WaitGroup
			for lane := uint32(0); lane < threads; lane++ {
				wg.Add(1)
				go processSegment(n, slice, lane, &wg)
			}
			wg.Wait()
		}
	}

}

func extractKey(B []block, memory, threads, keyLen uint32) []byte {
	lanes := memory / threads
	for lane := uint32(0); lane < threads-1; lane++ {
		for i, v := range B[(lane*lanes)+lanes-1] {
			B[memory-1][i] ^= v
		}
	}

	var block [1024]byte
	for i, v := range B[memory-1] {
		binary.LittleEndian.PutUint64(block[i*8:], v)
	}
	key := make([]byte, keyLen)
	blake2bHash(key, block[:])
	return key
}

func indexAlpha(rand uint64, lanes, segments, threads, n, slice, lane, index uint32) uint32 {
	refLane := uint32(rand>>32) % threads
	if n == 0 && slice == 0 {
		refLane = lane
	}
	m, s := 3*segments, ((slice+1)%syncPoints)*segments
	if lane == refLane {
		m += index
	}
	if n == 0 {
		m, s = slice*segments, 0
		if slice == 0 || lane == refLane {
			m += index
		}
	}
	if index == 0 || lane == refLane {
		m--
	}
	return phi(rand, uint64(m), uint64(s), refLane, lanes)
}

func phi(rand, m, s uint64, lane, lanes uint32) uint32 {
	p := rand & 0xFFFFFFFF
	p = (p * p) >> 32
	p = (p * m) >> 32
	return lane*lanes + uint32((s+m-(p+1))%uint64(lanes))
}
<commit_msg>argon2: add Argon2id and update parameter recommendations<commit_after>// Copyright 2017 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

// Package argon2 implements the key derivation function Argon2.
// Argon2 was selected as the winner of the Password Hashing Competition and can
// be used to derive cryptographic keys from passwords.
//
// For a detailed specification of Argon2 see [1].
//
// If you aren't sure which function you need, use Argon2id (IDKey) and
// the parameter recommendations for your scenario.
//
//
// Argon2i
//
// Argon2i (implemented by Key) is the side-channel resistant version of Argon2.
// It uses data-independent memory access, which is preferred for password
// hashing and password-based key derivation. Argon2i requires more passes over
// memory than Argon2id to protect from trade-off attacks. The recommended
// parameters (taken from [2]) for non-interactive operations are time=3 and to
// use the maximum available memory.
//
//
// Argon2id
//
// Argon2id (implemented by IDKey) is a hybrid version of Argon2 combining
// Argon2i and Argon2d. It uses data-independent memory access for the first
// half of the first iteration over the memory and data-dependent memory access
// for the rest. Argon2id is side-channel resistant and provides better brute-
// force cost savings due to time-memory tradeoffs than Argon2i. The recommended
// parameters for non-interactive operations (taken from [2]) are time=1 and to
// use the maximum available memory.
//
// [1] https://github.com/P-H-C/phc-winner-argon2/blob/master/argon2-specs.pdf
// [2] https://tools.ietf.org/html/draft-irtf-cfrg-argon2-03#section-9.3
package argon2

import (
	"encoding/binary"
	"sync"

	"golang.org/x/crypto/blake2b"
)

// The Argon2 version implemented by this package.
const Version = 0x13

const (
	argon2d = iota
	argon2i
	argon2id
)

// Key derives a key from the password, salt, and cost parameters using Argon2i
// returning a byte slice of length keyLen that can be used as cryptographic
// key. The CPU cost and parallism degree must be greater than zero.
//
// For example, you can get a derived key for e.g. AES-256 (which needs a
// 32-byte key) by doing: `key := argon2.Key([]byte("some password"), salt, 3,
// 32*1024, 4, 32)`
//
// The draft RFC recommends[2] time=3, and memory=32*1024 is a sensible number.
// If using that amount of memory (32 MB) is not possible in some contexts then
// the time parameter can be increased to compensate.
//
// The time parameter specifies the number of passes over the memory and the
// memory parameter specifies the size of the memory in KiB. For example
// memory=32*1024 sets the memory cost to ~32 MB. The number of threads can be
// adjusted to the number of available CPUs. The cost parameters should be
// increased as memory latency and CPU parallelism increases. Remember to get a
// good random salt.
func Key(password, salt []byte, time, memory uint32, threads uint8, keyLen uint32) []byte {
	return deriveKey(argon2i, password, salt, nil, nil, time, memory, threads, keyLen)
}

// IDKey derives a key from the password, salt, and cost parameters using
// Argon2id returning a byte slice of length keyLen that can be used as
// cryptographic key. The CPU cost and parallism degree must be greater than
// zero.
//
// For example, you can get a derived key for e.g. AES-256 (which needs a
// 32-byte key) by doing: `key := argon2.IDKey([]byte("some password"), salt, 1,
// 64*1024, 4, 32)`
//
// The draft RFC recommends[2] time=1, and memory=64*1024 is a sensible number.
// If using that amount of memory (64 MB) is not possible in some contexts then
// the time parameter can be increased to compensate.
//
// The time parameter specifies the number of passes over the memory and the
// memory parameter specifies the size of the memory in KiB. For example
// memory=64*1024 sets the memory cost to ~64 MB. The number of threads can be
// adjusted to the numbers of available CPUs. The cost parameters should be
// increased as memory latency and CPU parallelism increases. Remember to get a
// good random salt.
func IDKey(password, salt []byte, time, memory uint32, threads uint8, keyLen uint32) []byte {
	return deriveKey(argon2id, password, salt, nil, nil, time, memory, threads, keyLen)
}

func deriveKey(mode int, password, salt, secret, data []byte, time, memory uint32, threads uint8, keyLen uint32) []byte {
	if time < 1 {
		panic("argon2: number of rounds too small")
	}
	if threads < 1 {
		panic("argon2: parallelism degree too low")
	}
	h0 := initHash(password, salt, secret, data, time, memory, uint32(threads), keyLen, mode)

	memory = memory / (syncPoints * uint32(threads)) * (syncPoints * uint32(threads))
	if memory < 2*syncPoints*uint32(threads) {
		memory = 2 * syncPoints * uint32(threads)
	}
	B := initBlocks(&h0, memory, uint32(threads))
	processBlocks(B, time, memory, uint32(threads), mode)
	return extractKey(B, memory, uint32(threads), keyLen)
}

const (
	blockLength = 128
	syncPoints  = 4
)

type block [blockLength]uint64

func initHash(password, salt, key, data []byte, time, memory, threads, keyLen uint32, mode int) [blake2b.Size + 8]byte {
	var (
		h0     [blake2b.Size + 8]byte
		params [24]byte
		tmp    [4]byte
	)

	b2, _ := blake2b.New512(nil)
	binary.LittleEndian.PutUint32(params[0:4], threads)
	binary.LittleEndian.PutUint32(params[4:8], keyLen)
	binary.LittleEndian.PutUint32(params[8:12], memory)
	binary.LittleEndian.PutUint32(params[12:16], time)
	binary.LittleEndian.PutUint32(params[16:20], uint32(Version))
	binary.LittleEndian.PutUint32(params[20:24], uint32(mode))
	b2.Write(params[:])
	binary.LittleEndian.PutUint32(tmp[:], uint32(len(password)))
	b2.Write(tmp[:])
	b2.Write(password)
	binary.LittleEndian.PutUint32(tmp[:], uint32(len(salt)))
	b2.Write(tmp[:])
	b2.Write(salt)
	binary.LittleEndian.PutUint32(tmp[:], uint32(len(key)))
	b2.Write(tmp[:])
	b2.Write(key)
	binary.LittleEndian.PutUint32(tmp[:], uint32(len(data)))
	b2.Write(tmp[:])
	b2.Write(data)
	b2.Sum(h0[:0])
	return h0
}

func initBlocks(h0 *[blake2b.Size + 8]byte, memory, threads uint32) []block {
	var block0 [1024]byte
	B := make([]block, memory)
	for lane := uint32(0); lane < threads; lane++ {
		j := lane * (memory / threads)
		binary.LittleEndian.PutUint32(h0[blake2b.Size+4:], lane)

		binary.LittleEndian.PutUint32(h0[blake2b.Size:], 0)
		blake2bHash(block0[:], h0[:])
		for i := range B[j+0] {
			B[j+0][i] = binary.LittleEndian.Uint64(block0[i*8:])
		}

		binary.LittleEndian.PutUint32(h0[blake2b.Size:], 1)
		blake2bHash(block0[:], h0[:])
		for i := range B[j+1] {
			B[j+1][i] = binary.LittleEndian.Uint64(block0[i*8:])
		}
	}
	return B
}

func processBlocks(B []block, time, memory, threads uint32, mode int) {
	lanes := memory / threads
	segments := lanes / syncPoints

	processSegment := func(n, slice, lane uint32, wg *sync.WaitGroup) {
		var addresses, in, zero block
		if mode == argon2i || (mode == argon2id && n == 0 && slice < syncPoints/2) {
			in[0] = uint64(n)
			in[1] = uint64(lane)
			in[2] = uint64(slice)
			in[3] = uint64(memory)
			in[4] = uint64(time)
			in[5] = uint64(mode)
		}

		index := uint32(0)
		if n == 0 && slice == 0 {
			index = 2 // we have already generated the first two blocks
			if mode == argon2i || mode == argon2id {
				in[6]++
				processBlock(&addresses, &in, &zero)
				processBlock(&addresses, &addresses, &zero)
			}
		}

		offset := lane*lanes + slice*segments + index
		var random uint64
		for index < segments {
			prev := offset - 1
			if index == 0 && slice == 0 {
				prev += lanes // last block in lane
			}
			if mode == argon2i || (mode == argon2id && n == 0 && slice < syncPoints/2) {
				if index%blockLength == 0 {
					in[6]++
					processBlock(&addresses, &in, &zero)
					processBlock(&addresses, &addresses, &zero)
				}
				random = addresses[index%blockLength]
			} else {
				random = B[prev][0]
			}
			newOffset := indexAlpha(random, lanes, segments, threads, n, slice, lane, index)
			processBlockXOR(&B[offset], &B[prev], &B[newOffset])
			index, offset = index+1, offset+1
		}
		wg.Done()
	}

	for n := uint32(0); n < time; n++ {
		for slice := uint32(0); slice < syncPoints; slice++ {
			var wg sync.WaitGroup
			for lane := uint32(0); lane < threads; lane++ {
				wg.Add(1)
				go processSegment(n, slice, lane, &wg)
			}
			wg.Wait()
		}
	}

}

func extractKey(B []block, memory, threads, keyLen uint32) []byte {
	lanes := memory / threads
	for lane := uint32(0); lane < threads-1; lane++ {
		for i, v := range B[(lane*lanes)+lanes-1] {
			B[memory-1][i] ^= v
		}
	}

	var block [1024]byte
	for i, v := range B[memory-1] {
		binary.LittleEndian.PutUint64(block[i*8:], v)
	}
	key := make([]byte, keyLen)
	blake2bHash(key, block[:])
	return key
}

func indexAlpha(rand uint64, lanes, segments, threads, n, slice, lane, index uint32) uint32 {
	refLane := uint32(rand>>32) % threads
	if n == 0 && slice == 0 {
		refLane = lane
	}
	m, s := 3*segments, ((slice+1)%syncPoints)*segments
	if lane == refLane {
		m += index
	}
	if n == 0 {
		m, s = slice*segments, 0
		if slice == 0 || lane == refLane {
			m += index
		}
	}
	if index == 0 || lane == refLane {
		m--
	}
	return phi(rand, uint64(m), uint64(s), refLane, lanes)
}

func phi(rand, m, s uint64, lane, lanes uint32) uint32 {
	p := rand & 0xFFFFFFFF
	p = (p * p) >> 32
	p = (p * m) >> 32
	return lane*lanes + uint32((s+m-(p+1))%uint64(lanes))
}
<|endoftext|>


<commit_before>/*
Copyright 2021 The Skaffold Authors

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package integration

import (
	"io/ioutil"
	"os"
	"testing"
	"time"

	"k8s.io/apimachinery/pkg/util/wait"

	"github.com/GoogleContainerTools/skaffold/integration/skaffold"
)

func TestCustomTest(t *testing.T) {
	// TODO: This test shall pass once render v2 is completed.
	t.SkipNow()
	MarkIntegrationTest(t, CanRunWithoutGcp)

	config := "skaffold.yaml"
	expectedText := "bar\nbar\n"
	testDir := "testdata/custom-test"
	testFile := "testdata/custom-test/test"
	depFile := "testdata/custom-test/testdep"
	defer func() {
		os.Truncate(depFile, 0)
		os.Truncate(testFile, 0)
	}()

	// Run skaffold build first to fail quickly on a build failure
	skaffold.Build().InDir(testDir).WithConfig(config).RunOrFail(t)

	ns, client := SetupNamespace(t)

	skaffold.Dev().InDir(testDir).WithConfig(config).InNs(ns.Name).RunLive(t)

	client.WaitForPodsReady("custom-test-example")
	ioutil.WriteFile(depFile, []byte("foo"), 0644)

	err := wait.PollImmediate(time.Millisecond*500, 1*time.Minute, func() (bool, error) {
		out, e := ioutil.ReadFile(testFile)
		failNowIfError(t, e)
		return string(out) == expectedText, nil
	})
	failNowIfError(t, err)
}
<commit_msg>chore: re-enable integration/custom_test (#7509)<commit_after>/*
Copyright 2021 The Skaffold Authors

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package integration

import (
	"io/ioutil"
	"os"
	"testing"
	"time"

	"k8s.io/apimachinery/pkg/util/wait"

	"github.com/GoogleContainerTools/skaffold/integration/skaffold"
)

func TestCustomTest(t *testing.T) {
	MarkIntegrationTest(t, CanRunWithoutGcp)

	config := "skaffold.yaml"
	expectedText := "bar\nbar\n"
	testDir := "testdata/custom-test"
	testFile := "testdata/custom-test/test"
	depFile := "testdata/custom-test/testdep"
	defer func() {
		os.Truncate(depFile, 0)
		os.Truncate(testFile, 0)
	}()

	// Run skaffold build first to fail quickly on a build failure
	skaffold.Build().InDir(testDir).WithConfig(config).RunOrFail(t)

	ns, client := SetupNamespace(t)

	skaffold.Dev().InDir(testDir).WithConfig(config).InNs(ns.Name).RunLive(t)

	client.WaitForPodsReady("custom-test-example")
	ioutil.WriteFile(depFile, []byte("foo"), 0644)

	err := wait.PollImmediate(time.Millisecond*500, 1*time.Minute, func() (bool, error) {
		out, e := ioutil.ReadFile(testFile)
		failNowIfError(t, e)
		return string(out) == expectedText, nil
	})
	failNowIfError(t, err)
}
<|endoftext|>


<commit_before>package integration_test

import (
	"strings"

	. "github.com/onsi/ginkgo"
	. "github.com/onsi/gomega"
	"github.com/onsi/gomega/gbytes"
	"github.com/onsi/gomega/gexec"

	. "github.com/onsi/ginkgo/internal/test_helpers"
	"github.com/onsi/ginkgo/types"
)

var _ = FDescribe("Filter", func() {
	BeforeEach(func() {
		fm.MountFixture("filter")
	})

	It("honors the focus, skip, focus-file and skip-file flags", func() {
		session := startGinkgo(fm.PathTo("filter"),
			"--focus=dog", "--focus=fish",
			"--skip=cat",
			"--focus-file=sprocket", "--focus-file=widget:1-24", "--focus-file=_b:24-42",
			"--skip-file=_c",
			"--json-report=report.json",
		)
		Eventually(session).Should(gexec.Exit(0))
		specs := Reports(fm.LoadJSONReports("filter", "report.json")[0].SpecReports)

		passedSpecs := []string{
			"SprocketA dog fish",
			"SprocketB dog", "SprocketB dog fish", "SprocketB fish",
			"WidgetA dog", "WidgetA dog fish",
			"WidgetB dog", "WidgetB dog fish",
			// lines in _b > 24 are in --focus-file
			"More WidgetB dog", "More WidgetB dog fish",
		}

		skippedSpecs := []string{
			// nugget files are not in focus-file
			"NuggetA cat", "NuggetA dog", "NuggetA cat fish", "NuggetA dog fish",
			"NuggetB cat", "NuggetB dog", "NuggetB cat fish", "NuggetB dog fish",
			// cat is not in -focus
			"SprocketA cat", "SprocketB cat", "WidgetA cat", "WidgetB cat", "More WidgetB cat",
			// fish is in -focus but cat is in -skip
			"SprocketA cat fish", "SprocketB cat fish", "WidgetA cat fish", "WidgetB cat fish", "More WidgetB cat fish",
			// _c is in -skip-file
			"SprocketC cat", "SprocketC dog", "SprocketC cat fish", "SprocketC dog fish",
			// lines in widget > 24 are not in --focus-file
			"More WidgetA cat", "More WidgetA dog", "More WidgetA cat fish", "More WidgetA dog fish",
		}
		pendingSpecs := []string{
			"SprocketA pending dog",
		}

		Ω(specs).Should(HaveLen(len(passedSpecs) + len(skippedSpecs) + len(pendingSpecs)))

		for _, text := range passedSpecs {
			Ω(specs.FindByFullText(text)).Should(HavePassed(), text)
		}
		for _, text := range skippedSpecs {
			Ω(specs.FindByFullText(text)).Should(HaveBeenSkipped(), text)
		}
		for _, text := range pendingSpecs {
			Ω(specs.FindByFullText(text)).Should(BePending(), text)
		}
	})

	It("ignores empty filter flags", func() {
		session := startGinkgo(fm.PathTo("filter"),
			"--focus=", "--skip=",
			"--json-report=report.json",
		)
		Eventually(session).Should(gexec.Exit(types.GINKGO_FOCUS_EXIT_CODE))
		specs := Reports(fm.LoadJSONReports("filter", "report.json")[0].SpecReports)
		for _, spec := range specs {
			if strings.HasPrefix(spec.FullText(), "SprocketC") {
				Ω(spec).Should(HavePassed())
			} else {
				Ω(spec).Should(Or(HaveBeenSkipped(), BePending()))
			}
		}
	})

	It("errors if the file-filter format is wrong", func() {
		session := startGinkgo(fm.PathTo("filter"), "--focus-file=foo:bar", "--skip-file=")
		Eventually(session).Should(gexec.Exit(1))
		Ω(session).Should(gbytes.Say("Invalid File Filter"))
		Ω(session).Should(gbytes.Say("Invalid File Filter"))
	})
})
<commit_msg>remove accidental focused test<commit_after>package integration_test

import (
	"strings"

	. "github.com/onsi/ginkgo"
	. "github.com/onsi/gomega"
	"github.com/onsi/gomega/gbytes"
	"github.com/onsi/gomega/gexec"

	. "github.com/onsi/ginkgo/internal/test_helpers"
	"github.com/onsi/ginkgo/types"
)

var _ = Describe("Filter", func() {
	BeforeEach(func() {
		fm.MountFixture("filter")
	})

	It("honors the focus, skip, focus-file and skip-file flags", func() {
		session := startGinkgo(fm.PathTo("filter"),
			"--focus=dog", "--focus=fish",
			"--skip=cat",
			"--focus-file=sprocket", "--focus-file=widget:1-24", "--focus-file=_b:24-42",
			"--skip-file=_c",
			"--json-report=report.json",
		)
		Eventually(session).Should(gexec.Exit(0))
		specs := Reports(fm.LoadJSONReports("filter", "report.json")[0].SpecReports)

		passedSpecs := []string{
			"SprocketA dog fish",
			"SprocketB dog", "SprocketB dog fish", "SprocketB fish",
			"WidgetA dog", "WidgetA dog fish",
			"WidgetB dog", "WidgetB dog fish",
			// lines in _b > 24 are in --focus-file
			"More WidgetB dog", "More WidgetB dog fish",
		}

		skippedSpecs := []string{
			// nugget files are not in focus-file
			"NuggetA cat", "NuggetA dog", "NuggetA cat fish", "NuggetA dog fish",
			"NuggetB cat", "NuggetB dog", "NuggetB cat fish", "NuggetB dog fish",
			// cat is not in -focus
			"SprocketA cat", "SprocketB cat", "WidgetA cat", "WidgetB cat", "More WidgetB cat",
			// fish is in -focus but cat is in -skip
			"SprocketA cat fish", "SprocketB cat fish", "WidgetA cat fish", "WidgetB cat fish", "More WidgetB cat fish",
			// _c is in -skip-file
			"SprocketC cat", "SprocketC dog", "SprocketC cat fish", "SprocketC dog fish",
			// lines in widget > 24 are not in --focus-file
			"More WidgetA cat", "More WidgetA dog", "More WidgetA cat fish", "More WidgetA dog fish",
		}
		pendingSpecs := []string{
			"SprocketA pending dog",
		}

		Ω(specs).Should(HaveLen(len(passedSpecs) + len(skippedSpecs) + len(pendingSpecs)))

		for _, text := range passedSpecs {
			Ω(specs.FindByFullText(text)).Should(HavePassed(), text)
		}
		for _, text := range skippedSpecs {
			Ω(specs.FindByFullText(text)).Should(HaveBeenSkipped(), text)
		}
		for _, text := range pendingSpecs {
			Ω(specs.FindByFullText(text)).Should(BePending(), text)
		}
	})

	It("ignores empty filter flags", func() {
		session := startGinkgo(fm.PathTo("filter"),
			"--focus=", "--skip=",
			"--json-report=report.json",
		)
		Eventually(session).Should(gexec.Exit(types.GINKGO_FOCUS_EXIT_CODE))
		specs := Reports(fm.LoadJSONReports("filter", "report.json")[0].SpecReports)
		for _, spec := range specs {
			if strings.HasPrefix(spec.FullText(), "SprocketC") {
				Ω(spec).Should(HavePassed())
			} else {
				Ω(spec).Should(Or(HaveBeenSkipped(), BePending()))
			}
		}
	})

	It("errors if the file-filter format is wrong", func() {
		session := startGinkgo(fm.PathTo("filter"), "--focus-file=foo:bar", "--skip-file=")
		Eventually(session).Should(gexec.Exit(1))
		Ω(session).Should(gbytes.Say("Invalid File Filter"))
		Ω(session).Should(gbytes.Say("Invalid File Filter"))
	})
})
<|endoftext|>


<commit_before>// Copyright 2018 the u-root Authors. All rights reserved
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

package integration

import (
	"fmt"
	"io"
	"io/ioutil"
	"os"
	"os/exec"
	"path"
	"path/filepath"
	"strings"
	"testing"
	"time"

	"github.com/u-root/u-root/pkg/golang"
	"github.com/u-root/u-root/pkg/json2test"
	"github.com/u-root/u-root/pkg/uio"
)

// testPkgs returns a slice of tests to run.
func testPkgs(t *testing.T) []string {
	// Packages which do not contain tests (or do not contain tests for the
	// build target) will still compile a test binary which vacuously pass.
	cmd := exec.Command("go", "list",
		"github.com/u-root/u-root/cmds/core/...",
		"github.com/u-root/u-root/cmds/boot/...",
		// TODO: only running tests in cmds because tests in pkg have
		// duplicate names which confuses the test runner. This should
		// get fixed.
		// "github.com/u-root/u-root/xcmds/...",
		// "github.com/u-root/u-root/pkg/...",
	)
	cmd.Env = append(os.Environ(), "GOARCH="+TestArch())
	out, err := cmd.CombinedOutput()
	if err != nil {
		t.Fatal(err)
	}
	pkgs := strings.Fields(strings.TrimSpace(string(out)))

	// TODO: Some tests do not run properly in QEMU at the moment. They are
	// blacklisted. These tests fail for mostly two reasons:
	// 1. either it requires networking (not enabled in the kernel)
	// 2. or it depends on some test files (for example /bin/sleep)
	blacklist := []string{
		"github.com/u-root/u-root/cmds/core/cmp",
		"github.com/u-root/u-root/cmds/core/dd",
		"github.com/u-root/u-root/cmds/core/elvish/eval",
		"github.com/u-root/u-root/cmds/core/elvish/edit/tty",
		"github.com/u-root/u-root/cmds/core/fusermount",
		"github.com/u-root/u-root/cmds/core/wget",
		"github.com/u-root/u-root/cmds/core/which",
	}
	for i := 0; i < len(pkgs); i++ {
		for _, b := range blacklist {
			if pkgs[i] == b {
				pkgs = append(pkgs[:i], pkgs[i+1:]...)
			}
		}
	}

	return pkgs
}

func copyRelativeFiles(src string, dst string) error {
	return filepath.Walk(src, func(path string, fi os.FileInfo, err error) error {
		if err != nil {
			return err
		}

		rel, err := filepath.Rel(src, path)
		if err != nil {
			return err
		}

		if fi.Mode().IsDir() {
			return os.MkdirAll(filepath.Join(dst, rel), fi.Mode().Perm())
		} else if fi.Mode().IsRegular() {
			srcf, err := os.Open(path)
			if err != nil {
				return err
			}
			defer srcf.Close()
			dstf, err := os.Create(filepath.Join(dst, rel))
			if err != nil {
				return err
			}
			defer dstf.Close()
			_, err = io.Copy(dstf, srcf)
			return err
		}
		return nil
	})
}

// TestGoTest effectively runs "go test ./..." inside a QEMU instance. The
// tests run as root and can do all sorts of things not possible otherwise.
func TestGoTest(t *testing.T) {
	SkipWithoutQEMU(t)

	// TODO: support arm
	if TestArch() != "amd64" {
		t.Skipf("test not supported on %s", TestArch())
	}

	// Create a temporary directory.
	tmpDir, err := ioutil.TempDir("", "uroot-integration")
	if err != nil {
		t.Fatal(err)
	}

	env := golang.Default()
	env.CgoEnabled = false
	env.GOARCH = TestArch()

	// Statically build tests and add them to the temporary directory.
	pkgs := testPkgs(t)
	var tests []string
	os.Setenv("CGO_ENABLED", "0")
	testDir := filepath.Join(tmpDir, "tests")
	for _, pkg := range pkgs {
		pkgDir := filepath.Join(testDir, pkg)
		if err := os.MkdirAll(pkgDir, 0755); err != nil {
			t.Fatal(err)
		}

		testFile := filepath.Join(pkgDir, fmt.Sprintf("%s.test", path.Base(pkg)))
		cmd := exec.Command("go", "test", "-ldflags", "-s -w", "-c", pkg, "-o", testFile)
		if err := cmd.Run(); err != nil {
			t.Fatalf("could not build %s: %v", pkg, err)
		}

		// When a package does not contain any tests, the test
		// executable is not generated, so it is not included in the
		// `tests` list.
		if _, err := os.Stat(testFile); !os.IsNotExist(err) {
			tests = append(tests, pkg)

			p, err := env.Package(pkg)
			if err != nil {
				t.Fatal(err)
			}
			// Optimistically copy any files in the pkg's
			// directory, in case e.g. a testdata dir is there.
			if err := copyRelativeFiles(p.Dir, filepath.Join(testDir, pkg)); err != nil {
				t.Fatal(err)
			}
		}
	}

	tc := json2test.NewTestCollector()

	// Create the CPIO and start QEMU.
	q, cleanup := QEMUTest(t, &Options{
		Cmds: []string{
			"github.com/u-root/u-root/integration/testcmd/gotest/uinit",
			"github.com/u-root/u-root/cmds/core/init",
			// Used by gotest/uinit.
			"github.com/u-root/u-root/cmds/core/mkdir",
			"github.com/u-root/u-root/cmds/core/mount",
			// Used by an elvish test.
			"github.com/u-root/u-root/cmds/core/ls",
			"github.com/u-root/u-root/cmds/core/sleep",
		},
		TmpDir: tmpDir,
		SerialOutput: uio.ClosingMultiWriter(
			// Collect JSON test events in tc.
			json2test.EventParser(tc),
			// Write non-JSON output to log.
			JSONLessTestLineWriter(t, "serial"),
		),
	})
	defer cleanup()

	if err := q.ExpectTimeout("GoTest Done", 60*time.Second); err != nil {
		t.Errorf("Waiting for GoTest Done: %v", err)
	}

	for pkg, test := range tc.Tests {
		switch test.State {
		case json2test.StateFail:
			t.Errorf("Test %v failed:\n%v", pkg, test.FullOutput)
		case json2test.StateSkip:
			t.Logf("Test %v skipped", pkg)
		case json2test.StatePass:
			// Nothing.
		default:
			t.Errorf("Test %v left in state %v:\n%v", pkg, test.State, test.FullOutput)
		}
	}
}
<commit_msg>gotest: enable all pkg/ for Go QEMU testing<commit_after>// Copyright 2018 the u-root Authors. All rights reserved
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

package integration

import (
	"fmt"
	"io"
	"io/ioutil"
	"os"
	"os/exec"
	"path"
	"path/filepath"
	"strings"
	"testing"
	"time"

	"github.com/u-root/u-root/pkg/golang"
	"github.com/u-root/u-root/pkg/json2test"
	"github.com/u-root/u-root/pkg/uio"
)

// testPkgs returns a slice of tests to run.
func testPkgs(t *testing.T) []string {
	// Packages which do not contain tests (or do not contain tests for the
	// build target) will still compile a test binary which vacuously pass.
	cmd := exec.Command("go", "list",
		"github.com/u-root/u-root/cmds/core/...",
		"github.com/u-root/u-root/cmds/boot/...",
		"github.com/u-root/u-root/pkg/...",
		"github.com/u-root/u-root/cmds/exp/...",
	)
	cmd.Env = append(os.Environ(), "GOARCH="+TestArch())
	out, err := cmd.CombinedOutput()
	if err != nil {
		t.Fatal(err)
	}
	pkgs := strings.Fields(strings.TrimSpace(string(out)))

	// TODO: Some tests do not run properly in QEMU at the moment. They are
	// blacklisted. These tests fail for mostly two reasons:
	// 1. either it requires networking (not enabled in the kernel)
	// 2. or it depends on some test files (for example /bin/sleep)
	blacklist := []string{
		"github.com/u-root/u-root/cmds/core/cmp",
		"github.com/u-root/u-root/cmds/core/dd",
		"github.com/u-root/u-root/cmds/core/elvish/eval",
		"github.com/u-root/u-root/cmds/core/elvish/edit/tty",
		"github.com/u-root/u-root/cmds/core/fusermount",
		"github.com/u-root/u-root/cmds/core/wget",
		"github.com/u-root/u-root/cmds/core/which",
		"github.com/u-root/u-root/cmds/exp/rush",
		"github.com/u-root/u-root/cmds/exp/pox",
		"github.com/u-root/u-root/pkg/crypto",
		"github.com/u-root/u-root/pkg/tarutil",
		"github.com/u-root/u-root/pkg/ldd",
		//"github.com/u-root/u-root/pkg/pty",

		// Missing xzcat in VM.
		"github.com/u-root/u-root/cmds/exp/bzimage",
		"github.com/u-root/u-root/pkg/bzimage",

		// Missing /dev/mem and /sys/firmware/efi
		"github.com/u-root/u-root/pkg/acpi",

		// No Go compiler in VM.
		"github.com/u-root/u-root/pkg/bb",
		"github.com/u-root/u-root/pkg/uroot",
		"github.com/u-root/u-root/pkg/uroot/builder",
	}
	for i := 0; i < len(pkgs); i++ {
		for _, b := range blacklist {
			if pkgs[i] == b {
				pkgs = append(pkgs[:i], pkgs[i+1:]...)
			}
		}
	}

	return pkgs
}

func copyRelativeFiles(src string, dst string) error {
	return filepath.Walk(src, func(path string, fi os.FileInfo, err error) error {
		if err != nil {
			return err
		}

		rel, err := filepath.Rel(src, path)
		if err != nil {
			return err
		}

		if fi.Mode().IsDir() {
			return os.MkdirAll(filepath.Join(dst, rel), fi.Mode().Perm())
		} else if fi.Mode().IsRegular() {
			srcf, err := os.Open(path)
			if err != nil {
				return err
			}
			defer srcf.Close()
			dstf, err := os.Create(filepath.Join(dst, rel))
			if err != nil {
				return err
			}
			defer dstf.Close()
			_, err = io.Copy(dstf, srcf)
			return err
		}
		return nil
	})
}

// TestGoTest effectively runs "go test ./..." inside a QEMU instance. The
// tests run as root and can do all sorts of things not possible otherwise.
func TestGoTest(t *testing.T) {
	SkipWithoutQEMU(t)

	// TODO: support arm
	if TestArch() != "amd64" {
		t.Skipf("test not supported on %s", TestArch())
	}

	// Create a temporary directory.
	tmpDir, err := ioutil.TempDir("", "uroot-integration")
	if err != nil {
		t.Fatal(err)
	}

	env := golang.Default()
	env.CgoEnabled = false
	env.GOARCH = TestArch()

	// Statically build tests and add them to the temporary directory.
	pkgs := testPkgs(t)
	var tests []string
	os.Setenv("CGO_ENABLED", "0")
	testDir := filepath.Join(tmpDir, "tests")
	for _, pkg := range pkgs {
		pkgDir := filepath.Join(testDir, pkg)
		if err := os.MkdirAll(pkgDir, 0755); err != nil {
			t.Fatal(err)
		}

		testFile := filepath.Join(pkgDir, fmt.Sprintf("%s.test", path.Base(pkg)))
		cmd := exec.Command("go", "test", "-ldflags", "-s -w", "-c", pkg, "-o", testFile)
		if err := cmd.Run(); err != nil {
			t.Fatalf("could not build %s: %v", pkg, err)
		}

		// When a package does not contain any tests, the test
		// executable is not generated, so it is not included in the
		// `tests` list.
		if _, err := os.Stat(testFile); !os.IsNotExist(err) {
			tests = append(tests, pkg)

			p, err := env.Package(pkg)
			if err != nil {
				t.Fatal(err)
			}
			// Optimistically copy any files in the pkg's
			// directory, in case e.g. a testdata dir is there.
			if err := copyRelativeFiles(p.Dir, filepath.Join(testDir, pkg)); err != nil {
				t.Fatal(err)
			}
		}
	}

	tc := json2test.NewTestCollector()

	// Create the CPIO and start QEMU.
	q, cleanup := QEMUTest(t, &Options{
		Cmds: []string{
			"github.com/u-root/u-root/integration/testcmd/gotest/uinit",
			"github.com/u-root/u-root/cmds/core/init",
			// Used by gotest/uinit.
			"github.com/u-root/u-root/cmds/core/mkdir",
			"github.com/u-root/u-root/cmds/core/mount",
			// Used by an elvish test.
			"github.com/u-root/u-root/cmds/core/ls",
			"github.com/u-root/u-root/cmds/core/sleep",
			"github.com/u-root/u-root/cmds/core/echo",
		},
		TmpDir: tmpDir,
		SerialOutput: uio.ClosingMultiWriter(
			// Collect JSON test events in tc.
			json2test.EventParser(tc),
			// Write non-JSON output to log.
			JSONLessTestLineWriter(t, "serial"),
		),
	})
	defer cleanup()

	if err := q.ExpectTimeout("GoTest Done", 60*time.Second); err != nil {
		t.Errorf("Waiting for GoTest Done: %v", err)
	}

	for pkg, test := range tc.Tests {
		switch test.State {
		case json2test.StateFail:
			t.Errorf("Test %v failed:\n%v", pkg, test.FullOutput)
		case json2test.StateSkip:
			t.Logf("Test %v skipped", pkg)
		case json2test.StatePass:
			// Nothing.
		default:
			t.Errorf("Test %v left in state %v:\n%v", pkg, test.State, test.FullOutput)
		}
	}
}
<|endoftext|>